diff options
Diffstat (limited to 'src')
189 files changed, 29959 insertions, 12296 deletions
diff --git a/src/common/address.c b/src/common/address.c index 14a7b6bc96..29d4c0447e 100644 --- a/src/common/address.c +++ b/src/common/address.c @@ -14,6 +14,7 @@ #include "address.h" #include "torlog.h" #include "container.h" +#include "sandbox.h" #ifdef _WIN32 #include <process.h> @@ -181,7 +182,7 @@ tor_addr_make_unspec(tor_addr_t *a) a->family = AF_UNSPEC; } -/** Set address <a>a</b> to the null address in address family <b>family</b>. +/** Set address <b>a</b> to the null address in address family <b>family</b>. * The null address for AF_INET is 0.0.0.0. The null address for AF_INET6 is * [::]. AF_UNSPEC is all null. */ void @@ -234,8 +235,10 @@ tor_addr_lookup(const char *name, uint16_t family, tor_addr_t *addr) memset(&hints, 0, sizeof(hints)); hints.ai_family = family; hints.ai_socktype = SOCK_STREAM; - err = getaddrinfo(name, NULL, &hints, &res); - if (!err) { + err = sandbox_getaddrinfo(name, NULL, &hints, &res); + /* The check for 'res' here shouldn't be necessary, but it makes static + * analysis tools happy. */ + if (!err && res) { best = NULL; for (res_p = res; res_p; res_p = res_p->ai_next) { if (family == AF_UNSPEC) { @@ -261,7 +264,7 @@ tor_addr_lookup(const char *name, uint16_t family, tor_addr_t *addr) &((struct sockaddr_in6*)best->ai_addr)->sin6_addr); result = 0; } - freeaddrinfo(res); + sandbox_freeaddrinfo(res); return result; } return (err == EAI_AGAIN) ? 1 : -1; @@ -873,6 +876,32 @@ tor_addr_copy(tor_addr_t *dest, const tor_addr_t *src) memcpy(dest, src, sizeof(tor_addr_t)); } +/** Copy a tor_addr_t from <b>src</b> to <b>dest</b>, taking extra case to + * copy only the well-defined portions. Used for computing hashes of + * addresses. + */ +void +tor_addr_copy_tight(tor_addr_t *dest, const tor_addr_t *src) +{ + tor_assert(src != dest); + tor_assert(src); + tor_assert(dest); + memset(dest, 0, sizeof(tor_addr_t)); + dest->family = src->family; + switch (tor_addr_family(src)) + { + case AF_INET: + dest->addr.in_addr.s_addr = src->addr.in_addr.s_addr; + break; + case AF_INET6: + memcpy(dest->addr.in6_addr.s6_addr, src->addr.in6_addr.s6_addr, 16); + case AF_UNSPEC: + break; + default: + tor_fragile_assert(); + } +} + /** Given two addresses <b>addr1</b> and <b>addr2</b>, return 0 if the two * addresses are equivalent under the mask mbits, less than 0 if addr1 * precedes addr2, and greater than 0 otherwise. @@ -994,19 +1023,17 @@ tor_addr_compare_masked(const tor_addr_t *addr1, const tor_addr_t *addr2, } } -/** Return a hash code based on the address addr */ -unsigned int +/** Return a hash code based on the address addr. DOCDOC extra */ +uint64_t tor_addr_hash(const tor_addr_t *addr) { switch (tor_addr_family(addr)) { case AF_INET: - return tor_addr_to_ipv4h(addr); + return siphash24g(&addr->addr.in_addr.s_addr, 4); case AF_UNSPEC: return 0x4e4d5342; - case AF_INET6: { - const uint32_t *u = tor_addr_to_in6_addr32(addr); - return u[0] + u[1] + u[2] + u[3]; - } + case AF_INET6: + return siphash24g(&addr->addr.in6_addr.s6_addr, 16); default: tor_fragile_assert(); return 0; @@ -1420,31 +1447,22 @@ get_interface_address6(int severity, sa_family_t family, tor_addr_t *addr) * XXXX024 IPv6 deprecate some of these. */ -/** Return true iff <b>ip</b> (in host order) is an IP reserved to localhost, - * or reserved for local networks by RFC 1918. - */ -int -is_internal_IP(uint32_t ip, int for_listening) -{ - tor_addr_t myaddr; - myaddr.family = AF_INET; - myaddr.addr.in_addr.s_addr = htonl(ip); - - return tor_addr_is_internal(&myaddr, for_listening); -} - /** Given an address of the form "ip:port", try to divide it into its * ip and port portions, setting *<b>address_out</b> to a newly * allocated string holding the address portion and *<b>port_out</b> * to the port. * - * Don't do DNS lookups and don't allow domain names in the <ip> field. - * Don't accept <b>addrport</b> of the form "<ip>" or "<ip>:0". + * Don't do DNS lookups and don't allow domain names in the "ip" field. + * + * If <b>default_port</b> is less than 0, don't accept <b>addrport</b> of the + * form "ip" or "ip:0". Otherwise, accept those forms, and set + * *<b>port_out</b> to <b>default_port</b>. * * Return 0 on success, -1 on failure. */ int tor_addr_port_parse(int severity, const char *addrport, - tor_addr_t *address_out, uint16_t *port_out) + tor_addr_t *address_out, uint16_t *port_out, + int default_port) { int retval = -1; int r; @@ -1458,8 +1476,12 @@ tor_addr_port_parse(int severity, const char *addrport, if (r < 0) goto done; - if (!*port_out) - goto done; + if (!*port_out) { + if (default_port >= 0) + *port_out = default_port; + else + goto done; + } /* make sure that address_out is an IP address */ if (tor_addr_parse(address_out, addr_tmp) < 0) @@ -1480,9 +1502,18 @@ int tor_addr_port_split(int severity, const char *addrport, char **address_out, uint16_t *port_out) { + tor_addr_t a_tmp; tor_assert(addrport); tor_assert(address_out); tor_assert(port_out); + /* We need to check for IPv6 manually because addr_port_lookup() doesn't + * do a good job on IPv6 addresses that lack a port. */ + if (tor_addr_parse(&a_tmp, addrport) == AF_INET6) { + *port_out = 0; + *address_out = tor_strdup(addrport); + return 0; + } + return addr_port_lookup(severity, addrport, address_out, NULL, port_out); } @@ -1560,7 +1591,7 @@ addr_mask_get_bits(uint32_t mask) return 0; if (mask == 0xFFFFFFFFu) return 32; - for (i=0; i<=32; ++i) { + for (i=1; i<=32; ++i) { if (mask == (uint32_t) ~((1u<<(32-i))-1)) { return i; } diff --git a/src/common/address.h b/src/common/address.h index 77e5855346..8dc63b71c1 100644 --- a/src/common/address.h +++ b/src/common/address.h @@ -167,7 +167,7 @@ int tor_addr_compare_masked(const tor_addr_t *addr1, const tor_addr_t *addr2, * "exactly". */ #define tor_addr_eq(a,b) (0==tor_addr_compare((a),(b),CMP_EXACT)) -unsigned int tor_addr_hash(const tor_addr_t *addr); +uint64_t tor_addr_hash(const tor_addr_t *addr); int tor_addr_is_v4(const tor_addr_t *addr); int tor_addr_is_internal_(const tor_addr_t *ip, int for_listening, const char *filename, int lineno); @@ -192,6 +192,7 @@ const char * tor_addr_to_str(char *dest, const tor_addr_t *addr, size_t len, int decorate); int tor_addr_parse(tor_addr_t *addr, const char *src); void tor_addr_copy(tor_addr_t *dest, const tor_addr_t *src); +void tor_addr_copy_tight(tor_addr_t *dest, const tor_addr_t *src); void tor_addr_from_ipv4n(tor_addr_t *dest, uint32_t v4addr); /** Set <b>dest</b> to the IPv4 address encoded in <b>v4addr</b> in host * order. */ @@ -209,12 +210,12 @@ int tor_addr_port_split(int severity, const char *addrport, char **address_out, uint16_t *port_out); int tor_addr_port_parse(int severity, const char *addrport, - tor_addr_t *address_out, uint16_t *port_out); + tor_addr_t *address_out, uint16_t *port_out, + int default_port); int tor_addr_hostname_is_local(const char *name); /* IPv4 helpers */ -int is_internal_IP(uint32_t ip, int for_listening); int addr_port_lookup(int severity, const char *addrport, char **address, uint32_t *addr, uint16_t *port_out); int parse_port_range(const char *port, uint16_t *port_min_out, diff --git a/src/common/backtrace.c b/src/common/backtrace.c new file mode 100644 index 0000000000..3a073a8ff5 --- /dev/null +++ b/src/common/backtrace.c @@ -0,0 +1,233 @@ +/* Copyright (c) 2013, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#define __USE_GNU +#define _GNU_SOURCE 1 + +#include "orconfig.h" +#include "compat.h" +#include "util.h" +#include "torlog.h" + +#ifdef HAVE_EXECINFO_H +#include <execinfo.h> +#endif +#ifdef HAVE_FCNTL_H +#include <fcntl.h> +#endif +#ifdef HAVE_UNISTD_H +#include <unistd.h> +#endif +#ifdef HAVE_SIGNAL_H +#include <signal.h> +#endif + +#ifdef HAVE_CYGWIN_SIGNAL_H +#include <cygwin/signal.h> +#elif defined(HAVE_SYS_UCONTEXT_H) +#include <sys/ucontext.h> +#elif defined(HAVE_UCONTEXT_H) +#include <ucontext.h> +#endif + +#define EXPOSE_CLEAN_BACKTRACE +#include "backtrace.h" + +#if defined(HAVE_EXECINFO_H) && defined(HAVE_BACKTRACE) && \ + defined(HAVE_BACKTRACE_SYMBOLS_FD) && defined(HAVE_SIGACTION) +#define USE_BACKTRACE +#endif + +#if !defined(USE_BACKTRACE) +#define NO_BACKTRACE_IMPL +#endif + +/** Version of Tor to report in backtrace messages. */ +static char *bt_version = NULL; + +#ifdef USE_BACKTRACE +/** Largest stack depth to try to dump. */ +#define MAX_DEPTH 256 +/** Static allocation of stack to dump. This is static so we avoid stack + * pressure. */ +static void *cb_buf[MAX_DEPTH]; +/** Protects cb_buf from concurrent access */ +static tor_mutex_t cb_buf_mutex; + +/** Change a stacktrace in <b>stack</b> of depth <b>depth</b> so that it will + * log the correct function from which a signal was received with context + * <b>ctx</b>. (When we get a signal, the current function will not have + * called any other function, and will therefore have not pushed its address + * onto the stack. Fortunately, we usually have the program counter in the + * ucontext_t structure. + */ +void +clean_backtrace(void **stack, int depth, const ucontext_t *ctx) +{ +#ifdef PC_FROM_UCONTEXT +#if defined(__linux__) + const int n = 1; +#elif defined(__darwin__) || defined(__APPLE__) || defined(__OpenBSD__) \ + || defined(__FreeBSD__) + const int n = 2; +#else + const int n = 1; +#endif + if (depth <= n) + return; + + stack[n] = (void*) ctx->PC_FROM_UCONTEXT; +#else + (void) depth; + (void) ctx; +#endif +} + +/** Log a message <b>msg</b> at <b>severity</b> in <b>domain</b>, and follow + * that with a backtrace log. */ +void +log_backtrace(int severity, int domain, const char *msg) +{ + int depth; + char **symbols; + int i; + + tor_mutex_acquire(&cb_buf_mutex); + + depth = backtrace(cb_buf, MAX_DEPTH); + symbols = backtrace_symbols(cb_buf, depth); + + tor_log(severity, domain, "%s. Stack trace:", msg); + if (!symbols) { + tor_log(severity, domain, " Unable to generate backtrace."); + goto done; + } + for (i=0; i < depth; ++i) { + tor_log(severity, domain, " %s", symbols[i]); + } + free(symbols); + + done: + tor_mutex_release(&cb_buf_mutex); +} + +static void crash_handler(int sig, siginfo_t *si, void *ctx_) + __attribute__((noreturn)); + +/** Signal handler: write a crash message with a stack trace, and die. */ +static void +crash_handler(int sig, siginfo_t *si, void *ctx_) +{ + char buf[40]; + int depth; + ucontext_t *ctx = (ucontext_t *) ctx_; + int n_fds, i; + const int *fds = NULL; + + (void) si; + + depth = backtrace(cb_buf, MAX_DEPTH); + /* Clean up the top stack frame so we get the real function + * name for the most recently failing function. */ + clean_backtrace(cb_buf, depth, ctx); + + format_dec_number_sigsafe((unsigned)sig, buf, sizeof(buf)); + + tor_log_err_sigsafe(bt_version, " died: Caught signal ", buf, "\n", + NULL); + + n_fds = tor_log_get_sigsafe_err_fds(&fds); + for (i=0; i < n_fds; ++i) + backtrace_symbols_fd(cb_buf, depth, fds[i]); + + abort(); +} + +/** Install signal handlers as needed so that when we crash, we produce a + * useful stack trace. Return 0 on success, -1 on failure. */ +static int +install_bt_handler(void) +{ + int trap_signals[] = { SIGSEGV, SIGILL, SIGFPE, SIGBUS, SIGSYS, + SIGIO, -1 }; + int i, rv=0; + + struct sigaction sa; + + tor_mutex_init(&cb_buf_mutex); + + memset(&sa, 0, sizeof(sa)); + sa.sa_sigaction = crash_handler; + sa.sa_flags = SA_SIGINFO; + sigfillset(&sa.sa_mask); + + for (i = 0; trap_signals[i] >= 0; ++i) { + if (sigaction(trap_signals[i], &sa, NULL) == -1) { + log_warn(LD_BUG, "Sigaction failed: %s", strerror(errno)); + rv = -1; + } + } + + { + /* Now, generate (but do not log) a backtrace. This ensures that + * libc has pre-loaded the symbols we need to dump things, so that later + * reads won't be denied by the sandbox code */ + char **symbols; + int depth = backtrace(cb_buf, MAX_DEPTH); + symbols = backtrace_symbols(cb_buf, depth); + if (symbols) + free(symbols); + } + + return rv; +} + +/** Uninstall crash handlers. */ +static void +remove_bt_handler(void) +{ + tor_mutex_uninit(&cb_buf_mutex); +} +#endif + +#ifdef NO_BACKTRACE_IMPL +void +log_backtrace(int severity, int domain, const char *msg) +{ + tor_log(severity, domain, "%s. (Stack trace not available)", msg); +} + +static int +install_bt_handler(void) +{ + return 0; +} + +static void +remove_bt_handler(void) +{ +} +#endif + +/** Set up code to handle generating error messages on crashes. */ +int +configure_backtrace_handler(const char *tor_version) +{ + tor_free(bt_version); + if (!tor_version) + tor_version = ""; + tor_asprintf(&bt_version, "Tor %s", tor_version); + + return install_bt_handler(); +} + +/** Perform end-of-process cleanup for code that generates error messages on + * crashes. */ +void +clean_up_backtrace_handler(void) +{ + remove_bt_handler(); + + tor_free(bt_version); +} + diff --git a/src/common/backtrace.h b/src/common/backtrace.h new file mode 100644 index 0000000000..1f4d73339f --- /dev/null +++ b/src/common/backtrace.h @@ -0,0 +1,21 @@ +/* Copyright (c) 2013, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef TOR_BACKTRACE_H +#define TOR_BACKTRACE_H + +#include "orconfig.h" + +void log_backtrace(int severity, int domain, const char *msg); +int configure_backtrace_handler(const char *tor_version); +void clean_up_backtrace_handler(void); + +#ifdef EXPOSE_CLEAN_BACKTRACE +#if defined(HAVE_EXECINFO_H) && defined(HAVE_BACKTRACE) && \ + defined(HAVE_BACKTRACE_SYMBOLS_FD) && defined(HAVE_SIGACTION) +void clean_backtrace(void **stack, int depth, const ucontext_t *ctx); +#endif +#endif + +#endif + diff --git a/src/common/compat.c b/src/common/compat.c index d88c5f92de..e25ecc462d 100644 --- a/src/common/compat.c +++ b/src/common/compat.c @@ -23,6 +23,7 @@ * we can also take out the configure check. */ #define _GNU_SOURCE +#define COMPAT_PRIVATE #include "compat.h" #ifdef _WIN32 @@ -34,6 +35,15 @@ #ifdef HAVE_UNAME #include <sys/utsname.h> #endif +#ifdef HAVE_SYS_TYPES_H +#include <sys/types.h> +#endif +#ifdef HAVE_SYS_SYSCTL_H +#include <sys/sysctl.h> +#endif +#ifdef HAVE_SYS_STAT_H +#include <sys/stat.h> +#endif #ifdef HAVE_UNISTD_H #include <unistd.h> #endif @@ -104,11 +114,18 @@ /* Only use the linux prctl; the IRIX prctl is totally different */ #include <sys/prctl.h> #endif +#ifdef TOR_UNIT_TESTS +#if !defined(HAVE_USLEEP) && defined(HAVE_SYS_SELECT_H) +/* as fallback implementation for tor_sleep_msec */ +#include <sys/select.h> +#endif +#endif #include "torlog.h" #include "util.h" #include "container.h" #include "address.h" +#include "sandbox.h" /* Inline the strl functions if the platform doesn't have them. */ #ifndef HAVE_STRLCPY @@ -125,6 +142,7 @@ tor_open_cloexec(const char *path, int flags, unsigned mode) { int fd; #ifdef O_CLOEXEC + path = sandbox_intern_string(path); fd = open(path, flags|O_CLOEXEC, mode); if (fd >= 0) return fd; @@ -135,6 +153,7 @@ tor_open_cloexec(const char *path, int flags, unsigned mode) return -1; #endif + log_debug(LD_FS, "Opening %s with flags %x", path, flags); fd = open(path, flags, mode); #ifdef FD_CLOEXEC if (fd >= 0) { @@ -166,6 +185,15 @@ tor_fopen_cloexec(const char *path, const char *mode) return result; } +/** As rename(), but work correctly with the sandbox. */ +int +tor_rename(const char *path_old, const char *path_new) +{ + log_debug(LD_FS, "Renaming %s to %s", path_old, path_new); + return rename(sandbox_intern_string(path_old), + sandbox_intern_string(path_new)); +} + #if defined(HAVE_SYS_MMAN_H) || defined(RUNNING_DOXYGEN) /** Try to create a memory mapping for <b>filename</b> and return it. On * failure, return NULL. Sets errno properly, using ERANGE to mean @@ -175,9 +203,10 @@ tor_mmap_file(const char *filename) { int fd; /* router file */ char *string; - int page_size; + int page_size, result; tor_mmap_t *res; size_t size, filesize; + struct stat st; tor_assert(filename); @@ -191,9 +220,22 @@ tor_mmap_file(const char *filename) return NULL; } - /* XXXX why not just do fstat here? */ - size = filesize = (size_t) lseek(fd, 0, SEEK_END); - lseek(fd, 0, SEEK_SET); + /* Get the size of the file */ + result = fstat(fd, &st); + if (result != 0) { + int save_errno = errno; + log_warn(LD_FS, + "Couldn't fstat opened descriptor for \"%s\" during mmap: %s", + filename, strerror(errno)); + close(fd); + errno = save_errno; + return NULL; + } + size = filesize = (size_t)(st.st_size); + /* + * Should we check for weird crap like mmapping a named pipe here, + * or just wait for if (!size) below to fail? + */ /* ensure page alignment */ page_size = getpagesize(); size += (size%page_size) ? page_size-(size%page_size) : 0; @@ -224,12 +266,27 @@ tor_mmap_file(const char *filename) return res; } -/** Release storage held for a memory mapping. */ -void +/** Release storage held for a memory mapping; returns 0 on success, + * or -1 on failure (and logs a warning). */ +int tor_munmap_file(tor_mmap_t *handle) { - munmap((char*)handle->data, handle->mapping_size); - tor_free(handle); + int res; + + if (handle == NULL) + return 0; + + res = munmap((char*)handle->data, handle->mapping_size); + if (res == 0) { + /* munmap() succeeded */ + tor_free(handle); + } else { + log_warn(LD_FS, "Failed to munmap() in tor_munmap_file(): %s", + strerror(errno)); + res = -1; + } + + return res; } #elif defined(_WIN32) tor_mmap_t * @@ -311,17 +368,29 @@ tor_mmap_file(const char *filename) tor_munmap_file(res); return NULL; } -void + +/* Unmap the file, and return 0 for success or -1 for failure */ +int tor_munmap_file(tor_mmap_t *handle) { - if (handle->data) + if (handle == NULL) + return 0; + + if (handle->data) { /* This is an ugly cast, but without it, "data" in struct tor_mmap_t would have to be redefined as non-const. */ - UnmapViewOfFile( (LPVOID) handle->data); + BOOL ok = UnmapViewOfFile( (LPVOID) handle->data); + if (!ok) { + log_warn(LD_FS, "Failed to UnmapViewOfFile() in tor_munmap_file(): %d", + (int)GetLastError()); + } + } if (handle->mmap_handle != NULL) CloseHandle(handle->mmap_handle); tor_free(handle); + + return 0; } #else tor_mmap_t * @@ -337,13 +406,25 @@ tor_mmap_file(const char *filename) handle->size = st.st_size; return handle; } -void + +/** Unmap the file mapped with tor_mmap_file(), and return 0 for success + * or -1 for failure. + */ + +int tor_munmap_file(tor_mmap_t *handle) { - char *d = (char*)handle->data; + char *d = NULL; + if (handle == NULL) + return 0; + + d = (char*)handle->data; tor_free(d); memwipe(handle, 0, sizeof(tor_mmap_t)); tor_free(handle); + + /* Can't fail in this mmap()/munmap()-free case */ + return 0; } #endif @@ -498,21 +579,29 @@ tor_memmem(const void *_haystack, size_t hlen, #else /* This isn't as fast as the GLIBC implementation, but it doesn't need to * be. */ - const char *p, *end; + const char *p, *last_possible_start; const char *haystack = (const char*)_haystack; const char *needle = (const char*)_needle; char first; tor_assert(nlen); + if (nlen > hlen) + return NULL; + p = haystack; - end = haystack + hlen; + /* Last position at which the needle could start. */ + last_possible_start = haystack + hlen - nlen; first = *(const char*)needle; - while ((p = memchr(p, first, end-p))) { - if (p+nlen > end) - return NULL; + while ((p = memchr(p, first, last_possible_start + 1 - p))) { if (fast_memeq(p, needle, nlen)) return p; - ++p; + if (++p > last_possible_start) { + /* This comparison shouldn't be necessary, since if p was previously + * equal to last_possible_start, the next memchr call would be + * "memchr(p, first, 0)", which will return NULL. But it clarifies the + * logic. */ + return NULL; + } } return NULL; #endif @@ -729,7 +818,7 @@ int replace_file(const char *from, const char *to) { #ifndef _WIN32 - return rename(from,to); + return tor_rename(from, to); #else switch (file_status(to)) { @@ -744,7 +833,7 @@ replace_file(const char *from, const char *to) errno = EISDIR; return -1; } - return rename(from,to); + return tor_rename(from,to); #endif } @@ -948,24 +1037,40 @@ socket_accounting_unlock(void) } /** As close(), but guaranteed to work for sockets across platforms (including - * Windows, where close()ing a socket doesn't work. Returns 0 on success, -1 - * on failure. */ + * Windows, where close()ing a socket doesn't work. Returns 0 on success and + * the socket error code on failure. */ int -tor_close_socket(tor_socket_t s) +tor_close_socket_simple(tor_socket_t s) { int r = 0; /* On Windows, you have to call close() on fds returned by open(), - * and closesocket() on fds returned by socket(). On Unix, everything - * gets close()'d. We abstract this difference by always using - * tor_close_socket to close sockets, and always using close() on - * files. - */ -#if defined(_WIN32) - r = closesocket(s); -#else - r = close(s); -#endif + * and closesocket() on fds returned by socket(). On Unix, everything + * gets close()'d. We abstract this difference by always using + * tor_close_socket to close sockets, and always using close() on + * files. + */ + #if defined(_WIN32) + r = closesocket(s); + #else + r = close(s); + #endif + + if (r != 0) { + int err = tor_socket_errno(-1); + log_info(LD_NET, "Close returned an error: %s", tor_socket_strerror(err)); + return err; + } + + return r; +} + +/** As tor_close_socket_simple(), but keeps track of the number + * of open sockets. Returns 0 on success, -1 on failure. */ +int +tor_close_socket(tor_socket_t s) +{ + int r = tor_close_socket_simple(s); socket_accounting_lock(); #ifdef DEBUG_SOCKET_COUNTING @@ -980,13 +1085,11 @@ tor_close_socket(tor_socket_t s) if (r == 0) { --n_sockets_open; } else { - int err = tor_socket_errno(-1); - log_info(LD_NET, "Close returned an error: %s", tor_socket_strerror(err)); #ifdef _WIN32 - if (err != WSAENOTSOCK) + if (r != WSAENOTSOCK) --n_sockets_open; #else - if (err != EBADF) + if (r != EBADF) --n_sockets_open; #endif r = -1; @@ -1032,33 +1135,61 @@ mark_socket_open(tor_socket_t s) tor_socket_t tor_open_socket(int domain, int type, int protocol) { + return tor_open_socket_with_extensions(domain, type, protocol, 1, 0); +} + +/** As socket(), but creates a nonblocking socket and + * counts the number of open sockets. */ +tor_socket_t +tor_open_socket_nonblocking(int domain, int type, int protocol) +{ + return tor_open_socket_with_extensions(domain, type, protocol, 1, 1); +} + +/** As socket(), but counts the number of open sockets and handles + * socket creation with either of SOCK_CLOEXEC and SOCK_NONBLOCK specified. + * <b>cloexec</b> and <b>nonblock</b> should be either 0 or 1 to indicate + * if the corresponding extension should be used.*/ +tor_socket_t +tor_open_socket_with_extensions(int domain, int type, int protocol, + int cloexec, int nonblock) +{ tor_socket_t s; -#ifdef SOCK_CLOEXEC - s = socket(domain, type|SOCK_CLOEXEC, protocol); +#if defined(SOCK_CLOEXEC) && defined(SOCK_NONBLOCK) + int ext_flags = (cloexec ? SOCK_CLOEXEC : 0) | + (nonblock ? SOCK_NONBLOCK : 0); + s = socket(domain, type|ext_flags, protocol); if (SOCKET_OK(s)) goto socket_ok; /* If we got an error, see if it is EINVAL. EINVAL might indicate that, - * even though we were built on a system with SOCK_CLOEXEC support, we - * are running on one without. */ + * even though we were built on a system with SOCK_CLOEXEC and SOCK_NONBLOCK + * support, we are running on one without. */ if (errno != EINVAL) return s; -#endif /* SOCK_CLOEXEC */ +#endif /* SOCK_CLOEXEC && SOCK_NONBLOCK */ s = socket(domain, type, protocol); if (! SOCKET_OK(s)) return s; #if defined(FD_CLOEXEC) - if (fcntl(s, F_SETFD, FD_CLOEXEC) == -1) { - log_warn(LD_FS,"Couldn't set FD_CLOEXEC: %s", strerror(errno)); -#if defined(_WIN32) - closesocket(s); + if (cloexec) { + if (fcntl(s, F_SETFD, FD_CLOEXEC) == -1) { + log_warn(LD_FS,"Couldn't set FD_CLOEXEC: %s", strerror(errno)); + tor_close_socket_simple(s); + return TOR_INVALID_SOCKET; + } + } #else - close(s); + (void)cloexec; #endif - return -1; + + if (nonblock) { + if (set_socket_nonblocking(s) == -1) { + tor_close_socket_simple(s); + return TOR_INVALID_SOCKET; + } } -#endif goto socket_ok; /* So that socket_ok will not be unused. */ @@ -1070,19 +1201,41 @@ tor_open_socket(int domain, int type, int protocol) return s; } -/** As socket(), but counts the number of open sockets. */ +/** As accept(), but counts the number of open sockets. */ tor_socket_t tor_accept_socket(tor_socket_t sockfd, struct sockaddr *addr, socklen_t *len) { + return tor_accept_socket_with_extensions(sockfd, addr, len, 1, 0); +} + +/** As accept(), but returns a nonblocking socket and + * counts the number of open sockets. */ +tor_socket_t +tor_accept_socket_nonblocking(tor_socket_t sockfd, struct sockaddr *addr, + socklen_t *len) +{ + return tor_accept_socket_with_extensions(sockfd, addr, len, 1, 1); +} + +/** As accept(), but counts the number of open sockets and handles + * socket creation with either of SOCK_CLOEXEC and SOCK_NONBLOCK specified. + * <b>cloexec</b> and <b>nonblock</b> should be either 0 or 1 to indicate + * if the corresponding extension should be used.*/ +tor_socket_t +tor_accept_socket_with_extensions(tor_socket_t sockfd, struct sockaddr *addr, + socklen_t *len, int cloexec, int nonblock) +{ tor_socket_t s; -#if defined(HAVE_ACCEPT4) && defined(SOCK_CLOEXEC) - s = accept4(sockfd, addr, len, SOCK_CLOEXEC); +#if defined(HAVE_ACCEPT4) && defined(SOCK_CLOEXEC) && defined(SOCK_NONBLOCK) + int ext_flags = (cloexec ? SOCK_CLOEXEC : 0) | + (nonblock ? SOCK_NONBLOCK : 0); + s = accept4(sockfd, addr, len, ext_flags); if (SOCKET_OK(s)) goto socket_ok; /* If we got an error, see if it is ENOSYS. ENOSYS indicates that, * even though we were built on a system with accept4 support, we * are running on one without. Also, check for EINVAL, which indicates that - * we are missing SOCK_CLOEXEC support. */ + * we are missing SOCK_CLOEXEC/SOCK_NONBLOCK support. */ if (errno != EINVAL && errno != ENOSYS) return s; #endif @@ -1092,13 +1245,24 @@ tor_accept_socket(tor_socket_t sockfd, struct sockaddr *addr, socklen_t *len) return s; #if defined(FD_CLOEXEC) - if (fcntl(s, F_SETFD, FD_CLOEXEC) == -1) { - log_warn(LD_NET, "Couldn't set FD_CLOEXEC: %s", strerror(errno)); - close(s); - return TOR_INVALID_SOCKET; + if (cloexec) { + if (fcntl(s, F_SETFD, FD_CLOEXEC) == -1) { + log_warn(LD_NET, "Couldn't set FD_CLOEXEC: %s", strerror(errno)); + tor_close_socket_simple(s); + return TOR_INVALID_SOCKET; + } } +#else + (void)cloexec; #endif + if (nonblock) { + if (set_socket_nonblocking(s) == -1) { + tor_close_socket_simple(s); + return TOR_INVALID_SOCKET; + } + } + goto socket_ok; /* So that socket_ok will not be unused. */ socket_ok: @@ -1220,6 +1384,18 @@ tor_socketpair(int family, int type, int protocol, tor_socket_t fd[2]) return 0; #else + return tor_ersatz_socketpair(family, type, protocol, fd); +#endif +} + +#ifdef NEED_ERSATZ_SOCKETPAIR +/** + * Helper used to implement socketpair on systems that lack it, by + * making a direct connection to localhost. + */ +STATIC int +tor_ersatz_socketpair(int family, int type, int protocol, tor_socket_t fd[2]) +{ /* This socketpair does not work when localhost is down. So * it's really not the same thing at all. But it's close enough * for now, and really, when localhost is down sometimes, we @@ -1230,7 +1406,7 @@ tor_socketpair(int family, int type, int protocol, tor_socket_t fd[2]) tor_socket_t acceptor = TOR_INVALID_SOCKET; struct sockaddr_in listen_addr; struct sockaddr_in connect_addr; - int size; + socklen_t size; int saved_errno = -1; if (protocol @@ -1313,8 +1489,8 @@ tor_socketpair(int family, int type, int protocol, tor_socket_t fd[2]) if (SOCKET_OK(acceptor)) tor_close_socket(acceptor); return -saved_errno; -#endif } +#endif /** Number of extra file descriptors to keep in reserve beyond those that we * tell Tor it's allowed to use. */ @@ -1532,6 +1708,106 @@ log_credential_status(void) } #endif +#ifndef _WIN32 +/** Cached struct from the last getpwname() call we did successfully. */ +static struct passwd *passwd_cached = NULL; + +/** Helper: copy a struct passwd object. + * + * We only copy the fields pw_uid, pw_gid, pw_name, pw_dir. Tor doesn't use + * any others, and I don't want to run into incompatibilities. + */ +static struct passwd * +tor_passwd_dup(const struct passwd *pw) +{ + struct passwd *new_pw = tor_malloc_zero(sizeof(struct passwd)); + if (pw->pw_name) + new_pw->pw_name = tor_strdup(pw->pw_name); + if (pw->pw_dir) + new_pw->pw_dir = tor_strdup(pw->pw_dir); + new_pw->pw_uid = pw->pw_uid; + new_pw->pw_gid = pw->pw_gid; + + return new_pw; +} + +/** Helper: free one of our cached 'struct passwd' values. */ +static void +tor_passwd_free(struct passwd *pw) +{ + if (!pw) + return; + + tor_free(pw->pw_name); + tor_free(pw->pw_dir); + tor_free(pw); +} + +/** Wrapper around getpwnam() that caches result. Used so that we don't need + * to give the sandbox access to /etc/passwd. + * + * The following fields alone will definitely be copied in the output: pw_uid, + * pw_gid, pw_name, pw_dir. Other fields are not present in cached values. + * + * When called with a NULL argument, this function clears storage associated + * with static variables it uses. + **/ +const struct passwd * +tor_getpwnam(const char *username) +{ + struct passwd *pw; + + if (username == NULL) { + tor_passwd_free(passwd_cached); + passwd_cached = NULL; + return NULL; + } + + if ((pw = getpwnam(username))) { + tor_passwd_free(passwd_cached); + passwd_cached = tor_passwd_dup(pw); + log_notice(LD_GENERAL, "Caching new entry %s for %s", + passwd_cached->pw_name, username); + return pw; + } + + /* Lookup failed */ + if (! passwd_cached || ! passwd_cached->pw_name) + return NULL; + + if (! strcmp(username, passwd_cached->pw_name)) + return passwd_cached; + + return NULL; +} + +/** Wrapper around getpwnam() that can use cached result from + * tor_getpwnam(). Used so that we don't need to give the sandbox access to + * /etc/passwd. + * + * The following fields alone will definitely be copied in the output: pw_uid, + * pw_gid, pw_name, pw_dir. Other fields are not present in cached values. + */ +const struct passwd * +tor_getpwuid(uid_t uid) +{ + struct passwd *pw; + + if ((pw = getpwuid(uid))) { + return pw; + } + + /* Lookup failed */ + if (! passwd_cached) + return NULL; + + if (uid == passwd_cached->pw_uid) + return passwd_cached; + + return NULL; +} +#endif + /** Call setuid and setgid to run as <b>user</b> and switch to their * primary group. Return 0 on success. On failure, log and return -1. */ @@ -1539,7 +1815,7 @@ int switch_id(const char *user) { #ifndef _WIN32 - struct passwd *pw = NULL; + const struct passwd *pw = NULL; uid_t old_uid; gid_t old_gid; static int have_already_switched_id = 0; @@ -1560,7 +1836,7 @@ switch_id(const char *user) old_gid = getgid(); /* Lookup the user and group information, if we have a problem, bail out. */ - pw = getpwnam(user); + pw = tor_getpwnam(user); if (pw == NULL) { log_warn(LD_CONFIG, "Error setting configured user: %s not found", user); return -1; @@ -1731,10 +2007,10 @@ tor_disable_debugger_attach(void) char * get_user_homedir(const char *username) { - struct passwd *pw; + const struct passwd *pw; tor_assert(username); - if (!(pw = getpwnam(username))) { + if (!(pw = tor_getpwnam(username))) { log_err(LD_CONFIG,"User \"%s\" not found.", username); return NULL; } @@ -1746,6 +2022,15 @@ get_user_homedir(const char *username) * actually examine the filesystem; does a purely syntactic modification. * * The parent of the root director is considered to be iteself. + * + * Path separators are the forward slash (/) everywhere and additionally + * the backslash (\) on Win32. + * + * Cuts off any number of trailing path separators but otherwise ignores + * them for purposes of finding the parent directory. + * + * Returns 0 if a parent directory was successfully found, -1 otherwise (fname + * did not have any path separators or only had them at the end). * */ int get_parent_directory(char *fname) @@ -2019,8 +2304,10 @@ tor_inet_pton(int af, const char *src, void *dst) else { unsigned byte1,byte2,byte3,byte4; char more; - for (eow = dot-1; eow >= src && TOR_ISDIGIT(*eow); --eow) + for (eow = dot-1; eow > src && TOR_ISDIGIT(*eow); --eow) ; + if (*eow != ':') + return 0; ++eow; /* We use "scanf" because some platform inet_aton()s are too lax @@ -2248,6 +2535,12 @@ tor_pthread_helper_fn(void *_data) func(arg); return NULL; } +/** + * A pthread attribute to make threads start detached. + */ +static pthread_attr_t attr_detached; +/** True iff we've called tor_threads_init() */ +static int threads_initialized = 0; #endif /** Minimalist interface to run a void function in the background. On @@ -2271,12 +2564,12 @@ spawn_func(void (*func)(void *), void *data) #elif defined(USE_PTHREADS) pthread_t thread; tor_pthread_data_t *d; + if (PREDICT_UNLIKELY(!threads_initialized)) + tor_threads_init(); d = tor_malloc(sizeof(tor_pthread_data_t)); d->data = data; d->func = func; - if (pthread_create(&thread,NULL,tor_pthread_helper_fn,d)) - return -1; - if (pthread_detach(thread)) + if (pthread_create(&thread,&attr_detached,tor_pthread_helper_fn,d)) return -1; return 0; #else @@ -2633,8 +2926,6 @@ tor_get_thread_id(void) * "reentrant" mutexes (i.e., once we can re-lock if we're already holding * them.) */ static pthread_mutexattr_t attr_reentrant; -/** True iff we've called tor_threads_init() */ -static int threads_initialized = 0; /** Initialize <b>mutex</b> so it can be locked. Every mutex must be set * up with tor_mutex_init() or tor_mutex_new(); not both. */ void @@ -2778,6 +3069,8 @@ tor_threads_init(void) if (!threads_initialized) { pthread_mutexattr_init(&attr_reentrant); pthread_mutexattr_settype(&attr_reentrant, PTHREAD_MUTEX_RECURSIVE); + tor_assert(0==pthread_attr_init(&attr_detached)); + tor_assert(0==pthread_attr_setdetachstate(&attr_detached, 1)); threads_initialized = 1; set_main_thread(); } @@ -3153,3 +3446,139 @@ format_win32_error(DWORD err) } #endif +#if defined(HW_PHYSMEM64) +/* This appears to be an OpenBSD thing */ +#define INT64_HW_MEM HW_PHYSMEM64 +#elif defined(HW_MEMSIZE) +/* OSX defines this one */ +#define INT64_HW_MEM HW_MEMSIZE +#endif + +/** + * Helper: try to detect the total system memory, and return it. On failure, + * return 0. + */ +static uint64_t +get_total_system_memory_impl(void) +{ +#if defined(__linux__) + /* On linux, sysctl is deprecated. Because proc is so awesome that you + * shouldn't _want_ to write portable code, I guess? */ + unsigned long long result=0; + int fd = -1; + char *s = NULL; + const char *cp; + size_t file_size=0; + if (-1 == (fd = tor_open_cloexec("/proc/meminfo",O_RDONLY,0))) + return 0; + s = read_file_to_str_until_eof(fd, 65536, &file_size); + if (!s) + goto err; + cp = strstr(s, "MemTotal:"); + if (!cp) + goto err; + /* Use the system sscanf so that space will match a wider number of space */ + if (sscanf(cp, "MemTotal: %llu kB\n", &result) != 1) + goto err; + + close(fd); + tor_free(s); + return result * 1024; + + err: + tor_free(s); + close(fd); + return 0; +#elif defined (_WIN32) + /* Windows has MEMORYSTATUSEX; pretty straightforward. */ + MEMORYSTATUSEX ms; + memset(&ms, 0, sizeof(ms)); + ms.dwLength = sizeof(ms); + if (! GlobalMemoryStatusEx(&ms)) + return 0; + + return ms.ullTotalPhys; + +#elif defined(HAVE_SYSCTL) && defined(INT64_HW_MEM) + /* On many systems, HW_PYHSMEM is clipped to 32 bits; let's use a better + * variant if we know about it. */ + uint64_t memsize = 0; + size_t len = sizeof(memsize); + int mib[2] = {CTL_HW, INT64_HW_MEM}; + if (sysctl(mib,2,&memsize,&len,NULL,0)) + return 0; + + return memsize; + +#elif defined(HAVE_SYSCTL) && defined(HW_PHYSMEM) + /* On some systems (like FreeBSD I hope) you can use a size_t with + * HW_PHYSMEM. */ + size_t memsize=0; + size_t len = sizeof(memsize); + int mib[2] = {CTL_HW, HW_USERMEM}; + if (sysctl(mib,2,&memsize,&len,NULL,0)) + return -1; + + return memsize; + +#else + /* I have no clue. */ + return 0; +#endif +} + +/** + * Try to find out how much physical memory the system has. On success, + * return 0 and set *<b>mem_out</b> to that value. On failure, return -1. + */ +int +get_total_system_memory(size_t *mem_out) +{ + static size_t mem_cached=0; + uint64_t m = get_total_system_memory_impl(); + if (0 == m) { + /* We couldn't find our memory total */ + if (0 == mem_cached) { + /* We have no cached value either */ + *mem_out = 0; + return -1; + } + + *mem_out = mem_cached; + return 0; + } + +#if SIZE_T_MAX != UINT64_MAX + if (m > SIZE_T_MAX) { + /* I think this could happen if we're a 32-bit Tor running on a 64-bit + * system: we could have more system memory than would fit in a + * size_t. */ + m = SIZE_T_MAX; + } +#endif + + *mem_out = mem_cached = (size_t) m; + + return 0; +} + +#ifdef TOR_UNIT_TESTS +/** Delay for <b>msec</b> milliseconds. Only used in tests. */ +void +tor_sleep_msec(int msec) +{ +#ifdef _WIN32 + Sleep(msec); +#elif defined(HAVE_USLEEP) + sleep(msec / 1000); + /* Some usleep()s hate sleeping more than 1 sec */ + usleep((msec % 1000) * 1000); +#elif defined(HAVE_SYS_SELECT_H) + struct timeval tv = { msec / 1000, (msec % 1000) * 1000}; + select(0, NULL, NULL, NULL, &tv); +#else + sleep(CEIL_DIV(msec, 1000)); +#endif +} +#endif + diff --git a/src/common/compat.h b/src/common/compat.h index 51fb8c5273..ec7d2415ed 100644 --- a/src/common/compat.h +++ b/src/common/compat.h @@ -8,6 +8,7 @@ #include "orconfig.h" #include "torint.h" +#include "testsupport.h" #ifdef _WIN32 #ifndef _WIN32_WINNT #define _WIN32_WINNT 0x0501 @@ -84,13 +85,19 @@ /* ===== Compiler compatibility */ -/* GCC can check printf types on arbitrary functions. */ +/* GCC can check printf and scanf types on arbitrary functions. */ #ifdef __GNUC__ #define CHECK_PRINTF(formatIdx, firstArg) \ __attribute__ ((format(printf, formatIdx, firstArg))) #else #define CHECK_PRINTF(formatIdx, firstArg) #endif +#ifdef __GNUC__ +#define CHECK_SCANF(formatIdx, firstArg) \ + __attribute__ ((format(scanf, formatIdx, firstArg))) +#else +#define CHECK_SCANF(formatIdx, firstArg) +#endif /* inline is __inline on windows. */ #ifdef _WIN32 @@ -285,7 +292,7 @@ typedef struct tor_mmap_t { } tor_mmap_t; tor_mmap_t *tor_mmap_file(const char *filename) ATTR_NONNULL((1)); -void tor_munmap_file(tor_mmap_t *handle) ATTR_NONNULL((1)); +int tor_munmap_file(tor_mmap_t *handle) ATTR_NONNULL((1)); int tor_snprintf(char *str, size_t size, const char *format, ...) CHECK_PRINTF(3,4) ATTR_NONNULL((1,3)); @@ -314,7 +321,7 @@ tor_memstr(const void *haystack, size_t hlen, const char *needle) extern const uint32_t TOR_##name##_TABLE[]; \ static INLINE int TOR_##name(char c) { \ uint8_t u = c; \ - return !!(TOR_##name##_TABLE[(u >> 5) & 7] & (1 << (u & 31))); \ + return !!(TOR_##name##_TABLE[(u >> 5) & 7] & (1u << (u & 31))); \ } DECLARE_CTYPE_FN(ISALPHA) DECLARE_CTYPE_FN(ISALNUM) @@ -403,6 +410,7 @@ struct tm *tor_gmtime_r(const time_t *timep, struct tm *result); /* ===== File compatibility */ int tor_open_cloexec(const char *path, int flags, unsigned mode); FILE *tor_fopen_cloexec(const char *path, const char *mode); +int tor_rename(const char *path_old, const char *path_new); int replace_file(const char *from, const char *to); int touch_file(const char *fname); @@ -446,10 +454,22 @@ typedef int socklen_t; #define TOR_INVALID_SOCKET (-1) #endif +int tor_close_socket_simple(tor_socket_t s); int tor_close_socket(tor_socket_t s); +tor_socket_t tor_open_socket_with_extensions( + int domain, int type, int protocol, + int cloexec, int nonblock); tor_socket_t tor_open_socket(int domain, int type, int protocol); +tor_socket_t tor_open_socket_nonblocking(int domain, int type, int protocol); tor_socket_t tor_accept_socket(tor_socket_t sockfd, struct sockaddr *addr, socklen_t *len); +tor_socket_t tor_accept_socket_nonblocking(tor_socket_t sockfd, + struct sockaddr *addr, + socklen_t *len); +tor_socket_t tor_accept_socket_with_extensions(tor_socket_t sockfd, + struct sockaddr *addr, + socklen_t *len, + int cloexec, int nonblock); int get_n_open_sockets(void); #define tor_socket_send(s, buf, len, flags) send(s, buf, len, flags) @@ -613,11 +633,18 @@ int switch_id(const char *user); char *get_user_homedir(const char *username); #endif +#ifndef _WIN32 +const struct passwd *tor_getpwnam(const char *username); +const struct passwd *tor_getpwuid(uid_t uid); +#endif + int get_parent_directory(char *fname); char *make_path_absolute(char *fname); char **get_environment(void); +int get_total_system_memory(size_t *mem_out); + int spawn_func(void (*func)(void *), void *data); void spawn_exit(void) ATTR_NORETURN; @@ -722,5 +749,17 @@ char *format_win32_error(DWORD err); #endif +#ifdef TOR_UNIT_TESTS +void tor_sleep_msec(int msec); +#endif + +#ifdef COMPAT_PRIVATE +#if !defined(HAVE_SOCKETPAIR) || defined(_WIN32) || defined(TOR_UNIT_TESTS) +#define NEED_ERSATZ_SOCKETPAIR +STATIC int tor_ersatz_socketpair(int family, int type, int protocol, + tor_socket_t fd[2]); +#endif +#endif + #endif diff --git a/src/common/compat_libevent.c b/src/common/compat_libevent.c index 200a7c65fb..74b54bb855 100644 --- a/src/common/compat_libevent.c +++ b/src/common/compat_libevent.c @@ -13,6 +13,8 @@ #include "compat.h" #include "compat_libevent.h" +#include "crypto.h" + #include "util.h" #include "torlog.h" @@ -415,6 +417,14 @@ tor_check_libevent_version(const char *m, int server, #define HEADER_VERSION _EVENT_VERSION #endif +/** Return a string representation of the version of Libevent that was used +* at compilation time. */ +const char * +tor_libevent_get_header_version_str(void) +{ + return HEADER_VERSION; +} + /** See whether the headers we were built against differ from the library we * linked against so much that we're likely to crash. If so, warn the * user. */ @@ -618,7 +628,25 @@ tor_add_bufferevent_to_rate_limit_group(struct bufferevent *bev, } #endif -#if defined(LIBEVENT_VERSION_NUMBER) && LIBEVENT_VERSION_NUMBER >= V(2,1,1) +int +tor_init_libevent_rng(void) +{ + int rv = 0; +#ifdef HAVE_EVUTIL_SECURE_RNG_INIT + char buf[256]; + if (evutil_secure_rng_init() < 0) { + rv = -1; + } + /* Older libevent -- manually initialize the RNG */ + crypto_rand(buf, 32); + evutil_secure_rng_add_bytes(buf, 32); + evutil_secure_rng_get_bytes(buf, sizeof(buf)); +#endif + return rv; +} + +#if defined(LIBEVENT_VERSION_NUMBER) && LIBEVENT_VERSION_NUMBER >= V(2,1,1) \ + && !defined(TOR_UNIT_TESTS) void tor_gettimeofday_cached(struct timeval *tv) { @@ -651,5 +679,45 @@ tor_gettimeofday_cache_clear(void) { cached_time_hires.tv_sec = 0; } + +#ifdef TOR_UNIT_TESTS +/** For testing: force-update the cached time to a given value. */ +void +tor_gettimeofday_cache_set(const struct timeval *tv) +{ + tor_assert(tv); + memcpy(&cached_time_hires, tv, sizeof(*tv)); +} +#endif #endif +/** + * As tor_gettimeofday_cached, but can never move backwards in time. + * + * The returned value may diverge from wall-clock time, since wall-clock time + * can trivially be adjusted backwards, and this can't. Don't mix wall-clock + * time with these values in the same calculation. + * + * Depending on implementation, this function may or may not "smooth out" huge + * jumps forward in wall-clock time. It may or may not keep its results + * advancing forward (as opposed to stalling) if the wall-clock time goes + * backwards. The current implementation does neither of of these. + * + * This function is not thread-safe; do not call it outside the main thread. + * + * In future versions of Tor, this may return a time does not have its + * origin at the Unix epoch. + */ +void +tor_gettimeofday_cached_monotonic(struct timeval *tv) +{ + struct timeval last_tv = { 0, 0 }; + + tor_gettimeofday_cached(tv); + if (timercmp(tv, &last_tv, <)) { + memcpy(tv, &last_tv, sizeof(struct timeval)); + } else { + memcpy(&last_tv, tv, sizeof(struct timeval)); + } +} + diff --git a/src/common/compat_libevent.h b/src/common/compat_libevent.h index 2472e2c49e..9ee7b49cfb 100644 --- a/src/common/compat_libevent.h +++ b/src/common/compat_libevent.h @@ -78,6 +78,7 @@ void tor_check_libevent_version(const char *m, int server, const char **badness_out); void tor_check_libevent_header_compatibility(void); const char *tor_libevent_get_version_str(void); +const char *tor_libevent_get_header_version_str(void); #ifdef USE_BUFFEREVENTS const struct timeval *tor_libevent_get_one_tick_timeout(void); @@ -88,8 +89,14 @@ int tor_add_bufferevent_to_rate_limit_group(struct bufferevent *bev, struct bufferevent_rate_limit_group *g); #endif +int tor_init_libevent_rng(void); + void tor_gettimeofday_cached(struct timeval *tv); void tor_gettimeofday_cache_clear(void); +#ifdef TOR_UNIT_TESTS +void tor_gettimeofday_cache_set(const struct timeval *tv); +#endif +void tor_gettimeofday_cached_monotonic(struct timeval *tv); #endif diff --git a/src/common/container.c b/src/common/container.c index eec497a3e6..b937d544fc 100644 --- a/src/common/container.c +++ b/src/common/container.c @@ -243,6 +243,25 @@ smartlist_strings_eq(const smartlist_t *sl1, const smartlist_t *sl2) return 1; } +/** Return true iff the two lists contain the same int pointer values in + * the same order, or if they are both NULL. */ +int +smartlist_ints_eq(const smartlist_t *sl1, const smartlist_t *sl2) +{ + if (sl1 == NULL) + return sl2 == NULL; + if (sl2 == NULL) + return 0; + if (smartlist_len(sl1) != smartlist_len(sl2)) + return 0; + SMARTLIST_FOREACH(sl1, int *, cp1, { + int *cp2 = smartlist_get(sl2, cp1_sl_idx); + if (*cp1 != *cp2) + return 0; + }); + return 1; +} + /** Return true iff <b>sl</b> has some element E such that * tor_memeq(E,<b>element</b>,DIGEST_LEN) */ @@ -708,6 +727,26 @@ smartlist_uniq_strings(smartlist_t *sl) smartlist_uniq(sl, compare_string_ptrs_, tor_free_); } +/** Helper: compare two pointers. */ +static int +compare_ptrs_(const void **_a, const void **_b) +{ + const void *a = *_a, *b = *_b; + if (a<b) + return -1; + else if (a==b) + return 0; + else + return 1; +} + +/** Sort <b>sl</b> in ascending order of the pointers it contains. */ +void +smartlist_sort_pointers(smartlist_t *sl) +{ + smartlist_sort(sl, compare_ptrs_); +} + /* Heap-based priority queue implementation for O(lg N) insert and remove. * Recall that the heap property is that, for every index I, h[I] < * H[LEFT_CHILD[I]] and h[I] < H[RIGHT_CHILD[I]]. @@ -985,7 +1024,7 @@ strmap_entries_eq(const strmap_entry_t *a, const strmap_entry_t *b) static INLINE unsigned int strmap_entry_hash(const strmap_entry_t *a) { - return ht_string_hash(a->key); + return (unsigned) siphash24g(a->key, strlen(a->key)); } /** Helper: compare digestmap_entry_t objects by key value. */ @@ -999,13 +1038,7 @@ digestmap_entries_eq(const digestmap_entry_t *a, const digestmap_entry_t *b) static INLINE unsigned int digestmap_entry_hash(const digestmap_entry_t *a) { -#if SIZEOF_INT != 8 - const uint32_t *p = (const uint32_t*)a->key; - return p[0] ^ p[1] ^ p[2] ^ p[3] ^ p[4]; -#else - const uint64_t *p = (const uint64_t*)a->key; - return p[0] ^ p[1]; -#endif + return (unsigned) siphash24g(a->key, DIGEST_LEN); } HT_PROTOTYPE(strmap_impl, strmap_entry_t, node, strmap_entry_hash, diff --git a/src/common/container.h b/src/common/container.h index fb93747945..0d31f2093b 100644 --- a/src/common/container.h +++ b/src/common/container.h @@ -7,6 +7,7 @@ #define TOR_CONTAINER_H #include "util.h" +#include "siphash.h" /** A resizeable list of pointers, with associated helpful functionality. * @@ -42,6 +43,7 @@ int smartlist_contains_string_case(const smartlist_t *sl, const char *element); int smartlist_contains_int_as_string(const smartlist_t *sl, int num); int smartlist_strings_eq(const smartlist_t *sl1, const smartlist_t *sl2); int smartlist_contains_digest(const smartlist_t *sl, const char *element); +int smartlist_ints_eq(const smartlist_t *sl1, const smartlist_t *sl2); int smartlist_overlap(const smartlist_t *sl1, const smartlist_t *sl2); void smartlist_intersect(smartlist_t *sl1, const smartlist_t *sl2); void smartlist_subtract(smartlist_t *sl1, const smartlist_t *sl2); @@ -101,6 +103,7 @@ void smartlist_uniq(smartlist_t *sl, void smartlist_sort_strings(smartlist_t *sl); void smartlist_sort_digests(smartlist_t *sl); void smartlist_sort_digests256(smartlist_t *sl); +void smartlist_sort_pointers(smartlist_t *sl); char *smartlist_get_most_frequent_string(smartlist_t *sl); char *smartlist_get_most_frequent_digest256(smartlist_t *sl); @@ -619,11 +622,11 @@ typedef struct { static INLINE void digestset_add(digestset_t *set, const char *digest) { - const uint32_t *p = (const uint32_t *)digest; - const uint32_t d1 = p[0] + (p[1]>>16); - const uint32_t d2 = p[1] + (p[2]>>16); - const uint32_t d3 = p[2] + (p[3]>>16); - const uint32_t d4 = p[3] + (p[0]>>16); + const uint64_t x = siphash24g(digest, 20); + const uint32_t d1 = (uint32_t) x; + const uint32_t d2 = (uint32_t)( (x>>16) + x); + const uint32_t d3 = (uint32_t)( (x>>32) + x); + const uint32_t d4 = (uint32_t)( (x>>48) + x); bitarray_set(set->ba, BIT(d1)); bitarray_set(set->ba, BIT(d2)); bitarray_set(set->ba, BIT(d3)); @@ -635,11 +638,11 @@ digestset_add(digestset_t *set, const char *digest) static INLINE int digestset_contains(const digestset_t *set, const char *digest) { - const uint32_t *p = (const uint32_t *)digest; - const uint32_t d1 = p[0] + (p[1]>>16); - const uint32_t d2 = p[1] + (p[2]>>16); - const uint32_t d3 = p[2] + (p[3]>>16); - const uint32_t d4 = p[3] + (p[0]>>16); + const uint64_t x = siphash24g(digest, 20); + const uint32_t d1 = (uint32_t) x; + const uint32_t d2 = (uint32_t)( (x>>16) + x); + const uint32_t d3 = (uint32_t)( (x>>32) + x); + const uint32_t d4 = (uint32_t)( (x>>48) + x); return bitarray_is_set(set->ba, BIT(d1)) && bitarray_is_set(set->ba, BIT(d2)) && bitarray_is_set(set->ba, BIT(d3)) && diff --git a/src/common/crypto.c b/src/common/crypto.c index 925beb3529..a247a87d48 100644 --- a/src/common/crypto.c +++ b/src/common/crypto.c @@ -56,6 +56,7 @@ #include "../common/util.h" #include "container.h" #include "compat.h" +#include "sandbox.h" #if OPENSSL_VERSION_NUMBER < OPENSSL_V_SERIES(0,9,8) #error "We require OpenSSL >= 0.9.8" @@ -114,7 +115,6 @@ crypto_get_rsa_padding_overhead(int padding) switch (padding) { case RSA_PKCS1_OAEP_PADDING: return PKCS1_OAEP_PADDING_OVERHEAD; - case RSA_PKCS1_PADDING: return PKCS1_PADDING_OVERHEAD; default: tor_assert(0); return -1; } } @@ -126,13 +126,15 @@ crypto_get_rsa_padding(int padding) { switch (padding) { - case PK_PKCS1_PADDING: return RSA_PKCS1_PADDING; case PK_PKCS1_OAEP_PADDING: return RSA_PKCS1_OAEP_PADDING; default: tor_assert(0); return -1; } } /** Boolean: has OpenSSL's crypto been initialized? */ +static int crypto_early_initialized_ = 0; + +/** Boolean: has OpenSSL's crypto been initialized? */ static int crypto_global_initialized_ = 0; /** Log all pending crypto errors at level <b>severity</b>. Use @@ -197,6 +199,27 @@ try_load_engine(const char *path, const char *engine) } #endif +/* Returns a trimmed and human-readable version of an openssl version string +* <b>raw_version</b>. They are usually in the form of 'OpenSSL 1.0.0b 10 +* May 2012' and this will parse them into a form similar to '1.0.0b' */ +static char * +parse_openssl_version_str(const char *raw_version) +{ + const char *end_of_version = NULL; + /* The output should be something like "OpenSSL 1.0.0b 10 May 2012. Let's + trim that down. */ + if (!strcmpstart(raw_version, "OpenSSL ")) { + raw_version += strlen("OpenSSL "); + end_of_version = strchr(raw_version, ' '); + } + + if (end_of_version) + return tor_strndup(raw_version, + end_of_version-raw_version); + else + return tor_strdup(raw_version); +} + static char *crypto_openssl_version_str = NULL; /* Return a human-readable version of the run-time openssl version number. */ const char * @@ -204,32 +227,67 @@ crypto_openssl_get_version_str(void) { if (crypto_openssl_version_str == NULL) { const char *raw_version = SSLeay_version(SSLEAY_VERSION); - const char *end_of_version = NULL; - /* The output should be something like "OpenSSL 1.0.0b 10 May 2012. Let's - trim that down. */ - if (!strcmpstart(raw_version, "OpenSSL ")) { - raw_version += strlen("OpenSSL "); - end_of_version = strchr(raw_version, ' '); - } - - if (end_of_version) - crypto_openssl_version_str = tor_strndup(raw_version, - end_of_version-raw_version); - else - crypto_openssl_version_str = tor_strdup(raw_version); + crypto_openssl_version_str = parse_openssl_version_str(raw_version); } return crypto_openssl_version_str; } +static char *crypto_openssl_header_version_str = NULL; +/* Return a human-readable version of the compile-time openssl version +* number. */ +const char * +crypto_openssl_get_header_version_str(void) +{ + if (crypto_openssl_header_version_str == NULL) { + crypto_openssl_header_version_str = + parse_openssl_version_str(OPENSSL_VERSION_TEXT); + } + return crypto_openssl_header_version_str; +} + +/** Make sure that openssl is using its default PRNG. Return 1 if we had to + * adjust it; 0 otherwise. */ +static int +crypto_force_rand_ssleay(void) +{ + if (RAND_get_rand_method() != RAND_SSLeay()) { + log_notice(LD_CRYPTO, "It appears that one of our engines has provided " + "a replacement the OpenSSL RNG. Resetting it to the default " + "implementation."); + RAND_set_rand_method(RAND_SSLeay()); + return 1; + } + return 0; +} + +/** Set up the siphash key if we haven't already done so. */ +int +crypto_init_siphash_key(void) +{ + static int have_seeded_siphash = 0; + struct sipkey key; + if (have_seeded_siphash) + return 0; + + if (crypto_rand((char*) &key, sizeof(key)) < 0) + return -1; + siphash_set_global_key(&key); + have_seeded_siphash = 1; + return 0; +} + /** Initialize the crypto library. Return 0 on success, -1 on failure. */ int -crypto_global_init(int useAccel, const char *accelName, const char *accelDir) +crypto_early_init(void) { - if (!crypto_global_initialized_) { + if (!crypto_early_initialized_) { + + crypto_early_initialized_ = 1; + ERR_load_crypto_strings(); OpenSSL_add_all_algorithms(); - crypto_global_initialized_ = 1; + setup_openssl_threading(); if (SSLeay() == OPENSSL_VERSION_NUMBER && @@ -251,6 +309,26 @@ crypto_global_init(int useAccel, const char *accelName, const char *accelDir) crypto_openssl_get_version_str()); } + crypto_force_rand_ssleay(); + + if (crypto_seed_rng(1) < 0) + return -1; + if (crypto_init_siphash_key() < 0) + return -1; + } + return 0; +} + +/** Initialize the crypto library. Return 0 on success, -1 on failure. + */ +int +crypto_global_init(int useAccel, const char *accelName, const char *accelDir) +{ + if (!crypto_global_initialized_) { + crypto_early_init(); + + crypto_global_initialized_ = 1; + if (useAccel > 0) { #ifdef DISABLE_ENGINES (void)accelName; @@ -286,28 +364,41 @@ crypto_global_init(int useAccel, const char *accelName, const char *accelDir) " setting default ciphers."); ENGINE_set_default(e, ENGINE_METHOD_ALL); } + /* Log, if available, the intersection of the set of algorithms + used by Tor and the set of algorithms available in the engine */ log_engine("RSA", ENGINE_get_default_RSA()); log_engine("DH", ENGINE_get_default_DH()); + log_engine("ECDH", ENGINE_get_default_ECDH()); + log_engine("ECDSA", ENGINE_get_default_ECDSA()); + log_engine("RAND", ENGINE_get_default_RAND()); log_engine("RAND (which we will not use)", ENGINE_get_default_RAND()); log_engine("SHA1", ENGINE_get_digest_engine(NID_sha1)); - log_engine("3DES", ENGINE_get_cipher_engine(NID_des_ede3_ecb)); - log_engine("AES", ENGINE_get_cipher_engine(NID_aes_128_ecb)); + log_engine("3DES-CBC", ENGINE_get_cipher_engine(NID_des_ede3_cbc)); + log_engine("AES-128-ECB", ENGINE_get_cipher_engine(NID_aes_128_ecb)); + log_engine("AES-128-CBC", ENGINE_get_cipher_engine(NID_aes_128_cbc)); +#ifdef NID_aes_128_ctr + log_engine("AES-128-CTR", ENGINE_get_cipher_engine(NID_aes_128_ctr)); +#endif +#ifdef NID_aes_128_gcm + log_engine("AES-128-GCM", ENGINE_get_cipher_engine(NID_aes_128_gcm)); +#endif + log_engine("AES-256-CBC", ENGINE_get_cipher_engine(NID_aes_256_cbc)); +#ifdef NID_aes_256_gcm + log_engine("AES-256-GCM", ENGINE_get_cipher_engine(NID_aes_256_gcm)); +#endif + #endif } else { log_info(LD_CRYPTO, "NOT using OpenSSL engine support."); } - if (RAND_get_rand_method() != RAND_SSLeay()) { - log_notice(LD_CRYPTO, "It appears that one of our engines has provided " - "a replacement the OpenSSL RNG. Resetting it to the default " - "implementation."); - RAND_set_rand_method(RAND_SSLeay()); + if (crypto_force_rand_ssleay()) { + if (crypto_seed_rng(1) < 0) + return -1; } evaluate_evp_for_aes(-1); evaluate_ctr_for_aes(); - - return crypto_seed_rng(1); } return 0; } @@ -1161,22 +1252,21 @@ int crypto_pk_asn1_encode(crypto_pk_t *pk, char *dest, size_t dest_len) { int len; - unsigned char *buf, *cp; - len = i2d_RSAPublicKey(pk->key, NULL); - if (len < 0 || (size_t)len > dest_len || dest_len > SIZE_T_CEILING) + unsigned char *buf = NULL; + + len = i2d_RSAPublicKey(pk->key, &buf); + if (len < 0 || buf == NULL) return -1; - cp = buf = tor_malloc(len+1); - len = i2d_RSAPublicKey(pk->key, &cp); - if (len < 0) { - crypto_log_errors(LOG_WARN,"encoding public key"); - tor_free(buf); + + if ((size_t)len > dest_len || dest_len > SIZE_T_CEILING) { + OPENSSL_free(buf); return -1; } /* We don't encode directly into 'dest', because that would be illegal * type-punning. (C99 is smarter than me, C99 is smarter than me...) */ memcpy(dest,buf,len); - tor_free(buf); + OPENSSL_free(buf); return len; } @@ -1207,24 +1297,17 @@ crypto_pk_asn1_decode(const char *str, size_t len) int crypto_pk_get_digest(crypto_pk_t *pk, char *digest_out) { - unsigned char *buf, *bufp; + unsigned char *buf = NULL; int len; - len = i2d_RSAPublicKey(pk->key, NULL); - if (len < 0) - return -1; - buf = bufp = tor_malloc(len+1); - len = i2d_RSAPublicKey(pk->key, &bufp); - if (len < 0) { - crypto_log_errors(LOG_WARN,"encoding public key"); - tor_free(buf); + len = i2d_RSAPublicKey(pk->key, &buf); + if (len < 0 || buf == NULL) return -1; - } if (crypto_digest(digest_out, (char*)buf, len) < 0) { - tor_free(buf); + OPENSSL_free(buf); return -1; } - tor_free(buf); + OPENSSL_free(buf); return 0; } @@ -1233,31 +1316,24 @@ crypto_pk_get_digest(crypto_pk_t *pk, char *digest_out) int crypto_pk_get_all_digests(crypto_pk_t *pk, digests_t *digests_out) { - unsigned char *buf, *bufp; + unsigned char *buf = NULL; int len; - len = i2d_RSAPublicKey(pk->key, NULL); - if (len < 0) + len = i2d_RSAPublicKey(pk->key, &buf); + if (len < 0 || buf == NULL) return -1; - buf = bufp = tor_malloc(len+1); - len = i2d_RSAPublicKey(pk->key, &bufp); - if (len < 0) { - crypto_log_errors(LOG_WARN,"encoding public key"); - tor_free(buf); - return -1; - } if (crypto_digest_all(digests_out, (char*)buf, len) < 0) { - tor_free(buf); + OPENSSL_free(buf); return -1; } - tor_free(buf); + OPENSSL_free(buf); return 0; } /** Copy <b>in</b> to the <b>outlen</b>-byte buffer <b>out</b>, adding spaces * every four spaces. */ -/* static */ void -add_spaces_to_fp(char *out, size_t outlen, const char *in) +void +crypto_add_spaces_to_fp(char *out, size_t outlen, const char *in) { int n = 0; char *end = out+outlen; @@ -1294,13 +1370,35 @@ crypto_pk_get_fingerprint(crypto_pk_t *pk, char *fp_out, int add_space) } base16_encode(hexdigest,sizeof(hexdigest),digest,DIGEST_LEN); if (add_space) { - add_spaces_to_fp(fp_out, FINGERPRINT_LEN+1, hexdigest); + crypto_add_spaces_to_fp(fp_out, FINGERPRINT_LEN+1, hexdigest); } else { strncpy(fp_out, hexdigest, HEX_DIGEST_LEN+1); } return 0; } +/** Given a private or public key <b>pk</b>, put a hashed fingerprint of + * the public key into <b>fp_out</b> (must have at least FINGERPRINT_LEN+1 + * bytes of space). Return 0 on success, -1 on failure. + * + * Hashed fingerprints are computed as the SHA1 digest of the SHA1 digest + * of the ASN.1 encoding of the public key, converted to hexadecimal, in + * upper case. + */ +int +crypto_pk_get_hashed_fingerprint(crypto_pk_t *pk, char *fp_out) +{ + char digest[DIGEST_LEN], hashed_digest[DIGEST_LEN]; + if (crypto_pk_get_digest(pk, digest)) { + return -1; + } + if (crypto_digest(hashed_digest, digest, DIGEST_LEN)) { + return -1; + } + base16_encode(fp_out, FINGERPRINT_LEN + 1, hashed_digest, DIGEST_LEN); + return 0; +} + /* symmetric crypto */ /** Return a pointer to the key set for the cipher in <b>env</b>. @@ -1496,7 +1594,7 @@ struct crypto_digest_t { SHA256_CTX sha2; /**< state for SHA256 */ } d; /**< State for the digest we're using. Only one member of the * union is usable, depending on the value of <b>algorithm</b>. */ - ENUM_BF(digest_algorithm_t) algorithm : 8; /**< Which algorithm is in use? */ + digest_algorithm_bitfield_t algorithm : 8; /**< Which algorithm is in use? */ }; /** Allocate and return a new digest object to compute SHA1 digests. @@ -1644,21 +1742,6 @@ crypto_digest_smartlist(char *digest_out, size_t len_out, crypto_digest_free(d); } -/** Compute the HMAC-SHA-1 of the <b>msg_len</b> bytes in <b>msg</b>, using - * the <b>key</b> of length <b>key_len</b>. Store the DIGEST_LEN-byte result - * in <b>hmac_out</b>. - */ -void -crypto_hmac_sha1(char *hmac_out, - const char *key, size_t key_len, - const char *msg, size_t msg_len) -{ - tor_assert(key_len < INT_MAX); - tor_assert(msg_len < INT_MAX); - HMAC(EVP_sha1(), key, (int)key_len, (unsigned char*)msg, (int)msg_len, - (unsigned char*)hmac_out, NULL); -} - /** Compute the HMAC-SHA-256 of the <b>msg_len</b> bytes in <b>msg</b>, using * the <b>key</b> of length <b>key_len</b>. Store the DIGEST256_LEN-byte * result in <b>hmac_out</b>. @@ -1727,7 +1810,7 @@ crypto_store_dynamic_dh_modulus(const char *fname) { int len, new_len; DH *dh = NULL; - unsigned char *dh_string_repr = NULL, *cp = NULL; + unsigned char *dh_string_repr = NULL; char *base64_encoded_dh = NULL; char *file_string = NULL; int retval = -1; @@ -1751,15 +1834,8 @@ crypto_store_dynamic_dh_modulus(const char *fname) if (!BN_set_word(dh->g, DH_GENERATOR)) goto done; - len = i2d_DHparams(dh, NULL); - if (len < 0) { - log_warn(LD_CRYPTO, "Error occured while DER encoding DH modulus (1)."); - goto done; - } - - cp = dh_string_repr = tor_malloc_zero(len+1); - len = i2d_DHparams(dh, &cp); - if ((len < 0) || ((cp - dh_string_repr) != len)) { + len = i2d_DHparams(dh, &dh_string_repr); + if ((len < 0) || (dh_string_repr == NULL)) { log_warn(LD_CRYPTO, "Error occured while DER encoding DH modulus (2)."); goto done; } @@ -1786,7 +1862,8 @@ crypto_store_dynamic_dh_modulus(const char *fname) done: if (dh) DH_free(dh); - tor_free(dh_string_repr); + if (dh_string_repr) + OPENSSL_free(dh_string_repr); tor_free(base64_encoded_dh); tor_free(file_string); @@ -2394,7 +2471,8 @@ crypto_strongest_rand(uint8_t *out, size_t out_len) return 0; #else for (i = 0; filenames[i]; ++i) { - fd = open(filenames[i], O_RDONLY, 0); + log_debug(LD_FS, "Opening %s for entropy", filenames[i]); + fd = open(sandbox_intern_string(filenames[i]), O_RDONLY, 0); if (fd<0) continue; log_info(LD_CRYPTO, "Reading entropy from \"%s\"", filenames[i]); n = read_all(fd, (char*)out, out_len, 0); @@ -2449,8 +2527,8 @@ crypto_seed_rng(int startup) /** Write <b>n</b> bytes of strong random data to <b>to</b>. Return 0 on * success, -1 on failure. */ -int -crypto_rand(char *to, size_t n) +MOCK_IMPL(int, +crypto_rand, (char *to, size_t n)) { int r; tor_assert(n < INT_MAX); @@ -3026,7 +3104,7 @@ openssl_locking_cb_(int mode, int n, const char *file, int line) (void)file; (void)line; if (!openssl_mutexes_) - /* This is not a really good fix for the + /* This is not a really good fix for the * "release-freed-lock-from-separate-thread-on-shutdown" problem, but * it can't hurt. */ return; @@ -3144,6 +3222,7 @@ crypto_global_cleanup(void) } #endif tor_free(crypto_openssl_version_str); + tor_free(crypto_openssl_header_version_str); return 0; } diff --git a/src/common/crypto.h b/src/common/crypto.h index 2fbca4c260..aa4271aa33 100644 --- a/src/common/crypto.h +++ b/src/common/crypto.h @@ -15,6 +15,7 @@ #include <stdio.h> #include "torint.h" +#include "testsupport.h" /* Macro to create an arbitrary OpenSSL version number as used by @@ -69,13 +70,9 @@ * signs removed. */ #define BASE64_DIGEST256_LEN 43 -/** Constant used to indicate PKCS1 padding for public-key encryption */ -#define PK_PKCS1_PADDING 60001 /** Constant used to indicate OAEP padding for public-key encryption */ #define PK_PKCS1_OAEP_PADDING 60002 -/** Number of bytes added for PKCS1 padding. */ -#define PKCS1_PADDING_OVERHEAD 11 /** Number of bytes added for PKCS1-OAEP padding. */ #define PKCS1_OAEP_PADDING_OVERHEAD 42 @@ -92,6 +89,7 @@ typedef enum { DIGEST_SHA256 = 1, } digest_algorithm_t; #define N_DIGEST_ALGORITHMS (DIGEST_SHA256+1) +#define digest_algorithm_bitfield_t ENUM_BF(digest_algorithm_t) /** A set of all the digests we know how to compute, taken on a single * string. Any digests that are shorter than 256 bits are right-padded @@ -112,6 +110,8 @@ typedef struct crypto_dh_t crypto_dh_t; /* global state */ const char * crypto_openssl_get_version_str(void); +const char * crypto_openssl_get_header_version_str(void); +int crypto_early_init(void); int crypto_global_init(int hardwareAccel, const char *accelName, const char *accelPath); @@ -183,6 +183,7 @@ crypto_pk_t *crypto_pk_asn1_decode(const char *str, size_t len); int crypto_pk_get_digest(crypto_pk_t *pk, char *digest_out); int crypto_pk_get_all_digests(crypto_pk_t *pk, digests_t *digests_out); int crypto_pk_get_fingerprint(crypto_pk_t *pk, char *fp_out,int add_space); +int crypto_pk_get_hashed_fingerprint(crypto_pk_t *pk, char *fp_out); /* symmetric crypto */ const char *crypto_cipher_get_key(crypto_cipher_t *env); @@ -221,9 +222,6 @@ void crypto_digest_get_digest(crypto_digest_t *digest, crypto_digest_t *crypto_digest_dup(const crypto_digest_t *digest); void crypto_digest_assign(crypto_digest_t *into, const crypto_digest_t *from); -void crypto_hmac_sha1(char *hmac_out, - const char *key, size_t key_len, - const char *msg, size_t msg_len); void crypto_hmac_sha256(char *hmac_out, const char *key, size_t key_len, const char *msg, size_t msg_len); @@ -254,13 +252,14 @@ int crypto_expand_key_material_rfc5869_sha256( /* random numbers */ int crypto_seed_rng(int startup); -int crypto_rand(char *to, size_t n); +MOCK_DECL(int,crypto_rand,(char *to, size_t n)); int crypto_strongest_rand(uint8_t *out, size_t out_len); int crypto_rand_int(unsigned int max); uint64_t crypto_rand_uint64(uint64_t max); double crypto_rand_double(void); struct tor_weak_rng_t; void crypto_seed_weak_rng(struct tor_weak_rng_t *rng); +int crypto_init_siphash_key(void); char *crypto_random_hostname(int min_rand_len, int max_rand_len, const char *prefix, const char *suffix); @@ -290,7 +289,6 @@ void secret_to_key(char *key_out, size_t key_out_len, const char *secret, /** OpenSSL-based utility functions. */ void memwipe(void *mem, uint8_t byte, size_t sz); -#ifdef CRYPTO_PRIVATE /* Prototypes for private functions only used by tortls.c, crypto.c, and the * unit tests. */ struct rsa_st; @@ -301,9 +299,8 @@ crypto_pk_t *crypto_new_pk_from_rsa_(struct rsa_st *rsa); struct evp_pkey_st *crypto_pk_get_evp_pkey_(crypto_pk_t *env, int private); struct dh_st *crypto_dh_get_dh_(crypto_dh_t *dh); -/* Prototypes for private functions only used by crypto.c and test.c*/ -void add_spaces_to_fp(char *out, size_t outlen, const char *in); -#endif + +void crypto_add_spaces_to_fp(char *out, size_t outlen, const char *in); #endif diff --git a/src/common/crypto_curve25519.c b/src/common/crypto_curve25519.c index 88c723f37c..9e83440e16 100644 --- a/src/common/crypto_curve25519.c +++ b/src/common/crypto_curve25519.c @@ -29,7 +29,7 @@ int curve25519_donna(uint8_t *mypublic, #endif #endif -int +STATIC int curve25519_impl(uint8_t *output, const uint8_t *secret, const uint8_t *basepoint) { diff --git a/src/common/crypto_curve25519.h b/src/common/crypto_curve25519.h index 652f1883c6..57018ac2f5 100644 --- a/src/common/crypto_curve25519.h +++ b/src/common/crypto_curve25519.h @@ -4,6 +4,7 @@ #ifndef TOR_CRYPTO_CURVE25519_H #define TOR_CRYPTO_CURVE25519_H +#include "testsupport.h" #include "torint.h" /** Length of a curve25519 public key when encoded. */ @@ -30,6 +31,11 @@ typedef struct curve25519_keypair_t { } curve25519_keypair_t; #ifdef CURVE25519_ENABLED +/* These functions require that we actually know how to use curve25519 keys. + * The other data structures and functions in this header let us parse them, + * store them, and move them around. + */ + int curve25519_public_key_is_ok(const curve25519_public_key_t *); int curve25519_secret_key_generate(curve25519_secret_key_t *key_out, @@ -52,8 +58,8 @@ int curve25519_keypair_read_from_file(curve25519_keypair_t *keypair_out, const char *fname); #ifdef CRYPTO_CURVE25519_PRIVATE -int curve25519_impl(uint8_t *output, const uint8_t *secret, - const uint8_t *basepoint); +STATIC int curve25519_impl(uint8_t *output, const uint8_t *secret, + const uint8_t *basepoint); #endif #endif diff --git a/src/common/crypto_format.c b/src/common/crypto_format.c index 93932f839c..be669c8d2b 100644 --- a/src/common/crypto_format.c +++ b/src/common/crypto_format.c @@ -3,7 +3,6 @@ /* Formatting and parsing code for crypto-related data structures. */ -#define CRYPTO_CURVE25519_PRIVATE #include "orconfig.h" #ifdef HAVE_SYS_STAT_H #include <sys/stat.h> diff --git a/src/common/gen_server_ciphers.py b/src/common/gen_server_ciphers.py deleted file mode 100755 index 97ed9d0469..0000000000 --- a/src/common/gen_server_ciphers.py +++ /dev/null @@ -1,115 +0,0 @@ -#!/usr/bin/python -# Copyright 2014, The Tor Project, Inc -# See LICENSE for licensing information - -# This script parses openssl headers to find ciphersuite names, determines -# which ones we should be willing to use as a server, and sorts them according -# to preference rules. -# -# Run it on all the files in your openssl include directory. - -import re -import sys - -EPHEMERAL_INDICATORS = [ "_EDH_", "_DHE_", "_ECDHE_" ] -BAD_STUFF = [ "_DES_40_", "MD5", "_RC4_", "_DES_64_", - "_SEED_", "_CAMELLIA_", "_NULL" ] - -# these never get #ifdeffed. -MANDATORY = [ - "TLS1_TXT_DHE_RSA_WITH_AES_256_SHA", - "TLS1_TXT_DHE_RSA_WITH_AES_128_SHA", - "SSL3_TXT_EDH_RSA_DES_192_CBC3_SHA", -] - -def find_ciphers(filename): - with open(filename) as f: - for line in f: - m = re.search(r'(?:SSL3|TLS1)_TXT_\w+', line) - if m: - yield m.group(0) - -def usable_cipher(ciph): - ephemeral = False - for e in EPHEMERAL_INDICATORS: - if e in ciph: - ephemeral = True - if not ephemeral: - return False - - if "_RSA_" not in ciph: - return False - - for b in BAD_STUFF: - if b in ciph: - return False - return True - -# All fields we sort on, in order of priority. -FIELDS = [ 'cipher', 'fwsec', 'mode', 'digest', 'bitlength' ] -# Map from sorted fields to recognized value in descending order of goodness -FIELD_VALS = { 'cipher' : [ 'AES', 'DES'], - 'fwsec' : [ 'ECDHE', 'DHE' ], - 'mode' : [ 'GCM', 'CBC' ], - 'digest' : [ 'SHA384', 'SHA256', 'SHA' ], - 'bitlength' : [ '256', '128', '192' ], -} - -class Ciphersuite(object): - def __init__(self, name, fwsec, cipher, bitlength, mode, digest): - self.name = name - self.fwsec = fwsec - self.cipher = cipher - self.bitlength = bitlength - self.mode = mode - self.digest = digest - - for f in FIELDS: - assert(getattr(self, f) in FIELD_VALS[f]) - - def sort_key(self): - return tuple(FIELD_VALS[f].index(getattr(self,f)) for f in FIELDS) - - -def parse_cipher(ciph): - m = re.match('(?:TLS1|SSL3)_TXT_(EDH|DHE|ECDHE)_RSA(?:_WITH)?_(AES|DES)_(256|128|192)(|_CBC|_CBC3|_GCM)_(SHA|SHA256|SHA384)$', ciph) - - if not m: - print "/* Couldn't parse %s ! */"%ciph - return None - - fwsec, cipher, bits, mode, digest = m.groups() - if fwsec == 'EDH': - fwsec = 'DHE' - - if mode in [ '_CBC3', '_CBC', '' ]: - mode = 'CBC' - elif mode == '_GCM': - mode = 'GCM' - - return Ciphersuite(ciph, fwsec, cipher, bits, mode, digest) - -ALL_CIPHERS = [] - -for fname in sys.argv[1:]: - ALL_CIPHERS += (parse_cipher(c) - for c in find_ciphers(fname) - if usable_cipher(c) ) - -ALL_CIPHERS.sort(key=Ciphersuite.sort_key) - -for c in ALL_CIPHERS: - if c is ALL_CIPHERS[-1]: - colon = ';' - else: - colon = ' ":"' - - if c.name in MANDATORY: - print " /* Required */" - print ' %s%s'%(c.name,colon) - else: - print "#ifdef %s"%c.name - print ' %s%s'%(c.name,colon) - print "#endif" - - diff --git a/src/common/get_mozilla_ciphers.py b/src/common/get_mozilla_ciphers.py deleted file mode 100644 index 0636eb3658..0000000000 --- a/src/common/get_mozilla_ciphers.py +++ /dev/null @@ -1,210 +0,0 @@ -#!/usr/bin/python -# coding=utf-8 -# Copyright 2011, The Tor Project, Inc -# original version by Arturo Filastò -# See LICENSE for licensing information - -# This script parses Firefox and OpenSSL sources, and uses this information -# to generate a ciphers.inc file. -# -# It takes two arguments: the location of a firefox source directory, and the -# location of an openssl source directory. - -import os -import re -import sys - -if len(sys.argv) != 3: - print >>sys.stderr, "Syntax: get_mozilla_ciphers.py <firefox-source-dir> <openssl-source-dir>" - sys.exit(1) - -ff_root = sys.argv[1] -ossl_root = sys.argv[2] - -def ff(s): - return os.path.join(ff_root, s) -def ossl(s): - return os.path.join(ossl_root, s) - -##### -# Read the cpp file to understand what Ciphers map to what name : -# Make "ciphers" a map from name used in the javascript to a cipher macro name -fileA = open(ff('security/manager/ssl/src/nsNSSComponent.cpp'),'r') - -# The input format is a file containing exactly one section of the form: -# static CipherPref CipherPrefs[] = { -# {"name", MACRO_NAME}, // comment -# ... -# {NULL, 0} -# } - -inCipherSection = False -cipherLines = [] -for line in fileA: - if line.startswith('static const CipherPref sCipherPrefs[]'): - # Get the starting boundary of the Cipher Preferences - inCipherSection = True - elif inCipherSection: - line = line.strip() - if line.startswith('{ nullptr, 0}'): - # At the ending boundary of the Cipher Prefs - break - else: - cipherLines.append(line) -fileA.close() - -# Parse the lines and put them into a dict -ciphers = {} -cipher_pref = {} -key_pending = None -for line in cipherLines: - m = re.search(r'^{\s*\"([^\"]+)\",\s*(\S+)\s*(?:,\s*(true|false))?\s*}', line) - if m: - assert not key_pending - key,value,enabled = m.groups() - if enabled == 'true': - ciphers[key] = value - cipher_pref[value] = key - continue - m = re.search(r'^{\s*\"([^\"]+)\",', line) - if m: - assert not key_pending - key_pending = m.group(1) - continue - m = re.search(r'^\s*(\S+)(?:,\s*(true|false))?\s*}', line) - if m: - assert key_pending - key = key_pending - value,enabled = m.groups() - key_pending = None - if enabled == 'true': - ciphers[key] = value - cipher_pref[value] = key - -#### -# Now find the correct order for the ciphers -fileC = open(ff('security/nss/lib/ssl/ssl3con.c'), 'r') -firefox_ciphers = [] -inEnum=False -for line in fileC: - if not inEnum: - if "ssl3CipherSuiteCfg cipherSuites[" in line: - inEnum = True - continue - - if line.startswith("};"): - break - - m = re.match(r'^\s*\{\s*([A-Z_0-9]+),', line) - if m: - firefox_ciphers.append(m.group(1)) - -fileC.close() - -##### -# Read the JS file to understand what ciphers are enabled. The format is -# pref("name", true/false); -# Build a map enabled_ciphers from javascript name to "true" or "false", -# and an (unordered!) list of the macro names for those ciphers that are -# enabled. -fileB = open(ff('netwerk/base/public/security-prefs.js'), 'r') - -enabled_ciphers = {} -for line in fileB: - m = re.match(r'pref\(\"([^\"]+)\"\s*,\s*(\S*)\s*\)', line) - if not m: - continue - key, val = m.groups() - if key.startswith("security.ssl3"): - enabled_ciphers[key] = val -fileB.close() - -used_ciphers = [] -for k, v in enabled_ciphers.items(): - if v == "true": - used_ciphers.append(ciphers[k]) - -#oSSLinclude = ('/usr/include/openssl/ssl3.h', '/usr/include/openssl/ssl.h', -# '/usr/include/openssl/ssl2.h', '/usr/include/openssl/ssl23.h', -# '/usr/include/openssl/tls1.h') -oSSLinclude = ('ssl/ssl3.h', 'ssl/ssl.h', - 'ssl/ssl2.h', 'ssl/ssl23.h', - 'ssl/tls1.h') - -##### -# This reads the hex code for the ciphers that are used by firefox. -# sslProtoD is set to a map from macro name to macro value in sslproto.h; -# cipher_codes is set to an (unordered!) list of these hex values. -sslProto = open(ff('security/nss/lib/ssl/sslproto.h'), 'r') -sslProtoD = {} - -for line in sslProto: - m = re.match('#define\s+(\S+)\s+(\S+)', line) - if m: - key, value = m.groups() - sslProtoD[key] = value -sslProto.close() - -cipher_codes = [] -for x in used_ciphers: - cipher_codes.append(sslProtoD[x].lower()) - -#### -# Now read through all the openssl include files, and try to find the openssl -# macro names for those files. -openssl_macro_by_hex = {} -all_openssl_macros = {} -for fl in oSSLinclude: - fp = open(ossl(fl), 'r') - for line in fp.readlines(): - m = re.match('#define\s+(\S+)\s+(\S+)', line) - if m: - value,key = m.groups() - if key.startswith('0x') and "_CK_" in value: - key = key.replace('0x0300','0x').lower() - #print "%s %s" % (key, value) - openssl_macro_by_hex[key] = value - all_openssl_macros[value]=key - fp.close() - -# Now generate the output. -print """\ -/* This is an include file used to define the list of ciphers clients should - * advertise. Before including it, you should define the CIPHER and XCIPHER - * macros. - * - * This file was automatically generated by get_mozilla_ciphers.py. - */""" -# Go in order by the order in CipherPrefs -for firefox_macro in firefox_ciphers: - - try: - js_cipher_name = cipher_pref[firefox_macro] - except KeyError: - # This one has no javascript preference. - continue - - # The cipher needs to be enabled in security-prefs.js - if enabled_ciphers.get(js_cipher_name, 'false') != 'true': - continue - - hexval = sslProtoD[firefox_macro].lower() - - try: - openssl_macro = openssl_macro_by_hex[hexval.lower()] - openssl_macro = openssl_macro.replace("_CK_", "_TXT_") - if openssl_macro not in all_openssl_macros: - raise KeyError() - format = {'hex':hexval, 'macro':openssl_macro, 'note':""} - except KeyError: - # openssl doesn't have a macro for this. - format = {'hex':hexval, 'macro':firefox_macro, - 'note':"/* No openssl macro found for "+hexval+" */\n"} - - res = """\ -%(note)s#ifdef %(macro)s - CIPHER(%(hex)s, %(macro)s) -#else - XCIPHER(%(hex)s, %(macro)s) -#endif""" % format - print res diff --git a/src/common/include.am b/src/common/include.am index b796ebfae8..68e0110c26 100644 --- a/src/common/include.am +++ b/src/common/include.am @@ -1,5 +1,15 @@ -noinst_LIBRARIES+= src/common/libor.a src/common/libor-crypto.a src/common/libor-event.a +noinst_LIBRARIES += \ + src/common/libor.a \ + src/common/libor-crypto.a \ + src/common/libor-event.a + +if UNITTESTS_ENABLED +noinst_LIBRARIES += \ + src/common/libor-testing.a \ + src/common/libor-crypto-testing.a \ + src/common/libor-event-testing.a +endif EXTRA_DIST+= \ src/common/common_sha1.i \ @@ -14,9 +24,21 @@ else libor_extra_source= endif +if USE_MEMPOOLS +libor_mempool_source=src/common/mempool.c +libor_mempool_header=src/common/mempool.h +else +libor_mempool_source= +libor_mempool_header= +endif + +src_common_libcurve25519_donna_a_CFLAGS= + if BUILD_CURVE25519_DONNA src_common_libcurve25519_donna_a_SOURCES=\ src/ext/curve25519_donna/curve25519-donna.c +src_common_libcurve25519_donna_a_CFLAGS+=\ + @F_OMIT_FRAME_POINTER@ noinst_LIBRARIES+=src/common/libcurve25519_donna.a LIBDONNA=src/common/libcurve25519_donna.a else @@ -30,26 +52,27 @@ LIBDONNA= endif endif -src_common_libcurve25519_donna_a_CFLAGS = - if CURVE25519_ENABLED libcrypto_extra_source=src/common/crypto_curve25519.c endif -src_common_libor_a_SOURCES = \ +LIBOR_A_SOURCES = \ src/common/address.c \ + src/common/backtrace.c \ src/common/compat.c \ src/common/container.c \ src/common/di_ops.c \ src/common/log.c \ src/common/memarea.c \ - src/common/mempool.c \ - src/common/procmon.c \ src/common/util.c \ src/common/util_codedigest.c \ - $(libor_extra_source) + src/common/util_process.c \ + src/common/sandbox.c \ + src/ext/csiphash.c \ + $(libor_extra_source) \ + $(libor_mempool_source) -src_common_libor_crypto_a_SOURCES = \ +LIBOR_CRYPTO_A_SOURCES = \ src/common/aes.c \ src/common/crypto.c \ src/common/crypto_format.c \ @@ -57,10 +80,29 @@ src_common_libor_crypto_a_SOURCES = \ src/common/tortls.c \ $(libcrypto_extra_source) -src_common_libor_event_a_SOURCES = src/common/compat_libevent.c +LIBOR_EVENT_A_SOURCES = \ + src/common/compat_libevent.c \ + src/common/procmon.c + +src_common_libor_a_SOURCES = $(LIBOR_A_SOURCES) +src_common_libor_crypto_a_SOURCES = $(LIBOR_CRYPTO_A_SOURCES) +src_common_libor_event_a_SOURCES = $(LIBOR_EVENT_A_SOURCES) + +src_common_libor_testing_a_SOURCES = $(LIBOR_A_SOURCES) +src_common_libor_crypto_testing_a_SOURCES = $(LIBOR_CRYPTO_A_SOURCES) +src_common_libor_event_testing_a_SOURCES = $(LIBOR_EVENT_A_SOURCES) + +src_common_libor_testing_a_CPPFLAGS = -DTOR_UNIT_TESTS $(AM_CPPFLAGS) +src_common_libor_crypto_testing_a_CPPFLAGS = -DTOR_UNIT_TESTS $(AM_CPPFLAGS) +src_common_libor_event_testing_a_CPPFLAGS = -DTOR_UNIT_TESTS $(AM_CPPFLAGS) +src_common_libor_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +src_common_libor_crypto_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +src_common_libor_event_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) + COMMONHEADERS = \ src/common/address.h \ + src/common/backtrace.h \ src/common/aes.h \ src/common/ciphers.inc \ src/common/compat.h \ @@ -70,13 +112,17 @@ COMMONHEADERS = \ src/common/crypto_curve25519.h \ src/common/di_ops.h \ src/common/memarea.h \ - src/common/mempool.h \ + src/common/linux_syscalls.inc \ src/common/procmon.h \ + src/common/sandbox.h \ + src/common/testsupport.h \ src/common/torgzip.h \ src/common/torint.h \ src/common/torlog.h \ src/common/tortls.h \ - src/common/util.h + src/common/util.h \ + src/common/util_process.h \ + $(libor_mempool_header) noinst_HEADERS+= $(COMMONHEADERS) diff --git a/src/common/linux_syscalls.inc b/src/common/linux_syscalls.inc new file mode 100644 index 0000000000..cf47c73809 --- /dev/null +++ b/src/common/linux_syscalls.inc @@ -0,0 +1,1153 @@ +/* Automatically generated with + gen_linux_syscalls.pl /usr/include/asm/unistd*.h + Do not edit. + */ +static const struct { + int syscall_num; const char *syscall_name; +} SYSCALLS_BY_NUMBER[] = { +#ifdef __NR__llseek + { __NR__llseek, "_llseek" }, +#endif +#ifdef __NR__newselect + { __NR__newselect, "_newselect" }, +#endif +#ifdef __NR__sysctl + { __NR__sysctl, "_sysctl" }, +#endif +#ifdef __NR_accept + { __NR_accept, "accept" }, +#endif +#ifdef __NR_accept4 + { __NR_accept4, "accept4" }, +#endif +#ifdef __NR_access + { __NR_access, "access" }, +#endif +#ifdef __NR_acct + { __NR_acct, "acct" }, +#endif +#ifdef __NR_add_key + { __NR_add_key, "add_key" }, +#endif +#ifdef __NR_adjtimex + { __NR_adjtimex, "adjtimex" }, +#endif +#ifdef __NR_afs_syscall + { __NR_afs_syscall, "afs_syscall" }, +#endif +#ifdef __NR_alarm + { __NR_alarm, "alarm" }, +#endif +#ifdef __NR_arch_prctl + { __NR_arch_prctl, "arch_prctl" }, +#endif +#ifdef __NR_bdflush + { __NR_bdflush, "bdflush" }, +#endif +#ifdef __NR_bind + { __NR_bind, "bind" }, +#endif +#ifdef __NR_break + { __NR_break, "break" }, +#endif +#ifdef __NR_brk + { __NR_brk, "brk" }, +#endif +#ifdef __NR_capget + { __NR_capget, "capget" }, +#endif +#ifdef __NR_capset + { __NR_capset, "capset" }, +#endif +#ifdef __NR_chdir + { __NR_chdir, "chdir" }, +#endif +#ifdef __NR_chmod + { __NR_chmod, "chmod" }, +#endif +#ifdef __NR_chown + { __NR_chown, "chown" }, +#endif +#ifdef __NR_chown32 + { __NR_chown32, "chown32" }, +#endif +#ifdef __NR_chroot + { __NR_chroot, "chroot" }, +#endif +#ifdef __NR_clock_adjtime + { __NR_clock_adjtime, "clock_adjtime" }, +#endif +#ifdef __NR_clock_getres + { __NR_clock_getres, "clock_getres" }, +#endif +#ifdef __NR_clock_gettime + { __NR_clock_gettime, "clock_gettime" }, +#endif +#ifdef __NR_clock_nanosleep + { __NR_clock_nanosleep, "clock_nanosleep" }, +#endif +#ifdef __NR_clock_settime + { __NR_clock_settime, "clock_settime" }, +#endif +#ifdef __NR_clone + { __NR_clone, "clone" }, +#endif +#ifdef __NR_close + { __NR_close, "close" }, +#endif +#ifdef __NR_connect + { __NR_connect, "connect" }, +#endif +#ifdef __NR_creat + { __NR_creat, "creat" }, +#endif +#ifdef __NR_create_module + { __NR_create_module, "create_module" }, +#endif +#ifdef __NR_delete_module + { __NR_delete_module, "delete_module" }, +#endif +#ifdef __NR_dup + { __NR_dup, "dup" }, +#endif +#ifdef __NR_dup2 + { __NR_dup2, "dup2" }, +#endif +#ifdef __NR_dup3 + { __NR_dup3, "dup3" }, +#endif +#ifdef __NR_epoll_create + { __NR_epoll_create, "epoll_create" }, +#endif +#ifdef __NR_epoll_create1 + { __NR_epoll_create1, "epoll_create1" }, +#endif +#ifdef __NR_epoll_ctl + { __NR_epoll_ctl, "epoll_ctl" }, +#endif +#ifdef __NR_epoll_ctl_old + { __NR_epoll_ctl_old, "epoll_ctl_old" }, +#endif +#ifdef __NR_epoll_pwait + { __NR_epoll_pwait, "epoll_pwait" }, +#endif +#ifdef __NR_epoll_wait + { __NR_epoll_wait, "epoll_wait" }, +#endif +#ifdef __NR_epoll_wait_old + { __NR_epoll_wait_old, "epoll_wait_old" }, +#endif +#ifdef __NR_eventfd + { __NR_eventfd, "eventfd" }, +#endif +#ifdef __NR_eventfd2 + { __NR_eventfd2, "eventfd2" }, +#endif +#ifdef __NR_execve + { __NR_execve, "execve" }, +#endif +#ifdef __NR_exit + { __NR_exit, "exit" }, +#endif +#ifdef __NR_exit_group + { __NR_exit_group, "exit_group" }, +#endif +#ifdef __NR_faccessat + { __NR_faccessat, "faccessat" }, +#endif +#ifdef __NR_fadvise64 + { __NR_fadvise64, "fadvise64" }, +#endif +#ifdef __NR_fadvise64_64 + { __NR_fadvise64_64, "fadvise64_64" }, +#endif +#ifdef __NR_fallocate + { __NR_fallocate, "fallocate" }, +#endif +#ifdef __NR_fanotify_init + { __NR_fanotify_init, "fanotify_init" }, +#endif +#ifdef __NR_fanotify_mark + { __NR_fanotify_mark, "fanotify_mark" }, +#endif +#ifdef __NR_fchdir + { __NR_fchdir, "fchdir" }, +#endif +#ifdef __NR_fchmod + { __NR_fchmod, "fchmod" }, +#endif +#ifdef __NR_fchmodat + { __NR_fchmodat, "fchmodat" }, +#endif +#ifdef __NR_fchown + { __NR_fchown, "fchown" }, +#endif +#ifdef __NR_fchown32 + { __NR_fchown32, "fchown32" }, +#endif +#ifdef __NR_fchownat + { __NR_fchownat, "fchownat" }, +#endif +#ifdef __NR_fcntl + { __NR_fcntl, "fcntl" }, +#endif +#ifdef __NR_fcntl64 + { __NR_fcntl64, "fcntl64" }, +#endif +#ifdef __NR_fdatasync + { __NR_fdatasync, "fdatasync" }, +#endif +#ifdef __NR_fgetxattr + { __NR_fgetxattr, "fgetxattr" }, +#endif +#ifdef __NR_finit_module + { __NR_finit_module, "finit_module" }, +#endif +#ifdef __NR_flistxattr + { __NR_flistxattr, "flistxattr" }, +#endif +#ifdef __NR_flock + { __NR_flock, "flock" }, +#endif +#ifdef __NR_fork + { __NR_fork, "fork" }, +#endif +#ifdef __NR_fremovexattr + { __NR_fremovexattr, "fremovexattr" }, +#endif +#ifdef __NR_fsetxattr + { __NR_fsetxattr, "fsetxattr" }, +#endif +#ifdef __NR_fstat + { __NR_fstat, "fstat" }, +#endif +#ifdef __NR_fstat64 + { __NR_fstat64, "fstat64" }, +#endif +#ifdef __NR_fstatat64 + { __NR_fstatat64, "fstatat64" }, +#endif +#ifdef __NR_fstatfs + { __NR_fstatfs, "fstatfs" }, +#endif +#ifdef __NR_fstatfs64 + { __NR_fstatfs64, "fstatfs64" }, +#endif +#ifdef __NR_fsync + { __NR_fsync, "fsync" }, +#endif +#ifdef __NR_ftime + { __NR_ftime, "ftime" }, +#endif +#ifdef __NR_ftruncate + { __NR_ftruncate, "ftruncate" }, +#endif +#ifdef __NR_ftruncate64 + { __NR_ftruncate64, "ftruncate64" }, +#endif +#ifdef __NR_futex + { __NR_futex, "futex" }, +#endif +#ifdef __NR_futimesat + { __NR_futimesat, "futimesat" }, +#endif +#ifdef __NR_get_kernel_syms + { __NR_get_kernel_syms, "get_kernel_syms" }, +#endif +#ifdef __NR_get_mempolicy + { __NR_get_mempolicy, "get_mempolicy" }, +#endif +#ifdef __NR_get_robust_list + { __NR_get_robust_list, "get_robust_list" }, +#endif +#ifdef __NR_get_thread_area + { __NR_get_thread_area, "get_thread_area" }, +#endif +#ifdef __NR_getcpu + { __NR_getcpu, "getcpu" }, +#endif +#ifdef __NR_getcwd + { __NR_getcwd, "getcwd" }, +#endif +#ifdef __NR_getdents + { __NR_getdents, "getdents" }, +#endif +#ifdef __NR_getdents64 + { __NR_getdents64, "getdents64" }, +#endif +#ifdef __NR_getegid + { __NR_getegid, "getegid" }, +#endif +#ifdef __NR_getegid32 + { __NR_getegid32, "getegid32" }, +#endif +#ifdef __NR_geteuid + { __NR_geteuid, "geteuid" }, +#endif +#ifdef __NR_geteuid32 + { __NR_geteuid32, "geteuid32" }, +#endif +#ifdef __NR_getgid + { __NR_getgid, "getgid" }, +#endif +#ifdef __NR_getgid32 + { __NR_getgid32, "getgid32" }, +#endif +#ifdef __NR_getgroups + { __NR_getgroups, "getgroups" }, +#endif +#ifdef __NR_getgroups32 + { __NR_getgroups32, "getgroups32" }, +#endif +#ifdef __NR_getitimer + { __NR_getitimer, "getitimer" }, +#endif +#ifdef __NR_getpeername + { __NR_getpeername, "getpeername" }, +#endif +#ifdef __NR_getpgid + { __NR_getpgid, "getpgid" }, +#endif +#ifdef __NR_getpgrp + { __NR_getpgrp, "getpgrp" }, +#endif +#ifdef __NR_getpid + { __NR_getpid, "getpid" }, +#endif +#ifdef __NR_getpmsg + { __NR_getpmsg, "getpmsg" }, +#endif +#ifdef __NR_getppid + { __NR_getppid, "getppid" }, +#endif +#ifdef __NR_getpriority + { __NR_getpriority, "getpriority" }, +#endif +#ifdef __NR_getresgid + { __NR_getresgid, "getresgid" }, +#endif +#ifdef __NR_getresgid32 + { __NR_getresgid32, "getresgid32" }, +#endif +#ifdef __NR_getresuid + { __NR_getresuid, "getresuid" }, +#endif +#ifdef __NR_getresuid32 + { __NR_getresuid32, "getresuid32" }, +#endif +#ifdef __NR_getrlimit + { __NR_getrlimit, "getrlimit" }, +#endif +#ifdef __NR_getrusage + { __NR_getrusage, "getrusage" }, +#endif +#ifdef __NR_getsid + { __NR_getsid, "getsid" }, +#endif +#ifdef __NR_getsockname + { __NR_getsockname, "getsockname" }, +#endif +#ifdef __NR_getsockopt + { __NR_getsockopt, "getsockopt" }, +#endif +#ifdef __NR_gettid + { __NR_gettid, "gettid" }, +#endif +#ifdef __NR_gettimeofday + { __NR_gettimeofday, "gettimeofday" }, +#endif +#ifdef __NR_getuid + { __NR_getuid, "getuid" }, +#endif +#ifdef __NR_getuid32 + { __NR_getuid32, "getuid32" }, +#endif +#ifdef __NR_getxattr + { __NR_getxattr, "getxattr" }, +#endif +#ifdef __NR_gtty + { __NR_gtty, "gtty" }, +#endif +#ifdef __NR_idle + { __NR_idle, "idle" }, +#endif +#ifdef __NR_init_module + { __NR_init_module, "init_module" }, +#endif +#ifdef __NR_inotify_add_watch + { __NR_inotify_add_watch, "inotify_add_watch" }, +#endif +#ifdef __NR_inotify_init + { __NR_inotify_init, "inotify_init" }, +#endif +#ifdef __NR_inotify_init1 + { __NR_inotify_init1, "inotify_init1" }, +#endif +#ifdef __NR_inotify_rm_watch + { __NR_inotify_rm_watch, "inotify_rm_watch" }, +#endif +#ifdef __NR_io_cancel + { __NR_io_cancel, "io_cancel" }, +#endif +#ifdef __NR_io_destroy + { __NR_io_destroy, "io_destroy" }, +#endif +#ifdef __NR_io_getevents + { __NR_io_getevents, "io_getevents" }, +#endif +#ifdef __NR_io_setup + { __NR_io_setup, "io_setup" }, +#endif +#ifdef __NR_io_submit + { __NR_io_submit, "io_submit" }, +#endif +#ifdef __NR_ioctl + { __NR_ioctl, "ioctl" }, +#endif +#ifdef __NR_ioperm + { __NR_ioperm, "ioperm" }, +#endif +#ifdef __NR_iopl + { __NR_iopl, "iopl" }, +#endif +#ifdef __NR_ioprio_get + { __NR_ioprio_get, "ioprio_get" }, +#endif +#ifdef __NR_ioprio_set + { __NR_ioprio_set, "ioprio_set" }, +#endif +#ifdef __NR_ipc + { __NR_ipc, "ipc" }, +#endif +#ifdef __NR_kcmp + { __NR_kcmp, "kcmp" }, +#endif +#ifdef __NR_kexec_load + { __NR_kexec_load, "kexec_load" }, +#endif +#ifdef __NR_keyctl + { __NR_keyctl, "keyctl" }, +#endif +#ifdef __NR_kill + { __NR_kill, "kill" }, +#endif +#ifdef __NR_lchown + { __NR_lchown, "lchown" }, +#endif +#ifdef __NR_lchown32 + { __NR_lchown32, "lchown32" }, +#endif +#ifdef __NR_lgetxattr + { __NR_lgetxattr, "lgetxattr" }, +#endif +#ifdef __NR_link + { __NR_link, "link" }, +#endif +#ifdef __NR_linkat + { __NR_linkat, "linkat" }, +#endif +#ifdef __NR_listen + { __NR_listen, "listen" }, +#endif +#ifdef __NR_listxattr + { __NR_listxattr, "listxattr" }, +#endif +#ifdef __NR_llistxattr + { __NR_llistxattr, "llistxattr" }, +#endif +#ifdef __NR_lock + { __NR_lock, "lock" }, +#endif +#ifdef __NR_lookup_dcookie + { __NR_lookup_dcookie, "lookup_dcookie" }, +#endif +#ifdef __NR_lremovexattr + { __NR_lremovexattr, "lremovexattr" }, +#endif +#ifdef __NR_lseek + { __NR_lseek, "lseek" }, +#endif +#ifdef __NR_lsetxattr + { __NR_lsetxattr, "lsetxattr" }, +#endif +#ifdef __NR_lstat + { __NR_lstat, "lstat" }, +#endif +#ifdef __NR_lstat64 + { __NR_lstat64, "lstat64" }, +#endif +#ifdef __NR_madvise + { __NR_madvise, "madvise" }, +#endif +#ifdef __NR_mbind + { __NR_mbind, "mbind" }, +#endif +#ifdef __NR_migrate_pages + { __NR_migrate_pages, "migrate_pages" }, +#endif +#ifdef __NR_mincore + { __NR_mincore, "mincore" }, +#endif +#ifdef __NR_mkdir + { __NR_mkdir, "mkdir" }, +#endif +#ifdef __NR_mkdirat + { __NR_mkdirat, "mkdirat" }, +#endif +#ifdef __NR_mknod + { __NR_mknod, "mknod" }, +#endif +#ifdef __NR_mknodat + { __NR_mknodat, "mknodat" }, +#endif +#ifdef __NR_mlock + { __NR_mlock, "mlock" }, +#endif +#ifdef __NR_mlockall + { __NR_mlockall, "mlockall" }, +#endif +#ifdef __NR_mmap + { __NR_mmap, "mmap" }, +#endif +#ifdef __NR_mmap2 + { __NR_mmap2, "mmap2" }, +#endif +#ifdef __NR_modify_ldt + { __NR_modify_ldt, "modify_ldt" }, +#endif +#ifdef __NR_mount + { __NR_mount, "mount" }, +#endif +#ifdef __NR_move_pages + { __NR_move_pages, "move_pages" }, +#endif +#ifdef __NR_mprotect + { __NR_mprotect, "mprotect" }, +#endif +#ifdef __NR_mpx + { __NR_mpx, "mpx" }, +#endif +#ifdef __NR_mq_getsetattr + { __NR_mq_getsetattr, "mq_getsetattr" }, +#endif +#ifdef __NR_mq_notify + { __NR_mq_notify, "mq_notify" }, +#endif +#ifdef __NR_mq_open + { __NR_mq_open, "mq_open" }, +#endif +#ifdef __NR_mq_timedreceive + { __NR_mq_timedreceive, "mq_timedreceive" }, +#endif +#ifdef __NR_mq_timedsend + { __NR_mq_timedsend, "mq_timedsend" }, +#endif +#ifdef __NR_mq_unlink + { __NR_mq_unlink, "mq_unlink" }, +#endif +#ifdef __NR_mremap + { __NR_mremap, "mremap" }, +#endif +#ifdef __NR_msgctl + { __NR_msgctl, "msgctl" }, +#endif +#ifdef __NR_msgget + { __NR_msgget, "msgget" }, +#endif +#ifdef __NR_msgrcv + { __NR_msgrcv, "msgrcv" }, +#endif +#ifdef __NR_msgsnd + { __NR_msgsnd, "msgsnd" }, +#endif +#ifdef __NR_msync + { __NR_msync, "msync" }, +#endif +#ifdef __NR_munlock + { __NR_munlock, "munlock" }, +#endif +#ifdef __NR_munlockall + { __NR_munlockall, "munlockall" }, +#endif +#ifdef __NR_munmap + { __NR_munmap, "munmap" }, +#endif +#ifdef __NR_name_to_handle_at + { __NR_name_to_handle_at, "name_to_handle_at" }, +#endif +#ifdef __NR_nanosleep + { __NR_nanosleep, "nanosleep" }, +#endif +#ifdef __NR_newfstatat + { __NR_newfstatat, "newfstatat" }, +#endif +#ifdef __NR_nfsservctl + { __NR_nfsservctl, "nfsservctl" }, +#endif +#ifdef __NR_nice + { __NR_nice, "nice" }, +#endif +#ifdef __NR_oldfstat + { __NR_oldfstat, "oldfstat" }, +#endif +#ifdef __NR_oldlstat + { __NR_oldlstat, "oldlstat" }, +#endif +#ifdef __NR_oldolduname + { __NR_oldolduname, "oldolduname" }, +#endif +#ifdef __NR_oldstat + { __NR_oldstat, "oldstat" }, +#endif +#ifdef __NR_olduname + { __NR_olduname, "olduname" }, +#endif +#ifdef __NR_open + { __NR_open, "open" }, +#endif +#ifdef __NR_open_by_handle_at + { __NR_open_by_handle_at, "open_by_handle_at" }, +#endif +#ifdef __NR_openat + { __NR_openat, "openat" }, +#endif +#ifdef __NR_pause + { __NR_pause, "pause" }, +#endif +#ifdef __NR_perf_event_open + { __NR_perf_event_open, "perf_event_open" }, +#endif +#ifdef __NR_personality + { __NR_personality, "personality" }, +#endif +#ifdef __NR_pipe + { __NR_pipe, "pipe" }, +#endif +#ifdef __NR_pipe2 + { __NR_pipe2, "pipe2" }, +#endif +#ifdef __NR_pivot_root + { __NR_pivot_root, "pivot_root" }, +#endif +#ifdef __NR_poll + { __NR_poll, "poll" }, +#endif +#ifdef __NR_ppoll + { __NR_ppoll, "ppoll" }, +#endif +#ifdef __NR_prctl + { __NR_prctl, "prctl" }, +#endif +#ifdef __NR_pread64 + { __NR_pread64, "pread64" }, +#endif +#ifdef __NR_preadv + { __NR_preadv, "preadv" }, +#endif +#ifdef __NR_prlimit64 + { __NR_prlimit64, "prlimit64" }, +#endif +#ifdef __NR_process_vm_readv + { __NR_process_vm_readv, "process_vm_readv" }, +#endif +#ifdef __NR_process_vm_writev + { __NR_process_vm_writev, "process_vm_writev" }, +#endif +#ifdef __NR_prof + { __NR_prof, "prof" }, +#endif +#ifdef __NR_profil + { __NR_profil, "profil" }, +#endif +#ifdef __NR_pselect6 + { __NR_pselect6, "pselect6" }, +#endif +#ifdef __NR_ptrace + { __NR_ptrace, "ptrace" }, +#endif +#ifdef __NR_putpmsg + { __NR_putpmsg, "putpmsg" }, +#endif +#ifdef __NR_pwrite64 + { __NR_pwrite64, "pwrite64" }, +#endif +#ifdef __NR_pwritev + { __NR_pwritev, "pwritev" }, +#endif +#ifdef __NR_query_module + { __NR_query_module, "query_module" }, +#endif +#ifdef __NR_quotactl + { __NR_quotactl, "quotactl" }, +#endif +#ifdef __NR_read + { __NR_read, "read" }, +#endif +#ifdef __NR_readahead + { __NR_readahead, "readahead" }, +#endif +#ifdef __NR_readdir + { __NR_readdir, "readdir" }, +#endif +#ifdef __NR_readlink + { __NR_readlink, "readlink" }, +#endif +#ifdef __NR_readlinkat + { __NR_readlinkat, "readlinkat" }, +#endif +#ifdef __NR_readv + { __NR_readv, "readv" }, +#endif +#ifdef __NR_reboot + { __NR_reboot, "reboot" }, +#endif +#ifdef __NR_recvfrom + { __NR_recvfrom, "recvfrom" }, +#endif +#ifdef __NR_recvmmsg + { __NR_recvmmsg, "recvmmsg" }, +#endif +#ifdef __NR_recvmsg + { __NR_recvmsg, "recvmsg" }, +#endif +#ifdef __NR_remap_file_pages + { __NR_remap_file_pages, "remap_file_pages" }, +#endif +#ifdef __NR_removexattr + { __NR_removexattr, "removexattr" }, +#endif +#ifdef __NR_rename + { __NR_rename, "rename" }, +#endif +#ifdef __NR_renameat + { __NR_renameat, "renameat" }, +#endif +#ifdef __NR_request_key + { __NR_request_key, "request_key" }, +#endif +#ifdef __NR_restart_syscall + { __NR_restart_syscall, "restart_syscall" }, +#endif +#ifdef __NR_rmdir + { __NR_rmdir, "rmdir" }, +#endif +#ifdef __NR_rt_sigaction + { __NR_rt_sigaction, "rt_sigaction" }, +#endif +#ifdef __NR_rt_sigpending + { __NR_rt_sigpending, "rt_sigpending" }, +#endif +#ifdef __NR_rt_sigprocmask + { __NR_rt_sigprocmask, "rt_sigprocmask" }, +#endif +#ifdef __NR_rt_sigqueueinfo + { __NR_rt_sigqueueinfo, "rt_sigqueueinfo" }, +#endif +#ifdef __NR_rt_sigreturn + { __NR_rt_sigreturn, "rt_sigreturn" }, +#endif +#ifdef __NR_rt_sigsuspend + { __NR_rt_sigsuspend, "rt_sigsuspend" }, +#endif +#ifdef __NR_rt_sigtimedwait + { __NR_rt_sigtimedwait, "rt_sigtimedwait" }, +#endif +#ifdef __NR_rt_tgsigqueueinfo + { __NR_rt_tgsigqueueinfo, "rt_tgsigqueueinfo" }, +#endif +#ifdef __NR_sched_get_priority_max + { __NR_sched_get_priority_max, "sched_get_priority_max" }, +#endif +#ifdef __NR_sched_get_priority_min + { __NR_sched_get_priority_min, "sched_get_priority_min" }, +#endif +#ifdef __NR_sched_getaffinity + { __NR_sched_getaffinity, "sched_getaffinity" }, +#endif +#ifdef __NR_sched_getparam + { __NR_sched_getparam, "sched_getparam" }, +#endif +#ifdef __NR_sched_getscheduler + { __NR_sched_getscheduler, "sched_getscheduler" }, +#endif +#ifdef __NR_sched_rr_get_interval + { __NR_sched_rr_get_interval, "sched_rr_get_interval" }, +#endif +#ifdef __NR_sched_setaffinity + { __NR_sched_setaffinity, "sched_setaffinity" }, +#endif +#ifdef __NR_sched_setparam + { __NR_sched_setparam, "sched_setparam" }, +#endif +#ifdef __NR_sched_setscheduler + { __NR_sched_setscheduler, "sched_setscheduler" }, +#endif +#ifdef __NR_sched_yield + { __NR_sched_yield, "sched_yield" }, +#endif +#ifdef __NR_security + { __NR_security, "security" }, +#endif +#ifdef __NR_select + { __NR_select, "select" }, +#endif +#ifdef __NR_semctl + { __NR_semctl, "semctl" }, +#endif +#ifdef __NR_semget + { __NR_semget, "semget" }, +#endif +#ifdef __NR_semop + { __NR_semop, "semop" }, +#endif +#ifdef __NR_semtimedop + { __NR_semtimedop, "semtimedop" }, +#endif +#ifdef __NR_sendfile + { __NR_sendfile, "sendfile" }, +#endif +#ifdef __NR_sendfile64 + { __NR_sendfile64, "sendfile64" }, +#endif +#ifdef __NR_sendmmsg + { __NR_sendmmsg, "sendmmsg" }, +#endif +#ifdef __NR_sendmsg + { __NR_sendmsg, "sendmsg" }, +#endif +#ifdef __NR_sendto + { __NR_sendto, "sendto" }, +#endif +#ifdef __NR_set_mempolicy + { __NR_set_mempolicy, "set_mempolicy" }, +#endif +#ifdef __NR_set_robust_list + { __NR_set_robust_list, "set_robust_list" }, +#endif +#ifdef __NR_set_thread_area + { __NR_set_thread_area, "set_thread_area" }, +#endif +#ifdef __NR_set_tid_address + { __NR_set_tid_address, "set_tid_address" }, +#endif +#ifdef __NR_setdomainname + { __NR_setdomainname, "setdomainname" }, +#endif +#ifdef __NR_setfsgid + { __NR_setfsgid, "setfsgid" }, +#endif +#ifdef __NR_setfsgid32 + { __NR_setfsgid32, "setfsgid32" }, +#endif +#ifdef __NR_setfsuid + { __NR_setfsuid, "setfsuid" }, +#endif +#ifdef __NR_setfsuid32 + { __NR_setfsuid32, "setfsuid32" }, +#endif +#ifdef __NR_setgid + { __NR_setgid, "setgid" }, +#endif +#ifdef __NR_setgid32 + { __NR_setgid32, "setgid32" }, +#endif +#ifdef __NR_setgroups + { __NR_setgroups, "setgroups" }, +#endif +#ifdef __NR_setgroups32 + { __NR_setgroups32, "setgroups32" }, +#endif +#ifdef __NR_sethostname + { __NR_sethostname, "sethostname" }, +#endif +#ifdef __NR_setitimer + { __NR_setitimer, "setitimer" }, +#endif +#ifdef __NR_setns + { __NR_setns, "setns" }, +#endif +#ifdef __NR_setpgid + { __NR_setpgid, "setpgid" }, +#endif +#ifdef __NR_setpriority + { __NR_setpriority, "setpriority" }, +#endif +#ifdef __NR_setregid + { __NR_setregid, "setregid" }, +#endif +#ifdef __NR_setregid32 + { __NR_setregid32, "setregid32" }, +#endif +#ifdef __NR_setresgid + { __NR_setresgid, "setresgid" }, +#endif +#ifdef __NR_setresgid32 + { __NR_setresgid32, "setresgid32" }, +#endif +#ifdef __NR_setresuid + { __NR_setresuid, "setresuid" }, +#endif +#ifdef __NR_setresuid32 + { __NR_setresuid32, "setresuid32" }, +#endif +#ifdef __NR_setreuid + { __NR_setreuid, "setreuid" }, +#endif +#ifdef __NR_setreuid32 + { __NR_setreuid32, "setreuid32" }, +#endif +#ifdef __NR_setrlimit + { __NR_setrlimit, "setrlimit" }, +#endif +#ifdef __NR_setsid + { __NR_setsid, "setsid" }, +#endif +#ifdef __NR_setsockopt + { __NR_setsockopt, "setsockopt" }, +#endif +#ifdef __NR_settimeofday + { __NR_settimeofday, "settimeofday" }, +#endif +#ifdef __NR_setuid + { __NR_setuid, "setuid" }, +#endif +#ifdef __NR_setuid32 + { __NR_setuid32, "setuid32" }, +#endif +#ifdef __NR_setxattr + { __NR_setxattr, "setxattr" }, +#endif +#ifdef __NR_sgetmask + { __NR_sgetmask, "sgetmask" }, +#endif +#ifdef __NR_shmat + { __NR_shmat, "shmat" }, +#endif +#ifdef __NR_shmctl + { __NR_shmctl, "shmctl" }, +#endif +#ifdef __NR_shmdt + { __NR_shmdt, "shmdt" }, +#endif +#ifdef __NR_shmget + { __NR_shmget, "shmget" }, +#endif +#ifdef __NR_shutdown + { __NR_shutdown, "shutdown" }, +#endif +#ifdef __NR_sigaction + { __NR_sigaction, "sigaction" }, +#endif +#ifdef __NR_sigaltstack + { __NR_sigaltstack, "sigaltstack" }, +#endif +#ifdef __NR_signal + { __NR_signal, "signal" }, +#endif +#ifdef __NR_signalfd + { __NR_signalfd, "signalfd" }, +#endif +#ifdef __NR_signalfd4 + { __NR_signalfd4, "signalfd4" }, +#endif +#ifdef __NR_sigpending + { __NR_sigpending, "sigpending" }, +#endif +#ifdef __NR_sigprocmask + { __NR_sigprocmask, "sigprocmask" }, +#endif +#ifdef __NR_sigreturn + { __NR_sigreturn, "sigreturn" }, +#endif +#ifdef __NR_sigsuspend + { __NR_sigsuspend, "sigsuspend" }, +#endif +#ifdef __NR_socket + { __NR_socket, "socket" }, +#endif +#ifdef __NR_socketcall + { __NR_socketcall, "socketcall" }, +#endif +#ifdef __NR_socketpair + { __NR_socketpair, "socketpair" }, +#endif +#ifdef __NR_splice + { __NR_splice, "splice" }, +#endif +#ifdef __NR_ssetmask + { __NR_ssetmask, "ssetmask" }, +#endif +#ifdef __NR_stat + { __NR_stat, "stat" }, +#endif +#ifdef __NR_stat64 + { __NR_stat64, "stat64" }, +#endif +#ifdef __NR_statfs + { __NR_statfs, "statfs" }, +#endif +#ifdef __NR_statfs64 + { __NR_statfs64, "statfs64" }, +#endif +#ifdef __NR_stime + { __NR_stime, "stime" }, +#endif +#ifdef __NR_stty + { __NR_stty, "stty" }, +#endif +#ifdef __NR_swapoff + { __NR_swapoff, "swapoff" }, +#endif +#ifdef __NR_swapon + { __NR_swapon, "swapon" }, +#endif +#ifdef __NR_symlink + { __NR_symlink, "symlink" }, +#endif +#ifdef __NR_symlinkat + { __NR_symlinkat, "symlinkat" }, +#endif +#ifdef __NR_sync + { __NR_sync, "sync" }, +#endif +#ifdef __NR_sync_file_range + { __NR_sync_file_range, "sync_file_range" }, +#endif +#ifdef __NR_syncfs + { __NR_syncfs, "syncfs" }, +#endif +#ifdef __NR_sysfs + { __NR_sysfs, "sysfs" }, +#endif +#ifdef __NR_sysinfo + { __NR_sysinfo, "sysinfo" }, +#endif +#ifdef __NR_syslog + { __NR_syslog, "syslog" }, +#endif +#ifdef __NR_tee + { __NR_tee, "tee" }, +#endif +#ifdef __NR_tgkill + { __NR_tgkill, "tgkill" }, +#endif +#ifdef __NR_time + { __NR_time, "time" }, +#endif +#ifdef __NR_timer_create + { __NR_timer_create, "timer_create" }, +#endif +#ifdef __NR_timer_delete + { __NR_timer_delete, "timer_delete" }, +#endif +#ifdef __NR_timer_getoverrun + { __NR_timer_getoverrun, "timer_getoverrun" }, +#endif +#ifdef __NR_timer_gettime + { __NR_timer_gettime, "timer_gettime" }, +#endif +#ifdef __NR_timer_settime + { __NR_timer_settime, "timer_settime" }, +#endif +#ifdef __NR_timerfd_create + { __NR_timerfd_create, "timerfd_create" }, +#endif +#ifdef __NR_timerfd_gettime + { __NR_timerfd_gettime, "timerfd_gettime" }, +#endif +#ifdef __NR_timerfd_settime + { __NR_timerfd_settime, "timerfd_settime" }, +#endif +#ifdef __NR_times + { __NR_times, "times" }, +#endif +#ifdef __NR_tkill + { __NR_tkill, "tkill" }, +#endif +#ifdef __NR_truncate + { __NR_truncate, "truncate" }, +#endif +#ifdef __NR_truncate64 + { __NR_truncate64, "truncate64" }, +#endif +#ifdef __NR_tuxcall + { __NR_tuxcall, "tuxcall" }, +#endif +#ifdef __NR_ugetrlimit + { __NR_ugetrlimit, "ugetrlimit" }, +#endif +#ifdef __NR_ulimit + { __NR_ulimit, "ulimit" }, +#endif +#ifdef __NR_umask + { __NR_umask, "umask" }, +#endif +#ifdef __NR_umount + { __NR_umount, "umount" }, +#endif +#ifdef __NR_umount2 + { __NR_umount2, "umount2" }, +#endif +#ifdef __NR_uname + { __NR_uname, "uname" }, +#endif +#ifdef __NR_unlink + { __NR_unlink, "unlink" }, +#endif +#ifdef __NR_unlinkat + { __NR_unlinkat, "unlinkat" }, +#endif +#ifdef __NR_unshare + { __NR_unshare, "unshare" }, +#endif +#ifdef __NR_uselib + { __NR_uselib, "uselib" }, +#endif +#ifdef __NR_ustat + { __NR_ustat, "ustat" }, +#endif +#ifdef __NR_utime + { __NR_utime, "utime" }, +#endif +#ifdef __NR_utimensat + { __NR_utimensat, "utimensat" }, +#endif +#ifdef __NR_utimes + { __NR_utimes, "utimes" }, +#endif +#ifdef __NR_vfork + { __NR_vfork, "vfork" }, +#endif +#ifdef __NR_vhangup + { __NR_vhangup, "vhangup" }, +#endif +#ifdef __NR_vm86 + { __NR_vm86, "vm86" }, +#endif +#ifdef __NR_vm86old + { __NR_vm86old, "vm86old" }, +#endif +#ifdef __NR_vmsplice + { __NR_vmsplice, "vmsplice" }, +#endif +#ifdef __NR_vserver + { __NR_vserver, "vserver" }, +#endif +#ifdef __NR_wait4 + { __NR_wait4, "wait4" }, +#endif +#ifdef __NR_waitid + { __NR_waitid, "waitid" }, +#endif +#ifdef __NR_waitpid + { __NR_waitpid, "waitpid" }, +#endif +#ifdef __NR_write + { __NR_write, "write" }, +#endif +#ifdef __NR_writev + { __NR_writev, "writev" }, +#endif + {0, NULL} +}; + diff --git a/src/common/log.c b/src/common/log.c index e196a11287..517fa4faaa 100644 --- a/src/common/log.c +++ b/src/common/log.c @@ -36,6 +36,10 @@ #include "torlog.h" #include "container.h" +/** Given a severity, yields an index into log_severity_list_t.masks to use + * for that severity. */ +#define SEVERITY_MASK_IDX(sev) ((sev) - LOG_ERR) + /** @{ */ /** The string we stick at the end of a log message when it is too long, * and its length. */ @@ -83,12 +87,12 @@ should_log_function_name(log_domain_mask_t domain, int severity) case LOG_DEBUG: case LOG_INFO: /* All debugging messages occur in interesting places. */ - return 1; + return (domain & LD_NOFUNCNAME) == 0; case LOG_NOTICE: case LOG_WARN: case LOG_ERR: /* We care about places where bugs occur. */ - return (domain == LD_BUG); + return (domain & (LD_BUG|LD_NOFUNCNAME)) == LD_BUG; default: /* Call assert, not tor_assert, since tor_assert calls log on failure. */ assert(0); return 0; @@ -143,9 +147,6 @@ static INLINE char *format_msg(char *buf, size_t buf_len, const char *suffix, const char *format, va_list ap, size_t *msg_len_out) CHECK_PRINTF(7,0); -static void logv(int severity, log_domain_mask_t domain, const char *funcname, - const char *suffix, const char *format, va_list ap) - CHECK_PRINTF(5,0); /** Name of the application: used to generate the message we write at the * start of each new log. */ @@ -332,9 +333,9 @@ format_msg(char *buf, size_t buf_len, * <b>severity</b>. If provided, <b>funcname</b> is prepended to the * message. The actual message is derived as from tor_snprintf(format,ap). */ -static void -logv(int severity, log_domain_mask_t domain, const char *funcname, - const char *suffix, const char *format, va_list ap) +MOCK_IMPL(STATIC void, +logv,(int severity, log_domain_mask_t domain, const char *funcname, + const char *suffix, const char *format, va_list ap)) { char buf[10024]; size_t msg_len = 0; @@ -439,6 +440,149 @@ tor_log(int severity, log_domain_mask_t domain, const char *format, ...) va_end(ap); } +/** Maximum number of fds that will get notifications if we crash */ +#define MAX_SIGSAFE_FDS 8 +/** Array of fds to log crash-style warnings to. */ +static int sigsafe_log_fds[MAX_SIGSAFE_FDS] = { STDERR_FILENO }; +/** The number of elements used in sigsafe_log_fds */ +static int n_sigsafe_log_fds = 1; + +/** Write <b>s</b> to each element of sigsafe_log_fds. Return 0 on success, -1 + * on failure. */ +static int +tor_log_err_sigsafe_write(const char *s) +{ + int i; + ssize_t r; + size_t len = strlen(s); + int err = 0; + for (i=0; i < n_sigsafe_log_fds; ++i) { + r = write(sigsafe_log_fds[i], s, len); + err += (r != (ssize_t)len); + } + return err ? -1 : 0; +} + +/** Given a list of string arguments ending with a NULL, writes them + * to our logs and to stderr (if possible). This function is safe to call + * from within a signal handler. */ +void +tor_log_err_sigsafe(const char *m, ...) +{ + va_list ap; + const char *x; + char timebuf[33]; + time_t now = time(NULL); + + if (!m) + return; + if (log_time_granularity >= 2000) { + int g = log_time_granularity / 1000; + now -= now % g; + } + timebuf[0] = now < 0 ? '-' : ' '; + if (now < 0) now = -now; + timebuf[1] = '\0'; + format_dec_number_sigsafe(now, timebuf+1, sizeof(timebuf)-1); + tor_log_err_sigsafe_write("\n==========================================" + "================== T="); + tor_log_err_sigsafe_write(timebuf); + tor_log_err_sigsafe_write("\n"); + tor_log_err_sigsafe_write(m); + va_start(ap, m); + while ((x = va_arg(ap, const char*))) { + tor_log_err_sigsafe_write(x); + } + va_end(ap); +} + +/** Set *<b>out</b> to a pointer to an array of the fds to log errors to from + * inside a signal handler. Return the number of elements in the array. */ +int +tor_log_get_sigsafe_err_fds(const int **out) +{ + *out = sigsafe_log_fds; + return n_sigsafe_log_fds; +} + +/** Helper function; return true iff the <b>n</b>-element array <b>array</b> + * contains <b>item</b>. */ +static int +int_array_contains(const int *array, int n, int item) +{ + int j; + for (j = 0; j < n; ++j) { + if (array[j] == item) + return 1; + } + return 0; +} + +/** Function to call whenever the list of logs changes to get ready to log + * from signal handlers. */ +void +tor_log_update_sigsafe_err_fds(void) +{ + const logfile_t *lf; + int found_real_stderr = 0; + + LOCK_LOGS(); + /* Reserve the first one for stderr. This is safe because when we daemonize, + * we dup2 /dev/null to stderr, */ + sigsafe_log_fds[0] = STDERR_FILENO; + n_sigsafe_log_fds = 1; + + for (lf = logfiles; lf; lf = lf->next) { + /* Don't try callback to the control port, or syslogs: We can't + * do them from a signal handler. Don't try stdout: we always do stderr. + */ + if (lf->is_temporary || lf->is_syslog || + lf->callback || lf->seems_dead || lf->fd < 0) + continue; + if (lf->severities->masks[SEVERITY_MASK_IDX(LOG_ERR)] & + (LD_BUG|LD_GENERAL)) { + if (lf->fd == STDERR_FILENO) + found_real_stderr = 1; + /* Avoid duplicates */ + if (int_array_contains(sigsafe_log_fds, n_sigsafe_log_fds, lf->fd)) + continue; + sigsafe_log_fds[n_sigsafe_log_fds++] = lf->fd; + if (n_sigsafe_log_fds == MAX_SIGSAFE_FDS) + break; + } + } + + if (!found_real_stderr && + int_array_contains(sigsafe_log_fds, n_sigsafe_log_fds, STDOUT_FILENO)) { + /* Don't use a virtual stderr when we're also logging to stdout. */ + assert(n_sigsafe_log_fds >= 2); /* Don't use assert inside log functions*/ + sigsafe_log_fds[0] = sigsafe_log_fds[--n_sigsafe_log_fds]; + } + + UNLOCK_LOGS(); +} + +/** Add to <b>out</b> a copy of every currently configured log file name. Used + * to enable access to these filenames with the sandbox code. */ +void +tor_log_get_logfile_names(smartlist_t *out) +{ + logfile_t *lf; + tor_assert(out); + + LOCK_LOGS(); + + for (lf = logfiles; lf; lf = lf->next) { + if (lf->is_temporary || lf->is_syslog || lf->callback) + continue; + if (lf->filename == NULL) + continue; + smartlist_add(out, tor_strdup(lf->filename)); + } + + UNLOCK_LOGS(); +} + /** Output a message to the log, prefixed with a function name <b>fn</b>. */ #ifdef __GNUC__ /** GCC-based implementation of the log_fn backend, used when we have @@ -1153,38 +1297,3 @@ switch_logs_debug(void) UNLOCK_LOGS(); } -#if 0 -static void -dump_log_info(logfile_t *lf) -{ - const char *tp; - - if (lf->filename) { - printf("=== log into \"%s\" (%s-%s) (%stemporary)\n", lf->filename, - sev_to_string(lf->min_loglevel), - sev_to_string(lf->max_loglevel), - lf->is_temporary?"":"not "); - } else if (lf->is_syslog) { - printf("=== syslog (%s-%s) (%stemporary)\n", - sev_to_string(lf->min_loglevel), - sev_to_string(lf->max_loglevel), - lf->is_temporary?"":"not "); - } else { - printf("=== log (%s-%s) (%stemporary)\n", - sev_to_string(lf->min_loglevel), - sev_to_string(lf->max_loglevel), - lf->is_temporary?"":"not "); - } -} - -void -describe_logs(void) -{ - logfile_t *lf; - printf("==== BEGIN LOGS ====\n"); - for (lf = logfiles; lf; lf = lf->next) - dump_log_info(lf); - printf("==== END LOGS ====\n"); -} -#endif - diff --git a/src/common/memarea.c b/src/common/memarea.c index 0ae0ccca1d..bcaea0949e 100644 --- a/src/common/memarea.c +++ b/src/common/memarea.c @@ -29,6 +29,13 @@ #error "void* is neither 4 nor 8 bytes long. I don't know how to align stuff." #endif +#if defined(__GNUC__) && defined(FLEXIBLE_ARRAY_MEMBER) +#define USE_ALIGNED_ATTRIBUTE +#define U_MEM mem +#else +#define U_MEM u.mem +#endif + #ifdef USE_SENTINELS /** Magic value that we stick at the end of a memarea so we can make sure * there are no run-off-the-end bugs. */ @@ -39,12 +46,12 @@ * end, set those bytes. */ #define SET_SENTINEL(chunk) \ STMT_BEGIN \ - set_uint32( &(chunk)->u.mem[chunk->mem_size], SENTINEL_VAL ); \ + set_uint32( &(chunk)->U_MEM[chunk->mem_size], SENTINEL_VAL ); \ STMT_END /** Assert that the sentinel on a memarea is set correctly. */ #define CHECK_SENTINEL(chunk) \ STMT_BEGIN \ - uint32_t sent_val = get_uint32(&(chunk)->u.mem[chunk->mem_size]); \ + uint32_t sent_val = get_uint32(&(chunk)->U_MEM[chunk->mem_size]); \ tor_assert(sent_val == SENTINEL_VAL); \ STMT_END #else @@ -71,19 +78,23 @@ realign_pointer(void *ptr) typedef struct memarea_chunk_t { /** Next chunk in this area. Only kept around so we can free it. */ struct memarea_chunk_t *next_chunk; - size_t mem_size; /**< How much RAM is available in u.mem, total? */ - char *next_mem; /**< Next position in u.mem to allocate data at. If it's + size_t mem_size; /**< How much RAM is available in mem, total? */ + char *next_mem; /**< Next position in mem to allocate data at. If it's * greater than or equal to mem+mem_size, this chunk is * full. */ +#ifdef USE_ALIGNED_ATTRIBUTE + char mem[FLEXIBLE_ARRAY_MEMBER] __attribute__((aligned(MEMAREA_ALIGN))); +#else union { char mem[1]; /**< Memory space in this chunk. */ void *void_for_alignment_; /**< Dummy; used to make sure mem is aligned. */ } u; +#endif } memarea_chunk_t; /** How many bytes are needed for overhead before we get to the memory part * of a chunk? */ -#define CHUNK_HEADER_SIZE STRUCT_OFFSET(memarea_chunk_t, u) +#define CHUNK_HEADER_SIZE STRUCT_OFFSET(memarea_chunk_t, U_MEM) /** What's the smallest that we'll allocate a chunk? */ #define CHUNK_SIZE 4096 @@ -121,7 +132,7 @@ alloc_chunk(size_t sz, int freelist_ok) res = tor_malloc(chunk_size); res->next_chunk = NULL; res->mem_size = chunk_size - CHUNK_HEADER_SIZE - SENTINEL_LEN; - res->next_mem = res->u.mem; + res->next_mem = res->U_MEM; tor_assert(res->next_mem+res->mem_size+SENTINEL_LEN == ((char*)res)+chunk_size); tor_assert(realign_pointer(res->next_mem) == res->next_mem); @@ -140,7 +151,7 @@ chunk_free_unchecked(memarea_chunk_t *chunk) ++freelist_len; chunk->next_chunk = freelist; freelist = chunk; - chunk->next_mem = chunk->u.mem; + chunk->next_mem = chunk->U_MEM; } else { tor_free(chunk); } @@ -183,7 +194,7 @@ memarea_clear(memarea_t *area) } area->first->next_chunk = NULL; } - area->first->next_mem = area->first->u.mem; + area->first->next_mem = area->first->U_MEM; } /** Remove all unused memarea chunks from the internal freelist. */ @@ -207,7 +218,7 @@ memarea_owns_ptr(const memarea_t *area, const void *p) memarea_chunk_t *chunk; const char *ptr = p; for (chunk = area->first; chunk; chunk = chunk->next_chunk) { - if (ptr >= chunk->u.mem && ptr < chunk->next_mem) + if (ptr >= chunk->U_MEM && ptr < chunk->next_mem) return 1; } return 0; @@ -226,7 +237,7 @@ memarea_alloc(memarea_t *area, size_t sz) tor_assert(sz < SIZE_T_CEILING); if (sz == 0) sz = 1; - if (chunk->next_mem+sz > chunk->u.mem+chunk->mem_size) { + if (chunk->next_mem+sz > chunk->U_MEM+chunk->mem_size) { if (sz+CHUNK_HEADER_SIZE >= CHUNK_SIZE) { /* This allocation is too big. Stick it in a special chunk, and put * that chunk second in the list. */ @@ -244,8 +255,8 @@ memarea_alloc(memarea_t *area, size_t sz) result = chunk->next_mem; chunk->next_mem = chunk->next_mem + sz; /* Reinstate these if bug 930 ever comes back - tor_assert(chunk->next_mem >= chunk->u.mem); - tor_assert(chunk->next_mem <= chunk->u.mem+chunk->mem_size); + tor_assert(chunk->next_mem >= chunk->U_MEM); + tor_assert(chunk->next_mem <= chunk->U_MEM+chunk->mem_size); */ chunk->next_mem = realign_pointer(chunk->next_mem); return result; @@ -280,14 +291,11 @@ memarea_strdup(memarea_t *area, const char *s) char * memarea_strndup(memarea_t *area, const char *s, size_t n) { - size_t ln; + size_t ln = 0; char *result; - const char *cp, *end = s+n; tor_assert(n < SIZE_T_CEILING); - for (cp = s; cp < end && *cp; ++cp) + for (ln = 0; ln < n && s[ln]; ++ln) ; - /* cp now points to s+n, or to the 0 in the string. */ - ln = cp-s; result = memarea_alloc(area, ln+1); memcpy(result, s, ln); result[ln]='\0'; @@ -304,8 +312,8 @@ memarea_get_stats(memarea_t *area, size_t *allocated_out, size_t *used_out) for (chunk = area->first; chunk; chunk = chunk->next_chunk) { CHECK_SENTINEL(chunk); a += CHUNK_HEADER_SIZE + chunk->mem_size; - tor_assert(chunk->next_mem >= chunk->u.mem); - u += CHUNK_HEADER_SIZE + (chunk->next_mem - chunk->u.mem); + tor_assert(chunk->next_mem >= chunk->U_MEM); + u += CHUNK_HEADER_SIZE + (chunk->next_mem - chunk->U_MEM); } *allocated_out = a; *used_out = u; @@ -320,9 +328,9 @@ memarea_assert_ok(memarea_t *area) for (chunk = area->first; chunk; chunk = chunk->next_chunk) { CHECK_SENTINEL(chunk); - tor_assert(chunk->next_mem >= chunk->u.mem); + tor_assert(chunk->next_mem >= chunk->U_MEM); tor_assert(chunk->next_mem <= - (char*) realign_pointer(chunk->u.mem+chunk->mem_size)); + (char*) realign_pointer(chunk->U_MEM+chunk->mem_size)); } } diff --git a/src/common/procmon.c b/src/common/procmon.c index 0a49689e3a..7c9b7c3c88 100644 --- a/src/common/procmon.c +++ b/src/common/procmon.c @@ -162,6 +162,7 @@ tor_validate_process_specifier(const char *process_spec, return parse_process_specifier(process_spec, &ppspec, msg); } +/* XXXX we should use periodic_timer_new() for this stuff */ #ifdef HAVE_EVENT2_EVENT_H #define PERIODIC_TIMER_FLAGS EV_PERSIST #else diff --git a/src/common/sandbox.c b/src/common/sandbox.c new file mode 100644 index 0000000000..05b91be7be --- /dev/null +++ b/src/common/sandbox.c @@ -0,0 +1,1838 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2013, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file sandbox.c + * \brief Code to enable sandboxing. + **/ + +#include "orconfig.h" + +#ifndef _LARGEFILE64_SOURCE +/** + * Temporarily required for O_LARGEFILE flag. Needs to be removed + * with the libevent fix. + */ +#define _LARGEFILE64_SOURCE +#endif + +/** Malloc mprotect limit in bytes. */ +#define MALLOC_MP_LIM 1048576 + +#include <stdio.h> +#include <string.h> +#include <stdlib.h> + +#include "sandbox.h" +#include "container.h" +#include "torlog.h" +#include "torint.h" +#include "util.h" +#include "tor_queue.h" + +#include "ht.h" + +#define DEBUGGING_CLOSE + +#if defined(USE_LIBSECCOMP) + +#define _GNU_SOURCE + +#include <sys/mman.h> +#include <sys/syscall.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <sys/epoll.h> +#include <sys/prctl.h> +#include <linux/futex.h> +#include <bits/signum.h> + +#include <stdarg.h> +#include <seccomp.h> +#include <signal.h> +#include <unistd.h> +#include <fcntl.h> +#include <time.h> +#include <poll.h> + +#if defined(HAVE_EXECINFO_H) && defined(HAVE_BACKTRACE) && \ + defined(HAVE_BACKTRACE_SYMBOLS_FD) && defined(HAVE_SIGACTION) +#define USE_BACKTRACE +#define EXPOSE_CLEAN_BACKTRACE +#include "backtrace.h" +#endif + +#ifdef USE_BACKTRACE +#include <execinfo.h> +#endif + +/** + * Linux 32 bit definitions + */ +#if defined(__i386__) + +#define REG_SYSCALL REG_EAX +#define M_SYSCALL gregs[REG_SYSCALL] + +/** + * Linux 64 bit definitions + */ +#elif defined(__x86_64__) + +#define REG_SYSCALL REG_RAX +#define M_SYSCALL gregs[REG_SYSCALL] + +#elif defined(__arm__) + +#define M_SYSCALL arm_r7 + +#endif + +/**Determines if at least one sandbox is active.*/ +static int sandbox_active = 0; +/** Holds the parameter list configuration for the sandbox.*/ +static sandbox_cfg_t *filter_dynamic = NULL; + +#undef SCMP_CMP +#define SCMP_CMP(a,b,c) ((struct scmp_arg_cmp){(a),(b),(c),0}) +#define SCMP_CMP4(a,b,c,d) ((struct scmp_arg_cmp){(a),(b),(c),(d)}) +/* We use a wrapper here because these masked comparisons seem to be pretty + * verbose. Also, it's important to cast to scmp_datum_t before negating the + * mask, since otherwise the negation might get applied to a 32 bit value, and + * the high bits of the value might get masked out improperly. */ +#define SCMP_CMP_MASKED(a,b,c) \ + SCMP_CMP4((a), SCMP_CMP_MASKED_EQ, ~(scmp_datum_t)(b), (c)) + +/** Variable used for storing all syscall numbers that will be allowed with the + * stage 1 general Tor sandbox. + */ +static int filter_nopar_gen[] = { + SCMP_SYS(access), + SCMP_SYS(brk), + SCMP_SYS(clock_gettime), + SCMP_SYS(close), + SCMP_SYS(clone), + SCMP_SYS(epoll_create), + SCMP_SYS(epoll_wait), + SCMP_SYS(fcntl), + SCMP_SYS(fstat), +#ifdef __NR_fstat64 + SCMP_SYS(fstat64), +#endif + SCMP_SYS(getdents64), + SCMP_SYS(getegid), +#ifdef __NR_getegid32 + SCMP_SYS(getegid32), +#endif + SCMP_SYS(geteuid), +#ifdef __NR_geteuid32 + SCMP_SYS(geteuid32), +#endif + SCMP_SYS(getgid), +#ifdef __NR_getgid32 + SCMP_SYS(getgid32), +#endif +#ifdef __NR_getrlimit + SCMP_SYS(getrlimit), +#endif + SCMP_SYS(gettimeofday), + SCMP_SYS(gettid), + SCMP_SYS(getuid), +#ifdef __NR_getuid32 + SCMP_SYS(getuid32), +#endif + SCMP_SYS(lseek), +#ifdef __NR__llseek + SCMP_SYS(_llseek), +#endif + SCMP_SYS(mkdir), + SCMP_SYS(mlockall), +#ifdef __NR_mmap + /* XXXX restrict this in the same ways as mmap2 */ + SCMP_SYS(mmap), +#endif + SCMP_SYS(munmap), + SCMP_SYS(read), + SCMP_SYS(rt_sigreturn), + SCMP_SYS(sched_getaffinity), + SCMP_SYS(set_robust_list), +#ifdef __NR_sigreturn + SCMP_SYS(sigreturn), +#endif + SCMP_SYS(stat), + SCMP_SYS(uname), + SCMP_SYS(write), + SCMP_SYS(writev), + SCMP_SYS(exit_group), + SCMP_SYS(exit), + + SCMP_SYS(madvise), +#ifdef __NR_stat64 + // getaddrinfo uses this.. + SCMP_SYS(stat64), +#endif + + /* + * These socket syscalls are not required on x86_64 and not supported with + * some libseccomp versions (eg: 1.0.1) + */ +#if defined(__i386) + SCMP_SYS(recv), + SCMP_SYS(send), +#endif + + // socket syscalls + SCMP_SYS(bind), + SCMP_SYS(listen), + SCMP_SYS(connect), + SCMP_SYS(getsockname), + SCMP_SYS(recvmsg), + SCMP_SYS(recvfrom), + SCMP_SYS(sendto), + SCMP_SYS(unlink) +}; + +/* These macros help avoid the error where the number of filters we add on a + * single rule don't match the arg_cnt param. */ +#define seccomp_rule_add_0(ctx,act,call) \ + seccomp_rule_add((ctx),(act),(call),0) +#define seccomp_rule_add_1(ctx,act,call,f1) \ + seccomp_rule_add((ctx),(act),(call),1,(f1)) +#define seccomp_rule_add_2(ctx,act,call,f1,f2) \ + seccomp_rule_add((ctx),(act),(call),2,(f1),(f2)) +#define seccomp_rule_add_3(ctx,act,call,f1,f2,f3) \ + seccomp_rule_add((ctx),(act),(call),3,(f1),(f2),(f3)) +#define seccomp_rule_add_4(ctx,act,call,f1,f2,f3,f4) \ + seccomp_rule_add((ctx),(act),(call),4,(f1),(f2),(f3),(f4)) + +/** + * Function responsible for setting up the rt_sigaction syscall for + * the seccomp filter sandbox. + */ +static int +sb_rt_sigaction(scmp_filter_ctx ctx, sandbox_cfg_t *filter) +{ + unsigned i; + int rc; + int param[] = { SIGINT, SIGTERM, SIGPIPE, SIGUSR1, SIGUSR2, SIGHUP, SIGCHLD, +#ifdef SIGXFSZ + SIGXFSZ +#endif + }; + (void) filter; + + for (i = 0; i < ARRAY_LENGTH(param); i++) { + rc = seccomp_rule_add_1(ctx, SCMP_ACT_ALLOW, SCMP_SYS(rt_sigaction), + SCMP_CMP(0, SCMP_CMP_EQ, param[i])); + if (rc) + break; + } + + return rc; +} + +#if 0 +/** + * Function responsible for setting up the execve syscall for + * the seccomp filter sandbox. + */ +static int +sb_execve(scmp_filter_ctx ctx, sandbox_cfg_t *filter) +{ + int rc; + sandbox_cfg_t *elem = NULL; + + // for each dynamic parameter filters + for (elem = filter; elem != NULL; elem = elem->next) { + smp_param_t *param = elem->param; + + if (param != NULL && param->prot == 1 && param->syscall + == SCMP_SYS(execve)) { + rc = seccomp_rule_add_1(ctx, SCMP_ACT_ALLOW, SCMP_SYS(execve), + SCMP_CMP(0, SCMP_CMP_EQ, param->value)); + if (rc != 0) { + log_err(LD_BUG,"(Sandbox) failed to add execve syscall, received " + "libseccomp error %d", rc); + return rc; + } + } + } + + return 0; +} +#endif + +/** + * Function responsible for setting up the time syscall for + * the seccomp filter sandbox. + */ +static int +sb_time(scmp_filter_ctx ctx, sandbox_cfg_t *filter) +{ + (void) filter; +#ifdef __NR_time + return seccomp_rule_add_1(ctx, SCMP_ACT_ALLOW, SCMP_SYS(time), + SCMP_CMP(0, SCMP_CMP_EQ, 0)); +#else + return 0; +#endif +} + +/** + * Function responsible for setting up the accept4 syscall for + * the seccomp filter sandbox. + */ +static int +sb_accept4(scmp_filter_ctx ctx, sandbox_cfg_t *filter) +{ + int rc = 0; + (void)filter; + +#ifdef __i386__ + rc = seccomp_rule_add_1(ctx, SCMP_ACT_ALLOW, SCMP_SYS(socketcall), + SCMP_CMP(0, SCMP_CMP_EQ, 18)); + if (rc) { + return rc; + } +#endif + + rc = seccomp_rule_add_1(ctx, SCMP_ACT_ALLOW, SCMP_SYS(accept4), + SCMP_CMP_MASKED(3, SOCK_CLOEXEC|SOCK_NONBLOCK, 0)); + if (rc) { + return rc; + } + + return 0; +} + +#ifdef __NR_mmap2 +/** + * Function responsible for setting up the mmap2 syscall for + * the seccomp filter sandbox. + */ +static int +sb_mmap2(scmp_filter_ctx ctx, sandbox_cfg_t *filter) +{ + int rc = 0; + (void)filter; + + rc = seccomp_rule_add_2(ctx, SCMP_ACT_ALLOW, SCMP_SYS(mmap2), + SCMP_CMP(2, SCMP_CMP_EQ, PROT_READ), + SCMP_CMP(3, SCMP_CMP_EQ, MAP_PRIVATE)); + if (rc) { + return rc; + } + + rc = seccomp_rule_add_2(ctx, SCMP_ACT_ALLOW, SCMP_SYS(mmap2), + SCMP_CMP(2, SCMP_CMP_EQ, PROT_NONE), + SCMP_CMP(3, SCMP_CMP_EQ, MAP_PRIVATE|MAP_ANONYMOUS|MAP_NORESERVE)); + if (rc) { + return rc; + } + + rc = seccomp_rule_add_2(ctx, SCMP_ACT_ALLOW, SCMP_SYS(mmap2), + SCMP_CMP(2, SCMP_CMP_EQ, PROT_READ|PROT_WRITE), + SCMP_CMP(3, SCMP_CMP_EQ, MAP_PRIVATE|MAP_ANONYMOUS)); + if (rc) { + return rc; + } + + rc = seccomp_rule_add_2(ctx, SCMP_ACT_ALLOW, SCMP_SYS(mmap2), + SCMP_CMP(2, SCMP_CMP_EQ, PROT_READ|PROT_WRITE), + SCMP_CMP(3, SCMP_CMP_EQ,MAP_PRIVATE|MAP_ANONYMOUS|MAP_STACK)); + if (rc) { + return rc; + } + + rc = seccomp_rule_add_2(ctx, SCMP_ACT_ALLOW, SCMP_SYS(mmap2), + SCMP_CMP(2, SCMP_CMP_EQ, PROT_READ|PROT_WRITE), + SCMP_CMP(3, SCMP_CMP_EQ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE)); + if (rc) { + return rc; + } + + rc = seccomp_rule_add_2(ctx, SCMP_ACT_ALLOW, SCMP_SYS(mmap2), + SCMP_CMP(2, SCMP_CMP_EQ, PROT_READ|PROT_WRITE), + SCMP_CMP(3, SCMP_CMP_EQ, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS)); + if (rc) { + return rc; + } + + rc = seccomp_rule_add_2(ctx, SCMP_ACT_ALLOW, SCMP_SYS(mmap2), + SCMP_CMP(2, SCMP_CMP_EQ, PROT_READ|PROT_EXEC), + SCMP_CMP(3, SCMP_CMP_EQ, MAP_PRIVATE|MAP_DENYWRITE)); + if (rc) { + return rc; + } + + return 0; +} +#endif + +/** + * Function responsible for setting up the open syscall for + * the seccomp filter sandbox. + */ +static int +sb_open(scmp_filter_ctx ctx, sandbox_cfg_t *filter) +{ + int rc; + sandbox_cfg_t *elem = NULL; + + // for each dynamic parameter filters + for (elem = filter; elem != NULL; elem = elem->next) { + smp_param_t *param = elem->param; + + if (param != NULL && param->prot == 1 && param->syscall + == SCMP_SYS(open)) { + rc = seccomp_rule_add_1(ctx, SCMP_ACT_ALLOW, SCMP_SYS(open), + SCMP_CMP(0, SCMP_CMP_EQ, param->value)); + if (rc != 0) { + log_err(LD_BUG,"(Sandbox) failed to add open syscall, received " + "libseccomp error %d", rc); + return rc; + } + } + } + + rc = seccomp_rule_add_1(ctx, SCMP_ACT_ERRNO(EACCES), SCMP_SYS(open), + SCMP_CMP_MASKED(1, O_CLOEXEC|O_NONBLOCK|O_NOCTTY, O_RDONLY)); + if (rc != 0) { + log_err(LD_BUG,"(Sandbox) failed to add open syscall, received libseccomp " + "error %d", rc); + return rc; + } + + return 0; +} + +static int +sb__sysctl(scmp_filter_ctx ctx, sandbox_cfg_t *filter) +{ + int rc; + (void) filter; + (void) ctx; + + rc = seccomp_rule_add_0(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(_sysctl)); + if (rc != 0) { + log_err(LD_BUG,"(Sandbox) failed to add _sysctl syscall, " + "received libseccomp error %d", rc); + return rc; + } + + return 0; +} + +/** + * Function responsible for setting up the rename syscall for + * the seccomp filter sandbox. + */ +static int +sb_rename(scmp_filter_ctx ctx, sandbox_cfg_t *filter) +{ + int rc; + sandbox_cfg_t *elem = NULL; + + // for each dynamic parameter filters + for (elem = filter; elem != NULL; elem = elem->next) { + smp_param_t *param = elem->param; + + if (param != NULL && param->prot == 1 && + param->syscall == SCMP_SYS(rename)) { + + rc = seccomp_rule_add_2(ctx, SCMP_ACT_ALLOW, SCMP_SYS(rename), + SCMP_CMP(0, SCMP_CMP_EQ, param->value), + SCMP_CMP(1, SCMP_CMP_EQ, param->value2)); + if (rc != 0) { + log_err(LD_BUG,"(Sandbox) failed to add rename syscall, received " + "libseccomp error %d", rc); + return rc; + } + } + } + + return 0; +} + +/** + * Function responsible for setting up the openat syscall for + * the seccomp filter sandbox. + */ +static int +sb_openat(scmp_filter_ctx ctx, sandbox_cfg_t *filter) +{ + int rc; + sandbox_cfg_t *elem = NULL; + + // for each dynamic parameter filters + for (elem = filter; elem != NULL; elem = elem->next) { + smp_param_t *param = elem->param; + + if (param != NULL && param->prot == 1 && param->syscall + == SCMP_SYS(openat)) { + rc = seccomp_rule_add_3(ctx, SCMP_ACT_ALLOW, SCMP_SYS(openat), + SCMP_CMP(0, SCMP_CMP_EQ, AT_FDCWD), + SCMP_CMP(1, SCMP_CMP_EQ, param->value), + SCMP_CMP(2, SCMP_CMP_EQ, O_RDONLY|O_NONBLOCK|O_LARGEFILE|O_DIRECTORY| + O_CLOEXEC)); + if (rc != 0) { + log_err(LD_BUG,"(Sandbox) failed to add openat syscall, received " + "libseccomp error %d", rc); + return rc; + } + } + } + + return 0; +} + +/** + * Function responsible for setting up the socket syscall for + * the seccomp filter sandbox. + */ +static int +sb_socket(scmp_filter_ctx ctx, sandbox_cfg_t *filter) +{ + int rc = 0; + int i; + (void) filter; + +#ifdef __i386__ + rc = seccomp_rule_add_0(ctx, SCMP_ACT_ALLOW, SCMP_SYS(socket)); + if (rc) + return rc; +#endif + + rc = seccomp_rule_add_2(ctx, SCMP_ACT_ALLOW, SCMP_SYS(socket), + SCMP_CMP(0, SCMP_CMP_EQ, PF_FILE), + SCMP_CMP_MASKED(1, SOCK_CLOEXEC|SOCK_NONBLOCK, SOCK_STREAM)); + if (rc) + return rc; + + for (i = 0; i < 2; ++i) { + const int pf = i ? PF_INET : PF_INET6; + + rc = seccomp_rule_add_3(ctx, SCMP_ACT_ALLOW, SCMP_SYS(socket), + SCMP_CMP(0, SCMP_CMP_EQ, pf), + SCMP_CMP_MASKED(1, SOCK_CLOEXEC|SOCK_NONBLOCK, SOCK_STREAM), + SCMP_CMP(2, SCMP_CMP_EQ, IPPROTO_TCP)); + if (rc) + return rc; + + rc = seccomp_rule_add_3(ctx, SCMP_ACT_ALLOW, SCMP_SYS(socket), + SCMP_CMP(0, SCMP_CMP_EQ, pf), + SCMP_CMP_MASKED(1, SOCK_CLOEXEC|SOCK_NONBLOCK, SOCK_DGRAM), + SCMP_CMP(2, SCMP_CMP_EQ, IPPROTO_IP)); + if (rc) + return rc; + } + + rc = seccomp_rule_add_3(ctx, SCMP_ACT_ALLOW, SCMP_SYS(socket), + SCMP_CMP(0, SCMP_CMP_EQ, PF_NETLINK), + SCMP_CMP(1, SCMP_CMP_EQ, SOCK_RAW), + SCMP_CMP(2, SCMP_CMP_EQ, 0)); + if (rc) + return rc; + + return 0; +} + +/** + * Function responsible for setting up the socketpair syscall for + * the seccomp filter sandbox. + */ +static int +sb_socketpair(scmp_filter_ctx ctx, sandbox_cfg_t *filter) +{ + int rc = 0; + (void) filter; + +#ifdef __i386__ + rc = seccomp_rule_add_0(ctx, SCMP_ACT_ALLOW, SCMP_SYS(socketpair)); + if (rc) + return rc; +#endif + + rc = seccomp_rule_add_2(ctx, SCMP_ACT_ALLOW, SCMP_SYS(socketpair), + SCMP_CMP(0, SCMP_CMP_EQ, PF_FILE), + SCMP_CMP(1, SCMP_CMP_EQ, SOCK_STREAM|SOCK_CLOEXEC)); + if (rc) + return rc; + + return 0; +} + +/** + * Function responsible for setting up the setsockopt syscall for + * the seccomp filter sandbox. + */ +static int +sb_setsockopt(scmp_filter_ctx ctx, sandbox_cfg_t *filter) +{ + int rc = 0; + (void) filter; + +#ifdef __i386__ + rc = seccomp_rule_add_0(ctx, SCMP_ACT_ALLOW, SCMP_SYS(setsockopt)); + if (rc) + return rc; +#endif + + rc = seccomp_rule_add_2(ctx, SCMP_ACT_ALLOW, SCMP_SYS(setsockopt), + SCMP_CMP(1, SCMP_CMP_EQ, SOL_SOCKET), + SCMP_CMP(2, SCMP_CMP_EQ, SO_REUSEADDR)); + if (rc) + return rc; + + rc = seccomp_rule_add_2(ctx, SCMP_ACT_ALLOW, SCMP_SYS(setsockopt), + SCMP_CMP(1, SCMP_CMP_EQ, SOL_SOCKET), + SCMP_CMP(2, SCMP_CMP_EQ, SO_SNDBUF)); + if (rc) + return rc; + + rc = seccomp_rule_add_2(ctx, SCMP_ACT_ALLOW, SCMP_SYS(setsockopt), + SCMP_CMP(1, SCMP_CMP_EQ, SOL_SOCKET), + SCMP_CMP(2, SCMP_CMP_EQ, SO_RCVBUF)); + if (rc) + return rc; + +#ifdef IP_TRANSPARENT + rc = seccomp_rule_add_2(ctx, SCMP_ACT_ALLOW, SCMP_SYS(setsockopt), + SCMP_CMP(1, SCMP_CMP_EQ, SOL_IP), + SCMP_CMP(2, SCMP_CMP_EQ, IP_TRANSPARENT)); + if (rc) + return rc; +#endif + + return 0; +} + +/** + * Function responsible for setting up the getsockopt syscall for + * the seccomp filter sandbox. + */ +static int +sb_getsockopt(scmp_filter_ctx ctx, sandbox_cfg_t *filter) +{ + int rc = 0; + (void) filter; + +#ifdef __i386__ + rc = seccomp_rule_add_0(ctx, SCMP_ACT_ALLOW, SCMP_SYS(getsockopt)); + if (rc) + return rc; +#endif + + rc = seccomp_rule_add_2(ctx, SCMP_ACT_ALLOW, SCMP_SYS(getsockopt), + SCMP_CMP(1, SCMP_CMP_EQ, SOL_SOCKET), + SCMP_CMP(2, SCMP_CMP_EQ, SO_ERROR)); + if (rc) + return rc; + + return 0; +} + +#ifdef __NR_fcntl64 +/** + * Function responsible for setting up the fcntl64 syscall for + * the seccomp filter sandbox. + */ +static int +sb_fcntl64(scmp_filter_ctx ctx, sandbox_cfg_t *filter) +{ + int rc = 0; + (void) filter; + + rc = seccomp_rule_add_1(ctx, SCMP_ACT_ALLOW, SCMP_SYS(fcntl64), + SCMP_CMP(1, SCMP_CMP_EQ, F_GETFL)); + if (rc) + return rc; + + rc = seccomp_rule_add_2(ctx, SCMP_ACT_ALLOW, SCMP_SYS(fcntl64), + SCMP_CMP(1, SCMP_CMP_EQ, F_SETFL), + SCMP_CMP(2, SCMP_CMP_EQ, O_RDWR|O_NONBLOCK)); + if (rc) + return rc; + + rc = seccomp_rule_add_1(ctx, SCMP_ACT_ALLOW, SCMP_SYS(fcntl64), + SCMP_CMP(1, SCMP_CMP_EQ, F_GETFD)); + if (rc) + return rc; + + rc = seccomp_rule_add_2(ctx, SCMP_ACT_ALLOW, SCMP_SYS(fcntl64), + SCMP_CMP(1, SCMP_CMP_EQ, F_SETFD), + SCMP_CMP(2, SCMP_CMP_EQ, FD_CLOEXEC)); + if (rc) + return rc; + + return 0; +} +#endif + +/** + * Function responsible for setting up the epoll_ctl syscall for + * the seccomp filter sandbox. + * + * Note: basically allows everything but will keep for now.. + */ +static int +sb_epoll_ctl(scmp_filter_ctx ctx, sandbox_cfg_t *filter) +{ + int rc = 0; + (void) filter; + + rc = seccomp_rule_add_1(ctx, SCMP_ACT_ALLOW, SCMP_SYS(epoll_ctl), + SCMP_CMP(1, SCMP_CMP_EQ, EPOLL_CTL_ADD)); + if (rc) + return rc; + + rc = seccomp_rule_add_1(ctx, SCMP_ACT_ALLOW, SCMP_SYS(epoll_ctl), + SCMP_CMP(1, SCMP_CMP_EQ, EPOLL_CTL_MOD)); + if (rc) + return rc; + + rc = seccomp_rule_add_1(ctx, SCMP_ACT_ALLOW, SCMP_SYS(epoll_ctl), + SCMP_CMP(1, SCMP_CMP_EQ, EPOLL_CTL_DEL)); + if (rc) + return rc; + + return 0; +} + +/** + * Function responsible for setting up the fcntl64 syscall for + * the seccomp filter sandbox. + * + * NOTE: if multiple filters need to be added, the PR_SECCOMP parameter needs + * to be whitelisted in this function. + */ +static int +sb_prctl(scmp_filter_ctx ctx, sandbox_cfg_t *filter) +{ + int rc = 0; + (void) filter; + + rc = seccomp_rule_add_1(ctx, SCMP_ACT_ALLOW, SCMP_SYS(prctl), + SCMP_CMP(0, SCMP_CMP_EQ, PR_SET_DUMPABLE)); + if (rc) + return rc; + + return 0; +} + +/** + * Function responsible for setting up the fcntl64 syscall for + * the seccomp filter sandbox. + * + * NOTE: does not NEED to be here.. currently only occurs before filter; will + * keep just in case for the future. + */ +static int +sb_mprotect(scmp_filter_ctx ctx, sandbox_cfg_t *filter) +{ + int rc = 0; + (void) filter; + + rc = seccomp_rule_add_1(ctx, SCMP_ACT_ALLOW, SCMP_SYS(mprotect), + SCMP_CMP(2, SCMP_CMP_EQ, PROT_READ)); + if (rc) + return rc; + + rc = seccomp_rule_add_1(ctx, SCMP_ACT_ALLOW, SCMP_SYS(mprotect), + SCMP_CMP(2, SCMP_CMP_EQ, PROT_NONE)); + if (rc) + return rc; + + return 0; +} + +/** + * Function responsible for setting up the rt_sigprocmask syscall for + * the seccomp filter sandbox. + */ +static int +sb_rt_sigprocmask(scmp_filter_ctx ctx, sandbox_cfg_t *filter) +{ + int rc = 0; + (void) filter; + + rc = seccomp_rule_add_1(ctx, SCMP_ACT_ALLOW, SCMP_SYS(rt_sigprocmask), + SCMP_CMP(0, SCMP_CMP_EQ, SIG_UNBLOCK)); + if (rc) + return rc; + + rc = seccomp_rule_add_1(ctx, SCMP_ACT_ALLOW, SCMP_SYS(rt_sigprocmask), + SCMP_CMP(0, SCMP_CMP_EQ, SIG_SETMASK)); + if (rc) + return rc; + + return 0; +} + +/** + * Function responsible for setting up the flock syscall for + * the seccomp filter sandbox. + * + * NOTE: does not need to be here, occurs before filter is applied. + */ +static int +sb_flock(scmp_filter_ctx ctx, sandbox_cfg_t *filter) +{ + int rc = 0; + (void) filter; + + rc = seccomp_rule_add_1(ctx, SCMP_ACT_ALLOW, SCMP_SYS(flock), + SCMP_CMP(1, SCMP_CMP_EQ, LOCK_EX|LOCK_NB)); + if (rc) + return rc; + + rc = seccomp_rule_add_1(ctx, SCMP_ACT_ALLOW, SCMP_SYS(flock), + SCMP_CMP(1, SCMP_CMP_EQ, LOCK_UN)); + if (rc) + return rc; + + return 0; +} + +/** + * Function responsible for setting up the futex syscall for + * the seccomp filter sandbox. + */ +static int +sb_futex(scmp_filter_ctx ctx, sandbox_cfg_t *filter) +{ + int rc = 0; + (void) filter; + + // can remove + rc = seccomp_rule_add_1(ctx, SCMP_ACT_ALLOW, SCMP_SYS(futex), + SCMP_CMP(1, SCMP_CMP_EQ, + FUTEX_WAIT_BITSET_PRIVATE|FUTEX_CLOCK_REALTIME)); + if (rc) + return rc; + + rc = seccomp_rule_add_1(ctx, SCMP_ACT_ALLOW, SCMP_SYS(futex), + SCMP_CMP(1, SCMP_CMP_EQ, FUTEX_WAKE_PRIVATE)); + if (rc) + return rc; + + rc = seccomp_rule_add_1(ctx, SCMP_ACT_ALLOW, SCMP_SYS(futex), + SCMP_CMP(1, SCMP_CMP_EQ, FUTEX_WAIT_PRIVATE)); + if (rc) + return rc; + + return 0; +} + +/** + * Function responsible for setting up the mremap syscall for + * the seccomp filter sandbox. + * + * NOTE: so far only occurs before filter is applied. + */ +static int +sb_mremap(scmp_filter_ctx ctx, sandbox_cfg_t *filter) +{ + int rc = 0; + (void) filter; + + rc = seccomp_rule_add_1(ctx, SCMP_ACT_ALLOW, SCMP_SYS(mremap), + SCMP_CMP(3, SCMP_CMP_EQ, MREMAP_MAYMOVE)); + if (rc) + return rc; + + return 0; +} + +/** + * Function responsible for setting up the poll syscall for + * the seccomp filter sandbox. + */ +static int +sb_poll(scmp_filter_ctx ctx, sandbox_cfg_t *filter) +{ + int rc = 0; + (void) filter; + + rc = seccomp_rule_add_2(ctx, SCMP_ACT_ALLOW, SCMP_SYS(poll), + SCMP_CMP(1, SCMP_CMP_EQ, 1), + SCMP_CMP(2, SCMP_CMP_EQ, 10)); + if (rc) + return rc; + + return 0; +} + +#ifdef __NR_stat64 +/** + * Function responsible for setting up the stat64 syscall for + * the seccomp filter sandbox. + */ +static int +sb_stat64(scmp_filter_ctx ctx, sandbox_cfg_t *filter) +{ + int rc = 0; + sandbox_cfg_t *elem = NULL; + + // for each dynamic parameter filters + for (elem = filter; elem != NULL; elem = elem->next) { + smp_param_t *param = elem->param; + + if (param != NULL && param->prot == 1 && (param->syscall == SCMP_SYS(open) + || param->syscall == SCMP_SYS(stat64))) { + rc = seccomp_rule_add_1(ctx, SCMP_ACT_ALLOW, SCMP_SYS(stat64), + SCMP_CMP(0, SCMP_CMP_EQ, param->value)); + if (rc != 0) { + log_err(LD_BUG,"(Sandbox) failed to add open syscall, received " + "libseccomp error %d", rc); + return rc; + } + } + } + + return 0; +} +#endif + +/** + * Array of function pointers responsible for filtering different syscalls at + * a parameter level. + */ +static sandbox_filter_func_t filter_func[] = { + sb_rt_sigaction, + sb_rt_sigprocmask, +#if 0 + sb_execve, +#endif + sb_time, + sb_accept4, +#ifdef __NR_mmap2 + sb_mmap2, +#endif + sb_open, + sb_openat, + sb__sysctl, + sb_rename, +#ifdef __NR_fcntl64 + sb_fcntl64, +#endif + sb_epoll_ctl, + sb_prctl, + sb_mprotect, + sb_flock, + sb_futex, + sb_mremap, + sb_poll, +#ifdef __NR_stat64 + sb_stat64, +#endif + + sb_socket, + sb_setsockopt, + sb_getsockopt, + sb_socketpair +}; + +const char * +sandbox_intern_string(const char *str) +{ + sandbox_cfg_t *elem; + + if (str == NULL) + return NULL; + + for (elem = filter_dynamic; elem != NULL; elem = elem->next) { + smp_param_t *param = elem->param; + + if (param->prot) { + if (!strcmp(str, (char*)(param->value))) { + return (char*)param->value; + } + if (param->value2 && !strcmp(str, (char*)param->value2)) { + return (char*)param->value2; + } + } + } + + if (sandbox_active) + log_warn(LD_BUG, "No interned sandbox parameter found for %s", str); + return str; +} + +/** DOCDOC */ +static int +prot_strings_helper(strmap_t *locations, + char **pr_mem_next_p, + size_t *pr_mem_left_p, + intptr_t *value_p) +{ + char *param_val; + size_t param_size; + void *location; + + if (*value_p == 0) + return 0; + + param_val = (char*) *value_p; + param_size = strlen(param_val) + 1; + location = strmap_get(locations, param_val); + + if (location) { + // We already interned this string. + tor_free(param_val); + *value_p = (intptr_t) location; + return 0; + } else if (*pr_mem_left_p >= param_size) { + // copy to protected + location = *pr_mem_next_p; + memcpy(location, param_val, param_size); + + // re-point el parameter to protected + tor_free(param_val); + *value_p = (intptr_t) location; + + strmap_set(locations, location, location); /* good real estate advice */ + + // move next available protected memory + *pr_mem_next_p += param_size; + *pr_mem_left_p -= param_size; + return 0; + } else { + log_err(LD_BUG,"(Sandbox) insufficient protected memory!"); + return -1; + } +} + +/** + * Protects all the strings in the sandbox's parameter list configuration. It + * works by calculating the total amount of memory required by the parameter + * list, allocating the memory using mmap, and protecting it from writes with + * mprotect(). + */ +static int +prot_strings(scmp_filter_ctx ctx, sandbox_cfg_t* cfg) +{ + int ret = 0; + size_t pr_mem_size = 0, pr_mem_left = 0; + char *pr_mem_next = NULL, *pr_mem_base; + sandbox_cfg_t *el = NULL; + strmap_t *locations = NULL; + + // get total number of bytes required to mmap. (Overestimate.) + for (el = cfg; el != NULL; el = el->next) { + pr_mem_size += strlen((char*) el->param->value) + 1; + if (el->param->value2) + pr_mem_size += strlen((char*) el->param->value2) + 1; + } + + // allocate protected memory with MALLOC_MP_LIM canary + pr_mem_base = (char*) mmap(NULL, MALLOC_MP_LIM + pr_mem_size, + PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); + if (pr_mem_base == MAP_FAILED) { + log_err(LD_BUG,"(Sandbox) failed allocate protected memory! mmap: %s", + strerror(errno)); + ret = -1; + goto out; + } + + pr_mem_next = pr_mem_base + MALLOC_MP_LIM; + pr_mem_left = pr_mem_size; + + locations = strmap_new(); + + // change el value pointer to protected + for (el = cfg; el != NULL; el = el->next) { + if (prot_strings_helper(locations, &pr_mem_next, &pr_mem_left, + &el->param->value) < 0) { + ret = -2; + goto out; + } + if (prot_strings_helper(locations, &pr_mem_next, &pr_mem_left, + &el->param->value2) < 0) { + ret = -2; + goto out; + } + el->param->prot = 1; + } + + // protecting from writes + if (mprotect(pr_mem_base, MALLOC_MP_LIM + pr_mem_size, PROT_READ)) { + log_err(LD_BUG,"(Sandbox) failed to protect memory! mprotect: %s", + strerror(errno)); + ret = -3; + goto out; + } + + /* + * Setting sandbox restrictions so the string memory cannot be tampered with + */ + // no mremap of the protected base address + ret = seccomp_rule_add_1(ctx, SCMP_ACT_KILL, SCMP_SYS(mremap), + SCMP_CMP(0, SCMP_CMP_EQ, (intptr_t) pr_mem_base)); + if (ret) { + log_err(LD_BUG,"(Sandbox) mremap protected memory filter fail!"); + return ret; + } + + // no munmap of the protected base address + ret = seccomp_rule_add_1(ctx, SCMP_ACT_KILL, SCMP_SYS(munmap), + SCMP_CMP(0, SCMP_CMP_EQ, (intptr_t) pr_mem_base)); + if (ret) { + log_err(LD_BUG,"(Sandbox) munmap protected memory filter fail!"); + return ret; + } + + /* + * Allow mprotect with PROT_READ|PROT_WRITE because openssl uses it, but + * never over the memory region used by the protected strings. + * + * PROT_READ|PROT_WRITE was originally fully allowed in sb_mprotect(), but + * had to be removed due to limitation of libseccomp regarding intervals. + * + * There is a restriction on how much you can mprotect with R|W up to the + * size of the canary. + */ + ret = seccomp_rule_add_3(ctx, SCMP_ACT_ALLOW, SCMP_SYS(mprotect), + SCMP_CMP(0, SCMP_CMP_LT, (intptr_t) pr_mem_base), + SCMP_CMP(1, SCMP_CMP_LE, MALLOC_MP_LIM), + SCMP_CMP(2, SCMP_CMP_EQ, PROT_READ|PROT_WRITE)); + if (ret) { + log_err(LD_BUG,"(Sandbox) mprotect protected memory filter fail (LT)!"); + return ret; + } + + ret = seccomp_rule_add_3(ctx, SCMP_ACT_ALLOW, SCMP_SYS(mprotect), + SCMP_CMP(0, SCMP_CMP_GT, (intptr_t) pr_mem_base + pr_mem_size + + MALLOC_MP_LIM), + SCMP_CMP(1, SCMP_CMP_LE, MALLOC_MP_LIM), + SCMP_CMP(2, SCMP_CMP_EQ, PROT_READ|PROT_WRITE)); + if (ret) { + log_err(LD_BUG,"(Sandbox) mprotect protected memory filter fail (GT)!"); + return ret; + } + + out: + strmap_free(locations, NULL); + return ret; +} + +/** + * Auxiliary function used in order to allocate a sandbox_cfg_t element and set + * it's values according the the parameter list. All elements are initialised + * with the 'prot' field set to false, as the pointer is not protected at this + * point. + */ +static sandbox_cfg_t* +new_element2(int syscall, intptr_t value, intptr_t value2) +{ + smp_param_t *param = NULL; + + sandbox_cfg_t *elem = tor_malloc_zero(sizeof(sandbox_cfg_t)); + param = elem->param = tor_malloc_zero(sizeof(smp_param_t)); + + param->syscall = syscall; + param->value = value; + param->value2 = value2; + param->prot = 0; + + return elem; +} + +static sandbox_cfg_t* +new_element(int syscall, intptr_t value) +{ + return new_element2(syscall, value, 0); +} + +#ifdef __NR_stat64 +#define SCMP_stat SCMP_SYS(stat64) +#else +#define SCMP_stat SCMP_SYS(stat) +#endif + +int +sandbox_cfg_allow_stat_filename(sandbox_cfg_t **cfg, char *file) +{ + sandbox_cfg_t *elem = NULL; + + elem = new_element(SCMP_stat, (intptr_t)(void*) file); + if (!elem) { + log_err(LD_BUG,"(Sandbox) failed to register parameter!"); + return -1; + } + + elem->next = *cfg; + *cfg = elem; + + return 0; +} + +int +sandbox_cfg_allow_stat_filename_array(sandbox_cfg_t **cfg, ...) +{ + int rc = 0; + char *fn = NULL; + + va_list ap; + va_start(ap, cfg); + + while ((fn = va_arg(ap, char*)) != NULL) { + rc = sandbox_cfg_allow_stat_filename(cfg, fn); + if (rc) { + log_err(LD_BUG,"(Sandbox) sandbox_cfg_allow_stat_filename_array fail"); + goto end; + } + } + + end: + va_end(ap); + return 0; +} + +int +sandbox_cfg_allow_open_filename(sandbox_cfg_t **cfg, char *file) +{ + sandbox_cfg_t *elem = NULL; + + elem = new_element(SCMP_SYS(open), (intptr_t)(void *) file); + if (!elem) { + log_err(LD_BUG,"(Sandbox) failed to register parameter!"); + return -1; + } + + elem->next = *cfg; + *cfg = elem; + + return 0; +} + +int +sandbox_cfg_allow_rename(sandbox_cfg_t **cfg, char *file1, char *file2) +{ + sandbox_cfg_t *elem = NULL; + + elem = new_element2(SCMP_SYS(rename), + (intptr_t)(void *) file1, + (intptr_t)(void *) file2); + + if (!elem) { + log_err(LD_BUG,"(Sandbox) failed to register parameter!"); + return -1; + } + + elem->next = *cfg; + *cfg = elem; + + return 0; +} + +int +sandbox_cfg_allow_open_filename_array(sandbox_cfg_t **cfg, ...) +{ + int rc = 0; + char *fn = NULL; + + va_list ap; + va_start(ap, cfg); + + while ((fn = va_arg(ap, char*)) != NULL) { + rc = sandbox_cfg_allow_open_filename(cfg, fn); + if (rc) { + log_err(LD_BUG,"(Sandbox) sandbox_cfg_allow_open_filename_array fail"); + goto end; + } + } + + end: + va_end(ap); + return 0; +} + +int +sandbox_cfg_allow_openat_filename(sandbox_cfg_t **cfg, char *file) +{ + sandbox_cfg_t *elem = NULL; + + elem = new_element(SCMP_SYS(openat), (intptr_t)(void *) file); + if (!elem) { + log_err(LD_BUG,"(Sandbox) failed to register parameter!"); + return -1; + } + + elem->next = *cfg; + *cfg = elem; + + return 0; +} + +int +sandbox_cfg_allow_openat_filename_array(sandbox_cfg_t **cfg, ...) +{ + int rc = 0; + char *fn = NULL; + + va_list ap; + va_start(ap, cfg); + + while ((fn = va_arg(ap, char*)) != NULL) { + rc = sandbox_cfg_allow_openat_filename(cfg, fn); + if (rc) { + log_err(LD_BUG,"(Sandbox) sandbox_cfg_allow_openat_filename_array fail"); + goto end; + } + } + + end: + va_end(ap); + return 0; +} + +#if 0 +int +sandbox_cfg_allow_execve(sandbox_cfg_t **cfg, const char *com) +{ + sandbox_cfg_t *elem = NULL; + + elem = new_element(SCMP_SYS(execve), (intptr_t)(void *) com); + if (!elem) { + log_err(LD_BUG,"(Sandbox) failed to register parameter!"); + return -1; + } + + elem->next = *cfg; + *cfg = elem; + + return 0; +} + +int +sandbox_cfg_allow_execve_array(sandbox_cfg_t **cfg, ...) +{ + int rc = 0; + char *fn = NULL; + + va_list ap; + va_start(ap, cfg); + + while ((fn = va_arg(ap, char*)) != NULL) { + + rc = sandbox_cfg_allow_execve(cfg, fn); + if (rc) { + log_err(LD_BUG,"(Sandbox) sandbox_cfg_allow_execve_array failed"); + goto end; + } + } + + end: + va_end(ap); + return 0; +} +#endif + +/** Cache entry for getaddrinfo results; used when sandboxing is implemented + * so that we can consult the cache when the sandbox prevents us from doing + * getaddrinfo. + * + * We support only a limited range of getaddrinfo calls, where servname is null + * and hints contains only socktype=SOCK_STREAM, family in INET,INET6,UNSPEC. + */ +typedef struct cached_getaddrinfo_item_t { + HT_ENTRY(cached_getaddrinfo_item_t) node; + char *name; + int family; + /** set if no error; otherwise NULL */ + struct addrinfo *res; + /** 0 for no error; otherwise an EAI_* value */ + int err; +} cached_getaddrinfo_item_t; + +static unsigned +cached_getaddrinfo_item_hash(const cached_getaddrinfo_item_t *item) +{ + return (unsigned)siphash24g(item->name, strlen(item->name)) + item->family; +} + +static unsigned +cached_getaddrinfo_items_eq(const cached_getaddrinfo_item_t *a, + const cached_getaddrinfo_item_t *b) +{ + return (a->family == b->family) && 0 == strcmp(a->name, b->name); +} + +static void +cached_getaddrinfo_item_free(cached_getaddrinfo_item_t *item) +{ + if (item == NULL) + return; + + tor_free(item->name); + if (item->res) + freeaddrinfo(item->res); + tor_free(item); +} + +static HT_HEAD(getaddrinfo_cache, cached_getaddrinfo_item_t) + getaddrinfo_cache = HT_INITIALIZER(); + +HT_PROTOTYPE(getaddrinfo_cache, cached_getaddrinfo_item_t, node, + cached_getaddrinfo_item_hash, + cached_getaddrinfo_items_eq); +HT_GENERATE(getaddrinfo_cache, cached_getaddrinfo_item_t, node, + cached_getaddrinfo_item_hash, + cached_getaddrinfo_items_eq, + 0.6, tor_malloc_, tor_realloc_, tor_free_); + +int +sandbox_getaddrinfo(const char *name, const char *servname, + const struct addrinfo *hints, + struct addrinfo **res) +{ + int err; + struct cached_getaddrinfo_item_t search, *item; + + if (servname != NULL) { + log_warn(LD_BUG, "called with non-NULL servname"); + return EAI_NONAME; + } + if (name == NULL) { + log_warn(LD_BUG, "called with NULL name"); + return EAI_NONAME; + } + + *res = NULL; + + memset(&search, 0, sizeof(search)); + search.name = (char *) name; + search.family = hints ? hints->ai_family : AF_UNSPEC; + item = HT_FIND(getaddrinfo_cache, &getaddrinfo_cache, &search); + + if (! sandbox_is_active()) { + /* If the sandbox is not turned on yet, then getaddrinfo and store the + result. */ + + err = getaddrinfo(name, NULL, hints, res); + log_info(LD_NET,"(Sandbox) getaddrinfo %s.", err ? "failed" : "succeeded"); + + if (! item) { + item = tor_malloc_zero(sizeof(*item)); + item->name = tor_strdup(name); + item->family = hints ? hints->ai_family : AF_UNSPEC; + HT_INSERT(getaddrinfo_cache, &getaddrinfo_cache, item); + } + + if (item->res) { + freeaddrinfo(item->res); + item->res = NULL; + } + item->res = *res; + item->err = err; + return err; + } + + /* Otherwise, the sanbox is on. If we have an item, yield its cached + result. */ + if (item) { + *res = item->res; + return item->err; + } + + /* getting here means something went wrong */ + log_err(LD_BUG,"(Sandbox) failed to get address %s!", name); + return EAI_NONAME; +} + +int +sandbox_add_addrinfo(const char *name) +{ + struct addrinfo *res; + struct addrinfo hints; + int i; + static const int families[] = { AF_INET, AF_INET6, AF_UNSPEC }; + + memset(&hints, 0, sizeof(hints)); + hints.ai_socktype = SOCK_STREAM; + for (i = 0; i < 3; ++i) { + hints.ai_family = families[i]; + + res = NULL; + (void) sandbox_getaddrinfo(name, NULL, &hints, &res); + if (res) + sandbox_freeaddrinfo(res); + } + + return 0; +} + +void +sandbox_free_getaddrinfo_cache(void) +{ + cached_getaddrinfo_item_t **next, **item; + + for (item = HT_START(getaddrinfo_cache, &getaddrinfo_cache); + item; + item = next) { + next = HT_NEXT_RMV(getaddrinfo_cache, &getaddrinfo_cache, item); + cached_getaddrinfo_item_free(*item); + } + + HT_CLEAR(getaddrinfo_cache, &getaddrinfo_cache); +} + +/** + * Function responsible for going through the parameter syscall filters and + * call each function pointer in the list. + */ +static int +add_param_filter(scmp_filter_ctx ctx, sandbox_cfg_t* cfg) +{ + unsigned i; + int rc = 0; + + // function pointer + for (i = 0; i < ARRAY_LENGTH(filter_func); i++) { + if ((filter_func[i])(ctx, cfg)) { + log_err(LD_BUG,"(Sandbox) failed to add syscall %d, received libseccomp " + "error %d", i, rc); + return rc; + } + } + + return 0; +} + +/** + * Function responsible of loading the libseccomp syscall filters which do not + * have parameter filtering. + */ +static int +add_noparam_filter(scmp_filter_ctx ctx) +{ + unsigned i; + int rc = 0; + + // add general filters + for (i = 0; i < ARRAY_LENGTH(filter_nopar_gen); i++) { + rc = seccomp_rule_add_0(ctx, SCMP_ACT_ALLOW, filter_nopar_gen[i]); + if (rc != 0) { + log_err(LD_BUG,"(Sandbox) failed to add syscall index %d (NR=%d), " + "received libseccomp error %d", i, filter_nopar_gen[i], rc); + return rc; + } + } + + return 0; +} + +/** + * Function responsible for setting up and enabling a global syscall filter. + * The function is a prototype developed for stage 1 of sandboxing Tor. + * Returns 0 on success. + */ +static int +install_syscall_filter(sandbox_cfg_t* cfg) +{ + int rc = 0; + scmp_filter_ctx ctx; + + ctx = seccomp_init(SCMP_ACT_TRAP); + if (ctx == NULL) { + log_err(LD_BUG,"(Sandbox) failed to initialise libseccomp context"); + rc = -1; + goto end; + } + + // protectign sandbox parameter strings + if ((rc = prot_strings(ctx, cfg))) { + goto end; + } + + // add parameter filters + if ((rc = add_param_filter(ctx, cfg))) { + log_err(LD_BUG, "(Sandbox) failed to add param filters!"); + goto end; + } + + // adding filters with no parameters + if ((rc = add_noparam_filter(ctx))) { + log_err(LD_BUG, "(Sandbox) failed to add param filters!"); + goto end; + } + + // loading the seccomp2 filter + if ((rc = seccomp_load(ctx))) { + log_err(LD_BUG, "(Sandbox) failed to load: %d (%s)!", rc, + strerror(-rc)); + goto end; + } + + // marking the sandbox as active + sandbox_active = 1; + + end: + seccomp_release(ctx); + return (rc < 0 ? -rc : rc); +} + +#include "linux_syscalls.inc" +static const char * +get_syscall_name(int syscall_num) +{ + int i; + for (i = 0; SYSCALLS_BY_NUMBER[i].syscall_name; ++i) { + if (SYSCALLS_BY_NUMBER[i].syscall_num == syscall_num) + return SYSCALLS_BY_NUMBER[i].syscall_name; + } + + { + static char syscall_name_buf[64]; + format_dec_number_sigsafe(syscall_num, + syscall_name_buf, sizeof(syscall_name_buf)); + return syscall_name_buf; + } +} + +#ifdef USE_BACKTRACE +#define MAX_DEPTH 256 +static void *syscall_cb_buf[MAX_DEPTH]; +#endif + +/** + * Function called when a SIGSYS is caught by the application. It notifies the + * user that an error has occurred and either terminates or allows the + * application to continue execution, based on the DEBUGGING_CLOSE symbol. + */ +static void +sigsys_debugging(int nr, siginfo_t *info, void *void_context) +{ + ucontext_t *ctx = (ucontext_t *) (void_context); + const char *syscall_name; + int syscall; +#ifdef USE_BACKTRACE + int depth; + int n_fds, i; + const int *fds = NULL; +#endif + + (void) nr; + + if (info->si_code != SYS_SECCOMP) + return; + + if (!ctx) + return; + + syscall = (int) ctx->uc_mcontext.M_SYSCALL; + +#ifdef USE_BACKTRACE + depth = backtrace(syscall_cb_buf, MAX_DEPTH); + /* Clean up the top stack frame so we get the real function + * name for the most recently failing function. */ + clean_backtrace(syscall_cb_buf, depth, ctx); +#endif + + syscall_name = get_syscall_name(syscall); + + tor_log_err_sigsafe("(Sandbox) Caught a bad syscall attempt (syscall ", + syscall_name, + ")\n", + NULL); + +#ifdef USE_BACKTRACE + n_fds = tor_log_get_sigsafe_err_fds(&fds); + for (i=0; i < n_fds; ++i) + backtrace_symbols_fd(syscall_cb_buf, depth, fds[i]); +#endif + +#if defined(DEBUGGING_CLOSE) + _exit(1); +#endif // DEBUGGING_CLOSE +} + +/** + * Function that adds a handler for SIGSYS, which is the signal thrown + * when the application is issuing a syscall which is not allowed. The + * main purpose of this function is to help with debugging by identifying + * filtered syscalls. + */ +static int +install_sigsys_debugging(void) +{ + struct sigaction act; + sigset_t mask; + + memset(&act, 0, sizeof(act)); + sigemptyset(&mask); + sigaddset(&mask, SIGSYS); + + act.sa_sigaction = &sigsys_debugging; + act.sa_flags = SA_SIGINFO; + if (sigaction(SIGSYS, &act, NULL) < 0) { + log_err(LD_BUG,"(Sandbox) Failed to register SIGSYS signal handler"); + return -1; + } + + if (sigprocmask(SIG_UNBLOCK, &mask, NULL)) { + log_err(LD_BUG,"(Sandbox) Failed call to sigprocmask()"); + return -2; + } + + return 0; +} + +/** + * Function responsible of registering the sandbox_cfg_t list of parameter + * syscall filters to the existing parameter list. This is used for incipient + * multiple-sandbox support. + */ +static int +register_cfg(sandbox_cfg_t* cfg) +{ + sandbox_cfg_t *elem = NULL; + + if (filter_dynamic == NULL) { + filter_dynamic = cfg; + return 0; + } + + for (elem = filter_dynamic; elem->next != NULL; elem = elem->next) + ; + + elem->next = cfg; + + return 0; +} + +#endif // USE_LIBSECCOMP + +#ifdef USE_LIBSECCOMP +/** + * Initialises the syscall sandbox filter for any linux architecture, taking + * into account various available features for different linux flavours. + */ +static int +initialise_libseccomp_sandbox(sandbox_cfg_t* cfg) +{ + if (install_sigsys_debugging()) + return -1; + + if (install_syscall_filter(cfg)) + return -2; + + if (register_cfg(cfg)) + return -3; + + return 0; +} + +int +sandbox_is_active(void) +{ + return sandbox_active != 0; +} +#endif // USE_LIBSECCOMP + +sandbox_cfg_t* +sandbox_cfg_new(void) +{ + return NULL; +} + +int +sandbox_init(sandbox_cfg_t *cfg) +{ +#if defined(USE_LIBSECCOMP) + return initialise_libseccomp_sandbox(cfg); + +#elif defined(__linux__) + (void)cfg; + log_warn(LD_GENERAL, + "This version of Tor was built without support for sandboxing. To " + "build with support for sandboxing on Linux, you must have " + "libseccomp and its necessary header files (e.g. seccomp.h)."); + return 0; + +#else + (void)cfg; + log_warn(LD_GENERAL, + "Currently, sandboxing is only implemented on Linux. The feature " + "is disabled on your platform."); + return 0; +#endif +} + +#ifndef USE_LIBSECCOMP +int +sandbox_cfg_allow_open_filename(sandbox_cfg_t **cfg, char *file) +{ + (void)cfg; (void)file; + return 0; +} + +int +sandbox_cfg_allow_open_filename_array(sandbox_cfg_t **cfg, ...) +{ + (void)cfg; + return 0; +} + +int +sandbox_cfg_allow_openat_filename(sandbox_cfg_t **cfg, char *file) +{ + (void)cfg; (void)file; + return 0; +} + +int +sandbox_cfg_allow_openat_filename_array(sandbox_cfg_t **cfg, ...) +{ + (void)cfg; + return 0; +} + +#if 0 +int +sandbox_cfg_allow_execve(sandbox_cfg_t **cfg, const char *com) +{ + (void)cfg; (void)com; + return 0; +} + +int +sandbox_cfg_allow_execve_array(sandbox_cfg_t **cfg, ...) +{ + (void)cfg; + return 0; +} +#endif + +int +sandbox_cfg_allow_stat_filename(sandbox_cfg_t **cfg, char *file) +{ + (void)cfg; (void)file; + return 0; +} + +int +sandbox_cfg_allow_stat_filename_array(sandbox_cfg_t **cfg, ...) +{ + (void)cfg; + return 0; +} + +int +sandbox_cfg_allow_rename(sandbox_cfg_t **cfg, char *file1, char *file2) +{ + (void)cfg; (void)file1; (void)file2; + return 0; +} + +int +sandbox_is_active(void) +{ + return 0; +} +#endif + diff --git a/src/common/sandbox.h b/src/common/sandbox.h new file mode 100644 index 0000000000..20d5d5080c --- /dev/null +++ b/src/common/sandbox.h @@ -0,0 +1,212 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2013, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file sandbox.h + * \brief Header file for sandbox.c. + **/ + +#ifndef SANDBOX_H_ +#define SANDBOX_H_ + +#include "orconfig.h" +#include "torint.h" + +#ifndef SYS_SECCOMP + +/** + * Used by SIGSYS signal handler to check if the signal was issued due to a + * seccomp2 filter violation. + */ +#define SYS_SECCOMP 1 + +#endif + +#if defined(HAVE_SECCOMP_H) && defined(__linux__) +#define USE_LIBSECCOMP +#endif + +struct sandbox_cfg_elem; + +/** Typedef to structure used to manage a sandbox configuration. */ +typedef struct sandbox_cfg_elem sandbox_cfg_t; + +/** + * Linux definitions + */ +#ifdef USE_LIBSECCOMP + +#ifndef __USE_GNU +#define __USE_GNU +#endif +#include <sys/ucontext.h> +#include <seccomp.h> +#include <netdb.h> + +#define PARAM_PTR 0 +#define PARAM_NUM 1 + +/** + * Enum used to manage the type of the implementation for general purpose. + */ +typedef enum { + /** Libseccomp implementation based on seccomp2*/ + LIBSECCOMP2 = 0 +} SB_IMPL; + +/** + * Configuration parameter structure associated with the LIBSECCOMP2 + * implementation. + */ +typedef struct smp_param { + /** syscall associated with parameter. */ + int syscall; + + /** parameter value. */ + intptr_t value; + /** parameter value, second argument. */ + intptr_t value2; + + /** parameter flag (0 = not protected, 1 = protected). */ + int prot; +} smp_param_t; + +/** + * Structure used to manage a sandbox configuration. + * + * It is implemented as a linked list of parameters. Currently only controls + * parameters for open, openat, execve, stat64. + */ +struct sandbox_cfg_elem { + /** Sandbox implementation which dictates the parameter type. */ + SB_IMPL implem; + + /** Configuration parameter. */ + smp_param_t *param; + + /** Next element of the configuration*/ + struct sandbox_cfg_elem *next; +}; + +/** Function pointer defining the prototype of a filter function.*/ +typedef int (*sandbox_filter_func_t)(scmp_filter_ctx ctx, + sandbox_cfg_t *filter); + +/** Type that will be used in step 3 in order to manage multiple sandboxes.*/ +typedef struct { + /** function pointers associated with the filter */ + sandbox_filter_func_t *filter_func; + + /** filter function pointer parameters */ + sandbox_cfg_t *filter_dynamic; +} sandbox_t; + +#endif // USE_LIBSECCOMP + +#ifdef USE_LIBSECCOMP +/** Pre-calls getaddrinfo in order to pre-record result. */ +int sandbox_add_addrinfo(const char *addr); + +struct addrinfo; +/** Replacement for getaddrinfo(), using pre-recorded results. */ +int sandbox_getaddrinfo(const char *name, const char *servname, + const struct addrinfo *hints, + struct addrinfo **res); +#define sandbox_freeaddrinfo(addrinfo) ((void)0) +void sandbox_free_getaddrinfo_cache(void); +#else +#define sandbox_getaddrinfo(name, servname, hints, res) \ + getaddrinfo((name),(servname), (hints),(res)) +#define sandbox_add_addrinfo(name) \ + ((void)(name)) +#define sandbox_freeaddrinfo(addrinfo) \ + freeaddrinfo((addrinfo)) +#define sandbox_free_getaddrinfo_cache() +#endif + +#ifdef USE_LIBSECCOMP +/** Returns a registered protected string used with the sandbox, given that + * it matches the parameter. + */ +const char* sandbox_intern_string(const char *param); +#else +#define sandbox_intern_string(s) (s) +#endif + +/** Creates an empty sandbox configuration file.*/ +sandbox_cfg_t * sandbox_cfg_new(void); + +/** + * Function used to add a open allowed filename to a supplied configuration. + * The (char*) specifies the path to the allowed file; we take ownership + * of the pointer. + */ +int sandbox_cfg_allow_open_filename(sandbox_cfg_t **cfg, char *file); + +/**DOCDOC*/ +int sandbox_cfg_allow_rename(sandbox_cfg_t **cfg, char *file1, char *file2); + +/** Function used to add a series of open allowed filenames to a supplied + * configuration. + * @param cfg sandbox configuration. + * @param ... a list of stealable pointers to permitted files. The last + * one must be NULL. +*/ +int sandbox_cfg_allow_open_filename_array(sandbox_cfg_t **cfg, ...); + +/** + * Function used to add a openat allowed filename to a supplied configuration. + * The (char*) specifies the path to the allowed file; we steal the pointer to + * that file. + */ +int sandbox_cfg_allow_openat_filename(sandbox_cfg_t **cfg, char *file); + +/** Function used to add a series of openat allowed filenames to a supplied + * configuration. + * @param cfg sandbox configuration. + * @param ... a list of stealable pointers to permitted files. The last + * one must be NULL. + */ +int sandbox_cfg_allow_openat_filename_array(sandbox_cfg_t **cfg, ...); + +#if 0 +/** + * Function used to add a execve allowed filename to a supplied configuration. + * The (char*) specifies the path to the allowed file; that pointer is stolen. + */ +int sandbox_cfg_allow_execve(sandbox_cfg_t **cfg, const char *com); + +/** Function used to add a series of execve allowed filenames to a supplied + * configuration. + * @param cfg sandbox configuration. + * @param ... an array of stealable pointers to permitted files. The last + * one must be NULL. + */ +int sandbox_cfg_allow_execve_array(sandbox_cfg_t **cfg, ...); +#endif + +/** + * Function used to add a stat/stat64 allowed filename to a configuration. + * The (char*) specifies the path to the allowed file; that pointer is stolen. + */ +int sandbox_cfg_allow_stat_filename(sandbox_cfg_t **cfg, char *file); + +/** Function used to add a series of stat64 allowed filenames to a supplied + * configuration. + * @param cfg sandbox configuration. + * @param ... an array of stealable pointers to permitted files. The last + * one must be NULL. + */ +int sandbox_cfg_allow_stat_filename_array(sandbox_cfg_t **cfg, ...); + +/** Function used to initialise a sandbox configuration.*/ +int sandbox_init(sandbox_cfg_t* cfg); + +/** Return true iff the sandbox is turned on. */ +int sandbox_is_active(void); + +#endif /* SANDBOX_H_ */ + diff --git a/src/common/testsupport.h b/src/common/testsupport.h new file mode 100644 index 0000000000..4a4f50b69b --- /dev/null +++ b/src/common/testsupport.h @@ -0,0 +1,80 @@ +/* Copyright (c) 2013, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef TOR_TESTSUPPORT_H +#define TOR_TESTSUPPORT_H + +#ifdef TOR_UNIT_TESTS +#define STATIC +#else +#define STATIC static +#endif + +/** Quick and dirty macros to implement test mocking. + * + * To use them, suppose that you have a function you'd like to mock + * with the signature "void writebuf(size_t n, char *buf)". You can then + * declare the function as: + * + * MOCK_DECL(void, writebuf, (size_t n, char *buf)); + * + * and implement it as: + * + * MOCK_IMPL(void + * writebuf,(size_t n, char *buf) + * { + * ... + * } + * + * For the non-testing build, this will expand simply into: + * + * void writebuf(size_t n, char *buf); + * void + * writebuf(size_t n, char *buf) + * { + * ... + * } + * + * But for the testing case, it will expand into: + * + * void writebuf__real(size_t n, char *buf); + * extern void (*writebuf)(size_t n, char *buf); + * + * void (*writebuf)(size_t n, char *buf) = writebuf__real; + * void + * writebuf__real(size_t n, char *buf) + * { + * ... + * } + * + * This is not a great mocking system! It is deliberately "the simplest + * thing that could work", and pays for its simplicity in its lack of + * features, and in its uglification of the Tor code. Replacing it with + * something clever would be a fine thing. + * + * @{ */ +#ifdef TOR_UNIT_TESTS +#define MOCK_DECL(rv, funcname, arglist) \ + rv funcname ##__real arglist; \ + extern rv(*funcname) arglist +#define MOCK_IMPL(rv, funcname, arglist) \ + rv(*funcname) arglist = funcname ##__real; \ + rv funcname ##__real arglist +#define MOCK(func, replacement) \ + do { \ + (func) = (replacement); \ + } while (0) +#define UNMOCK(func) \ + do { \ + func = func ##__real; \ + } while (0) +#else +#define MOCK_DECL(rv, funcname, arglist) \ + rv funcname arglist +#define MOCK_IMPL(rv, funcname, arglist) \ + rv funcname arglist +#endif +/** @} */ + +#endif + diff --git a/src/common/torgzip.c b/src/common/torgzip.c index 4328c63c8b..15451ee30d 100644 --- a/src/common/torgzip.c +++ b/src/common/torgzip.c @@ -68,6 +68,22 @@ is_gzip_supported(void) return gzip_is_supported; } +/** Return a string representation of the version of the currently running + * version of zlib. */ +const char * +tor_zlib_get_version_str(void) +{ + return zlibVersion(); +} + +/** Return a string representation of the version of the version of zlib +* used at compilation. */ +const char * +tor_zlib_get_header_version_str(void) +{ + return ZLIB_VERSION; +} + /** Return the 'bits' value to tell zlib to use <b>method</b>.*/ static INLINE int method_bits(compress_method_t method) diff --git a/src/common/torgzip.h b/src/common/torgzip.h index be1016445b..5db03fe6e0 100644 --- a/src/common/torgzip.h +++ b/src/common/torgzip.h @@ -32,6 +32,12 @@ tor_gzip_uncompress(char **out, size_t *out_len, int is_gzip_supported(void); +const char * +tor_zlib_get_version_str(void); + +const char * +tor_zlib_get_header_version_str(void); + compress_method_t detect_compression_method(const char *in, size_t in_len); /** Return values from tor_zlib_process; see that function's documentation for diff --git a/src/common/torlog.h b/src/common/torlog.h index 8675d7b6e7..34f70f3c00 100644 --- a/src/common/torlog.h +++ b/src/common/torlog.h @@ -13,6 +13,7 @@ #ifndef TOR_TORLOG_H #include "compat.h" +#include "testsupport.h" #ifdef HAVE_SYSLOG_H #include <syslog.h> @@ -102,6 +103,9 @@ /** This log message is not safe to send to a callback-based logger * immediately. Used as a flag, not a log domain. */ #define LD_NOCB (1u<<31) +/** This log message should not include a function name, even if it otherwise + * would. Used as a flag, not a log domain. */ +#define LD_NOFUNCNAME (1u<<30) /** Mask of zero or more log domains, OR'd together. */ typedef uint32_t log_domain_mask_t; @@ -114,12 +118,6 @@ typedef struct log_severity_list_t { log_domain_mask_t masks[LOG_DEBUG-LOG_ERR+1]; } log_severity_list_t; -#ifdef LOG_PRIVATE -/** Given a severity, yields an index into log_severity_list_t.masks to use - * for that severity. */ -#define SEVERITY_MASK_IDX(sev) ((sev) - LOG_ERR) -#endif - /** Callback type used for add_callback_log. */ typedef void (*log_callback)(int severity, uint32_t domain, const char *msg); @@ -154,9 +152,16 @@ void set_log_time_granularity(int granularity_msec); void tor_log(int severity, log_domain_mask_t domain, const char *format, ...) CHECK_PRINTF(3,4); -#if defined(__GNUC__) || defined(RUNNING_DOXYGEN) +void tor_log_err_sigsafe(const char *m, ...); +int tor_log_get_sigsafe_err_fds(const int **out); +void tor_log_update_sigsafe_err_fds(void); + +struct smartlist_t; +void tor_log_get_logfile_names(struct smartlist_t *out); + extern int log_global_min_severity_; +#if defined(__GNUC__) || defined(RUNNING_DOXYGEN) void log_fn_(int severity, log_domain_mask_t domain, const char *funcname, const char *format, ...) CHECK_PRINTF(4,5); @@ -227,6 +232,12 @@ extern const char *log_fn_function_name_; #endif /* !GNUC */ +#ifdef LOG_PRIVATE +MOCK_DECL(STATIC void, logv, (int severity, log_domain_mask_t domain, + const char *funcname, const char *suffix, const char *format, + va_list ap) CHECK_PRINTF(5,0)); +#endif + # define TOR_TORLOG_H #endif diff --git a/src/common/tortls.c b/src/common/tortls.c index c13b12fd40..4fd9bba380 100644 --- a/src/common/tortls.c +++ b/src/common/tortls.c @@ -48,9 +48,6 @@ #include "compat_libevent.h" #endif -#define CRYPTO_PRIVATE /* to import prototypes from crypto.h */ -#define TORTLS_PRIVATE - #include "crypto.h" #include "tortls.h" #include "util.h" @@ -152,6 +149,7 @@ typedef enum { TOR_TLS_ST_SENTCLOSE, TOR_TLS_ST_CLOSED, TOR_TLS_ST_RENEGOTIATE, TOR_TLS_ST_BUFFEREVENT } tor_tls_state_t; +#define tor_tls_state_bitfield_t ENUM_BF(tor_tls_state_t) /** Holds a SSL object and its associated data. Members are only * accessed from within tortls.c. @@ -162,7 +160,7 @@ struct tor_tls_t { SSL *ssl; /**< An OpenSSL SSL object. */ int socket; /**< The underlying file descriptor for this TLS connection. */ char *address; /**< An address to log when describing this connection. */ - ENUM_BF(tor_tls_state_t) state : 3; /**< The current SSL state, + tor_tls_state_bitfield_t state : 3; /**< The current SSL state, * depending on which operations * have completed successfully. */ unsigned int isServer:1; /**< True iff this is a server-side connection */ @@ -822,24 +820,24 @@ tor_cert_new(X509 *x509_cert) tor_cert_t *cert; EVP_PKEY *pkey; RSA *rsa; - int length, length2; - unsigned char *cp; + int length; + unsigned char *buf = NULL; if (!x509_cert) return NULL; - length = i2d_X509(x509_cert, NULL); + length = i2d_X509(x509_cert, &buf); cert = tor_malloc_zero(sizeof(tor_cert_t)); - if (length <= 0) { + if (length <= 0 || buf == NULL) { tor_free(cert); log_err(LD_CRYPTO, "Couldn't get length of encoded x509 certificate"); X509_free(x509_cert); return NULL; } cert->encoded_len = (size_t) length; - cp = cert->encoded = tor_malloc(length); - length2 = i2d_X509(x509_cert, &cp); - tor_assert(length2 == length); + cert->encoded = tor_malloc(length); + memcpy(cert->encoded, buf, length); + OPENSSL_free(buf); cert->cert = x509_cert; @@ -995,31 +993,6 @@ tor_tls_cert_get_key(tor_cert_t *cert) return result; } -/** Return true iff <b>a</b> and <b>b</b> represent the same public key. */ -static int -pkey_eq(EVP_PKEY *a, EVP_PKEY *b) -{ - /* We'd like to do this, but openssl 0.9.7 doesn't have it: - return EVP_PKEY_cmp(a,b) == 1; - */ - unsigned char *a_enc=NULL, *b_enc=NULL, *a_ptr, *b_ptr; - int a_len1, b_len1, a_len2, b_len2, result; - a_len1 = i2d_PublicKey(a, NULL); - b_len1 = i2d_PublicKey(b, NULL); - if (a_len1 != b_len1) - return 0; - a_ptr = a_enc = tor_malloc(a_len1); - b_ptr = b_enc = tor_malloc(b_len1); - a_len2 = i2d_PublicKey(a, &a_ptr); - b_len2 = i2d_PublicKey(b, &b_ptr); - tor_assert(a_len2 == a_len1); - tor_assert(b_len2 == b_len1); - result = tor_memeq(a_enc, b_enc, a_len1); - tor_free(a_enc); - tor_free(b_enc); - return result; -} - /** Return true iff the other side of <b>tls</b> has authenticated to us, and * the key certified in <b>cert</b> is the same as the key they used to do it. */ @@ -1035,7 +1008,7 @@ tor_tls_cert_matches_key(const tor_tls_t *tls, const tor_cert_t *cert) link_key = X509_get_pubkey(peercert); cert_key = X509_get_pubkey(cert->cert); - result = link_key && cert_key && pkey_eq(cert_key, link_key); + result = link_key && cert_key && EVP_PKEY_cmp(cert_key, link_key) == 1; X509_free(peercert); if (link_key) @@ -1345,10 +1318,12 @@ tor_tls_context_new(crypto_pk_t *identity, unsigned int key_lifetime, SSL_CTX_set_options(result->ctx, SSL_OP_ALLOW_UNSAFE_LEGACY_RENEGOTIATION); } +#ifndef OPENSSL_NO_COMP /* Don't actually allow compression; it uses ram and time, but the data * we transmit is all encrypted anyway. */ if (result->ctx->comp_methods) result->ctx->comp_methods = NULL; +#endif #ifdef SSL_MODE_RELEASE_BUFFERS SSL_CTX_set_mode(result->ctx, SSL_MODE_RELEASE_BUFFERS); #endif @@ -1438,6 +1413,21 @@ tor_tls_context_new(crypto_pk_t *identity, unsigned int key_lifetime, return NULL; } +/** Invoked when a TLS state changes: log the change at severity 'debug' */ +static void +tor_tls_debug_state_callback(const SSL *ssl, int type, int val) +{ + log_debug(LD_HANDSHAKE, "SSL %p is now in state %s [type=%d,val=%d].", + ssl, SSL_state_string_long(ssl), type, val); +} + +/* Return the name of the negotiated ciphersuite in use on <b>tls</b> */ +const char * +tor_tls_get_ciphersuite_name(tor_tls_t *tls) +{ + return SSL_get_cipher(tls->ssl); +} + #ifdef V2_HANDSHAKE_SERVER /* Here's the old V2 cipher list we sent from 0.2.1.1-alpha up to @@ -1509,13 +1499,6 @@ prune_v2_cipher_list(void) v2_cipher_list_pruned = 1; } -/* Return the name of the negotiated ciphersuite in use on <b>tls</b> */ -const char * -tor_tls_get_ciphersuite_name(tor_tls_t *tls) -{ - return SSL_get_cipher(tls->ssl); -} - /** Examine the client cipher list in <b>ssl</b>, and determine what kind of * client it is. Return one of CIPHERS_ERR, CIPHERS_V1, CIPHERS_V2, * CIPHERS_UNRESTRICTED. @@ -1614,56 +1597,6 @@ tor_tls_client_is_using_v2_ciphers(const SSL *ssl) return tor_tls_classify_client_ciphers(ssl, session->ciphers) >= CIPHERS_V2; } -#if OPENSSL_VERSION_NUMBER >= OPENSSL_V_SERIES(1,0,0) -/** Callback to get invoked on a server after we've read the list of ciphers - * the client supports, but before we pick our own ciphersuite. - * - * We can't abuse an info_cb for this, since by the time one of the - * client_hello info_cbs is called, we've already picked which ciphersuite to - * use. - * - * Technically, this function is an abuse of this callback, since the point of - * a session_secret_cb is to try to set up and/or verify a shared-secret for - * authentication on the fly. But as long as we return 0, we won't actually be - * setting up a shared secret, and all will be fine. - */ -static int -tor_tls_session_secret_cb(SSL *ssl, void *secret, int *secret_len, - STACK_OF(SSL_CIPHER) *peer_ciphers, - SSL_CIPHER **cipher, void *arg) -{ - (void) secret; - (void) secret_len; - (void) peer_ciphers; - (void) cipher; - (void) arg; - - if (tor_tls_classify_client_ciphers(ssl, peer_ciphers) == - CIPHERS_UNRESTRICTED) { - SSL_set_cipher_list(ssl, UNRESTRICTED_SERVER_CIPHER_LIST); - } - - SSL_set_session_secret_cb(ssl, NULL, NULL); - - return 0; -} -static void -tor_tls_setup_session_secret_cb(tor_tls_t *tls) -{ - SSL_set_session_secret_cb(tls->ssl, tor_tls_session_secret_cb, NULL); -} -#else -#define tor_tls_setup_session_secret_cb(tls) STMT_NIL -#endif - -/** Invoked when a TLS state changes: log the change at severity 'debug' */ -static void -tor_tls_debug_state_callback(const SSL *ssl, int type, int val) -{ - log_debug(LD_HANDSHAKE, "SSL %p is now in state %s [type=%d,val=%d].", - ssl, SSL_state_string_long(ssl), type, val); -} - /** Invoked when we're accepting a connection on <b>ssl</b>, and the connection * changes state. We use this: * <ul><li>To alter the state of the handshake partway through, so we @@ -1723,6 +1656,48 @@ tor_tls_server_info_callback(const SSL *ssl, int type, int val) } #endif +#if OPENSSL_VERSION_NUMBER >= OPENSSL_V_SERIES(1,0,0) +/** Callback to get invoked on a server after we've read the list of ciphers + * the client supports, but before we pick our own ciphersuite. + * + * We can't abuse an info_cb for this, since by the time one of the + * client_hello info_cbs is called, we've already picked which ciphersuite to + * use. + * + * Technically, this function is an abuse of this callback, since the point of + * a session_secret_cb is to try to set up and/or verify a shared-secret for + * authentication on the fly. But as long as we return 0, we won't actually be + * setting up a shared secret, and all will be fine. + */ +static int +tor_tls_session_secret_cb(SSL *ssl, void *secret, int *secret_len, + STACK_OF(SSL_CIPHER) *peer_ciphers, + SSL_CIPHER **cipher, void *arg) +{ + (void) secret; + (void) secret_len; + (void) peer_ciphers; + (void) cipher; + (void) arg; + + if (tor_tls_classify_client_ciphers(ssl, peer_ciphers) == + CIPHERS_UNRESTRICTED) { + SSL_set_cipher_list(ssl, UNRESTRICTED_SERVER_CIPHER_LIST); + } + + SSL_set_session_secret_cb(ssl, NULL, NULL); + + return 0; +} +static void +tor_tls_setup_session_secret_cb(tor_tls_t *tls) +{ + SSL_set_session_secret_cb(tls->ssl, tor_tls_session_secret_cb, NULL); +} +#else +#define tor_tls_setup_session_secret_cb(tls) STMT_NIL +#endif + /** Explain which ciphers we're missing. */ static void log_unsupported_ciphers(smartlist_t *unsupported) @@ -2362,6 +2337,7 @@ log_cert_lifetime(int severity, const X509 *cert, const char *problem) char mytime[33]; time_t now = time(NULL); struct tm tm; + size_t n; if (problem) tor_log(severity, LD_GENERAL, @@ -2387,11 +2363,17 @@ log_cert_lifetime(int severity, const X509 *cert, const char *problem) BIO_get_mem_ptr(bio, &buf); s2 = tor_strndup(buf->data, buf->length); - strftime(mytime, 32, "%b %d %H:%M:%S %Y UTC", tor_gmtime_r(&now, &tm)); - - tor_log(severity, LD_GENERAL, - "(certificate lifetime runs from %s through %s. Your time is %s.)", - s1,s2,mytime); + n = strftime(mytime, 32, "%b %d %H:%M:%S %Y UTC", tor_gmtime_r(&now, &tm)); + if (n > 0) { + tor_log(severity, LD_GENERAL, + "(certificate lifetime runs from %s through %s. Your time is %s.)", + s1,s2,mytime); + } else { + tor_log(severity, LD_GENERAL, + "(certificate lifetime runs from %s through %s. " + "Couldn't get your time.)", + s1, s2); + } end: /* Not expected to get invoked */ @@ -2606,8 +2588,8 @@ tor_tls_get_n_raw_bytes(tor_tls_t *tls, size_t *n_read, size_t *n_written) /** Return a ratio of the bytes that TLS has sent to the bytes that we've told * it to send. Used to track whether our TLS records are getting too tiny. */ -double -tls_get_write_overhead_ratio(void) +MOCK_IMPL(double, +tls_get_write_overhead_ratio,(void)) { if (total_bytes_written_over_tls == 0) return 1.0; diff --git a/src/common/tortls.h b/src/common/tortls.h index 49c488b365..a76ba3bc7a 100644 --- a/src/common/tortls.h +++ b/src/common/tortls.h @@ -13,6 +13,7 @@ #include "crypto.h" #include "compat.h" +#include "testsupport.h" /* Opaque structure to hold a TLS connection. */ typedef struct tor_tls_t tor_tls_t; @@ -95,7 +96,7 @@ void tor_tls_get_buffer_sizes(tor_tls_t *tls, size_t *rbuf_capacity, size_t *rbuf_bytes, size_t *wbuf_capacity, size_t *wbuf_bytes); -double tls_get_write_overhead_ratio(void); +MOCK_DECL(double, tls_get_write_overhead_ratio, (void)); int tor_tls_used_v1_handshake(tor_tls_t *tls); int tor_tls_received_v3_certificate(tor_tls_t *tls); diff --git a/src/common/util.c b/src/common/util.c index 5eb0f9a69b..2d7893b38a 100644 --- a/src/common/util.c +++ b/src/common/util.c @@ -24,6 +24,9 @@ #include "torint.h" #include "container.h" #include "address.h" +#include "sandbox.h" +#include "backtrace.h" +#include "util_process.h" #ifdef _WIN32 #include <io.h> @@ -94,6 +97,23 @@ #endif /* ===== + * Assertion helper. + * ===== */ +/** Helper for tor_assert: report the assertion failure. */ +void +tor_assertion_failed_(const char *fname, unsigned int line, + const char *func, const char *expr) +{ + char buf[256]; + log_err(LD_BUG, "%s:%u: %s: Assertion %s failed; aborting.", + fname, line, func, expr); + tor_snprintf(buf, sizeof(buf), + "Assertion %s failed in %s at %s:%u", + expr, func, fname, line); + log_backtrace(LOG_ERR, LD_BUG, buf); +} + +/* ===== * Memory management * ===== */ #ifdef USE_DMALLOC @@ -284,7 +304,7 @@ tor_memdup_(const void *mem, size_t len DMALLOC_PARAMS) /** As tor_memdup(), but add an extra 0 byte at the end of the resulting * memory. */ void * -tor_memdup_nulterm(const void *mem, size_t len DMALLOC_PARAMS) +tor_memdup_nulterm_(const void *mem, size_t len DMALLOC_PARAMS) { char *dup; tor_assert(len < SIZE_T_CEILING+1); @@ -879,6 +899,39 @@ tor_digest_is_zero(const char *digest) return tor_memeq(digest, ZERO_DIGEST, DIGEST_LEN); } +/** Return true if <b>string</b> is a valid 'key=[value]' string. + * "value" is optional, to indicate the empty string. Log at logging + * <b>severity</b> if something ugly happens. */ +int +string_is_key_value(int severity, const char *string) +{ + /* position of equal sign in string */ + const char *equal_sign_pos = NULL; + + tor_assert(string); + + if (strlen(string) < 2) { /* "x=" is shortest args string */ + tor_log(severity, LD_GENERAL, "'%s' is too short to be a k=v value.", + escaped(string)); + return 0; + } + + equal_sign_pos = strchr(string, '='); + if (!equal_sign_pos) { + tor_log(severity, LD_GENERAL, "'%s' is not a k=v value.", escaped(string)); + return 0; + } + + /* validate that the '=' is not in the beginning of the string. */ + if (equal_sign_pos == string) { + tor_log(severity, LD_GENERAL, "'%s' is not a valid k=v value.", + escaped(string)); + return 0; + } + + return 1; +} + /** Return true iff the DIGEST256_LEN bytes in digest are all zero. */ int tor_digest256_is_zero(const char *digest) @@ -1190,6 +1243,43 @@ escaped(const char *s) return escaped_val_; } +/** Return a newly allocated string equal to <b>string</b>, except that every + * character in <b>chars_to_escape</b> is preceded by a backslash. */ +char * +tor_escape_str_for_pt_args(const char *string, const char *chars_to_escape) +{ + char *new_string = NULL; + char *new_cp = NULL; + size_t length, new_length; + + tor_assert(string); + + length = strlen(string); + + if (!length) /* If we were given the empty string, return the same. */ + return tor_strdup(""); + /* (new_length > SIZE_MAX) => ((length * 2) + 1 > SIZE_MAX) => + (length*2 > SIZE_MAX - 1) => (length > (SIZE_MAX - 1)/2) */ + if (length > (SIZE_MAX - 1)/2) /* check for overflow */ + return NULL; + + /* this should be enough even if all characters must be escaped */ + new_length = (length * 2) + 1; + + new_string = new_cp = tor_malloc(new_length); + + while (*string) { + if (strchr(chars_to_escape, *string)) + *new_cp++ = '\\'; + + *new_cp++ = *string++; + } + + *new_cp = '\0'; /* NUL-terminate the new string */ + + return new_string; +} + /* ===== * Time * ===== */ @@ -1427,7 +1517,7 @@ void format_iso_time_nospace_usec(char *buf, const struct timeval *tv) { tor_assert(tv); - format_iso_time_nospace(buf, tv->tv_sec); + format_iso_time_nospace(buf, (time_t)tv->tv_sec); tor_snprintf(buf+ISO_TIME_LEN, 8, ".%06d", (int)tv->tv_usec); } @@ -1741,7 +1831,8 @@ file_status(const char *fname) int r; f = tor_strdup(fname); clean_name_for_stat(f); - r = stat(f, &st); + log_debug(LD_FS, "stat()ing %s", f); + r = stat(sandbox_intern_string(f), &st); tor_free(f); if (r) { if (errno == ENOENT) { @@ -1781,7 +1872,7 @@ check_private_dir(const char *dirname, cpd_check_t check, char *f; #ifndef _WIN32 int mask; - struct passwd *pw = NULL; + const struct passwd *pw = NULL; uid_t running_uid; gid_t running_gid; #else @@ -1791,7 +1882,8 @@ check_private_dir(const char *dirname, cpd_check_t check, tor_assert(dirname); f = tor_strdup(dirname); clean_name_for_stat(f); - r = stat(f, &st); + log_debug(LD_FS, "stat()ing %s", f); + r = stat(sandbox_intern_string(f), &st); tor_free(f); if (r) { if (errno != ENOENT) { @@ -1827,7 +1919,7 @@ check_private_dir(const char *dirname, cpd_check_t check, if (effective_user) { /* Look up the user and group information. * If we have a problem, bail out. */ - pw = getpwnam(effective_user); + pw = tor_getpwnam(effective_user); if (pw == NULL) { log_warn(LD_CONFIG, "Error setting configured user: %s not found", effective_user); @@ -1841,13 +1933,13 @@ check_private_dir(const char *dirname, cpd_check_t check, } if (st.st_uid != running_uid) { - struct passwd *pw = NULL; + const struct passwd *pw = NULL; char *process_ownername = NULL; - pw = getpwuid(running_uid); + pw = tor_getpwuid(running_uid); process_ownername = pw ? tor_strdup(pw->pw_name) : tor_strdup("<unknown>"); - pw = getpwuid(st.st_uid); + pw = tor_getpwuid(st.st_uid); log_warn(LD_FS, "%s is not owned by this user (%s, %d) but by " "%s (%d). Perhaps you are running Tor as the wrong user?", @@ -1913,7 +2005,8 @@ write_str_to_file(const char *fname, const char *str, int bin) #ifdef _WIN32 if (!bin && strchr(str, '\r')) { log_warn(LD_BUG, - "We're writing a text string that already contains a CR."); + "We're writing a text string that already contains a CR to %s", + escaped(fname)); } #endif return write_bytes_to_file(fname, str, strlen(str), bin); @@ -1977,8 +2070,10 @@ start_writing_to_file(const char *fname, int open_flags, int mode, open_flags &= ~O_EXCL; new_file->rename_on_close = 1; } +#if O_BINARY != 0 if (open_flags & O_BINARY) new_file->binary = 1; +#endif new_file->fd = tor_open_cloexec(open_name, open_flags, mode); if (new_file->fd < 0) { @@ -2050,6 +2145,7 @@ static int finish_writing_to_file_impl(open_file_t *file_data, int abort_write) { int r = 0; + tor_assert(file_data && file_data->filename); if (file_data->stdio_file) { if (fclose(file_data->stdio_file)) { @@ -2066,7 +2162,13 @@ finish_writing_to_file_impl(open_file_t *file_data, int abort_write) if (file_data->rename_on_close) { tor_assert(file_data->tempname && file_data->filename); if (abort_write) { - unlink(file_data->tempname); + int res = unlink(file_data->tempname); + if (res != 0) { + /* We couldn't unlink and we'll leave a mess behind */ + log_warn(LD_FS, "Failed to unlink %s: %s", + file_data->tempname, strerror(errno)); + r = -1; + } } else { tor_assert(strcmp(file_data->filename, file_data->tempname)); if (replace_file(file_data->tempname, file_data->filename)) { @@ -2132,12 +2234,20 @@ write_chunks_to_file_impl(const char *fname, const smartlist_t *chunks, return -1; } -/** Given a smartlist of sized_chunk_t, write them atomically to a file - * <b>fname</b>, overwriting or creating the file as necessary. */ +/** Given a smartlist of sized_chunk_t, write them to a file + * <b>fname</b>, overwriting or creating the file as necessary. + * If <b>no_tempfile</b> is 0 then the file will be written + * atomically. */ int -write_chunks_to_file(const char *fname, const smartlist_t *chunks, int bin) +write_chunks_to_file(const char *fname, const smartlist_t *chunks, int bin, + int no_tempfile) { int flags = OPEN_FLAGS_REPLACE|(bin?O_BINARY:O_TEXT); + + if (no_tempfile) { + /* O_APPEND stops write_chunks_to_file from using tempfiles */ + flags |= O_APPEND; + } return write_chunks_to_file_impl(fname, chunks, flags); } @@ -2158,9 +2268,9 @@ write_bytes_to_file_impl(const char *fname, const char *str, size_t len, /** As write_str_to_file, but does not assume a NUL-terminated * string. Instead, we write <b>len</b> bytes, starting at <b>str</b>. */ -int -write_bytes_to_file(const char *fname, const char *str, size_t len, - int bin) +MOCK_IMPL(int, +write_bytes_to_file,(const char *fname, const char *str, size_t len, + int bin)) { return write_bytes_to_file_impl(fname, str, len, OPEN_FLAGS_REPLACE|(bin?O_BINARY:O_TEXT)); @@ -2927,7 +3037,7 @@ tor_vsscanf(const char *buf, const char *pattern, va_list ap) /** Minimal sscanf replacement: parse <b>buf</b> according to <b>pattern</b> * and store the results in the corresponding argument fields. Differs from * sscanf in that: - * <ul><li>It only handles %u, %lu, %x, %lx, %<NUM>s, %d, %ld, %lf, and %c. + * <ul><li>It only handles %u, %lu, %x, %lx, %[NUM]s, %d, %ld, %lf, and %c. * <li>It only handles decimal inputs for %lf. (12.3, not 1.23e1) * <li>It does not handle arbitrarily long widths. * <li>Numbers do not consume any space characters. @@ -3022,9 +3132,10 @@ tor_listdir(const char *dirname) FindClose(handle); tor_free(pattern); #else + const char *prot_dname = sandbox_intern_string(dirname); DIR *d; struct dirent *de; - if (!(d = opendir(dirname))) + if (!(d = opendir(prot_dname))) return NULL; result = smartlist_new(); @@ -3320,14 +3431,59 @@ tor_join_win_cmdline(const char *argv[]) return joined_argv; } +/* As format_{hex,dex}_number_sigsafe, but takes a <b>radix</b> argument + * in range 2..16 inclusive. */ +static int +format_number_sigsafe(unsigned long x, char *buf, int buf_len, + unsigned int radix) +{ + unsigned long tmp; + int len; + char *cp; + + /* NOT tor_assert. This needs to be safe to run from within a signal handler, + * and from within the 'tor_assert() has failed' code. */ + if (radix < 2 || radix > 16) + return 0; + + /* Count how many digits we need. */ + tmp = x; + len = 1; + while (tmp >= radix) { + tmp /= radix; + ++len; + } + + /* Not long enough */ + if (!buf || len >= buf_len) + return 0; + + cp = buf + len; + *cp = '\0'; + do { + unsigned digit = (unsigned) (x % radix); + tor_assert(cp > buf); + --cp; + *cp = "0123456789ABCDEF"[digit]; + x /= radix; + } while (x); + + /* NOT tor_assert; see above. */ + if (cp != buf) { + abort(); + } + + return len; +} + /** - * Helper function to output hex numbers, called by - * format_helper_exit_status(). This writes the hexadecimal digits of x into - * buf, up to max_len digits, and returns the actual number of digits written. - * If there is insufficient space, it will write nothing and return 0. + * Helper function to output hex numbers from within a signal handler. * - * This function DOES NOT add a terminating NUL character to its output: be - * careful! + * Writes the nul-terminated hexadecimal digits of <b>x</b> into a buffer + * <b>buf</b> of size <b>buf_len</b>, and return the actual number of digits + * written, not counting the terminal NUL. + * + * If there is insufficient space, write nothing and return 0. * * This accepts an unsigned int because format_helper_exit_status() needs to * call it with a signed int and an unsigned char, and since the C standard @@ -3342,46 +3498,19 @@ tor_join_win_cmdline(const char *argv[]) * arbitrary C functions. */ int -format_hex_number_for_helper_exit_status(unsigned int x, char *buf, - int max_len) +format_hex_number_sigsafe(unsigned long x, char *buf, int buf_len) { - int len; - unsigned int tmp; - char *cur; - - /* Sanity check */ - if (!buf || max_len <= 0) - return 0; - - /* How many chars do we need for x? */ - if (x > 0) { - len = 0; - tmp = x; - while (tmp > 0) { - tmp >>= 4; - ++len; - } - } else { - len = 1; - } - - /* Bail if we would go past the end of the buffer */ - if (len > max_len) - return 0; - - /* Point to last one */ - cur = buf + len - 1; - - /* Convert x to hex */ - do { - *cur-- = "0123456789ABCDEF"[x & 0xf]; - x >>= 4; - } while (x != 0 && cur >= buf); + return format_number_sigsafe(x, buf, buf_len, 16); +} - /* Return len */ - return len; +/** As format_hex_number_sigsafe, but format the number in base 10. */ +int +format_dec_number_sigsafe(unsigned long x, char *buf, int buf_len) +{ + return format_number_sigsafe(x, buf, buf_len, 10); } +#ifndef _WIN32 /** Format <b>child_state</b> and <b>saved_errno</b> as a hex string placed in * <b>hex_errno</b>. Called between fork and _exit, so must be signal-handler * safe. @@ -3397,7 +3526,7 @@ format_hex_number_for_helper_exit_status(unsigned int x, char *buf, * On success return the number of characters added to hex_errno, not counting * the terminating NUL; return -1 on error. */ -int +STATIC int format_helper_exit_status(unsigned char child_state, int saved_errno, char *hex_errno) { @@ -3428,8 +3557,8 @@ format_helper_exit_status(unsigned char child_state, int saved_errno, cur = hex_errno; /* Emit child_state */ - written = format_hex_number_for_helper_exit_status(child_state, - cur, left); + written = format_hex_number_sigsafe(child_state, cur, left); + if (written <= 0) goto err; @@ -3458,8 +3587,7 @@ format_helper_exit_status(unsigned char child_state, int saved_errno, } /* Emit unsigned_errno */ - written = format_hex_number_for_helper_exit_status(unsigned_errno, - cur, left); + written = format_hex_number_sigsafe(unsigned_errno, cur, left); if (written <= 0) goto err; @@ -3490,6 +3618,7 @@ format_helper_exit_status(unsigned char child_state, int saved_errno, done: return res; } +#endif /* Maximum number of file descriptors, if we cannot get it via sysconf() */ #define DEFAULT_MAX_FD 256 @@ -3501,13 +3630,7 @@ tor_terminate_process(process_handle_t *process_handle) { #ifdef _WIN32 if (tor_get_exit_code(process_handle, 0, NULL) == PROCESS_EXIT_RUNNING) { - HANDLE handle; - /* If the signal is outside of what GenerateConsoleCtrlEvent can use, - attempt to open and terminate the process. */ - handle = OpenProcess(PROCESS_ALL_ACCESS, FALSE, - process_handle->pid.dwProcessId); - if (!handle) - return -1; + HANDLE handle = process_handle->pid.hProcess; if (!TerminateProcess(handle, 0)) return -1; @@ -3515,7 +3638,10 @@ tor_terminate_process(process_handle_t *process_handle) return 0; } #else /* Unix */ - return kill(process_handle->pid, SIGTERM); + if (process_handle->waitpid_cb) { + /* We haven't got a waitpid yet, so we can just kill off the process. */ + return kill(process_handle->pid, SIGTERM); + } #endif return -1; @@ -3564,6 +3690,23 @@ process_handle_new(void) return out; } +#ifndef _WIN32 +/** Invoked when a process that we've launched via tor_spawn_background() has + * been found to have terminated. + */ +static void +process_handle_waitpid_cb(int status, void *arg) +{ + process_handle_t *process_handle = arg; + + process_handle->waitpid_exit_status = status; + clear_waitpid_callback(process_handle->waitpid_cb); + if (process_handle->status == PROCESS_STATUS_RUNNING) + process_handle->status = PROCESS_STATUS_NOTRUNNING; + process_handle->waitpid_cb = 0; +} +#endif + /** * @name child-process states * @@ -3685,7 +3828,7 @@ tor_spawn_background(const char *const filename, const char **argv, TRUE, // handles are inherited /*(TODO: set CREATE_NEW CONSOLE/PROCESS_GROUP to make GetExitCodeProcess() * work?) */ - 0, // creation flags + CREATE_NO_WINDOW, // creation flags (env==NULL) ? NULL : env->windows_environment_block, NULL, // use parent's current directory &siStartInfo, // STARTUPINFO pointer @@ -3880,6 +4023,10 @@ tor_spawn_background(const char *const filename, const char **argv, strerror(errno)); } + process_handle->waitpid_cb = set_waitpid_callback(pid, + process_handle_waitpid_cb, + process_handle); + process_handle->stderr_pipe = stderr_pipe[0]; retval = close(stderr_pipe[1]); @@ -3906,9 +4053,9 @@ tor_spawn_background(const char *const filename, const char **argv, * <b>process_handle</b>. * If <b>also_terminate_process</b> is true, also terminate the * process of the process handle. */ -void -tor_process_handle_destroy(process_handle_t *process_handle, - int also_terminate_process) +MOCK_IMPL(void, +tor_process_handle_destroy,(process_handle_t *process_handle, + int also_terminate_process)) { if (!process_handle) return; @@ -3944,6 +4091,8 @@ tor_process_handle_destroy(process_handle_t *process_handle, if (process_handle->stderr_handle) fclose(process_handle->stderr_handle); + + clear_waitpid_callback(process_handle->waitpid_cb); #endif memset(process_handle, 0x0f, sizeof(process_handle_t)); @@ -3961,7 +4110,7 @@ tor_process_handle_destroy(process_handle_t *process_handle, * probably not work in Tor, because waitpid() is called in main.c to reap any * terminated child processes.*/ int -tor_get_exit_code(const process_handle_t *process_handle, +tor_get_exit_code(process_handle_t *process_handle, int block, int *exit_code) { #ifdef _WIN32 @@ -4001,7 +4150,20 @@ tor_get_exit_code(const process_handle_t *process_handle, int stat_loc; int retval; - retval = waitpid(process_handle->pid, &stat_loc, block?0:WNOHANG); + if (process_handle->waitpid_cb) { + /* We haven't processed a SIGCHLD yet. */ + retval = waitpid(process_handle->pid, &stat_loc, block?0:WNOHANG); + if (retval == process_handle->pid) { + clear_waitpid_callback(process_handle->waitpid_cb); + process_handle->waitpid_cb = NULL; + process_handle->waitpid_exit_status = stat_loc; + } + } else { + /* We already got a SIGCHLD for this process, and handled it. */ + retval = process_handle->pid; + stat_loc = process_handle->waitpid_exit_status; + } + if (!block && 0 == retval) { /* Process has not exited */ return PROCESS_EXIT_RUNNING; @@ -4412,14 +4574,38 @@ stream_status_to_string(enum stream_status stream_status) } } +/* DOCDOC */ +static void +log_portfw_spawn_error_message(const char *buf, + const char *executable, int *child_status) +{ + /* Parse error message */ + int retval, child_state, saved_errno; + retval = tor_sscanf(buf, SPAWN_ERROR_MESSAGE "%x/%x", + &child_state, &saved_errno); + if (retval == 2) { + log_warn(LD_GENERAL, + "Failed to start child process \"%s\" in state %d: %s", + executable, child_state, strerror(saved_errno)); + if (child_status) + *child_status = 1; + } else { + /* Failed to parse message from child process, log it as a + warning */ + log_warn(LD_GENERAL, + "Unexpected message from port forwarding helper \"%s\": %s", + executable, buf); + } +} + #ifdef _WIN32 /** Return a smartlist containing lines outputted from * <b>handle</b>. Return NULL on error, and set * <b>stream_status_out</b> appropriately. */ -smartlist_t * -tor_get_lines_from_handle(HANDLE *handle, - enum stream_status *stream_status_out) +MOCK_IMPL(smartlist_t *, +tor_get_lines_from_handle, (HANDLE *handle, + enum stream_status *stream_status_out)) { int pos; char stdout_buf[600] = {0}; @@ -4507,8 +4693,9 @@ log_from_handle(HANDLE *pipe, int severity) /** Return a smartlist containing lines outputted from * <b>handle</b>. Return NULL on error, and set * <b>stream_status_out</b> appropriately. */ -smartlist_t * -tor_get_lines_from_handle(FILE *handle, enum stream_status *stream_status_out) +MOCK_IMPL(smartlist_t *, +tor_get_lines_from_handle, (FILE *handle, + enum stream_status *stream_status_out)) { enum stream_status stream_status; char stdout_buf[400]; @@ -4558,23 +4745,7 @@ log_from_pipe(FILE *stream, int severity, const char *executable, /* Check if buf starts with SPAWN_ERROR_MESSAGE */ if (strcmpstart(buf, SPAWN_ERROR_MESSAGE) == 0) { - /* Parse error message */ - int retval, child_state, saved_errno; - retval = tor_sscanf(buf, SPAWN_ERROR_MESSAGE "%x/%x", - &child_state, &saved_errno); - if (retval == 2) { - log_warn(LD_GENERAL, - "Failed to start child process \"%s\" in state %d: %s", - executable, child_state, strerror(saved_errno)); - if (child_status) - *child_status = 1; - } else { - /* Failed to parse message from child process, log it as a - warning */ - log_warn(LD_GENERAL, - "Unexpected message from port forwarding helper \"%s\": %s", - executable, buf); - } + log_portfw_spawn_error_message(buf, executable, child_status); } else { log_fn(severity, LD_GENERAL, "Port forwarding helper says: %s", buf); } @@ -4652,7 +4823,7 @@ get_string_from_pipe(FILE *stream, char *buf_out, size_t count) /** Parse a <b>line</b> from tor-fw-helper and issue an appropriate * log message to our user. */ static void -handle_fw_helper_line(const char *line) +handle_fw_helper_line(const char *executable, const char *line) { smartlist_t *tokens = smartlist_new(); char *message = NULL; @@ -4663,6 +4834,19 @@ handle_fw_helper_line(const char *line) int port = 0; int success = 0; + if (strcmpstart(line, SPAWN_ERROR_MESSAGE) == 0) { + /* We need to check for SPAWN_ERROR_MESSAGE again here, since it's + * possible that it got sent after we tried to read it in log_from_pipe. + * + * XXX Ideally, we should be using one of stdout/stderr for the real + * output, and one for the output of the startup code. We used to do that + * before cd05f35d2c. + */ + int child_status; + log_portfw_spawn_error_message(line, executable, &child_status); + goto done; + } + smartlist_split_string(tokens, line, NULL, SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, -1); @@ -4742,7 +4926,8 @@ handle_fw_helper_line(const char *line) /** Read what tor-fw-helper has to say in its stdout and handle it * appropriately */ static int -handle_fw_helper_output(process_handle_t *process_handle) +handle_fw_helper_output(const char *executable, + process_handle_t *process_handle) { smartlist_t *fw_helper_output = NULL; enum stream_status stream_status = 0; @@ -4757,7 +4942,7 @@ handle_fw_helper_output(process_handle_t *process_handle) /* Handle the lines we got: */ SMARTLIST_FOREACH_BEGIN(fw_helper_output, char *, line) { - handle_fw_helper_line(line); + handle_fw_helper_line(executable, line); tor_free(line); } SMARTLIST_FOREACH_END(line); @@ -4872,7 +5057,7 @@ tor_check_port_forwarding(const char *filename, stderr_status = log_from_pipe(child_handle->stderr_handle, LOG_INFO, filename, &retval); #endif - if (handle_fw_helper_output(child_handle) < 0) { + if (handle_fw_helper_output(filename, child_handle) < 0) { log_warn(LD_GENERAL, "Failed to handle fw helper output."); stdout_status = -1; retval = -1; diff --git a/src/common/util.h b/src/common/util.h index 73daa6e2a1..97367a9a7b 100644 --- a/src/common/util.h +++ b/src/common/util.h @@ -15,6 +15,7 @@ #include "torint.h" #include "compat.h" #include "di_ops.h" +#include "testsupport.h" #include <stdio.h> #include <stdlib.h> #ifdef _WIN32 @@ -47,13 +48,13 @@ /** Like assert(3), but send assertion failures to the log as well as to * stderr. */ #define tor_assert(expr) STMT_BEGIN \ - if (PREDICT_UNLIKELY(!(expr))) { \ - log_err(LD_BUG, "%s:%d: %s: Assertion %s failed; aborting.", \ - SHORT_FILE__, __LINE__, __func__, #expr); \ - fprintf(stderr,"%s:%d %s: Assertion %s failed; aborting.\n", \ - SHORT_FILE__, __LINE__, __func__, #expr); \ - abort(); \ - } STMT_END + if (PREDICT_UNLIKELY(!(expr))) { \ + tor_assertion_failed_(SHORT_FILE__, __LINE__, __func__, #expr); \ + abort(); \ + } STMT_END + +void tor_assertion_failed_(const char *fname, unsigned int line, + const char *func, const char *expr); /* If we're building with dmalloc, we want all of our memory allocation * functions to take an extra file/line pair of arguments. If not, not. @@ -222,23 +223,22 @@ const char *find_whitespace_eos(const char *s, const char *eos); const char *find_str_at_start_of_line(const char *haystack, const char *needle); int string_is_C_identifier(const char *string); +int string_is_key_value(int severity, const char *string); int tor_mem_is_zero(const char *mem, size_t len); int tor_digest_is_zero(const char *digest); int tor_digest256_is_zero(const char *digest); char *esc_for_log(const char *string) ATTR_MALLOC; const char *escaped(const char *string); + +char *tor_escape_str_for_pt_args(const char *string, + const char *chars_to_escape); + struct smartlist_t; -int tor_vsscanf(const char *buf, const char *pattern, va_list ap) -#ifdef __GNUC__ - __attribute__((format(scanf, 2, 0))) -#endif - ; +int tor_vsscanf(const char *buf, const char *pattern, va_list ap) \ + CHECK_SCANF(2, 0); int tor_sscanf(const char *buf, const char *pattern, ...) -#ifdef __GNUC__ - __attribute__((format(scanf, 2, 3))) -#endif - ; + CHECK_SCANF(2, 3); void smartlist_add_asprintf(struct smartlist_t *sl, const char *pattern, ...) CHECK_PRINTF(2, 3); @@ -356,8 +356,9 @@ FILE *fdopen_file(open_file_t *file_data); int finish_writing_to_file(open_file_t *file_data); int abort_writing_to_file(open_file_t *file_data); int write_str_to_file(const char *fname, const char *str, int bin); -int write_bytes_to_file(const char *fname, const char *str, size_t len, - int bin); +MOCK_DECL(int, +write_bytes_to_file,(const char *fname, const char *str, size_t len, + int bin)); /** An ad-hoc type to hold a string of characters and a count; used by * write_chunks_to_file. */ typedef struct sized_chunk_t { @@ -365,7 +366,7 @@ typedef struct sized_chunk_t { size_t len; } sized_chunk_t; int write_chunks_to_file(const char *fname, const struct smartlist_t *chunks, - int bin); + int bin, int no_tempfile); int append_bytes_to_file(const char *fname, const char *str, size_t len, int bin); int write_bytes_to_new_file(const char *fname, const char *str, size_t len, @@ -445,6 +446,7 @@ void set_environment_variable_in_smartlist(struct smartlist_t *env_vars, #define PROCESS_STATUS_ERROR -1 #ifdef UTIL_PRIVATE +struct waitpid_callback_t; /** Structure to represent the state of a process with which Tor is * communicating. The contents of this structure are private to util.c */ struct process_handle_t { @@ -460,6 +462,12 @@ struct process_handle_t { FILE *stdout_handle; FILE *stderr_handle; pid_t pid; + /** If the process has not given us a SIGCHLD yet, this has the + * waitpid_callback_t that gets invoked once it has. Otherwise this + * contains NULL. */ + struct waitpid_callback_t *waitpid_cb; + /** The exit status reported by waitpid. */ + int waitpid_exit_status; #endif // _WIN32 }; #endif @@ -468,7 +476,7 @@ struct process_handle_t { #define PROCESS_EXIT_RUNNING 1 #define PROCESS_EXIT_EXITED 0 #define PROCESS_EXIT_ERROR -1 -int tor_get_exit_code(const process_handle_t *process_handle, +int tor_get_exit_code(process_handle_t *process_handle, int block, int *exit_code); int tor_split_lines(struct smartlist_t *sl, char *buf, int len); #ifdef _WIN32 @@ -493,18 +501,21 @@ FILE *tor_process_get_stdout_pipe(process_handle_t *process_handle); #endif #ifdef _WIN32 -struct smartlist_t * -tor_get_lines_from_handle(HANDLE *handle, - enum stream_status *stream_status); +MOCK_DECL(struct smartlist_t *, +tor_get_lines_from_handle,(HANDLE *handle, + enum stream_status *stream_status)); #else -struct smartlist_t * -tor_get_lines_from_handle(FILE *handle, - enum stream_status *stream_status); +MOCK_DECL(struct smartlist_t *, +tor_get_lines_from_handle,(FILE *handle, + enum stream_status *stream_status)); #endif -int tor_terminate_process(process_handle_t *process_handle); -void tor_process_handle_destroy(process_handle_t *process_handle, - int also_terminate_process); +int +tor_terminate_process(process_handle_t *process_handle); + +MOCK_DECL(void, +tor_process_handle_destroy,(process_handle_t *process_handle, + int also_terminate_process)); /* ===== Insecure rng */ typedef struct tor_weak_rng_t { @@ -520,12 +531,14 @@ int32_t tor_weak_random_range(tor_weak_rng_t *rng, int32_t top); * <b>n</b> */ #define tor_weak_random_one_in_n(rng, n) (0==tor_weak_random_range((rng),(n))) +int format_hex_number_sigsafe(unsigned long x, char *buf, int max_len); +int format_dec_number_sigsafe(unsigned long x, char *buf, int max_len); + #ifdef UTIL_PRIVATE /* Prototypes for private functions only used by util.c (and unit tests) */ -int format_hex_number_for_helper_exit_status(unsigned int x, char *buf, - int max_len); -int format_helper_exit_status(unsigned char child_state, +#ifndef _WIN32 +STATIC int format_helper_exit_status(unsigned char child_state, int saved_errno, char *hex_errno); /* Space for hex values of child state, a slash, saved_errno (with @@ -534,7 +547,11 @@ int format_helper_exit_status(unsigned char child_state, 1 + sizeof(int) * 2 + 1) #endif +#endif + const char *libor_get_digests(void); +#define ARRAY_LENGTH(x) (sizeof(x)) / sizeof(x[0]) + #endif diff --git a/src/common/util_process.c b/src/common/util_process.c new file mode 100644 index 0000000000..d6ef590162 --- /dev/null +++ b/src/common/util_process.c @@ -0,0 +1,158 @@ +/* Copyright (c) 2003-2004, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2013, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file util_process.c + * \brief utility functions for launching processes and checking their + * status. These functions are kept separately from procmon so that they + * won't require linking against libevent. + **/ + +#include "orconfig.h" + +#ifdef HAVE_SYS_TYPES_H +#include <sys/types.h> +#endif +#ifdef HAVE_SYS_WAIT_H +#include <sys/wait.h> +#endif + +#include "compat.h" +#include "util.h" +#include "torlog.h" +#include "util_process.h" +#include "ht.h" + +/* ================================================== */ +/* Convenience structures for handlers for waitpid(). + * + * The tor_process_monitor*() code above doesn't use them, since it is for + * monitoring a non-child process. + */ + +#ifndef _WIN32 + +/** Mapping from a PID to a userfn/userdata pair. */ +struct waitpid_callback_t { + HT_ENTRY(waitpid_callback_t) node; + pid_t pid; + + void (*userfn)(int, void *userdata); + void *userdata; + + unsigned running; +}; + +static INLINE unsigned int +process_map_entry_hash_(const waitpid_callback_t *ent) +{ + return (unsigned) ent->pid; +} + +static INLINE unsigned int +process_map_entries_eq_(const waitpid_callback_t *a, + const waitpid_callback_t *b) +{ + return a->pid == b->pid; +} + +static HT_HEAD(process_map, waitpid_callback_t) process_map = HT_INITIALIZER(); + +HT_PROTOTYPE(process_map, waitpid_callback_t, node, process_map_entry_hash_, + process_map_entries_eq_); +HT_GENERATE(process_map, waitpid_callback_t, node, process_map_entry_hash_, + process_map_entries_eq_, 0.6, malloc, realloc, free); + +/** + * Begin monitoring the child pid <b>pid</b> to see if we get a SIGCHLD for + * it. If we eventually do, call <b>fn</b>, passing it the exit status (as + * yielded by waitpid) and the pointer <b>arg</b>. + * + * To cancel this, or clean up after it has triggered, call + * clear_waitpid_callback(). + */ +waitpid_callback_t * +set_waitpid_callback(pid_t pid, void (*fn)(int, void *), void *arg) +{ + waitpid_callback_t *old_ent; + waitpid_callback_t *ent = tor_malloc_zero(sizeof(waitpid_callback_t)); + ent->pid = pid; + ent->userfn = fn; + ent->userdata = arg; + ent->running = 1; + + old_ent = HT_REPLACE(process_map, &process_map, ent); + if (old_ent) { + log_warn(LD_BUG, "Replaced a waitpid monitor on pid %u. That should be " + "impossible.", (unsigned) pid); + old_ent->running = 0; + } + + return ent; +} + +/** + * Cancel a waitpid_callback_t, or clean up after one has triggered. Releases + * all storage held by <b>ent</b>. + */ +void +clear_waitpid_callback(waitpid_callback_t *ent) +{ + waitpid_callback_t *old_ent; + if (ent == NULL) + return; + + if (ent->running) { + old_ent = HT_REMOVE(process_map, &process_map, ent); + if (old_ent != ent) { + log_warn(LD_BUG, "Couldn't remove waitpid monitor for pid %u.", + (unsigned) ent->pid); + return; + } + } + + tor_free(ent); +} + +/** Helper: find the callack for <b>pid</b>; if there is one, run it, + * reporting the exit status as <b>status</b>. */ +static void +notify_waitpid_callback_by_pid(pid_t pid, int status) +{ + waitpid_callback_t search, *ent; + + search.pid = pid; + ent = HT_REMOVE(process_map, &process_map, &search); + if (!ent || !ent->running) { + log_info(LD_GENERAL, "Child process %u has exited; no callback was " + "registered", (unsigned)pid); + return; + } + + log_info(LD_GENERAL, "Child process %u has exited; running callback.", + (unsigned)pid); + + ent->running = 0; + ent->userfn(status, ent->userdata); +} + +/** Use waitpid() to wait for all children that have exited, and invoke any + * callbacks registered for them. */ +void +notify_pending_waitpid_callbacks(void) +{ + /* I was going to call this function reap_zombie_children(), but + * that makes it sound way more exciting than it really is. */ + pid_t child; + int status = 0; + + while ((child = waitpid(-1, &status, WNOHANG)) > 0) { + notify_waitpid_callback_by_pid(child, status); + status = 0; /* should be needless */ + } +} + +#endif + diff --git a/src/common/util_process.h b/src/common/util_process.h new file mode 100644 index 0000000000..0b268b85d3 --- /dev/null +++ b/src/common/util_process.h @@ -0,0 +1,26 @@ +/* Copyright (c) 2011-2013, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file util_process.h + * \brief Headers for util_process.c + **/ + +#ifndef TOR_UTIL_PROCESS_H +#define TOR_UTIL_PROCESS_H + +#ifndef _WIN32 +/** A callback structure waiting for us to get a SIGCHLD informing us that a + * PID has been closed. Created by set_waitpid_callback. Cancelled or cleaned- + * up from clear_waitpid_callback(). Do not access outside of the main thread; + * do not access from inside a signal handler. */ +typedef struct waitpid_callback_t waitpid_callback_t; + +waitpid_callback_t *set_waitpid_callback(pid_t pid, + void (*fn)(int, void *), void *arg); +void clear_waitpid_callback(waitpid_callback_t *ent); +void notify_pending_waitpid_callbacks(void); +#endif + +#endif + diff --git a/src/config/README.geoip b/src/config/README.geoip deleted file mode 100644 index 8520501405..0000000000 --- a/src/config/README.geoip +++ /dev/null @@ -1,90 +0,0 @@ -README.geoip -- information on the IP-to-country-code file shipped with tor -=========================================================================== - -The IP-to-country-code file in src/config/geoip is based on MaxMind's -GeoLite Country database with the following modifications: - - - Those "A1" ("Anonymous Proxy") entries lying inbetween two entries with - the same country code are automatically changed to that country code. - These changes can be overriden by specifying a different country code - in src/config/geoip-manual. - - - Other "A1" entries are replaced with country codes specified in - src/config/geoip-manual, or are left as is if there is no corresponding - entry in that file. Even non-"A1" entries can be modified by adding a - replacement entry to src/config/geoip-manual. Handle with care. - - -1. Updating the geoip file from a MaxMind database file -------------------------------------------------------- - -Download the most recent MaxMind GeoLite Country database: -http://geolite.maxmind.com/download/geoip/database/GeoIPCountryCSV.zip - -Run `python deanonymind.py` in the local directory. Review the output to -learn about applied automatic/manual changes and watch out for any -warnings. - -Possibly edit geoip-manual to make more/fewer/different manual changes and -re-run `python deanonymind.py`. - -When done, prepend the new geoip file with a comment like this: - - # Last updated based on $DATE Maxmind GeoLite Country - # See README.geoip for details on the conversion. - - -2. Verifying automatic and manual changes using diff ----------------------------------------------------- - -To unzip the original MaxMind file and look at the automatic changes, run: - - unzip GeoIPCountryCSV.zip - diff -U1 GeoIPCountryWhois.csv AutomaticGeoIPCountryWhois.csv - -To look at subsequent manual changes, run: - - diff -U1 AutomaticGeoIPCountryWhois.csv ManualGeoIPCountryWhois.csv - -To manually generate the geoip file and compare it to the automatically -created one, run: - - cut -d, -f3-5 < ManualGeoIPCountryWhois.csv | sed 's/"//g' > mygeoip - diff -U1 geoip mygeoip - - -3. Verifying automatic and manual changes using blockfinder ------------------------------------------------------------ - -Blockfinder is a powerful tool to handle multiple IP-to-country data -sources. Blockfinder has a function to specify a country code and compare -conflicting country code assignments in different data sources. - -We can use blockfinder to compare A1 entries in the original MaxMind file -with the same or overlapping blocks in the file generated above and in the -RIR delegation files: - - git clone https://github.com/ioerror/blockfinder - cd blockfinder/ - python blockfinder -i - python blockfinder -r ../GeoIPCountryWhois.csv - python blockfinder -r ../ManualGeoIPCountryWhois.csv - python blockfinder -p A1 > A1-comparison.txt - -The output marks conflicts between assignments using either '*' in case of -two different opinions or '#' for three or more different opinions about -the country code for a given block. - -The '*' conflicts are most likely harmless, because there will always be -at least two opinions with the original MaxMind file saying A1 and the -other two sources saying something more meaningful. - -However, watch out for '#' conflicts. In these cases, the original -MaxMind file ("A1"), the updated MaxMind file (hopefully the correct -country code), and the RIR delegation files (some other country code) all -disagree. - -There are perfectly valid cases where the updated MaxMind file and the RIR -delegation files don't agree. But each of those cases must be verified -manually. - diff --git a/src/config/deanonymind.py b/src/config/deanonymind.py deleted file mode 100755 index c86dadca99..0000000000 --- a/src/config/deanonymind.py +++ /dev/null @@ -1,194 +0,0 @@ -#!/usr/bin/env python -import optparse -import os -import sys -import zipfile - -""" -Take a MaxMind GeoLite Country database as input and replace A1 entries -with the country code and name of the preceding entry iff the preceding -(subsequent) entry ends (starts) directly before (after) the A1 entry and -both preceding and subsequent entries contain the same country code. - -Then apply manual changes, either replacing A1 entries that could not be -replaced automatically or overriding previously made automatic changes. -""" - -def main(): - options = parse_options() - assignments = read_file(options.in_maxmind) - assignments = apply_automatic_changes(assignments) - write_file(options.out_automatic, assignments) - manual_assignments = read_file(options.in_manual, must_exist=False) - assignments = apply_manual_changes(assignments, manual_assignments) - write_file(options.out_manual, assignments) - write_file(options.out_geoip, assignments, long_format=False) - -def parse_options(): - parser = optparse.OptionParser() - parser.add_option('-i', action='store', dest='in_maxmind', - default='GeoIPCountryCSV.zip', metavar='FILE', - help='use the specified MaxMind GeoLite Country .zip or .csv ' - 'file as input [default: %default]') - parser.add_option('-g', action='store', dest='in_manual', - default='geoip-manual', metavar='FILE', - help='use the specified .csv file for manual changes or to ' - 'override automatic changes [default: %default]') - parser.add_option('-a', action='store', dest='out_automatic', - default="AutomaticGeoIPCountryWhois.csv", metavar='FILE', - help='write full input file plus automatic changes to the ' - 'specified .csv file [default: %default]') - parser.add_option('-m', action='store', dest='out_manual', - default='ManualGeoIPCountryWhois.csv', metavar='FILE', - help='write full input file plus automatic and manual ' - 'changes to the specified .csv file [default: %default]') - parser.add_option('-o', action='store', dest='out_geoip', - default='geoip', metavar='FILE', - help='write full input file plus automatic and manual ' - 'changes to the specified .csv file that can be shipped ' - 'with tor [default: %default]') - (options, args) = parser.parse_args() - return options - -def read_file(path, must_exist=True): - if not os.path.exists(path): - if must_exist: - print 'File %s does not exist. Exiting.' % (path, ) - sys.exit(1) - else: - return - if path.endswith('.zip'): - zip_file = zipfile.ZipFile(path) - csv_content = zip_file.read('GeoIPCountryWhois.csv') - zip_file.close() - else: - csv_file = open(path) - csv_content = csv_file.read() - csv_file.close() - assignments = [] - for line in csv_content.split('\n'): - stripped_line = line.strip() - if len(stripped_line) > 0 and not stripped_line.startswith('#'): - assignments.append(stripped_line) - return assignments - -def apply_automatic_changes(assignments): - print '\nApplying automatic changes...' - result_lines = [] - prev_line = None - a1_lines = [] - for line in assignments: - if '"A1"' in line: - a1_lines.append(line) - else: - if len(a1_lines) > 0: - new_a1_lines = process_a1_lines(prev_line, a1_lines, line) - for new_a1_line in new_a1_lines: - result_lines.append(new_a1_line) - a1_lines = [] - result_lines.append(line) - prev_line = line - if len(a1_lines) > 0: - new_a1_lines = process_a1_lines(prev_line, a1_lines, None) - for new_a1_line in new_a1_lines: - result_lines.append(new_a1_line) - return result_lines - -def process_a1_lines(prev_line, a1_lines, next_line): - if not prev_line or not next_line: - return a1_lines # Can't merge first or last line in file. - if len(a1_lines) > 1: - return a1_lines # Can't merge more than 1 line at once. - a1_line = a1_lines[0].strip() - prev_entry = parse_line(prev_line) - a1_entry = parse_line(a1_line) - next_entry = parse_line(next_line) - touches_prev_entry = int(prev_entry['end_num']) + 1 == \ - int(a1_entry['start_num']) - touches_next_entry = int(a1_entry['end_num']) + 1 == \ - int(next_entry['start_num']) - same_country_code = prev_entry['country_code'] == \ - next_entry['country_code'] - if touches_prev_entry and touches_next_entry and same_country_code: - new_line = format_line_with_other_country(a1_entry, prev_entry) - print '-%s\n+%s' % (a1_line, new_line, ) - return [new_line] - else: - return a1_lines - -def parse_line(line): - if not line: - return None - keys = ['start_str', 'end_str', 'start_num', 'end_num', - 'country_code', 'country_name'] - stripped_line = line.replace('"', '').strip() - parts = stripped_line.split(',') - entry = dict((k, v) for k, v in zip(keys, parts)) - return entry - -def format_line_with_other_country(original_entry, other_entry): - return '"%s","%s","%s","%s","%s","%s"' % (original_entry['start_str'], - original_entry['end_str'], original_entry['start_num'], - original_entry['end_num'], other_entry['country_code'], - other_entry['country_name'], ) - -def apply_manual_changes(assignments, manual_assignments): - if not manual_assignments: - return assignments - print '\nApplying manual changes...' - manual_dict = {} - for line in manual_assignments: - start_num = parse_line(line)['start_num'] - if start_num in manual_dict: - print ('Warning: duplicate start number in manual ' - 'assignments:\n %s\n %s\nDiscarding first entry.' % - (manual_dict[start_num], line, )) - manual_dict[start_num] = line - result = [] - for line in assignments: - entry = parse_line(line) - start_num = entry['start_num'] - if start_num in manual_dict: - manual_line = manual_dict[start_num] - manual_entry = parse_line(manual_line) - if entry['start_str'] == manual_entry['start_str'] and \ - entry['end_str'] == manual_entry['end_str'] and \ - entry['end_num'] == manual_entry['end_num']: - if len(manual_entry['country_code']) != 2: - print '-%s' % (line, ) # only remove, don't replace - else: - new_line = format_line_with_other_country(entry, - manual_entry) - print '-%s\n+%s' % (line, new_line, ) - result.append(new_line) - del manual_dict[start_num] - else: - print ('Warning: only partial match between ' - 'original/automatically replaced assignment and ' - 'manual assignment:\n %s\n %s\nNot applying ' - 'manual change.' % (line, manual_line, )) - result.append(line) - else: - result.append(line) - if len(manual_dict) > 0: - print ('Warning: could not apply all manual assignments: %s' % - ('\n '.join(manual_dict.values())), ) - return result - -def write_file(path, assignments, long_format=True): - if long_format: - output_lines = assignments - else: - output_lines = [] - for long_line in assignments: - entry = parse_line(long_line) - short_line = "%s,%s,%s" % (entry['start_num'], - entry['end_num'], entry['country_code'], ) - output_lines.append(short_line) - out_file = open(path, 'w') - out_file.write('\n'.join(output_lines)) - out_file.close() - -if __name__ == '__main__': - main() - diff --git a/src/config/geoip b/src/config/geoip index bec1f0954b..28cb59ecb7 100644 --- a/src/config/geoip +++ b/src/config/geoip @@ -1,4 +1,4 @@ -# Last updated based on July 10 2014 Maxmind GeoLite2 Country +# Last updated based on August 7 2014 Maxmind GeoLite2 Country # wget https://geolite.maxmind.com/download/geoip/database/GeoLite2-Country.mmdb.gz # gunzip GeoLite2-Country.mmdb.gz # python mmdb-convert.py GeoLite2-Country.mmdb @@ -38,7 +38,8 @@ 18153472,18219007,JP 18219008,18350079,IN 18350080,18874367,CN -18874368,18939903,HK +18874368,18907135,MY +18907136,18939903,HK 18939904,19005439,JP 19005440,19136511,TW 19136512,19202047,HK @@ -76,6 +77,7 @@ 34612224,34612735,IL 34620416,34620927,SE 34620928,34621439,IT +34621952,34622463,NL 34636800,34637311,DE 34646528,34647551,DE 34648576,34649087,GR @@ -221,7 +223,9 @@ 84049920,84082687,RO 84082688,84148223,RU 84148224,84410367,DE -84410368,84434943,RU +84410368,84418559,RU +84418560,84420607,GB +84420608,84434943,RU 84434944,84443135,IT 84443136,84451327,LB 84451328,84457471,RU @@ -235,18 +239,22 @@ 84551680,84557823,DE 84557824,84557824,NL 84557825,84557825,US -84557826,84560635,NL +84557826,84558063,NL +84558064,84558071,US +84558072,84558879,NL +84558880,84558887,US +84558888,84560635,NL 84560636,84560639,US -84560640,84561215,NL +84560640,84561063,NL +84561064,84561071,US +84561072,84561215,NL 84561216,84561247,US 84561248,84561351,NL 84561352,84561359,MY 84561360,84561639,NL 84561640,84561647,ES 84561648,84561655,AF -84561656,84561687,NL -84561688,84561695,US -84561696,84561727,NL +84561656,84561727,NL 84561728,84561791,US 84561792,84562383,NL 84562384,84562391,US @@ -271,8 +279,59 @@ 84565664,84566303,NL 84566304,84566319,US 84566320,84566383,NL -84566384,84566399,US -84566400,84574207,NL +84566384,84566415,US +84566416,84566420,NL +84566421,84566421,US +84566422,84566495,NL +84566496,84566527,US +84566528,84566567,NL +84566568,84566583,US +84566584,84566591,NL +84566592,84566687,US +84566688,84566719,NL +84566720,84566743,US +84566744,84567039,NL +84567040,84567055,US +84567056,84567071,NL +84567072,84567103,US +84567104,84567119,NL +84567120,84567135,US +84567136,84567199,NL +84567200,84567231,US +84567232,84567375,NL +84567376,84567391,US +84567392,84567535,NL +84567536,84567551,US +84567552,84567599,NL +84567600,84567615,US +84567616,84567647,NL +84567648,84567679,US +84567680,84567903,NL +84567904,84567919,US +84567920,84567999,NL +84568000,84568015,US +84568016,84568223,NL +84568224,84568239,US +84568240,84568255,GB +84568256,84568287,NL +84568288,84568303,US +84568304,84568351,NL +84568352,84568399,US +84568400,84568591,NL +84568592,84568607,GB +84568608,84568847,NL +84568848,84568863,US +84568864,84568927,NL +84568928,84568943,GB +84568944,84568991,NL +84568992,84569023,GB +84569024,84569119,NL +84569120,84569135,US +84569136,84569183,NL +84569184,84569199,US +84569200,84570127,NL +84570128,84570143,US +84570144,84574207,NL 84574208,84576255,FR 84576256,84582399,GB 84582400,84590591,DE @@ -333,7 +392,8 @@ 85733376,85737471,GB 85737472,85753855,DE 85753856,85770239,IL -85770240,85786623,DE +85770240,85778431,MD +85778432,85786623,DE 85786624,85852159,IL 85852160,86015999,AE 86016000,86018047,BG @@ -413,7 +473,9 @@ 86444840,86444843,ES 86444844,86444879,FR 86444880,86444883,NL -86444884,86445059,FR +86444884,86444895,FR +86444896,86444903,NL +86444904,86445059,FR 86445060,86445063,NL 86445064,86445111,FR 86445112,86445115,NL @@ -431,7 +493,11 @@ 86446620,86446620,DE 86446621,86446679,FR 86446680,86446683,ES -86446684,86447095,FR +86446684,86446719,FR +86446720,86446727,NL +86446728,86446983,FR +86446984,86446991,IT +86446992,86447095,FR 86447096,86447103,ES 86447104,86447255,FR 86447256,86447263,PL @@ -452,7 +518,9 @@ 86449884,86449884,DE 86449885,86450235,FR 86450236,86450239,GB -86450240,86452067,FR +86450240,86451167,FR +86451168,86451175,NL +86451176,86452067,FR 86452068,86452071,ES 86452072,86452299,FR 86452300,86452303,IT @@ -466,11 +534,16 @@ 86453837,86453838,PT 86453839,86454187,FR 86454188,86454191,IT -86454192,86454335,FR +86454192,86454279,FR +86454280,86454287,ES +86454288,86454295,IT +86454296,86454335,FR 86454336,86454343,DE 86454344,86454615,FR 86454616,86454619,ES -86454620,86455623,FR +86454620,86454823,FR +86454824,86454831,ES +86454832,86455623,FR 86455624,86455624,DE 86455625,86456195,FR 86456196,86456211,DE @@ -492,7 +565,9 @@ 86457456,86457456,DE 86457457,86457799,FR 86457800,86457803,ES -86457804,86467320,FR +86457804,86466839,FR +86466840,86466847,NL +86466848,86467320,FR 86467321,86467321,FI 86467322,86467999,FR 86468000,86468003,PL @@ -767,7 +842,8 @@ 90540032,90544127,GB 90544128,90546175,RU 90546176,90548223,DE -90548224,90570751,RU +90548224,90550271,GB +90550272,90570751,RU 90570752,90578943,IT 90578944,90583039,IR 90583040,90587135,CZ @@ -869,9 +945,13 @@ 92736000,92736255,GB 92736256,92736479,FR 92736480,92736480,DE -92736481,92738719,FR +92736481,92738663,FR +92738664,92738679,ES +92738680,92738719,FR 92738720,92738727,GB -92738728,92741203,FR +92738728,92740447,FR +92740448,92740455,IT +92740456,92741203,FR 92741204,92741207,IT 92741208,92741387,FR 92741388,92741395,IT @@ -907,7 +987,9 @@ 92744556,92744559,IT 92744560,92747711,FR 92747712,92747775,GB -92747776,92749067,FR +92747776,92748773,FR +92748774,92748774,PT +92748775,92749067,FR 92749068,92749071,GB 92749072,92749747,FR 92749748,92749751,ES @@ -1010,9 +1092,7 @@ 93914320,93914323,AZ 93914324,93914357,NL 93914358,93914358,GB -93914359,93914671,NL -93914672,93914679,US -93914680,93914710,NL +93914359,93914710,NL 93914711,93914711,GB 93914712,93914951,NL 93914952,93914959,US @@ -1051,11 +1131,15 @@ 93920060,93920063,US 93920064,93920163,NL 93920164,93920167,US -93920168,93920575,NL +93920168,93920223,NL +93920224,93920231,US +93920232,93920575,NL 93920576,93920639,US 93920640,93920855,NL 93920856,93920863,US -93920864,93921055,NL +93920864,93920967,NL +93920968,93920975,US +93920976,93921055,NL 93921056,93921059,US 93921060,93921063,NL 93921064,93921071,GB @@ -1069,9 +1153,7 @@ 93924408,93924415,JP 93924416,93924591,NL 93924592,93924599,US -93924600,93924927,NL -93924928,93924935,ES -93924936,93925807,NL +93924600,93925807,NL 93925808,93925815,KE 93925816,93927143,NL 93927144,93927151,JP @@ -1199,12 +1281,15 @@ 95401472,95401727,GB 95401728,95401903,DE 95401904,95401911,AT -95401912,95402111,DE +95401912,95401983,DE +95401984,95402111,GB 95402112,95402239,KW -95402240,95402623,DE +95402240,95402495,DE +95402496,95402623,GB 95402624,95402695,US 95402696,95402703,HR -95402704,95402751,US +95402704,95402719,DE +95402720,95402751,US 95402752,95403183,DE 95403184,95403191,BE 95403192,95403519,DE @@ -1254,7 +1339,8 @@ 95666176,95668223,FR 95668224,95682559,DE 95944704,96075775,PL -96075776,96143359,DE +96075776,96141311,DK +96141312,96143359,DE 96143360,96145407,GE 96145408,96149503,GB 96149504,96151551,ES @@ -1331,7 +1417,9 @@ 98734080,98736127,CH 98736128,98738175,RU 98738176,98740223,NO -98740224,98742271,DE +98740224,98741503,DE +98741504,98741759,US +98741760,98742271,DE 98742272,98744319,GB 98744320,98746367,ES 98746368,98762751,TR @@ -1499,9 +1587,7 @@ 212788480,212788735,VI 212788736,212788863,US 212788864,212788991,PR -212788992,212789087,US -212789088,212789095,VI -212789096,212791831,US +212788992,212791831,US 212791832,212791839,VI 212791840,212791935,US 212791936,212792063,VI @@ -1509,9 +1595,7 @@ 212793088,212793343,PR 212793344,212794575,US 212794576,212794583,VI -212794584,214237247,US -214237248,214237311,PR -214237312,214698239,US +212794584,214698239,US 214698240,214698255,VI 214698256,214698303,US 214698304,214698311,VI @@ -1520,9 +1604,7 @@ 214699648,214699775,VI 214699776,214778367,US 214778368,214778623,PR -214778624,216417663,US -216417664,216417727,PR -216417728,217709055,US +214778624,217709055,US 217709056,217709311,PR 217709312,219512063,US 219512064,219512319,GB @@ -1586,8 +1668,7 @@ 247488512,247496703,JP 247496704,247504895,PK 247504896,247508991,AU -247508992,247509247,US -247509248,247513087,AU +247508992,247513087,US 247513088,247529471,MY 247529472,247595007,JP 247595008,247726079,IN @@ -1973,9 +2054,27 @@ 391897088,391905279,CA 391905280,391938047,US 391938048,391946239,CA -391946240,392765439,US +391946240,392429567,US +392429568,392433663,NL +392433664,392441855,US +392441856,392445951,IE +392445952,392452095,US +392452096,392454143,HK +392454144,392458239,JP +392458240,392460287,SG +392460288,392495103,US +392495104,392499199,HK +392499200,392503295,SG +392503296,392507391,US +392507392,392511487,IE +392511488,392515583,NL +392515584,392523775,JP +392523776,392548351,US +392548352,392556543,AU +392556544,392765439,US 392765440,392765695,GB 392765696,394264575,US +394264576,394264831,CA 398458880,398635007,US 398635008,398643199,NL 398643200,398647295,US @@ -1990,7 +2089,9 @@ 398856192,398876671,NL 398876672,398970879,US 398970880,398974975,NL -398974976,399007743,US +398974976,399004671,US +399004672,399006207,FR +399006208,399007743,US 399007744,399011839,NL 399011840,399020031,US 399020032,399028223,NL @@ -2129,6 +2230,7 @@ 406003712,406011903,US 406011904,406028287,BS 406028288,406052863,US +406052864,406061055,CA 406061056,406110207,US 406110208,406142975,CA 406142976,406147071,US @@ -2168,6 +2270,7 @@ 409337856,409354239,CA 409354240,409509887,US 409518080,409550847,US +409550848,409567231,CA 409567232,409731071,US 409731072,409862143,CA 409862144,410124287,US @@ -2192,7 +2295,7 @@ 411770880,411779071,US 411779072,411828223,PR 411828224,411885567,US -411893760,411975679,CA +411885568,411975679,CA 411975680,411979775,US 411979776,411983871,CA 411983872,411988991,US @@ -2242,7 +2345,7 @@ 417775616,417796095,CA 417796096,417800191,US 417800192,417808383,BS -417808384,417820671,CA +417808384,417816575,CA 417820672,417832191,US 417832192,417832447,VI 417832448,417857535,US @@ -2305,7 +2408,6 @@ 456542208,456544255,CN 456544256,456548351,AU 456548352,456553471,JP -456553472,456554495,MY 456554496,456555519,PK 456555520,456556543,JP 456556544,456560639,AU @@ -2334,7 +2436,9 @@ 459538432,459539455,AU 459540480,459541503,JP 459541504,459542527,IN -459542528,459544575,HK +459542528,459543295,HK +459543296,459543551,TW +459543552,459544575,HK 459545600,459547647,JP 459548672,459550719,TH 459550720,459554815,JP @@ -2421,8 +2525,7 @@ 460931072,460933119,AU 460933120,460935167,CN 460935168,460937215,ID -460937216,460937727,NZ -460937728,460938239,AU +460937216,460938239,AU 460938240,460939263,JP 460939264,460940287,NZ 460940288,460941311,IN @@ -2446,8 +2549,10 @@ 461047808,461049855,JP 461049856,461050879,TH 461050880,461051903,NZ -461051904,461053951,AU -461053952,461062143,HK +461051904,461054975,AU +461054976,461055999,HK +461056000,461058047,AU +461058048,461062143,HK 461062144,461078527,IN 461078528,461078783,FJ 461078784,461079039,AU @@ -2477,7 +2582,11 @@ 461281280,461282303,PH 461282304,461283327,MY 461283328,461287423,JP -461287424,461307903,HK +461287424,461294591,HK +461294592,461298687,US +461298688,461301759,HK +461301760,461302527,US +461302528,461307903,HK 461307904,461357055,JP 461357056,461369343,AU 461369344,461373439,JP @@ -2541,7 +2650,7 @@ 520339456,520343551,UA 520343552,520355839,GB 520355840,520421375,ES -520421376,520486911,AT +520421376,520486911,BE 520486912,520488959,NL 520488960,520489983,IT 520489984,520490495,RU @@ -2952,9 +3061,11 @@ 529596416,529661951,TR 529661952,529727487,GE 529727488,529793023,HR -529793024,529793535,CZ -529793536,529794047,RU -529794048,529818623,CZ +529793024,529793337,CZ +529793338,529793338,RU +529793339,529793535,CZ +529793536,529794303,RU +529794304,529818623,CZ 529818624,529826303,RU 529826304,529826815,CZ 529826816,529827839,RU @@ -3023,7 +3134,9 @@ 531263488,531265535,RU 531265536,531267583,GB 531267584,531275775,UA -531275776,531277823,US +531275776,531276799,US +531276800,531277311,GB +531277312,531277823,US 531277824,531279871,RU 531279872,531281919,CZ 531281920,531283967,RU @@ -3070,11 +3183,11 @@ 531429136,531429143,IT 531429144,531429167,GB 531429168,531429175,IT -531429176,531429239,GB +531429176,531429207,GB +531429208,531429215,IT +531429216,531429239,GB 531429240,531429247,IT -531429248,531429335,GB -531429336,531429343,IT -531429344,531429391,GB +531429248,531429391,GB 531429392,531429399,IT 531429400,531429407,GB 531429408,531429415,IT @@ -3086,7 +3199,9 @@ 531430824,531430831,IT 531430832,531430847,GB 531430848,531430855,IT -531430856,531431423,GB +531430856,531430927,GB +531430928,531430935,IT +531430936,531431423,GB 531431424,531496959,RO 531496960,531628031,PL 531628032,531660799,TR @@ -3326,7 +3441,8 @@ 534513408,534513663,SE 534513664,534515455,US 534515456,534515711,DE -534515712,534517759,US +534515712,534515967,GB +534515968,534517759,US 534517760,534518783,NL 534518784,534519039,DE 534519040,534519167,NL @@ -3350,9 +3466,7 @@ 534523136,534523391,NL 534523392,534523903,DE 534523904,534530047,US -534530048,534541311,DE -534541312,534543359,US -534543360,534544383,DE +534530048,534544383,DE 534544384,534546431,RO 534546432,534548479,DE 534548480,534550527,PL @@ -3813,7 +3927,9 @@ 623801600,623801855,SE 623801856,623802367,NL 623802368,623802879,SE -623802880,623804415,NL +623802880,623804148,NL +623804149,623804159,SE +623804160,623804415,NL 623804416,623806463,RU 623806464,623808511,NL 623808512,623810559,RU @@ -3861,7 +3977,11 @@ 624575400,624575403,US 624575404,624575679,NL 624575680,624575743,US -624575744,624576127,NL +624575744,624575871,NL +624575872,624575879,US +624575880,624576031,NL +624576032,624576039,US +624576040,624576127,NL 624576128,624576131,US 624576132,624576471,NL 624576472,624576479,US @@ -3875,7 +3995,11 @@ 624577308,624577311,US 624577312,624577483,NL 624577484,624577487,US -624577488,624578719,NL +624577488,624577863,NL +624577864,624577871,DK +624577872,624578415,NL +624578416,624578423,US +624578424,624578719,NL 624578720,624578723,GB 624578724,624578887,NL 624578888,624578895,US @@ -3911,9 +4035,7 @@ 624582288,624582295,US 624582296,624582399,NL 624582400,624582403,US -624582404,624583455,NL -624583456,624583463,US -624583464,624584111,NL +624582404,624584111,NL 624584112,624584119,US 624584120,624584159,NL 624584160,624584175,US @@ -3921,28 +4043,33 @@ 624584384,624584391,US 624584392,624584415,NL 624584416,624584423,US -624584424,624585359,NL -624585360,624585367,US -624585368,624587583,NL +624584424,624586279,NL +624586280,624586287,US +624586288,624587583,NL 624587584,624587599,US -624587600,624587751,NL -624587752,624587759,US -624587760,624587871,NL +624587600,624587871,NL 624587872,624587903,US -624587904,624588391,NL +624587904,624588383,NL +624588384,624588391,US 624588392,624588399,GB -624588400,624589159,NL +624588400,624588927,NL +624588928,624588943,GB +624588944,624589159,NL 624589160,624589167,IT 624589168,624589199,NL 624589200,624589215,KE -624589216,624589967,NL +624589216,624589223,NL +624589224,624589231,US +624589232,624589719,NL +624589720,624589727,US +624589728,624589967,NL 624589968,624589975,US -624589976,624590743,NL -624590744,624590751,US -624590752,624590815,NL -624590816,624590823,US -624590824,624590847,NL -624590848,624640951,FR +624589976,624590847,NL +624590848,624640527,FR +624640528,624640543,GB +624640544,624640759,FR +624640760,624640767,NL +624640768,624640951,FR 624640952,624640959,PT 624640960,624643019,FR 624643020,624643023,IT @@ -3950,9 +4077,11 @@ 624645148,624645151,IT 624645152,624646343,FR 624646344,624646347,NL -624646348,624647171,FR -624647172,624647175,ES -624647176,624648139,FR +624646348,624646583,FR +624646584,624646591,NL +624646592,624647171,FR +624647172,624647183,ES +624647184,624648139,FR 624648140,624648143,ES 624648144,624657711,FR 624657712,624657715,ES @@ -3962,7 +4091,9 @@ 624658324,624658327,IE 624658328,624658479,FR 624658480,624658483,DE -624658484,624660827,FR +624658484,624659031,FR +624659032,624659039,IT +624659040,624660827,FR 624660828,624660831,ES 624660832,624661247,FR 624661248,624661251,ES @@ -4004,7 +4135,9 @@ 624673452,624673455,ES 624673456,624673535,FR 624673536,624673791,ES -624673792,624673903,FR +624673792,624673803,FR +624673804,624673807,NL +624673808,624673903,FR 624673904,624673919,ES 624673920,624675691,FR 624675692,624675695,ES @@ -4020,13 +4153,25 @@ 624677464,624677467,ES 624677468,624677683,FR 624677684,624677687,NL -624677688,624679199,FR +624677688,624677767,FR +624677768,624677775,NL +624677776,624678391,FR +624678392,624678399,NL +624678400,624678487,FR +624678488,624678495,NL +624678496,624679175,FR +624679176,624679183,NL +624679184,624679199,FR 624679200,624679203,ES -624679204,624679843,FR +624679204,624679679,FR +624679680,624679687,NL +624679688,624679843,FR 624679844,624679847,ES 624679848,624680931,FR 624680932,624680935,NL -624680936,624681807,FR +624680936,624681351,FR +624681352,624681359,NL +624681360,624681807,FR 624681808,624681823,BE 624681824,624683775,FR 624683776,624683779,DE @@ -4034,7 +4179,9 @@ 624683784,624683787,DE 624683788,624683975,FR 624683976,624683983,GB -624683984,624684199,FR +624683984,624684183,FR +624684184,624684191,IT +624684192,624684199,FR 624684200,624684203,IT 624684204,624684207,FR 624684208,624684211,ES @@ -4059,7 +4206,9 @@ 624687347,624687347,ES 624687348,624687827,FR 624687828,624687831,ES -624687832,624688307,FR +624687832,624688007,FR +624688008,624688015,ES +624688016,624688307,FR 624688308,624688311,ES 624688312,624688487,FR 624688488,624688491,ES @@ -4135,7 +4284,9 @@ 625524480,625524735,SE 625524736,625541119,FR 625541120,625606655,UA -625606656,625672191,NL +625606656,625621667,NL +625621668,625621671,DE +625621672,625672191,NL 625672192,625674239,RU 625674240,625676287,TR 625676288,625680383,MD @@ -4561,8 +4712,8 @@ 635191296,635195391,RS 635195392,635197439,RU 635197440,635199591,GB -635199592,635199607,IT -635199608,635199647,GB +635199592,635199599,IT +635199600,635199647,GB 635199648,635199655,IT 635199656,635199775,GB 635199776,635199783,IT @@ -4607,7 +4758,9 @@ 635240448,635256831,IR 635273216,635281407,UA 635281408,635283455,RO -635283456,635284479,DE +635283456,635283967,DE +635283968,635284223,RO +635284224,635284479,DE 635284480,635284991,RO 635284992,635285503,US 635285504,635287551,ME @@ -4939,7 +5092,9 @@ 645873664,645874175,CA 645874176,645875711,US 645875712,645876735,CA -645876736,654311423,US +645876736,645989450,US +645989451,645989451,CA +645989452,654311423,US 654311424,654311679,CN 654311680,654311935,AU 654311936,654376959,CN @@ -5572,7 +5727,6 @@ 702447616,702449663,EG 702449664,702451711,ZA 702451712,702453759,NG -702453760,702455807,TZ 702455808,702457855,KE 702457856,702459903,EG 702459904,702461951,TZ @@ -5777,7 +5931,9 @@ 703753216,703753471,ZA 703753472,703754239,MG 703754240,703755263,GH -703755264,703757311,ZA +703755264,703755899,ZA +703755900,703755900,YT +703755901,703757311,ZA 703757312,703758335,RE 703758336,703759359,CD 703759360,703760383,ZA @@ -5980,11 +6136,25 @@ 737514496,737515519,IN 737515520,737516543,JP 737516544,737517567,IN -737517568,737529855,JP +737517568,737526783,JP +737526784,737527295,US +737527296,737529855,JP 737529856,737530879,IN 737530880,737941503,JP 737941504,737944575,IN -737944576,738197503,JP +737944576,737950719,JP +737950720,737951743,IN +737951744,737958911,JP +737958912,737959935,IN +737959936,737965055,JP +737965056,737966079,IN +737966080,737967103,JP +737967104,737967359,AU +737967360,737991679,JP +737991680,737992191,IN +737992192,737999320,JP +737999321,737999321,IN +737999322,738197503,JP 738197504,746717183,US 746717184,746782719,DE 746782720,755105791,US @@ -6086,7 +6256,9 @@ 772841472,772843519,GB 772843520,772845567,IT 772845568,772847615,RU -772847616,772848191,NL +772847616,772848103,NL +772848104,772848111,US +772848112,772848191,NL 772848192,772848223,US 772848224,772848871,NL 772848872,772848879,US @@ -6115,7 +6287,7 @@ 772884480,772886527,LB 772886528,772888575,FR 772888576,772890623,GB -772890624,772892671,RU +772890624,772892671,NL 772892672,772894719,GB 772894720,772896767,PL 772896768,772898815,RS @@ -6889,7 +7061,9 @@ 778247936,778248191,RS 778248192,778249727,AL 778249728,778249983,RS -778249984,778305535,AL +778249984,778304305,AL +778304306,778304306,SI +778304307,778305535,AL 778305536,778371071,IR 778371072,778436607,RU 778436608,778476031,RO @@ -6913,7 +7087,9 @@ 778668864,778668895,DE 778668896,778670975,FR 778670976,778670976,DE -778670977,778673187,FR +778670977,778671201,FR +778671202,778671202,ES +778671203,778673187,FR 778673188,778673191,ES 778673192,778673207,FR 778673208,778673211,ES @@ -6943,7 +7119,9 @@ 778680228,778680231,IE 778680232,778680683,FR 778680684,778680687,ES -778680688,778681503,FR +778680688,778681487,FR +778681488,778681495,NL +778681496,778681503,FR 778681504,778681507,DE 778681508,778681519,FR 778681520,778681523,ES @@ -7118,7 +7296,9 @@ 782663680,782667519,NL 782667520,782667775,LU 782667776,782671871,NL -782671872,782675967,LT +782671872,782672871,LT +782672872,782672879,IL +782672880,782675967,LT 782675968,782680063,NL 782680064,782696447,RU 782696448,782712831,DE @@ -7430,7 +7610,9 @@ 786920840,786920847,IT 786920848,786920855,GB 786920856,786920863,IT -786920864,786921455,GB +786920864,786921303,GB +786921304,786921311,IT +786921312,786921455,GB 786921456,786921463,IT 786921464,786921471,GB 786921472,786923519,ES @@ -7579,7 +7761,8 @@ 787703808,787705855,AT 787705856,787707903,RO 787707904,787709951,DE -787709952,787724287,RU +787709952,787718143,NL +787718144,787724287,RU 787724288,787726335,UA 787726336,787742719,RU 787742720,787759103,NL @@ -7619,7 +7802,9 @@ 788070400,788078591,RU 788078592,788086783,NL 788086784,788094975,BG -788094976,788103167,IR +788094976,788100095,IR +788100096,788101119,BG +788101120,788103167,IR 788103168,788111359,HU 788111360,788119551,LT 788119552,788127743,GB @@ -7802,7 +7987,8 @@ 830475264,830476287,AU 830476288,830480383,JP 830480384,830488575,SG -830488576,830496767,TW +830488576,830492671,HK +830492672,830496767,TW 830496768,830498815,JP 830498816,830499839,GU 830499840,830500863,IN @@ -8245,7 +8431,10 @@ 977764352,977797119,MY 977797120,978321407,KR 978321408,978452479,JP -978452480,978583551,CN +978452480,978485247,CN +978485248,978501631,TH +978501632,978518015,SG +978518016,978583551,CN 978599936,978640895,AU 978640896,978644991,NZ 978644992,978714623,JP @@ -8338,7 +8527,8 @@ 999751680,999784447,CN 999784448,999800831,JP 999800832,999817215,KR -999817216,999849983,BD +999817216,999821311,BD +999839744,999845887,BD 999849984,999866367,KR 999866368,999873919,HK 999873920,999873941,VN @@ -8441,20 +8631,28 @@ 1024352256,1024360447,AU 1024360448,1024361167,JP 1024361168,1024361183,HK -1024361184,1024363263,JP +1024361184,1024362911,JP +1024362912,1024362943,SG +1024362944,1024363263,JP 1024363264,1024363519,SG 1024363520,1024363775,JP 1024363776,1024364031,AU -1024364032,1024365727,JP +1024364032,1024364063,JP +1024364064,1024364079,AU +1024364080,1024365727,JP 1024365728,1024365759,SG -1024365760,1024372543,JP +1024365760,1024368639,JP +1024368640,1024369407,MY +1024369408,1024371199,JP +1024371200,1024371455,PH +1024371456,1024372543,JP 1024372544,1024372607,HK 1024372608,1024373263,JP 1024373264,1024373279,HK 1024373280,1024375295,JP 1024375296,1024375551,AU 1024375552,1024376831,JP -1024376832,1024393215,PH +1024376832,1024378879,PH 1024393216,1024458751,HK 1024458752,1024491519,SG 1024491520,1024589823,IN @@ -8506,8 +8704,10 @@ 1027866624,1027997695,AU 1027997696,1028128767,TW 1028128768,1029046271,KR -1029046272,1029144575,JP -1029144576,1029160959,SG +1029046272,1029148671,JP +1029148672,1029152767,HK +1029152768,1029156863,SG +1029156864,1029160959,AU 1029160960,1029177343,CN 1029177344,1029242879,AU 1029242880,1029308415,JP @@ -8619,7 +8819,9 @@ 1041710672,1041710687,FR 1041710688,1041711551,GB 1041711552,1041711559,FR -1041711560,1041712631,GB +1041711560,1041711943,GB +1041711944,1041711951,FR +1041711952,1041712631,GB 1041712632,1041712639,FR 1041712640,1041715071,GB 1041715072,1041715079,FR @@ -8750,7 +8952,13 @@ 1043103744,1043120127,DK 1043120128,1043136511,FI 1043136512,1043202047,NL -1043202048,1043333119,AT +1043202048,1043249919,AT +1043249920,1043250175,HU +1043250176,1043257087,AT +1043257088,1043257343,DE +1043257344,1043281151,AT +1043281152,1043281407,DE +1043281408,1043333119,AT 1043333120,1043341311,CH 1043341312,1043349503,IT 1043349504,1043357695,DE @@ -8896,7 +9104,9 @@ 1045158307,1045158307,SG 1045158308,1045159711,DE 1045159712,1045159712,EG -1045159713,1045168127,DE +1045159713,1045163508,DE +1045163509,1045163511,SG +1045163512,1045168127,DE 1045168128,1045233663,RU 1045233664,1045241855,GB 1045241856,1045250047,IT @@ -8988,7 +9198,9 @@ 1046488320,1046488575,DE 1046488576,1046489087,GB 1046489088,1046489119,DE -1046489120,1046489391,GB +1046489120,1046489311,GB +1046489312,1046489327,ES +1046489328,1046489391,GB 1046489392,1046489407,IT 1046489408,1046489471,GB 1046489472,1046489487,ES @@ -9137,7 +9349,9 @@ 1047658496,1047724031,EG 1047728128,1047732223,SE 1047787520,1047787775,ES -1047789568,1047822335,AT +1047789568,1047805439,AT +1047805440,1047805695,DE +1047805696,1047822335,AT 1047822336,1047838719,DE 1047838720,1047846911,DK 1047846912,1047855103,SE @@ -9202,7 +9416,6 @@ 1048707072,1048772607,GB 1048772608,1048903679,NL 1048903680,1048911871,GB -1048911872,1048920063,IT 1048920064,1048936447,NL 1048936448,1048944639,PL 1048944640,1048952831,RU @@ -9329,12 +9542,7 @@ 1051803648,1051820031,CZ 1051820032,1051852799,NL 1051852800,1051918335,AT -1051918336,1051918591,PL -1051918592,1051919359,AT -1051919360,1051920383,PL -1051920384,1051920895,AT -1051920896,1051921919,PL -1051921920,1051922431,AT +1051918336,1051922431,PL 1051922432,1051924479,CH 1051924480,1051948031,AT 1051948032,1051949055,CH @@ -9488,7 +9696,9 @@ 1053663232,1053671423,RU 1053671424,1053687807,LV 1053687808,1053753343,DE -1053753344,1053818879,NL +1053753344,1053788687,NL +1053788688,1053788695,DE +1053788696,1053818879,NL 1053818880,1053819391,DE 1053819424,1053819439,DE 1053819520,1053819563,DE @@ -9548,7 +9758,8 @@ 1053867976,1053867983,DE 1053867992,1053867999,DE 1053868008,1053868015,DE -1053868032,1053868447,ES +1053868032,1053868351,ES +1053868416,1053868447,ES 1053868448,1053868455,GB 1053868456,1053868463,ES 1053868480,1053868543,FR @@ -9718,7 +9929,8 @@ 1056473088,1056505855,FI 1056505856,1056514047,PT 1056514048,1056522239,IT -1056522240,1056538623,AT +1056522240,1056523007,DE +1056523008,1056538623,AT 1056538624,1056546815,RU 1056546816,1056555007,NO 1056555008,1056571391,GB @@ -9765,9 +9977,7 @@ 1063305728,1063305983,CA 1063305984,1063568895,US 1063568896,1063569151,TZ -1063569152,1063727615,US -1063727616,1063728127,GU -1063728128,1063748607,US +1063569152,1063748607,US 1063748608,1063749119,GH 1063749120,1063749631,US 1063749632,1063749887,LR @@ -10025,13 +10235,17 @@ 1071954304,1071954319,GB 1071954320,1071954391,US 1071954392,1071954399,GB -1071954400,1072157631,US +1071954400,1071998999,US +1071999000,1071999003,GB +1071999004,1072157631,US 1072157632,1072157663,DE 1072157664,1072228863,US 1072228864,1072229375,CA 1072229376,1072360703,US 1072360704,1072360959,HK -1072360960,1072707327,US +1072360960,1072361471,US +1072361472,1072361727,GB +1072361728,1072707327,US 1072707328,1072707583,IN 1072707584,1072923135,US 1072923136,1072923391,CA @@ -10437,7 +10651,9 @@ 1078280192,1078280447,CA 1078280448,1078280575,US 1078280576,1078280583,CA -1078280584,1078281300,US +1078280584,1078281071,US +1078281072,1078281079,CA +1078281080,1078281300,US 1078281301,1078281301,CA 1078281302,1078281511,US 1078281512,1078281519,CA @@ -10447,9 +10663,11 @@ 1078281728,1078281735,CA 1078281736,1078282239,US 1078282240,1078283015,CA -1078283016,1078284031,US -1078284032,1078284159,CA -1078284160,1078284479,US +1078283016,1078283831,US +1078283832,1078283839,CA +1078283840,1078284031,US +1078284032,1078284287,CA +1078284288,1078284479,US 1078284480,1078284543,CA 1078284544,1078284703,US 1078284704,1078284719,CA @@ -10542,109 +10760,66 @@ 1079377920,1079378943,CA 1079378944,1079379199,US 1079379200,1079379455,CA -1079379456,1079379711,US -1079379712,1079380927,CA -1079380928,1079380991,US -1079380992,1079381759,CA -1079381760,1079382271,US -1079382272,1079382527,CA -1079382528,1079383039,US +1079379456,1079383039,US 1079383040,1079383295,VG -1079383296,1079383551,CA -1079383552,1079383807,US +1079383296,1079383807,US 1079383808,1079384063,MH 1079384064,1079384319,LR 1079384320,1079384575,CA 1079384576,1079385087,ZW -1079385088,1079385343,US -1079385344,1079385471,CA -1079385472,1079385479,US -1079385480,1079386623,CA +1079385088,1079385599,US +1079385600,1079386623,CA 1079386624,1079386879,SG 1079386880,1079387135,EG 1079387136,1079387903,US 1079387904,1079388159,PH -1079388160,1079389695,CA -1079389696,1079389951,US -1079389952,1079390975,CA -1079390976,1079391487,US -1079391488,1079391743,CA +1079388160,1079389183,CA +1079389184,1079389951,US +1079389952,1079390207,CA +1079390208,1079391743,US 1079391744,1079392255,HT 1079392256,1079393791,CA -1079393792,1079394047,US -1079394048,1079394303,CA +1079393792,1079394303,US 1079394304,1079395327,EC 1079395328,1079396095,US 1079396096,1079396351,CA 1079396352,1079397375,MP 1079397376,1079397631,MH -1079397632,1079397887,CA -1079397888,1079398399,US -1079398400,1079399935,CA -1079399936,1079400447,US -1079400448,1079401215,CA -1079401216,1079401471,US -1079401472,1079403263,CA +1079397632,1079400959,US +1079400960,1079401215,CA +1079401216,1079402495,US +1079402496,1079403263,CA 1079403264,1079403519,US 1079403520,1079403775,CA -1079403776,1079403807,US -1079403808,1079403903,CA -1079403904,1079403935,US -1079403936,1079403999,CA -1079404000,1079404031,US -1079404032,1079404543,CA -1079404544,1079404799,US -1079404800,1079404959,CA -1079404960,1079404991,US -1079404992,1079405023,CA -1079405024,1079405055,US -1079405056,1079407103,CA -1079407104,1079407359,US -1079407360,1079407615,CA -1079407616,1079408127,US -1079408128,1079408639,CA -1079408640,1079408895,US +1079403776,1079405567,US +1079405568,1079406079,CA +1079406080,1079408895,US 1079408896,1079409407,PK 1079409408,1079409919,US -1079409920,1079410687,CA -1079410688,1079410943,US -1079410944,1079411199,CA -1079411200,1079411455,US +1079409920,1079410175,CA +1079410176,1079411455,US 1079411456,1079411711,PK -1079411712,1079411743,US -1079411744,1079412735,CA -1079412736,1079413247,US -1079413248,1079413311,CA -1079413312,1079413343,US -1079413344,1079414271,CA +1079411712,1079413759,US +1079413760,1079414271,CA 1079414272,1079415039,US 1079415040,1079415295,HN -1079415296,1079415871,US -1079415872,1079415879,CA -1079415880,1079415887,US -1079415888,1079421951,CA -1079421952,1079422207,US -1079422208,1079425023,CA -1079425024,1079425279,US -1079425280,1079427583,CA +1079415296,1079416831,US +1079416832,1079418879,CA +1079418880,1079422975,US +1079422976,1079423999,CA +1079424000,1079425791,US +1079425792,1079426047,CA +1079426048,1079427327,US +1079427328,1079427583,CA 1079427584,1079428095,PW -1079428096,1079429263,CA -1079429264,1079429295,US -1079429296,1079429375,CA -1079429376,1079429631,US -1079429632,1079431679,CA +1079428096,1079431679,US 1079431680,1079432191,ZM -1079432192,1079432703,CA -1079432704,1079432959,US +1079432192,1079432959,US 1079432960,1079433215,CA 1079433216,1079435263,CR 1079435264,1079435775,CO -1079435776,1079437311,CA -1079437312,1079439359,US -1079439360,1079442431,CA -1079442432,1079442687,US -1079442688,1079443455,CA -1079443456,1079459839,US +1079435776,1079436031,CA +1079436032,1079459839,US 1079459840,1079508991,CA 1079508992,1079566847,US 1079566848,1079567103,GB @@ -10795,9 +10970,7 @@ 1082952704,1082952959,CA 1082952960,1082982399,US 1082982400,1083015167,CA -1083015168,1083265023,US -1083265024,1083265279,CA -1083265280,1083396095,US +1083015168,1083396095,US 1083396096,1083400191,BM 1083400192,1083437055,US 1083437056,1083441151,CA @@ -10823,7 +10996,7 @@ 1085849600,1085857791,CA 1085857792,1085915135,US 1085915136,1085923327,PR -1085927424,1085997055,US +1085923328,1085997055,US 1085997056,1086013439,CA 1086013440,1086042111,US 1086042112,1086046207,CA @@ -11200,9 +11373,7 @@ 1101304064,1101304319,EC 1101304320,1101304831,US 1101304832,1101305855,HN -1101305856,1101306047,US -1101306048,1101306079,PR -1101306080,1101352959,US +1101305856,1101352959,US 1101352960,1101355007,HN 1101355008,1101402031,US 1101402032,1101402047,PR @@ -11533,7 +11704,9 @@ 1114507264,1114511871,US 1114511872,1114512127,CA 1114512128,1114513407,US -1114513408,1114515455,SA +1114513408,1114513471,SA +1114513472,1114513535,US +1114513536,1114515455,SA 1114515456,1114517503,US 1114517504,1114518015,CA 1114518016,1114520063,US @@ -11758,9 +11931,7 @@ 1119558144,1119558655,PR 1119558656,1119568383,US 1119568384,1119568639,GB -1119568640,1119568767,US -1119568768,1119568895,GB -1119568896,1119570175,US +1119568640,1119570175,US 1119570176,1119570303,GB 1119570304,1119571967,US 1119571968,1119576063,CA @@ -12028,6 +12199,7 @@ 1125478400,1125481215,US 1125481216,1125481727,CA 1125481728,1125489151,US +1125489152,1125490687,CA 1125490688,1125498879,US 1125498880,1125501439,CA 1125501440,1125501695,US @@ -12157,7 +12329,8 @@ 1138184192,1138184447,SG 1138184448,1138184959,US 1138184960,1138185215,AU -1138185216,1138188287,US +1138185216,1138185727,CA +1138185728,1138188287,US 1138188288,1138192383,PR 1138192384,1138196479,US 1138196480,1138204671,CA @@ -12251,7 +12424,9 @@ 1145430016,1145475071,US 1145475072,1145479167,CA 1145479168,1145503743,US -1145503744,1145520127,CA +1145503744,1145506815,CA +1145506816,1145507071,US +1145507072,1145520127,CA 1145520128,1145552895,US 1145552896,1145556991,CA 1145556992,1150043135,US @@ -12512,9 +12687,7 @@ 1161627704,1161627711,DE 1161627712,1161631623,US 1161631624,1161631631,KW -1161631632,1161634079,US -1161634080,1161634087,AF -1161634088,1161649407,US +1161631632,1161649407,US 1161649408,1161649663,AR 1161649664,1161764863,US 1161764864,1161773055,CA @@ -12984,7 +13157,9 @@ 1208935984,1208935991,IN 1208935992,1208935999,JP 1208936000,1208936003,HK -1208936004,1208936191,US +1208936004,1208936031,US +1208936032,1208936039,IN +1208936040,1208936191,US 1208936192,1208936199,AU 1208936200,1208936207,SG 1208936208,1208936215,HK @@ -13020,7 +13195,12 @@ 1209729024,1209786367,JM 1209786368,1209810943,US 1209810944,1209819135,CA -1209819136,1209824511,US +1209819136,1209823487,US +1209823488,1209823519,GB +1209823520,1209823543,US +1209823544,1209823551,GB +1209823552,1209824447,US +1209824448,1209824511,GB 1209824512,1209824767,CN 1209824768,1209861119,US 1209861120,1209861375,CA @@ -13070,7 +13250,9 @@ 1211308168,1211308175,CA 1211308176,1211308287,US 1211308288,1211308543,CA -1211308544,1211308799,US +1211308544,1211308719,US +1211308720,1211308727,CA +1211308728,1211308799,US 1211308800,1211310079,CA 1211310080,1211316479,US 1211316480,1211316991,CA @@ -13659,7 +13841,8 @@ 1296250496,1296250527,FR 1296250528,1296250559,DK 1296250560,1296250623,FR -1296250624,1296250879,BE +1296250624,1296250847,BE +1296250848,1296250879,GB 1296250880,1296251199,FR 1296251200,1296251231,NL 1296251232,1296251391,FR @@ -14163,8 +14346,7 @@ 1307636992,1307637247,LV 1307637248,1307637503,EE 1307637504,1307637759,LV -1307637760,1307639551,EE -1307639552,1307639807,LT +1307637760,1307639807,LT 1307639808,1307643903,IT 1307643904,1307652095,RU 1307652096,1307656191,ES @@ -14210,6 +14392,7 @@ 1307817984,1307818048,GB 1307818049,1307818049,BE 1307818050,1307818239,GB +1307819008,1307819263,GB 1307819520,1307819775,GB 1307820032,1307824127,ES 1307824128,1307828223,HU @@ -14240,8 +14423,7 @@ 1307926528,1307930623,KZ 1307930624,1307934719,RU 1307934720,1307938815,FR -1307938816,1307941119,US -1307941120,1307942911,TR +1307938816,1307942911,US 1307942912,1307947007,RU 1307947008,1307951103,CH 1307951104,1307959295,RU @@ -14617,7 +14799,7 @@ 1317928960,1317945343,BG 1317945344,1317978111,AT 1317978112,1317994495,RU -1317994496,1318010879,NL +1317994496,1318010879,DE 1318010880,1318027263,DK 1318027264,1318043647,IE 1318043648,1318584319,GB @@ -14779,7 +14961,6 @@ 1332609024,1332613119,PL 1332613120,1332617215,UA 1332617216,1332621311,CZ -1332621312,1332625407,UA 1332625408,1332629503,RU 1332629504,1332633599,DE 1332633600,1332637695,UA @@ -15179,7 +15360,6 @@ 1346412544,1346416639,RU 1346416640,1346420735,DE 1346420736,1346424831,NO -1346424832,1346428927,JO 1346428928,1346433023,FR 1346433024,1346439167,NL 1346439168,1346441215,ES @@ -15602,9 +15782,6 @@ 1347305472,1347309567,AL 1347309568,1347313663,DE 1347313664,1347321855,RU -1347321856,1347322111,KW -1347322112,1347322367,US -1347322368,1347325951,KW 1347325952,1347327743,CZ 1347327744,1347327999,SK 1347328000,1347330047,CZ @@ -15865,7 +16042,8 @@ 1348419584,1348427775,HU 1348427776,1348435967,CZ 1348435968,1348440063,FI -1348440064,1348448255,NL +1348440064,1348444159,DE +1348444160,1348448255,NL 1348448256,1348456447,GB 1348456448,1348460543,BH 1348460544,1348464639,SI @@ -15878,20 +16056,42 @@ 1348861952,1348993023,ES 1348993024,1349124095,IT 1349124096,1349255167,GR -1349255168,1349451775,AT +1349255168,1349400575,AT +1349400576,1349401087,DE +1349401088,1349451775,AT 1349451776,1349517311,IE 1349517312,1349763071,NL 1349763072,1349771263,RU 1349771264,1349779455,NL 1349779456,1349910527,IT 1349910528,1350041599,FR -1350041600,1350215679,AT +1350041600,1350067575,AT +1350067576,1350067579,CH +1350067580,1350110335,AT +1350110336,1350110463,DE +1350110464,1350140927,AT +1350140928,1350141055,CH +1350141056,1350166015,AT +1350166016,1350166271,SK +1350166272,1350188543,AT +1350188544,1350188799,DE +1350188800,1350195231,AT +1350195232,1350195235,DE +1350195236,1350215679,AT 1350215680,1350215935,IQ 1350215936,1350216959,AT 1350216960,1350217215,IQ 1350217216,1350217471,AT 1350217472,1350217727,IQ -1350217728,1350303743,AT +1350217728,1350230015,AT +1350230016,1350230271,DE +1350230272,1350251007,AT +1350251008,1350251263,HU +1350251264,1350259447,AT +1350259448,1350259451,CZ +1350259452,1350290223,AT +1350290224,1350290227,CH +1350290228,1350303743,AT 1350303744,1350434815,FR 1350434816,1350565887,NL 1350565888,1352299775,DE @@ -15996,6 +16196,7 @@ 1357347456,1357347583,FR 1357347616,1357347647,FR 1357347840,1357348095,PL +1357348384,1357348415,ES 1357351168,1357351423,PL 1357359872,1357360383,GB 1357361152,1357363199,GB @@ -16817,7 +17018,9 @@ 1365219168,1365219168,GB 1365219169,1365219391,NL 1365219392,1365219407,MY -1365219408,1365220231,NL +1365219408,1365219703,NL +1365219704,1365219711,US +1365219712,1365220231,NL 1365220232,1365220239,IE 1365220240,1365221239,NL 1365221240,1365221247,US @@ -17351,7 +17554,11 @@ 1385496576,1385504767,SI 1385504768,1385512959,IT 1385512960,1385521151,DE -1385521152,1385529343,AT +1385521152,1385523711,AT +1385523712,1385523967,HU +1385523968,1385524479,AT +1385524480,1385524735,HU +1385524736,1385529343,AT 1385529344,1385537535,RU 1385537536,1385545727,DE 1385545728,1385553919,RU @@ -17589,7 +17796,9 @@ 1389871104,1389887487,FI 1389887488,1389953023,FR 1389953024,1390018559,NL -1390018560,1390084095,AT +1390018560,1390074879,AT +1390074880,1390075135,HU +1390075136,1390084095,AT 1390084096,1390149631,GB 1390149632,1390215167,CH 1390215168,1390280703,IS @@ -17903,9 +18112,7 @@ 1403781120,1403797503,RU 1403797504,1403813887,SE 1403813888,1403830271,NL -1403830272,1403838719,CH -1403838720,1403838975,RU -1403838976,1403846655,CH +1403830272,1403846655,CH 1403846656,1403863039,IS 1403863040,1403879423,FR 1403879424,1403895807,ES @@ -18103,7 +18310,21 @@ 1404887040,1404927999,NL 1404928000,1404944383,SE 1404944384,1404960767,LT -1404960768,1405026303,SE +1404960768,1405009919,SE +1405009920,1405010687,LT +1405010688,1405010943,SE +1405010944,1405011199,LT +1405011200,1405011711,SE +1405011712,1405013759,LT +1405013760,1405014015,SE +1405014016,1405020159,LT +1405020160,1405020415,SE +1405020416,1405020927,LT +1405020928,1405021183,SE +1405021184,1405021439,LT +1405021440,1405021951,SE +1405021952,1405022207,LT +1405022208,1405026303,SE 1405026304,1405042687,NO 1405042688,1405048831,SE 1405048832,1405050879,HR @@ -18396,7 +18617,15 @@ 1410834432,1410842623,PL 1410842624,1410850815,PT 1410850816,1410859007,DE -1410859008,1411383295,NL +1410859008,1410972159,NL +1410972160,1410972415,DE +1410972416,1410972671,NL +1410972672,1410972927,DE +1410972928,1410973439,NL +1410973440,1410973695,DE +1410973696,1410975487,NL +1410975488,1410975743,DE +1410975744,1411383295,NL 1411383296,1411448831,LT 1411448832,1411449727,IT 1411449728,1411449791,DE @@ -18518,7 +18747,9 @@ 1415577600,1416101887,FR 1416101888,1416364031,NL 1416364032,1416626175,IL -1416626176,1416941567,AT +1416626176,1416876287,AT +1416876288,1416876543,CH +1416876544,1416941567,AT 1416941568,1416943615,CH 1416943616,1416944639,AT 1416944640,1416945663,CZ @@ -18527,11 +18758,67 @@ 1417019392,1417150463,DE 1417150464,1417674751,ES 1417674752,1421869055,DE -1421869056,1422393343,BE +1421869056,1421886975,BE +1421886976,1421887743,NL +1421887744,1421888511,BE +1421888512,1421888767,NL +1421888768,1421889279,BE +1421889280,1421889791,NL +1421889792,1421890047,BE +1421890048,1421890303,NL +1421890304,1421893119,BE +1421893120,1421893631,NL +1421893632,1421894143,BE +1421894144,1421894399,NL +1421894400,1421894783,BE +1421894784,1421894911,NL +1421894912,1421896063,BE +1421896064,1421896703,NL +1421896704,1421896831,BE +1421896832,1421896959,NL +1421896960,1421897727,BE +1421897728,1421898239,NL +1421898240,1421899263,BE +1421899264,1421899519,NL +1421899520,1421900543,BE +1421900544,1421900799,NL +1421900800,1421959679,BE +1421959680,1421959807,NL +1421959808,1421959935,BE +1421959936,1421960191,NL +1421960192,1422022143,BE +1422022144,1422022399,NL +1422022400,1422082559,BE +1422082560,1422083071,NL +1422083072,1422084223,BE +1422084224,1422084351,NL +1422084352,1422085119,BE +1422085120,1422085375,NL +1422085376,1422085887,BE +1422085888,1422086143,NL +1422086144,1422087679,BE +1422087680,1422088191,NL +1422088192,1422088447,BE +1422088448,1422088703,NL +1422088704,1422088959,BE +1422088960,1422089215,NL +1422089216,1422089471,BE +1422089472,1422089727,NL +1422089728,1422095743,BE +1422095744,1422095871,NL +1422095872,1422128639,BE +1422128640,1422129151,NL +1422129152,1422132479,BE +1422132480,1422132735,NL +1422132736,1422289663,BE +1422289664,1422289791,NL +1422289792,1422393343,BE 1422393344,1422413567,DE 1422413568,1422413695,AT 1422413696,1422413727,US -1422413728,1422491647,DE +1422413728,1422463231,DE +1422463232,1422463743,NL +1422463744,1422491647,DE 1422491648,1422495615,RU 1422495616,1422495679,NL 1422495680,1422508031,RU @@ -18561,7 +18848,9 @@ 1423966208,1424097279,HU 1424097280,1424228351,CH 1424228352,1424286719,IL -1424286720,1424287743,BG +1424286720,1424286938,BG +1424286939,1424286939,IL +1424286940,1424287743,BG 1424287744,1424359423,IL 1424359424,1424490495,FI 1424490496,1424503711,ES @@ -18579,7 +18868,9 @@ 1424600576,1424600831,FR 1424600832,1424602879,GB 1424602880,1424603135,US -1424603136,1424607743,GB +1424603136,1424604975,GB +1424604976,1424604983,NL +1424604984,1424607743,GB 1424607744,1424607775,DE 1424607776,1424617215,GB 1424617216,1424617231,IT @@ -18672,7 +18963,9 @@ 1425485312,1425489407,RO 1425489408,1425489663,NL 1425489664,1425506303,RO -1425506304,1425522687,NO +1425506304,1425518847,NO +1425518848,1425518975,RU +1425518976,1425522687,NO 1425522688,1425539071,IT 1425539072,1425801215,FI 1425801216,1425813759,BG @@ -19166,7 +19459,9 @@ 1439156107,1439156108,LB 1439156109,1439170559,GB 1439170560,1439236095,NO -1439236096,1439301631,BE +1439236096,1439261439,BE +1439261440,1439261695,FR +1439261696,1439301631,BE 1439301632,1439305727,RU 1439305728,1439309823,DK 1439309824,1439318015,PL @@ -19237,8 +19532,8 @@ 1440645120,1440669695,RS 1440669696,1440671743,NL 1440671744,1440672767,EE -1440672768,1440677887,RS -1440677888,1440710655,NL +1440672768,1440673791,RS +1440673792,1440710655,NL 1440710656,1440743423,UA 1440743424,1441267711,SE 1441267712,1441275903,DE @@ -19342,7 +19637,11 @@ 1442832384,1442836479,GB 1442836480,1442840575,PL 1442840576,1444937727,GB -1444937728,1445068799,AT +1444937728,1444985599,AT +1444985600,1444985855,DE +1444985856,1445061631,AT +1445061632,1445061887,DE +1445061888,1445068799,AT 1445068800,1445199871,RO 1445199872,1445330943,QA 1445330944,1445396479,LT @@ -19402,7 +19701,9 @@ 1449840640,1449852927,MD 1449852928,1449869311,RO 1449869312,1449870335,MD -1449870336,1449893887,RO +1449870336,1449883647,RO +1449883648,1449885695,BE +1449885696,1449893887,RO 1449893888,1449895935,MD 1449895936,1449918463,RO 1449918464,1449951231,JO @@ -19476,7 +19777,9 @@ 1466104468,1466104468,GB 1466104469,1466104942,FR 1466104943,1466104943,BE -1466104944,1466105343,FR +1466104944,1466105173,FR +1466105174,1466105174,BE +1466105175,1466105343,FR 1466105344,1466105599,GB 1466105600,1466105855,FR 1466105856,1466122239,PL @@ -19643,7 +19946,9 @@ 1475211264,1475213311,DE 1475213312,1475215359,FR 1475215360,1475223551,IT -1475223552,1475226495,SE +1475223552,1475226239,SE +1475226240,1475226367,NO +1475226368,1475226495,SE 1475226496,1475227647,NO 1475227648,1475231743,SE 1475231744,1475233791,NO @@ -19939,7 +20244,19 @@ 1482948608,1483210751,CZ 1483210752,1483735039,GB 1483735040,1483997183,FI -1483997184,1484128255,AT +1483997184,1484044287,AT +1484044288,1484044543,DE +1484044544,1484049911,AT +1484049912,1484049915,DE +1484049916,1484049921,AT +1484049922,1484049922,DE +1484049923,1484084607,AT +1484084608,1484084735,DE +1484084736,1484084991,AT +1484084992,1484085119,DE +1484085120,1484088831,AT +1484088832,1484089087,DE +1484089088,1484128255,AT 1484128256,1484259327,LT 1484259328,1484783615,FR 1484783616,1484849151,DE @@ -20265,7 +20582,6 @@ 1495203840,1495205887,DE 1495205888,1495207935,CZ 1495207936,1495209983,RU -1495209984,1495212031,KZ 1495212032,1495214079,RU 1495214080,1495216127,CZ 1495216128,1495220223,IT @@ -20319,7 +20635,9 @@ 1495468032,1495470079,MD 1495470080,1495476223,RO 1495476224,1495478271,MD -1495478272,1495492607,RO +1495478272,1495487487,RO +1495487488,1495488511,MD +1495488512,1495492607,RO 1495492608,1495494655,MD 1495494656,1495508991,RO 1495508992,1495510015,MD @@ -20329,7 +20647,9 @@ 1495571456,1495572479,MD 1495572480,1495608319,RO 1495608320,1495609343,MD -1495609344,1495670783,RO +1495609344,1495623679,RO +1495623680,1495623935,MD +1495623936,1495670783,RO 1495670784,1495671807,MD 1495671808,1495678975,RO 1495678976,1495679999,MD @@ -20337,7 +20657,9 @@ 1495682048,1495683071,MD 1495683072,1495747583,RO 1495747584,1495748607,MD -1495748608,1495749631,RO +1495748608,1495748863,RO +1495748864,1495749119,GB +1495749120,1495749631,RO 1495749632,1495750655,MD 1495750656,1495752703,RO 1495752704,1495755775,MD @@ -20347,11 +20669,15 @@ 1495759872,1495760127,MD 1495760128,1495762943,RO 1495762944,1495764991,MD -1495764992,1495790079,RO +1495764992,1495782655,RO +1495782656,1495782911,GB +1495782912,1495790079,RO 1495790080,1495790335,MD 1495790336,1495793663,RO 1495793664,1495795711,MD -1495795712,1495852031,RO +1495795712,1495845631,RO +1495845632,1495845887,GB +1495845888,1495852031,RO 1495852032,1495853055,MD 1495853056,1495875583,RO 1495875584,1495891967,MD @@ -20410,7 +20736,13 @@ 1499594752,1499725823,NL 1499725824,1499856895,IE 1499856896,1499987967,CZ -1499987968,1499996159,AT +1499987968,1499989247,AT +1499989248,1499989503,HU +1499989504,1499990911,AT +1499990912,1499991039,HU +1499991040,1499992575,AT +1499992576,1499992831,HU +1499992832,1499996159,AT 1499996160,1500004351,GB 1500004352,1500020735,RU 1500020736,1500028927,IS @@ -20486,7 +20818,11 @@ 1500479488,1500495871,RU 1500495872,1500512255,BA 1500512256,1500643327,RU -1500643328,1500774399,RO +1500643328,1500661247,PT +1500661248,1500661503,RO +1500661504,1500667647,PT +1500667648,1500667903,RO +1500667904,1500774399,PT 1500774400,1500905471,LT 1500905472,1501036543,IT 1501036544,1501298687,RO @@ -20635,9 +20971,7 @@ 1503898808,1503898815,PL 1503898816,1503898831,GB 1503898832,1503898839,RO -1503898840,1503898847,DE -1503898848,1503898855,MX -1503898856,1503898887,DE +1503898840,1503898887,DE 1503898888,1503898895,IT 1503898896,1503898927,DE 1503898928,1503898935,RO @@ -20882,7 +21216,9 @@ 1506445168,1506445183,DE 1506445184,1506446823,GB 1506446824,1506446831,NL -1506446832,1506450047,GB +1506446832,1506448319,GB +1506448320,1506448383,IT +1506448384,1506450047,GB 1506450048,1506450111,CH 1506450112,1506450863,GB 1506450864,1506450879,CZ @@ -20894,7 +21230,9 @@ 1506458245,1506458245,CH 1506458246,1506460151,GB 1506460152,1506460159,FR -1506460160,1506463679,GB +1506460160,1506462719,GB +1506462720,1506462975,IT +1506462976,1506463679,GB 1506463680,1506463695,DE 1506463696,1506464895,GB 1506464896,1506464911,NL @@ -20929,7 +21267,6 @@ 1506705408,1506727935,GB 1506727936,1506728959,FR 1506728960,1506740223,GB -1506740224,1506742271,FI 1506742272,1506744319,SE 1506744320,1506746367,NL 1506746368,1506750463,RU @@ -21367,20 +21704,7 @@ 1518407936,1518408191,SE 1518408192,1518409727,NL 1518409728,1518436351,SE -1518436352,1518439679,NO -1518439680,1518439935,SE -1518439936,1518441471,NO -1518441472,1518441727,SE -1518441728,1518444031,NO -1518444032,1518444287,SE -1518444288,1518444543,NO -1518444544,1518444799,SE -1518444800,1518445823,NO -1518445824,1518446335,SE -1518446336,1518448383,NO -1518448384,1518448895,SE -1518448896,1518452223,NO -1518452224,1518452735,SE +1518436352,1518452735,NO 1518452736,1518460927,AT 1518460928,1518470143,NL 1518470144,1518472191,SE @@ -21472,7 +21796,15 @@ 1518993408,1519190015,RU 1519190016,1519206399,SE 1519206400,1519208447,LV -1519208448,1519259647,SE +1519208448,1519214591,SE +1519214592,1519214847,LV +1519214848,1519215359,SE +1519215360,1519215615,LV +1519215616,1519215871,SE +1519215872,1519219455,LV +1519219456,1519219711,SE +1519219712,1519222783,LV +1519222784,1519259647,SE 1519259648,1519260671,NL 1519260672,1519263743,SE 1519263744,1519289343,NL @@ -21535,7 +21867,9 @@ 1520271360,1520304127,SI 1520304128,1520435199,TR 1520435200,1521483775,ES -1521483776,1522008063,CZ +1521483776,1521531391,CZ +1521531392,1521531647,PL +1521531648,1522008063,CZ 1522008064,1522139135,DK 1522139136,1522270207,DE 1522270208,1522401279,RU @@ -21556,7 +21890,9 @@ 1532199936,1532200959,RS 1532200960,1532231679,HU 1532231680,1532362751,GB -1532362752,1532493823,BE +1532362752,1532408063,BE +1532408064,1532408319,DE +1532408320,1532493823,BE 1532493824,1532559359,FR 1532559360,1532624895,DE 1532624896,1532626943,ES @@ -21701,7 +22037,29 @@ 1533929472,1533932799,GB 1533932800,1533933055,DE 1533933056,1534066687,GB -1534066688,1534328831,AT +1534066688,1534135295,AT +1534135296,1534135423,HU +1534135424,1534145279,AT +1534145280,1534145407,DE +1534145408,1534147711,AT +1534147712,1534147839,CZ +1534147840,1534156543,AT +1534156544,1534156671,SK +1534156672,1534241535,AT +1534241536,1534241663,DE +1534241664,1534268671,AT +1534268672,1534268927,HU +1534268928,1534279423,AT +1534279424,1534279551,SK +1534279552,1534298751,AT +1534298752,1534298879,HU +1534298880,1534299903,AT +1534299904,1534300159,HU +1534300160,1534302975,AT +1534302976,1534303231,HU +1534303232,1534303999,AT +1534304000,1534304255,SK +1534304256,1534328831,AT 1534328832,1534459903,ES 1534459904,1534590975,AT 1534590976,1534656511,HU @@ -21767,7 +22125,9 @@ 1534984192,1534985215,NO 1534985216,1535049727,ES 1535049728,1535115263,SK -1535115264,1535197183,AT +1535115264,1535146495,AT +1535146496,1535146751,CH +1535146752,1535197183,AT 1535197184,1535203071,EE 1535203072,1535246335,SE 1535246336,1535311871,AT @@ -21777,7 +22137,8 @@ 1535352832,1535361023,EE 1535361024,1535377407,NL 1535377408,1535442943,GR -1535442944,1535451135,FI +1535442944,1535450879,RU +1535450880,1535451135,FI 1535451136,1535459327,DK 1535459328,1535475711,AT 1535475712,1535508479,IR @@ -21795,7 +22156,9 @@ 1535602688,1535606783,HU 1535606784,1535610879,NO 1535610880,1535614975,RU -1535614976,1535619071,AT +1535614976,1535618559,AT +1535618560,1535618815,DE +1535618816,1535619071,AT 1535619072,1535623167,FR 1535623168,1535627263,KZ 1535627264,1535631359,RU @@ -21810,7 +22173,9 @@ 1535717376,1535721471,CH 1535721472,1535721727,DE 1535721728,1535737855,CH -1535737856,1535770623,HU +1535737856,1535746815,HU +1535746816,1535747071,RS +1535747072,1535770623,HU 1535770624,1535803391,CH 1535803392,1535836159,GR 1535836160,1535868927,CZ @@ -21912,7 +22277,9 @@ 1536684032,1536688127,GB 1536688128,1537212415,FI 1537212416,1538260991,FR -1538260992,1538785279,BE +1538260992,1538628607,BE +1538628608,1538629119,FR +1538629120,1538785279,BE 1538785280,1538793471,NL 1538793472,1538797567,DE 1538797568,1538801663,NL @@ -22946,7 +23313,6 @@ 1539927040,1539928063,UA 1539928064,1539930111,RU 1539930112,1539931135,PL -1539931136,1539932159,UA 1539932160,1539933183,DE 1539933184,1539934207,ES 1539934208,1539935231,RS @@ -23398,8 +23764,7 @@ 1540361472,1540361727,DE 1540361728,1540361983,IT 1540361984,1540362239,EE -1540362240,1540362495,RU -1540362496,1540363007,DE +1540362240,1540363263,DE 1540363264,1540363519,RU 1540363776,1540364031,IS 1540364032,1540364287,RU @@ -25027,7 +25392,7 @@ 1541163520,1541163775,DE 1541163776,1541164031,RO 1541164032,1541164287,RU -1541164288,1541164543,HR +1541164288,1541164543,CH 1541164544,1541164799,SI 1541164800,1541165055,IR 1541165056,1541165311,UA @@ -26030,7 +26395,7 @@ 1541707520,1541707775,NL 1541707776,1541708799,DE 1541708800,1541709823,PL -1541709824,1541710335,RU +1541710080,1541710335,RU 1541710336,1541710847,IL 1541710848,1541711871,SK 1541711872,1541712127,FR @@ -26269,7 +26634,7 @@ 1541836288,1541836543,AT 1541836800,1541837055,CZ 1541837056,1541837311,IT -1541837312,1541837823,RU +1541837312,1541837567,RU 1541837824,1541838079,PL 1541838080,1541839871,RU 1541839872,1541840383,PL @@ -26514,7 +26879,6 @@ 1541981952,1541982207,UA 1541982208,1541982719,RS 1541982720,1541984255,RU -1541984256,1541984511,UA 1541984512,1541984767,BG 1541984768,1541985279,RO 1541985280,1541986303,CZ @@ -26538,7 +26902,6 @@ 1541994496,1541995519,PL 1541995520,1541996031,RU 1541996032,1541996287,RO -1541996288,1541996543,RU 1541996544,1541997567,RO 1541997568,1541997823,RU 1541997824,1541998079,GB @@ -27693,7 +28056,7 @@ 1546088448,1546092543,GB 1546092544,1546096639,RU 1546096640,1546100735,IT -1546100736,1546104831,AT +1546100736,1546104831,DE 1546104832,1546108927,IE 1546108928,1546113023,IM 1546113024,1546121215,RU @@ -27853,7 +28216,9 @@ 1547565312,1547565823,US 1547565824,1547567103,NL 1547567104,1547571199,GB -1547571200,1547575295,AT +1547571200,1547572991,AT +1547572992,1547573247,HU +1547573248,1547575295,AT 1547575296,1547579391,NO 1547579392,1547583487,RU 1547583488,1547587583,KG @@ -27915,7 +28280,9 @@ 1547685888,1547689983,AT 1547689984,1547694079,IT 1547694080,1547698175,HU -1547698176,1548130303,NL +1547698176,1547925783,NL +1547925784,1547925791,DE +1547925792,1548130303,NL 1548130304,1548130559,BE 1548130560,1548158599,NL 1548158600,1548158607,GB @@ -28068,7 +28435,9 @@ 1554054656,1556086783,FR 1556086784,1557921791,DE 1557921792,1558052863,NO -1558052864,1558118399,FR +1558052864,1558054399,FR +1558054400,1558054655,DE +1558054656,1558118399,FR 1558118400,1558119423,DE 1558119424,1558122495,RU 1558122496,1558151167,AT @@ -28147,7 +28516,10 @@ 1560051712,1560084479,RU 1560084480,1560117247,JO 1560117248,1560133631,CZ -1560133632,1560137727,RU +1560133632,1560135679,RU +1560135680,1560135807,UA +1560135808,1560135935,CZ +1560135936,1560137727,RU 1560137728,1560150015,CZ 1560150016,1560182783,NL 1560182784,1560215551,SE @@ -28156,7 +28528,17 @@ 1562378240,1564999679,IT 1564999680,1565523967,UA 1565523968,1565655039,RU -1565655040,1565786111,AT +1565655040,1565673215,AT +1565673216,1565673471,DE +1565673472,1565675263,AT +1565675264,1565675391,DE +1565675392,1565713295,AT +1565713296,1565713299,CH +1565713300,1565718919,AT +1565718920,1565718927,DE +1565718928,1565723135,AT +1565723136,1565723391,HU +1565723392,1565786111,AT 1565786112,1565917183,BY 1565917184,1566048255,RS 1566048256,1566056447,RU @@ -28375,7 +28757,9 @@ 1567713280,1567714303,MD 1567714304,1567715327,RO 1567715328,1567717375,MD -1567717376,1567742975,RO +1567717376,1567727359,RO +1567727360,1567727615,GB +1567727616,1567742975,RO 1567742976,1567743487,MD 1567743488,1567749119,RO 1567749120,1567750143,MD @@ -28403,9 +28787,12 @@ 1567984640,1567987711,MD 1567987712,1567988735,RO 1567988736,1567992831,MD -1567992832,1567997951,RO +1567992832,1567993343,RO +1567993344,1567993599,GB +1567993600,1567997951,RO 1567997952,1568014335,NL -1568014336,1568026623,RO +1568014336,1568022527,DE +1568022528,1568026623,RO 1568026624,1568030719,MD 1568030720,1568059391,RO 1568059392,1568063487,MD @@ -28593,7 +28980,8 @@ 1571436288,1571436543,UA 1571436544,1571438591,RU 1571438592,1571440639,UA -1571440640,1571441407,CZ +1571440640,1571440895,RU +1571440896,1571441407,CZ 1571441408,1571441663,UA 1571441664,1571441919,RU 1571441920,1571442175,CZ @@ -28613,8 +29001,8 @@ 1571448832,1571449343,NL 1571449344,1571449855,CZ 1571449856,1571450879,RU -1571450880,1571451903,UA -1571451904,1571453951,CZ +1571450880,1571452927,UA +1571452928,1571453951,CZ 1571453952,1571455999,RU 1571456000,1571456511,UA 1571456512,1571456767,CZ @@ -28630,8 +29018,8 @@ 1571471104,1571471359,CZ 1571471360,1571475455,RU 1571475456,1571476479,CZ -1571476480,1571477503,RU -1571477504,1571487743,CZ +1571476480,1571478527,RU +1571478528,1571487743,CZ 1571487744,1571495935,SK 1571495936,1571504127,CZ 1571504128,1571508223,UA @@ -28647,8 +29035,7 @@ 1571529728,1571530239,RU 1571530240,1571531263,CZ 1571531264,1571531775,UA -1571531776,1571532799,RU -1571532800,1571534079,CZ +1571531776,1571534079,CZ 1571534080,1571534847,RU 1571534848,1571535103,LV 1571535104,1571535617,CZ @@ -28658,7 +29045,9 @@ 1571538944,1571540991,CZ 1571540992,1571541247,RU 1571541248,1571541503,UA -1571541504,1571543039,CZ +1571541504,1571542527,CZ +1571542528,1571542783,SK +1571542784,1571543039,CZ 1571543040,1571543551,NL 1571543552,1571543807,UA 1571543808,1571544063,CZ @@ -29011,7 +29400,9 @@ 1578500096,1578565631,FI 1578565632,1578582015,FR 1578582016,1578584063,IT -1578584064,1578585087,FR +1578584064,1578584355,FR +1578584356,1578584356,IT +1578584357,1578585087,FR 1578585088,1578586111,PT 1578586112,1578588159,ES 1578588160,1578590207,PL @@ -29030,11 +29421,15 @@ 1578592184,1578592191,PT 1578592192,1578592199,BE 1578592200,1578592207,CH -1578592208,1578593023,FR +1578592208,1578592295,FR +1578592296,1578592303,NL +1578592304,1578593023,FR 1578593024,1578593279,DE 1578593280,1578593439,FR 1578593440,1578593443,ES -1578593444,1578593955,FR +1578593444,1578593551,FR +1578593552,1578593559,NL +1578593560,1578593955,FR 1578593956,1578593959,ES 1578593960,1578595367,FR 1578595368,1578595371,IT @@ -29090,7 +29485,19 @@ 1580466176,1580597247,RO 1580597248,1580728319,TR 1580728320,1580990463,AE -1580990464,1581252607,RO +1580990464,1580999679,PT +1580999680,1580999935,RO +1580999936,1581001215,PT +1581001216,1581001471,RO +1581001472,1581118463,PT +1581118464,1581118719,RO +1581118720,1581126655,PT +1581126656,1581127167,RO +1581127168,1581132031,PT +1581132032,1581132287,RO +1581132288,1581208575,PT +1581208576,1581208831,RO +1581208832,1581252607,PT 1581252608,1581776895,GR 1581776896,1581793279,RU 1581793280,1581809663,PL @@ -29176,9 +29583,7 @@ 1583767552,1583771647,AT 1583771648,1583775743,RU 1583775744,1583779839,IT -1583779840,1583779847,GB -1583779848,1583779855,IT -1583779856,1583780263,GB +1583779840,1583780263,GB 1583780264,1583780271,IT 1583780272,1583780335,GB 1583780336,1583780343,IT @@ -29386,7 +29791,9 @@ 1586151424,1586159615,TR 1586159616,1586167807,MT 1586167808,1586175999,DE -1586176000,1586184191,BE +1586176000,1586177151,BE +1586177152,1586177279,DE +1586177280,1586184191,BE 1586184192,1586192383,NO 1586192384,1586200575,RU 1586200576,1586208767,MD @@ -29483,7 +29890,6 @@ 1587449856,1587453951,UA 1587453952,1587470335,RU 1587470336,1587474431,PL -1587474432,1587478527,UA 1587478528,1587511295,RU 1587511296,1587544063,IL 1587544064,1588068351,IT @@ -29554,8 +29960,8 @@ 1590073344,1590075391,NL 1590075392,1590077439,BE 1590077440,1590079487,GB -1590079488,1590079743,GP -1590079744,1590081535,FR +1590079488,1590080511,GP +1590080512,1590081535,MQ 1590081536,1590083583,GB 1590083584,1590085631,RU 1590085632,1590087679,FR @@ -29595,7 +30001,9 @@ 1590165504,1590689791,AE 1590689792,1591214079,NL 1591214080,1591738367,DE -1591738368,1592000511,BE +1591738368,1591977471,BE +1591977472,1591977599,NL +1591977600,1592000511,BE 1592000512,1592004607,ES 1592004608,1592008703,AM 1592008704,1592012799,GB @@ -29795,7 +30203,9 @@ 1596890624,1596891135,CZ 1596891136,1596900351,RU 1596900352,1596907519,BY -1596907520,1596915711,RU +1596907520,1596909567,RU +1596909568,1596911615,KZ +1596911616,1596915711,RU 1596915712,1596925951,CZ 1596925952,1596932095,RU 1596932096,1596940543,CZ @@ -29812,7 +30222,8 @@ 1596948480,1596950527,BY 1596950528,1596950783,UA 1596950784,1596951551,CZ -1596951552,1596952063,NL +1596951552,1596951807,RU +1596951808,1596952063,NL 1596952064,1596952575,UA 1596952576,1596952831,RU 1596952832,1596953599,CZ @@ -29880,7 +30291,9 @@ 1599127552,1599143935,CZ 1599143936,1599160319,UA 1599160320,1599176703,IR -1599176704,1599193087,FR +1599176704,1599188991,FR +1599188992,1599189503,DE +1599189504,1599193087,FR 1599193088,1599209471,RU 1599209472,1599242239,IR 1599242240,1599258623,CZ @@ -30056,7 +30469,8 @@ 1602781184,1602813951,FR 1602813952,1602846719,RU 1602846720,1602879487,GE -1602879488,1602879743,RE +1602879488,1602879488,MQ +1602879489,1602879743,RE 1602879744,1602882303,MQ 1602882304,1602882559,RE 1602882560,1602885631,MQ @@ -30168,7 +30582,11 @@ 1603252224,1603256319,RU 1603256320,1603260415,SE 1603260416,1603264511,RU -1603264512,1603268607,AT +1603264512,1603266559,AT +1603266560,1603266815,HU +1603266816,1603267711,AT +1603267712,1603267839,SK +1603267840,1603268607,AT 1603268608,1603272703,PL 1603272704,1603796991,GB 1603796992,1603813375,RU @@ -30272,7 +30690,9 @@ 1605112648,1605112655,IT 1605112656,1605112847,GB 1605112848,1605112855,IT -1605112856,1605113087,GB +1605112856,1605112983,GB +1605112984,1605112991,IT +1605112992,1605113087,GB 1605113088,1605113095,IT 1605113096,1605113383,GB 1605113384,1605113391,IT @@ -30290,9 +30710,7 @@ 1605114328,1605114335,IT 1605114336,1605114871,GB 1605114872,1605114879,IT -1605114880,1605114959,GB -1605114960,1605114967,IT -1605114968,1605115007,GB +1605114880,1605115007,GB 1605115008,1605115015,IT 1605115016,1605115599,GB 1605115600,1605115607,IT @@ -30408,9 +30826,13 @@ 1607622656,1607624703,ES 1607624704,1607625727,IT 1607625728,1607625983,ES -1607625984,1607626239,IT +1607625984,1607626121,IT +1607626122,1607626122,ES +1607626123,1607626239,IT 1607626240,1607626751,ES -1607626752,1607647231,IT +1607626752,1607640805,IT +1607640806,1607640806,CH +1607640807,1607647231,IT 1607647232,1607651327,DE 1607651328,1607655423,FR 1607655424,1607660287,IT @@ -30669,7 +31091,8 @@ 1701003264,1701011455,MY 1701011456,1701019647,CN 1701019648,1701052415,GU -1701052416,1701117951,NZ +1701052416,1701101567,NZ +1701101568,1701117951,SG 1701117952,1701134335,NC 1701134336,1701142527,CN 1701142528,1701143551,HK @@ -30704,7 +31127,9 @@ 1703411712,1703673855,TW 1703673856,1703935999,JP 1703936000,1704984575,CN -1704984576,1707081727,AU +1704984576,1705498623,AU +1705498624,1705500671,GB +1705500672,1707081727,AU 1707081728,1707737087,CN 1707737088,1707802623,KR 1707802624,1707835391,JP @@ -30952,7 +31377,6 @@ 1728420864,1728421887,NZ 1728421888,1728422911,JP 1728422912,1728423935,AU -1728423936,1728424959,TW 1728424960,1728425983,AF 1728425984,1728427007,JP 1728427008,1728428031,MY @@ -30991,7 +31415,7 @@ 1728458752,1728459775,HK 1728459776,1728460799,ID 1728460800,1728462847,JP -1728462848,1728463871,NZ +1728462848,1728463871,US 1728463872,1728464895,JP 1728464896,1728465919,KR 1728465920,1728466943,CN @@ -31204,7 +31628,6 @@ 1728668672,1728669695,SG 1728669696,1728670207,BD 1728670208,1728670463,NZ -1728670720,1728671743,JP 1728671744,1728672767,BD 1728672768,1728673791,TW 1728673792,1728674815,JP @@ -31504,7 +31927,8 @@ 1728951808,1728952063,SG 1728952064,1728952319,IN 1728952832,1728953343,AU -1728953344,1728954367,HK +1728953344,1728954112,GB +1728954113,1728954367,HK 1728954368,1728955391,JP 1728955392,1728956415,CN 1728956416,1728957439,ID @@ -31532,12 +31956,14 @@ 1728977920,1728978943,MY 1728978944,1728979967,JP 1728979968,1728980991,MN -1728980992,1728982015,NZ +1728980992,1728982015,AU 1728982016,1728982527,ID 1728982528,1728982783,IN 1728982784,1728985087,AU -1728985088,1728987135,PH -1728987136,1728988159,AU +1728985088,1728986111,SG +1728986112,1728987135,PH +1728987136,1728987391,JP +1728987392,1728988159,AU 1728988160,1728988191,NZ 1728988192,1728988199,US 1728988200,1728989183,NZ @@ -31645,7 +32071,7 @@ 1729078272,1729079295,BD 1729079296,1729080319,AU 1729080320,1729081343,ID -1729081344,1729081599,NP +1729081344,1729081599,IN 1729081600,1729082111,AU 1729082112,1729082367,IN 1729082368,1729083391,HK @@ -31773,6 +32199,7 @@ 1729189888,1729190911,VN 1729190912,1729191935,HK 1729191936,1729195007,IN +1729195008,1729195519,MN 1729195520,1729196031,BD 1729196032,1729197055,GU 1729197056,1729198079,HK @@ -32155,7 +32582,9 @@ 1729605632,1729606655,CN 1729606656,1729607679,ID 1729607680,1729609727,AU -1729609728,1729610751,HK +1729609728,1729609999,HK +1729610000,1729610003,JP +1729610004,1729610751,HK 1729610752,1729611775,JP 1729611776,1729612799,ID 1729613824,1729614847,SG @@ -32266,7 +32695,7 @@ 1729720320,1729721087,AU 1729721088,1729721343,IN 1729721344,1729722367,NZ -1729722368,1729726463,IN +1729723392,1729726463,IN 1729726464,1729727487,PK 1729727488,1729728511,KI 1729728512,1729729535,JP @@ -32418,9 +32847,12 @@ 1729876992,1729878015,JP 1729878016,1729879039,NZ 1729879040,1729879167,AU -1729879168,1729879679,JP +1729879168,1729879295,JP +1729879296,1729879423,KR +1729879424,1729879679,JP 1729879680,1729879807,AU -1729879808,1729880063,JP +1729879808,1729879935,JP +1729879936,1729880063,KR 1729880064,1729880831,AU 1729880832,1729881087,NZ 1729881088,1729881215,PH @@ -32619,7 +33051,9 @@ 1730074368,1730074623,SG 1730074624,1730075647,CN 1730075648,1730076671,ID -1730076672,1730080767,CN +1730077696,1730078719,CN +1730078720,1730079743,HK +1730079744,1730080767,CN 1730080768,1730081791,HK 1730081792,1730082815,ID 1730082816,1730083839,PW @@ -32691,8 +33125,8 @@ 1742737408,1742738431,SG 1742738432,1742738687,HK 1742738688,1742738943,AU -1742738944,1742739455,HK -1742739456,1742740479,IN +1742738944,1742739199,HK +1742739200,1742740479,IN 1742740480,1742741503,KH 1742741504,1742743551,IN 1742743552,1742745599,CN @@ -32819,8 +33253,8 @@ 1742869504,1742870015,IN 1742870016,1742870527,VU 1742870528,1742872575,IN -1742872576,1742873599,NZ -1742873600,1742874623,PH +1742872576,1742873855,NZ +1742873856,1742874623,PH 1742874624,1742875647,CN 1742875648,1742876671,JP 1742876672,1742877695,ID @@ -33053,7 +33487,10 @@ 1743108864,1743109119,AU 1743109120,1743110143,HK 1743110144,1743111167,VN -1743111168,1743112191,HK +1743111168,1743111423,SG +1743111424,1743111679,MY +1743111680,1743111935,ID +1743111936,1743112191,HK 1743112192,1743113215,ID 1743113216,1743114239,SG 1743114240,1743115263,IN @@ -33160,7 +33597,8 @@ 1743225856,1743226367,IN 1743226368,1743226623,PH 1743226624,1743226879,BD -1743226880,1743227903,AF +1743226880,1743227647,AF +1743227648,1743227903,IR 1743227904,1743228927,HK 1743228928,1743229951,CN 1743229952,1743230975,VN @@ -33174,7 +33612,7 @@ 1743236096,1743237119,US 1743237120,1743238143,BD 1743238144,1743240191,CN -1743240192,1743241215,VN +1743240192,1743241215,SG 1743241216,1743242239,JP 1743242240,1743244287,ID 1743244288,1743245311,AU @@ -33371,7 +33809,7 @@ 1743438592,1743438847,ID 1743438848,1743439871,JP 1743439872,1743440895,BD -1743440896,1743443967,TW +1743440896,1743441919,TW 1743443968,1743444991,ID 1743444992,1743446015,HK 1743446016,1743448063,TW @@ -33382,6 +33820,168 @@ 1743453184,1743454207,ID 1743454208,1743455231,IN 1743455232,1743456255,HK +1743456256,1743457279,IN +1743457280,1743458303,JP +1743458304,1743459327,CN +1743459328,1743459583,AU +1743459584,1743459839,ID +1743459840,1743460095,AU +1743460096,1743460351,IN +1743460352,1743461375,KR +1743461376,1743462399,MY +1743462400,1743463423,ID +1743463424,1743464447,AU +1743464448,1743465471,JP +1743465472,1743466495,HK +1743466496,1743467519,CN +1743467520,1743468543,AU +1743468544,1743470591,CN +1743470592,1743470847,IN +1743470848,1743471103,AU +1743471104,1743471615,ID +1743471616,1743472639,IN +1743472640,1743473663,KH +1743473664,1743474687,JP +1743474688,1743476735,CN +1743476736,1743477759,TW +1743477760,1743479807,AU +1743479808,1743480831,HK +1743480832,1743481855,IN +1743481856,1743482111,AU +1743482112,1743482367,ID +1743482368,1743482879,WF +1743482880,1743483903,JP +1743483904,1743484927,AU +1743484928,1743485951,IN +1743485952,1743486975,JP +1743486976,1743493119,CN +1743493120,1743493631,ID +1743493632,1743493887,AU +1743493888,1743495167,IN +1743495168,1743497215,JP +1743497216,1743499263,HK +1743499264,1743500287,AF +1743500288,1743501311,JP +1743501312,1743502335,CN +1743502336,1743503359,AU +1743503360,1743504383,CN +1743504384,1743505407,NP +1743505408,1743506431,CN +1743506432,1743506943,SG +1743506944,1743507455,IN +1743507456,1743509503,VN +1743509504,1743510527,HK +1743510528,1743545343,CN +1743585280,1743589375,CN +1743589376,1743590399,AU +1743590400,1743591423,KR +1743591424,1743593471,CN +1743593472,1743594495,ID +1743594496,1743595519,BD +1743595520,1743596543,NZ +1743596544,1743598591,IN +1743598592,1743599103,KH +1743599104,1743599615,HK +1743599616,1743600639,IN +1743600640,1743602687,VN +1743602688,1743603711,CN +1743603712,1743605759,HK +1743605760,1743606015,SG +1743606016,1743606527,AU +1743606528,1743606783,AF +1743606784,1743607807,PK +1743607808,1743608831,CN +1743608832,1743609855,AU +1743609856,1743610879,VN +1743610880,1743611903,MO +1743611904,1743613951,AU +1743613952,1743615999,IN +1743616000,1743617023,JP +1743617024,1743618047,MO +1743618048,1743619071,HK +1743619072,1743619583,SG +1743619584,1743619839,ID +1743619840,1743620095,IN +1743620096,1743621119,JP +1743621120,1743621631,ID +1743621632,1743622143,SG +1743622144,1743624191,VN +1743624192,1743625215,CN +1743625216,1743626239,IN +1743626240,1743627263,AU +1743627264,1743628287,NZ +1743628288,1743628799,SG +1743628800,1743629055,IN +1743629056,1743629311,AU +1743629312,1743630335,IN +1743630336,1743652863,CN +1743652864,1743654911,IN +1743654912,1743666175,CN +1743666176,1743666431,NZ +1743666432,1743666687,IN +1743666688,1743666943,AU +1743666944,1743667199,BD +1743667200,1743668223,KH +1743668224,1743672319,VN +1743672320,1743673343,NZ +1743673344,1743676415,CN +1743676416,1743679487,IN +1743679488,1743680511,JP +1743680512,1743681535,TL +1743681536,1743682559,AU +1743682560,1743683583,MY +1743683584,1743683839,JP +1743683840,1743684607,HK +1743684608,1743685631,CN +1743685632,1743686655,ID +1743686656,1743688703,CN +1743688704,1743689727,MY +1743689728,1743690751,IN +1743690752,1743691263,AS +1743691264,1743691775,IN +1743691776,1743699967,CN +1743699968,1743700479,IN +1743700480,1743700735,NZ +1743700736,1743700991,AU +1743700992,1743702015,CN +1743702016,1743703039,ID +1743703040,1743704063,CN +1743704064,1743706111,VN +1743706112,1743708159,IN +1743708160,1743709183,HK +1743709184,1743711231,IN +1743711232,1743712255,JP +1743712256,1743713279,IN +1743713280,1743714303,JP +1743714304,1743715327,KR +1743715328,1743717375,CN +1743717376,1743718399,BD +1743718400,1743719423,NZ +1743719424,1743720447,HK +1743720448,1743721471,JP +1743721472,1743722495,SG +1743722496,1743723519,AU +1743723520,1743724543,HK +1743724544,1743725567,VN +1743725568,1743726591,IN +1743726592,1743728639,CN +1743728640,1743729151,AU +1743729152,1743729407,IN +1743729408,1743729663,AU +1743729664,1743730687,KH +1743730688,1743731711,IN +1743731712,1743732735,TW +1743732736,1743733759,HK +1743733760,1743734783,CN +1743734784,1743735807,HK +1743735808,1743736319,AU +1743736320,1743736575,IN +1743736576,1743736831,AU +1743736832,1743738879,IN +1743738880,1743740927,JP +1743740928,1743741951,CN +1743741952,1743742975,HK +1743742976,1743743487,NZ 1743781888,1743783935,JP 1743783936,1743784959,IN 1743784960,1743785983,JP @@ -33407,7 +34007,7 @@ 1743806464,1743809535,IN 1743809536,1743810559,ID 1743810560,1743811583,KH -1743811584,1743813631,JP +1743811584,1743812607,JP 1743813632,1743814655,CN 1743814656,1743815679,SG 1743815680,1743815935,AU @@ -33631,8 +34231,8 @@ 1744056576,1744056831,MY 1744056832,1744057087,NZ 1744057088,1744057343,ID -1744057344,1744058367,HK -1744058368,1744066559,CN +1744057344,1744059391,HK +1744059392,1744066559,CN 1744066560,1744067583,HK 1744067584,1744068607,JP 1744068608,1744069631,ID @@ -33766,9 +34366,7 @@ 1744202240,1744202495,IN 1744202496,1744202751,ID 1744202752,1744203775,PK -1744203776,1744204287,MY -1744204288,1744204415,HK -1744204416,1744204799,MY +1744203776,1744204799,HK 1744204800,1744205823,ID 1744205824,1744207871,CN 1744207872,1744208127,AU @@ -33945,7 +34543,9 @@ 1744378880,1744379903,KR 1744379904,1744380927,HK 1744380928,1744383999,IN -1744384000,1744385023,AU +1744384000,1744384039,AU +1744384040,1744384047,GU +1744384048,1744385023,AU 1744385024,1744386047,CN 1744386048,1744386303,BD 1744386304,1744386559,AU @@ -33970,7 +34570,6 @@ 1744408576,1744409599,CN 1744409600,1744410623,AU 1744410624,1744411647,PK -1744411648,1744412671,JP 1744412672,1744413695,SG 1744413696,1744417791,CN 1744417792,1744418815,IN @@ -34108,7 +34707,8 @@ 1744555008,1744556031,CN 1744556032,1744562175,IN 1744562176,1744563199,BD -1744563200,1744565247,CN +1744563200,1744564223,HK +1744564224,1744565247,CN 1744565248,1744566271,HK 1744566272,1744567295,IN 1744567296,1744568319,JP @@ -34373,23 +34973,27 @@ 1747219456,1747220479,CA 1747220480,1747227647,US 1747227648,1747228671,CA -1747228672,1747229695,US -1747230720,1747256575,US +1747228672,1747256319,US +1747256320,1747256575,IE 1747256576,1747256831,AU -1747256832,1747259391,US +1747256832,1747257087,AE +1747257088,1747257343,IN +1747257344,1747260415,US 1747260416,1747261439,CA 1747261440,1747270655,US 1747270656,1747272703,CA 1747272704,1747273727,US 1747273728,1747274751,CA +1747274752,1747275775,US 1747275776,1747276799,CA -1747276800,1747277823,US -1747279872,1747283967,US +1747276800,1747283967,US 1747283968,1747284991,CA -1747284992,1747286015,US -1747288064,1747304447,US -1747306496,1747308543,CA -1747308544,1753251839,US +1747284992,1747293183,US +1747294208,1747304447,US +1747304448,1747308543,CA +1747308544,1747316735,US +1747316736,1747317759,CA +1747317760,1753251839,US 1753251840,1753252095,MN 1753252096,1753252351,SY 1753252352,1753252607,BY @@ -34411,11 +35015,124 @@ 1753280512,1753284607,CA 1753284608,1753309183,US 1753309184,1753317375,CA -1753333760,1753341951,US +1753317376,1753341951,US 1753341952,1753346047,CA -1753346048,1753481215,US -1753743360,1754071039,US +1753346048,1753483263,US +1753483264,1753483519,IE +1753483520,1753486335,US +1753486336,1753486591,IN +1753486592,1754136575,US 1754136576,1754169343,CA +1754169344,1754206719,US +1754206720,1754206975,GW +1754206976,1754207231,RW +1754207232,1754207487,GA +1754207488,1754207743,KI +1754207744,1754207999,MM +1754208000,1754208255,VU +1754208256,1754208511,NA +1754208512,1754208767,DJ +1754208768,1754209023,BF +1754209024,1754209279,BW +1754209280,1754209535,PW +1754209536,1754209791,BR +1754209792,1754210047,AR +1754210048,1754210303,BJ +1754210304,1754251519,US +1754251520,1754251775,LY +1754251776,1754252543,US +1754252544,1754252799,MZ +1754252800,1754253055,US +1754253056,1754253311,GN +1754253312,1754253567,US +1754253568,1754253823,ML +1754253824,1754254079,US +1754254080,1754254335,SO +1754254336,1754254591,CD +1754254592,1754254847,US +1754254848,1754255103,VE +1754255104,1754255359,US +1754255360,1754255615,EC +1754255616,1754255871,US +1754255872,1754256127,PY +1754256128,1754256383,US +1754256384,1754256639,BO +1754256640,1754256895,US +1754256896,1754257151,UY +1754257152,1754257407,US +1754257408,1754257663,TZ +1754257664,1754257919,US +1754257920,1754258175,TD +1754258176,1754258431,US +1754258432,1754258687,MG +1754258688,1754258943,US +1754258944,1754259199,TN +1754259200,1754792959,US +1754792960,1754793983,CA +1754793984,1754799103,US +1754799104,1754800127,CA +1754800128,1754804223,US +1754805248,1754822655,US +1754822656,1754823679,CA +1754823680,1754830847,US +1754830848,1754831871,CA +1754831872,1754832895,US +1754832896,1754836991,CA +1754836992,1754845183,US +1754845184,1754846207,CA +1754846208,1754849279,US +1754849280,1754850303,CA +1754850304,1754852351,US +1754852352,1754853375,JM +1754853376,1754863615,US +1754863616,1754864639,CA +1754864640,1754869759,US +1754869760,1754870783,CA +1754872832,1754890239,US +1754890240,1754892287,BB +1754923008,1755062271,US +1755062272,1755066367,CA +1755066368,1755070463,US +1755070464,1755074559,CA +1755074560,1755103487,US +1755103488,1755103743,MH +1755103744,1755103999,US +1755104000,1755104255,ZM +1755104256,1755104511,US +1755104512,1755104767,HT +1755104768,1755105023,US +1755105024,1755105279,BB +1755105280,1755106047,US +1755106048,1755106303,SR +1755106304,1755106559,US +1755106560,1755106815,BZ +1755106816,1755107071,US +1755107072,1755107327,MW +1755107328,1755107583,US +1755107584,1755107839,DM +1755107840,1755108095,US +1755108096,1755108351,NE +1755108352,1755108607,US +1755108608,1755108863,AG +1755108864,1755109119,US +1755109120,1755109375,TL +1755109376,1755109631,US +1755109632,1755109887,HN +1755109888,1755110143,US +1755110144,1755110399,SB +1755110400,1755110655,US +1755110656,1755110911,JO +1755110912,1755111167,US +1755111168,1755111423,SD +1755111424,1755119615,US +1755119616,1755283455,CA +1755283456,1755324415,US +1755324416,1755328511,CA +1755340800,1755348991,US +1755357184,1755365375,US +1755381760,1755512831,US +1755512832,1755545599,CA +1755578368,1755709439,US 1762656256,1763704831,MU 1763704832,1764753407,EG 1764753408,1765801983,KE @@ -34443,7 +35160,7 @@ 1777049600,1777053695,ZA 1777053696,1777057791,BF 1777057792,1777061887,NG -1777061888,1777063935,SD +1777061888,1777063935,SS 1777063936,1777065983,MZ 1777065984,1777070079,GQ 1777070080,1777074175,BW @@ -34488,7 +35205,9 @@ 1795559424,1795559679,AR 1795559680,1795560447,US 1795560448,1795560959,CA -1795560960,1795561471,US +1795560960,1795561247,US +1795561248,1795561343,CA +1795561344,1795561471,US 1795561472,1795562239,CA 1795562240,1795563263,US 1795563264,1795563519,CA @@ -34554,9 +35273,20 @@ 1807057664,1807057919,GB 1807057920,1807597567,US 1807597568,1807646719,CA -1807646720,1807695871,US +1807646720,1807655679,US +1807655680,1807655935,IE +1807655936,1807656191,LR +1807656192,1807656447,CZ +1807656448,1807656703,NL +1807656704,1807657983,US +1807657984,1807658239,SG +1807658240,1807658495,GB +1807658496,1807658751,BS +1807658752,1807695871,US 1807695872,1807699967,VI -1807699968,1807732735,US +1807699968,1807707311,US +1807707312,1807707312,JP +1807707313,1807732735,US 1807732736,1807736831,GP 1807736832,1815822335,US 1815822336,1815826431,CA @@ -35072,7 +35802,9 @@ 1835922456,1835922463,IT 1835922464,1835922559,GB 1835922560,1835922567,IT -1835922568,1835923351,GB +1835922568,1835922647,GB +1835922648,1835922655,IT +1835922656,1835923351,GB 1835923352,1835923359,IT 1835923360,1835923487,GB 1835923488,1835923495,IT @@ -35233,7 +35965,7 @@ 1841590272,1841594367,PL 1841594368,1841598463,RU 1841598464,1841602559,PL -1841602560,1841610751,UA +1841606656,1841610751,UA 1841610752,1841618943,RU 1841618944,1841627135,PL 1841627136,1841629183,NO @@ -35649,9 +36381,13 @@ 1847730176,1847732223,PK 1847734272,1847735295,NZ 1847735296,1847736319,AU -1847736320,1847738367,HK +1847736320,1847738367,JP 1847738368,1847754751,KR -1847754752,1847780351,TH +1847754752,1847757823,TH +1847757824,1847758847,CN +1847758848,1847770111,TH +1847770112,1847771135,SG +1847771136,1847780351,TH 1847780352,1847780607,CN 1847780608,1847783423,TH 1847783424,1847787519,US @@ -35702,7 +36438,8 @@ 1850400768,1850408959,JP 1850408960,1850490879,CN 1850490880,1850507263,KR -1850507264,1850510335,AU +1850507776,1850508031,AU +1850509312,1850510335,AU 1850510336,1850511359,KR 1850511360,1850513407,ID 1850513408,1850514431,TH @@ -35746,8 +36483,7 @@ 1855979520,1856241663,JP 1856241664,1856307199,TH 1856307200,1856315391,KR -1856315392,1856319487,HK -1856319488,1856323583,CN +1856315392,1856323583,CN 1856323584,1856339967,KR 1856339968,1856372735,JP 1856372736,1856503807,CN @@ -35848,7 +36584,8 @@ 1868300288,1868333055,IN 1868333056,1868341247,PK 1868341248,1868345343,ID -1868345344,1868346367,AU +1868345344,1868346111,GU +1868346112,1868346367,AU 1868346368,1868347391,TH 1868347392,1868348415,AU 1868348416,1868349439,KR @@ -35907,7 +36644,6 @@ 1876769792,1876770815,JP 1876770816,1876787199,SG 1876787200,1876885503,CN -1876885504,1876893695,PH 1876893696,1876901887,TH 1876901888,1876918271,SG 1876918272,1876934655,LK @@ -35940,7 +36676,8 @@ 1883770880,1883783167,KR 1883783168,1883799551,VN 1883799552,1883832319,KR -1883832320,1884028927,CN +1883832320,1883833855,HK +1883833856,1884028927,CN 1884028928,1884159999,KR 1884160000,1884164095,VN 1884164096,1884168191,TW @@ -35971,8 +36708,8 @@ 1886978048,1886986239,KR 1886986240,1886990335,TW 1886990336,1886994431,IN -1886994432,1887008767,TW -1887008768,1887010815,HK +1886994432,1887005695,TW +1887005696,1887010815,HK 1887010816,1887019007,TH 1887019008,1887027199,HK 1887027200,1887043583,KR @@ -35983,7 +36720,9 @@ 1887961088,1887993855,JP 1887993856,1888026623,KR 1888026624,1888030719,BD -1888030720,1888034815,HK +1888030720,1888030975,HK +1888030976,1888031231,GB +1888031232,1888034815,HK 1888034816,1888038911,JP 1888038912,1888040959,CN 1888040960,1888041471,JP @@ -36062,7 +36801,8 @@ 1897213952,1897218047,JP 1897218048,1897222143,IN 1897222144,1897226239,TW -1897226240,1897234431,NC +1897226240,1897231103,NC +1897231104,1897234431,VU 1897234432,1897242623,MY 1897242624,1897250815,TW 1897250816,1897259007,HK @@ -36170,7 +36910,9 @@ 1909161984,1909194751,PK 1909194752,1909456895,CN 1909456896,1909473279,JP -1909473280,1909481471,HK +1909473280,1909474303,HK +1909474304,1909475327,PH +1909475328,1909481471,HK 1909481472,1909587967,CN 1909587968,1909719039,MY 1909719040,1909735423,CN @@ -36211,7 +36953,7 @@ 1914601472,1914634239,KR 1914634240,1914642431,BD 1914642432,1914650623,KR -1914650624,1914652671,MN +1914652160,1914652415,MN 1914652672,1914654719,AU 1914654720,1914658815,JP 1914658816,1914660863,AU @@ -36219,7 +36961,8 @@ 1914662912,1914667007,KR 1914667008,1914683391,IN 1914683392,1914687487,AU -1914687488,1914689535,NZ +1914687488,1914688511,US +1914688512,1914689535,NZ 1914689536,1914691583,JP 1914691584,1914695679,IN 1914695680,1914697727,ID @@ -36256,10 +36999,12 @@ 1919909888,1919918079,AU 1919918080,1919926271,CN 1919926272,1919942655,KR -1919942656,1920006847,CN -1920006848,1920006911,HK -1920006912,1920069631,CN -1920069632,1920073727,HK +1919942656,1919999999,CN +1920000000,1920008191,HK +1920008192,1920057343,CN +1920057344,1920058111,HK +1920058112,1920072703,CN +1920072704,1920073727,HK 1920073728,1920466943,CN 1920466944,1920991231,ID 1920991232,1921056767,TH @@ -36277,14 +37022,15 @@ 1921253376,1921318911,CN 1921318912,1921384447,MY 1921384448,1921388543,NZ -1921388544,1921392639,PG 1921392640,1921400831,JP 1921400832,1921404927,ID 1921404928,1921405823,HK 1921405824,1921405855,MM 1921405856,1921405919,HK 1921405920,1921405951,MM -1921405952,1921406975,HK +1921405952,1921406463,HK +1921406464,1921406719,GB +1921406720,1921406975,HK 1921406976,1921409023,BD 1921409024,1921425407,JP 1921425408,1921431551,NZ @@ -36302,7 +37048,9 @@ 1921861632,1921863679,SG 1921865728,1921867775,ID 1921867776,1921871871,AU -1921871872,1921875967,US +1921871872,1921872895,US +1921872896,1921873663,NZ +1921873664,1921875967,US 1921875968,1921892351,CN 1921892352,1921896447,AU 1921896448,1921898495,SG @@ -36403,10 +37151,12 @@ 1937516544,1937518591,IN 1937518592,1937522687,JP 1937522688,1937530879,ID -1937530880,1937532927,US -1937532928,1937535999,IN +1937530880,1937534463,IN +1937534464,1937534975,GB +1937534976,1937535999,IN 1937536000,1937536511,GB -1937536512,1937539071,IN +1937536512,1937537023,US +1937537024,1937539071,IN 1937539072,1937637375,JP 1937637376,1937670143,HK 1937670144,1937672191,NZ @@ -36439,7 +37189,6 @@ 1940258816,1940275199,AU 1940275200,1940283391,CN 1940283392,1940291583,ID -1940291584,1940295679,HK 1940295680,1940324351,JP 1940324352,1940357119,PK 1940357120,1940389887,JP @@ -36480,7 +37229,8 @@ 1946163200,1946165247,CN 1946165248,1946173439,PK 1946173440,1946173695,PG -1946173696,1946176511,SG +1946173696,1946175487,SG +1946175488,1946176511,SY 1946176512,1946176767,PH 1946176768,1946178047,SG 1946178048,1946178303,HK @@ -36507,12 +37257,13 @@ 1949440000,1949442047,ID 1949442048,1949446143,TW 1949446144,1949448191,JP -1949448192,1949448703,HK +1949448192,1949448447,AU +1949448448,1949448703,HK 1949448704,1949448959,AU 1949448960,1949449215,IN 1949449216,1949449471,SG 1949449472,1949449727,JP -1949449728,1949450239,HK +1949449728,1949450239,AU 1949450240,1949466623,IN 1949466624,1949499391,PH 1949499392,1949564927,SG @@ -36528,7 +37279,12 @@ 1950089216,1950351359,CN 1950351360,1950482431,JP 1950482432,1950515199,CN -1950515200,1950523391,IN +1950515200,1950517247,IN +1950517248,1950518271,US +1950518272,1950519295,IN +1950519296,1950520319,US +1950520320,1950521343,PH +1950521344,1950523391,IN 1950523392,1950527487,AU 1950527488,1950531583,JP 1950531584,1950533631,NP @@ -36616,8 +37372,7 @@ 1958850560,1958852607,CN 1958852608,1958853631,AU 1958853632,1958854655,ID -1958854656,1958858751,AU -1958858752,1958860799,NZ +1958854656,1958860799,AU 1958860800,1958862847,BD 1958862848,1958871039,JP 1958871040,1959067647,CN @@ -36667,7 +37422,17 @@ 1960187904,1960189951,IN 1960189952,1960202239,CN 1960202240,1960206335,JP -1960206336,1960214527,SG +1960206336,1960210431,SG +1960210432,1960210943,TH +1960210944,1960211455,SG +1960211456,1960211903,AU +1960211904,1960212479,SG +1960212480,1960212582,IN +1960212583,1960212583,SG +1960212584,1960212735,IN +1960212736,1960214015,SG +1960214016,1960214271,IN +1960214272,1960214527,SG 1960214528,1960574975,CN 1960574976,1960837119,JP 1960837120,1961885695,CN @@ -36683,6 +37448,7 @@ 1962672128,1962803199,CN 1962803200,1962827775,JP 1962827776,1962829823,ID +1962829824,1962831871,JP 1962831872,1962835967,ID 1962835968,1962868735,CN 1962868736,1962885119,AU @@ -36694,7 +37460,9 @@ 1964113920,1964118015,HK 1964118016,1964120063,ID 1964120064,1964122111,JP -1964122112,1964126207,SG +1964122112,1964122367,SG +1964122368,1964122879,JP +1964122880,1964126207,SG 1964126208,1964126463,HK 1964126464,1964130303,SG 1964130304,1964134399,HK @@ -36763,7 +37531,7 @@ 1966800896,1967783935,CN 1967783936,1967800319,JP 1967800320,1967808511,CN -1967808512,1967812607,AT +1967808512,1967812607,AU 1967812608,1967816703,ID 1967816704,1969225727,CN 1969225728,1969487871,IN @@ -36886,7 +37654,7 @@ 1986498560,1986502655,HK 1986510848,1986519039,KR 1986519040,1986523135,PK -1986523136,1986525183,HK +1986523136,1986525183,CN 1986525184,1986527231,BN 1986527232,1986723839,JP 1986723840,1986740223,AU @@ -36925,7 +37693,9 @@ 1991376896,1991442431,CN 1991442432,1991499775,BD 1991499776,1991507967,NC -1991507968,1992097791,CN +1991507968,1991835647,CN +1991835648,1991901183,SG +1991901184,1992097791,CN 1992097792,1992163327,SG 1992163328,1992818687,CN 1992818688,1992949759,SG @@ -36945,7 +37715,7 @@ 1995702272,1996627967,CN 1996627968,1996630015,PH 1996630016,1996634111,ID -1996634112,1996636159,NP +1996634112,1996636159,AU 1996636160,1996644351,ID 1996644352,1996652543,BT 1996652544,1997078527,CN @@ -36992,9 +37762,13 @@ 1997723648,1997725695,JP 1997725696,1998061567,CN 1998061568,1998258175,JP -1998258176,1998271231,SG +1998258176,1998266367,SG +1998266368,1998268415,JP +1998268416,1998271231,SG 1998271232,1998271487,AU -1998271488,1998274559,SG +1998271488,1998271815,SG +1998271816,1998271823,JP +1998271824,1998274559,SG 1998274560,1998454783,CN 1998454784,1998456831,AU 1998456832,1998458879,JP @@ -37075,9 +37849,746 @@ 2001559552,2001567743,KR 2001567744,2001600511,TW 2001600512,2001797119,CN -2001797120,2001833007,SG +2001797120,2001798047,SG +2001798048,2001798079,US +2001798080,2001798087,SG +2001798088,2001798095,US +2001798096,2001798787,SG +2001798788,2001798791,AU +2001798792,2001799687,SG +2001799688,2001799695,US +2001799696,2001799711,SG +2001799712,2001799743,US +2001799744,2001799807,SG +2001799808,2001799935,US +2001799936,2001799967,SG +2001799968,2001799971,US +2001799972,2001800391,SG +2001800392,2001800399,US +2001800400,2001800927,SG +2001800928,2001800935,US +2001800936,2001801023,SG +2001801024,2001801039,US +2001801040,2001801055,SG +2001801056,2001801087,US +2001801088,2001801135,SG +2001801136,2001801151,US +2001801152,2001801339,SG +2001801340,2001801343,US +2001801344,2001801615,SG +2001801616,2001801623,NL +2001801624,2001801663,SG +2001801664,2001801703,US +2001801704,2001801711,SG +2001801712,2001801727,US +2001801728,2001801775,SG +2001801776,2001801807,US +2001801808,2001801919,SG +2001801920,2001801935,US +2001801936,2001801983,SG +2001801984,2001801999,US +2001802000,2001802191,SG +2001802192,2001802199,US +2001802200,2001802879,SG +2001802880,2001803007,US +2001803008,2001803215,SG +2001803216,2001803223,US +2001803224,2001803647,SG +2001803648,2001803679,US +2001803680,2001803735,SG +2001803736,2001803743,US +2001803744,2001803839,SG +2001803840,2001803871,US +2001803872,2001803903,SG +2001803904,2001803911,US +2001803912,2001803919,SG +2001803920,2001803935,US +2001803936,2001804015,SG +2001804016,2001804023,US +2001804024,2001805943,SG +2001805944,2001805951,US +2001805952,2001809383,SG +2001809384,2001809391,US +2001809392,2001810495,SG +2001810496,2001810527,MY +2001810528,2001810559,SG +2001810560,2001810623,MY +2001810624,2001812127,SG +2001812128,2001812159,SA +2001812160,2001812255,SG +2001812256,2001812263,US +2001812264,2001812671,SG +2001812672,2001812675,GB +2001812676,2001813655,SG +2001813656,2001813663,AU +2001813664,2001813783,SG +2001813784,2001813791,IN +2001813792,2001814431,SG +2001814432,2001814439,US +2001814440,2001814575,SG +2001814576,2001814583,US +2001814584,2001815191,SG +2001815192,2001815199,US +2001815200,2001817079,SG +2001817080,2001817087,US +2001817088,2001818639,SG +2001818640,2001818647,US +2001818648,2001818847,SG +2001818848,2001818851,AU +2001818852,2001819495,SG +2001819496,2001819503,US +2001819504,2001819887,SG +2001819888,2001819895,IE +2001819896,2001819999,SG +2001820000,2001820031,IE +2001820032,2001820719,SG +2001820720,2001820727,CA +2001820728,2001820991,SG +2001820992,2001820999,US +2001821000,2001821191,SG +2001821192,2001821199,US +2001821200,2001821335,SG +2001821336,2001821343,US +2001821344,2001821367,SG +2001821368,2001821375,US +2001821376,2001821423,SG +2001821424,2001821431,US +2001821432,2001821439,SG +2001821440,2001821447,US +2001821448,2001821623,SG +2001821624,2001821631,US +2001821632,2001823007,SG +2001823008,2001823015,US +2001823016,2001823447,SG +2001823448,2001823455,US +2001823456,2001824175,SG +2001824176,2001824183,US +2001824184,2001824287,SG +2001824288,2001824295,US +2001824296,2001824479,SG +2001824480,2001824487,US +2001824488,2001824495,SG +2001824496,2001824503,KR +2001824504,2001825183,SG +2001825184,2001825215,US +2001825216,2001825647,SG +2001825648,2001825655,US +2001825656,2001829891,SG +2001829892,2001829903,US +2001829904,2001830091,SG +2001830092,2001830095,US +2001830096,2001830175,SG +2001830176,2001830191,US +2001830192,2001830207,SG +2001830208,2001830223,US +2001830224,2001830335,SG +2001830336,2001830351,US +2001830352,2001830479,SG +2001830480,2001830495,US +2001830496,2001830567,SG +2001830568,2001830575,US +2001830576,2001830673,SG +2001830674,2001830674,HK +2001830675,2001830719,SG +2001830720,2001830783,US +2001830784,2001830927,SG +2001830928,2001830935,US +2001830936,2001830943,NL +2001830944,2001830975,SG +2001830976,2001830983,US +2001830984,2001831007,SG +2001831008,2001831015,US +2001831016,2001831027,SG +2001831028,2001831031,US +2001831032,2001831163,SG +2001831164,2001831167,US +2001831168,2001831287,SG +2001831288,2001831295,US +2001831296,2001831331,SG +2001831332,2001831335,US +2001831336,2001831683,SG +2001831684,2001831687,US +2001831688,2001831691,SG +2001831692,2001831703,US +2001831704,2001831915,SG +2001831916,2001831919,US +2001831920,2001831943,SG +2001831944,2001831951,US +2001831952,2001831963,SG +2001831964,2001831967,US +2001831968,2001831992,SG +2001831993,2001831993,US +2001831994,2001832135,SG +2001832136,2001832139,US +2001832140,2001832147,SG +2001832148,2001832151,US +2001832152,2001832315,SG +2001832316,2001832319,US +2001832320,2001832443,SG +2001832444,2001832447,US +2001832448,2001832615,SG +2001832616,2001832623,AZ +2001832624,2001832751,SG +2001832752,2001832759,IE +2001832760,2001832879,SG +2001832880,2001832887,US +2001832888,2001832903,SG +2001832904,2001832911,US +2001832912,2001832951,SG +2001832952,2001832955,US +2001832956,2001833007,SG 2001833008,2001833015,CN -2001833016,2001862655,SG +2001833016,2001833071,SG +2001833072,2001833079,US +2001833080,2001833099,SG +2001833100,2001833103,US +2001833104,2001833255,SG +2001833256,2001833263,US +2001833264,2001833419,SG +2001833420,2001833421,US +2001833422,2001833439,SG +2001833440,2001833447,US +2001833448,2001833487,SG +2001833488,2001833495,US +2001833496,2001833655,SG +2001833656,2001833663,GB +2001833664,2001833679,SG +2001833680,2001833687,GB +2001833688,2001833727,SG +2001833728,2001833791,US +2001833792,2001834096,SG +2001834097,2001834097,US +2001834098,2001834239,SG +2001834240,2001834495,HK +2001834496,2001834751,SG +2001834752,2001834815,US +2001834816,2001834935,SG +2001834936,2001834943,JP +2001834944,2001834947,US +2001834948,2001835035,SG +2001835036,2001835039,US +2001835040,2001835103,SG +2001835104,2001835135,US +2001835136,2001835147,SG +2001835148,2001835151,US +2001835152,2001835231,SG +2001835232,2001835239,US +2001835240,2001835247,SG +2001835248,2001835263,US +2001835264,2001835515,SG +2001835516,2001835519,US +2001835520,2001835587,SG +2001835588,2001835591,US +2001835592,2001835711,SG +2001835712,2001835719,US +2001835720,2001835793,SG +2001835794,2001835794,US +2001835795,2001835795,SG +2001835796,2001835799,US +2001835800,2001835959,SG +2001835960,2001835983,US +2001835984,2001836107,SG +2001836108,2001836111,US +2001836112,2001836267,SG +2001836268,2001836271,US +2001836272,2001836291,SG +2001836292,2001836295,US +2001836296,2001836419,SG +2001836420,2001836423,US +2001836424,2001836427,SG +2001836428,2001836431,US +2001836432,2001836435,SG +2001836436,2001836439,US +2001836440,2001836451,SG +2001836452,2001836455,US +2001836456,2001836471,SG +2001836472,2001836487,US +2001836488,2001836506,SG +2001836507,2001836507,US +2001836508,2001836543,SG +2001836544,2001838079,US +2001838080,2001838627,SG +2001838628,2001838631,US +2001838632,2001838633,SG +2001838634,2001838634,HK +2001838635,2001838755,SG +2001838756,2001838759,US +2001838760,2001838763,SG +2001838764,2001838767,US +2001838768,2001838847,SG +2001838848,2001839103,HK +2001839104,2001839111,SG +2001839112,2001839115,US +2001839116,2001839535,SG +2001839536,2001839543,US +2001839544,2001839595,SG +2001839596,2001839599,SA +2001839600,2001839739,SG +2001839740,2001839743,US +2001839744,2001839775,SG +2001839776,2001839783,US +2001839784,2001839803,SG +2001839804,2001839807,US +2001839808,2001840057,SG +2001840058,2001840059,SA +2001840060,2001840071,SG +2001840072,2001840079,US +2001840080,2001840175,SG +2001840176,2001840183,US +2001840184,2001840767,SG +2001840768,2001840799,US +2001840800,2001841159,SG +2001841160,2001841167,US +2001841168,2001841175,SG +2001841176,2001841176,US +2001841177,2001841177,SA +2001841178,2001841179,SG +2001841180,2001841247,US +2001841248,2001841279,SA +2001841280,2001841311,SG +2001841312,2001841319,US +2001841320,2001841343,SG +2001841344,2001841407,US +2001841408,2001841639,SG +2001841640,2001841647,US +2001841648,2001842047,SG +2001842048,2001842111,US +2001842112,2001842175,SG +2001842176,2001842271,US +2001842272,2001842319,SG +2001842320,2001842327,US +2001842328,2001842343,SG +2001842344,2001842351,US +2001842352,2001842687,SG +2001842688,2001842751,US +2001842752,2001842783,SG +2001842784,2001842815,US +2001842816,2001843249,SG +2001843250,2001843251,US +2001843252,2001843471,SG +2001843472,2001843475,US +2001843476,2001843839,SG +2001843840,2001843967,US +2001843968,2001844127,SG +2001844128,2001845247,US +2001845248,2001845335,SG +2001845336,2001845343,US +2001845344,2001845359,SG +2001845360,2001845375,US +2001845376,2001845391,SG +2001845392,2001845503,US +2001845504,2001845695,SG +2001845696,2001845703,US +2001845704,2001845799,SG +2001845800,2001845807,US +2001845808,2001846015,SG +2001846016,2001846311,US +2001846312,2001846335,SG +2001846336,2001846359,US +2001846360,2001846479,SG +2001846480,2001846495,US +2001846496,2001846503,SG +2001846504,2001846511,US +2001846512,2001846519,SG +2001846520,2001846523,US +2001846524,2001846531,SG +2001846532,2001846543,US +2001846544,2001846559,SG +2001846560,2001846567,US +2001846568,2001846571,SG +2001846572,2001846607,US +2001846608,2001846615,SG +2001846616,2001846727,US +2001846728,2001846735,SG +2001846736,2001846755,US +2001846756,2001846847,SG +2001846848,2001846911,US +2001846912,2001846927,SG +2001846928,2001846943,US +2001846944,2001846959,SG +2001846960,2001846975,IN +2001846976,2001847071,SG +2001847072,2001847103,US +2001847104,2001847167,SG +2001847168,2001847416,US +2001847417,2001847417,SG +2001847418,2001847429,US +2001847430,2001847439,SG +2001847440,2001847455,US +2001847456,2001847463,SG +2001847464,2001847479,US +2001847480,2001847487,SG +2001847488,2001847491,US +2001847492,2001847493,SG +2001847494,2001847495,US +2001847496,2001847519,SG +2001847520,2001847535,US +2001847536,2001847537,SG +2001847538,2001847559,US +2001847560,2001847567,SG +2001847568,2001847575,US +2001847576,2001847591,SG +2001847592,2001847599,US +2001847600,2001847623,SG +2001847624,2001847647,US +2001847648,2001847663,SG +2001847664,2001847671,US +2001847672,2001847679,SG +2001847680,2001847703,US +2001847704,2001847711,SG +2001847712,2001847727,US +2001847728,2001847743,SG +2001847744,2001847747,US +2001847748,2001847755,SG +2001847756,2001847771,US +2001847772,2001847783,SG +2001847784,2001847791,US +2001847792,2001847799,SG +2001847800,2001847839,US +2001847840,2001847855,SG +2001847856,2001847871,US +2001847872,2001847875,SG +2001847876,2001847879,US +2001847880,2001847895,SG +2001847896,2001847903,US +2001847904,2001847935,SG +2001847936,2001847951,US +2001847952,2001847955,SG +2001847956,2001847959,US +2001847960,2001847963,SG +2001847964,2001847967,US +2001847968,2001847975,SG +2001847976,2001847979,US +2001847980,2001847981,SG +2001847982,2001847991,US +2001847992,2001848007,SG +2001848008,2001848015,US +2001848016,2001848047,SG +2001848048,2001848055,US +2001848056,2001848071,SG +2001848072,2001848079,IN +2001848080,2001848103,SG +2001848104,2001848111,US +2001848112,2001848135,SG +2001848136,2001848143,US +2001848144,2001848151,SG +2001848152,2001848167,US +2001848168,2001848189,SG +2001848190,2001848199,US +2001848200,2001848207,SG +2001848208,2001848231,US +2001848232,2001848247,SG +2001848248,2001848253,US +2001848254,2001848254,SG +2001848255,2001848255,MY +2001848256,2001848279,SG +2001848280,2001848287,US +2001848288,2001848303,SG +2001848304,2001848327,US +2001848328,2001848335,SG +2001848336,2001848367,US +2001848368,2001848383,SG +2001848384,2001848407,US +2001848408,2001848415,SG +2001848416,2001848447,US +2001848448,2001848479,SG +2001848480,2001848575,US +2001848576,2001848623,SG +2001848624,2001848679,US +2001848680,2001848687,SG +2001848688,2001848767,US +2001848768,2001848863,SG +2001848864,2001848879,US +2001848880,2001848895,SG +2001848896,2001848943,US +2001848944,2001848951,SG +2001848952,2001848975,US +2001848976,2001848991,SG +2001848992,2001849015,US +2001849016,2001849023,SG +2001849024,2001849167,US +2001849168,2001849183,SG +2001849184,2001849247,US +2001849248,2001849255,SG +2001849256,2001849271,US +2001849272,2001849279,SG +2001849280,2001850391,US +2001850392,2001850399,SG +2001850400,2001850415,US +2001850416,2001850423,SG +2001850424,2001850435,US +2001850436,2001850439,SG +2001850440,2001850515,US +2001850516,2001850517,SG +2001850518,2001850519,US +2001850520,2001850543,SG +2001850544,2001850551,US +2001850552,2001850591,SG +2001850592,2001850599,US +2001850600,2001850607,SG +2001850608,2001850624,US +2001850625,2001850625,SG +2001850626,2001850655,US +2001850656,2001850663,SG +2001850664,2001850667,US +2001850668,2001850679,SG +2001850680,2001850727,US +2001850728,2001850735,SG +2001850736,2001850791,US +2001850792,2001850795,SG +2001850796,2001850823,US +2001850824,2001850847,SG +2001850848,2001850863,US +2001850864,2001850867,SG +2001850868,2001850871,US +2001850872,2001850911,SG +2001850912,2001850935,US +2001850936,2001851023,SG +2001851024,2001851083,US +2001851084,2001851087,SG +2001851088,2001851095,US +2001851096,2001851099,SG +2001851100,2001851159,US +2001851160,2001851167,SG +2001851168,2001851175,US +2001851176,2001851231,SG +2001851232,2001851239,US +2001851240,2001851247,SG +2001851248,2001851255,US +2001851256,2001851271,SG +2001851272,2001851287,US +2001851288,2001851295,SG +2001851296,2001851303,US +2001851304,2001851311,SG +2001851312,2001851407,US +2001851408,2001851423,SG +2001851424,2001851479,US +2001851480,2001851487,SG +2001851488,2001851503,US +2001851504,2001851527,SG +2001851528,2001851535,US +2001851536,2001851551,SG +2001851552,2001851591,US +2001851592,2001851599,SG +2001851600,2001851631,US +2001851632,2001851647,SG +2001851648,2001851655,US +2001851656,2001851663,SG +2001851664,2001851711,US +2001851712,2001851727,SG +2001851728,2001851743,US +2001851744,2001851759,SG +2001851760,2001851767,US +2001851768,2001851771,SG +2001851772,2001851775,US +2001851776,2001851783,SG +2001851784,2001851807,US +2001851808,2001851815,SG +2001851816,2001851823,US +2001851824,2001851831,MY +2001851832,2001851883,US +2001851884,2001851903,SG +2001851904,2001851905,US +2001851906,2001851927,SG +2001851928,2001851935,US +2001851936,2001851967,SG +2001851968,2001851975,US +2001851976,2001851991,SG +2001851992,2001851999,US +2001852000,2001852007,SG +2001852008,2001852015,US +2001852016,2001852031,SG +2001852032,2001852055,US +2001852056,2001852065,SG +2001852066,2001852067,PH +2001852068,2001852103,US +2001852104,2001852111,SG +2001852112,2001852127,US +2001852128,2001852135,SG +2001852136,2001852167,US +2001852168,2001852175,SG +2001852176,2001852183,US +2001852184,2001852207,SG +2001852208,2001852215,US +2001852216,2001852257,SG +2001852258,2001852259,US +2001852260,2001852263,SG +2001852264,2001852271,US +2001852272,2001852279,SG +2001852280,2001852351,US +2001852352,2001852357,SG +2001852358,2001852359,US +2001852360,2001852367,SG +2001852368,2001852399,US +2001852400,2001852403,SG +2001852404,2001853015,US +2001853016,2001853039,SG +2001853040,2001853055,US +2001853056,2001853063,SG +2001853064,2001853071,US +2001853072,2001853079,SG +2001853080,2001853083,US +2001853084,2001853087,SG +2001853088,2001853239,US +2001853240,2001853255,SG +2001853256,2001853257,US +2001853258,2001853258,MY +2001853259,2001853263,SG +2001853264,2001853271,US +2001853272,2001853295,SG +2001853296,2001853303,US +2001853304,2001853335,SG +2001853336,2001853375,US +2001853376,2001853379,SG +2001853380,2001853381,US +2001853382,2001853383,SG +2001853384,2001853399,US +2001853400,2001853423,SG +2001853424,2001853439,US +2001853440,2001853463,SG +2001853464,2001853471,US +2001853472,2001853479,SG +2001853480,2001853487,SA +2001853488,2001853503,SG +2001853504,2001853543,US +2001853544,2001853559,SG +2001853560,2001853575,US +2001853576,2001853639,SG +2001853640,2001853663,US +2001853664,2001853671,SG +2001853672,2001853676,US +2001853677,2001853679,SG +2001853680,2001853687,US +2001853688,2001853703,SG +2001853704,2001853727,US +2001853728,2001853743,SG +2001853744,2001853759,US +2001853760,2001853803,SG +2001853804,2001853823,US +2001853824,2001853831,SG +2001853832,2001853855,US +2001853856,2001853863,SG +2001853864,2001853895,US +2001853896,2001853907,SG +2001853908,2001853911,US +2001853912,2001853943,SG +2001853944,2001853947,US +2001853948,2001853951,SG +2001853952,2001854463,US +2001854464,2001854475,SG +2001854476,2001854479,US +2001854480,2001854531,SG +2001854532,2001854535,US +2001854536,2001854547,SG +2001854548,2001854551,US +2001854552,2001854611,SG +2001854612,2001854615,US +2001854616,2001854703,SG +2001854704,2001854707,US +2001854708,2001854759,SG +2001854760,2001854763,US +2001854764,2001854767,SG +2001854768,2001854775,AR +2001854776,2001854783,SG +2001854784,2001854787,US +2001854788,2001854802,SG +2001854803,2001854803,US +2001854804,2001854863,SG +2001854864,2001854867,US +2001854868,2001855075,SG +2001855076,2001855079,US +2001855080,2001855083,SG +2001855084,2001855087,US +2001855088,2001855159,SG +2001855160,2001855167,US +2001855168,2001855183,SG +2001855184,2001855187,US +2001855188,2001855195,SG +2001855196,2001855199,US +2001855200,2001855223,SG +2001855224,2001855231,IN +2001855232,2001855247,SG +2001855248,2001855251,US +2001855252,2001855415,SG +2001855416,2001855417,US +2001855418,2001855419,SG +2001855420,2001855423,US +2001855424,2001855455,SG +2001855456,2001855459,US +2001855460,2001855655,SG +2001855656,2001855663,US +2001855664,2001855831,SG +2001855832,2001855839,US +2001855840,2001856055,SG +2001856056,2001856059,US +2001856060,2001856295,SG +2001856296,2001856303,US +2001856304,2001856355,SG +2001856356,2001856363,US +2001856364,2001856599,SG +2001856600,2001856603,US +2001856604,2001856639,SG +2001856640,2001856703,US +2001856704,2001856815,SG +2001856816,2001856823,US +2001856824,2001856835,SG +2001856836,2001856847,US +2001856848,2001857283,SG +2001857284,2001857287,US +2001857288,2001857383,SG +2001857384,2001857391,US +2001857392,2001857399,SG +2001857400,2001857407,US +2001857408,2001857428,SG +2001857429,2001857429,US +2001857430,2001857430,NL +2001857431,2001857975,SG +2001857976,2001857983,US +2001857984,2001858007,SG +2001858008,2001858011,US +2001858012,2001858111,SG +2001858112,2001858143,US +2001858144,2001859007,SG +2001859008,2001859015,NL +2001859016,2001859039,SG +2001859040,2001859071,US +2001859072,2001859791,SG +2001859792,2001859799,US +2001859800,2001859807,SG +2001859808,2001859811,US +2001859812,2001859899,SG +2001859900,2001859903,US +2001859904,2001860119,SG +2001860120,2001860127,US +2001860128,2001860227,SG +2001860228,2001860231,US +2001860232,2001860335,SG +2001860336,2001860343,US +2001860344,2001860439,SG +2001860440,2001860443,US +2001860444,2001860527,SG +2001860528,2001860531,US +2001860532,2001860603,SG +2001860604,2001860607,US +2001860608,2001860791,SG +2001860792,2001860795,US +2001860796,2001860855,SG +2001860856,2001860863,US +2001860864,2001860903,SG +2001860904,2001860907,US +2001860908,2001860959,SG +2001860960,2001860963,US +2001860964,2001861087,SG +2001861088,2001861095,US +2001861096,2001861207,SG +2001861208,2001861215,US +2001861216,2001861399,SG +2001861400,2001861415,US +2001861416,2001861439,SG +2001861440,2001861447,AU +2001861448,2001862015,SG +2001862016,2001862047,US +2001862048,2001862655,SG 2001862656,2001864703,AU 2001864704,2001870847,JP 2001870848,2001879039,KR @@ -37177,7 +38688,9 @@ 2015207424,2015215615,JP 2015215616,2015216383,IN 2015216384,2015216639,AU -2015216640,2015219711,IN +2015216640,2015217663,IN +2015217664,2015219456,GU +2015219457,2015219711,IN 2015219712,2015219967,US 2015219968,2015220223,HK 2015220224,2015220479,IN @@ -37242,7 +38755,6 @@ 2022227968,2022244351,JP 2022244352,2022277119,CN 2022277120,2022309887,JP -2022309888,2022313983,PH 2022313984,2022318079,NZ 2022318080,2022319135,AU 2022319136,2022319151,SG @@ -37395,18 +38907,15 @@ 2046836736,2046885887,CN 2046885888,2046951423,JP 2046951424,2047082495,PH -2047082496,2047442943,CN -2047442944,2047492095,HK +2047082496,2047410175,CN +2047410176,2047492095,HK 2047492096,2047496191,KR -2047496192,2047508479,HK -2047508480,2047526911,CN -2047526912,2047531007,HK -2047531008,2047574015,CN +2047496192,2047506431,HK +2047506432,2047508479,US +2047508480,2047574015,CN 2047574016,2047606783,SG -2047606784,2047770623,CN -2047770624,2047787007,HK -2047787008,2047868415,CN -2047868416,2047868927,SG +2047606784,2047803391,CN +2047803392,2047868927,SG 2047868928,2048917503,JP 2048917504,2049966079,KR 2049966080,2050047999,CN @@ -37417,8 +38926,7 @@ 2050084864,2050088959,PH 2050088960,2050091007,ID 2050091008,2050097151,JP -2050097152,2050099199,SG -2050099200,2050101247,IN +2050097152,2050101247,SG 2050101248,2050113535,JP 2050113536,2050129663,SG 2050129664,2050129919,JP @@ -37455,8 +38963,10 @@ 2053534720,2053537791,IN 2053537792,2053636095,JP 2053636096,2054160383,AU -2054160384,2054373375,CN -2054373376,2054377471,HK +2054160384,2054189567,CN +2054189568,2054190591,HK +2054190592,2054376447,CN +2054376448,2054377471,HK 2054377472,2054422527,CN 2054422528,2054619135,TW 2054619136,2054684671,CN @@ -37499,7 +39009,9 @@ 2056806400,2056814591,KR 2056814592,2056816863,JP 2056816864,2056816895,MY -2056816896,2056817663,JP +2056816896,2056817335,JP +2056817336,2056817343,HK +2056817344,2056817663,JP 2056817664,2056817919,AU 2056817920,2056818431,JP 2056818432,2056818687,SG @@ -37561,21 +39073,24 @@ 2063085568,2063089663,CN 2063089664,2063097855,JP 2063097856,2063106047,MM -2063106048,2063106559,JP +2063106048,2063106559,SG 2063106560,2063106815,AU -2063106816,2063107623,JP +2063106816,2063107071,SG +2063107072,2063107327,JP +2063107328,2063107623,SG 2063107624,2063107631,AU -2063107632,2063111167,JP -2063111168,2063112191,AU -2063112192,2063113727,JP -2063113728,2063113983,AU -2063113984,2063114495,JP -2063114496,2063114751,IN -2063114752,2063115007,JP -2063115008,2063115263,IN +2063107632,2063108095,SG +2063108096,2063110143,HK +2063110144,2063111167,JP +2063111168,2063114239,AU +2063114240,2063115263,IN 2063115264,2063117311,JP 2063117312,2063117567,NZ -2063117568,2063118335,JP +2063117568,2063117823,JP +2063117824,2063117839,PH +2063117840,2063117951,JP +2063117952,2063118079,PH +2063118080,2063118335,JP 2063118336,2063118591,IN 2063118592,2063119871,JP 2063119872,2063120383,IN @@ -37585,8 +39100,7 @@ 2063122432,2063138815,SG 2063138816,2063335423,JP 2063335424,2063341567,AU -2063341568,2063343615,SG -2063343616,2063351807,JP +2063341568,2063351807,SG 2063351808,2063368191,KR 2063368192,2063372287,JP 2063372288,2063374335,AU @@ -37648,7 +39162,8 @@ 2070183936,2070192127,AU 2070192128,2070200319,KR 2070200320,2070208511,JP -2070208512,2070209535,SG +2070208512,2070209279,MY +2070209280,2070209535,SG 2070209536,2070210559,AU 2070210560,2070210815,SG 2070210816,2070211071,AU @@ -37767,8 +39282,7 @@ 2080768000,2080776191,TW 2080776192,2080777215,HK 2080777216,2080778239,GB -2080778240,2080779263,US -2080779264,2080780287,SG +2080778240,2080780287,US 2080780288,2080781311,ID 2080781312,2080784383,SG 2080784384,2080800767,CN @@ -37814,6 +39328,7 @@ 2084569088,2084732927,CN 2084732928,2084741119,SG 2084741120,2084743167,ID +2084743168,2084745215,JP 2084745216,2084749311,KR 2084749312,2084753407,JP 2084753408,2084757503,KR @@ -37835,8 +39350,10 @@ 2087460864,2087462911,JP 2087462912,2087464959,CN 2087464960,2087467007,KH -2087467008,2087469055,JP -2087469056,2087477247,HK +2087467008,2087471103,JP +2087471104,2087472127,SG +2087472128,2087476223,HK +2087476224,2087477247,TW 2087477248,2087478271,AU 2087478272,2087485439,HK 2087485440,2087501823,TW @@ -37885,7 +39402,6 @@ 2090696704,2090729471,TH 2090729472,2090733567,VN 2090733568,2090736639,AU -2090736640,2090737663,IN 2090737664,2090745855,PH 2090745856,2090762239,MN 2090762240,2090778623,ID @@ -37943,7 +39459,8 @@ 2096668672,2096676863,KH 2096676864,2096693247,HK 2096693248,2096889855,CN -2096889856,2096955391,AU +2096889856,2096902143,BE +2096902144,2096955391,AU 2096955392,2097020927,IN 2097020928,2097037311,CN 2097037312,2097053695,AU @@ -38035,7 +39552,10 @@ 2113688576,2113693599,JP 2113693600,2113693615,HK 2113693616,2113716223,JP -2113716224,2113732607,SG +2113716224,2113724927,SG +2113724928,2113725183,IN +2113725184,2113728511,SG +2113728512,2113732607,JP 2113732608,2113761279,AU 2113761280,2113765375,VN 2113765376,2113798143,HK @@ -38097,7 +39617,8 @@ 2151784448,2151792639,IR 2151792640,2151794687,CH 2151794688,2151796735,IT -2151796736,2151798783,NL +2151796736,2151797759,DE +2151797760,2151798783,NL 2151798784,2151799807,DE 2151799808,2151800831,NL 2151800832,2151809023,PT @@ -38162,7 +39683,9 @@ 2155827200,2155831295,PL 2155831296,2155833343,RU 2155833344,2155833855,SE -2155833856,2155834367,NL +2155833856,2155834084,NL +2155834085,2155834111,SE +2155834112,2155834367,NL 2155834368,2155834464,SE 2155834465,2155834512,NL 2155834513,2155834532,SE @@ -38219,8 +39742,7 @@ 2160908288,2160910335,PL 2160910336,2160914431,NL 2160914432,2160918527,SA -2160918528,2161180671,US -2161246208,2161508351,US +2160918528,2161508351,US 2161508352,2161573887,FI 2161573888,2162228223,US 2162228224,2162228479,CA @@ -38546,26 +40068,21 @@ 2193686528,2193688575,FR 2193688576,2193692671,CZ 2193692672,2193694719,FR -2193694720,2193695743,US -2193695744,2193698815,RU -2193698816,2193700863,US -2193700864,2193704959,RU +2193694720,2193704959,RU 2193704960,2193707007,IT 2193707008,2193707407,GB 2193707408,2193707415,IT 2193707416,2193707559,GB 2193707560,2193707567,IT -2193707568,2193707607,GB -2193707608,2193707615,IT -2193707616,2193707751,GB +2193707568,2193707615,GB +2193707616,2193707623,IT +2193707624,2193707751,GB 2193707752,2193707759,IT 2193707760,2193707831,GB 2193707832,2193707839,IT 2193707840,2193708511,GB 2193708512,2193708519,IT -2193708520,2193708663,GB -2193708664,2193708671,IT -2193708672,2193708735,GB +2193708520,2193708735,GB 2193708736,2193708743,IT 2193708744,2193709087,GB 2193709088,2193709095,IT @@ -38586,7 +40103,9 @@ 2194604032,2194669567,US 2194669568,2194735103,IS 2194735104,2194800639,GB -2194800640,2195193855,US +2194800640,2194888509,US +2194888510,2194888510,IE +2194888511,2195193855,US 2195193856,2195324927,NZ 2195324928,2195455999,US 2195456000,2195521535,AU @@ -38631,8 +40150,45 @@ 2197794816,2197796863,SA 2197796864,2197798911,DE 2197798912,2197815295,IR +2197815296,2197816319,BO +2197816320,2197828607,BR +2197828608,2197829631,AR +2197829632,2197833727,BR +2197833728,2197834751,CR +2197834752,2197841919,BR +2197841920,2197842943,AR +2197842944,2197843967,CL +2197843968,2197847039,BR +2197847040,2197848063,MX +2197848064,2197849087,AR +2197849088,2197850111,BR +2197850112,2197851135,CO +2197851136,2197858303,BR +2197858304,2197859327,CO +2197859328,2197860351,CL +2197860352,2197865471,BR +2197865472,2197866495,AR +2197867520,2197869567,BR +2197869568,2197870591,UY +2197870592,2197874687,BR +2197874688,2197875711,AR +2197875712,2197876735,MX +2197876736,2197880831,BR 2197880832,2197946367,IT 2197946368,2202533887,US +2202533888,2202534911,AR +2202534912,2202540031,BR +2202540032,2202541055,PY +2202541056,2202542079,AR +2202542080,2202550271,BR +2202552320,2202553343,AR +2202554368,2202559487,BR +2202562560,2202563583,CW +2202565632,2202566655,BR +2202567680,2202568703,AR +2202569728,2202570751,BR +2202573824,2202574847,AR +2202574848,2202578943,BR 2202599424,2204172287,US 2204172288,2204237823,SE 2204237824,2204303359,US @@ -38959,194 +40515,315 @@ 2258305024,2258370559,DE 2258370560,2258436095,US 2258436096,2258567167,FR -2258567168,2258568191,US +2258567168,2258568191,TW 2258568192,2258568447,HK -2258568448,2258582783,US +2258568448,2258571647,TW +2258571648,2258571711,US +2258571712,2258571719,TW +2258571720,2258571735,US +2258571736,2258571743,TW +2258571744,2258571751,US +2258571752,2258571763,TW +2258571764,2258571767,US +2258571768,2258579463,TW +2258579464,2258579467,US +2258579468,2258579967,TW +2258579968,2258580031,US +2258580032,2258582783,TW 2258582784,2258582791,GB -2258582792,2258583551,US -2258583552,2258591743,GB -2258591744,2258591935,AU +2258582792,2258583551,TW +2258583552,2258583575,GB +2258583576,2258583583,TW +2258583584,2258583775,GB +2258583776,2258583807,TW +2258583808,2258583935,GB +2258583936,2258583967,TW +2258583968,2258584007,GB +2258584008,2258584351,TW +2258584352,2258584383,GB +2258584384,2258584574,TW +2258584575,2258584575,GB +2258584576,2258591575,TW +2258591576,2258591579,GB +2258591580,2258591631,TW +2258591632,2258591639,GB +2258591640,2258591935,TW 2258591936,2258591967,HK -2258591968,2258592271,AU +2258591968,2258592271,TW 2258592272,2258592279,JP -2258592280,2258592287,AU +2258592280,2258592287,TW 2258592288,2258592291,JP -2258592292,2258592439,AU +2258592292,2258592439,TW 2258592440,2258592447,JP -2258592448,2258592791,AU +2258592448,2258592479,TW +2258592480,2258592495,AU +2258592496,2258592511,TW +2258592512,2258592767,AU +2258592768,2258592791,TW 2258592792,2258592803,HK -2258592804,2258593279,AU +2258592804,2258593279,TW 2258593280,2258593535,HK -2258593536,2258594047,AU +2258593536,2258594047,TW 2258594048,2258594111,HK -2258594112,2258594303,AU +2258594112,2258594303,TW 2258594304,2258594319,HK -2258594320,2258594559,PG +2258594320,2258594559,TW 2258594560,2258594815,HK -2258594816,2258595103,AU -2258595104,2258595167,TW -2258595168,2258595231,AU -2258595232,2258595263,TW -2258595264,2258595295,AU -2258595296,2258595327,TW -2258595328,2258595383,AU +2258594816,2258595167,TW +2258595168,2258595199,AU +2258595200,2258595383,TW 2258595384,2258595391,KR -2258595392,2258595887,AU +2258595392,2258595519,TW +2258595520,2258595551,AU +2258595552,2258595887,TW 2258595888,2258595895,NZ -2258595896,2258595967,AU -2258595968,2258595983,TW -2258595984,2258596095,AU +2258595896,2258595983,TW +2258595984,2258595991,AU +2258595992,2258596031,TW +2258596032,2258596063,AU +2258596064,2258596095,TW 2258596096,2258596103,HK -2258596104,2258596159,AU +2258596104,2258596159,TW 2258596160,2258596255,HK -2258596256,2258596351,AU +2258596256,2258596351,TW 2258596352,2258596863,HK -2258596864,2258596887,AU +2258596864,2258596887,TW 2258596888,2258596903,HK -2258596904,2258596991,AU +2258596904,2258596991,TW 2258596992,2258597023,HK -2258597024,2258597071,AU -2258597072,2258597079,TW -2258597080,2258597115,AU +2258597024,2258597115,TW 2258597116,2258597215,HK -2258597216,2258597263,PG +2258597216,2258597263,TW 2258597264,2258597293,HK 2258597294,2258597294,PG 2258597295,2258597303,HK -2258597304,2258597311,PG +2258597304,2258597311,TW 2258597312,2258597367,HK -2258597368,2258597371,PG +2258597368,2258597371,TW 2258597372,2258597535,HK -2258597536,2258597567,AU +2258597536,2258597567,TW 2258597568,2258597583,HK -2258597584,2258597599,AU +2258597584,2258597599,TW 2258597600,2258597631,HK 2258597632,2258597759,AU 2258597760,2258597887,HK -2258597888,2258597903,AU +2258597888,2258597895,AU +2258597896,2258597903,TW 2258597904,2258597919,JP 2258597920,2258597927,AU 2258597928,2258597935,JP -2258597936,2258598079,AU -2258598080,2258598087,TW -2258598088,2258598095,AU -2258598096,2258598111,TW +2258597936,2258597951,AU +2258597952,2258597975,TW +2258597976,2258597983,AU +2258597984,2258597999,TW +2258598000,2258598007,AU +2258598008,2258598111,TW 2258598112,2258598143,JP -2258598144,2258598495,AU +2258598144,2258598351,TW +2258598352,2258598367,AU +2258598368,2258598399,TW +2258598400,2258598439,AU +2258598440,2258598447,TW +2258598448,2258598479,AU +2258598480,2258598495,TW 2258598496,2258598511,NZ -2258598512,2258598519,AU +2258598512,2258598519,TW 2258598520,2258598527,JP -2258598528,2258598623,AU -2258598624,2258598655,TW -2258598656,2258599183,AU +2258598528,2258598567,TW +2258598568,2258598623,AU +2258598624,2258598667,TW +2258598668,2258598671,AU +2258598672,2258598991,TW +2258598992,2258598999,AU +2258599000,2258599011,TW +2258599012,2258599019,AU +2258599020,2258599183,TW 2258599184,2258599187,JP -2258599188,2258599675,AU +2258599188,2258599499,TW +2258599500,2258599503,AU +2258599504,2258599587,TW +2258599588,2258599591,AU +2258599592,2258599675,TW 2258599676,2258599679,JP -2258599680,2258600263,AU +2258599680,2258599743,TW +2258599744,2258599747,AU +2258599748,2258600263,TW 2258600264,2258600267,HK -2258600268,2258600515,AU +2258600268,2258600515,TW 2258600516,2258600519,IN -2258600520,2258600523,AU +2258600520,2258600523,TW 2258600524,2258600527,IN -2258600528,2258600959,AU +2258600528,2258600735,TW +2258600736,2258600739,AU +2258600740,2258600807,TW +2258600808,2258600811,AU +2258600812,2258600959,TW 2258600960,2258601087,SG -2258601088,2258601167,AU +2258601088,2258601119,AU +2258601120,2258601151,TW +2258601152,2258601167,AU 2258601168,2258601175,NZ -2258601176,2258601215,AU +2258601176,2258601199,AU +2258601200,2258601207,TW +2258601208,2258601215,AU 2258601216,2258601343,TW -2258601344,2258601471,AU +2258601344,2258601359,AU +2258601360,2258601367,TW +2258601368,2258601375,AU +2258601376,2258601407,TW +2258601408,2258601423,AU +2258601424,2258601471,TW 2258601472,2258601983,JP -2258601984,2258602303,AU +2258601984,2258602303,TW 2258602304,2258602327,HK -2258602328,2258602335,AU +2258602328,2258602335,TW 2258602336,2258602367,HK -2258602368,2258602399,AU +2258602368,2258602399,TW 2258602400,2258602447,HK -2258602448,2258602479,AU +2258602448,2258602479,TW 2258602480,2258602495,HK -2258602496,2258602815,AU +2258602496,2258602815,TW 2258602816,2258602879,HK -2258602880,2258603007,AU +2258602880,2258603007,TW 2258603008,2258603071,HK -2258603072,2258603087,PG +2258603072,2258603087,TW 2258603088,2258603089,HK 2258603090,2258603090,PG 2258603091,2258603103,HK -2258603104,2258603135,PG +2258603104,2258603135,TW 2258603136,2258603139,HK -2258603140,2258603199,PG +2258603140,2258603199,TW 2258603200,2258603207,HK -2258603208,2258603263,PG -2258603264,2258603775,AU +2258603208,2258603263,TW +2258603264,2258603279,AU +2258603280,2258603287,TW +2258603288,2258603295,AU +2258603296,2258603775,TW 2258603776,2258603839,HK -2258603840,2258603903,AU +2258603840,2258603903,TW 2258603904,2258603943,HK -2258603944,2258603951,AU +2258603944,2258603951,TW 2258603952,2258603967,HK -2258603968,2258604031,AU +2258603968,2258604031,TW 2258604032,2258604287,HK -2258604288,2258604671,AU +2258604288,2258604671,TW 2258604672,2258604735,SG 2258604736,2258604799,AU 2258604800,2258605055,HK -2258605056,2258605311,AU +2258605056,2258605311,TW 2258605312,2258605439,SG -2258605440,2258606143,AU +2258605440,2258605823,TW +2258605824,2258605951,AU +2258605952,2258605999,TW +2258606000,2258606015,AU +2258606016,2258606047,TW +2258606048,2258606079,AU +2258606080,2258606143,TW 2258606144,2258606147,HK -2258606148,2258606151,AU +2258606148,2258606151,TW 2258606152,2258606191,HK -2258606192,2258606199,AU +2258606192,2258606199,TW 2258606200,2258606367,HK -2258606368,2258606415,AU +2258606368,2258606415,TW 2258606416,2258606423,HK -2258606424,2258606463,AU +2258606424,2258606463,TW 2258606464,2258606471,HK -2258606472,2258606487,AU +2258606472,2258606487,TW 2258606488,2258606535,HK -2258606536,2258606963,AU +2258606536,2258606719,TW +2258606720,2258606751,AU +2258606752,2258606847,TW +2258606848,2258606879,AU +2258606880,2258606963,TW 2258606964,2258606967,NZ -2258606968,2258607091,AU +2258606968,2258606971,AU +2258606972,2258607007,TW +2258607008,2258607039,AU +2258607040,2258607087,TW +2258607088,2258607091,AU 2258607092,2258607095,NZ -2258607096,2258607351,AU +2258607096,2258607103,AU +2258607104,2258607263,TW +2258607264,2258607279,AU +2258607280,2258607351,TW 2258607352,2258607359,NZ -2258607360,2258607819,AU +2258607360,2258607519,AU +2258607520,2258607543,TW +2258607544,2258607551,AU +2258607552,2258607819,TW 2258607820,2258607823,HK -2258607824,2258607871,AU +2258607824,2258607871,TW 2258607872,2258607879,HK -2258607880,2258607903,AU +2258607880,2258607903,TW 2258607904,2258607999,HK -2258608000,2258608063,AU +2258608000,2258608063,TW 2258608064,2258608127,HK -2258608128,2258608183,AU -2258608184,2258608187,TW -2258608188,2258608255,AU +2258608128,2258608255,TW 2258608256,2258608259,JP -2258608260,2258608279,AU +2258608260,2258608279,TW 2258608280,2258608283,JP -2258608284,2258608655,AU +2258608284,2258608639,TW +2258608640,2258608647,AU +2258608648,2258608655,TW 2258608656,2258608663,JP -2258608664,2258608671,AU +2258608664,2258608671,TW 2258608672,2258608687,JP -2258608688,2258610175,AU -2258610176,2258610431,IN -2258610432,2258610703,AU -2258610704,2258610719,TW -2258610720,2258610751,AU -2258610752,2258610755,TW -2258610756,2258610759,AU -2258610760,2258610783,TW -2258610784,2258611071,AU +2258608688,2258608863,AU +2258608864,2258608879,TW +2258608880,2258608895,AU +2258608896,2258609087,TW +2258609088,2258609119,AU +2258609120,2258609151,TW +2258609152,2258609407,AU +2258609408,2258609423,TW +2258609424,2258609471,AU +2258609472,2258609567,TW +2258609568,2258609575,AU +2258609576,2258609919,TW +2258609920,2258609967,AU +2258609968,2258610179,TW +2258610180,2258610183,IN +2258610184,2258610239,TW +2258610240,2258610303,IN +2258610304,2258610431,TW +2258610432,2258610687,AU +2258610688,2258611071,TW 2258611072,2258611103,JP -2258611104,2258611119,AU +2258611104,2258611119,TW 2258611120,2258611167,JP -2258611168,2258611215,AU +2258611168,2258611215,TW 2258611216,2258611223,NZ -2258611224,2258614783,AU +2258611224,2258611567,TW +2258611568,2258611583,AU +2258611584,2258612223,TW +2258612224,2258612303,AU +2258612304,2258612351,TW +2258612352,2258612367,AU +2258612368,2258612383,TW +2258612384,2258612743,AU +2258612744,2258613503,TW +2258613504,2258613567,AU +2258613568,2258614783,TW 2258614784,2258614815,IN -2258614816,2258615039,AU +2258614816,2258615039,TW 2258615040,2258615055,IN -2258615056,2258616319,AU -2258616320,2258632703,HK +2258615056,2258615347,TW +2258615348,2258615355,AU +2258615356,2258615391,TW +2258615392,2258615395,AU +2258615396,2258616191,TW +2258616192,2258616203,AU +2258616204,2258616207,TW +2258616208,2258616303,AU +2258616304,2258616311,TW +2258616312,2258616319,AU +2258616320,2258620447,TW +2258620448,2258620455,HK +2258620456,2258620463,TW +2258620464,2258620467,HK +2258620468,2258620471,TW +2258620472,2258620475,HK +2258620476,2258632703,TW 2258632704,2258698239,JP 2258698240,2259222527,US 2259222528,2259288063,DE @@ -39368,6 +41045,7 @@ 2309226496,2309357567,US 2309357568,2309423103,AU 2309423104,2309685247,US +2309685248,2309750783,ZA 2309750784,2309816319,AU 2309816320,2309881855,US 2309881856,2309947391,NL @@ -39382,7 +41060,9 @@ 2311061504,2311127039,US 2311127040,2311192575,DE 2311192576,2311258111,FR -2311258112,2311323647,GB +2311258112,2311315455,GB +2311315456,2311319551,MY +2311319552,2311323647,GB 2311389184,2311847935,US 2311847936,2311913471,IT 2311913472,2311979007,GB @@ -39463,8 +41143,7 @@ 2319974400,2320039935,US 2320039936,2320105471,CA 2320105472,2320171007,US -2320171008,2320203775,NZ -2320203776,2320236543,SG +2320171008,2320236543,SG 2320236544,2320302079,US 2320302080,2320367615,AU 2320367616,2320433151,US @@ -39672,8 +41351,10 @@ 2344609024,2344609279,IT 2344615936,2344878079,ID 2344878080,2346188799,CN -2346188800,2346450943,AU -2346450944,2346582015,CN +2346188800,2346254335,AU +2346254336,2346319871,CN +2346319872,2346385407,AU +2346385408,2346582015,CN 2346582016,2346647551,GB 2346647552,2346713087,TW 2346713088,2346778623,CN @@ -39829,7 +41510,6 @@ 2371747840,2371878911,GB 2371878912,2371944447,BE 2371944448,2372009983,GB -2372009984,2372075519,PL 2372075520,2372206591,DE 2372206592,2372214783,UA 2372214784,2372218879,DE @@ -40158,7 +41838,8 @@ 2407661568,2407727103,AT 2407727104,2408054783,US 2408054784,2408120319,JP -2408185856,2409627647,US +2408185856,2409562111,US +2409562112,2409627647,GB 2409627648,2409693183,ZA 2409693184,2409758719,AT 2409758720,2409824255,US @@ -40168,7 +41849,6 @@ 2410151936,2410217471,US 2410217472,2410283007,BE 2410283008,2410348543,US -2410348544,2410414079,JP 2410414080,2410545151,US 2410545152,2410610687,AU 2410610688,2410676223,US @@ -40176,7 +41856,9 @@ 2410938368,2411003903,CH 2411003904,2411462655,US 2411462656,2411593727,AU -2411593728,2411986943,US +2411593728,2411749375,US +2411749376,2411753471,MY +2411753472,2411986943,US 2411986944,2412052479,FR 2412052480,2412314623,US 2412445696,2412576767,US @@ -40508,7 +42190,15 @@ 2455244800,2455245567,AU 2455245568,2455245823,US 2455245824,2455246847,AU -2455246848,2455371775,US +2455246848,2455247871,IN +2455247872,2455248895,US +2455248896,2455257087,TH +2455257088,2455261183,PH +2455261184,2455262207,US +2455262208,2455263231,KR +2455263232,2455273471,US +2455273472,2455275519,AU +2455275520,2455371775,US 2455371776,2455437311,GB 2455437312,2455830527,US 2455830528,2455896063,GB @@ -40538,7 +42228,8 @@ 2457360896,2457361151,RU 2457361152,2457361407,CZ 2457361408,2457365503,RU -2457365504,2457366527,CZ +2457365504,2457366271,CZ +2457366272,2457366527,RU 2457366528,2457367551,GB 2457367552,2457372671,CZ 2457372672,2457376767,RU @@ -40611,8 +42302,8 @@ 2462121984,2462187519,PT 2462187520,2462253055,GB 2462253056,2462326783,AU -2462384128,2462580735,US -2462580736,2462646271,GB +2462384128,2462449663,US +2462449664,2462646271,GB 2462646272,2463236095,US 2463236096,2463301631,AT 2463301632,2463367167,FI @@ -40977,13 +42668,17 @@ 2500199424,2500199679,IE 2500199680,2500201535,US 2500201536,2500201543,GB -2500201544,2500219135,US +2500201544,2500202879,US +2500202880,2500203007,ES +2500203008,2500219135,US 2500219136,2500219391,DE 2500219392,2500225551,US 2500225552,2500225559,ES 2500225560,2500228607,US 2500228608,2500228863,FR -2500228864,2500235775,US +2500228864,2500230041,US +2500230042,2500230042,GB +2500230043,2500235775,US 2500235776,2500236543,GB 2500236544,2500236837,US 2500236838,2500236838,ES @@ -41031,7 +42726,9 @@ 2500551680,2500551935,FR 2500551936,2500553759,US 2500553760,2500553767,GB -2500553768,2500555263,US +2500553768,2500554379,US +2500554380,2500554487,DE +2500554488,2500555263,US 2500555264,2500555519,FR 2500555520,2500558847,US 2500558848,2500559103,FR @@ -41083,7 +42780,9 @@ 2504491008,2504499199,IT 2504499200,2504916991,US 2504916992,2504982527,IL -2504982528,2506293247,US +2504982528,2505793535,US +2505793536,2505801727,ES +2505801728,2506293247,US 2506293248,2506358783,CA 2506358784,2506360831,US 2506360832,2506361087,ES @@ -41112,7 +42811,9 @@ 2508107776,2508109823,FI 2508109824,2508111871,DE 2508111872,2508128255,RU -2508128256,2508455935,US +2508128256,2508259327,US +2508259328,2508324863,MY +2508324864,2508455935,US 2508455936,2508521471,IT 2508521472,2508587007,CH 2508587008,2508652543,BE @@ -41278,6 +42979,8 @@ 2525038592,2525039615,HK 2525039616,2525040639,AU 2525040640,2525041151,BD +2525041152,2525041407,ID +2525041408,2525041663,IN 2525041664,2525042687,KH 2525042688,2525044735,IN 2525044736,2525045759,HK @@ -41306,6 +43009,13 @@ 2525085696,2525086719,CN 2525086720,2525089791,IN 2525089792,2525090815,HK +2525090816,2525091839,IN +2525091840,2525092863,CN +2525092864,2525093887,BD +2525093888,2525094911,KR +2525094912,2525095935,TW +2525095936,2525101055,IN +2525101056,2525102079,CN 2525102080,2525233151,US 2525233152,2525298687,SE 2525298688,2525626367,US @@ -41356,6 +43066,40 @@ 2532114432,2532179967,GB 2532179968,2532376575,US 2532376576,2532442111,ES +2532442112,2532445183,CN +2532445184,2532449279,IN +2532449280,2532450303,CN +2532450304,2532451327,AU +2532451328,2532452351,HK +2532452352,2532453375,NZ +2532453376,2532457471,CN +2532457472,2532461567,IN +2532461568,2532463615,CN +2532463616,2532465663,IN +2532465664,2532467711,CN +2532467712,2532468735,TW +2532468736,2532469759,BD +2532469760,2532470783,ID +2532470784,2532473855,CN +2532473856,2532474879,HK +2532474880,2532475903,NZ +2532475904,2532476927,KR +2532476928,2532477951,AU +2532477952,2532478975,IN +2532478976,2532479999,KR +2532480000,2532481023,IN +2532481024,2532486143,CN +2532486144,2532488191,IN +2532488192,2532489215,MY +2532489216,2532492287,CN +2532492288,2532495359,IN +2532495360,2532496383,HK +2532496384,2532497407,CN +2532497408,2532499455,HK +2532499456,2532500479,CN +2532500480,2532501503,HK +2532501504,2532506623,CN +2532506624,2532507647,IN 2532507648,2532573183,US 2532573184,2532638719,ES 2532638720,2533031935,US @@ -41363,7 +43107,8 @@ 2533097472,2533228543,US 2533228544,2533294079,PL 2533294080,2533359615,CN -2533359616,2533375999,UA +2533359616,2533373951,UA +2533373952,2533375999,BG 2533376000,2533392383,HU 2533392384,2533425151,RO 2533425152,2539978751,IT @@ -41386,7 +43131,6 @@ 2542338048,2543583231,US 2543583232,2543648767,SE 2543648768,2543714303,NO -2543714304,2543779839,JP 2543779840,2544154840,US 2544154841,2544154842,CA 2544154843,2544500735,US @@ -41410,8 +43154,7 @@ 2545811456,2547187711,US 2547187712,2547318783,GB 2547318784,2547515391,US -2547515392,2547523583,FR -2547531776,2547539967,UA +2547515392,2547523583,GB 2547553024,2547553279,RU 2548039680,2548563967,GB 2548563968,2548826111,IR @@ -41474,7 +43217,7 @@ 2549702656,2549704703,UA 2549704704,2549706751,CZ 2549706752,2549710847,NO -2549710848,2549743615,CZ +2549710848,2549743615,BY 2549743616,2549809151,TR 2549809152,2549874687,BG 2549874688,2549876735,GB @@ -41535,7 +43278,9 @@ 2556742080,2556742327,US 2556742328,2556742335,GB 2556742336,2556755967,US -2556755968,2556821503,HK +2556755968,2556780031,HK +2556780032,2556780799,SG +2556780800,2556821503,HK 2556821504,2556887039,SG 2556887040,2557018111,HK 2557018112,2557083647,GB @@ -41655,7 +43400,8 @@ 2585853952,2585985023,JP 2585985024,2586480639,US 2586480640,2586484735,IL -2586484736,2586510335,US +2586484736,2586486783,GB +2586486784,2586510335,US 2586510336,2586511359,ES 2586511360,2586566655,US 2586566656,2586566687,FR @@ -41670,7 +43416,9 @@ 2586622464,2586622975,ES 2586622976,2586640895,US 2586640896,2586641407,FR -2586641408,2586733567,US +2586641408,2586650687,US +2586650688,2586650703,DE +2586650704,2586733567,US 2586733568,2586733823,LT 2586733824,2586804223,US 2586804224,2586804479,ES @@ -41708,8 +43456,8 @@ 2587249418,2587249418,FR 2587249419,2587394047,US 2587394048,2587394559,ES -2587394560,2587398143,US -2587398144,2587399167,ES +2587394560,2587396095,US +2587396096,2587399167,ES 2587399168,2587443199,US 2587443200,2587447295,CH 2587447296,2587476760,US @@ -41722,7 +43470,9 @@ 2587492352,2587493375,ES 2587493376,2587508735,US 2587508736,2587525119,GB -2587525120,2587926527,US +2587525120,2587542527,US +2587542528,2587543551,ES +2587543552,2587926527,US 2587926528,2587930623,BG 2587930624,2587951103,US 2587951104,2587952127,ZA @@ -41821,17 +43571,24 @@ 2588488704,2588489727,LY 2588489728,2588490751,GQ 2588490752,2588491775,GW -2588493824,2588494847,ZA +2588491776,2588492799,TZ +2588492800,2588494847,ZA +2588494848,2588495871,SO 2588495872,2588496895,ZW +2588496896,2588497919,BW 2588497920,2588498943,SD +2588498944,2588499967,DZ 2588499968,2588500991,ZA 2588500992,2588502015,CI +2588502016,2588503039,BI 2588503040,2588504063,LY 2588504064,2588505087,TD 2588505088,2588506111,CG 2588506112,2588507135,ZA 2588507136,2588508159,SO 2588508160,2588510207,LY +2588510208,2588512255,ZA +2588522496,2588526591,ZA 2588526592,2588528639,CM 2588528640,2588530687,ZA 2588532736,2588534783,ZA @@ -41839,8 +43596,10 @@ 2588672000,2588934143,KE 2588934144,2589982719,SC 2589982720,2590507007,SD +2590507008,2591031295,TN 2591031296,2591096831,GA 2591293440,2591326207,GA +2591522816,2591526911,LR 2591531008,2591539199,ZA 2591539200,2591547391,GA 2591547392,2591555583,ZA @@ -41978,11 +43737,109 @@ 2616983552,2617049087,US 2617049088,2617114623,IT 2617114880,2617115135,US -2617118720,2617131007,US +2617118720,2617123839,US +2617123840,2617124095,DE +2617124096,2617131007,US 2617131008,2617139199,CA -2617139200,2617148159,US +2617139200,2617139455,PE +2617139456,2617139711,SZ +2617139712,2617139967,JM +2617139968,2617140223,PA +2617140224,2617140479,CO +2617140480,2617140735,SC +2617140736,2617140991,CA +2617140992,2617141247,LC +2617141248,2617141503,SV +2617141504,2617141759,GT +2617141760,2617142015,ZW +2617142016,2617142271,KR +2617142272,2617142527,US +2617142528,2617142783,LU +2617142784,2617143039,PH +2617143040,2617143295,UG +2617143296,2617143551,US +2617143552,2617143807,TT +2617143808,2617144063,US +2617144064,2617144319,ZA +2617144320,2617144575,MU +2617144576,2617144831,FJ +2617144832,2617145087,US +2617145088,2617145343,MX +2617145344,2617145599,NI +2617145600,2617145855,UA +2617145856,2617146111,US +2617146112,2617146367,CI +2617146368,2617146623,US +2617146624,2617146879,GY +2617146880,2617147135,DE +2617147136,2617147391,AO +2617147392,2617148159,US 2617151488,2617155583,CA -2617155584,2617180159,US +2617155584,2617163775,US +2617163776,2617164031,SK +2617164032,2617164287,MC +2617164288,2617164543,AZ +2617164544,2617164799,FI +2617164800,2617165055,CZ +2617165056,2617165311,UZ +2617165312,2617165567,BT +2617165568,2617165823,CN +2617165824,2617166079,ME +2617166080,2617166335,YE +2617166336,2617166591,KH +2617166592,2617166847,TJ +2617166848,2617167103,KG +2617167104,2617167359,IS +2617167360,2617167615,DK +2617167616,2617167871,IL +2617167872,2617168127,PL +2617168128,2617168383,LA +2617168384,2617168639,GH +2617168640,2617168895,BD +2617168896,2617169151,LI +2617169152,2617169407,ET +2617169408,2617169663,SM +2617169664,2617169919,BH +2617169920,2617170175,CY +2617170176,2617170431,NZ +2617170432,2617170687,PS +2617170688,2617170943,GI +2617170944,2617171199,RS +2617171200,2617171455,HK +2617171456,2617171711,HR +2617171712,2617171967,SN +2617171968,2617172223,MV +2617172224,2617172479,AF +2617172480,2617172735,JP +2617172736,2617172991,CM +2617172992,2617173247,US +2617173248,2617173503,JP +2617173504,2617173759,TM +2617173760,2617174015,GE +2617174016,2617174271,TR +2617174272,2617174527,TW +2617174528,2617174783,KW +2617174784,2617175039,EE +2617175040,2617175295,CN +2617175296,2617175551,MA +2617175552,2617175807,BA +2617175808,2617176063,IQ +2617176064,2617176319,NO +2617176320,2617176575,AU +2617176576,2617176831,TH +2617176832,2617177087,VN +2617177088,2617177343,AM +2617177344,2617177599,RU +2617177600,2617177855,IN +2617177856,2617178111,BE +2617178112,2617178367,IM +2617178368,2617178623,BN +2617178624,2617178879,AD +2617178880,2617179135,LT +2617179136,2617179391,MD +2617179392,2617179647,SI +2617179648,2617179903,KE +2617179904,2617180159,NG 2617311232,2617769983,US 2617769984,2617835519,ZA 2617835520,2617901055,US @@ -42000,7 +43857,7 @@ 2618884096,2618949631,CH 2618949632,2619080703,US 2619080704,2619146239,FR -2619146240,2619211775,US +2619146240,2619277311,US 2619277312,2619342847,BN 2619342848,2619473919,US 2619473920,2619539455,CA @@ -42053,7 +43910,9 @@ 2627076096,2627141631,NL 2627141632,2627403775,US 2634022912,2634088447,CN -2634088448,2635202559,JP +2634088448,2634350591,JP +2634416128,2635005951,JP +2635071488,2635202559,JP 2635202560,2635268095,CN 2635268096,2635399167,JP 2635399168,2635530239,US @@ -42352,13 +44211,29 @@ 2667642880,2667970559,US 2667970560,2668036095,CA 2668036096,2668101631,SE -2668101632,2668101791,NL +2668101632,2668101663,NL +2668101664,2668101671,US +2668101672,2668101791,NL 2668101792,2668101823,US 2668101824,2668101855,NL 2668101856,2668101863,US 2668101864,2668101887,NL 2668101888,2668101919,US -2668101920,2668167167,NL +2668101920,2668102063,NL +2668102064,2668102071,US +2668102072,2668102647,NL +2668102648,2668102655,US +2668102656,2668102991,NL +2668102992,2668102999,US +2668103000,2668103743,NL +2668103744,2668103775,KE +2668103776,2668104127,NL +2668104128,2668104135,GB +2668104136,2668104191,NL +2668104192,2668104207,KE +2668104208,2668104247,NL +2668104248,2668104255,US +2668104256,2668167167,NL 2668167168,2668363775,US 2668363776,2668429311,CH 2668429312,2668494847,AU @@ -42476,7 +44351,9 @@ 2677639680,2677639935,CA 2677639936,2677642239,US 2677642240,2677642495,ES -2677642496,2677648383,US +2677642496,2677642751,US +2677642752,2677643007,SE +2677643008,2677648383,US 2677648384,2677649407,JP 2677649408,2677650431,US 2677650432,2677650943,RO @@ -42534,7 +44411,8 @@ 2679242752,2679308287,US 2679308288,2679373823,CH 2679373824,2679406591,GB -2679406592,2679431167,US +2679406592,2679422975,BR +2679422976,2679431167,US 2679431168,2679439359,FR 2679439360,2679523327,US 2679523328,2679525375,GB @@ -42577,7 +44455,6 @@ 2682322944,2682388479,US 2682388480,2682454015,CN 2682454016,2682519551,US -2682519552,2682585087,JP 2682585088,2682716159,US 2682716160,2682781695,CA 2682781696,2682847231,CH @@ -42773,7 +44650,9 @@ 2684878848,2684944383,SE 2684944384,2685009919,GB 2685009920,2685075455,US -2685075456,2686386175,JP +2685075456,2685599743,JP +2685665280,2685992959,JP +2686058496,2686320639,JP 2686386176,2686844927,US 2686844928,2686910463,GB 2686910464,2686975999,US @@ -42858,7 +44737,9 @@ 2694381568,2694447103,US 2696151040,2696216575,IT 2696216576,2696282111,ZA -2696413184,2697789439,JP +2696413184,2696478719,ZA +2696478720,2696871935,JP +2696937472,2697789439,JP 2697789440,2697854975,US 2697854976,2697889791,AU 2697889792,2697891839,US @@ -42883,8 +44764,11 @@ 2699362304,2699624447,US 2699624448,2699689983,JP 2699755520,2699821055,JP -2699886592,2700935167,JP +2699886592,2699984895,JP +2700017664,2700738559,JP +2700804096,2700935167,JP 2700935168,2701066239,US +2701066240,2701131775,ZA 2701131776,2701139967,HN 2701139968,2701148159,NI 2701148160,2701149183,AR @@ -43430,9 +45314,7 @@ 2734588928,2734589951,CA 2734589952,2734599167,US 2734599168,2734600191,CA -2734600192,2734603263,US -2734603264,2734604287,AI -2734604288,2734633983,US +2734600192,2734633983,US 2734633984,2734635007,CA 2734635008,2734649343,US 2734649344,2734650367,CA @@ -43441,8 +45323,8 @@ 2734658560,2734659583,CA 2734659584,2734665727,US 2734665728,2734666751,CA -2734666752,2734673919,US -2734673920,2734675967,CA +2734666752,2734672895,US +2734672896,2734675967,CA 2734675968,2734678015,BM 2734678016,2734679039,US 2734679040,2734680063,CA @@ -43464,21 +45346,88 @@ 2736848896,2736914431,US 2736914432,2736979967,NO 2736979968,2737438719,US -2737438720,2737799167,JP +2737438720,2737767423,JP +2737767424,2737768447,CN +2737768448,2737769471,AU +2737769472,2737770495,IN +2737770496,2737771263,AU +2737771264,2737771519,JP +2737771520,2737772031,AU +2737772544,2737774591,JP +2737774592,2737776639,BD +2737776640,2737777663,AU +2737777664,2737778175,HK +2737778432,2737785855,AU +2737785856,2737788927,BD +2737788928,2737789951,MY +2737789952,2737793023,AU +2737793024,2737794047,NZ +2737794048,2737795071,AU +2737795072,2737796095,NZ +2737796096,2737799167,AU 2737799168,2737800191,NZ -2737800192,2737801215,JP -2737801216,2737802239,AU +2737800192,2737802239,AU 2737802240,2737803263,IN -2737803264,2737805311,JP +2737803264,2737804287,BD +2737804288,2737805311,NP 2737805312,2737806335,IN -2737806336,2737809407,JP +2737806336,2737807359,BD +2737807360,2737808383,AF +2737808384,2737809407,NZ 2737809408,2737810431,AU -2737810432,2737824767,JP -2737824768,2737825791,NZ -2737825792,2737828863,JP -2737828864,2737829887,NZ -2737829888,2738094079,JP -2738225152,2738749439,JP +2737810432,2737811455,KH +2737811456,2737811711,SG +2737811712,2737811967,HK +2737811968,2737812223,GB +2737812224,2737812479,US +2737812480,2737813503,AU +2737813504,2737815551,NZ +2737815552,2737816575,MY +2737816576,2737817599,AU +2737817600,2737818623,MY +2737818624,2737819647,NZ +2737819648,2737821695,IN +2737821696,2737822719,AU +2737822720,2737823743,NZ +2737823744,2737824767,NC +2737824768,2737829887,NZ +2737829888,2737830911,NC +2737830912,2737831935,AU +2737831936,2738094079,JP +2738159616,2738163711,CN +2738163712,2738164735,HK +2738164736,2738165759,AU +2738165760,2738166783,NP +2738166784,2738167807,KH +2738167808,2738168831,AU +2738168832,2738177023,CN +2738177024,2738178047,AU +2738178048,2738179071,HK +2738179072,2738182143,IN +2738182144,2738195455,CN +2738195456,2738196479,BD +2738196480,2738197503,AU +2738197504,2738198527,BD +2738198528,2738199551,MY +2738199552,2738200575,KR +2738200576,2738204671,CN +2738204672,2738205695,IN +2738205696,2738206719,BD +2738206720,2738207743,ID +2738207744,2738208767,CN +2738208768,2738209791,ID +2738209792,2738210815,KH +2738210816,2738213887,IN +2738213888,2738215935,HK +2738215936,2738216959,CN +2738216960,2738217983,HK +2738217984,2738220031,AU +2738220032,2738221055,PK +2738221056,2738222079,CN +2738222080,2738223103,MO +2738223104,2738224127,AU +2738224128,2738225151,IN +2738225152,2738683903,JP 2738749440,2742353919,FR 2742353920,2742419455,ES 2742419456,2742484991,US @@ -43712,8 +45661,8 @@ 2770337792,2770993151,US 2770993152,2771124223,IN 2771124224,2771451903,US +2771648512,2771910655,ZA 2772434944,2772631551,US -2772631552,2772697087,AU 2772697088,2772762623,US 2772762624,2772828159,AU 2772828160,2772959231,US @@ -43731,7 +45680,8 @@ 2774532096,2774597631,JP 2774597632,2774663167,US 2774663168,2774728703,NI -2774728704,2774990847,JP +2774728704,2774860799,JP +2774925312,2774990847,JP 2774990848,2775318527,US 2775318528,2775384063,JP 2775384064,2775711743,US @@ -43971,7 +45921,7 @@ 2813591552,2813908479,US 2813908480,2813908735,AU 2813908736,2814181375,US -2814181376,2814246911,AU +2814181376,2814246911,CN 2814246912,2815033343,US 2815033344,2815098879,NL 2815098880,2815111347,GB @@ -43996,7 +45946,8 @@ 2816271872,2816272383,IN 2816272384,2816273407,JP 2816273408,2816275455,IN -2816275456,2816671743,US +2816275456,2816278527,SG +2816278528,2816671743,US 2816671744,2816737279,CA 2816737280,2817061119,US 2817061120,2817062911,LU @@ -44005,11 +45956,11 @@ 2817294336,2817933055,US 2817933056,2817933311,CA 2817933312,2817933567,PR -2817933568,2818003711,US -2818003712,2818003722,GB +2817933568,2818002943,US +2818002944,2818003722,GB 2818003723,2818003723,US -2818003724,2818003967,GB -2818003968,2818113535,US +2818003724,2818004991,GB +2818004992,2818113535,US 2818244608,2818310143,US 2818310144,2818375679,AR 2818375680,2818572287,US @@ -44111,7 +46062,7 @@ 2831613952,2832007167,US 2832072704,2832138239,ZA 2832138240,2832269311,US -2832269312,2832400383,ZA +2832269312,2832465919,ZA 2832465920,2832793599,US 2832793600,2832859135,AU 2832859136,2832924671,US @@ -44129,7 +46080,8 @@ 2833973248,2834010111,US 2834010112,2834014207,CA 2834018304,2834030591,US -2834038784,2834497535,US +2834030592,2834034687,CA +2834034688,2834497535,US 2834497536,2834563071,SV 2834563072,2834825215,US 2834825216,2834956287,KR @@ -44166,7 +46118,6 @@ 2848244736,2848245759,GB 2848245760,2848276479,US 2848325632,2848522239,US -2848522240,2848587775,AU 2848587776,2848653311,ZA 2848653312,2848980991,US 2848980992,2849964031,KR @@ -44537,7 +46488,11 @@ 2918023168,2918043647,US 2918043648,2918047743,CA 2918047744,2918051839,US -2918051840,2918121471,CA +2918051840,2918111999,CA +2918112000,2918112127,US +2918112128,2918112895,CA +2918112896,2918113023,US +2918113024,2918121471,CA 2918121472,2918154239,US 2918154240,2918170623,CA 2918170624,2918187775,US @@ -44679,8 +46634,14 @@ 2925527040,2926575615,US 2926575616,2926777343,CA 2926777344,2926777855,US -2926777856,2927084799,CA -2927084800,2927085823,US +2926777856,2926778111,CA +2926778112,2926778239,US +2926778240,2926779391,CA +2926779392,2926779519,US +2926779520,2927084799,CA +2927084800,2927085055,US +2927085056,2927085567,CA +2927085568,2927085823,US 2927085824,2927099903,CA 2927099904,2927242751,US 2927242752,2927243263,AE @@ -44713,7 +46674,7 @@ 2936012800,2937847807,CN 2937847808,2937848831,KH 2937848832,2937849855,JP -2937849856,2937850879,SG +2937849856,2937850879,AU 2937850880,2937851903,MY 2937851904,2937855999,JP 2937856000,2937860095,KR @@ -44792,7 +46753,6 @@ 2943313920,2943314943,SG 2943315968,2943318015,ID 2943318016,2943320063,JP -2943320064,2943336447,PK 2943336448,2943352831,TW 2943352832,2944401407,KR 2944401408,2944925695,JP @@ -44814,7 +46774,13 @@ 2947584000,2947586047,TO 2947588096,2947590143,ID 2947590144,2947590655,JP -2947590656,2947592191,SG +2947590656,2947590911,SG +2947590912,2947590927,HK +2947590928,2947591167,SG +2947591168,2947591183,HK +2947591184,2947591679,SG +2947591680,2947591695,JP +2947591696,2947592191,SG 2947596288,2947597311,IN 2947597312,2947598335,JP 2947598336,2947602431,AU @@ -44948,7 +46914,8 @@ 2954837868,2954837871,ES 2954837872,2954838599,FR 2954838600,2954838607,GB -2954838608,2954839239,FR +2954838608,2954838615,NL +2954838616,2954839239,FR 2954839240,2954839243,IT 2954839244,2954839267,FR 2954839268,2954839271,ES @@ -44962,7 +46929,11 @@ 2954840928,2954840931,DE 2954840932,2954841567,FR 2954841568,2954841583,ES -2954841584,2954843503,FR +2954841584,2954841707,FR +2954841708,2954841711,NL +2954841712,2954841807,FR +2954841808,2954841815,NL +2954841816,2954843503,FR 2954843504,2954843507,ES 2954843508,2954843767,FR 2954843768,2954843771,ES @@ -44970,7 +46941,9 @@ 2954844148,2954844151,DE 2954844152,2954844999,FR 2954845000,2954845003,ES -2954845004,2954846107,FR +2954845004,2954846095,FR +2954846096,2954846103,NL +2954846104,2954846107,FR 2954846108,2954846111,ES 2954846112,2954846139,FR 2954846140,2954846143,ES @@ -45022,7 +46995,9 @@ 2954899456,2954901503,ES 2954901504,2954903551,IT 2954903552,2954911743,JP -2954911744,2954918911,NL +2954911744,2954911999,NL +2954912000,2954912767,US +2954912768,2954918911,NL 2954918912,2954919935,IE 2954919936,2954928127,RU 2954928128,2954932223,PL @@ -45269,7 +47244,9 @@ 2959282176,2959290367,RO 2959290368,2959292415,PL 2959292416,2959302655,RU -2959302656,2959310847,UA +2959302656,2959308877,UA +2959308878,2959308878,RU +2959308879,2959310847,UA 2959310848,2959343615,RU 2959343616,2959351807,UA 2959351808,2959353855,DE @@ -45498,7 +47475,6 @@ 2960594944,2960596991,UA 2960596992,2960605183,PL 2960605184,2960621567,UA -2960621568,2960629759,CZ 2960629760,2960646143,RU 2960646144,2960648191,KZ 2960648192,2960650239,UA @@ -45617,7 +47593,9 @@ 2961088512,2961088767,SE 2961088768,2961089535,UA 2961089536,2961090559,KG -2961090560,2961113087,RO +2961090560,2961108991,RO +2961108992,2961111039,GB +2961111040,2961113087,RO 2961113088,2961178623,CH 2961178624,2965372927,FR 2965372928,2965766143,RU @@ -45703,6 +47681,8 @@ 2969042944,2969567231,GB 2969567232,2984247295,BR 2984247296,2984935423,MX +2984935424,2984936447,AR +2984936448,2984937471,BR 2984937472,2984939519,HN 2984939520,2984951807,BR 2984951808,2984968191,EC @@ -45758,7 +47738,11 @@ 2987552768,2987556863,GB 2987556864,2987560959,NL 2987560960,2987565055,DE -2987565056,2987569151,AT +2987565056,2987566591,AT +2987566592,2987566847,HU +2987566848,2987568383,AT +2987568384,2987568639,HU +2987568640,2987569151,AT 2987569152,2987573247,FR 2987573248,2987577343,TR 2987577344,2987585535,RU @@ -45917,7 +47901,9 @@ 2988449216,2988451839,FR 2988451840,2988453887,BE 2988453888,2988457983,GB -2988457984,2988459119,FR +2988457984,2988458319,FR +2988458320,2988458327,NL +2988458328,2988459119,FR 2988459120,2988459127,IT 2988459128,2988459135,PL 2988459136,2988459535,FR @@ -45971,11 +47957,15 @@ 2988484832,2988484847,GB 2988484848,2988485683,FR 2988485684,2988485687,PL -2988485688,2988487071,FR +2988485688,2988486311,FR +2988486312,2988486319,NL +2988486320,2988487071,FR 2988487072,2988487075,ES 2988487076,2988487095,FR 2988487096,2988487099,ES -2988487100,2988489479,FR +2988487100,2988489119,FR +2988489120,2988489127,DE +2988489128,2988489479,FR 2988489480,2988489483,DE 2988489484,2988489675,FR 2988489676,2988489679,ES @@ -45986,7 +47976,9 @@ 2988490687,2988490687,BE 2988490688,2988492799,FR 2988492800,2988494847,PL -2988494848,2988499729,FR +2988494848,2988499663,FR +2988499664,2988499671,NL +2988499672,2988499729,FR 2988499730,2988499730,GB 2988499731,2988499736,FR 2988499737,2988499737,NL @@ -46096,7 +48088,8 @@ 2988558204,2988558207,DE 2988558208,2988558803,FR 2988558804,2988558807,ES -2988558808,2988561583,FR +2988558808,2988558815,NL +2988558816,2988561583,FR 2988561584,2988561591,GB 2988561592,2988561763,FR 2988561764,2988561767,IT @@ -46112,7 +48105,9 @@ 2989096960,2989228031,PL 2989228032,2989490175,RU 2989490176,2989555711,HU -2989555712,2989621247,RU +2989555712,2989588479,RU +2989588480,2989589503,CO +2989589504,2989621247,RU 2989621248,2989752319,BE 2989752320,2989817855,SY 2989817856,2989883391,KW @@ -46127,7 +48122,9 @@ 2990211072,2990276607,GR 2990276608,2990342143,ES 2990342144,2990407679,KW -2990407680,2990440447,GB +2990407680,2990421728,US +2990421729,2990421729,GB +2990421730,2990440447,US 2990440448,2990473215,NL 2990473216,2990475674,DE 2990475675,2990475675,HR @@ -46230,7 +48227,7 @@ 2996666368,2996682751,RU 2996682752,2996699135,DK 2996699136,2996764671,UA -2996764672,2996768767,NL +2996764672,2996768767,GB 2996768768,2996772863,RU 2996772864,2996776959,UA 2996776960,2996781055,RU @@ -46288,7 +48285,8 @@ 2997513355,2997515683,FR 2997515684,2997515684,GB 2997515685,2997518335,FR -2997518336,2997526527,NL +2997518336,2997520383,DE +2997520384,2997526527,NL 2997526528,2997583871,RU 2997583872,2997616639,SY 2997616640,2997649407,SI @@ -46307,7 +48305,21 @@ 2997878784,2998140927,RU 2998140928,2998403071,PL 2998403072,2998665215,RU -2998665216,2998927359,AT +2998665216,2998710451,AT +2998710452,2998710455,DE +2998710456,2998712975,AT +2998712976,2998712979,DE +2998712980,2998720227,AT +2998720228,2998720231,CZ +2998720232,2998820735,AT +2998820736,2998820863,CZ +2998820864,2998834559,AT +2998834560,2998834687,HU +2998834688,2998841343,AT +2998841344,2998841599,DE +2998841600,2998875391,AT +2998875392,2998875519,CZ +2998875520,2998927359,AT 2998927360,2998991615,CH 2998991616,2998991871,DE 2998991872,2999026943,CH @@ -46373,7 +48385,7 @@ 3000301568,3000302591,RU 3000302592,3000303103,LV 3000303104,3000303359,CH -3000303360,3000303615,PL +3000303360,3000303615,DE 3000303616,3000305663,UA 3000305664,3000313855,RU 3000313856,3000315903,CZ @@ -46790,6 +48802,7 @@ 3003173632,3003173887,AR 3003173888,3003174143,EC 3003174144,3003174399,CO +3003174400,3003174911,BR 3003174912,3003179007,CR 3003179008,3003187199,AR 3003187200,3003252735,CO @@ -47035,7 +49048,8 @@ 3007160064,3007160079,HR 3007160080,3007160575,CL 3007160576,3007160591,IE -3007160592,3007161087,CL +3007160592,3007160607,US +3007160608,3007161087,CL 3007161088,3007161103,GB 3007161104,3007161599,CL 3007161600,3007161615,GB @@ -47069,11 +49083,13 @@ 3007168512,3007168767,QA 3007168768,3007169023,CL 3007169024,3007169151,US -3007169152,3007175679,CL +3007169152,3007171855,CL +3007171856,3007171871,US +3007171872,3007175679,CL 3007175680,3007175935,GB 3007175936,3007183359,CL 3007183360,3007183615,AU -3007183616,3007183871,SA +3007183616,3007183871,IE 3007183872,3007184127,JO 3007184128,3007184383,KW 3007184384,3007184895,CL @@ -47142,24 +49158,32 @@ 3024879616,3025141759,CN 3025141760,3025403903,KR 3025403904,3025600511,CN -3025600512,3025603839,IN +3025600512,3025603071,IN +3025603072,3025603087,HK +3025603088,3025603839,IN 3025603840,3025604095,HK 3025604096,3025604381,IN -3025604382,3025604637,SG -3025604638,3025606815,IN +3025604382,3025606655,SG +3025606656,3025606815,IN 3025606816,3025606831,SG 3025606832,3025607167,IN 3025607168,3025607423,SG 3025607424,3025607679,HK 3025607680,3025608191,IN 3025608192,3025608203,JP -3025608204,3025616895,IN +3025608204,3025610751,IN +3025610752,3025612799,SG +3025612800,3025616895,IN 3025616896,3025617407,SG 3025617408,3025618943,IN 3025618944,3025619487,SG 3025619488,3025620991,IN 3025620992,3025621247,PH -3025621248,3025625343,IN +3025621248,3025621503,IN +3025621504,3025621759,PH +3025621760,3025623295,IN +3025623296,3025623551,JP +3025623552,3025625343,IN 3025625344,3025625375,SG 3025625376,3025625391,IN 3025625392,3025625395,SG @@ -47167,7 +49191,8 @@ 3025625400,3025625407,TH 3025625408,3025625471,SG 3025625472,3025625503,MY -3025625504,3025625599,IN +3025625504,3025625535,IN +3025625536,3025625599,CA 3025625600,3025625855,SG 3025625856,3025629439,IN 3025629440,3025629503,HK @@ -47183,7 +49208,9 @@ 3025632272,3025633535,IN 3025633536,3025633791,HK 3025633792,3025633807,AU -3025633808,3025637375,IN +3025633808,3025637151,IN +3025637152,3025637183,MY +3025637184,3025637375,IN 3025637376,3025637631,HK 3025637632,3025637887,MY 3025637888,3025638279,IN @@ -47210,7 +49237,8 @@ 3025647776,3025647791,SG 3025647792,3025648079,IN 3025648080,3025648087,SG -3025648088,3025649151,IN +3025648088,3025648091,US +3025648092,3025649151,IN 3025649152,3025649663,HK 3025649664,3025666047,IN 3025666048,3025928191,CN @@ -47410,7 +49438,8 @@ 3035103232,3035168767,PH 3035168768,3035193343,CN 3035193344,3035197439,JP -3035197440,3035199487,HK +3035197440,3035198463,SG +3035198464,3035199487,HK 3035199488,3035200511,IN 3035200512,3035202559,AU 3035202560,3035205631,JP @@ -47458,14 +49487,18 @@ 3039414272,3039414527,BR 3039414528,3039415295,CL 3039415296,3039415807,BR -3039415808,3039416831,CL +3039415808,3039416713,CL +3039416714,3039416715,US +3039416716,3039416735,CL +3039416736,3039416739,US +3039416740,3039416831,CL 3039416832,3039417343,BR 3039417344,3039417855,CL 3039417856,3039418111,BR 3039418112,3039418367,CL 3039418368,3039418879,BR -3039418880,3039419391,CL -3039419392,3039420159,BR +3039418880,3039419135,CL +3039419136,3039420159,BR 3039420160,3039420415,CL 3039420416,3039428607,AR 3039428608,3039559679,CL @@ -47569,7 +49602,8 @@ 3050700800,3050700815,JP 3050700816,3050701311,CL 3050701312,3050701327,IE -3050701328,3050701823,CL +3050701328,3050701343,US +3050701344,3050701823,CL 3050701824,3050701839,SG 3050701840,3050702335,CL 3050702336,3050702351,DK @@ -47804,7 +49838,8 @@ 3057052672,3057052927,IL 3057052928,3057053183,GB 3057053184,3057053695,AU -3057053696,3057057791,JP +3057053696,3057054719,JP +3057055744,3057057791,JP 3057057792,3057123327,SG 3057123328,3057451007,IN 3057451008,3057516543,CN @@ -47895,7 +49930,6 @@ 3074949120,3075342335,JP 3075342336,3075375103,MY 3075375104,3075383295,KR -3075383296,3075385343,IN 3075385344,3075386367,MY 3075386368,3075387391,AU 3075388416,3075389439,CN @@ -47941,7 +49975,12 @@ 3081437184,3081502719,MY 3081502720,3081764863,CN 3081764864,3081846783,JP -3081846784,3081854975,HK +3081846784,3081847807,TW +3081847808,3081848831,KR +3081848832,3081850879,SG +3081850880,3081851903,HK +3081851904,3081852927,JP +3081852928,3081854975,HK 3081854976,3081859071,MN 3081859072,3081861119,PH 3081861120,3081862143,AU @@ -47951,7 +49990,7 @@ 3082027008,3082158079,JP 3082158080,3082166271,CN 3082166272,3082174463,JP -3082174464,3082178559,PH +3082174464,3082178559,BZ 3082178560,3082179583,HK 3082179584,3082181631,IN 3082181632,3082182655,ID @@ -48098,7 +50137,7 @@ 3103855104,3103855359,DE 3103855360,3103855615,RU 3103855616,3103855871,ES -3103855872,3103856383,PL +3103856128,3103856383,PL 3103856384,3103856639,RU 3103856640,3103856895,HU 3103856896,3103857151,CZ @@ -48217,6 +50256,7 @@ 3104026624,3104027647,NL 3104027648,3104028671,RU 3104028672,3104029695,DE +3104029696,3104030719,AM 3104030720,3104031743,HR 3104031744,3104032767,FR 3104032768,3104033791,IR @@ -48502,9 +50542,7 @@ 3104331776,3104333823,GB 3104333824,3104334847,ES 3104334848,3104335871,SE -3104335872,3104336239,LT -3104336240,3104336255,DE -3104336256,3104336359,LT +3104335872,3104336359,LT 3104336360,3104336367,BE 3104336368,3104336407,LT 3104336408,3104336415,CH @@ -49309,7 +51347,7 @@ 3105212416,3105213439,NL 3105213440,3105214463,TR 3105214464,3105215487,GB -3105215488,3105216511,FR +3105215488,3105216511,GP 3105216512,3105217535,GB 3105217536,3105218559,CZ 3105218560,3105219583,DE @@ -49732,7 +51770,6 @@ 3105662976,3105663999,ES 3105664000,3105665023,DE 3105665024,3105666047,PL -3105666048,3105667071,GB 3105667072,3105668095,DE 3105668096,3105669119,NL 3105669120,3105670143,CZ @@ -49981,9 +52018,7 @@ 3105933344,3105933359,IT 3105933360,3105933463,GB 3105933464,3105933471,IT -3105933472,3105934007,GB -3105934008,3105934015,IT -3105934016,3105934127,GB +3105933472,3105934127,GB 3105934128,3105934135,IT 3105934136,3105934215,GB 3105934216,3105934223,IT @@ -50221,7 +52256,9 @@ 3106166784,3106167807,BH 3106167808,3106168831,RO 3106168832,3106169855,FI -3106169856,3106170879,CZ +3106169856,3106170000,NL +3106170001,3106170001,CZ +3106170002,3106170879,NL 3106170880,3106171903,RU 3106171904,3106172927,GB 3106172928,3106173951,NL @@ -50512,7 +52549,9 @@ 3106483424,3106483424,ES 3106483425,3106483425,NL 3106483426,3106483426,FR -3106483427,3106484223,GB +3106483427,3106483429,GB +3106483430,3106483430,IE +3106483431,3106484223,GB 3106484224,3106485247,NL 3106485248,3106486271,CZ 3106486272,3106488319,DE @@ -50672,8 +52711,7 @@ 3106661376,3106662399,TR 3106662400,3106663423,IE 3106663424,3106664447,UA -3106664448,3106665471,FR -3106665472,3106666495,CH +3106664448,3106666495,FR 3106666496,3106667519,UA 3106667520,3106668543,ES 3106668544,3106669567,RU @@ -51207,8 +53245,7 @@ 3107252224,3107253247,SY 3107253248,3107254271,NL 3107254272,3107255295,IT -3107255296,3107256319,DE -3107256320,3107257343,CZ +3107255296,3107257343,DE 3107257344,3107258367,GB 3107258368,3107259391,IT 3107259392,3107260415,CZ @@ -51437,12 +53474,12 @@ 3107492864,3107493887,DE 3107493888,3107494911,FR 3107494912,3107495935,IT -3107495936,3107495999,FR -3107496000,3107496031,BS +3107495936,3107496031,FR 3107496032,3107496063,VG 3107496064,3107496191,FR 3107496192,3107496255,DE -3107496256,3107496319,FR +3107496256,3107496287,BS +3107496288,3107496319,FR 3107496320,3107496703,NL 3107496704,3107496719,PE 3107496720,3107496735,KP @@ -51454,7 +53491,7 @@ 3107496816,3107496831,CN 3107496832,3107496847,BR 3107496848,3107496863,KP -3107496864,3107496879,FR +3107496864,3107496879,HK 3107496880,3107496911,IS 3107496912,3107496927,PL 3107496928,3107496959,FR @@ -51755,7 +53792,8 @@ 3107817472,3107818495,UA 3107818496,3107819519,GB 3107819520,3107820543,FR -3107820544,3107821567,SE +3107820544,3107821055,LV +3107821056,3107821567,SE 3107821568,3107822591,GB 3107822592,3107823615,UA 3107823616,3107824639,FR @@ -51808,6 +53846,225 @@ 3107876864,3107877887,RS 3107877888,3107878911,NL 3107878912,3107879935,IL +3107879936,3107880959,DE +3107880960,3107881983,HU +3107881984,3107883007,RS +3107883008,3107884031,US +3107884032,3107885055,NO +3107885056,3107887103,DE +3107887104,3107888127,BG +3107888128,3107889151,GB +3107889152,3107890175,NO +3107890176,3107891199,IE +3107891200,3107892223,CH +3107892224,3107893247,AL +3107893248,3107894271,SA +3107894272,3107895295,PL +3107895296,3107896319,FR +3107896320,3107897343,NL +3107897344,3107898367,RU +3107898368,3107899391,LV +3107899392,3107900415,RU +3107900416,3107901439,SE +3107901440,3107902463,NL +3107902464,3107904511,GB +3107904512,3107907583,FR +3107907584,3107908607,IR +3107908608,3107909631,BG +3107909632,3107910655,IT +3107910656,3107911679,DE +3107911680,3107912703,IT +3107912704,3107913727,SE +3107913728,3107914751,AT +3107914752,3107915775,RO +3107915776,3107916799,DE +3107916800,3107917823,IT +3107917824,3107918847,HU +3107918848,3107919871,NL +3107919872,3107920895,CZ +3107920896,3107921919,ES +3107921920,3107922943,UA +3107922944,3107923967,CH +3107923968,3107924991,IT +3107924992,3107926015,HU +3107926016,3107927039,FR +3107927040,3107928063,IT +3107928064,3107929087,NL +3107929088,3107930111,RU +3107930112,3107931135,CH +3107931136,3107932159,NL +3107932160,3107934207,MT +3107934208,3107935231,GB +3107935232,3107936255,PL +3107936256,3107937279,IT +3107937280,3107938303,FR +3107938304,3107939327,CZ +3107939328,3107940351,IT +3107940352,3107941375,CZ +3107941376,3107942399,PL +3107942400,3107943423,RU +3107943424,3107944447,NL +3107944448,3107945471,FR +3107945472,3107947519,DE +3107947520,3107948543,IT +3107948544,3107949567,RO +3107949568,3107950591,GB +3107950592,3107951615,US +3107951616,3107952639,CH +3107952640,3107953663,NL +3107953664,3107954687,SK +3107954688,3107955711,IT +3107955712,3107956735,NL +3107956736,3107957759,IT +3107957760,3107958783,FR +3107958784,3107959807,AL +3107959808,3107960831,PT +3107960832,3107961855,IM +3107961856,3107962879,RU +3107962880,3107963903,FR +3107963904,3107964927,RU +3107964928,3107965951,ES +3107965952,3107966975,DE +3107966976,3107967999,NO +3107968000,3107969023,DE +3107969024,3107970047,RU +3107970048,3107971071,GB +3107971072,3107972095,UZ +3107972096,3107973119,IT +3107973120,3107974143,SA +3107974144,3107976191,IR +3107976192,3107977215,NL +3107977216,3107978239,AT +3107978240,3107979263,NL +3107979264,3107980287,ME +3107980288,3107981311,BE +3107981312,3107982335,IL +3107982336,3107983359,GB +3107983360,3107984383,IE +3107984384,3107985407,BG +3107985408,3107986431,OM +3107986432,3107987455,PL +3107987456,3107988479,NL +3107988480,3107989503,GB +3107989504,3107990527,CZ +3107990528,3107991551,RU +3107991552,3107992575,AT +3107992576,3107993599,RU +3107993600,3107994623,RO +3107994624,3107995647,DE +3107995648,3107996671,BE +3107996672,3107997695,IT +3107997696,3107998719,SE +3107998720,3107999743,AE +3107999744,3108000767,TR +3108000768,3108001791,MK +3108001792,3108002815,GB +3108002816,3108003839,RO +3108003840,3108004863,DE +3108004864,3108005887,RU +3108005888,3108006911,RO +3108006912,3108007935,NL +3108007936,3108008959,DE +3108008960,3108009983,CH +3108009984,3108011007,NL +3108011008,3108012031,DE +3108012032,3108013055,FR +3108013056,3108014079,HR +3108014080,3108015103,NL +3108015104,3108016127,IE +3108016128,3108017151,DE +3108017152,3108018175,FR +3108018176,3108019199,AT +3108019200,3108020223,DE +3108020224,3108021247,US +3108021248,3108022271,AT +3108022272,3108023295,DE +3108023296,3108024319,RS +3108024320,3108025343,IR +3108025344,3108026367,BE +3108026368,3108027391,CH +3108027392,3108028415,GB +3108028416,3108029439,NL +3108029440,3108030463,CH +3108030464,3108031487,KW +3108031488,3108032511,IT +3108032512,3108033535,RU +3108033536,3108034559,GB +3108034560,3108036607,CZ +3108036608,3108037631,GB +3108037632,3108038655,RU +3108038656,3108039679,ES +3108039680,3108040703,MD +3108040704,3108041727,ES +3108041728,3108042751,DE +3108042752,3108044799,GB +3108044800,3108045823,LU +3108045824,3108046847,DE +3108046848,3108047871,HU +3108047872,3108048895,RU +3108048896,3108049919,UA +3108049920,3108050943,RU +3108050944,3108051967,GB +3108051968,3108052991,FR +3108052992,3108054015,IR +3108054016,3108055039,LV +3108055040,3108056063,GB +3108056064,3108057087,IT +3108057088,3108058111,LT +3108058112,3108059135,NL +3108059136,3108060159,FR +3108060160,3108061183,CH +3108061184,3108062207,ES +3108062208,3108063231,TR +3108063232,3108064255,BE +3108064256,3108065279,CZ +3108065280,3108066303,FR +3108066304,3108067327,GB +3108067328,3108068351,AT +3108068352,3108069375,CZ +3108069376,3108070399,FR +3108070400,3108073471,GB +3108073472,3108074495,FR +3108074496,3108075519,DK +3108075520,3108076543,BA +3108076544,3108077567,NL +3108077568,3108078591,GB +3108078592,3108079615,SE +3108079616,3108080639,MD +3108080640,3108081663,SE +3108081664,3108082687,GB +3108082688,3108083711,RU +3108083712,3108084735,NO +3108084736,3108085759,ES +3108085760,3108086783,LV +3108086784,3108087807,GB +3108087808,3108088831,CH +3108088832,3108089855,NL +3108089856,3108090879,SE +3108090880,3108091903,BE +3108091904,3108092927,ES +3108092928,3108093951,SE +3108093952,3108095999,DE +3108096000,3108097023,RU +3108097024,3108098047,TR +3108098048,3108099071,DE +3108099072,3108100095,NL +3108100096,3108101119,IT +3108101120,3108102143,SK +3108102144,3108103167,FR +3108103168,3108104191,SI +3108104192,3108105215,AT +3108105216,3108106239,GB +3108106240,3108107263,DE +3108107264,3108108287,UA +3108108288,3108109311,FR +3108109312,3108110335,IQ +3108110336,3108111359,DK +3108111360,3108112383,GB +3108112384,3108113407,UA +3108113408,3108114431,RU +3108114432,3108115455,AE +3108115456,3108116479,GB 3120562176,3120594943,CO 3120594944,3120599039,AR 3120599040,3120601087,EC @@ -52013,6 +54270,7 @@ 3133075456,3133079551,CW 3133079552,3133145087,AR 3133145088,3136985343,BR +3136985344,3136986111,AR 3136986112,3145727999,BR 3145728000,3149135871,MX 3149135872,3149398015,BR @@ -52036,7 +54294,31 @@ 3154575360,3154640895,FR 3154640896,3155165183,IT 3155165184,3155427327,RU -3155427328,3155689471,AT +3155427328,3155536647,AT +3155536648,3155536655,DE +3155536656,3155540735,AT +3155540736,3155540991,HU +3155540992,3155542323,AT +3155542324,3155542327,CH +3155542328,3155598079,AT +3155598080,3155598335,DE +3155598336,3155628031,AT +3155628032,3155628287,HU +3155628288,3155663743,AT +3155663744,3155663871,CH +3155663872,3155675647,AT +3155675648,3155675775,DE +3155675776,3155676415,AT +3155676416,3155676671,HU +3155676672,3155681791,AT +3155681792,3155682047,HU +3155682048,3155684095,AT +3155684096,3155684223,HU +3155684224,3155685119,AT +3155685120,3155685247,HU +3155685248,3155685375,AT +3155685376,3155685631,CZ +3155685632,3155689471,AT 3155689472,3155951615,RO 3155951616,3156213759,GB 3156213760,3156279295,RU @@ -52145,9 +54427,12 @@ 3158704128,3158835199,KW 3158835200,3158851583,IQ 3158851584,3158859775,RU -3158859776,3158860031,NL -3158860032,3158861055,DE -3158861056,3158862847,NL +3158859776,3158861567,NL +3158861568,3158861823,AE +3158861824,3158862079,IN +3158862080,3158862335,GB +3158862336,3158862591,NL +3158862592,3158862847,FR 3158862848,3158863103,PL 3158863104,3158863359,IT 3158863360,3158863615,MD @@ -52497,9 +54782,7 @@ 3163186535,3163186535,FR 3163186536,3163186673,DE 3163186674,3163186674,FR -3163186675,3163192621,DE -3163192622,3163192622,EG -3163192623,3163193343,DE +3163186675,3163193343,DE 3163193344,3163226111,MD 3163226112,3163258879,SA 3163258880,3163291647,SY @@ -52547,7 +54830,9 @@ 3164947580,3164947583,IT 3164947584,3164949135,FR 3164949136,3164949151,NL -3164949152,3164951663,FR +3164949152,3164950695,FR +3164950696,3164950703,NL +3164950704,3164951663,FR 3164951664,3164951671,PL 3164951672,3164952224,FR 3164952225,3164952231,GB @@ -52557,7 +54842,9 @@ 3164952237,3164952237,GB 3164952238,3164953255,FR 3164953256,3164953263,CZ -3164953264,3164953583,FR +3164953264,3164953391,FR +3164953392,3164953399,NL +3164953400,3164953583,FR 3164953584,3164953599,ES 3164953600,3164954511,FR 3164954512,3164954515,IT @@ -52586,11 +54873,15 @@ 3164968836,3164968839,ES 3164968840,3164969019,FR 3164969020,3164969023,BE -3164969024,3164970413,FR +3164969024,3164969039,FR +3164969040,3164969047,NL +3164969048,3164970413,FR 3164970414,3164970415,IT 3164970416,3164970923,FR 3164970924,3164970927,LT -3164970928,3164973419,FR +3164970928,3164971791,FR +3164971792,3164971799,IT +3164971800,3164973419,FR 3164973420,3164973423,ES 3164973424,3164973663,FR 3164973664,3164973695,GB @@ -52611,7 +54902,8 @@ 3165323264,3165388799,ES 3165388800,3165417471,RO 3165417472,3165421567,DE -3165421568,3165437951,RO +3165421568,3165425663,IE +3165425664,3165437951,RO 3165437952,3165454335,AT 3165454336,3165519871,RO 3165519872,3165585407,DE @@ -52648,14 +54940,19 @@ 3166697472,3166699519,RO 3166699520,3166961663,DE 3166961664,3167223807,SI -3167223808,3167748095,NL +3167223808,3167321095,NL +3167321096,3167321099,DE +3167321100,3167476583,NL +3167476584,3167476591,DE +3167476592,3167748095,NL 3167748096,3167762431,RO 3167762432,3167764479,MD 3167764480,3167772671,RO 3167772672,3167773695,MD 3167773696,3167774719,RO 3167774720,3167775743,MD -3167775744,3167776767,RO +3167775744,3167776511,RO +3167776512,3167776767,GB 3167776768,3167777791,MD 3167777792,3167778815,RO 3167778816,3167780863,MD @@ -52667,11 +54964,14 @@ 3167813632,3167815679,MD 3167815680,3167868927,RO 3167868928,3167879167,MD -3167879168,3167938559,RO +3167879168,3167895551,DE +3167895552,3167938559,RO 3167938560,3167939583,MD 3167939584,3167940607,RO 3167940608,3167943679,MD -3167943680,3167987711,RO +3167943680,3167944447,RO +3167944448,3167944703,GB +3167944704,3167987711,RO 3167987712,3167989759,MD 3167989760,3168005887,RO 3168005888,3168006143,NL @@ -52685,16 +54985,17 @@ 3168020480,3168022527,MD 3168022528,3168038911,RO 3168038912,3168039935,MD -3168039936,3168050431,RO +3168039936,3168040959,BE +3168040960,3168050431,RO 3168050432,3168050687,MD 3168050688,3168081919,RO -3168081920,3168082943,FR -3168082944,3168084991,RO +3168081920,3168083967,FR +3168083968,3168084991,RO 3168084992,3168086015,MD 3168086016,3168088063,IT 3168088064,3168089087,RO 3168089088,3168090111,MD -3168090112,3168092159,FR +3168090112,3168092159,ES 3168092160,3168096255,RO 3168096256,3168100351,MD 3168100352,3168129023,RO @@ -52709,15 +55010,21 @@ 3168156672,3168157695,MD 3168157696,3168165887,RO 3168165888,3168166911,MD -3168166912,3168176127,RO +3168166912,3168169983,RO +3168169984,3168172031,ES +3168172032,3168176127,RO 3168176128,3168177151,MD 3168177152,3168178175,RO 3168178176,3168179199,MD -3168179200,3168192511,RO +3168179200,3168191999,RO +3168192000,3168192255,GB +3168192256,3168192511,RO 3168192512,3168194559,MD 3168194560,3168199679,RO 3168199680,3168200703,MD -3168200704,3168267263,RO +3168200704,3168202751,RO +3168202752,3168203775,ES +3168203776,3168267263,RO 3168267264,3168269311,MD 3168269312,3168271359,RO 3168271360,3168272383,MD @@ -52745,8 +55052,8 @@ 3169189888,3169222655,UA 3169222656,3169255423,SI 3169255424,3169279999,KW -3169280000,3169282047,GB -3169282048,3169288191,KW +3169280000,3169285119,GB +3169285120,3169288191,KW 3169288192,3169320959,UA 3169320960,3169583103,RU 3169583104,3169648639,KW @@ -53710,7 +56017,8 @@ 3210740480,3210740735,US 3210740736,3210742015,BR 3210742016,3210742031,IT -3210742032,3210742271,BR +3210742032,3210742047,US +3210742048,3210742271,BR 3210742272,3210742527,CL 3210742528,3210742543,KR 3210742544,3210743039,CL @@ -53718,13 +56026,15 @@ 3210743056,3210743295,CL 3210743296,3210743551,US 3210743552,3210743567,TH -3210743568,3210744063,CL +3210743568,3210743583,US +3210743584,3210744063,CL 3210744064,3210744079,TR 3210744080,3210744575,CL 3210744576,3210744591,BE 3210744592,3210745343,CL 3210745344,3210745359,RU -3210745360,3210745855,CL +3210745360,3210745375,US +3210745376,3210745855,CL 3210745856,3210745871,IT 3210745872,3210746367,CL 3210746368,3210746383,SE @@ -53739,13 +56049,15 @@ 3210752000,3210752255,US 3210752256,3210755839,CL 3210755840,3210755855,MY -3210755856,3210757119,CL +3210755856,3210755871,US +3210755872,3210757119,CL 3210757120,3210757375,US 3210757376,3210763519,CL 3210763520,3210763775,US 3210763776,3210764031,CL 3210764032,3210764047,IE -3210764048,3210765055,CL +3210764048,3210764063,US +3210764064,3210765055,CL 3210765056,3210765071,SE 3210765072,3210769919,CL 3210769920,3210770175,US @@ -53779,7 +56091,9 @@ 3210786048,3210786063,AU 3210786064,3210786559,CL 3210786560,3210786575,GR -3210786576,3210803327,CL +3210786576,3210803071,CL +3210803072,3210803087,US +3210803088,3210803327,CL 3210803328,3210803455,BR 3210803456,3210805247,CL 3210805248,3210809343,PA @@ -53806,20 +56120,30 @@ 3210926080,3210928127,AR 3210928128,3210936319,NI 3210936320,3211067391,EC -3211067392,3211069439,US -3211069440,3211073023,CL +3211067392,3211073023,US 3211073024,3211073279,CA -3211073280,3211083775,CL +3211073280,3211073535,US +3211073536,3211075583,CL +3211075584,3211075839,US +3211075840,3211080703,CL +3211080704,3211080959,NL +3211080960,3211081215,CL +3211081216,3211081727,CH +3211081728,3211083775,CL 3211083776,3211083791,RU 3211083792,3211084287,CL 3211084288,3211084303,NL -3211084304,3211084799,CL +3211084304,3211084671,CL +3211084672,3211084799,BY 3211084800,3211084815,AT 3211084816,3211085311,CL 3211085312,3211085327,GB -3211085328,3211085823,CL +3211085328,3211085695,CL +3211085696,3211085823,GE 3211085824,3211085839,TH -3211085840,3211086335,CL +3211085840,3211086095,CL +3211086096,3211086111,US +3211086112,3211086335,CL 3211086336,3211086351,FR 3211086352,3211086847,CL 3211086848,3211086863,PL @@ -53859,7 +56183,13 @@ 3211095552,3211095567,GB 3211095568,3211096063,CL 3211096064,3211096079,GB -3211096080,3211129599,CL +3211096080,3211096575,CL +3211096576,3211096831,GB +3211096832,3211097087,CL +3211097088,3211097103,DE +3211097104,3211099647,CL +3211099648,3211099663,DE +3211099664,3211129599,CL 3211129600,3211129855,BR 3211129856,3211132927,CL 3211132928,3211137023,CO @@ -53957,7 +56287,9 @@ 3220041472,3220041727,BR 3220041728,3220042239,JP 3220042240,3220042751,BR -3220042752,3220062207,US +3220042752,3220043775,US +3220043776,3220045823,BR +3220045824,3220062207,US 3220062208,3220062719,SG 3220062720,3220063231,AU 3220063232,3220063743,JP @@ -53976,12 +56308,13 @@ 3220168704,3220172799,US 3220172800,3221225471,BR 3221225480,3221225727,GB -3221225728,3221225983,US 3221226240,3221226495,US 3221226496,3221227519,KY 3221227520,3221242879,US 3221242880,3221243391,GB -3221243392,3221258239,US +3221243392,3221243647,US +3221243648,3221243903,SG +3221243904,3221258239,US 3221258240,3221291007,CA 3221291008,3221334269,US 3221334270,3221334270,DZ @@ -54138,8 +56471,7 @@ 3223215104,3223215359,AU 3223215360,3223216383,US 3223216384,3223217151,CA -3223217152,3223219711,US -3223220224,3223223295,US +3223217152,3223223295,US 3223223296,3223223551,AI 3223223552,3223227903,US 3223227904,3223228159,CA @@ -54150,7 +56482,7 @@ 3223236608,3223237631,GB 3223237632,3223243263,US 3223243264,3223243519,CA -3223243520,3223248895,US +3223243520,3223249407,US 3223249408,3223249663,CA 3223249664,3223250943,US 3223250944,3223252991,CA @@ -54173,41 +56505,35 @@ 3223267584,3223269119,US 3223269376,3223270399,US 3223270400,3223271423,CA -3223271424,3223272447,US +3223271424,3223272959,US 3223272960,3223273215,GB 3223273216,3223283199,US 3223283200,3223283455,NL 3223283456,3223286783,US 3223286784,3223289087,JP -3223289088,3223289343,US -3223289856,3223299583,US +3223289088,3223299583,US 3223299584,3223301119,NL 3223301120,3223302399,US 3223302400,3223302655,CA +3223302656,3223303167,US 3223303168,3223303423,CA -3223303424,3223303679,US -3223304192,3223305215,US -3223305728,3223307519,US +3223303424,3223307519,US 3223307520,3223310079,JP -3223310080,3223310335,US -3223310848,3223311103,US +3223310080,3223311103,US 3223311104,3223311359,NL 3223311360,3223314431,US 3223314688,3223315455,US 3223315456,3223315711,CA 3223315712,3223316223,US 3223316224,3223316479,NL -3223316480,3223319039,US -3223319552,3223321343,US +3223316480,3223321343,US 3223321344,3223321599,CA 3223321600,3223321855,FR -3223321856,3223388159,US -3223388672,3223390719,US +3223321856,3223390719,US 3223390720,3223390975,MU 3223390976,3223391999,US 3223392000,3223392511,NL -3223392512,3223393791,US -3223394304,3223397375,US +3223392512,3223397375,US 3223397376,3223397631,NL 3223397632,3223410431,US 3223410432,3223416831,CH @@ -54217,8 +56543,8 @@ 3223418368,3223420927,CH 3223420928,3223421439,US 3223421440,3223421951,NL -3223422464,3223422719,US -3223422720,3223422975,AU +3223421952,3223422719,US +3223422720,3223422975,JP 3223422976,3223424767,US 3223424768,3223425535,NL 3223425536,3223425791,US @@ -54235,25 +56561,23 @@ 3223449088,3223449343,NL 3223449344,3223453183,US 3223453184,3223453439,NL -3223453440,3223457791,US -3223458304,3223458559,US +3223453440,3223458559,US 3223458560,3223458815,NL 3223458816,3223460351,US -3223460864,3223461887,US -3223462400,3223462911,US +3223460352,3223460863,CA +3223460864,3223462911,US 3223462912,3223463935,CA -3223463936,3223465983,US +3223463936,3223466495,US 3223466496,3223466751,NL 3223466752,3223467007,US 3223467008,3223468031,CA -3223468032,3223471103,US +3223468032,3223471615,US 3223471616,3223471871,CA -3223471872,3223472639,US +3223471872,3223473151,US 3223473152,3223473215,CA 3223473216,3223473231,US 3223473232,3223474175,CA -3223474176,3223476223,US -3223476736,3223477247,US +3223474176,3223477247,US 3223477248,3223478271,CA 3223478272,3223480319,US 3223480832,3223481087,US @@ -54280,8 +56604,7 @@ 3223534336,3223534591,AU 3223534592,3223535359,US 3223535360,3223537919,DE -3223537920,3223540991,US -3223541248,3223542271,US +3223537920,3223542271,US 3223542272,3223542527,NL 3223542528,3223543295,US 3223543296,3223543551,NL @@ -54304,7 +56627,7 @@ 3223566080,3223568639,NL 3223568640,3223569663,US 3223569664,3223570175,GB -3223570432,3223571711,US +3223570176,3223571711,US 3223571712,3223572223,GB 3223572224,3223572479,IE 3223572480,3223577855,US @@ -54314,8 +56637,7 @@ 3223581952,3223582207,US 3223582208,3223582719,NL 3223582720,3223582975,AU -3223582976,3223583231,US -3223583488,3223584767,US +3223582976,3223584767,US 3223584768,3223585023,GB 3223585024,3223585791,SE 3223585792,3223586047,GB @@ -54375,32 +56697,32 @@ 3223781376,3223847935,US 3223848448,3223855103,US 3223855104,3223857151,CA -3223857152,3223859711,US -3223859968,3223862271,US -3223862528,3223862783,US +3223857152,3223862783,US 3223863296,3223863807,US +3223864320,3223864575,US 3223864576,3223864831,AE 3223864832,3223865343,HR 3223865344,3223867391,FI 3223867392,3223867647,GB +3223867648,3223867903,CA 3223868416,3223869439,BM 3223869440,3223871487,US 3223871488,3223873535,CA -3223873536,3223874559,US -3223874816,3223875071,US -3223875584,3223876863,US +3223873536,3223875071,US +3223875584,3223877119,US 3223877632,3223881727,US 3223881728,3223882751,CA -3223883520,3223898111,US -3223898368,3223898623,US +3223883264,3223883519,CA +3223883520,3223898623,US 3223899136,3223902207,US 3223902464,3223902719,CA 3223903232,3223905279,US 3223905280,3223905535,FI +3223905536,3223905791,US 3223906304,3223909375,CA 3223909376,3223910911,US 3223911936,3223912191,CA -3223912448,3223938559,US +3223912192,3223938815,US 3223938816,3223946239,GB 3223946240,3223947519,CH 3223947520,3223947775,US @@ -54426,11 +56748,11 @@ 3223967744,3223967999,NL 3223968000,3223968255,US 3223968256,3223968511,NL -3223968512,3223970303,US +3223968512,3223970559,US 3223970560,3223970815,DE -3223971072,3223977983,US +3223970816,3223978239,US 3223978240,3223978495,NL -3223978752,3223979263,US +3223978496,3223979263,US 3223979264,3223979775,CA 3223979776,3223988735,US 3223988736,3223990271,CH @@ -54443,38 +56765,34 @@ 3223994112,3223994623,DE 3223994880,3223995391,US 3223995392,3223995647,CA -3223995648,3223996159,US -3223996416,3223999487,US +3223995648,3223999487,US 3224000256,3224000511,NL 3224000512,3224001023,US 3224001024,3224001279,CA -3224001280,3224002559,US -3224002816,3224003327,US +3224001280,3224003327,US 3224003328,3224003583,ZA 3224003584,3224003839,NL 3224003840,3224004095,US 3224004096,3224004351,NL 3224004352,3224005631,JP -3224005632,3224006655,US +3224005632,3224006911,US 3224006912,3224012031,NL 3224012032,3224014591,US 3224014592,3224014847,NL 3224014848,3224015615,US 3224015872,3224016639,US 3224016640,3224016895,AU -3224016896,3224017663,US -3224017920,3224024063,US +3224016896,3224024063,US 3224024064,3224029695,CH 3224029696,3224030463,CA +3224030464,3224030719,US 3224030720,3224030975,CA 3224030976,3224038655,US 3224038656,3224038911,AU 3224038912,3224039679,US -3224039936,3224040959,US -3224041216,3224042751,US +3224039936,3224042751,US 3224042752,3224043007,DE -3224043008,3224043263,US -3224043520,3224084991,US +3224043008,3224084991,US 3224084992,3224087551,SE 3224087552,3224088063,US 3224088064,3224088319,AU @@ -54482,32 +56800,34 @@ 3224090880,3224091135,AU 3224091648,3224091903,US 3224091904,3224092159,AU +3224092160,3224092415,CA 3224092416,3224092671,US 3224092672,3224093951,CH 3224093952,3224094207,US 3224094208,3224094463,AU 3224094464,3224094975,US 3224094976,3224095487,AU -3224095488,3224096255,US +3224095488,3224096511,US 3224096512,3224097279,AU 3224097280,3224097535,NL 3224097536,3224097791,US 3224097792,3224098047,NL -3224098048,3224098559,US -3224098816,3224099583,US +3224098048,3224099583,US 3224099584,3224099839,CA 3224099840,3224100863,US 3224101120,3224101375,US 3224101376,3224102399,AU 3224103424,3224103679,NL -3224103680,3224104447,US +3224103680,3224104703,US 3224104704,3224104959,AU 3224104960,3224105471,US 3224105728,3224106495,US -3224106752,3224108799,US +3224106752,3224109055,US 3224109056,3224109311,NL 3224109312,3224119551,DE -3224119552,3224129791,FR +3224119552,3224126463,FR +3224126976,3224127231,US +3224127232,3224129791,FR 3224129792,3224132351,DE 3224132352,3224170495,US 3224170496,3224173567,SE @@ -54859,7 +57179,7 @@ 3225664512,3225669887,DE 3225669888,3225671935,US 3225671936,3225672191,AU -3225672192,3225672447,NL +3225672192,3225672447,DE 3225672448,3225672703,US 3225672704,3225673215,NL 3225673472,3225673727,IE @@ -55626,7 +57946,19 @@ 3227724032,3227724287,US 3227724288,3227748035,CA 3227748036,3227748039,US -3227748040,3227779071,CA +3227748040,3227751868,CA +3227751869,3227751869,US +3227751870,3227756505,CA +3227756506,3227756506,US +3227756507,3227765503,CA +3227765504,3227765759,US +3227765760,3227777759,CA +3227777760,3227777763,US +3227777764,3227777951,CA +3227777952,3227777967,US +3227777968,3227777983,CA +3227777984,3227777999,US +3227778000,3227779071,CA 3227779328,3227779583,MU 3227779584,3227779839,AU 3227779840,3227783679,US @@ -55679,11 +58011,15 @@ 3227827200,3227829759,MX 3227829760,3227830015,BR 3227830016,3227831807,MX +3227831808,3227832063,BR 3227832064,3227833855,MX 3227833856,3227834367,BR 3227834368,3227837439,MX +3227837440,3227837951,BR 3227837952,3227842303,MX +3227842560,3227842815,BR 3227842816,3227843327,MX +3227843328,3227843583,BR 3227844096,3227844351,AR 3227844864,3227845119,ES 3227845120,3227845631,US @@ -55964,7 +58300,9 @@ 3228406016,3228406271,US 3228406272,3228407039,FR 3228407040,3228420095,DE -3228420608,3228426239,DE +3228420608,3228424703,DE +3228424704,3228424959,US +3228425216,3228426239,DE 3228426752,3228430847,DE 3228430848,3228431103,MU 3228431104,3228434431,DE @@ -56006,7 +58344,7 @@ 3228532480,3228532735,NL 3228532736,3228554751,US 3228555008,3228558591,US -3228558592,3228558847,BR +3228558592,3228559103,BR 3228559104,3228564479,US 3228564480,3228564735,AT 3228564992,3228565247,US @@ -56578,7 +58916,9 @@ 3230897664,3230897919,NL 3230898688,3230898943,US 3230898944,3230899199,NL -3230899200,3230913535,US +3230899200,3230913023,US +3230913024,3230913279,BR +3230913280,3230913535,US 3230913536,3230913791,NL 3230913792,3230914047,CA 3230914048,3230914303,US @@ -56934,7 +59274,8 @@ 3231483136,3231484927,US 3231484928,3231487999,JP 3231488512,3231488767,JP -3231489536,3231490047,AU +3231489536,3231489791,JP +3231489792,3231490047,AU 3231490048,3231490559,US 3231490560,3231490815,NL 3231490816,3231491071,US @@ -57359,15 +59700,7 @@ 3233288192,3233292287,CA 3233292288,3233431551,US 3233431552,3233447935,CA -3233447936,3233449095,US -3233449096,3233449103,GB -3233449104,3233449207,US -3233449208,3233449215,GB -3233449216,3233449359,US -3233449360,3233449367,GB -3233449368,3233450199,US -3233450200,3233450207,GB -3233450208,3233451519,US +3233447936,3233451519,US 3233451520,3233451775,GB 3233451776,3233453567,US 3233453568,3233453823,IT @@ -57399,7 +59732,8 @@ 3233564672,3233564927,US 3233564928,3233566719,JP 3233567744,3233568767,JP -3233568768,3233570047,AU +3233568768,3233569791,AU +3233569792,3233570047,JP 3233570048,3233570559,US 3233570816,3233571071,GB 3233571072,3233572095,US @@ -57428,7 +59762,7 @@ 3233583360,3233583615,NL 3233583616,3233584895,US 3233584896,3233585151,AU -3233585152,3233586431,US +3233585152,3233586175,US 3233586432,3233586943,NL 3233586944,3233588223,US 3233588224,3233589247,GA @@ -57722,7 +60056,9 @@ 3234226984,3234226991,CA 3234226992,3234227455,US 3234227456,3234227711,CA -3234227712,3234230015,US +3234227712,3234228223,US +3234228224,3234228351,CA +3234228352,3234230015,US 3234230016,3234230271,ES 3234230272,3234232319,US 3234232320,3234232575,EG @@ -57846,7 +60182,7 @@ 3234809088,3234810879,US 3234810880,3234811135,CA 3234811136,3234814719,US -3234814720,3234814975,AU +3234814720,3234814975,HK 3234814976,3234815999,US 3234816000,3234816767,AU 3234816768,3234820351,US @@ -57897,9 +60233,7 @@ 3235004416,3235021823,CA 3235021824,3235026943,US 3235026944,3235028991,CA -3235028992,3235038463,US -3235038464,3235038719,GB -3235038720,3235045375,US +3235028992,3235045375,US 3235045376,3235046143,CA 3235046144,3235085311,US 3235085312,3235086335,CA @@ -58207,7 +60541,7 @@ 3237328384,3237328639,US 3237328640,3237328895,CA 3237328896,3237329151,US -3237329152,3237329407,AU +3237329152,3237329407,NZ 3237329408,3237330943,US 3237330944,3237331199,AU 3237331456,3237331711,US @@ -58294,7 +60628,8 @@ 3237867776,3237868031,GB 3237868032,3237868287,HK 3237868288,3237869311,US -3237869312,3237869567,CA +3237869312,3237869439,KR +3237869440,3237869567,JP 3237869568,3237869823,AU 3237869824,3237869951,PH 3237869952,3237870079,VN @@ -58931,7 +61266,8 @@ 3239691520,3239691775,FR 3239691822,3239691822,GB 3239692032,3239692287,AT -3239692288,3239695871,DE +3239692288,3239694847,DE +3239695360,3239695871,DE 3239697408,3239697663,HR 3239697664,3239697919,RU 3239697920,3239698175,PL @@ -58975,7 +61311,6 @@ 3239741184,3239741439,RU 3239741440,3239759871,DE 3239760128,3239760383,UA -3239760640,3239760895,PL 3239761408,3239761663,RU 3239761920,3239762175,BG 3239762176,3239762431,RU @@ -58992,7 +61327,7 @@ 3239773952,3239774207,SA 3239774464,3239774719,ES 3239774976,3239775231,PT -3239775232,3239782399,DE +3239776256,3239782399,DE 3239782400,3239782655,AT 3239782656,3239782911,RU 3239782912,3239783167,GB @@ -59034,7 +61369,6 @@ 3239828992,3239829503,RU 3239830016,3239830527,CH 3239831040,3239831551,RU -3239832064,3239832575,UA 3239832576,3239834111,RU 3239834112,3239834623,UA 3239834624,3239835135,AT @@ -59042,7 +61376,6 @@ 3239836672,3239837183,DE 3239837184,3239837695,SE 3239837696,3239837951,PL -3239838208,3239838463,DE 3239838976,3239839231,DE 3239839232,3239839487,RU 3239839488,3239839743,DE @@ -59215,9 +61548,7 @@ 3240083456,3240085503,RU 3240085504,3240087551,KZ 3240087552,3240097791,DE -3240098304,3240098815,DE 3240098816,3240099327,CH -3240099584,3240099839,DE 3240100352,3240100863,GB 3240101376,3240101887,GB 3240102144,3240102399,GB @@ -59247,7 +61578,7 @@ 3240118272,3240120319,ES 3240120320,3240120831,IR 3240120832,3240121343,GB -3240122368,3240124927,GB +3240122368,3240123391,GB 3240125440,3240125695,RO 3240125696,3240132607,GB 3240140800,3240142847,GB @@ -59445,12 +61776,11 @@ 3240285184,3240286207,PL 3240286208,3240287231,UA 3240287232,3240288255,PL -3240288256,3240296447,GB +3240288256,3240292351,GB +3240294400,3240296447,GB 3240296448,3240296703,RO 3240297472,3240300543,GB 3240302848,3240303103,UA -3240303104,3240303359,GB -3240303616,3240304639,GB 3240304640,3240305663,RU 3240305664,3240305919,PL 3240306176,3240306687,RU @@ -59500,7 +61830,9 @@ 3240373760,3240374015,DE 3240374016,3240374271,GB 3240378368,3240394751,GB -3240395264,3240398847,GB +3240395264,3240395775,GB +3240396032,3240396287,GB +3240396800,3240398847,GB 3240400896,3240407039,GB 3240407040,3240407295,IL 3240407296,3240407551,NL @@ -59624,7 +61956,6 @@ 3240704000,3240705023,GR 3240705024,3240706047,UA 3240706048,3240707071,BG -3240707072,3240707839,NL 3240707840,3240709119,FR 3240709120,3240710143,RU 3240710144,3240710399,UA @@ -59674,7 +62005,7 @@ 3240739840,3240740095,DE 3240740096,3240740351,LT 3240740352,3240740607,IT -3240740608,3240741119,DE +3240740864,3240741119,DE 3240741120,3240741375,AT 3240741376,3240741631,IL 3240741632,3240741887,RU @@ -60037,7 +62368,9 @@ 3243238400,3243245567,NL 3243245568,3243376639,AT 3243376640,3243442175,GB -3243442176,3243507711,AT +3243442176,3243443199,AT +3243443200,3243443455,DE +3243443456,3243507711,AT 3243507712,3243507967,GB 3243509248,3243509759,CZ 3243509760,3243510015,RU @@ -61149,9 +63482,7 @@ 3246784512,3246825727,CH 3246825728,3246825983,GB 3246825984,3246826239,US -3246826240,3246874623,CH -3246874624,3246875647,DE -3246875648,3246915583,CH +3246826240,3246915583,CH 3246915584,3247046655,PT 3247046656,3247046911,AT 3247046912,3247048191,SI @@ -61243,7 +63574,9 @@ 3247112192,3247177727,FR 3247177728,3247243263,TR 3247243264,3247244287,DE -3247244288,3247253503,NL +3247244288,3247252319,NL +3247252320,3247252327,DE +3247252328,3247253503,NL 3247253504,3247254527,DE 3247254528,3247267839,NL 3247267840,3247268351,DE @@ -61262,7 +63595,6 @@ 3247322368,3247322623,DE 3247322624,3247323135,FI 3247323136,3247323647,RU -3247323648,3247323903,FI 3247324160,3247324415,FI 3247324416,3247324671,SE 3247324672,3247324927,CH @@ -61326,7 +63658,7 @@ 3247382528,3247390719,FI 3247392256,3247393791,FI 3247394048,3247394303,PL -3247394304,3247397887,FI +3247394560,3247397887,FI 3247397888,3247398143,RU 3247398144,3247399167,FI 3247399424,3247399679,RU @@ -61345,7 +63677,6 @@ 3247702016,3247702271,RO 3247702528,3247703551,ES 3247703552,3247704063,FR -3247705088,3247705855,ES 3247705856,3247706111,RU 3247708160,3247711743,ES 3247711744,3247712255,IT @@ -61355,9 +63686,7 @@ 3247714304,3247716351,CH 3247726592,3247742975,ES 3247742976,3247751167,DE -3247751168,3247764479,ES -3247764992,3247765247,ES -3247766528,3247769599,ES +3247751168,3247769599,ES 3247769600,3247771647,DE 3247775744,3247783935,DE 3247783936,3247792127,GB @@ -61443,8 +63772,7 @@ 3247909888,3247910911,DE 3247910912,3247912959,PL 3247912960,3247913983,UA -3247913984,3247914239,DE -3247914240,3247915007,AT +3247913984,3247915007,DE 3247915008,3247917055,PL 3247917056,3247918079,NL 3247918080,3247919103,PL @@ -61474,14 +63802,14 @@ 3248490752,3248491007,NO 3248491520,3248492031,RU 3248493568,3248493823,NO -3248494592,3248496639,NO +3248494592,3248496127,NO 3248496896,3248497151,IL 3248497152,3248498431,NO 3248498432,3248498687,DE 3248498688,3248504831,NO 3248513280,3248513535,UA 3248513536,3248514047,AT -3248514560,3248521215,NO +3248514816,3248521215,NO 3248521984,3248522239,RU 3248522240,3248522751,NO 3248523264,3248524287,NO @@ -61511,7 +63839,9 @@ 3248603648,3248604159,NO 3248604928,3248608767,NO 3248609280,3248619519,NO -3248619520,3248750591,DK +3248619520,3248624383,DK +3248624384,3248624639,US +3248624640,3248750591,DK 3248750592,3248750847,PT 3248751616,3248752127,PL 3248752640,3248752895,DE @@ -61692,7 +64022,9 @@ 3249141760,3249142271,RU 3249142784,3249143295,UA 3249143296,3249143807,GB -3249143808,3249274879,AT +3249143808,3249242879,AT +3249242880,3249243135,DE +3249243136,3249274879,AT 3249274880,3249405951,NL 3249405952,3249521279,DE 3249521280,3249521343,UA @@ -61710,7 +64042,8 @@ 3249574912,3249583103,NL 3249583616,3249590527,NL 3249590528,3249590783,FR -3249590784,3249600255,NL +3249590784,3249591295,NL +3249591808,3249600255,NL 3249600256,3249600511,AT 3249600512,3249601535,UA 3249601536,3249601791,RU @@ -61760,7 +64093,6 @@ 3249711360,3249711615,DE 3249711872,3249712127,AT 3249712384,3249712639,GB -3249712640,3249712895,BE 3249712896,3249713151,DK 3249713152,3249715199,LV 3249715200,3249715455,AT @@ -62140,7 +64472,6 @@ 3250847744,3250978815,DE 3250978816,3251044351,HR 3251044352,3251109887,FI -3251110144,3251110655,BG 3251110656,3251110911,IT 3251110912,3251111167,FR 3251111168,3251111423,CH @@ -62733,7 +65064,7 @@ 3252587264,3252587519,AT 3252587520,3252587775,UA 3252587776,3252588031,SE -3252588032,3252600319,CH +3252588032,3252599807,CH 3252600320,3252600575,NL 3252600576,3252600831,RU 3252600832,3252616703,CH @@ -63247,7 +65578,6 @@ 3254648832,3254649087,GB 3254649088,3254649855,AL 3254649856,3254650879,SE -3254650880,3254653439,CH 3254653440,3254654847,DE 3254654848,3254654975,DK 3254654976,3254655999,IT @@ -63386,7 +65716,6 @@ 3254828544,3254828799,RO 3254828800,3254829055,SI 3254829056,3254829311,NO -3254829312,3254829567,GB 3254829568,3254829823,NL 3254829824,3254830079,FR 3254830080,3254830335,IE @@ -63514,7 +65843,8 @@ 3255058432,3255067647,FR 3255068672,3255114751,FR 3255115264,3255117823,FR -3255118336,3255120639,FR +3255118336,3255120127,FR +3255120384,3255120639,FR 3255120640,3255120895,DE 3255120896,3255123711,FR 3255123712,3255123967,DE @@ -63742,7 +66072,8 @@ 3255544576,3255544831,AT 3255544832,3255558143,CH 3255558400,3255558655,UA -3255558656,3255563263,CH +3255558656,3255561983,CH +3255562240,3255563263,CH 3255563776,3255564031,CH 3255564032,3255564287,RU 3255564288,3255565311,CH @@ -64023,7 +66354,6 @@ 3256975360,3256988671,GB 3256988672,3256988927,RU 3256988928,3256989183,UA -3256989184,3256989439,FR 3256989440,3256989695,GB 3256989696,3256989951,HU 3256989952,3256990207,PL @@ -64036,8 +66366,9 @@ 3257011200,3257011455,BG 3257011456,3257024511,GB 3257024512,3257032703,AU -3257032704,3257051135,GB -3257051648,3257057279,GB +3257032704,3257040895,GB +3257042944,3257051135,GB +3257052160,3257057279,GB 3257058816,3257059071,PL 3257059072,3257092607,GB 3257092608,3257092863,RO @@ -64111,12 +66442,9 @@ 3257470976,3257471999,FI 3257472000,3257472511,SG 3257472512,3257475071,FI -3257475328,3257475583,FI -3257475584,3257475839,DE 3257476096,3257477119,DE -3257477120,3257477375,NL -3257477376,3257477631,SE -3257479168,3257481471,GB +3257480192,3257480447,GB +3257481216,3257481471,GB 3257481472,3257481727,DE 3257481728,3257481983,FI 3257481984,3257482239,FR @@ -64140,8 +66468,8 @@ 3257546688,3257546719,DE 3257546720,3257546751,DK 3257546752,3257548799,IE -3257548800,3257554943,GB -3257554944,3257555455,CH +3257548800,3257555199,GB +3257555200,3257555455,CH 3257555456,3257556991,GB 3257557504,3257558015,LU 3257558016,3257559039,RO @@ -64179,7 +66507,9 @@ 3257748480,3257749503,DE 3257749504,3257753087,NL 3257753088,3257753343,DE -3257753344,3257765887,NL +3257753344,3257759631,NL +3257759632,3257759639,DE +3257759640,3257765887,NL 3257765888,3257767935,DE 3257767936,3257782271,NL 3257782272,3257784319,DE @@ -64213,7 +66543,7 @@ 3257987584,3257988095,AT 3257989120,3257991167,AT 3257991168,3257995519,DE -3257995776,3258003967,DE +3257996032,3258003967,DE 3258003968,3258004479,RU 3258004480,3258018815,DE 3258019328,3258021887,DE @@ -64223,7 +66553,7 @@ 3258023424,3258023679,TR 3258023680,3258023935,RU 3258023936,3258056703,DE -3258057216,3258057727,CZ +3258057216,3258057471,CZ 3258058240,3258058495,RU 3258058496,3258059007,RO 3258059008,3258059263,UA @@ -64408,7 +66738,9 @@ 3258427648,3258427903,RO 3258427904,3258428159,DE 3258428416,3258449919,DE -3258449920,3258503935,CH +3258449920,3258486783,CH +3258486784,3258487807,LI +3258487808,3258503935,CH 3258503936,3258504191,PL 3258504192,3258504703,CH 3258504704,3258504959,DE @@ -64429,7 +66761,8 @@ 3258693888,3258694143,SI 3258694144,3258694399,RU 3258694656,3258694911,GB -3258694912,3258712063,DE +3258694912,3258701823,DE +3258702848,3258712063,DE 3258712064,3258728447,GB 3258728448,3258729471,FR 3258729472,3258729727,DE @@ -64449,12 +66782,13 @@ 3258733056,3258733311,RO 3258733312,3258734591,GB 3258735104,3258735359,GB -3258736640,3258762751,GB +3258736640,3258745855,GB +3258746880,3258762751,GB 3258763264,3258764287,GB 3258764288,3258764543,DE 3258764800,3258765055,BE 3258765056,3258765311,NL -3258765312,3258767615,GB +3258767360,3258767615,GB 3258767616,3258767871,CH 3258767872,3258769919,GB 3258769920,3258770431,PL @@ -64593,7 +66927,7 @@ 3259760640,3259814399,DE 3259814400,3259814655,AT 3259814656,3259816447,DE -3259816704,3259821823,DE +3259816704,3259821055,DE 3259821824,3259822079,AT 3259822080,3259823103,DE 3259823104,3259823615,RO @@ -64696,7 +67030,7 @@ 3260678144,3260743679,IL 3260743680,3260809215,IT 3260809216,3260874751,PL -3260874752,3260893439,DK +3260891136,3260893439,DK 3260893440,3260894207,SE 3260894208,3260895231,AT 3260895232,3260898303,SE @@ -64722,7 +67056,8 @@ 3261202432,3261213439,FR 3261213440,3261213695,AF 3261213696,3261267967,FR -3261267968,3261297663,DE +3261267968,3261281023,DE +3261281280,3261297663,DE 3261297664,3261297919,RU 3261297920,3261298175,PL 3261298176,3261333503,DE @@ -64833,7 +67168,7 @@ 3261857792,3261923327,CZ 3261923328,3261988863,NL 3261988864,3261989119,SE -3261989120,3261989887,FI +3261989120,3261989631,FI 3261990144,3261990399,FI 3261990400,3261990655,UA 3261990656,3261990911,FI @@ -64862,7 +67197,7 @@ 3262017536,3262018559,FI 3262018560,3262018815,PL 3262018816,3262019071,FI -3262019584,3262021119,FI +3262020096,3262021119,FI 3262021120,3262021375,UA 3262021376,3262021631,PL 3262021632,3262021887,CH @@ -64916,7 +67251,6 @@ 3262050816,3262051071,CH 3262051072,3262051583,GB 3262051584,3262051839,FR -3262052352,3262052607,UA 3262052608,3262052863,IL 3262052864,3262053119,GB 3262053120,3262053375,RU @@ -64968,7 +67302,8 @@ 3262227712,3262227967,RO 3262227968,3262229247,DE 3262229248,3262229503,NL -3262229504,3262283775,DE +3262229504,3262248191,DE +3262248448,3262283775,DE 3262283776,3262284799,RU 3262284800,3262286847,UA 3262286848,3262287871,SE @@ -65038,7 +67373,6 @@ 3262460672,3262460927,AE 3262460928,3262461055,CY 3262461056,3262461183,DK -3262461184,3262461311,SN 3262461312,3262461439,NO 3262461440,3262461567,DE 3262461568,3262461695,GB @@ -65168,7 +67502,11 @@ 3262511104,3262511615,GB 3262512128,3262512639,UA 3262512640,3262513151,DE -3262513152,3262578687,AT +3262513152,3262550271,AT +3262550272,3262550527,DE +3262550528,3262562815,AT +3262562816,3262563071,HU +3262563072,3262578687,AT 3262578688,3262611455,FR 3262611456,3262627839,GB 3262627840,3262636031,IT @@ -65177,7 +67515,9 @@ 3262648320,3262648575,DE 3262648576,3262654463,NL 3262654464,3262654719,DE -3262654720,3262664703,NL +3262654720,3262658967,NL +3262658968,3262658975,DE +3262658976,3262664703,NL 3262664704,3262670847,DE 3262670848,3262690815,NL 3262690816,3262691583,DE @@ -65185,7 +67525,9 @@ 3262693376,3262701567,DE 3262701568,3262715135,NL 3262715136,3262715391,DE -3262715392,3262722815,NL +3262715392,3262718463,NL +3262718464,3262718471,DE +3262718472,3262722815,NL 3262722816,3262723071,DE 3262723072,3262732799,NL 3262732800,3262733055,DE @@ -65194,7 +67536,9 @@ 3262754816,3262832639,NL 3262832640,3262840319,DE 3262840320,3262906367,NL -3262906368,3262964991,CH +3262906368,3262954495,CH +3262954496,3262955519,LI +3262955520,3262964991,CH 3262964992,3262965247,DE 3262965248,3262971903,CH 3262971904,3263029247,IE @@ -65358,7 +67702,6 @@ 3264014848,3264015103,IE 3264015104,3264015359,RO 3264015360,3264015615,DK -3264015616,3264015871,UA 3264015872,3264016127,PT 3264016128,3264016383,UA 3264016384,3264016639,SE @@ -65507,13 +67850,18 @@ 3264409600,3264410623,RU 3264410624,3264411647,NO 3264411648,3264413695,PL -3264417536,3264431103,CH +3264417536,3264419327,CH +3264423424,3264428031,CH +3264430592,3264431103,CH 3264431104,3264431615,LI 3264431616,3264431871,CH 3264432128,3264434687,CH -3264434944,3264441343,CH +3264434944,3264438783,CH +3264439296,3264439551,CH +3264439808,3264441343,CH 3264441344,3264441599,PL -3264442112,3264444927,CH +3264442112,3264444415,CH +3264444672,3264444927,CH 3264444928,3264445183,DE 3264445184,3264445439,CH 3264447488,3264447743,CH @@ -65564,7 +67912,6 @@ 3264624640,3264624671,US 3264624672,3264626687,GB 3264626688,3264627711,EE -3264627712,3264628735,GR 3264628736,3264630783,UA 3264630784,3264631807,DE 3264631808,3264632831,RO @@ -65661,7 +68008,6 @@ 3264831488,3264831743,CH 3264831744,3264831999,FR 3264832000,3264832255,IE -3264832256,3264832511,IT 3264832512,3264832767,DE 3264832768,3264833023,IL 3264833536,3264834047,DE @@ -65681,7 +68027,7 @@ 3264840192,3264840447,DE 3264840448,3264840703,IT 3264840704,3264840959,PT -3264840960,3264841471,PL +3264841216,3264841471,PL 3264841472,3264841727,LV 3264841728,3264844031,GB 3264844032,3264844287,BG @@ -65734,11 +68080,10 @@ 3264937984,3265003519,GB 3265003520,3265018879,DE 3265018880,3265019903,HK -3265019904,3265042943,DE +3265036288,3265042943,DE 3265043456,3265045759,DE 3265045760,3265046015,TR -3265046016,3265048575,DE -3265049600,3265055231,DE +3265046016,3265055231,DE 3265055232,3265055743,FR 3265055744,3265069055,DE 3265069056,3265134591,FI @@ -65965,7 +68310,8 @@ 3266617344,3266634383,DE 3266634392,3266634399,EE 3266634400,3266634431,DE -3266634464,3266637055,DE +3266634464,3266634751,DE +3266635520,3266637055,DE 3266637312,3266641919,DE 3266641920,3266707455,PL 3266707456,3266772991,DK @@ -66216,7 +68562,9 @@ 3268345856,3268411391,GB 3268411392,3268426751,AT 3268426752,3268427775,CH -3268427776,3268464127,AT +3268427776,3268463615,AT +3268463616,3268463871,CH +3268463872,3268464127,AT 3268464128,3268464383,LI 3268464384,3268476927,AT 3268476928,3268537087,CH @@ -66406,7 +68754,8 @@ 3270644224,3270644735,NL 3270644736,3270645247,DE 3270645760,3270646271,IL -3270646272,3270647807,RU +3270646272,3270646783,RU +3270647296,3270647807,RU 3270647808,3270648063,TR 3270648064,3270648319,RU 3270648320,3270648575,CH @@ -66928,7 +69277,6 @@ 3272480512,3272480767,CH 3272480768,3272481023,DE 3272481024,3272481279,SE -3272481536,3272481791,FR 3272481792,3272482047,IT 3272482048,3272482303,NL 3272482304,3272482559,PL @@ -67144,7 +69492,7 @@ 3273331712,3273331743,GB 3273331752,3273331791,GB 3273331808,3273331823,GB -3273331832,3273331887,GB +3273331828,3273331887,GB 3273331904,3273331967,GB 3273331968,3273332031,DE 3273332032,3273332095,GB @@ -67157,6 +69505,7 @@ 3273334784,3273335039,AE 3273335040,3273335295,DE 3273335296,3273335423,GB +3273335432,3273335439,GB 3273335936,3273335999,DE 3273336848,3273336863,DE 3273336864,3273336871,GB @@ -67354,8 +69703,6 @@ 3273873920,3273874431,GB 3273874432,3273875455,GR 3273875456,3273875711,PT -3273875712,3273875967,SE -3273875968,3273876223,DE 3273876224,3273876479,IT 3273876480,3273876991,SE 3273877248,3273877503,DE @@ -67640,7 +69987,6 @@ 3274690304,3274690559,GB 3274690560,3274690815,GR 3274690816,3274691071,ES -3274691072,3274691327,DK 3274691328,3274691583,SI 3274691840,3274692095,SI 3274692096,3274692351,RO @@ -67744,7 +70090,6 @@ 3275120640,3275137023,SE 3275137536,3275138047,UA 3275138048,3275138559,DE -3275138560,3275139071,UA 3275139072,3275139583,PT 3275140096,3275140607,AT 3275140608,3275141119,NL @@ -68166,7 +70511,7 @@ 3276109824,3276110847,BG 3276110848,3276111871,PL 3276111872,3276112895,DK -3276112896,3276114943,UA +3276112896,3276113919,UA 3276115968,3276116991,NL 3276116992,3276118015,RU 3276118016,3276119039,GB @@ -68314,6 +70659,7 @@ 3276695040,3276695551,RU 3276695552,3276696063,UA 3276696064,3276696575,RO +3276696576,3276697087,CZ 3276697088,3276697599,GB 3276697600,3276698111,UA 3276698112,3276699647,RU @@ -68369,7 +70715,9 @@ 3276865536,3276866303,NL 3276866304,3276866559,GB 3276866560,3276866815,IT -3276866816,3276873983,GB +3276866816,3276870911,GB +3276870912,3276871423,IT +3276871424,3276873983,GB 3276873984,3276874239,ES 3276874240,3276876383,GB 3276876384,3276876415,NL @@ -68387,7 +70735,9 @@ 3276893696,3276893951,IT 3276893952,3276898671,GB 3276898672,3276898687,CH -3276898688,3276902655,GB +3276898688,3276902141,GB +3276902142,3276902142,CH +3276902143,3276902655,GB 3276902656,3276902911,SE 3276902912,3276903167,GB 3276903168,3276903679,SE @@ -68406,7 +70756,9 @@ 3276912208,3276912215,IT 3276912216,3276912383,GB 3276912384,3276912511,IT -3276912512,3276912879,GB +3276912512,3276912671,GB +3276912672,3276912687,IT +3276912688,3276912879,GB 3276912880,3276912895,IT 3276912896,3276917231,GB 3276917232,3276917247,FR @@ -69211,7 +71563,6 @@ 3280576512,3280576767,BY 3280576768,3280577279,PL 3280577280,3280577535,DE -3280577536,3280577791,IL 3280577792,3280578047,NL 3280578048,3280578303,RO 3280578304,3280578559,UA @@ -69237,7 +71588,6 @@ 3280583936,3280584191,PL 3280584192,3280584447,KW 3280584448,3280584703,DE -3280584704,3280585215,UA 3280585216,3280585727,DK 3280585728,3280586239,UA 3280586240,3280586751,DE @@ -69905,7 +72255,8 @@ 3284016384,3284016639,CH 3284016640,3284017151,DK 3284017152,3284025343,GR -3284025344,3284030479,GB +3284025344,3284030471,GB +3284030472,3284030479,IL 3284030480,3284030495,FR 3284030496,3284033535,GB 3284033536,3284041727,RU @@ -70365,10 +72716,11 @@ 3285913656,3285913703,GB 3285913708,3285913711,FI 3285917696,3285917703,GB -3285917728,3285917759,GB +3285917712,3285917759,GB 3285919744,3285921791,QA 3285922048,3285922303,FR 3285924912,3285924919,CH +3285925164,3285925171,CH 3285926432,3285926463,CH 3285926592,3285926623,DE 3285928304,3285928311,GB @@ -70550,7 +72902,9 @@ 3286662400,3286662655,UA 3286662656,3286662911,DE 3286662912,3286671359,UA -3286671360,3286679551,AT +3286671360,3286672639,AT +3286672640,3286672895,HU +3286672896,3286679551,AT 3286679552,3286695935,IT 3286695936,3286761471,DK 3286761472,3286773759,GB @@ -70639,7 +72993,6 @@ 3286932736,3286932991,AT 3286932992,3286933247,IL 3286933504,3286933759,DK -3286933760,3286934015,RO 3286934016,3286934271,BE 3286934272,3286934527,CH 3286934528,3286934783,PL @@ -70900,7 +73253,6 @@ 3287671296,3287671551,PL 3287671552,3287671807,TR 3287671808,3287672063,CH -3287672064,3287672319,LT 3287672320,3287672575,DE 3287672576,3287672831,NL 3287672832,3287673087,RU @@ -71157,9 +73509,10 @@ 3288485888,3288489983,MA 3288489984,3288514559,ZA 3288514560,3288522751,EG +3288522752,3288530943,ZA 3288530944,3288532991,JM 3288532992,3288534527,PR -3288534528,3288535039,MG +3288534528,3288535039,EG 3288535040,3288539135,CW 3288539136,3288540415,US 3288540416,3288540671,BR @@ -71244,7 +73597,7 @@ 3289021440,3289024767,ZA 3289024768,3289025023,GH 3289025024,3289025535,UG -3289026048,3289027327,ZA +3289025536,3289027327,ZA 3289027328,3289027583,TZ 3289027584,3289027839,MZ 3289027840,3289041407,ZA @@ -71302,10 +73655,12 @@ 3289217280,3289217535,KE 3289218560,3289220351,ZA 3289220352,3289220607,TZ +3289220608,3289221119,ZA 3289221632,3289229311,ZA 3289229312,3289229567,SZ 3289229824,3289230591,ZA 3289233408,3289233919,ZA +3289233920,3289234175,TZ 3289234176,3289235199,ZA 3289235200,3289235455,KE 3289237504,3289243391,ZA @@ -71408,8 +73763,7 @@ 3291045888,3291078655,ZA 3291078656,3291086847,DZ 3291086848,3291103231,PR -3291103232,3291119615,ZA -3291152384,3291168767,ZA +3291103232,3291168767,ZA 3291168768,3291176959,TZ 3291176960,3291185151,ZW 3291185152,3291201535,UG @@ -71542,6 +73896,7 @@ 3291546368,3291546623,BI 3291546624,3291546879,SZ 3291546880,3291547135,TZ +3291547136,3291547391,AO 3291742208,3292004351,US 3292004352,3292266495,SC 3292397568,3292528639,ZA @@ -71552,7 +73907,7 @@ 3300937728,3300938751,MU 3300941824,3300950015,MU 3300953088,3300954111,MU -3300982784,3301113855,ZA +3300966400,3301113855,ZA 3301113856,3301146623,NG 3301146624,3301175295,ZA 3301175296,3301177343,AF @@ -71729,7 +74084,7 @@ 3302957824,3302958079,NA 3302958080,3302958335,BI 3302958336,3302958591,SZ -3302985728,3302987775,MU +3302958592,3302958847,DJ 3304062976,3304456191,SC 3304456192,3304521727,NG 3304521728,3304587263,SC @@ -72114,9 +74469,15 @@ 3323682960,3323684863,CA 3323684864,3323685375,US 3323685376,3323685887,CA -3323685888,3323687423,US -3323687424,3323688959,CA -3323688960,3323689471,US +3323685888,3323686399,US +3323686400,3323686911,CA +3323686912,3323687423,US +3323687424,3323687935,CA +3323687936,3323687999,US +3323688000,3323688959,CA +3323688960,3323689199,US +3323689200,3323689215,BY +3323689216,3323689471,US 3323689472,3323690495,CA 3323690496,3323741439,US 3323741440,3323741695,GB @@ -72565,13 +74926,17 @@ 3328479232,3328481279,CA 3328481280,3328481791,US 3328481792,3328482303,CA -3328482304,3328510064,US +3328482304,3328483071,US +3328483072,3328483327,CA +3328483328,3328510064,US 3328510065,3328510077,AE 3328510078,3328515071,US 3328515072,3328516095,DM 3328516096,3328617983,US 3328617984,3328618239,CA -3328618240,3328630015,US +3328618240,3328629631,US +3328629632,3328629695,GB +3328629696,3328630015,US 3328630016,3328630271,CA 3328630272,3328630783,US 3328630784,3328631807,CA @@ -72726,7 +75091,8 @@ 3332491264,3332492031,US 3332492032,3332500735,CA 3332500736,3332500991,US -3332500992,3332503039,CA +3332500992,3332501247,CA +3332501504,3332503039,CA 3332503040,3332503551,US 3332503552,3332505343,CA 3332505344,3332505855,US @@ -72839,7 +75205,9 @@ 3333624320,3333624575,CA 3333624576,3333675775,US 3333675776,3333676031,CA -3333676032,3333701887,US +3333676032,3333677311,US +3333677312,3333677567,HK +3333677568,3333701887,US 3333701888,3333702143,GB 3333702144,3333702399,US 3333702400,3333702655,CH @@ -73428,13 +75796,7 @@ 3343364096,3343365631,US 3343365632,3343366655,CA 3343366912,3343372543,CA -3343372800,3343378511,US -3343378512,3343378519,GB -3343378520,3343378535,US -3343378536,3343378543,GB -3343378544,3343378575,US -3343378576,3343378583,GB -3343378584,3343378823,US +3343372800,3343378823,US 3343378824,3343378839,SE 3343378840,3343379079,US 3343379080,3343379087,SE @@ -73530,6 +75892,7 @@ 3344678912,3344681983,US 3344681984,3344685055,CA 3344685056,3344691199,US +3344691200,3344693247,CA 3344693248,3344694271,US 3344694272,3344695295,CA 3344695296,3344937471,US @@ -73877,9 +76240,7 @@ 3351336960,3351339007,CA 3351339008,3351357439,US 3351357440,3351359487,CA -3351359488,3351364607,US -3351364608,3351365119,AU -3351365120,3351372799,US +3351359488,3351372799,US 3351372800,3351373823,BM 3351373824,3351380223,US 3351380224,3351380479,CA @@ -74066,7 +76427,9 @@ 3353653504,3353653759,GB 3353653760,3353722367,US 3353722368,3353722623,GB -3353722624,3353729023,US +3353722624,3353726975,US +3353726976,3353727231,IN +3353727232,3353729023,US 3353729024,3353729279,HK 3353729280,3353730047,US 3353730048,3353731071,CA @@ -74188,7 +76551,8 @@ 3355459840,3355460095,VE 3355460096,3355460351,CL 3355460352,3355460607,BR -3355460864,3355461631,BR +3355460608,3355460863,UY +3355460864,3355461887,BR 3355461888,3355463423,EC 3355463424,3355463935,AR 3355463936,3355464191,BR @@ -74196,7 +76560,9 @@ 3355464448,3355464959,MX 3355464960,3355465727,BR 3355465728,3355465983,UY +3355466240,3355466751,MX 3355466752,3355467007,BR +3355467008,3355467263,MX 3355467264,3355467519,US 3355467520,3355467775,MX 3355467776,3355468799,AR @@ -74867,10 +77233,13 @@ 3357581312,3357589503,CL 3357589504,3357605887,BZ 3357605888,3357606911,MX +3357606912,3357607167,AR 3357607168,3357613055,MX +3357613056,3357613311,AR 3357613312,3357616127,MX 3357616384,3357618943,MX 3357619200,3357623039,MX +3357623040,3357623295,AR 3357623296,3357626623,MX 3357627392,3357627647,MX 3357627904,3357628415,MX @@ -74888,9 +77257,12 @@ 3357648896,3357650431,MX 3357650944,3357657855,MX 3357658112,3357670655,MX -3357670912,3357712383,MX +3357670912,3357671423,MX +3357671424,3357671679,CO +3357671680,3357712383,MX 3357712384,3357713407,BR 3357713408,3357715199,MX +3357715200,3357715455,CO 3357715456,3357715711,MX 3357715968,3357723903,MX 3357724416,3357725183,MX @@ -75317,7 +77689,7 @@ 3361538048,3361570815,VE 3361570816,3361587199,CO 3361587200,3361599487,EC -3361599488,3361601279,UY +3361599488,3361601535,UY 3361601536,3361603583,EC 3361603584,3361665023,MX 3361665024,3361667071,BR @@ -75503,7 +77875,7 @@ 3363684352,3363708927,EC 3363708928,3363713023,CL 3363713024,3363713279,EC -3363713536,3363714047,CL +3363713280,3363714047,CL 3363714048,3363717119,PE 3363717120,3363733503,CL 3363733504,3363831807,AR @@ -75522,9 +77894,12 @@ 3368087552,3370188799,BR 3370188800,3370196991,MX 3370196992,3370487807,BR +3370487808,3370488831,CR 3370488832,3370489855,AR 3370489856,3370490879,VE -3370490880,3370514943,BR +3370490880,3370506239,BR +3370506240,3370507263,VE +3370507264,3370514943,BR 3370515456,3370516479,AR 3370516480,3376873471,BR 3376881664,3376922623,BR @@ -75538,13 +77913,16 @@ 3380745216,3380746239,BR 3380746240,3380747263,PE 3380747264,3380748287,BR -3380748288,3380760831,MX +3380748288,3380761087,MX +3380761088,3380761599,VE 3380761600,3380764671,BR 3380764672,3380808191,MX 3380808704,3380811775,MX 3380811776,3380813823,BR 3380813824,3380815103,MX +3380815104,3380815359,CR 3380815872,3380816127,MX +3380816128,3380816383,BO 3380816896,3380817151,MX 3380817920,3380818175,MX 3380818944,3380822527,MX @@ -75564,7 +77942,8 @@ 3380837376,3380840447,MX 3380840448,3380843519,BR 3380843520,3380844543,PA -3380844544,3380854783,BR +3380844544,3380853759,BR +3380853760,3380854783,CO 3380854784,3380858879,MX 3380858880,3380860927,BR 3380860928,3380861951,CO @@ -75573,7 +77952,8 @@ 3380865024,3380867071,MX 3380867072,3380869119,BR 3380869120,3381350399,MX -3381350400,3381354495,BR +3381350400,3381351423,AR +3381351424,3381354495,BR 3381354496,3381373951,MX 3381373952,3381374975,BR 3381374976,3381377023,MX @@ -75585,20 +77965,37 @@ 3381452800,3381453823,AR 3381453824,3381455871,BR 3381455872,3381456895,CO +3381456896,3381460991,BR 3381460992,3381944319,MX -3381947392,3381948415,BR +3381944320,3381946367,BR +3381946368,3381947391,CL +3381947392,3381952511,BR 3381952512,3381960703,MX +3381960704,3381962751,BR +3381962752,3381963775,HN +3381963776,3381968895,BR 3381968896,3381972991,MX 3381972992,3381974015,BR 3381974016,3381975039,CL -3381976064,3381977087,BR +3381975040,3381979135,BR +3381979136,3381980159,CR +3381980160,3381985279,BR 3381985280,3382009855,MX +3382009856,3382018047,BR 3382018048,3382053887,MX -3382053888,3382059007,BR +3382053888,3382057983,BR +3382057984,3382059007,CL 3382059008,3382063103,MX -3382063104,3382064127,BR +3382063104,3382066175,BR +3382066176,3382067199,PE 3382067200,3382071295,MX -3382071296,3382075391,BR +3382071296,3382072319,BR +3382072320,3382073343,SV +3382073344,3382075391,BR +3382075392,3382076415,CO +3382076416,3382077439,BR +3382077440,3382078463,HT +3382078464,3382083583,BR 3382083584,3382087679,MX 3382087680,3382091775,BR 3382091776,3382099967,BO @@ -75631,8 +78028,9 @@ 3382661120,3382662143,MX 3382662144,3382665215,BR 3382665216,3382669311,MX -3382669312,3382673407,BR -3382673408,3382677503,MX +3382669312,3382670335,GT +3382670336,3382672383,BR +3382672384,3382677503,MX 3382677504,3382681599,BR 3382681600,3382683647,MX 3382683648,3382685695,CL @@ -75642,7 +78040,7 @@ 3382695936,3382696959,AR 3382696960,3382697983,EC 3382697984,3382704127,MX -3382704128,3382705151,BR +3382704128,3382706175,BR 3382706176,3382853631,MX 3382853632,3382870015,BR 3382870016,3383630843,MX @@ -75737,7 +78135,7 @@ 3386761216,3386769407,PA 3386769408,3386773503,VE 3386773504,3386774527,AR -3386775552,3386776575,BR +3386774528,3386777599,BR 3386777600,3386781695,CW 3386781696,3386783743,PY 3386783744,3386784767,AR @@ -75880,11 +78278,7 @@ 3389197824,3389198079,ID 3389198080,3389198335,IN 3389198336,3389202431,KR -3389202432,3389203455,NZ -3389203456,3389203967,AU -3389203968,3389204223,NZ -3389204224,3389204479,AU -3389204480,3389210623,NZ +3389202432,3389210623,AU 3389210624,3389210879,IN 3389210880,3389211135,AU 3389211136,3389211391,IN @@ -76065,7 +78459,8 @@ 3389595648,3389595903,CN 3389595904,3389596159,AU 3389596160,3389596671,CN -3389596672,3389597695,MY +3389596672,3389597439,SG +3389597440,3389597695,MY 3389597696,3389599743,MN 3389599744,3389600255,CN 3389600256,3389600511,AU @@ -76102,7 +78497,7 @@ 3389661184,3389669375,ID 3389669376,3389673471,CN 3389673472,3389677567,AU -3389677568,3389681663,JP +3389677568,3389685759,JP 3389685760,3389718527,AU 3389718528,3389784063,JP 3389784064,3389784319,NZ @@ -76111,8 +78506,8 @@ 3389785088,3389786111,HK 3389786112,3389788159,JP 3389788160,3389788415,IN -3389788416,3389789183,CN -3389789184,3389789695,TH +3389788416,3389788927,CN +3389788928,3389789695,US 3389789696,3389790719,AU 3389790720,3389790975,BN 3389790976,3389791231,JP @@ -76157,7 +78552,7 @@ 3389919232,3389931519,NZ 3389931520,3389932031,CN 3389932032,3389932287,AU -3389932288,3389932799,HK +3389932288,3389932799,JP 3389932800,3389933055,CN 3389933056,3389933567,PK 3389933568,3389933823,IN @@ -76243,7 +78638,8 @@ 3390310400,3390316543,AU 3390316544,3390325247,NZ 3390325248,3390325503,CN -3390325504,3390327807,AU +3390325504,3390325759,AU +3390326016,3390327807,AU 3390327808,3390328575,NZ 3390328576,3390328831,CN 3390328832,3390329087,NZ @@ -76264,7 +78660,7 @@ 3390337024,3390337279,GU 3390337280,3390337535,NZ 3390337536,3390337791,CN -3390337792,3390338303,SG +3390337792,3390338303,JP 3390338304,3390338559,CN 3390338560,3390338815,HK 3390338816,3390339071,KR @@ -76291,7 +78687,7 @@ 3390409984,3390410495,CN 3390410496,3390410751,NZ 3390410752,3390411007,TH -3390411008,3390411519,SG +3390411008,3390411519,JP 3390411520,3390412031,CN 3390412032,3390412287,TH 3390412288,3390413567,CN @@ -76386,8 +78782,7 @@ 3391664128,3391668223,BD 3391668224,3391676415,HK 3391676416,3391684607,SG -3391684608,3391685631,IN -3391685632,3391686655,AU +3391684608,3391686655,IN 3391686656,3391687167,CN 3391687168,3391687423,IN 3391687424,3391688191,CN @@ -76407,85 +78802,16 @@ 3391725568,3391733759,TH 3391733760,3391734015,CN 3391734016,3391734783,AU +3391735808,3391736831,JP 3391736832,3391737855,IN 3391737856,3391741951,JP -3391741952,3391742207,HK -3391742208,3391742211,IN -3391742212,3391742223,HK -3391742224,3391742227,IN -3391742228,3391742231,HK -3391742232,3391742243,IN -3391742244,3391742287,HK -3391742288,3391742291,IN -3391742292,3391742319,HK -3391742320,3391742323,IN -3391742324,3391742327,HK -3391742328,3391742331,IN -3391742332,3391742335,HK -3391742336,3391742343,IN -3391742344,3391742347,HK -3391742348,3391742351,IN -3391742352,3391742399,HK -3391742400,3391742403,IN -3391742404,3391742423,HK -3391742424,3391742427,IN -3391742428,3391742443,HK -3391742444,3391742447,IN -3391742448,3391742471,HK -3391742472,3391742475,IN -3391742476,3391742523,HK -3391742524,3391742531,IN -3391742532,3391742571,HK -3391742572,3391742575,IN -3391742576,3391742591,HK -3391742592,3391742595,IN -3391742596,3391742687,HK -3391742688,3391742691,IN -3391742692,3391742719,HK -3391742720,3391742727,IN -3391742728,3391742731,HK -3391742732,3391742735,IN -3391742736,3391742847,HK -3391742848,3391742879,IN -3391742880,3391742903,HK -3391742904,3391742919,IN -3391742920,3391742927,HK -3391742928,3391742935,IN -3391742936,3391742943,HK -3391742944,3391742951,IN -3391742952,3391742983,HK -3391742984,3391742991,IN -3391742992,3391743135,HK -3391743136,3391743151,IN -3391743152,3391743175,HK -3391743176,3391743199,IN -3391743200,3391743359,HK -3391743360,3391743391,IN -3391743392,3391743743,HK -3391743744,3391743759,IN -3391743760,3391744039,HK -3391744040,3391744063,IN -3391744064,3391744111,HK -3391744112,3391744119,IN -3391744120,3391744271,HK -3391744272,3391744367,IN -3391744368,3391744383,HK -3391744384,3391744447,IN -3391744448,3391744511,HK -3391744512,3391744519,IN -3391744520,3391744535,HK -3391744536,3391744543,IN -3391744544,3391744791,HK -3391744792,3391744799,IN -3391744800,3391744823,HK -3391744824,3391744831,IN -3391744832,3391745055,HK -3391745056,3391745087,IN -3391745088,3391745279,HK -3391745280,3391745295,IN -3391745296,3391746047,HK +3391741952,3391746047,IN 3391746048,3391750143,CN -3391750144,3391815679,SG +3391750144,3391766783,SG +3391766784,3391767039,JP +3391767040,3391810047,SG +3391810048,3391810303,JP +3391810304,3391815679,SG 3391815680,3391817727,AU 3391817728,3391819775,TO 3391819776,3391823871,JP @@ -76514,7 +78840,9 @@ 3391852544,3391856639,CN 3391856640,3391864831,ID 3391864832,3391866879,TW -3391866880,3391873023,US +3391866880,3391868927,US +3391868928,3391870975,SG +3391870976,3391873023,US 3391873024,3391877119,AU 3391877120,3391879167,JP 3391879168,3391881215,ID @@ -76687,7 +79015,6 @@ 3392440832,3392441343,BD 3392441344,3392441855,ID 3392441856,3392442111,AU -3392442368,3392442623,HK 3392442624,3392442879,AU 3392442880,3392443391,ID 3392443392,3392443647,IN @@ -76711,7 +79038,8 @@ 3392479232,3392487423,AU 3392487424,3392499711,IN 3392499712,3392503807,JP -3392503808,3392511999,HK +3392503808,3392505343,HK +3392506880,3392507903,HK 3392512000,3392516095,BD 3392516096,3392520191,NZ 3392520192,3392524287,AU @@ -76926,7 +79254,9 @@ 3393314816,3393318911,ID 3393318912,3393320959,SG 3393320960,3393363967,HK -3393372160,3393388543,HK +3393372160,3393374975,HK +3393374976,3393375743,SG +3393375744,3393388543,HK 3393388544,3393389567,CN 3393389568,3393390591,IN 3393390592,3393392639,ID @@ -76977,7 +79307,7 @@ 3393620992,3393621247,JP 3393621248,3393622015,AU 3393622016,3393626111,PK -3393626112,3393630207,HK +3393626112,3393630207,CN 3393630208,3393634303,JP 3393634304,3393638399,CN 3393638400,3393650687,JP @@ -77048,9 +79378,11 @@ 3393867776,3393871871,CN 3393871872,3393874943,HK 3393874944,3393875967,AU -3393875968,3393878271,HK -3393878272,3393878527,IN -3393878528,3393880063,HK +3393875968,3393876991,HK +3393876992,3393878015,SG +3393878016,3393879039,IN +3393879040,3393879551,JP +3393879552,3393880063,HK 3393880064,3393896447,AU 3393896448,3393906687,NZ 3393908736,3393910783,BD @@ -77094,7 +79426,9 @@ 3394078720,3394078975,AU 3394078976,3394079231,IN 3394079232,3394079743,PH -3394079744,3394111487,HK +3394079744,3394087471,HK +3394087472,3394087487,CN +3394087488,3394111487,HK 3394111488,3394113535,CN 3394113536,3394117631,SG 3394117632,3394121727,AU @@ -77218,7 +79552,6 @@ 3394756608,3394760703,JP 3394760704,3394764799,ID 3394764800,3394772991,HK -3394772992,3394775039,PK 3394777088,3394781183,JP 3394781184,3394789375,MP 3394789376,3394797567,HK @@ -77229,12 +79562,24 @@ 3394830336,3394831359,HK 3394831360,3394832383,NZ 3394832384,3394834431,CN -3394834432,3394836479,HK +3394834432,3394834687,HK +3394834688,3394834943,JP +3394834944,3394834959,HK +3394834960,3394834975,JP +3394834976,3394834983,HK +3394834984,3394834991,SG +3394834992,3394835199,HK +3394835200,3394835455,JP +3394835456,3394835583,SG +3394835584,3394835711,JP +3394835712,3394835967,CN +3394835968,3394836223,HK +3394836224,3394836239,JP +3394836240,3394836247,AS +3394836248,3394836415,JP +3394836416,3394836479,HK 3394836480,3394838527,BT -3394838528,3394844671,NZ -3394844672,3394845695,AU -3394845696,3394846207,NZ -3394846208,3394846719,AU +3394838528,3394846719,NZ 3394846720,3394850815,IN 3394850816,3394854911,JP 3394855936,3394856959,AU @@ -77468,44 +79813,9 @@ 3397279744,3397283839,AU 3397283840,3397285887,KR 3397285888,3397287935,BD -3397287936,3397296175,SG -3397296176,3397296191,HK -3397296192,3397296271,SG -3397296272,3397296279,HK -3397296280,3397296479,SG -3397296480,3397297263,HK -3397297264,3397297271,SG -3397297272,3397297295,HK -3397297296,3397297311,SG -3397297312,3397297407,HK -3397297408,3397297439,SG -3397297440,3397297919,HK -3397297920,3397297983,SG -3397297984,3397298015,HK -3397298016,3397298071,SG -3397298072,3397298079,HK -3397298080,3397298431,SG -3397298432,3397298495,HK -3397298496,3397298511,SG -3397298512,3397298559,HK -3397298560,3397298943,SG -3397298944,3397298959,HK -3397298960,3397299047,SG -3397299048,3397299055,HK -3397299056,3397299183,SG -3397299184,3397299191,HK -3397299192,3397299359,SG +3397287936,3397299359,SG 3397299360,3397299375,IN -3397299376,3397299455,SG -3397299456,3397299483,HK -3397299484,3397299487,SG -3397299488,3397299599,HK -3397299600,3397299603,SG -3397299604,3397304211,HK -3397304212,3397304215,SG -3397304216,3397304299,HK -3397304300,3397304303,SG -3397304304,3397304319,HK +3397299376,3397304319,SG 3397304320,3397308415,NZ 3397308416,3397312511,HK 3397312512,3397320703,JP @@ -77637,9 +79947,19 @@ 3397812224,3397816319,CN 3397816320,3397832703,IN 3397836800,3397844991,AU -3397844992,3397869823,JP +3397844992,3397857791,JP +3397857792,3397858047,AU +3397858048,3397862143,JP +3397862144,3397862399,PH +3397862400,3397869823,JP 3397869824,3397870079,AU -3397870080,3397910527,JP +3397870080,3397878527,JP +3397878528,3397878783,TW +3397878784,3397881855,JP +3397881856,3397882111,HK +3397882112,3397887999,JP +3397888000,3397888768,IN +3397888769,3397910527,JP 3397910528,3397918719,SG 3397918720,3397922815,AU 3397922816,3397926911,CN @@ -77738,13 +80058,19 @@ 3398619136,3398621183,AU 3398621184,3398623231,HK 3398623232,3398631423,ID -3398631424,3398637119,JP +3398631424,3398634439,JP +3398634440,3398634447,KR +3398634448,3398636543,JP +3398636544,3398636575,HK +3398636576,3398637119,JP 3398637120,3398637183,HK 3398637184,3398637823,JP 3398637824,3398638079,PH 3398638080,3398639903,JP 3398639904,3398639907,MY -3398639908,3398641407,JP +3398639908,3398640671,JP +3398640672,3398640695,SG +3398640696,3398641407,JP 3398641408,3398641663,MY 3398641664,3398641919,JP 3398641920,3398642175,AU @@ -77957,14 +80283,17 @@ 3399946240,3399950335,HK 3399950336,3399954943,AU 3399954944,3399974911,US -3399974912,3399983744,MY +3399974912,3399979007,HK +3399979008,3399983744,US 3399983745,3399983745,SG -3399983746,3399991295,MY +3399983746,3399991295,US 3399991296,3399995391,IN 3399995392,3399999487,KR 3399999488,3400000255,SG 3400000256,3400000511,AU -3400000512,3400004607,SG +3400000512,3400002303,SG +3400002304,3400002367,HK +3400002368,3400004607,SG 3400004608,3400004863,AU 3400004864,3400005119,SG 3400005120,3400005375,AU @@ -78096,7 +80425,9 @@ 3400647936,3400648191,AU 3400648192,3400648815,SG 3400648816,3400648831,HK -3400648832,3400650239,SG +3400648832,3400649943,SG +3400649944,3400649951,HK +3400649952,3400650239,SG 3400650240,3400654847,AU 3400654848,3400663039,IN 3400663040,3400683519,MY @@ -78214,7 +80545,9 @@ 3405820160,3405820415,CN 3405820416,3405832191,AU 3405832192,3405832447,CN -3405832448,3405841407,AU +3405832448,3405839359,AU +3405839360,3405840383,HK +3405840384,3405841407,AU 3405841408,3405842431,CN 3405842432,3405844991,AU 3405844992,3405845247,CN @@ -78261,7 +80594,7 @@ 3405960704,3405961215,CN 3405961216,3405963775,AU 3405963776,3405964287,CN -3405964288,3405964543,AU +3405964288,3405964543,JP 3405964544,3405964799,CN 3405964800,3405966335,AU 3405966336,3405966847,CN @@ -78291,7 +80624,9 @@ 3406008064,3406008319,CN 3406008320,3406070783,AU 3406070784,3406071551,CN -3406071552,3406075647,AU +3406071552,3406071807,AU +3406071808,3406073855,US +3406073856,3406075647,AU 3406075648,3406076927,CN 3406076928,3406077951,AU 3406077952,3406078207,TH @@ -78394,7 +80729,8 @@ 3406299392,3406301183,AU 3406301184,3406301439,CN 3406301440,3406305023,AU -3406305024,3406307327,CN +3406305024,3406305279,CN +3406305280,3406307327,HK 3406307328,3406317055,AU 3406317056,3406317311,CN 3406317312,3406320127,AU @@ -78459,7 +80795,9 @@ 3406405120,3406405375,CN 3406405376,3406409727,AU 3406409728,3406411775,NZ -3406411776,3406434303,AU +3406411776,3406413311,AU +3406413312,3406413823,JP +3406413824,3406434303,AU 3406434304,3406436351,MY 3406436352,3406438911,AU 3406438912,3406439167,CN @@ -78518,15 +80856,20 @@ 3406566400,3406566911,ID 3406566912,3406567423,AU 3406567424,3406567679,CN -3406567680,3406575871,AU +3406567680,3406572031,AU +3406572032,3406572287,JP +3406572288,3406575871,AU 3406575872,3406576127,CN 3406576128,3406577919,AU 3406577920,3406578431,CN 3406578432,3406579199,AU 3406579200,3406579711,CN 3406579712,3406583295,AU -3406583552,3406585855,CN -3406585856,3406586879,AU +3406583552,3406583807,CN +3406583808,3406585855,HK +3406585856,3406586111,AU +3406586112,3406586367,JP +3406586368,3406586879,AU 3406586880,3406587391,CN 3406587392,3406587647,AU 3406587648,3406587903,CN @@ -78539,7 +80882,8 @@ 3406594816,3406596351,AU 3406596352,3406596607,CN 3406596608,3406611455,AU -3406611456,3406614527,CN +3406611456,3406612479,CN +3406612480,3406614527,HK 3406614528,3406615295,AU 3406615296,3406615551,CN 3406615552,3406617343,AU @@ -78562,7 +80906,8 @@ 3406637312,3406638079,AU 3406638080,3406638591,CN 3406638592,3406647295,AU -3406647296,3406649855,CN +3406647296,3406649343,HK +3406649344,3406649855,CN 3406649856,3406650367,AU 3406650368,3406651391,CN 3406651392,3406664191,AU @@ -78575,7 +80920,8 @@ 3406684160,3406684671,CN 3406684672,3406684927,AU 3406684928,3406685183,CN -3406685184,3406686463,AU +3406685184,3406685951,AU +3406686208,3406686463,AU 3406686464,3406686719,CN 3406686720,3406696959,AU 3406696960,3406697215,IN @@ -78601,7 +80947,8 @@ 3406737408,3406737663,ID 3406737664,3406739199,AU 3406739200,3406739455,ID -3406739456,3406741759,CN +3406739456,3406741503,HK +3406741504,3406741759,CN 3406741760,3406742015,AU 3406742016,3406742527,CN 3406742528,3406746623,AU @@ -78616,7 +80963,9 @@ 3406757888,3406761983,CN 3406761984,3406763007,AU 3406763008,3406763775,CN -3406763776,3406780159,AU +3406763776,3406775295,AU +3406775296,3406775551,JP +3406775552,3406780159,AU 3406780160,3406780927,CN 3406780928,3406784767,AU 3406784768,3406785023,CN @@ -78690,7 +81039,9 @@ 3406911488,3406911999,CN 3406912000,3406923775,AU 3406923776,3406924031,CN -3406924032,3406930943,AU +3406924032,3406930431,AU +3406930432,3406930687,JP +3406930688,3406930943,AU 3406930944,3406931199,CN 3406931200,3406936831,AU 3406936832,3406937087,CN @@ -78819,7 +81170,8 @@ 3407089920,3407090175,CN 3407090176,3407095807,AU 3407095808,3407096319,CN -3407096576,3407097855,AU +3407096576,3407096831,AU +3407097088,3407097855,AU 3407097856,3407098111,CN 3407098112,3407101183,AU 3407101184,3407101439,CN @@ -78926,7 +81278,9 @@ 3407261696,3407263743,CN 3407263744,3407266303,AU 3407266304,3407266559,CN -3407266560,3407278591,AU +3407266560,3407268863,AU +3407268864,3407269119,US +3407269120,3407278591,AU 3407278592,3407279103,CN 3407279104,3407279359,AU 3407279360,3407279871,CN @@ -79007,7 +81361,9 @@ 3407377408,3407377663,CN 3407377664,3407378943,AU 3407378944,3407379455,CN -3407379456,3407384831,AU +3407379456,3407382271,AU +3407382272,3407382527,JP +3407382528,3407384831,AU 3407384832,3407385087,CN 3407385088,3407386623,AU 3407386624,3407387135,CN @@ -79121,7 +81477,8 @@ 3407522304,3407522559,CN 3407522560,3407523071,AU 3407523072,3407523327,CN -3407523328,3407523839,AU +3407523328,3407523583,AU +3407523584,3407523839,JP 3407523840,3407524095,CN 3407524096,3407526143,AU 3407526144,3407526399,CN @@ -79161,7 +81518,9 @@ 3407558144,3407560959,AU 3407560960,3407561471,CN 3407561472,3407561727,NZ -3407561728,3407565055,AU +3407561728,3407561983,AU +3407561984,3407562239,JP +3407562240,3407565055,AU 3407565056,3407565311,CN 3407565312,3407566847,AU 3407566848,3407567103,CN @@ -79531,7 +81890,7 @@ 3408017408,3408017919,CN 3408017920,3408020223,AU 3408020224,3408020479,CN -3408020480,3408020735,AU +3408020480,3408020735,JP 3408020736,3408020991,CN 3408020992,3408022527,AU 3408022528,3408022783,CN @@ -79733,7 +82092,9 @@ 3409574144,3409574399,CN 3409574400,3409575167,AU 3409575168,3409575935,CN -3409575936,3409838335,AU +3409575936,3409707007,AU +3409707008,3409772543,HK +3409772544,3409838335,AU 3409838336,3409838591,MY 3409838592,3409838847,CN 3409838848,3409871615,AU @@ -79848,11 +82209,12 @@ 3411149824,3411150847,IN 3411150848,3411152895,HK 3411152896,3411154943,CN +3411154944,3411156991,JP 3411156992,3411161087,PH 3411161088,3411165183,PK 3411165184,3411173375,MY 3411173376,3411177471,AU -3411181568,3411189759,JP +3411177472,3411189759,JP 3411189760,3411197951,AU 3411197952,3411202047,BD 3411202048,3411204607,AU @@ -79898,7 +82260,9 @@ 3411335680,3411337215,HK 3411337216,3411341311,AU 3411341312,3411345407,KR -3411345408,3411410943,HK +3411345408,3411354879,HK +3411354880,3411355647,SG +3411355648,3411410943,HK 3411410944,3411443711,CN 3411443712,3411460095,HK 3411460096,3411464191,NZ @@ -79939,15 +82303,9 @@ 3411644928,3411645951,ID 3411645952,3411646207,SG 3411646208,3411647487,IN -3411647488,3411648767,AU -3411648768,3411649023,NZ -3411649024,3411649279,AU -3411649280,3411649535,NZ -3411649536,3411649791,AU -3411649792,3411650303,NZ -3411650304,3411654399,AU -3411654400,3411654655,NZ -3411654656,3411673087,AU +3411647488,3411648511,AU +3411648512,3411656703,NZ +3411656704,3411673087,AU 3411673088,3411674111,CN 3411674112,3411674623,IN 3411674624,3411675135,HK @@ -79988,7 +82346,12 @@ 3411859252,3411861503,JP 3411861504,3411869695,AU 3411869696,3411943423,CN -3411943424,3411951615,NZ +3411943424,3411946495,NZ +3411946496,3411947007,AU +3411947008,3411948031,NZ +3411948032,3411948543,AU +3411948544,3411950591,NZ +3411950592,3411951615,AU 3411951616,3411967999,LK 3411968000,3411984383,AU 3411984384,3412000767,IN @@ -80002,7 +82365,10 @@ 3412213760,3412221951,AU 3412221952,3412230143,IN 3412230144,3412246527,HK -3412246528,3412254719,AU +3412246528,3412249903,AU +3412249904,3412249919,SG +3412249920,3412253695,AU +3412253696,3412254719,JP 3412254720,3412262911,NR 3412262912,3412264959,JP 3412264960,3412271103,CN @@ -80047,14 +82413,11 @@ 3412381696,3412385791,NZ 3412385792,3412393983,AU 3412393984,3412426751,IN -3412426752,3412431487,HK -3412431488,3412431551,TH -3412431552,3412433679,HK -3412433680,3412433695,TH -3412433696,3412433759,HK +3412426752,3412433759,TH 3412433760,3412433775,ID -3412433776,3412433783,TH -3412433784,3412434943,HK +3412433776,3412433791,TH +3412433792,3412433823,HK +3412433824,3412434943,TH 3412434944,3412443135,PK 3412443136,3412451327,SG 3412451328,3412526079,AU @@ -80065,7 +82428,9 @@ 3412598784,3412602879,CN 3412602880,3412606975,NC 3412606976,3412615167,PH -3412615168,3412656127,JP +3412615168,3412631551,JP +3412631552,3412639743,KR +3412639744,3412656127,JP 3412656128,3412672511,HK 3412672512,3412675071,JP 3412675072,3412675327,US @@ -80113,8 +82478,8 @@ 3413180416,3413213183,TH 3413213184,3413229567,VN 3413229568,3413245951,AU -3413245952,3413262335,SG -3413262336,3413270527,PH +3413245952,3413262335,MY +3413262336,3413263359,PH 3413270528,3413278719,TH 3413278720,3413295103,NZ 3413295104,3413303295,JP @@ -80212,7 +82577,9 @@ 3413934080,3413946367,IN 3413946368,3413950463,AU 3413950464,3413966847,IN -3413966848,3414155519,SG +3413966848,3414050303,SG +3414050304,3414050559,US +3414050560,3414155519,SG 3414155520,3414155775,PH 3414155776,3414163455,SG 3414163456,3414171647,PK @@ -80278,11 +82645,13 @@ 3414670592,3414670847,SG 3414670848,3414671359,MY 3414671360,3415080959,JP -3415080960,3415083007,SG +3415080960,3415082239,MY +3415082240,3415083007,SG 3415083008,3415083519,AU 3415083520,3415084031,SG 3415084032,3415084159,CN -3415084160,3415089151,SG +3415084160,3415088127,SG +3415088128,3415089151,HK 3415089152,3415097343,MY 3415097344,3415103487,ID 3415103488,3415113727,HK @@ -80337,8 +82706,7 @@ 3415564288,3415568383,JP 3415568384,3415572479,KR 3415572480,3415605247,SG -3415605248,3415736319,TH -3415736320,3415752703,SG +3415605248,3415752703,TH 3415752704,3415760895,CN 3415760896,3415769087,NZ 3415769088,3415777279,CN @@ -80401,7 +82769,7 @@ 3416327168,3416328191,HK 3416328192,3416330239,AU 3416330240,3416334335,ID -3416334336,3416342527,MY +3416334336,3416342527,SG 3416342528,3416371199,AU 3416371200,3416371711,PH 3416371712,3416371967,VN @@ -80418,7 +82786,11 @@ 3416375296,3416383487,CN 3416383488,3416391679,HK 3416391680,3416457215,VN -3416457216,3416475391,JP +3416457216,3416474583,JP +3416474584,3416474599,AU +3416474600,3416474639,JP +3416474640,3416474647,NZ +3416474648,3416475391,JP 3416475392,3416475647,NZ 3416475648,3416482047,JP 3416482048,3416482303,SG @@ -80429,7 +82801,11 @@ 3416514560,3416522751,IN 3416522752,3416588287,AU 3416588288,3416653823,JP -3416653824,3416686591,AU +3416653824,3416667135,AU +3416667136,3416667647,US +3416667648,3416667775,AU +3416667776,3416668159,US +3416668160,3416686591,AU 3416686592,3416694783,SG 3416694784,3416702975,CN 3416702976,3416707071,ID @@ -80439,7 +82815,7 @@ 3416709632,3416710143,AU 3416710144,3416711167,HK 3416711168,3416719359,AU -3416719360,3416727551,PH +3416719360,3416723455,PH 3416727552,3416735743,JP 3416735744,3416752127,PH 3416752128,3416784895,NZ @@ -80455,8 +82831,28 @@ 3416856576,3416858623,BD 3416858624,3416862719,HK 3416862720,3416864767,MN -3416864768,3416866815,HK -3416866816,3416883199,SG +3416864768,3416864895,JP +3416864896,3416865023,SG +3416865024,3416865151,HK +3416865152,3416865279,SG +3416865280,3416865287,JP +3416865288,3416865295,HK +3416865296,3416865303,AU +3416865304,3416865311,HK +3416865312,3416865327,SG +3416865328,3416865343,HK +3416865344,3416865359,JP +3416865360,3416865407,HK +3416865408,3416865599,JP +3416865600,3416865655,HK +3416865656,3416865791,JP +3416865792,3416865919,SG +3416865920,3416866055,HK +3416866056,3416866071,SG +3416866072,3416866079,HK +3416866080,3416866095,SG +3416866096,3416866559,HK +3416866560,3416883199,SG 3416883200,3416915967,HK 3416915968,3416920063,AU 3416920064,3416921087,TH @@ -80662,7 +83058,9 @@ 3418292992,3418293503,HK 3418293760,3418294015,AU 3418294272,3418296319,VN -3418296320,3418300415,HK +3418296320,3418298367,CN +3418298368,3418299391,HK +3418299392,3418300415,CN 3418300416,3418300927,BD 3418300928,3418301439,IN 3418301440,3418302463,AU @@ -80703,7 +83101,9 @@ 3418472448,3418480639,AU 3418480640,3418488831,CN 3418488832,3418505215,AU -3418505216,3418513407,JP +3418505216,3418512319,JP +3418512320,3418512335,AU +3418512336,3418513407,JP 3418513408,3418517503,IN 3418517504,3418519551,MN 3418519552,3418521599,CN @@ -80723,7 +83123,11 @@ 3418642944,3418643199,ID 3418643200,3418643455,JP 3418643456,3418644479,AU -3418644480,3418652671,JP +3418644480,3418650807,JP +3418650808,3418650808,HK +3418650809,3418650823,JP +3418650824,3418650839,HK +3418650840,3418652671,JP 3418652672,3418750975,IN 3418750976,3418816511,HK 3418816512,3418947583,IN @@ -80870,16 +83274,20 @@ 3420061696,3420127231,HK 3420127232,3420323839,AU 3420323840,3420332031,TW -3420332032,3420369007,AU +3420332032,3420337439,AU +3420337440,3420337471,SG +3420337472,3420366959,AU +3420366960,3420366975,KR +3420366976,3420369007,AU 3420369008,3420369023,HK 3420369024,3420370559,AU 3420370560,3420370575,JP 3420370576,3420372991,AU -3420372992,3420374527,CN +3420372992,3420374527,HK 3420374528,3420374783,ID -3420374784,3420375551,CN +3420374784,3420375551,HK 3420375552,3420375807,ID -3420375808,3420377087,CN +3420375808,3420377087,HK 3420377088,3420389375,JP 3420389376,3420393471,US 3420393472,3420395519,CN @@ -80941,8 +83349,7 @@ 3423273984,3423285247,US 3423285248,3423287295,CA 3423287296,3423288319,VG -3423288320,3423301631,US -3423302656,3423303679,US +3423288320,3423303679,US 3423303680,3423304703,CA 3423304704,3423311871,US 3423311872,3423313151,VI @@ -80957,8 +83364,8 @@ 3423416320,3423417343,CA 3423417344,3423430655,US 3423430656,3423431167,TC -3423431168,3423458303,US -3423459328,3423468543,US +3423431168,3423468543,US +3423468544,3423469567,CA 3423469568,3423473663,US 3423473664,3423474687,CA 3423474688,3423480063,US @@ -80969,7 +83376,7 @@ 3423480989,3423481343,NG 3423481344,3423487999,US 3423488000,3423490047,CA -3423491072,3423493631,US +3423490048,3423493631,US 3423493632,3423493887,RU 3423493888,3423533055,US 3423533056,3423535103,AI @@ -80984,7 +83391,7 @@ 3423586304,3423588351,US 3423590400,3423602687,US 3423602688,3423603711,KN -3423603712,3423625215,US +3423603712,3423626239,US 3423626240,3423627263,CA 3423627264,3423629311,US 3423629312,3423630335,AG @@ -80992,8 +83399,7 @@ 3423637504,3423639551,CA 3423639552,3423651839,US 3423651840,3423653887,CA -3423653888,3423654911,US -3423655936,3423705599,US +3423653888,3423705599,US 3423705600,3423705855,CA 3423705856,3423725423,US 3423725424,3423725427,CA @@ -81028,8 +83434,7 @@ 3424378880,3424379135,PR 3424379136,3424412415,US 3424412416,3424412671,CA -3424412672,3424413695,US -3424415744,3424493823,US +3424412672,3424493823,US 3424493824,3424494079,CA 3424494080,3424507135,US 3424507136,3424507391,CA @@ -82253,7 +84658,8 @@ 3453180672,3453180927,US 3453180928,3453195263,CA 3453195264,3453195519,US -3453195520,3453199871,CA +3453195520,3453198335,CA +3453198592,3453199871,CA 3453200384,3453206527,CA 3453206528,3453207551,HN 3453207552,3453207807,NI @@ -82557,7 +84963,8 @@ 3458813952,3458818047,CA 3458818048,3458820095,US 3458820096,3458820351,CA -3458820352,3458820607,US +3458820352,3458820863,US +3458820864,3458821119,JM 3458822144,3459055615,US 3459055616,3459121151,CA 3459121152,3459186687,US @@ -82869,7 +85276,12 @@ 3461357568,3461408767,US 3461410816,3461414911,CA 3461414912,3461441535,US -3461447680,3461507071,US +3461443584,3461462015,US +3461462016,3461462271,AU +3461462272,3461462527,NL +3461462528,3461462783,US +3461462784,3461463039,SG +3461463040,3461507071,US 3461509120,3461513727,US 3461513728,3461513983,BF 3461513984,3461514495,US @@ -83068,7 +85480,9 @@ 3464664320,3464684543,CA 3464684800,3464688383,CA 3464688640,3464691711,CA -3464691712,3464769535,US +3464691712,3464740863,US +3464740864,3464744959,PH +3464744960,3464769535,US 3464769536,3464773631,CA 3464773632,3464774079,US 3464774080,3464774111,ES @@ -83319,8 +85733,7 @@ 3470573568,3470575615,CA 3470575616,3470610431,US 3470610432,3470614527,AR -3470614528,3470635007,US -3470639104,3470646591,US +3470614528,3470646591,US 3470646592,3470646599,CN 3470646600,3470651391,US 3470651392,3470655487,CA @@ -83460,7 +85873,9 @@ 3478192128,3478257663,CA 3478257664,3478274823,US 3478274824,3478274831,GB -3478274832,3478372351,US +3478274832,3478288671,US +3478288672,3478288703,GB +3478288704,3478372351,US 3478372352,3478380543,MX 3478380544,3478831103,US 3478847488,3479207935,US @@ -84274,15 +86689,13 @@ 3493866496,3493867519,VG 3493867520,3493881855,US 3493881856,3493882879,CA -3493882880,3493898239,US -3493900288,3493900371,US +3493882880,3493900371,US 3493900372,3493900375,GB 3493900376,3493901023,US 3493901024,3493901031,HK 3493901032,3493901311,US 3493901312,3493901567,AE -3493901568,3493928959,US -3493931008,3493936127,US +3493901568,3493936127,US 3493936128,3493937151,CA 3493937152,3493939455,US 3493939456,3493939711,CA @@ -84305,19 +86718,17 @@ 3494010880,3494014975,US 3494014976,3494017023,CA 3494017024,3494043647,US -3494044672,3494045695,CA +3494043648,3494045695,CA 3494045696,3494049791,US 3494049792,3494051839,CA -3494051840,3494059007,US -3494060032,3494075391,US +3494051840,3494075391,US 3494075392,3494076415,CA 3494076416,3494085887,US 3494086656,3494088703,US 3494088704,3494090751,CA 3494090752,3494094847,US 3494094848,3494095871,CA -3494095872,3494106111,US -3494107136,3494121471,US +3494095872,3494121471,US 3494121472,3494122495,CA 3494122496,3494125567,US 3494125568,3494126591,CA @@ -84325,20 +86736,22 @@ 3494135808,3494136831,CA 3494136832,3494139903,US 3494139904,3494141951,CA -3494141952,3494142975,US +3494141952,3494143999,US 3494144000,3494145023,CA -3494145024,3494168575,US +3494145024,3494163455,US +3494164480,3494168575,US 3494168576,3494170623,CA 3494170624,3494181631,US 3494181632,3494181887,SN -3494182912,3494184959,US +3494181888,3494184959,US 3494184960,3494187007,CA -3494187008,3494190079,US -3494191104,3494191359,US +3494187008,3494191359,US 3494191360,3494191615,DE 3494191616,3494191871,GB 3494191872,3494192127,NL -3494192128,3494197247,US +3494192128,3494192383,US +3494192384,3494192639,JP +3494192640,3494197247,US 3494197248,3494198271,CA 3494198272,3494244351,US 3494244352,3494246399,CA @@ -84351,7 +86764,7 @@ 3494273024,3494275071,US 3494275072,3494277119,CA 3494277120,3494294527,US -3494295040,3494301695,US +3494295040,3494302719,US 3494302720,3494303743,CA 3494303744,3494310911,US 3494310912,3494311935,CA @@ -84386,9 +86799,7 @@ 3494459392,3494460415,CA 3494460416,3494464511,US 3494464512,3494465535,CA -3494465536,3494470655,US -3494471680,3494486015,US -3494488064,3494496255,US +3494465536,3494496255,US 3494496512,3494496767,US 3494498304,3494510591,US 3494510592,3494512639,CA @@ -84428,6 +86839,7 @@ 3494729728,3494730751,US 3494730752,3494731775,CA 3494731776,3494743039,US +3494743040,3494744063,CA 3494744064,3494747135,US 3494747136,3494748159,CA 3494748160,3494749439,US @@ -84443,19 +86855,15 @@ 3494788096,3494788351,NG 3494788352,3494788607,LY 3494788608,3494789119,CA -3494789120,3494830079,US -3494831104,3494852607,US +3494789120,3494852607,US 3494852608,3494854655,CA 3494854656,3494862847,US 3494862848,3494863871,DM 3494863872,3494866943,US 3494866944,3494867967,CA -3494867968,3494868991,US -3494870016,3494874111,US +3494867968,3494874111,US 3494874112,3494874367,ES -3494874368,3494879231,US -3494879488,3494879743,US -3494881280,3494893567,US +3494874368,3494893567,US 3494893568,3494894591,CA 3494894592,3494917119,US 3494917120,3494917631,CA @@ -84500,16 +86908,16 @@ 3495123456,3495123967,GB 3495123968,3495153663,US 3495153664,3495155711,CA -3495155712,3495157759,US -3495159808,3495187199,US +3495155712,3495187199,US 3495187200,3495187455,IM 3495187456,3495192575,US 3495192576,3495193599,CA 3495193600,3495215103,US 3495215104,3495217151,VI 3495217152,3495219199,VC -3495219200,3495239679,US -3495241728,3495251967,US +3495219200,3495225343,US +3495226624,3495226879,US +3495227392,3495251967,US 3495251968,3495254015,CA 3495254016,3495260159,US 3495260160,3495261183,CA @@ -84549,17 +86957,14 @@ 3495479296,3495501823,US 3495502848,3495505919,US 3495505920,3495507967,CA -3495507968,3495508991,US -3495510016,3495515135,US +3495507968,3495515135,US 3495515136,3495516159,CA 3495516160,3495521279,US 3495522304,3495526399,US 3495526400,3495527423,CA 3495527424,3495544063,US 3495544064,3495544319,NG -3495544320,3495550207,US -3495550208,3495550463,CA -3495550464,3495551999,US +3495544320,3495551999,US 3495552000,3495553023,BM 3495553024,3495556095,US 3495557120,3495576575,US @@ -84569,9 +86974,7 @@ 3495583744,3495585791,CA 3495585792,3495587839,US 3495587840,3495588863,MS -3495588864,3495593983,US -3495594496,3495595519,US -3495596032,3495618559,US +3495588864,3495618559,US 3495618560,3495619583,CA 3495619584,3495620607,US 3495620608,3495622655,CA @@ -84589,8 +86992,7 @@ 3495696384,3495703551,US 3495703552,3495704063,GB 3495704064,3495704575,CA -3495704576,3495712767,US -3495714816,3495718911,US +3495704576,3495718911,US 3495718912,3495723007,CA 3495723008,3495724031,US 3495724032,3495727103,CA @@ -85501,7 +87903,9 @@ 3514592256,3514593279,SV 3514593280,3514596863,US 3514596864,3514597375,SV -3514597376,3514826751,US +3514597376,3514732071,US +3514732072,3514732075,UA +3514732076,3514826751,US 3514826752,3514843135,CA 3514843136,3514993919,US 3514993920,3514993983,GB @@ -85551,7 +87955,9 @@ 3516366848,3516370943,CA 3516370944,3516514303,US 3516514304,3516530687,CA -3516530688,3516899839,US +3516530688,3516643083,US +3516643084,3516643087,PR +3516643088,3516899839,US 3516899840,3516900095,NG 3516900096,3516900351,US 3516900352,3516900607,NG @@ -85734,7 +88140,9 @@ 3517597696,3517597951,IE 3517597952,3517598207,DE 3517598208,3517598463,IE -3517598464,3517599359,SE +3517598464,3517599263,SE +3517599264,3517599295,US +3517599296,3517599359,SE 3517599360,3517599615,US 3517599616,3517600767,SE 3517600768,3517601279,US @@ -85852,9 +88260,7 @@ 3519934464,3519938559,CA 3519938560,3520020479,US 3520020480,3520036863,CA -3520036864,3520086271,US -3520086272,3520086527,CA -3520086528,3520135167,US +3520036864,3520135167,US 3520167936,3520344063,US 3520348160,3520356351,US 3520356352,3520364543,ZA @@ -85930,7 +88336,9 @@ 3522123520,3522123775,GB 3522123776,3522125055,US 3522125056,3522125311,GB -3522125312,3522174975,US +3522125312,3522132395,US +3522132396,3522132396,BR +3522132397,3522174975,US 3522174976,3522179071,BM 3522179072,3522195455,US 3522195456,3522199551,CA @@ -85988,7 +88396,9 @@ 3523682304,3523686399,NZ 3523686400,3523690495,AU 3523690496,3523698687,IN -3523698688,3523707903,JP +3523698688,3523700735,JP +3523700736,3523701759,HK +3523701760,3523707903,JP 3523707904,3523708159,AU 3523708160,3523709183,JP 3523709184,3523709951,CN @@ -86020,22 +88430,7 @@ 3524745728,3524745983,GU 3524745984,3524747263,MP 3524747264,3524755455,PH -3524755456,3524756223,AU -3524756224,3524756479,SG -3524756480,3524756991,AU -3524756992,3524757247,SG -3524757248,3524758015,AU -3524758016,3524758271,SG -3524758272,3524758527,AU -3524758528,3524758783,SG -3524758784,3524759039,AU -3524759040,3524759295,SG -3524759296,3524759551,AU -3524759552,3524760575,SG -3524760576,3524761343,AU -3524761344,3524761599,SG -3524761600,3524761855,AU -3524761856,3524763647,SG +3524755456,3524763647,AU 3524763648,3524788223,PH 3524788224,3524853759,SG 3524853760,3526361087,CN @@ -86077,7 +88472,7 @@ 3528404992,3528407039,NZ 3528407040,3528409087,AU 3528409088,3528425471,CN -3528441856,3528445951,JP +3528425472,3528445951,JP 3528445952,3528450047,ID 3528450048,3528458239,CN 3528458240,3528474623,AF @@ -86107,20 +88502,19 @@ 3531603968,3533765631,JP 3533765632,3533765887,CN 3533765888,3534749695,JP -3534749696,3534758043,HK -3534758044,3534758143,AU +3534749696,3534757887,HK +3534757888,3534758143,AU 3534758144,3534758147,JP -3534758148,3534758911,HK +3534758148,3534758911,AU 3534758912,3534758927,PH -3534758928,3534758975,HK +3534758928,3534758975,AU 3534758976,3534759039,PH -3534759040,3534759167,HK +3534759040,3534759167,AU 3534759168,3534759183,JP -3534759184,3534760703,HK +3534759184,3534760703,AU 3534760704,3534760711,NZ -3534760712,3534761471,HK -3534761472,3534761535,AU -3534761536,3534863443,HK +3534760712,3534761983,AU +3534761984,3534863443,HK 3534863444,3534863444,CN 3534863445,3534880767,HK 3534880768,3535273983,KR @@ -86230,8 +88624,8 @@ 3557081088,3557089279,NL 3557089280,3557105663,DE 3557105664,3557111807,BG -3557111808,3557112830,GB -3557112831,3557113855,BG +3557111808,3557112831,GB +3557112832,3557113855,BG 3557113856,3557130239,RU 3557130240,3557138431,BG 3557138432,3557146623,RU @@ -86348,8 +88742,7 @@ 3557990400,3557998591,PT 3557998592,3558006783,DE 3558006784,3558010879,GB -3558010880,3558011903,FR -3558011904,3558012927,US +3558010880,3558012927,FR 3558012928,3558014975,GB 3558014976,3558023167,RU 3558023168,3558031359,DE @@ -86380,7 +88773,9 @@ 3558203392,3558211583,ES 3558211584,3558219775,GB 3558219776,3558227967,ES -3558227968,3558234623,RU +3558227968,3558232063,RU +3558232064,3558234111,ES +3558234112,3558234623,RU 3558234624,3558234879,UA 3558234880,3558235647,RU 3558235648,3558235903,LB @@ -86611,7 +89006,9 @@ 3559899648,3559899967,EE 3559899968,3559900223,UA 3559900224,3559900287,EE -3559900288,3559903231,UA +3559900288,3559902719,UA +3559902720,3559902975,EE +3559902976,3559903231,UA 3559903232,3559907327,EE 3559907328,3559915519,FR 3559915520,3559923711,IT @@ -86854,7 +89251,9 @@ 3561242624,3561259007,DE 3561259008,3561267199,IL 3561267200,3561275391,UA -3561275392,3561291775,BE +3561275392,3561288831,BE +3561288832,3561288959,FR +3561288960,3561291775,BE 3561291776,3561299967,RS 3561299968,3561308159,GB 3561308160,3561316351,PL @@ -87189,7 +89588,9 @@ 3563847680,3563848447,NL 3563848448,3563848703,ES 3563848704,3563848847,NL -3563848848,3563848855,ES +3563848848,3563848848,ES +3563848849,3563848849,NL +3563848850,3563848855,ES 3563848856,3563855871,NL 3563855872,3563864063,AT 3563864064,3563872255,GB @@ -87344,7 +89745,9 @@ 3564895744,3564896255,GB 3564896256,3564904447,RU 3564904448,3564912639,DE -3564912640,3564920831,BG +3564912640,3564915711,NL +3564915712,3564915967,BG +3564915968,3564920831,NL 3564920832,3564922111,DE 3564922112,3564929023,US 3564929024,3564937215,AT @@ -87502,8 +89905,7 @@ 3566600192,3566607359,IL 3566607360,3566607615,GN 3566607616,3566665727,IL -3566665728,3566678015,LT -3566678016,3566723071,SE +3566665728,3566723071,SE 3566723072,3566723327,RU 3566723328,3566739455,SE 3566739456,3566747647,RU @@ -87602,7 +90004,9 @@ 3567403008,3567419391,IT 3567419392,3567427583,SA 3567427584,3567435775,SE -3567435776,3567456407,GB +3567435776,3567453695,GB +3567453696,3567453951,ES +3567453952,3567456407,GB 3567456408,3567456415,ES 3567456416,3567458305,GB 3567458306,3567458306,ES @@ -87616,7 +90020,9 @@ 3567499008,3567499135,ES 3567499136,3567503359,GB 3567503360,3567504383,ES -3567504384,3567511487,GB +3567504384,3567505151,GB +3567505152,3567505407,ES +3567505408,3567511487,GB 3567511488,3567511495,ES 3567511496,3567516671,GB 3567516672,3567517695,ES @@ -87913,7 +90319,9 @@ 3575590912,3575640063,BE 3575640064,3575644159,TR 3575644160,3575709695,DK -3575709696,3575739391,AT +3575709696,3575715839,AT +3575715840,3575716351,SK +3575716352,3575739391,AT 3575739392,3575739647,FR 3575739648,3575742463,AT 3575742464,3575775231,RU @@ -88084,7 +90492,9 @@ 3576245248,3576245503,FR 3576245504,3576251135,GB 3576251136,3576251391,FR -3576251392,3576252415,GB +3576251392,3576251711,GB +3576251712,3576251775,FR +3576251776,3576252415,GB 3576252416,3576252671,FR 3576252672,3576253807,GB 3576253808,3576253815,FR @@ -88337,7 +90747,21 @@ 3581239296,3581241343,NL 3581242624,3581245439,FR 3581255680,3581258751,FR -3581280256,3581411327,BE +3581280256,3581365247,BE +3581365248,3581365503,NL +3581365504,3581365759,BE +3581365760,3581366015,NL +3581366016,3581366783,BE +3581366784,3581367039,NL +3581367040,3581367295,BE +3581367296,3581367807,NL +3581367808,3581368063,BE +3581368064,3581368319,NL +3581368320,3581371391,BE +3581371392,3581371647,NL +3581371648,3581371903,BE +3581371904,3581372159,NL +3581372160,3581411327,BE 3581411328,3581673471,GB 3581673472,3581935615,NL 3581935616,3581943807,RU @@ -89026,7 +91450,9 @@ 3587129344,3587145727,NL 3587145728,3587162111,CY 3587162112,3587178495,IR -3587178496,3587186687,AT +3587178496,3587182591,AT +3587182592,3587182847,HU +3587182848,3587186687,AT 3587186688,3587186943,DE 3587186944,3587187199,GB 3587187200,3587187455,DE @@ -89230,13 +91656,17 @@ 3589242880,3589259263,NL 3589259264,3589275647,DE 3589275648,3589292031,RS -3589292032,3589308415,AT +3589292032,3589305343,AT +3589305344,3589305599,DE +3589305600,3589308415,AT 3589308416,3589324799,DE 3589324800,3589341183,BG 3589341184,3589373951,PL 3589373952,3589390335,DE 3589390336,3589423103,RU -3589423104,3589431039,GB +3589423104,3589430591,GB +3589430592,3589430655,ES +3589430656,3589431039,GB 3589431040,3589431295,ES 3589431296,3589432831,GB 3589432832,3589433087,CH @@ -89366,39 +91796,36 @@ 3624255488,3624257535,ZA 3624257536,3624271871,US 3624271872,3624272383,SY -3624272384,3624272639,CA +3624272384,3624272639,US 3624272640,3624272895,DZ -3624272896,3624275967,CA -3624275968,3624276223,US -3624276224,3624281087,CA +3624272896,3624281087,US 3624281088,3624281343,HT -3624281344,3624283135,CA -3624283136,3624284159,US -3624284160,3624286193,CA +3624281344,3624284671,US +3624284672,3624284927,CA +3624284928,3624286193,US 3624286194,3624286194,ZA -3624286195,3624288255,CA +3624286195,3624287743,US +3624287744,3624287999,CA +3624288000,3624288255,US 3624288256,3624290303,IN 3624290304,3624292351,CA 3624292352,3624292607,MF -3624292608,3624292863,US -3624292864,3624294143,CA -3624294144,3624294399,US -3624294400,3624294911,CA -3624294912,3624295167,US -3624295168,3624295935,CA +3624292608,3624295423,US +3624295424,3624295679,CA +3624295680,3624295935,US 3624295936,3624296191,LY -3624296192,3624297215,CA -3624297216,3624297471,US +3624296192,3624296447,US +3624296448,3624296703,CA +3624296704,3624297471,US 3624297472,3624298495,CO 3624298496,3624299519,PH -3624299520,3624300031,CA +3624299520,3624300031,US 3624300032,3624300287,LY -3624300288,3624302847,CA +3624300288,3624302847,US 3624302848,3624303103,MY -3624303104,3624303359,CA -3624303360,3624303615,US -3624303616,3624304639,CA -3624304640,3624321023,US +3624303104,3624303871,US +3624303872,3624304127,CA +3624304128,3624321023,US 3624321024,3624325119,CA 3624325120,3624357887,US 3624357888,3624358143,KN @@ -89424,7 +91851,9 @@ 3624377904,3624377911,GB 3624377912,3624386559,US 3624386560,3624394751,CA -3624394752,3624435711,US +3624394752,3624402943,US +3624402944,3624407039,JP +3624407040,3624435711,US 3624435712,3624443391,CA 3624443392,3624443903,US 3624443904,3624452095,CA @@ -89521,9 +91950,7 @@ 3627679744,3627712511,CA 3627712512,3627753471,US 3627753472,3627753727,AR -3627753728,3627759359,US -3627759360,3627759615,NL -3627759616,3627802623,US +3627753728,3627802623,US 3627802624,3627810815,CA 3627810816,3627842047,US 3627842048,3627842303,IS @@ -89628,8 +92055,18 @@ 3629789952,3629790207,CA 3629790208,3629839103,US 3629839104,3629839359,CA -3629839360,3630039039,US -3630039040,3630039551,CA +3629839360,3630039047,US +3630039048,3630039055,CA +3630039056,3630039079,US +3630039080,3630039087,CA +3630039088,3630039119,US +3630039120,3630039135,CA +3630039136,3630039159,US +3630039160,3630039167,CA +3630039168,3630039183,US +3630039184,3630039199,CA +3630039200,3630039295,US +3630039296,3630039551,CA 3630039552,3630040063,US 3630040064,3630040319,CA 3630040320,3630040431,US @@ -89686,9 +92123,7 @@ 3630069504,3630069759,CA 3630069760,3630071295,US 3630071296,3630071551,CA -3630071552,3630072255,US -3630072256,3630072287,CA -3630072288,3630072575,US +3630071552,3630072575,US 3630072576,3630074111,CA 3630074112,3630074879,US 3630074880,3630075135,CA @@ -89753,9 +92188,7 @@ 3630141952,3630142207,CA 3630142208,3630143999,US 3630144000,3630144255,CA -3630144256,3630147327,US -3630147328,3630147583,CA -3630147584,3630148095,US +3630144256,3630148095,US 3630148096,3630148607,CA 3630148608,3630148863,US 3630148864,3630149119,CA @@ -89924,7 +92357,9 @@ 3633817088,3633817343,CA 3633817344,3633818623,US 3633818624,3633819135,CA -3633819136,3633844223,US +3633819136,3633823743,US +3633823744,3633827839,HK +3633827840,3633844223,US 3633848320,3633881087,US 3633881088,3633885183,CW 3633885184,3633889279,US @@ -90168,7 +92603,8 @@ 3636166400,3636166655,CA 3636166656,3636206079,US 3636206080,3636206335,AU -3636206336,3636396031,US +3636206336,3636301823,US +3636305920,3636396031,US 3636396032,3636461567,CA 3636461568,3636609023,US 3636609024,3636621311,CA @@ -90417,7 +92853,8 @@ 3641393152,3641397247,KZ 3641397248,3641401343,IT 3641401344,3641409535,ES -3641409536,3641417727,PL +3641409536,3641413631,PL +3641413632,3641417727,DE 3641417728,3641421823,GE 3641421824,3641425919,TJ 3641425920,3641430015,DE @@ -91093,7 +93530,9 @@ 3647988736,3647989759,BE 3647989760,3647995903,DE 3647995904,3648004095,RU -3648004096,3648008191,GB +3648004096,3648007167,GB +3648007168,3648007679,US +3648007680,3648008191,GB 3648008192,3648016383,FR 3648016384,3648020479,GB 3648020480,3648024575,IT @@ -91228,7 +93667,9 @@ 3648512000,3648516095,NL 3648516096,3648519167,RS 3648519168,3648520191,MK -3648520192,3648782335,NL +3648520192,3648523775,NL +3648523776,3648524031,DE +3648524032,3648782335,NL 3648782336,3649044479,ES 3649044480,3649110015,FR 3649110016,3649175551,PT @@ -91686,8 +94127,7 @@ 3653652480,3653656575,RU 3653656576,3653660671,GB 3653660672,3653664767,CZ -3653664768,3653665791,NL -3653665792,3653666815,DE +3653664768,3653666815,DE 3653666816,3653667839,NL 3653667840,3653668863,DE 3653668864,3653672959,SE @@ -91916,12 +94356,7 @@ 3707209728,3707215871,CN 3707215872,3707217919,BD 3707217920,3707219967,ID -3707219968,3707220479,NZ -3707220480,3707220735,AU -3707220736,3707220991,NZ -3707220992,3707221247,AU -3707221248,3707221759,NZ -3707221760,3707222015,AU +3707219968,3707222015,AU 3707222016,3707224063,JP 3707224064,3707240447,LK 3707240448,3707568127,CN @@ -92056,7 +94491,7 @@ 3743219712,3743252479,JP 3743252480,3743264767,NC 3743264768,3743268863,JP -3743268864,3743277055,IN +3743268864,3743272959,IN 3743277056,3743281151,PK 3743281152,3743282175,AU 3743282176,3743283199,JP @@ -92118,7 +94553,9 @@ 3757834240,3757867007,AU 3757867008,3757875519,CN 3757875520,3757875583,HK -3757875584,3757899775,CN +3757875584,3757880063,CN +3757880064,3757883391,HK +3757883392,3757899775,CN 3757899776,3757965311,KR 3757965312,3758063615,CN 3758063616,3758079999,HK diff --git a/src/config/geoip-manual b/src/config/geoip-manual deleted file mode 100644 index 99c897ff42..0000000000 --- a/src/config/geoip-manual +++ /dev/null @@ -1,116 +0,0 @@ -# This file contains manual overrides of A1 entries (and possibly others) -# in MaxMind's GeoLite Country database. Use deanonymind.py in the same -# directory to process this file when producing a new geoip file. See -# README.geoip in the same directory for details. - -# Remove MaxMind entry 0.116.0.0-0.119.255.255 which MaxMind says is AT, -# but which is part of reserved range 0.0.0.0/8. -KL 2012-06-13 -# Disabled, because MaxMind apparently removed this range from their -# database. -KL 2013-02-08 -#"0.116.0.0","0.119.255.255","7602176","7864319","","" - -# NL, because previous MaxMind entry 31.171.128.0-31.171.133.255 is NL, -# and RIR delegation files say 31.171.128.0-31.171.135.255 is NL. -# -KL 2012-11-27 -"31.171.134.0","31.171.135.255","531334656","531335167","NL","Netherlands" - -# EU, because next MaxMind entry 37.139.64.1-37.139.64.9 is EU, because -# RIR delegation files say 37.139.64.0-37.139.71.255 is EU, and because it -# just makes more sense for the next entry to start at .0 and not .1. -# -KL 2012-11-27 -"37.139.64.0","37.139.64.0","629882880","629882880","EU","Europe" - -# CH, because previous MaxMind entry 46.19.141.0-46.19.142.255 is CH, and -# RIR delegation files say 46.19.136.0-46.19.143.255 is CH. -# -KL 2012-11-27 -"46.19.143.0","46.19.143.255","773033728","773033983","CH","Switzerland" - -# GB, because next MaxMind entry 46.166.129.0-46.166.134.255 is GB, and -# RIR delegation files say 46.166.128.0-46.166.191.255 is GB. -# -KL 2012-11-27 -"46.166.128.0","46.166.128.255","782663680","782663935","GB","United Kingdom" - -# US, though could as well be CA. Previous MaxMind entry -# 64.237.32.52-64.237.34.127 is US, next MaxMind entry -# 64.237.34.144-64.237.34.151 is CA, and RIR delegation files say the -# entire block 64.237.32.0-64.237.63.255 is US. -KL 2012-11-27 -"64.237.34.128","64.237.34.143","1089282688","1089282703","US","United States" - -# US, though could as well be UY. Previous MaxMind entry -# 67.15.170.0-67.15.182.255 is US, next MaxMind entry -# 67.15.183.128-67.15.183.159 is UY, and RIR delegation files say the -# entire block 67.15.0.0-67.15.255.255 is US. -KL 2012-11-27 -"67.15.183.0","67.15.183.127","1125103360","1125103487","US","United States" - -# US, because next MaxMind entry 67.43.145.0-67.43.155.255 is US, and RIR -# delegation files say 67.43.144.0-67.43.159.255 is US. -# -KL 2012-11-27 -"67.43.144.0","67.43.144.255","1126928384","1126928639","US","United States" - -# US, because previous MaxMind entry 70.159.21.51-70.232.244.255 is US, -# because next MaxMind entry 70.232.245.58-70.232.245.59 is A2 ("Satellite -# Provider") which is a country information about as useless as A1, and -# because RIR delegation files say 70.224.0.0-70.239.255.255 is US. -# -KL 2012-11-27 -"70.232.245.0","70.232.245.57","1189672192","1189672249","US","United States" - -# US, because next MaxMind entry 70.232.246.0-70.240.141.255 is US, -# because previous MaxMind entry 70.232.245.58-70.232.245.59 is A2 -# ("Satellite Provider") which is a country information about as useless -# as A1, and because RIR delegation files say 70.224.0.0-70.239.255.255 is -# US. -KL 2012-11-27 -"70.232.245.60","70.232.245.255","1189672252","1189672447","US","United States" - -# GB, despite neither previous (GE) nor next (LV) MaxMind entry being GB, -# but because RIR delegation files agree with both previous and next -# MaxMind entry and say GB for 91.228.0.0-91.228.3.255. -KL 2012-11-27 -"91.228.0.0","91.228.3.255","1541668864","1541669887","GB","United Kingdom" - -# GB, because next MaxMind entry 91.232.125.0-91.232.125.255 is GB, and -# RIR delegation files say 91.232.124.0-91.232.125.255 is GB. -# -KL 2012-11-27 -"91.232.124.0","91.232.124.255","1541962752","1541963007","GB","United Kingdom" - -# GB, despite neither previous (RU) nor next (PL) MaxMind entry being GB, -# but because RIR delegation files agree with both previous and next -# MaxMind entry and say GB for 91.238.214.0-91.238.215.255. -# -KL 2012-11-27 -"91.238.214.0","91.238.215.255","1542379008","1542379519","GB","United Kingdom" - -# US, because next MaxMind entry 173.0.16.0-173.0.65.255 is US, and RIR -# delegation files say 173.0.0.0-173.0.15.255 is US. -KL 2012-11-27 -"173.0.0.0","173.0.15.255","2902458368","2902462463","US","United States" - -# US, because next MaxMind entry 176.67.84.0-176.67.84.79 is US, and RIR -# delegation files say 176.67.80.0-176.67.87.255 is US. -KL 2012-11-27 -"176.67.80.0","176.67.83.255","2957201408","2957202431","US","United States" - -# US, because previous MaxMind entry 176.67.84.192-176.67.85.255 is US, -# and RIR delegation files say 176.67.80.0-176.67.87.255 is US. -# -KL 2012-11-27 -"176.67.86.0","176.67.87.255","2957202944","2957203455","US","United States" - -# EU, despite neither previous (RU) nor next (UA) MaxMind entry being EU, -# but because RIR delegation files agree with both previous and next -# MaxMind entry and say EU for 193.200.150.0-193.200.150.255. -# -KL 2012-11-27 -"193.200.150.0","193.200.150.255","3251148288","3251148543","EU","Europe" - -# US, because previous MaxMind entry 199.96.68.0-199.96.87.127 is US, and -# RIR delegation files say 199.96.80.0-199.96.87.255 is US. -# -KL 2012-11-27 -"199.96.87.128","199.96.87.255","3344979840","3344979967","US","United States" - -# US, because previous MaxMind entry 209.58.176.144-209.59.31.255 is US, -# and RIR delegation files say 209.59.32.0-209.59.63.255 is US. -# -KL 2012-11-27 -"209.59.32.0","209.59.63.255","3510312960","3510321151","US","United States" - -# FR, because previous MaxMind entry 217.15.166.0-217.15.166.255 is FR, -# and RIR delegation files contain a block 217.15.160.0-217.15.175.255 -# which, however, is EU, not FR. But merging with next MaxMind entry -# 217.15.176.0-217.15.191.255 which is KZ and which fully matches what -# the RIR delegation files say seems unlikely to be correct. -# -KL 2012-11-27 -"217.15.167.0","217.15.175.255","3641681664","3641683967","FR","France" - diff --git a/src/config/geoip6 b/src/config/geoip6 index 70fa6e34bf..06bd578b85 100644 --- a/src/config/geoip6 +++ b/src/config/geoip6 @@ -1,4 +1,4 @@ -# Last updated based on July 10 2014 Maxmind GeoLite2 Country +# Last updated based on August 7 2014 Maxmind GeoLite2 Country # wget https://geolite.maxmind.com/download/geoip/database/GeoLite2-Country.mmdb.gz # gunzip GeoLite2-Country.mmdb.gz # python mmdb-convert.py GeoLite2-Country.mmdb @@ -60,165 +60,9 @@ 2001:3d8::,2001:3d8:ffff:ffff:ffff:ffff:ffff:ffff,JP 2001:3e0::,2001:3e0:ffff:ffff:ffff:ffff:ffff:ffff,JP 2001:3e8::,2001:3e8:ffff:ffff:ffff:ffff:ffff:ffff,JP -2001:400::,2001:400:ffff:ffff:ffff:ffff:ffff:ffff,US -2001:408::,2001:408:ffff:ffff:ffff:ffff:ffff:ffff,US -2001:410::,2001:410:ffff:ffff:ffff:ffff:ffff:ffff,CA -2001:418::,2001:418:ffff:ffff:ffff:ffff:ffff:ffff,US -2001:420::,2001:420:ffff:ffff:ffff:ffff:ffff:ffff,US -2001:428::,2001:428:ffff:ffff:ffff:ffff:ffff:ffff,US -2001:430::,2001:430:ffff:ffff:ffff:ffff:ffff:ffff,US -2001:438::,2001:438:ffff:ffff:ffff:ffff:ffff:ffff,US -2001:440::,2001:440:ffff:ffff:ffff:ffff:ffff:ffff,US -2001:448::,2001:448:ffff:ffff:ffff:ffff:ffff:ffff,MX -2001:450::,2001:450:ffff:ffff:ffff:ffff:ffff:ffff,US -2001:458::,2001:458:ffff:ffff:ffff:ffff:ffff:ffff,US -2001:460::,2001:460:ffff:ffff:ffff:ffff:ffff:ffff,US -2001:468::,2001:468:ffff:ffff:ffff:ffff:ffff:ffff,US -2001:470::,2001:470:ffff:ffff:ffff:ffff:ffff:ffff,US -2001:478::,2001:478:ffff:ffff:ffff:ffff:ffff:ffff,KN -2001:480::,2001:480:ffff:ffff:ffff:ffff:ffff:ffff,US -2001:490::,2001:490:ffff:ffff:ffff:ffff:ffff:ffff,US -2001:4a0::,2001:4a0:ffff:ffff:ffff:ffff:ffff:ffff,US -2001:4b0::,2001:4b0:ffff:ffff:ffff:ffff:ffff:ffff,US -2001:4b8::,2001:4b8:ffff:ffff:ffff:ffff:ffff:ffff,US -2001:4c0::,2001:4c0:ffff:ffff:ffff:ffff:ffff:ffff,CA -2001:4c8::,2001:4c8:ffff:ffff:ffff:ffff:ffff:ffff,CA -2001:4d0::,2001:4d0:ffff:ffff:ffff:ffff:ffff:ffff,US -2001:4e0::,2001:4e0:ffff:ffff:ffff:ffff:ffff:ffff,US -2001:4e8::,2001:4e8:ffff:ffff:ffff:ffff:ffff:ffff,CA -2001:4f0::,2001:4f0:ffff:ffff:ffff:ffff:ffff:ffff,US -2001:4f8::,2001:4f8:ffff:ffff:ffff:ffff:ffff:ffff,US -2001:500:1::,2001:500:4:ffff:ffff:ffff:ffff:ffff,US -2001:500:6::,2001:500:f:ffff:ffff:ffff:ffff:ffff,CA -2001:500:10::,2001:500:10:ffff:ffff:ffff:ffff:ffff,PR -2001:500:11::,2001:500:15:ffff:ffff:ffff:ffff:ffff,US -2001:500:16::,2001:500:2c:ffff:ffff:ffff:ffff:ffff,CA -2001:500:2d::,2001:500:31:ffff:ffff:ffff:ffff:ffff,US -2001:500:40::,2001:500:56:ffff:ffff:ffff:ffff:ffff,CA -2001:500:60::,2001:500:7d:ffff:ffff:ffff:ffff:ffff,US -2001:500:80::,2001:500:83:ffff:ffff:ffff:ffff:ffff,CA -2001:500:84::,2001:500:89:ffff:ffff:ffff:ffff:ffff,US -2001:500:8c::,2001:500:9a:ffff:ffff:ffff:ffff:ffff,US -2001:500:9c::,2001:500:9f:ffff:ffff:ffff:ffff:ffff,US -2001:500:a0::,2001:500:a7:ffff:ffff:ffff:ffff:ffff,CA -2001:500:a8::,2001:500:a8:ffff:ffff:ffff:ffff:ffff,US -2001:500:c0::,2001:500:ef:ffff:ffff:ffff:ffff:ffff,CA -2001:500:f0::,2001:500:f0:ffff:ffff:ffff:ffff:ffff,US -2001:500:f1::,2001:500:f1:ffff:ffff:ffff:ffff:ffff,CA -2001:500:100::,2001:500:109:ffff:ffff:ffff:ffff:ffff,CA -2001:500:3e5::,2001:500:3e5:ffff:ffff:ffff:ffff:ffff,US -2001:500:30ff::,2001:500:30ff:ffff:ffff:ffff:ffff:ffff,US -2001:500:3682::,2001:500:3682:ffff:ffff:ffff:ffff:ffff,US -2001:500:4431::,2001:500:4431:ffff:ffff:ffff:ffff:ffff,US -2001:500:7967::,2001:500:7967:ffff:ffff:ffff:ffff:ffff,US -2001:500:856e::,2001:500:856e:ffff:ffff:ffff:ffff:ffff,US -2001:500:d937::,2001:500:d937:ffff:ffff:ffff:ffff:ffff,US -2001:500:ed30::,2001:500:ed30:ffff:ffff:ffff:ffff:ffff,US -2001:501:8a29::,2001:501:8a29:ffff:ffff:ffff:ffff:ffff,US -2001:501:973c::,2001:501:973c:ffff:ffff:ffff:ffff:ffff,US -2001:501:b1f9::,2001:501:b1f9:ffff:ffff:ffff:ffff:ffff,US -2001:502:8cc::,2001:502:8cc:ffff:ffff:ffff:ffff:ffff,US -2001:502:100e::,2001:502:100e:ffff:ffff:ffff:ffff:ffff,US -2001:502:1ca1::,2001:502:1ca1:ffff:ffff:ffff:ffff:ffff,US -2001:502:2eda::,2001:502:2eda:ffff:ffff:ffff:ffff:ffff,US -2001:502:4612::,2001:502:4612:ffff:ffff:ffff:ffff:ffff,US -2001:502:63bd::,2001:502:63bd:ffff:ffff:ffff:ffff:ffff,US -2001:502:7094::,2001:502:7094:ffff:ffff:ffff:ffff:ffff,US -2001:502:7a71::,2001:502:7a71:ffff:ffff:ffff:ffff:ffff,US -2001:502:8c25::,2001:502:8c25:ffff:ffff:ffff:ffff:ffff,US -2001:502:ad09::,2001:502:ad09:ffff:ffff:ffff:ffff:ffff,US -2001:502:be98::,2001:502:be98:ffff:ffff:ffff:ffff:ffff,US -2001:502:cbe4::,2001:502:cbe4:ffff:ffff:ffff:ffff:ffff,US -2001:502:cfb5::,2001:502:cfb5:ffff:ffff:ffff:ffff:ffff,US -2001:502:d399::,2001:502:d399:ffff:ffff:ffff:ffff:ffff,US -2001:502:f3ff::,2001:502:f3ff:ffff:ffff:ffff:ffff:ffff,US -2001:503:c27::,2001:503:c27:ffff:ffff:ffff:ffff:ffff,US -2001:503:d2d::,2001:503:d2d:ffff:ffff:ffff:ffff:ffff,US -2001:503:231d::,2001:503:231d:ffff:ffff:ffff:ffff:ffff,US -2001:503:3227::,2001:503:3227:ffff:ffff:ffff:ffff:ffff,US -2001:503:39c1::,2001:503:39c1:ffff:ffff:ffff:ffff:ffff,US -2001:503:4872::,2001:503:4872:ffff:ffff:ffff:ffff:ffff,US -2001:503:5419::,2001:503:5419:ffff:ffff:ffff:ffff:ffff,US -2001:503:5ae2::,2001:503:5ae2:ffff:ffff:ffff:ffff:ffff,US -2001:503:6810::,2001:503:6810:ffff:ffff:ffff:ffff:ffff,US -2001:503:7bbb::,2001:503:7bbb:ffff:ffff:ffff:ffff:ffff,US -2001:503:7bbf::,2001:503:7bbf:ffff:ffff:ffff:ffff:ffff,US -2001:503:8028::,2001:503:8028:ffff:ffff:ffff:ffff:ffff,US -2001:503:83eb::,2001:503:83eb:ffff:ffff:ffff:ffff:ffff,US -2001:503:91ef::,2001:503:91ef:ffff:ffff:ffff:ffff:ffff,US -2001:503:a124::,2001:503:a124:ffff:ffff:ffff:ffff:ffff,US -2001:503:a83e::,2001:503:a83e:ffff:ffff:ffff:ffff:ffff,US -2001:503:ba3e::,2001:503:ba3e:ffff:ffff:ffff:ffff:ffff,US -2001:503:bfb0::,2001:503:bfb0:ffff:ffff:ffff:ffff:ffff,US -2001:503:c779::,2001:503:c779:ffff:ffff:ffff:ffff:ffff,US -2001:503:cc2c::,2001:503:cc2c:ffff:ffff:ffff:ffff:ffff,US -2001:503:d1ae::,2001:503:d1ae:ffff:ffff:ffff:ffff:ffff,US -2001:503:d414::,2001:503:d414:ffff:ffff:ffff:ffff:ffff,US -2001:503:e239::,2001:503:e239:ffff:ffff:ffff:ffff:ffff,US -2001:503:e8ef::,2001:503:e8ef:ffff:ffff:ffff:ffff:ffff,US -2001:503:eea3::,2001:503:eea3:ffff:ffff:ffff:ffff:ffff,US -2001:503:f189::,2001:503:f189:ffff:ffff:ffff:ffff:ffff,US -2001:503:f261::,2001:503:f261:ffff:ffff:ffff:ffff:ffff,US -2001:503:f3da::,2001:503:f3da:ffff:ffff:ffff:ffff:ffff,US -2001:503:ff39::,2001:503:ff39:ffff:ffff:ffff:ffff:ffff,US -2001:504::,2001:504:13:ffff:ffff:ffff:ffff:ffff,US -2001:504:15::,2001:504:15:ffff:ffff:ffff:ffff:ffff,CA -2001:504:16::,2001:504:19:ffff:ffff:ffff:ffff:ffff,US -2001:504:1a::,2001:504:1a:ffff:ffff:ffff:ffff:ffff,CA -2001:504:1b::,2001:504:1c:ffff:ffff:ffff:ffff:ffff,US -2001:504:1d::,2001:504:1d:ffff:ffff:ffff:ffff:ffff,PR -2001:504:20::,2001:504:23:ffff:ffff:ffff:ffff:ffff,CA -2001:504:24::,2001:504:24:ffff:ffff:ffff:ffff:ffff,US -2001:504:25::,2001:504:26:ffff:ffff:ffff:ffff:ffff,CA -2001:504:27::,2001:504:29:ffff:ffff:ffff:ffff:ffff,US -2001:504:2b::,2001:504:2d:ffff:ffff:ffff:ffff:ffff,CA -2001:504:2e::,2001:504:2e:ffff:ffff:ffff:ffff:ffff,US -2001:504:2f::,2001:504:2f:ffff:ffff:ffff:ffff:ffff,CA -2001:504:30::,2001:504:34:ffff:ffff:ffff:ffff:ffff,US -2001:504:35::,2001:504:35:ffff:ffff:ffff:ffff:ffff,GD -2001:504:36::,2001:504:36:ffff:ffff:ffff:ffff:ffff,US -2001:504:37::,2001:504:37:ffff:ffff:ffff:ffff:ffff,CA -2001:504:38::,2001:504:38:ffff:ffff:ffff:ffff:ffff,US -2001:504:39::,2001:504:39:ffff:ffff:ffff:ffff:ffff,CA -2001:504:3a::,2001:504:3d:ffff:ffff:ffff:ffff:ffff,US -2001:506::,2001:506:1:ffff:ffff:ffff:ffff:ffff,US -2001:506:8::,2001:506:8:ffff:ffff:ffff:ffff:ffff,US -2001:506:20::,2001:506:20:ffff:ffff:ffff:ffff:ffff,CA -2001:506:28::,2001:506:28:ffff:ffff:ffff:ffff:ffff,US -2001:506:100::,2001:506:100:ffff:ffff:ffff:ffff:ffff,US -2001:506:1000::,2001:506:2fff:ffff:ffff:ffff:ffff:ffff,US -2001:506:4000::,2001:506:7fff:ffff:ffff:ffff:ffff:ffff,US -2001:508::,2001:508:ffff:ffff:ffff:ffff:ffff:ffff,BM -2001:510::,2001:510:ffff:ffff:ffff:ffff:ffff:ffff,CA -2001:518::,2001:518:ffff:ffff:ffff:ffff:ffff:ffff,US -2001:520::,2001:520:ffff:ffff:ffff:ffff:ffff:ffff,US -2001:528::,2001:528:ffff:ffff:ffff:ffff:ffff:ffff,US -2001:530::,2001:530:ffff:ffff:ffff:ffff:ffff:ffff,US -2001:538::,2001:538:ffff:ffff:ffff:ffff:ffff:ffff,US -2001:540::,2001:540:ffff:ffff:ffff:ffff:ffff:ffff,US -2001:548::,2001:548:ffff:ffff:ffff:ffff:ffff:ffff,US -2001:550::,2001:550:ffff:ffff:ffff:ffff:ffff:ffff,US -2001:558::,2001:560:ffff:ffff:ffff:ffff:ffff:ffff,US -2001:568::,2001:56f:ffff:ffff:ffff:ffff:ffff:ffff,CA -2001:570::,2001:570:ffff:ffff:ffff:ffff:ffff:ffff,US -2001:578::,2001:57b:ffff:ffff:ffff:ffff:ffff:ffff,US -2001:580::,2001:580:ffff:ffff:ffff:ffff:ffff:ffff,US -2001:590::,2001:590:ffff:ffff:ffff:ffff:ffff:ffff,US -2001:598::,2001:598:ffff:ffff:ffff:ffff:ffff:ffff,CA -2001:5a0::,2001:5a0:ffff:ffff:ffff:ffff:ffff:ffff,US -2001:5a8::,2001:5a8:ffff:ffff:ffff:ffff:ffff:ffff,US -2001:5b0::,2001:5b0:ffff:ffff:ffff:ffff:ffff:ffff,US -2001:5b8::,2001:5b8:ffff:ffff:ffff:ffff:ffff:ffff,US -2001:5c0::,2001:5c0:ffff:ffff:ffff:ffff:ffff:ffff,CA -2001:5c8::,2001:5c8:ffff:ffff:ffff:ffff:ffff:ffff,US -2001:5d0::,2001:5d0:ffff:ffff:ffff:ffff:ffff:ffff,US -2001:5d8::,2001:5d8:ffff:ffff:ffff:ffff:ffff:ffff,US -2001:5e0::,2001:5e0:ffff:ffff:ffff:ffff:ffff:ffff,US -2001:5e8::,2001:5e8:ffff:ffff:ffff:ffff:ffff:ffff,US -2001:5f0::,2001:5f0:ffff:ffff:ffff:ffff:ffff:ffff,US -2001:5f8::,2001:5f8:ffff:ffff:ffff:ffff:ffff:ffff,US -2001:600::,2001:6ff:ffff:ffff:ffff:ffff:ffff:ffff,CH -2001:700::,2001:7ff:ffff:ffff:ffff:ffff:ffff:ffff,LU +2001:400::,2001:4ff:ffff:ffff:ffff:ffff:ffff:ffff,DE +2001:500::,2001:5ff:ffff:ffff:ffff:ffff:ffff:ffff,US +2001:600::,2001:7ff:ffff:ffff:ffff:ffff:ffff:ffff,DE 2001:808::,2001:808:ffff:ffff:ffff:ffff:ffff:ffff,PL 2001:810::,2001:810:ffff:ffff:ffff:ffff:ffff:ffff,FR 2001:818::,2001:81f:ffff:ffff:ffff:ffff:ffff:ffff,PT @@ -302,7 +146,7 @@ 2001:c98::,2001:c98:ffff:ffff:ffff:ffff:ffff:ffff,KR 2001:ca0::,2001:ca0:ffff:ffff:ffff:ffff:ffff:ffff,TW 2001:ca8::,2001:ca8:ffff:ffff:ffff:ffff:ffff:ffff,IN -2001:cb0::,2001:cb0:ffff:ffff:ffff:ffff:ffff:ffff,SG +2001:cb0::,2001:cb0:ffff:ffff:ffff:ffff:ffff:ffff,JP 2001:cb8::,2001:cb8:ffff:ffff:ffff:ffff:ffff:ffff,JP 2001:cc0::,2001:cc0:ffff:ffff:ffff:ffff:ffff:ffff,CN 2001:cc8::,2001:cc9:ffff:ffff:ffff:ffff:ffff:ffff,JP @@ -314,7 +158,37 @@ 2001:cf8::,2001:cf8:ffff:ffff:ffff:ffff:ffff:ffff,JP 2001:d00::,2001:db7:ffff:ffff:ffff:ffff:ffff:ffff,CN 2001:db9::,2001:dff:ffff:ffff:ffff:ffff:ffff:ffff,CN -2001:e00::,2001:eff:ffff:ffff:ffff:ffff:ffff:ffff,MY +2001:e00::,2001:e01:ffff:ffff:ffff:ffff:ffff:ffff,ID +2001:e08::,2001:e08:ffff:ffff:ffff:ffff:ffff:ffff,CN +2001:e10::,2001:e10:ffff:ffff:ffff:ffff:ffff:ffff,TW +2001:e18::,2001:e18:ffff:ffff:ffff:ffff:ffff:ffff,CN +2001:e20::,2001:e20:ffff:ffff:ffff:ffff:ffff:ffff,NZ +2001:e28::,2001:e28:ffff:ffff:ffff:ffff:ffff:ffff,JP +2001:e30::,2001:e30:ffff:ffff:ffff:ffff:ffff:ffff,IN +2001:e38::,2001:e38:ffff:ffff:ffff:ffff:ffff:ffff,JP +2001:e40::,2001:e47:ffff:ffff:ffff:ffff:ffff:ffff,JP +2001:e48::,2001:e48:ffff:ffff:ffff:ffff:ffff:ffff,IN +2001:e58::,2001:e58:ffff:ffff:ffff:ffff:ffff:ffff,JP +2001:e60::,2001:e60:ffff:ffff:ffff:ffff:ffff:ffff,KR +2001:e68::,2001:e68:ffff:ffff:ffff:ffff:ffff:ffff,MY +2001:e70::,2001:e70:ffff:ffff:ffff:ffff:ffff:ffff,KR +2001:e78::,2001:e78:ffff:ffff:ffff:ffff:ffff:ffff,KR +2001:e80::,2001:e80:ffff:ffff:ffff:ffff:ffff:ffff,CN +2001:e88::,2001:e88:ffff:ffff:ffff:ffff:ffff:ffff,CN +2001:e90::,2001:e90:ffff:ffff:ffff:ffff:ffff:ffff,JP +2001:e98::,2001:e98:ffff:ffff:ffff:ffff:ffff:ffff,KR +2001:ea0::,2001:ea0:ffff:ffff:ffff:ffff:ffff:ffff,KR +2001:ea8::,2001:ea8:ffff:ffff:ffff:ffff:ffff:ffff,KR +2001:eb0::,2001:eb0:ffff:ffff:ffff:ffff:ffff:ffff,HK +2001:eb8::,2001:eb8:ffff:ffff:ffff:ffff:ffff:ffff,KR +2001:ec0::,2001:ec0:ffff:ffff:ffff:ffff:ffff:ffff,TH +2001:ec8::,2001:ec8:ffff:ffff:ffff:ffff:ffff:ffff,PH +2001:ed0::,2001:ed0:ffff:ffff:ffff:ffff:ffff:ffff,KR +2001:ed8::,2001:ed8:ffff:ffff:ffff:ffff:ffff:ffff,TW +2001:ee0::,2001:ee0:ffff:ffff:ffff:ffff:ffff:ffff,VN +2001:ee8::,2001:ee8:ffff:ffff:ffff:ffff:ffff:ffff,KR +2001:ef0::,2001:ef0:ffff:ffff:ffff:ffff:ffff:ffff,KR +2001:ef8::,2001:ef8:ffff:ffff:ffff:ffff:ffff:ffff,KR 2001:f00::,2001:f00:ffff:ffff:ffff:ffff:ffff:ffff,TH 2001:f08::,2001:f08:ffff:ffff:ffff:ffff:ffff:ffff,JP 2001:f10::,2001:f10:ffff:ffff:ffff:ffff:ffff:ffff,TW @@ -385,8 +259,81 @@ 2001:12f8::,2001:12f8:1:ffff:ffff:ffff:ffff:ffff,BR 2001:12f8:4::,2001:12f8:4:ffff:ffff:ffff:ffff:ffff,BR 2001:12fe::,2001:12ff:ffff:ffff:ffff:ffff:ffff:ffff,BR -2001:1300::,2001:13ff:ffff:ffff:ffff:ffff:ffff:ffff,PE -2001:1400::,2001:14ff:ffff:ffff:ffff:ffff:ffff:ffff,SI +2001:1300::,2001:1300:ffff:ffff:ffff:ffff:ffff:ffff,PE +2001:1308::,2001:1308:ffff:ffff:ffff:ffff:ffff:ffff,DO +2001:1310::,2001:1310:ffff:ffff:ffff:ffff:ffff:ffff,CL +2001:1318::,2001:1318:ffff:ffff:ffff:ffff:ffff:ffff,AR +2001:1320::,2001:1320:ffff:ffff:ffff:ffff:ffff:ffff,PY +2001:1328::,2001:1328:ffff:ffff:ffff:ffff:ffff:ffff,UY +2001:1330::,2001:1337:ffff:ffff:ffff:ffff:ffff:ffff,CR +2001:1338::,2001:1338:ffff:ffff:ffff:ffff:ffff:ffff,VE +2001:1340::,2001:1340:ffff:ffff:ffff:ffff:ffff:ffff,CU +2001:1348::,2001:1348:ffff:ffff:ffff:ffff:ffff:ffff,UY +2001:1350::,2001:1350:ffff:ffff:ffff:ffff:ffff:ffff,VE +2001:1358::,2001:1358:ffff:ffff:ffff:ffff:ffff:ffff,CU +2001:1360::,2001:1360:ffff:ffff:ffff:ffff:ffff:ffff,GT +2001:1368::,2001:1368:ffff:ffff:ffff:ffff:ffff:ffff,PA +2001:1370::,2001:1370:ffff:ffff:ffff:ffff:ffff:ffff,HT +2001:1378::,2001:1378:ffff:ffff:ffff:ffff:ffff:ffff,BO +2001:1380::,2001:1380:ffff:ffff:ffff:ffff:ffff:ffff,PE +2001:1388::,2001:1388:ffff:ffff:ffff:ffff:ffff:ffff,PE +2001:1398::,2001:1398:ffff:ffff:ffff:ffff:ffff:ffff,CL +2001:13a0::,2001:13a0:ffff:ffff:ffff:ffff:ffff:ffff,PE +2001:13a8::,2001:13a8:ffff:ffff:ffff:ffff:ffff:ffff,MX +2001:13b0::,2001:13b7:ffff:ffff:ffff:ffff:ffff:ffff,AR +2001:13c7:6000::,2001:13c7:6000:ffff:ffff:ffff:ffff:ffff,CO +2001:13c7:6001::,2001:13c7:6001:ffff:ffff:ffff:ffff:ffff,AR +2001:13c7:6002::,2001:13c7:6002:ffff:ffff:ffff:ffff:ffff,SX +2001:13c7:6003::,2001:13c7:6003:ffff:ffff:ffff:ffff:ffff,HT +2001:13c7:6004::,2001:13c7:6005:ffff:ffff:ffff:ffff:ffff,CW +2001:13c7:6006::,2001:13c7:6006:ffff:ffff:ffff:ffff:ffff,EC +2001:13c7:6007::,2001:13c7:600e:ffff:ffff:ffff:ffff:ffff,AR +2001:13c7:6010::,2001:13c7:601f:ffff:ffff:ffff:ffff:ffff,AR +2001:13c7:6f00::,2001:13c7:6fff:ffff:ffff:ffff:ffff:ffff,EC +2001:13c7:7000::,2001:13c7:7000:ffff:ffff:ffff:ffff:ffff,MX +2001:13c7:7001::,2001:13c7:7003:ffff:ffff:ffff:ffff:ffff,UY +2001:13c7:7004::,2001:13c7:7004:ffff:ffff:ffff:ffff:ffff,CR +2001:13c7:7005::,2001:13c7:7009:ffff:ffff:ffff:ffff:ffff,UY +2001:13c7:7010::,2001:13c7:7013:ffff:ffff:ffff:ffff:ffff,UY +2001:13c7:7014::,2001:13c7:7014:ffff:ffff:ffff:ffff:ffff,MX +2001:13c8::,2001:13c8:ffff:ffff:ffff:ffff:ffff:ffff,CU +2001:13d0::,2001:13d7:ffff:ffff:ffff:ffff:ffff:ffff,AR +2001:13d8::,2001:13d8:ffff:ffff:ffff:ffff:ffff:ffff,CR +2001:13e0::,2001:13e0:ffff:ffff:ffff:ffff:ffff:ffff,DO +2001:13e8::,2001:13e8:ffff:ffff:ffff:ffff:ffff:ffff,AR +2001:13f0::,2001:13f0:ffff:ffff:ffff:ffff:ffff:ffff,DO +2001:13f8::,2001:13f8:ffff:ffff:ffff:ffff:ffff:ffff,CO +2001:1400::,2001:1400:ffff:ffff:ffff:ffff:ffff:ffff,SE +2001:1408::,2001:1408:ffff:ffff:ffff:ffff:ffff:ffff,AT +2001:1410::,2001:1410:ffff:ffff:ffff:ffff:ffff:ffff,DE +2001:1418::,2001:1418:ffff:ffff:ffff:ffff:ffff:ffff,IT +2001:1420::,2001:1420:ffff:ffff:ffff:ffff:ffff:ffff,GB +2001:1428::,2001:1428:ffff:ffff:ffff:ffff:ffff:ffff,RU +2001:1430::,2001:1430:ffff:ffff:ffff:ffff:ffff:ffff,FI +2001:1438::,2001:1438:ffff:ffff:ffff:ffff:ffff:ffff,DE +2001:1440::,2001:1440:ffff:ffff:ffff:ffff:ffff:ffff,DE +2001:1448::,2001:1448:ffff:ffff:ffff:ffff:ffff:ffff,DK +2001:1450::,2001:1450:ffff:ffff:ffff:ffff:ffff:ffff,IT +2001:1458::,2001:1459:ffff:ffff:ffff:ffff:ffff:ffff,CH +2001:1460::,2001:1460:ffff:ffff:ffff:ffff:ffff:ffff,NL +2001:1468::,2001:146f:ffff:ffff:ffff:ffff:ffff:ffff,CZ +2001:1470::,2001:1477:ffff:ffff:ffff:ffff:ffff:ffff,SI +2001:1478::,2001:1478:ffff:ffff:ffff:ffff:ffff:ffff,GB +2001:1488::,2001:1488:ffff:ffff:ffff:ffff:ffff:ffff,CZ +2001:1490::,2001:1490:ffff:ffff:ffff:ffff:ffff:ffff,SA +2001:1498::,2001:1498:ffff:ffff:ffff:ffff:ffff:ffff,ES +2001:14a0::,2001:14a0:ffff:ffff:ffff:ffff:ffff:ffff,NL +2001:14a8::,2001:14a8:ffff:ffff:ffff:ffff:ffff:ffff,CH +2001:14b0::,2001:14b0:ffff:ffff:ffff:ffff:ffff:ffff,DE +2001:14b8::,2001:14bf:ffff:ffff:ffff:ffff:ffff:ffff,FI +2001:14c0::,2001:14c0:ffff:ffff:ffff:ffff:ffff:ffff,DE +2001:14c8::,2001:14c8:ffff:ffff:ffff:ffff:ffff:ffff,FR +2001:14d0::,2001:14d0:ffff:ffff:ffff:ffff:ffff:ffff,DK +2001:14d8::,2001:14d8:ffff:ffff:ffff:ffff:ffff:ffff,DE +2001:14e0::,2001:14e0:ffff:ffff:ffff:ffff:ffff:ffff,DE +2001:14e8::,2001:14e8:ffff:ffff:ffff:ffff:ffff:ffff,IR +2001:14f0::,2001:14f0:ffff:ffff:ffff:ffff:ffff:ffff,DE +2001:14f8::,2001:14f8:ffff:ffff:ffff:ffff:ffff:ffff,DE 2001:1500::,2001:1500:ffff:ffff:ffff:ffff:ffff:ffff,IR 2001:1508::,2001:1508:ffff:ffff:ffff:ffff:ffff:ffff,CZ 2001:1510::,2001:1510:ffff:ffff:ffff:ffff:ffff:ffff,GB @@ -448,7 +395,34 @@ 2001:16f0::,2001:16f0:ffff:ffff:ffff:ffff:ffff:ffff,DE 2001:16f8::,2001:16f8:ffff:ffff:ffff:ffff:ffff:ffff,NL 2001:1700::,2001:171f:ffff:ffff:ffff:ffff:ffff:ffff,CH -2001:1800::,2001:1900:ffff:ffff:ffff:ffff:ffff:ffff,US +2001:1800::,2001:1800:ffff:ffff:ffff:ffff:ffff:ffff,US +2001:1808::,2001:1808:ffff:ffff:ffff:ffff:ffff:ffff,GD +2001:1810::,2001:1810:ffff:ffff:ffff:ffff:ffff:ffff,US +2001:1818::,2001:1818:ffff:ffff:ffff:ffff:ffff:ffff,US +2001:1820::,2001:1820:ffff:ffff:ffff:ffff:ffff:ffff,US +2001:1828::,2001:1828:ffff:ffff:ffff:ffff:ffff:ffff,US +2001:1830::,2001:1830:ffff:ffff:ffff:ffff:ffff:ffff,US +2001:1838::,2001:1838:ffff:ffff:ffff:ffff:ffff:ffff,US +2001:1840::,2001:1840:ffff:ffff:ffff:ffff:ffff:ffff,US +2001:1848::,2001:1848:ffff:ffff:ffff:ffff:ffff:ffff,US +2001:1850::,2001:1850:ffff:ffff:ffff:ffff:ffff:ffff,US +2001:1860::,2001:1860:ffff:ffff:ffff:ffff:ffff:ffff,US +2001:1868::,2001:1868:ffff:ffff:ffff:ffff:ffff:ffff,US +2001:1878::,2001:1878:ffff:ffff:ffff:ffff:ffff:ffff,US +2001:1888::,2001:1888:ffff:ffff:ffff:ffff:ffff:ffff,US +2001:1890::,2001:1898:ffff:ffff:ffff:ffff:ffff:ffff,US +2001:18a0::,2001:18a0:ffff:ffff:ffff:ffff:ffff:ffff,US +2001:18a8::,2001:18a8:ffff:ffff:ffff:ffff:ffff:ffff,US +2001:18b0::,2001:18b0:ffff:ffff:ffff:ffff:ffff:ffff,US +2001:18b8::,2001:18b8:ffff:ffff:ffff:ffff:ffff:ffff,US +2001:18c0::,2001:18c0:ffff:ffff:ffff:ffff:ffff:ffff,CA +2001:18c8::,2001:18c8:ffff:ffff:ffff:ffff:ffff:ffff,US +2001:18d8::,2001:18d8:ffff:ffff:ffff:ffff:ffff:ffff,US +2001:18e0::,2001:18e0:ffff:ffff:ffff:ffff:ffff:ffff,US +2001:18e8::,2001:18e8:ffff:ffff:ffff:ffff:ffff:ffff,US +2001:18f0::,2001:18f0:ffff:ffff:ffff:ffff:ffff:ffff,CA +2001:18f8::,2001:18f8:ffff:ffff:ffff:ffff:ffff:ffff,CA +2001:1900::,2001:1900:ffff:ffff:ffff:ffff:ffff:ffff,US 2001:1908::,2001:1908:ffff:ffff:ffff:ffff:ffff:ffff,US 2001:1910::,2001:1910:ffff:ffff:ffff:ffff:ffff:ffff,US 2001:1920::,2001:1920:ffff:ffff:ffff:ffff:ffff:ffff,CA @@ -494,7 +468,7 @@ 2001:1a68::,2001:1a68:ffff:ffff:ffff:ffff:ffff:ffff,PL 2001:1a70::,2001:1a70:ffff:ffff:ffff:ffff:ffff:ffff,MT 2001:1a78::,2001:1a78:ffff:ffff:ffff:ffff:ffff:ffff,DE -2001:1a80::,2001:1a80:ffff:ffff:ffff:ffff:ffff:ffff,DE +2001:1a80::,2001:1a87:ffff:ffff:ffff:ffff:ffff:ffff,DE 2001:1a88::,2001:1a8f:ffff:ffff:ffff:ffff:ffff:ffff,CH 2001:1a90::,2001:1a90:ffff:ffff:ffff:ffff:ffff:ffff,GB 2001:1a98::,2001:1a9f:ffff:ffff:ffff:ffff:ffff:ffff,IS @@ -628,7 +602,6 @@ 2001:42d8::,2001:42d8:ffff:ffff:ffff:ffff:ffff:ffff,CI 2001:42e0::,2001:42e0:ffff:ffff:ffff:ffff:ffff:ffff,SC 2001:42f0::,2001:42f0:ffff:ffff:ffff:ffff:ffff:ffff,ZA -2001:42f8::,2001:42f8:ffff:ffff:ffff:ffff:ffff:ffff,GH 2001:4300::,2001:4300:ffff:ffff:ffff:ffff:ffff:ffff,EG 2001:4308::,2001:4308:ffff:ffff:ffff:ffff:ffff:ffff,ZA 2001:4310::,2001:4310:ffff:ffff:ffff:ffff:ffff:ffff,MA @@ -668,7 +641,6 @@ 2001:43f8:a0::,2001:43f8:a0:ffff:ffff:ffff:ffff:ffff,ZA 2001:43f8:b0::,2001:43f8:b0:ffff:ffff:ffff:ffff:ffff,SL 2001:43f8:c0::,2001:43f8:c0:ffff:ffff:ffff:ffff:ffff,KE -2001:43f8:d0::,2001:43f8:d0:ffff:ffff:ffff:ffff:ffff,MU 2001:43f8:e0::,2001:43f8:e0:ffff:ffff:ffff:ffff:ffff,TZ 2001:43f8:100::,2001:43f8:100:ffff:ffff:ffff:ffff:ffff,ZA 2001:43f8:110::,2001:43f8:110:ffff:ffff:ffff:ffff:ffff,MU @@ -687,7 +659,6 @@ 2001:43f8:1f0::,2001:43f8:1f5:ffff:ffff:ffff:ffff:ffff,ZA 2001:43f8:200::,2001:43f8:200:ffff:ffff:ffff:ffff:ffff,KE 2001:43f8:210::,2001:43f8:210:ffff:ffff:ffff:ffff:ffff,LS -2001:43f8:220::,2001:43f8:220:ffff:ffff:ffff:ffff:ffff,MU 2001:43f8:230::,2001:43f8:230:ffff:ffff:ffff:ffff:ffff,ZA 2001:43f8:240::,2001:43f8:241:ffff:ffff:ffff:ffff:ffff,GH 2001:43f8:250::,2001:43f8:250:ffff:ffff:ffff:ffff:ffff,KE @@ -706,7 +677,7 @@ 2001:43f8:360::,2001:43f8:360:ffff:ffff:ffff:ffff:ffff,ZA 2001:43f8:370::,2001:43f8:370:ffff:ffff:ffff:ffff:ffff,NG 2001:43f8:380::,2001:43f8:380:ffff:ffff:ffff:ffff:ffff,MW -2001:43f8:390::,2001:43f8:390:ffff:ffff:ffff:ffff:ffff,AO +2001:43f8:390::,2001:43f8:391:ffff:ffff:ffff:ffff:ffff,AO 2001:43f8:3a0::,2001:43f8:3a0:ffff:ffff:ffff:ffff:ffff,ZA 2001:43f8:3b0::,2001:43f8:3b0:ffff:ffff:ffff:ffff:ffff,NA 2001:43f8:3c0::,2001:43f8:3c0:ffff:ffff:ffff:ffff:ffff,CD @@ -752,6 +723,7 @@ 2001:43f8:990::,2001:43f8:991:ffff:ffff:ffff:ffff:ffff,BI 2001:43f8:9a0::,2001:43f8:9a0:ffff:ffff:ffff:ffff:ffff,BJ 2001:43f8:9b0::,2001:43f8:9b1:ffff:ffff:ffff:ffff:ffff,SZ +2001:43f8:9c0::,2001:43f8:9c0:ffff:ffff:ffff:ffff:ffff,DJ 2001:4400::,2001:44ff:ffff:ffff:ffff:ffff:ffff:ffff,AU 2001:4500::,2001:4500:ffff:ffff:ffff:ffff:ffff:ffff,TW 2001:4508::,2001:4508:ffff:ffff:ffff:ffff:ffff:ffff,TW @@ -763,34 +735,7 @@ 2001:4540::,2001:455f:ffff:ffff:ffff:ffff:ffff:ffff,TW 2001:4580::,2001:45bf:ffff:ffff:ffff:ffff:ffff:ffff,TW 2001:4600::,2001:46ff:ffff:ffff:ffff:ffff:ffff:ffff,NO -2001:4800::,2001:4808:ffff:ffff:ffff:ffff:ffff:ffff,US -2001:4810::,2001:4810:ffff:ffff:ffff:ffff:ffff:ffff,US -2001:4818::,2001:4818:ffff:ffff:ffff:ffff:ffff:ffff,CA -2001:4828::,2001:4828:ffff:ffff:ffff:ffff:ffff:ffff,US -2001:4830::,2001:4830:ffff:ffff:ffff:ffff:ffff:ffff,US -2001:4838::,2001:4838:ffff:ffff:ffff:ffff:ffff:ffff,US -2001:4840::,2001:4840:ffff:ffff:ffff:ffff:ffff:ffff,US -2001:4848::,2001:4848:ffff:ffff:ffff:ffff:ffff:ffff,US -2001:4850::,2001:4850:ffff:ffff:ffff:ffff:ffff:ffff,US -2001:4858::,2001:4858:ffff:ffff:ffff:ffff:ffff:ffff,US -2001:4860::,2001:4860:ffff:ffff:ffff:ffff:ffff:ffff,US -2001:4868::,2001:4868:ffff:ffff:ffff:ffff:ffff:ffff,US -2001:4870::,2001:4871:ffff:ffff:ffff:ffff:ffff:ffff,US -2001:4878::,2001:4878:ffff:ffff:ffff:ffff:ffff:ffff,US -2001:4888::,2001:4888:ffff:ffff:ffff:ffff:ffff:ffff,US -2001:4890::,2001:4890:ffff:ffff:ffff:ffff:ffff:ffff,US -2001:4898::,2001:489a:ffff:ffff:ffff:ffff:ffff:ffff,US -2001:48a0::,2001:48a0:ffff:ffff:ffff:ffff:ffff:ffff,US -2001:48a8::,2001:48a8:ffff:ffff:ffff:ffff:ffff:ffff,US -2001:48b0::,2001:48b0:ffff:ffff:ffff:ffff:ffff:ffff,US -2001:48b8::,2001:48b8:ffff:ffff:ffff:ffff:ffff:ffff,US -2001:48c0::,2001:48c0:ffff:ffff:ffff:ffff:ffff:ffff,US -2001:48c8::,2001:48c8:ffff:ffff:ffff:ffff:ffff:ffff,US -2001:48d0::,2001:48d0:ffff:ffff:ffff:ffff:ffff:ffff,US -2001:48d8::,2001:48d8:ffff:ffff:ffff:ffff:ffff:ffff,US -2001:48e0::,2001:48e0:ffff:ffff:ffff:ffff:ffff:ffff,US -2001:48e8::,2001:48e8:ffff:ffff:ffff:ffff:ffff:ffff,US -2001:48f8::,2001:48f8:ffff:ffff:ffff:ffff:ffff:ffff,US +2001:4800::,2001:48ff:ffff:ffff:ffff:ffff:ffff:ffff,US 2001:4900::,2001:4900:ffff:ffff:ffff:ffff:ffff:ffff,CA 2001:4908::,2001:4908:ffff:ffff:ffff:ffff:ffff:ffff,US 2001:4910::,2001:4910:ffff:ffff:ffff:ffff:ffff:ffff,BM @@ -854,36 +799,7 @@ 2001:4be8::,2001:4be8:ffff:ffff:ffff:ffff:ffff:ffff,GB 2001:4bf0::,2001:4bf0:ffff:ffff:ffff:ffff:ffff:ffff,GB 2001:4bf8::,2001:4bf8:ffff:ffff:ffff:ffff:ffff:ffff,CH -2001:4c00::,2001:4cff:ffff:ffff:ffff:ffff:ffff:ffff,DE -2001:4d00::,2001:4d00:ffff:ffff:ffff:ffff:ffff:ffff,AM -2001:4d08::,2001:4d08:ffff:ffff:ffff:ffff:ffff:ffff,DE -2001:4d10::,2001:4d10:ffff:ffff:ffff:ffff:ffff:ffff,ES -2001:4d18::,2001:4d18:ffff:ffff:ffff:ffff:ffff:ffff,RO -2001:4d20::,2001:4d20:ffff:ffff:ffff:ffff:ffff:ffff,DE -2001:4d30::,2001:4d30:ffff:ffff:ffff:ffff:ffff:ffff,GB -2001:4d38::,2001:4d38:ffff:ffff:ffff:ffff:ffff:ffff,IT -2001:4d48::,2001:4d48:ffff:ffff:ffff:ffff:ffff:ffff,GB -2001:4d50::,2001:4d50:ffff:ffff:ffff:ffff:ffff:ffff,DE -2001:4d58::,2001:4d58:ffff:ffff:ffff:ffff:ffff:ffff,CH -2001:4d60::,2001:4d60:ffff:ffff:ffff:ffff:ffff:ffff,NL -2001:4d68::,2001:4d68:ffff:ffff:ffff:ffff:ffff:ffff,IE -2001:4d70::,2001:4d70:ffff:ffff:ffff:ffff:ffff:ffff,GR -2001:4d78::,2001:4d78:ffff:ffff:ffff:ffff:ffff:ffff,GB -2001:4d80::,2001:4d80:ffff:ffff:ffff:ffff:ffff:ffff,RO -2001:4d88::,2001:4d88:ffff:ffff:ffff:ffff:ffff:ffff,DE -2001:4d90::,2001:4d90:ffff:ffff:ffff:ffff:ffff:ffff,ES -2001:4d98::,2001:4d98:ffff:ffff:ffff:ffff:ffff:ffff,CH -2001:4da0::,2001:4da7:ffff:ffff:ffff:ffff:ffff:ffff,CH -2001:4da8::,2001:4da8:ffff:ffff:ffff:ffff:ffff:ffff,NO -2001:4db0::,2001:4db0:ffff:ffff:ffff:ffff:ffff:ffff,GB -2001:4db8::,2001:4db8:ffff:ffff:ffff:ffff:ffff:ffff,SE -2001:4dc0::,2001:4dc0:ffff:ffff:ffff:ffff:ffff:ffff,GB -2001:4dc8::,2001:4dc8:ffff:ffff:ffff:ffff:ffff:ffff,DE -2001:4dd0::,2001:4dd7:ffff:ffff:ffff:ffff:ffff:ffff,DE -2001:4dd8::,2001:4dd8:ffff:ffff:ffff:ffff:ffff:ffff,NO -2001:4de0::,2001:4de0:ffff:ffff:ffff:ffff:ffff:ffff,NL -2001:4de8::,2001:4de8:ffff:ffff:ffff:ffff:ffff:ffff,CZ -2001:4df0::,2001:4df0:ffff:ffff:ffff:ffff:ffff:ffff,IL +2001:4c00::,2001:4dff:ffff:ffff:ffff:ffff:ffff:ffff,DE 2001:8000::,2001:8fff:ffff:ffff:ffff:ffff:ffff:ffff,AU 2001:a000::,2001:a7ff:ffff:ffff:ffff:ffff:ffff:ffff,JP 2001:b000::,2001:b7ff:ffff:ffff:ffff:ffff:ffff:ffff,TW @@ -892,7 +808,6 @@ 2400:1000::,2400:1000:ffff:ffff:ffff:ffff:ffff:ffff,JP 2400:1080::,2400:1080:ffff:ffff:ffff:ffff:ffff:ffff,HK 2400:1100::,2400:1100:ffff:ffff:ffff:ffff:ffff:ffff,HK -2400:1180::,2400:1180:ffff:ffff:ffff:ffff:ffff:ffff,HK 2400:1200::,2400:1200:ffff:ffff:ffff:ffff:ffff:ffff,NZ 2400:1300::,2400:1300:ffff:ffff:ffff:ffff:ffff:ffff,TW 2400:1380::,2400:1380:ffff:ffff:ffff:ffff:ffff:ffff,CN @@ -934,7 +849,6 @@ 2400:3580::,2400:3580:ffff:ffff:ffff:ffff:ffff:ffff,ID 2400:3600::,2400:3600:ffff:ffff:ffff:ffff:ffff:ffff,CN 2400:3680::,2400:3680:ffff:ffff:ffff:ffff:ffff:ffff,IN -2400:3700::,2400:3700:ffff:ffff:ffff:ffff:ffff:ffff,MY 2400:3800::,2400:3800:ffff:ffff:ffff:ffff:ffff:ffff,JP 2400:3880::,2400:3880:ffff:ffff:ffff:ffff:ffff:ffff,AU 2400:3900::,2400:3900:ffff:ffff:ffff:ffff:ffff:ffff,AU @@ -1089,7 +1003,8 @@ 2400:8780::,2400:8780:ffff:ffff:ffff:ffff:ffff:ffff,CN 2400:8800::,2400:8800:ffff:ffff:ffff:ffff:ffff:ffff,HK 2400:8880::,2400:8880:ffff:ffff:ffff:ffff:ffff:ffff,IN -2400:8900::,2400:89ff:ffff:ffff:ffff:ffff:ffff:ffff,CN +2400:8900::,2400:8900:ffff:ffff:ffff:ffff:ffff:ffff,SG +2400:8980::,2400:8980:ffff:ffff:ffff:ffff:ffff:ffff,CN 2400:8a00::,2400:8a00:ffff:ffff:ffff:ffff:ffff:ffff,AU 2400:8a80::,2400:8a80:ffff:ffff:ffff:ffff:ffff:ffff,PH 2400:8b00::,2400:8b00:ffff:ffff:ffff:ffff:ffff:ffff,ID @@ -1326,7 +1241,7 @@ 2401:280::,2401:280:ffff:ffff:ffff:ffff:ffff:ffff,HK 2401:300::,2401:300:ffff:ffff:ffff:ffff:ffff:ffff,HK 2401:380::,2401:380:ffff:ffff:ffff:ffff:ffff:ffff,AU -2401:400::,2401:400:ffff:ffff:ffff:ffff:ffff:ffff,AU +2401:400::,2401:403:ffff:ffff:ffff:ffff:ffff:ffff,AU 2401:480::,2401:480:ffff:ffff:ffff:ffff:ffff:ffff,PH 2401:500::,2401:500:ffff:ffff:ffff:ffff:ffff:ffff,TH 2401:580::,2401:580:ffff:ffff:ffff:ffff:ffff:ffff,BD @@ -1351,7 +1266,6 @@ 2401:f00::,2401:f00:ffff:ffff:ffff:ffff:ffff:ffff,ID 2401:f80::,2401:f80:ffff:ffff:ffff:ffff:ffff:ffff,AU 2401:1000::,2401:1000:ffff:ffff:ffff:ffff:ffff:ffff,CN -2401:1080::,2401:1080:ffff:ffff:ffff:ffff:ffff:ffff,IN 2401:1100::,2401:1100:ffff:ffff:ffff:ffff:ffff:ffff,HK 2401:1180::,2401:1180:ffff:ffff:ffff:ffff:ffff:ffff,HK 2401:1200::,2401:1200:ffff:ffff:ffff:ffff:ffff:ffff,CN @@ -1408,45 +1322,86 @@ 2401:2b00::,2401:2b00:ffff:ffff:ffff:ffff:ffff:ffff,BD 2401:2b80::,2401:2b80:ffff:ffff:ffff:ffff:ffff:ffff,NP 2401:2c00::,2401:2c00:ffff:ffff:ffff:ffff:ffff:ffff,NZ +2401:2c80::,2401:2c80:ffff:ffff:ffff:ffff:ffff:ffff,HK 2401:2d00::,2401:2d00:ffff:ffff:ffff:ffff:ffff:ffff,IN +2401:2d80::,2401:2d80:ffff:ffff:ffff:ffff:ffff:ffff,SG 2401:2e00::,2401:2e00:ffff:ffff:ffff:ffff:ffff:ffff,CN +2401:2e80::,2401:2e80:ffff:ffff:ffff:ffff:ffff:ffff,NZ 2401:2f00::,2401:2f00:ffff:ffff:ffff:ffff:ffff:ffff,SG +2401:2f80::,2401:2f80:ffff:ffff:ffff:ffff:ffff:ffff,HK 2401:3000::,2401:3000:ffff:ffff:ffff:ffff:ffff:ffff,HK +2401:3080::,2401:3080:ffff:ffff:ffff:ffff:ffff:ffff,AU 2401:3100::,2401:3100:ffff:ffff:ffff:ffff:ffff:ffff,CN +2401:3180::,2401:3180:ffff:ffff:ffff:ffff:ffff:ffff,AU 2401:3200::,2401:3200:ffff:ffff:ffff:ffff:ffff:ffff,JP +2401:3280::,2401:3280:ffff:ffff:ffff:ffff:ffff:ffff,MO 2401:3300::,2401:3300:ffff:ffff:ffff:ffff:ffff:ffff,BD +2401:3380::,2401:3380:ffff:ffff:ffff:ffff:ffff:ffff,CN 2401:3400::,2401:3400:ffff:ffff:ffff:ffff:ffff:ffff,MY +2401:3480::,2401:3480:ffff:ffff:ffff:ffff:ffff:ffff,CN 2401:3500::,2401:3500:ffff:ffff:ffff:ffff:ffff:ffff,HK +2401:3580::,2401:3580:ffff:ffff:ffff:ffff:ffff:ffff,BD 2401:3600::,2401:3600:ffff:ffff:ffff:ffff:ffff:ffff,JP +2401:3680::,2401:3680:ffff:ffff:ffff:ffff:ffff:ffff,HK +2401:3780::,2401:3780:ffff:ffff:ffff:ffff:ffff:ffff,CN 2401:3800::,2401:3800:ffff:ffff:ffff:ffff:ffff:ffff,CN +2401:3880::,2401:3880:ffff:ffff:ffff:ffff:ffff:ffff,CN 2401:3900::,2401:3900:ffff:ffff:ffff:ffff:ffff:ffff,AU +2401:3980::,2401:3980:ffff:ffff:ffff:ffff:ffff:ffff,CN 2401:3a00::,2401:3a00:ffff:ffff:ffff:ffff:ffff:ffff,CN +2401:3a80::,2401:3a80:ffff:ffff:ffff:ffff:ffff:ffff,CN 2401:3b00::,2401:3b00:ffff:ffff:ffff:ffff:ffff:ffff,JP +2401:3b80::,2401:3b80:ffff:ffff:ffff:ffff:ffff:ffff,CN 2401:3c00::,2401:3c00:ffff:ffff:ffff:ffff:ffff:ffff,MY +2401:3c80::,2401:3c80:ffff:ffff:ffff:ffff:ffff:ffff,CN 2401:3d00::,2401:3d0f:ffff:ffff:ffff:ffff:ffff:ffff,SG +2401:3d80::,2401:3d80:ffff:ffff:ffff:ffff:ffff:ffff,CN 2401:3e00::,2401:3e00:ffff:ffff:ffff:ffff:ffff:ffff,KH +2401:3e80::,2401:3e80:ffff:ffff:ffff:ffff:ffff:ffff,CN 2401:3f00::,2401:3f00:ffff:ffff:ffff:ffff:ffff:ffff,SG +2401:3f80::,2401:3f80:ffff:ffff:ffff:ffff:ffff:ffff,CN 2401:4000::,2401:4000:ffff:ffff:ffff:ffff:ffff:ffff,KR +2401:4080::,2401:4080:ffff:ffff:ffff:ffff:ffff:ffff,CN 2401:4100::,2401:4100:ffff:ffff:ffff:ffff:ffff:ffff,PK +2401:4180::,2401:4180:ffff:ffff:ffff:ffff:ffff:ffff,CN 2401:4200::,2401:4200:ffff:ffff:ffff:ffff:ffff:ffff,ID +2401:4280::,2401:4280:ffff:ffff:ffff:ffff:ffff:ffff,CN 2401:4300::,2401:4300:ffff:ffff:ffff:ffff:ffff:ffff,IN +2401:4380::,2401:4380:ffff:ffff:ffff:ffff:ffff:ffff,CN 2401:4400::,2401:4400:ffff:ffff:ffff:ffff:ffff:ffff,AU +2401:4480::,2401:4480:ffff:ffff:ffff:ffff:ffff:ffff,CN 2401:4500::,2401:4500:ffff:ffff:ffff:ffff:ffff:ffff,JP +2401:4580::,2401:4580:ffff:ffff:ffff:ffff:ffff:ffff,CN +2401:4680::,2401:4680:ffff:ffff:ffff:ffff:ffff:ffff,CN 2401:4700::,2401:4700:ffff:ffff:ffff:ffff:ffff:ffff,IN +2401:4780::,2401:4780:ffff:ffff:ffff:ffff:ffff:ffff,CN 2401:4800::,2401:4800:ffff:ffff:ffff:ffff:ffff:ffff,IN +2401:4880::,2401:4880:ffff:ffff:ffff:ffff:ffff:ffff,CN 2401:4900::,2401:4900:ffff:ffff:ffff:ffff:ffff:ffff,IN 2401:4a00::,2401:4a00:ffff:ffff:ffff:ffff:ffff:ffff,HK +2401:4a80::,2401:4a80:ffff:ffff:ffff:ffff:ffff:ffff,CN 2401:4b00::,2401:4b00:ffff:ffff:ffff:ffff:ffff:ffff,CN +2401:4b80::,2401:4b80:ffff:ffff:ffff:ffff:ffff:ffff,TL 2401:4c00::,2401:4c00:ffff:ffff:ffff:ffff:ffff:ffff,HK +2401:4c80::,2401:4c80:ffff:ffff:ffff:ffff:ffff:ffff,AU 2401:4d00::,2401:4d00:ffff:ffff:ffff:ffff:ffff:ffff,AU +2401:4d80::,2401:4d80:ffff:ffff:ffff:ffff:ffff:ffff,KH 2401:4e00::,2401:4e00:ffff:ffff:ffff:ffff:ffff:ffff,ID +2401:4e80::,2401:4e80:ffff:ffff:ffff:ffff:ffff:ffff,HK 2401:4f00::,2401:4f00:ffff:ffff:ffff:ffff:ffff:ffff,TH +2401:4f80::,2401:4f80:ffff:ffff:ffff:ffff:ffff:ffff,CN 2401:5000::,2401:5000:ffff:ffff:ffff:ffff:ffff:ffff,MY +2401:5080::,2401:5080:ffff:ffff:ffff:ffff:ffff:ffff,HK 2401:5100::,2401:5100:ffff:ffff:ffff:ffff:ffff:ffff,FJ +2401:5180::,2401:5180:ffff:ffff:ffff:ffff:ffff:ffff,CN 2401:5200::,2401:5200:ffff:ffff:ffff:ffff:ffff:ffff,MY +2401:5280::,2401:5280:ffff:ffff:ffff:ffff:ffff:ffff,HK 2401:5300::,2401:5300:ffff:ffff:ffff:ffff:ffff:ffff,JP +2401:5380::,2401:5380:ffff:ffff:ffff:ffff:ffff:ffff,SG 2401:5400::,2401:5400:ffff:ffff:ffff:ffff:ffff:ffff,ID +2401:5480::,2401:5480:ffff:ffff:ffff:ffff:ffff:ffff,HK 2401:5500::,2401:5500:ffff:ffff:ffff:ffff:ffff:ffff,AU +2401:5580::,2401:5580:ffff:ffff:ffff:ffff:ffff:ffff,NZ 2401:5700::,2401:5700:ffff:ffff:ffff:ffff:ffff:ffff,TH 2401:5800::,2401:5800:ffff:ffff:ffff:ffff:ffff:ffff,BD 2401:5900::,2401:5900:ffff:ffff:ffff:ffff:ffff:ffff,HK @@ -1701,7 +1656,7 @@ 2402:6800::,2402:6800:ffff:ffff:ffff:ffff:ffff:ffff,JP 2402:6900::,2402:6900:ffff:ffff:ffff:ffff:ffff:ffff,AU 2402:6a00::,2402:6a00:ffff:ffff:ffff:ffff:ffff:ffff,CN -2402:6b00::,2402:6bff:ffff:ffff:ffff:ffff:ffff:ffff,JP +2402:6b00::,2402:6b00:ffff:ffff:ffff:ffff:ffff:ffff,JP 2402:6c00::,2402:6c00:ffff:ffff:ffff:ffff:ffff:ffff,MY 2402:6d00::,2402:6d00:ffff:ffff:ffff:ffff:ffff:ffff,PF 2402:6e00::,2402:6e00:ffff:ffff:ffff:ffff:ffff:ffff,CN @@ -2050,7 +2005,6 @@ 2403:d400::,2403:d400:ffff:ffff:ffff:ffff:ffff:ffff,CN 2403:d500::,2403:d500:ffff:ffff:ffff:ffff:ffff:ffff,AU 2403:d600::,2403:d600:ffff:ffff:ffff:ffff:ffff:ffff,AU -2403:d700::,2403:d700:ffff:ffff:ffff:ffff:ffff:ffff,MN 2403:d800::,2403:d800:ffff:ffff:ffff:ffff:ffff:ffff,NZ 2403:d900::,2403:d900:ffff:ffff:ffff:ffff:ffff:ffff,AU 2403:da00::,2403:da00:ffff:ffff:ffff:ffff:ffff:ffff,ID @@ -2187,7 +2141,7 @@ 2404:4c00::,2404:4c00:ffff:ffff:ffff:ffff:ffff:ffff,AU 2404:4d00::,2404:4d00:ffff:ffff:ffff:ffff:ffff:ffff,CN 2404:4e00::,2404:4e00:ffff:ffff:ffff:ffff:ffff:ffff,SG -2404:4f00::,2404:4f00:ffff:ffff:ffff:ffff:ffff:ffff,AU +2404:4f00::,2404:4f01:ffff:ffff:ffff:ffff:ffff:ffff,AU 2404:5000::,2404:5000:ffff:ffff:ffff:ffff:ffff:ffff,AU 2404:5100::,2404:5100:ffff:ffff:ffff:ffff:ffff:ffff,JP 2404:5200::,2404:5200:ffff:ffff:ffff:ffff:ffff:ffff,JP @@ -2887,7 +2841,6 @@ 2407:2000::,2407:2000:ffff:ffff:ffff:ffff:ffff:ffff,KR 2407:2100::,2407:2100:ffff:ffff:ffff:ffff:ffff:ffff,HK 2407:2200::,2407:2200:ffff:ffff:ffff:ffff:ffff:ffff,AU -2407:2300::,2407:2300:ffff:ffff:ffff:ffff:ffff:ffff,PG 2407:2400::,2407:2400:ffff:ffff:ffff:ffff:ffff:ffff,NZ 2407:2500::,2407:2500:ffff:ffff:ffff:ffff:ffff:ffff,AU 2407:2600::,2407:2600:ffff:ffff:ffff:ffff:ffff:ffff,AU @@ -3137,6 +3090,7 @@ 2600:1b00::,2600:1bff:ffff:ffff:ffff:ffff:ffff:ffff,JM 2600:1c00::,2600:1c0f:ffff:ffff:ffff:ffff:ffff:ffff,US 2600:1d00::,2600:1d0f:ffff:ffff:ffff:ffff:ffff:ffff,US +2600:1e00::,2600:1e0f:ffff:ffff:ffff:ffff:ffff:ffff,VC 2600:2000::,2600:200f:ffff:ffff:ffff:ffff:ffff:ffff,US 2600:2400::,2600:2407:ffff:ffff:ffff:ffff:ffff:ffff,US 2600:2800::,2600:2803:ffff:ffff:ffff:ffff:ffff:ffff,US @@ -3163,6 +3117,7 @@ 2600:7c00::,2600:7c0f:ffff:ffff:ffff:ffff:ffff:ffff,VC 2600:8000::,2600:80ff:ffff:ffff:ffff:ffff:ffff:ffff,US 2600:8400::,2600:840f:ffff:ffff:ffff:ffff:ffff:ffff,BB +2600:8800::,2600:880f:ffff:ffff:ffff:ffff:ffff:ffff,US 2600:e000::,2600:e00f:ffff:ffff:ffff:ffff:ffff:ffff,CA 2601::,2601:ff:ffff:ffff:ffff:ffff:ffff:ffff,US 2602::,2602:10f:ffff:ffff:ffff:ffff:ffff:ffff,US @@ -3173,6 +3128,10 @@ 2602:232::,2602:232:ffff:ffff:ffff:ffff:ffff:ffff,CA 2602:240::,2602:25f:ffff:ffff:ffff:ffff:ffff:ffff,US 2602:300::,2602:3ff:ffff:ffff:ffff:ffff:ffff:ffff,US +2602:ffb0::,2602:ffb0:fff:ffff:ffff:ffff:ffff:ffff,US +2602:ffb1::,2602:ffb1:fff:ffff:ffff:ffff:ffff:ffff,CA +2602:ffb2::,2602:ffb2:fff:ffff:ffff:ffff:ffff:ffff,US +2602:ffb3::,2602:ffb3:fff:ffff:ffff:ffff:ffff:ffff,US 2602:ffb4::,2602:ffb4:fff:ffff:ffff:ffff:ffff:ffff,US 2602:ffb5::,2602:ffb5:fff:ffff:ffff:ffff:ffff:ffff,US 2602:ffb6::,2602:ffb6:fff:ffff:ffff:ffff:ffff:ffff,CA @@ -3225,8 +3184,7 @@ 2602:ffe6::,2602:ffe6:fff:ffff:ffff:ffff:ffff:ffff,US 2602:ffe7::,2602:ffe7:fff:ffff:ffff:ffff:ffff:ffff,PR 2602:ffe8::,2602:ffe9:fff:ffff:ffff:ffff:ffff:ffff,US -2602:ffea::,2602:ffea:fff:ffff:ffff:ffff:ffff:ffff,US -2602:ffeb::,2602:ffeb:fff:ffff:ffff:ffff:ffff:ffff,US +2602:ffea::,2602:ffeb:fff:ffff:ffff:ffff:ffff:ffff,US 2602:ffec::,2602:ffec:fff:ffff:ffff:ffff:ffff:ffff,CA 2602:ffed::,2602:ffed:fff:ffff:ffff:ffff:ffff:ffff,US 2602:ffee::,2602:ffee:fff:ffff:ffff:ffff:ffff:ffff,US @@ -3362,6 +3320,7 @@ 2604:3900::,2604:3900:ffff:ffff:ffff:ffff:ffff:ffff,US 2604:3980::,2604:3980:ffff:ffff:ffff:ffff:ffff:ffff,US 2604:3a00::,2604:3a00:ffff:ffff:ffff:ffff:ffff:ffff,US +2604:3a80::,2604:3a80:ffff:ffff:ffff:ffff:ffff:ffff,CA 2604:3b00::,2604:3b00:ffff:ffff:ffff:ffff:ffff:ffff,CA 2604:3b80::,2604:3b80:ffff:ffff:ffff:ffff:ffff:ffff,US 2604:3c00::,2604:3c00:ffff:ffff:ffff:ffff:ffff:ffff,US @@ -3374,7 +3333,7 @@ 2604:3f80::,2604:3f80:ffff:ffff:ffff:ffff:ffff:ffff,US 2604:4000::,2604:4000:ffff:ffff:ffff:ffff:ffff:ffff,CA 2604:4080::,2604:4080:ffff:ffff:ffff:ffff:ffff:ffff,US -2604:4100::,2604:4100:ffff:ffff:ffff:ffff:ffff:ffff,US +2604:4100::,2604:4100:fff:ffff:ffff:ffff:ffff:ffff,US 2604:4180::,2604:4180:ffff:ffff:ffff:ffff:ffff:ffff,US 2604:4200::,2604:4200:ffff:ffff:ffff:ffff:ffff:ffff,US 2604:4280::,2604:4280:ffff:ffff:ffff:ffff:ffff:ffff,CA @@ -3473,7 +3432,6 @@ 2604:7200::,2604:7200:ffff:ffff:ffff:ffff:ffff:ffff,US 2604:7280::,2604:7280:ffff:ffff:ffff:ffff:ffff:ffff,US 2604:7300::,2604:7300:ffff:ffff:ffff:ffff:ffff:ffff,US -2604:7380::,2604:7380:ffff:ffff:ffff:ffff:ffff:ffff,US 2604:7400::,2604:7400:ffff:ffff:ffff:ffff:ffff:ffff,US 2604:7480::,2604:7480:ffff:ffff:ffff:ffff:ffff:ffff,CA 2604:7500::,2604:7500:ffff:ffff:ffff:ffff:ffff:ffff,US @@ -3540,6 +3498,7 @@ 2604:9400::,2604:9400:ffff:ffff:ffff:ffff:ffff:ffff,US 2604:9480::,2604:9480:ffff:ffff:ffff:ffff:ffff:ffff,US 2604:9500::,2604:9500:ffff:ffff:ffff:ffff:ffff:ffff,US +2604:9580::,2604:9580:ffff:ffff:ffff:ffff:ffff:ffff,US 2604:9600::,2604:9600:ffff:ffff:ffff:ffff:ffff:ffff,US 2604:9680::,2604:9680:ffff:ffff:ffff:ffff:ffff:ffff,US 2604:9700::,2604:9700:ffff:ffff:ffff:ffff:ffff:ffff,US @@ -3677,6 +3636,7 @@ 2604:da00::,2604:da00:fff:ffff:ffff:ffff:ffff:ffff,US 2604:da80::,2604:da80:ffff:ffff:ffff:ffff:ffff:ffff,US 2604:db00::,2604:db00:ffff:ffff:ffff:ffff:ffff:ffff,CA +2604:db80::,2604:db80:ffff:ffff:ffff:ffff:ffff:ffff,US 2604:dc00::,2604:dc00:ffff:ffff:ffff:ffff:ffff:ffff,CA 2604:dc80::,2604:dc80:ffff:ffff:ffff:ffff:ffff:ffff,US 2604:dd00::,2604:dd00:ffff:ffff:ffff:ffff:ffff:ffff,US @@ -3828,7 +3788,6 @@ 2605:2600::,2605:2600:ffff:ffff:ffff:ffff:ffff:ffff,CA 2605:2680::,2605:2680:ffff:ffff:ffff:ffff:ffff:ffff,US 2605:2700::,2605:2700:ffff:ffff:ffff:ffff:ffff:ffff,US -2605:2780::,2605:2780:ffff:ffff:ffff:ffff:ffff:ffff,US 2605:2800::,2605:2800:ffff:ffff:ffff:ffff:ffff:ffff,US 2605:2880::,2605:2880:ffff:ffff:ffff:ffff:ffff:ffff,US 2605:2900::,2605:2900:ffff:ffff:ffff:ffff:ffff:ffff,CA @@ -3941,6 +3900,7 @@ 2605:5f00::,2605:5f00:ffff:ffff:ffff:ffff:ffff:ffff,US 2605:5f80::,2605:5f80:ffff:ffff:ffff:ffff:ffff:ffff,US 2605:6000::,2605:6100:ffff:ffff:ffff:ffff:ffff:ffff,US +2605:6180::,2605:6180:ffff:ffff:ffff:ffff:ffff:ffff,US 2605:6200::,2605:6200:ffff:ffff:ffff:ffff:ffff:ffff,US 2605:6280::,2605:6280:ffff:ffff:ffff:ffff:ffff:ffff,AI 2605:6300::,2605:6300:ffff:ffff:ffff:ffff:ffff:ffff,US @@ -4032,16 +3992,27 @@ 2605:8e00::,2605:8e00:ffff:ffff:ffff:ffff:ffff:ffff,US 2605:8e80::,2605:8e80:ffff:ffff:ffff:ffff:ffff:ffff,US 2605:8f00::,2605:8f00:ffff:ffff:ffff:ffff:ffff:ffff,US +2605:8f80::,2605:8f80:ffff:ffff:ffff:ffff:ffff:ffff,US 2605:9000::,2605:9000:ffff:ffff:ffff:ffff:ffff:ffff,CA +2605:9080::,2605:9080:ffff:ffff:ffff:ffff:ffff:ffff,US 2605:9100::,2605:9100:ffff:ffff:ffff:ffff:ffff:ffff,US +2605:9180::,2605:9180:ffff:ffff:ffff:ffff:ffff:ffff,US 2605:9200::,2605:9200:ffff:ffff:ffff:ffff:ffff:ffff,US +2605:9280::,2605:9280:ffff:ffff:ffff:ffff:ffff:ffff,US 2605:9300::,2605:9300:ffff:ffff:ffff:ffff:ffff:ffff,CA +2605:9380::,2605:9380:ffff:ffff:ffff:ffff:ffff:ffff,US 2605:9400::,2605:9400:ffff:ffff:ffff:ffff:ffff:ffff,US +2605:9480::,2605:9480:ffff:ffff:ffff:ffff:ffff:ffff,US 2605:9500::,2605:9500:ffff:ffff:ffff:ffff:ffff:ffff,US +2605:9580::,2605:9580:ffff:ffff:ffff:ffff:ffff:ffff,US 2605:9600::,2605:9600:ffff:ffff:ffff:ffff:ffff:ffff,US +2605:9680::,2605:9680:ffff:ffff:ffff:ffff:ffff:ffff,CA 2605:9700::,2605:9700:ffff:ffff:ffff:ffff:ffff:ffff,US +2605:9780::,2605:9780:ffff:ffff:ffff:ffff:ffff:ffff,US 2605:9800::,2605:9800:ffff:ffff:ffff:ffff:ffff:ffff,US +2605:9880::,2605:9880:ffff:ffff:ffff:ffff:ffff:ffff,US 2605:9900::,2605:9900:ffff:ffff:ffff:ffff:ffff:ffff,US +2605:9980::,2605:9980:ffff:ffff:ffff:ffff:ffff:ffff,US 2605:9a00::,2605:9a00:ffff:ffff:ffff:ffff:ffff:ffff,US 2605:9b00::,2605:9b00:ffff:ffff:ffff:ffff:ffff:ffff,US 2605:9c00::,2605:9c00:ffff:ffff:ffff:ffff:ffff:ffff,US @@ -4322,6 +4293,7 @@ 2606:b400::,2606:b400:ffff:ffff:ffff:ffff:ffff:ffff,US 2606:b500::,2606:b500:ffff:ffff:ffff:ffff:ffff:ffff,US 2606:b600::,2606:b600:fff:ffff:ffff:ffff:ffff:ffff,US +2606:b700::,2606:b700:ffff:ffff:ffff:ffff:ffff:ffff,US 2606:b800::,2606:b800:ffff:ffff:ffff:ffff:ffff:ffff,CA 2606:b900::,2606:b900:ffff:ffff:ffff:ffff:ffff:ffff,US 2606:ba00::,2606:ba00:ffff:ffff:ffff:ffff:ffff:ffff,US @@ -4332,7 +4304,6 @@ 2606:bf00::,2606:bf00:ffff:ffff:ffff:ffff:ffff:ffff,US 2606:c000::,2606:c000:ffff:ffff:ffff:ffff:ffff:ffff,US 2606:c100::,2606:c100:ffff:ffff:ffff:ffff:ffff:ffff,US -2606:c200::,2606:c200:ffff:ffff:ffff:ffff:ffff:ffff,US 2606:c300::,2606:c300:ffff:ffff:ffff:ffff:ffff:ffff,US 2606:c400::,2606:c400:ffff:ffff:ffff:ffff:ffff:ffff,US 2606:c500::,2606:c500:ffff:ffff:ffff:ffff:ffff:ffff,JM @@ -4487,6 +4458,7 @@ 2607:5a00::,2607:5a00:ffff:ffff:ffff:ffff:ffff:ffff,US 2607:5b00::,2607:5b00:ffff:ffff:ffff:ffff:ffff:ffff,US 2607:5c00::,2607:5c00:ffff:ffff:ffff:ffff:ffff:ffff,US +2607:5d00::,2607:5d00:ffff:ffff:ffff:ffff:ffff:ffff,US 2607:5e00::,2607:5e00:ffff:ffff:ffff:ffff:ffff:ffff,CA 2607:5f00::,2607:5f00:ffff:ffff:ffff:ffff:ffff:ffff,US 2607:6000::,2607:6000:ffff:ffff:ffff:ffff:ffff:ffff,US @@ -4722,36 +4694,7 @@ 2607:f2e8::,2607:f2e8:ffff:ffff:ffff:ffff:ffff:ffff,US 2607:f2f0::,2607:f2f0:ffff:ffff:ffff:ffff:ffff:ffff,US 2607:f2f8::,2607:f2f8:ffff:ffff:ffff:ffff:ffff:ffff,US -2607:f300::,2607:f300:ffff:ffff:ffff:ffff:ffff:ffff,US -2607:f308::,2607:f308:ffff:ffff:ffff:ffff:ffff:ffff,US -2607:f310::,2607:f310:ffff:ffff:ffff:ffff:ffff:ffff,US -2607:f318::,2607:f318:ffff:ffff:ffff:ffff:ffff:ffff,US -2607:f330::,2607:f330:ffff:ffff:ffff:ffff:ffff:ffff,US -2607:f338::,2607:f338:ffff:ffff:ffff:ffff:ffff:ffff,CA -2607:f340::,2607:f340:ffff:ffff:ffff:ffff:ffff:ffff,US -2607:f348::,2607:f348:ffff:ffff:ffff:ffff:ffff:ffff,US -2607:f350::,2607:f350:ffff:ffff:ffff:ffff:ffff:ffff,US -2607:f358::,2607:f358:ffff:ffff:ffff:ffff:ffff:ffff,US -2607:f360::,2607:f360:ffff:ffff:ffff:ffff:ffff:ffff,US -2607:f368::,2607:f368:ffff:ffff:ffff:ffff:ffff:ffff,US -2607:f370::,2607:f370:ffff:ffff:ffff:ffff:ffff:ffff,US -2607:f378::,2607:f378:ffff:ffff:ffff:ffff:ffff:ffff,US -2607:f380::,2607:f380:ffff:ffff:ffff:ffff:ffff:ffff,US -2607:f388::,2607:f388:ffff:ffff:ffff:ffff:ffff:ffff,US -2607:f390::,2607:f390:ffff:ffff:ffff:ffff:ffff:ffff,US -2607:f398::,2607:f398:ffff:ffff:ffff:ffff:ffff:ffff,US -2607:f3a0::,2607:f3a0:ffff:ffff:ffff:ffff:ffff:ffff,US -2607:f3b0::,2607:f3b0:ffff:ffff:ffff:ffff:ffff:ffff,US -2607:f3b8::,2607:f3b8:ffff:ffff:ffff:ffff:ffff:ffff,US -2607:f3c0::,2607:f3c0:ffff:ffff:ffff:ffff:ffff:ffff,US -2607:f3c8::,2607:f3c8:ffff:ffff:ffff:ffff:ffff:ffff,US -2607:f3d0::,2607:f3d0:ffff:ffff:ffff:ffff:ffff:ffff,US -2607:f3d8::,2607:f3d8:ffff:ffff:ffff:ffff:ffff:ffff,CA -2607:f3e0::,2607:f3e0:ffff:ffff:ffff:ffff:ffff:ffff,CA -2607:f3e8::,2607:f3e8:ffff:ffff:ffff:ffff:ffff:ffff,US -2607:f3f0::,2607:f3f0:ffff:ffff:ffff:ffff:ffff:ffff,US -2607:f3f8::,2607:f3f8:ffff:ffff:ffff:ffff:ffff:ffff,US -2607:f400::,2607:f400:ffff:ffff:ffff:ffff:ffff:ffff,US +2607:f300::,2607:f400:ffff:ffff:ffff:ffff:ffff:ffff,US 2607:f408::,2607:f408:ffff:ffff:ffff:ffff:ffff:ffff,US 2607:f418::,2607:f418:ffff:ffff:ffff:ffff:ffff:ffff,US 2607:f420::,2607:f420:ffff:ffff:ffff:ffff:ffff:ffff,US @@ -4930,8 +4873,67 @@ 2607:f9e0::,2607:f9e0:fff:ffff:ffff:ffff:ffff:ffff,US 2607:f9f0::,2607:f9f1:ffff:ffff:ffff:ffff:ffff:ffff,US 2607:f9f8::,2607:f9f8:ffff:ffff:ffff:ffff:ffff:ffff,US -2607:fa00::,2607:faff:ffff:ffff:ffff:ffff:ffff:ffff,CA -2607:fb00::,2607:fd00:ffff:ffff:ffff:ffff:ffff:ffff,US +2607:fa00::,2607:fa00:ffff:ffff:ffff:ffff:ffff:ffff,US +2607:fa08::,2607:fa08:ffff:ffff:ffff:ffff:ffff:ffff,US +2607:fa10::,2607:fa10:ffff:ffff:ffff:ffff:ffff:ffff,US +2607:fa18::,2607:fa18:ffff:ffff:ffff:ffff:ffff:ffff,US +2607:fa20::,2607:fa20:ffff:ffff:ffff:ffff:ffff:ffff,US +2607:fa28::,2607:fa28:ffff:ffff:ffff:ffff:ffff:ffff,US +2607:fa38::,2607:fa38:ffff:ffff:ffff:ffff:ffff:ffff,US +2607:fa40::,2607:fa40:ffff:ffff:ffff:ffff:ffff:ffff,US +2607:fa48::,2607:fa48:ffff:ffff:ffff:ffff:ffff:ffff,CA +2607:fa58::,2607:fa58:ffff:ffff:ffff:ffff:ffff:ffff,US +2607:fa60::,2607:fa60:ffff:ffff:ffff:ffff:ffff:ffff,US +2607:fa68::,2607:fa68:ffff:ffff:ffff:ffff:ffff:ffff,US +2607:fa70::,2607:fa70:ffff:ffff:ffff:ffff:ffff:ffff,US +2607:fa78::,2607:fa78:ffff:ffff:ffff:ffff:ffff:ffff,US +2607:fa88::,2607:fa88:ffff:ffff:ffff:ffff:ffff:ffff,US +2607:fa90::,2607:fa90:ffff:ffff:ffff:ffff:ffff:ffff,US +2607:fa98::,2607:fa98:ffff:ffff:ffff:ffff:ffff:ffff,US +2607:faa0::,2607:faa0:ffff:ffff:ffff:ffff:ffff:ffff,US +2607:faa8::,2607:faa8:ffff:ffff:ffff:ffff:ffff:ffff,US +2607:fab0::,2607:fab0:ffff:ffff:ffff:ffff:ffff:ffff,US +2607:fab8::,2607:fab8:ffff:ffff:ffff:ffff:ffff:ffff,US +2607:fac0::,2607:fac0:ffff:ffff:ffff:ffff:ffff:ffff,US +2607:fac8::,2607:fac8:fff:ffff:ffff:ffff:ffff:ffff,US +2607:fad0::,2607:fad0:ffff:ffff:ffff:ffff:ffff:ffff,US +2607:fad8::,2607:fad8:ffff:ffff:ffff:ffff:ffff:ffff,CA +2607:fae0::,2607:fae0:ffff:ffff:ffff:ffff:ffff:ffff,US +2607:fae8::,2607:fae8:ffff:ffff:ffff:ffff:ffff:ffff,US +2607:faf0::,2607:faf0:ffff:ffff:ffff:ffff:ffff:ffff,US +2607:faf8::,2607:faf8:ffff:ffff:ffff:ffff:ffff:ffff,US +2607:fb00::,2607:fb00:ffff:ffff:ffff:ffff:ffff:ffff,US +2607:fb08::,2607:fb08:ffff:ffff:ffff:ffff:ffff:ffff,US +2607:fb10::,2607:fb10:ffff:ffff:ffff:ffff:ffff:ffff,US +2607:fb18::,2607:fb18:ffff:ffff:ffff:ffff:ffff:ffff,US +2607:fb20::,2607:fb20:ffff:ffff:ffff:ffff:ffff:ffff,US +2607:fb28::,2607:fb28:ffff:ffff:ffff:ffff:ffff:ffff,US +2607:fb30::,2607:fb30:ffff:ffff:ffff:ffff:ffff:ffff,US +2607:fb38::,2607:fb38:ffff:ffff:ffff:ffff:ffff:ffff,US +2607:fb40::,2607:fb40:ffff:ffff:ffff:ffff:ffff:ffff,US +2607:fb48::,2607:fb48:ffff:ffff:ffff:ffff:ffff:ffff,US +2607:fb50::,2607:fb50:ffff:ffff:ffff:ffff:ffff:ffff,US +2607:fb58::,2607:fb58:ffff:ffff:ffff:ffff:ffff:ffff,US +2607:fb60::,2607:fb60:ffff:ffff:ffff:ffff:ffff:ffff,US +2607:fb68::,2607:fb68:ffff:ffff:ffff:ffff:ffff:ffff,US +2607:fb70::,2607:fb70:ffff:ffff:ffff:ffff:ffff:ffff,US +2607:fb78::,2607:fb78:ffff:ffff:ffff:ffff:ffff:ffff,US +2607:fb80::,2607:fb80:ffff:ffff:ffff:ffff:ffff:ffff,US +2607:fb88::,2607:fb88:ffff:ffff:ffff:ffff:ffff:ffff,US +2607:fb90::,2607:fb90:ffff:ffff:ffff:ffff:ffff:ffff,US +2607:fb98::,2607:fb98:ffff:ffff:ffff:ffff:ffff:ffff,US +2607:fba0::,2607:fba0:ffff:ffff:ffff:ffff:ffff:ffff,US +2607:fba8::,2607:fba8:ffff:ffff:ffff:ffff:ffff:ffff,US +2607:fbb0::,2607:fbb0:ffff:ffff:ffff:ffff:ffff:ffff,US +2607:fbb8::,2607:fbb8:ffff:ffff:ffff:ffff:ffff:ffff,US +2607:fbc0::,2607:fbc0:ffff:ffff:ffff:ffff:ffff:ffff,US +2607:fbd0::,2607:fbd0:ffff:ffff:ffff:ffff:ffff:ffff,US +2607:fbd8::,2607:fbd8:ffff:ffff:ffff:ffff:ffff:ffff,CA +2607:fbe0::,2607:fbe0:ffff:ffff:ffff:ffff:ffff:ffff,US +2607:fbe8::,2607:fbe8:ffff:ffff:ffff:ffff:ffff:ffff,US +2607:fbf0::,2607:fbf0:ffff:ffff:ffff:ffff:ffff:ffff,US +2607:fbf8::,2607:fbf8:ffff:ffff:ffff:ffff:ffff:ffff,US +2607:fc00::,2607:fd00:ffff:ffff:ffff:ffff:ffff:ffff,US 2607:fd08::,2607:fd08:ffff:ffff:ffff:ffff:ffff:ffff,US 2607:fd10::,2607:fd10:ffff:ffff:ffff:ffff:ffff:ffff,US 2607:fd28::,2607:fd28:ffff:ffff:ffff:ffff:ffff:ffff,US @@ -5096,991 +5098,7 @@ 2610:1e8::,2610:1e8:ffff:ffff:ffff:ffff:ffff:ffff,CA 2610:1f0::,2610:1f0:ffff:ffff:ffff:ffff:ffff:ffff,US 2610:1f8::,2610:1f8:ffff:ffff:ffff:ffff:ffff:ffff,US -2620::,2620::ffff:ffff:ffff:ffff:ffff,US -2620:0:10::,2620::10:ffff:ffff:ffff:ffff:ffff,US -2620:0:20::,2620::20:ffff:ffff:ffff:ffff:ffff,US -2620:0:30::,2620::37:ffff:ffff:ffff:ffff:ffff,US -2620:0:40::,2620::40:ffff:ffff:ffff:ffff:ffff,US -2620:0:60::,2620::60:ffff:ffff:ffff:ffff:ffff,US -2620:0:70::,2620::70:ffff:ffff:ffff:ffff:ffff,US -2620:0:80::,2620::80:ffff:ffff:ffff:ffff:ffff,US -2620:0:90::,2620::90:ffff:ffff:ffff:ffff:ffff,US -2620:0:a0::,2620::a0:ffff:ffff:ffff:ffff:ffff,US -2620:0:b0::,2620::b0:ffff:ffff:ffff:ffff:ffff,US -2620:0:c0::,2620::c0:ffff:ffff:ffff:ffff:ffff,US -2620:0:f0::,2620::f0:ffff:ffff:ffff:ffff:ffff,CA -2620:0:100::,2620::100:ffff:ffff:ffff:ffff:ffff,US -2620:0:110::,2620::110:ffff:ffff:ffff:ffff:ffff,US -2620:0:120::,2620::120:ffff:ffff:ffff:ffff:ffff,US -2620:0:140::,2620::140:ffff:ffff:ffff:ffff:ffff,US -2620:0:150::,2620::150:ffff:ffff:ffff:ffff:ffff,US -2620:0:160::,2620::160:ffff:ffff:ffff:ffff:ffff,CA -2620:0:170::,2620::170:ffff:ffff:ffff:ffff:ffff,US -2620:0:180::,2620::180:ffff:ffff:ffff:ffff:ffff,US -2620:0:190::,2620::190:ffff:ffff:ffff:ffff:ffff,US -2620:0:1a0::,2620::1a0:ffff:ffff:ffff:ffff:ffff,US -2620:0:1b0::,2620::1b0:ffff:ffff:ffff:ffff:ffff,US -2620:0:1c0::,2620::1c0:ffff:ffff:ffff:ffff:ffff,US -2620:0:1d0::,2620::1d0:ffff:ffff:ffff:ffff:ffff,US -2620:0:1f0::,2620::1f0:ffff:ffff:ffff:ffff:ffff,US -2620:0:200::,2620::200:ffff:ffff:ffff:ffff:ffff,US -2620:0:210::,2620::210:ffff:ffff:ffff:ffff:ffff,US -2620:0:220::,2620::220:ffff:ffff:ffff:ffff:ffff,US -2620:0:230::,2620::230:ffff:ffff:ffff:ffff:ffff,CA -2620:0:240::,2620::240:ffff:ffff:ffff:ffff:ffff,US -2620:0:250::,2620::250:ffff:ffff:ffff:ffff:ffff,US -2620:0:260::,2620::260:ffff:ffff:ffff:ffff:ffff,US -2620:0:270::,2620::270:ffff:ffff:ffff:ffff:ffff,US -2620:0:280::,2620::280:ffff:ffff:ffff:ffff:ffff,US -2620:0:290::,2620::290:ffff:ffff:ffff:ffff:ffff,US -2620:0:2b0::,2620::2b0:ffff:ffff:ffff:ffff:ffff,US -2620:0:2c0::,2620::2c0:ffff:ffff:ffff:ffff:ffff,US -2620:0:2d0::,2620::2d0:ffff:ffff:ffff:ffff:ffff,US -2620:0:2f0::,2620::2f0:ffff:ffff:ffff:ffff:ffff,US -2620:0:300::,2620::300:ffff:ffff:ffff:ffff:ffff,US -2620:0:320::,2620::320:ffff:ffff:ffff:ffff:ffff,US -2620:0:350::,2620::353:ffff:ffff:ffff:ffff:ffff,US -2620:0:360::,2620::361:ffff:ffff:ffff:ffff:ffff,US -2620:0:380::,2620::380:ffff:ffff:ffff:ffff:ffff,US -2620:0:390::,2620::390:ffff:ffff:ffff:ffff:ffff,US -2620:0:3b0::,2620::3b0:ffff:ffff:ffff:ffff:ffff,US -2620:0:3c0::,2620::3c0:ffff:ffff:ffff:ffff:ffff,US -2620:0:3e0::,2620::3e0:ffff:ffff:ffff:ffff:ffff,US -2620:0:3f0::,2620::3f0:ffff:ffff:ffff:ffff:ffff,US -2620:0:400::,2620::57f:ffff:ffff:ffff:ffff:ffff,US -2620:0:600::,2620::600:ffff:ffff:ffff:ffff:ffff,US -2620:0:610::,2620::610:ffff:ffff:ffff:ffff:ffff,US -2620:0:630::,2620::630:ffff:ffff:ffff:ffff:ffff,US -2620:0:640::,2620::640:ffff:ffff:ffff:ffff:ffff,US -2620:0:650::,2620::650:ffff:ffff:ffff:ffff:ffff,US -2620:0:660::,2620::660:ffff:ffff:ffff:ffff:ffff,US -2620:0:670::,2620::671:ffff:ffff:ffff:ffff:ffff,US -2620:0:680::,2620::680:ffff:ffff:ffff:ffff:ffff,US -2620:0:690::,2620::691:ffff:ffff:ffff:ffff:ffff,US -2620:0:6a0::,2620::6a0:ffff:ffff:ffff:ffff:ffff,US -2620:0:6b0::,2620::6b0:ffff:ffff:ffff:ffff:ffff,US -2620:0:6c0::,2620::6c7:ffff:ffff:ffff:ffff:ffff,US -2620:0:6d0::,2620::6d0:ffff:ffff:ffff:ffff:ffff,US -2620:0:6e0::,2620::6e0:ffff:ffff:ffff:ffff:ffff,US -2620:0:6f0::,2620::6f0:ffff:ffff:ffff:ffff:ffff,US -2620:0:700::,2620::77f:ffff:ffff:ffff:ffff:ffff,US -2620:0:800::,2620::802:ffff:ffff:ffff:ffff:ffff,US -2620:0:810::,2620::810:ffff:ffff:ffff:ffff:ffff,CA -2620:0:820::,2620::820:ffff:ffff:ffff:ffff:ffff,US -2620:0:840::,2620::840:ffff:ffff:ffff:ffff:ffff,US -2620:0:850::,2620::850:ffff:ffff:ffff:ffff:ffff,US -2620:0:860::,2620::863:ffff:ffff:ffff:ffff:ffff,US -2620:0:870::,2620::877:ffff:ffff:ffff:ffff:ffff,US -2620:0:880::,2620::880:ffff:ffff:ffff:ffff:ffff,US -2620:0:890::,2620::890:ffff:ffff:ffff:ffff:ffff,US -2620:0:8a0::,2620::8a0:ffff:ffff:ffff:ffff:ffff,US -2620:0:8d0::,2620::8d0:ffff:ffff:ffff:ffff:ffff,US -2620:0:8e0::,2620::8e0:ffff:ffff:ffff:ffff:ffff,US -2620:0:8f0::,2620::8f0:ffff:ffff:ffff:ffff:ffff,US -2620:0:900::,2620::900:ffff:ffff:ffff:ffff:ffff,US -2620:0:910::,2620::910:ffff:ffff:ffff:ffff:ffff,US -2620:0:920::,2620::920:ffff:ffff:ffff:ffff:ffff,US -2620:0:930::,2620::930:ffff:ffff:ffff:ffff:ffff,US -2620:0:940::,2620::940:ffff:ffff:ffff:ffff:ffff,US -2620:0:950::,2620::950:ffff:ffff:ffff:ffff:ffff,US -2620:0:960::,2620::960:ffff:ffff:ffff:ffff:ffff,US -2620:0:970::,2620::970:ffff:ffff:ffff:ffff:ffff,US -2620:0:980::,2620::980:ffff:ffff:ffff:ffff:ffff,US -2620:0:990::,2620::990:ffff:ffff:ffff:ffff:ffff,US -2620:0:9a0::,2620::9a0:ffff:ffff:ffff:ffff:ffff,US -2620:0:9b0::,2620::9b0:ffff:ffff:ffff:ffff:ffff,US -2620:0:9c0::,2620::9c0:ffff:ffff:ffff:ffff:ffff,US -2620:0:9e0::,2620::9e0:ffff:ffff:ffff:ffff:ffff,US -2620:0:9f0::,2620::9f0:ffff:ffff:ffff:ffff:ffff,US -2620:0:a00::,2620::a1f:ffff:ffff:ffff:ffff:ffff,US -2620:0:b00::,2620::b00:ffff:ffff:ffff:ffff:ffff,US -2620:0:b10::,2620::b13:ffff:ffff:ffff:ffff:ffff,US -2620:0:b20::,2620::b20:ffff:ffff:ffff:ffff:ffff,US -2620:0:b30::,2620::b30:ffff:ffff:ffff:ffff:ffff,US -2620:0:b40::,2620::b40:ffff:ffff:ffff:ffff:ffff,US -2620:0:b50::,2620::b50:ffff:ffff:ffff:ffff:ffff,US -2620:0:b60::,2620::b61:ffff:ffff:ffff:ffff:ffff,US -2620:0:b80::,2620::b80:ffff:ffff:ffff:ffff:ffff,US -2620:0:b90::,2620::b90:ffff:ffff:ffff:ffff:ffff,US -2620:0:ba0::,2620::ba0:ffff:ffff:ffff:ffff:ffff,US -2620:0:bb0::,2620::bb0:ffff:ffff:ffff:ffff:ffff,US -2620:0:bd0::,2620::bd0:ffff:ffff:ffff:ffff:ffff,CA -2620:0:be0::,2620::be0:ffff:ffff:ffff:ffff:ffff,US -2620:0:bf0::,2620::bf0:ffff:ffff:ffff:ffff:ffff,US -2620:0:c10::,2620::c20:ffff:ffff:ffff:ffff:ffff,US -2620:0:c30::,2620::c30:ffff:ffff:ffff:ffff:ffff,US -2620:0:c40::,2620::c40:ffff:ffff:ffff:ffff:ffff,US -2620:0:c60::,2620::c60:ffff:ffff:ffff:ffff:ffff,US -2620:0:c70::,2620::c70:ffff:ffff:ffff:ffff:ffff,US -2620:0:c80::,2620::c80:ffff:ffff:ffff:ffff:ffff,US -2620:0:c90::,2620::ca0:ffff:ffff:ffff:ffff:ffff,US -2620:0:cb0::,2620::cb0:ffff:ffff:ffff:ffff:ffff,US -2620:0:cc0::,2620::ccf:ffff:ffff:ffff:ffff:ffff,US -2620:0:ce0::,2620::ce0:ffff:ffff:ffff:ffff:ffff,US -2620:0:cf0::,2620::cf0:ffff:ffff:ffff:ffff:ffff,US -2620:0:d00::,2620::d00:ffff:ffff:ffff:ffff:ffff,US -2620:0:d20::,2620::d20:ffff:ffff:ffff:ffff:ffff,US -2620:0:d30::,2620::d30:ffff:ffff:ffff:ffff:ffff,US -2620:0:d50::,2620::d50:ffff:ffff:ffff:ffff:ffff,US -2620:0:d60::,2620::d63:ffff:ffff:ffff:ffff:ffff,US -2620:0:d70::,2620::d77:ffff:ffff:ffff:ffff:ffff,US -2620:0:d80::,2620::d80:ffff:ffff:ffff:ffff:ffff,US -2620:0:d90::,2620::d90:ffff:ffff:ffff:ffff:ffff,US -2620:0:dc0::,2620::dc0:ffff:ffff:ffff:ffff:ffff,US -2620:0:dd0::,2620::dd0:ffff:ffff:ffff:ffff:ffff,US -2620:0:de0::,2620::de0:ffff:ffff:ffff:ffff:ffff,US -2620:0:df0::,2620::df0:ffff:ffff:ffff:ffff:ffff,US -2620:0:e00::,2620::e00:ffff:ffff:ffff:ffff:ffff,US -2620:0:e10::,2620::e10:ffff:ffff:ffff:ffff:ffff,US -2620:0:e20::,2620::e23:ffff:ffff:ffff:ffff:ffff,US -2620:0:e30::,2620::e30:ffff:ffff:ffff:ffff:ffff,US -2620:0:e50::,2620::e50:ffff:ffff:ffff:ffff:ffff,US -2620:0:e60::,2620::e60:ffff:ffff:ffff:ffff:ffff,US -2620:0:e80::,2620::e80:ffff:ffff:ffff:ffff:ffff,US -2620:0:e90::,2620::e90:ffff:ffff:ffff:ffff:ffff,US -2620:0:ea0::,2620::eb0:ffff:ffff:ffff:ffff:ffff,US -2620:0:ed0::,2620::ed0:ffff:ffff:ffff:ffff:ffff,US -2620:0:ee0::,2620::ee0:ffff:ffff:ffff:ffff:ffff,US -2620:0:ef0::,2620::ef0:ffff:ffff:ffff:ffff:ffff,US -2620:0:f00::,2620::f7f:ffff:ffff:ffff:ffff:ffff,US -2620:0:1000::,2620::10ff:ffff:ffff:ffff:ffff:ffff,US -2620:0:1400::,2620::143f:ffff:ffff:ffff:ffff:ffff,US -2620:0:1500::,2620::157f:ffff:ffff:ffff:ffff:ffff,US -2620:0:1600::,2620::167f:ffff:ffff:ffff:ffff:ffff,US -2620:0:1700::,2620::170f:ffff:ffff:ffff:ffff:ffff,US -2620:0:1800::,2620::181f:ffff:ffff:ffff:ffff:ffff,US -2620:0:1a00::,2620::1a00:ffff:ffff:ffff:ffff:ffff,US -2620:0:1a10::,2620::1a10:ffff:ffff:ffff:ffff:ffff,US -2620:0:1a20::,2620::1a20:ffff:ffff:ffff:ffff:ffff,US -2620:0:1a30::,2620::1a30:ffff:ffff:ffff:ffff:ffff,US -2620:0:1a40::,2620::1a40:ffff:ffff:ffff:ffff:ffff,US -2620:0:1a50::,2620::1a50:ffff:ffff:ffff:ffff:ffff,US -2620:0:1a60::,2620::1a60:ffff:ffff:ffff:ffff:ffff,US -2620:0:1a70::,2620::1a70:ffff:ffff:ffff:ffff:ffff,US -2620:0:1a80::,2620::1a80:ffff:ffff:ffff:ffff:ffff,US -2620:0:1aa0::,2620::1aa0:ffff:ffff:ffff:ffff:ffff,US -2620:0:1ab0::,2620::1ab0:ffff:ffff:ffff:ffff:ffff,US -2620:0:1ac0::,2620::1ac0:ffff:ffff:ffff:ffff:ffff,US -2620:0:1ad0::,2620::1ad7:ffff:ffff:ffff:ffff:ffff,US -2620:0:1ae0::,2620::1ae0:ffff:ffff:ffff:ffff:ffff,US -2620:0:1af0::,2620::1af0:ffff:ffff:ffff:ffff:ffff,CA -2620:0:1b00::,2620::1b07:ffff:ffff:ffff:ffff:ffff,US -2620:0:1c00::,2620::1cff:ffff:ffff:ffff:ffff:ffff,US -2620:0:2000::,2620::203f:ffff:ffff:ffff:ffff:ffff,US -2620:0:2100::,2620::213f:ffff:ffff:ffff:ffff:ffff,US -2620:0:2210::,2620::2210:ffff:ffff:ffff:ffff:ffff,US -2620:0:2220::,2620::2220:ffff:ffff:ffff:ffff:ffff,CA -2620:0:2240::,2620::2240:ffff:ffff:ffff:ffff:ffff,US -2620:0:2250::,2620::2250:ffff:ffff:ffff:ffff:ffff,US -2620:0:2260::,2620::2260:ffff:ffff:ffff:ffff:ffff,US -2620:0:2280::,2620::2280:ffff:ffff:ffff:ffff:ffff,US -2620:0:2290::,2620::2290:ffff:ffff:ffff:ffff:ffff,US -2620:0:22a0::,2620::22a0:ffff:ffff:ffff:ffff:ffff,US -2620:0:22b0::,2620::22b0:ffff:ffff:ffff:ffff:ffff,US -2620:0:22c0::,2620::22c0:ffff:ffff:ffff:ffff:ffff,US -2620:0:22d0::,2620::22d0:ffff:ffff:ffff:ffff:ffff,US -2620:0:22e0::,2620::22e0:ffff:ffff:ffff:ffff:ffff,CA -2620:0:22f0::,2620::22f0:ffff:ffff:ffff:ffff:ffff,US -2620:0:2300::,2620::230f:ffff:ffff:ffff:ffff:ffff,US -2620:0:2400::,2620::24ff:ffff:ffff:ffff:ffff:ffff,US -2620:0:2800::,2620::2800:ffff:ffff:ffff:ffff:ffff,US -2620:0:2810::,2620::2810:ffff:ffff:ffff:ffff:ffff,US -2620:0:2820::,2620::2820:ffff:ffff:ffff:ffff:ffff,US -2620:0:2830::,2620::2830:ffff:ffff:ffff:ffff:ffff,US -2620:0:2840::,2620::2840:ffff:ffff:ffff:ffff:ffff,US -2620:0:2850::,2620::2850:ffff:ffff:ffff:ffff:ffff,US -2620:0:2860::,2620::2860:ffff:ffff:ffff:ffff:ffff,US -2620:0:2870::,2620::2870:ffff:ffff:ffff:ffff:ffff,US -2620:0:2880::,2620::2880:ffff:ffff:ffff:ffff:ffff,US -2620:0:28a0::,2620::28a0:ffff:ffff:ffff:ffff:ffff,US -2620:0:28b0::,2620::28b0:ffff:ffff:ffff:ffff:ffff,US -2620:0:28d0::,2620::28d0:ffff:ffff:ffff:ffff:ffff,US -2620:0:28f0::,2620::28f0:ffff:ffff:ffff:ffff:ffff,US -2620:0:2900::,2620::290f:ffff:ffff:ffff:ffff:ffff,US -2620:0:2a00::,2620::2a1f:ffff:ffff:ffff:ffff:ffff,US -2620:0:2b00::,2620::2b00:ffff:ffff:ffff:ffff:ffff,US -2620:0:2b10::,2620::2b10:ffff:ffff:ffff:ffff:ffff,US -2620:0:2b20::,2620::2b20:ffff:ffff:ffff:ffff:ffff,US -2620:0:2b30::,2620::2b40:ffff:ffff:ffff:ffff:ffff,US -2620:0:2b50::,2620::2b50:ffff:ffff:ffff:ffff:ffff,US -2620:0:2b60::,2620::2b60:ffff:ffff:ffff:ffff:ffff,US -2620:0:2b70::,2620::2b8f:ffff:ffff:ffff:ffff:ffff,US -2620:0:2bc0::,2620::2bc3:ffff:ffff:ffff:ffff:ffff,US -2620:0:2be0::,2620::2be0:ffff:ffff:ffff:ffff:ffff,US -2620:0:2bf0::,2620::2bf0:ffff:ffff:ffff:ffff:ffff,US -2620:0:2d00::,2620::2d7f:ffff:ffff:ffff:ffff:ffff,US -2620:0:2e00::,2620::2e00:ffff:ffff:ffff:ffff:ffff,US -2620:0:2e10::,2620::2e10:ffff:ffff:ffff:ffff:ffff,US -2620:0:2e30::,2620::2e30:ffff:ffff:ffff:ffff:ffff,US -2620:0:2e40::,2620::2e40:ffff:ffff:ffff:ffff:ffff,US -2620:0:2e50::,2620::2e50:ffff:ffff:ffff:ffff:ffff,US -2620:0:2e60::,2620::2e60:ffff:ffff:ffff:ffff:ffff,US -2620:0:2e70::,2620::2e80:ffff:ffff:ffff:ffff:ffff,US -2620:0:2ea0::,2620::2ea0:ffff:ffff:ffff:ffff:ffff,US -2620:0:2eb0::,2620::2eb0:ffff:ffff:ffff:ffff:ffff,US -2620:0:2ed0::,2620::2ed0:ffff:ffff:ffff:ffff:ffff,US -2620:0:2ee0::,2620::2ee0:ffff:ffff:ffff:ffff:ffff,US -2620:0:2f00::,2620::2f7f:ffff:ffff:ffff:ffff:ffff,US -2620:0:5000::,2620::5000:ffff:ffff:ffff:ffff:ffff,US -2620:0:5010::,2620::5010:ffff:ffff:ffff:ffff:ffff,US -2620:0:5030::,2620::5030:ffff:ffff:ffff:ffff:ffff,US -2620:0:5040::,2620::5040:ffff:ffff:ffff:ffff:ffff,US -2620:0:5050::,2620::5050:ffff:ffff:ffff:ffff:ffff,US -2620:0:5060::,2620::5060:ffff:ffff:ffff:ffff:ffff,CA -2620:0:5070::,2620::5070:ffff:ffff:ffff:ffff:ffff,US -2620:0:5080::,2620::5080:ffff:ffff:ffff:ffff:ffff,US -2620:0:5090::,2620::5090:ffff:ffff:ffff:ffff:ffff,US -2620:0:50a0::,2620::50a0:ffff:ffff:ffff:ffff:ffff,US -2620:0:50b0::,2620::50b0:ffff:ffff:ffff:ffff:ffff,US -2620:0:50c0::,2620::50c0:ffff:ffff:ffff:ffff:ffff,US -2620:0:50d0::,2620::50d1:ffff:ffff:ffff:ffff:ffff,US -2620:0:50e0::,2620::50e0:ffff:ffff:ffff:ffff:ffff,US -2620:0:50f0::,2620::50f0:ffff:ffff:ffff:ffff:ffff,US -2620:0:5100::,2620::510f:ffff:ffff:ffff:ffff:ffff,US -2620:0:5200::,2620::5200:ffff:ffff:ffff:ffff:ffff,US -2620:0:5300::,2620::530f:ffff:ffff:ffff:ffff:ffff,US -2620:0:aa00::,2620::aa00:ffff:ffff:ffff:ffff:ffff,US -2620:1::,2620:1::ffff:ffff:ffff:ffff:ffff,US -2620:1:4000::,2620:1:4000:ffff:ffff:ffff:ffff:ffff,US -2620:1:8000::,2620:1:8000:ffff:ffff:ffff:ffff:ffff,US -2620:1:c000::,2620:1:c000:ffff:ffff:ffff:ffff:ffff,US -2620:2::,2620:2::ffff:ffff:ffff:ffff:ffff,US -2620:2:4000::,2620:2:4000:ffff:ffff:ffff:ffff:ffff,CA -2620:2:8000::,2620:2:8000:ffff:ffff:ffff:ffff:ffff,US -2620:2:c000::,2620:2:c000:ffff:ffff:ffff:ffff:ffff,US -2620:3::,2620:3::ffff:ffff:ffff:ffff:ffff,US -2620:3:4000::,2620:3:4000:ffff:ffff:ffff:ffff:ffff,US -2620:3:8000::,2620:3:8000:ffff:ffff:ffff:ffff:ffff,US -2620:3:c000::,2620:3:c000:ffff:ffff:ffff:ffff:ffff,US -2620:4::,2620:4::ffff:ffff:ffff:ffff:ffff,US -2620:4:4000::,2620:4:4000:ffff:ffff:ffff:ffff:ffff,US -2620:4:8000::,2620:4:8000:ffff:ffff:ffff:ffff:ffff,US -2620:4:c000::,2620:4:c000:ffff:ffff:ffff:ffff:ffff,US -2620:5::,2620:5::ffff:ffff:ffff:ffff:ffff,US -2620:5:4000::,2620:5:400f:ffff:ffff:ffff:ffff:ffff,US -2620:5:8000::,2620:5:8000:ffff:ffff:ffff:ffff:ffff,US -2620:5:c000::,2620:5:c000:ffff:ffff:ffff:ffff:ffff,US -2620:6::,2620:6::ffff:ffff:ffff:ffff:ffff,CA -2620:6:4000::,2620:6:4000:ffff:ffff:ffff:ffff:ffff,US -2620:6:8000::,2620:6:8000:ffff:ffff:ffff:ffff:ffff,US -2620:6:c000::,2620:6:c000:ffff:ffff:ffff:ffff:ffff,US -2620:7::,2620:7::ffff:ffff:ffff:ffff:ffff,US -2620:7:4000::,2620:7:4000:ffff:ffff:ffff:ffff:ffff,US -2620:7:8000::,2620:7:8000:ffff:ffff:ffff:ffff:ffff,US -2620:7:c000::,2620:7:c000:ffff:ffff:ffff:ffff:ffff,US -2620:8::,2620:8:7f:ffff:ffff:ffff:ffff:ffff,CA -2620:8:8200::,2620:8:8200:ffff:ffff:ffff:ffff:ffff,US -2620:9::,2620:9::ffff:ffff:ffff:ffff:ffff,US -2620:9:4000::,2620:9:4000:ffff:ffff:ffff:ffff:ffff,US -2620:9:8000::,2620:9:8000:ffff:ffff:ffff:ffff:ffff,US -2620:9:c000::,2620:9:c000:ffff:ffff:ffff:ffff:ffff,US -2620:a::,2620:a:f:ffff:ffff:ffff:ffff:ffff,CA -2620:a:4000::,2620:a:4000:ffff:ffff:ffff:ffff:ffff,US -2620:a:8000::,2620:a:8000:ffff:ffff:ffff:ffff:ffff,US -2620:a:c000::,2620:a:c000:ffff:ffff:ffff:ffff:ffff,US -2620:b::,2620:b::ffff:ffff:ffff:ffff:ffff,US -2620:b:4000::,2620:b:4000:ffff:ffff:ffff:ffff:ffff,US -2620:b:8000::,2620:b:8000:ffff:ffff:ffff:ffff:ffff,US -2620:b:c000::,2620:b:c000:ffff:ffff:ffff:ffff:ffff,US -2620:c::,2620:c::ffff:ffff:ffff:ffff:ffff,US -2620:c:4000::,2620:c:4000:ffff:ffff:ffff:ffff:ffff,US -2620:c:8000::,2620:c:8000:ffff:ffff:ffff:ffff:ffff,US -2620:c:c000::,2620:c:c000:ffff:ffff:ffff:ffff:ffff,US -2620:d::,2620:d::ffff:ffff:ffff:ffff:ffff,US -2620:d:4000::,2620:d:4000:ffff:ffff:ffff:ffff:ffff,US -2620:d:8000::,2620:d:8000:ffff:ffff:ffff:ffff:ffff,US -2620:d:c000::,2620:d:c000:ffff:ffff:ffff:ffff:ffff,US -2620:e::,2620:e::ffff:ffff:ffff:ffff:ffff,US -2620:e:4000::,2620:e:4000:ffff:ffff:ffff:ffff:ffff,US -2620:e:8000::,2620:e:8000:ffff:ffff:ffff:ffff:ffff,US -2620:e:c000::,2620:e:c000:ffff:ffff:ffff:ffff:ffff,US -2620:f::,2620:f:f:ffff:ffff:ffff:ffff:ffff,US -2620:f:4000::,2620:f:4000:ffff:ffff:ffff:ffff:ffff,US -2620:f:8000::,2620:f:8000:ffff:ffff:ffff:ffff:ffff,US -2620:f:c000::,2620:f:c000:ffff:ffff:ffff:ffff:ffff,US -2620:10::,2620:10::ffff:ffff:ffff:ffff:ffff,US -2620:10:4000::,2620:10:4000:ffff:ffff:ffff:ffff:ffff,US -2620:10:8000::,2620:10:8000:ffff:ffff:ffff:ffff:ffff,CA -2620:10:c000::,2620:10:c000:ffff:ffff:ffff:ffff:ffff,US -2620:11::,2620:11:ff:ffff:ffff:ffff:ffff:ffff,US -2620:11:4000::,2620:11:4000:ffff:ffff:ffff:ffff:ffff,US -2620:11:8000::,2620:11:8000:ffff:ffff:ffff:ffff:ffff,US -2620:11:c000::,2620:11:c000:ffff:ffff:ffff:ffff:ffff,US -2620:12::,2620:12::ffff:ffff:ffff:ffff:ffff,US -2620:12:4000::,2620:12:4000:ffff:ffff:ffff:ffff:ffff,US -2620:12:8000::,2620:12:8000:ffff:ffff:ffff:ffff:ffff,US -2620:12:c000::,2620:12:c000:ffff:ffff:ffff:ffff:ffff,US -2620:13::,2620:13::ffff:ffff:ffff:ffff:ffff,CA -2620:13:8000::,2620:13:8000:ffff:ffff:ffff:ffff:ffff,US -2620:13:c000::,2620:13:c000:ffff:ffff:ffff:ffff:ffff,US -2620:14::,2620:14::ffff:ffff:ffff:ffff:ffff,US -2620:14:4000::,2620:14:4000:ffff:ffff:ffff:ffff:ffff,US -2620:14:8000::,2620:14:8000:ffff:ffff:ffff:ffff:ffff,US -2620:14:c000::,2620:14:c000:ffff:ffff:ffff:ffff:ffff,CA -2620:15::,2620:15::ffff:ffff:ffff:ffff:ffff,US -2620:15:4000::,2620:15:4000:ffff:ffff:ffff:ffff:ffff,US -2620:15:8000::,2620:15:8000:ffff:ffff:ffff:ffff:ffff,US -2620:15:c000::,2620:15:c000:ffff:ffff:ffff:ffff:ffff,US -2620:16::,2620:16::ffff:ffff:ffff:ffff:ffff,CA -2620:16:4000::,2620:16:4000:ffff:ffff:ffff:ffff:ffff,US -2620:16:8000::,2620:16:8000:ffff:ffff:ffff:ffff:ffff,US -2620:16:c000::,2620:16:c000:ffff:ffff:ffff:ffff:ffff,US -2620:17::,2620:17::ffff:ffff:ffff:ffff:ffff,US -2620:17:4000::,2620:17:4000:ffff:ffff:ffff:ffff:ffff,US -2620:17:8000::,2620:17:800f:ffff:ffff:ffff:ffff:ffff,US -2620:17:c000::,2620:17:c000:ffff:ffff:ffff:ffff:ffff,US -2620:18::,2620:18::ffff:ffff:ffff:ffff:ffff,US -2620:18:4000::,2620:18:4000:ffff:ffff:ffff:ffff:ffff,US -2620:18:8000::,2620:18:8000:ffff:ffff:ffff:ffff:ffff,US -2620:18:c000::,2620:18:c000:ffff:ffff:ffff:ffff:ffff,KN -2620:19::,2620:19::ffff:ffff:ffff:ffff:ffff,US -2620:19:4000::,2620:19:4000:ffff:ffff:ffff:ffff:ffff,CA -2620:19:8000::,2620:19:8000:ffff:ffff:ffff:ffff:ffff,US -2620:19:c000::,2620:19:c000:ffff:ffff:ffff:ffff:ffff,US -2620:1a::,2620:1a::ffff:ffff:ffff:ffff:ffff,US -2620:1a:4000::,2620:1a:4000:ffff:ffff:ffff:ffff:ffff,US -2620:1a:8000::,2620:1a:8000:ffff:ffff:ffff:ffff:ffff,US -2620:1a:c000::,2620:1a:c000:ffff:ffff:ffff:ffff:ffff,US -2620:1b::,2620:1b:f:ffff:ffff:ffff:ffff:ffff,US -2620:1b:4000::,2620:1b:4000:ffff:ffff:ffff:ffff:ffff,US -2620:1b:8000::,2620:1b:8000:ffff:ffff:ffff:ffff:ffff,US -2620:1b:c000::,2620:1b:c000:ffff:ffff:ffff:ffff:ffff,US -2620:1c::,2620:1c::ffff:ffff:ffff:ffff:ffff,US -2620:1c:4000::,2620:1c:4000:ffff:ffff:ffff:ffff:ffff,US -2620:1c:8000::,2620:1c:8000:ffff:ffff:ffff:ffff:ffff,US -2620:1c:c000::,2620:1c:c000:ffff:ffff:ffff:ffff:ffff,US -2620:1d::,2620:1d::ffff:ffff:ffff:ffff:ffff,US -2620:1d:4000::,2620:1d:4000:ffff:ffff:ffff:ffff:ffff,US -2620:1d:8000::,2620:1d:8000:ffff:ffff:ffff:ffff:ffff,US -2620:1d:c000::,2620:1d:c000:ffff:ffff:ffff:ffff:ffff,US -2620:1e::,2620:1e::ffff:ffff:ffff:ffff:ffff,US -2620:1e:4000::,2620:1e:4000:ffff:ffff:ffff:ffff:ffff,US -2620:1e:8000::,2620:1e:8000:ffff:ffff:ffff:ffff:ffff,US -2620:1e:c000::,2620:1e:c000:ffff:ffff:ffff:ffff:ffff,US -2620:1f::,2620:1f::ffff:ffff:ffff:ffff:ffff,US -2620:1f:4000::,2620:1f:4000:ffff:ffff:ffff:ffff:ffff,CA -2620:1f:8000::,2620:1f:8000:ffff:ffff:ffff:ffff:ffff,US -2620:1f:c000::,2620:1f:c000:ffff:ffff:ffff:ffff:ffff,US -2620:20::,2620:20::ffff:ffff:ffff:ffff:ffff,US -2620:20:4000::,2620:20:4000:ffff:ffff:ffff:ffff:ffff,US -2620:20:8000::,2620:20:8000:ffff:ffff:ffff:ffff:ffff,US -2620:20:c000::,2620:20:c000:ffff:ffff:ffff:ffff:ffff,US -2620:21::,2620:21::ffff:ffff:ffff:ffff:ffff,US -2620:21:4000::,2620:21:4000:ffff:ffff:ffff:ffff:ffff,US -2620:21:8000::,2620:21:8000:ffff:ffff:ffff:ffff:ffff,US -2620:21:c000::,2620:21:c000:ffff:ffff:ffff:ffff:ffff,CA -2620:22::,2620:22::ffff:ffff:ffff:ffff:ffff,US -2620:22:4000::,2620:22:4000:ffff:ffff:ffff:ffff:ffff,CA -2620:22:8000::,2620:22:8000:ffff:ffff:ffff:ffff:ffff,US -2620:22:c000::,2620:22:c000:ffff:ffff:ffff:ffff:ffff,US -2620:23::,2620:23::ffff:ffff:ffff:ffff:ffff,US -2620:23:4000::,2620:23:4000:ffff:ffff:ffff:ffff:ffff,US -2620:23:8000::,2620:23:8000:ffff:ffff:ffff:ffff:ffff,US -2620:23:c000::,2620:23:c000:ffff:ffff:ffff:ffff:ffff,US -2620:24::,2620:24:1f:ffff:ffff:ffff:ffff:ffff,US -2620:24:8080::,2620:24:8080:ffff:ffff:ffff:ffff:ffff,US -2620:25::,2620:25::ffff:ffff:ffff:ffff:ffff,US -2620:25:4000::,2620:25:4000:ffff:ffff:ffff:ffff:ffff,US -2620:25:8000::,2620:25:8000:ffff:ffff:ffff:ffff:ffff,US -2620:25:c000::,2620:25:c00f:ffff:ffff:ffff:ffff:ffff,US -2620:26::,2620:26::ffff:ffff:ffff:ffff:ffff,US -2620:26:4000::,2620:26:400f:ffff:ffff:ffff:ffff:ffff,US -2620:26:8000::,2620:26:8000:ffff:ffff:ffff:ffff:ffff,US -2620:26:c000::,2620:26:c00f:ffff:ffff:ffff:ffff:ffff,US -2620:27::,2620:27:f:ffff:ffff:ffff:ffff:ffff,US -2620:27:8080::,2620:27:8080:ffff:ffff:ffff:ffff:ffff,US -2620:28::,2620:28::ffff:ffff:ffff:ffff:ffff,US -2620:28:4000::,2620:28:400f:ffff:ffff:ffff:ffff:ffff,US -2620:28:8000::,2620:28:8000:ffff:ffff:ffff:ffff:ffff,US -2620:28:c000::,2620:28:c000:ffff:ffff:ffff:ffff:ffff,US -2620:29::,2620:29::ffff:ffff:ffff:ffff:ffff,US -2620:29:4000::,2620:29:4000:ffff:ffff:ffff:ffff:ffff,US -2620:29:8000::,2620:29:8000:ffff:ffff:ffff:ffff:ffff,US -2620:29:c000::,2620:29:c00f:ffff:ffff:ffff:ffff:ffff,US -2620:2a::,2620:2a::ffff:ffff:ffff:ffff:ffff,US -2620:2a:4000::,2620:2a:400f:ffff:ffff:ffff:ffff:ffff,US -2620:2a:8000::,2620:2a:8000:ffff:ffff:ffff:ffff:ffff,US -2620:2a:c000::,2620:2a:c000:ffff:ffff:ffff:ffff:ffff,US -2620:2b::,2620:2b::ffff:ffff:ffff:ffff:ffff,US -2620:2b:4000::,2620:2b:400f:ffff:ffff:ffff:ffff:ffff,US -2620:2b:8000::,2620:2b:8000:ffff:ffff:ffff:ffff:ffff,US -2620:2b:c000::,2620:2b:c000:ffff:ffff:ffff:ffff:ffff,US -2620:2c::,2620:2c:f:ffff:ffff:ffff:ffff:ffff,US -2620:2c:8080::,2620:2c:8080:ffff:ffff:ffff:ffff:ffff,US -2620:2d::,2620:2d::ffff:ffff:ffff:ffff:ffff,US -2620:2d:4000::,2620:2d:400f:ffff:ffff:ffff:ffff:ffff,US -2620:2d:8000::,2620:2d:8000:ffff:ffff:ffff:ffff:ffff,US -2620:2d:c000::,2620:2d:c00f:ffff:ffff:ffff:ffff:ffff,US -2620:2e::,2620:2e:3f:ffff:ffff:ffff:ffff:ffff,US -2620:2e:8080::,2620:2e:8080:ffff:ffff:ffff:ffff:ffff,US -2620:2f::,2620:2f::ffff:ffff:ffff:ffff:ffff,CA -2620:2f:4000::,2620:2f:4000:ffff:ffff:ffff:ffff:ffff,US -2620:2f:8000::,2620:2f:8000:ffff:ffff:ffff:ffff:ffff,US -2620:2f:c000::,2620:2f:c000:ffff:ffff:ffff:ffff:ffff,US -2620:30::,2620:30::ffff:ffff:ffff:ffff:ffff,US -2620:30:4000::,2620:30:4000:ffff:ffff:ffff:ffff:ffff,US -2620:30:8000::,2620:30:8000:ffff:ffff:ffff:ffff:ffff,US -2620:30:c000::,2620:30:c00f:ffff:ffff:ffff:ffff:ffff,US -2620:31::,2620:31::ffff:ffff:ffff:ffff:ffff,US -2620:31:4000::,2620:31:4000:ffff:ffff:ffff:ffff:ffff,US -2620:31:8000::,2620:31:8000:ffff:ffff:ffff:ffff:ffff,US -2620:31:c000::,2620:31:c000:ffff:ffff:ffff:ffff:ffff,US -2620:32::,2620:32::ffff:ffff:ffff:ffff:ffff,US -2620:32:4000::,2620:32:4000:ffff:ffff:ffff:ffff:ffff,CA -2620:32:8000::,2620:32:8000:ffff:ffff:ffff:ffff:ffff,US -2620:32:c000::,2620:32:c00f:ffff:ffff:ffff:ffff:ffff,US -2620:33::,2620:33::ffff:ffff:ffff:ffff:ffff,US -2620:33:4000::,2620:33:400f:ffff:ffff:ffff:ffff:ffff,US -2620:33:8000::,2620:33:8000:ffff:ffff:ffff:ffff:ffff,US -2620:33:c000::,2620:33:c000:ffff:ffff:ffff:ffff:ffff,US -2620:34::,2620:34::ffff:ffff:ffff:ffff:ffff,US -2620:34:4000::,2620:34:4000:ffff:ffff:ffff:ffff:ffff,US -2620:34:8000::,2620:34:8000:ffff:ffff:ffff:ffff:ffff,US -2620:34:c000::,2620:34:c000:ffff:ffff:ffff:ffff:ffff,US -2620:35::,2620:35::ffff:ffff:ffff:ffff:ffff,US -2620:35:4000::,2620:35:400f:ffff:ffff:ffff:ffff:ffff,CA -2620:35:8000::,2620:35:8000:ffff:ffff:ffff:ffff:ffff,CA -2620:35:c000::,2620:35:c00f:ffff:ffff:ffff:ffff:ffff,US -2620:36::,2620:36::ffff:ffff:ffff:ffff:ffff,US -2620:36:4000::,2620:36:400f:ffff:ffff:ffff:ffff:ffff,CA -2620:36:8000::,2620:36:8000:ffff:ffff:ffff:ffff:ffff,US -2620:36:c000::,2620:36:c000:ffff:ffff:ffff:ffff:ffff,CA -2620:37::,2620:37::ffff:ffff:ffff:ffff:ffff,US -2620:37:4000::,2620:37:400f:ffff:ffff:ffff:ffff:ffff,US -2620:37:8000::,2620:37:8000:ffff:ffff:ffff:ffff:ffff,US -2620:37:c000::,2620:37:c000:ffff:ffff:ffff:ffff:ffff,US -2620:38::,2620:38::ffff:ffff:ffff:ffff:ffff,US -2620:38:4000::,2620:38:400f:ffff:ffff:ffff:ffff:ffff,US -2620:38:8000::,2620:38:8000:ffff:ffff:ffff:ffff:ffff,US -2620:38:c000::,2620:38:c000:ffff:ffff:ffff:ffff:ffff,US -2620:39::,2620:39::ffff:ffff:ffff:ffff:ffff,US -2620:39:4000::,2620:39:4000:ffff:ffff:ffff:ffff:ffff,US -2620:39:8000::,2620:39:8000:ffff:ffff:ffff:ffff:ffff,US -2620:39:c000::,2620:39:c00f:ffff:ffff:ffff:ffff:ffff,US -2620:3a::,2620:3a::ffff:ffff:ffff:ffff:ffff,US -2620:3a:4000::,2620:3a:400f:ffff:ffff:ffff:ffff:ffff,US -2620:3a:8000::,2620:3a:8000:ffff:ffff:ffff:ffff:ffff,US -2620:3a:c000::,2620:3a:c000:ffff:ffff:ffff:ffff:ffff,US -2620:3b::,2620:3b::ffff:ffff:ffff:ffff:ffff,US -2620:3b:4000::,2620:3b:4000:ffff:ffff:ffff:ffff:ffff,US -2620:3b:8000::,2620:3b:8000:ffff:ffff:ffff:ffff:ffff,US -2620:3b:c000::,2620:3b:c00f:ffff:ffff:ffff:ffff:ffff,US -2620:3c::,2620:3c:3f:ffff:ffff:ffff:ffff:ffff,US -2620:3c:8080::,2620:3c:8080:ffff:ffff:ffff:ffff:ffff,US -2620:3d::,2620:3d::ffff:ffff:ffff:ffff:ffff,US -2620:3d:4000::,2620:3d:4000:ffff:ffff:ffff:ffff:ffff,US -2620:3d:8000::,2620:3d:8000:ffff:ffff:ffff:ffff:ffff,US -2620:3d:c000::,2620:3d:c000:ffff:ffff:ffff:ffff:ffff,CA -2620:3e::,2620:3e::ffff:ffff:ffff:ffff:ffff,US -2620:3e:4000::,2620:3e:4000:ffff:ffff:ffff:ffff:ffff,US -2620:3e:8000::,2620:3e:8000:ffff:ffff:ffff:ffff:ffff,US -2620:3e:c000::,2620:3e:c000:ffff:ffff:ffff:ffff:ffff,US -2620:3f::,2620:3f::ffff:ffff:ffff:ffff:ffff,US -2620:3f:4000::,2620:3f:4000:ffff:ffff:ffff:ffff:ffff,US -2620:3f:8000::,2620:3f:8000:ffff:ffff:ffff:ffff:ffff,US -2620:3f:c000::,2620:3f:c00f:ffff:ffff:ffff:ffff:ffff,US -2620:40::,2620:40::ffff:ffff:ffff:ffff:ffff,US -2620:40:4000::,2620:40:4000:ffff:ffff:ffff:ffff:ffff,US -2620:40:8000::,2620:40:8000:ffff:ffff:ffff:ffff:ffff,US -2620:40:c000::,2620:40:c000:ffff:ffff:ffff:ffff:ffff,US -2620:41::,2620:41::ffff:ffff:ffff:ffff:ffff,US -2620:41:4000::,2620:41:4000:ffff:ffff:ffff:ffff:ffff,US -2620:41:8000::,2620:41:8000:ffff:ffff:ffff:ffff:ffff,US -2620:41:c000::,2620:41:c00f:ffff:ffff:ffff:ffff:ffff,US -2620:42::,2620:42::ffff:ffff:ffff:ffff:ffff,US -2620:42:4000::,2620:42:4000:ffff:ffff:ffff:ffff:ffff,US -2620:42:8000::,2620:42:8000:ffff:ffff:ffff:ffff:ffff,US -2620:42:c000::,2620:42:c000:ffff:ffff:ffff:ffff:ffff,CA -2620:43::,2620:43::ffff:ffff:ffff:ffff:ffff,US -2620:43:4000::,2620:43:4000:ffff:ffff:ffff:ffff:ffff,US -2620:43:8000::,2620:43:8000:ffff:ffff:ffff:ffff:ffff,US -2620:43:c000::,2620:43:c000:ffff:ffff:ffff:ffff:ffff,US -2620:44::,2620:44:1:ffff:ffff:ffff:ffff:ffff,US -2620:44:4000::,2620:44:4000:ffff:ffff:ffff:ffff:ffff,US -2620:44:8000::,2620:44:8000:ffff:ffff:ffff:ffff:ffff,US -2620:44:c000::,2620:44:c000:ffff:ffff:ffff:ffff:ffff,US -2620:45::,2620:45::ffff:ffff:ffff:ffff:ffff,CA -2620:45:4000::,2620:45:4000:ffff:ffff:ffff:ffff:ffff,US -2620:45:8000::,2620:45:8000:ffff:ffff:ffff:ffff:ffff,US -2620:45:c000::,2620:45:c000:ffff:ffff:ffff:ffff:ffff,US -2620:46::,2620:46::ffff:ffff:ffff:ffff:ffff,US -2620:46:4000::,2620:46:4000:ffff:ffff:ffff:ffff:ffff,CA -2620:46:8000::,2620:46:8000:ffff:ffff:ffff:ffff:ffff,US -2620:46:c000::,2620:46:c00f:ffff:ffff:ffff:ffff:ffff,US -2620:47::,2620:47::ffff:ffff:ffff:ffff:ffff,US -2620:47:4000::,2620:47:4000:ffff:ffff:ffff:ffff:ffff,US -2620:47:8000::,2620:47:8000:ffff:ffff:ffff:ffff:ffff,US -2620:47:c000::,2620:47:c000:ffff:ffff:ffff:ffff:ffff,US -2620:48::,2620:48::ffff:ffff:ffff:ffff:ffff,US -2620:48:4000::,2620:48:4000:ffff:ffff:ffff:ffff:ffff,US -2620:48:8000::,2620:48:8000:ffff:ffff:ffff:ffff:ffff,US -2620:48:c000::,2620:48:c00f:ffff:ffff:ffff:ffff:ffff,US -2620:49::,2620:49:f:ffff:ffff:ffff:ffff:ffff,CA -2620:49:8080::,2620:49:8080:ffff:ffff:ffff:ffff:ffff,US -2620:4a::,2620:4a::ffff:ffff:ffff:ffff:ffff,US -2620:4a:4000::,2620:4a:4000:ffff:ffff:ffff:ffff:ffff,US -2620:4a:8000::,2620:4a:8000:ffff:ffff:ffff:ffff:ffff,US -2620:4a:c000::,2620:4a:c000:ffff:ffff:ffff:ffff:ffff,US -2620:4b::,2620:4b::ffff:ffff:ffff:ffff:ffff,CA -2620:4b:4000::,2620:4b:400f:ffff:ffff:ffff:ffff:ffff,US -2620:4b:8000::,2620:4b:800f:ffff:ffff:ffff:ffff:ffff,US -2620:4b:c000::,2620:4b:c00f:ffff:ffff:ffff:ffff:ffff,US -2620:4c::,2620:4c:1:ffff:ffff:ffff:ffff:ffff,US -2620:4c:4000::,2620:4c:400f:ffff:ffff:ffff:ffff:ffff,US -2620:4c:c000::,2620:4c:c000:ffff:ffff:ffff:ffff:ffff,US -2620:4d::,2620:4d::ffff:ffff:ffff:ffff:ffff,US -2620:4d:4000::,2620:4d:400f:ffff:ffff:ffff:ffff:ffff,US -2620:4d:8000::,2620:4d:8000:ffff:ffff:ffff:ffff:ffff,US -2620:4d:c000::,2620:4d:c000:ffff:ffff:ffff:ffff:ffff,US -2620:4e::,2620:4e::ffff:ffff:ffff:ffff:ffff,US -2620:4e:4000::,2620:4e:4000:ffff:ffff:ffff:ffff:ffff,CA -2620:4e:8000::,2620:4e:8000:ffff:ffff:ffff:ffff:ffff,US -2620:4e:c000::,2620:4e:c000:ffff:ffff:ffff:ffff:ffff,US -2620:4f::,2620:4f::ffff:ffff:ffff:ffff:ffff,US -2620:4f:4000::,2620:4f:4000:ffff:ffff:ffff:ffff:ffff,US -2620:4f:8000::,2620:4f:8000:ffff:ffff:ffff:ffff:ffff,US -2620:4f:c000::,2620:4f:c000:ffff:ffff:ffff:ffff:ffff,US -2620:50::,2620:50:f:ffff:ffff:ffff:ffff:ffff,US -2620:50:8080::,2620:50:8080:ffff:ffff:ffff:ffff:ffff,US -2620:51::,2620:51::ffff:ffff:ffff:ffff:ffff,US -2620:51:4000::,2620:51:4000:ffff:ffff:ffff:ffff:ffff,CA -2620:51:8000::,2620:51:8000:ffff:ffff:ffff:ffff:ffff,CA -2620:51:c000::,2620:51:c000:ffff:ffff:ffff:ffff:ffff,US -2620:52::,2620:52:3:ffff:ffff:ffff:ffff:ffff,US -2620:52:4000::,2620:52:4000:ffff:ffff:ffff:ffff:ffff,US -2620:52:8000::,2620:52:8000:ffff:ffff:ffff:ffff:ffff,US -2620:52:c000::,2620:52:c000:ffff:ffff:ffff:ffff:ffff,US -2620:53::,2620:53::ffff:ffff:ffff:ffff:ffff,US -2620:53:4000::,2620:53:400f:ffff:ffff:ffff:ffff:ffff,US -2620:53:8000::,2620:53:8000:ffff:ffff:ffff:ffff:ffff,US -2620:53:c000::,2620:53:c000:ffff:ffff:ffff:ffff:ffff,US -2620:54::,2620:54::ffff:ffff:ffff:ffff:ffff,US -2620:54:4000::,2620:54:4000:ffff:ffff:ffff:ffff:ffff,US -2620:54:8000::,2620:54:8000:ffff:ffff:ffff:ffff:ffff,US -2620:54:c000::,2620:54:c000:ffff:ffff:ffff:ffff:ffff,CA -2620:55::,2620:55::ffff:ffff:ffff:ffff:ffff,US -2620:55:4000::,2620:55:400f:ffff:ffff:ffff:ffff:ffff,US -2620:55:8000::,2620:55:8000:ffff:ffff:ffff:ffff:ffff,US -2620:55:c000::,2620:55:c000:ffff:ffff:ffff:ffff:ffff,US -2620:56::,2620:56::ffff:ffff:ffff:ffff:ffff,US -2620:56:4000::,2620:56:4000:ffff:ffff:ffff:ffff:ffff,US -2620:56:8000::,2620:56:8000:ffff:ffff:ffff:ffff:ffff,US -2620:56:c000::,2620:56:c00f:ffff:ffff:ffff:ffff:ffff,US -2620:57::,2620:57::ffff:ffff:ffff:ffff:ffff,US -2620:57:4000::,2620:57:400f:ffff:ffff:ffff:ffff:ffff,KY -2620:57:8000::,2620:57:8000:ffff:ffff:ffff:ffff:ffff,US -2620:57:c000::,2620:57:c000:ffff:ffff:ffff:ffff:ffff,US -2620:58::,2620:58:ff:ffff:ffff:ffff:ffff:ffff,US -2620:58:8800::,2620:58:8800:ffff:ffff:ffff:ffff:ffff,US -2620:59::,2620:59::ffff:ffff:ffff:ffff:ffff,US -2620:59:4000::,2620:59:4000:ffff:ffff:ffff:ffff:ffff,US -2620:59:8000::,2620:59:8000:ffff:ffff:ffff:ffff:ffff,US -2620:59:c000::,2620:59:c000:ffff:ffff:ffff:ffff:ffff,US -2620:5a::,2620:5a::ffff:ffff:ffff:ffff:ffff,US -2620:5a:4000::,2620:5a:4000:ffff:ffff:ffff:ffff:ffff,CA -2620:5a:8000::,2620:5a:8000:ffff:ffff:ffff:ffff:ffff,US -2620:5a:c000::,2620:5a:c00f:ffff:ffff:ffff:ffff:ffff,US -2620:5b::,2620:5b::ffff:ffff:ffff:ffff:ffff,US -2620:5b:4000::,2620:5b:4000:ffff:ffff:ffff:ffff:ffff,US -2620:5b:8000::,2620:5b:8000:ffff:ffff:ffff:ffff:ffff,CA -2620:5b:c000::,2620:5b:c00f:ffff:ffff:ffff:ffff:ffff,CA -2620:5c::,2620:5c::ffff:ffff:ffff:ffff:ffff,US -2620:5c:4000::,2620:5c:4000:ffff:ffff:ffff:ffff:ffff,US -2620:5c:8000::,2620:5c:8000:ffff:ffff:ffff:ffff:ffff,US -2620:5c:c000::,2620:5c:c00f:ffff:ffff:ffff:ffff:ffff,US -2620:5d::,2620:5d::ffff:ffff:ffff:ffff:ffff,US -2620:5d:4000::,2620:5d:4000:ffff:ffff:ffff:ffff:ffff,US -2620:5d:8000::,2620:5d:8000:ffff:ffff:ffff:ffff:ffff,US -2620:5d:c000::,2620:5d:c000:ffff:ffff:ffff:ffff:ffff,US -2620:5e::,2620:5e::ffff:ffff:ffff:ffff:ffff,US -2620:5e:4000::,2620:5e:4000:ffff:ffff:ffff:ffff:ffff,CA -2620:5e:8000::,2620:5e:8000:ffff:ffff:ffff:ffff:ffff,US -2620:5e:c000::,2620:5e:c000:ffff:ffff:ffff:ffff:ffff,CA -2620:5f::,2620:5f::ffff:ffff:ffff:ffff:ffff,US -2620:5f:4000::,2620:5f:4000:ffff:ffff:ffff:ffff:ffff,US -2620:5f:8000::,2620:5f:8000:ffff:ffff:ffff:ffff:ffff,US -2620:5f:c000::,2620:5f:c000:ffff:ffff:ffff:ffff:ffff,US -2620:60::,2620:60::ffff:ffff:ffff:ffff:ffff,US -2620:60:4000::,2620:60:400f:ffff:ffff:ffff:ffff:ffff,US -2620:60:8000::,2620:60:8000:ffff:ffff:ffff:ffff:ffff,CA -2620:60:c000::,2620:60:c000:ffff:ffff:ffff:ffff:ffff,CA -2620:61::,2620:61::ffff:ffff:ffff:ffff:ffff,CA -2620:61:4000::,2620:61:400f:ffff:ffff:ffff:ffff:ffff,US -2620:61:8000::,2620:61:8000:ffff:ffff:ffff:ffff:ffff,US -2620:61:c000::,2620:61:c000:ffff:ffff:ffff:ffff:ffff,US -2620:62::,2620:62::ffff:ffff:ffff:ffff:ffff,US -2620:62:4000::,2620:62:400f:ffff:ffff:ffff:ffff:ffff,CA -2620:62:8000::,2620:62:8000:ffff:ffff:ffff:ffff:ffff,US -2620:62:c000::,2620:62:c000:ffff:ffff:ffff:ffff:ffff,US -2620:63::,2620:63::ffff:ffff:ffff:ffff:ffff,US -2620:63:4000::,2620:63:4000:ffff:ffff:ffff:ffff:ffff,US -2620:63:8000::,2620:63:8000:ffff:ffff:ffff:ffff:ffff,US -2620:63:c000::,2620:63:c000:ffff:ffff:ffff:ffff:ffff,US -2620:64::,2620:64::ffff:ffff:ffff:ffff:ffff,US -2620:64:4000::,2620:64:4000:ffff:ffff:ffff:ffff:ffff,US -2620:64:8000::,2620:64:8000:ffff:ffff:ffff:ffff:ffff,TW -2620:64:c000::,2620:64:c00f:ffff:ffff:ffff:ffff:ffff,US -2620:65::,2620:65:ff:ffff:ffff:ffff:ffff:ffff,US -2620:65:8000::,2620:65:8000:ffff:ffff:ffff:ffff:ffff,US -2620:65:c000::,2620:65:c000:ffff:ffff:ffff:ffff:ffff,US -2620:66::,2620:66::ffff:ffff:ffff:ffff:ffff,CA -2620:66:4000::,2620:66:400f:ffff:ffff:ffff:ffff:ffff,US -2620:66:8000::,2620:66:8000:ffff:ffff:ffff:ffff:ffff,US -2620:66:c000::,2620:66:c000:ffff:ffff:ffff:ffff:ffff,US -2620:67::,2620:67::ffff:ffff:ffff:ffff:ffff,US -2620:67:4000::,2620:67:4000:ffff:ffff:ffff:ffff:ffff,US -2620:67:8000::,2620:67:8000:ffff:ffff:ffff:ffff:ffff,US -2620:67:c000::,2620:67:c000:ffff:ffff:ffff:ffff:ffff,US -2620:68::,2620:68::ffff:ffff:ffff:ffff:ffff,US -2620:68:4000::,2620:68:4000:ffff:ffff:ffff:ffff:ffff,US -2620:68:8000::,2620:68:8000:ffff:ffff:ffff:ffff:ffff,CA -2620:68:c000::,2620:68:c00f:ffff:ffff:ffff:ffff:ffff,US -2620:69:4000::,2620:69:4000:ffff:ffff:ffff:ffff:ffff,US -2620:69:8000::,2620:69:8000:ffff:ffff:ffff:ffff:ffff,US -2620:69:c000::,2620:69:c000:ffff:ffff:ffff:ffff:ffff,CA -2620:6a::,2620:6a::ffff:ffff:ffff:ffff:ffff,US -2620:6a:4000::,2620:6a:4000:ffff:ffff:ffff:ffff:ffff,CA -2620:6a:8000::,2620:6a:8000:ffff:ffff:ffff:ffff:ffff,US -2620:6a:c000::,2620:6a:c000:ffff:ffff:ffff:ffff:ffff,CA -2620:6b::,2620:6b::ffff:ffff:ffff:ffff:ffff,US -2620:6b:4000::,2620:6b:4000:ffff:ffff:ffff:ffff:ffff,US -2620:6b:8000::,2620:6b:8000:ffff:ffff:ffff:ffff:ffff,US -2620:6b:c000::,2620:6b:c000:ffff:ffff:ffff:ffff:ffff,US -2620:6c::,2620:6c:3f:ffff:ffff:ffff:ffff:ffff,US -2620:6c:8080::,2620:6c:8080:ffff:ffff:ffff:ffff:ffff,US -2620:6d:40::,2620:6d:40:ffff:ffff:ffff:ffff:ffff,US -2620:6d:8000::,2620:6d:8000:ffff:ffff:ffff:ffff:ffff,US -2620:6d:c000::,2620:6d:c000:ffff:ffff:ffff:ffff:ffff,US -2620:6e::,2620:6e::ffff:ffff:ffff:ffff:ffff,US -2620:6e:4000::,2620:6e:4000:ffff:ffff:ffff:ffff:ffff,US -2620:6e:8000::,2620:6e:8000:ffff:ffff:ffff:ffff:ffff,US -2620:6e:c000::,2620:6e:c000:ffff:ffff:ffff:ffff:ffff,US -2620:6f::,2620:6f::ffff:ffff:ffff:ffff:ffff,US -2620:6f:4000::,2620:6f:4000:ffff:ffff:ffff:ffff:ffff,US -2620:6f:8000::,2620:6f:8000:ffff:ffff:ffff:ffff:ffff,US -2620:6f:c000::,2620:6f:c000:ffff:ffff:ffff:ffff:ffff,US -2620:70::,2620:70::ffff:ffff:ffff:ffff:ffff,US -2620:70:4000::,2620:70:4000:ffff:ffff:ffff:ffff:ffff,US -2620:70:8000::,2620:70:8000:ffff:ffff:ffff:ffff:ffff,US -2620:70:c000::,2620:70:c000:ffff:ffff:ffff:ffff:ffff,US -2620:71::,2620:71::ffff:ffff:ffff:ffff:ffff,US -2620:71:4000::,2620:71:4000:ffff:ffff:ffff:ffff:ffff,US -2620:71:8000::,2620:71:8000:ffff:ffff:ffff:ffff:ffff,US -2620:71:c000::,2620:71:c000:ffff:ffff:ffff:ffff:ffff,US -2620:72::,2620:72::ffff:ffff:ffff:ffff:ffff,US -2620:72:4000::,2620:72:4000:ffff:ffff:ffff:ffff:ffff,US -2620:72:8000::,2620:72:8000:ffff:ffff:ffff:ffff:ffff,US -2620:72:c000::,2620:72:c000:ffff:ffff:ffff:ffff:ffff,US -2620:73::,2620:73::ffff:ffff:ffff:ffff:ffff,US -2620:73:4000::,2620:73:4000:ffff:ffff:ffff:ffff:ffff,US -2620:73:8000::,2620:73:8000:ffff:ffff:ffff:ffff:ffff,US -2620:73:c000::,2620:73:c000:ffff:ffff:ffff:ffff:ffff,US -2620:74::,2620:74:1f:ffff:ffff:ffff:ffff:ffff,US -2620:74:8080::,2620:74:8080:ffff:ffff:ffff:ffff:ffff,US -2620:75::,2620:75::ffff:ffff:ffff:ffff:ffff,US -2620:75:4000::,2620:75:4000:ffff:ffff:ffff:ffff:ffff,US -2620:75:8000::,2620:75:8000:ffff:ffff:ffff:ffff:ffff,US -2620:75:c000::,2620:75:c000:ffff:ffff:ffff:ffff:ffff,US -2620:76::,2620:76::ffff:ffff:ffff:ffff:ffff,US -2620:76:4000::,2620:76:4000:ffff:ffff:ffff:ffff:ffff,US -2620:76:8000::,2620:76:8000:ffff:ffff:ffff:ffff:ffff,US -2620:76:c000::,2620:76:c000:ffff:ffff:ffff:ffff:ffff,US -2620:77::,2620:77::ffff:ffff:ffff:ffff:ffff,US -2620:77:4000::,2620:77:4000:ffff:ffff:ffff:ffff:ffff,US -2620:77:8000::,2620:77:8000:ffff:ffff:ffff:ffff:ffff,US -2620:77:c000::,2620:77:c000:ffff:ffff:ffff:ffff:ffff,US -2620:78::,2620:78::ffff:ffff:ffff:ffff:ffff,US -2620:78:4000::,2620:78:4000:ffff:ffff:ffff:ffff:ffff,US -2620:78:8000::,2620:78:8000:ffff:ffff:ffff:ffff:ffff,US -2620:78:c000::,2620:78:c000:ffff:ffff:ffff:ffff:ffff,CA -2620:79::,2620:79::ffff:ffff:ffff:ffff:ffff,US -2620:79:4000::,2620:79:4000:ffff:ffff:ffff:ffff:ffff,US -2620:79:8000::,2620:79:8000:ffff:ffff:ffff:ffff:ffff,US -2620:79:c000::,2620:79:c000:ffff:ffff:ffff:ffff:ffff,US -2620:7a::,2620:7a::ffff:ffff:ffff:ffff:ffff,US -2620:7a:4000::,2620:7a:4000:ffff:ffff:ffff:ffff:ffff,US -2620:7a:8000::,2620:7a:8000:ffff:ffff:ffff:ffff:ffff,US -2620:7a:c000::,2620:7a:c000:ffff:ffff:ffff:ffff:ffff,US -2620:7b::,2620:7b::ffff:ffff:ffff:ffff:ffff,US -2620:7b:4000::,2620:7b:4000:ffff:ffff:ffff:ffff:ffff,US -2620:7b:8000::,2620:7b:8000:ffff:ffff:ffff:ffff:ffff,US -2620:7b:e000::,2620:7b:e000:ffff:ffff:ffff:ffff:ffff,US -2620:7c:4000::,2620:7c:4000:ffff:ffff:ffff:ffff:ffff,US -2620:7c:a000::,2620:7c:a000:ffff:ffff:ffff:ffff:ffff,US -2620:7d::,2620:7d::ffff:ffff:ffff:ffff:ffff,US -2620:7d:4000::,2620:7d:4000:ffff:ffff:ffff:ffff:ffff,US -2620:7d:8000::,2620:7d:8000:ffff:ffff:ffff:ffff:ffff,US -2620:7d:c000::,2620:7d:c000:ffff:ffff:ffff:ffff:ffff,US -2620:7e::,2620:7e:f:ffff:ffff:ffff:ffff:ffff,US -2620:7e:60c0::,2620:7e:60c0:ffff:ffff:ffff:ffff:ffff,US -2620:7e:c080::,2620:7e:c080:ffff:ffff:ffff:ffff:ffff,US -2620:7f:2040::,2620:7f:2040:ffff:ffff:ffff:ffff:ffff,US -2620:7f:8000::,2620:7f:8000:ffff:ffff:ffff:ffff:ffff,CA -2620:7f:c000::,2620:7f:c000:ffff:ffff:ffff:ffff:ffff,US -2620:80::,2620:80::ffff:ffff:ffff:ffff:ffff,US -2620:80:4000::,2620:80:4000:ffff:ffff:ffff:ffff:ffff,US -2620:80:8000::,2620:80:8000:ffff:ffff:ffff:ffff:ffff,CA -2620:80:c000::,2620:80:c000:ffff:ffff:ffff:ffff:ffff,CA -2620:81::,2620:81::ffff:ffff:ffff:ffff:ffff,US -2620:81:4000::,2620:81:4000:ffff:ffff:ffff:ffff:ffff,US -2620:81:8000::,2620:81:8000:ffff:ffff:ffff:ffff:ffff,US -2620:81:c000::,2620:81:c000:ffff:ffff:ffff:ffff:ffff,US -2620:82:4000::,2620:82:4000:ffff:ffff:ffff:ffff:ffff,US -2620:82:8000::,2620:82:8000:ffff:ffff:ffff:ffff:ffff,US -2620:83::,2620:83::ffff:ffff:ffff:ffff:ffff,US -2620:83:8000::,2620:83:8000:ffff:ffff:ffff:ffff:ffff,US -2620:84::,2620:84:1:ffff:ffff:ffff:ffff:ffff,US -2620:84:8000::,2620:84:8000:ffff:ffff:ffff:ffff:ffff,US -2620:85::,2620:85::ffff:ffff:ffff:ffff:ffff,US -2620:85:8000::,2620:85:8000:ffff:ffff:ffff:ffff:ffff,US -2620:86::,2620:86::ffff:ffff:ffff:ffff:ffff,US -2620:86:8000::,2620:86:8000:ffff:ffff:ffff:ffff:ffff,US -2620:87::,2620:87::ffff:ffff:ffff:ffff:ffff,US -2620:87:8000::,2620:87:8000:ffff:ffff:ffff:ffff:ffff,US -2620:88::,2620:88::ffff:ffff:ffff:ffff:ffff,US -2620:88:8000::,2620:88:8000:ffff:ffff:ffff:ffff:ffff,US -2620:89::,2620:89::ffff:ffff:ffff:ffff:ffff,US -2620:89:8000::,2620:89:8000:ffff:ffff:ffff:ffff:ffff,US -2620:8a::,2620:8a::ffff:ffff:ffff:ffff:ffff,US -2620:8a:8000::,2620:8a:8000:ffff:ffff:ffff:ffff:ffff,US -2620:8b::,2620:8b::ffff:ffff:ffff:ffff:ffff,US -2620:8b:8000::,2620:8b:8000:ffff:ffff:ffff:ffff:ffff,CA -2620:8c::,2620:8c::ffff:ffff:ffff:ffff:ffff,US -2620:8c:8000::,2620:8c:8000:ffff:ffff:ffff:ffff:ffff,US -2620:8d::,2620:8d::ffff:ffff:ffff:ffff:ffff,US -2620:8d:8000::,2620:8d:8000:ffff:ffff:ffff:ffff:ffff,US -2620:8e::,2620:8e::ffff:ffff:ffff:ffff:ffff,US -2620:8e:8000::,2620:8e:8000:ffff:ffff:ffff:ffff:ffff,US -2620:8f::,2620:8f::ffff:ffff:ffff:ffff:ffff,US -2620:8f:8000::,2620:8f:8000:ffff:ffff:ffff:ffff:ffff,US -2620:90::,2620:90::ffff:ffff:ffff:ffff:ffff,CA -2620:90:8000::,2620:90:8000:ffff:ffff:ffff:ffff:ffff,US -2620:91::,2620:91::ffff:ffff:ffff:ffff:ffff,US -2620:91:8000::,2620:91:8000:ffff:ffff:ffff:ffff:ffff,US -2620:92::,2620:92:f:ffff:ffff:ffff:ffff:ffff,US -2620:92:8000::,2620:92:8000:ffff:ffff:ffff:ffff:ffff,US -2620:93::,2620:93::ffff:ffff:ffff:ffff:ffff,US -2620:93:8000::,2620:93:8000:ffff:ffff:ffff:ffff:ffff,US -2620:94::,2620:94::ffff:ffff:ffff:ffff:ffff,US -2620:94:8000::,2620:94:8000:ffff:ffff:ffff:ffff:ffff,US -2620:95::,2620:95::ffff:ffff:ffff:ffff:ffff,US -2620:95:8000::,2620:95:8000:ffff:ffff:ffff:ffff:ffff,US -2620:96::,2620:96::ffff:ffff:ffff:ffff:ffff,US -2620:96:8000::,2620:96:8000:ffff:ffff:ffff:ffff:ffff,US -2620:97::,2620:97::ffff:ffff:ffff:ffff:ffff,US -2620:97:8000::,2620:97:8000:ffff:ffff:ffff:ffff:ffff,US -2620:98::,2620:98::ffff:ffff:ffff:ffff:ffff,US -2620:98:8000::,2620:98:8000:ffff:ffff:ffff:ffff:ffff,US -2620:99::,2620:99::ffff:ffff:ffff:ffff:ffff,US -2620:99:8000::,2620:99:8000:ffff:ffff:ffff:ffff:ffff,US -2620:9a::,2620:9a::ffff:ffff:ffff:ffff:ffff,CA -2620:9a:8000::,2620:9a:8000:ffff:ffff:ffff:ffff:ffff,US -2620:9b::,2620:9b::ffff:ffff:ffff:ffff:ffff,US -2620:9b:8000::,2620:9b:8000:ffff:ffff:ffff:ffff:ffff,US -2620:9c::,2620:9c::ffff:ffff:ffff:ffff:ffff,US -2620:9c:8000::,2620:9c:8000:ffff:ffff:ffff:ffff:ffff,US -2620:9d::,2620:9d::ffff:ffff:ffff:ffff:ffff,US -2620:9d:8000::,2620:9d:8000:ffff:ffff:ffff:ffff:ffff,US -2620:9e::,2620:9e::ffff:ffff:ffff:ffff:ffff,US -2620:9e:8000::,2620:9e:8000:ffff:ffff:ffff:ffff:ffff,US -2620:9f::,2620:9f:ff:ffff:ffff:ffff:ffff:ffff,US -2620:9f:8000::,2620:9f:8000:ffff:ffff:ffff:ffff:ffff,US -2620:a0::,2620:a0::ffff:ffff:ffff:ffff:ffff,US -2620:a0:8000::,2620:a0:8000:ffff:ffff:ffff:ffff:ffff,US -2620:a1::,2620:a1::ffff:ffff:ffff:ffff:ffff,US -2620:a1:8000::,2620:a1:8000:ffff:ffff:ffff:ffff:ffff,US -2620:a2::,2620:a2::ffff:ffff:ffff:ffff:ffff,US -2620:a2:8000::,2620:a2:8000:ffff:ffff:ffff:ffff:ffff,US -2620:a3::,2620:a3::ffff:ffff:ffff:ffff:ffff,US -2620:a3:8000::,2620:a3:8000:ffff:ffff:ffff:ffff:ffff,US -2620:a3:c020::,2620:a3:c020:ffff:ffff:ffff:ffff:ffff,US -2620:a4:40::,2620:a4:40:ffff:ffff:ffff:ffff:ffff,US -2620:a4:4060::,2620:a4:4060:ffff:ffff:ffff:ffff:ffff,US -2620:a4:8080::,2620:a4:8080:ffff:ffff:ffff:ffff:ffff,US -2620:a5::,2620:a5::ffff:ffff:ffff:ffff:ffff,US -2620:a5:8000::,2620:a5:8000:ffff:ffff:ffff:ffff:ffff,US -2620:a6::,2620:a6::ffff:ffff:ffff:ffff:ffff,US -2620:a6:8000::,2620:a6:8000:ffff:ffff:ffff:ffff:ffff,US -2620:a7::,2620:a7::ffff:ffff:ffff:ffff:ffff,US -2620:a7:8000::,2620:a7:8000:ffff:ffff:ffff:ffff:ffff,US -2620:a8::,2620:a8::ffff:ffff:ffff:ffff:ffff,US -2620:a8:8000::,2620:a8:8000:ffff:ffff:ffff:ffff:ffff,US -2620:a9::,2620:a9::ffff:ffff:ffff:ffff:ffff,US -2620:a9:8000::,2620:a9:800f:ffff:ffff:ffff:ffff:ffff,US -2620:aa::,2620:aa::ffff:ffff:ffff:ffff:ffff,US -2620:aa:8000::,2620:aa:8000:ffff:ffff:ffff:ffff:ffff,US -2620:ab::,2620:ab::ffff:ffff:ffff:ffff:ffff,US -2620:ab:8000::,2620:ab:8000:ffff:ffff:ffff:ffff:ffff,US -2620:ac::,2620:ac::ffff:ffff:ffff:ffff:ffff,US -2620:ac:8000::,2620:ac:8000:ffff:ffff:ffff:ffff:ffff,US -2620:ad::,2620:ad::ffff:ffff:ffff:ffff:ffff,US -2620:ad:8000::,2620:ad:8000:ffff:ffff:ffff:ffff:ffff,US -2620:ae::,2620:ae::ffff:ffff:ffff:ffff:ffff,CA -2620:ae:8000::,2620:ae:8000:ffff:ffff:ffff:ffff:ffff,US -2620:af::,2620:af::ffff:ffff:ffff:ffff:ffff,US -2620:af:8000::,2620:af:8000:ffff:ffff:ffff:ffff:ffff,US -2620:b0::,2620:b0::ffff:ffff:ffff:ffff:ffff,CA -2620:b0:8000::,2620:b0:8000:ffff:ffff:ffff:ffff:ffff,US -2620:b1::,2620:b1::ffff:ffff:ffff:ffff:ffff,US -2620:b1:8000::,2620:b1:8000:ffff:ffff:ffff:ffff:ffff,US -2620:b2::,2620:b2::ffff:ffff:ffff:ffff:ffff,US -2620:b2:8000::,2620:b2:8000:ffff:ffff:ffff:ffff:ffff,US -2620:b3::,2620:b3::ffff:ffff:ffff:ffff:ffff,US -2620:b3:8000::,2620:b3:8000:ffff:ffff:ffff:ffff:ffff,US -2620:b4::,2620:b4::ffff:ffff:ffff:ffff:ffff,US -2620:b4:8000::,2620:b4:8000:ffff:ffff:ffff:ffff:ffff,CA -2620:b5::,2620:b5::ffff:ffff:ffff:ffff:ffff,US -2620:b5:8000::,2620:b5:8000:ffff:ffff:ffff:ffff:ffff,US -2620:b6::,2620:b6::ffff:ffff:ffff:ffff:ffff,US -2620:b6:8000::,2620:b6:8000:ffff:ffff:ffff:ffff:ffff,US -2620:b7::,2620:b7::ffff:ffff:ffff:ffff:ffff,US -2620:b7:8000::,2620:b7:8000:ffff:ffff:ffff:ffff:ffff,CA -2620:b8::,2620:b8::ffff:ffff:ffff:ffff:ffff,US -2620:b8:8000::,2620:b8:8000:ffff:ffff:ffff:ffff:ffff,US -2620:b9::,2620:b9::ffff:ffff:ffff:ffff:ffff,US -2620:b9:8000::,2620:b9:8000:ffff:ffff:ffff:ffff:ffff,US -2620:ba::,2620:ba::ffff:ffff:ffff:ffff:ffff,US -2620:ba:8000::,2620:ba:8000:ffff:ffff:ffff:ffff:ffff,US -2620:bb::,2620:bb::ffff:ffff:ffff:ffff:ffff,US -2620:bb:8000::,2620:bb:8000:ffff:ffff:ffff:ffff:ffff,US -2620:bc::,2620:bc::ffff:ffff:ffff:ffff:ffff,US -2620:bc:8000::,2620:bc:8000:ffff:ffff:ffff:ffff:ffff,US -2620:bd::,2620:bd::ffff:ffff:ffff:ffff:ffff,CA -2620:bd:8000::,2620:bd:8000:ffff:ffff:ffff:ffff:ffff,US -2620:be::,2620:be::ffff:ffff:ffff:ffff:ffff,US -2620:be:8000::,2620:be:8000:ffff:ffff:ffff:ffff:ffff,US -2620:bf::,2620:bf::ffff:ffff:ffff:ffff:ffff,US -2620:bf:8000::,2620:bf:8000:ffff:ffff:ffff:ffff:ffff,US -2620:c0::,2620:c0::ffff:ffff:ffff:ffff:ffff,US -2620:c0:8000::,2620:c0:8000:ffff:ffff:ffff:ffff:ffff,US -2620:c1::,2620:c1::ffff:ffff:ffff:ffff:ffff,US -2620:c1:8000::,2620:c1:8000:ffff:ffff:ffff:ffff:ffff,US -2620:c2::,2620:c2::ffff:ffff:ffff:ffff:ffff,CA -2620:c2:8000::,2620:c2:8000:ffff:ffff:ffff:ffff:ffff,US -2620:c3::,2620:c3::ffff:ffff:ffff:ffff:ffff,US -2620:c3:8000::,2620:c3:8000:ffff:ffff:ffff:ffff:ffff,US -2620:c4::,2620:c4::ffff:ffff:ffff:ffff:ffff,US -2620:c4:8000::,2620:c4:8000:ffff:ffff:ffff:ffff:ffff,US -2620:c5::,2620:c5::ffff:ffff:ffff:ffff:ffff,US -2620:c5:8000::,2620:c5:8000:ffff:ffff:ffff:ffff:ffff,US -2620:c6::,2620:c6::ffff:ffff:ffff:ffff:ffff,US -2620:c6:8000::,2620:c6:8000:ffff:ffff:ffff:ffff:ffff,US -2620:c7::,2620:c7::ffff:ffff:ffff:ffff:ffff,US -2620:c7:8000::,2620:c7:8000:ffff:ffff:ffff:ffff:ffff,US -2620:c8::,2620:c8::ffff:ffff:ffff:ffff:ffff,US -2620:c8:8000::,2620:c8:8000:ffff:ffff:ffff:ffff:ffff,US -2620:c9::,2620:c9::ffff:ffff:ffff:ffff:ffff,US -2620:c9:8000::,2620:c9:8000:ffff:ffff:ffff:ffff:ffff,US -2620:ca::,2620:ca::ffff:ffff:ffff:ffff:ffff,US -2620:ca:8000::,2620:ca:8000:ffff:ffff:ffff:ffff:ffff,US -2620:cb::,2620:cb::ffff:ffff:ffff:ffff:ffff,US -2620:cb:8000::,2620:cb:8000:ffff:ffff:ffff:ffff:ffff,CA -2620:cc::,2620:cc::ffff:ffff:ffff:ffff:ffff,US -2620:cc:8000::,2620:cc:8000:ffff:ffff:ffff:ffff:ffff,US -2620:cd::,2620:cd::ffff:ffff:ffff:ffff:ffff,US -2620:cd:8000::,2620:cd:8000:ffff:ffff:ffff:ffff:ffff,US -2620:ce::,2620:ce::ffff:ffff:ffff:ffff:ffff,US -2620:ce:8000::,2620:ce:8000:ffff:ffff:ffff:ffff:ffff,US -2620:cf::,2620:cf::ffff:ffff:ffff:ffff:ffff,US -2620:cf:8000::,2620:cf:8000:ffff:ffff:ffff:ffff:ffff,US -2620:d0::,2620:d0::ffff:ffff:ffff:ffff:ffff,US -2620:d0:8000::,2620:d0:8000:ffff:ffff:ffff:ffff:ffff,US -2620:d1::,2620:d1::ffff:ffff:ffff:ffff:ffff,US -2620:d1:8000::,2620:d1:8000:ffff:ffff:ffff:ffff:ffff,US -2620:d2::,2620:d2::ffff:ffff:ffff:ffff:ffff,US -2620:d2:8000::,2620:d2:8000:ffff:ffff:ffff:ffff:ffff,US -2620:d3::,2620:d3::ffff:ffff:ffff:ffff:ffff,US -2620:d3:8000::,2620:d3:8000:ffff:ffff:ffff:ffff:ffff,US -2620:d4::,2620:d4::ffff:ffff:ffff:ffff:ffff,US -2620:d4:8000::,2620:d4:8000:ffff:ffff:ffff:ffff:ffff,US -2620:d5::,2620:d5::ffff:ffff:ffff:ffff:ffff,US -2620:d5:8000::,2620:d5:8000:ffff:ffff:ffff:ffff:ffff,US -2620:d6::,2620:d6::ffff:ffff:ffff:ffff:ffff,US -2620:d6:8000::,2620:d6:8000:ffff:ffff:ffff:ffff:ffff,US -2620:d7::,2620:d7::ffff:ffff:ffff:ffff:ffff,US -2620:d7:8000::,2620:d7:8000:ffff:ffff:ffff:ffff:ffff,US -2620:d8::,2620:d8::ffff:ffff:ffff:ffff:ffff,US -2620:d8:8000::,2620:d8:8000:ffff:ffff:ffff:ffff:ffff,US -2620:d9::,2620:d9::ffff:ffff:ffff:ffff:ffff,US -2620:d9:8000::,2620:d9:8000:ffff:ffff:ffff:ffff:ffff,US -2620:da::,2620:da::ffff:ffff:ffff:ffff:ffff,US -2620:da:8000::,2620:da:8000:ffff:ffff:ffff:ffff:ffff,US -2620:db::,2620:db::ffff:ffff:ffff:ffff:ffff,US -2620:db:8000::,2620:db:8000:ffff:ffff:ffff:ffff:ffff,US -2620:dc::,2620:dc::ffff:ffff:ffff:ffff:ffff,US -2620:dc:8::,2620:dc:8:ffff:ffff:ffff:ffff:ffff,US -2620:dc:8000::,2620:dc:8000:ffff:ffff:ffff:ffff:ffff,CA -2620:dd::,2620:dd::ffff:ffff:ffff:ffff:ffff,CA -2620:dd:8000::,2620:dd:8000:ffff:ffff:ffff:ffff:ffff,US -2620:de::,2620:de::ffff:ffff:ffff:ffff:ffff,US -2620:de:8000::,2620:de:8000:ffff:ffff:ffff:ffff:ffff,US -2620:df::,2620:df::ffff:ffff:ffff:ffff:ffff,US -2620:df:8000::,2620:df:8000:ffff:ffff:ffff:ffff:ffff,US -2620:e0::,2620:e0::ffff:ffff:ffff:ffff:ffff,US -2620:e0:8000::,2620:e0:8000:ffff:ffff:ffff:ffff:ffff,US -2620:e1::,2620:e1::ffff:ffff:ffff:ffff:ffff,US -2620:e1:8000::,2620:e1:8000:ffff:ffff:ffff:ffff:ffff,US -2620:e2::,2620:e2::ffff:ffff:ffff:ffff:ffff,US -2620:e2:8000::,2620:e2:8000:ffff:ffff:ffff:ffff:ffff,US -2620:e3::,2620:e3::ffff:ffff:ffff:ffff:ffff,US -2620:e3:8000::,2620:e3:8000:ffff:ffff:ffff:ffff:ffff,US -2620:e4::,2620:e4::ffff:ffff:ffff:ffff:ffff,US -2620:e4:8000::,2620:e4:8000:ffff:ffff:ffff:ffff:ffff,US -2620:e5::,2620:e5::ffff:ffff:ffff:ffff:ffff,US -2620:e5:8000::,2620:e5:8000:ffff:ffff:ffff:ffff:ffff,US -2620:e6::,2620:e6::ffff:ffff:ffff:ffff:ffff,US -2620:e6:8000::,2620:e6:8000:ffff:ffff:ffff:ffff:ffff,US -2620:e7::,2620:e7::ffff:ffff:ffff:ffff:ffff,US -2620:e7:8000::,2620:e7:8000:ffff:ffff:ffff:ffff:ffff,CA -2620:e8::,2620:e8::ffff:ffff:ffff:ffff:ffff,US -2620:e8:8000::,2620:e8:8000:ffff:ffff:ffff:ffff:ffff,US -2620:e9::,2620:e9::ffff:ffff:ffff:ffff:ffff,US -2620:e9:8000::,2620:e9:8000:ffff:ffff:ffff:ffff:ffff,US -2620:ea::,2620:ea:f:ffff:ffff:ffff:ffff:ffff,US -2620:ea:8000::,2620:ea:8000:ffff:ffff:ffff:ffff:ffff,US -2620:eb::,2620:eb::ffff:ffff:ffff:ffff:ffff,US -2620:eb:8000::,2620:eb:8000:ffff:ffff:ffff:ffff:ffff,US -2620:ec::,2620:ec::ffff:ffff:ffff:ffff:ffff,US -2620:ec:8000::,2620:ec:8000:ffff:ffff:ffff:ffff:ffff,US -2620:ed::,2620:ed::ffff:ffff:ffff:ffff:ffff,US -2620:ed:8000::,2620:ed:8000:ffff:ffff:ffff:ffff:ffff,US -2620:ee::,2620:ee::ffff:ffff:ffff:ffff:ffff,US -2620:ee:8000::,2620:ee:8000:ffff:ffff:ffff:ffff:ffff,US -2620:ef::,2620:ef::ffff:ffff:ffff:ffff:ffff,US -2620:ef:8000::,2620:ef:8000:ffff:ffff:ffff:ffff:ffff,US -2620:f0::,2620:f0::ffff:ffff:ffff:ffff:ffff,US -2620:f0:8000::,2620:f0:8000:ffff:ffff:ffff:ffff:ffff,US -2620:f1::,2620:f1::ffff:ffff:ffff:ffff:ffff,US -2620:f1:8000::,2620:f1:8000:ffff:ffff:ffff:ffff:ffff,US -2620:f2::,2620:f2::ffff:ffff:ffff:ffff:ffff,CA -2620:f2:8000::,2620:f2:8000:ffff:ffff:ffff:ffff:ffff,US -2620:f3::,2620:f3::ffff:ffff:ffff:ffff:ffff,US -2620:f3:8000::,2620:f3:8000:ffff:ffff:ffff:ffff:ffff,US -2620:f4::,2620:f4::ffff:ffff:ffff:ffff:ffff,US -2620:f4:8000::,2620:f4:8000:ffff:ffff:ffff:ffff:ffff,CA -2620:f5::,2620:f5::ffff:ffff:ffff:ffff:ffff,US -2620:f5:8000::,2620:f5:8000:ffff:ffff:ffff:ffff:ffff,US -2620:f6::,2620:f6::ffff:ffff:ffff:ffff:ffff,CA -2620:f6:8000::,2620:f6:8000:ffff:ffff:ffff:ffff:ffff,US -2620:f7::,2620:f7::ffff:ffff:ffff:ffff:ffff,US -2620:f7:8000::,2620:f7:8000:ffff:ffff:ffff:ffff:ffff,US -2620:f8::,2620:f8::ffff:ffff:ffff:ffff:ffff,US -2620:f8:8000::,2620:f8:8000:ffff:ffff:ffff:ffff:ffff,US -2620:f9::,2620:f9:f:ffff:ffff:ffff:ffff:ffff,US -2620:f9:8000::,2620:f9:8000:ffff:ffff:ffff:ffff:ffff,US -2620:fa::,2620:fa::ffff:ffff:ffff:ffff:ffff,US -2620:fa:8000::,2620:fa:8000:ffff:ffff:ffff:ffff:ffff,CA -2620:fb::,2620:fb::ffff:ffff:ffff:ffff:ffff,US -2620:fb:8000::,2620:fb:8000:ffff:ffff:ffff:ffff:ffff,US -2620:fc::,2620:fc::ffff:ffff:ffff:ffff:ffff,CA -2620:fc:8000::,2620:fc:8000:ffff:ffff:ffff:ffff:ffff,US -2620:fd::,2620:fd::ffff:ffff:ffff:ffff:ffff,CA -2620:fd:8000::,2620:fd:8000:ffff:ffff:ffff:ffff:ffff,US -2620:fe::,2620:fe:ff:ffff:ffff:ffff:ffff:ffff,US -2620:fe:8000::,2620:fe:8000:ffff:ffff:ffff:ffff:ffff,US -2620:ff::,2620:ff::ffff:ffff:ffff:ffff:ffff,US -2620:ff:8000::,2620:ff:8000:ffff:ffff:ffff:ffff:ffff,US -2620:100::,2620:100:f:ffff:ffff:ffff:ffff:ffff,US +2620::,2620:100:f:ffff:ffff:ffff:ffff:ffff,US 2620:100:1000::,2620:100:1001:ffff:ffff:ffff:ffff:ffff,US 2620:100:3000::,2620:100:3007:ffff:ffff:ffff:ffff:ffff,US 2620:100:4000::,2620:100:403f:ffff:ffff:ffff:ffff:ffff,US @@ -6274,7 +5292,6 @@ 2620:10c:5000::,2620:10c:500f:ffff:ffff:ffff:ffff:ffff,US 2620:10c:6000::,2620:10c:600f:ffff:ffff:ffff:ffff:ffff,US 2620:10c:7000::,2620:10c:700f:ffff:ffff:ffff:ffff:ffff,US -2620:10c:8000::,2620:10c:80ff:ffff:ffff:ffff:ffff:ffff,US 2620:10c:9000::,2620:10c:90ff:ffff:ffff:ffff:ffff:ffff,US 2620:10c:a000::,2620:10c:a00f:ffff:ffff:ffff:ffff:ffff,US 2620:10c:b000::,2620:10c:b0ff:ffff:ffff:ffff:ffff:ffff,US @@ -6496,6 +5513,19 @@ 2620:11a:7000::,2620:11a:700f:ffff:ffff:ffff:ffff:ffff,US 2620:11a:8000::,2620:11a:800f:ffff:ffff:ffff:ffff:ffff,US 2620:11a:9000::,2620:11a:900f:ffff:ffff:ffff:ffff:ffff,US +2620:11a:a000::,2620:11a:a0ff:ffff:ffff:ffff:ffff:ffff,US +2620:11a:b000::,2620:11a:b00f:ffff:ffff:ffff:ffff:ffff,US +2620:11a:c000::,2620:11a:c0ff:ffff:ffff:ffff:ffff:ffff,US +2620:11a:d000::,2620:11a:d0ff:ffff:ffff:ffff:ffff:ffff,US +2620:11a:e000::,2620:11a:e00f:ffff:ffff:ffff:ffff:ffff,US +2620:11a:f000::,2620:11a:f0ff:ffff:ffff:ffff:ffff:ffff,US +2620:11b::,2620:11b:ff:ffff:ffff:ffff:ffff:ffff,US +2620:11b:1000::,2620:11b:10ff:ffff:ffff:ffff:ffff:ffff,US +2620:11b:2000::,2620:11b:20ff:ffff:ffff:ffff:ffff:ffff,US +2620:11b:3000::,2620:11b:30ff:ffff:ffff:ffff:ffff:ffff,US +2620:11b:4000::,2620:11b:40ff:ffff:ffff:ffff:ffff:ffff,US +2620:11b:5000::,2620:11b:500f:ffff:ffff:ffff:ffff:ffff,US +2620:11b:6000::,2620:11b:600f:ffff:ffff:ffff:ffff:ffff,US 2620:140::,2620:140:3ff:ffff:ffff:ffff:ffff:ffff,US 2620:141::,2620:141:fff:ffff:ffff:ffff:ffff:ffff,US 2620:143::,2620:143:7ff:ffff:ffff:ffff:ffff:ffff,US @@ -6555,7 +5585,8 @@ 2620:1b0::,2620:1b0:ffff:ffff:ffff:ffff:ffff:ffff,US 2620:1c0::,2620:1c0:ffff:ffff:ffff:ffff:ffff:ffff,US 2620:1d0::,2620:1d0:ffff:ffff:ffff:ffff:ffff:ffff,US -2620:1e0::,2620:1e0:ffff:ffff:ffff:ffff:ffff:ffff,US +2620:1e0::,2620:1e1:fff:ffff:ffff:ffff:ffff:ffff,US +2620:1e2::,2620:1e2:fff:ffff:ffff:ffff:ffff:ffff,US 2620:1f0::,2620:1f1:fff:ffff:ffff:ffff:ffff:ffff,US 2620:1f2::,2620:1f4:fff:ffff:ffff:ffff:ffff:ffff,US 2620:1f5::,2620:1f5:fff:ffff:ffff:ffff:ffff:ffff,CA @@ -6575,7 +5606,6 @@ 2800:48::,2800:48:ffff:ffff:ffff:ffff:ffff:ffff,AR 2800:68::,2800:68:ffff:ffff:ffff:ffff:ffff:ffff,EC 2800:70::,2800:70:ffff:ffff:ffff:ffff:ffff:ffff,TT -2800:78::,2800:78:ffff:ffff:ffff:ffff:ffff:ffff,CO 2800:80::,2800:80:ffff:ffff:ffff:ffff:ffff:ffff,CW 2800:88::,2800:88:ffff:ffff:ffff:ffff:ffff:ffff,BO 2800:90::,2800:90:ffff:ffff:ffff:ffff:ffff:ffff,SV @@ -6646,7 +5676,6 @@ 2800:4f0::,2800:4f0:ffff:ffff:ffff:ffff:ffff:ffff,EC 2800:500::,2800:500:ffff:ffff:ffff:ffff:ffff:ffff,VE 2800:510::,2800:510:ffff:ffff:ffff:ffff:ffff:ffff,CR -2800:520::,2800:520:ffff:ffff:ffff:ffff:ffff:ffff,TT 2800:530::,2800:530:ffff:ffff:ffff:ffff:ffff:ffff,CW 2800:540::,2800:540:ffff:ffff:ffff:ffff:ffff:ffff,CL 2800:550::,2800:550:ffff:ffff:ffff:ffff:ffff:ffff,CL @@ -6666,7 +5695,7 @@ 2800:630::,2800:630:ffff:ffff:ffff:ffff:ffff:ffff,AR 2800:640::,2800:640:ffff:ffff:ffff:ffff:ffff:ffff,CR 2800:650::,2800:650:ffff:ffff:ffff:ffff:ffff:ffff,PE -2800:660::,2800:660:ffff:ffff:ffff:ffff:ffff:ffff,EC +2800:660::,2800:661:ffff:ffff:ffff:ffff:ffff:ffff,EC 2800:670::,2800:670:ffff:ffff:ffff:ffff:ffff:ffff,CL 2800:680::,2800:680:ffff:ffff:ffff:ffff:ffff:ffff,CO 2800:690::,2800:690:ffff:ffff:ffff:ffff:ffff:ffff,PE @@ -6757,6 +5786,7 @@ 2800:d00::,2800:d00:ffff:ffff:ffff:ffff:ffff:ffff,AR 2800:d10::,2800:d10:ffff:ffff:ffff:ffff:ffff:ffff,AR 2800:d20::,2800:d20:ffff:ffff:ffff:ffff:ffff:ffff,DO +2800:e00::,2800:eff:ffff:ffff:ffff:ffff:ffff:ffff,TT 2800:1000::,2800:10ff:ffff:ffff:ffff:ffff:ffff:ffff,VE 2800:2000::,2800:2fff:ffff:ffff:ffff:ffff:ffff:ffff,AR 2800:a000::,2800:a000:ffff:ffff:ffff:ffff:ffff:ffff,VE @@ -6846,16 +5876,19 @@ 2801:10:2000::,2801:10:2000:ffff:ffff:ffff:ffff:ffff,AR 2801:10:4000::,2801:10:4000:ffff:ffff:ffff:ffff:ffff,AR 2801:10:8000::,2801:10:8000:ffff:ffff:ffff:ffff:ffff,AR +2801:10:a000::,2801:10:a000:ffff:ffff:ffff:ffff:ffff,AR 2801:10:c000::,2801:10:c000:ffff:ffff:ffff:ffff:ffff,CO 2801:11::,2801:11::ffff:ffff:ffff:ffff:ffff,AR 2801:11:2000::,2801:11:2000:ffff:ffff:ffff:ffff:ffff,AR 2801:11:4000::,2801:11:4000:ffff:ffff:ffff:ffff:ffff,CO 2801:11:8000::,2801:11:8000:ffff:ffff:ffff:ffff:ffff,CO +2801:11:a000::,2801:11:a000:ffff:ffff:ffff:ffff:ffff,VE 2801:11:c000::,2801:11:c000:ffff:ffff:ffff:ffff:ffff,AR 2801:12::,2801:12::ffff:ffff:ffff:ffff:ffff,PY 2801:12:2000::,2801:12:2000:ffff:ffff:ffff:ffff:ffff,HN 2801:12:4000::,2801:12:4000:ffff:ffff:ffff:ffff:ffff,CO 2801:12:8000::,2801:12:8000:ffff:ffff:ffff:ffff:ffff,AR +2801:12:a000::,2801:12:a000:ffff:ffff:ffff:ffff:ffff,CL 2801:12:c000::,2801:12:c000:ffff:ffff:ffff:ffff:ffff,AR 2801:13::,2801:13::ffff:ffff:ffff:ffff:ffff,VE 2801:13:2000::,2801:13:2000:ffff:ffff:ffff:ffff:ffff,AR @@ -6865,6 +5898,7 @@ 2801:14::,2801:14::ffff:ffff:ffff:ffff:ffff,CO 2801:14:2000::,2801:14:2000:ffff:ffff:ffff:ffff:ffff,AR 2801:14:4000::,2801:14:4000:ffff:ffff:ffff:ffff:ffff,CO +2801:14:a000::,2801:14:a001:ffff:ffff:ffff:ffff:ffff,UY 2801:14:c000::,2801:14:c000:ffff:ffff:ffff:ffff:ffff,BO 2801:15::,2801:15::ffff:ffff:ffff:ffff:ffff,EC 2801:15:2000::,2801:15:2000:ffff:ffff:ffff:ffff:ffff,CR @@ -6875,8 +5909,10 @@ 2801:16:2000::,2801:16:2000:ffff:ffff:ffff:ffff:ffff,HN 2801:16:4000::,2801:16:4000:ffff:ffff:ffff:ffff:ffff,AR 2801:16:8000::,2801:16:8000:ffff:ffff:ffff:ffff:ffff,CO +2801:16:a000::,2801:16:a000:ffff:ffff:ffff:ffff:ffff,CR 2801:16:c000::,2801:16:c000:ffff:ffff:ffff:ffff:ffff,AR 2801:17::,2801:17::ffff:ffff:ffff:ffff:ffff,CL +2801:17:2000::,2801:17:2000:ffff:ffff:ffff:ffff:ffff,PY 2801:17:4000::,2801:17:4000:ffff:ffff:ffff:ffff:ffff,CO 2801:17:8000::,2801:17:8000:ffff:ffff:ffff:ffff:ffff,CR 2801:17:c000::,2801:17:c000:ffff:ffff:ffff:ffff:ffff,PA @@ -6884,18 +5920,22 @@ 2801:18:2000::,2801:18:2000:ffff:ffff:ffff:ffff:ffff,CO 2801:18:4000::,2801:18:4000:ffff:ffff:ffff:ffff:ffff,CO 2801:18:8000::,2801:18:8000:ffff:ffff:ffff:ffff:ffff,AR +2801:18:a000::,2801:18:a000:ffff:ffff:ffff:ffff:ffff,BO 2801:18:c000::,2801:18:c000:ffff:ffff:ffff:ffff:ffff,AR 2801:19::,2801:19::ffff:ffff:ffff:ffff:ffff,AR 2801:19:2000::,2801:19:2000:ffff:ffff:ffff:ffff:ffff,CL 2801:19:4000::,2801:19:4000:ffff:ffff:ffff:ffff:ffff,PY 2801:19:8000::,2801:19:8000:ffff:ffff:ffff:ffff:ffff,EC +2801:19:a000::,2801:19:a000:ffff:ffff:ffff:ffff:ffff,BO 2801:19:c000::,2801:19:c000:ffff:ffff:ffff:ffff:ffff,AR 2801:1a::,2801:1a::ffff:ffff:ffff:ffff:ffff,CO 2801:1a:2000::,2801:1a:2000:ffff:ffff:ffff:ffff:ffff,AR 2801:1a:4000::,2801:1a:4000:ffff:ffff:ffff:ffff:ffff,CO 2801:1a:8000::,2801:1a:8000:ffff:ffff:ffff:ffff:ffff,CL +2801:1a:a000::,2801:1a:a000:ffff:ffff:ffff:ffff:ffff,AR 2801:1a:c000::,2801:1a:c000:ffff:ffff:ffff:ffff:ffff,CO 2801:1b::,2801:1b::ffff:ffff:ffff:ffff:ffff,CR +2801:1b:2000::,2801:1b:2000:ffff:ffff:ffff:ffff:ffff,UY 2801:1b:4000::,2801:1b:4000:ffff:ffff:ffff:ffff:ffff,CL 2801:1b:8000::,2801:1b:8000:ffff:ffff:ffff:ffff:ffff,CL 2801:1b:c000::,2801:1b:c000:ffff:ffff:ffff:ffff:ffff,PA @@ -6903,6 +5943,7 @@ 2801:1c:2000::,2801:1c:2000:ffff:ffff:ffff:ffff:ffff,PE 2801:1c:4000::,2801:1c:4000:ffff:ffff:ffff:ffff:ffff,CO 2801:1c:8000::,2801:1c:8000:ffff:ffff:ffff:ffff:ffff,EC +2801:1c:a000::,2801:1c:a000:ffff:ffff:ffff:ffff:ffff,CO 2801:1c:c000::,2801:1c:c000:ffff:ffff:ffff:ffff:ffff,HN 2801:1d::,2801:1d::ffff:ffff:ffff:ffff:ffff,PY 2801:1d:2000::,2801:1d:2000:ffff:ffff:ffff:ffff:ffff,GT @@ -6913,8 +5954,10 @@ 2801:1e:2000::,2801:1e:2000:ffff:ffff:ffff:ffff:ffff,AR 2801:1e:4000::,2801:1e:4007:ffff:ffff:ffff:ffff:ffff,AR 2801:1e:8000::,2801:1e:8000:ffff:ffff:ffff:ffff:ffff,CR +2801:1e:a000::,2801:1e:a000:ffff:ffff:ffff:ffff:ffff,AR 2801:1e:c000::,2801:1e:c000:ffff:ffff:ffff:ffff:ffff,AR 2801:1f::,2801:1f::ffff:ffff:ffff:ffff:ffff,AR +2801:1f:2000::,2801:1f:2000:ffff:ffff:ffff:ffff:ffff,CR 2801:1f:4000::,2801:1f:4000:ffff:ffff:ffff:ffff:ffff,CR 2801:1f:8000::,2801:1f:8000:ffff:ffff:ffff:ffff:ffff,AR 2801:1f:c000::,2801:1f:c000:ffff:ffff:ffff:ffff:ffff,CR @@ -7096,6 +6139,18 @@ 2801:80:b30::,2801:80:b30:ffff:ffff:ffff:ffff:ffff,BR 2801:80:b40::,2801:80:b40:ffff:ffff:ffff:ffff:ffff,BR 2801:80:b50::,2801:80:b50:ffff:ffff:ffff:ffff:ffff,BR +2801:80:b60::,2801:80:b60:ffff:ffff:ffff:ffff:ffff,BR +2801:80:b70::,2801:80:b70:ffff:ffff:ffff:ffff:ffff,BR +2801:80:b80::,2801:80:b80:ffff:ffff:ffff:ffff:ffff,BR +2801:80:b90::,2801:80:b90:ffff:ffff:ffff:ffff:ffff,BR +2801:80:ba0::,2801:80:baf:ffff:ffff:ffff:ffff:ffff,BR +2801:80:bc0::,2801:80:bcf:ffff:ffff:ffff:ffff:ffff,BR +2801:80:be0::,2801:80:be0:ffff:ffff:ffff:ffff:ffff,BR +2801:80:bf0::,2801:80:bf0:ffff:ffff:ffff:ffff:ffff,BR +2801:80:c00::,2801:80:c00:ffff:ffff:ffff:ffff:ffff,BR +2801:80:c10::,2801:80:c10:ffff:ffff:ffff:ffff:ffff,BR +2801:80:c20::,2801:80:c20:ffff:ffff:ffff:ffff:ffff,BR +2801:80:c30::,2801:80:c30:ffff:ffff:ffff:ffff:ffff,BR 2801:82::,2801:82:ffff:ffff:ffff:ffff:ffff:ffff,BR 2801:84::,2801:84:ffff:ffff:ffff:ffff:ffff:ffff,BR 2801:86::,2801:86:ffff:ffff:ffff:ffff:ffff:ffff,BR @@ -7145,6 +6200,7 @@ 2801:f0:20::,2801:f0:20:ffff:ffff:ffff:ffff:ffff,MX 2801:f0:28::,2801:f0:28:ffff:ffff:ffff:ffff:ffff,MX 2801:100::,2801:100:ff:ffff:ffff:ffff:ffff:ffff,AR +2801:108::,2801:108:f:ffff:ffff:ffff:ffff:ffff,CO 2801:110::,2801:110:1fff:ffff:ffff:ffff:ffff:ffff,CO 2801:120::,2801:120:ffff:ffff:ffff:ffff:ffff:ffff,AR 2801:130::,2801:130:fff:ffff:ffff:ffff:ffff:ffff,CO @@ -7159,6 +6215,7 @@ 2801:1c0::,2801:1c0:1ff:ffff:ffff:ffff:ffff:ffff,AR 2801:1d0::,2801:1d0:f:ffff:ffff:ffff:ffff:ffff,CO 2801:1e0::,2801:1e0:7f:ffff:ffff:ffff:ffff:ffff,AR +2801:1f0::,2801:1f0:ffff:ffff:ffff:ffff:ffff:ffff,AR 2802::,2802:3:ffff:ffff:ffff:ffff:ffff:ffff,CO 2803::,2803::ffff:ffff:ffff:ffff:ffff:ffff,AR 2803:80::,2803:80:ffff:ffff:ffff:ffff:ffff:ffff,CL @@ -7170,6 +6227,7 @@ 2803:400::,2803:400:ffff:ffff:ffff:ffff:ffff:ffff,AR 2803:480::,2803:480:ffff:ffff:ffff:ffff:ffff:ffff,EC 2803:500::,2803:500:ffff:ffff:ffff:ffff:ffff:ffff,PE +2803:580::,2803:580:ffff:ffff:ffff:ffff:ffff:ffff,AR 2803:600::,2803:600:ffff:ffff:ffff:ffff:ffff:ffff,PA 2803:680::,2803:680:ffff:ffff:ffff:ffff:ffff:ffff,AR 2803:700::,2803:700:ffff:ffff:ffff:ffff:ffff:ffff,AR @@ -7183,6 +6241,7 @@ 2803:c00::,2803:c00:ffff:ffff:ffff:ffff:ffff:ffff,AR 2803:c80::,2803:c80:ffff:ffff:ffff:ffff:ffff:ffff,PY 2803:d00::,2803:d00:ffff:ffff:ffff:ffff:ffff:ffff,GY +2803:d80::,2803:d8f:ffff:ffff:ffff:ffff:ffff:ffff,CO 2803:e00::,2803:e00:ffff:ffff:ffff:ffff:ffff:ffff,CL 2803:e80::,2803:e80:ffff:ffff:ffff:ffff:ffff:ffff,CO 2803:f00::,2803:f00:ffff:ffff:ffff:ffff:ffff:ffff,PE @@ -7196,6 +6255,7 @@ 2803:1400::,2803:1400:ffff:ffff:ffff:ffff:ffff:ffff,DO 2803:1480::,2803:1480:ffff:ffff:ffff:ffff:ffff:ffff,AR 2803:1500::,2803:1500:ffff:ffff:ffff:ffff:ffff:ffff,TT +2803:1580::,2803:1580:ffff:ffff:ffff:ffff:ffff:ffff,CO 2803:1600::,2803:1600:ffff:ffff:ffff:ffff:ffff:ffff,BQ 2803:1680::,2803:1680:ffff:ffff:ffff:ffff:ffff:ffff,GF 2803:1700::,2803:1700:ffff:ffff:ffff:ffff:ffff:ffff,AR @@ -7221,6 +6281,7 @@ 2803:2400::,2803:2400:ffff:ffff:ffff:ffff:ffff:ffff,AR 2803:2480::,2803:2480:ffff:ffff:ffff:ffff:ffff:ffff,PE 2803:2500::,2803:2500:ffff:ffff:ffff:ffff:ffff:ffff,PE +2803:2580::,2803:2580:ffff:ffff:ffff:ffff:ffff:ffff,HN 2803:2600::,2803:2600:ffff:ffff:ffff:ffff:ffff:ffff,AR 2803:2680::,2803:2680:ffff:ffff:ffff:ffff:ffff:ffff,UY 2803:2700::,2803:2700:ffff:ffff:ffff:ffff:ffff:ffff,AR @@ -7247,6 +6308,7 @@ 2803:3400::,2803:3400:ffff:ffff:ffff:ffff:ffff:ffff,PA 2803:3480::,2803:3480:ffff:ffff:ffff:ffff:ffff:ffff,AR 2803:3500::,2803:3500:ffff:ffff:ffff:ffff:ffff:ffff,AR +2803:3580::,2803:3580:ffff:ffff:ffff:ffff:ffff:ffff,PE 2803:3600::,2803:3600:ffff:ffff:ffff:ffff:ffff:ffff,CR 2803:3680::,2803:3680:ffff:ffff:ffff:ffff:ffff:ffff,AR 2803:3700::,2803:3700:ffff:ffff:ffff:ffff:ffff:ffff,DO @@ -7273,6 +6335,7 @@ 2803:4400::,2803:4400:ffff:ffff:ffff:ffff:ffff:ffff,AR 2803:4480::,2803:4480:ffff:ffff:ffff:ffff:ffff:ffff,PE 2803:4500::,2803:4500:ffff:ffff:ffff:ffff:ffff:ffff,CW +2803:4580::,2803:4580:ffff:ffff:ffff:ffff:ffff:ffff,CR 2803:4600::,2803:4600:ffff:ffff:ffff:ffff:ffff:ffff,HN 2803:4680::,2803:4680:ffff:ffff:ffff:ffff:ffff:ffff,TT 2803:4700::,2803:4700:ffff:ffff:ffff:ffff:ffff:ffff,AR @@ -7286,6 +6349,7 @@ 2803:4c00::,2803:4c00:ffff:ffff:ffff:ffff:ffff:ffff,EC 2803:4c80::,2803:4c80:ffff:ffff:ffff:ffff:ffff:ffff,AR 2803:4d00::,2803:4d00:ffff:ffff:ffff:ffff:ffff:ffff,CL +2803:4d80::,2803:4d80:ffff:ffff:ffff:ffff:ffff:ffff,CW 2803:4e00::,2803:4e00:ffff:ffff:ffff:ffff:ffff:ffff,UY 2803:4e80::,2803:4e80:ffff:ffff:ffff:ffff:ffff:ffff,CR 2803:4f00::,2803:4f00:ffff:ffff:ffff:ffff:ffff:ffff,VE @@ -7299,6 +6363,7 @@ 2803:5400::,2803:5400:ffff:ffff:ffff:ffff:ffff:ffff,AR 2803:5480::,2803:5480:ffff:ffff:ffff:ffff:ffff:ffff,AR 2803:5500::,2803:5500:ffff:ffff:ffff:ffff:ffff:ffff,AR +2803:5580::,2803:5580:ffff:ffff:ffff:ffff:ffff:ffff,CR 2803:5600::,2803:5600:ffff:ffff:ffff:ffff:ffff:ffff,HN 2803:5680::,2803:5680:ffff:ffff:ffff:ffff:ffff:ffff,VE 2803:5700::,2803:5700:ffff:ffff:ffff:ffff:ffff:ffff,BO @@ -7324,6 +6389,7 @@ 2803:6400::,2803:6400:ffff:ffff:ffff:ffff:ffff:ffff,DO 2803:6480::,2803:6480:ffff:ffff:ffff:ffff:ffff:ffff,BZ 2803:6500::,2803:6500:ffff:ffff:ffff:ffff:ffff:ffff,PE +2803:6580::,2803:6580:ffff:ffff:ffff:ffff:ffff:ffff,CL 2803:6600::,2803:6600:ffff:ffff:ffff:ffff:ffff:ffff,AR 2803:6680::,2803:6680:ffff:ffff:ffff:ffff:ffff:ffff,VE 2803:6700::,2803:6700:ffff:ffff:ffff:ffff:ffff:ffff,CO @@ -7350,12 +6416,14 @@ 2803:7400::,2803:7400:ffff:ffff:ffff:ffff:ffff:ffff,AR 2803:7480::,2803:7480:ffff:ffff:ffff:ffff:ffff:ffff,AR 2803:7500::,2803:7500:ffff:ffff:ffff:ffff:ffff:ffff,CL +2803:7580::,2803:7580:ffff:ffff:ffff:ffff:ffff:ffff,AR 2803:7600::,2803:7600:ffff:ffff:ffff:ffff:ffff:ffff,AR 2803:7680::,2803:7680:ffff:ffff:ffff:ffff:ffff:ffff,BO 2803:7700::,2803:7700:ffff:ffff:ffff:ffff:ffff:ffff,AR 2803:7800::,2803:7800:ffff:ffff:ffff:ffff:ffff:ffff,VE 2803:7880::,2803:7880:ffff:ffff:ffff:ffff:ffff:ffff,AR 2803:7900::,2803:7900:ffff:ffff:ffff:ffff:ffff:ffff,BZ +2803:7980::,2803:7980:ffff:ffff:ffff:ffff:ffff:ffff,VE 2803:7a00::,2803:7a00:ffff:ffff:ffff:ffff:ffff:ffff,CR 2803:7a80::,2803:7a80:ffff:ffff:ffff:ffff:ffff:ffff,CR 2803:7b00::,2803:7b00:ffff:ffff:ffff:ffff:ffff:ffff,CL @@ -7375,6 +6443,7 @@ 2803:8400::,2803:8400:ffff:ffff:ffff:ffff:ffff:ffff,CO 2803:8480::,2803:8480:ffff:ffff:ffff:ffff:ffff:ffff,AR 2803:8500::,2803:8500:ffff:ffff:ffff:ffff:ffff:ffff,AR +2803:8580::,2803:8580:ffff:ffff:ffff:ffff:ffff:ffff,CO 2803:8600::,2803:8600:ffff:ffff:ffff:ffff:ffff:ffff,HT 2803:8680::,2803:8680:ffff:ffff:ffff:ffff:ffff:ffff,AR 2803:8700::,2803:8700:ffff:ffff:ffff:ffff:ffff:ffff,CR @@ -7388,6 +6457,7 @@ 2803:8c00::,2803:8c00:ffff:ffff:ffff:ffff:ffff:ffff,BZ 2803:8c80::,2803:8c80:ffff:ffff:ffff:ffff:ffff:ffff,PA 2803:8d00::,2803:8d00:ffff:ffff:ffff:ffff:ffff:ffff,AR +2803:8d80::,2803:8d80:ffff:ffff:ffff:ffff:ffff:ffff,UY 2803:8e00::,2803:8e00:ffff:ffff:ffff:ffff:ffff:ffff,CL 2803:8e80::,2803:8e80:ffff:ffff:ffff:ffff:ffff:ffff,PY 2803:8f00::,2803:8f00:ffff:ffff:ffff:ffff:ffff:ffff,PA @@ -7401,6 +6471,7 @@ 2803:9400::,2803:9400:ffff:ffff:ffff:ffff:ffff:ffff,BO 2803:9480::,2803:9480:ffff:ffff:ffff:ffff:ffff:ffff,AR 2803:9500::,2803:9500:ffff:ffff:ffff:ffff:ffff:ffff,AR +2803:9580::,2803:9580:ffff:ffff:ffff:ffff:ffff:ffff,AR 2803:9600::,2803:9600:ffff:ffff:ffff:ffff:ffff:ffff,CW 2803:9680::,2803:9680:ffff:ffff:ffff:ffff:ffff:ffff,AR 2803:9700::,2803:9700:ffff:ffff:ffff:ffff:ffff:ffff,EC @@ -7427,6 +6498,7 @@ 2803:a400::,2803:a400:ffff:ffff:ffff:ffff:ffff:ffff,EC 2803:a480::,2803:a480:ffff:ffff:ffff:ffff:ffff:ffff,AR 2803:a500::,2803:a500:ffff:ffff:ffff:ffff:ffff:ffff,AR +2803:a580::,2803:a580:ffff:ffff:ffff:ffff:ffff:ffff,HT 2803:a600::,2803:a600:ffff:ffff:ffff:ffff:ffff:ffff,AR 2803:a680::,2803:a680:ffff:ffff:ffff:ffff:ffff:ffff,AR 2803:a700::,2803:a700:ffff:ffff:ffff:ffff:ffff:ffff,HN @@ -7453,6 +6525,7 @@ 2803:b400::,2803:b400:ffff:ffff:ffff:ffff:ffff:ffff,VE 2803:b480::,2803:b480:ffff:ffff:ffff:ffff:ffff:ffff,CL 2803:b500::,2803:b500:ffff:ffff:ffff:ffff:ffff:ffff,VE +2803:b580::,2803:b580:ffff:ffff:ffff:ffff:ffff:ffff,CL 2803:b600::,2803:b600:ffff:ffff:ffff:ffff:ffff:ffff,AR 2803:b680::,2803:b680:ffff:ffff:ffff:ffff:ffff:ffff,AR 2803:b700::,2803:b700:ffff:ffff:ffff:ffff:ffff:ffff,HN @@ -7479,6 +6552,7 @@ 2803:c400::,2803:c400:ffff:ffff:ffff:ffff:ffff:ffff,AR 2803:c480::,2803:c480:ffff:ffff:ffff:ffff:ffff:ffff,AR 2803:c500::,2803:c500:ffff:ffff:ffff:ffff:ffff:ffff,AR +2803:c580::,2803:c580:ffff:ffff:ffff:ffff:ffff:ffff,AR 2803:c600::,2803:c600:ffff:ffff:ffff:ffff:ffff:ffff,CL 2803:c680::,2803:c680:ffff:ffff:ffff:ffff:ffff:ffff,HT 2803:c700::,2803:c700:ffff:ffff:ffff:ffff:ffff:ffff,GF @@ -7505,6 +6579,7 @@ 2803:d400::,2803:d400:ffff:ffff:ffff:ffff:ffff:ffff,AR 2803:d480::,2803:d480:ffff:ffff:ffff:ffff:ffff:ffff,VE 2803:d500::,2803:d500:ffff:ffff:ffff:ffff:ffff:ffff,BZ +2803:d580::,2803:d580:ffff:ffff:ffff:ffff:ffff:ffff,CL 2803:d600::,2803:d600:ffff:ffff:ffff:ffff:ffff:ffff,UY 2803:d680::,2803:d680:ffff:ffff:ffff:ffff:ffff:ffff,PE 2803:d700::,2803:d700:ffff:ffff:ffff:ffff:ffff:ffff,VE @@ -7531,6 +6606,7 @@ 2803:e400::,2803:e400:ffff:ffff:ffff:ffff:ffff:ffff,AR 2803:e480::,2803:e480:ffff:ffff:ffff:ffff:ffff:ffff,AR 2803:e500::,2803:e500:ffff:ffff:ffff:ffff:ffff:ffff,PE +2803:e580::,2803:e580:ffff:ffff:ffff:ffff:ffff:ffff,CO 2803:e600::,2803:e600:ffff:ffff:ffff:ffff:ffff:ffff,PA 2803:e680::,2803:e680:ffff:ffff:ffff:ffff:ffff:ffff,HN 2803:e700::,2803:e700:ffff:ffff:ffff:ffff:ffff:ffff,HN @@ -7557,12 +6633,14 @@ 2803:f400::,2803:f400:ffff:ffff:ffff:ffff:ffff:ffff,HN 2803:f480::,2803:f480:ffff:ffff:ffff:ffff:ffff:ffff,CO 2803:f500::,2803:f500:ffff:ffff:ffff:ffff:ffff:ffff,CW +2803:f580::,2803:f580:ffff:ffff:ffff:ffff:ffff:ffff,AR 2803:f600::,2803:f600:ffff:ffff:ffff:ffff:ffff:ffff,CR 2803:f680::,2803:f680:ffff:ffff:ffff:ffff:ffff:ffff,AR 2803:f700::,2803:f700:ffff:ffff:ffff:ffff:ffff:ffff,AR 2803:f800::,2803:f800:ffff:ffff:ffff:ffff:ffff:ffff,CR 2803:f880::,2803:f880:ffff:ffff:ffff:ffff:ffff:ffff,CR 2803:f900::,2803:f900:ffff:ffff:ffff:ffff:ffff:ffff,BZ +2803:f980::,2803:f980:ffff:ffff:ffff:ffff:ffff:ffff,CL 2803:fa00::,2803:fa00:ffff:ffff:ffff:ffff:ffff:ffff,BO 2803:fa80::,2803:fa80:ffff:ffff:ffff:ffff:ffff:ffff,AR 2803:fb00::,2803:fb00:ffff:ffff:ffff:ffff:ffff:ffff,PA @@ -9202,6 +8280,107 @@ 2804:19e4::,2804:19e4:ffff:ffff:ffff:ffff:ffff:ffff,BR 2804:19e8::,2804:19e8:ffff:ffff:ffff:ffff:ffff:ffff,BR 2804:19ec::,2804:19ec:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:19f0::,2804:19f0:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:19f4::,2804:19f4:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:19f8::,2804:19f8:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:19fc::,2804:19fc:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1a00::,2804:1a00:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1a04::,2804:1a04:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1a08::,2804:1a08:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1a0c::,2804:1a0c:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1a10::,2804:1a10:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1a14::,2804:1a14:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1a18::,2804:1a18:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1a1c::,2804:1a1c:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1a20::,2804:1a20:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1a24::,2804:1a24:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1a28::,2804:1a28:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1a2c::,2804:1a2c:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1a30::,2804:1a30:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1a34::,2804:1a34:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1a38::,2804:1a38:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1a3c::,2804:1a3c:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1a40::,2804:1a40:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1a44::,2804:1a44:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1a48::,2804:1a48:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1a4c::,2804:1a4c:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1a50::,2804:1a50:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1a54::,2804:1a54:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1a58::,2804:1a58:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1a5c::,2804:1a5c:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1a60::,2804:1a60:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1a64::,2804:1a64:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1a68::,2804:1a68:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1a6c::,2804:1a6c:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1a70::,2804:1a70:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1a74::,2804:1a74:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1a78::,2804:1a78:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1a7c::,2804:1a7c:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1a80::,2804:1a80:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1a84::,2804:1a84:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1a88::,2804:1a88:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1a8c::,2804:1a8c:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1a90::,2804:1a90:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1a94::,2804:1a94:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1a98::,2804:1a98:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1a9c::,2804:1a9c:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1aa0::,2804:1aa0:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1aa4::,2804:1aa4:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1aa8::,2804:1aa8:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1aac::,2804:1aac:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1ab0::,2804:1ab0:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1ab4::,2804:1ab4:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1ab8::,2804:1ab8:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1abc::,2804:1abc:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1ac0::,2804:1ac0:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1ac4::,2804:1ac4:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1ac8::,2804:1ac8:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1acc::,2804:1acc:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1ad0::,2804:1ad0:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1ad4::,2804:1ad4:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1ad8::,2804:1ad8:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1adc::,2804:1adc:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1ae0::,2804:1ae0:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1ae4::,2804:1ae4:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1ae8::,2804:1ae8:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1aec::,2804:1aec:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1af0::,2804:1af0:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1af4::,2804:1af4:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1af8::,2804:1af8:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1afc::,2804:1afc:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1b00::,2804:1b00:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1b04::,2804:1b04:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1b08::,2804:1b08:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1b0c::,2804:1b0c:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1b10::,2804:1b10:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1b14::,2804:1b14:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1b18::,2804:1b18:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1b1c::,2804:1b1c:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1b20::,2804:1b20:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1b24::,2804:1b24:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1b28::,2804:1b28:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1b2c::,2804:1b2c:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1b30::,2804:1b30:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1b34::,2804:1b34:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1b38::,2804:1b38:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1b3c::,2804:1b3c:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1b40::,2804:1b40:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1b44::,2804:1b44:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1b48::,2804:1b48:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1b4c::,2804:1b4c:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1b50::,2804:1b50:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1b54::,2804:1b54:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1b58::,2804:1b58:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1b5c::,2804:1b5c:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1b60::,2804:1b60:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1b64::,2804:1b64:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1b68::,2804:1b68:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1b6c::,2804:1b6c:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1b70::,2804:1b70:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1b74::,2804:1b74:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1b78::,2804:1b78:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1b7c::,2804:1b7c:ffff:ffff:ffff:ffff:ffff:ffff,BR +2804:1b80::,2804:1b80:ffff:ffff:ffff:ffff:ffff:ffff,BR 2804:1f00::,2804:1f00:ffff:ffff:ffff:ffff:ffff:ffff,BR 2804:1f02::,2804:1f02:ffff:ffff:ffff:ffff:ffff:ffff,BR 2804:1f04::,2804:1f04:ffff:ffff:ffff:ffff:ffff:ffff,BR @@ -9209,7 +8388,8 @@ 2804:1f08::,2804:1f08:ffff:ffff:ffff:ffff:ffff:ffff,BR 2806::,2806:f:ffff:ffff:ffff:ffff:ffff:ffff,MX 2806:200::,2806:200:ffff:ffff:ffff:ffff:ffff:ffff,MX -2806:210::,2806:214:ffff:ffff:ffff:ffff:ffff:ffff,MX +2806:210::,2806:216::ffff:ffff:ffff:ffff:ffff,MX +2806:217::,2806:218:ffff:ffff:ffff:ffff:ffff:ffff,MX 2806:220::,2806:220:ffff:ffff:ffff:ffff:ffff:ffff,MX 2806:230::,2806:230:ffff:ffff:ffff:ffff:ffff:ffff,MX 2806:238::,2806:238::ffff:ffff:ffff:ffff:ffff,MX @@ -9329,7 +8509,7 @@ 2a00:ef8::,2a00:ef8:ffff:ffff:ffff:ffff:ffff:ffff,NO 2a00:f00::,2a00:f00:ffff:ffff:ffff:ffff:ffff:ffff,RU 2a00:f08::,2a00:f08:ffff:ffff:ffff:ffff:ffff:ffff,RU -2a00:f10::,2a00:f10:ffff:ffff:ffff:ffff:ffff:ffff,NL +2a00:f10::,2a00:f17:ffff:ffff:ffff:ffff:ffff:ffff,NL 2a00:f18::,2a00:f18:ffff:ffff:ffff:ffff:ffff:ffff,GB 2a00:f20::,2a00:f20:ffff:ffff:ffff:ffff:ffff:ffff,IT 2a00:f28::,2a00:f2f:ffff:ffff:ffff:ffff:ffff:ffff,AE @@ -9358,7 +8538,36 @@ 2a00:fe8::,2a00:fe8:ffff:ffff:ffff:ffff:ffff:ffff,BA 2a00:ff0::,2a00:ff0:ffff:ffff:ffff:ffff:ffff:ffff,RO 2a00:ff8::,2a00:fff:ffff:ffff:ffff:ffff:ffff:ffff,IR -2a00:1000::,2a00:10ff:ffff:ffff:ffff:ffff:ffff:ffff,CZ +2a00:1000::,2a00:1000:ffff:ffff:ffff:ffff:ffff:ffff,DE +2a00:1008::,2a00:1008:ffff:ffff:ffff:ffff:ffff:ffff,NO +2a00:1010::,2a00:1010:ffff:ffff:ffff:ffff:ffff:ffff,DE +2a00:1018::,2a00:1018:ffff:ffff:ffff:ffff:ffff:ffff,DE +2a00:1020::,2a00:1020:ffff:ffff:ffff:ffff:ffff:ffff,RU +2a00:1028::,2a00:1028:ffff:ffff:ffff:ffff:ffff:ffff,CZ +2a00:1030::,2a00:1030:ffff:ffff:ffff:ffff:ffff:ffff,DE +2a00:1038::,2a00:1038:ffff:ffff:ffff:ffff:ffff:ffff,CH +2a00:1040::,2a00:1040:ffff:ffff:ffff:ffff:ffff:ffff,DK +2a00:1048::,2a00:1048:ffff:ffff:ffff:ffff:ffff:ffff,NL +2a00:1050::,2a00:1050:ffff:ffff:ffff:ffff:ffff:ffff,DE +2a00:1058::,2a00:1058:ffff:ffff:ffff:ffff:ffff:ffff,FR +2a00:1060::,2a00:1060:ffff:ffff:ffff:ffff:ffff:ffff,MT +2a00:1068::,2a00:1068:ffff:ffff:ffff:ffff:ffff:ffff,GB +2a00:1070::,2a00:1070:ffff:ffff:ffff:ffff:ffff:ffff,GB +2a00:1078::,2a00:1078:ffff:ffff:ffff:ffff:ffff:ffff,RU +2a00:1080::,2a00:1080:ffff:ffff:ffff:ffff:ffff:ffff,FR +2a00:1088::,2a00:1088:ffff:ffff:ffff:ffff:ffff:ffff,IT +2a00:1098::,2a00:1098:ffff:ffff:ffff:ffff:ffff:ffff,GB +2a00:10a0::,2a00:10a0:ffff:ffff:ffff:ffff:ffff:ffff,PL +2a00:10a8::,2a00:10a8:ffff:ffff:ffff:ffff:ffff:ffff,RU +2a00:10b0::,2a00:10b7:ffff:ffff:ffff:ffff:ffff:ffff,NO +2a00:10b8::,2a00:10b8:ffff:ffff:ffff:ffff:ffff:ffff,HU +2a00:10c0::,2a00:10c0:ffff:ffff:ffff:ffff:ffff:ffff,CH +2a00:10c8::,2a00:10c8:ffff:ffff:ffff:ffff:ffff:ffff,IT +2a00:10d0::,2a00:10d0:ffff:ffff:ffff:ffff:ffff:ffff,HU +2a00:10d8::,2a00:10d8:ffff:ffff:ffff:ffff:ffff:ffff,SK +2a00:10e0::,2a00:10e0:ffff:ffff:ffff:ffff:ffff:ffff,MD +2a00:10e8::,2a00:10f0:ffff:ffff:ffff:ffff:ffff:ffff,GB +2a00:10f8::,2a00:10f8:ffff:ffff:ffff:ffff:ffff:ffff,LU 2a00:1100::,2a00:1107:ffff:ffff:ffff:ffff:ffff:ffff,DE 2a00:1108::,2a00:1108:ffff:ffff:ffff:ffff:ffff:ffff,RS 2a00:1110::,2a00:1117:ffff:ffff:ffff:ffff:ffff:ffff,HU @@ -9424,7 +8633,7 @@ 2a00:1300::,2a00:1300:ffff:ffff:ffff:ffff:ffff:ffff,SE 2a00:1308::,2a00:1308:ffff:ffff:ffff:ffff:ffff:ffff,PL 2a00:1318::,2a00:1318:ffff:ffff:ffff:ffff:ffff:ffff,GB -2a00:1320::,2a00:1320:ffff:ffff:ffff:ffff:ffff:ffff,GB +2a00:1320::,2a00:1327:ffff:ffff:ffff:ffff:ffff:ffff,GB 2a00:1328::,2a00:1328:ffff:ffff:ffff:ffff:ffff:ffff,DE 2a00:1338::,2a00:1338:ffff:ffff:ffff:ffff:ffff:ffff,DE 2a00:1340::,2a00:1340:ffff:ffff:ffff:ffff:ffff:ffff,DE @@ -9703,7 +8912,7 @@ 2a00:1c30::,2a00:1c30:ffff:ffff:ffff:ffff:ffff:ffff,FI 2a00:1c38::,2a00:1c3f:ffff:ffff:ffff:ffff:ffff:ffff,DE 2a00:1c40::,2a00:1c40:ffff:ffff:ffff:ffff:ffff:ffff,GB -2a00:1c48::,2a00:1c48:ffff:ffff:ffff:ffff:ffff:ffff,RU +2a00:1c48::,2a00:1c49:ffff:ffff:ffff:ffff:ffff:ffff,RU 2a00:1c50::,2a00:1c50:ffff:ffff:ffff:ffff:ffff:ffff,SE 2a00:1c58::,2a00:1c58:ffff:ffff:ffff:ffff:ffff:ffff,NL 2a00:1c60::,2a00:1c60:ffff:ffff:ffff:ffff:ffff:ffff,PT @@ -10213,7 +9422,6 @@ 2a00:7380::,2a00:7380:ffff:ffff:ffff:ffff:ffff:ffff,DE 2a00:73a0::,2a00:73a0:ffff:ffff:ffff:ffff:ffff:ffff,RS 2a00:73c0::,2a00:73c0:ffff:ffff:ffff:ffff:ffff:ffff,ES -2a00:73e0::,2a00:73e0:ffff:ffff:ffff:ffff:ffff:ffff,GB 2a00:7400::,2a00:7400:ffff:ffff:ffff:ffff:ffff:ffff,IE 2a00:7420::,2a00:7420:ffff:ffff:ffff:ffff:ffff:ffff,NL 2a00:7440::,2a00:7447:ffff:ffff:ffff:ffff:ffff:ffff,DE @@ -10278,7 +9486,6 @@ 2a00:7c00::,2a00:7c00:ffff:ffff:ffff:ffff:ffff:ffff,RU 2a00:7c20::,2a00:7c20:ffff:ffff:ffff:ffff:ffff:ffff,DE 2a00:7c40::,2a00:7c40:ffff:ffff:ffff:ffff:ffff:ffff,GB -2a00:7c60::,2a00:7c60:ffff:ffff:ffff:ffff:ffff:ffff,GB 2a00:7c80::,2a00:7c80:ffff:ffff:ffff:ffff:ffff:ffff,NL 2a00:7ca0::,2a00:7ca0:ffff:ffff:ffff:ffff:ffff:ffff,IT 2a00:7cc0::,2a00:7cc0:ffff:ffff:ffff:ffff:ffff:ffff,DE @@ -10830,7 +10037,6 @@ 2a00:c640::,2a00:c640:ffff:ffff:ffff:ffff:ffff:ffff,SE 2a00:c660::,2a00:c660:ffff:ffff:ffff:ffff:ffff:ffff,NL 2a00:c680::,2a00:c680:ffff:ffff:ffff:ffff:ffff:ffff,IT -2a00:c6a0::,2a00:c6a0:ffff:ffff:ffff:ffff:ffff:ffff,GB 2a00:c6c0::,2a00:c6c0:ffff:ffff:ffff:ffff:ffff:ffff,NL 2a00:c6e0::,2a00:c6e0:ffff:ffff:ffff:ffff:ffff:ffff,SE 2a00:c700::,2a00:c700:ffff:ffff:ffff:ffff:ffff:ffff,EE @@ -11174,7 +10380,14 @@ 2a00:f3a0::,2a00:f3a0:ffff:ffff:ffff:ffff:ffff:ffff,IT 2a00:f3c0::,2a00:f3c0:ffff:ffff:ffff:ffff:ffff:ffff,GB 2a00:f3e0::,2a00:f3e0:ffff:ffff:ffff:ffff:ffff:ffff,RU -2a00:f400::,2a00:f4ff:ffff:ffff:ffff:ffff:ffff:ffff,RU +2a00:f400::,2a00:f400:ffff:ffff:ffff:ffff:ffff:ffff,LU +2a00:f420::,2a00:f420:ffff:ffff:ffff:ffff:ffff:ffff,DE +2a00:f440::,2a00:f440:ffff:ffff:ffff:ffff:ffff:ffff,RU +2a00:f460::,2a00:f460:ffff:ffff:ffff:ffff:ffff:ffff,RU +2a00:f480::,2a00:f480:ffff:ffff:ffff:ffff:ffff:ffff,RU +2a00:f4a0::,2a00:f4a0:ffff:ffff:ffff:ffff:ffff:ffff,LV +2a00:f4c0::,2a00:f4c0:ffff:ffff:ffff:ffff:ffff:ffff,SE +2a00:f4e0::,2a00:f4e0:ffff:ffff:ffff:ffff:ffff:ffff,GB 2a00:f500::,2a00:f507:ffff:ffff:ffff:ffff:ffff:ffff,LT 2a00:f520::,2a00:f520:ffff:ffff:ffff:ffff:ffff:ffff,DE 2a00:f540::,2a00:f540:ffff:ffff:ffff:ffff:ffff:ffff,GB @@ -11260,42 +10473,12 @@ 2a00:ffa0::,2a00:ffa0:ffff:ffff:ffff:ffff:ffff:ffff,NL 2a00:ffc0::,2a00:ffc0:ffff:ffff:ffff:ffff:ffff:ffff,PL 2a00:ffe0::,2a00:ffe0:ffff:ffff:ffff:ffff:ffff:ffff,GB -2a01::,2a01::ffff:ffff:ffff:ffff:ffff:ffff,FR -2a01:8::,2a01:8:ffff:ffff:ffff:ffff:ffff:ffff,PT -2a01:10::,2a01:10:ffff:ffff:ffff:ffff:ffff:ffff,PT -2a01:18::,2a01:18:ffff:ffff:ffff:ffff:ffff:ffff,FI -2a01:20::,2a01:20:ffff:ffff:ffff:ffff:ffff:ffff,FR -2a01:28::,2a01:28:ffff:ffff:ffff:ffff:ffff:ffff,CZ -2a01:30::,2a01:30:ffff:ffff:ffff:ffff:ffff:ffff,DE -2a01:38::,2a01:38:ffff:ffff:ffff:ffff:ffff:ffff,BE -2a01:40::,2a01:40:ffff:ffff:ffff:ffff:ffff:ffff,GB -2a01:48::,2a01:48:ffff:ffff:ffff:ffff:ffff:ffff,ES -2a01:50::,2a01:50:ffff:ffff:ffff:ffff:ffff:ffff,GB -2a01:58::,2a01:58:ffff:ffff:ffff:ffff:ffff:ffff,SE -2a01:68::,2a01:68:ffff:ffff:ffff:ffff:ffff:ffff,FR -2a01:70::,2a01:70:ffff:ffff:ffff:ffff:ffff:ffff,GB -2a01:78::,2a01:78:ffff:ffff:ffff:ffff:ffff:ffff,DE -2a01:80::,2a01:80:ffff:ffff:ffff:ffff:ffff:ffff,EE -2a01:88::,2a01:88:ffff:ffff:ffff:ffff:ffff:ffff,IE -2a01:90::,2a01:90:ffff:ffff:ffff:ffff:ffff:ffff,GB -2a01:98::,2a01:98:ffff:ffff:ffff:ffff:ffff:ffff,GB -2a01:a0::,2a01:a0:ffff:ffff:ffff:ffff:ffff:ffff,MT -2a01:a8::,2a01:a8:ffff:ffff:ffff:ffff:ffff:ffff,IE -2a01:b0::,2a01:b1:ffff:ffff:ffff:ffff:ffff:ffff,BE -2a01:b8::,2a01:b8:ffff:ffff:ffff:ffff:ffff:ffff,VA -2a01:c0::,2a01:c0:ffff:ffff:ffff:ffff:ffff:ffff,GB -2a01:c8::,2a01:c8:ffff:ffff:ffff:ffff:ffff:ffff,FR -2a01:d0::,2a01:d0:ffff:ffff:ffff:ffff:ffff:ffff,UA -2a01:d8::,2a01:d8:ffff:ffff:ffff:ffff:ffff:ffff,RU -2a01:e0::,2a01:e0:ffff:ffff:ffff:ffff:ffff:ffff,SK -2a01:e8::,2a01:e8:ffff:ffff:ffff:ffff:ffff:ffff,SE -2a01:f0::,2a01:f0:ffff:ffff:ffff:ffff:ffff:ffff,ES -2a01:f8::,2a01:f8:ffff:ffff:ffff:ffff:ffff:ffff,AT +2a01::,2a01:ff:ffff:ffff:ffff:ffff:ffff:ffff,IE 2a01:100::,2a01:100:ffff:ffff:ffff:ffff:ffff:ffff,AT 2a01:108::,2a01:108:ffff:ffff:ffff:ffff:ffff:ffff,SK 2a01:120::,2a01:120:ffff:ffff:ffff:ffff:ffff:ffff,DE 2a01:130::,2a01:130:ffff:ffff:ffff:ffff:ffff:ffff,DE -2a01:138::,2a01:138:ffff:ffff:ffff:ffff:ffff:ffff,DE +2a01:138::,2a01:13f:ffff:ffff:ffff:ffff:ffff:ffff,DE 2a01:140::,2a01:140:ffff:ffff:ffff:ffff:ffff:ffff,GB 2a01:148::,2a01:148:ffff:ffff:ffff:ffff:ffff:ffff,IE 2a01:150::,2a01:150:ffff:ffff:ffff:ffff:ffff:ffff,GB @@ -11454,7 +10637,6 @@ 2a01:6a0::,2a01:6a0:ffff:ffff:ffff:ffff:ffff:ffff,IT 2a01:6a8::,2a01:6a8:ffff:ffff:ffff:ffff:ffff:ffff,CH 2a01:6b0::,2a01:6b0:ffff:ffff:ffff:ffff:ffff:ffff,UA -2a01:6b8::,2a01:6b8:ffff:ffff:ffff:ffff:ffff:ffff,FR 2a01:6c0::,2a01:6c0:ffff:ffff:ffff:ffff:ffff:ffff,DE 2a01:6c8::,2a01:6c8:ffff:ffff:ffff:ffff:ffff:ffff,GB 2a01:6d0::,2a01:6d0:ffff:ffff:ffff:ffff:ffff:ffff,SE @@ -11674,7 +10856,7 @@ 2a01:56e0::,2a01:56e0:ffff:ffff:ffff:ffff:ffff:ffff,PL 2a01:5700::,2a01:5700:ffff:ffff:ffff:ffff:ffff:ffff,SE 2a01:5720::,2a01:5720:ffff:ffff:ffff:ffff:ffff:ffff,IT -2a01:5740::,2a01:5740:ffff:ffff:ffff:ffff:ffff:ffff,AT +2a01:5740::,2a01:5741:ffff:ffff:ffff:ffff:ffff:ffff,AT 2a01:5760::,2a01:5760:ffff:ffff:ffff:ffff:ffff:ffff,IT 2a01:5780::,2a01:5780:ffff:ffff:ffff:ffff:ffff:ffff,ES 2a01:57a0::,2a01:57a0:ffff:ffff:ffff:ffff:ffff:ffff,ES @@ -12120,7 +11302,7 @@ 2a01:90a0::,2a01:90a0:ffff:ffff:ffff:ffff:ffff:ffff,NL 2a01:90c0::,2a01:90c0:ffff:ffff:ffff:ffff:ffff:ffff,GB 2a01:90e0::,2a01:90e0:ffff:ffff:ffff:ffff:ffff:ffff,AT -2a01:9100::,2a01:9100:ffff:ffff:ffff:ffff:ffff:ffff,BE +2a01:9100::,2a01:9107:ffff:ffff:ffff:ffff:ffff:ffff,BE 2a01:9120::,2a01:9120:ffff:ffff:ffff:ffff:ffff:ffff,IT 2a01:9140::,2a01:9140:ffff:ffff:ffff:ffff:ffff:ffff,ES 2a01:9160::,2a01:9160:ffff:ffff:ffff:ffff:ffff:ffff,RU @@ -12494,7 +11676,7 @@ 2a02:68::,2a02:68:ffff:ffff:ffff:ffff:ffff:ffff,GB 2a02:70::,2a02:70:ffff:ffff:ffff:ffff:ffff:ffff,UA 2a02:78::,2a02:78:ffff:ffff:ffff:ffff:ffff:ffff,PL -2a02:80::,2a02:80:ffff:ffff:ffff:ffff:ffff:ffff,SE +2a02:80::,2a02:87:ffff:ffff:ffff:ffff:ffff:ffff,SE 2a02:88::,2a02:88:ffff:ffff:ffff:ffff:ffff:ffff,EE 2a02:90::,2a02:97:ffff:ffff:ffff:ffff:ffff:ffff,FI 2a02:98::,2a02:98:ffff:ffff:ffff:ffff:ffff:ffff,GB @@ -13329,11 +12511,7 @@ 2a02:2dc0::,2a02:2dc0:ffff:ffff:ffff:ffff:ffff:ffff,DE 2a02:2de0::,2a02:2de0:ffff:ffff:ffff:ffff:ffff:ffff,PL 2a02:2e00::,2a02:2e1f:ffff:ffff:ffff:ffff:ffff:ffff,ES -2a02:2f00::,2a02:2f0f:ffff:ffff:ffff:ffff:ffff:ffff,RO -2a02:2f80::,2a02:2f80:ffff:ffff:ffff:ffff:ffff:ffff,ES -2a02:2fa0::,2a02:2fa0:ffff:ffff:ffff:ffff:ffff:ffff,NL -2a02:2fc0::,2a02:2fc0:ffff:ffff:ffff:ffff:ffff:ffff,DE -2a02:2fe0::,2a02:2fe0:ffff:ffff:ffff:ffff:ffff:ffff,IT +2a02:2f00::,2a02:2fff:ffff:ffff:ffff:ffff:ffff:ffff,RO 2a02:3000::,2a02:31ff:ffff:ffff:ffff:ffff:ffff:ffff,DE 2a02:4000::,2a02:4000:ffff:ffff:ffff:ffff:ffff:ffff,CZ 2a02:4020::,2a02:4020:ffff:ffff:ffff:ffff:ffff:ffff,SE @@ -13459,7 +12637,7 @@ 2a02:5060::,2a02:5060:ffff:ffff:ffff:ffff:ffff:ffff,NL 2a02:5080::,2a02:5080:ffff:ffff:ffff:ffff:ffff:ffff,IL 2a02:50a0::,2a02:50a0:ffff:ffff:ffff:ffff:ffff:ffff,PL -2a02:50c0::,2a02:50c0:ffff:ffff:ffff:ffff:ffff:ffff,KZ +2a02:50c0::,2a02:50c7:ffff:ffff:ffff:ffff:ffff:ffff,KZ 2a02:50e0::,2a02:50e0:ffff:ffff:ffff:ffff:ffff:ffff,NL 2a02:5100::,2a02:5100:ffff:ffff:ffff:ffff:ffff:ffff,ES 2a02:5120::,2a02:5120:ffff:ffff:ffff:ffff:ffff:ffff,SE @@ -13673,7 +12851,7 @@ 2a02:6c40::,2a02:6c40:ffff:ffff:ffff:ffff:ffff:ffff,FR 2a02:6c60::,2a02:6c60:ffff:ffff:ffff:ffff:ffff:ffff,UA 2a02:6c80::,2a02:6c80:ffff:ffff:ffff:ffff:ffff:ffff,RU -2a02:6ca0::,2a02:6ca0:ffff:ffff:ffff:ffff:ffff:ffff,SK +2a02:6ca0::,2a02:6ca7:ffff:ffff:ffff:ffff:ffff:ffff,SK 2a02:6cc0::,2a02:6cc0:ffff:ffff:ffff:ffff:ffff:ffff,RU 2a02:6ce0::,2a02:6ce0:ffff:ffff:ffff:ffff:ffff:ffff,IT 2a02:6d00::,2a02:6d00:ffff:ffff:ffff:ffff:ffff:ffff,RU @@ -13687,7 +12865,6 @@ 2a02:6e00::,2a02:6e00:ffff:ffff:ffff:ffff:ffff:ffff,RO 2a02:6e20::,2a02:6e20:ffff:ffff:ffff:ffff:ffff:ffff,DE 2a02:6e60::,2a02:6e60:ffff:ffff:ffff:ffff:ffff:ffff,RU -2a02:6e80::,2a02:6e80:ffff:ffff:ffff:ffff:ffff:ffff,FR 2a02:6ea0::,2a02:6ea0:ffff:ffff:ffff:ffff:ffff:ffff,GB 2a02:6ec0::,2a02:6ec0:ffff:ffff:ffff:ffff:ffff:ffff,IR 2a02:6ee0::,2a02:6ee0:ffff:ffff:ffff:ffff:ffff:ffff,PL @@ -13822,10 +12999,10 @@ 2a02:7fa0::,2a02:7fa0:ffff:ffff:ffff:ffff:ffff:ffff,FR 2a02:7fc0::,2a02:7fc0:ffff:ffff:ffff:ffff:ffff:ffff,RU 2a02:7fe0::,2a02:7fe0:ffff:ffff:ffff:ffff:ffff:ffff,IS -2a02:8000::,2a02:811f:ffff:ffff:ffff:ffff:ffff:ffff,DE -2a02:8200::,2a02:821f:ffff:ffff:ffff:ffff:ffff:ffff,DE -2a02:8300::,2a02:83ff:ffff:ffff:ffff:ffff:ffff:ffff,AT -2a02:8400::,2a02:84ff:ffff:ffff:ffff:ffff:ffff:ffff,FR +2a02:8000::,2a02:821f:ffff:ffff:ffff:ffff:ffff:ffff,DE +2a02:8300::,2a02:830f:ffff:ffff:ffff:ffff:ffff:ffff,CZ +2a02:8380::,2a02:838f:ffff:ffff:ffff:ffff:ffff:ffff,AT +2a02:8400::,2a02:847f:ffff:ffff:ffff:ffff:ffff:ffff,FR 2a02:8800::,2a02:88ff:ffff:ffff:ffff:ffff:ffff:ffff,GB 2a02:9000::,2a02:91ff:ffff:ffff:ffff:ffff:ffff:ffff,ES 2a02:a000::,2a02:a0ff:ffff:ffff:ffff:ffff:ffff:ffff,BE @@ -14142,115 +13319,221 @@ 2a03:340::,2a03:340:ffff:ffff:ffff:ffff:ffff:ffff,RU 2a03:360::,2a03:360:ffff:ffff:ffff:ffff:ffff:ffff,FR 2a03:380::,2a03:380:ffff:ffff:ffff:ffff:ffff:ffff,CH +2a03:3a0::,2a03:3a0:ffff:ffff:ffff:ffff:ffff:ffff,DE 2a03:3c0::,2a03:3c0:ffff:ffff:ffff:ffff:ffff:ffff,TR +2a03:3e0::,2a03:3e0:ffff:ffff:ffff:ffff:ffff:ffff,RS 2a03:400::,2a03:400:ffff:ffff:ffff:ffff:ffff:ffff,TR +2a03:420::,2a03:420:ffff:ffff:ffff:ffff:ffff:ffff,UA 2a03:440::,2a03:440:ffff:ffff:ffff:ffff:ffff:ffff,ES +2a03:460::,2a03:460:ffff:ffff:ffff:ffff:ffff:ffff,US 2a03:480::,2a03:480:ffff:ffff:ffff:ffff:ffff:ffff,GB +2a03:4a0::,2a03:4a0:ffff:ffff:ffff:ffff:ffff:ffff,NO 2a03:4c0::,2a03:4c0:ffff:ffff:ffff:ffff:ffff:ffff,IE +2a03:4e0::,2a03:4e0:ffff:ffff:ffff:ffff:ffff:ffff,GB 2a03:500::,2a03:500:ffff:ffff:ffff:ffff:ffff:ffff,RU +2a03:520::,2a03:520:ffff:ffff:ffff:ffff:ffff:ffff,NO 2a03:540::,2a03:540:ffff:ffff:ffff:ffff:ffff:ffff,UZ +2a03:560::,2a03:560:ffff:ffff:ffff:ffff:ffff:ffff,DE 2a03:580::,2a03:580:ffff:ffff:ffff:ffff:ffff:ffff,CH +2a03:5a0::,2a03:5a0:ffff:ffff:ffff:ffff:ffff:ffff,GB 2a03:5c0::,2a03:5c0:ffff:ffff:ffff:ffff:ffff:ffff,FR +2a03:5e0::,2a03:5e0:ffff:ffff:ffff:ffff:ffff:ffff,AM 2a03:600::,2a03:600:ffff:ffff:ffff:ffff:ffff:ffff,RU +2a03:620::,2a03:620:ffff:ffff:ffff:ffff:ffff:ffff,RS 2a03:640::,2a03:640:ffff:ffff:ffff:ffff:ffff:ffff,IT +2a03:660::,2a03:660:ffff:ffff:ffff:ffff:ffff:ffff,NL 2a03:680::,2a03:680:ffff:ffff:ffff:ffff:ffff:ffff,ES +2a03:6a0::,2a03:6a0:ffff:ffff:ffff:ffff:ffff:ffff,LV 2a03:6c0::,2a03:6c0:ffff:ffff:ffff:ffff:ffff:ffff,RU +2a03:6e0::,2a03:6e0:ffff:ffff:ffff:ffff:ffff:ffff,SK 2a03:700::,2a03:700:ffff:ffff:ffff:ffff:ffff:ffff,RU +2a03:720::,2a03:720:ffff:ffff:ffff:ffff:ffff:ffff,RU 2a03:740::,2a03:740:ffff:ffff:ffff:ffff:ffff:ffff,FR +2a03:760::,2a03:760:ffff:ffff:ffff:ffff:ffff:ffff,FR 2a03:780::,2a03:780:ffff:ffff:ffff:ffff:ffff:ffff,SE +2a03:7a0::,2a03:7a0:ffff:ffff:ffff:ffff:ffff:ffff,ME 2a03:7c0::,2a03:7c0:ffff:ffff:ffff:ffff:ffff:ffff,PS +2a03:7e0::,2a03:7e0:ffff:ffff:ffff:ffff:ffff:ffff,NL 2a03:800::,2a03:800:ffff:ffff:ffff:ffff:ffff:ffff,FR +2a03:820::,2a03:820:ffff:ffff:ffff:ffff:ffff:ffff,IT 2a03:840::,2a03:840:ffff:ffff:ffff:ffff:ffff:ffff,CZ +2a03:860::,2a03:860:ffff:ffff:ffff:ffff:ffff:ffff,DE 2a03:880::,2a03:880:ffff:ffff:ffff:ffff:ffff:ffff,FI +2a03:8a0::,2a03:8a0:ffff:ffff:ffff:ffff:ffff:ffff,SE 2a03:8c0::,2a03:8c0:ffff:ffff:ffff:ffff:ffff:ffff,AZ +2a03:8e0::,2a03:8e0:ffff:ffff:ffff:ffff:ffff:ffff,UA 2a03:900::,2a03:900:ffff:ffff:ffff:ffff:ffff:ffff,SE +2a03:920::,2a03:920:ffff:ffff:ffff:ffff:ffff:ffff,DE 2a03:940::,2a03:940:ffff:ffff:ffff:ffff:ffff:ffff,AT +2a03:960::,2a03:960:ffff:ffff:ffff:ffff:ffff:ffff,IT 2a03:980::,2a03:980:ffff:ffff:ffff:ffff:ffff:ffff,IL +2a03:9a0::,2a03:9a0:ffff:ffff:ffff:ffff:ffff:ffff,MT 2a03:9c0::,2a03:9c0:ffff:ffff:ffff:ffff:ffff:ffff,IT +2a03:9e0::,2a03:9e0:ffff:ffff:ffff:ffff:ffff:ffff,HU 2a03:a00::,2a03:a00:ffff:ffff:ffff:ffff:ffff:ffff,NL +2a03:a20::,2a03:a20:ffff:ffff:ffff:ffff:ffff:ffff,IT 2a03:a40::,2a03:a40:ffff:ffff:ffff:ffff:ffff:ffff,PL +2a03:a60::,2a03:a60:ffff:ffff:ffff:ffff:ffff:ffff,IT 2a03:a80::,2a03:a80:ffff:ffff:ffff:ffff:ffff:ffff,MD +2a03:aa0::,2a03:aa0:ffff:ffff:ffff:ffff:ffff:ffff,RU 2a03:ac0::,2a03:ac0:ffff:ffff:ffff:ffff:ffff:ffff,IE +2a03:ae0::,2a03:ae0:ffff:ffff:ffff:ffff:ffff:ffff,RU 2a03:b00::,2a03:b00:ffff:ffff:ffff:ffff:ffff:ffff,FR 2a03:b40::,2a03:b40:ffff:ffff:ffff:ffff:ffff:ffff,ES +2a03:b60::,2a03:b60:ffff:ffff:ffff:ffff:ffff:ffff,RU 2a03:b80::,2a03:b80:ffff:ffff:ffff:ffff:ffff:ffff,FI +2a03:ba0::,2a03:ba0:ffff:ffff:ffff:ffff:ffff:ffff,FR 2a03:bc0::,2a03:bc7:ffff:ffff:ffff:ffff:ffff:ffff,GB 2a03:c00::,2a03:c00:ffff:ffff:ffff:ffff:ffff:ffff,TR +2a03:c20::,2a03:c20:ffff:ffff:ffff:ffff:ffff:ffff,CZ 2a03:c40::,2a03:c40:ffff:ffff:ffff:ffff:ffff:ffff,PS +2a03:c60::,2a03:c60:ffff:ffff:ffff:ffff:ffff:ffff,IT 2a03:c80::,2a03:c80:ffff:ffff:ffff:ffff:ffff:ffff,FI +2a03:ca0::,2a03:ca0:ffff:ffff:ffff:ffff:ffff:ffff,CZ 2a03:cc0::,2a03:cc0:ffff:ffff:ffff:ffff:ffff:ffff,RO +2a03:ce0::,2a03:ce0:ffff:ffff:ffff:ffff:ffff:ffff,RU 2a03:d00::,2a03:d00:ffff:ffff:ffff:ffff:ffff:ffff,DK +2a03:d20::,2a03:d20:ffff:ffff:ffff:ffff:ffff:ffff,IR 2a03:d40::,2a03:d40:ffff:ffff:ffff:ffff:ffff:ffff,IT +2a03:d60::,2a03:d60:ffff:ffff:ffff:ffff:ffff:ffff,PL 2a03:d80::,2a03:d80:ffff:ffff:ffff:ffff:ffff:ffff,IT +2a03:da0::,2a03:da0:ffff:ffff:ffff:ffff:ffff:ffff,PT 2a03:dc0::,2a03:dc0:ffff:ffff:ffff:ffff:ffff:ffff,NL +2a03:de0::,2a03:de0:ffff:ffff:ffff:ffff:ffff:ffff,BE 2a03:e00::,2a03:e00:ffff:ffff:ffff:ffff:ffff:ffff,CZ +2a03:e20::,2a03:e20:ffff:ffff:ffff:ffff:ffff:ffff,RU 2a03:e40::,2a03:e40:ffff:ffff:ffff:ffff:ffff:ffff,GR +2a03:e60::,2a03:e60:ffff:ffff:ffff:ffff:ffff:ffff,RU 2a03:e80::,2a03:e80:ffff:ffff:ffff:ffff:ffff:ffff,FR +2a03:ea0::,2a03:ea0:ffff:ffff:ffff:ffff:ffff:ffff,IE 2a03:ec0::,2a03:ec0:ffff:ffff:ffff:ffff:ffff:ffff,RU +2a03:ee0::,2a03:ee0:ffff:ffff:ffff:ffff:ffff:ffff,DE 2a03:f00::,2a03:f07:ffff:ffff:ffff:ffff:ffff:ffff,DE 2a03:f40::,2a03:f40:ffff:ffff:ffff:ffff:ffff:ffff,CH +2a03:f60::,2a03:f60:ffff:ffff:ffff:ffff:ffff:ffff,NO 2a03:f80::,2a03:f87:ffff:ffff:ffff:ffff:ffff:ffff,AT 2a03:fc0::,2a03:fc0:ffff:ffff:ffff:ffff:ffff:ffff,DE +2a03:fe0::,2a03:fe0:ffff:ffff:ffff:ffff:ffff:ffff,BG 2a03:1000::,2a03:1000:ffff:ffff:ffff:ffff:ffff:ffff,AT +2a03:1020::,2a03:1020:ffff:ffff:ffff:ffff:ffff:ffff,UZ 2a03:1040::,2a03:1040:ffff:ffff:ffff:ffff:ffff:ffff,FR +2a03:1060::,2a03:1060:ffff:ffff:ffff:ffff:ffff:ffff,IT 2a03:1080::,2a03:1080:ffff:ffff:ffff:ffff:ffff:ffff,AM +2a03:10a0::,2a03:10a0:ffff:ffff:ffff:ffff:ffff:ffff,CZ 2a03:10c0::,2a03:10c3:ffff:ffff:ffff:ffff:ffff:ffff,NL +2a03:10e0::,2a03:10e0:ffff:ffff:ffff:ffff:ffff:ffff,IL 2a03:1100::,2a03:1100:ffff:ffff:ffff:ffff:ffff:ffff,DK +2a03:1120::,2a03:1120:ffff:ffff:ffff:ffff:ffff:ffff,GB 2a03:1140::,2a03:1140:ffff:ffff:ffff:ffff:ffff:ffff,IT +2a03:1160::,2a03:1160:ffff:ffff:ffff:ffff:ffff:ffff,CZ 2a03:1180::,2a03:1180:ffff:ffff:ffff:ffff:ffff:ffff,CZ +2a03:11a0::,2a03:11a0:ffff:ffff:ffff:ffff:ffff:ffff,OM 2a03:11c0::,2a03:11c0:ffff:ffff:ffff:ffff:ffff:ffff,CZ +2a03:11e0::,2a03:11e0:ffff:ffff:ffff:ffff:ffff:ffff,SA 2a03:1200::,2a03:1200:ffff:ffff:ffff:ffff:ffff:ffff,ES +2a03:1220::,2a03:1220:ffff:ffff:ffff:ffff:ffff:ffff,SE 2a03:1240::,2a03:1240:ffff:ffff:ffff:ffff:ffff:ffff,UA +2a03:1260::,2a03:1260:ffff:ffff:ffff:ffff:ffff:ffff,RU 2a03:1280::,2a03:1280:ffff:ffff:ffff:ffff:ffff:ffff,PL +2a03:12a0::,2a03:12a0:ffff:ffff:ffff:ffff:ffff:ffff,BE 2a03:12c0::,2a03:12c0:ffff:ffff:ffff:ffff:ffff:ffff,NO +2a03:12e0::,2a03:12e0:ffff:ffff:ffff:ffff:ffff:ffff,AE 2a03:1300::,2a03:1300:ffff:ffff:ffff:ffff:ffff:ffff,CY +2a03:1320::,2a03:1320:ffff:ffff:ffff:ffff:ffff:ffff,TR 2a03:1340::,2a03:1340:ffff:ffff:ffff:ffff:ffff:ffff,NL +2a03:1360::,2a03:1360:ffff:ffff:ffff:ffff:ffff:ffff,CH 2a03:1380::,2a03:1380:ffff:ffff:ffff:ffff:ffff:ffff,PL +2a03:13a0::,2a03:13a0:ffff:ffff:ffff:ffff:ffff:ffff,DE 2a03:13c0::,2a03:13c0:ffff:ffff:ffff:ffff:ffff:ffff,RU +2a03:13e0::,2a03:13e0:ffff:ffff:ffff:ffff:ffff:ffff,CH 2a03:1400::,2a03:1400:ffff:ffff:ffff:ffff:ffff:ffff,NO +2a03:1420::,2a03:1420:ffff:ffff:ffff:ffff:ffff:ffff,AT 2a03:1440::,2a03:1440:ffff:ffff:ffff:ffff:ffff:ffff,DE +2a03:1460::,2a03:1460:ffff:ffff:ffff:ffff:ffff:ffff,NL 2a03:1480::,2a03:1480:ffff:ffff:ffff:ffff:ffff:ffff,DE +2a03:14a0::,2a03:14a0:ffff:ffff:ffff:ffff:ffff:ffff,FR 2a03:14c0::,2a03:14c0:ffff:ffff:ffff:ffff:ffff:ffff,RU +2a03:14e0::,2a03:14e0:ffff:ffff:ffff:ffff:ffff:ffff,GB 2a03:1500::,2a03:1500:ffff:ffff:ffff:ffff:ffff:ffff,DK +2a03:1520::,2a03:1520:ffff:ffff:ffff:ffff:ffff:ffff,TR 2a03:1540::,2a03:1540:ffff:ffff:ffff:ffff:ffff:ffff,DE +2a03:1560::,2a03:1560:ffff:ffff:ffff:ffff:ffff:ffff,RS 2a03:1580::,2a03:1580:ffff:ffff:ffff:ffff:ffff:ffff,BE +2a03:15a0::,2a03:15a0:ffff:ffff:ffff:ffff:ffff:ffff,HK 2a03:15c0::,2a03:15c0:ffff:ffff:ffff:ffff:ffff:ffff,GB +2a03:15e0::,2a03:15e0:ffff:ffff:ffff:ffff:ffff:ffff,BE 2a03:1600::,2a03:1600:ffff:ffff:ffff:ffff:ffff:ffff,CZ +2a03:1620::,2a03:1620:ffff:ffff:ffff:ffff:ffff:ffff,GB 2a03:1640::,2a03:1640:ffff:ffff:ffff:ffff:ffff:ffff,RU +2a03:1660::,2a03:1660:ffff:ffff:ffff:ffff:ffff:ffff,KW 2a03:1680::,2a03:1680:ffff:ffff:ffff:ffff:ffff:ffff,IT +2a03:16a0::,2a03:16a0:ffff:ffff:ffff:ffff:ffff:ffff,ME 2a03:16c0::,2a03:16c0:ffff:ffff:ffff:ffff:ffff:ffff,FR +2a03:16e0::,2a03:16e0:ffff:ffff:ffff:ffff:ffff:ffff,DE 2a03:1700::,2a03:1707:ffff:ffff:ffff:ffff:ffff:ffff,IT 2a03:1740::,2a03:1740:ffff:ffff:ffff:ffff:ffff:ffff,RU +2a03:1760::,2a03:1760:ffff:ffff:ffff:ffff:ffff:ffff,GB 2a03:1780::,2a03:1780:ffff:ffff:ffff:ffff:ffff:ffff,FR +2a03:17a0::,2a03:17a0:ffff:ffff:ffff:ffff:ffff:ffff,GB 2a03:17c0::,2a03:17c0:ffff:ffff:ffff:ffff:ffff:ffff,RU +2a03:17e0::,2a03:17e0:ffff:ffff:ffff:ffff:ffff:ffff,CH 2a03:1800::,2a03:1800:ffff:ffff:ffff:ffff:ffff:ffff,RU +2a03:1820::,2a03:1820:ffff:ffff:ffff:ffff:ffff:ffff,DE 2a03:1840::,2a03:1840:ffff:ffff:ffff:ffff:ffff:ffff,CZ +2a03:1860::,2a03:1860:ffff:ffff:ffff:ffff:ffff:ffff,GB 2a03:1880::,2a03:1880:ffff:ffff:ffff:ffff:ffff:ffff,CH +2a03:18a0::,2a03:18a0:ffff:ffff:ffff:ffff:ffff:ffff,RU 2a03:18c0::,2a03:18c0:ffff:ffff:ffff:ffff:ffff:ffff,DE +2a03:18e0::,2a03:18e0:ffff:ffff:ffff:ffff:ffff:ffff,FR 2a03:1900::,2a03:1900:ffff:ffff:ffff:ffff:ffff:ffff,AM +2a03:1920::,2a03:1920:ffff:ffff:ffff:ffff:ffff:ffff,IT 2a03:1940::,2a03:1940:ffff:ffff:ffff:ffff:ffff:ffff,GB +2a03:1960::,2a03:1960:ffff:ffff:ffff:ffff:ffff:ffff,LT 2a03:1980::,2a03:1980:ffff:ffff:ffff:ffff:ffff:ffff,BE +2a03:19a0::,2a03:19a0:ffff:ffff:ffff:ffff:ffff:ffff,CH +2a03:19e0::,2a03:19e0:ffff:ffff:ffff:ffff:ffff:ffff,ES 2a03:1a00::,2a03:1a00:ffff:ffff:ffff:ffff:ffff:ffff,NL +2a03:1a20::,2a03:1a20:ffff:ffff:ffff:ffff:ffff:ffff,BE 2a03:1a40::,2a03:1a40:ffff:ffff:ffff:ffff:ffff:ffff,DE +2a03:1a60::,2a03:1a60:ffff:ffff:ffff:ffff:ffff:ffff,FR 2a03:1a80::,2a03:1a80:ffff:ffff:ffff:ffff:ffff:ffff,AT +2a03:1aa0::,2a03:1aa0:ffff:ffff:ffff:ffff:ffff:ffff,CA 2a03:1ac0::,2a03:1ac0:ffff:ffff:ffff:ffff:ffff:ffff,RU +2a03:1ae0::,2a03:1ae0:ffff:ffff:ffff:ffff:ffff:ffff,FR 2a03:1b00::,2a03:1b00:ffff:ffff:ffff:ffff:ffff:ffff,RU +2a03:1b20::,2a03:1b20:ffff:ffff:ffff:ffff:ffff:ffff,SE 2a03:1b40::,2a03:1b40:ffff:ffff:ffff:ffff:ffff:ffff,RS +2a03:1b60::,2a03:1b60:ffff:ffff:ffff:ffff:ffff:ffff,GB 2a03:1b80::,2a03:1b80:ffff:ffff:ffff:ffff:ffff:ffff,GB +2a03:1ba0::,2a03:1ba0:ffff:ffff:ffff:ffff:ffff:ffff,GB 2a03:1bc0::,2a03:1bc0:ffff:ffff:ffff:ffff:ffff:ffff,FR +2a03:1be0::,2a03:1be0:ffff:ffff:ffff:ffff:ffff:ffff,GB 2a03:1c00::,2a03:1c00:ffff:ffff:ffff:ffff:ffff:ffff,BH +2a03:1c20::,2a03:1c20:ffff:ffff:ffff:ffff:ffff:ffff,SE 2a03:1c40::,2a03:1c40:ffff:ffff:ffff:ffff:ffff:ffff,ES +2a03:1c60::,2a03:1c60:ffff:ffff:ffff:ffff:ffff:ffff,SE 2a03:1c80::,2a03:1c80:ffff:ffff:ffff:ffff:ffff:ffff,NL +2a03:1ca0::,2a03:1ca0:ffff:ffff:ffff:ffff:ffff:ffff,NO 2a03:1cc0::,2a03:1cc0:ffff:ffff:ffff:ffff:ffff:ffff,ES +2a03:1ce0::,2a03:1ce0:ffff:ffff:ffff:ffff:ffff:ffff,SE 2a03:1d00::,2a03:1d00:ffff:ffff:ffff:ffff:ffff:ffff,FR +2a03:1d20::,2a03:1d20:ffff:ffff:ffff:ffff:ffff:ffff,ES 2a03:1d40::,2a03:1d40:ffff:ffff:ffff:ffff:ffff:ffff,GB +2a03:1d60::,2a03:1d60:ffff:ffff:ffff:ffff:ffff:ffff,DE 2a03:1d80::,2a03:1d80:ffff:ffff:ffff:ffff:ffff:ffff,SK +2a03:1da0::,2a03:1da0:ffff:ffff:ffff:ffff:ffff:ffff,DE 2a03:1dc0::,2a03:1dc0:ffff:ffff:ffff:ffff:ffff:ffff,FR +2a03:1de0::,2a03:1de0:ffff:ffff:ffff:ffff:ffff:ffff,RU 2a03:1e00::,2a03:1e00:ffff:ffff:ffff:ffff:ffff:ffff,DE +2a03:1e20::,2a03:1e20:ffff:ffff:ffff:ffff:ffff:ffff,CZ 2a03:1e40::,2a03:1e40:ffff:ffff:ffff:ffff:ffff:ffff,SE +2a03:1e60::,2a03:1e60:ffff:ffff:ffff:ffff:ffff:ffff,GB 2a03:1e80::,2a03:1e80:ffff:ffff:ffff:ffff:ffff:ffff,DE +2a03:1ea0::,2a03:1ea0:ffff:ffff:ffff:ffff:ffff:ffff,GB 2a03:1ec0::,2a03:1ec0:ffff:ffff:ffff:ffff:ffff:ffff,BG +2a03:1ee0::,2a03:1ee0:ffff:ffff:ffff:ffff:ffff:ffff,RU 2a03:1f00::,2a03:1f00:ffff:ffff:ffff:ffff:ffff:ffff,PL +2a03:1f20::,2a03:1f20:ffff:ffff:ffff:ffff:ffff:ffff,GB 2a03:1f40::,2a03:1f40:ffff:ffff:ffff:ffff:ffff:ffff,IE 2a03:1f80::,2a03:1f80:ffff:ffff:ffff:ffff:ffff:ffff,CZ 2a03:2000::,2a03:2000:ffff:ffff:ffff:ffff:ffff:ffff,GB @@ -14335,7 +13618,7 @@ 2a03:3480::,2a03:3480:ffff:ffff:ffff:ffff:ffff:ffff,RU 2a03:34c0::,2a03:34c0:ffff:ffff:ffff:ffff:ffff:ffff,GB 2a03:3500::,2a03:3500:ffff:ffff:ffff:ffff:ffff:ffff,DE -2a03:3540::,2a03:3540:ffff:ffff:ffff:ffff:ffff:ffff,FR +2a03:3540::,2a03:3547:ffff:ffff:ffff:ffff:ffff:ffff,FR 2a03:3580::,2a03:3580:ffff:ffff:ffff:ffff:ffff:ffff,GE 2a03:35c0::,2a03:35c0:ffff:ffff:ffff:ffff:ffff:ffff,DE 2a03:3600::,2a03:3600:ffff:ffff:ffff:ffff:ffff:ffff,BE @@ -15621,7 +14904,7 @@ 2a04:8140::,2a04:8147:ffff:ffff:ffff:ffff:ffff:ffff,NL 2a04:8180::,2a04:8181:ffff:ffff:ffff:ffff:ffff:ffff,SY 2a04:8190::,2a04:8191:ffff:ffff:ffff:ffff:ffff:ffff,NL -2a04:81a0::,2a04:81a3:ffff:ffff:ffff:ffff:ffff:ffff,AT +2a04:81a0::,2a04:81a7:ffff:ffff:ffff:ffff:ffff:ffff,AT 2a04:81c0::,2a04:81c7:ffff:ffff:ffff:ffff:ffff:ffff,LU 2a04:8200::,2a04:8207:ffff:ffff:ffff:ffff:ffff:ffff,IT 2a04:8240::,2a04:8247:ffff:ffff:ffff:ffff:ffff:ffff,RU @@ -15877,7 +15160,6 @@ 2a04:c1c0::,2a04:c1c7:ffff:ffff:ffff:ffff:ffff:ffff,IT 2a04:c200::,2a04:c207:ffff:ffff:ffff:ffff:ffff:ffff,GE 2a04:c240::,2a04:c247:ffff:ffff:ffff:ffff:ffff:ffff,IT -2a04:c280::,2a04:c287:ffff:ffff:ffff:ffff:ffff:ffff,AT 2a04:c2c0::,2a04:c2c7:ffff:ffff:ffff:ffff:ffff:ffff,SA 2a04:c300::,2a04:c307:ffff:ffff:ffff:ffff:ffff:ffff,ES 2a04:c340::,2a04:c347:ffff:ffff:ffff:ffff:ffff:ffff,RU @@ -15925,7 +15207,6 @@ 2a04:ce40::,2a04:ce47:ffff:ffff:ffff:ffff:ffff:ffff,DE 2a04:ce80::,2a04:ce87:ffff:ffff:ffff:ffff:ffff:ffff,GB 2a04:cec0::,2a04:cec7:ffff:ffff:ffff:ffff:ffff:ffff,FR -2a04:cf00::,2a04:cf07:ffff:ffff:ffff:ffff:ffff:ffff,NL 2a04:cf40::,2a04:cf47:ffff:ffff:ffff:ffff:ffff:ffff,DE 2a04:cf80::,2a04:cf87:ffff:ffff:ffff:ffff:ffff:ffff,SA 2a04:cfc0::,2a04:cfc7:ffff:ffff:ffff:ffff:ffff:ffff,RU @@ -16040,6 +15321,7 @@ 2a04:eb80::,2a04:eb87:ffff:ffff:ffff:ffff:ffff:ffff,ES 2a04:ebc0::,2a04:ebc7:ffff:ffff:ffff:ffff:ffff:ffff,GB 2a04:ec00::,2a04:ec01:ffff:ffff:ffff:ffff:ffff:ffff,NL +2a04:ec20::,2a04:ec23:ffff:ffff:ffff:ffff:ffff:ffff,RS 2a04:ec40::,2a04:ec47:ffff:ffff:ffff:ffff:ffff:ffff,GB 2a04:ec80::,2a04:ec87:ffff:ffff:ffff:ffff:ffff:ffff,DE 2a04:ecc0::,2a04:ecc7:ffff:ffff:ffff:ffff:ffff:ffff,GB @@ -16055,6 +15337,81 @@ 2a04:ef80::,2a04:ef87:ffff:ffff:ffff:ffff:ffff:ffff,GB 2a04:efc0::,2a04:efc7:ffff:ffff:ffff:ffff:ffff:ffff,IT 2a04:f000::,2a04:f007:ffff:ffff:ffff:ffff:ffff:ffff,CH +2a04:f040::,2a04:f047:ffff:ffff:ffff:ffff:ffff:ffff,HU +2a04:f080::,2a04:f087:ffff:ffff:ffff:ffff:ffff:ffff,IR +2a04:f0c0::,2a04:f0c7:ffff:ffff:ffff:ffff:ffff:ffff,LV +2a04:f100::,2a04:f107:ffff:ffff:ffff:ffff:ffff:ffff,AT +2a04:f140::,2a04:f147:ffff:ffff:ffff:ffff:ffff:ffff,CY +2a04:f180::,2a04:f187:ffff:ffff:ffff:ffff:ffff:ffff,SE +2a04:f1c0::,2a04:f1c7:ffff:ffff:ffff:ffff:ffff:ffff,SA +2a04:f200::,2a04:f207:ffff:ffff:ffff:ffff:ffff:ffff,PL +2a04:f240::,2a04:f247:ffff:ffff:ffff:ffff:ffff:ffff,RU +2a04:f280::,2a04:f287:ffff:ffff:ffff:ffff:ffff:ffff,BG +2a04:f2c0::,2a04:f2c7:ffff:ffff:ffff:ffff:ffff:ffff,BA +2a04:f300::,2a04:f300:ffff:ffff:ffff:ffff:ffff:ffff,NL +2a04:f320::,2a04:f323:ffff:ffff:ffff:ffff:ffff:ffff,RU +2a04:f340::,2a04:f347:ffff:ffff:ffff:ffff:ffff:ffff,CH +2a04:f380::,2a04:f387:ffff:ffff:ffff:ffff:ffff:ffff,NL +2a04:f3c0::,2a04:f3c7:ffff:ffff:ffff:ffff:ffff:ffff,FR +2a04:f440::,2a04:f447:ffff:ffff:ffff:ffff:ffff:ffff,PL +2a04:f480::,2a04:f487:ffff:ffff:ffff:ffff:ffff:ffff,SK +2a04:f4c0::,2a04:f4c7:ffff:ffff:ffff:ffff:ffff:ffff,RO +2a04:f500::,2a04:f507:ffff:ffff:ffff:ffff:ffff:ffff,DE +2a04:f540::,2a04:f547:ffff:ffff:ffff:ffff:ffff:ffff,US +2a04:f580::,2a04:f587:ffff:ffff:ffff:ffff:ffff:ffff,GB +2a04:f5c0::,2a04:f5c7:ffff:ffff:ffff:ffff:ffff:ffff,IT +2a04:f600::,2a04:f607:ffff:ffff:ffff:ffff:ffff:ffff,FR +2a04:f680::,2a04:f687:ffff:ffff:ffff:ffff:ffff:ffff,AT +2a04:f6c0::,2a04:f6c7:ffff:ffff:ffff:ffff:ffff:ffff,SC +2a04:f700::,2a04:f707:ffff:ffff:ffff:ffff:ffff:ffff,IR +2a04:f740::,2a04:f747:ffff:ffff:ffff:ffff:ffff:ffff,NL +2a04:f780::,2a04:f787:ffff:ffff:ffff:ffff:ffff:ffff,IN +2a04:f7c0::,2a04:f7c7:ffff:ffff:ffff:ffff:ffff:ffff,NL +2a04:f800::,2a04:f807:ffff:ffff:ffff:ffff:ffff:ffff,RU +2a04:f840::,2a04:f847:ffff:ffff:ffff:ffff:ffff:ffff,NL +2a04:f880::,2a04:f887:ffff:ffff:ffff:ffff:ffff:ffff,DE +2a04:f8c0::,2a04:f8c7:ffff:ffff:ffff:ffff:ffff:ffff,DE +2a04:f900::,2a04:f907:ffff:ffff:ffff:ffff:ffff:ffff,DE +2a04:f940::,2a04:f947:ffff:ffff:ffff:ffff:ffff:ffff,CZ +2a04:f980::,2a04:f987:ffff:ffff:ffff:ffff:ffff:ffff,RO +2a04:f9c0::,2a04:f9c7:ffff:ffff:ffff:ffff:ffff:ffff,AT +2a04:fa00::,2a04:fa07:ffff:ffff:ffff:ffff:ffff:ffff,NL +2a04:fa40::,2a04:fa47:ffff:ffff:ffff:ffff:ffff:ffff,IR +2a04:fa80::,2a04:fa87:ffff:ffff:ffff:ffff:ffff:ffff,IE +2a04:fb40::,2a04:fb47:ffff:ffff:ffff:ffff:ffff:ffff,IR +2a04:fb80::,2a04:fb87:ffff:ffff:ffff:ffff:ffff:ffff,GB +2a04:fbc0::,2a04:fbc7:ffff:ffff:ffff:ffff:ffff:ffff,NL +2a04:fc00::,2a04:fc07:ffff:ffff:ffff:ffff:ffff:ffff,IT +2a04:fc40::,2a04:fc47:ffff:ffff:ffff:ffff:ffff:ffff,LU +2a04:fc80::,2a04:fc87:ffff:ffff:ffff:ffff:ffff:ffff,IR +2a04:fcc0::,2a04:fcc7:ffff:ffff:ffff:ffff:ffff:ffff,GB +2a04:fd00::,2a04:fd07:ffff:ffff:ffff:ffff:ffff:ffff,GB +2a04:ff40::,2a04:ff47:ffff:ffff:ffff:ffff:ffff:ffff,AT +2a04:ff80::,2a04:ff87:ffff:ffff:ffff:ffff:ffff:ffff,BA +2a04:ffc0::,2a04:ffc7:ffff:ffff:ffff:ffff:ffff:ffff,HU +2a05::,2a05:7:ffff:ffff:ffff:ffff:ffff:ffff,BE +2a05:40::,2a05:47:ffff:ffff:ffff:ffff:ffff:ffff,MD +2a05:80::,2a05:87:ffff:ffff:ffff:ffff:ffff:ffff,CH +2a05:c0::,2a05:c7:ffff:ffff:ffff:ffff:ffff:ffff,DK +2a05:100::,2a05:107:ffff:ffff:ffff:ffff:ffff:ffff,GB +2a05:140::,2a05:147:ffff:ffff:ffff:ffff:ffff:ffff,DE +2a05:180::,2a05:187:ffff:ffff:ffff:ffff:ffff:ffff,RU +2a05:1c0::,2a05:1c7:ffff:ffff:ffff:ffff:ffff:ffff,GB +2a05:200::,2a05:207:ffff:ffff:ffff:ffff:ffff:ffff,BE +2a05:240::,2a05:247:ffff:ffff:ffff:ffff:ffff:ffff,NL +2a05:280::,2a05:287:ffff:ffff:ffff:ffff:ffff:ffff,RU +2a05:2c0::,2a05:2c7:ffff:ffff:ffff:ffff:ffff:ffff,CZ +2a05:300::,2a05:307:ffff:ffff:ffff:ffff:ffff:ffff,TR +2a05:340::,2a05:347:ffff:ffff:ffff:ffff:ffff:ffff,ES +2a05:380::,2a05:387:ffff:ffff:ffff:ffff:ffff:ffff,DE +2a05:3c0::,2a05:3c7:ffff:ffff:ffff:ffff:ffff:ffff,FR +2a05:400::,2a05:407:ffff:ffff:ffff:ffff:ffff:ffff,DE +2a05:440::,2a05:447:ffff:ffff:ffff:ffff:ffff:ffff,PL +2a05:480::,2a05:487:ffff:ffff:ffff:ffff:ffff:ffff,UA +2a05:4c0::,2a05:4c7:ffff:ffff:ffff:ffff:ffff:ffff,IQ +2a05:500::,2a05:507:ffff:ffff:ffff:ffff:ffff:ffff,UA +2a05:540::,2a05:547:ffff:ffff:ffff:ffff:ffff:ffff,RU +2a05:580::,2a05:587:ffff:ffff:ffff:ffff:ffff:ffff,AE 2c0e::,2c0e:fff:ffff:ffff:ffff:ffff:ffff:ffff,EG 2c0e:2000::,2c0e:200f:ffff:ffff:ffff:ffff:ffff:ffff,ZA 2c0f:f600::,2c0f:f600:ffff:ffff:ffff:ffff:ffff:ffff,GN @@ -16095,6 +15452,7 @@ 2c0f:f718::,2c0f:f718:ffff:ffff:ffff:ffff:ffff:ffff,BW 2c0f:f720::,2c0f:f720:ffff:ffff:ffff:ffff:ffff:ffff,ZA 2c0f:f728::,2c0f:f728:ffff:ffff:ffff:ffff:ffff:ffff,BW +2c0f:f730::,2c0f:f730:ffff:ffff:ffff:ffff:ffff:ffff,TZ 2c0f:f800::,2c0f:f80f:ffff:ffff:ffff:ffff:ffff:ffff,ZA 2c0f:f810::,2c0f:f810:ffff:ffff:ffff:ffff:ffff:ffff,AO 2c0f:f818::,2c0f:f818:ffff:ffff:ffff:ffff:ffff:ffff,BJ @@ -16107,7 +15465,6 @@ 2c0f:f850::,2c0f:f850:ffff:ffff:ffff:ffff:ffff:ffff,ZA 2c0f:f858::,2c0f:f858:ffff:ffff:ffff:ffff:ffff:ffff,DZ 2c0f:f860::,2c0f:f860:ffff:ffff:ffff:ffff:ffff:ffff,RW -2c0f:f870::,2c0f:f870:ffff:ffff:ffff:ffff:ffff:ffff,NG 2c0f:f878::,2c0f:f878:ffff:ffff:ffff:ffff:ffff:ffff,KE 2c0f:f880::,2c0f:f880:ffff:ffff:ffff:ffff:ffff:ffff,NG 2c0f:f888::,2c0f:f888:ffff:ffff:ffff:ffff:ffff:ffff,AO @@ -16189,7 +15546,6 @@ 2c0f:fb10::,2c0f:fb10:ffff:ffff:ffff:ffff:ffff:ffff,LY 2c0f:fb18::,2c0f:fb18:ffff:ffff:ffff:ffff:ffff:ffff,ZA 2c0f:fb20::,2c0f:fb20:ffff:ffff:ffff:ffff:ffff:ffff,MA -2c0f:fb28::,2c0f:fb28:ffff:ffff:ffff:ffff:ffff:ffff,ZA 2c0f:fb30::,2c0f:fb30:ffff:ffff:ffff:ffff:ffff:ffff,ZA 2c0f:fb38::,2c0f:fb38:ffff:ffff:ffff:ffff:ffff:ffff,SO 2c0f:fb40::,2c0f:fb40:ffff:ffff:ffff:ffff:ffff:ffff,ZA diff --git a/src/config/mmdb-convert.py b/src/config/mmdb-convert.py new file mode 100644 index 0000000000..cbe9acdc5d --- /dev/null +++ b/src/config/mmdb-convert.py @@ -0,0 +1,466 @@ +#!/usr/bin/python3 + +# This software has been dedicated to the public domain under the CC0 +# public domain dedication. +# +# To the extent possible under law, the person who associated CC0 +# with mmdb-convert.py has waived all copyright and related or +# neighboring rights to mmdb-convert.py. +# +# You should have received a copy of the CC0 legalcode along with this +# work in doc/cc0.txt. If not, see +# <http://creativecommons.org/publicdomain/zero/1.0/>. + +# Nick Mathewson is responsible for this kludge, but takes no +# responsibility for it. + +"""This kludge is meant to + parse mmdb files in sufficient detail to dump out the old format + that Tor expects. It's also meant to be pure-python. + + When given a simplicity/speed tradeoff, it opts for simplicity. + + You will not understand the code without undestanding the MaxMind-DB + file format. It is specified at: + https://github.com/maxmind/MaxMind-DB/blob/master/MaxMind-DB-spec.md. + + This isn't so much tested. When it breaks, you get to keep both + pieces. +""" + +import struct +import bisect +import socket +import binascii +import sys +import time + +METADATA_MARKER = b'\xab\xcd\xefMaxMind.com' + +# Here's some python2/python3 junk. Better solutions wanted. +try: + ord(b"1"[0]) +except TypeError: + def byte_to_int(b): + "convert a single element of a bytestring to an integer." + return b +else: + byte_to_int = ord + +# Here's some more python2/python3 junk. Better solutions wanted. +try: + str(b"a", "utf8") +except TypeError: + bytesToStr = str +else: + def bytesToStr(b): + "convert a bytestring in utf8 to a string." + return str(b, 'utf8') + +def to_int(s): + "Parse a big-endian integer from bytestring s." + result = 0 + for c in s: + result *= 256 + result += byte_to_int(c) + return result + +def to_int24(s): + "Parse a pair of big-endian 24-bit integers from bytestring s." + a, b, c = struct.unpack("!HHH", s) + return ((a <<8)+(b>>8)), (((b&0xff)<<16)+c) + +def to_int32(s): + "Parse a pair of big-endian 32-bit integers from bytestring s." + a, b = struct.unpack("!LL", s) + return a, b + +def to_int28(s): + "Parse a pair of big-endian 28-bit integers from bytestring s." + a, b = unpack("!LL", s + b'\x00') + return (((a & 0xf0) << 20) + (a >> 8)), ((a & 0x0f) << 24) + (b >> 8) + +class Tree(object): + "Holds a node in the tree" + def __init__(self, left, right): + self.left = left + self.right = right + +def resolve_tree(tree, data): + """Fill in the left_item and right_item fields for all values in the tree + so that they point to another Tree, or to a Datum, or to None.""" + d = Datum(None, None, None, None) + def resolve_item(item): + "Helper: resolve a single index." + if item < len(tree): + return tree[item] + elif item == len(tree): + return None + else: + d.pos = (item - len(tree) - 16) + p = bisect.bisect_left(data, d) + assert data[p].pos == d.pos + return data[p] + + for t in tree: + t.left_item = resolve_item(t.left) + t.right_item = resolve_item(t.right) + +def parse_search_tree(s, record_size): + """Given a bytestring and a record size in bits, parse the tree. + Return a list of nodes.""" + record_bytes = (record_size*2) // 8 + nodes = [] + p = 0 + try: + to_leftright = { 24: to_int24, + 28: to_int28, + 32: to_int32 }[ record_size ] + except KeyError: + raise NotImplementedError("Unsupported record size in bits: %d" % + record_size) + while p < len(s): + left, right = to_leftright(s[p:p+record_bytes]) + p += record_bytes + + nodes.append( Tree(left, right ) ) + + return nodes + +class Datum(object): + """Holds a single entry from the Data section""" + def __init__(self, pos, kind, ln, data): + self.pos = pos # Position of this record within data section + self.kind = kind # Type of this record. one of TP_* + self.ln = ln # Length field, which might be overloaded. + self.data = data # Raw bytes data. + self.children = None # Used for arrays and maps. + + def __repr__(self): + return "Datum(%r,%r,%r,%r)" % (self.pos, self.kind, self.ln, self.data) + + # Comparison functions used for bsearch + def __lt__(self, other): + return self.pos < other.pos + + def __gt__(self, other): + return self.pos > other.pos + + def __eq__(self, other): + return self.pos == other.pos + + def build_maps(self): + """If this is a map or array, fill in its 'map' field if it's a map, + and the 'map' field of all its children.""" + + if not hasattr(self, 'nChildren'): + return + + if self.kind == TP_ARRAY: + del self.nChildren + for c in self.children: + c.build_maps() + + elif self.kind == TP_MAP: + del self.nChildren + self.map = {} + for i in range(0, len(self.children), 2): + k = self.children[i].deref() + v = self.children[i+1].deref() + v.build_maps() + if k.kind != TP_UTF8: + raise ValueError("Bad dictionary key type %d"% k.kind) + self.map[bytesToStr(k.data)] = v + + def int_val(self): + """If this is an integer type, return its value""" + assert self.kind in (TP_UINT16, TP_UINT32, TP_UINT64, + TP_UINT128, TP_SINT32) + i = to_int(self.data) + if self.kind == TP_SINT32: + if i & 0x80000000: + i = i - 0x100000000 + return i + + def deref(self): + """If this value is a pointer, return its pointed-to-value. Chase + through multiple layers of pointers if need be. If this isn't + a pointer, return it.""" + n = 0 + s = self + while s.kind == TP_PTR: + s = s.ptr + n += 1 + assert n < 100 + return s + +def resolve_pointers(data): + """Fill in the ptr field of every pointer in data.""" + search = Datum(None, None, None, None) + for d in data: + if d.kind == TP_PTR: + search.pos = d.ln + p = bisect.bisect_left(data, search) + assert data[p].pos == d.ln + d.ptr = data[p] + +TP_PTR = 1 +TP_UTF8 = 2 +TP_DBL = 3 +TP_BYTES = 4 +TP_UINT16 = 5 +TP_UINT32 = 6 +TP_MAP = 7 +TP_SINT32 = 8 +TP_UINT64 = 9 +TP_UINT128 = 10 +TP_ARRAY = 11 +TP_DCACHE = 12 +TP_END = 13 +TP_BOOL = 14 +TP_FLOAT = 15 + +def get_type_and_len(s): + """Data parsing helper: decode the type value and much-overloaded 'length' + field for the value starting at s. Return a 3-tuple of type, length, + and number of bytes used to encode type-plus-length.""" + c = byte_to_int(s[0]) + tp = c >> 5 + skip = 1 + if tp == 0: + tp = byte_to_int(s[1])+7 + skip = 2 + ln = c & 31 + + # I'm sure I don't know what they were thinking here... + if tp == TP_PTR: + len_len = (ln >> 3) + 1 + if len_len < 4: + ln &= 7 + ln <<= len_len * 8 + else: + ln = 0 + ln += to_int(s[skip:skip+len_len]) + ln += (0, 0, 2048, 526336, 0)[len_len] + skip += len_len + elif ln >= 29: + len_len = ln - 28 + ln = to_int(s[skip:skip+len_len]) + ln += (0, 29, 285, 65821)[len_len] + skip += len_len + + return tp, ln, skip + +# Set of types for which 'length' doesn't mean length. +IGNORE_LEN_TYPES = set([ + TP_MAP, # Length is number of key-value pairs that follow. + TP_ARRAY, # Length is number of members that follow. + TP_PTR, # Length is index to pointed-to data element. + TP_BOOL, # Length is 0 or 1. + TP_DCACHE, # Length isnumber of members that follow +]) + +def parse_data_section(s): + """Given a data section encoded in a bytestring, return a list of + Datum items.""" + + # Stack of possibly nested containers. We use the 'nChildren' member of + # the last one to tell how many moreitems nest directly inside. + stack = [] + + # List of all items, including nested ones. + data = [] + + # Byte index within the data section. + pos = 0 + + while s: + tp, ln, skip = get_type_and_len(s) + if tp in IGNORE_LEN_TYPES: + real_len = 0 + else: + real_len = ln + + d = Datum(pos, tp, ln, s[skip:skip+real_len]) + data.append(d) + pos += skip+real_len + s = s[skip+real_len:] + + if stack: + stack[-1].children.append(d) + stack[-1].nChildren -= 1 + if stack[-1].nChildren == 0: + del stack[-1] + + if d.kind == TP_ARRAY: + d.nChildren = d.ln + d.children = [] + stack.append(d) + elif d.kind == TP_MAP: + d.nChildren = d.ln * 2 + d.children = [] + stack.append(d) + + return data + +def parse_mm_file(s): + """Parse a MaxMind-DB file.""" + try: + metadata_ptr = s.rindex(METADATA_MARKER) + except ValueError: + raise ValueError("No metadata!") + + metadata = parse_data_section(s[metadata_ptr+len(METADATA_MARKER):]) + + if metadata[0].kind != TP_MAP: + raise ValueError("Bad map") + + metadata[0].build_maps() + mm = metadata[0].map + + tree_size = (((mm['record_size'].int_val() * 2) // 8 ) * + mm['node_count'].int_val()) + + if s[tree_size:tree_size+16] != b'\x00'*16: + raise ValueError("Missing section separator!") + + tree = parse_search_tree(s[:tree_size], mm['record_size'].int_val()) + + data = parse_data_section(s[tree_size+16:metadata_ptr]) + + resolve_pointers(data) + resolve_tree(tree, data) + + for d in data: + d.build_maps() + + return metadata, tree, data + +def format_datum(datum): + """Given a Datum at a leaf of the tree, return the string that we should + write as its value. + + We first try country->iso_code which is the two-character ISO 3166-1 + country code of the country where MaxMind believes the end user is + located. If there's no such key, we try registered_country->iso_code + which is the country in which the ISP has registered the IP address. + Without falling back to registered_country, we'd leave out all ranges + that MaxMind thinks belong to anonymous proxies, because those ranges + don't contain country but only registered_country. In short: let's + fill all A1 entries with what ARIN et. al think. + """ + try: + return bytesToStr(datum.map['country'].map['iso_code'].data) + except KeyError: + pass + try: + return bytesToStr(datum.map['registered_country'].map['iso_code'].data) + except KeyError: + pass + return None + +IPV4_PREFIX = "0"*96 + +def dump_item_ipv4(entries, prefix, val): + """Dump the information for an IPv4 address to entries, where 'prefix' + is a string holding a binary prefix for the address, and 'val' is the + value to dump. If the prefix is not an IPv4 address (it does not start + with 96 bits of 0), then print nothing. + """ + if not prefix.startswith(IPV4_PREFIX): + return + prefix = prefix[96:] + v = int(prefix, 2) + shift = 32 - len(prefix) + lo = v << shift + hi = ((v+1) << shift) - 1 + entries.append((lo, hi, val)) + +def fmt_item_ipv4(entry): + """Format an IPv4 range with lo and hi addresses in decimal form.""" + return "%d,%d,%s\n"%(entry[0], entry[1], entry[2]) + +def fmt_ipv6_addr(v): + """Given a 128-bit integer representing an ipv6 address, return a + string for that ipv6 address.""" + return socket.inet_ntop(socket.AF_INET6, binascii.unhexlify("%032x"%v)) + +def fmt_item_ipv6(entry): + """Format an IPv6 range with lo and hi addresses in hex form.""" + return "%s,%s,%s\n"%(fmt_ipv6_addr(entry[0]), + fmt_ipv6_addr(entry[1]), + entry[2]) + +IPV4_MAPPED_IPV6_PREFIX = "0"*80 + "1"*16 +IPV6_6TO4_PREFIX = "0010000000000010" +TEREDO_IPV6_PREFIX = "0010000000000001" + "0"*16 + +def dump_item_ipv6(entries, prefix, val): + """Dump the information for an IPv6 address prefix to entries, where + 'prefix' is a string holding a binary prefix for the address, + and 'val' is the value to dump. If the prefix is an IPv4 address + (starts with 96 bits of 0), is an IPv4-mapped IPv6 address + (::ffff:0:0/96), or is in the 6to4 mapping subnet (2002::/16), then + print nothing. + """ + if prefix.startswith(IPV4_PREFIX) or \ + prefix.startswith(IPV4_MAPPED_IPV6_PREFIX) or \ + prefix.startswith(IPV6_6TO4_PREFIX) or \ + prefix.startswith(TEREDO_IPV6_PREFIX): + return + v = int(prefix, 2) + shift = 128 - len(prefix) + lo = v << shift + hi = ((v+1) << shift) - 1 + entries.append((lo, hi, val)) + +def dump_tree(entries, node, dump_item, prefix=""): + """Walk the tree rooted at 'node', and call dump_item on the + format_datum output of every leaf of the tree.""" + + if isinstance(node, Tree): + dump_tree(entries, node.left_item, dump_item, prefix+"0") + dump_tree(entries, node.right_item, dump_item, prefix+"1") + elif isinstance(node, Datum): + assert node.kind == TP_MAP + code = format_datum(node) + if code: + dump_item(entries, prefix, code) + else: + assert node == None + +GEOIP_FILE_HEADER = """\ +# Last updated based on %s Maxmind GeoLite2 Country +# wget https://geolite.maxmind.com/download/geoip/database/GeoLite2-Country.mmdb.gz +# gunzip GeoLite2-Country.mmdb.gz +# python mmdb-convert.py GeoLite2-Country.mmdb +""" + +def write_geoip_file(filename, metadata, the_tree, dump_item, fmt_item): + """Write the entries in the_tree to filename.""" + entries = [] + dump_tree(entries, the_tree[0], dump_item) + fobj = open(filename, 'w') + + build_epoch = metadata[0].map['build_epoch'].int_val() + fobj.write(GEOIP_FILE_HEADER % + time.strftime('%B %-d %Y', time.gmtime(build_epoch))) + + unwritten = None + for entry in entries: + if not unwritten: + unwritten = entry + elif unwritten[1] + 1 == entry[0] and unwritten[2] == entry[2]: + unwritten = (unwritten[0], entry[1], unwritten[2]) + else: + fobj.write(fmt_item(unwritten)) + unwritten = entry + if unwritten: + fobj.write(fmt_item(unwritten)) + fobj.close() + +content = open(sys.argv[1], 'rb').read() +metadata, the_tree, _ = parse_mm_file(content) + +write_geoip_file('geoip', metadata, the_tree, dump_item_ipv4, fmt_item_ipv4) +write_geoip_file('geoip6', metadata, the_tree, dump_item_ipv6, fmt_item_ipv6) diff --git a/src/config/torrc.sample.in b/src/config/torrc.sample.in index c667efc5c9..d842fbcaf5 100644 --- a/src/config/torrc.sample.in +++ b/src/config/torrc.sample.in @@ -1,5 +1,5 @@ ## Configuration file for a typical Tor user -## Last updated 12 September 2012 for Tor 0.2.4.3-alpha. +## Last updated 9 October 2013 for Tor 0.2.5.2-alpha. ## (may or may not work for much older or much newer versions of Tor.) ## ## Lines that begin with "## " try to explain what's going on. Lines @@ -120,9 +120,12 @@ ## is per month) #AccountingStart month 3 15:00 -## Contact info to be published in the directory, so we can contact you -## if your relay is misconfigured or something else goes wrong. Google -## indexes this, so spammers might also collect it. +## Administrative contact information for this relay or bridge. This line +## can be used to contact you if your relay or bridge is misconfigured or +## something else goes wrong. Note that we archive and publish all +## descriptors containing these lines and that Google indexes them, so +## spammers might also collect them. You may want to obscure the fact that +## it's an email address and/or generate a new address for this purpose. #ContactInfo Random Person <nobody AT example dot com> ## You might also include your PGP or GPG fingerprint if you have one: #ContactInfo 0xFFFFFFFF Random Person <nobody AT example dot com> diff --git a/src/ext/README b/src/ext/README index 58ba7f699d..5d5a6e1518 100644 --- a/src/ext/README +++ b/src/ext/README @@ -42,3 +42,10 @@ curve25519_donna/*.c A copy of Adam Langley's curve25519-donna mostly-portable implementations of curve25519. + +csiphash.c +siphash.h + + Marek Majkowski's implementation of siphash 2-4, a secure keyed + hash algorithm to avoid collision-based DoS attacks against hash + tables. diff --git a/src/ext/csiphash.c b/src/ext/csiphash.c new file mode 100644 index 0000000000..c247886038 --- /dev/null +++ b/src/ext/csiphash.c @@ -0,0 +1,166 @@ +/* <MIT License> + Copyright (c) 2013 Marek Majkowski <marek@popcount.org> + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. + </MIT License> + + Original location: + https://github.com/majek/csiphash/ + + Solution inspired by code from: + Samuel Neves (supercop/crypto_auth/siphash24/little) + djb (supercop/crypto_auth/siphash24/little2) + Jean-Philippe Aumasson (https://131002.net/siphash/siphash24.c) +*/ + +#include "torint.h" +#include "siphash.h" +/* for tor_assert */ +#include "util.h" +/* for memcpy */ +#include <string.h> + +#if defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && \ + __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ +# define _le64toh(x) ((uint64_t)(x)) +#elif defined(_WIN32) +/* Windows is always little endian, unless you're on xbox360 + http://msdn.microsoft.com/en-us/library/b0084kay(v=vs.80).aspx */ +# define _le64toh(x) ((uint64_t)(x)) +#elif defined(__APPLE__) +# include <libkern/OSByteOrder.h> +# define _le64toh(x) OSSwapLittleToHostInt64(x) +#elif defined(sun) || defined(__sun) +# include <sys/byteorder.h> +# define _le64toh(x) LE_64(x) + +#else + +/* See: http://sourceforge.net/p/predef/wiki/Endianness/ */ +# if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) +# include <sys/endian.h> +# else +# include <endian.h> +# endif +# if defined(__BYTE_ORDER) && defined(__LITTLE_ENDIAN) && \ + __BYTE_ORDER == __LITTLE_ENDIAN +# define _le64toh(x) ((uint64_t)(x)) +# else +# if defined(__OpenBSD__) +# define _le64toh(x) letoh64(x) +# else +# define _le64toh(x) le64toh(x) +# endif +# endif + +#endif + +#define ROTATE(x, b) (uint64_t)( ((x) << (b)) | ( (x) >> (64 - (b))) ) + +#define HALF_ROUND(a,b,c,d,s,t) \ + a += b; c += d; \ + b = ROTATE(b, s) ^ a; \ + d = ROTATE(d, t) ^ c; \ + a = ROTATE(a, 32); + +#define DOUBLE_ROUND(v0,v1,v2,v3) \ + HALF_ROUND(v0,v1,v2,v3,13,16); \ + HALF_ROUND(v2,v1,v0,v3,17,21); \ + HALF_ROUND(v0,v1,v2,v3,13,16); \ + HALF_ROUND(v2,v1,v0,v3,17,21); + +#if 0 +/* This does not seem to save very much runtime in the fast case, and it's + * potentially a big loss in the slow case where we're misaligned and we cross + * a cache line. */ +#if (defined(__i386) || defined(__i386__) || defined(_M_IX86) || \ + defined(__x86_64) || defined(__x86_64__) || \ + defined(_M_AMD64) || defined(_M_X64) || defined(__INTEL__)) +# define UNALIGNED_OK 1 +#endif +#endif + +uint64_t siphash24(const void *src, unsigned long src_sz, const struct sipkey *key) { + uint64_t k0 = key->k0; + uint64_t k1 = key->k1; + uint64_t b = (uint64_t)src_sz << 56; + const uint64_t *in = (uint64_t*)src; + + uint64_t t; + uint8_t *pt, *m; + + uint64_t v0 = k0 ^ 0x736f6d6570736575ULL; + uint64_t v1 = k1 ^ 0x646f72616e646f6dULL; + uint64_t v2 = k0 ^ 0x6c7967656e657261ULL; + uint64_t v3 = k1 ^ 0x7465646279746573ULL; + + while (src_sz >= 8) { +#ifdef UNALIGNED_OK + uint64_t mi = _le64toh(*in); +#else + uint64_t mi; + memcpy(&mi, in, 8); + mi = _le64toh(mi); +#endif + in += 1; src_sz -= 8; + v3 ^= mi; + DOUBLE_ROUND(v0,v1,v2,v3); + v0 ^= mi; + } + + t = 0; pt = (uint8_t*)&t; m = (uint8_t*)in; + switch (src_sz) { + case 7: pt[6] = m[6]; + case 6: pt[5] = m[5]; + case 5: pt[4] = m[4]; +#ifdef UNALIGNED_OK + case 4: *((uint32_t*)&pt[0]) = *((uint32_t*)&m[0]); break; +#else + case 4: pt[3] = m[3]; +#endif + case 3: pt[2] = m[2]; + case 2: pt[1] = m[1]; + case 1: pt[0] = m[0]; + } + b |= _le64toh(t); + + v3 ^= b; + DOUBLE_ROUND(v0,v1,v2,v3); + v0 ^= b; v2 ^= 0xff; + DOUBLE_ROUND(v0,v1,v2,v3); + DOUBLE_ROUND(v0,v1,v2,v3); + return (v0 ^ v1) ^ (v2 ^ v3); +} + + +static int the_siphash_key_is_set = 0; +static struct sipkey the_siphash_key; + +uint64_t siphash24g(const void *src, unsigned long src_sz) { + tor_assert(the_siphash_key_is_set); + return siphash24(src, src_sz, &the_siphash_key); +} + +void siphash_set_global_key(const struct sipkey *key) +{ + tor_assert(! the_siphash_key_is_set); + the_siphash_key.k0 = key->k0; + the_siphash_key.k1 = key->k1; + the_siphash_key_is_set = 1; +} diff --git a/src/ext/eventdns.c b/src/ext/eventdns.c index 66280cccdb..2b2988f1ec 100644 --- a/src/ext/eventdns.c +++ b/src/ext/eventdns.c @@ -842,10 +842,11 @@ name_parse(u8 *packet, int length, int *idx, char *name_out, size_t name_out_len } if (label_len > 63) return -1; if (cp != name_out) { - if (cp + 1 >= end) return -1; + if (cp >= name_out + name_out_len - 1) return -1; *cp++ = '.'; } - if (cp + label_len >= end) return -1; + if (label_len > name_out_len || + cp >= name_out + name_out_len - label_len) return -1; memcpy(cp, packet + j, label_len); cp += label_len; j += label_len; @@ -2298,6 +2299,10 @@ _evdns_nameserver_add_impl(const struct sockaddr *address, evtimer_set(&ns->timeout_event, nameserver_prod_callback, ns); +#if 1 + ns->socket = tor_open_socket_nonblocking(address->sa_family, SOCK_DGRAM, 0); + if (!SOCKET_OK(ns->socket)) { err = 1; goto out1; } +#else ns->socket = tor_open_socket(address->sa_family, SOCK_DGRAM, 0); if (ns->socket < 0) { err = 1; goto out1; } #ifdef _WIN32 @@ -2314,6 +2319,7 @@ _evdns_nameserver_add_impl(const struct sockaddr *address, } #endif +#endif /* 1 */ if (global_bind_addr_is_set && !sockaddr_is_loopback((struct sockaddr*)&global_bind_address)) { if (bind(ns->socket, (struct sockaddr *)&global_bind_address, @@ -3009,7 +3015,8 @@ resolv_conf_parse_line(char *const start, int flags) { if (!strcmp(first_token, "nameserver") && (flags & DNS_OPTION_NAMESERVERS)) { const char *const nameserver = NEXT_TOKEN; - evdns_nameserver_ip_add(nameserver); + if (nameserver) + evdns_nameserver_ip_add(nameserver); } else if (!strcmp(first_token, "domain") && (flags & DNS_OPTION_SEARCH)) { const char *const domain = NEXT_TOKEN; if (domain) { @@ -3473,8 +3480,12 @@ main(int c, char **v) { if (servertest) { int sock; struct sockaddr_in my_addr; +#if 1 + sock = tor_open_socket_nonblocking(PF_INET, SOCK_DGRAM, 0) +#else sock = tor_open_socket(PF_INET, SOCK_DGRAM, 0); fcntl(sock, F_SETFL, O_NONBLOCK); +#endif my_addr.sin_family = AF_INET; my_addr.sin_port = htons(10053); my_addr.sin_addr.s_addr = INADDR_ANY; diff --git a/src/ext/ht.h b/src/ext/ht.h index 62c458ad0e..838710784f 100644 --- a/src/ext/ht.h +++ b/src/ext/ht.h @@ -58,6 +58,7 @@ #define HT_NEXT_RMV(name, head, elm) name##_HT_NEXT_RMV((head), (elm)) #define HT_CLEAR(name, head) name##_HT_CLEAR(head) #define HT_INIT(name, head) name##_HT_INIT(head) +#define HT_REP_IS_BAD_(name, head) name##_HT_REP_IS_BAD_(head) /* Helper: */ static INLINE unsigned ht_improve_hash(unsigned h) @@ -86,6 +87,7 @@ ht_string_hash(const char *s) } #endif +#if 0 /** Basic string hash function, from Python's str.__hash__() */ static INLINE unsigned ht_string_hash(const char *s) @@ -100,6 +102,7 @@ ht_string_hash(const char *s) h ^= (unsigned)(cp-(const unsigned char*)s); return h; } +#endif #ifndef HT_NO_CACHE_HASH_VALUES #define HT_SET_HASH_(elm, field, hashfn) \ @@ -157,7 +160,7 @@ ht_string_hash(const char *s) } \ /* Return a pointer to the element in the table 'head' matching 'elm', \ * or NULL if no such element exists */ \ - static INLINE struct type * \ + ATTR_UNUSED static INLINE struct type * \ name##_HT_FIND(const struct name *head, struct type *elm) \ { \ struct type **p; \ @@ -301,14 +304,16 @@ ht_string_hash(const char *s) #define HT_GENERATE(name, type, field, hashfn, eqfn, load, mallocfn, \ reallocfn, freefn) \ + /* Primes that aren't too far from powers of two. We stop at */ \ + /* P=402653189 because P*sizeof(void*) is less than SSIZE_MAX */ \ + /* even on a 32-bit platform. */ \ static unsigned name##_PRIMES[] = { \ 53, 97, 193, 389, \ 769, 1543, 3079, 6151, \ 12289, 24593, 49157, 98317, \ 196613, 393241, 786433, 1572869, \ 3145739, 6291469, 12582917, 25165843, \ - 50331653, 100663319, 201326611, 402653189, \ - 805306457, 1610612741 \ + 50331653, 100663319, 201326611, 402653189 \ }; \ static unsigned name##_N_PRIMES = \ (unsigned)(sizeof(name##_PRIMES)/sizeof(name##_PRIMES[0])); \ diff --git a/src/ext/include.am b/src/ext/include.am index ea7e58e79e..26e194e88e 100644 --- a/src/ext/include.am +++ b/src/ext/include.am @@ -10,7 +10,8 @@ EXTHEADERS = \ src/ext/strlcat.c \ src/ext/strlcpy.c \ src/ext/tinytest_macros.h \ - src/ext/tor_queue.h + src/ext/tor_queue.h \ + src/ext/siphash.h noinst_HEADERS+= $(EXTHEADERS) diff --git a/src/ext/siphash.h b/src/ext/siphash.h new file mode 100644 index 0000000000..d9b34b8980 --- /dev/null +++ b/src/ext/siphash.h @@ -0,0 +1,13 @@ +#ifndef SIPHASH_H +#define SIPHASH_H + +struct sipkey { + uint64_t k0; + uint64_t k1; +}; +uint64_t siphash24(const void *src, unsigned long src_sz, const struct sipkey *key); + +void siphash_set_global_key(const struct sipkey *key); +uint64_t siphash24g(const void *src, unsigned long src_sz); + +#endif diff --git a/src/ext/tinytest.c b/src/ext/tinytest.c index 4d9afacce4..cc054ad340 100644 --- a/src/ext/tinytest.c +++ b/src/ext/tinytest.c @@ -31,6 +31,8 @@ #include <string.h> #include <assert.h> +#ifndef NO_FORKING + #ifdef _WIN32 #include <windows.h> #else @@ -39,6 +41,17 @@ #include <unistd.h> #endif +#if defined(__APPLE__) && defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__) +#if (__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ >= 1060 && \ + __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 1070) +/* Workaround for a stupid bug in OSX 10.6 */ +#define FORK_BREAKS_GCOV +#include <vproc.h> +#endif +#endif + +#endif /* !NO_FORKING */ + #ifndef __GNUC__ #define __attribute__(x) #endif @@ -58,6 +71,8 @@ static int opt_nofork = 0; /**< Suppress calls to fork() for debugging. */ static int opt_verbosity = 1; /**< -==quiet,0==terse,1==normal,2==verbose */ const char *verbosity_flag = ""; +const struct testlist_alias_t *cfg_aliases=NULL; + enum outcome { SKIP=2, OK=1, FAIL=0 }; static enum outcome cur_test_outcome = 0; const char *cur_test_prefix = NULL; /**< prefix of the current test group */ @@ -71,6 +86,7 @@ static char commandname[MAX_PATH+1]; static void usage(struct testgroup_t *groups, int list_groups) __attribute__((noreturn)); +static int process_test_option(struct testgroup_t *groups, const char *test); static enum outcome testcase_run_bare_(const struct testcase_t *testcase) @@ -99,6 +115,8 @@ testcase_run_bare_(const struct testcase_t *testcase) #define MAGIC_EXITCODE 42 +#ifndef NO_FORKING + static enum outcome testcase_run_forked_(const struct testgroup_t *group, const struct testcase_t *testcase) @@ -160,6 +178,9 @@ testcase_run_forked_(const struct testgroup_t *group, if (opt_verbosity>0) printf("[forking] "); pid = fork(); +#ifdef FORK_BREAKS_GCOV + vproc_transaction_begin(0); +#endif if (!pid) { /* child. */ int test_r, write_r; @@ -196,16 +217,19 @@ testcase_run_forked_(const struct testgroup_t *group, #endif } +#endif /* !NO_FORKING */ + int testcase_run_one(const struct testgroup_t *group, const struct testcase_t *testcase) { enum outcome outcome; - if (testcase->flags & TT_SKIP) { + if (testcase->flags & (TT_SKIP|TT_OFF_BY_DEFAULT)) { if (opt_verbosity>0) - printf("%s%s: SKIPPED\n", - group->prefix, testcase->name); + printf("%s%s: %s\n", + group->prefix, testcase->name, + (testcase->flags & TT_SKIP) ? "SKIPPED" : "DISABLED"); ++n_skipped; return SKIP; } @@ -218,9 +242,13 @@ testcase_run_one(const struct testgroup_t *group, cur_test_name = testcase->name; } +#ifndef NO_FORKING if ((testcase->flags & TT_FORK) && !(opt_forked||opt_nofork)) { outcome = testcase_run_forked_(group, testcase); } else { +#else + { +#endif outcome = testcase_run_bare_(testcase); } @@ -247,7 +275,7 @@ testcase_run_one(const struct testgroup_t *group, } int -tinytest_set_flag_(struct testgroup_t *groups, const char *arg, unsigned long flag) +tinytest_set_flag_(struct testgroup_t *groups, const char *arg, int set, unsigned long flag) { int i, j; size_t length = LONGEST_TEST_NAME; @@ -257,12 +285,23 @@ tinytest_set_flag_(struct testgroup_t *groups, const char *arg, unsigned long fl length = strstr(arg,"..")-arg; for (i=0; groups[i].prefix; ++i) { for (j=0; groups[i].cases[j].name; ++j) { + struct testcase_t *testcase = &groups[i].cases[j]; snprintf(fullname, sizeof(fullname), "%s%s", - groups[i].prefix, groups[i].cases[j].name); - if (!flag) /* Hack! */ - printf(" %s\n", fullname); + groups[i].prefix, testcase->name); + if (!flag) { /* Hack! */ + printf(" %s", fullname); + if (testcase->flags & TT_OFF_BY_DEFAULT) + puts(" (Off by default)"); + else if (testcase->flags & TT_SKIP) + puts(" (DISABLED)"); + else + puts(""); + } if (!strncmp(fullname, arg, length)) { - groups[i].cases[j].flags |= flag; + if (set) + testcase->flags |= flag; + else + testcase->flags &= ~flag; ++found; } } @@ -275,15 +314,69 @@ usage(struct testgroup_t *groups, int list_groups) { puts("Options are: [--verbose|--quiet|--terse] [--no-fork]"); puts(" Specify tests by name, or using a prefix ending with '..'"); - puts(" To skip a test, list give its name prefixed with a colon."); + puts(" To skip a test, prefix its name with a colon."); + puts(" To enable a disabled test, prefix its name with a plus."); puts(" Use --list-tests for a list of tests."); if (list_groups) { puts("Known tests are:"); - tinytest_set_flag_(groups, "..", 0); + tinytest_set_flag_(groups, "..", 1, 0); } exit(0); } +static int +process_test_alias(struct testgroup_t *groups, const char *test) +{ + int i, j, n, r; + for (i=0; cfg_aliases && cfg_aliases[i].name; ++i) { + if (!strcmp(cfg_aliases[i].name, test)) { + n = 0; + for (j = 0; cfg_aliases[i].tests[j]; ++j) { + r = process_test_option(groups, cfg_aliases[i].tests[j]); + if (r<0) + return -1; + n += r; + } + return n; + } + } + printf("No such test alias as @%s!",test); + return -1; +} + +static int +process_test_option(struct testgroup_t *groups, const char *test) +{ + int flag = TT_ENABLED_; + int n = 0; + if (test[0] == '@') { + return process_test_alias(groups, test + 1); + } else if (test[0] == ':') { + ++test; + flag = TT_SKIP; + } else if (test[0] == '+') { + ++test; + ++n; + if (!tinytest_set_flag_(groups, test, 0, TT_OFF_BY_DEFAULT)) { + printf("No such test as %s!\n", test); + return -1; + } + } else { + ++n; + } + if (!tinytest_set_flag_(groups, test, 1, flag)) { + printf("No such test as %s!\n", test); + return -1; + } + return n; +} + +void +tinytest_set_aliases(const struct testlist_alias_t *aliases) +{ + cfg_aliases = aliases; +} + int tinytest_main(int c, const char **v, struct testgroup_t *groups) { @@ -321,24 +414,18 @@ tinytest_main(int c, const char **v, struct testgroup_t *groups) return -1; } } else { - const char *test = v[i]; - int flag = TT_ENABLED_; - if (test[0] == ':') { - ++test; - flag = TT_SKIP; - } else { - ++n; - } - if (!tinytest_set_flag_(groups, test, flag)) { - printf("No such test as %s!\n", v[i]); + int r = process_test_option(groups, v[i]); + if (r<0) return -1; - } + n += r; } } if (!n) - tinytest_set_flag_(groups, "..", TT_ENABLED_); + tinytest_set_flag_(groups, "..", 1, TT_ENABLED_); +#ifdef _IONBF setvbuf(stdout, NULL, _IONBF, 0); +#endif ++in_tinytest_main; for (i=0; groups[i].prefix; ++i) @@ -385,3 +472,29 @@ tinytest_set_test_skipped_(void) cur_test_outcome = SKIP; } +char * +tinytest_format_hex_(const void *val_, unsigned long len) +{ + const unsigned char *val = val_; + char *result, *cp; + size_t i; + int ellipses = 0; + + if (!val) + return strdup("null"); + if (len > 1024) { + ellipses = 3; + len = 1024; + } + if (!(result = malloc(len*2+4))) + return strdup("<allocation failure>"); + cp = result; + for (i=0;i<len;++i) { + *cp++ = "0123456789ABCDEF"[val[i] >> 4]; + *cp++ = "0123456789ABCDEF"[val[i] & 0x0f]; + } + while (ellipses--) + *cp++ = '.'; + *cp = 0; + return result; +} diff --git a/src/ext/tinytest.h b/src/ext/tinytest.h index bcac9f079c..ed07b26bc0 100644 --- a/src/ext/tinytest.h +++ b/src/ext/tinytest.h @@ -32,8 +32,10 @@ #define TT_SKIP (1<<1) /** Internal runtime flag for a test we've decided to run. */ #define TT_ENABLED_ (1<<2) +/** Flag for a test that's off by default. */ +#define TT_OFF_BY_DEFAULT (1<<3) /** If you add your own flags, make them start at this point. */ -#define TT_FIRST_USER_FLAG (1<<3) +#define TT_FIRST_USER_FLAG (1<<4) typedef void (*testcase_fn)(void *); @@ -64,6 +66,12 @@ struct testgroup_t { }; #define END_OF_GROUPS { NULL, NULL} +struct testlist_alias_t { + const char *name; + const char **tests; +}; +#define END_OF_ALIASES { NULL, NULL } + /** Implementation: called from a test to indicate failure, before logging. */ void tinytest_set_test_failed_(void); /** Implementation: called from a test to indicate that we're skipping. */ @@ -72,14 +80,19 @@ void tinytest_set_test_skipped_(void); int tinytest_get_verbosity_(void); /** Implementation: Set a flag on tests matching a name; returns number * of tests that matched. */ -int tinytest_set_flag_(struct testgroup_t *, const char *, unsigned long); +int tinytest_set_flag_(struct testgroup_t *, const char *, int set, unsigned long); +/** Implementation: Put a chunk of memory into hex. */ +char *tinytest_format_hex_(const void *, unsigned long); /** Set all tests in 'groups' matching the name 'named' to be skipped. */ #define tinytest_skip(groups, named) \ - tinytest_set_flag_(groups, named, TT_SKIP) + tinytest_set_flag_(groups, named, 1, TT_SKIP) /** Run a single testcase in a single group. */ int testcase_run_one(const struct testgroup_t *,const struct testcase_t *); + +void tinytest_set_aliases(const struct testlist_alias_t *aliases); + /** Run a set of testcases from an END_OF_GROUPS-terminated array of groups, as selected from the command line. */ int tinytest_main(int argc, const char **argv, struct testgroup_t *groups); diff --git a/src/ext/tinytest_demo.c b/src/ext/tinytest_demo.c index be95ce4c1d..634e112cb8 100644 --- a/src/ext/tinytest_demo.c +++ b/src/ext/tinytest_demo.c @@ -35,6 +35,13 @@ #include <stdlib.h> #include <string.h> #include <errno.h> +#include <time.h> + +#ifdef _WIN32 +#include <windows.h> +#else +#include <unistd.h> +#endif /* ============================================================ */ @@ -148,6 +155,10 @@ test_memcpy(void *ptr) memcpy(db->buffer2, db->buffer1, sizeof(db->buffer1)); tt_str_op(db->buffer1, ==, db->buffer2); + /* tt_mem_op() does a memcmp, as opposed to the strcmp in tt_str_op() */ + db->buffer2[100] = 3; /* Make the buffers unequal */ + tt_mem_op(db->buffer1, <, db->buffer2, sizeof(db->buffer1)); + /* Now we've allocated memory that's referenced by a local variable. The end block of the function will clean it up. */ mem = strdup("Hello world."); @@ -162,6 +173,27 @@ test_memcpy(void *ptr) free(mem); } +void +test_timeout(void *ptr) +{ + time_t t1, t2; + (void)ptr; + t1 = time(NULL); +#ifdef _WIN32 + Sleep(5000); +#else + sleep(5); +#endif + t2 = time(NULL); + + tt_int_op(t2-t1, >=, 4); + + tt_int_op(t2-t1, <=, 6); + + end: + ; +} + /* ============================================================ */ /* Now we need to make sure that our tests get invoked. First, you take @@ -178,6 +210,10 @@ struct testcase_t demo_tests[] = { its environment. */ { "memcpy", test_memcpy, TT_FORK, &data_buffer_setup }, + /* This flag is off-by-default, since it takes a while to run. You + * can enable it manually by passing +demo/timeout at the command line.*/ + { "timeout", test_timeout, TT_OFF_BY_DEFAULT }, + /* The array has to end with END_OF_TESTCASES. */ END_OF_TESTCASES }; @@ -192,6 +228,18 @@ struct testgroup_t groups[] = { END_OF_GROUPS }; +/* We can also define test aliases. These can be used for types of tests that + * cut across groups. */ +const char *alltests[] = { "+..", NULL }; +const char *slowtests[] = { "+demo/timeout", NULL }; +struct testlist_alias_t aliases[] = { + + { "ALL", alltests }, + { "SLOW", slowtests }, + + END_OF_ALIASES +}; + int main(int c, const char **v) @@ -211,5 +259,6 @@ main(int c, const char **v) "tinytest-demo" and "tinytest-demo .." mean the same thing. */ + tinytest_set_aliases(aliases); return tinytest_main(c, v, groups); } diff --git a/src/ext/tinytest_macros.h b/src/ext/tinytest_macros.h index 9ff69b1d50..c3728d1fdd 100644 --- a/src/ext/tinytest_macros.h +++ b/src/ext/tinytest_macros.h @@ -113,8 +113,8 @@ #define tt_assert_test_fmt_type(a,b,str_test,type,test,printf_type,printf_fmt, \ setup_block,cleanup_block,die_on_fail) \ TT_STMT_BEGIN \ - type val1_ = (type)(a); \ - type val2_ = (type)(b); \ + type val1_ = (a); \ + type val2_ = (b); \ int tt_status_ = (test); \ if (!tt_status_ || tinytest_get_verbosity_()>1) { \ printf_type print_; \ @@ -144,6 +144,10 @@ tt_assert_test_fmt_type(a,b,str_test,type,test,type,fmt, \ {print_=value_;},{},die_on_fail) +#define tt_assert_test_type_opt(a,b,str_test,type,test,fmt,die_on_fail) \ + tt_assert_test_fmt_type(a,b,str_test,type,test,type,fmt, \ + {print_=value_?value_:"<NULL>";},{},die_on_fail) + /* Helper: assert that a op b, when cast to type. Format the values with * printf format fmt on failure. */ #define tt_assert_op_type(a,op,b,type,fmt) \ @@ -159,12 +163,23 @@ (val1_ op val2_),"%lu",TT_EXIT_TEST_FUNCTION) #define tt_ptr_op(a,op,b) \ - tt_assert_test_type(a,b,#a" "#op" "#b,void*, \ + tt_assert_test_type(a,b,#a" "#op" "#b,const void*, \ (val1_ op val2_),"%p",TT_EXIT_TEST_FUNCTION) #define tt_str_op(a,op,b) \ - tt_assert_test_type(a,b,#a" "#op" "#b,const char *, \ - (strcmp(val1_,val2_) op 0),"<%s>",TT_EXIT_TEST_FUNCTION) + tt_assert_test_type_opt(a,b,#a" "#op" "#b,const char *, \ + (val1_ && val2_ && strcmp(val1_,val2_) op 0),"<%s>", \ + TT_EXIT_TEST_FUNCTION) + +#define tt_mem_op(expr1, op, expr2, len) \ + tt_assert_test_fmt_type(expr1,expr2,#expr1" "#op" "#expr2, \ + const void *, \ + (val1_ && val2_ && memcmp(val1_, val2_, len) op 0), \ + char *, "%s", \ + { print_ = tinytest_format_hex_(value_, (len)); }, \ + { if (print_) free(print_); }, \ + TT_EXIT_TEST_FUNCTION \ + ); #define tt_want_int_op(a,op,b) \ tt_assert_test_type(a,b,#a" "#op" "#b,long,(val1_ op val2_),"%ld",(void)0) @@ -174,7 +189,7 @@ (val1_ op val2_),"%lu",(void)0) #define tt_want_ptr_op(a,op,b) \ - tt_assert_test_type(a,b,#a" "#op" "#b,void*, \ + tt_assert_test_type(a,b,#a" "#op" "#b,const void*, \ (val1_ op val2_),"%p",(void)0) #define tt_want_str_op(a,op,b) \ diff --git a/src/or/addressmap.c b/src/or/addressmap.c index 79e4b7c5e2..998770a3db 100644 --- a/src/or/addressmap.c +++ b/src/or/addressmap.c @@ -45,7 +45,7 @@ typedef struct { char *new_address; time_t expires; - ENUM_BF(addressmap_entry_source_t) source:3; + addressmap_entry_source_bitfield_t source:3; unsigned src_wildcard:1; unsigned dst_wildcard:1; short num_resolve_failures; @@ -798,7 +798,7 @@ address_is_in_virtual_range(const char *address) /** Return a random address conforming to the virtual address configuration * in <b>conf</b>. */ -/* private */ void +STATIC void get_random_virtual_addr(const virtual_addr_conf_t *conf, tor_addr_t *addr_out) { uint8_t tmp[4]; diff --git a/src/or/addressmap.h b/src/or/addressmap.h index 40210ee990..417832b31f 100644 --- a/src/or/addressmap.h +++ b/src/or/addressmap.h @@ -7,6 +7,8 @@ #ifndef TOR_ADDRESSMAP_H #define TOR_ADDRESSMAP_H +#include "testsupport.h" + void addressmap_init(void); void addressmap_clear_excluded_trackexithosts(const or_options_t *options); void addressmap_clear_invalid_automaps(const or_options_t *options); @@ -52,8 +54,8 @@ typedef struct virtual_addr_conf_t { maskbits_t bits; } virtual_addr_conf_t; -void get_random_virtual_addr(const virtual_addr_conf_t *conf, - tor_addr_t *addr_out); +STATIC void get_random_virtual_addr(const virtual_addr_conf_t *conf, + tor_addr_t *addr_out); #endif #endif diff --git a/src/or/buffers.c b/src/or/buffers.c index c4c847ec87..033f86288e 100644 --- a/src/or/buffers.c +++ b/src/or/buffers.c @@ -19,6 +19,7 @@ #include "connection_or.h" #include "control.h" #include "reasons.h" +#include "ext_orport.h" #include "../common/util.h" #include "../common/torlog.h" #ifdef HAVE_UNISTD_H @@ -63,16 +64,6 @@ static int parse_socks_client(const uint8_t *data, size_t datalen, /* Chunk manipulation functions */ -/** A single chunk on a buffer or in a freelist. */ -typedef struct chunk_t { - struct chunk_t *next; /**< The next chunk on the buffer or freelist. */ - size_t datalen; /**< The number of bytes stored in this chunk */ - size_t memlen; /**< The number of usable bytes of storage in <b>mem</b>. */ - char *data; /**< A pointer to the first byte of data stored in <b>mem</b>. */ - char mem[FLEXIBLE_ARRAY_MEMBER]; /**< The actual memory used for storage in - * this chunk. */ -} chunk_t; - #define CHUNK_HEADER_LEN STRUCT_OFFSET(chunk_t, mem[0]) /** Return the number of bytes needed to allocate a chunk to hold @@ -109,6 +100,9 @@ chunk_repack(chunk_t *chunk) chunk->data = &chunk->mem[0]; } +/** Keep track of total size of allocated chunks for consistency asserts */ +static size_t total_bytes_allocated_in_chunks = 0; + #if defined(ENABLE_BUF_FREELISTS) || defined(RUNNING_DOXYGEN) /** A freelist of chunks. */ typedef struct chunk_freelist_t { @@ -173,6 +167,11 @@ chunk_free_unchecked(chunk_t *chunk) } else { if (freelist) ++freelist->n_free; +#ifdef DEBUG_CHUNK_ALLOC + tor_assert(alloc == chunk->DBG_alloc); +#endif + tor_assert(total_bytes_allocated_in_chunks >= alloc); + total_bytes_allocated_in_chunks -= alloc; tor_free(chunk); } } @@ -199,6 +198,10 @@ chunk_new_with_alloc_size(size_t alloc) else ++n_freelist_miss; ch = tor_malloc(alloc); +#ifdef DEBUG_CHUNK_ALLOC + ch->DBG_alloc = alloc; +#endif + total_bytes_allocated_in_chunks += alloc; } ch->next = NULL; ch->datalen = 0; @@ -210,6 +213,14 @@ chunk_new_with_alloc_size(size_t alloc) static void chunk_free_unchecked(chunk_t *chunk) { + if (!chunk) + return; +#ifdef DEBUG_CHUNK_ALLOC + tor_assert(CHUNK_ALLOC_SIZE(chunk->memlen) == chunk->DBG_alloc); +#endif + tor_assert(total_bytes_allocated_in_chunks >= + CHUNK_ALLOC_SIZE(chunk->memlen)); + total_bytes_allocated_in_chunks -= CHUNK_ALLOC_SIZE(chunk->memlen); tor_free(chunk); } static INLINE chunk_t * @@ -219,7 +230,11 @@ chunk_new_with_alloc_size(size_t alloc) ch = tor_malloc(alloc); ch->next = NULL; ch->datalen = 0; +#ifdef DEBUG_CHUNK_ALLOC + ch->DBG_alloc = alloc; +#endif ch->memlen = CHUNK_SIZE_WITH_ALLOC(alloc); + total_bytes_allocated_in_chunks += alloc; ch->data = &ch->mem[0]; return ch; } @@ -231,11 +246,18 @@ static INLINE chunk_t * chunk_grow(chunk_t *chunk, size_t sz) { off_t offset; + size_t memlen_orig = chunk->memlen; tor_assert(sz > chunk->memlen); offset = chunk->data - chunk->mem; chunk = tor_realloc(chunk, CHUNK_ALLOC_SIZE(sz)); chunk->memlen = sz; chunk->data = chunk->mem + offset; +#ifdef DEBUG_CHUNK_ALLOC + tor_assert(chunk->DBG_alloc == CHUNK_ALLOC_SIZE(memlen_orig)); + chunk->DBG_alloc = CHUNK_ALLOC_SIZE(sz); +#endif + total_bytes_allocated_in_chunks += + CHUNK_ALLOC_SIZE(sz) - CHUNK_ALLOC_SIZE(memlen_orig); return chunk; } @@ -260,12 +282,14 @@ preferred_chunk_size(size_t target) } /** Remove from the freelists most chunks that have not been used since the - * last call to buf_shrink_freelists(). */ -void + * last call to buf_shrink_freelists(). Return the amount of memory + * freed. */ +size_t buf_shrink_freelists(int free_all) { #ifdef ENABLE_BUF_FREELISTS int i; + size_t total_freed = 0; disable_control_logging(); for (i = 0; freelists[i].alloc_size; ++i) { int slack = freelists[i].slack; @@ -281,7 +305,7 @@ buf_shrink_freelists(int free_all) chunk_t **chp = &freelists[i].head; chunk_t *chunk; while (n_to_skip) { - if (! (*chp)->next) { + if (!(*chp) || ! (*chp)->next) { log_warn(LD_BUG, "I wanted to skip %d chunks in the freelist for " "%d-byte chunks, but only found %d. (Length %d)", orig_n_to_skip, (int)freelists[i].alloc_size, @@ -297,6 +321,13 @@ buf_shrink_freelists(int free_all) *chp = NULL; while (chunk) { chunk_t *next = chunk->next; +#ifdef DEBUG_CHUNK_ALLOC + tor_assert(chunk->DBG_alloc == CHUNK_ALLOC_SIZE(chunk->memlen)); +#endif + tor_assert(total_bytes_allocated_in_chunks >= + CHUNK_ALLOC_SIZE(chunk->memlen)); + total_bytes_allocated_in_chunks -= CHUNK_ALLOC_SIZE(chunk->memlen); + total_freed += CHUNK_ALLOC_SIZE(chunk->memlen); tor_free(chunk); chunk = next; --n_to_free; @@ -314,18 +345,21 @@ buf_shrink_freelists(int free_all) } // tor_assert(!n_to_free); freelists[i].cur_length = new_length; + tor_assert(orig_n_to_skip == new_length); log_info(LD_MM, "Cleaned freelist for %d-byte chunks: original " - "length %d, kept %d, dropped %d.", + "length %d, kept %d, dropped %d. New length is %d", (int)freelists[i].alloc_size, orig_length, - orig_n_to_skip, orig_n_to_free); + orig_n_to_skip, orig_n_to_free, new_length); } freelists[i].lowest_length = freelists[i].cur_length; assert_freelist_ok(&freelists[i]); } done: enable_control_logging(); + return total_freed; #else (void) free_all; + return 0; #endif } @@ -356,28 +390,16 @@ buf_dump_freelist_sizes(int severity) #endif } -/** Magic value for buf_t.magic, to catch pointer errors. */ -#define BUFFER_MAGIC 0xB0FFF312u -/** A resizeable buffer, optimized for reading and writing. */ -struct buf_t { - uint32_t magic; /**< Magic cookie for debugging: Must be set to - * BUFFER_MAGIC. */ - size_t datalen; /**< How many bytes is this buffer holding right now? */ - size_t default_chunk_size; /**< Don't allocate any chunks smaller than - * this for this buffer. */ - chunk_t *head; /**< First chunk in the list, or NULL for none. */ - chunk_t *tail; /**< Last chunk in the list, or NULL for none. */ -}; - /** Collapse data from the first N chunks from <b>buf</b> into buf->head, * growing it as necessary, until buf->head has the first <b>bytes</b> bytes * of data from the buffer, or until buf->head has all the data in <b>buf</b>. * * If <b>nulterminate</b> is true, ensure that there is a 0 byte in * buf->head->mem right after all the data. */ -static void +STATIC void buf_pullup(buf_t *buf, size_t bytes, int nulterminate) { + /* XXXX nothing uses nulterminate; remove it. */ chunk_t *dest, *src; size_t capacity; if (!buf->head) @@ -449,6 +471,20 @@ buf_pullup(buf_t *buf, size_t bytes, int nulterminate) check(); } +#ifdef TOR_UNIT_TESTS +void +buf_get_first_chunk_data(const buf_t *buf, const char **cp, size_t *sz) +{ + if (!buf || !buf->head) { + *cp = NULL; + *sz = 0; + } else { + *cp = buf->head->data; + *sz = buf->head->datalen; + } +} +#endif + /** Resize buf so it won't hold extra memory that we haven't been * using lately. */ @@ -503,6 +539,12 @@ buf_new(void) return buf; } +size_t +buf_get_default_chunk_size(const buf_t *buf) +{ + return buf->default_chunk_size; +} + /** Remove all data from <b>buf</b>. */ void buf_clear(buf_t *buf) @@ -530,7 +572,7 @@ buf_allocation(const buf_t *buf) size_t total = 0; const chunk_t *chunk; for (chunk = buf->head; chunk; chunk = chunk->next) { - total += chunk->memlen; + total += CHUNK_ALLOC_SIZE(chunk->memlen); } return total; } @@ -563,6 +605,10 @@ static chunk_t * chunk_copy(const chunk_t *in_chunk) { chunk_t *newch = tor_memdup(in_chunk, CHUNK_ALLOC_SIZE(in_chunk->memlen)); + total_bytes_allocated_in_chunks += CHUNK_ALLOC_SIZE(in_chunk->memlen); +#ifdef DEBUG_CHUNK_ALLOC + newch->DBG_alloc = CHUNK_ALLOC_SIZE(in_chunk->memlen); +#endif newch->next = NULL; if (in_chunk->data) { off_t offset = in_chunk->data - in_chunk->mem; @@ -598,6 +644,7 @@ static chunk_t * buf_add_chunk_with_capacity(buf_t *buf, size_t capacity, int capped) { chunk_t *chunk; + struct timeval now; if (CHUNK_ALLOC_SIZE(capacity) < buf->default_chunk_size) { chunk = chunk_new_with_alloc_size(buf->default_chunk_size); } else if (capped && CHUNK_ALLOC_SIZE(capacity) > MAX_CHUNK_ALLOC) { @@ -605,6 +652,10 @@ buf_add_chunk_with_capacity(buf_t *buf, size_t capacity, int capped) } else { chunk = chunk_new_with_alloc_size(preferred_chunk_size(capacity)); } + + tor_gettimeofday_cached_monotonic(&now); + chunk->inserted_time = (uint32_t)tv_to_msec(&now); + if (buf->tail) { tor_assert(buf->head); buf->tail->next = chunk; @@ -617,6 +668,26 @@ buf_add_chunk_with_capacity(buf_t *buf, size_t capacity, int capped) return chunk; } +/** Return the age of the oldest chunk in the buffer <b>buf</b>, in + * milliseconds. Requires the current time, in truncated milliseconds since + * the epoch, as its input <b>now</b>. + */ +uint32_t +buf_get_oldest_chunk_timestamp(const buf_t *buf, uint32_t now) +{ + if (buf->head) { + return now - buf->head->inserted_time; + } else { + return 0; + } +} + +size_t +buf_get_total_allocation(void) +{ + return total_bytes_allocated_in_chunks; +} + /** Read up to <b>at_most</b> bytes from the socket <b>fd</b> into * <b>chunk</b> (which must be on <b>buf</b>). If we get an EOF, set * *<b>reached_eof</b> to 1. Return -1 on error, 0 on eof or blocking, @@ -1294,7 +1365,7 @@ buf_matches_at_pos(const buf_pos_t *pos, const char *s, size_t n) /** Return the first position in <b>buf</b> at which the <b>n</b>-character * string <b>s</b> occurs, or -1 if it does not occur. */ -/*private*/ int +STATIC int buf_find_string_offset(const buf_t *buf, const char *s, size_t n) { buf_pos_t pos; @@ -1702,6 +1773,64 @@ fetch_from_evbuffer_socks(struct evbuffer *buf, socks_request_t *req, } #endif +/** The size of the header of an Extended ORPort message: 2 bytes for + * COMMAND, 2 bytes for BODYLEN */ +#define EXT_OR_CMD_HEADER_SIZE 4 + +/** Read <b>buf</b>, which should contain an Extended ORPort message + * from a transport proxy. If well-formed, create and populate + * <b>out</b> with the Extended ORport message. Return 0 if the + * buffer was incomplete, 1 if it was well-formed and -1 if we + * encountered an error while parsing it. */ +int +fetch_ext_or_command_from_buf(buf_t *buf, ext_or_cmd_t **out) +{ + char hdr[EXT_OR_CMD_HEADER_SIZE]; + uint16_t len; + + check(); + if (buf->datalen < EXT_OR_CMD_HEADER_SIZE) + return 0; + peek_from_buf(hdr, sizeof(hdr), buf); + len = ntohs(get_uint16(hdr+2)); + if (buf->datalen < (unsigned)len + EXT_OR_CMD_HEADER_SIZE) + return 0; + *out = ext_or_cmd_new(len); + (*out)->cmd = ntohs(get_uint16(hdr)); + (*out)->len = len; + buf_remove_from_front(buf, EXT_OR_CMD_HEADER_SIZE); + fetch_from_buf((*out)->body, len, buf); + return 1; +} + +#ifdef USE_BUFFEREVENTS +/** Read <b>buf</b>, which should contain an Extended ORPort message + * from a transport proxy. If well-formed, create and populate + * <b>out</b> with the Extended ORport message. Return 0 if the + * buffer was incomplete, 1 if it was well-formed and -1 if we + * encountered an error while parsing it. */ +int +fetch_ext_or_command_from_evbuffer(struct evbuffer *buf, ext_or_cmd_t **out) +{ + char hdr[EXT_OR_CMD_HEADER_SIZE]; + uint16_t len; + size_t buf_len = evbuffer_get_length(buf); + + if (buf_len < EXT_OR_CMD_HEADER_SIZE) + return 0; + evbuffer_copyout(buf, hdr, EXT_OR_CMD_HEADER_SIZE); + len = ntohs(get_uint16(hdr+2)); + if (buf_len < (unsigned)len + EXT_OR_CMD_HEADER_SIZE) + return 0; + *out = ext_or_cmd_new(len); + (*out)->cmd = ntohs(get_uint16(hdr)); + (*out)->len = len; + evbuffer_drain(buf, EXT_OR_CMD_HEADER_SIZE); + evbuffer_remove(buf, (*out)->body, len); + return 1; +} +#endif + /** Implementation helper to implement fetch_from_*_socks. Instead of looking * at a buffer's contents, we look at the <b>datalen</b> bytes of data in * <b>data</b>. Instead of removing data from the buffer, we set @@ -2348,6 +2477,7 @@ write_to_buf_zlib(buf_t *buf, tor_zlib_state_t *state, char *next; size_t old_avail, avail; int over = 0; + do { int need_new_chunk = 0; if (!buf->tail || ! CHUNK_REMAINING_CAPACITY(buf->tail)) { diff --git a/src/or/buffers.h b/src/or/buffers.h index c947f0ba98..c90e14750e 100644 --- a/src/or/buffers.h +++ b/src/or/buffers.h @@ -12,19 +12,25 @@ #ifndef TOR_BUFFERS_H #define TOR_BUFFERS_H +#include "testsupport.h" + buf_t *buf_new(void); buf_t *buf_new_with_capacity(size_t size); +size_t buf_get_default_chunk_size(const buf_t *buf); void buf_free(buf_t *buf); void buf_clear(buf_t *buf); buf_t *buf_copy(const buf_t *buf); void buf_shrink(buf_t *buf); -void buf_shrink_freelists(int free_all); +size_t buf_shrink_freelists(int free_all); void buf_dump_freelist_sizes(int severity); size_t buf_datalen(const buf_t *buf); size_t buf_allocation(const buf_t *buf); size_t buf_slack(const buf_t *buf); +uint32_t buf_get_oldest_chunk_timestamp(const buf_t *buf, uint32_t now); +size_t buf_get_total_allocation(void); + int read_to_buf(tor_socket_t s, size_t at_most, buf_t *buf, int *reached_eof, int *socket_error); int read_to_buf_tls(tor_tls_t *tls, size_t at_most, buf_t *buf); @@ -51,6 +57,8 @@ int fetch_from_buf_line(buf_t *buf, char *data_out, size_t *data_len); int peek_buf_has_control0_command(buf_t *buf); +int fetch_ext_or_command_from_buf(buf_t *buf, ext_or_cmd_t **out); + #ifdef USE_BUFFEREVENTS int fetch_var_cell_from_evbuffer(struct evbuffer *buf, var_cell_t **out, int linkproto); @@ -66,6 +74,8 @@ int peek_evbuffer_has_control0_command(struct evbuffer *buf); int write_to_evbuffer_zlib(struct evbuffer *buf, tor_zlib_state_t *state, const char *data, size_t data_len, int done); +int fetch_ext_or_command_from_evbuffer(struct evbuffer *buf, + ext_or_cmd_t **out); #endif #ifdef USE_BUFFEREVENTS @@ -75,6 +85,8 @@ int write_to_evbuffer_zlib(struct evbuffer *buf, tor_zlib_state_t *state, #define generic_buffer_get(b,buf,buflen) evbuffer_remove((b),(buf),(buflen)) #define generic_buffer_clear(b) evbuffer_drain((b), evbuffer_get_length((b))) #define generic_buffer_free(b) evbuffer_free((b)) +#define generic_buffer_fetch_ext_or_cmd(b, out) \ + fetch_ext_or_command_from_evbuffer((b), (out)) #else #define generic_buffer_new() buf_new() #define generic_buffer_len(b) buf_datalen((b)) @@ -82,6 +94,8 @@ int write_to_evbuffer_zlib(struct evbuffer *buf, tor_zlib_state_t *state, #define generic_buffer_get(b,buf,buflen) fetch_from_buf((buf),(buflen),(b)) #define generic_buffer_clear(b) buf_clear((b)) #define generic_buffer_free(b) buf_free((b)) +#define generic_buffer_fetch_ext_or_cmd(b, out) \ + fetch_ext_or_command_from_buf((b), (out)) #endif int generic_buffer_set_to_copy(generic_buffer_t **output, const generic_buffer_t *input); @@ -89,7 +103,38 @@ int generic_buffer_set_to_copy(generic_buffer_t **output, void assert_buf_ok(buf_t *buf); #ifdef BUFFERS_PRIVATE -int buf_find_string_offset(const buf_t *buf, const char *s, size_t n); +STATIC int buf_find_string_offset(const buf_t *buf, const char *s, size_t n); +STATIC void buf_pullup(buf_t *buf, size_t bytes, int nulterminate); +void buf_get_first_chunk_data(const buf_t *buf, const char **cp, size_t *sz); + +#define DEBUG_CHUNK_ALLOC +/** A single chunk on a buffer or in a freelist. */ +typedef struct chunk_t { + struct chunk_t *next; /**< The next chunk on the buffer or freelist. */ + size_t datalen; /**< The number of bytes stored in this chunk */ + size_t memlen; /**< The number of usable bytes of storage in <b>mem</b>. */ +#ifdef DEBUG_CHUNK_ALLOC + size_t DBG_alloc; +#endif + char *data; /**< A pointer to the first byte of data stored in <b>mem</b>. */ + uint32_t inserted_time; /**< Timestamp in truncated ms since epoch + * when this chunk was inserted. */ + char mem[FLEXIBLE_ARRAY_MEMBER]; /**< The actual memory used for storage in + * this chunk. */ +} chunk_t; + +/** Magic value for buf_t.magic, to catch pointer errors. */ +#define BUFFER_MAGIC 0xB0FFF312u +/** A resizeable buffer, optimized for reading and writing. */ +struct buf_t { + uint32_t magic; /**< Magic cookie for debugging: Must be set to + * BUFFER_MAGIC. */ + size_t datalen; /**< How many bytes is this buffer holding right now? */ + size_t default_chunk_size; /**< Don't allocate any chunks smaller than + * this for this buffer. */ + chunk_t *head; /**< First chunk in the list, or NULL for none. */ + chunk_t *tail; /**< Last chunk in the list, or NULL for none. */ +}; #endif #endif diff --git a/src/or/channel.c b/src/or/channel.c index cd55bd0dfd..ffd68493d0 100644 --- a/src/or/channel.c +++ b/src/or/channel.c @@ -19,6 +19,7 @@ #include "circuitbuild.h" #include "circuitlist.h" #include "circuitstats.h" +#include "config.h" #include "connection_or.h" /* For var_cell_free() */ #include "circuitmux.h" #include "entrynodes.h" @@ -95,12 +96,7 @@ typedef struct channel_idmap_entry_s { static INLINE unsigned channel_idmap_hash(const channel_idmap_entry_t *ent) { - const unsigned *a = (const unsigned *)ent->digest; -#if SIZEOF_INT == 4 - return a[0] ^ a[1] ^ a[2] ^ a[3] ^ a[4]; -#elif SIZEOF_INT == 8 - return a[0] ^ a[1]; -#endif + return (unsigned) siphash24g(ent->digest, DIGEST_LEN); } static INLINE int @@ -117,11 +113,15 @@ HT_GENERATE(channel_idmap, channel_idmap_entry_s, node, channel_idmap_hash, static cell_queue_entry_t * cell_queue_entry_dup(cell_queue_entry_t *q); static void cell_queue_entry_free(cell_queue_entry_t *q, int handed_off); +#if 0 static int cell_queue_entry_is_padding(cell_queue_entry_t *q); +#endif static cell_queue_entry_t * cell_queue_entry_new_fixed(cell_t *cell); static cell_queue_entry_t * cell_queue_entry_new_var(var_cell_t *var_cell); +static int is_destroy_cell(channel_t *chan, + const cell_queue_entry_t *q, circid_t *circid_out); /* Functions to maintain the digest map */ static void channel_add_to_digest_map(channel_t *chan); @@ -729,10 +729,10 @@ channel_init(channel_t *chan) chan->global_identifier = n_channels_allocated++; /* Init timestamp */ - chan->timestamp_last_added_nonpadding = time(NULL); + chan->timestamp_last_had_circuits = time(NULL); - /* Init next_circ_id */ - chan->next_circ_id = crypto_rand_int(1 << 15); + /* Warn about exhausted circuit IDs no more than hourly. */ + chan->last_warned_circ_ids_exhausted.rate = 3600; /* Initialize queues. */ TOR_SIMPLEQ_INIT(&chan->incoming_queue); @@ -803,7 +803,8 @@ channel_free(channel_t *chan) /* Get rid of cmux */ if (chan->cmux) { - circuitmux_detach_all_circuits(chan->cmux); + circuitmux_detach_all_circuits(chan->cmux, NULL); + circuitmux_mark_destroyed_circids_usable(chan->cmux, chan); circuitmux_free(chan->cmux); chan->cmux = NULL; } @@ -1597,6 +1598,7 @@ cell_queue_entry_free(cell_queue_entry_t *q, int handed_off) tor_free(q); } +#if 0 /** * Check whether a cell queue entry is padding; this is a helper function * for channel_write_cell_queue_entry() @@ -1625,6 +1627,7 @@ cell_queue_entry_is_padding(cell_queue_entry_t *q) return 0; } +#endif /** * Allocate a new cell queue entry for a fixed-size cell @@ -1683,9 +1686,11 @@ channel_write_cell_queue_entry(channel_t *chan, cell_queue_entry_t *q) chan->state == CHANNEL_STATE_OPEN || chan->state == CHANNEL_STATE_MAINT); - /* Increment the timestamp unless it's padding */ - if (!cell_queue_entry_is_padding(q)) { - chan->timestamp_last_added_nonpadding = approx_time(); + { + circid_t circ_id; + if (is_destroy_cell(chan, q, &circ_id)) { + channel_note_destroy_not_pending(chan, circ_id); + } } /* Can we send it right out? If so, try */ @@ -2355,7 +2360,7 @@ channel_do_open_actions(channel_t *chan) started_here = channel_is_outgoing(chan); if (started_here) { - circuit_build_times_network_is_live(&circ_times); + circuit_build_times_network_is_live(get_circuit_build_times_mutable()); rep_hist_note_connect_succeeded(chan->identity_digest, now); if (entry_guard_register_connect_status( chan->identity_digest, 1, 0, now) < 0) { @@ -2373,8 +2378,14 @@ channel_do_open_actions(channel_t *chan) /* only report it to the geoip module if it's not a known router */ if (!router_get_by_id_digest(chan->identity_digest)) { if (channel_get_addr_if_possible(chan, &remote_addr)) { - geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &remote_addr, + char *transport_name = NULL; + if (chan->get_transport_name(chan, &transport_name) < 0) + transport_name = NULL; + + geoip_note_client_seen(GEOIP_CLIENT_CONNECT, + &remote_addr, transport_name, now); + tor_free(transport_name); } /* Otherwise the underlying transport can't tell us this, so skip it */ } @@ -2611,6 +2622,54 @@ channel_queue_var_cell(channel_t *chan, var_cell_t *var_cell) } } +/** If <b>packed_cell</b> on <b>chan</b> is a destroy cell, then set + * *<b>circid_out</b> to its circuit ID, and return true. Otherwise, return + * false. */ +/* XXXX Move this function. */ +int +packed_cell_is_destroy(channel_t *chan, + const packed_cell_t *packed_cell, + circid_t *circid_out) +{ + if (chan->wide_circ_ids) { + if (packed_cell->body[4] == CELL_DESTROY) { + *circid_out = ntohl(get_uint32(packed_cell->body)); + return 1; + } + } else { + if (packed_cell->body[2] == CELL_DESTROY) { + *circid_out = ntohs(get_uint16(packed_cell->body)); + return 1; + } + } + return 0; +} + +/** DOCDOC */ +static int +is_destroy_cell(channel_t *chan, + const cell_queue_entry_t *q, circid_t *circid_out) +{ + *circid_out = 0; + switch (q->type) { + case CELL_QUEUE_FIXED: + if (q->u.fixed.cell->command == CELL_DESTROY) { + *circid_out = q->u.fixed.cell->circ_id; + return 1; + } + break; + case CELL_QUEUE_VAR: + if (q->u.var.var_cell->command == CELL_DESTROY) { + *circid_out = q->u.var.var_cell->circ_id; + return 1; + } + break; + case CELL_QUEUE_PACKED: + return packed_cell_is_destroy(chan, q->u.packed.packed_cell, circid_out); + } + return 0; +} + /** * Send destroy cell on a channel * @@ -2622,8 +2681,6 @@ channel_queue_var_cell(channel_t *chan, var_cell_t *var_cell) int channel_send_destroy(circid_t circ_id, channel_t *chan, int reason) { - cell_t cell; - tor_assert(chan); if (circ_id == 0) { log_warn(LD_BUG, "Attempted to send a destroy cell for circID 0 " @@ -2637,18 +2694,15 @@ channel_send_destroy(circid_t circ_id, channel_t *chan, int reason) /* Check to make sure we can send on this channel first */ if (!(chan->state == CHANNEL_STATE_CLOSING || chan->state == CHANNEL_STATE_CLOSED || - chan->state == CHANNEL_STATE_ERROR)) { - memset(&cell, 0, sizeof(cell_t)); - cell.circ_id = circ_id; - cell.command = CELL_DESTROY; - cell.payload[0] = (uint8_t) reason; + chan->state == CHANNEL_STATE_ERROR) && + chan->cmux) { + channel_note_destroy_pending(chan, circ_id); + circuitmux_append_destroy_cell(chan, chan->cmux, circ_id, reason); log_debug(LD_OR, "Sending destroy (circID %u) on channel %p " "(global ID " U64_FORMAT ")", (unsigned)circ_id, chan, U64_PRINTF_ARG(chan->global_identifier)); - - channel_write_cell(chan, &cell); } else { log_warn(LD_BUG, "Someone called channel_send_destroy() for circID %u " @@ -2814,7 +2868,7 @@ channel_free_list(smartlist_t *channels, int mark_for_close) channel_state_to_string(curr->state), curr->state); /* Detach circuits early so they can find the channel */ if (curr->cmux) { - circuitmux_detach_all_circuits(curr->cmux); + circuitmux_detach_all_circuits(curr->cmux, NULL); } channel_unregister(curr); if (mark_for_close) { @@ -3232,9 +3286,9 @@ channel_dump_statistics(channel_t *chan, int severity) " is %s, and gives a canonical description of \"%s\" and an " "actual description of \"%s\"", U64_PRINTF_ARG(chan->global_identifier), - remote_addr_str, - channel_get_canonical_remote_descr(chan), - actual); + safe_str(remote_addr_str), + safe_str(channel_get_canonical_remote_descr(chan)), + safe_str(actual)); tor_free(remote_addr_str); tor_free(actual); } else { @@ -3306,7 +3360,7 @@ channel_dump_statistics(channel_t *chan, int severity) U64_PRINTF_ARG(chan->timestamp_recv), U64_PRINTF_ARG(now - chan->timestamp_recv)); tor_log(severity, LD_GENERAL, - " * Channel " U64_FORMAT " last trasmitted a cell " + " * Channel " U64_FORMAT " last transmitted a cell " "at " U64_FORMAT " (" U64_FORMAT " seconds ago)", U64_PRINTF_ARG(chan->global_identifier), U64_PRINTF_ARG(chan->timestamp_xmit), diff --git a/src/or/channel.h b/src/or/channel.h index 29ba40e326..3e164c6892 100644 --- a/src/or/channel.h +++ b/src/or/channel.h @@ -10,7 +10,6 @@ #define TOR_CHANNEL_H #include "or.h" -#include "tor_queue.h" #include "circuitmux.h" /* Channel handler function pointer typedefs */ @@ -22,7 +21,7 @@ struct cell_queue_entry_s; TOR_SIMPLEQ_HEAD(chan_cell_queue, cell_queue_entry_s) incoming_queue; typedef struct chan_cell_queue chan_cell_queue_t; -/* +/** * Channel struct; see the channel_t typedef in or.h. A channel is an * abstract interface for the OR-to-OR connection, similar to connection_or_t, * but without the strong coupling to the underlying TLS implementation. They @@ -32,18 +31,18 @@ typedef struct chan_cell_queue chan_cell_queue_t; */ struct channel_s { - /* Magic number for type-checking cast macros */ + /** Magic number for type-checking cast macros */ uint32_t magic; - /* Current channel state */ + /** Current channel state */ channel_state_t state; - /* Globally unique ID number for a channel over the lifetime of a Tor + /** Globally unique ID number for a channel over the lifetime of a Tor * process. */ uint64_t global_identifier; - /* Should we expect to see this channel in the channel lists? */ + /** Should we expect to see this channel in the channel lists? */ unsigned char registered:1; /** has this channel ever been open? */ @@ -58,28 +57,28 @@ struct channel_s { CHANNEL_CLOSE_FOR_ERROR } reason_for_closing; - /* Timestamps for both cell channels and listeners */ + /** Timestamps for both cell channels and listeners */ time_t timestamp_created; /* Channel created */ time_t timestamp_active; /* Any activity */ /* Methods implemented by the lower layer */ - /* Free a channel */ + /** Free a channel */ void (*free)(channel_t *); - /* Close an open channel */ + /** Close an open channel */ void (*close)(channel_t *); - /* Describe the transport subclass for this channel */ + /** Describe the transport subclass for this channel */ const char * (*describe_transport)(channel_t *); - /* Optional method to dump transport-specific statistics on the channel */ + /** Optional method to dump transport-specific statistics on the channel */ void (*dumpstats)(channel_t *, int); - /* Registered handlers for incoming cells */ + /** Registered handlers for incoming cells */ channel_cell_handler_fn_ptr cell_handler; channel_var_cell_handler_fn_ptr var_cell_handler; /* Methods implemented by the lower layer */ - /* + /** * Ask the underlying transport what the remote endpoint address is, in * a tor_addr_t. This is optional and subclasses may leave this NULL. * If they implement it, they should write the address out to the @@ -87,79 +86,74 @@ struct channel_s { * available. */ int (*get_remote_addr)(channel_t *, tor_addr_t *); + int (*get_transport_name)(channel_t *chan, char **transport_out); + #define GRD_FLAG_ORIGINAL 1 #define GRD_FLAG_ADDR_ONLY 2 - /* + /** * Get a text description of the remote endpoint; canonicalized if the flag * GRD_FLAG_ORIGINAL is not set, or the one we originally connected * to/received from if it is. If GRD_FLAG_ADDR_ONLY is set, we return only * the original address. */ const char * (*get_remote_descr)(channel_t *, int); - /* Check if the lower layer has queued writes */ + /** Check if the lower layer has queued writes */ int (*has_queued_writes)(channel_t *); - /* + /** * If the second param is zero, ask the lower layer if this is * 'canonical', for a transport-specific definition of canonical; if * it is 1, ask if the answer to the preceding query is safe to rely * on. */ int (*is_canonical)(channel_t *, int); - /* Check if this channel matches a specified extend_info_t */ + /** Check if this channel matches a specified extend_info_t */ int (*matches_extend_info)(channel_t *, extend_info_t *); - /* Check if this channel matches a target address when extending */ + /** Check if this channel matches a target address when extending */ int (*matches_target)(channel_t *, const tor_addr_t *); - /* Write a cell to an open channel */ + /** Write a cell to an open channel */ int (*write_cell)(channel_t *, cell_t *); - /* Write a packed cell to an open channel */ + /** Write a packed cell to an open channel */ int (*write_packed_cell)(channel_t *, packed_cell_t *); - /* Write a variable-length cell to an open channel */ + /** Write a variable-length cell to an open channel */ int (*write_var_cell)(channel_t *, var_cell_t *); - /* + /** * Hash of the public RSA key for the other side's identity key, or * zeroes if the other side hasn't shown us a valid identity key. */ char identity_digest[DIGEST_LEN]; - /* Nickname of the OR on the other side, or NULL if none. */ + /** Nickname of the OR on the other side, or NULL if none. */ char *nickname; - /* + /** * Linked list of channels with the same identity digest, for the * digest->channel map */ TOR_LIST_ENTRY(channel_s) next_with_same_id; - /* List of incoming cells to handle */ + /** List of incoming cells to handle */ chan_cell_queue_t incoming_queue; - /* List of queued outgoing cells */ + /** List of queued outgoing cells */ chan_cell_queue_t outgoing_queue; - /* Circuit mux for circuits sending on this channel */ + /** Circuit mux for circuits sending on this channel */ circuitmux_t *cmux; - /* Circuit ID generation stuff for use by circuitbuild.c */ + /** Circuit ID generation stuff for use by circuitbuild.c */ - /* + /** * When we send CREATE cells along this connection, which half of the * space should we use? */ - ENUM_BF(circ_id_type_t) circ_id_type:2; + circ_id_type_bitfield_t circ_id_type:2; /** DOCDOC*/ unsigned wide_circ_ids:1; - /** Have we logged a warning about circID exhaustion on this channel? */ - unsigned warned_circ_ids_exhausted:1; - /* - * Which circ_id do we try to use next on this connection? This is - * always in the range 0..1<<15-1. - */ - circid_t next_circ_id; - /* For how many circuits are we n_chan? What about p_chan? */ + /** For how many circuits are we n_chan? What about p_chan? */ unsigned int num_n_circuits, num_p_circuits; - /* + /** * True iff this channel shouldn't get any new circs attached to it, * because the connection is too old, or because there's a better one. * More generally, this flag is used to note an unhealthy connection; @@ -183,14 +177,20 @@ struct channel_s { */ unsigned int is_local:1; + /** Have we logged a warning about circID exhaustion on this channel? + * If so, when? */ + ratelim_t last_warned_circ_ids_exhausted; + /** Channel timestamps for cell channels */ time_t timestamp_client; /* Client used this, according to relay.c */ time_t timestamp_drained; /* Output queue empty */ time_t timestamp_recv; /* Cell received from lower layer */ time_t timestamp_xmit; /* Cell sent to lower layer */ - /* Timestamp for relay.c */ - time_t timestamp_last_added_nonpadding; + /** Timestamp for run_connection_housekeeping(). We update this once a + * second when we run housekeeping and find a circuit on this channel, and + * whenever we add a circuit to the channel. */ + time_t timestamp_last_had_circuits; /** Unique ID for measuring direct network status requests;vtunneled ones * come over a circuit_t, which has a dirreq_id field as well, but is a @@ -211,7 +211,7 @@ struct channel_listener_s { */ uint64_t global_identifier; - /* Should we expect to see this channel in the channel lists? */ + /** Should we expect to see this channel in the channel lists? */ unsigned char registered:1; /** Why did we close? @@ -223,31 +223,31 @@ struct channel_listener_s { CHANNEL_LISTENER_CLOSE_FOR_ERROR } reason_for_closing; - /* Timestamps for both cell channels and listeners */ + /** Timestamps for both cell channels and listeners */ time_t timestamp_created; /* Channel created */ time_t timestamp_active; /* Any activity */ /* Methods implemented by the lower layer */ - /* Free a channel */ + /** Free a channel */ void (*free)(channel_listener_t *); - /* Close an open channel */ + /** Close an open channel */ void (*close)(channel_listener_t *); - /* Describe the transport subclass for this channel */ + /** Describe the transport subclass for this channel */ const char * (*describe_transport)(channel_listener_t *); - /* Optional method to dump transport-specific statistics on the channel */ + /** Optional method to dump transport-specific statistics on the channel */ void (*dumpstats)(channel_listener_t *, int); - /* Registered listen handler to call on incoming connection */ + /** Registered listen handler to call on incoming connection */ channel_listener_fn_ptr listener; - /* List of pending incoming connections */ + /** List of pending incoming connections */ smartlist_t *incoming_list; - /* Timestamps for listeners */ + /** Timestamps for listeners */ time_t timestamp_accepted; - /* Counters for listeners */ + /** Counters for listeners */ uint64_t n_accepted; }; @@ -482,5 +482,9 @@ uint64_t channel_count_xmitted(channel_t *chan); uint64_t channel_listener_count_accepted(channel_listener_t *chan_l); +int packed_cell_is_destroy(channel_t *chan, + const packed_cell_t *packed_cell, + circid_t *circid_out); + #endif diff --git a/src/or/channeltls.c b/src/or/channeltls.c index d5428c1abd..632bc328b7 100644 --- a/src/or/channeltls.c +++ b/src/or/channeltls.c @@ -56,6 +56,8 @@ static const char * channel_tls_describe_transport_method(channel_t *chan); static void channel_tls_free_method(channel_t *chan); static int channel_tls_get_remote_addr_method(channel_t *chan, tor_addr_t *addr_out); +static int +channel_tls_get_transport_name_method(channel_t *chan, char **transport_out); static const char * channel_tls_get_remote_descr_method(channel_t *chan, int flags); static int channel_tls_has_queued_writes_method(channel_t *chan); @@ -116,6 +118,7 @@ channel_tls_common_init(channel_tls_t *tlschan) chan->free = channel_tls_free_method; chan->get_remote_addr = channel_tls_get_remote_addr_method; chan->get_remote_descr = channel_tls_get_remote_descr_method; + chan->get_transport_name = channel_tls_get_transport_name_method; chan->has_queued_writes = channel_tls_has_queued_writes_method; chan->is_canonical = channel_tls_is_canonical_method; chan->matches_extend_info = channel_tls_matches_extend_info_method; @@ -286,8 +289,8 @@ channel_tls_handle_incoming(or_connection_t *orconn) if (is_local_addr(&(TO_CONN(orconn)->addr))) channel_mark_local(chan); channel_mark_incoming(chan); - /* If we got one, we should register it */ - if (chan) channel_register(chan); + /* Register it */ + channel_register(chan); return chan; } @@ -435,6 +438,30 @@ channel_tls_get_remote_addr_method(channel_t *chan, tor_addr_t *addr_out) } /** + * Get the name of the pluggable transport used by a channel_tls_t. + * + * This implements the get_transport_name for channel_tls_t. If the + * channel uses a pluggable transport, copy its name to + * <b>transport_out</b> and return 0. If the channel did not use a + * pluggable transport, return -1. */ + +static int +channel_tls_get_transport_name_method(channel_t *chan, char **transport_out) +{ + channel_tls_t *tlschan = BASE_CHAN_TO_TLS(chan); + + tor_assert(tlschan); + tor_assert(transport_out); + tor_assert(tlschan->conn); + + if (!tlschan->conn->ext_or_transport) + return -1; + + *transport_out = tor_strdup(tlschan->conn->ext_or_transport); + return 0; +} + +/** * Get endpoint description of a channel_tls_t * * This implements the get_remote_descr method for channel_tls_t; it returns @@ -1255,13 +1282,20 @@ static void channel_tls_process_versions_cell(var_cell_t *cell, channel_tls_t *chan) { int highest_supported_version = 0; - const uint8_t *cp, *end; int started_here = 0; tor_assert(cell); tor_assert(chan); tor_assert(chan->conn); + if ((cell->payload_len % 2) == 1) { + log_fn(LOG_PROTOCOL_WARN, LD_OR, + "Received a VERSION cell with odd payload length %d; " + "closing connection.",cell->payload_len); + connection_or_close_for_error(chan->conn, 0); + return; + } + started_here = connection_or_nonopen_was_started_here(chan->conn); if (chan->conn->link_proto != 0 || @@ -1287,11 +1321,15 @@ channel_tls_process_versions_cell(var_cell_t *cell, channel_tls_t *chan) } tor_assert(chan->conn->handshake_state); - end = cell->payload + cell->payload_len; - for (cp = cell->payload; cp+1 < end; cp += 2) { - uint16_t v = ntohs(get_uint16(cp)); - if (is_or_protocol_version_known(v) && v > highest_supported_version) - highest_supported_version = v; + + { + int i; + const uint8_t *cp = cell->payload; + for (i = 0; i < cell->payload_len / 2; ++i, cp += 2) { + uint16_t v = ntohs(get_uint16(cp)); + if (is_or_protocol_version_known(v) && v > highest_supported_version) + highest_supported_version = v; + } } if (!highest_supported_version) { log_fn(LOG_PROTOCOL_WARN, LD_OR, @@ -1489,12 +1527,14 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan) my_addr_ptr = (uint8_t*) cell->payload + 6; end = cell->payload + CELL_PAYLOAD_SIZE; cp = cell->payload + 6 + my_addr_len; - if (cp >= end) { - log_fn(LOG_PROTOCOL_WARN, LD_OR, - "Addresses too long in netinfo cell; closing connection."); - connection_or_close_for_error(chan->conn, 0); - return; - } else if (my_addr_type == RESOLVED_TYPE_IPV4 && my_addr_len == 4) { + + /* We used to check: + * if (my_addr_len >= CELL_PAYLOAD_SIZE - 6) { + * + * This is actually never going to happen, since my_addr_len is at most 255, + * and CELL_PAYLOAD_LEN - 6 is 503. So we know that cp is < end. */ + + if (my_addr_type == RESOLVED_TYPE_IPV4 && my_addr_len == 4) { tor_addr_from_ipv4n(&my_apparent_addr, get_uint32(my_addr_ptr)); } else if (my_addr_type == RESOLVED_TYPE_IPV6 && my_addr_len == 16) { tor_addr_from_ipv6_bytes(&my_apparent_addr, (const char *) my_addr_ptr); @@ -1514,7 +1554,7 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan) return; } if (tor_addr_eq(&addr, &(chan->conn->real_addr))) { - chan->conn->is_canonical = 1; + connection_or_set_canonical(chan->conn, 1); break; } cp = next; @@ -1648,12 +1688,16 @@ channel_tls_process_certs_cell(var_cell_t *cell, channel_tls_t *chan) for (i = 0; i < n_certs; ++i) { uint8_t cert_type; uint16_t cert_len; - if (ptr + 3 > cell->payload + cell->payload_len) { + if (cell->payload_len < 3) + goto truncated; + if (ptr > cell->payload + cell->payload_len - 3) { goto truncated; } cert_type = *ptr; cert_len = ntohs(get_uint16(ptr+1)); - if (ptr + 3 + cert_len > cell->payload + cell->payload_len) { + if (cell->payload_len < 3 + cert_len) + goto truncated; + if (ptr > cell->payload + cell->payload_len - cert_len - 3) { goto truncated; } if (cert_type == OR_CERT_TYPE_TLS_LINK || diff --git a/src/or/circpathbias.c b/src/or/circpathbias.c new file mode 100644 index 0000000000..51a75cf502 --- /dev/null +++ b/src/or/circpathbias.c @@ -0,0 +1,1538 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2013, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "or.h" +#include "channel.h" +#include "circpathbias.h" +#include "circuitbuild.h" +#include "circuitlist.h" +#include "circuituse.h" +#include "circuitstats.h" +#include "connection_edge.h" +#include "config.h" +#include "entrynodes.h" +#include "networkstatus.h" +#include "relay.h" + +static void pathbias_count_successful_close(origin_circuit_t *circ); +static void pathbias_count_collapse(origin_circuit_t *circ); +static void pathbias_count_use_failed(origin_circuit_t *circ); +static void pathbias_measure_use_rate(entry_guard_t *guard); +static void pathbias_measure_close_rate(entry_guard_t *guard); +static void pathbias_scale_use_rates(entry_guard_t *guard); +static void pathbias_scale_close_rates(entry_guard_t *guard); +static int entry_guard_inc_circ_attempt_count(entry_guard_t *guard); + +/** Increment the number of times we successfully extended a circuit to + * <b>guard</b>, first checking if the failure rate is high enough that + * we should eliminate the guard. Return -1 if the guard looks no good; + * return 0 if the guard looks fine. + */ +static int +entry_guard_inc_circ_attempt_count(entry_guard_t *guard) +{ + entry_guards_changed(); + + pathbias_measure_close_rate(guard); + + if (guard->path_bias_disabled) + return -1; + + pathbias_scale_close_rates(guard); + guard->circ_attempts++; + + log_info(LD_CIRC, "Got success count %f/%f for guard %s ($%s)", + guard->circ_successes, guard->circ_attempts, guard->nickname, + hex_str(guard->identity, DIGEST_LEN)); + return 0; +} + +/** The minimum number of circuit attempts before we start + * thinking about warning about path bias and dropping guards */ +static int +pathbias_get_min_circs(const or_options_t *options) +{ +#define DFLT_PATH_BIAS_MIN_CIRC 150 + if (options->PathBiasCircThreshold >= 5) + return options->PathBiasCircThreshold; + else + return networkstatus_get_param(NULL, "pb_mincircs", + DFLT_PATH_BIAS_MIN_CIRC, + 5, INT32_MAX); +} + +/** The circuit success rate below which we issue a notice */ +static double +pathbias_get_notice_rate(const or_options_t *options) +{ +#define DFLT_PATH_BIAS_NOTICE_PCT 70 + if (options->PathBiasNoticeRate >= 0.0) + return options->PathBiasNoticeRate; + else + return networkstatus_get_param(NULL, "pb_noticepct", + DFLT_PATH_BIAS_NOTICE_PCT, 0, 100)/100.0; +} + +/* XXXX024 I'd like to have this be static again, but entrynodes.c needs it. */ +/** The circuit success rate below which we issue a warn */ +static double +pathbias_get_warn_rate(const or_options_t *options) +{ +#define DFLT_PATH_BIAS_WARN_PCT 50 + if (options->PathBiasWarnRate >= 0.0) + return options->PathBiasWarnRate; + else + return networkstatus_get_param(NULL, "pb_warnpct", + DFLT_PATH_BIAS_WARN_PCT, 0, 100)/100.0; +} + +/* XXXX024 I'd like to have this be static again, but entrynodes.c needs it. */ +/** + * The extreme rate is the rate at which we would drop the guard, + * if pb_dropguard is also set. Otherwise we just warn. + */ +double +pathbias_get_extreme_rate(const or_options_t *options) +{ +#define DFLT_PATH_BIAS_EXTREME_PCT 30 + if (options->PathBiasExtremeRate >= 0.0) + return options->PathBiasExtremeRate; + else + return networkstatus_get_param(NULL, "pb_extremepct", + DFLT_PATH_BIAS_EXTREME_PCT, 0, 100)/100.0; +} + +/* XXXX024 I'd like to have this be static again, but entrynodes.c needs it. */ +/** + * If 1, we actually disable use of guards that fall below + * the extreme_pct. + */ +int +pathbias_get_dropguards(const or_options_t *options) +{ +#define DFLT_PATH_BIAS_DROP_GUARDS 0 + if (options->PathBiasDropGuards >= 0) + return options->PathBiasDropGuards; + else + return networkstatus_get_param(NULL, "pb_dropguards", + DFLT_PATH_BIAS_DROP_GUARDS, 0, 1); +} + +/** + * This is the number of circuits at which we scale our + * counts by mult_factor/scale_factor. Note, this count is + * not exact, as we only perform the scaling in the event + * of no integer truncation. + */ +static int +pathbias_get_scale_threshold(const or_options_t *options) +{ +#define DFLT_PATH_BIAS_SCALE_THRESHOLD 300 + if (options->PathBiasScaleThreshold >= 10) + return options->PathBiasScaleThreshold; + else + return networkstatus_get_param(NULL, "pb_scalecircs", + DFLT_PATH_BIAS_SCALE_THRESHOLD, 10, + INT32_MAX); +} + +/** + * Compute the path bias scaling ratio from the consensus + * parameters pb_multfactor/pb_scalefactor. + * + * Returns a value in (0, 1.0] which we multiply our pathbias + * counts with to scale them down. + */ +static double +pathbias_get_scale_ratio(const or_options_t *options) +{ + /* + * The scale factor is the denominator for our scaling + * of circuit counts for our path bias window. + * + * Note that our use of doubles for the path bias state + * file means that powers of 2 work best here. + */ + int denominator = networkstatus_get_param(NULL, "pb_scalefactor", + 2, 2, INT32_MAX); + (void) options; + /** + * The mult factor is the numerator for our scaling + * of circuit counts for our path bias window. It + * allows us to scale by fractions. + */ + return networkstatus_get_param(NULL, "pb_multfactor", + 1, 1, denominator)/((double)denominator); +} + +/** The minimum number of circuit usage attempts before we start + * thinking about warning about path use bias and dropping guards */ +static int +pathbias_get_min_use(const or_options_t *options) +{ +#define DFLT_PATH_BIAS_MIN_USE 20 + if (options->PathBiasUseThreshold >= 3) + return options->PathBiasUseThreshold; + else + return networkstatus_get_param(NULL, "pb_minuse", + DFLT_PATH_BIAS_MIN_USE, + 3, INT32_MAX); +} + +/** The circuit use success rate below which we issue a notice */ +static double +pathbias_get_notice_use_rate(const or_options_t *options) +{ +#define DFLT_PATH_BIAS_NOTICE_USE_PCT 80 + if (options->PathBiasNoticeUseRate >= 0.0) + return options->PathBiasNoticeUseRate; + else + return networkstatus_get_param(NULL, "pb_noticeusepct", + DFLT_PATH_BIAS_NOTICE_USE_PCT, + 0, 100)/100.0; +} + +/** + * The extreme use rate is the rate at which we would drop the guard, + * if pb_dropguard is also set. Otherwise we just warn. + */ +double +pathbias_get_extreme_use_rate(const or_options_t *options) +{ +#define DFLT_PATH_BIAS_EXTREME_USE_PCT 60 + if (options->PathBiasExtremeUseRate >= 0.0) + return options->PathBiasExtremeUseRate; + else + return networkstatus_get_param(NULL, "pb_extremeusepct", + DFLT_PATH_BIAS_EXTREME_USE_PCT, + 0, 100)/100.0; +} + +/** + * This is the number of circuits at which we scale our + * use counts by mult_factor/scale_factor. Note, this count is + * not exact, as we only perform the scaling in the event + * of no integer truncation. + */ +static int +pathbias_get_scale_use_threshold(const or_options_t *options) +{ +#define DFLT_PATH_BIAS_SCALE_USE_THRESHOLD 100 + if (options->PathBiasScaleUseThreshold >= 10) + return options->PathBiasScaleUseThreshold; + else + return networkstatus_get_param(NULL, "pb_scaleuse", + DFLT_PATH_BIAS_SCALE_USE_THRESHOLD, + 10, INT32_MAX); +} + +/** + * Convert a Guard's path state to string. + */ +const char * +pathbias_state_to_string(path_state_t state) +{ + switch (state) { + case PATH_STATE_NEW_CIRC: + return "new"; + case PATH_STATE_BUILD_ATTEMPTED: + return "build attempted"; + case PATH_STATE_BUILD_SUCCEEDED: + return "build succeeded"; + case PATH_STATE_USE_ATTEMPTED: + return "use attempted"; + case PATH_STATE_USE_SUCCEEDED: + return "use succeeded"; + case PATH_STATE_USE_FAILED: + return "use failed"; + case PATH_STATE_ALREADY_COUNTED: + return "already counted"; + } + + return "unknown"; +} + +/** + * This function decides if a circuit has progressed far enough to count + * as a circuit "attempt". As long as end-to-end tagging is possible, + * we assume the adversary will use it over hop-to-hop failure. Therefore, + * we only need to account bias for the last hop. This should make us + * much more resilient to ambient circuit failure, and also make that + * failure easier to measure (we only need to measure Exit failure rates). + */ +static int +pathbias_is_new_circ_attempt(origin_circuit_t *circ) +{ +#define N2N_TAGGING_IS_POSSIBLE +#ifdef N2N_TAGGING_IS_POSSIBLE + /* cpath is a circular list. We want circs with more than one hop, + * and the second hop must be waiting for keys still (it's just + * about to get them). */ + return circ->cpath && + circ->cpath->next != circ->cpath && + circ->cpath->next->state == CPATH_STATE_AWAITING_KEYS; +#else + /* If tagging attacks are no longer possible, we probably want to + * count bias from the first hop. However, one could argue that + * timing-based tagging is still more useful than per-hop failure. + * In which case, we'd never want to use this. + */ + return circ->cpath && + circ->cpath->state == CPATH_STATE_AWAITING_KEYS; +#endif +} + +/** + * Decide if the path bias code should count a circuit. + * + * @returns 1 if we should count it, 0 otherwise. + */ +static int +pathbias_should_count(origin_circuit_t *circ) +{ +#define PATHBIAS_COUNT_INTERVAL (600) + static ratelim_t count_limit = + RATELIM_INIT(PATHBIAS_COUNT_INTERVAL); + char *rate_msg = NULL; + + /* We can't do path bias accounting without entry guards. + * Testing and controller circuits also have no guards. + * + * We also don't count server-side rends, because their + * endpoint could be chosen maliciously. + * Similarly, we can't count client-side intro attempts, + * because clients can be manipulated into connecting to + * malicious intro points. */ + if (get_options()->UseEntryGuards == 0 || + circ->base_.purpose == CIRCUIT_PURPOSE_TESTING || + circ->base_.purpose == CIRCUIT_PURPOSE_CONTROLLER || + circ->base_.purpose == CIRCUIT_PURPOSE_S_CONNECT_REND || + circ->base_.purpose == CIRCUIT_PURPOSE_S_REND_JOINED || + (circ->base_.purpose >= CIRCUIT_PURPOSE_C_INTRODUCING && + circ->base_.purpose <= CIRCUIT_PURPOSE_C_INTRODUCE_ACKED)) { + + /* Check to see if the shouldcount result has changed due to a + * unexpected purpose change that would affect our results. + * + * The reason we check the path state too here is because for the + * cannibalized versions of these purposes, we count them as successful + * before their purpose change. + */ + if (circ->pathbias_shouldcount == PATHBIAS_SHOULDCOUNT_COUNTED + && circ->path_state != PATH_STATE_ALREADY_COUNTED) { + log_info(LD_BUG, + "Circuit %d is now being ignored despite being counted " + "in the past. Purpose is %s, path state is %s", + circ->global_identifier, + circuit_purpose_to_string(circ->base_.purpose), + pathbias_state_to_string(circ->path_state)); + } + circ->pathbias_shouldcount = PATHBIAS_SHOULDCOUNT_IGNORED; + return 0; + } + + /* Completely ignore one hop circuits */ + if (circ->build_state->onehop_tunnel || + circ->build_state->desired_path_len == 1) { + /* Check for inconsistency */ + if (circ->build_state->desired_path_len != 1 || + !circ->build_state->onehop_tunnel) { + if ((rate_msg = rate_limit_log(&count_limit, approx_time()))) { + log_info(LD_BUG, + "One-hop circuit has length %d. Path state is %s. " + "Circuit is a %s currently %s.%s", + circ->build_state->desired_path_len, + pathbias_state_to_string(circ->path_state), + circuit_purpose_to_string(circ->base_.purpose), + circuit_state_to_string(circ->base_.state), + rate_msg); + tor_free(rate_msg); + } + tor_fragile_assert(); + } + + /* Check to see if the shouldcount result has changed due to a + * unexpected change that would affect our results */ + if (circ->pathbias_shouldcount == PATHBIAS_SHOULDCOUNT_COUNTED) { + log_info(LD_BUG, + "One-hop circuit %d is now being ignored despite being counted " + "in the past. Purpose is %s, path state is %s", + circ->global_identifier, + circuit_purpose_to_string(circ->base_.purpose), + pathbias_state_to_string(circ->path_state)); + } + circ->pathbias_shouldcount = PATHBIAS_SHOULDCOUNT_IGNORED; + return 0; + } + + /* Check to see if the shouldcount result has changed due to a + * unexpected purpose change that would affect our results */ + if (circ->pathbias_shouldcount == PATHBIAS_SHOULDCOUNT_IGNORED) { + log_info(LD_BUG, + "Circuit %d is now being counted despite being ignored " + "in the past. Purpose is %s, path state is %s", + circ->global_identifier, + circuit_purpose_to_string(circ->base_.purpose), + pathbias_state_to_string(circ->path_state)); + } + circ->pathbias_shouldcount = PATHBIAS_SHOULDCOUNT_COUNTED; + + return 1; +} + +/** + * Check our circuit state to see if this is a successful circuit attempt. + * If so, record it in the current guard's path bias circ_attempt count. + * + * Also check for several potential error cases for bug #6475. + */ +int +pathbias_count_build_attempt(origin_circuit_t *circ) +{ +#define CIRC_ATTEMPT_NOTICE_INTERVAL (600) + static ratelim_t circ_attempt_notice_limit = + RATELIM_INIT(CIRC_ATTEMPT_NOTICE_INTERVAL); + char *rate_msg = NULL; + + if (!pathbias_should_count(circ)) { + return 0; + } + + if (pathbias_is_new_circ_attempt(circ)) { + /* Help track down the real cause of bug #6475: */ + if (circ->has_opened && circ->path_state != PATH_STATE_BUILD_ATTEMPTED) { + if ((rate_msg = rate_limit_log(&circ_attempt_notice_limit, + approx_time()))) { + log_info(LD_BUG, + "Opened circuit is in strange path state %s. " + "Circuit is a %s currently %s.%s", + pathbias_state_to_string(circ->path_state), + circuit_purpose_to_string(circ->base_.purpose), + circuit_state_to_string(circ->base_.state), + rate_msg); + tor_free(rate_msg); + } + } + + /* Don't re-count cannibalized circs.. */ + if (!circ->has_opened) { + entry_guard_t *guard = NULL; + + if (circ->cpath && circ->cpath->extend_info) { + guard = entry_guard_get_by_id_digest( + circ->cpath->extend_info->identity_digest); + } else if (circ->base_.n_chan) { + guard = + entry_guard_get_by_id_digest(circ->base_.n_chan->identity_digest); + } + + if (guard) { + if (circ->path_state == PATH_STATE_NEW_CIRC) { + circ->path_state = PATH_STATE_BUILD_ATTEMPTED; + + if (entry_guard_inc_circ_attempt_count(guard) < 0) { + /* Bogus guard; we already warned. */ + return -END_CIRC_REASON_TORPROTOCOL; + } + } else { + if ((rate_msg = rate_limit_log(&circ_attempt_notice_limit, + approx_time()))) { + log_info(LD_BUG, + "Unopened circuit has strange path state %s. " + "Circuit is a %s currently %s.%s", + pathbias_state_to_string(circ->path_state), + circuit_purpose_to_string(circ->base_.purpose), + circuit_state_to_string(circ->base_.state), + rate_msg); + tor_free(rate_msg); + } + } + } else { + if ((rate_msg = rate_limit_log(&circ_attempt_notice_limit, + approx_time()))) { + log_info(LD_CIRC, + "Unopened circuit has no known guard. " + "Circuit is a %s currently %s.%s", + circuit_purpose_to_string(circ->base_.purpose), + circuit_state_to_string(circ->base_.state), + rate_msg); + tor_free(rate_msg); + } + } + } + } + + return 0; +} + +/** + * Check our circuit state to see if this is a successful circuit + * completion. If so, record it in the current guard's path bias + * success count. + * + * Also check for several potential error cases for bug #6475. + */ +void +pathbias_count_build_success(origin_circuit_t *circ) +{ +#define SUCCESS_NOTICE_INTERVAL (600) + static ratelim_t success_notice_limit = + RATELIM_INIT(SUCCESS_NOTICE_INTERVAL); + char *rate_msg = NULL; + entry_guard_t *guard = NULL; + + if (!pathbias_should_count(circ)) { + return; + } + + /* Don't count cannibalized/reused circs for path bias + * "build" success, since they get counted under "use" success. */ + if (!circ->has_opened) { + if (circ->cpath && circ->cpath->extend_info) { + guard = entry_guard_get_by_id_digest( + circ->cpath->extend_info->identity_digest); + } + + if (guard) { + if (circ->path_state == PATH_STATE_BUILD_ATTEMPTED) { + circ->path_state = PATH_STATE_BUILD_SUCCEEDED; + guard->circ_successes++; + entry_guards_changed(); + + log_info(LD_CIRC, "Got success count %f/%f for guard %s ($%s)", + guard->circ_successes, guard->circ_attempts, + guard->nickname, hex_str(guard->identity, DIGEST_LEN)); + } else { + if ((rate_msg = rate_limit_log(&success_notice_limit, + approx_time()))) { + log_info(LD_BUG, + "Succeeded circuit is in strange path state %s. " + "Circuit is a %s currently %s.%s", + pathbias_state_to_string(circ->path_state), + circuit_purpose_to_string(circ->base_.purpose), + circuit_state_to_string(circ->base_.state), + rate_msg); + tor_free(rate_msg); + } + } + + if (guard->circ_attempts < guard->circ_successes) { + log_notice(LD_BUG, "Unexpectedly high successes counts (%f/%f) " + "for guard %s ($%s)", + guard->circ_successes, guard->circ_attempts, + guard->nickname, hex_str(guard->identity, DIGEST_LEN)); + } + /* In rare cases, CIRCUIT_PURPOSE_TESTING can get converted to + * CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT and have no guards here. + * No need to log that case. */ + } else if (circ->base_.purpose != CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT) { + if ((rate_msg = rate_limit_log(&success_notice_limit, + approx_time()))) { + log_info(LD_CIRC, + "Completed circuit has no known guard. " + "Circuit is a %s currently %s.%s", + circuit_purpose_to_string(circ->base_.purpose), + circuit_state_to_string(circ->base_.state), + rate_msg); + tor_free(rate_msg); + } + } + } else { + if (circ->path_state < PATH_STATE_BUILD_SUCCEEDED) { + if ((rate_msg = rate_limit_log(&success_notice_limit, + approx_time()))) { + log_info(LD_BUG, + "Opened circuit is in strange path state %s. " + "Circuit is a %s currently %s.%s", + pathbias_state_to_string(circ->path_state), + circuit_purpose_to_string(circ->base_.purpose), + circuit_state_to_string(circ->base_.state), + rate_msg); + tor_free(rate_msg); + } + } + } +} + +/** + * Record an attempt to use a circuit. Changes the circuit's + * path state and update its guard's usage counter. + * + * Used for path bias usage accounting. + */ +void +pathbias_count_use_attempt(origin_circuit_t *circ) +{ + entry_guard_t *guard; + + if (!pathbias_should_count(circ)) { + return; + } + + if (circ->path_state < PATH_STATE_BUILD_SUCCEEDED) { + log_notice(LD_BUG, + "Used circuit is in strange path state %s. " + "Circuit is a %s currently %s.", + pathbias_state_to_string(circ->path_state), + circuit_purpose_to_string(circ->base_.purpose), + circuit_state_to_string(circ->base_.state)); + } else if (circ->path_state < PATH_STATE_USE_ATTEMPTED) { + guard = entry_guard_get_by_id_digest( + circ->cpath->extend_info->identity_digest); + if (guard) { + pathbias_measure_use_rate(guard); + pathbias_scale_use_rates(guard); + guard->use_attempts++; + entry_guards_changed(); + + log_debug(LD_CIRC, + "Marked circuit %d (%f/%f) as used for guard %s ($%s).", + circ->global_identifier, + guard->use_successes, guard->use_attempts, + guard->nickname, hex_str(guard->identity, DIGEST_LEN)); + } + + circ->path_state = PATH_STATE_USE_ATTEMPTED; + } else { + /* Harmless but educational log message */ + log_info(LD_CIRC, + "Used circuit %d is already in path state %s. " + "Circuit is a %s currently %s.", + circ->global_identifier, + pathbias_state_to_string(circ->path_state), + circuit_purpose_to_string(circ->base_.purpose), + circuit_state_to_string(circ->base_.state)); + } + + return; +} + +/** + * Check the circuit's path state is appropriate and mark it as + * successfully used. Used for path bias usage accounting. + * + * We don't actually increment the guard's counters until + * pathbias_check_close(), because the circuit can still transition + * back to PATH_STATE_USE_ATTEMPTED if a stream fails later (this + * is done so we can probe the circuit for liveness at close). + */ +void +pathbias_mark_use_success(origin_circuit_t *circ) +{ + if (!pathbias_should_count(circ)) { + return; + } + + if (circ->path_state < PATH_STATE_USE_ATTEMPTED) { + log_notice(LD_BUG, + "Used circuit %d is in strange path state %s. " + "Circuit is a %s currently %s.", + circ->global_identifier, + pathbias_state_to_string(circ->path_state), + circuit_purpose_to_string(circ->base_.purpose), + circuit_state_to_string(circ->base_.state)); + + pathbias_count_use_attempt(circ); + } + + /* We don't do any accounting at the guard until actual circuit close */ + circ->path_state = PATH_STATE_USE_SUCCEEDED; + + return; +} + +/** + * If a stream ever detatches from a circuit in a retriable way, + * we need to mark this circuit as still needing either another + * successful stream, or in need of a probe. + * + * An adversary could let the first stream request succeed (ie the + * resolve), but then tag and timeout the remainder (via cell + * dropping), forcing them on new circuits. + * + * Rolling back the state will cause us to probe such circuits, which + * should lead to probe failures in the event of such tagging due to + * either unrecognized cells coming in while we wait for the probe, + * or the cipher state getting out of sync in the case of dropped cells. + */ +void +pathbias_mark_use_rollback(origin_circuit_t *circ) +{ + if (circ->path_state == PATH_STATE_USE_SUCCEEDED) { + log_info(LD_CIRC, + "Rolling back pathbias use state to 'attempted' for detached " + "circuit %d", circ->global_identifier); + circ->path_state = PATH_STATE_USE_ATTEMPTED; + } +} + +/** + * Actually count a circuit success towards a guard's usage counters + * if the path state is appropriate. + */ +static void +pathbias_count_use_success(origin_circuit_t *circ) +{ + entry_guard_t *guard; + + if (!pathbias_should_count(circ)) { + return; + } + + if (circ->path_state != PATH_STATE_USE_SUCCEEDED) { + log_notice(LD_BUG, + "Successfully used circuit %d is in strange path state %s. " + "Circuit is a %s currently %s.", + circ->global_identifier, + pathbias_state_to_string(circ->path_state), + circuit_purpose_to_string(circ->base_.purpose), + circuit_state_to_string(circ->base_.state)); + } else { + guard = entry_guard_get_by_id_digest( + circ->cpath->extend_info->identity_digest); + if (guard) { + guard->use_successes++; + entry_guards_changed(); + + if (guard->use_attempts < guard->use_successes) { + log_notice(LD_BUG, "Unexpectedly high use successes counts (%f/%f) " + "for guard %s=%s", + guard->use_successes, guard->use_attempts, + guard->nickname, hex_str(guard->identity, DIGEST_LEN)); + } + + log_debug(LD_CIRC, + "Marked circuit %d (%f/%f) as used successfully for guard " + "%s ($%s).", + circ->global_identifier, guard->use_successes, + guard->use_attempts, guard->nickname, + hex_str(guard->identity, DIGEST_LEN)); + } + } + + return; +} + +/** + * Send a probe down a circuit that the client attempted to use, + * but for which the stream timed out/failed. The probe is a + * RELAY_BEGIN cell with a 0.a.b.c destination address, which + * the exit will reject and reply back, echoing that address. + * + * The reason for such probes is because it is possible to bias + * a user's paths simply by causing timeouts, and these timeouts + * are not possible to differentiate from unresponsive servers. + * + * The probe is sent at the end of the circuit lifetime for two + * reasons: to prevent cryptographic taggers from being able to + * drop cells to cause timeouts, and to prevent easy recognition + * of probes before any real client traffic happens. + * + * Returns -1 if we couldn't probe, 0 otherwise. + */ +static int +pathbias_send_usable_probe(circuit_t *circ) +{ + /* Based on connection_ap_handshake_send_begin() */ + char payload[CELL_PAYLOAD_SIZE]; + int payload_len; + origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ); + crypt_path_t *cpath_layer = NULL; + char *probe_nonce = NULL; + + tor_assert(ocirc); + + cpath_layer = ocirc->cpath->prev; + + if (cpath_layer->state != CPATH_STATE_OPEN) { + /* This can happen for cannibalized circuits. Their + * last hop isn't yet open */ + log_info(LD_CIRC, + "Got pathbias probe request for unopened circuit %d. " + "Opened %d, len %d", ocirc->global_identifier, + ocirc->has_opened, ocirc->build_state->desired_path_len); + return -1; + } + + /* We already went down this road. */ + if (circ->purpose == CIRCUIT_PURPOSE_PATH_BIAS_TESTING && + ocirc->pathbias_probe_id) { + log_info(LD_CIRC, + "Got pathbias probe request for circuit %d with " + "outstanding probe", ocirc->global_identifier); + return -1; + } + + /* Can't probe if the channel isn't open */ + if (circ->n_chan == NULL || + (circ->n_chan->state != CHANNEL_STATE_OPEN + && circ->n_chan->state != CHANNEL_STATE_MAINT)) { + log_info(LD_CIRC, + "Skipping pathbias probe for circuit %d: Channel is not open.", + ocirc->global_identifier); + return -1; + } + + circuit_change_purpose(circ, CIRCUIT_PURPOSE_PATH_BIAS_TESTING); + + /* Update timestamp for when circuit_expire_building() should kill us */ + tor_gettimeofday(&circ->timestamp_began); + + /* Generate a random address for the nonce */ + crypto_rand((char*)ô->pathbias_probe_nonce, + sizeof(ocirc->pathbias_probe_nonce)); + ocirc->pathbias_probe_nonce &= 0x00ffffff; + probe_nonce = tor_dup_ip(ocirc->pathbias_probe_nonce); + + tor_snprintf(payload,RELAY_PAYLOAD_SIZE, "%s:25", probe_nonce); + payload_len = (int)strlen(payload)+1; + + // XXX: need this? Can we assume ipv4 will always be supported? + // If not, how do we tell? + //if (payload_len <= RELAY_PAYLOAD_SIZE - 4 && edge_conn->begincell_flags) { + // set_uint32(payload + payload_len, htonl(edge_conn->begincell_flags)); + // payload_len += 4; + //} + + /* Generate+Store stream id, make sure it's non-zero */ + ocirc->pathbias_probe_id = get_unique_stream_id_by_circ(ocirc); + + if (ocirc->pathbias_probe_id==0) { + log_warn(LD_CIRC, + "Ran out of stream IDs on circuit %u during " + "pathbias probe attempt.", ocirc->global_identifier); + tor_free(probe_nonce); + return -1; + } + + log_info(LD_CIRC, + "Sending pathbias testing cell to %s:25 on stream %d for circ %d.", + probe_nonce, ocirc->pathbias_probe_id, ocirc->global_identifier); + tor_free(probe_nonce); + + /* Send a test relay cell */ + if (relay_send_command_from_edge(ocirc->pathbias_probe_id, circ, + RELAY_COMMAND_BEGIN, payload, + payload_len, cpath_layer) < 0) { + log_notice(LD_CIRC, + "Failed to send pathbias probe cell on circuit %d.", + ocirc->global_identifier); + return -1; + } + + /* Mark it freshly dirty so it doesn't get expired in the meantime */ + circ->timestamp_dirty = time(NULL); + + return 0; +} + +/** + * Check the response to a pathbias probe, to ensure the + * cell is recognized and the nonce and other probe + * characteristics are as expected. + * + * If the response is valid, return 0. Otherwise return < 0. + */ +int +pathbias_check_probe_response(circuit_t *circ, const cell_t *cell) +{ + /* Based on connection_edge_process_relay_cell() */ + relay_header_t rh; + int reason; + uint32_t ipv4_host; + origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ); + + tor_assert(cell); + tor_assert(ocirc); + tor_assert(circ->purpose == CIRCUIT_PURPOSE_PATH_BIAS_TESTING); + + relay_header_unpack(&rh, cell->payload); + + reason = rh.length > 0 ? + get_uint8(cell->payload+RELAY_HEADER_SIZE) : END_STREAM_REASON_MISC; + + if (rh.command == RELAY_COMMAND_END && + reason == END_STREAM_REASON_EXITPOLICY && + ocirc->pathbias_probe_id == rh.stream_id) { + + /* Check length+extract host: It is in network order after the reason code. + * See connection_edge_end(). */ + if (rh.length < 9) { /* reason+ipv4+dns_ttl */ + log_notice(LD_PROTOCOL, + "Short path bias probe response length field (%d).", rh.length); + return - END_CIRC_REASON_TORPROTOCOL; + } + + ipv4_host = ntohl(get_uint32(cell->payload+RELAY_HEADER_SIZE+1)); + + /* Check nonce */ + if (ipv4_host == ocirc->pathbias_probe_nonce) { + pathbias_mark_use_success(ocirc); + circuit_mark_for_close(circ, END_CIRC_REASON_FINISHED); + log_info(LD_CIRC, + "Got valid path bias probe back for circ %d, stream %d.", + ocirc->global_identifier, ocirc->pathbias_probe_id); + return 0; + } else { + log_notice(LD_CIRC, + "Got strange probe value 0x%x vs 0x%x back for circ %d, " + "stream %d.", ipv4_host, ocirc->pathbias_probe_nonce, + ocirc->global_identifier, ocirc->pathbias_probe_id); + return -1; + } + } + log_info(LD_CIRC, + "Got another cell back back on pathbias probe circuit %d: " + "Command: %d, Reason: %d, Stream-id: %d", + ocirc->global_identifier, rh.command, reason, rh.stream_id); + return -1; +} + +/** + * Check if a circuit was used and/or closed successfully. + * + * If we attempted to use the circuit to carry a stream but failed + * for whatever reason, or if the circuit mysteriously died before + * we could attach any streams, record these two cases. + * + * If we *have* successfully used the circuit, or it appears to + * have been closed by us locally, count it as a success. + * + * Returns 0 if we're done making decisions with the circ, + * or -1 if we want to probe it first. + */ +int +pathbias_check_close(origin_circuit_t *ocirc, int reason) +{ + circuit_t *circ = ô->base_; + + if (!pathbias_should_count(ocirc)) { + return 0; + } + + switch (ocirc->path_state) { + /* If the circuit was closed after building, but before use, we need + * to ensure we were the ones who tried to close it (and not a remote + * actor). */ + case PATH_STATE_BUILD_SUCCEEDED: + if (reason & END_CIRC_REASON_FLAG_REMOTE) { + /* Remote circ close reasons on an unused circuit all could be bias */ + log_info(LD_CIRC, + "Circuit %d remote-closed without successful use for reason %d. " + "Circuit purpose %d currently %d,%s. Len %d.", + ocirc->global_identifier, + reason, circ->purpose, ocirc->has_opened, + circuit_state_to_string(circ->state), + ocirc->build_state->desired_path_len); + pathbias_count_collapse(ocirc); + } else if ((reason & ~END_CIRC_REASON_FLAG_REMOTE) + == END_CIRC_REASON_CHANNEL_CLOSED && + circ->n_chan && + circ->n_chan->reason_for_closing + != CHANNEL_CLOSE_REQUESTED) { + /* If we didn't close the channel ourselves, it could be bias */ + /* XXX: Only count bias if the network is live? + * What about clock jumps/suspends? */ + log_info(LD_CIRC, + "Circuit %d's channel closed without successful use for reason " + "%d, channel reason %d. Circuit purpose %d currently %d,%s. Len " + "%d.", ocirc->global_identifier, + reason, circ->n_chan->reason_for_closing, + circ->purpose, ocirc->has_opened, + circuit_state_to_string(circ->state), + ocirc->build_state->desired_path_len); + pathbias_count_collapse(ocirc); + } else { + pathbias_count_successful_close(ocirc); + } + break; + + /* If we tried to use a circuit but failed, we should probe it to ensure + * it has not been tampered with. */ + case PATH_STATE_USE_ATTEMPTED: + /* XXX: Only probe and/or count failure if the network is live? + * What about clock jumps/suspends? */ + if (pathbias_send_usable_probe(circ) == 0) + return -1; + else + pathbias_count_use_failed(ocirc); + + /* Any circuit where there were attempted streams but no successful + * streams could be bias */ + log_info(LD_CIRC, + "Circuit %d closed without successful use for reason %d. " + "Circuit purpose %d currently %d,%s. Len %d.", + ocirc->global_identifier, + reason, circ->purpose, ocirc->has_opened, + circuit_state_to_string(circ->state), + ocirc->build_state->desired_path_len); + break; + + case PATH_STATE_USE_SUCCEEDED: + pathbias_count_successful_close(ocirc); + pathbias_count_use_success(ocirc); + break; + + case PATH_STATE_USE_FAILED: + pathbias_count_use_failed(ocirc); + break; + + case PATH_STATE_NEW_CIRC: + case PATH_STATE_BUILD_ATTEMPTED: + case PATH_STATE_ALREADY_COUNTED: + default: + // Other states are uninteresting. No stats to count. + break; + } + + ocirc->path_state = PATH_STATE_ALREADY_COUNTED; + + return 0; +} + +/** + * Count a successfully closed circuit. + */ +static void +pathbias_count_successful_close(origin_circuit_t *circ) +{ + entry_guard_t *guard = NULL; + if (!pathbias_should_count(circ)) { + return; + } + + if (circ->cpath && circ->cpath->extend_info) { + guard = entry_guard_get_by_id_digest( + circ->cpath->extend_info->identity_digest); + } + + if (guard) { + /* In the long run: circuit_success ~= successful_circuit_close + + * circ_failure + stream_failure */ + guard->successful_circuits_closed++; + entry_guards_changed(); + } else if (circ->base_.purpose != CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT) { + /* In rare cases, CIRCUIT_PURPOSE_TESTING can get converted to + * CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT and have no guards here. + * No need to log that case. */ + log_info(LD_CIRC, + "Successfully closed circuit has no known guard. " + "Circuit is a %s currently %s", + circuit_purpose_to_string(circ->base_.purpose), + circuit_state_to_string(circ->base_.state)); + } +} + +/** + * Count a circuit that fails after it is built, but before it can + * carry any traffic. + * + * This is needed because there are ways to destroy a + * circuit after it has successfully completed. Right now, this is + * used for purely informational/debugging purposes. + */ +static void +pathbias_count_collapse(origin_circuit_t *circ) +{ + entry_guard_t *guard = NULL; + + if (!pathbias_should_count(circ)) { + return; + } + + if (circ->cpath && circ->cpath->extend_info) { + guard = entry_guard_get_by_id_digest( + circ->cpath->extend_info->identity_digest); + } + + if (guard) { + guard->collapsed_circuits++; + entry_guards_changed(); + } else if (circ->base_.purpose != CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT) { + /* In rare cases, CIRCUIT_PURPOSE_TESTING can get converted to + * CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT and have no guards here. + * No need to log that case. */ + log_info(LD_CIRC, + "Destroyed circuit has no known guard. " + "Circuit is a %s currently %s", + circuit_purpose_to_string(circ->base_.purpose), + circuit_state_to_string(circ->base_.state)); + } +} + +/** + * Count a known failed circuit (because we could not probe it). + * + * This counter is informational. + */ +static void +pathbias_count_use_failed(origin_circuit_t *circ) +{ + entry_guard_t *guard = NULL; + if (!pathbias_should_count(circ)) { + return; + } + + if (circ->cpath && circ->cpath->extend_info) { + guard = entry_guard_get_by_id_digest( + circ->cpath->extend_info->identity_digest); + } + + if (guard) { + guard->unusable_circuits++; + entry_guards_changed(); + } else if (circ->base_.purpose != CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT) { + /* In rare cases, CIRCUIT_PURPOSE_TESTING can get converted to + * CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT and have no guards here. + * No need to log that case. */ + /* XXX note cut-and-paste code in this function compared to nearby + * functions. Would be nice to refactor. -RD */ + log_info(LD_CIRC, + "Stream-failing circuit has no known guard. " + "Circuit is a %s currently %s", + circuit_purpose_to_string(circ->base_.purpose), + circuit_state_to_string(circ->base_.state)); + } +} + +/** + * Count timeouts for path bias log messages. + * + * These counts are purely informational. + */ +void +pathbias_count_timeout(origin_circuit_t *circ) +{ + entry_guard_t *guard = NULL; + + if (!pathbias_should_count(circ)) { + return; + } + + /* For hidden service circs, they can actually be used + * successfully and then time out later (because + * the other side declines to use them). */ + if (circ->path_state == PATH_STATE_USE_SUCCEEDED) { + return; + } + + if (circ->cpath && circ->cpath->extend_info) { + guard = entry_guard_get_by_id_digest( + circ->cpath->extend_info->identity_digest); + } + + if (guard) { + guard->timeouts++; + entry_guards_changed(); + } +} + +/** + * Helper function to count all of the currently opened circuits + * for a guard that are in a given path state range. The state + * range is inclusive on both ends. + */ +static int +pathbias_count_circs_in_states(entry_guard_t *guard, + path_state_t from, + path_state_t to) +{ + circuit_t *circ; + int open_circuits = 0; + + /* Count currently open circuits. Give them the benefit of the doubt. */ + TOR_LIST_FOREACH(circ, circuit_get_global_list(), head) { + origin_circuit_t *ocirc = NULL; + if (!CIRCUIT_IS_ORIGIN(circ) || /* didn't originate here */ + circ->marked_for_close) /* already counted */ + continue; + + ocirc = TO_ORIGIN_CIRCUIT(circ); + + if (!ocirc->cpath || !ocirc->cpath->extend_info) + continue; + + if (ocirc->path_state >= from && + ocirc->path_state <= to && + pathbias_should_count(ocirc) && + fast_memeq(guard->identity, + ocirc->cpath->extend_info->identity_digest, + DIGEST_LEN)) { + log_debug(LD_CIRC, "Found opened circuit %d in path_state %s", + ocirc->global_identifier, + pathbias_state_to_string(ocirc->path_state)); + open_circuits++; + } + } + + return open_circuits; +} + +/** + * Return the number of circuits counted as successfully closed for + * this guard. + * + * Also add in the currently open circuits to give them the benefit + * of the doubt. + */ +double +pathbias_get_close_success_count(entry_guard_t *guard) +{ + return guard->successful_circuits_closed + + pathbias_count_circs_in_states(guard, + PATH_STATE_BUILD_SUCCEEDED, + PATH_STATE_USE_SUCCEEDED); +} + +/** + * Return the number of circuits counted as successfully used + * this guard. + * + * Also add in the currently open circuits that we are attempting + * to use to give them the benefit of the doubt. + */ +double +pathbias_get_use_success_count(entry_guard_t *guard) +{ + return guard->use_successes + + pathbias_count_circs_in_states(guard, + PATH_STATE_USE_ATTEMPTED, + PATH_STATE_USE_SUCCEEDED); +} + +/** + * Check the path bias use rate against our consensus parameter limits. + * + * Emits a log message if the use success rates are too low. + * + * If pathbias_get_dropguards() is set, we also disable the use of + * very failure prone guards. + */ +static void +pathbias_measure_use_rate(entry_guard_t *guard) +{ + const or_options_t *options = get_options(); + + if (guard->use_attempts > pathbias_get_min_use(options)) { + /* Note: We rely on the < comparison here to allow us to set a 0 + * rate and disable the feature entirely. If refactoring, don't + * change to <= */ + if (pathbias_get_use_success_count(guard)/guard->use_attempts + < pathbias_get_extreme_use_rate(options)) { + /* Dropping is currently disabled by default. */ + if (pathbias_get_dropguards(options)) { + if (!guard->path_bias_disabled) { + log_warn(LD_CIRC, + "Your Guard %s ($%s) is failing to carry an extremely large " + "amount of stream on its circuits. " + "To avoid potential route manipulation attacks, Tor has " + "disabled use of this guard. " + "Use counts are %ld/%ld. Success counts are %ld/%ld. " + "%ld circuits completed, %ld were unusable, %ld collapsed, " + "and %ld timed out. " + "For reference, your timeout cutoff is %ld seconds.", + guard->nickname, hex_str(guard->identity, DIGEST_LEN), + tor_lround(pathbias_get_use_success_count(guard)), + tor_lround(guard->use_attempts), + tor_lround(pathbias_get_close_success_count(guard)), + tor_lround(guard->circ_attempts), + tor_lround(guard->circ_successes), + tor_lround(guard->unusable_circuits), + tor_lround(guard->collapsed_circuits), + tor_lround(guard->timeouts), + tor_lround(get_circuit_build_close_time_ms()/1000)); + guard->path_bias_disabled = 1; + guard->bad_since = approx_time(); + entry_guards_changed(); + return; + } + } else if (!guard->path_bias_use_extreme) { + guard->path_bias_use_extreme = 1; + log_warn(LD_CIRC, + "Your Guard %s ($%s) is failing to carry an extremely large " + "amount of streams on its circuits. " + "This could indicate a route manipulation attack, network " + "overload, bad local network connectivity, or a bug. " + "Use counts are %ld/%ld. Success counts are %ld/%ld. " + "%ld circuits completed, %ld were unusable, %ld collapsed, " + "and %ld timed out. " + "For reference, your timeout cutoff is %ld seconds.", + guard->nickname, hex_str(guard->identity, DIGEST_LEN), + tor_lround(pathbias_get_use_success_count(guard)), + tor_lround(guard->use_attempts), + tor_lround(pathbias_get_close_success_count(guard)), + tor_lround(guard->circ_attempts), + tor_lround(guard->circ_successes), + tor_lround(guard->unusable_circuits), + tor_lround(guard->collapsed_circuits), + tor_lround(guard->timeouts), + tor_lround(get_circuit_build_close_time_ms()/1000)); + } + } else if (pathbias_get_use_success_count(guard)/guard->use_attempts + < pathbias_get_notice_use_rate(options)) { + if (!guard->path_bias_use_noticed) { + guard->path_bias_use_noticed = 1; + log_notice(LD_CIRC, + "Your Guard %s ($%s) is failing to carry more streams on its " + "circuits than usual. " + "Most likely this means the Tor network is overloaded " + "or your network connection is poor. " + "Use counts are %ld/%ld. Success counts are %ld/%ld. " + "%ld circuits completed, %ld were unusable, %ld collapsed, " + "and %ld timed out. " + "For reference, your timeout cutoff is %ld seconds.", + guard->nickname, hex_str(guard->identity, DIGEST_LEN), + tor_lround(pathbias_get_use_success_count(guard)), + tor_lround(guard->use_attempts), + tor_lround(pathbias_get_close_success_count(guard)), + tor_lround(guard->circ_attempts), + tor_lround(guard->circ_successes), + tor_lround(guard->unusable_circuits), + tor_lround(guard->collapsed_circuits), + tor_lround(guard->timeouts), + tor_lround(get_circuit_build_close_time_ms()/1000)); + } + } + } +} + +/** + * Check the path bias circuit close status rates against our consensus + * parameter limits. + * + * Emits a log message if the use success rates are too low. + * + * If pathbias_get_dropguards() is set, we also disable the use of + * very failure prone guards. + * + * XXX: This function shares similar log messages and checks to + * pathbias_measure_use_rate(). It may be possible to combine them + * eventually, especially if we can ever remove the need for 3 + * levels of closure warns (if the overall circuit failure rate + * goes down with ntor). One way to do so would be to multiply + * the build rate with the use rate to get an idea of the total + * fraction of the total network paths the user is able to use. + * See ticket #8159. + */ +static void +pathbias_measure_close_rate(entry_guard_t *guard) +{ + const or_options_t *options = get_options(); + + if (guard->circ_attempts > pathbias_get_min_circs(options)) { + /* Note: We rely on the < comparison here to allow us to set a 0 + * rate and disable the feature entirely. If refactoring, don't + * change to <= */ + if (pathbias_get_close_success_count(guard)/guard->circ_attempts + < pathbias_get_extreme_rate(options)) { + /* Dropping is currently disabled by default. */ + if (pathbias_get_dropguards(options)) { + if (!guard->path_bias_disabled) { + log_warn(LD_CIRC, + "Your Guard %s ($%s) is failing an extremely large " + "amount of circuits. " + "To avoid potential route manipulation attacks, Tor has " + "disabled use of this guard. " + "Success counts are %ld/%ld. Use counts are %ld/%ld. " + "%ld circuits completed, %ld were unusable, %ld collapsed, " + "and %ld timed out. " + "For reference, your timeout cutoff is %ld seconds.", + guard->nickname, hex_str(guard->identity, DIGEST_LEN), + tor_lround(pathbias_get_close_success_count(guard)), + tor_lround(guard->circ_attempts), + tor_lround(pathbias_get_use_success_count(guard)), + tor_lround(guard->use_attempts), + tor_lround(guard->circ_successes), + tor_lround(guard->unusable_circuits), + tor_lround(guard->collapsed_circuits), + tor_lround(guard->timeouts), + tor_lround(get_circuit_build_close_time_ms()/1000)); + guard->path_bias_disabled = 1; + guard->bad_since = approx_time(); + entry_guards_changed(); + return; + } + } else if (!guard->path_bias_extreme) { + guard->path_bias_extreme = 1; + log_warn(LD_CIRC, + "Your Guard %s ($%s) is failing an extremely large " + "amount of circuits. " + "This could indicate a route manipulation attack, " + "extreme network overload, or a bug. " + "Success counts are %ld/%ld. Use counts are %ld/%ld. " + "%ld circuits completed, %ld were unusable, %ld collapsed, " + "and %ld timed out. " + "For reference, your timeout cutoff is %ld seconds.", + guard->nickname, hex_str(guard->identity, DIGEST_LEN), + tor_lround(pathbias_get_close_success_count(guard)), + tor_lround(guard->circ_attempts), + tor_lround(pathbias_get_use_success_count(guard)), + tor_lround(guard->use_attempts), + tor_lround(guard->circ_successes), + tor_lround(guard->unusable_circuits), + tor_lround(guard->collapsed_circuits), + tor_lround(guard->timeouts), + tor_lround(get_circuit_build_close_time_ms()/1000)); + } + } else if (pathbias_get_close_success_count(guard)/guard->circ_attempts + < pathbias_get_warn_rate(options)) { + if (!guard->path_bias_warned) { + guard->path_bias_warned = 1; + log_warn(LD_CIRC, + "Your Guard %s ($%s) is failing a very large " + "amount of circuits. " + "Most likely this means the Tor network is " + "overloaded, but it could also mean an attack against " + "you or potentially the guard itself. " + "Success counts are %ld/%ld. Use counts are %ld/%ld. " + "%ld circuits completed, %ld were unusable, %ld collapsed, " + "and %ld timed out. " + "For reference, your timeout cutoff is %ld seconds.", + guard->nickname, hex_str(guard->identity, DIGEST_LEN), + tor_lround(pathbias_get_close_success_count(guard)), + tor_lround(guard->circ_attempts), + tor_lround(pathbias_get_use_success_count(guard)), + tor_lround(guard->use_attempts), + tor_lround(guard->circ_successes), + tor_lround(guard->unusable_circuits), + tor_lround(guard->collapsed_circuits), + tor_lround(guard->timeouts), + tor_lround(get_circuit_build_close_time_ms()/1000)); + } + } else if (pathbias_get_close_success_count(guard)/guard->circ_attempts + < pathbias_get_notice_rate(options)) { + if (!guard->path_bias_noticed) { + guard->path_bias_noticed = 1; + log_notice(LD_CIRC, + "Your Guard %s ($%s) is failing more circuits than " + "usual. " + "Most likely this means the Tor network is overloaded. " + "Success counts are %ld/%ld. Use counts are %ld/%ld. " + "%ld circuits completed, %ld were unusable, %ld collapsed, " + "and %ld timed out. " + "For reference, your timeout cutoff is %ld seconds.", + guard->nickname, hex_str(guard->identity, DIGEST_LEN), + tor_lround(pathbias_get_close_success_count(guard)), + tor_lround(guard->circ_attempts), + tor_lround(pathbias_get_use_success_count(guard)), + tor_lround(guard->use_attempts), + tor_lround(guard->circ_successes), + tor_lround(guard->unusable_circuits), + tor_lround(guard->collapsed_circuits), + tor_lround(guard->timeouts), + tor_lround(get_circuit_build_close_time_ms()/1000)); + } + } + } +} + +/** + * This function scales the path bias use rates if we have + * more data than the scaling threshold. This allows us to + * be more sensitive to recent measurements. + * + * XXX: The attempt count transfer stuff here might be done + * better by keeping separate pending counters that get + * transfered at circuit close. See ticket #8160. + */ +static void +pathbias_scale_close_rates(entry_guard_t *guard) +{ + const or_options_t *options = get_options(); + + /* If we get a ton of circuits, just scale everything down */ + if (guard->circ_attempts > pathbias_get_scale_threshold(options)) { + double scale_ratio = pathbias_get_scale_ratio(options); + int opened_attempts = pathbias_count_circs_in_states(guard, + PATH_STATE_BUILD_ATTEMPTED, PATH_STATE_BUILD_ATTEMPTED); + int opened_built = pathbias_count_circs_in_states(guard, + PATH_STATE_BUILD_SUCCEEDED, + PATH_STATE_USE_FAILED); + /* Verify that the counts are sane before and after scaling */ + int counts_are_sane = (guard->circ_attempts >= guard->circ_successes); + + guard->circ_attempts -= (opened_attempts+opened_built); + guard->circ_successes -= opened_built; + + guard->circ_attempts *= scale_ratio; + guard->circ_successes *= scale_ratio; + guard->timeouts *= scale_ratio; + guard->successful_circuits_closed *= scale_ratio; + guard->collapsed_circuits *= scale_ratio; + guard->unusable_circuits *= scale_ratio; + + guard->circ_attempts += (opened_attempts+opened_built); + guard->circ_successes += opened_built; + + entry_guards_changed(); + + log_info(LD_CIRC, + "Scaled pathbias counts to (%f,%f)/%f (%d/%d open) for guard " + "%s ($%s)", + guard->circ_successes, guard->successful_circuits_closed, + guard->circ_attempts, opened_built, opened_attempts, + guard->nickname, hex_str(guard->identity, DIGEST_LEN)); + + /* Have the counts just become invalid by this scaling attempt? */ + if (counts_are_sane && guard->circ_attempts < guard->circ_successes) { + log_notice(LD_BUG, + "Scaling has mangled pathbias counts to %f/%f (%d/%d open) " + "for guard %s ($%s)", + guard->circ_successes, guard->circ_attempts, opened_built, + opened_attempts, guard->nickname, + hex_str(guard->identity, DIGEST_LEN)); + } + } +} + +/** + * This function scales the path bias circuit close rates if we have + * more data than the scaling threshold. This allows us to be more + * sensitive to recent measurements. + * + * XXX: The attempt count transfer stuff here might be done + * better by keeping separate pending counters that get + * transfered at circuit close. See ticket #8160. + */ +void +pathbias_scale_use_rates(entry_guard_t *guard) +{ + const or_options_t *options = get_options(); + + /* If we get a ton of circuits, just scale everything down */ + if (guard->use_attempts > pathbias_get_scale_use_threshold(options)) { + double scale_ratio = pathbias_get_scale_ratio(options); + int opened_attempts = pathbias_count_circs_in_states(guard, + PATH_STATE_USE_ATTEMPTED, PATH_STATE_USE_SUCCEEDED); + /* Verify that the counts are sane before and after scaling */ + int counts_are_sane = (guard->use_attempts >= guard->use_successes); + + guard->use_attempts -= opened_attempts; + + guard->use_attempts *= scale_ratio; + guard->use_successes *= scale_ratio; + + guard->use_attempts += opened_attempts; + + log_info(LD_CIRC, + "Scaled pathbias use counts to %f/%f (%d open) for guard %s ($%s)", + guard->use_successes, guard->use_attempts, opened_attempts, + guard->nickname, hex_str(guard->identity, DIGEST_LEN)); + + /* Have the counts just become invalid by this scaling attempt? */ + if (counts_are_sane && guard->use_attempts < guard->use_successes) { + log_notice(LD_BUG, + "Scaling has mangled pathbias usage counts to %f/%f " + "(%d open) for guard %s ($%s)", + guard->circ_successes, guard->circ_attempts, + opened_attempts, guard->nickname, + hex_str(guard->identity, DIGEST_LEN)); + } + + entry_guards_changed(); + } +} + diff --git a/src/or/circpathbias.h b/src/or/circpathbias.h new file mode 100644 index 0000000000..c95d801a4b --- /dev/null +++ b/src/or/circpathbias.h @@ -0,0 +1,29 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2013, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file circuitbuild.h + * \brief Header file for circuitbuild.c. + **/ + +#ifndef TOR_CIRCPATHBIAS_H +#define TOR_CIRCPATHBIAS_H + +double pathbias_get_extreme_rate(const or_options_t *options); +double pathbias_get_extreme_use_rate(const or_options_t *options); +int pathbias_get_dropguards(const or_options_t *options); +void pathbias_count_timeout(origin_circuit_t *circ); +void pathbias_count_build_success(origin_circuit_t *circ); +int pathbias_count_build_attempt(origin_circuit_t *circ); +int pathbias_check_close(origin_circuit_t *circ, int reason); +int pathbias_check_probe_response(circuit_t *circ, const cell_t *cell); +void pathbias_count_use_attempt(origin_circuit_t *circ); +void pathbias_mark_use_success(origin_circuit_t *circ); +void pathbias_mark_use_rollback(origin_circuit_t *circ); +const char *pathbias_state_to_string(path_state_t state); + +#endif + diff --git a/src/or/circuitbuild.c b/src/or/circuitbuild.c index 5325eff64a..897f90fe4c 100644 --- a/src/or/circuitbuild.c +++ b/src/or/circuitbuild.c @@ -9,8 +9,11 @@ * \brief The actual details of building circuits. **/ +#define CIRCUITBUILD_PRIVATE + #include "or.h" #include "channel.h" +#include "circpathbias.h" #include "circuitbuild.h" #include "circuitlist.h" #include "circuitstats.h" @@ -40,19 +43,11 @@ #include "routerparse.h" #include "routerset.h" #include "crypto.h" -#include "connection_edge.h" #ifndef MIN #define MIN(a,b) ((a)<(b)?(a):(b)) #endif -/********* START VARIABLES **********/ - -/** A global list of all circuits at this hop. */ -extern circuit_t *global_circuitlist; - -/********* END VARIABLES ************/ - static channel_t * channel_connect_for_circuit(const tor_addr_t *addr, uint16_t port, const char *id_digest); @@ -64,14 +59,6 @@ static crypt_path_t *onion_next_hop_in_cpath(crypt_path_t *cpath); static int onion_extend_cpath(origin_circuit_t *circ); static int count_acceptable_nodes(smartlist_t *routers); static int onion_append_hop(crypt_path_t **head_ptr, extend_info_t *choice); -static int entry_guard_inc_circ_attempt_count(entry_guard_t *guard); -static void pathbias_count_build_success(origin_circuit_t *circ); -static void pathbias_count_successful_close(origin_circuit_t *circ); -static void pathbias_count_collapse(origin_circuit_t *circ); -static void pathbias_count_use_failed(origin_circuit_t *circ); -static void pathbias_measure_use_rate(entry_guard_t *guard); -static void pathbias_measure_close_rate(entry_guard_t *guard); -static void pathbias_scale_use_rates(entry_guard_t *guard); #ifdef CURVE25519_ENABLED static int circuits_can_use_ntor(void); #endif @@ -92,18 +79,29 @@ channel_connect_for_circuit(const tor_addr_t *addr, uint16_t port, return chan; } -/** Iterate over values of circ_id, starting from conn-\>next_circ_id, - * and with the high bit specified by conn-\>circ_id_type, until we get - * a circ_id that is not in use by any other circuit on that conn. +/** Search for a value for circ_id that we can use on <b>chan</b> for an + * outbound circuit, until we get a circ_id that is not in use by any other + * circuit on that conn. * * Return it, or 0 if can't get a unique circ_id. */ -static circid_t +STATIC circid_t get_unique_circ_id_by_chan(channel_t *chan) { +/* This number is chosen somewhat arbitrarily; see comment below for more + * info. When the space is 80% full, it gives a one-in-a-million failure + * chance; when the space is 90% full, it gives a one-in-850 chance; and when + * the space is 95% full, it gives a one-in-26 failure chance. That seems + * okay, though you could make a case IMO for anything between N=32 and + * N=256. */ +#define MAX_CIRCID_ATTEMPTS 64 + int in_use; + unsigned n_with_circ = 0, n_pending_destroy = 0, n_weird_pending_destroy = 0; circid_t test_circ_id; circid_t attempts=0; - circid_t high_bit, max_range; + circid_t high_bit, max_range, mask; + int64_t pending_destroy_time_total = 0; + int64_t pending_destroy_time_max = 0; tor_assert(chan); @@ -113,32 +111,108 @@ get_unique_circ_id_by_chan(channel_t *chan) "a client with no identity."); return 0; } - max_range = (chan->wide_circ_ids) ? (1u<<31) : (1u<<15); + max_range = (chan->wide_circ_ids) ? (1u<<31) : (1u<<15); + mask = max_range - 1; high_bit = (chan->circ_id_type == CIRC_ID_TYPE_HIGHER) ? max_range : 0; do { - /* Sequentially iterate over test_circ_id=1...max_range until we find a - * circID such that (high_bit|test_circ_id) is not already used. */ - test_circ_id = chan->next_circ_id++; - if (test_circ_id == 0 || test_circ_id >= max_range) { - test_circ_id = 1; - chan->next_circ_id = 2; - } - if (++attempts > max_range) { - /* Make sure we don't loop forever if all circ_id's are used. This - * matters because it's an external DoS opportunity. + if (++attempts > MAX_CIRCID_ATTEMPTS) { + /* Make sure we don't loop forever because all circuit IDs are used. + * + * Once, we would try until we had tried every possible circuit ID. But + * that's quite expensive. Instead, we try MAX_CIRCID_ATTEMPTS random + * circuit IDs, and then give up. + * + * This potentially causes us to give up early if our circuit ID space + * is nearly full. If we have N circuit IDs in use, then we will reject + * a new circuit with probability (N / max_range) ^ MAX_CIRCID_ATTEMPTS. + * This means that in practice, a few percent of our circuit ID capacity + * will go unused. + * + * The alternative here, though, is to do a linear search over the + * whole circuit ID space every time we extend a circuit, which is + * not so great either. */ - if (! chan->warned_circ_ids_exhausted) { - chan->warned_circ_ids_exhausted = 1; - log_warn(LD_CIRC,"No unused circIDs found on channel %s wide " + int64_t queued_destroys; + char *m = rate_limit_log(&chan->last_warned_circ_ids_exhausted, + approx_time()); + if (m == NULL) + return 0; /* This message has been rate-limited away. */ + if (n_pending_destroy) + pending_destroy_time_total /= n_pending_destroy; + log_warn(LD_CIRC,"No unused circIDs found on channel %s wide " "circID support, with %u inbound and %u outbound circuits. " - "Failing a circuit.", + "Found %u circuit IDs in use by circuits, and %u with " + "pending destroy cells. (%u of those were marked bogusly.) " + "The ones with pending destroy cells " + "have been marked unusable for an average of %ld seconds " + "and a maximum of %ld seconds. This channel is %ld seconds " + "old. Failing a circuit.%s", chan->wide_circ_ids ? "with" : "without", - chan->num_p_circuits, chan->num_n_circuits); + chan->num_p_circuits, chan->num_n_circuits, + n_with_circ, n_pending_destroy, n_weird_pending_destroy, + (long)pending_destroy_time_total, + (long)pending_destroy_time_max, + (long)(approx_time() - chan->timestamp_created), + m); + tor_free(m); + + if (!chan->cmux) { + /* This warning should be impossible. */ + log_warn(LD_BUG, " This channel somehow has no cmux on it!"); + return 0; } + + /* analysis so far on 12184 suggests that we're running out of circuit + IDs because it looks like we have too many pending destroy + cells. Let's see how many we really have pending. + */ + queued_destroys = circuitmux_count_queued_destroy_cells(chan, + chan->cmux); + + log_warn(LD_CIRC, " Circuitmux on this channel has %u circuits, " + "of which %u are active. It says it has "I64_FORMAT + " destroy cells queued.", + circuitmux_num_circuits(chan->cmux), + circuitmux_num_active_circuits(chan->cmux), + I64_PRINTF_ARG(queued_destroys)); + + /* Change this into "if (1)" in order to get more information about + * possible failure modes here. You'll need to know how to use gdb with + * Tor: this will make Tor exit with an assertion failure if the cmux is + * corrupt. */ + if (0) + circuitmux_assert_okay(chan->cmux); + + channel_dump_statistics(chan, LOG_WARN); + return 0; } + + do { + crypto_rand((char*) &test_circ_id, sizeof(test_circ_id)); + test_circ_id &= mask; + } while (test_circ_id == 0); + test_circ_id |= high_bit; - } while (circuit_id_in_use_on_channel(test_circ_id, chan)); + + in_use = circuit_id_in_use_on_channel(test_circ_id, chan); + if (in_use == 1) + ++n_with_circ; + else if (in_use == 2) { + time_t since_when; + ++n_pending_destroy; + since_when = + circuit_id_when_marked_unusable_on_channel(test_circ_id, chan); + if (since_when) { + time_t waiting = approx_time() - since_when; + pending_destroy_time_total += waiting; + if (waiting > pending_destroy_time_max) + pending_destroy_time_max = waiting; + } else { + ++n_weird_pending_destroy; + } + } + } while (in_use); return test_circ_id; } @@ -299,9 +373,9 @@ circuit_rep_hist_note_result(origin_circuit_t *circ) static int circuit_cpath_supports_ntor(const origin_circuit_t *circ) { - crypt_path_t *head = circ->cpath, *cpath = circ->cpath; + crypt_path_t *head, *cpath; - cpath = head; + cpath = head = circ->cpath; do { if (cpath->extend_info && !tor_mem_is_zero( @@ -584,7 +658,9 @@ circuit_deliver_create_cell(circuit_t *circ, const create_cell_t *create_cell, id = get_unique_circ_id_by_chan(circ->n_chan); if (!id) { - log_warn(LD_CIRC,"failed to get unique circID."); + static ratelim_t circid_warning_limit = RATELIM_INIT(9600); + log_fn_ratelim(&circid_warning_limit, LOG_WARN, LD_CIRC, + "failed to get unique circID."); goto error; } @@ -632,27 +708,30 @@ int inform_testing_reachability(void) { char dirbuf[128]; + char *address; const routerinfo_t *me = router_get_my_routerinfo(); if (!me) return 0; + address = tor_dup_ip(me->addr); control_event_server_status(LOG_NOTICE, "CHECKING_REACHABILITY ORADDRESS=%s:%d", - me->address, me->or_port); + address, me->or_port); if (me->dir_port) { tor_snprintf(dirbuf, sizeof(dirbuf), " and DirPort %s:%d", - me->address, me->dir_port); + address, me->dir_port); control_event_server_status(LOG_NOTICE, "CHECKING_REACHABILITY DIRADDRESS=%s:%d", - me->address, me->dir_port); + address, me->dir_port); } log_notice(LD_OR, "Now checking whether ORPort %s:%d%s %s reachable... " "(this may take up to %d minutes -- look for log " "messages indicating success)", - me->address, me->or_port, + address, me->or_port, me->dir_port ? dirbuf : "", me->dir_port ? "are" : "is", TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT/60); + tor_free(address); return 1; } @@ -848,20 +927,24 @@ circuit_send_next_onion_skin(origin_circuit_t *circ) * it off at, we probably had a suspend event along this codepath, * and we should discard the value. */ - if (timediff < 0 || timediff > 2*circ_times.close_ms+1000) { + if (timediff < 0 || + timediff > 2*get_circuit_build_close_time_ms()+1000) { log_notice(LD_CIRC, "Strange value for circuit build time: %ldmsec. " "Assuming clock jump. Purpose %d (%s)", timediff, circ->base_.purpose, circuit_purpose_to_string(circ->base_.purpose)); } else if (!circuit_build_times_disabled()) { /* Only count circuit times if the network is live */ - if (circuit_build_times_network_check_live(&circ_times)) { - circuit_build_times_add_time(&circ_times, (build_time_t)timediff); - circuit_build_times_set_timeout(&circ_times); + if (circuit_build_times_network_check_live( + get_circuit_build_times())) { + circuit_build_times_add_time(get_circuit_build_times_mutable(), + (build_time_t)timediff); + circuit_build_times_set_timeout(get_circuit_build_times_mutable()); } if (circ->base_.purpose != CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT) { - circuit_build_times_network_circ_success(&circ_times); + circuit_build_times_network_circ_success( + get_circuit_build_times_mutable()); } } } @@ -1156,1516 +1239,6 @@ circuit_init_cpath_crypto(crypt_path_t *cpath, const char *key_data, return 0; } -/** The minimum number of circuit attempts before we start - * thinking about warning about path bias and dropping guards */ -static int -pathbias_get_min_circs(const or_options_t *options) -{ -#define DFLT_PATH_BIAS_MIN_CIRC 150 - if (options->PathBiasCircThreshold >= 5) - return options->PathBiasCircThreshold; - else - return networkstatus_get_param(NULL, "pb_mincircs", - DFLT_PATH_BIAS_MIN_CIRC, - 5, INT32_MAX); -} - -/** The circuit success rate below which we issue a notice */ -static double -pathbias_get_notice_rate(const or_options_t *options) -{ -#define DFLT_PATH_BIAS_NOTICE_PCT 70 - if (options->PathBiasNoticeRate >= 0.0) - return options->PathBiasNoticeRate; - else - return networkstatus_get_param(NULL, "pb_noticepct", - DFLT_PATH_BIAS_NOTICE_PCT, 0, 100)/100.0; -} - -/* XXXX024 I'd like to have this be static again, but entrynodes.c needs it. */ -/** The circuit success rate below which we issue a warn */ -static double -pathbias_get_warn_rate(const or_options_t *options) -{ -#define DFLT_PATH_BIAS_WARN_PCT 50 - if (options->PathBiasWarnRate >= 0.0) - return options->PathBiasWarnRate; - else - return networkstatus_get_param(NULL, "pb_warnpct", - DFLT_PATH_BIAS_WARN_PCT, 0, 100)/100.0; -} - -/* XXXX024 I'd like to have this be static again, but entrynodes.c needs it. */ -/** - * The extreme rate is the rate at which we would drop the guard, - * if pb_dropguard is also set. Otherwise we just warn. - */ -double -pathbias_get_extreme_rate(const or_options_t *options) -{ -#define DFLT_PATH_BIAS_EXTREME_PCT 30 - if (options->PathBiasExtremeRate >= 0.0) - return options->PathBiasExtremeRate; - else - return networkstatus_get_param(NULL, "pb_extremepct", - DFLT_PATH_BIAS_EXTREME_PCT, 0, 100)/100.0; -} - -/* XXXX024 I'd like to have this be static again, but entrynodes.c needs it. */ -/** - * If 1, we actually disable use of guards that fall below - * the extreme_pct. - */ -int -pathbias_get_dropguards(const or_options_t *options) -{ -#define DFLT_PATH_BIAS_DROP_GUARDS 0 - if (options->PathBiasDropGuards >= 0) - return options->PathBiasDropGuards; - else - return networkstatus_get_param(NULL, "pb_dropguards", - DFLT_PATH_BIAS_DROP_GUARDS, 0, 1); -} - -/** - * This is the number of circuits at which we scale our - * counts by mult_factor/scale_factor. Note, this count is - * not exact, as we only perform the scaling in the event - * of no integer truncation. - */ -static int -pathbias_get_scale_threshold(const or_options_t *options) -{ -#define DFLT_PATH_BIAS_SCALE_THRESHOLD 300 - if (options->PathBiasScaleThreshold >= 10) - return options->PathBiasScaleThreshold; - else - return networkstatus_get_param(NULL, "pb_scalecircs", - DFLT_PATH_BIAS_SCALE_THRESHOLD, 10, - INT32_MAX); -} - -/** - * Compute the path bias scaling ratio from the consensus - * parameters pb_multfactor/pb_scalefactor. - * - * Returns a value in (0, 1.0] which we multiply our pathbias - * counts with to scale them down. - */ -static double -pathbias_get_scale_ratio(const or_options_t *options) -{ - /* - * The scale factor is the denominator for our scaling - * of circuit counts for our path bias window. - * - * Note that our use of doubles for the path bias state - * file means that powers of 2 work best here. - */ - int denominator = networkstatus_get_param(NULL, "pb_scalefactor", - 2, 2, INT32_MAX); - (void) options; - /** - * The mult factor is the numerator for our scaling - * of circuit counts for our path bias window. It - * allows us to scale by fractions. - */ - return networkstatus_get_param(NULL, "pb_multfactor", - 1, 1, denominator)/((double)denominator); -} - -/** The minimum number of circuit usage attempts before we start - * thinking about warning about path use bias and dropping guards */ -static int -pathbias_get_min_use(const or_options_t *options) -{ -#define DFLT_PATH_BIAS_MIN_USE 20 - if (options->PathBiasUseThreshold >= 3) - return options->PathBiasUseThreshold; - else - return networkstatus_get_param(NULL, "pb_minuse", - DFLT_PATH_BIAS_MIN_USE, - 3, INT32_MAX); -} - -/** The circuit use success rate below which we issue a notice */ -static double -pathbias_get_notice_use_rate(const or_options_t *options) -{ -#define DFLT_PATH_BIAS_NOTICE_USE_PCT 80 - if (options->PathBiasNoticeUseRate >= 0.0) - return options->PathBiasNoticeUseRate; - else - return networkstatus_get_param(NULL, "pb_noticeusepct", - DFLT_PATH_BIAS_NOTICE_USE_PCT, - 0, 100)/100.0; -} - -/** - * The extreme use rate is the rate at which we would drop the guard, - * if pb_dropguard is also set. Otherwise we just warn. - */ -double -pathbias_get_extreme_use_rate(const or_options_t *options) -{ -#define DFLT_PATH_BIAS_EXTREME_USE_PCT 60 - if (options->PathBiasExtremeUseRate >= 0.0) - return options->PathBiasExtremeUseRate; - else - return networkstatus_get_param(NULL, "pb_extremeusepct", - DFLT_PATH_BIAS_EXTREME_USE_PCT, - 0, 100)/100.0; -} - -/** - * This is the number of circuits at which we scale our - * use counts by mult_factor/scale_factor. Note, this count is - * not exact, as we only perform the scaling in the event - * of no integer truncation. - */ -static int -pathbias_get_scale_use_threshold(const or_options_t *options) -{ -#define DFLT_PATH_BIAS_SCALE_USE_THRESHOLD 100 - if (options->PathBiasScaleUseThreshold >= 10) - return options->PathBiasScaleUseThreshold; - else - return networkstatus_get_param(NULL, "pb_scaleuse", - DFLT_PATH_BIAS_SCALE_USE_THRESHOLD, - 10, INT32_MAX); -} - -/** - * Convert a Guard's path state to string. - */ -const char * -pathbias_state_to_string(path_state_t state) -{ - switch (state) { - case PATH_STATE_NEW_CIRC: - return "new"; - case PATH_STATE_BUILD_ATTEMPTED: - return "build attempted"; - case PATH_STATE_BUILD_SUCCEEDED: - return "build succeeded"; - case PATH_STATE_USE_ATTEMPTED: - return "use attempted"; - case PATH_STATE_USE_SUCCEEDED: - return "use succeeded"; - case PATH_STATE_USE_FAILED: - return "use failed"; - case PATH_STATE_ALREADY_COUNTED: - return "already counted"; - } - - return "unknown"; -} - -/** - * This function decides if a circuit has progressed far enough to count - * as a circuit "attempt". As long as end-to-end tagging is possible, - * we assume the adversary will use it over hop-to-hop failure. Therefore, - * we only need to account bias for the last hop. This should make us - * much more resilient to ambient circuit failure, and also make that - * failure easier to measure (we only need to measure Exit failure rates). - */ -static int -pathbias_is_new_circ_attempt(origin_circuit_t *circ) -{ -#define N2N_TAGGING_IS_POSSIBLE -#ifdef N2N_TAGGING_IS_POSSIBLE - /* cpath is a circular list. We want circs with more than one hop, - * and the second hop must be waiting for keys still (it's just - * about to get them). */ - return circ->cpath && - circ->cpath->next != circ->cpath && - circ->cpath->next->state == CPATH_STATE_AWAITING_KEYS; -#else - /* If tagging attacks are no longer possible, we probably want to - * count bias from the first hop. However, one could argue that - * timing-based tagging is still more useful than per-hop failure. - * In which case, we'd never want to use this. - */ - return circ->cpath && - circ->cpath->state == CPATH_STATE_AWAITING_KEYS; -#endif -} - -/** - * Decide if the path bias code should count a circuit. - * - * @returns 1 if we should count it, 0 otherwise. - */ -static int -pathbias_should_count(origin_circuit_t *circ) -{ -#define PATHBIAS_COUNT_INTERVAL (600) - static ratelim_t count_limit = - RATELIM_INIT(PATHBIAS_COUNT_INTERVAL); - char *rate_msg = NULL; - - /* We can't do path bias accounting without entry guards. - * Testing and controller circuits also have no guards. - * - * We also don't count server-side rends, because their - * endpoint could be chosen maliciously. - * Similarly, we can't count client-side intro attempts, - * because clients can be manipulated into connecting to - * malicious intro points. */ - if (get_options()->UseEntryGuards == 0 || - circ->base_.purpose == CIRCUIT_PURPOSE_TESTING || - circ->base_.purpose == CIRCUIT_PURPOSE_CONTROLLER || - circ->base_.purpose == CIRCUIT_PURPOSE_S_CONNECT_REND || - circ->base_.purpose == CIRCUIT_PURPOSE_S_REND_JOINED || - (circ->base_.purpose >= CIRCUIT_PURPOSE_C_INTRODUCING && - circ->base_.purpose <= CIRCUIT_PURPOSE_C_INTRODUCE_ACKED)) { - - /* Check to see if the shouldcount result has changed due to a - * unexpected purpose change that would affect our results. - * - * The reason we check the path state too here is because for the - * cannibalized versions of these purposes, we count them as successful - * before their purpose change. - */ - if (circ->pathbias_shouldcount == PATHBIAS_SHOULDCOUNT_COUNTED - && circ->path_state != PATH_STATE_ALREADY_COUNTED) { - log_info(LD_BUG, - "Circuit %d is now being ignored despite being counted " - "in the past. Purpose is %s, path state is %s", - circ->global_identifier, - circuit_purpose_to_string(circ->base_.purpose), - pathbias_state_to_string(circ->path_state)); - } - circ->pathbias_shouldcount = PATHBIAS_SHOULDCOUNT_IGNORED; - return 0; - } - - /* Completely ignore one hop circuits */ - if (circ->build_state->onehop_tunnel || - circ->build_state->desired_path_len == 1) { - /* Check for inconsistency */ - if (circ->build_state->desired_path_len != 1 || - !circ->build_state->onehop_tunnel) { - if ((rate_msg = rate_limit_log(&count_limit, approx_time()))) { - log_info(LD_BUG, - "One-hop circuit has length %d. Path state is %s. " - "Circuit is a %s currently %s.%s", - circ->build_state->desired_path_len, - pathbias_state_to_string(circ->path_state), - circuit_purpose_to_string(circ->base_.purpose), - circuit_state_to_string(circ->base_.state), - rate_msg); - tor_free(rate_msg); - } - tor_fragile_assert(); - } - - /* Check to see if the shouldcount result has changed due to a - * unexpected change that would affect our results */ - if (circ->pathbias_shouldcount == PATHBIAS_SHOULDCOUNT_COUNTED) { - log_info(LD_BUG, - "One-hop circuit %d is now being ignored despite being counted " - "in the past. Purpose is %s, path state is %s", - circ->global_identifier, - circuit_purpose_to_string(circ->base_.purpose), - pathbias_state_to_string(circ->path_state)); - } - circ->pathbias_shouldcount = PATHBIAS_SHOULDCOUNT_IGNORED; - return 0; - } - - /* Check to see if the shouldcount result has changed due to a - * unexpected purpose change that would affect our results */ - if (circ->pathbias_shouldcount == PATHBIAS_SHOULDCOUNT_IGNORED) { - log_info(LD_BUG, - "Circuit %d is now being counted despite being ignored " - "in the past. Purpose is %s, path state is %s", - circ->global_identifier, - circuit_purpose_to_string(circ->base_.purpose), - pathbias_state_to_string(circ->path_state)); - } - circ->pathbias_shouldcount = PATHBIAS_SHOULDCOUNT_COUNTED; - - return 1; -} - -/** - * Check our circuit state to see if this is a successful circuit attempt. - * If so, record it in the current guard's path bias circ_attempt count. - * - * Also check for several potential error cases for bug #6475. - */ -static int -pathbias_count_build_attempt(origin_circuit_t *circ) -{ -#define CIRC_ATTEMPT_NOTICE_INTERVAL (600) - static ratelim_t circ_attempt_notice_limit = - RATELIM_INIT(CIRC_ATTEMPT_NOTICE_INTERVAL); - char *rate_msg = NULL; - - if (!pathbias_should_count(circ)) { - return 0; - } - - if (pathbias_is_new_circ_attempt(circ)) { - /* Help track down the real cause of bug #6475: */ - if (circ->has_opened && circ->path_state != PATH_STATE_BUILD_ATTEMPTED) { - if ((rate_msg = rate_limit_log(&circ_attempt_notice_limit, - approx_time()))) { - log_info(LD_BUG, - "Opened circuit is in strange path state %s. " - "Circuit is a %s currently %s.%s", - pathbias_state_to_string(circ->path_state), - circuit_purpose_to_string(circ->base_.purpose), - circuit_state_to_string(circ->base_.state), - rate_msg); - tor_free(rate_msg); - } - } - - /* Don't re-count cannibalized circs.. */ - if (!circ->has_opened) { - entry_guard_t *guard = NULL; - - if (circ->cpath && circ->cpath->extend_info) { - guard = entry_guard_get_by_id_digest( - circ->cpath->extend_info->identity_digest); - } else if (circ->base_.n_chan) { - guard = - entry_guard_get_by_id_digest(circ->base_.n_chan->identity_digest); - } - - if (guard) { - if (circ->path_state == PATH_STATE_NEW_CIRC) { - circ->path_state = PATH_STATE_BUILD_ATTEMPTED; - - if (entry_guard_inc_circ_attempt_count(guard) < 0) { - /* Bogus guard; we already warned. */ - return -END_CIRC_REASON_TORPROTOCOL; - } - } else { - if ((rate_msg = rate_limit_log(&circ_attempt_notice_limit, - approx_time()))) { - log_info(LD_BUG, - "Unopened circuit has strange path state %s. " - "Circuit is a %s currently %s.%s", - pathbias_state_to_string(circ->path_state), - circuit_purpose_to_string(circ->base_.purpose), - circuit_state_to_string(circ->base_.state), - rate_msg); - tor_free(rate_msg); - } - } - } else { - if ((rate_msg = rate_limit_log(&circ_attempt_notice_limit, - approx_time()))) { - log_info(LD_CIRC, - "Unopened circuit has no known guard. " - "Circuit is a %s currently %s.%s", - circuit_purpose_to_string(circ->base_.purpose), - circuit_state_to_string(circ->base_.state), - rate_msg); - tor_free(rate_msg); - } - } - } - } - - return 0; -} - -/** - * Check our circuit state to see if this is a successful circuit - * completion. If so, record it in the current guard's path bias - * success count. - * - * Also check for several potential error cases for bug #6475. - */ -static void -pathbias_count_build_success(origin_circuit_t *circ) -{ -#define SUCCESS_NOTICE_INTERVAL (600) - static ratelim_t success_notice_limit = - RATELIM_INIT(SUCCESS_NOTICE_INTERVAL); - char *rate_msg = NULL; - entry_guard_t *guard = NULL; - - if (!pathbias_should_count(circ)) { - return; - } - - /* Don't count cannibalized/reused circs for path bias - * "build" success, since they get counted under "use" success. */ - if (!circ->has_opened) { - if (circ->cpath && circ->cpath->extend_info) { - guard = entry_guard_get_by_id_digest( - circ->cpath->extend_info->identity_digest); - } - - if (guard) { - if (circ->path_state == PATH_STATE_BUILD_ATTEMPTED) { - circ->path_state = PATH_STATE_BUILD_SUCCEEDED; - guard->circ_successes++; - entry_guards_changed(); - - log_info(LD_CIRC, "Got success count %f/%f for guard %s ($%s)", - guard->circ_successes, guard->circ_attempts, - guard->nickname, hex_str(guard->identity, DIGEST_LEN)); - } else { - if ((rate_msg = rate_limit_log(&success_notice_limit, - approx_time()))) { - log_info(LD_BUG, - "Succeeded circuit is in strange path state %s. " - "Circuit is a %s currently %s.%s", - pathbias_state_to_string(circ->path_state), - circuit_purpose_to_string(circ->base_.purpose), - circuit_state_to_string(circ->base_.state), - rate_msg); - tor_free(rate_msg); - } - } - - if (guard->circ_attempts < guard->circ_successes) { - log_notice(LD_BUG, "Unexpectedly high successes counts (%f/%f) " - "for guard %s ($%s)", - guard->circ_successes, guard->circ_attempts, - guard->nickname, hex_str(guard->identity, DIGEST_LEN)); - } - /* In rare cases, CIRCUIT_PURPOSE_TESTING can get converted to - * CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT and have no guards here. - * No need to log that case. */ - } else if (circ->base_.purpose != CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT) { - if ((rate_msg = rate_limit_log(&success_notice_limit, - approx_time()))) { - log_info(LD_CIRC, - "Completed circuit has no known guard. " - "Circuit is a %s currently %s.%s", - circuit_purpose_to_string(circ->base_.purpose), - circuit_state_to_string(circ->base_.state), - rate_msg); - tor_free(rate_msg); - } - } - } else { - if (circ->path_state < PATH_STATE_BUILD_SUCCEEDED) { - if ((rate_msg = rate_limit_log(&success_notice_limit, - approx_time()))) { - log_info(LD_BUG, - "Opened circuit is in strange path state %s. " - "Circuit is a %s currently %s.%s", - pathbias_state_to_string(circ->path_state), - circuit_purpose_to_string(circ->base_.purpose), - circuit_state_to_string(circ->base_.state), - rate_msg); - tor_free(rate_msg); - } - } - } -} - -/** - * Record an attempt to use a circuit. Changes the circuit's - * path state and update its guard's usage counter. - * - * Used for path bias usage accounting. - */ -void -pathbias_count_use_attempt(origin_circuit_t *circ) -{ - entry_guard_t *guard; - - if (!pathbias_should_count(circ)) { - return; - } - - if (circ->path_state < PATH_STATE_BUILD_SUCCEEDED) { - log_notice(LD_BUG, - "Used circuit is in strange path state %s. " - "Circuit is a %s currently %s.", - pathbias_state_to_string(circ->path_state), - circuit_purpose_to_string(circ->base_.purpose), - circuit_state_to_string(circ->base_.state)); - } else if (circ->path_state < PATH_STATE_USE_ATTEMPTED) { - guard = entry_guard_get_by_id_digest( - circ->cpath->extend_info->identity_digest); - if (guard) { - pathbias_measure_use_rate(guard); - pathbias_scale_use_rates(guard); - guard->use_attempts++; - entry_guards_changed(); - - log_debug(LD_CIRC, - "Marked circuit %d (%f/%f) as used for guard %s ($%s).", - circ->global_identifier, - guard->use_successes, guard->use_attempts, - guard->nickname, hex_str(guard->identity, DIGEST_LEN)); - } - - circ->path_state = PATH_STATE_USE_ATTEMPTED; - } else { - /* Harmless but educational log message */ - log_info(LD_CIRC, - "Used circuit %d is already in path state %s. " - "Circuit is a %s currently %s.", - circ->global_identifier, - pathbias_state_to_string(circ->path_state), - circuit_purpose_to_string(circ->base_.purpose), - circuit_state_to_string(circ->base_.state)); - } - - return; -} - -/** - * Check the circuit's path state is appropriate and mark it as - * successfully used. Used for path bias usage accounting. - * - * We don't actually increment the guard's counters until - * pathbias_check_close(), because the circuit can still transition - * back to PATH_STATE_USE_ATTEMPTED if a stream fails later (this - * is done so we can probe the circuit for liveness at close). - */ -void -pathbias_mark_use_success(origin_circuit_t *circ) -{ - if (!pathbias_should_count(circ)) { - return; - } - - if (circ->path_state < PATH_STATE_USE_ATTEMPTED) { - log_notice(LD_BUG, - "Used circuit %d is in strange path state %s. " - "Circuit is a %s currently %s.", - circ->global_identifier, - pathbias_state_to_string(circ->path_state), - circuit_purpose_to_string(circ->base_.purpose), - circuit_state_to_string(circ->base_.state)); - - pathbias_count_use_attempt(circ); - } - - /* We don't do any accounting at the guard until actual circuit close */ - circ->path_state = PATH_STATE_USE_SUCCEEDED; - - return; -} - -/** - * If a stream ever detatches from a circuit in a retriable way, - * we need to mark this circuit as still needing either another - * successful stream, or in need of a probe. - * - * An adversary could let the first stream request succeed (ie the - * resolve), but then tag and timeout the remainder (via cell - * dropping), forcing them on new circuits. - * - * Rolling back the state will cause us to probe such circuits, which - * should lead to probe failures in the event of such tagging due to - * either unrecognized cells coming in while we wait for the probe, - * or the cipher state getting out of sync in the case of dropped cells. - */ -void -pathbias_mark_use_rollback(origin_circuit_t *circ) -{ - if (circ->path_state == PATH_STATE_USE_SUCCEEDED) { - log_info(LD_CIRC, - "Rolling back pathbias use state to 'attempted' for detached " - "circuit %d", circ->global_identifier); - circ->path_state = PATH_STATE_USE_ATTEMPTED; - } -} - -/** - * Actually count a circuit success towards a guard's usage counters - * if the path state is appropriate. - */ -static void -pathbias_count_use_success(origin_circuit_t *circ) -{ - entry_guard_t *guard; - - if (!pathbias_should_count(circ)) { - return; - } - - if (circ->path_state != PATH_STATE_USE_SUCCEEDED) { - log_notice(LD_BUG, - "Successfully used circuit %d is in strange path state %s. " - "Circuit is a %s currently %s.", - circ->global_identifier, - pathbias_state_to_string(circ->path_state), - circuit_purpose_to_string(circ->base_.purpose), - circuit_state_to_string(circ->base_.state)); - } else { - guard = entry_guard_get_by_id_digest( - circ->cpath->extend_info->identity_digest); - if (guard) { - guard->use_successes++; - entry_guards_changed(); - - if (guard->use_attempts < guard->use_successes) { - log_notice(LD_BUG, "Unexpectedly high use successes counts (%f/%f) " - "for guard %s=%s", - guard->use_successes, guard->use_attempts, - guard->nickname, hex_str(guard->identity, DIGEST_LEN)); - } - - log_debug(LD_CIRC, - "Marked circuit %d (%f/%f) as used successfully for guard " - "%s ($%s).", - circ->global_identifier, guard->use_successes, - guard->use_attempts, guard->nickname, - hex_str(guard->identity, DIGEST_LEN)); - } - } - - return; -} - -/** - * Send a probe down a circuit that the client attempted to use, - * but for which the stream timed out/failed. The probe is a - * RELAY_BEGIN cell with a 0.a.b.c destination address, which - * the exit will reject and reply back, echoing that address. - * - * The reason for such probes is because it is possible to bias - * a user's paths simply by causing timeouts, and these timeouts - * are not possible to differentiate from unresponsive servers. - * - * The probe is sent at the end of the circuit lifetime for two - * reasons: to prevent cryptographic taggers from being able to - * drop cells to cause timeouts, and to prevent easy recognition - * of probes before any real client traffic happens. - * - * Returns -1 if we couldn't probe, 0 otherwise. - */ -static int -pathbias_send_usable_probe(circuit_t *circ) -{ - /* Based on connection_ap_handshake_send_begin() */ - char payload[CELL_PAYLOAD_SIZE]; - int payload_len; - origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ); - crypt_path_t *cpath_layer = NULL; - char *probe_nonce = NULL; - - tor_assert(ocirc); - - cpath_layer = ocirc->cpath->prev; - - if (cpath_layer->state != CPATH_STATE_OPEN) { - /* This can happen for cannibalized circuits. Their - * last hop isn't yet open */ - log_info(LD_CIRC, - "Got pathbias probe request for unopened circuit %d. " - "Opened %d, len %d", ocirc->global_identifier, - ocirc->has_opened, ocirc->build_state->desired_path_len); - return -1; - } - - /* We already went down this road. */ - if (circ->purpose == CIRCUIT_PURPOSE_PATH_BIAS_TESTING && - ocirc->pathbias_probe_id) { - log_info(LD_CIRC, - "Got pathbias probe request for circuit %d with " - "outstanding probe", ocirc->global_identifier); - return -1; - } - - /* Can't probe if the channel isn't open */ - if (circ->n_chan == NULL || - (circ->n_chan->state != CHANNEL_STATE_OPEN - && circ->n_chan->state != CHANNEL_STATE_MAINT)) { - log_info(LD_CIRC, - "Skipping pathbias probe for circuit %d: Channel is not open.", - ocirc->global_identifier); - return -1; - } - - circuit_change_purpose(circ, CIRCUIT_PURPOSE_PATH_BIAS_TESTING); - - /* Update timestamp for when circuit_expire_building() should kill us */ - tor_gettimeofday(&circ->timestamp_began); - - /* Generate a random address for the nonce */ - crypto_rand((char*)ô->pathbias_probe_nonce, - sizeof(ocirc->pathbias_probe_nonce)); - ocirc->pathbias_probe_nonce &= 0x00ffffff; - probe_nonce = tor_dup_ip(ocirc->pathbias_probe_nonce); - - tor_snprintf(payload,RELAY_PAYLOAD_SIZE, "%s:25", probe_nonce); - payload_len = (int)strlen(payload)+1; - - // XXX: need this? Can we assume ipv4 will always be supported? - // If not, how do we tell? - //if (payload_len <= RELAY_PAYLOAD_SIZE - 4 && edge_conn->begincell_flags) { - // set_uint32(payload + payload_len, htonl(edge_conn->begincell_flags)); - // payload_len += 4; - //} - - /* Generate+Store stream id, make sure it's non-zero */ - ocirc->pathbias_probe_id = get_unique_stream_id_by_circ(ocirc); - - if (ocirc->pathbias_probe_id==0) { - log_warn(LD_CIRC, - "Ran out of stream IDs on circuit %u during " - "pathbias probe attempt.", ocirc->global_identifier); - tor_free(probe_nonce); - return -1; - } - - log_info(LD_CIRC, - "Sending pathbias testing cell to %s:25 on stream %d for circ %d.", - probe_nonce, ocirc->pathbias_probe_id, ocirc->global_identifier); - tor_free(probe_nonce); - - /* Send a test relay cell */ - if (relay_send_command_from_edge(ocirc->pathbias_probe_id, circ, - RELAY_COMMAND_BEGIN, payload, - payload_len, cpath_layer) < 0) { - log_notice(LD_CIRC, - "Failed to send pathbias probe cell on circuit %d.", - ocirc->global_identifier); - return -1; - } - - /* Mark it freshly dirty so it doesn't get expired in the meantime */ - circ->timestamp_dirty = time(NULL); - - return 0; -} - -/** - * Check the response to a pathbias probe, to ensure the - * cell is recognized and the nonce and other probe - * characteristics are as expected. - * - * If the response is valid, return 0. Otherwise return < 0. - */ -int -pathbias_check_probe_response(circuit_t *circ, const cell_t *cell) -{ - /* Based on connection_edge_process_relay_cell() */ - relay_header_t rh; - int reason; - uint32_t ipv4_host; - origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ); - - tor_assert(cell); - tor_assert(ocirc); - tor_assert(circ->purpose == CIRCUIT_PURPOSE_PATH_BIAS_TESTING); - - relay_header_unpack(&rh, cell->payload); - - reason = rh.length > 0 ? - get_uint8(cell->payload+RELAY_HEADER_SIZE) : END_STREAM_REASON_MISC; - - if (rh.command == RELAY_COMMAND_END && - reason == END_STREAM_REASON_EXITPOLICY && - ocirc->pathbias_probe_id == rh.stream_id) { - - /* Check length+extract host: It is in network order after the reason code. - * See connection_edge_end(). */ - if (rh.length < 9) { /* reason+ipv4+dns_ttl */ - log_notice(LD_PROTOCOL, - "Short path bias probe response length field (%d).", rh.length); - return - END_CIRC_REASON_TORPROTOCOL; - } - - ipv4_host = ntohl(get_uint32(cell->payload+RELAY_HEADER_SIZE+1)); - - /* Check nonce */ - if (ipv4_host == ocirc->pathbias_probe_nonce) { - pathbias_mark_use_success(ocirc); - circuit_mark_for_close(circ, END_CIRC_REASON_FINISHED); - log_info(LD_CIRC, - "Got valid path bias probe back for circ %d, stream %d.", - ocirc->global_identifier, ocirc->pathbias_probe_id); - return 0; - } else { - log_notice(LD_CIRC, - "Got strange probe value 0x%x vs 0x%x back for circ %d, " - "stream %d.", ipv4_host, ocirc->pathbias_probe_nonce, - ocirc->global_identifier, ocirc->pathbias_probe_id); - return -1; - } - } - log_info(LD_CIRC, - "Got another cell back back on pathbias probe circuit %d: " - "Command: %d, Reason: %d, Stream-id: %d", - ocirc->global_identifier, rh.command, reason, rh.stream_id); - return -1; -} - -/** - * Check if a circuit was used and/or closed successfully. - * - * If we attempted to use the circuit to carry a stream but failed - * for whatever reason, or if the circuit mysteriously died before - * we could attach any streams, record these two cases. - * - * If we *have* successfully used the circuit, or it appears to - * have been closed by us locally, count it as a success. - * - * Returns 0 if we're done making decisions with the circ, - * or -1 if we want to probe it first. - */ -int -pathbias_check_close(origin_circuit_t *ocirc, int reason) -{ - circuit_t *circ = ô->base_; - - if (!pathbias_should_count(ocirc)) { - return 0; - } - - switch (ocirc->path_state) { - /* If the circuit was closed after building, but before use, we need - * to ensure we were the ones who tried to close it (and not a remote - * actor). */ - case PATH_STATE_BUILD_SUCCEEDED: - if (reason & END_CIRC_REASON_FLAG_REMOTE) { - /* Remote circ close reasons on an unused circuit all could be bias */ - log_info(LD_CIRC, - "Circuit %d remote-closed without successful use for reason %d. " - "Circuit purpose %d currently %d,%s. Len %d.", - ocirc->global_identifier, - reason, circ->purpose, ocirc->has_opened, - circuit_state_to_string(circ->state), - ocirc->build_state->desired_path_len); - pathbias_count_collapse(ocirc); - } else if ((reason & ~END_CIRC_REASON_FLAG_REMOTE) - == END_CIRC_REASON_CHANNEL_CLOSED && - circ->n_chan && - circ->n_chan->reason_for_closing - != CHANNEL_CLOSE_REQUESTED) { - /* If we didn't close the channel ourselves, it could be bias */ - /* XXX: Only count bias if the network is live? - * What about clock jumps/suspends? */ - log_info(LD_CIRC, - "Circuit %d's channel closed without successful use for reason " - "%d, channel reason %d. Circuit purpose %d currently %d,%s. Len " - "%d.", ocirc->global_identifier, - reason, circ->n_chan->reason_for_closing, - circ->purpose, ocirc->has_opened, - circuit_state_to_string(circ->state), - ocirc->build_state->desired_path_len); - pathbias_count_collapse(ocirc); - } else { - pathbias_count_successful_close(ocirc); - } - break; - - /* If we tried to use a circuit but failed, we should probe it to ensure - * it has not been tampered with. */ - case PATH_STATE_USE_ATTEMPTED: - /* XXX: Only probe and/or count failure if the network is live? - * What about clock jumps/suspends? */ - if (pathbias_send_usable_probe(circ) == 0) - return -1; - else - pathbias_count_use_failed(ocirc); - - /* Any circuit where there were attempted streams but no successful - * streams could be bias */ - log_info(LD_CIRC, - "Circuit %d closed without successful use for reason %d. " - "Circuit purpose %d currently %d,%s. Len %d.", - ocirc->global_identifier, - reason, circ->purpose, ocirc->has_opened, - circuit_state_to_string(circ->state), - ocirc->build_state->desired_path_len); - break; - - case PATH_STATE_USE_SUCCEEDED: - pathbias_count_successful_close(ocirc); - pathbias_count_use_success(ocirc); - break; - - case PATH_STATE_USE_FAILED: - pathbias_count_use_failed(ocirc); - break; - - case PATH_STATE_NEW_CIRC: - case PATH_STATE_BUILD_ATTEMPTED: - case PATH_STATE_ALREADY_COUNTED: - default: - // Other states are uninteresting. No stats to count. - break; - } - - ocirc->path_state = PATH_STATE_ALREADY_COUNTED; - - return 0; -} - -/** - * Count a successfully closed circuit. - */ -static void -pathbias_count_successful_close(origin_circuit_t *circ) -{ - entry_guard_t *guard = NULL; - if (!pathbias_should_count(circ)) { - return; - } - - if (circ->cpath && circ->cpath->extend_info) { - guard = entry_guard_get_by_id_digest( - circ->cpath->extend_info->identity_digest); - } - - if (guard) { - /* In the long run: circuit_success ~= successful_circuit_close + - * circ_failure + stream_failure */ - guard->successful_circuits_closed++; - entry_guards_changed(); - } else if (circ->base_.purpose != CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT) { - /* In rare cases, CIRCUIT_PURPOSE_TESTING can get converted to - * CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT and have no guards here. - * No need to log that case. */ - log_info(LD_CIRC, - "Successfully closed circuit has no known guard. " - "Circuit is a %s currently %s", - circuit_purpose_to_string(circ->base_.purpose), - circuit_state_to_string(circ->base_.state)); - } -} - -/** - * Count a circuit that fails after it is built, but before it can - * carry any traffic. - * - * This is needed because there are ways to destroy a - * circuit after it has successfully completed. Right now, this is - * used for purely informational/debugging purposes. - */ -static void -pathbias_count_collapse(origin_circuit_t *circ) -{ - entry_guard_t *guard = NULL; - - if (!pathbias_should_count(circ)) { - return; - } - - if (circ->cpath && circ->cpath->extend_info) { - guard = entry_guard_get_by_id_digest( - circ->cpath->extend_info->identity_digest); - } - - if (guard) { - guard->collapsed_circuits++; - entry_guards_changed(); - } else if (circ->base_.purpose != CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT) { - /* In rare cases, CIRCUIT_PURPOSE_TESTING can get converted to - * CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT and have no guards here. - * No need to log that case. */ - log_info(LD_CIRC, - "Destroyed circuit has no known guard. " - "Circuit is a %s currently %s", - circuit_purpose_to_string(circ->base_.purpose), - circuit_state_to_string(circ->base_.state)); - } -} - -/** - * Count a known failed circuit (because we could not probe it). - * - * This counter is informational. - */ -static void -pathbias_count_use_failed(origin_circuit_t *circ) -{ - entry_guard_t *guard = NULL; - if (!pathbias_should_count(circ)) { - return; - } - - if (circ->cpath && circ->cpath->extend_info) { - guard = entry_guard_get_by_id_digest( - circ->cpath->extend_info->identity_digest); - } - - if (guard) { - guard->unusable_circuits++; - entry_guards_changed(); - } else if (circ->base_.purpose != CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT) { - /* In rare cases, CIRCUIT_PURPOSE_TESTING can get converted to - * CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT and have no guards here. - * No need to log that case. */ - /* XXX note cut-and-paste code in this function compared to nearby - * functions. Would be nice to refactor. -RD */ - log_info(LD_CIRC, - "Stream-failing circuit has no known guard. " - "Circuit is a %s currently %s", - circuit_purpose_to_string(circ->base_.purpose), - circuit_state_to_string(circ->base_.state)); - } -} - -/** - * Count timeouts for path bias log messages. - * - * These counts are purely informational. - */ -void -pathbias_count_timeout(origin_circuit_t *circ) -{ - entry_guard_t *guard = NULL; - - if (!pathbias_should_count(circ)) { - return; - } - - /* For hidden service circs, they can actually be used - * successfully and then time out later (because - * the other side declines to use them). */ - if (circ->path_state == PATH_STATE_USE_SUCCEEDED) { - return; - } - - if (circ->cpath && circ->cpath->extend_info) { - guard = entry_guard_get_by_id_digest( - circ->cpath->extend_info->identity_digest); - } - - if (guard) { - guard->timeouts++; - entry_guards_changed(); - } -} - -/** - * Helper function to count all of the currently opened circuits - * for a guard that are in a given path state range. The state - * range is inclusive on both ends. - */ -static int -pathbias_count_circs_in_states(entry_guard_t *guard, - path_state_t from, - path_state_t to) -{ - circuit_t *circ; - int open_circuits = 0; - - /* Count currently open circuits. Give them the benefit of the doubt. */ - for (circ = global_circuitlist; circ; circ = circ->next) { - origin_circuit_t *ocirc = NULL; - if (!CIRCUIT_IS_ORIGIN(circ) || /* didn't originate here */ - circ->marked_for_close) /* already counted */ - continue; - - ocirc = TO_ORIGIN_CIRCUIT(circ); - - if (!ocirc->cpath || !ocirc->cpath->extend_info) - continue; - - if (ocirc->path_state >= from && - ocirc->path_state <= to && - pathbias_should_count(ocirc) && - fast_memeq(guard->identity, - ocirc->cpath->extend_info->identity_digest, - DIGEST_LEN)) { - log_debug(LD_CIRC, "Found opened circuit %d in path_state %s", - ocirc->global_identifier, - pathbias_state_to_string(ocirc->path_state)); - open_circuits++; - } - } - - return open_circuits; -} - -/** - * Return the number of circuits counted as successfully closed for - * this guard. - * - * Also add in the currently open circuits to give them the benefit - * of the doubt. - */ -double -pathbias_get_close_success_count(entry_guard_t *guard) -{ - return guard->successful_circuits_closed + - pathbias_count_circs_in_states(guard, - PATH_STATE_BUILD_SUCCEEDED, - PATH_STATE_USE_SUCCEEDED); -} - -/** - * Return the number of circuits counted as successfully used - * this guard. - * - * Also add in the currently open circuits that we are attempting - * to use to give them the benefit of the doubt. - */ -double -pathbias_get_use_success_count(entry_guard_t *guard) -{ - return guard->use_successes + - pathbias_count_circs_in_states(guard, - PATH_STATE_USE_ATTEMPTED, - PATH_STATE_USE_SUCCEEDED); -} - -/** - * Check the path bias use rate against our consensus parameter limits. - * - * Emits a log message if the use success rates are too low. - * - * If pathbias_get_dropguards() is set, we also disable the use of - * very failure prone guards. - */ -static void -pathbias_measure_use_rate(entry_guard_t *guard) -{ - const or_options_t *options = get_options(); - - if (guard->use_attempts > pathbias_get_min_use(options)) { - /* Note: We rely on the < comparison here to allow us to set a 0 - * rate and disable the feature entirely. If refactoring, don't - * change to <= */ - if (pathbias_get_use_success_count(guard)/guard->use_attempts - < pathbias_get_extreme_use_rate(options)) { - /* Dropping is currently disabled by default. */ - if (pathbias_get_dropguards(options)) { - if (!guard->path_bias_disabled) { - log_warn(LD_CIRC, - "Your Guard %s ($%s) is failing to carry an extremely large " - "amount of stream on its circuits. " - "To avoid potential route manipulation attacks, Tor has " - "disabled use of this guard. " - "Use counts are %ld/%ld. Success counts are %ld/%ld. " - "%ld circuits completed, %ld were unusable, %ld collapsed, " - "and %ld timed out. " - "For reference, your timeout cutoff is %ld seconds.", - guard->nickname, hex_str(guard->identity, DIGEST_LEN), - tor_lround(pathbias_get_use_success_count(guard)), - tor_lround(guard->use_attempts), - tor_lround(pathbias_get_close_success_count(guard)), - tor_lround(guard->circ_attempts), - tor_lround(guard->circ_successes), - tor_lround(guard->unusable_circuits), - tor_lround(guard->collapsed_circuits), - tor_lround(guard->timeouts), - tor_lround(circ_times.close_ms/1000)); - guard->path_bias_disabled = 1; - guard->bad_since = approx_time(); - entry_guards_changed(); - return; - } - } else if (!guard->path_bias_use_extreme) { - guard->path_bias_use_extreme = 1; - log_warn(LD_CIRC, - "Your Guard %s ($%s) is failing to carry an extremely large " - "amount of streams on its circuits. " - "This could indicate a route manipulation attack, network " - "overload, bad local network connectivity, or a bug. " - "Use counts are %ld/%ld. Success counts are %ld/%ld. " - "%ld circuits completed, %ld were unusable, %ld collapsed, " - "and %ld timed out. " - "For reference, your timeout cutoff is %ld seconds.", - guard->nickname, hex_str(guard->identity, DIGEST_LEN), - tor_lround(pathbias_get_use_success_count(guard)), - tor_lround(guard->use_attempts), - tor_lround(pathbias_get_close_success_count(guard)), - tor_lround(guard->circ_attempts), - tor_lround(guard->circ_successes), - tor_lround(guard->unusable_circuits), - tor_lround(guard->collapsed_circuits), - tor_lround(guard->timeouts), - tor_lround(circ_times.close_ms/1000)); - } - } else if (pathbias_get_use_success_count(guard)/guard->use_attempts - < pathbias_get_notice_use_rate(options)) { - if (!guard->path_bias_use_noticed) { - guard->path_bias_use_noticed = 1; - log_notice(LD_CIRC, - "Your Guard %s ($%s) is failing to carry more streams on its " - "circuits than usual. " - "Most likely this means the Tor network is overloaded " - "or your network connection is poor. " - "Use counts are %ld/%ld. Success counts are %ld/%ld. " - "%ld circuits completed, %ld were unusable, %ld collapsed, " - "and %ld timed out. " - "For reference, your timeout cutoff is %ld seconds.", - guard->nickname, hex_str(guard->identity, DIGEST_LEN), - tor_lround(pathbias_get_use_success_count(guard)), - tor_lround(guard->use_attempts), - tor_lround(pathbias_get_close_success_count(guard)), - tor_lround(guard->circ_attempts), - tor_lround(guard->circ_successes), - tor_lround(guard->unusable_circuits), - tor_lround(guard->collapsed_circuits), - tor_lround(guard->timeouts), - tor_lround(circ_times.close_ms/1000)); - } - } - } -} - -/** - * Check the path bias circuit close status rates against our consensus - * parameter limits. - * - * Emits a log message if the use success rates are too low. - * - * If pathbias_get_dropguards() is set, we also disable the use of - * very failure prone guards. - * - * XXX: This function shares similar log messages and checks to - * pathbias_measure_use_rate(). It may be possible to combine them - * eventually, especially if we can ever remove the need for 3 - * levels of closure warns (if the overall circuit failure rate - * goes down with ntor). One way to do so would be to multiply - * the build rate with the use rate to get an idea of the total - * fraction of the total network paths the user is able to use. - * See ticket #8159. - */ -static void -pathbias_measure_close_rate(entry_guard_t *guard) -{ - const or_options_t *options = get_options(); - - if (guard->circ_attempts > pathbias_get_min_circs(options)) { - /* Note: We rely on the < comparison here to allow us to set a 0 - * rate and disable the feature entirely. If refactoring, don't - * change to <= */ - if (pathbias_get_close_success_count(guard)/guard->circ_attempts - < pathbias_get_extreme_rate(options)) { - /* Dropping is currently disabled by default. */ - if (pathbias_get_dropguards(options)) { - if (!guard->path_bias_disabled) { - log_warn(LD_CIRC, - "Your Guard %s ($%s) is failing an extremely large " - "amount of circuits. " - "To avoid potential route manipulation attacks, Tor has " - "disabled use of this guard. " - "Success counts are %ld/%ld. Use counts are %ld/%ld. " - "%ld circuits completed, %ld were unusable, %ld collapsed, " - "and %ld timed out. " - "For reference, your timeout cutoff is %ld seconds.", - guard->nickname, hex_str(guard->identity, DIGEST_LEN), - tor_lround(pathbias_get_close_success_count(guard)), - tor_lround(guard->circ_attempts), - tor_lround(pathbias_get_use_success_count(guard)), - tor_lround(guard->use_attempts), - tor_lround(guard->circ_successes), - tor_lround(guard->unusable_circuits), - tor_lround(guard->collapsed_circuits), - tor_lround(guard->timeouts), - tor_lround(circ_times.close_ms/1000)); - guard->path_bias_disabled = 1; - guard->bad_since = approx_time(); - entry_guards_changed(); - return; - } - } else if (!guard->path_bias_extreme) { - guard->path_bias_extreme = 1; - log_warn(LD_CIRC, - "Your Guard %s ($%s) is failing an extremely large " - "amount of circuits. " - "This could indicate a route manipulation attack, " - "extreme network overload, or a bug. " - "Success counts are %ld/%ld. Use counts are %ld/%ld. " - "%ld circuits completed, %ld were unusable, %ld collapsed, " - "and %ld timed out. " - "For reference, your timeout cutoff is %ld seconds.", - guard->nickname, hex_str(guard->identity, DIGEST_LEN), - tor_lround(pathbias_get_close_success_count(guard)), - tor_lround(guard->circ_attempts), - tor_lround(pathbias_get_use_success_count(guard)), - tor_lround(guard->use_attempts), - tor_lround(guard->circ_successes), - tor_lround(guard->unusable_circuits), - tor_lround(guard->collapsed_circuits), - tor_lround(guard->timeouts), - tor_lround(circ_times.close_ms/1000)); - } - } else if (pathbias_get_close_success_count(guard)/guard->circ_attempts - < pathbias_get_warn_rate(options)) { - if (!guard->path_bias_warned) { - guard->path_bias_warned = 1; - log_warn(LD_CIRC, - "Your Guard %s ($%s) is failing a very large " - "amount of circuits. " - "Most likely this means the Tor network is " - "overloaded, but it could also mean an attack against " - "you or potentially the guard itself. " - "Success counts are %ld/%ld. Use counts are %ld/%ld. " - "%ld circuits completed, %ld were unusable, %ld collapsed, " - "and %ld timed out. " - "For reference, your timeout cutoff is %ld seconds.", - guard->nickname, hex_str(guard->identity, DIGEST_LEN), - tor_lround(pathbias_get_close_success_count(guard)), - tor_lround(guard->circ_attempts), - tor_lround(pathbias_get_use_success_count(guard)), - tor_lround(guard->use_attempts), - tor_lround(guard->circ_successes), - tor_lround(guard->unusable_circuits), - tor_lround(guard->collapsed_circuits), - tor_lround(guard->timeouts), - tor_lround(circ_times.close_ms/1000)); - } - } else if (pathbias_get_close_success_count(guard)/guard->circ_attempts - < pathbias_get_notice_rate(options)) { - if (!guard->path_bias_noticed) { - guard->path_bias_noticed = 1; - log_notice(LD_CIRC, - "Your Guard %s ($%s) is failing more circuits than " - "usual. " - "Most likely this means the Tor network is overloaded. " - "Success counts are %ld/%ld. Use counts are %ld/%ld. " - "%ld circuits completed, %ld were unusable, %ld collapsed, " - "and %ld timed out. " - "For reference, your timeout cutoff is %ld seconds.", - guard->nickname, hex_str(guard->identity, DIGEST_LEN), - tor_lround(pathbias_get_close_success_count(guard)), - tor_lround(guard->circ_attempts), - tor_lround(pathbias_get_use_success_count(guard)), - tor_lround(guard->use_attempts), - tor_lround(guard->circ_successes), - tor_lround(guard->unusable_circuits), - tor_lround(guard->collapsed_circuits), - tor_lround(guard->timeouts), - tor_lround(circ_times.close_ms/1000)); - } - } - } -} - -/** - * This function scales the path bias use rates if we have - * more data than the scaling threshold. This allows us to - * be more sensitive to recent measurements. - * - * XXX: The attempt count transfer stuff here might be done - * better by keeping separate pending counters that get - * transfered at circuit close. See ticket #8160. - */ -static void -pathbias_scale_close_rates(entry_guard_t *guard) -{ - const or_options_t *options = get_options(); - - /* If we get a ton of circuits, just scale everything down */ - if (guard->circ_attempts > pathbias_get_scale_threshold(options)) { - double scale_ratio = pathbias_get_scale_ratio(options); - int opened_attempts = pathbias_count_circs_in_states(guard, - PATH_STATE_BUILD_ATTEMPTED, PATH_STATE_BUILD_ATTEMPTED); - int opened_built = pathbias_count_circs_in_states(guard, - PATH_STATE_BUILD_SUCCEEDED, - PATH_STATE_USE_FAILED); - /* Verify that the counts are sane before and after scaling */ - int counts_are_sane = (guard->circ_attempts >= guard->circ_successes); - - guard->circ_attempts -= (opened_attempts+opened_built); - guard->circ_successes -= opened_built; - - guard->circ_attempts *= scale_ratio; - guard->circ_successes *= scale_ratio; - guard->timeouts *= scale_ratio; - guard->successful_circuits_closed *= scale_ratio; - guard->collapsed_circuits *= scale_ratio; - guard->unusable_circuits *= scale_ratio; - - guard->circ_attempts += (opened_attempts+opened_built); - guard->circ_successes += opened_built; - - entry_guards_changed(); - - log_info(LD_CIRC, - "Scaled pathbias counts to (%f,%f)/%f (%d/%d open) for guard " - "%s ($%s)", - guard->circ_successes, guard->successful_circuits_closed, - guard->circ_attempts, opened_built, opened_attempts, - guard->nickname, hex_str(guard->identity, DIGEST_LEN)); - - /* Have the counts just become invalid by this scaling attempt? */ - if (counts_are_sane && guard->circ_attempts < guard->circ_successes) { - log_notice(LD_BUG, - "Scaling has mangled pathbias counts to %f/%f (%d/%d open) " - "for guard %s ($%s)", - guard->circ_successes, guard->circ_attempts, opened_built, - opened_attempts, guard->nickname, - hex_str(guard->identity, DIGEST_LEN)); - } - } -} - -/** - * This function scales the path bias circuit close rates if we have - * more data than the scaling threshold. This allows us to be more - * sensitive to recent measurements. - * - * XXX: The attempt count transfer stuff here might be done - * better by keeping separate pending counters that get - * transfered at circuit close. See ticket #8160. - */ -void -pathbias_scale_use_rates(entry_guard_t *guard) -{ - const or_options_t *options = get_options(); - - /* If we get a ton of circuits, just scale everything down */ - if (guard->use_attempts > pathbias_get_scale_use_threshold(options)) { - double scale_ratio = pathbias_get_scale_ratio(options); - int opened_attempts = pathbias_count_circs_in_states(guard, - PATH_STATE_USE_ATTEMPTED, PATH_STATE_USE_SUCCEEDED); - /* Verify that the counts are sane before and after scaling */ - int counts_are_sane = (guard->use_attempts >= guard->use_successes); - - guard->use_attempts -= opened_attempts; - - guard->use_attempts *= scale_ratio; - guard->use_successes *= scale_ratio; - - guard->use_attempts += opened_attempts; - - log_info(LD_CIRC, - "Scaled pathbias use counts to %f/%f (%d open) for guard %s ($%s)", - guard->use_successes, guard->use_attempts, opened_attempts, - guard->nickname, hex_str(guard->identity, DIGEST_LEN)); - - /* Have the counts just become invalid by this scaling attempt? */ - if (counts_are_sane && guard->use_attempts < guard->use_successes) { - log_notice(LD_BUG, - "Scaling has mangled pathbias usage counts to %f/%f " - "(%d open) for guard %s ($%s)", - guard->circ_successes, guard->circ_attempts, - opened_attempts, guard->nickname, - hex_str(guard->identity, DIGEST_LEN)); - } - - entry_guards_changed(); - } -} - -/** Increment the number of times we successfully extended a circuit to - * <b>guard</b>, first checking if the failure rate is high enough that - * we should eliminate the guard. Return -1 if the guard looks no good; - * return 0 if the guard looks fine. - */ -static int -entry_guard_inc_circ_attempt_count(entry_guard_t *guard) -{ - entry_guards_changed(); - - pathbias_measure_close_rate(guard); - - if (guard->path_bias_disabled) - return -1; - - pathbias_scale_close_rates(guard); - guard->circ_attempts++; - - log_info(LD_CIRC, "Got success count %f/%f for guard %s ($%s)", - guard->circ_successes, guard->circ_attempts, guard->nickname, - hex_str(guard->identity, DIGEST_LEN)); - return 0; -} - /** A "created" cell <b>reply</b> came back to us on circuit <b>circ</b>. * (The body of <b>reply</b> varies depending on what sort of handshake * this is.) @@ -2834,11 +1407,7 @@ onionskin_answer(or_circuit_t *circ, * number of endpoints that would give something away about our destination. * * If the routerlist <b>nodes</b> doesn't have enough routers - * to handle the desired path length, return as large a path length as - * is feasible, except if it's less than 2, in which case return -1. - * XXX ^^ I think this behavior is a hold-over from back when we had only a - * few relays in the network, and certainly back before guards existed. - * We should very likely get rid of it. -RD + * to handle the desired path length, return -1. */ static int new_route_len(uint8_t purpose, extend_info_t *exit, smartlist_t *nodes) @@ -2859,19 +1428,13 @@ new_route_len(uint8_t purpose, extend_info_t *exit, smartlist_t *nodes) log_debug(LD_CIRC,"Chosen route length %d (%d/%d routers suitable).", routelen, num_acceptable_routers, smartlist_len(nodes)); - if (num_acceptable_routers < 2) { + if (num_acceptable_routers < routelen) { log_info(LD_CIRC, - "Not enough acceptable routers (%d). Discarding this circuit.", - num_acceptable_routers); + "Not enough acceptable routers (%d/%d). Discarding this circuit.", + num_acceptable_routers, routelen); return -1; } - if (num_acceptable_routers < routelen) { - log_info(LD_CIRC,"Not enough routers: cutting routelen from %d to %d.", - routelen, num_acceptable_routers); - routelen = num_acceptable_routers; - } - return routelen; } diff --git a/src/or/circuitbuild.h b/src/or/circuitbuild.h index a3091707e8..71caea94ed 100644 --- a/src/or/circuitbuild.h +++ b/src/or/circuitbuild.h @@ -57,16 +57,10 @@ const char *build_state_get_exit_nickname(cpath_build_state_t *state); const node_t *choose_good_entry_server(uint8_t purpose, cpath_build_state_t *state); -double pathbias_get_extreme_rate(const or_options_t *options); -double pathbias_get_extreme_use_rate(const or_options_t *options); -int pathbias_get_dropguards(const or_options_t *options); -void pathbias_count_timeout(origin_circuit_t *circ); -int pathbias_check_close(origin_circuit_t *circ, int reason); -int pathbias_check_probe_response(circuit_t *circ, const cell_t *cell); -void pathbias_count_use_attempt(origin_circuit_t *circ); -void pathbias_mark_use_success(origin_circuit_t *circ); -void pathbias_mark_use_rollback(origin_circuit_t *circ); -const char *pathbias_state_to_string(path_state_t state); + +#ifdef CIRCUITBUILD_PRIVATE +STATIC circid_t get_unique_circ_id_by_chan(channel_t *chan); +#endif #endif diff --git a/src/or/circuitlist.c b/src/or/circuitlist.c index c7b15e40ba..f3a83503ef 100644 --- a/src/or/circuitlist.c +++ b/src/or/circuitlist.c @@ -8,9 +8,10 @@ * \file circuitlist.c * \brief Manage the global circuit list. **/ - +#define CIRCUITLIST_PRIVATE #include "or.h" #include "channel.h" +#include "circpathbias.h" #include "circuitbuild.h" #include "circuitlist.h" #include "circuituse.h" @@ -31,20 +32,23 @@ #include "rephist.h" #include "routerlist.h" #include "routerset.h" + #include "ht.h" /********* START VARIABLES **********/ /** A global list of all circuits at this hop. */ -circuit_t *global_circuitlist=NULL; +struct global_circuitlist_s global_circuitlist = + TOR_LIST_HEAD_INITIALIZER(global_circuitlist); /** A list of all the circuits in CIRCUIT_STATE_CHAN_WAIT. */ static smartlist_t *circuits_pending_chans = NULL; -static void circuit_free(circuit_t *circ); -static void circuit_free_cpath(crypt_path_t *cpath); static void circuit_free_cpath_node(crypt_path_t *victim); static void cpath_ref_decref(crypt_path_reference_t *cpath_ref); +//static void circuit_set_rend_token(or_circuit_t *circ, int is_rend_circ, +// const uint8_t *token); +static void circuit_clear_rend_token(or_circuit_t *circ); /********* END VARIABLES ************/ @@ -55,6 +59,8 @@ typedef struct chan_circid_circuit_map_t { channel_t *chan; circid_t circ_id; circuit_t *circuit; + /* For debugging 12184: when was this placeholder item added? */ + time_t made_placeholder_at; } chan_circid_circuit_map_t; /** Helper for hash tables: compare the channel and circuit ID for a and @@ -72,7 +78,15 @@ chan_circid_entries_eq_(chan_circid_circuit_map_t *a, static INLINE unsigned int chan_circid_entry_hash_(chan_circid_circuit_map_t *a) { - return ((unsigned)a->circ_id) ^ (unsigned)(uintptr_t)(a->chan); + /* Try to squeze the siphash input into 8 bytes to save any extra siphash + * rounds. This hash function is in the critical path. */ + uintptr_t chan = (uintptr_t) (void*) a->chan; + uint32_t array[2]; + array[0] = a->circ_id; + /* The low bits of the channel pointer are uninteresting, since the channel + * is a pretty big structure. */ + array[1] = (uint32_t) (chan >> 6); + return (unsigned) siphash24g(array, sizeof(array)); } /** Map from [chan,circid] to circuit. */ @@ -172,6 +186,7 @@ circuit_set_circid_chan_helper(circuit_t *circ, int direction, found = HT_FIND(chan_circid_map, &chan_circid_map, &search); if (found) { found->circuit = circ; + found->made_placeholder_at = 0; } else { found = tor_malloc_zero(sizeof(chan_circid_circuit_map_t)); found->circ_id = id; @@ -207,18 +222,129 @@ circuit_set_circid_chan_helper(circuit_t *circ, int direction, } } +/** Mark that circuit id <b>id</b> shouldn't be used on channel <b>chan</b>, + * even if there is no circuit on the channel. We use this to keep the + * circuit id from getting re-used while we have queued but not yet sent + * a destroy cell. */ +void +channel_mark_circid_unusable(channel_t *chan, circid_t id) +{ + chan_circid_circuit_map_t search; + chan_circid_circuit_map_t *ent; + + /* See if there's an entry there. That wouldn't be good. */ + memset(&search, 0, sizeof(search)); + search.chan = chan; + search.circ_id = id; + ent = HT_FIND(chan_circid_map, &chan_circid_map, &search); + + if (ent && ent->circuit) { + /* we have a problem. */ + log_warn(LD_BUG, "Tried to mark %u unusable on %p, but there was already " + "a circuit there.", (unsigned)id, chan); + } else if (ent) { + /* It's already marked. */ + if (!ent->made_placeholder_at) + ent->made_placeholder_at = approx_time(); + } else { + ent = tor_malloc_zero(sizeof(chan_circid_circuit_map_t)); + ent->chan = chan; + ent->circ_id = id; + /* leave circuit at NULL. */ + ent->made_placeholder_at = approx_time(); + HT_INSERT(chan_circid_map, &chan_circid_map, ent); + } +} + +/** Mark that a circuit id <b>id</b> can be used again on <b>chan</b>. + * We use this to re-enable the circuit ID after we've sent a destroy cell. + */ +void +channel_mark_circid_usable(channel_t *chan, circid_t id) +{ + chan_circid_circuit_map_t search; + chan_circid_circuit_map_t *ent; + + /* See if there's an entry there. That wouldn't be good. */ + memset(&search, 0, sizeof(search)); + search.chan = chan; + search.circ_id = id; + ent = HT_REMOVE(chan_circid_map, &chan_circid_map, &search); + if (ent && ent->circuit) { + log_warn(LD_BUG, "Tried to mark %u usable on %p, but there was already " + "a circuit there.", (unsigned)id, chan); + return; + } + if (_last_circid_chan_ent == ent) + _last_circid_chan_ent = NULL; + tor_free(ent); +} + +/** Called to indicate that a DESTROY is pending on <b>chan</b> with + * circuit ID <b>id</b>, but hasn't been sent yet. */ +void +channel_note_destroy_pending(channel_t *chan, circid_t id) +{ + circuit_t *circ = circuit_get_by_circid_channel_even_if_marked(id,chan); + if (circ) { + if (circ->n_chan == chan && circ->n_circ_id == id) { + circ->n_delete_pending = 1; + } else { + or_circuit_t *orcirc = TO_OR_CIRCUIT(circ); + if (orcirc->p_chan == chan && orcirc->p_circ_id == id) { + circ->p_delete_pending = 1; + } + } + return; + } + channel_mark_circid_unusable(chan, id); +} + +/** Called to indicate that a DESTROY is no longer pending on <b>chan</b> with + * circuit ID <b>id</b> -- typically, because it has been sent. */ +void +channel_note_destroy_not_pending(channel_t *chan, circid_t id) +{ + circuit_t *circ = circuit_get_by_circid_channel_even_if_marked(id,chan); + if (circ) { + if (circ->n_chan == chan && circ->n_circ_id == id) { + circ->n_delete_pending = 0; + } else { + or_circuit_t *orcirc = TO_OR_CIRCUIT(circ); + if (orcirc->p_chan == chan && orcirc->p_circ_id == id) { + circ->p_delete_pending = 0; + } + } + /* XXXX this shouldn't happen; log a bug here. */ + return; + } + channel_mark_circid_usable(chan, id); +} + /** Set the p_conn field of a circuit <b>circ</b>, along * with the corresponding circuit ID, and add the circuit as appropriate * to the (chan,id)-\>circuit map. */ void -circuit_set_p_circid_chan(or_circuit_t *circ, circid_t id, +circuit_set_p_circid_chan(or_circuit_t *or_circ, circid_t id, channel_t *chan) { - circuit_set_circid_chan_helper(TO_CIRCUIT(circ), CELL_DIRECTION_IN, - id, chan); + circuit_t *circ = TO_CIRCUIT(or_circ); + channel_t *old_chan = or_circ->p_chan; + circid_t old_id = or_circ->p_circ_id; + + circuit_set_circid_chan_helper(circ, CELL_DIRECTION_IN, id, chan); - if (chan) - tor_assert(bool_eq(circ->p_chan_cells.n, circ->next_active_on_p_chan)); + if (chan) { + tor_assert(bool_eq(or_circ->p_chan_cells.n, + or_circ->next_active_on_p_chan)); + + chan->timestamp_last_had_circuits = approx_time(); + } + + if (circ->p_delete_pending && old_chan) { + channel_mark_circid_unusable(old_chan, old_id); + circ->p_delete_pending = 0; + } } /** Set the n_conn field of a circuit <b>circ</b>, along @@ -228,10 +354,21 @@ void circuit_set_n_circid_chan(circuit_t *circ, circid_t id, channel_t *chan) { + channel_t *old_chan = circ->n_chan; + circid_t old_id = circ->n_circ_id; + circuit_set_circid_chan_helper(circ, CELL_DIRECTION_OUT, id, chan); - if (chan) + if (chan) { tor_assert(bool_eq(circ->n_chan_cells.n, circ->next_active_on_n_chan)); + + chan->timestamp_last_had_circuits = approx_time(); + } + + if (circ->n_delete_pending && old_chan) { + channel_mark_circid_unusable(old_chan, old_id); + circ->n_delete_pending = 0; + } } /** Change the state of <b>circ</b> to <b>state</b>, adding it to or removing @@ -257,21 +394,6 @@ circuit_set_state(circuit_t *circ, uint8_t state) circ->state = state; } -/** Add <b>circ</b> to the global list of circuits. This is called only from - * within circuit_new. - */ -static void -circuit_add(circuit_t *circ) -{ - if (!global_circuitlist) { /* first one */ - global_circuitlist = circ; - circ->next = NULL; - } else { - circ->next = global_circuitlist; - global_circuitlist = circ; - } -} - /** Append to <b>out</b> all circuits in state CHAN_WAIT waiting for * the given connection. */ void @@ -329,33 +451,17 @@ circuit_count_pending_on_channel(channel_t *chan) void circuit_close_all_marked(void) { - circuit_t *tmp,*m; - - while (global_circuitlist && global_circuitlist->marked_for_close) { - tmp = global_circuitlist->next; - circuit_free(global_circuitlist); - global_circuitlist = tmp; - } - - tmp = global_circuitlist; - while (tmp && tmp->next) { - if (tmp->next->marked_for_close) { - m = tmp->next->next; - circuit_free(tmp->next); - tmp->next = m; - /* Need to check new tmp->next; don't advance tmp. */ - } else { - /* Advance tmp. */ - tmp = tmp->next; - } - } + circuit_t *circ, *tmp; + TOR_LIST_FOREACH_SAFE(circ, &global_circuitlist, head, tmp) + if (circ->marked_for_close) + circuit_free(circ); } /** Return the head of the global linked list of circuits. */ -circuit_t * -circuit_get_global_list_(void) +MOCK_IMPL(struct global_circuitlist_s *, +circuit_get_global_list,(void)) { - return global_circuitlist; + return &global_circuitlist; } /** Function to make circ-\>state human-readable */ @@ -570,8 +676,9 @@ init_circuit_base(circuit_t *circ) circ->package_window = circuit_initial_package_window(); circ->deliver_window = CIRCWINDOW_START; + cell_queue_init(&circ->n_chan_cells); - circuit_add(circ); + TOR_LIST_INSERT_HEAD(&global_circuitlist, circ, head); } /** Allocate space for a new circuit, initializing with <b>p_circ_id</b> @@ -595,7 +702,7 @@ origin_circuit_new(void) init_circuit_base(TO_CIRCUIT(circ)); - circ_times.last_circ_at = approx_time(); + circuit_build_times_update_last_circ(get_circuit_build_times_mutable()); return circ; } @@ -615,6 +722,7 @@ or_circuit_new(circid_t p_circ_id, channel_t *p_chan) circuit_set_p_circid_chan(circ, p_circ_id, p_chan); circ->remaining_relay_early_cells = MAX_RELAY_EARLY_CELLS_PER_CIRCUIT; + cell_queue_init(&circ->p_chan_cells); init_circuit_base(TO_CIRCUIT(circ)); @@ -623,7 +731,7 @@ or_circuit_new(circid_t p_circ_id, channel_t *p_chan) /** Deallocate space associated with circ. */ -static void +STATIC void circuit_free(circuit_t *circ) { void *mem; @@ -643,7 +751,7 @@ circuit_free(circuit_t *circ) } tor_free(ocirc->build_state); - circuit_free_cpath(ocirc->cpath); + circuit_clear_cpath(ocirc); crypto_pk_free(ocirc->intro_key); rend_data_free(ocirc->rend_data); @@ -672,6 +780,8 @@ circuit_free(circuit_t *circ) crypto_cipher_free(ocirc->n_crypto); crypto_digest_free(ocirc->n_digest); + circuit_clear_rend_token(ocirc); + if (ocirc->rend_splice) { or_circuit_t *other = ocirc->rend_splice; tor_assert(other->base_.magic == OR_CIRCUIT_MAGIC); @@ -689,6 +799,8 @@ circuit_free(circuit_t *circ) extend_info_free(circ->n_hop); tor_free(circ->n_chan_create_cell); + TOR_LIST_REMOVE(circ, head); + /* Remove from map. */ circuit_set_n_circid_chan(circ, 0, NULL); @@ -700,11 +812,14 @@ circuit_free(circuit_t *circ) tor_free(mem); } -/** Deallocate space associated with the linked list <b>cpath</b>. */ -static void -circuit_free_cpath(crypt_path_t *cpath) +/** Deallocate the linked list circ-><b>cpath</b>, and remove the cpath from + * <b>circ</b>. */ +void +circuit_clear_cpath(origin_circuit_t *circ) { - crypt_path_t *victim, *head=cpath; + crypt_path_t *victim, *head, *cpath; + + head = cpath = circ->cpath; if (!cpath) return; @@ -718,13 +833,7 @@ circuit_free_cpath(crypt_path_t *cpath) } circuit_free_cpath_node(cpath); -} -/** Remove all the items in the cpath on <b>circ</b>.*/ -void -circuit_clear_cpath(origin_circuit_t *circ) -{ - circuit_free_cpath(circ->cpath); circ->cpath = NULL; } @@ -732,11 +841,11 @@ circuit_clear_cpath(origin_circuit_t *circ) void circuit_free_all(void) { - circuit_t *next; - while (global_circuitlist) { - next = global_circuitlist->next; - if (! CIRCUIT_IS_ORIGIN(global_circuitlist)) { - or_circuit_t *or_circ = TO_OR_CIRCUIT(global_circuitlist); + circuit_t *tmp, *tmp2; + + TOR_LIST_FOREACH_SAFE(tmp, &global_circuitlist, head, tmp2) { + if (! CIRCUIT_IS_ORIGIN(tmp)) { + or_circuit_t *or_circ = TO_OR_CIRCUIT(tmp); while (or_circ->resolving_streams) { edge_connection_t *next_conn; next_conn = or_circ->resolving_streams->next_stream; @@ -744,13 +853,24 @@ circuit_free_all(void) or_circ->resolving_streams = next_conn; } } - circuit_free(global_circuitlist); - global_circuitlist = next; + circuit_free(tmp); } smartlist_free(circuits_pending_chans); circuits_pending_chans = NULL; + { + chan_circid_circuit_map_t **elt, **next, *c; + for (elt = HT_START(chan_circid_map, &chan_circid_map); + elt; + elt = next) { + c = *elt; + next = HT_NEXT_RMV(chan_circid_map, &chan_circid_map, elt); + + tor_assert(c->circuit == NULL); + tor_free(c); + } + } HT_CLEAR(chan_circid_map, &chan_circid_map); } @@ -815,7 +935,7 @@ circuit_dump_by_conn(connection_t *conn, int severity) circuit_t *circ; edge_connection_t *tmpconn; - for (circ = global_circuitlist; circ; circ = circ->next) { + TOR_LIST_FOREACH(circ, &global_circuitlist, head) { circid_t n_circ_id = circ->n_circ_id, p_circ_id = 0; if (circ->marked_for_close) { @@ -848,79 +968,13 @@ circuit_dump_by_conn(connection_t *conn, int severity) } } -/** A helper function for circuit_dump_by_chan() below. Log a bunch - * of information about circuit <b>circ</b>. - */ -static void -circuit_dump_chan_details(int severity, - circuit_t *circ, - channel_t *chan, - const char *type, - circid_t this_circid, - circid_t other_circid) -{ - tor_log(severity, LD_CIRC, "Conn %p has %s circuit: circID %u " - "(other side %u), state %d (%s), born %ld:", - chan, type, (unsigned)this_circid, (unsigned)other_circid, circ->state, - circuit_state_to_string(circ->state), - (long)circ->timestamp_began.tv_sec); - if (CIRCUIT_IS_ORIGIN(circ)) { /* circ starts at this node */ - circuit_log_path(severity, LD_CIRC, TO_ORIGIN_CIRCUIT(circ)); - } -} - -/** Log, at severity <b>severity</b>, information about each circuit - * that is connected to <b>chan</b>. - */ -void -circuit_dump_by_chan(channel_t *chan, int severity) -{ - circuit_t *circ; - - tor_assert(chan); - - for (circ = global_circuitlist; circ; circ = circ->next) { - circid_t n_circ_id = circ->n_circ_id, p_circ_id = 0; - - if (circ->marked_for_close) { - continue; - } - - if (!CIRCUIT_IS_ORIGIN(circ)) { - p_circ_id = TO_OR_CIRCUIT(circ)->p_circ_id; - } - - if (! CIRCUIT_IS_ORIGIN(circ) && TO_OR_CIRCUIT(circ)->p_chan && - TO_OR_CIRCUIT(circ)->p_chan == chan) { - circuit_dump_chan_details(severity, circ, chan, "App-ward", - p_circ_id, n_circ_id); - } - - if (circ->n_chan && circ->n_chan == chan) { - circuit_dump_chan_details(severity, circ, chan, "Exit-ward", - n_circ_id, p_circ_id); - } - - if (!circ->n_chan && circ->n_hop && - channel_matches_extend_info(chan, circ->n_hop) && - tor_memeq(chan->identity_digest, - circ->n_hop->identity_digest, DIGEST_LEN)) { - circuit_dump_chan_details(severity, circ, chan, - (circ->state == CIRCUIT_STATE_OPEN && - !CIRCUIT_IS_ORIGIN(circ)) ? - "Endpoint" : "Pending", - n_circ_id, p_circ_id); - } - } -} - /** Return the circuit whose global ID is <b>id</b>, or NULL if no * such circuit exists. */ origin_circuit_t * circuit_get_by_global_id(uint32_t id) { circuit_t *circ; - for (circ=global_circuitlist;circ;circ = circ->next) { + TOR_LIST_FOREACH(circ, &global_circuitlist, head) { if (CIRCUIT_IS_ORIGIN(circ) && TO_ORIGIN_CIRCUIT(circ)->global_identifier == id) { if (circ->marked_for_close) @@ -936,9 +990,13 @@ circuit_get_by_global_id(uint32_t id) * - circ-\>n_circ_id or circ-\>p_circ_id is equal to <b>circ_id</b>, and * - circ is attached to <b>chan</b>, either as p_chan or n_chan. * Return NULL if no such circuit exists. + * + * If <b>found_entry_out</b> is provided, set it to true if we have a + * placeholder entry for circid/chan, and leave it unset otherwise. */ static INLINE circuit_t * -circuit_get_by_circid_channel_impl(circid_t circ_id, channel_t *chan) +circuit_get_by_circid_channel_impl(circid_t circ_id, channel_t *chan, + int *found_entry_out) { chan_circid_circuit_map_t search; chan_circid_circuit_map_t *found; @@ -959,21 +1017,27 @@ circuit_get_by_circid_channel_impl(circid_t circ_id, channel_t *chan) " circ_id %u, channel ID " U64_FORMAT " (%p)", found->circuit, (unsigned)circ_id, U64_PRINTF_ARG(chan->global_identifier), chan); + if (found_entry_out) + *found_entry_out = 1; return found->circuit; } log_debug(LD_CIRC, - "circuit_get_by_circid_channel_impl() found nothing for" + "circuit_get_by_circid_channel_impl() found %s for" " circ_id %u, channel ID " U64_FORMAT " (%p)", + found ? "placeholder" : "nothing", (unsigned)circ_id, U64_PRINTF_ARG(chan->global_identifier), chan); + if (found_entry_out) + *found_entry_out = found ? 1 : 0; + return NULL; /* The rest of this checks for bugs. Disabled by default. */ /* We comment it out because coverity complains otherwise. { circuit_t *circ; - for (circ=global_circuitlist;circ;circ = circ->next) { + TOR_LIST_FOREACH(circ, &global_circuitlist, head) { if (! CIRCUIT_IS_ORIGIN(circ)) { or_circuit_t *or_circ = TO_OR_CIRCUIT(circ); if (or_circ->p_chan == chan && or_circ->p_circ_id == circ_id) { @@ -1001,7 +1065,7 @@ circuit_get_by_circid_channel_impl(circid_t circ_id, channel_t *chan) circuit_t * circuit_get_by_circid_channel(circid_t circ_id, channel_t *chan) { - circuit_t *circ = circuit_get_by_circid_channel_impl(circ_id, chan); + circuit_t *circ = circuit_get_by_circid_channel_impl(circ_id, chan, NULL); if (!circ || circ->marked_for_close) return NULL; else @@ -1017,15 +1081,45 @@ circuit_t * circuit_get_by_circid_channel_even_if_marked(circid_t circ_id, channel_t *chan) { - return circuit_get_by_circid_channel_impl(circ_id, chan); + return circuit_get_by_circid_channel_impl(circ_id, chan, NULL); } /** Return true iff the circuit ID <b>circ_id</b> is currently used by a - * circuit, marked or not, on <b>chan</b>. */ + * circuit, marked or not, on <b>chan</b>, or if the circ ID is reserved until + * a queued destroy cell can be sent. + * + * (Return 1 if the circuit is present, marked or not; Return 2 + * if the circuit ID is pending a destroy.) + **/ int circuit_id_in_use_on_channel(circid_t circ_id, channel_t *chan) { - return circuit_get_by_circid_channel_impl(circ_id, chan) != NULL; + int found = 0; + if (circuit_get_by_circid_channel_impl(circ_id, chan, &found) != NULL) + return 1; + if (found) + return 2; + return 0; +} + +/** Helper for debugging 12184. Returns the time since which 'circ_id' has + * been marked unusable on 'chan'. */ +time_t +circuit_id_when_marked_unusable_on_channel(circid_t circ_id, channel_t *chan) +{ + chan_circid_circuit_map_t search; + chan_circid_circuit_map_t *found; + + memset(&search, 0, sizeof(search)); + search.circ_id = circ_id; + search.chan = chan; + + found = HT_FIND(chan_circid_map, &chan_circid_map, &search); + + if (! found || found->circuit) + return 0; + + return found->made_placeholder_at; } /** Return the circuit that a given edge connection is using. */ @@ -1049,13 +1143,59 @@ circuit_get_by_edge_conn(edge_connection_t *conn) void circuit_unlink_all_from_channel(channel_t *chan, int reason) { - circuit_t *circ; + smartlist_t *detached = smartlist_new(); + +/* #define DEBUG_CIRCUIT_UNLINK_ALL */ + + channel_unlink_all_circuits(chan, detached); + +#ifdef DEBUG_CIRCUIT_UNLINK_ALL + { + circuit_t *circ; + smartlist_t *detached_2 = smartlist_new(); + int mismatch = 0, badlen = 0; + + TOR_LIST_FOREACH(circ, &global_circuitlist, head) { + if (circ->n_chan == chan || + (!CIRCUIT_IS_ORIGIN(circ) && + TO_OR_CIRCUIT(circ)->p_chan == chan)) { + smartlist_add(detached_2, circ); + } + } + + if (smartlist_len(detached) != smartlist_len(detached_2)) { + log_warn(LD_BUG, "List of detached circuits had the wrong length! " + "(got %d, should have gotten %d)", + (int)smartlist_len(detached), + (int)smartlist_len(detached_2)); + badlen = 1; + } + smartlist_sort_pointers(detached); + smartlist_sort_pointers(detached_2); + + SMARTLIST_FOREACH(detached, circuit_t *, c, + if (c != smartlist_get(detached_2, c_sl_idx)) + mismatch = 1; + ); - channel_unlink_all_circuits(chan); + if (mismatch) + log_warn(LD_BUG, "Mismatch in list of detached circuits."); - for (circ = global_circuitlist; circ; circ = circ->next) { + if (badlen || mismatch) { + smartlist_free(detached); + detached = detached_2; + } else { + log_notice(LD_CIRC, "List of %d circuits was as expected.", + (int)smartlist_len(detached)); + smartlist_free(detached_2); + } + } +#endif + + SMARTLIST_FOREACH_BEGIN(detached, circuit_t *, circ) { int mark = 0; if (circ->n_chan == chan) { + circuit_set_n_circid_chan(circ, 0, NULL); mark = 1; @@ -1071,9 +1211,16 @@ circuit_unlink_all_from_channel(channel_t *chan, int reason) mark = 1; } } - if (mark && !circ->marked_for_close) + if (!mark) { + log_warn(LD_BUG, "Circuit on detached list which I had no reason " + "to mark"); + continue; + } + if (!circ->marked_for_close) circuit_mark_for_close(circ, reason); - } + } SMARTLIST_FOREACH_END(circ); + + smartlist_free(detached); } /** Return a circ such that @@ -1089,8 +1236,7 @@ origin_circuit_t * circuit_get_ready_rend_circ_by_rend_data(const rend_data_t *rend_data) { circuit_t *circ; - - for (circ = global_circuitlist; circ; circ = circ->next) { + TOR_LIST_FOREACH(circ, &global_circuitlist, head) { if (!circ->marked_for_close && circ->purpose == CIRCUIT_PURPOSE_C_REND_READY) { origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ); @@ -1118,11 +1264,11 @@ circuit_get_next_by_pk_and_purpose(origin_circuit_t *start, circuit_t *circ; tor_assert(CIRCUIT_PURPOSE_IS_ORIGIN(purpose)); if (start == NULL) - circ = global_circuitlist; + circ = TOR_LIST_FIRST(&global_circuitlist); else - circ = TO_CIRCUIT(start)->next; + circ = TOR_LIST_NEXT(TO_CIRCUIT(start), head); - for ( ; circ; circ = circ->next) { + for ( ; circ; circ = TOR_LIST_NEXT(circ, head)) { if (circ->marked_for_close) continue; if (circ->purpose != purpose) @@ -1137,43 +1283,175 @@ circuit_get_next_by_pk_and_purpose(origin_circuit_t *start, return NULL; } -/** Return the first OR circuit in the global list whose purpose is - * <b>purpose</b>, and whose rend_token is the <b>len</b>-byte - * <b>token</b>. */ +/** Map from rendezvous cookie to or_circuit_t */ +static digestmap_t *rend_cookie_map = NULL; + +/** Map from introduction point digest to or_circuit_t */ +static digestmap_t *intro_digest_map = NULL; + +/** Return the OR circuit whose purpose is <b>purpose</b>, and whose + * rend_token is the REND_TOKEN_LEN-byte <b>token</b>. If <b>is_rend_circ</b>, + * look for rendezvous point circuits; otherwise look for introduction point + * circuits. */ static or_circuit_t * -circuit_get_by_rend_token_and_purpose(uint8_t purpose, const char *token, - size_t len) +circuit_get_by_rend_token_and_purpose(uint8_t purpose, int is_rend_circ, + const char *token) { - circuit_t *circ; - for (circ = global_circuitlist; circ; circ = circ->next) { - if (! circ->marked_for_close && - circ->purpose == purpose && - tor_memeq(TO_OR_CIRCUIT(circ)->rend_token, token, len)) - return TO_OR_CIRCUIT(circ); + or_circuit_t *circ; + digestmap_t *map = is_rend_circ ? rend_cookie_map : intro_digest_map; + + if (!map) + return NULL; + + circ = digestmap_get(map, token); + if (!circ || + circ->base_.purpose != purpose || + circ->base_.marked_for_close) + return NULL; + + if (!circ->rendinfo) { + char *t = tor_strdup(hex_str(token, REND_TOKEN_LEN)); + log_warn(LD_BUG, "Wanted a circuit with %s:%d, but lookup returned a " + "circuit with no rendinfo set.", + safe_str(t), is_rend_circ); + tor_free(t); + return NULL; } - return NULL; + + if (! bool_eq(circ->rendinfo->is_rend_circ, is_rend_circ) || + tor_memneq(circ->rendinfo->rend_token, token, REND_TOKEN_LEN)) { + char *t = tor_strdup(hex_str(token, REND_TOKEN_LEN)); + log_warn(LD_BUG, "Wanted a circuit with %s:%d, but lookup returned %s:%d", + safe_str(t), is_rend_circ, + safe_str(hex_str(circ->rendinfo->rend_token, REND_TOKEN_LEN)), + (int)circ->rendinfo->is_rend_circ); + tor_free(t); + return NULL; + } + + return circ; +} + +/** Clear the rendezvous cookie or introduction point key digest that's + * configured on <b>circ</b>, if any, and remove it from any such maps. */ +static void +circuit_clear_rend_token(or_circuit_t *circ) +{ + or_circuit_t *found_circ; + digestmap_t *map; + + if (!circ || !circ->rendinfo) + return; + + map = circ->rendinfo->is_rend_circ ? rend_cookie_map : intro_digest_map; + + if (!map) { + log_warn(LD_BUG, "Tried to clear rend token on circuit, but found no map"); + return; + } + + found_circ = digestmap_get(map, circ->rendinfo->rend_token); + if (found_circ == circ) { + /* Great, this is the right one. */ + digestmap_remove(map, circ->rendinfo->rend_token); + } else if (found_circ) { + log_warn(LD_BUG, "Tried to clear rend token on circuit, but " + "it was already replaced in the map."); + } else { + log_warn(LD_BUG, "Tried to clear rend token on circuit, but " + "it not in the map at all."); + } + + tor_free(circ->rendinfo); /* Sets it to NULL too */ +} + +/** Set the rendezvous cookie (if is_rend_circ), or the introduction point + * digest (if ! is_rend_circ) of <b>circ</b> to the REND_TOKEN_LEN-byte value + * in <b>token</b>, and add it to the appropriate map. If it previously had a + * token, clear it. If another circuit previously had the same + * cookie/intro-digest, mark that circuit and remove it from the map. */ +static void +circuit_set_rend_token(or_circuit_t *circ, int is_rend_circ, + const uint8_t *token) +{ + digestmap_t **map_p, *map; + or_circuit_t *found_circ; + + /* Find the right map, creating it as needed */ + map_p = is_rend_circ ? &rend_cookie_map : &intro_digest_map; + + if (!*map_p) + *map_p = digestmap_new(); + + map = *map_p; + + /* If this circuit already has a token, we need to remove that. */ + if (circ->rendinfo) + circuit_clear_rend_token(circ); + + if (token == NULL) { + /* We were only trying to remove this token, not set a new one. */ + return; + } + + found_circ = digestmap_get(map, (const char *)token); + if (found_circ) { + tor_assert(found_circ != circ); + circuit_clear_rend_token(found_circ); + if (! found_circ->base_.marked_for_close) { + circuit_mark_for_close(TO_CIRCUIT(found_circ), END_CIRC_REASON_FINISHED); + if (is_rend_circ) { + log_fn(LOG_PROTOCOL_WARN, LD_REND, + "Duplicate rendezvous cookie (%s...) used on two circuits", + hex_str((const char*)token, 4)); /* only log first 4 chars */ + } + } + } + + /* Now set up the rendinfo */ + circ->rendinfo = tor_malloc(sizeof(*circ->rendinfo)); + memcpy(circ->rendinfo->rend_token, token, REND_TOKEN_LEN); + circ->rendinfo->is_rend_circ = is_rend_circ ? 1 : 0; + + digestmap_set(map, (const char *)token, circ); } /** Return the circuit waiting for a rendezvous with the provided cookie. * Return NULL if no such circuit is found. */ or_circuit_t * -circuit_get_rendezvous(const char *cookie) +circuit_get_rendezvous(const uint8_t *cookie) { return circuit_get_by_rend_token_and_purpose( CIRCUIT_PURPOSE_REND_POINT_WAITING, - cookie, REND_COOKIE_LEN); + 1, (const char*)cookie); } /** Return the circuit waiting for intro cells of the given digest. * Return NULL if no such circuit is found. */ or_circuit_t * -circuit_get_intro_point(const char *digest) +circuit_get_intro_point(const uint8_t *digest) { return circuit_get_by_rend_token_and_purpose( - CIRCUIT_PURPOSE_INTRO_POINT, digest, - DIGEST_LEN); + CIRCUIT_PURPOSE_INTRO_POINT, 0, + (const char *)digest); +} + +/** Set the rendezvous cookie of <b>circ</b> to <b>cookie</b>. If another + * circuit previously had that cookie, mark it. */ +void +circuit_set_rendezvous_cookie(or_circuit_t *circ, const uint8_t *cookie) +{ + circuit_set_rend_token(circ, 1, cookie); +} + +/** Set the intro point key digest of <b>circ</b> to <b>cookie</b>. If another + * circuit previously had that intro point digest, mark it. */ +void +circuit_set_intro_point_digest(or_circuit_t *circ, const uint8_t *digest) +{ + circuit_set_rend_token(circ, 0, digest); } /** Return a circuit that is open, is CIRCUIT_PURPOSE_C_GENERAL, @@ -1207,7 +1485,7 @@ circuit_find_to_cannibalize(uint8_t purpose, extend_info_t *info, "capacity %d, internal %d", purpose, need_uptime, need_capacity, internal); - for (circ_=global_circuitlist; circ_; circ_ = circ_->next) { + TOR_LIST_FOREACH(circ_, &global_circuitlist, head) { if (CIRCUIT_IS_ORIGIN(circ_) && circ_->state == CIRCUIT_STATE_OPEN && !circ_->marked_for_close && @@ -1297,8 +1575,7 @@ void circuit_mark_all_unused_circs(void) { circuit_t *circ; - - for (circ=global_circuitlist; circ; circ = circ->next) { + TOR_LIST_FOREACH(circ, &global_circuitlist, head) { if (CIRCUIT_IS_ORIGIN(circ) && !circ->marked_for_close && !circ->timestamp_dirty) @@ -1317,8 +1594,7 @@ void circuit_mark_all_dirty_circs_as_unusable(void) { circuit_t *circ; - - for (circ=global_circuitlist; circ; circ = circ->next) { + TOR_LIST_FOREACH(circ, &global_circuitlist, head) { if (CIRCUIT_IS_ORIGIN(circ) && !circ->marked_for_close && circ->timestamp_dirty) { @@ -1344,9 +1620,9 @@ circuit_mark_all_dirty_circs_as_unusable(void) * - If circ->rend_splice is set (we are the midpoint of a joined * rendezvous stream), then mark the other circuit to close as well. */ -void -circuit_mark_for_close_(circuit_t *circ, int reason, int line, - const char *file) +MOCK_IMPL(void, +circuit_mark_for_close_, (circuit_t *circ, int reason, int line, + const char *file)) { int orig_reason = reason; /* Passed to the controller */ assert_circuit_ok(circ); @@ -1450,6 +1726,7 @@ circuit_mark_for_close_(circuit_t *circ, int reason, int line, channel_send_destroy(circ->n_circ_id, circ->n_chan, reason); } circuitmux_detach_circuit(circ->n_chan->cmux, circ); + circuit_set_n_circid_chan(circ, 0, NULL); } if (! CIRCUIT_IS_ORIGIN(circ)) { @@ -1483,6 +1760,7 @@ circuit_mark_for_close_(circuit_t *circ, int reason, int line, channel_send_destroy(or_circ->p_circ_id, or_circ->p_chan, reason); } circuitmux_detach_circuit(or_circ->p_chan->cmux, circ); + circuit_set_p_circid_chan(or_circ, 0, NULL); } } else { origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ); @@ -1521,8 +1799,40 @@ marked_circuit_free_cells(circuit_t *circ) cell_queue_clear(& TO_OR_CIRCUIT(circ)->p_chan_cells); } -/** Return the number of cells used by the circuit <b>c</b>'s cell queues. */ +/** Aggressively free buffer contents on all the buffers of all streams in the + * list starting at <b>stream</b>. Return the number of bytes recovered. */ static size_t +marked_circuit_streams_free_bytes(edge_connection_t *stream) +{ + size_t result = 0; + for ( ; stream; stream = stream->next_stream) { + connection_t *conn = TO_CONN(stream); + if (conn->inbuf) { + result += buf_allocation(conn->inbuf); + buf_clear(conn->inbuf); + } + if (conn->outbuf) { + result += buf_allocation(conn->outbuf); + buf_clear(conn->outbuf); + } + } + return result; +} + +/** Aggressively free buffer contents on all the buffers of all streams on + * circuit <b>c</b>. Return the number of bytes recovered. */ +static size_t +marked_circuit_free_stream_bytes(circuit_t *c) +{ + if (CIRCUIT_IS_ORIGIN(c)) { + return marked_circuit_streams_free_bytes(TO_ORIGIN_CIRCUIT(c)->p_streams); + } else { + return marked_circuit_streams_free_bytes(TO_OR_CIRCUIT(c)->n_streams); + } +} + +/** Return the number of cells used by the circuit <b>c</b>'s cell queues. */ +STATIC size_t n_cells_in_circ_queues(const circuit_t *c) { size_t n = c->n_chan_cells.n; @@ -1541,17 +1851,19 @@ n_cells_in_circ_queues(const circuit_t *c) * This function will return incorrect results if the oldest cell queued on * the circuit is older than 2**32 msec (about 49 days) old. */ -static uint32_t +STATIC uint32_t circuit_max_queued_cell_age(const circuit_t *c, uint32_t now) { uint32_t age = 0; - if (c->n_chan_cells.head) - age = now - c->n_chan_cells.head->inserted_time; + packed_cell_t *cell; + + if (NULL != (cell = TOR_SIMPLEQ_FIRST(&c->n_chan_cells.head))) + age = now - cell->inserted_time; if (! CIRCUIT_IS_ORIGIN(c)) { - const or_circuit_t *orcirc = TO_OR_CIRCUIT((circuit_t*)c); - if (orcirc->p_chan_cells.head) { - uint32_t age2 = now - orcirc->p_chan_cells.head->inserted_time; + const or_circuit_t *orcirc = CONST_TO_OR_CIRCUIT(c); + if (NULL != (cell = TOR_SIMPLEQ_FIRST(&orcirc->p_chan_cells.head))) { + uint32_t age2 = now - cell->inserted_time; if (age2 > age) return age2; } @@ -1559,20 +1871,68 @@ circuit_max_queued_cell_age(const circuit_t *c, uint32_t now) return age; } -/** Temporary variable for circuits_compare_by_oldest_queued_cell_ This is a - * kludge to work around the fact that qsort doesn't provide a way for - * comparison functions to take an extra argument. */ -static uint32_t circcomp_now_tmp; +/** Return the age in milliseconds of the oldest buffer chunk on any stream in + * the linked list <b>stream</b>, where age is taken in milliseconds before + * the time <b>now</b> (in truncated milliseconds since the epoch). */ +static uint32_t +circuit_get_streams_max_data_age(const edge_connection_t *stream, uint32_t now) +{ + uint32_t age = 0, age2; + for (; stream; stream = stream->next_stream) { + const connection_t *conn = TO_CONN(stream); + if (conn->outbuf) { + age2 = buf_get_oldest_chunk_timestamp(conn->outbuf, now); + if (age2 > age) + age = age2; + } + if (conn->inbuf) { + age2 = buf_get_oldest_chunk_timestamp(conn->inbuf, now); + if (age2 > age) + age = age2; + } + } -/** Helper to sort a list of circuit_t by age of oldest cell, in descending - * order. Requires that circcomp_now_tmp is set correctly. */ + return age; +} + +/** Return the age in milliseconds of the oldest buffer chunk on any stream + * attached to the circuit <b>c</b>, where age is taken in milliseconds before + * the time <b>now</b> (in truncated milliseconds since the epoch). */ +STATIC uint32_t +circuit_max_queued_data_age(const circuit_t *c, uint32_t now) +{ + if (CIRCUIT_IS_ORIGIN(c)) { + return circuit_get_streams_max_data_age( + CONST_TO_ORIGIN_CIRCUIT(c)->p_streams, now); + } else { + return circuit_get_streams_max_data_age( + CONST_TO_OR_CIRCUIT(c)->n_streams, now); + } +} + +/** Return the age of the oldest cell or stream buffer chunk on the circuit + * <b>c</b>, where age is taken in milliseconds before the time <b>now</b> (in + * truncated milliseconds since the epoch). */ +STATIC uint32_t +circuit_max_queued_item_age(const circuit_t *c, uint32_t now) +{ + uint32_t cell_age = circuit_max_queued_cell_age(c, now); + uint32_t data_age = circuit_max_queued_data_age(c, now); + if (cell_age > data_age) + return cell_age; + else + return data_age; +} + +/** Helper to sort a list of circuit_t by age of oldest item, in descending + * order. */ static int -circuits_compare_by_oldest_queued_cell_(const void **a_, const void **b_) +circuits_compare_by_oldest_queued_item_(const void **a_, const void **b_) { const circuit_t *a = *a_; const circuit_t *b = *b_; - uint32_t age_a = circuit_max_queued_cell_age(a, circcomp_now_tmp); - uint32_t age_b = circuit_max_queued_cell_age(b, circcomp_now_tmp); + uint32_t age_a = a->age_tmp; + uint32_t age_b = b->age_tmp; if (age_a < age_b) return 1; @@ -1582,67 +1942,90 @@ circuits_compare_by_oldest_queued_cell_(const void **a_, const void **b_) return -1; } -#define FRACTION_OF_CELLS_TO_RETAIN_ON_OOM 0.90 +#define FRACTION_OF_DATA_TO_RETAIN_ON_OOM 0.90 /** We're out of memory for cells, having allocated <b>current_allocation</b> * bytes' worth. Kill the 'worst' circuits until we're under - * FRACTION_OF_CIRCS_TO_RETAIN_ON_OOM of our maximum usage. */ + * FRACTION_OF_DATA_TO_RETAIN_ON_OOM of our maximum usage. */ void circuits_handle_oom(size_t current_allocation) { /* Let's hope there's enough slack space for this allocation here... */ smartlist_t *circlist = smartlist_new(); circuit_t *circ; - size_t n_cells_removed=0, n_cells_to_remove; + size_t mem_to_recover; + size_t mem_recovered=0; int n_circuits_killed=0; struct timeval now; + uint32_t now_ms; log_notice(LD_GENERAL, "We're low on memory. Killing circuits with " "over-long queues. (This behavior is controlled by " - "MaxMemInCellQueues.)"); + "MaxMemInQueues.)"); + + { + const size_t recovered = buf_shrink_freelists(1); + if (recovered >= current_allocation) { + log_warn(LD_BUG, "We somehow recovered more memory from freelists " + "than we thought we had allocated"); + current_allocation = 0; + } else { + current_allocation -= recovered; + } + } { - size_t mem_target = (size_t)(get_options()->MaxMemInCellQueues * - FRACTION_OF_CELLS_TO_RETAIN_ON_OOM); - size_t mem_to_recover; + size_t mem_target = (size_t)(get_options()->MaxMemInQueues * + FRACTION_OF_DATA_TO_RETAIN_ON_OOM); if (current_allocation <= mem_target) return; mem_to_recover = current_allocation - mem_target; - n_cells_to_remove = CEIL_DIV(mem_to_recover, packed_cell_mem_cost()); } + tor_gettimeofday_cached_monotonic(&now); + now_ms = (uint32_t)tv_to_msec(&now); + /* This algorithm itself assumes that you've got enough memory slack * to actually run it. */ - for (circ = global_circuitlist; circ; circ = circ->next) + TOR_LIST_FOREACH(circ, &global_circuitlist, head) { + circ->age_tmp = circuit_max_queued_item_age(circ, now_ms); smartlist_add(circlist, circ); - - /* Set circcomp_now_tmp so that the sort can work. */ - tor_gettimeofday_cached(&now); - circcomp_now_tmp = (uint32_t)tv_to_msec(&now); + } /* This is O(n log n); there are faster algorithms we could use instead. * Let's hope this doesn't happen enough to be in the critical path. */ - smartlist_sort(circlist, circuits_compare_by_oldest_queued_cell_); + smartlist_sort(circlist, circuits_compare_by_oldest_queued_item_); /* Okay, now the worst circuits are at the front of the list. Let's mark * them, and reclaim their storage aggressively. */ SMARTLIST_FOREACH_BEGIN(circlist, circuit_t *, circ) { size_t n = n_cells_in_circ_queues(circ); + size_t freed; if (! circ->marked_for_close) { circuit_mark_for_close(circ, END_CIRC_REASON_RESOURCELIMIT); } marked_circuit_free_cells(circ); + freed = marked_circuit_free_stream_bytes(circ); ++n_circuits_killed; - n_cells_removed += n; - if (n_cells_removed >= n_cells_to_remove) + + mem_recovered += n * packed_cell_mem_cost(); + mem_recovered += freed; + + if (mem_recovered >= mem_to_recover) break; } SMARTLIST_FOREACH_END(circ); +#ifdef ENABLE_MEMPOOLS clean_cell_pool(); /* In case this helps. */ +#endif /* ENABLE_MEMPOOLS */ + buf_shrink_freelists(1); /* This is necessary to actually release buffer + chunks. */ - log_notice(LD_GENERAL, "Removed "U64_FORMAT" bytes by killing %d circuits.", - U64_PRINTF_ARG(n_cells_removed * packed_cell_mem_cost()), - n_circuits_killed); + log_notice(LD_GENERAL, "Removed "U64_FORMAT" bytes by killing %d circuits; " + "%d circuits remain alive.", + U64_PRINTF_ARG(mem_recovered), + n_circuits_killed, + smartlist_len(circlist) - n_circuits_killed); smartlist_free(circlist); } @@ -1716,15 +2099,10 @@ assert_circuit_ok(const circuit_t *c) tor_assert(c->purpose >= CIRCUIT_PURPOSE_MIN_ && c->purpose <= CIRCUIT_PURPOSE_MAX_); - { - /* Having a separate variable for this pleases GCC 4.2 in ways I hope I - * never understand. -NM. */ - circuit_t *nonconst_circ = (circuit_t*) c; - if (CIRCUIT_IS_ORIGIN(c)) - origin_circ = TO_ORIGIN_CIRCUIT(nonconst_circ); - else - or_circ = TO_OR_CIRCUIT(nonconst_circ); - } + if (CIRCUIT_IS_ORIGIN(c)) + origin_circ = CONST_TO_ORIGIN_CIRCUIT(c); + else + or_circ = CONST_TO_OR_CIRCUIT(c); if (c->n_chan) { tor_assert(!c->n_hop); @@ -1733,15 +2111,16 @@ assert_circuit_ok(const circuit_t *c) /* We use the _impl variant here to make sure we don't fail on marked * circuits, which would not be returned by the regular function. */ circuit_t *c2 = circuit_get_by_circid_channel_impl(c->n_circ_id, - c->n_chan); + c->n_chan, NULL); tor_assert(c == c2); } } if (or_circ && or_circ->p_chan) { if (or_circ->p_circ_id) { /* ibid */ - circuit_t *c2 = circuit_get_by_circid_channel_impl(or_circ->p_circ_id, - or_circ->p_chan); + circuit_t *c2 = + circuit_get_by_circid_channel_impl(or_circ->p_circ_id, + or_circ->p_chan, NULL); tor_assert(c == c2); } } diff --git a/src/or/circuitlist.h b/src/or/circuitlist.h index acc4b81fcd..d48d7c3963 100644 --- a/src/or/circuitlist.h +++ b/src/or/circuitlist.h @@ -12,17 +12,24 @@ #ifndef TOR_CIRCUITLIST_H #define TOR_CIRCUITLIST_H -circuit_t * circuit_get_global_list_(void); +#include "testsupport.h" + +TOR_LIST_HEAD(global_circuitlist_s, circuit_t); + +MOCK_DECL(struct global_circuitlist_s*, circuit_get_global_list, (void)); const char *circuit_state_to_string(int state); const char *circuit_purpose_to_controller_string(uint8_t purpose); const char *circuit_purpose_to_controller_hs_state_string(uint8_t purpose); const char *circuit_purpose_to_string(uint8_t purpose); void circuit_dump_by_conn(connection_t *conn, int severity); -void circuit_dump_by_chan(channel_t *chan, int severity); void circuit_set_p_circid_chan(or_circuit_t *circ, circid_t id, channel_t *chan); void circuit_set_n_circid_chan(circuit_t *circ, circid_t id, channel_t *chan); +void channel_mark_circid_unusable(channel_t *chan, circid_t id); +void channel_mark_circid_usable(channel_t *chan, circid_t id); +time_t circuit_id_when_marked_unusable_on_channel(circid_t circ_id, + channel_t *chan); void circuit_set_state(circuit_t *circ, uint8_t state); void circuit_close_all_marked(void); int32_t circuit_initial_package_window(void); @@ -41,14 +48,16 @@ origin_circuit_t *circuit_get_ready_rend_circ_by_rend_data( const rend_data_t *rend_data); origin_circuit_t *circuit_get_next_by_pk_and_purpose(origin_circuit_t *start, const char *digest, uint8_t purpose); -or_circuit_t *circuit_get_rendezvous(const char *cookie); -or_circuit_t *circuit_get_intro_point(const char *digest); +or_circuit_t *circuit_get_rendezvous(const uint8_t *cookie); +or_circuit_t *circuit_get_intro_point(const uint8_t *digest); +void circuit_set_rendezvous_cookie(or_circuit_t *circ, const uint8_t *cookie); +void circuit_set_intro_point_digest(or_circuit_t *circ, const uint8_t *digest); origin_circuit_t *circuit_find_to_cannibalize(uint8_t purpose, extend_info_t *info, int flags); void circuit_mark_all_unused_circs(void); void circuit_mark_all_dirty_circs_as_unusable(void); -void circuit_mark_for_close_(circuit_t *circ, int reason, - int line, const char *file); +MOCK_DECL(void, circuit_mark_for_close_, (circuit_t *circ, int reason, + int line, const char *file)); int circuit_get_cpath_len(origin_circuit_t *circ); void circuit_clear_cpath(origin_circuit_t *circ); crypt_path_t *circuit_get_cpath_hop(origin_circuit_t *circ, int hopnum); @@ -64,5 +73,16 @@ void assert_circuit_ok(const circuit_t *c); void circuit_free_all(void); void circuits_handle_oom(size_t current_allocation); +void channel_note_destroy_pending(channel_t *chan, circid_t id); +void channel_note_destroy_not_pending(channel_t *chan, circid_t id); + +#ifdef CIRCUITLIST_PRIVATE +STATIC void circuit_free(circuit_t *circ); +STATIC size_t n_cells_in_circ_queues(const circuit_t *c); +STATIC uint32_t circuit_max_queued_data_age(const circuit_t *c, uint32_t now); +STATIC uint32_t circuit_max_queued_cell_age(const circuit_t *c, uint32_t now); +STATIC uint32_t circuit_max_queued_item_age(const circuit_t *c, uint32_t now); +#endif + #endif diff --git a/src/or/circuitmux.c b/src/or/circuitmux.c index 545cfd0650..55580d5c29 100644 --- a/src/or/circuitmux.c +++ b/src/or/circuitmux.c @@ -10,6 +10,7 @@ #include "channel.h" #include "circuitlist.h" #include "circuitmux.h" +#include "relay.h" /* * Private typedefs for circuitmux.c @@ -115,6 +116,22 @@ struct circuitmux_s { */ struct circuit_t *active_circuits_head, *active_circuits_tail; + /** List of queued destroy cells */ + cell_queue_t destroy_cell_queue; + /** Boolean: True iff the last cell to circuitmux_get_first_active_circuit + * returned the destroy queue. Used to force alternation between + * destroy/non-destroy cells. + * + * XXXX There is no reason to think that alternating is a particularly good + * approach -- it's just designed to prevent destroys from starving other + * cells completely. + */ + unsigned int last_cell_was_destroy : 1; + /** Destroy counter: increment this when a destroy gets queued, decrement + * when we unqueue it, so we can test to make sure they don't starve. + */ + int64_t destroy_ctr; + /* * Circuitmux policy; if this is non-NULL, it can override the built- * in round-robin active circuits behavior. This is how EWMA works in @@ -193,6 +210,11 @@ static void circuitmux_assert_okay_pass_one(circuitmux_t *cmux); static void circuitmux_assert_okay_pass_two(circuitmux_t *cmux); static void circuitmux_assert_okay_pass_three(circuitmux_t *cmux); +/* Static global variables */ + +/** Count the destroy balance to debug destroy queue logic */ +static int64_t global_destroy_ctr = 0; + /* Function definitions */ /** @@ -361,16 +383,20 @@ circuitmux_alloc(void) rv = tor_malloc_zero(sizeof(*rv)); rv->chanid_circid_map = tor_malloc_zero(sizeof(*( rv->chanid_circid_map))); HT_INIT(chanid_circid_muxinfo_map, rv->chanid_circid_map); + cell_queue_init(&rv->destroy_cell_queue); return rv; } /** * Detach all circuits from a circuitmux (use before circuitmux_free()) + * + * If <b>detached_out</b> is non-NULL, add every detached circuit_t to + * detached_out. */ void -circuitmux_detach_all_circuits(circuitmux_t *cmux) +circuitmux_detach_all_circuits(circuitmux_t *cmux, smartlist_t *detached_out) { chanid_circid_muxinfo_t **i = NULL, *to_remove; channel_t *chan = NULL; @@ -386,7 +412,11 @@ circuitmux_detach_all_circuits(circuitmux_t *cmux) i = HT_START(chanid_circid_muxinfo_map, cmux->chanid_circid_map); while (i) { to_remove = *i; - if (to_remove) { + + if (! to_remove) { + log_warn(LD_BUG, "Somehow, an HT iterator gave us a NULL pointer."); + break; + } else { /* Find a channel and circuit */ chan = channel_find_by_global_id(to_remove->chan_id); if (chan) { @@ -407,6 +437,9 @@ circuitmux_detach_all_circuits(circuitmux_t *cmux) /* Clear n_mux */ circ->n_mux = NULL; + + if (detached_out) + smartlist_add(detached_out, circ); } else if (circ->magic == OR_CIRCUIT_MAGIC) { /* * Update active_circuits et al.; this does policy notifies, so @@ -422,6 +455,9 @@ circuitmux_detach_all_circuits(circuitmux_t *cmux) * so clear p_mux. */ TO_OR_CIRCUIT(circ)->p_mux = NULL; + + if (detached_out) + smartlist_add(detached_out, circ); } else { /* Complain and move on */ log_warn(LD_CIRC, @@ -476,6 +512,31 @@ circuitmux_detach_all_circuits(circuitmux_t *cmux) cmux->n_cells = 0; } +/** Reclaim all circuit IDs currently marked as unusable on <b>chan</b> because + * of pending destroy cells in <b>cmux</b>. + * + * This function must be called AFTER circuits are unlinked from the (channel, + * circuid-id) map with circuit_unlink_all_from_channel(), but before calling + * circuitmux_free(). + */ +void +circuitmux_mark_destroyed_circids_usable(circuitmux_t *cmux, channel_t *chan) +{ + packed_cell_t *cell; + int n_bad = 0; + TOR_SIMPLEQ_FOREACH(cell, &cmux->destroy_cell_queue.head, next) { + circid_t circid = 0; + if (packed_cell_is_destroy(chan, cell, &circid)) { + channel_mark_circid_usable(chan, circid); + } else { + ++n_bad; + } + } + if (n_bad) + log_warn(LD_BUG, "%d cell(s) on destroy queue did not look like a " + "DESTROY cell.", n_bad); +} + /** * Free a circuitmux_t; the circuits must be detached first with * circuitmux_detach_all_circuits(). @@ -508,6 +569,30 @@ circuitmux_free(circuitmux_t *cmux) tor_free(cmux->chanid_circid_map); } + /* + * We're throwing away some destroys; log the counter and + * adjust the global counter by the queue size. + */ + if (cmux->destroy_cell_queue.n > 0) { + cmux->destroy_ctr -= cmux->destroy_cell_queue.n; + global_destroy_ctr -= cmux->destroy_cell_queue.n; + log_debug(LD_CIRC, + "Freeing cmux at %p with %u queued destroys; the last cmux " + "destroy balance was "I64_FORMAT", global is "I64_FORMAT, + cmux, cmux->destroy_cell_queue.n, + I64_PRINTF_ARG(cmux->destroy_ctr), + I64_PRINTF_ARG(global_destroy_ctr)); + } else { + log_debug(LD_CIRC, + "Freeing cmux at %p with no queued destroys, the cmux destroy " + "balance was "I64_FORMAT", global is "I64_FORMAT, + cmux, + I64_PRINTF_ARG(cmux->destroy_ctr), + I64_PRINTF_ARG(global_destroy_ctr)); + } + + cell_queue_clear(&cmux->destroy_cell_queue); + tor_free(cmux); } @@ -816,7 +901,7 @@ circuitmux_num_cells(circuitmux_t *cmux) { tor_assert(cmux); - return cmux->n_cells; + return cmux->n_cells + cmux->destroy_cell_queue.n; } /** @@ -851,9 +936,9 @@ circuitmux_num_circuits(circuitmux_t *cmux) * Attach a circuit to a circuitmux, for the specified direction. */ -void -circuitmux_attach_circuit(circuitmux_t *cmux, circuit_t *circ, - cell_direction_t direction) +MOCK_IMPL(void, +circuitmux_attach_circuit,(circuitmux_t *cmux, circuit_t *circ, + cell_direction_t direction)) { channel_t *chan = NULL; uint64_t channel_id; @@ -1000,8 +1085,8 @@ circuitmux_attach_circuit(circuitmux_t *cmux, circuit_t *circ, * no-op if not attached. */ -void -circuitmux_detach_circuit(circuitmux_t *cmux, circuit_t *circ) +MOCK_IMPL(void, +circuitmux_detach_circuit,(circuitmux_t *cmux, circuit_t *circ)) { chanid_circid_muxinfo_t search, *hashent = NULL; /* @@ -1368,16 +1453,36 @@ circuitmux_set_num_cells(circuitmux_t *cmux, circuit_t *circ, /** * Pick a circuit to send from, using the active circuits list or a * circuitmux policy if one is available. This is called from channel.c. + * + * If we would rather send a destroy cell, return NULL and set + * *<b>destroy_queue_out</b> to the destroy queue. + * + * If we have nothing to send, set *<b>destroy_queue_out</b> to NULL and + * return NULL. */ circuit_t * -circuitmux_get_first_active_circuit(circuitmux_t *cmux) +circuitmux_get_first_active_circuit(circuitmux_t *cmux, + cell_queue_t **destroy_queue_out) { circuit_t *circ = NULL; tor_assert(cmux); + tor_assert(destroy_queue_out); + + *destroy_queue_out = NULL; + + if (cmux->destroy_cell_queue.n && + (!cmux->last_cell_was_destroy || cmux->n_active_circuits == 0)) { + /* We have destroy cells to send, and either we just sent a relay cell, + * or we have no relay cells to send. */ + + /* XXXX We should let the cmux policy have some say in this eventually. */ + /* XXXX Alternating is not a terribly brilliant approach here. */ + *destroy_queue_out = &cmux->destroy_cell_queue; - if (cmux->n_active_circuits > 0) { + cmux->last_cell_was_destroy = 1; + } else if (cmux->n_active_circuits > 0) { /* We also must have a cell available for this to be the case */ tor_assert(cmux->n_cells > 0); /* Do we have a policy-provided circuit selector? */ @@ -1389,7 +1494,11 @@ circuitmux_get_first_active_circuit(circuitmux_t *cmux) tor_assert(cmux->active_circuits_head); circ = cmux->active_circuits_head; } - } else tor_assert(cmux->n_cells == 0); + cmux->last_cell_was_destroy = 0; + } else { + tor_assert(cmux->n_cells == 0); + tor_assert(cmux->destroy_cell_queue.n == 0); + } return circ; } @@ -1463,6 +1572,26 @@ circuitmux_notify_xmit_cells(circuitmux_t *cmux, circuit_t *circ, circuitmux_assert_okay_paranoid(cmux); } +/** + * Notify the circuitmux that a destroy was sent, so we can update + * the counter. + */ + +void +circuitmux_notify_xmit_destroy(circuitmux_t *cmux) +{ + tor_assert(cmux); + + --(cmux->destroy_ctr); + --(global_destroy_ctr); + log_debug(LD_CIRC, + "Cmux at %p sent a destroy, cmux counter is now "I64_FORMAT", " + "global counter is now "I64_FORMAT, + cmux, + I64_PRINTF_ARG(cmux->destroy_ctr), + I64_PRINTF_ARG(global_destroy_ctr)); +} + /* * Circuitmux consistency checking assertions */ @@ -1743,3 +1872,76 @@ circuitmux_assert_okay_pass_three(circuitmux_t *cmux) } } +/*DOCDOC */ +void +circuitmux_append_destroy_cell(channel_t *chan, + circuitmux_t *cmux, + circid_t circ_id, + uint8_t reason) +{ + cell_t cell; + memset(&cell, 0, sizeof(cell_t)); + cell.circ_id = circ_id; + cell.command = CELL_DESTROY; + cell.payload[0] = (uint8_t) reason; + + cell_queue_append_packed_copy(NULL, &cmux->destroy_cell_queue, 0, &cell, + chan->wide_circ_ids, 0); + + /* Destroy entering the queue, update counters */ + ++(cmux->destroy_ctr); + ++global_destroy_ctr; + log_debug(LD_CIRC, + "Cmux at %p queued a destroy for circ %u, cmux counter is now " + I64_FORMAT", global counter is now "I64_FORMAT, + cmux, circ_id, + I64_PRINTF_ARG(cmux->destroy_ctr), + I64_PRINTF_ARG(global_destroy_ctr)); + + /* XXXX Duplicate code from append_cell_to_circuit_queue */ + if (!channel_has_queued_writes(chan)) { + /* There is no data at all waiting to be sent on the outbuf. Add a + * cell, so that we can notice when it gets flushed, flushed_some can + * get called, and we can start putting more data onto the buffer then. + */ + log_debug(LD_GENERAL, "Primed a buffer."); + channel_flush_from_first_active_circuit(chan, 1); + } +} + +/*DOCDOC; for debugging 12184. This runs slowly. */ +int64_t +circuitmux_count_queued_destroy_cells(const channel_t *chan, + const circuitmux_t *cmux) +{ + int64_t n_destroy_cells = cmux->destroy_ctr; + int64_t destroy_queue_size = cmux->destroy_cell_queue.n; + + int64_t manual_total = 0; + int64_t manual_total_in_map = 0; + packed_cell_t *cell; + + TOR_SIMPLEQ_FOREACH(cell, &cmux->destroy_cell_queue.head, next) { + circid_t id; + ++manual_total; + + id = packed_cell_get_circid(cell, chan->wide_circ_ids); + if (circuit_id_in_use_on_channel(id, (channel_t*)chan)) + ++manual_total_in_map; + } + + if (n_destroy_cells != destroy_queue_size || + n_destroy_cells != manual_total || + n_destroy_cells != manual_total_in_map) { + log_warn(LD_BUG, " Discrepancy in counts for queued destroy cells on " + "circuitmux. n="I64_FORMAT". queue_size="I64_FORMAT". " + "manual_total="I64_FORMAT". manual_total_in_map="I64_FORMAT".", + I64_PRINTF_ARG(n_destroy_cells), + I64_PRINTF_ARG(destroy_queue_size), + I64_PRINTF_ARG(manual_total), + I64_PRINTF_ARG(manual_total_in_map)); + } + + return n_destroy_cells; +} + diff --git a/src/or/circuitmux.h b/src/or/circuitmux.h index 25644ffab7..2b5fb7e51e 100644 --- a/src/or/circuitmux.h +++ b/src/or/circuitmux.h @@ -10,6 +10,7 @@ #define TOR_CIRCUITMUX_H #include "or.h" +#include "testsupport.h" typedef struct circuitmux_policy_s circuitmux_policy_t; typedef struct circuitmux_policy_data_s circuitmux_policy_data_t; @@ -98,7 +99,8 @@ void circuitmux_assert_okay(circuitmux_t *cmux); /* Create/destroy */ circuitmux_t * circuitmux_alloc(void); -void circuitmux_detach_all_circuits(circuitmux_t *cmux); +void circuitmux_detach_all_circuits(circuitmux_t *cmux, + smartlist_t *detached_out); void circuitmux_free(circuitmux_t *cmux); /* Policy control */ @@ -119,18 +121,32 @@ unsigned int circuitmux_num_cells(circuitmux_t *cmux); unsigned int circuitmux_num_circuits(circuitmux_t *cmux); unsigned int circuitmux_num_active_circuits(circuitmux_t *cmux); +/* Debuging interface - slow. */ +int64_t circuitmux_count_queued_destroy_cells(const channel_t *chan, + const circuitmux_t *cmux); + /* Channel interface */ -circuit_t * circuitmux_get_first_active_circuit(circuitmux_t *cmux); +circuit_t * circuitmux_get_first_active_circuit(circuitmux_t *cmux, + cell_queue_t **destroy_queue_out); void circuitmux_notify_xmit_cells(circuitmux_t *cmux, circuit_t *circ, unsigned int n_cells); +void circuitmux_notify_xmit_destroy(circuitmux_t *cmux); /* Circuit interface */ -void circuitmux_attach_circuit(circuitmux_t *cmux, circuit_t *circ, - cell_direction_t direction); -void circuitmux_detach_circuit(circuitmux_t *cmux, circuit_t *circ); +MOCK_DECL(void, circuitmux_attach_circuit, (circuitmux_t *cmux, + circuit_t *circ, + cell_direction_t direction)); +MOCK_DECL(void, circuitmux_detach_circuit, + (circuitmux_t *cmux, circuit_t *circ)); void circuitmux_clear_num_cells(circuitmux_t *cmux, circuit_t *circ); void circuitmux_set_num_cells(circuitmux_t *cmux, circuit_t *circ, unsigned int n_cells); +void circuitmux_append_destroy_cell(channel_t *chan, + circuitmux_t *cmux, circid_t circ_id, + uint8_t reason); +void circuitmux_mark_destroyed_circids_usable(circuitmux_t *cmux, + channel_t *chan); + #endif /* TOR_CIRCUITMUX_H */ diff --git a/src/or/circuitstats.c b/src/or/circuitstats.c index 1d7812bf2b..e362b1b49e 100644 --- a/src/or/circuitstats.c +++ b/src/or/circuitstats.c @@ -12,12 +12,17 @@ #include "config.h" #include "confparse.h" #include "control.h" +#include "main.h" #include "networkstatus.h" #include "statefile.h" #undef log #include <math.h> +static void cbt_control_event_buildtimeout_set( + const circuit_build_times_t *cbt, + buildtimeout_set_event_t type); + #define CBT_BIN_TO_MS(bin) ((bin)*CBT_BIN_WIDTH + (CBT_BIN_WIDTH/2)) /** Global list of circuit build times */ @@ -26,12 +31,46 @@ // vary in their own latency. The downside of this is that guards // can change frequently, so we'd be building a lot more circuits // most likely. -/* XXXX024 Make this static; add accessor functions. */ -circuit_build_times_t circ_times; +static circuit_build_times_t circ_times; +#ifdef TOR_UNIT_TESTS /** If set, we're running the unit tests: we should avoid clobbering * our state file or accessing get_options() or get_or_state() */ static int unit_tests = 0; +#else +#define unit_tests 0 +#endif + +/** Return a pointer to the data structure describing our current circuit + * build time history and computations. */ +const circuit_build_times_t * +get_circuit_build_times(void) +{ + return &circ_times; +} + +/** As get_circuit_build_times, but return a mutable pointer. */ +circuit_build_times_t * +get_circuit_build_times_mutable(void) +{ + return &circ_times; +} + +/** Return the time to wait before actually closing an under-construction, in + * milliseconds. */ +double +get_circuit_build_close_time_ms(void) +{ + return circ_times.close_ms; +} + +/** Return the time to wait before giving up on an under-construction circuit, + * in milliseconds. */ +double +get_circuit_build_timeout_ms(void) +{ + return circ_times.timeout_ms; +} /** * This function decides if CBT learning should be disabled. It returns @@ -56,18 +95,22 @@ circuit_build_times_disabled(void) if (consensus_disabled || config_disabled || dirauth_disabled || state_disabled) { +#if 0 log_debug(LD_CIRC, "CircuitBuildTime learning is disabled. " "Consensus=%d, Config=%d, AuthDir=%d, StateFile=%d", consensus_disabled, config_disabled, dirauth_disabled, state_disabled); +#endif return 1; } else { +#if 0 log_debug(LD_CIRC, "CircuitBuildTime learning is not disabled. " "Consensus=%d, Config=%d, AuthDir=%d, StateFile=%d", consensus_disabled, config_disabled, dirauth_disabled, state_disabled); +#endif return 0; } } @@ -154,7 +197,7 @@ circuit_build_times_min_circs_to_observe(void) /** Return true iff <b>cbt</b> has recorded enough build times that we * want to start acting on the timeout it implies. */ int -circuit_build_times_enough_to_compute(circuit_build_times_t *cbt) +circuit_build_times_enough_to_compute(const circuit_build_times_t *cbt) { return cbt->total_build_times >= circuit_build_times_min_circs_to_observe(); } @@ -438,7 +481,7 @@ circuit_build_times_get_initial_timeout(void) * Leave estimated parameters, timeout and network liveness intact * for future use. */ -void +STATIC void circuit_build_times_reset(circuit_build_times_t *cbt) { memset(cbt->circuit_build_times, 0, sizeof(cbt->circuit_build_times)); @@ -471,7 +514,7 @@ circuit_build_times_init(circuit_build_times_t *cbt) cbt->liveness.timeouts_after_firsthop = NULL; } cbt->close_ms = cbt->timeout_ms = circuit_build_times_get_initial_timeout(); - control_event_buildtimeout_set(cbt, BUILDTIMEOUT_SET_EVENT_RESET); + cbt_control_event_buildtimeout_set(cbt, BUILDTIMEOUT_SET_EVENT_RESET); } /** @@ -557,7 +600,7 @@ circuit_build_times_add_time(circuit_build_times_t *cbt, build_time_t time) * Return maximum circuit build time */ static build_time_t -circuit_build_times_max(circuit_build_times_t *cbt) +circuit_build_times_max(const circuit_build_times_t *cbt) { int i = 0; build_time_t max_build_time = 0; @@ -598,7 +641,7 @@ circuit_build_times_min(circuit_build_times_t *cbt) * The return value must be freed by the caller. */ static uint32_t * -circuit_build_times_create_histogram(circuit_build_times_t *cbt, +circuit_build_times_create_histogram(const circuit_build_times_t *cbt, build_time_t *nbins) { uint32_t *histogram; @@ -688,7 +731,7 @@ circuit_build_times_get_xm(circuit_build_times_t *cbt) * the or_state_t state structure. */ void -circuit_build_times_update_state(circuit_build_times_t *cbt, +circuit_build_times_update_state(const circuit_build_times_t *cbt, or_state_t *state) { uint32_t *histogram; @@ -949,7 +992,7 @@ circuit_build_times_parse_state(circuit_build_times_t *cbt, * an acceptable approximation because we are only concerned with the * accuracy of the CDF of the tail. */ -int +STATIC int circuit_build_times_update_alpha(circuit_build_times_t *cbt) { build_time_t *x=cbt->circuit_build_times; @@ -1033,7 +1076,7 @@ circuit_build_times_update_alpha(circuit_build_times_t *cbt) * * Return value is in milliseconds. */ -double +STATIC double circuit_build_times_calculate_timeout(circuit_build_times_t *cbt, double quantile) { @@ -1050,6 +1093,7 @@ circuit_build_times_calculate_timeout(circuit_build_times_t *cbt, return ret; } +#ifdef TOR_UNIT_TESTS /** Pareto CDF */ double circuit_build_times_cdf(circuit_build_times_t *cbt, double x) @@ -1060,7 +1104,9 @@ circuit_build_times_cdf(circuit_build_times_t *cbt, double x) tor_assert(0 <= ret && ret <= 1.0); return ret; } +#endif +#ifdef TOR_UNIT_TESTS /** * Generate a synthetic time using our distribution parameters. * @@ -1093,7 +1139,9 @@ circuit_build_times_generate_sample(circuit_build_times_t *cbt, tor_assert(ret > 0); return ret; } +#endif +#ifdef TOR_UNIT_TESTS /** * Estimate an initial alpha parameter by solving the quantile * function with a quantile point and a specific timeout value. @@ -1114,12 +1162,13 @@ circuit_build_times_initial_alpha(circuit_build_times_t *cbt, (tor_mathlog(cbt->Xm)-tor_mathlog(timeout_ms)); tor_assert(cbt->alpha > 0); } +#endif /** * Returns true if we need circuits to be built */ int -circuit_build_times_needs_circuits(circuit_build_times_t *cbt) +circuit_build_times_needs_circuits(const circuit_build_times_t *cbt) { /* Return true if < MIN_CIRCUITS_TO_OBSERVE */ return !circuit_build_times_enough_to_compute(cbt); @@ -1130,13 +1179,19 @@ circuit_build_times_needs_circuits(circuit_build_times_t *cbt) * right now. */ int -circuit_build_times_needs_circuits_now(circuit_build_times_t *cbt) +circuit_build_times_needs_circuits_now(const circuit_build_times_t *cbt) { return circuit_build_times_needs_circuits(cbt) && approx_time()-cbt->last_circ_at > circuit_build_times_test_frequency(); } /** + * How long should we be unreachable before we think we need to check if + * our published IP address has changed. + */ +#define CIRCUIT_TIMEOUT_BEFORE_RECHECK_IP (60*3) + +/** * Called to indicate that the network showed some signs of liveness, * i.e. we received a cell. * @@ -1151,12 +1206,15 @@ circuit_build_times_network_is_live(circuit_build_times_t *cbt) { time_t now = approx_time(); if (cbt->liveness.nonlive_timeouts > 0) { + time_t time_since_live = now - cbt->liveness.network_last_live; log_notice(LD_CIRC, "Tor now sees network activity. Restoring circuit build " "timeout recording. Network was down for %d seconds " "during %d circuit attempts.", - (int)(now - cbt->liveness.network_last_live), + (int)time_since_live, cbt->liveness.nonlive_timeouts); + if (time_since_live > CIRCUIT_TIMEOUT_BEFORE_RECHECK_IP) + reschedule_descriptor_update_check(); } cbt->liveness.network_last_live = now; cbt->liveness.nonlive_timeouts = 0; @@ -1263,7 +1321,7 @@ circuit_build_times_network_close(circuit_build_times_t *cbt, * in the case of recent liveness changes. */ int -circuit_build_times_network_check_live(circuit_build_times_t *cbt) +circuit_build_times_network_check_live(const circuit_build_times_t *cbt) { if (cbt->liveness.nonlive_timeouts > 0) { return 0; @@ -1282,7 +1340,7 @@ circuit_build_times_network_check_live(circuit_build_times_t *cbt) * to restart the process of building test circuits and estimating a * new timeout. */ -int +STATIC int circuit_build_times_network_check_changed(circuit_build_times_t *cbt) { int total_build_times = cbt->total_build_times; @@ -1329,7 +1387,7 @@ circuit_build_times_network_check_changed(circuit_build_times_t *cbt) = circuit_build_times_get_initial_timeout(); } - control_event_buildtimeout_set(cbt, BUILDTIMEOUT_SET_EVENT_RESET); + cbt_control_event_buildtimeout_set(cbt, BUILDTIMEOUT_SET_EVENT_RESET); log_notice(LD_CIRC, "Your network connection speed appears to have changed. Resetting " @@ -1511,7 +1569,7 @@ circuit_build_times_set_timeout(circuit_build_times_t *cbt) } } - control_event_buildtimeout_set(cbt, BUILDTIMEOUT_SET_EVENT_COMPUTED); + cbt_control_event_buildtimeout_set(cbt, BUILDTIMEOUT_SET_EVENT_COMPUTED); timeout_rate = circuit_build_times_timeout_rate(cbt); @@ -1546,6 +1604,8 @@ circuit_build_times_set_timeout(circuit_build_times_t *cbt) cbt->total_build_times); } } + +#ifdef TOR_UNIT_TESTS /** Make a note that we're running unit tests (rather than running Tor * itself), so we avoid clobbering our state file. */ void @@ -1553,4 +1613,46 @@ circuitbuild_running_unit_tests(void) { unit_tests = 1; } +#endif + +void +circuit_build_times_update_last_circ(circuit_build_times_t *cbt) +{ + cbt->last_circ_at = approx_time(); +} + +static void +cbt_control_event_buildtimeout_set(const circuit_build_times_t *cbt, + buildtimeout_set_event_t type) +{ + char *args = NULL; + double qnt; + + switch (type) { + case BUILDTIMEOUT_SET_EVENT_RESET: + case BUILDTIMEOUT_SET_EVENT_SUSPENDED: + case BUILDTIMEOUT_SET_EVENT_DISCARD: + qnt = 1.0; + break; + case BUILDTIMEOUT_SET_EVENT_COMPUTED: + case BUILDTIMEOUT_SET_EVENT_RESUME: + default: + qnt = circuit_build_times_quantile_cutoff(); + break; + } + + tor_asprintf(&args, "TOTAL_TIMES=%lu " + "TIMEOUT_MS=%lu XM=%lu ALPHA=%f CUTOFF_QUANTILE=%f " + "TIMEOUT_RATE=%f CLOSE_MS=%lu CLOSE_RATE=%f", + (unsigned long)cbt->total_build_times, + (unsigned long)cbt->timeout_ms, + (unsigned long)cbt->Xm, cbt->alpha, qnt, + circuit_build_times_timeout_rate(cbt), + (unsigned long)cbt->close_ms, + circuit_build_times_close_rate(cbt)); + + control_event_buildtimeout_set(type, args); + + tor_free(args); +} diff --git a/src/or/circuitstats.h b/src/or/circuitstats.h index 87dce99f4f..3343310b8e 100644 --- a/src/or/circuitstats.h +++ b/src/or/circuitstats.h @@ -12,11 +12,14 @@ #ifndef TOR_CIRCUITSTATS_H #define TOR_CIRCUITSTATS_H -extern circuit_build_times_t circ_times; +const circuit_build_times_t *get_circuit_build_times(void); +circuit_build_times_t *get_circuit_build_times_mutable(void); +double get_circuit_build_close_time_ms(void); +double get_circuit_build_timeout_ms(void); int circuit_build_times_disabled(void); -int circuit_build_times_enough_to_compute(circuit_build_times_t *cbt); -void circuit_build_times_update_state(circuit_build_times_t *cbt, +int circuit_build_times_enough_to_compute(const circuit_build_times_t *cbt); +void circuit_build_times_update_state(const circuit_build_times_t *cbt, or_state_t *state); int circuit_build_times_parse_state(circuit_build_times_t *cbt, or_state_t *state); @@ -27,9 +30,9 @@ int circuit_build_times_count_close(circuit_build_times_t *cbt, void circuit_build_times_set_timeout(circuit_build_times_t *cbt); int circuit_build_times_add_time(circuit_build_times_t *cbt, build_time_t time); -int circuit_build_times_needs_circuits(circuit_build_times_t *cbt); +int circuit_build_times_needs_circuits(const circuit_build_times_t *cbt); -int circuit_build_times_needs_circuits_now(circuit_build_times_t *cbt); +int circuit_build_times_needs_circuits_now(const circuit_build_times_t *cbt); void circuit_build_times_init(circuit_build_times_t *cbt); void circuit_build_times_free_timeouts(circuit_build_times_t *cbt); void circuit_build_times_new_consensus_params(circuit_build_times_t *cbt, @@ -37,29 +40,59 @@ void circuit_build_times_new_consensus_params(circuit_build_times_t *cbt, double circuit_build_times_timeout_rate(const circuit_build_times_t *cbt); double circuit_build_times_close_rate(const circuit_build_times_t *cbt); +void circuit_build_times_update_last_circ(circuit_build_times_t *cbt); + #ifdef CIRCUITSTATS_PRIVATE -double circuit_build_times_calculate_timeout(circuit_build_times_t *cbt, +STATIC double circuit_build_times_calculate_timeout(circuit_build_times_t *cbt, double quantile); +STATIC int circuit_build_times_update_alpha(circuit_build_times_t *cbt); +STATIC void circuit_build_times_reset(circuit_build_times_t *cbt); + +/* Network liveness functions */ +STATIC int circuit_build_times_network_check_changed( + circuit_build_times_t *cbt); +#endif + +#ifdef TOR_UNIT_TESTS build_time_t circuit_build_times_generate_sample(circuit_build_times_t *cbt, double q_lo, double q_hi); +double circuit_build_times_cdf(circuit_build_times_t *cbt, double x); void circuit_build_times_initial_alpha(circuit_build_times_t *cbt, double quantile, double time_ms); -int circuit_build_times_update_alpha(circuit_build_times_t *cbt); -double circuit_build_times_cdf(circuit_build_times_t *cbt, double x); void circuitbuild_running_unit_tests(void); -void circuit_build_times_reset(circuit_build_times_t *cbt); - -/* Network liveness functions */ -int circuit_build_times_network_check_changed(circuit_build_times_t *cbt); #endif /* Network liveness functions */ void circuit_build_times_network_is_live(circuit_build_times_t *cbt); -int circuit_build_times_network_check_live(circuit_build_times_t *cbt); +int circuit_build_times_network_check_live(const circuit_build_times_t *cbt); void circuit_build_times_network_circ_success(circuit_build_times_t *cbt); -/* DOCDOC circuit_build_times_get_bw_scale */ -int circuit_build_times_get_bw_scale(networkstatus_t *ns); +#ifdef CIRCUITSTATS_PRIVATE +/** Structure for circuit build times history */ +struct circuit_build_times_s { + /** The circular array of recorded build times in milliseconds */ + build_time_t circuit_build_times[CBT_NCIRCUITS_TO_OBSERVE]; + /** Current index in the circuit_build_times circular array */ + int build_times_idx; + /** Total number of build times accumulated. Max CBT_NCIRCUITS_TO_OBSERVE */ + int total_build_times; + /** Information about the state of our local network connection */ + network_liveness_t liveness; + /** Last time we built a circuit. Used to decide to build new test circs */ + time_t last_circ_at; + /** "Minimum" value of our pareto distribution (actually mode) */ + build_time_t Xm; + /** alpha exponent for pareto dist. */ + double alpha; + /** Have we computed a timeout? */ + int have_computed_timeout; + /** The exact value for that timeout in milliseconds. Stored as a double + * to maintain precision from calculations to and from quantile value. */ + double timeout_ms; + /** How long we wait before actually closing the circuit. */ + double close_ms; +}; +#endif #endif diff --git a/src/or/circuituse.c b/src/or/circuituse.c index 06a51a04a2..714754a672 100644 --- a/src/or/circuituse.c +++ b/src/or/circuituse.c @@ -12,6 +12,7 @@ #include "or.h" #include "addressmap.h" #include "channel.h" +#include "circpathbias.h" #include "circuitbuild.h" #include "circuitlist.h" #include "circuitstats.h" @@ -31,12 +32,6 @@ #include "router.h" #include "routerlist.h" -/********* START VARIABLES **********/ - -extern circuit_t *global_circuitlist; /* from circuitlist.c */ - -/********* END VARIABLES ************/ - static void circuit_expire_old_circuits_clientside(void); static void circuit_increment_failure_count(void); @@ -286,7 +281,7 @@ circuit_get_best(const entry_connection_t *conn, tor_gettimeofday(&now); - for (circ=global_circuitlist;circ;circ = circ->next) { + TOR_LIST_FOREACH(circ, circuit_get_global_list(), head) { origin_circuit_t *origin_circ; if (!CIRCUIT_IS_ORIGIN(circ)) continue; @@ -301,7 +296,7 @@ circuit_get_best(const entry_connection_t *conn, } if (!circuit_is_acceptable(origin_circ,conn,must_be_open,purpose, - need_uptime,need_internal,now.tv_sec)) + need_uptime,need_internal, (time_t)now.tv_sec)) continue; /* now this is an acceptable circ to hand back. but that doesn't @@ -327,7 +322,7 @@ count_pending_general_client_circuits(void) int count = 0; - for (circ = global_circuitlist; circ; circ = circ->next) { + TOR_LIST_FOREACH(circ, circuit_get_global_list(), head) { if (circ->marked_for_close || circ->state == CIRCUIT_STATE_OPEN || circ->purpose != CIRCUIT_PURPOSE_C_GENERAL || @@ -375,7 +370,7 @@ circuit_conforms_to_options(const origin_circuit_t *circ, void circuit_expire_building(void) { - circuit_t *victim, *next_circ = global_circuitlist; + circuit_t *victim, *next_circ; /* circ_times.timeout_ms and circ_times.close_ms are from * circuit_build_times_get_initial_timeout() if we haven't computed * custom timeouts yet */ @@ -393,10 +388,9 @@ circuit_expire_building(void) * we want to be more lenient with timeouts, in case the * user has relocated and/or changed network connections. * See bug #3443. */ - while (next_circ) { + TOR_LIST_FOREACH(next_circ, circuit_get_global_list(), head) { if (!CIRCUIT_IS_ORIGIN(next_circ) || /* didn't originate here */ next_circ->marked_for_close) { /* don't mess with marked circs */ - next_circ = next_circ->next; continue; } @@ -408,9 +402,7 @@ circuit_expire_building(void) any_opened_circs = 1; break; } - next_circ = next_circ->next; } - next_circ = global_circuitlist; #define SET_CUTOFF(target, msec) do { \ long ms = tor_lround(msec); \ @@ -451,12 +443,12 @@ circuit_expire_building(void) * RTTs = 4a + 3b + 2c * RTTs = 9h */ - SET_CUTOFF(general_cutoff, circ_times.timeout_ms); - SET_CUTOFF(begindir_cutoff, circ_times.timeout_ms); + SET_CUTOFF(general_cutoff, get_circuit_build_timeout_ms()); + SET_CUTOFF(begindir_cutoff, get_circuit_build_timeout_ms()); /* > 3hop circs seem to have a 1.0 second delay on their cannibalized * 4th hop. */ - SET_CUTOFF(fourhop_cutoff, circ_times.timeout_ms * (10/6.0) + 1000); + SET_CUTOFF(fourhop_cutoff, get_circuit_build_timeout_ms() * (10/6.0) + 1000); /* CIRCUIT_PURPOSE_C_ESTABLISH_REND behaves more like a RELAY cell. * Use the stream cutoff (more or less). */ @@ -465,26 +457,25 @@ circuit_expire_building(void) /* Be lenient with cannibalized circs. They already survived the official * CBT, and they're usually not performance-critical. */ SET_CUTOFF(cannibalized_cutoff, - MAX(circ_times.close_ms*(4/6.0), + MAX(get_circuit_build_close_time_ms()*(4/6.0), options->CircuitStreamTimeout * 1000) + 1000); /* Intro circs have an extra round trip (and are also 4 hops long) */ - SET_CUTOFF(c_intro_cutoff, circ_times.timeout_ms * (14/6.0) + 1000); + SET_CUTOFF(c_intro_cutoff, get_circuit_build_timeout_ms() * (14/6.0) + 1000); /* Server intro circs have an extra round trip */ - SET_CUTOFF(s_intro_cutoff, circ_times.timeout_ms * (9/6.0) + 1000); + SET_CUTOFF(s_intro_cutoff, get_circuit_build_timeout_ms() * (9/6.0) + 1000); - SET_CUTOFF(close_cutoff, circ_times.close_ms); - SET_CUTOFF(extremely_old_cutoff, circ_times.close_ms*2 + 1000); + SET_CUTOFF(close_cutoff, get_circuit_build_close_time_ms()); + SET_CUTOFF(extremely_old_cutoff, get_circuit_build_close_time_ms()*2 + 1000); SET_CUTOFF(hs_extremely_old_cutoff, - MAX(circ_times.close_ms*2 + 1000, + MAX(get_circuit_build_close_time_ms()*2 + 1000, options->SocksTimeout * 1000)); - while (next_circ) { + TOR_LIST_FOREACH(next_circ, circuit_get_global_list(), head) { struct timeval cutoff; victim = next_circ; - next_circ = next_circ->next; if (!CIRCUIT_IS_ORIGIN(victim) || /* didn't originate here */ victim->marked_for_close) /* don't mess with marked circs */ continue; @@ -546,7 +537,9 @@ circuit_expire_building(void) "%d guards are live.", TO_ORIGIN_CIRCUIT(victim)->global_identifier, circuit_purpose_to_string(victim->purpose), - TO_ORIGIN_CIRCUIT(victim)->build_state->desired_path_len, + TO_ORIGIN_CIRCUIT(victim)->build_state ? + TO_ORIGIN_CIRCUIT(victim)->build_state->desired_path_len : + -1, circuit_state_to_string(victim->state), channel_state_to_string(victim->n_chan->state), num_live_entry_guards(0)); @@ -555,12 +548,14 @@ circuit_expire_building(void) * was a timeout, and the timeout value needs to reset if we * see enough of them. Note this means we also need to avoid * double-counting below, too. */ - circuit_build_times_count_timeout(&circ_times, first_hop_succeeded); + circuit_build_times_count_timeout(get_circuit_build_times_mutable(), + first_hop_succeeded); TO_ORIGIN_CIRCUIT(victim)->relaxed_timeout = 1; } continue; } else { static ratelim_t relax_timeout_limit = RATELIM_INIT(3600); + const double build_close_ms = get_circuit_build_close_time_ms(); log_fn_ratelim(&relax_timeout_limit, LOG_NOTICE, LD_CIRC, "No circuits are opened. Relaxed timeout for circuit %d " "(a %s %d-hop circuit in state %s with channel state %s) to " @@ -568,10 +563,13 @@ circuit_expire_building(void) "anyway. %d guards are live.", TO_ORIGIN_CIRCUIT(victim)->global_identifier, circuit_purpose_to_string(victim->purpose), - TO_ORIGIN_CIRCUIT(victim)->build_state->desired_path_len, + TO_ORIGIN_CIRCUIT(victim)->build_state ? + TO_ORIGIN_CIRCUIT(victim)->build_state->desired_path_len : + -1, circuit_state_to_string(victim->state), channel_state_to_string(victim->n_chan->state), - (long)circ_times.close_ms, num_live_entry_guards(0)); + (long)build_close_ms, + num_live_entry_guards(0)); } } @@ -651,7 +649,7 @@ circuit_expire_building(void) } if (circuit_timeout_want_to_count_circ(TO_ORIGIN_CIRCUIT(victim)) && - circuit_build_times_enough_to_compute(&circ_times)) { + circuit_build_times_enough_to_compute(get_circuit_build_times())) { /* Circuits are allowed to last longer for measurement. * Switch their purpose and wait. */ if (victim->purpose != CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT) { @@ -665,8 +663,9 @@ circuit_expire_building(void) * have a timeout. We also want to avoid double-counting * already "relaxed" circuits, which are counted above. */ if (!TO_ORIGIN_CIRCUIT(victim)->relaxed_timeout) { - circuit_build_times_count_timeout(&circ_times, - first_hop_succeeded); + circuit_build_times_count_timeout( + get_circuit_build_times_mutable(), + first_hop_succeeded); } continue; } @@ -683,10 +682,11 @@ circuit_expire_building(void) (long)(now.tv_sec - victim->timestamp_began.tv_sec), victim->purpose, circuit_purpose_to_string(victim->purpose)); - } else if (circuit_build_times_count_close(&circ_times, - first_hop_succeeded, - victim->timestamp_created.tv_sec)) { - circuit_build_times_set_timeout(&circ_times); + } else if (circuit_build_times_count_close( + get_circuit_build_times_mutable(), + first_hop_succeeded, + (time_t)victim->timestamp_created.tv_sec)) { + circuit_build_times_set_timeout(get_circuit_build_times_mutable()); } } } @@ -711,7 +711,8 @@ circuit_expire_building(void) * and we have tried to send an INTRODUCE1 cell specifying it. * Thus, if the pending_final_cpath field *is* NULL, then we * want to not spare it. */ - if (TO_ORIGIN_CIRCUIT(victim)->build_state->pending_final_cpath == + if (TO_ORIGIN_CIRCUIT(victim)->build_state && + TO_ORIGIN_CIRCUIT(victim)->build_state->pending_final_cpath == NULL) break; /* fallthrough! */ @@ -750,23 +751,27 @@ circuit_expire_building(void) if (victim->n_chan) log_info(LD_CIRC, - "Abandoning circ %u %s:%d (state %d,%d:%s, purpose %d, " + "Abandoning circ %u %s:%u (state %d,%d:%s, purpose %d, " "len %d)", TO_ORIGIN_CIRCUIT(victim)->global_identifier, channel_get_canonical_remote_descr(victim->n_chan), (unsigned)victim->n_circ_id, TO_ORIGIN_CIRCUIT(victim)->has_opened, victim->state, circuit_state_to_string(victim->state), victim->purpose, - TO_ORIGIN_CIRCUIT(victim)->build_state->desired_path_len); + TO_ORIGIN_CIRCUIT(victim)->build_state ? + TO_ORIGIN_CIRCUIT(victim)->build_state->desired_path_len : + -1); else log_info(LD_CIRC, - "Abandoning circ %u %d (state %d,%d:%s, purpose %d, len %d)", + "Abandoning circ %u %u (state %d,%d:%s, purpose %d, len %d)", TO_ORIGIN_CIRCUIT(victim)->global_identifier, (unsigned)victim->n_circ_id, TO_ORIGIN_CIRCUIT(victim)->has_opened, victim->state, circuit_state_to_string(victim->state), victim->purpose, - TO_ORIGIN_CIRCUIT(victim)->build_state->desired_path_len); + TO_ORIGIN_CIRCUIT(victim)->build_state ? + TO_ORIGIN_CIRCUIT(victim)->build_state->desired_path_len : + -1); circuit_log_path(LOG_INFO,LD_CIRC,TO_ORIGIN_CIRCUIT(victim)); if (victim->purpose == CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT) @@ -778,6 +783,129 @@ circuit_expire_building(void) } } +/** For debugging #8387: track when we last called + * circuit_expire_old_circuits_clientside. */ +static time_t last_expired_clientside_circuits = 0; + +/** + * As a diagnostic for bug 8387, log information about how many one-hop + * circuits we have around that have been there for at least <b>age</b> + * seconds. Log a few of them. + */ +void +circuit_log_ancient_one_hop_circuits(int age) +{ +#define MAX_ANCIENT_ONEHOP_CIRCUITS_TO_LOG 10 + time_t now = time(NULL); + time_t cutoff = now - age; + int n_found = 0; + smartlist_t *log_these = smartlist_new(); + const circuit_t *circ; + + TOR_LIST_FOREACH(circ, circuit_get_global_list(), head) { + const origin_circuit_t *ocirc; + if (! CIRCUIT_IS_ORIGIN(circ)) + continue; + if (circ->timestamp_created.tv_sec >= cutoff) + continue; + ocirc = CONST_TO_ORIGIN_CIRCUIT(circ); + + if (ocirc->build_state && ocirc->build_state->onehop_tunnel) { + ++n_found; + + if (smartlist_len(log_these) < MAX_ANCIENT_ONEHOP_CIRCUITS_TO_LOG) + smartlist_add(log_these, (origin_circuit_t*) ocirc); + } + } + + if (n_found == 0) + goto done; + + log_notice(LD_HEARTBEAT, + "Diagnostic for issue 8387: Found %d one-hop circuits more " + "than %d seconds old! Logging %d...", + n_found, age, smartlist_len(log_these)); + + SMARTLIST_FOREACH_BEGIN(log_these, const origin_circuit_t *, ocirc) { + char created[ISO_TIME_LEN+1]; + int stream_num; + const edge_connection_t *conn; + char *dirty = NULL; + circ = TO_CIRCUIT(ocirc); + + format_local_iso_time(created, + (time_t)circ->timestamp_created.tv_sec); + + if (circ->timestamp_dirty) { + char dirty_since[ISO_TIME_LEN+1]; + format_local_iso_time(dirty_since, circ->timestamp_dirty); + + tor_asprintf(&dirty, "Dirty since %s (%ld seconds vs %ld-second cutoff)", + dirty_since, (long)(now - circ->timestamp_dirty), + (long) get_options()->MaxCircuitDirtiness); + } else { + dirty = tor_strdup("Not marked dirty"); + } + + log_notice(LD_HEARTBEAT, " #%d created at %s. %s, %s. %s for close. " + "%s for new conns. %s.", + ocirc_sl_idx, + created, + circuit_state_to_string(circ->state), + circuit_purpose_to_string(circ->purpose), + circ->marked_for_close ? "Marked" : "Not marked", + ocirc->unusable_for_new_conns ? "Not usable" : "usable", + dirty); + tor_free(dirty); + + stream_num = 0; + for (conn = ocirc->p_streams; conn; conn = conn->next_stream) { + const connection_t *c = TO_CONN(conn); + char stream_created[ISO_TIME_LEN+1]; + if (++stream_num >= 5) + break; + + format_local_iso_time(stream_created, c->timestamp_created); + + log_notice(LD_HEARTBEAT, " Stream#%d created at %s. " + "%s conn in state %s. " + "%s for close (%s:%d). Hold-open is %sset. " + "Has %ssent RELAY_END. %s on circuit.", + stream_num, + stream_created, + conn_type_to_string(c->type), + conn_state_to_string(c->type, c->state), + c->marked_for_close ? "Marked" : "Not marked", + c->marked_for_close_file ? c->marked_for_close_file : "--", + c->marked_for_close, + c->hold_open_until_flushed ? "" : "not ", + conn->edge_has_sent_end ? "" : "not ", + conn->edge_blocked_on_circ ? "Blocked" : "Not blocked"); + if (! c->linked_conn) + continue; + + c = c->linked_conn; + + log_notice(LD_HEARTBEAT, " Linked to %s connection in state %s " + "(Purpose %d). %s for close (%s:%d). Hold-open is %sset. ", + conn_type_to_string(c->type), + conn_state_to_string(c->type, c->state), + c->purpose, + c->marked_for_close ? "Marked" : "Not marked", + c->marked_for_close_file ? c->marked_for_close_file : "--", + c->marked_for_close, + c->hold_open_until_flushed ? "" : "not "); + } + } SMARTLIST_FOREACH_END(ocirc); + + log_notice(LD_HEARTBEAT, "It has been %ld seconds since I last called " + "circuit_expire_old_circuits_clientside().", + (long)(now - last_expired_clientside_circuits)); + + done: + smartlist_free(log_these); +} + /** Remove any elements in <b>needed_ports</b> that are handled by an * open or in-progress circuit. */ @@ -818,7 +946,7 @@ circuit_stream_is_being_handled(entry_connection_t *conn, get_options()->LongLivedPorts, conn ? conn->socks_request->port : port); - for (circ=global_circuitlist;circ;circ = circ->next) { + TOR_LIST_FOREACH(circ, circuit_get_global_list(), head) { if (CIRCUIT_IS_ORIGIN(circ) && !circ->marked_for_close && circ->purpose == CIRCUIT_PURPOSE_C_GENERAL && @@ -869,7 +997,7 @@ circuit_predict_and_launch_new(void) int flags = 0; /* First, count how many of each type of circuit we have already. */ - for (circ=global_circuitlist;circ;circ = circ->next) { + TOR_LIST_FOREACH(circ, circuit_get_global_list(), head) { cpath_build_state_t *build_state; origin_circuit_t *origin_circ; if (!CIRCUIT_IS_ORIGIN(circ)) @@ -949,7 +1077,7 @@ circuit_predict_and_launch_new(void) * we can still build circuits preemptively as needed. */ if (num < MAX_UNUSED_OPEN_CIRCUITS-2 && ! circuit_build_times_disabled() && - circuit_build_times_needs_circuits_now(&circ_times)) { + circuit_build_times_needs_circuits_now(get_circuit_build_times())) { flags = CIRCLAUNCH_NEED_CAPACITY; log_info(LD_CIRC, "Have %d clean circs need another buildtime test circ.", num); @@ -969,7 +1097,6 @@ circuit_predict_and_launch_new(void) void circuit_build_needed_circs(time_t now) { - static time_t time_to_new_circuit = 0; const or_options_t *options = get_options(); /* launch a new circ for any pending streams that need one */ @@ -978,14 +1105,34 @@ circuit_build_needed_circs(time_t now) /* make sure any hidden services have enough intro points */ rend_services_introduce(); - if (time_to_new_circuit < now) { + circuit_expire_old_circs_as_needed(now); + + if (!options->DisablePredictedCircuits) + circuit_predict_and_launch_new(); +} + +/** + * Called once a second either directly or from + * circuit_build_needed_circs(). As appropriate (once per NewCircuitPeriod) + * resets failure counts and expires old circuits. + */ +void +circuit_expire_old_circs_as_needed(time_t now) +{ + static time_t time_to_expire_and_reset = 0; + + if (time_to_expire_and_reset < now) { circuit_reset_failure_count(1); - time_to_new_circuit = now + options->NewCircuitPeriod; + time_to_expire_and_reset = now + get_options()->NewCircuitPeriod; if (proxy_mode(get_options())) addressmap_clean(now); circuit_expire_old_circuits_clientside(); #if 0 /* disable for now, until predict-and-launch-new can cull leftovers */ + + /* If we ever re-enable, this has to move into + * circuit_build_needed_circs */ + circ = circuit_get_youngest_clean_open(CIRCUIT_PURPOSE_C_GENERAL); if (get_options()->RunTesting && circ && @@ -995,8 +1142,6 @@ circuit_build_needed_circs(time_t now) } #endif } - if (!options->DisablePredictedCircuits) - circuit_predict_and_launch_new(); } /** If the stream <b>conn</b> is a member of any of the linked @@ -1083,9 +1228,10 @@ circuit_expire_old_circuits_clientside(void) tor_gettimeofday(&now); cutoff = now; + last_expired_clientside_circuits = now.tv_sec; if (! circuit_build_times_disabled() && - circuit_build_times_needs_circuits(&circ_times)) { + circuit_build_times_needs_circuits(get_circuit_build_times())) { /* Circuits should be shorter lived if we need more of them * for learning a good build timeout */ cutoff.tv_sec -= IDLE_TIMEOUT_WHILE_LEARNING; @@ -1093,7 +1239,7 @@ circuit_expire_old_circuits_clientside(void) cutoff.tv_sec -= get_options()->CircuitIdleTimeout; } - for (circ = global_circuitlist; circ; circ = circ->next) { + TOR_LIST_FOREACH(circ, circuit_get_global_list(), head) { if (circ->marked_for_close || !CIRCUIT_IS_ORIGIN(circ)) continue; /* If the circuit has been dirty for too long, and there are no streams @@ -1176,7 +1322,7 @@ circuit_expire_old_circuits_serverside(time_t now) or_circuit_t *or_circ; time_t cutoff = now - IDLE_ONE_HOP_CIRC_TIMEOUT; - for (circ = global_circuitlist; circ; circ = circ->next) { + TOR_LIST_FOREACH(circ, circuit_get_global_list(), head) { if (circ->marked_for_close || CIRCUIT_IS_ORIGIN(circ)) continue; or_circ = TO_OR_CIRCUIT(circ); @@ -1223,7 +1369,7 @@ circuit_enough_testing_circs(void) if (have_performed_bandwidth_test) return 1; - for (circ = global_circuitlist; circ; circ = circ->next) { + TOR_LIST_FOREACH(circ, circuit_get_global_list(), head) { if (!circ->marked_for_close && CIRCUIT_IS_ORIGIN(circ) && circ->purpose == CIRCUIT_PURPOSE_TESTING && circ->state == CIRCUIT_STATE_OPEN) diff --git a/src/or/circuituse.h b/src/or/circuituse.h index 11e5a64163..4c5977bee0 100644 --- a/src/or/circuituse.h +++ b/src/or/circuituse.h @@ -16,11 +16,13 @@ void circuit_expire_building(void); void circuit_remove_handled_ports(smartlist_t *needed_ports); int circuit_stream_is_being_handled(entry_connection_t *conn, uint16_t port, int min); +void circuit_log_ancient_one_hop_circuits(int age); #if 0 int circuit_conforms_to_options(const origin_circuit_t *circ, const or_options_t *options); #endif void circuit_build_needed_circs(time_t now); +void circuit_expire_old_circs_as_needed(time_t now); void circuit_detach_stream(circuit_t *circ, edge_connection_t *conn); void circuit_expire_old_circuits_serverside(time_t now); diff --git a/src/or/command.c b/src/or/command.c index 78fd4fad33..1f6f93a868 100644 --- a/src/or/command.c +++ b/src/or/command.c @@ -53,6 +53,33 @@ static void command_process_created_cell(cell_t *cell, channel_t *chan); static void command_process_relay_cell(cell_t *cell, channel_t *chan); static void command_process_destroy_cell(cell_t *cell, channel_t *chan); +/** Convert the cell <b>command</b> into a lower-case, human-readable + * string. */ +const char * +cell_command_to_string(uint8_t command) +{ + switch (command) { + case CELL_PADDING: return "padding"; + case CELL_CREATE: return "create"; + case CELL_CREATED: return "created"; + case CELL_RELAY: return "relay"; + case CELL_DESTROY: return "destroy"; + case CELL_CREATE_FAST: return "create_fast"; + case CELL_CREATED_FAST: return "created_fast"; + case CELL_VERSIONS: return "versions"; + case CELL_NETINFO: return "netinfo"; + case CELL_RELAY_EARLY: return "relay_early"; + case CELL_CREATE2: return "create2"; + case CELL_CREATED2: return "created2"; + case CELL_VPADDING: return "vpadding"; + case CELL_CERTS: return "certs"; + case CELL_AUTH_CHALLENGE: return "auth_challenge"; + case CELL_AUTHENTICATE: return "authenticate"; + case CELL_AUTHORIZE: return "authorize"; + default: return "unrecognized"; + } +} + #ifdef KEEP_TIMING_STATS /** This is a wrapper function around the actual function that processes the * <b>cell</b> that just arrived on <b>conn</b>. Increment <b>*time</b> @@ -200,6 +227,34 @@ command_process_create_cell(cell_t *cell, channel_t *chan) (unsigned)cell->circ_id, U64_PRINTF_ARG(chan->global_identifier), chan); + /* We check for the conditions that would make us drop the cell before + * we check for the conditions that would make us send a DESTROY back, + * since those conditions would make a DESTROY nonsensical. */ + if (cell->circ_id == 0) { + log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, + "Received a create cell (type %d) from %s with zero circID; " + " ignoring.", (int)cell->command, + channel_get_actual_remote_descr(chan)); + return; + } + + if (circuit_id_in_use_on_channel(cell->circ_id, chan)) { + const node_t *node = node_get_by_id(chan->identity_digest); + log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, + "Received CREATE cell (circID %u) for known circ. " + "Dropping (age %d).", + (unsigned)cell->circ_id, + (int)(time(NULL) - channel_when_created(chan))); + if (node) { + char *p = esc_for_log(node_get_platform(node)); + log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, + "Details: router %s, platform %s.", + node_describe(node), p); + tor_free(p); + } + return; + } + if (we_are_hibernating()) { log_info(LD_OR, "Received create cell but we're shutting down. Sending back " @@ -221,14 +276,6 @@ command_process_create_cell(cell_t *cell, channel_t *chan) return; } - if (cell->circ_id == 0) { - log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, - "Received a create cell (type %d) from %s with zero circID; " - " ignoring.", (int)cell->command, - channel_get_actual_remote_descr(chan)); - return; - } - /* If the high bit of the circuit ID is not as expected, close the * circ. */ if (chan->wide_circ_ids) @@ -247,23 +294,6 @@ command_process_create_cell(cell_t *cell, channel_t *chan) return; } - if (circuit_id_in_use_on_channel(cell->circ_id, chan)) { - const node_t *node = node_get_by_id(chan->identity_digest); - log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, - "Received CREATE cell (circID %u) for known circ. " - "Dropping (age %d).", - (unsigned)cell->circ_id, - (int)(time(NULL) - channel_when_created(chan))); - if (node) { - char *p = esc_for_log(node_get_platform(node)); - log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, - "Details: router %s, platform %s.", - node_describe(node), p); - tor_free(p); - } - return; - } - circ = or_circuit_new(cell->circ_id, chan); circ->base_.purpose = CIRCUIT_PURPOSE_OR; circuit_set_state(TO_CIRCUIT(circ), CIRCUIT_STATE_ONIONSKIN_PENDING); @@ -349,7 +379,7 @@ command_process_created_cell(cell_t *cell, channel_t *chan) return; } - if (circ->n_circ_id != cell->circ_id) { + if (circ->n_circ_id != cell->circ_id || circ->n_chan != chan) { log_fn(LOG_PROTOCOL_WARN,LD_PROTOCOL, "got created cell from Tor client? Closing."); circuit_mark_for_close(circ, END_CIRC_REASON_TORPROTOCOL); @@ -434,6 +464,7 @@ command_process_relay_cell(cell_t *cell, channel_t *chan) } if (!CIRCUIT_IS_ORIGIN(circ) && + chan == TO_OR_CIRCUIT(circ)->p_chan && cell->circ_id == TO_OR_CIRCUIT(circ)->p_circ_id) direction = CELL_DIRECTION_OUT; else @@ -514,6 +545,7 @@ command_process_destroy_cell(cell_t *cell, channel_t *chan) circ->received_destroy = 1; if (!CIRCUIT_IS_ORIGIN(circ) && + chan == TO_OR_CIRCUIT(circ)->p_chan && cell->circ_id == TO_OR_CIRCUIT(circ)->p_circ_id) { /* the destroy came from behind */ circuit_set_p_circid_chan(TO_OR_CIRCUIT(circ), 0, NULL); diff --git a/src/or/command.h b/src/or/command.h index 913f46a5cd..adea6adeaa 100644 --- a/src/or/command.h +++ b/src/or/command.h @@ -19,6 +19,8 @@ void command_process_var_cell(channel_t *chan, var_cell_t *cell); void command_setup_channel(channel_t *chan); void command_setup_listener(channel_listener_t *chan_l); +const char *cell_command_to_string(uint8_t command); + extern uint64_t stats_n_padding_cells_processed; extern uint64_t stats_n_create_cells_processed; extern uint64_t stats_n_created_cells_processed; diff --git a/src/or/config.c b/src/or/config.c index 1de91878b7..2661ce3b73 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -1,4 +1,4 @@ - /* Copyright (c) 2001 Matej Pfajfar. +/* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. * Copyright (c) 2007-2013, The Tor Project, Inc. */ @@ -10,7 +10,6 @@ **/ #define CONFIG_PRIVATE - #include "or.h" #include "addressmap.h" #include "channel.h" @@ -40,11 +39,14 @@ #include "rendservice.h" #include "rephist.h" #include "router.h" +#include "sandbox.h" #include "util.h" #include "routerlist.h" #include "routerset.h" #include "statefile.h" #include "transports.h" +#include "ext_orport.h" +#include "torgzip.h" #ifdef _WIN32 #include <shlobj.h> #endif @@ -83,6 +85,7 @@ static config_abbrev_t option_abbrevs_[] = { { "DirFetchPostPeriod", "StatusFetchPeriod", 0, 0}, { "DirServer", "DirAuthority", 0, 0}, /* XXXX024 later, make this warn? */ { "MaxConn", "ConnLimit", 0, 1}, + { "MaxMemInCellQueues", "MaxMemInQueues", 0, 0}, { "ORBindAddress", "ORListenAddress", 0, 0}, { "DirBindAddress", "DirListenAddress", 0, 0}, { "SocksBindAddress", "SocksListenAddress", 0, 0}, @@ -135,7 +138,7 @@ static config_var_t option_vars_[] = { V(AllowSingleHopExits, BOOL, "0"), V(AlternateBridgeAuthority, LINELIST, NULL), V(AlternateDirAuthority, LINELIST, NULL), - V(AlternateHSAuthority, LINELIST, NULL), + OBSOLETE("AlternateHSAuthority"), V(AssumeReachable, BOOL, "0"), V(AuthDirBadDir, LINELIST, NULL), V(AuthDirBadDirCCs, CSV, ""), @@ -144,7 +147,7 @@ static config_var_t option_vars_[] = { V(AuthDirInvalid, LINELIST, NULL), V(AuthDirInvalidCCs, CSV, ""), V(AuthDirFastGuarantee, MEMUNIT, "100 KB"), - V(AuthDirGuardBWGuarantee, MEMUNIT, "250 KB"), + V(AuthDirGuardBWGuarantee, MEMUNIT, "2 MB"), V(AuthDirReject, LINELIST, NULL), V(AuthDirRejectCCs, CSV, ""), V(AuthDirRejectUnlisted, BOOL, "0"), @@ -213,11 +216,14 @@ static config_var_t option_vars_[] = { V(DisableAllSwap, BOOL, "0"), V(DisableDebuggerAttachment, BOOL, "1"), V(DisableIOCP, BOOL, "1"), - V(DisableV2DirectoryInfo_, BOOL, "0"), + OBSOLETE("DisableV2DirectoryInfo_"), V(DynamicDHGroups, BOOL, "0"), VPORT(DNSPort, LINELIST, NULL), V(DNSListenAddress, LINELIST, NULL), V(DownloadExtraInfo, BOOL, "0"), + V(TestingEnableConnBwEvent, BOOL, "0"), + V(TestingEnableCellStatsEvent, BOOL, "0"), + V(TestingEnableTbEmptyEvent, BOOL, "0"), V(EnforceDistinctSubnets, BOOL, "1"), V(EntryNodes, ROUTERSET, NULL), V(EntryStatistics, BOOL, "0"), @@ -230,6 +236,8 @@ static config_var_t option_vars_[] = { V(ExitPolicyRejectPrivate, BOOL, "1"), V(ExitPortStatistics, BOOL, "0"), V(ExtendAllowPrivateAddresses, BOOL, "0"), + VPORT(ExtORPort, LINELIST, NULL), + V(ExtORPortCookieAuthFile, STRING, NULL), V(ExtraInfoStatistics, BOOL, "1"), V(FallbackDir, LINELIST, NULL), @@ -242,7 +250,7 @@ static config_var_t option_vars_[] = { V(FetchServerDescriptors, BOOL, "1"), V(FetchHidServDescriptors, BOOL, "1"), V(FetchUselessDescriptors, BOOL, "0"), - V(FetchV2Networkstatus, BOOL, "0"), + OBSOLETE("FetchV2Networkstatus"), V(GeoIPExcludeUnknown, AUTOBOOL, "auto"), #ifdef _WIN32 V(GeoIPFile, FILENAME, "<default>"), @@ -270,7 +278,7 @@ static config_var_t option_vars_[] = { VAR("HiddenServiceVersion",LINELIST_S, RendConfigLines, NULL), VAR("HiddenServiceAuthorizeClient",LINELIST_S,RendConfigLines, NULL), V(HidServAuth, LINELIST, NULL), - V(HSAuthoritativeDir, BOOL, "0"), + OBSOLETE("HSAuthoritativeDir"), OBSOLETE("HSAuthorityRecordStats"), V(CloseHSClientCircuitsImmediatelyOnTimeout, BOOL, "0"), V(CloseHSServiceRendCircuitsImmediatelyOnTimeout, BOOL, "0"), @@ -281,6 +289,7 @@ static config_var_t option_vars_[] = { V(IPv6Exit, BOOL, "0"), VAR("ServerTransportPlugin", LINELIST, ServerTransportPlugin, NULL), V(ServerTransportListenAddr, LINELIST, NULL), + V(ServerTransportOptions, LINELIST, NULL), V(Socks4Proxy, STRING, NULL), V(Socks5Proxy, STRING, NULL), V(Socks5ProxyUsername, STRING, NULL), @@ -299,7 +308,7 @@ static config_var_t option_vars_[] = { V(MaxAdvertisedBandwidth, MEMUNIT, "1 GB"), V(MaxCircuitDirtiness, INTERVAL, "10 minutes"), V(MaxClientCircuitsPending, UINT, "32"), - V(MaxMemInCellQueues, MEMUNIT, "8 GB"), + VAR("MaxMemInQueues", MEMUNIT, MaxMemInQueues_raw, "0"), OBSOLETE("MaxOnionsPending"), V(MaxOnionQueueDelay, MSEC_INTERVAL, "1750 msec"), V(MinMeasuredBWsForAuthToIgnoreAdvertised, INT, "500"), @@ -310,6 +319,7 @@ static config_var_t option_vars_[] = { V(NATDListenAddress, LINELIST, NULL), VPORT(NATDPort, LINELIST, NULL), V(Nickname, STRING, NULL), + V(PredictedPortsRelevanceTime, INTERVAL, "1 hour"), V(WarnUnsafeSocks, BOOL, "1"), OBSOLETE("NoPublish"), VAR("NodeFamily", LINELIST, NodeFamilies, NULL), @@ -347,7 +357,7 @@ static config_var_t option_vars_[] = { V(OptimisticData, AUTOBOOL, "auto"), V(PortForwarding, BOOL, "0"), V(PortForwardingHelper, FILENAME, "tor-fw-helper"), - V(PreferTunneledDirConns, BOOL, "1"), + OBSOLETE("PreferTunneledDirConns"), V(ProtocolWarnings, BOOL, "0"), V(PublishServerDescriptor, CSV, "1"), V(PublishHidServDescriptors, BOOL, "1"), @@ -370,6 +380,7 @@ static config_var_t option_vars_[] = { V(RunAsDaemon, BOOL, "0"), // V(RunTesting, BOOL, "0"), OBSOLETE("RunTesting"), // currently unused + V(Sandbox, BOOL, "0"), V(SafeLogging, STRING, "1"), V(SafeSocks, BOOL, "0"), V(ServerDNSAllowBrokenConfig, BOOL, "1"), @@ -400,21 +411,23 @@ static config_var_t option_vars_[] = { OBSOLETE("TrafficShaping"), V(TransListenAddress, LINELIST, NULL), VPORT(TransPort, LINELIST, NULL), - V(TunnelDirConns, BOOL, "1"), + V(TransProxyType, STRING, "default"), + OBSOLETE("TunnelDirConns"), V(UpdateBridgesFromAuthority, BOOL, "0"), V(UseBridges, BOOL, "0"), V(UseEntryGuards, BOOL, "1"), V(UseEntryGuardsAsDirGuards, BOOL, "1"), V(UseMicrodescriptors, AUTOBOOL, "auto"), - V(UseNTorHandshake, AUTOBOOL, "auto"), + V(UseNTorHandshake, AUTOBOOL, "1"), V(User, STRING, NULL), V(UserspaceIOCPBuffers, BOOL, "0"), - VAR("V1AuthoritativeDirectory",BOOL, V1AuthoritativeDir, "0"), - VAR("V2AuthoritativeDirectory",BOOL, V2AuthoritativeDir, "0"), + OBSOLETE("V1AuthoritativeDirectory"), + OBSOLETE("V2AuthoritativeDirectory"), VAR("V3AuthoritativeDirectory",BOOL, V3AuthoritativeDir, "0"), V(TestingV3AuthInitialVotingInterval, INTERVAL, "30 minutes"), V(TestingV3AuthInitialVoteDelay, INTERVAL, "5 minutes"), V(TestingV3AuthInitialDistDelay, INTERVAL, "5 minutes"), + V(TestingV3AuthVotingStartOffset, INTERVAL, "0"), V(V3AuthVotingInterval, INTERVAL, "1 hour"), V(V3AuthVoteDelay, INTERVAL, "5 minutes"), V(V3AuthDistDelay, INTERVAL, "5 minutes"), @@ -435,6 +448,24 @@ static config_var_t option_vars_[] = { VAR("__OwningControllerProcess",STRING,OwningControllerProcess, NULL), V(MinUptimeHidServDirectoryV2, INTERVAL, "25 hours"), V(VoteOnHidServDirectoriesV2, BOOL, "1"), + V(TestingServerDownloadSchedule, CSV_INTERVAL, "0, 0, 0, 60, 60, 120, " + "300, 900, 2147483647"), + V(TestingClientDownloadSchedule, CSV_INTERVAL, "0, 0, 60, 300, 600, " + "2147483647"), + V(TestingServerConsensusDownloadSchedule, CSV_INTERVAL, "0, 0, 60, " + "300, 600, 1800, 1800, 1800, 1800, " + "1800, 3600, 7200"), + V(TestingClientConsensusDownloadSchedule, CSV_INTERVAL, "0, 0, 60, " + "300, 600, 1800, 3600, 3600, 3600, " + "10800, 21600, 43200"), + V(TestingBridgeDownloadSchedule, CSV_INTERVAL, "3600, 900, 900, 3600"), + V(TestingClientMaxIntervalWithoutRequest, INTERVAL, "10 minutes"), + V(TestingDirConnectionMaxStall, INTERVAL, "5 minutes"), + V(TestingConsensusMaxDownloadTries, UINT, "8"), + V(TestingDescriptorMaxDownloadTries, UINT, "8"), + V(TestingMicrodescMaxDownloadTries, UINT, "8"), + V(TestingCertMaxDownloadTries, UINT, "8"), + V(TestingDirAuthVoteGuard, ROUTERSET, NULL), VAR("___UsingTestNetworkDefaults", BOOL, UsingTestNetworkDefaults_, "0"), { NULL, CONFIG_TYPE_OBSOLETE, 0, NULL } @@ -460,9 +491,28 @@ static const config_var_t testing_tor_network_defaults[] = { V(TestingV3AuthInitialVotingInterval, INTERVAL, "5 minutes"), V(TestingV3AuthInitialVoteDelay, INTERVAL, "20 seconds"), V(TestingV3AuthInitialDistDelay, INTERVAL, "20 seconds"), + V(TestingV3AuthVotingStartOffset, INTERVAL, "0"), V(TestingAuthDirTimeToLearnReachability, INTERVAL, "0 minutes"), V(TestingEstimatedDescriptorPropagationTime, INTERVAL, "0 minutes"), V(MinUptimeHidServDirectoryV2, INTERVAL, "0 minutes"), + V(TestingServerDownloadSchedule, CSV_INTERVAL, "0, 0, 0, 5, 10, 15, " + "20, 30, 60"), + V(TestingClientDownloadSchedule, CSV_INTERVAL, "0, 0, 5, 10, 15, 20, " + "30, 60"), + V(TestingServerConsensusDownloadSchedule, CSV_INTERVAL, "0, 0, 5, 10, " + "15, 20, 30, 60"), + V(TestingClientConsensusDownloadSchedule, CSV_INTERVAL, "0, 0, 5, 10, " + "15, 20, 30, 60"), + V(TestingBridgeDownloadSchedule, CSV_INTERVAL, "60, 30, 30, 60"), + V(TestingClientMaxIntervalWithoutRequest, INTERVAL, "5 seconds"), + V(TestingDirConnectionMaxStall, INTERVAL, "30 seconds"), + V(TestingConsensusMaxDownloadTries, UINT, "80"), + V(TestingDescriptorMaxDownloadTries, UINT, "80"), + V(TestingMicrodescMaxDownloadTries, UINT, "80"), + V(TestingCertMaxDownloadTries, UINT, "80"), + V(TestingEnableConnBwEvent, BOOL, "1"), + V(TestingEnableCellStatsEvent, BOOL, "1"), + V(TestingEnableTbEmptyEvent, BOOL, "1"), VAR("___UsingTestNetworkDefaults", BOOL, UsingTestNetworkDefaults_, "1"), { NULL, CONFIG_TYPE_OBSOLETE, 0, NULL } @@ -475,9 +525,6 @@ static const config_var_t testing_tor_network_defaults[] = { #ifdef _WIN32 static char *get_windows_conf_root(void); #endif -static int options_validate(or_options_t *old_options, - or_options_t *options, - int from_setconf, char **msg); static int options_act_reversible(const or_options_t *old_options, char **msg); static int options_act(const or_options_t *old_options); static int options_transition_allowed(const or_options_t *old, @@ -487,12 +534,13 @@ static int options_transition_affects_workers( const or_options_t *old_options, const or_options_t *new_options); static int options_transition_affects_descriptor( const or_options_t *old_options, const or_options_t *new_options); -static int check_nickname_list(const char *lst, const char *name, char **msg); +static int check_nickname_list(char **lst, const char *name, char **msg); -static int parse_bridge_line(const char *line, int validate_only); -static int parse_client_transport_line(const char *line, int validate_only); +static int parse_client_transport_line(const or_options_t *options, + const char *line, int validate_only); -static int parse_server_transport_line(const char *line, int validate_only); +static int parse_server_transport_line(const or_options_t *options, + const char *line, int validate_only); static char *get_bindaddr_from_transport_listen_line(const char *line, const char *transport); static int parse_dir_authority_line(const char *line, @@ -517,18 +565,23 @@ static int parse_outbound_addresses(or_options_t *options, int validate_only, char **msg); static void config_maybe_load_geoip_files_(const or_options_t *options, const or_options_t *old_options); +static int options_validate_cb(void *old_options, void *options, + void *default_options, + int from_setconf, char **msg); +static uint64_t compute_real_max_mem_in_queues(const uint64_t val, + int log_guess); /** Magic value for or_options_t. */ #define OR_OPTIONS_MAGIC 9090909 /** Configuration format for or_options_t. */ -static config_format_t options_format = { +STATIC config_format_t options_format = { sizeof(or_options_t), OR_OPTIONS_MAGIC, STRUCT_OFFSET(or_options_t, magic_), option_abbrevs_, option_vars_, - (validate_fn_t)options_validate, + options_validate_cb, NULL }; @@ -545,8 +598,12 @@ static or_options_t *global_default_options = NULL; static char *torrc_fname = NULL; /** Name of the most recently read torrc-defaults file.*/ static char *torrc_defaults_fname; -/** Configuration Options set by command line. */ +/** Configuration options set by command line. */ static config_line_t *global_cmdline_options = NULL; +/** Non-configuration options set by the command line */ +static config_line_t *global_cmdline_only_options = NULL; +/** Boolean: Have we parsed the command line? */ +static int have_parsed_cmdline = 0; /** Contents of most recently read DirPortFrontPage file. */ static char *global_dirfrontpagecontents = NULL; /** List of port_cfg_t for all configured ports. */ @@ -568,8 +625,8 @@ get_options_mutable(void) } /** Returns the currently configured options */ -const or_options_t * -get_options(void) +MOCK_IMPL(const or_options_t *, +get_options,(void)) { return get_options_mutable(); } @@ -678,7 +735,7 @@ get_short_version(void) /** Release additional memory allocated in options */ -static void +STATIC void or_options_free(or_options_t *options) { if (!options) @@ -691,6 +748,7 @@ or_options_free(or_options_t *options) smartlist_free(options->NodeFamilySets); } tor_free(options->BridgePassword_AuthDigest_); + tor_free(options->command_arg); config_free(&options_format, options); } @@ -707,6 +765,9 @@ config_free_all(void) config_free_lines(global_cmdline_options); global_cmdline_options = NULL; + config_free_lines(global_cmdline_only_options); + global_cmdline_only_options = NULL; + if (configured_ports) { SMARTLIST_FOREACH(configured_ports, port_cfg_t *, p, port_cfg_free(p)); @@ -787,30 +848,30 @@ add_default_trusted_dir_authorities(dirinfo_type_t type) { int i; const char *authorities[] = { - "moria1 orport=9101 no-v2 " + "moria1 orport=9101 " "v3ident=D586D18309DED4CD6D57C18FDB97EFA96D330566 " "128.31.0.39:9131 9695 DFC3 5FFE B861 329B 9F1A B04C 4639 7020 CE31", - "tor26 v1 orport=443 v3ident=14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4 " + "tor26 orport=443 v3ident=14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4 " "86.59.21.38:80 847B 1F85 0344 D787 6491 A548 92F9 0493 4E4E B85D", "dizum orport=443 v3ident=E8A9C45EDE6D711294FADF8E7951F4DE6CA56B58 " "194.109.206.212:80 7EA6 EAD6 FD83 083C 538F 4403 8BBF A077 587D D755", - "Tonga orport=443 bridge no-v2 82.94.251.203:80 " + "Tonga orport=443 bridge 82.94.251.203:80 " "4A0C CD2D DC79 9508 3D73 F5D6 6710 0C8A 5831 F16D", - "turtles orport=9090 no-v2 " + "turtles orport=9090 " "v3ident=27B6B5996C426270A5C95488AA5BCEB6BCC86956 " "76.73.17.194:9030 F397 038A DC51 3361 35E7 B80B D99C A384 4360 292B", - "gabelmoo orport=443 no-v2 " + "gabelmoo orport=443 " "v3ident=ED03BB616EB2F60BEC80151114BB25CEF515B226 " "212.112.245.170:80 F204 4413 DAC2 E02E 3D6B CF47 35A1 9BCA 1DE9 7281", - "dannenberg orport=443 no-v2 " + "dannenberg orport=443 " "v3ident=585769C78764D58426B8B52B6651A5A71137189A " "193.23.244.244:80 7BE6 83E6 5D48 1413 21C5 ED92 F075 C553 64AC 7123", - "urras orport=80 no-v2 v3ident=80550987E1D626E3EBA5E5E75A458DE0626D088C " + "urras orport=80 v3ident=80550987E1D626E3EBA5E5E75A458DE0626D088C " "208.83.223.34:443 0AD3 FA88 4D18 F89E EA2D 89C0 1937 9E0E 7FD9 4417", - "maatuska orport=80 no-v2 " + "maatuska orport=80 " "v3ident=49015F787433103580E3B66A1707A00E60F2D15B " "171.25.193.9:443 BD6A 8292 55CB 08E6 6FBE 7D37 4836 3586 E46B 3810", - "Faravahar orport=443 no-v2 " + "Faravahar orport=443 " "v3ident=EFCBE720AB3A82B99F9E953CD5BF50F7EEFC7B97 " "154.35.32.5:80 CF6D 0AAF B385 BE71 B8E1 11FC 5CFF 4B47 9237 33BC", NULL @@ -850,8 +911,7 @@ validate_dir_servers(or_options_t *options, or_options_t *old_options) config_line_t *cl; if (options->DirAuthorities && - (options->AlternateDirAuthority || options->AlternateBridgeAuthority || - options->AlternateHSAuthority)) { + (options->AlternateDirAuthority || options->AlternateBridgeAuthority)) { log_warn(LD_CONFIG, "You cannot set both DirAuthority and Alternate*Authority."); return -1; @@ -887,9 +947,6 @@ validate_dir_servers(or_options_t *options, or_options_t *old_options) for (cl = options->AlternateDirAuthority; cl; cl = cl->next) if (parse_dir_authority_line(cl->value, NO_DIRINFO, 1)<0) return -1; - for (cl = options->AlternateHSAuthority; cl; cl = cl->next) - if (parse_dir_authority_line(cl->value, NO_DIRINFO, 1)<0) - return -1; for (cl = options->FallbackDir; cl; cl = cl->next) if (parse_dir_fallback_line(cl->value, 1)<0) return -1; @@ -912,9 +969,7 @@ consider_adding_dir_servers(const or_options_t *options, !config_lines_eq(options->AlternateBridgeAuthority, old_options->AlternateBridgeAuthority) || !config_lines_eq(options->AlternateDirAuthority, - old_options->AlternateDirAuthority) || - !config_lines_eq(options->AlternateHSAuthority, - old_options->AlternateHSAuthority); + old_options->AlternateDirAuthority); if (!need_to_update) return 0; /* all done */ @@ -928,10 +983,7 @@ consider_adding_dir_servers(const or_options_t *options, if (!options->AlternateBridgeAuthority) type |= BRIDGE_DIRINFO; if (!options->AlternateDirAuthority) - type |= V1_DIRINFO | V2_DIRINFO | V3_DIRINFO | EXTRAINFO_DIRINFO | - MICRODESC_DIRINFO; - if (!options->AlternateHSAuthority) - type |= HIDSERV_DIRINFO; + type |= V3_DIRINFO | EXTRAINFO_DIRINFO | MICRODESC_DIRINFO; add_default_trusted_dir_authorities(type); } if (!options->FallbackDir) @@ -946,9 +998,6 @@ consider_adding_dir_servers(const or_options_t *options, for (cl = options->AlternateDirAuthority; cl; cl = cl->next) if (parse_dir_authority_line(cl->value, NO_DIRINFO, 0)<0) return -1; - for (cl = options->AlternateHSAuthority; cl; cl = cl->next) - if (parse_dir_authority_line(cl->value, NO_DIRINFO, 0)<0) - return -1; for (cl = options->FallbackDir; cl; cl = cl->next) if (parse_dir_fallback_line(cl->value, 0)<0) return -1; @@ -972,6 +1021,7 @@ options_act_reversible(const or_options_t *old_options, char **msg) int set_conn_limit = 0; int r = -1; int logs_marked = 0; + int old_min_log_level = get_min_log_level(); /* Daemonize _first_, since we only want to open most of this stuff in * the subprocess. Libevent bases can't be reliably inherited across @@ -998,12 +1048,18 @@ options_act_reversible(const or_options_t *old_options, char **msg) if (running_tor) { int n_ports=0; /* We need to set the connection limit before we can open the listeners. */ - if (set_max_file_descriptors((unsigned)options->ConnLimit, - &options->ConnLimit_) < 0) { - *msg = tor_strdup("Problem with ConnLimit value. See logs for details."); - goto rollback; + if (! sandbox_is_active()) { + if (set_max_file_descriptors((unsigned)options->ConnLimit, + &options->ConnLimit_) < 0) { + *msg = tor_strdup("Problem with ConnLimit value. " + "See logs for details."); + goto rollback; + } + set_conn_limit = 1; + } else { + tor_assert(old_options); + options->ConnLimit_ = old_options->ConnLimit_; } - set_conn_limit = 1; /* Set up libevent. (We need to do this before we can register the * listeners as listeners.) */ @@ -1044,7 +1100,8 @@ options_act_reversible(const or_options_t *old_options, char **msg) #if defined(HAVE_NET_IF_H) && defined(HAVE_NET_PFVAR_H) /* Open /dev/pf before dropping privileges. */ - if (options->TransPort_set) { + if (options->TransPort_set && + options->TransProxyType_parsed == TPT_DEFAULT) { if (get_pf_socket() < 0) { *msg = tor_strdup("Unable to open /dev/pf for transparent proxy."); goto rollback; @@ -1081,23 +1138,6 @@ options_act_reversible(const or_options_t *old_options, char **msg) /* No need to roll back, since you can't change the value. */ } - /* Write control ports to disk as appropriate */ - control_ports_write_to_file(); - - if (directory_caches_v2_dir_info(options)) { - char *fn = NULL; - tor_asprintf(&fn, "%s"PATH_SEPARATOR"cached-status", - options->DataDirectory); - if (check_private_dir(fn, running_tor ? CPD_CREATE : CPD_CHECK, - options->User) < 0) { - tor_asprintf(msg, - "Couldn't access/create private data directory \"%s\"", fn); - tor_free(fn); - goto done; - } - tor_free(fn); - } - /* Bail out at this point if we're not going to be a client or server: * we don't run Tor itself. */ if (!running_tor) @@ -1119,13 +1159,44 @@ options_act_reversible(const or_options_t *old_options, char **msg) add_callback_log(severity, control_event_logmsg); control_adjust_event_log_severity(); tor_free(severity); + tor_log_update_sigsafe_err_fds(); + } + + { + const char *badness = NULL; + int bad_safelog = 0, bad_severity = 0, new_badness = 0; + if (options->SafeLogging_ != SAFELOG_SCRUB_ALL) { + bad_safelog = 1; + if (!old_options || old_options->SafeLogging_ != options->SafeLogging_) + new_badness = 1; + } + if (get_min_log_level() >= LOG_INFO) { + bad_severity = 1; + if (get_min_log_level() != old_min_log_level) + new_badness = 1; + } + if (bad_safelog && bad_severity) + badness = "you disabled SafeLogging, and " + "you're logging more than \"notice\""; + else if (bad_safelog) + badness = "you disabled SafeLogging"; + else + badness = "you're logging more than \"notice\""; + if (new_badness) + log_warn(LD_GENERAL, "Your log may contain sensitive information - %s. " + "Don't log unless it serves an important reason. " + "Overwrite the log afterwards.", badness); } + SMARTLIST_FOREACH(replaced_listeners, connection_t *, conn, { + int marked = conn->marked_for_close; log_notice(LD_NET, "Closing old %s on %s:%d", conn_type_to_string(conn->type), conn->address, conn->port); connection_close_immediate(conn); - connection_mark_for_close(conn); + if (!marked) { + connection_mark_for_close(conn); + } }); goto done; @@ -1272,6 +1343,9 @@ options_act(const or_options_t *old_options) } } + /* Write control ports to disk as appropriate */ + control_ports_write_to_file(); + if (running_tor && !have_lockfile()) { if (try_locking(options, 1) < 0) return -1; @@ -1302,14 +1376,29 @@ options_act(const or_options_t *old_options) } #endif + /* If we are a bridge with a pluggable transport proxy but no + Extended ORPort, inform the user that she is missing out. */ + if (server_mode(options) && options->ServerTransportPlugin && + !options->ExtORPort_lines) { + log_notice(LD_CONFIG, "We use pluggable transports but the Extended " + "ORPort is disabled. Tor and your pluggable transports proxy " + "communicate with each other via the Extended ORPort so it " + "is suggested you enable it: it will also allow your Bridge " + "to collect statistics about its clients that use pluggable " + "transports. Please enable it using the ExtORPort torrc option " + "(e.g. set 'ExtORPort auto')."); + } + if (options->Bridges) { mark_bridge_list(); for (cl = options->Bridges; cl; cl = cl->next) { - if (parse_bridge_line(cl->value, 0)<0) { + bridge_line_t *bridge_line = parse_bridge_line(cl->value); + if (!bridge_line) { log_warn(LD_BUG, "Previously validated Bridge line could not be added!"); return -1; } + bridge_add_from_config(bridge_line); } sweep_bridge_list(); } @@ -1337,7 +1426,7 @@ options_act(const or_options_t *old_options) pt_prepare_proxy_list_for_config_read(); if (options->ClientTransportPlugin) { for (cl = options->ClientTransportPlugin; cl; cl = cl->next) { - if (parse_client_transport_line(cl->value, 0)<0) { + if (parse_client_transport_line(options, cl->value, 0)<0) { log_warn(LD_BUG, "Previously validated ClientTransportPlugin line " "could not be added!"); @@ -1348,7 +1437,7 @@ options_act(const or_options_t *old_options) if (options->ServerTransportPlugin && server_mode(options)) { for (cl = options->ServerTransportPlugin; cl; cl = cl->next) { - if (parse_server_transport_line(cl->value, 0)<0) { + if (parse_server_transport_line(options, cl->value, 0)<0) { log_warn(LD_BUG, "Previously validated ServerTransportPlugin line " "could not be added!"); @@ -1359,6 +1448,12 @@ options_act(const or_options_t *old_options) sweep_transport_list(); sweep_proxy_list(); + /* Start the PT proxy configuration. By doing this configuration + here, we also figure out which proxies need to be restarted and + which not. */ + if (pt_proxies_configuration_pending() && !net_is_disabled()) + pt_configure_remaining_proxies(); + /* Bail out at this point if we're not going to be a client or server: * we want to not fork, and to log stuff to stderr. */ if (!running_tor) @@ -1408,8 +1503,9 @@ options_act(const or_options_t *old_options) /* Write our PID to the PID file. If we do not have write permissions we * will log a warning */ - if (options->PidFile) + if (options->PidFile && !sandbox_is_active()) { write_pidfile(options->PidFile); + } /* Register addressmap directives */ config_register_addressmaps(options); @@ -1423,8 +1519,14 @@ options_act(const or_options_t *old_options) return -1; } - if (init_cookie_authentication(options->CookieAuthentication) < 0) { - log_warn(LD_CONFIG,"Error creating cookie authentication file."); + if (init_control_cookie_authentication(options->CookieAuthentication) < 0) { + log_warn(LD_CONFIG,"Error creating control cookie authentication file."); + return -1; + } + + /* If we have an ExtORPort, initialize its auth cookie. */ + if (init_ext_or_cookie_authentication(!!options->ExtORPort_lines) < 0) { + log_warn(LD_CONFIG,"Error creating Extended ORPort cookie file."); return -1; } @@ -1606,10 +1708,14 @@ options_act(const or_options_t *old_options) time_t now = time(NULL); int print_notice = 0; - /* If we aren't acting as a server, we can't collect stats anyway. */ + /* Only collect directory-request statistics on relays and bridges. */ if (!server_mode(options)) { - options->CellStatistics = 0; options->DirReqStatistics = 0; + } + + /* Only collect other relay-only statistics on relays. */ + if (!public_server_mode(options)) { + options->CellStatistics = 0; options->EntryStatistics = 0; options->ExitPortStatistics = 0; } @@ -1732,40 +1838,66 @@ options_act(const or_options_t *old_options) return 0; } -/** Helper: Read a list of configuration options from the command line. - * If successful, put them in *<b>result</b> and return 0, and return - * -1 and leave *<b>result</b> alone. */ -static int -config_get_commandlines(int argc, char **argv, config_line_t **result) +static const struct { + const char *name; + int takes_argument; +} CMDLINE_ONLY_OPTIONS[] = { + { "-f", 1 }, + { "--allow-missing-torrc", 0 }, + { "--defaults-torrc", 1 }, + { "--hash-password", 1 }, + { "--dump-config", 1 }, + { "--list-fingerprint", 0 }, + { "--verify-config", 0 }, + { "--ignore-missing-torrc", 0 }, + { "--quiet", 0 }, + { "--hush", 0 }, + { "--version", 0 }, + { "--library-versions", 0 }, + { "-h", 0 }, + { "--help", 0 }, + { "--list-torrc-options", 0 }, + { "--digests", 0 }, + { "--nt-service", 0 }, + { "-nt-service", 0 }, + { NULL, 0 }, +}; + +/** Helper: Read a list of configuration options from the command line. If + * successful, or if ignore_errors is set, put them in *<b>result</b>, put the + * commandline-only options in *<b>cmdline_result</b>, and return 0; + * otherwise, return -1 and leave *<b>result</b> and <b>cmdline_result</b> + * alone. */ +int +config_parse_commandline(int argc, char **argv, int ignore_errors, + config_line_t **result, + config_line_t **cmdline_result) { + config_line_t *param = NULL; + config_line_t *front = NULL; config_line_t **new = &front; - char *s; + + config_line_t *front_cmdline = NULL; + config_line_t **new_cmdline = &front_cmdline; + + char *s, *arg; int i = 1; while (i < argc) { unsigned command = CONFIG_LINE_NORMAL; int want_arg = 1; + int is_cmdline = 0; + int j; - if (!strcmp(argv[i],"-f") || - !strcmp(argv[i],"--defaults-torrc") || - !strcmp(argv[i],"--hash-password")) { - i += 2; /* command-line option with argument. ignore them. */ - continue; - } else if (!strcmp(argv[i],"--list-fingerprint") || - !strcmp(argv[i],"--verify-config") || - !strcmp(argv[i],"--ignore-missing-torrc") || - !strcmp(argv[i],"--quiet") || - !strcmp(argv[i],"--hush")) { - i += 1; /* command-line option. ignore it. */ - continue; - } else if (!strcmp(argv[i],"--nt-service") || - !strcmp(argv[i],"-nt-service")) { - i += 1; - continue; + for (j = 0; CMDLINE_ONLY_OPTIONS[j].name != NULL; ++j) { + if (!strcmp(argv[i], CMDLINE_ONLY_OPTIONS[j].name)) { + is_cmdline = 1; + want_arg = CMDLINE_ONLY_OPTIONS[j].takes_argument; + break; + } } - *new = tor_malloc_zero(sizeof(config_line_t)); s = argv[i]; /* Each keyword may be prefixed with one or two dashes. */ @@ -1785,22 +1917,38 @@ config_get_commandlines(int argc, char **argv, config_line_t **result) } if (want_arg && i == argc-1) { - log_warn(LD_CONFIG,"Command-line option '%s' with no value. Failing.", - argv[i]); - config_free_lines(front); - return -1; + if (ignore_errors) { + arg = strdup(""); + } else { + log_warn(LD_CONFIG,"Command-line option '%s' with no value. Failing.", + argv[i]); + config_free_lines(front); + config_free_lines(front_cmdline); + return -1; + } + } else { + arg = want_arg ? tor_strdup(argv[i+1]) : strdup(""); } - (*new)->key = tor_strdup(config_expand_abbrev(&options_format, s, 1, 1)); - (*new)->value = want_arg ? tor_strdup(argv[i+1]) : tor_strdup(""); - (*new)->command = command; - (*new)->next = NULL; + param = tor_malloc_zero(sizeof(config_line_t)); + param->key = is_cmdline ? tor_strdup(argv[i]) : tor_strdup(s); + param->value = arg; + param->command = command; + param->next = NULL; log_debug(LD_CONFIG, "command line: parsed keyword '%s', value '%s'", - (*new)->key, (*new)->value); + param->key, param->value); + + if (is_cmdline) { + *new_cmdline = param; + new_cmdline = &((*new_cmdline)->next); + } else { + *new = param; + new = &((*new)->next); + } - new = &((*new)->next); i += want_arg ? 2 : 1; } + *cmdline_result = front_cmdline; *result = front; return 0; } @@ -1852,7 +2000,8 @@ options_trial_assign(config_line_t *list, int use_defaults, return r; } - if (options_validate(get_options_mutable(), trial_options, 1, msg) < 0) { + if (options_validate(get_options_mutable(), trial_options, + global_default_options, 1, msg) < 0) { config_free(&options_format, trial_options); return SETOPT_ERR_PARSE; /*XXX make this a separate return value. */ } @@ -1945,6 +2094,7 @@ resolve_my_address(int warn_severity, const or_options_t *options, int notice_severity = warn_severity <= LOG_NOTICE ? LOG_NOTICE : warn_severity; + tor_addr_t myaddr; tor_assert(addr_out); /* @@ -1995,24 +2145,26 @@ resolve_my_address(int warn_severity, const or_options_t *options, "local interface. Using that.", fmt_addr32(addr)); strlcpy(hostname, "<guessed from interfaces>", sizeof(hostname)); } else { /* resolved hostname into addr */ + tor_addr_from_ipv4h(&myaddr, addr); + if (!explicit_hostname && - is_internal_IP(addr, 0)) { - uint32_t interface_ip; + tor_addr_is_internal(&myaddr, 0)) { + tor_addr_t interface_ip; log_fn(notice_severity, LD_CONFIG, "Guessed local hostname '%s' " "resolves to a private IP address (%s). Trying something " "else.", hostname, fmt_addr32(addr)); - if (get_interface_address(warn_severity, &interface_ip)) { + if (get_interface_address6(warn_severity, AF_INET, &interface_ip)<0) { log_fn(warn_severity, LD_CONFIG, "Could not get local interface IP address. Too bad."); - } else if (is_internal_IP(interface_ip, 0)) { + } else if (tor_addr_is_internal(&interface_ip, 0)) { log_fn(notice_severity, LD_CONFIG, "Interface IP address '%s' is a private address too. " - "Ignoring.", fmt_addr32(interface_ip)); + "Ignoring.", fmt_addr(&interface_ip)); } else { from_interface = 1; - addr = interface_ip; + addr = tor_addr_to_ipv4h(&interface_ip); log_fn(notice_severity, LD_CONFIG, "Learned IP address '%s' for local interface." " Using that.", fmt_addr32(addr)); @@ -2030,8 +2182,10 @@ resolve_my_address(int warn_severity, const or_options_t *options, * out if it is and we don't want that. */ + tor_addr_from_ipv4h(&myaddr,addr); + addr_string = tor_dup_ip(addr); - if (is_internal_IP(addr, 0)) { + if (tor_addr_is_internal(&myaddr, 0)) { /* make sure we're ok with publishing an internal IP */ if (!options->DirAuthorities && !options->AlternateDirAuthority) { /* if they are using the default authorities, disallow internal IPs @@ -2137,7 +2291,7 @@ is_local_addr(const tor_addr_t *addr) * resolve_my_address will never be called at all). In those cases, * last_resolved_addr will be 0, and so checking to see whether ip is on * the same /24 as last_resolved_addr will be the same as checking whether - * it was on net 0, which is already done by is_internal_IP. + * it was on net 0, which is already done by tor_addr_is_internal. */ if ((last_resolved_addr & (uint32_t)0xffffff00ul) == (ip & (uint32_t)0xffffff00ul)) @@ -2166,10 +2320,29 @@ options_init(or_options_t *options) * include options that are the same as Tor's defaults. */ char * -options_dump(const or_options_t *options, int minimal) +options_dump(const or_options_t *options, int how_to_dump) { - return config_dump(&options_format, global_default_options, - options, minimal, 0); + const or_options_t *use_defaults; + int minimal; + switch (how_to_dump) { + case OPTIONS_DUMP_MINIMAL: + use_defaults = global_default_options; + minimal = 1; + break; + case OPTIONS_DUMP_DEFAULTS: + use_defaults = NULL; + minimal = 1; + break; + case OPTIONS_DUMP_ALL: + use_defaults = NULL; + minimal = 0; + break; + default: + log_warn(LD_BUG, "Bogus value for how_to_dump==%d", how_to_dump); + return NULL; + } + + return config_dump(&options_format, use_defaults, options, minimal, 0); } /** Return 0 if every element of sl is a string holding a decimal @@ -2218,7 +2391,7 @@ ensure_bandwidth_cap(uint64_t *value, const char *desc, char **msg) /** Parse an authority type from <b>options</b>-\>PublishServerDescriptor * and write it to <b>options</b>-\>PublishServerDescriptor_. Treat "1" - * as "v2,v3" unless BridgeRelay is 1, in which case treat it as "bridge". + * as "v3" unless BridgeRelay is 1, in which case treat it as "bridge". * Treat "0" as "". * Return 0 on success or -1 if not a recognized authority type (in which * case the value of PublishServerDescriptor_ is undefined). */ @@ -2232,14 +2405,16 @@ compute_publishserverdescriptor(or_options_t *options) return 0; SMARTLIST_FOREACH_BEGIN(list, const char *, string) { if (!strcasecmp(string, "v1")) - *auth |= V1_DIRINFO; + log_warn(LD_CONFIG, "PublishServerDescriptor v1 has no effect, because " + "there are no v1 directory authorities anymore."); else if (!strcmp(string, "1")) if (options->BridgeRelay) *auth |= BRIDGE_DIRINFO; else - *auth |= V2_DIRINFO | V3_DIRINFO; + *auth |= V3_DIRINFO; else if (!strcasecmp(string, "v2")) - *auth |= V2_DIRINFO; + log_warn(LD_CONFIG, "PublishServerDescriptor v2 has no effect, because " + "there are no v2 directory authorities anymore."); else if (!strcasecmp(string, "v3")) *auth |= V3_DIRINFO; else if (!strcasecmp(string, "bridge")) @@ -2260,6 +2435,11 @@ compute_publishserverdescriptor(or_options_t *options) * services can overload the directory system. */ #define MIN_REND_POST_PERIOD (10*60) +/** Higest allowable value for PredictedPortsRelevanceTime; if this is + * too high, our selection of exits will decrease for an extended + * period of time to an uncomfortable level .*/ +#define MAX_PREDICTED_CIRCS_RELEVANCE (60*60) + /** Highest allowable value for RendPostPeriod. */ #define MAX_DIR_PERIOD (MIN_ONION_KEY_LIFETIME/2) @@ -2286,10 +2466,19 @@ compute_publishserverdescriptor(or_options_t *options) * */ #define RECOMMENDED_MIN_CIRCUIT_BUILD_TIMEOUT (10) -/** Return 0 if every setting in <b>options</b> is reasonable, and a - * permissible transition from <b>old_options</b>. Else return -1. - * Should have no side effects, except for normalizing the contents of - * <b>options</b>. +static int +options_validate_cb(void *old_options, void *options, void *default_options, + int from_setconf, char **msg) +{ + return options_validate(old_options, options, default_options, + from_setconf, msg); +} + +/** Return 0 if every setting in <b>options</b> is reasonable, is a + * permissible transition from <b>old_options</b>, and none of the + * testing-only settings differ from <b>default_options</b> unless in + * testing mode. Else return -1. Should have no side effects, except for + * normalizing the contents of <b>options</b>. * * On error, tor_strdup an error explanation into *<b>msg</b>. * @@ -2298,9 +2487,9 @@ compute_publishserverdescriptor(or_options_t *options) * Log line should stay empty. If it's 0, then give us a default log * if there are no logs defined. */ -static int +STATIC int options_validate(or_options_t *old_options, or_options_t *options, - int from_setconf, char **msg) + or_options_t *default_options, int from_setconf, char **msg) { int i; config_line_t *cl; @@ -2381,10 +2570,43 @@ options_validate(or_options_t *old_options, or_options_t *options, "undefined, and there aren't any hidden services configured. " "Tor will still run, but probably won't do anything."); -#ifndef USE_TRANSPARENT - /* XXXX024 I think we can remove this TransListenAddress */ - if (options->TransPort_set || options->TransListenAddress) - REJECT("TransPort and TransListenAddress are disabled in this build."); + options->TransProxyType_parsed = TPT_DEFAULT; +#ifdef USE_TRANSPARENT + if (options->TransProxyType) { + if (!strcasecmp(options->TransProxyType, "default")) { + options->TransProxyType_parsed = TPT_DEFAULT; + } else if (!strcasecmp(options->TransProxyType, "pf-divert")) { +#ifndef __OpenBSD__ + REJECT("pf-divert is a OpenBSD-specific feature."); +#else + options->TransProxyType_parsed = TPT_PF_DIVERT; +#endif + } else if (!strcasecmp(options->TransProxyType, "tproxy")) { +#ifndef __linux__ + REJECT("TPROXY is a Linux-specific feature."); +#else + options->TransProxyType_parsed = TPT_TPROXY; +#endif + } else if (!strcasecmp(options->TransProxyType, "ipfw")) { +#ifndef __FreeBSD__ + REJECT("ipfw is a FreeBSD-specific feature."); +#else + options->TransProxyType_parsed = TPT_IPFW; +#endif + } else { + REJECT("Unrecognized value for TransProxyType"); + } + + if (strcasecmp(options->TransProxyType, "default") && + !options->TransPort_set) { + REJECT("Cannot use TransProxyType without any valid TransPort or " + "TransListenAddress."); + } + } +#else + if (options->TransPort_set) + REJECT("TransPort and TransListenAddress are disabled " + "in this build."); #endif if (options->TokenBucketRefillInterval <= 0 @@ -2392,10 +2614,6 @@ options_validate(or_options_t *old_options, or_options_t *options, REJECT("TokenBucketRefillInterval must be between 1 and 1000 inclusive."); } - if (options->DisableV2DirectoryInfo_ && ! authdir_mode(options)) { - REJECT("DisableV2DirectoryInfo_ set, but we aren't an authority."); - } - if (options->ExcludeExitNodes || options->ExcludeNodes) { options->ExcludeExitNodesUnion_ = routerset_new(); routerset_union(options->ExcludeExitNodesUnion_,options->ExcludeExitNodes); @@ -2429,8 +2647,6 @@ options_validate(or_options_t *old_options, or_options_t *options, if (options->AuthoritativeDir) { if (!options->ContactInfo && !options->TestingTorNetwork) REJECT("Authoritative directory servers must set ContactInfo"); - if (options->V1AuthoritativeDir && !options->RecommendedVersions) - REJECT("V1 authoritative dir servers must set RecommendedVersions."); if (!options->RecommendedClientVersions) options->RecommendedClientVersions = config_lines_dup(options->RecommendedVersions); @@ -2452,11 +2668,10 @@ options_validate(or_options_t *old_options, or_options_t *options, "extra-info documents. Setting DownloadExtraInfo."); options->DownloadExtraInfo = 1; } - if (!(options->BridgeAuthoritativeDir || options->HSAuthoritativeDir || - options->V1AuthoritativeDir || options->V2AuthoritativeDir || + if (!(options->BridgeAuthoritativeDir || options->V3AuthoritativeDir)) REJECT("AuthoritativeDir is set, but none of " - "(Bridge/HS/V1/V2/V3)AuthoritativeDir is set."); + "(Bridge/V3)AuthoritativeDir is set."); /* If we have a v3bandwidthsfile and it's broken, complain on startup */ if (options->V3BandwidthsFile && !old_options) { dirserv_read_measured_bandwidths(options->V3BandwidthsFile, NULL); @@ -2476,10 +2691,6 @@ options_validate(or_options_t *old_options, or_options_t *options, REJECT("FetchDirInfoExtraEarly requires that you also set " "FetchDirInfoEarly"); - if (options->HSAuthoritativeDir && proxy_mode(options)) - REJECT("Running as authoritative v0 HS directory, but also configured " - "as a client."); - if (options->ConnLimit <= 0) { tor_asprintf(msg, "ConnLimit must be greater than 0, but was set to %d", @@ -2616,11 +2827,9 @@ options_validate(or_options_t *old_options, or_options_t *options, REJECT("If EntryNodes is set, UseEntryGuards must be enabled."); } - if (options->MaxMemInCellQueues < (256 << 20)) { - log_warn(LD_CONFIG, "MaxMemInCellQueues must be at least 256 MB for now. " - "Ideally, have it as large as you can afford."); - options->MaxMemInCellQueues = (256 << 20); - } + options->MaxMemInQueues = + compute_real_max_mem_in_queues(options->MaxMemInQueues_raw, + server_mode(options)); options->AllowInvalid_ = 0; @@ -2665,8 +2874,7 @@ options_validate(or_options_t *old_options, or_options_t *options, if ((options->BridgeRelay || options->PublishServerDescriptor_ & BRIDGE_DIRINFO) - && (options->PublishServerDescriptor_ - & (V1_DIRINFO|V2_DIRINFO|V3_DIRINFO))) { + && (options->PublishServerDescriptor_ & V3_DIRINFO)) { REJECT("Bridges are not supposed to publish router descriptors to the " "directory authorities. Please correct your " "PublishServerDescriptor line."); @@ -2698,6 +2906,13 @@ options_validate(or_options_t *old_options, or_options_t *options, options->RendPostPeriod = MAX_DIR_PERIOD; } + if (options->PredictedPortsRelevanceTime > + MAX_PREDICTED_CIRCS_RELEVANCE) { + log_warn(LD_CONFIG, "PredictedPortsRelevanceTime is too large; " + "clipping to %ds.", MAX_PREDICTED_CIRCS_RELEVANCE); + options->PredictedPortsRelevanceTime = MAX_PREDICTED_CIRCS_RELEVANCE; + } + if (options->Tor2webMode && options->LearnCircuitBuildTimeout) { /* LearnCircuitBuildTimeout and Tor2webMode are incompatible in * two ways: @@ -2814,6 +3029,11 @@ options_validate(or_options_t *old_options, or_options_t *options, if (options->KeepalivePeriod < 1) REJECT("KeepalivePeriod option must be positive."); + if (options->PortForwarding && options->Sandbox) { + REJECT("PortForwarding is not compatible with Sandbox; at most one can " + "be set"); + } + if (ensure_bandwidth_cap(&options->BandwidthRate, "BandwidthRate", msg) < 0) return -1; @@ -2973,14 +3193,14 @@ options_validate(or_options_t *old_options, or_options_t *options, size_t len; len = strlen(options->Socks5ProxyUsername); - if (len < 1 || len > 255) + if (len < 1 || len > MAX_SOCKS5_AUTH_FIELD_SIZE) REJECT("Socks5ProxyUsername must be between 1 and 255 characters."); if (!options->Socks5ProxyPassword) REJECT("Socks5ProxyPassword must be included with Socks5ProxyUsername."); len = strlen(options->Socks5ProxyPassword); - if (len < 1 || len > 255) + if (len < 1 || len > MAX_SOCKS5_AUTH_FIELD_SIZE) REJECT("Socks5ProxyPassword must be between 1 and 255 characters."); } else if (options->Socks5ProxyPassword) REJECT("Socks5ProxyPassword must be included with Socks5ProxyUsername."); @@ -3037,7 +3257,7 @@ options_validate(or_options_t *old_options, or_options_t *options, "You should also make sure you aren't listing this bridge's " "fingerprint in any other MyFamily."); } - if (check_nickname_list(options->MyFamily, "MyFamily", msg)) + if (check_nickname_list(&options->MyFamily, "MyFamily", msg)) return -1; for (cl = options->NodeFamilies; cl; cl = cl->next) { routerset_t *rs = routerset_new(); @@ -3057,26 +3277,22 @@ options_validate(or_options_t *old_options, or_options_t *options, if (options->UseBridges && !options->Bridges) REJECT("If you set UseBridges, you must specify at least one bridge."); - if (options->UseBridges && !options->TunnelDirConns) - REJECT("If you set UseBridges, you must set TunnelDirConns."); - if (options->RendConfigLines && - (!options->TunnelDirConns || !options->PreferTunneledDirConns)) - REJECT("If you are running a hidden service, you must set TunnelDirConns " - "and PreferTunneledDirConns"); for (cl = options->Bridges; cl; cl = cl->next) { - if (parse_bridge_line(cl->value, 1)<0) - REJECT("Bridge line did not parse. See logs for details."); + bridge_line_t *bridge_line = parse_bridge_line(cl->value); + if (!bridge_line) + REJECT("Bridge line did not parse. See logs for details."); + bridge_line_free(bridge_line); } for (cl = options->ClientTransportPlugin; cl; cl = cl->next) { - if (parse_client_transport_line(cl->value, 1)<0) - REJECT("Transport line did not parse. See logs for details."); + if (parse_client_transport_line(options, cl->value, 1)<0) + REJECT("Invalid client transport line. See logs for details."); } for (cl = options->ServerTransportPlugin; cl; cl = cl->next) { - if (parse_server_transport_line(cl->value, 1)<0) - REJECT("Server transport line did not parse. See logs for details."); + if (parse_server_transport_line(options, cl->value, 1)<0) + REJECT("Invalid server transport line. See logs for details."); } if (options->ServerTransportPlugin && !server_mode(options)) { @@ -3102,6 +3318,19 @@ options_validate(or_options_t *old_options, or_options_t *options, "ServerTransportListenAddr line will be ignored."); } + for (cl = options->ServerTransportOptions; cl; cl = cl->next) { + /** If get_options_from_transport_options_line() fails with + 'transport' being NULL, it means that something went wrong + while parsing the ServerTransportOptions line. */ + smartlist_t *options_sl = + get_options_from_transport_options_line(cl->value, NULL); + if (!options_sl) + REJECT("ServerTransportOptions did not parse. See logs for details."); + + SMARTLIST_FOREACH(options_sl, char *, cp, tor_free(cp)); + smartlist_free(options_sl); + } + if (options->ConstrainedSockets) { /* If the user wants to constrain socket buffer use, make sure the desired * limit is between MIN|MAX_TCPSOCK_BUFFER in k increments. */ @@ -3160,15 +3389,6 @@ options_validate(or_options_t *old_options, or_options_t *options, AF_INET6, 1, msg)<0) return -1; - if (options->PreferTunneledDirConns && !options->TunnelDirConns) - REJECT("Must set TunnelDirConns if PreferTunneledDirConns is set."); - - if ((options->Socks4Proxy || options->Socks5Proxy) && - !options->HTTPProxy && !options->PreferTunneledDirConns) - REJECT("When Socks4Proxy or Socks5Proxy is configured, " - "PreferTunneledDirConns and TunnelDirConns must both be " - "set to 1, or HTTPProxy must be configured."); - if (options->AutomapHostsSuffixes) { SMARTLIST_FOREACH(options->AutomapHostsSuffixes, char *, suf, { @@ -3194,35 +3414,46 @@ options_validate(or_options_t *old_options, or_options_t *options, "ignore you."); } - /*XXXX checking for defaults manually like this is a bit fragile.*/ - - /* Keep changes to hard-coded values synchronous to man page and default - * values table. */ - if (options->TestingV3AuthInitialVotingInterval != 30*60 && - !options->TestingTorNetwork && !options->UsingTestNetworkDefaults_) { - REJECT("TestingV3AuthInitialVotingInterval may only be changed in testing " - "Tor networks!"); - } else if (options->TestingV3AuthInitialVotingInterval < MIN_VOTE_INTERVAL) { +#define CHECK_DEFAULT(arg) \ + STMT_BEGIN \ + if (!options->TestingTorNetwork && \ + !options->UsingTestNetworkDefaults_ && \ + !config_is_same(&options_format,options, \ + default_options,#arg)) { \ + REJECT(#arg " may only be changed in testing Tor " \ + "networks!"); \ + } STMT_END + CHECK_DEFAULT(TestingV3AuthInitialVotingInterval); + CHECK_DEFAULT(TestingV3AuthInitialVoteDelay); + CHECK_DEFAULT(TestingV3AuthInitialDistDelay); + CHECK_DEFAULT(TestingV3AuthVotingStartOffset); + CHECK_DEFAULT(TestingAuthDirTimeToLearnReachability); + CHECK_DEFAULT(TestingEstimatedDescriptorPropagationTime); + CHECK_DEFAULT(TestingServerDownloadSchedule); + CHECK_DEFAULT(TestingClientDownloadSchedule); + CHECK_DEFAULT(TestingServerConsensusDownloadSchedule); + CHECK_DEFAULT(TestingClientConsensusDownloadSchedule); + CHECK_DEFAULT(TestingBridgeDownloadSchedule); + CHECK_DEFAULT(TestingClientMaxIntervalWithoutRequest); + CHECK_DEFAULT(TestingDirConnectionMaxStall); + CHECK_DEFAULT(TestingConsensusMaxDownloadTries); + CHECK_DEFAULT(TestingDescriptorMaxDownloadTries); + CHECK_DEFAULT(TestingMicrodescMaxDownloadTries); + CHECK_DEFAULT(TestingCertMaxDownloadTries); +#undef CHECK_DEFAULT + + if (options->TestingV3AuthInitialVotingInterval < MIN_VOTE_INTERVAL) { REJECT("TestingV3AuthInitialVotingInterval is insanely low."); } else if (((30*60) % options->TestingV3AuthInitialVotingInterval) != 0) { REJECT("TestingV3AuthInitialVotingInterval does not divide evenly into " "30 minutes."); } - if (options->TestingV3AuthInitialVoteDelay != 5*60 && - !options->TestingTorNetwork && !options->UsingTestNetworkDefaults_) { - - REJECT("TestingV3AuthInitialVoteDelay may only be changed in testing " - "Tor networks!"); - } else if (options->TestingV3AuthInitialVoteDelay < MIN_VOTE_SECONDS) { + if (options->TestingV3AuthInitialVoteDelay < MIN_VOTE_SECONDS) { REJECT("TestingV3AuthInitialVoteDelay is way too low."); } - if (options->TestingV3AuthInitialDistDelay != 5*60 && - !options->TestingTorNetwork && !options->UsingTestNetworkDefaults_) { - REJECT("TestingV3AuthInitialDistDelay may only be changed in testing " - "Tor networks!"); - } else if (options->TestingV3AuthInitialDistDelay < MIN_DIST_SECONDS) { + if (options->TestingV3AuthInitialDistDelay < MIN_DIST_SECONDS) { REJECT("TestingV3AuthInitialDistDelay is way too low."); } @@ -3233,26 +3464,79 @@ options_validate(or_options_t *old_options, or_options_t *options, "must be less than half TestingV3AuthInitialVotingInterval"); } - if (options->TestingAuthDirTimeToLearnReachability != 30*60 && - !options->TestingTorNetwork && !options->UsingTestNetworkDefaults_) { - REJECT("TestingAuthDirTimeToLearnReachability may only be changed in " - "testing Tor networks!"); - } else if (options->TestingAuthDirTimeToLearnReachability < 0) { + if (options->TestingV3AuthVotingStartOffset > + MIN(options->TestingV3AuthInitialVotingInterval, + options->V3AuthVotingInterval)) { + REJECT("TestingV3AuthVotingStartOffset is higher than the voting " + "interval."); + } + + if (options->TestingAuthDirTimeToLearnReachability < 0) { REJECT("TestingAuthDirTimeToLearnReachability must be non-negative."); } else if (options->TestingAuthDirTimeToLearnReachability > 2*60*60) { COMPLAIN("TestingAuthDirTimeToLearnReachability is insanely high."); } - if (options->TestingEstimatedDescriptorPropagationTime != 10*60 && - !options->TestingTorNetwork && !options->UsingTestNetworkDefaults_) { - REJECT("TestingEstimatedDescriptorPropagationTime may only be changed in " - "testing Tor networks!"); - } else if (options->TestingEstimatedDescriptorPropagationTime < 0) { + if (options->TestingEstimatedDescriptorPropagationTime < 0) { REJECT("TestingEstimatedDescriptorPropagationTime must be non-negative."); } else if (options->TestingEstimatedDescriptorPropagationTime > 60*60) { COMPLAIN("TestingEstimatedDescriptorPropagationTime is insanely high."); } + if (options->TestingClientMaxIntervalWithoutRequest < 1) { + REJECT("TestingClientMaxIntervalWithoutRequest is way too low."); + } else if (options->TestingClientMaxIntervalWithoutRequest > 3600) { + COMPLAIN("TestingClientMaxIntervalWithoutRequest is insanely high."); + } + + if (options->TestingDirConnectionMaxStall < 5) { + REJECT("TestingDirConnectionMaxStall is way too low."); + } else if (options->TestingDirConnectionMaxStall > 3600) { + COMPLAIN("TestingDirConnectionMaxStall is insanely high."); + } + + if (options->TestingConsensusMaxDownloadTries < 2) { + REJECT("TestingConsensusMaxDownloadTries must be greater than 1."); + } else if (options->TestingConsensusMaxDownloadTries > 800) { + COMPLAIN("TestingConsensusMaxDownloadTries is insanely high."); + } + + if (options->TestingDescriptorMaxDownloadTries < 2) { + REJECT("TestingDescriptorMaxDownloadTries must be greater than 1."); + } else if (options->TestingDescriptorMaxDownloadTries > 800) { + COMPLAIN("TestingDescriptorMaxDownloadTries is insanely high."); + } + + if (options->TestingMicrodescMaxDownloadTries < 2) { + REJECT("TestingMicrodescMaxDownloadTries must be greater than 1."); + } else if (options->TestingMicrodescMaxDownloadTries > 800) { + COMPLAIN("TestingMicrodescMaxDownloadTries is insanely high."); + } + + if (options->TestingCertMaxDownloadTries < 2) { + REJECT("TestingCertMaxDownloadTries must be greater than 1."); + } else if (options->TestingCertMaxDownloadTries > 800) { + COMPLAIN("TestingCertMaxDownloadTries is insanely high."); + } + + if (options->TestingEnableConnBwEvent && + !options->TestingTorNetwork && !options->UsingTestNetworkDefaults_) { + REJECT("TestingEnableConnBwEvent may only be changed in testing " + "Tor networks!"); + } + + if (options->TestingEnableCellStatsEvent && + !options->TestingTorNetwork && !options->UsingTestNetworkDefaults_) { + REJECT("TestingEnableCellStatsEvent may only be changed in testing " + "Tor networks!"); + } + + if (options->TestingEnableTbEmptyEvent && + !options->TestingTorNetwork && !options->UsingTestNetworkDefaults_) { + REJECT("TestingEnableTbEmptyEvent may only be changed in testing " + "Tor networks!"); + } + if (options->TestingTorNetwork) { log_warn(LD_CONFIG, "TestingTorNetwork is set. This will make your node " "almost unusable in the public Tor network, and is " @@ -3286,6 +3570,68 @@ options_validate(or_options_t *old_options, or_options_t *options, #undef COMPLAIN } +/* Given the value that the user has set for MaxMemInQueues, compute the + * actual maximum value. We clip this value if it's too low, and autodetect + * it if it's set to 0. */ +static uint64_t +compute_real_max_mem_in_queues(const uint64_t val, int log_guess) +{ + uint64_t result; + + if (val == 0) { +#define ONE_GIGABYTE (U64_LITERAL(1) << 30) +#define ONE_MEGABYTE (U64_LITERAL(1) << 20) +#if SIZEOF_VOID_P >= 8 +#define MAX_DEFAULT_MAXMEM (8*ONE_GIGABYTE) +#else +#define MAX_DEFAULT_MAXMEM (2*ONE_GIGABYTE) +#endif + /* The user didn't pick a memory limit. Choose a very large one + * that is still smaller than the system memory */ + static int notice_sent = 0; + size_t ram = 0; + if (get_total_system_memory(&ram) < 0) { + /* We couldn't determine our total system memory! */ +#if SIZEOF_VOID_P >= 8 + /* 64-bit system. Let's hope for 8 GB. */ + result = 8 * ONE_GIGABYTE; +#else + /* (presumably) 32-bit system. Let's hope for 1 GB. */ + result = ONE_GIGABYTE; +#endif + } else { + /* We detected it, so let's pick 3/4 of the total RAM as our limit. */ + const uint64_t avail = (ram / 4) * 3; + + /* Make sure it's in range from 0.25 GB to 8 GB. */ + if (avail > MAX_DEFAULT_MAXMEM) { + /* If you want to use more than this much RAM, you need to configure + it yourself */ + result = MAX_DEFAULT_MAXMEM; + } else if (avail < ONE_GIGABYTE / 4) { + result = ONE_GIGABYTE / 4; + } else { + result = avail; + } + } + if (log_guess && ! notice_sent) { + log_notice(LD_CONFIG, "%sMaxMemInQueues is set to "U64_FORMAT" MB. " + "You can override this by setting MaxMemInQueues by hand.", + ram ? "Based on detected system memory, " : "", + U64_PRINTF_ARG(result / ONE_MEGABYTE)); + notice_sent = 1; + } + return result; + } else if (val < ONE_GIGABYTE / 4) { + log_warn(LD_CONFIG, "MaxMemInQueues must be at least 256 MB for now. " + "Ideally, have it as large as you can afford."); + return ONE_GIGABYTE / 4; + } else { + /* The value was fine all along */ + return val; + } +} + /** Helper: return true iff s1 and s2 are both NULL, or both non-NULL * equal strings. */ static int @@ -3314,6 +3660,12 @@ options_transition_allowed(const or_options_t *old, return -1; } + if (old->Sandbox != new_val->Sandbox) { + *msg = tor_strdup("While Tor is running, changing Sandbox " + "is not allowed."); + return -1; + } + if (strcmp(old->DataDirectory,new_val->DataDirectory)!=0) { tor_asprintf(msg, "While Tor is running, changing DataDirectory " @@ -3366,6 +3718,38 @@ options_transition_allowed(const or_options_t *old, return -1; } + if (sandbox_is_active()) { +#define SB_NOCHANGE_STR(opt) \ + do { \ + if (! opt_streq(old->opt, new_val->opt)) { \ + *msg = tor_strdup("Can't change " #opt " while Sandbox is active"); \ + return -1; \ + } \ + } while (0) + + SB_NOCHANGE_STR(PidFile); + SB_NOCHANGE_STR(ServerDNSResolvConfFile); + SB_NOCHANGE_STR(DirPortFrontPage); + SB_NOCHANGE_STR(CookieAuthFile); + SB_NOCHANGE_STR(ExtORPortCookieAuthFile); + +#undef SB_NOCHANGE_STR + + if (! config_lines_eq(old->Logs, new_val->Logs)) { + *msg = tor_strdup("Can't change Logs while Sandbox is active"); + return -1; + } + if (old->ConnLimit != new_val->ConnLimit) { + *msg = tor_strdup("Can't change ConnLimit while Sandbox is active"); + return -1; + } + if (server_mode(old) != server_mode(new_val)) { + *msg = tor_strdup("Can't start/stop being a server while " + "Sandbox is active"); + return -1; + } + } + return 0; } @@ -3511,31 +3895,63 @@ get_default_conf_file(int defaults_file) } /** Verify whether lst is a string containing valid-looking comma-separated - * nicknames, or NULL. Return 0 on success. Warn and return -1 on failure. + * nicknames, or NULL. Will normalise <b>lst</b> to prefix '$' to any nickname + * or fingerprint that needs it. Return 0 on success. + * Warn and return -1 on failure. */ static int -check_nickname_list(const char *lst, const char *name, char **msg) +check_nickname_list(char **lst, const char *name, char **msg) { int r = 0; smartlist_t *sl; + int changes = 0; - if (!lst) + if (!*lst) return 0; sl = smartlist_new(); - smartlist_split_string(sl, lst, ",", + smartlist_split_string(sl, *lst, ",", SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK|SPLIT_STRIP_SPACE, 0); - SMARTLIST_FOREACH(sl, const char *, s, + SMARTLIST_FOREACH_BEGIN(sl, char *, s) { if (!is_legal_nickname_or_hexdigest(s)) { + // check if first char is dollar + if (s[0] != '$') { + // Try again but with a dollar symbol prepended + char *prepended; + tor_asprintf(&prepended, "$%s", s); + + if (is_legal_nickname_or_hexdigest(prepended)) { + // The nickname is valid when it's prepended, swap the current + // version with a prepended one + tor_free(s); + SMARTLIST_REPLACE_CURRENT(sl, s, prepended); + changes = 1; + continue; + } + + // Still not valid, free and fallback to error message + tor_free(prepended); + } + tor_asprintf(msg, "Invalid nickname '%s' in %s line", s, name); r = -1; break; } - }); + } + SMARTLIST_FOREACH_END(s); + + // Replace the caller's nickname list with a fixed one + if (changes && r == 0) { + char *newNicknames = smartlist_join_strings(sl, ", ", 0, NULL); + tor_free(*lst); + *lst = newNicknames; + } + SMARTLIST_FOREACH(sl, char *, s, tor_free(s)); smartlist_free(sl); + return r; } @@ -3551,26 +3967,26 @@ check_nickname_list(const char *lst, const char *name, char **msg) * filename if it doesn't exist. */ static char * -find_torrc_filename(int argc, char **argv, +find_torrc_filename(config_line_t *cmd_arg, int defaults_file, int *using_default_fname, int *ignore_missing_torrc) { char *fname=NULL; - int i; + config_line_t *p_index; const char *fname_opt = defaults_file ? "--defaults-torrc" : "-f"; const char *ignore_opt = defaults_file ? NULL : "--ignore-missing-torrc"; if (defaults_file) *ignore_missing_torrc = 1; - for (i = 1; i < argc; ++i) { - if (i < argc-1 && !strcmp(argv[i],fname_opt)) { + for (p_index = cmd_arg; p_index; p_index = p_index->next) { + if (!strcmp(p_index->key, fname_opt)) { if (fname) { log_warn(LD_CONFIG, "Duplicate %s options on command line.", fname_opt); tor_free(fname); } - fname = expand_filename(argv[i+1]); + fname = expand_filename(p_index->value); { char *absfname; @@ -3580,8 +3996,7 @@ find_torrc_filename(int argc, char **argv, } *using_default_fname = 0; - ++i; - } else if (ignore_opt && !strcmp(argv[i],ignore_opt)) { + } else if (ignore_opt && !strcmp(p_index->key,ignore_opt)) { *ignore_missing_torrc = 1; } } @@ -3618,7 +4033,7 @@ find_torrc_filename(int argc, char **argv, * Return the contents of the file on success, and NULL on failure. */ static char * -load_torrc_from_disk(int argc, char **argv, int defaults_file) +load_torrc_from_disk(config_line_t *cmd_arg, int defaults_file) { char *fname=NULL; char *cf = NULL; @@ -3626,7 +4041,7 @@ load_torrc_from_disk(int argc, char **argv, int defaults_file) int ignore_missing_torrc = 0; char **fname_var = defaults_file ? &torrc_defaults_fname : &torrc_fname; - fname = find_torrc_filename(argc, argv, defaults_file, + fname = find_torrc_filename(cmd_arg, defaults_file, &using_default_torrc, &ignore_missing_torrc); tor_assert(fname); log_debug(LD_CONFIG, "Opening config file \"%s\"", fname); @@ -3668,59 +4083,75 @@ int options_init_from_torrc(int argc, char **argv) { char *cf=NULL, *cf_defaults=NULL; - int i, command; + int command; int retval = -1; - static char **backup_argv; - static int backup_argc; char *command_arg = NULL; char *errmsg=NULL; + config_line_t *p_index = NULL; + config_line_t *cmdline_only_options = NULL; - if (argv) { /* first time we're called. save command line args */ - backup_argv = argv; - backup_argc = argc; - } else { /* we're reloading. need to clean up old options first. */ - argv = backup_argv; - argc = backup_argc; + /* Go through command-line variables */ + if (! have_parsed_cmdline) { + /* Or we could redo the list every time we pass this place. + * It does not really matter */ + if (config_parse_commandline(argc, argv, 0, &global_cmdline_options, + &global_cmdline_only_options) < 0) { + goto err; + } + have_parsed_cmdline = 1; } - if (argc > 1 && (!strcmp(argv[1], "-h") || !strcmp(argv[1],"--help"))) { + cmdline_only_options = global_cmdline_only_options; + + if (config_line_find(cmdline_only_options, "-h") || + config_line_find(cmdline_only_options, "--help")) { print_usage(); exit(0); } - if (argc > 1 && !strcmp(argv[1], "--list-torrc-options")) { + if (config_line_find(cmdline_only_options, "--list-torrc-options")) { /* For documenting validating whether we've documented everything. */ list_torrc_options(); exit(0); } - if (argc > 1 && (!strcmp(argv[1],"--version"))) { + if (config_line_find(cmdline_only_options, "--version")) { printf("Tor version %s.\n",get_version()); exit(0); } - if (argc > 1 && (!strcmp(argv[1],"--digests"))) { + + if (config_line_find(cmdline_only_options, "--digests")) { printf("Tor version %s.\n",get_version()); printf("%s", libor_get_digests()); printf("%s", tor_get_digests()); exit(0); } - /* Go through command-line variables */ - if (!global_cmdline_options) { - /* Or we could redo the list every time we pass this place. - * It does not really matter */ - if (config_get_commandlines(argc, argv, &global_cmdline_options) < 0) { - goto err; - } + if (config_line_find(cmdline_only_options, "--library-versions")) { + printf("Tor version %s. \n", get_version()); + printf("Library versions\tCompiled\t\tRuntime\n"); + printf("Libevent\t\t%-15s\t\t%s\n", + tor_libevent_get_header_version_str(), + tor_libevent_get_version_str()); + printf("OpenSSL \t\t%-15s\t\t%s\n", + crypto_openssl_get_header_version_str(), + crypto_openssl_get_version_str()); + printf("Zlib \t\t%-15s\t\t%s\n", + tor_zlib_get_header_version_str(), + tor_zlib_get_version_str()); + //TODO: Hex versions? + exit(0); } command = CMD_RUN_TOR; - for (i = 1; i < argc; ++i) { - if (!strcmp(argv[i],"--list-fingerprint")) { + for (p_index = cmdline_only_options; p_index; p_index = p_index->next) { + if (!strcmp(p_index->key,"--list-fingerprint")) { command = CMD_LIST_FINGERPRINT; - } else if (!strcmp(argv[i],"--hash-password")) { + } else if (!strcmp(p_index->key, "--hash-password")) { command = CMD_HASH_PASSWORD; - command_arg = tor_strdup( (i < argc-1) ? argv[i+1] : ""); - ++i; - } else if (!strcmp(argv[i],"--verify-config")) { + command_arg = p_index->value; + } else if (!strcmp(p_index->key, "--dump-config")) { + command = CMD_DUMP_CONFIG; + command_arg = p_index->value; + } else if (!strcmp(p_index->key, "--verify-config")) { command = CMD_VERIFY_CONFIG; } } @@ -3729,10 +4160,15 @@ options_init_from_torrc(int argc, char **argv) cf_defaults = tor_strdup(""); cf = tor_strdup(""); } else { - cf_defaults = load_torrc_from_disk(argc, argv, 1); - cf = load_torrc_from_disk(argc, argv, 0); - if (!cf) - goto err; + cf_defaults = load_torrc_from_disk(cmdline_only_options, 1); + cf = load_torrc_from_disk(cmdline_only_options, 0); + if (!cf) { + if (config_line_find(cmdline_only_options, "--allow-missing-torrc")) { + cf = tor_strdup(""); + } else { + goto err; + } + } } retval = options_init_from_string(cf_defaults, cf, command, command_arg, @@ -3776,7 +4212,7 @@ options_init_from_string(const char *cf_defaults, const char *cf, newoptions->magic_ = OR_OPTIONS_MAGIC; options_init(newoptions); newoptions->command = command; - newoptions->command_arg = command_arg; + newoptions->command_arg = command_arg ? tor_strdup(command_arg) : NULL; for (i = 0; i < 2; ++i) { const char *body = i==0 ? cf_defaults : cf; @@ -3840,7 +4276,7 @@ options_init_from_string(const char *cf_defaults, const char *cf, newoptions->magic_ = OR_OPTIONS_MAGIC; options_init(newoptions); newoptions->command = command; - newoptions->command_arg = command_arg; + newoptions->command_arg = command_arg ? tor_strdup(command_arg) : NULL; /* Assign all options a second time. */ for (i = 0; i < 2; ++i) { @@ -3872,7 +4308,8 @@ options_init_from_string(const char *cf_defaults, const char *cf, } /* Validate newoptions */ - if (options_validate(oldoptions, newoptions, 0, msg) < 0) { + if (options_validate(oldoptions, newoptions, newdefaultoptions, + 0, msg) < 0) { err = SETOPT_ERR_PARSE; /*XXX make this a separate return value.*/ goto err; } @@ -4129,21 +4566,72 @@ options_init_logs(or_options_t *options, int validate_only) return ok?0:-1; } +/** Given a smartlist of SOCKS arguments to be passed to a transport + * proxy in <b>args</b>, validate them and return -1 if they are + * corrupted. Return 0 if they seem OK. */ +static int +validate_transport_socks_arguments(const smartlist_t *args) +{ + char *socks_string = NULL; + size_t socks_string_len; + + tor_assert(args); + tor_assert(smartlist_len(args) > 0); + + SMARTLIST_FOREACH_BEGIN(args, const char *, s) { + if (!string_is_key_value(LOG_WARN, s)) { /* items should be k=v items */ + log_warn(LD_CONFIG, "'%s' is not a k=v item.", s); + return -1; + } + } SMARTLIST_FOREACH_END(s); + + socks_string = pt_stringify_socks_args(args); + if (!socks_string) + return -1; + + socks_string_len = strlen(socks_string); + tor_free(socks_string); + + if (socks_string_len > MAX_SOCKS5_AUTH_SIZE_TOTAL) { + log_warn(LD_CONFIG, "SOCKS arguments can't be more than %u bytes (%lu).", + MAX_SOCKS5_AUTH_SIZE_TOTAL, + (unsigned long) socks_string_len); + return -1; + } + + return 0; +} + +/** Deallocate a bridge_line_t structure. */ +/* private */ void +bridge_line_free(bridge_line_t *bridge_line) +{ + if (!bridge_line) + return; + + if (bridge_line->socks_args) { + SMARTLIST_FOREACH(bridge_line->socks_args, char*, s, tor_free(s)); + smartlist_free(bridge_line->socks_args); + } + tor_free(bridge_line->transport_name); + tor_free(bridge_line); +} + /** Read the contents of a Bridge line from <b>line</b>. Return 0 * if the line is well-formed, and -1 if it isn't. If * <b>validate_only</b> is 0, and the line is well-formed, then add - * the bridge described in the line to our internal bridge list. */ -static int -parse_bridge_line(const char *line, int validate_only) + * the bridge described in the line to our internal bridge list. + * + * Bridge line format: + * Bridge [transport] IP:PORT [id-fingerprint] [k=v] [k=v] ... + */ +/* private */ bridge_line_t * +parse_bridge_line(const char *line) { smartlist_t *items = NULL; - int r; char *addrport=NULL, *fingerprint=NULL; - char *transport_name=NULL; - char *field1=NULL; - tor_addr_t addr; - uint16_t port = 0; - char digest[DIGEST_LEN]; + char *field=NULL; + bridge_line_t *bridge_line = tor_malloc_zero(sizeof(bridge_line_t)); items = smartlist_new(); smartlist_split_string(items, line, NULL, @@ -4153,80 +4641,109 @@ parse_bridge_line(const char *line, int validate_only) goto err; } - /* field1 is either a transport name or addrport */ - field1 = smartlist_get(items, 0); + /* first field is either a transport name or addrport */ + field = smartlist_get(items, 0); smartlist_del_keeporder(items, 0); - if (!(strstr(field1, ".") || strstr(field1, ":"))) { - /* new-style bridge line */ - transport_name = field1; + if (string_is_C_identifier(field)) { + /* It's a transport name. */ + bridge_line->transport_name = field; if (smartlist_len(items) < 1) { log_warn(LD_CONFIG, "Too few items to Bridge line."); goto err; } - addrport = smartlist_get(items, 0); + addrport = smartlist_get(items, 0); /* Next field is addrport then. */ smartlist_del_keeporder(items, 0); } else { - addrport = field1; + addrport = field; } - if (tor_addr_port_lookup(addrport, &addr, &port)<0) { + if (tor_addr_port_parse(LOG_INFO, addrport, + &bridge_line->addr, &bridge_line->port, 443)<0) { log_warn(LD_CONFIG, "Error parsing Bridge address '%s'", addrport); goto err; } - if (!port) { - log_info(LD_CONFIG, - "Bridge address '%s' has no port; using default port 443.", - addrport); - port = 443; - } + /* If transports are enabled, next field could be a fingerprint or a + socks argument. If transports are disabled, next field must be + a fingerprint. */ if (smartlist_len(items)) { - fingerprint = smartlist_join_strings(items, "", 0, NULL); + if (bridge_line->transport_name) { /* transports enabled: */ + field = smartlist_get(items, 0); + smartlist_del_keeporder(items, 0); + + /* If it's a key=value pair, then it's a SOCKS argument for the + transport proxy... */ + if (string_is_key_value(LOG_DEBUG, field)) { + bridge_line->socks_args = smartlist_new(); + smartlist_add(bridge_line->socks_args, field); + } else { /* ...otherwise, it's the bridge fingerprint. */ + fingerprint = field; + } + + } else { /* transports disabled: */ + fingerprint = smartlist_join_strings(items, "", 0, NULL); + } + } + + /* Handle fingerprint, if it was provided. */ + if (fingerprint) { if (strlen(fingerprint) != HEX_DIGEST_LEN) { log_warn(LD_CONFIG, "Key digest for Bridge is wrong length."); goto err; } - if (base16_decode(digest, DIGEST_LEN, fingerprint, HEX_DIGEST_LEN)<0) { + if (base16_decode(bridge_line->digest, DIGEST_LEN, + fingerprint, HEX_DIGEST_LEN)<0) { log_warn(LD_CONFIG, "Unable to decode Bridge key digest."); goto err; } } - if (!validate_only) { - log_debug(LD_DIR, "Bridge at %s (transport: %s) (%s)", - fmt_addrport(&addr, port), - transport_name ? transport_name : "no transport", - fingerprint ? fingerprint : "no key listed"); - bridge_add_from_config(&addr, port, - fingerprint ? digest : NULL, transport_name); + /* If we are using transports, any remaining items in the smartlist + should be k=v values. */ + if (bridge_line->transport_name && smartlist_len(items)) { + if (!bridge_line->socks_args) + bridge_line->socks_args = smartlist_new(); + + /* append remaining items of 'items' to 'socks_args' */ + smartlist_add_all(bridge_line->socks_args, items); + smartlist_clear(items); + + tor_assert(smartlist_len(bridge_line->socks_args) > 0); + } + + if (bridge_line->socks_args) { + if (validate_transport_socks_arguments(bridge_line->socks_args) < 0) + goto err; } - r = 0; goto done; err: - r = -1; + bridge_line_free(bridge_line); + bridge_line = NULL; done: SMARTLIST_FOREACH(items, char*, s, tor_free(s)); smartlist_free(items); tor_free(addrport); - tor_free(transport_name); tor_free(fingerprint); - return r; + + return bridge_line; } /** Read the contents of a ClientTransportPlugin line from * <b>line</b>. Return 0 if the line is well-formed, and -1 if it * isn't. * - * If <b>validate_only</b> is 0, and the line is well-formed: + * If <b>validate_only</b> is 0, the line is well-formed, and the + * transport is needed by some bridge: * - If it's an external proxy line, add the transport described in the line to * our internal transport list. * - If it's a managed proxy line, launch the managed proxy. */ static int -parse_client_transport_line(const char *line, int validate_only) +parse_client_transport_line(const or_options_t *options, + const char *line, int validate_only) { smartlist_t *items = NULL; int r; @@ -4243,7 +4760,8 @@ parse_client_transport_line(const char *line, int validate_only) int is_managed=0; char **proxy_argv=NULL; char **tmp=NULL; - int proxy_argc,i; + int proxy_argc, i; + int is_useless_proxy=1; int line_length; @@ -4265,11 +4783,16 @@ parse_client_transport_line(const char *line, int validate_only) smartlist_split_string(transport_list, transports, ",", SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); SMARTLIST_FOREACH_BEGIN(transport_list, const char *, transport_name) { + /* validate transport names */ if (!string_is_C_identifier(transport_name)) { log_warn(LD_CONFIG, "Transport name is not a C identifier (%s).", transport_name); goto err; } + + /* see if we actually need the transports provided by this proxy */ + if (!validate_only && transport_is_needed(transport_name)) + is_useless_proxy = 0; } SMARTLIST_FOREACH_END(transport_name); /* field2 is either a SOCKS version or "exec" */ @@ -4287,10 +4810,22 @@ parse_client_transport_line(const char *line, int validate_only) goto err; } + if (is_managed && options->Sandbox) { + log_warn(LD_CONFIG, "Managed proxies are not compatible with Sandbox mode." + "(ClientTransportPlugin line was %s)", escaped(line)); + goto err; + } + if (is_managed) { /* managed */ - if (!validate_only) { /* if we are not just validating, use the - rest of the line as the argv of the proxy - to be launched */ + if (!validate_only && is_useless_proxy) { + log_notice(LD_GENERAL, "Pluggable transport proxy (%s) does not provide " + "any needed transports and will not be launched.", line); + } + + /* If we are not just validating, use the rest of the line as the + argv of the proxy to be launched. Also, make sure that we are + only launching proxies that contribute useful transports. */ + if (!validate_only && !is_useless_proxy) { proxy_argc = line_length-2; tor_assert(proxy_argc > 0); proxy_argv = tor_malloc_zero(sizeof(char*)*(proxy_argc+1)); @@ -4385,7 +4920,7 @@ get_bindaddr_from_transport_listen_line(const char *line,const char *transport) goto err; /* Validate addrport */ - if (tor_addr_port_parse(LOG_WARN, addrport, &addr, &port)<0) { + if (tor_addr_port_parse(LOG_WARN, addrport, &addr, &port, -1)<0) { log_warn(LD_CONFIG, "Error parsing ServerTransportListenAddr " "address '%s'", addrport); goto err; @@ -4404,6 +4939,63 @@ get_bindaddr_from_transport_listen_line(const char *line,const char *transport) return addrport; } +/** Given a ServerTransportOptions <b>line</b>, return a smartlist + * with the options. Return NULL if the line was not well-formed. + * + * If <b>transport</b> is set, return NULL if the line is not + * referring to <b>transport</b>. + * + * The returned smartlist and its strings are allocated on the heap + * and it's the responsibility of the caller to free it. */ +smartlist_t * +get_options_from_transport_options_line(const char *line,const char *transport) +{ + smartlist_t *items = smartlist_new(); + smartlist_t *options = smartlist_new(); + const char *parsed_transport = NULL; + + smartlist_split_string(items, line, NULL, + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, -1); + + if (smartlist_len(items) < 2) { + log_warn(LD_CONFIG,"Too few arguments on ServerTransportOptions line."); + goto err; + } + + parsed_transport = smartlist_get(items, 0); + /* If 'transport' is given, check if it matches the one on the line */ + if (transport && strcmp(transport, parsed_transport)) + goto err; + + SMARTLIST_FOREACH_BEGIN(items, const char *, option) { + if (option_sl_idx == 0) /* skip the transport field (first field)*/ + continue; + + /* validate that it's a k=v value */ + if (!string_is_key_value(LOG_WARN, option)) { + log_warn(LD_CONFIG, "%s is not a k=v value.", escaped(option)); + goto err; + } + + /* add it to the options smartlist */ + smartlist_add(options, tor_strdup(option)); + log_debug(LD_CONFIG, "Added %s to the list of options", escaped(option)); + } SMARTLIST_FOREACH_END(option); + + goto done; + + err: + SMARTLIST_FOREACH(options, char*, s, tor_free(s)); + smartlist_free(options); + options = NULL; + + done: + SMARTLIST_FOREACH(items, char*, s, tor_free(s)); + smartlist_free(items); + + return options; +} + /** Given the name of a pluggable transport in <b>transport</b>, check * the configuration file to see if the user has explicitly asked for * it to listen on a specific port. Return a <address:port> string if @@ -4424,13 +5016,34 @@ get_transport_bindaddr_from_config(const char *transport) return NULL; } +/** Given the name of a pluggable transport in <b>transport</b>, check + * the configuration file to see if the user has asked us to pass any + * parameters to the pluggable transport. Return a smartlist + * containing the parameters, otherwise NULL. */ +smartlist_t * +get_options_for_server_transport(const char *transport) +{ + config_line_t *cl; + const or_options_t *options = get_options(); + + for (cl = options->ServerTransportOptions; cl; cl = cl->next) { + smartlist_t *options_sl = + get_options_from_transport_options_line(cl->value, transport); + if (options_sl) + return options_sl; + } + + return NULL; +} + /** Read the contents of a ServerTransportPlugin line from * <b>line</b>. Return 0 if the line is well-formed, and -1 if it * isn't. * If <b>validate_only</b> is 0, the line is well-formed, and it's a * managed proxy line, launch the managed proxy. */ static int -parse_server_transport_line(const char *line, int validate_only) +parse_server_transport_line(const or_options_t *options, + const char *line, int validate_only) { smartlist_t *items = NULL; int r; @@ -4485,6 +5098,12 @@ parse_server_transport_line(const char *line, int validate_only) goto err; } + if (is_managed && options->Sandbox) { + log_warn(LD_CONFIG, "Managed proxies are not compatible with Sandbox mode." + "(ServerTransportPlugin line was %s)", escaped(line)); + goto err; + } + if (is_managed) { /* managed */ if (!validate_only) { proxy_argc = line_length-2; @@ -4560,8 +5179,7 @@ parse_dir_authority_line(const char *line, dirinfo_type_t required_type, uint16_t dir_port = 0, or_port = 0; char digest[DIGEST_LEN]; char v3_digest[DIGEST_LEN]; - dirinfo_type_t type = V2_DIRINFO; - int is_not_hidserv_authority = 0, is_not_v2_authority = 0; + dirinfo_type_t type = 0; double weight = 1.0; items = smartlist_new(); @@ -4581,16 +5199,15 @@ parse_dir_authority_line(const char *line, dirinfo_type_t required_type, char *flag = smartlist_get(items, 0); if (TOR_ISDIGIT(flag[0])) break; - if (!strcasecmp(flag, "v1")) { - type |= (V1_DIRINFO | HIDSERV_DIRINFO); - } else if (!strcasecmp(flag, "hs")) { - type |= HIDSERV_DIRINFO; - } else if (!strcasecmp(flag, "no-hs")) { - is_not_hidserv_authority = 1; + if (!strcasecmp(flag, "hs") || + !strcasecmp(flag, "no-hs")) { + log_warn(LD_CONFIG, "The DirAuthority options 'hs' and 'no-hs' are " + "obsolete; you don't need them any more."); } else if (!strcasecmp(flag, "bridge")) { type |= BRIDGE_DIRINFO; } else if (!strcasecmp(flag, "no-v2")) { - is_not_v2_authority = 1; + /* obsolete, but may still be contained in DirAuthority lines generated + by various tools */; } else if (!strcasecmpstart(flag, "orport=")) { int ok; char *portstring = flag + strlen("orport="); @@ -4622,10 +5239,6 @@ parse_dir_authority_line(const char *line, dirinfo_type_t required_type, tor_free(flag); smartlist_del_keeporder(items, 0); } - if (is_not_hidserv_authority) - type &= ~HIDSERV_DIRINFO; - if (is_not_v2_authority) - type &= ~V2_DIRINFO; if (smartlist_len(items) < 2) { log_warn(LD_CONFIG, "Too few arguments to DirAuthority line."); @@ -4828,6 +5441,27 @@ warn_nonlocal_client_ports(const smartlist_t *ports, const char *portname, } SMARTLIST_FOREACH_END(port); } +/** Warn for every Extended ORPort port in <b>ports</b> that is on a + * publicly routable address. */ +static void +warn_nonlocal_ext_orports(const smartlist_t *ports, const char *portname) +{ + SMARTLIST_FOREACH_BEGIN(ports, const port_cfg_t *, port) { + if (port->type != CONN_TYPE_EXT_OR_LISTENER) + continue; + if (port->is_unix_addr) + continue; + /* XXX maybe warn even if address is RFC1918? */ + if (!tor_addr_is_internal(&port->addr, 1)) { + log_warn(LD_CONFIG, "You specified a public address '%s' for %sPort. " + "This is not advised; this address is supposed to only be " + "exposed on localhost so that your pluggable transport " + "proxies can connect to it.", + fmt_addrport(&port->addr, port->port), portname); + } + } SMARTLIST_FOREACH_END(port); +} + /** Given a list of port_cfg_t in <b>ports</b>, warn any controller port there * is listening on any non-loopback address. If <b>forbid</b> is true, * then emit a stronger warning and remove the port from the list. @@ -4928,6 +5562,7 @@ parse_port_config(smartlist_t *out, smartlist_t *elts; int retval = -1; const unsigned is_control = (listener_type == CONN_TYPE_CONTROL_LISTENER); + const unsigned is_ext_orport = (listener_type == CONN_TYPE_EXT_OR_LISTENER); const unsigned allow_no_options = flags & CL_PORT_NO_OPTIONS; const unsigned use_server_options = flags & CL_PORT_SERVER_OPTIONS; const unsigned warn_nonlocal = flags & CL_PORT_WARN_NONLOCAL; @@ -5005,6 +5640,8 @@ parse_port_config(smartlist_t *out, if (warn_nonlocal && out) { if (is_control) warn_nonlocal_controller_ports(out, forbid_nonlocal); + else if (is_ext_orport) + warn_nonlocal_ext_orports(out, portname); else warn_nonlocal_client_ports(out, portname, listener_type); } @@ -5278,6 +5915,8 @@ parse_port_config(smartlist_t *out, if (warn_nonlocal && out) { if (is_control) warn_nonlocal_controller_ports(out, forbid_nonlocal); + else if (is_ext_orport) + warn_nonlocal_ext_orports(out, portname); else warn_nonlocal_client_ports(out, portname, listener_type); } @@ -5424,6 +6063,14 @@ parse_ports(or_options_t *options, int validate_only, goto err; } if (parse_port_config(ports, + options->ExtORPort_lines, NULL, + "ExtOR", CONN_TYPE_EXT_OR_LISTENER, + "127.0.0.1", 0, + CL_PORT_SERVER_OPTIONS|CL_PORT_WARN_NONLOCAL) < 0) { + *msg = tor_strdup("Invalid ExtORPort configuration"); + goto err; + } + if (parse_port_config(ports, options->DirPort_lines, options->DirListenAddress, "Dir", CONN_TYPE_DIR_LISTENER, "0.0.0.0", 0, @@ -5458,6 +6105,8 @@ parse_ports(or_options_t *options, int validate_only, !! count_real_listeners(ports, CONN_TYPE_DIR_LISTENER); options->DNSPort_set = !! count_real_listeners(ports, CONN_TYPE_AP_DNS_LISTENER); + options->ExtORPort_set = + !! count_real_listeners(ports, CONN_TYPE_EXT_OR_LISTENER); if (!validate_only) { if (configured_ports) { @@ -5745,7 +6394,7 @@ write_configuration_file(const char *fname, const or_options_t *options) return -1; } - if (!(new_conf = options_dump(options, 1))) { + if (!(new_conf = options_dump(options, OPTIONS_DUMP_MINIMAL))) { log_warn(LD_BUG, "Couldn't get configuration string"); goto err; } @@ -5764,7 +6413,7 @@ write_configuration_file(const char *fname, const or_options_t *options) ++i; } log_notice(LD_CONFIG, "Renaming old configuration file to \"%s\"", fn_tmp); - if (rename(fname, fn_tmp) < 0) { + if (tor_rename(fname, fn_tmp) < 0) {//XXXX sandbox doesn't allow log_warn(LD_FS, "Couldn't rename configuration file \"%s\" to \"%s\": %s", fname, fn_tmp, strerror(errno)); @@ -5905,6 +6554,43 @@ options_get_datadir_fname2_suffix(const or_options_t *options, return fname; } +/** Check wether the data directory has a private subdirectory + * <b>subdir</b>. If not, try to create it. Return 0 on success, + * -1 otherwise. */ +int +check_or_create_data_subdir(const char *subdir) +{ + char *statsdir = get_datadir_fname(subdir); + int return_val = 0; + + if (check_private_dir(statsdir, CPD_CREATE, get_options()->User) < 0) { + log_warn(LD_HIST, "Unable to create %s/ directory!", subdir); + return_val = -1; + } + tor_free(statsdir); + return return_val; +} + +/** Create a file named <b>fname</b> with contents <b>str</b> in the + * subdirectory <b>subdir</b> of the data directory. <b>descr</b> + * should be a short description of the file's content and will be + * used for the warning message, if it's present and the write process + * fails. Return 0 on success, -1 otherwise.*/ +int +write_to_data_subdir(const char* subdir, const char* fname, + const char* str, const char* descr) +{ + char *filename = get_datadir_fname2(subdir, fname); + int return_val = 0; + + if (write_str_to_file(filename, str, 0) < 0) { + log_warn(LD_HIST, "Unable to write %s to disk!", descr ? descr : fname); + return_val = -1; + } + tor_free(filename); + return return_val; +} + /** Given a file name check to see whether the file exists but has not been * modified for a very long time. If so, remove it. */ void @@ -5913,12 +6599,17 @@ remove_file_if_very_old(const char *fname, time_t now) #define VERY_OLD_FILE_AGE (28*24*60*60) struct stat st; - if (stat(fname, &st)==0 && st.st_mtime < now-VERY_OLD_FILE_AGE) { + log_debug(LD_FS, "stat()ing %s", fname); + if (stat(sandbox_intern_string(fname), &st)==0 && + st.st_mtime < now-VERY_OLD_FILE_AGE) { char buf[ISO_TIME_LEN+1]; format_local_iso_time(buf, st.st_mtime); log_notice(LD_GENERAL, "Obsolete file %s hasn't been modified since %s. " "Removing it.", fname, buf); - unlink(fname); + if (unlink(fname) != 0) { + log_warn(LD_FS, "Failed to unlink %s: %s", + fname, strerror(errno)); + } } } @@ -5994,6 +6685,7 @@ getinfo_helper_config(control_connection_t *conn, case CONFIG_TYPE_ISOTIME: type = "Time"; break; case CONFIG_TYPE_ROUTERSET: type = "RouterList"; break; case CONFIG_TYPE_CSV: type = "CommaList"; break; + case CONFIG_TYPE_CSV_INTERVAL: type = "TimeIntervalCommaList"; break; case CONFIG_TYPE_LINELIST: type = "LineList"; break; case CONFIG_TYPE_LINELIST_S: type = "Dependant"; break; case CONFIG_TYPE_LINELIST_V: type = "Virtual"; break; @@ -6125,3 +6817,58 @@ config_maybe_load_geoip_files_(const or_options_t *options, config_load_geoip_file_(AF_INET6, options->GeoIPv6File, "geoip6"); } +/** Initialize cookie authentication (used so far by the ControlPort + * and Extended ORPort). + * + * Allocate memory and create a cookie (of length <b>cookie_len</b>) + * in <b>cookie_out</b>. + * Then write it down to <b>fname</b> and prepend it with <b>header</b>. + * + * If the whole procedure was successful, set + * <b>cookie_is_set_out</b> to True. */ +int +init_cookie_authentication(const char *fname, const char *header, + int cookie_len, + uint8_t **cookie_out, int *cookie_is_set_out) +{ + char cookie_file_str_len = strlen(header) + cookie_len; + char *cookie_file_str = tor_malloc(cookie_file_str_len); + int retval = -1; + + /* We don't want to generate a new cookie every time we call + * options_act(). One should be enough. */ + if (*cookie_is_set_out) { + retval = 0; /* we are all set */ + goto done; + } + + /* If we've already set the cookie, free it before re-setting + it. This can happen if we previously generated a cookie, but + couldn't write it to a disk. */ + if (*cookie_out) + tor_free(*cookie_out); + + /* Generate the cookie */ + *cookie_out = tor_malloc(cookie_len); + if (crypto_rand((char *)*cookie_out, cookie_len) < 0) + goto done; + + /* Create the string that should be written on the file. */ + memcpy(cookie_file_str, header, strlen(header)); + memcpy(cookie_file_str+strlen(header), *cookie_out, cookie_len); + if (write_bytes_to_file(fname, cookie_file_str, cookie_file_str_len, 1)) { + log_warn(LD_FS,"Error writing auth cookie to %s.", escaped(fname)); + goto done; + } + + /* Success! */ + log_info(LD_GENERAL, "Generated auth cookie file in '%s'.", escaped(fname)); + *cookie_is_set_out = 1; + retval = 0; + + done: + memwipe(cookie_file_str, 0, cookie_file_str_len); + tor_free(cookie_file_str); + return retval; +} + diff --git a/src/or/config.h b/src/or/config.h index ef4acac514..bf386134b8 100644 --- a/src/or/config.h +++ b/src/or/config.h @@ -12,8 +12,10 @@ #ifndef TOR_CONFIG_H #define TOR_CONFIG_H +#include "testsupport.h" + const char *get_dirportfrontpage(void); -const or_options_t *get_options(void); +MOCK_DECL(const or_options_t *,get_options,(void)); or_options_t *get_options_mutable(void); int set_options(or_options_t *new_val, char **msg); void config_free_all(void); @@ -32,7 +34,11 @@ int resolve_my_address(int warn_severity, const or_options_t *options, const char **method_out, char **hostname_out); int is_local_addr(const tor_addr_t *addr); void options_init(or_options_t *options); -char *options_dump(const or_options_t *options, int minimal); + +#define OPTIONS_DUMP_MINIMAL 1 +#define OPTIONS_DUMP_DEFAULTS 2 +#define OPTIONS_DUMP_ALL 3 +char *options_dump(const or_options_t *options, int how_to_dump); int options_init_from_torrc(int argc, char **argv); setopt_err_t options_init_from_string(const char *cf_defaults, const char *cf, int command, const char *command_arg, char **msg); @@ -59,6 +65,10 @@ char *options_get_datadir_fname2_suffix(const or_options_t *options, #define get_datadir_fname_suffix(sub1, suffix) \ get_datadir_fname2_suffix((sub1), NULL, (suffix)) +int check_or_create_data_subdir(const char *subdir); +int write_to_data_subdir(const char* subdir, const char* fname, + const char* str, const char* descr); + int get_num_cpus(const or_options_t *options); const smartlist_t *get_configured_ports(void); @@ -86,10 +96,15 @@ uint32_t get_effective_bwburst(const or_options_t *options); char *get_transport_bindaddr_from_config(const char *transport); -#ifdef CONFIG_PRIVATE -/* Used only by config.c and test.c */ +int init_cookie_authentication(const char *fname, const char *header, + int cookie_len, + uint8_t **cookie_out, int *cookie_is_set_out); + or_options_t *options_new(void); -#endif + +int config_parse_commandline(int argc, char **argv, int ignore_errors, + config_line_t **result, + config_line_t **cmdline_result); void config_register_addressmaps(const or_options_t *options); /* XXXX024 move to connection_edge.h */ @@ -98,5 +113,34 @@ int addressmap_register_auto(const char *from, const char *to, addressmap_entry_source_t addrmap_source, const char **msg); +/** Represents the information stored in a torrc Bridge line. */ +typedef struct bridge_line_t { + tor_addr_t addr; /* The IP address of the bridge. */ + uint16_t port; /* The TCP port of the bridge. */ + char *transport_name; /* The name of the pluggable transport that + should be used to connect to the bridge. */ + char digest[DIGEST_LEN]; /* The bridge's identity key digest. */ + smartlist_t *socks_args; /* SOCKS arguments for the pluggable + transport proxy. */ +} bridge_line_t; + +void bridge_line_free(bridge_line_t *bridge_line); +bridge_line_t *parse_bridge_line(const char *line); +smartlist_t *get_options_from_transport_options_line(const char *line, + const char *transport); +smartlist_t *get_options_for_server_transport(const char *transport); + +#ifdef CONFIG_PRIVATE +#ifdef TOR_UNIT_TESTS +extern struct config_format_t options_format; +#endif + +STATIC void or_options_free(or_options_t *options); +STATIC int options_validate(or_options_t *old_options, + or_options_t *options, + or_options_t *default_options, + int from_setconf, char **msg); +#endif + #endif diff --git a/src/or/confparse.c b/src/or/confparse.c index 8863d92409..c5400a6512 100644 --- a/src/or/confparse.c +++ b/src/or/confparse.c @@ -79,6 +79,21 @@ config_line_append(config_line_t **lst, (*lst) = newline; } +/** Return the line in <b>lines</b> whose key is exactly <b>key</b>, or NULL + * if no such key exists. For handling commandline-only options only; other + * options should be looked up in the appropriate data structure. */ +const config_line_t * +config_line_find(const config_line_t *lines, + const char *key) +{ + const config_line_t *cl; + for (cl = lines; cl; cl = cl->next) { + if (!strcmp(cl->key, key)) + return cl; + } + return NULL; +} + /** Helper: parse the config string and strdup into key/value * strings. Set *result to the list, or NULL if parsing the string * failed. Return 0 on success, -1 on failure. Warn and ignore any @@ -223,6 +238,8 @@ config_assign_value(const config_format_t *fmt, void *options, int i, ok; const config_var_t *var; void *lvalue; + int *csv_int; + smartlist_t *csv_str; CONFIG_CHECK(fmt, options); @@ -357,6 +374,36 @@ config_assign_value(const config_format_t *fmt, void *options, SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); break; + case CONFIG_TYPE_CSV_INTERVAL: + if (*(smartlist_t**)lvalue) { + SMARTLIST_FOREACH(*(smartlist_t**)lvalue, int *, cp, tor_free(cp)); + smartlist_clear(*(smartlist_t**)lvalue); + } else { + *(smartlist_t**)lvalue = smartlist_new(); + } + csv_str = smartlist_new(); + smartlist_split_string(csv_str, c->value, ",", + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); + SMARTLIST_FOREACH_BEGIN(csv_str, char *, str) + { + i = config_parse_interval(str, &ok); + if (!ok) { + tor_asprintf(msg, + "Interval in '%s %s' is malformed or out of bounds.", + c->key, c->value); + SMARTLIST_FOREACH(csv_str, char *, cp, tor_free(cp)); + smartlist_free(csv_str); + return -1; + } + csv_int = tor_malloc_zero(sizeof(int)); + *csv_int = i; + smartlist_add(*(smartlist_t**)lvalue, csv_int); + } + SMARTLIST_FOREACH_END(str); + SMARTLIST_FOREACH(csv_str, char *, cp, tor_free(cp)); + smartlist_free(csv_str); + break; + case CONFIG_TYPE_LINELIST: case CONFIG_TYPE_LINELIST_S: { @@ -555,6 +602,7 @@ config_get_assigned_option(const config_format_t *fmt, const void *options, const config_var_t *var; const void *value; config_line_t *result; + smartlist_t *csv_str; tor_assert(options && key); CONFIG_CHECK(fmt, options); @@ -637,6 +685,20 @@ config_get_assigned_option(const config_format_t *fmt, const void *options, else result->value = tor_strdup(""); break; + case CONFIG_TYPE_CSV_INTERVAL: + if (*(smartlist_t**)value) { + csv_str = smartlist_new(); + SMARTLIST_FOREACH_BEGIN(*(smartlist_t**)value, int *, i) + { + smartlist_add_asprintf(csv_str, "%d", *i); + } + SMARTLIST_FOREACH_END(i); + result->value = smartlist_join_strings(csv_str, ",", 0, NULL); + SMARTLIST_FOREACH(csv_str, char *, cp, tor_free(cp)); + smartlist_free(csv_str); + } else + result->value = tor_strdup(""); + break; case CONFIG_TYPE_OBSOLETE: log_fn(LOG_INFO, LD_CONFIG, "You asked me for the value of an obsolete config option '%s'.", @@ -826,6 +888,13 @@ config_clear(const config_format_t *fmt, void *options, *(smartlist_t **)lvalue = NULL; } break; + case CONFIG_TYPE_CSV_INTERVAL: + if (*(smartlist_t**)lvalue) { + SMARTLIST_FOREACH(*(smartlist_t **)lvalue, int *, cp, tor_free(cp)); + smartlist_free(*(smartlist_t **)lvalue); + *(smartlist_t **)lvalue = NULL; + } + break; case CONFIG_TYPE_LINELIST: case CONFIG_TYPE_LINELIST_S: config_free_lines(*(config_line_t **)lvalue); @@ -1005,8 +1074,8 @@ config_dump(const config_format_t *fmt, const void *default_options, /* XXX use a 1 here so we don't add a new log line while dumping */ if (default_options == NULL) { - if (fmt->validate_fn(NULL, defaults_tmp, 1, &msg) < 0) { - log_err(LD_BUG, "Failed to validate default config."); + if (fmt->validate_fn(NULL, defaults_tmp, defaults_tmp, 1, &msg) < 0) { + log_err(LD_BUG, "Failed to validate default config: %s", msg); tor_free(msg); tor_assert(0); } @@ -1072,20 +1141,36 @@ static struct unit_table_t memory_units[] = { { "kbytes", 1<<10 }, { "kilobyte", 1<<10 }, { "kilobytes", 1<<10 }, + { "kilobits", 1<<7 }, + { "kilobit", 1<<7 }, + { "kbits", 1<<7 }, + { "kbit", 1<<7 }, { "m", 1<<20 }, { "mb", 1<<20 }, { "mbyte", 1<<20 }, { "mbytes", 1<<20 }, { "megabyte", 1<<20 }, { "megabytes", 1<<20 }, + { "megabits", 1<<17 }, + { "megabit", 1<<17 }, + { "mbits", 1<<17 }, + { "mbit", 1<<17 }, { "gb", 1<<30 }, { "gbyte", 1<<30 }, { "gbytes", 1<<30 }, { "gigabyte", 1<<30 }, { "gigabytes", 1<<30 }, + { "gigabits", 1<<27 }, + { "gigabit", 1<<27 }, + { "gbits", 1<<27 }, + { "gbit", 1<<27 }, { "tb", U64_LITERAL(1)<<40 }, { "terabyte", U64_LITERAL(1)<<40 }, { "terabytes", U64_LITERAL(1)<<40 }, + { "terabits", U64_LITERAL(1)<<37 }, + { "terabit", U64_LITERAL(1)<<37 }, + { "tbits", U64_LITERAL(1)<<37 }, + { "tbit", U64_LITERAL(1)<<37 }, { NULL, 0 }, }; diff --git a/src/or/confparse.h b/src/or/confparse.h index 1b987f3bf9..2cd6c49a2a 100644 --- a/src/or/confparse.h +++ b/src/or/confparse.h @@ -26,6 +26,9 @@ typedef enum config_type_t { CONFIG_TYPE_ISOTIME, /**< An ISO-formatted time relative to UTC. */ CONFIG_TYPE_CSV, /**< A list of strings, separated by commas and * optional whitespace. */ + CONFIG_TYPE_CSV_INTERVAL, /**< A list of strings, separated by commas and + * optional whitespace, representing intervals in + * seconds, with optional units */ CONFIG_TYPE_LINELIST, /**< Uninterpreted config lines */ CONFIG_TYPE_LINELIST_S, /**< Uninterpreted, context-sensitive config lines, * mixed with other keywords. */ @@ -68,12 +71,12 @@ typedef struct config_var_description_t { /** Type of a callback to validate whether a given configuration is * well-formed and consistent. See options_trial_assign() for documentation * of arguments. */ -typedef int (*validate_fn_t)(void*,void*,int,char**); +typedef int (*validate_fn_t)(void*,void*,void*,int,char**); /** Information on the keys, value types, key-to-struct-member mappings, * variable descriptions, validation functions, and abbreviations for a * configuration or storage format. */ -typedef struct { +typedef struct config_format_t { size_t size; /**< Size of the struct that everything gets parsed into. */ uint32_t magic; /**< Required 'magic value' to make sure we have a struct * of the right type. */ @@ -100,6 +103,8 @@ void *config_new(const config_format_t *fmt); void config_line_append(config_line_t **lst, const char *key, const char *val); config_line_t *config_lines_dup(const config_line_t *inp); +const config_line_t *config_line_find(const config_line_t *lines, + const char *key); void config_free(const config_format_t *fmt, void *options); int config_lines_eq(config_line_t *a, config_line_t *b); int config_count_key(const config_line_t *a, const char *key); diff --git a/src/or/connection.c b/src/or/connection.c index 4f74a1d04b..4788bdf950 100644 --- a/src/or/connection.c +++ b/src/or/connection.c @@ -10,6 +10,7 @@ * on connections. **/ +#define CONNECTION_PRIVATE #include "or.h" #include "buffers.h" /* @@ -17,6 +18,7 @@ * part of a subclass (channel_tls_t). */ #define TOR_CHANNEL_INTERNAL_ +#define CONNECTION_PRIVATE #include "channel.h" #include "channeltls.h" #include "circuitbuild.h" @@ -33,6 +35,7 @@ #include "dns.h" #include "dnsserv.h" #include "entrynodes.h" +#include "ext_orport.h" #include "geoip.h" #include "main.h" #include "policies.h" @@ -44,6 +47,7 @@ #include "router.h" #include "transports.h" #include "routerparse.h" +#include "transports.h" #ifdef USE_BUFFEREVENTS #include <event2/event.h> @@ -97,6 +101,7 @@ static smartlist_t *outgoing_addrs = NULL; #define CASE_ANY_LISTENER_TYPE \ case CONN_TYPE_OR_LISTENER: \ + case CONN_TYPE_EXT_OR_LISTENER: \ case CONN_TYPE_AP_LISTENER: \ case CONN_TYPE_DIR_LISTENER: \ case CONN_TYPE_CONTROL_LISTENER: \ @@ -128,6 +133,8 @@ conn_type_to_string(int type) case CONN_TYPE_CPUWORKER: return "CPU worker"; case CONN_TYPE_CONTROL_LISTENER: return "Control listener"; case CONN_TYPE_CONTROL: return "Control"; + case CONN_TYPE_EXT_OR: return "Extended OR"; + case CONN_TYPE_EXT_OR_LISTENER: return "Extended OR listener"; default: log_warn(LD_BUG, "unknown connection type %d", type); tor_snprintf(buf, sizeof(buf), "unknown [%d]", type); @@ -164,6 +171,18 @@ conn_state_to_string(int type, int state) case OR_CONN_STATE_OPEN: return "open"; } break; + case CONN_TYPE_EXT_OR: + switch (state) { + case EXT_OR_CONN_STATE_AUTH_WAIT_AUTH_TYPE: + return "waiting for authentication type"; + case EXT_OR_CONN_STATE_AUTH_WAIT_CLIENT_NONCE: + return "waiting for client nonce"; + case EXT_OR_CONN_STATE_AUTH_WAIT_CLIENT_HASH: + return "waiting for client hash"; + case EXT_OR_CONN_STATE_OPEN: return "open"; + case EXT_OR_CONN_STATE_FLUSHING: return "flushing final OKAY"; + } + break; case CONN_TYPE_EXIT: switch (state) { case EXIT_CONN_STATE_RESOLVING: return "waiting for dest info"; @@ -228,6 +247,7 @@ connection_type_uses_bufferevent(connection_t *conn) case CONN_TYPE_DIR: case CONN_TYPE_CONTROL: case CONN_TYPE_OR: + case CONN_TYPE_EXT_OR: case CONN_TYPE_CPUWORKER: return 1; default: @@ -249,22 +269,22 @@ dir_connection_new(int socket_family) /** Allocate and return a new or_connection_t, initialized as by * connection_init(). * - * Set timestamp_last_added_nonpadding to now. - * - * Assign a pseudorandom next_circ_id between 0 and 2**15. - * * Initialize active_circuit_pqueue. * * Set active_circuit_pqueue_last_recalibrated to current cell_ewma tick. */ or_connection_t * -or_connection_new(int socket_family) +or_connection_new(int type, int socket_family) { or_connection_t *or_conn = tor_malloc_zero(sizeof(or_connection_t)); time_t now = time(NULL); - connection_init(now, TO_CONN(or_conn), CONN_TYPE_OR, socket_family); + tor_assert(type == CONN_TYPE_OR || type == CONN_TYPE_EXT_OR); + connection_init(now, TO_CONN(or_conn), type, socket_family); - or_conn->timestamp_last_added_nonpadding = time(NULL); + connection_or_set_canonical(or_conn, 0); + + if (type == CONN_TYPE_EXT_OR) + connection_or_set_ext_or_identifier(or_conn); return or_conn; } @@ -311,7 +331,6 @@ control_connection_new(int socket_family) tor_malloc_zero(sizeof(control_connection_t)); connection_init(time(NULL), TO_CONN(control_conn), CONN_TYPE_CONTROL, socket_family); - log_notice(LD_CONTROL, "New control connection opened."); return control_conn; } @@ -334,7 +353,8 @@ connection_new(int type, int socket_family) { switch (type) { case CONN_TYPE_OR: - return TO_CONN(or_connection_new(socket_family)); + case CONN_TYPE_EXT_OR: + return TO_CONN(or_connection_new(type, socket_family)); case CONN_TYPE_EXIT: return TO_CONN(edge_connection_new(type, socket_family)); @@ -376,6 +396,7 @@ connection_init(time_t now, connection_t *conn, int type, int socket_family) switch (type) { case CONN_TYPE_OR: + case CONN_TYPE_EXT_OR: conn->magic = OR_CONNECTION_MAGIC; break; case CONN_TYPE_EXIT: @@ -434,7 +455,7 @@ connection_link_connections(connection_t *conn_a, connection_t *conn_b) * necessary, close its socket if necessary, and mark the directory as dirty * if <b>conn</b> is an OR or OP connection. */ -static void +STATIC void connection_free_(connection_t *conn) { void *mem; @@ -444,6 +465,7 @@ connection_free_(connection_t *conn) switch (conn->type) { case CONN_TYPE_OR: + case CONN_TYPE_EXT_OR: tor_assert(conn->magic == OR_CONNECTION_MAGIC); mem = TO_OR_CONN(conn); memlen = sizeof(or_connection_t); @@ -590,6 +612,13 @@ connection_free_(connection_t *conn) log_warn(LD_BUG, "called on OR conn with non-zeroed identity_digest"); connection_or_remove_from_identity_map(TO_OR_CONN(conn)); } + if (conn->type == CONN_TYPE_OR || conn->type == CONN_TYPE_EXT_OR) { + connection_or_remove_from_ext_or_id_map(TO_OR_CONN(conn)); + tor_free(TO_OR_CONN(conn)->ext_or_conn_id); + tor_free(TO_OR_CONN(conn)->ext_or_auth_correct_client_hash); + tor_free(TO_OR_CONN(conn)->ext_or_transport); + } + #ifdef USE_BUFFEREVENTS if (conn->type == CONN_TYPE_OR && TO_OR_CONN(conn)->bucket_cfg) { ev_token_bucket_cfg_free(TO_OR_CONN(conn)->bucket_cfg); @@ -653,6 +682,7 @@ connection_about_to_close_connection(connection_t *conn) connection_dir_about_to_close(TO_DIR_CONN(conn)); break; case CONN_TYPE_OR: + case CONN_TYPE_EXT_OR: connection_or_about_to_close(TO_OR_CONN(conn)); break; case CONN_TYPE_AP: @@ -892,8 +922,11 @@ check_location_for_unix_socket(const or_options_t *options, const char *path) int r = -1; char *p = tor_strdup(path); cpd_check_t flags = CPD_CHECK_MODE_ONLY; - if (get_parent_directory(p)<0) + if (get_parent_directory(p)<0 || p[0] != '/') { + log_warn(LD_GENERAL, "Bad unix socket address '%s'. Tor does not support " + "relative paths for unix sockets.", path); goto done; + } if (options->ControlSocketsGroupWritable) flags |= CPD_GROUP_OK; @@ -921,12 +954,14 @@ check_location_for_unix_socket(const or_options_t *options, const char *path) #endif /** Tell the TCP stack that it shouldn't wait for a long time after - * <b>sock</b> has closed before reusing its port. */ -static void + * <b>sock</b> has closed before reusing its port. Return 0 on success, + * -1 on failure. */ +static int make_socket_reuseable(tor_socket_t sock) { #ifdef _WIN32 (void) sock; + return 0; #else int one=1; @@ -936,9 +971,9 @@ make_socket_reuseable(tor_socket_t sock) * already has it bound_. So, don't do that on Win32. */ if (setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, (void*) &one, (socklen_t)sizeof(one)) == -1) { - log_warn(LD_NET, "Error setting SO_REUSEADDR flag: %s", - tor_socket_strerror(errno)); + return -1; } + return 0; #endif } @@ -971,16 +1006,16 @@ tor_listen(tor_socket_t fd) */ static connection_t * connection_listener_new(const struct sockaddr *listensockaddr, - socklen_t socklen, - int type, const char *address, - const port_cfg_t *port_cfg) + socklen_t socklen, + int type, const char *address, + const port_cfg_t *port_cfg) { listener_connection_t *lis_conn; - connection_t *conn; - tor_socket_t s; /* the socket we're going to make */ + connection_t *conn = NULL; + tor_socket_t s = TOR_INVALID_SOCKET; /* the socket we're going to make */ or_options_t const *options = get_options(); #if defined(HAVE_PWD_H) && defined(HAVE_SYS_UN_H) - struct passwd *pw = NULL; + const struct passwd *pw = NULL; #endif uint16_t usePort = 0, gotPort = 0; int start_reading = 0; @@ -1003,7 +1038,7 @@ connection_listener_new(const struct sockaddr *listensockaddr, log_notice(LD_NET, "Opening %s on %s", conn_type_to_string(type), fmt_addrport(&addr, usePort)); - s = tor_open_socket(tor_addr_family(&addr), + s = tor_open_socket_nonblocking(tor_addr_family(&addr), is_tcp ? SOCK_STREAM : SOCK_DGRAM, is_tcp ? IPPROTO_TCP: IPPROTO_UDP); if (!SOCKET_OK(s)) { @@ -1012,7 +1047,27 @@ connection_listener_new(const struct sockaddr *listensockaddr, goto err; } - make_socket_reuseable(s); + if (make_socket_reuseable(s) < 0) { + log_warn(LD_NET, "Error setting SO_REUSEADDR flag on %s: %s", + conn_type_to_string(type), + tor_socket_strerror(errno)); + } + +#if defined USE_TRANSPARENT && defined(IP_TRANSPARENT) + if (options->TransProxyType_parsed == TPT_TPROXY && + type == CONN_TYPE_AP_TRANS_LISTENER) { + int one = 1; + if (setsockopt(s, SOL_IP, IP_TRANSPARENT, &one, sizeof(one)) < 0) { + const char *extra = ""; + int e = tor_socket_errno(s); + if (e == EPERM) + extra = "TransTPROXY requires root privileges or similar" + " capabilities."; + log_warn(LD_NET, "Error setting IP_TRANSPARENT flag: %s.%s", + tor_socket_strerror(e), extra); + } + } +#endif #ifdef IPV6_V6ONLY if (listensockaddr->sa_family == AF_INET6) { @@ -1025,7 +1080,7 @@ connection_listener_new(const struct sockaddr *listensockaddr, /* We need to set IPV6_V6ONLY so that this socket can't get used for * IPv4 connections. */ if (setsockopt(s,IPPROTO_IPV6, IPV6_V6ONLY, - (void*)&one, sizeof(one))<0) { + (void*)&one, sizeof(one)) < 0) { int e = tor_socket_errno(s); log_warn(LD_NET, "Error setting IPV6_V6ONLY flag: %s", tor_socket_strerror(e)); @@ -1041,7 +1096,6 @@ connection_listener_new(const struct sockaddr *listensockaddr, helpfulhint = ". Is Tor already running?"; log_warn(LD_NET, "Could not bind to %s:%u: %s%s", address, usePort, tor_socket_strerror(e), helpfulhint); - tor_close_socket(s); goto err; } @@ -1049,7 +1103,6 @@ connection_listener_new(const struct sockaddr *listensockaddr, if (tor_listen(s) < 0) { log_warn(LD_NET, "Could not listen on %s:%u: %s", address, usePort, tor_socket_strerror(tor_socket_errno(s))); - tor_close_socket(s); goto err; } } @@ -1089,7 +1142,7 @@ connection_listener_new(const struct sockaddr *listensockaddr, strerror(errno)); goto err; } - s = tor_open_socket(AF_UNIX, SOCK_STREAM, 0); + s = tor_open_socket_nonblocking(AF_UNIX, SOCK_STREAM, 0); if (! SOCKET_OK(s)) { log_warn(LD_NET,"Socket creation failed: %s.", strerror(errno)); goto err; @@ -1098,21 +1151,18 @@ connection_listener_new(const struct sockaddr *listensockaddr, if (bind(s, listensockaddr, (socklen_t)sizeof(struct sockaddr_un)) == -1) { log_warn(LD_NET,"Bind to %s failed: %s.", address, tor_socket_strerror(tor_socket_errno(s))); - tor_close_socket(s); goto err; } #ifdef HAVE_PWD_H if (options->User) { - pw = getpwnam(options->User); + pw = tor_getpwnam(options->User); if (pw == NULL) { log_warn(LD_NET,"Unable to chown() %s socket: user %s not found.", address, options->User); - tor_close_socket(s); goto err; } else if (chown(address, pw->pw_uid, pw->pw_gid) < 0) { log_warn(LD_NET,"Unable to chown() %s socket: %s.", address, strerror(errno)); - tor_close_socket(s); goto err; } } @@ -1122,35 +1172,29 @@ connection_listener_new(const struct sockaddr *listensockaddr, * platforms. */ if (chmod(address, 0660) < 0) { log_warn(LD_FS,"Unable to make %s group-writable.", address); - tor_close_socket(s); goto err; } } - if (listen(s,SOMAXCONN) < 0) { + if (listen(s, SOMAXCONN) < 0) { log_warn(LD_NET, "Could not listen on %s: %s", address, tor_socket_strerror(tor_socket_errno(s))); - tor_close_socket(s); goto err; } #else (void)options; #endif /* HAVE_SYS_UN_H */ } else { - log_err(LD_BUG,"Got unexpected address family %d.", - listensockaddr->sa_family); - tor_assert(0); - } - - if (set_socket_nonblocking(s) == -1) { - tor_close_socket(s); - goto err; + log_err(LD_BUG, "Got unexpected address family %d.", + listensockaddr->sa_family); + tor_assert(0); } lis_conn = listener_connection_new(type, listensockaddr->sa_family); conn = TO_CONN(lis_conn); conn->socket_family = listensockaddr->sa_family; conn->s = s; + s = TOR_INVALID_SOCKET; /* Prevent double-close */ conn->address = tor_strdup(address); conn->port = gotPort; tor_addr_copy(&conn->addr, &addr); @@ -1186,7 +1230,6 @@ connection_listener_new(const struct sockaddr *listensockaddr, if (connection_add(conn) < 0) { /* no space, forget it */ log_warn(LD_NET,"connection_add for listener failed. Giving up."); - connection_free(conn); goto err; } @@ -1205,6 +1248,11 @@ connection_listener_new(const struct sockaddr *listensockaddr, return conn; err: + if (SOCKET_OK(s)) + tor_close_socket(s); + if (conn) + connection_free(conn); + return NULL; } @@ -1289,7 +1337,7 @@ connection_handle_listener_read(connection_t *conn, int new_type) tor_assert((size_t)remotelen >= sizeof(struct sockaddr_in)); memset(&addrbuf, 0, sizeof(addrbuf)); - news = tor_accept_socket(conn->s,remote,&remotelen); + news = tor_accept_socket_nonblocking(conn->s,remote,&remotelen); if (!SOCKET_OK(news)) { /* accept() error */ int e = tor_socket_errno(conn->s); if (ERRNO_IS_ACCEPT_EAGAIN(e)) { @@ -1308,8 +1356,15 @@ connection_handle_listener_read(connection_t *conn, int new_type) "Connection accepted on socket %d (child of fd %d).", (int)news,(int)conn->s); - make_socket_reuseable(news); - if (set_socket_nonblocking(news) == -1) { + if (make_socket_reuseable(news) < 0) { + if (tor_socket_errno(news) == EINVAL) { + /* This can happen on OSX if we get a badly timed shutdown. */ + log_debug(LD_NET, "make_socket_reuseable returned EINVAL"); + } else { + log_warn(LD_NET, "Error setting SO_REUSEADDR flag on %s: %s", + conn_type_to_string(new_type), + tor_socket_strerror(errno)); + } tor_close_socket(news); return 0; } @@ -1367,11 +1422,17 @@ connection_handle_listener_read(connection_t *conn, int new_type) TO_ENTRY_CONN(newconn)->socks_request->socks_prefer_no_auth = TO_LISTENER_CONN(conn)->socks_prefer_no_auth; } + if (new_type == CONN_TYPE_CONTROL) { + log_notice(LD_CONTROL, "New control connection opened from %s.", + fmt_and_decorate_addr(&addr)); + } } else if (conn->socket_family == AF_UNIX) { /* For now only control ports can be Unix domain sockets * and listeners at the same time */ tor_assert(conn->type == CONN_TYPE_CONTROL_LISTENER); + tor_assert(new_type == CONN_TYPE_CONTROL); + log_notice(LD_CONTROL, "New control connection opened."); newconn = connection_new(new_type, conn->socket_family); newconn->s = news; @@ -1411,6 +1472,9 @@ connection_init_accepted_conn(connection_t *conn, connection_start_reading(conn); switch (conn->type) { + case CONN_TYPE_EXT_OR: + /* Initiate Extended ORPort authentication. */ + return connection_ext_or_start_auth(TO_OR_CONN(conn)); case CONN_TYPE_OR: control_event_or_conn_status(TO_OR_CONN(conn), OR_CONN_EVENT_NEW, 0); rv = connection_tls_start_handshake(TO_OR_CONN(conn), 1); @@ -1504,7 +1568,7 @@ connection_connect(connection_t *conn, const char *address, return -1; } - s = tor_open_socket(protocol_family,SOCK_STREAM,IPPROTO_TCP); + s = tor_open_socket_nonblocking(protocol_family,SOCK_STREAM,IPPROTO_TCP); if (! SOCKET_OK(s)) { *socket_error = tor_socket_errno(-1); log_warn(LD_NET,"Error creating network socket: %s", @@ -1512,7 +1576,10 @@ connection_connect(connection_t *conn, const char *address, return -1; } - make_socket_reuseable(s); + if (make_socket_reuseable(s) < 0) { + log_warn(LD_NET, "Error setting SO_REUSEADDR flag on new connection: %s", + tor_socket_strerror(errno)); + } if (!tor_addr_is_loopback(addr)) { const tor_addr_t *ext_addr = NULL; @@ -1546,12 +1613,6 @@ connection_connect(connection_t *conn, const char *address, } } - if (set_socket_nonblocking(s) == -1) { - *socket_error = tor_socket_errno(s); - tor_close_socket(s); - return -1; - } - if (options->ConstrainedSockets) set_constrained_socket_buffers(s, (int)options->ConstrainedSockSize); @@ -1617,6 +1678,32 @@ connection_proxy_state_to_string(int state) return states[state]; } +/** Returns the global proxy type used by tor. Use this function for + * logging or high-level purposes, don't use it to fill the + * <b>proxy_type</b> field of or_connection_t; use the actual proxy + * protocol instead.*/ +static int +get_proxy_type(void) +{ + const or_options_t *options = get_options(); + + if (options->HTTPSProxy) + return PROXY_CONNECT; + else if (options->Socks4Proxy) + return PROXY_SOCKS4; + else if (options->Socks5Proxy) + return PROXY_SOCKS5; + else if (options->ClientTransportPlugin) + return PROXY_PLUGGABLE; + else + return PROXY_NONE; +} + +/* One byte for the version, one for the command, two for the + port, and four for the addr... and, one more for the + username NUL: */ +#define SOCKS4_STANDARD_BUFFER_SIZE (1 + 1 + 2 + 4 + 1) + /** Write a proxy request of <b>type</b> (socks4, socks5, https) to conn * for conn->addr:conn->port, authenticating with the auth details given * in the configuration (if available). SOCKS 5 and HTTP CONNECT proxies @@ -1671,17 +1758,45 @@ connection_proxy_connect(connection_t *conn, int type) } case PROXY_SOCKS4: { - unsigned char buf[9]; + unsigned char *buf; uint16_t portn; uint32_t ip4addr; + size_t buf_size = 0; + char *socks_args_string = NULL; - /* Send a SOCKS4 connect request with empty user id */ + /* Send a SOCKS4 connect request */ if (tor_addr_family(&conn->addr) != AF_INET) { log_warn(LD_NET, "SOCKS4 client is incompatible with IPv6"); return -1; } + { /* If we are here because we are trying to connect to a + pluggable transport proxy, check if we have any SOCKS + arguments to transmit. If we do, compress all arguments to + a single string in 'socks_args_string': */ + + if (get_proxy_type() == PROXY_PLUGGABLE) { + socks_args_string = + pt_get_socks_args_for_proxy_addrport(&conn->addr, conn->port); + if (socks_args_string) + log_debug(LD_NET, "Sending out '%s' as our SOCKS argument string.", + socks_args_string); + } + } + + { /* Figure out the buffer size we need for the SOCKS message: */ + + buf_size = SOCKS4_STANDARD_BUFFER_SIZE; + + /* If we have a SOCKS argument string, consider its size when + calculating the buffer size: */ + if (socks_args_string) + buf_size += strlen(socks_args_string); + } + + buf = tor_malloc_zero(buf_size); + ip4addr = tor_addr_to_ipv4n(&conn->addr); portn = htons(conn->port); @@ -1689,9 +1804,23 @@ connection_proxy_connect(connection_t *conn, int type) buf[1] = SOCKS_COMMAND_CONNECT; /* command */ memcpy(buf + 2, &portn, 2); /* port */ memcpy(buf + 4, &ip4addr, 4); /* addr */ - buf[8] = 0; /* userid (empty) */ - connection_write_to_buf((char *)buf, sizeof(buf), conn); + /* Next packet field is the userid. If we have pluggable + transport SOCKS arguments, we have to embed them + there. Otherwise, we use an empty userid. */ + if (socks_args_string) { /* place the SOCKS args string: */ + tor_assert(strlen(socks_args_string) > 0); + tor_assert(buf_size >= + SOCKS4_STANDARD_BUFFER_SIZE + strlen(socks_args_string)); + strlcpy((char *)buf + 8, socks_args_string, buf_size - 8); + tor_free(socks_args_string); + } else { + buf[8] = 0; /* no userid */ + } + + connection_write_to_buf((char *)buf, buf_size, conn); + tor_free(buf); + conn->proxy_state = PROXY_SOCKS4_WANT_CONNECT_OK; break; } @@ -1703,8 +1832,13 @@ connection_proxy_connect(connection_t *conn, int type) buf[0] = 5; /* version */ + /* We have to use SOCKS5 authentication, if we have a + Socks5ProxyUsername or if we want to pass arguments to our + pluggable transport proxy: */ + if ((options->Socks5ProxyUsername) || + (get_proxy_type() == PROXY_PLUGGABLE && + (get_socks_args_by_bridge_addrport(&conn->addr, conn->port)))) { /* number of auth methods */ - if (options->Socks5ProxyUsername) { buf[1] = 2; buf[2] = 0x00; /* no authentication */ buf[3] = 0x02; /* rfc1929 Username/Passwd auth */ @@ -1898,15 +2032,49 @@ connection_read_proxy_handshake(connection_t *conn) unsigned char buf[1024]; size_t reqsize, usize, psize; const char *user, *pass; + char *socks_args_string = NULL; + + if (get_proxy_type() == PROXY_PLUGGABLE) { + socks_args_string = + pt_get_socks_args_for_proxy_addrport(&conn->addr, conn->port); + if (!socks_args_string) { + log_warn(LD_NET, "Could not create SOCKS args string."); + ret = -1; + break; + } + + log_debug(LD_NET, "SOCKS5 arguments: %s", socks_args_string); + tor_assert(strlen(socks_args_string) > 0); + tor_assert(strlen(socks_args_string) <= MAX_SOCKS5_AUTH_SIZE_TOTAL); + + if (strlen(socks_args_string) > MAX_SOCKS5_AUTH_FIELD_SIZE) { + user = socks_args_string; + usize = MAX_SOCKS5_AUTH_FIELD_SIZE; + pass = socks_args_string + MAX_SOCKS5_AUTH_FIELD_SIZE; + psize = strlen(socks_args_string) - MAX_SOCKS5_AUTH_FIELD_SIZE; + } else { + user = socks_args_string; + usize = strlen(socks_args_string); + pass = "\0"; + psize = 1; + } + } else if (get_options()->Socks5ProxyUsername) { + user = get_options()->Socks5ProxyUsername; + pass = get_options()->Socks5ProxyPassword; + tor_assert(user && pass); + usize = strlen(user); + psize = strlen(pass); + } else { + log_err(LD_BUG, "We entered %s for no reason!", __func__); + tor_fragile_assert(); + ret = -1; + break; + } - user = get_options()->Socks5ProxyUsername; - pass = get_options()->Socks5ProxyPassword; - tor_assert(user && pass); - - /* XXX len of user and pass must be <= 255 !!! */ - usize = strlen(user); - psize = strlen(pass); - tor_assert(usize <= 255 && psize <= 255); + /* Username and password lengths should have been checked + above and during torrc parsing. */ + tor_assert(usize <= MAX_SOCKS5_AUTH_FIELD_SIZE && + psize <= MAX_SOCKS5_AUTH_FIELD_SIZE); reqsize = 3 + usize + psize; buf[0] = 1; /* negotiation version */ @@ -1915,6 +2083,9 @@ connection_read_proxy_handshake(connection_t *conn) buf[2 + usize] = psize; memcpy(buf + 3 + usize, pass, psize); + if (socks_args_string) + tor_free(socks_args_string); + connection_write_to_buf((char *)buf, reqsize, conn); conn->proxy_state = PROXY_SOCKS5_WANT_AUTH_RFC1929_OK; @@ -2072,7 +2243,7 @@ retry_listener_ports(smartlist_t *old_conns, if (listensockaddr) { conn = connection_listener_new(listensockaddr, listensocklen, - port->type, address, port); + port->type, address, port); tor_free(listensockaddr); tor_free(address); } else { @@ -2184,6 +2355,20 @@ connection_mark_all_noncontrol_connections(void) connection_mark_unattached_ap(TO_ENTRY_CONN(conn), END_STREAM_REASON_HIBERNATING); break; + case CONN_TYPE_OR: + { + or_connection_t *orconn = TO_OR_CONN(conn); + if (orconn->chan) { + connection_or_close_normally(orconn, 0); + } else { + /* + * There should have been one, but mark for close and hope + * for the best.. + */ + connection_mark_for_close(conn); + } + } + break; default: connection_mark_for_close(conn); break; @@ -2358,9 +2543,8 @@ connection_bucket_write_limit(connection_t *conn, time_t now) * shouldn't send <b>attempt</b> bytes of low-priority directory stuff * out to <b>conn</b>. Else return 0. - * Priority is 1 for v1 requests (directories and running-routers), - * and 2 for v2 requests (statuses and descriptors). But see FFFF in - * directory_handle_command_get() for why we don't use priority 2 yet. + * Priority was 1 for v1 requests (directories and running-routers), + * and 2 for v2 requests and later (statuses and descriptors). * * There are a lot of parameters we could use here: * - global_relayed_write_bucket. Low is bad. @@ -2466,7 +2650,58 @@ record_num_bytes_transferred(connection_t *conn, } #endif +/** Helper: convert given <b>tvnow</b> time value to milliseconds since + * midnight. */ +static uint32_t +msec_since_midnight(const struct timeval *tvnow) +{ + return (uint32_t)(((tvnow->tv_sec % 86400L) * 1000L) + + ((uint32_t)tvnow->tv_usec / (uint32_t)1000L)); +} + +/** Helper: return the time in milliseconds since <b>last_empty_time</b> + * when a bucket ran empty that previously had <b>tokens_before</b> tokens + * now has <b>tokens_after</b> tokens after refilling at timestamp + * <b>tvnow</b>, capped at <b>milliseconds_elapsed</b> milliseconds since + * last refilling that bucket. Return 0 if the bucket has not been empty + * since the last refill or has not been refilled. */ +uint32_t +bucket_millis_empty(int tokens_before, uint32_t last_empty_time, + int tokens_after, int milliseconds_elapsed, + const struct timeval *tvnow) +{ + uint32_t result = 0, refilled; + if (tokens_before <= 0 && tokens_after > tokens_before) { + refilled = msec_since_midnight(tvnow); + result = (uint32_t)((refilled + 86400L * 1000L - last_empty_time) % + (86400L * 1000L)); + if (result > (uint32_t)milliseconds_elapsed) + result = (uint32_t)milliseconds_elapsed; + } + return result; +} + +/** Check if a bucket which had <b>tokens_before</b> tokens and which got + * <b>tokens_removed</b> tokens removed at timestamp <b>tvnow</b> has run + * out of tokens, and if so, note the milliseconds since midnight in + * <b>timestamp_var</b> for the next TB_EMPTY event. */ +void +connection_buckets_note_empty_ts(uint32_t *timestamp_var, + int tokens_before, size_t tokens_removed, + const struct timeval *tvnow) +{ + if (tokens_before > 0 && (uint32_t)tokens_before <= tokens_removed) + *timestamp_var = msec_since_midnight(tvnow); +} + #ifndef USE_BUFFEREVENTS +/** Last time at which the global or relay buckets were emptied in msec + * since midnight. */ +static uint32_t global_relayed_read_emptied = 0, + global_relayed_write_emptied = 0, + global_read_emptied = 0, + global_write_emptied = 0; + /** We just read <b>num_read</b> and wrote <b>num_written</b> bytes * onto <b>conn</b>. Decrement buckets appropriately. */ static void @@ -2489,6 +2724,30 @@ connection_buckets_decrement(connection_t *conn, time_t now, if (!connection_is_rate_limited(conn)) return; /* local IPs are free */ + /* If one or more of our token buckets ran dry just now, note the + * timestamp for TB_EMPTY events. */ + if (get_options()->TestingEnableTbEmptyEvent) { + struct timeval tvnow; + tor_gettimeofday_cached(&tvnow); + if (connection_counts_as_relayed_traffic(conn, now)) { + connection_buckets_note_empty_ts(&global_relayed_read_emptied, + global_relayed_read_bucket, num_read, &tvnow); + connection_buckets_note_empty_ts(&global_relayed_write_emptied, + global_relayed_write_bucket, num_written, &tvnow); + } + connection_buckets_note_empty_ts(&global_read_emptied, + global_read_bucket, num_read, &tvnow); + connection_buckets_note_empty_ts(&global_write_emptied, + global_write_bucket, num_written, &tvnow); + if (connection_speaks_cells(conn) && conn->state == OR_CONN_STATE_OPEN) { + or_connection_t *or_conn = TO_OR_CONN(conn); + connection_buckets_note_empty_ts(&or_conn->read_emptied_time, + or_conn->read_bucket, num_read, &tvnow); + connection_buckets_note_empty_ts(&or_conn->write_emptied_time, + or_conn->write_bucket, num_written, &tvnow); + } + } + if (connection_counts_as_relayed_traffic(conn, now)) { global_relayed_read_bucket -= (int)num_read; global_relayed_write_bucket -= (int)num_written; @@ -2508,6 +2767,9 @@ connection_consider_empty_read_buckets(connection_t *conn) { const char *reason; + if (!connection_is_rate_limited(conn)) + return; /* Always okay. */ + if (global_read_bucket <= 0) { reason = "global read bucket exhausted. Pausing."; } else if (connection_counts_as_relayed_traffic(conn, approx_time()) && @@ -2520,9 +2782,6 @@ connection_consider_empty_read_buckets(connection_t *conn) } else return; /* all good, no need to stop it */ - if (conn->type == CONN_TYPE_CPUWORKER) - return; /* Always okay. */ - LOG_FN_CONN(conn, (LOG_DEBUG, LD_NET, "%s", reason)); conn->read_blocked_on_bw = 1; connection_stop_reading(conn); @@ -2535,6 +2794,9 @@ connection_consider_empty_write_buckets(connection_t *conn) { const char *reason; + if (!connection_is_rate_limited(conn)) + return; /* Always okay. */ + if (global_write_bucket <= 0) { reason = "global write bucket exhausted. Pausing."; } else if (connection_counts_as_relayed_traffic(conn, approx_time()) && @@ -2547,9 +2809,6 @@ connection_consider_empty_write_buckets(connection_t *conn) } else return; /* all good, no need to stop it */ - if (conn->type == CONN_TYPE_CPUWORKER) - return; /* Always okay. */ - LOG_FN_CONN(conn, (LOG_DEBUG, LD_NET, "%s", reason)); conn->write_blocked_on_bw = 1; connection_stop_writing(conn); @@ -2609,6 +2868,12 @@ connection_bucket_refill(int milliseconds_elapsed, time_t now) smartlist_t *conns = get_connection_array(); int bandwidthrate, bandwidthburst, relayrate, relayburst; + int prev_global_read = global_read_bucket; + int prev_global_write = global_write_bucket; + int prev_relay_read = global_relayed_read_bucket; + int prev_relay_write = global_relayed_write_bucket; + struct timeval tvnow; /*< Only used if TB_EMPTY events are enabled. */ + bandwidthrate = (int)options->BandwidthRate; bandwidthburst = (int)options->BandwidthBurst; @@ -2643,12 +2908,42 @@ connection_bucket_refill(int milliseconds_elapsed, time_t now) milliseconds_elapsed, "global_relayed_write_bucket"); + /* If buckets were empty before and have now been refilled, tell any + * interested controllers. */ + if (get_options()->TestingEnableTbEmptyEvent) { + uint32_t global_read_empty_time, global_write_empty_time, + relay_read_empty_time, relay_write_empty_time; + tor_gettimeofday_cached(&tvnow); + global_read_empty_time = bucket_millis_empty(prev_global_read, + global_read_emptied, global_read_bucket, + milliseconds_elapsed, &tvnow); + global_write_empty_time = bucket_millis_empty(prev_global_write, + global_write_emptied, global_write_bucket, + milliseconds_elapsed, &tvnow); + control_event_tb_empty("GLOBAL", global_read_empty_time, + global_write_empty_time, milliseconds_elapsed); + relay_read_empty_time = bucket_millis_empty(prev_relay_read, + global_relayed_read_emptied, + global_relayed_read_bucket, + milliseconds_elapsed, &tvnow); + relay_write_empty_time = bucket_millis_empty(prev_relay_write, + global_relayed_write_emptied, + global_relayed_write_bucket, + milliseconds_elapsed, &tvnow); + control_event_tb_empty("RELAY", relay_read_empty_time, + relay_write_empty_time, milliseconds_elapsed); + } + /* refill the per-connection buckets */ SMARTLIST_FOREACH_BEGIN(conns, connection_t *, conn) { if (connection_speaks_cells(conn)) { or_connection_t *or_conn = TO_OR_CONN(conn); int orbandwidthrate = or_conn->bandwidthrate; int orbandwidthburst = or_conn->bandwidthburst; + + int prev_conn_read = or_conn->read_bucket; + int prev_conn_write = or_conn->write_bucket; + if (connection_bucket_should_increase(or_conn->read_bucket, or_conn)) { connection_bucket_refill_helper(&or_conn->read_bucket, orbandwidthrate, @@ -2663,6 +2958,27 @@ connection_bucket_refill(int milliseconds_elapsed, time_t now) milliseconds_elapsed, "or_conn->write_bucket"); } + + /* If buckets were empty before and have now been refilled, tell any + * interested controllers. */ + if (get_options()->TestingEnableTbEmptyEvent) { + char *bucket; + uint32_t conn_read_empty_time, conn_write_empty_time; + tor_asprintf(&bucket, "ORCONN ID="U64_FORMAT, + U64_PRINTF_ARG(or_conn->base_.global_identifier)); + conn_read_empty_time = bucket_millis_empty(prev_conn_read, + or_conn->read_emptied_time, + or_conn->read_bucket, + milliseconds_elapsed, &tvnow); + conn_write_empty_time = bucket_millis_empty(prev_conn_write, + or_conn->write_emptied_time, + or_conn->write_bucket, + milliseconds_elapsed, &tvnow); + control_event_tb_empty(bucket, conn_read_empty_time, + conn_write_empty_time, + milliseconds_elapsed); + tor_free(bucket); + } } if (conn->read_blocked_on_bw == 1 /* marked to turn reading back on now */ @@ -2819,6 +3135,8 @@ connection_handle_read_impl(connection_t *conn) switch (conn->type) { case CONN_TYPE_OR_LISTENER: return connection_handle_listener_read(conn, CONN_TYPE_OR); + case CONN_TYPE_EXT_OR_LISTENER: + return connection_handle_listener_read(conn, CONN_TYPE_EXT_OR); case CONN_TYPE_AP_LISTENER: case CONN_TYPE_AP_TRANS_LISTENER: case CONN_TYPE_AP_NATD_LISTENER: @@ -3071,14 +3389,37 @@ connection_read_to_buf(connection_t *conn, ssize_t *max_to_read, /* change *max_to_read */ *max_to_read = at_most - n_read; - /* Update edge_conn->n_read */ + /* Update edge_conn->n_read and ocirc->n_read_circ_bw */ if (conn->type == CONN_TYPE_AP) { edge_connection_t *edge_conn = TO_EDGE_CONN(conn); + circuit_t *circ = circuit_get_by_edge_conn(edge_conn); + origin_circuit_t *ocirc; + /* Check for overflow: */ if (PREDICT_LIKELY(UINT32_MAX - edge_conn->n_read > n_read)) edge_conn->n_read += (int)n_read; else edge_conn->n_read = UINT32_MAX; + + if (circ && CIRCUIT_IS_ORIGIN(circ)) { + ocirc = TO_ORIGIN_CIRCUIT(circ); + if (PREDICT_LIKELY(UINT32_MAX - ocirc->n_read_circ_bw > n_read)) + ocirc->n_read_circ_bw += (int)n_read; + else + ocirc->n_read_circ_bw = UINT32_MAX; + } + } + + /* If CONN_BW events are enabled, update conn->n_read_conn_bw for + * OR/DIR/EXIT connections, checking for overflow. */ + if (get_options()->TestingEnableConnBwEvent && + (conn->type == CONN_TYPE_OR || + conn->type == CONN_TYPE_DIR || + conn->type == CONN_TYPE_EXIT)) { + if (PREDICT_LIKELY(UINT32_MAX - conn->n_read_conn_bw > n_read)) + conn->n_read_conn_bw += (int)n_read; + else + conn->n_read_conn_bw = UINT32_MAX; } } @@ -3331,8 +3672,8 @@ connection_outbuf_too_full(connection_t *conn) /** Try to flush more bytes onto <b>conn</b>-\>s. * - * This function gets called either from conn_write() in main.c - * when poll() has declared that conn wants to write, or below + * This function gets called either from conn_write_callback() in main.c + * when libevent tells us that conn wants to write, or below * from connection_write_to_buf() when an entire TLS record is ready. * * Update <b>conn</b>-\>timestamp_lastwritten to now, and call flush_buf @@ -3518,12 +3859,34 @@ connection_handle_write_impl(connection_t *conn, int force) if (n_written && conn->type == CONN_TYPE_AP) { edge_connection_t *edge_conn = TO_EDGE_CONN(conn); + circuit_t *circ = circuit_get_by_edge_conn(edge_conn); + origin_circuit_t *ocirc; /* Check for overflow: */ if (PREDICT_LIKELY(UINT32_MAX - edge_conn->n_written > n_written)) edge_conn->n_written += (int)n_written; else edge_conn->n_written = UINT32_MAX; + + if (circ && CIRCUIT_IS_ORIGIN(circ)) { + ocirc = TO_ORIGIN_CIRCUIT(circ); + if (PREDICT_LIKELY(UINT32_MAX - ocirc->n_written_circ_bw > n_written)) + ocirc->n_written_circ_bw += (int)n_written; + else + ocirc->n_written_circ_bw = UINT32_MAX; + } + } + + /* If CONN_BW events are enabled, update conn->n_written_conn_bw for + * OR/DIR/EXIT connections, checking for overflow. */ + if (n_written && get_options()->TestingEnableConnBwEvent && + (conn->type == CONN_TYPE_OR || + conn->type == CONN_TYPE_DIR || + conn->type == CONN_TYPE_EXIT)) { + if (PREDICT_LIKELY(UINT32_MAX - conn->n_written_conn_bw > n_written)) + conn->n_written_conn_bw += (int)n_written; + else + conn->n_written_conn_bw = UINT32_MAX; } connection_buckets_decrement(conn, approx_time(), n_read, n_written); @@ -3609,9 +3972,9 @@ connection_flush(connection_t *conn) * it all, so we don't end up with many megabytes of controller info queued at * once. */ -void -connection_write_to_buf_impl_(const char *string, size_t len, - connection_t *conn, int zlib) +MOCK_IMPL(void, +connection_write_to_buf_impl_,(const char *string, size_t len, + connection_t *conn, int zlib)) { /* XXXX This function really needs to return -1 on failure. */ int r; @@ -3656,6 +4019,12 @@ connection_write_to_buf_impl_(const char *string, size_t len, "write_to_buf failed. Closing circuit (fd %d).", (int)conn->s); circuit_mark_for_close(circuit_get_by_edge_conn(TO_EDGE_CONN(conn)), END_CIRC_REASON_INTERNAL); + } else if (conn->type == CONN_TYPE_OR) { + or_connection_t *orconn = TO_OR_CONN(conn); + log_warn(LD_NET, + "write_to_buf failed on an orconn; notifying of error " + "(fd %d)", (int)(conn->s)); + connection_or_close_for_error(orconn, 0); } else { log_warn(LD_NET, "write_to_buf failed. Closing connection (fd %d).", @@ -3830,20 +4199,29 @@ connection_dir_get_by_purpose_and_resource(int purpose, return NULL; } -/** Return an open, non-marked connection of a given type and purpose, or NULL - * if no such connection exists. */ -connection_t * -connection_get_by_type_purpose(int type, int purpose) +/** Return 1 if there are any active OR connections apart from + * <b>this_conn</b>. + * + * We use this to guess if we should tell the controller that we + * didn't manage to connect to any of our bridges. */ +int +any_other_active_or_conns(const or_connection_t *this_conn) { smartlist_t *conns = get_connection_array(); - SMARTLIST_FOREACH(conns, connection_t *, conn, - { - if (conn->type == type && - !conn->marked_for_close && - (purpose == conn->purpose)) - return conn; - }); - return NULL; + SMARTLIST_FOREACH_BEGIN(conns, connection_t *, conn) { + if (conn == TO_CONN(this_conn)) { /* don't consider this conn */ + continue; + } + + if (conn->type == CONN_TYPE_OR && + !conn->marked_for_close) { + log_debug(LD_DIR, "%s: Found an OR connection: %s", + __func__, conn->address); + return 1; + } + } SMARTLIST_FOREACH_END(conn); + + return 0; } /** Return 1 if <b>conn</b> is a listener conn, else return 0. */ @@ -3851,6 +4229,7 @@ int connection_is_listener(connection_t *conn) { if (conn->type == CONN_TYPE_OR_LISTENER || + conn->type == CONN_TYPE_EXT_OR_LISTENER || conn->type == CONN_TYPE_AP_LISTENER || conn->type == CONN_TYPE_AP_TRANS_LISTENER || conn->type == CONN_TYPE_AP_DNS_LISTENER || @@ -3873,6 +4252,7 @@ connection_state_is_open(connection_t *conn) return 0; if ((conn->type == CONN_TYPE_OR && conn->state == OR_CONN_STATE_OPEN) || + (conn->type == CONN_TYPE_EXT_OR) || (conn->type == CONN_TYPE_AP && conn->state == AP_CONN_STATE_OPEN) || (conn->type == CONN_TYPE_EXIT && conn->state == EXIT_CONN_STATE_OPEN) || (conn->type == CONN_TYPE_CONTROL && @@ -4042,6 +4422,8 @@ connection_process_inbuf(connection_t *conn, int package_partial) switch (conn->type) { case CONN_TYPE_OR: return connection_or_process_inbuf(TO_OR_CONN(conn)); + case CONN_TYPE_EXT_OR: + return connection_ext_or_process_inbuf(TO_OR_CONN(conn)); case CONN_TYPE_EXIT: case CONN_TYPE_AP: return connection_edge_process_inbuf(TO_EDGE_CONN(conn), @@ -4102,6 +4484,8 @@ connection_finished_flushing(connection_t *conn) switch (conn->type) { case CONN_TYPE_OR: return connection_or_finished_flushing(TO_OR_CONN(conn)); + case CONN_TYPE_EXT_OR: + return connection_ext_or_finished_flushing(TO_OR_CONN(conn)); case CONN_TYPE_AP: case CONN_TYPE_EXIT: return connection_edge_finished_flushing(TO_EDGE_CONN(conn)); @@ -4157,6 +4541,7 @@ connection_reached_eof(connection_t *conn) { switch (conn->type) { case CONN_TYPE_OR: + case CONN_TYPE_EXT_OR: return connection_or_reached_eof(TO_OR_CONN(conn)); case CONN_TYPE_AP: case CONN_TYPE_EXIT: @@ -4243,6 +4628,7 @@ assert_connection_ok(connection_t *conn, time_t now) switch (conn->type) { case CONN_TYPE_OR: + case CONN_TYPE_EXT_OR: tor_assert(conn->magic == OR_CONNECTION_MAGIC); break; case CONN_TYPE_AP: @@ -4348,6 +4734,10 @@ assert_connection_ok(connection_t *conn, time_t now) tor_assert(conn->state >= OR_CONN_STATE_MIN_); tor_assert(conn->state <= OR_CONN_STATE_MAX_); break; + case CONN_TYPE_EXT_OR: + tor_assert(conn->state >= EXT_OR_CONN_STATE_MIN_); + tor_assert(conn->state <= EXT_OR_CONN_STATE_MAX_); + break; case CONN_TYPE_EXIT: tor_assert(conn->state >= EXIT_CONN_STATE_MIN_); tor_assert(conn->state <= EXIT_CONN_STATE_MAX_); @@ -4409,7 +4799,7 @@ get_proxy_addrport(tor_addr_t *addr, uint16_t *port, int *proxy_type, options->Bridges) { const transport_t *transport = NULL; int r; - r = find_transport_by_bridge_addrport(&conn->addr, conn->port, &transport); + r = get_transport_by_bridge_addrport(&conn->addr, conn->port, &transport); if (r<0) return -1; if (transport) { /* transport found */ @@ -4420,28 +4810,12 @@ get_proxy_addrport(tor_addr_t *addr, uint16_t *port, int *proxy_type, } } + tor_addr_make_unspec(addr); + *port = 0; *proxy_type = PROXY_NONE; return 0; } -/** Returns the global proxy type used by tor. */ -static int -get_proxy_type(void) -{ - const or_options_t *options = get_options(); - - if (options->HTTPSProxy) - return PROXY_CONNECT; - else if (options->Socks4Proxy) - return PROXY_SOCKS4; - else if (options->Socks5Proxy) - return PROXY_SOCKS5; - else if (options->ClientTransportPlugin) - return PROXY_PLUGGABLE; - else - return PROXY_NONE; -} - /** Log a failed connection to a proxy server. * <b>conn</b> is the connection we use the proxy server for. */ void @@ -4498,6 +4872,7 @@ connection_free_all(void) /* Unlink everything from the identity map. */ connection_or_clear_identity_map(); + connection_or_clear_ext_or_id_map(); /* Clear out our list of broken connections */ clear_broken_connection_map(0); diff --git a/src/or/connection.h b/src/or/connection.h index c78fe6e652..13dcbcd919 100644 --- a/src/or/connection.h +++ b/src/or/connection.h @@ -19,7 +19,7 @@ const char *conn_type_to_string(int type); const char *conn_state_to_string(int type, int state); dir_connection_t *dir_connection_new(int socket_family); -or_connection_t *or_connection_new(int socket_family); +or_connection_t *or_connection_new(int type, int socket_family); edge_connection_t *edge_connection_new(int type, int socket_family); entry_connection_t *entry_connection_new(int type, int socket_family); control_connection_t *control_connection_new(int socket_family); @@ -89,6 +89,14 @@ int connection_connect(connection_t *conn, const char *address, const tor_addr_t *addr, uint16_t port, int *socket_error); +/** Maximum size of information that we can fit into SOCKS5 username + or password fields. */ +#define MAX_SOCKS5_AUTH_FIELD_SIZE 255 + +/** Total maximum size of information that we can fit into SOCKS5 + username and password fields. */ +#define MAX_SOCKS5_AUTH_SIZE_TOTAL 2*MAX_SOCKS5_AUTH_FIELD_SIZE + int connection_proxy_connect(connection_t *conn, int type); int connection_read_proxy_handshake(connection_t *conn); void log_failed_proxy_connection(connection_t *conn); @@ -122,8 +130,8 @@ int connection_outbuf_too_full(connection_t *conn); int connection_handle_write(connection_t *conn, int force); int connection_flush(connection_t *conn); -void connection_write_to_buf_impl_(const char *string, size_t len, - connection_t *conn, int zlib); +MOCK_DECL(void, connection_write_to_buf_impl_, + (const char *string, size_t len, connection_t *conn, int zlib)); /* DOCDOC connection_write_to_buf */ static void connection_write_to_buf(const char *string, size_t len, connection_t *conn); @@ -170,7 +178,6 @@ connection_get_outbuf_len(connection_t *conn) connection_t *connection_get_by_global_id(uint64_t id); connection_t *connection_get_by_type(int type); -connection_t *connection_get_by_type_purpose(int type, int purpose); connection_t *connection_get_by_type_addr_port_purpose(int type, const tor_addr_t *addr, uint16_t port, int purpose); @@ -180,6 +187,8 @@ connection_t *connection_get_by_type_state_rendquery(int type, int state, dir_connection_t *connection_dir_get_by_purpose_and_resource( int state, const char *resource); +int any_other_active_or_conns(const or_connection_t *this_conn); + #define connection_speaks_cells(conn) ((conn)->type == CONN_TYPE_OR) int connection_is_listener(connection_t *conn); int connection_state_is_open(connection_t *conn); @@ -206,5 +215,18 @@ void connection_enable_rate_limiting(connection_t *conn); #define connection_type_uses_bufferevent(c) (0) #endif +#ifdef CONNECTION_PRIVATE +STATIC void connection_free_(connection_t *conn); + +/* Used only by connection.c and test*.c */ +uint32_t bucket_millis_empty(int tokens_before, uint32_t last_empty_time, + int tokens_after, int milliseconds_elapsed, + const struct timeval *tvnow); +void connection_buckets_note_empty_ts(uint32_t *timestamp_var, + int tokens_before, + size_t tokens_removed, + const struct timeval *tvnow); +#endif + #endif diff --git a/src/or/connection_edge.c b/src/or/connection_edge.c index 895c0f7f01..49f9ba4978 100644 --- a/src/or/connection_edge.c +++ b/src/or/connection_edge.c @@ -14,6 +14,7 @@ #include "addressmap.h" #include "buffers.h" #include "channel.h" +#include "circpathbias.h" #include "circuitlist.h" #include "circuituse.h" #include "config.h" @@ -61,19 +62,14 @@ static int connection_ap_process_natd(entry_connection_t *conn); static int connection_exit_connect_dir(edge_connection_t *exitconn); static int consider_plaintext_ports(entry_connection_t *conn, uint16_t port); static int connection_ap_supports_optimistic_data(const entry_connection_t *); -static void connection_ap_handshake_socks_resolved_addr( - entry_connection_t *conn, - const tor_addr_t *answer, - int ttl, - time_t expires); /** An AP stream has failed/finished. If it hasn't already sent back * a socks reply, send one now (based on endreason). Also set * has_sent_end to 1, and mark the conn. */ -void -connection_mark_unattached_ap_(entry_connection_t *conn, int endreason, - int line, const char *file) +MOCK_IMPL(void, +connection_mark_unattached_ap_,(entry_connection_t *conn, int endreason, + int line, const char *file)) { connection_t *base_conn = ENTRY_TO_CONN(conn); edge_connection_t *edge_conn = ENTRY_TO_EDGE_CONN(conn); @@ -412,7 +408,7 @@ connection_edge_finished_flushing(edge_connection_t *conn) * that the name resolution that led us to <b>addr</b> will be valid for * <b>ttl</b> seconds. Return -1 on error, or the number of bytes used on * success. */ -/* private */int +STATIC int connected_cell_format_payload(uint8_t *payload_out, const tor_addr_t *addr, uint32_t ttl) @@ -1395,35 +1391,48 @@ get_pf_socket(void) } #endif -/** Fetch the original destination address and port from a - * system-specific interface and put them into a - * socks_request_t as if they came from a socks request. - * - * Return -1 if an error prevents fetching the destination, - * else return 0. - */ +#if defined(TRANS_NETFILTER) || defined(TRANS_PF) +/** Try fill in the address of <b>req</b> from the socket configured + * with <b>conn</b>. */ static int -connection_ap_get_original_destination(entry_connection_t *conn, - socks_request_t *req) +destination_from_socket(entry_connection_t *conn, socks_request_t *req) { -#ifdef TRANS_NETFILTER - /* Linux 2.4+ */ struct sockaddr_storage orig_dst; socklen_t orig_dst_len = sizeof(orig_dst); tor_addr_t addr; +#ifdef TRANS_NETFILTER if (getsockopt(ENTRY_TO_CONN(conn)->s, SOL_IP, SO_ORIGINAL_DST, (struct sockaddr*)&orig_dst, &orig_dst_len) < 0) { int e = tor_socket_errno(ENTRY_TO_CONN(conn)->s); log_warn(LD_NET, "getsockopt() failed: %s", tor_socket_strerror(e)); return -1; } +#elif defined(TRANS_PF) + if (getsockname(ENTRY_TO_CONN(conn)->s, (struct sockaddr*)&orig_dst, + &orig_dst_len) < 0) { + int e = tor_socket_errno(ENTRY_TO_CONN(conn)->s); + log_warn(LD_NET, "getsockname() failed: %s", tor_socket_strerror(e)); + return -1; + } +#else + (void)conn; + (void)req; + log_warn(LD_BUG, "Unable to determine destination from socket."); + return -1; +#endif tor_addr_from_sockaddr(&addr, (struct sockaddr*)&orig_dst, &req->port); tor_addr_to_str(req->address, &addr, sizeof(req->address), 1); return 0; -#elif defined(TRANS_PF) +} +#endif + +#ifdef TRANS_PF +static int +destination_from_pf(entry_connection_t *conn, socks_request_t *req) +{ struct sockaddr_storage proxy_addr; socklen_t proxy_addr_len = sizeof(proxy_addr); struct sockaddr *proxy_sa = (struct sockaddr*) &proxy_addr; @@ -1439,6 +1448,21 @@ connection_ap_get_original_destination(entry_connection_t *conn, return -1; } +#ifdef __FreeBSD__ + if (get_options()->TransProxyType_parsed == TPT_IPFW) { + /* ipfw(8) is used and in this case getsockname returned the original + destination */ + if (tor_addr_from_sockaddr(&addr, proxy_sa, &req->port) < 0) { + tor_fragile_assert(); + return -1; + } + + tor_addr_to_str(req->address, &addr, sizeof(req->address), 0); + + return 0; + } +#endif + memset(&pnl, 0, sizeof(pnl)); pnl.proto = IPPROTO_TCP; pnl.direction = PF_OUT; @@ -1485,6 +1509,36 @@ connection_ap_get_original_destination(entry_connection_t *conn, req->port = ntohs(pnl.rdport); return 0; +} +#endif + +/** Fetch the original destination address and port from a + * system-specific interface and put them into a + * socks_request_t as if they came from a socks request. + * + * Return -1 if an error prevents fetching the destination, + * else return 0. + */ +static int +connection_ap_get_original_destination(entry_connection_t *conn, + socks_request_t *req) +{ +#ifdef TRANS_NETFILTER + return destination_from_socket(conn, req); +#elif defined(TRANS_PF) + const or_options_t *options = get_options(); + + if (options->TransProxyType_parsed == TPT_PF_DIVERT) + return destination_from_socket(conn, req); + + if (options->TransProxyType_parsed == TPT_DEFAULT) + return destination_from_pf(conn, req); + + (void)conn; + (void)req; + log_warn(LD_BUG, "Proxy destination determination mechanism %s unknown.", + options->TransProxyType); + return -1; #else (void)conn; (void)req; @@ -2064,7 +2118,7 @@ tell_controller_about_resolved_result(entry_connection_t *conn, * As connection_ap_handshake_socks_resolved, but take a tor_addr_t to send * as the answer. */ -static void +void connection_ap_handshake_socks_resolved_addr(entry_connection_t *conn, const tor_addr_t *answer, int ttl, @@ -2097,13 +2151,13 @@ connection_ap_handshake_socks_resolved_addr(entry_connection_t *conn, **/ /* XXXX the use of the ttl and expires fields is nutty. Let's make this * interface and those that use it less ugly. */ -void -connection_ap_handshake_socks_resolved(entry_connection_t *conn, +MOCK_IMPL(void, +connection_ap_handshake_socks_resolved,(entry_connection_t *conn, int answer_type, size_t answer_len, const uint8_t *answer, int ttl, - time_t expires) + time_t expires)) { char buf[384]; size_t replylen; @@ -2241,13 +2295,21 @@ connection_ap_handshake_socks_reply(entry_connection_t *conn, char *reply, endreason == END_STREAM_REASON_RESOURCELIMIT) { if (!conn->edge_.on_circuit || !CIRCUIT_IS_ORIGIN(conn->edge_.on_circuit)) { - // DNS remaps can trigger this. So can failed hidden service - // lookups. - log_info(LD_BUG, - "No origin circuit for successful SOCKS stream "U64_FORMAT - ". Reason: %d", - U64_PRINTF_ARG(ENTRY_TO_CONN(conn)->global_identifier), - endreason); + if (endreason != END_STREAM_REASON_RESOLVEFAILED) { + log_info(LD_BUG, + "No origin circuit for successful SOCKS stream "U64_FORMAT + ". Reason: %d", + U64_PRINTF_ARG(ENTRY_TO_CONN(conn)->global_identifier), + endreason); + } + /* + * Else DNS remaps and failed hidden service lookups can send us + * here with END_STREAM_REASON_RESOLVEFAILED; ignore it + * + * Perhaps we could make the test more precise; we can tell hidden + * services by conn->edge_.renddata != NULL; anything analogous for + * the DNS remap case? + */ } else { // XXX: Hrmm. It looks like optimistic data can't go through this // codepath, but someone should probably test it and make sure. @@ -2272,13 +2334,24 @@ connection_ap_handshake_socks_reply(entry_connection_t *conn, char *reply, /* leave version, destport, destip zero */ connection_write_to_buf(buf, SOCKS4_NETWORK_LEN, ENTRY_TO_CONN(conn)); } else if (conn->socks_request->socks_version == 5) { - buf[0] = 5; /* version 5 */ - buf[1] = (char)status; - buf[2] = 0; - buf[3] = 1; /* ipv4 addr */ - memset(buf+4,0,6); /* Set external addr/port to 0. - The spec doesn't seem to say what to do here. -RD */ - connection_write_to_buf(buf,10,ENTRY_TO_CONN(conn)); + size_t buf_len; + memset(buf,0,sizeof(buf)); + if (tor_addr_family(&conn->edge_.base_.addr) == AF_INET) { + buf[0] = 5; /* version 5 */ + buf[1] = (char)status; + buf[2] = 0; + buf[3] = 1; /* ipv4 addr */ + /* 4 bytes for the header, 2 bytes for the port, 4 for the address. */ + buf_len = 10; + } else { /* AF_INET6. */ + buf[0] = 5; /* version 5 */ + buf[1] = (char)status; + buf[2] = 0; + buf[3] = 4; /* ipv6 addr */ + /* 4 bytes for the header, 2 bytes for the port, 16 for the address. */ + buf_len = 22; + } + connection_write_to_buf(buf,buf_len,ENTRY_TO_CONN(conn)); } /* If socks_version isn't 4 or 5, don't send anything. * This can happen in the case of AP bridges. */ @@ -2294,7 +2367,7 @@ connection_ap_handshake_socks_reply(entry_connection_t *conn, char *reply, * Return -1 in the case where want to send a RELAY_END cell, and < -1 when * we don't. **/ -/* static */ int +STATIC int begin_cell_parse(const cell_t *cell, begin_cell_t *bcell, uint8_t *end_reason_out) { diff --git a/src/or/connection_edge.h b/src/or/connection_edge.h index ea284cbcfd..3c0e30a973 100644 --- a/src/or/connection_edge.h +++ b/src/or/connection_edge.h @@ -12,11 +12,14 @@ #ifndef TOR_CONNECTION_EDGE_H #define TOR_CONNECTION_EDGE_H +#include "testsupport.h" + #define connection_mark_unattached_ap(conn, endreason) \ connection_mark_unattached_ap_((conn), (endreason), __LINE__, SHORT_FILE__) -void connection_mark_unattached_ap_(entry_connection_t *conn, int endreason, - int line, const char *file); +MOCK_DECL(void,connection_mark_unattached_ap_, + (entry_connection_t *conn, int endreason, + int line, const char *file)); int connection_edge_reached_eof(edge_connection_t *conn); int connection_edge_process_inbuf(edge_connection_t *conn, int package_partial); @@ -42,12 +45,17 @@ entry_connection_t *connection_ap_make_link(connection_t *partner, void connection_ap_handshake_socks_reply(entry_connection_t *conn, char *reply, size_t replylen, int endreason); -void connection_ap_handshake_socks_resolved(entry_connection_t *conn, - int answer_type, - size_t answer_len, - const uint8_t *answer, - int ttl, - time_t expires); +MOCK_DECL(void,connection_ap_handshake_socks_resolved, + (entry_connection_t *conn, + int answer_type, + size_t answer_len, + const uint8_t *answer, + int ttl, + time_t expires)); +void connection_ap_handshake_socks_resolved_addr(entry_connection_t *conn, + const tor_addr_t *answer, + int ttl, + time_t expires); int connection_exit_begin_conn(cell_t *cell, circuit_t *circ); int connection_exit_begin_resolve(cell_t *cell, or_circuit_t *circ); @@ -130,9 +138,9 @@ typedef struct begin_cell_t { unsigned is_begindir : 1; } begin_cell_t; -int begin_cell_parse(const cell_t *cell, begin_cell_t *bcell, +STATIC int begin_cell_parse(const cell_t *cell, begin_cell_t *bcell, uint8_t *end_reason_out); -int connected_cell_format_payload(uint8_t *payload_out, +STATIC int connected_cell_format_payload(uint8_t *payload_out, const tor_addr_t *addr, uint32_t ttl); #endif diff --git a/src/or/connection_or.c b/src/or/connection_or.c index 8e7cd9ea51..16f87349fc 100644 --- a/src/or/connection_or.c +++ b/src/or/connection_or.c @@ -37,7 +37,7 @@ #include "rephist.h" #include "router.h" #include "routerlist.h" - +#include "ext_orport.h" #ifdef USE_BUFFEREVENTS #include <event2/bufferevent_ssl.h> #endif @@ -75,6 +75,10 @@ static void connection_or_handle_event_cb(struct bufferevent *bufev, * they form a linked list, with next_with_same_id as the next pointer. */ static digestmap_t *orconn_identity_map = NULL; +/** Global map between Extended ORPort identifiers and OR + * connections. */ +static digestmap_t *orconn_ext_or_id_map = NULL; + /** If conn is listed in orconn_identity_map, remove it, and clear * conn->identity_digest. Otherwise do nothing. */ void @@ -174,6 +178,71 @@ connection_or_set_identity_digest(or_connection_t *conn, const char *digest) #endif } +/** Remove the Extended ORPort identifier of <b>conn</b> from the + * global identifier list. Also, clear the identifier from the + * connection itself. */ +void +connection_or_remove_from_ext_or_id_map(or_connection_t *conn) +{ + or_connection_t *tmp; + if (!orconn_ext_or_id_map) + return; + if (!conn->ext_or_conn_id) + return; + + tmp = digestmap_remove(orconn_ext_or_id_map, conn->ext_or_conn_id); + if (!tor_digest_is_zero(conn->ext_or_conn_id)) + tor_assert(tmp == conn); + + memset(conn->ext_or_conn_id, 0, EXT_OR_CONN_ID_LEN); +} + +/** Return the connection whose ext_or_id is <b>id</b>. Return NULL if no such + * connection is found. */ +or_connection_t * +connection_or_get_by_ext_or_id(const char *id) +{ + if (!orconn_ext_or_id_map) + return NULL; + return digestmap_get(orconn_ext_or_id_map, id); +} + +/** Deallocate the global Extended ORPort identifier list */ +void +connection_or_clear_ext_or_id_map(void) +{ + digestmap_free(orconn_ext_or_id_map, NULL); + orconn_ext_or_id_map = NULL; +} + +/** Creates an Extended ORPort identifier for <b>conn</b> and deposits + * it into the global list of identifiers. */ +void +connection_or_set_ext_or_identifier(or_connection_t *conn) +{ + char random_id[EXT_OR_CONN_ID_LEN]; + or_connection_t *tmp; + + if (!orconn_ext_or_id_map) + orconn_ext_or_id_map = digestmap_new(); + + /* Remove any previous identifiers: */ + if (conn->ext_or_conn_id && !tor_digest_is_zero(conn->ext_or_conn_id)) + connection_or_remove_from_ext_or_id_map(conn); + + do { + crypto_rand(random_id, sizeof(random_id)); + } while (digestmap_get(orconn_ext_or_id_map, random_id)); + + if (!conn->ext_or_conn_id) + conn->ext_or_conn_id = tor_malloc_zero(EXT_OR_CONN_ID_LEN); + + memcpy(conn->ext_or_conn_id, random_id, EXT_OR_CONN_ID_LEN); + + tmp = digestmap_set(orconn_ext_or_id_map, random_id, conn); + tor_assert(!tmp); +} + /**************************************************************/ /** Map from a string describing what a non-open OR connection was doing when @@ -228,7 +297,7 @@ connection_or_get_state_description(or_connection_t *orconn, const char *conn_state; char tls_state[256]; - tor_assert(conn->type == CONN_TYPE_OR); + tor_assert(conn->type == CONN_TYPE_OR || conn->type == CONN_TYPE_EXT_OR); conn_state = conn_state_to_string(conn->type, conn->state); tor_tls_get_state_description(orconn->tls, tls_state, sizeof(tls_state)); @@ -645,7 +714,8 @@ connection_or_about_to_close(or_connection_t *or_conn) reason); if (!authdir_mode_tests_reachability(options)) control_event_bootstrap_problem( - orconn_end_reason_to_control_string(reason), reason); + orconn_end_reason_to_control_string(reason), + reason, or_conn); } } } else if (conn->hold_open_until_flushed) { @@ -756,6 +826,45 @@ connection_or_update_token_buckets(smartlist_t *conns, }); } +/** How long do we wait before killing non-canonical OR connections with no + * circuits? In Tor versions up to 0.2.1.25 and 0.2.2.12-alpha, we waited 15 + * minutes before cancelling these connections, which caused fast relays to + * accrue many many idle connections. Hopefully 3-4.5 minutes is low enough + * that it kills most idle connections, without being so low that we cause + * clients to bounce on and off. + * + * For canonical connections, the limit is higher, at 15-22.5 minutes. + * + * For each OR connection, we randomly add up to 50% extra to its idle_timeout + * field, to avoid exposing when exactly the last circuit closed. Since we're + * storing idle_timeout in a uint16_t, don't let these values get higher than + * 12 hours or so without revising connection_or_set_canonical and/or expanding + * idle_timeout. + */ +#define IDLE_OR_CONN_TIMEOUT_NONCANONICAL 180 +#define IDLE_OR_CONN_TIMEOUT_CANONICAL 900 + +/* Mark <b>or_conn</b> as canonical if <b>is_canonical</b> is set, and + * non-canonical otherwise. Adjust idle_timeout accordingly. + */ +void +connection_or_set_canonical(or_connection_t *or_conn, + int is_canonical) +{ + const unsigned int timeout_base = is_canonical ? + IDLE_OR_CONN_TIMEOUT_CANONICAL : IDLE_OR_CONN_TIMEOUT_NONCANONICAL; + + if (bool_eq(is_canonical, or_conn->is_canonical) && + or_conn->idle_timeout != 0) { + /* Don't recalculate an existing idle_timeout unless the canonical + * status changed. */ + return; + } + + or_conn->is_canonical = !! is_canonical; /* force to a 1-bit boolean */ + or_conn->idle_timeout = timeout_base + crypto_rand_int(timeout_base / 2); +} + /** If we don't necessarily know the router we're connecting to, but we * have an addr/port/id_digest, then fill in as much as we can. Start * by checking to see if this describes a router we know. @@ -780,7 +889,7 @@ connection_or_init_conn_from_address(or_connection_t *conn, /* XXXX proposal 186 is making this more complex. For now, a conn is canonical when it uses the _preferred_ address. */ if (tor_addr_eq(&conn->base_.addr, &node_ap.addr)) - conn->is_canonical = 1; + connection_or_set_canonical(conn, 1); if (!started_here) { /* Override the addr/port, so our log messages will make sense. * This is dangerous, since if we ever try looking up a conn by @@ -1008,7 +1117,7 @@ connection_or_connect_failed(or_connection_t *conn, { control_event_or_conn_status(conn, OR_CONN_EVENT_FAILED, reason); if (!authdir_mode_tests_reachability(get_options())) - control_event_bootstrap_problem(msg, reason); + control_event_bootstrap_problem(msg, reason, conn); } /** <b>conn</b> got an error in connection_handle_read_impl() or @@ -1082,7 +1191,7 @@ connection_or_connect(const tor_addr_t *_addr, uint16_t port, return NULL; } - conn = or_connection_new(tor_addr_family(&addr)); + conn = or_connection_new(CONN_TYPE_OR, tor_addr_family(&addr)); /* * Set up conn so it's got all the data we need to remember for channels @@ -1125,6 +1234,12 @@ connection_or_connect(const tor_addr_t *_addr, uint16_t port, "your pluggable transport proxy stopped running.", fmt_addrport(&TO_CONN(conn)->addr, TO_CONN(conn)->port), transport_name, transport_name); + + control_event_bootstrap_problem( + "Can't connect to bridge", + END_OR_CONN_REASON_PT_MISSING, + conn); + } else { log_warn(LD_GENERAL, "Tried to connect to '%s' through a proxy, but " "the proxy address could not be found.", @@ -1227,8 +1342,8 @@ connection_or_close_for_error(or_connection_t *orconn, int flush) * * Return -1 if <b>conn</b> is broken, else return 0. */ -int -connection_tls_start_handshake(or_connection_t *conn, int receiving) +MOCK_IMPL(int, +connection_tls_start_handshake,(or_connection_t *conn, int receiving)) { channel_listener_t *chan_listener; channel_t *chan; @@ -1485,7 +1600,8 @@ connection_or_handle_event_cb(struct bufferevent *bufev, short event, int connection_or_nonopen_was_started_here(or_connection_t *conn) { - tor_assert(conn->base_.type == CONN_TYPE_OR); + tor_assert(conn->base_.type == CONN_TYPE_OR || + conn->base_.type == CONN_TYPE_EXT_OR); if (!conn->tls) return 1; /* it's still in proxy states or something */ if (conn->handshake_state) @@ -1638,7 +1754,8 @@ connection_or_client_learned_peer_id(or_connection_t *conn, if (!authdir_mode_tests_reachability(options)) control_event_bootstrap_problem( "Unexpected identity in router certificate", - END_OR_CONN_REASON_OR_IDENTITY); + END_OR_CONN_REASON_OR_IDENTITY, + conn); return -1; } if (authdir_mode_tests_reachability(options)) { @@ -1688,13 +1805,11 @@ connection_tls_finish_handshake(or_connection_t *conn) safe_str_client(conn->base_.address), tor_tls_get_ciphersuite_name(conn->tls)); - directory_set_dirty(); - if (connection_or_check_valid_tls_handshake(conn, started_here, digest_rcvd) < 0) return -1; - circuit_build_times_network_is_live(&circ_times); + circuit_build_times_network_is_live(get_circuit_build_times_mutable()); if (tor_tls_used_v1_handshake(conn->tls)) { conn->link_proto = 1; @@ -1728,7 +1843,7 @@ connection_or_launch_v3_or_handshake(or_connection_t *conn) tor_assert(connection_or_nonopen_was_started_here(conn)); tor_assert(tor_tls_received_v3_certificate(conn->tls)); - circuit_build_times_network_is_live(&circ_times); + circuit_build_times_network_is_live(get_circuit_build_times_mutable()); connection_or_change_state(conn, OR_CONN_STATE_OR_HANDSHAKING_V3); if (connection_init_or_handshake_state(conn, 1) < 0) @@ -1890,9 +2005,6 @@ connection_or_write_cell_to_buf(const cell_t *cell, or_connection_t *conn) if (conn->base_.state == OR_CONN_STATE_OR_HANDSHAKING_V3) or_handshake_state_record_cell(conn, conn->handshake_state, cell, 0); - - if (cell->command != CELL_PADDING) - conn->timestamp_last_added_nonpadding = approx_time(); } /** Pack a variable-length <b>cell</b> into wire-format, and write it onto @@ -1913,8 +2025,6 @@ connection_or_write_var_cell_to_buf(const var_cell_t *cell, cell->payload_len, TO_CONN(conn)); if (conn->base_.state == OR_CONN_STATE_OR_HANDSHAKING_V3) or_handshake_state_record_var_cell(conn, conn->handshake_state, cell, 0); - if (cell->command != CELL_PADDING) - conn->timestamp_last_added_nonpadding = approx_time(); /* Touch the channel's active timestamp if there is one */ if (conn->chan) @@ -1961,7 +2071,7 @@ connection_or_process_cells_from_inbuf(or_connection_t *conn) if (conn->chan) channel_timestamp_active(TLS_CHAN_TO_BASE(conn->chan)); - circuit_build_times_network_is_live(&circ_times); + circuit_build_times_network_is_live(get_circuit_build_times_mutable()); channel_tls_handle_var_cell(var_cell, conn); var_cell_free(var_cell); } else { @@ -1977,7 +2087,7 @@ connection_or_process_cells_from_inbuf(or_connection_t *conn) if (conn->chan) channel_timestamp_active(TLS_CHAN_TO_BASE(conn->chan)); - circuit_build_times_network_is_live(&circ_times); + circuit_build_times_network_is_live(get_circuit_build_times_mutable()); connection_fetch_from_buf(buf, cell_network_size, TO_CONN(conn)); /* retrieve cell info from buf (create the host-order struct from the diff --git a/src/or/connection_or.h b/src/or/connection_or.h index 85e68f1a33..143540edd9 100644 --- a/src/or/connection_or.h +++ b/src/or/connection_or.h @@ -45,8 +45,11 @@ void connection_or_close_for_error(or_connection_t *orconn, int flush); void connection_or_report_broken_states(int severity, int domain); -int connection_tls_start_handshake(or_connection_t *conn, int receiving); +MOCK_DECL(int,connection_tls_start_handshake,(or_connection_t *conn, + int receiving)); int connection_tls_continue_handshake(or_connection_t *conn); +void connection_or_set_canonical(or_connection_t *or_conn, + int is_canonical); int connection_init_or_handshake_state(or_connection_t *conn, int started_here); diff --git a/src/or/control.c b/src/or/control.c index ae9dd69d21..9285fc564a 100644 --- a/src/or/control.c +++ b/src/or/control.c @@ -19,6 +19,7 @@ #include "circuitlist.h" #include "circuitstats.h" #include "circuituse.h" +#include "command.h" #include "config.h" #include "confparse.h" #include "connection.h" @@ -52,46 +53,13 @@ * finished authentication and is accepting commands. */ #define STATE_IS_OPEN(s) ((s) == CONTROL_CONN_STATE_OPEN) -/* Recognized asynchronous event types. It's okay to expand this list - * because it is used both as a list of v0 event types, and as indices - * into the bitfield to determine which controllers want which events. - */ -#define EVENT_MIN_ 0x0001 -#define EVENT_CIRCUIT_STATUS 0x0001 -#define EVENT_STREAM_STATUS 0x0002 -#define EVENT_OR_CONN_STATUS 0x0003 -#define EVENT_BANDWIDTH_USED 0x0004 -#define EVENT_CIRCUIT_STATUS_MINOR 0x0005 -#define EVENT_NEW_DESC 0x0006 -#define EVENT_DEBUG_MSG 0x0007 -#define EVENT_INFO_MSG 0x0008 -#define EVENT_NOTICE_MSG 0x0009 -#define EVENT_WARN_MSG 0x000A -#define EVENT_ERR_MSG 0x000B -#define EVENT_ADDRMAP 0x000C -// #define EVENT_AUTHDIR_NEWDESCS 0x000D -#define EVENT_DESCCHANGED 0x000E -// #define EVENT_NS 0x000F -#define EVENT_STATUS_CLIENT 0x0010 -#define EVENT_STATUS_SERVER 0x0011 -#define EVENT_STATUS_GENERAL 0x0012 -#define EVENT_GUARD 0x0013 -#define EVENT_STREAM_BANDWIDTH_USED 0x0014 -#define EVENT_CLIENTS_SEEN 0x0015 -#define EVENT_NEWCONSENSUS 0x0016 -#define EVENT_BUILDTIMEOUT_SET 0x0017 -#define EVENT_SIGNAL 0x0018 -#define EVENT_CONF_CHANGED 0x0019 -#define EVENT_MAX_ 0x0019 -/* If EVENT_MAX_ ever hits 0x0020, we need to make the mask wider. */ - /** Bitfield: The bit 1<<e is set if <b>any</b> open control * connection is interested in events of type <b>e</b>. We use this * so that we can decide to skip generating event messages that nobody * has interest in without having to walk over the global connection * list to find out. **/ -typedef uint32_t event_mask_t; +typedef uint64_t event_mask_t; /** An event mask of all the events that any controller is interested in * receiving. */ @@ -103,7 +71,7 @@ static int disable_log_messages = 0; /** Macro: true if any control connection is interested in events of type * <b>e</b>. */ #define EVENT_IS_INTERESTING(e) \ - (global_event_mask & (1<<(e))) + (!! (global_event_mask & (((uint64_t)1)<<(e)))) /** If we're using cookie-type authentication, how long should our cookies be? */ @@ -115,7 +83,7 @@ static int authentication_cookie_is_set = 0; /** If authentication_cookie_is_set, a secret cookie that we've stored to disk * and which we're using to authenticate controllers. (If the controller can * read it off disk, it has permission to connect.) */ -static char authentication_cookie[AUTHENTICATION_COOKIE_LEN]; +static uint8_t *authentication_cookie = NULL; #define SAFECOOKIE_SERVER_TO_CONTROLLER_CONSTANT \ "Tor safe cookie authentication server-to-controller hash" @@ -130,15 +98,6 @@ static char authentication_cookie[AUTHENTICATION_COOKIE_LEN]; * of this so we can respond to getinfo status/bootstrap-phase queries. */ static char last_sent_bootstrap_message[BOOTSTRAP_MSG_LEN]; -/** Flag for event_format_t. Indicates that we should use the one standard - format. - */ -#define ALL_FORMATS 1 - -/** Bit field of flags to select how to format a controller event. Recognized - * flag is ALL_FORMATS. */ -typedef int event_format_t; - static void connection_printf_to_buf(control_connection_t *conn, const char *format, ...) CHECK_PRINTF(2,3); @@ -201,7 +160,6 @@ static int write_stream_target_to_buf(entry_connection_t *conn, char *buf, size_t len); static void orconn_target_get_name(char *buf, size_t len, or_connection_t *conn); -static char *get_cookie_file(void); /** Given a control event code for a message event, return the corresponding * log severity. */ @@ -232,6 +190,20 @@ log_severity_to_event(int severity) } } +/** Helper: clear bandwidth counters of all origin circuits. */ +static void +clear_circ_bw_fields(void) +{ + circuit_t *circ; + origin_circuit_t *ocirc; + TOR_LIST_FOREACH(circ, circuit_get_global_list(), head) { + if (!CIRCUIT_IS_ORIGIN(circ)) + continue; + ocirc = TO_ORIGIN_CIRCUIT(circ); + ocirc->n_written_circ_bw = ocirc->n_read_circ_bw = 0; + } +} + /** Set <b>global_event_mask*</b> to the bitwise OR of each live control * connection's event_mask field. */ void @@ -257,8 +229,8 @@ control_update_global_event_mask(void) * we want to hear...*/ control_adjust_event_log_severity(); - /* ...then, if we've started logging stream bw, clear the appropriate - * fields. */ + /* ...then, if we've started logging stream or circ bw, clear the + * appropriate fields. */ if (! (old_mask & EVENT_STREAM_BANDWIDTH_USED) && (new_mask & EVENT_STREAM_BANDWIDTH_USED)) { SMARTLIST_FOREACH(conns, connection_t *, conn, @@ -269,6 +241,10 @@ control_update_global_event_mask(void) } }); } + if (! (old_mask & EVENT_CIRC_BANDWIDTH_USED) && + (new_mask & EVENT_CIRC_BANDWIDTH_USED)) { + clear_circ_bw_fields(); + } } /** Adjust the log severities that result in control_event_logmsg being called @@ -334,7 +310,7 @@ connection_write_str_to_buf(const char *s, control_connection_t *conn) * the end. Replace all LF characters sequences with CRLF. Return the number * of bytes in *<b>out</b>. */ -/* static */ size_t +STATIC size_t write_escaped_data(const char *data, size_t len, char **out) { size_t sz_out = len+8; @@ -382,7 +358,7 @@ write_escaped_data(const char *data, size_t len, char **out) * that appears at the start of a line, and replacing all CRLF sequences * with LF. Return the number of * bytes in *<b>out</b>. */ -/* static */ size_t +STATIC size_t read_escaped_data(const char *data, size_t len, char **out) { char *outp; @@ -592,9 +568,9 @@ send_control_done(control_connection_t *conn) * * The EXTENDED_FORMAT and NONEXTENDED_FORMAT flags behave similarly with * respect to the EXTENDED_EVENTS feature. */ -static void -send_control_event_string(uint16_t event, event_format_t which, - const char *msg) +MOCK_IMPL(STATIC void, +send_control_event_string,(uint16_t event, event_format_t which, + const char *msg)) { smartlist_t *conns = get_connection_array(); (void)which; @@ -958,6 +934,12 @@ static const struct control_event_t control_event_table[] = { { EVENT_BUILDTIMEOUT_SET, "BUILDTIMEOUT_SET" }, { EVENT_SIGNAL, "SIGNAL" }, { EVENT_CONF_CHANGED, "CONF_CHANGED"}, + { EVENT_CONN_BW, "CONN_BW" }, + { EVENT_CELL_STATS, "CELL_STATS" }, + { EVENT_TB_EMPTY, "TB_EMPTY" }, + { EVENT_CIRC_BANDWIDTH_USED, "CIRC_BW" }, + { EVENT_TRANSPORT_LAUNCHED, "TRANSPORT_LAUNCHED" }, + { EVENT_HS_DESC, "HS_DESC" }, { 0, NULL }, }; @@ -1447,7 +1429,7 @@ getinfo_helper_misc(control_connection_t *conn, const char *question, } else if (!strcmp(question, "config-defaults-file")) { *answer = tor_strdup(get_torrc_fname(1)); } else if (!strcmp(question, "config-text")) { - *answer = options_dump(get_options(), 1); + *answer = options_dump(get_options(), OPTIONS_DUMP_MINIMAL); } else if (!strcmp(question, "info/names")) { *answer = list_getinfo_options(); } else if (!strcmp(question, "dormant")) { @@ -1509,7 +1491,7 @@ getinfo_helper_misc(control_connection_t *conn, const char *question, *answer = tor_strdup(""); #else int myUid = geteuid(); - struct passwd *myPwEntry = getpwuid(myUid); + const struct passwd *myPwEntry = tor_getpwuid(myUid); if (myPwEntry) { *answer = tor_strdup(myPwEntry->pw_name); @@ -1521,6 +1503,9 @@ getinfo_helper_misc(control_connection_t *conn, const char *question, int max_fds=-1; set_max_file_descriptors(0, &max_fds); tor_asprintf(answer, "%d", max_fds); + } else if (!strcmp(question, "limits/max-mem-in-queues")) { + tor_asprintf(answer, U64_FORMAT, + U64_PRINTF_ARG(get_options()->MaxMemInQueues)); } else if (!strcmp(question, "dir-usage")) { *answer = directory_dump_request_log(); } else if (!strcmp(question, "fingerprint")) { @@ -1567,12 +1552,13 @@ munge_extrainfo_into_routerinfo(const char *ri_body, outp += router_sig-ri_body; for (i=0; i < 2; ++i) { - const char *kwd = i?"\nwrite-history ":"\nread-history "; + const char *kwd = i ? "\nwrite-history " : "\nread-history "; const char *cp, *eol; if (!(cp = tor_memstr(ei_body, ei_len, kwd))) continue; ++cp; - eol = memchr(cp, '\n', ei_len - (cp-ei_body)); + if (!(eol = memchr(cp, '\n', ei_len - (cp-ei_body)))) + continue; memcpy(outp, cp, eol-cp+1); outp += eol-cp+1; } @@ -1764,39 +1750,7 @@ getinfo_helper_dir(control_connection_t *control_conn, tor_free(url); smartlist_free(descs); } else if (!strcmpstart(question, "dir/status/")) { - if (directory_permits_controller_requests(get_options())) { - size_t len=0; - char *cp; - smartlist_t *status_list = smartlist_new(); - dirserv_get_networkstatus_v2(status_list, - question+strlen("dir/status/")); - SMARTLIST_FOREACH(status_list, cached_dir_t *, d, len += d->dir_len); - cp = *answer = tor_malloc(len+1); - SMARTLIST_FOREACH(status_list, cached_dir_t *, d, { - memcpy(cp, d->dir, d->dir_len); - cp += d->dir_len; - }); - *cp = '\0'; - smartlist_free(status_list); - } else { - smartlist_t *fp_list = smartlist_new(); - smartlist_t *status_list = smartlist_new(); - dirserv_get_networkstatus_v2_fingerprints( - fp_list, question+strlen("dir/status/")); - SMARTLIST_FOREACH(fp_list, const char *, fp, { - char *s; - char *fname = networkstatus_get_cache_filename(fp); - s = read_file_to_str(fname, 0, NULL); - if (s) - smartlist_add(status_list, s); - tor_free(fname); - }); - SMARTLIST_FOREACH(fp_list, char *, fp, tor_free(fp)); - smartlist_free(fp_list); - *answer = smartlist_join_strings(status_list, "", 0, NULL); - SMARTLIST_FOREACH(status_list, char *, s, tor_free(s)); - smartlist_free(status_list); - } + *answer = tor_strdup(""); } else if (!strcmp(question, "dir/status-vote/current/consensus")) { /* v3 */ if (directory_caches_dir_info(get_options())) { const cached_dir_t *consensus = dirserv_get_consensus("ns"); @@ -1927,7 +1881,7 @@ getinfo_helper_events(control_connection_t *control_conn, if (!strcmp(question, "circuit-status")) { circuit_t *circ_; smartlist_t *status = smartlist_new(); - for (circ_ = circuit_get_global_list_(); circ_; circ_ = circ_->next) { + TOR_LIST_FOREACH(circ_, circuit_get_global_list(), head) { origin_circuit_t *circ; char *circdesc; const char *state; @@ -2232,6 +2186,7 @@ static const getinfo_item_t getinfo_items[] = { ITEM("process/user", misc, "Username under which the tor process is running."), ITEM("process/descriptor-limit", misc, "File descriptor limit."), + ITEM("limits/max-mem-in-queues", misc, "Actual limit on memory in queues"), ITEM("dir-usage", misc, "Breakdown of bytes transferred over DirPort."), PREFIX("desc-annotations/id/", dir, "Router annotations by hexdigest."), PREFIX("dir/server/", dir,"Router descriptors as retrieved from a DirPort."), @@ -2241,6 +2196,9 @@ static const getinfo_item_t getinfo_items[] = { "v3 Networkstatus consensus as retrieved from a DirPort."), ITEM("exit-policy/default", policies, "The default value appended to the configured exit policy."), + ITEM("exit-policy/full", policies, "The entire exit policy of onion router"), + ITEM("exit-policy/ipv4", policies, "IPv4 parts of exit policy"), + ITEM("exit-policy/ipv6", policies, "IPv6 parts of exit policy"), PREFIX("ip-to-country/", geoip, "Perform a GEOIP lookup"), { NULL, NULL, NULL, 0 } }; @@ -2985,7 +2943,7 @@ handle_control_protocolinfo(control_connection_t *conn, uint32_t len, } else { const or_options_t *options = get_options(); int cookies = options->CookieAuthentication; - char *cfile = get_cookie_file(); + char *cfile = get_controller_cookie_file_name(); char *abs_cfile; char *esc_cfile; char *methods; @@ -3181,6 +3139,30 @@ handle_control_usefeature(control_connection_t *conn, return 0; } +/** Implementation for the DROPGUARDS command. */ +static int +handle_control_dropguards(control_connection_t *conn, + uint32_t len, + const char *body) +{ + smartlist_t *args; + (void) len; /* body is nul-terminated; it's safe to ignore the length */ + args = smartlist_new(); + smartlist_split_string(args, body, " ", + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); + + if (smartlist_len(args)) { + connection_printf_to_buf(conn, "512 Too many arguments to DROPGUARDS\r\n"); + } else { + remove_all_entry_guards(); + send_control_done(conn); + } + + SMARTLIST_FOREACH(args, char *, cp, tor_free(cp)); + smartlist_free(args); + return 0; +} + /** Called when <b>conn</b> has no more bytes left on its outbuf. */ int connection_control_finished_flushing(control_connection_t *conn) @@ -3200,27 +3182,22 @@ connection_control_reached_eof(control_connection_t *conn) return 0; } +static void lost_owning_controller(const char *owner_type, + const char *loss_manner) + ATTR_NORETURN; + /** Shut down this Tor instance in the same way that SIGINT would, but * with a log message appropriate for the loss of an owning controller. */ static void lost_owning_controller(const char *owner_type, const char *loss_manner) { - int shutdown_slowly = server_mode(get_options()); - - log_notice(LD_CONTROL, "Owning controller %s has %s -- %s.", - owner_type, loss_manner, - shutdown_slowly ? "shutting down" : "exiting now"); + log_notice(LD_CONTROL, "Owning controller %s has %s -- exiting now.", + owner_type, loss_manner); /* XXXX Perhaps this chunk of code should be a separate function, * called here and by process_signal(SIGINT). */ - - if (!shutdown_slowly) { - tor_cleanup(); - exit(0); - } - /* XXXX This will close all listening sockets except control-port - * listeners. Perhaps we should close those too. */ - hibernate_begin_shutdown(); + tor_cleanup(); + exit(0); } /** Called when <b>conn</b> is being freed. */ @@ -3480,6 +3457,9 @@ connection_control_process_inbuf(control_connection_t *conn) } else if (!strcasecmp(conn->incoming_cmd, "AUTHCHALLENGE")) { if (handle_control_authchallenge(conn, cmd_data_len, args)) return -1; + } else if (!strcasecmp(conn->incoming_cmd, "DROPGUARDS")) { + if (handle_control_dropguards(conn, cmd_data_len, args)) + return -1; } else { connection_printf_to_buf(conn, "510 Unrecognized command \"%s\"\r\n", conn->incoming_cmd); @@ -3847,17 +3827,17 @@ control_event_or_conn_status(or_connection_t *conn, or_conn_status_event_t tp, } ncircs += connection_or_get_num_circuits(conn); if (ncircs && (tp == OR_CONN_EVENT_FAILED || tp == OR_CONN_EVENT_CLOSED)) { - tor_snprintf(ncircs_buf, sizeof(ncircs_buf), "%sNCIRCS=%d", - reason ? " " : "", ncircs); + tor_snprintf(ncircs_buf, sizeof(ncircs_buf), " NCIRCS=%d", ncircs); } orconn_target_get_name(name, sizeof(name), conn); send_control_event(EVENT_OR_CONN_STATUS, ALL_FORMATS, - "650 ORCONN %s %s %s%s%s\r\n", + "650 ORCONN %s %s%s%s%s ID="U64_FORMAT"\r\n", name, status, - reason ? "REASON=" : "", + reason ? " REASON=" : "", orconn_end_reason_to_control_string(reason), - ncircs_buf); + ncircs_buf, + U64_PRINTF_ARG(conn->base_.global_identifier)); return 0; } @@ -3868,6 +3848,8 @@ control_event_or_conn_status(or_connection_t *conn, or_conn_status_event_t tp, int control_event_stream_bandwidth(edge_connection_t *edge_conn) { + circuit_t *circ; + origin_circuit_t *ocirc; if (EVENT_IS_INTERESTING(EVENT_STREAM_BANDWIDTH_USED)) { if (!edge_conn->n_read && !edge_conn->n_written) return 0; @@ -3878,6 +3860,12 @@ control_event_stream_bandwidth(edge_connection_t *edge_conn) (unsigned long)edge_conn->n_read, (unsigned long)edge_conn->n_written); + circ = circuit_get_by_edge_conn(edge_conn); + if (circ && CIRCUIT_IS_ORIGIN(circ)) { + ocirc = TO_ORIGIN_CIRCUIT(circ); + ocirc->n_read_circ_bw += edge_conn->n_read; + ocirc->n_written_circ_bw += edge_conn->n_written; + } edge_conn->n_written = edge_conn->n_read = 0; } @@ -3915,6 +3903,235 @@ control_event_stream_bandwidth_used(void) return 0; } +/** A second or more has elapsed: tell any interested control connections + * how much bandwidth origin circuits have used. */ +int +control_event_circ_bandwidth_used(void) +{ + circuit_t *circ; + origin_circuit_t *ocirc; + if (!EVENT_IS_INTERESTING(EVENT_CIRC_BANDWIDTH_USED)) + return 0; + + TOR_LIST_FOREACH(circ, circuit_get_global_list(), head) { + if (!CIRCUIT_IS_ORIGIN(circ)) + continue; + ocirc = TO_ORIGIN_CIRCUIT(circ); + if (!ocirc->n_read_circ_bw && !ocirc->n_written_circ_bw) + continue; + send_control_event(EVENT_CIRC_BANDWIDTH_USED, ALL_FORMATS, + "650 CIRC_BW ID=%d READ=%lu WRITTEN=%lu\r\n", + ocirc->global_identifier, + (unsigned long)ocirc->n_read_circ_bw, + (unsigned long)ocirc->n_written_circ_bw); + ocirc->n_written_circ_bw = ocirc->n_read_circ_bw = 0; + } + + return 0; +} + +/** Print out CONN_BW event for a single OR/DIR/EXIT <b>conn</b> and reset + * bandwidth counters. */ +int +control_event_conn_bandwidth(connection_t *conn) +{ + const char *conn_type_str; + if (!get_options()->TestingEnableConnBwEvent || + !EVENT_IS_INTERESTING(EVENT_CONN_BW)) + return 0; + if (!conn->n_read_conn_bw && !conn->n_written_conn_bw) + return 0; + switch (conn->type) { + case CONN_TYPE_OR: + conn_type_str = "OR"; + break; + case CONN_TYPE_DIR: + conn_type_str = "DIR"; + break; + case CONN_TYPE_EXIT: + conn_type_str = "EXIT"; + break; + default: + return 0; + } + send_control_event(EVENT_CONN_BW, ALL_FORMATS, + "650 CONN_BW ID="U64_FORMAT" TYPE=%s " + "READ=%lu WRITTEN=%lu\r\n", + U64_PRINTF_ARG(conn->global_identifier), + conn_type_str, + (unsigned long)conn->n_read_conn_bw, + (unsigned long)conn->n_written_conn_bw); + conn->n_written_conn_bw = conn->n_read_conn_bw = 0; + return 0; +} + +/** A second or more has elapsed: tell any interested control + * connections how much bandwidth connections have used. */ +int +control_event_conn_bandwidth_used(void) +{ + if (get_options()->TestingEnableConnBwEvent && + EVENT_IS_INTERESTING(EVENT_CONN_BW)) { + SMARTLIST_FOREACH(get_connection_array(), connection_t *, conn, + control_event_conn_bandwidth(conn)); + } + return 0; +} + +/** Helper: iterate over cell statistics of <b>circ</b> and sum up added + * cells, removed cells, and waiting times by cell command and direction. + * Store results in <b>cell_stats</b>. Free cell statistics of the + * circuit afterwards. */ +void +sum_up_cell_stats_by_command(circuit_t *circ, cell_stats_t *cell_stats) +{ + memset(cell_stats, 0, sizeof(cell_stats_t)); + SMARTLIST_FOREACH_BEGIN(circ->testing_cell_stats, + testing_cell_stats_entry_t *, ent) { + tor_assert(ent->command <= CELL_COMMAND_MAX_); + if (!ent->removed && !ent->exitward) { + cell_stats->added_cells_appward[ent->command] += 1; + } else if (!ent->removed && ent->exitward) { + cell_stats->added_cells_exitward[ent->command] += 1; + } else if (!ent->exitward) { + cell_stats->removed_cells_appward[ent->command] += 1; + cell_stats->total_time_appward[ent->command] += ent->waiting_time * 10; + } else { + cell_stats->removed_cells_exitward[ent->command] += 1; + cell_stats->total_time_exitward[ent->command] += ent->waiting_time * 10; + } + tor_free(ent); + } SMARTLIST_FOREACH_END(ent); + smartlist_free(circ->testing_cell_stats); + circ->testing_cell_stats = NULL; +} + +/** Helper: append a cell statistics string to <code>event_parts</code>, + * prefixed with <code>key</code>=. Statistics consist of comma-separated + * key:value pairs with lower-case command strings as keys and cell + * numbers or total waiting times as values. A key:value pair is included + * if the entry in <code>include_if_non_zero</code> is not zero, but with + * the (possibly zero) entry from <code>number_to_include</code>. Both + * arrays are expected to have a length of CELL_COMMAND_MAX_ + 1. If no + * entry in <code>include_if_non_zero</code> is positive, no string will + * be added to <code>event_parts</code>. */ +void +append_cell_stats_by_command(smartlist_t *event_parts, const char *key, + const uint64_t *include_if_non_zero, + const uint64_t *number_to_include) +{ + smartlist_t *key_value_strings = smartlist_new(); + int i; + for (i = 0; i <= CELL_COMMAND_MAX_; i++) { + if (include_if_non_zero[i] > 0) { + smartlist_add_asprintf(key_value_strings, "%s:"U64_FORMAT, + cell_command_to_string(i), + U64_PRINTF_ARG(number_to_include[i])); + } + } + if (smartlist_len(key_value_strings) > 0) { + char *joined = smartlist_join_strings(key_value_strings, ",", 0, NULL); + smartlist_add_asprintf(event_parts, "%s=%s", key, joined); + SMARTLIST_FOREACH(key_value_strings, char *, cp, tor_free(cp)); + tor_free(joined); + } + smartlist_free(key_value_strings); +} + +/** Helper: format <b>cell_stats</b> for <b>circ</b> for inclusion in a + * CELL_STATS event and write result string to <b>event_string</b>. */ +void +format_cell_stats(char **event_string, circuit_t *circ, + cell_stats_t *cell_stats) +{ + smartlist_t *event_parts = smartlist_new(); + if (CIRCUIT_IS_ORIGIN(circ)) { + origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ); + smartlist_add_asprintf(event_parts, "ID=%lu", + (unsigned long)ocirc->global_identifier); + } else if (TO_OR_CIRCUIT(circ)->p_chan) { + or_circuit_t *or_circ = TO_OR_CIRCUIT(circ); + smartlist_add_asprintf(event_parts, "InboundQueue=%lu", + (unsigned long)or_circ->p_circ_id); + smartlist_add_asprintf(event_parts, "InboundConn="U64_FORMAT, + U64_PRINTF_ARG(or_circ->p_chan->global_identifier)); + append_cell_stats_by_command(event_parts, "InboundAdded", + cell_stats->added_cells_appward, + cell_stats->added_cells_appward); + append_cell_stats_by_command(event_parts, "InboundRemoved", + cell_stats->removed_cells_appward, + cell_stats->removed_cells_appward); + append_cell_stats_by_command(event_parts, "InboundTime", + cell_stats->removed_cells_appward, + cell_stats->total_time_appward); + } + if (circ->n_chan) { + smartlist_add_asprintf(event_parts, "OutboundQueue=%lu", + (unsigned long)circ->n_circ_id); + smartlist_add_asprintf(event_parts, "OutboundConn="U64_FORMAT, + U64_PRINTF_ARG(circ->n_chan->global_identifier)); + append_cell_stats_by_command(event_parts, "OutboundAdded", + cell_stats->added_cells_exitward, + cell_stats->added_cells_exitward); + append_cell_stats_by_command(event_parts, "OutboundRemoved", + cell_stats->removed_cells_exitward, + cell_stats->removed_cells_exitward); + append_cell_stats_by_command(event_parts, "OutboundTime", + cell_stats->removed_cells_exitward, + cell_stats->total_time_exitward); + } + *event_string = smartlist_join_strings(event_parts, " ", 0, NULL); + SMARTLIST_FOREACH(event_parts, char *, cp, tor_free(cp)); + smartlist_free(event_parts); +} + +/** A second or more has elapsed: tell any interested control connection + * how many cells have been processed for a given circuit. */ +int +control_event_circuit_cell_stats(void) +{ + circuit_t *circ; + cell_stats_t *cell_stats; + char *event_string; + if (!get_options()->TestingEnableCellStatsEvent || + !EVENT_IS_INTERESTING(EVENT_CELL_STATS)) + return 0; + cell_stats = tor_malloc(sizeof(cell_stats_t));; + TOR_LIST_FOREACH(circ, circuit_get_global_list(), head) { + if (!circ->testing_cell_stats) + continue; + sum_up_cell_stats_by_command(circ, cell_stats); + format_cell_stats(&event_string, circ, cell_stats); + send_control_event(EVENT_CELL_STATS, ALL_FORMATS, + "650 CELL_STATS %s\r\n", event_string); + tor_free(event_string); + } + tor_free(cell_stats); + return 0; +} + +/** Tokens in <b>bucket</b> have been refilled: the read bucket was empty + * for <b>read_empty_time</b> millis, the write bucket was empty for + * <b>write_empty_time</b> millis, and buckets were last refilled + * <b>milliseconds_elapsed</b> millis ago. Only emit TB_EMPTY event if + * either read or write bucket have been empty before. */ +int +control_event_tb_empty(const char *bucket, uint32_t read_empty_time, + uint32_t write_empty_time, + int milliseconds_elapsed) +{ + if (get_options()->TestingEnableTbEmptyEvent && + EVENT_IS_INTERESTING(EVENT_TB_EMPTY) && + (read_empty_time > 0 || write_empty_time > 0)) { + send_control_event(EVENT_TB_EMPTY, ALL_FORMATS, + "650 TB_EMPTY %s READ=%d WRITTEN=%d " + "LAST=%d\r\n", + bucket, read_empty_time, write_empty_time, + milliseconds_elapsed); + } + return 0; +} + /** A second or more has elapsed: tell any interested control * connections how much bandwidth we used. */ int @@ -4162,32 +4379,26 @@ control_event_newconsensus(const networkstatus_t *consensus) /** Called when we compute a new circuitbuildtimeout */ int -control_event_buildtimeout_set(const circuit_build_times_t *cbt, - buildtimeout_set_event_t type) +control_event_buildtimeout_set(buildtimeout_set_event_t type, + const char *args) { const char *type_string = NULL; - double qnt; if (!control_event_is_interesting(EVENT_BUILDTIMEOUT_SET)) return 0; - qnt = circuit_build_times_quantile_cutoff(); - switch (type) { case BUILDTIMEOUT_SET_EVENT_COMPUTED: type_string = "COMPUTED"; break; case BUILDTIMEOUT_SET_EVENT_RESET: type_string = "RESET"; - qnt = 1.0; break; case BUILDTIMEOUT_SET_EVENT_SUSPENDED: type_string = "SUSPENDED"; - qnt = 1.0; break; case BUILDTIMEOUT_SET_EVENT_DISCARD: type_string = "DISCARD"; - qnt = 1.0; break; case BUILDTIMEOUT_SET_EVENT_RESUME: type_string = "RESUME"; @@ -4198,15 +4409,8 @@ control_event_buildtimeout_set(const circuit_build_times_t *cbt, } send_control_event(EVENT_BUILDTIMEOUT_SET, ALL_FORMATS, - "650 BUILDTIMEOUT_SET %s TOTAL_TIMES=%lu " - "TIMEOUT_MS=%lu XM=%lu ALPHA=%f CUTOFF_QUANTILE=%f " - "TIMEOUT_RATE=%f CLOSE_MS=%lu CLOSE_RATE=%f\r\n", - type_string, (unsigned long)cbt->total_build_times, - (unsigned long)cbt->timeout_ms, - (unsigned long)cbt->Xm, cbt->alpha, qnt, - circuit_build_times_timeout_rate(cbt), - (unsigned long)cbt->close_ms, - circuit_build_times_close_rate(cbt)); + "650 BUILDTIMEOUT_SET %s %s\r\n", + type_string, args); return 0; } @@ -4434,8 +4638,8 @@ control_event_conf_changed(const smartlist_t *elements) /** Helper: Return a newly allocated string containing a path to the * file where we store our authentication cookie. */ -static char * -get_cookie_file(void) +char * +get_controller_cookie_file_name(void) { const or_options_t *options = get_options(); if (options->CookieAuthFile && strlen(options->CookieAuthFile)) { @@ -4445,44 +4649,27 @@ get_cookie_file(void) } } -/** Choose a random authentication cookie and write it to disk. - * Anybody who can read the cookie from disk will be considered - * authorized to use the control connection. Return -1 if we can't - * write the file, or 0 on success. */ +/* Initialize the cookie-based authentication system of the + * ControlPort. If <b>enabled</b> is 0, then disable the cookie + * authentication system. */ int -init_cookie_authentication(int enabled) +init_control_cookie_authentication(int enabled) { - char *fname; + char *fname = NULL; + int retval; + if (!enabled) { authentication_cookie_is_set = 0; return 0; } - /* We don't want to generate a new cookie every time we call - * options_act(). One should be enough. */ - if (authentication_cookie_is_set) - return 0; /* all set */ - - fname = get_cookie_file(); - crypto_rand(authentication_cookie, AUTHENTICATION_COOKIE_LEN); - authentication_cookie_is_set = 1; - if (write_bytes_to_file(fname, authentication_cookie, - AUTHENTICATION_COOKIE_LEN, 1)) { - log_warn(LD_FS,"Error writing authentication cookie to %s.", - escaped(fname)); - tor_free(fname); - return -1; - } -#ifndef _WIN32 - if (get_options()->CookieAuthFileGroupReadable) { - if (chmod(fname, 0640)) { - log_warn(LD_FS,"Unable to make %s group-readable.", escaped(fname)); - } - } -#endif - + fname = get_controller_cookie_file_name(); + retval = init_cookie_authentication(fname, "", /* no header */ + AUTHENTICATION_COOKIE_LEN, + &authentication_cookie, + &authentication_cookie_is_set); tor_free(fname); - return 0; + return retval; } /** A copy of the process specifier of Tor's owning controller, or @@ -4493,6 +4680,8 @@ static char *owning_controller_process_spec = NULL; * if this Tor instance is not currently owned by a process. */ static tor_process_monitor_t *owning_controller_process_monitor = NULL; +static void owning_controller_procmon_cb(void *unused) ATTR_NORETURN; + /** Process-termination monitor callback for Tor's owning controller * process. */ static void @@ -4636,16 +4825,28 @@ bootstrap_status_to_string(bootstrap_status_t s, const char **tag, * Tor initializes. */ static int bootstrap_percent = BOOTSTRAP_STATUS_UNDEF; +/** As bootstrap_percent, but holds the bootstrapping level at which we last + * logged a NOTICE-level message. We use this, plus BOOTSTRAP_PCT_INCREMENT, + * to avoid flooding the log with a new message every time we get a few more + * microdescriptors */ +static int notice_bootstrap_percent = 0; + /** How many problems have we had getting to the next bootstrapping phase? * These include failure to establish a connection to a Tor relay, * failures to finish the TLS handshake, failures to validate the * consensus document, etc. */ static int bootstrap_problems = 0; -/* We only tell the controller once we've hit a threshold of problems +/** We only tell the controller once we've hit a threshold of problems * for the current phase. */ #define BOOTSTRAP_PROBLEM_THRESHOLD 10 +/** When our bootstrapping progress level changes, but our bootstrapping + * status has not advanced, we only log at NOTICE when we have made at least + * this much progress. + */ +#define BOOTSTRAP_PCT_INCREMENT 5 + /** Called when Tor has made progress at bootstrapping its directory * information and initial circuits. * @@ -4665,7 +4866,7 @@ control_event_bootstrap(bootstrap_status_t status, int progress) * can't distinguish what the connection is going to be for. */ if (status == BOOTSTRAP_STATUS_HANDSHAKE) { if (bootstrap_percent < BOOTSTRAP_STATUS_CONN_OR) { - status = BOOTSTRAP_STATUS_HANDSHAKE_DIR; + status = BOOTSTRAP_STATUS_HANDSHAKE_DIR; } else { status = BOOTSTRAP_STATUS_HANDSHAKE_OR; } @@ -4673,9 +4874,19 @@ control_event_bootstrap(bootstrap_status_t status, int progress) if (status > bootstrap_percent || (progress && progress > bootstrap_percent)) { + int loglevel = LOG_NOTICE; bootstrap_status_to_string(status, &tag, &summary); - tor_log(status ? LOG_NOTICE : LOG_INFO, LD_CONTROL, - "Bootstrapped %d%%: %s.", progress ? progress : status, summary); + + if (status <= bootstrap_percent && + (progress < notice_bootstrap_percent + BOOTSTRAP_PCT_INCREMENT)) { + /* We log the message at info if the status hasn't advanced, and if less + * than BOOTSTRAP_PCT_INCREMENT progress has been made. + */ + loglevel = LOG_INFO; + } + + tor_log(loglevel, LD_CONTROL, + "Bootstrapped %d%%: %s", progress ? progress : status, summary); tor_snprintf(buf, sizeof(buf), "BOOTSTRAP PROGRESS=%d TAG=%s SUMMARY=\"%s\"", progress ? progress : status, tag, summary); @@ -4691,15 +4902,22 @@ control_event_bootstrap(bootstrap_status_t status, int progress) bootstrap_percent = progress; bootstrap_problems = 0; /* Progress! Reset our problem counter. */ } + if (loglevel == LOG_NOTICE && + bootstrap_percent > notice_bootstrap_percent) { + /* Remember that we gave a notice at this level. */ + notice_bootstrap_percent = bootstrap_percent; + } } } /** Called when Tor has failed to make bootstrapping progress in a way * that indicates a problem. <b>warn</b> gives a hint as to why, and - * <b>reason</b> provides an "or_conn_end_reason" tag. + * <b>reason</b> provides an "or_conn_end_reason" tag. <b>or_conn</b> + * is the connection that caused this problem. */ -void -control_event_bootstrap_problem(const char *warn, int reason) +MOCK_IMPL(void, + control_event_bootstrap_problem, (const char *warn, int reason, + or_connection_t *or_conn)) { int status = bootstrap_percent; const char *tag, *summary; @@ -4710,6 +4928,11 @@ control_event_bootstrap_problem(const char *warn, int reason) /* bootstrap_percent must not be in "undefined" state here. */ tor_assert(status >= 0); + if (or_conn->have_noted_bootstrap_problem) + return; + + or_conn->have_noted_bootstrap_problem = 1; + if (bootstrap_percent == 100) return; /* already bootstrapped; nothing to be done here. */ @@ -4721,9 +4944,10 @@ control_event_bootstrap_problem(const char *warn, int reason) if (reason == END_OR_CONN_REASON_NO_ROUTE) recommendation = "warn"; - if (get_options()->UseBridges && - !any_bridge_descriptors_known() && - !any_pending_bridge_descriptor_fetches()) + /* If we are using bridges and all our OR connections are now + closed, it means that we totally failed to connect to our + bridges. Throw a warning. */ + if (get_options()->UseBridges && !any_other_active_or_conns(or_conn)) recommendation = "warn"; if (we_are_hibernating()) @@ -4766,3 +4990,159 @@ control_event_clients_seen(const char *controller_str) "650 CLIENTS_SEEN %s\r\n", controller_str); } +/** A new pluggable transport called <b>transport_name</b> was + * launched on <b>addr</b>:<b>port</b>. <b>mode</b> is either + * "server" or "client" depending on the mode of the pluggable + * transport. + * "650" SP "TRANSPORT_LAUNCHED" SP Mode SP Name SP Address SP Port + */ +void +control_event_transport_launched(const char *mode, const char *transport_name, + tor_addr_t *addr, uint16_t port) +{ + send_control_event(EVENT_TRANSPORT_LAUNCHED, ALL_FORMATS, + "650 TRANSPORT_LAUNCHED %s %s %s %u\r\n", + mode, transport_name, fmt_addr(addr), port); +} + +/** Convert rendezvous auth type to string for HS_DESC control events + */ +const char * +rend_auth_type_to_string(rend_auth_type_t auth_type) +{ + const char *str; + + switch (auth_type) { + case REND_NO_AUTH: + str = "NO_AUTH"; + break; + case REND_BASIC_AUTH: + str = "BASIC_AUTH"; + break; + case REND_STEALTH_AUTH: + str = "STEALTH_AUTH"; + break; + default: + str = "UNKNOWN"; + } + + return str; +} + +/** Return a longname the node whose identity is <b>id_digest</b>. If + * node_get_by_id() returns NULL, base 16 encoding of <b>id_digest</b> is + * returned instead. + * + * This function is not thread-safe. Each call to this function invalidates + * previous values returned by this function. + */ +MOCK_IMPL(const char *, +node_describe_longname_by_id,(const char *id_digest)) +{ + static char longname[MAX_VERBOSE_NICKNAME_LEN+1]; + node_get_verbose_nickname_by_id(id_digest, longname); + return longname; +} + +/** send HS_DESC requested event. + * + * <b>rend_query</b> is used to fetch requested onion address and auth type. + * <b>hs_dir</b> is the description of contacting hs directory. + * <b>desc_id_base32</b> is the ID of requested hs descriptor. + */ +void +control_event_hs_descriptor_requested(const rend_data_t *rend_query, + const char *id_digest, + const char *desc_id_base32) +{ + if (!id_digest || !rend_query || !desc_id_base32) { + log_warn(LD_BUG, "Called with rend_query==%p, " + "id_digest==%p, desc_id_base32==%p", + rend_query, id_digest, desc_id_base32); + return; + } + + send_control_event(EVENT_HS_DESC, ALL_FORMATS, + "650 HS_DESC REQUESTED %s %s %s %s\r\n", + rend_query->onion_address, + rend_auth_type_to_string(rend_query->auth_type), + node_describe_longname_by_id(id_digest), + desc_id_base32); +} + +/** send HS_DESC event after got response from hs directory. + * + * NOTE: this is an internal function used by following functions: + * control_event_hs_descriptor_received + * control_event_hs_descriptor_failed + * + * So do not call this function directly. + */ +void +control_event_hs_descriptor_receive_end(const char *action, + const rend_data_t *rend_query, + const char *id_digest) +{ + if (!action || !rend_query || !id_digest) { + log_warn(LD_BUG, "Called with action==%p, rend_query==%p, " + "id_digest==%p", action, rend_query, id_digest); + return; + } + + send_control_event(EVENT_HS_DESC, ALL_FORMATS, + "650 HS_DESC %s %s %s %s\r\n", + action, + rend_query->onion_address, + rend_auth_type_to_string(rend_query->auth_type), + node_describe_longname_by_id(id_digest)); +} + +/** send HS_DESC RECEIVED event + * + * called when a we successfully received a hidden service descriptor. + */ +void +control_event_hs_descriptor_received(const rend_data_t *rend_query, + const char *id_digest) +{ + if (!rend_query || !id_digest) { + log_warn(LD_BUG, "Called with rend_query==%p, id_digest==%p", + rend_query, id_digest); + return; + } + control_event_hs_descriptor_receive_end("RECEIVED", rend_query, id_digest); +} + +/** send HS_DESC FAILED event + * + * called when request for hidden service descriptor returned failure. + */ +void +control_event_hs_descriptor_failed(const rend_data_t *rend_query, + const char *id_digest) +{ + if (!rend_query || !id_digest) { + log_warn(LD_BUG, "Called with rend_query==%p, id_digest==%p", + rend_query, id_digest); + return; + } + control_event_hs_descriptor_receive_end("FAILED", rend_query, id_digest); +} + +/** Free any leftover allocated memory of the control.c subsystem. */ +void +control_free_all(void) +{ + if (authentication_cookie) /* Free the auth cookie */ + tor_free(authentication_cookie); +} + +#ifdef TOR_UNIT_TESTS +/* For testing: change the value of global_event_mask */ +void +control_testing_set_global_event_mask(uint64_t mask) +{ + global_event_mask = mask; +} +#endif + diff --git a/src/or/control.h b/src/or/control.h index 61062da2c4..68a6c244d0 100644 --- a/src/or/control.h +++ b/src/or/control.h @@ -50,6 +50,13 @@ int control_event_or_conn_status(or_connection_t *conn, int control_event_bandwidth_used(uint32_t n_read, uint32_t n_written); int control_event_stream_bandwidth(edge_connection_t *edge_conn); int control_event_stream_bandwidth_used(void); +int control_event_circ_bandwidth_used(void); +int control_event_conn_bandwidth(connection_t *conn); +int control_event_conn_bandwidth_used(void); +int control_event_circuit_cell_stats(void); +int control_event_tb_empty(const char *bucket, uint32_t read_empty_time, + uint32_t write_empty_time, + int milliseconds_elapsed); void control_event_logmsg(int severity, uint32_t domain, const char *msg); int control_event_descriptors_changed(smartlist_t *routers); int control_event_address_mapped(const char *from, const char *to, @@ -73,11 +80,12 @@ int control_event_server_status(int severity, const char *format, ...) int control_event_guard(const char *nickname, const char *digest, const char *status); int control_event_conf_changed(const smartlist_t *elements); -int control_event_buildtimeout_set(const circuit_build_times_t *cbt, - buildtimeout_set_event_t type); +int control_event_buildtimeout_set(buildtimeout_set_event_t type, + const char *args); int control_event_signal(uintptr_t signal); -int init_cookie_authentication(int enabled); +int init_control_cookie_authentication(int enabled); +char *get_controller_cookie_file_name(void); smartlist_t *decode_hashed_passwords(config_line_t *passwords); void disable_control_logging(void); void enable_control_logging(void); @@ -85,14 +93,114 @@ void enable_control_logging(void); void monitor_owning_controller_process(const char *process_spec); void control_event_bootstrap(bootstrap_status_t status, int progress); -void control_event_bootstrap_problem(const char *warn, int reason); +MOCK_DECL(void, control_event_bootstrap_problem,(const char *warn, + int reason, + or_connection_t *or_conn)); void control_event_clients_seen(const char *controller_str); +void control_event_transport_launched(const char *mode, + const char *transport_name, + tor_addr_t *addr, uint16_t port); +const char *rend_auth_type_to_string(rend_auth_type_t auth_type); +MOCK_DECL(const char *, node_describe_longname_by_id,(const char *id_digest)); +void control_event_hs_descriptor_requested(const rend_data_t *rend_query, + const char *desc_id_base32, + const char *hs_dir); +void control_event_hs_descriptor_receive_end(const char *action, + const rend_data_t *rend_query, + const char *hs_dir); +void control_event_hs_descriptor_received(const rend_data_t *rend_query, + const char *hs_dir); +void control_event_hs_descriptor_failed(const rend_data_t *rend_query, + const char *hs_dir); + +void control_free_all(void); #ifdef CONTROL_PRIVATE +/* Recognized asynchronous event types. It's okay to expand this list + * because it is used both as a list of v0 event types, and as indices + * into the bitfield to determine which controllers want which events. + */ +#define EVENT_MIN_ 0x0001 +#define EVENT_CIRCUIT_STATUS 0x0001 +#define EVENT_STREAM_STATUS 0x0002 +#define EVENT_OR_CONN_STATUS 0x0003 +#define EVENT_BANDWIDTH_USED 0x0004 +#define EVENT_CIRCUIT_STATUS_MINOR 0x0005 +#define EVENT_NEW_DESC 0x0006 +#define EVENT_DEBUG_MSG 0x0007 +#define EVENT_INFO_MSG 0x0008 +#define EVENT_NOTICE_MSG 0x0009 +#define EVENT_WARN_MSG 0x000A +#define EVENT_ERR_MSG 0x000B +#define EVENT_ADDRMAP 0x000C +/* Exposed above */ +// #define EVENT_AUTHDIR_NEWDESCS 0x000D +#define EVENT_DESCCHANGED 0x000E +/* Exposed above */ +// #define EVENT_NS 0x000F +#define EVENT_STATUS_CLIENT 0x0010 +#define EVENT_STATUS_SERVER 0x0011 +#define EVENT_STATUS_GENERAL 0x0012 +#define EVENT_GUARD 0x0013 +#define EVENT_STREAM_BANDWIDTH_USED 0x0014 +#define EVENT_CLIENTS_SEEN 0x0015 +#define EVENT_NEWCONSENSUS 0x0016 +#define EVENT_BUILDTIMEOUT_SET 0x0017 +#define EVENT_SIGNAL 0x0018 +#define EVENT_CONF_CHANGED 0x0019 +#define EVENT_CONN_BW 0x001A +#define EVENT_CELL_STATS 0x001B +#define EVENT_TB_EMPTY 0x001C +#define EVENT_CIRC_BANDWIDTH_USED 0x001D +#define EVENT_TRANSPORT_LAUNCHED 0x0020 +#define EVENT_HS_DESC 0x0021 +#define EVENT_MAX_ 0x0021 +/* If EVENT_MAX_ ever hits 0x0040, we need to make the mask into a + * different structure. */ + /* Used only by control.c and test.c */ -size_t write_escaped_data(const char *data, size_t len, char **out); -size_t read_escaped_data(const char *data, size_t len, char **out); +STATIC size_t write_escaped_data(const char *data, size_t len, char **out); +STATIC size_t read_escaped_data(const char *data, size_t len, char **out); +/** Flag for event_format_t. Indicates that we should use the one standard + format. (Other formats previous existed, and are now deprecated) + */ +#define ALL_FORMATS 1 +/** Bit field of flags to select how to format a controller event. Recognized + * flag is ALL_FORMATS. */ +typedef int event_format_t; + +#ifdef TOR_UNIT_TESTS +MOCK_DECL(STATIC void, +send_control_event_string,(uint16_t event, event_format_t which, + const char *msg)); + +void control_testing_set_global_event_mask(uint64_t mask); +#endif + +/** Helper structure: temporarily stores cell statistics for a circuit. */ +typedef struct cell_stats_t { + /** Number of cells added in app-ward direction by command. */ + uint64_t added_cells_appward[CELL_COMMAND_MAX_ + 1]; + /** Number of cells added in exit-ward direction by command. */ + uint64_t added_cells_exitward[CELL_COMMAND_MAX_ + 1]; + /** Number of cells removed in app-ward direction by command. */ + uint64_t removed_cells_appward[CELL_COMMAND_MAX_ + 1]; + /** Number of cells removed in exit-ward direction by command. */ + uint64_t removed_cells_exitward[CELL_COMMAND_MAX_ + 1]; + /** Total waiting time of cells in app-ward direction by command. */ + uint64_t total_time_appward[CELL_COMMAND_MAX_ + 1]; + /** Total waiting time of cells in exit-ward direction by command. */ + uint64_t total_time_exitward[CELL_COMMAND_MAX_ + 1]; +} cell_stats_t; +void sum_up_cell_stats_by_command(circuit_t *circ, + cell_stats_t *cell_stats); +void append_cell_stats_by_command(smartlist_t *event_parts, + const char *key, + const uint64_t *include_if_non_zero, + const uint64_t *number_to_include); +void format_cell_stats(char **event_string, circuit_t *circ, + cell_stats_t *cell_stats); #endif #endif diff --git a/src/or/cpuworker.c b/src/or/cpuworker.c index ecf0d2035d..61b2c29b38 100644 --- a/src/or/cpuworker.c +++ b/src/or/cpuworker.c @@ -436,7 +436,7 @@ cpuworker_main(void *data) if (req.task == CPUWORKER_TASK_ONION) { const create_cell_t *cc = &req.create_cell; created_cell_t *cell_out = &rpl.created_cell; - struct timeval tv_start, tv_end; + struct timeval tv_start = {0,0}, tv_end; int n; rpl.timed = req.timed; rpl.started_at = req.started_at; @@ -528,7 +528,12 @@ spawn_cpuworker(void) tor_assert(SOCKET_OK(fdarray[1])); fd = fdarray[0]; - spawn_func(cpuworker_main, (void*)fdarray); + if (spawn_func(cpuworker_main, (void*)fdarray) < 0) { + tor_close_socket(fdarray[0]); + tor_close_socket(fdarray[1]); + tor_free(fdarray); + return -1; + } log_debug(LD_OR,"just spawned a cpu worker."); #ifndef TOR_IS_MULTITHREADED tor_close_socket(fdarray[1]); /* don't need the worker's side of the pipe */ @@ -686,7 +691,7 @@ assign_onionskin_to_cpuworker(connection_t *cpuworker, } if (connection_or_digest_is_known_relay(circ->p_chan->identity_digest)) - rep_hist_note_circuit_handshake_completed(onionskin->handshake_type); + rep_hist_note_circuit_handshake_assigned(onionskin->handshake_type); should_time = should_time_request(onionskin->handshake_type); memset(&req, 0, sizeof(req)); diff --git a/src/or/directory.c b/src/or/directory.c index 3752367c44..51ea312689 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -67,15 +67,11 @@ static int purpose_needs_anonymity(uint8_t dir_purpose, uint8_t router_purpose); static char *http_get_header(const char *headers, const char *which); static void http_set_address_origin(const char *headers, connection_t *conn); -static void connection_dir_download_v2_networkstatus_failed( - dir_connection_t *conn, int status_code); static void connection_dir_download_routerdesc_failed(dir_connection_t *conn); static void connection_dir_bridge_routerdesc_failed(dir_connection_t *conn); static void connection_dir_download_cert_failed( dir_connection_t *conn, int status_code); static void connection_dir_retry_bridges(smartlist_t *descs); -static void dir_networkstatus_download_failed(smartlist_t *failed, - int status_code); static void dir_routerdesc_download_failed(smartlist_t *failed, int status_code, int router_purpose, @@ -86,8 +82,7 @@ static void dir_microdesc_download_failed(smartlist_t *failed, static void note_client_request(int purpose, int compressed, size_t bytes); static int client_likes_consensus(networkstatus_t *v, const char *want_url); -static void directory_initiate_command_rend(const char *address, - const tor_addr_t *addr, +static void directory_initiate_command_rend(const tor_addr_t *addr, uint16_t or_port, uint16_t dir_port, const char *digest, @@ -135,7 +130,6 @@ purpose_needs_anonymity(uint8_t dir_purpose, uint8_t router_purpose) if (dir_purpose == DIR_PURPOSE_UPLOAD_DIR || dir_purpose == DIR_PURPOSE_UPLOAD_VOTE || dir_purpose == DIR_PURPOSE_UPLOAD_SIGNATURES || - dir_purpose == DIR_PURPOSE_FETCH_V2_NETWORKSTATUS || dir_purpose == DIR_PURPOSE_FETCH_STATUS_VOTE || dir_purpose == DIR_PURPOSE_FETCH_DETACHED_SIGNATURES || dir_purpose == DIR_PURPOSE_FETCH_CONSENSUS || @@ -154,16 +148,10 @@ authdir_type_to_string(dirinfo_type_t auth) { char *result; smartlist_t *lst = smartlist_new(); - if (auth & V1_DIRINFO) - smartlist_add(lst, (void*)"V1"); - if (auth & V2_DIRINFO) - smartlist_add(lst, (void*)"V2"); if (auth & V3_DIRINFO) smartlist_add(lst, (void*)"V3"); if (auth & BRIDGE_DIRINFO) smartlist_add(lst, (void*)"Bridge"); - if (auth & HIDSERV_DIRINFO) - smartlist_add(lst, (void*)"Hidden service"); if (smartlist_len(lst)) { result = smartlist_join_strings(lst, ", ", 0, NULL); } else { @@ -179,18 +167,12 @@ dir_conn_purpose_to_string(int purpose) { switch (purpose) { - case DIR_PURPOSE_FETCH_RENDDESC: - return "hidden-service descriptor fetch"; case DIR_PURPOSE_UPLOAD_DIR: return "server descriptor upload"; - case DIR_PURPOSE_UPLOAD_RENDDESC: - return "hidden-service descriptor upload"; case DIR_PURPOSE_UPLOAD_VOTE: return "server vote upload"; case DIR_PURPOSE_UPLOAD_SIGNATURES: return "consensus signature upload"; - case DIR_PURPOSE_FETCH_V2_NETWORKSTATUS: - return "network-status fetch"; case DIR_PURPOSE_FETCH_SERVERDESC: return "server descriptor fetch"; case DIR_PURPOSE_FETCH_EXTRAINFO: @@ -258,13 +240,13 @@ directories_have_accepted_server_descriptor(void) /** Start a connection to every suitable directory authority, using * connection purpose <b>dir_purpose</b> and uploading <b>payload</b> * (of length <b>payload_len</b>). The dir_purpose should be one of - * 'DIR_PURPOSE_UPLOAD_DIR' or 'DIR_PURPOSE_UPLOAD_RENDDESC'. + * 'DIR_PURPOSE_UPLOAD_{DIR|VOTE|SIGNATURES}'. * * <b>router_purpose</b> describes the type of descriptor we're * publishing, if we're publishing a descriptor -- e.g. general or bridge. * - * <b>type</b> specifies what sort of dir authorities (V1, V2, - * HIDSERV, BRIDGE) we should upload to. + * <b>type</b> specifies what sort of dir authorities (V3, + * BRIDGE, etc) we should upload to. * * If <b>extrainfo_len</b> is nonzero, the first <b>payload_len</b> bytes of * <b>payload</b> hold a router descriptor, and the next <b>extrainfo_len</b> @@ -279,7 +261,7 @@ directory_post_to_dirservers(uint8_t dir_purpose, uint8_t router_purpose, size_t payload_len, size_t extrainfo_len) { const or_options_t *options = get_options(); - int post_via_tor; + dir_indirection_t indirection; const smartlist_t *dirservers = router_get_trusted_dir_servers(); int found = 0; const int exclude_self = (dir_purpose == DIR_PURPOSE_UPLOAD_VOTE || @@ -296,8 +278,12 @@ directory_post_to_dirservers(uint8_t dir_purpose, uint8_t router_purpose, if ((type & ds->type) == 0) continue; - if (exclude_self && router_digest_is_me(ds->digest)) + if (exclude_self && router_digest_is_me(ds->digest)) { + /* we don't upload to ourselves, but at least there's now at least + * one authority of this type that has what we wanted to upload. */ + found = 1; continue; + } if (options->StrictNodes && routerset_contains_routerstatus(options->ExcludeNodes, rs, -1)) { @@ -319,11 +305,19 @@ directory_post_to_dirservers(uint8_t dir_purpose, uint8_t router_purpose, (int) extrainfo_len); } tor_addr_from_ipv4h(&ds_addr, ds->addr); - post_via_tor = purpose_needs_anonymity(dir_purpose, router_purpose) || - !fascist_firewall_allows_address_dir(&ds_addr, ds->dir_port); + if (purpose_needs_anonymity(dir_purpose, router_purpose)) { + indirection = DIRIND_ANONYMOUS; + } else if (!fascist_firewall_allows_address_dir(&ds_addr,ds->dir_port)) { + if (fascist_firewall_allows_address_or(&ds_addr,ds->or_port)) + indirection = DIRIND_ONEHOP; + else + indirection = DIRIND_ANONYMOUS; + } else { + indirection = DIRIND_DIRECT_CONN; + } directory_initiate_command_routerstatus(rs, dir_purpose, router_purpose, - post_via_tor, + indirection, NULL, payload, upload_len, 0); } SMARTLIST_FOREACH_END(ds); if (!found) { @@ -350,15 +344,12 @@ should_use_directory_guards(const or_options_t *options) /* If we're configured to fetch directory info aggressively or of a * nonstandard type, don't use directory guards. */ if (options->DownloadExtraInfo || options->FetchDirInfoEarly || - options->FetchDirInfoExtraEarly || options->FetchUselessDescriptors || - options->FetchV2Networkstatus) - return 0; - if (! options->PreferTunneledDirConns) + options->FetchDirInfoExtraEarly || options->FetchUselessDescriptors) return 0; return 1; } -/** Pick an unconsetrained directory server from among our guards, the latest +/** Pick an unconstrained directory server from among our guards, the latest * networkstatus, or the fallback dirservers, for use in downloading * information of type <b>type</b>, and return its routerstatus. */ static const routerstatus_t * @@ -414,18 +405,10 @@ directory_get_from_dirserver(uint8_t dir_purpose, uint8_t router_purpose, (router_purpose == ROUTER_PURPOSE_BRIDGE ? BRIDGE_DIRINFO : V3_DIRINFO); break; - case DIR_PURPOSE_FETCH_V2_NETWORKSTATUS: - type = V2_DIRINFO; - prefer_authority = 1; /* Only v2 authorities have these anyway. */ - require_authority = 1; /* Don't fallback to asking a non-authority */ - break; case DIR_PURPOSE_FETCH_SERVERDESC: type = (router_purpose == ROUTER_PURPOSE_BRIDGE ? BRIDGE_DIRINFO : V3_DIRINFO); break; - case DIR_PURPOSE_FETCH_RENDDESC: - type = HIDSERV_DIRINFO; - break; case DIR_PURPOSE_FETCH_STATUS_VOTE: case DIR_PURPOSE_FETCH_DETACHED_SIGNATURES: case DIR_PURPOSE_FETCH_CERTIFICATE: @@ -465,7 +448,7 @@ directory_get_from_dirserver(uint8_t dir_purpose, uint8_t router_purpose, } } - if (!options->FetchServerDescriptors && type != HIDSERV_DIRINFO) + if (!options->FetchServerDescriptors) return; if (!get_via_tor) { @@ -484,7 +467,7 @@ directory_get_from_dirserver(uint8_t dir_purpose, uint8_t router_purpose, tor_addr_t addr; routerinfo_t *ri = node->ri; node_get_addr(node, &addr); - directory_initiate_command(ri->address, &addr, + directory_initiate_command(&addr, ri->or_port, 0/*no dirport*/, ri->cache_info.identity_digest, dir_purpose, @@ -536,11 +519,7 @@ directory_get_from_dirserver(uint8_t dir_purpose, uint8_t router_purpose, } } else { /* get_via_tor */ /* Never use fascistfirewall; we're going via Tor. */ - if (dir_purpose == DIR_PURPOSE_FETCH_RENDDESC) { - /* only ask hidserv authorities, any of them will do */ - pds_flags |= PDS_IGNORE_FASCISTFIREWALL|PDS_ALLOW_SELF; - rs = router_pick_trusteddirserver(HIDSERV_DIRINFO, pds_flags); - } else { + if (1) { /* anybody with a non-zero dirport will do. Disregard firewalls. */ pds_flags |= PDS_IGNORE_FASCISTFIREWALL; rs = router_pick_directory_server(type, pds_flags); @@ -617,9 +596,6 @@ directory_initiate_command_routerstatus_rend(const routerstatus_t *status, { const or_options_t *options = get_options(); const node_t *node; - char address_buf[INET_NTOA_BUF_LEN+1]; - struct in_addr in; - const char *address; tor_addr_t addr; const int anonymized_connection = dirind_is_anon(indirection); node = node_get_by_id(status->identity_digest); @@ -629,13 +605,6 @@ directory_initiate_command_routerstatus_rend(const routerstatus_t *status, "don't have its router descriptor.", routerstatus_describe(status)); return; - } else if (node) { - node_get_address_string(node, address_buf, sizeof(address_buf)); - address = address_buf; - } else { - in.s_addr = htonl(status->addr); - tor_inet_ntoa(&in, address_buf, sizeof(address_buf)); - address = address_buf; } tor_addr_from_ipv4h(&addr, status->addr); @@ -649,7 +618,7 @@ directory_initiate_command_routerstatus_rend(const routerstatus_t *status, return; } - directory_initiate_command_rend(address, &addr, + directory_initiate_command_rend(&addr, status->or_port, status->dir_port, status->identity_digest, dir_purpose, router_purpose, @@ -662,7 +631,7 @@ directory_initiate_command_routerstatus_rend(const routerstatus_t *status, * upload or download a server or rendezvous * descriptor. <b>dir_purpose</b> determines what * kind of directory connection we're launching, and must be one of - * DIR_PURPOSE_{FETCH|UPLOAD}_{DIR|RENDDESC|RENDDESC_V2}. <b>router_purpose</b> + * DIR_PURPOSE_{FETCH|UPLOAD}_{DIR|RENDDESC_V2}. <b>router_purpose</b> * specifies the descriptor purposes we have in mind (currently only * used for FETCH_DIR). * @@ -719,11 +688,7 @@ connection_dir_request_failed(dir_connection_t *conn) } if (!entry_list_is_constrained(get_options())) router_set_status(conn->identity_digest, 0); /* don't try him again */ - if (conn->base_.purpose == DIR_PURPOSE_FETCH_V2_NETWORKSTATUS) { - log_info(LD_DIR, "Giving up on directory server at '%s'; retrying", - conn->base_.address); - connection_dir_download_v2_networkstatus_failed(conn, -1); - } else if (conn->base_.purpose == DIR_PURPOSE_FETCH_SERVERDESC || + if (conn->base_.purpose == DIR_PURPOSE_FETCH_SERVERDESC || conn->base_.purpose == DIR_PURPOSE_FETCH_EXTRAINFO) { log_info(LD_DIR, "Giving up on serverdesc/extrainfo fetch from " "directory server at '%s'; retrying", @@ -747,48 +712,11 @@ connection_dir_request_failed(dir_connection_t *conn) conn->base_.address); } else if (conn->base_.purpose == DIR_PURPOSE_FETCH_MICRODESC) { log_info(LD_DIR, "Giving up on downloading microdescriptors from " - " directory server at '%s'; will retry", conn->base_.address); + "directory server at '%s'; will retry", conn->base_.address); connection_dir_download_routerdesc_failed(conn); } } -/** Called when an attempt to download one or more network status - * documents on connection <b>conn</b> failed. Decide whether to - * retry the fetch now, later, or never. - */ -static void -connection_dir_download_v2_networkstatus_failed(dir_connection_t *conn, - int status_code) -{ - if (!conn->requested_resource) { - /* We never reached directory_send_command, which means that we never - * opened a network connection. Either we're out of sockets, or the - * network is down. Either way, retrying would be pointless. */ - return; - } - if (!strcmpstart(conn->requested_resource, "all")) { - /* We're a non-authoritative directory cache; try again. Ignore status - * code, since we don't want to keep trying forever in a tight loop - * if all the authorities are shutting us out. */ - const smartlist_t *trusted_dirs = router_get_trusted_dir_servers(); - SMARTLIST_FOREACH(trusted_dirs, dir_server_t *, ds, - download_status_failed(&ds->v2_ns_dl_status, 0)); - directory_get_from_dirserver(conn->base_.purpose, conn->router_purpose, - "all.z", 0 /* don't retry_if_no_servers */); - } else if (!strcmpstart(conn->requested_resource, "fp/")) { - /* We were trying to download by fingerprint; mark them all as having - * failed, and possibly retry them later.*/ - smartlist_t *failed = smartlist_new(); - dir_split_resource_into_fingerprints(conn->requested_resource+3, - failed, NULL, 0); - if (smartlist_len(failed)) { - dir_networkstatus_download_failed(failed, status_code); - SMARTLIST_FOREACH(failed, char *, cp, tor_free(cp)); - } - smartlist_free(failed); - } -} - /** Helper: Attempt to fetch directly the descriptors of each bridge * listed in <b>failed</b>. */ @@ -912,6 +840,7 @@ directory_command_should_use_begindir(const or_options_t *options, int or_port, uint8_t router_purpose, dir_indirection_t indirection) { + (void) router_purpose; if (!or_port) return 0; /* We don't know an ORPort -- no chance. */ if (indirection == DIRIND_DIRECT_CONN || indirection == DIRIND_ANON_DIRPORT) @@ -920,9 +849,6 @@ directory_command_should_use_begindir(const or_options_t *options, if (!fascist_firewall_allows_address_or(addr, or_port) || directory_fetches_from_authorities(options)) return 0; /* We're firewalled or are acting like a relay -- also no. */ - if (!options->TunnelDirConns && - router_purpose != ROUTER_PURPOSE_BRIDGE) - return 0; /* We prefer to avoid using begindir conns. Fine. */ return 1; } @@ -932,7 +858,7 @@ directory_command_should_use_begindir(const or_options_t *options, * <b>supports_begindir</b>, and whose identity key digest is * <b>digest</b>. */ void -directory_initiate_command(const char *address, const tor_addr_t *_addr, +directory_initiate_command(const tor_addr_t *_addr, uint16_t or_port, uint16_t dir_port, const char *digest, uint8_t dir_purpose, uint8_t router_purpose, @@ -940,7 +866,7 @@ directory_initiate_command(const char *address, const tor_addr_t *_addr, const char *payload, size_t payload_len, time_t if_modified_since) { - directory_initiate_command_rend(address, _addr, or_port, dir_port, + directory_initiate_command_rend(_addr, or_port, dir_port, digest, dir_purpose, router_purpose, indirection, resource, payload, payload_len, @@ -954,9 +880,7 @@ directory_initiate_command(const char *address, const tor_addr_t *_addr, static int is_sensitive_dir_purpose(uint8_t dir_purpose) { - return ((dir_purpose == DIR_PURPOSE_FETCH_RENDDESC) || - (dir_purpose == DIR_PURPOSE_HAS_FETCHED_RENDDESC) || - (dir_purpose == DIR_PURPOSE_UPLOAD_RENDDESC) || + return ((dir_purpose == DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2) || (dir_purpose == DIR_PURPOSE_UPLOAD_RENDDESC_V2) || (dir_purpose == DIR_PURPOSE_FETCH_RENDDESC_V2)); } @@ -964,7 +888,7 @@ is_sensitive_dir_purpose(uint8_t dir_purpose) /** Same as directory_initiate_command(), but accepts rendezvous data to * fetch a hidden service descriptor. */ static void -directory_initiate_command_rend(const char *address, const tor_addr_t *_addr, +directory_initiate_command_rend(const tor_addr_t *_addr, uint16_t or_port, uint16_t dir_port, const char *digest, uint8_t dir_purpose, uint8_t router_purpose, @@ -982,7 +906,6 @@ directory_initiate_command_rend(const char *address, const tor_addr_t *_addr, const int anonymized_connection = dirind_is_anon(indirection); tor_addr_t addr; - tor_assert(address); tor_assert(_addr); tor_assert(or_port || dir_port); tor_assert(digest); @@ -1015,7 +938,7 @@ directory_initiate_command_rend(const char *address, const tor_addr_t *_addr, /* set up conn so it's got all the data we need to remember */ tor_addr_copy(&conn->base_.addr, &addr); conn->base_.port = use_begindir ? or_port : dir_port; - conn->base_.address = tor_strdup(address); + conn->base_.address = tor_dup_addr(&addr); memcpy(conn->identity_digest, digest, DIGEST_LEN); conn->base_.purpose = dir_purpose; @@ -1250,11 +1173,6 @@ directory_send_command(dir_connection_t *conn, } switch (purpose) { - case DIR_PURPOSE_FETCH_V2_NETWORKSTATUS: - tor_assert(resource); - httpcommand = "GET"; - tor_asprintf(&url, "/tor/status/%s", resource); - break; case DIR_PURPOSE_FETCH_CONSENSUS: /* resource is optional. If present, it's a flavor name */ tor_assert(!payload); @@ -1326,12 +1244,6 @@ directory_send_command(dir_connection_t *conn, httpcommand = "GET"; tor_asprintf(&url, "/tor/rendezvous2/%s", resource); break; - case DIR_PURPOSE_UPLOAD_RENDDESC: - tor_assert(!resource); - tor_assert(payload); - httpcommand = "POST"; - url = tor_strdup("/tor/rendezvous/publish"); - break; case DIR_PURPOSE_UPLOAD_RENDDESC_V2: tor_assert(!resource); tor_assert(payload); @@ -1387,7 +1299,7 @@ directory_send_command(dir_connection_t *conn, * so it does. Return 0. * Otherwise, return -1. */ -static int +STATIC int parse_http_url(const char *headers, char **url) { char *s, *start, *tmp; @@ -1416,6 +1328,19 @@ parse_http_url(const char *headers, char **url) } } + /* Check if the header is well formed (next sequence + * should be HTTP/1.X\r\n). Assumes we're supporting 1.0? */ + { + unsigned minor_ver; + char ch; + char *e = (char *)eat_whitespace_no_nl(s); + if (2 != tor_sscanf(e, "HTTP/1.%u%c", &minor_ver, &ch)) { + return -1; + } + if (ch != '\r') + return -1; + } + if (s-start < 5 || strcmpstart(start,"/tor/")) { /* need to rewrite it */ *url = tor_malloc(s - start + 5); strlcpy(*url,"/tor", s-start+5); @@ -1462,13 +1387,14 @@ http_set_address_origin(const char *headers, connection_t *conn) if (!fwd) fwd = http_get_header(headers, "X-Forwarded-For: "); if (fwd) { - struct in_addr in; - if (!tor_inet_aton(fwd, &in) || is_internal_IP(ntohl(in.s_addr), 0)) { - log_debug(LD_DIR, "Ignoring unrecognized or internal IP %s", - escaped(fwd)); + tor_addr_t toraddr; + if (tor_addr_parse(&toraddr,fwd) == -1 || + tor_addr_is_internal(&toraddr,0)) { + log_debug(LD_DIR, "Ignoring local/internal IP %s", escaped(fwd)); tor_free(fwd); return; } + tor_free(conn->address); conn->address = tor_strdup(fwd); tor_free(fwd); @@ -1565,8 +1491,8 @@ parse_http_response(const char *headers, int *code, time_t *date, } /** Return true iff <b>body</b> doesn't start with a plausible router or - * running-list or directory opening. This is a sign of possible compression. - **/ + * network-status or microdescriptor opening. This is a sign of possible + * compression. */ static int body_is_plausible(const char *body, size_t len, int purpose) { @@ -1578,20 +1504,16 @@ body_is_plausible(const char *body, size_t len, int purpose) if (purpose == DIR_PURPOSE_FETCH_MICRODESC) { return (!strcmpstart(body,"onion-key")); } - if (purpose != DIR_PURPOSE_FETCH_RENDDESC) { + if (1) { if (!strcmpstart(body,"router") || - !strcmpstart(body,"signed-directory") || - !strcmpstart(body,"network-status") || - !strcmpstart(body,"running-routers")) - return 1; + !strcmpstart(body,"network-status")) + return 1; for (i=0;i<32;++i) { if (!TOR_ISPRINT(body[i]) && !TOR_ISSPACE(body[i])) return 0; } - return 1; - } else { - return 1; } + return 1; } /** Called when we've just fetched a bunch of router descriptors in @@ -1626,8 +1548,9 @@ load_downloaded_routers(const char *body, smartlist_t *which, added = router_load_routers_from_string(body, NULL, SAVED_NOWHERE, which, descriptor_digests, buf); - control_event_bootstrap(BOOTSTRAP_STATUS_LOADING_DESCRIPTORS, - count_loading_descriptors_progress()); + if (general) + control_event_bootstrap(BOOTSTRAP_STATUS_LOADING_DESCRIPTORS, + count_loading_descriptors_progress()); return added; } @@ -1646,17 +1569,17 @@ connection_dir_client_reached_eof(dir_connection_t *conn) char *body; char *headers; char *reason = NULL; - size_t body_len=0, orig_len=0; + size_t body_len = 0, orig_len = 0; int status_code; - time_t date_header=0; + time_t date_header = 0; long delta; compress_method_t compression; int plausible; - int skewed=0; + int skewed = 0; int allow_partial = (conn->base_.purpose == DIR_PURPOSE_FETCH_SERVERDESC || conn->base_.purpose == DIR_PURPOSE_FETCH_EXTRAINFO || conn->base_.purpose == DIR_PURPOSE_FETCH_MICRODESC); - int was_compressed=0; + int was_compressed = 0; time_t now = time(NULL); int src_code; @@ -1810,77 +1733,6 @@ connection_dir_client_reached_eof(dir_connection_t *conn) } } - if (conn->base_.purpose == DIR_PURPOSE_FETCH_V2_NETWORKSTATUS) { - smartlist_t *which = NULL; - v2_networkstatus_source_t source; - char *cp; - log_info(LD_DIR,"Received networkstatus objects (size %d) from server " - "'%s:%d'", (int)body_len, conn->base_.address, conn->base_.port); - if (status_code != 200) { - static ratelim_t warning_limit = RATELIM_INIT(3600); - char *m; - if ((m = rate_limit_log(&warning_limit, now))) { - log_warn(LD_DIR, - "Received http status code %d (%s) from server " - "'%s:%d' while fetching \"/tor/status/%s\". " - "I'll try again soon.%s", - status_code, escaped(reason), conn->base_.address, - conn->base_.port, conn->requested_resource, m); - tor_free(m); - } - tor_free(body); tor_free(headers); tor_free(reason); - connection_dir_download_v2_networkstatus_failed(conn, status_code); - return -1; - } - if (conn->requested_resource && - !strcmpstart(conn->requested_resource,"fp/")) { - source = NS_FROM_DIR_BY_FP; - which = smartlist_new(); - dir_split_resource_into_fingerprints(conn->requested_resource+3, - which, NULL, 0); - } else if (conn->requested_resource && - !strcmpstart(conn->requested_resource, "all")) { - source = NS_FROM_DIR_ALL; - which = smartlist_new(); - SMARTLIST_FOREACH(router_get_trusted_dir_servers(), - dir_server_t *, ds, - { - char *hex = tor_malloc(HEX_DIGEST_LEN+1); - base16_encode(hex, HEX_DIGEST_LEN+1, ds->digest, DIGEST_LEN); - smartlist_add(which, hex); - }); - } else { - /* XXXX Can we even end up here? -- weasel*/ - source = NS_FROM_DIR_BY_FP; - log_warn(LD_BUG, "We received a networkstatus but we didn't ask " - "for it by fp, nor did we ask for all."); - } - cp = body; - while (*cp) { - char *next = strstr(cp, "\nnetwork-status-version"); - if (next) - next[1] = '\0'; - /* learn from it, and then remove it from 'which' */ - if (router_set_networkstatus_v2(cp, now, source, which)<0) - break; - if (next) { - next[1] = 'n'; - cp = next+1; - } else - break; - } - /* launches router downloads as needed */ - routers_update_all_from_networkstatus(now, 2); - directory_info_has_arrived(now, 0); - if (which) { - if (smartlist_len(which)) { - dir_networkstatus_download_failed(which, status_code); - } - SMARTLIST_FOREACH(which, char *, s, tor_free(s)); - smartlist_free(which); - } - } - if (conn->base_.purpose == DIR_PURPOSE_FETCH_CONSENSUS) { int r; const char *flavname = conn->requested_resource; @@ -2220,47 +2072,10 @@ connection_dir_client_reached_eof(dir_connection_t *conn) * dirservers down just because they don't like us. */ } - if (conn->base_.purpose == DIR_PURPOSE_FETCH_RENDDESC) { - tor_assert(conn->rend_data); - log_info(LD_REND,"Received rendezvous descriptor (size %d, status %d " - "(%s))", - (int)body_len, status_code, escaped(reason)); - switch (status_code) { - case 200: - if (rend_cache_store(body, body_len, 0, - conn->rend_data->onion_address) < -1) { - log_warn(LD_REND,"Failed to parse rendezvous descriptor."); - /* Any pending rendezvous attempts will notice when - * connection_about_to_close_connection() - * cleans this dir conn up. */ - /* We could retry. But since v0 descriptors are going out of - * style, it isn't worth the hassle. We'll do better in v2. */ - } else { - /* Success, or at least there's a v2 descriptor already - * present. Notify pending connections about this. */ - conn->base_.purpose = DIR_PURPOSE_HAS_FETCHED_RENDDESC; - rend_client_desc_trynow(conn->rend_data->onion_address); - } - break; - case 404: - /* Not there. Pending connections will be notified when - * connection_about_to_close_connection() cleans this conn up. */ - break; - case 400: - log_warn(LD_REND, - "http status 400 (%s). Dirserver didn't like our " - "rendezvous query?", escaped(reason)); - break; - default: - log_warn(LD_REND,"http status %d (%s) response unexpected while " - "fetching hidden service descriptor (server '%s:%d').", - status_code, escaped(reason), conn->base_.address, - conn->base_.port); - break; - } - } - if (conn->base_.purpose == DIR_PURPOSE_FETCH_RENDDESC_V2) { + #define SEND_HS_DESC_FAILED_EVENT() ( \ + control_event_hs_descriptor_failed(conn->rend_data, \ + conn->identity_digest) ) tor_assert(conn->rend_data); log_info(LD_REND,"Received rendezvous descriptor (size %d, status %d " "(%s))", @@ -2268,24 +2083,22 @@ connection_dir_client_reached_eof(dir_connection_t *conn) switch (status_code) { case 200: switch (rend_cache_store_v2_desc_as_client(body, conn->rend_data)) { - case -2: + case RCS_BADDESC: + case RCS_NOTDIR: /* Impossible */ log_warn(LD_REND,"Fetching v2 rendezvous descriptor failed. " "Retrying at another directory."); /* We'll retry when connection_about_to_close_connection() * cleans this dir conn up. */ + SEND_HS_DESC_FAILED_EVENT(); break; - case -1: - /* We already have a v0 descriptor here. Ignoring this one - * and _not_ performing another request. */ - log_info(LD_REND, "Successfully fetched v2 rendezvous " - "descriptor, but we already have a v0 descriptor."); - conn->base_.purpose = DIR_PURPOSE_HAS_FETCHED_RENDDESC; - break; + case RCS_OKAY: default: /* success. notify pending connections about this. */ log_info(LD_REND, "Successfully fetched v2 rendezvous " "descriptor."); - conn->base_.purpose = DIR_PURPOSE_HAS_FETCHED_RENDDESC; + control_event_hs_descriptor_received(conn->rend_data, + conn->identity_digest); + conn->base_.purpose = DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2; rend_client_desc_trynow(conn->rend_data->onion_address); break; } @@ -2295,12 +2108,14 @@ connection_dir_client_reached_eof(dir_connection_t *conn) * connection_about_to_close_connection() cleans this conn up. */ log_info(LD_REND,"Fetching v2 rendezvous descriptor failed: " "Retrying at another directory."); + SEND_HS_DESC_FAILED_EVENT(); break; case 400: log_warn(LD_REND, "Fetching v2 rendezvous descriptor failed: " "http status 400 (%s). Dirserver didn't like our " "v2 rendezvous query? Retrying at another directory.", escaped(reason)); + SEND_HS_DESC_FAILED_EVENT(); break; default: log_warn(LD_REND, "Fetching v2 rendezvous descriptor failed: " @@ -2309,12 +2124,12 @@ connection_dir_client_reached_eof(dir_connection_t *conn) "Retrying at another directory.", status_code, escaped(reason), conn->base_.address, conn->base_.port); + SEND_HS_DESC_FAILED_EVENT(); break; } } - if (conn->base_.purpose == DIR_PURPOSE_UPLOAD_RENDDESC || - conn->base_.purpose == DIR_PURPOSE_UPLOAD_RENDDESC_V2) { + if (conn->base_.purpose == DIR_PURPOSE_UPLOAD_RENDDESC_V2) { log_info(LD_REND,"Uploaded rendezvous descriptor (status %d " "(%s))", status_code, escaped(reason)); @@ -2418,7 +2233,7 @@ connection_dir_about_to_close(dir_connection_t *dir_conn) } /* If we were trying to fetch a v2 rend desc and did not succeed, * retry as needed. (If a fetch is successful, the connection state - * is changed to DIR_PURPOSE_HAS_FETCHED_RENDDESC to mark that + * is changed to DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2 to mark that * refetching is unnecessary.) */ if (conn->purpose == DIR_PURPOSE_FETCH_RENDDESC_V2 && dir_conn->rend_data && @@ -2492,7 +2307,7 @@ write_http_response_header_impl(dir_connection_t *conn, ssize_t length, } if (cache_lifetime > 0) { char expbuf[RFC1123_TIME_LEN+1]; - format_rfc1123_time(expbuf, now + cache_lifetime); + format_rfc1123_time(expbuf, (time_t)(now + cache_lifetime)); /* We could say 'Cache-control: max-age=%d' here if we start doing * http/1.1 */ tor_snprintf(cp, sizeof(tmp)-(cp-tmp), @@ -2549,7 +2364,6 @@ note_client_request(int purpose, int compressed, size_t bytes) char *key; const char *kind = NULL; switch (purpose) { - case DIR_PURPOSE_FETCH_V2_NETWORKSTATUS: kind = "dl/status"; break; case DIR_PURPOSE_FETCH_CONSENSUS: kind = "dl/consensus"; break; case DIR_PURPOSE_FETCH_CERTIFICATE: kind = "dl/cert"; break; case DIR_PURPOSE_FETCH_STATUS_VOTE: kind = "dl/vote"; break; @@ -2560,9 +2374,7 @@ note_client_request(int purpose, int compressed, size_t bytes) case DIR_PURPOSE_UPLOAD_DIR: kind = "dl/ul-dir"; break; case DIR_PURPOSE_UPLOAD_VOTE: kind = "dl/ul-vote"; break; case DIR_PURPOSE_UPLOAD_SIGNATURES: kind = "dl/ul-sig"; break; - case DIR_PURPOSE_FETCH_RENDDESC: kind = "dl/rend"; break; case DIR_PURPOSE_FETCH_RENDDESC_V2: kind = "dl/rend2"; break; - case DIR_PURPOSE_UPLOAD_RENDDESC: kind = "dl/ul-rend"; break; case DIR_PURPOSE_UPLOAD_RENDDESC_V2: kind = "dl/ul-rend2"; break; } if (kind) { @@ -2774,109 +2586,13 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, /* if no disclaimer file, fall through and continue */ } - if (!strcmp(url,"/tor/") || !strcmp(url,"/tor/dir")) { /* v1 dir fetch */ - cached_dir_t *d = dirserv_get_directory(); - - if (!d) { - log_info(LD_DIRSERV,"Client asked for the mirrored directory, but we " - "don't have a good one yet. Sending 503 Dir not available."); - write_http_status_line(conn, 503, "Directory unavailable"); - goto done; - } - if (d->published < if_modified_since) { - write_http_status_line(conn, 304, "Not modified"); - goto done; - } - - dlen = compressed ? d->dir_z_len : d->dir_len; - - if (global_write_bucket_low(TO_CONN(conn), dlen, 1)) { - log_debug(LD_DIRSERV, - "Client asked for the mirrored directory, but we've been " - "writing too many bytes lately. Sending 503 Dir busy."); - write_http_status_line(conn, 503, "Directory busy, try again later"); - goto done; - } - - note_request(url, dlen); - - log_debug(LD_DIRSERV,"Dumping %sdirectory to client.", - compressed?"compressed ":""); - write_http_response_header(conn, dlen, compressed, - FULL_DIR_CACHE_LIFETIME); - conn->cached_dir = d; - conn->cached_dir_offset = 0; - if (!compressed) - conn->zlib_state = tor_zlib_new(0, ZLIB_METHOD); - ++d->refcnt; - - /* Prime the connection with some data. */ - conn->dir_spool_src = DIR_SPOOL_CACHED_DIR; - connection_dirserv_flushed_some(conn); - goto done; - } - - if (!strcmp(url,"/tor/running-routers")) { /* running-routers fetch */ - cached_dir_t *d = dirserv_get_runningrouters(); - if (!d) { - write_http_status_line(conn, 503, "Directory unavailable"); - goto done; - } - if (d->published < if_modified_since) { - write_http_status_line(conn, 304, "Not modified"); - goto done; - } - dlen = compressed ? d->dir_z_len : d->dir_len; - - if (global_write_bucket_low(TO_CONN(conn), dlen, 1)) { - log_info(LD_DIRSERV, - "Client asked for running-routers, but we've been " - "writing too many bytes lately. Sending 503 Dir busy."); - write_http_status_line(conn, 503, "Directory busy, try again later"); - goto done; - } - note_request(url, dlen); - write_http_response_header(conn, dlen, compressed, - RUNNINGROUTERS_CACHE_LIFETIME); - connection_write_to_buf(compressed ? d->dir_z : d->dir, dlen, - TO_CONN(conn)); - goto done; - } - - if (!strcmpstart(url,"/tor/status/") - || !strcmpstart(url, "/tor/status-vote/current/consensus")) { - /* v2 or v3 network status fetch. */ + if (!strcmpstart(url, "/tor/status-vote/current/consensus")) { + /* v3 network status fetch. */ smartlist_t *dir_fps = smartlist_new(); - int is_v3 = !strcmpstart(url, "/tor/status-vote"); const char *request_type = NULL; - const char *key = url + strlen("/tor/status/"); long lifetime = NETWORKSTATUS_CACHE_LIFETIME; - if (options->DisableV2DirectoryInfo_ && !is_v3) { - static ratelim_t reject_v2_ratelim = RATELIM_INIT(1800); - char *m; - write_http_status_line(conn, 404, "Not found"); - smartlist_free(dir_fps); - geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND); - if ((m = rate_limit_log(&reject_v2_ratelim, approx_time()))) { - log_notice(LD_DIR, "Rejected a v2 networkstatus request.%s", m); - tor_free(m); - } - goto done; - } - - if (!is_v3) { - dirserv_get_networkstatus_v2_fingerprints(dir_fps, key); - if (!strcmpstart(key, "fp/")) - request_type = compressed?"/tor/status/fp.z":"/tor/status/fp"; - else if (!strcmpstart(key, "authority")) - request_type = compressed?"/tor/status/authority.z": - "/tor/status/authority"; - else if (!strcmpstart(key, "all")) - request_type = compressed?"/tor/status/all.z":"/tor/status/all"; - else - request_type = "/tor/status/?"; - } else { + if (1) { networkstatus_t *v; time_t now = time(NULL); const char *want_fps = NULL; @@ -2929,8 +2645,7 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, if (!smartlist_len(dir_fps)) { /* we failed to create/cache cp */ write_http_status_line(conn, 503, "Network status object unavailable"); smartlist_free(dir_fps); - if (is_v3) - geoip_note_ns_response(GEOIP_REJECT_UNAVAILABLE); + geoip_note_ns_response(GEOIP_REJECT_UNAVAILABLE); goto done; } @@ -2938,15 +2653,13 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, write_http_status_line(conn, 404, "Not found"); SMARTLIST_FOREACH(dir_fps, char *, cp, tor_free(cp)); smartlist_free(dir_fps); - if (is_v3) - geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND); + geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND); goto done; } else if (!smartlist_len(dir_fps)) { write_http_status_line(conn, 304, "Not modified"); SMARTLIST_FOREACH(dir_fps, char *, cp, tor_free(cp)); smartlist_free(dir_fps); - if (is_v3) - geoip_note_ns_response(GEOIP_REJECT_NOT_MODIFIED); + geoip_note_ns_response(GEOIP_REJECT_NOT_MODIFIED); goto done; } @@ -2958,17 +2671,19 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, write_http_status_line(conn, 503, "Directory busy, try again later"); SMARTLIST_FOREACH(dir_fps, char *, fp, tor_free(fp)); smartlist_free(dir_fps); - if (is_v3) - geoip_note_ns_response(GEOIP_REJECT_BUSY); + + geoip_note_ns_response(GEOIP_REJECT_BUSY); goto done; } - if (is_v3) { + if (1) { struct in_addr in; tor_addr_t addr; if (tor_inet_aton((TO_CONN(conn))->address, &in)) { tor_addr_from_ipv4h(&addr, ntohl(in.s_addr)); - geoip_note_client_seen(GEOIP_CLIENT_NETWORKSTATUS, &addr, time(NULL)); + geoip_note_client_seen(GEOIP_CLIENT_NETWORKSTATUS, + &addr, NULL, + time(NULL)); geoip_note_ns_response(GEOIP_SUCCESS); /* Note that a request for a network status has started, so that we * can measure the download time later on. */ @@ -3310,32 +3025,6 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, goto done; } - if (options->HSAuthoritativeDir && !strcmpstart(url,"/tor/rendezvous/")) { - /* rendezvous descriptor fetch */ - const char *descp; - size_t desc_len; - const char *query = url+strlen("/tor/rendezvous/"); - - log_info(LD_REND, "Handling rendezvous descriptor get"); - switch (rend_cache_lookup_desc(query, 0, &descp, &desc_len)) { - case 1: /* valid */ - write_http_response_header_impl(conn, desc_len, - "application/octet-stream", - NULL, NULL, 0); - note_request("/tor/rendezvous?/", desc_len); - /* need to send descp separately, because it may include NULs */ - connection_write_to_buf(descp, desc_len, TO_CONN(conn)); - break; - case 0: /* well-formed but not present */ - write_http_status_line(conn, 404, "Not found"); - break; - case -1: /* not well-formed */ - write_http_status_line(conn, 400, "Bad request"); - break; - } - goto done; - } - if (options->BridgeAuthoritativeDir && options->BridgePassword_AuthDigest_ && connection_dir_is_encrypted(conn) && @@ -3384,22 +3073,6 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, goto done; } - if (!strcmp(url,"/tor/dbg-stability.txt")) { - const char *stability; - size_t len; - if (options->BridgeAuthoritativeDir || - ! authdir_mode_tests_reachability(options) || - ! (stability = rep_hist_get_router_stability_doc(time(NULL)))) { - write_http_status_line(conn, 404, "Not found."); - goto done; - } - - len = strlen(stability); - write_http_response_header(conn, len, 0, 0); - connection_write_to_buf(stability, len, TO_CONN(conn)); - goto done; - } - #if defined(EXPORTMALLINFO) && defined(HAVE_MALLOC_H) && defined(HAVE_MALLINFO) #define ADD_MALLINFO_LINE(x) do { \ smartlist_add_asprintf(lines, "%s %d\n", #x, mi.x); \ @@ -3474,19 +3147,20 @@ directory_handle_command_post(dir_connection_t *conn, const char *headers, connection_dir_is_encrypted(conn) && !strcmpstart(url,"/tor/rendezvous2/publish")) { switch (rend_cache_store_v2_desc_as_dir(body)) { - case -2: + case RCS_NOTDIR: log_info(LD_REND, "Rejected v2 rend descriptor (length %d) from %s " "since we're not currently a hidden service directory.", (int)body_len, conn->base_.address); write_http_status_line(conn, 503, "Currently not acting as v2 " "hidden service directory"); break; - case -1: + case RCS_BADDESC: log_warn(LD_REND, "Rejected v2 rend descriptor (length %d) from %s.", (int)body_len, conn->base_.address); write_http_status_line(conn, 400, "Invalid v2 service descriptor rejected"); break; + case RCS_OKAY: default: write_http_status_line(conn, 200, "Service descriptor (v2) stored"); log_info(LD_REND, "Handled v2 rendezvous descriptor post: accepted"); @@ -3510,8 +3184,6 @@ directory_handle_command_post(dir_connection_t *conn, const char *headers, was_router_added_t r = dirserv_add_multiple_descriptors(body, purpose, conn->base_.address, &msg); tor_assert(msg); - if (WRA_WAS_ADDED(r)) - dirserv_get_directory(); /* rebuild and write to disk */ if (r == ROUTER_ADDED_NOTIFY_GENERATOR) { /* Accepted with a message. */ @@ -3535,22 +3207,6 @@ directory_handle_command_post(dir_connection_t *conn, const char *headers, goto done; } - if (options->HSAuthoritativeDir && - !strcmpstart(url,"/tor/rendezvous/publish")) { - /* rendezvous descriptor post */ - log_info(LD_REND, "Handling rendezvous descriptor post."); - if (rend_cache_store(body, body_len, 1, NULL) < 0) { - log_fn(LOG_PROTOCOL_WARN, LD_DIRSERV, - "Rejected rend descriptor (length %d) from %s.", - (int)body_len, conn->base_.address); - write_http_status_line(conn, 400, - "Invalid v0 service descriptor rejected"); - } else { - write_http_status_line(conn, 200, "Service descriptor (v0) stored"); - } - goto done; - } - if (authdir_mode_v3(options) && !strcmp(url,"/tor/post/vote")) { /* v3 networkstatus vote */ const char *msg = "OK"; @@ -3702,80 +3358,27 @@ connection_dir_finished_connecting(dir_connection_t *conn) return 0; } -/** Called when one or more networkstatus fetches have failed (with uppercase - * fingerprints listed in <b>failed</b>). Mark those fingerprints as having - * failed once, unless they failed with status code 503. */ -static void -dir_networkstatus_download_failed(smartlist_t *failed, int status_code) -{ - if (status_code == 503) - return; - SMARTLIST_FOREACH_BEGIN(failed, const char *, fp) { - char digest[DIGEST_LEN]; - dir_server_t *dir; - if (base16_decode(digest, DIGEST_LEN, fp, strlen(fp))<0) { - log_warn(LD_BUG, "Called with bad fingerprint in list: %s", - escaped(fp)); - continue; - } - dir = router_get_fallback_dirserver_by_digest(digest); - - if (dir) - download_status_failed(&dir->v2_ns_dl_status, status_code); - } SMARTLIST_FOREACH_END(fp); -} - -/** Schedule for when servers should download things in general. */ -static const int server_dl_schedule[] = { - 0, 0, 0, 60, 60, 60*2, 60*5, 60*15, INT_MAX -}; -/** Schedule for when clients should download things in general. */ -static const int client_dl_schedule[] = { - 0, 0, 60, 60*5, 60*10, INT_MAX -}; -/** Schedule for when servers should download consensuses. */ -static const int server_consensus_dl_schedule[] = { - 0, 0, 60, 60*5, 60*10, 60*30, 60*30, 60*30, 60*30, 60*30, 60*60, 60*60*2 -}; -/** Schedule for when clients should download consensuses. */ -static const int client_consensus_dl_schedule[] = { - 0, 0, 60, 60*5, 60*10, 60*30, 60*60, 60*60, 60*60, 60*60*3, 60*60*6, 60*60*12 -}; -/** Schedule for when clients should download bridge descriptors. */ -static const int bridge_dl_schedule[] = { - 60*60, 15*60, 15*60, 60*60 -}; - -/** Decide which download schedule we want to use, and then return a - * pointer to it along with a pointer to its length. Helper function for - * download_status_increment_failure() and download_status_reset(). */ -static void -find_dl_schedule_and_len(download_status_t *dls, int server, - const int **schedule, size_t *schedule_len) +/** Decide which download schedule we want to use based on descriptor type + * in <b>dls</b> and whether we are acting as directory <b>server</b>, and + * then return a list of int pointers defining download delays in seconds. + * Helper function for download_status_increment_failure() and + * download_status_reset(). */ +static const smartlist_t * +find_dl_schedule_and_len(download_status_t *dls, int server) { switch (dls->schedule) { case DL_SCHED_GENERIC: - if (server) { - *schedule = server_dl_schedule; - *schedule_len = sizeof(server_dl_schedule)/sizeof(int); - } else { - *schedule = client_dl_schedule; - *schedule_len = sizeof(client_dl_schedule)/sizeof(int); - } - break; + if (server) + return get_options()->TestingServerDownloadSchedule; + else + return get_options()->TestingClientDownloadSchedule; case DL_SCHED_CONSENSUS: - if (server) { - *schedule = server_consensus_dl_schedule; - *schedule_len = sizeof(server_consensus_dl_schedule)/sizeof(int); - } else { - *schedule = client_consensus_dl_schedule; - *schedule_len = sizeof(client_consensus_dl_schedule)/sizeof(int); - } - break; + if (server) + return get_options()->TestingServerConsensusDownloadSchedule; + else + return get_options()->TestingClientConsensusDownloadSchedule; case DL_SCHED_BRIDGE: - *schedule = bridge_dl_schedule; - *schedule_len = sizeof(bridge_dl_schedule)/sizeof(int); - break; + return get_options()->TestingBridgeDownloadSchedule; default: tor_assert(0); } @@ -3789,8 +3392,7 @@ time_t download_status_increment_failure(download_status_t *dls, int status_code, const char *item, int server, time_t now) { - const int *schedule; - size_t schedule_len; + const smartlist_t *schedule; int increment; tor_assert(dls); if (status_code != 503 || server) { @@ -3798,14 +3400,14 @@ download_status_increment_failure(download_status_t *dls, int status_code, ++dls->n_download_failures; } - find_dl_schedule_and_len(dls, server, &schedule, &schedule_len); + schedule = find_dl_schedule_and_len(dls, server); - if (dls->n_download_failures < schedule_len) - increment = schedule[dls->n_download_failures]; + if (dls->n_download_failures < smartlist_len(schedule)) + increment = *(int *)smartlist_get(schedule, dls->n_download_failures); else if (dls->n_download_failures == IMPOSSIBLE_TO_DOWNLOAD) increment = INT_MAX; else - increment = schedule[schedule_len-1]; + increment = *(int *)smartlist_get(schedule, smartlist_len(schedule) - 1); if (increment < INT_MAX) dls->next_attempt_at = now+increment; @@ -3838,14 +3440,11 @@ download_status_increment_failure(download_status_t *dls, int status_code, void download_status_reset(download_status_t *dls) { - const int *schedule; - size_t schedule_len; - - find_dl_schedule_and_len(dls, get_options()->DirPort_set, - &schedule, &schedule_len); + const smartlist_t *schedule = find_dl_schedule_and_len( + dls, get_options()->DirPort_set); dls->n_download_failures = 0; - dls->next_attempt_at = time(NULL) + schedule[0]; + dls->next_attempt_at = time(NULL) + *(int *)smartlist_get(schedule, 0); } /** Return the number of failures on <b>dls</b> since the last success (if @@ -3890,7 +3489,8 @@ dir_routerdesc_download_failed(smartlist_t *failed, int status_code, } else { dls = router_get_dl_status_by_descriptor_digest(digest); } - if (!dls || dls->n_download_failures >= MAX_ROUTERDESC_DOWNLOAD_FAILURES) + if (!dls || dls->n_download_failures >= + get_options()->TestingDescriptorMaxDownloadTries) continue; download_status_increment_failure(dls, status_code, cp, server, now); } SMARTLIST_FOREACH_END(cp); @@ -3921,7 +3521,8 @@ dir_microdesc_download_failed(smartlist_t *failed, if (!rs) continue; dls = &rs->dl_status; - if (dls->n_download_failures >= MAX_MICRODESC_DOWNLOAD_FAILURES) + if (dls->n_download_failures >= + get_options()->TestingMicrodescMaxDownloadTries) continue; { char buf[BASE64_DIGEST256_LEN+1]; diff --git a/src/or/directory.h b/src/or/directory.h index 41f18a1725..bc200797d4 100644 --- a/src/or/directory.h +++ b/src/or/directory.h @@ -30,7 +30,7 @@ typedef enum { DIRIND_ONEHOP=0, /** Connect over a multi-hop anonymizing Tor circuit */ DIRIND_ANONYMOUS=1, - /** Conncet to the DirPort directly */ + /** Connect to the DirPort directly */ DIRIND_DIRECT_CONN, /** Connect over a multi-hop anonymizing Tor circuit to our dirport */ DIRIND_ANON_DIRPORT, @@ -63,7 +63,7 @@ int connection_dir_process_inbuf(dir_connection_t *conn); int connection_dir_finished_flushing(dir_connection_t *conn); int connection_dir_finished_connecting(dir_connection_t *conn); void connection_dir_about_to_close(dir_connection_t *dir_conn); -void directory_initiate_command(const char *address, const tor_addr_t *addr, +void directory_initiate_command(const tor_addr_t *addr, uint16_t or_port, uint16_t dir_port, const char *digest, uint8_t dir_purpose, uint8_t router_purpose, @@ -118,5 +118,10 @@ download_status_mark_impossible(download_status_t *dl) int download_status_get_n_failures(const download_status_t *dls); +#ifdef TOR_UNIT_TESTS +/* Used only by directory.c and test_dir.c */ +STATIC int parse_http_url(const char *headers, char **url); +#endif + #endif diff --git a/src/or/dirserv.c b/src/or/dirserv.c index 3e46153a55..49fafafab2 100644 --- a/src/or/dirserv.c +++ b/src/or/dirserv.c @@ -26,6 +26,7 @@ #include "router.h" #include "routerlist.h" #include "routerparse.h" +#include "routerset.h" /** * \file dirserv.c @@ -41,31 +42,10 @@ * directory authorities. */ #define MAX_UNTRUSTED_NETWORKSTATUSES 16 -/** If a v1 directory is older than this, discard it. */ -#define MAX_V1_DIRECTORY_AGE (30*24*60*60) -/** If a v1 running-routers is older than this, discard it. */ -#define MAX_V1_RR_AGE (7*24*60*60) - extern time_t time_of_process_start; /* from main.c */ extern long stats_n_seconds_working; /* from main.c */ -/** Do we need to regenerate the v1 directory when someone asks for it? */ -static time_t the_directory_is_dirty = 1; -/** Do we need to regenerate the v1 runningrouters document when somebody - * asks for it? */ -static time_t runningrouters_is_dirty = 1; -/** Do we need to regenerate our v2 networkstatus document when somebody asks - * for it? */ -static time_t the_v2_networkstatus_is_dirty = 1; - -/** Most recently generated encoded signed v1 directory. (v1 auth dirservers - * only.) */ -static cached_dir_t *the_directory = NULL; - -/** For authoritative directories: the current (v1) network status. */ -static cached_dir_t the_runningrouters; - /** Total number of routers with measured bandwidth; this is set by * dirserv_count_measured_bws() before the loop in * dirserv_generate_networkstatus_vote_obj() and checked by @@ -74,14 +54,12 @@ static cached_dir_t the_runningrouters; static int routers_with_measured_bw = 0; static void directory_remove_invalid(void); -static cached_dir_t *dirserv_regenerate_directory(void); static char *format_versions_list(config_line_t *ln); struct authdir_config_t; static int add_fingerprint_to_dir(const char *nickname, const char *fp, struct authdir_config_t *list); static uint32_t dirserv_get_status_impl(const char *fp, const char *nickname, - const char *address, uint32_t addr, uint16_t or_port, const char *platform, const char *contact, const char **msg, int should_log); @@ -329,7 +307,6 @@ dirserv_router_get_status(const routerinfo_t *router, const char **msg) } return dirserv_get_status_impl(d, router->nickname, - router->address, router->addr, router->or_port, router->platform, router->contact_info, msg, 1); @@ -343,7 +320,6 @@ dirserv_would_reject_router(const routerstatus_t *rs) uint32_t res; res = dirserv_get_status_impl(rs->identity_digest, rs->nickname, - "", /* address is only used in logs */ rs->addr, rs->or_port, NULL, NULL, NULL, 0); @@ -382,7 +358,6 @@ dirserv_get_name_status(const char *id_digest, const char *nickname) */ static uint32_t dirserv_get_status_impl(const char *id_digest, const char *nickname, - const char *address, uint32_t addr, uint16_t or_port, const char *platform, const char *contact, const char **msg, int should_log) @@ -399,13 +374,15 @@ dirserv_get_status_impl(const char *id_digest, const char *nickname, strmap_size(fingerprint_list->fp_by_name), digestmap_size(fingerprint_list->status_by_digest)); - /* Versions before Tor 0.2.2.35 have known security issues that - * make them unsuitable for the current network. */ - if (platform && !tor_version_as_new_as(platform,"0.2.2.35")) { + /* Versions before Tor 0.2.3.16-alpha are too old to support, and are + * missing some important security fixes too. Disable them. */ + if (platform && !tor_version_as_new_as(platform,"0.2.3.16-alpha")) { if (msg) *msg = "Tor version is insecure or unsupported. Please upgrade!"; return FP_REJECT; - } else if (platform && tor_version_as_new_as(platform,"0.2.3.0-alpha")) { + } +#if 0 + else if (platform && tor_version_as_new_as(platform,"0.2.3.0-alpha")) { /* Versions from 0.2.3-alpha...0.2.3.9-alpha have known security * issues that make them unusable for the current network */ if (!tor_version_as_new_as(platform, "0.2.3.10-alpha")) { @@ -414,6 +391,7 @@ dirserv_get_status_impl(const char *id_digest, const char *nickname, return FP_REJECT; } } +#endif result = dirserv_get_name_status(id_digest, nickname); if (result & FP_NAMED) { @@ -454,14 +432,14 @@ dirserv_get_status_impl(const char *id_digest, const char *nickname, if (should_log) log_info(LD_DIRSERV, "Marking '%s' as bad directory because of address '%s'", - nickname, address); + nickname, fmt_addr32(addr)); result |= FP_BADDIR; } if (authdir_policy_badexit_address(addr, or_port)) { if (should_log) log_info(LD_DIRSERV, "Marking '%s' as bad exit because of address '%s'", - nickname, address); + nickname, fmt_addr32(addr)); result |= FP_BADEXIT; } @@ -469,7 +447,7 @@ dirserv_get_status_impl(const char *id_digest, const char *nickname, if (!authdir_policy_permits_address(addr, or_port)) { if (should_log) log_info(LD_DIRSERV, "Rejecting '%s' because of address '%s'", - nickname, address); + nickname, fmt_addr32(addr)); if (msg) *msg = "Authdir is rejecting routers in this range."; return FP_REJECT; @@ -477,7 +455,7 @@ dirserv_get_status_impl(const char *id_digest, const char *nickname, if (!authdir_policy_valid_address(addr, or_port)) { if (should_log) log_info(LD_DIRSERV, "Not marking '%s' valid because of address '%s'", - nickname, address); + nickname, fmt_addr32(addr)); result |= FP_INVALID; } if (reject_unlisted) { @@ -526,19 +504,15 @@ dirserv_free_fingerprint_list(void) static int dirserv_router_has_valid_address(routerinfo_t *ri) { - struct in_addr iaddr; + tor_addr_t addr; if (get_options()->DirAllowPrivateAddresses) return 0; /* whatever it is, we're fine with it */ - if (!tor_inet_aton(ri->address, &iaddr)) { - log_info(LD_DIRSERV,"Router %s published non-IP address '%s'. Refusing.", - router_describe(ri), - ri->address); - return -1; - } - if (is_internal_IP(ntohl(iaddr.s_addr), 0)) { + tor_addr_from_ipv4h(&addr, ri->addr); + + if (tor_addr_is_internal(&addr, 0)) { log_info(LD_DIRSERV, - "Router %s published internal IP address '%s'. Refusing.", - router_describe(ri), ri->address); + "Router %s published internal IP address. Refusing.", + router_describe(ri)); return -1; /* it's a private IP, we should reject it */ } return 0; @@ -590,12 +564,10 @@ authdir_wants_to_reject_router(routerinfo_t *ri, const char **msg, } if (dirserv_router_has_valid_address(ri) < 0) { log_fn(severity, LD_DIRSERV, - "Router %s has invalid address '%s'. " - "Not adding (%s).", + "Router %s has invalid address. Not adding (%s).", router_describe(ri), - ri->address, esc_router_info(ri)); - *msg = "Rejected: Address is not an IP, or IP is a private address."; + *msg = "Rejected: Address is a private address."; return -1; } @@ -839,7 +811,6 @@ dirserv_add_extrainfo(extrainfo_t *ei, const char **msg) static void directory_remove_invalid(void) { - int changed = 0; routerlist_t *rl = router_get_routerlist(); smartlist_t *nodes = smartlist_new(); smartlist_add_all(nodes, nodelist_get_list()); @@ -857,7 +828,6 @@ directory_remove_invalid(void) log_info(LD_DIRSERV, "Router %s is now rejected: %s", description, msg?msg:""); routerlist_remove(rl, ent, 0, time(NULL)); - changed = 1; continue; } #if 0 @@ -866,72 +836,35 @@ directory_remove_invalid(void) "Router %s is now %snamed.", description, (r&FP_NAMED)?"":"un"); ent->is_named = (r&FP_NAMED)?1:0; - changed = 1; } if (bool_neq((r & FP_UNNAMED), ent->auth_says_is_unnamed)) { log_info(LD_DIRSERV, "Router '%s' is now %snamed. (FP_UNNAMED)", description, (r&FP_NAMED)?"":"un"); ent->is_named = (r&FP_NUNAMED)?0:1; - changed = 1; } #endif if (bool_neq((r & FP_INVALID), !node->is_valid)) { log_info(LD_DIRSERV, "Router '%s' is now %svalid.", description, (r&FP_INVALID) ? "in" : ""); node->is_valid = (r&FP_INVALID)?0:1; - changed = 1; } if (bool_neq((r & FP_BADDIR), node->is_bad_directory)) { log_info(LD_DIRSERV, "Router '%s' is now a %s directory", description, (r & FP_BADDIR) ? "bad" : "good"); node->is_bad_directory = (r&FP_BADDIR) ? 1: 0; - changed = 1; } if (bool_neq((r & FP_BADEXIT), node->is_bad_exit)) { log_info(LD_DIRSERV, "Router '%s' is now a %s exit", description, (r & FP_BADEXIT) ? "bad" : "good"); node->is_bad_exit = (r&FP_BADEXIT) ? 1: 0; - changed = 1; } } SMARTLIST_FOREACH_END(node); - if (changed) - directory_set_dirty(); routerlist_assert_ok(rl); smartlist_free(nodes); } -/** Mark the directory as <b>dirty</b> -- when we're next asked for a - * directory, we will rebuild it instead of reusing the most recently - * generated one. - */ -void -directory_set_dirty(void) -{ - time_t now = time(NULL); - int set_v1_dirty=0; - - /* Regenerate stubs only every 8 hours. - * XXXX It would be nice to generate less often, but these are just - * stubs: it doesn't matter. */ -#define STUB_REGENERATE_INTERVAL (8*60*60) - if (!the_directory || !the_runningrouters.dir) - set_v1_dirty = 1; - else if (the_directory->published < now - STUB_REGENERATE_INTERVAL || - the_runningrouters.published < now - STUB_REGENERATE_INTERVAL) - set_v1_dirty = 1; - - if (set_v1_dirty) { - if (!the_directory_is_dirty) - the_directory_is_dirty = now; - if (!runningrouters_is_dirty) - runningrouters_is_dirty = now; - } - if (!the_v2_networkstatus_is_dirty) - the_v2_networkstatus_is_dirty = now; -} - /** * Allocate and return a description of the status of the server <b>desc</b>, * for use in a v1-style router-status line. The server is listed @@ -1271,14 +1204,6 @@ directory_fetches_dir_info_later(const or_options_t *options) return options->UseBridges != 0; } -/** Return 1 if we want to cache v2 dir info (each status file). - */ -int -directory_caches_v2_dir_info(const or_options_t *options) -{ - return options->DirPort_set; -} - /** Return true iff we want to fetch and keep certificates for authorities * that we don't acknowledge as aurthorities ourself. */ @@ -1313,15 +1238,6 @@ directory_permits_begindir_requests(const or_options_t *options) return options->BridgeRelay != 0 || options->DirPort_set; } -/** Return 1 if we want to allow controllers to ask us directory - * requests via the controller interface, which doesn't require - * having any separate port open. */ -int -directory_permits_controller_requests(const or_options_t *options) -{ - return options->DirPort_set; -} - /** Return 1 if we have no need to fetch new descriptors. This generally * happens when we're not a dir cache and we haven't built any circuits * lately. @@ -1337,55 +1253,10 @@ directory_too_idle_to_fetch_descriptors(const or_options_t *options, /********************************************************************/ -/* Used only by non-v1-auth dirservers: The v1 directory and - * runningrouters we'll serve when requested. */ - -/** The v1 directory we'll serve (as a cache or as an authority) if - * requested. */ -static cached_dir_t *cached_directory = NULL; -/** The v1 runningrouters document we'll serve (as a cache or as an authority) - * if requested. */ -static cached_dir_t cached_runningrouters; - -/** Used for other dirservers' v2 network statuses. Map from hexdigest to - * cached_dir_t. */ -static digestmap_t *cached_v2_networkstatus = NULL; - /** Map from flavor name to the cached_dir_t for the v3 consensuses that we're * currently serving. */ static strmap_t *cached_consensuses = NULL; -/** Possibly replace the contents of <b>d</b> with the value of - * <b>directory</b> published on <b>when</b>, unless <b>when</b> is older than - * the last value, or too far in the future. - * - * Does not copy <b>directory</b>; frees it if it isn't used. - */ -static void -set_cached_dir(cached_dir_t *d, char *directory, time_t when) -{ - time_t now = time(NULL); - if (when<=d->published) { - log_info(LD_DIRSERV, "Ignoring old directory; not caching."); - tor_free(directory); - } else if (when>=now+ROUTER_MAX_AGE_TO_PUBLISH) { - log_info(LD_DIRSERV, "Ignoring future directory; not caching."); - tor_free(directory); - } else { - /* if (when>d->published && when<now+ROUTER_MAX_AGE) */ - log_debug(LD_DIRSERV, "Caching directory."); - tor_free(d->dir); - d->dir = directory; - d->dir_len = strlen(directory); - tor_free(d->dir_z); - if (tor_gzip_compress(&(d->dir_z), &(d->dir_z_len), d->dir, d->dir_len, - ZLIB_METHOD)) { - log_warn(LD_BUG,"Error compressing cached directory"); - } - d->published = when; - } -} - /** Decrement the reference count on <b>d</b>, and free it if it no longer has * any references. */ void @@ -1435,86 +1306,6 @@ free_cached_dir_(void *_d) cached_dir_decref(d); } -/** If we have no cached v1 directory, or it is older than <b>published</b>, - * then replace it with <b>directory</b>, published at <b>published</b>. - * - * If <b>published</b> is too old, do nothing. - * - * If <b>is_running_routers</b>, this is really a v1 running_routers - * document rather than a v1 directory. - */ -static void -dirserv_set_cached_directory(const char *directory, time_t published) -{ - - cached_dir_decref(cached_directory); - cached_directory = new_cached_dir(tor_strdup(directory), published); -} - -/** If <b>networkstatus</b> is non-NULL, we've just received a v2 - * network-status for an authoritative directory with identity digest - * <b>identity</b> published at <b>published</b> -- store it so we can - * serve it to others. - * - * If <b>networkstatus</b> is NULL, remove the entry with the given - * identity fingerprint from the v2 cache. - */ -void -dirserv_set_cached_networkstatus_v2(const char *networkstatus, - const char *identity, - time_t published) -{ - cached_dir_t *d, *old_d; - if (!cached_v2_networkstatus) - cached_v2_networkstatus = digestmap_new(); - - old_d = digestmap_get(cached_v2_networkstatus, identity); - if (!old_d && !networkstatus) - return; - - if (networkstatus) { - if (!old_d || published > old_d->published) { - d = new_cached_dir(tor_strdup(networkstatus), published); - digestmap_set(cached_v2_networkstatus, identity, d); - if (old_d) - cached_dir_decref(old_d); - } - } else { - if (old_d) { - digestmap_remove(cached_v2_networkstatus, identity); - cached_dir_decref(old_d); - } - } - - /* Now purge old entries. */ - - if (digestmap_size(cached_v2_networkstatus) > - get_n_authorities(V2_DIRINFO) + MAX_UNTRUSTED_NETWORKSTATUSES) { - /* We need to remove the oldest untrusted networkstatus. */ - const char *oldest = NULL; - time_t oldest_published = TIME_MAX; - digestmap_iter_t *iter; - - for (iter = digestmap_iter_init(cached_v2_networkstatus); - !digestmap_iter_done(iter); - iter = digestmap_iter_next(cached_v2_networkstatus, iter)) { - const char *ident; - void *val; - digestmap_iter_get(iter, &ident, &val); - d = val; - if (d->published < oldest_published && - !router_digest_is_trusted_dir(ident)) { - oldest = ident; - oldest_published = d->published; - } - } - tor_assert(oldest); - d = digestmap_remove(cached_v2_networkstatus, oldest); - if (d) - cached_dir_decref(d); - } -} - /** Replace the v3 consensus networkstatus of type <b>flavor_name</b> that * we're serving with <b>networkstatus</b>, published at <b>published</b>. No * validation is performed. */ @@ -1537,186 +1328,6 @@ dirserv_set_cached_consensus_networkstatus(const char *networkstatus, cached_dir_decref(old_networkstatus); } -/** Remove any v2 networkstatus from the directory cache that was published - * before <b>cutoff</b>. */ -void -dirserv_clear_old_networkstatuses(time_t cutoff) -{ - if (!cached_v2_networkstatus) - return; - - DIGESTMAP_FOREACH_MODIFY(cached_v2_networkstatus, id, cached_dir_t *, dir) { - if (dir->published < cutoff) { - char *fname; - fname = networkstatus_get_cache_filename(id); - if (file_status(fname) == FN_FILE) { - log_info(LD_DIR, "Removing too-old untrusted networkstatus in %s", - fname); - unlink(fname); - } - tor_free(fname); - cached_dir_decref(dir); - MAP_DEL_CURRENT(id); - } - } DIGESTMAP_FOREACH_END -} - -/** Remove any v1 info from the directory cache that was published - * too long ago. */ -void -dirserv_clear_old_v1_info(time_t now) -{ - if (cached_directory && - cached_directory->published < (now - MAX_V1_DIRECTORY_AGE)) { - cached_dir_decref(cached_directory); - cached_directory = NULL; - } - if (cached_runningrouters.published < (now - MAX_V1_RR_AGE)) { - clear_cached_dir(&cached_runningrouters); - } -} - -/** Helper: If we're an authority for the right directory version (v1 or v2) - * (based on <b>auth_type</b>), try to regenerate - * auth_src as appropriate and return it, falling back to cache_src on - * failure. If we're a cache, simply return cache_src. - */ -static cached_dir_t * -dirserv_pick_cached_dir_obj(cached_dir_t *cache_src, - cached_dir_t *auth_src, - time_t dirty, cached_dir_t *(*regenerate)(void), - const char *name, - dirinfo_type_t auth_type) -{ - const or_options_t *options = get_options(); - int authority = (auth_type == V1_DIRINFO && authdir_mode_v1(options)) || - (auth_type == V2_DIRINFO && authdir_mode_v2(options)); - - if (!authority || authdir_mode_bridge(options)) { - return cache_src; - } else { - /* We're authoritative. */ - if (regenerate != NULL) { - if (dirty && dirty + DIR_REGEN_SLACK_TIME < time(NULL)) { - if (!(auth_src = regenerate())) { - log_err(LD_BUG, "Couldn't generate %s?", name); - exit(1); - } - } else { - log_info(LD_DIRSERV, "The %s is still clean; reusing.", name); - } - } - return auth_src ? auth_src : cache_src; - } -} - -/** Return the most recently generated encoded signed v1 directory, - * generating a new one as necessary. If not a v1 authoritative directory - * may return NULL if no directory is yet cached. */ -cached_dir_t * -dirserv_get_directory(void) -{ - return dirserv_pick_cached_dir_obj(cached_directory, the_directory, - the_directory_is_dirty, - dirserv_regenerate_directory, - "v1 server directory", V1_DIRINFO); -} - -/** Only called by v1 auth dirservers. - * Generate a fresh v1 directory; set the_directory and return a pointer - * to the new value. - */ -static cached_dir_t * -dirserv_regenerate_directory(void) -{ - /* XXXX 024 Get rid of this function if we can confirm that nobody's - * fetching these any longer */ - char *new_directory=NULL; - - if (dirserv_dump_directory_to_string(&new_directory, - get_server_identity_key())) { - log_warn(LD_BUG, "Error creating directory."); - tor_free(new_directory); - return NULL; - } - cached_dir_decref(the_directory); - the_directory = new_cached_dir(new_directory, time(NULL)); - log_info(LD_DIRSERV,"New directory (size %d) has been built.", - (int)the_directory->dir_len); - log_debug(LD_DIRSERV,"New directory (size %d):\n%s", - (int)the_directory->dir_len, the_directory->dir); - - the_directory_is_dirty = 0; - - /* Save the directory to disk so we re-load it quickly on startup. - */ - dirserv_set_cached_directory(the_directory->dir, time(NULL)); - - return the_directory; -} - -/** Only called by v1 auth dirservers. - * Replace the current running-routers list with a newly generated one. */ -static cached_dir_t * -generate_runningrouters(void) -{ - char *s=NULL; - char digest[DIGEST_LEN]; - char published[ISO_TIME_LEN+1]; - size_t len; - crypto_pk_t *private_key = get_server_identity_key(); - char *identity_pkey; /* Identity key, DER64-encoded. */ - size_t identity_pkey_len; - - if (crypto_pk_write_public_key_to_string(private_key,&identity_pkey, - &identity_pkey_len)<0) { - log_warn(LD_BUG,"write identity_pkey to string failed!"); - goto err; - } - format_iso_time(published, time(NULL)); - - len = 2048; - s = tor_malloc_zero(len); - tor_snprintf(s, len, - "network-status\n" - "published %s\n" - "router-status %s\n" - "dir-signing-key\n%s" - "directory-signature %s\n", - published, "", identity_pkey, - get_options()->Nickname); - tor_free(identity_pkey); - if (router_get_runningrouters_hash(s,digest)) { - log_warn(LD_BUG,"couldn't compute digest"); - goto err; - } - note_crypto_pk_op(SIGN_DIR); - if (router_append_dirobj_signature(s, len, digest, DIGEST_LEN, - private_key)<0) - goto err; - - set_cached_dir(&the_runningrouters, s, time(NULL)); - runningrouters_is_dirty = 0; - - return &the_runningrouters; - err: - tor_free(s); - return NULL; -} - -/** Set *<b>rr</b> to the most recently generated encoded signed - * running-routers list, generating a new one as necessary. Return the - * size of the directory on success, and 0 on failure. */ -cached_dir_t * -dirserv_get_runningrouters(void) -{ - return dirserv_pick_cached_dir_obj( - &cached_runningrouters, &the_runningrouters, - runningrouters_is_dirty, - generate_runningrouters, - "v1 network status list", V1_DIRINFO); -} - /** Return the latest downloaded consensus networkstatus in encoded, signed, * optionally compressed format, suitable for sending to clients. */ cached_dir_t * @@ -1727,19 +1338,6 @@ dirserv_get_consensus(const char *flavor_name) return strmap_get(cached_consensuses, flavor_name); } -/** For authoritative directories: the current (v2) network status. */ -static cached_dir_t *the_v2_networkstatus = NULL; - -/** Return true iff our opinion of the routers has been stale for long - * enough that we should generate a new v2 network status doc. */ -static int -should_generate_v2_networkstatus(void) -{ - return authdir_mode_v2(get_options()) && - the_v2_networkstatus_is_dirty && - the_v2_networkstatus_is_dirty + DIR_REGEN_SLACK_TIME < time(NULL); -} - /** If a router's uptime is at least this value, then it is always * considered stable, regardless of the rest of the network. This * way we resist attacks where an attacker doubles the size of the @@ -1907,7 +1505,7 @@ router_counts_toward_thresholds(const node_t *node, time_t now, * the Weighted Fractional Uptime history, and use them to set thresholds for * the Stable, Fast, and Guard flags. Update the fields stable_uptime, * stable_mtbf, enough_mtbf_info, guard_wfu, guard_tk, fast_bandwidth, - * guard_bandwidh_including_exits, guard_bandwidth_excluding_exits, + * guard_bandwidth_including_exits, and guard_bandwidth_excluding_exits. * * Also, set the is_exit flag of each router appropriately. */ static void @@ -1956,6 +1554,10 @@ dirserv_compute_performance_thresholds(routerlist_t *rl, /* Now, fill in the arrays. */ SMARTLIST_FOREACH_BEGIN(nodelist_get_list(), node_t *, node) { + if (options->BridgeAuthoritativeDir && + node->ri && + node->ri->purpose != ROUTER_PURPOSE_BRIDGE) + continue; if (router_counts_toward_thresholds(node, now, omit_as_sybil, require_mbw)) { routerinfo_t *ri = node->ri; @@ -1986,7 +1588,7 @@ dirserv_compute_performance_thresholds(routerlist_t *rl, /* (Now bandwidths is sorted.) */ if (fast_bandwidth_kb < ROUTER_REQUIRED_MIN_BANDWIDTH/(2 * 1000)) fast_bandwidth_kb = bandwidths_kb[n_active/4]; - guard_bandwidth_including_exits_kb = bandwidths_kb[(n_active-1)/2]; + guard_bandwidth_including_exits_kb = bandwidths_kb[n_active*3/4]; guard_tk = find_nth_long(tks, n_active, n_active/8); } @@ -2044,7 +1646,8 @@ dirserv_compute_performance_thresholds(routerlist_t *rl, if (n_active_nonexit) { guard_bandwidth_excluding_exits_kb = - median_uint32(bandwidths_excluding_exits_kb, n_active_nonexit); + find_nth_uint32(bandwidths_excluding_exits_kb, + n_active_nonexit, n_active_nonexit*3/4); } log_info(LD_DIRSERV, @@ -2070,6 +1673,21 @@ dirserv_compute_performance_thresholds(routerlist_t *rl, tor_free(wfus); } +/* Use dirserv_compute_performance_thresholds() to compute the thresholds + * for the status flags, specifically for bridges. + * + * This is only called by a Bridge Authority from + * networkstatus_getinfo_by_purpose(). + */ +void +dirserv_compute_bridge_flag_thresholds(routerlist_t *rl) +{ + + digestmap_t *omit_as_sybil = digestmap_new(); + dirserv_compute_performance_thresholds(rl, omit_as_sybil); + digestmap_free(omit_as_sybil, NULL); +} + /** Measured bandwidth cache entry */ typedef struct mbw_cache_entry_s { long mbw_kb; @@ -2082,7 +1700,7 @@ static digestmap_t *mbw_cache = NULL; /** Store a measured bandwidth cache entry when reading the measured * bandwidths file. */ -void +STATIC void dirserv_cache_measured_bw(const measured_bw_line_t *parsed_line, time_t as_of) { @@ -2112,7 +1730,7 @@ dirserv_cache_measured_bw(const measured_bw_line_t *parsed_line, } /** Clear and free the measured bandwidth cache */ -void +STATIC void dirserv_clear_measured_bw_cache(void) { if (mbw_cache) { @@ -2123,7 +1741,7 @@ dirserv_clear_measured_bw_cache(void) } /** Scan the measured bandwidth cache and remove expired entries */ -void +STATIC void dirserv_expire_measured_bw_cache(time_t now) { @@ -2145,7 +1763,7 @@ dirserv_expire_measured_bw_cache(time_t now) } /** Get the current size of the measured bandwidth cache */ -int +STATIC int dirserv_get_measured_bw_cache_size(void) { if (mbw_cache) return digestmap_size(mbw_cache); @@ -2155,7 +1773,7 @@ dirserv_get_measured_bw_cache_size(void) /** Query the cache by identity digest, return value indicates whether * we found it. The bw_out and as_of_out pointers receive the cached * bandwidth value and the time it was cached if not NULL. */ -int +STATIC int dirserv_query_measured_bw_cache_kb(const char *node_id, long *bw_kb_out, time_t *as_of_out) { @@ -2176,7 +1794,7 @@ dirserv_query_measured_bw_cache_kb(const char *node_id, long *bw_kb_out, } /** Predicate wrapper for dirserv_query_measured_bw_cache() */ -int +STATIC int dirserv_has_measured_bw(const char *node_id) { return dirserv_query_measured_bw_cache_kb(node_id, NULL, NULL); @@ -2391,7 +2009,7 @@ routerstatus_format_entry(const routerstatus_t *rs, const char *version, rs->is_flagged_running?" Running":"", rs->is_stable?" Stable":"", rs->is_unnamed?" Unnamed":"", - rs->is_v2_dir?" V2Dir":"", + (rs->dir_port!=0)?" V2Dir":"", rs->is_valid?" Valid":""); /* length of "opt v \n" */ @@ -2705,12 +2323,16 @@ set_routerstatus_from_routerinfo(routerstatus_t *rs, } else { rs->is_possible_guard = 0; } + if (options->TestingTorNetwork && + routerset_contains_routerstatus(options->TestingDirAuthVoteGuard, + rs, 0)) { + rs->is_possible_guard = 1; + } rs->is_bad_directory = listbaddirs && node->is_bad_directory; rs->is_bad_exit = listbadexits && node->is_bad_exit; node->is_hs_dir = dirserv_thinks_router_is_hs_dir(ri, node, now); rs->is_hs_dir = vote_on_hsdirs && node->is_hs_dir; - rs->is_v2_dir = ri->dir_port != 0; if (!strcasecmp(ri->nickname, UNNAMED_ROUTER_NICKNAME)) rs->is_named = rs->is_unnamed = 0; @@ -2741,7 +2363,7 @@ static void clear_status_flags_on_sybil(routerstatus_t *rs) { rs->is_authority = rs->is_exit = rs->is_stable = rs->is_fast = - rs->is_flagged_running = rs->is_named = rs->is_valid = rs->is_v2_dir = + rs->is_flagged_running = rs->is_named = rs->is_valid = rs->is_hs_dir = rs->is_possible_guard = rs->is_bad_exit = rs->is_bad_directory = 0; /* FFFF we might want some mechanism to check later on if we @@ -2754,7 +2376,7 @@ clear_status_flags_on_sybil(routerstatus_t *rs) * into a measured_bw_line_t output structure. Returns -1 on failure * or 0 on success. */ -int +STATIC int measured_bw_line_parse(measured_bw_line_t *out, const char *orig_line) { char *line = tor_strdup(orig_line); @@ -2835,7 +2457,7 @@ measured_bw_line_parse(measured_bw_line_t *out, const char *orig_line) * of bandwidth statuses. Returns true if a line is found, * false otherwise. */ -int +STATIC int measured_bw_line_apply(measured_bw_line_t *parsed_line, smartlist_t *routerstatuses) { @@ -2886,7 +2508,7 @@ dirserv_read_measured_bandwidths(const char *from_file, } line[strlen(line)-1] = '\0'; - file_time = tor_parse_ulong(line, 10, 0, ULONG_MAX, &ok, NULL); + file_time = (time_t)tor_parse_ulong(line, 10, 0, ULONG_MAX, &ok, NULL); if (!ok) { log_warn(LD_DIRSERV, "Non-integer time in bandwidth file: %s", escaped(line)); @@ -2957,14 +2579,6 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key, tor_assert(private_key); tor_assert(cert); - if (resolve_my_address(LOG_WARN, options, &addr, NULL, &hostname)<0) { - log_warn(LD_NET, "Couldn't resolve my hostname"); - return NULL; - } - if (!hostname || !strchr(hostname, '.')) { - tor_free(hostname); - hostname = tor_dup_ip(addr); - } if (crypto_pk_get_digest(private_key, signing_key_digest)<0) { log_err(LD_BUG, "Error computing signing key digest"); return NULL; @@ -2973,6 +2587,14 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key, log_err(LD_BUG, "Error computing identity key digest"); return NULL; } + if (resolve_my_address(LOG_WARN, options, &addr, NULL, &hostname)<0) { + log_warn(LD_NET, "Couldn't resolve my hostname"); + return NULL; + } + if (!hostname || !strchr(hostname, '.')) { + tor_free(hostname); + hostname = tor_dup_ip(addr); + } if (options->VersioningAuthoritativeDir) { client_versions = format_versions_list(options->RecommendedClientVersions); @@ -3093,7 +2715,8 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key, else last_consensus_interval = options->TestingV3AuthInitialVotingInterval; v3_out->valid_after = - dirvote_get_start_of_next_interval(now, (int)last_consensus_interval); + dirvote_get_start_of_next_interval(now, (int)last_consensus_interval, + options->TestingV3AuthVotingStartOffset); format_iso_time(tbuf, v3_out->valid_after); log_notice(LD_DIR,"Choosing valid-after time in vote as %s: " "consensus_set=%d, last_interval=%d", @@ -3164,270 +2787,6 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key, return v3_out; } -/** For v2 authoritative directories only: Replace the contents of - * <b>the_v2_networkstatus</b> with a newly generated network status - * object. */ -cached_dir_t * -generate_v2_networkstatus_opinion(void) -{ - cached_dir_t *r = NULL; - size_t identity_pkey_len; - char *status = NULL, *client_versions = NULL, *server_versions = NULL, - *identity_pkey = NULL, *hostname = NULL; - const or_options_t *options = get_options(); - char fingerprint[FINGERPRINT_LEN+1]; - char published[ISO_TIME_LEN+1]; - char digest[DIGEST_LEN]; - uint32_t addr; - crypto_pk_t *private_key; - routerlist_t *rl = router_get_routerlist(); - time_t now = time(NULL); - time_t cutoff = now - ROUTER_MAX_AGE_TO_PUBLISH; - int naming = options->NamingAuthoritativeDir; - int versioning = options->VersioningAuthoritativeDir; - int listbaddirs = options->AuthDirListBadDirs; - int listbadexits = options->AuthDirListBadExits; - int vote_on_hsdirs = options->VoteOnHidServDirectoriesV2; - const char *contact; - char *version_lines = NULL; - smartlist_t *routers = NULL; - digestmap_t *omit_as_sybil = NULL; - smartlist_t *chunks = NULL; - - private_key = get_server_identity_key(); - - if (resolve_my_address(LOG_WARN, options, &addr, NULL, &hostname)<0) { - log_warn(LD_NET, "Couldn't resolve my hostname"); - goto done; - } - if (!hostname) - hostname = tor_dup_ip(addr); - - format_iso_time(published, now); - - client_versions = format_versions_list(options->RecommendedClientVersions); - server_versions = format_versions_list(options->RecommendedServerVersions); - - if (crypto_pk_write_public_key_to_string(private_key, &identity_pkey, - &identity_pkey_len)<0) { - log_warn(LD_BUG,"Writing public key to string failed."); - goto done; - } - - if (crypto_pk_get_fingerprint(private_key, fingerprint, 0)<0) { - log_err(LD_BUG, "Error computing fingerprint"); - goto done; - } - - contact = options->ContactInfo; - if (!contact) - contact = "(none)"; - - if (versioning) { - tor_asprintf(&version_lines, - "client-versions %s\nserver-versions %s\n", - client_versions, server_versions); - } else { - version_lines = tor_strdup(""); - } - - chunks = smartlist_new(); - smartlist_add_asprintf(chunks, - "network-status-version 2\n" - "dir-source %s %s %d\n" - "fingerprint %s\n" - "contact %s\n" - "published %s\n" - "dir-options%s%s%s%s\n" - "%s" /* client version line, server version line. */ - "dir-signing-key\n%s", - hostname, fmt_addr32(addr), - (int)router_get_advertised_dir_port(options, 0), - fingerprint, - contact, - published, - naming ? " Names" : "", - listbaddirs ? " BadDirectories" : "", - listbadexits ? " BadExits" : "", - versioning ? " Versions" : "", - version_lines, - identity_pkey); - - /* precompute this part, since we need it to decide what "stable" - * means. */ - SMARTLIST_FOREACH(rl->routers, routerinfo_t *, ri, { - dirserv_set_router_is_running(ri, now); - }); - - routers = smartlist_new(); - smartlist_add_all(routers, rl->routers); - routers_sort_by_identity(routers); - omit_as_sybil = get_possible_sybil_list(routers); - - dirserv_compute_performance_thresholds(rl, omit_as_sybil); - - SMARTLIST_FOREACH_BEGIN(routers, routerinfo_t *, ri) { - if (ri->cache_info.published_on >= cutoff) { - routerstatus_t rs; - char *version = version_from_platform(ri->platform); - node_t *node = node_get_mutable_by_id(ri->cache_info.identity_digest); - if (!node) { - tor_free(version); - continue; - } - set_routerstatus_from_routerinfo(&rs, node, ri, now, - naming, listbadexits, listbaddirs, - vote_on_hsdirs); - - if (digestmap_get(omit_as_sybil, ri->cache_info.identity_digest)) - clear_status_flags_on_sybil(&rs); - - { - char *rsf = routerstatus_format_entry(&rs, version, NS_V2, NULL); - if (rsf) - smartlist_add(chunks, rsf); - } - tor_free(version); - } - } SMARTLIST_FOREACH_END(ri); - - smartlist_add_asprintf(chunks, "directory-signature %s\n", - options->Nickname); - - crypto_digest_smartlist(digest, DIGEST_LEN, chunks, "", DIGEST_SHA1); - - note_crypto_pk_op(SIGN_DIR); - { - char *sig; - if (!(sig = router_get_dirobj_signature(digest,DIGEST_LEN, - private_key))) { - log_warn(LD_BUG, "Unable to sign router status."); - goto done; - } - smartlist_add(chunks, sig); - } - - status = smartlist_join_strings(chunks, "", 0, NULL); - - { - networkstatus_v2_t *ns; - if (!(ns = networkstatus_v2_parse_from_string(status))) { - log_err(LD_BUG,"Generated a networkstatus we couldn't parse."); - goto done; - } - networkstatus_v2_free(ns); - } - - { - cached_dir_t **ns_ptr = &the_v2_networkstatus; - if (*ns_ptr) - cached_dir_decref(*ns_ptr); - *ns_ptr = new_cached_dir(status, now); - status = NULL; /* So it doesn't get double-freed. */ - the_v2_networkstatus_is_dirty = 0; - router_set_networkstatus_v2((*ns_ptr)->dir, now, NS_GENERATED, NULL); - r = *ns_ptr; - } - - done: - if (chunks) { - SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp)); - smartlist_free(chunks); - } - tor_free(client_versions); - tor_free(server_versions); - tor_free(version_lines); - tor_free(status); - tor_free(hostname); - tor_free(identity_pkey); - smartlist_free(routers); - digestmap_free(omit_as_sybil, NULL); - return r; -} - -/** Given the portion of a networkstatus request URL after "tor/status/" in - * <b>key</b>, append to <b>result</b> the digests of the identity keys of the - * networkstatus objects that the client has requested. */ -void -dirserv_get_networkstatus_v2_fingerprints(smartlist_t *result, - const char *key) -{ - tor_assert(result); - - if (!cached_v2_networkstatus) - cached_v2_networkstatus = digestmap_new(); - - if (should_generate_v2_networkstatus()) - generate_v2_networkstatus_opinion(); - - if (!strcmp(key,"authority")) { - if (authdir_mode_v2(get_options())) { - const routerinfo_t *me = router_get_my_routerinfo(); - if (me) - smartlist_add(result, - tor_memdup(me->cache_info.identity_digest, DIGEST_LEN)); - } - } else if (!strcmp(key, "all")) { - if (digestmap_size(cached_v2_networkstatus)) { - digestmap_iter_t *iter; - iter = digestmap_iter_init(cached_v2_networkstatus); - while (!digestmap_iter_done(iter)) { - const char *ident; - void *val; - digestmap_iter_get(iter, &ident, &val); - smartlist_add(result, tor_memdup(ident, DIGEST_LEN)); - iter = digestmap_iter_next(cached_v2_networkstatus, iter); - } - } else { - SMARTLIST_FOREACH(router_get_trusted_dir_servers(), - dir_server_t *, ds, - if (ds->type & V2_DIRINFO) - smartlist_add(result, tor_memdup(ds->digest, DIGEST_LEN))); - } - smartlist_sort_digests(result); - if (smartlist_len(result) == 0) - log_info(LD_DIRSERV, - "Client requested 'all' network status objects; we have none."); - } else if (!strcmpstart(key, "fp/")) { - dir_split_resource_into_fingerprints(key+3, result, NULL, - DSR_HEX|DSR_SORT_UNIQ); - } -} - -/** Look for a network status object as specified by <b>key</b>, which should - * be either "authority" (to find a network status generated by us), a hex - * identity digest (to find a network status generated by given directory), or - * "all" (to return all the v2 network status objects we have). - */ -void -dirserv_get_networkstatus_v2(smartlist_t *result, - const char *key) -{ - cached_dir_t *cached; - smartlist_t *fingerprints = smartlist_new(); - tor_assert(result); - - if (!cached_v2_networkstatus) - cached_v2_networkstatus = digestmap_new(); - - dirserv_get_networkstatus_v2_fingerprints(fingerprints, key); - SMARTLIST_FOREACH_BEGIN(fingerprints, const char *, fp) { - if (router_digest_is_me(fp) && should_generate_v2_networkstatus()) - generate_v2_networkstatus_opinion(); - cached = digestmap_get(cached_v2_networkstatus, fp); - if (cached) { - smartlist_add(result, cached); - } else { - char hexbuf[HEX_DIGEST_LEN+1]; - base16_encode(hexbuf, sizeof(hexbuf), fp, DIGEST_LEN); - log_info(LD_DIRSERV, "Don't know about any network status with " - "fingerprint '%s'", hexbuf); - } - } SMARTLIST_FOREACH_END(fp); - SMARTLIST_FOREACH(fingerprints, char *, cp, tor_free(cp)); - smartlist_free(fingerprints); -} - /** As dirserv_get_routerdescs(), but instead of getting signed_descriptor_t * pointers, adds copies of digests to fps_out, and doesn't use the * /tor/server/ prefix. For a /d/ request, adds descriptor digests; for other @@ -3661,7 +3020,7 @@ dirserv_single_reachability_test(time_t now, routerinfo_t *router) /* IPv4. */ log_debug(LD_OR,"Testing reachability of %s at %s:%u.", - router->nickname, router->address, router->or_port); + router->nickname, fmt_addr32(router->addr), router->or_port); tor_addr_from_ipv4h(&router_addr, router->addr); chan = channel_tls_connect(&router_addr, router->or_port, router->cache_info.identity_digest); @@ -3727,15 +3086,12 @@ static cached_dir_t * lookup_cached_dir_by_fp(const char *fp) { cached_dir_t *d = NULL; - if (tor_digest_is_zero(fp) && cached_consensuses) + if (tor_digest_is_zero(fp) && cached_consensuses) { d = strmap_get(cached_consensuses, "ns"); - else if (memchr(fp, '\0', DIGEST_LEN) && cached_consensuses && + } else if (memchr(fp, '\0', DIGEST_LEN) && cached_consensuses && (d = strmap_get(cached_consensuses, fp))) { /* this here interface is a nasty hack XXXX024 */; - } else if (router_digest_is_me(fp) && the_v2_networkstatus) - d = the_v2_networkstatus; - else if (cached_v2_networkstatus) - d = digestmap_get(cached_v2_networkstatus, fp); + } return d; } @@ -3941,8 +3297,6 @@ connection_dirserv_add_servers_to_outbuf(dir_connection_t *conn) } body = signed_descriptor_get_body(sd); if (conn->zlib_state) { - /* XXXX024 This 'last' business should actually happen on the last - * routerinfo, not on the last fingerprint. */ int last = ! smartlist_len(conn->fingerprint_stack); connection_write_to_buf_zlib(body, sd->signed_descriptor_len, conn, last); @@ -3959,6 +3313,11 @@ connection_dirserv_add_servers_to_outbuf(dir_connection_t *conn) if (!smartlist_len(conn->fingerprint_stack)) { /* We just wrote the last one; finish up. */ + if (conn->zlib_state) { + connection_write_to_buf_zlib("", 0, conn, 1); + tor_zlib_free(conn->zlib_state); + conn->zlib_state = NULL; + } conn->dir_spool_src = DIR_SPOOL_NONE; smartlist_free(conn->fingerprint_stack); conn->fingerprint_stack = NULL; @@ -3984,8 +3343,6 @@ connection_dirserv_add_microdescs_to_outbuf(dir_connection_t *conn) if (!md || !md->body) continue; if (conn->zlib_state) { - /* XXXX024 This 'last' business should actually happen on the last - * routerinfo, not on the last fingerprint. */ int last = !smartlist_len(conn->fingerprint_stack); connection_write_to_buf_zlib(md->body, md->bodylen, conn, last); if (last) { @@ -3997,6 +3354,11 @@ connection_dirserv_add_microdescs_to_outbuf(dir_connection_t *conn) } } if (!smartlist_len(conn->fingerprint_stack)) { + if (conn->zlib_state) { + connection_write_to_buf_zlib("", 0, conn, 1); + tor_zlib_free(conn->zlib_state); + conn->zlib_state = NULL; + } conn->dir_spool_src = DIR_SPOOL_NONE; smartlist_free(conn->fingerprint_stack); conn->fingerprint_stack = NULL; @@ -4128,14 +3490,6 @@ dirserv_free_all(void) { dirserv_free_fingerprint_list(); - cached_dir_decref(the_directory); - clear_cached_dir(&the_runningrouters); - cached_dir_decref(the_v2_networkstatus); - cached_dir_decref(cached_directory); - clear_cached_dir(&cached_runningrouters); - - digestmap_free(cached_v2_networkstatus, free_cached_dir_); - cached_v2_networkstatus = NULL; strmap_free(cached_consensuses, free_cached_dir_); cached_consensuses = NULL; diff --git a/src/or/dirserv.h b/src/or/dirserv.h index f9d36d760f..858e6e3a07 100644 --- a/src/or/dirserv.h +++ b/src/or/dirserv.h @@ -12,6 +12,8 @@ #ifndef TOR_DIRSERV_H #define TOR_DIRSERV_H +#include "testsupport.h" + /** What fraction (1 over this number) of the relay ID space do we * (as a directory authority) launch connections to at each reachability * test? */ @@ -49,34 +51,23 @@ int list_server_status_v1(smartlist_t *routers, char **router_status_out, int dirserv_dump_directory_to_string(char **dir_out, crypto_pk_t *private_key); char *dirserv_get_flag_thresholds_line(void); +void dirserv_compute_bridge_flag_thresholds(routerlist_t *rl); int directory_fetches_from_authorities(const or_options_t *options); int directory_fetches_dir_info_early(const or_options_t *options); int directory_fetches_dir_info_later(const or_options_t *options); -int directory_caches_v2_dir_info(const or_options_t *options); int directory_caches_unknown_auth_certs(const or_options_t *options); int directory_caches_dir_info(const or_options_t *options); int directory_permits_begindir_requests(const or_options_t *options); -int directory_permits_controller_requests(const or_options_t *options); int directory_too_idle_to_fetch_descriptors(const or_options_t *options, time_t now); -void directory_set_dirty(void); -cached_dir_t *dirserv_get_directory(void); -cached_dir_t *dirserv_get_runningrouters(void); cached_dir_t *dirserv_get_consensus(const char *flavor_name); -void dirserv_set_cached_networkstatus_v2(const char *directory, - const char *identity, - time_t published); void dirserv_set_cached_consensus_networkstatus(const char *consensus, const char *flavor_name, const digests_t *digests, time_t published); void dirserv_clear_old_networkstatuses(time_t cutoff); -void dirserv_clear_old_v1_info(time_t now); -void dirserv_get_networkstatus_v2(smartlist_t *result, const char *key); -void dirserv_get_networkstatus_v2_fingerprints(smartlist_t *result, - const char *key); int dirserv_get_routerdesc_fingerprints(smartlist_t *fps_out, const char *key, const char **msg, int for_unencrypted_conn, @@ -119,20 +110,20 @@ cached_dir_t *new_cached_dir(char *s, time_t published); /* Put the MAX_MEASUREMENT_AGE #define here so unit tests can see it */ #define MAX_MEASUREMENT_AGE (3*24*60*60) /* 3 days */ -int measured_bw_line_parse(measured_bw_line_t *out, const char *line); +STATIC int measured_bw_line_parse(measured_bw_line_t *out, const char *line); -int measured_bw_line_apply(measured_bw_line_t *parsed_line, +STATIC int measured_bw_line_apply(measured_bw_line_t *parsed_line, smartlist_t *routerstatuses); -void dirserv_cache_measured_bw(const measured_bw_line_t *parsed_line, +STATIC void dirserv_cache_measured_bw(const measured_bw_line_t *parsed_line, time_t as_of); -void dirserv_clear_measured_bw_cache(void); -void dirserv_expire_measured_bw_cache(time_t now); -int dirserv_get_measured_bw_cache_size(void); -int dirserv_query_measured_bw_cache_kb(const char *node_id, long *bw_out, - time_t *as_of_out); -int dirserv_has_measured_bw(const char *node_id); -cached_dir_t *generate_v2_networkstatus_opinion(void); +STATIC void dirserv_clear_measured_bw_cache(void); +STATIC void dirserv_expire_measured_bw_cache(time_t now); +STATIC int dirserv_get_measured_bw_cache_size(void); +STATIC int dirserv_query_measured_bw_cache_kb(const char *node_id, + long *bw_out, + time_t *as_of_out); +STATIC int dirserv_has_measured_bw(const char *node_id); #endif int dirserv_read_measured_bandwidths(const char *from_file, diff --git a/src/or/dirvote.c b/src/or/dirvote.c index c6d1244902..c7be343ca2 100644 --- a/src/or/dirvote.c +++ b/src/or/dirvote.c @@ -60,7 +60,7 @@ static char *make_consensus_method_list(int low, int high, const char *sep); /** Return a new string containing the string representation of the vote in * <b>v3_ns</b>, signed with our v3 signing key <b>private_signing_key</b>. * For v3 authorities. */ -char * +STATIC char * format_networkstatus_vote(crypto_pk_t *private_signing_key, networkstatus_t *v3_ns) { @@ -587,7 +587,7 @@ compute_consensus_versions_list(smartlist_t *lst, int n_versioning) /** Helper: given a list of valid networkstatus_t, return a new string * containing the contents of the consensus network parameter set. */ -/* private */ char * +STATIC char * dirvote_compute_params(smartlist_t *votes, int method, int total_authorities) { int i; @@ -2533,12 +2533,13 @@ dirvote_get_preferred_voting_intervals(vote_timing_t *timing_out) timing_out->dist_delay = options->V3AuthDistDelay; } -/** Return the start of the next interval of size <b>interval</b> (in seconds) - * after <b>now</b>. Midnight always starts a fresh interval, and if the last - * interval of a day would be truncated to less than half its size, it is - * rolled into the previous interval. */ +/** Return the start of the next interval of size <b>interval</b> (in + * seconds) after <b>now</b>, plus <b>offset</b>. Midnight always + * starts a fresh interval, and if the last interval of a day would be + * truncated to less than half its size, it is rolled into the + * previous interval. */ time_t -dirvote_get_start_of_next_interval(time_t now, int interval) +dirvote_get_start_of_next_interval(time_t now, int interval, int offset) { struct tm tm; time_t midnight_today=0; @@ -2566,6 +2567,10 @@ dirvote_get_start_of_next_interval(time_t now, int interval) if (next + interval/2 > midnight_tomorrow) next = midnight_tomorrow; + next += offset; + if (next - interval > now) + next -= interval; + return next; } @@ -2629,8 +2634,10 @@ dirvote_recalculate_timing(const or_options_t *options, time_t now) vote_delay = dist_delay = interval / 4; start = voting_schedule.interval_starts = - dirvote_get_start_of_next_interval(now,interval); - end = dirvote_get_start_of_next_interval(start+1, interval); + dirvote_get_start_of_next_interval(now,interval, + options->TestingV3AuthVotingStartOffset); + end = dirvote_get_start_of_next_interval(start+1, interval, + options->TestingV3AuthVotingStartOffset); tor_assert(end > start); @@ -3136,7 +3143,7 @@ dirvote_compute_consensuses(void) }); votefile = get_datadir_fname("v3-status-votes"); - write_chunks_to_file(votefile, votestrings, 0); + write_chunks_to_file(votefile, votestrings, 0, 0); tor_free(votefile); SMARTLIST_FOREACH(votestrings, sized_chunk_t *, c, tor_free(c)); smartlist_free(votestrings); @@ -3581,6 +3588,12 @@ dirvote_create_microdescriptor(const routerinfo_t *ri, int consensus_method) tor_free(p6); } + if (consensus_method >= MIN_METHOD_FOR_ID_HASH_IN_MD) { + char idbuf[BASE64_DIGEST_LEN+1]; + digest_to_base64(idbuf, ri->cache_info.identity_digest); + smartlist_add_asprintf(chunks, "id rsa1024 %s\n", idbuf); + } + output = smartlist_join_strings(chunks, "", 0, NULL); { @@ -3650,7 +3663,8 @@ static const struct consensus_method_range_t { {MIN_METHOD_FOR_MICRODESC, MIN_METHOD_FOR_A_LINES - 1}, {MIN_METHOD_FOR_A_LINES, MIN_METHOD_FOR_P6_LINES - 1}, {MIN_METHOD_FOR_P6_LINES, MIN_METHOD_FOR_NTOR_KEY - 1}, - {MIN_METHOD_FOR_NTOR_KEY, MAX_SUPPORTED_CONSENSUS_METHOD}, + {MIN_METHOD_FOR_NTOR_KEY, MIN_METHOD_FOR_ID_HASH_IN_MD - 1}, + {MIN_METHOD_FOR_ID_HASH_IN_MD, MAX_SUPPORTED_CONSENSUS_METHOD}, {-1, -1} }; diff --git a/src/or/dirvote.h b/src/or/dirvote.h index b236452122..4c57e43661 100644 --- a/src/or/dirvote.h +++ b/src/or/dirvote.h @@ -12,15 +12,17 @@ #ifndef TOR_DIRVOTE_H #define TOR_DIRVOTE_H +#include "testsupport.h" + /** Lowest allowable value for VoteSeconds. */ -#define MIN_VOTE_SECONDS 20 +#define MIN_VOTE_SECONDS 2 /** Lowest allowable value for DistSeconds. */ -#define MIN_DIST_SECONDS 20 +#define MIN_DIST_SECONDS 2 /** Smallest allowable voting interval. */ #define MIN_VOTE_INTERVAL 300 /** The highest consensus method that we currently support. */ -#define MAX_SUPPORTED_CONSENSUS_METHOD 17 +#define MAX_SUPPORTED_CONSENSUS_METHOD 18 /** Lowest consensus method that contains a 'directory-footer' marker */ #define MIN_METHOD_FOR_FOOTER 9 @@ -59,6 +61,10 @@ * Unmeasured=1 flag for unmeasured bandwidths */ #define MIN_METHOD_TO_CLIP_UNMEASURED_BW 17 +/** Lowest consensus method where authorities may include an "id" line in + * microdescriptors. */ +#define MIN_METHOD_FOR_ID_HASH_IN_MD 18 + /** Default bandwidth to clip unmeasured bandwidths to using method >= * MIN_METHOD_TO_CLIP_UNMEASURED_BW */ #define DEFAULT_MAX_UNMEASURED_BW_KB 20 @@ -86,7 +92,9 @@ authority_cert_t *authority_cert_dup(authority_cert_t *cert); /* vote scheduling */ void dirvote_get_preferred_voting_intervals(vote_timing_t *timing_out); -time_t dirvote_get_start_of_next_interval(time_t now, int interval); +time_t dirvote_get_start_of_next_interval(time_t now, + int interval, + int offset); void dirvote_recalculate_timing(const or_options_t *options, time_t now); void dirvote_act(const or_options_t *options, time_t now); @@ -134,9 +142,9 @@ document_signature_t *voter_get_sig_by_algorithm( digest_algorithm_t alg); #ifdef DIRVOTE_PRIVATE -char *format_networkstatus_vote(crypto_pk_t *private_key, +STATIC char *format_networkstatus_vote(crypto_pk_t *private_key, networkstatus_t *v3_ns); -char *dirvote_compute_params(smartlist_t *votes, int method, +STATIC char *dirvote_compute_params(smartlist_t *votes, int method, int total_authorities); #endif diff --git a/src/or/dns.c b/src/or/dns.c index fb1b10d82c..a9c4318651 100644 --- a/src/or/dns.c +++ b/src/or/dns.c @@ -24,6 +24,7 @@ #include "relay.h" #include "router.h" #include "ht.h" +#include "../common/sandbox.h" #ifdef HAVE_EVENT2_DNS_H #include <event2/event.h> #include <event2/dns.h> @@ -238,7 +239,7 @@ cached_resolves_eq(cached_resolve_t *a, cached_resolve_t *b) static INLINE unsigned int cached_resolve_hash(cached_resolve_t *a) { - return ht_string_hash(a->address); + return (unsigned) siphash24g((const uint8_t*)a->address, strlen(a->address)); } HT_PROTOTYPE(cache_map, cached_resolve_t, node, cached_resolve_hash, @@ -1444,13 +1445,14 @@ configure_nameservers(int force) const or_options_t *options; const char *conf_fname; struct stat st; - int r; + int r, flags; options = get_options(); conf_fname = options->ServerDNSResolvConfFile; #ifndef _WIN32 if (!conf_fname) conf_fname = "/etc/resolv.conf"; #endif + flags = DNS_OPTIONS_ALL; if (!the_evdns_base) { if (!(the_evdns_base = evdns_base_new(tor_libevent_get_base(), 0))) { @@ -1478,7 +1480,8 @@ configure_nameservers(int force) evdns_set_log_fn(evdns_log_cb); if (conf_fname) { - if (stat(conf_fname, &st)) { + log_debug(LD_FS, "stat()ing %s", conf_fname); + if (stat(sandbox_intern_string(conf_fname), &st)) { log_warn(LD_EXIT, "Unable to stat resolver configuration in '%s': %s", conf_fname, strerror(errno)); goto err; @@ -1492,9 +1495,17 @@ configure_nameservers(int force) evdns_base_search_clear(the_evdns_base); evdns_base_clear_nameservers_and_suspend(the_evdns_base); } +#if defined(DNS_OPTION_HOSTSFILE) && defined(USE_LIBSECCOMP) + if (flags & DNS_OPTION_HOSTSFILE) { + flags ^= DNS_OPTION_HOSTSFILE; + log_debug(LD_FS, "Loading /etc/hosts"); + evdns_base_load_hosts(the_evdns_base, + sandbox_intern_string("/etc/hosts")); + } +#endif log_info(LD_EXIT, "Parsing resolver configuration in '%s'", conf_fname); - if ((r = evdns_base_resolv_conf_parse(the_evdns_base, - DNS_OPTIONS_ALL, conf_fname))) { + if ((r = evdns_base_resolv_conf_parse(the_evdns_base, flags, + sandbox_intern_string(conf_fname)))) { log_warn(LD_EXIT, "Unable to parse '%s', or no nameservers in '%s' (%d)", conf_fname, conf_fname, r); goto err; @@ -2163,7 +2174,7 @@ static void assert_cache_ok_(void) { cached_resolve_t **resolve; - int bad_rep = _cache_map_HT_REP_IS_BAD(&cache_root); + int bad_rep = HT_REP_IS_BAD_(cache_map, &cache_root); if (bad_rep) { log_err(LD_BUG, "Bad rep type %d on dns cache hash table", bad_rep); tor_assert(!bad_rep); diff --git a/src/or/dnsserv.c b/src/or/dnsserv.c index ebff7b524c..ecd45be77c 100644 --- a/src/or/dnsserv.c +++ b/src/or/dnsserv.c @@ -35,7 +35,7 @@ evdns_server_callback(struct evdns_server_request *req, void *data_) entry_connection_t *entry_conn; edge_connection_t *conn; int i = 0; - struct evdns_server_question *q = NULL; + struct evdns_server_question *q = NULL, *supported_q = NULL; struct sockaddr_storage addr; struct sockaddr *sa; int addrlen; @@ -87,31 +87,37 @@ evdns_server_callback(struct evdns_server_request *req, void *data_) for (i = 0; i < req->nquestions; ++i) { if (req->questions[i]->dns_question_class != EVDNS_CLASS_INET) continue; + if (! q) + q = req->questions[i]; switch (req->questions[i]->type) { case EVDNS_TYPE_A: case EVDNS_TYPE_AAAA: case EVDNS_TYPE_PTR: - q = req->questions[i]; + /* We always pick the first one of these questions, if there is + one. */ + if (! supported_q) + supported_q = q; + break; default: break; } } + if (supported_q) + q = supported_q; if (!q) { log_info(LD_APP, "None of the questions we got were ones we're willing " "to support. Sending NOTIMPL."); evdns_server_request_respond(req, DNS_ERR_NOTIMPL); return; } - if (q->type != EVDNS_TYPE_A && q->type != EVDNS_TYPE_AAAA) { - tor_assert(q->type == EVDNS_TYPE_PTR); - } /* Make sure the name isn't too long: This should be impossible, I think. */ if (err == DNS_ERR_NONE && strlen(q->name) > MAX_SOCKS_ADDR_LEN-1) err = DNS_ERR_FORMAT; - if (err != DNS_ERR_NONE) { - /* We got an error? Then send back an answer immediately; we're done. */ + if (err != DNS_ERR_NONE || !supported_q) { + /* We got an error? There's no question we're willing to answer? Then + * send back an answer immediately; we're done. */ evdns_server_request_respond(req, err); return; } @@ -126,10 +132,23 @@ evdns_server_callback(struct evdns_server_request *req, void *data_) TO_CONN(conn)->port = port; TO_CONN(conn)->address = tor_dup_addr(&tor_addr); - if (q->type == EVDNS_TYPE_A || q->type == EVDNS_TYPE_AAAA) + if (q->type == EVDNS_TYPE_A || q->type == EVDNS_TYPE_AAAA || + q->type == EVDNS_QTYPE_ALL) { entry_conn->socks_request->command = SOCKS_COMMAND_RESOLVE; - else + } else { + tor_assert(q->type == EVDNS_TYPE_PTR); entry_conn->socks_request->command = SOCKS_COMMAND_RESOLVE_PTR; + } + + if (q->type == EVDNS_TYPE_A || q->type == EVDNS_QTYPE_ALL) { + entry_conn->ipv4_traffic_ok = 1; + entry_conn->ipv6_traffic_ok = 0; + entry_conn->prefer_ipv6_traffic = 0; + } else if (q->type == EVDNS_TYPE_AAAA) { + entry_conn->ipv4_traffic_ok = 0; + entry_conn->ipv6_traffic_ok = 1; + entry_conn->prefer_ipv6_traffic = 1; + } strlcpy(entry_conn->socks_request->address, q->name, sizeof(entry_conn->socks_request->address)); diff --git a/src/or/entrynodes.c b/src/or/entrynodes.c index 484b88dbf8..66b7201187 100644 --- a/src/or/entrynodes.c +++ b/src/or/entrynodes.c @@ -13,6 +13,7 @@ **/ #include "or.h" +#include "circpathbias.h" #include "circuitbuild.h" #include "circuitstats.h" #include "config.h" @@ -54,6 +55,10 @@ typedef struct { /** When should we next try to fetch a descriptor for this bridge? */ download_status_t fetch_status; + + /** A smartlist of k=v values to be passed to the SOCKS proxy, if + transports are used for this bridge. */ + smartlist_t *socks_args; } bridge_info_t; /** A list of our chosen entry guards. */ @@ -65,7 +70,9 @@ static int entry_guards_dirty = 0; static void bridge_free(bridge_info_t *bridge); static const node_t *choose_random_entry_impl(cpath_build_state_t *state, int for_directory, - dirinfo_type_t dirtype); + dirinfo_type_t dirtype, + int *n_options_out); +static int num_bridges_usable(void); /** Return the list of entry guards, creating it if necessary. */ const smartlist_t * @@ -359,7 +366,7 @@ add_an_entry_guard(const node_t *chosen, int reset_status, int prepend, entry->can_retry = 1; } entry->is_dir_cache = node->rs && - node->rs->version_supports_microdesc_cache; + node->rs->version_supports_microdesc_cache; if (get_options()->UseBridges && node_is_a_configured_bridge(node)) entry->is_dir_cache = 1; return NULL; @@ -371,7 +378,7 @@ add_an_entry_guard(const node_t *chosen, int reset_status, int prepend, } else { const routerstatus_t *rs; rs = router_pick_directory_server(MICRODESC_DIRINFO|V3_DIRINFO, - PDS_PREFER_TUNNELED_DIR_CONNS_|PDS_FOR_GUARD); + PDS_FOR_GUARD); if (!rs) return NULL; node = node_get_by_id(rs->identity_digest); @@ -392,8 +399,8 @@ add_an_entry_guard(const node_t *chosen, int reset_status, int prepend, node_describe(node)); strlcpy(entry->nickname, node_get_nickname(node), sizeof(entry->nickname)); memcpy(entry->identity, node->identity, DIGEST_LEN); - entry->is_dir_cache = node_is_dir(node) && - node->rs && node->rs->version_supports_microdesc_cache; + entry->is_dir_cache = node_is_dir(node) && node->rs && + node->rs->version_supports_microdesc_cache; if (get_options()->UseBridges && node_is_a_configured_bridge(node)) entry->is_dir_cache = 1; @@ -605,6 +612,25 @@ remove_dead_entry_guards(time_t now) return changed ? 1 : 0; } +/** Remove all currently listed entry guards. So new ones will be chosen. */ +void +remove_all_entry_guards(void) +{ + char dbuf[HEX_DIGEST_LEN+1]; + + while (smartlist_len(entry_guards)) { + entry_guard_t *entry = smartlist_get(entry_guards, 0); + base16_encode(dbuf, sizeof(dbuf), entry->identity, DIGEST_LEN); + log_info(LD_CIRC, "Entry guard '%s' (%s) has been dropped.", + entry->nickname, dbuf); + control_event_guard(entry->nickname, entry->identity, "DROPPED"); + entry_guard_free(entry); + smartlist_del(entry_guards, 0); + } + log_entry_guards(LOG_INFO); + entry_guards_changed(); +} + /** A new directory or router-status has arrived; update the down/listed * status of the entry guards. * @@ -970,7 +996,7 @@ node_can_handle_dirinfo(const node_t *node, dirinfo_type_t dirinfo) const node_t * choose_random_entry(cpath_build_state_t *state) { - return choose_random_entry_impl(state, 0, 0); + return choose_random_entry_impl(state, 0, 0, NULL); } /** Pick a live (up and listed) directory guard from entry_guards for @@ -978,13 +1004,13 @@ choose_random_entry(cpath_build_state_t *state) const node_t * choose_random_dirguard(dirinfo_type_t type) { - return choose_random_entry_impl(NULL, 1, type); + return choose_random_entry_impl(NULL, 1, type, NULL); } /** Helper for choose_random{entry,dirguard}. */ static const node_t * choose_random_entry_impl(cpath_build_state_t *state, int for_directory, - dirinfo_type_t dirinfo_type) + dirinfo_type_t dirinfo_type, int *n_options_out) { const or_options_t *options = get_options(); smartlist_t *live_entry_guards = smartlist_new(); @@ -998,6 +1024,9 @@ choose_random_entry_impl(cpath_build_state_t *state, int for_directory, int need_descriptor = !for_directory; const int num_needed = decide_num_guards(options, for_directory); + if (n_options_out) + *n_options_out = 0; + if (chosen_exit) { nodelist_add_node_and_family(exit_family, chosen_exit); consider_exit_family = 1; @@ -1124,6 +1153,8 @@ choose_random_entry_impl(cpath_build_state_t *state, int for_directory, * *double*-weight our guard selection. */ node = smartlist_choose(live_entry_guards); } + if (n_options_out) + *n_options_out = smartlist_len(live_entry_guards); smartlist_free(live_entry_guards); smartlist_free(exit_family); return node; @@ -1607,6 +1638,11 @@ bridge_free(bridge_info_t *bridge) return; tor_free(bridge->transport_name); + if (bridge->socks_args) { + SMARTLIST_FOREACH(bridge->socks_args, char*, s, tor_free(s)); + smartlist_free(bridge->socks_args); + } + tor_free(bridge); } @@ -1640,7 +1676,8 @@ get_configured_bridge_by_orports_digest(const char *digest, /** If we have a bridge configured whose digest matches <b>digest</b>, or a * bridge with no known digest whose address matches <b>addr</b>:<b>/port</b>, - * return that bridge. Else return NULL. */ + * return that bridge. Else return NULL. If <b>digest</b> is NULL, check for + * address/port matches only. */ static bridge_info_t * get_configured_bridge_by_addr_port_digest(const tor_addr_t *addr, uint16_t port, @@ -1650,7 +1687,7 @@ get_configured_bridge_by_addr_port_digest(const tor_addr_t *addr, return NULL; SMARTLIST_FOREACH_BEGIN(bridge_list, bridge_info_t *, bridge) { - if (tor_digest_is_zero(bridge->identity) && + if ((tor_digest_is_zero(bridge->identity) || digest == NULL) && !tor_addr_compare(&bridge->addr, addr, CMP_EXACT) && bridge->port == port) return bridge; @@ -1785,30 +1822,68 @@ bridge_resolve_conflicts(const tor_addr_t *addr, uint16_t port, } SMARTLIST_FOREACH_END(bridge); } -/** Remember a new bridge at <b>addr</b>:<b>port</b>. If <b>digest</b> - * is set, it tells us the identity key too. If we already had the - * bridge in our list, unmark it, and don't actually add anything new. - * If <b>transport_name</b> is non-NULL - the bridge is associated with a - * pluggable transport - we assign the transport to the bridge. */ +/** Return True if we have a bridge that uses a transport with name + * <b>transport_name</b>. */ +int +transport_is_needed(const char *transport_name) +{ + if (!bridge_list) + return 0; + + SMARTLIST_FOREACH_BEGIN(bridge_list, const bridge_info_t *, bridge) { + if (bridge->transport_name && + !strcmp(bridge->transport_name, transport_name)) + return 1; + } SMARTLIST_FOREACH_END(bridge); + + return 0; +} + +/** Register the bridge information in <b>bridge_line</b> to the + * bridge subsystem. Steals reference of <b>bridge_line</b>. */ void -bridge_add_from_config(const tor_addr_t *addr, uint16_t port, - const char *digest, const char *transport_name) +bridge_add_from_config(bridge_line_t *bridge_line) { bridge_info_t *b; - bridge_resolve_conflicts(addr, port, digest, transport_name); + { /* Log the bridge we are about to register: */ + log_debug(LD_GENERAL, "Registering bridge at %s (transport: %s) (%s)", + fmt_addrport(&bridge_line->addr, bridge_line->port), + bridge_line->transport_name ? + bridge_line->transport_name : "no transport", + tor_digest_is_zero(bridge_line->digest) ? + "no key listed" : hex_str(bridge_line->digest, DIGEST_LEN)); + + if (bridge_line->socks_args) { /* print socks arguments */ + int i = 0; + + tor_assert(smartlist_len(bridge_line->socks_args) > 0); + + log_debug(LD_GENERAL, "Bridge uses %d SOCKS arguments:", + smartlist_len(bridge_line->socks_args)); + SMARTLIST_FOREACH(bridge_line->socks_args, const char *, arg, + log_debug(LD_CONFIG, "%d: %s", ++i, arg)); + } + } + + bridge_resolve_conflicts(&bridge_line->addr, + bridge_line->port, + bridge_line->digest, + bridge_line->transport_name); b = tor_malloc_zero(sizeof(bridge_info_t)); - tor_addr_copy(&b->addr, addr); - b->port = port; - if (digest) - memcpy(b->identity, digest, DIGEST_LEN); - if (transport_name) - b->transport_name = tor_strdup(transport_name); + tor_addr_copy(&b->addr, &bridge_line->addr); + b->port = bridge_line->port; + memcpy(b->identity, bridge_line->digest, DIGEST_LEN); + if (bridge_line->transport_name) + b->transport_name = bridge_line->transport_name; b->fetch_status.schedule = DL_SCHED_BRIDGE; + b->socks_args = bridge_line->socks_args; if (!bridge_list) bridge_list = smartlist_new(); + tor_free(bridge_line); /* Deallocate bridge_line now. */ + smartlist_add(bridge_list, b); } @@ -1869,7 +1944,7 @@ find_transport_name_by_bridge_addrport(const tor_addr_t *addr, uint16_t port) * transport, but the transport could not be found. */ int -find_transport_by_bridge_addrport(const tor_addr_t *addr, uint16_t port, +get_transport_by_bridge_addrport(const tor_addr_t *addr, uint16_t port, const transport_t **transport) { *transport = NULL; @@ -1896,11 +1971,21 @@ find_transport_by_bridge_addrport(const tor_addr_t *addr, uint16_t port, return 0; } +/** Return a smartlist containing all the SOCKS arguments that we + * should pass to the SOCKS proxy. */ +const smartlist_t * +get_socks_args_by_bridge_addrport(const tor_addr_t *addr, uint16_t port) +{ + bridge_info_t *bridge = get_configured_bridge_by_addr_port_digest(addr, + port, + NULL); + return bridge ? bridge->socks_args : NULL; +} + /** We need to ask <b>bridge</b> for its server descriptor. */ static void launch_direct_bridge_descriptor_fetch(bridge_info_t *bridge) { - char *address; const or_options_t *options = get_options(); if (connection_get_by_type_addr_port_purpose( @@ -1915,15 +2000,12 @@ launch_direct_bridge_descriptor_fetch(bridge_info_t *bridge) return; } - address = tor_dup_addr(&bridge->addr); - - directory_initiate_command(address, &bridge->addr, + directory_initiate_command(&bridge->addr, bridge->port, 0/*no dirport*/, bridge->identity, DIR_PURPOSE_FETCH_SERVERDESC, ROUTER_PURPOSE_BRIDGE, DIRIND_ONEHOP, "authority.z", NULL, 0, 0); - tor_free(address); } /** Fetching the bridge descriptor from the bridge authority returned a @@ -2041,13 +2123,11 @@ rewrite_node_address_for_bridge(const bridge_info_t *bridge, node_t *node) } else { if (tor_addr_family(&bridge->addr) == AF_INET) { ri->addr = tor_addr_to_ipv4h(&bridge->addr); - tor_free(ri->address); - ri->address = tor_dup_ip(ri->addr); ri->or_port = bridge->port; log_info(LD_DIR, "Adjusted bridge routerinfo for '%s' to match configured " "address %s:%d.", - ri->nickname, ri->address, ri->or_port); + ri->nickname, fmt_addr32(ri->addr), ri->or_port); } else if (tor_addr_family(&bridge->addr) == AF_INET6) { tor_addr_copy(&ri->ipv6_addr, &bridge->addr); ri->ipv6_orport = bridge->port; @@ -2105,7 +2185,7 @@ learned_bridge_descriptor(routerinfo_t *ri, int from_cache) tor_assert(ri); tor_assert(ri->purpose == ROUTER_PURPOSE_BRIDGE); if (get_options()->UseBridges) { - int first = !any_bridge_descriptors_known(); + int first = num_bridges_usable() <= 1; bridge_info_t *bridge = get_configured_bridge_by_routerinfo(ri); time_t now = time(NULL); router_set_status(ri->cache_info.identity_digest, 1); @@ -2128,17 +2208,14 @@ learned_bridge_descriptor(routerinfo_t *ri, int from_cache) entry_guard_register_connect_status(ri->cache_info.identity_digest, 1, 0, now); if (first) { - /* XXXX apparently, this is never called. See bug #9229. */ routerlist_retry_directory_downloads(now); } - - update_networkstatus_downloads(now); } } } -/** Return 1 if any of our entry guards have descriptors that - * are marked with purpose 'bridge' and are running. Else return 0. +/** Return the number of bridges that have descriptors that + * are marked with purpose 'bridge' and are running. * * We use this function to decide if we're ready to start building * circuits through our bridges, or if we need to wait until the @@ -2150,25 +2227,16 @@ any_bridge_descriptors_known(void) return choose_random_entry(NULL) != NULL; } -/** Return 1 if there are any directory conns fetching bridge descriptors - * that aren't marked for close. We use this to guess if we should tell - * the controller that we have a problem. */ -int -any_pending_bridge_descriptor_fetches(void) +/** Return the number of bridges that have descriptors that are marked with + * purpose 'bridge' and are running. + */ +static int +num_bridges_usable(void) { - smartlist_t *conns = get_connection_array(); - SMARTLIST_FOREACH_BEGIN(conns, connection_t *, conn) { - if (conn->type == CONN_TYPE_DIR && - conn->purpose == DIR_PURPOSE_FETCH_SERVERDESC && - TO_DIR_CONN(conn)->router_purpose == ROUTER_PURPOSE_BRIDGE && - !conn->marked_for_close && - conn->linked && - conn->linked_conn && !conn->linked_conn->marked_for_close) { - log_debug(LD_DIR, "found one: %s", conn->address); - return 1; - } - } SMARTLIST_FOREACH_END(conn); - return 0; + int n_options = 0; + tor_assert(get_options()->UseBridges); + (void) choose_random_entry_impl(NULL, 0, 0, &n_options); + return n_options; } /** Return 1 if we have at least one descriptor for an entry guard @@ -2266,6 +2334,6 @@ entry_guards_free_all(void) clear_bridge_list(); smartlist_free(bridge_list); bridge_list = NULL; - circuit_build_times_free_timeouts(&circ_times); + circuit_build_times_free_timeouts(get_circuit_build_times_mutable()); } diff --git a/src/or/entrynodes.h b/src/or/entrynodes.h index 52b8dc00e4..e229f3b79a 100644 --- a/src/or/entrynodes.h +++ b/src/or/entrynodes.h @@ -5,7 +5,7 @@ /* See LICENSE for licensing information */ /** - * \file guardnodes.h + * \file entrynodes.h * \brief Header file for circuitbuild.c. **/ @@ -77,6 +77,8 @@ int num_live_entry_guards(int for_directory); #endif +void remove_all_entry_guards(void); + void entry_guards_compute_status(const or_options_t *options, time_t now); int entry_guard_register_connect_status(const char *digest, int succeeded, int mark_relay_status, time_t now); @@ -97,27 +99,30 @@ int routerinfo_is_a_configured_bridge(const routerinfo_t *ri); int node_is_a_configured_bridge(const node_t *node); void learned_router_identity(const tor_addr_t *addr, uint16_t port, const char *digest); -void bridge_add_from_config(const tor_addr_t *addr, uint16_t port, - const char *digest, - const char *transport_name); +struct bridge_line_t; +void bridge_add_from_config(struct bridge_line_t *bridge_line); void retry_bridge_descriptor_fetch_directly(const char *digest); void fetch_bridge_descriptors(const or_options_t *options, time_t now); void learned_bridge_descriptor(routerinfo_t *ri, int from_cache); int any_bridge_descriptors_known(void); -int any_pending_bridge_descriptor_fetches(void); int entries_known_but_down(const or_options_t *options); void entries_retry_all(const or_options_t *options); int any_bridge_supports_microdescriptors(void); +const smartlist_t *get_socks_args_by_bridge_addrport(const tor_addr_t *addr, + uint16_t port); + +int any_bridges_dont_support_microdescriptors(void); void entry_guards_free_all(void); const char *find_transport_name_by_bridge_addrport(const tor_addr_t *addr, uint16_t port); struct transport_t; -int find_transport_by_bridge_addrport(const tor_addr_t *addr, uint16_t port, +int get_transport_by_bridge_addrport(const tor_addr_t *addr, uint16_t port, const struct transport_t **transport); +int transport_is_needed(const char *transport_name); int validate_pluggable_transports_config(void); double pathbias_get_close_success_count(entry_guard_t *guard); diff --git a/src/or/ext_orport.c b/src/or/ext_orport.c new file mode 100644 index 0000000000..0d28a9199a --- /dev/null +++ b/src/or/ext_orport.c @@ -0,0 +1,648 @@ +/* Copyright (c) 2012, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file ext_orport.c + * \brief Code implementing the Extended ORPort. +*/ + +#define EXT_ORPORT_PRIVATE +#include "or.h" +#include "connection.h" +#include "connection_or.h" +#include "ext_orport.h" +#include "control.h" +#include "config.h" +#include "util.h" +#include "main.h" + +/** Allocate and return a structure capable of holding an Extended + * ORPort message of body length <b>len</b>. */ +ext_or_cmd_t * +ext_or_cmd_new(uint16_t len) +{ + size_t size = STRUCT_OFFSET(ext_or_cmd_t, body) + len; + ext_or_cmd_t *cmd = tor_malloc(size); + cmd->len = len; + return cmd; +} + +/** Deallocate the Extended ORPort message in <b>cmd</b>. */ +void +ext_or_cmd_free(ext_or_cmd_t *cmd) +{ + tor_free(cmd); +} + +/** Get an Extended ORPort message from <b>conn</b>, and place it in + * <b>out</b>. Return -1 on fail, 0 if we need more data, and 1 if we + * successfully extracted an Extended ORPort command from the + * buffer. */ +static int +connection_fetch_ext_or_cmd_from_buf(connection_t *conn, ext_or_cmd_t **out) +{ + IF_HAS_BUFFEREVENT(conn, { + struct evbuffer *input = bufferevent_get_input(conn->bufev); + return fetch_ext_or_command_from_evbuffer(input, out); + }) ELSE_IF_NO_BUFFEREVENT { + return fetch_ext_or_command_from_buf(conn->inbuf, out); + } +} + +/** Write an Extended ORPort message to <b>conn</b>. Use + * <b>command</b> as the command type, <b>bodylen</b> as the body + * length, and <b>body</b>, if it's present, as the body of the + * message. */ +STATIC int +connection_write_ext_or_command(connection_t *conn, + uint16_t command, + const char *body, + size_t bodylen) +{ + char header[4]; + if (bodylen > UINT16_MAX) + return -1; + set_uint16(header, htons(command)); + set_uint16(header+2, htons(bodylen)); + connection_write_to_buf(header, 4, conn); + if (bodylen) { + tor_assert(body); + connection_write_to_buf(body, bodylen, conn); + } + return 0; +} + +/** Transition from an Extended ORPort which accepts Extended ORPort + * messages, to an Extended ORport which accepts OR traffic. */ +static void +connection_ext_or_transition(or_connection_t *conn) +{ + tor_assert(conn->base_.type == CONN_TYPE_EXT_OR); + + conn->base_.type = CONN_TYPE_OR; + TO_CONN(conn)->state = 0; // set the state to a neutral value + control_event_or_conn_status(conn, OR_CONN_EVENT_NEW, 0); + connection_tls_start_handshake(conn, 1); +} + +/** Length of authentication cookie. */ +#define EXT_OR_PORT_AUTH_COOKIE_LEN 32 +/** Length of the header of the cookie file. */ +#define EXT_OR_PORT_AUTH_COOKIE_HEADER_LEN 32 +/** Static cookie file header. */ +#define EXT_OR_PORT_AUTH_COOKIE_HEADER "! Extended ORPort Auth Cookie !\x0a" +/** Length of safe-cookie protocol hashes. */ +#define EXT_OR_PORT_AUTH_HASH_LEN DIGEST256_LEN +/** Length of safe-cookie protocol nonces. */ +#define EXT_OR_PORT_AUTH_NONCE_LEN 32 +/** Safe-cookie protocol constants. */ +#define EXT_OR_PORT_AUTH_SERVER_TO_CLIENT_CONST \ + "ExtORPort authentication server-to-client hash" +#define EXT_OR_PORT_AUTH_CLIENT_TO_SERVER_CONST \ + "ExtORPort authentication client-to-server hash" + +/* Code to indicate cookie authentication */ +#define EXT_OR_AUTHTYPE_SAFECOOKIE 0x01 + +/** If true, we've set ext_or_auth_cookie to a secret code and stored + * it to disk. */ +STATIC int ext_or_auth_cookie_is_set = 0; +/** If ext_or_auth_cookie_is_set, a secret cookie that we've stored to disk + * and which we're using to authenticate controllers. (If the controller can + * read it off disk, it has permission to connect.) */ +STATIC uint8_t *ext_or_auth_cookie = NULL; + +/** Helper: Return a newly allocated string containing a path to the + * file where we store our authentication cookie. */ +char * +get_ext_or_auth_cookie_file_name(void) +{ + const or_options_t *options = get_options(); + if (options->ExtORPortCookieAuthFile && + strlen(options->ExtORPortCookieAuthFile)) { + return tor_strdup(options->ExtORPortCookieAuthFile); + } else { + return get_datadir_fname("extended_orport_auth_cookie"); + } +} + +/* Initialize the cookie-based authentication system of the + * Extended ORPort. If <b>is_enabled</b> is 0, then disable the cookie + * authentication system. */ +int +init_ext_or_cookie_authentication(int is_enabled) +{ + char *fname = NULL; + int retval; + + if (!is_enabled) { + ext_or_auth_cookie_is_set = 0; + return 0; + } + + fname = get_ext_or_auth_cookie_file_name(); + retval = init_cookie_authentication(fname, EXT_OR_PORT_AUTH_COOKIE_HEADER, + EXT_OR_PORT_AUTH_COOKIE_HEADER_LEN, + &ext_or_auth_cookie, + &ext_or_auth_cookie_is_set); + tor_free(fname); + return retval; +} + +/** Read data from <b>conn</b> and see if the client sent us the + * authentication type that she prefers to use in this session. + * + * Return -1 if we received corrupted data or if we don't support the + * authentication type. Return 0 if we need more data in + * <b>conn</b>. Return 1 if the authentication type negotiation was + * successful. */ +static int +connection_ext_or_auth_neg_auth_type(connection_t *conn) +{ + char authtype[1] = {0}; + + if (connection_get_inbuf_len(conn) < 1) + return 0; + + if (connection_fetch_from_buf(authtype, 1, conn) < 0) + return -1; + + log_debug(LD_GENERAL, "Client wants us to use %d auth type", authtype[0]); + if (authtype[0] != EXT_OR_AUTHTYPE_SAFECOOKIE) { + /* '1' is the only auth type supported atm */ + return -1; + } + + conn->state = EXT_OR_CONN_STATE_AUTH_WAIT_CLIENT_NONCE; + return 1; +} + +/** DOCDOC */ +STATIC int +handle_client_auth_nonce(const char *client_nonce, size_t client_nonce_len, + char **client_hash_out, + char **reply_out, size_t *reply_len_out) +{ + char server_hash[EXT_OR_PORT_AUTH_HASH_LEN] = {0}; + char server_nonce[EXT_OR_PORT_AUTH_NONCE_LEN] = {0}; + char *reply; + size_t reply_len; + + if (client_nonce_len != EXT_OR_PORT_AUTH_NONCE_LEN) + return -1; + + /* Get our nonce */ + if (crypto_rand(server_nonce, EXT_OR_PORT_AUTH_NONCE_LEN) < 0) + return -1; + + { /* set up macs */ + size_t hmac_s_msg_len = strlen(EXT_OR_PORT_AUTH_SERVER_TO_CLIENT_CONST) + + 2*EXT_OR_PORT_AUTH_NONCE_LEN; + size_t hmac_c_msg_len = strlen(EXT_OR_PORT_AUTH_CLIENT_TO_SERVER_CONST) + + 2*EXT_OR_PORT_AUTH_NONCE_LEN; + + char *hmac_s_msg = tor_malloc_zero(hmac_s_msg_len); + char *hmac_c_msg = tor_malloc_zero(hmac_c_msg_len); + char *correct_client_hash = tor_malloc_zero(EXT_OR_PORT_AUTH_HASH_LEN); + + memcpy(hmac_s_msg, + EXT_OR_PORT_AUTH_SERVER_TO_CLIENT_CONST, + strlen(EXT_OR_PORT_AUTH_SERVER_TO_CLIENT_CONST)); + memcpy(hmac_s_msg + strlen(EXT_OR_PORT_AUTH_SERVER_TO_CLIENT_CONST), + client_nonce, EXT_OR_PORT_AUTH_NONCE_LEN); + memcpy(hmac_s_msg + strlen(EXT_OR_PORT_AUTH_SERVER_TO_CLIENT_CONST) + + EXT_OR_PORT_AUTH_NONCE_LEN, + server_nonce, EXT_OR_PORT_AUTH_NONCE_LEN); + + memcpy(hmac_c_msg, + EXT_OR_PORT_AUTH_CLIENT_TO_SERVER_CONST, + strlen(EXT_OR_PORT_AUTH_CLIENT_TO_SERVER_CONST)); + memcpy(hmac_c_msg + strlen(EXT_OR_PORT_AUTH_CLIENT_TO_SERVER_CONST), + client_nonce, EXT_OR_PORT_AUTH_NONCE_LEN); + memcpy(hmac_c_msg + strlen(EXT_OR_PORT_AUTH_CLIENT_TO_SERVER_CONST) + + EXT_OR_PORT_AUTH_NONCE_LEN, + server_nonce, EXT_OR_PORT_AUTH_NONCE_LEN); + + crypto_hmac_sha256(server_hash, + (char*)ext_or_auth_cookie, + EXT_OR_PORT_AUTH_COOKIE_LEN, + hmac_s_msg, + hmac_s_msg_len); + + crypto_hmac_sha256(correct_client_hash, + (char*)ext_or_auth_cookie, + EXT_OR_PORT_AUTH_COOKIE_LEN, + hmac_c_msg, + hmac_c_msg_len); + + /* Store the client hash we generated. We will need to compare it + with the hash sent by the client. */ + *client_hash_out = correct_client_hash; + + memwipe(hmac_s_msg, 0, hmac_s_msg_len); + memwipe(hmac_c_msg, 0, hmac_c_msg_len); + + tor_free(hmac_s_msg); + tor_free(hmac_c_msg); + } + + { /* debug logging */ /* XXX disable this codepath if not logging on debug?*/ + char server_hash_encoded[(2*EXT_OR_PORT_AUTH_HASH_LEN) + 1]; + char server_nonce_encoded[(2*EXT_OR_PORT_AUTH_NONCE_LEN) + 1]; + char client_nonce_encoded[(2*EXT_OR_PORT_AUTH_NONCE_LEN) + 1]; + + base16_encode(server_hash_encoded, sizeof(server_hash_encoded), + server_hash, sizeof(server_hash)); + base16_encode(server_nonce_encoded, sizeof(server_nonce_encoded), + server_nonce, sizeof(server_nonce)); + base16_encode(client_nonce_encoded, sizeof(client_nonce_encoded), + client_nonce, EXT_OR_PORT_AUTH_NONCE_LEN); + + log_debug(LD_GENERAL, + "server_hash: '%s'\nserver_nonce: '%s'\nclient_nonce: '%s'", + server_hash_encoded, server_nonce_encoded, client_nonce_encoded); + + memwipe(server_hash_encoded, 0, sizeof(server_hash_encoded)); + memwipe(server_nonce_encoded, 0, sizeof(server_nonce_encoded)); + memwipe(client_nonce_encoded, 0, sizeof(client_nonce_encoded)); + } + + { /* write reply: (server_hash, server_nonce) */ + + reply_len = EXT_OR_PORT_AUTH_COOKIE_LEN+EXT_OR_PORT_AUTH_NONCE_LEN; + reply = tor_malloc_zero(reply_len); + memcpy(reply, server_hash, EXT_OR_PORT_AUTH_HASH_LEN); + memcpy(reply + EXT_OR_PORT_AUTH_HASH_LEN, server_nonce, + EXT_OR_PORT_AUTH_NONCE_LEN); + } + + *reply_out = reply; + *reply_len_out = reply_len; + + return 0; +} + +/** Read the client's nonce out of <b>conn</b>, setup the safe-cookie + * crypto, and then send our own hash and nonce to the client + * + * Return -1 if there was an error; return 0 if we need more data in + * <b>conn</b>, and return 1 if we successfully retrieved the + * client's nonce and sent our own. */ +static int +connection_ext_or_auth_handle_client_nonce(connection_t *conn) +{ + char client_nonce[EXT_OR_PORT_AUTH_NONCE_LEN]; + char *reply=NULL; + size_t reply_len=0; + + if (!ext_or_auth_cookie_is_set) { /* this should not happen */ + log_warn(LD_BUG, "Extended ORPort authentication cookie was not set. " + "That's weird since we should have done that on startup. " + "This might be a Tor bug, please file a bug report. "); + return -1; + } + + if (connection_get_inbuf_len(conn) < EXT_OR_PORT_AUTH_NONCE_LEN) + return 0; + + if (connection_fetch_from_buf(client_nonce, + EXT_OR_PORT_AUTH_NONCE_LEN, conn) < 0) + return -1; + + /* We extract the ClientNonce from the received data, and use it to + calculate ServerHash and ServerNonce according to proposal 217. + + We also calculate our own ClientHash value and save it in the + connection state. We validate it later against the ClientHash + sent by the client. */ + if (handle_client_auth_nonce(client_nonce, sizeof(client_nonce), + &TO_OR_CONN(conn)->ext_or_auth_correct_client_hash, + &reply, &reply_len) < 0) + return -1; + + connection_write_to_buf(reply, reply_len, conn); + + memwipe(reply, 0, reply_len); + tor_free(reply); + + log_debug(LD_GENERAL, "Got client nonce, and sent our own nonce and hash."); + + conn->state = EXT_OR_CONN_STATE_AUTH_WAIT_CLIENT_HASH; + return 1; +} + +#define connection_ext_or_auth_send_result_success(c) \ + connection_ext_or_auth_send_result(c, 1) +#define connection_ext_or_auth_send_result_fail(c) \ + connection_ext_or_auth_send_result(c, 0) + +/** Send authentication results to <b>conn</b>. Successful results if + * <b>success</b> is set; failure results otherwise. */ +static void +connection_ext_or_auth_send_result(connection_t *conn, int success) +{ + if (success) + connection_write_to_buf("\x01", 1, conn); + else + connection_write_to_buf("\x00", 1, conn); +} + +/** Receive the client's hash from <b>conn</b>, validate that it's + * correct, and then send the authentication results to the client. + * + * Return -1 if there was an error during validation; return 0 if we + * need more data in <b>conn</b>, and return 1 if we successfully + * validated the client's hash and sent a happy authentication + * result. */ +static int +connection_ext_or_auth_handle_client_hash(connection_t *conn) +{ + char provided_client_hash[EXT_OR_PORT_AUTH_HASH_LEN] = {0}; + + if (connection_get_inbuf_len(conn) < EXT_OR_PORT_AUTH_HASH_LEN) + return 0; + + if (connection_fetch_from_buf(provided_client_hash, + EXT_OR_PORT_AUTH_HASH_LEN, conn) < 0) + return -1; + + if (tor_memneq(TO_OR_CONN(conn)->ext_or_auth_correct_client_hash, + provided_client_hash, EXT_OR_PORT_AUTH_HASH_LEN)) { + log_warn(LD_GENERAL, "Incorrect client hash. Authentication failed."); + connection_ext_or_auth_send_result_fail(conn); + return -1; + } + + log_debug(LD_GENERAL, "Got client's hash and it was legit."); + + /* send positive auth result */ + connection_ext_or_auth_send_result_success(conn); + conn->state = EXT_OR_CONN_STATE_OPEN; + return 1; +} + +/** Handle data from <b>or_conn</b> received on Extended ORPort. + * Return -1 on error. 0 on unsufficient data. 1 on correct. */ +static int +connection_ext_or_auth_process_inbuf(or_connection_t *or_conn) +{ + connection_t *conn = TO_CONN(or_conn); + + /* State transitions of the Extended ORPort authentication protocol: + + EXT_OR_CONN_STATE_AUTH_WAIT_AUTH_TYPE (start state) -> + EXT_OR_CONN_STATE_AUTH_WAIT_CLIENT_NONCE -> + EXT_OR_CONN_STATE_AUTH_WAIT_CLIENT_HASH -> + EXT_OR_CONN_STATE_OPEN + + During EXT_OR_CONN_STATE_OPEN, data is handled by + connection_ext_or_process_inbuf(). + */ + + switch (conn->state) { /* Functionify */ + case EXT_OR_CONN_STATE_AUTH_WAIT_AUTH_TYPE: + return connection_ext_or_auth_neg_auth_type(conn); + + case EXT_OR_CONN_STATE_AUTH_WAIT_CLIENT_NONCE: + return connection_ext_or_auth_handle_client_nonce(conn); + + case EXT_OR_CONN_STATE_AUTH_WAIT_CLIENT_HASH: + return connection_ext_or_auth_handle_client_hash(conn); + + default: + log_warn(LD_BUG, "Encountered unexpected connection state %d while trying " + "to process Extended ORPort authentication data.", conn->state); + return -1; + } +} + +/** Extended ORPort commands (Transport-to-Bridge) */ +#define EXT_OR_CMD_TB_DONE 0x0000 +#define EXT_OR_CMD_TB_USERADDR 0x0001 +#define EXT_OR_CMD_TB_TRANSPORT 0x0002 + +/** Extended ORPort commands (Bridge-to-Transport) */ +#define EXT_OR_CMD_BT_OKAY 0x1000 +#define EXT_OR_CMD_BT_DENY 0x1001 +#define EXT_OR_CMD_BT_CONTROL 0x1002 + +/** Process a USERADDR command from the Extended + * ORPort. <b>payload</b> is a payload of size <b>len</b>. + * + * If the USERADDR command was well formed, change the address of + * <b>conn</b> to the address on the USERADDR command. + * + * Return 0 on success and -1 on error. */ +static int +connection_ext_or_handle_cmd_useraddr(connection_t *conn, + const char *payload, uint16_t len) +{ + /* Copy address string. */ + tor_addr_t addr; + uint16_t port; + char *addr_str; + char *address_part=NULL; + int res; + if (memchr(payload, '\0', len)) { + log_fn(LOG_PROTOCOL_WARN, LD_NET, "Unexpected NUL in ExtORPort UserAddr"); + return -1; + } + + addr_str = tor_memdup_nulterm(payload, len); + + res = tor_addr_port_split(LOG_INFO, addr_str, &address_part, &port); + tor_free(addr_str); + if (res<0) + return -1; + + res = tor_addr_parse(&addr, address_part); + tor_free(address_part); + if (res<0) + return -1; + + { /* do some logging */ + char *old_address = tor_dup_addr(&conn->addr); + char *new_address = tor_dup_addr(&addr); + + log_debug(LD_NET, "Received USERADDR." + "We rewrite our address from '%s:%u' to '%s:%u'.", + safe_str(old_address), conn->port, safe_str(new_address), port); + + tor_free(old_address); + tor_free(new_address); + } + + /* record the address */ + tor_addr_copy(&conn->addr, &addr); + conn->port = port; + if (conn->address) { + tor_free(conn->address); + } + conn->address = tor_dup_addr(&addr); + + return 0; +} + +/** Process a TRANSPORT command from the Extended + * ORPort. <b>payload</b> is a payload of size <b>len</b>. + * + * If the TRANSPORT command was well formed, register the name of the + * transport on <b>conn</b>. + * + * Return 0 on success and -1 on error. */ +static int +connection_ext_or_handle_cmd_transport(or_connection_t *conn, + const char *payload, uint16_t len) +{ + char *transport_str; + if (memchr(payload, '\0', len)) { + log_fn(LOG_PROTOCOL_WARN, LD_NET, "Unexpected NUL in ExtORPort Transport"); + return -1; + } + + transport_str = tor_memdup_nulterm(payload, len); + + /* Transport names MUST be C-identifiers. */ + if (!string_is_C_identifier(transport_str)) { + tor_free(transport_str); + return -1; + } + + /* If ext_or_transport is already occupied (because the PT sent two + * TRANSPORT commands), deallocate the old name and keep the new + * one */ + if (conn->ext_or_transport) + tor_free(conn->ext_or_transport); + + conn->ext_or_transport = transport_str; + return 0; +} + +#define EXT_OR_CONN_STATE_IS_AUTHENTICATING(st) \ + ((st) <= EXT_OR_CONN_STATE_AUTH_MAX) + +/** Process Extended ORPort messages from <b>or_conn</b>. */ +int +connection_ext_or_process_inbuf(or_connection_t *or_conn) +{ + connection_t *conn = TO_CONN(or_conn); + ext_or_cmd_t *command; + int r; + + /* DOCDOC Document the state machine and transitions in this function */ + + /* If we are still in the authentication stage, process traffic as + authentication data: */ + while (EXT_OR_CONN_STATE_IS_AUTHENTICATING(conn->state)) { + log_debug(LD_GENERAL, "Got Extended ORPort authentication data (%u).", + (unsigned int) connection_get_inbuf_len(conn)); + r = connection_ext_or_auth_process_inbuf(or_conn); + if (r < 0) { + connection_mark_for_close(conn); + return -1; + } else if (r == 0) { + return 0; + } + /* if r > 0, loop and process more data (if any). */ + } + + while (1) { + log_debug(LD_GENERAL, "Got Extended ORPort data."); + command = NULL; + r = connection_fetch_ext_or_cmd_from_buf(conn, &command); + if (r < 0) + goto err; + else if (r == 0) + return 0; /* need to wait for more data */ + + /* Got a command! */ + tor_assert(command); + + if (command->cmd == EXT_OR_CMD_TB_DONE) { + if (connection_get_inbuf_len(conn)) { + /* The inbuf isn't empty; the client is misbehaving. */ + goto err; + } + + log_debug(LD_NET, "Received DONE."); + + /* If the transport proxy did not use the TRANSPORT command to + * specify the transport name, mark this as unknown transport. */ + if (!or_conn->ext_or_transport) { + /* We write this string this way to avoid ??>, which is a C + * trigraph. */ + or_conn->ext_or_transport = tor_strdup("<?" "?>"); + } + + connection_write_ext_or_command(conn, EXT_OR_CMD_BT_OKAY, NULL, 0); + + /* can't transition immediately; need to flush first. */ + conn->state = EXT_OR_CONN_STATE_FLUSHING; + connection_stop_reading(conn); + } else if (command->cmd == EXT_OR_CMD_TB_USERADDR) { + if (connection_ext_or_handle_cmd_useraddr(conn, + command->body, command->len) < 0) + goto err; + } else if (command->cmd == EXT_OR_CMD_TB_TRANSPORT) { + if (connection_ext_or_handle_cmd_transport(or_conn, + command->body, command->len) < 0) + goto err; + } else { + log_notice(LD_NET,"Got Extended ORPort command we don't regognize (%u).", + command->cmd); + } + + ext_or_cmd_free(command); + } + + return 0; + + err: + ext_or_cmd_free(command); + connection_mark_for_close(conn); + return -1; +} + +/** <b>conn</b> finished flushing Extended ORPort messages to the + * network, and is now ready to accept OR traffic. This function + * does the transition. */ +int +connection_ext_or_finished_flushing(or_connection_t *conn) +{ + if (conn->base_.state == EXT_OR_CONN_STATE_FLUSHING) { + connection_start_reading(TO_CONN(conn)); + connection_ext_or_transition(conn); + } + return 0; +} + +/** Initiate Extended ORPort authentication, by sending the list of + * supported authentication types to the client. */ +int +connection_ext_or_start_auth(or_connection_t *or_conn) +{ + connection_t *conn = TO_CONN(or_conn); + const uint8_t authtypes[] = { + /* We only support authtype '1' for now. */ + EXT_OR_AUTHTYPE_SAFECOOKIE, + /* Marks the end of the list. */ + 0 + }; + + log_debug(LD_GENERAL, + "ExtORPort authentication: Sending supported authentication types"); + + connection_write_to_buf((const char *)authtypes, sizeof(authtypes), conn); + conn->state = EXT_OR_CONN_STATE_AUTH_WAIT_AUTH_TYPE; + + return 0; +} + +/** Free any leftover allocated memory of the ext_orport.c subsystem. */ +void +ext_orport_free_all(void) +{ + if (ext_or_auth_cookie) /* Free the auth cookie */ + tor_free(ext_or_auth_cookie); +} + diff --git a/src/or/ext_orport.h b/src/or/ext_orport.h new file mode 100644 index 0000000000..ce45e5f418 --- /dev/null +++ b/src/or/ext_orport.h @@ -0,0 +1,42 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2013, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef EXT_ORPORT_H +#define EXT_ORPORT_H + +int connection_ext_or_start_auth(or_connection_t *or_conn); + +ext_or_cmd_t *ext_or_cmd_new(uint16_t len); +void ext_or_cmd_free(ext_or_cmd_t *cmd); +void connection_or_set_ext_or_identifier(or_connection_t *conn); +void connection_or_remove_from_ext_or_id_map(or_connection_t *conn); +void connection_or_clear_ext_or_id_map(void); +or_connection_t *connection_or_get_by_ext_or_id(const char *id); + +int connection_ext_or_finished_flushing(or_connection_t *conn); +int connection_ext_or_process_inbuf(or_connection_t *or_conn); + +int init_ext_or_cookie_authentication(int is_enabled); +char *get_ext_or_auth_cookie_file_name(void); +void ext_orport_free_all(void); + +#ifdef EXT_ORPORT_PRIVATE +STATIC int connection_write_ext_or_command(connection_t *conn, + uint16_t command, + const char *body, + size_t bodylen); +STATIC int handle_client_auth_nonce(const char *client_nonce, + size_t client_nonce_len, + char **client_hash_out, + char **reply_out, size_t *reply_len_out); +#ifdef TOR_UNIT_TESTS +extern uint8_t *ext_or_auth_cookie; +extern int ext_or_auth_cookie_is_set; +#endif +#endif + +#endif + diff --git a/src/or/fp_pair.c b/src/or/fp_pair.c index 4d8a835c83..55e4c89a42 100644 --- a/src/or/fp_pair.c +++ b/src/or/fp_pair.c @@ -32,17 +32,8 @@ fp_pair_map_entries_eq(const fp_pair_map_entry_t *a, static INLINE unsigned int fp_pair_map_entry_hash(const fp_pair_map_entry_t *a) { - const uint32_t *p; - unsigned int hash; - - p = (const uint32_t *)(a->key.first); - /* Hashes are 20 bytes long, so 5 times uint32_t */ - hash = p[0] ^ p[1] ^ p[2] ^ p[3] ^ p[4]; - /* Now XOR in the second fingerprint */ - p = (const uint32_t *)(a->key.second); - hash ^= p[0] ^ p[1] ^ p[2] ^ p[3] ^ p[4]; - - return hash; + tor_assert(sizeof(a->key) == DIGEST_LEN*2); + return (unsigned) siphash24g(&a->key, DIGEST_LEN*2); } /* diff --git a/src/or/geoip.c b/src/or/geoip.c index e2e98e8ec4..f722bac468 100644 --- a/src/or/geoip.c +++ b/src/or/geoip.c @@ -120,7 +120,7 @@ geoip_add_entry(const tor_addr_t *low, const tor_addr_t *high, /** Add an entry to the GeoIP table indicated by <b>family</b>, * parsing it from <b>line</b>. The format is as for geoip_load_file(). */ -/*private*/ int +STATIC int geoip_parse_entry(const char *line, sa_family_t family) { tor_addr_t low_addr, high_addr; @@ -363,7 +363,7 @@ geoip_load_file(sa_family_t family, const char *filename) * be less than geoip_get_n_countries(). To decode it, call * geoip_get_country_name(). */ -int +STATIC int geoip_get_country_by_ipv4(uint32_t ipaddr) { geoip_ipv4_entry_t *ent; @@ -379,7 +379,7 @@ geoip_get_country_by_ipv4(uint32_t ipaddr) * 0 for the 'unknown country'. The return value will always be less than * geoip_get_n_countries(). To decode it, call geoip_get_country_name(). */ -int +STATIC int geoip_get_country_by_ipv6(const struct in6_addr *addr) { geoip_ipv6_entry_t *ent; @@ -461,6 +461,10 @@ geoip_db_digest(sa_family_t family) typedef struct clientmap_entry_t { HT_ENTRY(clientmap_entry_t) node; tor_addr_t addr; + /* Name of pluggable transport used by this client. NULL if no + pluggable transport was used. */ + char *transport_name; + /** Time when we last saw this IP address, in MINUTES since the epoch. * * (This will run out of space around 4011 CE. If Tor is still in use around @@ -482,12 +486,20 @@ static HT_HEAD(clientmap, clientmap_entry_t) client_history = static INLINE unsigned clientmap_entry_hash(const clientmap_entry_t *a) { - return ht_improve_hash(tor_addr_hash(&a->addr)); + unsigned h = (unsigned) tor_addr_hash(&a->addr); + + if (a->transport_name) + h += (unsigned) siphash24g(a->transport_name, strlen(a->transport_name)); + + return h; } /** Hashtable helper: compare two clientmap_entry_t values for equality. */ static INLINE int clientmap_entries_eq(const clientmap_entry_t *a, const clientmap_entry_t *b) { + if (strcmp_opt(a->transport_name, b->transport_name)) + return 0; + return !tor_addr_compare(&a->addr, &b->addr, CMP_EXACT) && a->action == b->action; } @@ -497,6 +509,17 @@ HT_PROTOTYPE(clientmap, clientmap_entry_t, node, clientmap_entry_hash, HT_GENERATE(clientmap, clientmap_entry_t, node, clientmap_entry_hash, clientmap_entries_eq, 0.6, malloc, realloc, free); +/** Free all storage held by <b>ent</b>. */ +static void +clientmap_entry_free(clientmap_entry_t *ent) +{ + if (!ent) + return; + + tor_free(ent->transport_name); + tor_free(ent); +} + /** Clear history of connecting clients used by entry and bridge stats. */ static void client_history_clear(void) @@ -507,7 +530,7 @@ client_history_clear(void) if ((*ent)->action == GEOIP_CLIENT_CONNECT) { this = *ent; next = HT_NEXT_RMV(clientmap, &client_history, ent); - tor_free(this); + clientmap_entry_free(this); } else { next = HT_NEXT(clientmap, &client_history, ent); } @@ -519,27 +542,40 @@ client_history_clear(void) * configured accordingly. */ void geoip_note_client_seen(geoip_client_action_t action, - const tor_addr_t *addr, time_t now) + const tor_addr_t *addr, + const char *transport_name, + time_t now) { const or_options_t *options = get_options(); clientmap_entry_t lookup, *ent; + memset(&lookup, 0, sizeof(clientmap_entry_t)); + if (action == GEOIP_CLIENT_CONNECT) { /* Only remember statistics as entry guard or as bridge. */ if (!options->EntryStatistics && (!(options->BridgeRelay && options->BridgeRecordUsageByCountry))) return; } else { - if (options->BridgeRelay || options->BridgeAuthoritativeDir || - !options->DirReqStatistics) + /* Only gather directory-request statistics if configured, and + * forcibly disable them on bridge authorities. */ + if (!options->DirReqStatistics || options->BridgeAuthoritativeDir) return; } + log_debug(LD_GENERAL, "Seen client from '%s' with transport '%s'.", + safe_str_client(fmt_addr((addr))), + transport_name ? transport_name : "<no transport>"); + tor_addr_copy(&lookup.addr, addr); lookup.action = (int)action; + lookup.transport_name = (char*) transport_name; ent = HT_FIND(clientmap, &client_history, &lookup); + if (! ent) { ent = tor_malloc_zero(sizeof(clientmap_entry_t)); tor_addr_copy(&ent->addr, addr); + if (transport_name) + ent->transport_name = tor_strdup(transport_name); ent->action = (int)action; HT_INSERT(clientmap, &client_history, ent); } @@ -566,7 +602,7 @@ remove_old_client_helper_(struct clientmap_entry_t *ent, void *_cutoff) { time_t cutoff = *(time_t*)_cutoff / 60; if (ent->last_seen_in_minutes < cutoff) { - tor_free(ent); + clientmap_entry_free(ent); return 1; } else { return 0; @@ -769,6 +805,106 @@ geoip_change_dirreq_state(uint64_t dirreq_id, dirreq_type_t type, } } +/** Return the bridge-ip-transports string that should be inserted in + * our extra-info descriptor. Return NULL if the bridge-ip-transports + * line should be empty. */ +char * +geoip_get_transport_history(void) +{ + unsigned granularity = IP_GRANULARITY; + /** String hash table (name of transport) -> (number of users). */ + strmap_t *transport_counts = strmap_new(); + + /** Smartlist that contains copies of the names of the transports + that have been used. */ + smartlist_t *transports_used = smartlist_new(); + + /* Special string to signify that no transport was used for this + connection. Pluggable transport names can't have symbols in their + names, so this string will never collide with a real transport. */ + static const char* no_transport_str = "<OR>"; + + clientmap_entry_t **ent; + const char *transport_name = NULL; + smartlist_t *string_chunks = smartlist_new(); + char *the_string = NULL; + + /* If we haven't seen any clients yet, return NULL. */ + if (HT_EMPTY(&client_history)) + goto done; + + /** We do the following steps to form the transport history string: + * a) Foreach client that uses a pluggable transport, we increase the + * times that transport was used by one. If the client did not use + * a transport, we increase the number of times someone connected + * without obfuscation. + * b) Foreach transport we observed, we write its transport history + * string and push it to string_chunks. So, for example, if we've + * seen 665 obfs2 clients, we write "obfs2=665". + * c) We concatenate string_chunks to form the final string. + */ + + log_debug(LD_GENERAL,"Starting iteration for transport history. %d clients.", + HT_SIZE(&client_history)); + + /* Loop through all clients. */ + HT_FOREACH(ent, clientmap, &client_history) { + uintptr_t val; + void *ptr; + transport_name = (*ent)->transport_name; + if (!transport_name) + transport_name = no_transport_str; + + /* Increase the count for this transport name. */ + ptr = strmap_get(transport_counts, transport_name); + val = (uintptr_t)ptr; + val++; + ptr = (void*)val; + strmap_set(transport_counts, transport_name, ptr); + + /* If it's the first time we see this transport, note it. */ + if (val == 1) + smartlist_add(transports_used, tor_strdup(transport_name)); + + log_debug(LD_GENERAL, "Client from '%s' with transport '%s'. " + "I've now seen %d clients.", + safe_str_client(fmt_addr(&(*ent)->addr)), + transport_name ? transport_name : "<no transport>", + (int)val); + } + + /* Sort the transport names (helps with unit testing). */ + smartlist_sort_strings(transports_used); + + /* Loop through all seen transports. */ + SMARTLIST_FOREACH_BEGIN(transports_used, const char *, transport_name) { + void *transport_count_ptr = strmap_get(transport_counts, transport_name); + uintptr_t transport_count = (uintptr_t) transport_count_ptr; + + log_debug(LD_GENERAL, "We got "U64_FORMAT" clients with transport '%s'.", + U64_PRINTF_ARG((uint64_t)transport_count), transport_name); + + smartlist_add_asprintf(string_chunks, "%s="U64_FORMAT, + transport_name, + U64_PRINTF_ARG(round_uint64_to_next_multiple_of( + (uint64_t)transport_count, + granularity))); + } SMARTLIST_FOREACH_END(transport_name); + + the_string = smartlist_join_strings(string_chunks, ",", 0, NULL); + + log_debug(LD_GENERAL, "Final bridge-ip-transports string: '%s'", the_string); + + done: + strmap_free(transport_counts, NULL); + SMARTLIST_FOREACH(transports_used, char *, s, tor_free(s)); + smartlist_free(transports_used); + SMARTLIST_FOREACH(string_chunks, char *, s, tor_free(s)); + smartlist_free(string_chunks); + + return the_string; +} + /** Return a newly allocated comma-separated string containing statistics * on network status downloads. The string contains the number of completed * requests, timeouts, and still running requests as well as the download @@ -1037,7 +1173,7 @@ geoip_reset_dirreq_stats(time_t now) if ((*ent)->action == GEOIP_CLIENT_NETWORKSTATUS) { this = *ent; next = HT_NEXT_RMV(clientmap, &client_history, ent); - tor_free(this); + clientmap_entry_free(this); } else { next = HT_NEXT(clientmap, &client_history, ent); } @@ -1132,7 +1268,7 @@ geoip_format_dirreq_stats(time_t now) time_t geoip_dirreq_stats_write(time_t now) { - char *statsdir = NULL, *filename = NULL, *str = NULL; + char *str = NULL; if (!start_of_dirreq_stats_interval) return 0; /* Not initialized. */ @@ -1146,21 +1282,13 @@ geoip_dirreq_stats_write(time_t now) str = geoip_format_dirreq_stats(now); /* Write dirreq-stats string to disk. */ - statsdir = get_datadir_fname("stats"); - if (check_private_dir(statsdir, CPD_CREATE, get_options()->User) < 0) { - log_warn(LD_HIST, "Unable to create stats/ directory!"); - goto done; + if (!check_or_create_data_subdir("stats")) { + write_to_data_subdir("stats", "dirreq-stats", str, "dirreq statistics"); + /* Reset measurement interval start. */ + geoip_reset_dirreq_stats(now); } - filename = get_datadir_fname2("stats", "dirreq-stats"); - if (write_str_to_file(filename, str, 0) < 0) - log_warn(LD_HIST, "Unable to write dirreq statistics to disk!"); - - /* Reset measurement interval start. */ - geoip_reset_dirreq_stats(now); done: - tor_free(statsdir); - tor_free(filename); tor_free(str); return start_of_dirreq_stats_interval + WRITE_STATS_INTERVAL; } @@ -1197,6 +1325,8 @@ validate_bridge_stats(const char *stats_str, time_t now) const char *BRIDGE_STATS_END = "bridge-stats-end "; const char *BRIDGE_IPS = "bridge-ips "; const char *BRIDGE_IPS_EMPTY_LINE = "bridge-ips\n"; + const char *BRIDGE_TRANSPORTS = "bridge-ip-transports "; + const char *BRIDGE_TRANSPORTS_EMPTY_LINE = "bridge-ip-transports\n"; const char *tmp; time_t stats_end_time; int seconds; @@ -1231,6 +1361,15 @@ validate_bridge_stats(const char *stats_str, time_t now) return 0; } + /* Parse: "bridge-ip-transports PT=N,PT=N,..." */ + tmp = find_str_at_start_of_line(stats_str, BRIDGE_TRANSPORTS); + if (!tmp) { + /* Look if there is an empty "bridge-ip-transports" line */ + tmp = find_str_at_start_of_line(stats_str, BRIDGE_TRANSPORTS_EMPTY_LINE); + if (!tmp) + return 0; + } + return 1; } @@ -1244,7 +1383,8 @@ static char *bridge_stats_extrainfo = NULL; char * geoip_format_bridge_stats(time_t now) { - char *out = NULL, *country_data = NULL, *ipver_data = NULL; + char *out = NULL; + char *country_data = NULL, *ipver_data = NULL, *transport_data = NULL; long duration = now - start_of_bridge_stats_interval; char written[ISO_TIME_LEN+1]; @@ -1255,16 +1395,20 @@ geoip_format_bridge_stats(time_t now) format_iso_time(written, now); geoip_get_client_history(GEOIP_CLIENT_CONNECT, &country_data, &ipver_data); + transport_data = geoip_get_transport_history(); tor_asprintf(&out, "bridge-stats-end %s (%ld s)\n" "bridge-ips %s\n" - "bridge-ip-versions %s\n", + "bridge-ip-versions %s\n" + "bridge-ip-transports %s\n", written, duration, country_data ? country_data : "", - ipver_data ? ipver_data : ""); + ipver_data ? ipver_data : "", + transport_data ? transport_data : ""); tor_free(country_data); tor_free(ipver_data); + tor_free(transport_data); return out; } @@ -1297,7 +1441,7 @@ format_bridge_stats_controller(time_t now) time_t geoip_bridge_stats_write(time_t now) { - char *filename = NULL, *val = NULL, *statsdir = NULL; + char *val = NULL; /* Check if 24 hours have passed since starting measurements. */ if (now < start_of_bridge_stats_interval + WRITE_STATS_INTERVAL) @@ -1317,24 +1461,20 @@ geoip_bridge_stats_write(time_t now) start_of_bridge_stats_interval = now; /* Write it to disk. */ - statsdir = get_datadir_fname("stats"); - if (check_private_dir(statsdir, CPD_CREATE, get_options()->User) < 0) - goto done; - filename = get_datadir_fname2("stats", "bridge-stats"); - - write_str_to_file(filename, bridge_stats_extrainfo, 0); - - /* Tell the controller, "hey, there are clients!" */ - { - char *controller_str = format_bridge_stats_controller(now); - if (controller_str) - control_event_clients_seen(controller_str); - tor_free(controller_str); + if (!check_or_create_data_subdir("stats")) { + write_to_data_subdir("stats", "bridge-stats", + bridge_stats_extrainfo, "bridge statistics"); + + /* Tell the controller, "hey, there are clients!" */ + { + char *controller_str = format_bridge_stats_controller(now); + if (controller_str) + control_event_clients_seen(controller_str); + tor_free(controller_str); + } } - done: - tor_free(filename); - tor_free(statsdir); + done: return start_of_bridge_stats_interval + WRITE_STATS_INTERVAL; } @@ -1436,7 +1576,7 @@ geoip_format_entry_stats(time_t now) time_t geoip_entry_stats_write(time_t now) { - char *statsdir = NULL, *filename = NULL, *str = NULL; + char *str = NULL; if (!start_of_entry_stats_interval) return 0; /* Not initialized. */ @@ -1450,21 +1590,14 @@ geoip_entry_stats_write(time_t now) str = geoip_format_entry_stats(now); /* Write entry-stats string to disk. */ - statsdir = get_datadir_fname("stats"); - if (check_private_dir(statsdir, CPD_CREATE, get_options()->User) < 0) { - log_warn(LD_HIST, "Unable to create stats/ directory!"); - goto done; - } - filename = get_datadir_fname2("stats", "entry-stats"); - if (write_str_to_file(filename, str, 0) < 0) - log_warn(LD_HIST, "Unable to write entry statistics to disk!"); + if (!check_or_create_data_subdir("stats")) { + write_to_data_subdir("stats", "entry-stats", str, "entry statistics"); - /* Reset measurement interval start. */ - geoip_reset_entry_stats(now); + /* Reset measurement interval start. */ + geoip_reset_entry_stats(now); + } done: - tor_free(statsdir); - tor_free(filename); tor_free(str); return start_of_entry_stats_interval + WRITE_STATS_INTERVAL; } @@ -1534,7 +1667,7 @@ geoip_free_all(void) for (ent = HT_START(clientmap, &client_history); ent != NULL; ent = next) { this = *ent; next = HT_NEXT_RMV(clientmap, &client_history, ent); - tor_free(this); + clientmap_entry_free(this); } HT_CLEAR(clientmap, &client_history); } @@ -1549,5 +1682,6 @@ geoip_free_all(void) } clear_geoip_db(); + tor_free(bridge_stats_extrainfo); } diff --git a/src/or/geoip.h b/src/or/geoip.h index ebefee5f4e..b9b53c3006 100644 --- a/src/or/geoip.h +++ b/src/or/geoip.h @@ -12,10 +12,12 @@ #ifndef TOR_GEOIP_H #define TOR_GEOIP_H +#include "testsupport.h" + #ifdef GEOIP_PRIVATE -int geoip_parse_entry(const char *line, sa_family_t family); -int geoip_get_country_by_ipv4(uint32_t ipaddr); -int geoip_get_country_by_ipv6(const struct in6_addr *addr); +STATIC int geoip_parse_entry(const char *line, sa_family_t family); +STATIC int geoip_get_country_by_ipv4(uint32_t ipaddr); +STATIC int geoip_get_country_by_ipv6(const struct in6_addr *addr); #endif int should_record_bridge_info(const or_options_t *options); int geoip_load_file(sa_family_t family, const char *filename); @@ -27,10 +29,12 @@ const char *geoip_db_digest(sa_family_t family); country_t geoip_get_country(const char *countrycode); void geoip_note_client_seen(geoip_client_action_t action, - const tor_addr_t *addr, time_t now); + const tor_addr_t *addr, const char *transport_name, + time_t now); void geoip_remove_old_clients(time_t cutoff); void geoip_note_ns_response(geoip_ns_response_t response); +char *geoip_get_transport_history(void); int geoip_get_client_history(geoip_client_action_t action, char **country_str, char **ipver_str); char *geoip_get_request_history(void); diff --git a/src/or/hibernate.c b/src/or/hibernate.c index a412571331..c433ac1be9 100644 --- a/src/or/hibernate.c +++ b/src/or/hibernate.c @@ -239,8 +239,8 @@ accounting_parse_options(const or_options_t *options, int validate_only) /** If we want to manage the accounting system and potentially * hibernate, return 1, else return 0. */ -int -accounting_is_enabled(const or_options_t *options) +MOCK_IMPL(int, +accounting_is_enabled,(const or_options_t *options)) { if (options->AccountingMax) return 1; @@ -255,6 +255,13 @@ accounting_get_interval_length(void) return (int)(interval_end_time - interval_start_time); } +/** Return the time at which the current accounting interval will end. */ +MOCK_IMPL(time_t, +accounting_get_end_time,(void)) +{ + return interval_end_time; +} + /** Called from main.c to tell us that <b>seconds</b> seconds have * passed, <b>n_read</b> bytes have been read, and <b>n_written</b> * bytes have been written. */ @@ -641,7 +648,15 @@ read_bandwidth_usage(void) { char *fname = get_datadir_fname("bw_accounting"); - unlink(fname); + int res; + + res = unlink(fname); + if (res != 0) { + log_warn(LD_FS, + "Failed to unlink %s: %s", + fname, strerror(errno)); + } + tor_free(fname); } @@ -808,8 +823,8 @@ hibernate_begin_shutdown(void) } /** Return true iff we are currently hibernating. */ -int -we_are_hibernating(void) +MOCK_IMPL(int, +we_are_hibernating,(void)) { return hibernate_state != HIBERNATE_STATE_LIVE; } @@ -1010,6 +1025,7 @@ getinfo_helper_accounting(control_connection_t *conn, return 0; } +#ifdef TOR_UNIT_TESTS /** * Manually change the hibernation state. Private; used only by the unit * tests. @@ -1019,4 +1035,5 @@ hibernate_set_state_for_testing_(hibernate_state_t newstate) { hibernate_state = newstate; } +#endif diff --git a/src/or/hibernate.h b/src/or/hibernate.h index d2d6989e10..38ecb75129 100644 --- a/src/or/hibernate.h +++ b/src/or/hibernate.h @@ -12,15 +12,18 @@ #ifndef TOR_HIBERNATE_H #define TOR_HIBERNATE_H +#include "testsupport.h" + int accounting_parse_options(const or_options_t *options, int validate_only); -int accounting_is_enabled(const or_options_t *options); +MOCK_DECL(int, accounting_is_enabled, (const or_options_t *options)); int accounting_get_interval_length(void); +MOCK_DECL(time_t, accounting_get_end_time, (void)); void configure_accounting(time_t now); void accounting_run_housekeeping(time_t now); void accounting_add_bytes(size_t n_read, size_t n_written, int seconds); int accounting_record_bandwidth_usage(time_t now, or_state_t *state); void hibernate_begin_shutdown(void); -int we_are_hibernating(void); +MOCK_DECL(int, we_are_hibernating, (void)); void consider_hibernation(time_t now); int getinfo_helper_accounting(control_connection_t *conn, const char *question, char **answer, @@ -45,8 +48,10 @@ typedef enum { HIBERNATE_STATE_INITIAL=5 } hibernate_state_t; +#ifdef TOR_UNIT_TESTS void hibernate_set_state_for_testing_(hibernate_state_t newstate); #endif +#endif #endif diff --git a/src/or/include.am b/src/or/include.am index 65dbeff53e..47bdd09901 100644 --- a/src/or/include.am +++ b/src/or/include.am @@ -1,5 +1,13 @@ bin_PROGRAMS+= src/or/tor -noinst_LIBRARIES+= src/or/libtor.a +noinst_LIBRARIES += \ + src/or/libtor.a +if UNITTESTS_ENABLED +noinst_LIBRARIES += \ + src/or/libtor-testing.a +endif +if COVERAGE_ENABLED +noinst_PROGRAMS+= src/or/tor-cov +endif if BUILD_NT_SERVICES tor_platform_source=src/or/ntmain.c @@ -21,11 +29,12 @@ else onion_ntor_source= endif -src_or_libtor_a_SOURCES = \ +LIBTOR_A_SOURCES = \ src/or/addressmap.c \ src/or/buffers.c \ src/or/channel.c \ src/or/channeltls.c \ + src/or/circpathbias.c \ src/or/circuitbuild.c \ src/or/circuitlist.c \ src/or/circuitmux.c \ @@ -48,6 +57,7 @@ src_or_libtor_a_SOURCES = \ src/or/fp_pair.c \ src/or/geoip.c \ src/or/entrynodes.c \ + src/or/ext_orport.c \ src/or/hibernate.c \ src/or/main.c \ src/or/microdesc.c \ @@ -77,6 +87,9 @@ src_or_libtor_a_SOURCES = \ $(onion_ntor_source) \ src/or/config_codedigest.c +src_or_libtor_a_SOURCES = $(LIBTOR_A_SOURCES) +src_or_libtor_testing_a_SOURCES = $(LIBTOR_A_SOURCES) + #libtor_a_LIBADD = ../common/libor.a ../common/libor-crypto.a \ # ../common/libor-event.a @@ -90,6 +103,9 @@ AM_CPPFLAGS += -DSHARE_DATADIR="\"$(datadir)\"" \ -DLOCALSTATEDIR="\"$(localstatedir)\"" \ -DBINDIR="\"$(bindir)\"" +src_or_libtor_testing_a_CPPFLAGS = -DTOR_UNIT_TESTS $(AM_CPPFLAGS) +src_or_libtor_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) + # -L flags need to go in LDFLAGS. -l flags need to go in LDADD. # This seems to matter nowhere but on windows, but I assure you that it # matters a lot there, and is quite hard to debug if you forget to do it. @@ -102,11 +118,24 @@ src_or_tor_LDADD = src/or/libtor.a src/common/libor.a \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ \ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ +if COVERAGE_ENABLED +src_or_tor_cov_SOURCES = src/or/tor_main.c +src_or_tor_cov_CPPFLAGS = -DTOR_UNIT_TESTS $(AM_CPPFLAGS) +src_or_tor_cov_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +src_or_tor_cov_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ @TOR_LDFLAGS_libevent@ +src_or_tor_cov_LDADD = src/or/libtor-testing.a src/common/libor-testing.a \ + src/common/libor-crypto-testing.a $(LIBDONNA) \ + src/common/libor-event-testing.a \ + @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ \ + @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ +endif + ORHEADERS = \ src/or/addressmap.h \ src/or/buffers.h \ src/or/channel.h \ src/or/channeltls.h \ + src/or/circpathbias.h \ src/or/circuitbuild.h \ src/or/circuitlist.h \ src/or/circuitmux.h \ @@ -127,6 +156,7 @@ ORHEADERS = \ src/or/dns.h \ src/or/dnsserv.h \ src/or/eventdns_tor.h \ + src/or/ext_orport.h \ src/or/fp_pair.h \ src/or/geoip.h \ src/or/entrynodes.h \ diff --git a/src/or/main.c b/src/or/main.c index bd23141b97..9c1cabf037 100644 --- a/src/or/main.c +++ b/src/or/main.c @@ -13,6 +13,7 @@ #define MAIN_PRIVATE #include "or.h" #include "addressmap.h" +#include "backtrace.h" #include "buffers.h" #include "channel.h" #include "channeltls.h" @@ -21,6 +22,7 @@ #include "circuituse.h" #include "command.h" #include "config.h" +#include "confparse.h" #include "connection.h" #include "connection_edge.h" #include "connection_or.h" @@ -52,11 +54,14 @@ #include "routerparse.h" #include "statefile.h" #include "status.h" +#include "util_process.h" +#include "ext_orport.h" #ifdef USE_DMALLOC #include <dmalloc.h> #include <openssl/crypto.h> #endif #include "memarea.h" +#include "../common/sandbox.h" #ifdef HAVE_EVENT2_EVENT_H #include <event2/event.h> @@ -155,8 +160,6 @@ int can_complete_circuit=0; /** How often do we 'forgive' undownloadable router descriptors and attempt * to download them again? */ #define DESCRIPTOR_FAILURE_RESET_INTERVAL (60*60) -/** How long do we let a directory connection stall before expiring it? */ -#define DIR_CONN_MAX_STALL (5*60) /** Decides our behavior when no logs are configured/before any * logs have been configured. For 0, we log notice to stdout as normal. @@ -351,6 +354,8 @@ connection_remove(connection_t *conn) (int)conn->s, conn_type_to_string(conn->type), smartlist_len(connection_array)); + control_event_conn_bandwidth(conn); + tor_assert(conn->conn_array_index >= 0); current_index = conn->conn_array_index; connection_unregister_events(conn); /* This is redundant, but cheap. */ @@ -414,6 +419,19 @@ connection_unlink(connection_t *conn) connection_free(conn); } +/** Initialize the global connection list, closeable connection list, + * and active connection list. */ +STATIC void +init_connection_lists(void) +{ + if (!connection_array) + connection_array = smartlist_new(); + if (!closeable_connection_lst) + closeable_connection_lst = smartlist_new(); + if (!active_linked_connection_lst) + active_linked_connection_lst = smartlist_new(); +} + /** Schedule <b>conn</b> to be closed. **/ void add_connection_to_closeable_list(connection_t *conn) @@ -452,15 +470,15 @@ get_connection_array(void) /** Provides the traffic read and written over the life of the process. */ -uint64_t -get_bytes_read(void) +MOCK_IMPL(uint64_t, +get_bytes_read,(void)) { return stats_n_bytes_read; } /* DOCDOC get_bytes_written */ -uint64_t -get_bytes_written(void) +MOCK_IMPL(uint64_t, +get_bytes_written,(void)) { return stats_n_bytes_written; } @@ -507,8 +525,8 @@ connection_is_reading(connection_t *conn) } /** Tell the main loop to stop notifying <b>conn</b> of any read events. */ -void -connection_stop_reading(connection_t *conn) +MOCK_IMPL(void, +connection_stop_reading,(connection_t *conn)) { tor_assert(conn); @@ -532,8 +550,8 @@ connection_stop_reading(connection_t *conn) } /** Tell the main loop to start notifying <b>conn</b> of any read events. */ -void -connection_start_reading(connection_t *conn) +MOCK_IMPL(void, +connection_start_reading,(connection_t *conn)) { tor_assert(conn); @@ -572,8 +590,8 @@ connection_is_writing(connection_t *conn) } /** Tell the main loop to stop notifying <b>conn</b> of any write events. */ -void -connection_stop_writing(connection_t *conn) +MOCK_IMPL(void, +connection_stop_writing,(connection_t *conn)) { tor_assert(conn); @@ -598,8 +616,8 @@ connection_stop_writing(connection_t *conn) } /** Tell the main loop to start notifying <b>conn</b> of any write events. */ -void -connection_start_writing(connection_t *conn) +MOCK_IMPL(void, +connection_start_writing,(connection_t *conn)) { tor_assert(conn); @@ -687,7 +705,7 @@ connection_stop_reading_from_linked_conn(connection_t *conn) } /** Close all connections that have been scheduled to get closed. */ -static void +STATIC void close_closeable_connections(void) { int i; @@ -902,16 +920,7 @@ conn_close_if_marked(int i) return 0; } if (connection_wants_to_flush(conn)) { - int severity; - if (conn->type == CONN_TYPE_EXIT || - (conn->type == CONN_TYPE_OR && server_mode(get_options())) || - (conn->type == CONN_TYPE_DIR && conn->purpose == DIR_PURPOSE_SERVER)) - severity = LOG_INFO; - else - severity = LOG_NOTICE; - /* XXXX Maybe allow this to happen a certain amount per hour; it usually - * is meaningless. */ - log_fn(severity, LD_NET, "We stalled too much while trying to write %d " + log_fn(LOG_INFO, LD_NET, "We stalled too much while trying to write %d " "bytes to address %s. If this happens a lot, either " "something is wrong with your network connection, or " "something is wrong with theirs. " @@ -993,15 +1002,6 @@ directory_info_has_arrived(time_t now, int from_cache) consider_testing_reachability(1, 1); } -/** How long do we wait before killing OR connections with no circuits? - * In Tor versions up to 0.2.1.25 and 0.2.2.12-alpha, we waited 15 minutes - * before cancelling these connections, which caused fast relays to accrue - * many many idle connections. Hopefully 3 minutes is low enough that - * it kills most idle connections, without being so low that we cause - * clients to bounce on and off. - */ -#define IDLE_OR_CONN_TIMEOUT 180 - /** Perform regular maintenance tasks for a single connection. This * function gets run once per second per connection by run_scheduled_events. */ @@ -1012,6 +1012,8 @@ run_connection_housekeeping(int i, time_t now) connection_t *conn = smartlist_get(connection_array, i); const or_options_t *options = get_options(); or_connection_t *or_conn; + channel_t *chan = NULL; + int have_any_circuits; int past_keepalive = now >= conn->timestamp_lastwritten + options->KeepalivePeriod; @@ -1028,9 +1030,11 @@ run_connection_housekeeping(int i, time_t now) * if a server or received if a client) for 5 min */ if (conn->type == CONN_TYPE_DIR && ((DIR_CONN_IS_SERVER(conn) && - conn->timestamp_lastwritten + DIR_CONN_MAX_STALL < now) || + conn->timestamp_lastwritten + + options->TestingDirConnectionMaxStall < now) || (!DIR_CONN_IS_SERVER(conn) && - conn->timestamp_lastread + DIR_CONN_MAX_STALL < now))) { + conn->timestamp_lastread + + options->TestingDirConnectionMaxStall < now))) { log_info(LD_DIR,"Expiring wedged directory conn (fd %d, purpose %d)", (int)conn->s, conn->purpose); /* This check is temporary; it's to let us know whether we should consider @@ -1059,8 +1063,18 @@ run_connection_housekeeping(int i, time_t now) tor_assert(conn->outbuf); #endif + chan = TLS_CHAN_TO_BASE(or_conn->chan); + tor_assert(chan); + + if (channel_num_circuits(chan) != 0) { + have_any_circuits = 1; + chan->timestamp_last_had_circuits = now; + } else { + have_any_circuits = 0; + } + if (channel_is_bad_for_new_circs(TLS_CHAN_TO_BASE(or_conn->chan)) && - !connection_or_get_num_circuits(or_conn)) { + ! have_any_circuits) { /* It's bad for new circuits, and has no unmarked circuits on it: * mark it now. */ log_info(LD_OR, @@ -1079,19 +1093,22 @@ run_connection_housekeeping(int i, time_t now) connection_or_close_normally(TO_OR_CONN(conn), 0); } } else if (we_are_hibernating() && - !connection_or_get_num_circuits(or_conn) && + ! have_any_circuits && !connection_get_outbuf_len(conn)) { /* We're hibernating, there's no circuits, and nothing to flush.*/ log_info(LD_OR,"Expiring non-used OR connection to fd %d (%s:%d) " "[Hibernating or exiting].", (int)conn->s,conn->address, conn->port); connection_or_close_normally(TO_OR_CONN(conn), 1); - } else if (!connection_or_get_num_circuits(or_conn) && - now >= or_conn->timestamp_last_added_nonpadding + - IDLE_OR_CONN_TIMEOUT) { + } else if (!have_any_circuits && + now - or_conn->idle_timeout >= + chan->timestamp_last_had_circuits) { log_info(LD_OR,"Expiring non-used OR connection to fd %d (%s:%d) " - "[idle %d].", (int)conn->s,conn->address, conn->port, - (int)(now - or_conn->timestamp_last_added_nonpadding)); + "[no circuits for %d; timeout %d; %scanonical].", + (int)conn->s, conn->address, conn->port, + (int)(now - chan->timestamp_last_had_circuits), + or_conn->idle_timeout, + or_conn->is_canonical ? "" : "non"); connection_or_close_normally(TO_OR_CONN(conn), 0); } else if ( now >= or_conn->timestamp_lastempty + options->KeepalivePeriod*10 && @@ -1143,6 +1160,18 @@ get_signewnym_epoch(void) return newnym_epoch; } +static time_t time_to_check_descriptor = 0; +/** + * Update our schedule so that we'll check whether we need to update our + * descriptor immediately, rather than after up to CHECK_DESCRIPTOR_INTERVAL + * seconds. + */ +void +reschedule_descriptor_update_check(void) +{ + time_to_check_descriptor = 0; +} + /** Perform regular maintenance tasks. This function gets run once per * second by second_elapsed_callback(). */ @@ -1152,7 +1181,7 @@ run_scheduled_events(time_t now) static time_t last_rotated_x509_certificate = 0; static time_t time_to_check_v3_certificate = 0; static time_t time_to_check_listeners = 0; - static time_t time_to_check_descriptor = 0; + static time_t time_to_download_networkstatus = 0; static time_t time_to_shrink_memory = 0; static time_t time_to_try_getting_descriptors = 0; static time_t time_to_reset_descriptor_failures = 0; @@ -1176,22 +1205,12 @@ run_scheduled_events(time_t now) int i; int have_dir_info; - /** 0. See if we've been asked to shut down and our timeout has + /* 0. See if we've been asked to shut down and our timeout has * expired; or if our bandwidth limits are exhausted and we * should hibernate; or if it's time to wake up from hibernation. */ consider_hibernation(now); -#if 0 - { - static time_t nl_check_time = 0; - if (nl_check_time <= now) { - nodelist_assert_ok(); - nl_check_time = now + 30; - } - } -#endif - /* 0b. If we've deferred a signewnym, make sure it gets handled * eventually. */ if (signewnym_is_pending && @@ -1203,7 +1222,7 @@ run_scheduled_events(time_t now) /* 0c. If we've deferred log messages for the controller, handle them now */ flush_pending_log_callbacks(); - /** 1a. Every MIN_ONION_KEY_LIFETIME seconds, rotate the onion keys, + /* 1a. Every MIN_ONION_KEY_LIFETIME seconds, rotate the onion keys, * shut down and restart all cpuworkers, and update the directory if * necessary. */ @@ -1219,7 +1238,8 @@ run_scheduled_events(time_t now) router_upload_dir_desc_to_dirservers(0); } - if (!options->DisableNetwork && time_to_try_getting_descriptors < now) { + if (!should_delay_dir_fetches(options, NULL) && + time_to_try_getting_descriptors < now) { update_all_descriptor_downloads(now); update_extrainfo_downloads(now); if (router_have_minimum_dir_info()) @@ -1234,10 +1254,10 @@ run_scheduled_events(time_t now) now + DESCRIPTOR_FAILURE_RESET_INTERVAL; } - if (options->UseBridges) + if (options->UseBridges && !options->DisableNetwork) fetch_bridge_descriptors(options, now); - /** 1b. Every MAX_SSL_KEY_LIFETIME_INTERNAL seconds, we change our + /* 1b. Every MAX_SSL_KEY_LIFETIME_INTERNAL seconds, we change our * TLS context. */ if (!last_rotated_x509_certificate) last_rotated_x509_certificate = now; @@ -1263,7 +1283,7 @@ run_scheduled_events(time_t now) time_to_add_entropy = now + ENTROPY_INTERVAL; } - /** 1c. If we have to change the accounting interval or record + /* 1c. If we have to change the accounting interval or record * bandwidth used in this accounting interval, do so. */ if (accounting_is_enabled(options)) accounting_run_housekeeping(now); @@ -1276,7 +1296,7 @@ run_scheduled_events(time_t now) dirserv_test_reachability(now); } - /** 1d. Periodically, we discount older stability information so that new + /* 1d. Periodically, we discount older stability information so that new * stability info counts more, and save the stability information to disk as * appropriate. */ if (time_to_downrate_stability < now) @@ -1395,7 +1415,7 @@ run_scheduled_events(time_t now) dns_init(); } - /** 2. Periodically, we consider force-uploading our descriptor + /* 2. Periodically, we consider force-uploading our descriptor * (if we've passed our internal checks). */ /** How often do we check whether part of our router info has changed in a @@ -1438,22 +1458,29 @@ run_scheduled_events(time_t now) /* If any networkstatus documents are no longer recent, we need to * update all the descriptors' running status. */ - /* purge obsolete entries */ - networkstatus_v2_list_clean(now); /* Remove dead routers. */ routerlist_remove_old_routers(); + } - /* Also, once per minute, check whether we want to download any - * networkstatus documents. - */ + /* 2c. Every minute (or every second if TestingTorNetwork), check + * whether we want to download any networkstatus documents. */ + +/* How often do we check whether we should download network status + * documents? */ +#define networkstatus_dl_check_interval(o) ((o)->TestingTorNetwork ? 1 : 60) + + if (!should_delay_dir_fetches(options, NULL) && + time_to_download_networkstatus < now) { + time_to_download_networkstatus = + now + networkstatus_dl_check_interval(options); update_networkstatus_downloads(now); } - /** 2c. Let directory voting happen. */ + /* 2c. Let directory voting happen. */ if (authdir_mode_v3(options)) dirvote_act(options, now); - /** 3a. Every second, we examine pending circuits and prune the + /* 3a. Every second, we examine pending circuits and prune the * ones which have been pending for more than a few seconds. * We do this before step 4, so it can try building more if * it's not comfortable with the number of available circuits. @@ -1462,37 +1489,40 @@ run_scheduled_events(time_t now) * it can't, currently), we should do this more often.) */ circuit_expire_building(); - /** 3b. Also look at pending streams and prune the ones that 'began' + /* 3b. Also look at pending streams and prune the ones that 'began' * a long time ago but haven't gotten a 'connected' yet. * Do this before step 4, so we can put them back into pending * state to be picked up by the new circuit. */ connection_ap_expire_beginning(); - /** 3c. And expire connections that we've held open for too long. + /* 3c. And expire connections that we've held open for too long. */ connection_expire_held_open(); - /** 3d. And every 60 seconds, we relaunch listeners if any died. */ + /* 3d. And every 60 seconds, we relaunch listeners if any died. */ if (!net_is_disabled() && time_to_check_listeners < now) { retry_all_listeners(NULL, NULL, 0); time_to_check_listeners = now+60; } - /** 4. Every second, we try a new circuit if there are no valid + /* 4. Every second, we try a new circuit if there are no valid * circuits. Every NewCircuitPeriod seconds, we expire circuits * that became dirty more than MaxCircuitDirtiness seconds ago, * and we make a new circ if there are no clean circuits. */ have_dir_info = router_have_minimum_dir_info(); - if (have_dir_info && !net_is_disabled()) + if (have_dir_info && !net_is_disabled()) { circuit_build_needed_circs(now); + } else { + circuit_expire_old_circs_as_needed(now); + } /* every 10 seconds, but not at the same second as other such events */ if (now % 10 == 5) circuit_expire_old_circuits_serverside(now); - /** 5. We do housekeeping for each connection... */ + /* 5. We do housekeeping for each connection... */ connection_or_set_bad_connections(NULL, 0); for (i=0;i<smartlist_len(connection_array);i++) { run_connection_housekeeping(i, now); @@ -1504,7 +1534,9 @@ run_scheduled_events(time_t now) if (conn->inbuf) buf_shrink(conn->inbuf); }); +#ifdef ENABLE_MEMPOOL clean_cell_pool(); +#endif /* ENABLE_MEMPOOL */ buf_shrink_freelists(0); /** How often do we check buffers and pools for empty space that can be * deallocated? */ @@ -1512,33 +1544,35 @@ run_scheduled_events(time_t now) time_to_shrink_memory = now + MEM_SHRINK_INTERVAL; } - /** 6. And remove any marked circuits... */ + /* 6. And remove any marked circuits... */ circuit_close_all_marked(); - /** 7. And upload service descriptors if necessary. */ + /* 7. And upload service descriptors if necessary. */ if (can_complete_circuit && !net_is_disabled()) { rend_consider_services_upload(now); rend_consider_descriptor_republication(); } - /** 8. and blow away any connections that need to die. have to do this now, + /* 8. and blow away any connections that need to die. have to do this now, * because if we marked a conn for close and left its socket -1, then * we'll pass it to poll/select and bad things will happen. */ close_closeable_connections(); - /** 8b. And if anything in our state is ready to get flushed to disk, we + /* 8b. And if anything in our state is ready to get flushed to disk, we * flush it. */ or_state_save(now); - /** 8c. Do channel cleanup just like for connections */ + /* 8c. Do channel cleanup just like for connections */ channel_run_cleanup(); channel_listener_run_cleanup(); - /** 9. and if we're a server, check whether our DNS is telling stories to - * us. */ + /* 9. and if we're an exit node, check whether our DNS is telling stories + * to us. */ if (!net_is_disabled() && - public_server_mode(options) && time_to_check_for_correct_dns < now) { + public_server_mode(options) && + time_to_check_for_correct_dns < now && + ! router_my_exit_policy_is_reject_star()) { if (!time_to_check_for_correct_dns) { time_to_check_for_correct_dns = now + 60 + crypto_rand_int(120); } else { @@ -1548,7 +1582,7 @@ run_scheduled_events(time_t now) } } - /** 10. write bridge networkstatus file to disk */ + /* 10. write bridge networkstatus file to disk */ if (options->BridgeAuthoritativeDir && time_to_write_bridge_status_file < now) { networkstatus_dump_bridge_status_to_file(now); @@ -1556,7 +1590,7 @@ run_scheduled_events(time_t now) time_to_write_bridge_status_file = now+BRIDGE_STATUSFILE_INTERVAL; } - /** 11. check the port forwarding app */ + /* 11. check the port forwarding app */ if (!net_is_disabled() && time_to_check_port_forwarding < now && options->PortForwarding && @@ -1574,11 +1608,11 @@ run_scheduled_events(time_t now) time_to_check_port_forwarding = now+PORT_FORWARDING_CHECK_INTERVAL; } - /** 11b. check pending unconfigured managed proxies */ + /* 11b. check pending unconfigured managed proxies */ if (!net_is_disabled() && pt_proxies_configuration_pending()) pt_configure_remaining_proxies(); - /** 12. write the heartbeat message */ + /* 12. write the heartbeat message */ if (options->HeartbeatPeriod && time_to_next_heartbeat <= now) { if (time_to_next_heartbeat) /* don't log the first heartbeat */ @@ -1638,6 +1672,9 @@ second_elapsed_callback(periodic_timer_t *timer, void *arg) control_event_bandwidth_used((uint32_t)bytes_read,(uint32_t)bytes_written); control_event_stream_bandwidth_used(); + control_event_conn_bandwidth_used(); + control_event_circ_bandwidth_used(); + control_event_circuit_cell_stats(); if (server_mode(options) && !net_is_disabled() && @@ -1649,24 +1686,28 @@ second_elapsed_callback(periodic_timer_t *timer, void *arg) /* every 20 minutes, check and complain if necessary */ const routerinfo_t *me = router_get_my_routerinfo(); if (me && !check_whether_orport_reachable()) { + char *address = tor_dup_ip(me->addr); log_warn(LD_CONFIG,"Your server (%s:%d) has not managed to confirm that " "its ORPort is reachable. Please check your firewalls, ports, " "address, /etc/hosts file, etc.", - me->address, me->or_port); + address, me->or_port); control_event_server_status(LOG_WARN, "REACHABILITY_FAILED ORADDRESS=%s:%d", - me->address, me->or_port); + address, me->or_port); + tor_free(address); } if (me && !check_whether_dirport_reachable()) { + char *address = tor_dup_ip(me->addr); log_warn(LD_CONFIG, "Your server (%s:%d) has not managed to confirm that its " "DirPort is reachable. Please check your firewalls, ports, " "address, /etc/hosts file, etc.", - me->address, me->dir_port); + address, me->dir_port); control_event_server_status(LOG_WARN, "REACHABILITY_FAILED DIRADDRESS=%s:%d", - me->address, me->dir_port); + address, me->dir_port); + tor_free(address); } } @@ -1728,7 +1769,7 @@ refill_callback(periodic_timer_t *timer, void *arg) accounting_add_bytes(bytes_read, bytes_written, seconds_rolled_over); if (milliseconds_elapsed > 0) - connection_bucket_refill(milliseconds_elapsed, now.tv_sec); + connection_bucket_refill(milliseconds_elapsed, (time_t)now.tv_sec); stats_prev_global_read_bucket = global_read_bucket; stats_prev_global_write_bucket = global_write_bucket; @@ -1865,7 +1906,7 @@ do_hup(void) } /** Tor main loop. */ -/* static */ int +int do_main_loop(void) { int loop_result; @@ -1900,8 +1941,10 @@ do_main_loop(void) } } +#ifdef ENABLE_MEMPOOLS /* Set up the packed_cell_t memory pool. */ init_cell_pool(); +#endif /* ENABLE_MEMPOOLS */ /* Set up our buckets */ connection_bucket_init(); @@ -1917,9 +1960,6 @@ do_main_loop(void) log_warn(LD_DIR, "Couldn't load all cached v3 certificates. Starting anyway."); } - if (router_reload_v2_networkstatus()) { - return -1; - } if (router_reload_consensus_networkstatus()) { return -1; } @@ -2073,8 +2113,7 @@ process_signal(uintptr_t sig) break; #ifdef SIGCHLD case SIGCHLD: - while (waitpid(-1,NULL,WNOHANG) > 0) ; /* keep reaping until no more - zombies */ + notify_pending_waitpid_callbacks(); break; #endif case SIGNEWNYM: { @@ -2097,8 +2136,8 @@ process_signal(uintptr_t sig) } /** Returns Tor's uptime. */ -long -get_uptime(void) +MOCK_IMPL(long, +get_uptime,(void)) { return stats_n_seconds_working; } @@ -2292,21 +2331,24 @@ handle_signals(int is_parent) /** Main entry point for the Tor command-line client. */ -/* static */ int +int tor_init(int argc, char *argv[]) { - char buf[256]; - int i, quiet = 0; + char progname[256]; + int quiet = 0; + time_of_process_start = time(NULL); - if (!connection_array) - connection_array = smartlist_new(); - if (!closeable_connection_lst) - closeable_connection_lst = smartlist_new(); - if (!active_linked_connection_lst) - active_linked_connection_lst = smartlist_new(); + init_connection_lists(); /* Have the log set up with our application name. */ - tor_snprintf(buf, sizeof(buf), "Tor %s", get_version()); - log_set_application_name(buf); + tor_snprintf(progname, sizeof(progname), "Tor %s", get_version()); + log_set_application_name(progname); + + /* Set up the crypto nice and early */ + if (crypto_early_init() < 0) { + log_err(LD_GENERAL, "Unable to initialize the crypto subsystem!"); + return 1; + } + /* Initialize the history structures. */ rep_hist_init(); /* Initialize the service cache. */ @@ -2314,17 +2356,31 @@ tor_init(int argc, char *argv[]) addressmap_init(); /* Init the client dns cache. Do it always, since it's * cheap. */ + { /* We search for the "quiet" option first, since it decides whether we * will log anything at all to the command line. */ - for (i=1;i<argc;++i) { - if (!strcmp(argv[i], "--hush")) - quiet = 1; - if (!strcmp(argv[i], "--quiet")) - quiet = 2; - /* --version implies --quiet */ - if (!strcmp(argv[i], "--version")) - quiet = 2; + config_line_t *opts = NULL, *cmdline_opts = NULL; + const config_line_t *cl; + (void) config_parse_commandline(argc, argv, 1, &opts, &cmdline_opts); + for (cl = cmdline_opts; cl; cl = cl->next) { + if (!strcmp(cl->key, "--hush")) + quiet = 1; + if (!strcmp(cl->key, "--quiet") || + !strcmp(cl->key, "--dump-config")) + quiet = 2; + /* --version, --digests, and --help imply --hush */ + if (!strcmp(cl->key, "--version") || !strcmp(cl->key, "--digests") || + !strcmp(cl->key, "--list-torrc-options") || + !strcmp(cl->key, "--library-versions") || + !strcmp(cl->key, "-h") || !strcmp(cl->key, "--help")) { + if (quiet < 1) + quiet = 1; + } + } + config_free_lines(opts); + config_free_lines(cmdline_opts); } + /* give it somewhere to log to initially */ switch (quiet) { case 2: @@ -2346,11 +2402,12 @@ tor_init(int argc, char *argv[]) #else ""; #endif - log_notice(LD_GENERAL, "Tor v%s %srunning on %s with Libevent %s " - "and OpenSSL %s.", version, bev_str, + log_notice(LD_GENERAL, "Tor v%s %srunning on %s with Libevent %s, " + "OpenSSL %s and Zlib %s.", version, bev_str, get_uname(), tor_libevent_get_version_str(), - crypto_openssl_get_version_str()); + crypto_openssl_get_version_str(), + tor_zlib_get_version_str()); log_notice(LD_GENERAL, "Tor can't help you if you use it wrong! " "Learn how to be safe at " @@ -2390,6 +2447,9 @@ tor_init(int argc, char *argv[]) return -1; } stream_choice_seed_weak_rng(); + if (tor_init_libevent_rng() < 0) { + log_warn(LD_NET, "Problem initializing libevent RNG."); + } return 0; } @@ -2492,15 +2552,23 @@ tor_free_all(int postfork) memarea_clear_freelist(); nodelist_free_all(); microdesc_free_all(); + ext_orport_free_all(); + control_free_all(); + sandbox_free_getaddrinfo_cache(); if (!postfork) { config_free_all(); or_state_free_all(); router_free_all(); policies_free_all(); } +#ifdef ENABLE_MEMPOOLS free_cell_pool(); +#endif /* ENABLE_MEMPOOLS */ if (!postfork) { tor_tls_free_all(); +#ifndef _WIN32 + tor_getpwnam(NULL); +#endif } /* stuff in main.c */ @@ -2532,10 +2600,19 @@ tor_cleanup(void) time_t now = time(NULL); /* Remove our pid file. We don't care if there was an error when we * unlink, nothing we could do about it anyways. */ - if (options->PidFile) - unlink(options->PidFile); - if (options->ControlPortWriteToFile) - unlink(options->ControlPortWriteToFile); + if (options->PidFile) { + if (unlink(options->PidFile) != 0) { + log_warn(LD_FS, "Couldn't unlink pid file %s: %s", + options->PidFile, strerror(errno)); + } + } + if (options->ControlPortWriteToFile) { + if (unlink(options->ControlPortWriteToFile) != 0) { + log_warn(LD_FS, "Couldn't unlink control port file %s: %s", + options->ControlPortWriteToFile, + strerror(errno)); + } + } if (accounting_is_enabled(options)) accounting_record_bandwidth_usage(now, get_or_state()); or_state_mark_dirty(get_or_state(), 0); /* force an immediate save. */ @@ -2558,7 +2635,7 @@ tor_cleanup(void) } /** Read/create keys as needed, and echo our fingerprint to stdout. */ -/* static */ int +static int do_list_fingerprint(void) { char buf[FINGERPRINT_LEN+1]; @@ -2566,7 +2643,7 @@ do_list_fingerprint(void) const char *nickname = get_options()->Nickname; if (!server_mode(get_options())) { log_err(LD_GENERAL, - "Clients don't have long-term identity keys. Exiting.\n"); + "Clients don't have long-term identity keys. Exiting."); return -1; } tor_assert(nickname); @@ -2588,7 +2665,7 @@ do_list_fingerprint(void) /** Entry point for password hashing: take the desired password from * the command line, and print its salted hash to stdout. **/ -/* static */ void +static void do_hash_password(void) { @@ -2604,6 +2681,34 @@ do_hash_password(void) printf("16:%s\n",output); } +/** Entry point for configuration dumping: write the configuration to + * stdout. */ +static int +do_dump_config(void) +{ + const or_options_t *options = get_options(); + const char *arg = options->command_arg; + int how; + char *opts; + if (!strcmp(arg, "short")) { + how = OPTIONS_DUMP_MINIMAL; + } else if (!strcmp(arg, "non-builtin")) { + how = OPTIONS_DUMP_DEFAULTS; + } else if (!strcmp(arg, "full")) { + how = OPTIONS_DUMP_ALL; + } else { + printf("%s is not a recognized argument to --dump-config. " + "Please select 'short', 'non-builtin', or 'full'", arg); + return -1; + } + + opts = options_dump(options, how); + printf("%s", opts); + tor_free(opts); + + return 0; +} + #if defined (WINCE) int find_flashcard_path(PWCHAR path, size_t size) @@ -2629,6 +2734,227 @@ find_flashcard_path(PWCHAR path, size_t size) } #endif +static void +init_addrinfo(void) +{ + char hname[256]; + + // host name to sandbox + gethostname(hname, sizeof(hname)); + sandbox_add_addrinfo(hname); +} + +static sandbox_cfg_t* +sandbox_init_filter(void) +{ + const or_options_t *options = get_options(); + sandbox_cfg_t *cfg = sandbox_cfg_new(); + int i; + + sandbox_cfg_allow_openat_filename(&cfg, + get_datadir_fname("cached-status")); + + sandbox_cfg_allow_open_filename_array(&cfg, + get_datadir_fname("cached-certs"), + get_datadir_fname("cached-certs.tmp"), + get_datadir_fname("cached-consensus"), + get_datadir_fname("cached-consensus.tmp"), + get_datadir_fname("unverified-consensus"), + get_datadir_fname("unverified-consensus.tmp"), + get_datadir_fname("unverified-microdesc-consensus"), + get_datadir_fname("unverified-microdesc-consensus.tmp"), + get_datadir_fname("cached-microdesc-consensus"), + get_datadir_fname("cached-microdesc-consensus.tmp"), + get_datadir_fname("cached-microdescs"), + get_datadir_fname("cached-microdescs.tmp"), + get_datadir_fname("cached-microdescs.new"), + get_datadir_fname("cached-microdescs.new.tmp"), + get_datadir_fname("cached-descriptors"), + get_datadir_fname("cached-descriptors.new"), + get_datadir_fname("cached-descriptors.tmp"), + get_datadir_fname("cached-descriptors.new.tmp"), + get_datadir_fname("cached-descriptors.tmp.tmp"), + get_datadir_fname("cached-extrainfo"), + get_datadir_fname("cached-extrainfo.new"), + get_datadir_fname("cached-extrainfo.tmp"), + get_datadir_fname("cached-extrainfo.new.tmp"), + get_datadir_fname("cached-extrainfo.tmp.tmp"), + get_datadir_fname("state.tmp"), + get_datadir_fname("unparseable-desc.tmp"), + get_datadir_fname("unparseable-desc"), + get_datadir_fname("v3-status-votes"), + get_datadir_fname("v3-status-votes.tmp"), + tor_strdup("/dev/srandom"), + tor_strdup("/dev/urandom"), + tor_strdup("/dev/random"), + tor_strdup("/etc/hosts"), + tor_strdup("/proc/meminfo"), + NULL, 0 + ); + if (options->ServerDNSResolvConfFile) + sandbox_cfg_allow_open_filename(&cfg, + tor_strdup(options->ServerDNSResolvConfFile)); + else + sandbox_cfg_allow_open_filename(&cfg, tor_strdup("/etc/resolv.conf")); + + for (i = 0; i < 2; ++i) { + if (get_torrc_fname(i)) { + sandbox_cfg_allow_open_filename(&cfg, tor_strdup(get_torrc_fname(i))); + } + } + +#define RENAME_SUFFIX(name, suffix) \ + sandbox_cfg_allow_rename(&cfg, \ + get_datadir_fname(name suffix), \ + get_datadir_fname(name)) + +#define RENAME_SUFFIX2(prefix, name, suffix) \ + sandbox_cfg_allow_rename(&cfg, \ + get_datadir_fname2(prefix, name suffix), \ + get_datadir_fname2(prefix, name)) + + RENAME_SUFFIX("cached-certs", ".tmp"); + RENAME_SUFFIX("cached-consensus", ".tmp"); + RENAME_SUFFIX("unverified-consensus", ".tmp"); + RENAME_SUFFIX("unverified-microdesc-consensus", ".tmp"); + RENAME_SUFFIX("cached-microdesc-consensus", ".tmp"); + RENAME_SUFFIX("cached-microdescs", ".tmp"); + RENAME_SUFFIX("cached-microdescs", ".new"); + RENAME_SUFFIX("cached-microdescs.new", ".tmp"); + RENAME_SUFFIX("cached-descriptors", ".tmp"); + RENAME_SUFFIX("cached-descriptors", ".new"); + RENAME_SUFFIX("cached-descriptors.new", ".tmp"); + RENAME_SUFFIX("cached-extrainfo", ".tmp"); + RENAME_SUFFIX("cached-extrainfo", ".new"); + RENAME_SUFFIX("cached-extrainfo.new", ".tmp"); + RENAME_SUFFIX("state", ".tmp"); + RENAME_SUFFIX("unparseable-desc", ".tmp"); + RENAME_SUFFIX("v3-status-votes", ".tmp"); + + sandbox_cfg_allow_stat_filename_array(&cfg, + get_datadir_fname(NULL), + get_datadir_fname("lock"), + get_datadir_fname("state"), + get_datadir_fname("router-stability"), + get_datadir_fname("cached-extrainfo.new"), + NULL, 0 + ); + + { + smartlist_t *files = smartlist_new(); + tor_log_get_logfile_names(files); + SMARTLIST_FOREACH(files, char *, file_name, { + /* steals reference */ + sandbox_cfg_allow_open_filename(&cfg, file_name); + }); + smartlist_free(files); + } + + { + smartlist_t *files = smartlist_new(); + smartlist_t *dirs = smartlist_new(); + rend_services_add_filenames_to_lists(files, dirs); + SMARTLIST_FOREACH(files, char *, file_name, { + char *tmp_name = NULL; + tor_asprintf(&tmp_name, "%s.tmp", file_name); + sandbox_cfg_allow_rename(&cfg, + tor_strdup(tmp_name), tor_strdup(file_name)); + /* steals references */ + sandbox_cfg_allow_open_filename_array(&cfg, file_name, tmp_name, NULL); + }); + SMARTLIST_FOREACH(dirs, char *, dir, { + /* steals reference */ + sandbox_cfg_allow_stat_filename(&cfg, dir); + }); + smartlist_free(files); + smartlist_free(dirs); + } + + { + char *fname; + if ((fname = get_controller_cookie_file_name())) { + sandbox_cfg_allow_open_filename(&cfg, fname); + } + if ((fname = get_ext_or_auth_cookie_file_name())) { + sandbox_cfg_allow_open_filename(&cfg, fname); + } + } + + if (options->DirPortFrontPage) { + sandbox_cfg_allow_open_filename(&cfg, + tor_strdup(options->DirPortFrontPage)); + } + + // orport + if (server_mode(get_options())) { + sandbox_cfg_allow_open_filename_array(&cfg, + get_datadir_fname2("keys", "secret_id_key"), + get_datadir_fname2("keys", "secret_onion_key"), + get_datadir_fname2("keys", "secret_onion_key_ntor"), + get_datadir_fname2("keys", "secret_onion_key_ntor.tmp"), + get_datadir_fname2("keys", "secret_id_key.old"), + get_datadir_fname2("keys", "secret_onion_key.old"), + get_datadir_fname2("keys", "secret_onion_key_ntor.old"), + get_datadir_fname2("keys", "secret_onion_key.tmp"), + get_datadir_fname2("keys", "secret_id_key.tmp"), + get_datadir_fname2("stats", "bridge-stats"), + get_datadir_fname2("stats", "bridge-stats.tmp"), + get_datadir_fname2("stats", "dirreq-stats"), + get_datadir_fname2("stats", "dirreq-stats.tmp"), + get_datadir_fname2("stats", "entry-stats"), + get_datadir_fname2("stats", "entry-stats.tmp"), + get_datadir_fname2("stats", "exit-stats"), + get_datadir_fname2("stats", "exit-stats.tmp"), + get_datadir_fname2("stats", "buffer-stats"), + get_datadir_fname2("stats", "buffer-stats.tmp"), + get_datadir_fname2("stats", "conn-stats"), + get_datadir_fname2("stats", "conn-stats.tmp"), + get_datadir_fname("approved-routers"), + get_datadir_fname("fingerprint"), + get_datadir_fname("fingerprint.tmp"), + get_datadir_fname("hashed-fingerprint"), + get_datadir_fname("hashed-fingerprint.tmp"), + get_datadir_fname("router-stability"), + get_datadir_fname("router-stability.tmp"), + tor_strdup("/etc/resolv.conf"), + NULL, 0 + ); + + RENAME_SUFFIX("fingerprint", ".tmp"); + RENAME_SUFFIX2("keys", "secret_onion_key_ntor", ".tmp"); + RENAME_SUFFIX2("keys", "secret_id_key", ".tmp"); + RENAME_SUFFIX2("keys", "secret_id_key.old", ".tmp"); + RENAME_SUFFIX2("keys", "secret_onion_key", ".tmp"); + RENAME_SUFFIX2("keys", "secret_onion_key.old", ".tmp"); + RENAME_SUFFIX2("stats", "bridge-stats", ".tmp"); + RENAME_SUFFIX2("stats", "dirreq-stats", ".tmp"); + RENAME_SUFFIX2("stats", "entry-stats", ".tmp"); + RENAME_SUFFIX2("stats", "exit-stats", ".tmp"); + RENAME_SUFFIX2("stats", "buffer-stats", ".tmp"); + RENAME_SUFFIX2("stats", "conn-stats", ".tmp"); + RENAME_SUFFIX("hashed-fingerprint", ".tmp"); + RENAME_SUFFIX("router-stability", ".tmp"); + + sandbox_cfg_allow_rename(&cfg, + get_datadir_fname2("keys", "secret_onion_key"), + get_datadir_fname2("keys", "secret_onion_key.old")); + sandbox_cfg_allow_rename(&cfg, + get_datadir_fname2("keys", "secret_onion_key_ntor"), + get_datadir_fname2("keys", "secret_onion_key_ntor.old")); + + sandbox_cfg_allow_stat_filename_array(&cfg, + get_datadir_fname("keys"), + get_datadir_fname("stats"), + get_datadir_fname2("stats", "dirreq-stats"), + NULL, 0 + ); + } + + init_addrinfo(); + + return cfg; +} + /** Main entry point for the Tor process. Called from main(). */ /* This function is distinct from main() only so we can link main.c into * the unittest binary without conflicting with the unittests' main. */ @@ -2675,6 +3001,8 @@ tor_main(int argc, char *argv[]) } #endif + configure_backtrace_handler(get_version()); + update_approx_time(time(NULL)); tor_threads_init(); init_logging(); @@ -2695,6 +3023,22 @@ tor_main(int argc, char *argv[]) #endif if (tor_init(argc, argv)<0) return -1; + + if (get_options()->Sandbox && get_options()->command == CMD_RUN_TOR) { + sandbox_cfg_t* cfg = sandbox_init_filter(); + + if (sandbox_init(cfg)) { + log_err(LD_BUG,"Failed to create syscall sandbox filter"); + return -1; + } + + // registering libevent rng +#ifdef HAVE_EVUTIL_SECURE_RNG_SET_URANDOM_DEVICE_FILE + evutil_secure_rng_set_urandom_device_file( + (char*) sandbox_intern_string("/dev/urandom")); +#endif + } + switch (get_options()->command) { case CMD_RUN_TOR: #ifdef NT_SERVICE @@ -2713,6 +3057,9 @@ tor_main(int argc, char *argv[]) printf("Configuration was valid\n"); result = 0; break; + case CMD_DUMP_CONFIG: + result = do_dump_config(); + break; case CMD_RUN_UNITTESTS: /* only set by test.c */ default: log_warn(LD_BUG,"Illegal command number %d: internal error.", diff --git a/src/or/main.h b/src/or/main.h index 338449b6a6..a3bce3486f 100644 --- a/src/or/main.h +++ b/src/or/main.h @@ -24,8 +24,8 @@ void add_connection_to_closeable_list(connection_t *conn); int connection_is_on_closeable_list(connection_t *conn); smartlist_t *get_connection_array(void); -uint64_t get_bytes_read(void); -uint64_t get_bytes_written(void); +MOCK_DECL(uint64_t,get_bytes_read,(void)); +MOCK_DECL(uint64_t,get_bytes_written,(void)); /** Bitmask for events that we can turn on and off with * connection_watch_events. */ @@ -36,12 +36,12 @@ typedef enum watchable_events { } watchable_events_t; void connection_watch_events(connection_t *conn, watchable_events_t events); int connection_is_reading(connection_t *conn); -void connection_stop_reading(connection_t *conn); -void connection_start_reading(connection_t *conn); +MOCK_DECL(void,connection_stop_reading,(connection_t *conn)); +MOCK_DECL(void,connection_start_reading,(connection_t *conn)); int connection_is_writing(connection_t *conn); -void connection_stop_writing(connection_t *conn); -void connection_start_writing(connection_t *conn); +MOCK_DECL(void,connection_stop_writing,(connection_t *conn)); +MOCK_DECL(void,connection_start_writing,(connection_t *conn)); void connection_stop_reading_from_linked_conn(connection_t *conn); @@ -50,8 +50,10 @@ void directory_info_has_arrived(time_t now, int from_cache); void ip_address_changed(int at_interface); void dns_servers_relaunch_checks(void); +void reschedule_descriptor_update_check(void); + +MOCK_DECL(long,get_uptime,(void)); -long get_uptime(void); unsigned get_signewnym_epoch(void); void handle_signals(int is_parent); @@ -66,11 +68,12 @@ void tor_free_all(int postfork); int tor_main(int argc, char *argv[]); -#ifdef MAIN_PRIVATE int do_main_loop(void); -int do_list_fingerprint(void); -void do_hash_password(void); int tor_init(int argc, char **argv); + +#ifdef MAIN_PRIVATE +STATIC void init_connection_lists(void); +STATIC void close_closeable_connections(void); #endif #endif diff --git a/src/or/microdesc.c b/src/or/microdesc.c index 0e72c0b89b..fdb549a9ac 100644 --- a/src/or/microdesc.c +++ b/src/or/microdesc.c @@ -45,12 +45,7 @@ struct microdesc_cache_t { static INLINE unsigned int microdesc_hash_(microdesc_t *md) { - unsigned *d = (unsigned*)md->digest; -#if SIZEOF_INT == 4 - return d[0] ^ d[1] ^ d[2] ^ d[3] ^ d[4] ^ d[5] ^ d[6] ^ d[7]; -#else - return d[0] ^ d[1] ^ d[2] ^ d[3]; -#endif + return (unsigned) siphash24g(md->digest, sizeof(md->digest)); } /** Helper: compares <b>a</b> and </b> for equality for hash-table purposes. */ @@ -139,7 +134,7 @@ get_microdesc_cache(void) * ending at <b>eos</b>, and store them in <b>cache</b>. If <b>no_save</b>, * mark them as non-writable to disk. If <b>where</b> is SAVED_IN_CACHE, * leave their bodies as pointers to the mmap'd cache. If where is - * <b>SAVED_NOWHERE</b>, do not allow annotations. If listed_at is positive, + * <b>SAVED_NOWHERE</b>, do not allow annotations. If listed_at is not -1, * set the last_listed field of every microdesc to listed_at. If * requested_digests is non-null, then it contains a list of digests we mean * to allow, so we should reject any non-requested microdesc with a different @@ -158,7 +153,7 @@ microdescs_add_to_cache(microdesc_cache_t *cache, descriptors = microdescs_parse_from_string(s, eos, allow_annotations, where); - if (listed_at > 0) { + if (listed_at != (time_t)-1) { SMARTLIST_FOREACH(descriptors, microdesc_t *, md, md->last_listed = listed_at); } @@ -280,6 +275,7 @@ void microdesc_cache_clear(microdesc_cache_t *cache) { microdesc_t **entry, **next; + for (entry = HT_START(microdesc_map, &cache->map); entry; entry = next) { microdesc_t *md = *entry; next = HT_NEXT_RMV(microdesc_map, &cache->map, entry); @@ -288,7 +284,13 @@ microdesc_cache_clear(microdesc_cache_t *cache) } HT_CLEAR(microdesc_map, &cache->map); if (cache->cache_content) { - tor_munmap_file(cache->cache_content); + int res = tor_munmap_file(cache->cache_content); + if (res != 0) { + log_warn(LD_FS, + "tor_munmap_file() failed clearing microdesc cache; " + "we are probably about to leak memory."); + /* TODO something smarter? */ + } cache->cache_content = NULL; } cache->total_len_seen = 0; @@ -368,7 +370,9 @@ microdesc_cache_clean(microdesc_cache_t *cache, time_t cutoff, int force) cutoff = now - TOLERATE_MICRODESC_AGE; for (mdp = HT_START(microdesc_map, &cache->map); mdp != NULL; ) { - if ((*mdp)->last_listed < cutoff) { + const int is_old = (*mdp)->last_listed < cutoff; + const unsigned held_by_nodes = (*mdp)->held_by_nodes; + if (is_old && !held_by_nodes) { ++dropped; victim = *mdp; mdp = HT_NEXT_RMV(microdesc_map, &cache->map, mdp); @@ -376,6 +380,57 @@ microdesc_cache_clean(microdesc_cache_t *cache, time_t cutoff, int force) bytes_dropped += victim->bodylen; microdesc_free(victim); } else { + if (is_old) { + /* It's old, but it has held_by_nodes set. That's not okay. */ + /* Let's try to diagnose and fix #7164 . */ + smartlist_t *nodes = nodelist_find_nodes_with_microdesc(*mdp); + const networkstatus_t *ns = networkstatus_get_latest_consensus(); + long networkstatus_age = -1; + const int ht_badness = HT_REP_IS_BAD_(microdesc_map, &cache->map); + if (ns) { + networkstatus_age = now - ns->valid_after; + } + log_warn(LD_BUG, "Microdescriptor seemed very old " + "(last listed %d hours ago vs %d hour cutoff), but is still " + "marked as being held by %d node(s). I found %d node(s) " + "holding it. Current networkstatus is %ld hours old. " + "Hashtable badness is %d.", + (int)((now - (*mdp)->last_listed) / 3600), + (int)((now - cutoff) / 3600), + held_by_nodes, + smartlist_len(nodes), + networkstatus_age / 3600, + ht_badness); + + SMARTLIST_FOREACH_BEGIN(nodes, const node_t *, node) { + const char *rs_match = "No RS"; + const char *rs_present = ""; + if (node->rs) { + if (tor_memeq(node->rs->descriptor_digest, + (*mdp)->digest, DIGEST256_LEN)) { + rs_match = "Microdesc digest in RS matches"; + } else { + rs_match = "Microdesc digest in RS does match"; + } + if (ns) { + /* This should be impossible, but let's see! */ + rs_present = " RS not present in networkstatus."; + SMARTLIST_FOREACH(ns->routerstatus_list, routerstatus_t *,rs, { + if (rs == node->rs) { + rs_present = " RS okay in networkstatus."; + } + }); + } + } + log_warn(LD_BUG, " [%d]: ID=%s. md=%p, rs=%p, ri=%p. %s.%s", + node_sl_idx, + hex_str(node->identity, DIGEST_LEN), + node->md, node->rs, node->ri, rs_match, rs_present); + } SMARTLIST_FOREACH_END(node); + smartlist_free(nodes); + (*mdp)->last_listed = now; + } + ++kept; mdp = HT_NEXT(microdesc_map, &cache->map, mdp); } @@ -434,7 +489,7 @@ int microdesc_cache_rebuild(microdesc_cache_t *cache, int force) { open_file_t *open_file; - int fd = -1; + int fd = -1, res; microdesc_t **mdp; smartlist_t *wrote; ssize_t size; @@ -489,7 +544,8 @@ microdesc_cache_rebuild(microdesc_cache_t *cache, int force) "By my count, I'm at "I64_FORMAT ", but I should be at "I64_FORMAT, I64_PRINTF_ARG(off), I64_PRINTF_ARG(off_real)); - off = off_real; + if (off_real >= 0) + off = off_real; } if (md->saved_location != SAVED_IN_CACHE) { tor_free(md->body); @@ -500,8 +556,14 @@ microdesc_cache_rebuild(microdesc_cache_t *cache, int force) /* We must do this unmap _before_ we call finish_writing_to_file(), or * windows will not actually replace the file. */ - if (cache->cache_content) - tor_munmap_file(cache->cache_content); + if (cache->cache_content) { + res = tor_munmap_file(cache->cache_content); + if (res != 0) { + log_warn(LD_FS, + "Failed to unmap old microdescriptor cache while rebuilding"); + } + cache->cache_content = NULL; + } if (finish_writing_to_file(open_file) < 0) { log_warn(LD_DIR, "Error rebuilding microdescriptor cache: %s", @@ -605,8 +667,10 @@ microdesc_free_(microdesc_t *md, const char *fname, int lineno) tor_fragile_assert(); } if (md->held_by_nodes) { + microdesc_cache_t *cache = get_microdesc_cache(); int found=0; const smartlist_t *nodes = nodelist_get_list(); + const int ht_badness = HT_REP_IS_BAD_(microdesc_map, &cache->map); SMARTLIST_FOREACH(nodes, node_t *, node, { if (node->md == md) { ++found; @@ -614,13 +678,14 @@ microdesc_free_(microdesc_t *md, const char *fname, int lineno) } }); if (found) { - log_info(LD_BUG, "microdesc_free() called from %s:%d, but md was still " - "referenced %d node(s); held_by_nodes == %u", - fname, lineno, found, md->held_by_nodes); + log_warn(LD_BUG, "microdesc_free() called from %s:%d, but md was still " + "referenced %d node(s); held_by_nodes == %u, ht_badness == %d", + fname, lineno, found, md->held_by_nodes, ht_badness); } else { log_warn(LD_BUG, "microdesc_free() called from %s:%d with held_by_nodes " - "set to %u, but md was not referenced by any nodes", - fname, lineno, md->held_by_nodes); + "set to %u, but md was not referenced by any nodes. " + "ht_badness == %d", + fname, lineno, md->held_by_nodes, ht_badness); } tor_fragile_assert(); } @@ -696,7 +761,7 @@ microdesc_list_missing_digest256(networkstatus_t *ns, microdesc_cache_t *cache, continue; if (downloadable_only && !download_status_is_ready(&rs->dl_status, now, - MAX_MICRODESC_DOWNLOAD_FAILURES)) + get_options()->TestingMicrodescMaxDownloadTries)) continue; if (skip && digestmap_get(skip, rs->descriptor_digest)) continue; @@ -725,7 +790,7 @@ update_microdesc_downloads(time_t now) smartlist_t *missing; digestmap_t *pending; - if (should_delay_dir_fetches(options)) + if (should_delay_dir_fetches(options, NULL)) return; if (directory_too_idle_to_fetch_descriptors(options, now)) return; diff --git a/src/or/networkstatus.c b/src/or/networkstatus.c index 23b7304b39..890da0ad17 100644 --- a/src/or/networkstatus.c +++ b/src/or/networkstatus.c @@ -10,6 +10,7 @@ * client or cache. */ +#define NETWORKSTATUS_PRIVATE #include "or.h" #include "channel.h" #include "circuitmux.h" @@ -31,18 +32,7 @@ #include "router.h" #include "routerlist.h" #include "routerparse.h" - -/* For tracking v2 networkstatus documents. Only caches do this now. */ - -/** Map from descriptor digest of routers listed in the v2 networkstatus - * documents to download_status_t* */ -static digestmap_t *v2_download_status_map = NULL; -/** Global list of all of the current v2 network_status documents that we know - * about. This list is kept sorted by published_on. */ -static smartlist_t *networkstatus_v2_list = NULL; -/** True iff any member of networkstatus_v2_list has changed since the last - * time we called download_status_map_update_from_v2_networkstatus() */ -static int networkstatus_v2_list_has_changed = 0; +#include "transports.h" /** Map from lowercase nickname to identity digest of named server, if any. */ static strmap_t *named_server_map = NULL; @@ -88,11 +78,6 @@ typedef struct consensus_waiting_for_certs_t { static consensus_waiting_for_certs_t consensus_waiting_for_certs[N_CONSENSUS_FLAVORS]; -/** The last time we tried to download a networkstatus, or 0 for "never". We - * use this to rate-limit download attempts for directory caches (including - * mirrors). Clients don't use this now. */ -static time_t last_networkstatus_download_attempted = 0; - /** A time before which we shouldn't try to replace the current consensus: * this will be at some point after the next consensus becomes valid, but * before the current consensus becomes invalid. */ @@ -107,7 +92,6 @@ static int have_warned_about_old_version = 0; * listed by the authorities. */ static int have_warned_about_new_version = 0; -static void download_status_map_update_from_v2_networkstatus(void); static void routerstatus_list_update_named_server_map(void); /** Forget that we've warned about anything networkstatus-related, so we will @@ -131,86 +115,9 @@ void networkstatus_reset_download_failures(void) { int i; - const smartlist_t *networkstatus_v2_list = networkstatus_get_v2_list(); - SMARTLIST_FOREACH_BEGIN(networkstatus_v2_list, networkstatus_v2_t *, ns) { - SMARTLIST_FOREACH_BEGIN(ns->entries, routerstatus_t *, rs) { - if (!router_get_by_descriptor_digest(rs->descriptor_digest)) - rs->need_to_mirror = 1; - } SMARTLIST_FOREACH_END(rs); - } SMARTLIST_FOREACH_END(ns); for (i=0; i < N_CONSENSUS_FLAVORS; ++i) download_status_reset(&consensus_dl_status[i]); - if (v2_download_status_map) { - digestmap_iter_t *iter; - digestmap_t *map = v2_download_status_map; - const char *key; - void *val; - download_status_t *dls; - for (iter = digestmap_iter_init(map); !digestmap_iter_done(iter); - iter = digestmap_iter_next(map, iter) ) { - digestmap_iter_get(iter, &key, &val); - dls = val; - download_status_reset(dls); - } - } -} - -/** Repopulate our list of network_status_t objects from the list cached on - * disk. Return 0 on success, -1 on failure. */ -int -router_reload_v2_networkstatus(void) -{ - smartlist_t *entries; - struct stat st; - char *s; - char *filename = get_datadir_fname("cached-status"); - int maybe_delete = !directory_caches_v2_dir_info(get_options()); - time_t now = time(NULL); - if (!networkstatus_v2_list) - networkstatus_v2_list = smartlist_new(); - - entries = tor_listdir(filename); - if (!entries) { /* dir doesn't exist */ - tor_free(filename); - return 0; - } else if (!smartlist_len(entries) && maybe_delete) { - rmdir(filename); - tor_free(filename); - smartlist_free(entries); - return 0; - } - tor_free(filename); - SMARTLIST_FOREACH_BEGIN(entries, const char *, fn) { - char buf[DIGEST_LEN]; - if (maybe_delete) { - filename = get_datadir_fname2("cached-status", fn); - remove_file_if_very_old(filename, now); - tor_free(filename); - continue; - } - if (strlen(fn) != HEX_DIGEST_LEN || - base16_decode(buf, sizeof(buf), fn, strlen(fn))) { - log_info(LD_DIR, - "Skipping cached-status file with unexpected name \"%s\"",fn); - continue; - } - filename = get_datadir_fname2("cached-status", fn); - s = read_file_to_str(filename, 0, &st); - if (s) { - if (router_set_networkstatus_v2(s, st.st_mtime, NS_FROM_CACHE, - NULL)<0) { - log_warn(LD_FS, "Couldn't load networkstatus from \"%s\"",filename); - } - tor_free(s); - } - tor_free(filename); - } SMARTLIST_FOREACH_END(fn); - SMARTLIST_FOREACH(entries, char *, fn, tor_free(fn)); - smartlist_free(entries); - networkstatus_v2_list_clean(time(NULL)); - routers_update_all_from_networkstatus(time(NULL), 2); - return 0; } /** Read every cached v3 consensus networkstatus from the disk. */ @@ -277,7 +184,7 @@ router_reload_consensus_networkstatus(void) } /** Free all storage held by the vote_routerstatus object <b>rs</b>. */ -static void +STATIC void vote_routerstatus_free(vote_routerstatus_t *rs) { vote_microdesc_hash_t *h, *next; @@ -303,26 +210,6 @@ routerstatus_free(routerstatus_t *rs) tor_free(rs); } -/** Free all storage held by the networkstatus object <b>ns</b>. */ -void -networkstatus_v2_free(networkstatus_v2_t *ns) -{ - if (!ns) - return; - tor_free(ns->source_address); - tor_free(ns->contact); - if (ns->signing_key) - crypto_pk_free(ns->signing_key); - tor_free(ns->client_versions); - tor_free(ns->server_versions); - if (ns->entries) { - SMARTLIST_FOREACH(ns->entries, routerstatus_t *, rs, - routerstatus_free(rs)); - smartlist_free(ns->entries); - } - tor_free(ns); -} - /** Free all storage held in <b>sig</b> */ void document_signature_free(document_signature_t *sig) @@ -648,296 +535,10 @@ networkstatus_check_consensus_signature(networkstatus_t *consensus, return -2; } -/** Helper: return a newly allocated string containing the name of the filename - * where we plan to cache the network status with the given identity digest. */ -char * -networkstatus_get_cache_filename(const char *identity_digest) -{ - char fp[HEX_DIGEST_LEN+1]; - base16_encode(fp, HEX_DIGEST_LEN+1, identity_digest, DIGEST_LEN); - return get_datadir_fname2("cached-status", fp); -} - -/** Helper for smartlist_sort: Compare two networkstatus objects by - * publication date. */ -static int -compare_networkstatus_v2_published_on_(const void **_a, const void **_b) -{ - const networkstatus_v2_t *a = *_a, *b = *_b; - if (a->published_on < b->published_on) - return -1; - else if (a->published_on > b->published_on) - return 1; - else - return 0; -} - -/** Add the parsed v2 networkstatus in <b>ns</b> (with original document in - * <b>s</b>) to the disk cache (and the in-memory directory server cache) as - * appropriate. */ -static int -add_networkstatus_to_cache(const char *s, - v2_networkstatus_source_t source, - networkstatus_v2_t *ns) -{ - if (source != NS_FROM_CACHE) { - char *fn = networkstatus_get_cache_filename(ns->identity_digest); - if (write_str_to_file(fn, s, 0)<0) { - log_notice(LD_FS, "Couldn't write cached network status to \"%s\"", fn); - } - tor_free(fn); - } - - if (directory_caches_v2_dir_info(get_options())) - dirserv_set_cached_networkstatus_v2(s, - ns->identity_digest, - ns->published_on); - - return 0; -} - /** How far in the future do we allow a network-status to get before removing * it? (seconds) */ #define NETWORKSTATUS_ALLOW_SKEW (24*60*60) -/** Given a string <b>s</b> containing a network status that we received at - * <b>arrived_at</b> from <b>source</b>, try to parse it, see if we want to - * store it, and put it into our cache as necessary. - * - * If <b>source</b> is NS_FROM_DIR or NS_FROM_CACHE, do not replace our - * own networkstatus_t (if we're an authoritative directory server). - * - * If <b>source</b> is NS_FROM_CACHE, do not write our networkstatus_t to the - * cache. - * - * If <b>requested_fingerprints</b> is provided, it must contain a list of - * uppercased identity fingerprints. Do not update any networkstatus whose - * fingerprint is not on the list; after updating a networkstatus, remove its - * fingerprint from the list. - * - * Return 0 on success, -1 on failure. - * - * Callers should make sure that routers_update_all_from_networkstatus() is - * invoked after this function succeeds. - */ -int -router_set_networkstatus_v2(const char *s, time_t arrived_at, - v2_networkstatus_source_t source, - smartlist_t *requested_fingerprints) -{ - networkstatus_v2_t *ns; - int i, found; - time_t now; - int skewed = 0; - dir_server_t *trusted_dir = NULL; - const char *source_desc = NULL; - char fp[HEX_DIGEST_LEN+1]; - char published[ISO_TIME_LEN+1]; - - if (!directory_caches_v2_dir_info(get_options())) - return 0; /* Don't bother storing it. */ - - ns = networkstatus_v2_parse_from_string(s); - if (!ns) { - log_warn(LD_DIR, "Couldn't parse network status."); - return -1; - } - base16_encode(fp, HEX_DIGEST_LEN+1, ns->identity_digest, DIGEST_LEN); - if (!(trusted_dir = - router_get_trusteddirserver_by_digest(ns->identity_digest)) || - !(trusted_dir->type & V2_DIRINFO)) { - log_info(LD_DIR, "Network status was signed, but not by an authoritative " - "directory we recognize."); - source_desc = fp; - } else { - source_desc = trusted_dir->description; - } - now = time(NULL); - if (arrived_at > now) - arrived_at = now; - - ns->received_on = arrived_at; - - format_iso_time(published, ns->published_on); - - if (ns->published_on > now + NETWORKSTATUS_ALLOW_SKEW) { - char dbuf[64]; - long delta = now - ns->published_on; - format_time_interval(dbuf, sizeof(dbuf), delta); - log_warn(LD_GENERAL, "Network status from %s was published %s in the " - "future (%s UTC). Check your time and date settings! " - "Not caching.", - source_desc, dbuf, published); - control_event_general_status(LOG_WARN, - "CLOCK_SKEW MIN_SKEW=%ld SOURCE=NETWORKSTATUS:%s:%d", - delta, ns->source_address, ns->source_dirport); - skewed = 1; - } - - if (!networkstatus_v2_list) - networkstatus_v2_list = smartlist_new(); - - if ( (source == NS_FROM_DIR_BY_FP || source == NS_FROM_DIR_ALL) && - router_digest_is_me(ns->identity_digest)) { - /* Don't replace our own networkstatus when we get it from somebody else.*/ - networkstatus_v2_free(ns); - return 0; - } - - if (requested_fingerprints) { - if (smartlist_contains_string(requested_fingerprints, fp)) { - smartlist_string_remove(requested_fingerprints, fp); - } else { - if (source != NS_FROM_DIR_ALL) { - char *requested = - smartlist_join_strings(requested_fingerprints," ",0,NULL); - log_warn(LD_DIR, - "We received a network status with a fingerprint (%s) that we " - "never requested. (We asked for: %s.) Dropping.", - fp, requested); - tor_free(requested); - return 0; - } - } - } - - if (!trusted_dir) { - if (!skewed) { - /* We got a non-trusted networkstatus, and we're a directory cache. - * This means that we asked an authority, and it told us about another - * authority we didn't recognize. */ - log_info(LD_DIR, - "We do not recognize authority (%s) but we are willing " - "to cache it.", fp); - add_networkstatus_to_cache(s, source, ns); - networkstatus_v2_free(ns); - } - return 0; - } - - found = 0; - for (i=0; i < smartlist_len(networkstatus_v2_list); ++i) { - networkstatus_v2_t *old_ns = smartlist_get(networkstatus_v2_list, i); - - if (tor_memeq(old_ns->identity_digest, ns->identity_digest, DIGEST_LEN)) { - if (tor_memeq(old_ns->networkstatus_digest, - ns->networkstatus_digest, DIGEST_LEN)) { - /* Same one we had before. */ - networkstatus_v2_free(ns); - tor_assert(trusted_dir); - log_info(LD_DIR, - "Not replacing network-status from %s (published %s); " - "we already have it.", - trusted_dir->description, published); - if (old_ns->received_on < arrived_at) { - if (source != NS_FROM_CACHE) { - char *fn; - fn = networkstatus_get_cache_filename(old_ns->identity_digest); - /* We use mtime to tell when it arrived, so update that. */ - touch_file(fn); - tor_free(fn); - } - old_ns->received_on = arrived_at; - } - download_status_failed(&trusted_dir->v2_ns_dl_status, 0); - return 0; - } else if (old_ns->published_on >= ns->published_on) { - char old_published[ISO_TIME_LEN+1]; - format_iso_time(old_published, old_ns->published_on); - tor_assert(trusted_dir); - log_info(LD_DIR, - "Not replacing network-status from %s (published %s);" - " we have a newer one (published %s) for this authority.", - trusted_dir->description, published, - old_published); - networkstatus_v2_free(ns); - download_status_failed(&trusted_dir->v2_ns_dl_status, 0); - return 0; - } else { - networkstatus_v2_free(old_ns); - smartlist_set(networkstatus_v2_list, i, ns); - found = 1; - break; - } - } - } - - if (source != NS_FROM_CACHE && trusted_dir) { - download_status_reset(&trusted_dir->v2_ns_dl_status); - } - - if (!found) - smartlist_add(networkstatus_v2_list, ns); - -/** Retain any routerinfo mentioned in a V2 networkstatus for at least this - * long. */ -#define V2_NETWORKSTATUS_ROUTER_LIFETIME (3*60*60) - - { - time_t live_until = ns->published_on + V2_NETWORKSTATUS_ROUTER_LIFETIME; - SMARTLIST_FOREACH_BEGIN(ns->entries, routerstatus_t *, rs) { - signed_descriptor_t *sd = - router_get_by_descriptor_digest(rs->descriptor_digest); - if (sd) { - if (sd->last_listed_as_valid_until < live_until) - sd->last_listed_as_valid_until = live_until; - } else { - rs->need_to_mirror = 1; - } - } SMARTLIST_FOREACH_END(rs); - } - - log_info(LD_DIR, "Setting networkstatus %s %s (published %s)", - source == NS_FROM_CACHE?"cached from": - ((source == NS_FROM_DIR_BY_FP || source == NS_FROM_DIR_ALL) ? - "downloaded from":"generated for"), - trusted_dir->description, published); - networkstatus_v2_list_has_changed = 1; - - smartlist_sort(networkstatus_v2_list, - compare_networkstatus_v2_published_on_); - - if (!skewed) - add_networkstatus_to_cache(s, source, ns); - - return 0; -} - -/** Remove all very-old network_status_t objects from memory and from the - * disk cache. */ -void -networkstatus_v2_list_clean(time_t now) -{ - int i; - if (!networkstatus_v2_list) - return; - - for (i = 0; i < smartlist_len(networkstatus_v2_list); ++i) { - networkstatus_v2_t *ns = smartlist_get(networkstatus_v2_list, i); - char *fname = NULL; - if (ns->published_on + MAX_NETWORKSTATUS_AGE > now) - continue; - /* Okay, this one is too old. Remove it from the list, and delete it - * from the cache. */ - smartlist_del(networkstatus_v2_list, i--); - fname = networkstatus_get_cache_filename(ns->identity_digest); - if (file_status(fname) == FN_FILE) { - log_info(LD_DIR, "Removing too-old networkstatus in %s", fname); - unlink(fname); - } - tor_free(fname); - if (directory_caches_v2_dir_info(get_options())) { - dirserv_set_cached_networkstatus_v2(NULL, ns->identity_digest, 0); - } - networkstatus_v2_free(ns); - } - - /* And now go through the directory cache for any cached untrusted - * networkstatuses and other network info. */ - dirserv_clear_old_networkstatuses(now - MAX_NETWORKSTATUS_AGE); - dirserv_clear_old_v1_info(now); -} - /** Helper for bsearching a list of routerstatus_t pointers: compare a * digest in the key to the identity digest of a routerstatus_t. */ int @@ -959,22 +560,6 @@ compare_digest_to_vote_routerstatus_entry(const void *_key, return tor_memcmp(key, vrs->status.identity_digest, DIGEST_LEN); } -/** As networkstatus_v2_find_entry, but do not return a const pointer */ -routerstatus_t * -networkstatus_v2_find_mutable_entry(networkstatus_v2_t *ns, const char *digest) -{ - return smartlist_bsearch(ns->entries, digest, - compare_digest_to_routerstatus_entry); -} - -/** Return the entry in <b>ns</b> for the identity digest <b>digest</b>, or - * NULL if none was found. */ -const routerstatus_t * -networkstatus_v2_find_entry(networkstatus_v2_t *ns, const char *digest) -{ - return networkstatus_v2_find_mutable_entry(ns, digest); -} - /** As networkstatus_find_entry, but do not return a const pointer */ routerstatus_t * networkstatus_vote_find_mutable_entry(networkstatus_t *ns, const char *digest) @@ -1004,15 +589,6 @@ networkstatus_vote_find_entry_idx(networkstatus_t *ns, found_out); } -/** Return a list of the v2 networkstatus documents. */ -const smartlist_t * -networkstatus_get_v2_list(void) -{ - if (!networkstatus_v2_list) - networkstatus_v2_list = smartlist_new(); - return networkstatus_v2_list; -} - /** As router_get_consensus_status_by_descriptor_digest, but does not return * a const pointer. */ routerstatus_t * @@ -1057,8 +633,6 @@ router_get_dl_status_by_descriptor_digest(const char *d) if ((rs = router_get_mutable_consensus_status_by_descriptor_digest( current_ns_consensus, d))) return &rs->dl_status; - if (v2_download_status_map) - return digestmap_get(v2_download_status_map, d); return NULL; } @@ -1124,72 +698,6 @@ networkstatus_nickname_is_unnamed(const char *nickname) * networkstatus documents? */ #define NONAUTHORITY_NS_CACHE_INTERVAL (60*60) -/** We are a directory server, and so cache network_status documents. - * Initiate downloads as needed to update them. For v2 authorities, - * this means asking each trusted directory for its network-status. - * For caches, this means asking a random v2 authority for all - * network-statuses. - */ -static void -update_v2_networkstatus_cache_downloads(time_t now) -{ - int authority = authdir_mode_v2(get_options()); - int interval = - authority ? AUTHORITY_NS_CACHE_INTERVAL : NONAUTHORITY_NS_CACHE_INTERVAL; - const smartlist_t *trusted_dir_servers = router_get_trusted_dir_servers(); - - if (last_networkstatus_download_attempted + interval >= now) - return; - - last_networkstatus_download_attempted = now; - - if (authority) { - /* An authority launches a separate connection for everybody. */ - SMARTLIST_FOREACH_BEGIN(trusted_dir_servers, dir_server_t *, ds) - { - char resource[HEX_DIGEST_LEN+6]; /* fp/hexdigit.z\0 */ - tor_addr_t addr; - if (!(ds->type & V2_DIRINFO)) - continue; - if (router_digest_is_me(ds->digest)) - continue; - tor_addr_from_ipv4h(&addr, ds->addr); - /* Is this quite sensible with IPv6 or multiple addresses? */ - if (connection_get_by_type_addr_port_purpose( - CONN_TYPE_DIR, &addr, ds->dir_port, - DIR_PURPOSE_FETCH_V2_NETWORKSTATUS)) { - /* XXX the above dir_port won't be accurate if we're - * doing a tunneled conn. In that case it should be or_port. - * How to guess from here? Maybe make the function less general - * and have it know that it's looking for dir conns. -RD */ - /* Only directory caches download v2 networkstatuses, and they - * don't use tunneled connections. I think it's okay to ignore - * this. */ - continue; - } - strlcpy(resource, "fp/", sizeof(resource)); - base16_encode(resource+3, sizeof(resource)-3, ds->digest, DIGEST_LEN); - strlcat(resource, ".z", sizeof(resource)); - directory_initiate_command_routerstatus( - &ds->fake_status, DIR_PURPOSE_FETCH_V2_NETWORKSTATUS, - ROUTER_PURPOSE_GENERAL, - DIRIND_ONEHOP, - resource, - NULL, 0 /* No payload. */, - 0 /* No I-M-S. */); - } - SMARTLIST_FOREACH_END(ds); - } else { - /* A non-authority cache launches one connection to a random authority. */ - /* (Check whether we're currently fetching network-status objects.) */ - if (!connection_get_by_type_purpose(CONN_TYPE_DIR, - DIR_PURPOSE_FETCH_V2_NETWORKSTATUS)) - directory_get_from_dirserver(DIR_PURPOSE_FETCH_V2_NETWORKSTATUS, - ROUTER_PURPOSE_GENERAL, "all.z", - PDS_RETRY_IF_NO_SERVERS); - } -} - /** Return true iff, given the options listed in <b>options</b>, <b>flavor</b> * is the flavor of a consensus networkstatus that we would like to fetch. */ static int @@ -1214,8 +722,6 @@ we_want_to_fetch_flavor(const or_options_t *options, int flavor) return flavor == usable_consensus_flavor(); } -/** How many times will we try to fetch a consensus before we give up? */ -#define CONSENSUS_NETWORKSTATUS_MAX_DL_TRIES 8 /** How long will we hang onto a possibly live consensus for which we're * fetching certs before we check whether there is a better one? */ #define DELAY_WHILE_FETCHING_CERTS (20*60) @@ -1249,7 +755,7 @@ update_consensus_networkstatus_downloads(time_t now) resource = networkstatus_get_flavor_name(i); if (!download_status_is_ready(&consensus_dl_status[i], now, - CONSENSUS_NETWORKSTATUS_MAX_DL_TRIES)) + options->TestingConsensusMaxDownloadTries)) continue; /* We failed downloading a consensus too recently. */ if (connection_dir_get_by_purpose_and_resource( DIR_PURPOSE_FETCH_CONSENSUS, resource)) @@ -1324,7 +830,7 @@ update_consensus_networkstatus_fetch_time_impl(time_t now, int flav) if (directory_fetches_dir_info_early(options)) { /* We want to cache the next one at some point after this one * is no longer fresh... */ - start = c->fresh_until + min_sec_before_caching; + start = (time_t)(c->fresh_until + min_sec_before_caching); /* Some clients may need the consensus sooner than others. */ if (options->FetchDirInfoExtraEarly || authdir_mode_v3(options)) { dl_interval = 60; @@ -1337,7 +843,7 @@ update_consensus_networkstatus_fetch_time_impl(time_t now, int flav) } else { /* We're an ordinary client or a bridge. Give all the caches enough * time to download the consensus. */ - start = c->fresh_until + (interval*3)/4; + start = (time_t)(c->fresh_until + (interval*3)/4); /* But download the next one well before this one is expired. */ dl_interval = ((c->valid_until - start) * 7 )/ 8; @@ -1345,7 +851,7 @@ update_consensus_networkstatus_fetch_time_impl(time_t now, int flav) * to choose the rest of the interval *after* them. */ if (directory_fetches_dir_info_later(options)) { /* Give all the *clients* enough time to download the consensus. */ - start = start + dl_interval + min_sec_before_caching; + start = (time_t)(start + dl_interval + min_sec_before_caching); /* But try to get it before ours actually expires. */ dl_interval = (c->valid_until - start) - min_sec_before_caching; } @@ -1391,14 +897,45 @@ update_consensus_networkstatus_fetch_time(time_t now) /** Return 1 if there's a reason we shouldn't try any directory * fetches yet (e.g. we demand bridges and none are yet known). - * Else return 0. */ + * Else return 0. + + * If we return 1 and <b>msg_out</b> is provided, set <b>msg_out</b> + * to an explanation of why directory fetches are delayed. (If we + * return 0, we set msg_out to NULL.) + */ int -should_delay_dir_fetches(const or_options_t *options) +should_delay_dir_fetches(const or_options_t *options, const char **msg_out) { - if (options->UseBridges && !any_bridge_descriptors_known()) { - log_info(LD_DIR, "delaying dir fetches (no running bridges known)"); + if (msg_out) { + *msg_out = NULL; + } + + if (options->DisableNetwork) { + if (msg_out) { + *msg_out = "DisableNetwork is set."; + } + log_info(LD_DIR, "Delaying dir fetches (DisableNetwork is set)"); return 1; } + + if (options->UseBridges) { + if (!any_bridge_descriptors_known()) { + if (msg_out) { + *msg_out = "No running bridges"; + } + log_info(LD_DIR, "Delaying dir fetches (no running bridges known)"); + return 1; + } + + if (pt_proxies_configuration_pending()) { + if (msg_out) { + *msg_out = "Pluggable transport proxies still configuring"; + } + log_info(LD_DIR, "Delaying dir fetches (pt proxies still configuring)"); + return 1; + } + } + return 0; } @@ -1408,10 +945,8 @@ void update_networkstatus_downloads(time_t now) { const or_options_t *options = get_options(); - if (should_delay_dir_fetches(options)) + if (should_delay_dir_fetches(options, NULL)) return; - if (authdir_mode_any_main(options) || options->FetchV2Networkstatus) - update_v2_networkstatus_cache_downloads(now); update_consensus_networkstatus_downloads(now); update_certificate_downloads(now); } @@ -1518,7 +1053,6 @@ routerstatus_has_changed(const routerstatus_t *a, const routerstatus_t *b) a->is_named != b->is_named || a->is_unnamed != b->is_unnamed || a->is_valid != b->is_valid || - a->is_v2_dir != b->is_v2_dir || a->is_possible_guard != b->is_possible_guard || a->is_bad_exit != b->is_bad_exit || a->is_bad_directory != b->is_bad_directory || @@ -1740,7 +1274,11 @@ networkstatus_set_current_consensus(const char *consensus, /* Even if we had enough signatures, we'd never use this as the * latest consensus. */ if (was_waiting_for_certs && from_cache) - unlink(unverified_fname); + if (unlink(unverified_fname) != 0) { + log_warn(LD_FS, + "Failed to unlink %s: %s", + unverified_fname, strerror(errno)); + } } goto done; } else { @@ -1750,8 +1288,13 @@ networkstatus_set_current_consensus(const char *consensus, "consensus"); result = -2; } - if (was_waiting_for_certs && (r < -1) && from_cache) - unlink(unverified_fname); + if (was_waiting_for_certs && (r < -1) && from_cache) { + if (unlink(unverified_fname) != 0) { + log_warn(LD_FS, + "Failed to unlink %s: %s", + unverified_fname, strerror(errno)); + } + } goto done; } } @@ -1799,7 +1342,11 @@ networkstatus_set_current_consensus(const char *consensus, waiting->body = NULL; waiting->set_at = 0; waiting->dl_failed = 0; - unlink(unverified_fname); + if (unlink(unverified_fname) != 0) { + log_warn(LD_FS, + "Failed to unlink %s: %s", + unverified_fname, strerror(errno)); + } } /* Reset the failure count only if this consensus is actually valid. */ @@ -1835,7 +1382,8 @@ networkstatus_set_current_consensus(const char *consensus, * current consensus really alter our view of any OR's rate limits? */ connection_or_update_token_buckets(get_connection_array(), options); - circuit_build_times_new_consensus_params(&circ_times, current_consensus); + circuit_build_times_new_consensus_params(get_circuit_build_times_mutable(), + current_consensus); } if (directory_caches_dir_info(options)) { @@ -1912,9 +1460,6 @@ routers_update_all_from_networkstatus(time_t now, int dir_version) networkstatus_t *consensus = networkstatus_get_reasonably_live_consensus(now, FLAV_NS); - if (networkstatus_v2_list_has_changed) - download_status_map_update_from_v2_networkstatus(); - if (!consensus || dir_version < 3) /* nothing more we should do */ return; @@ -1969,35 +1514,6 @@ routers_update_all_from_networkstatus(time_t now, int dir_version) } } -/** Update v2_download_status_map to contain an entry for every router - * descriptor listed in the v2 networkstatuses. */ -static void -download_status_map_update_from_v2_networkstatus(void) -{ - digestmap_t *dl_status; - if (!networkstatus_v2_list) - return; - if (!v2_download_status_map) - v2_download_status_map = digestmap_new(); - - dl_status = digestmap_new(); - SMARTLIST_FOREACH_BEGIN(networkstatus_v2_list, networkstatus_v2_t *, ns) { - SMARTLIST_FOREACH_BEGIN(ns->entries, const routerstatus_t *, rs) { - const char *d = rs->descriptor_digest; - download_status_t *s; - if (digestmap_get(dl_status, d)) - continue; - if (!(s = digestmap_remove(v2_download_status_map, d))) { - s = tor_malloc_zero(sizeof(download_status_t)); - } - digestmap_set(dl_status, d, s); - } SMARTLIST_FOREACH_END(rs); - } SMARTLIST_FOREACH_END(ns); - digestmap_free(v2_download_status_map, tor_free_); - v2_download_status_map = dl_status; - networkstatus_v2_list_has_changed = 0; -} - /** Update our view of the list of named servers from the most recently * retrieved networkstatus consensus. */ static void @@ -2029,14 +1545,11 @@ void routers_update_status_from_consensus_networkstatus(smartlist_t *routers, int reset_failures) { - dir_server_t *ds; const or_options_t *options = get_options(); - int authdir = authdir_mode_v2(options) || authdir_mode_v3(options); + int authdir = authdir_mode_v3(options); networkstatus_t *ns = current_consensus; if (!ns || !smartlist_len(ns->routerstatus_list)) return; - if (!networkstatus_v2_list) - networkstatus_v2_list = smartlist_new(); routers_sort_by_identity(routers); @@ -2046,11 +1559,6 @@ routers_update_status_from_consensus_networkstatus(smartlist_t *routers, router->cache_info.identity_digest, DIGEST_LEN), { }) { - /* We have a routerstatus for this router. */ - const char *digest = router->cache_info.identity_digest; - - ds = router_get_fallback_dirserver_by_digest(digest); - /* Is it the same descriptor, or only the same identity? */ if (tor_memeq(router->cache_info.signed_descriptor_digest, rs->descriptor_digest, DIGEST_LEN)) { @@ -2068,30 +1576,11 @@ routers_update_status_from_consensus_networkstatus(smartlist_t *routers, dirserv_should_launch_reachability_test(router, old_router); } } - if (rs->is_flagged_running && ds) { - download_status_reset(&ds->v2_ns_dl_status); - } if (reset_failures) { download_status_reset(&rs->dl_status); } } SMARTLIST_FOREACH_JOIN_END(rs, router); - /* Now update last_listed_as_valid_until from v2 networkstatuses. */ - SMARTLIST_FOREACH_BEGIN(networkstatus_v2_list, networkstatus_v2_t *, ns) { - time_t live_until = ns->published_on + V2_NETWORKSTATUS_ROUTER_LIFETIME; - SMARTLIST_FOREACH_JOIN(ns->entries, const routerstatus_t *, rs, - routers, routerinfo_t *, ri, - tor_memcmp(rs->identity_digest, - ri->cache_info.identity_digest, DIGEST_LEN), - STMT_NIL) { - if (tor_memeq(ri->cache_info.signed_descriptor_digest, - rs->descriptor_digest, DIGEST_LEN)) { - if (live_until > ri->cache_info.last_listed_as_valid_until) - ri->cache_info.last_listed_as_valid_until = live_until; - } - } SMARTLIST_FOREACH_JOIN_END(rs, ri); - } SMARTLIST_FOREACH_END(ns); - router_dir_info_changed(); } @@ -2183,9 +1672,17 @@ networkstatus_dump_bridge_status_to_file(time_t now) char *status = networkstatus_getinfo_by_purpose("bridge", now); const or_options_t *options = get_options(); char *fname = NULL; + char *thresholds = NULL, *thresholds_and_status = NULL; + routerlist_t *rl = router_get_routerlist(); + dirserv_compute_bridge_flag_thresholds(rl); + thresholds = dirserv_get_flag_thresholds_line(); + tor_asprintf(&thresholds_and_status, "flag-thresholds %s\n%s", + thresholds, status); tor_asprintf(&fname, "%s"PATH_SEPARATOR"networkstatus-bridges", options->DataDirectory); - write_str_to_file(fname,status,0); + write_str_to_file(fname,thresholds_and_status,0); + tor_free(thresholds); + tor_free(thresholds_and_status); tor_free(fname); tor_free(status); } @@ -2402,15 +1899,6 @@ void networkstatus_free_all(void) { int i; - if (networkstatus_v2_list) { - SMARTLIST_FOREACH(networkstatus_v2_list, networkstatus_v2_t *, ns, - networkstatus_v2_free(ns)); - smartlist_free(networkstatus_v2_list); - networkstatus_v2_list = NULL; - } - - digestmap_free(v2_download_status_map, tor_free_); - v2_download_status_map = NULL; networkstatus_vote_free(current_ns_consensus); networkstatus_vote_free(current_md_consensus); current_md_consensus = current_ns_consensus = NULL; diff --git a/src/or/networkstatus.h b/src/or/networkstatus.h index 761f8e7f0e..be0a86cdd8 100644 --- a/src/or/networkstatus.h +++ b/src/or/networkstatus.h @@ -12,16 +12,10 @@ #ifndef TOR_NETWORKSTATUS_H #define TOR_NETWORKSTATUS_H -/** How old do we allow a v2 network-status to get before removing it - * completely? */ -#define MAX_NETWORKSTATUS_AGE (10*24*60*60) - void networkstatus_reset_warnings(void); void networkstatus_reset_download_failures(void); -int router_reload_v2_networkstatus(void); int router_reload_consensus_networkstatus(void); void routerstatus_free(routerstatus_t *rs); -void networkstatus_v2_free(networkstatus_v2_t *ns); void networkstatus_vote_free(networkstatus_t *ns); networkstatus_voter_info_t *networkstatus_get_voter_by_id( networkstatus_t *vote, @@ -31,26 +25,16 @@ int networkstatus_check_consensus_signature(networkstatus_t *consensus, int networkstatus_check_document_signature(const networkstatus_t *consensus, document_signature_t *sig, const authority_cert_t *cert); -char *networkstatus_get_cache_filename(const char *identity_digest); -int router_set_networkstatus_v2(const char *s, time_t arrived_at, - v2_networkstatus_source_t source, - smartlist_t *requested_fingerprints); -void networkstatus_v2_list_clean(time_t now); int compare_digest_to_routerstatus_entry(const void *_key, const void **_member); int compare_digest_to_vote_routerstatus_entry(const void *_key, const void **_member); -const routerstatus_t *networkstatus_v2_find_entry(networkstatus_v2_t *ns, - const char *digest); const routerstatus_t *networkstatus_vote_find_entry(networkstatus_t *ns, const char *digest); -routerstatus_t *networkstatus_v2_find_mutable_entry(networkstatus_v2_t *ns, - const char *digest); routerstatus_t *networkstatus_vote_find_mutable_entry(networkstatus_t *ns, const char *digest); int networkstatus_vote_find_entry_idx(networkstatus_t *ns, const char *digest, int *found_out); -const smartlist_t *networkstatus_get_v2_list(void); download_status_t *router_get_dl_status_by_descriptor_digest(const char *d); const routerstatus_t *router_get_consensus_status_by_id(const char *digest); routerstatus_t *router_get_mutable_consensus_status_by_id( @@ -69,7 +53,7 @@ int networkstatus_nickname_is_unnamed(const char *nickname); void networkstatus_consensus_download_failed(int status_code, const char *flavname); void update_consensus_networkstatus_fetch_time(time_t now); -int should_delay_dir_fetches(const or_options_t *options); +int should_delay_dir_fetches(const or_options_t *options,const char **msg_out); void update_networkstatus_downloads(time_t now); void update_certificate_downloads(time_t now); int consensus_is_waiting_for_certs(void); @@ -115,5 +99,9 @@ document_signature_t *document_signature_dup(const document_signature_t *sig); void networkstatus_free_all(void); int networkstatus_get_weight_scale_param(networkstatus_t *ns); +#ifdef NETWORKSTATUS_PRIVATE +STATIC void vote_routerstatus_free(vote_routerstatus_t *rs); +#endif + #endif diff --git a/src/or/nodelist.c b/src/or/nodelist.c index 178f084b69..7b1f338bd4 100644 --- a/src/or/nodelist.c +++ b/src/or/nodelist.c @@ -25,6 +25,8 @@ static void nodelist_drop_node(node_t *node, int remove_from_ht); static void node_free(node_t *node); static void update_router_have_minimum_dir_info(void); +static double get_frac_paths_needed_for_circs(const or_options_t *options, + const networkstatus_t *ns); /** A nodelist_t holds a node_t object for every router we're "willing to use * for something". Specifically, it should hold a node_t for every node that @@ -41,14 +43,7 @@ typedef struct nodelist_t { static INLINE unsigned int node_id_hash(const node_t *node) { -#if SIZEOF_INT == 4 - const uint32_t *p = (const uint32_t*)node->identity; - return p[0] ^ p[1] ^ p[2] ^ p[3] ^ p[4]; -#elif SIZEOF_INT == 8 - const uint64_t *p = (const uint32_t*)node->identity; - const uint32_t *p32 = (const uint32_t*)node->identity; - return p[0] ^ p[1] ^ p32[4]; -#endif + return (unsigned) siphash24g(node->identity, DIGEST_LEN); } static INLINE unsigned int @@ -90,8 +85,8 @@ node_get_mutable_by_id(const char *identity_digest) /** Return the node_t whose identity is <b>identity_digest</b>, or NULL * if no such node exists. */ -const node_t * -node_get_by_id(const char *identity_digest) +MOCK_IMPL(const node_t *, +node_get_by_id,(const char *identity_digest)) { return node_get_mutable_by_id(identity_digest); } @@ -211,7 +206,7 @@ void nodelist_set_consensus(networkstatus_t *ns) { const or_options_t *options = get_options(); - int authdir = authdir_mode_v2(options) || authdir_mode_v3(options); + int authdir = authdir_mode_v3(options); int client = !server_mode(options); init_nodelist(); @@ -337,6 +332,25 @@ nodelist_drop_node(node_t *node, int remove_from_ht) node->nodelist_idx = -1; } +/** Return a newly allocated smartlist of the nodes that have <b>md</b> as + * their microdescriptor. */ +smartlist_t * +nodelist_find_nodes_with_microdesc(const microdesc_t *md) +{ + smartlist_t *result = smartlist_new(); + + if (the_nodelist == NULL) + return result; + + SMARTLIST_FOREACH_BEGIN(the_nodelist->nodes, node_t *, node) { + if (node->md == md) { + smartlist_add(result, node); + } + } SMARTLIST_FOREACH_END(node); + + return result; +} + /** Release storage held by <b>node</b> */ static void node_free(node_t *node) @@ -644,7 +658,7 @@ node_get_purpose(const node_t *node) /** Compute the verbose ("extended") nickname of <b>node</b> and store it * into the MAX_VERBOSE_NICKNAME_LEN+1 character buffer at - * <b>verbose_nickname_out</b> */ + * <b>verbose_name_out</b> */ void node_get_verbose_nickname(const node_t *node, char *verbose_name_out) @@ -660,6 +674,25 @@ node_get_verbose_nickname(const node_t *node, strlcpy(verbose_name_out+1+HEX_DIGEST_LEN+1, nickname, MAX_NICKNAME_LEN+1); } +/** Compute the verbose ("extended") nickname of node with + * given <b>id_digest</b> and store it into the MAX_VERBOSE_NICKNAME_LEN+1 + * character buffer at <b>verbose_name_out</b> + * + * If node_get_by_id() returns NULL, base 16 encoding of + * <b>id_digest</b> is returned instead. */ +void +node_get_verbose_nickname_by_id(const char *id_digest, + char *verbose_name_out) +{ + const node_t *node = node_get_by_id(id_digest); + if (!node) { + verbose_name_out[0] = '$'; + base16_encode(verbose_name_out+1, HEX_DIGEST_LEN+1, id_digest, DIGEST_LEN); + } else { + node_get_verbose_nickname(node, verbose_name_out); + } +} + /** Return true iff it seems that <b>node</b> allows circuits to exit * through it directlry from the client. */ int @@ -771,7 +804,7 @@ void node_get_address_string(const node_t *node, char *buf, size_t len) { if (node->ri) { - strlcpy(buf, node->ri->address, len); + strlcpy(buf, fmt_addr32(node->ri->addr), len); } else if (node->rs) { tor_addr_t addr; tor_addr_from_ipv4h(&addr, node->rs->addr); @@ -1216,10 +1249,12 @@ router_set_status(const char *digest, int up) if (!up && node_is_me(node) && !net_is_disabled()) log_warn(LD_NET, "We just marked ourself as down. Are your external " "addresses reachable?"); + + if (bool_neq(node->is_running, up)) + router_dir_info_changed(); + node->is_running = up; } - - router_dir_info_changed(); } /** True iff, the last time we checked whether we had enough directory info @@ -1240,10 +1275,21 @@ static char dir_info_status[256] = ""; int router_have_minimum_dir_info(void) { + static int logged_delay=0; + const char *delay_fetches_msg = NULL; + if (should_delay_dir_fetches(get_options(), &delay_fetches_msg)) { + if (!logged_delay) + log_notice(LD_DIR, "Delaying directory fetches: %s", delay_fetches_msg); + logged_delay=1; + strlcpy(dir_info_status, delay_fetches_msg, sizeof(dir_info_status)); + return 0; + } + logged_delay = 0; /* reset it if we get this far */ + if (PREDICT_UNLIKELY(need_to_update_have_min_dir_info)) { update_router_have_minimum_dir_info(); - need_to_update_have_min_dir_info = 0; } + return have_min_dir_info; } @@ -1317,7 +1363,7 @@ count_usable_descriptors(int *num_present, int *num_usable, md ? "microdesc" : "desc", exit_only ? " exits" : "s"); } -/** Return an extimate of which fraction of usable paths through the Tor +/** Return an estimate of which fraction of usable paths through the Tor * network we have available for use. */ static double compute_frac_paths_available(const networkstatus_t *consensus, @@ -1329,9 +1375,10 @@ compute_frac_paths_available(const networkstatus_t *consensus, smartlist_t *mid = smartlist_new(); smartlist_t *exits = smartlist_new(); smartlist_t *myexits= smartlist_new(); - double f_guard, f_mid, f_exit, f_myexit; + smartlist_t *myexits_unflagged = smartlist_new(); + double f_guard, f_mid, f_exit, f_myexit, f_myexit_unflagged; int np, nu; /* Ignored */ - const int authdir = authdir_mode_v2(options) || authdir_mode_v3(options); + const int authdir = authdir_mode_v3(options); count_usable_descriptors(num_present_out, num_usable_out, mid, consensus, options, now, NULL, 0); @@ -1350,20 +1397,42 @@ compute_frac_paths_available(const networkstatus_t *consensus, }); } + /* All nodes with exit flag */ count_usable_descriptors(&np, &nu, exits, consensus, options, now, NULL, 1); + /* All nodes with exit flag in ExitNodes option */ count_usable_descriptors(&np, &nu, myexits, consensus, options, now, options->ExitNodes, 1); + /* Now compute the nodes in the ExitNodes option where which we don't know + * what their exit policy is, or we know it permits something. */ + count_usable_descriptors(&np, &nu, myexits_unflagged, + consensus, options, now, + options->ExitNodes, 0); + SMARTLIST_FOREACH_BEGIN(myexits_unflagged, const node_t *, node) { + if (node_has_descriptor(node) && node_exit_policy_rejects_all(node)) + SMARTLIST_DEL_CURRENT(myexits_unflagged, node); + } SMARTLIST_FOREACH_END(node); f_guard = frac_nodes_with_descriptors(guards, WEIGHT_FOR_GUARD); f_mid = frac_nodes_with_descriptors(mid, WEIGHT_FOR_MID); f_exit = frac_nodes_with_descriptors(exits, WEIGHT_FOR_EXIT); f_myexit= frac_nodes_with_descriptors(myexits,WEIGHT_FOR_EXIT); + f_myexit_unflagged= + frac_nodes_with_descriptors(myexits_unflagged,WEIGHT_FOR_EXIT); + + /* If our ExitNodes list has eliminated every possible Exit node, and there + * were some possible Exit nodes, then instead consider nodes that permit + * exiting to some ports. */ + if (smartlist_len(myexits) == 0 && + smartlist_len(myexits_unflagged)) { + f_myexit = f_myexit_unflagged; + } smartlist_free(guards); smartlist_free(mid); smartlist_free(exits); smartlist_free(myexits); + smartlist_free(myexits_unflagged); /* This is a tricky point here: we don't want to make it easy for a * directory to trickle exits to us until it learns which exits we have @@ -1372,13 +1441,14 @@ compute_frac_paths_available(const networkstatus_t *consensus, if (f_myexit < f_exit) f_exit = f_myexit; - tor_asprintf(status_out, - "%d%% of guards bw, " - "%d%% of midpoint bw, and " - "%d%% of exit bw", - (int)(f_guard*100), - (int)(f_mid*100), - (int)(f_exit*100)); + if (status_out) + tor_asprintf(status_out, + "%d%% of guards bw, " + "%d%% of midpoint bw, and " + "%d%% of exit bw", + (int)(f_guard*100), + (int)(f_mid*100), + (int)(f_exit*100)); return f_guard * f_mid * f_exit; } @@ -1391,19 +1461,19 @@ count_loading_descriptors_progress(void) { int num_present = 0, num_usable=0; time_t now = time(NULL); + const or_options_t *options = get_options(); const networkstatus_t *consensus = networkstatus_get_reasonably_live_consensus(now,usable_consensus_flavor()); - double fraction; + double paths, fraction; if (!consensus) return 0; /* can't count descriptors if we have no list of them */ - count_usable_descriptors(&num_present, &num_usable, NULL, - consensus, get_options(), now, NULL, 0); + paths = compute_frac_paths_available(consensus, options, now, + &num_present, &num_usable, + NULL); - if (num_usable == 0) - return 0; /* don't div by 0 */ - fraction = num_present / (num_usable/4.); + fraction = paths / get_frac_paths_needed_for_circs(options,consensus); if (fraction > 1.0) return 0; /* it's not the number of descriptors holding us back */ return BOOTSTRAP_STATUS_LOADING_DESCRIPTORS + (int) @@ -1451,14 +1521,6 @@ update_router_have_minimum_dir_info(void) goto done; } - if (should_delay_dir_fetches(get_options())) { - log_notice(LD_DIR, "no known bridge descriptors running yet; stalling"); - strlcpy(dir_info_status, "No live bridge descriptors.", - sizeof(dir_info_status)); - res = 0; - goto done; - } - using_md = consensus->flavor == FLAV_MICRODESC; { diff --git a/src/or/nodelist.h b/src/or/nodelist.h index 8a4665a8bf..8e719e012d 100644 --- a/src/or/nodelist.h +++ b/src/or/nodelist.h @@ -17,7 +17,7 @@ } STMT_END node_t *node_get_mutable_by_id(const char *identity_digest); -const node_t *node_get_by_id(const char *identity_digest); +MOCK_DECL(const node_t *, node_get_by_id, (const char *identity_digest)); const node_t *node_get_by_hex_id(const char *identity_digest); node_t *nodelist_set_routerinfo(routerinfo_t *ri, routerinfo_t **ri_old_out); node_t *nodelist_add_microdesc(microdesc_t *md); @@ -26,6 +26,7 @@ void nodelist_set_consensus(networkstatus_t *ns); void nodelist_remove_microdesc(const char *identity_digest, microdesc_t *md); void nodelist_remove_routerinfo(routerinfo_t *ri); void nodelist_purge(void); +smartlist_t *nodelist_find_nodes_with_microdesc(const microdesc_t *md); void nodelist_free_all(void); void nodelist_assert_ok(void); @@ -33,6 +34,8 @@ void nodelist_assert_ok(void); const node_t *node_get_by_nickname(const char *nickname, int warn_if_unnamed); void node_get_verbose_nickname(const node_t *node, char *verbose_name_out); +void node_get_verbose_nickname_by_id(const char *id_digest, + char *verbose_name_out); int node_is_named(const node_t *node); int node_is_dir(const node_t *node); int node_has_descriptor(const node_t *node); diff --git a/src/or/ntmain.c b/src/or/ntmain.c index 8b67b86822..e848314043 100644 --- a/src/or/ntmain.c +++ b/src/or/ntmain.c @@ -3,7 +3,6 @@ * Copyright (c) 2007-2013, The Tor Project, Inc. */ /* See LICENSE for licensing information */ -#define MAIN_PRIVATE #include "or.h" #include "config.h" #include "main.h" @@ -315,6 +314,7 @@ nt_service_main(void) case CMD_LIST_FINGERPRINT: case CMD_HASH_PASSWORD: case CMD_VERIFY_CONFIG: + case CMD_DUMP_CONFIG: log_err(LD_CONFIG, "Unsupported command (--list-fingerprint, " "--hash-password, or --verify-config) in NT service."); break; diff --git a/src/or/onion.c b/src/or/onion.c index 1a0bcf106e..ae39f451f4 100644 --- a/src/or/onion.c +++ b/src/or/onion.c @@ -22,7 +22,6 @@ #include "relay.h" #include "rephist.h" #include "router.h" -#include "tor_queue.h" /** Type for a linked list of circuits that are waiting for a free CPU worker * to process a waiting onion handshake. */ @@ -59,7 +58,7 @@ static void onion_queue_entry_remove(onion_queue_t *victim); * MAX_ONIONSKIN_CHALLENGE/REPLY_LEN." Also, make sure that we can pass * over-large values via EXTEND2/EXTENDED2, for future-compatibility.*/ -/** Return true iff we have room to queue another oninoskin of type +/** Return true iff we have room to queue another onionskin of type * <b>type</b>. */ static int have_room_for_onionskin(uint16_t type) @@ -330,12 +329,14 @@ onion_queue_entry_remove(onion_queue_t *victim) void clear_pending_onions(void) { - onion_queue_t *victim; + onion_queue_t *victim, *next; int i; for (i=0; i<=MAX_ONION_HANDSHAKE_TYPE; i++) { - while ((victim = TOR_TAILQ_FIRST(&ol_list[i]))) { + for (victim = TOR_TAILQ_FIRST(&ol_list[i]); victim; victim = next) { + next = TOR_TAILQ_NEXT(victim,next); onion_queue_entry_remove(victim); } + tor_assert(TOR_TAILQ_EMPTY(&ol_list[i])); } memset(ol_entries, 0, sizeof(ol_entries)); } @@ -553,8 +554,10 @@ onion_skin_client_handshake(int type, switch (type) { case ONION_HANDSHAKE_TYPE_TAP: - if (reply_len != TAP_ONIONSKIN_REPLY_LEN) + if (reply_len != TAP_ONIONSKIN_REPLY_LEN) { + log_warn(LD_CIRC, "TAP reply was not of the correct length."); return -1; + } if (onion_skin_TAP_client_handshake(handshake_state->u.tap, (const char*)reply, (char *)keys_out, keys_out_len) < 0) @@ -564,8 +567,10 @@ onion_skin_client_handshake(int type, return 0; case ONION_HANDSHAKE_TYPE_FAST: - if (reply_len != CREATED_FAST_LEN) + if (reply_len != CREATED_FAST_LEN) { + log_warn(LD_CIRC, "CREATED_FAST reply was not of the correct length."); return -1; + } if (fast_client_handshake(handshake_state->u.fast, reply, keys_out, keys_out_len) < 0) return -1; @@ -574,8 +579,10 @@ onion_skin_client_handshake(int type, return 0; #ifdef CURVE25519_ENABLED case ONION_HANDSHAKE_TYPE_NTOR: - if (reply_len < NTOR_REPLY_LEN) + if (reply_len < NTOR_REPLY_LEN) { + log_warn(LD_CIRC, "ntor reply was not of the correct length."); return -1; + } { size_t keys_tmp_len = keys_out_len + DIGEST_LEN; uint8_t *keys_tmp = tor_malloc(keys_tmp_len); @@ -861,16 +868,19 @@ extend_cell_parse(extend_cell_t *cell_out, const uint8_t command, } case RELAY_COMMAND_EXTEND2: { - uint8_t n_specs = *payload, spectype, speclen; + uint8_t n_specs, spectype, speclen; int i; int found_ipv4 = 0, found_ipv6 = 0, found_id = 0; tor_addr_make_unspec(&cell_out->orport_ipv4.addr); tor_addr_make_unspec(&cell_out->orport_ipv6.addr); + if (payload_length == 0) + return -1; + cell_out->cell_type = RELAY_COMMAND_EXTEND2; - ++payload; + n_specs = *payload++; /* Parse the specifiers. We'll only take the first IPv4 and first IPv6 - * addres, and the node ID, and ignore everything else */ + * address, and the node ID, and ignore everything else */ for (i = 0; i < n_specs; ++i) { if (eop - payload < 2) return -1; diff --git a/src/or/onion_fast.c b/src/or/onion_fast.c index aa034a8bd6..38b62decc3 100644 --- a/src/or/onion_fast.c +++ b/src/or/onion_fast.c @@ -22,7 +22,7 @@ fast_handshake_state_free(fast_handshake_state_t *victim) tor_free(victim); } -/** Create the state needed to perform a CREATE_FAST hasnshake. Return 0 +/** Create the state needed to perform a CREATE_FAST handshake. Return 0 * on success, -1 on failure. */ int fast_onionskin_create(fast_handshake_state_t **handshake_state_out, @@ -104,6 +104,7 @@ fast_client_handshake(const fast_handshake_state_t *handshake_state, out_len = key_out_len+DIGEST_LEN; out = tor_malloc(out_len); if (crypto_expand_key_material_TAP(tmp, sizeof(tmp), out, out_len)) { + log_warn(LD_CIRC, "Failed to expand key material"); goto done; } if (tor_memneq(out, handshake_reply_out+DIGEST_LEN, DIGEST_LEN)) { diff --git a/src/or/onion_ntor.c b/src/or/onion_ntor.c index 9cf7d5dd6e..ef501f69da 100644 --- a/src/or/onion_ntor.c +++ b/src/or/onion_ntor.c @@ -256,7 +256,7 @@ onion_skin_ntor_client_handshake( si += CURVE25519_OUTPUT_LEN; curve25519_handshake(si, &handshake_state->seckey_x, &handshake_state->pubkey_B); - bad |= safe_mem_is_zero(si, CURVE25519_OUTPUT_LEN); + bad |= (safe_mem_is_zero(si, CURVE25519_OUTPUT_LEN) << 1); si += CURVE25519_OUTPUT_LEN; APPEND(si, handshake_state->router_id, DIGEST_LEN); APPEND(si, handshake_state->pubkey_B.public_key, CURVE25519_PUBKEY_LEN); @@ -281,7 +281,7 @@ onion_skin_ntor_client_handshake( /* Compute auth */ h_tweak(s.auth, s.auth_input, sizeof(s.auth_input), T->t_mac); - bad |= tor_memneq(s.auth, auth_candidate, DIGEST256_LEN); + bad |= (tor_memneq(s.auth, auth_candidate, DIGEST256_LEN) << 2); crypto_expand_key_material_rfc5869_sha256( s.secret_input, sizeof(s.secret_input), @@ -290,6 +290,11 @@ onion_skin_ntor_client_handshake( key_out, key_out_len); memwipe(&s, 0, sizeof(s)); + + if (bad) { + log_warn(LD_PROTOCOL, "Invalid result from curve25519 handshake: %d", bad); + } + return bad ? -1 : 0; } diff --git a/src/or/onion_tap.c b/src/or/onion_tap.c index 3782e75abf..9a9f374b93 100644 --- a/src/or/onion_tap.c +++ b/src/or/onion_tap.c @@ -194,8 +194,10 @@ onion_skin_TAP_client_handshake(crypto_dh_t *handshake_state, len = crypto_dh_compute_secret(LOG_PROTOCOL_WARN, handshake_state, handshake_reply, DH_KEY_LEN, key_material, key_material_len); - if (len < 0) + if (len < 0) { + log_warn(LD_PROTOCOL,"DH computation failed."); goto err; + } if (tor_memneq(key_material, handshake_reply+DH_KEY_LEN, DIGEST_LEN)) { /* H(K) does *not* match. Something fishy. */ diff --git a/src/or/or.h b/src/or/or.h index 34f055cf06..131bce3e11 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -42,9 +42,6 @@ #include <sys/param.h> /* FreeBSD needs this to know what version it is */ #endif #include "torint.h" -#ifdef HAVE_SYS_WAIT_H -#include <sys/wait.h> -#endif #ifdef HAVE_SYS_FCNTL_H #include <sys/fcntl.h> #endif @@ -99,6 +96,7 @@ #include "ht.h" #include "replaycache.h" #include "crypto_curve25519.h" +#include "tor_queue.h" /* These signals are defined to help handle_control_signal work. */ @@ -195,6 +193,7 @@ typedef enum { * and let it use any circuit ID it wants. */ CIRC_ID_TYPE_NEITHER=2 } circ_id_type_t; +#define circ_id_type_bitfield_t ENUM_BF(circ_id_type_t) #define CONN_TYPE_MIN_ 3 /** Type for sockets listening for OR connections. */ @@ -227,8 +226,14 @@ typedef enum { #define CONN_TYPE_AP_NATD_LISTENER 14 /** Type for sockets listening for DNS requests. */ #define CONN_TYPE_AP_DNS_LISTENER 15 -#define CONN_TYPE_MAX_ 15 -/* !!!! If CONN_TYPE_MAX_ is ever over 15, we must grow the type field in + +/** Type for connections from the Extended ORPort. */ +#define CONN_TYPE_EXT_OR 16 +/** Type for sockets listening for Extended ORPort connections. */ +#define CONN_TYPE_EXT_OR_LISTENER 17 + +#define CONN_TYPE_MAX_ 17 +/* !!!! If _CONN_TYPE_MAX is ever over 31, we must grow the type field in * connection_t. */ /* Proxy client types */ @@ -238,7 +243,9 @@ typedef enum { #define PROXY_SOCKS5 3 /* !!!! If there is ever a PROXY_* type over 2, we must grow the proxy_type * field in or_connection_t */ -/* pluggable transports proxy type */ + +/* Pluggable transport proxy type. Don't use this in or_connection_t, + * instead use the actual underlying proxy type (see above). */ #define PROXY_PLUGGABLE 4 /* Proxy client handshake states */ @@ -306,6 +313,25 @@ typedef enum { #define OR_CONN_STATE_OPEN 8 #define OR_CONN_STATE_MAX_ 8 +/** States of the Extended ORPort protocol. Be careful before changing + * the numbers: they matter. */ +#define EXT_OR_CONN_STATE_MIN_ 1 +/** Extended ORPort authentication is waiting for the authentication + * type selected by the client. */ +#define EXT_OR_CONN_STATE_AUTH_WAIT_AUTH_TYPE 1 +/** Extended ORPort authentication is waiting for the client nonce. */ +#define EXT_OR_CONN_STATE_AUTH_WAIT_CLIENT_NONCE 2 +/** Extended ORPort authentication is waiting for the client hash. */ +#define EXT_OR_CONN_STATE_AUTH_WAIT_CLIENT_HASH 3 +#define EXT_OR_CONN_STATE_AUTH_MAX 3 +/** Authentication finished and the Extended ORPort is now accepting + * traffic. */ +#define EXT_OR_CONN_STATE_OPEN 4 +/** Extended ORPort is flushing its last messages and preparing to + * start accepting OR connections. */ +#define EXT_OR_CONN_STATE_FLUSHING 5 +#define EXT_OR_CONN_STATE_MAX_ 5 + #define EXIT_CONN_STATE_MIN_ 1 /** State for an exit connection: waiting for response from DNS farm. */ #define EXIT_CONN_STATE_RESOLVING 1 @@ -372,16 +398,10 @@ typedef enum { #define CONTROL_CONN_STATE_NEEDAUTH 2 #define CONTROL_CONN_STATE_MAX_ 2 -#define DIR_PURPOSE_MIN_ 3 -/** A connection to a directory server: download a rendezvous - * descriptor. */ -#define DIR_PURPOSE_FETCH_RENDDESC 3 -/** A connection to a directory server: set after a rendezvous +#define DIR_PURPOSE_MIN_ 4 +/** A connection to a directory server: set after a v2 rendezvous * descriptor is downloaded. */ -#define DIR_PURPOSE_HAS_FETCHED_RENDDESC 4 -/** A connection to a directory server: download one or more v2 - * network-status objects */ -#define DIR_PURPOSE_FETCH_V2_NETWORKSTATUS 5 +#define DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2 4 /** A connection to a directory server: download one or more server * descriptors. */ #define DIR_PURPOSE_FETCH_SERVERDESC 6 @@ -390,9 +410,6 @@ typedef enum { #define DIR_PURPOSE_FETCH_EXTRAINFO 7 /** A connection to a directory server: upload a server descriptor. */ #define DIR_PURPOSE_UPLOAD_DIR 8 -/** A connection to a directory server: upload a rendezvous - * descriptor. */ -#define DIR_PURPOSE_UPLOAD_RENDDESC 9 /** A connection to a directory server: upload a v3 networkstatus vote. */ #define DIR_PURPOSE_UPLOAD_VOTE 10 /** A connection to a directory server: upload a v3 consensus signature */ @@ -426,7 +443,6 @@ typedef enum { * directory server. */ #define DIR_PURPOSE_IS_UPLOAD(p) \ ((p)==DIR_PURPOSE_UPLOAD_DIR || \ - (p)==DIR_PURPOSE_UPLOAD_RENDDESC || \ (p)==DIR_PURPOSE_UPLOAD_VOTE || \ (p)==DIR_PURPOSE_UPLOAD_SIGNATURES) @@ -585,7 +601,8 @@ typedef enum { #define END_OR_CONN_REASON_NO_ROUTE 6 /* no route to host/net */ #define END_OR_CONN_REASON_IO_ERROR 7 /* read/write error */ #define END_OR_CONN_REASON_RESOURCE_LIMIT 8 /* sockets, buffers, etc */ -#define END_OR_CONN_REASON_MISC 9 +#define END_OR_CONN_REASON_PT_MISSING 9 /* PT failed or not available */ +#define END_OR_CONN_REASON_MISC 10 /* Reasons why we (or a remote OR) might close a stream. See tor-spec.txt for * documentation of these. The values must match. */ @@ -823,9 +840,15 @@ typedef enum { /** Maximum number of queued cells on a circuit for which we are the * midpoint before we give up and kill it. This must be >= circwindow * to avoid killing innocent circuits, and >= circwindow*2 to give - * leaky-pipe a chance for being useful someday. + * leaky-pipe a chance of working someday. The ORCIRC_MAX_MIDDLE_KILL_THRESH + * ratio controls the margin of error between emitting a warning and + * killing the circuit. + */ +#define ORCIRC_MAX_MIDDLE_CELLS (CIRCWINDOW_START_MAX*2) +/** Ratio of hard (circuit kill) to soft (warning) thresholds for the + * ORCIRC_MAX_MIDDLE_CELLS tests. */ -#define ORCIRC_MAX_MIDDLE_CELLS (21*(CIRCWINDOW_START_MAX)/10) +#define ORCIRC_MAX_MIDDLE_KILL_THRESH (1.1f) /* Cell commands. These values are defined in tor-spec.txt. */ #define CELL_PADDING 0 @@ -846,6 +869,7 @@ typedef enum { #define CELL_AUTH_CHALLENGE 130 #define CELL_AUTHENTICATE 131 #define CELL_AUTHORIZE 132 +#define CELL_COMMAND_MAX_ 132 /** How long to test reachability before complaining to the user. */ #define TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT (20*60) @@ -1073,9 +1097,17 @@ typedef struct var_cell_t { uint8_t payload[FLEXIBLE_ARRAY_MEMBER]; } var_cell_t; +/** A parsed Extended ORPort message. */ +typedef struct ext_or_cmd_t { + uint16_t cmd; /** Command type */ + uint16_t len; /** Body length */ + char body[FLEXIBLE_ARRAY_MEMBER]; /** Message body */ +} ext_or_cmd_t; + /** A cell as packed for writing to the network. */ typedef struct packed_cell_t { - struct packed_cell_t *next; /**< Next cell queued on this circuit. */ + /** Next cell queued on this circuit. */ + TOR_SIMPLEQ_ENTRY(packed_cell_t) next; char body[CELL_MAX_NETWORK_SIZE]; /**< Cell as packed for network. */ uint32_t inserted_time; /**< Time (in milliseconds since epoch, with high * bits truncated) when this cell was inserted. */ @@ -1084,8 +1116,8 @@ typedef struct packed_cell_t { /** A queue of cells on a circuit, waiting to be added to the * or_connection_t's outbuf. */ typedef struct cell_queue_t { - packed_cell_t *head; /**< The first cell, or NULL if the queue is empty. */ - packed_cell_t *tail; /**< The last cell, or NULL if the queue is empty. */ + /** Linked list of packed_cell_t*/ + TOR_SIMPLEQ_HEAD(cell_simpleq, packed_cell_t) head; int n; /**< The number of cells in the queue. */ } cell_queue_t; @@ -1139,7 +1171,7 @@ typedef struct connection_t { * *_CONNECTION_MAGIC. */ uint8_t state; /**< Current state of this connection. */ - unsigned int type:4; /**< What kind of connection is this? */ + unsigned int type:5; /**< What kind of connection is this? */ unsigned int purpose:5; /**< Only used for DIR and EXIT types currently. */ /* The next fields are all one-bit booleans. Some are only applicable to @@ -1223,6 +1255,14 @@ typedef struct connection_t { /** Unique identifier for this connection on this Tor instance. */ uint64_t global_identifier; + + /** Bytes read since last call to control_event_conn_bandwidth_used(). + * Only used if we're configured to emit CONN_BW events. */ + uint32_t n_read_conn_bw; + + /** Bytes written since last call to control_event_conn_bandwidth_used(). + * Only used if we're configured to emit CONN_BW events. */ + uint32_t n_written_conn_bw; } connection_t; /** Subtype of connection_t; used for a listener socket. */ @@ -1384,6 +1424,9 @@ typedef struct or_handshake_state_t { /**@}*/ } or_handshake_state_t; +/** Length of Extended ORPort connection identifier. */ +#define EXT_OR_CONN_ID_LEN DIGEST_LEN /* 20 */ + /** Subtype of connection_t for an "OR connection" -- that is, one that speaks * cells over TLS. */ typedef struct or_connection_t { @@ -1392,6 +1435,20 @@ typedef struct or_connection_t { /** Hash of the public RSA key for the other side's identity key, or zeroes * if the other side hasn't shown us a valid identity key. */ char identity_digest[DIGEST_LEN]; + + /** Extended ORPort connection identifier. */ + char *ext_or_conn_id; + /** This is the ClientHash value we expect to receive from the + * client during the Extended ORPort authentication protocol. We + * compute it upon receiving the ClientNoce from the client, and we + * compare it with the acual ClientHash value sent by the + * client. */ + char *ext_or_auth_correct_client_hash; + /** String carrying the name of the pluggable transport + * (e.g. "obfs2") that is obfuscating this connection. If no + * pluggable transports are used, it's NULL. */ + char *ext_or_transport; + char *nickname; /**< Nickname of OR on other side (if any). */ tor_tls_t *tls; /**< TLS connection state. */ @@ -1422,15 +1479,20 @@ typedef struct or_connection_t { unsigned int is_outgoing:1; unsigned int proxy_type:2; /**< One of PROXY_NONE...PROXY_SOCKS5 */ unsigned int wide_circ_ids:1; + /** True iff this connection has had its bootstrap failure logged with + * control_event_bootstrap_problem. */ + unsigned int have_noted_bootstrap_problem:1; + uint16_t link_proto; /**< What protocol version are we using? 0 for * "none negotiated yet." */ - + uint16_t idle_timeout; /**< How long can this connection sit with no + * circuits on it before we close it? Based on + * IDLE_CIRCUIT_TIMEOUT_{NON,}CANONICAL and + * on is_canonical, randomized. */ or_handshake_state_t *handshake_state; /**< If we are setting this connection * up, state information to do so. */ time_t timestamp_lastempty; /**< When was the outbuf last completely empty?*/ - time_t timestamp_last_added_nonpadding; /** When did we last add a - * non-padding cell to the outbuf? */ /* bandwidth* and *_bucket only used by ORs in OPEN state: */ int bandwidthrate; /**< Bytes/s added to the bucket. (OPEN ORs only.) */ @@ -1449,6 +1511,12 @@ typedef struct or_connection_t { struct or_connection_t *next_with_same_id; /**< Next connection with same * identity digest as this one. */ + /** Last emptied read token bucket in msec since midnight; only used if + * TB_EMPTY events are enabled. */ + uint32_t read_emptied_time; + /** Last emptied write token bucket in msec since midnight; only used if + * TB_EMPTY events are enabled. */ + uint32_t write_emptied_time; } or_connection_t; /** Subtype of connection_t for an "edge connection" -- that is, an entry (ap) @@ -1619,6 +1687,7 @@ typedef enum { DIR_SPOOL_CACHED_DIR, DIR_SPOOL_NETWORKSTATUS, DIR_SPOOL_MICRODESC, /* NOTE: if we add another entry, add another bit. */ } dir_spool_source_t; +#define dir_spool_source_bitfield_t ENUM_BF(dir_spool_source_t) /** Subtype of connection_t for an "directory connection" -- that is, an HTTP * connection to retrieve or serve directory material. */ @@ -1638,7 +1707,7 @@ typedef struct dir_connection_t { * "spooling" of directory material to the outbuf. Otherwise, we'd have * to append everything to the outbuf in one enormous chunk. */ /** What exactly are we spooling right now? */ - ENUM_BF(dir_spool_source_t) dir_spool_src : 3; + dir_spool_source_bitfield_t dir_spool_src : 3; /** If we're fetching descriptors, what router purpose shall we assign * to them? */ @@ -1811,12 +1880,13 @@ typedef enum { ADDR_POLICY_ACCEPT=1, ADDR_POLICY_REJECT=2, } addr_policy_action_t; +#define addr_policy_action_bitfield_t ENUM_BF(addr_policy_action_t) /** A reference-counted address policy rule. */ typedef struct addr_policy_t { int refcnt; /**< Reference count */ /** What to do when the policy matches.*/ - ENUM_BF(addr_policy_action_t) policy_type:2; + addr_policy_action_bitfield_t policy_type:2; unsigned int is_private:1; /**< True iff this is the pseudo-address, * "private". */ unsigned int is_canonical:1; /**< True iff this policy is the canonical @@ -1868,6 +1938,7 @@ typedef enum { */ SAVED_IN_JOURNAL } saved_location_t; +#define saved_location_bitfield_t ENUM_BF(saved_location_t) /** Enumeration: what kind of download schedule are we using for a given * object? */ @@ -1876,6 +1947,7 @@ typedef enum { DL_SCHED_CONSENSUS = 1, DL_SCHED_BRIDGE = 2, } download_schedule_t; +#define download_schedule_bitfield_t ENUM_BF(download_schedule_t) /** Information about our plans for retrying downloads for a downloadable * object. */ @@ -1884,7 +1956,7 @@ typedef struct download_status_t { * again? */ uint8_t n_download_failures; /**< Number of failures trying to download the * most recent descriptor. */ - ENUM_BF(download_schedule_t) schedule : 8; + download_schedule_bitfield_t schedule : 8; } download_status_t; /** If n_download_failures is this high, the download can never happen. */ @@ -1926,9 +1998,7 @@ typedef struct signed_descriptor_t { * routerlist->old_routers? -1 for none. */ int routerlist_index; /** The valid-until time of the most recent consensus that listed this - * descriptor, or a bit after the publication time of the most recent v2 - * networkstatus that listed it. 0 for "never listed in a consensus or - * status, so far as we know." */ + * descriptor. 0 for "never listed in a consensus, so far as we know." */ time_t last_listed_as_valid_until; /* If true, we do not ever try to save this object in the cache. */ unsigned int do_not_cache : 1; @@ -1947,7 +2017,6 @@ typedef int16_t country_t; /** Information about another onion router in the network. */ typedef struct { signed_descriptor_t cache_info; - char *address; /**< Location of OR: either a hostname or an IP address. */ char *nickname; /**< Human-readable OR name. */ uint32_t addr; /**< IPv4 address of OR, in host order. */ @@ -2065,10 +2134,6 @@ typedef struct routerstatus_t { unsigned int is_unnamed:1; /**< True iff "nickname" belongs to another * router. */ unsigned int is_valid:1; /**< True iff this router isn't invalid. */ - unsigned int is_v2_dir:1; /**< True iff this router can serve directory - * information with v2 of the directory - * protocol. (All directory caches cache v1 - * directories.) */ unsigned int is_possible_guard:1; /**< True iff this router would be a good * choice as an entry guard. */ unsigned int is_bad_exit:1; /**< True iff this node is a bad choice for @@ -2105,12 +2170,6 @@ typedef struct routerstatus_t { /* ---- The fields below aren't derived from the networkstatus; they * hold local information only. */ - /** True if we, as a directory mirror, want to download the corresponding - * routerinfo from the authority who gave us this routerstatus. (That is, - * if we don't have the routerinfo, and if we haven't already tried to get it - * from this authority.) Applies in v2 networkstatus document only. - */ - unsigned int need_to_mirror:1; time_t last_dir_503_at; /**< When did this router last tell us that it * was too busy to serve directory info? */ download_status_t dl_status; @@ -2152,7 +2211,7 @@ typedef struct microdesc_t { */ time_t last_listed; /** Where is this microdescriptor currently stored? */ - ENUM_BF(saved_location_t) saved_location : 3; + saved_location_bitfield_t saved_location : 3; /** If true, do not attempt to cache this microdescriptor on disk. */ unsigned int no_save : 1; /** If true, this microdesc has an entry in the microdesc_map */ @@ -2275,52 +2334,6 @@ typedef struct node_t { } node_t; -/** How many times will we try to download a router's descriptor before giving - * up? */ -#define MAX_ROUTERDESC_DOWNLOAD_FAILURES 8 - -/** How many times will we try to download a microdescriptor before giving - * up? */ -#define MAX_MICRODESC_DOWNLOAD_FAILURES 8 - -/** Contents of a v2 (non-consensus, non-vote) network status object. */ -typedef struct networkstatus_v2_t { - /** When did we receive the network-status document? */ - time_t received_on; - - /** What was the digest of the document? */ - char networkstatus_digest[DIGEST_LEN]; - - /* These fields come from the actual network-status document.*/ - time_t published_on; /**< Declared publication date. */ - - char *source_address; /**< Canonical directory server hostname. */ - uint32_t source_addr; /**< Canonical directory server IP. */ - uint16_t source_dirport; /**< Canonical directory server dirport. */ - - unsigned int binds_names:1; /**< True iff this directory server binds - * names. */ - unsigned int recommends_versions:1; /**< True iff this directory server - * recommends client and server software - * versions. */ - unsigned int lists_bad_exits:1; /**< True iff this directory server marks - * malfunctioning exits as bad. */ - /** True iff this directory server marks malfunctioning directories as - * bad. */ - unsigned int lists_bad_directories:1; - - char identity_digest[DIGEST_LEN]; /**< Digest of signing key. */ - char *contact; /**< How to contact directory admin? (may be NULL). */ - crypto_pk_t *signing_key; /**< Key used to sign this directory. */ - char *client_versions; /**< comma-separated list of recommended client - * versions. */ - char *server_versions; /**< comma-separated list of recommended server - * versions. */ - - smartlist_t *entries; /**< List of routerstatus_t*. This list is kept - * sorted by identity_digest. */ -} networkstatus_v2_t; - /** Linked list of microdesc hash lines for a single router in a directory * vote. */ @@ -2408,8 +2421,8 @@ typedef enum { /** A common structure to hold a v3 network status vote, or a v3 network * status consensus. */ typedef struct networkstatus_t { - ENUM_BF(networkstatus_type_t) type : 8; /**< Vote, consensus, or opinion? */ - ENUM_BF(consensus_flavor_t) flavor : 8; /**< If a consensus, what kind? */ + networkstatus_type_t type; /**< Vote, consensus, or opinion? */ + consensus_flavor_t flavor; /**< If a consensus, what kind? */ unsigned int has_measured_bws : 1;/**< True iff this networkstatus contains * measured= bandwidth values. */ @@ -2492,10 +2505,6 @@ typedef struct desc_store_t { * filename for a temporary file when rebuilding the store, and .new to this * filename for the journal. */ const char *fname_base; - /** Alternative (obsolete) value for fname_base: if the file named by - * fname_base isn't present, we read from here instead, but we never write - * here. */ - const char *fname_alt_base; /** Human-readable description of what this store contains. */ const char *description; @@ -2572,9 +2581,6 @@ typedef struct authority_cert_t { uint32_t addr; /** This authority's directory port. */ uint16_t dir_port; - /** True iff this certificate was cross-certified by signing the identity - * key with the signing key. */ - uint8_t is_cross_certified; } authority_cert_t; /** Bitfield enum type listing types of information that directory authorities @@ -2588,15 +2594,8 @@ typedef struct authority_cert_t { */ typedef enum { NO_DIRINFO = 0, - /** Serves/signs v1 directory information: Big lists of routers, and short - * routerstatus documents. */ - V1_DIRINFO = 1 << 0, - /** Serves/signs v2 directory information: i.e. v2 networkstatus documents */ - V2_DIRINFO = 1 << 1, /** Serves/signs v3 directory information: votes, consensuses, certs */ V3_DIRINFO = 1 << 2, - /** Serves hidden service descriptors. */ - HIDSERV_DIRINFO = 1 << 3, /** Serves bridge descriptors. */ BRIDGE_DIRINFO = 1 << 4, /** Serves extrainfo documents. */ @@ -2724,6 +2723,19 @@ typedef struct { struct create_cell_t; +/** Entry in the cell stats list of a circuit; used only if CELL_STATS + * events are enabled. */ +typedef struct testing_cell_stats_entry_t { + uint8_t command; /**< cell command number. */ + /** Waiting time in centiseconds if this event is for a removed cell, + * or 0 if this event is for adding a cell to the queue. 22 bits can + * store more than 11 hours, enough to assume that a circuit with this + * delay would long have been closed. */ + unsigned int waiting_time:22; + unsigned int removed:1; /**< 0 for added to, 1 for removed from queue. */ + unsigned int exitward:1; /**< 0 for app-ward, 1 for exit-ward. */ +} testing_cell_stats_entry_t; + /** * A circuit is a path over the onion routing * network. Applications can connect to one end of the circuit, and can @@ -2785,6 +2797,13 @@ typedef struct circuit_t { * allowing n_streams to add any more cells. (OR circuit only.) */ unsigned int streams_blocked_on_p_chan : 1; + /** True iff we have queued a delete backwards on this circuit, but not put + * it on the output buffer. */ + unsigned int p_delete_pending : 1; + /** True iff we have queued a delete forwards on this circuit, but not put + * it on the output buffer. */ + unsigned int n_delete_pending : 1; + /** True iff this circuit has received a DESTROY cell in either direction */ unsigned int received_destroy : 1; @@ -2801,6 +2820,9 @@ typedef struct circuit_t { * more. */ int deliver_window; + /** Temporary field used during circuits_handle_oom. */ + uint32_t age_tmp; + /** For storage while n_chan is pending (state CIRCUIT_STATE_CHAN_WAIT). */ struct create_cell_t *n_chan_create_cell; @@ -2842,7 +2864,8 @@ typedef struct circuit_t { /** Unique ID for measuring tunneled network status requests. */ uint64_t dirreq_id; - struct circuit_t *next; /**< Next circuit in linked list of all circuits. */ + /** Next circuit in linked list of all circuits (global_circuitlist). */ + TOR_LIST_ENTRY(circuit_t) head; /** Next circuit in the doubly-linked ring of circuits waiting to add * cells to n_conn. NULL if we have no cells pending, or if we're not @@ -2852,6 +2875,11 @@ typedef struct circuit_t { * cells to n_conn. NULL if we have no cells pending, or if we're not * linked to an OR connection. */ struct circuit_t *prev_active_on_n_chan; + + /** Various statistics about cells being added to or removed from this + * circuit's queues; used only if CELL_STATS events are enabled and + * cleared after being sent to control port. */ + smartlist_t *testing_cell_stats; } circuit_t; /** Largest number of relay_early cells that we can send on a given @@ -2913,6 +2941,7 @@ typedef enum { */ PATH_STATE_ALREADY_COUNTED = 6, } path_state_t; +#define path_state_bitfield_t ENUM_BF(path_state_t) /** An origin_circuit_t holds data necessary to build and use a circuit. */ @@ -2922,6 +2951,17 @@ typedef struct origin_circuit_t { /** Linked list of AP streams (or EXIT streams if hidden service) * associated with this circuit. */ edge_connection_t *p_streams; + + /** Bytes read from any attached stream since last call to + * control_event_circ_bandwidth_used(). Only used if we're configured + * to emit CIRC_BW events. */ + uint32_t n_read_circ_bw; + + /** Bytes written to any attached stream since last call to + * control_event_circ_bandwidth_used(). Only used if we're configured + * to emit CIRC_BW events. */ + uint32_t n_written_circ_bw; + /** Build state for this circuit. It includes the intended path * length, the chosen exit router, rendezvous information, etc. */ @@ -2952,7 +2992,7 @@ typedef struct origin_circuit_t { * circuit building and usage accounting. See path_state_t * for more details. */ - ENUM_BF(path_state_t) path_state : 3; + path_state_bitfield_t path_state : 3; /* If this flag is set, we should not consider attaching any more * connections to this circuit. */ @@ -3136,20 +3176,8 @@ typedef struct or_circuit_t { * is not marked for close. */ struct or_circuit_t *rend_splice; -#if REND_COOKIE_LEN >= DIGEST_LEN -#define REND_TOKEN_LEN REND_COOKIE_LEN -#else -#define REND_TOKEN_LEN DIGEST_LEN -#endif + struct or_circuit_rendinfo_s *rendinfo; - /** A hash of location-hidden service's PK if purpose is INTRO_POINT, or a - * rendezvous cookie if purpose is REND_POINT_WAITING. Filled with zeroes - * otherwise. - * ???? move to a subtype or adjunct structure? Wastes 20 bytes. -NM - */ - char rend_token[REND_TOKEN_LEN]; - - /* ???? move to a subtype or adjunct structure? Wastes 20 bytes -NM */ /** Stores KH for the handshake. */ char rend_circ_nonce[DIGEST_LEN];/* KH in tor-spec.txt */ @@ -3168,28 +3196,66 @@ typedef struct or_circuit_t { * exit-ward queues of this circuit; reset every time when writing * buffer stats to disk. */ uint64_t total_cell_waiting_time; + + /** Maximum cell queue size for a middle relay; this is stored per circuit + * so append_cell_to_circuit_queue() can adjust it if it changes. If set + * to zero, it is initialized to the default value. + */ + uint32_t max_middle_cells; } or_circuit_t; +typedef struct or_circuit_rendinfo_s { + +#if REND_COOKIE_LEN != DIGEST_LEN +#error "The REND_TOKEN_LEN macro assumes REND_COOKIE_LEN == DIGEST_LEN" +#endif +#define REND_TOKEN_LEN DIGEST_LEN + + /** A hash of location-hidden service's PK if purpose is INTRO_POINT, or a + * rendezvous cookie if purpose is REND_POINT_WAITING. Filled with zeroes + * otherwise. + */ + char rend_token[REND_TOKEN_LEN]; + + /** True if this is a rendezvous point circuit; false if this is an + * introduction point. */ + unsigned is_rend_circ; + +} or_circuit_rendinfo_t; + /** Convert a circuit subtype to a circuit_t. */ #define TO_CIRCUIT(x) (&((x)->base_)) /** Convert a circuit_t* to a pointer to the enclosing or_circuit_t. Assert * if the cast is impossible. */ static or_circuit_t *TO_OR_CIRCUIT(circuit_t *); +static const or_circuit_t *CONST_TO_OR_CIRCUIT(const circuit_t *); /** Convert a circuit_t* to a pointer to the enclosing origin_circuit_t. * Assert if the cast is impossible. */ static origin_circuit_t *TO_ORIGIN_CIRCUIT(circuit_t *); +static const origin_circuit_t *CONST_TO_ORIGIN_CIRCUIT(const circuit_t *); static INLINE or_circuit_t *TO_OR_CIRCUIT(circuit_t *x) { tor_assert(x->magic == OR_CIRCUIT_MAGIC); return DOWNCAST(or_circuit_t, x); } +static INLINE const or_circuit_t *CONST_TO_OR_CIRCUIT(const circuit_t *x) +{ + tor_assert(x->magic == OR_CIRCUIT_MAGIC); + return DOWNCAST(or_circuit_t, x); +} static INLINE origin_circuit_t *TO_ORIGIN_CIRCUIT(circuit_t *x) { tor_assert(x->magic == ORIGIN_CIRCUIT_MAGIC); return DOWNCAST(origin_circuit_t, x); } +static INLINE const origin_circuit_t *CONST_TO_ORIGIN_CIRCUIT( + const circuit_t *x) +{ + tor_assert(x->magic == ORIGIN_CIRCUIT_MAGIC); + return DOWNCAST(origin_circuit_t, x); +} /** Bitfield type: things that we're willing to use invalid routers for. */ typedef enum invalid_router_usage_t { @@ -3326,9 +3392,9 @@ typedef struct { /** What should the tor process actually do? */ enum { CMD_RUN_TOR=0, CMD_LIST_FINGERPRINT, CMD_HASH_PASSWORD, - CMD_VERIFY_CONFIG, CMD_RUN_UNITTESTS + CMD_VERIFY_CONFIG, CMD_RUN_UNITTESTS, CMD_DUMP_CONFIG } command; - const char *command_arg; /**< Argument for command-line option. */ + char *command_arg; /**< Argument for command-line option. */ config_line_t *Logs; /**< New-style list of configuration lines * for logs */ @@ -3409,10 +3475,21 @@ typedef struct { char *User; /**< Name of user to run Tor as. */ char *Group; /**< Name of group to run Tor as. */ config_line_t *ORPort_lines; /**< Ports to listen on for OR connections. */ + /** Ports to listen on for extended OR connections. */ + config_line_t *ExtORPort_lines; /** Ports to listen on for SOCKS connections. */ config_line_t *SocksPort_lines; /** Ports to listen on for transparent pf/netfilter connections. */ config_line_t *TransPort_lines; + const char *TransProxyType; /**< What kind of transparent proxy + * implementation are we using? */ + /** Parsed value of TransProxyType. */ + enum { + TPT_DEFAULT, + TPT_PF_DIVERT, + TPT_IPFW, + TPT_TPROXY, + } TransProxyType_parsed; config_line_t *NATDPort_lines; /**< Ports to listen on for transparent natd * connections. */ config_line_t *ControlPort_lines; /**< Ports to listen on for control @@ -3425,9 +3502,11 @@ typedef struct { config_line_t *DirPort_lines; config_line_t *DNSPort_lines; /**< Ports to listen on for DNS requests. */ - uint64_t MaxMemInCellQueues; /**< If we have more memory than this allocated - * for circuit cell queues, run the OOM handler - */ + /* MaxMemInQueues value as input by the user. We clean this up to be + * MaxMemInQueues. */ + uint64_t MaxMemInQueues_raw; + uint64_t MaxMemInQueues;/**< If we have more memory than this allocated + * for queues and buffers, run the OOM handler */ /** @name port booleans * @@ -3444,18 +3523,13 @@ typedef struct { unsigned int ControlPort_set : 1; unsigned int DirPort_set : 1; unsigned int DNSPort_set : 1; + unsigned int ExtORPort_set : 1; /**@}*/ int AssumeReachable; /**< Whether to publish our descriptor regardless. */ int AuthoritativeDir; /**< Boolean: is this an authoritative directory? */ - int V1AuthoritativeDir; /**< Boolean: is this an authoritative directory - * for version 1 directories? */ - int V2AuthoritativeDir; /**< Boolean: is this an authoritative directory - * for version 2 directories? */ int V3AuthoritativeDir; /**< Boolean: is this an authoritative directory * for version 3 directories? */ - int HSAuthoritativeDir; /**< Boolean: does this an authoritative directory - * handle hidden service requests? */ int NamingAuthoritativeDir; /**< Boolean: is this an authoritative directory * that's willing to bind names? */ int VersioningAuthoritativeDir; /**< Boolean: is this an authoritative @@ -3483,6 +3557,9 @@ typedef struct { /** List of TCP/IP addresses that transports should listen at. */ config_line_t *ServerTransportListenAddr; + /** List of options that must be passed to pluggable transports. */ + config_line_t *ServerTransportOptions; + int BridgeRelay; /**< Boolean: are we acting as a bridge relay? We make * this explicit so we can change how we behave in the * future. */ @@ -3503,8 +3580,6 @@ typedef struct { int PublishHidServDescriptors; int FetchServerDescriptors; /**< Do we fetch server descriptors as normal? */ int FetchHidServDescriptors; /**< and hidden service descriptors? */ - int FetchV2Networkstatus; /**< Do we fetch v2 networkstatus documents when - * we don't need to? */ int HidServDirectoryV2; /**< Do we participate in the HS DHT? */ int VoteOnHidServDirectoriesV2; /**< As a directory authority, vote on @@ -3595,6 +3670,10 @@ typedef struct { * a new one? */ int MaxCircuitDirtiness; /**< Never use circs that were first used more than this interval ago. */ + int PredictedPortsRelevanceTime; /** How long after we've requested a + * connection for a given port, do we want + * to continue to pick exits that support + * that port? */ uint64_t BandwidthRate; /**< How much bandwidth, on average, are we willing * to use in a second? */ uint64_t BandwidthBurst; /**< How much bandwidth, at maximum, are we willing @@ -3658,9 +3737,6 @@ typedef struct { /** If set, use these bridge authorities and not the default one. */ config_line_t *AlternateBridgeAuthority; - /** If set, use these HS authorities and not the default ones. */ - config_line_t *AlternateHSAuthority; - char *MyFamily; /**< Declared family for this OR. */ config_line_t *NodeFamilies; /**< List of config lines for * node families */ @@ -3720,7 +3796,10 @@ typedef struct { int CookieAuthentication; /**< Boolean: do we enable cookie-based auth for * the control system? */ - char *CookieAuthFile; /**< Location of a cookie authentication file. */ + char *CookieAuthFile; /**< Filesystem location of a ControlPort + * authentication cookie. */ + char *ExtORPortCookieAuthFile; /**< Filesystem location of Extended + * ORPort authentication cookie. */ int CookieAuthFileGroupReadable; /**< Boolean: Is the CookieAuthFile g+r? */ int LeaveStreamsUnattached; /**< Boolean: Does Tor attach new streams to * circuits itself (0), or does it expect a controller @@ -3742,6 +3821,7 @@ typedef struct { SAFELOG_SCRUB_ALL, SAFELOG_SCRUB_RELAY, SAFELOG_SCRUB_NONE } SafeLogging_; + int Sandbox; /**< Boolean: should sandboxing be enabled? */ int SafeSocks; /**< Boolean: should we outright refuse application * connections that use socks4 or socks5-with-local-dns? */ #define LOG_PROTOCOL_WARN (get_options()->ProtocolWarnings ? \ @@ -3804,10 +3884,6 @@ typedef struct { * testing our DNS server. */ int EnforceDistinctSubnets; /**< If true, don't allow multiple routers in the * same network zone in the same circuit. */ - int TunnelDirConns; /**< If true, use BEGIN_DIR rather than BEGIN when - * possible. */ - int PreferTunneledDirConns; /**< If true, avoid dirservers that don't - * support BEGIN_DIR, when possible. */ int PortForwarding; /**< If true, use NAT-PMP or UPnP to automatically * forward the DirPort and ORPort on the NAT device */ char *PortForwardingHelper; /** < Filename or full path of the port @@ -3913,6 +3989,10 @@ typedef struct { * signatures. Only altered on testing networks.*/ int TestingV3AuthInitialDistDelay; + /** Offset in seconds added to the starting time for consensus + voting. Only altered on testing networks. */ + int TestingV3AuthVotingStartOffset; + /** If an authority has been around for less than this amount of time, it * does not believe its reachability information is accurate. Only * altered on testing networks. */ @@ -3923,6 +4003,51 @@ typedef struct { * networks. */ int TestingEstimatedDescriptorPropagationTime; + /** Schedule for when servers should download things in general. Only + * altered on testing networks. */ + smartlist_t *TestingServerDownloadSchedule; + + /** Schedule for when clients should download things in general. Only + * altered on testing networks. */ + smartlist_t *TestingClientDownloadSchedule; + + /** Schedule for when servers should download consensuses. Only altered + * on testing networks. */ + smartlist_t *TestingServerConsensusDownloadSchedule; + + /** Schedule for when clients should download consensuses. Only altered + * on testing networks. */ + smartlist_t *TestingClientConsensusDownloadSchedule; + + /** Schedule for when clients should download bridge descriptors. Only + * altered on testing networks. */ + smartlist_t *TestingBridgeDownloadSchedule; + + /** When directory clients have only a few descriptors to request, they + * batch them until they have more, or until this amount of time has + * passed. Only altered on testing networks. */ + int TestingClientMaxIntervalWithoutRequest; + + /** How long do we let a directory connection stall before expiring + * it? Only altered on testing networks. */ + int TestingDirConnectionMaxStall; + + /** How many times will we try to fetch a consensus before we give + * up? Only altered on testing networks. */ + int TestingConsensusMaxDownloadTries; + + /** How many times will we try to download a router's descriptor before + * giving up? Only altered on testing networks. */ + int TestingDescriptorMaxDownloadTries; + + /** How many times will we try to download a microdescriptor before + * giving up? Only altered on testing networks. */ + int TestingMicrodescMaxDownloadTries; + + /** How many times will we try to fetch a certificate before giving + * up? Only altered on testing networks. */ + int TestingCertMaxDownloadTries; + /** If true, we take part in a testing network. Change the defaults of a * couple of other configuration options and allow to change the values * of certain configuration options. */ @@ -3934,6 +4059,19 @@ typedef struct { /** Minimum value for the Fast flag threshold on testing networks. */ uint64_t TestingMinFastFlagThreshold; + /** Relays in a testing network which should be voted Guard + * regardless of uptime and bandwidth. */ + routerset_t *TestingDirAuthVoteGuard; + + /** Enable CONN_BW events. Only altered on testing networks. */ + int TestingEnableConnBwEvent; + + /** Enable CELL_STATS events. Only altered on testing networks. */ + int TestingEnableCellStatsEvent; + + /** Enable TB_EMPTY events. Only altered on testing networks. */ + int TestingEnableTbEmptyEvent; + /** If true, and we have GeoIP data, and we're a bridge, keep a per-country * count of how many client addresses have contacted us so that we can help * the bridge authority guess which countries have blocked access to us. */ @@ -4072,16 +4210,6 @@ typedef struct { /** Fraction: */ double PathsNeededToBuildCircuits; - /** Do we serve v2 directory info at all? This is a temporary option, since - * we'd like to disable v2 directory serving entirely, but we need a way to - * make it temporarily disableable, in order to do fast testing and be - * able to turn it back on if it turns out to be non-workable. - * - * XXXX025 Make this always-on, or always-off. Right now, it's only - * enableable for authorities. - */ - int DisableV2DirectoryInfo_; - /** What expiry time shall we place on our SSL certs? "0" means we * should guess a suitable value. */ int SSLKeyLifetime; @@ -4346,30 +4474,7 @@ typedef struct { int after_firsthop_idx; } network_liveness_t; -/** Structure for circuit build times history */ -typedef struct { - /** The circular array of recorded build times in milliseconds */ - build_time_t circuit_build_times[CBT_NCIRCUITS_TO_OBSERVE]; - /** Current index in the circuit_build_times circular array */ - int build_times_idx; - /** Total number of build times accumulated. Max CBT_NCIRCUITS_TO_OBSERVE */ - int total_build_times; - /** Information about the state of our local network connection */ - network_liveness_t liveness; - /** Last time we built a circuit. Used to decide to build new test circs */ - time_t last_circ_at; - /** "Minimum" value of our pareto distribution (actually mode) */ - build_time_t Xm; - /** alpha exponent for pareto dist. */ - double alpha; - /** Have we computed a timeout? */ - int have_computed_timeout; - /** The exact value for that timeout in milliseconds. Stored as a double - * to maintain precision from calculations to and from quantile value. */ - double timeout_ms; - /** How long we wait before actually closing the circuit. */ - double close_ms; -} circuit_build_times_t; +typedef struct circuit_build_times_s circuit_build_times_t; /********************************* config.c ***************************/ @@ -4406,6 +4511,7 @@ typedef enum { * did this remapping happen." */ ADDRMAPSRC_NONE } addressmap_entry_source_t; +#define addressmap_entry_source_bitfield_t ENUM_BF(addressmap_entry_source_t) /********************************* control.c ***************************/ @@ -4557,8 +4663,6 @@ typedef enum { GEOIP_CLIENT_CONNECT = 0, /** We've served a networkstatus consensus as a directory server. */ GEOIP_CLIENT_NETWORKSTATUS = 1, - /** We've served a v2 networkstatus consensus as a directory server. */ - GEOIP_CLIENT_NETWORKSTATUS_V2 = 2, } geoip_client_action_t; /** Indicates either a positive reply or a reason for rejectng a network * status request that will be included in geoip statistics. */ @@ -4616,11 +4720,6 @@ typedef struct microdesc_cache_t microdesc_cache_t; /********************************* networkstatus.c *********************/ -/** Location where we found a v2 networkstatus. */ -typedef enum { - NS_FROM_CACHE, NS_FROM_DIR_BY_FP, NS_FROM_DIR_ALL, NS_GENERATED -} v2_networkstatus_source_t; - /** Possible statuses of a version of Tor, given opinions from the directory * servers. */ typedef enum version_status_t { @@ -4771,9 +4870,9 @@ typedef struct rend_service_descriptor_t { crypto_pk_t *pk; /**< This service's public key. */ int version; /**< Version of the descriptor format: 0 or 2. */ time_t timestamp; /**< Time when the descriptor was generated. */ - /** Bitmask: which rendezvous protocols are supported? - * (We allow bits '0', '1', and '2' to be set.) */ - int protocols : REND_PROTOCOL_VERSION_BITMASK_WIDTH; + /** Bitmask: which introduce/rendezvous protocols are supported? + * (We allow bits '0', '1', '2' and '3' to be set.) */ + unsigned protocols : REND_PROTOCOL_VERSION_BITMASK_WIDTH; /** List of the service's introduction points. Elements are removed if * introduction attempts fail. */ smartlist_t *intro_nodes; @@ -4821,8 +4920,6 @@ typedef struct dir_server_t { /** What kind of authority is this? (Bitfield.) */ dirinfo_type_t type; - download_status_t v2_ns_dl_status; /**< Status of downloading this server's - * v2 network status. */ time_t addr_current_at; /**< When was the document that we derived the * address information from published? */ @@ -4871,8 +4968,6 @@ typedef struct dir_server_t { * node that's currently a guard. */ #define PDS_FOR_GUARD (1<<5) -#define PDS_PREFER_TUNNELED_DIR_CONNS_ (1<<16) - /** Possible ways to weight routers when choosing one randomly. See * routerlist_sl_choose_by_bandwidth() for more information.*/ typedef enum bandwidth_weight_rule_t { diff --git a/src/or/policies.c b/src/or/policies.c index be4da55061..8a91509a77 100644 --- a/src/or/policies.c +++ b/src/or/policies.c @@ -13,6 +13,7 @@ #include "dirserv.h" #include "nodelist.h" #include "policies.h" +#include "router.h" #include "routerparse.h" #include "geoip.h" #include "ht.h" @@ -438,7 +439,7 @@ validate_addr_policies(const or_options_t *options, char **msg) if (policies_parse_exit_policy(options->ExitPolicy, &addr_policy, options->IPv6Exit, - options->ExitPolicyRejectPrivate, NULL, + options->ExitPolicyRejectPrivate, 0, !options->BridgeRelay)) REJECT("Error in ExitPolicy entry."); @@ -482,10 +483,12 @@ validate_addr_policies(const or_options_t *options, char **msg) * Ignore port specifiers. */ static int -load_policy_from_option(config_line_t *config, smartlist_t **policy, +load_policy_from_option(config_line_t *config, const char *option_name, + smartlist_t **policy, int assume_action) { int r; + int killed_any_ports = 0; addr_policy_list_free(*policy); *policy = NULL; r = parse_addr_policy(config, policy, assume_action); @@ -504,9 +507,13 @@ load_policy_from_option(config_line_t *config, smartlist_t **policy, c = addr_policy_get_canonical_entry(&newp); SMARTLIST_REPLACE_CURRENT(*policy, n, c); addr_policy_free(n); + killed_any_ports = 1; } } SMARTLIST_FOREACH_END(n); } + if (killed_any_ports) { + log_warn(LD_CONFIG, "Ignoring ports in %s option.", option_name); + } return 0; } @@ -516,20 +523,22 @@ int policies_parse_from_options(const or_options_t *options) { int ret = 0; - if (load_policy_from_option(options->SocksPolicy, &socks_policy, -1) < 0) + if (load_policy_from_option(options->SocksPolicy, "SocksPolicy", + &socks_policy, -1) < 0) ret = -1; - if (load_policy_from_option(options->DirPolicy, &dir_policy, -1) < 0) + if (load_policy_from_option(options->DirPolicy, "DirPolicy", + &dir_policy, -1) < 0) ret = -1; - if (load_policy_from_option(options->AuthDirReject, + if (load_policy_from_option(options->AuthDirReject, "AuthDirReject", &authdir_reject_policy, ADDR_POLICY_REJECT) < 0) ret = -1; - if (load_policy_from_option(options->AuthDirInvalid, + if (load_policy_from_option(options->AuthDirInvalid, "AuthDirInvalid", &authdir_invalid_policy, ADDR_POLICY_REJECT) < 0) ret = -1; - if (load_policy_from_option(options->AuthDirBadDir, + if (load_policy_from_option(options->AuthDirBadDir, "AuthDirBadDir", &authdir_baddir_policy, ADDR_POLICY_REJECT) < 0) ret = -1; - if (load_policy_from_option(options->AuthDirBadExit, + if (load_policy_from_option(options->AuthDirBadExit, "AuthDirBadExit", &authdir_badexit_policy, ADDR_POLICY_REJECT) < 0) ret = -1; if (parse_reachable_addresses() < 0) @@ -597,21 +606,25 @@ policy_eq(policy_map_ent_t *a, policy_map_ent_t *b) /** Return a hashcode for <b>ent</b> */ static unsigned int -policy_hash(policy_map_ent_t *ent) +policy_hash(const policy_map_ent_t *ent) { - addr_policy_t *a = ent->policy; - unsigned int r; - if (a->is_private) - r = 0x1234abcd; - else - r = tor_addr_hash(&a->addr); - r += a->prt_min << 8; - r += a->prt_max << 16; - r += a->maskbits; - if (a->policy_type == ADDR_POLICY_REJECT) - r ^= 0xffffffff; + const addr_policy_t *a = ent->policy; + addr_policy_t aa; + memset(&aa, 0, sizeof(aa)); + + aa.prt_min = a->prt_min; + aa.prt_max = a->prt_max; + aa.maskbits = a->maskbits; + aa.policy_type = a->policy_type; + aa.is_private = a->is_private; + + if (a->is_private) { + aa.is_private = 1; + } else { + tor_addr_copy_tight(&aa.addr, &a->addr); + } - return r; + return (unsigned) siphash24g(&aa, sizeof(aa)); } HT_PROTOTYPE(policy_map, policy_map_ent_t, node, policy_hash, @@ -958,7 +971,7 @@ exit_policy_remove_redundancies(smartlist_t *dest) int policies_parse_exit_policy(config_line_t *cfg, smartlist_t **dest, int ipv6_exit, - int rejectprivate, const char *local_address, + int rejectprivate, uint32_t local_address, int add_default_policy) { if (!ipv6_exit) { @@ -968,7 +981,7 @@ policies_parse_exit_policy(config_line_t *cfg, smartlist_t **dest, append_exit_policy_string(dest, "reject private:*"); if (local_address) { char buf[POLICY_BUF_LEN]; - tor_snprintf(buf, sizeof(buf), "reject %s:*", local_address); + tor_snprintf(buf, sizeof(buf), "reject %s:*", fmt_addr32(local_address)); append_exit_policy_string(dest, buf); } } @@ -1680,6 +1693,28 @@ getinfo_helper_policies(control_connection_t *conn, (void) errmsg; if (!strcmp(question, "exit-policy/default")) { *answer = tor_strdup(DEFAULT_EXIT_POLICY); + } else if (!strcmpstart(question, "exit-policy/")) { + const routerinfo_t *me = router_get_my_routerinfo(); + + int include_ipv4 = 0; + int include_ipv6 = 0; + + if (!strcmp(question, "exit-policy/ipv4")) { + include_ipv4 = 1; + } else if (!strcmp(question, "exit-policy/ipv6")) { + include_ipv6 = 1; + } else if (!strcmp(question, "exit-policy/full")) { + include_ipv4 = include_ipv6 = 1; + } else { + return 0; /* No such key. */ + } + + if (!me) { + *errmsg = "router_get_my_routerinfo returned NULL"; + return -1; + } + + *answer = router_dump_exit_policy_to_string(me,include_ipv4,include_ipv6); } return 0; } diff --git a/src/or/policies.h b/src/or/policies.h index facbbb6b5a..91ac427492 100644 --- a/src/or/policies.h +++ b/src/or/policies.h @@ -45,7 +45,7 @@ addr_policy_result_t compare_tor_addr_to_node_policy(const tor_addr_t *addr, int policies_parse_exit_policy(config_line_t *cfg, smartlist_t **dest, int ipv6exit, - int rejectprivate, const char *local_address, + int rejectprivate, uint32_t local_address, int add_default_policy); void policies_exit_policy_append_reject_star(smartlist_t **dest); void addr_policy_append_reject_addr(smartlist_t **dest, diff --git a/src/or/reasons.c b/src/or/reasons.c index 0674474e72..750e89bbe7 100644 --- a/src/or/reasons.c +++ b/src/or/reasons.c @@ -231,6 +231,8 @@ orconn_end_reason_to_control_string(int r) return "RESOURCELIMIT"; case END_OR_CONN_REASON_MISC: return "MISC"; + case END_OR_CONN_REASON_PT_MISSING: + return "PT_MISSING"; case 0: return ""; default: diff --git a/src/or/relay.c b/src/or/relay.c index 7f06c6e145..f42602d412 100644 --- a/src/or/relay.c +++ b/src/or/relay.c @@ -15,6 +15,7 @@ #include "addressmap.h" #include "buffers.h" #include "channel.h" +#include "circpathbias.h" #include "circuitbuild.h" #include "circuitlist.h" #include "circuituse.h" @@ -25,7 +26,9 @@ #include "control.h" #include "geoip.h" #include "main.h" +#ifdef ENABLE_MEMPOOLS #include "mempool.h" +#endif #include "networkstatus.h" #include "nodelist.h" #include "onion.h" @@ -58,6 +61,9 @@ static void adjust_exit_policy_from_exitpolicy_failure(origin_circuit_t *circ, entry_connection_t *conn, node_t *node, const tor_addr_t *addr); +#if 0 +static int get_max_middle_cells(void); +#endif /** Stop reading on edge connections when we have this many cells * waiting on the appropriate queue. */ @@ -105,14 +111,14 @@ relay_set_digest(crypto_digest_t *digest, cell_t *cell) static int relay_digest_matches(crypto_digest_t *digest, cell_t *cell) { - char received_integrity[4], calculated_integrity[4]; + uint32_t received_integrity, calculated_integrity; relay_header_t rh; crypto_digest_t *backup_digest=NULL; backup_digest = crypto_digest_dup(digest); relay_header_unpack(&rh, cell->payload); - memcpy(received_integrity, rh.integrity, 4); + memcpy(&received_integrity, rh.integrity, 4); memset(rh.integrity, 0, 4); relay_header_pack(cell->payload, &rh); @@ -121,15 +127,15 @@ relay_digest_matches(crypto_digest_t *digest, cell_t *cell) // received_integrity[2], received_integrity[3]); crypto_digest_add_bytes(digest, (char*) cell->payload, CELL_PAYLOAD_SIZE); - crypto_digest_get_digest(digest, calculated_integrity, 4); + crypto_digest_get_digest(digest, (char*) &calculated_integrity, 4); - if (tor_memneq(received_integrity, calculated_integrity, 4)) { + if (calculated_integrity != received_integrity) { // log_fn(LOG_INFO,"Recognized=0 but bad digest. Not recognizing."); // (%d vs %d).", received_integrity, calculated_integrity); /* restore digest to its old form */ crypto_digest_assign(digest, backup_digest); /* restore the relay header */ - memcpy(rh.integrity, received_integrity, 4); + memcpy(rh.integrity, &received_integrity, 4); relay_header_pack(cell->payload, &rh); crypto_digest_free(backup_digest); return 0; @@ -968,7 +974,7 @@ remap_event_helper(entry_connection_t *conn, const tor_addr_t *new_addr) * <b>addr_out</b> to the address we're connected to, and <b>ttl_out</b> to * the ttl of that address, in seconds, and return 0. On failure, return * -1. */ -int +STATIC int connected_cell_parse(const relay_header_t *rh, const cell_t *cell, tor_addr_t *addr_out, int *ttl_out) { @@ -1005,6 +1011,254 @@ connected_cell_parse(const relay_header_t *rh, const cell_t *cell, return 0; } +/** Drop all storage held by <b>addr</b>. */ +STATIC void +address_ttl_free(address_ttl_t *addr) +{ + if (!addr) + return; + tor_free(addr->hostname); + tor_free(addr); +} + +/** Parse a resolved cell in <b>cell</b>, with parsed header in <b>rh</b>. + * Return -1 on parse error. On success, add one or more newly allocated + * address_ttl_t to <b>addresses_out</b>; set *<b>errcode_out</b> to + * one of 0, RESOLVED_TYPE_ERROR, or RESOLVED_TYPE_ERROR_TRANSIENT, and + * return 0. */ +STATIC int +resolved_cell_parse(const cell_t *cell, const relay_header_t *rh, + smartlist_t *addresses_out, int *errcode_out) +{ + const uint8_t *cp; + uint8_t answer_type; + size_t answer_len; + address_ttl_t *addr; + size_t remaining; + int errcode = 0; + smartlist_t *addrs; + + tor_assert(cell); + tor_assert(rh); + tor_assert(addresses_out); + tor_assert(errcode_out); + + *errcode_out = 0; + + if (rh->length > RELAY_PAYLOAD_SIZE) + return -1; + + addrs = smartlist_new(); + + cp = cell->payload + RELAY_HEADER_SIZE; + + remaining = rh->length; + while (remaining) { + const uint8_t *cp_orig = cp; + if (remaining < 2) + goto err; + answer_type = *cp++; + answer_len = *cp++; + if (remaining < 2 + answer_len + 4) { + goto err; + } + if (answer_type == RESOLVED_TYPE_IPV4) { + if (answer_len != 4) { + goto err; + } + addr = tor_malloc_zero(sizeof(*addr)); + tor_addr_from_ipv4n(&addr->addr, get_uint32(cp)); + cp += 4; + addr->ttl = ntohl(get_uint32(cp)); + cp += 4; + smartlist_add(addrs, addr); + } else if (answer_type == RESOLVED_TYPE_IPV6) { + if (answer_len != 16) + goto err; + addr = tor_malloc_zero(sizeof(*addr)); + tor_addr_from_ipv6_bytes(&addr->addr, (const char*) cp); + cp += 16; + addr->ttl = ntohl(get_uint32(cp)); + cp += 4; + smartlist_add(addrs, addr); + } else if (answer_type == RESOLVED_TYPE_HOSTNAME) { + if (answer_len == 0) { + goto err; + } + addr = tor_malloc_zero(sizeof(*addr)); + addr->hostname = tor_memdup_nulterm(cp, answer_len); + cp += answer_len; + addr->ttl = ntohl(get_uint32(cp)); + cp += 4; + smartlist_add(addrs, addr); + } else if (answer_type == RESOLVED_TYPE_ERROR_TRANSIENT || + answer_type == RESOLVED_TYPE_ERROR) { + errcode = answer_type; + /* Ignore the error contents */ + cp += answer_len + 4; + } else { + cp += answer_len + 4; + } + tor_assert(((ssize_t)remaining) >= (cp - cp_orig)); + remaining -= (cp - cp_orig); + } + + if (errcode && smartlist_len(addrs) == 0) { + /* Report an error only if there were no results. */ + *errcode_out = errcode; + } + + smartlist_add_all(addresses_out, addrs); + smartlist_free(addrs); + + return 0; + + err: + /* On parse error, don't report any results */ + SMARTLIST_FOREACH(addrs, address_ttl_t *, a, address_ttl_free(a)); + smartlist_free(addrs); + return -1; +} + +/** Helper for connection_edge_process_resolved_cell: given an error code, + * an entry_connection, and a list of address_ttl_t *, report the best answer + * to the entry_connection. */ +static void +connection_ap_handshake_socks_got_resolved_cell(entry_connection_t *conn, + int error_code, + smartlist_t *results) +{ + address_ttl_t *addr_ipv4 = NULL; + address_ttl_t *addr_ipv6 = NULL; + address_ttl_t *addr_hostname = NULL; + address_ttl_t *addr_best = NULL; + + /* If it's an error code, that's easy. */ + if (error_code) { + tor_assert(error_code == RESOLVED_TYPE_ERROR || + error_code == RESOLVED_TYPE_ERROR_TRANSIENT); + connection_ap_handshake_socks_resolved(conn, + error_code,0,NULL,-1,-1); + return; + } + + /* Get the first answer of each type. */ + SMARTLIST_FOREACH_BEGIN(results, address_ttl_t *, addr) { + if (addr->hostname) { + if (!addr_hostname) { + addr_hostname = addr; + } + } else if (tor_addr_family(&addr->addr) == AF_INET) { + if (!addr_ipv4 && conn->ipv4_traffic_ok) { + addr_ipv4 = addr; + } + } else if (tor_addr_family(&addr->addr) == AF_INET6) { + if (!addr_ipv6 && conn->ipv6_traffic_ok) { + addr_ipv6 = addr; + } + } + } SMARTLIST_FOREACH_END(addr); + + /* Now figure out which type we wanted to deliver. */ + if (conn->socks_request->command == SOCKS_COMMAND_RESOLVE_PTR) { + if (addr_hostname) { + connection_ap_handshake_socks_resolved(conn, + RESOLVED_TYPE_HOSTNAME, + strlen(addr_hostname->hostname), + (uint8_t*)addr_hostname->hostname, + addr_hostname->ttl,-1); + } else { + connection_ap_handshake_socks_resolved(conn, + RESOLVED_TYPE_ERROR,0,NULL,-1,-1); + } + return; + } + + if (conn->prefer_ipv6_traffic) { + addr_best = addr_ipv6 ? addr_ipv6 : addr_ipv4; + } else { + addr_best = addr_ipv4 ? addr_ipv4 : addr_ipv6; + } + + /* Now convert it to the ugly old interface */ + if (! addr_best) { + connection_ap_handshake_socks_resolved(conn, + RESOLVED_TYPE_ERROR,0,NULL,-1,-1); + return; + } + + connection_ap_handshake_socks_resolved_addr(conn, + &addr_best->addr, + addr_best->ttl, + -1); + + remap_event_helper(conn, &addr_best->addr); +} + +/** Handle a RELAY_COMMAND_RESOLVED cell that we received on a non-open AP + * stream. */ +STATIC int +connection_edge_process_resolved_cell(edge_connection_t *conn, + const cell_t *cell, + const relay_header_t *rh) +{ + entry_connection_t *entry_conn = EDGE_TO_ENTRY_CONN(conn); + smartlist_t *resolved_addresses = NULL; + int errcode = 0; + + if (conn->base_.state != AP_CONN_STATE_RESOLVE_WAIT) { + log_fn(LOG_PROTOCOL_WARN, LD_APP, "Got a 'resolved' cell while " + "not in state resolve_wait. Dropping."); + return 0; + } + tor_assert(SOCKS_COMMAND_IS_RESOLVE(entry_conn->socks_request->command)); + + resolved_addresses = smartlist_new(); + if (resolved_cell_parse(cell, rh, resolved_addresses, &errcode)) { + log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, + "Dropping malformed 'resolved' cell"); + connection_mark_unattached_ap(entry_conn, END_STREAM_REASON_TORPROTOCOL); + goto done; + } + + if (get_options()->ClientDNSRejectInternalAddresses) { + int orig_len = smartlist_len(resolved_addresses); + SMARTLIST_FOREACH_BEGIN(resolved_addresses, address_ttl_t *, addr) { + if (addr->hostname == NULL && tor_addr_is_internal(&addr->addr, 0)) { + log_info(LD_APP, "Got a resolved cell with answer %s; dropping that " + "answer.", + safe_str_client(fmt_addr(&addr->addr))); + address_ttl_free(addr); + SMARTLIST_DEL_CURRENT(resolved_addresses, addr); + } + } SMARTLIST_FOREACH_END(addr); + if (orig_len && smartlist_len(resolved_addresses) == 0) { + log_info(LD_APP, "Got a resolved cell with only private addresses; " + "dropping it."); + connection_ap_handshake_socks_resolved(entry_conn, + RESOLVED_TYPE_ERROR_TRANSIENT, + 0, NULL, 0, TIME_MAX); + connection_mark_unattached_ap(entry_conn, + END_STREAM_REASON_TORPROTOCOL); + goto done; + } + } + + connection_ap_handshake_socks_got_resolved_cell(entry_conn, + errcode, + resolved_addresses); + + connection_mark_unattached_ap(entry_conn, + END_STREAM_REASON_DONE | + END_STREAM_REASON_FLAG_ALREADY_SOCKS_REPLIED); + + done: + SMARTLIST_FOREACH(resolved_addresses, address_ttl_t *, addr, + address_ttl_free(addr)); + smartlist_free(resolved_addresses); + return 0; +} + /** An incoming relay cell has arrived from circuit <b>circ</b> to * stream <b>conn</b>. * @@ -1106,8 +1360,9 @@ connection_edge_process_relay_cell_not_open( break; case DIR_PURPOSE_FETCH_SERVERDESC: case DIR_PURPOSE_FETCH_MICRODESC: - control_event_bootstrap(BOOTSTRAP_STATUS_LOADING_DESCRIPTORS, - count_loading_descriptors_progress()); + if (TO_DIR_CONN(dirconn)->router_purpose == ROUTER_PURPOSE_GENERAL) + control_event_bootstrap(BOOTSTRAP_STATUS_LOADING_DESCRIPTORS, + count_loading_descriptors_progress()); break; } } @@ -1128,67 +1383,7 @@ connection_edge_process_relay_cell_not_open( } if (conn->base_.type == CONN_TYPE_AP && rh->command == RELAY_COMMAND_RESOLVED) { - int ttl; - int answer_len; - uint8_t answer_type; - entry_connection_t *entry_conn = EDGE_TO_ENTRY_CONN(conn); - if (conn->base_.state != AP_CONN_STATE_RESOLVE_WAIT) { - log_fn(LOG_PROTOCOL_WARN, LD_APP, "Got a 'resolved' cell while " - "not in state resolve_wait. Dropping."); - return 0; - } - tor_assert(SOCKS_COMMAND_IS_RESOLVE(entry_conn->socks_request->command)); - answer_len = cell->payload[RELAY_HEADER_SIZE+1]; - if (rh->length < 2 || answer_len+2>rh->length) { - log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, - "Dropping malformed 'resolved' cell"); - connection_mark_unattached_ap(entry_conn, END_STREAM_REASON_TORPROTOCOL); - return 0; - } - answer_type = cell->payload[RELAY_HEADER_SIZE]; - if (rh->length >= answer_len+6) - ttl = (int)ntohl(get_uint32(cell->payload+RELAY_HEADER_SIZE+ - 2+answer_len)); - else - ttl = -1; - if (answer_type == RESOLVED_TYPE_IPV4 || - answer_type == RESOLVED_TYPE_IPV6) { - tor_addr_t addr; - if (decode_address_from_payload(&addr, cell->payload+RELAY_HEADER_SIZE, - rh->length) && - tor_addr_is_internal(&addr, 0) && - get_options()->ClientDNSRejectInternalAddresses) { - log_info(LD_APP,"Got a resolve with answer %s. Rejecting.", - fmt_addr(&addr)); - connection_ap_handshake_socks_resolved(entry_conn, - RESOLVED_TYPE_ERROR_TRANSIENT, - 0, NULL, 0, TIME_MAX); - connection_mark_unattached_ap(entry_conn, - END_STREAM_REASON_TORPROTOCOL); - return 0; - } - } - connection_ap_handshake_socks_resolved(entry_conn, - answer_type, - cell->payload[RELAY_HEADER_SIZE+1], /*answer_len*/ - cell->payload+RELAY_HEADER_SIZE+2, /*answer*/ - ttl, - -1); - if (answer_type == RESOLVED_TYPE_IPV4 && answer_len == 4) { - tor_addr_t addr; - tor_addr_from_ipv4n(&addr, - get_uint32(cell->payload+RELAY_HEADER_SIZE+2)); - remap_event_helper(entry_conn, &addr); - } else if (answer_type == RESOLVED_TYPE_IPV6 && answer_len == 16) { - tor_addr_t addr; - tor_addr_from_ipv6_bytes(&addr, - (char*)(cell->payload+RELAY_HEADER_SIZE+2)); - remap_event_helper(entry_conn, &addr); - } - connection_mark_unattached_ap(entry_conn, - END_STREAM_REASON_DONE | - END_STREAM_REASON_FLAG_ALREADY_SOCKS_REPLIED); - return 0; + return connection_edge_process_resolved_cell(conn, cell, rh); } log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, @@ -1242,7 +1437,6 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, switch (rh.command) { case RELAY_COMMAND_BEGIN: case RELAY_COMMAND_CONNECTED: - case RELAY_COMMAND_DATA: case RELAY_COMMAND_END: case RELAY_COMMAND_RESOLVE: case RELAY_COMMAND_RESOLVED: @@ -1267,6 +1461,9 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, * EXIT_CONN_STATE_CONNECTING or EXIT_CONN_STATE_RESOLVING. * This speeds up HTTP, for example. */ optimistic_data = 1; + } else if (rh.stream_id == 0 && rh.command == RELAY_COMMAND_DATA) { + log_warn(LD_BUG, "Somehow I had a connection that matched a " + "data cell with stream ID 0."); } else { return connection_edge_process_relay_cell_not_open( &rh, cell, circ, conn, layer_hint); @@ -1327,7 +1524,11 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, circuit_consider_sending_sendme(circ, layer_hint); - if (!conn) { + if (rh.stream_id == 0) { + log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, "Relay data cell with zero " + "stream_id. Dropping."); + return 0; + } else if (!conn) { log_info(domain,"data cell dropped, unknown stream (streamid %d).", rh.stream_id); return 0; @@ -1497,7 +1698,8 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, if (layer_hint) { if (layer_hint->package_window + CIRCWINDOW_INCREMENT > CIRCWINDOW_START_MAX) { - log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, + static struct ratelim_t exit_warn_ratelim = RATELIM_INIT(600); + log_fn_ratelim(&exit_warn_ratelim, LOG_WARN, LD_PROTOCOL, "Unexpected sendme cell from exit relay. " "Closing circ."); return -END_CIRC_REASON_TORPROTOCOL; @@ -1509,7 +1711,8 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, } else { if (circ->package_window + CIRCWINDOW_INCREMENT > CIRCWINDOW_START_MAX) { - log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, + static struct ratelim_t client_warn_ratelim = RATELIM_INIT(600); + log_fn_ratelim(&client_warn_ratelim, LOG_WARN, LD_PROTOCOL, "Unexpected sendme cell from client. " "Closing circ (window %d).", circ->package_window); @@ -2036,9 +2239,10 @@ circuit_consider_sending_sendme(circuit_t *circ, crypt_path_t *layer_hint) #define assert_cmux_ok_paranoid(chan) #endif -/** The total number of cells we have allocated from the memory pool. */ +/** The total number of cells we have allocated. */ static size_t total_cells_allocated = 0; +#ifdef ENABLE_MEMPOOLS /** A memory pool to allocate packed_cell_t objects. */ static mp_pool_t *cell_pool = NULL; @@ -2050,8 +2254,8 @@ init_cell_pool(void) cell_pool = mp_pool_new(sizeof(packed_cell_t), 128*1024); } -/** Free all storage used to hold cells (and insertion times if we measure - * cell statistics). */ +/** Free all storage used to hold cells (and insertion times/commands if we + * measure cell statistics and/or if CELL_STATS events are enabled). */ void free_cell_pool(void) { @@ -2070,26 +2274,46 @@ clean_cell_pool(void) mp_pool_clean(cell_pool, 0, 1); } +#define relay_alloc_cell() \ + mp_pool_get(cell_pool) +#define relay_free_cell(cell) \ + mp_pool_release(cell) + +#define RELAY_CELL_MEM_COST (sizeof(packed_cell_t) + MP_POOL_ITEM_OVERHEAD) + +#else /* !ENABLE_MEMPOOLS case */ + +#define relay_alloc_cell() \ + tor_malloc_zero(sizeof(packed_cell_t)) +#define relay_free_cell(cell) \ + tor_free(cell) + +#define RELAY_CELL_MEM_COST (sizeof(packed_cell_t)) + +#endif /* ENABLE_MEMPOOLS */ + /** Release storage held by <b>cell</b>. */ static INLINE void packed_cell_free_unchecked(packed_cell_t *cell) { --total_cells_allocated; - mp_pool_release(cell); + relay_free_cell(cell); } /** Allocate and return a new packed_cell_t. */ -static INLINE packed_cell_t * +STATIC packed_cell_t * packed_cell_new(void) { ++total_cells_allocated; - return mp_pool_get(cell_pool); + return relay_alloc_cell(); } /** Return a packed cell used outside by channel_t lower layer */ void packed_cell_free(packed_cell_t *cell) { + if (!cell) + return; packed_cell_free_unchecked(cell); } @@ -2101,7 +2325,7 @@ dump_cell_pool_usage(int severity) circuit_t *c; int n_circs = 0; int n_cells = 0; - for (c = circuit_get_global_list_(); c; c = c->next) { + TOR_LIST_FOREACH(c, circuit_get_global_list(), head) { n_cells += c->n_chan_cells.n; if (!CIRCUIT_IS_ORIGIN(c)) n_cells += TO_OR_CIRCUIT(c)->p_chan_cells.n; @@ -2110,7 +2334,9 @@ dump_cell_pool_usage(int severity) tor_log(severity, LD_MM, "%d cells allocated on %d circuits. %d cells leaked.", n_cells, n_circs, (int)total_cells_allocated - n_cells); +#ifdef ENABLE_MEMPOOLS mp_pool_log_status(cell_pool, severity); +#endif } /** Allocate a new copy of packed <b>cell</b>. */ @@ -2119,7 +2345,6 @@ packed_cell_copy(const cell_t *cell, int wide_circ_ids) { packed_cell_t *c = packed_cell_new(); cell_pack(c, cell, wide_circ_ids); - c->next = NULL; return c; } @@ -2127,58 +2352,61 @@ packed_cell_copy(const cell_t *cell, int wide_circ_ids) void cell_queue_append(cell_queue_t *queue, packed_cell_t *cell) { - if (queue->tail) { - tor_assert(!queue->tail->next); - queue->tail->next = cell; - } else { - queue->head = cell; - } - queue->tail = cell; - cell->next = NULL; + TOR_SIMPLEQ_INSERT_TAIL(&queue->head, cell, next); ++queue->n; } -/** Append a newly allocated copy of <b>cell</b> to the end of <b>queue</b> */ +/** Append a newly allocated copy of <b>cell</b> to the end of the + * <b>exitward</b> (or app-ward) <b>queue</b> of <b>circ</b>. If + * <b>use_stats</b> is true, record statistics about the cell. + */ void -cell_queue_append_packed_copy(cell_queue_t *queue, const cell_t *cell, - int wide_circ_ids) +cell_queue_append_packed_copy(circuit_t *circ, cell_queue_t *queue, + int exitward, const cell_t *cell, + int wide_circ_ids, int use_stats) { struct timeval now; packed_cell_t *copy = packed_cell_copy(cell, wide_circ_ids); - tor_gettimeofday_cached(&now); + (void)circ; + (void)exitward; + (void)use_stats; + tor_gettimeofday_cached_monotonic(&now); + copy->inserted_time = (uint32_t)tv_to_msec(&now); cell_queue_append(queue, copy); } +/** Initialize <b>queue</b> as an empty cell queue. */ +void +cell_queue_init(cell_queue_t *queue) +{ + memset(queue, 0, sizeof(cell_queue_t)); + TOR_SIMPLEQ_INIT(&queue->head); +} + /** Remove and free every cell in <b>queue</b>. */ void cell_queue_clear(cell_queue_t *queue) { - packed_cell_t *cell, *next; - cell = queue->head; - while (cell) { - next = cell->next; + packed_cell_t *cell; + while ((cell = TOR_SIMPLEQ_FIRST(&queue->head))) { + TOR_SIMPLEQ_REMOVE_HEAD(&queue->head, next); packed_cell_free_unchecked(cell); - cell = next; } - queue->head = queue->tail = NULL; + TOR_SIMPLEQ_INIT(&queue->head); queue->n = 0; } /** Extract and return the cell at the head of <b>queue</b>; return NULL if * <b>queue</b> is empty. */ -static INLINE packed_cell_t * +STATIC packed_cell_t * cell_queue_pop(cell_queue_t *queue) { - packed_cell_t *cell = queue->head; + packed_cell_t *cell = TOR_SIMPLEQ_FIRST(&queue->head); if (!cell) return NULL; - queue->head = cell->next; - if (cell == queue->tail) { - tor_assert(!queue->head); - queue->tail = NULL; - } + TOR_SIMPLEQ_REMOVE_HEAD(&queue->head, next); --queue->n; return cell; } @@ -2188,16 +2416,24 @@ cell_queue_pop(cell_queue_t *queue) size_t packed_cell_mem_cost(void) { - return sizeof(packed_cell_t) + MP_POOL_ITEM_OVERHEAD; + return RELAY_CELL_MEM_COST; +} + +/** DOCDOC */ +STATIC size_t +cell_queues_get_total_allocation(void) +{ + return total_cells_allocated * packed_cell_mem_cost(); } /** Check whether we've got too much space used for cells. If so, * call the OOM handler and return 1. Otherwise, return 0. */ -static int +STATIC int cell_queues_check_size(void) { - size_t alloc = total_cells_allocated * packed_cell_mem_cost(); - if (alloc >= get_options()->MaxMemInCellQueues) { + size_t alloc = cell_queues_get_total_allocation(); + alloc += buf_get_total_allocation(); + if (alloc >= get_options()->MaxMemInQueues) { circuits_handle_oom(alloc); return 1; } @@ -2252,14 +2488,18 @@ update_circuit_on_cmux_(circuit_t *circ, cell_direction_t direction, assert_cmux_ok_paranoid(chan); } -/** Remove all circuits from the cmux on <b>chan</b>. */ +/** Remove all circuits from the cmux on <b>chan</b>. + * + * If <b>circuits_out</b> is non-NULL, add all detached circuits to + * <b>circuits_out</b>. + **/ void -channel_unlink_all_circuits(channel_t *chan) +channel_unlink_all_circuits(channel_t *chan, smartlist_t *circuits_out) { tor_assert(chan); tor_assert(chan->cmux); - circuitmux_detach_all_circuits(chan->cmux); + circuitmux_detach_all_circuits(chan->cmux, circuits_out); chan->num_n_circuits = 0; chan->num_p_circuits = 0; } @@ -2318,6 +2558,28 @@ set_streams_blocked_on_circ(circuit_t *circ, channel_t *chan, return n; } +/** Extract the command from a packed cell. */ +static uint8_t +packed_cell_get_command(const packed_cell_t *cell, int wide_circ_ids) +{ + if (wide_circ_ids) { + return get_uint8(cell->body+4); + } else { + return get_uint8(cell->body+2); + } +} + +/** Extract the circuit ID from a packed cell. */ +circid_t +packed_cell_get_circid(const packed_cell_t *cell, int wide_circ_ids) +{ + if (wide_circ_ids) { + return ntohl(get_uint32(cell->body)); + } else { + return ntohs(get_uint16(cell->body)); + } +} + /** Pull as many cells as possible (but no more than <b>max</b>) from the * queue of the first active circuit on <b>chan</b>, and write them to * <b>chan</b>->outbuf. Return the number of cells written. Advance @@ -2327,7 +2589,7 @@ channel_flush_from_first_active_circuit(channel_t *chan, int max) { circuitmux_t *cmux = NULL; int n_flushed = 0; - cell_queue_t *queue; + cell_queue_t *queue, *destroy_queue=NULL; circuit_t *circ; or_circuit_t *or_circ; int streams_blocked; @@ -2340,7 +2602,18 @@ channel_flush_from_first_active_circuit(channel_t *chan, int max) /* Main loop: pick a circuit, send a cell, update the cmux */ while (n_flushed < max) { - circ = circuitmux_get_first_active_circuit(cmux); + circ = circuitmux_get_first_active_circuit(cmux, &destroy_queue); + if (destroy_queue) { + /* this code is duplicated from some of the logic below. Ugly! XXXX */ + tor_assert(destroy_queue->n > 0); + cell = cell_queue_pop(destroy_queue); + channel_write_packed_cell(chan, cell); + /* Update the cmux destroy counter */ + circuitmux_notify_xmit_destroy(cmux); + cell = NULL; + ++n_flushed; + continue; + } /* If it returns NULL, no cells left to send */ if (!circ) break; assert_cmux_ok_paranoid(chan); @@ -2366,15 +2639,33 @@ channel_flush_from_first_active_circuit(channel_t *chan, int max) cell = cell_queue_pop(queue); /* Calculate the exact time that this cell has spent in the queue. */ - if (get_options()->CellStatistics && !CIRCUIT_IS_ORIGIN(circ)) { + if (get_options()->CellStatistics || + get_options()->TestingEnableCellStatsEvent) { uint32_t msec_waiting; struct timeval tvnow; - or_circ = TO_OR_CIRCUIT(circ); tor_gettimeofday_cached(&tvnow); msec_waiting = ((uint32_t)tv_to_msec(&tvnow)) - cell->inserted_time; - or_circ->total_cell_waiting_time += msec_waiting; - or_circ->processed_cells++; + if (get_options()->CellStatistics && !CIRCUIT_IS_ORIGIN(circ)) { + or_circ = TO_OR_CIRCUIT(circ); + or_circ->total_cell_waiting_time += msec_waiting; + or_circ->processed_cells++; + } + + if (get_options()->TestingEnableCellStatsEvent) { + uint8_t command = packed_cell_get_command(cell, chan->wide_circ_ids); + + testing_cell_stats_entry_t *ent = + tor_malloc_zero(sizeof(testing_cell_stats_entry_t)); + ent->command = command; + ent->waiting_time = msec_waiting / 10; + ent->removed = 1; + if (circ->n_chan == chan) + ent->exitward = 1; + if (!circ->testing_cell_stats) + circ->testing_cell_stats = smartlist_new(); + smartlist_add(circ->testing_cell_stats, ent); + } } /* If we just flushed our queue and this circuit is used for a @@ -2420,6 +2711,20 @@ channel_flush_from_first_active_circuit(channel_t *chan, int max) return n_flushed; } +#if 0 +/** Indicate the current preferred cap for middle circuits; zero disables + * the cap. Right now it's just a constant, ORCIRC_MAX_MIDDLE_CELLS, but + * the logic in append_cell_to_circuit_queue() is written to be correct + * if we want to base it on a consensus param or something that might change + * in the future. + */ +static int +get_max_middle_cells(void) +{ + return ORCIRC_MAX_MIDDLE_CELLS; +} +#endif + /** Add <b>cell</b> to the queue of <b>circ</b> writing to <b>chan</b> * transmitting in <b>direction</b>. */ void @@ -2430,11 +2735,16 @@ append_cell_to_circuit_queue(circuit_t *circ, channel_t *chan, or_circuit_t *orcirc = NULL; cell_queue_t *queue; int streams_blocked; +#if 0 + uint32_t tgt_max_middle_cells, p_len, n_len, tmp, hard_max_middle_cells; +#endif + int exitward; if (circ->marked_for_close) return; - if (direction == CELL_DIRECTION_OUT) { + exitward = (direction == CELL_DIRECTION_OUT); + if (exitward) { queue = &circ->n_chan_cells; streams_blocked = circ->streams_blocked_on_n_chan; } else { @@ -2451,28 +2761,82 @@ append_cell_to_circuit_queue(circuit_t *circ, channel_t *chan, if ((circ->n_chan != NULL) && CIRCUIT_IS_ORCIRC(circ)) { orcirc = TO_OR_CIRCUIT(circ); if (orcirc->p_chan) { - if (queue->n + 1 >= ORCIRC_MAX_MIDDLE_CELLS) { - /* Queueing this cell would put queue over the cap */ - log_warn(LD_CIRC, - "Got a cell exceeding the cap of %u in the %s direction " - "on middle circ ID %u on chan ID " U64_FORMAT - "; killing the circuit.", - ORCIRC_MAX_MIDDLE_CELLS, - (direction == CELL_DIRECTION_OUT) ? "n" : "p", - (direction == CELL_DIRECTION_OUT) ? - circ->n_circ_id : orcirc->p_circ_id, - U64_PRINTF_ARG( + /* We are a middle circuit if we have both n_chan and p_chan */ + /* We'll need to know the current preferred maximum */ + tgt_max_middle_cells = get_max_middle_cells(); + if (tgt_max_middle_cells > 0) { + /* Do we need to initialize middle_max_cells? */ + if (orcirc->max_middle_cells == 0) { + orcirc->max_middle_cells = tgt_max_middle_cells; + } else { + if (tgt_max_middle_cells > orcirc->max_middle_cells) { + /* If we want to increase the cap, we can do so right away */ + orcirc->max_middle_cells = tgt_max_middle_cells; + } else if (tgt_max_middle_cells < orcirc->max_middle_cells) { + /* + * If we're shrinking the cap, we can't shrink past either queue; + * compare tgt_max_middle_cells rather than tgt_max_middle_cells * + * ORCIRC_MAX_MIDDLE_KILL_THRESH so the queues don't shrink enough + * to generate spurious warnings, either. + */ + n_len = circ->n_chan_cells.n; + p_len = orcirc->p_chan_cells.n; + tmp = tgt_max_middle_cells; + if (tmp < n_len) tmp = n_len; + if (tmp < p_len) tmp = p_len; + orcirc->max_middle_cells = tmp; + } + /* else no change */ + } + } else { + /* tgt_max_middle_cells == 0 indicates we should disable the cap */ + orcirc->max_middle_cells = 0; + } + + /* Now we know orcirc->max_middle_cells is set correctly */ + if (orcirc->max_middle_cells > 0) { + hard_max_middle_cells = + (uint32_t)(((double)orcirc->max_middle_cells) * + ORCIRC_MAX_MIDDLE_KILL_THRESH); + + if ((unsigned)queue->n + 1 >= hard_max_middle_cells) { + /* Queueing this cell would put queue over the kill theshold */ + log_warn(LD_CIRC, + "Got a cell exceeding the hard cap of %u in the " + "%s direction on middle circ ID %u on chan ID " + U64_FORMAT "; killing the circuit.", + hard_max_middle_cells, + (direction == CELL_DIRECTION_OUT) ? "n" : "p", (direction == CELL_DIRECTION_OUT) ? - circ->n_chan->global_identifier : - orcirc->p_chan->global_identifier)); - circuit_mark_for_close(circ, END_CIRC_REASON_RESOURCELIMIT); - return; + circ->n_circ_id : orcirc->p_circ_id, + U64_PRINTF_ARG( + (direction == CELL_DIRECTION_OUT) ? + circ->n_chan->global_identifier : + orcirc->p_chan->global_identifier)); + circuit_mark_for_close(circ, END_CIRC_REASON_RESOURCELIMIT); + return; + } else if ((unsigned)queue->n + 1 == orcirc->max_middle_cells) { + /* Only use ==, not >= for this test so we don't spam the log */ + log_warn(LD_CIRC, + "While trying to queue a cell, reached the soft cap of %u " + "in the %s direction on middle circ ID %u " + "on chan ID " U64_FORMAT ".", + orcirc->max_middle_cells, + (direction == CELL_DIRECTION_OUT) ? "n" : "p", + (direction == CELL_DIRECTION_OUT) ? + circ->n_circ_id : orcirc->p_circ_id, + U64_PRINTF_ARG( + (direction == CELL_DIRECTION_OUT) ? + circ->n_chan->global_identifier : + orcirc->p_chan->global_identifier)); + } } } } #endif - cell_queue_append_packed_copy(queue, cell, chan->wide_circ_ids); + cell_queue_append_packed_copy(circ, queue, exitward, cell, + chan->wide_circ_ids, 1); if (PREDICT_UNLIKELY(cell_queues_check_size())) { /* We ran the OOM handler */ diff --git a/src/or/relay.h b/src/or/relay.h index 1fef10a7da..969c6fb61d 100644 --- a/src/or/relay.h +++ b/src/or/relay.h @@ -42,24 +42,28 @@ extern uint64_t stats_n_data_bytes_packaged; extern uint64_t stats_n_data_cells_received; extern uint64_t stats_n_data_bytes_received; +#ifdef ENABLE_MEMPOOLS void init_cell_pool(void); void free_cell_pool(void); void clean_cell_pool(void); +#endif /* ENABLE_MEMPOOLS */ void dump_cell_pool_usage(int severity); size_t packed_cell_mem_cost(void); /* For channeltls.c */ void packed_cell_free(packed_cell_t *cell); +void cell_queue_init(cell_queue_t *queue); void cell_queue_clear(cell_queue_t *queue); void cell_queue_append(cell_queue_t *queue, packed_cell_t *cell); -void cell_queue_append_packed_copy(cell_queue_t *queue, const cell_t *cell, - int wide_circ_ids); +void cell_queue_append_packed_copy(circuit_t *circ, cell_queue_t *queue, + int exitward, const cell_t *cell, + int wide_circ_ids, int use_stats); void append_cell_to_circuit_queue(circuit_t *circ, channel_t *chan, cell_t *cell, cell_direction_t direction, streamid_t fromstream); -void channel_unlink_all_circuits(channel_t *chan); +void channel_unlink_all_circuits(channel_t *chan, smartlist_t *detached_out); int channel_flush_from_first_active_circuit(channel_t *chan, int max); void assert_circuit_mux_okay(channel_t *chan); void update_circuit_on_cmux_(circuit_t *circ, cell_direction_t direction, @@ -75,11 +79,30 @@ void circuit_clear_cell_queue(circuit_t *circ, channel_t *chan); void stream_choice_seed_weak_rng(void); -#ifdef RELAY_PRIVATE int relay_crypt(circuit_t *circ, cell_t *cell, cell_direction_t cell_direction, crypt_path_t **layer_hint, char *recognized); -int connected_cell_parse(const relay_header_t *rh, const cell_t *cell, + +circid_t packed_cell_get_circid(const packed_cell_t *cell, int wide_circ_ids); + +#ifdef RELAY_PRIVATE +STATIC int connected_cell_parse(const relay_header_t *rh, const cell_t *cell, tor_addr_t *addr_out, int *ttl_out); +/** An address-and-ttl tuple as yielded by resolved_cell_parse */ +typedef struct address_ttl_s { + tor_addr_t addr; + char *hostname; + int ttl; +} address_ttl_t; +STATIC void address_ttl_free(address_ttl_t *addr); +STATIC int resolved_cell_parse(const cell_t *cell, const relay_header_t *rh, + smartlist_t *addresses_out, int *errcode_out); +STATIC int connection_edge_process_resolved_cell(edge_connection_t *conn, + const cell_t *cell, + const relay_header_t *rh); +STATIC packed_cell_t *packed_cell_new(void); +STATIC packed_cell_t *cell_queue_pop(cell_queue_t *queue); +STATIC size_t cell_queues_get_total_allocation(void); +STATIC int cell_queues_check_size(void); #endif #endif diff --git a/src/or/rendclient.c b/src/or/rendclient.c index bb4bd9bfd4..d42024010d 100644 --- a/src/or/rendclient.c +++ b/src/or/rendclient.c @@ -8,6 +8,7 @@ **/ #include "or.h" +#include "circpathbias.h" #include "circuitbuild.h" #include "circuitlist.h" #include "circuituse.h" @@ -25,6 +26,7 @@ #include "router.h" #include "routerlist.h" #include "routerset.h" +#include "control.h" static extend_info_t *rend_client_get_random_intro_impl( const rend_cache_entry_t *rend_query, @@ -376,7 +378,7 @@ rend_client_close_other_intros(const char *onion_address) { circuit_t *c; /* abort parallel intro circs, if any */ - for (c = circuit_get_global_list_(); c; c = c->next) { + TOR_LIST_FOREACH(c, circuit_get_global_list(), head) { if ((c->purpose == CIRCUIT_PURPOSE_C_INTRODUCING || c->purpose == CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT) && !c->marked_for_close && CIRCUIT_IS_ORIGIN(c)) { @@ -617,11 +619,14 @@ static int directory_get_from_hs_dir(const char *desc_id, const rend_data_t *rend_query) { smartlist_t *responsible_dirs = smartlist_new(); + smartlist_t *usable_responsible_dirs = smartlist_new(); + const or_options_t *options = get_options(); routerstatus_t *hs_dir; char desc_id_base32[REND_DESC_ID_V2_LEN_BASE32 + 1]; time_t now = time(NULL); char descriptor_cookie_base64[3*REND_DESC_COOKIE_LEN_BASE64]; - int tor2web_mode = get_options()->Tor2webMode; + const int tor2web_mode = options->Tor2webMode; + int excluded_some; tor_assert(desc_id); tor_assert(rend_query); /* Determine responsible dirs. Even if we can't get all we want, @@ -642,16 +647,33 @@ directory_get_from_hs_dir(const char *desc_id, const rend_data_t *rend_query) dir, desc_id_base32, rend_query, 0, 0); const node_t *node = node_get_by_id(dir->identity_digest); if (last + REND_HID_SERV_DIR_REQUERY_PERIOD >= now || - !node || !node_has_descriptor(node)) - SMARTLIST_DEL_CURRENT(responsible_dirs, dir); + !node || !node_has_descriptor(node)) { + SMARTLIST_DEL_CURRENT(responsible_dirs, dir); + continue; + } + if (! routerset_contains_node(options->ExcludeNodes, node)) { + smartlist_add(usable_responsible_dirs, dir); + } }); - hs_dir = smartlist_choose(responsible_dirs); + excluded_some = + smartlist_len(usable_responsible_dirs) < smartlist_len(responsible_dirs); + + hs_dir = smartlist_choose(usable_responsible_dirs); + if (! hs_dir && ! options->StrictNodes) + hs_dir = smartlist_choose(responsible_dirs); + smartlist_free(responsible_dirs); + smartlist_free(usable_responsible_dirs); if (!hs_dir) { log_info(LD_REND, "Could not pick one of the responsible hidden " "service directories, because we requested them all " "recently without success."); + if (options->StrictNodes && excluded_some) { + log_warn(LD_REND, "Could not pick a hidden service directory for the " + "requested hidden service: they are all either down or " + "excluded, and StrictNodes is set."); + } return 0; } @@ -693,6 +715,9 @@ directory_get_from_hs_dir(const char *desc_id, const rend_data_t *rend_query) (rend_query->auth_type == REND_NO_AUTH ? "[none]" : escaped_safe_str_client(descriptor_cookie_base64)), routerstatus_describe(hs_dir)); + control_event_hs_descriptor_requested(rend_query, + hs_dir->identity_digest, + desc_id_base32); return 1; } @@ -772,8 +797,7 @@ rend_client_cancel_descriptor_fetches(void) SMARTLIST_FOREACH_BEGIN(connection_array, connection_t *, conn) { if (conn->type == CONN_TYPE_DIR && - (conn->purpose == DIR_PURPOSE_FETCH_RENDDESC || - conn->purpose == DIR_PURPOSE_FETCH_RENDDESC_V2)) { + conn->purpose == DIR_PURPOSE_FETCH_RENDDESC_V2) { /* It's a rendezvous descriptor fetch in progress -- cancel it * by marking the connection for close. * diff --git a/src/or/rendcommon.c b/src/or/rendcommon.c index d1b49411cd..a664b5d501 100644 --- a/src/or/rendcommon.c +++ b/src/or/rendcommon.c @@ -672,79 +672,6 @@ rend_encode_v2_descriptors(smartlist_t *descs_out, return seconds_valid; } -/** Parse a service descriptor at <b>str</b> (<b>len</b> bytes). On - * success, return a newly alloced service_descriptor_t. On failure, - * return NULL. - */ -rend_service_descriptor_t * -rend_parse_service_descriptor(const char *str, size_t len) -{ - rend_service_descriptor_t *result = NULL; - int i, n_intro_points; - size_t keylen, asn1len; - const char *end, *cp, *eos; - rend_intro_point_t *intro; - - result = tor_malloc_zero(sizeof(rend_service_descriptor_t)); - cp = str; - end = str+len; - if (end-cp<2) goto truncated; - result->version = 0; - if (end-cp < 2) goto truncated; - asn1len = ntohs(get_uint16(cp)); - cp += 2; - if ((size_t)(end-cp) < asn1len) goto truncated; - result->pk = crypto_pk_asn1_decode(cp, asn1len); - if (!result->pk) goto truncated; - cp += asn1len; - if (end-cp < 4) goto truncated; - result->timestamp = (time_t) ntohl(get_uint32(cp)); - cp += 4; - result->protocols = 1<<2; /* always use intro format 2 */ - if (end-cp < 2) goto truncated; - n_intro_points = ntohs(get_uint16(cp)); - cp += 2; - - result->intro_nodes = smartlist_new(); - for (i=0;i<n_intro_points;++i) { - if (end-cp < 2) goto truncated; - eos = (const char *)memchr(cp,'\0',end-cp); - if (!eos) goto truncated; - /* Write nickname to extend info, but postpone the lookup whether - * we know that router. It's not part of the parsing process. */ - intro = tor_malloc_zero(sizeof(rend_intro_point_t)); - intro->extend_info = tor_malloc_zero(sizeof(extend_info_t)); - strlcpy(intro->extend_info->nickname, cp, - sizeof(intro->extend_info->nickname)); - smartlist_add(result->intro_nodes, intro); - cp = eos+1; - } - keylen = crypto_pk_keysize(result->pk); - tor_assert(end-cp >= 0); - if ((size_t)(end-cp) < keylen) goto truncated; - if ((size_t)(end-cp) > keylen) { - log_warn(LD_PROTOCOL, - "Signature is %d bytes too long on service descriptor.", - (int)((size_t)(end-cp) - keylen)); - goto error; - } - note_crypto_pk_op(REND_CLIENT); - if (crypto_pk_public_checksig_digest(result->pk, - (char*)str,cp-str, /* data */ - (char*)cp,end-cp /* signature*/ - )<0) { - log_warn(LD_PROTOCOL, "Bad signature on service descriptor."); - goto error; - } - - return result; - truncated: - log_warn(LD_PROTOCOL, "Truncated service descriptor."); - error: - rend_service_descriptor_free(result); - return NULL; -} - /** Sets <b>out</b> to the first 10 bytes of the digest of <b>pk</b>, * base32 encoded. NUL-terminates out. (We use this string to * identify services in directory requests and .onion URLs.) @@ -843,7 +770,7 @@ void rend_cache_purge(void) { if (rend_cache) { - log_info(LD_REND, "Purging client/v0-HS-authority HS descriptor cache"); + log_info(LD_REND, "Purging HS descriptor cache"); strmap_free(rend_cache, rend_cache_entry_free_); } rend_cache = strmap_new(); @@ -954,27 +881,6 @@ rend_cache_lookup_entry(const char *query, int version, rend_cache_entry_t **e) return 1; } -/** <b>query</b> is a base32'ed service id. If it's malformed, return -1. - * Else look it up. - * - If it is found, point *desc to it, and write its length into - * *desc_len, and return 1. - * - If it is not found, return 0. - * Note: calls to rend_cache_clean or rend_cache_store may invalidate - * *desc. - */ -int -rend_cache_lookup_desc(const char *query, int version, const char **desc, - size_t *desc_len) -{ - rend_cache_entry_t *e; - int r; - r = rend_cache_lookup_entry(query,version,&e); - if (r <= 0) return r; - *desc = e->desc; - *desc_len = e->len; - return 1; -} - /** Lookup the v2 service descriptor with base32-encoded <b>desc_id</b> and * copy the pointer to it to *<b>desc</b>. Return 1 on success, 0 on * well-formed-but-not-found, and -1 on failure. @@ -1006,130 +912,16 @@ rend_cache_lookup_v2_desc_as_dir(const char *desc_id, const char **desc) * descriptor */ #define MAX_INTRO_POINTS 10 -/** Parse *desc, calculate its service id, and store it in the cache. - * If we have a newer v0 descriptor with the same ID, ignore this one. - * If we have an older descriptor with the same ID, replace it. - * If we are acting as client due to the published flag and have any v2 - * descriptor with the same ID, reject this one in order to not get - * confused with having both versions for the same service. - * - * Return -2 if it's malformed or otherwise rejected; return -1 if we - * already have a v2 descriptor here; return 0 if it's the same or older - * than one we've already got; return 1 if it's novel. - * - * The published flag tells us if we store the descriptor - * in our role as directory (1) or if we cache it as client (0). - * - * If <b>service_id</b> is non-NULL and the descriptor is not for that - * service ID, reject it. <b>service_id</b> must be specified if and - * only if <b>published</b> is 0 (we fetched this descriptor). - */ -int -rend_cache_store(const char *desc, size_t desc_len, int published, - const char *service_id) -{ - rend_cache_entry_t *e; - rend_service_descriptor_t *parsed; - char query[REND_SERVICE_ID_LEN_BASE32+1]; - char key[REND_SERVICE_ID_LEN_BASE32+2]; /* 0<query>\0 */ - time_t now; - tor_assert(rend_cache); - parsed = rend_parse_service_descriptor(desc,desc_len); - if (!parsed) { - log_warn(LD_PROTOCOL,"Couldn't parse service descriptor."); - return -2; - } - if (rend_get_service_id(parsed->pk, query)<0) { - log_warn(LD_BUG,"Couldn't compute service ID."); - rend_service_descriptor_free(parsed); - return -2; - } - if ((service_id != NULL) && strcmp(query, service_id)) { - log_warn(LD_REND, "Received service descriptor for service ID %s; " - "expected descriptor for service ID %s.", - query, safe_str(service_id)); - rend_service_descriptor_free(parsed); - return -2; - } - now = time(NULL); - if (parsed->timestamp < now-REND_CACHE_MAX_AGE-REND_CACHE_MAX_SKEW) { - log_fn(LOG_PROTOCOL_WARN, LD_REND, - "Service descriptor %s is too old.", - safe_str_client(query)); - rend_service_descriptor_free(parsed); - return -2; - } - if (parsed->timestamp > now+REND_CACHE_MAX_SKEW) { - log_fn(LOG_PROTOCOL_WARN, LD_REND, - "Service descriptor %s is too far in the future.", - safe_str_client(query)); - rend_service_descriptor_free(parsed); - return -2; - } - /* Do we have a v2 descriptor and fetched this descriptor as a client? */ - tor_snprintf(key, sizeof(key), "2%s", query); - if (!published && strmap_get_lc(rend_cache, key)) { - log_info(LD_REND, "We already have a v2 descriptor for service %s.", - safe_str_client(query)); - rend_service_descriptor_free(parsed); - return -1; - } - if (parsed->intro_nodes && - smartlist_len(parsed->intro_nodes) > MAX_INTRO_POINTS) { - log_warn(LD_REND, "Found too many introduction points on a hidden " - "service descriptor for %s. This is probably a (misguided) " - "attempt to improve reliability, but it could also be an " - "attempt to do a guard enumeration attack. Rejecting.", - safe_str_client(query)); - rend_service_descriptor_free(parsed); - return -2; - } - tor_snprintf(key, sizeof(key), "0%s", query); - e = (rend_cache_entry_t*) strmap_get_lc(rend_cache, key); - if (e && e->parsed->timestamp > parsed->timestamp) { - log_info(LD_REND,"We already have a newer service descriptor %s with the " - "same ID and version.", - safe_str_client(query)); - rend_service_descriptor_free(parsed); - return 0; - } - if (e && e->len == desc_len && tor_memeq(desc,e->desc,desc_len)) { - log_info(LD_REND,"We already have this service descriptor %s.", - safe_str_client(query)); - e->received = time(NULL); - rend_service_descriptor_free(parsed); - return 0; - } - if (!e) { - e = tor_malloc_zero(sizeof(rend_cache_entry_t)); - strmap_set_lc(rend_cache, key, e); - } else { - rend_service_descriptor_free(e->parsed); - tor_free(e->desc); - } - e->received = time(NULL); - e->parsed = parsed; - e->len = desc_len; - e->desc = tor_malloc(desc_len); - memcpy(e->desc, desc, desc_len); - - log_debug(LD_REND,"Successfully stored rend desc '%s', len %d.", - safe_str_client(query), (int)desc_len); - return 1; -} - /** Parse the v2 service descriptor(s) in <b>desc</b> and store it/them to the * local rend cache. Don't attempt to decrypt the included list of introduction * points (as we don't have a descriptor cookie for it). * * If we have a newer descriptor with the same ID, ignore this one. * If we have an older descriptor with the same ID, replace it. - * Return -2 if we are not acting as hidden service directory; - * return -1 if the descriptor(s) were not parsable; return 0 if all - * descriptors are the same or older than those we've already got; - * return a positive number for the number of novel stored descriptors. + * + * Return an appropriate rend_cache_store_status_t. */ -int +rend_cache_store_status_t rend_cache_store_v2_desc_as_dir(const char *desc) { rend_service_descriptor_t *parsed; @@ -1149,7 +941,7 @@ rend_cache_store_v2_desc_as_dir(const char *desc) /* Cannot store descs, because we are (currently) not acting as * hidden service directory. */ log_info(LD_REND, "Cannot store descs: Not acting as hs dir"); - return -2; + return RCS_NOTDIR; } while (rend_parse_v2_service_descriptor(&parsed, desc_id, &intro_content, &intro_size, &encoded_size, @@ -1225,11 +1017,11 @@ rend_cache_store_v2_desc_as_dir(const char *desc) } if (!number_parsed) { log_info(LD_REND, "Could not parse any descriptor."); - return -1; + return RCS_BADDESC; } log_info(LD_REND, "Parsed %d and added %d descriptor%s.", number_parsed, number_stored, number_stored != 1 ? "s" : ""); - return number_stored; + return RCS_OKAY; } /** Parse the v2 service descriptor in <b>desc</b>, decrypt the included list @@ -1239,15 +1031,12 @@ rend_cache_store_v2_desc_as_dir(const char *desc) * * If we have a newer v2 descriptor with the same ID, ignore this one. * If we have an older descriptor with the same ID, replace it. - * If we have any v0 descriptor with the same ID, reject this one in order - * to not get confused with having both versions for the same service. * If the descriptor's service ID does not match * <b>rend_query</b>-\>onion_address, reject it. - * Return -2 if it's malformed or otherwise rejected; return -1 if we - * already have a v0 descriptor here; return 0 if it's the same or older - * than one we've already got; return 1 if it's novel. + * + * Return an appropriate rend_cache_store_status_t. */ -int +rend_cache_store_status_t rend_cache_store_v2_desc_as_client(const char *desc, const rend_data_t *rend_query) { @@ -1276,7 +1065,7 @@ rend_cache_store_v2_desc_as_client(const char *desc, char key[REND_SERVICE_ID_LEN_BASE32+2]; char service_id[REND_SERVICE_ID_LEN_BASE32+1]; rend_cache_entry_t *e; - int retval; + rend_cache_store_status_t retval = RCS_BADDESC; tor_assert(rend_cache); tor_assert(desc); /* Parse the descriptor. */ @@ -1284,20 +1073,17 @@ rend_cache_store_v2_desc_as_client(const char *desc, &intro_size, &encoded_size, &next_desc, desc) < 0) { log_warn(LD_REND, "Could not parse descriptor."); - retval = -2; goto err; } /* Compute service ID from public key. */ if (rend_get_service_id(parsed->pk, service_id)<0) { log_warn(LD_REND, "Couldn't compute service ID."); - retval = -2; goto err; } if (strcmp(rend_query->onion_address, service_id)) { log_warn(LD_REND, "Received service descriptor for service ID %s; " "expected descriptor for service ID %s.", service_id, safe_str(rend_query->onion_address)); - retval = -2; goto err; } /* Decode/decrypt introduction points. */ @@ -1329,7 +1115,6 @@ rend_cache_store_v2_desc_as_client(const char *desc, log_warn(LD_REND, "Failed to parse introduction points. Either the " "service has published a corrupt descriptor or you have " "provided invalid authorization data."); - retval = -2; goto err; } else if (n_intro_points > MAX_INTRO_POINTS) { log_warn(LD_REND, "Found too many introduction points on a hidden " @@ -1337,7 +1122,7 @@ rend_cache_store_v2_desc_as_client(const char *desc, "attempt to improve reliability, but it could also be an " "attempt to do a guard enumeration attack. Rejecting.", safe_str_client(rend_query->onion_address)); - retval = -2; + goto err; } } else { @@ -1350,22 +1135,12 @@ rend_cache_store_v2_desc_as_client(const char *desc, if (parsed->timestamp < now - REND_CACHE_MAX_AGE-REND_CACHE_MAX_SKEW) { log_warn(LD_REND, "Service descriptor with service ID %s is too old.", safe_str_client(service_id)); - retval = -2; goto err; } /* Is descriptor too far in the future? */ if (parsed->timestamp > now + REND_CACHE_MAX_SKEW) { log_warn(LD_REND, "Service descriptor with service ID %s is too far in " "the future.", safe_str_client(service_id)); - retval = -2; - goto err; - } - /* Do we have a v0 descriptor? */ - tor_snprintf(key, sizeof(key), "0%s", service_id); - if (strmap_get_lc(rend_cache, key)) { - log_info(LD_REND, "We already have a v0 descriptor for service ID %s.", - safe_str_client(service_id)); - retval = -1; goto err; } /* Do we already have a newer descriptor? */ @@ -1375,16 +1150,14 @@ rend_cache_store_v2_desc_as_client(const char *desc, log_info(LD_REND, "We already have a newer service descriptor for " "service ID %s with the same desc ID and version.", safe_str_client(service_id)); - retval = 0; - goto err; + goto okay; } /* Do we already have this descriptor? */ if (e && !strcmp(desc, e->desc)) { log_info(LD_REND,"We already have this service descriptor %s.", safe_str_client(service_id)); e->received = time(NULL); - retval = 0; - goto err; + goto okay; } if (!e) { e = tor_malloc_zero(sizeof(rend_cache_entry_t)); @@ -1400,7 +1173,10 @@ rend_cache_store_v2_desc_as_client(const char *desc, e->len = encoded_size; log_debug(LD_REND,"Successfully stored rend desc '%s', len %d.", safe_str_client(service_id), (int)encoded_size); - return 1; + return RCS_OKAY; + + okay: + retval = RCS_OKAY; err: rend_service_descriptor_free(parsed); diff --git a/src/or/rendcommon.h b/src/or/rendcommon.h index f476593d2b..07a47accfe 100644 --- a/src/or/rendcommon.h +++ b/src/or/rendcommon.h @@ -26,8 +26,6 @@ void rend_process_relay_cell(circuit_t *circ, const crypt_path_t *layer_hint, const uint8_t *payload); void rend_service_descriptor_free(rend_service_descriptor_t *desc); -rend_service_descriptor_t *rend_parse_service_descriptor(const char *str, - size_t len); int rend_get_service_id(crypto_pk_t *pk, char *out); void rend_encoded_v2_service_descriptor_free( rend_encoded_v2_service_descriptor_t *desc); @@ -39,16 +37,20 @@ void rend_cache_clean_v2_descs_as_dir(time_t now); void rend_cache_purge(void); void rend_cache_free_all(void); int rend_valid_service_id(const char *query); -int rend_cache_lookup_desc(const char *query, int version, const char **desc, - size_t *desc_len); int rend_cache_lookup_entry(const char *query, int version, rend_cache_entry_t **entry_out); int rend_cache_lookup_v2_desc_as_dir(const char *query, const char **desc); -int rend_cache_store(const char *desc, size_t desc_len, int published, - const char *service_id); -int rend_cache_store_v2_desc_as_client(const char *desc, +/** Return value from rend_cache_store_v2_desc_as_{dir,client}. */ +typedef enum { + RCS_NOTDIR = -2, /**< We're not a directory */ + RCS_BADDESC = -1, /**< This descriptor is no good. */ + RCS_OKAY = 0 /**< All worked as expected */ +} rend_cache_store_status_t; + +rend_cache_store_status_t rend_cache_store_v2_desc_as_dir(const char *desc); +rend_cache_store_status_t rend_cache_store_v2_desc_as_client(const char *desc, const rend_data_t *rend_query); -int rend_cache_store_v2_desc_as_dir(const char *desc); + int rend_encode_v2_descriptors(smartlist_t *descs_out, rend_service_descriptor_t *desc, time_t now, uint8_t period, rend_auth_type_t auth_type, diff --git a/src/or/rendmid.c b/src/or/rendmid.c index 1bd11f6dc0..1103816806 100644 --- a/src/or/rendmid.c +++ b/src/or/rendmid.c @@ -94,7 +94,7 @@ rend_mid_establish_intro(or_circuit_t *circ, const uint8_t *request, /* Close any other intro circuits with the same pk. */ c = NULL; - while ((c = circuit_get_intro_point(pk_digest))) { + while ((c = circuit_get_intro_point((const uint8_t *)pk_digest))) { log_info(LD_REND, "Replacing old circuit for service %s", safe_str(serviceid)); circuit_mark_for_close(TO_CIRCUIT(c), END_CIRC_REASON_FINISHED); @@ -111,7 +111,7 @@ rend_mid_establish_intro(or_circuit_t *circ, const uint8_t *request, /* Now, set up this circuit. */ circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_INTRO_POINT); - memcpy(circ->rend_token, pk_digest, DIGEST_LEN); + circuit_set_intro_point_digest(circ, (uint8_t *)pk_digest); log_info(LD_REND, "Established introduction point on circuit %u for service %s", @@ -165,7 +165,7 @@ rend_mid_introduce(or_circuit_t *circ, const uint8_t *request, (char*)request, REND_SERVICE_ID_LEN); /* The first 20 bytes are all we look at: they have a hash of Bob's PK. */ - intro_circ = circuit_get_intro_point((char*)request); + intro_circ = circuit_get_intro_point((const uint8_t*)request); if (!intro_circ) { log_info(LD_REND, "No intro circ found for INTRODUCE1 cell (%s) from circuit %u; " @@ -231,11 +231,12 @@ rend_mid_establish_rendezvous(or_circuit_t *circ, const uint8_t *request, } if (request_len != REND_COOKIE_LEN) { - log_warn(LD_PROTOCOL, "Invalid length on ESTABLISH_RENDEZVOUS."); + log_fn(LOG_PROTOCOL_WARN, + LD_PROTOCOL, "Invalid length on ESTABLISH_RENDEZVOUS."); goto err; } - if (circuit_get_rendezvous((char*)request)) { + if (circuit_get_rendezvous(request)) { log_warn(LD_PROTOCOL, "Duplicate rendezvous cookie in ESTABLISH_RENDEZVOUS."); goto err; @@ -251,7 +252,7 @@ rend_mid_establish_rendezvous(or_circuit_t *circ, const uint8_t *request, } circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_REND_POINT_WAITING); - memcpy(circ->rend_token, request, REND_COOKIE_LEN); + circuit_set_rendezvous_cookie(circ, request); base16_encode(hexid,9,(char*)request,4); @@ -299,7 +300,7 @@ rend_mid_rendezvous(or_circuit_t *circ, const uint8_t *request, "Got request for rendezvous from circuit %u to cookie %s.", (unsigned)circ->p_circ_id, hexid); - rend_circ = circuit_get_rendezvous((char*)request); + rend_circ = circuit_get_rendezvous(request); if (!rend_circ) { log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, "Rejecting RENDEZVOUS1 cell with unrecognized rendezvous cookie %s.", @@ -327,7 +328,7 @@ rend_mid_rendezvous(or_circuit_t *circ, const uint8_t *request, circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_REND_ESTABLISHED); circuit_change_purpose(TO_CIRCUIT(rend_circ), CIRCUIT_PURPOSE_REND_ESTABLISHED); - memset(circ->rend_token, 0, REND_COOKIE_LEN); + circuit_set_rendezvous_cookie(circ, NULL); rend_circ->rend_splice = circ; circ->rend_splice = rend_circ; diff --git a/src/or/rendservice.c b/src/or/rendservice.c index 8a4a11e475..a7c1e32f15 100644 --- a/src/or/rendservice.c +++ b/src/or/rendservice.c @@ -10,6 +10,7 @@ #define RENDSERVICE_PRIVATE #include "or.h" +#include "circpathbias.h" #include "circuitbuild.h" #include "circuitlist.h" #include "circuituse.h" @@ -81,7 +82,7 @@ typedef struct rend_service_port_config_t { #define MAX_INTRO_CIRCS_PER_PERIOD 10 /** How many times will a hidden service operator attempt to connect to * a requested rendezvous point before giving up? */ -#define MAX_REND_FAILURES 30 +#define MAX_REND_FAILURES 8 /** How many seconds should we spend trying to connect to a requested * rendezvous point before giving up? */ #define MAX_REND_TIMEOUT 30 @@ -543,7 +544,7 @@ rend_config_services(const or_options_t *options, int validate_only) /* XXXX it would be nicer if we had a nicer abstraction to use here, * so we could just iterate over the list of services to close, but * once again, this isn't critical-path code. */ - for (circ = circuit_get_global_list_(); circ; circ = circ->next) { + TOR_LIST_FOREACH(circ, circuit_get_global_list(), head) { if (!circ->marked_for_close && circ->state == CIRCUIT_STATE_OPEN && (circ->purpose == CIRCUIT_PURPOSE_S_ESTABLISH_INTRO || @@ -655,6 +656,35 @@ rend_service_load_all_keys(void) return 0; } +/** Add to <b>lst</b> every filename used by <b>s</b>. */ +static void +rend_service_add_filenames_to_list(smartlist_t *lst, const rend_service_t *s) +{ + tor_assert(lst); + tor_assert(s); + smartlist_add_asprintf(lst, "%s"PATH_SEPARATOR"private_key", + s->directory); + smartlist_add_asprintf(lst, "%s"PATH_SEPARATOR"hostname", + s->directory); + smartlist_add_asprintf(lst, "%s"PATH_SEPARATOR"client_keys", + s->directory); +} + +/** Add to <b>open_lst</b> every filename used by a configured hidden service, + * and to <b>stat_lst</b> every directory used by a configured hidden + * service */ +void +rend_services_add_filenames_to_lists(smartlist_t *open_lst, + smartlist_t *stat_lst) +{ + if (!rend_service_list) + return; + SMARTLIST_FOREACH_BEGIN(rend_service_list, rend_service_t *, s) { + rend_service_add_filenames_to_list(open_lst, s); + smartlist_add(stat_lst, tor_strdup(s->directory)); + } SMARTLIST_FOREACH_END(s); +} + /** Load and/or generate private keys for the hidden service <b>s</b>, * possibly including keys for client authorization. Return 0 on success, -1 * on failure. */ @@ -1208,7 +1238,7 @@ rend_service_introduce(origin_circuit_t *circuit, const uint8_t *request, /* check for replay of PK-encrypted portion. */ replay = replaycache_add_test_and_elapsed( intro_point->accepted_intro_rsa_parts, - parsed_req->ciphertext, (int)parsed_req->ciphertext_len, + parsed_req->ciphertext, parsed_req->ciphertext_len, &elapsed); if (replay) { @@ -1502,27 +1532,6 @@ find_rp_for_intro(const rend_intro_cell_t *intro, return rp; } -/** Remove unnecessary parts from a rend_intro_cell_t - the ciphertext if - * already decrypted, the plaintext too if already parsed - */ - -void -rend_service_compact_intro(rend_intro_cell_t *request) -{ - if (!request) return; - - if ((request->plaintext && request->plaintext_len > 0) || - request->parsed) { - tor_free(request->ciphertext); - request->ciphertext_len = 0; - } - - if (request->parsed) { - tor_free(request->plaintext); - request->plaintext_len = 0; - } -} - /** Free a parsed INTRODUCE1 or INTRODUCE2 cell that was allocated by * rend_service_parse_intro(). */ @@ -2061,7 +2070,7 @@ rend_service_decrypt_intro( if (err_msg_out && !err_msg) { tor_asprintf(&err_msg, "unknown INTRODUCE%d error decrypting encrypted part", - (int)(intro->type)); + intro ? (int)(intro->type) : -1); } if (status >= 0) status = -1; @@ -2167,7 +2176,7 @@ rend_service_parse_intro_plaintext( if (err_msg_out && !err_msg) { tor_asprintf(&err_msg, "unknown INTRODUCE%d error parsing encrypted part", - (int)(intro->type)); + intro ? (int)(intro->type) : -1); } if (status >= 0) status = -1; @@ -2376,7 +2385,7 @@ count_established_intro_points(const char *query) { int num_ipos = 0; circuit_t *circ; - for (circ = circuit_get_global_list_(); circ; circ = circ->next) { + TOR_LIST_FOREACH(circ, circuit_get_global_list(), head) { if (!circ->marked_for_close && circ->state == CIRCUIT_STATE_OPEN && (circ->purpose == CIRCUIT_PURPOSE_S_ESTABLISH_INTRO || diff --git a/src/or/rendservice.h b/src/or/rendservice.h index caf88a3d64..40198b07ec 100644 --- a/src/or/rendservice.h +++ b/src/or/rendservice.h @@ -71,6 +71,8 @@ struct rend_intro_cell_s { int num_rend_services(void); int rend_config_services(const or_options_t *options, int validate_only); int rend_service_load_all_keys(void); +void rend_services_add_filenames_to_lists(smartlist_t *open_lst, + smartlist_t *stat_lst); void rend_services_introduce(void); void rend_consider_services_upload(time_t now); void rend_hsdir_routers_changed(void); @@ -83,7 +85,6 @@ int rend_service_intro_established(origin_circuit_t *circuit, void rend_service_rendezvous_has_opened(origin_circuit_t *circuit); int rend_service_introduce(origin_circuit_t *circuit, const uint8_t *request, size_t request_len); -void rend_service_compact_intro(rend_intro_cell_t *request); int rend_service_decrypt_intro(rend_intro_cell_t *request, crypto_pk_t *key, char **err_msg_out); diff --git a/src/or/rephist.c b/src/or/rephist.c index 2948bf8f00..5446c25e36 100644 --- a/src/or/rephist.c +++ b/src/or/rephist.c @@ -879,126 +879,6 @@ rep_hist_record_mtbf_data(time_t now, int missing_means_down) return -1; } -/** Format the current tracked status of the router in <b>hist</b> at time - * <b>now</b> for analysis; return it in a newly allocated string. */ -static char * -rep_hist_format_router_status(or_history_t *hist, time_t now) -{ - char sor_buf[ISO_TIME_LEN+1]; - char sod_buf[ISO_TIME_LEN+1]; - double wfu; - double mtbf; - int up = 0, down = 0; - char *cp = NULL; - - if (hist->start_of_run) { - format_iso_time(sor_buf, hist->start_of_run); - up = 1; - } - if (hist->start_of_downtime) { - format_iso_time(sod_buf, hist->start_of_downtime); - down = 1; - } - - wfu = get_weighted_fractional_uptime(hist, now); - mtbf = get_stability(hist, now); - tor_asprintf(&cp, - "%s%s%s" - "%s%s%s" - "wfu %0.3f\n" - " weighted-time %lu\n" - " weighted-uptime %lu\n" - "mtbf %0.1f\n" - " weighted-run-length %lu\n" - " total-run-weights %f\n", - up?"uptime-started ":"", up?sor_buf:"", up?" UTC\n":"", - down?"downtime-started ":"", down?sod_buf:"", down?" UTC\n":"", - wfu, - hist->total_weighted_time, - hist->weighted_uptime, - mtbf, - hist->weighted_run_length, - hist->total_run_weights - ); - return cp; -} - -/** The last stability analysis document that we created, or NULL if we never - * have created one. */ -static char *last_stability_doc = NULL; -/** The last time we created a stability analysis document, or 0 if we never - * have created one. */ -static time_t built_last_stability_doc_at = 0; -/** Shortest allowable time between building two stability documents. */ -#define MAX_STABILITY_DOC_BUILD_RATE (3*60) - -/** Return a pointer to a NUL-terminated document describing our view of the - * stability of the routers we've been tracking. Return NULL on failure. */ -const char * -rep_hist_get_router_stability_doc(time_t now) -{ - char *result; - smartlist_t *chunks; - if (built_last_stability_doc_at + MAX_STABILITY_DOC_BUILD_RATE > now) - return last_stability_doc; - - if (!history_map) - return NULL; - - tor_free(last_stability_doc); - chunks = smartlist_new(); - - if (rep_hist_have_measured_enough_stability()) { - smartlist_add(chunks, tor_strdup("we-have-enough-measurements\n")); - } else { - smartlist_add(chunks, tor_strdup("we-do-not-have-enough-measurements\n")); - } - - DIGESTMAP_FOREACH(history_map, id, or_history_t *, hist) { - const node_t *node; - char dbuf[BASE64_DIGEST_LEN+1]; - char *info; - digest_to_base64(dbuf, id); - node = node_get_by_id(id); - if (node) { - char ip[INET_NTOA_BUF_LEN+1]; - char tbuf[ISO_TIME_LEN+1]; - time_t published = node_get_published_on(node); - node_get_address_string(node,ip,sizeof(ip)); - if (published > 0) - format_iso_time(tbuf, published); - else - strlcpy(tbuf, "???", sizeof(tbuf)); - smartlist_add_asprintf(chunks, - "router %s %s %s\n" - "published %s\n" - "relevant-flags %s%s%s\n" - "declared-uptime %ld\n", - dbuf, node_get_nickname(node), ip, - tbuf, - node->is_running ? "Running " : "", - node->is_valid ? "Valid " : "", - node->ri && node->ri->is_hibernating ? "Hibernating " : "", - node_get_declared_uptime(node)); - } else { - smartlist_add_asprintf(chunks, - "router %s {no descriptor}\n", dbuf); - } - info = rep_hist_format_router_status(hist, now); - if (info) - smartlist_add(chunks, info); - - } DIGESTMAP_FOREACH_END; - - result = smartlist_join_strings(chunks, "", 0, NULL); - SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp)); - smartlist_free(chunks); - - last_stability_doc = result; - built_last_stability_doc_at = time(NULL); - return result; -} - /** Helper: return the first j >= i such that !strcmpstart(sl[j], prefix) and * such that no line sl[k] with i <= k < j starts with "R ". Return -1 if no * such line exists. */ @@ -1051,7 +931,7 @@ correct_time(time_t t, time_t now, time_t stored_at, time_t started_measuring) return 0; else { long run_length = stored_at - t; - t = now - run_length; + t = (time_t)(now - run_length); if (t < started_measuring) t = started_measuring; return t; @@ -1212,7 +1092,7 @@ rep_hist_load_mtbf_data(time_t now) hist->start_of_run = correct_time(start_of_run, now, stored_at, tracked_since); if (hist->start_of_run < latest_possible_start + wrl) - latest_possible_start = hist->start_of_run - wrl; + latest_possible_start = (time_t)(hist->start_of_run - wrl); hist->weighted_run_length = wrl; hist->total_run_weights = trw; @@ -1862,22 +1742,20 @@ rep_hist_note_used_port(time_t now, uint16_t port) add_predicted_port(now, port); } -/** For this long after we've seen a request for a given port, assume that - * we'll want to make connections to the same port in the future. */ -#define PREDICTED_CIRCS_RELEVANCE_TIME (60*60) - /** Return a newly allocated pointer to a list of uint16_t * for ports that * are likely to be asked for in the near future. */ smartlist_t * rep_hist_get_predicted_ports(time_t now) { + int predicted_circs_relevance_time; smartlist_t *out = smartlist_new(); tor_assert(predicted_ports_list); + predicted_circs_relevance_time = get_options()->PredictedPortsRelevanceTime; /* clean out obsolete entries */ SMARTLIST_FOREACH_BEGIN(predicted_ports_list, predicted_port_t *, pp) { - if (pp->time + PREDICTED_CIRCS_RELEVANCE_TIME < now) { + if (pp->time + predicted_circs_relevance_time < now) { log_debug(LD_CIRC, "Expiring predicted port %d", pp->port); rephist_total_alloc -= sizeof(predicted_port_t); @@ -1944,14 +1822,17 @@ int rep_hist_get_predicted_internal(time_t now, int *need_uptime, int *need_capacity) { + int predicted_circs_relevance_time; + predicted_circs_relevance_time = get_options()->PredictedPortsRelevanceTime; + if (!predicted_internal_time) { /* initialize it */ predicted_internal_time = now; predicted_internal_uptime_time = now; predicted_internal_capacity_time = now; } - if (predicted_internal_time + PREDICTED_CIRCS_RELEVANCE_TIME < now) + if (predicted_internal_time + predicted_circs_relevance_time < now) return 0; /* too long ago */ - if (predicted_internal_uptime_time + PREDICTED_CIRCS_RELEVANCE_TIME >= now) + if (predicted_internal_uptime_time + predicted_circs_relevance_time >= now) *need_uptime = 1; // Always predict that we need capacity. *need_capacity = 1; @@ -1963,8 +1844,11 @@ rep_hist_get_predicted_internal(time_t now, int *need_uptime, int any_predicted_circuits(time_t now) { + int predicted_circs_relevance_time; + predicted_circs_relevance_time = get_options()->PredictedPortsRelevanceTime; + return smartlist_len(predicted_ports_list) || - predicted_internal_time + PREDICTED_CIRCS_RELEVANCE_TIME >= now; + predicted_internal_time + predicted_circs_relevance_time >= now; } /** Return 1 if we have no need for circuits currently, else return 0. */ @@ -2313,7 +2197,7 @@ rep_hist_format_exit_stats(time_t now) time_t rep_hist_exit_stats_write(time_t now) { - char *statsdir = NULL, *filename = NULL, *str = NULL; + char *str = NULL; if (!start_of_exit_stats_interval) return 0; /* Not initialized. */ @@ -2329,19 +2213,12 @@ rep_hist_exit_stats_write(time_t now) rep_hist_reset_exit_stats(now); /* Try to write to disk. */ - statsdir = get_datadir_fname("stats"); - if (check_private_dir(statsdir, CPD_CREATE, get_options()->User) < 0) { - log_warn(LD_HIST, "Unable to create stats/ directory!"); - goto done; + if (!check_or_create_data_subdir("stats")) { + write_to_data_subdir("stats", "exit-stats", str, "exit port statistics"); } - filename = get_datadir_fname2("stats", "exit-stats"); - if (write_str_to_file(filename, str, 0) < 0) - log_warn(LD_HIST, "Unable to write exit port statistics to disk!"); done: tor_free(str); - tor_free(statsdir); - tor_free(filename); return start_of_exit_stats_interval + WRITE_STATS_INTERVAL; } @@ -2434,7 +2311,7 @@ rep_hist_buffer_stats_add_circ(circuit_t *circ, time_t end_of_interval) return; start_of_interval = (circ->timestamp_created.tv_sec > start_of_buffer_stats_interval) ? - circ->timestamp_created.tv_sec : + (time_t)circ->timestamp_created.tv_sec : start_of_buffer_stats_interval; interval_length = (int) (end_of_interval - start_of_interval); if (interval_length <= 0) @@ -2598,7 +2475,7 @@ time_t rep_hist_buffer_stats_write(time_t now) { circuit_t *circ; - char *statsdir = NULL, *filename = NULL, *str = NULL; + char *str = NULL; if (!start_of_buffer_stats_interval) return 0; /* Not initialized. */ @@ -2606,7 +2483,7 @@ rep_hist_buffer_stats_write(time_t now) goto done; /* Not ready to write */ /* Add open circuits to the history. */ - for (circ = circuit_get_global_list_(); circ; circ = circ->next) { + TOR_LIST_FOREACH(circ, circuit_get_global_list(), head) { rep_hist_buffer_stats_add_circ(circ, now); } @@ -2617,19 +2494,12 @@ rep_hist_buffer_stats_write(time_t now) rep_hist_reset_buffer_stats(now); /* Try to write to disk. */ - statsdir = get_datadir_fname("stats"); - if (check_private_dir(statsdir, CPD_CREATE, get_options()->User) < 0) { - log_warn(LD_HIST, "Unable to create stats/ directory!"); - goto done; + if (!check_or_create_data_subdir("stats")) { + write_to_data_subdir("stats", "buffer-stats", str, "buffer statistics"); } - filename = get_datadir_fname2("stats", "buffer-stats"); - if (write_str_to_file(filename, str, 0) < 0) - log_warn(LD_HIST, "Unable to write buffer stats to disk!"); done: tor_free(str); - tor_free(filename); - tor_free(statsdir); return start_of_buffer_stats_interval + WRITE_STATS_INTERVAL; } @@ -2741,7 +2611,7 @@ rep_hist_format_desc_stats(time_t now) time_t rep_hist_desc_stats_write(time_t now) { - char *statsdir = NULL, *filename = NULL, *str = NULL; + char *filename = NULL, *str = NULL; if (!start_of_served_descs_stats_interval) return 0; /* We're not collecting stats. */ @@ -2751,10 +2621,8 @@ rep_hist_desc_stats_write(time_t now) str = rep_hist_format_desc_stats(now); tor_assert(str != NULL); - statsdir = get_datadir_fname("stats"); - if (check_private_dir(statsdir, CPD_CREATE, get_options()->User) < 0) { - log_warn(LD_HIST, "Unable to create stats/ directory!"); - goto done; + if (check_or_create_data_subdir("stats") < 0) { + goto done; } filename = get_datadir_fname2("stats", "served-desc-stats"); if (append_bytes_to_file(filename, str, strlen(str), 0) < 0) @@ -2763,7 +2631,6 @@ rep_hist_desc_stats_write(time_t now) rep_hist_reset_desc_stats(now); done: - tor_free(statsdir); tor_free(filename); tor_free(str); return start_of_served_descs_stats_interval + WRITE_STATS_INTERVAL; @@ -2981,7 +2848,7 @@ rep_hist_format_conn_stats(time_t now) time_t rep_hist_conn_stats_write(time_t now) { - char *statsdir = NULL, *filename = NULL, *str = NULL; + char *str = NULL; if (!start_of_conn_stats_interval) return 0; /* Not initialized. */ @@ -2995,28 +2862,21 @@ rep_hist_conn_stats_write(time_t now) rep_hist_reset_conn_stats(now); /* Try to write to disk. */ - statsdir = get_datadir_fname("stats"); - if (check_private_dir(statsdir, CPD_CREATE, get_options()->User) < 0) { - log_warn(LD_HIST, "Unable to create stats/ directory!"); - goto done; + if (!check_or_create_data_subdir("stats")) { + write_to_data_subdir("stats", "conn-stats", str, "connection statistics"); } - filename = get_datadir_fname2("stats", "conn-stats"); - if (write_str_to_file(filename, str, 0) < 0) - log_warn(LD_HIST, "Unable to write conn stats to disk!"); done: tor_free(str); - tor_free(filename); - tor_free(statsdir); return start_of_conn_stats_interval + WRITE_STATS_INTERVAL; } /** Internal statistics to track how many requests of each type of - * handshake we've received, and how many we've completed. Useful for - * seeing trends in cpu load. + * handshake we've received, and how many we've assigned to cpuworkers. + * Useful for seeing trends in cpu load. * @{ */ -static int onion_handshakes_requested[MAX_ONION_HANDSHAKE_TYPE+1] = {0}; -static int onion_handshakes_completed[MAX_ONION_HANDSHAKE_TYPE+1] = {0}; +STATIC int onion_handshakes_requested[MAX_ONION_HANDSHAKE_TYPE+1] = {0}; +STATIC int onion_handshakes_assigned[MAX_ONION_HANDSHAKE_TYPE+1] = {0}; /**@}*/ /** A new onionskin (using the <b>type</b> handshake) has arrived. */ @@ -3030,10 +2890,10 @@ rep_hist_note_circuit_handshake_requested(uint16_t type) /** We've sent an onionskin (using the <b>type</b> handshake) to a * cpuworker. */ void -rep_hist_note_circuit_handshake_completed(uint16_t type) +rep_hist_note_circuit_handshake_assigned(uint16_t type) { if (type <= MAX_ONION_HANDSHAKE_TYPE) - onion_handshakes_completed[type]++; + onion_handshakes_assigned[type]++; } /** Log our onionskin statistics since the last time we were called. */ @@ -3041,13 +2901,13 @@ void rep_hist_log_circuit_handshake_stats(time_t now) { (void)now; - log_notice(LD_HIST, "Circuit handshake stats since last time: " + log_notice(LD_HEARTBEAT, "Circuit handshake stats since last time: " "%d/%d TAP, %d/%d NTor.", - onion_handshakes_completed[ONION_HANDSHAKE_TYPE_TAP], + onion_handshakes_assigned[ONION_HANDSHAKE_TYPE_TAP], onion_handshakes_requested[ONION_HANDSHAKE_TYPE_TAP], - onion_handshakes_completed[ONION_HANDSHAKE_TYPE_NTOR], + onion_handshakes_assigned[ONION_HANDSHAKE_TYPE_NTOR], onion_handshakes_requested[ONION_HANDSHAKE_TYPE_NTOR]); - memset(onion_handshakes_completed, 0, sizeof(onion_handshakes_completed)); + memset(onion_handshakes_assigned, 0, sizeof(onion_handshakes_assigned)); memset(onion_handshakes_requested, 0, sizeof(onion_handshakes_requested)); } @@ -3061,11 +2921,9 @@ rep_hist_free_all(void) tor_free(write_array); tor_free(dir_read_array); tor_free(dir_write_array); - tor_free(last_stability_doc); tor_free(exit_bytes_read); tor_free(exit_bytes_written); tor_free(exit_streams); - built_last_stability_doc_at = 0; predicted_ports_free(); bidi_map_free(); diff --git a/src/or/rephist.h b/src/or/rephist.h index de824749b4..cd6231e6e4 100644 --- a/src/or/rephist.h +++ b/src/or/rephist.h @@ -47,7 +47,6 @@ double rep_hist_get_stability(const char *id, time_t when); double rep_hist_get_weighted_fractional_uptime(const char *id, time_t when); long rep_hist_get_weighted_time_known(const char *id, time_t when); int rep_hist_have_measured_enough_stability(void); -const char *rep_hist_get_router_stability_doc(time_t now); void rep_hist_note_used_port(time_t now, uint16_t port); smartlist_t *rep_hist_get_predicted_ports(time_t now); @@ -97,7 +96,7 @@ time_t rep_hist_conn_stats_write(time_t now); void rep_hist_conn_stats_term(void); void rep_hist_note_circuit_handshake_requested(uint16_t type); -void rep_hist_note_circuit_handshake_completed(uint16_t type); +void rep_hist_note_circuit_handshake_assigned(uint16_t type); void rep_hist_log_circuit_handshake_stats(time_t now); void rep_hist_free_all(void); diff --git a/src/or/replaycache.c b/src/or/replaycache.c index 59b98489b7..90f87c12d5 100644 --- a/src/or/replaycache.c +++ b/src/or/replaycache.c @@ -63,9 +63,9 @@ replaycache_new(time_t horizon, time_t interval) /** See documentation for replaycache_add_and_test() */ -int +STATIC int replaycache_add_and_test_internal( - time_t present, replaycache_t *r, const void *data, int len, + time_t present, replaycache_t *r, const void *data, size_t len, time_t *elapsed) { int rv = 0; @@ -73,7 +73,7 @@ replaycache_add_and_test_internal( time_t *access_time; /* sanity check */ - if (present <= 0 || !r || !data || len <= 0) { + if (present <= 0 || !r || !data || len == 0) { log_info(LD_BUG, "replaycache_add_and_test_internal() called with stupid" " parameters; please fix this."); goto done; @@ -127,14 +127,13 @@ replaycache_add_and_test_internal( /** See documentation for replaycache_scrub_if_needed() */ -void +STATIC void replaycache_scrub_if_needed_internal(time_t present, replaycache_t *r) { digestmap_iter_t *itr = NULL; const char *digest; void *valp; time_t *access_time; - char scrub_this; /* sanity check */ if (!r || !(r->digests_seen)) { @@ -152,20 +151,10 @@ replaycache_scrub_if_needed_internal(time_t present, replaycache_t *r) /* okay, scrub time */ itr = digestmap_iter_init(r->digests_seen); while (!digestmap_iter_done(itr)) { - scrub_this = 0; digestmap_iter_get(itr, &digest, &valp); access_time = (time_t *)valp; - if (access_time) { - /* aged out yet? */ - if (*access_time < present - r->horizon) scrub_this = 1; - } else { - /* Buh? Get rid of it, anyway */ - log_info(LD_BUG, "replaycache_scrub_if_needed_internal() saw a NULL" - " entry in the digestmap."); - scrub_this = 1; - } - - if (scrub_this) { + /* aged out yet? */ + if (*access_time < present - r->horizon) { /* Advance the iterator and remove this one */ itr = digestmap_iter_next_rmv(r->digests_seen, itr); /* Free the value removed */ @@ -187,7 +176,7 @@ replaycache_scrub_if_needed_internal(time_t present, replaycache_t *r) */ int -replaycache_add_and_test(replaycache_t *r, const void *data, int len) +replaycache_add_and_test(replaycache_t *r, const void *data, size_t len) { return replaycache_add_and_test_internal(time(NULL), r, data, len, NULL); } @@ -198,7 +187,7 @@ replaycache_add_and_test(replaycache_t *r, const void *data, int len) int replaycache_add_test_and_elapsed( - replaycache_t *r, const void *data, int len, time_t *elapsed) + replaycache_t *r, const void *data, size_t len, time_t *elapsed) { return replaycache_add_and_test_internal(time(NULL), r, data, len, elapsed); } diff --git a/src/or/replaycache.h b/src/or/replaycache.h index de20cab627..cd713fe891 100644 --- a/src/or/replaycache.h +++ b/src/or/replaycache.h @@ -45,10 +45,10 @@ replaycache_t * replaycache_new(time_t horizon, time_t interval); * testing. For everything else, use the wrappers below instead. */ -int replaycache_add_and_test_internal( - time_t present, replaycache_t *r, const void *data, int len, +STATIC int replaycache_add_and_test_internal( + time_t present, replaycache_t *r, const void *data, size_t len, time_t *elapsed); -void replaycache_scrub_if_needed_internal( +STATIC void replaycache_scrub_if_needed_internal( time_t present, replaycache_t *r); #endif /* REPLAYCACHE_PRIVATE */ @@ -57,9 +57,9 @@ void replaycache_scrub_if_needed_internal( * replaycache_t methods */ -int replaycache_add_and_test(replaycache_t *r, const void *data, int len); +int replaycache_add_and_test(replaycache_t *r, const void *data, size_t len); int replaycache_add_test_and_elapsed( - replaycache_t *r, const void *data, int len, time_t *elapsed); + replaycache_t *r, const void *data, size_t len, time_t *elapsed); void replaycache_scrub_if_needed(replaycache_t *r); #endif diff --git a/src/or/router.c b/src/or/router.c index eabd9c3f59..2cdbb0c8bb 100644 --- a/src/or/router.c +++ b/src/or/router.c @@ -232,7 +232,8 @@ get_server_identity_key(void) return server_identitykey; } -/** Return true iff the server identity key has been set. */ +/** Return true iff we are a server and the server identity key + * has been set. */ int server_identity_key_is_set(void) { @@ -683,6 +684,63 @@ router_initialize_tls_context(void) (unsigned int)lifetime); } +/** Compute fingerprint (or hashed fingerprint if hashed is 1) and write + * it to 'fingerprint' (or 'hashed-fingerprint'). Return 0 on success, or + * -1 if Tor should die, + */ +STATIC int +router_write_fingerprint(int hashed) +{ + char *keydir = NULL, *cp = NULL; + const char *fname = hashed ? "hashed-fingerprint" : + "fingerprint"; + char fingerprint[FINGERPRINT_LEN+1]; + const or_options_t *options = get_options(); + char *fingerprint_line = NULL; + int result = -1; + + keydir = get_datadir_fname(fname); + log_info(LD_GENERAL,"Dumping %sfingerprint to \"%s\"...", + hashed ? "hashed " : "", keydir); + if (!hashed) { + if (crypto_pk_get_fingerprint(get_server_identity_key(), + fingerprint, 0) < 0) { + log_err(LD_GENERAL,"Error computing fingerprint"); + goto done; + } + } else { + if (crypto_pk_get_hashed_fingerprint(get_server_identity_key(), + fingerprint) < 0) { + log_err(LD_GENERAL,"Error computing hashed fingerprint"); + goto done; + } + } + + tor_asprintf(&fingerprint_line, "%s %s\n", options->Nickname, fingerprint); + + /* Check whether we need to write the (hashed-)fingerprint file. */ + + cp = read_file_to_str(keydir, RFTS_IGNORE_MISSING, NULL); + if (!cp || strcmp(cp, fingerprint_line)) { + if (write_str_to_file(keydir, fingerprint_line, 0)) { + log_err(LD_FS, "Error writing %sfingerprint line to file", + hashed ? "hashed " : ""); + goto done; + } + } + + log_notice(LD_GENERAL, "Your Tor %s identity key fingerprint is '%s %s'", + hashed ? "bridge's hashed" : "server's", options->Nickname, + fingerprint); + + result = 0; + done: + tor_free(cp); + tor_free(keydir); + tor_free(fingerprint_line); + return result; +} + /** Initialize all OR private keys, and the TLS context, as necessary. * On OPs, this only initializes the tls context. Return 0 on success, * or -1 if Tor should die. @@ -691,14 +749,10 @@ int init_keys(void) { char *keydir; - char fingerprint[FINGERPRINT_LEN+1]; - /*nickname<space>fp\n\0 */ - char fingerprint_line[MAX_NICKNAME_LEN+FINGERPRINT_LEN+3]; const char *mydesc; crypto_pk_t *prkey; char digest[DIGEST_LEN]; char v3_digest[DIGEST_LEN]; - char *cp; const or_options_t *options = get_options(); dirinfo_type_t type; time_t now = time(NULL); @@ -888,40 +942,16 @@ init_keys(void) } } - /* 5. Dump fingerprint to 'fingerprint' */ - keydir = get_datadir_fname("fingerprint"); - log_info(LD_GENERAL,"Dumping fingerprint to \"%s\"...",keydir); - if (crypto_pk_get_fingerprint(get_server_identity_key(), - fingerprint, 0) < 0) { - log_err(LD_GENERAL,"Error computing fingerprint"); - tor_free(keydir); + /* 5. Dump fingerprint and possibly hashed fingerprint to files. */ + if (router_write_fingerprint(0)) { + log_err(LD_FS, "Error writing fingerprint to file"); return -1; } - tor_assert(strlen(options->Nickname) <= MAX_NICKNAME_LEN); - if (tor_snprintf(fingerprint_line, sizeof(fingerprint_line), - "%s %s\n",options->Nickname, fingerprint) < 0) { - log_err(LD_GENERAL,"Error writing fingerprint line"); - tor_free(keydir); + if (!public_server_mode(options) && router_write_fingerprint(1)) { + log_err(LD_FS, "Error writing hashed fingerprint to file"); return -1; } - /* Check whether we need to write the fingerprint file. */ - cp = NULL; - if (file_status(keydir) == FN_FILE) - cp = read_file_to_str(keydir, 0, NULL); - if (!cp || strcmp(cp, fingerprint_line)) { - if (write_str_to_file(keydir, fingerprint_line, 0)) { - log_err(LD_FS, "Error writing fingerprint line to file"); - tor_free(keydir); - tor_free(cp); - return -1; - } - } - tor_free(cp); - tor_free(keydir); - log_notice(LD_GENERAL, - "Your Tor server's identity key fingerprint is '%s %s'", - options->Nickname, fingerprint); if (!authdir_mode(options)) return 0; /* 6. [authdirserver only] load approved-routers file */ @@ -931,12 +961,9 @@ init_keys(void) } /* 6b. [authdirserver only] add own key to approved directories. */ crypto_pk_get_digest(get_server_identity_key(), digest); - type = ((options->V1AuthoritativeDir ? V1_DIRINFO : NO_DIRINFO) | - (options->V2AuthoritativeDir ? V2_DIRINFO : NO_DIRINFO) | - (options->V3AuthoritativeDir ? + type = ((options->V3AuthoritativeDir ? (V3_DIRINFO|MICRODESC_DIRINFO|EXTRAINFO_DIRINFO) : NO_DIRINFO) | - (options->BridgeAuthoritativeDir ? BRIDGE_DIRINFO : NO_DIRINFO) | - (options->HSAuthoritativeDir ? HIDSERV_DIRINFO : NO_DIRINFO)); + (options->BridgeAuthoritativeDir ? BRIDGE_DIRINFO : NO_DIRINFO)); ds = router_get_trusteddirserver_by_digest(digest); if (!ds) { @@ -1149,7 +1176,7 @@ consider_testing_reachability(int test_or, int test_dir) /* XXX IPv6 self testing */ log_info(LD_CIRC, "Testing %s of my ORPort: %s:%d.", !orport_reachable ? "reachability" : "bandwidth", - me->address, me->or_port); + fmt_addr32(me->addr), me->or_port); circuit_launch_by_extend_info(CIRCUIT_PURPOSE_TESTING, ei, CIRCLAUNCH_NEED_CAPACITY|CIRCLAUNCH_IS_INTERNAL); extend_info_free(ei); @@ -1161,7 +1188,7 @@ consider_testing_reachability(int test_or, int test_dir) CONN_TYPE_DIR, &addr, me->dir_port, DIR_PURPOSE_FETCH_SERVERDESC)) { /* ask myself, via tor, for my server descriptor. */ - directory_initiate_command(me->address, &addr, + directory_initiate_command(&addr, me->or_port, me->dir_port, me->cache_info.identity_digest, DIR_PURPOSE_FETCH_SERVERDESC, @@ -1176,6 +1203,7 @@ router_orport_found_reachable(void) { const routerinfo_t *me = router_get_my_routerinfo(); if (!can_reach_or_port && me) { + char *address = tor_dup_ip(me->addr); log_notice(LD_OR,"Self-testing indicates your ORPort is reachable from " "the outside. Excellent.%s", get_options()->PublishServerDescriptor_ != NO_DIRINFO ? @@ -1184,7 +1212,8 @@ router_orport_found_reachable(void) mark_my_descriptor_dirty("ORPort found reachable"); control_event_server_status(LOG_NOTICE, "REACHABILITY_SUCCEEDED ORADDRESS=%s:%d", - me->address, me->or_port); + address, me->or_port); + tor_free(address); } } @@ -1194,6 +1223,7 @@ router_dirport_found_reachable(void) { const routerinfo_t *me = router_get_my_routerinfo(); if (!can_reach_dir_port && me) { + char *address = tor_dup_ip(me->addr); log_notice(LD_DIRSERV,"Self-testing indicates your DirPort is reachable " "from the outside. Excellent."); can_reach_dir_port = 1; @@ -1201,7 +1231,8 @@ router_dirport_found_reachable(void) mark_my_descriptor_dirty("DirPort found reachable"); control_event_server_status(LOG_NOTICE, "REACHABILITY_SUCCEEDED DIRADDRESS=%s:%d", - me->address, me->dir_port); + address, me->dir_port); + tor_free(address); } } @@ -1236,7 +1267,8 @@ router_perform_bandwidth_test(int num_circs, time_t now) } /** Return true iff our network is in some sense disabled: either we're - * hibernating, entering hibernation, or */ + * hibernating, entering hibernation, or the network is turned off with + * DisableNetwork. */ int net_is_disabled(void) { @@ -1251,22 +1283,6 @@ authdir_mode(const or_options_t *options) { return options->AuthoritativeDir != 0; } -/** Return true iff we believe ourselves to be a v1 authoritative - * directory server. - */ -int -authdir_mode_v1(const or_options_t *options) -{ - return authdir_mode(options) && options->V1AuthoritativeDir != 0; -} -/** Return true iff we believe ourselves to be a v2 authoritative - * directory server. - */ -int -authdir_mode_v2(const or_options_t *options) -{ - return authdir_mode(options) && options->V2AuthoritativeDir != 0; -} /** Return true iff we believe ourselves to be a v3 authoritative * directory server. */ @@ -1275,13 +1291,11 @@ authdir_mode_v3(const or_options_t *options) { return authdir_mode(options) && options->V3AuthoritativeDir != 0; } -/** Return true iff we are a v1, v2, or v3 directory authority. */ +/** Return true iff we are a v3 directory authority. */ int authdir_mode_any_main(const or_options_t *options) { - return options->V1AuthoritativeDir || - options->V2AuthoritativeDir || - options->V3AuthoritativeDir; + return options->V3AuthoritativeDir; } /** Return true if we believe ourselves to be any kind of * authoritative directory beyond just a hidserv authority. */ @@ -1335,8 +1349,8 @@ authdir_mode_bridge(const or_options_t *options) /** Return true iff we are trying to be a server. */ -int -server_mode(const or_options_t *options) +MOCK_IMPL(int, +server_mode,(const or_options_t *options)) { if (options->ClientOnly) return 0; /* XXXX024 I believe we can kill off ORListenAddress here.*/ @@ -1345,8 +1359,8 @@ server_mode(const or_options_t *options) /** Return true iff we are trying to be a non-bridge server. */ -int -public_server_mode(const or_options_t *options) +MOCK_IMPL(int, +public_server_mode,(const or_options_t *options)) { if (!server_mode(options)) return 0; return (!options->BridgeRelay); @@ -1674,22 +1688,10 @@ router_is_me(const routerinfo_t *router) return router_digest_is_me(router->cache_info.identity_digest); } -/** Return true iff <b>fp</b> is a hex fingerprint of my identity digest. */ -int -router_fingerprint_is_me(const char *fp) -{ - char digest[DIGEST_LEN]; - if (strlen(fp) == HEX_DIGEST_LEN && - base16_decode(digest, sizeof(digest), fp, HEX_DIGEST_LEN) == 0) - return router_digest_is_me(digest); - - return 0; -} - /** Return a routerinfo for this OR, rebuilding a fresh one if * necessary. Return NULL on error, or if called on an OP. */ -const routerinfo_t * -router_get_my_routerinfo(void) +MOCK_IMPL(const routerinfo_t *, +router_get_my_routerinfo,(void)) { if (!server_mode(get_options())) return NULL; @@ -1793,7 +1795,6 @@ router_rebuild_descriptor(int force) ri = tor_malloc_zero(sizeof(routerinfo_t)); ri->cache_info.routerlist_index = -1; - ri->address = tor_dup_ip(addr); ri->nickname = tor_strdup(options->Nickname); ri->addr = addr; ri->or_port = router_get_advertised_or_port(options); @@ -1858,7 +1859,7 @@ router_rebuild_descriptor(int force) policies_parse_exit_policy(options->ExitPolicy, &ri->exit_policy, options->IPv6Exit, options->ExitPolicyRejectPrivate, - ri->address, !options->BridgeRelay); + ri->addr, !options->BridgeRelay); } ri->policy_is_reject_star = policy_is_reject_star(ri->exit_policy, AF_INET) && @@ -1871,12 +1872,6 @@ router_rebuild_descriptor(int force) tor_free(p_tmp); } -#if 0 - /* XXXX NM NM I belive this is safe to remove */ - if (authdir_mode(options)) - ri->is_valid = ri->is_named = 1; /* believe in yourself */ -#endif - if (options->MyFamily && ! options->BridgeRelay) { smartlist_t *family; if (!warned_nonexistent_family) @@ -2249,7 +2244,7 @@ router_guess_address_from_dir_headers(uint32_t *guess) * string describing the version of Tor and the operating system we're * currently running on. */ -void +STATIC void get_platform_str(char *platform, size_t len) { tor_snprintf(platform, len, "Tor %s on %s", @@ -2270,8 +2265,7 @@ char * router_dump_router_to_string(routerinfo_t *router, crypto_pk_t *ident_key) { - /* XXXX025 Make this look entirely at its arguments, and not at globals. - */ + char *address = NULL; char *onion_pkey = NULL; /* Onion key, PEM-encoded. */ char *identity_pkey = NULL; /* Identity key, PEM-encoded. */ char digest[DIGEST_LEN]; @@ -2345,7 +2339,9 @@ router_dump_router_to_string(routerinfo_t *router, } } + address = tor_dup_ip(router->addr); chunks = smartlist_new(); + /* Generate the easy portion of the router descriptor. */ smartlist_add_asprintf(chunks, "router %s %s %d 0 %d\n" @@ -2361,7 +2357,7 @@ router_dump_router_to_string(routerinfo_t *router, "signing-key\n%s" "%s%s%s%s", router->nickname, - router->address, + address, router->or_port, decide_to_advertise_dirport(options, router->dir_port), extra_or_address ? extra_or_address : "", @@ -2403,20 +2399,13 @@ router_dump_router_to_string(routerinfo_t *router, if (!router->exit_policy || !smartlist_len(router->exit_policy)) { smartlist_add(chunks, tor_strdup("reject *:*\n")); } else if (router->exit_policy) { - int i; - for (i = 0; i < smartlist_len(router->exit_policy); ++i) { - char pbuf[POLICY_BUF_LEN]; - addr_policy_t *tmpe = smartlist_get(router->exit_policy, i); - int result; - if (tor_addr_family(&tmpe->addr) == AF_INET6) - continue; /* Don't include IPv6 parts of address policy */ - result = policy_write_item(pbuf, POLICY_BUF_LEN, tmpe, 1); - if (result < 0) { - log_warn(LD_BUG,"descriptor policy_write_item ran out of room!"); - goto err; - } - smartlist_add_asprintf(chunks, "%s\n", pbuf); - } + char *exit_policy = router_dump_exit_policy_to_string(router,1,0); + + if (!exit_policy) + goto err; + + smartlist_add_asprintf(chunks, "%s\n", exit_policy); + tor_free(exit_policy); } if (router->ipv6_exit_policy) { @@ -2475,6 +2464,7 @@ router_dump_router_to_string(routerinfo_t *router, SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp)); smartlist_free(chunks); } + tor_free(address); tor_free(family_line); tor_free(onion_pkey); tor_free(identity_pkey); @@ -2483,6 +2473,56 @@ router_dump_router_to_string(routerinfo_t *router, return output; } +/** + * OR only: Given <b>router</b>, produce a string with its exit policy. + * If <b>include_ipv4</b> is true, include IPv4 entries. + * If <b>include_ipv6</b> is true, include IPv6 entries. + */ +char * +router_dump_exit_policy_to_string(const routerinfo_t *router, + int include_ipv4, + int include_ipv6) +{ + smartlist_t *exit_policy_strings; + char *policy_string = NULL; + + if ((!router->exit_policy) || (router->policy_is_reject_star)) { + return tor_strdup("reject *:*"); + } + + exit_policy_strings = smartlist_new(); + + SMARTLIST_FOREACH_BEGIN(router->exit_policy, addr_policy_t *, tmpe) { + char *pbuf; + int bytes_written_to_pbuf; + if ((tor_addr_family(&tmpe->addr) == AF_INET6) && (!include_ipv6)) { + continue; /* Don't include IPv6 parts of address policy */ + } + if ((tor_addr_family(&tmpe->addr) == AF_INET) && (!include_ipv4)) { + continue; /* Don't include IPv4 parts of address policy */ + } + + pbuf = tor_malloc(POLICY_BUF_LEN); + bytes_written_to_pbuf = policy_write_item(pbuf,POLICY_BUF_LEN, tmpe, 1); + + if (bytes_written_to_pbuf < 0) { + log_warn(LD_BUG, "router_dump_exit_policy_to_string ran out of room!"); + tor_free(pbuf); + goto done; + } + + smartlist_add(exit_policy_strings,pbuf); + } SMARTLIST_FOREACH_END(tmpe); + + policy_string = smartlist_join_strings(exit_policy_strings, "\n", 0, NULL); + + done: + SMARTLIST_FOREACH(exit_policy_strings, char *, str, tor_free(str)); + smartlist_free(exit_policy_strings); + + return policy_string; +} + /** Copy the primary (IPv4) OR port (IP address and TCP port) for * <b>router</b> into *<b>ap_out</b>. */ void diff --git a/src/or/router.h b/src/or/router.h index 60095d087b..d18ff065ea 100644 --- a/src/or/router.h +++ b/src/or/router.h @@ -12,6 +12,8 @@ #ifndef TOR_ROUTER_H #define TOR_ROUTER_H +#include "testsupport.h" + crypto_pk_t *get_onion_key(void); time_t get_onion_key_set_at(void); void set_server_identity_key(crypto_pk_t *k); @@ -48,8 +50,6 @@ void router_perform_bandwidth_test(int num_circs, time_t now); int net_is_disabled(void); int authdir_mode(const or_options_t *options); -int authdir_mode_v1(const or_options_t *options); -int authdir_mode_v2(const or_options_t *options); int authdir_mode_v3(const or_options_t *options); int authdir_mode_any_main(const or_options_t *options); int authdir_mode_any_nonhidserv(const or_options_t *options); @@ -66,8 +66,8 @@ uint16_t router_get_advertised_or_port_by_af(const or_options_t *options, uint16_t router_get_advertised_dir_port(const or_options_t *options, uint16_t dirport); -int server_mode(const or_options_t *options); -int public_server_mode(const or_options_t *options); +MOCK_DECL(int, server_mode, (const or_options_t *options)); +MOCK_DECL(int, public_server_mode, (const or_options_t *options)); int advertised_server_mode(void); int proxy_mode(const or_options_t *options); void consider_publishable_server(int force); @@ -82,7 +82,7 @@ void router_new_address_suggestion(const char *suggestion, const dir_connection_t *d_conn); int router_compare_to_my_exit_policy(const tor_addr_t *addr, uint16_t port); int router_my_exit_policy_is_reject_star(void); -const routerinfo_t *router_get_my_routerinfo(void); +MOCK_DECL(const routerinfo_t *, router_get_my_routerinfo, (void)); extrainfo_t *router_get_my_extrainfo(void); const char *router_get_my_descriptor(void); const char *router_get_descriptor_gen_reason(void); @@ -90,11 +90,13 @@ int router_digest_is_me(const char *digest); const uint8_t *router_get_my_id_digest(void); int router_extrainfo_digest_is_me(const char *digest); int router_is_me(const routerinfo_t *router); -int router_fingerprint_is_me(const char *fp); int router_pick_published_address(const or_options_t *options, uint32_t *addr); int router_rebuild_descriptor(int force); char *router_dump_router_to_string(routerinfo_t *router, crypto_pk_t *ident_key); +char *router_dump_exit_policy_to_string(const routerinfo_t *router, + int include_ipv4, + int include_ipv6); void router_get_prim_orport(const routerinfo_t *router, tor_addr_port_t *addr_port_out); void router_get_pref_orport(const routerinfo_t *router, @@ -146,7 +148,8 @@ smartlist_t *router_get_all_orports(const routerinfo_t *ri); #ifdef ROUTER_PRIVATE /* Used only by router.c and test.c */ -void get_platform_str(char *platform, size_t len); +STATIC void get_platform_str(char *platform, size_t len); +STATIC int router_write_fingerprint(int hashed); #endif #endif diff --git a/src/or/routerlist.c b/src/or/routerlist.c index 8fe496b51e..b5e924522e 100644 --- a/src/or/routerlist.c +++ b/src/or/routerlist.c @@ -37,7 +37,7 @@ #include "routerlist.h" #include "routerparse.h" #include "routerset.h" - +#include "../common/sandbox.h" // #define DEBUG_ROUTERLIST /****************************************************************************/ @@ -98,7 +98,8 @@ static smartlist_t *trusted_dir_servers = NULL; * and all fallback directory servers. */ static smartlist_t *fallback_dir_servers = NULL; -/** List of for a given authority, and download status for latest certificate. +/** List of certificates for a single authority, and download status for + * latest certificate. */ struct cert_list_t { /* @@ -130,16 +131,6 @@ static smartlist_t *warned_nicknames = NULL; * download is low. */ static time_t last_descriptor_download_attempted = 0; -/** When we last computed the weights to use for bandwidths on directory - * requests, what were the total weighted bandwidth, and our share of that - * bandwidth? Used to determine what fraction of directory requests we should - * expect to see. - * - * @{ */ -static uint64_t sl_last_total_weighted_bw = 0, - sl_last_weighted_bw_of_me = 0; -/**@}*/ - /** Return the number of directory authorities whose type matches some bit set * in <b>type</b> */ int @@ -220,8 +211,6 @@ download_status_is_ready_by_sk_in_cl(cert_list_t *cl, return rv; } -#define get_n_v2_authorities() get_n_authorities(V2_DIRINFO) - /** Helper: Return the cert_list_t for an authority whose authority ID is * <b>id_digest</b>, allocating a new list if necessary. */ static cert_list_t * @@ -449,7 +438,7 @@ trusted_dirs_flush_certs_to_disk(void) } DIGESTMAP_FOREACH_END; filename = get_datadir_fname("cached-certs"); - if (write_chunks_to_file(filename, chunks, 0)) { + if (write_chunks_to_file(filename, chunks, 0, 0)) { log_warn(LD_FS, "Error writing certificates to disk."); } tor_free(filename); @@ -681,9 +670,6 @@ authority_cert_dl_looks_uncertain(const char *id_digest) return n_failures >= N_AUTH_CERT_DL_FAILURES_TO_BUG_USER; } -/** How many times will we try to fetch a certificate before giving up? */ -#define MAX_CERT_DL_FAILURES 8 - /** Try to download any v3 authority certificates that we may be missing. If * <b>status</b> is provided, try to get all the ones that were used to sign * <b>status</b>. Additionally, try to have a non-expired certificate for @@ -715,7 +701,7 @@ authority_certs_fetch_missing(networkstatus_t *status, time_t now) char id_digest_str[2*DIGEST_LEN+1]; char sk_digest_str[2*DIGEST_LEN+1]; - if (should_delay_dir_fetches(get_options())) + if (should_delay_dir_fetches(get_options(), NULL)) return; pending_cert = fp_pair_map_new(); @@ -755,7 +741,7 @@ authority_certs_fetch_missing(networkstatus_t *status, time_t now) } SMARTLIST_FOREACH_END(cert); if (!found && download_status_is_ready(&(cl->dl_status_by_id), now, - MAX_CERT_DL_FAILURES) && + get_options()->TestingCertMaxDownloadTries) && !digestmap_get(pending_id, ds->v3_identity_digest)) { log_info(LD_DIR, "No current certificate known for authority %s " @@ -817,7 +803,7 @@ authority_certs_fetch_missing(networkstatus_t *status, time_t now) } if (download_status_is_ready_by_sk_in_cl( cl, sig->signing_key_digest, - now, MAX_CERT_DL_FAILURES) && + now, get_options()->TestingCertMaxDownloadTries) && !fp_pair_map_get_by_digests(pending_cert, voter->identity_digest, sig->signing_key_digest)) { @@ -1103,15 +1089,18 @@ router_rebuild_store(int flags, desc_store_t *store) smartlist_add(chunk_list, c); } SMARTLIST_FOREACH_END(sd); - if (write_chunks_to_file(fname_tmp, chunk_list, 1)<0) { + if (write_chunks_to_file(fname_tmp, chunk_list, 1, 1)<0) { log_warn(LD_FS, "Error writing router store to disk."); goto done; } /* Our mmap is now invalid. */ if (store->mmap) { - tor_munmap_file(store->mmap); + int res = tor_munmap_file(store->mmap); store->mmap = NULL; + if (res != 0) { + log_warn(LD_FS, "Unable to munmap route store in %s", fname); + } } if (replace_file(fname_tmp, fname)<0) { @@ -1178,32 +1167,25 @@ router_rebuild_store(int flags, desc_store_t *store) static int router_reload_router_list_impl(desc_store_t *store) { - char *fname = NULL, *altname = NULL, *contents = NULL; + char *fname = NULL, *contents = NULL; struct stat st; - int read_from_old_location = 0; int extrainfo = (store->type == EXTRAINFO_STORE); - time_t now = time(NULL); store->journal_len = store->store_len = 0; fname = get_datadir_fname(store->fname_base); - if (store->fname_alt_base) - altname = get_datadir_fname(store->fname_alt_base); - if (store->mmap) /* get rid of it first */ - tor_munmap_file(store->mmap); - store->mmap = NULL; + if (store->mmap) { + /* get rid of it first */ + int res = tor_munmap_file(store->mmap); + store->mmap = NULL; + if (res != 0) { + log_warn(LD_FS, "Failed to munmap %s", fname); + tor_free(fname); + return -1; + } + } store->mmap = tor_mmap_file(fname); - if (!store->mmap && altname && file_status(altname) == FN_FILE) { - read_from_old_location = 1; - log_notice(LD_DIR, "Couldn't read %s; trying to load routers from old " - "location %s.", fname, altname); - if ((store->mmap = tor_mmap_file(altname))) - read_from_old_location = 1; - } - if (altname && !read_from_old_location) { - remove_file_if_very_old(altname, now); - } if (store->mmap) { store->store_len = store->mmap->size; if (extrainfo) @@ -1220,14 +1202,6 @@ router_reload_router_list_impl(desc_store_t *store) fname = get_datadir_fname_suffix(store->fname_base, ".new"); if (file_status(fname) == FN_FILE) contents = read_file_to_str(fname, RFTS_BIN|RFTS_IGNORE_MISSING, &st); - if (read_from_old_location) { - tor_free(altname); - altname = get_datadir_fname_suffix(store->fname_alt_base, ".new"); - if (!contents) - contents = read_file_to_str(altname, RFTS_BIN|RFTS_IGNORE_MISSING, &st); - else - remove_file_if_very_old(altname, now); - } if (contents) { if (extrainfo) router_load_extrainfo_from_string(contents, NULL,SAVED_IN_JOURNAL, @@ -1240,9 +1214,8 @@ router_reload_router_list_impl(desc_store_t *store) } tor_free(fname); - tor_free(altname); - if (store->journal_len || read_from_old_location) { + if (store->journal_len) { /* Always clear the journal on startup.*/ router_rebuild_store(RRS_FORCE, store); } else if (!extrainfo) { @@ -1309,8 +1282,6 @@ const routerstatus_t * router_pick_directory_server(dirinfo_type_t type, int flags) { const routerstatus_t *choice; - if (get_options()->PreferTunneledDirConns) - flags |= PDS_PREFER_TUNNELED_DIR_CONNS_; if (!routerlist) return NULL; @@ -1329,47 +1300,6 @@ router_pick_directory_server(dirinfo_type_t type, int flags) return choice; } -/** Try to determine which fraction of v2 and v3 directory requests aimed at - * caches will be sent to us. Set *<b>v2_share_out</b> and - * *<b>v3_share_out</b> to the fractions of v2 and v3 protocol shares we - * expect to see, respectively. Return 0 on success, negative on failure. */ -int -router_get_my_share_of_directory_requests(double *v2_share_out, - double *v3_share_out) -{ - const routerinfo_t *me = router_get_my_routerinfo(); - const routerstatus_t *rs; - const int pds_flags = PDS_ALLOW_SELF|PDS_IGNORE_FASCISTFIREWALL; - *v2_share_out = *v3_share_out = 0.0; - if (!me) - return -1; - rs = router_get_consensus_status_by_id(me->cache_info.identity_digest); - if (!rs) - return -1; - - /* Calling for side effect */ - /* XXXX This is a bit of a kludge */ - if (rs->is_v2_dir) { - sl_last_total_weighted_bw = 0; - router_pick_directory_server(V2_DIRINFO, pds_flags); - if (sl_last_total_weighted_bw != 0) { - *v2_share_out = U64_TO_DBL(sl_last_weighted_bw_of_me) / - U64_TO_DBL(sl_last_total_weighted_bw); - } - } - - { - sl_last_total_weighted_bw = 0; - router_pick_directory_server(V3_DIRINFO, pds_flags); - if (sl_last_total_weighted_bw != 0) { - *v3_share_out = U64_TO_DBL(sl_last_weighted_bw_of_me) / - U64_TO_DBL(sl_last_total_weighted_bw); - } - } - - return 0; -} - /** Return the dir_server_t for the directory authority whose identity * key hashes to <b>digest</b>, or NULL if no such authority is known. */ @@ -1435,7 +1365,7 @@ router_pick_trusteddirserver(dirinfo_type_t type, int flags) return router_pick_dirserver_generic(trusted_dir_servers, type, flags); } -/** Try to find a running fallback directory Flags are as for +/** Try to find a running fallback directory. Flags are as for * router_pick_directory_server. */ const routerstatus_t * @@ -1444,7 +1374,7 @@ router_pick_fallback_dirserver(dirinfo_type_t type, int flags) return router_pick_dirserver_generic(fallback_dir_servers, type, flags); } -/** Try to find a running fallback directory Flags are as for +/** Try to find a running fallback directory. Flags are as for * router_pick_directory_server. */ static const routerstatus_t * @@ -1453,8 +1383,6 @@ router_pick_dirserver_generic(smartlist_t *sourcelist, { const routerstatus_t *choice; int busy = 0; - if (get_options()->PreferTunneledDirConns) - flags |= PDS_PREFER_TUNNELED_DIR_CONNS_; choice = router_pick_trusteddirserver_impl(sourcelist, type, flags, &busy); if (choice || !(flags & PDS_RETRY_IF_NO_SERVERS)) @@ -1479,10 +1407,7 @@ router_pick_dirserver_generic(smartlist_t *sourcelist, /** Pick a random running valid directory server/mirror from our * routerlist. Arguments are as for router_pick_directory_server(), except - * that RETRY_IF_NO_SERVERS is ignored, and: - * - * If the PDS_PREFER_TUNNELED_DIR_CONNS_ flag is set, prefer directory servers - * that we can use with BEGINDIR. + * that RETRY_IF_NO_SERVERS is ignored. */ static const routerstatus_t * router_pick_directory_server_impl(dirinfo_type_t type, int flags) @@ -1496,7 +1421,6 @@ router_pick_directory_server_impl(dirinfo_type_t type, int flags) const networkstatus_t *consensus = networkstatus_get_latest_consensus(); int requireother = ! (flags & PDS_ALLOW_SELF); int fascistfirewall = ! (flags & PDS_IGNORE_FASCISTFIREWALL); - int prefer_tunnel = (flags & PDS_PREFER_TUNNELED_DIR_CONNS_); int for_guard = (flags & PDS_FOR_GUARD); int try_excluding = 1, n_excluded = 0; @@ -1529,8 +1453,6 @@ router_pick_directory_server_impl(dirinfo_type_t type, int flags) if (requireother && router_digest_is_me(node->identity)) continue; is_trusted = router_digest_is_trusted_dir(node->identity); - if ((type & V2_DIRINFO) && !(node->rs->is_v2_dir || is_trusted)) - continue; if ((type & EXTRAINFO_DIRINFO) && !router_supports_extrainfo(node->identity, 0)) continue; @@ -1551,8 +1473,7 @@ router_pick_directory_server_impl(dirinfo_type_t type, int flags) is_overloaded = status->last_dir_503_at + DIR_503_TIMEOUT > now; - if (prefer_tunnel && - (!fascistfirewall || + if ((!fascistfirewall || fascist_firewall_allows_address_or(&addr, status->or_port))) smartlist_add(is_trusted ? trusted_tunnel : is_overloaded ? overloaded_tunnel : tunnel, (void*)node); @@ -1639,7 +1560,6 @@ router_pick_trusteddirserver_impl(const smartlist_t *sourcelist, time_t now = time(NULL); const int requireother = ! (flags & PDS_ALLOW_SELF); const int fascistfirewall = ! (flags & PDS_IGNORE_FASCISTFIREWALL); - const int prefer_tunnel = (flags & PDS_PREFER_TUNNELED_DIR_CONNS_); const int no_serverdesc_fetching =(flags & PDS_NO_EXISTING_SERVERDESC_FETCH); const int no_microdesc_fetching =(flags & PDS_NO_EXISTING_MICRODESC_FETCH); const double auth_weight = (sourcelist == fallback_dir_servers) ? @@ -1700,8 +1620,7 @@ router_pick_trusteddirserver_impl(const smartlist_t *sourcelist, } } - if (prefer_tunnel && - d->or_port && + if (d->or_port && (!fascistfirewall || fascist_firewall_allows_address_or(&addr, d->or_port))) smartlist_add(is_overloaded ? overloaded_tunnel : tunnel, (void*)d); @@ -1757,7 +1676,6 @@ mark_all_dirservers_up(smartlist_t *server_list) routerstatus_t *rs; node_t *node; dir->is_running = 1; - download_status_reset(&dir->v2_ns_dl_status); node = node_get_mutable_by_id(dir->digest); if (node) node->is_running = 1; @@ -1879,7 +1797,7 @@ router_get_advertised_bandwidth_capped(const routerinfo_t *router) * doubles, convert them to uint64_t, and try to scale them linearly so as to * much of the range of uint64_t. If <b>total_out</b> is provided, set it to * the sum of all elements in the array _before_ scaling. */ -/* private */ void +STATIC void scale_array_elements_to_u64(u64_dbl_t *entries, int n_entries, uint64_t *total_out) { @@ -1922,7 +1840,7 @@ gt_i64_timei(uint64_t a, uint64_t b) * value, and return the index of that element. If all elements are 0, choose * an index at random. Return -1 on error. */ -/* private */ int +STATIC int choose_array_element_by_weight(const u64_dbl_t *entries, int n_entries) { int i, i_chosen=-1, n_chosen=0; @@ -2015,8 +1933,7 @@ smartlist_choose_node_by_bandwidth_weights(const smartlist_t *sl, if (compute_weighted_bandwidths(sl, rule, &bandwidths) < 0) return NULL; - scale_array_elements_to_u64(bandwidths, smartlist_len(sl), - &sl_last_total_weighted_bw); + scale_array_elements_to_u64(bandwidths, smartlist_len(sl), NULL); { int idx = choose_array_element_by_weight(bandwidths, @@ -2125,7 +2042,7 @@ compute_weighted_bandwidths(const smartlist_t *sl, // Cycle through smartlist and total the bandwidth. SMARTLIST_FOREACH_BEGIN(sl, const node_t *, node) { - int is_exit = 0, is_guard = 0, is_dir = 0, this_bw = 0, is_me = 0; + int is_exit = 0, is_guard = 0, is_dir = 0, this_bw = 0; double weight = 1; is_exit = node->is_exit && ! node->is_bad_exit; is_guard = node->is_possible_guard; @@ -2148,7 +2065,6 @@ compute_weighted_bandwidths(const smartlist_t *sl, /* We can't use this one. */ continue; } - is_me = router_digest_is_me(node->identity); if (is_guard && is_exit) { weight = (is_dir ? Wdb*Wd : Wd); @@ -2167,8 +2083,6 @@ compute_weighted_bandwidths(const smartlist_t *sl, weight = 0.0; bandwidths[node_sl_idx].dbl = weight*this_bw + 0.5; - if (is_me) - sl_last_weighted_bw_of_me = (uint64_t) bandwidths[node_sl_idx].dbl; } SMARTLIST_FOREACH_END(node); log_debug(LD_CIRC, "Generated weighted bandwidths for rule %s based " @@ -2250,7 +2164,6 @@ smartlist_choose_node_by_bandwidth(const smartlist_t *sl, bitarray_t *fast_bits; bitarray_t *exit_bits; bitarray_t *guard_bits; - int me_idx = -1; // This function does not support WEIGHT_FOR_DIR // or WEIGHT_FOR_MID @@ -2284,9 +2197,6 @@ smartlist_choose_node_by_bandwidth(const smartlist_t *sl, uint32_t this_bw = 0; i = node_sl_idx; - if (router_digest_is_me(node->identity)) - me_idx = node_sl_idx; - is_exit = node->is_exit; is_guard = node->is_possible_guard; if (node->rs) { @@ -2390,7 +2300,6 @@ smartlist_choose_node_by_bandwidth(const smartlist_t *sl, if (guard_weight <= 0.0) guard_weight = 0.0; - sl_last_weighted_bw_of_me = 0; for (i=0; i < (unsigned)smartlist_len(sl); i++) { tor_assert(bandwidths[i].dbl >= 0.0); @@ -2402,9 +2311,6 @@ smartlist_choose_node_by_bandwidth(const smartlist_t *sl, bandwidths[i].dbl *= guard_weight; else if (is_exit) bandwidths[i].dbl *= exit_weight; - - if (i == (unsigned) me_idx) - sl_last_weighted_bw_of_me = (uint64_t) bandwidths[i].dbl; } } @@ -2423,8 +2329,7 @@ smartlist_choose_node_by_bandwidth(const smartlist_t *sl, guard_weight, (int)(rule == WEIGHT_FOR_GUARD)); #endif - scale_array_elements_to_u64(bandwidths, smartlist_len(sl), - &sl_last_total_weighted_bw); + scale_array_elements_to_u64(bandwidths, smartlist_len(sl), NULL); { int idx = choose_array_element_by_weight(bandwidths, @@ -2816,7 +2721,6 @@ router_get_routerlist(void) routerlist->extra_info_map = eimap_new(); routerlist->desc_store.fname_base = "cached-descriptors"; - routerlist->desc_store.fname_alt_base = "cached-routers"; routerlist->extrainfo_store.fname_base = "cached-extrainfo"; routerlist->desc_store.type = ROUTER_STORE; @@ -2836,7 +2740,6 @@ routerinfo_free(routerinfo_t *router) return; tor_free(router->cache_info.signed_descriptor_body); - tor_free(router->address); tor_free(router->nickname); tor_free(router->platform); tor_free(router->contact_info); @@ -2922,10 +2825,18 @@ routerlist_free(routerlist_t *rl) signed_descriptor_free(sd)); smartlist_free(rl->routers); smartlist_free(rl->old_routers); - if (routerlist->desc_store.mmap) - tor_munmap_file(routerlist->desc_store.mmap); - if (routerlist->extrainfo_store.mmap) - tor_munmap_file(routerlist->extrainfo_store.mmap); + if (rl->desc_store.mmap) { + int res = tor_munmap_file(routerlist->desc_store.mmap); + if (res != 0) { + log_warn(LD_FS, "Failed to munmap routerlist->desc_store.mmap"); + } + } + if (rl->extrainfo_store.mmap) { + int res = tor_munmap_file(routerlist->extrainfo_store.mmap); + if (res != 0) { + log_warn(LD_FS, "Failed to munmap routerlist->extrainfo_store.mmap"); + } + } tor_free(rl); router_dir_info_changed(); @@ -3412,7 +3323,6 @@ router_add_to_routerlist(routerinfo_t *router, const char **msg, routerinfo_t *old_router; networkstatus_t *consensus = networkstatus_get_latest_consensus_by_flavor(FLAV_NS); - const smartlist_t *networkstatus_v2_list = networkstatus_get_v2_list(); int in_consensus = 0; tor_assert(msg); @@ -3483,15 +3393,6 @@ router_add_to_routerlist(routerinfo_t *router, const char **msg, } /* We no longer need a router with this descriptor digest. */ - SMARTLIST_FOREACH(networkstatus_v2_list, networkstatus_v2_t *, ns, - { - routerstatus_t *rs = - networkstatus_v2_find_mutable_entry(ns, id_digest); - if (rs && tor_memeq(rs->descriptor_digest, - router->cache_info.signed_descriptor_digest, - DIGEST_LEN)) - rs->need_to_mirror = 0; - }); if (consensus) { routerstatus_t *rs = networkstatus_vote_find_mutable_entry( consensus, id_digest); @@ -3499,7 +3400,6 @@ router_add_to_routerlist(routerinfo_t *router, const char **msg, router->cache_info.signed_descriptor_digest, DIGEST_LEN)) { in_consensus = 1; - rs->need_to_mirror = 0; } } @@ -3553,7 +3453,6 @@ router_add_to_routerlist(routerinfo_t *router, const char **msg, signed_desc_append_to_journal(&router->cache_info, &routerlist->desc_store); } - directory_set_dirty(); *msg = authdir_believes_valid ? "Valid server updated" : ("Invalid server updated. (This dirserver is marking your " "server as unapproved.)"); @@ -3575,7 +3474,6 @@ router_add_to_routerlist(routerinfo_t *router, const char **msg, signed_desc_append_to_journal(&router->cache_info, &routerlist->desc_store); } - directory_set_dirty(); return ROUTER_ADDED_SUCCESSFULLY; } @@ -3738,11 +3636,7 @@ routerlist_remove_old_routers(void) routerinfo_t *router; signed_descriptor_t *sd; digestset_t *retain; - int caches = directory_caches_dir_info(get_options()); const networkstatus_t *consensus = networkstatus_get_latest_consensus(); - const smartlist_t *networkstatus_v2_list = networkstatus_get_v2_list(); - int have_enough_v2; - const or_options_t *options = get_options(); trusted_dirs_remove_old_certs(); @@ -3758,38 +3652,10 @@ routerlist_remove_old_routers(void) { /* We'll probably retain everything in the consensus. */ int n_max_retain = smartlist_len(consensus->routerstatus_list); - if (caches && networkstatus_v2_list) { - /* If we care about v2 statuses, we'll retain at most as many as are - listed any of the v2 statues. This will be at least the length of - the largest v2 networkstatus, and in the worst case, this set will be - equal to the sum of the lengths of all v2 consensuses. Take the - worst case. - */ - SMARTLIST_FOREACH(networkstatus_v2_list, networkstatus_v2_t *, ns, - n_max_retain += smartlist_len(ns->entries)); - } retain = digestset_new(n_max_retain); } cutoff = now - OLD_ROUTER_DESC_MAX_AGE; - /* Build a list of all the descriptors that _anybody_ lists. */ - if (caches && networkstatus_v2_list) { - SMARTLIST_FOREACH_BEGIN(networkstatus_v2_list, networkstatus_v2_t *, ns) { - /* XXXX The inner loop here gets pretty expensive, and actually shows up - * on some profiles. It may be the reason digestmap_set shows up in - * profiles too. If instead we kept a per-descriptor digest count of - * how many networkstatuses recommended each descriptor, and changed - * that only when the networkstatuses changed, that would be a speed - * improvement, possibly 1-4% if it also removes digestmap_set from the - * profile. Not worth it for 0.1.2.x, though. The new directory - * system will obsolete this whole thing in 0.2.0.x. */ - SMARTLIST_FOREACH_BEGIN(ns->entries, routerstatus_t *, rs) { - if (rs->published_on >= cutoff) - digestset_add(retain, rs->descriptor_digest); - } SMARTLIST_FOREACH_END(rs); - } SMARTLIST_FOREACH_END(ns); - } - /* Retain anything listed in the consensus. */ if (consensus) { SMARTLIST_FOREACH(consensus->routerstatus_list, routerstatus_t *, rs, @@ -3797,18 +3663,11 @@ routerlist_remove_old_routers(void) digestset_add(retain, rs->descriptor_digest)); } - /* If we have a consensus, and nearly as many v2 networkstatuses as we want, - * we should consider pruning current routers that are too old and that - * nobody recommends. (If we don't have a consensus or enough v2 - * networkstatuses, then we should get more before we decide to kill - * routers.) */ - /* we set this to true iff we don't care about v2 info, or we have enough. */ - have_enough_v2 = !caches || - !(authdir_mode_any_main(options) || options->FetchV2Networkstatus) || - (networkstatus_v2_list && - smartlist_len(networkstatus_v2_list) > get_n_v2_authorities() / 2); - - if (have_enough_v2 && consensus) { + /* If we have a consensus, we should consider pruning current routers that + * are too old and that nobody recommends. (If we don't have a consensus, + * then we should get one before we decide to kill routers.) */ + + if (consensus) { cutoff = now - ROUTER_MAX_AGE; /* Remove too-old unrecommended members of routerlist->routers. */ for (i = 0; i < smartlist_len(routerlist->routers); ++i) { @@ -4107,8 +3966,6 @@ signed_desc_digest_is_recognized(signed_descriptor_t *desc) { const routerstatus_t *rs; networkstatus_t *consensus = networkstatus_get_latest_consensus(); - int caches = directory_caches_dir_info(get_options()); - const smartlist_t *networkstatus_v2_list = networkstatus_get_v2_list(); if (consensus) { rs = networkstatus_vote_find_entry(consensus, desc->identity_digest); @@ -4116,16 +3973,6 @@ signed_desc_digest_is_recognized(signed_descriptor_t *desc) desc->signed_descriptor_digest, DIGEST_LEN)) return 1; } - if (caches && networkstatus_v2_list) { - SMARTLIST_FOREACH(networkstatus_v2_list, networkstatus_v2_t *, ns, - { - if (!(rs = networkstatus_v2_find_entry(ns, desc->identity_digest))) - continue; - if (tor_memeq(rs->descriptor_digest, - desc->signed_descriptor_digest, DIGEST_LEN)) - return 1; - }); - } return 0; } @@ -4141,7 +3988,7 @@ update_all_descriptor_downloads(time_t now) launch_dummy_descriptor_download_as_needed(now, get_options()); } -/** Clear all our timeouts for fetching v2 and v3 directory stuff, and then +/** Clear all our timeouts for fetching v3 directory stuff, and then * give it all a try again. */ void routerlist_retry_directory_downloads(time_t now) @@ -4520,12 +4367,8 @@ initiate_descriptor_downloads(const routerstatus_t *source, * try to split our requests into at least this many requests. */ #define MIN_REQUESTS 3 /** If we want fewer than this many descriptors, wait until we - * want more, or until MAX_CLIENT_INTERVAL_WITHOUT_REQUEST has - * passed. */ + * want more, or until TestingClientMaxIntervalWithoutRequest has passed. */ #define MAX_DL_TO_DELAY 16 -/** When directory clients have only a few servers to request, they batch - * them until they have more, or until this amount of time has passed. */ -#define MAX_CLIENT_INTERVAL_WITHOUT_REQUEST (10*60) /** Given a <b>purpose</b> (FETCH_MICRODESC or FETCH_SERVERDESC) and a list of * router descriptor digests or microdescriptor digest256s in @@ -4557,7 +4400,7 @@ launch_descriptor_downloads(int purpose, should_delay = 0; } else { should_delay = (last_descriptor_download_attempted + - MAX_CLIENT_INTERVAL_WITHOUT_REQUEST) > now; + options->TestingClientMaxIntervalWithoutRequest) > now; if (!should_delay && n_downloadable) { if (last_descriptor_download_attempted) { log_info(LD_DIR, @@ -4626,152 +4469,6 @@ launch_descriptor_downloads(int purpose, } } -/** Launch downloads for router status as needed, using the strategy used by - * authorities and caches: based on the v2 networkstatuses we have, download - * every descriptor we don't have but would serve, from a random authority - * that lists it. */ -static void -update_router_descriptor_cache_downloads_v2(time_t now) -{ - smartlist_t **downloadable; /* For each authority, what can we dl from it? */ - smartlist_t **download_from; /* ... and, what will we dl from it? */ - digestmap_t *map; /* Which descs are in progress, or assigned? */ - int i, j, n; - int n_download; - const or_options_t *options = get_options(); - const smartlist_t *networkstatus_v2_list = networkstatus_get_v2_list(); - - if (! directory_fetches_dir_info_early(options)) { - log_warn(LD_BUG, "Called update_router_descriptor_cache_downloads_v2() " - "on a non-dir-mirror?"); - } - - if (!networkstatus_v2_list || !smartlist_len(networkstatus_v2_list)) - return; - - map = digestmap_new(); - n = smartlist_len(networkstatus_v2_list); - - downloadable = tor_malloc_zero(sizeof(smartlist_t*) * n); - download_from = tor_malloc_zero(sizeof(smartlist_t*) * n); - - /* Set map[d]=1 for the digest of every descriptor that we are currently - * downloading. */ - list_pending_descriptor_downloads(map, 0); - - /* For the digest of every descriptor that we don't have, and that we aren't - * downloading, add d to downloadable[i] if the i'th networkstatus knows - * about that descriptor, and we haven't already failed to get that - * descriptor from the corresponding authority. - */ - n_download = 0; - SMARTLIST_FOREACH_BEGIN(networkstatus_v2_list, networkstatus_v2_t *, ns) { - dir_server_t *ds; - smartlist_t *dl; - dl = downloadable[ns_sl_idx] = smartlist_new(); - download_from[ns_sl_idx] = smartlist_new(); - if (ns->published_on + MAX_NETWORKSTATUS_AGE+10*60 < now) { - /* Don't download if the networkstatus is almost ancient. */ - /* Actually, I suspect what's happening here is that we ask - * for the descriptor when we have a given networkstatus, - * and then we get a newer networkstatus, and then we receive - * the descriptor. Having a networkstatus actually expire is - * probably a rare event, and we'll probably be happiest if - * we take this clause out. -RD */ - continue; - } - - /* Don't try dirservers that we think are down -- we might have - * just tried them and just marked them as down. */ - ds = router_get_trusteddirserver_by_digest(ns->identity_digest); - if (ds && !ds->is_running) - continue; - - SMARTLIST_FOREACH_BEGIN(ns->entries, routerstatus_t * , rs) { - if (!rs->need_to_mirror) - continue; - if (router_get_by_descriptor_digest(rs->descriptor_digest)) { - log_warn(LD_BUG, - "We have a router descriptor, but need_to_mirror=1."); - rs->need_to_mirror = 0; - continue; - } - if (authdir_mode(options) && dirserv_would_reject_router(rs)) { - rs->need_to_mirror = 0; - continue; - } - if (digestmap_get(map, rs->descriptor_digest)) { - /* We're downloading it already. */ - continue; - } else { - /* We could download it from this guy. */ - smartlist_add(dl, rs->descriptor_digest); - ++n_download; - } - } SMARTLIST_FOREACH_END(rs); - } SMARTLIST_FOREACH_END(ns); - - /* At random, assign descriptors to authorities such that: - * - if d is a member of some downloadable[x], d is a member of some - * download_from[y]. (Everything we want to download, we try to download - * from somebody.) - * - If d is a member of download_from[y], d is a member of downloadable[y]. - * (We only try to download descriptors from authorities who claim to have - * them.) - * - No d is a member of download_from[x] and download_from[y] s.t. x != y. - * (We don't try to download anything from two authorities concurrently.) - */ - while (n_download) { - int which_ns = crypto_rand_int(n); - smartlist_t *dl = downloadable[which_ns]; - int idx; - char *d; - if (!smartlist_len(dl)) - continue; - idx = crypto_rand_int(smartlist_len(dl)); - d = smartlist_get(dl, idx); - if (! digestmap_get(map, d)) { - smartlist_add(download_from[which_ns], d); - digestmap_set(map, d, (void*) 1); - } - smartlist_del(dl, idx); - --n_download; - } - - /* Now, we can actually launch our requests. */ - for (i=0; i<n; ++i) { - networkstatus_v2_t *ns = smartlist_get(networkstatus_v2_list, i); - dir_server_t *ds = - router_get_trusteddirserver_by_digest(ns->identity_digest); - smartlist_t *dl = download_from[i]; - int pds_flags = PDS_RETRY_IF_NO_SERVERS; - if (! authdir_mode_any_nonhidserv(options)) - pds_flags |= PDS_NO_EXISTING_SERVERDESC_FETCH; /* XXXX ignored*/ - - if (!ds) { - log_info(LD_DIR, "Networkstatus with no corresponding authority!"); - continue; - } - if (! smartlist_len(dl)) - continue; - log_info(LD_DIR, "Requesting %d descriptors from authority \"%s\"", - smartlist_len(dl), ds->nickname); - for (j=0; j < smartlist_len(dl); j += MAX_DL_PER_REQUEST) { - initiate_descriptor_downloads(&(ds->fake_status), - DIR_PURPOSE_FETCH_SERVERDESC, dl, j, - j+MAX_DL_PER_REQUEST, pds_flags); - } - } - - for (i=0; i<n; ++i) { - smartlist_free(download_from[i]); - smartlist_free(downloadable[i]); - } - tor_free(download_from); - tor_free(downloadable); - digestmap_free(map,NULL); -} - /** For any descriptor that we want that's currently listed in * <b>consensus</b>, download it as appropriate. */ void @@ -4830,7 +4527,7 @@ update_consensus_router_descriptor_downloads(time_t now, int is_vote, continue; /* We have an in-progress download. */ } if (!download_status_is_ready(&rs->dl_status, now, - MAX_ROUTERDESC_DOWNLOAD_FAILURES)) { + options->TestingDescriptorMaxDownloadTries)) { ++n_delayed; /* Not ready for retry. */ continue; } @@ -4932,13 +4629,10 @@ void update_router_descriptor_downloads(time_t now) { const or_options_t *options = get_options(); - if (should_delay_dir_fetches(options)) + if (should_delay_dir_fetches(options, NULL)) return; if (!we_fetch_router_descriptors(options)) return; - if (directory_fetches_dir_info_early(options)) { - update_router_descriptor_cache_downloads_v2(now); - } update_consensus_router_descriptor_downloads(now, 0, networkstatus_get_reasonably_live_consensus(now, FLAV_NS)); @@ -4956,7 +4650,7 @@ update_extrainfo_downloads(time_t now) int n_no_ei = 0, n_pending = 0, n_have = 0, n_delay = 0; if (! options->DownloadExtraInfo) return; - if (should_delay_dir_fetches(options)) + if (should_delay_dir_fetches(options, NULL)) return; if (!router_have_minimum_dir_info()) return; @@ -4990,7 +4684,7 @@ update_extrainfo_downloads(time_t now) continue; } if (!download_status_is_ready(&sd->ei_dl_status, now, - MAX_ROUTERDESC_DOWNLOAD_FAILURES)) { + options->TestingDescriptorMaxDownloadTries)) { ++n_delay; continue; } @@ -5062,7 +4756,7 @@ router_differences_are_cosmetic(const routerinfo_t *r1, const routerinfo_t *r2) } /* If any key fields differ, they're different. */ - if (strcasecmp(r1->address, r2->address) || + if (r1->addr != r2->addr || strcasecmp(r1->nickname, r2->nickname) || r1->or_port != r2->or_port || !tor_addr_eq(&r1->ipv6_addr, &r2->ipv6_addr) || diff --git a/src/or/routerlist.h b/src/or/routerlist.h index 505685897f..6e2f2eaea0 100644 --- a/src/or/routerlist.h +++ b/src/or/routerlist.h @@ -11,6 +11,8 @@ #ifndef TOR_ROUTERLIST_H #define TOR_ROUTERLIST_H +#include "testsupport.h" + int get_n_authorities(dirinfo_type_t type); int trusted_dirs_reload_certs(void); @@ -53,8 +55,7 @@ const routerstatus_t *router_pick_trusteddirserver(dirinfo_type_t type, int flags); const routerstatus_t *router_pick_fallback_dirserver(dirinfo_type_t type, int flags); -int router_get_my_share_of_directory_requests(double *v2_share_out, - double *v3_share_out); +int router_get_my_share_of_directory_requests(double *v3_share_out); void router_reset_status_download_failures(void); int routers_have_same_or_addrs(const routerinfo_t *r1, const routerinfo_t *r2); const routerinfo_t *routerlist_find_my_routerinfo(void); @@ -207,9 +208,10 @@ typedef union u64_dbl_t { double dbl; } u64_dbl_t; -int choose_array_element_by_weight(const u64_dbl_t *entries, int n_entries); -void scale_array_elements_to_u64(u64_dbl_t *entries, int n_entries, - uint64_t *total_out); +STATIC int choose_array_element_by_weight(const u64_dbl_t *entries, + int n_entries); +STATIC void scale_array_elements_to_u64(u64_dbl_t *entries, int n_entries, + uint64_t *total_out); #endif #endif diff --git a/src/or/routerparse.c b/src/or/routerparse.c index 01f65f262b..5add728d6d 100644 --- a/src/or/routerparse.c +++ b/src/or/routerparse.c @@ -339,7 +339,7 @@ static token_rule_t extrainfo_token_table[] = { END_OF_TABLE }; -/** List of tokens recognized in the body part of v2 and v3 networkstatus +/** List of tokens recognized in the body part of v3 networkstatus * documents. */ static token_rule_t rtrstatus_token_table[] = { T01("p", K_P, CONCAT_ARGS, NO_OBJ ), @@ -353,31 +353,6 @@ static token_rule_t rtrstatus_token_table[] = { END_OF_TABLE }; -/** List of tokens recognized in the header part of v2 networkstatus documents. - */ -static token_rule_t netstatus_token_table[] = { - T1( "published", K_PUBLISHED, CONCAT_ARGS, NO_OBJ ), - T0N("opt", K_OPT, CONCAT_ARGS, OBJ_OK ), - T1( "contact", K_CONTACT, CONCAT_ARGS, NO_OBJ ), - T1( "dir-signing-key", K_DIR_SIGNING_KEY, NO_ARGS, NEED_KEY_1024 ), - T1( "fingerprint", K_FINGERPRINT, CONCAT_ARGS, NO_OBJ ), - T1_START("network-status-version", K_NETWORK_STATUS_VERSION, - GE(1), NO_OBJ ), - T1( "dir-source", K_DIR_SOURCE, GE(3), NO_OBJ ), - T01("dir-options", K_DIR_OPTIONS, ARGS, NO_OBJ ), - T01("client-versions", K_CLIENT_VERSIONS, CONCAT_ARGS, NO_OBJ ), - T01("server-versions", K_SERVER_VERSIONS, CONCAT_ARGS, NO_OBJ ), - - END_OF_TABLE -}; - -/** List of tokens recognized in the footer of v1/v2 directory/networkstatus - * footers. */ -static token_rule_t dir_footer_token_table[] = { - T1("directory-signature", K_DIRECTORY_SIGNATURE, EQ(1), NEED_OBJ ), - END_OF_TABLE -}; - /** List of tokens common to V3 authority certificates and V3 consensuses. */ #define CERTIFICATE_MEMBERS \ T1("dir-key-certificate-version", K_DIR_KEY_CERTIFICATE_VERSION, \ @@ -386,7 +361,7 @@ static token_rule_t dir_footer_token_table[] = { T1("dir-key-published",K_DIR_KEY_PUBLISHED, CONCAT_ARGS, NO_OBJ), \ T1("dir-key-expires", K_DIR_KEY_EXPIRES, CONCAT_ARGS, NO_OBJ), \ T1("dir-signing-key", K_DIR_SIGNING_KEY, NO_ARGS, NEED_KEY ),\ - T01("dir-key-crosscert", K_DIR_KEY_CROSSCERT, NO_ARGS, NEED_OBJ ),\ + T1("dir-key-crosscert", K_DIR_KEY_CROSSCERT, NO_ARGS, NEED_OBJ ),\ T1("dir-key-certification", K_DIR_KEY_CERTIFICATION, \ NO_ARGS, NEED_OBJ), \ T01("dir-address", K_DIR_ADDRESS, GE(1), NO_OBJ), @@ -486,8 +461,7 @@ static token_rule_t networkstatus_consensus_token_table[] = { END_OF_TABLE }; -/** List of tokens recognized in the footer of v1/v2 directory/networkstatus - * footers. */ +/** List of tokens recognized in the footer of v1 directory footers. */ static token_rule_t networkstatus_vote_footer_token_table[] = { T01("directory-footer", K_DIRECTORY_FOOTER, NO_ARGS, NO_OBJ ), T01("bandwidth-weights", K_BW_WEIGHTS, ARGS, NO_OBJ ), @@ -598,7 +572,7 @@ dump_desc(const char *desc, const char *type) char *content = tor_malloc_zero(filelen); tor_snprintf(content, filelen, "Unable to parse descriptor of type " "%s:\n%s", type, desc); - write_str_to_file(debugfile, content, 0); + write_str_to_file(debugfile, content, 1); log_info(LD_DIR, "Unable to parse descriptor of type %s. See file " "unparseable-desc in data directory for details.", type); tor_free(content); @@ -629,28 +603,6 @@ router_get_router_hash(const char *s, size_t s_len, char *digest) DIGEST_SHA1); } -/** Set <b>digest</b> to the SHA-1 digest of the hash of the running-routers - * string in <b>s</b>. Return 0 on success, -1 on failure. - */ -int -router_get_runningrouters_hash(const char *s, char *digest) -{ - return router_get_hash_impl(s, strlen(s), digest, - "network-status","\ndirectory-signature", '\n', - DIGEST_SHA1); -} - -/** Set <b>digest</b> to the SHA-1 digest of the hash of the network-status - * string in <b>s</b>. Return 0 on success, -1 on failure. */ -int -router_get_networkstatus_v2_hash(const char *s, char *digest) -{ - return router_get_hash_impl(s, strlen(s), digest, - "network-status-version","\ndirectory-signature", - '\n', - DIGEST_SHA1); -} - /** Set <b>digests</b> to all the digests of the consensus document in * <b>s</b> */ int @@ -728,7 +680,7 @@ router_get_dirobj_signature(const char *digest, /** Helper: used to generate signatures for routers, directories and * network-status objects. Given a digest in <b>digest</b> and a secret - * <b>private_key</b>, generate an PKCS1-padded signature, BASE64-encode it, + * <b>private_key</b>, generate a PKCS1-padded signature, BASE64-encode it, * surround it with -----BEGIN/END----- pairs, and write it to the * <b>buf_len</b>-byte buffer at <b>buf</b>. Return 0 on success, -1 on * failure. @@ -751,6 +703,7 @@ router_append_dirobj_signature(char *buf, size_t buf_len, const char *digest, return -1; } memcpy(buf+s_len, sig, sig_len+1); + tor_free(sig); return 0; } @@ -1232,8 +1185,7 @@ router_parse_entry_from_string(const char *s, const char *end, log_warn(LD_DIR,"Router nickname is invalid"); goto err; } - router->address = tor_strdup(tok->args[1]); - if (!tor_inet_aton(router->address, &in)) { + if (!tor_inet_aton(tok->args[1], &in)) { log_warn(LD_DIR,"Router address is not an IP address."); goto err; } @@ -1728,7 +1680,6 @@ authority_cert_parse_from_string(const char *s, const char **end_of_string) log_debug(LD_DIR, "We already checked the signature on this " "certificate; no need to do so again."); found = 1; - cert->is_cross_certified = old_cert->is_cross_certified; } } if (!found) { @@ -1737,18 +1688,14 @@ authority_cert_parse_from_string(const char *s, const char **end_of_string) goto err; } - if ((tok = find_opt_by_keyword(tokens, K_DIR_KEY_CROSSCERT))) { - /* XXXX Once all authorities generate cross-certified certificates, - * make this field mandatory. */ - if (check_signature_token(cert->cache_info.identity_digest, - DIGEST_LEN, - tok, - cert->signing_key, - CST_NO_CHECK_OBJTYPE, - "key cross-certification")) { - goto err; - } - cert->is_cross_certified = 1; + tok = find_by_keyword(tokens, K_DIR_KEY_CROSSCERT); + if (check_signature_token(cert->cache_info.identity_digest, + DIGEST_LEN, + tok, + cert->signing_key, + CST_NO_CHECK_OBJTYPE, + "key cross-certification")) { + goto err; } } @@ -1948,8 +1895,6 @@ routerstatus_parse_entry_from_string(memarea_t *area, rs->is_named = 1; else if (!strcmp(tok->args[i], "Valid")) rs->is_valid = 1; - else if (!strcmp(tok->args[i], "V2Dir")) - rs->is_v2_dir = 1; else if (!strcmp(tok->args[i], "Guard")) rs->is_possible_guard = 1; else if (!strcmp(tok->args[i], "BadExit")) @@ -2084,14 +2029,6 @@ routerstatus_parse_entry_from_string(memarea_t *area, return rs; } -/** Helper to sort a smartlist of pointers to routerstatus_t */ -int -compare_routerstatus_entries(const void **_a, const void **_b) -{ - const routerstatus_t *a = *_a, *b = *_b; - return fast_memcmp(a->identity_digest, b->identity_digest, DIGEST_LEN); -} - int compare_vote_routerstatus_entries(const void **_a, const void **_b) { @@ -2100,188 +2037,6 @@ compare_vote_routerstatus_entries(const void **_a, const void **_b) DIGEST_LEN); } -/** Helper: used in call to _smartlist_uniq to clear out duplicate entries. */ -static void -free_duplicate_routerstatus_entry_(void *e) -{ - log_warn(LD_DIR, - "Network-status has two entries for the same router. " - "Dropping one."); - routerstatus_free(e); -} - -/** Given a v2 network-status object in <b>s</b>, try to - * parse it and return the result. Return NULL on failure. Check the - * signature of the network status, but do not (yet) check the signing key for - * authority. - */ -networkstatus_v2_t * -networkstatus_v2_parse_from_string(const char *s) -{ - const char *eos, *s_dup = s; - smartlist_t *tokens = smartlist_new(); - smartlist_t *footer_tokens = smartlist_new(); - networkstatus_v2_t *ns = NULL; - char ns_digest[DIGEST_LEN]; - char tmp_digest[DIGEST_LEN]; - struct in_addr in; - directory_token_t *tok; - int i; - memarea_t *area = NULL; - - if (router_get_networkstatus_v2_hash(s, ns_digest)) { - log_warn(LD_DIR, "Unable to compute digest of network-status"); - goto err; - } - - area = memarea_new(); - eos = find_start_of_next_routerstatus(s); - if (tokenize_string(area, s, eos, tokens, netstatus_token_table,0)) { - log_warn(LD_DIR, "Error tokenizing network-status header."); - goto err; - } - ns = tor_malloc_zero(sizeof(networkstatus_v2_t)); - memcpy(ns->networkstatus_digest, ns_digest, DIGEST_LEN); - - tok = find_by_keyword(tokens, K_NETWORK_STATUS_VERSION); - tor_assert(tok->n_args >= 1); - if (strcmp(tok->args[0], "2")) { - log_warn(LD_BUG, "Got a non-v2 networkstatus. Version was " - "%s", escaped(tok->args[0])); - goto err; - } - - tok = find_by_keyword(tokens, K_DIR_SOURCE); - tor_assert(tok->n_args >= 3); - ns->source_address = tor_strdup(tok->args[0]); - if (tor_inet_aton(tok->args[1], &in) == 0) { - log_warn(LD_DIR, "Error parsing network-status source address %s", - escaped(tok->args[1])); - goto err; - } - ns->source_addr = ntohl(in.s_addr); - ns->source_dirport = - (uint16_t) tor_parse_long(tok->args[2],10,0,65535,NULL,NULL); - if (ns->source_dirport == 0) { - log_warn(LD_DIR, "Directory source without dirport; skipping."); - goto err; - } - - tok = find_by_keyword(tokens, K_FINGERPRINT); - tor_assert(tok->n_args); - if (base16_decode(ns->identity_digest, DIGEST_LEN, tok->args[0], - strlen(tok->args[0]))) { - log_warn(LD_DIR, "Couldn't decode networkstatus fingerprint %s", - escaped(tok->args[0])); - goto err; - } - - if ((tok = find_opt_by_keyword(tokens, K_CONTACT))) { - tor_assert(tok->n_args); - ns->contact = tor_strdup(tok->args[0]); - } - - tok = find_by_keyword(tokens, K_DIR_SIGNING_KEY); - tor_assert(tok->key); - ns->signing_key = tok->key; - tok->key = NULL; - - if (crypto_pk_get_digest(ns->signing_key, tmp_digest)<0) { - log_warn(LD_DIR, "Couldn't compute signing key digest"); - goto err; - } - if (tor_memneq(tmp_digest, ns->identity_digest, DIGEST_LEN)) { - log_warn(LD_DIR, - "network-status fingerprint did not match dir-signing-key"); - goto err; - } - - if ((tok = find_opt_by_keyword(tokens, K_DIR_OPTIONS))) { - for (i=0; i < tok->n_args; ++i) { - if (!strcmp(tok->args[i], "Names")) - ns->binds_names = 1; - if (!strcmp(tok->args[i], "Versions")) - ns->recommends_versions = 1; - if (!strcmp(tok->args[i], "BadExits")) - ns->lists_bad_exits = 1; - if (!strcmp(tok->args[i], "BadDirectories")) - ns->lists_bad_directories = 1; - } - } - - if (ns->recommends_versions) { - if (!(tok = find_opt_by_keyword(tokens, K_CLIENT_VERSIONS))) { - log_warn(LD_DIR, "Missing client-versions on versioning directory"); - goto err; - } - ns->client_versions = tor_strdup(tok->args[0]); - - if (!(tok = find_opt_by_keyword(tokens, K_SERVER_VERSIONS)) || - tok->n_args<1) { - log_warn(LD_DIR, "Missing server-versions on versioning directory"); - goto err; - } - ns->server_versions = tor_strdup(tok->args[0]); - } - - tok = find_by_keyword(tokens, K_PUBLISHED); - tor_assert(tok->n_args == 1); - if (parse_iso_time(tok->args[0], &ns->published_on) < 0) { - goto err; - } - - ns->entries = smartlist_new(); - s = eos; - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); - smartlist_clear(tokens); - memarea_clear(area); - while (!strcmpstart(s, "r ")) { - routerstatus_t *rs; - if ((rs = routerstatus_parse_entry_from_string(area, &s, tokens, - NULL, NULL, 0, 0))) - smartlist_add(ns->entries, rs); - } - smartlist_sort(ns->entries, compare_routerstatus_entries); - smartlist_uniq(ns->entries, compare_routerstatus_entries, - free_duplicate_routerstatus_entry_); - - if (tokenize_string(area,s, NULL, footer_tokens, dir_footer_token_table,0)) { - log_warn(LD_DIR, "Error tokenizing network-status footer."); - goto err; - } - if (smartlist_len(footer_tokens) < 1) { - log_warn(LD_DIR, "Too few items in network-status footer."); - goto err; - } - tok = smartlist_get(footer_tokens, smartlist_len(footer_tokens)-1); - if (tok->tp != K_DIRECTORY_SIGNATURE) { - log_warn(LD_DIR, - "Expected network-status footer to end with a signature."); - goto err; - } - - note_crypto_pk_op(VERIFY_DIR); - if (check_signature_token(ns_digest, DIGEST_LEN, tok, ns->signing_key, 0, - "network-status") < 0) - goto err; - - goto done; - err: - dump_desc(s_dup, "v2 networkstatus"); - networkstatus_v2_free(ns); - ns = NULL; - done: - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); - smartlist_free(tokens); - SMARTLIST_FOREACH(footer_tokens, directory_token_t *, t, token_clear(t)); - smartlist_free(footer_tokens); - if (area) { - DUMP_AREA(area, "v2 networkstatus"); - memarea_drop_all(area); - } - return ns; -} - /** Verify the bandwidth weights of a network status document */ int networkstatus_verify_bw_weights(networkstatus_t *ns, int consensus_method) diff --git a/src/or/routerparse.h b/src/or/routerparse.h index eb2e885cb1..5d5d9e59ef 100644 --- a/src/or/routerparse.h +++ b/src/or/routerparse.h @@ -14,8 +14,6 @@ int router_get_router_hash(const char *s, size_t s_len, char *digest); int router_get_dir_hash(const char *s, char *digest); -int router_get_runningrouters_hash(const char *s, char *digest); -int router_get_networkstatus_v2_hash(const char *s, char *digest); int router_get_networkstatus_v3_hashes(const char *s, digests_t *digests); int router_get_extrainfo_hash(const char *s, size_t s_len, char *digest); #define DIROBJ_MAX_SIG_LEN 256 @@ -52,9 +50,7 @@ void sort_version_list(smartlist_t *lst, int remove_duplicates); void assert_addr_policy_ok(smartlist_t *t); void dump_distinct_digest_count(int severity); -int compare_routerstatus_entries(const void **_a, const void **_b); int compare_vote_routerstatus_entries(const void **_a, const void **_b); -networkstatus_v2_t *networkstatus_v2_parse_from_string(const char *s); int networkstatus_verify_bw_weights(networkstatus_t *ns, int); networkstatus_t *networkstatus_parse_vote_from_string(const char *s, const char **eos_out, diff --git a/src/or/routerset.c b/src/or/routerset.c index 2e41f7f6c4..7aee90d6db 100644 --- a/src/or/routerset.c +++ b/src/or/routerset.c @@ -358,39 +358,6 @@ routerset_get_all_nodes(smartlist_t *out, const routerset_t *routerset, } } -#if 0 -/** Add to <b>target</b> every node_t from <b>source</b> except: - * - * 1) Don't add it if <b>include</b> is non-empty and the relay isn't in - * <b>include</b>; and - * 2) Don't add it if <b>exclude</b> is non-empty and the relay is - * excluded in a more specific fashion by <b>exclude</b>. - * 3) If <b>running_only</b>, don't add non-running routers. - */ -void -routersets_get_node_disjunction(smartlist_t *target, - const smartlist_t *source, - const routerset_t *include, - const routerset_t *exclude, int running_only) -{ - SMARTLIST_FOREACH(source, const node_t *, node, { - int include_result; - if (running_only && !node->is_running) - continue; - if (!routerset_is_empty(include)) - include_result = routerset_contains_node(include, node); - else - include_result = 1; - - if (include_result) { - int exclude_result = routerset_contains_node(exclude, node); - if (include_result >= exclude_result) - smartlist_add(target, (void*)node); - } - }); -} -#endif - /** Remove every node_t from <b>lst</b> that is in <b>routerset</b>. */ void routerset_subtract_nodes(smartlist_t *lst, const routerset_t *routerset) diff --git a/src/or/routerset.h b/src/or/routerset.h index bfa0c59ac1..8261c7fb09 100644 --- a/src/or/routerset.h +++ b/src/or/routerset.h @@ -32,12 +32,6 @@ void routerset_get_all_nodes(smartlist_t *out, const routerset_t *routerset, const routerset_t *excludeset, int running_only); int routerset_add_unknown_ccs(routerset_t **setp, int only_if_some_cc_set); -#if 0 -void routersets_get_node_disjunction(smartlist_t *target, - const smartlist_t *source, - const routerset_t *include, - const routerset_t *exclude, int running_only); -#endif void routerset_subtract_nodes(smartlist_t *out, const routerset_t *routerset); diff --git a/src/or/statefile.c b/src/or/statefile.c index bcb7b07417..7b9998fc1a 100644 --- a/src/or/statefile.c +++ b/src/or/statefile.c @@ -4,6 +4,7 @@ * Copyright (c) 2007-2013, The Tor Project, Inc. */ /* See LICENSE for licensing information */ +#define STATEFILE_PRIVATE #include "or.h" #include "circuitstats.h" #include "config.h" @@ -12,6 +13,7 @@ #include "hibernate.h" #include "rephist.h" #include "router.h" +#include "sandbox.h" #include "statefile.h" /** A list of state-file "abbreviations," for compatibility. */ @@ -90,8 +92,11 @@ static config_var_t state_vars_[] = { #undef VAR #undef V -static int or_state_validate(or_state_t *old_options, or_state_t *options, - int from_setconf, char **msg); +static int or_state_validate(or_state_t *state, char **msg); + +static int or_state_validate_cb(void *old_options, void *options, + void *default_options, + int from_setconf, char **msg); /** Magic value for or_state_t. */ #define OR_STATE_MAGIC 0x57A73f57 @@ -109,7 +114,7 @@ static const config_format_t state_format = { STRUCT_OFFSET(or_state_t, magic_), state_abbrevs_, state_vars_, - (validate_fn_t)or_state_validate, + or_state_validate_cb, &state_extra_var, }; @@ -117,8 +122,8 @@ static const config_format_t state_format = { static or_state_t *global_state = NULL; /** Return the persistent state struct for this Tor. */ -or_state_t * -get_or_state(void) +MOCK_IMPL(or_state_t *, +get_or_state, (void)) { tor_assert(global_state); return global_state; @@ -194,21 +199,27 @@ validate_transports_in_state(or_state_t *state) return 0; } -/** Return 0 if every setting in <b>state</b> is reasonable, and a - * permissible transition from <b>old_state</b>. Else warn and return -1. - * Should have no side effects, except for normalizing the contents of - * <b>state</b>. - */ -/* XXX from_setconf is here because of bug 238 */ static int -or_state_validate(or_state_t *old_state, or_state_t *state, - int from_setconf, char **msg) +or_state_validate_cb(void *old_state, void *state, void *default_state, + int from_setconf, char **msg) { /* We don't use these; only options do. Still, we need to match that * signature. */ (void) from_setconf; + (void) default_state; (void) old_state; + return or_state_validate(state, msg); +} + +/** Return 0 if every setting in <b>state</b> is reasonable, and a + * permissible transition from <b>old_state</b>. Else warn and return -1. + * Should have no side effects, except for normalizing the contents of + * <b>state</b>. + */ +static int +or_state_validate(or_state_t *state, char **msg) +{ if (entry_guards_parse_state(state, 0, msg)<0) return -1; @@ -237,7 +248,8 @@ or_state_set(or_state_t *new_state) tor_free(err); ret = -1; } - if (circuit_build_times_parse_state(&circ_times, global_state) < 0) { + if (circuit_build_times_parse_state( + get_circuit_build_times_mutable(),global_state) < 0) { ret = -1; } return ret; @@ -249,7 +261,7 @@ or_state_set(or_state_t *new_state) static void or_state_save_broken(char *fname) { - int i; + int i, res; file_status_t status; char *fname2 = NULL; for (i = 0; i < 100; ++i) { @@ -263,12 +275,18 @@ or_state_save_broken(char *fname) log_warn(LD_BUG, "Unable to parse state in \"%s\"; too many saved bad " "state files to move aside. Discarding the old state file.", fname); - unlink(fname); + res = unlink(fname); + if (res != 0) { + log_warn(LD_FS, + "Also couldn't discard old state file \"%s\" because " + "unlink() failed: %s", + fname, strerror(errno)); + } } else { log_warn(LD_BUG, "Unable to parse state in \"%s\". Moving it aside " "to \"%s\". This could be a bug in Tor; please tell " "the developers.", fname, fname2); - if (rename(fname, fname2) < 0) { + if (tor_rename(fname, fname2) < 0) {//XXXX sandbox prohibits log_warn(LD_BUG, "Weirdly, I couldn't even move the state aside. The " "OS gave an error of %s", strerror(errno)); } @@ -276,6 +294,16 @@ or_state_save_broken(char *fname) tor_free(fname2); } +STATIC or_state_t * +or_state_new(void) +{ + or_state_t *new_state = tor_malloc_zero(sizeof(or_state_t)); + new_state->magic_ = OR_STATE_MAGIC; + config_init(&state_format, new_state); + + return new_state; +} + /** Reload the persistent state from disk, generating a new state as needed. * Return 0 on success, less than 0 on failure. */ @@ -303,9 +331,7 @@ or_state_load(void) log_warn(LD_GENERAL,"State file \"%s\" is not a file? Failing.", fname); goto done; } - new_state = tor_malloc_zero(sizeof(or_state_t)); - new_state->magic_ = OR_STATE_MAGIC; - config_init(&state_format, new_state); + new_state = or_state_new(); if (contents) { config_line_t *lines=NULL; int assign_retval; @@ -322,7 +348,7 @@ or_state_load(void) } } - if (!badstate && or_state_validate(NULL, new_state, 1, &errmsg) < 0) + if (!badstate && or_state_validate(new_state, &errmsg) < 0) badstate = 1; if (errmsg) { @@ -340,9 +366,7 @@ or_state_load(void) tor_free(contents); config_free(&state_format, new_state); - new_state = tor_malloc_zero(sizeof(or_state_t)); - new_state->magic_ = OR_STATE_MAGIC; - config_init(&state_format, new_state); + new_state = or_state_new(); } else if (contents) { log_info(LD_GENERAL, "Loaded state from \"%s\"", fname); } else { @@ -404,7 +428,7 @@ or_state_save(time_t now) * to avoid redundant writes. */ entry_guards_update_state(global_state); rep_hist_update_state(global_state); - circuit_build_times_update_state(&circ_times, global_state); + circuit_build_times_update_state(get_circuit_build_times(), global_state); if (accounting_is_enabled(get_options())) accounting_run_housekeeping(now); @@ -449,7 +473,7 @@ or_state_save(time_t now) /** Return the config line for transport <b>transport</b> in the current state. * Return NULL if there is no config line for <b>transport</b>. */ -static config_line_t * +STATIC config_line_t * get_transport_in_state_by_name(const char *transport) { or_state_t *or_state = get_or_state(); @@ -607,10 +631,19 @@ save_transport_to_state(const char *transport, tor_free(transport_addrport); } +STATIC void +or_state_free(or_state_t *state) +{ + if (!state) + return; + + config_free(&state_format, state); +} + void or_state_free_all(void) { - config_free(&state_format, global_state); + or_state_free(global_state); global_state = NULL; } diff --git a/src/or/statefile.h b/src/or/statefile.h index dcdee6c604..15bb0b4aae 100644 --- a/src/or/statefile.h +++ b/src/or/statefile.h @@ -7,7 +7,7 @@ #ifndef TOR_STATEFILE_H #define TOR_STATEFILE_H -or_state_t *get_or_state(void); +MOCK_DECL(or_state_t *,get_or_state,(void)); int did_last_state_file_write_fail(void); int or_state_save(time_t now); @@ -18,5 +18,11 @@ int or_state_load(void); int or_state_loaded(void); void or_state_free_all(void); +#ifdef STATEFILE_PRIVATE +STATIC config_line_t *get_transport_in_state_by_name(const char *transport); +STATIC void or_state_free(or_state_t *state); +STATIC or_state_t *or_state_new(void); +#endif + #endif diff --git a/src/or/status.c b/src/or/status.c index 69f92ed097..afaa9de840 100644 --- a/src/or/status.c +++ b/src/or/status.c @@ -6,7 +6,10 @@ * \brief Keep status information and log the heartbeat messages. **/ +#define STATUS_PRIVATE + #include "or.h" +#include "circuituse.h" #include "config.h" #include "status.h" #include "nodelist.h" @@ -14,17 +17,21 @@ #include "router.h" #include "circuitlist.h" #include "main.h" +#include "rephist.h" #include "hibernate.h" #include "rephist.h" +#include "statefile.h" + +static void log_accounting(const time_t now, const or_options_t *options); /** Return the total number of circuits. */ -static int +STATIC int count_circuits(void) { circuit_t *circ; int nr=0; - for (circ = circuit_get_global_list_(); circ; circ = circ->next) + TOR_LIST_FOREACH(circ, circuit_get_global_list(), head) nr++; return nr; @@ -32,7 +39,7 @@ count_circuits(void) /** Take seconds <b>secs</b> and return a newly allocated human-readable * uptime string */ -static char * +STATIC char * secs_to_uptime(long secs) { long int days = secs / 86400; @@ -59,7 +66,7 @@ secs_to_uptime(long secs) /** Take <b>bytes</b> and returns a newly allocated human-readable usage * string. */ -static char * +STATIC char * bytes_to_usage(uint64_t bytes) { char *bw_string = NULL; @@ -112,6 +119,10 @@ log_heartbeat(time_t now) uptime, count_circuits(),bw_sent,bw_rcvd, hibernating?" We are currently hibernating.":""); + if (server_mode(options) && accounting_is_enabled(options) && !hibernating) { + log_accounting(now, options); + } + if (stats_n_data_cells_packaged && !hibernating) log_notice(LD_HEARTBEAT, "Average packaged cell fullness: %2.3f%%", 100*(U64_TO_DBL(stats_n_data_bytes_packaged) / @@ -125,6 +136,8 @@ log_heartbeat(time_t now) if (public_server_mode(options)) rep_hist_log_circuit_handshake_stats(now); + circuit_log_ancient_one_hop_circuits(1800); + tor_free(uptime); tor_free(bw_sent); tor_free(bw_rcvd); @@ -132,3 +145,27 @@ log_heartbeat(time_t now) return 0; } +static void +log_accounting(const time_t now, const or_options_t *options) +{ + or_state_t *state = get_or_state(); + char *acc_rcvd = bytes_to_usage(state->AccountingBytesReadInInterval); + char *acc_sent = bytes_to_usage(state->AccountingBytesWrittenInInterval); + char *acc_max = bytes_to_usage(options->AccountingMax); + time_t interval_end = accounting_get_end_time(); + char end_buf[ISO_TIME_LEN + 1]; + char *remaining = NULL; + format_local_iso_time(end_buf, interval_end); + remaining = secs_to_uptime(interval_end - now); + + log_notice(LD_HEARTBEAT, "Heartbeat: Accounting enabled. " + "Sent: %s / %s, Received: %s / %s. The " + "current accounting interval ends on %s, in %s.", + acc_sent, acc_max, acc_rcvd, acc_max, end_buf, remaining); + + tor_free(acc_rcvd); + tor_free(acc_sent); + tor_free(acc_max); + tor_free(remaining); +} + diff --git a/src/or/status.h b/src/or/status.h index 7c3b969c86..13458ea476 100644 --- a/src/or/status.h +++ b/src/or/status.h @@ -4,7 +4,15 @@ #ifndef TOR_STATUS_H #define TOR_STATUS_H +#include "testsupport.h" + int log_heartbeat(time_t now); +#ifdef STATUS_PRIVATE +STATIC int count_circuits(void); +STATIC char *secs_to_uptime(long secs); +STATIC char *bytes_to_usage(uint64_t bytes); +#endif + #endif diff --git a/src/or/transports.c b/src/or/transports.c index 3749d6bb21..dc30754162 100644 --- a/src/or/transports.c +++ b/src/or/transports.c @@ -51,35 +51,37 @@ * logic, because of race conditions that can cause dangling * pointers. ] * - * <b>In even more detail, this is what happens when a SIGHUP - * occurs:</b> + * <b>In even more detail, this is what happens when a config read + * (like a SIGHUP or a SETCONF) occurs:</b> * * We immediately destroy all unconfigured proxies (We shouldn't have - * unconfigured proxies in the first place, except when SIGHUP rings - * immediately after tor is launched.). + * unconfigured proxies in the first place, except when the config + * read happens immediately after tor is launched.). * * We mark all managed proxies and transports to signify that they * must be removed if they don't contribute by the new torrc * (we mark using the <b>marked_for_removal</b> element). * We also mark all managed proxies to signify that they might need to * be restarted so that they end up supporting all the transports the - * new torrc wants them to support (using the <b>got_hup</b> element). + * new torrc wants them to support + * (we mark using the <b>was_around_before_config_read</b> element). * We also clear their <b>transports_to_launch</b> list so that we can * put there the transports we need to launch according to the new * torrc. * * We then start parsing torrc again. * - * Everytime we encounter a transport line using a known pre-SIGHUP - * managed proxy, we cleanse that proxy from the removal mark. - * We also mark it as unconfigured so that on the next scheduled - * events tick, we investigate whether we need to restart the proxy - * so that it also spawns the new transports. - * If the post-SIGHUP <b>transports_to_launch</b> list is identical to - * the pre-SIGHUP one, it means that no changes were introduced to - * this proxy during the SIGHUP and no restart has to take place. + * Everytime we encounter a transport line using a managed proxy that + * was around before the config read, we cleanse that proxy from the + * removal mark. We also toggle the <b>check_if_restarts_needed</b> + * flag, so that on the next <b>pt_configure_remaining_proxies</b> + * tick, we investigate whether we need to restart the proxy so that + * it also spawns the new transports. If the post-config-read + * <b>transports_to_launch</b> list is identical to the pre-config-read + * one, it means that no changes were introduced to this proxy during + * the config read and no restart has to take place. * - * During the post-SIGHUP torrc parsing, we unmark all transports + * During the post-config-read torrc parsing, we unmark all transports * spawned by managed proxies that we find in our torrc. * We do that so that if we don't need to restart a managed proxy, we * can continue using its old transports normally. @@ -95,18 +97,17 @@ #include "util.h" #include "router.h" #include "statefile.h" +#include "entrynodes.h" +#include "connection_or.h" +#include "ext_orport.h" +#include "control.h" static process_environment_t * create_managed_proxy_environment(const managed_proxy_t *mp); static INLINE int proxy_configuration_finished(const managed_proxy_t *mp); -static void managed_proxy_destroy(managed_proxy_t *mp, - int also_terminate_process); - static void handle_finished_proxy(managed_proxy_t *mp); -static int configure_proxy(managed_proxy_t *mp); - static void parse_method_error(const char *line, int is_server_method); #define parse_server_method_error(l) parse_method_error(l, 1) #define parse_client_method_error(l) parse_method_error(l, 0) @@ -136,7 +137,8 @@ static smartlist_t *transport_list = NULL; SOCKS version <b>socks_ver</b>. */ static transport_t * transport_new(const tor_addr_t *addr, uint16_t port, - const char *name, int socks_ver) + const char *name, int socks_ver, + const char *extra_info_args) { transport_t *t = tor_malloc_zero(sizeof(transport_t)); @@ -144,6 +146,8 @@ transport_new(const tor_addr_t *addr, uint16_t port, t->port = port; t->name = tor_strdup(name); t->socks_version = socks_ver; + if (extra_info_args) + t->extra_info_args = tor_strdup(extra_info_args); return t; } @@ -156,6 +160,7 @@ transport_free(transport_t *transport) return; tor_free(transport->name); + tor_free(transport->extra_info_args); tor_free(transport); } @@ -323,7 +328,7 @@ int transport_add_from_config(const tor_addr_t *addr, uint16_t port, const char *name, int socks_ver) { - transport_t *t = transport_new(addr, port, name, socks_ver); + transport_t *t = transport_new(addr, port, name, socks_ver, NULL); int r = transport_add(t); @@ -531,8 +536,7 @@ launch_managed_proxy(managed_proxy_t *mp) } /** Check if any of the managed proxies we are currently trying to - * configure have anything new to say. This is called from - * run_scheduled_events(). */ + * configure has anything new to say. */ void pt_configure_remaining_proxies(void) { @@ -549,14 +553,15 @@ pt_configure_remaining_proxies(void) assert_unconfigured_count_ok(); SMARTLIST_FOREACH_BEGIN(tmp, managed_proxy_t *, mp) { - tor_assert(mp->conf_state != PT_PROTO_BROKEN || + tor_assert(mp->conf_state != PT_PROTO_BROKEN && mp->conf_state != PT_PROTO_FAILED_LAUNCH); - if (mp->got_hup) { - mp->got_hup = 0; + if (mp->was_around_before_config_read) { + /* This proxy is marked by a config read. Check whether we need + to restart it. */ + + mp->was_around_before_config_read = 0; - /* This proxy is marked by a SIGHUP. Check whether we need to - restart it. */ if (proxy_needs_restart(mp)) { log_info(LD_GENERAL, "Preparing managed proxy '%s' for restart.", mp->argv[0]); @@ -589,7 +594,7 @@ pt_configure_remaining_proxies(void) * Return 1 if the transport configuration finished, and return 0 * otherwise (if we still have more configuring to do for this * proxy). */ -static int +STATIC int configure_proxy(managed_proxy_t *mp) { int configuration_finished = 0; @@ -657,6 +662,7 @@ register_server_proxy(const managed_proxy_t *mp) save_transport_to_state(t->name, &t->addr, t->port); log_notice(LD_GENERAL, "Registered server transport '%s' at '%s'", t->name, fmt_addrport(&t->addr, t->port)); + control_event_transport_launched("server", t->name, &t->addr, t->port); } SMARTLIST_FOREACH_END(t); } @@ -679,9 +685,11 @@ register_client_proxy(const managed_proxy_t *mp) break; case 0: log_info(LD_GENERAL, "Successfully registered transport %s", t->name); + control_event_transport_launched("client", t->name, &t->addr, t->port); break; case 1: log_info(LD_GENERAL, "Successfully registered transport %s", t->name); + control_event_transport_launched("client", t->name, &t->addr, t->port); transport_free(transport_tmp); break; } @@ -699,7 +707,7 @@ register_proxy(const managed_proxy_t *mp) } /** Free memory allocated by managed proxy <b>mp</b>. */ -static void +STATIC void managed_proxy_destroy(managed_proxy_t *mp, int also_terminate_process) { @@ -713,7 +721,8 @@ managed_proxy_destroy(managed_proxy_t *mp, smartlist_free(mp->transports_to_launch); /* remove it from the list of managed proxies */ - smartlist_remove(managed_proxy_list, mp); + if (managed_proxy_list) + smartlist_remove(managed_proxy_list, mp); /* free the argv */ free_execve_args(mp->argv); @@ -750,7 +759,6 @@ handle_finished_proxy(managed_proxy_t *mp) } unconfigured_proxies_n--; - tor_assert(unconfigured_proxies_n >= 0); } /** Return true if the configuration of the managed proxy <b>mp</b> is @@ -781,7 +789,7 @@ handle_methods_done(const managed_proxy_t *mp) /** Handle a configuration protocol <b>line</b> received from a * managed proxy <b>mp</b>. */ -void +STATIC void handle_proxy_line(const char *line, managed_proxy_t *mp) { log_info(LD_GENERAL, "Got a line from managed proxy '%s': (%s)", @@ -882,7 +890,7 @@ handle_proxy_line(const char *line, managed_proxy_t *mp) } /** Parses an ENV-ERROR <b>line</b> and warns the user accordingly. */ -void +STATIC void parse_env_error(const char *line) { /* (Length of the protocol string) plus (a space) and (the first char of @@ -898,7 +906,7 @@ parse_env_error(const char *line) /** Handles a VERSION <b>line</b>. Updates the configuration protocol * version in <b>mp</b>. */ -int +STATIC int parse_version(const char *line, managed_proxy_t *mp) { if (strlen(line) < (strlen(PROTO_NEG_SUCCESS) + 2)) { @@ -939,14 +947,14 @@ parse_method_error(const char *line, int is_server) /** Parses an SMETHOD <b>line</b> and if well-formed it registers the * new transport in <b>mp</b>. */ -int +STATIC int parse_smethod_line(const char *line, managed_proxy_t *mp) { int r; smartlist_t *items = NULL; char *method_name=NULL; - + char *args_string=NULL; char *addrport=NULL; tor_addr_t tor_addr; char *address=NULL; @@ -963,6 +971,9 @@ parse_smethod_line(const char *line, managed_proxy_t *mp) goto err; } + /* Example of legit SMETHOD line: + SMETHOD obfs2 0.0.0.0:25612 ARGS:secret=supersekrit,key=superkey */ + tor_assert(!strcmp(smartlist_get(items,0),PROTO_SMETHOD)); method_name = smartlist_get(items,1); @@ -990,7 +1001,19 @@ parse_smethod_line(const char *line, managed_proxy_t *mp) goto err; } - transport = transport_new(&tor_addr, port, method_name, PROXY_NONE); + if (smartlist_len(items) > 3) { + /* Seems like there are also some [options] in the SMETHOD line. + Let's see if we can parse them. */ + char *options_string = smartlist_get(items, 3); + log_debug(LD_CONFIG, "Got options_string: %s", options_string); + if (!strcmpstart(options_string, "ARGS:")) { + args_string = options_string+strlen("ARGS:"); + log_debug(LD_CONFIG, "Got ARGS: %s", args_string); + } + } + + transport = transport_new(&tor_addr, port, method_name, + PROXY_NONE, args_string); if (!transport) goto err; @@ -1016,7 +1039,7 @@ parse_smethod_line(const char *line, managed_proxy_t *mp) /** Parses a CMETHOD <b>line</b>, and if well-formed it registers * the new transport in <b>mp</b>. */ -int +STATIC int parse_cmethod_line(const char *line, managed_proxy_t *mp) { int r; @@ -1082,7 +1105,7 @@ parse_cmethod_line(const char *line, managed_proxy_t *mp) goto err; } - transport = transport_new(&tor_addr, port, method_name, socks_ver); + transport = transport_new(&tor_addr, port, method_name, socks_ver, NULL); if (!transport) goto err; @@ -1105,6 +1128,50 @@ parse_cmethod_line(const char *line, managed_proxy_t *mp) return r; } +/** Return a newly allocated string that tor should place in + * TOR_PT_SERVER_TRANSPORT_OPTIONS while configuring the server + * manged proxy in <b>mp</b>. Return NULL if no such options are found. */ +STATIC char * +get_transport_options_for_server_proxy(const managed_proxy_t *mp) +{ + char *options_string = NULL; + smartlist_t *string_sl = smartlist_new(); + + tor_assert(mp->is_server); + + /** Loop over the transports of the proxy. If we have options for + any of them, format them appropriately and place them in our + smartlist. Finally, join our smartlist to get the final + string. */ + SMARTLIST_FOREACH_BEGIN(mp->transports_to_launch, const char *, transport) { + smartlist_t *options_tmp_sl = NULL; + options_tmp_sl = get_options_for_server_transport(transport); + if (!options_tmp_sl) + continue; + + /** Loop over the options of this transport, escape them, and + place them in the smartlist. */ + SMARTLIST_FOREACH_BEGIN(options_tmp_sl, const char *, options) { + char *escaped_opts = tor_escape_str_for_pt_args(options, ":;\\"); + smartlist_add_asprintf(string_sl, "%s:%s", + transport, escaped_opts); + tor_free(escaped_opts); + } SMARTLIST_FOREACH_END(options); + + SMARTLIST_FOREACH(options_tmp_sl, char *, c, tor_free(c)); + smartlist_free(options_tmp_sl); + } SMARTLIST_FOREACH_END(transport); + + if (smartlist_len(string_sl)) { + options_string = smartlist_join_strings(string_sl, ";", 0, NULL); + } + + SMARTLIST_FOREACH(string_sl, char *, t, tor_free(t)); + smartlist_free(string_sl); + + return options_string; +} + /** Return the string that tor should place in TOR_PT_SERVER_BINDADDR * while configuring the server managed proxy in <b>mp</b>. The * string is stored in the heap, and it's the the responsibility of @@ -1139,6 +1206,8 @@ get_bindaddr_for_server_proxy(const managed_proxy_t *mp) static process_environment_t * create_managed_proxy_environment(const managed_proxy_t *mp) { + const or_options_t *options = get_options(); + /* Environment variables to be added to or set in mp's environment. */ smartlist_t *envs = smartlist_new(); /* XXXX The next time someone touches this code, shorten the name of @@ -1176,8 +1245,10 @@ create_managed_proxy_environment(const managed_proxy_t *mp) { char *orport_tmp = get_first_listener_addrport_string(CONN_TYPE_OR_LISTENER); - smartlist_add_asprintf(envs, "TOR_PT_ORPORT=%s", orport_tmp); - tor_free(orport_tmp); + if (orport_tmp) { + smartlist_add_asprintf(envs, "TOR_PT_ORPORT=%s", orport_tmp); + tor_free(orport_tmp); + } } { @@ -1186,13 +1257,41 @@ create_managed_proxy_environment(const managed_proxy_t *mp) tor_free(bindaddr_tmp); } + { + char *server_transport_options = + get_transport_options_for_server_proxy(mp); + if (server_transport_options) { + smartlist_add_asprintf(envs, "TOR_PT_SERVER_TRANSPORT_OPTIONS=%s", + server_transport_options); + tor_free(server_transport_options); + } + } + /* XXX024 Remove the '=' here once versions of obfsproxy which * assert that this env var exists are sufficiently dead. * * (If we remove this line entirely, some joker will stick this * variable in Tor's environment and crash PTs that try to parse * it even when not run in server mode.) */ - smartlist_add(envs, tor_strdup("TOR_PT_EXTENDED_SERVER_PORT=")); + + if (options->ExtORPort_lines) { + char *ext_or_addrport_tmp = + get_first_listener_addrport_string(CONN_TYPE_EXT_OR_LISTENER); + char *cookie_file_loc = get_ext_or_auth_cookie_file_name(); + + if (ext_or_addrport_tmp) { + smartlist_add_asprintf(envs, "TOR_PT_EXTENDED_SERVER_PORT=%s", + ext_or_addrport_tmp); + } + smartlist_add_asprintf(envs, "TOR_PT_AUTH_COOKIE_FILE=%s", + cookie_file_loc); + + tor_free(ext_or_addrport_tmp); + tor_free(cookie_file_loc); + + } else { + smartlist_add_asprintf(envs, "TOR_PT_EXTENDED_SERVER_PORT="); + } } SMARTLIST_FOREACH_BEGIN(envs, const char *, env_var) { @@ -1216,7 +1315,7 @@ create_managed_proxy_environment(const managed_proxy_t *mp) * <b>proxy_argv</b>. * * Requires that proxy_argv have at least one element. */ -static managed_proxy_t * +STATIC managed_proxy_t * managed_proxy_create(const smartlist_t *transport_list, char **proxy_argv, int is_server) { @@ -1267,19 +1366,20 @@ pt_kickstart_proxy(const smartlist_t *transport_list, managed_proxy_create(transport_list, proxy_argv, is_server); } else { /* known proxy. add its transport to its transport list */ - if (mp->got_hup) { - /* If the managed proxy we found is marked by a SIGHUP, it means - that it's not useless and should be kept. If it's marked for - removal, unmark it and increase the unconfigured proxies so - that we try to restart it if we need to. Afterwards, check if - a transport_t for 'transport' used to exist before the SIGHUP - and make sure it doesn't get deleted because we might reuse - it. */ + if (mp->was_around_before_config_read) { + /* If this managed proxy was around even before we read the + config this time, it means that it was already enabled before + and is not useless and should be kept. If it's marked for + removal, unmark it and make sure that we check whether it + needs to be restarted. */ if (mp->marked_for_removal) { mp->marked_for_removal = 0; check_if_restarts_needed = 1; } + /* For each new transport, check if the managed proxy used to + support it before the SIGHUP. If that was the case, make sure + it doesn't get removed because we might reuse it. */ SMARTLIST_FOREACH_BEGIN(transport_list, const char *, transport) { old_transport = transport_get_by_name(transport); if (old_transport) @@ -1328,8 +1428,10 @@ pt_prepare_proxy_list_for_config_read(void) tor_assert(mp->conf_state == PT_PROTO_COMPLETED); + /* Mark all proxies for removal, and also note that they have been + here before the config read. */ mp->marked_for_removal = 1; - mp->got_hup = 1; + mp->was_around_before_config_read = 1; SMARTLIST_FOREACH(mp->transports_to_launch, char *, t, tor_free(t)); smartlist_clear(mp->transports_to_launch); } SMARTLIST_FOREACH_END(mp); @@ -1390,6 +1492,8 @@ pt_get_extra_info_descriptor_string(void) tor_assert(mp->transports); SMARTLIST_FOREACH_BEGIN(mp->transports, const transport_t *, t) { + char *transport_args = NULL; + /* If the transport proxy returned "0.0.0.0" as its address, and * we know our external IP address, use it. Otherwise, use the * returned address. */ @@ -1405,9 +1509,16 @@ pt_get_extra_info_descriptor_string(void) addrport = fmt_addrport(&t->addr, t->port); } + /* If this transport has any arguments with it, prepend a space + to them so that we can add them to the transport line. */ + if (t->extra_info_args) + tor_asprintf(&transport_args, " %s", t->extra_info_args); + smartlist_add_asprintf(string_chunks, - "transport %s %s", - t->name, addrport); + "transport %s %s%s", + t->name, addrport, + transport_args ? transport_args : ""); + tor_free(transport_args); } SMARTLIST_FOREACH_END(t); } SMARTLIST_FOREACH_END(mp); @@ -1426,6 +1537,57 @@ pt_get_extra_info_descriptor_string(void) return the_string; } +/** Stringify the SOCKS arguments in <b>socks_args</b> according to + * 180_pluggable_transport.txt. The string is allocated on the heap + * and it's the responsibility of the caller to free it after use. */ +char * +pt_stringify_socks_args(const smartlist_t *socks_args) +{ + /* tmp place to store escaped socks arguments, so that we can + concatenate them up afterwards */ + smartlist_t *sl_tmp = NULL; + char *escaped_string = NULL; + char *new_string = NULL; + + tor_assert(socks_args); + tor_assert(smartlist_len(socks_args) > 0); + + sl_tmp = smartlist_new(); + + SMARTLIST_FOREACH_BEGIN(socks_args, const char *, s) { + /* Escape ';' and '\'. */ + escaped_string = tor_escape_str_for_pt_args(s, ";\\"); + if (!escaped_string) + goto done; + + smartlist_add(sl_tmp, escaped_string); + } SMARTLIST_FOREACH_END(s); + + new_string = smartlist_join_strings(sl_tmp, ";", 0, NULL); + + done: + SMARTLIST_FOREACH(sl_tmp, char *, s, tor_free(s)); + smartlist_free(sl_tmp); + + return new_string; +} + +/** Return a string of the SOCKS arguments that we should pass to the + * pluggable transports proxy in <b>addr</b>:<b>port</b> according to + * 180_pluggable_transport.txt. The string is allocated on the heap + * and it's the responsibility of the caller to free it after use. */ +char * +pt_get_socks_args_for_proxy_addrport(const tor_addr_t *addr, uint16_t port) +{ + const smartlist_t *socks_args = NULL; + + socks_args = get_socks_args_by_bridge_addrport(addr, port); + if (!socks_args) + return NULL; + + return pt_stringify_socks_args(socks_args); +} + /** The tor config was read. * Destroy all managed proxies that were marked by a previous call to * prepare_proxy_list_for_config_read() and are not used by the new diff --git a/src/or/transports.h b/src/or/transports.h index 6ee82f4556..1365ead006 100644 --- a/src/or/transports.h +++ b/src/or/transports.h @@ -25,6 +25,9 @@ typedef struct transport_t { /** Boolean: We are re-parsing our transport list, and we are going to remove * this one if we don't find it in the list of configured transports. */ unsigned marked_for_removal : 1; + /** Arguments for this transport that must be written to the + extra-info descriptor. */ + char *extra_info_args; } transport_t; void mark_transport_list(void); @@ -55,6 +58,10 @@ void pt_prepare_proxy_list_for_config_read(void); void sweep_proxy_list(void); smartlist_t *get_transport_proxy_ports(void); +char *pt_stringify_socks_args(const smartlist_t *socks_args); + +char *pt_get_socks_args_for_proxy_addrport(const tor_addr_t *addr, + uint16_t port); #ifdef PT_PRIVATE /** State of the managed proxy configuration protocol. */ @@ -90,7 +97,7 @@ typedef struct { * this flag to signify that this proxy might need to be restarted * so that it can listen for other transports according to the new * torrc. */ - unsigned int got_hup : 1; + unsigned int was_around_before_config_read : 1; /* transports to-be-launched by this proxy */ smartlist_t *transports_to_launch; @@ -100,12 +107,21 @@ typedef struct { smartlist_t *transports; } managed_proxy_t; -int parse_cmethod_line(const char *line, managed_proxy_t *mp); -int parse_smethod_line(const char *line, managed_proxy_t *mp); +STATIC int parse_cmethod_line(const char *line, managed_proxy_t *mp); +STATIC int parse_smethod_line(const char *line, managed_proxy_t *mp); + +STATIC int parse_version(const char *line, managed_proxy_t *mp); +STATIC void parse_env_error(const char *line); +STATIC void handle_proxy_line(const char *line, managed_proxy_t *mp); +STATIC char *get_transport_options_for_server_proxy(const managed_proxy_t *mp); + +STATIC void managed_proxy_destroy(managed_proxy_t *mp, + int also_terminate_process); + +STATIC managed_proxy_t *managed_proxy_create(const smartlist_t *transport_list, + char **proxy_argv, int is_server); -int parse_version(const char *line, managed_proxy_t *mp); -void parse_env_error(const char *line); -void handle_proxy_line(const char *line, managed_proxy_t *mp); +STATIC int configure_proxy(managed_proxy_t *mp); #endif diff --git a/src/test/Makefile.nmake b/src/test/Makefile.nmake index 562c8df8b5..822431f3b8 100644 --- a/src/test/Makefile.nmake +++ b/src/test/Makefile.nmake @@ -12,9 +12,10 @@ LIBS = ..\..\..\build-alpha\lib\libevent.lib \ crypt32.lib gdi32.lib user32.lib TEST_OBJECTS = test.obj test_addr.obj test_containers.obj \ - test_crypto.obj test_data.obj test_dir.obj test_microdesc.obj \ - test_pt.obj test_util.obj test_config.obj test_cell_formats.obj \ - test_replay.obj test_introduce.obj tinytest.obj + test_controller_events.ogj test_crypto.obj test_data.obj test_dir.obj \ + test_microdesc.obj test_pt.obj test_util.obj test_config.obj \ + test_cell_formats.obj test_replay.obj test_introduce.obj tinytest.obj \ + test_hs.obj tinytest.obj: ..\ext\tinytest.c $(CC) $(CFLAGS) /D snprintf=_snprintf /c ..\ext\tinytest.c diff --git a/src/test/bench.c b/src/test/bench.c index 706b8bc7fb..f6c33626f2 100644 --- a/src/test/bench.c +++ b/src/test/bench.c @@ -14,9 +14,6 @@ const char tor_git_revision[] = ""; #include "orconfig.h" -#define RELAY_PRIVATE -#define CONFIG_PRIVATE - #include "or.h" #include "onion_tap.h" #include "relay.h" @@ -204,6 +201,7 @@ bench_onion_ntor(void) for (i = 0; i < iters; ++i) { onion_skin_ntor_create(nodeid, &keypair1.pubkey, &state, os); ntor_handshake_state_free(state); + state = NULL; } end = perftime(); printf("Client-side, part 1: %f usec.\n", NANOCOUNT(start, end, iters)/1e3); @@ -340,6 +338,28 @@ bench_dmap(void) } static void +bench_siphash(void) +{ + char buf[128]; + int lens[] = { 7, 8, 15, 16, 20, 32, 111, 128, -1 }; + int i, j; + uint64_t start, end; + const int N = 300000; + crypto_rand(buf, sizeof(buf)); + + for (i = 0; lens[i] > 0; ++i) { + reset_perftime(); + start = perftime(); + for (j = 0; j < N; ++j) { + siphash24g(buf, lens[i]); + } + end = perftime(); + printf("siphash24g(%d): %.2f ns per call\n", + lens[i], NANOCOUNT(start,end,N)); + } +} + +static void bench_cell_ops(void) { const int iters = 1<<16; @@ -489,6 +509,7 @@ typedef struct benchmark_t { static struct benchmark_t benchmarks[] = { ENT(dmap), + ENT(siphash), ENT(aes), ENT(onion_TAP), #ifdef CURVE25519_ENABLED @@ -546,6 +567,7 @@ main(int argc, const char **argv) reset_perftime(); crypto_seed_rng(1); + crypto_init_siphash_key(); options = options_new(); init_logging(); options->command = CMD_RUN_UNITTESTS; diff --git a/src/test/bt_test.py b/src/test/bt_test.py new file mode 100755 index 0000000000..8290509fa7 --- /dev/null +++ b/src/test/bt_test.py @@ -0,0 +1,42 @@ +# Copyright 2013, The Tor Project, Inc +# See LICENSE for licensing information + +""" +bt_test.py + +This file tests the output from test-bt-cl to make sure it's as expected. + +Example usage: + +$ ./src/test/test-bt-cl crash | ./src/test/bt_test.py +OK +$ ./src/test/test-bt-cl assert | ./src/test/bt_test.py +OK + +""" + +import sys + + +def matches(lines, funcs): + if len(lines) < len(funcs): + return False + try: + for l, f in zip(lines, funcs): + l.index(f) + except ValueError: + return False + else: + return True + +FUNCNAMES = "crash oh_what a_tangled_web we_weave main".split() + +LINES = sys.stdin.readlines() + +for I in range(len(LINES)): + if matches(LINES[I:], FUNCNAMES): + print("OK") + break +else: + print("BAD") + diff --git a/src/test/include.am b/src/test/include.am index 112d1a79d8..fba439a616 100644 --- a/src/test/include.am +++ b/src/test/include.am @@ -1,11 +1,15 @@ -TESTS+= src/test/test +TESTS += src/test/test -noinst_PROGRAMS+= src/test/test src/test/test-child src/test/bench +noinst_PROGRAMS+= src/test/bench +if UNITTESTS_ENABLED +noinst_PROGRAMS+= src/test/test src/test/test-child +endif src_test_AM_CPPFLAGS = -DSHARE_DATADIR="\"$(datadir)\"" \ -DLOCALSTATEDIR="\"$(localstatedir)\"" \ -DBINDIR="\"$(bindir)\"" \ - -I"$(top_srcdir)/src/or" -I"$(top_srcdir)/src/ext" + -I"$(top_srcdir)/src/or" -I"$(top_srcdir)/src/ext" \ + -DTOR_UNIT_TESTS # -L flags need to go in LDFLAGS. -l flags need to go in LDADD. # This seems to matter nowhere but on Windows, but I assure you that it @@ -14,31 +18,47 @@ src_test_AM_CPPFLAGS = -DSHARE_DATADIR="\"$(datadir)\"" \ src_test_test_SOURCES = \ src/test/test.c \ src/test/test_addr.c \ + src/test/test_buffers.c \ src/test/test_cell_formats.c \ + src/test/test_circuitlist.c \ + src/test/test_circuitmux.c \ src/test/test_containers.c \ + src/test/test_controller_events.c \ src/test/test_crypto.c \ + src/test/test_cell_queue.c \ src/test/test_data.c \ src/test/test_dir.c \ + src/test/test_extorport.c \ src/test/test_introduce.c \ + src/test/test_logging.c \ src/test/test_microdesc.c \ + src/test/test_oom.c \ + src/test/test_options.c \ src/test/test_pt.c \ + src/test/test_relaycell.c \ src/test/test_replay.c \ + src/test/test_routerkeys.c \ + src/test/test_socks.c \ src/test/test_util.c \ src/test/test_config.c \ + src/test/test_hs.c \ + src/test/test_nodelist.c \ + src/test/test_policy.c \ + src/test/test_status.c \ src/ext/tinytest.c +src_test_test_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) + src_test_test_CPPFLAGS= $(src_test_AM_CPPFLAGS) src_test_bench_SOURCES = \ src/test/bench.c -src_test_bench_CPPFLAGS= $(src_test_AM_CPPFLAGS) - src_test_test_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ \ @TOR_LDFLAGS_libevent@ -src_test_test_LDADD = src/or/libtor.a src/common/libor.a \ - src/common/libor-crypto.a $(LIBDONNA) \ - src/common/libor-event.a \ +src_test_test_LDADD = src/or/libtor-testing.a src/common/libor-testing.a \ + src/common/libor-crypto-testing.a $(LIBDONNA) \ + src/common/libor-event-testing.a \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ \ @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ @@ -63,6 +83,39 @@ src_test_test_ntor_cl_LDADD = src/or/libtor.a src/common/libor.a \ @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ src_test_test_ntor_cl_AM_CPPFLAGS = \ -I"$(top_srcdir)/src/or" +NTOR_TEST_DEPS=src/test/test-ntor-cl +else +NTOR_TEST_DEPS= +endif +if COVERAGE_ENABLED +CMDLINE_TEST_TOR = ./src/or/tor-cov +else +CMDLINE_TEST_TOR = ./src/or/tor +endif + +noinst_PROGRAMS += src/test/test-bt-cl +src_test_test_bt_cl_SOURCES = src/test/test_bt_cl.c +src_test_test_bt_cl_LDADD = src/common/libor-testing.a \ + @TOR_LIB_MATH@ \ + @TOR_LIB_WS32@ @TOR_LIB_GDI@ +src_test_test_bt_cl_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +src_test_test_bt_cl_CPPFLAGS= $(src_test_AM_CPPFLAGS) + + +check-local: $(NTOR_TEST_DEPS) $(CMDLINE_TEST_TOR) +if USEPYTHON + $(PYTHON) $(top_srcdir)/src/test/test_cmdline_args.py $(CMDLINE_TEST_TOR) "${top_srcdir}" +if CURVE25519_ENABLED + $(PYTHON) $(top_srcdir)/src/test/ntor_ref.py test-tor + $(PYTHON) $(top_srcdir)/src/test/ntor_ref.py self-test +endif + ./src/test/test-bt-cl assert | $(PYTHON) $(top_srcdir)/src/test/bt_test.py + ./src/test/test-bt-cl crash | $(PYTHON) $(top_srcdir)/src/test/bt_test.py endif +EXTRA_DIST += \ + src/test/bt_test.py \ + src/test/ntor_ref.py \ + src/test/slownacl_curve25519.py \ + src/test/test_cmdline_args.py diff --git a/src/test/ntor_ref.py b/src/test/ntor_ref.py index ade468da7d..7d6e43e716 100644..100755 --- a/src/test/ntor_ref.py +++ b/src/test/ntor_ref.py @@ -1,3 +1,4 @@ +#!/usr/bin/python # Copyright 2012-2013, The Tor Project, Inc # See LICENSE for licensing information @@ -27,17 +28,25 @@ commands: """ import binascii -import curve25519 +try: + import curve25519 + curve25519mod = curve25519.keys +except ImportError: + curve25519 = None + import slownacl_curve25519 + curve25519mod = slownacl_curve25519 + import hashlib import hmac import subprocess +import sys # ********************************************************************** # Helpers and constants def HMAC(key,msg): "Return the HMAC-SHA256 of 'msg' using the key 'key'." - H = hmac.new(key, "", hashlib.sha256) + H = hmac.new(key, b"", hashlib.sha256) H.update(msg) return H.digest() @@ -59,31 +68,38 @@ G_LENGTH = 32 H_LENGTH = 32 PROTOID = b"ntor-curve25519-sha256-1" -M_EXPAND = PROTOID + ":key_expand" -T_MAC = PROTOID + ":mac" -T_KEY = PROTOID + ":key_extract" -T_VERIFY = PROTOID + ":verify" +M_EXPAND = PROTOID + b":key_expand" +T_MAC = PROTOID + b":mac" +T_KEY = PROTOID + b":key_extract" +T_VERIFY = PROTOID + b":verify" def H_mac(msg): return H(msg, tweak=T_MAC) def H_verify(msg): return H(msg, tweak=T_VERIFY) -class PrivateKey(curve25519.keys.Private): - """As curve25519.keys.Private, but doesn't regenerate its public key +class PrivateKey(curve25519mod.Private): + """As curve25519mod.Private, but doesn't regenerate its public key every time you ask for it. """ def __init__(self): - curve25519.keys.Private.__init__(self) + curve25519mod.Private.__init__(self) self._memo_public = None def get_public(self): if self._memo_public is None: - self._memo_public = curve25519.keys.Private.get_public(self) + self._memo_public = curve25519mod.Private.get_public(self) return self._memo_public # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -def kdf_rfc5869(key, salt, info, n): +if sys.version < '3': + def int2byte(i): + return chr(i) +else: + def int2byte(i): + return bytes([i]) + +def kdf_rfc5869(key, salt, info, n): prk = HMAC(key=salt, msg=key) @@ -91,7 +107,7 @@ def kdf_rfc5869(key, salt, info, n): last = b"" i = 1 while len(out) < n: - m = last + info + chr(i) + m = last + info + int2byte(i) last = h = HMAC(key=prk, msg=m) out += h i = i + 1 @@ -177,7 +193,7 @@ def server(seckey_b, my_node_id, message, keyBytes=72): badness = (keyid(seckey_b.get_public()) != message[NODE_ID_LENGTH:NODE_ID_LENGTH+H_LENGTH]) - pubkey_X = curve25519.keys.Public(message[NODE_ID_LENGTH+H_LENGTH:]) + pubkey_X = curve25519mod.Public(message[NODE_ID_LENGTH+H_LENGTH:]) seckey_y = PrivateKey() pubkey_Y = seckey_y.get_public() pubkey_B = seckey_b.get_public() @@ -200,7 +216,7 @@ def server(seckey_b, my_node_id, message, keyBytes=72): pubkey_Y.serialize() + pubkey_X.serialize() + PROTOID + - "Server") + b"Server") msg = pubkey_Y.serialize() + H_mac(auth_input) @@ -240,7 +256,7 @@ def client_part2(seckey_x, msg, node_id, pubkey_B, keyBytes=72): """ assert len(msg) == G_LENGTH + H_LENGTH - pubkey_Y = curve25519.keys.Public(msg[:G_LENGTH]) + pubkey_Y = curve25519mod.Public(msg[:G_LENGTH]) their_auth = msg[G_LENGTH:] pubkey_X = seckey_x.get_public() @@ -262,7 +278,7 @@ def client_part2(seckey_x, msg, node_id, pubkey_B, keyBytes=72): pubkey_B.serialize() + pubkey_Y.serialize() + pubkey_X.serialize() + PROTOID + - "Server") + b"Server") my_auth = H_mac(auth_input) @@ -276,7 +292,7 @@ def client_part2(seckey_x, msg, node_id, pubkey_B, keyBytes=72): # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -def demo(node_id="iToldYouAboutStairs.", server_key=PrivateKey()): +def demo(node_id=b"iToldYouAboutStairs.", server_key=PrivateKey()): """ Try to handshake with ourself. """ @@ -286,6 +302,7 @@ def demo(node_id="iToldYouAboutStairs.", server_key=PrivateKey()): assert len(skeys) == 72 assert len(ckeys) == 72 assert skeys == ckeys + print("OK") # ====================================================================== def timing(): @@ -295,7 +312,7 @@ def timing(): import timeit t = timeit.Timer(stmt="ntor_ref.demo(N,SK)", setup="import ntor_ref,curve25519;N='ABCD'*5;SK=ntor_ref.PrivateKey()") - print t.timeit(number=1000) + print(t.timeit(number=1000)) # ====================================================================== @@ -306,7 +323,7 @@ def kdf_vectors(): import binascii def kdf_vec(inp): k = kdf(inp, T_KEY, M_EXPAND, 100) - print repr(inp), "\n\""+ binascii.b2a_hex(k)+ "\"" + print(repr(inp), "\n\""+ binascii.b2a_hex(k)+ "\"") kdf_vec("") kdf_vec("Tor") kdf_vec("AN ALARMING ITEM TO FIND ON YOUR CREDIT-RATING STATEMENT") @@ -319,13 +336,13 @@ def test_tor(): Call the test-ntor-cl command-line program to make sure we can interoperate with Tor's ntor program """ - enhex=binascii.b2a_hex + enhex=lambda s: binascii.b2a_hex(s) dehex=lambda s: binascii.a2b_hex(s.strip()) - PROG = "./src/test/test-ntor-cl" + PROG = b"./src/test/test-ntor-cl" def tor_client1(node_id, pubkey_B): " returns (msg, state) " - p = subprocess.Popen([PROG, "client1", enhex(node_id), + p = subprocess.Popen([PROG, b"client1", enhex(node_id), enhex(pubkey_B.serialize())], stdout=subprocess.PIPE) return map(dehex, p.stdout.readlines()) @@ -343,7 +360,7 @@ def test_tor(): return map(dehex, p.stdout.readlines()) - node_id = "thisisatornodeid$#%^" + node_id = b"thisisatornodeid$#%^" seckey_b = PrivateKey() pubkey_B = seckey_b.get_public() @@ -368,13 +385,14 @@ def test_tor(): assert c_keys == s_keys assert len(c_keys) == 90 - print "We just interoperated." + print("OK") # ====================================================================== if __name__ == '__main__': - import sys - if sys.argv[1] == 'gen_kdf_vectors': + if len(sys.argv) < 2: + print(__doc__) + elif sys.argv[1] == 'gen_kdf_vectors': kdf_vectors() elif sys.argv[1] == 'timing': timing() @@ -384,4 +402,4 @@ if __name__ == '__main__': test_tor() else: - print __doc__ + print(__doc__) diff --git a/src/test/slownacl_curve25519.py b/src/test/slownacl_curve25519.py new file mode 100644 index 0000000000..4dabab61b6 --- /dev/null +++ b/src/test/slownacl_curve25519.py @@ -0,0 +1,117 @@ +# This is the curve25519 implementation from Matthew Dempsky's "Slownacl" +# library. It is in the public domain. +# +# It isn't constant-time. Don't use it except for testing. +# +# Nick got the slownacl source from: +# https://github.com/mdempsky/dnscurve/tree/master/slownacl + +__all__ = ['smult_curve25519_base', 'smult_curve25519'] + +import sys + +P = 2 ** 255 - 19 +A = 486662 + +def expmod(b, e, m): + if e == 0: return 1 + t = expmod(b, e // 2, m) ** 2 % m + if e & 1: t = (t * b) % m + return t + +def inv(x): + return expmod(x, P - 2, P) + +# Addition and doubling formulas taken from Appendix D of "Curve25519: +# new Diffie-Hellman speed records". + +def add(n,m,d): + (xn,zn), (xm,zm), (xd,zd) = n, m, d + x = 4 * (xm * xn - zm * zn) ** 2 * zd + z = 4 * (xm * zn - zm * xn) ** 2 * xd + return (x % P, z % P) + +def double(n): + (xn,zn) = n + x = (xn ** 2 - zn ** 2) ** 2 + z = 4 * xn * zn * (xn ** 2 + A * xn * zn + zn ** 2) + return (x % P, z % P) + +def curve25519(n, base): + one = (base,1) + two = double(one) + # f(m) evaluates to a tuple containing the mth multiple and the + # (m+1)th multiple of base. + def f(m): + if m == 1: return (one, two) + (pm, pm1) = f(m // 2) + if (m & 1): + return (add(pm, pm1, one), double(pm1)) + return (double(pm), add(pm, pm1, one)) + ((x,z), _) = f(n) + return (x * inv(z)) % P + +if sys.version < '3': + def b2i(c): + return ord(c) + def i2b(i): + return chr(i) + def ba2bs(ba): + return "".join(ba) +else: + def b2i(c): + return c + def i2b(i): + return i + def ba2bs(ba): + return bytes(ba) + +def unpack(s): + if len(s) != 32: raise ValueError('Invalid Curve25519 argument') + return sum(b2i(s[i]) << (8 * i) for i in range(32)) + +def pack(n): + return ba2bs([i2b((n >> (8 * i)) & 255) for i in range(32)]) + +def clamp(n): + n &= ~7 + n &= ~(128 << 8 * 31) + n |= 64 << 8 * 31 + return n + +def smult_curve25519(n, p): + n = clamp(unpack(n)) + p = unpack(p) + return pack(curve25519(n, p)) + +def smult_curve25519_base(n): + n = clamp(unpack(n)) + return pack(curve25519(n, 9)) + + +# +# This part I'm adding in for compatibility with the curve25519 python +# module. -Nick +# +import os + +class Private: + def __init__(self, secret=None, seed=None): + self.private = pack(clamp(unpack(os.urandom(32)))) + + def get_public(self): + return Public(smult_curve25519_base(self.private)) + + def get_shared_key(self, public, hashfn): + return hashfn(smult_curve25519(self.private, public.public)) + + def serialize(self): + return self.private + +class Public: + def __init__(self, public): + self.public = public + + def serialize(self): + return self.public + diff --git a/src/test/test-child.c b/src/test/test-child.c index ef10fbb922..756782e70b 100644 --- a/src/test/test-child.c +++ b/src/test/test-child.c @@ -9,6 +9,13 @@ #else #include <unistd.h> #endif +#include <string.h> + +#ifdef _WIN32 +#define SLEEP(sec) Sleep((sec)*1000) +#else +#define SLEEP(sec) sleep(sec) +#endif /** Trivial test program which prints out its command line arguments so we can * check if tor_spawn_background() works */ @@ -16,27 +23,38 @@ int main(int argc, char **argv) { int i; + int delay = 1; + int fast = 0; + + if (argc > 1) { + if (!strcmp(argv[1], "--hang")) { + delay = 60; + } else if (!strcmp(argv[1], "--fast")) { + fast = 1; + delay = 0; + } + } fprintf(stdout, "OUT\n"); fprintf(stderr, "ERR\n"); for (i = 1; i < argc; i++) fprintf(stdout, "%s\n", argv[i]); - fprintf(stdout, "SLEEPING\n"); + if (!fast) + fprintf(stdout, "SLEEPING\n"); /* We need to flush stdout so that test_util_spawn_background_partial_read() succeed. Otherwise ReadFile() will get the entire output in one */ // XXX: Can we make stdio flush on newline? fflush(stdout); -#ifdef _WIN32 - Sleep(1000); -#else - sleep(1); -#endif + if (!fast) + SLEEP(1); fprintf(stdout, "DONE\n"); -#ifdef _WIN32 - Sleep(1000); -#else - sleep(1); -#endif + fflush(stdout); + if (fast) + return 0; + + while (--delay) { + SLEEP(1); + } return 0; } diff --git a/src/test/test-network.sh b/src/test/test-network.sh new file mode 100755 index 0000000000..7b59864166 --- /dev/null +++ b/src/test/test-network.sh @@ -0,0 +1,47 @@ +#! /bin/sh + +until [ -z $1 ] +do + case $1 in + --chutney-path) + export CHUTNEY_PATH="$2" + shift + ;; + --tor-path) + export TOR_DIR="$2" + shift + ;; + --flavo?r|--network-flavo?r) + export NETWORK_FLAVOUR="$2" + shift + ;; + *) + echo "Sorry, I don't know what to do with '$1'." + exit 2 + ;; + esac + shift +done + +TOR_DIR="${TOR_DIR:-$PWD}" +NETWORK_FLAVOUR=${NETWORK_FLAVOUR:-basic} +CHUTNEY_NETWORK=networks/$NETWORK_FLAVOUR +myname=$(basename $0) + +[ -d "$CHUTNEY_PATH" ] && [ -x "$CHUTNEY_PATH/chutney" ] || { + echo "$myname: missing 'chutney' in CHUTNEY_PATH ($CHUTNEY_PATH)" + exit 1 +} +cd "$CHUTNEY_PATH" +# For picking up the right tor binaries. +PATH="$TOR_DIR/src/or:$TOR_DIR/src/tools:$PATH" +./tools/bootstrap-network.sh $NETWORK_FLAVOUR || exit 2 + +# Sleep some, waiting for the network to bootstrap. +# TODO: Add chutney command 'bootstrap-status' and use that instead. +BOOTSTRAP_TIME=18 +echo -n "$myname: sleeping for $BOOTSTRAP_TIME seconds" +n=$BOOTSTRAP_TIME; while [ $n -gt 0 ]; do + sleep 1; n=$(expr $n - 1); echo -n . +done; echo "" +./chutney verify $CHUTNEY_NETWORK diff --git a/src/test/test.c b/src/test/test.c index c2911d842c..8bce9c91f4 100644 --- a/src/test/test.c +++ b/src/test/test.c @@ -28,11 +28,11 @@ const char tor_git_revision[] = ""; /* These macros pull in declarations for some functions and structures that * are typically file-private. */ -#define BUFFERS_PRIVATE -#define CONFIG_PRIVATE #define GEOIP_PRIVATE #define ROUTER_PRIVATE #define CIRCUITSTATS_PRIVATE +#define CIRCUITLIST_PRIVATE +#define STATEFILE_PRIVATE /* * Linux doesn't provide lround in math.h by default, but mac os does... @@ -52,16 +52,20 @@ double fabs(double x); #include "rendcommon.h" #include "test.h" #include "torgzip.h" +#ifdef ENABLE_MEMPOOLS #include "mempool.h" +#endif #include "memarea.h" #include "onion.h" -#include "onion_tap.h" #include "onion_ntor.h" +#include "onion_tap.h" #include "policies.h" #include "rephist.h" #include "routerparse.h" +#include "statefile.h" #ifdef CURVE25519_ENABLED #include "crypto_curve25519.h" +#include "onion_ntor.h" #endif #ifdef USE_DMALLOC @@ -218,667 +222,138 @@ free_pregenerated_keys(void) } } -typedef struct socks_test_data_t { - socks_request_t *req; - buf_t *buf; -} socks_test_data_t; - -static void * -socks_test_setup(const struct testcase_t *testcase) -{ - socks_test_data_t *data = tor_malloc(sizeof(socks_test_data_t)); - (void)testcase; - data->buf = buf_new_with_capacity(256); - data->req = socks_request_new(); - config_register_addressmaps(get_options()); - return data; -} -static int -socks_test_cleanup(const struct testcase_t *testcase, void *ptr) -{ - socks_test_data_t *data = ptr; - (void)testcase; - buf_free(data->buf); - socks_request_free(data->req); - tor_free(data); - return 1; -} - -const struct testcase_setup_t socks_setup = { - socks_test_setup, socks_test_cleanup -}; - -#define SOCKS_TEST_INIT() \ - socks_test_data_t *testdata = ptr; \ - buf_t *buf = testdata->buf; \ - socks_request_t *socks = testdata->req; -#define ADD_DATA(buf, s) \ - write_to_buf(s, sizeof(s)-1, buf) - -static void -socks_request_clear(socks_request_t *socks) -{ - tor_free(socks->username); - tor_free(socks->password); - memset(socks, 0, sizeof(socks_request_t)); -} - -/** Perform unsupported SOCKS 4 commands */ -static void -test_socks_4_unsupported_commands(void *ptr) -{ - SOCKS_TEST_INIT(); - - /* SOCKS 4 Send BIND [02] to IP address 2.2.2.2:4369 */ - ADD_DATA(buf, "\x04\x02\x11\x11\x02\x02\x02\x02\x00"); - test_assert(fetch_from_buf_socks(buf, socks, get_options()->TestSocks, - get_options()->SafeSocks) == -1); - test_eq(4, socks->socks_version); - test_eq(0, socks->replylen); /* XXX: shouldn't tor reply? */ - - done: - ; -} - -/** Perform supported SOCKS 4 commands */ -static void -test_socks_4_supported_commands(void *ptr) -{ - SOCKS_TEST_INIT(); - - test_eq(0, buf_datalen(buf)); - - /* SOCKS 4 Send CONNECT [01] to IP address 2.2.2.2:4370 */ - ADD_DATA(buf, "\x04\x01\x11\x12\x02\x02\x02\x03\x00"); - test_assert(fetch_from_buf_socks(buf, socks, get_options()->TestSocks, - get_options()->SafeSocks) == 1); - test_eq(4, socks->socks_version); - test_eq(0, socks->replylen); /* XXX: shouldn't tor reply? */ - test_eq(SOCKS_COMMAND_CONNECT, socks->command); - test_streq("2.2.2.3", socks->address); - test_eq(4370, socks->port); - test_assert(socks->got_auth == 0); - test_assert(! socks->username); - - test_eq(0, buf_datalen(buf)); - socks_request_clear(socks); - - /* SOCKS 4 Send CONNECT [01] to IP address 2.2.2.2:4369 with userid*/ - ADD_DATA(buf, "\x04\x01\x11\x12\x02\x02\x02\x04me\x00"); - test_assert(fetch_from_buf_socks(buf, socks, get_options()->TestSocks, - get_options()->SafeSocks) == 1); - test_eq(4, socks->socks_version); - test_eq(0, socks->replylen); /* XXX: shouldn't tor reply? */ - test_eq(SOCKS_COMMAND_CONNECT, socks->command); - test_streq("2.2.2.4", socks->address); - test_eq(4370, socks->port); - test_assert(socks->got_auth == 1); - test_assert(socks->username); - test_eq(2, socks->usernamelen); - test_memeq("me", socks->username, 2); - - test_eq(0, buf_datalen(buf)); - socks_request_clear(socks); - - /* SOCKS 4a Send RESOLVE [F0] request for torproject.org */ - ADD_DATA(buf, "\x04\xF0\x01\x01\x00\x00\x00\x02me\x00torproject.org\x00"); - test_assert(fetch_from_buf_socks(buf, socks, get_options()->TestSocks, - get_options()->SafeSocks) == 1); - test_eq(4, socks->socks_version); - test_eq(0, socks->replylen); /* XXX: shouldn't tor reply? */ - test_streq("torproject.org", socks->address); - - test_eq(0, buf_datalen(buf)); - - done: - ; -} - -/** Perform unsupported SOCKS 5 commands */ -static void -test_socks_5_unsupported_commands(void *ptr) -{ - SOCKS_TEST_INIT(); - - /* SOCKS 5 Send unsupported BIND [02] command */ - ADD_DATA(buf, "\x05\x02\x00\x01"); - - test_eq(fetch_from_buf_socks(buf, socks, get_options()->TestSocks, - get_options()->SafeSocks), 0); - test_eq(0, buf_datalen(buf)); - test_eq(5, socks->socks_version); - test_eq(2, socks->replylen); - test_eq(5, socks->reply[0]); - test_eq(0, socks->reply[1]); - ADD_DATA(buf, "\x05\x02\x00\x01\x02\x02\x02\x01\x01\x01"); - test_eq(fetch_from_buf_socks(buf, socks, get_options()->TestSocks, - get_options()->SafeSocks), -1); - /* XXX: shouldn't tor reply 'command not supported' [07]? */ - - buf_clear(buf); - socks_request_clear(socks); - - /* SOCKS 5 Send unsupported UDP_ASSOCIATE [03] command */ - ADD_DATA(buf, "\x05\x03\x00\x01\x02"); - test_eq(fetch_from_buf_socks(buf, socks, get_options()->TestSocks, - get_options()->SafeSocks), 0); - test_eq(5, socks->socks_version); - test_eq(2, socks->replylen); - test_eq(5, socks->reply[0]); - test_eq(2, socks->reply[1]); - ADD_DATA(buf, "\x05\x03\x00\x01\x02\x02\x02\x01\x01\x01"); - test_eq(fetch_from_buf_socks(buf, socks, get_options()->TestSocks, - get_options()->SafeSocks), -1); - /* XXX: shouldn't tor reply 'command not supported' [07]? */ - - done: - ; -} - -/** Perform supported SOCKS 5 commands */ -static void -test_socks_5_supported_commands(void *ptr) -{ - SOCKS_TEST_INIT(); - - /* SOCKS 5 Send CONNECT [01] to IP address 2.2.2.2:4369 */ - ADD_DATA(buf, "\x05\x01\x00"); - test_eq(fetch_from_buf_socks(buf, socks, get_options()->TestSocks, - get_options()->SafeSocks), 0); - test_eq(5, socks->socks_version); - test_eq(2, socks->replylen); - test_eq(5, socks->reply[0]); - test_eq(0, socks->reply[1]); - - ADD_DATA(buf, "\x05\x01\x00\x01\x02\x02\x02\x02\x11\x11"); - test_eq(fetch_from_buf_socks(buf, socks, get_options()->TestSocks, - get_options()->SafeSocks), 1); - test_streq("2.2.2.2", socks->address); - test_eq(4369, socks->port); - - test_eq(0, buf_datalen(buf)); - socks_request_clear(socks); - - /* SOCKS 5 Send CONNECT [01] to FQDN torproject.org:4369 */ - ADD_DATA(buf, "\x05\x01\x00"); - ADD_DATA(buf, "\x05\x01\x00\x03\x0Etorproject.org\x11\x11"); - test_eq(fetch_from_buf_socks(buf, socks, get_options()->TestSocks, - get_options()->SafeSocks), 1); - - test_eq(5, socks->socks_version); - test_eq(2, socks->replylen); - test_eq(5, socks->reply[0]); - test_eq(0, socks->reply[1]); - test_streq("torproject.org", socks->address); - test_eq(4369, socks->port); - - test_eq(0, buf_datalen(buf)); - socks_request_clear(socks); - - /* SOCKS 5 Send RESOLVE [F0] request for torproject.org:4369 */ - ADD_DATA(buf, "\x05\x01\x00"); - ADD_DATA(buf, "\x05\xF0\x00\x03\x0Etorproject.org\x01\x02"); - test_assert(fetch_from_buf_socks(buf, socks, get_options()->TestSocks, - get_options()->SafeSocks) == 1); - test_eq(5, socks->socks_version); - test_eq(2, socks->replylen); - test_eq(5, socks->reply[0]); - test_eq(0, socks->reply[1]); - test_streq("torproject.org", socks->address); - - test_eq(0, buf_datalen(buf)); - socks_request_clear(socks); - - /* SOCKS 5 Send RESOLVE_PTR [F1] for IP address 2.2.2.5 */ - ADD_DATA(buf, "\x05\x01\x00"); - ADD_DATA(buf, "\x05\xF1\x00\x01\x02\x02\x02\x05\x01\x03"); - test_assert(fetch_from_buf_socks(buf, socks, get_options()->TestSocks, - get_options()->SafeSocks) == 1); - test_eq(5, socks->socks_version); - test_eq(2, socks->replylen); - test_eq(5, socks->reply[0]); - test_eq(0, socks->reply[1]); - test_streq("2.2.2.5", socks->address); - - test_eq(0, buf_datalen(buf)); - - done: - ; -} - -/** Perform SOCKS 5 authentication */ -static void -test_socks_5_no_authenticate(void *ptr) -{ - SOCKS_TEST_INIT(); - - /*SOCKS 5 No Authentication */ - ADD_DATA(buf,"\x05\x01\x00"); - test_assert(!fetch_from_buf_socks(buf, socks, - get_options()->TestSocks, - get_options()->SafeSocks)); - test_eq(2, socks->replylen); - test_eq(5, socks->reply[0]); - test_eq(SOCKS_NO_AUTH, socks->reply[1]); - - test_eq(0, buf_datalen(buf)); - - /*SOCKS 5 Send username/password anyway - pretend to be broken */ - ADD_DATA(buf,"\x01\x02\x01\x01\x02\x01\x01"); - test_assert(!fetch_from_buf_socks(buf, socks, - get_options()->TestSocks, - get_options()->SafeSocks)); - test_eq(5, socks->socks_version); - test_eq(2, socks->replylen); - test_eq(1, socks->reply[0]); - test_eq(0, socks->reply[1]); - - test_eq(2, socks->usernamelen); - test_eq(2, socks->passwordlen); - - test_memeq("\x01\x01", socks->username, 2); - test_memeq("\x01\x01", socks->password, 2); - - done: - ; -} - -/** Perform SOCKS 5 authentication */ -static void -test_socks_5_authenticate(void *ptr) -{ - SOCKS_TEST_INIT(); - - /* SOCKS 5 Negotiate username/password authentication */ - ADD_DATA(buf, "\x05\x01\x02"); - - test_assert(!fetch_from_buf_socks(buf, socks, - get_options()->TestSocks, - get_options()->SafeSocks)); - test_eq(2, socks->replylen); - test_eq(5, socks->reply[0]); - test_eq(SOCKS_USER_PASS, socks->reply[1]); - test_eq(5, socks->socks_version); - - test_eq(0, buf_datalen(buf)); - - /* SOCKS 5 Send username/password */ - ADD_DATA(buf, "\x01\x02me\x08mypasswd"); - test_assert(!fetch_from_buf_socks(buf, socks, - get_options()->TestSocks, - get_options()->SafeSocks)); - test_eq(5, socks->socks_version); - test_eq(2, socks->replylen); - test_eq(1, socks->reply[0]); - test_eq(0, socks->reply[1]); - - test_eq(2, socks->usernamelen); - test_eq(8, socks->passwordlen); - - test_memeq("me", socks->username, 2); - test_memeq("mypasswd", socks->password, 8); - - done: - ; -} - -/** Perform SOCKS 5 authentication and send data all in one go */ -static void -test_socks_5_authenticate_with_data(void *ptr) -{ - SOCKS_TEST_INIT(); - - /* SOCKS 5 Negotiate username/password authentication */ - ADD_DATA(buf, "\x05\x01\x02"); - - test_assert(!fetch_from_buf_socks(buf, socks, - get_options()->TestSocks, - get_options()->SafeSocks)); - test_eq(2, socks->replylen); - test_eq(5, socks->reply[0]); - test_eq(SOCKS_USER_PASS, socks->reply[1]); - test_eq(5, socks->socks_version); - - test_eq(0, buf_datalen(buf)); - - /* SOCKS 5 Send username/password */ - /* SOCKS 5 Send CONNECT [01] to IP address 2.2.2.2:4369 */ - ADD_DATA(buf, "\x01\x02me\x03you\x05\x01\x00\x01\x02\x02\x02\x02\x11\x11"); - test_assert(fetch_from_buf_socks(buf, socks, - get_options()->TestSocks, - get_options()->SafeSocks) == 1); - test_eq(5, socks->socks_version); - test_eq(2, socks->replylen); - test_eq(1, socks->reply[0]); - test_eq(0, socks->reply[1]); - - test_streq("2.2.2.2", socks->address); - test_eq(4369, socks->port); - - test_eq(2, socks->usernamelen); - test_eq(3, socks->passwordlen); - test_memeq("me", socks->username, 2); - test_memeq("you", socks->password, 3); - - done: - ; -} - -/** Perform SOCKS 5 authentication before method negotiated */ -static void -test_socks_5_auth_before_negotiation(void *ptr) -{ - SOCKS_TEST_INIT(); - - /* SOCKS 5 Send username/password */ - ADD_DATA(buf, "\x01\x02me\x02me"); - test_assert(fetch_from_buf_socks(buf, socks, - get_options()->TestSocks, - get_options()->SafeSocks) == -1); - test_eq(0, socks->socks_version); - test_eq(0, socks->replylen); - test_eq(0, socks->reply[0]); - test_eq(0, socks->reply[1]); - - done: - ; -} - +/** Run unit tests for the onion handshake code. */ static void -test_buffer_copy(void *arg) +test_onion_handshake(void) { - generic_buffer_t *buf=NULL, *buf2=NULL; - const char *s; - size_t len; - char b[256]; + /* client-side */ + crypto_dh_t *c_dh = NULL; + char c_buf[TAP_ONIONSKIN_CHALLENGE_LEN]; + char c_keys[40]; + /* server-side */ + char s_buf[TAP_ONIONSKIN_REPLY_LEN]; + char s_keys[40]; int i; - (void)arg; - - buf = generic_buffer_new(); - tt_assert(buf); - - /* Copy an empty buffer. */ - tt_int_op(0, ==, generic_buffer_set_to_copy(&buf2, buf)); - tt_assert(buf2); - tt_int_op(0, ==, generic_buffer_len(buf2)); - - /* Now try with a short buffer. */ - s = "And now comes an act of enormous enormance!"; - len = strlen(s); - generic_buffer_add(buf, s, len); - tt_int_op(len, ==, generic_buffer_len(buf)); - /* Add junk to buf2 so we can test replacing.*/ - generic_buffer_add(buf2, "BLARG", 5); - tt_int_op(0, ==, generic_buffer_set_to_copy(&buf2, buf)); - tt_int_op(len, ==, generic_buffer_len(buf2)); - generic_buffer_get(buf2, b, len); - test_mem_op(b, ==, s, len); - /* Now free buf2 and retry so we can test allocating */ - generic_buffer_free(buf2); - buf2 = NULL; - tt_int_op(0, ==, generic_buffer_set_to_copy(&buf2, buf)); - tt_int_op(len, ==, generic_buffer_len(buf2)); - generic_buffer_get(buf2, b, len); - test_mem_op(b, ==, s, len); - /* Clear buf for next test */ - generic_buffer_get(buf, b, len); - tt_int_op(generic_buffer_len(buf),==,0); - - /* Okay, now let's try a bigger buffer. */ - s = "Quis autem vel eum iure reprehenderit qui in ea voluptate velit " - "esse quam nihil molestiae consequatur, vel illum qui dolorem eum " - "fugiat quo voluptas nulla pariatur?"; - len = strlen(s); - for (i = 0; i < 256; ++i) { - b[0]=i; - generic_buffer_add(buf, b, 1); - generic_buffer_add(buf, s, len); - } - tt_int_op(0, ==, generic_buffer_set_to_copy(&buf2, buf)); - tt_int_op(generic_buffer_len(buf2), ==, generic_buffer_len(buf)); - for (i = 0; i < 256; ++i) { - generic_buffer_get(buf2, b, len+1); - tt_int_op((unsigned char)b[0],==,i); - test_mem_op(b+1, ==, s, len); - } - - done: - if (buf) - generic_buffer_free(buf); - if (buf2) - generic_buffer_free(buf2); -} - -/** Run unit tests for buffers.c */ -static void -test_buffers(void) -{ - char str[256]; - char str2[256]; - - buf_t *buf = NULL, *buf2 = NULL; - const char *cp; + /* shared */ + crypto_pk_t *pk = NULL, *pk2 = NULL; - int j; - size_t r; + pk = pk_generate(0); + pk2 = pk_generate(1); - /**** - * buf_new - ****/ - if (!(buf = buf_new())) - test_fail(); + /* client handshake 1. */ + memset(c_buf, 0, TAP_ONIONSKIN_CHALLENGE_LEN); + test_assert(! onion_skin_TAP_create(pk, &c_dh, c_buf)); - //test_eq(buf_capacity(buf), 4096); - test_eq(buf_datalen(buf), 0); + for (i = 1; i <= 3; ++i) { + crypto_pk_t *k1, *k2; + if (i==1) { + /* server handshake: only one key known. */ + k1 = pk; k2 = NULL; + } else if (i==2) { + /* server handshake: try the right key first. */ + k1 = pk; k2 = pk2; + } else { + /* server handshake: try the right key second. */ + k1 = pk2; k2 = pk; + } - /**** - * General pointer frobbing - */ - for (j=0;j<256;++j) { - str[j] = (char)j; - } - write_to_buf(str, 256, buf); - write_to_buf(str, 256, buf); - test_eq(buf_datalen(buf), 512); - fetch_from_buf(str2, 200, buf); - test_memeq(str, str2, 200); - test_eq(buf_datalen(buf), 312); - memset(str2, 0, sizeof(str2)); - - fetch_from_buf(str2, 256, buf); - test_memeq(str+200, str2, 56); - test_memeq(str, str2+56, 200); - test_eq(buf_datalen(buf), 56); - memset(str2, 0, sizeof(str2)); - /* Okay, now we should be 512 bytes into the 4096-byte buffer. If we add - * another 3584 bytes, we hit the end. */ - for (j=0;j<15;++j) { - write_to_buf(str, 256, buf); - } - assert_buf_ok(buf); - test_eq(buf_datalen(buf), 3896); - fetch_from_buf(str2, 56, buf); - test_eq(buf_datalen(buf), 3840); - test_memeq(str+200, str2, 56); - for (j=0;j<15;++j) { - memset(str2, 0, sizeof(str2)); - fetch_from_buf(str2, 256, buf); - test_memeq(str, str2, 256); - } - test_eq(buf_datalen(buf), 0); - buf_free(buf); - buf = NULL; - - /* Okay, now make sure growing can work. */ - buf = buf_new_with_capacity(16); - //test_eq(buf_capacity(buf), 16); - write_to_buf(str+1, 255, buf); - //test_eq(buf_capacity(buf), 256); - fetch_from_buf(str2, 254, buf); - test_memeq(str+1, str2, 254); - //test_eq(buf_capacity(buf), 256); - assert_buf_ok(buf); - write_to_buf(str, 32, buf); - //test_eq(buf_capacity(buf), 256); - assert_buf_ok(buf); - write_to_buf(str, 256, buf); - assert_buf_ok(buf); - //test_eq(buf_capacity(buf), 512); - test_eq(buf_datalen(buf), 33+256); - fetch_from_buf(str2, 33, buf); - test_eq(*str2, str[255]); - - test_memeq(str2+1, str, 32); - //test_eq(buf_capacity(buf), 512); - test_eq(buf_datalen(buf), 256); - fetch_from_buf(str2, 256, buf); - test_memeq(str, str2, 256); - - /* now try shrinking: case 1. */ - buf_free(buf); - buf = buf_new_with_capacity(33668); - for (j=0;j<67;++j) { - write_to_buf(str,255, buf); - } - //test_eq(buf_capacity(buf), 33668); - test_eq(buf_datalen(buf), 17085); - for (j=0; j < 40; ++j) { - fetch_from_buf(str2, 255,buf); - test_memeq(str2, str, 255); - } + memset(s_buf, 0, TAP_ONIONSKIN_REPLY_LEN); + memset(s_keys, 0, 40); + test_assert(! onion_skin_TAP_server_handshake(c_buf, k1, k2, + s_buf, s_keys, 40)); - /* now try shrinking: case 2. */ - buf_free(buf); - buf = buf_new_with_capacity(33668); - for (j=0;j<67;++j) { - write_to_buf(str,255, buf); - } - for (j=0; j < 20; ++j) { - fetch_from_buf(str2, 255,buf); - test_memeq(str2, str, 255); - } - for (j=0;j<80;++j) { - write_to_buf(str,255, buf); - } - //test_eq(buf_capacity(buf),33668); - for (j=0; j < 120; ++j) { - fetch_from_buf(str2, 255,buf); - test_memeq(str2, str, 255); - } + /* client handshake 2 */ + memset(c_keys, 0, 40); + test_assert(! onion_skin_TAP_client_handshake(c_dh, s_buf, c_keys, 40)); - /* Move from buf to buf. */ - buf_free(buf); - buf = buf_new_with_capacity(4096); - buf2 = buf_new_with_capacity(4096); - for (j=0;j<100;++j) - write_to_buf(str, 255, buf); - test_eq(buf_datalen(buf), 25500); - for (j=0;j<100;++j) { - r = 10; - move_buf_to_buf(buf2, buf, &r); - test_eq(r, 0); - } - test_eq(buf_datalen(buf), 24500); - test_eq(buf_datalen(buf2), 1000); - for (j=0;j<3;++j) { - fetch_from_buf(str2, 255, buf2); - test_memeq(str2, str, 255); - } - r = 8192; /*big move*/ - move_buf_to_buf(buf2, buf, &r); - test_eq(r, 0); - r = 30000; /* incomplete move */ - move_buf_to_buf(buf2, buf, &r); - test_eq(r, 13692); - for (j=0;j<97;++j) { - fetch_from_buf(str2, 255, buf2); - test_memeq(str2, str, 255); + test_memeq(c_keys, s_keys, 40); + memset(s_buf, 0, 40); + test_memneq(c_keys, s_buf, 40); } - buf_free(buf); - buf_free(buf2); - buf = buf2 = NULL; - - buf = buf_new_with_capacity(5); - cp = "Testing. This is a moderately long Testing string."; - for (j = 0; cp[j]; j++) - write_to_buf(cp+j, 1, buf); - test_eq(0, buf_find_string_offset(buf, "Testing", 7)); - test_eq(1, buf_find_string_offset(buf, "esting", 6)); - test_eq(1, buf_find_string_offset(buf, "est", 3)); - test_eq(39, buf_find_string_offset(buf, "ing str", 7)); - test_eq(35, buf_find_string_offset(buf, "Testing str", 11)); - test_eq(32, buf_find_string_offset(buf, "ng ", 3)); - test_eq(43, buf_find_string_offset(buf, "string.", 7)); - test_eq(-1, buf_find_string_offset(buf, "shrdlu", 6)); - test_eq(-1, buf_find_string_offset(buf, "Testing thing", 13)); - test_eq(-1, buf_find_string_offset(buf, "ngx", 3)); - buf_free(buf); - buf = NULL; - - /* Try adding a string too long for any freelist. */ - { - char *cp = tor_malloc_zero(65536); - buf = buf_new(); - write_to_buf(cp, 65536, buf); - tor_free(cp); - - tt_int_op(buf_datalen(buf), ==, 65536); - buf_free(buf); - buf = NULL; - } - done: - if (buf) - buf_free(buf); - if (buf2) - buf_free(buf2); + crypto_dh_free(c_dh); + crypto_pk_free(pk); + crypto_pk_free(pk2); } -/** Run unit tests for the onion handshake code. */ static void -test_onion_handshake(void) +test_bad_onion_handshake(void *arg) { + char junk_buf[TAP_ONIONSKIN_CHALLENGE_LEN]; + char junk_buf2[TAP_ONIONSKIN_CHALLENGE_LEN]; /* client-side */ crypto_dh_t *c_dh = NULL; char c_buf[TAP_ONIONSKIN_CHALLENGE_LEN]; char c_keys[40]; - /* server-side */ char s_buf[TAP_ONIONSKIN_REPLY_LEN]; char s_keys[40]; - /* shared */ - crypto_pk_t *pk = NULL; + crypto_pk_t *pk = NULL, *pk2 = NULL; + + (void)arg; pk = pk_generate(0); + pk2 = pk_generate(1); - /* client handshake 1. */ + /* Server: Case 1: the encrypted data is degenerate. */ + memset(junk_buf, 0, sizeof(junk_buf)); + crypto_pk_public_hybrid_encrypt(pk, junk_buf2, TAP_ONIONSKIN_CHALLENGE_LEN, + junk_buf, DH_KEY_LEN, PK_PKCS1_OAEP_PADDING, 1); + tt_int_op(-1, ==, + onion_skin_TAP_server_handshake(junk_buf2, pk, NULL, + s_buf, s_keys, 40)); + + /* Server: Case 2: the encrypted data is not long enough. */ + memset(junk_buf, 0, sizeof(junk_buf)); + memset(junk_buf2, 0, sizeof(junk_buf2)); + crypto_pk_public_encrypt(pk, junk_buf2, sizeof(junk_buf2), + junk_buf, 48, PK_PKCS1_OAEP_PADDING); + tt_int_op(-1, ==, + onion_skin_TAP_server_handshake(junk_buf2, pk, NULL, + s_buf, s_keys, 40)); + + /* client handshake 1: do it straight. */ memset(c_buf, 0, TAP_ONIONSKIN_CHALLENGE_LEN); test_assert(! onion_skin_TAP_create(pk, &c_dh, c_buf)); - /* server handshake */ - memset(s_buf, 0, TAP_ONIONSKIN_REPLY_LEN); - memset(s_keys, 0, 40); - test_assert(! onion_skin_TAP_server_handshake(c_buf, pk, NULL, + /* Server: Case 3: we just don't have the right key. */ + tt_int_op(-1, ==, + onion_skin_TAP_server_handshake(c_buf, pk2, NULL, s_buf, s_keys, 40)); - /* client handshake 2 */ - memset(c_keys, 0, 40); - test_assert(! onion_skin_TAP_client_handshake(c_dh, s_buf, c_keys, 40)); + /* Server: Case 4: The RSA-encrypted portion is corrupt. */ + c_buf[64] ^= 33; + tt_int_op(-1, ==, + onion_skin_TAP_server_handshake(c_buf, pk, NULL, + s_buf, s_keys, 40)); + c_buf[64] ^= 33; - if (memcmp(c_keys, s_keys, 40)) { - puts("Aiiiie"); - exit(1); - } - test_memeq(c_keys, s_keys, 40); - memset(s_buf, 0, 40); - test_memneq(c_keys, s_buf, 40); + /* (Let the server procede) */ + tt_int_op(0, ==, + onion_skin_TAP_server_handshake(c_buf, pk, NULL, + s_buf, s_keys, 40)); + + /* Client: Case 1: The server sent back junk. */ + s_buf[64] ^= 33; + tt_int_op(-1, ==, + onion_skin_TAP_client_handshake(c_dh, s_buf, c_keys, 40)); + s_buf[64] ^= 33; + + /* Let the client finish; make sure it can. */ + tt_int_op(0, ==, + onion_skin_TAP_client_handshake(c_dh, s_buf, c_keys, 40)); + test_memeq(s_keys, c_keys, 40); + + /* Client: Case 2: The server sent back a degenerate DH. */ + memset(s_buf, 0, sizeof(s_buf)); + tt_int_op(-1, ==, + onion_skin_TAP_client_handshake(c_dh, s_buf, c_keys, 40)); done: - if (c_dh) - crypto_dh_free(c_dh); - if (pk) - crypto_pk_free(pk); + crypto_dh_free(c_dh); + crypto_pk_free(pk); + crypto_pk_free(pk2); } #ifdef CURVE25519_ENABLED @@ -945,9 +420,10 @@ test_onion_queues(void) or_circuit_t *circ1 = or_circuit_new(0, NULL); or_circuit_t *circ2 = or_circuit_new(0, NULL); - create_cell_t *onionskin = NULL; + create_cell_t *onionskin = NULL, *create2_ptr; create_cell_t *create1 = tor_malloc_zero(sizeof(create_cell_t)); create_cell_t *create2 = tor_malloc_zero(sizeof(create_cell_t)); + create2_ptr = create2; /* remember, but do not free */ create_cell_init(create1, CELL_CREATE, ONION_HANDSHAKE_TYPE_TAP, TAP_ONIONSKIN_CHALLENGE_LEN, buf1); @@ -956,26 +432,29 @@ test_onion_queues(void) test_eq(0, onion_num_pending(ONION_HANDSHAKE_TYPE_TAP)); test_eq(0, onion_pending_add(circ1, create1)); + create1 = NULL; test_eq(1, onion_num_pending(ONION_HANDSHAKE_TYPE_TAP)); test_eq(0, onion_num_pending(ONION_HANDSHAKE_TYPE_NTOR)); test_eq(0, onion_pending_add(circ2, create2)); + create2 = NULL; test_eq(1, onion_num_pending(ONION_HANDSHAKE_TYPE_NTOR)); test_eq_ptr(circ2, onion_next_task(&onionskin)); test_eq(1, onion_num_pending(ONION_HANDSHAKE_TYPE_TAP)); test_eq(0, onion_num_pending(ONION_HANDSHAKE_TYPE_NTOR)); + tt_ptr_op(onionskin, ==, create2_ptr); clear_pending_onions(); test_eq(0, onion_num_pending(ONION_HANDSHAKE_TYPE_TAP)); test_eq(0, onion_num_pending(ONION_HANDSHAKE_TYPE_NTOR)); done: - ; -// circuit_free(circ1); -// circuit_free(circ2); - /* and free create1 and create2 */ - /* XXX leaks everything here */ + circuit_free(TO_CIRCUIT(circ1)); + circuit_free(TO_CIRCUIT(circ2)); + tor_free(create1); + tor_free(create2); + tor_free(onionskin); } static void @@ -994,14 +473,14 @@ test_circuit_timeout(void) circuit_build_times_t estimate; circuit_build_times_t final; double timeout1, timeout2; - or_state_t state; + or_state_t *state=NULL; int i, runs; double close_ms; circuit_build_times_init(&initial); circuit_build_times_init(&estimate); circuit_build_times_init(&final); - memset(&state, 0, sizeof(or_state_t)); + state = or_state_new(); circuitbuild_running_unit_tests(); #define timeout0 (build_time_t)(30*1000.0) @@ -1033,8 +512,9 @@ test_circuit_timeout(void) test_assert(estimate.total_build_times <= CBT_NCIRCUITS_TO_OBSERVE); - circuit_build_times_update_state(&estimate, &state); - test_assert(circuit_build_times_parse_state(&final, &state) == 0); + circuit_build_times_update_state(&estimate, state); + circuit_build_times_free_timeouts(&final); + test_assert(circuit_build_times_parse_state(&final, state) == 0); circuit_build_times_update_alpha(&final); timeout2 = circuit_build_times_calculate_timeout(&final, @@ -1123,336 +603,10 @@ test_circuit_timeout(void) } done: - return; -} - -/* Helper: assert that short_policy parses and writes back out as itself, - or as <b>expected</b> if that's provided. */ -static void -test_short_policy_parse(const char *input, - const char *expected) -{ - short_policy_t *short_policy = NULL; - char *out = NULL; - - if (expected == NULL) - expected = input; - - short_policy = parse_short_policy(input); - tt_assert(short_policy); - out = write_short_policy(short_policy); - tt_str_op(out, ==, expected); - - done: - tor_free(out); - short_policy_free(short_policy); -} - -/** Helper: Parse the exit policy string in <b>policy_str</b>, and make sure - * that policies_summarize() produces the string <b>expected_summary</b> from - * it. */ -static void -test_policy_summary_helper(const char *policy_str, - const char *expected_summary) -{ - config_line_t line; - smartlist_t *policy = smartlist_new(); - char *summary = NULL; - char *summary_after = NULL; - int r; - short_policy_t *short_policy = NULL; - - line.key = (char*)"foo"; - line.value = (char *)policy_str; - line.next = NULL; - - r = policies_parse_exit_policy(&line, &policy, 1, 0, NULL, 1); - test_eq(r, 0); - summary = policy_summarize(policy, AF_INET); - - test_assert(summary != NULL); - test_streq(summary, expected_summary); - - short_policy = parse_short_policy(summary); - tt_assert(short_policy); - summary_after = write_short_policy(short_policy); - test_streq(summary, summary_after); - - done: - tor_free(summary_after); - tor_free(summary); - if (policy) - addr_policy_list_free(policy); - short_policy_free(short_policy); -} - -/** Run unit tests for generating summary lines of exit policies */ -static void -test_policies(void) -{ - int i; - smartlist_t *policy = NULL, *policy2 = NULL, *policy3 = NULL, - *policy4 = NULL, *policy5 = NULL, *policy6 = NULL, - *policy7 = NULL; - addr_policy_t *p; - tor_addr_t tar; - config_line_t line; - smartlist_t *sm = NULL; - char *policy_str = NULL; - - policy = smartlist_new(); - - p = router_parse_addr_policy_item_from_string("reject 192.168.0.0/16:*",-1); - test_assert(p != NULL); - test_eq(ADDR_POLICY_REJECT, p->policy_type); - tor_addr_from_ipv4h(&tar, 0xc0a80000u); - test_eq(0, tor_addr_compare(&p->addr, &tar, CMP_EXACT)); - test_eq(16, p->maskbits); - test_eq(1, p->prt_min); - test_eq(65535, p->prt_max); - - smartlist_add(policy, p); - - tor_addr_from_ipv4h(&tar, 0x01020304u); - test_assert(ADDR_POLICY_ACCEPTED == - compare_tor_addr_to_addr_policy(&tar, 2, policy)); - tor_addr_make_unspec(&tar); - test_assert(ADDR_POLICY_PROBABLY_ACCEPTED == - compare_tor_addr_to_addr_policy(&tar, 2, policy)); - tor_addr_from_ipv4h(&tar, 0xc0a80102); - test_assert(ADDR_POLICY_REJECTED == - compare_tor_addr_to_addr_policy(&tar, 2, policy)); - - test_assert(0 == policies_parse_exit_policy(NULL, &policy2, 1, 1, NULL, 1)); - test_assert(policy2); - - policy3 = smartlist_new(); - p = router_parse_addr_policy_item_from_string("reject *:*",-1); - test_assert(p != NULL); - smartlist_add(policy3, p); - p = router_parse_addr_policy_item_from_string("accept *:*",-1); - test_assert(p != NULL); - smartlist_add(policy3, p); - - policy4 = smartlist_new(); - p = router_parse_addr_policy_item_from_string("accept *:443",-1); - test_assert(p != NULL); - smartlist_add(policy4, p); - p = router_parse_addr_policy_item_from_string("accept *:443",-1); - test_assert(p != NULL); - smartlist_add(policy4, p); - - policy5 = smartlist_new(); - p = router_parse_addr_policy_item_from_string("reject 0.0.0.0/8:*",-1); - test_assert(p != NULL); - smartlist_add(policy5, p); - p = router_parse_addr_policy_item_from_string("reject 169.254.0.0/16:*",-1); - test_assert(p != NULL); - smartlist_add(policy5, p); - p = router_parse_addr_policy_item_from_string("reject 127.0.0.0/8:*",-1); - test_assert(p != NULL); - smartlist_add(policy5, p); - p = router_parse_addr_policy_item_from_string("reject 192.168.0.0/16:*",-1); - test_assert(p != NULL); - smartlist_add(policy5, p); - p = router_parse_addr_policy_item_from_string("reject 10.0.0.0/8:*",-1); - test_assert(p != NULL); - smartlist_add(policy5, p); - p = router_parse_addr_policy_item_from_string("reject 172.16.0.0/12:*",-1); - test_assert(p != NULL); - smartlist_add(policy5, p); - p = router_parse_addr_policy_item_from_string("reject 80.190.250.90:*",-1); - test_assert(p != NULL); - smartlist_add(policy5, p); - p = router_parse_addr_policy_item_from_string("reject *:1-65534",-1); - test_assert(p != NULL); - smartlist_add(policy5, p); - p = router_parse_addr_policy_item_from_string("reject *:65535",-1); - test_assert(p != NULL); - smartlist_add(policy5, p); - p = router_parse_addr_policy_item_from_string("accept *:1-65535",-1); - test_assert(p != NULL); - smartlist_add(policy5, p); - - policy6 = smartlist_new(); - p = router_parse_addr_policy_item_from_string("accept 43.3.0.0/9:*",-1); - test_assert(p != NULL); - smartlist_add(policy6, p); - - policy7 = smartlist_new(); - p = router_parse_addr_policy_item_from_string("accept 0.0.0.0/8:*",-1); - test_assert(p != NULL); - smartlist_add(policy7, p); - - test_assert(!exit_policy_is_general_exit(policy)); - test_assert(exit_policy_is_general_exit(policy2)); - test_assert(!exit_policy_is_general_exit(NULL)); - test_assert(!exit_policy_is_general_exit(policy3)); - test_assert(!exit_policy_is_general_exit(policy4)); - test_assert(!exit_policy_is_general_exit(policy5)); - test_assert(!exit_policy_is_general_exit(policy6)); - test_assert(!exit_policy_is_general_exit(policy7)); - - test_assert(cmp_addr_policies(policy, policy2)); - test_assert(cmp_addr_policies(policy, NULL)); - test_assert(!cmp_addr_policies(policy2, policy2)); - test_assert(!cmp_addr_policies(NULL, NULL)); - - test_assert(!policy_is_reject_star(policy2, AF_INET)); - test_assert(policy_is_reject_star(policy, AF_INET)); - test_assert(policy_is_reject_star(NULL, AF_INET)); - - addr_policy_list_free(policy); - policy = NULL; - - /* make sure compacting logic works. */ - policy = NULL; - line.key = (char*)"foo"; - line.value = (char*)"accept *:80,reject private:*,reject *:*"; - line.next = NULL; - test_assert(0 == policies_parse_exit_policy(&line, &policy, 1, 0, NULL, 1)); - test_assert(policy); - //test_streq(policy->string, "accept *:80"); - //test_streq(policy->next->string, "reject *:*"); - test_eq(smartlist_len(policy), 4); - - /* test policy summaries */ - /* check if we properly ignore private IP addresses */ - test_policy_summary_helper("reject 192.168.0.0/16:*," - "reject 0.0.0.0/8:*," - "reject 10.0.0.0/8:*," - "accept *:10-30," - "accept *:90," - "reject *:*", - "accept 10-30,90"); - /* check all accept policies, and proper counting of rejects */ - test_policy_summary_helper("reject 11.0.0.0/9:80," - "reject 12.0.0.0/9:80," - "reject 13.0.0.0/9:80," - "reject 14.0.0.0/9:80," - "accept *:*", "accept 1-65535"); - test_policy_summary_helper("reject 11.0.0.0/9:80," - "reject 12.0.0.0/9:80," - "reject 13.0.0.0/9:80," - "reject 14.0.0.0/9:80," - "reject 15.0.0.0:81," - "accept *:*", "accept 1-65535"); - test_policy_summary_helper("reject 11.0.0.0/9:80," - "reject 12.0.0.0/9:80," - "reject 13.0.0.0/9:80," - "reject 14.0.0.0/9:80," - "reject 15.0.0.0:80," - "accept *:*", - "reject 80"); - /* no exits */ - test_policy_summary_helper("accept 11.0.0.0/9:80," - "reject *:*", - "reject 1-65535"); - /* port merging */ - test_policy_summary_helper("accept *:80," - "accept *:81," - "accept *:100-110," - "accept *:111," - "reject *:*", - "accept 80-81,100-111"); - /* border ports */ - test_policy_summary_helper("accept *:1," - "accept *:3," - "accept *:65535," - "reject *:*", - "accept 1,3,65535"); - /* holes */ - test_policy_summary_helper("accept *:1," - "accept *:3," - "accept *:5," - "accept *:7," - "reject *:*", - "accept 1,3,5,7"); - test_policy_summary_helper("reject *:1," - "reject *:3," - "reject *:5," - "reject *:7," - "accept *:*", - "reject 1,3,5,7"); - - /* Short policies with unrecognized formats should get accepted. */ - test_short_policy_parse("accept fred,2,3-5", "accept 2,3-5"); - test_short_policy_parse("accept 2,fred,3", "accept 2,3"); - test_short_policy_parse("accept 2,fred,3,bob", "accept 2,3"); - test_short_policy_parse("accept 2,-3,500-600", "accept 2,500-600"); - /* Short policies with nil entries are accepted too. */ - test_short_policy_parse("accept 1,,3", "accept 1,3"); - test_short_policy_parse("accept 100-200,,", "accept 100-200"); - test_short_policy_parse("reject ,1-10,,,,30-40", "reject 1-10,30-40"); - - /* Try parsing various broken short policies */ - tt_ptr_op(NULL, ==, parse_short_policy("accept 200-199")); - tt_ptr_op(NULL, ==, parse_short_policy("")); - tt_ptr_op(NULL, ==, parse_short_policy("rejekt 1,2,3")); - tt_ptr_op(NULL, ==, parse_short_policy("reject ")); - tt_ptr_op(NULL, ==, parse_short_policy("reject")); - tt_ptr_op(NULL, ==, parse_short_policy("rej")); - tt_ptr_op(NULL, ==, parse_short_policy("accept 2,3,100000")); - tt_ptr_op(NULL, ==, parse_short_policy("accept 2,3x,4")); - tt_ptr_op(NULL, ==, parse_short_policy("accept 2,3x,4")); - tt_ptr_op(NULL, ==, parse_short_policy("accept 2-")); - tt_ptr_op(NULL, ==, parse_short_policy("accept 2-x")); - tt_ptr_op(NULL, ==, parse_short_policy("accept 1-,3")); - tt_ptr_op(NULL, ==, parse_short_policy("accept 1-,3")); - /* Test a too-long policy. */ - { - int i; - char *policy = NULL; - smartlist_t *chunks = smartlist_new(); - smartlist_add(chunks, tor_strdup("accept ")); - for (i=1; i<10000; ++i) - smartlist_add_asprintf(chunks, "%d,", i); - smartlist_add(chunks, tor_strdup("20000")); - policy = smartlist_join_strings(chunks, "", 0, NULL); - SMARTLIST_FOREACH(chunks, char *, ch, tor_free(ch)); - smartlist_free(chunks); - tt_ptr_op(NULL, ==, parse_short_policy(policy));/* shouldn't be accepted */ - tor_free(policy); /* could leak. */ - } - - /* truncation ports */ - sm = smartlist_new(); - for (i=1; i<2000; i+=2) { - char buf[POLICY_BUF_LEN]; - tor_snprintf(buf, sizeof(buf), "reject *:%d", i); - smartlist_add(sm, tor_strdup(buf)); - } - smartlist_add(sm, tor_strdup("accept *:*")); - policy_str = smartlist_join_strings(sm, ",", 0, NULL); - test_policy_summary_helper( policy_str, - "accept 2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40,42,44," - "46,48,50,52,54,56,58,60,62,64,66,68,70,72,74,76,78,80,82,84,86,88,90," - "92,94,96,98,100,102,104,106,108,110,112,114,116,118,120,122,124,126,128," - "130,132,134,136,138,140,142,144,146,148,150,152,154,156,158,160,162,164," - "166,168,170,172,174,176,178,180,182,184,186,188,190,192,194,196,198,200," - "202,204,206,208,210,212,214,216,218,220,222,224,226,228,230,232,234,236," - "238,240,242,244,246,248,250,252,254,256,258,260,262,264,266,268,270,272," - "274,276,278,280,282,284,286,288,290,292,294,296,298,300,302,304,306,308," - "310,312,314,316,318,320,322,324,326,328,330,332,334,336,338,340,342,344," - "346,348,350,352,354,356,358,360,362,364,366,368,370,372,374,376,378,380," - "382,384,386,388,390,392,394,396,398,400,402,404,406,408,410,412,414,416," - "418,420,422,424,426,428,430,432,434,436,438,440,442,444,446,448,450,452," - "454,456,458,460,462,464,466,468,470,472,474,476,478,480,482,484,486,488," - "490,492,494,496,498,500,502,504,506,508,510,512,514,516,518,520,522"); - - done: - addr_policy_list_free(policy); - addr_policy_list_free(policy2); - addr_policy_list_free(policy3); - addr_policy_list_free(policy4); - addr_policy_list_free(policy5); - addr_policy_list_free(policy6); - addr_policy_list_free(policy7); - tor_free(policy_str); - if (sm) { - SMARTLIST_FOREACH(sm, char *, s, tor_free(s)); - smartlist_free(sm); - } + circuit_build_times_free_timeouts(&initial); + circuit_build_times_free_timeouts(&estimate); + circuit_build_times_free_timeouts(&final); + or_state_free(state); } /** Test encoding and parsing of rendezvous service descriptors. */ @@ -1579,6 +733,34 @@ test_rend_fns(void) tor_free(intro_points_encrypted); } + /* Record odd numbered fake-IPs using ipv6, even numbered fake-IPs + * using ipv4. Since our fake geoip database is the same between + * ipv4 and ipv6, we should get the same result no matter which + * address family we pick for each IP. */ +#define SET_TEST_ADDRESS(i) do { \ + if ((i) & 1) { \ + SET_TEST_IPV6(i); \ + tor_addr_from_in6(&addr, &in6); \ + } else { \ + tor_addr_from_ipv4h(&addr, (uint32_t) i); \ + } \ + } while (0) + + /* Make sure that country ID actually works. */ +#define SET_TEST_IPV6(i) \ + do { \ + set_uint32(in6.s6_addr + 12, htonl((uint32_t) (i))); \ + } while (0) +#define CHECK_COUNTRY(country, val) do { \ + /* test ipv4 country lookup */ \ + test_streq(country, \ + geoip_get_country_name(geoip_get_country_by_ipv4(val))); \ + /* test ipv6 country lookup */ \ + SET_TEST_IPV6(val); \ + test_streq(country, \ + geoip_get_country_name(geoip_get_country_by_ipv6(&in6))); \ + } while (0) + /** Run unit tests for GeoIP code. */ static void test_geoip(void) @@ -1589,7 +771,8 @@ test_geoip(void) const char *bridge_stats_1 = "bridge-stats-end 2010-08-12 13:27:30 (86400 s)\n" "bridge-ips zz=24,xy=8\n" - "bridge-ip-versions v4=16,v6=16\n", + "bridge-ip-versions v4=16,v6=16\n" + "bridge-ip-transports <OR>=24\n", *dirreq_stats_1 = "dirreq-stats-end 2010-08-12 13:27:30 (86400 s)\n" "dirreq-v3-ips ab=8\n" @@ -1653,21 +836,6 @@ test_geoip(void) test_eq(4, geoip_get_n_countries()); memset(&in6, 0, sizeof(in6)); - /* Make sure that country ID actually works. */ -#define SET_TEST_IPV6(i) \ - do { \ - set_uint32(in6.s6_addr + 12, htonl((uint32_t) (i))); \ - } while (0) -#define CHECK_COUNTRY(country, val) do { \ - /* test ipv4 country lookup */ \ - test_streq(country, \ - geoip_get_country_name(geoip_get_country_by_ipv4(val))); \ - /* test ipv6 country lookup */ \ - SET_TEST_IPV6(val); \ - test_streq(country, \ - geoip_get_country_name(geoip_get_country_by_ipv6(&in6))); \ - } while (0) - CHECK_COUNTRY("??", 3); CHECK_COUNTRY("ab", 32); CHECK_COUNTRY("??", 5); @@ -1680,40 +848,25 @@ test_geoip(void) SET_TEST_IPV6(3); test_eq(0, geoip_get_country_by_ipv6(&in6)); -#undef CHECK_COUNTRY - - /* Record odd numbered fake-IPs using ipv6, even numbered fake-IPs - * using ipv4. Since our fake geoip database is the same between - * ipv4 and ipv6, we should get the same result no matter which - * address family we pick for each IP. */ -#define SET_TEST_ADDRESS(i) do { \ - if ((i) & 1) { \ - SET_TEST_IPV6(i); \ - tor_addr_from_in6(&addr, &in6); \ - } else { \ - tor_addr_from_ipv4h(&addr, (uint32_t) i); \ - } \ - } while (0) - get_options_mutable()->BridgeRelay = 1; get_options_mutable()->BridgeRecordUsageByCountry = 1; /* Put 9 observations in AB... */ for (i=32; i < 40; ++i) { SET_TEST_ADDRESS(i); - geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, now-7200); + geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, now-7200); } SET_TEST_ADDRESS(225); - geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, now-7200); + geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, now-7200); /* and 3 observations in XY, several times. */ for (j=0; j < 10; ++j) for (i=52; i < 55; ++i) { SET_TEST_ADDRESS(i); - geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, now-3600); + geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, now-3600); } /* and 17 observations in ZZ... */ for (i=110; i < 127; ++i) { SET_TEST_ADDRESS(i); - geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, now); + geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, now); } geoip_get_client_history(GEOIP_CLIENT_CONNECT, &s, &v); test_assert(s); @@ -1762,7 +915,7 @@ test_geoip(void) /* Start testing dirreq statistics by making sure that we don't collect * dirreq stats without initializing them. */ SET_TEST_ADDRESS(100); - geoip_note_client_seen(GEOIP_CLIENT_NETWORKSTATUS, &addr, now); + geoip_note_client_seen(GEOIP_CLIENT_NETWORKSTATUS, &addr, NULL, now); s = geoip_format_dirreq_stats(now + 86400); test_assert(!s); @@ -1770,7 +923,7 @@ test_geoip(void) * dirreq-stats history string. */ geoip_dirreq_stats_init(now); SET_TEST_ADDRESS(100); - geoip_note_client_seen(GEOIP_CLIENT_NETWORKSTATUS, &addr, now); + geoip_note_client_seen(GEOIP_CLIENT_NETWORKSTATUS, &addr, NULL, now); s = geoip_format_dirreq_stats(now + 86400); test_streq(dirreq_stats_1, s); tor_free(s); @@ -1779,7 +932,7 @@ test_geoip(void) * don't generate a history string. */ geoip_dirreq_stats_term(); SET_TEST_ADDRESS(101); - geoip_note_client_seen(GEOIP_CLIENT_NETWORKSTATUS, &addr, now); + geoip_note_client_seen(GEOIP_CLIENT_NETWORKSTATUS, &addr, NULL, now); s = geoip_format_dirreq_stats(now + 86400); test_assert(!s); @@ -1787,7 +940,7 @@ test_geoip(void) * that we get an all empty history string. */ geoip_dirreq_stats_init(now); SET_TEST_ADDRESS(100); - geoip_note_client_seen(GEOIP_CLIENT_NETWORKSTATUS, &addr, now); + geoip_note_client_seen(GEOIP_CLIENT_NETWORKSTATUS, &addr, NULL, now); geoip_reset_dirreq_stats(now); s = geoip_format_dirreq_stats(now + 86400); test_streq(dirreq_stats_2, s); @@ -1804,6 +957,7 @@ test_geoip(void) geoip_start_dirreq((uint64_t) 1, 1024, DIRREQ_TUNNELED); s = geoip_format_dirreq_stats(now + 86400); test_streq(dirreq_stats_4, s); + tor_free(s); /* Stop collecting directory request statistics and start gathering * entry stats. */ @@ -1814,7 +968,7 @@ test_geoip(void) /* Start testing entry statistics by making sure that we don't collect * anything without initializing entry stats. */ SET_TEST_ADDRESS(100); - geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, now); + geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, now); s = geoip_format_entry_stats(now + 86400); test_assert(!s); @@ -1822,7 +976,7 @@ test_geoip(void) * entry-stats history string. */ geoip_entry_stats_init(now); SET_TEST_ADDRESS(100); - geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, now); + geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, now); s = geoip_format_entry_stats(now + 86400); test_streq(entry_stats_1, s); tor_free(s); @@ -1831,7 +985,7 @@ test_geoip(void) * don't generate a history string. */ geoip_entry_stats_term(); SET_TEST_ADDRESS(101); - geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, now); + geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, now); s = geoip_format_entry_stats(now + 86400); test_assert(!s); @@ -1839,15 +993,12 @@ test_geoip(void) * that we get an all empty history string. */ geoip_entry_stats_init(now); SET_TEST_ADDRESS(100); - geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, now); + geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, now); geoip_reset_entry_stats(now); s = geoip_format_entry_stats(now + 86400); test_streq(entry_stats_2, s); tor_free(s); -#undef SET_TEST_ADDRESS -#undef SET_TEST_IPV6 - /* Stop collecting entry statistics. */ geoip_entry_stats_term(); get_options_mutable()->EntryStatistics = 0; @@ -1857,6 +1008,81 @@ test_geoip(void) tor_free(v); } +static void +test_geoip_with_pt(void) +{ + time_t now = 1281533250; /* 2010-08-11 13:27:30 UTC */ + char *s = NULL; + int i; + tor_addr_t addr; + struct in6_addr in6; + + get_options_mutable()->BridgeRelay = 1; + get_options_mutable()->BridgeRecordUsageByCountry = 1; + + memset(&in6, 0, sizeof(in6)); + + /* No clients seen yet. */ + s = geoip_get_transport_history(); + tor_assert(!s); + + /* 4 connections without a pluggable transport */ + for (i=0; i < 4; ++i) { + SET_TEST_ADDRESS(i); + geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, now-7200); + } + + /* 9 connections with "alpha" */ + for (i=4; i < 13; ++i) { + SET_TEST_ADDRESS(i); + geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, "alpha", now-7200); + } + + /* one connection with "beta" */ + SET_TEST_ADDRESS(13); + geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, "beta", now-7200); + + /* 14 connections with "charlie" */ + for (i=14; i < 28; ++i) { + SET_TEST_ADDRESS(i); + geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, "charlie", now-7200); + } + + /* 131 connections with "ddr" */ + for (i=28; i < 159; ++i) { + SET_TEST_ADDRESS(i); + geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, "ddr", now-7200); + } + + /* 8 connections with "entropy" */ + for (i=159; i < 167; ++i) { + SET_TEST_ADDRESS(i); + geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, "entropy", now-7200); + } + + /* 2 connections from the same IP with two different transports. */ + SET_TEST_ADDRESS(++i); + geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, "fire", now-7200); + geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, "google", now-7200); + + /* Test the transport history string. */ + s = geoip_get_transport_history(); + tor_assert(s); + test_streq(s, "<OR>=8,alpha=16,beta=8,charlie=16,ddr=136," + "entropy=8,fire=8,google=8"); + + /* Stop collecting entry statistics. */ + geoip_entry_stats_term(); + get_options_mutable()->EntryStatistics = 0; + + done: + tor_free(s); +} + +#undef SET_TEST_ADDRESS +#undef SET_TEST_IPV6 +#undef CHECK_COUNTRY + /** Run unit tests for stats code. */ static void test_stats(void) @@ -2047,40 +1273,23 @@ const struct testcase_setup_t legacy_setup = { { #name, legacy_test_helper, TT_FORK, &legacy_setup, test_ ## name } static struct testcase_t test_array[] = { - ENT(buffers), - { "buffer_copy", test_buffer_copy, 0, NULL, NULL }, ENT(onion_handshake), + { "bad_onion_handshake", test_bad_onion_handshake, 0, NULL, NULL }, ENT(onion_queues), #ifdef CURVE25519_ENABLED { "ntor_handshake", test_ntor_handshake, 0, NULL, NULL }, #endif ENT(circuit_timeout), - ENT(policies), ENT(rend_fns), ENT(geoip), + FORK(geoip_with_pt), FORK(stats), END_OF_TESTCASES }; -#define SOCKSENT(name) \ - { #name, test_socks_##name, TT_FORK, &socks_setup, NULL } - -static struct testcase_t socks_tests[] = { - SOCKSENT(4_unsupported_commands), - SOCKSENT(4_supported_commands), - - SOCKSENT(5_unsupported_commands), - SOCKSENT(5_supported_commands), - SOCKSENT(5_no_authenticate), - SOCKSENT(5_auth_before_negotiation), - SOCKSENT(5_authenticate), - SOCKSENT(5_authenticate_with_data), - - END_OF_TESTCASES -}; - extern struct testcase_t addr_tests[]; +extern struct testcase_t buffer_tests[]; extern struct testcase_t crypto_tests[]; extern struct testcase_t container_tests[]; extern struct testcase_t util_tests[]; @@ -2090,22 +1299,52 @@ extern struct testcase_t pt_tests[]; extern struct testcase_t config_tests[]; extern struct testcase_t introduce_tests[]; extern struct testcase_t replaycache_tests[]; +extern struct testcase_t relaycell_tests[]; extern struct testcase_t cell_format_tests[]; +extern struct testcase_t circuitlist_tests[]; +extern struct testcase_t circuitmux_tests[]; +extern struct testcase_t cell_queue_tests[]; +extern struct testcase_t options_tests[]; +extern struct testcase_t socks_tests[]; +extern struct testcase_t extorport_tests[]; +extern struct testcase_t controller_event_tests[]; +extern struct testcase_t logging_tests[]; +extern struct testcase_t hs_tests[]; +extern struct testcase_t nodelist_tests[]; +extern struct testcase_t routerkeys_tests[]; +extern struct testcase_t oom_tests[]; +extern struct testcase_t policy_tests[]; +extern struct testcase_t status_tests[]; static struct testgroup_t testgroups[] = { { "", test_array }, + { "buffer/", buffer_tests }, { "socks/", socks_tests }, { "addr/", addr_tests }, { "crypto/", crypto_tests }, { "container/", container_tests }, { "util/", util_tests }, + { "util/logging/", logging_tests }, { "cellfmt/", cell_format_tests }, + { "cellqueue/", cell_queue_tests }, { "dir/", dir_tests }, { "dir/md/", microdesc_tests }, { "pt/", pt_tests }, { "config/", config_tests }, { "replaycache/", replaycache_tests }, + { "relaycell/", relaycell_tests }, { "introduce/", introduce_tests }, + { "circuitlist/", circuitlist_tests }, + { "circuitmux/", circuitmux_tests }, + { "options/", options_tests }, + { "extorport/", extorport_tests }, + { "control/", controller_event_tests }, + { "hs/", hs_tests }, + { "nodelist/", nodelist_tests }, + { "routerkeys/", routerkeys_tests }, + { "oom/", oom_tests }, + { "policy/" , policy_tests }, + { "status/" , status_tests }, END_OF_GROUPS }; @@ -2118,6 +1357,7 @@ main(int c, const char **v) char *errmsg = NULL; int i, i_out; int loglevel = LOG_ERR; + int accel_crypto = 0; #ifdef USE_DMALLOC { @@ -2140,6 +1380,8 @@ main(int c, const char **v) loglevel = LOG_INFO; } else if (!strcmp(v[i], "--debug")) { loglevel = LOG_DEBUG; + } else if (!strcmp(v[i], "--accel")) { + accel_crypto = 1; } else { v[i_out++] = v[i]; } @@ -2154,7 +1396,7 @@ main(int c, const char **v) } options->command = CMD_RUN_UNITTESTS; - if (crypto_global_init(0, NULL, NULL)) { + if (crypto_global_init(accel_crypto, NULL, NULL)) { printf("Can't initialize crypto subsystem; exiting.\n"); return 1; } diff --git a/src/test/test.h b/src/test/test.h index a89b558e5a..b9e4d5bdb4 100644 --- a/src/test/test.h +++ b/src/test/test.h @@ -36,17 +36,7 @@ #define test_strneq(expr1, expr2) tt_str_op((expr1), !=, (expr2)) #define test_mem_op(expr1, op, expr2, len) \ - tt_assert_test_fmt_type(expr1,expr2,#expr1" "#op" "#expr2, \ - const char *, \ - (memcmp(val1_, val2_, len) op 0), \ - char *, "%s", \ - { size_t printlen = (len)*2+1; \ - print_ = tor_malloc(printlen); \ - base16_encode(print_, printlen, value_, \ - (len)); }, \ - { tor_free(print_); }, \ - TT_EXIT_TEST_FUNCTION \ - ); + tt_mem_op((expr1), op, (expr2), (len)) #define test_memeq(expr1, expr2, len) test_mem_op((expr1), ==, (expr2), len) #define test_memneq(expr1, expr2, len) test_mem_op((expr1), !=, (expr2), len) @@ -69,11 +59,126 @@ tt_assert_test_type(a,b,#a" "#op" "#b,double,(val1_ op val2_),"%f", \ TT_EXIT_TEST_FUNCTION) +#ifdef _MSC_VER +#define U64_PRINTF_TYPE uint64_t +#define I64_PRINTF_TYPE int64_t +#else +#define U64_PRINTF_TYPE unsigned long long +#define I64_PRINTF_TYPE long long +#endif + +#define tt_size_op(a,op,b) \ + tt_assert_test_fmt_type(a,b,#a" "#op" "#b,size_t,(val1_ op val2_), \ + U64_PRINTF_TYPE, U64_FORMAT, \ + {print_ = (U64_PRINTF_TYPE) value_;}, {}, TT_EXIT_TEST_FUNCTION) + +#define tt_u64_op(a,op,b) \ + tt_assert_test_fmt_type(a,b,#a" "#op" "#b,uint64_t,(val1_ op val2_), \ + U64_PRINTF_TYPE, U64_FORMAT, \ + {print_ = (U64_PRINTF_TYPE) value_;}, {}, TT_EXIT_TEST_FUNCTION) + +#define tt_i64_op(a,op,b) \ + tt_assert_test_fmt_type(a,b,#a" "#op" "#b,int64_t,(val1_ op val2_), \ + I64_PRINTF_TYPE, I64_FORMAT, \ + {print_ = (I64_PRINTF_TYPE) value_;}, {}, TT_EXIT_TEST_FUNCTION) + const char *get_fname(const char *name); crypto_pk_t *pk_generate(int idx); void legacy_test_helper(void *data); extern const struct testcase_setup_t legacy_setup; +#define US2_CONCAT_2__(a, b) a ## __ ## b +#define US_CONCAT_2__(a, b) a ## _ ## b +#define US_CONCAT_3__(a, b, c) a ## _ ## b ## _ ## c +#define US_CONCAT_2_(a, b) US_CONCAT_2__(a, b) +#define US_CONCAT_3_(a, b, c) US_CONCAT_3__(a, b, c) + +/* + * These macros are helpful for streamlining the authorship of several test + * cases that use mocks. + * + * The pattern is as follows. + * * Declare a top level namespace: + * #define NS_MODULE foo + * + * * For each test case you want to write, create a new submodule in the + * namespace. All mocks and other information should belong to a single + * submodule to avoid interference with other test cases. + * You can simply name the submodule after the function in the module you + * are testing: + * #define NS_SUBMODULE some_function + * or, if you're wanting to write several tests against the same function, + * ie., you are testing an aspect of that function, you can use: + * #define NS_SUBMODULE ASPECT(some_function, behavior) + * + * * Declare all the mocks you will use. The NS_DECL macro serves to declare + * the mock in the current namespace (defined by NS_MODULE and NS_SUBMODULE). + * It behaves like MOCK_DECL: + * NS_DECL(int, dependent_function, (void *)); + * Here, dependent_function must be declared and implemented with the + * MOCK_DECL and MOCK_IMPL macros. The NS_DECL macro also defines an integer + * global for use for tracking how many times a mock was called, and can be + * accessed by CALLED(mock_name). For example, you might put + * CALLED(dependent_function)++; + * in your mock body. + * + * * Define a function called NS(main) that will contain the body of the + * test case. The NS macro can be used to reference a name in the current + * namespace. + * + * * In NS(main), indicate that a mock function in the current namespace, + * declared with NS_DECL is to override that in the global namespace, + * with the NS_MOCK macro: + * NS_MOCK(dependent_function) + * Unmock with: + * NS_UNMOCK(dependent_function) + * + * * Define the mocks with the NS macro, eg., + * int + * NS(dependent_function)(void *) + * { + * CALLED(dependent_function)++; + * } + * + * * In the struct testcase_t array, you can use the TEST_CASE and + * TEST_CASE_ASPECT macros to define the cases without having to do so + * explicitly nor without having to reset NS_SUBMODULE, eg., + * struct testcase_t foo_tests[] = { + * TEST_CASE_ASPECT(some_function, behavior), + * ... + * END_OF_TESTCASES + * which will define a test case named "some_function__behavior". + */ + +#define NAME_TEST_(name) #name +#define NAME_TEST(name) NAME_TEST_(name) +#define ASPECT(test_module, test_name) US2_CONCAT_2__(test_module, test_name) +#define TEST_CASE(function) \ + { \ + NAME_TEST(function), \ + NS_FULL(NS_MODULE, function, test_main), \ + TT_FORK, \ + NULL, \ + NULL, \ + } +#define TEST_CASE_ASPECT(function, aspect) \ + { \ + NAME_TEST(ASPECT(function, aspect)), \ + NS_FULL(NS_MODULE, ASPECT(function, aspect), test_main), \ + TT_FORK, \ + NULL, \ + NULL, \ + } + +#define NS(name) US_CONCAT_3_(NS_MODULE, NS_SUBMODULE, name) +#define NS_FULL(module, submodule, name) US_CONCAT_3_(module, submodule, name) + +#define CALLED(mock_name) US_CONCAT_2_(NS(mock_name), called) +#define NS_DECL(retval, mock_fn, args) \ + static retval NS(mock_fn) args; int CALLED(mock_fn) = 0 +#define NS_MOCK(name) MOCK(name, NS(name)) +#define NS_UNMOCK(name) UNMOCK(name) + #endif diff --git a/src/test/test_addr.c b/src/test/test_addr.c index fec85a4696..50011e606b 100644 --- a/src/test/test_addr.c +++ b/src/test/test_addr.c @@ -44,6 +44,10 @@ test_addr_basic(void) test_eq(u32, 0x7f000001u); test_eq(u16, 0); tor_free(cp); + + test_assert(addr_port_lookup(LOG_WARN, "localhost:3", &cp, &u32, NULL)); + tor_free(cp); + test_eq(0, addr_mask_get_bits(0x0u)); test_eq(32, addr_mask_get_bits(0xFFFFFFFFu)); test_eq(16, addr_mask_get_bits(0xFFFF0000u)); @@ -69,7 +73,7 @@ test_addr_basic(void) } done: - ; + tor_free(cp); } #define test_op_ip6_(a,op,b,e1,e2) \ @@ -217,11 +221,12 @@ test_addr_ip6_helpers(void) /* ==== Converting to and from sockaddr_t. */ sin = (struct sockaddr_in *)&sa_storage; sin->sin_family = AF_INET; - sin->sin_port = 9090; + sin->sin_port = htons(9090); sin->sin_addr.s_addr = htonl(0x7f7f0102); /*127.127.1.2*/ - tor_addr_from_sockaddr(&t1, (struct sockaddr *)sin, NULL); + tor_addr_from_sockaddr(&t1, (struct sockaddr *)sin, &port1); test_eq(tor_addr_family(&t1), AF_INET); test_eq(tor_addr_to_ipv4h(&t1), 0x7f7f0102); + tt_int_op(port1, ==, 9090); memset(&sa_storage, 0, sizeof(sa_storage)); test_eq(sizeof(struct sockaddr_in), @@ -235,8 +240,9 @@ test_addr_ip6_helpers(void) sin6->sin6_family = AF_INET6; sin6->sin6_port = htons(7070); sin6->sin6_addr.s6_addr[0] = 128; - tor_addr_from_sockaddr(&t1, (struct sockaddr *)sin6, NULL); + tor_addr_from_sockaddr(&t1, (struct sockaddr *)sin6, &port1); test_eq(tor_addr_family(&t1), AF_INET6); + tt_int_op(port1, ==, 7070); p1 = tor_addr_to_str(buf, &t1, sizeof(buf), 0); test_streq(p1, "8000::"); @@ -340,6 +346,9 @@ test_addr_ip6_helpers(void) test_pton6_bad("a:::b:c"); test_pton6_bad(":::a:b:c"); test_pton6_bad("a:b:c:::"); + test_pton6_bad("1.2.3.4"); + test_pton6_bad(":1.2.3.4"); + test_pton6_bad(".2.3.4"); /* test internal checking */ test_external_ip("fbff:ffff::2:7", 0); @@ -396,7 +405,6 @@ test_addr_ip6_helpers(void) test_internal_ip("::ffff:169.254.0.0", 0); test_internal_ip("::ffff:169.254.255.255", 0); test_external_ip("::ffff:169.255.0.0", 0); - test_assert(is_internal_IP(0x7f000001, 0)); /* tor_addr_compare(tor_addr_t x2) */ test_addr_compare("ffff::", ==, "ffff::0"); @@ -464,6 +472,9 @@ test_addr_ip6_helpers(void) test_eq(0, i); i = tor_addr_parse_PTR_name(&t1, "Foobar.baz", AF_UNSPEC, 1); test_eq(0, i); + i = tor_addr_parse_PTR_name(&t1, "9999999999999999999999999999.in-addr.arpa", + AF_UNSPEC, 1); + test_eq(-1, i); i = tor_addr_parse_PTR_name(&t1, "1.0.168.192.in-addr.arpa", AF_UNSPEC, 1); test_eq(1, i); @@ -735,42 +746,89 @@ test_addr_parse(void) /* Correct call. */ r= tor_addr_port_parse(LOG_DEBUG, "192.0.2.1:1234", - &addr, &port); + &addr, &port, -1); test_assert(r == 0); tor_addr_to_str(buf, &addr, sizeof(buf), 0); test_streq(buf, "192.0.2.1"); test_eq(port, 1234); + r= tor_addr_port_parse(LOG_DEBUG, + "[::1]:1234", + &addr, &port, -1); + test_assert(r == 0); + tor_addr_to_str(buf, &addr, sizeof(buf), 0); + test_streq(buf, "::1"); + test_eq(port, 1234); + /* Domain name. */ r= tor_addr_port_parse(LOG_DEBUG, "torproject.org:1234", - &addr, &port); + &addr, &port, -1); test_assert(r == -1); /* Only IP. */ r= tor_addr_port_parse(LOG_DEBUG, "192.0.2.2", - &addr, &port); + &addr, &port, -1); + test_assert(r == -1); + + r= tor_addr_port_parse(LOG_DEBUG, + "192.0.2.2", + &addr, &port, 200); + test_assert(r == 0); + tt_int_op(port,==,200); + + r= tor_addr_port_parse(LOG_DEBUG, + "[::1]", + &addr, &port, -1); test_assert(r == -1); + r= tor_addr_port_parse(LOG_DEBUG, + "[::1]", + &addr, &port, 400); + test_assert(r == 0); + tt_int_op(port,==,400); + /* Bad port. */ r= tor_addr_port_parse(LOG_DEBUG, "192.0.2.2:66666", - &addr, &port); + &addr, &port, -1); + test_assert(r == -1); + r= tor_addr_port_parse(LOG_DEBUG, + "192.0.2.2:66666", + &addr, &port, 200); test_assert(r == -1); /* Only domain name */ r= tor_addr_port_parse(LOG_DEBUG, "torproject.org", - &addr, &port); + &addr, &port, -1); + test_assert(r == -1); + r= tor_addr_port_parse(LOG_DEBUG, + "torproject.org", + &addr, &port, 200); test_assert(r == -1); /* Bad IP address */ r= tor_addr_port_parse(LOG_DEBUG, "192.0.2:1234", - &addr, &port); + &addr, &port, -1); test_assert(r == -1); + /* Make sure that the default port has lower priority than the real + one */ + r= tor_addr_port_parse(LOG_DEBUG, + "192.0.2.2:1337", + &addr, &port, 200); + test_assert(r == 0); + tt_int_op(port,==,1337); + + r= tor_addr_port_parse(LOG_DEBUG, + "[::1]:1369", + &addr, &port, 200); + test_assert(r == 0); + tt_int_op(port,==,1369); + done: ; } @@ -844,6 +902,90 @@ test_virtaddrmap(void *data) } static void +test_addr_localname(void *arg) +{ + (void)arg; + tt_assert(tor_addr_hostname_is_local("localhost")); + tt_assert(tor_addr_hostname_is_local("LOCALHOST")); + tt_assert(tor_addr_hostname_is_local("LocalHost")); + tt_assert(tor_addr_hostname_is_local("local")); + tt_assert(tor_addr_hostname_is_local("LOCAL")); + tt_assert(tor_addr_hostname_is_local("here.now.local")); + tt_assert(tor_addr_hostname_is_local("here.now.LOCAL")); + + tt_assert(!tor_addr_hostname_is_local(" localhost")); + tt_assert(!tor_addr_hostname_is_local("www.torproject.org")); + done: + ; +} + +static void +test_addr_dup_ip(void *arg) +{ + char *v = NULL; + (void)arg; +#define CHECK(ip, s) do { \ + v = tor_dup_ip(ip); \ + tt_str_op(v,==,(s)); \ + tor_free(v); \ + } while (0) + + CHECK(0xffffffff, "255.255.255.255"); + CHECK(0x00000000, "0.0.0.0"); + CHECK(0x7f000001, "127.0.0.1"); + CHECK(0x01020304, "1.2.3.4"); + +#undef CHECK + done: + tor_free(v); +} + +static void +test_addr_sockaddr_to_str(void *arg) +{ + char *v = NULL; + struct sockaddr_in sin; + struct sockaddr_in6 sin6; + struct sockaddr_storage ss; +#ifdef HAVE_SYS_UN_H + struct sockaddr_un s_un; +#endif +#define CHECK(sa, s) do { \ + v = tor_sockaddr_to_str((const struct sockaddr*) &(sa)); \ + tt_str_op(v,==,(s)); \ + tor_free(v); \ + } while (0) + (void)arg; + + memset(&ss,0,sizeof(ss)); + ss.ss_family = AF_UNSPEC; + CHECK(ss, "unspec"); + + memset(&sin,0,sizeof(sin)); + sin.sin_family = AF_INET; + sin.sin_addr.s_addr = htonl(0x7f808001); + sin.sin_port = htons(1234); + CHECK(sin, "127.128.128.1:1234"); + +#ifdef HAVE_SYS_UN_H + memset(&s_un,0,sizeof(s_un)); + s_un.sun_family = AF_UNIX; + strlcpy(s_un.sun_path, "/here/is/a/path", sizeof(s_un.sun_path)); + CHECK(s_un, "unix:/here/is/a/path"); +#endif + + memset(&sin6,0,sizeof(sin6)); + sin6.sin6_family = AF_INET6; + memcpy(sin6.sin6_addr.s6_addr, "\x20\x00\x00\x00\x00\x00\x00\x00" + "\x00\x1a\x2b\x3c\x4d\x5e\x00\x01", 16); + sin6.sin6_port = htons(1234); + CHECK(sin6, "[2000::1a:2b3c:4d5e:1]:1234"); + + done: + tor_free(v); +} + +static void test_addr_is_loopback(void *data) { static const struct loopback_item { @@ -878,6 +1020,32 @@ test_addr_is_loopback(void *data) ; } +static void +test_addr_make_null(void *data) +{ + tor_addr_t *addr = tor_malloc(sizeof(*addr)); + tor_addr_t *zeros = tor_malloc_zero(sizeof(*addr)); + char buf[TOR_ADDR_BUF_LEN]; + (void) data; + /* Ensure that before tor_addr_make_null, addr != 0's */ + memset(addr, 1, sizeof(*addr)); + tt_int_op(memcmp(addr, zeros, sizeof(*addr)), !=, 0); + /* Test with AF == AF_INET */ + zeros->family = AF_INET; + tor_addr_make_null(addr, AF_INET); + tt_int_op(memcmp(addr, zeros, sizeof(*addr)), ==, 0); + tt_str_op(tor_addr_to_str(buf, addr, sizeof(buf), 0), ==, "0.0.0.0"); + /* Test with AF == AF_INET6 */ + memset(addr, 1, sizeof(*addr)); + zeros->family = AF_INET6; + tor_addr_make_null(addr, AF_INET6); + tt_int_op(memcmp(addr, zeros, sizeof(*addr)), ==, 0); + tt_str_op(tor_addr_to_str(buf, addr, sizeof(buf), 0), ==, "::"); + done: + tor_free(addr); + tor_free(zeros); +} + #define ADDR_LEGACY(name) \ { #name, legacy_test_helper, 0, &legacy_setup, test_addr_ ## name } @@ -886,7 +1054,11 @@ struct testcase_t addr_tests[] = { ADDR_LEGACY(ip6_helpers), ADDR_LEGACY(parse), { "virtaddr", test_virtaddrmap, 0, NULL, NULL }, + { "localname", test_addr_localname, 0, NULL, NULL }, + { "dup_ip", test_addr_dup_ip, 0, NULL, NULL }, + { "sockaddr_to_str", test_addr_sockaddr_to_str, 0, NULL, NULL }, { "is_loopback", test_addr_is_loopback, 0, NULL, NULL }, + { "make_null", test_addr_make_null, 0, NULL, NULL }, END_OF_TESTCASES }; diff --git a/src/test/test_bt_cl.c b/src/test/test_bt_cl.c new file mode 100644 index 0000000000..45ae82fb85 --- /dev/null +++ b/src/test/test_bt_cl.c @@ -0,0 +1,109 @@ +/* Copyright (c) 2012-2013, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "orconfig.h" +#include <stdio.h> +#include <stdlib.h> + +#include "or.h" +#include "util.h" +#include "backtrace.h" +#include "torlog.h" + +/* -1: no crash. + * 0: crash with a segmentation fault. + * 1x: crash with an assertion failure. */ +static int crashtype = 0; + +#ifdef __GNUC__ +#define NOINLINE __attribute__((noinline)) +#define NORETURN __attribute__((noreturn)) +#endif + +int crash(int x) NOINLINE; +int oh_what(int x) NOINLINE; +int a_tangled_web(int x) NOINLINE; +int we_weave(int x) NOINLINE; +static void abort_handler(int s) NORETURN; + +int +crash(int x) +{ + if (crashtype == 0) { + *(volatile int *)0 = 0; + } else if (crashtype == 1) { + tor_assert(1 == 0); + } else if (crashtype == -1) { + ; + } + + crashtype *= x; + return crashtype; +} + +int +oh_what(int x) +{ + /* We call crash() twice here, so that the compiler won't try to do a + * tail-call optimization. Only the first call will actually happen, but + * telling the compiler to maybe do the second call will prevent it from + * replacing the first call with a jump. */ + return crash(x) + crash(x*2); +} + +int +a_tangled_web(int x) +{ + return oh_what(x) * 99 + oh_what(x); +} + +int +we_weave(int x) +{ + return a_tangled_web(x) + a_tangled_web(x+1); +} + +static void +abort_handler(int s) +{ + (void)s; + exit(0); +} + +int +main(int argc, char **argv) +{ + log_severity_list_t severity; + + if (argc < 2) { + puts("I take an argument. It should be \"assert\" or \"crash\" or " + "\"none\""); + return 1; + } + if (!strcmp(argv[1], "assert")) { + crashtype = 1; + } else if (!strcmp(argv[1], "crash")) { + crashtype = 0; + } else if (!strcmp(argv[1], "none")) { + crashtype = -1; + } else { + puts("Argument should be \"assert\" or \"crash\" or \"none\""); + return 1; + } + + init_logging(); + set_log_severity_config(LOG_WARN, LOG_ERR, &severity); + add_stream_log(&severity, "stdout", STDOUT_FILENO); + tor_log_update_sigsafe_err_fds(); + + configure_backtrace_handler(NULL); + + signal(SIGABRT, abort_handler); + + printf("%d\n", we_weave(2)); + + clean_up_backtrace_handler(); + + return 0; +} + diff --git a/src/test/test_buffers.c b/src/test/test_buffers.c new file mode 100644 index 0000000000..61ac5bc36b --- /dev/null +++ b/src/test/test_buffers.c @@ -0,0 +1,732 @@ +/* Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2013, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#define BUFFERS_PRIVATE +#include "or.h" +#include "buffers.h" +#include "ext_orport.h" +#include "test.h" + +/** Run unit tests for buffers.c */ +static void +test_buffers_basic(void *arg) +{ + char str[256]; + char str2[256]; + + buf_t *buf = NULL, *buf2 = NULL; + const char *cp; + + int j; + size_t r; + (void) arg; + + /**** + * buf_new + ****/ + if (!(buf = buf_new())) + test_fail(); + + //test_eq(buf_capacity(buf), 4096); + test_eq(buf_datalen(buf), 0); + + /**** + * General pointer frobbing + */ + for (j=0;j<256;++j) { + str[j] = (char)j; + } + write_to_buf(str, 256, buf); + write_to_buf(str, 256, buf); + test_eq(buf_datalen(buf), 512); + fetch_from_buf(str2, 200, buf); + test_memeq(str, str2, 200); + test_eq(buf_datalen(buf), 312); + memset(str2, 0, sizeof(str2)); + + fetch_from_buf(str2, 256, buf); + test_memeq(str+200, str2, 56); + test_memeq(str, str2+56, 200); + test_eq(buf_datalen(buf), 56); + memset(str2, 0, sizeof(str2)); + /* Okay, now we should be 512 bytes into the 4096-byte buffer. If we add + * another 3584 bytes, we hit the end. */ + for (j=0;j<15;++j) { + write_to_buf(str, 256, buf); + } + assert_buf_ok(buf); + test_eq(buf_datalen(buf), 3896); + fetch_from_buf(str2, 56, buf); + test_eq(buf_datalen(buf), 3840); + test_memeq(str+200, str2, 56); + for (j=0;j<15;++j) { + memset(str2, 0, sizeof(str2)); + fetch_from_buf(str2, 256, buf); + test_memeq(str, str2, 256); + } + test_eq(buf_datalen(buf), 0); + buf_free(buf); + buf = NULL; + + /* Okay, now make sure growing can work. */ + buf = buf_new_with_capacity(16); + //test_eq(buf_capacity(buf), 16); + write_to_buf(str+1, 255, buf); + //test_eq(buf_capacity(buf), 256); + fetch_from_buf(str2, 254, buf); + test_memeq(str+1, str2, 254); + //test_eq(buf_capacity(buf), 256); + assert_buf_ok(buf); + write_to_buf(str, 32, buf); + //test_eq(buf_capacity(buf), 256); + assert_buf_ok(buf); + write_to_buf(str, 256, buf); + assert_buf_ok(buf); + //test_eq(buf_capacity(buf), 512); + test_eq(buf_datalen(buf), 33+256); + fetch_from_buf(str2, 33, buf); + test_eq(*str2, str[255]); + + test_memeq(str2+1, str, 32); + //test_eq(buf_capacity(buf), 512); + test_eq(buf_datalen(buf), 256); + fetch_from_buf(str2, 256, buf); + test_memeq(str, str2, 256); + + /* now try shrinking: case 1. */ + buf_free(buf); + buf = buf_new_with_capacity(33668); + for (j=0;j<67;++j) { + write_to_buf(str,255, buf); + } + //test_eq(buf_capacity(buf), 33668); + test_eq(buf_datalen(buf), 17085); + for (j=0; j < 40; ++j) { + fetch_from_buf(str2, 255,buf); + test_memeq(str2, str, 255); + } + + /* now try shrinking: case 2. */ + buf_free(buf); + buf = buf_new_with_capacity(33668); + for (j=0;j<67;++j) { + write_to_buf(str,255, buf); + } + for (j=0; j < 20; ++j) { + fetch_from_buf(str2, 255,buf); + test_memeq(str2, str, 255); + } + for (j=0;j<80;++j) { + write_to_buf(str,255, buf); + } + //test_eq(buf_capacity(buf),33668); + for (j=0; j < 120; ++j) { + fetch_from_buf(str2, 255,buf); + test_memeq(str2, str, 255); + } + + /* Move from buf to buf. */ + buf_free(buf); + buf = buf_new_with_capacity(4096); + buf2 = buf_new_with_capacity(4096); + for (j=0;j<100;++j) + write_to_buf(str, 255, buf); + test_eq(buf_datalen(buf), 25500); + for (j=0;j<100;++j) { + r = 10; + move_buf_to_buf(buf2, buf, &r); + test_eq(r, 0); + } + test_eq(buf_datalen(buf), 24500); + test_eq(buf_datalen(buf2), 1000); + for (j=0;j<3;++j) { + fetch_from_buf(str2, 255, buf2); + test_memeq(str2, str, 255); + } + r = 8192; /*big move*/ + move_buf_to_buf(buf2, buf, &r); + test_eq(r, 0); + r = 30000; /* incomplete move */ + move_buf_to_buf(buf2, buf, &r); + test_eq(r, 13692); + for (j=0;j<97;++j) { + fetch_from_buf(str2, 255, buf2); + test_memeq(str2, str, 255); + } + buf_free(buf); + buf_free(buf2); + buf = buf2 = NULL; + + buf = buf_new_with_capacity(5); + cp = "Testing. This is a moderately long Testing string."; + for (j = 0; cp[j]; j++) + write_to_buf(cp+j, 1, buf); + test_eq(0, buf_find_string_offset(buf, "Testing", 7)); + test_eq(1, buf_find_string_offset(buf, "esting", 6)); + test_eq(1, buf_find_string_offset(buf, "est", 3)); + test_eq(39, buf_find_string_offset(buf, "ing str", 7)); + test_eq(35, buf_find_string_offset(buf, "Testing str", 11)); + test_eq(32, buf_find_string_offset(buf, "ng ", 3)); + test_eq(43, buf_find_string_offset(buf, "string.", 7)); + test_eq(-1, buf_find_string_offset(buf, "shrdlu", 6)); + test_eq(-1, buf_find_string_offset(buf, "Testing thing", 13)); + test_eq(-1, buf_find_string_offset(buf, "ngx", 3)); + buf_free(buf); + buf = NULL; + + /* Try adding a string too long for any freelist. */ + { + char *cp = tor_malloc_zero(65536); + buf = buf_new(); + write_to_buf(cp, 65536, buf); + tor_free(cp); + + tt_int_op(buf_datalen(buf), ==, 65536); + buf_free(buf); + buf = NULL; + } + + done: + if (buf) + buf_free(buf); + if (buf2) + buf_free(buf2); + buf_shrink_freelists(1); +} + +static void +test_buffer_pullup(void *arg) +{ + buf_t *buf; + char *stuff, *tmp; + const char *cp; + size_t sz; + (void)arg; + stuff = tor_malloc(16384); + tmp = tor_malloc(16384); + + /* Note: this test doesn't check the nulterminate argument to buf_pullup, + since nothing actually uses it. We should remove it some time. */ + + buf = buf_new_with_capacity(3000); /* rounds up to next power of 2. */ + + tt_assert(buf); + tt_int_op(buf_get_default_chunk_size(buf), ==, 4096); + + tt_int_op(buf_get_total_allocation(), ==, 0); + + /* There are a bunch of cases for pullup. One is the trivial case. Let's + mess around with an empty buffer. */ + buf_pullup(buf, 16, 1); + buf_get_first_chunk_data(buf, &cp, &sz); + tt_ptr_op(cp, ==, NULL); + tt_uint_op(sz, ==, 0); + + /* Let's make sure nothing got allocated */ + tt_int_op(buf_get_total_allocation(), ==, 0); + + /* Case 1: everything puts into the first chunk with some moving. */ + + /* Let's add some data. */ + crypto_rand(stuff, 16384); + write_to_buf(stuff, 3000, buf); + write_to_buf(stuff+3000, 3000, buf); + buf_get_first_chunk_data(buf, &cp, &sz); + tt_ptr_op(cp, !=, NULL); + tt_int_op(sz, <=, 4096); + + /* Make room for 3000 bytes in the first chunk, so that the pullup-move code + * can get tested. */ + tt_int_op(fetch_from_buf(tmp, 3000, buf), ==, 3000); + test_memeq(tmp, stuff, 3000); + buf_pullup(buf, 2048, 0); + assert_buf_ok(buf); + buf_get_first_chunk_data(buf, &cp, &sz); + tt_ptr_op(cp, !=, NULL); + tt_int_op(sz, >=, 2048); + test_memeq(cp, stuff+3000, 2048); + tt_int_op(3000, ==, buf_datalen(buf)); + tt_int_op(fetch_from_buf(tmp, 3000, buf), ==, 0); + test_memeq(tmp, stuff+3000, 2048); + + buf_free(buf); + + /* Now try the large-chunk case. */ + buf = buf_new_with_capacity(3000); /* rounds up to next power of 2. */ + write_to_buf(stuff, 4000, buf); + write_to_buf(stuff+4000, 4000, buf); + write_to_buf(stuff+8000, 4000, buf); + write_to_buf(stuff+12000, 4000, buf); + tt_int_op(buf_datalen(buf), ==, 16000); + buf_get_first_chunk_data(buf, &cp, &sz); + tt_ptr_op(cp, !=, NULL); + tt_int_op(sz, <=, 4096); + + buf_pullup(buf, 12500, 0); + assert_buf_ok(buf); + buf_get_first_chunk_data(buf, &cp, &sz); + tt_ptr_op(cp, !=, NULL); + tt_int_op(sz, >=, 12500); + test_memeq(cp, stuff, 12500); + tt_int_op(buf_datalen(buf), ==, 16000); + + fetch_from_buf(tmp, 12400, buf); + test_memeq(tmp, stuff, 12400); + tt_int_op(buf_datalen(buf), ==, 3600); + fetch_from_buf(tmp, 3500, buf); + test_memeq(tmp, stuff+12400, 3500); + fetch_from_buf(tmp, 100, buf); + test_memeq(tmp, stuff+15900, 10); + + buf_free(buf); + + /* Make sure that the pull-up-whole-buffer case works */ + buf = buf_new_with_capacity(3000); /* rounds up to next power of 2. */ + write_to_buf(stuff, 4000, buf); + write_to_buf(stuff+4000, 4000, buf); + fetch_from_buf(tmp, 100, buf); /* dump 100 bytes from first chunk */ + buf_pullup(buf, 16000, 0); /* Way too much. */ + assert_buf_ok(buf); + buf_get_first_chunk_data(buf, &cp, &sz); + tt_ptr_op(cp, !=, NULL); + tt_int_op(sz, ==, 7900); + test_memeq(cp, stuff+100, 7900); + + buf_free(buf); + buf = NULL; + + buf_shrink_freelists(1); + + tt_int_op(buf_get_total_allocation(), ==, 0); + done: + buf_free(buf); + buf_shrink_freelists(1); + tor_free(stuff); + tor_free(tmp); +} + +static void +test_buffer_copy(void *arg) +{ + generic_buffer_t *buf=NULL, *buf2=NULL; + const char *s; + size_t len; + char b[256]; + int i; + (void)arg; + + buf = generic_buffer_new(); + tt_assert(buf); + + /* Copy an empty buffer. */ + tt_int_op(0, ==, generic_buffer_set_to_copy(&buf2, buf)); + tt_assert(buf2); + tt_int_op(0, ==, generic_buffer_len(buf2)); + + /* Now try with a short buffer. */ + s = "And now comes an act of enormous enormance!"; + len = strlen(s); + generic_buffer_add(buf, s, len); + tt_int_op(len, ==, generic_buffer_len(buf)); + /* Add junk to buf2 so we can test replacing.*/ + generic_buffer_add(buf2, "BLARG", 5); + tt_int_op(0, ==, generic_buffer_set_to_copy(&buf2, buf)); + tt_int_op(len, ==, generic_buffer_len(buf2)); + generic_buffer_get(buf2, b, len); + test_mem_op(b, ==, s, len); + /* Now free buf2 and retry so we can test allocating */ + generic_buffer_free(buf2); + buf2 = NULL; + tt_int_op(0, ==, generic_buffer_set_to_copy(&buf2, buf)); + tt_int_op(len, ==, generic_buffer_len(buf2)); + generic_buffer_get(buf2, b, len); + test_mem_op(b, ==, s, len); + /* Clear buf for next test */ + generic_buffer_get(buf, b, len); + tt_int_op(generic_buffer_len(buf),==,0); + + /* Okay, now let's try a bigger buffer. */ + s = "Quis autem vel eum iure reprehenderit qui in ea voluptate velit " + "esse quam nihil molestiae consequatur, vel illum qui dolorem eum " + "fugiat quo voluptas nulla pariatur?"; + len = strlen(s); + for (i = 0; i < 256; ++i) { + b[0]=i; + generic_buffer_add(buf, b, 1); + generic_buffer_add(buf, s, len); + } + tt_int_op(0, ==, generic_buffer_set_to_copy(&buf2, buf)); + tt_int_op(generic_buffer_len(buf2), ==, generic_buffer_len(buf)); + for (i = 0; i < 256; ++i) { + generic_buffer_get(buf2, b, len+1); + tt_int_op((unsigned char)b[0],==,i); + test_mem_op(b+1, ==, s, len); + } + + done: + if (buf) + generic_buffer_free(buf); + if (buf2) + generic_buffer_free(buf2); + buf_shrink_freelists(1); +} + +static void +test_buffer_ext_or_cmd(void *arg) +{ + ext_or_cmd_t *cmd = NULL; + generic_buffer_t *buf = generic_buffer_new(); + char *tmp = NULL; + (void) arg; + + /* Empty -- should give "not there. */ + tt_int_op(0, ==, generic_buffer_fetch_ext_or_cmd(buf, &cmd)); + tt_ptr_op(NULL, ==, cmd); + + /* Three bytes: shouldn't work. */ + generic_buffer_add(buf, "\x00\x20\x00", 3); + tt_int_op(0, ==, generic_buffer_fetch_ext_or_cmd(buf, &cmd)); + tt_ptr_op(NULL, ==, cmd); + tt_int_op(3, ==, generic_buffer_len(buf)); + + /* 0020 0000: That's a nil command. It should work. */ + generic_buffer_add(buf, "\x00", 1); + tt_int_op(1, ==, generic_buffer_fetch_ext_or_cmd(buf, &cmd)); + tt_ptr_op(NULL, !=, cmd); + tt_int_op(0x20, ==, cmd->cmd); + tt_int_op(0, ==, cmd->len); + tt_int_op(0, ==, generic_buffer_len(buf)); + ext_or_cmd_free(cmd); + cmd = NULL; + + /* Now try a length-6 command with one byte missing. */ + generic_buffer_add(buf, "\x10\x21\x00\x06""abcde", 9); + tt_int_op(0, ==, generic_buffer_fetch_ext_or_cmd(buf, &cmd)); + tt_ptr_op(NULL, ==, cmd); + generic_buffer_add(buf, "f", 1); + tt_int_op(1, ==, generic_buffer_fetch_ext_or_cmd(buf, &cmd)); + tt_ptr_op(NULL, !=, cmd); + tt_int_op(0x1021, ==, cmd->cmd); + tt_int_op(6, ==, cmd->len); + test_mem_op("abcdef", ==, cmd->body, 6); + tt_int_op(0, ==, generic_buffer_len(buf)); + ext_or_cmd_free(cmd); + cmd = NULL; + + /* Now try a length-10 command with 4 extra bytes. */ + generic_buffer_add(buf, "\xff\xff\x00\x0a" + "loremipsum\x10\x00\xff\xff", 18); + tt_int_op(1, ==, generic_buffer_fetch_ext_or_cmd(buf, &cmd)); + tt_ptr_op(NULL, !=, cmd); + tt_int_op(0xffff, ==, cmd->cmd); + tt_int_op(10, ==, cmd->len); + test_mem_op("loremipsum", ==, cmd->body, 10); + tt_int_op(4, ==, generic_buffer_len(buf)); + ext_or_cmd_free(cmd); + cmd = NULL; + + /* Finally, let's try a maximum-length command. We already have the header + * waiting. */ + tt_int_op(0, ==, generic_buffer_fetch_ext_or_cmd(buf, &cmd)); + tmp = tor_malloc_zero(65535); + generic_buffer_add(buf, tmp, 65535); + tt_int_op(1, ==, generic_buffer_fetch_ext_or_cmd(buf, &cmd)); + tt_ptr_op(NULL, !=, cmd); + tt_int_op(0x1000, ==, cmd->cmd); + tt_int_op(0xffff, ==, cmd->len); + test_mem_op(tmp, ==, cmd->body, 65535); + tt_int_op(0, ==, generic_buffer_len(buf)); + ext_or_cmd_free(cmd); + cmd = NULL; + + done: + ext_or_cmd_free(cmd); + generic_buffer_free(buf); + tor_free(tmp); + buf_shrink_freelists(1); +} + +static void +test_buffer_allocation_tracking(void *arg) +{ + char *junk = tor_malloc(16384); + buf_t *buf1 = NULL, *buf2 = NULL; + int i; + + (void)arg; + + crypto_rand(junk, 16384); + tt_int_op(buf_get_total_allocation(), ==, 0); + + buf1 = buf_new(); + tt_assert(buf1); + buf2 = buf_new(); + tt_assert(buf2); + + tt_int_op(buf_allocation(buf1), ==, 0); + tt_int_op(buf_get_total_allocation(), ==, 0); + + write_to_buf(junk, 4000, buf1); + write_to_buf(junk, 4000, buf1); + write_to_buf(junk, 4000, buf1); + write_to_buf(junk, 4000, buf1); + tt_int_op(buf_allocation(buf1), ==, 16384); + fetch_from_buf(junk, 100, buf1); + tt_int_op(buf_allocation(buf1), ==, 16384); /* still 4 4k chunks */ + + tt_int_op(buf_get_total_allocation(), ==, 16384); + + fetch_from_buf(junk, 4096, buf1); /* drop a 1k chunk... */ + tt_int_op(buf_allocation(buf1), ==, 3*4096); /* now 3 4k chunks */ + +#ifdef ENABLE_BUF_FREELISTS + tt_int_op(buf_get_total_allocation(), ==, 16384); /* that chunk went onto + the freelist. */ +#else + tt_int_op(buf_get_total_allocation(), ==, 12288); /* that chunk was really + freed. */ +#endif + + write_to_buf(junk, 4000, buf2); + tt_int_op(buf_allocation(buf2), ==, 4096); /* another 4k chunk. */ + /* + * If we're using freelists, size stays at 16384 because we just pulled a + * chunk from the freelist. If we aren't, we bounce back up to 16384 by + * allocating a new chunk. + */ + tt_int_op(buf_get_total_allocation(), ==, 16384); + write_to_buf(junk, 4000, buf2); + tt_int_op(buf_allocation(buf2), ==, 8192); /* another 4k chunk. */ + tt_int_op(buf_get_total_allocation(), ==, 5*4096); /* that chunk was new. */ + + /* Make a really huge buffer */ + for (i = 0; i < 1000; ++i) { + write_to_buf(junk, 4000, buf2); + } + tt_int_op(buf_allocation(buf2), >=, 4008000); + tt_int_op(buf_get_total_allocation(), >=, 4008000); + buf_free(buf2); + buf2 = NULL; + + tt_int_op(buf_get_total_allocation(), <, 4008000); + buf_shrink_freelists(1); + tt_int_op(buf_get_total_allocation(), ==, buf_allocation(buf1)); + buf_free(buf1); + buf1 = NULL; + buf_shrink_freelists(1); + tt_int_op(buf_get_total_allocation(), ==, 0); + + done: + buf_free(buf1); + buf_free(buf2); + buf_shrink_freelists(1); + tor_free(junk); +} + +static void +test_buffer_time_tracking(void *arg) +{ + buf_t *buf=NULL, *buf2=NULL; + struct timeval tv0; + const time_t START = 1389288246; + const uint32_t START_MSEC = (uint32_t) ((uint64_t)START * 1000); + int i; + char tmp[4096]; + (void)arg; + + crypto_rand(tmp, sizeof(tmp)); + + tv0.tv_sec = START; + tv0.tv_usec = 0; + + buf = buf_new_with_capacity(3000); /* rounds up to next power of 2. */ + tt_assert(buf); + + /* Empty buffer means the timestamp is 0. */ + tt_int_op(0, ==, buf_get_oldest_chunk_timestamp(buf, START_MSEC)); + tt_int_op(0, ==, buf_get_oldest_chunk_timestamp(buf, START_MSEC+1000)); + + tor_gettimeofday_cache_set(&tv0); + write_to_buf("ABCDEFG", 7, buf); + tt_int_op(1000, ==, buf_get_oldest_chunk_timestamp(buf, START_MSEC+1000)); + + buf2 = buf_copy(buf); + tt_assert(buf2); + tt_int_op(1234, ==, buf_get_oldest_chunk_timestamp(buf2, START_MSEC+1234)); + + /* Now add more bytes; enough to overflow the first chunk. */ + tv0.tv_usec += 123 * 1000; + tor_gettimeofday_cache_set(&tv0); + for (i = 0; i < 600; ++i) + write_to_buf("ABCDEFG", 7, buf); + tt_int_op(4207, ==, buf_datalen(buf)); + + /* The oldest bytes are still in the front. */ + tt_int_op(2000, ==, buf_get_oldest_chunk_timestamp(buf, START_MSEC+2000)); + + /* Once those bytes are dropped, the chunk is still on the first + * timestamp. */ + fetch_from_buf(tmp, 100, buf); + tt_int_op(2000, ==, buf_get_oldest_chunk_timestamp(buf, START_MSEC+2000)); + + /* But once we discard the whole first chunk, we get the data in the second + * chunk. */ + fetch_from_buf(tmp, 4000, buf); + tt_int_op(107, ==, buf_datalen(buf)); + tt_int_op(2000, ==, buf_get_oldest_chunk_timestamp(buf, START_MSEC+2123)); + + /* This time we'll be grabbing a chunk from the freelist, and making sure + its time gets updated */ + tv0.tv_sec += 5; + tv0.tv_usec = 617*1000; + tor_gettimeofday_cache_set(&tv0); + for (i = 0; i < 600; ++i) + write_to_buf("ABCDEFG", 7, buf); + tt_int_op(4307, ==, buf_datalen(buf)); + + tt_int_op(2000, ==, buf_get_oldest_chunk_timestamp(buf, START_MSEC+2123)); + fetch_from_buf(tmp, 4000, buf); + fetch_from_buf(tmp, 306, buf); + tt_int_op(0, ==, buf_get_oldest_chunk_timestamp(buf, START_MSEC+5617)); + tt_int_op(383, ==, buf_get_oldest_chunk_timestamp(buf, START_MSEC+6000)); + + done: + buf_free(buf); + buf_free(buf2); +} + +static void +test_buffers_zlib_impl(int finalize_with_nil) +{ + char *msg = NULL; + char *contents = NULL; + char *expanded = NULL; + buf_t *buf = NULL; + tor_zlib_state_t *zlib_state = NULL; + size_t out_len, in_len; + int done; + + buf = buf_new_with_capacity(128); /* will round up */ + zlib_state = tor_zlib_new(1, ZLIB_METHOD); + + msg = tor_malloc(512); + crypto_rand(msg, 512); + tt_int_op(write_to_buf_zlib(buf, zlib_state, msg, 128, 0), ==, 0); + tt_int_op(write_to_buf_zlib(buf, zlib_state, msg+128, 128, 0), ==, 0); + tt_int_op(write_to_buf_zlib(buf, zlib_state, msg+256, 256, 0), ==, 0); + done = !finalize_with_nil; + tt_int_op(write_to_buf_zlib(buf, zlib_state, "all done", 9, done), ==, 0); + if (finalize_with_nil) { + tt_int_op(write_to_buf_zlib(buf, zlib_state, "", 0, 1), ==, 0); + } + + in_len = buf_datalen(buf); + contents = tor_malloc(in_len); + + tt_int_op(fetch_from_buf(contents, in_len, buf), ==, 0); + + tt_int_op(0, ==, tor_gzip_uncompress(&expanded, &out_len, + contents, in_len, + ZLIB_METHOD, 1, + LOG_WARN)); + + tt_int_op(out_len, >=, 128); + tt_mem_op(msg, ==, expanded, 128); + tt_int_op(out_len, >=, 512); + tt_mem_op(msg, ==, expanded, 512); + tt_int_op(out_len, ==, 512+9); + tt_mem_op("all done", ==, expanded+512, 9); + + done: + buf_free(buf); + tor_zlib_free(zlib_state); + tor_free(contents); + tor_free(expanded); + tor_free(msg); +} + +static void +test_buffers_zlib(void *arg) +{ + (void) arg; + test_buffers_zlib_impl(0); +} +static void +test_buffers_zlib_fin_with_nil(void *arg) +{ + (void) arg; + test_buffers_zlib_impl(1); +} + +static void +test_buffers_zlib_fin_at_chunk_end(void *arg) +{ + char *msg = NULL; + char *contents = NULL; + char *expanded = NULL; + buf_t *buf = NULL; + tor_zlib_state_t *zlib_state = NULL; + size_t out_len, in_len; + size_t sz, headerjunk; + (void) arg; + + buf = buf_new_with_capacity(128); /* will round up */ + sz = buf_get_default_chunk_size(buf); + msg = tor_malloc_zero(sz); + + write_to_buf(msg, 1, buf); + tt_assert(buf->head); + + /* Fill up the chunk so the zlib stuff won't fit in one chunk. */ + tt_uint_op(buf->head->memlen, <, sz); + headerjunk = buf->head->memlen - 7; + write_to_buf(msg, headerjunk-1, buf); + tt_uint_op(buf->head->datalen, ==, headerjunk); + printf("<%u>\n", (unsigned)buf_datalen(buf)); + tt_uint_op(buf_datalen(buf), ==, headerjunk); + /* Write an empty string, with finalization on. */ + zlib_state = tor_zlib_new(1, ZLIB_METHOD); + tt_int_op(write_to_buf_zlib(buf, zlib_state, "", 0, 1), ==, 0); + + printf("<%u>\n", (unsigned)buf_datalen(buf)); + + in_len = buf_datalen(buf); + contents = tor_malloc(in_len); + + tt_int_op(fetch_from_buf(contents, in_len, buf), ==, 0); + + tt_uint_op(in_len, >, headerjunk); + + tt_int_op(0, ==, tor_gzip_uncompress(&expanded, &out_len, + contents + headerjunk, in_len - headerjunk, + ZLIB_METHOD, 1, + LOG_WARN)); + + tt_int_op(out_len, ==, 0); + tt_assert(expanded); + + done: + buf_free(buf); + tor_zlib_free(zlib_state); + tor_free(contents); + tor_free(expanded); + tor_free(msg); +} + +struct testcase_t buffer_tests[] = { + { "basic", test_buffers_basic, TT_FORK, NULL, NULL }, + { "copy", test_buffer_copy, TT_FORK, NULL, NULL }, + { "pullup", test_buffer_pullup, TT_FORK, NULL, NULL }, + { "ext_or_cmd", test_buffer_ext_or_cmd, TT_FORK, NULL, NULL }, + { "allocation_tracking", test_buffer_allocation_tracking, TT_FORK, + NULL, NULL }, + { "time_tracking", test_buffer_time_tracking, TT_FORK, NULL, NULL }, + { "zlib", test_buffers_zlib, TT_FORK, NULL, NULL }, + { "zlib_fin_with_nil", test_buffers_zlib_fin_with_nil, TT_FORK, NULL, NULL }, + { "zlib_fin_at_chunk_end", test_buffers_zlib_fin_at_chunk_end, TT_FORK, + NULL, NULL}, + END_OF_TESTCASES +}; + diff --git a/src/test/test_cell_formats.c b/src/test/test_cell_formats.c index 55d8d0f00f..d7f60680c2 100644 --- a/src/test/test_cell_formats.c +++ b/src/test/test_cell_formats.c @@ -8,7 +8,9 @@ #define CONNECTION_EDGE_PRIVATE #define RELAY_PRIVATE #include "or.h" +#include "channel.h" #include "connection_edge.h" +#include "connection_or.h" #include "onion.h" #include "onion_tap.h" #include "onion_fast.h" @@ -872,6 +874,387 @@ test_cfmt_extended_cells(void *arg) tor_free(mem_op_hex_tmp); } +static void +test_cfmt_resolved_cells(void *arg) +{ + smartlist_t *addrs = smartlist_new(); + relay_header_t rh; + cell_t cell; + int r, errcode; + address_ttl_t *a; + + (void)arg; +#define CLEAR_CELL() do { \ + memset(&cell, 0, sizeof(cell)); \ + memset(&rh, 0, sizeof(rh)); \ + } while (0) +#define CLEAR_ADDRS() do { \ + SMARTLIST_FOREACH(addrs, address_ttl_t *, a, \ + address_ttl_free(a); ); \ + smartlist_clear(addrs); \ + } while (0) +#define SET_CELL(s) do { \ + CLEAR_CELL(); \ + memcpy(cell.payload + RELAY_HEADER_SIZE, (s), sizeof((s))-1); \ + rh.length = sizeof((s))-1; \ + rh.command = RELAY_COMMAND_RESOLVED; \ + errcode = -1; \ + } while (0) + + /* The cell format is one or more answers; each of the form + * type [1 byte---0:hostname, 4:ipv4, 6:ipv6, f0:err-transient, f1:err] + * length [1 byte] + * body [length bytes] + * ttl [4 bytes] + */ + + /* Let's try an empty cell */ + SET_CELL(""); + r = resolved_cell_parse(&cell, &rh, addrs, &errcode); + tt_int_op(errcode, ==, 0); + tt_int_op(r, ==, 0); + tt_int_op(smartlist_len(addrs), ==, 0); + CLEAR_ADDRS(); /* redundant but let's be consistent */ + + /* Cell with one ipv4 addr */ + SET_CELL("\x04\x04" "\x7f\x00\x02\x0a" "\x00\00\x01\x00"); + tt_int_op(rh.length, ==, 10); + r = resolved_cell_parse(&cell, &rh, addrs, &errcode); + tt_int_op(errcode, ==, 0); + tt_int_op(r, ==, 0); + tt_int_op(smartlist_len(addrs), ==, 1); + a = smartlist_get(addrs, 0); + tt_str_op(fmt_addr(&a->addr), ==, "127.0.2.10"); + tt_ptr_op(a->hostname, ==, NULL); + tt_int_op(a->ttl, ==, 256); + CLEAR_ADDRS(); + + /* Cell with one ipv6 addr */ + SET_CELL("\x06\x10" + "\x20\x02\x90\x90\x00\x00\x00\x00" + "\x00\x00\x00\x00\xf0\xf0\xab\xcd" + "\x02\00\x00\x01"); + tt_int_op(rh.length, ==, 22); + r = resolved_cell_parse(&cell, &rh, addrs, &errcode); + tt_int_op(errcode, ==, 0); + tt_int_op(r, ==, 0); + tt_int_op(smartlist_len(addrs), ==, 1); + a = smartlist_get(addrs, 0); + tt_str_op(fmt_addr(&a->addr), ==, "2002:9090::f0f0:abcd"); + tt_ptr_op(a->hostname, ==, NULL); + tt_int_op(a->ttl, ==, 0x2000001); + CLEAR_ADDRS(); + + /* Cell with one hostname */ + SET_CELL("\x00\x11" + "motherbrain.zebes" + "\x00\00\x00\x00"); + tt_int_op(rh.length, ==, 23); + r = resolved_cell_parse(&cell, &rh, addrs, &errcode); + tt_int_op(errcode, ==, 0); + tt_int_op(r, ==, 0); + tt_int_op(smartlist_len(addrs), ==, 1); + a = smartlist_get(addrs, 0); + tt_assert(tor_addr_is_null(&a->addr)); + tt_str_op(a->hostname, ==, "motherbrain.zebes"); + tt_int_op(a->ttl, ==, 0); + CLEAR_ADDRS(); + +#define LONG_NAME \ + "this-hostname-has-255-characters.in-order-to-test-whether-very-long.ho" \ + "stnames-are-accepted.i-am-putting-it-in-a-macro-because-although.this-" \ + "function-is-already-very-full.of-copy-and-pasted-stuff.having-this-app" \ + "ear-more-than-once-would-bother-me-somehow.is" + + tt_int_op(strlen(LONG_NAME), ==, 255); + SET_CELL("\x00\xff" + LONG_NAME + "\x00\01\x00\x00"); + tt_int_op(rh.length, ==, 261); + r = resolved_cell_parse(&cell, &rh, addrs, &errcode); + tt_int_op(errcode, ==, 0); + tt_int_op(r, ==, 0); + tt_int_op(smartlist_len(addrs), ==, 1); + a = smartlist_get(addrs, 0); + tt_assert(tor_addr_is_null(&a->addr)); + tt_str_op(a->hostname, ==, LONG_NAME); + tt_int_op(a->ttl, ==, 65536); + CLEAR_ADDRS(); + + /* Cells with an error */ + SET_CELL("\xf0\x2b" + "I'm sorry, Dave. I'm afraid I can't do that" + "\x00\x11\x22\x33"); + tt_int_op(rh.length, ==, 49); + r = resolved_cell_parse(&cell, &rh, addrs, &errcode); + tt_int_op(errcode, ==, RESOLVED_TYPE_ERROR_TRANSIENT); + tt_int_op(r, ==, 0); + tt_int_op(smartlist_len(addrs), ==, 0); + CLEAR_ADDRS(); + + SET_CELL("\xf1\x40" + "This hostname is too important for me to allow you to resolve it" + "\x00\x00\x00\x00"); + tt_int_op(rh.length, ==, 70); + r = resolved_cell_parse(&cell, &rh, addrs, &errcode); + tt_int_op(errcode, ==, RESOLVED_TYPE_ERROR); + tt_int_op(r, ==, 0); + tt_int_op(smartlist_len(addrs), ==, 0); + CLEAR_ADDRS(); + + /* Cell with an unrecognized type */ + SET_CELL("\xee\x16" + "fault in the AE35 unit" + "\x09\x09\x01\x01"); + tt_int_op(rh.length, ==, 28); + r = resolved_cell_parse(&cell, &rh, addrs, &errcode); + tt_int_op(errcode, ==, 0); + tt_int_op(r, ==, 0); + tt_int_op(smartlist_len(addrs), ==, 0); + CLEAR_ADDRS(); + + /* Cell with one of each */ + SET_CELL(/* unrecognized: */ + "\xee\x16" + "fault in the AE35 unit" + "\x09\x09\x01\x01" + /* error: */ + "\xf0\x2b" + "I'm sorry, Dave. I'm afraid I can't do that" + "\x00\x11\x22\x33" + /* IPv6: */ + "\x06\x10" + "\x20\x02\x90\x90\x00\x00\x00\x00" + "\x00\x00\x00\x00\xf0\xf0\xab\xcd" + "\x02\00\x00\x01" + /* IPv4: */ + "\x04\x04" "\x7f\x00\x02\x0a" "\x00\00\x01\x00" + /* Hostname: */ + "\x00\x11" + "motherbrain.zebes" + "\x00\00\x00\x00" + ); + r = resolved_cell_parse(&cell, &rh, addrs, &errcode); + tt_int_op(errcode, ==, 0); /* no error reported; we got answers */ + tt_int_op(r, ==, 0); + tt_int_op(smartlist_len(addrs), ==, 3); + a = smartlist_get(addrs, 0); + tt_str_op(fmt_addr(&a->addr), ==, "2002:9090::f0f0:abcd"); + tt_ptr_op(a->hostname, ==, NULL); + tt_int_op(a->ttl, ==, 0x2000001); + a = smartlist_get(addrs, 1); + tt_str_op(fmt_addr(&a->addr), ==, "127.0.2.10"); + tt_ptr_op(a->hostname, ==, NULL); + tt_int_op(a->ttl, ==, 256); + a = smartlist_get(addrs, 2); + tt_assert(tor_addr_is_null(&a->addr)); + tt_str_op(a->hostname, ==, "motherbrain.zebes"); + tt_int_op(a->ttl, ==, 0); + CLEAR_ADDRS(); + + /* Cell with several of similar type */ + SET_CELL(/* IPv4 */ + "\x04\x04" "\x7f\x00\x02\x0a" "\x00\00\x01\x00" + "\x04\x04" "\x08\x08\x08\x08" "\x00\00\x01\x05" + "\x04\x04" "\x7f\xb0\x02\xb0" "\x00\01\xff\xff" + /* IPv6 */ + "\x06\x10" + "\x20\x02\x90\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\xca\xfe\xf0\x0d" + "\x00\00\x00\x01" + "\x06\x10" + "\x20\x02\x90\x01\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\xfa\xca\xde" + "\x00\00\x00\x03"); + r = resolved_cell_parse(&cell, &rh, addrs, &errcode); + tt_int_op(errcode, ==, 0); + tt_int_op(r, ==, 0); + tt_int_op(smartlist_len(addrs), ==, 5); + a = smartlist_get(addrs, 0); + tt_str_op(fmt_addr(&a->addr), ==, "127.0.2.10"); + tt_ptr_op(a->hostname, ==, NULL); + tt_int_op(a->ttl, ==, 256); + a = smartlist_get(addrs, 1); + tt_str_op(fmt_addr(&a->addr), ==, "8.8.8.8"); + tt_ptr_op(a->hostname, ==, NULL); + tt_int_op(a->ttl, ==, 261); + a = smartlist_get(addrs, 2); + tt_str_op(fmt_addr(&a->addr), ==, "127.176.2.176"); + tt_ptr_op(a->hostname, ==, NULL); + tt_int_op(a->ttl, ==, 131071); + a = smartlist_get(addrs, 3); + tt_str_op(fmt_addr(&a->addr), ==, "2002:9000::cafe:f00d"); + tt_ptr_op(a->hostname, ==, NULL); + tt_int_op(a->ttl, ==, 1); + a = smartlist_get(addrs, 4); + tt_str_op(fmt_addr(&a->addr), ==, "2002:9001::fa:cade"); + tt_ptr_op(a->hostname, ==, NULL); + tt_int_op(a->ttl, ==, 3); + CLEAR_ADDRS(); + + /* Full cell */ +#define LONG_NAME2 \ + "this-name-has-231-characters.so-that-it-plus-LONG_NAME-can-completely-" \ + "fill-up-the-payload-of-a-cell.its-important-to-check-for-the-full-thin" \ + "g-case.to-avoid-off-by-one-errors.where-full-things-are-misreported-as" \ + ".overflowing-by-one.z" + + tt_int_op(strlen(LONG_NAME2), ==, 231); + SET_CELL("\x00\xff" + LONG_NAME + "\x00\01\x00\x00" + "\x00\xe7" + LONG_NAME2 + "\x00\01\x00\x00"); + tt_int_op(rh.length, ==, RELAY_PAYLOAD_SIZE); + r = resolved_cell_parse(&cell, &rh, addrs, &errcode); + tt_int_op(errcode, ==, 0); + tt_int_op(r, ==, 0); + tt_int_op(smartlist_len(addrs), ==, 2); + a = smartlist_get(addrs, 0); + tt_str_op(a->hostname, ==, LONG_NAME); + a = smartlist_get(addrs, 1); + tt_str_op(a->hostname, ==, LONG_NAME2); + CLEAR_ADDRS(); + + /* BAD CELLS */ + + /* Invalid length on an IPv4 */ + SET_CELL("\x04\x03zzz1234"); + r = resolved_cell_parse(&cell, &rh, addrs, &errcode); + tt_int_op(errcode, ==, 0); + tt_int_op(r, ==, -1); + tt_int_op(smartlist_len(addrs), ==, 0); + SET_CELL("\x04\x04" "\x7f\x00\x02\x0a" "\x00\00\x01\x00" + "\x04\x05zzzzz1234"); + r = resolved_cell_parse(&cell, &rh, addrs, &errcode); + tt_int_op(errcode, ==, 0); + tt_int_op(r, ==, -1); + tt_int_op(smartlist_len(addrs), ==, 0); + + /* Invalid length on an IPv6 */ + SET_CELL("\x06\x03zzz1234"); + r = resolved_cell_parse(&cell, &rh, addrs, &errcode); + tt_int_op(errcode, ==, 0); + tt_int_op(r, ==, -1); + tt_int_op(smartlist_len(addrs), ==, 0); + SET_CELL("\x04\x04" "\x7f\x00\x02\x0a" "\x00\00\x01\x00" + "\x06\x17wwwwwwwwwwwwwwwww1234"); + r = resolved_cell_parse(&cell, &rh, addrs, &errcode); + tt_int_op(errcode, ==, 0); + tt_int_op(r, ==, -1); + tt_int_op(smartlist_len(addrs), ==, 0); + SET_CELL("\x04\x04" "\x7f\x00\x02\x0a" "\x00\00\x01\x00" + "\x06\x10xxxx"); + r = resolved_cell_parse(&cell, &rh, addrs, &errcode); + tt_int_op(errcode, ==, 0); + tt_int_op(r, ==, -1); + tt_int_op(smartlist_len(addrs), ==, 0); + + /* Empty hostname */ + SET_CELL("\x00\x00xxxx"); + r = resolved_cell_parse(&cell, &rh, addrs, &errcode); + tt_int_op(errcode, ==, 0); + tt_int_op(r, ==, -1); + tt_int_op(smartlist_len(addrs), ==, 0); + + /* rh.length out of range */ + CLEAR_CELL(); + rh.length = 499; + r = resolved_cell_parse(&cell, &rh, addrs, &errcode); + tt_int_op(errcode, ==, 0); + tt_int_op(r, ==, -1); + tt_int_op(smartlist_len(addrs), ==, 0); + + /* Item length extends beyond rh.length */ + CLEAR_CELL(); + SET_CELL("\x00\xff" + LONG_NAME + "\x00\01\x00\x00"); + rh.length -= 1; + r = resolved_cell_parse(&cell, &rh, addrs, &errcode); + tt_int_op(r, ==, -1); + tt_int_op(smartlist_len(addrs), ==, 0); + rh.length -= 5; + r = resolved_cell_parse(&cell, &rh, addrs, &errcode); + tt_int_op(r, ==, -1); + tt_int_op(smartlist_len(addrs), ==, 0); + + SET_CELL("\x04\x04" "\x7f\x00\x02\x0a" "\x00\00\x01\x00"); + rh.length -= 1; + r = resolved_cell_parse(&cell, &rh, addrs, &errcode); + tt_int_op(r, ==, -1); + tt_int_op(smartlist_len(addrs), ==, 0); + + SET_CELL("\xee\x10" + "\x20\x02\x90\x01\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\xfa\xca\xde" + "\x00\00\x00\x03"); + rh.length -= 1; + r = resolved_cell_parse(&cell, &rh, addrs, &errcode); + tt_int_op(r, ==, -1); + tt_int_op(smartlist_len(addrs), ==, 0); + + /* Truncated item after first character */ + SET_CELL("\x04"); + r = resolved_cell_parse(&cell, &rh, addrs, &errcode); + tt_int_op(r, ==, -1); + tt_int_op(smartlist_len(addrs), ==, 0); + + SET_CELL("\xee"); + r = resolved_cell_parse(&cell, &rh, addrs, &errcode); + tt_int_op(r, ==, -1); + tt_int_op(smartlist_len(addrs), ==, 0); + + done: + CLEAR_ADDRS(); + CLEAR_CELL(); + smartlist_free(addrs); +#undef CLEAR_ADDRS +#undef CLEAR_CELL +} + +static void +test_cfmt_is_destroy(void *arg) +{ + cell_t cell; + packed_cell_t packed; + circid_t circid = 0; + channel_t *chan; + (void)arg; + + chan = tor_malloc_zero(sizeof(channel_t)); + + memset(&cell, 0xff, sizeof(cell)); + cell.circ_id = 3003; + cell.command = CELL_RELAY; + + cell_pack(&packed, &cell, 0); + chan->wide_circ_ids = 0; + tt_assert(! packed_cell_is_destroy(chan, &packed, &circid)); + tt_int_op(circid, ==, 0); + + cell_pack(&packed, &cell, 1); + chan->wide_circ_ids = 1; + tt_assert(! packed_cell_is_destroy(chan, &packed, &circid)); + tt_int_op(circid, ==, 0); + + cell.command = CELL_DESTROY; + + cell_pack(&packed, &cell, 0); + chan->wide_circ_ids = 0; + tt_assert(packed_cell_is_destroy(chan, &packed, &circid)); + tt_int_op(circid, ==, 3003); + + circid = 0; + cell_pack(&packed, &cell, 1); + chan->wide_circ_ids = 1; + tt_assert(packed_cell_is_destroy(chan, &packed, &circid)); + + done: + tor_free(chan); +} + #define TEST(name, flags) \ { #name, test_cfmt_ ## name, flags, 0, NULL } @@ -883,6 +1266,8 @@ struct testcase_t cell_format_tests[] = { TEST(created_cells, 0), TEST(extend_cells, 0), TEST(extended_cells, 0), + TEST(resolved_cells, 0), + TEST(is_destroy, 0), END_OF_TESTCASES }; diff --git a/src/test/test_cell_queue.c b/src/test/test_cell_queue.c new file mode 100644 index 0000000000..92629823ec --- /dev/null +++ b/src/test/test_cell_queue.c @@ -0,0 +1,158 @@ +/* Copyright (c) 2013, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#define CIRCUITLIST_PRIVATE +#define RELAY_PRIVATE +#include "or.h" +#include "circuitlist.h" +#include "relay.h" +#include "test.h" + +static void +test_cq_manip(void *arg) +{ + packed_cell_t *pc1=NULL, *pc2=NULL, *pc3=NULL, *pc4=NULL, *pc_tmp=NULL; + cell_queue_t cq; + cell_t cell; + (void) arg; + +#ifdef ENABLE_MEMPOOLS + init_cell_pool(); +#endif /* ENABLE_MEMPOOLS */ + + cell_queue_init(&cq); + tt_int_op(cq.n, ==, 0); + + pc1 = packed_cell_new(); + pc2 = packed_cell_new(); + pc3 = packed_cell_new(); + pc4 = packed_cell_new(); + tt_assert(pc1 && pc2 && pc3 && pc4); + + tt_ptr_op(NULL, ==, cell_queue_pop(&cq)); + + /* Add and remove a singleton. */ + cell_queue_append(&cq, pc1); + tt_int_op(cq.n, ==, 1); + tt_ptr_op(pc1, ==, cell_queue_pop(&cq)); + tt_int_op(cq.n, ==, 0); + + /* Add and remove four items */ + cell_queue_append(&cq, pc4); + cell_queue_append(&cq, pc3); + cell_queue_append(&cq, pc2); + cell_queue_append(&cq, pc1); + tt_int_op(cq.n, ==, 4); + tt_ptr_op(pc4, ==, cell_queue_pop(&cq)); + tt_ptr_op(pc3, ==, cell_queue_pop(&cq)); + tt_ptr_op(pc2, ==, cell_queue_pop(&cq)); + tt_ptr_op(pc1, ==, cell_queue_pop(&cq)); + tt_int_op(cq.n, ==, 0); + tt_ptr_op(NULL, ==, cell_queue_pop(&cq)); + + /* Try a packed copy (wide, then narrow, which is a bit of a cheat, since a + * real cell queue has only one type.) */ + memset(&cell, 0, sizeof(cell)); + cell.circ_id = 0x12345678; + cell.command = 10; + strlcpy((char*)cell.payload, "Lorax ipsum gruvvulus thneed amet, snergelly " + "once-ler lerkim, sed do barbaloot tempor gluppitus ut labore et " + "truffula magna aliqua.", + sizeof(cell.payload)); + cell_queue_append_packed_copy(NULL /*circ*/, &cq, 0 /*exitward*/, &cell, + 1 /*wide*/, 0 /*stats*/); + cell.circ_id = 0x2013; + cell_queue_append_packed_copy(NULL /*circ*/, &cq, 0 /*exitward*/, &cell, + 0 /*wide*/, 0 /*stats*/); + tt_int_op(cq.n, ==, 2); + + pc_tmp = cell_queue_pop(&cq); + tt_int_op(cq.n, ==, 1); + tt_ptr_op(pc_tmp, !=, NULL); + test_mem_op(pc_tmp->body, ==, "\x12\x34\x56\x78\x0a", 5); + test_mem_op(pc_tmp->body+5, ==, cell.payload, sizeof(cell.payload)); + packed_cell_free(pc_tmp); + + pc_tmp = cell_queue_pop(&cq); + tt_int_op(cq.n, ==, 0); + tt_ptr_op(pc_tmp, !=, NULL); + test_mem_op(pc_tmp->body, ==, "\x20\x13\x0a", 3); + test_mem_op(pc_tmp->body+3, ==, cell.payload, sizeof(cell.payload)); + packed_cell_free(pc_tmp); + pc_tmp = NULL; + + tt_ptr_op(NULL, ==, cell_queue_pop(&cq)); + + /* Now make sure cell_queue_clear works. */ + cell_queue_append(&cq, pc2); + cell_queue_append(&cq, pc1); + tt_int_op(cq.n, ==, 2); + cell_queue_clear(&cq); + pc2 = pc1 = NULL; /* prevent double-free */ + tt_int_op(cq.n, ==, 0); + + done: + packed_cell_free(pc1); + packed_cell_free(pc2); + packed_cell_free(pc3); + packed_cell_free(pc4); + packed_cell_free(pc_tmp); + + cell_queue_clear(&cq); + +#ifdef ENABLE_MEMPOOLS + free_cell_pool(); +#endif /* ENABLE_MEMPOOLS */ +} + +static void +test_circuit_n_cells(void *arg) +{ + packed_cell_t *pc1=NULL, *pc2=NULL, *pc3=NULL, *pc4=NULL, *pc5=NULL; + origin_circuit_t *origin_c=NULL; + or_circuit_t *or_c=NULL; + + (void)arg; + +#ifdef ENABLE_MEMPOOLS + init_cell_pool(); +#endif /* ENABLE_MEMPOOLS */ + + pc1 = packed_cell_new(); + pc2 = packed_cell_new(); + pc3 = packed_cell_new(); + pc4 = packed_cell_new(); + pc5 = packed_cell_new(); + tt_assert(pc1 && pc2 && pc3 && pc4 && pc5); + + or_c = or_circuit_new(0, NULL); + origin_c = origin_circuit_new(); + origin_c->base_.purpose = CIRCUIT_PURPOSE_C_GENERAL; + + tt_int_op(n_cells_in_circ_queues(TO_CIRCUIT(or_c)), ==, 0); + cell_queue_append(&or_c->p_chan_cells, pc1); + tt_int_op(n_cells_in_circ_queues(TO_CIRCUIT(or_c)), ==, 1); + cell_queue_append(&or_c->base_.n_chan_cells, pc2); + cell_queue_append(&or_c->base_.n_chan_cells, pc3); + tt_int_op(n_cells_in_circ_queues(TO_CIRCUIT(or_c)), ==, 3); + + tt_int_op(n_cells_in_circ_queues(TO_CIRCUIT(origin_c)), ==, 0); + cell_queue_append(&origin_c->base_.n_chan_cells, pc4); + cell_queue_append(&origin_c->base_.n_chan_cells, pc5); + tt_int_op(n_cells_in_circ_queues(TO_CIRCUIT(origin_c)), ==, 2); + + done: + circuit_free(TO_CIRCUIT(or_c)); + circuit_free(TO_CIRCUIT(origin_c)); + +#ifdef ENABLE_MEMPOOLS + free_cell_pool(); +#endif /* ENABLE_MEMPOOLS */ +} + +struct testcase_t cell_queue_tests[] = { + { "basic", test_cq_manip, TT_FORK, NULL, NULL, }, + { "circ_n_cells", test_circuit_n_cells, TT_FORK, NULL, NULL }, + END_OF_TESTCASES +}; + diff --git a/src/test/test_circuitlist.c b/src/test/test_circuitlist.c new file mode 100644 index 0000000000..b19edd1fd4 --- /dev/null +++ b/src/test/test_circuitlist.c @@ -0,0 +1,342 @@ +/* Copyright (c) 2013, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#define TOR_CHANNEL_INTERNAL_ +#define CIRCUITBUILD_PRIVATE +#define CIRCUITLIST_PRIVATE +#include "or.h" +#include "channel.h" +#include "circuitbuild.h" +#include "circuitlist.h" +#include "test.h" + +static channel_t * +new_fake_channel(void) +{ + channel_t *chan = tor_malloc_zero(sizeof(channel_t)); + channel_init(chan); + return chan; +} + +static struct { + int ncalls; + void *cmux; + void *circ; + cell_direction_t dir; +} cam; + +static void +circuitmux_attach_mock(circuitmux_t *cmux, circuit_t *circ, + cell_direction_t dir) +{ + ++cam.ncalls; + cam.cmux = cmux; + cam.circ = circ; + cam.dir = dir; +} + +static struct { + int ncalls; + void *cmux; + void *circ; +} cdm; + +static void +circuitmux_detach_mock(circuitmux_t *cmux, circuit_t *circ) +{ + ++cdm.ncalls; + cdm.cmux = cmux; + cdm.circ = circ; +} + +#define GOT_CMUX_ATTACH(mux_, circ_, dir_) do { \ + tt_int_op(cam.ncalls, ==, 1); \ + tt_ptr_op(cam.cmux, ==, (mux_)); \ + tt_ptr_op(cam.circ, ==, (circ_)); \ + tt_int_op(cam.dir, ==, (dir_)); \ + memset(&cam, 0, sizeof(cam)); \ + } while (0) + +#define GOT_CMUX_DETACH(mux_, circ_) do { \ + tt_int_op(cdm.ncalls, ==, 1); \ + tt_ptr_op(cdm.cmux, ==, (mux_)); \ + tt_ptr_op(cdm.circ, ==, (circ_)); \ + memset(&cdm, 0, sizeof(cdm)); \ + } while (0) + +static void +test_clist_maps(void *arg) +{ + channel_t *ch1 = new_fake_channel(); + channel_t *ch2 = new_fake_channel(); + channel_t *ch3 = new_fake_channel(); + or_circuit_t *or_c1=NULL, *or_c2=NULL; + + (void) arg; + + MOCK(circuitmux_attach_circuit, circuitmux_attach_mock); + MOCK(circuitmux_detach_circuit, circuitmux_detach_mock); + memset(&cam, 0, sizeof(cam)); + memset(&cdm, 0, sizeof(cdm)); + + ch1->cmux = (void*)0x1001; + ch2->cmux = (void*)0x1002; + ch3->cmux = (void*)0x1003; + + or_c1 = or_circuit_new(100, ch2); + tt_assert(or_c1); + GOT_CMUX_ATTACH(ch2->cmux, or_c1, CELL_DIRECTION_IN); + tt_int_op(or_c1->p_circ_id, ==, 100); + tt_ptr_op(or_c1->p_chan, ==, ch2); + + or_c2 = or_circuit_new(100, ch1); + tt_assert(or_c2); + GOT_CMUX_ATTACH(ch1->cmux, or_c2, CELL_DIRECTION_IN); + tt_int_op(or_c2->p_circ_id, ==, 100); + tt_ptr_op(or_c2->p_chan, ==, ch1); + + circuit_set_n_circid_chan(TO_CIRCUIT(or_c1), 200, ch1); + GOT_CMUX_ATTACH(ch1->cmux, or_c1, CELL_DIRECTION_OUT); + + circuit_set_n_circid_chan(TO_CIRCUIT(or_c2), 200, ch2); + GOT_CMUX_ATTACH(ch2->cmux, or_c2, CELL_DIRECTION_OUT); + + tt_ptr_op(circuit_get_by_circid_channel(200, ch1), ==, TO_CIRCUIT(or_c1)); + tt_ptr_op(circuit_get_by_circid_channel(200, ch2), ==, TO_CIRCUIT(or_c2)); + tt_ptr_op(circuit_get_by_circid_channel(100, ch2), ==, TO_CIRCUIT(or_c1)); + /* Try the same thing again, to test the "fast" path. */ + tt_ptr_op(circuit_get_by_circid_channel(100, ch2), ==, TO_CIRCUIT(or_c1)); + tt_assert(circuit_id_in_use_on_channel(100, ch2)); + tt_assert(! circuit_id_in_use_on_channel(101, ch2)); + + /* Try changing the circuitid and channel of that circuit. */ + circuit_set_p_circid_chan(or_c1, 500, ch3); + GOT_CMUX_DETACH(ch2->cmux, TO_CIRCUIT(or_c1)); + GOT_CMUX_ATTACH(ch3->cmux, TO_CIRCUIT(or_c1), CELL_DIRECTION_IN); + tt_ptr_op(circuit_get_by_circid_channel(100, ch2), ==, NULL); + tt_assert(! circuit_id_in_use_on_channel(100, ch2)); + tt_ptr_op(circuit_get_by_circid_channel(500, ch3), ==, TO_CIRCUIT(or_c1)); + + /* Now let's see about destroy handling. */ + tt_assert(! circuit_id_in_use_on_channel(205, ch2)); + tt_assert(circuit_id_in_use_on_channel(200, ch2)); + channel_note_destroy_pending(ch2, 200); + channel_note_destroy_pending(ch2, 205); + channel_note_destroy_pending(ch1, 100); + tt_assert(circuit_id_in_use_on_channel(205, ch2)) + tt_assert(circuit_id_in_use_on_channel(200, ch2)); + tt_assert(circuit_id_in_use_on_channel(100, ch1)); + + tt_assert(TO_CIRCUIT(or_c2)->n_delete_pending != 0); + tt_ptr_op(circuit_get_by_circid_channel(200, ch2), ==, TO_CIRCUIT(or_c2)); + tt_ptr_op(circuit_get_by_circid_channel(100, ch1), ==, TO_CIRCUIT(or_c2)); + + /* Okay, now free ch2 and make sure that the circuit ID is STILL not + * usable, because we haven't declared the destroy to be nonpending */ + tt_int_op(cdm.ncalls, ==, 0); + circuit_free(TO_CIRCUIT(or_c2)); + or_c2 = NULL; /* prevent free */ + tt_int_op(cdm.ncalls, ==, 2); + memset(&cdm, 0, sizeof(cdm)); + tt_assert(circuit_id_in_use_on_channel(200, ch2)); + tt_assert(circuit_id_in_use_on_channel(100, ch1)); + tt_ptr_op(circuit_get_by_circid_channel(200, ch2), ==, NULL); + tt_ptr_op(circuit_get_by_circid_channel(100, ch1), ==, NULL); + + /* Now say that the destroy is nonpending */ + channel_note_destroy_not_pending(ch2, 200); + tt_ptr_op(circuit_get_by_circid_channel(200, ch2), ==, NULL); + channel_note_destroy_not_pending(ch1, 100); + tt_ptr_op(circuit_get_by_circid_channel(100, ch1), ==, NULL); + tt_assert(! circuit_id_in_use_on_channel(200, ch2)); + tt_assert(! circuit_id_in_use_on_channel(100, ch1)); + + done: + if (or_c1) + circuit_free(TO_CIRCUIT(or_c1)); + if (or_c2) + circuit_free(TO_CIRCUIT(or_c2)); + tor_free(ch1); + tor_free(ch2); + tor_free(ch3); + UNMOCK(circuitmux_attach_circuit); + UNMOCK(circuitmux_detach_circuit); +} + +static void +test_rend_token_maps(void *arg) +{ + or_circuit_t *c1, *c2, *c3, *c4; + const uint8_t tok1[REND_TOKEN_LEN] = "The cat can't tell y"; + const uint8_t tok2[REND_TOKEN_LEN] = "ou its name, and it "; + const uint8_t tok3[REND_TOKEN_LEN] = "doesn't really care."; + /* -- Adapted from a quote by Fredrik Lundh. */ + + (void)arg; + (void)tok1; //xxxx + c1 = or_circuit_new(0, NULL); + c2 = or_circuit_new(0, NULL); + c3 = or_circuit_new(0, NULL); + c4 = or_circuit_new(0, NULL); + + /* Make sure we really filled up the tok* variables */ + tt_int_op(tok1[REND_TOKEN_LEN-1], ==, 'y'); + tt_int_op(tok2[REND_TOKEN_LEN-1], ==, ' '); + tt_int_op(tok3[REND_TOKEN_LEN-1], ==, '.'); + + /* No maps; nothing there. */ + tt_ptr_op(NULL, ==, circuit_get_rendezvous(tok1)); + tt_ptr_op(NULL, ==, circuit_get_intro_point(tok1)); + + circuit_set_rendezvous_cookie(c1, tok1); + circuit_set_intro_point_digest(c2, tok2); + + tt_ptr_op(NULL, ==, circuit_get_rendezvous(tok3)); + tt_ptr_op(NULL, ==, circuit_get_intro_point(tok3)); + tt_ptr_op(NULL, ==, circuit_get_rendezvous(tok2)); + tt_ptr_op(NULL, ==, circuit_get_intro_point(tok1)); + + /* Without purpose set, we don't get the circuits */ + tt_ptr_op(NULL, ==, circuit_get_rendezvous(tok1)); + tt_ptr_op(NULL, ==, circuit_get_intro_point(tok2)); + + c1->base_.purpose = CIRCUIT_PURPOSE_REND_POINT_WAITING; + c2->base_.purpose = CIRCUIT_PURPOSE_INTRO_POINT; + + /* Okay, make sure they show up now. */ + tt_ptr_op(c1, ==, circuit_get_rendezvous(tok1)); + tt_ptr_op(c2, ==, circuit_get_intro_point(tok2)); + + /* Two items at the same place with the same token. */ + c3->base_.purpose = CIRCUIT_PURPOSE_REND_POINT_WAITING; + circuit_set_rendezvous_cookie(c3, tok2); + tt_ptr_op(c2, ==, circuit_get_intro_point(tok2)); + tt_ptr_op(c3, ==, circuit_get_rendezvous(tok2)); + + /* Marking a circuit makes it not get returned any more */ + circuit_mark_for_close(TO_CIRCUIT(c1), END_CIRC_REASON_FINISHED); + tt_ptr_op(NULL, ==, circuit_get_rendezvous(tok1)); + circuit_free(TO_CIRCUIT(c1)); + c1 = NULL; + + /* Freeing a circuit makes it not get returned any more. */ + circuit_free(TO_CIRCUIT(c2)); + c2 = NULL; + tt_ptr_op(NULL, ==, circuit_get_intro_point(tok2)); + + /* c3 -- are you still there? */ + tt_ptr_op(c3, ==, circuit_get_rendezvous(tok2)); + /* Change its cookie. This never happens in Tor per se, but hey. */ + c3->base_.purpose = CIRCUIT_PURPOSE_INTRO_POINT; + circuit_set_intro_point_digest(c3, tok3); + + tt_ptr_op(NULL, ==, circuit_get_rendezvous(tok2)); + tt_ptr_op(c3, ==, circuit_get_intro_point(tok3)); + + /* Now replace c3 with c4. */ + c4->base_.purpose = CIRCUIT_PURPOSE_INTRO_POINT; + circuit_set_intro_point_digest(c4, tok3); + + tt_ptr_op(c4, ==, circuit_get_intro_point(tok3)); + + tt_ptr_op(c3->rendinfo, ==, NULL); + tt_ptr_op(c4->rendinfo, !=, NULL); + test_mem_op(c4->rendinfo, ==, tok3, REND_TOKEN_LEN); + + /* Now clear c4's cookie. */ + circuit_set_intro_point_digest(c4, NULL); + tt_ptr_op(c4->rendinfo, ==, NULL); + tt_ptr_op(NULL, ==, circuit_get_intro_point(tok3)); + + done: + if (c1) + circuit_free(TO_CIRCUIT(c1)); + if (c2) + circuit_free(TO_CIRCUIT(c2)); + if (c3) + circuit_free(TO_CIRCUIT(c3)); + if (c4) + circuit_free(TO_CIRCUIT(c4)); +} + +static void +test_pick_circid(void *arg) +{ + bitarray_t *ba = NULL; + channel_t *chan1, *chan2; + circid_t circid; + int i; + (void) arg; + + chan1 = tor_malloc_zero(sizeof(channel_t)); + chan2 = tor_malloc_zero(sizeof(channel_t)); + chan2->wide_circ_ids = 1; + + chan1->circ_id_type = CIRC_ID_TYPE_NEITHER; + tt_int_op(0, ==, get_unique_circ_id_by_chan(chan1)); + + /* Basic tests, with no collisions */ + chan1->circ_id_type = CIRC_ID_TYPE_LOWER; + for (i = 0; i < 50; ++i) { + circid = get_unique_circ_id_by_chan(chan1); + tt_uint_op(0, <, circid); + tt_uint_op(circid, <, (1<<15)); + } + chan1->circ_id_type = CIRC_ID_TYPE_HIGHER; + for (i = 0; i < 50; ++i) { + circid = get_unique_circ_id_by_chan(chan1); + tt_uint_op((1<<15), <, circid); + tt_uint_op(circid, <, (1<<16)); + } + + chan2->circ_id_type = CIRC_ID_TYPE_LOWER; + for (i = 0; i < 50; ++i) { + circid = get_unique_circ_id_by_chan(chan2); + tt_uint_op(0, <, circid); + tt_uint_op(circid, <, (1u<<31)); + } + chan2->circ_id_type = CIRC_ID_TYPE_HIGHER; + for (i = 0; i < 50; ++i) { + circid = get_unique_circ_id_by_chan(chan2); + tt_uint_op((1u<<31), <, circid); + } + + /* Now make sure that we can behave well when we are full up on circuits */ + chan1->circ_id_type = CIRC_ID_TYPE_LOWER; + chan2->circ_id_type = CIRC_ID_TYPE_LOWER; + chan1->wide_circ_ids = chan2->wide_circ_ids = 0; + ba = bitarray_init_zero((1<<15)); + for (i = 0; i < (1<<15); ++i) { + circid = get_unique_circ_id_by_chan(chan1); + if (circid == 0) { + tt_int_op(i, >, (1<<14)); + break; + } + tt_uint_op(circid, <, (1<<15)); + tt_assert(! bitarray_is_set(ba, circid)); + bitarray_set(ba, circid); + channel_mark_circid_unusable(chan1, circid); + } + tt_int_op(i, <, (1<<15)); + /* Make sure that being full on chan1 does not interfere with chan2 */ + for (i = 0; i < 100; ++i) { + circid = get_unique_circ_id_by_chan(chan2); + tt_uint_op(circid, >, 0); + tt_uint_op(circid, <, (1<<15)); + channel_mark_circid_unusable(chan2, circid); + } + + done: + tor_free(chan1); + tor_free(chan2); + bitarray_free(ba); + circuit_free_all(); +} + +struct testcase_t circuitlist_tests[] = { + { "maps", test_clist_maps, TT_FORK, NULL, NULL }, + { "rend_token_maps", test_rend_token_maps, TT_FORK, NULL, NULL }, + { "pick_circid", test_pick_circid, TT_FORK, NULL, NULL }, + END_OF_TESTCASES +}; + diff --git a/src/test/test_circuitmux.c b/src/test/test_circuitmux.c new file mode 100644 index 0000000000..b9c0436ebf --- /dev/null +++ b/src/test/test_circuitmux.c @@ -0,0 +1,88 @@ +/* Copyright (c) 2013, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#define TOR_CHANNEL_INTERNAL_ +#define CIRCUITMUX_PRIVATE +#define RELAY_PRIVATE +#include "or.h" +#include "channel.h" +#include "circuitmux.h" +#include "relay.h" +#include "test.h" + +/* XXXX duplicated function from test_circuitlist.c */ +static channel_t * +new_fake_channel(void) +{ + channel_t *chan = tor_malloc_zero(sizeof(channel_t)); + channel_init(chan); + return chan; +} + +static int +has_queued_writes(channel_t *c) +{ + (void) c; + return 1; +} + +/** Test destroy cell queue with no interference from other queues. */ +static void +test_cmux_destroy_cell_queue(void *arg) +{ + circuitmux_t *cmux = NULL; + channel_t *ch = NULL; + circuit_t *circ = NULL; + cell_queue_t *cq = NULL; + packed_cell_t *pc = NULL; + +#ifdef ENABLE_MEMPOOLS + init_cell_pool(); +#endif /* ENABLE_MEMPOOLS */ + (void) arg; + + cmux = circuitmux_alloc(); + tt_assert(cmux); + ch = new_fake_channel(); + ch->has_queued_writes = has_queued_writes; + ch->wide_circ_ids = 1; + + circ = circuitmux_get_first_active_circuit(cmux, &cq); + tt_assert(!circ); + tt_assert(!cq); + + circuitmux_append_destroy_cell(ch, cmux, 100, 10); + circuitmux_append_destroy_cell(ch, cmux, 190, 6); + circuitmux_append_destroy_cell(ch, cmux, 30, 1); + + tt_int_op(circuitmux_num_cells(cmux), ==, 3); + + circ = circuitmux_get_first_active_circuit(cmux, &cq); + tt_assert(!circ); + tt_assert(cq); + + tt_int_op(cq->n, ==, 3); + + pc = cell_queue_pop(cq); + tt_assert(pc); + test_mem_op(pc->body, ==, "\x00\x00\x00\x64\x04\x0a\x00\x00\x00", 9); + packed_cell_free(pc); + pc = NULL; + + tt_int_op(circuitmux_num_cells(cmux), ==, 2); + + done: + circuitmux_free(cmux); + channel_free(ch); + packed_cell_free(pc); + +#ifdef ENABLE_MEMPOOLS + free_cell_pool(); +#endif /* ENABLE_MEMPOOLS */ +} + +struct testcase_t circuitmux_tests[] = { + { "destroy_cell_queue", test_cmux_destroy_cell_queue, TT_FORK, NULL, NULL }, + END_OF_TESTCASES +}; + diff --git a/src/test/test_cmdline_args.py b/src/test/test_cmdline_args.py new file mode 100755 index 0000000000..55d1cdb805 --- /dev/null +++ b/src/test/test_cmdline_args.py @@ -0,0 +1,292 @@ +#!/usr/bin/python + +import binascii +import hashlib +import os +import re +import shutil +import subprocess +import sys +import tempfile +import unittest + +TOR = "./src/or/tor" +TOP_SRCDIR = "." + +if len(sys.argv) > 1: + TOR = sys.argv[1] + del sys.argv[1] + +if len(sys.argv) > 1: + TOP_SRCDIR = sys.argv[1] + del sys.argv[1] + +class UnexpectedSuccess(Exception): + pass + +class UnexpectedFailure(Exception): + pass + +if sys.version < '3': + def b2s(b): + return b + def s2b(s): + return s + def NamedTemporaryFile(): + return tempfile.NamedTemporaryFile(delete=False) +else: + def b2s(b): + return str(b, 'ascii') + def s2b(s): + return s.encode('ascii') + def NamedTemporaryFile(): + return tempfile.NamedTemporaryFile(mode="w",delete=False,encoding="ascii") + +def contents(fn): + f = open(fn) + try: + return f.read() + finally: + f.close() + +def run_tor(args, failure=False): + p = subprocess.Popen([TOR] + args, stdout=subprocess.PIPE) + output, _ = p.communicate() + result = p.poll() + if result and not failure: + raise UnexpectedFailure() + elif not result and failure: + raise UnexpectedSuccess() + return b2s(output) + +def spaceify_fp(fp): + for i in range(0, len(fp), 4): + yield fp[i:i+4] + +def lines(s): + out = s.split("\n") + if out and out[-1] == '': + del out[-1] + return out + +def strip_log_junk(line): + m = re.match(r'([^\[]+\[[a-z]*\] *)(.*)', line) + if not m: + return ""+line + return m.group(2).strip() + +def randstring(entropy_bytes): + s = os.urandom(entropy_bytes) + return b2s(binascii.b2a_hex(s)) + +def findLineContaining(lines, s): + for ln in lines: + if s in ln: + return True + return False + +class CmdlineTests(unittest.TestCase): + + def test_version(self): + out = run_tor(["--version"]) + self.assertTrue(out.startswith("Tor version ")) + self.assertEqual(len(lines(out)), 1) + + def test_quiet(self): + out = run_tor(["--quiet", "--quumblebluffin", "1"], failure=True) + self.assertEqual(out, "") + + def test_help(self): + out = run_tor(["--help"], failure=False) + out2 = run_tor(["-h"], failure=False) + self.assertTrue(out.startswith("Copyright (c) 2001")) + self.assertTrue(out.endswith( + "tor -f <torrc> [args]\n" + "See man page for options, or https://www.torproject.org/ for documentation.\n")) + self.assertTrue(out == out2) + + def test_hush(self): + torrc = NamedTemporaryFile() + torrc.close() + try: + out = run_tor(["--hush", "-f", torrc.name, + "--quumblebluffin", "1"], failure=True) + finally: + os.unlink(torrc.name) + self.assertEqual(len(lines(out)), 2) + ln = [ strip_log_junk(l) for l in lines(out) ] + self.assertEqual(ln[0], "Failed to parse/validate config: Unknown option 'quumblebluffin'. Failing.") + self.assertEqual(ln[1], "Reading config failed--see warnings above.") + + def test_missing_argument(self): + out = run_tor(["--hush", "--hash-password"], failure=True) + self.assertEqual(len(lines(out)), 2) + ln = [ strip_log_junk(l) for l in lines(out) ] + self.assertEqual(ln[0], "Command-line option '--hash-password' with no value. Failing.") + + def test_hash_password(self): + out = run_tor(["--hash-password", "woodwose"]) + result = lines(out)[-1] + self.assertEqual(result[:3], "16:") + self.assertEqual(len(result), 61) + r = binascii.a2b_hex(result[3:]) + self.assertEqual(len(r), 29) + + salt, how, hashed = r[:8], r[8], r[9:] + self.assertEqual(len(hashed), 20) + if type(how) == type("A"): + how = ord(how) + + count = (16 + (how & 15)) << ((how >> 4) + 6) + stuff = salt + s2b("woodwose") + repetitions = count // len(stuff) + 1 + inp = stuff * repetitions + inp = inp[:count] + + self.assertEqual(hashlib.sha1(inp).digest(), hashed) + + def test_digests(self): + main_c = os.path.join(TOP_SRCDIR, "src", "or", "main.c") + + if os.stat(TOR).st_mtime < os.stat(main_c).st_mtime: + self.skipTest(TOR+" not up to date") + out = run_tor(["--digests"]) + main_line = [ l for l in lines(out) if l.endswith("/main.c") ] + digest, name = main_line[0].split() + f = open(main_c, 'rb') + actual = hashlib.sha1(f.read()).hexdigest() + f.close() + self.assertEqual(digest, actual) + + def test_dump_options(self): + default_torrc = NamedTemporaryFile() + torrc = NamedTemporaryFile() + torrc.write("SocksPort 9999") + torrc.close() + default_torrc.write("SafeLogging 0") + default_torrc.close() + out_sh = out_nb = out_fl = None + opts = [ "-f", torrc.name, + "--defaults-torrc", default_torrc.name ] + try: + out_sh = run_tor(["--dump-config", "short"]+opts) + out_nb = run_tor(["--dump-config", "non-builtin"]+opts) + out_fl = run_tor(["--dump-config", "full"]+opts) + out_nr = run_tor(["--dump-config", "bliznert"]+opts, + failure=True) + + out_verif = run_tor(["--verify-config"]+opts) + finally: + os.unlink(torrc.name) + os.unlink(default_torrc.name) + + self.assertEqual(len(lines(out_sh)), 2) + self.assertTrue(lines(out_sh)[0].startswith("DataDirectory ")) + self.assertEqual(lines(out_sh)[1:], + [ "SocksPort 9999" ]) + + self.assertEqual(len(lines(out_nb)), 2) + self.assertEqual(lines(out_nb), + [ "SafeLogging 0", + "SocksPort 9999" ]) + + out_fl = lines(out_fl) + self.assertTrue(len(out_fl) > 100) + self.assertTrue("SocksPort 9999" in out_fl) + self.assertTrue("SafeLogging 0" in out_fl) + self.assertTrue("ClientOnly 0" in out_fl) + + self.assertTrue(out_verif.endswith("Configuration was valid\n")) + + def test_list_fingerprint(self): + tmpdir = tempfile.mkdtemp(prefix='ttca_') + torrc = NamedTemporaryFile() + torrc.write("ORPort 9999\n") + torrc.write("DataDirectory %s\n"%tmpdir) + torrc.write("Nickname tippi") + torrc.close() + opts = ["-f", torrc.name] + try: + out = run_tor(["--list-fingerprint"]+opts) + fp = contents(os.path.join(tmpdir, "fingerprint")) + finally: + os.unlink(torrc.name) + shutil.rmtree(tmpdir) + + out = lines(out) + lastlog = strip_log_junk(out[-2]) + lastline = out[-1] + fp = fp.strip() + nn_fp = fp.split()[0] + space_fp = " ".join(spaceify_fp(fp.split()[1])) + self.assertEqual(lastlog, + "Your Tor server's identity key fingerprint is '%s'"%fp) + self.assertEqual(lastline, "tippi %s"%space_fp) + self.assertEqual(nn_fp, "tippi") + + def test_list_options(self): + out = lines(run_tor(["--list-torrc-options"])) + self.assertTrue(len(out)>100) + self.assertTrue(out[0] <= 'AccountingMax') + self.assertTrue("UseBridges" in out) + self.assertTrue("SocksPort" in out) + + def test_cmdline_args(self): + default_torrc = NamedTemporaryFile() + torrc = NamedTemporaryFile() + torrc.write("SocksPort 9999\n") + torrc.write("SocksPort 9998\n") + torrc.write("ORPort 9000\n") + torrc.write("ORPort 9001\n") + torrc.write("Nickname eleventeen\n") + torrc.write("ControlPort 9500\n") + torrc.close() + default_torrc.write("") + default_torrc.close() + out_sh = out_nb = out_fl = None + opts = [ "-f", torrc.name, + "--defaults-torrc", default_torrc.name, + "--dump-config", "short" ] + try: + out_1 = run_tor(opts) + out_2 = run_tor(opts+["+ORPort", "9003", + "SocksPort", "9090", + "/ControlPort", + "/TransPort", + "+ExtORPort", "9005"]) + finally: + os.unlink(torrc.name) + os.unlink(default_torrc.name) + + out_1 = [ l for l in lines(out_1) if not l.startswith("DataDir") ] + out_2 = [ l for l in lines(out_2) if not l.startswith("DataDir") ] + + self.assertEqual(out_1, + ["ControlPort 9500", + "Nickname eleventeen", + "ORPort 9000", + "ORPort 9001", + "SocksPort 9999", + "SocksPort 9998"]) + self.assertEqual(out_2, + ["ExtORPort 9005", + "Nickname eleventeen", + "ORPort 9000", + "ORPort 9001", + "ORPort 9003", + "SocksPort 9090"]) + + def test_missing_torrc(self): + fname = "nonexistent_file_"+randstring(8) + out = run_tor(["-f", fname, "--verify-config"], failure=True) + ln = [ strip_log_junk(l) for l in lines(out) ] + self.assertTrue("Unable to open configuration file" in ln[-2]) + self.assertTrue("Reading config failed" in ln[-1]) + + out = run_tor(["-f", fname, "--verify-config", "--ignore-missing-torrc"]) + ln = [ strip_log_junk(l) for l in lines(out) ] + self.assertTrue(findLineContaining(ln, ", using reasonable defaults")) + self.assertTrue("Configuration was valid" in ln[-1]) + +if __name__ == '__main__': + unittest.main() diff --git a/src/test/test_config.c b/src/test/test_config.c index e20fe73295..94ac4dca13 100644 --- a/src/test/test_config.c +++ b/src/test/test_config.c @@ -4,12 +4,16 @@ /* See LICENSE for licensing information */ #include "orconfig.h" + +#define CONFIG_PRIVATE #include "or.h" #include "addressmap.h" #include "config.h" #include "confparse.h" #include "connection_edge.h" #include "test.h" +#include "util.h" +#include "address.h" static void test_config_addressmap(void *arg) @@ -120,6 +124,7 @@ test_config_addressmap(void *arg) test_assert(!addressmap_rewrite(address, sizeof(address), &expires, NULL)); /* Test top-level-domain matching a bit harder */ + config_free_lines(get_options_mutable()->AddressMap); addressmap_clear_configured(); strlcpy(buf, "MapAddress *.com *.torserver.exit\n" "MapAddress *.torproject.org 1.1.1.1\n" @@ -149,6 +154,7 @@ test_config_addressmap(void *arg) test_streq(address, "2.2.2.2"); /* We don't support '*' as a mapping directive */ + config_free_lines(get_options_mutable()->AddressMap); addressmap_clear_configured(); strlcpy(buf, "MapAddress * *.torserver.exit\n", sizeof(buf)); config_get_lines(buf, &(get_options_mutable()->AddressMap), 0); @@ -166,7 +172,421 @@ test_config_addressmap(void *arg) #undef addressmap_rewrite done: - ; + config_free_lines(get_options_mutable()->AddressMap); + get_options_mutable()->AddressMap = NULL; +} + +static int +is_private_dir(const char* path) +{ + struct stat st; + int r = stat(path, &st); + if (r) { + return 0; + } +#if !defined (_WIN32) || defined (WINCE) + if ((st.st_mode & (S_IFDIR | 0777)) != (S_IFDIR | 0700)) { + return 0; + } +#endif + return 1; +} + +static void +test_config_check_or_create_data_subdir(void *arg) +{ + or_options_t *options = get_options_mutable(); + char *datadir; + const char *subdir = "test_stats"; + char *subpath; + struct stat st; + int r; +#if !defined (_WIN32) || defined (WINCE) + unsigned group_permission; +#endif + (void)arg; + + tor_free(options->DataDirectory); + datadir = options->DataDirectory = tor_strdup(get_fname("datadir-0")); + subpath = get_datadir_fname(subdir); + +#if defined (_WIN32) && !defined (WINCE) + tt_int_op(mkdir(options->DataDirectory), ==, 0); +#else + tt_int_op(mkdir(options->DataDirectory, 0700), ==, 0); +#endif + + r = stat(subpath, &st); + + // The subdirectory shouldn't exist yet, + // but should be created by the call to check_or_create_data_subdir. + test_assert(r && (errno == ENOENT)); + test_assert(!check_or_create_data_subdir(subdir)); + test_assert(is_private_dir(subpath)); + + // The check should return 0, if the directory already exists + // and is private to the user. + test_assert(!check_or_create_data_subdir(subdir)); + + r = stat(subpath, &st); + if (r) { + tt_abort_perror("stat"); + } + +#if !defined (_WIN32) || defined (WINCE) + group_permission = st.st_mode | 0070; + r = chmod(subpath, group_permission); + + if (r) { + tt_abort_perror("chmod"); + } + + // If the directory exists, but its mode is too permissive + // a call to check_or_create_data_subdir should reset the mode. + test_assert(!is_private_dir(subpath)); + test_assert(!check_or_create_data_subdir(subdir)); + test_assert(is_private_dir(subpath)); +#endif + + done: + rmdir(subpath); + tor_free(datadir); + tor_free(subpath); +} + +static void +test_config_write_to_data_subdir(void *arg) +{ + or_options_t* options = get_options_mutable(); + char *datadir; + char *cp = NULL; + const char* subdir = "test_stats"; + const char* fname = "test_file"; + const char* str = + "Lorem ipsum dolor sit amet, consetetur sadipscing\n" + "elitr, sed diam nonumy eirmod\n" + "tempor invidunt ut labore et dolore magna aliquyam\n" + "erat, sed diam voluptua.\n" + "At vero eos et accusam et justo duo dolores et ea\n" + "rebum. Stet clita kasd gubergren,\n" + "no sea takimata sanctus est Lorem ipsum dolor sit amet.\n" + "Lorem ipsum dolor sit amet,\n" + "consetetur sadipscing elitr, sed diam nonumy eirmod\n" + "tempor invidunt ut labore et dolore\n" + "magna aliquyam erat, sed diam voluptua. At vero eos et\n" + "accusam et justo duo dolores et\n" + "ea rebum. Stet clita kasd gubergren, no sea takimata\n" + "sanctus est Lorem ipsum dolor sit amet."; + char* filepath = NULL; + (void)arg; + + tor_free(options->DataDirectory); + datadir = options->DataDirectory = tor_strdup(get_fname("datadir-1")); + filepath = get_datadir_fname2(subdir, fname); + +#if defined (_WIN32) && !defined (WINCE) + tt_int_op(mkdir(options->DataDirectory), ==, 0); +#else + tt_int_op(mkdir(options->DataDirectory, 0700), ==, 0); +#endif + + // Write attempt shoudl fail, if subdirectory doesn't exist. + test_assert(write_to_data_subdir(subdir, fname, str, NULL)); + test_assert(! check_or_create_data_subdir(subdir)); + + // Content of file after write attempt should be + // equal to the original string. + test_assert(!write_to_data_subdir(subdir, fname, str, NULL)); + cp = read_file_to_str(filepath, 0, NULL); + test_streq(cp, str); + tor_free(cp); + + // A second write operation should overwrite the old content. + test_assert(!write_to_data_subdir(subdir, fname, str, NULL)); + cp = read_file_to_str(filepath, 0, NULL); + test_streq(cp, str); + tor_free(cp); + + done: + (void) unlink(filepath); + rmdir(options->DataDirectory); + tor_free(datadir); + tor_free(filepath); + tor_free(cp); +} + +/* Test helper function: Make sure that a bridge line gets parsed + * properly. Also make sure that the resulting bridge_line_t structure + * has its fields set correctly. */ +static void +good_bridge_line_test(const char *string, const char *test_addrport, + const char *test_digest, const char *test_transport, + const smartlist_t *test_socks_args) +{ + char *tmp = NULL; + bridge_line_t *bridge_line = parse_bridge_line(string); + test_assert(bridge_line); + + /* test addrport */ + tmp = tor_strdup(fmt_addrport(&bridge_line->addr, bridge_line->port)); + test_streq(test_addrport, tmp); + tor_free(tmp); + + /* If we were asked to validate a digest, but we did not get a + digest after parsing, we failed. */ + if (test_digest && tor_digest_is_zero(bridge_line->digest)) + test_assert(0); + + /* If we were not asked to validate a digest, and we got a digest + after parsing, we failed again. */ + if (!test_digest && !tor_digest_is_zero(bridge_line->digest)) + test_assert(0); + + /* If we were asked to validate a digest, and we got a digest after + parsing, make sure it's correct. */ + if (test_digest) { + tmp = tor_strdup(hex_str(bridge_line->digest, DIGEST_LEN)); + tor_strlower(tmp); + test_streq(test_digest, tmp); + tor_free(tmp); + } + + /* If we were asked to validate a transport name, make sure tha it + matches with the transport name that was parsed. */ + if (test_transport && !bridge_line->transport_name) + test_assert(0); + if (!test_transport && bridge_line->transport_name) + test_assert(0); + if (test_transport) + test_streq(test_transport, bridge_line->transport_name); + + /* Validate the SOCKS argument smartlist. */ + if (test_socks_args && !bridge_line->socks_args) + test_assert(0); + if (!test_socks_args && bridge_line->socks_args) + test_assert(0); + if (test_socks_args) + test_assert(smartlist_strings_eq(test_socks_args, + bridge_line->socks_args)); + + done: + tor_free(tmp); + bridge_line_free(bridge_line); +} + +/* Test helper function: Make sure that a bridge line is + * unparseable. */ +static void +bad_bridge_line_test(const char *string) +{ + bridge_line_t *bridge_line = parse_bridge_line(string); + if (bridge_line) + TT_FAIL(("%s was supposed to fail, but it didn't.", string)); + test_assert(!bridge_line); + + done: + bridge_line_free(bridge_line); +} + +static void +test_config_parse_bridge_line(void *arg) +{ + (void) arg; + good_bridge_line_test("192.0.2.1:4123", + "192.0.2.1:4123", NULL, NULL, NULL); + + good_bridge_line_test("192.0.2.1", + "192.0.2.1:443", NULL, NULL, NULL); + + good_bridge_line_test("transport [::1]", + "[::1]:443", NULL, "transport", NULL); + + good_bridge_line_test("transport 192.0.2.1:12 " + "4352e58420e68f5e40bf7c74faddccd9d1349413", + "192.0.2.1:12", + "4352e58420e68f5e40bf7c74faddccd9d1349413", + "transport", NULL); + + { + smartlist_t *sl_tmp = smartlist_new(); + smartlist_add_asprintf(sl_tmp, "twoandtwo=five"); + + good_bridge_line_test("transport 192.0.2.1:12 " + "4352e58420e68f5e40bf7c74faddccd9d1349413 twoandtwo=five", + "192.0.2.1:12", "4352e58420e68f5e40bf7c74faddccd9d1349413", + "transport", sl_tmp); + + SMARTLIST_FOREACH(sl_tmp, char *, s, tor_free(s)); + smartlist_free(sl_tmp); + } + + { + smartlist_t *sl_tmp = smartlist_new(); + smartlist_add_asprintf(sl_tmp, "twoandtwo=five"); + smartlist_add_asprintf(sl_tmp, "z=z"); + + good_bridge_line_test("transport 192.0.2.1:12 twoandtwo=five z=z", + "192.0.2.1:12", NULL, "transport", sl_tmp); + + SMARTLIST_FOREACH(sl_tmp, char *, s, tor_free(s)); + smartlist_free(sl_tmp); + } + + { + smartlist_t *sl_tmp = smartlist_new(); + smartlist_add_asprintf(sl_tmp, "dub=come"); + smartlist_add_asprintf(sl_tmp, "save=me"); + + good_bridge_line_test("transport 192.0.2.1:12 " + "4352e58420e68f5e40bf7c74faddccd9d1349666 " + "dub=come save=me", + + "192.0.2.1:12", + "4352e58420e68f5e40bf7c74faddccd9d1349666", + "transport", sl_tmp); + + SMARTLIST_FOREACH(sl_tmp, char *, s, tor_free(s)); + smartlist_free(sl_tmp); + } + + good_bridge_line_test("192.0.2.1:1231 " + "4352e58420e68f5e40bf7c74faddccd9d1349413", + "192.0.2.1:1231", + "4352e58420e68f5e40bf7c74faddccd9d1349413", + NULL, NULL); + + /* Empty line */ + bad_bridge_line_test(""); + /* bad transport name */ + bad_bridge_line_test("tr$n_sp0r7 190.20.2.2"); + /* weird ip address */ + bad_bridge_line_test("a.b.c.d"); + /* invalid fpr */ + bad_bridge_line_test("2.2.2.2:1231 4352e58420e68f5e40bf7c74faddccd9d1349"); + /* no k=v in the end */ + bad_bridge_line_test("obfs2 2.2.2.2:1231 " + "4352e58420e68f5e40bf7c74faddccd9d1349413 what"); + /* no addrport */ + bad_bridge_line_test("asdw"); + /* huge k=v value that can't fit in SOCKS fields */ + bad_bridge_line_test( + "obfs2 2.2.2.2:1231 aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + "aa=b"); +} + +static void +test_config_parse_transport_options_line(void *arg) +{ + smartlist_t *options_sl = NULL, *sl_tmp = NULL; + + (void) arg; + + { /* too small line */ + options_sl = get_options_from_transport_options_line("valley", NULL); + test_assert(!options_sl); + } + + { /* no k=v values */ + options_sl = get_options_from_transport_options_line("hit it!", NULL); + test_assert(!options_sl); + } + + { /* correct line, but wrong transport specified */ + options_sl = + get_options_from_transport_options_line("trebuchet k=v", "rook"); + test_assert(!options_sl); + } + + { /* correct -- no transport specified */ + sl_tmp = smartlist_new(); + smartlist_add_asprintf(sl_tmp, "ladi=dadi"); + smartlist_add_asprintf(sl_tmp, "weliketo=party"); + + options_sl = + get_options_from_transport_options_line("rook ladi=dadi weliketo=party", + NULL); + test_assert(options_sl); + test_assert(smartlist_strings_eq(options_sl, sl_tmp)); + + SMARTLIST_FOREACH(sl_tmp, char *, s, tor_free(s)); + smartlist_free(sl_tmp); + sl_tmp = NULL; + SMARTLIST_FOREACH(options_sl, char *, s, tor_free(s)); + smartlist_free(options_sl); + options_sl = NULL; + } + + { /* correct -- correct transport specified */ + sl_tmp = smartlist_new(); + smartlist_add_asprintf(sl_tmp, "ladi=dadi"); + smartlist_add_asprintf(sl_tmp, "weliketo=party"); + + options_sl = + get_options_from_transport_options_line("rook ladi=dadi weliketo=party", + "rook"); + test_assert(options_sl); + test_assert(smartlist_strings_eq(options_sl, sl_tmp)); + SMARTLIST_FOREACH(sl_tmp, char *, s, tor_free(s)); + smartlist_free(sl_tmp); + sl_tmp = NULL; + SMARTLIST_FOREACH(options_sl, char *, s, tor_free(s)); + smartlist_free(options_sl); + options_sl = NULL; + } + + done: + if (options_sl) { + SMARTLIST_FOREACH(options_sl, char *, s, tor_free(s)); + smartlist_free(options_sl); + } + if (sl_tmp) { + SMARTLIST_FOREACH(sl_tmp, char *, s, tor_free(s)); + smartlist_free(sl_tmp); + } +} + +// Tests if an options with MyFamily fingerprints missing '$' normalises +// them correctly and also ensure it also works with multiple fingerprints +static void +test_config_fix_my_family(void *arg) +{ + char *err = NULL; + const char *family = "$1111111111111111111111111111111111111111, " + "1111111111111111111111111111111111111112, " + "$1111111111111111111111111111111111111113"; + + or_options_t* options = options_new(); + or_options_t* defaults = options_new(); + (void) arg; + + options_init(options); + options_init(defaults); + options->MyFamily = tor_strdup(family); + + options_validate(NULL, options, defaults, 0, &err) ; + + if (err != NULL) { + TT_FAIL(("options_validate failed: %s", err)); + } + + test_streq(options->MyFamily, "$1111111111111111111111111111111111111111, " + "$1111111111111111111111111111111111111112, " + "$1111111111111111111111111111111111111113"); + + done: + if (err != NULL) { + tor_free(err); + } + + or_options_free(options); + or_options_free(defaults); } #define CONFIG_TEST(name, flags) \ @@ -174,6 +594,11 @@ test_config_addressmap(void *arg) struct testcase_t config_tests[] = { CONFIG_TEST(addressmap, 0), + CONFIG_TEST(parse_bridge_line, 0), + CONFIG_TEST(parse_transport_options_line, 0), + CONFIG_TEST(check_or_create_data_subdir, TT_FORK), + CONFIG_TEST(write_to_data_subdir, TT_FORK), + CONFIG_TEST(fix_my_family, 0), END_OF_TESTCASES }; diff --git a/src/test/test_containers.c b/src/test/test_containers.c index 005e102e25..067c4c1907 100644 --- a/src/test/test_containers.c +++ b/src/test/test_containers.c @@ -469,6 +469,51 @@ test_container_smartlist_join(void) tor_free(joined); } +static void +test_container_smartlist_ints_eq(void *arg) +{ + smartlist_t *sl1 = NULL, *sl2 = NULL; + int x; + (void)arg; + + tt_assert(smartlist_ints_eq(NULL, NULL)); + + sl1 = smartlist_new(); + tt_assert(!smartlist_ints_eq(sl1, NULL)); + tt_assert(!smartlist_ints_eq(NULL, sl1)); + + sl2 = smartlist_new(); + tt_assert(smartlist_ints_eq(sl1, sl2)); + + x = 5; + smartlist_add(sl1, tor_memdup(&x, sizeof(int))); + smartlist_add(sl2, tor_memdup(&x, sizeof(int))); + x = 90; + smartlist_add(sl1, tor_memdup(&x, sizeof(int))); + smartlist_add(sl2, tor_memdup(&x, sizeof(int))); + tt_assert(smartlist_ints_eq(sl1, sl2)); + + x = -50; + smartlist_add(sl1, tor_memdup(&x, sizeof(int))); + tt_assert(! smartlist_ints_eq(sl1, sl2)); + tt_assert(! smartlist_ints_eq(sl2, sl1)); + smartlist_add(sl2, tor_memdup(&x, sizeof(int))); + tt_assert(smartlist_ints_eq(sl1, sl2)); + + *(int*)smartlist_get(sl1, 1) = 101010; + tt_assert(! smartlist_ints_eq(sl2, sl1)); + *(int*)smartlist_get(sl2, 1) = 101010; + tt_assert(smartlist_ints_eq(sl1, sl2)); + + done: + if (sl1) + SMARTLIST_FOREACH(sl1, int *, ip, tor_free(ip)); + if (sl2) + SMARTLIST_FOREACH(sl2, int *, ip, tor_free(ip)); + smartlist_free(sl1); + smartlist_free(sl2); +} + /** Run unit tests for bitarray code */ static void test_container_bitarray(void) @@ -784,7 +829,7 @@ test_container_order_functions(void) } static void -test_di_map(void *arg) +test_container_di_map(void *arg) { di_digest256_map_t *map = NULL; const uint8_t key1[] = "In view of the fact that it was "; @@ -856,12 +901,12 @@ test_container_fp_pair_map(void) memset(fp6.second, 0x62, DIGEST_LEN); v = fp_pair_map_set(map, &fp1, (void*)99); - test_eq(v, NULL); + tt_ptr_op(v, ==, NULL); test_assert(!fp_pair_map_isempty(map)); v = fp_pair_map_set(map, &fp2, (void*)101); - test_eq(v, NULL); + tt_ptr_op(v, ==, NULL); v = fp_pair_map_set(map, &fp1, (void*)100); - test_eq(v, (void*)99); + tt_ptr_op(v, ==, (void*)99); test_eq_ptr(fp_pair_map_get(map, &fp1), (void*)100); test_eq_ptr(fp_pair_map_get(map, &fp2), (void*)101); test_eq_ptr(fp_pair_map_get(map, &fp3), NULL); @@ -912,18 +957,22 @@ test_container_fp_pair_map(void) #define CONTAINER_LEGACY(name) \ { #name, legacy_test_helper, 0, &legacy_setup, test_container_ ## name } +#define CONTAINER(name, flags) \ + { #name, test_container_ ## name, (flags), NULL, NULL } + struct testcase_t container_tests[] = { CONTAINER_LEGACY(smartlist_basic), CONTAINER_LEGACY(smartlist_strings), CONTAINER_LEGACY(smartlist_overlap), CONTAINER_LEGACY(smartlist_digests), CONTAINER_LEGACY(smartlist_join), + CONTAINER(smartlist_ints_eq, 0), CONTAINER_LEGACY(bitarray), CONTAINER_LEGACY(digestset), CONTAINER_LEGACY(strmap), CONTAINER_LEGACY(pqueue), CONTAINER_LEGACY(order_functions), - { "di_map", test_di_map, 0, NULL, NULL }, + CONTAINER(di_map, 0), CONTAINER_LEGACY(fp_pair_map), END_OF_TESTCASES }; diff --git a/src/test/test_controller_events.c b/src/test/test_controller_events.c new file mode 100644 index 0000000000..b45e97a417 --- /dev/null +++ b/src/test/test_controller_events.c @@ -0,0 +1,307 @@ +/* Copyright (c) 2013, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#define CONNECTION_PRIVATE +#define TOR_CHANNEL_INTERNAL_ +#define CONTROL_PRIVATE +#include "or.h" +#include "channel.h" +#include "channeltls.h" +#include "connection.h" +#include "control.h" +#include "test.h" + +static void +help_test_bucket_note_empty(uint32_t expected_msec_since_midnight, + int tokens_before, size_t tokens_removed, + uint32_t msec_since_epoch) +{ + uint32_t timestamp_var = 0; + struct timeval tvnow; + tvnow.tv_sec = msec_since_epoch / 1000; + tvnow.tv_usec = (msec_since_epoch % 1000) * 1000; + connection_buckets_note_empty_ts(×tamp_var, tokens_before, + tokens_removed, &tvnow); + tt_int_op(expected_msec_since_midnight, ==, timestamp_var); + + done: + ; +} + +static void +test_cntev_bucket_note_empty(void *arg) +{ + (void)arg; + + /* Two cases with nothing to note, because bucket was empty before; + * 86442200 == 1970-01-02 00:00:42.200000 */ + help_test_bucket_note_empty(0, 0, 0, 86442200); + help_test_bucket_note_empty(0, -100, 100, 86442200); + + /* Nothing to note, because bucket has not been emptied. */ + help_test_bucket_note_empty(0, 101, 100, 86442200); + + /* Bucket was emptied, note 42200 msec since midnight. */ + help_test_bucket_note_empty(42200, 101, 101, 86442200); + help_test_bucket_note_empty(42200, 101, 102, 86442200); +} + +static void +test_cntev_bucket_millis_empty(void *arg) +{ + struct timeval tvnow; + (void)arg; + + /* 1970-01-02 00:00:42.200000 */ + tvnow.tv_sec = 86400 + 42; + tvnow.tv_usec = 200000; + + /* Bucket has not been refilled. */ + tt_int_op(0, ==, bucket_millis_empty(0, 42120, 0, 100, &tvnow)); + tt_int_op(0, ==, bucket_millis_empty(-10, 42120, -10, 100, &tvnow)); + + /* Bucket was not empty. */ + tt_int_op(0, ==, bucket_millis_empty(10, 42120, 20, 100, &tvnow)); + + /* Bucket has been emptied 80 msec ago and has just been refilled. */ + tt_int_op(80, ==, bucket_millis_empty(-20, 42120, -10, 100, &tvnow)); + tt_int_op(80, ==, bucket_millis_empty(-10, 42120, 0, 100, &tvnow)); + tt_int_op(80, ==, bucket_millis_empty(0, 42120, 10, 100, &tvnow)); + + /* Bucket has been emptied 180 msec ago, last refill was 100 msec ago + * which was insufficient to make it positive, so cap msec at 100. */ + tt_int_op(100, ==, bucket_millis_empty(0, 42020, 1, 100, &tvnow)); + + /* 1970-01-02 00:00:00:050000 */ + tvnow.tv_sec = 86400; + tvnow.tv_usec = 50000; + + /* Last emptied 30 msec before midnight, tvnow is 50 msec after + * midnight, that's 80 msec in total. */ + tt_int_op(80, ==, bucket_millis_empty(0, 86400000 - 30, 1, 100, &tvnow)); + + done: + ; +} + +static void +add_testing_cell_stats_entry(circuit_t *circ, uint8_t command, + unsigned int waiting_time, + unsigned int removed, unsigned int exitward) +{ + testing_cell_stats_entry_t *ent = tor_malloc_zero( + sizeof(testing_cell_stats_entry_t)); + ent->command = command; + ent->waiting_time = waiting_time; + ent->removed = removed; + ent->exitward = exitward; + if (!circ->testing_cell_stats) + circ->testing_cell_stats = smartlist_new(); + smartlist_add(circ->testing_cell_stats, ent); +} + +static void +test_cntev_sum_up_cell_stats(void *arg) +{ + or_circuit_t *or_circ; + circuit_t *circ; + cell_stats_t *cell_stats = NULL; + (void)arg; + + /* This circuit is fake. */ + or_circ = tor_malloc_zero(sizeof(or_circuit_t)); + or_circ->base_.magic = OR_CIRCUIT_MAGIC; + or_circ->base_.purpose = CIRCUIT_PURPOSE_OR; + circ = TO_CIRCUIT(or_circ); + + /* A single RELAY cell was added to the appward queue. */ + cell_stats = tor_malloc_zero(sizeof(cell_stats_t)); + add_testing_cell_stats_entry(circ, CELL_RELAY, 0, 0, 0); + sum_up_cell_stats_by_command(circ, cell_stats); + tt_u64_op(1, ==, cell_stats->added_cells_appward[CELL_RELAY]); + + /* A single RELAY cell was added to the exitward queue. */ + add_testing_cell_stats_entry(circ, CELL_RELAY, 0, 0, 1); + sum_up_cell_stats_by_command(circ, cell_stats); + tt_u64_op(1, ==, cell_stats->added_cells_exitward[CELL_RELAY]); + + /* A single RELAY cell was removed from the appward queue where it spent + * 20 msec. */ + add_testing_cell_stats_entry(circ, CELL_RELAY, 2, 1, 0); + sum_up_cell_stats_by_command(circ, cell_stats); + tt_u64_op(20, ==, cell_stats->total_time_appward[CELL_RELAY]); + tt_u64_op(1, ==, cell_stats->removed_cells_appward[CELL_RELAY]); + + /* A single RELAY cell was removed from the exitward queue where it + * spent 30 msec. */ + add_testing_cell_stats_entry(circ, CELL_RELAY, 3, 1, 1); + sum_up_cell_stats_by_command(circ, cell_stats); + tt_u64_op(30, ==, cell_stats->total_time_exitward[CELL_RELAY]); + tt_u64_op(1, ==, cell_stats->removed_cells_exitward[CELL_RELAY]); + + done: + tor_free(cell_stats); + tor_free(or_circ); +} + +static void +test_cntev_append_cell_stats(void *arg) +{ + smartlist_t *event_parts; + char *cp = NULL; + const char *key = "Z"; + uint64_t include_if_non_zero[CELL_COMMAND_MAX_ + 1], + number_to_include[CELL_COMMAND_MAX_ + 1]; + (void)arg; + + event_parts = smartlist_new(); + memset(include_if_non_zero, 0, + (CELL_COMMAND_MAX_ + 1) * sizeof(uint64_t)); + memset(number_to_include, 0, + (CELL_COMMAND_MAX_ + 1) * sizeof(uint64_t)); + + /* All array entries empty. */ + append_cell_stats_by_command(event_parts, key, + include_if_non_zero, + number_to_include); + tt_int_op(0, ==, smartlist_len(event_parts)); + + /* There's a RELAY cell to include, but the corresponding field in + * include_if_non_zero is still zero. */ + number_to_include[CELL_RELAY] = 1; + append_cell_stats_by_command(event_parts, key, + include_if_non_zero, + number_to_include); + tt_int_op(0, ==, smartlist_len(event_parts)); + + /* Now include single RELAY cell. */ + include_if_non_zero[CELL_RELAY] = 2; + append_cell_stats_by_command(event_parts, key, + include_if_non_zero, + number_to_include); + cp = smartlist_pop_last(event_parts); + tt_str_op("Z=relay:1", ==, cp); + tor_free(cp); + + /* Add four CREATE cells. */ + include_if_non_zero[CELL_CREATE] = 3; + number_to_include[CELL_CREATE] = 4; + append_cell_stats_by_command(event_parts, key, + include_if_non_zero, + number_to_include); + cp = smartlist_pop_last(event_parts); + tt_str_op("Z=create:4,relay:1", ==, cp); + + done: + tor_free(cp); + smartlist_free(event_parts); +} + +static void +test_cntev_format_cell_stats(void *arg) +{ + char *event_string = NULL; + origin_circuit_t *ocirc = NULL; + or_circuit_t *or_circ = NULL; + cell_stats_t *cell_stats = NULL; + channel_tls_t *n_chan=NULL, *p_chan=NULL; + (void)arg; + + n_chan = tor_malloc_zero(sizeof(channel_tls_t)); + n_chan->base_.global_identifier = 1; + + ocirc = tor_malloc_zero(sizeof(origin_circuit_t)); + ocirc->base_.magic = ORIGIN_CIRCUIT_MAGIC; + ocirc->base_.purpose = CIRCUIT_PURPOSE_C_GENERAL; + ocirc->global_identifier = 2; + ocirc->base_.n_circ_id = 3; + ocirc->base_.n_chan = &(n_chan->base_); + + /* Origin circuit was completely idle. */ + cell_stats = tor_malloc_zero(sizeof(cell_stats_t)); + format_cell_stats(&event_string, TO_CIRCUIT(ocirc), cell_stats); + tt_str_op("ID=2 OutboundQueue=3 OutboundConn=1", ==, event_string); + tor_free(event_string); + + /* Origin circuit had 4 RELAY cells added to its exitward queue. */ + cell_stats->added_cells_exitward[CELL_RELAY] = 4; + format_cell_stats(&event_string, TO_CIRCUIT(ocirc), cell_stats); + tt_str_op("ID=2 OutboundQueue=3 OutboundConn=1 OutboundAdded=relay:4", + ==, event_string); + tor_free(event_string); + + /* Origin circuit also had 5 CREATE2 cells added to its exitward + * queue. */ + cell_stats->added_cells_exitward[CELL_CREATE2] = 5; + format_cell_stats(&event_string, TO_CIRCUIT(ocirc), cell_stats); + tt_str_op("ID=2 OutboundQueue=3 OutboundConn=1 OutboundAdded=relay:4," + "create2:5", ==, event_string); + tor_free(event_string); + + /* Origin circuit also had 7 RELAY cells removed from its exitward queue + * which together spent 6 msec in the queue. */ + cell_stats->total_time_exitward[CELL_RELAY] = 6; + cell_stats->removed_cells_exitward[CELL_RELAY] = 7; + format_cell_stats(&event_string, TO_CIRCUIT(ocirc), cell_stats); + tt_str_op("ID=2 OutboundQueue=3 OutboundConn=1 OutboundAdded=relay:4," + "create2:5 OutboundRemoved=relay:7 OutboundTime=relay:6", + ==, event_string); + tor_free(event_string); + + p_chan = tor_malloc_zero(sizeof(channel_tls_t)); + p_chan->base_.global_identifier = 2; + + or_circ = tor_malloc_zero(sizeof(or_circuit_t)); + or_circ->base_.magic = OR_CIRCUIT_MAGIC; + or_circ->base_.purpose = CIRCUIT_PURPOSE_OR; + or_circ->p_circ_id = 8; + or_circ->p_chan = &(p_chan->base_); + or_circ->base_.n_circ_id = 9; + or_circ->base_.n_chan = &(n_chan->base_); + + tor_free(cell_stats); + + /* OR circuit was idle. */ + cell_stats = tor_malloc_zero(sizeof(cell_stats_t)); + format_cell_stats(&event_string, TO_CIRCUIT(or_circ), cell_stats); + tt_str_op("InboundQueue=8 InboundConn=2 OutboundQueue=9 OutboundConn=1", + ==, event_string); + tor_free(event_string); + + /* OR circuit had 3 RELAY cells added to its appward queue. */ + cell_stats->added_cells_appward[CELL_RELAY] = 3; + format_cell_stats(&event_string, TO_CIRCUIT(or_circ), cell_stats); + tt_str_op("InboundQueue=8 InboundConn=2 InboundAdded=relay:3 " + "OutboundQueue=9 OutboundConn=1", ==, event_string); + tor_free(event_string); + + /* OR circuit had 7 RELAY cells removed from its appward queue which + * together spent 6 msec in the queue. */ + cell_stats->total_time_appward[CELL_RELAY] = 6; + cell_stats->removed_cells_appward[CELL_RELAY] = 7; + format_cell_stats(&event_string, TO_CIRCUIT(or_circ), cell_stats); + tt_str_op("InboundQueue=8 InboundConn=2 InboundAdded=relay:3 " + "InboundRemoved=relay:7 InboundTime=relay:6 " + "OutboundQueue=9 OutboundConn=1", ==, event_string); + + done: + tor_free(cell_stats); + tor_free(event_string); + tor_free(or_circ); + tor_free(ocirc); + tor_free(p_chan); + tor_free(n_chan); +} + +#define TEST(name, flags) \ + { #name, test_cntev_ ## name, flags, 0, NULL } + +struct testcase_t controller_event_tests[] = { + TEST(bucket_note_empty, 0), + TEST(bucket_millis_empty, 0), + TEST(sum_up_cell_stats, 0), + TEST(append_cell_stats, 0), + TEST(format_cell_stats, 0), + END_OF_TESTCASES +}; + diff --git a/src/test/test_crypto.c b/src/test/test_crypto.c index f92bfd673e..5d8edb6550 100644 --- a/src/test/test_crypto.c +++ b/src/test/test_crypto.c @@ -4,16 +4,20 @@ /* See LICENSE for licensing information */ #include "orconfig.h" -#define CRYPTO_PRIVATE #define CRYPTO_CURVE25519_PRIVATE #include "or.h" #include "test.h" #include "aes.h" #include "util.h" +#include "siphash.h" #ifdef CURVE25519_ENABLED #include "crypto_curve25519.h" #endif +extern const char AUTHORITY_SIGNKEY_3[]; +extern const char AUTHORITY_SIGNKEY_A_DIGEST[]; +extern const char AUTHORITY_SIGNKEY_A_DIGEST256[]; + /** Run unit tests for Diffie-Hellman functionality. */ static void test_crypto_dh(void) @@ -269,34 +273,6 @@ test_crypto_sha(void) "96177A9CB410FF61F20015AD"); tt_int_op(i, ==, 0); - /* Test HMAC-SHA-1 with test cases from RFC2202. */ - - /* Case 1. */ - memset(key, 0x0b, 20); - crypto_hmac_sha1(digest, key, 20, "Hi There", 8); - test_streq(hex_str(digest, 20), - "B617318655057264E28BC0B6FB378C8EF146BE00"); - /* Case 2. */ - crypto_hmac_sha1(digest, "Jefe", 4, "what do ya want for nothing?", 28); - test_streq(hex_str(digest, 20), - "EFFCDF6AE5EB2FA2D27416D5F184DF9C259A7C79"); - - /* Case 4. */ - base16_decode(key, 25, - "0102030405060708090a0b0c0d0e0f10111213141516171819", 50); - memset(data, 0xcd, 50); - crypto_hmac_sha1(digest, key, 25, data, 50); - test_streq(hex_str(digest, 20), - "4C9007F4026250C6BC8414F9BF50C86C2D7235DA"); - - /* Case 5. */ - memset(key, 0xaa, 80); - crypto_hmac_sha1(digest, key, 80, - "Test Using Larger Than Block-Size Key - Hash Key First", - 54); - test_streq(hex_str(digest, 20), - "AA4AE5E15272D00E95705637CE8A3B55ED402112"); - /* Test HMAC-SHA256 with test cases from wikipedia and RFC 4231 */ /* Case empty (wikipedia) */ @@ -422,7 +398,7 @@ test_crypto_pk(void) char *encoded = NULL; char data1[1024], data2[1024], data3[1024]; size_t size; - int i, j, p, len; + int i, len; /* Public-key ciphers */ pk1 = pk_generate(0); @@ -506,19 +482,16 @@ test_crypto_pk(void) /* Try with hybrid encryption wrappers. */ crypto_rand(data1, 1024); - for (i = 0; i < 2; ++i) { - for (j = 85; j < 140; ++j) { - memset(data2,0,1024); - memset(data3,0,1024); - p = (i==0)?PK_PKCS1_PADDING:PK_PKCS1_OAEP_PADDING; - len = crypto_pk_public_hybrid_encrypt(pk1,data2,sizeof(data2), - data1,j,p,0); - test_assert(len>=0); - len = crypto_pk_private_hybrid_decrypt(pk1,data3,sizeof(data3), - data2,len,p,1); - test_eq(len,j); - test_memeq(data1,data3,j); - } + for (i = 85; i < 140; ++i) { + memset(data2,0,1024); + memset(data3,0,1024); + len = crypto_pk_public_hybrid_encrypt(pk1,data2,sizeof(data2), + data1,i,PK_PKCS1_OAEP_PADDING,0); + test_assert(len>=0); + len = crypto_pk_private_hybrid_decrypt(pk1,data3,sizeof(data3), + data2,len,PK_PKCS1_OAEP_PADDING,1); + test_eq(len,i); + test_memeq(data1,data3,i); } /* Try copy_full */ @@ -536,6 +509,85 @@ test_crypto_pk(void) tor_free(encoded); } +static void +test_crypto_pk_fingerprints(void *arg) +{ + crypto_pk_t *pk = NULL; + char encoded[512]; + char d[DIGEST_LEN], d2[DIGEST_LEN]; + char fingerprint[FINGERPRINT_LEN+1]; + int n; + unsigned i; + char *mem_op_hex_tmp=NULL; + + (void)arg; + + pk = pk_generate(1); + tt_assert(pk); + n = crypto_pk_asn1_encode(pk, encoded, sizeof(encoded)); + tt_int_op(n, >, 0); + tt_int_op(n, >, 128); + tt_int_op(n, <, 256); + + /* Is digest as expected? */ + crypto_digest(d, encoded, n); + tt_int_op(0, ==, crypto_pk_get_digest(pk, d2)); + test_memeq(d, d2, DIGEST_LEN); + + /* Is fingerprint right? */ + tt_int_op(0, ==, crypto_pk_get_fingerprint(pk, fingerprint, 0)); + tt_int_op(strlen(fingerprint), ==, DIGEST_LEN * 2); + test_memeq_hex(d, fingerprint); + + /* Are spaces right? */ + tt_int_op(0, ==, crypto_pk_get_fingerprint(pk, fingerprint, 1)); + for (i = 4; i < strlen(fingerprint); i += 5) { + tt_int_op(fingerprint[i], ==, ' '); + } + tor_strstrip(fingerprint, " "); + tt_int_op(strlen(fingerprint), ==, DIGEST_LEN * 2); + test_memeq_hex(d, fingerprint); + + /* Now hash again and check crypto_pk_get_hashed_fingerprint. */ + crypto_digest(d2, d, sizeof(d)); + tt_int_op(0, ==, crypto_pk_get_hashed_fingerprint(pk, fingerprint)); + tt_int_op(strlen(fingerprint), ==, DIGEST_LEN * 2); + test_memeq_hex(d2, fingerprint); + + done: + crypto_pk_free(pk); + tor_free(mem_op_hex_tmp); +} + +/** Sanity check for crypto pk digests */ +static void +test_crypto_digests(void) +{ + crypto_pk_t *k = NULL; + ssize_t r; + digests_t pkey_digests; + char digest[DIGEST_LEN]; + + k = crypto_pk_new(); + test_assert(k); + r = crypto_pk_read_private_key_from_string(k, AUTHORITY_SIGNKEY_3, -1); + test_assert(!r); + + r = crypto_pk_get_digest(k, digest); + test_assert(r == 0); + test_memeq(hex_str(digest, DIGEST_LEN), + AUTHORITY_SIGNKEY_A_DIGEST, HEX_DIGEST_LEN); + + r = crypto_pk_get_all_digests(k, &pkey_digests); + + test_memeq(hex_str(pkey_digests.d[DIGEST_SHA1], DIGEST_LEN), + AUTHORITY_SIGNKEY_A_DIGEST, HEX_DIGEST_LEN); + test_memeq(hex_str(pkey_digests.d[DIGEST_SHA256], DIGEST256_LEN), + AUTHORITY_SIGNKEY_A_DIGEST256, HEX_DIGEST256_LEN); + done: + crypto_pk_free(k); +} + /** Run unit tests for misc crypto formatting functionality (base64, base32, * fingerprints, etc) */ static void @@ -630,7 +682,7 @@ test_crypto_formats(void) data1 = tor_strdup("ABCD1234ABCD56780000ABCD1234ABCD56780000"); test_eq(strlen(data1), 40); data2 = tor_malloc(FINGERPRINT_LEN+1); - add_spaces_to_fp(data2, FINGERPRINT_LEN+1, data1); + crypto_add_spaces_to_fp(data2, FINGERPRINT_LEN+1, data1); test_streq(data2, "ABCD 1234 ABCD 5678 0000 ABCD 1234 ABCD 5678 0000"); tor_free(data1); tor_free(data2); @@ -730,11 +782,13 @@ test_crypto_aes_iv(void *arg) /* Decrypt with the wrong key. */ decrypted_size = crypto_cipher_decrypt_with_iv(key2, decrypted2, 4095, encrypted1, encrypted_size); + test_eq(decrypted_size, 4095); test_memneq(plain, decrypted2, decrypted_size); /* Alter the initialization vector. */ encrypted1[0] += 42; decrypted_size = crypto_cipher_decrypt_with_iv(key1, decrypted1, 4095, encrypted1, encrypted_size); + test_eq(decrypted_size, 4095); test_memneq(plain, decrypted2, 4095); /* Special length case: 1. */ encrypted_size = crypto_cipher_encrypt_with_iv(key1, encrypted1, 16 + 1, @@ -1078,7 +1132,8 @@ test_crypto_curve25519_persist(void *arg) content = read_file_to_str(fname, RFTS_BIN, &st); tt_assert(content); taglen = strlen("== c25519v1: testing =="); - tt_int_op(st.st_size, ==, 32+CURVE25519_PUBKEY_LEN+CURVE25519_SECKEY_LEN); + tt_u64_op((uint64_t)st.st_size, ==, + 32+CURVE25519_PUBKEY_LEN+CURVE25519_SECKEY_LEN); tt_assert(fast_memeq(content, "== c25519v1: testing ==", taglen)); tt_assert(tor_mem_is_zero(content+taglen, 32-taglen)); cp = content + 32; @@ -1108,6 +1163,102 @@ test_crypto_curve25519_persist(void *arg) #endif +static void +test_crypto_siphash(void *arg) +{ + /* From the reference implementation, taking + k = 00 01 02 ... 0f + and in = 00; 00 01; 00 01 02; ... + */ + const uint8_t VECTORS[64][8] = + { + { 0x31, 0x0e, 0x0e, 0xdd, 0x47, 0xdb, 0x6f, 0x72, }, + { 0xfd, 0x67, 0xdc, 0x93, 0xc5, 0x39, 0xf8, 0x74, }, + { 0x5a, 0x4f, 0xa9, 0xd9, 0x09, 0x80, 0x6c, 0x0d, }, + { 0x2d, 0x7e, 0xfb, 0xd7, 0x96, 0x66, 0x67, 0x85, }, + { 0xb7, 0x87, 0x71, 0x27, 0xe0, 0x94, 0x27, 0xcf, }, + { 0x8d, 0xa6, 0x99, 0xcd, 0x64, 0x55, 0x76, 0x18, }, + { 0xce, 0xe3, 0xfe, 0x58, 0x6e, 0x46, 0xc9, 0xcb, }, + { 0x37, 0xd1, 0x01, 0x8b, 0xf5, 0x00, 0x02, 0xab, }, + { 0x62, 0x24, 0x93, 0x9a, 0x79, 0xf5, 0xf5, 0x93, }, + { 0xb0, 0xe4, 0xa9, 0x0b, 0xdf, 0x82, 0x00, 0x9e, }, + { 0xf3, 0xb9, 0xdd, 0x94, 0xc5, 0xbb, 0x5d, 0x7a, }, + { 0xa7, 0xad, 0x6b, 0x22, 0x46, 0x2f, 0xb3, 0xf4, }, + { 0xfb, 0xe5, 0x0e, 0x86, 0xbc, 0x8f, 0x1e, 0x75, }, + { 0x90, 0x3d, 0x84, 0xc0, 0x27, 0x56, 0xea, 0x14, }, + { 0xee, 0xf2, 0x7a, 0x8e, 0x90, 0xca, 0x23, 0xf7, }, + { 0xe5, 0x45, 0xbe, 0x49, 0x61, 0xca, 0x29, 0xa1, }, + { 0xdb, 0x9b, 0xc2, 0x57, 0x7f, 0xcc, 0x2a, 0x3f, }, + { 0x94, 0x47, 0xbe, 0x2c, 0xf5, 0xe9, 0x9a, 0x69, }, + { 0x9c, 0xd3, 0x8d, 0x96, 0xf0, 0xb3, 0xc1, 0x4b, }, + { 0xbd, 0x61, 0x79, 0xa7, 0x1d, 0xc9, 0x6d, 0xbb, }, + { 0x98, 0xee, 0xa2, 0x1a, 0xf2, 0x5c, 0xd6, 0xbe, }, + { 0xc7, 0x67, 0x3b, 0x2e, 0xb0, 0xcb, 0xf2, 0xd0, }, + { 0x88, 0x3e, 0xa3, 0xe3, 0x95, 0x67, 0x53, 0x93, }, + { 0xc8, 0xce, 0x5c, 0xcd, 0x8c, 0x03, 0x0c, 0xa8, }, + { 0x94, 0xaf, 0x49, 0xf6, 0xc6, 0x50, 0xad, 0xb8, }, + { 0xea, 0xb8, 0x85, 0x8a, 0xde, 0x92, 0xe1, 0xbc, }, + { 0xf3, 0x15, 0xbb, 0x5b, 0xb8, 0x35, 0xd8, 0x17, }, + { 0xad, 0xcf, 0x6b, 0x07, 0x63, 0x61, 0x2e, 0x2f, }, + { 0xa5, 0xc9, 0x1d, 0xa7, 0xac, 0xaa, 0x4d, 0xde, }, + { 0x71, 0x65, 0x95, 0x87, 0x66, 0x50, 0xa2, 0xa6, }, + { 0x28, 0xef, 0x49, 0x5c, 0x53, 0xa3, 0x87, 0xad, }, + { 0x42, 0xc3, 0x41, 0xd8, 0xfa, 0x92, 0xd8, 0x32, }, + { 0xce, 0x7c, 0xf2, 0x72, 0x2f, 0x51, 0x27, 0x71, }, + { 0xe3, 0x78, 0x59, 0xf9, 0x46, 0x23, 0xf3, 0xa7, }, + { 0x38, 0x12, 0x05, 0xbb, 0x1a, 0xb0, 0xe0, 0x12, }, + { 0xae, 0x97, 0xa1, 0x0f, 0xd4, 0x34, 0xe0, 0x15, }, + { 0xb4, 0xa3, 0x15, 0x08, 0xbe, 0xff, 0x4d, 0x31, }, + { 0x81, 0x39, 0x62, 0x29, 0xf0, 0x90, 0x79, 0x02, }, + { 0x4d, 0x0c, 0xf4, 0x9e, 0xe5, 0xd4, 0xdc, 0xca, }, + { 0x5c, 0x73, 0x33, 0x6a, 0x76, 0xd8, 0xbf, 0x9a, }, + { 0xd0, 0xa7, 0x04, 0x53, 0x6b, 0xa9, 0x3e, 0x0e, }, + { 0x92, 0x59, 0x58, 0xfc, 0xd6, 0x42, 0x0c, 0xad, }, + { 0xa9, 0x15, 0xc2, 0x9b, 0xc8, 0x06, 0x73, 0x18, }, + { 0x95, 0x2b, 0x79, 0xf3, 0xbc, 0x0a, 0xa6, 0xd4, }, + { 0xf2, 0x1d, 0xf2, 0xe4, 0x1d, 0x45, 0x35, 0xf9, }, + { 0x87, 0x57, 0x75, 0x19, 0x04, 0x8f, 0x53, 0xa9, }, + { 0x10, 0xa5, 0x6c, 0xf5, 0xdf, 0xcd, 0x9a, 0xdb, }, + { 0xeb, 0x75, 0x09, 0x5c, 0xcd, 0x98, 0x6c, 0xd0, }, + { 0x51, 0xa9, 0xcb, 0x9e, 0xcb, 0xa3, 0x12, 0xe6, }, + { 0x96, 0xaf, 0xad, 0xfc, 0x2c, 0xe6, 0x66, 0xc7, }, + { 0x72, 0xfe, 0x52, 0x97, 0x5a, 0x43, 0x64, 0xee, }, + { 0x5a, 0x16, 0x45, 0xb2, 0x76, 0xd5, 0x92, 0xa1, }, + { 0xb2, 0x74, 0xcb, 0x8e, 0xbf, 0x87, 0x87, 0x0a, }, + { 0x6f, 0x9b, 0xb4, 0x20, 0x3d, 0xe7, 0xb3, 0x81, }, + { 0xea, 0xec, 0xb2, 0xa3, 0x0b, 0x22, 0xa8, 0x7f, }, + { 0x99, 0x24, 0xa4, 0x3c, 0xc1, 0x31, 0x57, 0x24, }, + { 0xbd, 0x83, 0x8d, 0x3a, 0xaf, 0xbf, 0x8d, 0xb7, }, + { 0x0b, 0x1a, 0x2a, 0x32, 0x65, 0xd5, 0x1a, 0xea, }, + { 0x13, 0x50, 0x79, 0xa3, 0x23, 0x1c, 0xe6, 0x60, }, + { 0x93, 0x2b, 0x28, 0x46, 0xe4, 0xd7, 0x06, 0x66, }, + { 0xe1, 0x91, 0x5f, 0x5c, 0xb1, 0xec, 0xa4, 0x6c, }, + { 0xf3, 0x25, 0x96, 0x5c, 0xa1, 0x6d, 0x62, 0x9f, }, + { 0x57, 0x5f, 0xf2, 0x8e, 0x60, 0x38, 0x1b, 0xe5, }, + { 0x72, 0x45, 0x06, 0xeb, 0x4c, 0x32, 0x8a, 0x95, } + }; + + const struct sipkey K = { U64_LITERAL(0x0706050403020100), + U64_LITERAL(0x0f0e0d0c0b0a0908) }; + uint8_t input[64]; + int i, j; + + (void)arg; + + for (i = 0; i < 64; ++i) + input[i] = i; + + for (i = 0; i < 64; ++i) { + uint64_t r = siphash24(input, i, &K); + for (j = 0; j < 8; ++j) { + tt_int_op( (r >> (j*8)) & 0xff, ==, VECTORS[i][j]); + } + } + + done: + ; +} + static void * pass_data_setup_fn(const struct testcase_t *testcase) { @@ -1134,6 +1285,8 @@ struct testcase_t crypto_tests[] = { { "aes_EVP", test_crypto_aes, TT_FORK, &pass_data, (void*)"evp" }, CRYPTO_LEGACY(sha), CRYPTO_LEGACY(pk), + { "pk_fingerprints", test_crypto_pk_fingerprints, TT_FORK, NULL, NULL }, + CRYPTO_LEGACY(digests), CRYPTO_LEGACY(dh), CRYPTO_LEGACY(s2k), { "aes_iv_AES", test_crypto_aes_iv, TT_FORK, &pass_data, (void*)"aes" }, @@ -1148,6 +1301,7 @@ struct testcase_t crypto_tests[] = { { "curve25519_encode", test_crypto_curve25519_encode, 0, NULL, NULL }, { "curve25519_persist", test_crypto_curve25519_persist, 0, NULL, NULL }, #endif + { "siphash", test_crypto_siphash, 0, NULL, NULL }, END_OF_TESTCASES }; diff --git a/src/test/test_data.c b/src/test/test_data.c index 5f0f7cba01..0c51c98f1e 100644 --- a/src/test/test_data.c +++ b/src/test/test_data.c @@ -3,8 +3,18 @@ * Copyright (c) 2007-2013, The Tor Project, Inc. */ /* See LICENSE for licensing information */ +/* Our unit test expect that the AUTHORITY_CERT_* public keys will sort + * in this order. */ +#define AUTHORITY_CERT_A AUTHORITY_CERT_3 +#define AUTHORITY_CERT_B AUTHORITY_CERT_1 +#define AUTHORITY_CERT_C AUTHORITY_CERT_2 + +#define AUTHORITY_SIGNKEY_A AUTHORITY_SIGNKEY_3 +#define AUTHORITY_SIGNKEY_B AUTHORITY_SIGNKEY_1 +#define AUTHORITY_SIGNKEY_C AUTHORITY_SIGNKEY_2 + /** First of 3 example authority certificates for unit testing. */ -const char AUTHORITY_CERT_1[] = +const char AUTHORITY_CERT_A[] = "dir-key-certificate-version 3\n" "fingerprint D867ACF56A9D229B35C25F0090BC9867E906BE69\n" "dir-key-published 2008-12-12 18:07:24\n" @@ -46,7 +56,7 @@ const char AUTHORITY_CERT_1[] = "-----END SIGNATURE-----\n"; /** The private signing key for AUTHORITY_CERT_1 */ -const char AUTHORITY_SIGNKEY_1[] = +const char AUTHORITY_SIGNKEY_A[] = "-----BEGIN RSA PRIVATE KEY-----\n" "MIICWwIBAAKBgQCz0lCJ8rhLujVdzY6M6ZWp4iBAc0FxI79cff/pqp8GQAaWFZrs\n" "vQPJ8XqMmN7GRbJ2MDVvyGYwIBtt6RJnr7txfi+JsjI42mujkZdzIEWEOIJrhaqX\n" @@ -63,111 +73,128 @@ const char AUTHORITY_SIGNKEY_1[] = "Yx4lqK0ca5IkTp3HevwnlWaJgbaOTUspCVshzJBhDA==\n" "-----END RSA PRIVATE KEY-----\n"; +const char AUTHORITY_SIGNKEY_A_DIGEST[] = + "CBF56A83368A5150F1A9AAADAFB4D77F8C4170E2"; +const char AUTHORITY_SIGNKEY_A_DIGEST256[] = + "AF7C5468DBE3BA54A052726038D7F15F3C4CA511B1952645B3D96D83A8DFB51C"; + /** Second of 3 example authority certificates for unit testing. */ -const char AUTHORITY_CERT_2[] = +const char AUTHORITY_CERT_B[] = "dir-key-certificate-version 3\n" -"fingerprint 4D44AE0470B9E88FD4558EFEC82698FB33715400\n" -"dir-key-published 2007-06-13 16:52:32\n" -"dir-key-expires 2008-06-13 16:52:32\n" +"fingerprint AD011E25302925A9D39A80E0E32576442E956467\n" +"dir-key-published 2013-11-14 14:12:05\n" +"dir-key-expires 2014-11-14 14:12:05\n" "dir-identity-key\n" "-----BEGIN RSA PUBLIC KEY-----\n" -"MIIBigKCAYEAqukDwQRm1Oy1pPY+7GNRnRNFJzEVPUBfJwC4tBH19tkvdRQPuIGI\n" -"2jiTy/rmZ6CLcl1G0oulSgxfKEX75QdptOasZu+rKUrRRSxx0QrXhs9a7up0rpXh\n" -"13fw3mh1Vl/As3rJYF30Hjk01BTOJMxi/HY2y0ALQytFWjiMGY74A9Y6+uDcHkB2\n" -"KflBjxIl8zpCsXsTTnUhN5kXqaOOnK46XaUShSpXsyOxTMJXuJEtgLz9XCyA8XjW\n" -"d75QLHucEnlTqxUAdI5YSN2KIlIJiySCVnAorDpJey2mE9VncpHQWMCv/FPFdnSU\n" -"EMMPUc4bBShcoNFf0mMJeV2sv+dBkgKAL0GLM19PuJIThJhfN/B6+YQTxw4HEpPV\n" -"plfUqYRN0fYC+5hCTS6rroO/uCfDR7NBtoeDNm9dQrvjfk3b/Mywah1rdWNjnVqs\n" -"tPJaz3fc/CVBOUUexhmyktgLuwSNEYIQilQ+BydkWN/4RObhV+YSV5BgekEDVaoS\n" -"RHw4IbYBDHVxAgMBAAE=\n" +"MIIBigKCAYEAyXYEMlGNRAixXdg65xf2WPkskYj2Wo8ysKMTls1JCXdIOAPvC2k2\n" +"+AC6i3x9JHzUgCjWr4Jd5PSi7ODGyFC543igYl4wzkxNTU2L+SQ+hMe9qbEuUNhH\n" +"sRR0xofdoH//3UuKj+HXEiMhhHbRWQGtWFuJqtGBruJqjZqIGOrp5nFjdlP0R98n\n" +"Rx5wWlPgdJzifkXjKouu4mV+KzLl7f0gAtngA9DkSjt1wzga5IlL/lxDciD0SyJU\n" +"tKMmls056omrZNbTnBxnY2pOlq9nx/zFrt/KQm1fTAQMjMBCf9KnDIV7NhaaHx7F\n" +"7Nk8L7Hha353SvR+bsOFpiu05/EMZFTTIhO3MhUxZiCVZ0hKXvW1xe0HoGC5wbB+\n" +"NyXu8oa4fIKLJ+WJ8Z60BNc0DcxJiQOf1eolGM/qrBul1lFZznds5/7182d+nF2W\n" +"+bEjSm0fgXIxPfSD/7hB0FvgtmB3TXybHGBfPZgX0sTzFB6LNtP0BHicRoMXKdLF\n" +"hM3tgIjEAsoZAgMBAAE=\n" "-----END RSA PUBLIC KEY-----\n" "dir-signing-key\n" "-----BEGIN RSA PUBLIC KEY-----\n" -"MIGJAoGBAOu3dgrQth3iqvi/UzfywaANw0bBUuMOBhnMBeiLEcRLneJHUJkVvrpR\n" -"/EDQkdMov1e7CX6aqBKygVnbDNYjJ+bcQej8MKpuuW+zIknnz5lfnAVZO5uAmo3Y\n" -"DpG574oQ2FFMdkWHSBloIRxSj/E4Jn1M2qJjElBXP0E33Ka/Noo7AgMBAAE=\n" +"MIGJAoGBAJ567PZIGG/mYWEY4szYi/C5XXvf0BkquzKTHKrqVjysZEys9giz56Gv\n" +"B08kIRxsxYKEWkq60rv0xtTc1WyEMcDpV1WLU0KSTQSVXzLu7BT8jbTsWzGsxdTV\n" +"TdeyOirwHh8Cyyon5lppuMH5twUHrL5O7pWWbxjjrQjAHCn3gd+NAgMBAAE=\n" "-----END RSA PUBLIC KEY-----\n" +"dir-key-crosscert\n" +"-----BEGIN ID SIGNATURE-----\n" +"OC+gaukd4K7xJOsgTPbRhacf5mDUGxsu3ho/J1oJdtni4CK9WscVs6/Goj1o5Lot\n" +"H1nCAMaR96Jnqq5c63Aaj1sEXdeYHlu5cI7YHgtGI5MmtjiUNXUCWMjCwSQYwGKe\n" +"2YDYGAKAGt97n7XMKhJWGjAmv1TgmK3DvL1jt/aazL8=\n" +"-----END ID SIGNATURE-----\n" "dir-key-certification\n" "-----BEGIN SIGNATURE-----\n" -"Fv0Li68QUdAiChY3OklZOakHzwXAUfCzDNxkqe+HLC0n6ZECE9ZCvLVo69XmgVhH\n" -"L5qYr2rxT6QpF+9yuOHbN9gWn8EsDcli06MlhX9TUt/IYVxHa/9tJwNoTfEw2w2D\n" -"tyHhWm94IfOK7/Sea6jHnjckl80X+kk0ZNtAGs3/6fP4iltKNGXnvBwfgLpEgW7X\n" -"NpDl0OLeDuA79zem2GogwQZQdoDbePByU0TJVx9jYi2Bzx2Nb2H0hRTPP6+dY0HQ\n" -"MHb7yyyTQRad5iAUnExKhhyt22p7X3a6lgkAhq4YrNn/zVPkpnT2dzjsOydTHOW8\n" -"2BQs33QlGNe095i47pJBDYsUgmJaXfqB/RG6dFg7jwIsc3/7dZcvcqfxY7wKcD/T\n" -"wtogCIKxDvWbZn7f0hqYkT6uQC8Zom8bcnedmyzufOZCyA2SqQ2wvio6lznR4RIB\n" -"a8qDHR0tPS9/VkqTPcvUWCZeY3UiDeWPjoK1nea1pz6DHDWglKPx86a0amjjayZQ\n" -"-----END SIGNATURE-----\n"; +"BddmCKsvS6VoFXIf9Aj9OZnfyVCx527517QtsQHN+NaVm20LzUkJ5MWGXYx4wgh3\n" +"ExsHvVQguiVfnonkQpEHHKg+TbldlkuDhIdlb9f7dL7V3HLCsEdmS1c3A+TEyrPH\n" +"i44p6QB5IMFAdgUMV/9ueKMh7pMoam6VNakMOd+Axx9BSJTrCRzcepjtM4Z0cPsj\n" +"nmDgZi0df1+ca1t+HnuWyt3trxlqoUxRcPZKz28kEFDJsgnRNvoHrIvNTuy9qY4x\n" +"rONnPuLr5kTO7VQVVZxgxt6WX3p6d8tj+WYHubydr2pG0dwu2vGDTy4qXvDIm/I4\n" +"Gyo6OAoPbYV8fl0584EgiEbAWcX/Pze8mXr9lmXbf73xbSBHqveAs0UfB+4sBI98\n" +"v4ax4NZkGs8cCIfugtAOLgZE0WCh/TQYnQ3PFcrUtj0RW+tM1z7S8P3UfEVBHVkJ\n" +"8SqSB+pbsY6PwMuy6TC3WujW7gmjVanbwkbW19El9l9jRzteFerz7grG/WQkshqF\n" + "-----END SIGNATURE-----\n"; /** The private signing key for AUTHORITY_CERT_2 */ -const char AUTHORITY_SIGNKEY_2[] = +const char AUTHORITY_SIGNKEY_B[] = "-----BEGIN RSA PRIVATE KEY-----\n" -"MIICXgIBAAKBgQDrt3YK0LYd4qr4v1M38sGgDcNGwVLjDgYZzAXoixHES53iR1CZ\n" -"Fb66UfxA0JHTKL9Xuwl+mqgSsoFZ2wzWIyfm3EHo/DCqbrlvsyJJ58+ZX5wFWTub\n" -"gJqN2A6Rue+KENhRTHZFh0gZaCEcUo/xOCZ9TNqiYxJQVz9BN9ymvzaKOwIDAQAB\n" -"AoGAJ+I9/ex8tCfTSA2PdisEKiHKBeHWNYb870Z/RW6qje1BhLUOZSixwfL3XLwt\n" -"wG3nml+SZrKid69uhZaz4FPIf0tqCgURf6dDrF5vuzzr7VLVqkZHYSBp0vE6bu0R\n" -"Sgc5QNxI2talgc4bsp0O0C+Zd4n3Yto0pXl/I6NHVAxlFBECQQD2mahkY+QEHWPV\n" -"yRY3w3HhRmWBcrkY2zVyvPpqfn/sdHRPYW/yj4Xr/d1CO9VyFmEs4k324lIvu6LT\n" -"WDdpPlcJAkEA9LOZv5aNeAm8ckvvXH7iv8KiONiSz0n9wlisxMhNYTEkOCo1g7jG\n" -"AX5ZknRC9s4sWCPOBpMhloUvemdQ5FCEIwJBAMqCFwoSCf7jD8hRcUBr7QodoF/0\n" -"kVJ7OeI2lMJ9jZnlbFp/3snn2Qeam2e38SnWfQi582KKKwnt4eIDMMXpntkCQQDI\n" -"v1Lh11wl3y7nQZ6T7lCNatp08k+2mQgCWYcbRQweMRd6sD4I2xwt+372ZETPfyLo\n" -"CC+sOyYx+v+RVpMJS3irAkEA6l98nMteZKmhOgyKSjdolP+ahpZunb+WnCdAtP97\n" -"rjZyXmEZS3oe7TRCDD28GAGMmxSDvNfOOpyn14ishEs5AQ==\n" +"MIICWwIBAAKBgQCeeuz2SBhv5mFhGOLM2IvwuV1739AZKrsykxyq6lY8rGRMrPYI\n" +"s+ehrwdPJCEcbMWChFpKutK79MbU3NVshDHA6VdVi1NCkk0ElV8y7uwU/I207Fsx\n" +"rMXU1U3Xsjoq8B4fAssqJ+ZaabjB+bcFB6y+Tu6Vlm8Y460IwBwp94HfjQIDAQAB\n" +"AoGAfHQ4ZmfTmPyoeGHcqdVcgBxxh3gJqdnezCavGqGQO3F+CqDBTbBKNLSI3uOW\n" +"hQX+TTK23Xy9RRFCm6MYj3F4x7OOrSHSFyhMmzRnAZi3zGbtQZn30XoqTwCmVevY\n" +"p5JbVvhP2BJcvdsyQhiIG23FRQ7MMHWtksAxmovTto1h/hkCQQDNCfMqSztgJZDn\n" +"JSf5ASHBOw8QzfZBeYi3hqfiDtAN1RxT1uQnEiFQFJqwCz5lCbcwVrfQbrrk5M+h\n" +"ooYrX7tTAkEAxd6Tl0N0WM3zCKz+3/Hoiyty6olnnpzNoPCg7LLBJcetABQi0KUv\n" +"swYWlKP3eOFZkiBzTqa9nBK7eYLKV3d9nwJAKNM3WI98Nguky3FJgTnpd6kDuevY\n" +"gXbqcuhb2xXp9Sceqc7axLDGc0R2/GBwvvttPzG1DcpOai7o7J0Iq/A2wwJAYuKI\n" +"/99GFdtWyc8q0OAkRui/1VY14p6aZQPcaG4s+KSBYLivbXYgEGfKgR4wXsi/6rcs\n" +"6PGLcKQr7N3gITYmIQJAaQn6djUWygCn1noKyWU+Sa7G5qqU2GWkLq9dMaRLm1/I\n" +"nqi+2K1mN15rra0QtFVqSH4JXr8h3KAGyU45voGM7A==\n" "-----END RSA PRIVATE KEY-----\n"; /** Third of 3 example authority certificates for unit testing. */ -const char AUTHORITY_CERT_3[] = +const char AUTHORITY_CERT_C[] = "dir-key-certificate-version 3\n" -"fingerprint ED3719BF554DE9D7D59F5CA5A4F5AD121D020ED9\n" -"dir-key-published 2007-06-13 16:52:40\n" -"dir-key-expires 2008-06-13 16:52:40\n" +"fingerprint 628C2086EC29C9D26E638C5A8B2065BFBD35829B\n" +"dir-key-published 2013-11-14 14:12:18\n" +"dir-key-expires 2014-11-14 14:12:18\n" "dir-identity-key\n" "-----BEGIN RSA PUBLIC KEY-----\n" -"MIIBigKCAYEAtB+yw4BNxtZAG4cPaedkhWNmeij7IuNWmXjh58ZYEGurvGyHs1w4\n" -"QlwNYI2UftSIeIGdWZ5fJ17h9P3xvO6eeJuOt4KPrNOxUbSGrELEx1Lje1fDAJ1X\n" -"SvN+dvptusxtyFUr8afgTPrFIvYuazQ6q/Rw+NDagjmDx3h/A/enihpBnjwzeH8j\n" -"Xzu7b+HKnzFnNfveTDdvSy0NSC6tCOnrfXo31XbXRXtlesnMIpbJClUcAv55eyai\n" -"/PrVPCCUz8mk0sQnn2Xhv1YJmwOlQTGMfg0a0kWLmh+UWcHsGQ4VWxBZJcuzgFHG\n" -"hu2/Fz6DXSpX5Q6B9HKoGmnH1oBh24l0kUW1jL8BxPY4YDU1Lt5t3qgcDn9dXYcI\n" -"o8VvyI0ecSc26Q2PYFWX1hpN4VIBZ8uGaW3IpyTdNiRq0g3iMGRFEXcDlWuyMB9E\n" -"EbSM7m/79V/z7SjDd75EP8Z0qDPESEVB8a8LbuSJtzFVE0KHd7RzkIEN5sorXspZ\n" -"/THukftSmkIvAgMBAAE=\n" +"MIIBigKCAYEAuzPA82lRVUAc1uZgfDehhK0rBU5xt+qhJXUSH0DxsuocYCLW//q+\n" +"7+L7q9SochqZK3R5+SxJaZRlVK4rAeIHsxXFxsnGvuqasGM3he80EV1RpVRkvLaO\n" +"2dDmHcfEjYBadft2DEq811yvqSRqbFXmK0hLucA6LI6NnEw9VNWlguaV6ACVLyKQ\n" +"iYVFz2JOJIAi0Zz57WZg7eHypUAGoyXjtYTJPsh6pUe/0NLFJVd3JHcJX+bNqU2a\n" +"QU37r+CQ9f3T+8fZGJQ/CXNnYUNHa0j+toOFuPEiZBBh8C4PE7FJWjidvhe9uI7T\n" +"Py41RZhy8e05MAQmUBNRKBHWPKHoy2zWZZxTkcfWFdJJz/dzsNrIjrqf2fYId9To\n" +"fDpHzYd/UjzZaaVYRVS/Oyf3pN8DKw8LMhEArS0X9pblPVkWWjmYMU6f0VR7pelc\n" +"gGYuML3gOiKdNbeMWgAv3HNRsVsuW0HZLrhXUGYzTRPJ/GxVCwA/NmYgMTNVWRwF\n" +"7M78YHpayyEPAgMBAAE=\n" "-----END RSA PUBLIC KEY-----\n" "dir-signing-key\n" "-----BEGIN RSA PUBLIC KEY-----\n" -"MIGJAoGBANrSZlUq38Boz3iuUOydYTJV57rTbq1bz805FP2QG2Z+2bwpgKIOZag/\n" -"gN2A1ySJaIYLgZIg9irxrLkqlY/UAjC23y6V9fJXP1S3TXoqLmHleW8PsaDLuwTo\n" -"hCWaR61Mx9WG7IXcodn2Z7RiCfZpSW4Rztbk5WtjQa5jPXSFOuBJAgMBAAE=\n" +"MIGJAoGBANESf/hRRWCK3TLQyNb9Y42tYedCORUc8Rl+Q4wrvdz3R0TNr6rztE9N\n" +"u8v3Wbvjtiqm1xL1I5PaOObFQQj61QZxKiCm1yU4eFH15dNmcvBEy5BjEXVYiDgy\n" +"zKRyePzjHYQIZF3ZaQTABUplkXVpY0YvAurluhEy+dKEvZMwWFZTAgMBAAE=\n" "-----END RSA PUBLIC KEY-----\n" +"dir-key-crosscert\n" +"-----BEGIN ID SIGNATURE-----\n" +"NHNBya6Dt7Ww3qSGA0DBEl6pZFBzmYXM+QdqF+ESpdyYCQ54EYimaxl4VcXoGaxy\n" +"xk8/VOXPC6h7hVnTWDTsC86G6eXug1yzpd/uhQbcDJMH5q8/Yg5WXGOnGhMWNCBh\n" +"u2UmbtAjdjLrObQaB50FfOpuOV9kdG4SEzaPUBR2ayU=\n" +"-----END ID SIGNATURE-----\n" "dir-key-certification\n" "-----BEGIN SIGNATURE-----\n" -"UNXZy+4OQ8iat+gw+vg2ynvKj2BYbqZt+EAZAV3rmw6gux44U9TLRECRd6LsA08N\n" -"4+Vz01TU81xqMgfrUy94ei2YvcfpO8art9/muWHTP9SmOX8S1uqDqLWA+n723C9A\n" -"HyVXn4aINncO2081gJcIW5+Ul8WTCeZe/n3LVPTCKbTdqxvmrPUdCWlJTQUmb19M\n" -"T+kcCjaEfgQGLC+Y2MHqYe/nxz+aBKqpjiWUDdjc35va6r/2e3c0jGi1B1xRZxN1\n" -"xThPZ+CifjDoWBxJdDGlIfZRK1lMnOCJY9w9ibTXQ1UnvE4whFvmB55/t9/XLq4q\n" -"3pnZz0H7funey3+ilmTxDohoAYT1GX+4a+3xYH07UmAFqlTzqKClj84XEHn+Cer7\n" -"Nun9kJlJFuBgUpQjwCkzedFZKKLOHgB2h7trJfnqcBpAM8Rup1Bb5u/RcBx9gy1q\n" -"pMc65FviIrc/Q5TUku6NNbCbnGll1599PvWuUzkG42lJ17V6psKHIsqGtVdHlCUc\n" +"NocTkLl9iKglVo+yrpY0slsqgPviuScMyEfOJ3i65KeJb4Dr1huIs0Fip40zFD8D\n" +"cz/SYu09FbANuRwBJIRdVWZLLwVFLBj5F8U65iJRAPBw/O/xgSVBvWoOhBUZqmJA\n" +"Jp1IUutQHYFfnAOO9za4r8Ox6yPaOWF9Ks5gL0kU/fI8Bdi5E9p3e9fMtoM7hROg\n" +"oX1AoV/za3LcM0oMsGsdXQ7B8vRqY0eUX523kpRpF1fUDyvBUvvMsXdZDN6anCV6\n" +"NtSq2UaM/msTX1oQ8gzyD1gMXH0Ek26YMhd+6WZE6KUeb1x5HJgXtKtYzMLB6nQM\n" +"4Q/OA4NND/Veflofy6xx8uzXe8H+MoUHK9WiORtwqvBl0E9qk6SVCuo4ipR4Ybgk\n" +"PAFOXA58j80dlNYYEVgV8MXF1Y/g/thuXlf2dWiLAExdHTtE0AzC4quWshegaImC\n" +"4aziHeA43TRDszAXcJorREAM0AhSxp3aWDde4Jt46ODOJR8t+gHreks29eDttEIn\n" "-----END SIGNATURE-----\n"; /** The private signing key for AUTHORITY_CERT_3 */ -const char AUTHORITY_SIGNKEY_3[] = +const char AUTHORITY_SIGNKEY_C[] = "-----BEGIN RSA PRIVATE KEY-----\n" -"MIICXgIBAAKBgQDa0mZVKt/AaM94rlDsnWEyVee6026tW8/NORT9kBtmftm8KYCi\n" -"DmWoP4DdgNckiWiGC4GSIPYq8ay5KpWP1AIwtt8ulfXyVz9Ut016Ki5h5XlvD7Gg\n" -"y7sE6IQlmketTMfVhuyF3KHZ9me0Ygn2aUluEc7W5OVrY0GuYz10hTrgSQIDAQAB\n" -"AoGBAIyoeG1AnQmildKeQpiGZackf0uhg2BeRwpFKg//5Q0Sd0Wza+M/2+q1v1Ei\n" -"86ihxxV7KfPTykk6hmuUSwVkI28Z+5J9NYTr35EzPiUlqpo0iclTkFqrlbqSPULx\n" -"9fQhvcOGv1c0m5CnYrHsM8eu3tagLg+6OE4abLOYX4Az5pkxAkEA/NwHhVaVJrXH\n" -"lGDrRAfGtaD5Tzeeg1H9DNZi5lmFiSNR0O11sgDLkiZNP5oM8knyqo8Gq08hwxEb\n" -"yqMXM3XtJQJBAN2KJbFhOjDIkvJyYvbmcP6P7vV2c9j+oUTKkFMF7vvfWunxMi9j\n" -"ghbdUKgl7tU0VFpw7ufDDD0pkN6sua3gp1UCQQCvNzTK861U7p/GtMYyFQVf9JTt\n" -"jMf9jYHBNInBvwTme6AFG5bz6tMlif77dJ9GAXHzODrR2Hq3thJA/3RjR3M1AkBg\n" -"+6M4ncmtpYC+5lhwob0Bk90WU/6vFflfdhXsYoKWfNb95vsDR9qhS82Nbt25NClh\n" -"VmMfzoFDHTkwYgj/F4PpAkEA+RaaSRP7BmbvFNqvlm8J/m0RVdAH4+p/Q5Z5u6Yo\n" -"N7xC/gFi0qFPGKsDvD2CncAYmt+KNsd8S0JGDN4eieKn+Q==\n" +"MIICXAIBAAKBgQDREn/4UUVgit0y0MjW/WONrWHnQjkVHPEZfkOMK73c90dEza+q\n" +"87RPTbvL91m747YqptcS9SOT2jjmxUEI+tUGcSogptclOHhR9eXTZnLwRMuQYxF1\n" +"WIg4Msykcnj84x2ECGRd2WkEwAVKZZF1aWNGLwLq5boRMvnShL2TMFhWUwIDAQAB\n" +"AoGAU68L+eDN3C65CzX2rdcOmg7kOSSQpJrJBmM7tkdr3546sJeD0PFrIrMCkEmZ\n" +"aVNj/v545+WnL+8RB4280lNUIF4AMNaMZUL+4FAtwekqWua3QvvqgRMjCdG3/h/d\n" +"bOAUiiKKEimflTaIVHNVSCvOIntftOu3PhebctuabnZzg0ECQQD9i+FX7M9UXT1A\n" +"bVm+bRIJuQtG+u9jD3VxrvHsmh0QnOAL3oa/ofTCwoTJLZs8Qy0GeAoJNf28rY1q\n" +"AgNMEeEXAkEA0xhxNX2fDQ2yvKwPkPMrRycJVWry+KHvSZG2+XYh+V5sVGQ5H7Gu\n" +"krc6IzRZlIKQhEGktkw8ih0DEHQbAihiJQJBAKi/SnFcePjrPXL91Hb63MB/2dOZ\n" +"+21wwnexOe6A+8ssvajop8IvJlnhYMMMiX7oLrVZe0R6HLBQyge94zfjxm0CQGye\n" +"dRIrE34qAEBo4JGbLjesdHcJUwBwgqn+WoI+MPkZhvBdqa8PRF6l/TpEI5vxGt+S\n" +"z2gmDjia+QqsU4FmuikCQDDOs85uwNSKJFax9XMzd1qd1QwX20F8lvnOsWErXiDw\n" +"Fy2+rmIRHoSxn4D+rE5ivqkO99E9jAlz+uuQz/6WqwE=\n" "-----END RSA PRIVATE KEY-----\n"; diff --git a/src/test/test_dir.c b/src/test/test_dir.c index 56ac3b34c7..c03b63be27 100644 --- a/src/test/test_dir.c +++ b/src/test/test_dir.c @@ -11,6 +11,7 @@ #define ROUTER_PRIVATE #define ROUTERLIST_PRIVATE #define HIBERNATE_PRIVATE +#define NETWORKSTATUS_PRIVATE #include "or.h" #include "config.h" #include "directory.h" @@ -97,7 +98,6 @@ test_dir_formats(void) get_platform_str(platform, sizeof(platform)); r1 = tor_malloc_zero(sizeof(routerinfo_t)); - r1->address = tor_strdup("18.244.0.1"); r1->addr = 0xc0a80001u; /* 192.168.0.1 */ r1->cache_info.published_on = 0; r1->or_port = 9000; @@ -124,7 +124,6 @@ test_dir_formats(void) ex2->maskbits = 8; ex2->prt_min = ex2->prt_max = 24; r2 = tor_malloc_zero(sizeof(routerinfo_t)); - r2->address = tor_strdup("1.1.1.1"); r2->addr = 0x0a030201u; /* 10.3.2.1 */ r2->platform = tor_strdup(platform); r2->cache_info.published_on = 5; @@ -153,7 +152,7 @@ test_dir_formats(void) tor_free(options->ContactInfo); test_assert(buf); - strlcpy(buf2, "router Magri 18.244.0.1 9000 0 9003\n" + strlcpy(buf2, "router Magri 192.168.0.1 9000 0 9003\n" "or-address [1:2:3:4::]:9999\n" "platform Tor "VERSION" on ", sizeof(buf2)); strlcat(buf2, get_uname(), sizeof(buf2)); @@ -187,7 +186,7 @@ test_dir_formats(void) cp = buf; rp1 = router_parse_entry_from_string((const char*)cp,NULL,1,0,NULL); test_assert(rp1); - test_streq(rp1->address, r1->address); + test_eq(rp1->addr, r1->addr); test_eq(rp1->or_port, r1->or_port); //test_eq(rp1->dir_port, r1->dir_port); test_eq(rp1->bandwidthrate, r1->bandwidthrate); @@ -196,9 +195,10 @@ test_dir_formats(void) test_assert(crypto_pk_cmp_keys(rp1->onion_pkey, pk1) == 0); test_assert(crypto_pk_cmp_keys(rp1->identity_pkey, pk2) == 0); //test_assert(rp1->exit_policy == NULL); + tor_free(buf); strlcpy(buf2, - "router Fred 1.1.1.1 9005 0 0\n" + "router Fred 10.3.2.1 9005 0 0\n" "platform Tor "VERSION" on ", sizeof(buf2)); strlcat(buf2, get_uname(), sizeof(buf2)); strlcat(buf2, "\n" @@ -214,8 +214,10 @@ test_dir_formats(void) strlcat(buf2, "signing-key\n", sizeof(buf2)); strlcat(buf2, pk1_str, sizeof(buf2)); strlcat(buf2, "hidden-service-dir\n", sizeof(buf2)); +#ifdef CURVE25519_ENABLED strlcat(buf2, "ntor-onion-key " "skyinAnvardNostarsNomoonNowindormistsorsnow=\n", sizeof(buf2)); +#endif strlcat(buf2, "accept *:80\nreject 18.0.0.0/8:24\n", sizeof(buf2)); strlcat(buf2, "router-signature\n", sizeof(buf2)); @@ -229,15 +231,17 @@ test_dir_formats(void) cp = buf; rp2 = router_parse_entry_from_string((const char*)cp,NULL,1,0,NULL); test_assert(rp2); - test_streq(rp2->address, r2->address); + test_eq(rp2->addr, r2->addr); test_eq(rp2->or_port, r2->or_port); test_eq(rp2->dir_port, r2->dir_port); test_eq(rp2->bandwidthrate, r2->bandwidthrate); test_eq(rp2->bandwidthburst, r2->bandwidthburst); test_eq(rp2->bandwidthcapacity, r2->bandwidthcapacity); +#ifdef CURVE25519_ENABLED test_memeq(rp2->onion_curve25519_pkey->public_key, r2->onion_curve25519_pkey->public_key, CURVE25519_PUBKEY_LEN); +#endif test_assert(crypto_pk_cmp_keys(rp2->onion_pkey, pk2) == 0); test_assert(crypto_pk_cmp_keys(rp2->identity_pkey, pk1) == 0); @@ -275,6 +279,8 @@ test_dir_formats(void) routerinfo_free(r1); if (r2) routerinfo_free(r2); + if (rp2) + routerinfo_free(rp2); tor_free(buf); tor_free(pk1_str); @@ -937,7 +943,7 @@ gen_routerstatus_for_v3ns(int idx, time_t now) tor_addr_copy(&rs->ipv6_addr, &addr_ipv6); rs->ipv6_orport = 4711; rs->is_exit = rs->is_stable = rs->is_fast = rs->is_flagged_running = - rs->is_valid = rs->is_v2_dir = rs->is_possible_guard = 1; + rs->is_valid = rs->is_possible_guard = 1; break; case 2: /* Generate the third routerstatus. */ @@ -952,7 +958,7 @@ gen_routerstatus_for_v3ns(int idx, time_t now) rs->or_port = 400; rs->dir_port = 9999; rs->is_authority = rs->is_exit = rs->is_stable = rs->is_fast = - rs->is_flagged_running = rs->is_valid = rs->is_v2_dir = + rs->is_flagged_running = rs->is_valid = rs->is_possible_guard = 1; break; case 3: @@ -1009,16 +1015,14 @@ vote_tweaks_for_v3ns(networkstatus_t *v, int voter, time_t now) /* Monkey around with the list a bit */ vrs = smartlist_get(v->routerstatus_list, 2); smartlist_del_keeporder(v->routerstatus_list, 2); - tor_free(vrs->version); - tor_free(vrs); + vote_routerstatus_free(vrs); vrs = smartlist_get(v->routerstatus_list, 0); vrs->status.is_fast = 1; if (voter == 3) { vrs = smartlist_get(v->routerstatus_list, 0); smartlist_del_keeporder(v->routerstatus_list, 0); - tor_free(vrs->version); - tor_free(vrs); + vote_routerstatus_free(vrs); vrs = smartlist_get(v->routerstatus_list, 0); memset(vrs->status.descriptor_digest, (int)'Z', DIGEST_LEN); test_assert(router_add_to_routerlist( @@ -1061,7 +1065,8 @@ test_vrs_for_v3ns(vote_routerstatus_t *vrs, int voter, time_t now) test_eq(rs->addr, 0x99008801); test_eq(rs->or_port, 443); test_eq(rs->dir_port, 8000); - test_eq(vrs->flags, U64_LITERAL(16)); // no flags except "running" + /* no flags except "running" (16) and "v2dir" (64) */ + tt_u64_op(vrs->flags, ==, U64_LITERAL(80)); } else if (tor_memeq(rs->identity_digest, "\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5" "\x5\x5\x5\x5", @@ -1086,10 +1091,11 @@ test_vrs_for_v3ns(vote_routerstatus_t *vrs, int voter, time_t now) test_assert(tor_addr_eq(&rs->ipv6_addr, &addr_ipv6)); test_eq(rs->ipv6_orport, 4711); if (voter == 1) { - test_eq(vrs->flags, U64_LITERAL(254)); // all flags except "authority." + /* all except "authority" (1) and "v2dir" (64) */ + tt_u64_op(vrs->flags, ==, U64_LITERAL(190)); } else { - /* 1023 - authority(1) - madeofcheese(16) - madeoftin(32) */ - test_eq(vrs->flags, U64_LITERAL(974)); + /* 1023 - authority(1) - madeofcheese(16) - madeoftin(32) - v2dir(256) */ + tt_u64_op(vrs->flags, ==, U64_LITERAL(718)); } } else if (tor_memeq(rs->identity_digest, "\x33\x33\x33\x33\x33\x33\x33\x33\x33\x33" @@ -1157,7 +1163,6 @@ test_routerstatus_for_v3ns(routerstatus_t *rs, time_t now) test_assert(!rs->is_stable); /* (If it wasn't running it wouldn't be here) */ test_assert(rs->is_flagged_running); - test_assert(!rs->is_v2_dir); test_assert(!rs->is_valid); test_assert(!rs->is_named); /* XXXX check version */ @@ -1184,7 +1189,6 @@ test_routerstatus_for_v3ns(routerstatus_t *rs, time_t now) test_assert(rs->is_possible_guard); test_assert(rs->is_stable); test_assert(rs->is_flagged_running); - test_assert(rs->is_v2_dir); test_assert(rs->is_valid); test_assert(!rs->is_named); /* XXXX check version */ @@ -1226,7 +1230,8 @@ test_a_networkstatus( vote_routerstatus_t *vrs; routerstatus_t *rs; int idx, n_rs, n_vrs; - char *v1_text=NULL, *v2_text=NULL, *v3_text=NULL, *consensus_text=NULL, *cp; + char *v1_text=NULL, *v2_text=NULL, *v3_text=NULL, *consensus_text=NULL, + *cp=NULL; smartlist_t *votes = smartlist_new(); /* For generating the two other consensuses. */ @@ -1244,7 +1249,6 @@ test_a_networkstatus( /* Parse certificates and keys. */ cert1 = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL); test_assert(cert1); - test_assert(cert1->is_cross_certified); cert2 = authority_cert_parse_from_string(AUTHORITY_CERT_2, NULL); test_assert(cert2); cert3 = authority_cert_parse_from_string(AUTHORITY_CERT_3, NULL); @@ -1358,7 +1362,8 @@ test_a_networkstatus( vote->dist_seconds = 300; authority_cert_free(vote->cert); vote->cert = authority_cert_dup(cert2); - vote->net_params = smartlist_new(); + SMARTLIST_FOREACH(vote->net_params, char *, c, tor_free(c)); + smartlist_clear(vote->net_params); smartlist_split_string(vote->net_params, "bar=2000000000 circuitwindow=20", NULL, 0, 0); tor_free(vote->client_versions); @@ -1402,7 +1407,8 @@ test_a_networkstatus( vote->dist_seconds = 250; authority_cert_free(vote->cert); vote->cert = authority_cert_dup(cert3); - vote->net_params = smartlist_new(); + SMARTLIST_FOREACH(vote->net_params, char *, c, tor_free(c)); + smartlist_clear(vote->net_params); smartlist_split_string(vote->net_params, "circuitwindow=80 foo=660", NULL, 0, 0); smartlist_add(vote->supported_methods, tor_strdup("4")); @@ -1645,6 +1651,7 @@ test_a_networkstatus( } done: + tor_free(cp); smartlist_free(votes); tor_free(v1_text); tor_free(v2_text); @@ -1768,7 +1775,7 @@ test_dir_random_weighted(void *testdata) inp[i].u64 = vals[i]; total += vals[i]; } - tt_int_op(total, ==, 45); + tt_u64_op(total, ==, 45); for (i=0; i<n; ++i) { choice = choose_array_element_by_weight(inp, 10); tt_int_op(choice, >=, 0); @@ -1886,7 +1893,7 @@ gen_routerstatus_for_umbw(int idx, time_t now) tor_addr_copy(&rs->ipv6_addr, &addr_ipv6); rs->ipv6_orport = 4711; rs->is_exit = rs->is_stable = rs->is_fast = rs->is_flagged_running = - rs->is_valid = rs->is_v2_dir = rs->is_possible_guard = 1; + rs->is_valid = rs->is_possible_guard = 1; /* * This one has measured bandwidth above the clip cutoff, and * so shouldn't be clipped; we'll have to test that it isn't @@ -1909,7 +1916,7 @@ gen_routerstatus_for_umbw(int idx, time_t now) rs->or_port = 400; rs->dir_port = 9999; rs->is_authority = rs->is_exit = rs->is_stable = rs->is_fast = - rs->is_flagged_running = rs->is_valid = rs->is_v2_dir = + rs->is_flagged_running = rs->is_valid = rs->is_possible_guard = 1; /* * This one has unmeasured bandwidth above the clip cutoff, and @@ -1978,6 +1985,7 @@ vote_tweaks_for_umbw(networkstatus_t *v, int voter, time_t now) (void)now; test_assert(v->supported_methods); + SMARTLIST_FOREACH(v->supported_methods, char *, c, tor_free(c)); smartlist_clear(v->supported_methods); /* Method 17 is MIN_METHOD_TO_CLIP_UNMEASURED_BW_KB */ smartlist_split_string(v->supported_methods, @@ -2145,7 +2153,6 @@ test_routerstatus_for_umbw(routerstatus_t *rs, time_t now) test_assert(!rs->is_stable); /* (If it wasn't running it wouldn't be here) */ test_assert(rs->is_flagged_running); - test_assert(!rs->is_v2_dir); test_assert(!rs->is_valid); test_assert(!rs->is_named); /* This one should have measured bandwidth below the clip cutoff */ @@ -2176,7 +2183,6 @@ test_routerstatus_for_umbw(routerstatus_t *rs, time_t now) test_assert(rs->is_possible_guard); test_assert(rs->is_stable); test_assert(rs->is_flagged_running); - test_assert(rs->is_v2_dir); test_assert(rs->is_valid); test_assert(!rs->is_named); /* This one should have measured bandwidth above the clip cutoff */ @@ -2254,82 +2260,6 @@ test_dir_clip_unmeasured_bw_kb_alt(void) test_routerstatus_for_umbw); } -extern time_t time_of_process_start; /* from main.c */ - -static void -test_dir_v2_dir(void *arg) -{ - /* Runs in a forked process: acts like a v2 directory just enough to make and - * sign a v2 networkstatus opinion */ - - cached_dir_t *v2 = NULL; - or_options_t *options = get_options_mutable(); - crypto_pk_t *id_key = pk_generate(4); - (void) arg; - - options->ORPort_set = 1; /* So we believe we're a server. */ - options->DirPort_set = 1; - options->Address = tor_strdup("99.99.99.99"); - options->Nickname = tor_strdup("TestV2Auth"); - options->ContactInfo = tor_strdup("TestV2Auth <testv2auth@example.com>"); - { - /* Give it a DirPort */ - smartlist_t *ports = (smartlist_t *)get_configured_ports(); - port_cfg_t *port = tor_malloc_zero(sizeof(port_cfg_t)); - port->type = CONN_TYPE_DIR_LISTENER; - port->port = 9999; - smartlist_add(ports, port); - } - set_server_identity_key(id_key); - set_client_identity_key(id_key); - - /* Add a router. */ - { - was_router_added_t wra; - const char *msg = NULL; - routerinfo_t *r1 = tor_malloc_zero(sizeof(routerinfo_t)); - r1->address = tor_strdup("18.244.0.1"); - r1->addr = 0xc0a80001u; /* 192.168.0.1 */ - r1->cache_info.published_on = time(NULL)-60; - r1->or_port = 9000; - r1->dir_port = 9003; - tor_addr_parse(&r1->ipv6_addr, "1:2:3:4::"); - r1->ipv6_orport = 9999; - r1->onion_pkey = pk_generate(1); - r1->identity_pkey = pk_generate(2); - r1->bandwidthrate = 1000; - r1->bandwidthburst = 5000; - r1->bandwidthcapacity = 10000; - r1->exit_policy = NULL; - r1->nickname = tor_strdup("Magri"); - r1->platform = tor_strdup("Tor 0.2.7.7-gamma"); - r1->cache_info.routerlist_index = -1; - r1->cache_info.signed_descriptor_body = - router_dump_router_to_string(r1, r1->identity_pkey); - r1->cache_info.signed_descriptor_len = - strlen(r1->cache_info.signed_descriptor_body); - wra = router_add_to_routerlist(r1, &msg, 0, 0); - tt_int_op(wra, ==, ROUTER_ADDED_SUCCESSFULLY); - } - - /* Prevent call of rep_hist_note_router_unreachable(). */ - time_of_process_start = time(NULL); - - /* Make a directory so there's somewhere to store the thing */ -#ifdef _WIN32 - mkdir(get_fname("cached-status")); -#else - mkdir(get_fname("cached-status"), 0700); -#endif - - v2 = generate_v2_networkstatus_opinion(); - tt_assert(v2); - - done: - crypto_pk_free(id_key); - cached_dir_decref(v2); -} - static void test_dir_fmt_control_ns(void *arg) { @@ -2357,13 +2287,81 @@ test_dir_fmt_control_ns(void *arg) "r TetsuoMilk U3RhdGVseSwgcGx1bXAgQnVjayA " "TXVsbGlnYW4gY2FtZSB1cCBmcm8 2013-04-02 17:53:18 " "32.48.64.80 9001 9002\n" - "s Exit Fast Running\n" + "s Exit Fast Running V2Dir\n" "w Bandwidth=1000\n"); done: tor_free(s); } +static void +test_dir_http_handling(void *args) +{ + char *url = NULL; + (void)args; + + /* Parse http url tests: */ + /* Good headers */ + test_eq(parse_http_url("GET /tor/a/b/c.txt HTTP/1.1\r\n" + "Host: example.com\r\n" + "User-Agent: Mozilla/5.0 (Windows;" + " U; Windows NT 6.1; en-US; rv:1.9.1.5)\r\n", + &url), 0); + test_streq(url, "/tor/a/b/c.txt"); + tor_free(url); + + test_eq(parse_http_url("GET /tor/a/b/c.txt HTTP/1.0\r\n", &url), 0); + test_streq(url, "/tor/a/b/c.txt"); + tor_free(url); + + test_eq(parse_http_url("GET /tor/a/b/c.txt HTTP/1.600\r\n", &url), 0); + test_streq(url, "/tor/a/b/c.txt"); + tor_free(url); + + /* Should prepend '/tor/' to url if required */ + test_eq(parse_http_url("GET /a/b/c.txt HTTP/1.1\r\n" + "Host: example.com\r\n" + "User-Agent: Mozilla/5.0 (Windows;" + " U; Windows NT 6.1; en-US; rv:1.9.1.5)\r\n", + &url), 0); + test_streq(url, "/tor/a/b/c.txt"); + tor_free(url); + + /* Bad headers -- no HTTP/1.x*/ + test_eq(parse_http_url("GET /a/b/c.txt\r\n" + "Host: example.com\r\n" + "User-Agent: Mozilla/5.0 (Windows;" + " U; Windows NT 6.1; en-US; rv:1.9.1.5)\r\n", + &url), -1); + tt_assert(!url); + + /* Bad headers */ + test_eq(parse_http_url("GET /a/b/c.txt\r\n" + "Host: example.com\r\n" + "User-Agent: Mozilla/5.0 (Windows;" + " U; Windows NT 6.1; en-US; rv:1.9.1.5)\r\n", + &url), -1); + tt_assert(!url); + + test_eq(parse_http_url("GET /tor/a/b/c.txt", &url), -1); + tt_assert(!url); + + test_eq(parse_http_url("GET /tor/a/b/c.txt HTTP/1.1", &url), -1); + tt_assert(!url); + + test_eq(parse_http_url("GET /tor/a/b/c.txt HTTP/1.1x\r\n", &url), -1); + tt_assert(!url); + + test_eq(parse_http_url("GET /tor/a/b/c.txt HTTP/1.", &url), -1); + tt_assert(!url); + + test_eq(parse_http_url("GET /tor/a/b/c.txt HTTP/1.\r", &url), -1); + tt_assert(!url); + + done: + tor_free(url); +} + #define DIR_LEGACY(name) \ { #name, legacy_test_helper, TT_FORK, &legacy_setup, test_dir_ ## name } @@ -2384,8 +2382,8 @@ struct testcase_t dir_tests[] = { DIR(scale_bw, 0), DIR_LEGACY(clip_unmeasured_bw_kb), DIR_LEGACY(clip_unmeasured_bw_kb_alt), - DIR(v2_dir, TT_FORK), DIR(fmt_control_ns, 0), + DIR(http_handling, 0), END_OF_TESTCASES }; diff --git a/src/test/test_extorport.c b/src/test/test_extorport.c new file mode 100644 index 0000000000..93c8f77d5b --- /dev/null +++ b/src/test/test_extorport.c @@ -0,0 +1,607 @@ +/* Copyright (c) 2013, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#define CONNECTION_PRIVATE +#define EXT_ORPORT_PRIVATE +#define MAIN_PRIVATE +#include "or.h" +#include "buffers.h" +#include "connection.h" +#include "connection_or.h" +#include "config.h" +#include "control.h" +#include "ext_orport.h" +#include "main.h" +#include "test.h" + +/* Test connection_or_remove_from_ext_or_id_map and + * connection_or_set_ext_or_identifier */ +static void +test_ext_or_id_map(void *arg) +{ + or_connection_t *c1 = NULL, *c2 = NULL, *c3 = NULL; + char *idp = NULL, *idp2 = NULL; + (void)arg; + + /* pre-initialization */ + tt_ptr_op(NULL, ==, connection_or_get_by_ext_or_id("xxxxxxxxxxxxxxxxxxxx")); + + c1 = or_connection_new(CONN_TYPE_EXT_OR, AF_INET); + c2 = or_connection_new(CONN_TYPE_EXT_OR, AF_INET); + c3 = or_connection_new(CONN_TYPE_OR, AF_INET); + + tt_ptr_op(c1->ext_or_conn_id, !=, NULL); + tt_ptr_op(c2->ext_or_conn_id, !=, NULL); + tt_ptr_op(c3->ext_or_conn_id, ==, NULL); + + tt_ptr_op(c1, ==, connection_or_get_by_ext_or_id(c1->ext_or_conn_id)); + tt_ptr_op(c2, ==, connection_or_get_by_ext_or_id(c2->ext_or_conn_id)); + tt_ptr_op(NULL, ==, connection_or_get_by_ext_or_id("xxxxxxxxxxxxxxxxxxxx")); + + idp = tor_memdup(c2->ext_or_conn_id, EXT_OR_CONN_ID_LEN); + + /* Give c2 a new ID. */ + connection_or_set_ext_or_identifier(c2); + test_mem_op(idp, !=, c2->ext_or_conn_id, EXT_OR_CONN_ID_LEN); + idp2 = tor_memdup(c2->ext_or_conn_id, EXT_OR_CONN_ID_LEN); + tt_assert(!tor_digest_is_zero(idp2)); + + tt_ptr_op(NULL, ==, connection_or_get_by_ext_or_id(idp)); + tt_ptr_op(c2, ==, connection_or_get_by_ext_or_id(idp2)); + + /* Now remove it. */ + connection_or_remove_from_ext_or_id_map(c2); + tt_ptr_op(NULL, ==, connection_or_get_by_ext_or_id(idp)); + tt_ptr_op(NULL, ==, connection_or_get_by_ext_or_id(idp2)); + + done: + if (c1) + connection_free_(TO_CONN(c1)); + if (c2) + connection_free_(TO_CONN(c2)); + if (c3) + connection_free_(TO_CONN(c3)); + tor_free(idp); + tor_free(idp2); + connection_or_clear_ext_or_id_map(); +} + +/* Simple connection_write_to_buf_impl_ replacement that unconditionally + * writes to outbuf. */ +static void +connection_write_to_buf_impl_replacement(const char *string, size_t len, + connection_t *conn, int zlib) +{ + (void) zlib; + + tor_assert(string); + tor_assert(conn); + write_to_buf(string, len, conn->outbuf); +} + +static char * +buf_get_contents(buf_t *buf, size_t *sz_out) +{ + char *out; + *sz_out = buf_datalen(buf); + if (*sz_out >= ULONG_MAX) + return NULL; /* C'mon, really? */ + out = tor_malloc(*sz_out + 1); + if (fetch_from_buf(out, (unsigned long)*sz_out, buf) != 0) { + tor_free(out); + return NULL; + } + out[*sz_out] = '\0'; /* Hopefully gratuitous. */ + return out; +} + +static void +test_ext_or_write_command(void *arg) +{ + or_connection_t *c1; + char *cp = NULL; + char *buf = NULL; + size_t sz; + + (void) arg; + MOCK(connection_write_to_buf_impl_, + connection_write_to_buf_impl_replacement); + + c1 = or_connection_new(CONN_TYPE_EXT_OR, AF_INET); + tt_assert(c1); + + /* Length too long */ + tt_int_op(connection_write_ext_or_command(TO_CONN(c1), 100, "X", 100000), + <, 0); + + /* Empty command */ + tt_int_op(connection_write_ext_or_command(TO_CONN(c1), 0x99, NULL, 0), + ==, 0); + cp = buf_get_contents(TO_CONN(c1)->outbuf, &sz); + tt_int_op(sz, ==, 4); + test_mem_op(cp, ==, "\x00\x99\x00\x00", 4); + tor_free(cp); + + /* Medium command. */ + tt_int_op(connection_write_ext_or_command(TO_CONN(c1), 0x99, + "Wai\0Hello", 9), ==, 0); + cp = buf_get_contents(TO_CONN(c1)->outbuf, &sz); + tt_int_op(sz, ==, 13); + test_mem_op(cp, ==, "\x00\x99\x00\x09Wai\x00Hello", 13); + tor_free(cp); + + /* Long command */ + buf = tor_malloc(65535); + memset(buf, 'x', 65535); + tt_int_op(connection_write_ext_or_command(TO_CONN(c1), 0xf00d, + buf, 65535), ==, 0); + cp = buf_get_contents(TO_CONN(c1)->outbuf, &sz); + tt_int_op(sz, ==, 65539); + test_mem_op(cp, ==, "\xf0\x0d\xff\xff", 4); + test_mem_op(cp+4, ==, buf, 65535); + tor_free(cp); + + done: + if (c1) + connection_free_(TO_CONN(c1)); + tor_free(cp); + tor_free(buf); + UNMOCK(connection_write_to_buf_impl_); +} + +static int +write_bytes_to_file_fail(const char *fname, const char *str, size_t len, + int bin) +{ + (void) fname; + (void) str; + (void) len; + (void) bin; + + return -1; +} + +static void +test_ext_or_init_auth(void *arg) +{ + or_options_t *options = get_options_mutable(); + const char *fn; + char *cp = NULL; + struct stat st; + char cookie0[32]; + (void)arg; + + /* Check default filename location */ + tor_free(options->DataDirectory); + options->DataDirectory = tor_strdup("foo"); + cp = get_ext_or_auth_cookie_file_name(); + tt_str_op(cp, ==, "foo"PATH_SEPARATOR"extended_orport_auth_cookie"); + tor_free(cp); + + /* Shouldn't be initialized already, or our tests will be a bit + * meaningless */ + ext_or_auth_cookie = tor_malloc_zero(32); + test_assert(tor_mem_is_zero((char*)ext_or_auth_cookie, 32)); + + /* Now make sure we use a temporary file */ + fn = get_fname("ext_cookie_file"); + options->ExtORPortCookieAuthFile = tor_strdup(fn); + cp = get_ext_or_auth_cookie_file_name(); + tt_str_op(cp, ==, fn); + tor_free(cp); + + /* Test the initialization function with a broken + write_bytes_to_file(). See if the problem is handled properly. */ + MOCK(write_bytes_to_file, write_bytes_to_file_fail); + tt_int_op(-1, ==, init_ext_or_cookie_authentication(1)); + tt_int_op(ext_or_auth_cookie_is_set, ==, 0); + UNMOCK(write_bytes_to_file); + + /* Now do the actual initialization. */ + tt_int_op(0, ==, init_ext_or_cookie_authentication(1)); + tt_int_op(ext_or_auth_cookie_is_set, ==, 1); + cp = read_file_to_str(fn, RFTS_BIN, &st); + tt_ptr_op(cp, !=, NULL); + tt_u64_op((uint64_t)st.st_size, ==, 64); + test_memeq(cp, "! Extended ORPort Auth Cookie !\x0a", 32); + test_memeq(cp+32, ext_or_auth_cookie, 32); + memcpy(cookie0, ext_or_auth_cookie, 32); + test_assert(!tor_mem_is_zero((char*)ext_or_auth_cookie, 32)); + + /* Operation should be idempotent. */ + tt_int_op(0, ==, init_ext_or_cookie_authentication(1)); + test_memeq(cookie0, ext_or_auth_cookie, 32); + + done: + tor_free(cp); + ext_orport_free_all(); +} + +static void +test_ext_or_cookie_auth(void *arg) +{ + char *reply=NULL, *reply2=NULL, *client_hash=NULL, *client_hash2=NULL; + size_t reply_len=0; + char hmac1[32], hmac2[32]; + + const char client_nonce[32] = + "Who is the third who walks alway"; + char server_hash_input[] = + "ExtORPort authentication server-to-client hash" + "Who is the third who walks alway" + "................................"; + char client_hash_input[] = + "ExtORPort authentication client-to-server hash" + "Who is the third who walks alway" + "................................"; + + (void)arg; + + tt_int_op(strlen(client_hash_input), ==, 46+32+32); + tt_int_op(strlen(server_hash_input), ==, 46+32+32); + + ext_or_auth_cookie = tor_malloc_zero(32); + memcpy(ext_or_auth_cookie, "s beside you? When I count, ther", 32); + ext_or_auth_cookie_is_set = 1; + + /* For this authentication, the client sends 32 random bytes (ClientNonce) + * The server replies with 32 byte ServerHash and 32 byte ServerNonce, + * where ServerHash is: + * HMAC-SHA256(CookieString, + * "ExtORPort authentication server-to-client hash" | ClientNonce | + * ServerNonce)" + * The client must reply with 32-byte ClientHash, which we compute as: + * ClientHash is computed as: + * HMAC-SHA256(CookieString, + * "ExtORPort authentication client-to-server hash" | ClientNonce | + * ServerNonce) + */ + + /* Wrong length */ + tt_int_op(-1, ==, + handle_client_auth_nonce(client_nonce, 33, &client_hash, &reply, + &reply_len)); + tt_int_op(-1, ==, + handle_client_auth_nonce(client_nonce, 31, &client_hash, &reply, + &reply_len)); + + /* Now let's try this for real! */ + tt_int_op(0, ==, + handle_client_auth_nonce(client_nonce, 32, &client_hash, &reply, + &reply_len)); + tt_int_op(reply_len, ==, 64); + tt_ptr_op(reply, !=, NULL); + tt_ptr_op(client_hash, !=, NULL); + /* Fill in the server nonce into the hash inputs... */ + memcpy(server_hash_input+46+32, reply+32, 32); + memcpy(client_hash_input+46+32, reply+32, 32); + /* Check the HMACs are correct... */ + crypto_hmac_sha256(hmac1, (char*)ext_or_auth_cookie, 32, server_hash_input, + 46+32+32); + crypto_hmac_sha256(hmac2, (char*)ext_or_auth_cookie, 32, client_hash_input, + 46+32+32); + test_memeq(hmac1, reply, 32); + test_memeq(hmac2, client_hash, 32); + + /* Now do it again and make sure that the results are *different* */ + tt_int_op(0, ==, + handle_client_auth_nonce(client_nonce, 32, &client_hash2, &reply2, + &reply_len)); + test_memneq(reply2, reply, reply_len); + test_memneq(client_hash2, client_hash, 32); + /* But that this one checks out too. */ + memcpy(server_hash_input+46+32, reply2+32, 32); + memcpy(client_hash_input+46+32, reply2+32, 32); + /* Check the HMACs are correct... */ + crypto_hmac_sha256(hmac1, (char*)ext_or_auth_cookie, 32, server_hash_input, + 46+32+32); + crypto_hmac_sha256(hmac2, (char*)ext_or_auth_cookie, 32, client_hash_input, + 46+32+32); + test_memeq(hmac1, reply2, 32); + test_memeq(hmac2, client_hash2, 32); + + done: + tor_free(reply); + tor_free(client_hash); + tor_free(reply2); + tor_free(client_hash2); +} + +static int +crypto_rand_return_tse_str(char *to, size_t n) +{ + if (n != 32) { + TT_FAIL(("Asked for %d bytes, not 32", (int)n)); + return -1; + } + memcpy(to, "te road There is always another ", 32); + return 0; +} + +static void +test_ext_or_cookie_auth_testvec(void *arg) +{ + char *reply=NULL, *client_hash=NULL; + size_t reply_len; + char *mem_op_hex_tmp=NULL; + + const char client_nonce[] = "But when I look ahead up the whi"; + (void)arg; + + ext_or_auth_cookie = tor_malloc_zero(32); + memcpy(ext_or_auth_cookie, "Gliding wrapt in a brown mantle," , 32); + ext_or_auth_cookie_is_set = 1; + + MOCK(crypto_rand, crypto_rand_return_tse_str); + + tt_int_op(0, ==, + handle_client_auth_nonce(client_nonce, 32, &client_hash, &reply, + &reply_len)); + tt_ptr_op(reply, !=, NULL ); + tt_uint_op(reply_len, ==, 64); + test_memeq(reply+32, "te road There is always another ", 32); + /* HMACSHA256("Gliding wrapt in a brown mantle," + * "ExtORPort authentication server-to-client hash" + * "But when I look ahead up the write road There is always another "); + */ + test_memeq_hex(reply, + "ec80ed6e546d3b36fdfc22fe1315416b" + "029f1ade7610d910878b62eeb7403821"); + /* HMACSHA256("Gliding wrapt in a brown mantle," + * "ExtORPort authentication client-to-server hash" + * "But when I look ahead up the write road There is always another "); + * (Both values computed using Python CLI.) + */ + test_memeq_hex(client_hash, + "ab391732dd2ed968cd40c087d1b1f25b" + "33b3cd77ff79bd80c2074bbf438119a2"); + + done: + UNMOCK(crypto_rand); + tor_free(reply); + tor_free(client_hash); + tor_free(mem_op_hex_tmp); +} + +static void +ignore_bootstrap_problem(const char *warn, int reason, + or_connection_t *conn) +{ + (void)warn; + (void)reason; + (void)conn; +} + +static int is_reading = 1; +static int handshake_start_called = 0; + +static void +note_read_stopped(connection_t *conn) +{ + (void)conn; + is_reading=0; +} +static void +note_read_started(connection_t *conn) +{ + (void)conn; + is_reading=1; +} +static int +handshake_start(or_connection_t *conn, int receiving) +{ + if (!conn || !receiving) + TT_FAIL(("Bad arguments to handshake_start")); + handshake_start_called = 1; + return 0; +} + +#define WRITE(s,n) \ + do { \ + write_to_buf((s), (n), TO_CONN(conn)->inbuf); \ + } while (0) +#define CONTAINS(s,n) \ + do { \ + tt_int_op((n), <=, sizeof(b)); \ + tt_int_op(buf_datalen(TO_CONN(conn)->outbuf), ==, (n)); \ + if ((n)) { \ + fetch_from_buf(b, (n), TO_CONN(conn)->outbuf); \ + test_memeq(b, (s), (n)); \ + } \ + } while (0) + +/* Helper: Do a successful Extended ORPort authentication handshake. */ +static void +do_ext_or_handshake(or_connection_t *conn) +{ + char b[256]; + + tt_int_op(0, ==, connection_ext_or_start_auth(conn)); + CONTAINS("\x01\x00", 2); + WRITE("\x01", 1); + WRITE("But when I look ahead up the whi", 32); + MOCK(crypto_rand, crypto_rand_return_tse_str); + tt_int_op(0, ==, connection_ext_or_process_inbuf(conn)); + UNMOCK(crypto_rand); + tt_int_op(TO_CONN(conn)->state, ==, EXT_OR_CONN_STATE_AUTH_WAIT_CLIENT_HASH); + CONTAINS("\xec\x80\xed\x6e\x54\x6d\x3b\x36\xfd\xfc\x22\xfe\x13\x15\x41\x6b" + "\x02\x9f\x1a\xde\x76\x10\xd9\x10\x87\x8b\x62\xee\xb7\x40\x38\x21" + "te road There is always another ", 64); + /* Send the right response this time. */ + WRITE("\xab\x39\x17\x32\xdd\x2e\xd9\x68\xcd\x40\xc0\x87\xd1\xb1\xf2\x5b" + "\x33\xb3\xcd\x77\xff\x79\xbd\x80\xc2\x07\x4b\xbf\x43\x81\x19\xa2", + 32); + tt_int_op(0, ==, connection_ext_or_process_inbuf(conn)); + CONTAINS("\x01", 1); + tt_assert(! TO_CONN(conn)->marked_for_close); + tt_int_op(TO_CONN(conn)->state, ==, EXT_OR_CONN_STATE_OPEN); + + done: ; +} + +static void +test_ext_or_handshake(void *arg) +{ + or_connection_t *conn=NULL; + char b[256]; + + (void) arg; + MOCK(connection_write_to_buf_impl_, + connection_write_to_buf_impl_replacement); + /* Use same authenticators as for test_ext_or_cookie_auth_testvec */ + ext_or_auth_cookie = tor_malloc_zero(32); + memcpy(ext_or_auth_cookie, "Gliding wrapt in a brown mantle," , 32); + ext_or_auth_cookie_is_set = 1; + + init_connection_lists(); + + conn = or_connection_new(CONN_TYPE_EXT_OR, AF_INET); + tt_int_op(0, ==, connection_ext_or_start_auth(conn)); + /* The server starts by telling us about the one supported authtype. */ + CONTAINS("\x01\x00", 2); + /* Say the client hasn't responded yet. */ + tt_int_op(0, ==, connection_ext_or_process_inbuf(conn)); + /* Let's say the client replies badly. */ + WRITE("\x99", 1); + tt_int_op(-1, ==, connection_ext_or_process_inbuf(conn)); + CONTAINS("", 0); + tt_assert(TO_CONN(conn)->marked_for_close); + close_closeable_connections(); + conn = NULL; + + /* Okay, try again. */ + conn = or_connection_new(CONN_TYPE_EXT_OR, AF_INET); + tt_int_op(0, ==, connection_ext_or_start_auth(conn)); + CONTAINS("\x01\x00", 2); + /* Let's say the client replies sensibly this time. "Yes, AUTHTYPE_COOKIE + * sounds delicious. Let's have some of that!" */ + WRITE("\x01", 1); + /* Let's say that the client also sends part of a nonce. */ + WRITE("But when I look ", 16); + tt_int_op(0, ==, connection_ext_or_process_inbuf(conn)); + CONTAINS("", 0); + tt_int_op(TO_CONN(conn)->state, ==, + EXT_OR_CONN_STATE_AUTH_WAIT_CLIENT_NONCE); + /* Pump it again. Nothing should happen. */ + tt_int_op(0, ==, connection_ext_or_process_inbuf(conn)); + /* send the rest of the nonce. */ + WRITE("ahead up the whi", 16); + MOCK(crypto_rand, crypto_rand_return_tse_str); + tt_int_op(0, ==, connection_ext_or_process_inbuf(conn)); + UNMOCK(crypto_rand); + /* We should get the right reply from the server. */ + CONTAINS("\xec\x80\xed\x6e\x54\x6d\x3b\x36\xfd\xfc\x22\xfe\x13\x15\x41\x6b" + "\x02\x9f\x1a\xde\x76\x10\xd9\x10\x87\x8b\x62\xee\xb7\x40\x38\x21" + "te road There is always another ", 64); + /* Send the wrong response. */ + WRITE("not with a bang but a whimper...", 32); + MOCK(control_event_bootstrap_problem, ignore_bootstrap_problem); + tt_int_op(-1, ==, connection_ext_or_process_inbuf(conn)); + CONTAINS("\x00", 1); + tt_assert(TO_CONN(conn)->marked_for_close); + /* XXXX Hold-open-until-flushed. */ + close_closeable_connections(); + conn = NULL; + UNMOCK(control_event_bootstrap_problem); + + MOCK(connection_start_reading, note_read_started); + MOCK(connection_stop_reading, note_read_stopped); + MOCK(connection_tls_start_handshake, handshake_start); + + /* Okay, this time let's succeed. */ + conn = or_connection_new(CONN_TYPE_EXT_OR, AF_INET); + do_ext_or_handshake(conn); + + /* Now let's run through some messages. */ + /* First let's send some junk and make sure it's ignored. */ + WRITE("\xff\xf0\x00\x03""ABC", 7); + tt_int_op(0, ==, connection_ext_or_process_inbuf(conn)); + CONTAINS("", 0); + /* Now let's send a USERADDR command. */ + WRITE("\x00\x01\x00\x0c""1.2.3.4:5678", 16); + tt_int_op(0, ==, connection_ext_or_process_inbuf(conn)); + tt_int_op(TO_CONN(conn)->port, ==, 5678); + tt_int_op(tor_addr_to_ipv4h(&TO_CONN(conn)->addr), ==, 0x01020304); + /* Now let's send a TRANSPORT command. */ + WRITE("\x00\x02\x00\x07""rfc1149", 11); + tt_int_op(0, ==, connection_ext_or_process_inbuf(conn)); + tt_ptr_op(NULL, !=, conn->ext_or_transport); + tt_str_op("rfc1149", ==, conn->ext_or_transport); + tt_int_op(is_reading,==,1); + tt_int_op(TO_CONN(conn)->state, ==, EXT_OR_CONN_STATE_OPEN); + /* DONE */ + WRITE("\x00\x00\x00\x00", 4); + tt_int_op(0, ==, connection_ext_or_process_inbuf(conn)); + tt_int_op(TO_CONN(conn)->state, ==, EXT_OR_CONN_STATE_FLUSHING); + tt_int_op(is_reading,==,0); + CONTAINS("\x10\x00\x00\x00", 4); + tt_int_op(handshake_start_called,==,0); + tt_int_op(0, ==, connection_ext_or_finished_flushing(conn)); + tt_int_op(is_reading,==,1); + tt_int_op(handshake_start_called,==,1); + tt_int_op(TO_CONN(conn)->type, ==, CONN_TYPE_OR); + tt_int_op(TO_CONN(conn)->state, ==, 0); + close_closeable_connections(); + conn = NULL; + + /* Okay, this time let's succeed the handshake but fail the USERADDR + command. */ + conn = or_connection_new(CONN_TYPE_EXT_OR, AF_INET); + do_ext_or_handshake(conn); + /* USERADDR command with an extra NUL byte */ + WRITE("\x00\x01\x00\x0d""1.2.3.4:5678\x00", 17); + MOCK(control_event_bootstrap_problem, ignore_bootstrap_problem); + tt_int_op(-1, ==, connection_ext_or_process_inbuf(conn)); + CONTAINS("", 0); + tt_assert(TO_CONN(conn)->marked_for_close); + close_closeable_connections(); + conn = NULL; + UNMOCK(control_event_bootstrap_problem); + + /* Now fail the TRANSPORT command. */ + conn = or_connection_new(CONN_TYPE_EXT_OR, AF_INET); + do_ext_or_handshake(conn); + /* TRANSPORT command with an extra NUL byte */ + WRITE("\x00\x02\x00\x08""rfc1149\x00", 12); + MOCK(control_event_bootstrap_problem, ignore_bootstrap_problem); + tt_int_op(-1, ==, connection_ext_or_process_inbuf(conn)); + CONTAINS("", 0); + tt_assert(TO_CONN(conn)->marked_for_close); + close_closeable_connections(); + conn = NULL; + UNMOCK(control_event_bootstrap_problem); + + /* Now fail the TRANSPORT command. */ + conn = or_connection_new(CONN_TYPE_EXT_OR, AF_INET); + do_ext_or_handshake(conn); + /* TRANSPORT command with transport name with symbols (not a + C-identifier) */ + WRITE("\x00\x02\x00\x07""rf*1149", 11); + MOCK(control_event_bootstrap_problem, ignore_bootstrap_problem); + tt_int_op(-1, ==, connection_ext_or_process_inbuf(conn)); + CONTAINS("", 0); + tt_assert(TO_CONN(conn)->marked_for_close); + close_closeable_connections(); + conn = NULL; + UNMOCK(control_event_bootstrap_problem); + + done: + UNMOCK(connection_write_to_buf_impl_); + UNMOCK(crypto_rand); + if (conn) + connection_free_(TO_CONN(conn)); +#undef CONTAINS +#undef WRITE +} + +struct testcase_t extorport_tests[] = { + { "id_map", test_ext_or_id_map, TT_FORK, NULL, NULL }, + { "write_command", test_ext_or_write_command, TT_FORK, NULL, NULL }, + { "init_auth", test_ext_or_init_auth, TT_FORK, NULL, NULL }, + { "cookie_auth", test_ext_or_cookie_auth, TT_FORK, NULL, NULL }, + { "cookie_auth_testvec", test_ext_or_cookie_auth_testvec, TT_FORK, + NULL, NULL }, + { "handshake", test_ext_or_handshake, TT_FORK, NULL, NULL }, + END_OF_TESTCASES +}; + diff --git a/src/test/test_hs.c b/src/test/test_hs.c new file mode 100644 index 0000000000..99ef7dd570 --- /dev/null +++ b/src/test/test_hs.c @@ -0,0 +1,129 @@ +/* Copyright (c) 2007-2013, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file test_hs.c + * \brief Unit tests for hidden service. + **/ + +#define CONTROL_PRIVATE +#include "or.h" +#include "test.h" +#include "control.h" + +/* mock ID digest and longname for node that's in nodelist */ +#define HSDIR_EXIST_ID "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA" \ + "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA" +#define STR_HSDIR_EXIST_LONGNAME \ + "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=TestDir" +/* mock ID digest and longname for node that's not in nodelist */ +#define HSDIR_NONE_EXIST_ID "\xBB\xBB\xBB\xBB\xBB\xBB\xBB\xBB\xBB\xBB" \ + "\xBB\xBB\xBB\xBB\xBB\xBB\xBB\xBB\xBB\xBB" +#define STR_HSDIR_NONE_EXIST_LONGNAME \ + "$BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB" + +/* Helper global variable for hidden service descriptor event test. + * It's used as a pointer to dynamically created message buffer in + * send_control_event_string_replacement function, which mocks + * send_control_event_string function. + * + * Always free it after use! */ +static char *received_msg = NULL; + +/** Mock function for send_control_event_string + */ +static void +send_control_event_string_replacement(uint16_t event, event_format_t which, + const char *msg) +{ + (void) event; + (void) which; + tor_free(received_msg); + received_msg = tor_strdup(msg); +} + +/** Mock function for node_describe_longname_by_id, it returns either + * STR_HSDIR_EXIST_LONGNAME or STR_HSDIR_NONE_EXIST_LONGNAME + */ +static const char * +node_describe_longname_by_id_replacement(const char *id_digest) +{ + if (!strcmp(id_digest, HSDIR_EXIST_ID)) { + return STR_HSDIR_EXIST_LONGNAME; + } else { + return STR_HSDIR_NONE_EXIST_LONGNAME; + } +} + +/** Make sure each hidden service descriptor async event generation + * + * function generates the message in expected format. + */ +static void +test_hs_desc_event(void *arg) +{ + #define STR_HS_ADDR "ajhb7kljbiru65qo" + #define STR_HS_ID "b3oeducbhjmbqmgw2i3jtz4fekkrinwj" + + rend_data_t rend_query; + const char *expected_msg; + + (void) arg; + MOCK(send_control_event_string, + send_control_event_string_replacement); + MOCK(node_describe_longname_by_id, + node_describe_longname_by_id_replacement); + + /* setup rend_query struct */ + strncpy(rend_query.onion_address, STR_HS_ADDR, + REND_SERVICE_ID_LEN_BASE32+1); + rend_query.auth_type = 0; + + /* test request event */ + control_event_hs_descriptor_requested(&rend_query, HSDIR_EXIST_ID, + STR_HS_ID); + expected_msg = "650 HS_DESC REQUESTED "STR_HS_ADDR" NO_AUTH "\ + STR_HSDIR_EXIST_LONGNAME" "STR_HS_ID"\r\n"; + test_assert(received_msg); + test_streq(received_msg, expected_msg); + tor_free(received_msg); + + /* test received event */ + rend_query.auth_type = 1; + control_event_hs_descriptor_received(&rend_query, HSDIR_EXIST_ID); + expected_msg = "650 HS_DESC RECEIVED "STR_HS_ADDR" BASIC_AUTH "\ + STR_HSDIR_EXIST_LONGNAME"\r\n"; + test_assert(received_msg); + test_streq(received_msg, expected_msg); + tor_free(received_msg); + + /* test failed event */ + rend_query.auth_type = 2; + control_event_hs_descriptor_failed(&rend_query, HSDIR_NONE_EXIST_ID); + expected_msg = "650 HS_DESC FAILED "STR_HS_ADDR" STEALTH_AUTH "\ + STR_HSDIR_NONE_EXIST_LONGNAME"\r\n"; + test_assert(received_msg); + test_streq(received_msg, expected_msg); + tor_free(received_msg); + + /* test invalid auth type */ + rend_query.auth_type = 999; + control_event_hs_descriptor_failed(&rend_query, HSDIR_EXIST_ID); + expected_msg = "650 HS_DESC FAILED "STR_HS_ADDR" UNKNOWN "\ + STR_HSDIR_EXIST_LONGNAME"\r\n"; + test_assert(received_msg); + test_streq(received_msg, expected_msg); + tor_free(received_msg); + + done: + UNMOCK(send_control_event_string); + UNMOCK(node_describe_longname_by_id); + tor_free(received_msg); +} + +struct testcase_t hs_tests[] = { + { "hs_desc_event", test_hs_desc_event, TT_FORK, + NULL, NULL }, + END_OF_TESTCASES +}; + diff --git a/src/test/test_logging.c b/src/test/test_logging.c new file mode 100644 index 0000000000..7e558f83b1 --- /dev/null +++ b/src/test/test_logging.c @@ -0,0 +1,135 @@ +/* Copyright (c) 2013, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "orconfig.h" +#include "or.h" +#include "torlog.h" +#include "test.h" + +static void +dummy_cb_fn(int severity, uint32_t domain, const char *msg) +{ + (void)severity; (void)domain; (void)msg; +} + +static void +test_get_sigsafe_err_fds(void *arg) +{ + const int *fds; + int n; + log_severity_list_t include_bug, no_bug, no_bug2; + (void) arg; + init_logging(); + + n = tor_log_get_sigsafe_err_fds(&fds); + tt_int_op(n, ==, 1); + tt_int_op(fds[0], ==, STDERR_FILENO); + + set_log_severity_config(LOG_WARN, LOG_ERR, &include_bug); + set_log_severity_config(LOG_WARN, LOG_ERR, &no_bug); + no_bug.masks[0] &= ~(LD_BUG|LD_GENERAL); + set_log_severity_config(LOG_INFO, LOG_NOTICE, &no_bug2); + + /* Add some logs; make sure the output is as expected. */ + mark_logs_temp(); + add_stream_log(&include_bug, "dummy-1", 3); + add_stream_log(&no_bug, "dummy-2", 4); + add_stream_log(&no_bug2, "dummy-3", 5); + add_callback_log(&include_bug, dummy_cb_fn); + close_temp_logs(); + tor_log_update_sigsafe_err_fds(); + + n = tor_log_get_sigsafe_err_fds(&fds); + tt_int_op(n, ==, 2); + tt_int_op(fds[0], ==, STDERR_FILENO); + tt_int_op(fds[1], ==, 3); + + /* Allow STDOUT to replace STDERR. */ + add_stream_log(&include_bug, "dummy-4", STDOUT_FILENO); + tor_log_update_sigsafe_err_fds(); + n = tor_log_get_sigsafe_err_fds(&fds); + tt_int_op(n, ==, 2); + tt_int_op(fds[0], ==, 3); + tt_int_op(fds[1], ==, STDOUT_FILENO); + + /* But don't allow it to replace explicit STDERR. */ + add_stream_log(&include_bug, "dummy-5", STDERR_FILENO); + tor_log_update_sigsafe_err_fds(); + n = tor_log_get_sigsafe_err_fds(&fds); + tt_int_op(n, ==, 3); + tt_int_op(fds[0], ==, STDERR_FILENO); + tt_int_op(fds[1], ==, STDOUT_FILENO); + tt_int_op(fds[2], ==, 3); + + /* Don't overflow the array. */ + { + int i; + for (i=5; i<20; ++i) { + add_stream_log(&include_bug, "x-dummy", i); + } + } + tor_log_update_sigsafe_err_fds(); + n = tor_log_get_sigsafe_err_fds(&fds); + tt_int_op(n, ==, 8); + + done: + ; +} + +static void +test_sigsafe_err(void *arg) +{ + const char *fn=get_fname("sigsafe_err_log"); + char *content=NULL; + log_severity_list_t include_bug; + smartlist_t *lines = smartlist_new(); + (void)arg; + + set_log_severity_config(LOG_WARN, LOG_ERR, &include_bug); + + init_logging(); + mark_logs_temp(); + add_file_log(&include_bug, fn); + tor_log_update_sigsafe_err_fds(); + close_temp_logs(); + + close(STDERR_FILENO); + log_err(LD_BUG, "Say, this isn't too cool."); + tor_log_err_sigsafe("Minimal.\n", NULL); + + set_log_time_granularity(100*1000); + tor_log_err_sigsafe("Testing any ", + "attempt to manually log ", + "from a signal.\n", + NULL); + mark_logs_temp(); + close_temp_logs(); + close(STDERR_FILENO); + content = read_file_to_str(fn, 0, NULL); + + tt_assert(content != NULL); + tor_split_lines(lines, content, (int)strlen(content)); + tt_int_op(smartlist_len(lines), >=, 5); + + if (strstr(smartlist_get(lines, 0), "opening new log file")) + smartlist_del_keeporder(lines, 0); + tt_assert(strstr(smartlist_get(lines, 0), "Say, this isn't too cool")); + /* Next line is blank. */ + tt_assert(!strcmpstart(smartlist_get(lines, 1), "==============")); + tt_assert(!strcmpstart(smartlist_get(lines, 2), "Minimal.")); + /* Next line is blank. */ + tt_assert(!strcmpstart(smartlist_get(lines, 3), "==============")); + tt_str_op(smartlist_get(lines, 4), ==, + "Testing any attempt to manually log from a signal."); + + done: + tor_free(content); + smartlist_free(lines); +} + +struct testcase_t logging_tests[] = { + { "sigsafe_err_fds", test_get_sigsafe_err_fds, TT_FORK, NULL, NULL }, + { "sigsafe_err", test_sigsafe_err, TT_FORK, NULL, NULL }, + END_OF_TESTCASES +}; + diff --git a/src/test/test_microdesc.c b/src/test/test_microdesc.c index 53a03a48ad..78f4823b87 100644 --- a/src/test/test_microdesc.c +++ b/src/test/test_microdesc.c @@ -5,7 +5,10 @@ #include "or.h" #include "config.h" +#include "dirvote.h" #include "microdesc.h" +#include "routerlist.h" +#include "routerparse.h" #include "test.h" @@ -261,6 +264,7 @@ test_md_cache_broken(void *data) options = get_options_mutable(); tt_assert(options); + tor_free(options->DataDirectory); options->DataDirectory = tor_strdup(get_fname("md_datadir_test2")); #ifdef _WIN32 @@ -284,9 +288,113 @@ test_md_cache_broken(void *data) microdesc_free_all(); } +/* Generated by chutney. */ +static const char test_ri[] = + "router test005r 127.0.0.1 5005 0 7005\n" + "platform Tor 0.2.5.4-alpha-dev on Linux\n" + "protocols Link 1 2 Circuit 1\n" + "published 2014-05-06 22:57:55\n" + "fingerprint 09DE 3BA2 48C2 1C3F 3760 6CD3 8460 43A6 D5EC F59E\n" + "uptime 0\n" + "bandwidth 1073741824 1073741824 0\n" + "extra-info-digest 361F9428F9FA4DD854C03DDBCC159D0D9FA996C9\n" + "onion-key\n" + "-----BEGIN RSA PUBLIC KEY-----\n" + "MIGJAoGBANBJz8Vldl12aFeSMPLiA4nOetLDN0oxU8bB1SDhO7Uu2zdWYVYAF5J0\n" + "st7WvrVy/jA9v/fsezNAPskBanecHRSkdMTpkcgRPMHE7CTGEwIy1Yp1X4bPgDlC\n" + "VCnbs5Pcts5HnWEYNK7qHDAUn+IlmjOO+pTUY8uyq+GQVz6H9wFlAgMBAAE=\n" + "-----END RSA PUBLIC KEY-----\n" + "signing-key\n" + "-----BEGIN RSA PUBLIC KEY-----\n" + "MIGJAoGBANbGUC4802Ke6C3nOVxN0U0HhIRrs32cQFEL4v+UUMJPgjbistHBvOax\n" + "CWVR/sMXM2kKJeGThJ9ZUs2p9dDG4WHPUXgkMqzTTEeeFa7pQKU0brgbmLaJq0Pi\n" + "mxmqC5RkTHa5bQvq6QlSFprAEoovV27cWqBM9jVdV9hyc//6kwPzAgMBAAE=\n" + "-----END RSA PUBLIC KEY-----\n" + "hidden-service-dir\n" + "ntor-onion-key Gg73xH7+kTfT6bi1uNVx9gwQdQas9pROIfmc4NpAdC4=\n" + "reject *:25\n" + "reject *:119\n" + "reject *:135-139\n" + "reject *:445\n" + "reject *:563\n" + "reject *:1214\n" + "reject *:4661-4666\n" + "reject *:6346-6429\n" + "reject *:6699\n" + "reject *:6881-6999\n" + "accept *:*\n" + "router-signature\n" + "-----BEGIN SIGNATURE-----\n" + "ImzX5PF2vRCrG1YzGToyjoxYhgh1vtHEDjmP+tIS/iil1DSnHZNpHSuHp0L1jE9S\n" + "yZyrtKaqpBE/aecAM3j4CWCn/ipnAAQkHcyRLin1bYvqBtRzyopVCRlUhF+uWrLq\n" + "t0xkIE39ss/EwmQr7iIgkdVH4oRIMsjYnFFJBG26nYY=\n" + "-----END SIGNATURE-----\n"; + +static const char test_md_8[] = + "onion-key\n" + "-----BEGIN RSA PUBLIC KEY-----\n" + "MIGJAoGBANBJz8Vldl12aFeSMPLiA4nOetLDN0oxU8bB1SDhO7Uu2zdWYVYAF5J0\n" + "st7WvrVy/jA9v/fsezNAPskBanecHRSkdMTpkcgRPMHE7CTGEwIy1Yp1X4bPgDlC\n" + "VCnbs5Pcts5HnWEYNK7qHDAUn+IlmjOO+pTUY8uyq+GQVz6H9wFlAgMBAAE=\n" + "-----END RSA PUBLIC KEY-----\n" + "p reject 25,119,135-139,445,563,1214,4661-4666,6346-6429,6699,6881-6999\n"; + +static const char test_md_16[] = + "onion-key\n" + "-----BEGIN RSA PUBLIC KEY-----\n" + "MIGJAoGBANBJz8Vldl12aFeSMPLiA4nOetLDN0oxU8bB1SDhO7Uu2zdWYVYAF5J0\n" + "st7WvrVy/jA9v/fsezNAPskBanecHRSkdMTpkcgRPMHE7CTGEwIy1Yp1X4bPgDlC\n" + "VCnbs5Pcts5HnWEYNK7qHDAUn+IlmjOO+pTUY8uyq+GQVz6H9wFlAgMBAAE=\n" + "-----END RSA PUBLIC KEY-----\n" + "ntor-onion-key Gg73xH7+kTfT6bi1uNVx9gwQdQas9pROIfmc4NpAdC4=\n" + "p reject 25,119,135-139,445,563,1214,4661-4666,6346-6429,6699,6881-6999\n"; + +static const char test_md_18[] = + "onion-key\n" + "-----BEGIN RSA PUBLIC KEY-----\n" + "MIGJAoGBANBJz8Vldl12aFeSMPLiA4nOetLDN0oxU8bB1SDhO7Uu2zdWYVYAF5J0\n" + "st7WvrVy/jA9v/fsezNAPskBanecHRSkdMTpkcgRPMHE7CTGEwIy1Yp1X4bPgDlC\n" + "VCnbs5Pcts5HnWEYNK7qHDAUn+IlmjOO+pTUY8uyq+GQVz6H9wFlAgMBAAE=\n" + "-----END RSA PUBLIC KEY-----\n" + "ntor-onion-key Gg73xH7+kTfT6bi1uNVx9gwQdQas9pROIfmc4NpAdC4=\n" + "p reject 25,119,135-139,445,563,1214,4661-4666,6346-6429,6699,6881-6999\n" + "id rsa1024 Cd47okjCHD83YGzThGBDptXs9Z4\n"; + +static void +test_md_generate(void *arg) +{ + routerinfo_t *ri; + microdesc_t *md = NULL; + (void)arg; + + ri = router_parse_entry_from_string(test_ri, NULL, 0, 0, NULL); + tt_assert(ri); + md = dirvote_create_microdescriptor(ri, 8); + tt_str_op(md->body, ==, test_md_8); + + /* XXXX test family lines. */ + /* XXXX test method 14 for A lines. */ + /* XXXX test method 15 for P6 lines. */ + + microdesc_free(md); + md = NULL; + md = dirvote_create_microdescriptor(ri, 16); + tt_str_op(md->body, ==, test_md_16); + + microdesc_free(md); + md = NULL; + md = dirvote_create_microdescriptor(ri, 18); + tt_str_op(md->body, ==, test_md_18); + + done: + microdesc_free(md); + routerinfo_free(ri); +} + struct testcase_t microdesc_tests[] = { { "cache", test_md_cache, TT_FORK, NULL, NULL }, { "broken_cache", test_md_cache_broken, TT_FORK, NULL, NULL }, + { "generate", test_md_generate, 0, NULL, NULL }, END_OF_TESTCASES }; diff --git a/src/test/test_nodelist.c b/src/test/test_nodelist.c new file mode 100644 index 0000000000..600e6a89d4 --- /dev/null +++ b/src/test/test_nodelist.c @@ -0,0 +1,71 @@ +/* Copyright (c) 2007-2013, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file test_nodelist.c + * \brief Unit tests for nodelist related functions. + **/ + +#include "or.h" +#include "nodelist.h" +#include "test.h" + +/** Tese the case when node_get_by_id() returns NULL, + * node_get_verbose_nickname_by_id should return the base 16 encoding + * of the id. + */ +static void +test_nodelist_node_get_verbose_nickname_by_id_null_node(void *arg) +{ + char vname[MAX_VERBOSE_NICKNAME_LEN+1]; + const char ID[] = "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA" + "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA"; + (void) arg; + + /* make sure node_get_by_id returns NULL */ + test_assert(!node_get_by_id(ID)); + node_get_verbose_nickname_by_id(ID, vname); + test_streq(vname, "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"); + done: + return; +} + +/** For routers without named flag, get_verbose_nickname should return + * "Fingerprint~Nickname" + */ +static void +test_nodelist_node_get_verbose_nickname_not_named(void *arg) +{ + node_t mock_node; + routerstatus_t mock_rs; + + char vname[MAX_VERBOSE_NICKNAME_LEN+1]; + + (void) arg; + + memset(&mock_node, 0, sizeof(node_t)); + memset(&mock_rs, 0, sizeof(routerstatus_t)); + + /* verbose nickname should use ~ instead of = for unnamed routers */ + strlcpy(mock_rs.nickname, "TestOR", sizeof(mock_rs.nickname)); + mock_node.rs = &mock_rs; + memcpy(mock_node.identity, + "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA" + "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA", + DIGEST_LEN); + node_get_verbose_nickname(&mock_node, vname); + test_streq(vname, "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR"); + + done: + return; +} + +#define NODE(name, flags) \ + { #name, test_nodelist_##name, (flags), NULL, NULL } + +struct testcase_t nodelist_tests[] = { + NODE(node_get_verbose_nickname_by_id_null_node, TT_FORK), + NODE(node_get_verbose_nickname_not_named, TT_FORK), + END_OF_TESTCASES +}; + diff --git a/src/test/test_oom.c b/src/test/test_oom.c new file mode 100644 index 0000000000..2726056b80 --- /dev/null +++ b/src/test/test_oom.c @@ -0,0 +1,381 @@ +/* Copyright (c) 2014, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/* Unit tests for OOM handling logic */ + +#define RELAY_PRIVATE +#define BUFFERS_PRIVATE +#define CIRCUITLIST_PRIVATE +#define CONNECTION_PRIVATE +#include "or.h" +#include "buffers.h" +#include "circuitlist.h" +#include "compat_libevent.h" +#include "connection.h" +#include "config.h" +#ifdef ENABLE_MEMPOOLS +#include "mempool.h" +#endif +#include "relay.h" +#include "test.h" + +/* small replacement mock for circuit_mark_for_close_ to avoid doing all + * the other bookkeeping that comes with marking circuits. */ +static void +circuit_mark_for_close_dummy_(circuit_t *circ, int reason, int line, + const char *file) +{ + (void) reason; + if (circ->marked_for_close) { + TT_FAIL(("Circuit already marked for close at %s:%d, but we are marking " + "it again at %s:%d", + circ->marked_for_close_file, (int)circ->marked_for_close, + file, line)); + } + + circ->marked_for_close = line; + circ->marked_for_close_file = file; +} + +static circuit_t * +dummy_or_circuit_new(int n_p_cells, int n_n_cells) +{ + or_circuit_t *circ = or_circuit_new(0, NULL); + int i; + cell_t cell; + + for (i=0; i < n_p_cells; ++i) { + crypto_rand((void*)&cell, sizeof(cell)); + cell_queue_append_packed_copy(TO_CIRCUIT(circ), &circ->p_chan_cells, + 0, &cell, 1, 0); + } + + for (i=0; i < n_n_cells; ++i) { + crypto_rand((void*)&cell, sizeof(cell)); + cell_queue_append_packed_copy(TO_CIRCUIT(circ), + &TO_CIRCUIT(circ)->n_chan_cells, + 1, &cell, 1, 0); + } + + TO_CIRCUIT(circ)->purpose = CIRCUIT_PURPOSE_OR; + return TO_CIRCUIT(circ); +} + +static circuit_t * +dummy_origin_circuit_new(int n_cells) +{ + origin_circuit_t *circ = origin_circuit_new(); + int i; + cell_t cell; + + for (i=0; i < n_cells; ++i) { + crypto_rand((void*)&cell, sizeof(cell)); + cell_queue_append_packed_copy(TO_CIRCUIT(circ), + &TO_CIRCUIT(circ)->n_chan_cells, + 1, &cell, 1, 0); + } + + TO_CIRCUIT(circ)->purpose = CIRCUIT_PURPOSE_C_GENERAL; + return TO_CIRCUIT(circ); +} + +static void +add_bytes_to_buf(generic_buffer_t *buf, size_t n_bytes) +{ + char b[3000]; + + while (n_bytes) { + size_t this_add = n_bytes > sizeof(b) ? sizeof(b) : n_bytes; + crypto_rand(b, this_add); + generic_buffer_add(buf, b, this_add); + n_bytes -= this_add; + } +} + +static edge_connection_t * +dummy_edge_conn_new(circuit_t *circ, + int type, size_t in_bytes, size_t out_bytes) +{ + edge_connection_t *conn; + generic_buffer_t *inbuf, *outbuf; + + if (type == CONN_TYPE_EXIT) + conn = edge_connection_new(type, AF_INET); + else + conn = ENTRY_TO_EDGE_CONN(entry_connection_new(type, AF_INET)); + +#ifdef USE_BUFFEREVENTS + inbuf = bufferevent_get_input(TO_CONN(conn)->bufev); + outbuf = bufferevent_get_output(TO_CONN(conn)->bufev); +#else + inbuf = TO_CONN(conn)->inbuf; + outbuf = TO_CONN(conn)->outbuf; +#endif + + /* We add these bytes directly to the buffers, to avoid all the + * edge connection read/write machinery. */ + add_bytes_to_buf(inbuf, in_bytes); + add_bytes_to_buf(outbuf, out_bytes); + + conn->on_circuit = circ; + if (type == CONN_TYPE_EXIT) { + or_circuit_t *oc = TO_OR_CIRCUIT(circ); + conn->next_stream = oc->n_streams; + oc->n_streams = conn; + } else { + origin_circuit_t *oc = TO_ORIGIN_CIRCUIT(circ); + conn->next_stream = oc->p_streams; + oc->p_streams = conn; + } + + return conn; +} + +/** Run unit tests for buffers.c */ +static void +test_oom_circbuf(void *arg) +{ + or_options_t *options = get_options_mutable(); + circuit_t *c1 = NULL, *c2 = NULL, *c3 = NULL, *c4 = NULL; + struct timeval tv = { 1389631048, 0 }; + + (void) arg; + + MOCK(circuit_mark_for_close_, circuit_mark_for_close_dummy_); + +#ifdef ENABLE_MEMPOOLS + init_cell_pool(); +#endif /* ENABLE_MEMPOOLS */ + + /* Far too low for real life. */ + options->MaxMemInQueues = 256*packed_cell_mem_cost(); + options->CellStatistics = 0; + + tt_int_op(cell_queues_check_size(), ==, 0); /* We don't start out OOM. */ + tt_int_op(cell_queues_get_total_allocation(), ==, 0); + tt_int_op(buf_get_total_allocation(), ==, 0); + + /* Now we're going to fake up some circuits and get them added to the global + circuit list. */ + tv.tv_usec = 0; + tor_gettimeofday_cache_set(&tv); + c1 = dummy_origin_circuit_new(30); + tv.tv_usec = 10*1000; + tor_gettimeofday_cache_set(&tv); + c2 = dummy_or_circuit_new(20, 20); + +#ifdef ENABLE_MEMPOOLS + tt_int_op(packed_cell_mem_cost(), ==, + sizeof(packed_cell_t) + MP_POOL_ITEM_OVERHEAD); +#else + tt_int_op(packed_cell_mem_cost(), ==, + sizeof(packed_cell_t)); +#endif /* ENABLE_MEMPOOLS */ + tt_int_op(cell_queues_get_total_allocation(), ==, + packed_cell_mem_cost() * 70); + tt_int_op(cell_queues_check_size(), ==, 0); /* We are still not OOM */ + + tv.tv_usec = 20*1000; + tor_gettimeofday_cache_set(&tv); + c3 = dummy_or_circuit_new(100, 85); + tt_int_op(cell_queues_check_size(), ==, 0); /* We are still not OOM */ + tt_int_op(cell_queues_get_total_allocation(), ==, + packed_cell_mem_cost() * 255); + + tv.tv_usec = 30*1000; + tor_gettimeofday_cache_set(&tv); + /* Adding this cell will trigger our OOM handler. */ + c4 = dummy_or_circuit_new(2, 0); + + tt_int_op(cell_queues_get_total_allocation(), ==, + packed_cell_mem_cost() * 257); + + tt_int_op(cell_queues_check_size(), ==, 1); /* We are now OOM */ + + tt_assert(c1->marked_for_close); + tt_assert(! c2->marked_for_close); + tt_assert(! c3->marked_for_close); + tt_assert(! c4->marked_for_close); + + tt_int_op(cell_queues_get_total_allocation(), ==, + packed_cell_mem_cost() * (257 - 30)); + + circuit_free(c1); + tv.tv_usec = 0; + tor_gettimeofday_cache_set(&tv); /* go back in time */ + c1 = dummy_or_circuit_new(90, 0); + + tv.tv_usec = 40*1000; /* go back to the future */ + tor_gettimeofday_cache_set(&tv); + + tt_int_op(cell_queues_check_size(), ==, 1); /* We are now OOM */ + + tt_assert(c1->marked_for_close); + tt_assert(! c2->marked_for_close); + tt_assert(! c3->marked_for_close); + tt_assert(! c4->marked_for_close); + + tt_int_op(cell_queues_get_total_allocation(), ==, + packed_cell_mem_cost() * (257 - 30)); + + done: + circuit_free(c1); + circuit_free(c2); + circuit_free(c3); + circuit_free(c4); + + UNMOCK(circuit_mark_for_close_); +} + +/** Run unit tests for buffers.c */ +static void +test_oom_streambuf(void *arg) +{ + or_options_t *options = get_options_mutable(); + circuit_t *c1 = NULL, *c2 = NULL, *c3 = NULL, *c4 = NULL, *c5 = NULL; + struct timeval tv = { 1389641159, 0 }; + uint32_t tvms; + int i; + smartlist_t *edgeconns = smartlist_new(); + + (void) arg; + + MOCK(circuit_mark_for_close_, circuit_mark_for_close_dummy_); + +#ifdef ENABLE_MEMPOOLS + init_cell_pool(); +#endif /* ENABLE_MEMPOOLS */ + + /* Far too low for real life. */ + options->MaxMemInQueues = 81*packed_cell_mem_cost() + 4096 * 34; + options->CellStatistics = 0; + + tt_int_op(cell_queues_check_size(), ==, 0); /* We don't start out OOM. */ + tt_int_op(cell_queues_get_total_allocation(), ==, 0); + tt_int_op(buf_get_total_allocation(), ==, 0); + + /* Start all circuits with a bit of data queued in cells */ + tv.tv_usec = 500*1000; /* go halfway into the second. */ + tor_gettimeofday_cache_set(&tv); + c1 = dummy_or_circuit_new(10,10); + tv.tv_usec = 510*1000; + tor_gettimeofday_cache_set(&tv); + c2 = dummy_origin_circuit_new(20); + tv.tv_usec = 520*1000; + tor_gettimeofday_cache_set(&tv); + c3 = dummy_or_circuit_new(20,20); + tv.tv_usec = 530*1000; + tor_gettimeofday_cache_set(&tv); + c4 = dummy_or_circuit_new(0,0); + tt_int_op(cell_queues_get_total_allocation(), ==, + packed_cell_mem_cost() * 80); + + tv.tv_usec = 600*1000; + tor_gettimeofday_cache_set(&tv); + + /* Add some connections to c1...c4. */ + for (i = 0; i < 4; ++i) { + edge_connection_t *ec; + /* link it to a circuit */ + tv.tv_usec += 10*1000; + tor_gettimeofday_cache_set(&tv); + ec = dummy_edge_conn_new(c1, CONN_TYPE_EXIT, 1000, 1000); + tt_assert(ec); + smartlist_add(edgeconns, ec); + tv.tv_usec += 10*1000; + tor_gettimeofday_cache_set(&tv); + ec = dummy_edge_conn_new(c2, CONN_TYPE_AP, 1000, 1000); + tt_assert(ec); + smartlist_add(edgeconns, ec); + tv.tv_usec += 10*1000; + tor_gettimeofday_cache_set(&tv); + ec = dummy_edge_conn_new(c4, CONN_TYPE_EXIT, 1000, 1000); /* Yes, 4 twice*/ + tt_assert(ec); + smartlist_add(edgeconns, ec); + tv.tv_usec += 10*1000; + tor_gettimeofday_cache_set(&tv); + ec = dummy_edge_conn_new(c4, CONN_TYPE_EXIT, 1000, 1000); + smartlist_add(edgeconns, ec); + tt_assert(ec); + } + + tv.tv_sec += 1; + tv.tv_usec = 0; + tvms = (uint32_t) tv_to_msec(&tv); + + tt_int_op(circuit_max_queued_cell_age(c1, tvms), ==, 500); + tt_int_op(circuit_max_queued_cell_age(c2, tvms), ==, 490); + tt_int_op(circuit_max_queued_cell_age(c3, tvms), ==, 480); + tt_int_op(circuit_max_queued_cell_age(c4, tvms), ==, 0); + + tt_int_op(circuit_max_queued_data_age(c1, tvms), ==, 390); + tt_int_op(circuit_max_queued_data_age(c2, tvms), ==, 380); + tt_int_op(circuit_max_queued_data_age(c3, tvms), ==, 0); + tt_int_op(circuit_max_queued_data_age(c4, tvms), ==, 370); + + tt_int_op(circuit_max_queued_item_age(c1, tvms), ==, 500); + tt_int_op(circuit_max_queued_item_age(c2, tvms), ==, 490); + tt_int_op(circuit_max_queued_item_age(c3, tvms), ==, 480); + tt_int_op(circuit_max_queued_item_age(c4, tvms), ==, 370); + + tt_int_op(cell_queues_get_total_allocation(), ==, + packed_cell_mem_cost() * 80); + tt_int_op(buf_get_total_allocation(), ==, 4096*16*2); + + /* Now give c4 a very old buffer of modest size */ + { + edge_connection_t *ec; + tv.tv_sec -= 1; + tv.tv_usec = 0; + tor_gettimeofday_cache_set(&tv); + ec = dummy_edge_conn_new(c4, CONN_TYPE_EXIT, 1000, 1000); + tt_assert(ec); + smartlist_add(edgeconns, ec); + } + tt_int_op(buf_get_total_allocation(), ==, 4096*17*2); + tt_int_op(circuit_max_queued_item_age(c4, tvms), ==, 1000); + + tt_int_op(cell_queues_check_size(), ==, 0); + + /* And run over the limit. */ + tv.tv_usec = 800*1000; + tor_gettimeofday_cache_set(&tv); + c5 = dummy_or_circuit_new(0,5); + + tt_int_op(cell_queues_get_total_allocation(), ==, + packed_cell_mem_cost() * 85); + tt_int_op(buf_get_total_allocation(), ==, 4096*17*2); + + tt_int_op(cell_queues_check_size(), ==, 1); /* We are now OOM */ + + /* C4 should have died. */ + tt_assert(! c1->marked_for_close); + tt_assert(! c2->marked_for_close); + tt_assert(! c3->marked_for_close); + tt_assert(c4->marked_for_close); + tt_assert(! c5->marked_for_close); + + tt_int_op(cell_queues_get_total_allocation(), ==, + packed_cell_mem_cost() * 85); + tt_int_op(buf_get_total_allocation(), ==, 4096*8*2); + + done: + circuit_free(c1); + circuit_free(c2); + circuit_free(c3); + circuit_free(c4); + circuit_free(c5); + + SMARTLIST_FOREACH(edgeconns, edge_connection_t *, ec, + connection_free_(TO_CONN(ec))); + smartlist_free(edgeconns); + + UNMOCK(circuit_mark_for_close_); +} + +struct testcase_t oom_tests[] = { + { "circbuf", test_oom_circbuf, TT_FORK, NULL, NULL }, + { "streambuf", test_oom_streambuf, TT_FORK, NULL, NULL }, + END_OF_TESTCASES +}; + diff --git a/src/test/test_options.c b/src/test/test_options.c new file mode 100644 index 0000000000..737f658e2c --- /dev/null +++ b/src/test/test_options.c @@ -0,0 +1,170 @@ +/* Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2013, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#define CONFIG_PRIVATE +#include "or.h" +#include "confparse.h" +#include "config.h" +#include "test.h" + +typedef struct { + int severity; + uint32_t domain; + char *msg; +} logmsg_t; + +static smartlist_t *messages = NULL; + +static void +log_cback(int severity, uint32_t domain, const char *msg) +{ + logmsg_t *x = tor_malloc(sizeof(*x)); + x->severity = severity; + x->domain = domain; + x->msg = tor_strdup(msg); + if (!messages) + messages = smartlist_new(); + smartlist_add(messages, x); +} + +static void +setup_log_callback(void) +{ + log_severity_list_t lst; + memset(&lst, 0, sizeof(lst)); + lst.masks[LOG_ERR - LOG_ERR] = ~0; + lst.masks[LOG_WARN - LOG_ERR] = ~0; + lst.masks[LOG_NOTICE - LOG_ERR] = ~0; + add_callback_log(&lst, log_cback); +} + +static char * +dump_logs(void) +{ + smartlist_t *msgs; + char *out; + if (! messages) + return tor_strdup(""); + msgs = smartlist_new(); + SMARTLIST_FOREACH_BEGIN(messages, logmsg_t *, x) { + smartlist_add_asprintf(msgs, "[%s] %s", + log_level_to_string(x->severity), x->msg); + } SMARTLIST_FOREACH_END(x); + out = smartlist_join_strings(msgs, "", 0, NULL); + SMARTLIST_FOREACH(msgs, char *, cp, tor_free(cp)); + smartlist_free(msgs); + return out; +} + +static void +clear_log_messages(void) +{ + if (!messages) + return; + SMARTLIST_FOREACH(messages, logmsg_t *, m, + { tor_free(m->msg); tor_free(m); }); + smartlist_free(messages); + messages = NULL; +} + +static void +test_options_validate_impl(const char *configuration, + const char *expect_errmsg, + int expect_log_severity, + const char *expect_log) +{ + or_options_t *opt = options_new(); + or_options_t *dflt; + config_line_t *cl=NULL; + char *msg=NULL; + int r; + opt->command = CMD_RUN_TOR; + options_init(opt); + + dflt = config_dup(&options_format, opt); + clear_log_messages(); + + r = config_get_lines(configuration, &cl, 1); + tt_int_op(r, ==, 0); + + r = config_assign(&options_format, opt, cl, 0, 0, &msg); + tt_int_op(r, ==, 0); + + r = options_validate(NULL, opt, dflt, 0, &msg); + if (expect_errmsg && !msg) { + TT_DIE(("Expected error message <%s> from <%s>, but got none.", + expect_errmsg, configuration)); + } else if (expect_errmsg && !strstr(msg, expect_errmsg)) { + TT_DIE(("Expected error message <%s> from <%s>, but got <%s>.", + expect_errmsg, configuration, msg)); + } else if (!expect_errmsg && msg) { + TT_DIE(("Expected no error message from <%s> but got <%s>.", + configuration, msg)); + } + tt_int_op((r == 0), ==, (msg == NULL)); + + if (expect_log) { + int found = 0; + if (messages) { + SMARTLIST_FOREACH_BEGIN(messages, logmsg_t *, m) { + if (m->severity == expect_log_severity && + strstr(m->msg, expect_log)) { + found = 1; + break; + } + } SMARTLIST_FOREACH_END(m); + } + if (!found) { + tor_free(msg); + msg = dump_logs(); + TT_DIE(("Expected log message [%s] %s from <%s>, but got <%s>.", + log_level_to_string(expect_log_severity), expect_log, + configuration, msg)); + } + } + + done: + config_free_lines(cl); + or_options_free(opt); + or_options_free(dflt); + tor_free(msg); + clear_log_messages(); +} + +#define WANT_ERR(config, msg) \ + test_options_validate_impl((config), (msg), 0, NULL) +#define WANT_LOG(config, severity, msg) \ + test_options_validate_impl((config), NULL, (severity), (msg)) +#define WANT_ERR_LOG(config, msg, severity, logmsg) \ + test_options_validate_impl((config), (msg), (severity), (logmsg)) +#define OK(config) \ + test_options_validate_impl((config), NULL, 0, NULL) + +static void +test_options_validate(void *arg) +{ + (void)arg; + setup_log_callback(); + + WANT_ERR("ExtORPort 500000", "Invalid ExtORPort"); + + WANT_ERR_LOG("ServerTransportOptions trebuchet", + "ServerTransportOptions did not parse", + LOG_WARN, "Too few arguments"); + OK("ServerTransportOptions trebuchet sling=snappy"); + OK("ServerTransportOptions trebuchet sling="); + WANT_ERR_LOG("ServerTransportOptions trebuchet slingsnappy", + "ServerTransportOptions did not parse", + LOG_WARN, "\"slingsnappy\" is not a k=v"); + + clear_log_messages(); + return; +} + +struct testcase_t options_tests[] = { + { "validate", test_options_validate, TT_FORK, NULL, NULL }, + END_OF_TESTCASES +}; + diff --git a/src/test/test_policy.c b/src/test/test_policy.c new file mode 100644 index 0000000000..4cdcd034bb --- /dev/null +++ b/src/test/test_policy.c @@ -0,0 +1,437 @@ +/* Copyright (c) 2013, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "or.h" +#include "router.h" +#include "routerparse.h" +#include "policies.h" +#include "test.h" + +/* Helper: assert that short_policy parses and writes back out as itself, + or as <b>expected</b> if that's provided. */ +static void +test_short_policy_parse(const char *input, + const char *expected) +{ + short_policy_t *short_policy = NULL; + char *out = NULL; + + if (expected == NULL) + expected = input; + + short_policy = parse_short_policy(input); + tt_assert(short_policy); + out = write_short_policy(short_policy); + tt_str_op(out, ==, expected); + + done: + tor_free(out); + short_policy_free(short_policy); +} + +/** Helper: Parse the exit policy string in <b>policy_str</b>, and make sure + * that policies_summarize() produces the string <b>expected_summary</b> from + * it. */ +static void +test_policy_summary_helper(const char *policy_str, + const char *expected_summary) +{ + config_line_t line; + smartlist_t *policy = smartlist_new(); + char *summary = NULL; + char *summary_after = NULL; + int r; + short_policy_t *short_policy = NULL; + + line.key = (char*)"foo"; + line.value = (char *)policy_str; + line.next = NULL; + + r = policies_parse_exit_policy(&line, &policy, 1, 0, 0, 1); + test_eq(r, 0); + summary = policy_summarize(policy, AF_INET); + + test_assert(summary != NULL); + test_streq(summary, expected_summary); + + short_policy = parse_short_policy(summary); + tt_assert(short_policy); + summary_after = write_short_policy(short_policy); + test_streq(summary, summary_after); + + done: + tor_free(summary_after); + tor_free(summary); + if (policy) + addr_policy_list_free(policy); + short_policy_free(short_policy); +} + +/** Run unit tests for generating summary lines of exit policies */ +static void +test_policies_general(void *arg) +{ + int i; + smartlist_t *policy = NULL, *policy2 = NULL, *policy3 = NULL, + *policy4 = NULL, *policy5 = NULL, *policy6 = NULL, + *policy7 = NULL; + addr_policy_t *p; + tor_addr_t tar; + config_line_t line; + smartlist_t *sm = NULL; + char *policy_str = NULL; + short_policy_t *short_parsed = NULL; + (void)arg; + + policy = smartlist_new(); + + p = router_parse_addr_policy_item_from_string("reject 192.168.0.0/16:*",-1); + test_assert(p != NULL); + test_eq(ADDR_POLICY_REJECT, p->policy_type); + tor_addr_from_ipv4h(&tar, 0xc0a80000u); + test_eq(0, tor_addr_compare(&p->addr, &tar, CMP_EXACT)); + test_eq(16, p->maskbits); + test_eq(1, p->prt_min); + test_eq(65535, p->prt_max); + + smartlist_add(policy, p); + + tor_addr_from_ipv4h(&tar, 0x01020304u); + test_assert(ADDR_POLICY_ACCEPTED == + compare_tor_addr_to_addr_policy(&tar, 2, policy)); + tor_addr_make_unspec(&tar); + test_assert(ADDR_POLICY_PROBABLY_ACCEPTED == + compare_tor_addr_to_addr_policy(&tar, 2, policy)); + tor_addr_from_ipv4h(&tar, 0xc0a80102); + test_assert(ADDR_POLICY_REJECTED == + compare_tor_addr_to_addr_policy(&tar, 2, policy)); + + test_assert(0 == policies_parse_exit_policy(NULL, &policy2, 1, 1, 0, 1)); + test_assert(policy2); + + policy3 = smartlist_new(); + p = router_parse_addr_policy_item_from_string("reject *:*",-1); + test_assert(p != NULL); + smartlist_add(policy3, p); + p = router_parse_addr_policy_item_from_string("accept *:*",-1); + test_assert(p != NULL); + smartlist_add(policy3, p); + + policy4 = smartlist_new(); + p = router_parse_addr_policy_item_from_string("accept *:443",-1); + test_assert(p != NULL); + smartlist_add(policy4, p); + p = router_parse_addr_policy_item_from_string("accept *:443",-1); + test_assert(p != NULL); + smartlist_add(policy4, p); + + policy5 = smartlist_new(); + p = router_parse_addr_policy_item_from_string("reject 0.0.0.0/8:*",-1); + test_assert(p != NULL); + smartlist_add(policy5, p); + p = router_parse_addr_policy_item_from_string("reject 169.254.0.0/16:*",-1); + test_assert(p != NULL); + smartlist_add(policy5, p); + p = router_parse_addr_policy_item_from_string("reject 127.0.0.0/8:*",-1); + test_assert(p != NULL); + smartlist_add(policy5, p); + p = router_parse_addr_policy_item_from_string("reject 192.168.0.0/16:*",-1); + test_assert(p != NULL); + smartlist_add(policy5, p); + p = router_parse_addr_policy_item_from_string("reject 10.0.0.0/8:*",-1); + test_assert(p != NULL); + smartlist_add(policy5, p); + p = router_parse_addr_policy_item_from_string("reject 172.16.0.0/12:*",-1); + test_assert(p != NULL); + smartlist_add(policy5, p); + p = router_parse_addr_policy_item_from_string("reject 80.190.250.90:*",-1); + test_assert(p != NULL); + smartlist_add(policy5, p); + p = router_parse_addr_policy_item_from_string("reject *:1-65534",-1); + test_assert(p != NULL); + smartlist_add(policy5, p); + p = router_parse_addr_policy_item_from_string("reject *:65535",-1); + test_assert(p != NULL); + smartlist_add(policy5, p); + p = router_parse_addr_policy_item_from_string("accept *:1-65535",-1); + test_assert(p != NULL); + smartlist_add(policy5, p); + + policy6 = smartlist_new(); + p = router_parse_addr_policy_item_from_string("accept 43.3.0.0/9:*",-1); + test_assert(p != NULL); + smartlist_add(policy6, p); + + policy7 = smartlist_new(); + p = router_parse_addr_policy_item_from_string("accept 0.0.0.0/8:*",-1); + test_assert(p != NULL); + smartlist_add(policy7, p); + + test_assert(!exit_policy_is_general_exit(policy)); + test_assert(exit_policy_is_general_exit(policy2)); + test_assert(!exit_policy_is_general_exit(NULL)); + test_assert(!exit_policy_is_general_exit(policy3)); + test_assert(!exit_policy_is_general_exit(policy4)); + test_assert(!exit_policy_is_general_exit(policy5)); + test_assert(!exit_policy_is_general_exit(policy6)); + test_assert(!exit_policy_is_general_exit(policy7)); + + test_assert(cmp_addr_policies(policy, policy2)); + test_assert(cmp_addr_policies(policy, NULL)); + test_assert(!cmp_addr_policies(policy2, policy2)); + test_assert(!cmp_addr_policies(NULL, NULL)); + + test_assert(!policy_is_reject_star(policy2, AF_INET)); + test_assert(policy_is_reject_star(policy, AF_INET)); + test_assert(policy_is_reject_star(NULL, AF_INET)); + + addr_policy_list_free(policy); + policy = NULL; + + /* make sure compacting logic works. */ + policy = NULL; + line.key = (char*)"foo"; + line.value = (char*)"accept *:80,reject private:*,reject *:*"; + line.next = NULL; + test_assert(0 == policies_parse_exit_policy(&line, &policy, 1, 0, 0, 1)); + test_assert(policy); + //test_streq(policy->string, "accept *:80"); + //test_streq(policy->next->string, "reject *:*"); + test_eq(smartlist_len(policy), 4); + + /* test policy summaries */ + /* check if we properly ignore private IP addresses */ + test_policy_summary_helper("reject 192.168.0.0/16:*," + "reject 0.0.0.0/8:*," + "reject 10.0.0.0/8:*," + "accept *:10-30," + "accept *:90," + "reject *:*", + "accept 10-30,90"); + /* check all accept policies, and proper counting of rejects */ + test_policy_summary_helper("reject 11.0.0.0/9:80," + "reject 12.0.0.0/9:80," + "reject 13.0.0.0/9:80," + "reject 14.0.0.0/9:80," + "accept *:*", "accept 1-65535"); + test_policy_summary_helper("reject 11.0.0.0/9:80," + "reject 12.0.0.0/9:80," + "reject 13.0.0.0/9:80," + "reject 14.0.0.0/9:80," + "reject 15.0.0.0:81," + "accept *:*", "accept 1-65535"); + test_policy_summary_helper("reject 11.0.0.0/9:80," + "reject 12.0.0.0/9:80," + "reject 13.0.0.0/9:80," + "reject 14.0.0.0/9:80," + "reject 15.0.0.0:80," + "accept *:*", + "reject 80"); + /* no exits */ + test_policy_summary_helper("accept 11.0.0.0/9:80," + "reject *:*", + "reject 1-65535"); + /* port merging */ + test_policy_summary_helper("accept *:80," + "accept *:81," + "accept *:100-110," + "accept *:111," + "reject *:*", + "accept 80-81,100-111"); + /* border ports */ + test_policy_summary_helper("accept *:1," + "accept *:3," + "accept *:65535," + "reject *:*", + "accept 1,3,65535"); + /* holes */ + test_policy_summary_helper("accept *:1," + "accept *:3," + "accept *:5," + "accept *:7," + "reject *:*", + "accept 1,3,5,7"); + test_policy_summary_helper("reject *:1," + "reject *:3," + "reject *:5," + "reject *:7," + "accept *:*", + "reject 1,3,5,7"); + + /* Short policies with unrecognized formats should get accepted. */ + test_short_policy_parse("accept fred,2,3-5", "accept 2,3-5"); + test_short_policy_parse("accept 2,fred,3", "accept 2,3"); + test_short_policy_parse("accept 2,fred,3,bob", "accept 2,3"); + test_short_policy_parse("accept 2,-3,500-600", "accept 2,500-600"); + /* Short policies with nil entries are accepted too. */ + test_short_policy_parse("accept 1,,3", "accept 1,3"); + test_short_policy_parse("accept 100-200,,", "accept 100-200"); + test_short_policy_parse("reject ,1-10,,,,30-40", "reject 1-10,30-40"); + + /* Try parsing various broken short policies */ +#define TT_BAD_SHORT_POLICY(s) \ + do { \ + tt_ptr_op(NULL, ==, (short_parsed = parse_short_policy((s)))); \ + } while (0) + TT_BAD_SHORT_POLICY("accept 200-199"); + TT_BAD_SHORT_POLICY(""); + TT_BAD_SHORT_POLICY("rejekt 1,2,3"); + TT_BAD_SHORT_POLICY("reject "); + TT_BAD_SHORT_POLICY("reject"); + TT_BAD_SHORT_POLICY("rej"); + TT_BAD_SHORT_POLICY("accept 2,3,100000"); + TT_BAD_SHORT_POLICY("accept 2,3x,4"); + TT_BAD_SHORT_POLICY("accept 2,3x,4"); + TT_BAD_SHORT_POLICY("accept 2-"); + TT_BAD_SHORT_POLICY("accept 2-x"); + TT_BAD_SHORT_POLICY("accept 1-,3"); + TT_BAD_SHORT_POLICY("accept 1-,3"); + + /* Test a too-long policy. */ + { + int i; + char *policy = NULL; + smartlist_t *chunks = smartlist_new(); + smartlist_add(chunks, tor_strdup("accept ")); + for (i=1; i<10000; ++i) + smartlist_add_asprintf(chunks, "%d,", i); + smartlist_add(chunks, tor_strdup("20000")); + policy = smartlist_join_strings(chunks, "", 0, NULL); + SMARTLIST_FOREACH(chunks, char *, ch, tor_free(ch)); + smartlist_free(chunks); + short_parsed = parse_short_policy(policy);/* shouldn't be accepted */ + tor_free(policy); + tt_ptr_op(NULL, ==, short_parsed); + } + + /* truncation ports */ + sm = smartlist_new(); + for (i=1; i<2000; i+=2) { + char buf[POLICY_BUF_LEN]; + tor_snprintf(buf, sizeof(buf), "reject *:%d", i); + smartlist_add(sm, tor_strdup(buf)); + } + smartlist_add(sm, tor_strdup("accept *:*")); + policy_str = smartlist_join_strings(sm, ",", 0, NULL); + test_policy_summary_helper( policy_str, + "accept 2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40,42,44," + "46,48,50,52,54,56,58,60,62,64,66,68,70,72,74,76,78,80,82,84,86,88,90," + "92,94,96,98,100,102,104,106,108,110,112,114,116,118,120,122,124,126,128," + "130,132,134,136,138,140,142,144,146,148,150,152,154,156,158,160,162,164," + "166,168,170,172,174,176,178,180,182,184,186,188,190,192,194,196,198,200," + "202,204,206,208,210,212,214,216,218,220,222,224,226,228,230,232,234,236," + "238,240,242,244,246,248,250,252,254,256,258,260,262,264,266,268,270,272," + "274,276,278,280,282,284,286,288,290,292,294,296,298,300,302,304,306,308," + "310,312,314,316,318,320,322,324,326,328,330,332,334,336,338,340,342,344," + "346,348,350,352,354,356,358,360,362,364,366,368,370,372,374,376,378,380," + "382,384,386,388,390,392,394,396,398,400,402,404,406,408,410,412,414,416," + "418,420,422,424,426,428,430,432,434,436,438,440,442,444,446,448,450,452," + "454,456,458,460,462,464,466,468,470,472,474,476,478,480,482,484,486,488," + "490,492,494,496,498,500,502,504,506,508,510,512,514,516,518,520,522"); + + done: + addr_policy_list_free(policy); + addr_policy_list_free(policy2); + addr_policy_list_free(policy3); + addr_policy_list_free(policy4); + addr_policy_list_free(policy5); + addr_policy_list_free(policy6); + addr_policy_list_free(policy7); + tor_free(policy_str); + if (sm) { + SMARTLIST_FOREACH(sm, char *, s, tor_free(s)); + smartlist_free(sm); + } + short_policy_free(short_parsed); +} + +static void +test_dump_exit_policy_to_string(void *arg) +{ + char *ep; + addr_policy_t *policy_entry; + + routerinfo_t *ri = tor_malloc_zero(sizeof(routerinfo_t)); + + (void)arg; + + ri->policy_is_reject_star = 1; + ri->exit_policy = NULL; // expecting "reject *:*" + ep = router_dump_exit_policy_to_string(ri,1,1); + + test_streq("reject *:*",ep); + + tor_free(ep); + + ri->exit_policy = smartlist_new(); + ri->policy_is_reject_star = 0; + + policy_entry = router_parse_addr_policy_item_from_string("accept *:*",-1); + + smartlist_add(ri->exit_policy,policy_entry); + + ep = router_dump_exit_policy_to_string(ri,1,1); + + test_streq("accept *:*",ep); + + tor_free(ep); + + policy_entry = router_parse_addr_policy_item_from_string("reject *:25",-1); + + smartlist_add(ri->exit_policy,policy_entry); + + ep = router_dump_exit_policy_to_string(ri,1,1); + + test_streq("accept *:*\nreject *:25",ep); + + tor_free(ep); + + policy_entry = + router_parse_addr_policy_item_from_string("reject 8.8.8.8:*",-1); + + smartlist_add(ri->exit_policy,policy_entry); + + ep = router_dump_exit_policy_to_string(ri,1,1); + + test_streq("accept *:*\nreject *:25\nreject 8.8.8.8:*",ep); + tor_free(ep); + + policy_entry = + router_parse_addr_policy_item_from_string("reject6 [FC00::]/7:*",-1); + + smartlist_add(ri->exit_policy,policy_entry); + + ep = router_dump_exit_policy_to_string(ri,1,1); + + test_streq("accept *:*\nreject *:25\nreject 8.8.8.8:*\n" + "reject6 [fc00::]/7:*",ep); + tor_free(ep); + + policy_entry = + router_parse_addr_policy_item_from_string("accept6 [c000::]/3:*",-1); + + smartlist_add(ri->exit_policy,policy_entry); + + ep = router_dump_exit_policy_to_string(ri,1,1); + + test_streq("accept *:*\nreject *:25\nreject 8.8.8.8:*\n" + "reject6 [fc00::]/7:*\naccept6 [c000::]/3:*",ep); + + done: + + if (ri->exit_policy) { + SMARTLIST_FOREACH(ri->exit_policy, addr_policy_t *, + entry, addr_policy_free(entry)); + smartlist_free(ri->exit_policy); + } + tor_free(ri); + tor_free(ep); +} + +struct testcase_t policy_tests[] = { + { "router_dump_exit_policy_to_string", test_dump_exit_policy_to_string, 0, + NULL, NULL }, + { "general", test_policies_general, 0, NULL, NULL }, + END_OF_TESTCASES +}; + diff --git a/src/test/test_pt.c b/src/test/test_pt.c index 80707f4379..f71627df1e 100644 --- a/src/test/test_pt.c +++ b/src/test/test_pt.c @@ -5,9 +5,17 @@ #include "orconfig.h" #define PT_PRIVATE +#define UTIL_PRIVATE +#define STATEFILE_PRIVATE +#define CONTROL_PRIVATE #include "or.h" +#include "config.h" +#include "confparse.h" +#include "control.h" #include "transports.h" #include "circuitbuild.h" +#include "util.h" +#include "statefile.h" #include "test.h" static void @@ -22,71 +30,163 @@ static void test_pt_parsing(void) { char line[200]; + transport_t *transport = NULL; + tor_addr_t test_addr; managed_proxy_t *mp = tor_malloc(sizeof(managed_proxy_t)); mp->conf_state = PT_PROTO_INFANT; mp->transports = smartlist_new(); /* incomplete cmethod */ - strcpy(line,"CMETHOD trebuchet"); + strlcpy(line,"CMETHOD trebuchet",sizeof(line)); test_assert(parse_cmethod_line(line, mp) < 0); reset_mp(mp); /* wrong proxy type */ - strcpy(line,"CMETHOD trebuchet dog 127.0.0.1:1999"); + strlcpy(line,"CMETHOD trebuchet dog 127.0.0.1:1999",sizeof(line)); test_assert(parse_cmethod_line(line, mp) < 0); reset_mp(mp); /* wrong addrport */ - strcpy(line,"CMETHOD trebuchet socks4 abcd"); + strlcpy(line,"CMETHOD trebuchet socks4 abcd",sizeof(line)); test_assert(parse_cmethod_line(line, mp) < 0); reset_mp(mp); /* correct line */ - strcpy(line,"CMETHOD trebuchet socks5 127.0.0.1:1999"); + strlcpy(line,"CMETHOD trebuchet socks5 127.0.0.1:1999",sizeof(line)); test_assert(parse_cmethod_line(line, mp) == 0); - test_assert(smartlist_len(mp->transports)); + test_assert(smartlist_len(mp->transports) == 1); + transport = smartlist_get(mp->transports, 0); + /* test registered address of transport */ + tor_addr_parse(&test_addr, "127.0.0.1"); + test_assert(tor_addr_eq(&test_addr, &transport->addr)); + /* test registered port of transport */ + test_assert(transport->port == 1999); + /* test registered SOCKS version of transport */ + test_assert(transport->socks_version == PROXY_SOCKS5); + /* test registered name of transport */ + test_streq(transport->name, "trebuchet"); reset_mp(mp); /* incomplete smethod */ - strcpy(line,"SMETHOD trebuchet"); + strlcpy(line,"SMETHOD trebuchet",sizeof(line)); test_assert(parse_smethod_line(line, mp) < 0); reset_mp(mp); /* wrong addr type */ - strcpy(line,"SMETHOD trebuchet abcd"); + strlcpy(line,"SMETHOD trebuchet abcd",sizeof(line)); test_assert(parse_smethod_line(line, mp) < 0); reset_mp(mp); /* cowwect */ - strcpy(line,"SMETHOD trebuchy 127.0.0.1:1999"); + strlcpy(line,"SMETHOD trebuchy 127.0.0.2:2999",sizeof(line)); test_assert(parse_smethod_line(line, mp) == 0); + test_assert(smartlist_len(mp->transports) == 1); + transport = smartlist_get(mp->transports, 0); + /* test registered address of transport */ + tor_addr_parse(&test_addr, "127.0.0.2"); + test_assert(tor_addr_eq(&test_addr, &transport->addr)); + /* test registered port of transport */ + test_assert(transport->port == 2999); + /* test registered name of transport */ + test_streq(transport->name, "trebuchy"); reset_mp(mp); + /* Include some arguments. Good ones. */ + strlcpy(line,"SMETHOD trebuchet 127.0.0.1:9999 " + "ARGS:counterweight=3,sling=snappy", + sizeof(line)); + test_assert(parse_smethod_line(line, mp) == 0); + tt_int_op(1, ==, smartlist_len(mp->transports)); + { + const transport_t *transport = smartlist_get(mp->transports, 0); + tt_assert(transport); + tt_str_op(transport->name, ==, "trebuchet"); + tt_int_op(transport->port, ==, 9999); + tt_str_op(fmt_addr(&transport->addr), ==, "127.0.0.1"); + tt_str_op(transport->extra_info_args, ==, + "counterweight=3,sling=snappy"); + } + reset_mp(mp); + /* unsupported version */ - strcpy(line,"VERSION 666"); + strlcpy(line,"VERSION 666",sizeof(line)); test_assert(parse_version(line, mp) < 0); /* incomplete VERSION */ - strcpy(line,"VERSION "); + strlcpy(line,"VERSION ",sizeof(line)); test_assert(parse_version(line, mp) < 0); /* correct VERSION */ - strcpy(line,"VERSION 1"); + strlcpy(line,"VERSION 1",sizeof(line)); test_assert(parse_version(line, mp) == 0); done: + reset_mp(mp); + smartlist_free(mp->transports); tor_free(mp); } static void +test_pt_get_transport_options(void *arg) +{ + char **execve_args; + smartlist_t *transport_list = smartlist_new(); + managed_proxy_t *mp; + or_options_t *options = get_options_mutable(); + char *opt_str = NULL; + config_line_t *cl = NULL; + (void)arg; + + execve_args = tor_malloc(sizeof(char*)*2); + execve_args[0] = tor_strdup("cheeseshop"); + execve_args[1] = NULL; + + mp = managed_proxy_create(transport_list, execve_args, 1); + tt_ptr_op(mp, !=, NULL); + opt_str = get_transport_options_for_server_proxy(mp); + tt_ptr_op(opt_str, ==, NULL); + + smartlist_add(mp->transports_to_launch, tor_strdup("gruyere")); + smartlist_add(mp->transports_to_launch, tor_strdup("roquefort")); + smartlist_add(mp->transports_to_launch, tor_strdup("stnectaire")); + + tt_assert(options); + + cl = tor_malloc_zero(sizeof(config_line_t)); + cl->value = tor_strdup("gruyere melty=10 hardness=se;ven"); + options->ServerTransportOptions = cl; + + cl = tor_malloc_zero(sizeof(config_line_t)); + cl->value = tor_strdup("stnectaire melty=4 hardness=three"); + cl->next = options->ServerTransportOptions; + options->ServerTransportOptions = cl; + + cl = tor_malloc_zero(sizeof(config_line_t)); + cl->value = tor_strdup("pepperjack melty=12 hardness=five"); + cl->next = options->ServerTransportOptions; + options->ServerTransportOptions = cl; + + opt_str = get_transport_options_for_server_proxy(mp); + tt_str_op(opt_str, ==, + "gruyere:melty=10;gruyere:hardness=se\\;ven;" + "stnectaire:melty=4;stnectaire:hardness=three"); + + done: + tor_free(opt_str); + config_free_lines(cl); + managed_proxy_destroy(mp, 0); + smartlist_free(transport_list); +} + +static void test_pt_protocol(void) { char line[200]; @@ -99,36 +199,254 @@ test_pt_protocol(void) /* various wrong protocol runs: */ - strcpy(line,"VERSION 1"); + strlcpy(line,"VERSION 1",sizeof(line)); handle_proxy_line(line, mp); test_assert(mp->conf_state == PT_PROTO_ACCEPTING_METHODS); - strcpy(line,"VERSION 1"); + strlcpy(line,"VERSION 1",sizeof(line)); handle_proxy_line(line, mp); test_assert(mp->conf_state == PT_PROTO_BROKEN); reset_mp(mp); - strcpy(line,"CMETHOD trebuchet socks5 127.0.0.1:1999"); + strlcpy(line,"CMETHOD trebuchet socks5 127.0.0.1:1999",sizeof(line)); handle_proxy_line(line, mp); test_assert(mp->conf_state == PT_PROTO_BROKEN); reset_mp(mp); /* correct protocol run: */ - strcpy(line,"VERSION 1"); + strlcpy(line,"VERSION 1",sizeof(line)); handle_proxy_line(line, mp); test_assert(mp->conf_state == PT_PROTO_ACCEPTING_METHODS); - strcpy(line,"CMETHOD trebuchet socks5 127.0.0.1:1999"); + strlcpy(line,"CMETHOD trebuchet socks5 127.0.0.1:1999",sizeof(line)); handle_proxy_line(line, mp); test_assert(mp->conf_state == PT_PROTO_ACCEPTING_METHODS); - strcpy(line,"CMETHODS DONE"); + strlcpy(line,"CMETHODS DONE",sizeof(line)); handle_proxy_line(line, mp); test_assert(mp->conf_state == PT_PROTO_CONFIGURED); done: + reset_mp(mp); + smartlist_free(mp->transports); + tor_free(mp->argv[0]); + tor_free(mp->argv); + tor_free(mp); +} + +static void +test_pt_get_extrainfo_string(void *arg) +{ + managed_proxy_t *mp1 = NULL, *mp2 = NULL; + char **argv1, **argv2; + smartlist_t *t1 = smartlist_new(), *t2 = smartlist_new(); + int r; + char *s = NULL; + (void) arg; + + argv1 = tor_malloc_zero(sizeof(char*)*3); + argv1[0] = tor_strdup("ewige"); + argv1[1] = tor_strdup("Blumenkraft"); + argv1[2] = NULL; + argv2 = tor_malloc_zero(sizeof(char*)*4); + argv2[0] = tor_strdup("und"); + argv2[1] = tor_strdup("ewige"); + argv2[2] = tor_strdup("Schlangenkraft"); + argv2[3] = NULL; + + mp1 = managed_proxy_create(t1, argv1, 1); + mp2 = managed_proxy_create(t2, argv2, 1); + + r = parse_smethod_line("SMETHOD hagbard 127.0.0.1:5555", mp1); + tt_int_op(r, ==, 0); + r = parse_smethod_line("SMETHOD celine 127.0.0.1:1723 ARGS:card=no-enemy", + mp2); + tt_int_op(r, ==, 0); + + /* Force these proxies to look "completed" or they won't generate output. */ + mp1->conf_state = mp2->conf_state = PT_PROTO_COMPLETED; + + s = pt_get_extra_info_descriptor_string(); + tt_assert(s); + tt_str_op(s, ==, + "transport hagbard 127.0.0.1:5555\n" + "transport celine 127.0.0.1:1723 card=no-enemy\n"); + + done: + /* XXXX clean up better */ + smartlist_free(t1); + smartlist_free(t2); + tor_free(s); +} + +#ifdef _WIN32 +#define STDIN_HANDLE HANDLE +#else +#define STDIN_HANDLE FILE +#endif + +static smartlist_t * +tor_get_lines_from_handle_replacement(STDIN_HANDLE *handle, + enum stream_status *stream_status_out) +{ + static int times_called = 0; + smartlist_t *retval_sl = smartlist_new(); + + (void) handle; + (void) stream_status_out; + + /* Generate some dummy CMETHOD lines the first 5 times. The 6th + time, send 'CMETHODS DONE' to finish configuring the proxy. */ + if (times_called++ != 5) { + smartlist_add_asprintf(retval_sl, "SMETHOD mock%d 127.0.0.1:555%d", + times_called, times_called); + } else { + smartlist_add(retval_sl, tor_strdup("SMETHODS DONE")); + } + + return retval_sl; +} + +/* NOP mock */ +static void +tor_process_handle_destroy_replacement(process_handle_t *process_handle, + int also_terminate_process) +{ + (void) process_handle; + (void) also_terminate_process; +} + +static or_state_t *dummy_state = NULL; + +static or_state_t * +get_or_state_replacement(void) +{ + return dummy_state; +} + +static int controlevent_n = 0; +static uint16_t controlevent_event = 0; +static smartlist_t *controlevent_msgs = NULL; + +static void +send_control_event_string_replacement(uint16_t event, event_format_t which, + const char *msg) +{ + (void) which; + ++controlevent_n; + controlevent_event = event; + if (!controlevent_msgs) + controlevent_msgs = smartlist_new(); + smartlist_add(controlevent_msgs, tor_strdup(msg)); +} + +/* Test the configure_proxy() function. */ +static void +test_pt_configure_proxy(void *arg) +{ + int i, retval; + managed_proxy_t *mp = NULL; + (void) arg; + + dummy_state = tor_malloc_zero(sizeof(or_state_t)); + + MOCK(tor_get_lines_from_handle, + tor_get_lines_from_handle_replacement); + MOCK(tor_process_handle_destroy, + tor_process_handle_destroy_replacement); + MOCK(get_or_state, + get_or_state_replacement); + MOCK(send_control_event_string, + send_control_event_string_replacement); + + control_testing_set_global_event_mask(EVENT_TRANSPORT_LAUNCHED); + + mp = tor_malloc(sizeof(managed_proxy_t)); + mp->conf_state = PT_PROTO_ACCEPTING_METHODS; + mp->transports = smartlist_new(); + mp->transports_to_launch = smartlist_new(); + mp->process_handle = tor_malloc_zero(sizeof(process_handle_t)); + mp->argv = tor_malloc_zero(sizeof(char*)*2); + mp->argv[0] = tor_strdup("<testcase>"); + mp->is_server = 1; + + /* Test the return value of configure_proxy() by calling it some + times while it is uninitialized and then finally finalizing its + configuration. */ + for (i = 0 ; i < 5 ; i++) { + retval = configure_proxy(mp); + /* retval should be zero because proxy hasn't finished configuring yet */ + test_assert(retval == 0); + /* check the number of registered transports */ + test_assert(smartlist_len(mp->transports) == i+1); + /* check that the mp is still waiting for transports */ + test_assert(mp->conf_state == PT_PROTO_ACCEPTING_METHODS); + } + + /* this last configure_proxy() should finalize the proxy configuration. */ + retval = configure_proxy(mp); + /* retval should be 1 since the proxy finished configuring */ + test_assert(retval == 1); + /* check the mp state */ + test_assert(mp->conf_state == PT_PROTO_COMPLETED); + + tt_int_op(controlevent_n, ==, 5); + tt_int_op(controlevent_event, ==, EVENT_TRANSPORT_LAUNCHED); + tt_int_op(smartlist_len(controlevent_msgs), ==, 5); + smartlist_sort_strings(controlevent_msgs); + tt_str_op(smartlist_get(controlevent_msgs, 0), ==, + "650 TRANSPORT_LAUNCHED server mock1 127.0.0.1 5551\r\n"); + tt_str_op(smartlist_get(controlevent_msgs, 1), ==, + "650 TRANSPORT_LAUNCHED server mock2 127.0.0.1 5552\r\n"); + tt_str_op(smartlist_get(controlevent_msgs, 2), ==, + "650 TRANSPORT_LAUNCHED server mock3 127.0.0.1 5553\r\n"); + tt_str_op(smartlist_get(controlevent_msgs, 3), ==, + "650 TRANSPORT_LAUNCHED server mock4 127.0.0.1 5554\r\n"); + tt_str_op(smartlist_get(controlevent_msgs, 4), ==, + "650 TRANSPORT_LAUNCHED server mock5 127.0.0.1 5555\r\n"); + + { /* check that the transport info were saved properly in the tor state */ + config_line_t *transport_in_state = NULL; + smartlist_t *transport_info_sl = smartlist_new(); + char *name_of_transport = NULL; + char *bindaddr = NULL; + + /* Get the bindaddr for "mock1" and check it against the bindaddr + that the mocked tor_get_lines_from_handle() generated. */ + transport_in_state = get_transport_in_state_by_name("mock1"); + test_assert(transport_in_state); + smartlist_split_string(transport_info_sl, transport_in_state->value, + NULL, 0, 0); + name_of_transport = smartlist_get(transport_info_sl, 0); + bindaddr = smartlist_get(transport_info_sl, 1); + tt_str_op(name_of_transport, ==, "mock1"); + tt_str_op(bindaddr, ==, "127.0.0.1:5551"); + + SMARTLIST_FOREACH(transport_info_sl, char *, cp, tor_free(cp)); + smartlist_free(transport_info_sl); + } + + done: + or_state_free(dummy_state); + UNMOCK(tor_get_lines_from_handle); + UNMOCK(tor_process_handle_destroy); + UNMOCK(get_or_state); + UNMOCK(send_control_event_string); + if (controlevent_msgs) { + SMARTLIST_FOREACH(controlevent_msgs, char *, cp, tor_free(cp)); + smartlist_free(controlevent_msgs); + controlevent_msgs = NULL; + } + if (mp->transports) { + SMARTLIST_FOREACH(mp->transports, transport_t *, t, transport_free(t)); + smartlist_free(mp->transports); + } + smartlist_free(mp->transports_to_launch); + tor_free(mp->process_handle); + tor_free(mp->argv[0]); + tor_free(mp->argv); tor_free(mp); } @@ -138,6 +456,12 @@ test_pt_protocol(void) struct testcase_t pt_tests[] = { PT_LEGACY(parsing), PT_LEGACY(protocol), + { "get_transport_options", test_pt_get_transport_options, TT_FORK, + NULL, NULL }, + { "get_extrainfo_string", test_pt_get_extrainfo_string, TT_FORK, + NULL, NULL }, + { "configure_proxy",test_pt_configure_proxy, TT_FORK, + NULL, NULL }, END_OF_TESTCASES }; diff --git a/src/test/test_relaycell.c b/src/test/test_relaycell.c new file mode 100644 index 0000000000..5deb36260f --- /dev/null +++ b/src/test/test_relaycell.c @@ -0,0 +1,249 @@ +/* Copyright (c) 2014, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/* Unit tests for handling different kinds of relay cell */ + +#define RELAY_PRIVATE +#include "or.h" +#include "config.h" +#include "connection.h" +#include "connection_edge.h" +#include "relay.h" +#include "test.h" + +static int srm_ncalls; +static entry_connection_t *srm_conn; +static int srm_atype; +static size_t srm_alen; +static int srm_answer_is_set; +static uint8_t srm_answer[512]; +static int srm_ttl; +static time_t srm_expires; + +/* Mock replacement for connection_ap_hannshake_socks_resolved() */ +static void +socks_resolved_mock(entry_connection_t *conn, + int answer_type, + size_t answer_len, + const uint8_t *answer, + int ttl, + time_t expires) +{ + srm_ncalls++; + srm_conn = conn; + srm_atype = answer_type; + srm_alen = answer_len; + if (answer) { + memset(srm_answer, 0, sizeof(srm_answer)); + memcpy(srm_answer, answer, answer_len < 512 ? answer_len : 512); + srm_answer_is_set = 1; + } else { + srm_answer_is_set = 0; + } + srm_ttl = ttl; + srm_expires = expires; +} + +static int mum_ncalls; +static entry_connection_t *mum_conn; +static int mum_endreason; + +/* Mock replacement for connection_mark_unattached_ap_() */ +static void +mark_unattached_mock(entry_connection_t *conn, int endreason, + int line, const char *file) +{ + ++mum_ncalls; + mum_conn = conn; + mum_endreason = endreason; + (void) line; + (void) file; +} + +/* Tests for connection_edge_process_resolved_cell(). + + The point of ..process_resolved_cell() is to handle an incoming cell + on an entry connection, and call connection_mark_unattached_ap() and/or + connection_ap_handshake_socks_resolved(). + */ +static void +test_relaycell_resolved(void *arg) +{ + entry_connection_t *entryconn; + edge_connection_t *edgeconn; + cell_t cell; + relay_header_t rh; + int r; + or_options_t *options = get_options_mutable(); + +#define SET_CELL(s) do { \ + memset(&cell, 0, sizeof(cell)); \ + memset(&rh, 0, sizeof(rh)); \ + memcpy(cell.payload + RELAY_HEADER_SIZE, (s), sizeof((s))-1); \ + rh.length = sizeof((s))-1; \ + rh.command = RELAY_COMMAND_RESOLVED; \ + } while (0) +#define MOCK_RESET() do { \ + srm_ncalls = mum_ncalls = 0; \ + } while (0) +#define ASSERT_MARK_CALLED(reason) do { \ + tt_int_op(mum_ncalls, ==, 1); \ + tt_ptr_op(mum_conn, ==, entryconn); \ + tt_int_op(mum_endreason, ==, (reason)); \ + } while (0) +#define ASSERT_RESOLVED_CALLED(atype, answer, ttl, expires) do { \ + tt_int_op(srm_ncalls, ==, 1); \ + tt_ptr_op(srm_conn, ==, entryconn); \ + tt_int_op(srm_atype, ==, (atype)); \ + if (answer) { \ + tt_int_op(srm_alen, ==, sizeof(answer)-1); \ + tt_int_op(srm_alen, <, 512); \ + tt_int_op(srm_answer_is_set, ==, 1); \ + tt_mem_op(srm_answer, ==, answer, sizeof(answer)-1); \ + } else { \ + tt_int_op(srm_answer_is_set, ==, 0); \ + } \ + tt_int_op(srm_ttl, ==, ttl); \ + tt_int_op(srm_expires, ==, expires); \ + } while (0) + + (void)arg; + + MOCK(connection_mark_unattached_ap_, mark_unattached_mock); + MOCK(connection_ap_handshake_socks_resolved, socks_resolved_mock); + + options->ClientDNSRejectInternalAddresses = 0; + + SET_CELL(/* IPv4: 127.0.1.2, ttl 256 */ + "\x04\x04\x7f\x00\x01\x02\x00\x00\x01\x00" + /* IPv4: 18.0.0.1, ttl 512 */ + "\x04\x04\x12\x00\x00\x01\x00\x00\x02\x00" + /* IPv6: 2003::3, ttl 1024 */ + "\x06\x10" + "\x20\x02\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x03" + "\x00\x00\x04\x00"); + + entryconn = entry_connection_new(CONN_TYPE_AP, AF_INET); + edgeconn = ENTRY_TO_EDGE_CONN(entryconn); + + /* Try with connection in non-RESOLVE_WAIT state: cell gets ignored */ + MOCK_RESET(); + r = connection_edge_process_resolved_cell(edgeconn, &cell, &rh); + tt_int_op(r, ==, 0); + tt_int_op(srm_ncalls, ==, 0); + tt_int_op(mum_ncalls, ==, 0); + + /* Now put it in the right state. */ + ENTRY_TO_CONN(entryconn)->state = AP_CONN_STATE_RESOLVE_WAIT; + entryconn->socks_request->command = SOCKS_COMMAND_RESOLVE; + entryconn->ipv4_traffic_ok = 1; + entryconn->ipv6_traffic_ok = 1; + entryconn->prefer_ipv6_traffic = 0; + + /* We prefer ipv4, so we should get the first ipv4 answer */ + MOCK_RESET(); + r = connection_edge_process_resolved_cell(edgeconn, &cell, &rh); + tt_int_op(r, ==, 0); + ASSERT_MARK_CALLED(END_STREAM_REASON_DONE| + END_STREAM_REASON_FLAG_ALREADY_SOCKS_REPLIED); + ASSERT_RESOLVED_CALLED(RESOLVED_TYPE_IPV4, "\x7f\x00\x01\x02", 256, -1); + + /* But we may be discarding private answers. */ + MOCK_RESET(); + options->ClientDNSRejectInternalAddresses = 1; + r = connection_edge_process_resolved_cell(edgeconn, &cell, &rh); + tt_int_op(r, ==, 0); + ASSERT_MARK_CALLED(END_STREAM_REASON_DONE| + END_STREAM_REASON_FLAG_ALREADY_SOCKS_REPLIED); + ASSERT_RESOLVED_CALLED(RESOLVED_TYPE_IPV4, "\x12\x00\x00\x01", 512, -1); + + /* now prefer ipv6, and get the first ipv6 answer */ + entryconn->prefer_ipv6_traffic = 1; + MOCK_RESET(); + r = connection_edge_process_resolved_cell(edgeconn, &cell, &rh); + tt_int_op(r, ==, 0); + ASSERT_MARK_CALLED(END_STREAM_REASON_DONE| + END_STREAM_REASON_FLAG_ALREADY_SOCKS_REPLIED); + ASSERT_RESOLVED_CALLED(RESOLVED_TYPE_IPV6, + "\x20\x02\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x03", + 1024, -1); + + /* With a cell that only has IPv4, we report IPv4 even if we prefer IPv6 */ + MOCK_RESET(); + SET_CELL("\x04\x04\x12\x00\x00\x01\x00\x00\x02\x00"); + r = connection_edge_process_resolved_cell(edgeconn, &cell, &rh); + tt_int_op(r, ==, 0); + ASSERT_MARK_CALLED(END_STREAM_REASON_DONE| + END_STREAM_REASON_FLAG_ALREADY_SOCKS_REPLIED); + ASSERT_RESOLVED_CALLED(RESOLVED_TYPE_IPV4, "\x12\x00\x00\x01", 512, -1); + + /* But if we don't allow IPv4, we report nothing if the cell contains only + * ipv4 */ + MOCK_RESET(); + entryconn->ipv4_traffic_ok = 0; + r = connection_edge_process_resolved_cell(edgeconn, &cell, &rh); + tt_int_op(r, ==, 0); + ASSERT_MARK_CALLED(END_STREAM_REASON_DONE| + END_STREAM_REASON_FLAG_ALREADY_SOCKS_REPLIED); + ASSERT_RESOLVED_CALLED(RESOLVED_TYPE_ERROR, NULL, -1, -1); + + /* If we wanted hostnames, we report nothing, since we only had IPs. */ + MOCK_RESET(); + entryconn->ipv4_traffic_ok = 1; + entryconn->socks_request->command = SOCKS_COMMAND_RESOLVE_PTR; + r = connection_edge_process_resolved_cell(edgeconn, &cell, &rh); + tt_int_op(r, ==, 0); + ASSERT_MARK_CALLED(END_STREAM_REASON_DONE| + END_STREAM_REASON_FLAG_ALREADY_SOCKS_REPLIED); + ASSERT_RESOLVED_CALLED(RESOLVED_TYPE_ERROR, NULL, -1, -1); + + /* A hostname cell is fine though. */ + MOCK_RESET(); + SET_CELL("\x00\x0fwww.example.com\x00\x01\x00\x00"); + r = connection_edge_process_resolved_cell(edgeconn, &cell, &rh); + tt_int_op(r, ==, 0); + ASSERT_MARK_CALLED(END_STREAM_REASON_DONE| + END_STREAM_REASON_FLAG_ALREADY_SOCKS_REPLIED); + ASSERT_RESOLVED_CALLED(RESOLVED_TYPE_HOSTNAME, "www.example.com", 65536, -1); + + /* error on malformed cell */ + MOCK_RESET(); + entryconn->socks_request->command = SOCKS_COMMAND_RESOLVE; + SET_CELL("\x04\x04\x01\x02\x03\x04"); /* no ttl */ + r = connection_edge_process_resolved_cell(edgeconn, &cell, &rh); + tt_int_op(r, ==, 0); + ASSERT_MARK_CALLED(END_STREAM_REASON_TORPROTOCOL); + tt_int_op(srm_ncalls, ==, 0); + + /* error on all addresses private */ + MOCK_RESET(); + SET_CELL(/* IPv4: 127.0.1.2, ttl 256 */ + "\x04\x04\x7f\x00\x01\x02\x00\x00\x01\x00" + /* IPv4: 192.168.1.1, ttl 256 */ + "\x04\x04\xc0\xa8\x01\x01\x00\x00\x01\x00"); + r = connection_edge_process_resolved_cell(edgeconn, &cell, &rh); + tt_int_op(r, ==, 0); + ASSERT_MARK_CALLED(END_STREAM_REASON_TORPROTOCOL); + ASSERT_RESOLVED_CALLED(RESOLVED_TYPE_ERROR_TRANSIENT, NULL, 0, TIME_MAX); + + /* Legit error code */ + MOCK_RESET(); + SET_CELL("\xf0\x15" "quiet and meaningless" "\x00\x00\x0f\xff"); + r = connection_edge_process_resolved_cell(edgeconn, &cell, &rh); + tt_int_op(r, ==, 0); + ASSERT_MARK_CALLED(END_STREAM_REASON_DONE| + END_STREAM_REASON_FLAG_ALREADY_SOCKS_REPLIED); + ASSERT_RESOLVED_CALLED(RESOLVED_TYPE_ERROR_TRANSIENT, NULL, -1, -1); + + done: + UNMOCK(connection_mark_unattached_ap_); + UNMOCK(connection_ap_handshake_socks_resolved); +} + +struct testcase_t relaycell_tests[] = { + { "resolved", test_relaycell_resolved, TT_FORK, NULL, NULL }, + END_OF_TESTCASES +}; + diff --git a/src/test/test_replay.c b/src/test/test_replay.c index de841ad594..b48f582f5e 100644 --- a/src/test/test_replay.c +++ b/src/test/test_replay.c @@ -32,6 +32,40 @@ test_replaycache_alloc(void) } static void +test_replaycache_badalloc(void) +{ + replaycache_t *r = NULL; + + /* Negative horizon should fail */ + r = replaycache_new(-600, 300); + test_assert(r == NULL); + /* Negative interval should get adjusted to zero */ + r = replaycache_new(600, -300); + test_assert(r != NULL); + test_eq(r->scrub_interval, 0); + replaycache_free(r); + /* Negative horizon and negative interval should still fail */ + r = replaycache_new(-600, -300); + test_assert(r == NULL); + + done: + if (r) replaycache_free(r); + + return; +} + +static void +test_replaycache_free_null(void) +{ + replaycache_free(NULL); + /* Assert that we're here without horrible death */ + test_assert(1); + + done: + return; +} + +static void test_replaycache_miss(void) { replaycache_t *r = NULL; @@ -42,7 +76,13 @@ test_replaycache_miss(void) result = replaycache_add_and_test_internal(1200, r, test_buffer, - (int)strlen(test_buffer), NULL); + strlen(test_buffer), NULL); + test_eq(result, 0); + + /* poke the bad-parameter error case too */ + result = + replaycache_add_and_test_internal(1200, NULL, test_buffer, + strlen(test_buffer), NULL); test_eq(result, 0); done: @@ -62,12 +102,12 @@ test_replaycache_hit(void) result = replaycache_add_and_test_internal(1200, r, test_buffer, - (int)strlen(test_buffer), NULL); + strlen(test_buffer), NULL); test_eq(result, 0); result = replaycache_add_and_test_internal(1300, r, test_buffer, - (int)strlen(test_buffer), NULL); + strlen(test_buffer), NULL); test_eq(result, 1); done: @@ -87,17 +127,17 @@ test_replaycache_age(void) result = replaycache_add_and_test_internal(1200, r, test_buffer, - (int)strlen(test_buffer), NULL); + strlen(test_buffer), NULL); test_eq(result, 0); result = replaycache_add_and_test_internal(1300, r, test_buffer, - (int)strlen(test_buffer), NULL); + strlen(test_buffer), NULL); test_eq(result, 1); result = replaycache_add_and_test_internal(3000, r, test_buffer, - (int)strlen(test_buffer), NULL); + strlen(test_buffer), NULL); test_eq(result, 0); done: @@ -118,12 +158,12 @@ test_replaycache_elapsed(void) result = replaycache_add_and_test_internal(1200, r, test_buffer, - (int)strlen(test_buffer), NULL); + strlen(test_buffer), NULL); test_eq(result, 0); result = replaycache_add_and_test_internal(1300, r, test_buffer, - (int)strlen(test_buffer), &elapsed); + strlen(test_buffer), &elapsed); test_eq(result, 1); test_eq(elapsed, 100); @@ -144,18 +184,102 @@ test_replaycache_noexpire(void) result = replaycache_add_and_test_internal(1200, r, test_buffer, - (int)strlen(test_buffer), NULL); + strlen(test_buffer), NULL); test_eq(result, 0); result = replaycache_add_and_test_internal(1300, r, test_buffer, - (int)strlen(test_buffer), NULL); + strlen(test_buffer), NULL); test_eq(result, 1); result = replaycache_add_and_test_internal(3000, r, test_buffer, - (int)strlen(test_buffer), NULL); + strlen(test_buffer), NULL); + test_eq(result, 1); + + done: + if (r) replaycache_free(r); + + return; +} + +static void +test_replaycache_scrub(void) +{ + replaycache_t *r = NULL; + int result; + + r = replaycache_new(600, 300); + test_assert(r != NULL); + + /* Set up like in test_replaycache_hit() */ + result = + replaycache_add_and_test_internal(100, r, test_buffer, + strlen(test_buffer), NULL); + test_eq(result, 0); + + result = + replaycache_add_and_test_internal(200, r, test_buffer, + strlen(test_buffer), NULL); + test_eq(result, 1); + + /* + * Poke a few replaycache_scrub_if_needed_internal() error cases that + * can't happen through replaycache_add_and_test_internal() + */ + + /* Null cache */ + replaycache_scrub_if_needed_internal(300, NULL); + /* Assert we're still here */ + test_assert(1); + + /* Make sure we hit the aging-out case too */ + replaycache_scrub_if_needed_internal(1500, r); + /* Assert that we aged it */ + test_eq(digestmap_size(r->digests_seen), 0); + + done: + if (r) replaycache_free(r); + + return; +} + +static void +test_replaycache_future(void) +{ + replaycache_t *r = NULL; + int result; + time_t elapsed = 0; + + r = replaycache_new(600, 300); + test_assert(r != NULL); + + /* Set up like in test_replaycache_hit() */ + result = + replaycache_add_and_test_internal(100, r, test_buffer, + strlen(test_buffer), &elapsed); + test_eq(result, 0); + /* elapsed should still be 0, since it wasn't written */ + test_eq(elapsed, 0); + + result = + replaycache_add_and_test_internal(200, r, test_buffer, + strlen(test_buffer), &elapsed); + test_eq(result, 1); + /* elapsed should be the time since the last hit */ + test_eq(elapsed, 100); + + /* + * Now let's turn the clock back to get coverage on the cache entry from the + * future not-supposed-to-happen case. + */ + result = + replaycache_add_and_test_internal(150, r, test_buffer, + strlen(test_buffer), &elapsed); + /* We should still get a hit */ test_eq(result, 1); + /* ...but it shouldn't let us see a negative elapsed time */ + test_eq(elapsed, 0); done: if (r) replaycache_free(r); @@ -163,16 +287,62 @@ test_replaycache_noexpire(void) return; } +static void +test_replaycache_realtime(void) +{ + replaycache_t *r = NULL; + /* + * Negative so we fail if replaycache_add_test_and_elapsed() doesn't + * write to elapsed. + */ + time_t elapsed = -1; + int result; + + /* Test the realtime as well as *_internal() entry points */ + r = replaycache_new(600, 300); + test_assert(r != NULL); + + /* This should miss */ + result = + replaycache_add_and_test(r, test_buffer, strlen(test_buffer)); + test_eq(result, 0); + + /* This should hit */ + result = + replaycache_add_and_test(r, test_buffer, strlen(test_buffer)); + test_eq(result, 1); + + /* This should hit and return a small elapsed time */ + result = + replaycache_add_test_and_elapsed(r, test_buffer, + strlen(test_buffer), &elapsed); + test_eq(result, 1); + test_assert(elapsed >= 0); + test_assert(elapsed <= 5); + + /* Scrub it to exercise that entry point too */ + replaycache_scrub_if_needed(r); + + done: + if (r) replaycache_free(r); + return; +} + #define REPLAYCACHE_LEGACY(name) \ { #name, legacy_test_helper, 0, &legacy_setup, test_replaycache_ ## name } struct testcase_t replaycache_tests[] = { REPLAYCACHE_LEGACY(alloc), + REPLAYCACHE_LEGACY(badalloc), + REPLAYCACHE_LEGACY(free_null), REPLAYCACHE_LEGACY(miss), REPLAYCACHE_LEGACY(hit), REPLAYCACHE_LEGACY(age), REPLAYCACHE_LEGACY(elapsed), REPLAYCACHE_LEGACY(noexpire), + REPLAYCACHE_LEGACY(scrub), + REPLAYCACHE_LEGACY(future), + REPLAYCACHE_LEGACY(realtime), END_OF_TESTCASES }; diff --git a/src/test/test_routerkeys.c b/src/test/test_routerkeys.c new file mode 100644 index 0000000000..182e0f6f87 --- /dev/null +++ b/src/test/test_routerkeys.c @@ -0,0 +1,85 @@ +/* Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2013, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "orconfig.h" +#define ROUTER_PRIVATE +#include "or.h" +#include "config.h" +#include "router.h" +#include "util.h" +#include "crypto.h" + +#include "test.h" + +static void +test_routerkeys_write_fingerprint(void *arg) +{ + crypto_pk_t *key = pk_generate(2); + or_options_t *options = get_options_mutable(); + const char *ddir = get_fname("write_fingerprint"); + char *cp = NULL, *cp2 = NULL; + char fp[FINGERPRINT_LEN+1]; + + (void)arg; + + tt_assert(key); + + options->ORPort_set = 1; /* So that we can get the server ID key */ + tor_free(options->DataDirectory); + options->DataDirectory = tor_strdup(ddir); + options->Nickname = tor_strdup("haflinger"); + set_server_identity_key(key); + set_client_identity_key(crypto_pk_dup_key(key)); + + tt_int_op(0, ==, check_private_dir(ddir, CPD_CREATE, NULL)); + tt_int_op(crypto_pk_cmp_keys(get_server_identity_key(),key),==,0); + + /* Write fingerprint file */ + tt_int_op(0, ==, router_write_fingerprint(0)); + cp = read_file_to_str(get_fname("write_fingerprint/fingerprint"), + 0, NULL); + crypto_pk_get_fingerprint(key, fp, 0); + tor_asprintf(&cp2, "haflinger %s\n", fp); + tt_str_op(cp, ==, cp2); + tor_free(cp); + tor_free(cp2); + + /* Write hashed-fingerprint file */ + tt_int_op(0, ==, router_write_fingerprint(1)); + cp = read_file_to_str(get_fname("write_fingerprint/hashed-fingerprint"), + 0, NULL); + crypto_pk_get_hashed_fingerprint(key, fp); + tor_asprintf(&cp2, "haflinger %s\n", fp); + tt_str_op(cp, ==, cp2); + tor_free(cp); + tor_free(cp2); + + /* Replace outdated file */ + write_str_to_file(get_fname("write_fingerprint/hashed-fingerprint"), + "junk goes here", 0); + tt_int_op(0, ==, router_write_fingerprint(1)); + cp = read_file_to_str(get_fname("write_fingerprint/hashed-fingerprint"), + 0, NULL); + crypto_pk_get_hashed_fingerprint(key, fp); + tor_asprintf(&cp2, "haflinger %s\n", fp); + tt_str_op(cp, ==, cp2); + tor_free(cp); + tor_free(cp2); + + done: + crypto_pk_free(key); + set_client_identity_key(NULL); + tor_free(cp); + tor_free(cp2); +} + +#define TEST(name, flags) \ + { #name , test_routerkeys_ ## name, (flags), NULL, NULL } + +struct testcase_t routerkeys_tests[] = { + TEST(write_fingerprint, TT_FORK), + END_OF_TESTCASES +}; + diff --git a/src/test/test_socks.c b/src/test/test_socks.c new file mode 100644 index 0000000000..4ce61e068b --- /dev/null +++ b/src/test/test_socks.c @@ -0,0 +1,393 @@ +/* Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2013, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "or.h" +#include "buffers.h" +#include "config.h" +#include "test.h" + +typedef struct socks_test_data_t { + socks_request_t *req; + buf_t *buf; +} socks_test_data_t; + +static void * +socks_test_setup(const struct testcase_t *testcase) +{ + socks_test_data_t *data = tor_malloc(sizeof(socks_test_data_t)); + (void)testcase; + data->buf = buf_new_with_capacity(256); + data->req = socks_request_new(); + config_register_addressmaps(get_options()); + return data; +} +static int +socks_test_cleanup(const struct testcase_t *testcase, void *ptr) +{ + socks_test_data_t *data = ptr; + (void)testcase; + buf_free(data->buf); + socks_request_free(data->req); + tor_free(data); + return 1; +} + +const struct testcase_setup_t socks_setup = { + socks_test_setup, socks_test_cleanup +}; + +#define SOCKS_TEST_INIT() \ + socks_test_data_t *testdata = ptr; \ + buf_t *buf = testdata->buf; \ + socks_request_t *socks = testdata->req; +#define ADD_DATA(buf, s) \ + write_to_buf(s, sizeof(s)-1, buf) + +static void +socks_request_clear(socks_request_t *socks) +{ + tor_free(socks->username); + tor_free(socks->password); + memset(socks, 0, sizeof(socks_request_t)); +} + +/** Perform unsupported SOCKS 4 commands */ +static void +test_socks_4_unsupported_commands(void *ptr) +{ + SOCKS_TEST_INIT(); + + /* SOCKS 4 Send BIND [02] to IP address 2.2.2.2:4369 */ + ADD_DATA(buf, "\x04\x02\x11\x11\x02\x02\x02\x02\x00"); + test_assert(fetch_from_buf_socks(buf, socks, get_options()->TestSocks, + get_options()->SafeSocks) == -1); + test_eq(4, socks->socks_version); + test_eq(0, socks->replylen); /* XXX: shouldn't tor reply? */ + + done: + ; +} + +/** Perform supported SOCKS 4 commands */ +static void +test_socks_4_supported_commands(void *ptr) +{ + SOCKS_TEST_INIT(); + + test_eq(0, buf_datalen(buf)); + + /* SOCKS 4 Send CONNECT [01] to IP address 2.2.2.2:4370 */ + ADD_DATA(buf, "\x04\x01\x11\x12\x02\x02\x02\x03\x00"); + test_assert(fetch_from_buf_socks(buf, socks, get_options()->TestSocks, + get_options()->SafeSocks) == 1); + test_eq(4, socks->socks_version); + test_eq(0, socks->replylen); /* XXX: shouldn't tor reply? */ + test_eq(SOCKS_COMMAND_CONNECT, socks->command); + test_streq("2.2.2.3", socks->address); + test_eq(4370, socks->port); + test_assert(socks->got_auth == 0); + test_assert(! socks->username); + + test_eq(0, buf_datalen(buf)); + socks_request_clear(socks); + + /* SOCKS 4 Send CONNECT [01] to IP address 2.2.2.2:4369 with userid*/ + ADD_DATA(buf, "\x04\x01\x11\x12\x02\x02\x02\x04me\x00"); + test_assert(fetch_from_buf_socks(buf, socks, get_options()->TestSocks, + get_options()->SafeSocks) == 1); + test_eq(4, socks->socks_version); + test_eq(0, socks->replylen); /* XXX: shouldn't tor reply? */ + test_eq(SOCKS_COMMAND_CONNECT, socks->command); + test_streq("2.2.2.4", socks->address); + test_eq(4370, socks->port); + test_assert(socks->got_auth == 1); + test_assert(socks->username); + test_eq(2, socks->usernamelen); + test_memeq("me", socks->username, 2); + + test_eq(0, buf_datalen(buf)); + socks_request_clear(socks); + + /* SOCKS 4a Send RESOLVE [F0] request for torproject.org */ + ADD_DATA(buf, "\x04\xF0\x01\x01\x00\x00\x00\x02me\x00torproject.org\x00"); + test_assert(fetch_from_buf_socks(buf, socks, get_options()->TestSocks, + get_options()->SafeSocks) == 1); + test_eq(4, socks->socks_version); + test_eq(0, socks->replylen); /* XXX: shouldn't tor reply? */ + test_streq("torproject.org", socks->address); + + test_eq(0, buf_datalen(buf)); + + done: + ; +} + +/** Perform unsupported SOCKS 5 commands */ +static void +test_socks_5_unsupported_commands(void *ptr) +{ + SOCKS_TEST_INIT(); + + /* SOCKS 5 Send unsupported BIND [02] command */ + ADD_DATA(buf, "\x05\x02\x00\x01"); + + test_eq(fetch_from_buf_socks(buf, socks, get_options()->TestSocks, + get_options()->SafeSocks), 0); + test_eq(0, buf_datalen(buf)); + test_eq(5, socks->socks_version); + test_eq(2, socks->replylen); + test_eq(5, socks->reply[0]); + test_eq(0, socks->reply[1]); + ADD_DATA(buf, "\x05\x02\x00\x01\x02\x02\x02\x01\x01\x01"); + test_eq(fetch_from_buf_socks(buf, socks, get_options()->TestSocks, + get_options()->SafeSocks), -1); + /* XXX: shouldn't tor reply 'command not supported' [07]? */ + + buf_clear(buf); + socks_request_clear(socks); + + /* SOCKS 5 Send unsupported UDP_ASSOCIATE [03] command */ + ADD_DATA(buf, "\x05\x03\x00\x01\x02"); + test_eq(fetch_from_buf_socks(buf, socks, get_options()->TestSocks, + get_options()->SafeSocks), 0); + test_eq(5, socks->socks_version); + test_eq(2, socks->replylen); + test_eq(5, socks->reply[0]); + test_eq(2, socks->reply[1]); + ADD_DATA(buf, "\x05\x03\x00\x01\x02\x02\x02\x01\x01\x01"); + test_eq(fetch_from_buf_socks(buf, socks, get_options()->TestSocks, + get_options()->SafeSocks), -1); + /* XXX: shouldn't tor reply 'command not supported' [07]? */ + + done: + ; +} + +/** Perform supported SOCKS 5 commands */ +static void +test_socks_5_supported_commands(void *ptr) +{ + SOCKS_TEST_INIT(); + + /* SOCKS 5 Send CONNECT [01] to IP address 2.2.2.2:4369 */ + ADD_DATA(buf, "\x05\x01\x00"); + test_eq(fetch_from_buf_socks(buf, socks, get_options()->TestSocks, + get_options()->SafeSocks), 0); + test_eq(5, socks->socks_version); + test_eq(2, socks->replylen); + test_eq(5, socks->reply[0]); + test_eq(0, socks->reply[1]); + + ADD_DATA(buf, "\x05\x01\x00\x01\x02\x02\x02\x02\x11\x11"); + test_eq(fetch_from_buf_socks(buf, socks, get_options()->TestSocks, + get_options()->SafeSocks), 1); + test_streq("2.2.2.2", socks->address); + test_eq(4369, socks->port); + + test_eq(0, buf_datalen(buf)); + socks_request_clear(socks); + + /* SOCKS 5 Send CONNECT [01] to FQDN torproject.org:4369 */ + ADD_DATA(buf, "\x05\x01\x00"); + ADD_DATA(buf, "\x05\x01\x00\x03\x0Etorproject.org\x11\x11"); + test_eq(fetch_from_buf_socks(buf, socks, get_options()->TestSocks, + get_options()->SafeSocks), 1); + + test_eq(5, socks->socks_version); + test_eq(2, socks->replylen); + test_eq(5, socks->reply[0]); + test_eq(0, socks->reply[1]); + test_streq("torproject.org", socks->address); + test_eq(4369, socks->port); + + test_eq(0, buf_datalen(buf)); + socks_request_clear(socks); + + /* SOCKS 5 Send RESOLVE [F0] request for torproject.org:4369 */ + ADD_DATA(buf, "\x05\x01\x00"); + ADD_DATA(buf, "\x05\xF0\x00\x03\x0Etorproject.org\x01\x02"); + test_assert(fetch_from_buf_socks(buf, socks, get_options()->TestSocks, + get_options()->SafeSocks) == 1); + test_eq(5, socks->socks_version); + test_eq(2, socks->replylen); + test_eq(5, socks->reply[0]); + test_eq(0, socks->reply[1]); + test_streq("torproject.org", socks->address); + + test_eq(0, buf_datalen(buf)); + socks_request_clear(socks); + + /* SOCKS 5 Send RESOLVE_PTR [F1] for IP address 2.2.2.5 */ + ADD_DATA(buf, "\x05\x01\x00"); + ADD_DATA(buf, "\x05\xF1\x00\x01\x02\x02\x02\x05\x01\x03"); + test_assert(fetch_from_buf_socks(buf, socks, get_options()->TestSocks, + get_options()->SafeSocks) == 1); + test_eq(5, socks->socks_version); + test_eq(2, socks->replylen); + test_eq(5, socks->reply[0]); + test_eq(0, socks->reply[1]); + test_streq("2.2.2.5", socks->address); + + test_eq(0, buf_datalen(buf)); + + done: + ; +} + +/** Perform SOCKS 5 authentication */ +static void +test_socks_5_no_authenticate(void *ptr) +{ + SOCKS_TEST_INIT(); + + /*SOCKS 5 No Authentication */ + ADD_DATA(buf,"\x05\x01\x00"); + test_assert(!fetch_from_buf_socks(buf, socks, + get_options()->TestSocks, + get_options()->SafeSocks)); + test_eq(2, socks->replylen); + test_eq(5, socks->reply[0]); + test_eq(SOCKS_NO_AUTH, socks->reply[1]); + + test_eq(0, buf_datalen(buf)); + + /*SOCKS 5 Send username/password anyway - pretend to be broken */ + ADD_DATA(buf,"\x01\x02\x01\x01\x02\x01\x01"); + test_assert(!fetch_from_buf_socks(buf, socks, + get_options()->TestSocks, + get_options()->SafeSocks)); + test_eq(5, socks->socks_version); + test_eq(2, socks->replylen); + test_eq(1, socks->reply[0]); + test_eq(0, socks->reply[1]); + + test_eq(2, socks->usernamelen); + test_eq(2, socks->passwordlen); + + test_memeq("\x01\x01", socks->username, 2); + test_memeq("\x01\x01", socks->password, 2); + + done: + ; +} + +/** Perform SOCKS 5 authentication */ +static void +test_socks_5_authenticate(void *ptr) +{ + SOCKS_TEST_INIT(); + + /* SOCKS 5 Negotiate username/password authentication */ + ADD_DATA(buf, "\x05\x01\x02"); + + test_assert(!fetch_from_buf_socks(buf, socks, + get_options()->TestSocks, + get_options()->SafeSocks)); + test_eq(2, socks->replylen); + test_eq(5, socks->reply[0]); + test_eq(SOCKS_USER_PASS, socks->reply[1]); + test_eq(5, socks->socks_version); + + test_eq(0, buf_datalen(buf)); + + /* SOCKS 5 Send username/password */ + ADD_DATA(buf, "\x01\x02me\x08mypasswd"); + test_assert(!fetch_from_buf_socks(buf, socks, + get_options()->TestSocks, + get_options()->SafeSocks)); + test_eq(5, socks->socks_version); + test_eq(2, socks->replylen); + test_eq(1, socks->reply[0]); + test_eq(0, socks->reply[1]); + + test_eq(2, socks->usernamelen); + test_eq(8, socks->passwordlen); + + test_memeq("me", socks->username, 2); + test_memeq("mypasswd", socks->password, 8); + + done: + ; +} + +/** Perform SOCKS 5 authentication and send data all in one go */ +static void +test_socks_5_authenticate_with_data(void *ptr) +{ + SOCKS_TEST_INIT(); + + /* SOCKS 5 Negotiate username/password authentication */ + ADD_DATA(buf, "\x05\x01\x02"); + + test_assert(!fetch_from_buf_socks(buf, socks, + get_options()->TestSocks, + get_options()->SafeSocks)); + test_eq(2, socks->replylen); + test_eq(5, socks->reply[0]); + test_eq(SOCKS_USER_PASS, socks->reply[1]); + test_eq(5, socks->socks_version); + + test_eq(0, buf_datalen(buf)); + + /* SOCKS 5 Send username/password */ + /* SOCKS 5 Send CONNECT [01] to IP address 2.2.2.2:4369 */ + ADD_DATA(buf, "\x01\x02me\x03you\x05\x01\x00\x01\x02\x02\x02\x02\x11\x11"); + test_assert(fetch_from_buf_socks(buf, socks, + get_options()->TestSocks, + get_options()->SafeSocks) == 1); + test_eq(5, socks->socks_version); + test_eq(2, socks->replylen); + test_eq(1, socks->reply[0]); + test_eq(0, socks->reply[1]); + + test_streq("2.2.2.2", socks->address); + test_eq(4369, socks->port); + + test_eq(2, socks->usernamelen); + test_eq(3, socks->passwordlen); + test_memeq("me", socks->username, 2); + test_memeq("you", socks->password, 3); + + done: + ; +} + +/** Perform SOCKS 5 authentication before method negotiated */ +static void +test_socks_5_auth_before_negotiation(void *ptr) +{ + SOCKS_TEST_INIT(); + + /* SOCKS 5 Send username/password */ + ADD_DATA(buf, "\x01\x02me\x02me"); + test_assert(fetch_from_buf_socks(buf, socks, + get_options()->TestSocks, + get_options()->SafeSocks) == -1); + test_eq(0, socks->socks_version); + test_eq(0, socks->replylen); + test_eq(0, socks->reply[0]); + test_eq(0, socks->reply[1]); + + done: + ; +} + +#define SOCKSENT(name) \ + { #name, test_socks_##name, TT_FORK, &socks_setup, NULL } + +struct testcase_t socks_tests[] = { + SOCKSENT(4_unsupported_commands), + SOCKSENT(4_supported_commands), + + SOCKSENT(5_unsupported_commands), + SOCKSENT(5_supported_commands), + SOCKSENT(5_no_authenticate), + SOCKSENT(5_auth_before_negotiation), + SOCKSENT(5_authenticate), + SOCKSENT(5_authenticate_with_data), + + END_OF_TESTCASES +}; + diff --git a/src/test/test_status.c b/src/test/test_status.c new file mode 100644 index 0000000000..46dd473132 --- /dev/null +++ b/src/test/test_status.c @@ -0,0 +1,1114 @@ +#define STATUS_PRIVATE +#define HIBERNATE_PRIVATE +#define LOG_PRIVATE +#define REPHIST_PRIVATE + +#include <float.h> +#include <math.h> + +#include "or.h" +#include "torlog.h" +#include "tor_queue.h" +#include "status.h" +#include "circuitlist.h" +#include "config.h" +#include "hibernate.h" +#include "rephist.h" +#include "relay.h" +#include "router.h" +#include "main.h" +#include "nodelist.h" +#include "statefile.h" +#include "test.h" + +#define NS_MODULE status + +#define NS_SUBMODULE count_circuits + +/* + * Test that count_circuits() is correctly counting the number of + * global circuits. + */ + +struct global_circuitlist_s mock_global_circuitlist = + TOR_LIST_HEAD_INITIALIZER(global_circuitlist); + +NS_DECL(struct global_circuitlist_s *, circuit_get_global_list, (void)); + +static void +NS(test_main)(void *arg) +{ + /* Choose origin_circuit_t wlog. */ + origin_circuit_t *mock_circuit1, *mock_circuit2; + circuit_t *circ, *tmp; + int expected_circuits = 2, actual_circuits; + + (void)arg; + + mock_circuit1 = tor_malloc_zero(sizeof(origin_circuit_t)); + mock_circuit2 = tor_malloc_zero(sizeof(origin_circuit_t)); + TOR_LIST_INSERT_HEAD( + &mock_global_circuitlist, TO_CIRCUIT(mock_circuit1), head); + TOR_LIST_INSERT_HEAD( + &mock_global_circuitlist, TO_CIRCUIT(mock_circuit2), head); + + NS_MOCK(circuit_get_global_list); + + actual_circuits = count_circuits(); + + tt_assert(expected_circuits == actual_circuits); + + done: + TOR_LIST_FOREACH_SAFE( + circ, NS(circuit_get_global_list)(), head, tmp); + tor_free(circ); + NS_UNMOCK(circuit_get_global_list); +} + +static struct global_circuitlist_s * +NS(circuit_get_global_list)(void) +{ + return &mock_global_circuitlist; +} + +#undef NS_SUBMODULE +#define NS_SUBMODULE secs_to_uptime + +/* + * Test that secs_to_uptime() is converting the number of seconds that + * Tor is up for into the appropriate string form containing hours and minutes. + */ + +static void +NS(test_main)(void *arg) +{ + const char *expected; + char *actual; + (void)arg; + + expected = "0:00 hours"; + actual = secs_to_uptime(0); + tt_str_op(actual, ==, expected); + tor_free(actual); + + expected = "0:00 hours"; + actual = secs_to_uptime(1); + tt_str_op(actual, ==, expected); + tor_free(actual); + + expected = "0:01 hours"; + actual = secs_to_uptime(60); + tt_str_op(actual, ==, expected); + tor_free(actual); + + expected = "0:59 hours"; + actual = secs_to_uptime(60 * 59); + tt_str_op(actual, ==, expected); + tor_free(actual); + + expected = "1:00 hours"; + actual = secs_to_uptime(60 * 60); + tt_str_op(actual, ==, expected); + tor_free(actual); + + expected = "23:59 hours"; + actual = secs_to_uptime(60 * 60 * 23 + 60 * 59); + tt_str_op(actual, ==, expected); + tor_free(actual); + + expected = "1 day 0:00 hours"; + actual = secs_to_uptime(60 * 60 * 23 + 60 * 60); + tt_str_op(actual, ==, expected); + tor_free(actual); + + expected = "1 day 0:00 hours"; + actual = secs_to_uptime(86400 + 1); + tt_str_op(actual, ==, expected); + tor_free(actual); + + expected = "1 day 0:01 hours"; + actual = secs_to_uptime(86400 + 60); + tt_str_op(actual, ==, expected); + tor_free(actual); + + expected = "10 days 0:00 hours"; + actual = secs_to_uptime(86400 * 10); + tt_str_op(actual, ==, expected); + tor_free(actual); + + expected = "10 days 0:00 hours"; + actual = secs_to_uptime(864000 + 1); + tt_str_op(actual, ==, expected); + tor_free(actual); + + expected = "10 days 0:01 hours"; + actual = secs_to_uptime(864000 + 60); + tt_str_op(actual, ==, expected); + tor_free(actual); + + done: + if (actual != NULL) + tor_free(actual); +} + +#undef NS_SUBMODULE +#define NS_SUBMODULE bytes_to_usage + +/* + * Test that bytes_to_usage() is correctly converting the number of bytes that + * Tor has read/written into the appropriate string form containing kilobytes, + * megabytes, or gigabytes. + */ + +static void +NS(test_main)(void *arg) +{ + const char *expected; + char *actual; + (void)arg; + + expected = "0 kB"; + actual = bytes_to_usage(0); + tt_str_op(actual, ==, expected); + tor_free(actual); + + expected = "0 kB"; + actual = bytes_to_usage(1); + tt_str_op(actual, ==, expected); + tor_free(actual); + + expected = "1 kB"; + actual = bytes_to_usage(1024); + tt_str_op(actual, ==, expected); + tor_free(actual); + + expected = "1023 kB"; + actual = bytes_to_usage((1 << 20) - 1); + tt_str_op(actual, ==, expected); + tor_free(actual); + + expected = "1.00 MB"; + actual = bytes_to_usage((1 << 20)); + tt_str_op(actual, ==, expected); + tor_free(actual); + + expected = "1.00 MB"; + actual = bytes_to_usage((1 << 20) + 5242); + tt_str_op(actual, ==, expected); + tor_free(actual); + + expected = "1.01 MB"; + actual = bytes_to_usage((1 << 20) + 5243); + tt_str_op(actual, ==, expected); + tor_free(actual); + + expected = "1024.00 MB"; + actual = bytes_to_usage((1 << 30) - 1); + tt_str_op(actual, ==, expected); + tor_free(actual); + + expected = "1.00 GB"; + actual = bytes_to_usage((1 << 30)); + tt_str_op(actual, ==, expected); + tor_free(actual); + + expected = "1.00 GB"; + actual = bytes_to_usage((1 << 30) + 5368709); + tt_str_op(actual, ==, expected); + tor_free(actual); + + expected = "1.01 GB"; + actual = bytes_to_usage((1 << 30) + 5368710); + tt_str_op(actual, ==, expected); + tor_free(actual); + + expected = "10.00 GB"; + actual = bytes_to_usage((U64_LITERAL(1) << 30) * 10L); + tt_str_op(actual, ==, expected); + tor_free(actual); + + done: + if (actual != NULL) + tor_free(actual); +} + +#undef NS_SUBMODULE +#define NS_SUBMODULE ASPECT(log_heartbeat, fails) + +/* + * Tests that log_heartbeat() fails when in the public server mode, + * not hibernating, and we couldn't get the current routerinfo. + */ + +NS_DECL(double, tls_get_write_overhead_ratio, (void)); +NS_DECL(int, we_are_hibernating, (void)); +NS_DECL(const or_options_t *, get_options, (void)); +NS_DECL(int, public_server_mode, (const or_options_t *options)); +NS_DECL(const routerinfo_t *, router_get_my_routerinfo, (void)); + +static void +NS(test_main)(void *arg) +{ + int expected, actual; + (void)arg; + + NS_MOCK(tls_get_write_overhead_ratio); + NS_MOCK(we_are_hibernating); + NS_MOCK(get_options); + NS_MOCK(public_server_mode); + NS_MOCK(router_get_my_routerinfo); + + expected = -1; + actual = log_heartbeat(0); + + tt_int_op(actual, ==, expected); + + done: + NS_UNMOCK(tls_get_write_overhead_ratio); + NS_UNMOCK(we_are_hibernating); + NS_UNMOCK(get_options); + NS_UNMOCK(public_server_mode); + NS_UNMOCK(router_get_my_routerinfo); +} + +static double +NS(tls_get_write_overhead_ratio)(void) +{ + return 2.0; +} + +static int +NS(we_are_hibernating)(void) +{ + return 0; +} + +static const or_options_t * +NS(get_options)(void) +{ + return NULL; +} + +static int +NS(public_server_mode)(const or_options_t *options) +{ + (void)options; + + return 1; +} + +static const routerinfo_t * +NS(router_get_my_routerinfo)(void) +{ + return NULL; +} + +#undef NS_SUBMODULE +#define NS_SUBMODULE ASPECT(log_heartbeat, not_in_consensus) + +/* + * Tests that log_heartbeat() logs appropriately if we are not in the cached + * consensus. + */ + +NS_DECL(double, tls_get_write_overhead_ratio, (void)); +NS_DECL(int, we_are_hibernating, (void)); +NS_DECL(const or_options_t *, get_options, (void)); +NS_DECL(int, public_server_mode, (const or_options_t *options)); +NS_DECL(const routerinfo_t *, router_get_my_routerinfo, (void)); +NS_DECL(const node_t *, node_get_by_id, (const char *identity_digest)); +NS_DECL(void, logv, (int severity, log_domain_mask_t domain, + const char *funcname, const char *suffix, const char *format, va_list ap)); +NS_DECL(int, server_mode, (const or_options_t *options)); + +static routerinfo_t *mock_routerinfo; +extern int onion_handshakes_requested[MAX_ONION_HANDSHAKE_TYPE+1]; +extern int onion_handshakes_assigned[MAX_ONION_HANDSHAKE_TYPE+1]; + +static void +NS(test_main)(void *arg) +{ + int expected, actual; + (void)arg; + + NS_MOCK(tls_get_write_overhead_ratio); + NS_MOCK(we_are_hibernating); + NS_MOCK(get_options); + NS_MOCK(public_server_mode); + NS_MOCK(router_get_my_routerinfo); + NS_MOCK(node_get_by_id); + NS_MOCK(logv); + NS_MOCK(server_mode); + + log_global_min_severity_ = LOG_DEBUG; + onion_handshakes_requested[ONION_HANDSHAKE_TYPE_TAP] = 1; + onion_handshakes_assigned[ONION_HANDSHAKE_TYPE_TAP] = 1; + onion_handshakes_requested[ONION_HANDSHAKE_TYPE_NTOR] = 1; + onion_handshakes_assigned[ONION_HANDSHAKE_TYPE_NTOR] = 1; + + expected = 0; + actual = log_heartbeat(0); + + tt_int_op(actual, ==, expected); + tt_int_op(CALLED(logv), ==, 3); + + done: + NS_UNMOCK(tls_get_write_overhead_ratio); + NS_UNMOCK(we_are_hibernating); + NS_UNMOCK(get_options); + NS_UNMOCK(public_server_mode); + NS_UNMOCK(router_get_my_routerinfo); + NS_UNMOCK(node_get_by_id); + NS_UNMOCK(logv); + NS_UNMOCK(server_mode); + tor_free(mock_routerinfo); +} + +static double +NS(tls_get_write_overhead_ratio)(void) +{ + return 1.0; +} + +static int +NS(we_are_hibernating)(void) +{ + return 0; +} + +static const or_options_t * +NS(get_options)(void) +{ + return NULL; +} + +static int +NS(public_server_mode)(const or_options_t *options) +{ + (void)options; + + return 1; +} + +static const routerinfo_t * +NS(router_get_my_routerinfo)(void) +{ + mock_routerinfo = tor_malloc(sizeof(routerinfo_t)); + + return mock_routerinfo; +} + +static const node_t * +NS(node_get_by_id)(const char *identity_digest) +{ + (void)identity_digest; + + return NULL; +} + +static void +NS(logv)(int severity, log_domain_mask_t domain, + const char *funcname, const char *suffix, const char *format, va_list ap) +{ + switch (CALLED(logv)) + { + case 0: + tt_int_op(severity, ==, LOG_NOTICE); + tt_int_op(domain, ==, LD_HEARTBEAT); + tt_ptr_op(strstr(funcname, "log_heartbeat"), !=, NULL); + tt_ptr_op(suffix, ==, NULL); + tt_str_op(format, ==, + "Heartbeat: It seems like we are not in the cached consensus."); + break; + case 1: + tt_int_op(severity, ==, LOG_NOTICE); + tt_int_op(domain, ==, LD_HEARTBEAT); + tt_ptr_op(strstr(funcname, "log_heartbeat"), !=, NULL); + tt_ptr_op(suffix, ==, NULL); + tt_str_op(format, ==, + "Heartbeat: Tor's uptime is %s, with %d circuits open. " + "I've sent %s and received %s.%s"); + tt_str_op(va_arg(ap, char *), ==, "0:00 hours"); /* uptime */ + tt_int_op(va_arg(ap, int), ==, 0); /* count_circuits() */ + tt_str_op(va_arg(ap, char *), ==, "0 kB"); /* bw_sent */ + tt_str_op(va_arg(ap, char *), ==, "0 kB"); /* bw_rcvd */ + tt_str_op(va_arg(ap, char *), ==, ""); /* hibernating */ + break; + case 2: + tt_int_op(severity, ==, LOG_NOTICE); + tt_int_op(domain, ==, LD_HEARTBEAT); + tt_ptr_op( + strstr(funcname, "rep_hist_log_circuit_handshake_stats"), !=, NULL); + tt_ptr_op(suffix, ==, NULL); + tt_str_op(format, ==, + "Circuit handshake stats since last time: %d/%d TAP, %d/%d NTor."); + tt_int_op(va_arg(ap, int), ==, 1); /* handshakes assigned (TAP) */ + tt_int_op(va_arg(ap, int), ==, 1); /* handshakes requested (TAP) */ + tt_int_op(va_arg(ap, int), ==, 1); /* handshakes assigned (NTOR) */ + tt_int_op(va_arg(ap, int), ==, 1); /* handshakes requested (NTOR) */ + break; + default: + tt_abort_msg("unexpected call to logv()"); // TODO: prettyprint args + break; + } + + done: + CALLED(logv)++; +} + +static int +NS(server_mode)(const or_options_t *options) +{ + (void)options; + + return 0; +} + +#undef NS_SUBMODULE +#define NS_SUBMODULE ASPECT(log_heartbeat, simple) + +/* + * Tests that log_heartbeat() correctly logs heartbeat information + * normally. + */ + +NS_DECL(double, tls_get_write_overhead_ratio, (void)); +NS_DECL(int, we_are_hibernating, (void)); +NS_DECL(const or_options_t *, get_options, (void)); +NS_DECL(int, public_server_mode, (const or_options_t *options)); +NS_DECL(long, get_uptime, (void)); +NS_DECL(uint64_t, get_bytes_read, (void)); +NS_DECL(uint64_t, get_bytes_written, (void)); +NS_DECL(void, logv, (int severity, log_domain_mask_t domain, + const char *funcname, const char *suffix, const char *format, va_list ap)); +NS_DECL(int, server_mode, (const or_options_t *options)); + +static void +NS(test_main)(void *arg) +{ + int expected, actual; + (void)arg; + + NS_MOCK(tls_get_write_overhead_ratio); + NS_MOCK(we_are_hibernating); + NS_MOCK(get_options); + NS_MOCK(public_server_mode); + NS_MOCK(get_uptime); + NS_MOCK(get_bytes_read); + NS_MOCK(get_bytes_written); + NS_MOCK(logv); + NS_MOCK(server_mode); + + log_global_min_severity_ = LOG_DEBUG; + + expected = 0; + actual = log_heartbeat(0); + + tt_int_op(actual, ==, expected); + + done: + NS_UNMOCK(tls_get_write_overhead_ratio); + NS_UNMOCK(we_are_hibernating); + NS_UNMOCK(get_options); + NS_UNMOCK(public_server_mode); + NS_UNMOCK(get_uptime); + NS_UNMOCK(get_bytes_read); + NS_UNMOCK(get_bytes_written); + NS_UNMOCK(logv); + NS_UNMOCK(server_mode); +} + +static double +NS(tls_get_write_overhead_ratio)(void) +{ + return 1.0; +} + +static int +NS(we_are_hibernating)(void) +{ + return 1; +} + +static const or_options_t * +NS(get_options)(void) +{ + return NULL; +} + +static int +NS(public_server_mode)(const or_options_t *options) +{ + (void)options; + + return 0; +} + +static long +NS(get_uptime)(void) +{ + return 0; +} + +static uint64_t +NS(get_bytes_read)(void) +{ + return 0; +} + +static uint64_t +NS(get_bytes_written)(void) +{ + return 0; +} + +static void +NS(logv)(int severity, log_domain_mask_t domain, const char *funcname, + const char *suffix, const char *format, va_list ap) +{ + tt_int_op(severity, ==, LOG_NOTICE); + tt_int_op(domain, ==, LD_HEARTBEAT); + tt_ptr_op(strstr(funcname, "log_heartbeat"), !=, NULL); + tt_ptr_op(suffix, ==, NULL); + tt_str_op(format, ==, + "Heartbeat: Tor's uptime is %s, with %d circuits open. " + "I've sent %s and received %s.%s"); + tt_str_op(va_arg(ap, char *), ==, "0:00 hours"); /* uptime */ + tt_int_op(va_arg(ap, int), ==, 0); /* count_circuits() */ + tt_str_op(va_arg(ap, char *), ==, "0 kB"); /* bw_sent */ + tt_str_op(va_arg(ap, char *), ==, "0 kB"); /* bw_rcvd */ + tt_str_op(va_arg(ap, char *), ==, " We are currently hibernating."); + + done: + ; +} + +static int +NS(server_mode)(const or_options_t *options) +{ + (void)options; + + return 0; +} + +#undef NS_SUBMODULE +#define NS_SUBMODULE ASPECT(log_heartbeat, calls_log_accounting) + +/* + * Tests that log_heartbeat() correctly logs heartbeat information + * and accounting information when configured. + */ + +NS_DECL(double, tls_get_write_overhead_ratio, (void)); +NS_DECL(int, we_are_hibernating, (void)); +NS_DECL(const or_options_t *, get_options, (void)); +NS_DECL(int, public_server_mode, (const or_options_t *options)); +NS_DECL(long, get_uptime, (void)); +NS_DECL(uint64_t, get_bytes_read, (void)); +NS_DECL(uint64_t, get_bytes_written, (void)); +NS_DECL(void, logv, (int severity, log_domain_mask_t domain, + const char *funcname, const char *suffix, const char *format, va_list ap)); +NS_DECL(int, server_mode, (const or_options_t *options)); +NS_DECL(or_state_t *, get_or_state, (void)); +NS_DECL(int, accounting_is_enabled, (const or_options_t *options)); +NS_DECL(time_t, accounting_get_end_time, (void)); + +static or_state_t * NS(mock_state) = NULL; +static or_options_t * NS(mock_options) = NULL; + +static void +NS(test_main)(void *arg) +{ + int expected, actual; + (void)arg; + + NS_MOCK(tls_get_write_overhead_ratio); + NS_MOCK(we_are_hibernating); + NS_MOCK(get_options); + NS_MOCK(public_server_mode); + NS_MOCK(get_uptime); + NS_MOCK(get_bytes_read); + NS_MOCK(get_bytes_written); + NS_MOCK(logv); + NS_MOCK(server_mode); + NS_MOCK(get_or_state); + NS_MOCK(accounting_is_enabled); + NS_MOCK(accounting_get_end_time); + + log_global_min_severity_ = LOG_DEBUG; + + expected = 0; + actual = log_heartbeat(0); + + tt_int_op(actual, ==, expected); + tt_int_op(CALLED(logv), ==, 2); + + done: + NS_UNMOCK(tls_get_write_overhead_ratio); + NS_UNMOCK(we_are_hibernating); + NS_UNMOCK(get_options); + NS_UNMOCK(public_server_mode); + NS_UNMOCK(get_uptime); + NS_UNMOCK(get_bytes_read); + NS_UNMOCK(get_bytes_written); + NS_UNMOCK(logv); + NS_UNMOCK(server_mode); + NS_UNMOCK(accounting_is_enabled); + NS_UNMOCK(accounting_get_end_time); + tor_free_(NS(mock_state)); + tor_free_(NS(mock_options)); +} + +static double +NS(tls_get_write_overhead_ratio)(void) +{ + return 1.0; +} + +static int +NS(we_are_hibernating)(void) +{ + return 0; +} + +static const or_options_t * +NS(get_options)(void) +{ + NS(mock_options) = tor_malloc_zero(sizeof(or_options_t)); + NS(mock_options)->AccountingMax = 0; + + return NS(mock_options); +} + +static int +NS(public_server_mode)(const or_options_t *options) +{ + (void)options; + + return 0; +} + +static long +NS(get_uptime)(void) +{ + return 0; +} + +static uint64_t +NS(get_bytes_read)(void) +{ + return 0; +} + +static uint64_t +NS(get_bytes_written)(void) +{ + return 0; +} + +static void +NS(logv)(int severity, log_domain_mask_t domain, + const char *funcname, const char *suffix, const char *format, va_list ap) +{ + switch (CALLED(logv)) + { + case 0: + tt_int_op(severity, ==, LOG_NOTICE); + tt_int_op(domain, ==, LD_HEARTBEAT); + tt_ptr_op(strstr(funcname, "log_heartbeat"), !=, NULL); + tt_ptr_op(suffix, ==, NULL); + tt_str_op(format, ==, + "Heartbeat: Tor's uptime is %s, with %d circuits open. " + "I've sent %s and received %s.%s"); + tt_str_op(va_arg(ap, char *), ==, "0:00 hours"); /* uptime */ + tt_int_op(va_arg(ap, int), ==, 0); /* count_circuits() */ + tt_str_op(va_arg(ap, char *), ==, "0 kB"); /* bw_sent */ + tt_str_op(va_arg(ap, char *), ==, "0 kB"); /* bw_rcvd */ + tt_str_op(va_arg(ap, char *), ==, ""); /* hibernating */ + break; + case 1: + tt_int_op(severity, ==, LOG_NOTICE); + tt_int_op(domain, ==, LD_HEARTBEAT); + tt_ptr_op(strstr(funcname, "log_accounting"), !=, NULL); + tt_ptr_op(suffix, ==, NULL); + tt_str_op(format, ==, + "Heartbeat: Accounting enabled. Sent: %s / %s, Received: %s / %s. " + "The current accounting interval ends on %s, in %s."); + tt_str_op(va_arg(ap, char *), ==, "0 kB"); /* acc_sent */ + tt_str_op(va_arg(ap, char *), ==, "0 kB"); /* acc_max */ + tt_str_op(va_arg(ap, char *), ==, "0 kB"); /* acc_rcvd */ + tt_str_op(va_arg(ap, char *), ==, "0 kB"); /* acc_max */ + /* format_local_iso_time uses local tz, just check mins and secs. */ + tt_ptr_op(strstr(va_arg(ap, char *), ":01:00"), !=, NULL); /* end_buf */ + tt_str_op(va_arg(ap, char *), ==, "0:01 hours"); /* remaining */ + break; + default: + tt_abort_msg("unexpected call to logv()"); // TODO: prettyprint args + break; + } + + done: + CALLED(logv)++; +} + +static int +NS(server_mode)(const or_options_t *options) +{ + (void)options; + + return 1; +} + +static int +NS(accounting_is_enabled)(const or_options_t *options) +{ + (void)options; + + return 1; +} + +static time_t +NS(accounting_get_end_time)(void) +{ + return 60; +} + +static or_state_t * +NS(get_or_state)(void) +{ + NS(mock_state) = tor_malloc_zero(sizeof(or_state_t)); + NS(mock_state)->AccountingBytesReadInInterval = 0; + NS(mock_state)->AccountingBytesWrittenInInterval = 0; + + return NS(mock_state); +} + +#undef NS_SUBMODULE +#define NS_SUBMODULE ASPECT(log_heartbeat, packaged_cell_fullness) + +/* + * Tests that log_heartbeat() correctly logs packaged cell + * fullness information. + */ + +NS_DECL(double, tls_get_write_overhead_ratio, (void)); +NS_DECL(int, we_are_hibernating, (void)); +NS_DECL(const or_options_t *, get_options, (void)); +NS_DECL(int, public_server_mode, (const or_options_t *options)); +NS_DECL(long, get_uptime, (void)); +NS_DECL(uint64_t, get_bytes_read, (void)); +NS_DECL(uint64_t, get_bytes_written, (void)); +NS_DECL(void, logv, (int severity, log_domain_mask_t domain, + const char *funcname, const char *suffix, const char *format, va_list ap)); +NS_DECL(int, server_mode, (const or_options_t *options)); +NS_DECL(int, accounting_is_enabled, (const or_options_t *options)); + +static void +NS(test_main)(void *arg) +{ + int expected, actual; + (void)arg; + + NS_MOCK(tls_get_write_overhead_ratio); + NS_MOCK(we_are_hibernating); + NS_MOCK(get_options); + NS_MOCK(public_server_mode); + NS_MOCK(get_uptime); + NS_MOCK(get_bytes_read); + NS_MOCK(get_bytes_written); + NS_MOCK(logv); + NS_MOCK(server_mode); + NS_MOCK(accounting_is_enabled); + log_global_min_severity_ = LOG_DEBUG; + + stats_n_data_bytes_packaged = RELAY_PAYLOAD_SIZE; + stats_n_data_cells_packaged = 1; + expected = 0; + actual = log_heartbeat(0); + + tt_int_op(actual, ==, expected); + tt_int_op(CALLED(logv), ==, 2); + + done: + stats_n_data_bytes_packaged = 0; + stats_n_data_cells_packaged = 0; + NS_UNMOCK(tls_get_write_overhead_ratio); + NS_UNMOCK(we_are_hibernating); + NS_UNMOCK(get_options); + NS_UNMOCK(public_server_mode); + NS_UNMOCK(get_uptime); + NS_UNMOCK(get_bytes_read); + NS_UNMOCK(get_bytes_written); + NS_UNMOCK(logv); + NS_UNMOCK(server_mode); + NS_UNMOCK(accounting_is_enabled); +} + +static double +NS(tls_get_write_overhead_ratio)(void) +{ + return 1.0; +} + +static int +NS(we_are_hibernating)(void) +{ + return 0; +} + +static const or_options_t * +NS(get_options)(void) +{ + return NULL; +} + +static int +NS(public_server_mode)(const or_options_t *options) +{ + (void)options; + + return 0; +} + +static long +NS(get_uptime)(void) +{ + return 0; +} + +static uint64_t +NS(get_bytes_read)(void) +{ + return 0; +} + +static uint64_t +NS(get_bytes_written)(void) +{ + return 0; +} + +static void +NS(logv)(int severity, log_domain_mask_t domain, const char *funcname, + const char *suffix, const char *format, va_list ap) +{ + switch (CALLED(logv)) + { + case 0: + tt_int_op(severity, ==, LOG_NOTICE); + tt_int_op(domain, ==, LD_HEARTBEAT); + tt_ptr_op(strstr(funcname, "log_heartbeat"), !=, NULL); + tt_ptr_op(suffix, ==, NULL); + tt_str_op(format, ==, + "Heartbeat: Tor's uptime is %s, with %d circuits open. " + "I've sent %s and received %s.%s"); + tt_str_op(va_arg(ap, char *), ==, "0:00 hours"); /* uptime */ + tt_int_op(va_arg(ap, int), ==, 0); /* count_circuits() */ + tt_str_op(va_arg(ap, char *), ==, "0 kB"); /* bw_sent */ + tt_str_op(va_arg(ap, char *), ==, "0 kB"); /* bw_rcvd */ + tt_str_op(va_arg(ap, char *), ==, ""); /* hibernating */ + break; + case 1: + tt_int_op(severity, ==, LOG_NOTICE); + tt_int_op(domain, ==, LD_HEARTBEAT); + tt_ptr_op(strstr(funcname, "log_heartbeat"), !=, NULL); + tt_ptr_op(suffix, ==, NULL); + tt_str_op(format, ==, + "Average packaged cell fullness: %2.3f%%"); + tt_int_op(fabs(va_arg(ap, double) - 100.0) <= DBL_EPSILON, ==, 1); + break; + default: + tt_abort_msg("unexpected call to logv()"); // TODO: prettyprint args + break; + } + + done: + CALLED(logv)++; +} + +static int +NS(server_mode)(const or_options_t *options) +{ + (void)options; + + return 0; +} + +static int +NS(accounting_is_enabled)(const or_options_t *options) +{ + (void)options; + + return 0; +} + +#undef NS_SUBMODULE +#define NS_SUBMODULE ASPECT(log_heartbeat, tls_write_overhead) + +/* + * Tests that log_heartbeat() correctly logs the TLS write overhead information + * when the TLS write overhead ratio exceeds 1. + */ + +NS_DECL(double, tls_get_write_overhead_ratio, (void)); +NS_DECL(int, we_are_hibernating, (void)); +NS_DECL(const or_options_t *, get_options, (void)); +NS_DECL(int, public_server_mode, (const or_options_t *options)); +NS_DECL(long, get_uptime, (void)); +NS_DECL(uint64_t, get_bytes_read, (void)); +NS_DECL(uint64_t, get_bytes_written, (void)); +NS_DECL(void, logv, (int severity, log_domain_mask_t domain, + const char *funcname, const char *suffix, const char *format, va_list ap)); +NS_DECL(int, server_mode, (const or_options_t *options)); +NS_DECL(int, accounting_is_enabled, (const or_options_t *options)); + +static void +NS(test_main)(void *arg) +{ + int expected, actual; + (void)arg; + + NS_MOCK(tls_get_write_overhead_ratio); + NS_MOCK(we_are_hibernating); + NS_MOCK(get_options); + NS_MOCK(public_server_mode); + NS_MOCK(get_uptime); + NS_MOCK(get_bytes_read); + NS_MOCK(get_bytes_written); + NS_MOCK(logv); + NS_MOCK(server_mode); + NS_MOCK(accounting_is_enabled); + stats_n_data_cells_packaged = 0; + log_global_min_severity_ = LOG_DEBUG; + + expected = 0; + actual = log_heartbeat(0); + + tt_int_op(actual, ==, expected); + tt_int_op(CALLED(logv), ==, 2); + + done: + NS_UNMOCK(tls_get_write_overhead_ratio); + NS_UNMOCK(we_are_hibernating); + NS_UNMOCK(get_options); + NS_UNMOCK(public_server_mode); + NS_UNMOCK(get_uptime); + NS_UNMOCK(get_bytes_read); + NS_UNMOCK(get_bytes_written); + NS_UNMOCK(logv); + NS_UNMOCK(server_mode); + NS_UNMOCK(accounting_is_enabled); +} + +static double +NS(tls_get_write_overhead_ratio)(void) +{ + return 2.0; +} + +static int +NS(we_are_hibernating)(void) +{ + return 0; +} + +static const or_options_t * +NS(get_options)(void) +{ + return NULL; +} + +static int +NS(public_server_mode)(const or_options_t *options) +{ + (void)options; + + return 0; +} + +static long +NS(get_uptime)(void) +{ + return 0; +} + +static uint64_t +NS(get_bytes_read)(void) +{ + return 0; +} + +static uint64_t +NS(get_bytes_written)(void) +{ + return 0; +} + +static void +NS(logv)(int severity, log_domain_mask_t domain, + const char *funcname, const char *suffix, const char *format, va_list ap) +{ + switch (CALLED(logv)) + { + case 0: + tt_int_op(severity, ==, LOG_NOTICE); + tt_int_op(domain, ==, LD_HEARTBEAT); + tt_ptr_op(strstr(funcname, "log_heartbeat"), !=, NULL); + tt_ptr_op(suffix, ==, NULL); + tt_str_op(format, ==, + "Heartbeat: Tor's uptime is %s, with %d circuits open. " + "I've sent %s and received %s.%s"); + tt_str_op(va_arg(ap, char *), ==, "0:00 hours"); /* uptime */ + tt_int_op(va_arg(ap, int), ==, 0); /* count_circuits() */ + tt_str_op(va_arg(ap, char *), ==, "0 kB"); /* bw_sent */ + tt_str_op(va_arg(ap, char *), ==, "0 kB"); /* bw_rcvd */ + tt_str_op(va_arg(ap, char *), ==, ""); /* hibernating */ + break; + case 1: + tt_int_op(severity, ==, LOG_NOTICE); + tt_int_op(domain, ==, LD_HEARTBEAT); + tt_ptr_op(strstr(funcname, "log_heartbeat"), !=, NULL); + tt_ptr_op(suffix, ==, NULL); + tt_str_op(format, ==, "TLS write overhead: %.f%%"); + tt_int_op(fabs(va_arg(ap, double) - 100.0) <= DBL_EPSILON, ==, 1); + break; + default: + tt_abort_msg("unexpected call to logv()"); // TODO: prettyprint args + break; + } + + done: + CALLED(logv)++; +} + +static int +NS(server_mode)(const or_options_t *options) +{ + (void)options; + + return 0; +} + +static int +NS(accounting_is_enabled)(const or_options_t *options) +{ + (void)options; + + return 0; +} + +#undef NS_SUBMODULE + +struct testcase_t status_tests[] = { + TEST_CASE(count_circuits), + TEST_CASE(secs_to_uptime), + TEST_CASE(bytes_to_usage), + TEST_CASE_ASPECT(log_heartbeat, fails), + TEST_CASE_ASPECT(log_heartbeat, simple), + TEST_CASE_ASPECT(log_heartbeat, not_in_consensus), + TEST_CASE_ASPECT(log_heartbeat, calls_log_accounting), + TEST_CASE_ASPECT(log_heartbeat, packaged_cell_fullness), + TEST_CASE_ASPECT(log_heartbeat, tls_write_overhead), + END_OF_TESTCASES +}; + diff --git a/src/test/test_util.c b/src/test/test_util.c index 65d9d2f878..151ec69127 100644 --- a/src/test/test_util.c +++ b/src/test/test_util.c @@ -4,6 +4,7 @@ /* See LICENSE for licensing information */ #include "orconfig.h" +#define COMPAT_PRIVATE #define CONTROL_PRIVATE #define MEMPOOL_PRIVATE #define UTIL_PRIVATE @@ -11,8 +12,11 @@ #include "config.h" #include "control.h" #include "test.h" +#ifdef ENABLE_MEMPOOLS #include "mempool.h" +#endif /* ENABLE_MEMPOOLS */ #include "memarea.h" +#include "util_process.h" #ifdef _WIN32 #include <tchar.h> @@ -101,6 +105,107 @@ test_util_read_file_eof_zero_bytes(void *arg) test_util_read_until_eof_impl("tor_test_fifo_empty", 0, 10000); } +/* Test the basic expected behaviour for write_chunks_to_file. + * NOTE: This will need to be updated if we ever change the tempfile location + * or extension */ +static void +test_util_write_chunks_to_file(void *arg) +{ + char *fname = NULL; + char *tempname = NULL; + char *str = NULL; + int r; + struct stat st; + + /* These should be two different sizes to ensure the data is different + * between the data file and the temp file's 'known string' */ + int temp_str_len = 1024; + int data_str_len = 512; + char *data_str = tor_malloc(data_str_len); + char *temp_str = tor_malloc(temp_str_len); + + smartlist_t *chunks = smartlist_new(); + sized_chunk_t c = {data_str, data_str_len/2}; + sized_chunk_t c2 = {data_str + data_str_len/2, data_str_len/2}; + (void)arg; + + crypto_rand(temp_str, temp_str_len); + crypto_rand(data_str, data_str_len); + + // Ensure it can write multiple chunks + + smartlist_add(chunks, &c); + smartlist_add(chunks, &c2); + + /* + * Check if it writes using a tempfile + */ + fname = tor_strdup(get_fname("write_chunks_with_tempfile")); + tor_asprintf(&tempname, "%s.tmp", fname); + + // write a known string to a file where the tempfile will be + r = write_bytes_to_file(tempname, temp_str, temp_str_len, 1); + tt_int_op(r, ==, 0); + + // call write_chunks_to_file + r = write_chunks_to_file(fname, chunks, 1, 0); + tt_int_op(r, ==, 0); + + // assert the file has been written (expected size) + str = read_file_to_str(fname, RFTS_BIN, &st); + tt_assert(str != NULL); + tt_u64_op((uint64_t)st.st_size, ==, data_str_len); + test_mem_op(data_str, ==, str, data_str_len); + tor_free(str); + + // assert that the tempfile is removed (should not leave artifacts) + str = read_file_to_str(tempname, RFTS_BIN|RFTS_IGNORE_MISSING, &st); + tt_assert(str == NULL); + + // Remove old testfile for second test + r = unlink(fname); + tt_int_op(r, ==, 0); + tor_free(fname); + tor_free(tempname); + + /* + * Check if it skips using a tempfile with flags + */ + fname = tor_strdup(get_fname("write_chunks_with_no_tempfile")); + tor_asprintf(&tempname, "%s.tmp", fname); + + // write a known string to a file where the tempfile will be + r = write_bytes_to_file(tempname, temp_str, temp_str_len, 1); + tt_int_op(r, ==, 0); + + // call write_chunks_to_file with no_tempfile = true + r = write_chunks_to_file(fname, chunks, 1, 1); + tt_int_op(r, ==, 0); + + // assert the file has been written (expected size) + str = read_file_to_str(fname, RFTS_BIN, &st); + tt_assert(str != NULL); + tt_u64_op((uint64_t)st.st_size, ==, data_str_len); + test_mem_op(data_str, ==, str, data_str_len); + tor_free(str); + + // assert the tempfile still contains the known string + str = read_file_to_str(tempname, RFTS_BIN, &st); + tt_assert(str != NULL); + tt_u64_op((uint64_t)st.st_size, ==, temp_str_len); + test_mem_op(temp_str, ==, str, temp_str_len); + + done: + unlink(fname); + unlink(tempname); + smartlist_free(chunks); + tor_free(fname); + tor_free(tempname); + tor_free(str); + tor_free(data_str); + tor_free(temp_str); +} + static void test_util_time(void) { @@ -242,7 +347,7 @@ test_util_time(void) tv.tv_sec = (time_t)1326296338; tv.tv_usec = 3060; - format_iso_time(timestr, tv.tv_sec); + format_iso_time(timestr, (time_t)tv.tv_sec); test_streq("2012-01-11 15:38:58", timestr); /* The output of format_local_iso_time will vary by timezone, and setting our timezone for testing purposes would be a nontrivial flaky pain. @@ -250,7 +355,7 @@ test_util_time(void) format_local_iso_time(timestr, tv.tv_sec); test_streq("2012-01-11 10:38:58", timestr); */ - format_iso_time_nospace(timestr, tv.tv_sec); + format_iso_time_nospace(timestr, (time_t)tv.tv_sec); test_streq("2012-01-11T15:38:58", timestr); test_eq(strlen(timestr), ISO_TIME_LEN); format_iso_time_nospace_usec(timestr, &tv); @@ -796,6 +901,64 @@ test_util_expand_filename(void) } #endif +/** Test tor_escape_str_for_pt_args(). */ +static void +test_util_escape_string_socks(void) +{ + char *escaped_string = NULL; + + /** Simple backslash escape. */ + escaped_string = tor_escape_str_for_pt_args("This is a backslash: \\",";\\"); + test_assert(escaped_string); + test_streq(escaped_string, "This is a backslash: \\\\"); + tor_free(escaped_string); + + /** Simple semicolon escape. */ + escaped_string = tor_escape_str_for_pt_args("First rule:Do not use ;",";\\"); + test_assert(escaped_string); + test_streq(escaped_string, "First rule:Do not use \\;"); + tor_free(escaped_string); + + /** Empty string. */ + escaped_string = tor_escape_str_for_pt_args("", ";\\"); + test_assert(escaped_string); + test_streq(escaped_string, ""); + tor_free(escaped_string); + + /** Escape all characters. */ + escaped_string = tor_escape_str_for_pt_args(";\\;\\", ";\\"); + test_assert(escaped_string); + test_streq(escaped_string, "\\;\\\\\\;\\\\"); + tor_free(escaped_string); + + escaped_string = tor_escape_str_for_pt_args(";", ";\\"); + test_assert(escaped_string); + test_streq(escaped_string, "\\;"); + tor_free(escaped_string); + + done: + tor_free(escaped_string); +} + +static void +test_util_string_is_key_value(void *ptr) +{ + (void)ptr; + test_assert(string_is_key_value(LOG_WARN, "key=value")); + test_assert(string_is_key_value(LOG_WARN, "k=v")); + test_assert(string_is_key_value(LOG_WARN, "key=")); + test_assert(string_is_key_value(LOG_WARN, "x=")); + test_assert(string_is_key_value(LOG_WARN, "xx=")); + test_assert(!string_is_key_value(LOG_WARN, "=value")); + test_assert(!string_is_key_value(LOG_WARN, "=x")); + test_assert(!string_is_key_value(LOG_WARN, "=")); + + /* ??? */ + /* test_assert(!string_is_key_value(LOG_WARN, "===")); */ + done: + ; +} + /** Test basic string functionality. */ static void test_util_strmisc(void) @@ -867,6 +1030,8 @@ test_util_strmisc(void) test_eq(0L, tor_parse_long("10",-2,0,100,NULL,NULL)); test_eq(68284L, tor_parse_long("10abc",16,0,70000,NULL,NULL)); test_eq(68284L, tor_parse_long("10ABC",16,0,70000,NULL,NULL)); + test_eq(0, tor_parse_long("10ABC",-1,0,70000,&i,NULL)); + test_eq(i, 0); /* Test parse_ulong */ test_eq(0UL, tor_parse_ulong("",10,0,100,NULL,NULL)); @@ -878,6 +1043,8 @@ test_util_strmisc(void) test_eq(0UL, tor_parse_ulong("8",8,0,100,NULL,NULL)); test_eq(50UL, tor_parse_ulong("50",10,50,100,NULL,NULL)); test_eq(0UL, tor_parse_ulong("-50",10,-100,100,NULL,NULL)); + test_eq(0UL, tor_parse_ulong("50",-1,50,100,&i,NULL)); + test_eq(0, i); /* Test parse_uint64 */ test_assert(U64_LITERAL(10) == tor_parse_uint64("10 x",10,0,100, &i, &cp)); @@ -890,6 +1057,9 @@ test_util_strmisc(void) test_assert(U64_LITERAL(0) == tor_parse_uint64("12345678901",10,500,INT32_MAX, &i, &cp)); test_eq(0, i); + test_assert(U64_LITERAL(0) == + tor_parse_uint64("123",-1,0,INT32_MAX, &i, &cp)); + test_eq(0, i); { /* Test parse_double */ @@ -923,7 +1093,7 @@ test_util_strmisc(void) test_eq(i, 0); test_eq(0UL, tor_parse_ulong(TOOBIG, 10, 0, ULONG_MAX, &i, NULL)); test_eq(i, 0); - test_eq(U64_LITERAL(0), tor_parse_uint64(TOOBIG, 10, + tt_u64_op(U64_LITERAL(0), ==, tor_parse_uint64(TOOBIG, 10, 0, UINT64_MAX, &i, NULL)); test_eq(i, 0); } @@ -1022,19 +1192,19 @@ test_util_strmisc(void) } /* Test str-foo functions */ - cp = tor_strdup("abcdef"); - test_assert(tor_strisnonupper(cp)); - cp[3] = 'D'; - test_assert(!tor_strisnonupper(cp)); - tor_strupper(cp); - test_streq(cp, "ABCDEF"); - tor_strlower(cp); - test_streq(cp, "abcdef"); - test_assert(tor_strisnonupper(cp)); - test_assert(tor_strisprint(cp)); - cp[3] = 3; - test_assert(!tor_strisprint(cp)); - tor_free(cp); + cp_tmp = tor_strdup("abcdef"); + test_assert(tor_strisnonupper(cp_tmp)); + cp_tmp[3] = 'D'; + test_assert(!tor_strisnonupper(cp_tmp)); + tor_strupper(cp_tmp); + test_streq(cp_tmp, "ABCDEF"); + tor_strlower(cp_tmp); + test_streq(cp_tmp, "abcdef"); + test_assert(tor_strisnonupper(cp_tmp)); + test_assert(tor_strisprint(cp_tmp)); + cp_tmp[3] = 3; + test_assert(!tor_strisprint(cp_tmp)); + tor_free(cp_tmp); /* Test memmem and memstr */ { @@ -1045,6 +1215,10 @@ test_util_strmisc(void) test_assert(!tor_memmem(haystack, 4, "cde", 3)); haystack = "ababcad"; test_eq_ptr(tor_memmem(haystack, 7, "abc", 3), haystack + 2); + test_eq_ptr(tor_memmem(haystack, 7, "ad", 2), haystack + 5); + test_eq_ptr(tor_memmem(haystack, 7, "cad", 3), haystack + 4); + test_assert(!tor_memmem(haystack, 7, "dadad", 5)); + test_assert(!tor_memmem(haystack, 7, "abcdefghij", 10)); /* memstr */ test_eq_ptr(tor_memstr(haystack, 7, "abc"), haystack + 2); test_eq_ptr(tor_memstr(haystack, 7, "cad"), haystack + 4); @@ -1117,21 +1291,21 @@ test_util_pow2(void) test_eq(tor_log2(UINT64_MAX), 63); /* Test round_to_power_of_2 */ - test_eq(round_to_power_of_2(120), 128); - test_eq(round_to_power_of_2(128), 128); - test_eq(round_to_power_of_2(130), 128); - test_eq(round_to_power_of_2(U64_LITERAL(40000000000000000)), - U64_LITERAL(1)<<55); - test_eq(round_to_power_of_2(U64_LITERAL(0xffffffffffffffff)), + tt_u64_op(round_to_power_of_2(120), ==, 128); + tt_u64_op(round_to_power_of_2(128), ==, 128); + tt_u64_op(round_to_power_of_2(130), ==, 128); + tt_u64_op(round_to_power_of_2(U64_LITERAL(40000000000000000)), ==, + U64_LITERAL(1)<<55); + tt_u64_op(round_to_power_of_2(U64_LITERAL(0xffffffffffffffff)), ==, U64_LITERAL(1)<<63); - test_eq(round_to_power_of_2(0), 1); - test_eq(round_to_power_of_2(1), 1); - test_eq(round_to_power_of_2(2), 2); - test_eq(round_to_power_of_2(3), 2); - test_eq(round_to_power_of_2(4), 4); - test_eq(round_to_power_of_2(5), 4); - test_eq(round_to_power_of_2(6), 4); - test_eq(round_to_power_of_2(7), 8); + tt_u64_op(round_to_power_of_2(0), ==, 1); + tt_u64_op(round_to_power_of_2(1), ==, 1); + tt_u64_op(round_to_power_of_2(2), ==, 2); + tt_u64_op(round_to_power_of_2(3), ==, 2); + tt_u64_op(round_to_power_of_2(4), ==, 4); + tt_u64_op(round_to_power_of_2(5), ==, 4); + tt_u64_op(round_to_power_of_2(6), ==, 4); + tt_u64_op(round_to_power_of_2(7), ==, 8); done: ; @@ -1410,14 +1584,14 @@ test_util_mmap(void) test_eq(mapping->size, strlen("Short file.")); test_streq(mapping->data, "Short file."); #ifdef _WIN32 - tor_munmap_file(mapping); + tt_int_op(0, ==, tor_munmap_file(mapping)); mapping = NULL; test_assert(unlink(fname1) == 0); #else /* make sure we can unlink. */ test_assert(unlink(fname1) == 0); test_streq(mapping->data, "Short file."); - tor_munmap_file(mapping); + tt_int_op(0, ==, tor_munmap_file(mapping)); mapping = NULL; #endif @@ -1438,7 +1612,7 @@ test_util_mmap(void) test_assert(mapping); test_eq(mapping->size, buflen); test_memeq(mapping->data, buf, buflen); - tor_munmap_file(mapping); + tt_int_op(0, ==, tor_munmap_file(mapping)); mapping = NULL; /* Now try a big aligned file. */ @@ -1447,7 +1621,7 @@ test_util_mmap(void) test_assert(mapping); test_eq(mapping->size, 16384); test_memeq(mapping->data, buf, 16384); - tor_munmap_file(mapping); + tt_int_op(0, ==, tor_munmap_file(mapping)); mapping = NULL; done: @@ -1460,8 +1634,7 @@ test_util_mmap(void) tor_free(fname3); tor_free(buf); - if (mapping) - tor_munmap_file(mapping); + tor_munmap_file(mapping); } /** Run unit tests for escaping/unescaping data for use by controllers. */ @@ -1729,6 +1902,8 @@ test_util_path_is_relative(void) ; } +#ifdef ENABLE_MEMPOOLS + /** Run unittests for memory pool allocator */ static void test_util_mempool(void) @@ -1787,6 +1962,8 @@ test_util_mempool(void) mp_pool_destroy(pool); } +#endif /* ENABLE_MEMPOOLS */ + /** Run unittests for memory area allocator */ static void test_util_memarea(void) @@ -2071,18 +2248,21 @@ test_util_asprintf(void *ptr) test_assert(cp); test_streq("simple string 100% safe", cp); test_eq(strlen(cp), r); + tor_free(cp); /* empty string */ r = tor_asprintf(&cp, "%s", ""); test_assert(cp); test_streq("", cp); test_eq(strlen(cp), r); + tor_free(cp); /* numbers (%i) */ r = tor_asprintf(&cp, "I like numbers-%2i, %i, etc.", -1, 2); test_assert(cp); test_streq("I like numbers--1, 2, etc.", cp); test_eq(strlen(cp), r); + /* don't free cp; next test uses it. */ /* numbers (%d) */ r = tor_asprintf(&cp2, "First=%d, Second=%d", 101, 202); @@ -2155,6 +2335,8 @@ test_util_listdir(void *ptr) done: tor_free(fname1); tor_free(fname2); + tor_free(fname3); + tor_free(dir1); tor_free(dirname); if (dir_contents) { SMARTLIST_FOREACH(dir_contents, char *, cp, tor_free(cp)); @@ -2223,6 +2405,7 @@ test_util_load_win_lib(void *ptr) } #endif +#ifndef _WIN32 static void clear_hex_errno(char *hex_errno) { @@ -2267,6 +2450,7 @@ test_util_exit_status(void *ptr) done: ; } +#endif #ifndef _WIN32 /** Check that fgets waits until a full line, and not return a partial line, on @@ -2355,6 +2539,19 @@ test_util_fgets_eagain(void *ptr) } #endif +#ifndef BUILDDIR +#define BUILDDIR "." +#endif + +#ifdef _WIN32 +#define notify_pending_waitpid_callbacks() STMT_NIL +#define TEST_CHILD "test-child.exe" +#define EOL "\r\n" +#else +#define TEST_CHILD (BUILDDIR "/src/test/test-child") +#define EOL "\n" +#endif + /** Helper function for testing tor_spawn_background */ static void run_util_spawn_background(const char *argv[], const char *expected_out, @@ -2374,19 +2571,28 @@ run_util_spawn_background(const char *argv[], const char *expected_out, status = tor_spawn_background(argv[0], argv, NULL, &process_handle); #endif + notify_pending_waitpid_callbacks(); + test_eq(expected_status, status); - if (status == PROCESS_STATUS_ERROR) + if (status == PROCESS_STATUS_ERROR) { + tt_ptr_op(process_handle, ==, NULL); return; + } test_assert(process_handle != NULL); test_eq(expected_status, process_handle->status); +#ifndef _WIN32 + notify_pending_waitpid_callbacks(); + tt_ptr_op(process_handle->waitpid_cb, !=, NULL); +#endif + #ifdef _WIN32 test_assert(process_handle->stdout_pipe != INVALID_HANDLE_VALUE); test_assert(process_handle->stderr_pipe != INVALID_HANDLE_VALUE); #else - test_assert(process_handle->stdout_pipe > 0); - test_assert(process_handle->stderr_pipe > 0); + test_assert(process_handle->stdout_pipe >= 0); + test_assert(process_handle->stderr_pipe >= 0); #endif /* Check stdout */ @@ -2397,12 +2603,19 @@ run_util_spawn_background(const char *argv[], const char *expected_out, test_eq(strlen(expected_out), pos); test_streq(expected_out, stdout_buf); + notify_pending_waitpid_callbacks(); + /* Check it terminated correctly */ retval = tor_get_exit_code(process_handle, 1, &exit_code); test_eq(PROCESS_EXIT_EXITED, retval); test_eq(expected_exit, exit_code); // TODO: Make test-child exit with something other than 0 +#ifndef _WIN32 + notify_pending_waitpid_callbacks(); + tt_ptr_op(process_handle->waitpid_cb, ==, NULL); +#endif + /* Check stderr */ pos = tor_read_all_from_process_stderr(process_handle, stderr_buf, sizeof(stderr_buf) - 1); @@ -2411,6 +2624,8 @@ run_util_spawn_background(const char *argv[], const char *expected_out, test_streq(expected_err, stderr_buf); test_eq(strlen(expected_err), pos); + notify_pending_waitpid_callbacks(); + done: if (process_handle) tor_process_handle_destroy(process_handle, 1); @@ -2420,29 +2635,20 @@ run_util_spawn_background(const char *argv[], const char *expected_out, static void test_util_spawn_background_ok(void *ptr) { -#ifdef _WIN32 - const char *argv[] = {"test-child.exe", "--test", NULL}; - const char *expected_out = "OUT\r\n--test\r\nSLEEPING\r\nDONE\r\n"; - const char *expected_err = "ERR\r\n"; -#else - const char *argv[] = {BUILDDIR "/src/test/test-child", "--test", NULL}; - const char *expected_out = "OUT\n--test\nSLEEPING\nDONE\n"; - const char *expected_err = "ERR\n"; -#endif + const char *argv[] = {TEST_CHILD, "--test", NULL}; + const char *expected_out = "OUT"EOL "--test"EOL "SLEEPING"EOL "DONE" EOL; + const char *expected_err = "ERR"EOL; (void)ptr; run_util_spawn_background(argv, expected_out, expected_err, 0, - PROCESS_STATUS_RUNNING); + PROCESS_STATUS_RUNNING); } /** Check that failing to find the executable works as expected */ static void test_util_spawn_background_fail(void *ptr) { -#ifndef BUILDDIR -#define BUILDDIR "." -#endif const char *argv[] = {BUILDDIR "/src/test/no-such-file", "--test", NULL}; const char *expected_err = ""; char expected_out[1024]; @@ -2463,13 +2669,13 @@ test_util_spawn_background_fail(void *ptr) "ERR: Failed to spawn background process - code %s\n", code); run_util_spawn_background(argv, expected_out, expected_err, 255, - expected_status); + expected_status); } /** Test that reading from a handle returns a partial read rather than * blocking */ static void -test_util_spawn_background_partial_read(void *ptr) +test_util_spawn_background_partial_read_impl(int exit_early) { const int expected_exit = 0; const int expected_status = PROCESS_STATUS_RUNNING; @@ -2479,22 +2685,22 @@ test_util_spawn_background_partial_read(void *ptr) process_handle_t *process_handle=NULL; int status; char stdout_buf[100], stderr_buf[100]; -#ifdef _WIN32 - const char *argv[] = {"test-child.exe", "--test", NULL}; - const char *expected_out[] = { "OUT\r\n--test\r\nSLEEPING\r\n", - "DONE\r\n", - NULL }; - const char *expected_err = "ERR\r\n"; -#else - const char *argv[] = {BUILDDIR "/src/test/test-child", "--test", NULL}; - const char *expected_out[] = { "OUT\n--test\nSLEEPING\n", - "DONE\n", + + const char *argv[] = {TEST_CHILD, "--test", NULL}; + const char *expected_out[] = { "OUT" EOL "--test" EOL "SLEEPING" EOL, + "DONE" EOL, NULL }; - const char *expected_err = "ERR\n"; + const char *expected_err = "ERR" EOL; + +#ifndef _WIN32 int eof = 0; #endif int expected_out_ctr; - (void)ptr; + + if (exit_early) { + argv[1] = "--hang"; + expected_out[0] = "OUT"EOL "--hang"EOL "SLEEPING" EOL; + } /* Start the program */ #ifdef _WIN32 @@ -2530,6 +2736,12 @@ test_util_spawn_background_partial_read(void *ptr) expected_out_ctr++; } + if (exit_early) { + tor_process_handle_destroy(process_handle, 1); + process_handle = NULL; + goto done; + } + /* The process should have exited without writing more */ #ifdef _WIN32 pos = tor_read_all_handle(process_handle->stdout_pipe, stdout_buf, @@ -2567,15 +2779,84 @@ test_util_spawn_background_partial_read(void *ptr) tor_process_handle_destroy(process_handle, 1); } +static void +test_util_spawn_background_partial_read(void *arg) +{ + (void)arg; + test_util_spawn_background_partial_read_impl(0); +} + +static void +test_util_spawn_background_exit_early(void *arg) +{ + (void)arg; + test_util_spawn_background_partial_read_impl(1); +} + +static void +test_util_spawn_background_waitpid_notify(void *arg) +{ + int retval, exit_code; + process_handle_t *process_handle=NULL; + int status; + int ms_timer; + + const char *argv[] = {TEST_CHILD, "--fast", NULL}; + + (void) arg; + +#ifdef _WIN32 + status = tor_spawn_background(NULL, argv, NULL, &process_handle); +#else + status = tor_spawn_background(argv[0], argv, NULL, &process_handle); +#endif + + tt_int_op(status, ==, PROCESS_STATUS_RUNNING); + tt_ptr_op(process_handle, !=, NULL); + + /* We're not going to look at the stdout/stderr output this time. Instead, + * we're testing whether notify_pending_waitpid_calbacks() can report the + * process exit (on unix) and/or whether tor_get_exit_code() can notice it + * (on windows) */ + +#ifndef _WIN32 + ms_timer = 30*1000; + tt_ptr_op(process_handle->waitpid_cb, !=, NULL); + while (process_handle->waitpid_cb && ms_timer > 0) { + tor_sleep_msec(100); + ms_timer -= 100; + notify_pending_waitpid_callbacks(); + } + tt_int_op(ms_timer, >, 0); + tt_ptr_op(process_handle->waitpid_cb, ==, NULL); +#endif + + ms_timer = 30*1000; + while (((retval = tor_get_exit_code(process_handle, 0, &exit_code)) + == PROCESS_EXIT_RUNNING) && ms_timer > 0) { + tor_sleep_msec(100); + ms_timer -= 100; + } + tt_int_op(ms_timer, >, 0); + + tt_int_op(retval, ==, PROCESS_EXIT_EXITED); + + done: + tor_process_handle_destroy(process_handle, 1); +} + +#undef TEST_CHILD +#undef EOL + /** - * Test for format_hex_number_for_helper_exit_status() + * Test for format_hex_number_sigsafe() */ static void test_util_format_hex_number(void *ptr) { int i, len; - char buf[HEX_ERRNO_SIZE + 1]; + char buf[33]; const struct { const char *str; unsigned int x; @@ -2584,6 +2865,8 @@ test_util_format_hex_number(void *ptr) {"1", 1}, {"273A", 0x273a}, {"FFFF", 0xffff}, + {"7FFFFFFF", 0x7fffffff}, + {"FFFFFFFF", 0xffffffff}, #if UINT_MAX >= 0xffffffff {"31BC421D", 0x31bc421d}, {"FFFFFFFF", 0xffffffff}, @@ -2594,19 +2877,73 @@ test_util_format_hex_number(void *ptr) (void)ptr; for (i = 0; test_data[i].str != NULL; ++i) { - len = format_hex_number_for_helper_exit_status(test_data[i].x, - buf, HEX_ERRNO_SIZE); + len = format_hex_number_sigsafe(test_data[i].x, buf, sizeof(buf)); + test_neq(len, 0); + test_eq(len, strlen(buf)); + test_streq(buf, test_data[i].str); + } + + test_eq(4, format_hex_number_sigsafe(0xffff, buf, 5)); + test_streq(buf, "FFFF"); + test_eq(0, format_hex_number_sigsafe(0xffff, buf, 4)); + test_eq(0, format_hex_number_sigsafe(0, buf, 1)); + + done: + return; +} + +/** + * Test for format_hex_number_sigsafe() + */ + +static void +test_util_format_dec_number(void *ptr) +{ + int i, len; + char buf[33]; + const struct { + const char *str; + unsigned int x; + } test_data[] = { + {"0", 0}, + {"1", 1}, + {"1234", 1234}, + {"12345678", 12345678}, + {"99999999", 99999999}, + {"100000000", 100000000}, + {"4294967295", 4294967295u}, +#if UINT_MAX > 0xffffffff + {"18446744073709551615", 18446744073709551615u }, +#endif + {NULL, 0} + }; + + (void)ptr; + + for (i = 0; test_data[i].str != NULL; ++i) { + len = format_dec_number_sigsafe(test_data[i].x, buf, sizeof(buf)); test_neq(len, 0); - buf[len] = '\0'; + test_eq(len, strlen(buf)); + test_streq(buf, test_data[i].str); + + len = format_dec_number_sigsafe(test_data[i].x, buf, + (int)(strlen(test_data[i].str) + 1)); + test_eq(len, strlen(buf)); test_streq(buf, test_data[i].str); } + test_eq(4, format_dec_number_sigsafe(7331, buf, 5)); + test_streq(buf, "7331"); + test_eq(0, format_dec_number_sigsafe(7331, buf, 4)); + test_eq(1, format_dec_number_sigsafe(0, buf, 2)); + test_eq(0, format_dec_number_sigsafe(0, buf, 1)); + done: return; } /** - * Test that we can properly format q Windows command line + * Test that we can properly format a Windows command line */ static void test_util_join_win_cmdline(void *ptr) @@ -2817,7 +3154,7 @@ test_util_eat_whitespace(void *ptr) (void)ptr; /* Try one leading ws */ - strcpy(str, "fuubaar"); + strlcpy(str, "fuubaar", sizeof(str)); for (i = 0; i < sizeof(ws); ++i) { str[0] = ws[i]; test_eq_ptr(str + 1, eat_whitespace(str)); @@ -2832,14 +3169,14 @@ test_util_eat_whitespace(void *ptr) test_eq_ptr(str, eat_whitespace_eos_no_nl(str, str + strlen(str))); /* Empty string */ - strcpy(str, ""); + strlcpy(str, "", sizeof(str)); test_eq_ptr(str, eat_whitespace(str)); test_eq_ptr(str, eat_whitespace_eos(str, str)); test_eq_ptr(str, eat_whitespace_no_nl(str)); test_eq_ptr(str, eat_whitespace_eos_no_nl(str, str)); /* Only ws */ - strcpy(str, " \t\r\n"); + strlcpy(str, " \t\r\n", sizeof(str)); test_eq_ptr(str + strlen(str), eat_whitespace(str)); test_eq_ptr(str + strlen(str), eat_whitespace_eos(str, str + strlen(str))); test_eq_ptr(str + strlen(str) - 1, @@ -2847,7 +3184,7 @@ test_util_eat_whitespace(void *ptr) test_eq_ptr(str + strlen(str) - 1, eat_whitespace_eos_no_nl(str, str + strlen(str))); - strcpy(str, " \t\r "); + strlcpy(str, " \t\r ", sizeof(str)); test_eq_ptr(str + strlen(str), eat_whitespace(str)); test_eq_ptr(str + strlen(str), eat_whitespace_eos(str, str + strlen(str))); @@ -2856,7 +3193,7 @@ test_util_eat_whitespace(void *ptr) eat_whitespace_eos_no_nl(str, str + strlen(str))); /* Multiple ws */ - strcpy(str, "fuubaar"); + strlcpy(str, "fuubaar", sizeof(str)); for (i = 0; i < sizeof(ws); ++i) str[i] = ws[i]; test_eq_ptr(str + sizeof(ws), eat_whitespace(str)); @@ -2866,28 +3203,28 @@ test_util_eat_whitespace(void *ptr) eat_whitespace_eos_no_nl(str, str + strlen(str))); /* Eat comment */ - strcpy(str, "# Comment \n No Comment"); + strlcpy(str, "# Comment \n No Comment", sizeof(str)); test_streq("No Comment", eat_whitespace(str)); test_streq("No Comment", eat_whitespace_eos(str, str + strlen(str))); test_eq_ptr(str, eat_whitespace_no_nl(str)); test_eq_ptr(str, eat_whitespace_eos_no_nl(str, str + strlen(str))); /* Eat comment & ws mix */ - strcpy(str, " # \t Comment \n\t\nNo Comment"); + strlcpy(str, " # \t Comment \n\t\nNo Comment", sizeof(str)); test_streq("No Comment", eat_whitespace(str)); test_streq("No Comment", eat_whitespace_eos(str, str + strlen(str))); test_eq_ptr(str + 1, eat_whitespace_no_nl(str)); test_eq_ptr(str + 1, eat_whitespace_eos_no_nl(str, str + strlen(str))); /* Eat entire comment */ - strcpy(str, "#Comment"); + strlcpy(str, "#Comment", sizeof(str)); test_eq_ptr(str + strlen(str), eat_whitespace(str)); test_eq_ptr(str + strlen(str), eat_whitespace_eos(str, str + strlen(str))); test_eq_ptr(str, eat_whitespace_no_nl(str)); test_eq_ptr(str, eat_whitespace_eos_no_nl(str, str + strlen(str))); /* Blank line, then comment */ - strcpy(str, " \t\n # Comment"); + strlcpy(str, " \t\n # Comment", sizeof(str)); test_eq_ptr(str + strlen(str), eat_whitespace(str)); test_eq_ptr(str + strlen(str), eat_whitespace_eos(str, str + strlen(str))); test_eq_ptr(str + 2, eat_whitespace_no_nl(str)); @@ -2913,6 +3250,8 @@ smartlist_new_from_text_lines(const char *lines) last_line = smartlist_pop_last(sl); if (last_line != NULL && *last_line != '\0') { smartlist_add(sl, last_line); + } else { + tor_free(last_line); } return sl; @@ -3212,12 +3551,204 @@ test_util_mathlog(void *arg) ; } +static void +test_util_round_to_next_multiple_of(void *arg) +{ + (void)arg; + + test_assert(round_uint64_to_next_multiple_of(0,1) == 0); + test_assert(round_uint64_to_next_multiple_of(0,7) == 0); + + test_assert(round_uint64_to_next_multiple_of(99,1) == 99); + test_assert(round_uint64_to_next_multiple_of(99,7) == 105); + test_assert(round_uint64_to_next_multiple_of(99,9) == 99); + + done: + ; +} + +static void +test_util_strclear(void *arg) +{ + static const char *vals[] = { "", "a", "abcdef", "abcdefgh", NULL }; + int i; + char *v = NULL; + (void)arg; + + for (i = 0; vals[i]; ++i) { + size_t n; + v = tor_strdup(vals[i]); + n = strlen(v); + tor_strclear(v); + tt_assert(tor_mem_is_zero(v, n+1)); + tor_free(v); + } + done: + tor_free(v); +} + #define UTIL_LEGACY(name) \ { #name, legacy_test_helper, 0, &legacy_setup, test_util_ ## name } #define UTIL_TEST(name, flags) \ { #name, test_util_ ## name, flags, NULL, NULL } +#ifdef FD_CLOEXEC +#define CAN_CHECK_CLOEXEC +static int +fd_is_cloexec(tor_socket_t fd) +{ + int flags = fcntl(fd, F_GETFD, 0); + return (flags & FD_CLOEXEC) == FD_CLOEXEC; +} +#endif + +#ifndef _WIN32 +#define CAN_CHECK_NONBLOCK +static int +fd_is_nonblocking(tor_socket_t fd) +{ + int flags = fcntl(fd, F_GETFL, 0); + return (flags & O_NONBLOCK) == O_NONBLOCK; +} +#endif + +static void +test_util_socket(void *arg) +{ + tor_socket_t fd1 = TOR_INVALID_SOCKET; + tor_socket_t fd2 = TOR_INVALID_SOCKET; + tor_socket_t fd3 = TOR_INVALID_SOCKET; + tor_socket_t fd4 = TOR_INVALID_SOCKET; + int n = get_n_open_sockets(); + + TT_BLATHER(("Starting with %d open sockets.", n)); + + (void)arg; + + fd1 = tor_open_socket_with_extensions(AF_INET, SOCK_STREAM, 0, 0, 0); + fd2 = tor_open_socket_with_extensions(AF_INET, SOCK_STREAM, 0, 0, 1); + tt_assert(SOCKET_OK(fd1)); + tt_assert(SOCKET_OK(fd2)); + tt_int_op(get_n_open_sockets(), ==, n + 2); + //fd3 = tor_open_socket_with_extensions(AF_INET, SOCK_STREAM, 0, 1, 0); + //fd4 = tor_open_socket_with_extensions(AF_INET, SOCK_STREAM, 0, 1, 1); + fd3 = tor_open_socket(AF_INET, SOCK_STREAM, 0); + fd4 = tor_open_socket_nonblocking(AF_INET, SOCK_STREAM, 0); + tt_assert(SOCKET_OK(fd3)); + tt_assert(SOCKET_OK(fd4)); + tt_int_op(get_n_open_sockets(), ==, n + 4); + +#ifdef CAN_CHECK_CLOEXEC + tt_int_op(fd_is_cloexec(fd1), ==, 0); + tt_int_op(fd_is_cloexec(fd2), ==, 0); + tt_int_op(fd_is_cloexec(fd3), ==, 1); + tt_int_op(fd_is_cloexec(fd4), ==, 1); +#endif +#ifdef CAN_CHECK_NONBLOCK + tt_int_op(fd_is_nonblocking(fd1), ==, 0); + tt_int_op(fd_is_nonblocking(fd2), ==, 1); + tt_int_op(fd_is_nonblocking(fd3), ==, 0); + tt_int_op(fd_is_nonblocking(fd4), ==, 1); +#endif + + tor_close_socket(fd1); + tor_close_socket(fd2); + fd1 = fd2 = TOR_INVALID_SOCKET; + tt_int_op(get_n_open_sockets(), ==, n + 2); + tor_close_socket(fd3); + tor_close_socket(fd4); + fd3 = fd4 = TOR_INVALID_SOCKET; + tt_int_op(get_n_open_sockets(), ==, n); + + done: + if (SOCKET_OK(fd1)) + tor_close_socket(fd1); + if (SOCKET_OK(fd2)) + tor_close_socket(fd2); + if (SOCKET_OK(fd3)) + tor_close_socket(fd3); + if (SOCKET_OK(fd4)) + tor_close_socket(fd4); +} + +static void * +socketpair_test_setup(const struct testcase_t *testcase) +{ + return testcase->setup_data; +} +static int +socketpair_test_cleanup(const struct testcase_t *testcase, void *ptr) +{ + (void)testcase; + (void)ptr; + return 1; +} + +static const struct testcase_setup_t socketpair_setup = { + socketpair_test_setup, socketpair_test_cleanup +}; + +/* Test for socketpair and ersatz_socketpair(). We test them both, since + * the latter is a tolerably good way to exersize tor_accept_socket(). */ +static void +test_util_socketpair(void *arg) +{ + const int ersatz = !strcmp(arg, "1"); + int (*const tor_socketpair_fn)(int, int, int, tor_socket_t[2]) = + ersatz ? tor_ersatz_socketpair : tor_socketpair; + int n = get_n_open_sockets(); + tor_socket_t fds[2] = {TOR_INVALID_SOCKET, TOR_INVALID_SOCKET}; + const int family = AF_UNIX; + + tt_int_op(0, ==, tor_socketpair_fn(family, SOCK_STREAM, 0, fds)); + tt_assert(SOCKET_OK(fds[0])); + tt_assert(SOCKET_OK(fds[1])); + tt_int_op(get_n_open_sockets(), ==, n + 2); +#ifdef CAN_CHECK_CLOEXEC + tt_int_op(fd_is_cloexec(fds[0]), ==, 1); + tt_int_op(fd_is_cloexec(fds[1]), ==, 1); +#endif +#ifdef CAN_CHECK_NONBLOCK + tt_int_op(fd_is_nonblocking(fds[0]), ==, 0); + tt_int_op(fd_is_nonblocking(fds[1]), ==, 0); +#endif + + done: + if (SOCKET_OK(fds[0])) + tor_close_socket(fds[0]); + if (SOCKET_OK(fds[1])) + tor_close_socket(fds[1]); +} + +static void +test_util_max_mem(void *arg) +{ + size_t memory1, memory2; + int r, r2; + (void) arg; + + r = get_total_system_memory(&memory1); + r2 = get_total_system_memory(&memory2); + tt_int_op(r, ==, r2); + tt_uint_op(memory2, ==, memory1); + + TT_BLATHER(("System memory: "U64_FORMAT, U64_PRINTF_ARG(memory1))); + + if (r==0) { + /* You have at least a megabyte. */ + tt_uint_op(memory1, >, (1<<20)); + } else { + /* You do not have a petabyte. */ +#if SIZEOF_SIZE_T == SIZEOF_UINT64_T + tt_uint_op(memory1, <, (U64_LITERAL(1)<<50)); +#endif + } + + done: + ; +} + struct testcase_t util_tests[] = { UTIL_LEGACY(time), UTIL_TEST(parse_http_time, 0), @@ -3228,11 +3759,15 @@ struct testcase_t util_tests[] = { #ifndef _WIN32 UTIL_LEGACY(expand_filename), #endif + UTIL_LEGACY(escape_string_socks), + UTIL_LEGACY(string_is_key_value), UTIL_LEGACY(strmisc), UTIL_LEGACY(pow2), UTIL_LEGACY(gzip), UTIL_LEGACY(datadir), +#ifdef ENABLE_MEMPOOLS UTIL_LEGACY(mempool), +#endif UTIL_LEGACY(memarea), UTIL_LEGACY(control_formats), UTIL_LEGACY(mmap), @@ -3241,6 +3776,8 @@ struct testcase_t util_tests[] = { UTIL_LEGACY(path_is_relative), UTIL_LEGACY(strtok), UTIL_LEGACY(di_ops), + UTIL_TEST(round_to_next_multiple_of, 0), + UTIL_TEST(strclear, 0), UTIL_TEST(find_str_at_start_of_line, 0), UTIL_TEST(string_is_C_identifier, 0), UTIL_TEST(asprintf, 0), @@ -3249,14 +3786,17 @@ struct testcase_t util_tests[] = { #ifdef _WIN32 UTIL_TEST(load_win_lib, 0), #endif - UTIL_TEST(exit_status, 0), #ifndef _WIN32 + UTIL_TEST(exit_status, 0), UTIL_TEST(fgets_eagain, TT_SKIP), #endif UTIL_TEST(spawn_background_ok, 0), UTIL_TEST(spawn_background_fail, 0), UTIL_TEST(spawn_background_partial_read, 0), + UTIL_TEST(spawn_background_exit_early, 0), + UTIL_TEST(spawn_background_waitpid_notify, 0), UTIL_TEST(format_hex_number, 0), + UTIL_TEST(format_dec_number, 0), UTIL_TEST(join_win_cmdline, 0), UTIL_TEST(split_lines, 0), UTIL_TEST(n_bits_set, 0), @@ -3268,8 +3808,15 @@ struct testcase_t util_tests[] = { UTIL_TEST(read_file_eof_tiny_limit, 0), UTIL_TEST(read_file_eof_two_loops, 0), UTIL_TEST(read_file_eof_zero_bytes, 0), + UTIL_TEST(write_chunks_to_file, 0), UTIL_TEST(mathlog, 0), UTIL_TEST(weak_random, 0), + UTIL_TEST(socket, TT_FORK), + { "socketpair", test_util_socketpair, TT_FORK, &socketpair_setup, + (void*)"0" }, + { "socketpair_ersatz", test_util_socketpair, TT_FORK, + &socketpair_setup, (void*)"1" }, + UTIL_TEST(max_mem, 0), END_OF_TESTCASES }; diff --git a/src/tools/tor-checkkey.c b/src/tools/tor-checkkey.c index a3860ca4b7..d50f12ed2a 100644 --- a/src/tools/tor-checkkey.c +++ b/src/tools/tor-checkkey.c @@ -1,8 +1,6 @@ /* Copyright (c) 2008-2013, The Tor Project, Inc. */ /* See LICENSE for licensing information */ -#define CRYPTO_PRIVATE - #include "orconfig.h" #include <stdio.h> diff --git a/src/tools/tor-fw-helper/include.am b/src/tools/tor-fw-helper/include.am index 275a0e237c..1f862e6f06 100644 --- a/src/tools/tor-fw-helper/include.am +++ b/src/tools/tor-fw-helper/include.am @@ -33,4 +33,4 @@ endif src_tools_tor_fw_helper_tor_fw_helper_LDFLAGS = $(nat_pmp_ldflags) $(miniupnpc_ldflags) src_tools_tor_fw_helper_tor_fw_helper_LDADD = src/common/libor.a $(nat_pmp_ldadd) $(miniupnpc_ldadd) -lm @TOR_LIB_WS32@ -src_tools_tor_fw_helper_tor_fw_helper_CPPFLAGS = $(nat_pmp_cppflags) $(miniupnpc_cppflags) +src_tools_tor_fw_helper_tor_fw_helper_CPPFLAGS = $(nat_pmp_cppflags) $(miniupnpc_cppflags) -I"$(top_srcdir)/src/ext" diff --git a/src/tools/tor-fw-helper/tor-fw-helper.c b/src/tools/tor-fw-helper/tor-fw-helper.c index bb6e70aaa3..84cc21e346 100644 --- a/src/tools/tor-fw-helper/tor-fw-helper.c +++ b/src/tools/tor-fw-helper/tor-fw-helper.c @@ -496,6 +496,6 @@ main(int argc, char **argv) smartlist_free(tor_fw_options.ports_to_forward); } - exit(r); + exit(0); } diff --git a/src/tools/tor-gencert.c b/src/tools/tor-gencert.c index 3809b22d43..e799df5cad 100644 --- a/src/tools/tor-gencert.c +++ b/src/tools/tor-gencert.c @@ -27,8 +27,6 @@ #include <assert.h> #endif -#define CRYPTO_PRIVATE - #include "compat.h" #include "../common/util.h" #include "../common/torlog.h" @@ -36,7 +34,7 @@ #include "address.h" #define IDENTITY_KEY_BITS 3072 -#define SIGNING_KEY_BITS 1024 +#define SIGNING_KEY_BITS 2048 #define DEFAULT_LIFETIME 12 /* These globals are set via command line options. */ @@ -304,6 +302,7 @@ load_identity_key(void) if (!identity_key) { log_err(LD_GENERAL, "Couldn't read identity key from %s", identity_key_file); + fclose(f); return 1; } fclose(f); @@ -324,6 +323,7 @@ load_signing_key(void) } if (!(signing_key = PEM_read_PrivateKey(f, NULL, NULL, NULL))) { log_err(LD_GENERAL, "Couldn't read siging key from %s", signing_key_file); + fclose(f); return 1; } fclose(f); @@ -549,6 +549,9 @@ main(int argc, char **argv) if (signing_key) EVP_PKEY_free(signing_key); tor_free(address); + tor_free(identity_key_file); + tor_free(signing_key_file); + tor_free(certificate_file); crypto_global_cleanup(); return r; diff --git a/src/win32/orconfig.h b/src/win32/orconfig.h index c86e2b0fae..8c2472c323 100644 --- a/src/win32/orconfig.h +++ b/src/win32/orconfig.h @@ -241,7 +241,7 @@ #define USING_TWOS_COMPLEMENT /* Version number of package */ -#define VERSION "0.2.4.23" +#define VERSION "0.2.5.6-alpha" |