diff options
Diffstat (limited to 'src')
125 files changed, 32584 insertions, 15725 deletions
diff --git a/src/Makefile.am b/src/Makefile.am index ae647b2d61..fa2dd560a6 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -1,5 +1,5 @@ # leave in dependency order, since common must be built first -SUBDIRS = common or tools win32 config -DIST_SUBDIRS = common or tools win32 config +SUBDIRS = common or test tools win32 config +DIST_SUBDIRS = common or test tools win32 config diff --git a/src/common/Makefile.am b/src/common/Makefile.am index db94a984ca..2d009bd4fa 100644 --- a/src/common/Makefile.am +++ b/src/common/Makefile.am @@ -1,5 +1,7 @@ -noinst_LIBRARIES = libor.a libor-crypto.a +noinst_LIBRARIES = libor.a libor-crypto.a libor-event.a + +EXTRA_DIST = common_sha1.i sha256.c #CFLAGS = -Wall -Wpointer-arith -O2 @@ -10,7 +12,21 @@ libor_extra_source= endif libor_a_SOURCES = address.c log.c util.c compat.c container.c mempool.c \ - memarea.c di_ops.c $(libor_extra_source) + memarea.c di_ops.c util_codedigest.c $(libor_extra_source) libor_crypto_a_SOURCES = crypto.c aes.c tortls.c torgzip.c +libor_event_a_SOURCES = compat_libevent.c + +noinst_HEADERS = address.h torlog.h crypto.h util.h compat.h aes.h torint.h tortls.h strlcpy.c strlcat.c torgzip.h container.h ht.h mempool.h memarea.h ciphers.inc compat_libevent.h tortls_states.h di_ops.h + +common_sha1.i: $(libor_SOURCES) $(libor_crypto_a_SOURCES) $(noinst_HEADERS) + if test "@SHA1SUM@" != none; then \ + @SHA1SUM@ $(libor_SOURCES) $(libor_crypto_a_SOURCES) $(noinst_HEADERS) | @SED@ -n 's/^\(.*\)$$/"\1\\n"/p' > common_sha1.i; \ + elif test "@OPENSSL@" != none; then \ + @OPENSSL@ sha1 $(libor_SOURCES) $(libor_crypto_a_SOURCES) $(noinst_HEADERS) | @SED@ -n 's/SHA1(\(.*\))= \(.*\)/"\2 \1\\n"/p' > common_sha1.i; \ + else \ + rm common_sha1.i; \ + touch common_sha1.i; \ + fi -noinst_HEADERS = address.h log.h crypto.h test.h util.h compat.h aes.h torint.h tortls.h strlcpy.c strlcat.c torgzip.h container.h ht.h mempool.h memarea.h di_ops.h ciphers.inc +util_codedigest.o: common_sha1.i +crypto.o: sha256.c diff --git a/src/common/address.c b/src/common/address.c index 14f1d3f604..34e109fcf4 100644 --- a/src/common/address.c +++ b/src/common/address.c @@ -12,7 +12,7 @@ #include "compat.h" #include "util.h" #include "address.h" -#include "log.h" +#include "torlog.h" #ifdef MS_WINDOWS #include <process.h> @@ -50,11 +50,13 @@ #include <assert.h> /** Convert the tor_addr_t in <b>a</b>, with port in <b>port</b>, into a - * socklen object in *<b>sa_out</b> of object size <b>len</b>. If not enough - * room is free, or on error, return -1. Else return the length of the - * sockaddr. */ -/* XXXX021 This returns socklen_t. socklen_t is sometimes unsigned. This - * function claims to return -1 sometimes. Problematic! */ + * sockaddr object in *<b>sa_out</b> of object size <b>len</b>. If not enough + * room is available in sa_out, or on error, return 0. On success, return + * the length of the sockaddr. + * + * Interface note: ordinarily, we return -1 for error. We can't do that here, + * since socklen_t is unsigned on some platforms. + **/ socklen_t tor_addr_to_sockaddr(const tor_addr_t *a, uint16_t port, @@ -65,7 +67,7 @@ tor_addr_to_sockaddr(const tor_addr_t *a, if (family == AF_INET) { struct sockaddr_in *sin; if (len < (int)sizeof(struct sockaddr_in)) - return -1; + return 0; sin = (struct sockaddr_in *)sa_out; memset(sin, 0, sizeof(struct sockaddr_in)); #ifdef HAVE_STRUCT_SOCKADDR_IN_SIN_LEN @@ -78,7 +80,7 @@ tor_addr_to_sockaddr(const tor_addr_t *a, } else if (family == AF_INET6) { struct sockaddr_in6 *sin6; if (len < (int)sizeof(struct sockaddr_in6)) - return -1; + return 0; sin6 = (struct sockaddr_in6 *)sa_out; memset(sin6, 0, sizeof(struct sockaddr_in6)); #ifdef HAVE_STRUCT_SOCKADDR_IN6_SIN6_LEN @@ -89,7 +91,7 @@ tor_addr_to_sockaddr(const tor_addr_t *a, memcpy(&sin6->sin6_addr, tor_addr_to_in6(a), sizeof(struct in6_addr)); return sizeof(struct sockaddr_in6); } else { - return -1; + return 0; } } @@ -508,7 +510,8 @@ tor_addr_parse_mask_ports(const char *s, tor_addr_t *addr_out, tor_assert(s); tor_assert(addr_out); - /* IP, [], /mask, ports */ + /** Longest possible length for an address, mask, and port-range combination. + * Includes IP, [], /mask, :, ports */ #define MAX_ADDRESS_LENGTH (TOR_ADDR_BUF_LEN+2+(1+INET_NTOA_BUF_LEN)+12+1) if (strlen(s) > MAX_ADDRESS_LENGTH) { @@ -601,7 +604,7 @@ tor_addr_parse_mask_ports(const char *s, tor_addr_t *addr_out, if (family == AF_INET6 && v4map) { if (bits > 32 && bits < 96) { /* Crazy */ log_warn(LD_GENERAL, - "Bad mask bits %i for V4-mapped V6 address; rejecting.", + "Bad mask bits %d for V4-mapped V6 address; rejecting.", bits); goto err; } @@ -726,7 +729,7 @@ tor_addr_is_loopback(const tor_addr_t *addr) } /** Set <b>dest</b> to equal the IPv4 address in <b>v4addr</b> (given in - * network order. */ + * network order). */ void tor_addr_from_ipv4n(tor_addr_t *dest, uint32_t v4addr) { @@ -760,6 +763,8 @@ tor_addr_from_in6(tor_addr_t *dest, const struct in6_addr *in6) void tor_addr_copy(tor_addr_t *dest, const tor_addr_t *src) { + if (src == dest) + return; tor_assert(src); tor_assert(dest); memcpy(dest, src, sizeof(tor_addr_t)); @@ -786,7 +791,7 @@ tor_addr_compare(const tor_addr_t *addr1, const tor_addr_t *addr2, * Reduce over-specific masks (>128 for ipv6, >32 for ipv4) to 128 or 32. * * The mask is interpreted relative to <b>addr1</b>, so that if a is - * ::ffff:1.2.3.4, and b is 3.4.5.6, + * \::ffff:1.2.3.4, and b is 3.4.5.6, * tor_addr_compare_masked(a,b,100,CMP_SEMANTIC) is the same as * -tor_addr_compare_masked(b,a,4,CMP_SEMANTIC). * @@ -798,6 +803,8 @@ int tor_addr_compare_masked(const tor_addr_t *addr1, const tor_addr_t *addr2, maskbits_t mbits, tor_addr_comparison_t how) { + /** Helper: Evaluates to -1 if a is less than b, 0 if a equals b, or 1 if a + * is greater than b. May evaluate a and b more than once. */ #define TRISTATE(a,b) (((a)<(b))?-1: (((a)==(b))?0:1)) sa_family_t family1, family2, v_family1, v_family2; @@ -912,13 +919,6 @@ tor_dup_addr(const tor_addr_t *addr) return tor_strdup(buf); } -/** Copy the address in <b>src</b> to <b>dest</b> */ -void -tor_addr_assign(tor_addr_t *dest, const tor_addr_t *src) -{ - memcpy(dest, src, sizeof(tor_addr_t)); -} - /** Return a string representing the address <b>addr</b>. This string is * statically allocated, and must not be freed. Each call to * <b>fmt_addr</b> invalidates the last result of the function. This @@ -1030,27 +1030,28 @@ get_interface_address6(int severity, sa_family_t family, tor_addr_t *addr) { int sock=-1, r=-1; struct sockaddr_storage my_addr, target_addr; - socklen_t my_addr_len; + socklen_t addr_len; tor_assert(addr); memset(addr, 0, sizeof(tor_addr_t)); memset(&target_addr, 0, sizeof(target_addr)); - my_addr_len = (socklen_t)sizeof(my_addr); - /* Use the "discard" service port */ - ((struct sockaddr_in*)&target_addr)->sin_port = 9; /* Don't worry: no packets are sent. We just need to use a real address * on the actual Internet. */ if (family == AF_INET6) { struct sockaddr_in6 *sin6 = (struct sockaddr_in6*)&target_addr; + /* Use the "discard" service port */ + sin6->sin6_port = htons(9); sock = tor_open_socket(PF_INET6,SOCK_DGRAM,IPPROTO_UDP); - my_addr_len = (socklen_t)sizeof(struct sockaddr_in6); + addr_len = (socklen_t)sizeof(struct sockaddr_in6); sin6->sin6_family = AF_INET6; S6_ADDR16(sin6->sin6_addr)[0] = htons(0x2002); /* 2002:: */ } else if (family == AF_INET) { struct sockaddr_in *sin = (struct sockaddr_in*)&target_addr; + /* Use the "discard" service port */ + sin->sin_port = htons(9); sock = tor_open_socket(PF_INET,SOCK_DGRAM,IPPROTO_UDP); - my_addr_len = (socklen_t)sizeof(struct sockaddr_in); + addr_len = (socklen_t)sizeof(struct sockaddr_in); sin->sin_family = AF_INET; sin->sin_addr.s_addr = htonl(0x12000001); /* 18.0.0.1 */ } else { @@ -1063,14 +1064,13 @@ get_interface_address6(int severity, sa_family_t family, tor_addr_t *addr) goto err; } - if (connect(sock,(struct sockaddr *)&target_addr, - (socklen_t)sizeof(target_addr))<0) { + if (connect(sock,(struct sockaddr *)&target_addr, addr_len) < 0) { int e = tor_socket_errno(sock); log_fn(severity, LD_NET, "connect() failed: %s", tor_socket_strerror(e)); goto err; } - if (getsockname(sock,(struct sockaddr*)&my_addr, &my_addr_len)) { + if (getsockname(sock,(struct sockaddr*)&my_addr, &addr_len)) { int e = tor_socket_errno(sock); log_fn(severity, LD_NET, "getsockname() to determine interface failed: %s", tor_socket_strerror(e)); @@ -1087,7 +1087,7 @@ get_interface_address6(int severity, sa_family_t family, tor_addr_t *addr) /* ====== * IPv4 helpers - * XXXX022 IPv6 deprecate some of these. + * XXXX023 IPv6 deprecate some of these. */ /** Return true iff <b>ip</b> (in host order) is an IP reserved to localhost, diff --git a/src/common/address.h b/src/common/address.h index cb2438791f..d05a3de2e7 100644 --- a/src/common/address.h +++ b/src/common/address.h @@ -53,8 +53,20 @@ tor_addr_to_in6(const tor_addr_t *a) return a->family == AF_INET6 ? &a->addr.in6_addr : NULL; } +/** Given an IPv6 address <b>x</b>, yield it as an array of uint8_t. + * + * Requires that <b>x</b> is actually an IPv6 address. + */ #define tor_addr_to_in6_addr8(x) tor_addr_to_in6(x)->s6_addr +/** Given an IPv6 address <b>x</b>, yield it as an array of uint16_t. + * + * Requires that <b>x</b> is actually an IPv6 address. + */ #define tor_addr_to_in6_addr16(x) S6_ADDR16(*tor_addr_to_in6(x)) +/** Given an IPv6 address <b>x</b>, yield it as an array of uint32_t. + * + * Requires that <b>x</b> is actually an IPv6 address. + */ #define tor_addr_to_in6_addr32(x) S6_ADDR32(*tor_addr_to_in6(x)) /** Return an IPv4 address in network order for <b>a</b>, or 0 if @@ -71,7 +83,7 @@ tor_addr_to_ipv4h(const tor_addr_t *a) { return ntohl(tor_addr_to_ipv4n(a)); } -/* Given an IPv6 address, return its mapped IPv4 address in host order, or +/** Given an IPv6 address, return its mapped IPv4 address in host order, or * 0 if <b>a</b> is not an IPv6 address. * * (Does not check whether the address is really a mapped address */ @@ -102,12 +114,17 @@ tor_addr_eq_ipv4h(const tor_addr_t *a, uint32_t u) return a->family == AF_INET ? (tor_addr_to_ipv4h(a) == u) : 0; } -#define TOR_ADDR_BUF_LEN 48 /* [ffff:ffff:ffff:ffff:ffff:ffff:255.255.255.255] - */ +/** Length of a buffer that you need to allocate to be sure you can encode + * any tor_addr_t. + * + * This allows enough space for + * "[ffff:ffff:ffff:ffff:ffff:ffff:255.255.255.255]", + * plus a terminating NUL. + */ +#define TOR_ADDR_BUF_LEN 48 int tor_addr_lookup(const char *name, uint16_t family, tor_addr_t *addr_out); char *tor_dup_addr(const tor_addr_t *addr) ATTR_MALLOC; -void tor_addr_assign(tor_addr_t *dest, const tor_addr_t *src); const char *fmt_addr(const tor_addr_t *addr); int get_interface_address6(int severity, sa_family_t family, tor_addr_t *addr); @@ -155,6 +172,7 @@ void tor_addr_from_ipv4n(tor_addr_t *dest, uint32_t v4addr); #define tor_addr_from_ipv4h(dest, v4addr) \ tor_addr_from_ipv4n((dest), htonl(v4addr)) void tor_addr_from_ipv6_bytes(tor_addr_t *dest, const char *bytes); +/** Set <b>dest</b> to the IPv4 address incoded in <b>in</b>. */ #define tor_addr_from_in(dest, in) \ tor_addr_from_ipv4n((dest), (in)->s_addr); void tor_addr_from_in6(tor_addr_t *dest, const struct in6_addr *in6); diff --git a/src/common/aes.c b/src/common/aes.c index 7e332b0502..c2fdeb594a 100644 --- a/src/common/aes.c +++ b/src/common/aes.c @@ -21,7 +21,7 @@ #include "compat.h" #include "aes.h" #include "util.h" -#include "log.h" +#include "torlog.h" /* We have 3 strategies for getting AES: Via OpenSSL's AES_encrypt function, * via OpenSSL's EVP_EncryptUpdate function, or via the built-in AES @@ -100,6 +100,9 @@ /* Figure out which AES optimizations to use. */ #ifdef USE_BUILTIN_AES +/** If this is defined, we take advantage of the fact that AES treats its + * input as a set of 4 32-bit words, so that there is no need to encode and + * decode the 128-bit counter before every block encryption */ # define USE_RIJNDAEL_COUNTER_OPTIMIZATION # if 0 && (defined(__powerpc__) || defined(__powerpc64__)) /* XXXX do more experimentation before concluding this is actually @@ -263,7 +266,8 @@ aes_set_key(aes_cnt_cipher_t *cipher, const char *key, int key_bits) void aes_free_cipher(aes_cnt_cipher_t *cipher) { - tor_assert(cipher); + if (!cipher) + return; #ifdef USE_OPENSSL_EVP EVP_CIPHER_CTX_cleanup(&cipher->key); #endif diff --git a/src/common/compat.c b/src/common/compat.c index 39651084a0..ea7f9d7efc 100644 --- a/src/common/compat.c +++ b/src/common/compat.c @@ -15,6 +15,8 @@ /* This is required on rh7 to make strptime not complain. * We also need it to make memmem get defined (where available) */ +/* XXXX023 We should just use AC_USE_SYSTEM_EXTENSIONS in our autoconf, + * and get this (and other important stuff!) automatically */ #define _GNU_SOURCE #include "compat.h" @@ -88,7 +90,7 @@ #include <sys/prctl.h> #endif -#include "log.h" +#include "torlog.h" #include "util.h" #include "container.h" #include "address.h" @@ -170,12 +172,17 @@ tor_munmap_file(tor_mmap_t *handle) tor_mmap_t * tor_mmap_file(const char *filename) { + TCHAR tfilename[MAX_PATH]= {0}; tor_mmap_t *res = tor_malloc_zero(sizeof(tor_mmap_t)); int empty = 0; res->file_handle = INVALID_HANDLE_VALUE; res->mmap_handle = NULL; - - res->file_handle = CreateFile(filename, +#ifdef UNICODE + mbstowcs(tfilename,filename,MAX_PATH); +#else + strlcpy(tfilename,filename,MAX_PATH); +#endif + res->file_handle = CreateFile(tfilename, GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, @@ -308,6 +315,100 @@ tor_vsnprintf(char *str, size_t size, const char *format, va_list args) return r; } +/** + * Portable asprintf implementation. Does a printf() into a newly malloc'd + * string. Sets *<b>strp</b> to this string, and returns its length (not + * including the terminating NUL character). + * + * You can treat this function as if its implementation were something like + <pre> + char buf[_INFINITY_]; + tor_snprintf(buf, sizeof(buf), fmt, args); + *strp = tor_strdup(buf); + return strlen(*strp): + </pre> + * Where _INFINITY_ is an imaginary constant so big that any string can fit + * into it. + */ +int +tor_asprintf(char **strp, const char *fmt, ...) +{ + int r; + va_list args; + va_start(args, fmt); + r = tor_vasprintf(strp, fmt, args); + va_end(args); + if (!*strp || r < 0) { + log_err(LD_BUG, "Internal error in asprintf"); + tor_assert(0); + } + return r; +} + +/** + * Portable vasprintf implementation. Does a printf() into a newly malloc'd + * string. Differs from regular vasprintf in the same ways that + * tor_asprintf() differs from regular asprintf. + */ +int +tor_vasprintf(char **strp, const char *fmt, va_list args) +{ + /* use a temporary variable in case *strp is in args. */ + char *strp_tmp=NULL; +#ifdef HAVE_VASPRINTF + /* If the platform gives us one, use it. */ + int r = vasprintf(&strp_tmp, fmt, args); + if (r < 0) + *strp = NULL; + else + *strp = strp_tmp; + return r; +#elif defined(_MSC_VER) + /* On Windows, _vsnprintf won't tell us the length of the string if it + * overflows, so we need to use _vcsprintf to tell how much to allocate */ + int len, r; + char *res; + len = _vscprintf(fmt, args); + if (len < 0) { + *strp = NULL; + return -1; + } + strp_tmp = tor_malloc(len + 1); + r = _vsnprintf(strp_tmp, len+1, fmt, args); + if (r != len) { + tor_free(strp_tmp); + *strp = NULL; + return -1; + } + *strp = strp_tmp; + return len; +#else + /* Everywhere else, we have a decent vsnprintf that tells us how many + * characters we need. We give it a try on a short buffer first, since + * it might be nice to avoid the second vsnprintf call. + */ + char buf[128]; + int len, r; + va_list tmp_args; + va_copy(tmp_args, args); + len = vsnprintf(buf, sizeof(buf), fmt, tmp_args); + va_end(tmp_args); + if (len < (int)sizeof(buf)) { + *strp = tor_strdup(buf); + return len; + } + strp_tmp = tor_malloc(len+1); + r = vsnprintf(strp_tmp, len+1, fmt, args); + if (r != len) { + tor_free(strp_tmp); + *strp = NULL; + return -1; + } + *strp = strp_tmp; + return len; +#endif +} + /** Given <b>hlen</b> bytes at <b>haystack</b> and <b>nlen</b> bytes at * <b>needle</b>, return a pointer to the first occurrence of the needle * within the haystack, or NULL if there is no such occurrence. @@ -401,6 +502,37 @@ const char TOR_TOLOWER_TABLE[256] = { 240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255, }; +/** Implementation of strtok_r for platforms whose coders haven't figured out + * how to write one. Hey guys! You can use this code here for free! */ +char * +tor_strtok_r_impl(char *str, const char *sep, char **lasts) +{ + char *cp, *start; + if (str) + start = cp = *lasts = str; + else if (!*lasts) + return NULL; + else + start = cp = *lasts; + + tor_assert(*sep); + if (sep[1]) { + while (*cp && !strchr(sep, *cp)) + ++cp; + } else { + tor_assert(strlen(sep) == 1); + cp = strchr(cp, *sep); + } + + if (!cp || !*cp) { + *lasts = NULL; + } else { + *cp++ = '\0'; + *lasts = cp; + } + return start; +} + #ifdef MS_WINDOWS /** Take a filename and return a pointer to its final element. This * function is called on __FILE__ to fix a MSVC nit where __FILE__ @@ -452,8 +584,8 @@ get_uint32(const void *cp) return v; } /** - * Read a 32-bit value beginning at <b>cp</b>. Equivalent to - * *(uint32_t*)(cp), but will not cause segfaults on platforms that forbid + * Read a 64-bit value beginning at <b>cp</b>. Equivalent to + * *(uint64_t*)(cp), but will not cause segfaults on platforms that forbid * unaligned memory access. */ uint64_t @@ -532,7 +664,9 @@ touch_file(const char *fname) /** Represents a lockfile on which we hold the lock. */ struct tor_lockfile_t { + /** Name of the file */ char *filename; + /** File descriptor used to hold the file open */ int fd; }; @@ -548,7 +682,10 @@ struct tor_lockfile_t { * * (Implementation note: because we need to fall back to fcntl on some * platforms, these locks are per-process, not per-thread. If you want - * to do in-process locking, use tor_mutex_t like a normal person.) + * to do in-process locking, use tor_mutex_t like a normal person. + * On Windows, when <b>blocking</b> is true, the maximum time that + * is actually waited is 10 seconds, after which NULL is returned + * and <b>locked_out</b> is set to 1.) */ tor_lockfile_t * tor_lockfile_lock(const char *filename, int blocking, int *locked_out) @@ -568,7 +705,7 @@ tor_lockfile_lock(const char *filename, int blocking, int *locked_out) #ifdef WIN32 _lseek(fd, 0, SEEK_SET); if (_locking(fd, blocking ? _LK_LOCK : _LK_NBLCK, 1) < 0) { - if (errno != EDEADLOCK) + if (errno != EACCES && errno != EDEADLOCK) log_warn(LD_FS,"Couldn't lock \"%s\": %s", filename, strerror(errno)); else *locked_out = 1; @@ -635,7 +772,8 @@ tor_lockfile_unlock(tor_lockfile_t *lockfile) tor_free(lockfile); } -/* Some old versions of Unix didn't define constants for these values, +/** @{ */ +/** Some old versions of Unix didn't define constants for these values, * and instead expect you to say 0, 1, or 2. */ #ifndef SEEK_CUR #define SEEK_CUR 1 @@ -643,6 +781,7 @@ tor_lockfile_unlock(tor_lockfile_t *lockfile) #ifndef SEEK_END #define SEEK_END 2 #endif +/** @} */ /** Return the position of <b>fd</b> with respect to the start of the file. */ off_t @@ -682,6 +821,7 @@ static int n_sockets_open = 0; /** Mutex to protect open_sockets, max_socket, and n_sockets_open. */ static tor_mutex_t *socket_accounting_mutex = NULL; +/** Helper: acquire the socket accounting lock. */ static INLINE void socket_accounting_lock(void) { @@ -690,6 +830,7 @@ socket_accounting_lock(void) tor_mutex_acquire(socket_accounting_mutex); } +/** Helper: release the socket accounting lock. */ static INLINE void socket_accounting_unlock(void) { @@ -748,6 +889,7 @@ tor_close_socket(int s) return r; } +/** @{ */ #ifdef DEBUG_SOCKET_COUNTING /** Helper: if DEBUG_SOCKET_COUNTING is enabled, remember that <b>s</b> is * now an open socket. */ @@ -772,6 +914,7 @@ mark_socket_open(int s) #else #define mark_socket_open(s) STMT_NIL #endif +/** @} */ /** As socket(), but counts the number of open sockets. */ int @@ -959,6 +1102,8 @@ tor_socketpair(int family, int type, int protocol, int fd[2]) #endif } +/** Number of extra file descriptors to keep in reserve beyond those that we + * tell Tor it's allowed to use. */ #define ULIMIT_BUFFER 32 /* keep 32 extra fd's beyond _ConnLimit */ /** Learn the maximum allowed number of file descriptors. (Some systems @@ -981,9 +1126,6 @@ set_max_file_descriptors(rlim_t limit, int *max_out) #if defined(CYGWIN) || defined(__CYGWIN__) const char *platform = "Cygwin"; const unsigned long MAX_CONNECTIONS = 3200; -#elif defined(IPHONE) - const char *platform = "iPhone"; - const unsigned long MAX_CONNECTIONS = 9999; #elif defined(MS_WINDOWS) const char *platform = "Windows"; const unsigned long MAX_CONNECTIONS = 15000; @@ -1074,6 +1216,7 @@ set_max_file_descriptors(rlim_t limit, int *max_out) static int log_credential_status(void) { +/** Log level to use when describing non-error UID/GID status. */ #define CREDENTIAL_LOG_LEVEL LOG_INFO /* Real, effective and saved UIDs */ uid_t ruid, euid, suid; @@ -1555,6 +1698,30 @@ tor_lookup_hostname(const char *name, uint32_t *addr) return -1; } +/** Initialize the insecure libc RNG. */ +void +tor_init_weak_random(unsigned seed) +{ +#ifdef MS_WINDOWS + srand(seed); +#else + srandom(seed); +#endif +} + +/** Return a randomly chosen value in the range 0..TOR_RAND_MAX. This + * entropy will not be cryptographically strong; do not rely on it + * for anything an adversary should not be able to predict. */ +long +tor_weak_random(void) +{ +#ifdef MS_WINDOWS + return rand(); +#else + return random(); +#endif +} + /** Hold the result of our call to <b>uname</b>. */ static char uname_result[256]; /** True iff uname_result is set. */ @@ -1580,13 +1747,14 @@ get_uname(void) #ifdef MS_WINDOWS OSVERSIONINFOEX info; int i; - unsigned int leftover_mask; const char *plat = NULL; const char *extra = NULL; + char acsd[MAX_PATH] = {0}; static struct { unsigned major; unsigned minor; const char *version; } win_version_table[] = { - { 6, 0, "Windows \"Longhorn\"" }, + { 6, 1, "Windows 7" }, + { 6, 0, "Windows Vista" }, { 5, 2, "Windows Server 2003" }, { 5, 1, "Windows XP" }, { 5, 0, "Windows 2000" }, @@ -1597,25 +1765,6 @@ get_uname(void) { 3, 51, "Windows NT 3.51" }, { 0, 0, NULL } }; -#ifdef VER_SUITE_BACKOFFICE - static struct { - unsigned int mask; const char *str; - } win_mask_table[] = { - { VER_SUITE_BACKOFFICE, " {backoffice}" }, - { VER_SUITE_BLADE, " {\"blade\" (2003, web edition)}" }, - { VER_SUITE_DATACENTER, " {datacenter}" }, - { VER_SUITE_ENTERPRISE, " {enterprise}" }, - { VER_SUITE_EMBEDDEDNT, " {embedded}" }, - { VER_SUITE_PERSONAL, " {personal}" }, - { VER_SUITE_SINGLEUSERTS, - " {terminal services, single user}" }, - { VER_SUITE_SMALLBUSINESS, " {small business}" }, - { VER_SUITE_SMALLBUSINESS_RESTRICTED, - " {small business, restricted}" }, - { VER_SUITE_TERMINAL, " {terminal services}" }, - { 0, NULL }, - }; -#endif memset(&info, 0, sizeof(info)); info.dwOSVersionInfoSize = sizeof(info); if (! GetVersionEx((LPOSVERSIONINFO)&info)) { @@ -1624,14 +1773,19 @@ get_uname(void) uname_result_is_set = 1; return uname_result; } +#ifdef UNICODE + wcstombs(acsd, info.szCSDVersion, MAX_PATH); +#else + strlcpy(acsd, info.szCSDVersion, sizeof(acsd)); +#endif if (info.dwMajorVersion == 4 && info.dwMinorVersion == 0) { if (info.dwPlatformId == VER_PLATFORM_WIN32_NT) plat = "Windows NT 4.0"; else plat = "Windows 95"; - if (info.szCSDVersion[1] == 'B') + if (acsd[1] == 'B') extra = "OSR2 (B)"; - else if (info.szCSDVersion[1] == 'C') + else if (acsd[1] == 'C') extra = "OSR2 (C)"; } else { for (i=0; win_version_table[i].major>0; ++i) { @@ -1643,29 +1797,30 @@ get_uname(void) } } if (plat && !strcmp(plat, "Windows 98")) { - if (info.szCSDVersion[1] == 'A') + if (acsd[1] == 'A') extra = "SE (A)"; - else if (info.szCSDVersion[1] == 'B') + else if (acsd[1] == 'B') extra = "SE (B)"; } if (plat) { if (!extra) - extra = info.szCSDVersion; + extra = acsd; tor_snprintf(uname_result, sizeof(uname_result), "%s %s", plat, extra); } else { if (info.dwMajorVersion > 6 || - (info.dwMajorVersion==6 && info.dwMinorVersion>0)) + (info.dwMajorVersion==6 && info.dwMinorVersion>1)) tor_snprintf(uname_result, sizeof(uname_result), "Very recent version of Windows [major=%d,minor=%d] %s", (int)info.dwMajorVersion,(int)info.dwMinorVersion, - info.szCSDVersion); + acsd); else tor_snprintf(uname_result, sizeof(uname_result), "Unrecognized version of Windows [major=%d,minor=%d] %s", (int)info.dwMajorVersion,(int)info.dwMinorVersion, - info.szCSDVersion); + acsd); } +#if !defined (WINCE) #ifdef VER_SUITE_BACKOFFICE if (info.wProductType == VER_NT_DOMAIN_CONTROLLER) { strlcat(uname_result, " [domain controller]", sizeof(uname_result)); @@ -1674,18 +1829,7 @@ get_uname(void) } else if (info.wProductType == VER_NT_WORKSTATION) { strlcat(uname_result, " [workstation]", sizeof(uname_result)); } - leftover_mask = info.wSuiteMask; - for (i = 0; win_mask_table[i].mask; ++i) { - if (info.wSuiteMask & win_mask_table[i].mask) { - strlcat(uname_result, win_mask_table[i].str, sizeof(uname_result)); - leftover_mask &= ~win_mask_table[i].mask; - } - } - if (leftover_mask) { - size_t len = strlen(uname_result); - tor_snprintf(uname_result+len, sizeof(uname_result)-len, - " {0x%x}", info.wSuiteMask); - } +#endif #endif #else strlcpy(uname_result, "Unknown platform", sizeof(uname_result)); @@ -1795,7 +1939,6 @@ spawn_exit(void) * call _exit, not exit, from child processes. */ _exit(0); #endif - } /** Set *timeval to the current time of day. On error, log and terminate. @@ -1815,8 +1958,15 @@ tor_gettimeofday(struct timeval *timeval) uint64_t ft_64; FILETIME ft_ft; } ft; +#if defined (WINCE) + /* wince do not have GetSystemTimeAsFileTime */ + SYSTEMTIME stime; + GetSystemTime(&stime); + SystemTimeToFileTime(&stime,&ft.ft_ft); +#else /* number of 100-nsec units since Jan 1, 1601 */ GetSystemTimeAsFileTime(&ft.ft_ft); +#endif if (ft.ft_64 < EPOCH_BIAS) { log_err(LD_GENERAL,"System time is before 1970; failing."); exit(1); @@ -1848,8 +1998,87 @@ tor_gettimeofday(struct timeval *timeval) #define TIME_FNS_NEED_LOCKS #endif -#ifndef HAVE_LOCALTIME_R -#ifdef TIME_FNS_NEED_LOCKS +static struct tm * +correct_tm(int islocal, const time_t *timep, struct tm *resultbuf, + struct tm *r) +{ + const char *outcome; + + if (PREDICT_LIKELY(r)) { + if (r->tm_year > 8099) { /* We can't strftime dates after 9999 CE. */ + r->tm_year = 8099; + r->tm_mon = 11; + r->tm_mday = 31; + r->tm_yday = 365; + r->tm_hour = 23; + r->tm_min = 59; + r->tm_sec = 59; + } + return r; + } + + /* If we get here, gmtime or localtime returned NULL. It might have done + * this because of overrun or underrun, or it might have done it because of + * some other weird issue. */ + if (timep) { + if (*timep < 0) { + r = resultbuf; + r->tm_year = 70; /* 1970 CE */ + r->tm_mon = 0; + r->tm_mday = 1; + r->tm_yday = 1; + r->tm_hour = 0; + r->tm_min = 0 ; + r->tm_sec = 0; + outcome = "Rounding up to 1970"; + goto done; + } else if (*timep >= INT32_MAX) { + /* Rounding down to INT32_MAX isn't so great, but keep in mind that we + * only do it if gmtime/localtime tells us NULL. */ + r = resultbuf; + r->tm_year = 137; /* 2037 CE */ + r->tm_mon = 11; + r->tm_mday = 31; + r->tm_yday = 365; + r->tm_hour = 23; + r->tm_min = 59; + r->tm_sec = 59; + outcome = "Rounding down to 2037"; + goto done; + } + } + + /* If we get here, then gmtime/localtime failed without getting an extreme + * value for *timep */ + + tor_fragile_assert(); + r = resultbuf; + memset(resultbuf, 0, sizeof(struct tm)); + outcome="can't recover"; + done: + log_warn(LD_BUG, "%s("I64_FORMAT") failed with error %s: %s", + islocal?"localtime":"gmtime", + timep?I64_PRINTF_ARG(*timep):0, + strerror(errno), + outcome); + return r; +} + +/** @{ */ +/** As localtime_r, but defined for platforms that don't have it: + * + * Convert *<b>timep</b> to a struct tm in local time, and store the value in + * *<b>result</b>. Return the result on success, or NULL on failure. + */ +#ifdef HAVE_LOCALTIME_R +struct tm * +tor_localtime_r(const time_t *timep, struct tm *result) +{ + struct tm *r; + r = localtime_r(timep, result); + return correct_tm(1, timep, result, r); +} +#elif defined(TIME_FNS_NEED_LOCKS) struct tm * tor_localtime_r(const time_t *timep, struct tm *result) { @@ -1859,9 +2088,10 @@ tor_localtime_r(const time_t *timep, struct tm *result) tor_assert(result); tor_mutex_acquire(m); r = localtime(timep); - memcpy(result, r, sizeof(struct tm)); + if (r) + memcpy(result, r, sizeof(struct tm)); tor_mutex_release(m); - return result; + return correct_tm(1, timep, result, r); } #else struct tm * @@ -1870,14 +2100,28 @@ tor_localtime_r(const time_t *timep, struct tm *result) struct tm *r; tor_assert(result); r = localtime(timep); - memcpy(result, r, sizeof(struct tm)); - return result; + if (r) + memcpy(result, r, sizeof(struct tm)); + return correct_tm(1, timep, result, r); } #endif -#endif +/** @} */ -#ifndef HAVE_GMTIME_R -#ifdef TIME_FNS_NEED_LOCKS +/** @{ */ +/** As gmtimee_r, but defined for platforms that don't have it: + * + * Convert *<b>timep</b> to a struct tm in UTC, and store the value in + * *<b>result</b>. Return the result on success, or NULL on failure. + */ +#ifdef HAVE_GMTIME_R +struct tm * +tor_gmtime_r(const time_t *timep, struct tm *result) +{ + struct tm *r; + r = gmtime_r(timep, result); + return correct_tm(0, timep, result, r); +} +#elif defined(TIME_FNS_NEED_LOCKS) struct tm * tor_gmtime_r(const time_t *timep, struct tm *result) { @@ -1887,9 +2131,10 @@ tor_gmtime_r(const time_t *timep, struct tm *result) tor_assert(result); tor_mutex_acquire(m); r = gmtime(timep); - memcpy(result, r, sizeof(struct tm)); + if (r) + memcpy(result, r, sizeof(struct tm)); tor_mutex_release(m); - return result; + return correct_tm(0, timep, result, r); } #else struct tm * @@ -1898,11 +2143,11 @@ tor_gmtime_r(const time_t *timep, struct tm *result) struct tm *r; tor_assert(result); r = gmtime(timep); - memcpy(result, r, sizeof(struct tm)); - return result; + if (r) + memcpy(result, r, sizeof(struct tm)); + return correct_tm(0, timep, result, r); } #endif -#endif #if defined(USE_WIN32_THREADS) void @@ -2016,6 +2261,8 @@ tor_mutex_new(void) void tor_mutex_free(tor_mutex_t *m) { + if (!m) + return; tor_mutex_uninit(m); tor_free(m); } @@ -2043,7 +2290,8 @@ tor_cond_new(void) void tor_cond_free(tor_cond_t *cond) { - tor_assert(cond); + if (!cond) + return; if (pthread_cond_destroy(&cond->cond)) { log_warn(LD_GENERAL,"Error freeing condition: %s", strerror(errno)); return; @@ -2100,7 +2348,8 @@ tor_cond_new(void) void tor_cond_free(tor_cond_t *cond) { - tor_assert(cond); + if (!cond) + return; DeleteCriticalSection(&cond->mutex); /* XXXX notify? */ smartlist_free(cond->events); @@ -2176,6 +2425,89 @@ tor_threads_init(void) } #endif +#if defined(HAVE_MLOCKALL) && HAVE_DECL_MLOCKALL && defined(RLIMIT_MEMLOCK) +/** Attempt to raise the current and max rlimit to infinity for our process. + * This only needs to be done once and can probably only be done when we have + * not already dropped privileges. + */ +static int +tor_set_max_memlock(void) +{ + /* Future consideration for Windows is probably SetProcessWorkingSetSize + * This is similar to setting the memory rlimit of RLIMIT_MEMLOCK + * http://msdn.microsoft.com/en-us/library/ms686234(VS.85).aspx + */ + + struct rlimit limit; + + /* RLIM_INFINITY is -1 on some platforms. */ + limit.rlim_cur = RLIM_INFINITY; + limit.rlim_max = RLIM_INFINITY; + + if (setrlimit(RLIMIT_MEMLOCK, &limit) == -1) { + if (errno == EPERM) { + log_warn(LD_GENERAL, "You appear to lack permissions to change memory " + "limits. Are you root?"); + } + log_warn(LD_GENERAL, "Unable to raise RLIMIT_MEMLOCK: %s", + strerror(errno)); + return -1; + } + + return 0; +} +#endif + +/** Attempt to lock all current and all future memory pages. + * This should only be called once and while we're privileged. + * Like mlockall() we return 0 when we're successful and -1 when we're not. + * Unlike mlockall() we return 1 if we've already attempted to lock memory. + */ +int +tor_mlockall(void) +{ + static int memory_lock_attempted = 0; + + if (memory_lock_attempted) { + return 1; + } + + memory_lock_attempted = 1; + + /* + * Future consideration for Windows may be VirtualLock + * VirtualLock appears to implement mlock() but not mlockall() + * + * http://msdn.microsoft.com/en-us/library/aa366895(VS.85).aspx + */ + +#if defined(HAVE_MLOCKALL) && HAVE_DECL_MLOCKALL && defined(RLIMIT_MEMLOCK) + if (tor_set_max_memlock() == 0) { + log_debug(LD_GENERAL, "RLIMIT_MEMLOCK is now set to RLIM_INFINITY."); + } + + if (mlockall(MCL_CURRENT|MCL_FUTURE) == 0) { + log_info(LD_GENERAL, "Insecure OS paging is effectively disabled."); + return 0; + } else { + if (errno == ENOSYS) { + /* Apple - it's 2009! I'm looking at you. Grrr. */ + log_notice(LD_GENERAL, "It appears that mlockall() is not available on " + "your platform."); + } else if (errno == EPERM) { + log_notice(LD_GENERAL, "It appears that you lack the permissions to " + "lock memory. Are you root?"); + } + log_notice(LD_GENERAL, "Unable to lock all current and future memory " + "pages: %s", strerror(errno)); + return -1; + } +#else + log_warn(LD_GENERAL, "Unable to lock memory pages. mlockall() unsupported?"); + return -1; +#endif +} + /** Identity of the "main" thread */ static unsigned long main_thread_id = -1; @@ -2327,20 +2659,26 @@ network_init(void) char * format_win32_error(DWORD err) { - LPVOID str = NULL; + TCHAR *str = NULL; char *result; /* Somebody once decided that this interface was better than strerror(). */ - FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | + FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, NULL, err, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), - (LPTSTR) &str, + (LPVOID)&str, 0, NULL); if (str) { - result = tor_strdup((char*)str); +#ifdef UNICODE + char abuf[1024] = {0}; + wcstombs(abuf,str,1024); + result = tor_strdup(abuf); +#else + result = tor_strdup(str); +#endif LocalFree(str); /* LocalFree != free() */ } else { result = tor_strdup("<unformattable error>"); diff --git a/src/common/compat.h b/src/common/compat.h index c7ed32b0d8..af795ffba9 100644 --- a/src/common/compat.h +++ b/src/common/compat.h @@ -51,6 +51,22 @@ #include <netinet6/in6.h> #endif +#if defined (WINCE) +#include <fcntl.h> +#include <io.h> +#include <math.h> +#include <projects.h> +#define snprintf _snprintf +/* this is not exported as W .... */ +#define SHGetPathFromIDListW SHGetPathFromIDList +/* wcecompat has vasprintf */ +#define HAVE_VASPRINTF +/* no service here */ +#ifdef NT_SERVICE +#undef NT_SERVICE +#endif +#endif // WINCE + #ifndef NULL_REP_IS_ZERO_BYTES #error "It seems your platform does not represent NULL as zero. We can't cope." #endif @@ -177,8 +193,8 @@ extern INLINE double U64_TO_DBL(uint64_t x) { /* ===== String compatibility */ #ifdef MS_WINDOWS /* Windows names string functions differently from most other platforms. */ -#define strncasecmp strnicmp -#define strcasecmp stricmp +#define strncasecmp _strnicmp +#define strcasecmp _stricmp #endif #ifndef HAVE_STRLCAT size_t strlcat(char *dst, const char *src, size_t siz) ATTR_NONNULL((1,2)); @@ -196,18 +212,26 @@ size_t strlcpy(char *dst, const char *src, size_t siz) ATTR_NONNULL((1,2)); #define U64_SCANF_ARG(a) (a) /** Expands to a literal uint64_t-typed constant for the value <b>n</b>. */ #define U64_LITERAL(n) (n ## ui64) +#define I64_PRINTF_ARG(a) (a) +#define I64_SCANF_ARG(a) (a) +#define I64_LITERAL(n) (n ## i64) #else #define U64_PRINTF_ARG(a) ((long long unsigned int)(a)) #define U64_SCANF_ARG(a) ((long long unsigned int*)(a)) #define U64_LITERAL(n) (n ## llu) +#define I64_PRINTF_ARG(a) ((long long signed int)(a)) +#define I64_SCANF_ARG(a) ((long long signed int*)(a)) +#define I64_LITERAL(n) (n ## ll) #endif #if defined(_MSC_VER) || defined(__MINGW32__) || defined(__MINGW64__) /** The formatting string used to put a uint64_t value in a printf() or * scanf() function. See also U64_PRINTF_ARG and U64_SCANF_ARG. */ #define U64_FORMAT "%I64u" +#define I64_FORMAT "%I64d" #else #define U64_FORMAT "%llu" +#define I64_FORMAT "%lld" #endif /** Represents an mmaped file. Allocated via tor_mmap_file; freed with @@ -235,6 +259,10 @@ int tor_snprintf(char *str, size_t size, const char *format, ...) int tor_vsnprintf(char *str, size_t size, const char *format, va_list args) ATTR_NONNULL((1,3)); +int tor_asprintf(char **strp, const char *fmt, ...) + CHECK_PRINTF(2,3); +int tor_vasprintf(char **strp, const char *fmt, va_list args); + const void *tor_memmem(const void *haystack, size_t hlen, const void *needle, size_t nlen) ATTR_PURE ATTR_NONNULL((1,3)); static const void *tor_memstr(const void *haystack, size_t hlen, @@ -267,6 +295,13 @@ extern const char TOR_TOLOWER_TABLE[]; #define TOR_TOLOWER(c) (TOR_TOLOWER_TABLE[(uint8_t)c]) #define TOR_TOUPPER(c) (TOR_TOUPPER_TABLE[(uint8_t)c]) +char *tor_strtok_r_impl(char *str, const char *sep, char **lasts); +#ifdef HAVE_STRTOK_R +#define tor_strtok_r(str, sep, lasts) strtok_r(str, sep, lasts) +#else +#define tor_strtok_r(str, sep, lasts) tor_strtok_r_impl(str, sep, lasts) +#endif + #ifdef MS_WINDOWS #define _SHORT_FILE_ (tor_fix_source_file(__FILE__)) const char *tor_fix_source_file(const char *fname); @@ -277,6 +312,7 @@ const char *tor_fix_source_file(const char *fname); /* ===== Time compatibility */ #if !defined(HAVE_GETTIMEOFDAY) && !defined(HAVE_STRUCT_TIMEVAL_TV_SEC) +/** Implementation of timeval for platforms that don't have it. */ struct timeval { time_t tv_sec; unsigned int tv_usec; @@ -285,16 +321,49 @@ struct timeval { void tor_gettimeofday(struct timeval *timeval); -#ifdef HAVE_LOCALTIME_R -#define tor_localtime_r localtime_r -#else struct tm *tor_localtime_r(const time_t *timep, struct tm *result); -#endif - -#ifdef HAVE_GMTIME_R -#define tor_gmtime_r gmtime_r -#else struct tm *tor_gmtime_r(const time_t *timep, struct tm *result); + +#ifndef timeradd +/** Replacement for timeradd on platforms that do not have it: sets tvout to + * the sum of tv1 and tv2. */ +#define timeradd(tv1,tv2,tvout) \ + do { \ + (tvout)->tv_sec = (tv1)->tv_sec + (tv2)->tv_sec; \ + (tvout)->tv_usec = (tv2)->tv_usec + (tv2)->tv_usec; \ + if ((tvout)->tv_usec >= 1000000) { \ + (tvout)->tv_usec -= 1000000; \ + (tvout)->tv_sec++; \ + } \ + } while (0) +#endif + +#ifndef timersub +/** Replacement for timersub on platforms that do not have it: sets tvout to + * tv1 minus tv2. */ +#define timersub(tv1,tv2,tvout) \ + do { \ + (tvout)->tv_sec = (tv1)->tv_sec - (tv2)->tv_sec; \ + (tvout)->tv_usec = (tv2)->tv_usec - (tv2)->tv_usec; \ + if ((tvout)->tv_usec < 0) { \ + (tvout)->tv_usec += 1000000; \ + (tvout)->tv_sec--; \ + } \ + } while (0) +#endif + +#ifndef timercmp +/** Replacement for timersub on platforms that do not have it: returns true + * iff the relational operator "op" makes the expression tv1 op tv2 true. + * + * Note that while this definition should work for all boolean opeators, some + * platforms' native timercmp definitions do not support >=, <=, or ==. So + * don't use those. + */ +#define timercmp(tv1,tv2,op) \ + (((tv1)->tv_sec == (tv2)->tv_sec) ? \ + ((tv1)->tv_usec op (tv2)->tv_usec) : \ + ((tv1)->tv_sec op (tv2)->tv_sec)) #endif /* ===== File compatibility */ @@ -329,9 +398,9 @@ int get_n_open_sockets(void); #define tor_socket_send(s, buf, len, flags) send(s, buf, len, flags) #define tor_socket_recv(s, buf, len, flags) recv(s, buf, len, flags) -/* Define struct in6_addr on platforms that do not have it. Generally, - * these platforms are ones without IPv6 support, but we want to have - * a working in6_addr there anyway, so we can use it to parse IPv6 +/** Implementatino of struct in6_addr for platforms that do not have it. + * Generally, these platforms are ones without IPv6 support, but we want to + * have a working in6_addr there anyway, so we can use it to parse IPv6 * addresses. */ #if !defined(HAVE_STRUCT_IN6_ADDR) struct in6_addr @@ -347,9 +416,10 @@ struct in6_addr }; #endif +/** @{ */ +/** Many BSD variants seem not to define these. */ #if defined(__APPLE__) || defined(__darwin__) || defined(__FreeBSD__) \ || defined(__NetBSD__) || defined(__OpenBSD__) -/* Many BSD variants seem not to define these. */ #ifndef s6_addr16 #define s6_addr16 __u6_addr.__u6_addr16 #endif @@ -357,12 +427,15 @@ struct in6_addr #define s6_addr32 __u6_addr.__u6_addr32 #endif #endif +/** @} */ #ifndef HAVE_SA_FAMILY_T typedef uint16_t sa_family_t; #endif -/* Apparently, MS and Solaris don't define s6_addr16 or s6_addr32. */ +/** @{ */ +/** Apparently, MS and Solaris don't define s6_addr16 or s6_addr32; these + * macros get you a pointer to s6_addr32 or local equivalent. */ #ifdef HAVE_STRUCT_IN6_ADDR_S6_ADDR32 #define S6_ADDR32(x) ((uint32_t*)(x).s6_addr32) #else @@ -373,9 +446,10 @@ typedef uint16_t sa_family_t; #else #define S6_ADDR16(x) ((uint16_t*)((char*)&(x).s6_addr)) #endif +/** @} */ -/* Define struct sockaddr_in6 on platforms that do not have it. See notes - * on struct in6_addr. */ +/** Implementation of struct sockaddr_in6 on platforms that do not have + * it. See notes on struct in6_addr. */ #if !defined(HAVE_STRUCT_SOCKADDR_IN6) struct sockaddr_in6 { sa_family_t sin6_family; @@ -445,6 +519,11 @@ typedef enum { SOCKS5_ADDRESS_TYPE_NOT_SUPPORTED = 0x08, } socks5_reply_status_t; +/* ===== Insecure rng */ +void tor_init_weak_random(unsigned seed); +long tor_weak_random(void); +#define TOR_RAND_MAX (RAND_MAX) + /* ===== OS compatibility */ const char *get_uname(void); @@ -494,14 +573,20 @@ void spawn_exit(void) ATTR_NORETURN; /** A generic lock structure for multithreaded builds. */ typedef struct tor_mutex_t { #if defined(USE_WIN32_THREADS) + /** Windows-only: on windows, we implement locks with CRITICAL_SECTIONS. */ CRITICAL_SECTION mutex; #elif defined(USE_PTHREADS) + /** Pthreads-only: with pthreads, we implement locks with + * pthread_mutex_t. */ pthread_mutex_t mutex; #else + /** No-threads only: Dummy variable so that tor_mutex_t takes up space. */ int _unused; #endif } tor_mutex_t; +int tor_mlockall(void); + #ifdef TOR_IS_MULTITHREADED tor_mutex_t *tor_mutex_new(void); void tor_mutex_init(tor_mutex_t *m); @@ -536,6 +621,19 @@ void tor_cond_signal_all(tor_cond_t *cond); #endif #endif +/** Macros for MIN/MAX. Never use these when the arguments could have + * side-effects. + * {With GCC extensions we could probably define a safer MIN/MAX. But + * depending on that safety would be dangerous, since not every platform + * has it.} + **/ +#ifndef MAX +#define MAX(a,b) ( ((a)<(b)) ? (b) : (a) ) +#endif +#ifndef MIN +#define MIN(a,b) ( ((a)>(b)) ? (b) : (a) ) +#endif + /* Platform-specific helpers. */ #ifdef MS_WINDOWS char *format_win32_error(DWORD err); diff --git a/src/common/compat_libevent.c b/src/common/compat_libevent.c new file mode 100644 index 0000000000..3ad9be145d --- /dev/null +++ b/src/common/compat_libevent.c @@ -0,0 +1,554 @@ +/* Copyright (c) 2009-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file compat_libevent.c + * \brief Wrappers to handle porting between different versions of libevent. + * + * In an ideal world, we'd just use Libevent 2.0 from now on. But as of June + * 2009, Libevent 2.0 is still in alpha, and we will have old versions of + * Libevent for the forseeable future. + **/ + +#include "orconfig.h" +#include "compat.h" +#include "compat_libevent.h" + +#include "util.h" +#include "torlog.h" + +#ifdef HAVE_EVENT2_EVENT_H +#include <event2/event.h> +#else +#include <event.h> +#endif + +/** A number representing a version of Libevent. + + This is a 4-byte number, with the first three bytes representing the + major, minor, and patchlevel respectively of the library. The fourth + byte is unused. + + This is equivalent to the format of LIBEVENT_VERSION_NUMBER on Libevent + 2.0.1 or later. For versions of Libevent before 1.4.0, which followed the + format of "1.0, 1.0a, 1.0b", we define 1.0 to be equivalent to 1.0.0, 1.0a + to be equivalent to 1.0.1, and so on. +*/ +typedef uint32_t le_version_t; + +/** @{ */ +/** Macros: returns the number of a libevent version as a le_version_t */ +#define V(major, minor, patch) \ + (((major) << 24) | ((minor) << 16) | ((patch) << 8)) +#define V_OLD(major, minor, patch) \ + V((major), (minor), (patch)-'a'+1) +/** @} */ + +/** Represetns a version of libevent so old we can't figure out what version + * it is. */ +#define LE_OLD V(0,0,0) +/** Represents a version of libevent so weird we can't figure out what version + * it it. */ +#define LE_OTHER V(0,0,99) + +static le_version_t tor_get_libevent_version(const char **v_out); + +#ifdef HAVE_EVENT_SET_LOG_CALLBACK +/** A string which, if it appears in a libevent log, should be ignored. */ +static const char *suppress_msg = NULL; +/** Callback function passed to event_set_log() so we can intercept + * log messages from libevent. */ +static void +libevent_logging_callback(int severity, const char *msg) +{ + char buf[1024]; + size_t n; + if (suppress_msg && strstr(msg, suppress_msg)) + return; + n = strlcpy(buf, msg, sizeof(buf)); + if (n && n < sizeof(buf) && buf[n-1] == '\n') { + buf[n-1] = '\0'; + } + switch (severity) { + case _EVENT_LOG_DEBUG: + log(LOG_DEBUG, LD_NOCB|LD_NET, "Message from libevent: %s", buf); + break; + case _EVENT_LOG_MSG: + log(LOG_INFO, LD_NOCB|LD_NET, "Message from libevent: %s", buf); + break; + case _EVENT_LOG_WARN: + log(LOG_WARN, LD_NOCB|LD_GENERAL, "Warning from libevent: %s", buf); + break; + case _EVENT_LOG_ERR: + log(LOG_ERR, LD_NOCB|LD_GENERAL, "Error from libevent: %s", buf); + break; + default: + log(LOG_WARN, LD_NOCB|LD_GENERAL, "Message [%d] from libevent: %s", + severity, buf); + break; + } +} +/** Set hook to intercept log messages from libevent. */ +void +configure_libevent_logging(void) +{ + event_set_log_callback(libevent_logging_callback); +} +/** Ignore any libevent log message that contains <b>msg</b>. */ +void +suppress_libevent_log_msg(const char *msg) +{ + suppress_msg = msg; +} +#else +void +configure_libevent_logging(void) +{ +} +void +suppress_libevent_log_msg(const char *msg) +{ + (void)msg; +} +#endif + +#ifndef HAVE_EVENT2_EVENT_H +/** Work-alike replacement for event_new() on pre-Libevent-2.0 systems. */ +struct event * +tor_event_new(struct event_base *base, int sock, short what, + void (*cb)(int, short, void *), void *arg) +{ + struct event *e = tor_malloc_zero(sizeof(struct event)); + event_set(e, sock, what, cb, arg); + if (! base) + base = tor_libevent_get_base(); + event_base_set(base, e); + return e; +} +/** Work-alike replacement for evtimer_new() on pre-Libevent-2.0 systems. */ +struct event * +tor_evtimer_new(struct event_base *base, + void (*cb)(int, short, void *), void *arg) +{ + return tor_event_new(base, -1, 0, cb, arg); +} +/** Work-alike replacement for evsignal_new() on pre-Libevent-2.0 systems. */ +struct event * +tor_evsignal_new(struct event_base * base, int sig, + void (*cb)(int, short, void *), void *arg) +{ + return tor_event_new(base, sig, EV_SIGNAL|EV_PERSIST, cb, arg); +} +/** Work-alike replacement for event_free() on pre-Libevent-2.0 systems. */ +void +tor_event_free(struct event *ev) +{ + event_del(ev); + tor_free(ev); +} +#endif + +/** Global event base for use by the main thread. */ +struct event_base *the_event_base = NULL; + +/* This is what passes for version detection on OSX. We set + * MACOSX_KQUEUE_IS_BROKEN to true iff we're on a version of OSX before + * 10.4.0 (aka 1040). */ +#ifdef __APPLE__ +#ifdef __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ +#define MACOSX_KQUEUE_IS_BROKEN \ + (__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 1040) +#else +#define MACOSX_KQUEUE_IS_BROKEN 0 +#endif +#endif + +/** Initialize the Libevent library and set up the event base. */ +void +tor_libevent_initialize(void) +{ + tor_assert(the_event_base == NULL); + +#ifdef __APPLE__ + if (MACOSX_KQUEUE_IS_BROKEN || + tor_get_libevent_version(NULL) < V_OLD(1,1,'b')) { + setenv("EVENT_NOKQUEUE","1",1); + } +#endif + +#ifdef HAVE_EVENT2_EVENT_H + the_event_base = event_base_new(); +#else + the_event_base = event_init(); +#endif + +#if defined(HAVE_EVENT_GET_VERSION) && defined(HAVE_EVENT_GET_METHOD) + /* Making this a NOTICE for now so we can link bugs to a libevent versions + * or methods better. */ + log(LOG_NOTICE, LD_GENERAL, + "Initialized libevent version %s using method %s. Good.", + event_get_version(), tor_libevent_get_method()); +#else + log(LOG_NOTICE, LD_GENERAL, + "Initialized old libevent (version 1.0b or earlier)."); + log(LOG_WARN, LD_GENERAL, + "You have a *VERY* old version of libevent. It is likely to be buggy; " + "please build Tor with a more recent version."); +#endif +} + +/** Return the current Libevent event base that we're set up to use. */ +struct event_base * +tor_libevent_get_base(void) +{ + return the_event_base; +} + +#ifndef HAVE_EVENT_BASE_LOOPEXIT +/** Replacement for event_base_loopexit on some very old versions of Libevent + * that we are not yet brave enough to deprecate. */ +int +tor_event_base_loopexit(struct event_base *base, struct timeval *tv) +{ + tor_assert(base == the_event_base); + return event_loopexit(tv); +} +#endif + +/** Return the name of the Libevent backend we're using. */ +const char * +tor_libevent_get_method(void) +{ +#ifdef HAVE_EVENT2_EVENT_H + return event_base_get_method(the_event_base); +#elif defined(HAVE_EVENT_GET_METHOD) + return event_get_method(); +#else + return "<unknown>"; +#endif +} + +/** Return the le_version_t for the version of libevent specified in the + * string <b>v</b>. If the version is very new or uses an unrecognized + * version, format, return LE_OTHER. */ +static le_version_t +tor_decode_libevent_version(const char *v) +{ + unsigned major, minor, patchlevel; + char c, e, extra; + int fields; + + /* Try the new preferred "1.4.11-stable" format. + * Also accept "1.4.14b-stable". */ + fields = sscanf(v, "%u.%u.%u%c%c", &major, &minor, &patchlevel, &c, &e); + if (fields == 3 || + ((fields == 4 || fields == 5 ) && (c == '-' || c == '_')) || + (fields == 5 && TOR_ISALPHA(c) && (e == '-' || e == '_'))) { + return V(major,minor,patchlevel); + } + + /* Try the old "1.3e" format. */ + fields = sscanf(v, "%u.%u%c%c", &major, &minor, &c, &extra); + if (fields == 3 && TOR_ISALPHA(c)) { + return V_OLD(major, minor, c); + } else if (fields == 2) { + return V(major, minor, 0); + } + + return LE_OTHER; +} + +/** Return an integer representing the binary interface of a Libevent library. + * Two different versions with different numbers are sure not to be binary + * compatible. Two different versions with the same numbers have a decent + * chance of binary compatibility.*/ +static int +le_versions_compatibility(le_version_t v) +{ + if (v == LE_OTHER) + return 0; + if (v < V_OLD(1,0,'c')) + return 1; + else if (v < V(1,4,0)) + return 2; + else if (v < V(1,4,99)) + return 3; + else if (v < V(2,0,1)) + return 4; + else /* Everything 2.0 and later should be compatible. */ + return 5; +} + +/** Return the version number of the currently running version of Libevent. + * See le_version_t for info on the format. + */ +static le_version_t +tor_get_libevent_version(const char **v_out) +{ + const char *v; + le_version_t r; +#if defined(HAVE_EVENT_GET_VERSION_NUMBER) + v = event_get_version(); + r = event_get_version_number(); +#elif defined (HAVE_EVENT_GET_VERSION) + v = event_get_version(); + r = tor_decode_libevent_version(v); +#else + v = "pre-1.0c"; + r = LE_OLD; +#endif + if (v_out) + *v_out = v; + return r; +} + +/** Return a string representation of the version of the currently running + * version of Libevent. */ +const char * +tor_libevent_get_version_str(void) +{ +#ifdef HAVE_EVENT_GET_VERSION + return event_get_version(); +#else + return "pre-1.0c"; +#endif +} + +/** + * Compare the current Libevent method and version to a list of versions + * which are known not to work. Warn the user as appropriate. + */ +void +tor_check_libevent_version(const char *m, int server, + const char **badness_out) +{ + int buggy = 0, iffy = 0, slow = 0, thread_unsafe = 0; + le_version_t version; + const char *v = NULL; + const char *badness = NULL; + const char *sad_os = ""; + + version = tor_get_libevent_version(&v); + + /* It would be better to disable known-buggy methods rather than warning + * about them. But the problem is that with older versions of Libevent, + * it's not trivial to get them to change their methods once they're + * initialized... and with newer versions of Libevent, they aren't actually + * broken. But we should revisit this if we ever find a post-1.4 version + * of Libevent where we need to disable a given method. */ + if (!strcmp(m, "kqueue")) { + if (version < V_OLD(1,1,'b')) + buggy = 1; + } else if (!strcmp(m, "epoll")) { + if (version < V(1,1,0)) + iffy = 1; + } else if (!strcmp(m, "poll")) { + if (version < V_OLD(1,0,'e')) + buggy = 1; + if (version < V(1,1,0)) + slow = 1; + } else if (!strcmp(m, "select")) { + if (version < V(1,1,0)) + slow = 1; + } else if (!strcmp(m, "win32")) { + if (version < V_OLD(1,1,'b')) + buggy = 1; + } + + /* Libevent versions before 1.3b do very badly on operating systems with + * user-space threading implementations. */ +#if defined(__OpenBSD__) || defined(__FreeBSD__) || defined(__NetBSD__) + if (server && version < V_OLD(1,3,'b')) { + thread_unsafe = 1; + sad_os = "BSD variants"; + } +#elif defined(__APPLE__) || defined(__darwin__) + if (server && version < V_OLD(1,3,'b')) { + thread_unsafe = 1; + sad_os = "Mac OS X"; + } +#endif + + if (thread_unsafe) { + log(LOG_WARN, LD_GENERAL, + "Libevent version %s often crashes when running a Tor server with %s. " + "Please use the latest version of libevent (1.3b or later)",v,sad_os); + badness = "BROKEN"; + } else if (buggy) { + log(LOG_WARN, LD_GENERAL, + "There are serious bugs in using %s with libevent %s. " + "Please use the latest version of libevent.", m, v); + badness = "BROKEN"; + } else if (iffy) { + log(LOG_WARN, LD_GENERAL, + "There are minor bugs in using %s with libevent %s. " + "You may want to use the latest version of libevent.", m, v); + badness = "BUGGY"; + } else if (slow && server) { + log(LOG_WARN, LD_GENERAL, + "libevent %s can be very slow with %s. " + "When running a server, please use the latest version of libevent.", + v,m); + badness = "SLOW"; + } + + *badness_out = badness; +} + +#if defined(LIBEVENT_VERSION) +#define HEADER_VERSION LIBEVENT_VERSION +#elif defined(_EVENT_VERSION) +#define HEADER_VERSION _EVENT_VERSION +#endif + +/** See whether the headers we were built against differ from the library we + * linked against so much that we're likely to crash. If so, warn the + * user. */ +void +tor_check_libevent_header_compatibility(void) +{ + (void) le_versions_compatibility; + (void) tor_decode_libevent_version; + + /* In libevent versions before 2.0, it's hard to keep binary compatibility + * between upgrades, and unpleasant to detect when the version we compiled + * against is unlike the version we have linked against. Here's how. */ +#if defined(HEADER_VERSION) && defined(HAVE_EVENT_GET_VERSION) + /* We have a header-file version and a function-call version. Easy. */ + if (strcmp(HEADER_VERSION, event_get_version())) { + le_version_t v1, v2; + int compat1 = -1, compat2 = -1; + int verybad; + v1 = tor_decode_libevent_version(HEADER_VERSION); + v2 = tor_decode_libevent_version(event_get_version()); + compat1 = le_versions_compatibility(v1); + compat2 = le_versions_compatibility(v2); + + verybad = compat1 != compat2; + + log(verybad ? LOG_WARN : LOG_NOTICE, + LD_GENERAL, "We were compiled with headers from version %s " + "of Libevent, but we're using a Libevent library that says it's " + "version %s.", HEADER_VERSION, event_get_version()); + if (verybad) + log_warn(LD_GENERAL, "This will almost certainly make Tor crash."); + else + log_info(LD_GENERAL, "I think these versions are binary-compatible."); + } +#elif defined(HAVE_EVENT_GET_VERSION) + /* event_get_version but no _EVENT_VERSION. We might be in 1.4.0-beta or + earlier, where that's normal. To see whether we were compiled with an + earlier version, let's see whether the struct event defines MIN_HEAP_IDX. + */ +#ifdef HAVE_STRUCT_EVENT_MIN_HEAP_IDX + /* The header files are 1.4.0-beta or later. If the version is not + * 1.4.0-beta, we are incompatible. */ + { + if (strcmp(event_get_version(), "1.4.0-beta")) { + log_warn(LD_GENERAL, "It's a little hard to tell, but you seem to have " + "Libevent 1.4.0-beta header files, whereas you have linked " + "against Libevent %s. This will probably make Tor crash.", + event_get_version()); + } + } +#else + /* Our headers are 1.3e or earlier. If the library version is not 1.4.x or + later, we're probably fine. */ + { + const char *v = event_get_version(); + if ((v[0] == '1' && v[2] == '.' && v[3] > '3') || v[0] > '1') { + log_warn(LD_GENERAL, "It's a little hard to tell, but you seem to have " + "Libevent header file from 1.3e or earlier, whereas you have " + "linked against Libevent %s. This will probably make Tor " + "crash.", event_get_version()); + } + } +#endif + +#elif defined(HEADER_VERSION) +#warn "_EVENT_VERSION is defined but not get_event_version(): Libevent is odd." +#else + /* Your libevent is ancient. */ +#endif +} + +/* + If possible, we're going to try to use Libevent's periodic timer support, + since it does a pretty good job of making sure that periodic events get + called exactly M seconds apart, rather than starting each one exactly M + seconds after the time that the last one was run. + */ +#ifdef HAVE_EVENT2_EVENT_H +#define HAVE_PERIODIC +#define PERIODIC_FLAGS EV_PERSIST +#else +#define PERIODIC_FLAGS 0 +#endif + +/** Represents a timer that's run every N microseconds by Libevent. */ +struct periodic_timer_t { + /** Underlying event used to implement this periodic event. */ + struct event *ev; + /** The callback we'll be invoking whenever the event triggers */ + void (*cb)(struct periodic_timer_t *, void *); + /** User-supplied data for the callback */ + void *data; +#ifndef HAVE_PERIODIC + /** If Libevent doesn't know how to invoke events every N microseconds, + * we'll need to remember the timeout interval here. */ + struct timeval tv; +#endif +}; + +/** Libevent callback to implement a periodic event. */ +static void +periodic_timer_cb(evutil_socket_t fd, short what, void *arg) +{ + periodic_timer_t *timer = arg; + (void) what; + (void) fd; +#ifndef HAVE_PERIODIC + /** reschedule the event as needed. */ + event_add(timer->ev, &timer->tv); +#endif + timer->cb(timer, timer->data); +} + +/** Create and schedule a new timer that will run every <b>tv</b> in + * the event loop of <b>base</b>. When the timer fires, it will + * run the timer in <b>cb</b> with the user-supplied data in <b>data</b>. */ +periodic_timer_t * +periodic_timer_new(struct event_base *base, + const struct timeval *tv, + void (*cb)(periodic_timer_t *timer, void *data), + void *data) +{ + periodic_timer_t *timer; + tor_assert(base); + tor_assert(tv); + tor_assert(cb); + timer = tor_malloc_zero(sizeof(periodic_timer_t)); + if (!(timer->ev = tor_event_new(base, -1, PERIODIC_FLAGS, + periodic_timer_cb, timer))) { + tor_free(timer); + return NULL; + } + timer->cb = cb; + timer->data = data; +#ifndef HAVE_PERIODIC + memcpy(&timer->tv, tv, sizeof(struct timeval)); +#endif + event_add(timer->ev, (struct timeval *)tv); /*drop const for old libevent*/ + return timer; +} + +/** Stop and free a periodic timer */ +void +periodic_timer_free(periodic_timer_t *timer) +{ + if (!timer) + return; + tor_event_free(timer->ev); + tor_free(timer); +} + diff --git a/src/common/compat_libevent.h b/src/common/compat_libevent.h new file mode 100644 index 0000000000..fdf5e0a18f --- /dev/null +++ b/src/common/compat_libevent.h @@ -0,0 +1,65 @@ +/* Copyright (c) 2009, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef _TOR_COMPAT_LIBEVENT_H +#define _TOR_COMPAT_LIBEVENT_H + +#include "orconfig.h" + +struct event; +struct event_base; + +#ifdef HAVE_EVENT2_EVENT_H +#include <event2/util.h> +#else +#define evutil_socket_t int +#endif + +void configure_libevent_logging(void); +void suppress_libevent_log_msg(const char *msg); + +#ifdef HAVE_EVENT2_EVENT_H +#define tor_event_new event_new +#define tor_evtimer_new evtimer_new +#define tor_evsignal_new evsignal_new +#define tor_event_free event_free +#define tor_evdns_add_server_port(sock, tcp, cb, data) \ + evdns_add_server_port_with_base(tor_libevent_get_base(), \ + (sock),(tcp),(cb),(data)); + +#else +struct event *tor_event_new(struct event_base * base, evutil_socket_t sock, + short what, void (*cb)(evutil_socket_t, short, void *), void *arg); +struct event *tor_evtimer_new(struct event_base * base, + void (*cb)(evutil_socket_t, short, void *), void *arg); +struct event *tor_evsignal_new(struct event_base * base, int sig, + void (*cb)(evutil_socket_t, short, void *), void *arg); +void tor_event_free(struct event *ev); +#define tor_evdns_add_server_port evdns_add_server_port +#endif + +typedef struct periodic_timer_t periodic_timer_t; + +periodic_timer_t *periodic_timer_new(struct event_base *base, + const struct timeval *tv, + void (*cb)(periodic_timer_t *timer, void *data), + void *data); +void periodic_timer_free(periodic_timer_t *); + +#ifdef HAVE_EVENT_BASE_LOOPEXIT +#define tor_event_base_loopexit event_base_loopexit +#else +struct timeval; +int tor_event_base_loopexit(struct event_base *base, struct timeval *tv); +#endif + +void tor_libevent_initialize(void); +struct event_base *tor_libevent_get_base(void); +const char *tor_libevent_get_method(void); +void tor_check_libevent_version(const char *m, int server, + const char **badness_out); +void tor_check_libevent_header_compatibility(void); +const char *tor_libevent_get_version_str(void); + +#endif + diff --git a/src/common/container.c b/src/common/container.c index c741eb0206..5476c6008f 100644 --- a/src/common/container.c +++ b/src/common/container.c @@ -13,7 +13,7 @@ #include "compat.h" #include "util.h" -#include "log.h" +#include "torlog.h" #include "container.h" #include "crypto.h" @@ -44,7 +44,8 @@ smartlist_create(void) void smartlist_free(smartlist_t *sl) { - tor_assert(sl != NULL); + if (!sl) + return; tor_free(sl->list); tor_free(sl); } @@ -459,6 +460,42 @@ smartlist_sort(smartlist_t *sl, int (*compare)(const void **a, const void **b)) (int (*)(const void *,const void*))compare); } +/** Given a smartlist <b>sl</b> sorted with the function <b>compare</b>, + * return the most frequent member in the list. Break ties in favor of + * later elements. If the list is empty, return NULL. + */ +void * +smartlist_get_most_frequent(const smartlist_t *sl, + int (*compare)(const void **a, const void **b)) +{ + const void *most_frequent = NULL; + int most_frequent_count = 0; + + const void *cur = NULL; + int i, count=0; + + if (!sl->num_used) + return NULL; + for (i = 0; i < sl->num_used; ++i) { + const void *item = sl->list[i]; + if (cur && 0 == compare(&cur, &item)) { + ++count; + } else { + if (cur && count >= most_frequent_count) { + most_frequent = cur; + most_frequent_count = count; + } + cur = item; + count = 1; + } + } + if (cur && count >= most_frequent_count) { + most_frequent = cur; + most_frequent_count = count; + } + return (void*)most_frequent; +} + /** Given a sorted smartlist <b>sl</b> and the comparison function used to * sort it, remove all duplicate members. If free_fn is provided, calls * free_fn on each duplicate. Otherwise, just removes them. Preserves order. @@ -550,6 +587,13 @@ smartlist_sort_strings(smartlist_t *sl) smartlist_sort(sl, _compare_string_ptrs); } +/** Return the most frequent string in the sorted list <b>sl</b> */ +char * +smartlist_get_most_frequent_string(smartlist_t *sl) +{ + return smartlist_get_most_frequent(sl, _compare_string_ptrs); +} + /** Remove duplicate strings from a sorted list, and free them with tor_free(). */ void @@ -561,16 +605,70 @@ smartlist_uniq_strings(smartlist_t *sl) /* Heap-based priority queue implementation for O(lg N) insert and remove. * Recall that the heap property is that, for every index I, h[I] < * H[LEFT_CHILD[I]] and h[I] < H[RIGHT_CHILD[I]]. + * + * For us to remove items other than the topmost item, each item must store + * its own index within the heap. When calling the pqueue functions, tell + * them about the offset of the field that stores the index within the item. + * + * Example: + * + * typedef struct timer_t { + * struct timeval tv; + * int heap_index; + * } timer_t; + * + * static int compare(const void *p1, const void *p2) { + * const timer_t *t1 = p1, *t2 = p2; + * if (t1->tv.tv_sec < t2->tv.tv_sec) { + * return -1; + * } else if (t1->tv.tv_sec > t2->tv.tv_sec) { + * return 1; + * } else { + * return t1->tv.tv_usec - t2->tv_usec; + * } + * } + * + * void timer_heap_insert(smartlist_t *heap, timer_t *timer) { + * smartlist_pqueue_add(heap, compare, STRUCT_OFFSET(timer_t, heap_index), + * timer); + * } + * + * void timer_heap_pop(smartlist_t *heap) { + * return smartlist_pqueue_pop(heap, compare, + * STRUCT_OFFSET(timer_t, heap_index)); + * } */ -/* For a 1-indexed array, we would use LEFT_CHILD[x] = 2*x and RIGHT_CHILD[x] - * = 2*x + 1. But this is C, so we have to adjust a little. */ +/** @{ */ +/** Functions to manipulate heap indices to find a node's parent and children. + * + * For a 1-indexed array, we would use LEFT_CHILD[x] = 2*x and RIGHT_CHILD[x] + * = 2*x + 1. But this is C, so we have to adjust a little. */ //#define LEFT_CHILD(i) ( ((i)+1)*2 - 1) //#define RIGHT_CHILD(i) ( ((i)+1)*2 ) //#define PARENT(i) ( ((i)+1)/2 - 1) #define LEFT_CHILD(i) ( 2*(i) + 1 ) #define RIGHT_CHILD(i) ( 2*(i) + 2 ) #define PARENT(i) ( ((i)-1) / 2 ) +/** }@ */ + +/** @{ */ +/** Helper macros for heaps: Given a local variable <b>idx_field_offset</b> + * set to the offset of an integer index within the heap element structure, + * IDX_OF_ITEM(p) gives you the index of p, and IDXP(p) gives you a pointer to + * where p's index is stored. Given additionally a local smartlist <b>sl</b>, + * UPDATE_IDX(i) sets the index of the element at <b>i</b> to the correct + * value (that is, to <b>i</b>). + */ +#define IDXP(p) ((int*)STRUCT_VAR_P(p, idx_field_offset)) + +#define UPDATE_IDX(i) do { \ + void *updated = sl->list[i]; \ + *IDXP(updated) = i; \ + } while (0) + +#define IDX_OF_ITEM(p) (*IDXP(p)) +/** @} */ /** Helper. <b>sl</b> may have at most one violation of the heap property: * the item at <b>idx</b> may be greater than one or both of its children. @@ -578,6 +676,7 @@ smartlist_uniq_strings(smartlist_t *sl) static INLINE void smartlist_heapify(smartlist_t *sl, int (*compare)(const void *a, const void *b), + int idx_field_offset, int idx) { while (1) { @@ -600,21 +699,28 @@ smartlist_heapify(smartlist_t *sl, void *tmp = sl->list[idx]; sl->list[idx] = sl->list[best_idx]; sl->list[best_idx] = tmp; + UPDATE_IDX(idx); + UPDATE_IDX(best_idx); idx = best_idx; } } } -/** Insert <b>item</b> into the heap stored in <b>sl</b>, where order - * is determined by <b>compare</b>. */ +/** Insert <b>item</b> into the heap stored in <b>sl</b>, where order is + * determined by <b>compare</b> and the offset of the item in the heap is + * stored in an int-typed field at position <b>idx_field_offset</b> within + * item. + */ void smartlist_pqueue_add(smartlist_t *sl, int (*compare)(const void *a, const void *b), + int idx_field_offset, void *item) { int idx; smartlist_add(sl,item); + UPDATE_IDX(sl->num_used-1); for (idx = sl->num_used - 1; idx; ) { int parent = PARENT(idx); @@ -622,6 +728,8 @@ smartlist_pqueue_add(smartlist_t *sl, void *tmp = sl->list[parent]; sl->list[parent] = sl->list[idx]; sl->list[idx] = tmp; + UPDATE_IDX(parent); + UPDATE_IDX(idx); idx = parent; } else { return; @@ -630,32 +738,63 @@ smartlist_pqueue_add(smartlist_t *sl, } /** Remove and return the top-priority item from the heap stored in <b>sl</b>, - * where order is determined by <b>compare</b>. <b>sl</b> must not be - * empty. */ + * where order is determined by <b>compare</b> and the item's position is + * stored at position <b>idx_field_offset</b> within the item. <b>sl</b> must + * not be empty. */ void * smartlist_pqueue_pop(smartlist_t *sl, - int (*compare)(const void *a, const void *b)) + int (*compare)(const void *a, const void *b), + int idx_field_offset) { void *top; tor_assert(sl->num_used); top = sl->list[0]; + *IDXP(top)=-1; if (--sl->num_used) { sl->list[0] = sl->list[sl->num_used]; - smartlist_heapify(sl, compare, 0); + UPDATE_IDX(0); + smartlist_heapify(sl, compare, idx_field_offset, 0); } return top; } +/** Remove the item <b>item</b> from the heap stored in <b>sl</b>, + * where order is determined by <b>compare</b> and the item's position is + * stored at position <b>idx_field_offset</b> within the item. <b>sl</b> must + * not be empty. */ +void +smartlist_pqueue_remove(smartlist_t *sl, + int (*compare)(const void *a, const void *b), + int idx_field_offset, + void *item) +{ + int idx = IDX_OF_ITEM(item); + tor_assert(idx >= 0); + tor_assert(sl->list[idx] == item); + --sl->num_used; + *IDXP(item) = -1; + if (idx == sl->num_used) { + return; + } else { + sl->list[idx] = sl->list[sl->num_used]; + UPDATE_IDX(idx); + smartlist_heapify(sl, compare, idx_field_offset, idx); + } +} + /** Assert that the heap property is correctly maintained by the heap stored * in <b>sl</b>, where order is determined by <b>compare</b>. */ void smartlist_pqueue_assert_ok(smartlist_t *sl, - int (*compare)(const void *a, const void *b)) + int (*compare)(const void *a, const void *b), + int idx_field_offset) { int i; - for (i = sl->num_used - 1; i > 0; --i) { - tor_assert(compare(sl->list[PARENT(i)], sl->list[i]) <= 0); + for (i = sl->num_used - 1; i >= 0; --i) { + if (i>0) + tor_assert(compare(sl->list[PARENT(i)], sl->list[i]) <= 0); + tor_assert(IDX_OF_ITEM(sl->list[i]) == i); } } @@ -681,6 +820,37 @@ smartlist_uniq_digests(smartlist_t *sl) smartlist_uniq(sl, _compare_digests, _tor_free); } +/** Helper: compare two DIGEST256_LEN digests. */ +static int +_compare_digests256(const void **_a, const void **_b) +{ + return memcmp((const char*)*_a, (const char*)*_b, DIGEST256_LEN); +} + +/** Sort the list of DIGEST256_LEN-byte digests into ascending order. */ +void +smartlist_sort_digests256(smartlist_t *sl) +{ + smartlist_sort(sl, _compare_digests256); +} + +/** Return the most frequent member of the sorted list of DIGEST256_LEN + * digests in <b>sl</b> */ +char * +smartlist_get_most_frequent_digest256(smartlist_t *sl) +{ + return smartlist_get_most_frequent(sl, _compare_digests256); +} + +/** Remove duplicate 256-bit digests from a sorted list, and free them with + * tor_free(). + */ +void +smartlist_uniq_digests256(smartlist_t *sl) +{ + smartlist_uniq(sl, _compare_digests256, _tor_free); +} + /** Helper: Declare an entry type and a map type to implement a mapping using * ht.h. The map type will be called <b>maptype</b>. The key part of each * entry is declared using the C declaration <b>keydecl</b>. All functions @@ -1113,6 +1283,9 @@ void strmap_free(strmap_t *map, void (*free_val)(void*)) { strmap_entry_t **ent, **next, *this; + if (!map) + return; + for (ent = HT_START(strmap_impl, &map->head); ent != NULL; ent = next) { this = *ent; next = HT_NEXT_RMV(strmap_impl, &map->head, ent); @@ -1134,6 +1307,8 @@ void digestmap_free(digestmap_t *map, void (*free_val)(void*)) { digestmap_entry_t **ent, **next, *this; + if (!map) + return; for (ent = HT_START(digestmap_impl, &map->head); ent != NULL; ent = next) { this = *ent; next = HT_NEXT_RMV(digestmap_impl, &map->head, ent); @@ -1220,6 +1395,7 @@ IMPLEMENT_ORDER_FUNC(find_nth_int, int) IMPLEMENT_ORDER_FUNC(find_nth_time, time_t) IMPLEMENT_ORDER_FUNC(find_nth_double, double) IMPLEMENT_ORDER_FUNC(find_nth_uint32, uint32_t) +IMPLEMENT_ORDER_FUNC(find_nth_int32, int32_t) IMPLEMENT_ORDER_FUNC(find_nth_long, long) /** Return a newly allocated digestset_t, optimized to hold a total of @@ -1248,6 +1424,8 @@ digestset_new(int max_elements) void digestset_free(digestset_t *set) { + if (!set) + return; bitarray_free(set->ba); tor_free(set); } diff --git a/src/common/container.h b/src/common/container.h index bb7cb89fc5..b39d4ca07e 100644 --- a/src/common/container.h +++ b/src/common/container.h @@ -15,6 +15,7 @@ * and macros defined here. **/ typedef struct smartlist_t { + /** @{ */ /** <b>list</b> has enough capacity to store exactly <b>capacity</b> elements * before it needs to be resized. Only the first <b>num_used</b> (\<= * capacity) elements point to valid data. @@ -22,6 +23,7 @@ typedef struct smartlist_t { void **list; int num_used; int capacity; + /** @} */ } smartlist_t; smartlist_t *smartlist_create(void); @@ -93,13 +95,22 @@ void smartlist_del_keeporder(smartlist_t *sl, int idx); void smartlist_insert(smartlist_t *sl, int idx, void *val); void smartlist_sort(smartlist_t *sl, int (*compare)(const void **a, const void **b)); +void *smartlist_get_most_frequent(const smartlist_t *sl, + int (*compare)(const void **a, const void **b)); void smartlist_uniq(smartlist_t *sl, int (*compare)(const void **a, const void **b), void (*free_fn)(void *elt)); + void smartlist_sort_strings(smartlist_t *sl); void smartlist_sort_digests(smartlist_t *sl); +void smartlist_sort_digests256(smartlist_t *sl); + +char *smartlist_get_most_frequent_string(smartlist_t *sl); +char *smartlist_get_most_frequent_digest256(smartlist_t *sl); + void smartlist_uniq_strings(smartlist_t *sl); void smartlist_uniq_digests(smartlist_t *sl); +void smartlist_uniq_digests256(smartlist_t *sl); void *smartlist_bsearch(smartlist_t *sl, const void *key, int (*compare)(const void *key, const void **member)) ATTR_PURE; @@ -109,11 +120,18 @@ int smartlist_bsearch_idx(const smartlist_t *sl, const void *key, void smartlist_pqueue_add(smartlist_t *sl, int (*compare)(const void *a, const void *b), + int idx_field_offset, void *item); void *smartlist_pqueue_pop(smartlist_t *sl, - int (*compare)(const void *a, const void *b)); + int (*compare)(const void *a, const void *b), + int idx_field_offset); +void smartlist_pqueue_remove(smartlist_t *sl, + int (*compare)(const void *a, const void *b), + int idx_field_offset, + void *item); void smartlist_pqueue_assert_ok(smartlist_t *sl, - int (*compare)(const void *a, const void *b)); + int (*compare)(const void *a, const void *b), + int idx_field_offset); #define SPLIT_SKIP_SPACE 0x01 #define SPLIT_IGNORE_BLANK 0x02 @@ -579,9 +597,9 @@ bitarray_is_set(bitarray_t *b, int bit) /** A set of digests, implemented as a Bloom filter. */ typedef struct { - int mask; /* One less than the number of bits in <b>ba</b>; always one less + int mask; /**< One less than the number of bits in <b>ba</b>; always one less * than a power of two. */ - bitarray_t *ba; /* A bit array to implement the Bloom filter. */ + bitarray_t *ba; /**< A bit array to implement the Bloom filter. */ } digestset_t; #define BIT(n) ((n) & set->mask) @@ -627,6 +645,7 @@ void digestset_free(digestset_t* set); int find_nth_int(int *array, int n_elements, int nth); time_t find_nth_time(time_t *array, int n_elements, int nth); double find_nth_double(double *array, int n_elements, int nth); +int32_t find_nth_int32(int32_t *array, int n_elements, int nth); uint32_t find_nth_uint32(uint32_t *array, int n_elements, int nth); long find_nth_long(long *array, int n_elements, int nth); static INLINE int @@ -649,6 +668,11 @@ median_uint32(uint32_t *array, int n_elements) { return find_nth_uint32(array, n_elements, (n_elements-1)/2); } +static INLINE int32_t +median_int32(int32_t *array, int n_elements) +{ + return find_nth_int32(array, n_elements, (n_elements-1)/2); +} static INLINE long median_long(long *array, int n_elements) { diff --git a/src/common/crypto.c b/src/common/crypto.c index f3268fe183..ff117e929f 100644 --- a/src/common/crypto.c +++ b/src/common/crypto.c @@ -27,6 +27,7 @@ #include <openssl/rsa.h> #include <openssl/pem.h> #include <openssl/evp.h> +#include <openssl/engine.h> #include <openssl/rand.h> #include <openssl/opensslv.h> #include <openssl/bn.h> @@ -49,9 +50,9 @@ #define CRYPTO_PRIVATE #include "crypto.h" -#include "log.h" +#include "../common/torlog.h" #include "aes.h" -#include "util.h" +#include "../common/util.h" #include "container.h" #include "compat.h" @@ -61,6 +62,35 @@ #include <openssl/engine.h> +#ifdef ANDROID +/* Android's OpenSSL seems to have removed all of its Engine support. */ +#define DISABLE_ENGINES +#endif + +#if OPENSSL_VERSION_NUMBER < 0x00908000l +/** @{ */ +/** On OpenSSL versions before 0.9.8, there is no working SHA256 + * implementation, so we use Tom St Denis's nice speedy one, slightly adapted + * to our needs. These macros make it usable by us. */ +#define SHA256_CTX sha256_state +#define SHA256_Init sha256_init +#define SHA256_Update sha256_process +#define LTC_ARGCHK(x) tor_assert(x) +/** @} */ +#include "sha256.c" +#define SHA256_Final(a,b) sha256_done(b,a) + +static unsigned char * +SHA256(const unsigned char *m, size_t len, unsigned char *d) +{ + SHA256_CTX ctx; + SHA256_Init(&ctx); + SHA256_Update(&ctx, m, len); + SHA256_Final(d, &ctx); + return d; +} +#endif + /** Macro: is k a valid RSA public or private key? */ #define PUBLIC_KEY_OK(k) ((k) && (k)->key && (k)->key->n) /** Macro: is k a valid RSA private key? */ @@ -76,25 +106,26 @@ static int _n_openssl_mutexes = 0; /** A public key, or a public/private key-pair. */ struct crypto_pk_env_t { - int refs; /* reference counting so we don't have to copy keys */ - RSA *key; + int refs; /**< reference count, so we don't have to copy keys */ + RSA *key; /**< The key itself */ }; /** Key and stream information for a stream cipher. */ struct crypto_cipher_env_t { - char key[CIPHER_KEY_LEN]; - aes_cnt_cipher_t *cipher; + char key[CIPHER_KEY_LEN]; /**< The raw key. */ + aes_cnt_cipher_t *cipher; /**< The key in format usable for counter-mode AES + * encryption */ }; /** A structure to hold the first half (x, g^x) of a Diffie-Hellman handshake * while we're waiting for the second.*/ struct crypto_dh_env_t { - DH *dh; + DH *dh; /**< The openssl DH object */ }; static int setup_openssl_threading(void); -static int tor_check_dh_key(BIGNUM *bn); +static int tor_check_dh_key(int severity, BIGNUM *bn); /** Return the number of bytes added by padding method <b>padding</b>. */ @@ -151,6 +182,7 @@ crypto_log_errors(int severity, const char *doing) } } +#ifndef DISABLE_ENGINES /** Log any OpenSSL engines we're using at NOTICE. */ static void log_engine(const char *fn, ENGINE *e) @@ -165,37 +197,82 @@ log_engine(const char *fn, ENGINE *e) log(LOG_INFO, LD_CRYPTO, "Using default implementation for %s", fn); } } +#endif + +#ifndef DISABLE_ENGINES +/** Try to load an engine in a shared library via fully qualified path. + */ +static ENGINE * +try_load_engine(const char *path, const char *engine) +{ + ENGINE *e = ENGINE_by_id("dynamic"); + if (e) { + if (!ENGINE_ctrl_cmd_string(e, "ID", engine, 0) || + !ENGINE_ctrl_cmd_string(e, "DIR_LOAD", "2", 0) || + !ENGINE_ctrl_cmd_string(e, "DIR_ADD", path, 0) || + !ENGINE_ctrl_cmd_string(e, "LOAD", NULL, 0)) { + ENGINE_free(e); + e = NULL; + } + } + return e; +} +#endif /** Initialize the crypto library. Return 0 on success, -1 on failure. */ int -crypto_global_init(int useAccel) +crypto_global_init(int useAccel, const char *accelName, const char *accelDir) { if (!_crypto_global_initialized) { ERR_load_crypto_strings(); OpenSSL_add_all_algorithms(); _crypto_global_initialized = 1; setup_openssl_threading(); - /* XXX the below is a bug, since we can't know if we're supposed - * to be using hardware acceleration or not. we should arrange - * for this function to be called before init_keys. But make it - * not complain loudly, at least until we make acceleration work. */ - if (useAccel < 0) { - log_info(LD_CRYPTO, "Initializing OpenSSL via tor_tls_init()."); - } if (useAccel > 0) { +#ifdef DISABLE_ENGINES + (void)accelName; + (void)accelDir; + log_warn(LD_CRYPTO, "No OpenSSL hardware acceleration support enabled."); +#else + ENGINE *e = NULL; + log_info(LD_CRYPTO, "Initializing OpenSSL engine support."); ENGINE_load_builtin_engines(); - if (!ENGINE_register_all_complete()) - return -1; - - /* XXXX make sure this isn't leaking. */ + ENGINE_register_all_complete(); + + if (accelName) { + if (accelDir) { + log_info(LD_CRYPTO, "Trying to load dynamic OpenSSL engine \"%s\"" + " via path \"%s\".", accelName, accelDir); + e = try_load_engine(accelName, accelDir); + } else { + log_info(LD_CRYPTO, "Initializing dynamic OpenSSL engine \"%s\"" + " acceleration support.", accelName); + e = ENGINE_by_id(accelName); + } + if (!e) { + log_warn(LD_CRYPTO, "Unable to load dynamic OpenSSL engine \"%s\".", + accelName); + } else { + log_info(LD_CRYPTO, "Loaded dynamic OpenSSL engine \"%s\".", + accelName); + } + } + if (e) { + log_info(LD_CRYPTO, "Loaded OpenSSL hardware acceleration engine," + " setting default ciphers."); + ENGINE_set_default(e, ENGINE_METHOD_ALL); + } log_engine("RSA", ENGINE_get_default_RSA()); log_engine("DH", ENGINE_get_default_DH()); log_engine("RAND", ENGINE_get_default_RAND()); log_engine("SHA1", ENGINE_get_digest_engine(NID_sha1)); log_engine("3DES", ENGINE_get_cipher_engine(NID_des_ede3_ecb)); log_engine("AES", ENGINE_get_cipher_engine(NID_aes_128_ecb)); +#endif + } else { + log_info(LD_CRYPTO, "NOT using OpenSSL engine support."); } return crypto_seed_rng(1); } @@ -217,7 +294,11 @@ crypto_global_cleanup(void) EVP_cleanup(); ERR_remove_state(0); ERR_free_strings(); + +#ifndef DISABLE_ENGINES ENGINE_cleanup(); +#endif + CONF_modules_unload(1); CRYPTO_cleanup_all_ex_data(); #ifdef TOR_IS_MULTITHREADED @@ -248,18 +329,8 @@ _crypto_new_pk_env_rsa(RSA *rsa) return env; } -/** used by tortls.c: wrap the RSA from an evp_pkey in a crypto_pk_env_t. - * returns NULL if this isn't an RSA key. */ -crypto_pk_env_t * -_crypto_new_pk_env_evp_pkey(EVP_PKEY *pkey) -{ - RSA *rsa; - if (!(rsa = EVP_PKEY_get1_RSA(pkey))) - return NULL; - return _crypto_new_pk_env_rsa(rsa); -} - -/** Helper, used by tor-checkkey.c. Return the RSA from a crypto_pk_env_t. */ +/** Helper, used by tor-checkkey.c and tor-gencert.c. Return the RSA from a + * crypto_pk_env_t. */ RSA * _crypto_pk_env_get_rsa(crypto_pk_env_t *env) { @@ -311,7 +382,7 @@ crypto_new_pk_env(void) RSA *rsa; rsa = RSA_new(); - if (!rsa) return NULL; + tor_assert(rsa); return _crypto_new_pk_env_rsa(rsa); } @@ -321,10 +392,12 @@ crypto_new_pk_env(void) void crypto_free_pk_env(crypto_pk_env_t *env) { - tor_assert(env); + if (!env) + return; if (--env->refs > 0) return; + tor_assert(env->refs == 0); if (env->key) RSA_free(env->key); @@ -347,10 +420,7 @@ crypto_create_init_cipher(const char *key, int encrypt_mode) return NULL; } - if (crypto_cipher_set_key(crypto, key)) { - crypto_log_errors(LOG_WARN, "setting symmetric key"); - goto error; - } + crypto_cipher_set_key(crypto, key); if (encrypt_mode) r = crypto_cipher_encrypt_init_cipher(crypto); @@ -384,7 +454,8 @@ crypto_new_cipher_env(void) void crypto_free_cipher_env(crypto_cipher_env_t *env) { - tor_assert(env); + if (!env) + return; tor_assert(env->cipher); aes_free_cipher(env->cipher); @@ -394,11 +465,11 @@ crypto_free_cipher_env(crypto_cipher_env_t *env) /* public key crypto */ -/** Generate a new public/private keypair in <b>env</b>. Return 0 on - * success, -1 on failure. +/** Generate a <b>bits</b>-bit new public/private keypair in <b>env</b>. + * Return 0 on success, -1 on failure. */ int -crypto_pk_generate_key(crypto_pk_env_t *env) +crypto_pk_generate_key_with_bits(crypto_pk_env_t *env, int bits) { tor_assert(env); @@ -406,7 +477,7 @@ crypto_pk_generate_key(crypto_pk_env_t *env) RSA_free(env->key); #if OPENSSL_VERSION_NUMBER < 0x00908000l /* In OpenSSL 0.9.7, RSA_generate_key is all we have. */ - env->key = RSA_generate_key(PK_BYTES*8,65537, NULL, NULL); + env->key = RSA_generate_key(bits, 65537, NULL, NULL); #else /* In OpenSSL 0.9.8, RSA_generate_key is deprecated. */ { @@ -419,7 +490,7 @@ crypto_pk_generate_key(crypto_pk_env_t *env) r = RSA_new(); if (!r) goto done; - if (RSA_generate_key_ex(r, PK_BYTES*8, e, NULL) == -1) + if (RSA_generate_key_ex(r, bits, e, NULL) == -1) goto done; env->key = r; @@ -456,6 +527,8 @@ crypto_pk_read_private_key_from_string(crypto_pk_env_t *env, /* Create a read-only memory BIO, backed by the string 's' */ b = BIO_new_mem_buf((char*)s, (int)len); + if (!b) + return -1; if (env->key) RSA_free(env->key); @@ -516,6 +589,8 @@ crypto_pk_write_key_to_string_impl(crypto_pk_env_t *env, char **dest, tor_assert(dest); b = BIO_new(BIO_s_mem()); /* Create a memory BIO */ + if (!b) + return -1; /* Now you can treat b as if it were a file. Just use the * PEM_*_bio_* functions instead of the non-bio variants. @@ -583,6 +658,8 @@ crypto_pk_read_public_key_from_string(crypto_pk_env_t *env, const char *src, tor_assert(len<INT_MAX); b = BIO_new(BIO_s_mem()); /* Create a memory BIO */ + if (!b) + return -1; BIO_write(b, src, (int)len); @@ -705,14 +782,25 @@ crypto_pk_env_t * crypto_pk_copy_full(crypto_pk_env_t *env) { RSA *new_key; + int privatekey = 0; tor_assert(env); tor_assert(env->key); if (PRIVATE_KEY_OK(env)) { new_key = RSAPrivateKey_dup(env->key); + privatekey = 1; } else { new_key = RSAPublicKey_dup(env->key); } + if (!new_key) { + log_err(LD_CRYPTO, "Unable to duplicate a %s key: openssl failed.", + privatekey?"private":"public"); + crypto_log_errors(LOG_ERR, + privatekey ? "Duplicating a private key" : + "Duplicating a public key"); + tor_fragile_assert(); + return NULL; + } return _crypto_new_pk_env_rsa(new_key); } @@ -1213,19 +1301,14 @@ crypto_cipher_generate_key(crypto_cipher_env_t *env) /** Set the symmetric key for the cipher in <b>env</b> to the first * CIPHER_KEY_LEN bytes of <b>key</b>. Does not initialize the cipher. - * Return 0 on success, -1 on failure. */ -int +void crypto_cipher_set_key(crypto_cipher_env_t *env, const char *key) { tor_assert(env); tor_assert(key); - if (!env->key) - return -1; - memcpy(env->key, key, CIPHER_KEY_LEN); - return 0; } /** Generate an initialization vector for our AES-CTR cipher; store it @@ -1390,7 +1473,7 @@ crypto_cipher_decrypt_with_iv(crypto_cipher_env_t *cipher, /* SHA-1 */ -/** Compute the SHA1 digest of <b>len</b> bytes in data stored in +/** Compute the SHA1 digest of the <b>len</b> bytes on data stored in * <b>m</b>. Write the DIGEST_LEN byte result into <b>digest</b>. * Return 0 on success, -1 on failure. */ @@ -1402,19 +1485,97 @@ crypto_digest(char *digest, const char *m, size_t len) return (SHA1((const unsigned char*)m,len,(unsigned char*)digest) == NULL); } +/** Compute a 256-bit digest of <b>len</b> bytes in data stored in <b>m</b>, + * using the algorithm <b>algorithm</b>. Write the DIGEST_LEN256-byte result + * into <b>digest</b>. Return 0 on success, -1 on failure. */ +int +crypto_digest256(char *digest, const char *m, size_t len, + digest_algorithm_t algorithm) +{ + tor_assert(m); + tor_assert(digest); + tor_assert(algorithm == DIGEST_SHA256); + return (SHA256((const unsigned char*)m,len,(unsigned char*)digest) == NULL); +} + +/** Set the digests_t in <b>ds_out</b> to contain every digest on the + * <b>len</b> bytes in <b>m</b> that we know how to compute. Return 0 on + * success, -1 on failure. */ +int +crypto_digest_all(digests_t *ds_out, const char *m, size_t len) +{ + digest_algorithm_t i; + tor_assert(ds_out); + memset(ds_out, 0, sizeof(*ds_out)); + if (crypto_digest(ds_out->d[DIGEST_SHA1], m, len) < 0) + return -1; + for (i = DIGEST_SHA256; i < N_DIGEST_ALGORITHMS; ++i) { + if (crypto_digest256(ds_out->d[i], m, len, i) < 0) + return -1; + } + return 0; +} + +/** Return the name of an algorithm, as used in directory documents. */ +const char * +crypto_digest_algorithm_get_name(digest_algorithm_t alg) +{ + switch (alg) { + case DIGEST_SHA1: + return "sha1"; + case DIGEST_SHA256: + return "sha256"; + default: + tor_fragile_assert(); + return "??unknown_digest??"; + } +} + +/** Given the name of a digest algorithm, return its integer value, or -1 if + * the name is not recognized. */ +int +crypto_digest_algorithm_parse_name(const char *name) +{ + if (!strcmp(name, "sha1")) + return DIGEST_SHA1; + else if (!strcmp(name, "sha256")) + return DIGEST_SHA256; + else + return -1; +} + /** Intermediate information about the digest of a stream of data. */ struct crypto_digest_env_t { - SHA_CTX d; + union { + SHA_CTX sha1; /**< state for SHA1 */ + SHA256_CTX sha2; /**< state for SHA256 */ + } d; /**< State for the digest we're using. Only one member of the + * union is usable, depending on the value of <b>algorithm</b>. */ + digest_algorithm_t algorithm : 8; /**< Which algorithm is in use? */ }; -/** Allocate and return a new digest object. +/** Allocate and return a new digest object to compute SHA1 digests. */ crypto_digest_env_t * crypto_new_digest_env(void) { crypto_digest_env_t *r; r = tor_malloc(sizeof(crypto_digest_env_t)); - SHA1_Init(&r->d); + SHA1_Init(&r->d.sha1); + r->algorithm = DIGEST_SHA1; + return r; +} + +/** Allocate and return a new digest object to compute 256-bit digests + * using <b>algorithm</b>. */ +crypto_digest_env_t * +crypto_new_digest256_env(digest_algorithm_t algorithm) +{ + crypto_digest_env_t *r; + tor_assert(algorithm == DIGEST_SHA256); + r = tor_malloc(sizeof(crypto_digest_env_t)); + SHA256_Init(&r->d.sha2); + r->algorithm = algorithm; return r; } @@ -1423,6 +1584,8 @@ crypto_new_digest_env(void) void crypto_free_digest_env(crypto_digest_env_t *digest) { + if (!digest) + return; memset(digest, 0, sizeof(crypto_digest_env_t)); tor_free(digest); } @@ -1435,30 +1598,51 @@ crypto_digest_add_bytes(crypto_digest_env_t *digest, const char *data, { tor_assert(digest); tor_assert(data); - /* Using the SHA1_*() calls directly means we don't support doing - * SHA1 in hardware. But so far the delay of getting the question + /* Using the SHA*_*() calls directly means we don't support doing + * SHA in hardware. But so far the delay of getting the question * to the hardware, and hearing the answer, is likely higher than * just doing it ourselves. Hashes are fast. */ - SHA1_Update(&digest->d, (void*)data, len); + switch (digest->algorithm) { + case DIGEST_SHA1: + SHA1_Update(&digest->d.sha1, (void*)data, len); + break; + case DIGEST_SHA256: + SHA256_Update(&digest->d.sha2, (void*)data, len); + break; + default: + tor_fragile_assert(); + break; + } } /** Compute the hash of the data that has been passed to the digest * object; write the first out_len bytes of the result to <b>out</b>. - * <b>out_len</b> must be \<= DIGEST_LEN. + * <b>out_len</b> must be \<= DIGEST256_LEN. */ void crypto_digest_get_digest(crypto_digest_env_t *digest, char *out, size_t out_len) { - unsigned char r[DIGEST_LEN]; - SHA_CTX tmpctx; + unsigned char r[DIGEST256_LEN]; + crypto_digest_env_t tmpenv; tor_assert(digest); tor_assert(out); - tor_assert(out_len <= DIGEST_LEN); - /* memcpy into a temporary ctx, since SHA1_Final clears the context */ - memcpy(&tmpctx, &digest->d, sizeof(SHA_CTX)); - SHA1_Final(r, &tmpctx); + /* memcpy into a temporary ctx, since SHA*_Final clears the context */ + memcpy(&tmpenv, digest, sizeof(crypto_digest_env_t)); + switch (digest->algorithm) { + case DIGEST_SHA1: + tor_assert(out_len <= DIGEST_LEN); + SHA1_Final(r, &tmpenv.d.sha1); + break; + case DIGEST_SHA256: + tor_assert(out_len <= DIGEST256_LEN); + SHA256_Final(r, &tmpenv.d.sha2); + break; + default: + tor_fragile_assert(); + break; + } memcpy(out, r, out_len); memset(r, 0, sizeof(r)); } @@ -1557,6 +1741,10 @@ init_dh_param(void) dh_param_g = g; } +/** Number of bits to use when choosing the x or y value in a Diffie-Hellman + * handshake. Since we exponentiate by this value, choosing a smaller one + * lets our handhake go faster. + */ #define DH_PRIVATE_KEY_BITS 320 /** Allocate and return a new DH object for a key exchange. @@ -1616,7 +1804,7 @@ crypto_dh_generate_public(crypto_dh_env_t *dh) crypto_log_errors(LOG_WARN, "generating DH key"); return -1; } - if (tor_check_dh_key(dh->dh->pub_key)<0) { + if (tor_check_dh_key(LOG_WARN, dh->dh->pub_key)<0) { log_warn(LD_CRYPTO, "Weird! Our own DH key was invalid. I guess once-in-" "the-universe chances really do happen. Trying again."); /* Free and clear the keys, so OpenSSL will actually try again. */ @@ -1663,7 +1851,7 @@ crypto_dh_get_public(crypto_dh_env_t *dh, char *pubkey, size_t pubkey_len) * See http://www.cl.cam.ac.uk/ftp/users/rja14/psandqs.ps.gz for some tips. */ static int -tor_check_dh_key(BIGNUM *bn) +tor_check_dh_key(int severity, BIGNUM *bn) { BIGNUM *x; char *s; @@ -1674,13 +1862,13 @@ tor_check_dh_key(BIGNUM *bn) init_dh_param(); BN_set_word(x, 1); if (BN_cmp(bn,x)<=0) { - log_warn(LD_CRYPTO, "DH key must be at least 2."); + log_fn(severity, LD_CRYPTO, "DH key must be at least 2."); goto err; } BN_copy(x,dh_param_p); BN_sub_word(x, 1); if (BN_cmp(bn,x)>=0) { - log_warn(LD_CRYPTO, "DH key must be at most p-2."); + log_fn(severity, LD_CRYPTO, "DH key must be at most p-2."); goto err; } BN_free(x); @@ -1688,7 +1876,7 @@ tor_check_dh_key(BIGNUM *bn) err: BN_free(x); s = BN_bn2hex(bn); - log_warn(LD_CRYPTO, "Rejecting insecure DH key [%s]", s); + log_fn(severity, LD_CRYPTO, "Rejecting insecure DH key [%s]", s); OPENSSL_free(s); return -1; } @@ -1706,7 +1894,7 @@ tor_check_dh_key(BIGNUM *bn) * where || is concatenation.) */ ssize_t -crypto_dh_compute_secret(crypto_dh_env_t *dh, +crypto_dh_compute_secret(int severity, crypto_dh_env_t *dh, const char *pubkey, size_t pubkey_len, char *secret_out, size_t secret_bytes_out) { @@ -1721,9 +1909,9 @@ crypto_dh_compute_secret(crypto_dh_env_t *dh, if (!(pubkey_bn = BN_bin2bn((const unsigned char*)pubkey, (int)pubkey_len, NULL))) goto error; - if (tor_check_dh_key(pubkey_bn)<0) { + if (tor_check_dh_key(severity, pubkey_bn)<0) { /* Check for invalid public keys. */ - log_warn(LD_CRYPTO,"Rejected invalid g^x"); + log_fn(severity, LD_CRYPTO,"Rejected invalid g^x"); goto error; } secret_tmp_len = crypto_dh_get_bytes(dh); @@ -1799,7 +1987,8 @@ crypto_expand_key_material(const char *key_in, size_t key_in_len, void crypto_dh_free(crypto_dh_env_t *dh) { - tor_assert(dh); + if (!dh) + return; tor_assert(dh->dh); DH_free(dh->dh); tor_free(dh); @@ -1807,15 +1996,22 @@ crypto_dh_free(crypto_dh_env_t *dh) /* random numbers */ -/* This is how much entropy OpenSSL likes to add right now, so maybe it will +/** How many bytes of entropy we add at once. + * + * This is how much entropy OpenSSL likes to add right now, so maybe it will * work for us too. */ #define ADD_ENTROPY 32 -/* Use RAND_poll if OpenSSL is 0.9.6 release or later. (The "f" means - "release".) */ +/** True iff we should use OpenSSL's RAND_poll function to add entropy to its + * pool. + * + * Use RAND_poll if OpenSSL is 0.9.6 release or later. (The "f" means + *"release".) */ #define HAVE_RAND_POLL (OPENSSL_VERSION_NUMBER >= 0x0090600fl) -/* Versions of OpenSSL prior to 0.9.7k and 0.9.8c had a bug where RAND_poll +/** True iff it's safe to use RAND_poll after setup. + * + * Versions of OpenSSL prior to 0.9.7k and 0.9.8c had a bug where RAND_poll * would allocate an fd_set on the stack, open a new file, and try to FD_SET * that fd without checking whether it fit in the fd_set. Thus, if the * system has not just been started up, it is unsafe to call */ @@ -1824,6 +2020,15 @@ crypto_dh_free(crypto_dh_env_t *dh) OPENSSL_VERSION_NUMBER <= 0x00907fffl) || \ (OPENSSL_VERSION_NUMBER >= 0x0090803fl)) +/** Set the seed of the weak RNG to a random value. */ +static void +seed_weak_rng(void) +{ + unsigned seed; + crypto_rand((void*)&seed, sizeof(seed)); + tor_init_weak_random(seed); +} + /** Seed OpenSSL's random number generator with bytes from the operating * system. <b>startup</b> should be true iff we have just started Tor and * have not yet allocated a bunch of fds. Return 0 on success, -1 on failure. @@ -1831,14 +2036,15 @@ crypto_dh_free(crypto_dh_env_t *dh) int crypto_seed_rng(int startup) { - char buf[ADD_ENTROPY]; int rand_poll_status = 0; /* local variables */ #ifdef MS_WINDOWS + unsigned char buf[ADD_ENTROPY]; static int provider_set = 0; static HCRYPTPROV provider; #else + char buf[ADD_ENTROPY]; static const char *filenames[] = { "/dev/srandom", "/dev/urandom", "/dev/random", NULL }; @@ -1874,6 +2080,7 @@ crypto_seed_rng(int startup) } RAND_seed(buf, sizeof(buf)); memset(buf, 0, sizeof(buf)); + seed_weak_rng(); return 0; #else for (i = 0; filenames[i]; ++i) { @@ -1890,6 +2097,7 @@ crypto_seed_rng(int startup) } RAND_seed(buf, (int)sizeof(buf)); memset(buf, 0, sizeof(buf)); + seed_weak_rng(); return 0; } @@ -1957,6 +2165,26 @@ crypto_rand_uint64(uint64_t max) } } +/** Return a pseudorandom double d, chosen uniformly from the range + * 0.0 <= d < 1.0. + */ +double +crypto_rand_double(void) +{ + /* We just use an unsigned int here; we don't really care about getting + * more than 32 bits of resolution */ + unsigned int uint; + crypto_rand((char*)&uint, sizeof(uint)); +#if SIZEOF_INT == 4 +#define UINT_MAX_AS_DOUBLE 4294967296.0 +#elif SIZEOF_INT == 8 +#define UINT_MAX_AS_DOUBLE 1.8446744073709552e+19 +#else +#error SIZEOF_INT is neither 4 nor 8 +#endif + return ((double)uint) / UINT_MAX_AS_DOUBLE; +} + /** Generate and return a new random hostname starting with <b>prefix</b>, * ending with <b>suffix</b>, and containing no less than * <b>min_rand_len</b> and no more than <b>max_rand_len</b> random base32 @@ -2046,9 +2274,12 @@ base64_encode(char *dest, size_t destlen, const char *src, size_t srclen) return ret; } +/** @{ */ +/** Special values used for the base64_decode_table */ #define X 255 #define SP 64 #define PAD 65 +/** @} */ /** Internal table mapping byte values to what they represent in base64. * Numbers 0..63 are 6-bit integers. SPs are spaces, and should be * skipped. Xs are invalid and must not appear in base64. PAD indicates @@ -2217,15 +2448,54 @@ digest_from_base64(char *digest, const char *d64) #endif } +/** Base-64 encode DIGEST256_LINE bytes from <b>digest</b>, remove the + * trailing = and newline characters, and store the nul-terminated result in + * the first BASE64_DIGEST256_LEN+1 bytes of <b>d64</b>. */ +int +digest256_to_base64(char *d64, const char *digest) +{ + char buf[256]; + base64_encode(buf, sizeof(buf), digest, DIGEST256_LEN); + buf[BASE64_DIGEST256_LEN] = '\0'; + memcpy(d64, buf, BASE64_DIGEST256_LEN+1); + return 0; +} + +/** Given a base-64 encoded, nul-terminated digest in <b>d64</b> (without + * trailing newline or = characters), decode it and store the result in the + * first DIGEST256_LEN bytes at <b>digest</b>. */ +int +digest256_from_base64(char *digest, const char *d64) +{ +#ifdef USE_OPENSSL_BASE64 + char buf_in[BASE64_DIGEST256_LEN+3]; + char buf[256]; + if (strlen(d64) != BASE64_DIGEST256_LEN) + return -1; + memcpy(buf_in, d64, BASE64_DIGEST256_LEN); + memcpy(buf_in+BASE64_DIGEST256_LEN, "=\n\0", 3); + if (base64_decode(buf, sizeof(buf), buf_in, strlen(buf_in)) != DIGEST256_LEN) + return -1; + memcpy(digest, buf, DIGEST256_LEN); + return 0; +#else + if (base64_decode(digest, DIGEST256_LEN, d64, strlen(d64)) == DIGEST256_LEN) + return 0; + else + return -1; +#endif +} + /** Implements base32 encoding as in rfc3548. Limitation: Requires * that srclen*8 is a multiple of 5. */ void base32_encode(char *dest, size_t destlen, const char *src, size_t srclen) { - unsigned int i, bit, v, u; - size_t nbits = srclen * 8; + unsigned int i, v, u; + size_t nbits = srclen * 8, bit; + tor_assert(srclen < SIZE_T_CEILING/8); tor_assert((nbits%5) == 0); /* We need an even multiple of 5 bits. */ tor_assert((nbits/5)+1 <= destlen); /* We need enough space. */ tor_assert(destlen < SIZE_T_CEILING); @@ -2249,11 +2519,12 @@ base32_decode(char *dest, size_t destlen, const char *src, size_t srclen) { /* XXXX we might want to rewrite this along the lines of base64_decode, if * it ever shows up in the profile. */ - unsigned int i, j, bit; - size_t nbits; + unsigned int i; + size_t nbits, j, bit; char *tmp; nbits = srclen * 5; + tor_assert(srclen < SIZE_T_CEILING / 5); tor_assert((nbits%8) == 0); /* We need an even multiple of 8 bits. */ tor_assert((nbits/8) <= destlen); /* We need enough space. */ tor_assert(destlen < SIZE_T_CEILING); @@ -2412,6 +2683,7 @@ _openssl_dynlock_destroy_cb(struct CRYPTO_dynlock_value *v, tor_free(v); } +/** @{ */ /** Helper: Construct mutexes, and set callbacks to help OpenSSL handle being * multithreaded. */ static int @@ -2437,4 +2709,5 @@ setup_openssl_threading(void) return 0; } #endif +/** @} */ diff --git a/src/common/crypto.h b/src/common/crypto.h index 576c03dc30..05185f3f18 100644 --- a/src/common/crypto.h +++ b/src/common/crypto.h @@ -18,6 +18,9 @@ /** Length of the output of our message digest. */ #define DIGEST_LEN 20 +/** Length of the output of our second (improved) message digests. (For now + * this is just sha256, but any it can be any other 256-byte digest). */ +#define DIGEST256_LEN 32 /** Length of our symmetric cipher's keys. */ #define CIPHER_KEY_LEN 16 /** Length of our symmetric cipher's IV. */ @@ -27,9 +30,12 @@ /** Length of our DH keys. */ #define DH_BYTES (1024/8) -/** Length of a message digest when encoded in base64 with trailing = signs - * removed. */ +/** Length of a sha1 message digest when encoded in base64 with trailing = + * signs removed. */ #define BASE64_DIGEST_LEN 27 +/** Length of a sha256 message digest when encoded in base64 with trailing = + * signs removed. */ +#define BASE64_DIGEST256_LEN 43 /** Constants used to indicate no padding for public-key encryption */ #define PK_NO_PADDING 60000 @@ -48,6 +54,26 @@ #define FINGERPRINT_LEN 49 /** Length of hex encoding of SHA1 digest, not including final NUL. */ #define HEX_DIGEST_LEN 40 +/** Length of hex encoding of SHA256 digest, not including final NUL. */ +#define HEX_DIGEST256_LEN 64 + +typedef enum { + DIGEST_SHA1 = 0, + DIGEST_SHA256 = 1, +} digest_algorithm_t; +#define N_DIGEST_ALGORITHMS (DIGEST_SHA256+1) + +/** A set of all the digests we know how to compute, taken on a single + * string. Any digests that are shorter than 256 bits are right-padded + * with 0 bits. + * + * Note that this representation wastes 12 bytes for the SHA1 case, so + * don't use it for anything where we need to allocate a whole bunch at + * once. + **/ +typedef struct { + char d[N_DIGEST_ALGORITHMS][DIGEST256_LEN]; +} digests_t; typedef struct crypto_pk_env_t crypto_pk_env_t; typedef struct crypto_cipher_env_t crypto_cipher_env_t; @@ -55,7 +81,9 @@ typedef struct crypto_digest_env_t crypto_digest_env_t; typedef struct crypto_dh_env_t crypto_dh_env_t; /* global state */ -int crypto_global_init(int hardwareAccel); +int crypto_global_init(int hardwareAccel, + const char *accelName, + const char *accelPath); void crypto_thread_cleanup(void); int crypto_global_cleanup(void); @@ -71,7 +99,9 @@ crypto_cipher_env_t *crypto_new_cipher_env(void); void crypto_free_cipher_env(crypto_cipher_env_t *env); /* public key crypto */ -int crypto_pk_generate_key(crypto_pk_env_t *env); +int crypto_pk_generate_key_with_bits(crypto_pk_env_t *env, int bits); +#define crypto_pk_generate_key(env) \ + crypto_pk_generate_key_with_bits((env), (PK_BYTES*8)) int crypto_pk_read_private_key_from_filename(crypto_pk_env_t *env, const char *keyfile); @@ -123,7 +153,7 @@ int crypto_pk_check_fingerprint_syntax(const char *s); /* symmetric crypto */ int crypto_cipher_generate_key(crypto_cipher_env_t *env); -int crypto_cipher_set_key(crypto_cipher_env_t *env, const char *key); +void crypto_cipher_set_key(crypto_cipher_env_t *env, const char *key); void crypto_cipher_generate_iv(char *iv_out); int crypto_cipher_set_iv(crypto_cipher_env_t *env, const char *iv); const char *crypto_cipher_get_key(crypto_cipher_env_t *env); @@ -143,9 +173,15 @@ int crypto_cipher_decrypt_with_iv(crypto_cipher_env_t *env, char *to, size_t tolen, const char *from, size_t fromlen); -/* SHA-1 */ +/* SHA-1 and other digests. */ int crypto_digest(char *digest, const char *m, size_t len); +int crypto_digest256(char *digest, const char *m, size_t len, + digest_algorithm_t algorithm); +int crypto_digest_all(digests_t *ds_out, const char *m, size_t len); +const char *crypto_digest_algorithm_get_name(digest_algorithm_t alg); +int crypto_digest_algorithm_parse_name(const char *name); crypto_digest_env_t *crypto_new_digest_env(void); +crypto_digest_env_t *crypto_new_digest256_env(digest_algorithm_t algorithm); void crypto_free_digest_env(crypto_digest_env_t *digest); void crypto_digest_add_bytes(crypto_digest_env_t *digest, const char *data, size_t len); @@ -167,7 +203,7 @@ int crypto_dh_get_bytes(crypto_dh_env_t *dh); int crypto_dh_generate_public(crypto_dh_env_t *dh); int crypto_dh_get_public(crypto_dh_env_t *dh, char *pubkey_out, size_t pubkey_out_len); -ssize_t crypto_dh_compute_secret(crypto_dh_env_t *dh, +ssize_t crypto_dh_compute_secret(int severity, crypto_dh_env_t *dh, const char *pubkey, size_t pubkey_len, char *secret_out, size_t secret_out_len); void crypto_dh_free(crypto_dh_env_t *dh); @@ -179,6 +215,7 @@ int crypto_seed_rng(int startup); int crypto_rand(char *to, size_t n); int crypto_rand_int(unsigned int max); uint64_t crypto_rand_uint64(uint64_t max); +double crypto_rand_double(void); char *crypto_random_hostname(int min_rand_len, int max_rand_len, const char *prefix, const char *suffix); @@ -196,6 +233,8 @@ int base32_decode(char *dest, size_t destlen, const char *src, size_t srclen); int digest_to_base64(char *d64, const char *digest); int digest_from_base64(char *digest, const char *d64); +int digest256_to_base64(char *d64, const char *digest); +int digest256_from_base64(char *digest, const char *d64); /** Length of RFC2440-style S2K specifier: the first 8 bytes are a salt, the * 9th describes how much iteration to do. */ @@ -210,7 +249,6 @@ struct evp_pkey_st; struct dh_st; struct rsa_st *_crypto_pk_env_get_rsa(crypto_pk_env_t *env); crypto_pk_env_t *_crypto_new_pk_env_rsa(struct rsa_st *rsa); -crypto_pk_env_t *_crypto_new_pk_env_evp_pkey(struct evp_pkey_st *pkey); struct evp_pkey_st *_crypto_pk_env_get_evp_pkey(crypto_pk_env_t *env, int private); struct dh_st *_crypto_dh_env_get_dh(crypto_dh_env_t *dh); diff --git a/src/common/ht.h b/src/common/ht.h index 033cd89ab0..0850c07092 100644 --- a/src/common/ht.h +++ b/src/common/ht.h @@ -42,6 +42,10 @@ #define HT_SIZE(head) \ ((head)->hth_n_entries) +/* Return memory usage for a hashtable (not counting the entries themselves) */ +#define HT_MEM_USAGE(head) \ + (sizeof(*head) + (head)->hth_table_length * sizeof(void*)) + #define HT_FIND(name, head, elm) name##_HT_FIND((head), (elm)) #define HT_INSERT(name, head, elm) name##_HT_INSERT((head), (elm)) #define HT_REPLACE(name, head, elm) name##_HT_REPLACE((head), (elm)) diff --git a/src/common/log.c b/src/common/log.c index fe93bd7f57..d14563c885 100644 --- a/src/common/log.c +++ b/src/common/log.c @@ -33,13 +33,15 @@ #include "compat.h" #include "util.h" #define LOG_PRIVATE -#include "log.h" +#include "torlog.h" #include "container.h" -#include <event.h> - +/** @{ */ +/** The string we stick at the end of a log message when it is too long, + * and its length. */ #define TRUNCATED_STR "[...truncated]" #define TRUNCATED_STR_LEN 14 +/** @} */ /** Information for a single logfile; only used in log.c */ typedef struct logfile_t { @@ -83,7 +85,7 @@ should_log_function_name(log_domain_mask_t domain, int severity) /* All debugging messages occur in interesting places. */ return 1; case LOG_NOTICE: - case LOG_WARN: + case LOG_WARN: case LOG_ERR: /* We care about places where bugs occur. */ return (domain == LD_BUG); @@ -99,15 +101,31 @@ static int log_mutex_initialized = 0; /** Linked list of logfile_t. */ static logfile_t *logfiles = NULL; +/** Boolean: do we report logging domains? */ +static int log_domains_are_logged = 0; + #ifdef HAVE_SYSLOG_H /** The number of open syslog log handlers that we have. When this reaches 0, * we can close our connection to the syslog facility. */ static int syslog_count = 0; #endif +/** Represents a log message that we are going to send to callback-driven + * loggers once we can do so in a non-reentrant way. */ +typedef struct pending_cb_message_t { + int severity; /**< The severity of the message */ + log_domain_mask_t domain; /**< The domain of the message */ + char *msg; /**< The content of the message */ +} pending_cb_message_t; + +/** Log messages waiting to be replayed onto callback-based logs */ +static smartlist_t *pending_cb_messages = NULL; + +/** Lock the log_mutex to prevent others from changing the logfile_t list */ #define LOCK_LOGS() STMT_BEGIN \ tor_mutex_acquire(&log_mutex); \ STMT_END +/** Unlock the log_mutex */ #define UNLOCK_LOGS() STMT_BEGIN tor_mutex_release(&log_mutex); STMT_END /** What's the lowest log level anybody cares about? Checking this lets us @@ -117,6 +135,9 @@ int _log_global_min_severity = LOG_NOTICE; static void delete_log(logfile_t *victim); static void close_log(logfile_t *victim); +static char *domain_to_string(log_domain_mask_t domain, + char *buf, size_t buflen); + /** Name of the application: used to generate the message we write at the * start of each new log. */ static char *appname = NULL; @@ -208,13 +229,34 @@ format_msg(char *buf, size_t buf_len, size_t n; int r; char *end_of_prefix; + char *buf_end; - assert(buf_len >= 2); /* prevent integer underflow */ + assert(buf_len >= 16); /* prevent integer underflow and general stupidity */ buf_len -= 2; /* subtract 2 characters so we have room for \n\0 */ + buf_end = buf+buf_len; /* point *after* the last char we can write to */ n = _log_prefix(buf, buf_len, severity); end_of_prefix = buf+n; + if (log_domains_are_logged) { + char *cp = buf+n; + if (cp == buf_end) goto format_msg_no_room_for_domains; + *cp++ = '{'; + if (cp == buf_end) goto format_msg_no_room_for_domains; + cp = domain_to_string(domain, cp, (buf+buf_len-cp)); + if (cp == buf_end) goto format_msg_no_room_for_domains; + *cp++ = '}'; + if (cp == buf_end) goto format_msg_no_room_for_domains; + *cp++ = ' '; + if (cp == buf_end) goto format_msg_no_room_for_domains; + end_of_prefix = cp; + n = cp-buf; + format_msg_no_room_for_domains: + /* This will leave end_of_prefix and n unchanged, and thus cause + * whatever log domain string we had written to be clobbered. */ + ; + } + if (funcname && should_log_function_name(domain, severity)) { r = tor_snprintf(buf+n, buf_len-n, "%s(): ", funcname); if (r<0) @@ -263,6 +305,7 @@ logv(int severity, log_domain_mask_t domain, const char *funcname, int formatted = 0; logfile_t *lf; char *end_of_prefix=NULL; + int callbacks_deferred = 0; /* Call assert, not tor_assert, since tor_assert calls log on failure. */ assert(format); @@ -270,6 +313,10 @@ logv(int severity, log_domain_mask_t domain, const char *funcname, * interesting and hard to diagnose effects */ assert(severity >= LOG_ERR && severity <= LOG_DEBUG); LOCK_LOGS(); + + if ((! (domain & LD_NOCB)) && smartlist_len(pending_cb_messages)) + flush_pending_log_callbacks(); + lf = logfiles; while (lf) { if (! (lf->severities->masks[SEVERITY_MASK_IDX(severity)] & domain)) { @@ -280,10 +327,6 @@ logv(int severity, log_domain_mask_t domain, const char *funcname, lf = lf->next; continue; } - if (lf->callback && (domain & LD_NOCB)) { - lf = lf->next; - continue; - } if (lf->seems_dead) { lf = lf->next; continue; @@ -295,6 +338,7 @@ logv(int severity, log_domain_mask_t domain, const char *funcname, &msg_len); formatted = 1; } + if (lf->is_syslog) { #ifdef HAVE_SYSLOG_H char *m = end_of_prefix; @@ -318,7 +362,19 @@ logv(int severity, log_domain_mask_t domain, const char *funcname, lf = lf->next; continue; } else if (lf->callback) { - lf->callback(severity, domain, end_of_prefix); + if (domain & LD_NOCB) { + if (!callbacks_deferred && pending_cb_messages) { + pending_cb_message_t *msg = tor_malloc(sizeof(pending_cb_message_t)); + msg->severity = severity; + msg->domain = domain; + msg->msg = tor_strdup(end_of_prefix); + smartlist_add(pending_cb_messages, msg); + + callbacks_deferred = 1; + } + } else { + lf->callback(severity, domain, end_of_prefix); + } lf = lf->next; continue; } @@ -332,9 +388,12 @@ logv(int severity, log_domain_mask_t domain, const char *funcname, UNLOCK_LOGS(); } -/** Output a message to the log. */ +/** Output a message to the log. It gets logged to all logfiles that + * care about messages with <b>severity</b> in <b>domain</b>. The content + * is formatted printf style basedc on <b>format</b> and extra arguments. + * */ void -_log(int severity, log_domain_mask_t domain, const char *format, ...) +tor_log(int severity, log_domain_mask_t domain, const char *format, ...) { va_list ap; if (severity > _log_global_min_severity) @@ -346,6 +405,9 @@ _log(int severity, log_domain_mask_t domain, const char *format, ...) /** Output a message to the log, prefixed with a function name <b>fn</b>. */ #ifdef __GNUC__ +/** GCC-based implementation of the log_fn backend, used when we have + * variadic macros. All arguments are as for log_fn, except for + * <b>fn</b>, which is the name of the calling functions. */ void _log_fn(int severity, log_domain_mask_t domain, const char *fn, const char *format, ...) @@ -358,6 +420,11 @@ _log_fn(int severity, log_domain_mask_t domain, const char *fn, va_end(ap); } #else +/** @{ */ +/** Variant implementation of log_fn, log_debug, log_info,... for C compilers + * without variadic macros. In this case, the calling function sets + * _log_fn_function_name to the name of the function, then invokes the + * appropriate _log_fn, _log_debug, etc. */ const char *_log_fn_function_name=NULL; void _log_fn(int severity, log_domain_mask_t domain, const char *format, ...) @@ -426,12 +493,15 @@ _log_err(log_domain_mask_t domain, const char *format, ...) va_end(ap); _log_fn_function_name = NULL; } +/** @} */ #endif /** Free all storage held by <b>victim</b>. */ static void log_free(logfile_t *victim) { + if (!victim) + return; tor_free(victim->severities); tor_free(victim->filename); tor_free(victim); @@ -442,9 +512,12 @@ void logs_free_all(void) { logfile_t *victim, *next; + smartlist_t *messages; LOCK_LOGS(); next = logfiles; logfiles = NULL; + messages = pending_cb_messages; + pending_cb_messages = NULL; UNLOCK_LOGS(); while (next) { victim = next; @@ -454,6 +527,12 @@ logs_free_all(void) } tor_free(appname); + SMARTLIST_FOREACH(messages, pending_cb_message_t *, msg, { + tor_free(msg->msg); + tor_free(msg); + }); + smartlist_free(messages); + /* We _could_ destroy the log mutex here, but that would screw up any logs * that happened between here and the end of execution. */ } @@ -539,8 +618,7 @@ add_stream_log_impl(const log_severity_list_t *severity, * to <b>fd</b>. Steals a reference to <b>severity</b>; the caller must * not use it after calling this function. */ void -add_stream_log(const log_severity_list_t *severity, - const char *name, int fd) +add_stream_log(const log_severity_list_t *severity, const char *name, int fd) { LOCK_LOGS(); add_stream_log_impl(severity, name, fd); @@ -555,6 +633,18 @@ init_logging(void) tor_mutex_init(&log_mutex); log_mutex_initialized = 1; } + if (pending_cb_messages == NULL) + pending_cb_messages = smartlist_create(); +} + +/** Set whether we report logging domains as a part of our log messages. + */ +void +logs_set_domain_logging(int enabled) +{ + LOCK_LOGS(); + log_domains_are_logged = enabled; + UNLOCK_LOGS(); } /** Add a log handler to receive messages during startup (before the real @@ -613,6 +703,48 @@ change_callback_log_severity(int loglevelMin, int loglevelMax, UNLOCK_LOGS(); } +/** If there are any log messages that were genered with LD_NOCB waiting to + * be sent to callback-based loggers, send them now. */ +void +flush_pending_log_callbacks(void) +{ + logfile_t *lf; + smartlist_t *messages, *messages_tmp; + + LOCK_LOGS(); + if (0 == smartlist_len(pending_cb_messages)) { + UNLOCK_LOGS(); + return; + } + + messages = pending_cb_messages; + pending_cb_messages = smartlist_create(); + do { + SMARTLIST_FOREACH_BEGIN(messages, pending_cb_message_t *, msg) { + const int severity = msg->severity; + const int domain = msg->domain; + for (lf = logfiles; lf; lf = lf->next) { + if (! lf->callback || lf->seems_dead || + ! (lf->severities->masks[SEVERITY_MASK_IDX(severity)] & domain)) { + continue; + } + lf->callback(severity, domain, msg->msg); + } + tor_free(msg->msg); + tor_free(msg); + } SMARTLIST_FOREACH_END(msg); + smartlist_clear(messages); + + messages_tmp = pending_cb_messages; + pending_cb_messages = messages; + messages = messages_tmp; + } while (smartlist_len(messages)); + + smartlist_free(messages); + + UNLOCK_LOGS(); +} + /** Close any log handlers added by add_temp_log() or marked by * mark_logs_temp(). */ void @@ -682,13 +814,11 @@ add_file_log(const log_severity_list_t *severity, const char *filename) logfiles->needs_close = 1; lf = logfiles; _log_global_min_severity = get_min_log_level(); - UNLOCK_LOGS(); if (log_tor_version(lf, 0) < 0) { - LOCK_LOGS(); delete_log(lf); - UNLOCK_LOGS(); } + UNLOCK_LOGS(); return 0; } @@ -709,7 +839,6 @@ add_syslog_log(const log_severity_list_t *severity) lf->fd = -1; lf->severities = tor_memdup(severity, sizeof(log_severity_list_t)); lf->filename = tor_strdup("<syslog>"); - lf->is_syslog = 1; LOCK_LOGS(); @@ -751,7 +880,7 @@ log_level_to_string(int level) static const char *domain_list[] = { "GENERAL", "CRYPTO", "NET", "CONFIG", "FS", "PROTOCOL", "MM", "HTTP", "APP", "CONTROL", "CIRC", "REND", "BUG", "DIR", "DIRSERV", - "OR", "EDGE", "ACCT", "HIST", NULL + "OR", "EDGE", "ACCT", "HIST", "HANDSHAKE", NULL }; /** Return a bitmask for the log domain for which <b>domain</b> is the name, @@ -766,18 +895,41 @@ parse_log_domain(const char *domain) } return 0; } -#if 0 -/** Translate a bitmask of log domains to a string, or NULL if the bitmask - * is undecodable. */ -static const char * -domain_to_string(log_domain_mask_t domain) + +/** Translate a bitmask of log domains to a string. */ +static char * +domain_to_string(log_domain_mask_t domain, char *buf, size_t buflen) { - int bit = tor_log2(domain); - if ((bit == 0 && domain == 0) || bit >= N_LOGGING_DOMAINS) - return NULL; - return domain_list[bit]; + char *cp = buf; + char *eos = buf+buflen; + + buf[0] = '\0'; + if (! domain) + return buf; + while (1) { + const char *d; + int bit = tor_log2(domain); + size_t n; + if (bit >= N_LOGGING_DOMAINS) { + tor_snprintf(buf, buflen, "<BUG:Unknown domain %lx>", (long)domain); + return buf+strlen(buf); + } + d = domain_list[bit]; + n = strlcpy(cp, d, eos-cp); + if (n >= buflen) { + tor_snprintf(buf, buflen, "<BUG:Truncating domain %lx>", (long)domain); + return buf+strlen(buf); + } + cp += n; + domain &= ~(1<<bit); + + if (domain == 0 || (eos-cp) < 2) + return cp; + + memcpy(cp, ",", 2); /*Nul-terminated ,"*/ + cp++; + } } -#endif /** Parse a log severity pattern in *<b>cfg_ptr</b>. Advance cfg_ptr after * the end of the severityPattern. Set the value of <b>severity_out</b> to @@ -853,7 +1005,10 @@ parse_log_severity_config(const char **cfg_ptr, smartlist_free(domains_list); if (err) return -1; - domains &= ~neg_domains; + if (domains == 0 && neg_domains) + domains = ~neg_domains; + else + domains &= ~neg_domains; cfg = eat_whitespace(closebracket+1); } else { ++got_an_unqualified_range; @@ -929,65 +1084,6 @@ switch_logs_debug(void) UNLOCK_LOGS(); } -#ifdef HAVE_EVENT_SET_LOG_CALLBACK -/** A string which, if it appears in a libevent log, should be ignored. */ -static const char *suppress_msg = NULL; -/** Callback function passed to event_set_log() so we can intercept - * log messages from libevent. */ -static void -libevent_logging_callback(int severity, const char *msg) -{ - char buf[1024]; - size_t n; - if (suppress_msg && strstr(msg, suppress_msg)) - return; - n = strlcpy(buf, msg, sizeof(buf)); - if (n && n < sizeof(buf) && buf[n-1] == '\n') { - buf[n-1] = '\0'; - } - switch (severity) { - case _EVENT_LOG_DEBUG: - log(LOG_DEBUG, LD_NOCB|LD_NET, "Message from libevent: %s", buf); - break; - case _EVENT_LOG_MSG: - log(LOG_INFO, LD_NOCB|LD_NET, "Message from libevent: %s", buf); - break; - case _EVENT_LOG_WARN: - log(LOG_WARN, LD_NOCB|LD_GENERAL, "Warning from libevent: %s", buf); - break; - case _EVENT_LOG_ERR: - log(LOG_ERR, LD_NOCB|LD_GENERAL, "Error from libevent: %s", buf); - break; - default: - log(LOG_WARN, LD_NOCB|LD_GENERAL, "Message [%d] from libevent: %s", - severity, buf); - break; - } -} -/** Set hook to intercept log messages from libevent. */ -void -configure_libevent_logging(void) -{ - event_set_log_callback(libevent_logging_callback); -} -/** Ignore any libevent log message that contains <b>msg</b>. */ -void -suppress_libevent_log_msg(const char *msg) -{ - suppress_msg = msg; -} -#else -void -configure_libevent_logging(void) -{ -} -void -suppress_libevent_log_msg(const char *msg) -{ - (void)msg; -} -#endif - #if 0 static void dump_log_info(logfile_t *lf) diff --git a/src/common/memarea.c b/src/common/memarea.c index ac26c5fd90..a6b8c4ee9c 100644 --- a/src/common/memarea.c +++ b/src/common/memarea.c @@ -11,7 +11,11 @@ #include "memarea.h" #include "util.h" #include "compat.h" -#include "log.h" +#include "torlog.h" + +/** If true, we try to detect any attempts to write beyond the length of a + * memarea. */ +#define USE_SENTINELS /** All returned pointers should be aligned to the nearest multiple of this * value. */ @@ -25,13 +29,39 @@ #error "void* is neither 4 nor 8 bytes long. I don't know how to align stuff." #endif +#ifdef USE_SENTINELS +/** Magic value that we stick at the end of a memarea so we can make sure + * there are no run-off-the-end bugs. */ +#define SENTINEL_VAL 0x90806622u +/** How many bytes per area do we devote to the sentinel? */ +#define SENTINEL_LEN sizeof(uint32_t) +/** Given a mem_area_chunk_t with SENTINEL_LEN extra bytes allocated at the + * end, set those bytes. */ +#define SET_SENTINEL(chunk) \ + STMT_BEGIN \ + set_uint32( &(chunk)->u.mem[chunk->mem_size], SENTINEL_VAL ); \ + STMT_END +/** Assert that the sentinel on a memarea is set correctly. */ +#define CHECK_SENTINEL(chunk) \ + STMT_BEGIN \ + uint32_t sent_val = get_uint32(&(chunk)->u.mem[chunk->mem_size]); \ + tor_assert(sent_val == SENTINEL_VAL); \ + STMT_END +#else +#define SENTINEL_LEN 0 +#define SET_SENTINEL(chunk) STMT_NIL +#define CHECK_SENTINEL(chunk) STMT_NIL +#endif + /** Increment <b>ptr</b> until it is aligned to MEMAREA_ALIGN. */ static INLINE void * realign_pointer(void *ptr) { uintptr_t x = (uintptr_t)ptr; x = (x+MEMAREA_ALIGN_MASK) & ~MEMAREA_ALIGN_MASK; - tor_assert(((void*)x) >= ptr); // XXXX021 remove this once bug 930 is solved + /* Reinstate this if bug 930 ever reappears + tor_assert(((void*)x) >= ptr); + */ return (void*)x; } @@ -51,8 +81,11 @@ typedef struct memarea_chunk_t { } u; } memarea_chunk_t; +/** How many bytes are needed for overhead before we get to the memory part + * of a chunk? */ #define CHUNK_HEADER_SIZE STRUCT_OFFSET(memarea_chunk_t, u) +/** What's the smallest that we'll allocate a chunk? */ #define CHUNK_SIZE 4096 /** A memarea_t is an allocation region for a set of small memory requests @@ -79,15 +112,20 @@ alloc_chunk(size_t sz, int freelist_ok) freelist = res->next_chunk; res->next_chunk = NULL; --freelist_len; + CHECK_SENTINEL(res); return res; } else { size_t chunk_size = freelist_ok ? CHUNK_SIZE : sz; - memarea_chunk_t *res = tor_malloc_roundup(&chunk_size); + memarea_chunk_t *res; + chunk_size += SENTINEL_LEN; + res = tor_malloc_roundup(&chunk_size); res->next_chunk = NULL; - res->mem_size = chunk_size - CHUNK_HEADER_SIZE; + res->mem_size = chunk_size - CHUNK_HEADER_SIZE - SENTINEL_LEN; res->next_mem = res->u.mem; - tor_assert(res->next_mem+res->mem_size == ((char*)res)+chunk_size); + tor_assert(res->next_mem+res->mem_size+SENTINEL_LEN == + ((char*)res)+chunk_size); tor_assert(realign_pointer(res->next_mem) == res->next_mem); + SET_SENTINEL(res); return res; } } @@ -95,8 +133,9 @@ alloc_chunk(size_t sz, int freelist_ok) /** Release <b>chunk</b> from a memarea, either by adding it to the freelist * or by freeing it if the freelist is already too big. */ static void -chunk_free(memarea_chunk_t *chunk) +chunk_free_unchecked(memarea_chunk_t *chunk) { + CHECK_SENTINEL(chunk); if (freelist_len < MAX_FREELIST_LEN) { ++freelist_len; chunk->next_chunk = freelist; @@ -124,7 +163,7 @@ memarea_drop_all(memarea_t *area) memarea_chunk_t *chunk, *next; for (chunk = area->first; chunk; chunk = next) { next = chunk->next_chunk; - chunk_free(chunk); + chunk_free_unchecked(chunk); } area->first = NULL; /*fail fast on */ tor_free(area); @@ -140,7 +179,7 @@ memarea_clear(memarea_t *area) if (area->first->next_chunk) { for (chunk = area->first->next_chunk; chunk; chunk = next) { next = chunk->next_chunk; - chunk_free(chunk); + chunk_free_unchecked(chunk); } area->first->next_chunk = NULL; } @@ -183,6 +222,7 @@ memarea_alloc(memarea_t *area, size_t sz) memarea_chunk_t *chunk = area->first; char *result; tor_assert(chunk); + CHECK_SENTINEL(chunk); tor_assert(sz < SIZE_T_CEILING); if (sz == 0) sz = 1; @@ -203,9 +243,10 @@ memarea_alloc(memarea_t *area, size_t sz) } result = chunk->next_mem; chunk->next_mem = chunk->next_mem + sz; - // XXXX021 remove these once bug 930 is solved. + /* Reinstate these if bug 930 ever comes back tor_assert(chunk->next_mem >= chunk->u.mem); tor_assert(chunk->next_mem <= chunk->u.mem+chunk->mem_size); + */ chunk->next_mem = realign_pointer(chunk->next_mem); return result; } @@ -261,6 +302,7 @@ memarea_get_stats(memarea_t *area, size_t *allocated_out, size_t *used_out) size_t a = 0, u = 0; memarea_chunk_t *chunk; for (chunk = area->first; chunk; chunk = chunk->next_chunk) { + CHECK_SENTINEL(chunk); a += CHUNK_HEADER_SIZE + chunk->mem_size; tor_assert(chunk->next_mem >= chunk->u.mem); u += CHUNK_HEADER_SIZE + (chunk->next_mem - chunk->u.mem); @@ -277,6 +319,7 @@ memarea_assert_ok(memarea_t *area) tor_assert(area->first); for (chunk = area->first; chunk; chunk = chunk->next_chunk) { + CHECK_SENTINEL(chunk); tor_assert(chunk->next_mem >= chunk->u.mem); tor_assert(chunk->next_mem <= (char*) realign_pointer(chunk->u.mem+chunk->mem_size)); diff --git a/src/common/mempool.c b/src/common/mempool.c index 9538a05489..c444923189 100644 --- a/src/common/mempool.c +++ b/src/common/mempool.c @@ -65,7 +65,7 @@ #include "orconfig.h" #include "util.h" #include "compat.h" -#include "log.h" +#include "torlog.h" #define ALLOC(x) tor_malloc(x) #define FREE(x) tor_free(x) #define ASSERT(x) tor_assert(x) diff --git a/src/common/sha256.c b/src/common/sha256.c new file mode 100644 index 0000000000..258b7e062a --- /dev/null +++ b/src/common/sha256.c @@ -0,0 +1,331 @@ +/* Copyright (c) 2009-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ +/* This SHA256 implementation is adapted from the public domain one in + LibTomCrypt, version 1.6. Tor uses it on platforms where OpenSSL doesn't + have a SHA256. */ + + +typedef struct sha256_state { + uint64_t length; + uint32_t state[8], curlen; + unsigned char buf[64]; +} sha256_state; + +#define CRYPT_OK 0 +#define CRYPT_NOP -1 +#define CRYPT_INVALID_ARG -2 + +#define LOAD32H(x,y) STMT_BEGIN x = ntohl(get_uint32((const char*)y)); STMT_END +#define STORE32H(x,y) STMT_BEGIN set_uint32((char*)y, htonl(x)); STMT_END +#define STORE64H(x,y) STMT_BEGIN \ + set_uint32((char*)y, htonl((uint32_t)((x)>>32))); \ + set_uint32(((char*)y)+4, htonl((uint32_t)((x)&0xffffffff))); \ + STMT_END +#define RORc(x, y) ( ((((unsigned long)(x)&0xFFFFFFFFUL)>>(unsigned long)((y)&31)) | ((unsigned long)(x)<<(unsigned long)(32-((y)&31)))) & 0xFFFFFFFFUL) +#ifndef MIN + #define MIN(x, y) ( ((x)<(y))?(x):(y) ) +#endif + + +/* LibTomCrypt, modular cryptographic library -- Tom St Denis + * + * LibTomCrypt is a library that provides various cryptographic + * algorithms in a highly modular and flexible manner. + * + * The library is free for all purposes without any express + * guarantee it works. + * + * Tom St Denis, tomstdenis@gmail.com, http://libtomcrypt.com + */ + +/** + @file sha256.c + SHA256 by Tom St Denis +*/ + + +#ifdef LTC_SMALL_CODE +/* the K array */ +static const uint32_t K[64] = { + 0x428a2f98UL, 0x71374491UL, 0xb5c0fbcfUL, 0xe9b5dba5UL, 0x3956c25bUL, + 0x59f111f1UL, 0x923f82a4UL, 0xab1c5ed5UL, 0xd807aa98UL, 0x12835b01UL, + 0x243185beUL, 0x550c7dc3UL, 0x72be5d74UL, 0x80deb1feUL, 0x9bdc06a7UL, + 0xc19bf174UL, 0xe49b69c1UL, 0xefbe4786UL, 0x0fc19dc6UL, 0x240ca1ccUL, + 0x2de92c6fUL, 0x4a7484aaUL, 0x5cb0a9dcUL, 0x76f988daUL, 0x983e5152UL, + 0xa831c66dUL, 0xb00327c8UL, 0xbf597fc7UL, 0xc6e00bf3UL, 0xd5a79147UL, + 0x06ca6351UL, 0x14292967UL, 0x27b70a85UL, 0x2e1b2138UL, 0x4d2c6dfcUL, + 0x53380d13UL, 0x650a7354UL, 0x766a0abbUL, 0x81c2c92eUL, 0x92722c85UL, + 0xa2bfe8a1UL, 0xa81a664bUL, 0xc24b8b70UL, 0xc76c51a3UL, 0xd192e819UL, + 0xd6990624UL, 0xf40e3585UL, 0x106aa070UL, 0x19a4c116UL, 0x1e376c08UL, + 0x2748774cUL, 0x34b0bcb5UL, 0x391c0cb3UL, 0x4ed8aa4aUL, 0x5b9cca4fUL, + 0x682e6ff3UL, 0x748f82eeUL, 0x78a5636fUL, 0x84c87814UL, 0x8cc70208UL, + 0x90befffaUL, 0xa4506cebUL, 0xbef9a3f7UL, 0xc67178f2UL +}; +#endif + +/* Various logical functions */ +#define Ch(x,y,z) (z ^ (x & (y ^ z))) +#define Maj(x,y,z) (((x | y) & z) | (x & y)) +#define S(x, n) RORc((x),(n)) +#define R(x, n) (((x)&0xFFFFFFFFUL)>>(n)) +#define Sigma0(x) (S(x, 2) ^ S(x, 13) ^ S(x, 22)) +#define Sigma1(x) (S(x, 6) ^ S(x, 11) ^ S(x, 25)) +#define Gamma0(x) (S(x, 7) ^ S(x, 18) ^ R(x, 3)) +#define Gamma1(x) (S(x, 17) ^ S(x, 19) ^ R(x, 10)) + +/* compress 512-bits */ +#ifdef LTC_CLEAN_STACK +static int _sha256_compress(sha256_state * md, unsigned char *buf) +#else +static int sha256_compress(sha256_state * md, unsigned char *buf) +#endif +{ + uint32_t S[8], W[64], t0, t1; +#ifdef LTC_SMALL_CODE + uint32_t t; +#endif + int i; + + /* copy state into S */ + for (i = 0; i < 8; i++) { + S[i] = md->state[i]; + } + + /* copy the state into 512-bits into W[0..15] */ + for (i = 0; i < 16; i++) { + LOAD32H(W[i], buf + (4*i)); + } + + /* fill W[16..63] */ + for (i = 16; i < 64; i++) { + W[i] = Gamma1(W[i - 2]) + W[i - 7] + Gamma0(W[i - 15]) + W[i - 16]; + } + + /* Compress */ +#ifdef LTC_SMALL_CODE +#define RND(a,b,c,d,e,f,g,h,i) \ + t0 = h + Sigma1(e) + Ch(e, f, g) + K[i] + W[i]; \ + t1 = Sigma0(a) + Maj(a, b, c); \ + d += t0; \ + h = t0 + t1; + + for (i = 0; i < 64; ++i) { + RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],i); + t = S[7]; S[7] = S[6]; S[6] = S[5]; S[5] = S[4]; + S[4] = S[3]; S[3] = S[2]; S[2] = S[1]; S[1] = S[0]; S[0] = t; + } +#else +#define RND(a,b,c,d,e,f,g,h,i,ki) \ + t0 = h + Sigma1(e) + Ch(e, f, g) + ki + W[i]; \ + t1 = Sigma0(a) + Maj(a, b, c); \ + d += t0; \ + h = t0 + t1; + + RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],0,0x428a2f98); + RND(S[7],S[0],S[1],S[2],S[3],S[4],S[5],S[6],1,0x71374491); + RND(S[6],S[7],S[0],S[1],S[2],S[3],S[4],S[5],2,0xb5c0fbcf); + RND(S[5],S[6],S[7],S[0],S[1],S[2],S[3],S[4],3,0xe9b5dba5); + RND(S[4],S[5],S[6],S[7],S[0],S[1],S[2],S[3],4,0x3956c25b); + RND(S[3],S[4],S[5],S[6],S[7],S[0],S[1],S[2],5,0x59f111f1); + RND(S[2],S[3],S[4],S[5],S[6],S[7],S[0],S[1],6,0x923f82a4); + RND(S[1],S[2],S[3],S[4],S[5],S[6],S[7],S[0],7,0xab1c5ed5); + RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],8,0xd807aa98); + RND(S[7],S[0],S[1],S[2],S[3],S[4],S[5],S[6],9,0x12835b01); + RND(S[6],S[7],S[0],S[1],S[2],S[3],S[4],S[5],10,0x243185be); + RND(S[5],S[6],S[7],S[0],S[1],S[2],S[3],S[4],11,0x550c7dc3); + RND(S[4],S[5],S[6],S[7],S[0],S[1],S[2],S[3],12,0x72be5d74); + RND(S[3],S[4],S[5],S[6],S[7],S[0],S[1],S[2],13,0x80deb1fe); + RND(S[2],S[3],S[4],S[5],S[6],S[7],S[0],S[1],14,0x9bdc06a7); + RND(S[1],S[2],S[3],S[4],S[5],S[6],S[7],S[0],15,0xc19bf174); + RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],16,0xe49b69c1); + RND(S[7],S[0],S[1],S[2],S[3],S[4],S[5],S[6],17,0xefbe4786); + RND(S[6],S[7],S[0],S[1],S[2],S[3],S[4],S[5],18,0x0fc19dc6); + RND(S[5],S[6],S[7],S[0],S[1],S[2],S[3],S[4],19,0x240ca1cc); + RND(S[4],S[5],S[6],S[7],S[0],S[1],S[2],S[3],20,0x2de92c6f); + RND(S[3],S[4],S[5],S[6],S[7],S[0],S[1],S[2],21,0x4a7484aa); + RND(S[2],S[3],S[4],S[5],S[6],S[7],S[0],S[1],22,0x5cb0a9dc); + RND(S[1],S[2],S[3],S[4],S[5],S[6],S[7],S[0],23,0x76f988da); + RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],24,0x983e5152); + RND(S[7],S[0],S[1],S[2],S[3],S[4],S[5],S[6],25,0xa831c66d); + RND(S[6],S[7],S[0],S[1],S[2],S[3],S[4],S[5],26,0xb00327c8); + RND(S[5],S[6],S[7],S[0],S[1],S[2],S[3],S[4],27,0xbf597fc7); + RND(S[4],S[5],S[6],S[7],S[0],S[1],S[2],S[3],28,0xc6e00bf3); + RND(S[3],S[4],S[5],S[6],S[7],S[0],S[1],S[2],29,0xd5a79147); + RND(S[2],S[3],S[4],S[5],S[6],S[7],S[0],S[1],30,0x06ca6351); + RND(S[1],S[2],S[3],S[4],S[5],S[6],S[7],S[0],31,0x14292967); + RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],32,0x27b70a85); + RND(S[7],S[0],S[1],S[2],S[3],S[4],S[5],S[6],33,0x2e1b2138); + RND(S[6],S[7],S[0],S[1],S[2],S[3],S[4],S[5],34,0x4d2c6dfc); + RND(S[5],S[6],S[7],S[0],S[1],S[2],S[3],S[4],35,0x53380d13); + RND(S[4],S[5],S[6],S[7],S[0],S[1],S[2],S[3],36,0x650a7354); + RND(S[3],S[4],S[5],S[6],S[7],S[0],S[1],S[2],37,0x766a0abb); + RND(S[2],S[3],S[4],S[5],S[6],S[7],S[0],S[1],38,0x81c2c92e); + RND(S[1],S[2],S[3],S[4],S[5],S[6],S[7],S[0],39,0x92722c85); + RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],40,0xa2bfe8a1); + RND(S[7],S[0],S[1],S[2],S[3],S[4],S[5],S[6],41,0xa81a664b); + RND(S[6],S[7],S[0],S[1],S[2],S[3],S[4],S[5],42,0xc24b8b70); + RND(S[5],S[6],S[7],S[0],S[1],S[2],S[3],S[4],43,0xc76c51a3); + RND(S[4],S[5],S[6],S[7],S[0],S[1],S[2],S[3],44,0xd192e819); + RND(S[3],S[4],S[5],S[6],S[7],S[0],S[1],S[2],45,0xd6990624); + RND(S[2],S[3],S[4],S[5],S[6],S[7],S[0],S[1],46,0xf40e3585); + RND(S[1],S[2],S[3],S[4],S[5],S[6],S[7],S[0],47,0x106aa070); + RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],48,0x19a4c116); + RND(S[7],S[0],S[1],S[2],S[3],S[4],S[5],S[6],49,0x1e376c08); + RND(S[6],S[7],S[0],S[1],S[2],S[3],S[4],S[5],50,0x2748774c); + RND(S[5],S[6],S[7],S[0],S[1],S[2],S[3],S[4],51,0x34b0bcb5); + RND(S[4],S[5],S[6],S[7],S[0],S[1],S[2],S[3],52,0x391c0cb3); + RND(S[3],S[4],S[5],S[6],S[7],S[0],S[1],S[2],53,0x4ed8aa4a); + RND(S[2],S[3],S[4],S[5],S[6],S[7],S[0],S[1],54,0x5b9cca4f); + RND(S[1],S[2],S[3],S[4],S[5],S[6],S[7],S[0],55,0x682e6ff3); + RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],56,0x748f82ee); + RND(S[7],S[0],S[1],S[2],S[3],S[4],S[5],S[6],57,0x78a5636f); + RND(S[6],S[7],S[0],S[1],S[2],S[3],S[4],S[5],58,0x84c87814); + RND(S[5],S[6],S[7],S[0],S[1],S[2],S[3],S[4],59,0x8cc70208); + RND(S[4],S[5],S[6],S[7],S[0],S[1],S[2],S[3],60,0x90befffa); + RND(S[3],S[4],S[5],S[6],S[7],S[0],S[1],S[2],61,0xa4506ceb); + RND(S[2],S[3],S[4],S[5],S[6],S[7],S[0],S[1],62,0xbef9a3f7); + RND(S[1],S[2],S[3],S[4],S[5],S[6],S[7],S[0],63,0xc67178f2); + +#undef RND + +#endif + + /* feedback */ + for (i = 0; i < 8; i++) { + md->state[i] = md->state[i] + S[i]; + } + return CRYPT_OK; +} + +#ifdef LTC_CLEAN_STACK +static int sha256_compress(sha256_state * md, unsigned char *buf) +{ + int err; + err = _sha256_compress(md, buf); + burn_stack(sizeof(uint32_t) * 74); + return err; +} +#endif + +/** + Initialize the hash state + @param md The hash state you wish to initialize + @return CRYPT_OK if successful +*/ +static int sha256_init(sha256_state * md) +{ + LTC_ARGCHK(md != NULL); + + md->curlen = 0; + md->length = 0; + md->state[0] = 0x6A09E667UL; + md->state[1] = 0xBB67AE85UL; + md->state[2] = 0x3C6EF372UL; + md->state[3] = 0xA54FF53AUL; + md->state[4] = 0x510E527FUL; + md->state[5] = 0x9B05688CUL; + md->state[6] = 0x1F83D9ABUL; + md->state[7] = 0x5BE0CD19UL; + return CRYPT_OK; +} + +/** + Process a block of memory though the hash + @param md The hash state + @param in The data to hash + @param inlen The length of the data (octets) + @return CRYPT_OK if successful +*/ +static int sha256_process (sha256_state * md, const unsigned char *in, unsigned long inlen) +{ + unsigned long n; + int err; + LTC_ARGCHK(md != NULL); + LTC_ARGCHK(in != NULL); + if (md->curlen > sizeof(md->buf)) { + return CRYPT_INVALID_ARG; + } + while (inlen > 0) { + if (md->curlen == 0 && inlen >= 64) { + if ((err = sha256_compress (md, (unsigned char *)in)) != CRYPT_OK) { + return err; + } + md->length += 64 * 8; + in += 64; + inlen -= 64; + } else { + n = MIN(inlen, (64 - md->curlen)); + memcpy(md->buf + md->curlen, in, (size_t)n); + md->curlen += n; + in += n; + inlen -= n; + if (md->curlen == 64) { + if ((err = sha256_compress (md, md->buf)) != CRYPT_OK) { + return err; + } + md->length += 8*64; + md->curlen = 0; + } + } + } + return CRYPT_OK; +} + +/** + Terminate the hash to get the digest + @param md The hash state + @param out [out] The destination of the hash (32 bytes) + @return CRYPT_OK if successful +*/ +static int sha256_done(sha256_state * md, unsigned char *out) +{ + int i; + + LTC_ARGCHK(md != NULL); + LTC_ARGCHK(out != NULL); + + if (md->curlen >= sizeof(md->buf)) { + return CRYPT_INVALID_ARG; + } + + + /* increase the length of the message */ + md->length += md->curlen * 8; + + /* append the '1' bit */ + md->buf[md->curlen++] = (unsigned char)0x80; + + /* if the length is currently above 56 bytes we append zeros + * then compress. Then we can fall back to padding zeros and length + * encoding like normal. + */ + if (md->curlen > 56) { + while (md->curlen < 64) { + md->buf[md->curlen++] = (unsigned char)0; + } + sha256_compress(md, md->buf); + md->curlen = 0; + } + + /* pad upto 56 bytes of zeroes */ + while (md->curlen < 56) { + md->buf[md->curlen++] = (unsigned char)0; + } + + /* store length */ + STORE64H(md->length, md->buf+56); + sha256_compress(md, md->buf); + + /* copy output */ + for (i = 0; i < 8; i++) { + STORE32H(md->state[i], out+(4*i)); + } +#ifdef LTC_CLEAN_STACK + zeromem(md, sizeof(sha256_state)); +#endif + return CRYPT_OK; +} + +/* $Source: /cvs/libtom/libtomcrypt/src/hashes/sha2/sha256.c,v $ */ +/* $Revision: 1.9 $ */ +/* $Date: 2006/11/01 09:28:17 $ */ diff --git a/src/common/test.h b/src/common/test.h deleted file mode 100644 index c5995d1454..0000000000 --- a/src/common/test.h +++ /dev/null @@ -1,184 +0,0 @@ -/* Copyright (c) 2001-2003, Roger Dingledine. - * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2011, The Tor Project, Inc. */ -/* See LICENSE for licensing information */ - -#ifndef _TOR_TEST_H -#define _TOR_TEST_H - -/** - * \file test.h - * \brief Macros used by unit tests. - */ - -#include "compat.h" - -#ifdef __GNUC__ -#define PRETTY_FUNCTION __PRETTY_FUNCTION__ -#else -#define PRETTY_FUNCTION "" -#endif - -#define test_fail_msg(msg) \ - STMT_BEGIN \ - have_failed = 1; \ - printf("\nFile %s: line %d (%s): %s", \ - _SHORT_FILE_, \ - __LINE__, \ - PRETTY_FUNCTION, \ - msg); \ - goto done; \ - STMT_END - -#define test_fail() test_fail_msg("Assertion failed.") - -#define test_assert(expr) \ - STMT_BEGIN \ - if (expr) { printf("."); fflush(stdout); } else { \ - have_failed = 1; \ - printf("\nFile %s: line %d (%s): assertion failed: (%s)\n", \ - _SHORT_FILE_, \ - __LINE__, \ - PRETTY_FUNCTION, \ - #expr); \ - goto done; \ - } STMT_END - -#define test_eq_type(tp, fmt, expr1, expr2) \ - STMT_BEGIN \ - tp _test_v1=(tp)(expr1); \ - tp _test_v2=(tp)(expr2); \ - if (_test_v1==_test_v2) { printf("."); fflush(stdout); } else { \ - have_failed = 1; \ - printf("\nFile %s: line %d (%s): Assertion failed: (%s==%s)\n" \ - " "fmt "!="fmt"\n", \ - _SHORT_FILE_, \ - __LINE__, \ - PRETTY_FUNCTION, \ - #expr1, #expr2, \ - _test_v1, _test_v2); \ - goto done; \ - } STMT_END - -#define test_eq(expr1, expr2) \ - test_eq_type(long, "%ld", expr1, expr2) - -#define test_eq_ptr(expr1, expr2) \ - test_eq_type(void*, "%p", expr1, expr2) - -#define test_neq_type(tp, fmt, expr1, expr2) \ - STMT_BEGIN \ - tp _test_v1=(tp)(expr1); \ - tp _test_v2=(tp)(expr2); \ - if (_test_v1!=_test_v2) { printf("."); fflush(stdout); } else { \ - have_failed = 1; \ - printf("\nFile %s: line %d (%s): Assertion failed: (%s!=%s)\n" \ - " ("fmt" == "fmt")\n", \ - _SHORT_FILE_, \ - __LINE__, \ - PRETTY_FUNCTION, \ - #expr1, #expr2, \ - _test_v1, _test_v2); \ - goto done; \ - } STMT_END - -#define test_neq(expr1, expr2) \ - test_neq_type(long, "%ld", expr1, expr2) - -#define test_neq_ptr(expr1, expr2) \ - test_neq_type(void *, "%p", expr1, expr2) - -#define test_streq(expr1, expr2) \ - STMT_BEGIN \ - const char *_test_v1=(expr1), *_test_v2=(expr2); \ - if (!strcmp(_test_v1,_test_v2)) { printf("."); fflush(stdout); } else { \ - have_failed = 1; \ - printf("\nFile %s: line %d (%s): Assertion failed: (%s==%s)\n"\ - " (\"%s\" != \"%s\")\n", \ - _SHORT_FILE_, \ - __LINE__, \ - PRETTY_FUNCTION, \ - #expr1, #expr2, \ - _test_v1, _test_v2); \ - goto done; \ - } STMT_END - -#define test_strneq(expr1, expr2) \ - STMT_BEGIN \ - const char *_test_v1=(expr1), *_test_v2=(expr2); \ - if (strcmp(_test_v1,_test_v2)) { printf("."); fflush(stdout); } else { \ - have_failed = 1; \ - printf("\nFile %s: line %d (%s): Assertion failed: (%s!=%s)\n"\ - " (\"%s\" == \"%s\")\n", \ - _SHORT_FILE_, \ - __LINE__, \ - PRETTY_FUNCTION, \ - #expr1, #expr2, \ - _test_v1, _test_v2); \ - goto done; \ - } STMT_END - -#define test_memeq(expr1, expr2, len) \ - STMT_BEGIN \ - const void *_test_v1=(expr1), *_test_v2=(expr2); \ - char *mem1, *mem2; \ - if (!memcmp(_test_v1,_test_v2,(len))) { \ - printf("."); fflush(stdout); } else { \ - have_failed = 1; \ - mem1 = tor_malloc(len*2+1); \ - mem2 = tor_malloc(len*2+1); \ - base16_encode(mem1, len*2+1, _test_v1, len); \ - base16_encode(mem2, len*2+1, _test_v2, len); \ - printf("\nFile %s: line %d (%s): Assertion failed: (%s==%s)\n" \ - " %s != %s\n", \ - _SHORT_FILE_, \ - __LINE__, \ - PRETTY_FUNCTION, \ - #expr1, #expr2, mem1, mem2); \ - tor_free(mem1); \ - tor_free(mem2); \ - goto done; \ - } STMT_END - -#define test_memeq_hex(expr1, hex) \ - STMT_BEGIN \ - const char *_test_v1 = (char*)(expr1); \ - const char *_test_v2 = (hex); \ - size_t _len_v2 = strlen(_test_v2); \ - char *_mem2 = tor_malloc(_len_v2/2); \ - tor_assert((_len_v2 & 1) == 0); \ - base16_decode(_mem2, _len_v2/2, _test_v2, _len_v2); \ - if (!memcmp(_mem2, _test_v1, _len_v2/2)) { \ - printf("."); fflush(stdout); } else { \ - char *_mem1 = tor_malloc(_len_v2+1); \ - base16_encode(_mem1, _len_v2+1, _test_v1, _len_v2/2); \ - printf("\nFile %s: line %d (%s): Assertion failed: (%s==%s)\n" \ - " %s != %s\n", \ - _SHORT_FILE_, \ - __LINE__, \ - PRETTY_FUNCTION, \ - #expr1, _test_v2, _mem1, _test_v2); \ - tor_free(_mem1); \ - tor_free(_mem2); \ - goto done; \ - } \ - tor_free(_mem2); \ - STMT_END - -#define test_memneq(expr1, expr2, len) \ - STMT_BEGIN \ - void *_test_v1=(expr1), *_test_v2=(expr2); \ - if (memcmp(_test_v1,_test_v2,(len))) { \ - printf("."); fflush(stdout); \ - } else { \ - have_failed = 1; \ - printf("\nFile %s: line %d (%s): Assertion failed: (%s!=%s)\n", \ - _SHORT_FILE_, \ - __LINE__, \ - PRETTY_FUNCTION, \ - #expr1, #expr2); \ - goto done; \ - } STMT_END - -#endif - diff --git a/src/common/torgzip.c b/src/common/torgzip.c index f5709aaf3c..2937c67de2 100644 --- a/src/common/torgzip.c +++ b/src/common/torgzip.c @@ -13,20 +13,42 @@ #include <stdlib.h> #include <stdio.h> #include <assert.h> -#ifdef _MSC_VER -#include "..\..\contrib\zlib\zlib.h" -#else -#include <zlib.h> -#endif #include <string.h> #ifdef HAVE_NETINET_IN_H #include <netinet/in.h> #endif +#include "torint.h" #include "util.h" -#include "log.h" +#include "torlog.h" #include "torgzip.h" +/* zlib 1.2.4 and 1.2.5 do some "clever" things with macros. Instead of + saying "(defined(FOO) ? FOO : 0)" they like to say "FOO-0", on the theory + that nobody will care if the compile outputs a no-such-identifier warning. + + Sorry, but we like -Werror over here, so I guess we need to define these. + I hope that zlib 1.2.6 doesn't break these too. +*/ +#ifndef _LARGEFILE64_SOURCE +#define _LARGEFILE64_SOURCE 0 +#endif +#ifndef _LFS64_LARGEFILE +#define _LFS64_LARGEFILE 0 +#endif +#ifndef _FILE_OFFSET_BITS +#define _FILE_OFFSET_BITS 0 +#endif +#ifndef off64_t +#define off64_t int64_t +#endif + +#ifdef _MSC_VER +#include "..\..\contrib\zlib\zlib.h" +#else +#include <zlib.h> +#endif + /** Set to 1 if zlib is a version that supports gzip; set to 0 if it doesn't; * set to -1 if we haven't checked yet. */ static int gzip_is_supported = -1; @@ -57,6 +79,7 @@ method_bits(compress_method_t method) return method == GZIP_METHOD ? 15+16 : 15; } +/** @{ */ /* These macros define the maximum allowable compression factor. Anything of * size greater than CHECK_FOR_COMPRESSION_BOMB_AFTER is not allowed to * have an uncompression factor (uncompressed size:compressed size ratio) of @@ -72,6 +95,7 @@ method_bits(compress_method_t method) */ #define MAX_UNCOMPRESSION_FACTOR 25 #define CHECK_FOR_COMPRESSION_BOMB_AFTER (1024*64) +/** @} */ /** Return true if uncompressing an input of size <b>in_size</b> to an input * of size at least <b>size_out</b> looks like a compression bomb. */ @@ -198,9 +222,7 @@ tor_gzip_compress(char **out, size_t *out_len, deflateEnd(stream); tor_free(stream); } - if (*out) { - tor_free(*out); - } + tor_free(*out); return -1; } @@ -369,12 +391,12 @@ detect_compression_method(const char *in, size_t in_len) /** Internal state for an incremental zlib compression/decompression. The * body of this struct is not exposed. */ struct tor_zlib_state_t { - struct z_stream_s stream; - int compress; + struct z_stream_s stream; /**< The zlib stream */ + int compress; /**< True if we are compressing; false if we are inflating */ - /* Number of bytes read so far. Used to detect zlib bombs. */ + /** Number of bytes read so far. Used to detect zlib bombs. */ size_t input_so_far; - /* Number of bytes written so far. Used to detect zlib bombs. */ + /** Number of bytes written so far. Used to detect zlib bombs. */ size_t output_so_far; }; @@ -479,7 +501,8 @@ tor_zlib_process(tor_zlib_state_t *state, void tor_zlib_free(tor_zlib_state_t *state) { - tor_assert(state); + if (!state) + return; if (state->compress) deflateEnd(&state->stream); diff --git a/src/common/torint.h b/src/common/torint.h index d489684656..0b5c29adc0 100644 --- a/src/common/torint.h +++ b/src/common/torint.h @@ -117,11 +117,10 @@ typedef unsigned int uint32_t; #ifndef INT32_MAX #define INT32_MAX 0x7fffffff #endif -#endif - #ifndef INT32_MIN #define INT32_MIN (-2147483647-1) #endif +#endif #if (SIZEOF_LONG == 4) #ifndef HAVE_INT32_T @@ -330,9 +329,9 @@ typedef uint32_t uintptr_t; #endif #endif -/* Any ssize_t larger than this amount is likely to be an underflow. */ +/** Any ssize_t larger than this amount is likely to be an underflow. */ #define SSIZE_T_CEILING ((ssize_t)(SSIZE_T_MAX-16)) -/* Any size_t larger than this amount is likely to be an underflow. */ +/** Any size_t larger than this amount is likely to be an underflow. */ #define SIZE_T_CEILING ((size_t)(SSIZE_T_MAX-16)) #endif /* __TORINT_H */ diff --git a/src/common/log.h b/src/common/torlog.h index 2520ed1633..000e32ddab 100644 --- a/src/common/log.h +++ b/src/common/torlog.h @@ -5,7 +5,7 @@ /* See LICENSE for licensing information */ /** - * \file log.h + * \file torlog.h * * \brief Headers for log.c **/ @@ -90,14 +90,16 @@ #define LD_ACCT (1u<<17) /** Router history */ #define LD_HIST (1u<<18) - +/** OR handshaking */ +#define LD_HANDSHAKE (1u<<19) /** Number of logging domains in the code. */ -#define N_LOGGING_DOMAINS 19 +#define N_LOGGING_DOMAINS 20 -/** This log message is not safe to send to a callback-based logger. - * Used as a flag, not a log domain. */ +/** This log message is not safe to send to a callback-based logger + * immediately. Used as a flag, not a log domain. */ #define LD_NOCB (1u<<31) +/** Mask of zero or more log domains, OR'd together. */ typedef uint32_t log_domain_mask_t; /** Configures which severities are logged for each logging domain for a given @@ -131,6 +133,7 @@ int add_file_log(const log_severity_list_t *severity, const char *filename); int add_syslog_log(const log_severity_list_t *severity); #endif int add_callback_log(const log_severity_list_t *severity, log_callback cb); +void logs_set_domain_logging(int enabled); int get_min_log_level(void); void switch_logs_debug(void); void logs_free_all(void); @@ -138,16 +141,15 @@ void add_temp_log(int min_severity); void close_temp_logs(void); void rollback_log_changes(void); void mark_logs_temp(void); -void configure_libevent_logging(void); -void suppress_libevent_log_msg(const char *msg); void change_callback_log_severity(int loglevelMin, int loglevelMax, log_callback cb); +void flush_pending_log_callbacks(void); void log_set_application_name(const char *name); /* Outputs a message to stdout */ -void _log(int severity, log_domain_mask_t domain, const char *format, ...) +void tor_log(int severity, log_domain_mask_t domain, const char *format, ...) CHECK_PRINTF(3,4); -#define log _log /* hack it so we don't conflict with log() as much */ +#define log tor_log /* hack it so we don't conflict with log() as much */ #ifdef __GNUC__ extern int _log_global_min_severity; diff --git a/src/common/tortls.c b/src/common/tortls.c index 7735618ea2..10f4440cb4 100644 --- a/src/common/tortls.c +++ b/src/common/tortls.c @@ -16,6 +16,10 @@ #include "orconfig.h" +#if defined (WINCE) +#include <WinSock2.h> +#endif + #include <assert.h> #ifdef MS_WINDOWS /*wrkard for dtls1.h >= 0.9.8m of "#include <winsock.h>"*/ #define WIN32_WINNT 0x400 @@ -45,7 +49,7 @@ #include "crypto.h" #include "tortls.h" #include "util.h" -#include "log.h" +#include "torlog.h" #include "container.h" #include "ht.h" #include <string.h> @@ -82,7 +86,9 @@ static int use_unsafe_renegotiation_op = 0; * SSL3_FLAGS_ALLOW_UNSAFE_LEGACY_RENEGOTIATION? */ static int use_unsafe_renegotiation_flag = 0; -/** Structure holding the TLS state for a single connection. */ +/** Holds a SSL_CTX object and related state used to configure TLS + * connections. + */ typedef struct tor_tls_context_t { int refcnt; SSL_CTX *ctx; @@ -184,10 +190,16 @@ static X509* tor_tls_create_certificate(crypto_pk_env_t *rsa, const char *cname_sign, unsigned int lifetime); static void tor_tls_unblock_renegotiation(tor_tls_t *tls); - -/** Global tls context. We keep it here because nobody else needs to - * touch it. */ -static tor_tls_context_t *global_tls_context = NULL; +static int tor_tls_context_init_one(tor_tls_context_t **ppcontext, + crypto_pk_env_t *identity, + unsigned int key_lifetime); +static tor_tls_context_t *tor_tls_context_new(crypto_pk_env_t *identity, + unsigned int key_lifetime); + +/** Global TLS contexts. We keep them here because nobody else needs + * to touch them. */ +static tor_tls_context_t *server_tls_context = NULL; +static tor_tls_context_t *client_tls_context = NULL; /** True iff tor_tls_init() has been called. */ static int tls_library_is_initialized = 0; @@ -195,30 +207,51 @@ static int tls_library_is_initialized = 0; #define _TOR_TLS_SYSCALL (_MIN_TOR_TLS_ERROR_VAL - 2) #define _TOR_TLS_ZERORETURN (_MIN_TOR_TLS_ERROR_VAL - 1) +#include "tortls_states.h" + +/** Return the symbolic name of an OpenSSL state. */ +static const char * +ssl_state_to_string(int ssl_state) +{ + static char buf[40]; + int i; + for (i = 0; state_map[i].name; ++i) { + if (state_map[i].state == ssl_state) + return state_map[i].name; + } + tor_snprintf(buf, sizeof(buf), "Unknown state %d", ssl_state); + return buf; +} + /** Log all pending tls errors at level <b>severity</b>. Use * <b>doing</b> to describe our current activities. */ static void -tls_log_errors(tor_tls_t *tls, int severity, const char *doing) +tls_log_errors(tor_tls_t *tls, int severity, int domain, const char *doing) { + const char *state = NULL; + int st; unsigned long err; const char *msg, *lib, *func, *addr; addr = tls ? tls->address : NULL; + st = (tls && tls->ssl) ? tls->ssl->state : -1; while ((err = ERR_get_error()) != 0) { msg = (const char*)ERR_reason_error_string(err); lib = (const char*)ERR_lib_error_string(err); func = (const char*)ERR_func_error_string(err); + if (!state) + state = (st>=0)?ssl_state_to_string(st):"---"; if (!msg) msg = "(null)"; if (!lib) lib = "(null)"; if (!func) func = "(null)"; if (doing) { - log(severity, LD_NET, "TLS error while %s%s%s: %s (in %s:%s)", + log(severity, domain, "TLS error while %s%s%s: %s (in %s:%s:%s)", doing, addr?" with ":"", addr?addr:"", - msg, lib, func); + msg, lib, func, state); } else { - log(severity, LD_NET, "TLS error%s%s: %s (in %s:%s)", + log(severity, domain, "TLS error%s%s: %s (in %s:%s:%s)", addr?" with ":"", addr?addr:"", - msg, lib, func); + msg, lib, func, state); } } } @@ -294,7 +327,7 @@ tor_tls_err_to_string(int err) */ static int tor_tls_get_error(tor_tls_t *tls, int r, int extra, - const char *doing, int severity) + const char *doing, int severity, int domain) { int err = SSL_get_error(tls->ssl, r); int tor_error = TOR_TLS_ERROR_MISC; @@ -309,25 +342,28 @@ tor_tls_get_error(tor_tls_t *tls, int r, int extra, if (extra&CATCH_SYSCALL) return _TOR_TLS_SYSCALL; if (r == 0) { - log(severity, LD_NET, "TLS error: unexpected close while %s", doing); + log(severity, LD_NET, "TLS error: unexpected close while %s (%s)", + doing, ssl_state_to_string(tls->ssl->state)); tor_error = TOR_TLS_ERROR_IO; } else { int e = tor_socket_errno(tls->socket); log(severity, LD_NET, - "TLS error: <syscall error while %s> (errno=%d: %s)", - doing, e, tor_socket_strerror(e)); + "TLS error: <syscall error while %s> (errno=%d: %s; state=%s)", + doing, e, tor_socket_strerror(e), + ssl_state_to_string(tls->ssl->state)); tor_error = tor_errno_to_tls_error(e); } - tls_log_errors(tls, severity, doing); + tls_log_errors(tls, severity, domain, doing); return tor_error; case SSL_ERROR_ZERO_RETURN: if (extra&CATCH_ZERO) return _TOR_TLS_ZERORETURN; - log(severity, LD_NET, "TLS connection closed while %s", doing); - tls_log_errors(tls, severity, doing); + log(severity, LD_NET, "TLS connection closed while %s in state %s", + doing, ssl_state_to_string(tls->ssl->state)); + tls_log_errors(tls, severity, domain, doing); return TOR_TLS_CLOSE; default: - tls_log_errors(tls, severity, doing); + tls_log_errors(tls, severity, domain, doing); return TOR_TLS_ERROR_MISC; } } @@ -341,11 +377,10 @@ tor_tls_init(void) long version; SSL_library_init(); SSL_load_error_strings(); - crypto_global_init(-1); version = SSLeay(); - /* OpenSSL 0.9.8l introdeced SSL3_FLAGS_ALLOW_UNSAGE_LEGACY_RENEGOTIATION + /* OpenSSL 0.9.8l introduced SSL3_FLAGS_ALLOW_UNSAFE_LEGACY_RENEGOTIATION * here, but without thinking too hard about it: it turns out that the * flag in question needed to be set at the last minute, and that it * conflicted with an existing flag number that had already been added @@ -364,8 +399,8 @@ tor_tls_init(void) * leave their headers out of sync with their libraries. * * Yes, it _is_ almost as if the OpenSSL developers decided that no - * program should be allowed to use renegotiation its first passed an - * test of intelligence and determination. + * program should be allowed to use renegotiation unless it first passed + * a test of intelligence and determination. */ if (version >= 0x009080c0L && version < 0x009080d0L) { log_notice(LD_GENERAL, "OpenSSL %s looks like version 0.9.8l; " @@ -400,9 +435,15 @@ tor_tls_init(void) void tor_tls_free_all(void) { - if (global_tls_context) { - tor_tls_context_decref(global_tls_context); - global_tls_context = NULL; + if (server_tls_context) { + tor_tls_context_t *ctx = server_tls_context; + server_tls_context = NULL; + tor_tls_context_decref(ctx); + } + if (client_tls_context) { + tor_tls_context_t *ctx = client_tls_context; + client_tls_context = NULL; + tor_tls_context_decref(ctx); } if (!HT_EMPTY(&tlsmap_root)) { log_warn(LD_MM, "Still have entries in the tlsmap at shutdown."); @@ -511,7 +552,7 @@ tor_tls_create_certificate(crypto_pk_env_t *rsa, x509 = NULL; } done: - tls_log_errors(NULL, LOG_WARN, "generating certificate"); + tls_log_errors(NULL, LOG_WARN, LD_NET, "generating certificate"); if (sign_pkey) EVP_PKEY_free(sign_pkey); if (pkey) @@ -528,9 +569,9 @@ tor_tls_create_certificate(crypto_pk_env_t *rsa, (TLS1_TXT_DHE_RSA_WITH_AES_256_SHA ":" \ TLS1_TXT_DHE_RSA_WITH_AES_128_SHA ":" \ SSL3_TXT_EDH_RSA_DES_192_CBC3_SHA) -/* Note: for setting up your own private testing network with link crypto - * disabled, set the cipher lists to your cipher list to - * SSL3_TXT_RSA_NULL_SHA. If you do this, you won't be able to communicate +/* Note: to set up your own private testing network with link crypto + * disabled, set your Tors' cipher list to + * (SSL3_TXT_RSA_NULL_SHA). If you do this, you won't be able to communicate * with any of the "real" Tors, though. */ #ifdef V2_HANDSHAKE_CLIENT @@ -589,15 +630,97 @@ tor_tls_context_incref(tor_tls_context_t *ctx) ++ctx->refcnt; } -/** Create a new TLS context for use with Tor TLS handshakes. - * <b>identity</b> should be set to the identity key used to sign the - * certificate, and <b>nickname</b> set to the nickname to use. +/** Create new global client and server TLS contexts. + * + * If <b>server_identity</b> is NULL, this will not generate a server + * TLS context. If <b>is_public_server</b> is non-zero, this will use + * the same TLS context for incoming and outgoing connections, and + * ignore <b>client_identity</b>. */ +int +tor_tls_context_init(int is_public_server, + crypto_pk_env_t *client_identity, + crypto_pk_env_t *server_identity, + unsigned int key_lifetime) +{ + int rv1 = 0; + int rv2 = 0; + + if (is_public_server) { + tor_tls_context_t *new_ctx; + tor_tls_context_t *old_ctx; + + tor_assert(server_identity != NULL); + + rv1 = tor_tls_context_init_one(&server_tls_context, + server_identity, + key_lifetime); + + if (rv1 >= 0) { + new_ctx = server_tls_context; + tor_tls_context_incref(new_ctx); + old_ctx = client_tls_context; + client_tls_context = new_ctx; + + if (old_ctx != NULL) { + tor_tls_context_decref(old_ctx); + } + } + } else { + if (server_identity != NULL) { + rv1 = tor_tls_context_init_one(&server_tls_context, + server_identity, + key_lifetime); + } else { + tor_tls_context_t *old_ctx = server_tls_context; + server_tls_context = NULL; + + if (old_ctx != NULL) { + tor_tls_context_decref(old_ctx); + } + } + + rv2 = tor_tls_context_init_one(&client_tls_context, + client_identity, + key_lifetime); + } + + return MIN(rv1, rv2); +} + +/** Create a new global TLS context. * * You can call this function multiple times. Each time you call it, * it generates new certificates; all new connections will use * the new SSL context. */ -int +static int +tor_tls_context_init_one(tor_tls_context_t **ppcontext, + crypto_pk_env_t *identity, + unsigned int key_lifetime) +{ + tor_tls_context_t *new_ctx = tor_tls_context_new(identity, + key_lifetime); + tor_tls_context_t *old_ctx = *ppcontext; + + if (new_ctx != NULL) { + *ppcontext = new_ctx; + + /* Free the old context if one existed. */ + if (old_ctx != NULL) { + /* This is safe even if there are open connections: we reference- + * count tor_tls_context_t objects. */ + tor_tls_context_decref(old_ctx); + } + } + + return ((new_ctx != NULL) ? 0 : -1); +} + +/** Create a new TLS context for use with Tor TLS handshakes. + * <b>identity</b> should be set to the identity key used to sign the + * certificate. + */ +static tor_tls_context_t * tor_tls_context_new(crypto_pk_env_t *identity, unsigned int key_lifetime) { crypto_pk_env_t *rsa = NULL; @@ -685,6 +808,7 @@ tor_tls_context_new(crypto_pk_env_t *identity, unsigned int key_lifetime) goto error; { crypto_dh_env_t *dh = crypto_dh_new(DH_TYPE_TLS); + tor_assert(dh); SSL_CTX_set_tmp_dh(result->ctx, _crypto_dh_env_get_dh(dh)); crypto_dh_free(dh); } @@ -692,21 +816,15 @@ tor_tls_context_new(crypto_pk_env_t *identity, unsigned int key_lifetime) always_accept_verify_cb); /* let us realloc bufs that we're writing from */ SSL_CTX_set_mode(result->ctx, SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER); - /* Free the old context if one exists. */ - if (global_tls_context) { - /* This is safe even if there are open connections: OpenSSL does - * reference counting with SSL and SSL_CTX objects. */ - tor_tls_context_decref(global_tls_context); - } - global_tls_context = result; + if (rsa) crypto_free_pk_env(rsa); tor_free(nickname); tor_free(nn2); - return 0; + return result; error: - tls_log_errors(NULL, LOG_WARN, "creating TLS context"); + tls_log_errors(NULL, LOG_WARN, LD_NET, "creating TLS context"); tor_free(nickname); tor_free(nn2); if (pkey) @@ -719,7 +837,7 @@ tor_tls_context_new(crypto_pk_env_t *identity, unsigned int key_lifetime) X509_free(cert); if (idcert) X509_free(idcert); - return -1; + return NULL; } #ifdef V2_HANDSHAKE_SERVER @@ -734,11 +852,11 @@ tor_tls_client_is_using_v2_ciphers(const SSL *ssl, const char *address) /* If we reached this point, we just got a client hello. See if there is * a cipher list. */ if (!(session = SSL_get_session((SSL *)ssl))) { - log_warn(LD_NET, "No session on TLS?"); + log_info(LD_NET, "No session on TLS?"); return 0; } if (!session->ciphers) { - log_warn(LD_NET, "No ciphers on session"); + log_info(LD_NET, "No ciphers on session"); return 0; } /* Now we need to see if there are any ciphers whose presence means we're @@ -750,8 +868,7 @@ tor_tls_client_is_using_v2_ciphers(const SSL *ssl, const char *address) strcmp(ciphername, TLS1_TXT_DHE_RSA_WITH_AES_256_SHA) && strcmp(ciphername, SSL3_TXT_EDH_RSA_DES_192_CBC3_SHA) && strcmp(ciphername, "(NONE)")) { - /* XXXX should be ld_debug */ - log_info(LD_NET, "Got a non-version-1 cipher called '%s'", ciphername); + log_debug(LD_NET, "Got a non-version-1 cipher called '%s'", ciphername); // return 1; goto dump_list; } @@ -767,8 +884,8 @@ tor_tls_client_is_using_v2_ciphers(const SSL *ssl, const char *address) smartlist_add(elts, (char*)ciphername); } s = smartlist_join_strings(elts, ":", 0, NULL); - log_info(LD_NET, "Got a non-version-1 cipher list from %s. It is: '%s'", - address, s); + log_debug(LD_NET, "Got a non-version-1 cipher list from %s. It is: '%s'", + address, s); tor_free(s); smartlist_free(elts); } @@ -899,10 +1016,12 @@ tor_tls_new(int sock, int isServer) { BIO *bio = NULL; tor_tls_t *result = tor_malloc_zero(sizeof(tor_tls_t)); + tor_tls_context_t *context = isServer ? server_tls_context : + client_tls_context; - tor_assert(global_tls_context); /* make sure somebody made it first */ - if (!(result->ssl = SSL_new(global_tls_context->ctx))) { - tls_log_errors(NULL, LOG_WARN, "generating TLS context"); + tor_assert(context); /* make sure somebody made it first */ + if (!(result->ssl = SSL_new(context->ctx))) { + tls_log_errors(NULL, LOG_WARN, LD_NET, "creating SSL object"); tor_free(result); return NULL; } @@ -918,7 +1037,7 @@ tor_tls_new(int sock, int isServer) if (!SSL_set_cipher_list(result->ssl, isServer ? SERVER_CIPHER_LIST : CLIENT_CIPHER_LIST)) { - tls_log_errors(NULL, LOG_WARN, "setting ciphers"); + tls_log_errors(NULL, LOG_WARN, LD_NET, "setting ciphers"); #ifdef SSL_set_tlsext_host_name SSL_set_tlsext_host_name(result->ssl, NULL); #endif @@ -931,7 +1050,7 @@ tor_tls_new(int sock, int isServer) result->socket = sock; bio = BIO_new_socket(sock, BIO_NOCLOSE); if (! bio) { - tls_log_errors(NULL, LOG_WARN, "opening BIO"); + tls_log_errors(NULL, LOG_WARN, LD_NET, "opening BIO"); #ifdef SSL_set_tlsext_host_name SSL_set_tlsext_host_name(result->ssl, NULL); #endif @@ -941,8 +1060,8 @@ tor_tls_new(int sock, int isServer) } HT_INSERT(tlsmap, &tlsmap_root, result); SSL_set_bio(result->ssl, bio, bio); - tor_tls_context_incref(global_tls_context); - result->context = global_tls_context; + tor_tls_context_incref(context); + result->context = context; result->state = TOR_TLS_ST_HANDSHAKE; result->isServer = isServer; result->wantwrite_n = 0; @@ -959,7 +1078,7 @@ tor_tls_new(int sock, int isServer) #endif /* Not expected to get called. */ - tls_log_errors(NULL, LOG_WARN, "generating TLS context"); + tls_log_errors(NULL, LOG_WARN, LD_NET, "creating tor_tls_t object"); return result; } @@ -1038,7 +1157,9 @@ void tor_tls_free(tor_tls_t *tls) { tor_tls_t *removed; - tor_assert(tls && tls->ssl); + if (!tls) + return; + tor_assert(tls->ssl); removed = HT_REMOVE(tlsmap, &tlsmap_root, tls); if (!removed) { log_warn(LD_BUG, "Freeing a TLS that was not in the ssl->tls map."); @@ -1081,7 +1202,7 @@ tor_tls_read(tor_tls_t *tls, char *cp, size_t len) #endif return r; } - err = tor_tls_get_error(tls, r, CATCH_ZERO, "reading", LOG_DEBUG); + err = tor_tls_get_error(tls, r, CATCH_ZERO, "reading", LOG_DEBUG, LD_NET); if (err == _TOR_TLS_ZERORETURN || err == TOR_TLS_CLOSE) { log_debug(LD_NET,"read returned r=%d; TLS is closed",r); tls->state = TOR_TLS_ST_CLOSED; @@ -1117,7 +1238,7 @@ tor_tls_write(tor_tls_t *tls, const char *cp, size_t n) tls->wantwrite_n = 0; } r = SSL_write(tls->ssl, cp, (int)n); - err = tor_tls_get_error(tls, r, 0, "writing", LOG_INFO); + err = tor_tls_get_error(tls, r, 0, "writing", LOG_INFO, LD_NET); if (err == TOR_TLS_DONE) { return r; } @@ -1135,21 +1256,30 @@ int tor_tls_handshake(tor_tls_t *tls) { int r; + int oldstate; tor_assert(tls); tor_assert(tls->ssl); tor_assert(tls->state == TOR_TLS_ST_HANDSHAKE); check_no_tls_errors(); + oldstate = tls->ssl->state; if (tls->isServer) { + log_debug(LD_HANDSHAKE, "About to call SSL_accept on %p (%s)", tls, + ssl_state_to_string(tls->ssl->state)); r = SSL_accept(tls->ssl); } else { + log_debug(LD_HANDSHAKE, "About to call SSL_connect on %p (%s)", tls, + ssl_state_to_string(tls->ssl->state)); r = SSL_connect(tls->ssl); } + if (oldstate != tls->ssl->state) + log_debug(LD_HANDSHAKE, "After call, %p was in state %s", + tls, ssl_state_to_string(tls->ssl->state)); /* We need to call this here and not earlier, since OpenSSL has a penchant * for clearing its flags when you say accept or connect. */ tor_tls_unblock_renegotiation(tls); - r = tor_tls_get_error(tls,r,0, "handshaking", LOG_INFO); + r = tor_tls_get_error(tls,r,0, "handshaking", LOG_INFO, LD_HANDSHAKE); if (ERR_peek_error() != 0) { - tls_log_errors(tls, tls->isServer ? LOG_INFO : LOG_WARN, + tls_log_errors(tls, tls->isServer ? LOG_INFO : LOG_WARN, LD_HANDSHAKE, "handshaking"); return TOR_TLS_ERROR_MISC; } @@ -1170,7 +1300,8 @@ tor_tls_handshake(tor_tls_t *tls) " get set. Fixing that."); } tls->wasV2Handshake = 1; - log_debug(LD_NET, "Completed V2 TLS handshake with client; waiting " + log_debug(LD_HANDSHAKE, + "Completed V2 TLS handshake with client; waiting " "for renegotiation."); } else { tls->wasV2Handshake = 0; @@ -1182,10 +1313,13 @@ tor_tls_handshake(tor_tls_t *tls) X509 *cert = SSL_get_peer_certificate(tls->ssl); STACK_OF(X509) *chain = SSL_get_peer_cert_chain(tls->ssl); int n_certs = sk_X509_num(chain); - if (n_certs > 1 || (n_certs == 1 && cert != sk_X509_value(chain, 0))) + if (n_certs > 1 || (n_certs == 1 && cert != sk_X509_value(chain, 0))) { + log_debug(LD_HANDSHAKE, "Server sent back multiple certificates; it " + "looks like a v1 handshake on %p", tls); tls->wasV2Handshake = 0; - else { - log_debug(LD_NET, "Server sent back a single certificate; looks like " + } else { + log_debug(LD_HANDSHAKE, + "Server sent back a single certificate; looks like " "a v2 handshake on %p.", tls); tls->wasV2Handshake = 1; } @@ -1193,7 +1327,7 @@ tor_tls_handshake(tor_tls_t *tls) X509_free(cert); #endif if (SSL_set_cipher_list(tls->ssl, SERVER_CIPHER_LIST) == 0) { - tls_log_errors(NULL, LOG_WARN, "re-setting ciphers"); + tls_log_errors(NULL, LOG_WARN, LD_HANDSHAKE, "re-setting ciphers"); r = TOR_TLS_ERROR_MISC; } } @@ -1216,7 +1350,8 @@ tor_tls_renegotiate(tor_tls_t *tls) if (tls->state != TOR_TLS_ST_RENEGOTIATE) { int r = SSL_renegotiate(tls->ssl); if (r <= 0) { - return tor_tls_get_error(tls, r, 0, "renegotiating", LOG_WARN); + return tor_tls_get_error(tls, r, 0, "renegotiating", LOG_WARN, + LD_HANDSHAKE); } tls->state = TOR_TLS_ST_RENEGOTIATE; } @@ -1225,7 +1360,8 @@ tor_tls_renegotiate(tor_tls_t *tls) tls->state = TOR_TLS_ST_OPEN; return TOR_TLS_DONE; } else - return tor_tls_get_error(tls, r, 0, "renegotiating handshake", LOG_INFO); + return tor_tls_get_error(tls, r, 0, "renegotiating handshake", LOG_INFO, + LD_HANDSHAKE); } /** Shut down an open tls connection <b>tls</b>. When finished, returns @@ -1249,7 +1385,7 @@ tor_tls_shutdown(tor_tls_t *tls) r = SSL_read(tls->ssl, buf, 128); } while (r>0); err = tor_tls_get_error(tls, r, CATCH_ZERO, "reading to shut down", - LOG_INFO); + LOG_INFO, LD_NET); if (err == _TOR_TLS_ZERORETURN) { tls->state = TOR_TLS_ST_GOTCLOSE; /* fall through... */ @@ -1265,7 +1401,7 @@ tor_tls_shutdown(tor_tls_t *tls) return TOR_TLS_DONE; } err = tor_tls_get_error(tls, r, CATCH_SYSCALL|CATCH_ZERO, "shutting down", - LOG_INFO); + LOG_INFO, LD_NET); if (err == _TOR_TLS_SYSCALL) { /* The underlying TCP connection closed while we were shutting down. */ tls->state = TOR_TLS_ST_CLOSED; @@ -1297,7 +1433,7 @@ tor_tls_peer_has_cert(tor_tls_t *tls) { X509 *cert; cert = SSL_get_peer_certificate(tls->ssl); - tls_log_errors(tls, LOG_WARN, "getting peer certificate"); + tls_log_errors(tls, LOG_WARN, LD_HANDSHAKE, "getting peer certificate"); if (!cert) return 0; X509_free(cert); @@ -1324,7 +1460,7 @@ log_cert_lifetime(X509 *cert, const char *problem) log_warn(LD_GENERAL, "Couldn't allocate BIO!"); goto end; } if (!(ASN1_TIME_print(bio, X509_get_notBefore(cert)))) { - tls_log_errors(NULL, LOG_WARN, "printing certificate lifetime"); + tls_log_errors(NULL, LOG_WARN, LD_NET, "printing certificate lifetime"); goto end; } BIO_get_mem_ptr(bio, &buf); @@ -1332,7 +1468,7 @@ log_cert_lifetime(X509 *cert, const char *problem) (void)BIO_reset(bio); if (!(ASN1_TIME_print(bio, X509_get_notAfter(cert)))) { - tls_log_errors(NULL, LOG_WARN, "printing certificate lifetime"); + tls_log_errors(NULL, LOG_WARN, LD_NET, "printing certificate lifetime"); goto end; } BIO_get_mem_ptr(bio, &buf); @@ -1346,13 +1482,11 @@ log_cert_lifetime(X509 *cert, const char *problem) end: /* Not expected to get invoked */ - tls_log_errors(NULL, LOG_WARN, "getting certificate lifetime"); + tls_log_errors(NULL, LOG_WARN, LD_NET, "getting certificate lifetime"); if (bio) BIO_free(bio); - if (s1) - tor_free(s1); - if (s2) - tor_free(s2); + tor_free(s1); + tor_free(s2); } /** Helper function: try to extract a link certificate and an identity @@ -1420,7 +1554,7 @@ tor_tls_verify(int severity, tor_tls_t *tls, crypto_pk_env_t **identity_key) if (!(id_pkey = X509_get_pubkey(id_cert)) || X509_verify(cert, id_pkey) <= 0) { log_fn(severity,LD_PROTOCOL,"X509_verify on cert and pkey returned <= 0"); - tls_log_errors(tls, severity,"verifying certificate"); + tls_log_errors(tls, severity, LD_HANDSHAKE, "verifying certificate"); goto done; } @@ -1439,7 +1573,7 @@ tor_tls_verify(int severity, tor_tls_t *tls, crypto_pk_env_t **identity_key) /* This should never get invoked, but let's make sure in case OpenSSL * acts unexpectedly. */ - tls_log_errors(tls, LOG_WARN, "finishing tor_tls_verify"); + tls_log_errors(tls, LOG_WARN, LD_HANDSHAKE, "finishing tor_tls_verify"); return r; } @@ -1478,7 +1612,7 @@ tor_tls_check_lifetime(tor_tls_t *tls, int tolerance) if (cert) X509_free(cert); /* Not expected to get invoked */ - tls_log_errors(tls, LOG_WARN, "checking certificate lifetime"); + tls_log_errors(tls, LOG_WARN, LD_NET, "checking certificate lifetime"); return r; } @@ -1546,7 +1680,7 @@ _check_no_tls_errors(const char *fname, int line) return; log(LOG_WARN, LD_CRYPTO, "Unhandled OpenSSL errors found at %s:%d: ", tor_fix_source_file(fname), line); - tls_log_errors(NULL, LOG_WARN, NULL); + tls_log_errors(NULL, LOG_WARN, LD_NET, NULL); } /** Return true iff the initial TLS connection at <b>tls</b> did not use a v2 diff --git a/src/common/tortls.h b/src/common/tortls.h index 9644b87f2c..55fee81aea 100644 --- a/src/common/tortls.h +++ b/src/common/tortls.h @@ -50,7 +50,10 @@ typedef struct tor_tls_t tor_tls_t; const char *tor_tls_err_to_string(int err); void tor_tls_free_all(void); -int tor_tls_context_new(crypto_pk_env_t *rsa, unsigned int key_lifetime); +int tor_tls_context_init(int is_public_server, + crypto_pk_env_t *client_identity, + crypto_pk_env_t *server_identity, + unsigned int key_lifetime); tor_tls_t *tor_tls_new(int sock, int is_server); void tor_tls_set_logged_address(tor_tls_t *tls, const char *address); void tor_tls_set_renegotiate_callback(tor_tls_t *tls, diff --git a/src/common/tortls_states.h b/src/common/tortls_states.h new file mode 100644 index 0000000000..dcff2479f6 --- /dev/null +++ b/src/common/tortls_states.h @@ -0,0 +1,414 @@ +/* Copyright (c) 2003, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/* Helper file: included only in tortls.c */ + +#ifndef _TORTLS_STATES_H +#define _TORTLS_STATES_H + +/* The main body of this file was mechanically generated with this + perl script: + + my %keys = (); + for $fn (@ARGV) { + open(F, $fn); + while (<F>) { + next unless /^#define ((?:SSL|DTLS)\w*_ST_\w*)/; + $keys{$1} = 1; + } + close(F); + } + for $k (sort keys %keys) { + print "#ifdef $k\n S($k),\n#endif\n" + } +*/ + +/** Mapping from allowed value of SSL.state to the name of C macro for that + * state. Used for debugging an openssl connection. */ +static const struct { int state; const char *name; } state_map[] = { +#define S(state) { state, #state } +#ifdef DTLS1_ST_CR_HELLO_VERIFY_REQUEST_A + S(DTLS1_ST_CR_HELLO_VERIFY_REQUEST_A), +#endif +#ifdef DTLS1_ST_CR_HELLO_VERIFY_REQUEST_B + S(DTLS1_ST_CR_HELLO_VERIFY_REQUEST_B), +#endif +#ifdef DTLS1_ST_SW_HELLO_VERIFY_REQUEST_A + S(DTLS1_ST_SW_HELLO_VERIFY_REQUEST_A), +#endif +#ifdef DTLS1_ST_SW_HELLO_VERIFY_REQUEST_B + S(DTLS1_ST_SW_HELLO_VERIFY_REQUEST_B), +#endif +#ifdef SSL23_ST_CR_SRVR_HELLO_A + S(SSL23_ST_CR_SRVR_HELLO_A), +#endif +#ifdef SSL23_ST_CR_SRVR_HELLO_B + S(SSL23_ST_CR_SRVR_HELLO_B), +#endif +#ifdef SSL23_ST_CW_CLNT_HELLO_A + S(SSL23_ST_CW_CLNT_HELLO_A), +#endif +#ifdef SSL23_ST_CW_CLNT_HELLO_B + S(SSL23_ST_CW_CLNT_HELLO_B), +#endif +#ifdef SSL23_ST_SR_CLNT_HELLO_A + S(SSL23_ST_SR_CLNT_HELLO_A), +#endif +#ifdef SSL23_ST_SR_CLNT_HELLO_B + S(SSL23_ST_SR_CLNT_HELLO_B), +#endif +#ifdef SSL2_ST_CLIENT_START_ENCRYPTION + S(SSL2_ST_CLIENT_START_ENCRYPTION), +#endif +#ifdef SSL2_ST_GET_CLIENT_FINISHED_A + S(SSL2_ST_GET_CLIENT_FINISHED_A), +#endif +#ifdef SSL2_ST_GET_CLIENT_FINISHED_B + S(SSL2_ST_GET_CLIENT_FINISHED_B), +#endif +#ifdef SSL2_ST_GET_CLIENT_HELLO_A + S(SSL2_ST_GET_CLIENT_HELLO_A), +#endif +#ifdef SSL2_ST_GET_CLIENT_HELLO_B + S(SSL2_ST_GET_CLIENT_HELLO_B), +#endif +#ifdef SSL2_ST_GET_CLIENT_HELLO_C + S(SSL2_ST_GET_CLIENT_HELLO_C), +#endif +#ifdef SSL2_ST_GET_CLIENT_MASTER_KEY_A + S(SSL2_ST_GET_CLIENT_MASTER_KEY_A), +#endif +#ifdef SSL2_ST_GET_CLIENT_MASTER_KEY_B + S(SSL2_ST_GET_CLIENT_MASTER_KEY_B), +#endif +#ifdef SSL2_ST_GET_SERVER_FINISHED_A + S(SSL2_ST_GET_SERVER_FINISHED_A), +#endif +#ifdef SSL2_ST_GET_SERVER_FINISHED_B + S(SSL2_ST_GET_SERVER_FINISHED_B), +#endif +#ifdef SSL2_ST_GET_SERVER_HELLO_A + S(SSL2_ST_GET_SERVER_HELLO_A), +#endif +#ifdef SSL2_ST_GET_SERVER_HELLO_B + S(SSL2_ST_GET_SERVER_HELLO_B), +#endif +#ifdef SSL2_ST_GET_SERVER_VERIFY_A + S(SSL2_ST_GET_SERVER_VERIFY_A), +#endif +#ifdef SSL2_ST_GET_SERVER_VERIFY_B + S(SSL2_ST_GET_SERVER_VERIFY_B), +#endif +#ifdef SSL2_ST_SEND_CLIENT_CERTIFICATE_A + S(SSL2_ST_SEND_CLIENT_CERTIFICATE_A), +#endif +#ifdef SSL2_ST_SEND_CLIENT_CERTIFICATE_B + S(SSL2_ST_SEND_CLIENT_CERTIFICATE_B), +#endif +#ifdef SSL2_ST_SEND_CLIENT_CERTIFICATE_C + S(SSL2_ST_SEND_CLIENT_CERTIFICATE_C), +#endif +#ifdef SSL2_ST_SEND_CLIENT_CERTIFICATE_D + S(SSL2_ST_SEND_CLIENT_CERTIFICATE_D), +#endif +#ifdef SSL2_ST_SEND_CLIENT_FINISHED_A + S(SSL2_ST_SEND_CLIENT_FINISHED_A), +#endif +#ifdef SSL2_ST_SEND_CLIENT_FINISHED_B + S(SSL2_ST_SEND_CLIENT_FINISHED_B), +#endif +#ifdef SSL2_ST_SEND_CLIENT_HELLO_A + S(SSL2_ST_SEND_CLIENT_HELLO_A), +#endif +#ifdef SSL2_ST_SEND_CLIENT_HELLO_B + S(SSL2_ST_SEND_CLIENT_HELLO_B), +#endif +#ifdef SSL2_ST_SEND_CLIENT_MASTER_KEY_A + S(SSL2_ST_SEND_CLIENT_MASTER_KEY_A), +#endif +#ifdef SSL2_ST_SEND_CLIENT_MASTER_KEY_B + S(SSL2_ST_SEND_CLIENT_MASTER_KEY_B), +#endif +#ifdef SSL2_ST_SEND_REQUEST_CERTIFICATE_A + S(SSL2_ST_SEND_REQUEST_CERTIFICATE_A), +#endif +#ifdef SSL2_ST_SEND_REQUEST_CERTIFICATE_B + S(SSL2_ST_SEND_REQUEST_CERTIFICATE_B), +#endif +#ifdef SSL2_ST_SEND_REQUEST_CERTIFICATE_C + S(SSL2_ST_SEND_REQUEST_CERTIFICATE_C), +#endif +#ifdef SSL2_ST_SEND_REQUEST_CERTIFICATE_D + S(SSL2_ST_SEND_REQUEST_CERTIFICATE_D), +#endif +#ifdef SSL2_ST_SEND_SERVER_FINISHED_A + S(SSL2_ST_SEND_SERVER_FINISHED_A), +#endif +#ifdef SSL2_ST_SEND_SERVER_FINISHED_B + S(SSL2_ST_SEND_SERVER_FINISHED_B), +#endif +#ifdef SSL2_ST_SEND_SERVER_HELLO_A + S(SSL2_ST_SEND_SERVER_HELLO_A), +#endif +#ifdef SSL2_ST_SEND_SERVER_HELLO_B + S(SSL2_ST_SEND_SERVER_HELLO_B), +#endif +#ifdef SSL2_ST_SEND_SERVER_VERIFY_A + S(SSL2_ST_SEND_SERVER_VERIFY_A), +#endif +#ifdef SSL2_ST_SEND_SERVER_VERIFY_B + S(SSL2_ST_SEND_SERVER_VERIFY_B), +#endif +#ifdef SSL2_ST_SEND_SERVER_VERIFY_C + S(SSL2_ST_SEND_SERVER_VERIFY_C), +#endif +#ifdef SSL2_ST_SERVER_START_ENCRYPTION + S(SSL2_ST_SERVER_START_ENCRYPTION), +#endif +#ifdef SSL2_ST_X509_GET_CLIENT_CERTIFICATE + S(SSL2_ST_X509_GET_CLIENT_CERTIFICATE), +#endif +#ifdef SSL2_ST_X509_GET_SERVER_CERTIFICATE + S(SSL2_ST_X509_GET_SERVER_CERTIFICATE), +#endif +#ifdef SSL3_ST_CR_CERT_A + S(SSL3_ST_CR_CERT_A), +#endif +#ifdef SSL3_ST_CR_CERT_B + S(SSL3_ST_CR_CERT_B), +#endif +#ifdef SSL3_ST_CR_CERT_REQ_A + S(SSL3_ST_CR_CERT_REQ_A), +#endif +#ifdef SSL3_ST_CR_CERT_REQ_B + S(SSL3_ST_CR_CERT_REQ_B), +#endif +#ifdef SSL3_ST_CR_CERT_STATUS_A + S(SSL3_ST_CR_CERT_STATUS_A), +#endif +#ifdef SSL3_ST_CR_CERT_STATUS_B + S(SSL3_ST_CR_CERT_STATUS_B), +#endif +#ifdef SSL3_ST_CR_CHANGE_A + S(SSL3_ST_CR_CHANGE_A), +#endif +#ifdef SSL3_ST_CR_CHANGE_B + S(SSL3_ST_CR_CHANGE_B), +#endif +#ifdef SSL3_ST_CR_FINISHED_A + S(SSL3_ST_CR_FINISHED_A), +#endif +#ifdef SSL3_ST_CR_FINISHED_B + S(SSL3_ST_CR_FINISHED_B), +#endif +#ifdef SSL3_ST_CR_KEY_EXCH_A + S(SSL3_ST_CR_KEY_EXCH_A), +#endif +#ifdef SSL3_ST_CR_KEY_EXCH_B + S(SSL3_ST_CR_KEY_EXCH_B), +#endif +#ifdef SSL3_ST_CR_SESSION_TICKET_A + S(SSL3_ST_CR_SESSION_TICKET_A), +#endif +#ifdef SSL3_ST_CR_SESSION_TICKET_B + S(SSL3_ST_CR_SESSION_TICKET_B), +#endif +#ifdef SSL3_ST_CR_SRVR_DONE_A + S(SSL3_ST_CR_SRVR_DONE_A), +#endif +#ifdef SSL3_ST_CR_SRVR_DONE_B + S(SSL3_ST_CR_SRVR_DONE_B), +#endif +#ifdef SSL3_ST_CR_SRVR_HELLO_A + S(SSL3_ST_CR_SRVR_HELLO_A), +#endif +#ifdef SSL3_ST_CR_SRVR_HELLO_B + S(SSL3_ST_CR_SRVR_HELLO_B), +#endif +#ifdef SSL3_ST_CW_CERT_A + S(SSL3_ST_CW_CERT_A), +#endif +#ifdef SSL3_ST_CW_CERT_B + S(SSL3_ST_CW_CERT_B), +#endif +#ifdef SSL3_ST_CW_CERT_C + S(SSL3_ST_CW_CERT_C), +#endif +#ifdef SSL3_ST_CW_CERT_D + S(SSL3_ST_CW_CERT_D), +#endif +#ifdef SSL3_ST_CW_CERT_VRFY_A + S(SSL3_ST_CW_CERT_VRFY_A), +#endif +#ifdef SSL3_ST_CW_CERT_VRFY_B + S(SSL3_ST_CW_CERT_VRFY_B), +#endif +#ifdef SSL3_ST_CW_CHANGE_A + S(SSL3_ST_CW_CHANGE_A), +#endif +#ifdef SSL3_ST_CW_CHANGE_B + S(SSL3_ST_CW_CHANGE_B), +#endif +#ifdef SSL3_ST_CW_CLNT_HELLO_A + S(SSL3_ST_CW_CLNT_HELLO_A), +#endif +#ifdef SSL3_ST_CW_CLNT_HELLO_B + S(SSL3_ST_CW_CLNT_HELLO_B), +#endif +#ifdef SSL3_ST_CW_FINISHED_A + S(SSL3_ST_CW_FINISHED_A), +#endif +#ifdef SSL3_ST_CW_FINISHED_B + S(SSL3_ST_CW_FINISHED_B), +#endif +#ifdef SSL3_ST_CW_FLUSH + S(SSL3_ST_CW_FLUSH), +#endif +#ifdef SSL3_ST_CW_KEY_EXCH_A + S(SSL3_ST_CW_KEY_EXCH_A), +#endif +#ifdef SSL3_ST_CW_KEY_EXCH_B + S(SSL3_ST_CW_KEY_EXCH_B), +#endif +#ifdef SSL3_ST_SR_CERT_A + S(SSL3_ST_SR_CERT_A), +#endif +#ifdef SSL3_ST_SR_CERT_B + S(SSL3_ST_SR_CERT_B), +#endif +#ifdef SSL3_ST_SR_CERT_VRFY_A + S(SSL3_ST_SR_CERT_VRFY_A), +#endif +#ifdef SSL3_ST_SR_CERT_VRFY_B + S(SSL3_ST_SR_CERT_VRFY_B), +#endif +#ifdef SSL3_ST_SR_CHANGE_A + S(SSL3_ST_SR_CHANGE_A), +#endif +#ifdef SSL3_ST_SR_CHANGE_B + S(SSL3_ST_SR_CHANGE_B), +#endif +#ifdef SSL3_ST_SR_CLNT_HELLO_A + S(SSL3_ST_SR_CLNT_HELLO_A), +#endif +#ifdef SSL3_ST_SR_CLNT_HELLO_B + S(SSL3_ST_SR_CLNT_HELLO_B), +#endif +#ifdef SSL3_ST_SR_CLNT_HELLO_C + S(SSL3_ST_SR_CLNT_HELLO_C), +#endif +#ifdef SSL3_ST_SR_FINISHED_A + S(SSL3_ST_SR_FINISHED_A), +#endif +#ifdef SSL3_ST_SR_FINISHED_B + S(SSL3_ST_SR_FINISHED_B), +#endif +#ifdef SSL3_ST_SR_KEY_EXCH_A + S(SSL3_ST_SR_KEY_EXCH_A), +#endif +#ifdef SSL3_ST_SR_KEY_EXCH_B + S(SSL3_ST_SR_KEY_EXCH_B), +#endif +#ifdef SSL3_ST_SW_CERT_A + S(SSL3_ST_SW_CERT_A), +#endif +#ifdef SSL3_ST_SW_CERT_B + S(SSL3_ST_SW_CERT_B), +#endif +#ifdef SSL3_ST_SW_CERT_REQ_A + S(SSL3_ST_SW_CERT_REQ_A), +#endif +#ifdef SSL3_ST_SW_CERT_REQ_B + S(SSL3_ST_SW_CERT_REQ_B), +#endif +#ifdef SSL3_ST_SW_CERT_STATUS_A + S(SSL3_ST_SW_CERT_STATUS_A), +#endif +#ifdef SSL3_ST_SW_CERT_STATUS_B + S(SSL3_ST_SW_CERT_STATUS_B), +#endif +#ifdef SSL3_ST_SW_CHANGE_A + S(SSL3_ST_SW_CHANGE_A), +#endif +#ifdef SSL3_ST_SW_CHANGE_B + S(SSL3_ST_SW_CHANGE_B), +#endif +#ifdef SSL3_ST_SW_FINISHED_A + S(SSL3_ST_SW_FINISHED_A), +#endif +#ifdef SSL3_ST_SW_FINISHED_B + S(SSL3_ST_SW_FINISHED_B), +#endif +#ifdef SSL3_ST_SW_FLUSH + S(SSL3_ST_SW_FLUSH), +#endif +#ifdef SSL3_ST_SW_HELLO_REQ_A + S(SSL3_ST_SW_HELLO_REQ_A), +#endif +#ifdef SSL3_ST_SW_HELLO_REQ_B + S(SSL3_ST_SW_HELLO_REQ_B), +#endif +#ifdef SSL3_ST_SW_HELLO_REQ_C + S(SSL3_ST_SW_HELLO_REQ_C), +#endif +#ifdef SSL3_ST_SW_KEY_EXCH_A + S(SSL3_ST_SW_KEY_EXCH_A), +#endif +#ifdef SSL3_ST_SW_KEY_EXCH_B + S(SSL3_ST_SW_KEY_EXCH_B), +#endif +#ifdef SSL3_ST_SW_SESSION_TICKET_A + S(SSL3_ST_SW_SESSION_TICKET_A), +#endif +#ifdef SSL3_ST_SW_SESSION_TICKET_B + S(SSL3_ST_SW_SESSION_TICKET_B), +#endif +#ifdef SSL3_ST_SW_SRVR_DONE_A + S(SSL3_ST_SW_SRVR_DONE_A), +#endif +#ifdef SSL3_ST_SW_SRVR_DONE_B + S(SSL3_ST_SW_SRVR_DONE_B), +#endif +#ifdef SSL3_ST_SW_SRVR_HELLO_A + S(SSL3_ST_SW_SRVR_HELLO_A), +#endif +#ifdef SSL3_ST_SW_SRVR_HELLO_B + S(SSL3_ST_SW_SRVR_HELLO_B), +#endif +#ifdef SSL_ST_ACCEPT + S(SSL_ST_ACCEPT), +#endif +#ifdef SSL_ST_BEFORE + S(SSL_ST_BEFORE), +#endif +#ifdef SSL_ST_CONNECT + S(SSL_ST_CONNECT), +#endif +#ifdef SSL_ST_INIT + S(SSL_ST_INIT), +#endif +#ifdef SSL_ST_MASK + S(SSL_ST_MASK), +#endif +#ifdef SSL_ST_OK + S(SSL_ST_OK), +#endif +#ifdef SSL_ST_READ_BODY + S(SSL_ST_READ_BODY), +#endif +#ifdef SSL_ST_READ_DONE + S(SSL_ST_READ_DONE), +#endif +#ifdef SSL_ST_READ_HEADER + S(SSL_ST_READ_HEADER), +#endif +#ifdef SSL_ST_RENEGOTIATE + S(SSL_ST_RENEGOTIATE), +#endif + { 0, NULL } +}; + +#endif + diff --git a/src/common/util.c b/src/common/util.c index 879a0e4bd3..86f4141674 100644 --- a/src/common/util.c +++ b/src/common/util.c @@ -15,7 +15,8 @@ #include "orconfig.h" #include "util.h" -#include "log.h" +#include "torlog.h" +#undef log #include "crypto.h" #include "torint.h" #include "container.h" @@ -25,11 +26,17 @@ #include <io.h> #include <direct.h> #include <process.h> +#include <tchar.h> #else #include <dirent.h> #include <pwd.h> #endif +/* math.h needs this on Linux */ +#ifndef __USE_ISOC99 +#define __USE_ISOC99 1 +#endif +#include <math.h> #include <stdlib.h> #include <stdio.h> #include <string.h> @@ -287,7 +294,7 @@ tor_log_mallinfo(int severity) struct mallinfo mi; memset(&mi, 0, sizeof(mi)); mi = mallinfo(); - log(severity, LD_MM, + tor_log(severity, LD_MM, "mallinfo() said: arena=%d, ordblks=%d, smblks=%d, hblks=%d, " "hblkhd=%d, usmblks=%d, fsmblks=%d, uordblks=%d, fordblks=%d, " "keepcost=%d", @@ -310,6 +317,25 @@ tor_log_mallinfo(int severity) * Math * ===== */ +/** + * Returns the natural logarithm of d base 2. We define this wrapper here so + * as to make it easier not to conflict with Tor's log() macro. + */ +double +tor_mathlog(double d) +{ + return log(d); +} + +/** Return the long integer closest to d. We define this wrapper here so + * that not all users of math.h need to use the right incancations to get + * the c99 functions. */ +long +tor_lround(double d) +{ + return lround(d); +} + /** Returns floor(log2(u64)). If u64 is 0, (incorrectly) returns 0. */ int tor_log2(uint64_t u64) @@ -354,6 +380,36 @@ round_to_power_of_2(uint64_t u64) return low; } +/** Return the lowest x such that x is at least <b>number</b>, and x modulo + * <b>divisor</b> == 0. */ +unsigned +round_to_next_multiple_of(unsigned number, unsigned divisor) +{ + number += divisor - 1; + number -= number % divisor; + return number; +} + +/** Return the lowest x such that x is at least <b>number</b>, and x modulo + * <b>divisor</b> == 0. */ +uint32_t +round_uint32_to_next_multiple_of(uint32_t number, uint32_t divisor) +{ + number += divisor - 1; + number -= number % divisor; + return number; +} + +/** Return the lowest x such that x is at least <b>number</b>, and x modulo + * <b>divisor</b> == 0. */ +uint64_t +round_uint64_to_next_multiple_of(uint64_t number, uint64_t divisor) +{ + number += divisor - 1; + number -= number % divisor; + return number; +} + /* ===== * String manipulation * ===== */ @@ -636,6 +692,29 @@ find_whitespace_eos(const char *s, const char *eos) return s; } +/** Return the first occurrence of <b>needle</b> in <b>haystack</b> that + * occurs at the start of a line (that is, at the beginning of <b>haystack</b> + * or immediately after a newline). Return NULL if no such string is found. + */ +const char * +find_str_at_start_of_line(const char *haystack, const char *needle) +{ + size_t needle_len = strlen(needle); + + do { + if (!strncmp(haystack, needle, needle_len)) + return haystack; + + haystack = strchr(haystack, '\n'); + if (!haystack) + return NULL; + else + ++haystack; + } while (*haystack); + + return NULL; +} + /** Return true iff the 'len' bytes at 'mem' are all zero. */ int tor_mem_is_zero(const char *mem, size_t len) @@ -668,6 +747,13 @@ tor_digest_is_zero(const char *digest) return tor_memeq(digest, ZERO_DIGEST, DIGEST_LEN); } +/** Return true iff the DIGEST256_LEN bytes in digest are all zero. */ +int +tor_digest256_is_zero(const char *digest) +{ + return tor_mem_is_zero(digest, DIGEST256_LEN); +} + /* Helper: common code to check whether the result of a strtol or strtoul or * strtoll is correct. */ #define CHECK_STRTOX_RESULT() \ @@ -719,7 +805,18 @@ tor_parse_ulong(const char *s, int base, unsigned long min, CHECK_STRTOX_RESULT(); } -/** As tor_parse_log, but return a unit64_t. Only base 10 is guaranteed to +/** As tor_parse_long(), but return a double. */ +double +tor_parse_double(const char *s, double min, double max, int *ok, char **next) +{ + char *endptr; + double r; + + r = strtod(s, &endptr); + CHECK_STRTOX_RESULT(); +} + +/** As tor_parse_long, but return a uint64_t. Only base 10 is guaranteed to * work for now. */ uint64_t tor_parse_uint64(const char *s, int base, uint64_t min, @@ -858,6 +955,9 @@ esc_for_log(const char *s) case '\\': case '\"': case '\'': + case '\r': + case '\n': + case '\t': len += 2; break; default: @@ -919,8 +1019,7 @@ const char * escaped(const char *s) { static char *_escaped_val = NULL; - if (_escaped_val) - tor_free(_escaped_val); + tor_free(_escaped_val); if (s) _escaped_val = esc_for_log(s); @@ -1007,6 +1106,42 @@ wrap_string(smartlist_t *out, const char *string, size_t width, * Time * ===== */ +/** + * Converts struct timeval to a double value. + * Preserves microsecond precision, but just barely. + * Error is approx +/- 0.1 usec when dealing with epoch values. + */ +double +tv_to_double(const struct timeval *tv) +{ + double conv = tv->tv_sec; + conv += tv->tv_usec/1000000.0; + return conv; +} + +/** + * Converts timeval to milliseconds. + */ +int64_t +tv_to_msec(const struct timeval *tv) +{ + int64_t conv = ((int64_t)tv->tv_sec)*1000L; + /* Round ghetto-style */ + conv += ((int64_t)tv->tv_usec+500)/1000L; + return conv; +} + +/** + * Converts timeval to microseconds. + */ +int64_t +tv_to_usec(const struct timeval *tv) +{ + int64_t conv = ((int64_t)tv->tv_sec)*1000000L; + conv += tv->tv_usec; + return conv; +} + /** Return the number of microseconds elapsed between *start and *end. */ long @@ -1016,7 +1151,8 @@ tv_udiff(const struct timeval *start, const struct timeval *end) long secdiff = end->tv_sec - start->tv_sec; if (labs(secdiff+1) > LONG_MAX/1000000) { - log_warn(LD_GENERAL, "comparing times too far apart."); + log_warn(LD_GENERAL, "comparing times on microsecond detail too far " + "apart: %ld seconds", secdiff); return LONG_MAX; } @@ -1024,6 +1160,26 @@ tv_udiff(const struct timeval *start, const struct timeval *end) return udiff; } +/** Return the number of milliseconds elapsed between *start and *end. + */ +long +tv_mdiff(const struct timeval *start, const struct timeval *end) +{ + long mdiff; + long secdiff = end->tv_sec - start->tv_sec; + + if (labs(secdiff+1) > LONG_MAX/1000) { + log_warn(LD_GENERAL, "comparing times on millisecond detail too far " + "apart: %ld seconds", secdiff); + return LONG_MAX; + } + + /* Subtract and round */ + mdiff = secdiff*1000L + + ((long)end->tv_usec - (long)start->tv_usec + 500L) / 1000L; + return mdiff; +} + /** Yield true iff <b>y</b> is a leap-year. */ #define IS_LEAPYEAR(y) (!(y % 4) && ((y % 100) || !(y % 400))) /** Helper: Return the number of leap-days between Jan 1, y1 and Jan 1, y2. */ @@ -1099,7 +1255,7 @@ format_rfc1123_time(char *buf, time_t t) memcpy(buf+8, MONTH_NAMES[tm.tm_mon], 3); } -/** Parse the the RFC1123 encoding of some time (in GMT) from <b>buf</b>, +/** Parse the RFC1123 encoding of some time (in GMT) from <b>buf</b>, * and store the result in *<b>t</b>. * * Return 0 on success, -1 on failure. @@ -1360,80 +1516,46 @@ update_approx_time(time_t now) #endif /* ===== - * Fuzzy time - * XXXX022 Use this consistently or rip most of it out. + * Rate limiting * ===== */ -/* In a perfect world, everybody would run NTP, and NTP would be perfect, so - * if we wanted to know "Is the current time before time X?" we could just say - * "time(NULL) < X". - * - * But unfortunately, many users are running Tor in an imperfect world, on - * even more imperfect computers. Hence, we need to track time oddly. We - * model the user's computer as being "skewed" from accurate time by - * -<b>ftime_skew</b> seconds, such that our best guess of the current time is - * time(NULL)+ftime_skew. We also assume that our measurements of time may - * have up to <b>ftime_slop</b> seconds of inaccuracy; IOW, our window of - * estimate for the current time is now + ftime_skew +/- ftime_slop. - */ -/** Our current estimate of our skew, such that we think the current time is - * closest to time(NULL)+ftime_skew. */ -static int ftime_skew = 0; -/** Tolerance during time comparisons, in seconds. */ -static int ftime_slop = 60; -/** Set the largest amount of sloppiness we'll allow in fuzzy time - * comparisons. */ -void -ftime_set_maximum_sloppiness(int seconds) -{ - tor_assert(seconds >= 0); - ftime_slop = seconds; -} -/** Set the amount by which we believe our system clock to differ from - * real time. */ -void -ftime_set_estimated_skew(int seconds) -{ - ftime_skew = seconds; -} -#if 0 -void -ftime_get_window(time_t now, ftime_t *ft_out) -{ - ft_out->earliest = now + ftime_skew - ftime_slop; - ft_out->latest = now + ftime_skew + ftime_slop; -} -#endif -/** Return true iff we think that <b>now</b> might be after <b>when</b>. */ -int -ftime_maybe_after(time_t now, time_t when) -{ - /* It may be after when iff the latest possible current time is after when */ - return (now + ftime_skew + ftime_slop) >= when; -} -/** Return true iff we think that <b>now</b> might be before <b>when</b>. */ -int -ftime_maybe_before(time_t now, time_t when) -{ - /* It may be before when iff the earliest possible current time is before */ - return (now + ftime_skew - ftime_slop) < when; -} -/** Return true if we think that <b>now</b> is definitely after <b>when</b>. */ -int -ftime_definitely_after(time_t now, time_t when) +/** If the rate-limiter <b>lim</b> is ready at <b>now</b>, return the number + * of calls to rate_limit_is_ready (including this one!) since the last time + * rate_limit_is_ready returned nonzero. Otherwise return 0. */ +static int +rate_limit_is_ready(ratelim_t *lim, time_t now) { - /* It is definitely after when if the earliest time it could be is still - * after when. */ - return (now + ftime_skew - ftime_slop) >= when; + if (lim->rate + lim->last_allowed <= now) { + int res = lim->n_calls_since_last_time + 1; + lim->last_allowed = now; + lim->n_calls_since_last_time = 0; + return res; + } else { + ++lim->n_calls_since_last_time; + return 0; + } } -/** Return true if we think that <b>now</b> is definitely before <b>when</b>. - */ -int -ftime_definitely_before(time_t now, time_t when) + +/** If the rate-limiter <b>lim</b> is ready at <b>now</b>, return a newly + * allocated string indicating how many messages were suppressed, suitable to + * append to a log message. Otherwise return NULL. */ +char * +rate_limit_log(ratelim_t *lim, time_t now) { - /* It is definitely before when if the latest time it could be is still - * before when. */ - return (now + ftime_skew + ftime_slop) < when; + int n; + if ((n = rate_limit_is_ready(lim, now))) { + if (n == 1) { + return tor_strdup(""); + } else { + char *cp=NULL; + tor_asprintf(&cp, + " [%d similar message(s) suppressed in last %d seconds]", + n-1, lim->rate); + return cp; + } + } else { + return NULL; + } } /* ===== @@ -1560,22 +1682,22 @@ check_private_dir(const char *dirname, cpd_check_t check) tor_free(f); if (r) { if (errno != ENOENT) { - log(LOG_WARN, LD_FS, "Directory %s cannot be read: %s", dirname, - strerror(errno)); + log_warn(LD_FS, "Directory %s cannot be read: %s", dirname, + strerror(errno)); return -1; } if (check == CPD_NONE) { - log(LOG_WARN, LD_FS, "Directory %s does not exist.", dirname); + log_warn(LD_FS, "Directory %s does not exist.", dirname); return -1; } else if (check == CPD_CREATE) { log_info(LD_GENERAL, "Creating directory %s", dirname); -#ifdef MS_WINDOWS +#if defined (MS_WINDOWS) && !defined (WINCE) r = mkdir(dirname); #else r = mkdir(dirname, 0700); #endif if (r) { - log(LOG_WARN, LD_FS, "Error creating directory %s: %s", dirname, + log_warn(LD_FS, "Error creating directory %s: %s", dirname, strerror(errno)); return -1; } @@ -1585,7 +1707,7 @@ check_private_dir(const char *dirname, cpd_check_t check) return 0; } if (!(st.st_mode & S_IFDIR)) { - log(LOG_WARN, LD_FS, "%s is not a directory", dirname); + log_warn(LD_FS, "%s is not a directory", dirname); return -1; } #ifndef MS_WINDOWS @@ -1598,7 +1720,7 @@ check_private_dir(const char *dirname, cpd_check_t check) pw = getpwuid(st.st_uid); - log(LOG_WARN, LD_FS, "%s is not owned by this user (%s, %d) but by " + log_warn(LD_FS, "%s is not owned by this user (%s, %d) but by " "%s (%d). Perhaps you are running Tor as the wrong user?", dirname, process_ownername, (int)getuid(), pw ? pw->pw_name : "<unknown>", (int)st.st_uid); @@ -1607,9 +1729,9 @@ check_private_dir(const char *dirname, cpd_check_t check) return -1; } if (st.st_mode & 0077) { - log(LOG_WARN, LD_FS, "Fixing permissions on directory %s", dirname); + log_warn(LD_FS, "Fixing permissions on directory %s", dirname); if (chmod(dirname, 0700)) { - log(LOG_WARN, LD_FS, "Could not chmod directory %s: %s", dirname, + log_warn(LD_FS, "Could not chmod directory %s: %s", dirname, strerror(errno)); return -1; } else { @@ -1640,12 +1762,13 @@ write_str_to_file(const char *fname, const char *str, int bin) } /** Represents a file that we're writing to, with support for atomic commit: - * we can write into a a temporary file, and either remove the file on + * we can write into a temporary file, and either remove the file on * failure, or replace the original file on success. */ struct open_file_t { char *tempname; /**< Name of the temporary file. */ char *filename; /**< Name of the original file. */ - int rename_on_close; /**< Are we using the temporary file or not? */ + unsigned rename_on_close:1; /**< Are we using the temporary file or not? */ + unsigned binary:1; /**< Did we open in binary mode? */ int fd; /**< fd for the open file. */ FILE *stdio_file; /**< stdio wrapper for <b>fd</b>. */ }; @@ -1693,7 +1816,7 @@ start_writing_to_file(const char *fname, int open_flags, int mode, } else { open_name = new_file->tempname = tor_malloc(tempname_len); if (tor_snprintf(new_file->tempname, tempname_len, "%s.tmp", fname)<0) { - log(LOG_WARN, LD_GENERAL, "Failed to generate filename"); + log_warn(LD_GENERAL, "Failed to generate filename"); goto err; } /* We always replace an existing temporary file if there is one. */ @@ -1701,9 +1824,12 @@ start_writing_to_file(const char *fname, int open_flags, int mode, open_flags &= ~O_EXCL; new_file->rename_on_close = 1; } + if (open_flags & O_BINARY) + new_file->binary = 1; - if ((new_file->fd = open(open_name, open_flags, mode)) < 0) { - log(LOG_WARN, LD_FS, "Couldn't open \"%s\" (%s) for writing: %s", + new_file->fd = open(open_name, open_flags, mode); + if (new_file->fd < 0) { + log_warn(LD_FS, "Couldn't open \"%s\" (%s) for writing: %s", open_name, fname, strerror(errno)); goto err; } @@ -1739,7 +1865,8 @@ fdopen_file(open_file_t *file_data) if (file_data->stdio_file) return file_data->stdio_file; tor_assert(file_data->fd >= 0); - if (!(file_data->stdio_file = fdopen(file_data->fd, "a"))) { + if (!(file_data->stdio_file = fdopen(file_data->fd, + file_data->binary?"ab":"a"))) { log_warn(LD_FS, "Couldn't fdopen \"%s\" [%d]: %s", file_data->filename, file_data->fd, strerror(errno)); } @@ -1839,7 +1966,7 @@ write_chunks_to_file_impl(const char *fname, const smartlist_t *chunks, { result = write_all(fd, chunk->bytes, chunk->len, 0); if (result < 0) { - log(LOG_WARN, LD_FS, "Error writing to \"%s\": %s", fname, + log_warn(LD_FS, "Error writing to \"%s\": %s", fname, strerror(errno)); goto err; } @@ -2095,7 +2222,40 @@ unescape_string(const char *s, char **result, size_t *size_out) const char * parse_config_line_from_str(const char *line, char **key_out, char **value_out) { + /* I believe the file format here is supposed to be: + FILE = (EMPTYLINE | LINE)* (EMPTYLASTLINE | LASTLINE)? + + EMPTYLASTLINE = SPACE* | COMMENT + EMPTYLINE = EMPTYLASTLINE NL + SPACE = ' ' | '\r' | '\t' + COMMENT = '#' NOT-NL* + NOT-NL = Any character except '\n' + NL = '\n' + + LASTLINE = SPACE* KEY SPACE* VALUES + LINE = LASTLINE NL + KEY = KEYCHAR+ + KEYCHAR = Any character except ' ', '\r', '\n', '\t', '#', "\" + + VALUES = QUOTEDVALUE | NORMALVALUE + QUOTEDVALUE = QUOTE QVITEM* QUOTE EOLSPACE? + QUOTE = '"' + QVCHAR = KEYCHAR | ESC ('n' | 't' | 'r' | '"' | ESC |'\'' | OCTAL | HEX) + ESC = "\\" + OCTAL = ODIGIT (ODIGIT ODIGIT?)? + HEX = ('x' | 'X') HEXDIGIT HEXDIGIT + ODIGIT = '0' .. '7' + HEXDIGIT = '0'..'9' | 'a' .. 'f' | 'A' .. 'F' + EOLSPACE = SPACE* COMMENT? + + NORMALVALUE = (VALCHAR | ESC ESC_IGNORE | CONTINUATION)* EOLSPACE? + VALCHAR = Any character except ESC, '#', and '\n' + ESC_IGNORE = Any character except '#' or '\n' + CONTINUATION = ESC NL ( COMMENT NL )* + */ + const char *key, *val, *cp; + int continuation = 0; tor_assert(key_out); tor_assert(value_out); @@ -2119,9 +2279,10 @@ parse_config_line_from_str(const char *line, char **key_out, char **value_out) return line; } - /* Skip until the next space. */ + /* Skip until the next space or \ followed by newline. */ key = line; - while (*line && !TOR_ISSPACE(*line) && *line != '#') + while (*line && !TOR_ISSPACE(*line) && *line != '#' && + ! (line[0] == '\\' && line[1] == '\n')) ++line; *key_out = tor_strndup(key, line-key); @@ -2132,7 +2293,7 @@ parse_config_line_from_str(const char *line, char **key_out, char **value_out) val = line; /* Find the end of the line. */ - if (*line == '\"') { + if (*line == '\"') { // XXX No continuation handling is done here if (!(line = unescape_string(line, value_out, NULL))) return NULL; while (*line == ' ' || *line == '\t') @@ -2140,18 +2301,53 @@ parse_config_line_from_str(const char *line, char **key_out, char **value_out) if (*line && *line != '#' && *line != '\n') return NULL; } else { - while (*line && *line != '\n' && *line != '#') - ++line; + /* Look for the end of the line. */ + while (*line && *line != '\n' && (*line != '#' || continuation)) { + if (*line == '\\' && line[1] == '\n') { + continuation = 1; + line += 2; + } else if (*line == '#') { + do { + ++line; + } while (*line && *line != '\n'); + if (*line == '\n') + ++line; + } else { + ++line; + } + } + if (*line == '\n') { cp = line++; } else { cp = line; } + /* Now back cp up to be the last nonspace character */ while (cp>val && TOR_ISSPACE(*(cp-1))) --cp; tor_assert(cp >= val); + + /* Now copy out and decode the value. */ *value_out = tor_strndup(val, cp-val); + if (continuation) { + char *v_out, *v_in; + v_out = v_in = *value_out; + while (*v_in) { + if (*v_in == '#') { + do { + ++v_in; + } while (*v_in && *v_in != '\n'); + if (*v_in == '\n') + ++v_in; + } else if (v_in[0] == '\\' && v_in[1] == '\n') { + v_in += 2; + } else { + *v_out++ = *v_in++; + } + } + *v_out = '\0'; + } } if (*line == '#') { @@ -2170,19 +2366,22 @@ char * expand_filename(const char *filename) { tor_assert(filename); +#ifdef MS_WINDOWS + return tor_strdup(filename); +#else if (*filename == '~') { - size_t len; - char *home, *result; + char *home, *result=NULL; const char *rest; if (filename[1] == '/' || filename[1] == '\0') { home = getenv("HOME"); if (!home) { log_warn(LD_CONFIG, "Couldn't find $HOME environment variable while " - "expanding \"%s\"", filename); - return NULL; + "expanding \"%s\"; defaulting to \"\".", filename); + home = tor_strdup(""); + } else { + home = tor_strdup(home); } - home = tor_strdup(home); rest = strlen(filename)>=2?(filename+2):""; } else { #ifdef HAVE_PWD_H @@ -2209,21 +2408,19 @@ expand_filename(const char *filename) if (strlen(home)>1 && !strcmpend(home,PATH_SEPARATOR)) { home[strlen(home)-1] = '\0'; } - /* Plus one for /, plus one for NUL. - * Round up to 16 in case we can't do math. */ - len = strlen(home)+strlen(rest)+16; - result = tor_malloc(len); - tor_snprintf(result,len,"%s"PATH_SEPARATOR"%s",home,rest); + tor_asprintf(&result,"%s"PATH_SEPARATOR"%s",home,rest); tor_free(home); return result; } else { return tor_strdup(filename); } +#endif } #define MAX_SCANF_WIDTH 9999 -/** DOCDOC */ +/** Helper: given an ASCII-encoded decimal digit, return its numeric value. + * NOTE: requires that its input be in-bounds. */ static int digit_to_num(char d) { @@ -2232,7 +2429,10 @@ digit_to_num(char d) return num; } -/** DOCDOC */ +/** Helper: Read an unsigned int from *<b>bufp</b> of up to <b>width</b> + * characters. (Handle arbitrary width if <b>width</b> is less than 0.) On + * success, store the result in <b>out</b>, advance bufp to the next + * character, and return 0. On failure, return -1. */ static int scan_unsigned(const char **bufp, unsigned *out, int width) { @@ -2259,7 +2459,9 @@ scan_unsigned(const char **bufp, unsigned *out, int width) return 0; } -/** DOCDOC */ +/** Helper: copy up to <b>width</b> non-space characters from <b>bufp</b> to + * <b>out</b>. Make sure <b>out</b> is nul-terminated. Advance <b>bufp</b> + * to the next non-space character or the EOS. */ static int scan_string(const char **bufp, char *out, int width) { @@ -2348,7 +2550,12 @@ tor_vsscanf(const char *buf, const char *pattern, va_list ap) * and store the results in the corresponding argument fields. Differs from * sscanf in that it: Only handles %u and %Ns. Does not handle arbitrarily * long widths. %u does not consume any space. Is locale-independent. - * Returns -1 on malformed patterns. */ + * Returns -1 on malformed patterns. + * + * (As with other locale-independent functions, we need this to parse data that + * is in ASCII without worrying that the C library's locale-handling will make + * miscellaneous characters look like numbers, spaces, and so on.) + */ int tor_sscanf(const char *buf, const char *pattern, ...) { @@ -2369,20 +2576,32 @@ tor_listdir(const char *dirname) smartlist_t *result; #ifdef MS_WINDOWS char *pattern; + TCHAR tpattern[MAX_PATH] = {0}; + char name[MAX_PATH] = {0}; HANDLE handle; WIN32_FIND_DATA findData; size_t pattern_len = strlen(dirname)+16; pattern = tor_malloc(pattern_len); tor_snprintf(pattern, pattern_len, "%s\\*", dirname); - if (INVALID_HANDLE_VALUE == (handle = FindFirstFile(pattern, &findData))) { +#ifdef UNICODE + mbstowcs(tpattern,pattern,MAX_PATH); +#else + strlcpy(tpattern, pattern, MAX_PATH); +#endif + if (INVALID_HANDLE_VALUE == (handle = FindFirstFile(tpattern, &findData))) { tor_free(pattern); return NULL; } result = smartlist_create(); while (1) { - if (strcmp(findData.cFileName, ".") && - strcmp(findData.cFileName, "..")) { - smartlist_add(result, tor_strdup(findData.cFileName)); +#ifdef UNICODE + wcstombs(name,findData.cFileName,MAX_PATH); +#else + strlcpy(name,findData.cFileName,sizeof(name)); +#endif + if (strcmp(name, ".") && + strcmp(name, "..")) { + smartlist_add(result, tor_strdup(name)); } if (!FindNextFile(handle, &findData)) { DWORD err; @@ -2581,3 +2800,18 @@ write_pidfile(char *filename) } } +#ifdef MS_WINDOWS +HANDLE +load_windows_system_library(const TCHAR *library_name) +{ + TCHAR path[MAX_PATH]; + unsigned n; + n = GetSystemDirectory(path, MAX_PATH); + if (n == 0 || n + _tcslen(library_name) + 2 >= MAX_PATH) + return 0; + _tcscat(path, TEXT("\\")); + _tcscat(path, library_name); + return LoadLibrary(path); +} +#endif + diff --git a/src/common/util.h b/src/common/util.h index 1012a111da..961b5875ad 100644 --- a/src/common/util.h +++ b/src/common/util.h @@ -44,7 +44,7 @@ * stderr. */ #define tor_assert(expr) STMT_BEGIN \ if (PREDICT_UNLIKELY(!(expr))) { \ - log(LOG_ERR, LD_BUG, "%s:%d: %s: Assertion %s failed; aborting.", \ + log_err(LD_BUG, "%s:%d: %s: Assertion %s failed; aborting.", \ _SHORT_FILE_, __LINE__, __func__, #expr); \ fprintf(stderr,"%s:%d %s: Assertion %s failed; aborting.\n", \ _SHORT_FILE_, __LINE__, __func__, #expr); \ @@ -153,8 +153,18 @@ void tor_log_mallinfo(int severity); #define bool_neq(a,b) (!(a)!=!(b)) /* Math functions */ +double tor_mathlog(double d) ATTR_CONST; +long tor_lround(double d) ATTR_CONST; int tor_log2(uint64_t u64) ATTR_CONST; uint64_t round_to_power_of_2(uint64_t u64); +unsigned round_to_next_multiple_of(unsigned number, unsigned divisor); +uint32_t round_uint32_to_next_multiple_of(uint32_t number, uint32_t divisor); +uint64_t round_uint64_to_next_multiple_of(uint64_t number, uint64_t divisor); + +/* Compute the CEIL of <b>a</b> divided by <b>b</b>, for nonnegative <b>a</b> + * and positive <b>b</b>. Works on integer types only. Not defined if a+b can + * overflow. */ +#define CEIL_DIV(a,b) (((a)+(b)-1)/(b)) /* String manipulation */ @@ -180,6 +190,8 @@ long tor_parse_long(const char *s, int base, long min, long max, int *ok, char **next); unsigned long tor_parse_ulong(const char *s, int base, unsigned long min, unsigned long max, int *ok, char **next); +double tor_parse_double(const char *s, double min, double max, int *ok, + char **next); uint64_t tor_parse_uint64(const char *s, int base, uint64_t min, uint64_t max, int *ok, char **next); const char *hex_str(const char *from, size_t fromlen) ATTR_NONNULL((1)); @@ -189,8 +201,11 @@ const char *eat_whitespace_no_nl(const char *s) ATTR_PURE; const char *eat_whitespace_eos_no_nl(const char *s, const char *eos) ATTR_PURE; const char *find_whitespace(const char *s) ATTR_PURE; const char *find_whitespace_eos(const char *s, const char *eos) ATTR_PURE; +const char *find_str_at_start_of_line(const char *haystack, const char *needle) + ATTR_PURE; int tor_mem_is_zero(const char *mem, size_t len) ATTR_PURE; int tor_digest_is_zero(const char *digest) ATTR_PURE; +int tor_digest256_is_zero(const char *digest) ATTR_PURE; char *esc_for_log(const char *string) ATTR_MALLOC; const char *escaped(const char *string); struct smartlist_t; @@ -208,7 +223,11 @@ void base16_encode(char *dest, size_t destlen, const char *src, size_t srclen); int base16_decode(char *dest, size_t destlen, const char *src, size_t srclen); /* Time helpers */ +double tv_to_double(const struct timeval *tv); +int64_t tv_to_msec(const struct timeval *tv); +int64_t tv_to_usec(const struct timeval *tv); long tv_udiff(const struct timeval *start, const struct timeval *end); +long tv_mdiff(const struct timeval *start, const struct timeval *end); time_t tor_timegm(struct tm *tm); #define RFC1123_TIME_LEN 29 void format_rfc1123_time(char *buf, time_t t); @@ -229,15 +248,32 @@ time_t approx_time(void); void update_approx_time(time_t now); #endif -/* Fuzzy time. */ -void ftime_set_maximum_sloppiness(int seconds); -void ftime_set_estimated_skew(int seconds); -/* typedef struct ftime_t { time_t earliest; time_t latest; } ftime_t; */ -/* void ftime_get_window(time_t now, ftime_t *ft_out); */ -int ftime_maybe_after(time_t now, time_t when); -int ftime_maybe_before(time_t now, time_t when); -int ftime_definitely_after(time_t now, time_t when); -int ftime_definitely_before(time_t now, time_t when); +/* Rate-limiter */ + +/** A ratelim_t remembers how often an event is occurring, and how often + * it's allowed to occur. Typical usage is something like: + * + <pre> + if (possibly_very_frequent_event()) { + const int INTERVAL = 300; + static ratelim_t warning_limit = RATELIM_INIT(INTERVAL); + char *m; + if ((m = rate_limit_log(&warning_limit, approx_time()))) { + log_warn(LD_GENERAL, "The event occurred!%s", m); + tor_free(m); + } + } + </pre> + */ +typedef struct ratelim_t { + int rate; + time_t last_allowed; + int n_calls_since_last_time; +} ratelim_t; + +#define RATELIM_INIT(r) { (r), 0, 0 } + +char *rate_limit_log(ratelim_t *lim, time_t now); /* File helpers */ ssize_t write_all(int fd, const char *buf, size_t count, int isSocket); @@ -295,5 +331,11 @@ void start_daemon(void); void finish_daemon(const char *desired_cwd); void write_pidfile(char *filename); +#ifdef MS_WINDOWS +HANDLE load_windows_system_library(const TCHAR *library_name); +#endif + +const char *libor_get_digests(void); + #endif diff --git a/src/common/util_codedigest.c b/src/common/util_codedigest.c new file mode 100644 index 0000000000..88fe508b92 --- /dev/null +++ b/src/common/util_codedigest.c @@ -0,0 +1,11 @@ + +#include "util.h" + +const char * +libor_get_digests(void) +{ + return "" +#include "common_sha1.i" + ; +} + diff --git a/src/config/geoip b/src/config/geoip index 39aceddd04..465c13be7a 100644 --- a/src/config/geoip +++ b/src/config/geoip @@ -1,23 +1,46 @@ -# Last updated based on April 1 2011 Maxmind GeoLite Country +# Last updated based on May 1 2011 Maxmind GeoLite Country # wget http://geolite.maxmind.com/download/geoip/database/GeoIPCountryCSV.zip # cut -d, -f3-5 < GeoIPCountryWhois.csv|sed 's/"//g' > geoip -16777216,16908287,AU -16908800,16909055,CN +16777216,16777471,AU +16777472,16778239,CN +16778240,16779263,AU +16779264,16781311,CN +16781312,16785407,JP +16785408,16793599,CN +16793600,16809983,JP +16809984,16842751,TH +16842752,16843007,CN +16843008,16843263,AU +16843264,16859135,CN +16859136,16875519,JP +16875520,16908287,TH +16908288,16909055,CN 16909056,16909311,AU +16909312,16941055,CN +16941056,16973823,TH +16973824,17039359,CN 17039360,17039615,AU -17040384,17040639,CN +17039616,17072127,CN +17072128,17104895,TH +17104896,17170431,JP 17170432,17301503,IN +17301504,17367039,CN 17367040,17432575,MY +17432576,17435135,CN 17435136,17435391,AU +17435392,17465343,CN +17465344,17498111,TH 17498112,17563647,KR 17563648,17825791,CN 17825792,18087935,KR +18087936,18153471,TH 18153472,18219007,JP 18219008,18350079,IN -18350080,18874367,CN +18350080,18939903,CN 18939904,19005439,JP 19005440,19136511,TW 19136512,19202047,HK +19202048,19267583,PH 19267584,19398655,IN 19398656,19726335,AU 19726336,19791871,CN @@ -36,10 +59,12 @@ 24641536,27262975,AU 27262976,28311551,TW 28311552,28442623,KR -28442624,28573695,AU +28442624,28540927,AU +28540928,28573695,TH 28573696,28966911,CN -28966912,29032447,IN +28966912,29097983,IN 29097984,29884415,CN +29884416,29949951,TW 29949952,30015487,KR 30015488,30408703,CN 30408704,33554431,KR @@ -72,15 +97,35 @@ 34911232,34911743,DE 34911744,34911999,AT 34912000,34912255,GB -34912256,34913279,DE -34913280,34930687,EU +34912256,34912511,DE +34912512,34912767,ES +34912768,34913279,DE +34913280,34928383,EU +34928384,34928639,DE +34928640,34930687,EU 34930688,34938879,DE -34938880,35127295,EU +34938880,34952703,EU +34952704,34953215,ES +34953216,34954751,AT +34954752,34955263,NL +34955264,34959359,AT +34959360,34963455,NL +34963456,34992127,EU +34992128,34993151,NL +34993152,34993663,EU +34993664,34993919,AT +34993920,35127295,EU 35127296,35651583,GB 35651584,36700159,IT 36700160,36962303,AE 36962304,37224447,IL -37224448,37486591,UA +37224448,37265407,UA +37265408,37268479,CZ +37268480,37289983,UA +37289984,37298175,RU +37298176,37355519,UA +37355520,37421055,RU +37421056,37486591,UA 37486592,37748735,RU 37748736,38273023,SE 38273024,38797311,KZ @@ -622,11 +667,14 @@ 217028008,217046775,US 217046776,217046783,PR 217046784,234881023,US +234881024,234883071,CN 234883072,234884095,JP +234884096,234885119,CN 234885120,234889215,VN 234889216,234913791,KR 234913792,234946559,HK -234946560,234950655,JP +234946560,234947583,CN +234947584,234950655,JP 234950656,234951679,AU 234951680,234952703,HK 234952704,234954751,JP @@ -643,6 +691,7 @@ 241434624,241500159,SG 241500160,241565695,JP 241565696,241598463,IN +241598464,241599487,CN 241599488,241600511,JP 241600512,241602559,AU 241602560,241604607,MY @@ -676,6 +725,7 @@ 247479296,247480319,CN 247480320,247482367,MY 247482368,247483391,PG +247483392,247484415,CN 247484416,247488511,KR 247488512,247496703,JP 247496704,247504895,PK @@ -846,8 +896,8 @@ 417775616,417796095,CA 417796096,417800191,US 417800192,417808383,BS -417808384,417824767,CA -417824768,417857535,US +417808384,417820671,CA +417820672,417857535,US 417857536,417923071,AR 417923072,418062335,US 418062336,418070527,CA @@ -866,8 +916,8 @@ 418709504,418766847,US 418766848,418770943,CA 418770944,418775039,US -418775040,418807807,CA -418807808,419430399,US +418775040,418799615,CA +418799616,419430399,US 419430400,436207615,GB 436207616,452984831,US 452984832,452985855,JP @@ -889,6 +939,7 @@ 455258112,455262207,SG 455262208,455270399,JP 455270400,455272447,AU +455272448,455274495,CN 455274496,455278591,IN 455278592,455344127,KR 455344128,456130559,CN @@ -899,18 +950,22 @@ 456264704,456265727,JP 456265728,456269823,ID 456269824,456271871,HK +456271872,456273919,CN 456273920,456286207,AU 456286208,456294399,JP 456294400,456327167,CN 456327168,456523775,TW 456523776,456540159,SG -456540160,456548351,AU +456540160,456542207,AU +456542208,456544255,CN +456544256,456548351,AU 456548352,456553471,JP 456553472,456554495,MY 456554496,456555519,PK 456555520,456556543,JP 456556544,456560639,AU 456560640,456562687,IN +456562688,456564735,CN 456564736,456572927,IN 456572928,456589311,CN 456589312,456654847,TH @@ -1023,6 +1078,7 @@ 460734464,460865535,KR 460865536,460931071,JP 460931072,460933119,AU +460933120,460935167,CN 460935168,460937215,ID 460937216,460938239,NZ 460938240,460939263,JP @@ -1031,6 +1087,7 @@ 460941312,460942335,AU 460942336,460943359,MY 460943360,460945407,AU +460945408,460947455,CN 460947456,460980223,JP 460980224,460981247,NC 460981248,460983295,JP @@ -1115,6 +1172,102 @@ 469712896,469729279,KR 469729280,469762047,IN 469762048,520093695,US +520093696,520257535,PL +520257536,520290303,RO +520290304,520292351,TR +520292352,520294399,NL +520294400,520296447,RU +520296448,520298495,UA +520298496,520306687,SK +520306688,520308735,GB +520308736,520310783,DK +520310784,520312831,DE +520312832,520314879,MK +520314880,520318975,NL +520318976,520323071,ES +520323072,520325119,CH +520325120,520327167,FR +520329216,520331263,SE +520331264,520339455,IT +520339456,520343551,UA +520343552,520355839,GB +520355840,520421375,ES +520421376,520486911,AT +520486912,520488959,NL +520488960,520491007,IT +520491008,520493567,GB +520493568,520501759,IT +520501760,520503295,GB +520505344,520507391,GB +520507392,520511487,TR +520511488,520519679,UA +520519680,520552447,PL +520552448,520554495,NL +520554496,520556543,GB +520556544,520560639,FI +520562688,520564735,PL +520564736,520566783,IE +520566784,520568831,CH +520568832,520589311,IR +520589312,520593407,IT +520593408,520595455,AM +520595456,520597503,MK +520597504,520601599,DE +520601600,520609791,SI +520609792,520613887,RU +520613888,520615935,CZ +520615936,520617983,FR +520617984,520683519,RU +520683520,520749055,SY +520749056,520753151,RU +520753152,520757247,LI +520757248,520761343,GB +520761344,520763391,IT +520763392,520765439,CZ +520765440,520781823,RU +520781824,520822783,CH +520822784,520824831,IT +520824832,520826879,RO +520826880,520828927,QA +520828928,520830975,NL +520830976,520847359,MK +520847360,520880127,PL +520880128,520882175,AT +520882176,520884223,IT +520884224,520888319,MK +520888320,520896511,DE +520896512,520898559,BE +520898560,520912895,GB +520912896,520945663,UA +520945664,520947711,GB +520947712,520949759,SE +520949760,520951807,RU +520951808,520953855,IE +520953856,520962047,RU +520962048,520978431,IE +520978432,520980479,RU +520980480,520982527,IT +520982528,520984575,RU +520984576,520984831,NG +520984832,520986623,GB +520986624,520988671,PS +520988672,520990719,DE +520990720,520992767,RU +520992768,520994815,ES +520994816,521011199,BG +521011200,521076735,RO +521076736,521078783,ES +521078784,521080831,CZ +521080832,521082879,RU +521082880,521084927,FR +521084928,521093119,SE +521093120,521095167,DE +521095168,521097215,IT +521097216,521101311,RU +521101312,521103359,IT +521103360,521105407,GB +521105408,521107455,IS +521109504,521142271,SI 521142272,521404415,DE 521404416,521535487,NL 521535488,521537535,GB @@ -1207,7 +1360,6 @@ 522059776,522125311,CZ 522125312,522133503,MD 522133504,522135551,NL -522135552,522137599,SE 522137600,522141695,CH 522141696,522143743,RU 522143744,522145791,CZ @@ -1223,11 +1375,18 @@ 522180608,522182655,KZ 522182656,522190847,CZ 522190848,522715135,FR +522715136,522717183,IR +522717184,522719231,RU +522719232,522721279,UA +522721280,522739711,RU +522747904,522780671,UA 522780672,522782719,RU 522782720,522784767,UA 522784768,522786815,BG 522786816,522788863,PL -522788864,522797055,RU +522788864,522792959,RU +522792960,522795007,UA +522795008,522797055,RU 522797056,522801151,UA 522801152,522803199,PL 522803200,522805247,UA @@ -1240,6 +1399,12 @@ 522821632,522823679,RU 522823680,522827775,PL 522827776,522831871,RU +522831872,522833919,CZ +522833920,522835967,PL +522835968,522838015,UA +522838016,522840063,RU +522840064,522842111,PL +522842112,522846207,RU 522846208,522854399,PL 522854400,522858495,RU 522858496,522866687,UA @@ -1300,7 +1465,24 @@ 528580608,528588799,RU 528588800,528596991,UA 528596992,528605183,RU -528875520,528879615,RU +528605184,528613375,UA +528613376,528637951,RU +528637952,528642047,PL +528642048,528646143,RU +528809984,528812031,PL +528812032,528814079,CZ +528814080,528816127,PL +528816128,528818175,RO +528818176,528834559,UA +528834560,528836607,RO +528836608,528838655,RU +528838656,528840703,UA +528840704,528842751,RU +528875520,528887807,RU +528887808,528891903,PL +528891904,528900095,UA +528900096,528902143,SK +528902144,528908287,UA 529006592,529268735,NL 529268736,529530879,TR 529530880,529596415,UA @@ -1349,8 +1531,7 @@ 531177472,531183615,GB 531183616,531185663,NL 531185664,531193855,TR -531193856,531194111,IE -531194112,531195903,GB +531193856,531195903,IE 531195904,531197951,IT 531197952,531199999,GB 531200000,531202047,RU @@ -1373,9 +1554,36 @@ 531281920,531283967,RU 531283968,531292159,DE 531292160,531333119,AZ -531333120,531335167,NL +531333120,531333599,NL +531333600,531333631,RU +531333632,531334399,NL +531334400,531334655,SE +531334656,531335167,US 531335168,531337215,IT +531337216,531339263,CH +531339264,531341311,AL 531341312,531349503,KZ +531349504,531351551,RU +531351552,531355647,NL +531357696,531361791,HU +531361792,531365887,CH +531365888,531366399,DE +531366400,531367935,CH +531367936,531369983,DE +531369984,531371007,CH +531371008,531371519,US +531371520,531372031,JP +531372032,531398655,CH +531398656,531400703,RU +531400704,531402751,UA +531402752,531404799,LU +531404800,531406847,FI +531406848,531415039,IM +531415040,531423231,RU +531423232,531425279,NO +531425280,531431423,GB +531431424,531496959,RO +531496960,531628031,PL 531628032,531660799,TR 531660800,531693567,BA 531693568,531695615,RU @@ -1397,7 +1605,9 @@ 532152320,532168703,DE 532168704,532185087,NL 532185088,532201471,IR -532201472,532221951,RU +532201472,532211711,RU +532211712,532212223,LU +532212224,532221951,RU 532221952,532223999,IT 532224000,532226047,NO 532226048,532234239,GB @@ -1430,6 +1640,62 @@ 532377600,532381695,DE 532381696,532414463,NL 532414464,532676607,IT +532676608,532692991,GE +532692992,532701183,CZ +532703232,532705279,RU +532705280,532709375,NL +532709376,532725759,RU +532725760,532729855,SE +532729856,532731903,TR +532731904,532733951,PL +532733952,532735999,SE +532736000,532738047,RU +532738048,532740095,GB +532740096,532742143,KG +532742144,532746239,GB +532746240,532750335,IT +532750336,532752383,SM +532752384,532754431,BE +532754432,532756479,FR +532756480,532758527,IT +532758528,532762623,SY +532762624,532766719,UA +532766720,532768767,PL +532768768,532770815,NL +532770816,532772863,IR +532772864,532774911,RU +532774912,532779007,GB +532779008,532783103,IR +532783104,532785151,AT +532785152,532787199,GB +532787200,532789247,BE +532789248,532791295,DK +532791296,532793343,LT +532793344,532795391,SE +532795392,532797439,CH +532797440,532799487,IE +532799488,532801535,ES +532801536,532803583,DK +532803584,532805631,FR +532805632,532807679,SE +532807680,533200895,IT +533200896,533233663,TR +533233664,533250047,IE +533250048,533262335,RU +533262336,533264383,ES +533264384,533266431,RU +533266432,533331967,FR +533331968,533397503,UA +533397504,533463039,KW +533463040,533479423,RU +533481472,533483519,NO +533483520,533485567,FR +533487616,533491711,RU +533491712,533495807,DE +533495808,533503999,NL +533512192,533528575,ME +533528576,533594111,GB +533594112,533659647,TR 536870912,539624567,US 539624568,539624575,IE 539624576,539624703,US @@ -1571,14 +1837,15 @@ 543691008,543844351,US 543844352,543844607,CH 543844608,603979775,US +603979776,603980799,CN 603980800,603981823,NP -603981824,603983871,CN -603996160,604110847,CN +603981824,604110847,CN 604110848,604241919,JP 604241920,604504063,CN 604504064,605028351,JP -605028352,606404607,CN +605028352,606412799,CN 606412800,606413823,HK +606413824,606420991,CN 606420992,606437375,ID 606437376,606470143,KH 606470144,606601215,KR @@ -1602,11 +1869,19 @@ 644840448,645225471,US 645225472,645225503,CA 645225504,654311423,US +654311424,654311679,CN 654311680,654311935,AU +654311936,654376959,CN 654376960,654442495,TW +654442496,654573567,JP +654573568,654835711,KR +654835712,655359999,TW 655360000,656408575,KR 656408576,658505727,PK -658505728,660602879,CN +658505728,661651455,CN +661651456,662700031,KR +662700032,666894335,CN +666894336,671088639,ID 671088640,687865855,US 687865856,689963007,ZA 689963008,691011583,EG @@ -1709,7 +1984,8 @@ 692918272,692920319,ZA 692920320,692922367,MZ 692922368,692924415,NG -692924416,692926463,ZA +692924416,692928511,ZA +692928512,692930559,GH 692969472,692971519,TZ 692973568,692975615,MZ 692975616,692977663,EG @@ -1773,7 +2049,7 @@ 693044224,693045247,NG 693045248,693046271,KE 693046272,693047295,ZW -693047296,693048319,ZA +693047296,693049343,ZA 693101568,693102591,KE 693102592,693103615,CD 693103616,693104639,GN @@ -2183,10 +2459,13 @@ 703594496,704118783,ZA 704118784,704380927,MA 704380928,704643071,LY +704643072,704644095,CN 704644096,704645119,BD +704645120,704650239,CN 704650240,704651263,MY +704651264,704659455,CN 704659456,704675839,TW -704675840,704708607,CN +704675840,704723967,CN 704723968,704724991,MY 704724992,704741375,VN 704741376,704774143,CN @@ -2195,42 +2474,48 @@ 705167360,707788799,KR 707788800,708575231,CN 708575232,708706303,SG -708706304,708739071,CN +708706304,708751359,CN 708751360,708752383,ID +708752384,708755455,CN 708755456,708771839,AU 708771840,708837375,CN 708837376,709885951,TW 709885952,710017023,CN 710017024,710082559,KR 710082560,710098943,JP +710098944,710104063,CN 710104064,710105087,PK -710115328,710934527,CN +710105088,710934527,CN 710934528,710950911,VN +710950912,710961151,CN 710961152,710962175,TH -710967296,711065599,CN +710962176,711065599,CN 711065600,711131135,HK +711131136,711160831,CN 711160832,711161855,BD +711161856,711163903,CN 711163904,711196671,JP 711196672,711458815,CN 711458816,711983103,IN 711983104,712507391,VN -712507392,712703999,CN +712507392,712712191,CN 712712192,712713215,JP -712736768,712769535,CN +712713216,712769535,CN 712769536,713031679,JP 713031680,714080255,CN 714080256,714604543,JP 714604544,714866687,MY +714866688,714874879,CN 714874880,714875903,MY -714899456,716898303,CN +714875904,716930047,CN 716930048,716931071,JP 716931072,716963839,CN 716963840,717225983,MY 717225984,717750271,CN 717750272,717815807,HK +717815808,717848575,CN 717848576,717881343,PK -717881344,720371711,CN -720404480,720437247,CN +717881344,720437247,CN 720437248,720502783,AU 720502784,721420287,CN 721420288,738197503,JP @@ -2395,7 +2680,9 @@ 773048320,773050367,LV 773050368,773052415,IE 773052416,773054463,NL -773054464,773056511,AL +773054464,773055487,AL +773055488,773055743,RS +773055744,773056511,AL 773056512,773058559,IT 773058560,773060607,BE 773060608,773062655,DK @@ -2405,8 +2692,8 @@ 773063425,773063436,US 773063437,773064447,TR 773064448,773064703,US -773064704,773064959,TR -773064960,773065215,US +773064704,773065088,TR +773065089,773065215,US 773065216,773066751,TR 773066752,773070847,AT 773070848,773074943,DE @@ -2436,20 +2723,21 @@ 773152768,773153791,SE 773153792,773154815,US 773154816,773156863,SE -773156864,773165055,FR +773156864,773160447,FR +773160448,773160703,ES +773160704,773165055,FR 773165056,773168127,NL -773168128,773168383,US -773168384,773168399,NL -773168400,773168415,US -773168416,773168511,NL -773168512,773168639,US -773168640,773169023,NL -773169024,773169151,US +773168128,773168639,US +773168640,773168895,NL +773168896,773169151,US 773169152,773169375,NL 773169376,773169407,BE 773169408,773171343,NL 773171344,773171359,BE -773171360,773173247,NL +773171360,773172223,NL +773172224,773172287,BE +773172288,773172351,GB +773172352,773173247,NL 773173248,773177343,IT 773177344,773181439,FR 773181440,773185535,PL @@ -2467,8 +2755,7 @@ 773234688,773238783,PL 773238784,773242879,NL 773242880,773246975,FR -773246976,773247231,RU -773247232,773247999,EE +773246976,773247999,EE 773248000,773251071,US 773251072,773255167,AZ 773255168,773259263,RU @@ -2514,7 +2801,6 @@ 773632000,773634047,CY 773634048,773636095,DE 773636096,773638143,UA -773638144,773640191,GB 773640192,773642239,ES 773642240,773644287,HU 773644288,773646335,RU @@ -2874,8 +3160,7 @@ 774184960,774193151,GB 774193152,774209535,ES 774209536,774217727,RU -774217728,774217759,PH -774217760,774217791,GB +774217728,774217791,GB 774217792,774217823,US 774217824,774217855,AE 774217856,774217919,US @@ -2916,7 +3201,8 @@ 774218744,774219007,GB 774219008,774219263,US 774219264,774219271,CN -774219272,774219287,GB +774219272,774219279,TR +774219280,774219287,GB 774219288,774219295,CY 774219296,774219327,GB 774219328,774219335,TW @@ -2948,12 +3234,11 @@ 774219600,774219607,CN 774219608,774219679,GB 774219680,774219687,GR -774219688,774219695,GB -774219696,774219711,US -774219712,774219727,GB +774219688,774219711,GB +774219712,774219727,BE 774219728,774219743,RU 774219744,774219751,GB -774219752,774219759,CN +774219752,774219759,PK 774219760,774219775,RU 774219776,774219839,CN 774219840,774219847,UA @@ -2964,32 +3249,44 @@ 774219896,774219903,LK 774219904,774219967,US 774219968,774220031,CN -774220032,774220287,US +774220032,774220159,GB +774220160,774220191,US +774220192,774220207,GB +774220208,774220223,CN +774220224,774220287,GB 774220288,774220543,CN 774220544,774221055,GB 774221056,774221311,US 774221312,774221567,CN 774221568,774221823,US -774221824,774222463,GB +774221824,774222079,GB +774222080,774222335,US +774222336,774222463,GB 774222464,774222495,CN 774222496,774222511,FI 774222512,774222527,CN -774222528,774222847,GB +774222528,774222591,GB +774222592,774222655,CN +774222656,774222719,GB +774222720,774222783,CN +774222784,774222847,GB 774222848,774222863,UA 774222864,774222879,TR -774222880,774222911,US +774222880,774222895,US +774222896,774222911,GB 774222912,774222943,MA -774222944,774222951,GR +774222944,774222951,GB 774222952,774222967,CN 774222968,774222975,GB 774222976,774223039,CN 774223040,774223071,IT 774223072,774223103,CN 774223104,774223119,UA -774223120,774223135,TR -774223136,774223151,CN +774223120,774223151,CN 774223152,774223167,TR -774223168,774223239,CN +774223168,774223215,GB +774223216,774223231,BE +774223232,774223239,CN 774223240,774223247,ME 774223248,774223255,TR 774223256,774223263,ID @@ -3007,7 +3304,11 @@ 774223360,774223375,UA 774223376,774223391,RU 774223392,774223423,TR -774223424,774223951,GB +774223424,774223743,GB +774223744,774223807,US +774223808,774223839,CN +774223840,774223871,US +774223872,774223951,GB 774223952,774223959,RU 774223960,774223967,CN 774223968,774223975,RU @@ -3028,7 +3329,8 @@ 774224112,774224119,CN 774224120,774224127,IN 774224128,774224159,BE -774224160,774224223,US +774224160,774224191,GB +774224192,774224223,US 774224224,774224255,LT 774224256,774224383,GB 774224384,774224399,UA @@ -3056,27 +3358,35 @@ 774225344,774225351,CN 774225352,774225359,LT 774225360,774225375,UA -774225376,774225391,GB +774225376,774225391,JO 774225392,774225415,UA 774225416,774225423,CN 774225424,774225431,TR 774225432,774225439,DE 774225440,774225447,CN -774225448,774225511,GB +774225448,774225455,BY +774225456,774225463,GB +774225464,774225471,CN +774225472,774225479,IT +774225480,774225487,IN +774225488,774225495,CN +774225496,774225503,UA +774225504,774225511,CN 774225512,774225519,TR -774225520,774225567,GB +774225520,774225527,RU +774225528,774225535,CA +774225536,774225567,GB 774225568,774225599,RU 774225600,774225615,PK 774225616,774225647,TR 774225648,774225663,UA 774225664,774225671,LK -774225672,774225679,IT +774225672,774225679,RU 774225680,774225687,GB 774225688,774225695,CA 774225696,774225727,GB 774225728,774225743,LT -774225744,774225775,GB -774225776,774225791,US +774225744,774225791,GB 774225792,774225807,UA 774225808,774225823,GB 774225824,774225855,US @@ -3242,7 +3552,272 @@ 778666160,778666175,PL 778666176,778666207,FR 778666208,778666239,PL -778666240,778698751,FR +778666240,778666367,FR +778666368,778666371,PL +778666372,778666375,GB +778666376,778666383,FR +778666384,778666391,DE +778666392,778666399,PL +778666400,778666479,FR +778666480,778666495,DE +778666496,778666751,FR +778666752,778666783,CZ +778666784,778666815,FR +778666816,778666847,GB +778666848,778666863,FR +778666864,778666871,GB +778666872,778666879,IE +778666880,778666967,FR +778666968,778666975,PL +778666976,778667279,FR +778667280,778667287,IE +778667288,778667291,NL +778667292,778667295,CH +778667296,778667327,BE +778667328,778667331,GB +778667332,778667335,FR +778667336,778667343,ES +778667344,778667347,FR +778667348,778667351,GB +778667352,778667391,PL +778667392,778667407,FR +778667408,778667415,PL +778667416,778667423,FR +778667424,778667455,LT +778667456,778667471,DE +778667472,778667479,FR +778667480,778667483,GB +778667484,778667487,NL +778667488,778667491,PL +778667492,778667499,FR +778667500,778667503,GB +778667504,778667839,FR +778667840,778667855,PL +778667856,778667871,FR +778667872,778667875,ES +778667876,778667879,LT +778667880,778667887,FI +778667888,778667895,IT +778667896,778667903,IE +778667904,778667911,GB +778667912,778667915,PL +778667916,778667919,PT +778667920,778667927,NL +778667928,778667931,GB +778667932,778667935,CZ +778667936,778667943,FR +778667944,778667947,DE +778667948,778667951,LT +778667952,778667967,GB +778667968,778667999,FR +778668000,778668019,DE +778668020,778668023,ES +778668024,778668027,PT +778668028,778668319,FR +778668320,778668351,GB +778668352,778668359,FR +778668360,778668367,PL +778668368,778668371,DE +778668372,778668375,ES +778668376,778668379,GB +778668380,778668415,FR +778668416,778668495,ES +778668496,778668499,PL +778668500,778668503,FR +778668504,778668507,ES +778668508,778668511,PT +778668512,778668515,GB +778668516,778668527,PL +778668528,778668543,NL +778668544,778668559,FR +778668560,778668567,IT +778668568,778668575,PL +778668576,778668607,FR +778668608,778668615,IE +778668616,778668619,PL +778668620,778668623,GB +778668624,778668639,FR +778668640,778668671,PL +778668672,778668703,NL +778668704,778668719,FR +778668720,778668723,PT +778668724,778668735,PL +778668736,778668799,IT +778668800,778669103,FR +778669104,778669107,PL +778669108,778669119,GB +778669120,778669151,FI +778669152,778669183,FR +778669184,778669191,DE +778669192,778669199,FR +778669200,778669207,PL +778669208,778669211,CH +778669212,778669215,ES +778669216,778669247,GB +778669248,778669295,FR +778669296,778669303,PL +778669304,778669439,FR +778669440,778669447,GB +778669448,778669451,PL +778669452,778669455,FI +778669456,778669471,FR +778669472,778669503,CZ +778669504,778669535,ES +778669536,778669539,PL +778669540,778669543,GB +778669544,778669547,FR +778669548,778669551,IT +778669552,778669567,PL +778669568,778669615,FR +778669616,778669623,GB +778669624,778669631,PL +778669632,778669695,FR +778669696,778669727,PT +778669728,778669759,NL +778669760,778669767,DE +778669768,778669771,GB +778669772,778669775,PL +778669776,778669791,FR +778669792,778669807,DE +778669808,778669823,FR +778669824,778669855,ES +778669856,778669887,FR +778669888,778669903,PL +778669904,778669919,FR +778669920,778669935,BE +778669936,778669943,DE +778669944,778669951,CH +778669952,778669959,NL +778669960,778669967,GB +778669968,778669983,IE +778669984,778669999,IT +778670000,778670007,PL +778670008,778670011,CZ +778670012,778670015,LT +778670016,778670143,FR +778670144,778670151,CZ +778670152,778670159,CH +778670160,778670163,FR +778670164,778670167,GB +778670168,778670175,FR +778670176,778670207,GB +778670208,778670211,DE +778670212,778670215,IT +778670216,778670223,FR +778670224,778670239,PL +778670240,778670243,PT +778670244,778670247,PL +778670248,778670255,CZ +778670256,778670291,FR +778670292,778670295,NL +778670296,778670303,DE +778670304,778670335,FR +778670336,778670343,DE +778670344,778670347,CZ +778670348,778670351,PL +778670352,778670355,DE +778670356,778670359,FR +778670360,778670383,DE +778670384,778670387,PL +778670388,778670395,GB +778670396,778670399,CZ +778670400,778670407,PL +778670408,778670411,GB +778670412,778670415,NL +778670416,778670431,FR +778670432,778670435,NL +778670436,778670439,ES +778670440,778670447,PL +778670448,778670455,CH +778670456,778670495,CZ +778670496,778670503,FR +778670504,778670511,ES +778670512,778670519,NL +778670520,778670523,FR +778670524,778670527,PT +778670528,778670559,FR +778670560,778670563,PL +778670564,778670567,FI +778670568,778670571,CH +778670572,778670575,FR +778670576,778670591,IT +778670592,778671103,ES +778671104,778671119,FR +778671120,778671127,DE +778671128,778671135,PT +778671136,778671167,BE +778671168,778671203,FR +778671204,778671211,PL +778671212,778671231,FR +778671232,778671239,GB +778671240,778671243,PL +778671244,778671247,FR +778671248,778671263,PL +778671264,778671279,GB +778671280,778671311,FR +778671312,778671327,PT +778671328,778671331,FR +778671332,778671335,DE +778671336,778671631,FR +778671632,778671647,PL +778671648,778671807,FR +778671808,778671839,ES +778671840,778671867,FR +778671868,778671871,GB +778671872,778671875,PL +778671876,778671879,ES +778671880,778671883,NL +778671884,778671887,ES +778671888,778671903,GB +778671904,778671907,NL +778671908,778671915,FR +778671916,778671919,GB +778671920,778671935,PT +778671936,778671967,FR +778671968,778672015,ES +778672016,778672055,FR +778672056,778672063,PL +778672064,778672067,GB +778672068,778672071,FR +778672072,778672079,GB +778672080,778672095,BE +778672096,778672103,CH +778672104,778672111,NL +778672112,778672115,GB +778672116,778672119,IT +778672120,778672123,ES +778672124,778672127,CH +778672128,778672255,FR +778672256,778672319,BE +778672320,778672383,FR +778672384,778672447,PL +778672448,778672479,DE +778672480,778672511,ES +778672512,778672543,FR +778672544,778672551,ES +778672552,778672559,CH +778672560,778672563,GB +778672564,778672567,IT +778672568,778672639,FR +778672640,778672703,DE +778672704,778672767,CH +778672768,778672803,FR +778672804,778672807,DE +778672808,778672811,PL +778672812,778672815,ES +778672816,778672819,LT +778672820,778672823,PL +778672824,778672831,DE +778672832,778672851,FR +778672852,778672879,PL +778672880,778672887,IT +778672888,778672891,PL +778672892,778672911,FR +778672912,778672919,PL +778672920,778672923,FR +778672924,778672927,ES +778672928,778672959,DE +778672960,778698751,FR 778698752,778764287,TR 778764288,778829823,HU 778829824,778895359,RO @@ -3392,7 +3967,133 @@ 780696832,780697087,LU 780697088,780697343,MK 780697344,780697599,MT -780697600,780730367,FR +780697600,780697855,NL +780697856,780698111,IT +780698112,780698367,ES +780698368,780698623,DE +780698624,780698879,FR +780698880,780699135,RU +780699136,780699391,RO +780699392,780699647,BG +780699648,780699903,DK +780699904,780700159,AT +780700160,780700415,FI +780700416,780700671,GR +780700672,780700927,PL +780700928,780701183,PT +780701184,780701439,SE +780701440,780701695,CH +780701696,780701951,SA +780701952,780702207,AD +780702208,780702463,AE +780702464,780702719,AF +780702720,780702975,AG +780702976,780703231,AI +780703232,780703487,AL +780703488,780703743,AM +780703744,780703999,AO +780704000,780704255,AQ +780704256,780704511,AR +780704512,780704767,AS +780704768,780705023,AT +780705024,780705279,AU +780705280,780705535,AW +780705536,780705791,AX +780705792,780706047,AZ +780706048,780706303,BA +780706304,780706559,BB +780706560,780706815,BD +780706816,780707071,BE +780707072,780707327,BF +780707328,780707583,BG +780707584,780707839,BH +780707840,780708095,BI +780708096,780708351,BJ +780708352,780708607,BM +780708608,780708863,BN +780708864,780709119,BO +780709120,780709375,BR +780709376,780709631,BS +780709632,780709887,BT +780709888,780710143,BV +780710144,780710399,BW +780710400,780710655,BY +780710656,780710911,BZ +780710912,780711167,CA +780711168,780711423,CC +780711424,780711679,CD +780711680,780711935,CF +780711936,780712191,CG +780712192,780712447,CH +780712448,780712703,CI +780712704,780712959,CK +780712960,780713215,CL +780713216,780713471,CM +780713472,780713727,CN +780713728,780713983,CO +780713984,780714239,CR +780714240,780714495,CU +780714496,780714751,CV +780714752,780715007,FR +780715008,780715263,CY +780715264,780715519,CZ +780715520,780715775,DE +780715776,780716031,DJ +780716032,780716287,DK +780716288,780716543,DM +780716544,780716799,DO +780716800,780717055,DZ +780717056,780717311,EC +780717312,780717567,EE +780717568,780717823,EG +780717824,780718079,EH +780718080,780718335,ER +780718336,780718591,ES +780718592,780718847,ET +780718848,780719103,FI +780719104,780719359,FJ +780719360,780719615,FK +780719616,780719871,FM +780719872,780720127,FO +780720128,780720383,FR +780720384,780720639,GA +780720640,780720895,GB +780720896,780721151,GD +780721152,780721407,GE +780721408,780721663,GF +780721664,780721919,GG +780721920,780722175,GH +780722176,780722431,GI +780722432,780722687,GL +780722688,780722943,GM +780722944,780723199,GN +780723200,780723455,GP +780723456,780723711,GQ +780723712,780723967,GR +780723968,780724223,GS +780724224,780724479,GT +780724480,780724735,GU +780724736,780724991,GW +780724992,780725247,GY +780725248,780725759,HK +780725760,780726015,HN +780726016,780726271,HR +780726272,780726527,HT +780726528,780726783,HU +780726784,780727039,ID +780727040,780727295,IE +780727296,780727551,IL +780727552,780727807,IM +780727808,780728063,IN +780728064,780728319,IO +780728320,780728575,IQ +780728576,780728831,IR +780728832,780729087,IS +780729088,780729343,IT +780729344,780729599,JE +780729600,780729855,JM +780729856,780730111,JO +780730112,780730367,JP 780730368,780795903,IE 780795904,780861439,RU 780861440,780926975,HU @@ -3468,9 +4169,7 @@ 782270464,782305791,RU 782305792,782306303,RO 782306304,782310399,RU -782310400,782310527,RO -782310528,782310655,RU -782310656,782310911,RO +782310400,782310911,RO 782310912,782319615,RU 782319616,782335999,ME 782336000,782352383,RU @@ -3479,7 +4178,8 @@ 782385152,782401535,SE 782401536,782417919,FR 782417920,782434303,AM -782434304,782450687,SI +782434304,782450175,SI +782450176,782450687,BA 782450688,782467071,DE 782467072,782483455,RU 782483456,782499839,FI @@ -3491,16 +4191,19 @@ 782598144,782630911,DE 782630912,782647295,MD 782647296,782663679,RU -782663680,782664191,GB -782664192,782664447,DK +782663680,782664447,GB 782664448,782664703,LU 782664704,782664704,GB 782664705,782665471,NL -782665472,782680063,LU +782665472,782667519,GB +782667520,782667775,LU +782667776,782680063,GB 782680064,782696447,RU 782696448,782712831,DE 782712832,782729215,RU -782729216,782745599,DE +782729216,782735359,DE +782735360,782735871,CH +782735872,782745599,DE 782745600,782761983,CZ 782761984,783024127,PL 783024128,783040511,RU @@ -3757,7 +4460,8 @@ 786788352,786792447,CZ 786792448,786796543,RU 786796544,786800639,PL -786800640,786804735,UA +786800640,786800895,US +786800896,786804735,UA 786804736,786808831,RU 786808832,786812927,BG 786812928,786817023,RU @@ -3948,7 +4652,8 @@ 788250624,788258815,KG 788258816,788259583,DE 788259584,788260863,NL -788260864,788267007,DE +788260864,788266495,DE +788266496,788267007,IN 788267008,788271103,SE 788271104,788275199,DE 788275200,788279295,AL @@ -4043,7 +4748,7 @@ 788527104,788529151,GB 788529152,805306367,CA 805306368,822083583,US -822084608,822085631,ID +822083584,822085631,ID 822085632,822087679,AU 822087680,822089727,JP 822089728,822090751,ID @@ -4069,14 +4774,18 @@ 825419776,825420799,TH 825420800,825421823,MY 825421824,825425919,NZ -825491456,825753599,CN +825425920,825753599,CN 825753600,826277887,KR 826277888,828375039,CN 828375040,829423615,JP 829423616,830210047,CN 830210048,830341119,MY 830341120,830406655,NP -830472192,830474239,AU +830406656,830472191,AU +830472192,830472447,CN +830472448,830472703,AU +830472704,830473215,CN +830473216,830474239,AU 830474240,830475263,SG 830475264,830476287,AU 830476288,830480383,JP @@ -4200,7 +4909,9 @@ 843055104,843644927,CA 843644928,844890111,US 844890112,844988415,CA -844988416,845217791,US +844988416,845283327,US +845283328,845414399,CA +845545472,846200831,US 847249408,855638015,US 855638016,872415231,GB 872415232,889192447,US @@ -4239,7 +4950,6 @@ 977797120,978321407,KR 978321408,978452479,JP 978452480,978583551,CN -978583552,978599935,JP 978599936,978640895,AU 978640896,978644991,NZ 978644992,978714623,JP @@ -4333,7 +5043,6 @@ 999817216,999849983,BD 999849984,999866367,KR 999866368,999882751,HK -999882752,999948287,JP 999948288,1000013823,AU 1000013824,1000079359,CN 1000079360,1000341503,JP @@ -4717,11 +5426,7 @@ 1040468480,1040468607,EU 1040468608,1040468735,DE 1040468736,1040468767,NL -1040468768,1040468815,EU -1040468816,1040468831,NL -1040468832,1040468927,EU -1040468928,1040468991,NL -1040468992,1040469055,EU +1040468768,1040469055,EU 1040469056,1040469071,FR 1040469072,1040469119,EU 1040469120,1040469183,FR @@ -4860,7 +5565,9 @@ 1041701648,1041701663,GB 1041701664,1041701719,FR 1041701720,1041701727,GB -1041701728,1041701967,FR +1041701728,1041701863,FR +1041701864,1041701871,GB +1041701872,1041701967,FR 1041701968,1041701975,GB 1041701976,1041702167,FR 1041702168,1041702175,GB @@ -4872,7 +5579,9 @@ 1041702352,1041702399,GB 1041702400,1041703575,FR 1041703576,1041703583,GB -1041703584,1041704119,FR +1041703584,1041703783,FR +1041703784,1041703791,GB +1041703792,1041704119,FR 1041704120,1041704127,GB 1041704128,1041704159,FR 1041704160,1041704175,GB @@ -4978,11 +5687,7 @@ 1041708000,1041708007,GB 1041708008,1041708023,FR 1041708024,1041708031,GB -1041708032,1041708327,FR -1041708328,1041708335,GB -1041708336,1041708375,FR -1041708376,1041708383,GB -1041708384,1041708415,FR +1041708032,1041708415,FR 1041708416,1041708431,GB 1041708432,1041708447,FR 1041708448,1041708487,GB @@ -5094,9 +5799,7 @@ 1041714912,1041714919,GB 1041714920,1041714999,FR 1041715000,1041715015,GB -1041715016,1041715063,FR -1041715064,1041715071,GB -1041715072,1041715079,FR +1041715016,1041715079,FR 1041715080,1041715087,GB 1041715088,1041715095,FR 1041715096,1041715103,GB @@ -5165,8 +5868,8 @@ 1041718136,1041718151,FR 1041718152,1041718159,GB 1041718160,1041718191,FR -1041718192,1041718215,GB -1041718216,1041718231,FR +1041718192,1041718223,GB +1041718224,1041718231,FR 1041718232,1041718255,GB 1041718256,1041718343,FR 1041718344,1041718351,GB @@ -5214,9 +5917,7 @@ 1041719920,1041719935,GB 1041719936,1041720015,FR 1041720016,1041720031,GB -1041720032,1041720343,FR -1041720344,1041720351,GB -1041720352,1041720551,FR +1041720032,1041720551,FR 1041720552,1041720559,GB 1041720560,1041720567,FR 1041720568,1041720607,GB @@ -5247,8 +5948,8 @@ 1041722384,1041722391,FR 1041722392,1041722399,GB 1041722400,1041722423,FR -1041722424,1041722431,GB -1041722432,1041722479,FR +1041722424,1041722463,GB +1041722464,1041722479,FR 1041722480,1041722503,GB 1041722504,1041722535,FR 1041722536,1041722551,GB @@ -5429,8 +6130,8 @@ 1041736256,1041736375,FR 1041736376,1041736383,GB 1041736384,1041736423,FR -1041736424,1041736463,GB -1041736464,1041736527,FR +1041736424,1041736455,GB +1041736456,1041736527,FR 1041736528,1041736535,GB 1041736536,1041736543,FR 1041736544,1041736551,GB @@ -5677,8 +6378,8 @@ 1041746064,1041746079,GB 1041746080,1041746111,FR 1041746112,1041746135,GB -1041746136,1041746703,FR -1041746704,1041746735,GB +1041746136,1041746711,FR +1041746712,1041746735,GB 1041746736,1041746743,FR 1041746744,1041746751,GB 1041746752,1041746799,FR @@ -5749,8 +6450,8 @@ 1041756464,1041756471,GB 1041756472,1041756543,FR 1041756544,1041756567,GB -1041756568,1041756599,FR -1041756600,1041756607,GB +1041756568,1041756591,FR +1041756592,1041756607,GB 1041756608,1041756655,FR 1041756656,1041756663,GB 1041756664,1041756695,FR @@ -5819,7 +6520,9 @@ 1041959680,1042022399,DE 1042022400,1042045891,PT 1042045892,1042045895,A2 -1042045896,1042087935,PT +1042045896,1042086739,PT +1042086740,1042086743,A2 +1042086744,1042087935,PT 1042087936,1042120703,TR 1042120704,1042153471,PL 1042153472,1042284543,GB @@ -5917,16 +6620,16 @@ 1042888960,1042889215,NL 1042889216,1042889471,DE 1042889472,1042889983,NL -1042889984,1042890111,GB -1042890112,1042890239,NL +1042889984,1042890119,GB +1042890120,1042890239,NL 1042890240,1042890495,GB 1042890496,1042890751,FR 1042890752,1042890815,GB 1042890816,1042890819,FR 1042890820,1042890944,NL 1042890945,1042890950,GB -1042890951,1042890975,NL -1042890976,1042890991,GB +1042890951,1042890959,NL +1042890960,1042890991,GB 1042890992,1042891775,NL 1042891776,1042891839,GB 1042891840,1042891871,BE @@ -6016,28 +6719,26 @@ 1043466320,1043466327,GB 1043466328,1043466335,NL 1043466336,1043466351,GB -1043466352,1043466391,NL -1043466392,1043466399,GB -1043466400,1043466431,NL +1043466352,1043466431,NL 1043466432,1043466447,GB 1043466448,1043466455,NL 1043466456,1043466463,GB 1043466464,1043466495,NL 1043466496,1043466503,GB 1043466504,1043466511,NL -1043466512,1043466527,GB -1043466528,1043466559,NL +1043466512,1043466519,GB +1043466520,1043466559,NL 1043466560,1043466575,GB 1043466576,1043466583,NL 1043466584,1043466607,GB -1043466608,1043466623,NL -1043466624,1043466639,GB -1043466640,1043466671,NL -1043466672,1043466687,GB -1043466688,1043466887,NL +1043466608,1043466887,NL 1043466888,1043466895,GB 1043466896,1043466911,NL -1043466912,1043467039,GB +1043466912,1043466927,GB +1043466928,1043466943,NL +1043466944,1043466991,GB +1043466992,1043467007,NL +1043467008,1043467039,GB 1043467040,1043467071,NL 1043467072,1043467087,GB 1043467088,1043467103,NL @@ -6085,8 +6786,8 @@ 1043468968,1043469023,NL 1043469024,1043469055,GB 1043469056,1043469087,NL -1043469088,1043469143,GB -1043469144,1043469159,NL +1043469088,1043469151,GB +1043469152,1043469159,NL 1043469160,1043469183,GB 1043469184,1043469199,NL 1043469200,1043469207,GB @@ -6094,14 +6795,10 @@ 1043469224,1043469231,GB 1043469232,1043469239,NL 1043469240,1043469247,GB -1043469248,1043469335,NL -1043469336,1043469343,GB -1043469344,1043469359,NL +1043469248,1043469359,NL 1043469360,1043469375,GB 1043469376,1043469399,NL -1043469400,1043469415,GB -1043469416,1043469423,NL -1043469424,1043469439,GB +1043469400,1043469439,GB 1043469440,1043469559,NL 1043469560,1043469663,GB 1043469664,1043469679,NL @@ -6119,11 +6816,15 @@ 1043470272,1043470303,NL 1043470304,1043470335,GB 1043470336,1043470847,NL -1043470848,1043472415,GB -1043472416,1043472423,DE +1043470848,1043472383,GB +1043472384,1043472395,DE +1043472396,1043472399,GB +1043472400,1043472423,DE 1043472424,1043472431,GB -1043472432,1043472439,DE -1043472440,1043472487,GB +1043472432,1043472443,DE +1043472444,1043472467,GB +1043472468,1043472475,DE +1043472476,1043472487,GB 1043472488,1043472495,DE 1043472496,1043472503,GB 1043472504,1043472895,DE @@ -6768,7 +7469,9 @@ 1044931552,1044931567,GB 1044931568,1044931583,BE 1044931584,1044931623,GB -1044931624,1044931663,BE +1044931624,1044931631,BE +1044931632,1044931639,GB +1044931640,1044931663,BE 1044931664,1044931671,GB 1044931672,1044931759,BE 1044931760,1044931775,GB @@ -6896,13 +7599,13 @@ 1044935744,1044935751,GB 1044935752,1044935839,BE 1044935840,1044935847,GB -1044935848,1044935855,BE -1044935856,1044935863,GB +1044935848,1044935851,BE +1044935852,1044935863,GB 1044935864,1044935879,BE 1044935880,1044935887,GB 1044935888,1044935903,BE -1044935904,1044936063,GB -1044936064,1044936103,BE +1044935904,1044936095,GB +1044936096,1044936103,BE 1044936104,1044936111,GB 1044936112,1044936123,BE 1044936124,1044936151,GB @@ -6930,8 +7633,8 @@ 1044937392,1044937399,GB 1044937400,1044937487,BE 1044937488,1044937503,GB -1044937504,1044937527,BE -1044937528,1044937531,GB +1044937504,1044937515,BE +1044937516,1044937531,GB 1044937532,1044937535,BE 1044937536,1044937551,GB 1044937552,1044937567,BE @@ -7012,7 +7715,7 @@ 1045154624,1045154655,GB 1045154656,1045154687,DE 1045154688,1045154719,US -1045154720,1045154751,DE +1045154720,1045154751,RU 1045154752,1045154783,SE 1045154784,1045155071,DE 1045155072,1045155327,CH @@ -7084,8 +7787,9 @@ 1045716992,1045725183,RU 1045725184,1045733375,CZ 1045733376,1045741567,GB -1045741568,1045741835,SE -1045741836,1045741839,GB +1045741568,1045741823,SE +1045741824,1045741831,GB +1045741832,1045741839,BE 1045741840,1045741887,SE 1045741888,1045742039,GB 1045742040,1045742047,SE @@ -7267,9 +7971,7 @@ 1046337536,1046338047,DE 1046338048,1046339839,EU 1046339840,1046340095,FR -1046340096,1046342143,EU -1046342144,1046342655,NL -1046342656,1046343423,EU +1046340096,1046343423,EU 1046343424,1046343935,NL 1046343936,1046344959,EU 1046344960,1046345215,DE @@ -7815,7 +8517,9 @@ 1047563408,1047563411,CH 1047563412,1047563415,DE 1047563416,1047563419,BE -1047563420,1047563443,DE +1047563420,1047563431,DE +1047563432,1047563435,CH +1047563436,1047563443,DE 1047563444,1047563447,CH 1047563448,1047563451,DE 1047563452,1047563455,NL @@ -7829,7 +8533,9 @@ 1047566364,1047566367,CH 1047566368,1047566403,DE 1047566404,1047566415,CH -1047566416,1047566443,DE +1047566416,1047566435,DE +1047566436,1047566439,CH +1047566440,1047566443,DE 1047566444,1047566447,AT 1047566448,1047566451,DE 1047566452,1047566459,CH @@ -8893,9 +9599,10 @@ 1049709312,1049709567,GB 1049709568,1049709823,A2 1049709824,1049710079,US -1049710080,1049710591,NL -1049710592,1049712639,GB -1049712640,1049713055,IR +1049710080,1049710335,GB +1049710336,1049710591,NL +1049710592,1049712895,GB +1049712896,1049713055,IR 1049713056,1049713087,MA 1049713088,1049713151,IR 1049713152,1049713663,NL @@ -9053,9 +9760,7 @@ 1050619504,1050619511,CH 1050619512,1050621407,DE 1050621408,1050621439,BE -1050621440,1050625959,DE -1050625960,1050625967,AT -1050625968,1050626559,DE +1050621440,1050626559,DE 1050626560,1050626815,GB 1050626816,1050647431,DE 1050647432,1050647439,NL @@ -9236,9 +9941,7 @@ 1051918336,1051919359,PL 1051919360,1051920383,AT 1051920384,1051920895,PL -1051920896,1051934719,AT -1051934720,1051938815,DE -1051938816,1051949823,AT +1051920896,1051949823,AT 1051949824,1051949951,NL 1051949952,1051983871,AT 1051983872,1051986687,EU @@ -9339,9 +10042,7 @@ 1052005888,1052005911,DE 1052005912,1052006543,EU 1052006544,1052006559,DE -1052006560,1052006575,EU -1052006576,1052006607,DE -1052006608,1052007039,EU +1052006560,1052007039,EU 1052007040,1052007103,DE 1052007104,1052007431,EU 1052007432,1052007435,DE @@ -9531,7 +10232,9 @@ 1052090368,1052098559,PL 1052098560,1052099471,SE 1052099472,1052099479,NO -1052099480,1052116991,SE +1052099480,1052104095,SE +1052104096,1052104127,DK +1052104128,1052116991,SE 1052116992,1052119039,RU 1052119040,1052121087,RO 1052121088,1052129279,RU @@ -10422,9 +11125,7 @@ 1053297028,1053297039,EU 1053297040,1053297055,IT 1053297056,1053297071,EU -1053297072,1053297087,IT -1053297088,1053297111,EU -1053297112,1053297135,IT +1053297072,1053297135,IT 1053297136,1053297143,EU 1053297144,1053297151,IT 1053297152,1053298175,EU @@ -10440,9 +11141,7 @@ 1053301064,1053301071,EU 1053301072,1053301135,FR 1053301136,1053301167,EU -1053301168,1053301183,FR -1053301184,1053301199,EU -1053301200,1053301295,FR +1053301168,1053301295,FR 1053301296,1053301303,EU 1053301304,1053301359,FR 1053301360,1053301367,EU @@ -10860,7 +11559,11 @@ 1054130176,1054138367,LT 1054138368,1054146559,AZ 1054146560,1054179327,RU -1054179328,1054187519,GB +1054179328,1054180095,GB +1054180096,1054180351,DE +1054180352,1054181375,GB +1054181376,1054186495,DE +1054186496,1054187519,GB 1054187520,1054195711,BG 1054195712,1054212095,BE 1054212096,1054212519,DE @@ -11150,7 +11853,11 @@ 1056244480,1056251903,CH 1056251904,1056260095,RU 1056260096,1056276479,CZ -1056276480,1056374783,DE +1056276480,1056286511,DE +1056286512,1056286519,A2 +1056286520,1056286591,DE +1056286592,1056286655,A2 +1056286656,1056374783,DE 1056374784,1056440319,SE 1056440320,1056473087,TR 1056473088,1056505087,FI @@ -11971,7 +12678,9 @@ 1074962432,1074970623,CA 1074970624,1075117287,US 1075117288,1075117311,IN -1075117312,1075421183,US +1075117312,1075265535,US +1075265536,1075269631,KR +1075269632,1075421183,US 1075421184,1075429375,CA 1075429376,1075478527,US 1075478528,1075479103,CA @@ -12120,7 +12829,9 @@ 1075974144,1075975167,US 1075975168,1075976031,CA 1075976032,1075976063,US -1075976064,1075976511,CA +1075976064,1075976127,CA +1075976128,1075976191,US +1075976192,1075976511,CA 1075976512,1075976543,US 1075976544,1075976647,CA 1075976648,1075976655,US @@ -14008,7 +14719,9 @@ 1077960728,1077960735,SE 1077960736,1077960751,US 1077960752,1077960759,CA -1077960760,1077965855,US +1077960760,1077960775,US +1077960776,1077960783,FR +1077960784,1077965855,US 1077965856,1077965887,BA 1077965888,1077965911,US 1077965912,1077965919,CA @@ -14571,7 +15284,9 @@ 1079396096,1079396351,CA 1079396352,1079397375,MP 1079397376,1079397631,MH -1079397632,1079399583,CA +1079397632,1079397887,CA +1079397888,1079398399,US +1079398400,1079399583,CA 1079399584,1079399599,US 1079399600,1079400447,CA 1079400448,1079400511,FR @@ -15248,7 +15963,9 @@ 1083263744,1083263999,GB 1083264000,1083264447,US 1083264448,1083264463,GB -1083264464,1083396095,US +1083264464,1083265023,US +1083265024,1083265279,CA +1083265280,1083396095,US 1083396096,1083400191,BM 1083400192,1083417727,US 1083417728,1083417791,CA @@ -15421,7 +16138,9 @@ 1086029728,1086029743,CA 1086029744,1086309887,US 1086309888,1086310143,AU -1086310144,1086358143,US +1086310144,1086317823,US +1086317824,1086318079,CA +1086318080,1086358143,US 1086358144,1086358271,PA 1086358272,1086359231,US 1086359232,1086359295,IL @@ -15890,11 +16609,7 @@ 1091800320,1091800327,JP 1091800328,1091802111,US 1091802112,1091802367,CA -1091802368,1091802831,US -1091802832,1091802847,CN -1091802848,1091802863,US -1091802864,1091802871,HK -1091802872,1091803135,US +1091802368,1091803135,US 1091803136,1091803391,CN 1091803392,1091803711,US 1091803712,1091803775,TH @@ -16245,9 +16960,7 @@ 1093730304,1093730559,US 1093730560,1093730815,HK 1093730816,1093731071,A1 -1093731072,1093732095,US -1093732096,1093732351,GB -1093732352,1093733887,US +1093731072,1093733887,US 1093733888,1093734143,A1 1093734144,1093737247,US 1093737248,1093737263,AE @@ -16257,8 +16970,8 @@ 1093740096,1093740159,JP 1093740160,1093740167,US 1093740168,1093740191,CN -1093740192,1093740223,US -1093740224,1093740239,CN +1093740192,1093740231,US +1093740232,1093740239,CN 1093740240,1093740247,US 1093740248,1093740255,HK 1093740256,1093740271,US @@ -16815,7 +17528,9 @@ 1096968128,1096968159,CA 1096968160,1096968191,US 1096968192,1096968319,GB -1096968320,1097057623,US +1096968320,1096969471,US +1096969472,1096969479,IN +1096969480,1097057623,US 1097057624,1097057631,IT 1097057632,1097057655,US 1097057656,1097057663,NZ @@ -17079,7 +17794,9 @@ 1102016256,1102016287,AR 1102016288,1102016351,US 1102016352,1102016383,MY -1102016384,1102018431,US +1102016384,1102017087,US +1102017088,1102017119,DM +1102017120,1102018431,US 1102018432,1102018495,TZ 1102018496,1102019583,US 1102019584,1102019711,IN @@ -17333,8 +18050,8 @@ 1108055424,1108055439,US 1108055440,1108055455,CA 1108055456,1108055471,US -1108055472,1108055487,CA -1108055488,1108055551,US +1108055472,1108055519,CA +1108055520,1108055551,US 1108055552,1108055903,CA 1108055904,1108055919,US 1108055920,1108055967,CA @@ -17686,9 +18403,7 @@ 1112357376,1112357503,CA 1112357504,1112360959,US 1112360960,1112361023,CA -1112361024,1112369343,US -1112369344,1112369375,AU -1112369376,1112432639,US +1112361024,1112432639,US 1112432640,1112433147,CA 1112433148,1112433151,US 1112433152,1112440831,CA @@ -17767,12 +18482,15 @@ 1114507328,1114507391,US 1114507392,1114507423,CA 1114507424,1114507431,GB -1114507432,1114511359,CA +1114507432,1114508287,CA +1114508288,1114508799,US +1114508800,1114511359,CA 1114511360,1114511871,US 1114511872,1114513407,CA 1114513408,1114515455,SA 1114515456,1114517503,US -1114517504,1114520063,CA +1114517504,1114519551,CA +1114519552,1114520063,US 1114520064,1114520319,PH 1114520320,1114520575,US 1114520576,1114520831,PH @@ -17819,8 +18537,8 @@ 1114880096,1114880383,US 1114880384,1114880399,CY 1114880400,1114880407,GB -1114880408,1114881343,US -1114881344,1114881407,CY +1114880408,1114881279,US +1114881280,1114881407,CY 1114881408,1114881471,US 1114881472,1114881535,CY 1114881536,1114928863,US @@ -18133,7 +18851,9 @@ 1117142272,1117142527,CA 1117142528,1117163023,US 1117163024,1117163031,CA -1117163032,1117169503,US +1117163032,1117167855,US +1117167856,1117167871,GB +1117167872,1117169503,US 1117169504,1117169535,CA 1117169536,1117171071,US 1117171072,1117171103,CA @@ -18353,7 +19073,9 @@ 1118151464,1118151471,CR 1118151472,1118151631,US 1118151632,1118151647,MX -1118151648,1118151791,US +1118151648,1118151759,US +1118151760,1118151775,ES +1118151776,1118151791,US 1118151792,1118151795,BR 1118151796,1118152015,US 1118152016,1118152031,CO @@ -19125,11 +19847,9 @@ 1120374512,1120374783,US 1120374784,1120375243,CA 1120375244,1120375263,US -1120375264,1120375359,CA -1120375360,1120375551,US -1120375552,1120376063,CA -1120376064,1120376079,US -1120376080,1120376095,CA +1120375264,1120375423,CA +1120375424,1120375551,US +1120375552,1120376095,CA 1120376096,1120376127,US 1120376128,1120376223,CA 1120376224,1120376239,US @@ -19410,7 +20130,7 @@ 1121247544,1121247551,CA 1121247552,1121247559,VG 1121247560,1121247567,CA -1121247568,1121247583,PA +1121247568,1121247583,MT 1121247584,1121247591,CY 1121247592,1121247615,CA 1121247616,1121247631,PA @@ -19499,8 +20219,8 @@ 1121251048,1121251055,AW 1121251056,1121251071,CA 1121251072,1121251079,GB -1121251080,1121251087,CY -1121251088,1121251103,CA +1121251080,1121251095,CY +1121251096,1121251103,CA 1121251104,1121251119,BZ 1121251120,1121251135,CA 1121251136,1121251167,AG @@ -19513,7 +20233,8 @@ 1121251312,1121251327,PA 1121251328,1121251583,CA 1121251584,1121251591,BZ -1121251592,1121251607,CA +1121251592,1121251599,MT +1121251600,1121251607,CA 1121251608,1121251615,BZ 1121251616,1121251647,NL 1121251648,1121251663,CA @@ -19533,19 +20254,19 @@ 1121251896,1121251903,CA 1121251904,1121251935,AG 1121251936,1121251943,CY -1121251944,1121251951,PA +1121251944,1121251951,MT 1121251952,1121251967,GB -1121251968,1121251983,PA +1121251968,1121251983,MT 1121251984,1121251991,CA 1121251992,1121251999,AG 1121252000,1121252063,CA 1121252064,1121252359,AG 1121252360,1121252367,ZA -1121252368,1121252375,CA -1121252376,1121252383,PA +1121252368,1121252375,GG +1121252376,1121252383,MT 1121252384,1121252391,CR -1121252392,1121252399,CA -1121252400,1121252415,PA +1121252392,1121252399,GG +1121252400,1121252415,MT 1121252416,1121252479,GI 1121252480,1121252607,CR 1121252608,1121252671,AW @@ -19855,7 +20576,9 @@ 1121483760,1121483775,FR 1121483776,1121654863,US 1121654864,1121654879,IN -1121654880,1121655231,US +1121654880,1121654975,US +1121654976,1121654991,FR +1121654992,1121655231,US 1121655232,1121655263,AR 1121655264,1121655439,US 1121655440,1121655447,IN @@ -20196,7 +20919,13 @@ 1125120832,1125120863,US 1125120864,1125120895,GB 1125120896,1125121023,RO -1125121024,1125454111,US +1125121024,1125396483,US +1125396484,1125396491,LK +1125396492,1125396999,US +1125397000,1125397007,ES +1125397008,1125454007,US +1125454008,1125454015,GB +1125454016,1125454111,US 1125454112,1125454119,ES 1125454120,1125454143,US 1125454144,1125454151,GB @@ -20259,7 +20988,7 @@ 1125455516,1125455519,CA 1125455520,1125455523,ES 1125455524,1125455527,US -1125455528,1125455531,ES +1125455528,1125455531,GB 1125455532,1125455535,NO 1125455536,1125455543,US 1125455544,1125455547,ES @@ -20350,7 +21079,13 @@ 1125550336,1125552127,CA 1125552128,1125572607,US 1125572608,1125576703,CA -1125576704,1125613567,US +1125576704,1125595695,US +1125595696,1125595711,NG +1125595712,1125596343,US +1125596344,1125596351,NO +1125596352,1125596479,US +1125596480,1125596503,GB +1125596504,1125613567,US 1125613568,1125617663,CA 1125617664,1125623295,US 1125623296,1125623551,IN @@ -20450,9 +21185,7 @@ 1128641536,1128792063,CA 1128792064,1128818687,US 1128818688,1128818719,CA -1128818720,1130536959,US -1130536960,1130539007,GU -1130539008,1131440135,US +1128818720,1131440135,US 1131440136,1131440143,PR 1131440144,1131949295,US 1131949296,1131949303,PR @@ -20468,7 +21201,9 @@ 1132910112,1132910143,AU 1132910144,1132947431,US 1132947432,1132947439,CA -1132947440,1133785375,US +1132947440,1132954319,US +1132954320,1132954335,IS +1132954336,1133785375,US 1133785376,1133785383,NE 1133785384,1133785391,US 1133785392,1133785407,GB @@ -21225,7 +21960,9 @@ 1137278976,1137283071,CA 1137283072,1137287167,US 1137287168,1137295359,CA -1137295360,1137369087,US +1137295360,1137341063,US +1137341064,1137341064,GB +1137341065,1137369087,US 1137369088,1137369519,CA 1137369520,1137369535,US 1137369536,1137370111,CA @@ -21380,16 +22117,16 @@ 1137711280,1137711287,NL 1137711288,1137711295,US 1137711296,1137711303,NL -1137711304,1137711343,US -1137711344,1137711367,CA +1137711304,1137711335,US +1137711336,1137711367,CA 1137711368,1137711375,US 1137711376,1137711399,CA 1137711400,1137711407,US 1137711408,1137711439,CA 1137711440,1137711455,US 1137711456,1137711471,CA -1137711472,1137711479,US -1137711480,1137711503,CA +1137711472,1137711487,US +1137711488,1137711503,CA 1137711504,1137711511,US 1137711512,1137711559,CA 1137711560,1137711567,US @@ -21414,8 +22151,10 @@ 1137712120,1137712127,CA 1137712128,1137712135,US 1137712136,1137712151,CA -1137712152,1137712207,US -1137712208,1137712223,CA +1137712152,1137712183,US +1137712184,1137712191,CA +1137712192,1137712215,US +1137712216,1137712223,CA 1137712224,1137712239,US 1137712240,1137712255,CA 1137712256,1137712263,US @@ -21424,8 +22163,8 @@ 1137712288,1137712295,CA 1137712296,1137712327,US 1137712328,1137712343,CA -1137712344,1137712351,US -1137712352,1137712383,CA +1137712344,1137712359,US +1137712360,1137712383,CA 1137712384,1137712671,US 1137712672,1137712687,CA 1137712688,1137712711,US @@ -21433,8 +22172,8 @@ 1137712720,1137712759,US 1137712760,1137712767,CA 1137712768,1137712919,US -1137712920,1137712935,CA -1137712936,1137712959,US +1137712920,1137712927,CA +1137712928,1137712959,US 1137712960,1137712975,CA 1137712976,1137713015,US 1137713016,1137713023,CA @@ -21442,7 +22181,9 @@ 1137713072,1137713087,CA 1137713088,1137713103,US 1137713104,1137713111,CA -1137713112,1137724495,US +1137713112,1137713127,US +1137713128,1137713135,CA +1137713136,1137724495,US 1137724496,1137724511,CA 1137724512,1137724543,US 1137724544,1137724575,CA @@ -21529,7 +22270,9 @@ 1137946146,1137946153,DE 1137946154,1137946585,US 1137946586,1137946593,NO -1137946594,1137953023,US +1137946594,1137950975,US +1137950976,1137951231,CA +1137951232,1137953023,US 1137953024,1137954815,CA 1137954816,1137963007,US 1137963008,1137967103,VI @@ -21541,7 +22284,9 @@ 1138000096,1138001519,US 1138001520,1138001535,CA 1138001536,1138049023,US -1138049024,1138061311,CA +1138049024,1138053631,CA +1138053632,1138053887,US +1138053888,1138061311,CA 1138061312,1138163711,US 1138163712,1138163967,CA 1138163968,1138163975,MA @@ -21658,7 +22403,9 @@ 1138417728,1138421759,US 1138421760,1138421791,DE 1138421792,1138425855,US -1138425856,1138429951,KN +1138425856,1138427519,KN +1138427520,1138427647,US +1138427648,1138429951,KN 1138429952,1138450959,US 1138450960,1138450967,JM 1138450968,1138450991,US @@ -21746,7 +22493,9 @@ 1138512640,1138512671,ID 1138512672,1138512895,US 1138512896,1138512927,ID -1138512928,1138548735,US +1138512928,1138544895,US +1138544896,1138545151,GB +1138545152,1138548735,US 1138548736,1138556927,CA 1138556928,1138593791,US 1138593792,1138597887,CA @@ -21827,7 +22576,8 @@ 1138659498,1138659593,US 1138659594,1138659609,LK 1138659610,1138659642,GB -1138659643,1138659673,US +1138659643,1138659650,ID +1138659651,1138659673,US 1138659674,1138659681,MA 1138659682,1138659705,US 1138659706,1138659713,CA @@ -22042,7 +22792,9 @@ 1145259264,1145259327,IN 1145259328,1145260031,US 1145260032,1145260095,IN -1145260096,1145261055,US +1145260096,1145260623,US +1145260624,1145260631,IN +1145260632,1145261055,US 1145261056,1145261119,IN 1145261120,1145261311,US 1145261312,1145261375,IN @@ -22061,7 +22813,8 @@ 1145333248,1145333327,US 1145333328,1145333343,CN 1145333344,1145333351,US -1145333352,1145333367,CN +1145333352,1145333359,CN +1145333360,1145333367,US 1145333368,1145333375,PA 1145333376,1145333503,BD 1145333504,1145333863,US @@ -22875,8 +23628,8 @@ 1159514880,1159515135,US 1159515136,1159515647,CA 1159515648,1159515711,US -1159515712,1159515871,CA -1159515872,1159515895,US +1159515712,1159515887,CA +1159515888,1159515895,US 1159515896,1159515903,MX 1159515904,1159516159,CA 1159516160,1159516255,US @@ -22929,9 +23682,7 @@ 1159525376,1159526399,CA 1159526400,1159527935,US 1159527936,1159528191,CA -1159528192,1159530671,US -1159530672,1159530679,MO -1159530680,1159532103,US +1159528192,1159532103,US 1159532104,1159532111,MO 1159532112,1159560207,US 1159560208,1159560215,MO @@ -22939,8 +23690,8 @@ 1159643440,1159643455,TR 1159643456,1159643471,JP 1159643472,1159656487,US -1159656488,1159656511,BR -1159656512,1159657023,US +1159656488,1159656495,BR +1159656496,1159657023,US 1159657024,1159657039,AU 1159657040,1159657071,US 1159657072,1159657087,NZ @@ -22968,8 +23719,7 @@ 1159997584,1159997591,BB 1159997592,1159997623,US 1159997624,1159997631,SC -1159997632,1159997639,GB -1159997640,1159998575,US +1159997632,1159998575,US 1159998576,1159998583,UG 1159998584,1159998703,US 1159998704,1159998711,RS @@ -23031,9 +23781,7 @@ 1160017160,1160019967,CA 1160019968,1160364031,US 1160364032,1160368127,CA -1160368128,1160393727,US -1160393728,1160396799,CA -1160396800,1160397007,US +1160368128,1160397007,US 1160397008,1160397023,GB 1160397024,1160405631,US 1160405632,1160406015,DO @@ -23047,7 +23795,8 @@ 1160406480,1160406487,A2 1160406488,1160408095,US 1160408096,1160408111,GD -1160408112,1160408319,US +1160408112,1160408127,ES +1160408128,1160408319,US 1160408320,1160408575,CA 1160408576,1160409423,US 1160409424,1160409439,PT @@ -23115,7 +23864,20 @@ 1160487424,1160503295,US 1160503296,1160503871,A2 1160503872,1160503903,US -1160503904,1160507391,A2 +1160503904,1160504159,A2 +1160504160,1160504175,US +1160504176,1160504191,A2 +1160504192,1160504207,AU +1160504208,1160504287,A2 +1160504288,1160504303,US +1160504304,1160504319,A2 +1160504320,1160504383,US +1160504384,1160504511,A2 +1160504512,1160504543,NP +1160504544,1160504575,AF +1160504576,1160505343,A2 +1160505344,1160505855,AU +1160505856,1160507391,A2 1160507392,1160542207,US 1160542208,1160542239,LB 1160542240,1160543327,US @@ -23130,7 +23892,21 @@ 1160563200,1160563711,MP 1160563712,1160609791,US 1160609792,1160610815,MX -1160610816,1160667135,US +1160610816,1160660111,US +1160660112,1160660119,GB +1160660120,1160661639,US +1160661640,1160661647,GB +1160661648,1160662487,US +1160662488,1160662495,GB +1160662496,1160662743,US +1160662744,1160662751,GB +1160662752,1160665623,US +1160665624,1160665631,GB +1160665632,1160665807,US +1160665808,1160665815,GB +1160665816,1160667087,US +1160667088,1160667095,GB +1160667096,1160667135,US 1160667136,1160675327,CA 1160675328,1160677015,US 1160677016,1160677023,AR @@ -23266,9 +24042,13 @@ 1160921088,1160925183,AG 1160925184,1160938879,US 1160938880,1160938887,NL -1160938888,1160941463,US +1160938888,1160938895,US +1160938896,1160938911,CY +1160938912,1160941463,US 1160941464,1160941471,RU -1160941472,1160941535,US +1160941472,1160941503,US +1160941504,1160941519,CY +1160941520,1160941535,US 1160941536,1160941567,CA 1160941568,1160945663,US 1160953856,1160973439,US @@ -23355,26 +24135,25 @@ 1161617408,1161625599,CA 1161625600,1161627663,US 1161627664,1161627679,SE -1161627680,1161627695,US -1161627696,1161627703,AR +1161627680,1161627687,US +1161627688,1161627695,HR +1161627696,1161627703,US 1161627704,1161627711,DE 1161627712,1161627727,US 1161627728,1161627743,AR 1161627744,1161627751,RO 1161627752,1161627759,US 1161627760,1161627775,GB -1161627776,1161627791,US -1161627792,1161627799,AZ -1161627800,1161627807,US +1161627776,1161627807,US 1161627808,1161627815,HR 1161627816,1161627823,US 1161627824,1161627831,AU 1161627832,1161627839,US 1161627840,1161627863,AR -1161627864,1161627871,US -1161627872,1161627879,GB +1161627864,1161627871,CA +1161627872,1161627879,HR 1161627880,1161627895,US -1161627896,1161627903,AZ +1161627896,1161627903,CA 1161627904,1161628447,US 1161628448,1161628455,NL 1161628456,1161628463,GB @@ -23386,7 +24165,8 @@ 1161628648,1161628663,US 1161628664,1161628671,AR 1161628672,1161629199,US -1161629200,1161629215,HR +1161629200,1161629207,SI +1161629208,1161629215,HR 1161629216,1161629223,US 1161629224,1161629231,GB 1161629232,1161629239,US @@ -23396,7 +24176,10 @@ 1161629272,1161629343,US 1161629344,1161629375,FI 1161629376,1161629383,CA -1161629384,1161629415,US +1161629384,1161629391,AR +1161629392,1161629399,IE +1161629400,1161629407,IL +1161629408,1161629415,GB 1161629416,1161629423,HR 1161629424,1161629439,MY 1161629440,1161629519,US @@ -23408,9 +24191,15 @@ 1161629600,1161629615,HR 1161629616,1161629695,US 1161629696,1161629951,PL -1161629952,1161630335,US +1161629952,1161630263,US +1161630264,1161630271,AR +1161630272,1161630335,US 1161630336,1161630343,PL -1161630344,1161630399,US +1161630344,1161630367,US +1161630368,1161630375,GB +1161630376,1161630383,US +1161630384,1161630391,AR +1161630392,1161630399,US 1161630400,1161630431,GB 1161630432,1161630727,US 1161630728,1161630735,EG @@ -23492,12 +24281,15 @@ 1161633216,1161633231,EE 1161633232,1161633631,US 1161633632,1161633639,CA -1161633640,1161634063,US +1161633640,1161634055,US +1161634056,1161634063,IN 1161634064,1161634071,CA 1161634072,1161634127,US 1161634128,1161634135,AR 1161634136,1161634143,PH -1161634144,1161634199,US +1161634144,1161634175,US +1161634176,1161634191,PL +1161634192,1161634199,US 1161634200,1161634207,NZ 1161634208,1161634223,US 1161634224,1161634231,VN @@ -23509,7 +24301,9 @@ 1161634336,1161634352,PL 1161634353,1161634495,US 1161634496,1161634503,GB -1161634504,1161634943,US +1161634504,1161634519,US +1161634520,1161634527,BG +1161634528,1161634943,US 1161634944,1161634959,PL 1161634960,1161634975,US 1161634976,1161635007,PL @@ -23527,9 +24321,12 @@ 1161636464,1161636471,EG 1161636472,1161636495,US 1161636496,1161636503,CA -1161636504,1161637159,US +1161636504,1161637135,US +1161637136,1161637143,CA +1161637144,1161637159,US 1161637160,1161637167,NZ -1161637168,1161637199,US +1161637168,1161637175,GB +1161637176,1161637199,US 1161637200,1161637207,GB 1161637208,1161637215,US 1161637216,1161637223,PL @@ -23537,7 +24334,9 @@ 1161637296,1161637303,AR 1161637304,1161637343,US 1161637344,1161637351,RU -1161637352,1161637671,US +1161637352,1161637655,US +1161637656,1161637663,IL +1161637664,1161637671,US 1161637672,1161637679,IL 1161637680,1161637695,AR 1161637696,1161637775,US @@ -23548,7 +24347,9 @@ 1161638968,1161638975,GB 1161638976,1161638991,US 1161638992,1161638999,PL -1161639000,1161639039,US +1161639000,1161639007,US +1161639008,1161639023,BG +1161639024,1161639039,US 1161639040,1161639047,CA 1161639048,1161639063,US 1161639064,1161639071,PL @@ -23565,18 +24366,23 @@ 1161639520,1161639527,PL 1161639528,1161639575,US 1161639576,1161639583,BG -1161639584,1161639703,US +1161639584,1161639687,US +1161639688,1161639695,KW +1161639696,1161639703,US 1161639704,1161639711,BZ 1161639712,1161639719,PL 1161639720,1161639727,US 1161639728,1161639743,GB 1161639744,1161639831,US 1161639832,1161639839,AR -1161639840,1161639887,US +1161639840,1161639879,US +1161639880,1161639887,NL 1161639888,1161639895,AT 1161639896,1161639959,US 1161639960,1161639967,GB -1161639968,1161640015,US +1161639968,1161639991,US +1161639992,1161639999,HR +1161640000,1161640015,US 1161640016,1161640023,PL 1161640024,1161640031,AR 1161640032,1161640095,US @@ -23586,7 +24392,12 @@ 1161640784,1161640791,GB 1161640792,1161640847,US 1161640848,1161640863,PL -1161640864,1161641887,US +1161640864,1161640895,AR +1161640896,1161641375,US +1161641376,1161641383,KW +1161641384,1161641463,US +1161641464,1161641471,IN +1161641472,1161641887,US 1161641888,1161641911,KW 1161641912,1161641919,US 1161641920,1161641983,PL @@ -23621,19 +24432,26 @@ 1161649408,1161649663,AR 1161649664,1161650175,US 1161650176,1161650183,HR -1161650184,1161650191,IL +1161650184,1161650191,AR 1161650192,1161650199,US 1161650200,1161650207,BG -1161650208,1161650303,US +1161650208,1161650215,US +1161650216,1161650223,FI +1161650224,1161650239,CA +1161650240,1161650303,US 1161650304,1161650311,NL 1161650312,1161650327,US -1161650328,1161650335,AR -1161650336,1161650359,US -1161650360,1161650367,RO -1161650368,1161650687,US -1161650688,1161650695,AR -1161650696,1161650703,BR -1161650704,1161650823,US +1161650328,1161650343,AR +1161650344,1161650359,US +1161650360,1161650367,IL +1161650368,1161650375,ES +1161650376,1161650391,US +1161650392,1161650399,CA +1161650400,1161650415,US +1161650416,1161650423,PL +1161650424,1161650799,US +1161650800,1161650815,SI +1161650816,1161650823,US 1161650824,1161650831,AU 1161650832,1161650847,US 1161650848,1161650863,AR @@ -23641,12 +24459,18 @@ 1161650880,1161650895,GB 1161650896,1161650927,US 1161650928,1161650935,HR -1161650936,1161651055,US +1161650936,1161650943,US +1161650944,1161650951,SI +1161650952,1161650967,US +1161650968,1161650975,SI +1161650976,1161651055,US 1161651056,1161651071,PL -1161651072,1161651095,US +1161651072,1161651079,HR +1161651080,1161651095,US 1161651096,1161651103,HR 1161651104,1161651135,GB -1161651136,1161651183,US +1161651136,1161651143,BG +1161651144,1161651183,US 1161651184,1161651199,GB 1161651200,1161651487,US 1161651488,1161651503,GB @@ -23735,16 +24559,16 @@ 1161835342,1161835353,GR 1161835354,1161836031,US 1161836032,1161836063,CA -1161836064,1161836159,US +1161836064,1161836095,JP +1161836096,1161836159,US 1161836160,1161836191,BR -1161836192,1161836255,US -1161836256,1161836287,NL +1161836192,1161836287,US 1161836288,1161836319,RS 1161836320,1161836383,US 1161836384,1161836415,CY 1161836416,1161836447,BR 1161836448,1161836479,US -1161836480,1161836511,UA +1161836480,1161836511,GB 1161836512,1161837567,US 1161837568,1161837823,JP 1161837824,1161838548,US @@ -23906,7 +24730,10 @@ 1162926016,1162926071,US 1162926072,1162926079,AU 1162926080,1163395071,US -1163395072,1163395839,A2 +1163395072,1163395823,A2 +1163395824,1163395827,NP +1163395828,1163395831,AF +1163395832,1163395839,A2 1163395840,1163395847,BD 1163395848,1163395855,ID 1163395856,1163395863,BD @@ -23970,7 +24797,9 @@ 1163397576,1163397583,BD 1163397584,1163397663,A2 1163397664,1163397695,BD -1163397696,1163397855,A2 +1163397696,1163397791,A2 +1163397792,1163397807,AF +1163397808,1163397855,A2 1163397856,1163397887,US 1163397888,1163398143,NP 1163398144,1163398239,A2 @@ -23980,7 +24809,7 @@ 1163399040,1163399103,A2 1163399104,1163399295,BD 1163399296,1163399679,A2 -1163399680,1163399807,BD +1163399680,1163399807,US 1163399808,1163399935,A2 1163399936,1163400063,US 1163400064,1163400447,A2 @@ -24027,23 +24856,18 @@ 1163477696,1163477727,JP 1163477728,1163526143,US 1163526144,1163526463,CA -1163526464,1163526911,US -1163526912,1163527007,CA -1163527008,1163527023,US +1163526464,1163526655,US +1163526656,1163527023,CA 1163527024,1163527039,BV 1163527040,1163527059,CA 1163527060,1163527071,US 1163527072,1163527103,CA 1163527104,1163527135,US -1163527136,1163527143,CA -1163527144,1163527151,US -1163527152,1163527167,CA -1163527168,1163527743,US +1163527136,1163527679,CA +1163527680,1163527743,US 1163527744,1163527775,CA 1163527776,1163527791,US -1163527792,1163528191,CA -1163528192,1163528703,US -1163528704,1163529215,CA +1163527792,1163529215,CA 1163529216,1163530239,US 1163530240,1163530399,CA 1163530400,1163530431,US @@ -24067,12 +24891,14 @@ 1163533696,1163533727,US 1163533728,1163533791,CA 1163533792,1163533807,US -1163533808,1163533823,CA -1163533824,1163534015,US +1163533808,1163533951,CA +1163533952,1163534015,US 1163534016,1163534031,CA 1163534032,1163534047,US 1163534048,1163534063,CA -1163534064,1163534143,US +1163534064,1163534071,US +1163534072,1163534079,MX +1163534080,1163534143,US 1163534144,1163534175,CA 1163534176,1163534255,US 1163534256,1163534311,CA @@ -24097,9 +24923,7 @@ 1163542848,1163542855,US 1163542856,1163542919,CA 1163542920,1163542927,US -1163542928,1163542975,CA -1163542976,1163543007,US -1163543008,1163543687,CA +1163542928,1163543687,CA 1163543688,1163543695,FI 1163543696,1163543839,CA 1163543840,1163543847,US @@ -24107,20 +24931,17 @@ 1163543984,1163543991,US 1163543992,1163544319,CA 1163544320,1163544327,US -1163544328,1163544383,CA +1163544328,1163544335,GB +1163544336,1163544383,CA 1163544384,1163544423,US 1163544424,1163544575,CA 1163544576,1163544607,US -1163544608,1163544671,CA -1163544672,1163544687,US -1163544688,1163544751,CA +1163544608,1163544751,CA 1163544752,1163544759,SG 1163544760,1163544783,CA 1163544784,1163544799,US -1163544800,1163545215,CA -1163545216,1163545247,FI -1163545248,1163545279,CA -1163545280,1163545303,US +1163544800,1163545295,CA +1163545296,1163545303,US 1163545304,1163545311,CA 1163545312,1163545343,BV 1163545344,1163545351,US @@ -24139,45 +24960,36 @@ 1163546048,1163546119,CA 1163546120,1163546127,US 1163546128,1163546135,CA -1163546136,1163546175,US -1163546176,1163546199,CA -1163546200,1163546239,US -1163546240,1163546391,CA -1163546392,1163546423,US +1163546136,1163546143,US +1163546144,1163546199,CA +1163546200,1163546207,US +1163546208,1163546399,CA +1163546400,1163546423,US 1163546424,1163546447,CA 1163546448,1163546455,US 1163546456,1163546527,CA 1163546528,1163546535,US -1163546536,1163546543,CA -1163546544,1163546559,US +1163546536,1163546551,CA +1163546552,1163546559,US 1163546560,1163547455,CA 1163547456,1163547463,US 1163547464,1163547487,CA 1163547488,1163547519,US 1163547520,1163547535,CA 1163547536,1163547539,US -1163547540,1163547551,CA -1163547552,1163547567,US +1163547540,1163547567,CA 1163547568,1163547583,VG 1163547584,1163547647,CA 1163547648,1163547903,US 1163547904,1163547951,CA -1163547952,1163547999,US -1163548000,1163549007,CA -1163549008,1163549023,US -1163549024,1163549183,CA -1163549184,1163549695,US -1163549696,1163550271,CA -1163550272,1163550303,US -1163550304,1163550335,CA +1163547952,1163547983,US +1163547984,1163550335,CA 1163550336,1163550351,US 1163550352,1163550367,CA 1163550368,1163550375,US 1163550376,1163550383,CA 1163550384,1163550399,US -1163550400,1163550591,CA -1163550592,1163550655,US -1163550656,1163550783,CA +1163550400,1163550783,CA 1163550784,1163550815,US 1163550816,1163551071,CA 1163551072,1163551087,US @@ -24264,8 +25076,10 @@ 1163577376,1163577407,CA 1163577408,1163577423,US 1163577424,1163577439,CA -1163577440,1163577871,US -1163577872,1163577919,CA +1163577440,1163577471,US +1163577472,1163577503,CA +1163577504,1163577863,US +1163577864,1163577919,CA 1163577920,1163577951,US 1163577952,1163578111,CA 1163578112,1163578191,US @@ -24300,7 +25114,9 @@ 1163582168,1163582183,CA 1163582184,1163582191,US 1163582192,1163583487,CA -1163583488,1163585855,US +1163583488,1163585199,US +1163585200,1163585215,CA +1163585216,1163585855,US 1163585856,1163585887,CA 1163585888,1163585889,US 1163585890,1163585905,GB @@ -24320,9 +25136,9 @@ 1163588608,1163588608,US 1163588609,1163588655,CA 1163588656,1163588671,US -1163588672,1163588703,CA -1163588704,1163588711,US -1163588712,1163588727,CA +1163588672,1163588695,CA +1163588696,1163588703,US +1163588704,1163588727,CA 1163588728,1163588863,US 1163588864,1163589631,CA 1163589632,1163870575,US @@ -24422,7 +25238,9 @@ 1168952888,1168952895,CA 1168952896,1168952959,US 1168952960,1168953343,CA -1168953344,1168954075,US +1168953344,1168954015,US +1168954016,1168954047,IN +1168954048,1168954075,US 1168954076,1168954079,CA 1168954080,1168955647,US 1168955648,1168956159,CA @@ -24534,7 +25352,8 @@ 1170539330,1170539330,CN 1170539331,1170539331,TR 1170539332,1170539332,CA -1170539333,1170539460,US +1170539333,1170539333,RO +1170539334,1170539460,US 1170539461,1170539461,PK 1170539462,1170539522,US 1170539523,1170539523,AU @@ -24579,10 +25398,15 @@ 1170540232,1170540232,CN 1170540233,1170540288,US 1170540289,1170540290,CN -1170540291,1170540294,US +1170540291,1170540291,US +1170540292,1170540292,TR +1170540293,1170540294,US 1170540295,1170540295,CN 1170540296,1170540296,TR -1170540297,1170540360,US +1170540297,1170540353,US +1170540354,1170540354,CN +1170540355,1170540359,US +1170540360,1170540360,CN 1170540361,1170540361,CA 1170540362,1170540369,US 1170540370,1170540370,CN @@ -24590,16 +25414,27 @@ 1170540372,1170540372,CN 1170540373,1170540416,US 1170540417,1170540417,CN -1170540418,1170540480,US +1170540418,1170540428,US +1170540429,1170540429,TR +1170540430,1170540480,US 1170540481,1170540481,TR -1170540482,1170540492,US -1170540493,1170540493,TR -1170540494,1170544127,US +1170540482,1170540485,US +1170540486,1170540487,TR +1170540488,1170540491,US +1170540492,1170540494,TR +1170540495,1170540608,US +1170540609,1170540609,CN +1170540610,1170540616,US +1170540617,1170540617,TR +1170540618,1170544127,US 1170544128,1170544191,CN 1170544192,1170544199,US 1170544200,1170544207,TR -1170544208,1170544255,US -1170544256,1170544383,IN +1170544208,1170544239,US +1170544240,1170544247,CN +1170544248,1170544251,US +1170544252,1170544255,CN +1170544256,1170544383,PK 1170544384,1170544387,US 1170544388,1170544391,BD 1170544392,1170544395,TR @@ -24608,25 +25443,38 @@ 1170544440,1170544443,AU 1170544444,1170544511,US 1170544512,1170544543,TR -1170544544,1170544831,US +1170544544,1170544807,US +1170544808,1170544815,CN +1170544816,1170544831,US 1170544832,1170544959,CN -1170544960,1170545407,US +1170544960,1170544983,US +1170544984,1170544987,CN +1170544988,1170545055,US +1170545056,1170545087,CN +1170545088,1170545407,US 1170545408,1170545439,TR -1170545440,1170552959,US +1170545440,1170545595,US +1170545596,1170545599,CN +1170545600,1170552959,US 1170552960,1170553023,CN -1170553024,1170553279,US +1170553024,1170553087,US +1170553088,1170553151,GB +1170553152,1170553231,US +1170553232,1170553247,TR +1170553248,1170553279,US 1170553280,1170553407,CN -1170553408,1170553431,US +1170553408,1170553423,US +1170553424,1170553427,CN +1170553428,1170553431,US 1170553432,1170553435,TR -1170553436,1170553471,US +1170553436,1170553439,US +1170553440,1170553471,TR 1170553472,1170553535,PK 1170553536,1170553599,US 1170553600,1170553603,TR 1170553604,1170553615,US 1170553616,1170553619,SA -1170553620,1170553623,TR -1170553624,1170553627,US -1170553628,1170553631,TR +1170553620,1170553631,TR 1170553632,1170553635,US 1170553636,1170553643,SA 1170553644,1170553647,TR @@ -24634,7 +25482,9 @@ 1170553652,1170553655,TR 1170553656,1170553663,US 1170553664,1170553667,SA -1170553668,1170553707,US +1170553668,1170553683,US +1170553684,1170553687,CN +1170553688,1170553707,US 1170553708,1170553711,TR 1170553712,1170553855,US 1170553856,1170553983,MY @@ -24646,7 +25496,9 @@ 1170554480,1170554483,TR 1170554484,1170554487,BD 1170554488,1170554495,CN -1170554496,1170573375,US +1170554496,1170554683,US +1170554684,1170554687,TR +1170554688,1170573375,US 1170573376,1170573439,RU 1170573440,1175977983,US 1175977984,1176068167,CA @@ -24695,8 +25547,16 @@ 1176620968,1176620975,US 1176620976,1176620991,CA 1176620992,1176620999,US -1176621000,1176621567,CA -1176621568,1176622591,US +1176621000,1176621599,CA +1176621600,1176621631,US +1176621632,1176621679,CA +1176621680,1176621823,US +1176621824,1176621855,CA +1176621856,1176621887,US +1176621888,1176621951,CA +1176621952,1176622079,US +1176622080,1176622335,GB +1176622336,1176622591,US 1176622592,1176623359,CA 1176623360,1176623487,US 1176623488,1176623567,CA @@ -25024,7 +25884,9 @@ 1178000920,1178075135,CA 1178075136,1178599423,US 1178599424,1179910143,CA -1179910144,1189130447,US +1179910144,1180113919,US +1180113920,1180114431,AE +1180114432,1189130447,US 1189130448,1189130463,IN 1189130464,1190170111,US 1190170112,1190170127,UY @@ -25427,16 +26289,28 @@ 1208516624,1208516639,US 1208516640,1208516735,TR 1208516736,1208516799,ID -1208516800,1208517010,US +1208516800,1208516855,US +1208516856,1208516859,CN +1208516860,1208516955,US +1208516956,1208516959,CN +1208516960,1208516999,US +1208517000,1208517000,TR +1208517001,1208517001,US +1208517002,1208517002,CN +1208517003,1208517010,US 1208517011,1208517011,TR 1208517012,1208517012,GB -1208517013,1208517119,US +1208517013,1208517017,US +1208517018,1208517018,GB +1208517019,1208517119,US 1208517120,1208517375,KH -1208517376,1208517503,US +1208517376,1208517471,US +1208517472,1208517487,CN +1208517488,1208517503,US 1208517504,1208517567,CN 1208517568,1208517631,US 1208517632,1208517887,KH -1208517888,1208518015,US +1208517888,1208518015,TR 1208518016,1208518079,CN 1208518080,1208518143,US 1208518144,1208518207,CN @@ -25452,38 +26326,58 @@ 1208518300,1208518303,BD 1208518304,1208518335,US 1208518336,1208518399,TR -1208518400,1208519167,US +1208518400,1208518655,ID +1208518656,1208519167,US 1208519168,1208519423,CN 1208519424,1208519551,TR 1208519552,1208521983,US 1208521984,1208522239,CN 1208522240,1208522271,US 1208522272,1208522303,TR -1208522304,1208522367,US +1208522304,1208522319,US +1208522320,1208522335,TR +1208522336,1208522367,US 1208522368,1208522431,CN -1208522432,1208523991,US +1208522432,1208523975,US +1208523976,1208523983,CN +1208523984,1208523991,US 1208523992,1208523995,TR 1208523996,1208523999,US 1208524000,1208524031,TR 1208524032,1208524047,US 1208524048,1208524055,TR -1208524056,1208524063,US +1208524056,1208524059,CN +1208524060,1208524063,US 1208524064,1208524079,CN 1208524080,1208524083,US 1208524084,1208524087,KR -1208524088,1208524159,US +1208524088,1208524091,US +1208524092,1208524095,TR +1208524096,1208524159,US 1208524160,1208524223,ID -1208524224,1208524287,US +1208524224,1208524227,US +1208524228,1208524231,TR +1208524232,1208524287,US 1208524288,1208524351,PK 1208524352,1208524415,CN 1208524416,1208524863,US 1208524864,1208524927,PK -1208524928,1208525823,US +1208524928,1208525311,US +1208525312,1208525343,CN +1208525344,1208525375,TR +1208525376,1208525823,US 1208525824,1208526079,KH 1208526080,1208526335,US 1208526336,1208526591,CN -1208526592,1208527871,US -1208527872,1208528127,VN +1208526592,1208526607,TR +1208526608,1208526623,US +1208526624,1208526655,TR +1208526656,1208526815,US +1208526816,1208526847,CN +1208526848,1208527871,US +1208527872,1208527887,VN +1208527888,1208527903,US +1208527904,1208528127,VN 1208528128,1208531007,US 1208531008,1208531071,CN 1208531072,1208531455,US @@ -25573,17 +26467,25 @@ 1208588104,1208588111,MA 1208588112,1208588211,US 1208588212,1208588219,CA -1208588220,1208588427,US +1208588220,1208588408,US +1208588409,1208588416,PE +1208588417,1208588427,US 1208588428,1208588435,MO 1208588436,1208588463,US 1208588464,1208588476,AU 1208588477,1208588591,US 1208588592,1208588599,IN -1208588600,1208588728,US +1208588600,1208588706,US +1208588707,1208588707,PE +1208588708,1208588711,US +1208588712,1208588714,PE +1208588715,1208588728,US 1208588729,1208588760,GB 1208588761,1208588771,US 1208588772,1208588779,GB -1208588780,1208588953,US +1208588780,1208588861,US +1208588862,1208588869,PE +1208588870,1208588953,US 1208588954,1208588961,TR 1208588962,1208588972,US 1208588973,1208588985,GB @@ -25731,9 +26633,7 @@ 1208832392,1208832407,CN 1208832408,1208832455,US 1208832456,1208832463,CN -1208832464,1208832543,US -1208832544,1208832551,RS -1208832552,1208832567,US +1208832464,1208832567,US 1208832568,1208832575,CN 1208832576,1208832583,US 1208832584,1208832591,CN @@ -25800,7 +26700,10 @@ 1209271128,1209271131,SA 1209271132,1209271187,US 1209271188,1209271191,TR -1209271192,1209271367,US +1209271192,1209271195,RO +1209271196,1209271215,US +1209271216,1209271219,IN +1209271220,1209271367,US 1209271368,1209271371,TR 1209271372,1209271375,PE 1209271376,1209271435,US @@ -25813,9 +26716,12 @@ 1209271488,1209271495,CN 1209271496,1209271499,CA 1209271500,1209271503,MX -1209271504,1209271543,US +1209271504,1209271519,US +1209271520,1209271523,RO +1209271524,1209271543,US 1209271544,1209271551,CN -1209271552,1209271571,US +1209271552,1209271567,US +1209271568,1209271571,PL 1209271572,1209271575,TR 1209271576,1209271579,US 1209271580,1209271583,BD @@ -25823,7 +26729,7 @@ 1209271608,1209271611,KH 1209271612,1209271615,CA 1209271616,1209271639,US -1209271640,1209271643,CZ +1209271640,1209271643,CA 1209271644,1209271659,US 1209271660,1209271663,PK 1209271664,1209271691,US @@ -25835,7 +26741,15 @@ 1209271724,1209271727,US 1209271728,1209271731,PK 1209271732,1209271735,CN -1209271736,1209271847,US +1209271736,1209271767,US +1209271768,1209271771,RO +1209271772,1209271783,US +1209271784,1209271787,RO +1209271788,1209271827,US +1209271828,1209271831,CN +1209271832,1209271839,US +1209271840,1209271843,CN +1209271844,1209271847,US 1209271848,1209271855,IN 1209271856,1209271867,US 1209271868,1209271871,BR @@ -25849,7 +26763,8 @@ 1209271980,1209271983,ZA 1209271984,1209271999,US 1209272000,1209272007,TR -1209272008,1209272043,US +1209272008,1209272011,PL +1209272012,1209272043,US 1209272044,1209272047,TR 1209272048,1209272079,US 1209272080,1209272095,TR @@ -25862,12 +26777,13 @@ 1209272240,1209272255,TR 1209272256,1209272287,CN 1209272288,1209272383,US -1209272384,1209272399,CA +1209272384,1209272399,CN 1209272400,1209272407,US 1209272408,1209272415,PE 1209272416,1209272447,US 1209272448,1209272479,KH -1209272480,1209272511,US +1209272480,1209272495,CN +1209272496,1209272511,US 1209272512,1209272575,CN 1209272576,1209272583,US 1209272584,1209272591,PH @@ -25893,8 +26809,8 @@ 1209273216,1209273279,US 1209273280,1209273311,BR 1209273312,1209273359,US -1209273360,1209273375,TR -1209273376,1209273555,US +1209273360,1209273407,TR +1209273408,1209273555,US 1209273556,1209273559,CA 1209273560,1209273647,US 1209273648,1209273655,CN @@ -25902,7 +26818,8 @@ 1209273664,1209273671,BD 1209273672,1209273999,US 1209274000,1209274015,TR -1209274016,1209274031,US +1209274016,1209274023,US +1209274024,1209274031,RO 1209274032,1209274039,TR 1209274040,1209274047,US 1209274048,1209274111,CN @@ -25922,7 +26839,9 @@ 1209274928,1209274943,TR 1209274944,1209274959,US 1209274960,1209274967,PH -1209274968,1209275679,US +1209274968,1209275071,US +1209275072,1209275135,GB +1209275136,1209275679,US 1209275680,1209275695,TR 1209275696,1209275791,US 1209275792,1209275799,TR @@ -25936,17 +26855,22 @@ 1209275918,1209275918,KH 1209275919,1209275924,US 1209275925,1209275925,TR -1209275926,1209275928,US +1209275926,1209275927,US +1209275928,1209275928,CN 1209275929,1209275929,MY 1209275930,1209275930,US 1209275931,1209275931,AU 1209275932,1209275932,CN -1209275933,1209275999,US +1209275933,1209275935,US +1209275936,1209275951,RO +1209275952,1209275999,US 1209276000,1209276031,MY 1209276032,1209276063,BR 1209276064,1209276351,US 1209276352,1209276415,CN -1209276416,1209276671,US +1209276416,1209276591,US +1209276592,1209276607,RO +1209276608,1209276671,US 1209276672,1209276703,CN 1209276704,1209276799,US 1209276800,1209276803,SA @@ -25956,7 +26880,8 @@ 1209276912,1209276927,TR 1209276928,1209277103,US 1209277104,1209277119,TR -1209277120,1209277143,US +1209277120,1209277135,RO +1209277136,1209277143,US 1209277144,1209277147,TR 1209277148,1209277154,US 1209277155,1209277155,TR @@ -25968,35 +26893,60 @@ 1209277161,1209277166,US 1209277167,1209277167,CN 1209277168,1209277168,AU -1209277169,1209277215,US +1209277169,1209277171,US +1209277172,1209277172,AU +1209277173,1209277215,US 1209277216,1209277231,BR 1209277232,1209277247,US 1209277248,1209277263,BR 1209277264,1209277279,US 1209277280,1209277283,CA -1209277284,1209277575,US +1209277284,1209277287,US +1209277288,1209277295,RO +1209277296,1209277375,US +1209277376,1209277379,PL +1209277380,1209277399,US +1209277400,1209277419,CN +1209277420,1209277575,US 1209277576,1209277583,BD -1209277584,1209278083,US +1209277584,1209277599,TR +1209277600,1209277695,US +1209277696,1209277951,KH +1209277952,1209278083,US 1209278084,1209278087,CN 1209278088,1209278095,AU -1209278096,1209278207,US -1209278208,1209278463,IE -1209278464,1209278523,US +1209278096,1209278111,US +1209278112,1209278143,TR +1209278144,1209278207,US +1209278208,1209278463,KH +1209278464,1209278495,US +1209278496,1209278511,TR +1209278512,1209278523,US 1209278524,1209278527,CN 1209278528,1209278591,US 1209278592,1209278719,CN -1209278720,1209278827,US +1209278720,1209278795,US +1209278796,1209278799,CN +1209278800,1209278815,US +1209278816,1209278823,CN +1209278824,1209278827,US 1209278828,1209278831,CN -1209278832,1209278911,US +1209278832,1209278839,US +1209278840,1209278847,CN +1209278848,1209278911,US 1209278912,1209278919,TR 1209278920,1209278975,US 1209278976,1209279103,CA 1209279104,1209279119,US 1209279120,1209279135,CA -1209279136,1209279167,US +1209279136,1209279139,US +1209279140,1209279143,PL +1209279144,1209279167,US 1209279168,1209279231,VN 1209279232,1209279295,BR -1209279296,1209279855,US +1209279296,1209279615,US +1209279616,1209279743,GB +1209279744,1209279855,US 1209279856,1209279871,TR 1209279872,1209279935,MY 1209279936,1209279951,GB @@ -26007,14 +26957,19 @@ 1209280028,1209280031,SA 1209280032,1209280047,IN 1209280048,1209280191,US -1209280192,1209280255,VN -1209280256,1209280495,US -1209280496,1209280511,CA +1209280192,1209280255,ID +1209280256,1209280419,US +1209280420,1209280423,CN +1209280424,1209280439,US +1209280440,1209280447,CN +1209280448,1209280463,IN +1209280464,1209280495,US +1209280496,1209280511,CN 1209280512,1209280515,US 1209280516,1209280519,TR 1209280520,1209280527,AU 1209280528,1209280623,US -1209280624,1209280639,CN +1209280624,1209280639,TR 1209280640,1209280895,US 1209280896,1209280959,BR 1209280960,1209281007,US @@ -26027,19 +26982,27 @@ 1209281536,1209281791,IN 1209281792,1209281871,US 1209281872,1209281887,TR -1209281888,1209281983,US +1209281888,1209281927,US +1209281928,1209281935,IN +1209281936,1209281983,US 1209281984,1209282047,TR 1209282048,1209282063,US 1209282064,1209282067,SA -1209282068,1209282559,US +1209282068,1209282159,US +1209282160,1209282175,CN +1209282176,1209282559,US 1209282560,1209282623,CN 1209282624,1209283487,US 1209283488,1209283503,TR 1209283504,1209283543,US 1209283544,1209283547,SA -1209283548,1209283583,US +1209283548,1209283567,US +1209283568,1209283575,CN +1209283576,1209283583,US 1209283584,1209283587,SA -1209283588,1209283751,US +1209283588,1209283599,US +1209283600,1209283607,IN +1209283608,1209283751,US 1209283752,1209283759,BD 1209283760,1209283775,US 1209283776,1209283807,ZA @@ -26053,7 +27016,8 @@ 1209283920,1209284159,US 1209284160,1209284223,TR 1209284224,1209284351,IE -1209284352,1209284399,US +1209284352,1209284391,US +1209284392,1209284399,RO 1209284400,1209284415,TR 1209284416,1209284431,US 1209284432,1209284447,TR @@ -26066,7 +27030,7 @@ 1209284832,1209284839,US 1209284840,1209284843,TR 1209284844,1209284855,US -1209284856,1209284863,TR +1209284856,1209284863,CN 1209284864,1209284993,US 1209284994,1209284994,TR 1209284995,1209284995,US @@ -26076,11 +27040,12 @@ 1209284999,1209284999,US 1209285000,1209285000,CA 1209285001,1209285002,US -1209285003,1209285003,CN -1209285004,1209285004,CA +1209285003,1209285004,CN 1209285005,1209285009,US 1209285010,1209285010,CA -1209285011,1209285017,US +1209285011,1209285011,US +1209285012,1209285012,TR +1209285013,1209285017,US 1209285018,1209285018,CA 1209285019,1209285021,US 1209285022,1209285022,CA @@ -26626,7 +27591,9 @@ 1210101728,1210101743,GB 1210101744,1210101879,US 1210101880,1210101887,CA -1210101888,1210102399,US +1210101888,1210102287,US +1210102288,1210102295,IL +1210102296,1210102399,US 1210102400,1210102407,AU 1210102408,1210102815,US 1210102816,1210102823,CA @@ -26863,7 +27830,8 @@ 1211035664,1211035711,US 1211035712,1211035775,CA 1211035776,1211035791,US -1211035792,1211035823,CR +1211035792,1211035807,CR +1211035808,1211035823,US 1211035824,1211035839,CH 1211035840,1211036031,US 1211036032,1211036095,EC @@ -26873,7 +27841,10 @@ 1211036736,1211036751,GT 1211036752,1211036991,US 1211036992,1211037055,BR -1211037056,1211037519,US +1211037056,1211037135,US +1211037136,1211037151,GB +1211037152,1211037183,DO +1211037184,1211037519,US 1211037520,1211037535,UY 1211037536,1211037679,US 1211037680,1211037695,AR @@ -27216,12 +28187,15 @@ 1249010688,1249011711,CA 1249011712,1249019903,US 1249019904,1249020927,CA -1249020928,1249026703,US +1249020928,1249026423,US +1249026424,1249026431,CA +1249026432,1249026703,US 1249026704,1249026711,ZA 1249026712,1249026719,CA 1249026720,1249026767,US 1249026768,1249026775,CA -1249026776,1249027127,US +1249026776,1249026783,MX +1249026784,1249027127,US 1249027128,1249027135,ZA 1249027136,1249027143,AU 1249027144,1249027175,US @@ -27373,7 +28347,13 @@ 1249568328,1249571839,US 1249571840,1249572863,CA 1249572864,1249577087,US -1249577088,1249577983,CA +1249577088,1249577279,CA +1249577280,1249577343,US +1249577344,1249577480,CA +1249577481,1249577545,US +1249577546,1249577730,CA +1249577731,1249577794,US +1249577795,1249577983,CA 1249577984,1249592319,US 1249592320,1249593343,CA 1249593344,1249598463,US @@ -27535,9 +28515,7 @@ 1254953616,1254953623,GB 1254953624,1254953639,US 1254953640,1254953647,TW -1254953648,1254953671,US -1254953672,1254953679,MT -1254953680,1254953687,US +1254953648,1254953687,US 1254953688,1254953703,CA 1254953704,1254953743,US 1254953744,1254953751,IN @@ -28034,13 +29012,15 @@ 1255063552,1255071743,PR 1255071744,1255210495,US 1255210496,1255211007,DE -1255211008,1255276543,US +1255211008,1255265279,US +1255265280,1255266303,SG +1255266304,1255274495,US +1255274496,1255274751,SG +1255274752,1255276543,US 1255276544,1255342079,CA 1255342080,1255369055,US 1255369056,1255369087,DE -1255369088,1255370239,US -1255370240,1255370495,DO -1255370496,1255373759,US +1255369088,1255373759,US 1255373760,1255373791,DE 1255373792,1255489535,US 1255489536,1255505919,PR @@ -28078,7 +29058,9 @@ 1256079360,1256087551,KY 1256087552,1256098559,US 1256098560,1256098815,CA -1256098816,1263264305,US +1256098816,1263263999,US +1263264000,1263264127,CA +1263264128,1263264305,US 1263264306,1263264321,PK 1263264322,1263266623,US 1263266624,1263266655,CA @@ -28086,7 +29068,9 @@ 1263267328,1263267583,CA 1263267584,1263267647,US 1263267648,1263267679,CA -1263267680,1263267839,US +1263267680,1263267711,US +1263267712,1263267775,CA +1263267776,1263267839,US 1263267840,1263268191,CA 1263268192,1263268275,US 1263268276,1263268340,CA @@ -28103,7 +29087,7 @@ 1264718336,1264718591,US 1264718592,1264718599,GB 1264718600,1264718719,US -1264718720,1264718847,GB +1264718720,1264718847,CA 1264718848,1264719103,US 1264719104,1264719359,CA 1264719360,1264736255,US @@ -28512,7 +29496,9 @@ 1279953056,1279953087,US 1279953088,1279953671,CA 1279953672,1279953679,US -1279953680,1279953951,CA +1279953680,1279953759,CA +1279953760,1279953791,GB +1279953792,1279953951,CA 1279953952,1279953967,US 1279953968,1279954095,CA 1279954096,1279954119,US @@ -28521,8 +29507,10 @@ 1279954432,1279954527,CA 1279954528,1279954559,US 1279954560,1279954623,CA -1279954624,1279954687,FI -1279954688,1279954783,CA +1279954624,1279954655,FI +1279954656,1279954719,CA +1279954720,1279954727,US +1279954728,1279954783,CA 1279954784,1279954839,US 1279954840,1279954879,CA 1279954880,1279954911,US @@ -28607,9 +29595,12 @@ 1279967232,1279971327,CA 1279971328,1279971583,US 1279971584,1279972095,CA -1279972096,1279973951,US +1279972096,1279973887,US +1279973888,1279973951,IN 1279973952,1279973967,CA -1279973968,1279974175,US +1279973968,1279973983,US +1279973984,1279974015,IN +1279974016,1279974175,US 1279974176,1279974207,CN 1279974208,1279974271,CA 1279974272,1279974391,US @@ -29004,8 +29995,8 @@ 1296259584,1296259839,FR 1296259840,1296260351,NL 1296260352,1296260607,US -1296260608,1296261119,DE -1296261120,1296262399,FR +1296260608,1296262143,DE +1296262144,1296262399,FR 1296262400,1296262655,US 1296262656,1296262911,CA 1296262912,1296263935,US @@ -29354,7 +30345,9 @@ 1297867584,1297867647,SC 1297867648,1297867687,RU 1297867688,1297867695,CY -1297867696,1297867855,RU +1297867696,1297867743,RU +1297867744,1297867775,AQ +1297867776,1297867855,RU 1297867856,1297867871,MK 1297867872,1297867879,ES 1297867880,1297868799,RU @@ -29439,7 +30432,6 @@ 1298126848,1298127615,SA 1298127616,1298128127,IR 1298128128,1298128895,SA -1298128896,1298130943,FR 1298130944,1298132991,BG 1298132992,1298135039,FI 1298135040,1298137087,NL @@ -29521,7 +30513,11 @@ 1306001408,1306132479,RU 1306132480,1306198015,SE 1306198016,1306206207,LV -1306206208,1306263551,SE +1306206208,1306214399,HR +1306214400,1306222591,LT +1306222592,1306230783,HR +1306230784,1306238975,LT +1306238976,1306263551,SE 1306263552,1306271743,KE 1306271744,1306279935,RU 1306279936,1306286079,IT @@ -29696,8 +30692,7 @@ 1307756288,1307756431,GB 1307756432,1307756447,FR 1307756448,1307756543,GB -1307756544,1307756847,FR -1307756848,1307756863,US +1307756544,1307756863,FR 1307756864,1307756903,GB 1307756904,1307756911,US 1307756912,1307757063,GB @@ -29790,7 +30785,12 @@ 1307979776,1307981823,ZW 1307981824,1307982847,ZA 1307982848,1307983359,ZW -1307983360,1307983871,GB +1307983360,1307983423,ZA +1307983424,1307983487,BW +1307983488,1307983551,ZM +1307983552,1307983583,MZ +1307983584,1307983615,ZA +1307983616,1307983871,GB 1307983872,1307987967,LB 1307987968,1307992063,FR 1307992064,1307996159,RU @@ -29826,7 +30826,9 @@ 1308073984,1308078079,RU 1308078080,1308078879,NL 1308078880,1308078911,FR -1308078912,1308080127,NL +1308078912,1308079519,NL +1308079520,1308079535,DE +1308079536,1308080127,NL 1308080128,1308082175,RU 1308082176,1308084223,GB 1308084224,1308086271,RS @@ -30109,7 +31111,9 @@ 1311263376,1311263407,CH 1311263408,1311263423,BE 1311263424,1311263615,CH -1311263616,1311264767,FR +1311263616,1311263743,FR +1311263744,1311263871,CH +1311263872,1311264767,FR 1311264768,1311266815,RU 1311266816,1311268863,FR 1311268864,1311270911,BE @@ -30220,7 +31224,9 @@ 1311757440,1311757447,ES 1311757448,1311757463,DE 1311757464,1311757471,ES -1311757472,1312292863,DE +1311757472,1311767311,DE +1311767312,1311767319,IT +1311767320,1312292863,DE 1312292864,1312817151,LT 1312817152,1313865727,SE 1313865728,1313931263,CZ @@ -30411,7 +31417,9 @@ 1315917824,1315921919,RU 1315921920,1315926015,TR 1315926016,1315930111,CZ -1315930112,1315934207,DE +1315930112,1315930623,DE +1315930624,1315930879,LB +1315930880,1315934207,DE 1315934208,1315938303,RU 1315938304,1315942399,DK 1315942400,1315946495,UA @@ -30517,7 +31525,7 @@ 1317667048,1317667055,A2 1317667056,1317667063,LR 1317667064,1317667103,A2 -1317667104,1317667111,NG +1317667104,1317667111,UG 1317667112,1317667135,A2 1317667136,1317667143,AO 1317667144,1317667151,A2 @@ -30530,8 +31538,8 @@ 1317667240,1317667247,NG 1317667248,1317667263,A2 1317667264,1317667271,GB -1317667272,1317667279,A2 -1317667280,1317667295,NG +1317667272,1317667287,A2 +1317667288,1317667295,NG 1317667296,1317667311,A2 1317667312,1317667335,NG 1317667336,1317667343,A2 @@ -30564,8 +31572,7 @@ 1317667808,1317667815,NG 1317667816,1317668095,A2 1317668096,1317668103,GH -1317668104,1317668111,ZM -1317668112,1317668143,A2 +1317668104,1317668143,A2 1317668144,1317668151,AO 1317668152,1317668167,NG 1317668168,1317668183,A2 @@ -30575,12 +31582,10 @@ 1317668208,1317668215,LR 1317668216,1317668223,A2 1317668224,1317668239,NG -1317668240,1317668247,A2 -1317668248,1317668255,NG -1317668256,1317668271,A2 +1317668240,1317668271,A2 1317668272,1317668279,SL 1317668280,1317668303,A2 -1317668304,1317668311,NG +1317668304,1317668311,UG 1317668312,1317668319,IQ 1317668320,1317668343,A2 1317668344,1317668351,AO @@ -30596,7 +31601,7 @@ 1317668464,1317668471,AO 1317668472,1317668479,NG 1317668480,1317668487,A2 -1317668488,1317668495,NG +1317668488,1317668495,UG 1317668496,1317668503,GH 1317668504,1317668511,CD 1317668512,1317668519,IQ @@ -30608,9 +31613,9 @@ 1317668560,1317668575,NG 1317668576,1317668583,A2 1317668584,1317668591,LR -1317668592,1317668599,NG +1317668592,1317668599,UG 1317668600,1317668615,A2 -1317668616,1317668623,NG +1317668616,1317668623,UG 1317668624,1317668631,IQ 1317668632,1317668639,A2 1317668640,1317668655,NG @@ -30624,24 +31629,20 @@ 1317668784,1317668791,NG 1317668792,1317668807,A2 1317668808,1317668815,NG -1317668816,1317668831,A2 -1317668832,1317668839,NG -1317668840,1317668847,A2 +1317668816,1317668847,A2 1317668848,1317668855,NG 1317668856,1317668863,A2 1317668864,1317668871,LR 1317668872,1317668911,A2 1317668912,1317668919,NG -1317668920,1317668935,A2 -1317668936,1317668943,NG -1317668944,1317668959,A2 -1317668960,1317668983,NG -1317668984,1317668999,A2 +1317668920,1317668959,A2 +1317668960,1317668991,UG +1317668992,1317668999,A2 1317669000,1317669007,CD 1317669008,1317669015,NG 1317669016,1317669055,A2 -1317669056,1317669079,NG -1317669080,1317669095,A2 +1317669056,1317669071,NG +1317669072,1317669095,A2 1317669096,1317669103,NG 1317669104,1317669111,A2 1317669112,1317669119,NG @@ -30657,14 +31658,10 @@ 1317669992,1317669999,NG 1317670000,1317670007,A2 1317670008,1317670015,NG -1317670016,1317670071,A2 -1317670072,1317670079,NG -1317670080,1317670087,A2 -1317670088,1317670103,NG -1317670104,1317670111,A2 -1317670112,1317670119,NG -1317670120,1317670127,A2 -1317670128,1317670143,NG +1317670016,1317670063,A2 +1317670064,1317670103,NG +1317670104,1317670135,A2 +1317670136,1317670143,NG 1317670144,1317670175,SL 1317670176,1317670215,A2 1317670216,1317670223,NG @@ -30682,12 +31679,15 @@ 1317670512,1317670519,CI 1317670520,1317670543,NG 1317670544,1317670551,IQ -1317670552,1317670567,NG +1317670552,1317670559,UG +1317670560,1317670567,NG 1317670568,1317670575,A2 1317670576,1317670583,NG 1317670584,1317670591,GH 1317670592,1317670607,A2 -1317670608,1317670639,NG +1317670608,1317670615,NG +1317670616,1317670631,UG +1317670632,1317670639,NG 1317670640,1317670647,A2 1317670648,1317670655,NG 1317670656,1317670663,AO @@ -30714,13 +31714,12 @@ 1317670897,1317670911,NG 1317670912,1317671175,A2 1317671176,1317671191,NG -1317671192,1317671231,A2 -1317671232,1317671239,NG +1317671192,1317671239,A2 1317671240,1317671247,CI 1317671248,1317671255,NG 1317671256,1317671263,CI -1317671264,1317671287,A2 -1317671288,1317671303,NG +1317671264,1317671279,A2 +1317671280,1317671303,NG 1317671304,1317671319,A2 1317671320,1317671327,NG 1317671328,1317671335,A2 @@ -30734,12 +31733,10 @@ 1317671400,1317671407,IQ 1317671408,1317671415,UG 1317671416,1317671439,A2 -1317671440,1317671463,NG -1317671464,1317671471,A2 -1317671472,1317671487,NG +1317671440,1317671447,ZW +1317671448,1317671487,NG 1317671488,1317671527,A2 -1317671528,1317671535,NG -1317671536,1317671543,A2 +1317671528,1317671543,NG 1317671544,1317671551,GH 1317671552,1317671567,NG 1317671568,1317671583,A2 @@ -30758,10 +31755,11 @@ 1317671808,1317671823,CI 1317671824,1317671831,NG 1317671832,1317671847,A2 -1317671848,1317671863,NG +1317671848,1317671855,NG +1317671856,1317671863,UG 1317671864,1317671887,A2 1317671888,1317671895,NG -1317671896,1317671903,CM +1317671896,1317671903,A2 1317671904,1317671911,NG 1317671912,1317671919,AO 1317671920,1317671927,NG @@ -30783,7 +31781,7 @@ 1317672664,1317672671,A2 1317672672,1317672679,NG 1317672680,1317672727,A2 -1317672728,1317672735,NG +1317672728,1317672735,UG 1317672736,1317672743,LR 1317672744,1317672759,A2 1317672760,1317672767,NG @@ -30794,12 +31792,14 @@ 1317672840,1317672847,NG 1317672848,1317672863,A2 1317672864,1317672871,NG -1317672872,1317672879,A2 -1317672880,1317672895,NG +1317672872,1317672887,A2 +1317672888,1317672895,NG 1317672896,1317672903,A2 1317672904,1317672919,CI 1317672920,1317672927,GH -1317672928,1317672951,NG +1317672928,1317672935,A2 +1317672936,1317672943,NG +1317672944,1317672951,UG 1317672952,1317672959,A2 1317672960,1317673247,NG 1317673248,1317673263,A2 @@ -30824,18 +31824,17 @@ 1317673480,1317673495,A2 1317673496,1317673527,NG 1317673528,1317673535,A2 -1317673536,1317673551,NG +1317673536,1317673551,ZW 1317673552,1317673559,A2 1317673560,1317673575,NG 1317673576,1317673583,CD -1317673584,1317673599,A2 -1317673600,1317673615,NG +1317673584,1317673607,A2 +1317673608,1317673615,NG 1317673616,1317673623,A2 1317673624,1317673631,NG 1317673632,1317673639,A2 1317673640,1317673647,AO -1317673648,1317673671,A2 -1317673672,1317673679,CM +1317673648,1317673679,A2 1317673680,1317673687,IQ 1317673688,1317673695,A2 1317673696,1317673703,NG @@ -30855,8 +31854,7 @@ 1317673920,1317673927,CI 1317673928,1317673943,NG 1317673944,1317673951,AO -1317673952,1317673959,A2 -1317673960,1317673967,NG +1317673952,1317673967,NG 1317673968,1317673975,A2 1317673976,1317674239,NG 1317674240,1317674271,A2 @@ -30871,14 +31869,14 @@ 1317674384,1317674399,NG 1317674400,1317674415,A2 1317674416,1317674423,AO -1317674424,1317674431,NG +1317674424,1317674431,UG 1317674432,1317674439,A2 1317674440,1317674440,AO 1317674441,1317674447,A2 1317674448,1317674455,AO 1317674456,1317674471,NG 1317674472,1317674487,A2 -1317674488,1317674495,NG +1317674488,1317674495,UG 1317674496,1317674527,A2 1317674528,1317674535,NG 1317674536,1317674543,IQ @@ -30899,7 +31897,9 @@ 1317674744,1317674751,IQ 1317674752,1317674759,NG 1317674760,1317674767,A2 -1317674768,1317674791,NG +1317674768,1317674775,NG +1317674776,1317674783,A2 +1317674784,1317674791,NG 1317674792,1317674799,A2 1317674800,1317674807,NG 1317674808,1317674823,A2 @@ -30919,28 +31919,23 @@ 1317674984,1317674999,A2 1317675000,1317675007,NG 1317675008,1317675031,A2 -1317675032,1317675047,NG -1317675048,1317675055,A2 +1317675032,1317675039,NG +1317675040,1317675055,A2 1317675056,1317675063,NG 1317675064,1317675071,AO 1317675072,1317675095,NG -1317675096,1317675183,A2 -1317675184,1317675199,CM +1317675096,1317675199,A2 1317675200,1317675215,NG 1317675216,1317675247,A2 1317675248,1317675255,AO 1317675256,1317675271,A2 1317675272,1317675279,NG -1317675280,1317675287,CM -1317675288,1317675295,A2 +1317675280,1317675295,A2 1317675296,1317675319,NG -1317675320,1317675343,CM -1317675344,1317675351,A2 +1317675320,1317675351,A2 1317675352,1317675359,IQ 1317675360,1317675367,NG -1317675368,1317675375,A2 -1317675376,1317675383,CM -1317675384,1317675407,A2 +1317675368,1317675407,A2 1317675408,1317675415,NG 1317675416,1317675423,A2 1317675424,1317675439,NG @@ -30977,33 +31972,34 @@ 1317675760,1317675775,GH 1317675776,1317675783,NG 1317675784,1317675799,A2 -1317675800,1317675807,NG +1317675800,1317675807,GB 1317675808,1317675823,A2 -1317675824,1317675839,NG -1317675840,1317675855,A2 +1317675824,1317675847,NG +1317675848,1317675855,A2 1317675856,1317675863,NG 1317675864,1317675887,A2 1317675888,1317675895,CD 1317675896,1317675911,A2 -1317675912,1317675927,NG +1317675912,1317675919,NG +1317675920,1317675927,UG 1317675928,1317675943,A2 -1317675944,1317675951,GB +1317675944,1317675951,UG 1317675952,1317675975,NG -1317675976,1317675991,A2 +1317675976,1317675983,A2 +1317675984,1317675991,ML 1317675992,1317675999,NG 1317676000,1317676007,GN 1317676008,1317676015,GH 1317676016,1317676023,SD 1317676024,1317676031,GQ -1317676032,1317676039,NG +1317676032,1317676039,A2 1317676040,1317676047,CD 1317676048,1317676055,A2 1317676056,1317676063,NG -1317676064,1317676071,CM -1317676072,1317676079,A2 +1317676064,1317676079,A2 1317676080,1317676087,AO 1317676088,1317676095,NG -1317676096,1317676103,A2 +1317676096,1317676103,CM 1317676104,1317676111,NG 1317676112,1317676119,CM 1317676120,1317676127,A2 @@ -31015,13 +32011,16 @@ 1317676168,1317676175,ML 1317676176,1317676191,A2 1317676192,1317676207,SD -1317676208,1317676231,A2 +1317676208,1317676215,A2 +1317676216,1317676223,NG +1317676224,1317676231,A2 1317676232,1317676239,NG 1317676240,1317676247,SD 1317676248,1317676255,NG 1317676256,1317676263,CD 1317676264,1317676271,NG -1317676272,1317676287,A2 +1317676272,1317676279,A2 +1317676280,1317676287,NG 1317676288,1317676543,LR 1317676544,1317676551,A2 1317676552,1317676559,AO @@ -31053,9 +32052,10 @@ 1317676848,1317676855,NG 1317676856,1317676863,A2 1317676864,1317676871,TG -1317676872,1317676911,A2 -1317676912,1317676919,NG -1317676920,1317676943,A2 +1317676872,1317676879,NG +1317676880,1317676903,A2 +1317676904,1317676927,NG +1317676928,1317676943,A2 1317676944,1317676951,CM 1317676952,1317676983,A2 1317676984,1317676991,NG @@ -31069,14 +32069,12 @@ 1317677056,1317677063,UG 1317677064,1317677071,A2 1317677072,1317677079,NG -1317677080,1317677087,SD +1317677080,1317677087,UG 1317677088,1317677095,A2 1317677096,1317677103,BF 1317677104,1317677127,A2 1317677128,1317677135,NG -1317677136,1317677143,A2 -1317677144,1317677151,SD -1317677152,1317677191,A2 +1317677136,1317677191,A2 1317677192,1317677199,SD 1317677200,1317677215,NG 1317677216,1317677231,A2 @@ -31095,20 +32093,19 @@ 1317677376,1317677399,A2 1317677400,1317677407,NG 1317677408,1317677415,CD -1317677416,1317677423,A2 +1317677416,1317677423,LR 1317677424,1317677431,CD 1317677432,1317677439,LR 1317677440,1317677447,NG 1317677448,1317677455,CD 1317677456,1317677463,AO -1317677464,1317677471,A2 +1317677464,1317677471,GQ 1317677472,1317677479,NG 1317677480,1317677487,AO -1317677488,1317677495,A2 +1317677488,1317677495,CD 1317677496,1317677527,NG 1317677528,1317677535,GQ -1317677536,1317677543,A2 -1317677544,1317677551,CD +1317677536,1317677551,CD 1317677552,1317677567,NG 1317677568,1317677663,A2 1317677664,1317677671,GB @@ -31127,7 +32124,7 @@ 1317678056,1317678079,A2 1317678080,1317678095,NG 1317678096,1317678103,GQ -1317678104,1317678111,BE +1317678104,1317678111,NG 1317678112,1317678127,A2 1317678128,1317678135,SD 1317678136,1317678143,NG @@ -31135,16 +32132,19 @@ 1317678152,1317678159,NG 1317678160,1317678167,GQ 1317678168,1317678175,NG -1317678176,1317678183,A2 +1317678176,1317678183,GR 1317678184,1317678191,NG 1317678192,1317678199,A2 1317678200,1317678207,CD -1317678208,1317678231,A2 +1317678208,1317678223,A2 +1317678224,1317678231,NG 1317678232,1317678239,BF -1317678240,1317678271,A2 +1317678240,1317678247,CD +1317678248,1317678271,A2 1317678272,1317678287,NG 1317678288,1317678295,GR -1317678296,1317678319,A2 +1317678296,1317678311,A2 +1317678312,1317678319,SD 1317678320,1317678327,NG 1317678328,1317678335,A2 1317678336,1317678343,NG @@ -31154,7 +32154,9 @@ 1317678368,1317678375,CD 1317678376,1317678383,NG 1317678384,1317678391,GL -1317678392,1317678415,A2 +1317678392,1317678399,A2 +1317678400,1317678407,ML +1317678408,1317678415,A2 1317678416,1317678439,NG 1317678440,1317678447,CD 1317678448,1317678455,A2 @@ -31163,43 +32165,48 @@ 1317678472,1317678479,AO 1317678480,1317678487,CD 1317678488,1317678495,ML -1317678496,1317678511,A2 -1317678512,1317678527,NG +1317678496,1317678503,A2 +1317678504,1317678527,NG 1317678528,1317678543,A2 1317678544,1317678551,CD -1317678552,1317678559,A2 +1317678552,1317678559,ML 1317678560,1317678567,NG 1317678568,1317678575,CD -1317678576,1317679615,A2 +1317678576,1317678583,NG +1317678584,1317678591,AO +1317678592,1317679615,A2 1317679616,1317679631,AO -1317679632,1317679639,A2 +1317679632,1317679639,NG 1317679640,1317679647,ZW -1317679648,1317679663,A2 +1317679648,1317679655,CD +1317679656,1317679663,A2 1317679664,1317679671,AO -1317679672,1317679679,A2 -1317679680,1317679687,NG +1317679672,1317679687,NG 1317679688,1317679695,A2 1317679696,1317679703,LU -1317679704,1317679719,NG -1317679720,1317679727,A2 +1317679704,1317679727,NG 1317679728,1317679735,GN 1317679736,1317679743,NG 1317679744,1317679751,AO 1317679752,1317679767,NG 1317679768,1317679775,A2 1317679776,1317679799,NG -1317679800,1317679831,A2 -1317679832,1317679847,NG +1317679800,1317679807,A2 +1317679808,1317679847,NG 1317679848,1317679855,A2 -1317679856,1317679863,NG -1317679864,1317683199,A2 +1317679856,1317679871,NG +1317679872,1317681183,A2 +1317681184,1317681191,NG +1317681192,1317683199,A2 1317683200,1317683839,DE 1317683840,1317683855,CH 1317683856,1317683863,DE 1317683864,1317683871,FR 1317683872,1317685503,DE 1317685504,1317685631,BE -1317685632,1317686335,DE +1317685632,1317686303,DE +1317686304,1317686319,NL +1317686320,1317686335,DE 1317686336,1317686399,NL 1317686400,1317695743,DE 1317695744,1317695999,CH @@ -31213,9 +32220,9 @@ 1317765120,1317781503,GE 1317781504,1317814271,RU 1317814272,1317830655,DE -1317830656,1317831807,NL -1317831808,1317831935,CA -1317831936,1317832191,NL +1317830656,1317831839,NL +1317831840,1317831871,CA +1317831872,1317832191,NL 1317832192,1317832447,GB 1317832448,1317832511,CY 1317832512,1317832575,NL @@ -31278,8 +32285,8 @@ 1317994496,1317995519,NL 1317995520,1317996095,DE 1317996096,1317996287,NL -1317996288,1317997567,DE -1317997568,1317998591,NL +1317996288,1317998079,DE +1317998080,1317998591,NL 1317998592,1318000383,DE 1318000384,1318000447,NL 1318000448,1318002175,DE @@ -31493,27 +32500,47 @@ 1331931136,1331933183,SE 1331933184,1331935231,TR 1331935232,1331937279,NL -1331937280,1331937535,GB +1331937280,1331937407,PL +1331937408,1331937471,GB +1331937472,1331937519,NG +1331937520,1331937535,GB 1331937536,1331937567,SM 1331937568,1331937583,US 1331937584,1331937599,ZA -1331937600,1331938111,GB +1331937600,1331937663,GB +1331937664,1331937727,DE +1331937728,1331938111,GB 1331938112,1331938127,AE 1331938128,1331938135,SA -1331938136,1331938191,GB +1331938136,1331938143,AE +1331938144,1331938151,GB +1331938152,1331938159,PL +1331938160,1331938167,GB +1331938168,1331938175,KW +1331938176,1331938191,GB 1331938192,1331938207,AE -1331938208,1331938375,GB +1331938208,1331938319,GB +1331938320,1331938335,AE +1331938336,1331938375,GB 1331938376,1331938383,AE 1331938384,1331938399,IE -1331938400,1331938823,GB +1331938400,1331938815,GB +1331938816,1331938823,NG 1331938824,1331938831,US 1331938832,1331938839,NG 1331938840,1331938847,IE -1331938848,1331938879,GB -1331938880,1331938895,NG -1331938896,1331938943,GB +1331938848,1331938871,GB +1331938872,1331938895,NG +1331938896,1331938911,GB +1331938912,1331938927,NG +1331938928,1331938935,RU +1331938936,1331938943,NG 1331938944,1331938958,US -1331938959,1331939327,GB +1331938959,1331938959,GB +1331938960,1331938967,US +1331938968,1331938983,GB +1331938984,1331938999,NG +1331939000,1331939327,GB 1331939328,1331941375,BE 1331941376,1331943423,ES 1331943424,1331945471,RU @@ -31664,9 +32691,7 @@ 1334647808,1334648063,LB 1334648064,1334648319,IQ 1334648320,1334648831,A2 -1334648832,1334650959,GB -1334650960,1334650975,NL -1334650976,1334651391,GB +1334648832,1334651391,GB 1334651392,1334651647,NL 1334651648,1334651903,FR 1334651904,1334652159,DE @@ -31705,7 +32730,9 @@ 1334725632,1334726143,SE 1334726144,1334726655,NL 1334726656,1334729983,RU -1334729984,1334730431,KZ +1334729984,1334730015,KZ +1334730016,1334730239,RU +1334730240,1334730431,KZ 1334730432,1334730439,RU 1334730440,1334730443,KZ 1334730444,1334730447,RU @@ -32356,9 +33383,7 @@ 1347327232,1347327487,SK 1347327488,1347327743,CZ 1347327744,1347327999,SK -1347328000,1347329535,CZ -1347329536,1347329599,SK -1347329600,1347330047,CZ +1347328000,1347330047,CZ 1347330048,1347338239,DE 1347338240,1347342335,RU 1347342336,1347346431,SE @@ -32373,11 +33398,19 @@ 1347379200,1347383295,NL 1347383296,1347384111,EE 1347384112,1347384115,FI -1347384116,1347385063,EE +1347384116,1347384703,EE +1347384704,1347384831,US +1347384832,1347385063,EE 1347385064,1347385071,RU -1347385072,1347386559,EE +1347385072,1347385215,EE +1347385216,1347385343,US +1347385344,1347385599,EE +1347385600,1347385855,US +1347385856,1347386559,EE 1347386560,1347386567,RU -1347386568,1347386751,EE +1347386568,1347386655,EE +1347386656,1347386671,US +1347386672,1347386751,EE 1347386752,1347386815,MY 1347386816,1347387011,EE 1347387012,1347387015,DE @@ -32999,7 +34032,11 @@ 1352149744,1352149751,AT 1352149752,1352149783,DE 1352149784,1352149791,SE -1352149792,1352277535,DE +1352149792,1352149807,DE +1352149808,1352149823,SE +1352149824,1352149855,DE +1352149856,1352149871,SE +1352149872,1352277535,DE 1352277536,1352277567,IT 1352277568,1352287399,DE 1352287400,1352287407,NL @@ -33032,11 +34069,15 @@ 1353258264,1353258271,NO 1353258272,1353258303,SE 1353258304,1353258367,FI -1353258368,1353258415,SE +1353258368,1353258391,SE +1353258392,1353258399,GB +1353258400,1353258415,SE 1353258416,1353258423,DK 1353258424,1353258495,SE 1353258496,1353258503,GB -1353258504,1353258639,SE +1353258504,1353258527,SE +1353258528,1353258559,GB +1353258560,1353258639,SE 1353258640,1353258783,GB 1353258784,1353258807,SE 1353258808,1353259271,GB @@ -33067,13 +34108,17 @@ 1353272040,1353272047,GB 1353272048,1353272055,ES 1353272056,1353272063,GB -1353272064,1353272245,ES +1353272064,1353272127,ES +1353272128,1353272135,GB +1353272136,1353272245,ES 1353272246,1353272247,GB 1353272248,1353272691,ES 1353272692,1353272695,GB 1353272696,1353272807,ES 1353272808,1353272815,GB -1353272816,1353273343,ES +1353272816,1353272887,ES +1353272888,1353272895,GB +1353272896,1353273343,ES 1353273344,1353273631,BE 1353273632,1353273639,ES 1353273640,1353273711,BE @@ -33098,16 +34143,16 @@ 1353280120,1353280127,GB 1353280128,1353280143,IT 1353280144,1353280151,GB -1353280152,1353280287,IT -1353280288,1353280295,GB -1353280296,1353280663,IT -1353280664,1353280679,GB +1353280152,1353280671,IT +1353280672,1353280679,GB 1353280680,1353281023,IT 1353281024,1353281535,BE 1353281536,1353282047,GB 1353282048,1353282159,IT 1353282160,1353282167,GB -1353282168,1353282559,IT +1353282168,1353282215,IT +1353282216,1353282223,GB +1353282224,1353282559,IT 1353282560,1353283071,GB 1353283072,1353283327,IT 1353283328,1353287327,GB @@ -33437,7 +34482,7 @@ 1357334016,1357334271,TZ 1357334272,1357334527,CA 1357334528,1357335039,FR -1357335040,1357335295,LB +1357335040,1357335295,GB 1357335296,1357335551,FR 1357335552,1357335807,GB 1357335808,1357336063,KE @@ -33452,9 +34497,10 @@ 1357339648,1357339903,DZ 1357339904,1357340159,GB 1357340160,1357340415,DE -1357340416,1357340671,LB -1357340672,1357342719,GB -1357342720,1357348863,EU +1357340416,1357342719,GB +1357342720,1357344767,EU +1357344768,1357346815,CA +1357346816,1357348863,EU 1357348864,1357349119,DE 1357349120,1357349375,LU 1357349376,1357349887,GB @@ -33463,7 +34509,8 @@ 1357350912,1357351167,GB 1357351168,1357351423,PL 1357351424,1357352959,GB -1357352960,1357355007,EU +1357352960,1357353983,CA +1357353984,1357355007,EU 1357355008,1357355263,NL 1357355264,1357355775,FR 1357355776,1357356031,GB @@ -33476,9 +34523,10 @@ 1357358592,1357358847,DE 1357358848,1357359103,PL 1357359104,1357359999,ES -1357360000,1357360127,EU -1357360128,1357360255,GB -1357360256,1357360383,EU +1357360000,1357360031,GB +1357360032,1357360127,EU +1357360128,1357360279,GB +1357360280,1357360383,EU 1357360384,1357360639,ES 1357360640,1357360895,GB 1357360896,1357361151,DE @@ -33501,8 +34549,10 @@ 1357366816,1357366847,FR 1357366848,1357366863,GB 1357366864,1357366879,ES -1357366880,1357367039,EU -1357367040,1357367295,GB +1357366880,1357366911,EU +1357366912,1357366927,GB +1357366928,1357366975,EU +1357366976,1357367295,GB 1357367296,1357367551,FR 1357367552,1357367807,EU 1357367808,1357368063,GB @@ -33624,7 +34674,7 @@ 1357884160,1357884415,GB 1357884416,1357884423,FR 1357884424,1357884427,BE -1357884428,1357884431,EU +1357884428,1357884431,DE 1357884432,1357884439,US 1357884440,1357884447,FR 1357884448,1357884511,EU @@ -33842,8 +34892,8 @@ 1358223696,1358223719,DE 1358223720,1358223783,GB 1358223784,1358223791,DE -1358223792,1358223799,GB -1358223800,1358223843,DE +1358223792,1358223807,GB +1358223808,1358223843,DE 1358223844,1358223871,GB 1358223872,1358223887,NL 1358223888,1358223895,DE @@ -34033,7 +35083,16 @@ 1358475264,1358479359,GB 1358479360,1358483455,LI 1358483456,1358487551,FR -1358487552,1358491647,SE +1358487552,1358487711,SE +1358487712,1358487727,US +1358487728,1358487999,FR +1358488000,1358488223,SE +1358488224,1358488239,US +1358488240,1358488255,FR +1358488256,1358488479,SE +1358488480,1358488495,US +1358488496,1358488511,FR +1358488512,1358491647,SE 1358491648,1358493823,FI 1358493824,1358493887,AX 1358493888,1358495743,FI @@ -34083,9 +35142,7 @@ 1358668480,1358668495,GB 1358668496,1358668535,PT 1358668536,1358668543,GB -1358668544,1358668559,PT -1358668560,1358668575,GB -1358668576,1358668799,PT +1358668544,1358668799,PT 1358668800,1358668807,GB 1358668808,1358668927,PT 1358668928,1358668959,ES @@ -34236,7 +35293,8 @@ 1358861312,1358861823,DE 1358861824,1358862335,FR 1358862336,1358862847,US -1358862848,1358863359,DK +1358862848,1358863103,DK +1358863104,1358863359,CH 1358863360,1358864383,US 1358864384,1358864399,CH 1358864400,1358864407,IN @@ -34284,28 +35342,23 @@ 1358899392,1358899407,A2 1358899408,1358899415,DE 1358899416,1358899423,CD -1358899424,1358899431,US -1358899432,1358899455,NG +1358899424,1358899439,A2 +1358899440,1358899455,NG 1358899456,1358899463,A2 1358899464,1358899471,US 1358899472,1358899479,SV -1358899480,1358899583,A2 -1358899584,1358899591,IN -1358899592,1358899599,NG +1358899480,1358899599,A2 1358899600,1358899607,LK 1358899608,1358899615,A2 1358899616,1358899631,NG 1358899632,1358899647,NP -1358899648,1358899654,NG -1358899655,1358899983,A2 +1358899648,1358899983,A2 1358899984,1358899991,KY 1358899992,1358900767,A2 1358900768,1358900783,BI 1358900784,1358900959,A2 1358900960,1358900975,SL -1358900976,1358901543,A2 -1358901544,1358901551,DE -1358901552,1358901807,A2 +1358900976,1358901807,A2 1358901808,1358901815,CD 1358901816,1358901935,A2 1358901936,1358901951,NG @@ -34397,8 +35450,8 @@ 1359036416,1359052799,GB 1359069184,1359101951,RU 1359101952,1359118335,GB -1359118336,1359118655,DE -1359118656,1359118719,NL +1359118336,1359118591,DE +1359118592,1359118719,NL 1359118720,1359118815,DE 1359118816,1359118831,NL 1359118832,1359120383,DE @@ -34617,7 +35670,9 @@ 1360265216,1360269311,GB 1360269312,1360273407,KG 1360273408,1360281599,GB -1360281600,1360282751,NL +1360281600,1360281727,NL +1360281728,1360281855,ES +1360281856,1360282751,NL 1360282752,1360282783,GB 1360282784,1360285695,NL 1360285696,1360289791,DE @@ -35220,11 +36275,7 @@ 1362397440,1362397503,US 1362397504,1362398463,FR 1362398464,1362398719,DE -1362398720,1362399231,FR -1362399232,1362399263,GB -1362399264,1362399487,FR -1362399488,1362400767,GB -1362400768,1362403583,FR +1362398720,1362403583,FR 1362403584,1362405887,DZ 1362405888,1362406143,FR 1362406144,1362407167,SV @@ -35275,7 +36326,9 @@ 1364526592,1364528639,GB 1364528640,1364528895,UA 1364528896,1364530175,GB -1364530176,1364540671,NL +1364530176,1364531455,NL +1364531456,1364531711,IT +1364531712,1364540671,NL 1364540672,1364540927,US 1364540928,1364577023,NL 1364577024,1364577279,GB @@ -35401,7 +36454,9 @@ 1365004064,1365004287,GB 1365004288,1365008383,FR 1365008384,1365012479,CH -1365012480,1365016575,FR +1365012480,1365015927,FR +1365015928,1365015935,ES +1365015936,1365016575,FR 1365016576,1365020671,ES 1365020672,1365024767,CZ 1365024768,1365027839,DE @@ -35685,8 +36740,7 @@ 1365220424,1365220431,GB 1365220432,1365220435,JO 1365220436,1365220439,US -1365220440,1365220471,GB -1365220472,1365220479,US +1365220440,1365220479,GB 1365220480,1365220487,AU 1365220488,1365220523,US 1365220524,1365220527,GB @@ -36191,9 +37245,7 @@ 1382233424,1382252543,SE 1382252544,1382268927,CZ 1382268928,1382285311,IR -1382285312,1382301391,CZ -1382301392,1382301407,SK -1382301408,1382301695,CZ +1382285312,1382301695,CZ 1382301696,1382318079,SE 1382318080,1382334463,RU 1382334464,1382350847,DE @@ -36231,7 +37283,9 @@ 1383097088,1383097343,ES 1383097344,1383098111,GB 1383098112,1383098367,DE -1383098368,1383099391,GB +1383098368,1383099135,GB +1383099136,1383099151,DE +1383099152,1383099391,GB 1383099392,1383099903,DE 1383099904,1383100415,FR 1383100416,1383100831,GB @@ -36390,12 +37444,9 @@ 1384192576,1384192639,SE 1384192640,1384192767,DE 1384192768,1384193023,BG -1384193024,1384194047,NL -1384194048,1384194191,DE +1384193024,1384194191,DE 1384194192,1384194207,AT -1384194208,1384194271,DE -1384194272,1384194303,NL -1384194304,1384194559,DE +1384194208,1384194559,DE 1384194560,1384194815,NL 1384194816,1384195711,DE 1384195712,1384195743,NL @@ -36639,9 +37690,7 @@ 1385565440,1385566207,EU 1385566208,1385566399,FR 1385566400,1385566431,EU -1385566432,1385566447,FR -1385566448,1385566455,EU -1385566456,1385566847,FR +1385566432,1385566847,FR 1385566848,1385566927,EU 1385566928,1385566935,FR 1385566936,1385566959,EU @@ -36863,12 +37912,15 @@ 1388689648,1388691455,CH 1388691456,1388699647,NL 1388699648,1388707839,SE -1388707840,1388708863,RU -1388708864,1388709119,LT +1388707840,1388708607,RU +1388708608,1388709119,LT 1388709120,1388709375,IQ 1388709376,1388709887,RU 1388709888,1388710143,LB -1388710144,1388711679,RU +1388710144,1388710399,RU +1388710400,1388710911,LB +1388710912,1388711167,IQ +1388711168,1388711679,RU 1388711680,1388711935,IQ 1388711936,1388712191,RU 1388712192,1388712703,LT @@ -37497,16 +38549,15 @@ 1389723648,1389756415,ES 1389756416,1389772799,SE 1389772800,1389778431,SI -1389778432,1389778943,US +1389778432,1389778943,CA 1389778944,1389780991,RS 1389780992,1389782527,HR 1389782528,1389782543,SI 1389782544,1389782559,HR 1389782560,1389783039,SI -1389783040,1389784063,BA -1389784064,1389785087,SI -1389785088,1389785855,MK -1389785856,1389788671,SI +1389783040,1389785087,BA +1389785088,1389787135,MK +1389787136,1389788671,SI 1389788672,1389789183,RS 1389789184,1389805567,PL 1389805568,1389821951,US @@ -37843,7 +38894,7 @@ 1401746020,1401746175,SE 1401746176,1401746191,DK 1401746192,1401746215,SE -1401746216,1401746223,GB +1401746216,1401746223,BG 1401746224,1401746239,SE 1401746240,1401746255,NL 1401746256,1401746271,ES @@ -37854,7 +38905,8 @@ 1401746432,1401746447,NL 1401746448,1401746467,SE 1401746468,1401746471,NL -1401746472,1401746487,GB +1401746472,1401746479,US +1401746480,1401746487,GB 1401746488,1401746495,SE 1401746496,1401746511,NL 1401746512,1401746527,ES @@ -38058,7 +39110,13 @@ 1403518976,1403535359,CY 1403535360,1403551743,PL 1403551744,1403568127,SK -1403568128,1403580159,SE +1403568128,1403573247,SE +1403573248,1403573503,ES +1403573504,1403574783,SE +1403574784,1403575039,IT +1403575040,1403576063,SE +1403576064,1403576319,DE +1403576320,1403580159,SE 1403580160,1403580415,GB 1403580416,1403584511,SE 1403584512,1403600895,DE @@ -38067,63 +39125,75 @@ 1403601536,1403617279,FR 1403617280,1403633663,DE 1403633664,1403650047,RU -1403650048,1403655935,DE +1403650048,1403651647,A2 +1403651648,1403651663,DE +1403651664,1403651727,A2 +1403651728,1403651743,DE +1403651744,1403651807,A2 +1403651808,1403651831,DE +1403651832,1403651839,A2 +1403651840,1403652095,DE +1403652096,1403652319,A2 +1403652320,1403652351,DE +1403652352,1403655935,A2 1403655936,1403655943,IR -1403655944,1403656703,DE +1403655944,1403656703,A2 1403656704,1403656959,IR -1403656960,1403658495,DE +1403656960,1403658495,A2 1403658496,1403658527,IR 1403658528,1403658559,KW -1403658560,1403658847,DE +1403658560,1403658847,A2 1403658848,1403658879,IR -1403658880,1403658911,DE +1403658880,1403658911,A2 1403658912,1403658975,IR -1403658976,1403660735,DE +1403658976,1403660735,A2 1403660736,1403660799,IR -1403660800,1403661183,DE +1403660800,1403661183,A2 1403661184,1403661215,IR 1403661216,1403661279,AE 1403661280,1403661310,IR -1403661311,1403661631,DE +1403661311,1403661631,A2 1403661632,1403661663,IR -1403661664,1403661727,DE +1403661664,1403661727,A2 1403661728,1403661759,IR -1403661760,1403661887,DE +1403661760,1403661887,A2 1403661888,1403661919,IR -1403661920,1403661951,DE +1403661920,1403661951,A2 1403661952,1403661983,IR -1403661984,1403662015,DE +1403661984,1403662015,A2 1403662016,1403662047,IR -1403662048,1403662111,DE +1403662048,1403662111,A2 1403662112,1403662143,IR -1403662144,1403662175,DE +1403662144,1403662175,A2 1403662176,1403662207,AE -1403662208,1403662271,DE +1403662208,1403662271,A2 1403662272,1403662303,AE -1403662304,1403662495,DE +1403662304,1403662495,A2 1403662496,1403662527,IR -1403662528,1403662655,DE +1403662528,1403662655,A2 1403662656,1403662687,IR -1403662688,1403662815,DE +1403662688,1403662815,A2 1403662816,1403662911,IR -1403662912,1403662943,DE +1403662912,1403662943,A2 1403662944,1403662975,AE 1403662976,1403663039,IR -1403663040,1403663135,DE +1403663040,1403663135,A2 1403663136,1403663199,AE -1403663200,1403663359,DE +1403663200,1403663359,A2 1403663360,1403663487,IR -1403663488,1403663511,DE +1403663488,1403663511,A2 1403663512,1403663519,IR -1403663520,1403663527,DE +1403663520,1403663527,A2 1403663528,1403663535,IQ -1403663536,1403663567,DE +1403663536,1403663567,A2 1403663568,1403663575,AE -1403663576,1403663589,DE +1403663576,1403663589,A2 1403663590,1403663590,IR -1403663591,1403664919,DE +1403663591,1403664919,A2 1403664920,1403664927,TZ -1403664928,1403666431,DE +1403664928,1403665151,A2 +1403665152,1403665183,DE +1403665184,1403666431,A2 1403666432,1403682815,GB 1403682816,1403688959,A2 1403688960,1403692031,GB @@ -38224,8 +39294,10 @@ 1404803072,1404804095,LV 1404804096,1404805119,SE 1404805120,1404813311,AT -1404813312,1404815615,EE -1404815616,1404829695,SE +1404813312,1404815871,EE +1404815872,1404816383,LT +1404816384,1404821503,NL +1404821504,1404829695,SE 1404829696,1404870655,RU 1404870656,1404872703,LT 1404872704,1404874751,SE @@ -38275,7 +39347,9 @@ 1406716928,1406717439,AT 1406717440,1406717695,NL 1406717696,1406717951,DE -1406717952,1406719487,GB +1406717952,1406718015,GB +1406718016,1406718019,AT +1406718020,1406719487,GB 1406719488,1406719743,AT 1406719744,1406719999,GB 1406720000,1406721023,AT @@ -38406,10 +39480,22 @@ 1407516872,1407516879,A2 1407516880,1407516895,NG 1407516896,1407516911,A2 -1407516912,1407516927,NG -1407516928,1407516943,A2 -1407516944,1407516951,NG -1407516952,1407517183,A2 +1407516912,1407516951,NG +1407516952,1407516959,A2 +1407516960,1407516967,NG +1407516968,1407516983,A2 +1407516984,1407516991,KE +1407516992,1407516999,CD +1407517000,1407517023,A2 +1407517024,1407517031,GQ +1407517032,1407517039,NG +1407517040,1407517047,A2 +1407517048,1407517055,GQ +1407517056,1407517151,A2 +1407517152,1407517159,CD +1407517160,1407517167,ML +1407517168,1407517175,A2 +1407517176,1407517183,NG 1407517184,1407517311,GB 1407517312,1407517383,A2 1407517384,1407517391,CD @@ -38424,8 +39510,7 @@ 1407517952,1407518015,SL 1407518016,1407518031,ZA 1407518032,1407518039,A2 -1407518040,1407518047,CD -1407518048,1407518055,A2 +1407518040,1407518055,CD 1407518056,1407518063,NG 1407518064,1407518079,A2 1407518080,1407518111,LR @@ -38441,8 +39526,7 @@ 1407518200,1407518207,AE 1407518208,1407518215,ZW 1407518216,1407518239,A2 -1407518240,1407518247,NG -1407518248,1407518255,A2 +1407518240,1407518255,NG 1407518256,1407518263,CD 1407518264,1407518269,GH 1407518270,1407518335,A2 @@ -38481,7 +39565,7 @@ 1407519160,1407519167,AO 1407519168,1407519175,A2 1407519176,1407519183,SL -1407519184,1407519191,NG +1407519184,1407519191,UG 1407519192,1407519199,CD 1407519200,1407519207,A2 1407519208,1407519215,TG @@ -38499,39 +39583,42 @@ 1407519816,1407519823,NG 1407519824,1407519831,A2 1407519832,1407519839,SD -1407519840,1407519847,A2 +1407519840,1407519847,ML 1407519848,1407519855,NG 1407519856,1407519863,GN 1407519864,1407519871,NG 1407519872,1407519879,IL -1407519880,1407519895,NG +1407519880,1407519887,NG +1407519888,1407519895,CD 1407519896,1407519903,GN 1407519904,1407519911,NG 1407519912,1407519919,GN 1407519920,1407519959,A2 1407519960,1407519967,GB 1407519968,1407519983,NG -1407519984,1407519991,A2 -1407519992,1407519999,NG -1407520000,1407520007,CD -1407520008,1407520023,NG +1407519984,1407519991,SD +1407519992,1407520023,NG 1407520024,1407520031,A2 1407520032,1407520039,GN -1407520040,1407520071,A2 +1407520040,1407520047,A2 +1407520048,1407520055,GB +1407520056,1407520071,A2 1407520072,1407520079,CM 1407520080,1407520087,NG -1407520088,1407520103,GN +1407520088,1407520103,A2 1407520104,1407520111,SD -1407520112,1407520119,GN +1407520112,1407520119,A2 1407520120,1407520127,NE -1407520128,1407520151,GN +1407520128,1407520135,GN +1407520136,1407520143,A2 +1407520144,1407520151,GN 1407520152,1407520152,A2 1407520153,1407520159,GN 1407520160,1407520167,IL 1407520168,1407520175,GN 1407520176,1407520207,A2 1407520208,1407520215,NG -1407520216,1407520223,NE +1407520216,1407520223,CD 1407520224,1407520231,SD 1407520232,1407520271,A2 1407520272,1407520279,NG @@ -38560,15 +39647,14 @@ 1407520600,1407520607,A2 1407520608,1407520615,AO 1407520616,1407520623,A2 -1407520624,1407520631,NG -1407520632,1407520639,A2 -1407520640,1407520655,NG +1407520624,1407520655,NG 1407520656,1407520663,GA 1407520664,1407520687,A2 1407520688,1407520695,NG 1407520696,1407520711,A2 1407520712,1407520719,NG -1407520720,1407520735,A2 +1407520720,1407520727,A2 +1407520728,1407520735,AO 1407520736,1407520743,LR 1407520744,1407520751,NG 1407520752,1407520759,A2 @@ -38578,23 +39664,23 @@ 1407520784,1407520791,SD 1407520792,1407520799,ZA 1407520800,1407520807,A2 -1407520808,1407520815,CI -1407520816,1407520823,GQ +1407520808,1407520815,MU +1407520816,1407520823,A2 1407520824,1407520831,CI 1407520832,1407520839,SO 1407520840,1407520847,A2 1407520848,1407520855,ZM 1407520856,1407520863,A2 1407520864,1407520871,TZ -1407520872,1407520879,CD -1407520880,1407520895,A2 +1407520872,1407520887,CD +1407520888,1407520895,TZ 1407520896,1407520903,AO 1407520904,1407520911,NG 1407520912,1407520919,ZW 1407520920,1407520927,IQ 1407520928,1407520935,A2 1407520936,1407520943,CD -1407520944,1407520951,A2 +1407520944,1407520951,TZ 1407520952,1407520959,ZM 1407520960,1407520967,NG 1407520968,1407520975,A2 @@ -38606,7 +39692,7 @@ 1407521048,1407521055,NG 1407521056,1407521063,KE 1407521064,1407521071,NG -1407521072,1407521079,A2 +1407521072,1407521079,TZ 1407521080,1407521087,UG 1407521088,1407521095,NG 1407521096,1407521103,A2 @@ -38623,7 +39709,7 @@ 1407521184,1407521199,CD 1407521200,1407521207,A2 1407521208,1407521215,IQ -1407521216,1407521223,A2 +1407521216,1407521223,MU 1407521224,1407521231,TZ 1407521232,1407521239,LR 1407521240,1407521247,A2 @@ -38642,12 +39728,8 @@ 1407522384,1407522391,A2 1407522392,1407522407,NG 1407522408,1407522415,ZW -1407522416,1407522423,NG -1407522424,1407522431,A2 -1407522432,1407522439,IQ -1407522440,1407522440,NG -1407522441,1407522447,A2 -1407522448,1407522455,UG +1407522416,1407522439,A2 +1407522440,1407522455,UG 1407522456,1407522463,ML 1407522464,1407522471,A2 1407522472,1407522479,LR @@ -38655,17 +39737,19 @@ 1407522488,1407522495,ZW 1407522496,1407522503,LR 1407522504,1407522511,ZW -1407522512,1407522527,A2 +1407522512,1407522519,UG +1407522520,1407522527,A2 1407522528,1407522535,ZM -1407522536,1407522551,NG -1407522552,1407522559,UG +1407522536,1407522543,NG +1407522544,1407522559,UG 1407522560,1407522567,MZ 1407522568,1407522575,NG 1407522576,1407522583,A2 1407522584,1407522591,GL -1407522592,1407522599,A2 +1407522592,1407522599,UG 1407522600,1407522607,ZM -1407522608,1407522623,A2 +1407522608,1407522615,AO +1407522616,1407522623,A2 1407522624,1407522631,NG 1407522632,1407522639,MW 1407522640,1407522647,GQ @@ -38686,21 +39770,27 @@ 1407522768,1407522775,A2 1407522776,1407522791,NG 1407522792,1407522799,A2 -1407522800,1407522807,ZM +1407522800,1407522807,NG 1407522808,1407522815,A2 1407522816,1407522823,CD 1407522824,1407522831,NG 1407522832,1407522839,MZ -1407522840,1407522855,A2 +1407522840,1407522847,NG +1407522848,1407522855,TZ 1407522856,1407522863,ZW -1407522864,1407522871,A2 +1407522864,1407522871,TZ 1407522872,1407522879,IQ -1407522880,1407522903,A2 +1407522880,1407522895,A2 +1407522896,1407522903,UG 1407522904,1407522911,KE 1407522912,1407522919,CG -1407522920,1407522943,A2 +1407522920,1407522927,A2 +1407522928,1407522935,GR +1407522936,1407522943,A2 1407522944,1407522951,CD -1407522952,1407522991,A2 +1407522952,1407522967,ZW +1407522968,1407522975,MU +1407522976,1407522991,A2 1407522992,1407522999,CD 1407523000,1407523007,A2 1407523008,1407523015,NG @@ -38715,15 +39805,16 @@ 1407523136,1407523143,BJ 1407523144,1407523159,A2 1407523160,1407523167,LR -1407523168,1407523183,A2 +1407523168,1407523175,A2 +1407523176,1407523183,UG 1407523184,1407523191,NG 1407523192,1407523199,A2 1407523200,1407523207,NG -1407523208,1407523215,A2 +1407523208,1407523215,SD 1407523216,1407523223,BI 1407523224,1407523247,AO -1407523248,1407523263,A2 -1407523264,1407523271,NG +1407523248,1407523255,A2 +1407523256,1407523271,NG 1407523272,1407523279,US 1407523280,1407523287,MZ 1407523288,1407523295,NG @@ -38741,14 +39832,16 @@ 1407523536,1407523551,NG 1407523552,1407523559,CM 1407523560,1407523567,KE -1407523568,1407523839,A2 +1407523568,1407523583,A2 +1407523584,1407523591,IQ +1407523592,1407523839,A2 1407523840,1407524351,MW 1407524352,1407524607,GB 1407524608,1407524615,ZW 1407524616,1407524623,CM -1407524624,1407524631,A2 +1407524624,1407524631,ZA 1407524632,1407524639,NG -1407524640,1407524647,A2 +1407524640,1407524647,KE 1407524648,1407524655,CD 1407524656,1407524663,NG 1407524664,1407524671,CD @@ -38767,7 +39860,8 @@ 1407524840,1407524847,LR 1407524848,1407524855,A2 1407524856,1407524863,CD -1407524864,1407524895,A2 +1407524864,1407524887,A2 +1407524888,1407524895,TZ 1407524896,1407524903,LR 1407524904,1407524935,A2 1407524936,1407524943,GR @@ -38775,14 +39869,14 @@ 1407524952,1407524959,CM 1407524960,1407524967,TZ 1407524968,1407524975,NG -1407524976,1407524983,A2 +1407524976,1407524983,CD 1407524984,1407524991,ZA 1407524992,1407524999,TZ 1407525000,1407525007,A2 1407525008,1407525015,TZ 1407525016,1407525023,BI 1407525024,1407525031,NG -1407525032,1407525039,A2 +1407525032,1407525039,IL 1407525040,1407525047,ZA 1407525048,1407525055,NG 1407525056,1407525063,ZA @@ -38793,10 +39887,10 @@ 1407525096,1407525103,A2 1407525104,1407525111,NG 1407525112,1407525119,ZA -1407525120,1407525127,NG +1407525120,1407525127,A2 1407525128,1407525135,SD 1407525136,1407525143,CD -1407525144,1407525151,A2 +1407525144,1407525151,TZ 1407525152,1407525167,CD 1407525168,1407525175,A2 1407525176,1407525183,CD @@ -38815,14 +39909,15 @@ 1407525384,1407525391,CD 1407525392,1407525399,A2 1407525400,1407525407,KE -1407525408,1407525471,A2 +1407525408,1407525415,UG +1407525416,1407525423,A2 +1407525424,1407525431,UG +1407525432,1407525471,A2 1407525472,1407525479,UG 1407525480,1407525487,CD 1407525488,1407525543,A2 1407525544,1407525551,KE -1407525552,1407525583,A2 -1407525584,1407525591,GL -1407525592,1407526711,A2 +1407525552,1407526711,A2 1407526712,1407526719,GN 1407526720,1407529023,A2 1407529024,1407529087,NG @@ -38830,7 +39925,8 @@ 1407529104,1407529111,NG 1407529112,1407529143,A2 1407529144,1407529151,ZW -1407529152,1407529183,A2 +1407529152,1407529175,A2 +1407529176,1407529183,SO 1407529184,1407529191,NG 1407529192,1407529207,A2 1407529208,1407529215,NG @@ -38838,7 +39934,7 @@ 1407531008,1407531519,NG 1407531520,1407531551,CD 1407531552,1407531559,SD -1407531560,1407531567,NG +1407531560,1407531567,A2 1407531568,1407531575,CD 1407531576,1407531583,A2 1407531584,1407531591,CD @@ -38851,8 +39947,7 @@ 1407531656,1407531663,CD 1407531664,1407531671,MZ 1407531672,1407531687,CD -1407531688,1407531695,A2 -1407531696,1407531703,CD +1407531688,1407531703,A2 1407531704,1407531711,IQ 1407531712,1407531719,A2 1407531720,1407531735,CD @@ -38905,7 +40000,7 @@ 1407534336,1407534343,CM 1407534344,1407534359,A2 1407534360,1407534383,NG -1407534384,1407534391,A2 +1407534384,1407534391,GR 1407534392,1407534399,AO 1407534400,1407534407,NG 1407534408,1407534415,CI @@ -38928,7 +40023,7 @@ 1407535616,1407535623,CD 1407535624,1407535631,A2 1407535632,1407535639,SD -1407535640,1407535647,A2 +1407535640,1407535647,CD 1407535648,1407535655,NG 1407535656,1407535663,GB 1407535664,1407535671,A2 @@ -38936,18 +40031,20 @@ 1407535680,1407535687,GN 1407535688,1407535695,A2 1407535696,1407535703,GN -1407535704,1407535711,A2 +1407535704,1407535711,CD 1407535712,1407535719,GB 1407535720,1407535735,A2 1407535736,1407535743,NG -1407535744,1407535775,A2 +1407535744,1407535751,SD +1407535752,1407535759,GB +1407535760,1407535775,A2 1407535776,1407535783,GN 1407535784,1407535815,A2 1407535816,1407535823,NG 1407535824,1407535831,GB 1407535832,1407535847,A2 1407535848,1407535855,CD -1407535856,1407535863,NG +1407535856,1407535863,SD 1407535864,1407536127,A2 1407536128,1407536639,GA 1407536640,1407536895,CD @@ -38971,16 +40068,18 @@ 1407537112,1407537119,LR 1407537120,1407537135,A2 1407537136,1407537143,NG -1407537144,1407537183,A2 +1407537144,1407537151,A2 +1407537152,1407537159,UG +1407537160,1407537183,A2 1407537184,1407537191,NG 1407537192,1407537207,A2 1407537208,1407537215,BE 1407537216,1407537239,A2 1407537240,1407537255,NG 1407537256,1407537287,A2 -1407537288,1407537295,NG -1407537296,1407537303,UG -1407537304,1407537319,A2 +1407537288,1407537303,UG +1407537304,1407537311,LR +1407537312,1407537319,A2 1407537320,1407537327,NG 1407537328,1407537343,A2 1407537344,1407537351,NG @@ -38991,7 +40090,8 @@ 1407537408,1407537415,A2 1407537416,1407537423,GH 1407537424,1407537431,A2 -1407537432,1407537455,NG +1407537432,1407537447,NG +1407537448,1407537455,UG 1407537456,1407537463,AO 1407537464,1407537479,A2 1407537480,1407537487,CM @@ -39048,14 +40148,13 @@ 1407538144,1407538151,GN 1407538152,1407538167,A2 1407538168,1407538175,IQ -1407538176,1407538207,A2 -1407538208,1407538223,NG -1407538224,1407538263,A2 -1407538264,1407538271,NG +1407538176,1407538199,A2 +1407538200,1407538271,NG 1407538272,1407538279,CO -1407538280,1407538295,A2 +1407538280,1407538287,A2 +1407538288,1407538295,NG 1407538296,1407538303,AO -1407538304,1407538311,A2 +1407538304,1407538311,NG 1407538312,1407538319,CD 1407538320,1407538327,A2 1407538328,1407538335,CD @@ -39065,7 +40164,9 @@ 1407538384,1407538391,CM 1407538392,1407538407,A2 1407538408,1407538415,NG -1407538416,1407538439,A2 +1407538416,1407538423,A2 +1407538424,1407538431,NG +1407538432,1407538439,A2 1407538440,1407538447,NG 1407538448,1407538463,A2 1407538464,1407538471,ZM @@ -39102,21 +40203,26 @@ 1407539712,1407539719,BE 1407539720,1407539727,A2 1407539728,1407539735,GR -1407539736,1407539751,A2 +1407539736,1407539743,NG +1407539744,1407539751,A2 1407539752,1407539759,NG -1407539760,1407539767,A2 +1407539760,1407539767,GH 1407539768,1407539775,GR 1407539776,1407539783,A2 1407539784,1407539791,GR 1407539792,1407539799,NG -1407539800,1407539831,A2 -1407539832,1407539839,NG -1407539840,1407539863,A2 +1407539800,1407539823,A2 +1407539824,1407539839,NG +1407539840,1407539847,A2 +1407539848,1407539855,NG +1407539856,1407539863,CD 1407539864,1407539879,NG -1407539880,1407539911,A2 -1407539912,1407539919,NG -1407539920,1407539935,A2 -1407539936,1407539943,CM +1407539880,1407539895,A2 +1407539896,1407539903,CD +1407539904,1407539911,A2 +1407539912,1407539927,NG +1407539928,1407539935,GN +1407539936,1407539943,A2 1407539944,1407539951,CD 1407539952,1407539967,A2 1407539968,1407539975,CD @@ -39182,7 +40288,17 @@ 1407543256,1407543263,NG 1407543264,1407543279,A2 1407543280,1407543287,NG -1407543288,1407546367,A2 +1407543288,1407545855,A2 +1407545856,1407545863,CI +1407545864,1407545895,A2 +1407545896,1407545903,CD +1407545904,1407545951,A2 +1407545952,1407545959,NG +1407545960,1407545967,A2 +1407545968,1407545975,GQ +1407545976,1407545983,A2 +1407545984,1407545991,MU +1407545992,1407546367,A2 1407546368,1407546495,AE 1407546496,1407546799,A2 1407546800,1407546815,GH @@ -39194,7 +40310,8 @@ 1407547152,1407547167,A2 1407547168,1407547175,NG 1407547176,1407547183,GH -1407547184,1407547199,NG +1407547184,1407547191,UG +1407547192,1407547199,NG 1407547200,1407547207,AO 1407547208,1407547215,TD 1407547216,1407547239,A2 @@ -39228,19 +40345,35 @@ 1407548416,1407548543,GE 1407548544,1407548711,A2 1407548712,1407548719,CD -1407548720,1407549183,A2 +1407548720,1407548927,A2 +1407548928,1407548943,NG +1407548944,1407548967,A2 +1407548968,1407548975,CM +1407548976,1407548983,NG +1407548984,1407548991,CD +1407548992,1407549039,A2 +1407549040,1407549047,NG +1407549048,1407549055,A2 +1407549056,1407549063,NG +1407549064,1407549071,A2 +1407549072,1407549079,NG +1407549080,1407549127,A2 +1407549128,1407549135,NG +1407549136,1407549183,A2 1407549184,1407549439,GE 1407549440,1407582207,RU 1407582208,1407614975,PL 1407614976,1407680511,ES 1407680512,1407680591,FR 1407680592,1407680607,GB -1407680608,1407680623,FR -1407680624,1407681279,GB -1407681280,1407681287,ES -1407681288,1407681343,GB -1407681344,1407681407,ES -1407681408,1407681535,GB +1407680608,1407680639,FR +1407680640,1407681023,GB +1407681024,1407681087,ES +1407681088,1407681279,GB +1407681280,1407681291,ES +1407681292,1407681295,GB +1407681296,1407681503,ES +1407681504,1407681535,GB 1407681536,1407681639,FR 1407681640,1407681983,GB 1407681984,1407681999,ES @@ -39304,7 +40437,13 @@ 1407778816,1407844351,SE 1407844352,1407909887,RU 1407909888,1407975423,GR -1407975424,1408040959,DE +1407975424,1408013647,DE +1408013648,1408013655,A2 +1408013656,1408015312,DE +1408015313,1408015313,A2 +1408015314,1408035046,DE +1408035047,1408035047,A2 +1408035048,1408040959,DE 1408040960,1408106495,RU 1408106496,1408172031,PL 1408172032,1408237567,RU @@ -39313,9 +40452,17 @@ 1408303104,1408335871,LV 1408335872,1408336879,SE 1408336880,1408336887,FI -1408336888,1408340047,SE +1408336888,1408337695,SE +1408337696,1408337703,DK +1408337704,1408338807,SE +1408338808,1408338815,DK +1408338816,1408338959,SE +1408338960,1408338967,NO +1408338968,1408340047,SE 1408340048,1408340055,NO -1408340056,1408360935,SE +1408340056,1408353815,SE +1408353816,1408353823,DK +1408353824,1408360935,SE 1408360936,1408360943,DK 1408360944,1408368639,SE 1408368640,1408376831,NO @@ -39370,36 +40517,39 @@ 1409548288,1409810431,FR 1409810432,1409941503,GB 1409941504,1410007039,PL -1410007040,1410007551,DE +1410007040,1410007551,A2 1410007552,1410007807,IR -1410007808,1410008575,DE +1410007808,1410008575,A2 1410008576,1410008607,AE -1410008608,1410008831,DE +1410008608,1410008831,A2 1410008832,1410008911,US -1410008912,1410009343,DE +1410008912,1410009343,A2 1410009344,1410009479,US -1410009480,1410009487,DE +1410009480,1410009487,A2 1410009488,1410009535,GI -1410009536,1410010367,DE +1410009536,1410010367,A2 1410010368,1410010399,IR 1410010400,1410010431,LY -1410010432,1410010543,DE +1410010432,1410010543,A2 1410010544,1410010575,LY -1410010576,1410010599,DE +1410010576,1410010599,A2 1410010600,1410010607,IR 1410010608,1410010623,LY -1410010624,1410010687,DE +1410010624,1410010639,DE +1410010640,1410010687,A2 1410010688,1410010703,IR -1410010704,1410010719,DE +1410010704,1410010719,A2 1410010720,1410010751,LY -1410010752,1410010823,DE +1410010752,1410010823,A2 1410010824,1410010831,IR -1410010832,1410010879,DE +1410010832,1410010879,A2 1410010880,1410011135,IR -1410011136,1410012159,DE +1410011136,1410011647,DE +1410011648,1410012159,A2 1410012160,1410012415,IQ 1410012416,1410012447,AF -1410012448,1410013183,DE +1410012448,1410012671,A2 +1410012672,1410013183,DE 1410013184,1410013439,ES 1410013440,1410013471,IR 1410013472,1410013535,AE @@ -39410,35 +40560,35 @@ 1410013696,1410013727,KW 1410013728,1410013759,AE 1410013760,1410013791,IR -1410013792,1410013823,DE +1410013792,1410013823,A2 1410013824,1410013887,AF -1410013888,1410013919,DE +1410013888,1410013919,A2 1410013920,1410013951,KW 1410013952,1410013983,AE 1410013984,1410014015,IR 1410014016,1410014047,KW 1410014048,1410014079,AE -1410014080,1410014207,DE +1410014080,1410014207,A2 1410014208,1410014239,AE 1410014240,1410014335,IR 1410014336,1410014399,AE 1410014400,1410014463,ES 1410014464,1410014495,KW 1410014496,1410014591,IR -1410014592,1410014885,DE +1410014592,1410014885,A2 1410014886,1410014886,ES -1410014887,1410014895,DE +1410014887,1410014895,A2 1410014896,1410014927,IR 1410014928,1410014935,PA 1410014936,1410014943,IR 1410014944,1410014951,AE -1410014952,1410014966,DE +1410014952,1410014966,A2 1410014967,1410014967,IR -1410014968,1410014968,DE +1410014968,1410014968,A2 1410014969,1410014969,IR -1410014970,1410014972,DE +1410014970,1410014972,A2 1410014973,1410014974,AE -1410014975,1410015007,DE +1410014975,1410015007,A2 1410015008,1410015103,IR 1410015104,1410015135,ES 1410015136,1410015263,IR @@ -39454,114 +40604,120 @@ 1410016384,1410016415,IR 1410016416,1410016447,PA 1410016448,1410016543,IR -1410016544,1410016575,DE +1410016544,1410016575,A2 1410016576,1410016607,AE 1410016608,1410016639,PA 1410016640,1410016671,IR 1410016672,1410016703,PA -1410016704,1410016767,DE +1410016704,1410016767,A2 1410016768,1410016831,IR 1410016832,1410016863,AE 1410016864,1410016895,IR 1410016896,1410017023,AE 1410017024,1410017055,PA 1410017056,1410017119,IR -1410017120,1410017279,DE +1410017120,1410017279,A2 1410017280,1410017407,IR -1410017408,1410017791,DE +1410017408,1410017791,A2 1410017792,1410018047,IR -1410018048,1410018303,DE +1410018048,1410018303,A2 1410018304,1410018559,AE -1410018560,1410018591,DE +1410018560,1410018591,A2 1410018592,1410018607,IR 1410018608,1410018623,LY -1410018624,1410020103,DE +1410018624,1410018815,A2 +1410018816,1410018831,DE +1410018832,1410018847,A2 +1410018848,1410018863,DE +1410018864,1410020103,A2 1410020104,1410020111,US -1410020112,1410020351,DE +1410020112,1410020351,A2 1410020352,1410020863,US -1410020864,1410021375,DE +1410020864,1410021375,A2 1410021376,1410021407,IR -1410021408,1410021631,DE +1410021408,1410021631,A2 1410021632,1410021663,IR -1410021664,1410021791,DE +1410021664,1410021791,A2 1410021792,1410021823,IR -1410021824,1410022527,DE +1410021824,1410022527,A2 1410022528,1410022591,AE 1410022592,1410022655,IR -1410022656,1410022911,DE +1410022656,1410022911,A2 1410022912,1410023423,IR -1410023424,1410024287,DE +1410023424,1410024287,A2 1410024288,1410024319,IR -1410024320,1410024447,DE +1410024320,1410024447,A2 1410024448,1410025087,IR -1410025088,1410025215,DE +1410025088,1410025215,A2 1410025216,1410025247,ES -1410025248,1410025279,DE +1410025248,1410025279,A2 1410025280,1410025311,AE 1410025312,1410025343,ES 1410025344,1410025407,IR 1410025408,1410025439,AE -1410025440,1410025503,DE +1410025440,1410025503,A2 1410025504,1410025519,AE 1410025520,1410025535,IR 1410025536,1410025567,ES -1410025568,1410025599,DE +1410025568,1410025599,A2 1410025600,1410025727,IR 1410025728,1410025759,AE -1410025760,1410025791,DE +1410025760,1410025791,A2 1410025792,1410025823,AE 1410025824,1410025855,IR -1410025856,1410025983,DE +1410025856,1410025983,A2 1410025984,1410026015,IR 1410026016,1410026047,AE 1410026048,1410026111,ES -1410026112,1410026143,DE +1410026112,1410026143,A2 1410026144,1410026175,IR 1410026176,1410026207,AE 1410026208,1410026239,IR 1410026240,1410026367,AE 1410026368,1410026431,IR -1410026432,1410026463,DE +1410026432,1410026463,A2 1410026464,1410026751,IR 1410026752,1410026815,CA -1410026816,1410026847,DE +1410026816,1410026847,A2 1410026848,1410026911,IR -1410026912,1410026943,DE +1410026912,1410026943,A2 1410026944,1410027006,IR -1410027007,1410027007,DE +1410027007,1410027007,A2 1410027008,1410027263,KW 1410027264,1410027519,AF -1410027520,1410027711,DE +1410027520,1410027711,A2 1410027712,1410027775,IR -1410027776,1410028799,DE +1410027776,1410028799,A2 1410028800,1410028831,IR -1410028832,1410035327,DE +1410028832,1410035327,A2 1410035328,1410035343,IR -1410035344,1410035983,DE +1410035344,1410035983,A2 1410035984,1410035999,PA -1410036000,1410036031,DE +1410036000,1410036031,A2 1410036032,1410036063,PA -1410036064,1410036111,DE +1410036064,1410036111,A2 1410036112,1410036127,PA -1410036128,1410036191,DE +1410036128,1410036191,A2 1410036192,1410036207,PA -1410036208,1410036735,DE +1410036208,1410036735,A2 1410036736,1410036751,PA -1410036752,1410036991,DE +1410036752,1410036991,A2 1410036992,1410037247,LB -1410037248,1410037759,DE +1410037248,1410037759,A2 1410037760,1410038015,US -1410038016,1410039807,DE +1410038016,1410039807,A2 1410039808,1410041855,AE -1410041856,1410042815,DE +1410041856,1410042815,A2 1410042816,1410042831,US -1410042832,1410043903,DE +1410042832,1410043903,A2 1410043904,1410044415,JO -1410044416,1410044927,DE +1410044416,1410044927,A2 1410044928,1410045183,VG -1410045184,1410045439,DE +1410045184,1410045439,IQ 1410045440,1410045695,LB -1410045696,1410072319,DE +1410045696,1410071815,A2 +1410071816,1410071839,DE +1410071840,1410072319,A2 1410072320,1410072575,AE 1410072576,1410203647,GB 1410203648,1410204439,FR @@ -39574,7 +40730,9 @@ 1410234840,1410234847,A2 1410234848,1410250551,FR 1410250552,1410250559,GB -1410250560,1410261007,FR +1410250560,1410258527,FR +1410258528,1410258535,ES +1410258536,1410261007,FR 1410261008,1410261015,GB 1410261016,1410261663,FR 1410261664,1410261671,GB @@ -39653,7 +40811,9 @@ 1410573776,1410573799,DE 1410573800,1410573815,IT 1410573816,1410573823,RU -1410573824,1410574527,DE +1410573824,1410574511,DE +1410574512,1410574519,CH +1410574520,1410574527,DE 1410574528,1410574543,IT 1410574544,1410574559,DE 1410574560,1410574575,RU @@ -39682,9 +40842,7 @@ 1410575840,1410575855,IT 1410575856,1410575871,DE 1410575872,1410575999,DK -1410576000,1410576063,DE -1410576064,1410576127,IT -1410576128,1410576255,DE +1410576000,1410576255,DE 1410576256,1410576383,IT 1410576384,1410588671,DE 1410588672,1410596863,GB @@ -39874,16 +41032,16 @@ 1411923968,1411940351,BG 1411940352,1411973119,PL 1411973120,1411999743,SI -1411999744,1411999759,BA -1411999760,1411999767,SI -1411999768,1411999783,BA +1411999744,1411999783,BA 1411999784,1411999791,SI 1411999792,1411999799,BA 1411999800,1411999807,SI 1411999808,1411999847,BA 1411999848,1411999871,SI 1411999872,1411999887,BA -1411999888,1411999999,SI +1411999888,1411999903,SI +1411999904,1411999943,BA +1411999944,1411999999,SI 1412000000,1412000767,BA 1412000768,1412000783,SI 1412000784,1412000791,BA @@ -39893,7 +41051,9 @@ 1412000904,1412000911,BA 1412000912,1412000943,SI 1412000944,1412000991,BA -1412000992,1412001311,SI +1412000992,1412001007,SI +1412001008,1412001015,BA +1412001016,1412001311,SI 1412001312,1412001319,DE 1412001320,1412001807,SI 1412001808,1412001855,RO @@ -39923,8 +41083,8 @@ 1412003272,1412003303,BA 1412003304,1412003327,SI 1412003328,1412003351,BA -1412003352,1412003359,SI -1412003360,1412003383,BA +1412003352,1412003375,SI +1412003376,1412003383,BA 1412003384,1412003391,SI 1412003392,1412003407,BA 1412003408,1412003423,SI @@ -39934,10 +41094,12 @@ 1412003552,1412003583,SI 1412003584,1412003599,BA 1412003600,1412003623,SI -1412003624,1412003783,BA +1412003624,1412003647,BA +1412003648,1412003711,SI +1412003712,1412003783,BA 1412003784,1412003791,SI -1412003792,1412003839,BA -1412003840,1412003855,SI +1412003792,1412003823,BA +1412003824,1412003855,SI 1412003856,1412003903,BG 1412003904,1412003935,SI 1412003936,1412004351,BG @@ -40034,11 +41196,7 @@ 1422403840,1422403903,IT 1422403904,1422406399,DE 1422406400,1422406463,GB -1422406464,1422452495,DE -1422452496,1422452503,CA -1422452504,1422462207,DE -1422462208,1422462463,TR -1422462464,1422468207,DE +1422406464,1422468207,DE 1422468208,1422468223,IT 1422468224,1422468671,DE 1422468672,1422468735,IT @@ -40136,8 +41294,8 @@ 1424595820,1424595823,GB 1424595824,1424595831,IT 1424595832,1424595839,GB -1424595840,1424595903,IT -1424595904,1424595935,GB +1424595840,1424595919,IT +1424595920,1424595935,GB 1424595936,1424596479,IT 1424596480,1424596991,FR 1424596992,1424597255,CZ @@ -40190,9 +41348,7 @@ 1424603136,1424603391,SK 1424603392,1424603647,DE 1424603648,1424603903,GB -1424603904,1424604067,ES -1424604068,1424604071,GB -1424604072,1424604159,ES +1424603904,1424604159,ES 1424604160,1424604543,NL 1424604544,1424604671,GB 1424604672,1424604799,NL @@ -40213,15 +41369,7 @@ 1424605696,1424605951,NL 1424605952,1424605959,CH 1424605960,1424606023,AT -1424606024,1424606199,GB -1424606200,1424606207,AT -1424606208,1424606455,GB -1424606456,1424606463,AT -1424606464,1424606711,GB -1424606712,1424606719,AT -1424606720,1424606967,GB -1424606968,1424606975,AT -1424606976,1424607167,GB +1424606024,1424607167,GB 1424607168,1424607199,SK 1424607200,1424607215,GB 1424607216,1424607223,SK @@ -40234,9 +41382,7 @@ 1424608128,1424608279,GB 1424608280,1424608383,FR 1424608384,1424608399,ES -1424608400,1424608407,FR -1424608408,1424608415,GB -1424608416,1424608511,FR +1424608400,1424608511,FR 1424608512,1424608687,ES 1424608688,1424608695,GB 1424608696,1424609023,ES @@ -40273,9 +41419,7 @@ 1424614400,1424614415,FR 1424614416,1424614431,IT 1424614432,1424614439,GB -1424614440,1424614559,IT -1424614560,1424614591,GB -1424614592,1424614655,IT +1424614440,1424614655,IT 1424614656,1424614911,FR 1424614912,1424615167,RO 1424615168,1424615679,GB @@ -40313,7 +41457,9 @@ 1424618048,1424618239,IT 1424618240,1424618495,NL 1424618496,1424618751,GB -1424618752,1424619007,FR +1424618752,1424618927,FR +1424618928,1424618943,IT +1424618944,1424619007,FR 1424619008,1424619775,IT 1424619776,1424619807,BE 1424619808,1424619815,GB @@ -40416,7 +41562,9 @@ 1425326080,1425342463,SE 1425342464,1425358847,NL 1425358848,1425375231,SK -1425375232,1425391615,DE +1425375232,1425377339,DE +1425377340,1425377340,A2 +1425377341,1425391615,DE 1425391616,1425407999,LV 1425408000,1425424383,NL 1425424384,1425424543,A2 @@ -40762,7 +41910,9 @@ 1426751488,1426767871,FR 1426767872,1426778991,DE 1426778992,1426778999,IT -1426779000,1426784255,DE +1426779000,1426781655,DE +1426781656,1426781663,IT +1426781664,1426784255,DE 1426784256,1426800639,SE 1426800640,1426817023,PL 1426817024,1426833407,BG @@ -40889,12 +42039,10 @@ 1427729312,1427729343,GR 1427729344,1427742719,DE 1427742720,1427742751,IO -1427742752,1427742879,DE -1427742880,1427742911,TR -1427742912,1427742975,DE +1427742752,1427742975,DE 1427742976,1427743007,RU 1427743008,1427743039,US -1427743040,1427743071,BG +1427743040,1427743071,DE 1427743072,1427743103,CY 1427743104,1427743199,DE 1427743200,1427743231,TR @@ -40910,7 +42058,7 @@ 1427743616,1427743647,DK 1427743648,1427743775,DE 1427743776,1427743807,ZA -1427743808,1427743839,DE +1427743808,1427743839,US 1427743840,1427743871,RU 1427743872,1427743935,DE 1427743936,1427743967,RU @@ -40927,8 +42075,7 @@ 1427744416,1427744447,UA 1427744448,1427744639,DE 1427744640,1427744671,TR -1427744672,1427744703,DE -1427744704,1427744735,CA +1427744672,1427744735,DE 1427744736,1427744767,DK 1427744768,1427744863,DE 1427744864,1427744927,TR @@ -40938,9 +42085,7 @@ 1427745056,1427745151,DE 1427745152,1427745183,RO 1427745184,1427745215,CH -1427745216,1427745247,DE -1427745248,1427745279,TR -1427745280,1427745439,DE +1427745216,1427745439,DE 1427745440,1427745471,US 1427745472,1427745503,DE 1427745504,1427745535,RU @@ -40962,13 +42107,15 @@ 1427746240,1427746271,US 1427746272,1427747839,DE 1427747840,1427747871,FI -1427747872,1427747967,DE -1427747968,1427747999,BG -1427748000,1427748063,DE +1427747872,1427747935,DE +1427747936,1427747967,US +1427747968,1427748063,DE 1427748064,1427748095,US 1427748096,1427748255,DE 1427748256,1427748287,US -1427748288,1427748447,DE +1427748288,1427748351,DE +1427748352,1427748383,US +1427748384,1427748447,DE 1427748448,1427748479,RU 1427748480,1427748511,TR 1427748512,1427748543,DE @@ -40983,8 +42130,7 @@ 1427749888,1427749919,BR 1427749920,1427749951,DE 1427749952,1427749983,RU -1427749984,1427750015,BG -1427750016,1427750079,DE +1427749984,1427750079,DE 1427750080,1427750111,BG 1427750112,1427750239,DE 1427750240,1427750271,US @@ -41003,7 +42149,7 @@ 1427760416,1427760575,DE 1427760576,1427760607,CH 1427760608,1427760799,DE -1427760800,1427760831,NL +1427760800,1427760831,US 1427760832,1427760959,DE 1427760960,1427760991,CZ 1427760992,1427761023,BG @@ -41015,7 +42161,7 @@ 1427761216,1427761247,TR 1427761248,1427761279,DE 1427761280,1427761311,RU -1427761312,1427761343,DE +1427761312,1427761343,US 1427761344,1427761375,CY 1427761376,1427761407,TW 1427761408,1427761503,DE @@ -41023,7 +42169,8 @@ 1427761536,1427761567,US 1427761568,1427761599,RU 1427761600,1427761631,EG -1427761632,1427767295,DE +1427761632,1427761663,TR +1427761664,1427767295,DE 1427767296,1427800063,BE 1427800064,1427832831,RU 1427832832,1427865599,BE @@ -41197,9 +42344,7 @@ 1432322048,1432338431,RU 1432338432,1432346623,FR 1432346624,1433403391,TR -1433403392,1433406255,ES -1433406256,1433406271,US -1433406272,1433406431,ES +1433403392,1433406431,ES 1433406432,1433406447,US 1433406448,1433407487,ES 1433407488,1433410047,NL @@ -41302,7 +42447,9 @@ 1433862976,1433862991,GB 1433862992,1433863055,CH 1433863056,1433863071,GB -1433863072,1433863423,CH +1433863072,1433863095,CH +1433863096,1433863103,US +1433863104,1433863423,CH 1433863424,1433863487,US 1433863488,1433864191,CH 1433864192,1433866239,HU @@ -41648,7 +42795,7 @@ 1439154176,1439170559,GB 1439170560,1439236095,NO 1439236096,1439301631,BE -1439301632,1439305727,RU +1439301632,1439305727,CZ 1439305728,1439309823,DK 1439309824,1439318015,PL 1439318016,1439322111,DK @@ -41686,40 +42833,30 @@ 1439482368,1439482879,GB 1439482880,1439498239,RO 1439498240,1439503103,DE -1439503104,1439504127,GB -1439504128,1439504383,DE -1439504384,1439514623,GB +1439503104,1439503359,PT +1439503360,1439514111,DE +1439514112,1439514367,GB +1439514368,1439514623,DE 1439514624,1439516671,IT -1439516672,1439517439,DE -1439517440,1439517695,GB -1439517696,1439518207,DE +1439516672,1439518207,DE 1439518208,1439518719,IT -1439518720,1439520767,GB -1439520768,1439522815,DE -1439522816,1439528959,GB -1439528960,1439529215,DE -1439529216,1439529471,GB -1439529472,1439529599,DE -1439529600,1439529983,GB +1439518720,1439529599,DE +1439529600,1439529727,GB +1439529728,1439529983,DE 1439529984,1439530239,EG 1439530240,1439535103,DE 1439535104,1439538175,IT -1439538176,1439538687,GB +1439538176,1439538687,DE 1439538688,1439539199,IT 1439539200,1439549439,DE 1439549440,1439551487,IT -1439551488,1439555839,DE -1439555840,1439556095,GB +1439551488,1439556095,DE 1439556096,1439556607,IT -1439556608,1439557119,DE -1439557120,1439557375,GB -1439557376,1439557887,DE -1439557888,1439558143,GB +1439556608,1439558143,DE 1439558144,1439558911,IT -1439558912,1439559423,GB -1439559424,1439560191,DE +1439558912,1439560191,DE 1439560192,1439560447,IT -1439560448,1439560703,GB +1439560448,1439560703,DE 1439560704,1439561215,IT 1439561216,1439561727,DE 1439561728,1439562239,GB @@ -43220,14 +44357,15 @@ 1467369520,1467369535,RU 1467369536,1467369599,DE 1467369600,1467369663,IT -1467369664,1467369727,AT -1467369728,1467369759,DE +1467369664,1467369759,DE 1467369760,1467369791,HR 1467369792,1467369855,SE 1467369856,1467369871,RU 1467369872,1467369887,DE 1467369888,1467369903,RU -1467369904,1467383807,DE +1467369904,1467369919,DE +1467369920,1467369951,SG +1467369952,1467383807,DE 1467383808,1467400191,BG 1467400192,1467416575,RU 1467416576,1467432959,PL @@ -43277,7 +44415,11 @@ 1472264808,1472266239,IE 1472266240,1472304610,DE 1472304611,1472304611,A2 -1472304612,1472331775,DE +1472304612,1472314335,DE +1472314336,1472314343,A2 +1472314344,1472330699,DE +1472330700,1472330700,A2 +1472330701,1472331775,DE 1472331776,1472397311,GB 1472397312,1472462847,NL 1472462848,1472528383,PT @@ -43311,7 +44453,11 @@ 1474944536,1474953215,NL 1474953216,1474966473,DE 1474966474,1474966474,A2 -1474966475,1475018751,DE +1474966475,1474968895,DE +1474968896,1474968903,A2 +1474968904,1475009582,DE +1475009583,1475009583,A2 +1475009584,1475018751,DE 1475018752,1475084287,ES 1475084288,1475086335,NL 1475086336,1475092479,RU @@ -43515,7 +44661,9 @@ 1475362816,1475379199,FR 1475379200,1475395583,RU 1475395584,1475411967,LU -1475411968,1475412471,IT +1475411968,1475412383,IT +1475412384,1475412391,ES +1475412392,1475412471,IT 1475412472,1475412479,ES 1475412480,1475417975,IT 1475417976,1475417983,A2 @@ -43976,13 +45124,15 @@ 1489731584,1489764351,BG 1489764352,1489797119,RU 1489797120,1489829887,KZ -1489829888,1489855519,DE +1489829888,1489855503,DE +1489855504,1489855519,RU 1489855520,1489855535,PL 1489855536,1489855543,DE 1489855544,1489855551,PL 1489855552,1489855999,DE 1489856000,1489856031,PL -1489856032,1489862655,DE +1489856032,1489856063,RU +1489856064,1489862655,DE 1489862656,1489928191,RU 1489928192,1489960959,SE 1489960960,1489993727,HR @@ -43990,8 +45140,8 @@ 1490026496,1490028543,US 1490028544,1490029055,UA 1490029056,1490040847,NL -1490040848,1490041855,UA -1490041856,1490042879,NL +1490040848,1490041599,UA +1490041600,1490042879,NL 1490042880,1490049879,CZ 1490049880,1490049887,AT 1490049888,1490049919,CZ @@ -44049,7 +45199,11 @@ 1490518016,1490534399,RU 1490534400,1490550783,DE 1490550784,1490616319,LT -1490616320,1490681855,DE +1490616320,1490680643,DE +1490680644,1490680647,A2 +1490680648,1490680663,DE +1490680664,1490680667,A2 +1490680668,1490681855,DE 1490681856,1490747391,GR 1490747392,1490812927,FR 1490812928,1490878463,PL @@ -44095,7 +45249,6 @@ 1494294528,1494302719,RU 1494302720,1494310911,FI 1494310912,1494319103,LB -1494319104,1494327295,MT 1494327296,1494335487,IT 1494335488,1494343679,ES 1494343680,1494351871,PL @@ -44227,7 +45380,6 @@ 1495209984,1495212031,KZ 1495212032,1495214079,RU 1495214080,1495216127,CZ -1495216128,1495218175,GB 1495218176,1495220223,IT 1495220224,1495222271,PL 1495222272,1495224319,IT @@ -44601,13 +45753,19 @@ 1503789056,1503821823,NO 1503821824,1503854591,UA 1503854592,1503887359,RU -1503887360,1503895631,DE +1503887360,1503895599,DE +1503895600,1503895607,BE +1503895608,1503895631,DE 1503895632,1503895639,AT 1503895640,1503895647,GR 1503895648,1503895671,DE 1503895672,1503895679,PL 1503895680,1503895687,IT -1503895688,1503895767,DE +1503895688,1503895695,DE +1503895696,1503895703,GB +1503895704,1503895751,DE +1503895752,1503895759,IT +1503895760,1503895767,DE 1503895768,1503895783,IT 1503895784,1503895799,DE 1503895800,1503895807,CA @@ -44624,7 +45782,9 @@ 1503896544,1503896551,FR 1503896552,1503897303,DE 1503897304,1503897311,BE -1503897312,1503897367,DE +1503897312,1503897335,DE +1503897336,1503897343,PT +1503897344,1503897367,DE 1503897368,1503897375,BE 1503897376,1503897383,GR 1503897384,1503897407,DE @@ -44701,9 +45861,7 @@ 1504152128,1504152191,IE 1504152192,1504152415,GB 1504152416,1504152431,IE -1504152432,1504152447,GB -1504152448,1504152575,IE -1504152576,1504154623,GB +1504152432,1504154623,GB 1504154624,1504155647,IE 1504155648,1504156927,GB 1504156928,1504157183,IE @@ -44745,11 +45903,7 @@ 1505271808,1505273087,NL 1505273088,1505273095,NZ 1505273096,1505279999,NL -1505280000,1505280007,IR -1505280008,1505284095,AE -1505284096,1505284607,IR -1505284608,1505284863,AE -1505284864,1505288191,IR +1505280000,1505288191,IR 1505288192,1505296383,RU 1505296384,1505304575,UA 1505304576,1505305351,FR @@ -44835,7 +45989,10 @@ 1505452104,1505452111,US 1505452112,1505452327,GB 1505452328,1505452335,NO -1505452336,1505453439,GB +1505452336,1505453167,GB +1505453168,1505453175,SE +1505453176,1505453183,US +1505453184,1505453439,GB 1505453440,1505453567,US 1505453568,1505454367,GB 1505454368,1505454375,US @@ -44865,7 +46022,9 @@ 1505455760,1505455767,DE 1505455768,1505455791,GB 1505455792,1505455799,US -1505455800,1505456127,GB +1505455800,1505455999,GB +1505456000,1505456063,US +1505456064,1505456127,GB 1505456128,1505456255,US 1505456256,1505456639,GB 1505456640,1505456895,US @@ -44982,7 +46141,8 @@ 1506438864,1506438871,DE 1506438872,1506438879,FR 1506438880,1506438883,US -1506438884,1506438895,DE +1506438884,1506438887,DE +1506438888,1506438895,NL 1506438896,1506438911,CH 1506438912,1506439039,US 1506439040,1506439463,DE @@ -45058,11 +46218,11 @@ 1506444288,1506445311,DE 1506445312,1506445337,FR 1506445338,1506445343,GB -1506445344,1506445519,FR +1506445344,1506445407,FR +1506445408,1506445439,GB +1506445440,1506445519,FR 1506445520,1506445527,GB -1506445528,1506445566,FR -1506445567,1506445567,GB -1506445568,1506445703,FR +1506445528,1506445703,FR 1506445704,1506445711,NL 1506445712,1506445719,GB 1506445720,1506445759,FR @@ -45083,14 +46243,12 @@ 1506448648,1506448663,AT 1506448664,1506448671,GB 1506448672,1506448703,AT -1506448704,1506448727,GB -1506448728,1506448735,AT -1506448736,1506448895,GB +1506448704,1506448895,GB 1506448896,1506449159,BE 1506449160,1506449171,GB -1506449172,1506449247,BE -1506449248,1506449279,GB -1506449280,1506449407,BE +1506449172,1506449255,BE +1506449256,1506449263,GB +1506449264,1506449407,BE 1506449408,1506449663,NL 1506449664,1506449919,SK 1506449920,1506449927,CH @@ -45111,11 +46269,11 @@ 1506451056,1506451059,DK 1506451060,1506451199,GB 1506451200,1506451791,ES -1506451792,1506451807,GB -1506451808,1506451895,ES +1506451792,1506451799,GB +1506451800,1506451895,ES 1506451896,1506451903,GB -1506451904,1506452127,ES -1506452128,1506452159,GB +1506451904,1506452143,ES +1506452144,1506452159,GB 1506452160,1506452223,ES 1506452224,1506452479,GB 1506452480,1506452735,US @@ -45190,15 +46348,15 @@ 1506465792,1506466047,DE 1506466048,1506466303,BE 1506466304,1506466559,DE -1506466560,1506467071,GB +1506466560,1506466627,GB +1506466628,1506466631,NL +1506466632,1506467071,GB 1506467072,1506467327,DE 1506467328,1506467583,GB 1506467584,1506468351,IT 1506468352,1506468607,GB 1506468608,1506468863,TZ -1506468864,1506469047,IT -1506469048,1506469055,GB -1506469056,1506470143,IT +1506468864,1506470143,IT 1506470144,1506470399,GB 1506470400,1506470655,DE 1506470656,1506471679,IT @@ -45214,13 +46372,13 @@ 1506472448,1506472703,GB 1506472704,1506473215,IT 1506473216,1506473471,GB -1506473472,1506474271,IT -1506474272,1506474303,SE -1506474304,1506474495,IT +1506473472,1506474247,IT +1506474248,1506474271,GB +1506474272,1506474495,IT 1506474496,1506474751,FR 1506474752,1506475519,IT -1506475520,1506475551,AT -1506475552,1506475775,GB +1506475520,1506475559,AT +1506475560,1506475775,GB 1506475776,1506476031,DE 1506476032,1506508799,KW 1506508800,1506541567,CZ @@ -45299,7 +46457,9 @@ 1507664384,1507664767,DE 1507664768,1507664895,US 1507664896,1507665407,GR -1507665408,1507665919,IT +1507665408,1507665663,IT +1507665664,1507665791,TZ +1507665792,1507665919,IT 1507665920,1507666431,US 1507666432,1507666559,GR 1507666560,1507666591,FR @@ -45370,8 +46530,8 @@ 1508647808,1508648447,SE 1508648448,1508648703,DK 1508648704,1508650751,SE -1508650752,1508650847,DK -1508650848,1508650879,SE +1508650752,1508650855,DK +1508650856,1508650879,SE 1508650880,1508651263,DK 1508651264,1508652543,SE 1508652544,1508654079,DK @@ -45454,9 +46614,7 @@ 1509507072,1509511167,GB 1509511168,1509515263,LT 1509515264,1509519359,HR -1509519360,1509531647,RU -1509531648,1509535743,GB -1509535744,1509539839,RU +1509519360,1509539839,RU 1509539840,1509543935,AM 1509543936,1509543975,LB 1509543976,1509543983,DE @@ -45834,7 +46992,8 @@ 1533505536,1533507583,DE 1533507584,1533509631,UA 1533509632,1533511679,GB -1533511680,1533513023,FR +1533511680,1533511935,IT +1533511936,1533513023,FR 1533513024,1533513087,ES 1533513088,1533513215,FR 1533513216,1533513471,GB @@ -45995,7 +47154,7 @@ 1534715792,1534715807,ES 1534715808,1534715875,FR 1534715876,1534715879,NL -1534715880,1534715883,CH +1534715880,1534715883,PT 1534715884,1534715887,PL 1534715888,1534715919,FR 1534715920,1534715935,ES @@ -46066,7 +47225,8 @@ 1534717324,1534717343,FR 1534717344,1534717359,GB 1534717360,1534717375,PL -1534717376,1534717519,FR +1534717376,1534717503,FR +1534717504,1534717519,BE 1534717520,1534717535,DE 1534717536,1534717551,PL 1534717552,1534717567,CZ @@ -46173,7 +47333,8 @@ 1534719632,1534719639,ES 1534719640,1534719647,FR 1534719648,1534719655,ES -1534719656,1534719679,FR +1534719656,1534719663,FR +1534719664,1534719679,IE 1534719680,1534719695,GB 1534719696,1534719699,IT 1534719700,1534719703,DE @@ -46211,7 +47372,10 @@ 1534720240,1534720255,BE 1534720256,1534720271,NL 1534720272,1534720287,PL -1534720288,1534720351,FR +1534720288,1534720291,IT +1534720292,1534720295,PL +1534720296,1534720299,GB +1534720300,1534720351,FR 1534720352,1534720367,ES 1534720368,1534720383,PL 1534720384,1534720431,FR @@ -46292,9 +47456,7 @@ 1534721320,1534721327,PT 1534721328,1534721343,NL 1534721344,1534721359,PL -1534721360,1534721367,CH -1534721368,1534721371,GB -1534721372,1534721375,DE +1534721360,1534721375,GB 1534721376,1534721391,ES 1534721392,1534721407,PL 1534721408,1534721439,DE @@ -46334,7 +47496,7 @@ 1534721968,1534721971,DE 1534721972,1534721975,PL 1534721976,1534721979,ES -1534721980,1534721983,GB +1534721980,1534721983,FR 1534721984,1534722007,PL 1534722008,1534722011,PT 1534722012,1534722015,ES @@ -46519,11 +47681,9 @@ 1536655360,1536659455,GE 1536659456,1536659759,DE 1536659760,1536659775,BI -1536659776,1536659791,DE +1536659776,1536659791,AO 1536659792,1536659823,SD -1536659824,1536659839,DE -1536659840,1536659903,LB -1536659904,1536659991,DE +1536659824,1536659991,DE 1536659992,1536659999,EG 1536660000,1536660007,SO 1536660008,1536660015,DE @@ -46535,8 +47695,7 @@ 1536660736,1536660991,SA 1536660992,1536661247,DE 1536661248,1536661759,GQ -1536661760,1536662015,DE -1536662016,1536662271,LB +1536661760,1536662271,DE 1536662272,1536662303,SO 1536662304,1536662335,LB 1536662336,1536662367,DE @@ -48735,7 +49894,7 @@ 1540589568,1540593663,RU 1540593664,1540594687,GB 1540594688,1540595711,IT -1540595712,1540596735,KZ +1540595712,1540596735,UZ 1540596736,1540597759,FR 1540597760,1540598783,SE 1540598784,1540600831,UA @@ -48765,7 +49924,6 @@ 1540621824,1540622335,RU 1540622336,1540622591,KW 1540622592,1540622847,PL -1540622848,1540623103,RU 1540623104,1540623359,SE 1540623360,1540623615,GB 1540623616,1540623871,BG @@ -49426,7 +50584,7 @@ 1540910848,1540911103,SE 1540911104,1540911359,GB 1540911360,1540911615,NL -1540911616,1540912127,RU +1540911616,1540911871,RU 1540912128,1540912383,GB 1540912384,1540912639,PT 1540912640,1540912895,DK @@ -49496,7 +50654,6 @@ 1540930816,1540931071,KZ 1540931072,1540931327,EU 1540931328,1540931583,PL -1540931584,1540931839,RU 1540931840,1540932095,GB 1540932096,1540932351,PL 1540932352,1540932607,FR @@ -49743,7 +50900,6 @@ 1541027840,1541028863,RU 1541028864,1541029887,PL 1541029888,1541030911,UA -1541030912,1541031935,DE 1541031936,1541032959,UA 1541032960,1541033983,PL 1541033984,1541035007,BG @@ -50075,7 +51231,9 @@ 1541208320,1541208575,NL 1541208576,1541208831,BY 1541208832,1541209087,HU -1541209088,1541210111,UA +1541209088,1541209599,UA +1541209600,1541209855,SE +1541209856,1541210111,UA 1541210112,1541210623,RU 1541210624,1541211135,CZ 1541211136,1541211647,SK @@ -50189,6 +51347,7 @@ 1541270528,1541271039,IT 1541271040,1541271551,FI 1541271552,1541272063,RO +1541272064,1541272575,RU 1541272576,1541273087,FR 1541273088,1541274111,RU 1541274112,1541274623,DE @@ -50648,6 +51807,107 @@ 1541536768,1541537791,RU 1541537792,1541538303,GB 1541538304,1541538815,RO +1541538816,1541539327,UA +1541539328,1541539839,PL +1541539840,1541540351,HR +1541540352,1541541375,RU +1541541376,1541541887,UA +1541541888,1541542399,PL +1541542400,1541542911,FR +1541542912,1541543423,MK +1541543424,1541543935,PL +1541543936,1541544447,GB +1541544448,1541544959,PL +1541544960,1541545471,FR +1541545472,1541545983,RU +1541545984,1541546495,LV +1541546496,1541547007,UA +1541547008,1541547519,SE +1541547520,1541548543,PL +1541548544,1541549567,UA +1541549568,1541550079,DE +1541550592,1541551103,PL +1541551104,1541552127,RO +1541552128,1541553151,UA +1541553152,1541555199,RU +1541555200,1541556223,PL +1541556224,1541556479,UA +1541556480,1541556735,RU +1541556736,1541557247,IT +1541557248,1541557503,RU +1541557504,1541557759,SI +1541557760,1541558015,RU +1541558016,1541558271,HU +1541558272,1541559295,RU +1541559296,1541560319,RO +1541560320,1541561343,DE +1541561344,1541562879,RU +1541562880,1541563135,FR +1541563136,1541563391,NL +1541563392,1541564415,PL +1541564416,1541565439,RU +1541565440,1541565951,IT +1541565952,1541566463,PL +1541566464,1541567487,SK +1541567488,1541567743,RU +1541567744,1541567999,PL +1541568000,1541568511,SE +1541568512,1541569535,RU +1541569536,1541570559,CZ +1541570560,1541571583,RU +1541571584,1541572607,UA +1541572608,1541574655,RU +1541574656,1541575167,DK +1541575168,1541575423,PL +1541575424,1541575679,IT +1541575680,1541577727,RU +1541577728,1541578751,AT +1541578752,1541579007,GB +1541579008,1541579263,RU +1541579264,1541579775,DE +1541579776,1541580799,RU +1541580800,1541581311,KZ +1541581312,1541581567,HR +1541581568,1541582847,RU +1541582848,1541583359,ES +1541583360,1541583615,RU +1541583616,1541583871,GE +1541583872,1541584127,SE +1541584384,1541584895,BE +1541584896,1541585151,DE +1541585152,1541585663,RU +1541585664,1541585919,UA +1541585920,1541586431,RU +1541586432,1541587199,UA +1541587200,1541587455,FR +1541587456,1541588991,PL +1541588992,1541589247,RU +1541589248,1541589503,UA +1541589504,1541590015,CH +1541590016,1541590527,RU +1541590528,1541590783,DK +1541590784,1541591039,UA +1541591040,1541592063,RU +1541592064,1541592575,UA +1541592576,1541593087,FR +1541593088,1541594111,CZ +1541594112,1541594367,EU +1541594368,1541595135,RU +1541595136,1541595647,IR +1541595648,1541596159,BG +1541596160,1541597695,PL +1541597696,1541597951,RU +1541598208,1541599231,PL +1541599232,1541600255,RS +1541600256,1541600511,HR +1541600512,1541600767,IL +1541600768,1541601023,SI +1541601024,1541601279,RU +1541601280,1541601791,DE +1541601792,1541602047,RU +1541602048,1541602303,PL +1541602304,1541603327,UA +1541603328,1541604351,PL 1543503872,1545601023,GB 1545601024,1545673167,SE 1545673168,1545673175,FI @@ -50742,9 +52002,7 @@ 1546323968,1546325503,GE 1546325504,1546325759,SE 1546325760,1546326015,NO -1546326016,1546327295,ES -1546327296,1546327551,FR -1546327552,1546328063,ES +1546326016,1546328063,ES 1546328064,1546330111,CZ 1546330112,1546332159,SE 1546332160,1546334207,GB @@ -51031,7 +52289,13 @@ 1551892480,1556086783,FR 1556086784,1556486924,DE 1556486925,1556486925,A2 -1556486926,1558708223,DE +1556486926,1556490839,DE +1556490840,1556490847,A2 +1556490848,1556493903,DE +1556493904,1556493911,A2 +1556493912,1556497655,DE +1556497656,1556497663,A2 +1556497664,1558708223,DE 1558708224,1559232511,GB 1559232512,1559240703,IL 1559240704,1559248895,BA @@ -51236,8 +52500,7 @@ 1566400640,1566400671,NO 1566400672,1566400703,DE 1566400704,1566400735,NL -1566400736,1566400767,DE -1566400768,1566400832,NO +1566400736,1566400832,NO 1566400833,1566400895,GB 1566400896,1566401023,NO 1566401024,1566401055,US @@ -51356,7 +52619,9 @@ 1566568448,1566570495,KZ 1566570496,1566570943,NL 1566570944,1566570951,DE -1566570952,1566572543,NL +1566570952,1566571479,NL +1566571480,1566571483,DE +1566571484,1566572543,NL 1566572544,1566703615,GB 1566703616,1566769151,SA 1566769152,1566773759,CZ @@ -51439,7 +52704,9 @@ 1570439168,1570471935,TR 1570471936,1570480895,BG 1570480896,1570481151,MK -1570481152,1570504703,BG +1570481152,1570492159,BG +1570492160,1570492415,ES +1570492416,1570504703,BG 1570504704,1570570239,ES 1570570240,1570572287,NL 1570572288,1570574335,UA @@ -51611,7 +52878,8 @@ 1571429376,1571432447,UA 1571432448,1571434495,CZ 1571434496,1571435519,UA -1571435520,1571438591,CZ +1571435520,1571435775,NL +1571435776,1571438591,CZ 1571438592,1571441663,UA 1571441664,1571442175,KZ 1571442176,1571442687,NL @@ -51710,7 +52978,9 @@ 1572225024,1572241407,TR 1572241408,1572257791,SE 1572257792,1572274175,FR -1572274176,1572282111,SG +1572274176,1572276223,SG +1572276224,1572277247,US +1572277248,1572282111,SG 1572282112,1572282367,PT 1572282368,1572290559,SG 1572290560,1572306943,RU @@ -52038,7 +53308,8 @@ 1578590472,1578590479,DE 1578590480,1578590495,IT 1578590496,1578590527,PL -1578590528,1578590543,FR +1578590528,1578590535,ES +1578590536,1578590543,FR 1578590544,1578590559,GB 1578590560,1578590575,ES 1578590576,1578590603,FR @@ -52148,7 +53419,8 @@ 1578592200,1578592207,CH 1578592208,1578592223,FR 1578592224,1578592239,ES -1578592240,1578592275,FR +1578592240,1578592271,FR +1578592272,1578592275,PL 1578592276,1578592279,FI 1578592280,1578592283,BE 1578592284,1578592287,NL @@ -52181,13 +53453,11 @@ 1578592736,1578592743,PL 1578592744,1578592747,PT 1578592748,1578592783,DE -1578592784,1578592799,FR -1578592800,1578592815,GB -1578592816,1578592831,FR +1578592784,1578592831,FR 1578592832,1578592847,PL 1578592848,1578592851,BE 1578592852,1578592855,DE -1578592856,1578592859,FR +1578592856,1578592859,CH 1578592860,1578592879,PL 1578592880,1578592883,FR 1578592884,1578592891,GB @@ -52204,7 +53474,8 @@ 1578593344,1578593359,FR 1578593360,1578593375,PL 1578593376,1578593407,GB -1578593408,1578593415,FR +1578593408,1578593411,IT +1578593412,1578593415,FR 1578593416,1578593423,PL 1578593424,1578593439,FR 1578593440,1578593455,PL @@ -52315,7 +53586,10 @@ 1578594776,1578594783,ES 1578594784,1578594799,FR 1578594800,1578594815,DE -1578594816,1578595039,FR +1578594816,1578594847,FR +1578594848,1578594863,IT +1578594864,1578594871,GB +1578594872,1578595039,FR 1578595040,1578595055,GB 1578595056,1578595103,FR 1578595104,1578595119,GB @@ -52369,7 +53643,8 @@ 1578595760,1578595763,FR 1578595764,1578595775,ES 1578595776,1578595807,CH -1578595808,1578595871,FR +1578595808,1578595855,FR +1578595856,1578595871,GB 1578595872,1578595907,PL 1578595908,1578595911,BE 1578595912,1578595935,FR @@ -52382,7 +53657,7 @@ 1578596100,1578596103,GB 1578596104,1578596111,FR 1578596112,1578596115,DE -1578596116,1578596119,NL +1578596116,1578596119,FR 1578596120,1578596123,PT 1578596124,1578596127,ES 1578596128,1578596143,PL @@ -52413,8 +53688,7 @@ 1578610768,1578610771,CH 1578610772,1578610775,GB 1578610776,1578610779,DE -1578610780,1578610783,FR -1578610784,1578610799,NL +1578610780,1578610799,FR 1578610800,1578610803,PL 1578610804,1578610807,GB 1578610808,1578610943,FR @@ -52447,13 +53721,13 @@ 1578611328,1578611343,BE 1578611344,1578611359,FR 1578611360,1578611375,CH -1578611376,1578611399,FR -1578611400,1578611403,BE -1578611404,1578611407,ES +1578611376,1578611391,FR +1578611392,1578611399,GB +1578611400,1578611407,ES 1578611408,1578611423,IT 1578611424,1578611439,GB 1578611440,1578611443,ES -1578611444,1578611447,FR +1578611444,1578611447,NL 1578611448,1578611455,ES 1578611456,1578611711,FR 1578611712,1578611775,CH @@ -52482,7 +53756,7 @@ 1578612136,1578612139,NL 1578612140,1578612143,ES 1578612144,1578612223,PL -1578612224,1578612239,GB +1578612224,1578612239,IT 1578612240,1578612255,FR 1578612256,1578612259,PL 1578612260,1578612263,IT @@ -52502,7 +53776,7 @@ 1578612904,1578612907,DE 1578612908,1578612911,FR 1578612912,1578612959,ES -1578612960,1578612975,GB +1578612960,1578612975,FR 1578612976,1578612983,IT 1578612984,1578612991,FR 1578612992,1578613247,DE @@ -52532,7 +53806,8 @@ 1578614000,1578614015,CH 1578614016,1578614031,FR 1578614032,1578614047,BE -1578614048,1578614071,FR +1578614048,1578614055,FI +1578614056,1578614071,FR 1578614072,1578614079,ES 1578614080,1578614175,FR 1578614176,1578614191,ES @@ -52551,11 +53826,9 @@ 1578991616,1579024383,KW 1579024384,1579057151,GB 1579057152,1579089919,LV -1579089920,1579090175,GB -1579090176,1579090575,NL -1579090576,1579090687,GB -1579090688,1579090943,NL -1579090944,1579091759,GB +1579089920,1579090431,GB +1579090432,1579090463,NL +1579090464,1579091759,GB 1579091760,1579091775,IL 1579091776,1579091839,GB 1579091840,1579091855,US @@ -52563,15 +53836,16 @@ 1579091968,1579092223,DE 1579092224,1579093759,GB 1579093760,1579094015,NL -1579094016,1579104511,GB -1579104512,1579104767,NL -1579104768,1579105023,GB +1579094016,1579094271,GB +1579094272,1579094527,NL +1579094528,1579094783,GB +1579094784,1579095039,NL +1579095040,1579098367,GB +1579098368,1579098623,NL +1579098624,1579105023,GB 1579105024,1579105087,NL 1579105088,1579105119,DE -1579105120,1579105151,NL -1579105152,1579105279,GB -1579105280,1579105343,NL -1579105344,1579105535,GB +1579105120,1579105535,GB 1579105536,1579105599,NL 1579105600,1579106303,GB 1579106304,1579122687,DE @@ -52588,7 +53862,9 @@ 1580064768,1580072959,DE 1580072960,1580085247,PT 1580085248,1580089343,MZ -1580089344,1580134399,PT +1580089344,1580104959,PT +1580104960,1580105215,CH +1580105216,1580134399,PT 1580134400,1580136447,ES 1580136448,1580138495,PT 1580138496,1580204031,IT @@ -52907,7 +54183,9 @@ 1585991168,1585991171,US 1585991172,1585991683,SE 1585991684,1585991687,US -1585991688,1585991935,SE +1585991688,1585991727,SE +1585991728,1585991731,RU +1585991732,1585991935,SE 1585991936,1585991939,US 1585991940,1585994543,SE 1585994544,1585994547,MY @@ -53113,7 +54391,6 @@ 1590132992,1590134783,GB 1590134784,1590136831,ES 1590136832,1590138879,GB -1590138880,1590140927,CZ 1590140928,1590142975,UA 1590142976,1590145023,AT 1590145024,1590147071,HU @@ -53237,7 +54514,9 @@ 1592524800,1592540415,GB 1592540416,1592540423,A2 1592540424,1592557567,GB -1592557568,1592590335,BG +1592557568,1592584447,BG +1592584448,1592584703,GB +1592584704,1592590335,BG 1592590336,1592623103,FI 1592623104,1592655871,RU 1592655872,1592786943,FR @@ -53385,7 +54664,9 @@ 1599129008,1599129011,EE 1599129012,1599133695,CZ 1599133696,1599133823,SK -1599133824,1599143935,CZ +1599133824,1599137055,CZ +1599137056,1599137071,SK +1599137072,1599143935,CZ 1599143936,1599160319,UA 1599160320,1599176703,IR 1599176704,1599188991,FR @@ -53474,7 +54755,8 @@ 1602252800,1602254847,SE 1602254848,1602255359,HU 1602255360,1602255615,US -1602255616,1602256895,HU +1602255616,1602255871,PT +1602255872,1602256895,HU 1602256896,1602258943,GB 1602258944,1602260991,RU 1602260992,1602263039,FR @@ -53560,10 +54842,10 @@ 1602392064,1602394111,GB 1602394112,1602396159,FR 1602396160,1602398207,DE -1602398208,1602398400,TR -1602398401,1602398406,US -1602398407,1602399231,TR -1602399232,1602399487,US +1602398208,1602399231,TR +1602399232,1602399359,US +1602399360,1602399360,TR +1602399361,1602399487,US 1602399488,1602400255,TR 1602400256,1602402303,RU 1602402304,1602404351,LU @@ -53641,7 +54923,11 @@ 1603080192,1603083007,DE 1603083008,1603083263,UA 1603083264,1603084287,DE -1603084288,1603088383,IT +1603084288,1603087515,IT +1603087516,1603087517,CH +1603087518,1603087544,IT +1603087545,1603087546,CH +1603087547,1603088383,IT 1603088384,1603092479,LB 1603092480,1603100671,NO 1603100672,1603108863,FR @@ -53666,7 +54952,9 @@ 1603153920,1603158015,NL 1603158016,1603159167,DE 1603159168,1603159183,NL -1603159184,1603159535,DE +1603159184,1603159487,DE +1603159488,1603159519,AT +1603159520,1603159535,DE 1603159536,1603159551,GB 1603159552,1603161007,DE 1603161008,1603161023,GB @@ -53724,7 +55012,8 @@ 1603224400,1603224415,IT 1603224416,1603224431,ES 1603224432,1603224447,US -1603224448,1603224575,GB +1603224448,1603224463,MX +1603224464,1603224575,GB 1603224576,1603224775,FR 1603224776,1603224783,GB 1603224784,1603224895,FR @@ -53881,8 +55170,8 @@ 1605125280,1605125335,GB 1605125336,1605125343,US 1605125344,1605125375,GB -1605125376,1605125887,US -1605125888,1605126143,GB +1605125376,1605125919,US +1605125920,1605126143,GB 1605126144,1605127679,US 1605127680,1605127935,GB 1605127936,1605128703,US @@ -54001,14 +55290,13 @@ 1607966720,1607967743,RU 1607967744,1607968767,UA 1607968768,1607969791,SE -1607969792,1607971839,RU +1607969792,1607970815,RU 1607972864,1607974911,NL 1607974912,1607975935,LV 1607976960,1607977983,KZ 1607979008,1607980031,RU 1607980032,1607981055,DE 1607981056,1607982079,UA -1607982080,1607983103,RU 1607983104,1607984127,GB 1607984128,1607985151,RU 1607985152,1607986175,PL @@ -54131,7 +55419,10 @@ 1614741504,1614757887,CA 1614757888,1614774271,US 1614774272,1614786559,CA -1614786560,1632305151,US +1614786560,1618886655,US +1618968576,1618984959,US +1618984960,1619001343,CA +1619001344,1632305151,US 1632305152,1632321535,CA 1632321536,1632354303,US 1632354304,1632354607,CA @@ -54766,7 +56057,9 @@ 1654554624,1654558719,CA 1654558720,1654583103,US 1654583104,1654583135,CA -1654583136,1654599455,US +1654583136,1654597823,US +1654597824,1654597887,GB +1654597888,1654599455,US 1654599456,1654599471,CA 1654599472,1654599487,US 1654599488,1654599519,BA @@ -54819,9 +56112,7 @@ 1673566976,1673567007,CA 1673567008,1673567087,US 1673567088,1673567167,GB -1673567168,1673567231,US -1673567232,1673567247,CA -1673567248,1673567263,US +1673567168,1673567263,US 1673567264,1673567279,AT 1673567280,1673567311,US 1673567312,1673567327,GB @@ -54864,9 +56155,7 @@ 1673572096,1673572351,LT 1673572352,1673572383,US 1673572384,1673572399,EC -1673572400,1673572415,US -1673572416,1673572431,CA -1673572432,1673572895,US +1673572400,1673572895,US 1673572896,1673572911,CA 1673572912,1673573183,US 1673573184,1673573247,CA @@ -54881,16 +56170,20 @@ 1673578288,1673578303,CA 1673578304,1673585343,US 1673585344,1673585407,SC -1673585408,1673986047,US +1673585408,1673590783,US +1673590784,1673590911,EC +1673590912,1673986047,US 1673986048,1674051583,CA 1674051584,1674575871,US 1674575872,1677721599,CA 1677721600,1681915903,US +1694498816,1694499839,CN 1694499840,1694500863,ID 1694500864,1694507007,JP 1694507008,1694515199,IN 1694515200,1694531583,AU 1694531584,1694564351,TW +1694564352,1694565375,CN 1694565376,1694566399,HK 1694566400,1694568447,KR 1694568448,1694572543,HK @@ -54900,6 +56193,7 @@ 1694662656,1694670847,JP 1694670848,1694672895,BD 1694672896,1694673919,AU +1694673920,1694674943,CN 1694674944,1694679039,LK 1694679040,1694695423,AU 1694695424,1694760959,TW @@ -54909,20 +56203,26 @@ 1697775616,1697776639,ID 1697776640,1697779711,JP 1697779712,1697783807,ID -1697783808,1697791999,JP +1697783808,1697789951,JP +1697789952,1697790975,CN +1697790976,1697791999,JP 1697792000,1697808383,PK 1697808384,1697841151,JP 1697841152,1697906687,TH 1697906688,1697972223,CN 1697972224,1697988607,VN 1697988608,1697996799,KR -1697996800,1698004991,JP +1697996800,1697997823,JP +1697997824,1697998847,CN +1697998848,1698004991,JP 1698004992,1698037759,AU 1698037760,1698103295,CN 1698103296,1698136063,KR -1698136064,1698168831,JP +1698136064,1698160639,JP +1698160640,1698162687,CN +1698162688,1698168831,JP 1698168832,1698693119,IN -1698693120,1699610623,CN +1698693120,1699611647,CN 1699611648,1699612671,JP 1699612672,1699614719,LA 1699614720,1699618815,PH @@ -54930,29 +56230,41 @@ 1699627008,1699643391,SG 1699643392,1699676159,HK 1699676160,1699741695,KR -1699741696,1700790271,CN +1699741696,1700793343,CN 1700793344,1700794367,VN +1700794368,1700798463,CN 1700798464,1700806655,JP 1700806656,1700823039,VN 1700823040,1700855807,CN +1700855808,1700921343,JP 1700921344,1700986879,NZ 1700986880,1701003263,VN 1701003264,1701011455,MY +1701011456,1701019647,CN 1701019648,1701052415,GU 1701052416,1701117951,NZ 1701117952,1701134335,NC +1701134336,1701142527,CN 1701142528,1701143551,HK +1701143552,1701150719,CN 1701150720,1701183487,KR -1701183488,1701249023,JP +1701183488,1701199871,JP +1701199872,1701208063,CN +1701208064,1701209087,JP +1701209088,1701216255,CN +1701216256,1701249023,JP 1701249024,1701314559,AU 1701314560,1701576703,CN 1701576704,1701707775,TH 1701707776,1701724159,JP +1701724160,1701736447,CN 1701736448,1701737471,NZ +1701737472,1701740543,CN 1701740544,1701838847,JP 1701838848,1702363135,AU -1702363136,1702887423,CN +1702363136,1702888447,CN 1702888448,1702889471,AU +1702889472,1702903807,CN 1702903808,1702920191,ID 1702920192,1702952959,JP 1702952960,1703411711,CN @@ -54963,33 +56275,143 @@ 1707081728,1707737087,CN 1707737088,1707802623,KR 1707802624,1707835391,JP +1707835392,1707845631,CN 1707845632,1707846655,ID +1707846656,1707851775,CN 1707851776,1707868159,JP 1707868160,1708130303,CN 1708130304,1709178879,IN 1709178880,1709834239,CN 1709834240,1709850623,SG +1709850624,1709852671,CN 1709852672,1709853695,AU +1709853696,1709867007,CN 1709867008,1709899775,AU 1709899776,1709965311,KR 1709965312,1710882815,CN 1710882816,1710948351,KR +1710948352,1710949375,CN 1710949376,1710950399,NP -1710964736,1711210495,CN +1710950400,1711210495,CN 1711210496,1711276031,ID +1728053248,1728120831,AU +1728120832,1728121855,CN +1728121856,1728123903,HK +1728123904,1728125951,CN +1728125952,1728126975,LA +1728126976,1728132095,HK +1728132096,1728135167,AU +1728135168,1728136191,HK +1728136192,1728137215,MY +1728137216,1728138239,CN +1728138240,1728139263,AU +1728139264,1728140287,IN +1728140288,1728141311,SG +1728141312,1728142335,CN +1728142336,1728143359,NP +1728143360,1728144383,MP +1728144384,1728145407,IN +1728145408,1728146431,MY +1728146432,1728147455,AU +1728147456,1728148479,IN +1728148480,1728149503,PH +1728149504,1728150527,JP +1728150528,1728152575,IN +1728152576,1728153599,MY +1728153600,1728154623,SG +1728154624,1728155647,JP +1728155648,1728158719,MY +1728158720,1728159743,HK +1728159744,1728161791,TH +1728161792,1728162815,CN +1728162816,1728163839,SG +1728163840,1728164863,LK +1728164864,1728165887,FJ +1728165888,1728168959,AU +1728168960,1728169983,IN +1728169984,1728171007,VN +1728171008,1728172031,AU +1728172032,1728173055,VN +1728173056,1728175103,AU +1728175104,1728177151,HK +1728177152,1728178175,AU +1728178176,1728179199,LA +1728179200,1728180223,VN +1728180224,1728181247,AU +1728181248,1728203775,JP +1728203776,1728204799,KR +1728204800,1728205823,IN +1728205824,1728206847,KR +1728206848,1728207871,SB +1728207872,1728210943,JP +1728210944,1728211967,SG +1728211968,1728212991,CN +1728212992,1728214015,TH +1728214016,1728215039,AU +1728215040,1728216063,NZ +1728216064,1728218111,JP +1728218112,1728219135,IN +1728219136,1728220159,JP +1728220160,1728221183,NZ +1728221184,1728222207,ID +1728222208,1728224255,LK +1728224256,1728225279,CN +1728225280,1728226303,JP +1728226304,1728227327,CN +1728227328,1728230399,AU +1728230400,1728231423,SG +1728231424,1728232447,NC +1728232448,1728235519,AU +1728235520,1728239615,CN +1728239616,1728240639,TW +1728240640,1728243711,VN +1728243712,1728246783,IN +1728246784,1728254975,JP +1728254976,1728255999,MY +1728256000,1728257023,HK +1728257024,1728258047,MN +1728258048,1728259071,IN +1728259072,1728260095,KR +1728260096,1728261119,IN +1728261120,1728262143,ID +1728262144,1728264191,JP +1728264192,1728265215,ID +1728265216,1728266239,SG +1728266240,1728267263,TH +1728267264,1728268287,ID +1728268288,1728269311,MY +1728269312,1728270335,ID +1728270336,1728271359,PH +1728271360,1728286719,CN +1728286720,1728287743,AU +1728287744,1728290815,CN +1728290816,1728291839,AU +1728291840,1728292863,SG +1728292864,1728293887,PG +1728293888,1728294911,MY +1778384896,1778385151,CN 1778385152,1778385407,AU +1778385408,1778393087,CN +1778393088,1778401279,AU +1778401280,1778417663,CN 1778417664,1778450431,TH 1778450432,1778515967,TW 1778515968,1779040255,CN +1779040256,1779073023,KR 1779073024,1779105791,SG 1779105792,1781727231,CN 1781727232,1781792767,IN -1782054912,1782579199,CN -1782972416,1783103487,AU +1781792768,1782579199,CN +1782579200,1782710271,TW +1782710272,1782841343,IN +1782841344,1783103487,AU +1783103488,1783234559,JP 1783234560,1783365631,CN +1783365632,1783627775,IN 1783627776,1784676351,CN 1784676352,1785200639,KR -1785724928,1786249215,CN +1785200640,1785462783,TW +1785462784,1786773503,CN 1786773504,1790967807,JP 1790967808,1793064959,IN 1793064960,1794113535,CN @@ -55112,8 +56534,8 @@ 1833332736,1833334783,CH 1833334784,1833336831,IT 1833336832,1833336959,ES -1833336960,1833336975,FR -1833336976,1833338879,ES +1833336960,1833337071,FR +1833337072,1833338879,ES 1833338880,1833342975,GB 1833342976,1833345023,CH 1833345024,1833347071,FI @@ -55209,7 +56631,9 @@ 1833541632,1833542143,IL 1833542144,1833542655,GB 1833542656,1833542911,IN -1833542912,1833545727,GB +1833542912,1833543168,GB +1833543169,1833543423,IN +1833543424,1833545727,GB 1833545728,1833549823,IT 1833549824,1833553919,RU 1833553920,1833558015,CZ @@ -55286,7 +56710,9 @@ 1834956800,1834960895,IR 1834960896,1834964991,RU 1834964992,1834967039,PL -1834967040,1834975231,RU +1834967040,1834971135,RU +1834971136,1834973183,PL +1834973184,1834975231,RU 1834975232,1834977279,IL 1834977280,1834983423,PL 1834983424,1834985471,RU @@ -55361,7 +56787,11 @@ 1835913216,1835917311,RU 1835917312,1835920127,GB 1835920128,1835920143,PT -1835920144,1835925503,GB +1835920144,1835920151,GB +1835920152,1835920159,PT +1835920160,1835920263,GB +1835920264,1835920279,PT +1835920280,1835925503,GB 1835925504,1835933695,LV 1835933696,1835941887,RU 1835941888,1835950079,UA @@ -55381,8 +56811,8 @@ 1836048472,1836056575,RS 1836056576,1836580863,IT 1836580864,1836597247,RU -1836597248,1836598015,LU -1836598016,1836606463,DE +1836597248,1836597759,LU +1836597760,1836606463,DE 1836606464,1836613631,LU 1836613632,1836630015,RU 1836630016,1836646399,BG @@ -55413,7 +56843,9 @@ 1837088768,1837105151,SI 1837105152,1838153727,BE 1838153728,1839202303,GB -1839202304,1839235071,BG +1839202304,1839218687,BG +1839218688,1839220735,ES +1839220736,1839235071,BG 1839235072,1839267839,IL 1839267840,1839300607,RU 1839300608,1839333375,BH @@ -55440,9 +56872,7 @@ 1839795152,1839795167,US 1839795168,1839796607,GB 1839796608,1839796671,US -1839796672,1839796735,GB -1839796736,1839796991,TR -1839796992,1839797759,GB +1839796672,1839797759,GB 1839797760,1839798015,GR 1839798016,1839798527,GB 1839798528,1839798559,US @@ -55768,7 +57198,8 @@ 1844125696,1844127743,NL 1844127744,1844129791,DE 1844129792,1844131583,NL -1844131584,1844131839,SC +1844131584,1844131711,SC +1844131712,1844131839,NL 1844131840,1844133887,DE 1844133888,1844135935,LT 1844135936,1844137983,NL @@ -55793,7 +57224,10 @@ 1844169520,1844169599,SE 1844169600,1844169647,US 1844169648,1844169655,AF -1844169656,1844169727,SE +1844169656,1844169663,SE +1844169664,1844169679,US +1844169680,1844169687,ZM +1844169688,1844169727,SE 1844169728,1844169767,DE 1844169768,1844169951,US 1844169952,1844169983,DE @@ -55801,7 +57235,9 @@ 1844169988,1844169991,DE 1844169992,1844169995,AF 1844169996,1844169999,TM -1844170000,1844170751,DE +1844170000,1844170007,DE +1844170008,1844170015,AF +1844170016,1844170751,DE 1844170752,1844174847,RU 1844174848,1844178943,DE 1844178944,1844180991,EE @@ -55813,25 +57249,7 @@ 1844181953,1844181958,GB 1844181959,1844181984,TR 1844181985,1844181990,GB -1844181991,1844182271,TR -1844182272,1844182302,US -1844182303,1844182309,TR -1844182310,1844182329,US -1844182330,1844182343,TR -1844182344,1844182403,US -1844182404,1844182416,TR -1844182417,1844182417,US -1844182418,1844182424,TR -1844182425,1844182426,US -1844182427,1844182432,TR -1844182433,1844182442,US -1844182443,1844182459,TR -1844182460,1844182480,US -1844182481,1844182485,TR -1844182486,1844182489,US -1844182490,1844182501,TR -1844182502,1844182511,US -1844182512,1844183039,TR +1844181991,1844183039,TR 1844183040,1844191231,IT 1844191232,1844195327,AL 1844195328,1844203519,RU @@ -55839,10 +57257,9 @@ 1844207616,1844211711,RU 1844211712,1844215807,SK 1844215808,1844219903,BE -1844219904,1844220031,A2 -1844220032,1844220191,IQ -1844220192,1844220287,DE -1844220288,1844220415,JO +1844219904,1844220159,A2 +1844220160,1844220191,IQ +1844220192,1844220415,DE 1844220416,1844220431,A2 1844220432,1844223743,DE 1844223744,1844223999,A2 @@ -56006,9 +57423,11 @@ 1850510336,1850511359,KR 1850511360,1850513407,ID 1850513408,1850514431,TH +1850514432,1850515455,CN 1850515456,1850519551,IN 1850519552,1850520575,AU 1850520576,1850521599,JP +1850521600,1850522623,CN 1850522624,1850523647,HK 1850523648,1850572799,CN 1850572800,1850671103,TH @@ -56029,6 +57448,7 @@ 1851591680,1851592703,ID 1851592704,1851594751,AU 1851594752,1851596799,KR +1851596800,1851604991,CN 1851604992,1851613183,PH 1851613184,1851617279,JP 1851617280,1851637759,KR @@ -56057,7 +57477,7 @@ 1856798720,1856815103,IN 1856815104,1856843775,CN 1856843776,1856847871,HK -1856856064,1856864255,CN +1856847872,1856864255,CN 1856864256,1856872447,AU 1856872448,1856876543,NZ 1856876544,1856880639,IN @@ -56222,7 +57642,7 @@ 1877707776,1877709823,AU 1877709824,1877710847,IN 1877710848,1877711871,HK -1877712896,1877721087,CN +1877711872,1877721087,CN 1877721088,1877737471,TW 1877737472,1877999615,JP 1877999616,1879048191,TW @@ -56280,7 +57700,9 @@ 1887993856,1888026623,KR 1888026624,1888030719,BD 1888030720,1888034815,HK -1888034816,1888059391,JP +1888034816,1888038911,JP +1888038912,1888040959,CN +1888040960,1888059391,JP 1888059392,1888063487,VN 1888063488,1888067583,JP 1888067584,1888071679,MY @@ -56400,6 +57822,7 @@ 1899364352,1899724799,CN 1899724800,1899741183,KR 1899741184,1899749375,LK +1899749376,1899750399,CN 1899750400,1899751423,JP 1899751424,1899753471,ID 1899753472,1899757567,HK @@ -56461,6 +57884,7 @@ 1909760000,1909762047,ID 1909762048,1909764095,AU 1909764096,1909766143,PH +1909766144,1909768191,CN 1909768192,1909784575,HK 1909784576,1909817343,CN 1909817344,1909850111,JP @@ -56511,6 +57935,7 @@ 1917190144,1917321215,KR 1917321216,1917779967,AU 1917779968,1917796351,ID +1917796352,1917812735,CN 1917812736,1917845503,IN 1917845504,1919680511,CN 1919680512,1919729663,KR @@ -57419,6 +58844,7 @@ 2013003776,2013011967,AU 2013011968,2013020159,JP 2013020160,2013028351,AU +2013028352,2013030399,CN 2013030400,2013032447,ID 2013032448,2013036543,FM 2013036544,2013038591,ID @@ -57896,6 +59322,7 @@ 2066882560,2066890751,TW 2066890752,2066907135,PF 2066907136,2066915327,AU +2066915328,2066923519,CN 2066923520,2066939903,JP 2066939904,2066972671,AU 2066972672,2067005439,TW @@ -59964,7 +61391,7 @@ 2425487360,2426667007,US 2426667008,2426732543,NO 2426732544,2426798079,FR -2426798080,2427256831,US +2426798080,2427207679,US 2427256832,2427322367,GB 2427322368,2427453439,US 2427453440,2427584511,NO @@ -60012,7 +61439,50 @@ 2447679820,2447679839,IT 2447679840,2447704063,DE 2447704064,2447769599,GB -2447769600,2447835135,DE +2447769600,2447769855,DE +2447769856,2447770111,GB +2447770112,2447775743,DE +2447775744,2447777791,GB +2447777792,2447781887,DE +2447781888,2447782911,GB +2447782912,2447783935,LU +2447783936,2447784959,GB +2447784960,2447785983,LU +2447785984,2447788031,DE +2447788032,2447790079,QA +2447790080,2447792127,DE +2447792128,2447792383,GB +2447792384,2447793919,DE +2447793920,2447794175,QA +2447794176,2447796223,DE +2447796224,2447798271,GB +2447798272,2447801343,DE +2447801344,2447802367,IT +2447802368,2447806463,DE +2447806464,2447808511,HU +2447808512,2447810559,DE +2447810560,2447812607,GB +2447812608,2447813631,HU +2447813632,2447813887,IE +2447813888,2447814143,DE +2447814144,2447814655,GB +2447814656,2447815679,DE +2447815680,2447816191,IT +2447816192,2447816447,GB +2447816448,2447816703,DE +2447816704,2447817727,GB +2447817728,2447821055,DE +2447821056,2447821311,QA +2447821312,2447821823,GB +2447821824,2447824895,IT +2447824896,2447826943,DE +2447826944,2447827967,GB +2447827968,2447830015,DE +2447830016,2447830527,IT +2447830528,2447833087,DE +2447833088,2447833599,GB +2447833600,2447834111,DE +2447834112,2447835135,IT 2447835136,2447900671,FR 2447900672,2447966207,CH 2447966208,2448031743,GB @@ -61519,8 +62989,14 @@ 2747465728,2748055551,ZA 2748055552,2748121087,CN 2748121088,2748317695,US -2748317696,2749628415,JP -2749628416,2749890559,US +2748317696,2748645375,JP +2748645376,2748710911,KR +2748710912,2749235199,JP +2749235200,2749300735,KR +2749300736,2749628415,JP +2749628416,2749693951,US +2749693952,2749759487,KR +2749759488,2749890559,US 2749890560,2750021631,AU 2750021632,2750349311,US 2750349312,2750414847,KR @@ -61528,7 +63004,8 @@ 2750873600,2750939135,CL 2750939136,2751070207,US 2751070208,2751135743,CL -2751135744,2751463423,US +2751135744,2751397887,US +2751397888,2751463423,KR 2751528960,2751660031,FR 2751660032,2751725567,AT 2751725568,2751791103,SE @@ -61818,7 +63295,9 @@ 2788243472,2788243487,SG 2788243488,2788245007,US 2788245008,2788245023,CA -2788245024,2788261887,US +2788245024,2788253183,US +2788253184,2788253191,JP +2788253192,2788261887,US 2788261888,2788294655,GB 2788294656,2789212159,US 2789212160,2789277695,AU @@ -62239,7 +63718,11 @@ 2905428968,2905428975,AE 2905428976,2905432975,US 2905432976,2905432983,AE -2905432984,2905449983,US +2905432984,2905441535,US +2905441536,2905441791,DE +2905441792,2905446655,US +2905446656,2905446911,DE +2905446912,2905449983,US 2905449984,2905451007,CA 2905451008,2905451519,US 2905451520,2905451647,PA @@ -62294,7 +63777,7 @@ 2915767904,2915768191,US 2915768192,2915768199,GB 2915768200,2915768303,US -2915768304,2915768311,IL +2915768304,2915768311,MX 2915768312,2915768359,US 2915768360,2915768367,GB 2915768368,2915768375,US @@ -62315,17 +63798,29 @@ 2915769240,2915769247,CA 2915769248,2915769295,US 2915769296,2915769303,BR -2915769304,2915770311,US +2915769304,2915770167,US +2915770168,2915770175,BR +2915770176,2915770255,US +2915770256,2915770263,CA +2915770264,2915770279,JE +2915770280,2915770311,US 2915770312,2915770319,MX -2915770320,2915772671,US +2915770320,2915771103,US +2915771104,2915771135,CA +2915771136,2915772671,US 2915772672,2915772679,IN 2915772680,2915772711,US 2915772712,2915772719,GB -2915772720,2915773023,US +2915772720,2915772751,US +2915772752,2915772759,DE +2915772760,2915773023,US 2915773024,2915773039,IN 2915773040,2915773175,US 2915773176,2915773183,KW -2915773184,2915794959,US +2915773184,2915773791,US +2915773792,2915773807,CA +2915773808,2915773839,JE +2915773840,2915794959,US 2915794960,2915794975,MX 2915794976,2915795007,US 2915795008,2915795023,BR @@ -62565,9 +64060,15 @@ 2916434560,2916434591,CA 2916434592,2916434623,US 2916434624,2916434655,CA -2916434656,2916436735,US +2916434656,2916436543,US +2916436544,2916436607,CA +2916436608,2916436735,US 2916436736,2916436743,CA -2916436744,2916437503,US +2916436744,2916437039,US +2916437040,2916437047,DE +2916437048,2916437055,CN +2916437056,2916437063,GB +2916437064,2916437503,US 2916437504,2916437567,CA 2916437568,2916440143,US 2916440144,2916440159,CA @@ -62654,7 +64155,9 @@ 2917257216,2917261311,KY 2917261312,2917265407,US 2917265408,2917269503,JM -2917269504,2917572607,US +2917269504,2917449727,US +2917449728,2917466111,PR +2917466112,2917572607,US 2917572608,2917580799,CA 2917580800,2917597439,US 2917597440,2917597695,GB @@ -62743,8 +64246,12 @@ 2918416384,2918420479,CA 2918420480,2918432767,US 2918432768,2918436863,CA -2918436864,2918469631,US -2918469632,2918473727,CA +2918436864,2918460095,US +2918460096,2918460159,GB +2918460160,2918469631,US +2918469632,2918471423,CA +2918471424,2918471679,US +2918471680,2918473727,CA 2918473728,2918477823,US 2918477824,2918481919,CA 2918481920,2918514943,US @@ -62765,7 +64272,11 @@ 2918533216,2918534687,US 2918534688,2918534695,CN 2918534696,2918534703,US -2918534704,2918534807,CN +2918534704,2918534735,CN +2918534736,2918534751,US +2918534752,2918534767,CN +2918534768,2918534783,US +2918534784,2918534807,CN 2918534808,2918534815,US 2918534816,2918534887,CN 2918534888,2918534911,US @@ -62779,7 +64290,13 @@ 2918535680,2918535807,US 2918535808,2918536191,CN 2918536192,2918536703,US -2918536704,2918536959,CN +2918536704,2918536711,CN +2918536712,2918536767,US +2918536768,2918536791,CN +2918536792,2918536799,US +2918536800,2918536815,CN +2918536816,2918536823,US +2918536824,2918536959,CN 2918536960,2918537215,US 2918537216,2918537615,CN 2918537616,2918537623,US @@ -63157,7 +64674,8 @@ 2928173520,2928173527,JP 2928173528,2928173551,US 2928173552,2928173559,CA -2928173560,2928173679,US +2928173560,2928173671,US +2928173672,2928173679,NZ 2928173680,2928173695,JP 2928173696,2928173711,US 2928173712,2928173727,JP @@ -63195,7 +64713,9 @@ 2928175816,2928175823,CA 2928175824,2928175967,US 2928175968,2928175975,ZA -2928175976,2928176207,US +2928175976,2928176007,US +2928176008,2928176015,AU +2928176016,2928176207,US 2928176208,2928176223,ZA 2928176224,2928176231,JP 2928176232,2928176383,US @@ -64080,6 +65600,7 @@ 2942763008,2942767103,JP 2942767104,2942771199,AU 2942771200,2942779391,ID +2942779392,2942795775,VN 2942795776,2942959615,JP 2942959616,2942960639,VN 2942960640,2942961663,AU @@ -64102,7 +65623,6 @@ 2943311872,2943312895,HK 2943312896,2943313919,NZ 2943313920,2943314943,SG -2943314944,2943315967,BD 2943315968,2943318015,ID 2943318016,2943320063,JP 2943320064,2943336447,PK @@ -64117,7 +65637,6 @@ 2946375680,2946383871,ID 2946383872,2946392063,IN 2946392064,2946393087,BD -2946393088,2946394111,JP 2946394112,2946396159,AU 2946396160,2946400255,JP 2946400256,2946416639,NC @@ -64127,10 +65646,8 @@ 2947547136,2947579903,PH 2947579904,2947583999,KR 2947584000,2947586047,TO -2947586048,2947588095,JP 2947588096,2947590143,ID 2947590144,2947592191,SG -2947592192,2947596287,HK 2947596288,2947597311,IN 2947597312,2947598335,JP 2947598336,2947602431,AU @@ -64161,9 +65678,7 @@ 2987417600,2987425791,PL 2987425792,2987429887,BG 2987429888,2987433215,RU -2987433216,2987433235,KZ -2987433236,2987433239,RU -2987433240,2987433287,KZ +2987433216,2987433287,KZ 2987433288,2987433291,RU 2987433292,2987433331,KZ 2987433332,2987433335,RU @@ -64192,14 +65707,12 @@ 2987487232,2987491327,HR 2987491328,2987495423,RU 2987495424,2987499519,NO -2987499520,2987500103,MD -2987500104,2987500111,US -2987500112,2987500239,MD -2987500240,2987500255,DE -2987500256,2987503615,MD +2987499520,2987503615,MD 2987503616,2987511807,RU 2987511808,2987515903,JO -2987515904,2987519999,A2 +2987515904,2987519487,A2 +2987519488,2987519743,KE +2987519744,2987519999,A2 2987520000,2987524095,GB 2987524096,2987528191,RU 2987528192,2987529215,US @@ -64378,9 +65891,19 @@ 2987917312,2988179455,DE 2988179456,2988441599,SE 2988441600,2988441603,CH -2988441604,2988441655,FR +2988441604,2988441607,GB +2988441608,2988441611,DE +2988441612,2988441615,CH +2988441616,2988441647,FR +2988441648,2988441651,GB +2988441652,2988441655,FR 2988441656,2988441663,DE -2988441664,2988441839,FR +2988441664,2988441695,IT +2988441696,2988441791,FR +2988441792,2988441807,PL +2988441808,2988441815,FR +2988441816,2988441819,GB +2988441820,2988441839,FR 2988441840,2988441843,PL 2988441844,2988441847,BE 2988441848,2988441855,FR @@ -64389,7 +65912,9 @@ 2988441896,2988441903,PL 2988441904,2988441911,IT 2988441912,2988441919,ES -2988441920,2988441939,FR +2988441920,2988441931,PL +2988441932,2988441935,GB +2988441936,2988441939,FR 2988441940,2988441943,ES 2988441944,2988441951,FR 2988441952,2988441967,PL @@ -64412,8 +65937,7 @@ 2988442432,2988442439,CZ 2988442440,2988442447,ES 2988442448,2988442463,GB -2988442464,2988442623,FR -2988442624,2988442639,NL +2988442464,2988442639,FR 2988442640,2988442647,PL 2988442648,2988442651,DE 2988442652,2988442655,GB @@ -64444,7 +65968,7 @@ 2988443088,2988443111,PL 2988443112,2988443119,ES 2988443120,2988443391,FR -2988443392,2988443407,GB +2988443392,2988443407,DE 2988443408,2988443439,FR 2988443440,2988443443,CZ 2988443444,2988443447,DE @@ -64453,7 +65977,7 @@ 2988443488,2988443539,FR 2988443540,2988443547,ES 2988443548,2988443551,IT -2988443552,2988443555,FR +2988443552,2988443555,CH 2988443556,2988443559,PL 2988443560,2988443563,GB 2988443564,2988443567,PL @@ -64470,7 +65994,8 @@ 2988444204,2988444207,NL 2988444208,2988444415,FR 2988444416,2988444679,ES -2988444680,2988444695,PL +2988444680,2988444687,FR +2988444688,2988444695,LT 2988444696,2988444703,FR 2988444704,2988444719,GB 2988444720,2988444735,ES @@ -64495,7 +66020,9 @@ 2988445036,2988445039,PL 2988445040,2988445119,FR 2988445120,2988445127,ES -2988445128,2988445167,FR +2988445128,2988445139,FR +2988445140,2988445143,GB +2988445144,2988445167,FR 2988445168,2988445183,ES 2988445184,2988445951,DE 2988445952,2988445967,FR @@ -64528,7 +66055,8 @@ 2988448000,2988448127,DE 2988448128,2988448255,ES 2988448256,2988448511,DE -2988448512,2988448519,FR +2988448512,2988448515,GB +2988448516,2988448519,FR 2988448520,2988448543,PL 2988448544,2988448551,FR 2988448552,2988448559,PL @@ -64567,7 +66095,7 @@ 2988449580,2988449583,PL 2988449584,2988449631,FR 2988449632,2988449647,ES -2988449648,2988449663,GB +2988449648,2988449663,FR 2988449664,2988449695,DE 2988449696,2988449727,IT 2988449728,2988449743,BE @@ -64579,9 +66107,7 @@ 2988457984,2988457987,FR 2988457988,2988457991,PL 2988457992,2988457995,CH -2988457996,2988457999,FR -2988458000,2988458015,GB -2988458016,2988458031,FR +2988457996,2988458031,FR 2988458032,2988458047,IT 2988458048,2988458055,PL 2988458056,2988458063,CH @@ -64719,8 +66245,7 @@ 2988460576,2988460591,PT 2988460592,2988460607,GB 2988460608,2988460615,IT -2988460616,2988460619,CZ -2988460620,2988460623,IT +2988460616,2988460623,PT 2988460624,2988460679,FR 2988460680,2988460687,DE 2988460688,2988460719,FR @@ -64729,14 +66254,13 @@ 2988460768,2988460799,PL 2988460800,2988460863,DE 2988460864,2988460927,FR -2988460928,2988460931,DE +2988460928,2988460931,ES 2988460932,2988460943,PL 2988460944,2988460959,DE 2988460960,2988460991,GB 2988460992,2988461087,FR 2988461088,2988461103,PL -2988461104,2988461119,FR -2988461120,2988461151,NL +2988461104,2988461151,FR 2988461152,2988461183,GB 2988461184,2988461255,FR 2988461256,2988461259,IT @@ -64776,7 +66300,7 @@ 2988461616,2988461623,IT 2988461624,2988461695,FR 2988461696,2988461699,DE -2988461700,2988461703,FR +2988461700,2988461703,PL 2988461704,2988461707,NL 2988461708,2988461711,BE 2988461712,2988461719,FR @@ -64800,7 +66324,7 @@ 2988461860,2988461871,DE 2988461872,2988461879,FR 2988461880,2988461883,ES -2988461884,2988461887,IT +2988461884,2988461887,PL 2988461888,2988461903,FR 2988461904,2988461911,ES 2988461912,2988461915,IT @@ -64903,7 +66427,9 @@ 2988463832,2988463835,DE 2988463836,2988463839,FR 2988463840,2988463871,PL -2988463872,2988463915,FR +2988463872,2988463907,FR +2988463908,2988463911,PL +2988463912,2988463915,FR 2988463916,2988463919,GB 2988463920,2988463947,FR 2988463948,2988463951,GB @@ -64916,7 +66442,7 @@ 2988464028,2988464031,PL 2988464032,2988464055,FR 2988464056,2988464059,ES -2988464060,2988464063,FR +2988464060,2988464063,CZ 2988464064,2988464095,PL 2988464096,2988464271,FR 2988464272,2988464275,ES @@ -64925,7 +66451,8 @@ 2988464284,2988464287,PL 2988464288,2988464303,FR 2988464304,2988464307,IT -2988464308,2988464351,FR +2988464308,2988464319,FR +2988464320,2988464351,GB 2988464352,2988464355,DE 2988464356,2988464359,FR 2988464360,2988464363,PL @@ -65013,10 +66540,19 @@ 2988466128,2988466131,PT 2988466132,2988466139,FR 2988466140,2988466143,PL -2988466144,2988466175,NL -2988466176,2988476415,FR +2988466144,2988466159,NL +2988466160,2988476415,FR 2988476416,2988478463,IT -2988478464,2988482935,FR +2988478464,2988482799,FR +2988482800,2988482807,ES +2988482808,2988482811,GB +2988482812,2988482815,FR +2988482816,2988482819,NL +2988482820,2988482823,DE +2988482824,2988482827,PL +2988482828,2988482895,FR +2988482896,2988482911,GB +2988482912,2988482935,FR 2988482936,2988482939,PL 2988482940,2988482943,LT 2988482944,2988482959,PL @@ -65033,8 +66569,8 @@ 2988483108,2988483111,PL 2988483112,2988483115,ES 2988483116,2988483119,PL -2988483120,2988483135,GB -2988483136,2988483151,FR +2988483120,2988483127,ES +2988483128,2988483151,FR 2988483152,2988483155,CH 2988483156,2988483159,FR 2988483160,2988483167,ES @@ -65069,7 +66605,7 @@ 2988483744,2988483767,GB 2988483768,2988483775,FR 2988483776,2988483871,PL -2988483872,2988483879,FR +2988483872,2988483879,DE 2988483880,2988483887,ES 2988483888,2988483895,GB 2988483896,2988483903,DE @@ -65078,11 +66614,11 @@ 2988483952,2988483963,IT 2988483964,2988483967,FR 2988483968,2988483983,IE -2988483984,2988483987,DE -2988483988,2988483991,NL -2988483992,2988483999,FR -2988484000,2988484003,GB -2988484004,2988484007,DE +2988483984,2988483991,DE +2988483992,2988483995,GB +2988483996,2988483999,PT +2988484000,2988484003,DE +2988484004,2988484007,GB 2988484008,2988484011,FR 2988484012,2988484015,IE 2988484016,2988484019,FR @@ -65090,7 +66626,9 @@ 2988484024,2988484031,FR 2988484032,2988484039,IT 2988484040,2988484047,NL -2988484048,2988484095,FR +2988484048,2988484051,FR +2988484052,2988484055,GB +2988484056,2988484095,FR 2988484096,2988484111,DE 2988484112,2988484127,FR 2988484128,2988484131,PT @@ -65121,7 +66659,19 @@ 2988484512,2988484543,IT 2988484544,2988484591,FR 2988484592,2988484607,ES -2988484608,2988485599,FR +2988484608,2988484863,FR +2988484864,2988484879,ES +2988484880,2988484991,FR +2988484992,2988485007,LT +2988485008,2988485023,PL +2988485024,2988485071,FR +2988485072,2988485087,IE +2988485088,2988485327,FR +2988485328,2988485335,FI +2988485336,2988485343,PT +2988485344,2988485487,FR +2988485488,2988485503,DE +2988485504,2988485599,FR 2988485600,2988485607,PL 2988485608,2988485611,GB 2988485612,2988485615,PL @@ -65152,7 +66702,7 @@ 2988486072,2988486075,FR 2988486076,2988486079,ES 2988486080,2988486083,BE -2988486084,2988486087,CZ +2988486084,2988486087,NL 2988486088,2988486111,FR 2988486112,2988486127,PL 2988486128,2988486159,NL @@ -65262,8 +66812,24 @@ 2988489280,2988489283,FI 2988489284,2988489287,FR 2988489288,2988489295,PL -2988489296,2988489343,FR -2988489344,2988489471,DE +2988489296,2988489311,FR +2988489312,2988489327,PL +2988489328,2988489331,DE +2988489332,2988489335,PL +2988489336,2988489339,DE +2988489340,2988489343,PL +2988489344,2988489347,DE +2988489348,2988489351,PL +2988489352,2988489355,FR +2988489356,2988489359,PT +2988489360,2988489379,ES +2988489380,2988489383,GB +2988489384,2988489391,FR +2988489392,2988489399,NL +2988489400,2988489407,FR +2988489408,2988489439,PT +2988489440,2988489455,GB +2988489456,2988489471,BE 2988489472,2988489475,ES 2988489476,2988489479,GB 2988489480,2988489483,IT @@ -65303,7 +66869,7 @@ 2988490060,2988490063,FR 2988490064,2988490079,GB 2988490080,2988490095,FR -2988490096,2988490103,DE +2988490096,2988490103,PT 2988490104,2988490107,ES 2988490108,2988490111,FR 2988490112,2988490143,PL @@ -65350,7 +66916,7 @@ 2988507216,2988507231,FR 2988507232,2988507239,ES 2988507240,2988507243,FR -2988507244,2988507247,ES +2988507244,2988507247,GB 2988507248,2988507263,FR 2988507264,2988507279,PL 2988507280,2988507287,GB @@ -65450,9 +67016,9 @@ 2988508344,2988508351,GB 2988508352,2988508367,ES 2988508368,2988508383,FR -2988508384,2988508419,PL -2988508420,2988508423,ES -2988508424,2988508435,FR +2988508384,2988508423,PL +2988508424,2988508431,IE +2988508432,2988508435,FR 2988508436,2988508439,DE 2988508440,2988508447,PL 2988508448,2988508479,FR @@ -65546,7 +67112,10 @@ 2988509904,2988509907,PL 2988509908,2988509919,IT 2988509920,2988509927,FR -2988509928,2988509951,PL +2988509928,2988509935,PL +2988509936,2988509939,DE +2988509940,2988509943,IT +2988509944,2988509951,FR 2988509952,2988509955,IT 2988509956,2988509959,PL 2988509960,2988509963,IT @@ -65557,7 +67126,7 @@ 2988509992,2988509995,FR 2988509996,2988509999,DE 2988510000,2988510015,PL -2988510016,2988510023,ES +2988510016,2988510023,GB 2988510024,2988510031,FR 2988510032,2988510079,PL 2988510080,2988510095,ES @@ -65582,7 +67151,7 @@ 2988510324,2988510327,GB 2988510328,2988510399,FR 2988510400,2988510403,DE -2988510404,2988510407,FR +2988510404,2988510407,GB 2988510408,2988510415,PL 2988510416,2988510431,FR 2988510432,2988510435,IT @@ -65593,7 +67162,10 @@ 2988510776,2988510847,FR 2988510848,2988510943,PL 2988510944,2988510975,GB -2988510976,2988510991,PL +2988510976,2988510979,PL +2988510980,2988510983,IT +2988510984,2988510987,GB +2988510988,2988510991,PL 2988510992,2988511007,FR 2988511008,2988511015,PL 2988511016,2988511023,IT @@ -65615,7 +67187,7 @@ 2988511576,2988511583,GB 2988511584,2988511663,FR 2988511664,2988511671,DE -2988511672,2988511675,FR +2988511672,2988511675,PL 2988511676,2988511679,GB 2988511680,2988511683,NL 2988511684,2988511687,ES @@ -65691,12 +67263,13 @@ 2988512460,2988512467,FR 2988512468,2988512471,GB 2988512472,2988512479,FR -2988512480,2988512483,ES +2988512480,2988512483,DE 2988512484,2988512523,FR 2988512524,2988512527,IT 2988512528,2988512543,FR 2988512544,2988512551,PL -2988512552,2988512579,FR +2988512552,2988512575,FR +2988512576,2988512579,NL 2988512580,2988512583,ES 2988512584,2988512587,FR 2988512588,2988512591,PL @@ -65789,7 +67362,9 @@ 2988513680,2988513683,ES 2988513684,2988513691,FR 2988513692,2988513695,DE -2988513696,2988513723,FR +2988513696,2988513703,FR +2988513704,2988513707,DE +2988513708,2988513723,FR 2988513724,2988513727,DE 2988513728,2988513731,FR 2988513732,2988513735,BE @@ -65809,10 +67384,11 @@ 2988513984,2988513991,GB 2988513992,2988513999,IE 2988514000,2988514015,FR -2988514016,2988514019,PT +2988514016,2988514019,ES 2988514020,2988514023,NL 2988514024,2988514027,PT -2988514028,2988514047,FR +2988514028,2988514031,PL +2988514032,2988514047,FR 2988514048,2988514079,ES 2988514080,2988514095,PL 2988514096,2988514099,DE @@ -65821,7 +67397,8 @@ 2988514112,2988514115,DE 2988514116,2988514127,FR 2988514128,2988514131,IE -2988514132,2988514139,IT +2988514132,2988514135,IT +2988514136,2988514139,ES 2988514140,2988514143,PL 2988514144,2988514159,FR 2988514160,2988514163,IT @@ -65875,7 +67452,7 @@ 2988514880,2988514943,PL 2988514944,2988514959,ES 2988514960,2988514975,PL -2988514976,2988514979,FR +2988514976,2988514979,DE 2988514980,2988514983,PL 2988514984,2988514991,IT 2988514992,2988514995,FR @@ -65944,7 +67521,9 @@ 2988524488,2988524495,FR 2988524496,2988524543,PL 2988524544,2988524575,IE -2988524576,2988524607,ES +2988524576,2988524591,FR +2988524592,2988524603,PL +2988524604,2988524607,ES 2988524608,2988524623,GB 2988524624,2988524639,FR 2988524640,2988524671,DE @@ -65955,10 +67534,10 @@ 2988525584,2988525647,PL 2988525648,2988525655,IE 2988525656,2988525659,FR -2988525660,2988525663,GB +2988525660,2988525663,DE 2988525664,2988525695,FR 2988525696,2988525823,PL -2988525824,2988525839,GB +2988525824,2988525839,FR 2988525840,2988525847,ES 2988525848,2988525851,IT 2988525852,2988525887,DE @@ -65967,7 +67546,11 @@ 2988526080,2988526423,PL 2988526424,2988526427,NL 2988526428,2988526431,ES -2988526432,2988526527,FR +2988526432,2988526447,FR +2988526448,2988526451,ES +2988526452,2988526455,FR +2988526456,2988526463,ES +2988526464,2988526527,FR 2988526528,2988526543,GB 2988526544,2988526559,IT 2988526560,2988526579,FR @@ -65994,7 +67577,10 @@ 2988526996,2988526999,ES 2988527000,2988527055,FR 2988527056,2988527071,NL -2988527072,2988527127,FR +2988527072,2988527095,GB +2988527096,2988527099,IE +2988527100,2988527103,LT +2988527104,2988527127,FR 2988527128,2988527167,PL 2988527168,2988527183,FR 2988527184,2988527187,IT @@ -66049,10 +67635,13 @@ 2988528400,2988528415,FR 2988528416,2988528423,GB 2988528424,2988528431,NL -2988528432,2988528451,CH -2988528452,2988528455,DE -2988528456,2988528459,ES -2988528460,2988528463,NL +2988528432,2988528435,PL +2988528436,2988528439,FR +2988528440,2988528447,ES +2988528448,2988528451,DE +2988528452,2988528455,IT +2988528456,2988528459,GB +2988528460,2988528463,ES 2988528464,2988528467,DE 2988528468,2988528471,GB 2988528472,2988528475,ES @@ -66135,8 +67724,7 @@ 2988529768,2988529771,FR 2988529772,2988529775,DE 2988529776,2988529783,ES -2988529784,2988529791,CH -2988529792,2988529823,FR +2988529784,2988529823,FR 2988529824,2988529855,ES 2988529856,2988529887,FR 2988529888,2988529891,DE @@ -66147,7 +67735,7 @@ 2988529920,2988529935,FR 2988529936,2988529939,CH 2988529940,2988529943,GB -2988529944,2988529951,NL +2988529944,2988529951,ES 2988529952,2988529955,GB 2988529956,2988529959,DE 2988529960,2988529967,GB @@ -66159,7 +67747,7 @@ 2988530040,2988530043,ES 2988530044,2988530047,FR 2988530048,2988530063,GB -2988530064,2988530079,DE +2988530064,2988530079,IE 2988530080,2988530095,FR 2988530096,2988530099,DE 2988530100,2988530103,FR @@ -66196,7 +67784,9 @@ 2988530872,2988530879,IT 2988530880,2988530887,PL 2988530888,2988530895,DE -2988530896,2988530975,FR +2988530896,2988530943,FR +2988530944,2988530959,BE +2988530960,2988530975,FR 2988530976,2988531007,PL 2988531008,2988531015,DE 2988531016,2988531019,NL @@ -66210,7 +67800,11 @@ 2988531056,2988531071,FR 2988531072,2988531075,DE 2988531076,2988531079,GB -2988531080,2988531151,FR +2988531080,2988531083,FR +2988531084,2988531087,PL +2988531088,2988531103,FR +2988531104,2988531119,BE +2988531120,2988531151,FR 2988531152,2988531167,PL 2988531168,2988531175,NL 2988531176,2988531183,GB @@ -66225,11 +67819,12 @@ 2988531312,2988531319,FR 2988531320,2988531323,PT 2988531324,2988531327,GB -2988531328,2988531391,FR +2988531328,2988531343,IE +2988531344,2988531391,FR 2988531392,2988531399,PL 2988531400,2988531403,DE -2988531404,2988531423,PL -2988531424,2988531451,FR +2988531404,2988531427,PL +2988531428,2988531451,FR 2988531452,2988531455,CH 2988531456,2988535807,FR 2988535808,2988537855,ES @@ -66443,25 +68038,21 @@ 2988542784,2988542847,CZ 2988542848,2988542919,DE 2988542920,2988542923,PL -2988542924,2988542943,GB +2988542924,2988542927,CH +2988542928,2988542935,FR +2988542936,2988542939,IT +2988542940,2988542943,PL 2988542944,2988542959,DE 2988542960,2988542963,FR -2988542964,2988542967,CH -2988542968,2988542971,NL -2988542972,2988542975,DE -2988542976,2988542991,FR -2988542992,2988542995,FI -2988542996,2988542999,IE -2988543000,2988543003,IT -2988543004,2988543007,LT +2988542964,2988542967,PL +2988542968,2988542975,DE +2988542976,2988543007,FR 2988543008,2988543011,PL -2988543012,2988543015,FR +2988543012,2988543015,IT 2988543016,2988543023,CH 2988543024,2988543039,DE -2988543040,2988543043,CZ -2988543044,2988543047,ES -2988543048,2988543051,IE -2988543052,2988543067,FR +2988543040,2988543047,GB +2988543048,2988543067,FR 2988543068,2988543071,PL 2988543072,2988543103,FR 2988543104,2988543167,GB @@ -66492,7 +68083,10 @@ 2988543384,2988543399,DE 2988543400,2988543403,GB 2988543404,2988543407,NL -2988543408,2988543423,GB +2988543408,2988543411,PL +2988543412,2988543415,FR +2988543416,2988543419,ES +2988543420,2988543423,PL 2988543424,2988543431,DE 2988543432,2988543439,GB 2988543440,2988543447,FR @@ -66521,7 +68115,8 @@ 2988543968,2988544015,FR 2988544016,2988544023,GB 2988544024,2988544031,IE -2988544032,2988544039,FR +2988544032,2988544035,PL +2988544036,2988544039,FR 2988544040,2988544043,DE 2988544044,2988544047,GB 2988544048,2988544063,FR @@ -66561,8 +68156,7 @@ 2988544480,2988544511,PL 2988544512,2988544527,FR 2988544528,2988544535,ES -2988544536,2988544539,PL -2988544540,2988544543,LT +2988544536,2988544543,FR 2988544544,2988544639,ES 2988544640,2988544647,PL 2988544648,2988544655,FR @@ -66575,7 +68169,12 @@ 2988544704,2988544719,IT 2988544720,2988544723,FR 2988544724,2988544727,DE -2988544728,2988544799,FR +2988544728,2988544767,FR +2988544768,2988544775,ES +2988544776,2988544783,PT +2988544784,2988544787,GB +2988544788,2988544791,PL +2988544792,2988544799,PT 2988544800,2988544831,FI 2988544832,2988544863,FR 2988544864,2988544895,PL @@ -66624,9 +68223,7 @@ 2988545384,2988545387,ES 2988545388,2988545391,FR 2988545392,2988545395,DE -2988545396,2988545407,FR -2988545408,2988545439,IE -2988545440,2988545471,FR +2988545396,2988545471,FR 2988545472,2988545503,ES 2988545504,2988545507,FR 2988545508,2988545511,DE @@ -66661,7 +68258,8 @@ 2988545920,2988545923,GB 2988545924,2988545927,PL 2988545928,2988545931,BE -2988545932,2988545943,FR +2988545932,2988545935,DE +2988545936,2988545943,FR 2988545944,2988545967,NL 2988545968,2988545971,IT 2988545972,2988545979,FR @@ -66710,7 +68308,8 @@ 2988546540,2988546547,ES 2988546548,2988546559,FR 2988546560,2988546567,NL -2988546568,2988546575,FR +2988546568,2988546571,PL +2988546572,2988546575,DE 2988546576,2988546579,ES 2988546580,2988546583,GB 2988546584,2988546591,FR @@ -66796,7 +68395,7 @@ 2988547476,2988547479,GB 2988547480,2988547487,FR 2988547488,2988547519,ES -2988547520,2988547523,PL +2988547520,2988547523,NL 2988547524,2988547527,CZ 2988547528,2988547531,IT 2988547532,2988547535,ES @@ -66898,8 +68497,7 @@ 2988557400,2988557407,DE 2988557408,2988557427,ES 2988557428,2988557431,DE -2988557432,2988557435,PL -2988557436,2988557439,FR +2988557432,2988557439,PL 2988557440,2988557471,GB 2988557472,2988557487,PL 2988557488,2988557495,DE @@ -66920,7 +68518,7 @@ 2988557772,2988557823,FR 2988557824,2988557951,DE 2988557952,2988557983,LT -2988557984,2988557999,GB +2988557984,2988557999,FR 2988558000,2988558015,IE 2988558016,2988558047,FR 2988558048,2988558063,PL @@ -66929,8 +68527,8 @@ 2988558076,2988558079,PL 2988558080,2988558083,FR 2988558084,2988558087,PL -2988558088,2988558099,FR -2988558100,2988558103,PL +2988558088,2988558095,FR +2988558096,2988558103,PL 2988558104,2988558119,FR 2988558120,2988558123,GB 2988558124,2988558127,FR @@ -66949,7 +68547,8 @@ 2988558656,2988558727,FR 2988558728,2988558731,IT 2988558732,2988558735,PT -2988558736,2988558767,FR +2988558736,2988558751,GB +2988558752,2988558767,FR 2988558768,2988558783,CH 2988558784,2988558831,FR 2988558832,2988558847,IT @@ -67019,13 +68618,18 @@ 2988560800,2988560815,DE 2988560816,2988560831,FR 2988560832,2988560863,GB -2988560864,2988560895,DE +2988560864,2988560871,FR +2988560872,2988560895,GB 2988560896,2988560911,PL 2988560912,2988560919,FR 2988560920,2988560923,DE 2988560924,2988560951,FR 2988560952,2988560959,PL -2988560960,2988561023,IE +2988560960,2988560975,FR +2988560976,2988560991,ES +2988560992,2988560995,FR +2988560996,2988561007,DE +2988561008,2988561023,FR 2988561024,2988561031,ES 2988561032,2988561039,FR 2988561040,2988561043,GB @@ -67046,7 +68650,10 @@ 2988561172,2988561175,PL 2988561176,2988561179,ES 2988561180,2988561183,PL -2988561184,2988561199,GB +2988561184,2988561187,GB +2988561188,2988561191,FR +2988561192,2988561195,CZ +2988561196,2988561199,LT 2988561200,2988561203,PL 2988561204,2988561207,GB 2988561208,2988561215,PL @@ -67055,8 +68662,7 @@ 2988561288,2988561291,ES 2988561292,2988561295,PT 2988561296,2988561303,PL -2988561304,2988561311,FR -2988561312,2988561343,IE +2988561304,2988561343,FR 2988561344,2988561375,PL 2988561376,2988561391,FR 2988561392,2988561403,ES @@ -67081,8 +68687,7 @@ 2988561872,2988561875,CH 2988561876,2988561887,FR 2988561888,2988561903,PL -2988561904,2988561919,IE -2988561920,2988561983,FR +2988561904,2988561983,FR 2988561984,2988562015,DE 2988562016,2988562023,PT 2988562024,2988562027,ES @@ -67094,9 +68699,9 @@ 2988562112,2988562127,FR 2988562128,2988562143,PL 2988562144,2988562151,FR -2988562152,2988562159,PL -2988562160,2988562175,IT -2988562176,2988562431,NL +2988562152,2988562163,PL +2988562164,2988562167,DE +2988562168,2988562431,NL 2988562432,2988562591,FR 2988562592,2988562595,GB 2988562596,2988562607,FR @@ -67108,7 +68713,11 @@ 2988562708,2988562711,DE 2988562712,2988562719,GB 2988562720,2988562815,BE -2988562816,2988562847,FR +2988562816,2988562823,PL +2988562824,2988562831,PT +2988562832,2988562835,FR +2988562836,2988562839,NL +2988562840,2988562847,FR 2988562848,2988562855,NL 2988562856,2988562863,DE 2988562864,2988562911,FR @@ -67157,7 +68766,15 @@ 2988564020,2988564023,PL 2988564024,2988564031,FR 2988564032,2988564063,BE -2988564064,2988572671,FR +2988564064,2988564191,FR +2988564192,2988564195,ES +2988564196,2988564203,DE +2988564204,2988564383,FR +2988564384,2988564387,IE +2988564388,2988564391,GB +2988564392,2988564395,NL +2988564396,2988564399,CH +2988564400,2988572671,FR 2988572672,2988703743,RU 2988703744,2988834815,PL 2988834816,2988965887,CH @@ -67206,139 +68823,162 @@ 2991185968,2991185983,DE 2991185984,2991185999,DK 2991186000,2991186015,ES -2991186016,2991186175,SE +2991186016,2991186019,FR +2991186020,2991186175,SE 2991186176,2991186207,GB 2991186208,2991186223,NL 2991186224,2991186239,DE 2991186240,2991186255,DK 2991186256,2991186271,ES -2991186272,2991186431,SE +2991186272,2991186275,FR +2991186276,2991186431,SE 2991186432,2991186463,GB 2991186464,2991186479,NL 2991186480,2991186495,DE 2991186496,2991186511,DK 2991186512,2991186527,ES -2991186528,2991186687,SE +2991186528,2991186531,FR +2991186532,2991186687,SE 2991186688,2991186719,GB 2991186720,2991186735,NL 2991186736,2991186751,DE 2991186752,2991186767,DK 2991186768,2991186783,ES -2991186784,2991186943,SE +2991186784,2991186787,FR +2991186788,2991186943,SE 2991186944,2991186975,GB 2991186976,2991186991,NL 2991186992,2991187007,DE 2991187008,2991187023,DK 2991187024,2991187039,ES -2991187040,2991187199,SE +2991187040,2991187043,FR +2991187044,2991187199,SE 2991187200,2991187231,GB 2991187232,2991187247,NL 2991187248,2991187263,DE 2991187264,2991187279,DK 2991187280,2991187295,ES -2991187296,2991187455,SE +2991187296,2991187299,FR +2991187300,2991187455,SE 2991187456,2991187487,GB 2991187488,2991187503,NL 2991187504,2991187519,DE 2991187520,2991187535,DK 2991187536,2991187551,ES -2991187552,2991187711,SE +2991187552,2991187555,FR +2991187556,2991187711,SE 2991187712,2991187743,GB 2991187744,2991187759,NL 2991187760,2991187775,DE 2991187776,2991187791,DK 2991187792,2991187807,ES -2991187808,2991187967,SE +2991187808,2991187811,FR +2991187812,2991187967,SE 2991187968,2991187999,GB 2991188000,2991188015,NL 2991188016,2991188031,DE 2991188032,2991188047,DK 2991188048,2991188063,ES -2991188064,2991188223,SE +2991188064,2991188067,FR +2991188068,2991188223,SE 2991188224,2991188255,GB 2991188256,2991188271,NL 2991188272,2991188287,DE 2991188288,2991188303,DK 2991188304,2991188319,ES -2991188320,2991188479,SE +2991188320,2991188323,FR +2991188324,2991188479,SE 2991188480,2991188511,GB 2991188512,2991188527,NL 2991188528,2991188543,DE 2991188544,2991188559,DK 2991188560,2991188575,ES -2991188576,2991188735,SE +2991188576,2991188579,FR +2991188580,2991188735,SE 2991188736,2991188767,GB 2991188768,2991188783,NL 2991188784,2991188799,DE 2991188800,2991188815,DK 2991188816,2991188831,ES -2991188832,2991188991,SE +2991188832,2991188835,FR +2991188836,2991188991,SE 2991188992,2991189023,GB 2991189024,2991189039,NL 2991189040,2991189055,DE 2991189056,2991189071,DK 2991189072,2991189087,ES -2991189088,2991189247,SE +2991189088,2991189091,FR +2991189092,2991189247,SE 2991189248,2991189279,GB 2991189280,2991189295,NL 2991189296,2991189311,DE 2991189312,2991189327,DK 2991189328,2991189343,ES -2991189344,2991189503,SE +2991189344,2991189347,FR +2991189348,2991189503,SE 2991189504,2991189535,GB 2991189536,2991189551,NL 2991189552,2991189567,DE 2991189568,2991189583,DK 2991189584,2991189599,ES -2991189600,2991189759,SE +2991189600,2991189603,FR +2991189604,2991189759,SE 2991189760,2991189791,GB 2991189792,2991189807,NL 2991189808,2991189823,DE 2991189824,2991189839,DK 2991189840,2991189855,ES -2991189856,2991190015,SE +2991189856,2991189859,FR +2991189860,2991190015,SE 2991190016,2991190047,GB 2991190048,2991190063,NL 2991190064,2991190079,DE 2991190080,2991190095,DK 2991190096,2991190111,ES -2991190112,2991190271,SE +2991190112,2991190115,FR +2991190116,2991190271,SE 2991190272,2991190303,GB 2991190304,2991190319,NL 2991190320,2991190335,DE 2991190336,2991190351,DK 2991190352,2991190367,ES -2991190368,2991190527,SE +2991190368,2991190371,FR +2991190372,2991190527,SE 2991190528,2991190559,GB 2991190560,2991190575,NL 2991190576,2991190591,DE 2991190592,2991190607,DK 2991190608,2991190623,ES -2991190624,2991190783,SE +2991190624,2991190627,FR +2991190628,2991190783,SE 2991190784,2991190815,GB 2991190816,2991190831,NL 2991190832,2991190847,DE 2991190848,2991190863,DK 2991190864,2991190879,ES -2991190880,2991191039,SE +2991190880,2991190883,FR +2991190884,2991191039,SE 2991191040,2991191071,GB 2991191072,2991191087,NL 2991191088,2991191103,DE 2991191104,2991191119,DK 2991191120,2991191135,ES -2991191136,2991191295,SE +2991191136,2991191139,FR +2991191140,2991191295,SE 2991191296,2991191327,GB 2991191328,2991191343,NL 2991191344,2991191359,DE 2991191360,2991191375,DK 2991191376,2991191391,ES -2991191392,2991191551,SE +2991191392,2991191395,FR +2991191396,2991191551,SE 2991191552,2991191583,GB 2991191584,2991191599,NL 2991191600,2991191615,DE 2991191616,2991191631,DK 2991191632,2991191647,ES -2991191648,2991191807,SE +2991191648,2991191651,FR +2991191652,2991191807,SE 2991191808,2991192063,FI 2991192064,2991192255,DK 2991192256,2991192319,IT @@ -67453,14 +69093,17 @@ 2996995648,2996995711,BZ 2996995712,2996995775,BY 2996995776,2996995839,RU -2996995840,2996996127,DE +2996995840,2996995871,DE +2996995872,2996995903,BR +2996995904,2996996127,DE 2996996128,2996996159,CA 2996996160,2996996287,DE 2996996288,2996996351,RU 2996996352,2996996383,DE 2996996384,2996996447,RU 2996996448,2996996575,UA -2996996576,2996996639,DE +2996996576,2996996583,US +2996996584,2996996639,DE 2996996640,2996996767,UA 2996996768,2996996831,RU 2996996832,2996997119,CN @@ -67480,7 +69123,8 @@ 2996999952,2996999967,LT 2996999968,2997000447,DE 2997000448,2997000703,RU -2997000704,2997000831,DE +2997000704,2997000767,DE +2997000768,2997000831,CY 2997000832,2997000959,CA 2997000960,2997000991,DE 2997000992,2997001119,RO @@ -67597,17 +69241,19 @@ 2999985744,2999985759,NL 2999985760,2999988991,BE 2999988992,2999989007,FR -2999989008,2999992319,NL +2999989008,2999989247,NL +2999989248,2999990527,BE +2999990528,2999992319,NL 2999992320,3000000511,RU 3000000512,3000008703,DE 3000008704,3000016895,RU -3000016896,3000025087,GB +3000016896,3000020991,GB +3000020992,3000023039,US +3000023040,3000025087,GB 3000025088,3000033279,GI 3000033280,3000041471,RU 3000041472,3000049663,BA -3000049664,3000052319,CH -3000052320,3000052351,AT -3000052352,3000057855,CH +3000049664,3000057855,CH 3000057856,3000066047,UA 3000066048,3000074239,RU 3000074240,3000082431,CZ @@ -67692,7 +69338,7 @@ 3000506368,3000508415,PL 3000508416,3000510463,UA 3000510464,3000512511,PL -3000512512,3000514559,EE +3000512512,3000514559,SE 3000514560,3000516607,SI 3000516608,3000520703,RU 3000520704,3000522751,CZ @@ -67775,16 +69421,35 @@ 3001823232,3001827327,GE 3001827328,3001827647,SE 3001827648,3001827743,GB -3001827744,3001827839,SE -3001827840,3001827871,GB -3001827872,3001830399,SE -3001830400,3001830431,GB -3001830432,3001830655,SE -3001830656,3001830687,GB -3001830688,3001830911,SE -3001830912,3001830943,GB -3001830944,3001830975,US -3001830976,3001831423,SE +3001827744,3001828864,SE +3001828865,3001828896,US +3001828897,3001829120,SE +3001829121,3001829152,US +3001829153,3001830400,SE +3001830401,3001830432,GB +3001830433,3001830527,SE +3001830528,3001830559,IT +3001830560,3001830591,FR +3001830592,3001830623,US +3001830624,3001830656,SE +3001830657,3001830688,GB +3001830689,3001830783,SE +3001830784,3001830815,IT +3001830816,3001830847,FR +3001830848,3001830879,US +3001830880,3001830912,SE +3001830913,3001830944,GB +3001830945,3001831039,SE +3001831040,3001831071,IT +3001831072,3001831103,FR +3001831104,3001831135,US +3001831136,3001831167,IT +3001831168,3001831199,GB +3001831200,3001831295,SE +3001831296,3001831327,IT +3001831328,3001831359,FR +3001831360,3001831391,US +3001831392,3001831423,SE 3001831424,3001835519,BA 3001835520,3001839615,RU 3001839616,3001843711,ES @@ -67795,15 +69460,28 @@ 3001851904,3001855999,IT 3001856000,3001859071,NL 3001859072,3001860095,MD -3001860096,3001864191,EE +3001860096,3001861119,LV +3001861120,3001862143,LT +3001862144,3001864191,EE 3001864192,3001868287,RU -3001868288,3001869312,FR +3001868288,3001869055,FR +3001869056,3001869311,RU +3001869312,3001869312,FR 3001869313,3001869599,SA 3001869600,3001869600,FR 3001869601,3001869696,IR 3001869697,3001869823,YE 3001869824,3001869887,IR -3001869888,3001872383,FR +3001869888,3001870079,FR +3001870080,3001870335,RU +3001870336,3001870591,IT +3001870592,3001870847,FR +3001870848,3001871103,DE +3001871104,3001871359,ES +3001871360,3001871615,GR +3001871616,3001871871,PL +3001871872,3001872127,PT +3001872128,3001872383,RO 3001872384,3001876479,RU 3001876480,3001880575,IT 3001880576,3001884671,RU @@ -67833,7 +69511,8 @@ 3001954304,3001958399,IT 3001958400,3001962495,KZ 3001962496,3001966591,GB -3001966592,3001970687,NL +3001966592,3001968639,NL +3001968640,3001970687,BE 3001970688,3001974783,RU 3001974784,3001975567,GB 3001975568,3001975575,LK @@ -67852,7 +69531,7 @@ 3001975704,3001975711,DE 3001975712,3001975719,MY 3001975720,3001975727,US -3001975728,3001975735,GB +3001975728,3001975735,RU 3001975736,3001975743,MY 3001975744,3001975751,CN 3001975752,3001975759,UA @@ -67873,9 +69552,7 @@ 3001975896,3001975919,GB 3001975920,3001975927,CN 3001975928,3001975935,AU -3001975936,3001976575,GB -3001976576,3001976607,US -3001976608,3001976623,GB +3001975936,3001976623,GB 3001976624,3001976639,CN 3001976640,3001976655,RU 3001976656,3001976671,GB @@ -67892,8 +69569,7 @@ 3001977144,3001977151,TW 3001977152,3001977167,IN 3001977168,3001977183,LK -3001977184,3001977199,GB -3001977200,3001977215,SI +3001977184,3001977215,GB 3001977216,3001977247,RU 3001977248,3001977311,GB 3001977312,3001977343,CN @@ -68005,7 +69681,9 @@ 3002667008,3002669055,GE 3002669056,3002669199,DE 3002669200,3002669207,CH -3002669208,3002671103,DE +3002669208,3002669951,DE +3002669952,3002669983,US +3002669984,3002671103,DE 3002671104,3002673151,LV 3002673152,3002675199,GB 3002675200,3002677247,DK @@ -68203,18 +69881,20 @@ 3029598208,3029600255,VN 3029600256,3029601279,AU 3029601280,3029602303,IN +3029602304,3029604351,CN 3029604352,3029605375,AU 3029605376,3029606399,JP 3029606400,3029614591,IN 3029614592,3029630975,AU 3029630976,3029635071,VN 3029635072,3029637119,JP +3029637120,3029639167,CN 3029639168,3029643263,JP 3029643264,3029644287,AU 3029644288,3029645311,KR 3029645312,3029651455,JP 3029651456,3029653503,BD -3029655552,3029663743,CN +3029653504,3029663743,CN 3029663744,3029671935,BD 3029671936,3029680127,IN 3029680128,3029696511,MN @@ -68379,7 +70059,6 @@ 3054501888,3054534655,HK 3054534656,3054537727,PH 3054537728,3054538751,SG -3054538752,3054540799,BD 3054540800,3054541823,NZ 3054541824,3054542847,BD 3054542848,3054551039,ID @@ -68400,7 +70079,9 @@ 3054997504,3055001599,SG 3055001600,3055005695,ID 3055005696,3055007743,NZ +3055007744,3055009791,CN 3055009792,3055011839,AU +3055011840,3055013887,CN 3055013888,3055014911,JP 3055014912,3055015935,AU 3055015936,3055026175,JP @@ -68538,7 +70219,7 @@ 3075383296,3075385343,IN 3075385344,3075386367,MY 3075386368,3075387391,AU -3075387392,3075388415,JP +3075388416,3075389439,CN 3075389440,3075390463,IN 3075390464,3075391487,JP 3075391488,3075407871,KR @@ -68573,7 +70254,7 @@ 3076218880,3076227071,JP 3076227072,3076228095,CN 3076228096,3076229119,NP -3076231168,3076235263,CN +3076229120,3076235263,CN 3076235264,3076243455,VN 3076243456,3076259839,KR 3076259840,3076521983,CN @@ -68761,7 +70442,9 @@ 3098281712,3098281719,PA 3098281720,3098476543,US 3098476544,3098492927,CA -3098492928,3103784959,US +3098492928,3098502143,US +3098502144,3098502207,IN +3098502208,3103784959,US 3103784960,3107979263,EU 3120562176,3120594943,CO 3120594944,3120599039,AR @@ -68807,10 +70490,12 @@ 3123707904,3123970047,UY 3124232192,3124772863,AR 3124789248,3124822015,CR +3124822016,3124838399,EC 3124854784,3124887551,CL 3124887552,3124953087,EC 3125018624,3125149695,EC 3125280768,3125542911,PA +3125673984,3125805055,CL 3125805056,3126329343,CO 3126329344,3126853631,VE 3126853632,3126919167,AR @@ -68828,9 +70513,10 @@ 3129999360,3130261503,CO 3130523648,3130654719,AR 3131047936,3131310079,PE +3131572224,3131703295,CO 3132096512,3132162047,CR 3132227584,3132293119,EC -3132358656,3132424191,CO +3132358656,3132489727,CO 3132489728,3132555263,AR 3132620800,3132915711,VE 3132915712,3132948479,PA @@ -68938,7 +70624,9 @@ 3158395432,3158395439,DE 3158395440,3158395647,AT 3158395648,3158395663,DE -3158395664,3158396927,AT +3158395664,3158396287,AT +3158396288,3158396319,DE +3158396320,3158396927,AT 3158396928,3158398975,IT 3158398976,3158401023,ES 3158401024,3158403071,GB @@ -68952,7 +70640,12 @@ 3158417408,3158419455,NL 3158419456,3158421503,FR 3158421504,3158423551,GB -3158423552,3158425599,MT +3158423552,3158424063,MT +3158424064,3158424095,NL +3158424096,3158424127,MT +3158424128,3158424159,IE +3158424160,3158425567,MT +3158425568,3158425599,IE 3158425600,3158427647,NL 3158427648,3158429695,DE 3158429696,3158431743,RU @@ -69232,14 +70925,23 @@ 3162071040,3162087423,IR 3162087424,3162095615,SK 3162095616,3162103807,GE -3162103808,3162104319,FR -3162104320,3162108415,NL -3162108416,3162108671,FR -3162108672,3162108927,NL -3162108928,3162109951,FR -3162109952,3162110975,NL +3162103808,3162104831,FR +3162104832,3162110975,NL 3162110976,3162111103,FR -3162111104,3162111999,NL +3162111104,3162111231,NL +3162111232,3162111295,CA +3162111296,3162111359,NL +3162111360,3162111391,CA +3162111392,3162111471,NL +3162111472,3162111487,BE +3162111488,3162111519,CA +3162111520,3162111599,NL +3162111600,3162111615,BE +3162111616,3162111727,NL +3162111728,3162111775,CH +3162111776,3162111871,NL +3162111872,3162111903,CH +3162111904,3162111999,NL 3162112000,3162120191,PL 3162120192,3162128383,GB 3162128384,3162129407,NL @@ -69324,9 +71026,11 @@ 3163161632,3163161663,DK 3163161664,3163161695,DE 3163161696,3163161727,BG -3163161728,3163161887,DE -3163161888,3163161919,US -3163161920,3163162111,DE +3163161728,3163161759,DE +3163161760,3163161791,BR +3163161792,3163161887,DE +3163161888,3163161951,US +3163161952,3163162111,DE 3163162112,3163162143,TR 3163162144,3163162207,DE 3163162208,3163162239,TR @@ -69334,7 +71038,9 @@ 3163162272,3163162303,NL 3163162304,3163162431,DE 3163162432,3163162463,CY -3163162464,3163163871,DE +3163162464,3163163807,DE +3163163808,3163163839,CY +3163163840,3163163871,DE 3163163872,3163163903,RU 3163163904,3163163935,GR 3163163936,3163163967,VG @@ -69344,9 +71050,9 @@ 3163164128,3163164159,US 3163164160,3163164191,DE 3163164192,3163164223,RU -3163164224,3163164351,DE -3163164352,3163164383,BG -3163164384,3163164447,DE +3163164224,3163164287,DE +3163164288,3163164319,CL +3163164320,3163164447,DE 3163164448,3163164479,RU 3163164480,3163164511,US 3163164512,3163164543,HU @@ -69365,13 +71071,16 @@ 3163166592,3163167775,DE 3163167776,3163167807,GR 3163167808,3163167839,US -3163167840,3163167967,DE +3163167840,3163167871,BR +3163167872,3163167967,DE 3163167968,3163167999,TR -3163168000,3163168031,BG +3163168000,3163168031,DE 3163168032,3163168095,RU 3163168096,3163168127,TR 3163168128,3163168159,SA -3163168160,3163168319,DE +3163168160,3163168191,BR +3163168192,3163168223,AT +3163168224,3163168319,DE 3163168320,3163168351,DK 3163168352,3163168383,DE 3163168384,3163168415,TR @@ -69381,7 +71090,8 @@ 3163168576,3163168607,US 3163168608,3163168671,DE 3163168672,3163168703,US -3163168704,3163169823,DE +3163168704,3163168735,GR +3163168736,3163169823,DE 3163169824,3163169855,RU 3163169856,3163169951,DE 3163169952,3163169983,GB @@ -69395,7 +71105,8 @@ 3163170208,3163170239,RU 3163170240,3163170303,DE 3163170304,3163170335,IT -3163170336,3163170495,DE +3163170336,3163170367,RO +3163170368,3163170495,DE 3163170496,3163170527,BG 3163170528,3163170591,DE 3163170592,3163170623,BR @@ -69409,7 +71120,8 @@ 3163171936,3163171967,BR 3163171968,3163172127,DE 3163172128,3163172159,US -3163172160,3163172319,DE +3163172160,3163172191,TR +3163172192,3163172319,DE 3163172320,3163172351,NL 3163172352,3163172383,GB 3163172384,3163172447,DE @@ -69424,13 +71136,13 @@ 3163174048,3163174079,DK 3163174080,3163174111,DE 3163174112,3163174143,BE -3163174144,3163174271,DE -3163174272,3163174303,BG +3163174144,3163174303,DE 3163174304,3163174335,IL 3163174336,3163174367,RU 3163174368,3163174495,DE 3163174496,3163174527,US -3163174528,3163174591,DE +3163174528,3163174559,DE +3163174560,3163174591,PL 3163174592,3163174623,SE 3163174624,3163174655,GB 3163174656,3163174719,DE @@ -69439,13 +71151,12 @@ 3163174784,3163174815,DE 3163174816,3163174847,CA 3163174848,3163174879,TR -3163174880,3163176063,DE -3163176064,3163176095,US -3163176096,3163176127,DE +3163174880,3163176127,DE 3163176128,3163176159,US 3163176160,3163176255,DE 3163176256,3163176287,UG -3163176288,3163176479,DE +3163176288,3163176319,PT +3163176320,3163176479,DE 3163176480,3163176511,GB 3163176512,3163176575,DE 3163176576,3163176607,GR @@ -69508,8 +71219,8 @@ 3164947552,3164947583,DE 3164947584,3164947587,FR 3164947588,3164947591,DE -3164947592,3164947615,GB -3164947616,3164947619,FR +3164947592,3164947599,GB +3164947600,3164947619,FR 3164947620,3164947623,ES 3164947624,3164947627,GB 3164947628,3164947635,ES @@ -69538,12 +71249,16 @@ 3164949156,3164949159,FR 3164949160,3164949163,PL 3164949164,3164949183,FR -3164949184,3164949215,DE +3164949184,3164949199,GB +3164949200,3164949215,FR 3164949216,3164949219,PL 3164949220,3164949223,FR 3164949224,3164949231,PL 3164949232,3164949247,BE -3164949248,3164949279,FR +3164949248,3164949263,FR +3164949264,3164949271,PL +3164949272,3164949275,GB +3164949276,3164949279,CH 3164949280,3164949295,ES 3164949296,3164949327,FR 3164949328,3164949331,PL @@ -69615,8 +71330,7 @@ 3164951616,3164951663,GB 3164951664,3164951671,PL 3164951672,3164951675,FR -3164951676,3164951679,IT -3164951680,3164951683,ES +3164951676,3164951683,ES 3164951684,3164951687,GB 3164951688,3164951691,FR 3164951692,3164951695,CH @@ -69644,7 +71358,8 @@ 3164952032,3164952063,PL 3164952064,3164952191,FR 3164952192,3164952207,ES -3164952208,3164952219,NL +3164952208,3164952215,FR +3164952216,3164952219,NL 3164952220,3164952239,FR 3164952240,3164952243,DE 3164952244,3164952247,BE @@ -69799,7 +71514,7 @@ 3164960928,3164960935,ES 3164960936,3164960939,GB 3164960940,3164960943,DE -3164960944,3164960959,FR +3164960944,3164960959,PL 3164960960,3164961023,CH 3164961024,3164961151,FR 3164961152,3164961167,DE @@ -69819,7 +71534,8 @@ 3164961404,3164961471,FR 3164961472,3164961503,DE 3164961504,3164961519,GB -3164961520,3164961535,FR +3164961520,3164961527,IT +3164961528,3164961535,FR 3164961536,3164961551,ES 3164961552,3164961559,FR 3164961560,3164961563,DE @@ -69835,7 +71551,7 @@ 3164961744,3164961763,PL 3164961764,3164961767,IT 3164961768,3164961775,BE -3164961776,3164961783,FR +3164961776,3164961783,PL 3164961784,3164961791,ES 3164961792,3164961855,FR 3164961856,3164961871,PL @@ -69853,7 +71569,7 @@ 3164962032,3164962047,NL 3164962048,3164962079,FR 3164962080,3164962095,ES -3164962096,3164962111,FR +3164962096,3164962111,DE 3164962112,3164962143,GB 3164962144,3164962203,FR 3164962204,3164962207,PT @@ -69889,7 +71605,8 @@ 3164962640,3164962647,ES 3164962648,3164962655,PT 3164962656,3164962687,ES -3164962688,3164962715,FR +3164962688,3164962703,PL +3164962704,3164962715,FR 3164962716,3164962719,NL 3164962720,3164962723,FR 3164962724,3164962727,ES @@ -70066,7 +71783,7 @@ 3164973884,3164973887,DE 3164973888,3164973919,ES 3164973920,3164973935,FR -3164973936,3164973939,IE +3164973936,3164973939,GB 3164973940,3164973943,FR 3164973944,3164973951,PL 3164973952,3164974647,FR @@ -70154,10 +71871,12 @@ 3164977744,3164977759,PT 3164977760,3164977775,NL 3164977776,3164977791,PL -3164977792,3164977807,FR -3164977808,3164977823,DE +3164977792,3164977823,FR 3164977824,3164977839,GB -3164977840,3164977883,FR +3164977840,3164977871,FR +3164977872,3164977875,DE +3164977876,3164977879,ES +3164977880,3164977883,DE 3164977884,3164977887,GB 3164977888,3164977903,IT 3164977904,3164977919,NL @@ -70178,7 +71897,7 @@ 3164978512,3164978527,ES 3164978528,3164978543,FR 3164978544,3164978559,PT -3164978560,3164978575,GB +3164978560,3164978575,FR 3164978576,3164978591,ES 3164978592,3164978607,FR 3164978608,3164978623,CZ @@ -70399,6 +72118,7 @@ 3187965952,3187982335,AN 3187982336,3187998719,CL 3187998720,3188006911,AR +3188006912,3188015103,CL 3188015104,3188031487,HN 3188031488,3188039679,SV 3188047872,3188051967,CO @@ -70462,7 +72182,7 @@ 3188621312,3188625407,GT 3188625408,3188627455,AR 3188627456,3188628479,CR -3188629504,3188637695,AR +3188628480,3188637695,AR 3188637696,3188645887,PA 3188645888,3188662271,CO 3188670464,3188674559,HN @@ -70592,6 +72312,8 @@ 3194519552,3194535935,PY 3194552320,3194585087,AR 3194585088,3194589183,HN +3194589184,3194591231,AR +3194591232,3194592255,PA 3194593280,3194595327,AR 3194595328,3194596351,PA 3194596352,3194597375,HT @@ -70686,6 +72408,7 @@ 3195715584,3195731967,AR 3195731968,3195736063,EC 3195740160,3195744255,PA +3195744256,3195748351,EC 3195748352,3195752447,CL 3195756544,3195764735,AR 3195764736,3195768831,CR @@ -70697,6 +72420,7 @@ 3195809792,3195811839,PE 3195811840,3195813887,AR 3195813888,3195822079,DO +3195822080,3195830271,CO 3195830272,3195838463,AR 3195838464,3195840511,HN 3195846656,3195852799,AR @@ -70774,6 +72498,8 @@ 3201875968,3201880063,CO 3201880064,3201888255,AR 3201888256,3201892351,VE +3201892352,3201925119,AR +3201925120,3201957887,CL 3201957888,3202088959,PA 3202088960,3202220031,AR 3202220032,3202351103,PE @@ -72561,7 +74287,20 @@ 3229391872,3229394943,US 3229394944,3229408255,RU 3229408256,3229412095,US -3229412096,3229483007,DE +3229412096,3229470719,DE +3229470720,3229471231,IE +3229471232,3229471999,DE +3229472000,3229472255,IE +3229472256,3229472767,GB +3229472768,3229473791,DE +3229473792,3229474047,GB +3229474048,3229474303,DE +3229474304,3229475839,GB +3229475840,3229478911,DE +3229478912,3229480959,IE +3229480960,3229481471,DE +3229481472,3229482239,GB +3229482240,3229483007,DE 3229483008,3229499647,FI 3229499648,3229500671,US 3229548544,3229679615,US @@ -72579,7 +74318,9 @@ 3229705216,3229708287,US 3229745152,3229749759,FI 3229749760,3229750015,BE -3229750016,3229810687,FI +3229750016,3229764063,FI +3229764064,3229764095,AX +3229764096,3229810687,FI 3229810688,3229814015,US 3229814016,3229814271,AU 3229814272,3229815807,US @@ -73522,7 +75263,26 @@ 3231873024,3231875071,RU 3231875072,3231876095,PL 3231876096,3231877119,UA -3231906816,3231907839,RU +3231877120,3231878143,NL +3231878144,3231879167,UA +3231879168,3231881215,PL +3231881216,3231882239,UA +3231882240,3231883263,RU +3231883264,3231884287,UA +3231884288,3231885311,PL +3231885312,3231886335,DE +3231886336,3231888383,PL +3231888384,3231889407,RU +3231889408,3231890431,RO +3231890432,3231893503,RU +3231893504,3231894527,UA +3231894528,3231895551,RU +3231895552,3231896575,UA +3231896576,3231897599,RU +3231897600,3231898623,IE +3231898624,3231899647,SE +3231899648,3231903743,UA +3231903744,3231907839,RU 3231907840,3231916031,US 3231916032,3231948799,FI 3231973376,3232038911,AT @@ -73535,7 +75295,9 @@ 3232094208,3232094719,CH 3232094720,3232095231,US 3232095232,3232096255,GB -3232096256,3232104447,SE +3232096256,3232097279,SE +3232097280,3232097535,IT +3232097536,3232104447,SE 3232104448,3232107519,DE 3232107520,3232108543,RU 3232108544,3232129023,DE @@ -74512,6 +76274,10 @@ 3238630656,3238632959,GB 3238632960,3238633215,UA 3238633216,3238653951,DK +3238653952,3238655999,RU +3238656000,3238656255,GB +3238656256,3238656511,RU +3238656512,3238657023,UA 3238657024,3238657535,AT 3238657536,3238657791,DK 3238657792,3238658047,AT @@ -74755,7 +76521,6 @@ 3239452416,3239452671,HR 3239452672,3239464959,DE 3239464960,3239465215,IL -3239465216,3239465471,HR 3239465472,3239465727,AT 3239465728,3239465983,PL 3239465984,3239466239,UA @@ -74839,7 +76604,7 @@ 3239541504,3239541759,FR 3239541760,3239542015,GB 3239542016,3239542271,PL -3239542272,3239542527,UA +3239542272,3239542527,RU 3239542528,3239542783,FR 3239542784,3239544831,DE 3239544832,3239545087,GB @@ -75286,7 +77051,7 @@ 3240183616,3240183647,CR 3240183648,3240183807,NL 3240183808,3240184319,GB -3240184320,3240184831,UA +3240184320,3240184831,RU 3240184832,3240185343,GB 3240185344,3240185855,FR 3240185856,3240187391,RU @@ -75441,7 +77206,10 @@ 3240282368,3240282495,RO 3240282496,3240282879,SE 3240282880,3240283007,UA -3240283008,3240283135,PL +3240283008,3240283391,PL +3240283392,3240283647,TR +3240283648,3240283903,AT +3240283904,3240284159,DE 3240284160,3240285183,RU 3240285184,3240286207,PL 3240286208,3240287231,UA @@ -76240,6 +78008,7 @@ 3244826368,3244826623,CH 3244826624,3244826879,DE 3244826880,3244827135,MK +3244827136,3244827391,AT 3244827392,3244827647,GB 3244827648,3244827903,FR 3244827904,3244828159,BE @@ -76442,6 +78211,7 @@ 3244883200,3244883455,UA 3244883456,3244883711,CZ 3244883712,3244883967,NL +3244883968,3244884223,DE 3244884224,3244884479,FR 3244884480,3244884735,IR 3244884736,3244884991,NL @@ -76516,7 +78286,6 @@ 3244903168,3244903423,PL 3244903424,3244903679,FI 3244903680,3244903935,NO -3244903936,3244904191,TR 3244904192,3244904447,IT 3244904448,3244904703,GR 3244904704,3244904959,FR @@ -76720,6 +78489,9 @@ 3244999680,3245000703,IQ 3245000704,3245001727,UA 3245001728,3245002751,IL +3245002752,3245003263,PL +3245003264,3245003519,SE +3245003520,3245003775,CH 3245003776,3245004799,RU 3245004800,3245005823,PL 3245005824,3245006847,UA @@ -76734,7 +78506,7 @@ 3245017088,3245018111,PL 3245018112,3245019135,RU 3245019136,3245020159,SC -3245020160,3245021183,RU +3245020160,3245021183,CZ 3245021184,3245022207,UA 3245022208,3245023231,NO 3245023232,3245024255,PL @@ -77888,9 +79660,7 @@ 3248790784,3248791039,PL 3248791040,3248791295,BE 3248791296,3248791551,DE -3248791552,3248792487,GB -3248792488,3248792495,EU -3248792496,3248792511,GB +3248791552,3248792511,GB 3248792512,3248796607,EU 3248796608,3248796863,GB 3248796864,3248798975,EU @@ -78130,7 +79900,7 @@ 3249716736,3249717247,UA 3249717248,3249718271,LV 3249718272,3249719295,DE -3249719296,3249720319,GB +3249719296,3249720319,IR 3249720320,3249721343,IT 3249721344,3249721599,AT 3249721600,3249721855,BE @@ -78951,6 +80721,7 @@ 3252342144,3252342207,LU 3252342208,3252342239,GB 3252342240,3252342271,CH +3252342272,3252342527,IL 3252342528,3252342543,NO 3252342544,3252342559,GB 3252342560,3252342591,FR @@ -79026,8 +80797,8 @@ 3252408392,3252408415,LT 3252408416,3252408479,NO 3252408480,3252408511,LT -3252408512,3252408519,AF -3252408520,3252408575,LT +3252408512,3252408527,AF +3252408528,3252408575,LT 3252408576,3252408607,SO 3252408608,3252408639,LT 3252408640,3252408671,CF @@ -79059,7 +80830,8 @@ 3252409216,3252409223,BW 3252409224,3252409231,LT 3252409232,3252409247,UG -3252409248,3252409279,LT +3252409248,3252409263,CD +3252409264,3252409279,LT 3252409280,3252409295,SO 3252409296,3252409303,LT 3252409304,3252409343,SO @@ -79078,7 +80850,9 @@ 3252409632,3252409647,AO 3252409648,3252409727,LT 3252409728,3252409735,LR -3252409736,3252410383,LT +3252409736,3252409759,LT +3252409760,3252409775,BI +3252409776,3252410383,LT 3252410384,3252410391,BW 3252410392,3252410431,LT 3252410432,3252410463,BI @@ -79123,8 +80897,8 @@ 3252415128,3252415135,LT 3252415136,3252415159,IQ 3252415160,3252415167,BE -3252415168,3252415223,IQ -3252415224,3252415487,LT +3252415168,3252415231,IQ +3252415232,3252415487,LT 3252415488,3252415743,IQ 3252415744,3252415775,GB 3252415776,3252415967,LT @@ -79135,8 +80909,8 @@ 3252416960,3252417023,GN 3252417024,3252417279,LT 3252417280,3252417287,IQ -3252417288,3252417455,AF -3252417456,3252417791,LT +3252417288,3252417463,AF +3252417464,3252417791,LT 3252417792,3252417855,NG 3252417856,3252417919,LT 3252417920,3252417935,MW @@ -79280,8 +81054,8 @@ 3252436352,3252436383,GN 3252436384,3252436399,LR 3252436400,3252436407,SL -3252436408,3252436415,ML -3252436416,3252436447,LT +3252436408,3252436431,ML +3252436432,3252436447,LT 3252436448,3252436479,ER 3252436480,3252436991,LT 3252436992,3252437503,NG @@ -79373,7 +81147,7 @@ 3252456960,3252457471,NG 3252457472,3252460799,LT 3252460800,3252460831,AF -3252460832,3252460847,KP +3252460832,3252460847,US 3252460848,3252461055,LT 3252461056,3252461567,NO 3252461568,3252464383,LT @@ -80402,9 +82176,7 @@ 3255260336,3255260343,BE 3255260344,3255260347,LU 3255260348,3255261471,BE -3255261472,3255261503,LU -3255261504,3255261519,BE -3255261520,3255261535,LU +3255261472,3255261535,LU 3255261536,3255262799,BE 3255262800,3255262815,LU 3255262816,3255263295,BE @@ -80433,7 +82205,9 @@ 3255276576,3255276607,LU 3255276608,3255276671,BE 3255276672,3255276703,LU -3255276704,3255277247,BE +3255276704,3255276783,BE +3255276784,3255276799,LU +3255276800,3255277247,BE 3255277248,3255277255,LU 3255277256,3255277955,BE 3255277956,3255277959,LU @@ -80552,7 +82326,7 @@ 3255366144,3255367167,DK 3255367168,3255367679,RU 3255367680,3255368191,UA -3255368192,3255368703,LV +3255368192,3255368703,MD 3255368704,3255369215,CZ 3255369216,3255369727,GB 3255369728,3255370239,LU @@ -80736,7 +82510,13 @@ 3255743488,3255745535,SE 3255745536,3255746047,DK 3255746048,3255762943,SE -3255762944,3255799039,DE +3255762944,3255791615,DE +3255791616,3255792639,UA +3255792640,3255793663,RU +3255793664,3255794943,PL +3255794944,3255795199,RU +3255795200,3255795711,UA +3255795712,3255799039,DE 3255799040,3255799295,SE 3255799296,3255800575,DE 3255800576,3255800831,UA @@ -82273,7 +84053,9 @@ 3258948608,3258949631,RU 3258949632,3258972159,GR 3258972160,3258974207,NO -3258974208,3259023103,DE +3258974208,3259006079,DE +3259006080,3259006111,BE +3259006112,3259023103,DE 3259023104,3259023107,ES 3259023108,3259031655,DE 3259031656,3259031659,ES @@ -82509,11 +84291,7 @@ 3259657728,3259657887,BE 3259657888,3259657919,GB 3259657920,3259657983,BE -3259657984,3259660287,GB -3259660288,3259660295,CH -3259660296,3259660327,GB -3259660328,3259660335,CH -3259660336,3259660543,GB +3259657984,3259660543,GB 3259660544,3259660799,CH 3259660800,3259695871,GB 3259695872,3259695903,ES @@ -82521,11 +84299,7 @@ 3259696640,3259696895,ES 3259696896,3259701759,GB 3259701760,3259702303,DE -3259702304,3259751423,GB -3259751424,3259751431,FR -3259751432,3259751551,GB -3259751552,3259751615,FR -3259751616,3259752191,GB +3259702304,3259752191,GB 3259752192,3259752447,FR 3259752448,3259760639,GB 3259760640,3259814399,DE @@ -82785,6 +84559,7 @@ 3261820928,3261821183,RO 3261821184,3261821439,AT 3261821440,3261821695,NL +3261821696,3261821951,UA 3261821952,3261822207,RU 3261822208,3261822463,UA 3261822464,3261822719,GB @@ -83251,7 +85026,7 @@ 3262472992,3262472995,DE 3262472996,3262472999,FR 3262473000,3262473003,HU -3262473004,3262473007,NO +3262473004,3262473007,DE 3262473008,3262473011,US 3262473012,3262473015,DE 3262473016,3262473019,AR @@ -83647,8 +85422,7 @@ 3262475460,3262475463,FR 3262475464,3262475467,DE 3262475468,3262475471,ES -3262475472,3262475475,DE -3262475476,3262475479,ES +3262475472,3262475479,DE 3262475480,3262475483,US 3262475484,3262475487,IT 3262475488,3262475491,MX @@ -84563,7 +86337,7 @@ 3262478749,3262478751,FR 3262478752,3262478752,PT 3262478753,3262478753,IE -3262478754,3262478754,ES +3262478754,3262478754,DE 3262478755,3262478755,DK 3262478756,3262478756,DE 3262478757,3262478757,AT @@ -84610,7 +86384,7 @@ 3262478801,3262478801,DE 3262478802,3262478802,IT 3262478803,3262478803,ES -3262478804,3262478804,SK +3262478804,3262478804,DE 3262478805,3262478806,GR 3262478807,3262478807,CH 3262478808,3262478808,HU @@ -85368,9 +87142,7 @@ 3262479903,3262479903,CH 3262479904,3262479908,DE 3262479909,3262479909,SE -3262479910,3262479912,DE -3262479913,3262479913,NO -3262479914,3262479914,DE +3262479910,3262479914,DE 3262479915,3262479915,FR 3262479916,3262479917,DE 3262479918,3262479918,GB @@ -86290,13 +88062,14 @@ 3264612512,3264612591,FR 3264612592,3264613119,GB 3264613120,3264614143,FR -3264614144,3264614431,GB +3264614144,3264614399,GB +3264614400,3264614431,NL 3264614432,3264614447,FR -3264614448,3264614463,GB +3264614448,3264614463,NL 3264614464,3264614527,FR -3264614528,3264614559,GB +3264614528,3264614559,NL 3264614560,3264614591,FR -3264614592,3264614623,GB +3264614592,3264614623,NL 3264614624,3264614655,FR 3264614656,3264615935,GB 3264615936,3264615999,FR @@ -86716,6 +88489,7 @@ 3265918976,3265919231,NL 3265919232,3265919487,FR 3265919488,3265919743,RU +3265919744,3265919999,DE 3265920000,3265920255,CZ 3265920256,3265920511,TR 3265920512,3265921023,GB @@ -87249,11 +89023,9 @@ 3267672576,3267672831,AT 3267672832,3267672847,FR 3267672848,3267672855,EU -3267672856,3267672927,FR -3267672928,3267672943,EU -3267672944,3267672999,FR -3267673000,3267673007,EU -3267673008,3267673015,FR +3267672856,3267672935,FR +3267672936,3267672943,EU +3267672944,3267673015,FR 3267673016,3267673023,EU 3267673024,3267673087,FR 3267673088,3267673439,DE @@ -87715,7 +89487,6 @@ 3268739584,3268739839,DE 3268740096,3268740351,IL 3268740352,3268740607,DE -3268740608,3268740863,UA 3268740864,3268741119,CH 3268741120,3268741375,FR 3268741376,3268741887,LV @@ -87850,7 +89621,8 @@ 3269285216,3269285311,EU 3269285312,3269285327,DE 3269285328,3269285343,FR -3269285344,3269285631,EU +3269285344,3269285375,GB +3269285376,3269285631,EU 3269285632,3269285887,DE 3269285888,3269286399,EU 3269286400,3269286463,DE @@ -88966,7 +90738,8 @@ 3272400904,3272400927,GB 3272400928,3272401023,EU 3272401024,3272401087,GB -3272401088,3272401407,EU +3272401088,3272401279,EU +3272401280,3272401407,NL 3272401408,3272401919,ES 3272401920,3272401951,GB 3272401952,3272401983,EU @@ -89460,6 +91233,9 @@ 3273025536,3273026559,RU 3273026560,3273028607,UA 3273028608,3273029631,PL +3273029632,3273029887,CH +3273029888,3273030143,GB +3273030144,3273030655,RU 3273030656,3273031679,PL 3273031680,3273032191,GB 3273032192,3273033215,UA @@ -89490,7 +91266,9 @@ 3273051968,3273052039,IT 3273052040,3273052047,NL 3273052048,3273052063,GR -3273052064,3273052415,IT +3273052064,3273052087,IT +3273052088,3273052095,DE +3273052096,3273052415,IT 3273052416,3273052671,GB 3273052672,3273052927,FR 3273052928,3273053183,DE @@ -90561,7 +92339,7 @@ 3274964928,3274964991,GB 3274964992,3275030527,DE 3275030528,3275096063,ES -3275096064,3275105279,RU +3275096064,3275104767,RU 3275105280,3275105791,GB 3275105792,3275106303,NL 3275106304,3275106815,RU @@ -90614,8 +92392,8 @@ 3275423752,3275423775,EU 3275423776,3275423807,GB 3275423808,3275423871,EU -3275423872,3275424711,GB -3275424712,3275425791,EU +3275423872,3275424719,GB +3275424720,3275425791,EU 3275425792,3275427271,GB 3275427272,3275427279,EU 3275427280,3275427615,GB @@ -90637,16 +92415,16 @@ 3275433984,3275437567,GB 3275437568,3275438079,EU 3275438080,3275438623,GB -3275438624,3275440127,EU +3275438624,3275438719,EU +3275438720,3275439103,GB +3275439104,3275440127,EU 3275440128,3275440639,GB 3275440640,3275442175,EU 3275442176,3275442719,GB 3275442720,3275444223,EU 3275444224,3275444735,GB 3275444736,3275446271,EU -3275446272,3275447063,GB -3275447064,3275447071,EU -3275447072,3275447087,GB +3275446272,3275447087,GB 3275447088,3275448319,EU 3275448320,3275449519,GB 3275449520,3275449527,FR @@ -90654,7 +92432,9 @@ 3275450880,3275451231,EU 3275451232,3275451263,GB 3275451264,3275452415,EU -3275452416,3275454127,GB +3275452416,3275453423,GB +3275453424,3275453439,EU +3275453440,3275454127,GB 3275454128,3275454143,EU 3275454144,3275457023,GB 3275457024,3275457791,FK @@ -90676,11 +92456,7 @@ 3275468768,3275468799,IE 3275468800,3275469071,GB 3275469072,3275469087,IE -3275469088,3275469823,GB -3275469824,3275469951,EU -3275469952,3275469983,GB -3275469984,3275470335,EU -3275470336,3275471871,GB +3275469088,3275471871,GB 3275471872,3275489279,EU 3275489280,3275497471,GB 3275497472,3275505663,DE @@ -91162,7 +92938,9 @@ 3276015416,3276015423,GB 3276015424,3276015559,FR 3276015560,3276015599,GB -3276015600,3276015879,FR +3276015600,3276015855,FR +3276015856,3276015863,GB +3276015864,3276015879,FR 3276015880,3276015887,GB 3276015888,3276016391,FR 3276016392,3276016399,GB @@ -91204,7 +92982,9 @@ 3276018096,3276018127,GB 3276018128,3276018143,FR 3276018144,3276018159,GB -3276018160,3276018375,FR +3276018160,3276018239,FR +3276018240,3276018271,GB +3276018272,3276018375,FR 3276018376,3276018383,GB 3276018384,3276018431,FR 3276018432,3276018447,GB @@ -91484,18 +93264,18 @@ 3276033928,3276033951,GB 3276033952,3276036271,FR 3276036272,3276036279,DE -3276036280,3276036343,FR -3276036344,3276036351,GB -3276036352,3276036383,FR +3276036280,3276036383,FR 3276036384,3276036415,GB 3276036416,3276036591,FR 3276036592,3276036607,GB -3276036608,3276037199,FR +3276036608,3276037127,FR +3276037128,3276037135,GB +3276037136,3276037199,FR 3276037200,3276037215,GB 3276037216,3276037359,FR 3276037360,3276037375,GB -3276037376,3276037695,FR -3276037696,3276037743,GB +3276037376,3276037703,FR +3276037704,3276037743,GB 3276037744,3276037887,FR 3276037888,3276037903,GB 3276037904,3276037959,FR @@ -91512,7 +93292,9 @@ 3276038800,3276038815,GB 3276038816,3276038847,FR 3276038848,3276038911,GB -3276038912,3276039199,FR +3276038912,3276038919,FR +3276038920,3276039167,GB +3276039168,3276039199,FR 3276039200,3276039247,GB 3276039248,3276039279,FR 3276039280,3276039295,GB @@ -91574,10 +93356,8 @@ 3276041920,3276041951,GB 3276041952,3276041967,FR 3276041968,3276041983,GB -3276041984,3276041999,FR -3276042000,3276042007,GB -3276042008,3276042015,FR -3276042016,3276042143,GB +3276041984,3276042031,FR +3276042032,3276042143,GB 3276042144,3276042175,FR 3276042176,3276042191,GB 3276042192,3276042207,FR @@ -91640,9 +93420,9 @@ 3276045480,3276045487,GB 3276045488,3276045543,FR 3276045544,3276045551,GB -3276045552,3276045567,FR -3276045568,3276045599,GB -3276045600,3276045631,FR +3276045552,3276045575,FR +3276045576,3276045607,GB +3276045608,3276045631,FR 3276045632,3276045647,GB 3276045648,3276045767,FR 3276045768,3276045783,GB @@ -93817,6 +95597,7 @@ 3276425472,3276425727,DE 3276425728,3276425983,RU 3276425984,3276426239,IT +3276426240,3276426495,CZ 3276426496,3276426751,DK 3276426752,3276427007,RO 3276427008,3276427263,PL @@ -93863,9 +95644,7 @@ 3276474880,3276475015,EU 3276475016,3276475135,IT 3276475136,3276475903,EU -3276475904,3276475951,IT -3276475952,3276475983,EU -3276475984,3276476039,IT +3276475904,3276476039,IT 3276476040,3276476047,EU 3276476048,3276476111,IT 3276476112,3276476119,GB @@ -93891,13 +95670,11 @@ 3276479616,3276479647,EU 3276479648,3276479743,FR 3276479744,3276479999,EU -3276480000,3276480095,FR -3276480096,3276480111,EU -3276480112,3276480159,FR +3276480000,3276480159,FR 3276480160,3276480191,EU 3276480192,3276480319,FR -3276480320,3276480351,EU -3276480352,3276480375,FR +3276480320,3276480335,EU +3276480336,3276480375,FR 3276480376,3276480399,EU 3276480400,3276480439,FR 3276480440,3276480455,EU @@ -93974,8 +95751,8 @@ 3276499520,3276499615,DE 3276499616,3276499679,EU 3276499680,3276499759,DE -3276499760,3276499799,EU -3276499800,3276499823,DE +3276499760,3276499791,EU +3276499792,3276499823,DE 3276499824,3276499935,EU 3276499936,3276499999,DE 3276500000,3276500031,CH @@ -94087,8 +95864,8 @@ 3276520192,3276520223,BE 3276520224,3276520255,GB 3276520256,3276520415,BE -3276520416,3276520431,EU -3276520432,3276520447,BE +3276520416,3276520423,EU +3276520424,3276520447,BE 3276520448,3276520463,EU 3276520464,3276520591,SE 3276520592,3276520607,EU @@ -94102,8 +95879,8 @@ 3276520928,3276520943,EU 3276520944,3276521215,SE 3276521216,3276521471,EU -3276521472,3276521479,RO -3276521480,3276521983,EU +3276521472,3276521487,RO +3276521488,3276521983,EU 3276521984,3276523519,NL 3276523520,3276523791,EU 3276523792,3276523911,NO @@ -94153,7 +95930,9 @@ 3276528368,3276528399,BE 3276528400,3276528447,EU 3276528448,3276528503,BE -3276528504,3276528559,EU +3276528504,3276528511,EU +3276528512,3276528519,BE +3276528520,3276528559,EU 3276528560,3276528567,BE 3276528568,3276528575,EU 3276528576,3276528719,BE @@ -94204,8 +95983,8 @@ 3276533920,3276533951,IE 3276533952,3276533983,EU 3276533984,3276534015,IE -3276534016,3276534399,EU -3276534400,3276534415,GB +3276534016,3276534271,EU +3276534272,3276534415,GB 3276534416,3276534431,NL 3276534432,3276534463,GB 3276534464,3276534495,FR @@ -94304,6 +96083,7 @@ 3276701696,3276709887,SE 3276709888,3276718079,DE 3276718080,3276726271,IT +3276726272,3276727295,SE 3276727296,3276728319,ES 3276728320,3276729343,UA 3276729344,3276730367,PL @@ -94726,8 +96506,8 @@ 3276872480,3276872511,GB 3276872512,3276872703,DE 3276872704,3276873727,GB -3276873728,3276874495,ES -3276874496,3276874751,GB +3276873728,3276874559,ES +3276874560,3276874751,GB 3276874752,3276875007,NL 3276875008,3276875263,CH 3276875264,3276875775,NL @@ -94735,18 +96515,10 @@ 3276876032,3276876287,DK 3276876288,3276876415,NL 3276876416,3276876431,GB -3276876432,3276876479,NL -3276876480,3276876799,GB -3276876800,3276876823,AT -3276876824,3276877303,GB -3276877304,3276877311,AT -3276877312,3276877535,GB +3276876432,3276876541,NL +3276876542,3276877535,GB 3276877536,3276877551,AT -3276877552,3276877559,GB -3276877560,3276877567,AT -3276877568,3276877815,GB -3276877816,3276877823,AT -3276877824,3276878079,GB +3276877552,3276878079,GB 3276878080,3276878335,BG 3276878336,3276878591,GB 3276878592,3276878847,FR @@ -94759,10 +96531,7 @@ 3276881152,3276881279,GB 3276881280,3276881407,DE 3276881408,3276881919,FR -3276881920,3276883391,IT -3276883392,3276883455,GB -3276883456,3276883839,IT -3276883840,3276883967,GB +3276881920,3276883967,IT 3276883968,3276884487,PL 3276884488,3276884735,GB 3276884736,3276884991,PL @@ -94770,7 +96539,9 @@ 3276886016,3276886271,RO 3276886272,3276886527,GB 3276886528,3276886943,DE -3276886944,3276887071,GB +3276886944,3276886959,GB +3276886960,3276886975,DE +3276886976,3276887071,GB 3276887072,3276888063,DE 3276888064,3276888575,GB 3276888576,3276888831,AT @@ -94782,11 +96553,11 @@ 3276890192,3276890367,GB 3276890368,3276890623,US 3276890624,3276891135,IT -3276891136,3276891391,GB +3276891136,3276891391,BE 3276891392,3276892159,US -3276892160,3276893695,IT -3276893696,3276894207,GB -3276894208,3276895999,IT +3276892160,3276893950,IT +3276893951,3276893951,GB +3276893952,3276895999,IT 3276896000,3276896255,CZ 3276896256,3276896767,BE 3276896768,3276896831,GB @@ -94803,14 +96574,12 @@ 3276900040,3276900047,GB 3276900048,3276900351,CH 3276900352,3276900607,GB -3276900608,3276901391,CH -3276901392,3276901407,GB -3276901408,3276901439,CH -3276901440,3276901631,GB +3276900608,3276901471,CH +3276901472,3276901631,GB 3276901632,3276902151,CH 3276902152,3276902159,GB -3276902160,3276902175,CH -3276902176,3276902207,GB +3276902160,3276902191,CH +3276902192,3276902207,GB 3276902208,3276902271,CH 3276902272,3276902399,GB 3276902400,3276902583,SE @@ -94837,11 +96606,14 @@ 3276906496,3276906823,NL 3276906824,3276906831,CH 3276906832,3276907551,NL -3276907552,3276907775,GB +3276907552,3276907567,BE +3276907568,3276907643,NL +3276907644,3276907775,GB 3276907776,3276908159,NL -3276908160,3276908287,GB -3276908288,3276908799,NL -3276908800,3276909567,GB +3276908160,3276908175,CH +3276908176,3276908287,GB +3276908288,3276909055,NL +3276909056,3276909567,GB 3276909568,3276910591,NL 3276910592,3276910967,IT 3276910968,3276910975,GB @@ -94849,9 +96621,13 @@ 3276912616,3276912623,GB 3276912624,3276913183,IT 3276913184,3276913215,GB -3276913216,3276913919,IT +3276913216,3276913359,IT +3276913360,3276913375,GB +3276913376,3276913919,IT 3276913920,3276913983,US -3276913984,3276914143,IT +3276913984,3276914079,IT +3276914080,3276914095,GB +3276914096,3276914143,IT 3276914144,3276914159,GB 3276914160,3276914687,IT 3276914688,3276914943,ES @@ -94883,12 +96659,12 @@ 3276921344,3276921399,GB 3276921400,3276921403,DK 3276921404,3276921599,GB -3276921600,3276921607,DE -3276921608,3276921615,GB -3276921616,3276922623,DE +3276921600,3276922623,DE 3276922624,3276923431,FR 3276923432,3276923439,GB -3276923440,3276924071,FR +3276923440,3276923447,FR +3276923448,3276923455,DE +3276923456,3276924071,FR 3276924072,3276924079,GB 3276924080,3276926751,FR 3276926752,3276926783,GB @@ -95295,8 +97071,11 @@ 3277885440,3277885727,LB 3277885728,3277885951,IT 3277885952,3277886207,LB -3277886208,3277886975,IR -3277886976,3277888255,IT +3277886208,3277886463,IT +3277886464,3277886719,IQ +3277886720,3277886975,IR +3277886976,3277887487,IQ +3277887488,3277888255,IT 3277888256,3277888319,LB 3277888320,3277889535,IT 3277889536,3277897727,RU @@ -95605,8 +97384,7 @@ 3278939048,3278939055,FR 3278939056,3278939063,DE 3278939064,3278939067,ES -3278939068,3278939071,IT -3278939072,3278939075,DE +3278939068,3278939075,DE 3278939076,3278939079,CH 3278939080,3278939083,DE 3278939084,3278939087,IT @@ -96309,9 +98087,7 @@ 3278942836,3278942837,DE 3278942838,3278942838,ES 3278942839,3278942839,FR -3278942840,3278942841,DE -3278942842,3278942842,IT -3278942843,3278942843,DE +3278942840,3278942843,DE 3278942844,3278942844,ES 3278942845,3278942850,DE 3278942851,3278942851,HU @@ -98098,11 +99874,15 @@ 3279601664,3279609855,CZ 3279609856,3279618047,RU 3279618048,3279683583,UA -3279683584,3279744391,DE +3279683584,3279723215,DE +3279723216,3279723223,A2 +3279723224,3279744391,DE 3279744392,3279744399,PL 3279744400,3279752959,DE 3279752960,3279753215,CH -3279753216,3279897583,DE +3279753216,3279840663,DE +3279840664,3279840671,A2 +3279840672,3279897583,DE 3279897584,3279897591,GB 3279897592,3279946751,DE 3279946752,3279947775,SE @@ -98118,6 +99898,34 @@ 3279972352,3279974399,AT 3279974400,3279976447,PL 3279976448,3279978495,RU +3279978496,3279978751,SI +3279978752,3279979007,NL +3279979008,3279979263,RO +3279979264,3279979519,SA +3279979520,3279979775,BG +3279979776,3279980031,RU +3279980032,3279980287,BG +3279980288,3279980543,PL +3279980544,3279980799,RO +3279980800,3279981055,RU +3279981056,3279981311,GB +3279981312,3279981567,UA +3279981568,3279981823,PL +3279981824,3279982079,SI +3279982080,3279982335,PL +3279982336,3279982591,FR +3279982592,3279982847,DE +3279982848,3279983103,IL +3279983104,3279983615,FR +3279983616,3279983871,PL +3279983872,3279984127,FR +3279984128,3279984383,AT +3279984384,3279984639,RO +3279984640,3279985151,RU +3279985152,3279985407,PL +3279985408,3279985663,CZ +3279985664,3279985919,DE +3279985920,3279986687,RU 3279986688,3279987199,NL 3279987200,3279987711,RU 3279987712,3279988223,RO @@ -99021,10 +100829,8 @@ 3280949760,3280950655,GB 3280950656,3280950687,DE 3280950688,3280951039,GB -3280951040,3280951807,DE -3280951808,3280951819,GB -3280951820,3280951831,DE -3280951832,3280952067,GB +3280951040,3280952063,DE +3280952064,3280952067,GB 3280952068,3280952079,DE 3280952080,3280952087,GB 3280952088,3280952095,DE @@ -99093,9 +100899,7 @@ 3280954496,3280955391,GB 3280955392,3280955419,DE 3280955420,3280955423,GB -3280955424,3280955519,DE -3280955520,3280955647,GB -3280955648,3280955791,DE +3280955424,3280955791,DE 3280955792,3280955799,GB 3280955800,3280956143,DE 3280956144,3280956415,GB @@ -99990,6 +101794,7 @@ 3281351168,3281351423,UA 3281351424,3281351679,AT 3281351680,3281351935,TR +3281351936,3281352191,UA 3281352192,3281352447,PL 3281352448,3281352703,RO 3281352704,3281352959,DE @@ -100002,6 +101807,9 @@ 3281354752,3281359071,SE 3281359072,3281359103,AN 3281359104,3281371135,SE +3281371136,3281372159,RS +3281372160,3281372671,RU +3281372672,3281373183,PL 3281373184,3281375231,RU 3281375232,3281377279,ES 3281377280,3281379327,AT @@ -100218,6 +102026,7 @@ 3282743040,3282743295,CH 3282743296,3282743551,CY 3282743552,3282743807,SI +3282743808,3282744063,BG 3282744064,3282744319,UA 3282744320,3282744575,SA 3282744576,3282744831,LT @@ -100358,7 +102167,7 @@ 3283485184,3283485439,DE 3283485440,3283485695,UA 3283485696,3283485951,GB -3283485952,3283486207,SE +3283485952,3283486207,US 3283486208,3283486463,AT 3283486464,3283486719,UA 3283486720,3283486975,RU @@ -100634,8 +102443,7 @@ 3283584032,3283584063,DE 3283584064,3283584127,IE 3283584128,3283585023,ZA -3283585024,3283585031,EU -3283585032,3283585279,ES +3283585024,3283585279,ES 3283585280,3283585535,ZA 3283585536,3283585679,ES 3283585680,3283585695,IE @@ -101109,7 +102917,7 @@ 3284079616,3284080127,DE 3284080128,3284080639,BE 3284080640,3284081151,UA -3284081152,3284081663,CH +3284081152,3284081663,DE 3284081664,3284082175,IE 3284082176,3284082687,NO 3284082688,3284083199,RU @@ -101191,7 +102999,7 @@ 3284125184,3284125695,RO 3284125696,3284126207,UA 3284126208,3284127231,CZ -3284127232,3284127743,NL +3284127232,3284127743,HK 3284127744,3284128255,RU 3284128256,3284128767,GB 3284128768,3284129279,SE @@ -101465,11 +103273,8 @@ 3285456352,3285456383,EU 3285456384,3285456639,DE 3285456640,3285456703,DK -3285456704,3285456711,DE -3285456712,3285456735,EU -3285456736,3285456743,DK -3285456744,3285456751,EU -3285456752,3285456767,DK +3285456704,3285456735,EU +3285456736,3285456767,DK 3285456768,3285456831,EU 3285456832,3285456871,DK 3285456872,3285456879,EU @@ -101608,8 +103413,8 @@ 3285467392,3285467663,EU 3285467664,3285467679,DE 3285467680,3285467775,EU -3285467776,3285467807,DE -3285467808,3285467831,EU +3285467776,3285467823,DE +3285467824,3285467831,EU 3285467832,3285467839,DE 3285467840,3285467935,EU 3285467936,3285467951,DE @@ -101631,9 +103436,7 @@ 3285471744,3285471807,EU 3285471808,3285471871,DE 3285471872,3285471935,EU -3285471936,3285471967,DE -3285471968,3285471999,EU -3285472000,3285472127,DE +3285471936,3285472127,DE 3285472128,3285472159,EU 3285472160,3285472175,DE 3285472176,3285472183,EU @@ -101647,9 +103450,7 @@ 3285472512,3285473327,EU 3285473328,3285473343,DE 3285473344,3285473439,EU -3285473440,3285473527,DE -3285473528,3285473535,EU -3285473536,3285473567,DE +3285473440,3285473567,DE 3285473568,3285473583,EU 3285473584,3285473591,GB 3285473592,3285473599,DE @@ -101660,9 +103461,7 @@ 3285474048,3285474079,EU 3285474080,3285474095,DE 3285474096,3285474175,EU -3285474176,3285474231,DE -3285474232,3285474239,EU -3285474240,3285474271,DE +3285474176,3285474271,DE 3285474272,3285474303,EU 3285474304,3285474319,DE 3285474320,3285474335,EU @@ -101784,9 +103583,7 @@ 3285492736,3285493759,EU 3285493760,3285493775,ES 3285493776,3285493783,EU -3285493784,3285493791,ES -3285493792,3285493807,EU -3285493808,3285493887,ES +3285493784,3285493887,ES 3285493888,3285493967,EU 3285493968,3285493983,ES 3285493984,3285493991,GB @@ -102718,6 +104515,8 @@ 3285766656,3285767679,UA 3285767680,3285768191,FR 3285768192,3285768703,RU +3285768704,3285768959,RO +3285768960,3285769215,PL 3285769216,3285769727,DE 3285769728,3285770239,RO 3285770240,3285770495,NL @@ -102815,8 +104614,10 @@ 3285926408,3285926415,ES 3285926416,3285926431,GB 3285926432,3285926463,CH -3285926464,3285926591,GB -3285926592,3285926847,EU +3285926464,3285926479,GB +3285926480,3285926783,EU +3285926784,3285926799,GB +3285926800,3285926847,EU 3285926848,3285926911,GB 3285926912,3285927423,DE 3285927424,3285927679,GB @@ -102842,7 +104643,8 @@ 3285930656,3285930671,NL 3285930672,3285930679,BE 3285930680,3285930687,DE -3285930688,3285930751,EU +3285930688,3285930703,GB +3285930704,3285930751,EU 3285930752,3285931007,GB 3285931008,3285932031,EU 3285932032,3285932287,NL @@ -102881,8 +104683,8 @@ 3285943552,3285943567,GB 3285943568,3285943575,ES 3285943576,3285943583,DE -3285943584,3285943615,GB -3285943616,3285943679,EU +3285943584,3285943631,GB +3285943632,3285943679,EU 3285943680,3285943807,GB 3285943808,3285944319,EU 3285944320,3285944831,US @@ -102907,7 +104709,8 @@ 3285949696,3285949823,ES 3285949824,3285949855,EU 3285949856,3285949887,ES -3285949888,3285949951,EU +3285949888,3285949903,GB +3285949904,3285949951,EU 3285949952,3285950463,ES 3285950464,3285950719,GB 3285950720,3285950783,US @@ -102921,7 +104724,8 @@ 3285951744,3285951999,GB 3285952000,3285952255,IT 3285952256,3285952511,SA -3285952512,3285955583,EU +3285952512,3285953535,CA +3285953536,3285955583,EU 3285955584,3285955839,DE 3285955840,3285956095,ES 3285956096,3285956351,FR @@ -102939,7 +104743,8 @@ 3285959936,3285960191,ES 3285960192,3285960447,FR 3285960448,3285960703,GB -3285960704,3285962751,EU +3285960704,3285961727,CA +3285961728,3285962751,EU 3285962752,3285963775,DE 3285963776,3285964287,GB 3285964288,3285964799,FR @@ -103189,7 +104994,76 @@ 3286790912,3286791679,GB 3286791680,3286791935,AT 3286791936,3286794239,GB -3286794240,3286892543,DE +3286794240,3286795263,DE +3286795264,3286795519,LR +3286795520,3286795775,DE +3286795776,3286796031,LR +3286796032,3286796287,DE +3286796288,3286796543,LR +3286796544,3286796799,DE +3286796800,3286797055,RO +3286797056,3286797311,DE +3286797312,3286797567,HU +3286797568,3286798847,DE +3286798848,3286799359,IT +3286799360,3286801407,DE +3286801408,3286801663,NL +3286801664,3286802943,DE +3286802944,3286803199,AU +3286803200,3286805503,DE +3286805504,3286806527,GB +3286806528,3286808063,DE +3286808064,3286808575,GR +3286808576,3286809087,EG +3286809088,3286809599,DE +3286809600,3286809855,GB +3286809856,3286810111,PT +3286810112,3286812927,DE +3286812928,3286813183,PT +3286813184,3286813695,EG +3286813696,3286813951,DE +3286813952,3286814207,AU +3286814208,3286814719,HU +3286814720,3286819839,DE +3286819840,3286820863,NZ +3286820864,3286826751,DE +3286826752,3286827007,IT +3286827008,3286827263,DE +3286827264,3286827519,GB +3286827520,3286832127,DE +3286832128,3286832895,GB +3286832896,3286835711,DE +3286835712,3286836223,ES +3286836224,3286836479,IT +3286836480,3286836735,DE +3286836736,3286836991,IT +3286836992,3286837247,DE +3286837248,3286839295,GB +3286839296,3286839807,DE +3286839808,3286840319,ES +3286840320,3286841087,DE +3286841088,3286841343,IT +3286841344,3286842623,DE +3286842624,3286843391,IT +3286843392,3286844415,IE +3286844416,3286844671,IT +3286844672,3286847487,DE +3286847488,3286848511,GB +3286848512,3286849535,DE +3286849536,3286849791,IT +3286849792,3286851583,DE +3286851584,3286852607,IE +3286852608,3286855679,DE +3286855680,3286857727,IE +3286857728,3286864895,DE +3286864896,3286865151,IT +3286865152,3286872063,DE +3286872064,3286879231,IE +3286879232,3286882303,DE +3286882304,3286883327,IE +3286883328,3286888447,DE +3286888448,3286889471,IE +3286889472,3286892543,DE 3286892544,3286893055,LI 3286893056,3286893567,RU 3286893568,3286894591,UA @@ -104103,7 +105977,9 @@ 3291204352,3291204863,ZA 3291204864,3291205119,KE 3291205120,3291205631,GH -3291205632,3291205887,ZA +3291205632,3291206143,ZA +3291206144,3291206399,AO +3291206400,3291206911,KE 3291217920,3291230207,ZA 3291230208,3291234303,GH 3291234304,3291242495,ZA @@ -104307,7 +106183,9 @@ 3302954496,3302955007,ZA 3302955008,3302955263,LS 3305111552,3307208703,TN -3309305856,3313500159,ZA +3309305856,3310354431,ZA +3311403008,3312451583,ZA +3312975872,3313500159,EG 3313500160,3313762303,MA 3313762304,3314024447,EG 3314024448,3314286591,KE @@ -104324,14 +106202,18 @@ 3319791616,3320053759,MU 3320578048,3320643583,ZA 3320643584,3320709119,KE +3320709120,3320840191,ZA +3321708544,3321724927,GH 3321724928,3321757695,MA 3321757696,3321790463,KE 3321790464,3321806847,LS 3321806848,3321823231,SD 3321823232,3321839615,NG -3321839616,3321855999,GH +3321839616,3321855999,MU 3321856000,3321860095,CV 3321860096,3321864191,ZA +3321864192,3321868287,NG +3321868288,3321872383,CG 3321954304,3321970687,US 3322019840,3322023935,US 3322023936,3322028031,CL @@ -105422,7 +107304,10 @@ 3345666592,3345666639,NO 3345666640,3345667103,US 3345667104,3345667119,IN -3345667120,3346282495,US +3345667120,3346219007,US +3346235392,3346241535,US +3346241536,3346243583,CA +3346243584,3346282495,US 3346282496,3346284543,PR 3346333696,3346923519,US 3346923520,3346989055,CA @@ -106139,7 +108024,6 @@ 3356380928,3356381183,DO 3356381184,3356381439,CL 3356381440,3356381695,PA -3356381696,3356382207,UY 3356382208,3356389375,CL 3356389376,3356389887,CO 3356389888,3356390399,CL @@ -106943,7 +108827,7 @@ 3363471360,3363487743,CR 3363487744,3363504127,CO 3363504128,3363512319,PE -3363520512,3363553791,AR +3363512320,3363553791,AR 3363553792,3363554047,PE 3363554048,3363557375,AR 3363561472,3363565567,CO @@ -107151,6 +109035,7 @@ 3389195776,3389196287,HK 3389196288,3389197567,AU 3389197568,3389197823,IN +3389197824,3389198079,ID 3389198080,3389198335,IN 3389198336,3389202431,NZ 3389202432,3389210623,AU @@ -107241,8 +109126,10 @@ 3389412352,3389412863,NZ 3389412864,3389413119,AU 3389413120,3389413375,NZ +3389413376,3389413887,CN 3389413888,3389414143,TH 3389414144,3389414655,AU +3389414656,3389414911,CN 3389414912,3389415167,HK 3389415168,3389415423,KR 3389415424,3389415935,VN @@ -107299,6 +109186,7 @@ 3389534208,3389538303,JP 3389538304,3389538559,AU 3389538560,3389540351,TH +3389540352,3389541375,KH 3389541376,3389541631,AU 3389541632,3389541887,JP 3389541888,3389542399,TH @@ -107437,7 +109325,9 @@ 3389957376,3389957631,KR 3389957632,3389957887,AU 3389957888,3389958399,NZ -3389958400,3389970431,AU +3389958400,3389969663,AU +3389969664,3389969919,CN +3389969920,3389970431,AU 3389970432,3389971199,NZ 3389971200,3389971967,AU 3389971968,3389972223,HK @@ -107561,7 +109451,7 @@ 3391526656,3391526911,SG 3391526912,3391528191,CN 3391528192,3391528447,AU -3391528960,3391529471,CN +3391528448,3391529471,CN 3391529472,3391529983,AU 3391529984,3391533567,CN 3391533568,3391534079,HK @@ -107597,7 +109487,7 @@ 3391725568,3391733759,TH 3391733760,3391734015,IN 3391734016,3391734783,AU -3391734784,3391736831,JP +3391735808,3391736831,JP 3391736832,3391737855,IN 3391737856,3391741951,JP 3391741952,3391744959,IN @@ -107706,6 +109596,7 @@ 3392086016,3392094207,AU 3392094208,3392098559,ID 3392098560,3392098815,AU +3392098816,3392099327,CN 3392099328,3392100095,AU 3392100096,3392100351,VN 3392100352,3392100607,ID @@ -107733,11 +109624,13 @@ 3392208896,3392286975,NZ 3392286976,3392287231,US 3392287232,3392287743,NZ +3392287744,3392288767,NP 3392288768,3392295935,NZ 3392295936,3392296191,AU 3392296192,3392324607,NZ 3392324608,3392325119,AU 3392325120,3392325631,NZ +3392325632,3392326655,ID 3392326656,3392339967,NZ 3392339968,3392340991,NP 3392340992,3392344063,JP @@ -107746,6 +109639,7 @@ 3392354304,3392356351,ID 3392356352,3392364543,NP 3392364544,3392372735,ID +3392372736,3392380927,HK 3392380928,3392385023,AU 3392385024,3392389119,BD 3392389120,3392401407,ID @@ -107870,6 +109764,7 @@ 3392765952,3392782335,TH 3392782336,3392790527,HK 3392790528,3392794623,JP +3392794624,3392798719,CN 3392798720,3392798975,LA 3392798976,3392799231,JP 3392799232,3392799487,PH @@ -107905,7 +109800,7 @@ 3392864512,3392864767,AU 3392864768,3392865279,IN 3392865280,3392866303,NU -3392868352,3392880639,AU +3392866304,3392880639,AU 3392880640,3392888831,PK 3392888832,3392892927,AU 3392892928,3392897023,JP @@ -107943,6 +109838,7 @@ 3392944128,3392945151,MY 3392945152,3392946175,IN 3392946176,3392954367,AU +3392954368,3392956415,CN 3392956416,3392958463,VN 3392958464,3392962559,CN 3392962560,3392970751,IN @@ -107987,6 +109883,7 @@ 3393069056,3393077247,AU 3393077248,3393085439,IN 3393085440,3393089535,LA +3393089536,3393090559,CN 3393090560,3393091071,IN 3393091072,3393093631,FJ 3393093632,3393101823,AU @@ -107997,9 +109894,11 @@ 3393123328,3393123583,IN 3393123584,3393123839,NZ 3393123840,3393124351,IN +3393124352,3393125375,CN 3393125376,3393125631,IN 3393125632,3393125887,JP 3393125888,3393126143,AU +3393126144,3393126399,CN 3393126400,3393134591,HK 3393134592,3393146879,AU 3393146880,3393150975,PK @@ -108064,6 +109963,7 @@ 3393593344,3393597439,MN 3393597440,3393601535,ID 3393601536,3393609727,NP +3393609728,3393613823,CN 3393613824,3393617919,AS 3393617920,3393618431,AU 3393618432,3393618687,NZ @@ -108129,6 +110029,7 @@ 3393814528,3393815551,HK 3393815552,3393816575,KR 3393816576,3393818623,JP +3393818624,3393822719,AU 3393822720,3393830911,PH 3393830912,3393835007,NZ 3393835008,3393839103,JP @@ -108244,6 +110145,7 @@ 3394284352,3394285567,SG 3394285568,3394289663,AU 3394289664,3394290687,HK +3394290688,3394293759,CN 3394293760,3394297855,ID 3394297856,3394306047,TH 3394306048,3394307071,HK @@ -108275,6 +110177,7 @@ 3394501632,3394507263,HK 3394507264,3394507775,JP 3394507776,3394508799,PH +3394508800,3394510847,CN 3394510848,3394514943,BD 3394514944,3394519039,JP 3394519040,3394521087,BD @@ -108312,6 +110215,7 @@ 3394697472,3394697727,PK 3394697728,3394697983,AU 3394697984,3394698239,IN +3394698240,3394699263,CN 3394699264,3394700287,HK 3394700288,3394707455,AU 3394707456,3394715647,IN @@ -108380,6 +110284,7 @@ 3394906112,3394906367,IN 3394906368,3394906623,AU 3394906624,3394907135,IN +3394907136,3394908159,NZ 3394908160,3394910207,AU 3394910208,3394912255,PH 3394912256,3394920447,PF @@ -108411,6 +110316,7 @@ 3394971648,3394973695,SG 3394973696,3394977791,JP 3394977792,3394985983,IN +3394985984,3394990079,CN 3394990080,3394994175,JP 3394994176,3394995199,CN 3394995200,3394998271,IN @@ -108474,6 +110380,7 @@ 3395180288,3395180543,HK 3395180544,3395181055,VN 3395181056,3395181567,HK +3395181568,3395182591,CN 3395182592,3395190783,SG 3395190784,3395198975,JP 3395198976,3395203071,MY @@ -108522,7 +110429,6 @@ 3397070848,3397074943,PH 3397074944,3397083135,HK 3397083136,3397087231,CN -3397087232,3397091327,JP 3397091328,3397099519,GU 3397099520,3397103615,HK 3397103616,3397105663,LA @@ -108531,7 +110437,7 @@ 3397115904,3397119999,ID 3397120000,3397124095,PK 3397124096,3397128191,JP -3397130240,3397131263,CN +3397128192,3397131263,CN 3397131264,3397132287,HK 3397132288,3397136383,AU 3397136384,3397140479,JP @@ -108676,7 +110582,6 @@ 3397525504,3397526527,AU 3397526528,3397527039,VN 3397527040,3397527295,AU -3397527296,3397527551,JP 3397527552,3397528575,IN 3397528576,3397530623,AU 3397530624,3397531647,ID @@ -108784,6 +110689,7 @@ 3397963776,3397971967,CN 3397971968,3397974015,LA 3397974016,3397975039,IN +3397975040,3397976063,AU 3397976064,3397984255,ID 3397984256,3397992447,JP 3397992448,3398004735,AU @@ -108798,6 +110704,7 @@ 3398033664,3398033919,ID 3398033920,3398034943,IN 3398034944,3398035199,ID +3398035200,3398035455,CN 3398035456,3398037503,IN 3398037504,3398039551,NZ 3398039552,3398040575,IN @@ -108850,6 +110757,7 @@ 3398481920,3398483967,LA 3398483968,3398488063,MY 3398488064,3398492159,TW +3398492160,3398500351,JP 3398500352,3398504447,ID 3398504448,3398508543,JP 3398508544,3398565887,TW @@ -108866,6 +110774,7 @@ 3398612992,3398613503,PH 3398613504,3398613759,NZ 3398613760,3398614015,AU +3398614016,3398615039,CN 3398615040,3398619135,IN 3398619136,3398621183,AU 3398621184,3398623231,HK @@ -109000,13 +110909,15 @@ 3398647296,3398647551,TW 3398647552,3398647807,AP 3398647808,3398668287,AU +3398668288,3398672383,CN 3398672384,3398680575,PK 3398680576,3398684671,ID 3398684672,3398688767,JP 3398688768,3398705151,ID +3398705152,3398709247,CN 3398709248,3398711295,AU 3398711296,3398713343,BD -3398721536,3398729727,CN +3398713344,3398729727,CN 3398729728,3398737919,AU 3398737920,3398742015,SG 3398742016,3398746111,TH @@ -109035,6 +110946,7 @@ 3398852608,3398860799,NZ 3398860800,3398873087,ID 3398873088,3398877183,KR +3398877184,3398881279,CN 3398881280,3398885375,SG 3398885376,3398894591,ID 3398894592,3398895615,TH @@ -109130,6 +111042,7 @@ 3399332864,3399333375,HK 3399333376,3399335423,MY 3399335424,3399335935,IN +3399335936,3399344127,CN 3399344128,3399352319,JP 3399352320,3399389183,ID 3399389184,3399393279,KR @@ -109187,10 +111100,12 @@ 3399614464,3399622655,MY 3399622656,3399626751,ID 3399626752,3399630847,IN -3399630848,3399631871,AU +3399630848,3399631615,AU +3399631616,3399631871,CN 3399631872,3399632895,SG 3399632896,3399633407,NZ 3399633408,3399633663,AU +3399633664,3399633919,CN 3399633920,3399634943,TH 3399634944,3399639039,JP 3399639040,3399643135,AU @@ -109215,6 +111130,7 @@ 3399752704,3399753727,NZ 3399753728,3399761919,IN 3399761920,3399770111,JP +3399770112,3399778303,CN 3399778304,3399786495,IN 3399786496,3399794687,PH 3399794688,3399798783,AU @@ -109317,7 +111233,7 @@ 3400183808,3400187903,JP 3400187904,3400191999,AU 3400192000,3400194047,JP -3400194048,3400196095,CN +3400194048,3400204287,CN 3400204288,3400212479,ID 3400212480,3400220671,MY 3400220672,3400221055,NC @@ -109329,10 +111245,12 @@ 3400245248,3400253439,AU 3400253440,3400257535,MY 3400257536,3400259583,HK +3400259584,3400261631,CN 3400261632,3400263679,JP 3400263680,3400263935,AU 3400263936,3400264191,ID 3400264192,3400264447,IN +3400264448,3400264703,CN 3400264704,3400265215,ID 3400265216,3400265471,AU 3400265472,3400265727,IN @@ -109359,7 +111277,7 @@ 3400341504,3400343551,AU 3400343552,3400351743,TW 3400351744,3400359935,ID -3400359936,3400368127,JP +3400359936,3400364031,JP 3400368128,3400388607,AU 3400388608,3400392703,TH 3400392704,3400400895,CN @@ -109616,6 +111534,7 @@ 3400536064,3400548351,JP 3400548352,3400581119,TH 3400581120,3400589311,SG +3400589312,3400597503,CN 3400597504,3400605695,HK 3400605696,3400607743,JP 3400607744,3400608767,AU @@ -109668,16 +111587,18 @@ 3400826880,3400835071,CN 3400835072,3400839167,HK 3400839168,3400847359,JP +3400847360,3400849407,CN 3400849408,3400851455,MN -3400851456,3400867839,AU -3400876032,3400884223,AU +3400851456,3400884223,AU 3400884224,3400888319,JP 3400888320,3400892415,CN 3400892416,3400925183,HK 3400925184,3400933375,TH +3400933376,3400937471,CN 3400937472,3400941567,ID 3400941568,3400966143,AU 3400966144,3400974335,ID +3400974336,3400982527,CN 3400982528,3400990719,HK 3400990720,3400998911,ID 3400998912,3401003007,PH @@ -109687,6 +111608,7 @@ 3401015296,3401023487,AU 3401023488,3401056255,TH 3401056256,3401383935,MY +3401383936,3401400319,CN 3401400320,3401404415,AU 3401404416,3401408511,CN 3401408512,3401416703,HK @@ -109694,6 +111616,7 @@ 3401420800,3401424895,JP 3401424896,3401428991,NZ 3401428992,3401431039,JP +3401431040,3401433087,CN 3401433088,3401441279,JP 3401441280,3401449471,IN 3401449472,3401515007,MY @@ -109719,9 +111642,13 @@ 3404857956,3404857966,JP 3404857967,3404857967,IN 3404857968,3405774847,JP -3405774848,3405846783,AU +3405774848,3405795583,AU +3405795584,3405796095,CN +3405796096,3405846783,AU 3405846784,3405847039,ID -3405847040,3406005247,AU +3405847040,3405934591,AU +3405934592,3405936639,CN +3405936640,3406005247,AU 3406005248,3406005503,HK 3406005504,3406071807,AU 3406071808,3406073855,US @@ -109731,10 +111658,13 @@ 3406109696,3406109951,NZ 3406109952,3406205951,AU 3406205952,3406206463,ID -3406206464,3406277375,AU +3406206464,3406271231,AU +3406271232,3406271487,CN +3406271488,3406277375,AU 3406277376,3406277631,ID 3406277632,3406327039,AU 3406327040,3406327295,IN +3406327296,3406327807,CN 3406327808,3406328831,AU 3406328832,3406329343,IN 3406329344,3406331647,AU @@ -109743,7 +111673,11 @@ 3406343168,3406343423,VN 3406343424,3406350591,AU 3406350592,3406350847,IN -3406350848,3406384639,AU +3406350848,3406380799,AU +3406380800,3406381055,CN +3406381056,3406382591,AU +3406382592,3406383103,CN +3406383104,3406384639,AU 3406384640,3406385151,SG 3406385152,3406409727,AU 3406409728,3406411775,NZ @@ -109755,7 +111689,9 @@ 3406512384,3406512639,IN 3406512640,3406514687,AU 3406514688,3406514943,TH -3406514944,3406542847,AU +3406514944,3406521343,AU +3406521344,3406522367,CN +3406522368,3406542847,AU 3406542848,3406543103,SG 3406543104,3406565887,AU 3406565888,3406566143,PH @@ -109793,7 +111729,9 @@ 3406894336,3406894591,ID 3406894592,3406896895,AU 3406896896,3406897151,IN -3406897152,3406938623,AU +3406897152,3406923775,AU +3406923776,3406924031,CN +3406924032,3406938623,AU 3406938624,3406938879,IN 3406938880,3406946815,AU 3406946816,3406947071,KR @@ -109813,13 +111751,25 @@ 3407020544,3407020799,AU 3407020800,3407021055,IN 3407021056,3407021311,ID -3407021312,3407045887,AU +3407021312,3407027711,AU +3407027712,3407027967,CN +3407027968,3407045887,AU 3407045888,3407046143,HK 3407046144,3407057663,AU 3407057664,3407057919,JP -3407057920,3407096831,AU +3407057920,3407058175,AU +3407058176,3407058431,CN +3407058432,3407059967,AU +3407059968,3407060223,CN +3407060224,3407078399,AU +3407078400,3407079423,CN +3407079424,3407085311,AU +3407085312,3407085567,CN +3407085568,3407096831,AU 3407096832,3407097087,JP -3407097088,3407112447,AU +3407097088,3407101183,AU +3407101184,3407101439,CN +3407101440,3407112447,AU 3407112448,3407112703,SG 3407112704,3407152895,AU 3407152896,3407153151,IN @@ -109860,27 +111810,79 @@ 3407159240,3407159263,AU 3407159264,3407159295,NZ 3407159296,3407161599,AU +3407161600,3407161855,CN 3407161856,3407162367,TH 3407162368,3407170047,AU 3407170048,3407170559,ID -3407170560,3407243263,AU +3407170560,3407240959,AU +3407240960,3407241215,CN +3407241216,3407243263,AU 3407243264,3407243775,HK -3407243776,3407268863,AU +3407243776,3407250175,AU +3407250176,3407250431,CN +3407250432,3407268863,AU 3407268864,3407269119,US -3407269120,3407360511,AU +3407269120,3407294207,AU +3407294208,3407294463,CN +3407294464,3407310847,AU +3407310848,3407311103,CN +3407311104,3407315455,AU +3407315456,3407315711,CN +3407315712,3407326207,AU +3407326208,3407326463,CN +3407326464,3407328767,AU +3407328768,3407329023,CN +3407329024,3407329791,AU +3407330048,3407330303,CN +3407330304,3407360511,AU 3407360512,3407361023,ID -3407361024,3407367167,AU +3407361024,3407362047,AU +3407362048,3407362303,CN +3407362304,3407367167,AU 3407367168,3407367679,ID 3407367680,3407367935,AU -3407368192,3407369983,AU +3407367936,3407368447,CN +3407368448,3407369983,AU 3407369984,3407370239,IN -3407370240,3407498495,AU +3407370240,3407370751,AU +3407370752,3407371007,CN +3407371008,3407386623,AU +3407386624,3407386879,CN +3407386880,3407388927,AU +3407388928,3407389183,CN +3407389184,3407398655,AU +3407398656,3407398911,CN +3407398912,3407440383,AU +3407440384,3407440639,CN +3407440640,3407464191,AU +3407464192,3407464447,CN +3407464448,3407466495,AU +3407466496,3407470591,CN +3407470592,3407475199,AU +3407475200,3407475455,CN +3407475456,3407495423,AU +3407495424,3407495679,CN +3407495680,3407498495,AU 3407498496,3407498751,PK -3407498752,3407524607,AU +3407498752,3407499263,AU +3407499264,3407499519,CN +3407499520,3407504895,AU +3407504896,3407505407,CN +3407505408,3407508223,AU +3407508224,3407508479,CN +3407508480,3407523071,AU +3407523072,3407523327,CN +3407523328,3407524607,AU 3407524608,3407524863,NZ 3407524864,3407545855,AU 3407545856,3407546367,ID -3407546368,3407602943,AU +3407546368,3407546879,AU +3407546880,3407547135,CN +3407547136,3407574271,AU +3407574272,3407574527,CN +3407574528,3407575807,AU +3407575808,3407576063,CN +3407576064,3407602943,AU 3407602944,3407603199,JP 3407603200,3407604479,AU 3407604480,3407604735,IN @@ -109888,17 +111890,41 @@ 3407608716,3407608736,JP 3407608737,3407642623,AU 3407642624,3407643135,TH -3407643136,3407682047,AU +3407643136,3407675903,AU +3407675904,3407676159,CN +3407676160,3407678975,AU +3407678976,3407679231,CN +3407679232,3407682047,AU 3407682048,3407682559,ID -3407682560,3407732223,AU +3407682560,3407682815,CN +3407682816,3407701759,AU +3407701760,3407702015,CN +3407702016,3407704063,AU +3407704064,3407704319,CN +3407704320,3407727871,AU +3407727872,3407728127,CN +3407728128,3407729151,AU +3407729152,3407729407,CN +3407729408,3407732223,AU 3407732224,3407732479,HK -3407732480,3407750655,AU +3407732480,3407747839,AU +3407747840,3407748095,CN +3407748096,3407750655,AU 3407750656,3407751167,SG 3407751168,3407753215,AU 3407753216,3407753727,HK -3407753728,3407785471,AU +3407753728,3407779839,AU +3407779840,3407780095,CN +3407780096,3407780863,AU +3407780864,3407781119,CN +3407781120,3407785471,AU 3407785472,3407785727,NZ -3407785728,3407801343,AU +3407785728,3407790591,AU +3407790592,3407790847,CN +3407790848,3407797247,AU +3407797248,3407797503,CN +3407797504,3407801087,AU +3407801088,3407801343,CN 3407801344,3407801855,ID 3407801856,3407805951,AU 3407805952,3407806463,ID @@ -109906,44 +111932,79 @@ 3407808512,3407809023,IN 3407809024,3407814655,AU 3407814656,3407815167,HK -3407815168,3407828991,AU +3407815168,3407824127,AU +3407824128,3407824383,CN +3407824384,3407828991,AU 3407828992,3407829503,US 3407829504,3407837183,AU 3407837184,3407837439,US -3407837440,3407848447,AU +3407837440,3407847935,AU +3407847936,3407848191,CN +3407848192,3407848447,AU 3407848448,3407848959,ID -3407848960,3407866367,AU +3407848960,3407862783,AU +3407862784,3407863039,CN +3407863040,3407866367,AU 3407866368,3407866623,GB 3407866624,3407873023,AU 3407873024,3407873535,IN -3407873536,3407928575,AU +3407873536,3407877119,AU +3407877120,3407877375,CN +3407877376,3407887871,AU +3407887872,3407888127,CN +3407888128,3407905279,AU +3407905280,3407905535,CN +3407905536,3407907839,AU +3407907840,3407908095,CN +3407908096,3407919615,AU +3407919616,3407920127,CN +3407920128,3407928575,AU 3407928576,3407928831,IN -3407928832,3407985919,AU +3407928832,3407977471,AU +3407977472,3407977727,CN +3407977728,3407985919,AU 3407985920,3407986175,KH 3407986176,3407987711,AU 3407987712,3407987967,PH 3407987968,3407988223,AU 3407988224,3407988735,IN -3407988736,3407998975,AU +3407988736,3407989759,AU +3407989760,3407990015,CN +3407990016,3407994879,AU +3407994880,3407995135,CN +3407995136,3407997183,AU +3407997184,3407997439,CN +3407997440,3407998975,AU 3407998976,3407999231,TH -3407999232,3408012543,AU +3407999232,3408009983,AU +3408009984,3408010239,CN +3408010240,3408012543,AU 3408012544,3408012799,MN -3408012800,3408023807,AU +3408012800,3408020735,AU +3408020736,3408020991,CN +3408020992,3408023807,AU 3408023808,3408024063,JP -3408024064,3408032767,AU +3408024064,3408031999,AU +3408032000,3408032255,CN +3408032256,3408032767,AU 3408032768,3408033279,IN 3408033280,3408033791,ID 3408033792,3408039935,AU 3408039936,3408040191,VN 3408040192,3408040703,AU +3408040704,3408040959,CN 3408040960,3408041983,AU 3408041984,3408042495,SG 3408042496,3408042751,HK -3408042752,3408066047,AU +3408042752,3408064511,AU +3408064512,3408064767,CN +3408064768,3408066047,AU 3408066048,3408066303,PH 3408066304,3409396479,AU 3409396480,3409396735,PH -3409396736,3409418495,AU +3409396736,3409409023,AU +3409409024,3409409535,CN +3409409536,3409418495,AU 3409418496,3409418751,PL 3409418752,3409420287,AU 3409420288,3409420543,IN @@ -109951,9 +112012,21 @@ 3409423616,3409423871,IN 3409423872,3409425663,AU 3409425664,3409425919,AP -3409425920,3409491711,AU +3409425920,3409429503,AU +3409429504,3409429759,CN +3409429760,3409435135,AU +3409435136,3409435391,CN +3409435392,3409466623,AU +3409466624,3409466879,CN +3409466880,3409475839,AU +3409475840,3409476095,CN +3409476096,3409488127,AU +3409488128,3409488383,CN +3409488384,3409491711,AU 3409491712,3409491967,SG -3409491968,3409503999,AU +3409491968,3409498111,AU +3409498112,3409498879,CN +3409498880,3409503999,AU 3409504000,3409504255,HK 3409504256,3409505023,AU 3409505024,3409505279,US @@ -109965,26 +112038,45 @@ 3409510368,3409510383,IN 3409510384,3409516543,AU 3409516544,3409517055,ID -3409517056,3409547519,AU +3409517056,3409520383,AU +3409520384,3409520639,CN +3409520640,3409522175,AU +3409522176,3409522431,CN +3409522432,3409547519,AU 3409547520,3409547775,NZ -3409547776,3409802831,AU +3409547776,3409550591,AU +3409550592,3409550847,CN +3409550848,3409567231,AU +3409567232,3409567487,CN +3409567488,3409574143,AU +3409574144,3409574399,CN +3409574400,3409802831,AU 3409802832,3409802847,MT 3409802848,3409838335,AU 3409838336,3409838591,MY +3409838592,3409838847,CN 3409838848,3409876991,AU 3409876992,3409878015,TH 3409878016,3409882111,AU 3409882112,3409883135,IN 3409883136,3409887999,AU 3409888000,3409888255,SG -3409888256,3409969151,AU +3409888256,3409896447,AU +3409896448,3409897471,CN +3409897472,3409897983,AU +3409897984,3409898239,CN +3409898240,3409901055,AU +3409901056,3409901311,CN +3409901312,3409969151,AU 3409969152,3410755583,TW 3410755584,3410771967,AU 3410771968,3410780159,JP 3410780160,3410788351,BD 3410788352,3410792447,IN 3410792448,3410796543,BD +3410796544,3410797567,CN 3410797568,3410798591,JP +3410798592,3410799615,CN 3410799616,3410800639,SG 3410800640,3410804735,IN 3410804736,3410821119,PH @@ -110005,6 +112097,7 @@ 3410888704,3410890751,SG 3410890752,3410894847,AU 3410894848,3410898943,HK +3410898944,3410903039,CN 3410903040,3410911231,HK 3410911232,3410915327,TH 3410915328,3410919423,ID @@ -110031,8 +112124,10 @@ 3411019776,3411021823,ID 3411021824,3411023871,MY 3411023872,3411025919,JP +3411025920,3411030015,CN 3411030016,3411032063,NC 3411032064,3411032319,TH +3411032320,3411032575,CN 3411032576,3411033087,AU 3411033088,3411034111,NZ 3411034112,3411050495,HK @@ -110043,6 +112138,7 @@ 3411062784,3411063231,HK 3411063232,3411063295,PK 3411063296,3411083263,HK +3411083264,3411085311,CN 3411085312,3411086335,KR 3411086336,3411087359,JP 3411087360,3411091455,CN @@ -110055,12 +112151,12 @@ 3411128320,3411130367,HK 3411130368,3411132415,ID 3411132416,3411144703,PK -3411144704,3411146751,JP 3411146752,3411147775,ID 3411147776,3411149311,HK 3411149312,3411149823,MV 3411149824,3411150847,IN 3411150848,3411152895,HK +3411152896,3411154943,CN 3411154944,3411156991,JP 3411156992,3411161087,PH 3411161088,3411165183,PK @@ -110079,7 +112175,6 @@ 3411212800,3411213311,IN 3411213312,3411215359,HK 3411215360,3411216383,AU -3411216384,3411218431,JP 3411218432,3411220479,PG 3411220480,3411226623,ID 3411226624,3411228671,IO @@ -110122,6 +112217,7 @@ 3411472384,3411475199,JP 3411475200,3411475455,AU 3411475456,3411475967,HK +3411475968,3411476479,CN 3411476480,3411509247,AU 3411509248,3411542015,PH 3411542016,3411550207,IN @@ -110236,6 +112332,7 @@ 3411608576,3411608831,IN 3411608832,3411609087,AU 3411609088,3411609599,HK +3411609600,3411611647,CN 3411611648,3411615743,ID 3411615744,3411623935,JP 3411623936,3411640319,AU @@ -110243,6 +112340,7 @@ 3411641344,3411641599,IN 3411641600,3411641855,HK 3411641856,3411642367,IN +3411642368,3411643391,CN 3411643392,3411644415,VN 3411644416,3411644927,AU 3411644928,3411645951,ID @@ -110253,6 +112351,7 @@ 3411673088,3411674111,CN 3411674112,3411674623,IN 3411674624,3411675135,HK +3411675136,3411676159,CN 3411676160,3411677183,PK 3411677184,3411679231,JP 3411679232,3411681279,AU @@ -110425,7 +112524,7 @@ 3412253696,3412254719,JP 3412254720,3412262911,NR 3412262912,3412264959,JP -3412267008,3412271103,CN +3412264960,3412271103,CN 3412271104,3412273151,NZ 3412273152,3412275199,IN 3412275200,3412279295,PK @@ -110451,12 +112550,13 @@ 3412327936,3412328191,HK 3412328192,3412328447,WS 3412328448,3412336639,AU -3412336640,3412340735,CN +3412336640,3412342783,CN 3412342784,3412343039,AU 3412343040,3412343295,IN 3412343296,3412343551,AP 3412343552,3412344319,AU 3412344320,3412344575,SG +3412344576,3412344831,CN 3412344832,3412348927,IN 3412348928,3412361215,CN 3412361216,3412369407,AP @@ -110471,16 +112571,19 @@ 3412451328,3412594687,AU 3412594688,3412596735,IN 3412596736,3412598783,MV +3412598784,3412602879,CN 3412602880,3412606975,NC 3412606976,3412615167,PH 3412615168,3412656127,JP 3412656128,3412672511,HK 3412672512,3412680703,JP +3412680704,3412697087,CN 3412697088,3412705279,IN 3412705280,3412713471,AU 3412713472,3412721663,TW 3412721664,3412787199,MY 3412787200,3412803583,TW +3412803584,3412819967,CN 3412819968,3412852735,TH 3412852736,3412918271,AU 3412918272,3412926463,KR @@ -110495,10 +112598,13 @@ 3413032960,3413037055,AU 3413037056,3413041151,IN 3413041152,3413043199,JP -3413043200,3413044223,AU +3413043200,3413043711,AU +3413043712,3413043967,CN +3413043968,3413044223,AU 3413044224,3413045247,AP 3413045248,3413047295,IN 3413047296,3413098495,AU +3413098496,3413102591,JP 3413102592,3413106687,TW 3413106688,3413110783,PH 3413110784,3413112831,JP @@ -110506,6 +112612,7 @@ 3413113856,3413133311,JP 3413133312,3413135359,BD 3413135360,3413139455,HK +3413139456,3413147647,AU 3413147648,3413155839,IN 3413155840,3413164031,SG 3413164032,3413172223,BD @@ -110536,6 +112643,7 @@ 3413557248,3413565439,CN 3413565440,3413569535,TW 3413569536,3413569791,SG +3413569792,3413570047,CN 3413570048,3413570303,KH 3413570304,3413570559,AU 3413570560,3413571583,PH @@ -110579,6 +112687,7 @@ 3413597696,3413597951,TW 3413597952,3413602303,AU 3413602304,3413602559,ID +3413602560,3413639167,CN 3413639168,3413704703,SG 3413704704,3413737471,MY 3413737472,3413753855,TH @@ -110619,7 +112728,9 @@ 3414171648,3414179839,CN 3414179840,3414188031,ID 3414188032,3414196223,CN +3414196224,3414204415,AU 3414204416,3414220799,KR +3414220800,3414222847,CN 3414222848,3414223871,AU 3414223872,3414224895,KR 3414224896,3414226943,VN @@ -110627,6 +112738,7 @@ 3414227968,3414230015,PK 3414230016,3414230527,PH 3414230528,3414231039,KR +3414231040,3414233087,CN 3414233088,3414245375,AU 3414245376,3414253567,HK 3414253568,3414261759,JP @@ -110708,6 +112820,7 @@ 3415228416,3415236607,KH 3415236608,3415244799,IN 3415244800,3415277567,TH +3415277568,3415285759,CN 3415285760,3415293951,AU 3415293952,3415302143,HK 3415302144,3415306239,AU @@ -110727,7 +112840,9 @@ 3415441408,3415474175,AU 3415474176,3415490559,CN 3415490560,3415491583,PK +3415491584,3415495679,CN 3415495680,3415496191,ID +3415496192,3415496703,CN 3415496704,3415497727,MY 3415497728,3415497983,TW 3415497984,3415498751,AU @@ -110741,6 +112856,7 @@ 3415605248,3415752703,TH 3415752704,3415760895,CN 3415760896,3415769087,NZ +3415769088,3415777279,CN 3415777280,3415785471,KR 3415785472,3415793663,JP 3415793664,3415801855,AU @@ -110786,8 +112902,10 @@ 3416274944,3416276991,ID 3416276992,3416285183,HK 3416285184,3416287231,VN +3416287232,3416289279,CN 3416289280,3416293375,NZ 3416293376,3416293631,ID +3416293632,3416293887,CN 3416293888,3416294399,PH 3416294400,3416295423,AU 3416295424,3416295679,IN @@ -110796,6 +112914,7 @@ 3416296448,3416297471,KR 3416297472,3416301567,TW 3416301568,3416309759,PH +3416309760,3416317951,CN 3416317952,3416326143,TW 3416326144,3416327167,VN 3416327168,3416328191,HK @@ -111027,6 +113146,7 @@ 3416489775,3416489783,AU 3416489784,3416489787,JP 3416489788,3416489983,AU +3416489984,3416506367,VN 3416506368,3416514559,TW 3416514560,3416522751,IN 3416522752,3416588287,AU @@ -111051,6 +113171,7 @@ 3416727552,3416735743,JP 3416735744,3416752127,PH 3416752128,3416784895,NZ +3416784896,3416793087,CN 3416793088,3416801279,AU 3416801280,3416817663,JP 3416817664,3416850431,HK @@ -111092,6 +113213,7 @@ 3416929280,3416930303,JP 3416930304,3416930559,NZ 3416930560,3416930815,AU +3416930816,3416931327,CN 3416931328,3416932351,IN 3416932352,3416936447,PK 3416936448,3416938495,AU @@ -111131,6 +113253,7 @@ 3417137152,3417145343,KR 3417145344,3417178111,NZ 3417178112,3417179135,PH +3417179136,3417179391,CN 3417179392,3417179647,ID 3417179648,3417179903,IN 3417179904,3417180159,CN @@ -111165,7 +113288,7 @@ 3417289728,3417291263,IN 3417291264,3417291775,AU 3417291776,3417292799,KR -3417292800,3417309183,CN +3417292800,3417333759,CN 3417333760,3417337855,AU 3417337856,3417338367,IN 3417338368,3417338879,HK @@ -111223,6 +113346,7 @@ 3418157056,3418161663,BD 3418161664,3418162431,AU 3418162432,3418162687,IN +3418162688,3418163199,CN 3418163200,3418165247,PH 3418165248,3418167295,MY 3418167296,3418167551,IN @@ -111236,6 +113360,7 @@ 3418184192,3418184959,IN 3418184960,3418185727,AU 3418185728,3418189823,PK +3418189824,3418190847,CN 3418190848,3418191871,TH 3418191872,3418192895,ID 3418192896,3418193919,AU @@ -111243,6 +113368,7 @@ 3418202112,3418206207,HK 3418206208,3418208255,IN 3418208256,3418210303,LK +3418210304,3418218495,CN 3418218496,3418227711,BD 3418227712,3418228735,TW 3418228736,3418230783,BD @@ -111430,6 +113556,7 @@ 3418453760,3418456063,HK 3418456064,3418472447,IN 3418472448,3418480639,AU +3418480640,3418488831,CN 3418488832,3418505215,AU 3418505216,3418506831,JP 3418506832,3418506879,ID @@ -111501,17 +113628,21 @@ 3418513232,3418513407,JP 3418513408,3418517503,IN 3418517504,3418519551,MN +3418519552,3418521599,CN 3418521600,3418524574,HK 3418524575,3418524606,CN 3418524607,3418524638,TH 3418524639,3418554367,HK +3418554368,3418570751,VN +3418570752,3418578943,CN 3418578944,3418583039,TH +3418583040,3418585087,CN 3418585088,3418586111,TH 3418586112,3418586367,AU 3418586368,3418586623,SG 3418586624,3418586879,PK 3418586880,3418587135,AU -3418619904,3418621951,CN +3418587136,3418623999,CN 3418624000,3418626047,JP 3418626048,3418628095,KI 3418628096,3418636287,AU @@ -111569,6 +113700,7 @@ 3418955776,3418959871,TW 3418959872,3418960383,BD 3418960384,3418960895,ID +3418960896,3418961919,JP 3418961920,3418962943,VN 3418962944,3418963967,IN 3418963968,3418988543,AU @@ -111603,6 +113735,7 @@ 3419357184,3419411455,CN 3419411456,3419411711,HK 3419411712,3419411967,NZ +3419411968,3419412223,HK 3419412224,3419412479,JP 3419412480,3419414527,PH 3419414528,3419422719,CN @@ -111622,6 +113755,7 @@ 3419512832,3419516927,AU 3419516928,3419517951,JP 3419517952,3419518975,VN +3419518976,3419519999,JP 3419520000,3419520767,ID 3419520768,3419521023,TH 3419521024,3419529215,AU @@ -111663,6 +113797,7 @@ 3419877632,3419877887,KH 3419877888,3419878143,ID 3419878144,3419878399,IN +3419878400,3419879423,GU 3419879424,3419880447,JP 3419880448,3419881471,MY 3419881472,3419897855,PH @@ -111681,6 +113816,7 @@ 3419906048,3419914239,PK 3419914240,3419922431,KR 3419922432,3419924479,JP +3419924480,3419926527,CN 3419926528,3419930623,HK 3419930624,3419971583,JP 3419971584,3419979775,KR @@ -111856,6 +113992,7 @@ 3420372992,3420377087,HK 3420377088,3420389375,JP 3420389376,3420393471,US +3420393472,3420395519,CN 3420395520,3420397567,JP 3420397568,3420401663,KH 3420401664,3420411903,JP @@ -111896,7 +114033,9 @@ 3423161614,3423161621,CA 3423161622,3423162303,US 3423162304,3423162311,GB -3423162312,3423182847,US +3423162312,3423162367,US +3423162368,3423163391,CA +3423163392,3423182847,US 3423182848,3423183199,CA 3423183200,3423183231,BM 3423183232,3423183263,CA @@ -111981,7 +114120,9 @@ 3423339376,3423339383,DE 3423339384,3423340399,US 3423340400,3423340407,AU -3423340408,3423341511,US +3423340408,3423341015,US +3423341016,3423341023,CA +3423341024,3423341511,US 3423341512,3423341519,PE 3423341520,3423341543,US 3423341544,3423341551,GB @@ -112030,9 +114171,12 @@ 3423346128,3423346143,CA 3423346144,3423346319,US 3423346320,3423346327,DK -3423346328,3423346487,US +3423346328,3423346431,US +3423346432,3423346447,GB +3423346448,3423346487,US 3423346488,3423346495,AE -3423346496,3423346831,US +3423346496,3423346503,SA +3423346504,3423346831,US 3423346832,3423346847,IM 3423346848,3423346943,US 3423346944,3423346951,CA @@ -112045,13 +114189,19 @@ 3423347448,3423347455,CA 3423347456,3423347503,US 3423347504,3423347519,DE -3423347520,3423347783,US +3423347520,3423347639,US +3423347640,3423347663,GB +3423347664,3423347783,US 3423347784,3423347791,SA 3423347792,3423347823,US 3423347824,3423347831,FR -3423347832,3423347919,US +3423347832,3423347879,US +3423347880,3423347903,CA +3423347904,3423347919,US 3423347920,3423347927,AU -3423347928,3423348007,US +3423347928,3423347951,US +3423347952,3423347959,TH +3423347960,3423348007,US 3423348008,3423348023,GB 3423348024,3423348071,US 3423348072,3423348095,AU @@ -112073,19 +114223,28 @@ 3423350720,3423350727,CA 3423350728,3423350735,US 3423350736,3423350743,SA -3423350744,3423351831,US +3423350744,3423351551,US +3423351552,3423351615,PR +3423351616,3423351831,US 3423351832,3423351839,AU -3423351840,3423352047,US +3423351840,3423352023,US +3423352024,3423352047,CA 3423352048,3423352063,GB -3423352064,3423352255,US +3423352064,3423352071,US +3423352072,3423352079,CA +3423352080,3423352255,US 3423352256,3423352271,CA 3423352272,3423352439,US 3423352440,3423352447,IL -3423352448,3423352503,US +3423352448,3423352495,US +3423352496,3423352503,CA 3423352504,3423352511,IL 3423352512,3423352679,US 3423352680,3423352687,NL -3423352688,3423353031,US +3423352688,3423352695,US +3423352696,3423352703,MX +3423352704,3423352711,BR +3423352712,3423353031,US 3423353032,3423353039,AU 3423353040,3423353143,US 3423353144,3423353151,KH @@ -112095,11 +114254,15 @@ 3423353352,3423353367,PA 3423353368,3423353471,US 3423353472,3423353479,KH -3423353480,3423353543,US +3423353480,3423353503,US +3423353504,3423353511,GB +3423353512,3423353543,US 3423353544,3423353551,IN 3423353552,3423353791,US 3423353792,3423353807,GB -3423353808,3423353919,US +3423353808,3423353855,US +3423353856,3423353871,CA +3423353872,3423353919,US 3423353920,3423353927,BB 3423353928,3423354031,US 3423354032,3423354039,IL @@ -112113,15 +114276,30 @@ 3423354336,3423354359,PE 3423354360,3423354583,US 3423354584,3423354591,AU -3423354592,3423354655,US +3423354592,3423354623,GB +3423354624,3423354655,US 3423354656,3423354663,AE -3423354664,3423354783,US +3423354664,3423354719,US +3423354720,3423354727,BR +3423354728,3423354783,US 3423354784,3423354791,PA -3423354792,3423357111,US +3423354792,3423356087,US +3423356088,3423356119,BR +3423356120,3423356287,US +3423356288,3423356319,CA +3423356320,3423356383,US +3423356384,3423356399,IN +3423356400,3423357111,US 3423357112,3423357127,AE 3423357128,3423357511,US 3423357512,3423357519,DE -3423357520,3423363463,US +3423357520,3423357839,US +3423357840,3423357855,IN +3423357856,3423357863,US +3423357864,3423357871,AE +3423357872,3423361023,US +3423361024,3423361279,AU +3423361280,3423363463,US 3423363464,3423363471,AU 3423363472,3423363487,CA 3423363488,3423363511,US @@ -112216,8 +114394,7 @@ 3423417471,3423417480,AU 3423417481,3423462655,US 3423462656,3423462671,CA -3423462672,3423463423,US -3423465472,3423473663,US +3423462672,3423473663,US 3423473664,3423474655,CA 3423474656,3423474671,CY 3423474672,3423474687,CA @@ -112387,17 +114564,7 @@ 3426618912,3426618943,NZ 3426618944,3426646015,US 3426646016,3426647039,CA -3426647040,3426680831,US -3426680832,3426680959,KN -3426680960,3426681087,US -3426681088,3426682111,KN -3426682112,3426682367,US -3426682368,3426682879,KN -3426682880,3426683647,US -3426683648,3426683903,KN -3426683904,3426684159,US -3426684160,3426684415,KN -3426684416,3426744319,US +3426647040,3426744319,US 3426744320,3426746367,CA 3426746368,3427127295,US 3427127296,3427127551,CA @@ -112405,7 +114572,9 @@ 3427128064,3427128831,US 3427129344,3427618303,US 3427618304,3427618559,CA -3427618560,3427729407,US +3427618560,3427647743,US +3427648000,3427648511,CA +3427648512,3427729407,US 3427729408,3427729663,CA 3427729664,3427730431,US 3427730432,3427730687,BE @@ -112714,8 +114883,8 @@ 3428606384,3428606415,US 3428606416,3428606431,CA 3428606432,3428606463,US -3428606464,3428606559,CA -3428606560,3428606655,US +3428606464,3428606591,CA +3428606592,3428606655,US 3428606656,3428606687,CA 3428606688,3428606911,US 3428606912,3428606975,IS @@ -112840,7 +115009,9 @@ 3430468864,3430468871,PR 3430468872,3430701055,US 3430701056,3430702079,CA -3430702080,3430705151,US +3430702080,3430703871,US +3430703872,3430704127,PR +3430704128,3430705151,US 3430705152,3430706175,MX 3430706176,3430747903,US 3430747904,3430748159,CA @@ -112890,7 +115061,9 @@ 3430842368,3430842879,DO 3430842880,3430845439,US 3430845440,3430845951,MX -3430845952,3431114495,US +3430845952,3430849535,US +3430849536,3430850047,CA +3430850048,3431114495,US 3431114496,3431114751,CA 3431114752,3431468031,US 3431468032,3431469055,CA @@ -113341,7 +115514,9 @@ 3437805232,3437805239,FR 3437805240,3437805271,US 3437805272,3437805279,IL -3437805280,3437815807,US +3437805280,3437814623,US +3437814624,3437814631,CA +3437814632,3437815807,US 3437815808,3437815815,IN 3437815816,3437815991,US 3437815992,3437815999,CA @@ -113926,7 +116101,11 @@ 3453039168,3453039183,AU 3453039184,3453039199,US 3453039200,3453039215,CA -3453039216,3453091839,US +3453039216,3453039623,US +3453039624,3453039631,AU +3453039632,3453039919,US +3453039920,3453039935,AU +3453039936,3453091839,US 3453091840,3453101055,CA 3453101056,3453101311,US 3453101312,3453139455,CA @@ -114658,8 +116837,8 @@ 3460113048,3460114431,US 3460114432,3460116479,SR 3460116480,3460161535,US -3460161536,3460163583,PR -3460163584,3460374527,US +3460161536,3460165631,PR +3460165632,3460374527,US 3460374528,3460375551,MX 3460375552,3460453631,US 3460453632,3460453887,BS @@ -114671,32 +116850,41 @@ 3460854832,3460854847,VE 3460854848,3460854911,US 3460854912,3460854943,GB -3460854944,3460855247,US +3460854944,3460855015,US +3460855016,3460855031,AU +3460855032,3460855195,US +3460855196,3460855199,AU +3460855200,3460855247,US 3460855248,3460855255,AU 3460855256,3460855263,VE 3460855264,3460855271,US 3460855272,3460855279,GB 3460855280,3460855287,CA -3460855288,3460855463,US +3460855288,3460855311,US +3460855312,3460855319,NL +3460855320,3460855463,US 3460855464,3460855471,SA 3460855472,3460855495,US 3460855496,3460855503,CA 3460855504,3460855535,US 3460855536,3460855543,MX 3460855544,3460855551,AU -3460855552,3460855703,US +3460855552,3460855631,US +3460855632,3460855647,CA +3460855648,3460855703,US 3460855704,3460855711,AU 3460855712,3460855743,US 3460855744,3460855775,CA 3460855776,3460855807,US 3460855808,3460855815,GB -3460855816,3460855823,NZ -3460855824,3460855855,US +3460855816,3460855855,US 3460855856,3460855863,GB 3460855864,3460855871,US 3460855872,3460855879,GB 3460855880,3460855887,GU -3460855888,3460856007,US +3460855888,3460855991,US +3460855992,3460855999,GB +3460856000,3460856007,US 3460856008,3460856015,CA 3460856016,3460856119,US 3460856120,3460856127,GB @@ -114707,21 +116895,28 @@ 3460856216,3460856223,GB 3460856224,3460856287,US 3460856288,3460856295,IN -3460856296,3460856383,US +3460856296,3460856351,US +3460856352,3460856355,TZ +3460856356,3460856383,US 3460856384,3460856399,GB 3460856400,3460856447,US 3460856448,3460856495,GB 3460856496,3460856559,US 3460856560,3460856575,CA -3460856576,3460856607,US -3460856608,3460856639,NZ +3460856576,3460856623,US +3460856624,3460856639,NZ 3460856640,3460856815,US 3460856816,3460856831,NZ 3460856832,3460857055,US 3460857056,3460857087,NZ 3460857088,3460857151,US 3460857152,3460857183,CA -3460857184,3460857431,US +3460857184,3460857343,US +3460857344,3460857359,CA +3460857360,3460857367,US +3460857368,3460857375,PL +3460857376,3460857383,GB +3460857384,3460857431,US 3460857432,3460857439,VE 3460857440,3460857463,US 3460857464,3460857471,GB @@ -115316,7 +117511,9 @@ 3464341760,3464341775,JP 3464341776,3464341823,US 3464341824,3464341831,PT -3464341832,3464342543,US +3464341832,3464341951,US +3464341952,3464341959,CZ +3464341960,3464342543,US 3464342544,3464342559,SE 3464342560,3464342567,US 3464342568,3464342575,PT @@ -115394,7 +117591,8 @@ 3465177088,3465179135,PE 3465179136,3465412607,US 3465412608,3465412871,HK -3465412872,3465412927,US +3465412872,3465412895,GB +3465412896,3465412927,US 3465412928,3465413055,HK 3465413056,3465413119,US 3465413120,3465413127,HK @@ -115402,7 +117600,15 @@ 3465413376,3465413383,HK 3465413384,3465462783,US 3465462784,3465463039,GB -3465463040,3465510911,US +3465463040,3465463279,US +3465463280,3465463295,GB +3465463296,3465466687,US +3465466688,3465466703,GB +3465466704,3465466879,US +3465466880,3465466975,GB +3465466976,3465467071,US +3465467072,3465467079,GB +3465467080,3465510911,US 3465510912,3465543679,JP 3465543680,3465982991,US 3465982992,3465983007,GB @@ -115887,7 +118093,9 @@ 3470148864,3470149119,CA 3470149120,3470150655,US 3470150656,3470150911,CA -3470150912,3470151935,US +3470150912,3470151295,US +3470151296,3470151359,CA +3470151360,3470151935,US 3470151936,3470152703,CA 3470152704,3470152959,US 3470152960,3470152975,CA @@ -116376,7 +118584,9 @@ 3470360800,3470360807,CA 3470360808,3470361039,US 3470361040,3470361055,CA -3470361056,3470361663,US +3470361056,3470361487,US +3470361488,3470361495,SG +3470361496,3470361663,US 3470361664,3470361671,CA 3470361672,3470361703,US 3470361704,3470361711,AF @@ -116402,9 +118612,12 @@ 3470362720,3470362727,CA 3470362728,3470362783,US 3470362784,3470362791,AF -3470362792,3470362847,US +3470362792,3470362799,SG +3470362800,3470362847,US 3470362848,3470362855,AR -3470362856,3470363423,US +3470362856,3470362895,US +3470362896,3470362911,AU +3470362912,3470363423,US 3470363424,3470363439,CA 3470363440,3470363535,US 3470363536,3470363543,CA @@ -116435,7 +118648,7 @@ 3470645512,3470645519,GB 3470645520,3470645527,FR 3470645528,3470645559,US -3470645560,3470645567,IT +3470645560,3470645567,RU 3470645568,3470645575,CA 3470645576,3470645583,US 3470645584,3470645591,FR @@ -116961,7 +119174,9 @@ 3479290744,3479290751,EC 3479290752,3479290767,US 3479290768,3479290783,CA -3479290784,3479291263,US +3479290784,3479291071,US +3479291072,3479291087,CA +3479291088,3479291263,US 3479291264,3479291287,CA 3479291288,3479291671,US 3479291672,3479291679,CA @@ -117012,13 +119227,16 @@ 3479294664,3479294671,AU 3479294672,3479294775,US 3479294776,3479294783,CA -3479294784,3479295015,US +3479294784,3479294911,US +3479294912,3479294919,CA +3479294920,3479295015,US 3479295016,3479295023,MY 3479295024,3479295071,US 3479295072,3479295079,CR 3479295080,3479295095,US 3479295096,3479295103,PR -3479295104,3479295359,US +3479295104,3479295111,CA +3479295112,3479295359,US 3479295360,3479295367,AU 3479295368,3479295519,US 3479295520,3479295527,CA @@ -117026,7 +119244,9 @@ 3479295584,3479295591,NL 3479295592,3479295703,US 3479295704,3479295735,MY -3479295736,3479296007,US +3479295736,3479295767,US +3479295768,3479295775,MY +3479295776,3479296007,US 3479296008,3479296015,CA 3479296016,3479296071,US 3479296072,3479296079,GB @@ -117470,9 +119690,7 @@ 3484472184,3484472199,US 3484472200,3484472223,CA 3484472224,3484472263,US -3484472264,3484472311,CA -3484472312,3484472319,US -3484472320,3484472839,CA +3484472264,3484472839,CA 3484472840,3484472855,US 3484472856,3484473007,CA 3484473008,3484473015,US @@ -117482,11 +119700,11 @@ 3484473048,3484473055,US 3484473056,3484473087,CA 3484473088,3484473151,US -3484473152,3484473343,CA -3484473344,3484473727,US +3484473152,3484473599,CA +3484473600,3484473727,US 3484473728,3484473791,CA -3484473792,3484474111,US -3484474112,3484477183,CA +3484473792,3484473855,US +3484473856,3484477183,CA 3484477184,3484477695,US 3484477696,3484478207,CA 3484478208,3484478719,US @@ -117814,11 +120032,13 @@ 3487203072,3487203327,DK 3487203328,3487236095,US 3487236096,3487301631,CA -3487301632,3487524607,US -3487524608,3487524863,GB -3487524864,3487525119,US -3487525120,3487526911,GB -3487526912,3487559711,US +3487301632,3487507327,US +3487507328,3487507335,CA +3487507336,3487507343,US +3487507344,3487507359,CA +3487507360,3487507375,US +3487507376,3487507391,CA +3487507392,3487559711,US 3487559712,3487559743,AU 3487559744,3487559855,US 3487559856,3487559871,AU @@ -118068,7 +120288,8 @@ 3493901024,3493901031,HK 3493901032,3493901311,US 3493901312,3493901567,AE -3493901568,3493901583,US +3493901568,3493901579,US +3493901580,3493901583,TT 3493901584,3493901599,VG 3493901600,3493901759,US 3493901760,3493901767,AE @@ -118109,14 +120330,15 @@ 3494044672,3494045695,CA 3494045696,3494049791,US 3494049792,3494051839,CA -3494051840,3494072319,US -3494074368,3494075391,US +3494051840,3494075391,US 3494075392,3494076415,CA 3494076416,3494088703,US 3494088704,3494090751,CA 3494090752,3494094847,US 3494094848,3494095871,CA -3494095872,3494101319,US +3494095872,3494101089,US +3494101090,3494101097,JM +3494101098,3494101319,US 3494101320,3494101327,GB 3494101328,3494101377,US 3494101378,3494101385,JM @@ -118138,8 +120360,7 @@ 3494110092,3494110109,CA 3494110110,3494110145,US 3494110146,3494110161,CA -3494110162,3494114303,US -3494115328,3494115471,US +3494110162,3494115471,US 3494115472,3494115487,AU 3494115488,3494115495,US 3494115496,3494115503,CA @@ -118243,9 +120464,7 @@ 3494291744,3494291751,GB 3494291752,3494291903,US 3494291904,3494291967,GB -3494291968,3494295039,US -3494295040,3494295551,UM -3494295552,3494299663,US +3494291968,3494299663,US 3494299664,3494299679,TH 3494299680,3494299687,US 3494299688,3494299695,SC @@ -118295,7 +120514,7 @@ 3494361088,3494362111,CA 3494362112,3494380543,US 3494380544,3494381567,CA -3494381568,3494409215,US +3494381568,3494410239,US 3494410240,3494412287,CA 3494412288,3494418511,US 3494418512,3494418527,GB @@ -118342,8 +120561,8 @@ 3494420224,3494420239,BO 3494420240,3494420415,US 3494420416,3494420447,CA -3494420448,3494424575,US -3494424576,3494425599,CA +3494420448,3494422527,US +3494422528,3494425599,CA 3494425600,3494427199,US 3494427200,3494427215,CA 3494427216,3494428223,US @@ -118388,8 +120607,7 @@ 3494459392,3494460415,CA 3494460416,3494464511,US 3494464512,3494465535,CA -3494465536,3494467583,US -3494469632,3494501023,US +3494465536,3494501023,US 3494501024,3494501039,AU 3494501040,3494510591,US 3494510592,3494512639,CA @@ -118497,7 +120715,9 @@ 3494745952,3494745959,GB 3494745960,3494746019,US 3494746020,3494746023,AU -3494746024,3494757375,US +3494746024,3494747135,US +3494747136,3494748159,CA +3494748160,3494757375,US 3494757376,3494758399,CA 3494758400,3494763007,US 3494763008,3494763015,ES @@ -118545,7 +120765,9 @@ 3494866944,3494867967,CA 3494867968,3494893567,US 3494893568,3494894591,CA -3494894592,3494917119,US +3494894592,3494906455,US +3494906456,3494906463,GB +3494906464,3494917119,US 3494917120,3494917631,CA 3494917632,3494928383,US 3494928384,3494930431,CA @@ -118569,7 +120791,7 @@ 3494964224,3494965247,PR 3494965248,3494968319,US 3494968320,3494972415,CA -3494972416,3494978559,US +3494972416,3494979583,US 3494979584,3494981631,CA 3494981632,3495000063,US 3495000064,3495001087,CA @@ -118583,7 +120805,9 @@ 3495014449,3495014456,SG 3495014457,3495023615,US 3495023616,3495024639,CA -3495024640,3495065599,US +3495024640,3495057407,US +3495057408,3495059455,CA +3495059456,3495065599,US 3495065600,3495066623,CA 3495066624,3495068031,US 3495068032,3495068047,PL @@ -118632,35 +120856,29 @@ 3495160480,3495160511,CA 3495160512,3495160535,US 3495160536,3495160543,BR -3495160544,3495160639,US -3495160640,3495160647,CA -3495160648,3495160663,US -3495160664,3495160671,NL -3495160672,3495160863,US +3495160544,3495160863,US 3495160864,3495160895,TR 3495160896,3495161055,US 3495161056,3495161087,TR 3495161088,3495161151,US -3495161152,3495161183,TR -3495161184,3495161599,US +3495161152,3495161167,TR +3495161168,3495161599,US 3495161600,3495161855,BR 3495161856,3495164239,US 3495164240,3495164247,CA 3495164248,3495187199,US 3495187200,3495187455,IM -3495187456,3495190527,US +3495187456,3495192575,US 3495192576,3495193599,CA 3495193600,3495215103,US 3495215104,3495217151,VI 3495217152,3495219199,VC -3495219200,3495235663,US -3495235664,3495235671,BR +3495219200,3495235671,US 3495235672,3495235679,AU -3495235680,3495235687,BR +3495235680,3495235687,US 3495235688,3495235695,IN 3495235696,3495235703,FR -3495235704,3495235775,US -3495235776,3495235783,MY +3495235704,3495235783,US 3495235784,3495235791,HR 3495235792,3495235823,US 3495235824,3495235831,GB @@ -118670,11 +120888,7 @@ 3495235992,3495235999,CA 3495236000,3495236015,US 3495236016,3495236023,GB -3495236024,3495236111,US -3495236112,3495236119,CZ -3495236120,3495236135,US -3495236136,3495236143,CR -3495236144,3495236367,US +3495236024,3495236367,US 3495236368,3495236375,CA 3495236376,3495251967,US 3495251968,3495254015,CA @@ -118776,8 +120990,8 @@ 3495441920,3495441935,PL 3495441936,3495442319,US 3495442320,3495442335,PL -3495442336,3495454719,US -3495454720,3495456767,CA +3495442336,3495455743,US +3495455744,3495456767,CA 3495456768,3495463935,US 3495463936,3495464959,CA 3495464960,3495475199,US @@ -118856,7 +121070,7 @@ 3495620608,3495622655,CA 3495622656,3495653375,US 3495653376,3495654399,CA -3495655424,3495657551,US +3495654400,3495657551,US 3495657552,3495657567,GB 3495657568,3495658015,US 3495658016,3495658023,IN @@ -118865,8 +121079,8 @@ 3495658328,3495673855,US 3495673856,3495674623,GP 3495674624,3495674879,MF -3495674880,3495686143,US -3495687168,3495688191,US +3495674880,3495675903,VG +3495675904,3495688191,US 3495688192,3495689215,CA 3495689216,3495694335,US 3495694336,3495696383,CA @@ -119249,7 +121463,9 @@ 3497447424,3497451519,CA 3497451520,3497713415,US 3497713416,3497713423,EC -3497713424,3497717759,US +3497713424,3497713543,US +3497713544,3497713551,CA +3497713552,3497717759,US 3497717760,3497719807,A2 3497719808,3497719839,MR 3497719840,3497721343,A2 @@ -119359,7 +121575,10 @@ 3500810248,3500810255,CA 3500810256,3500921279,US 3500921280,3500921311,AU -3500921312,3501181703,US +3500921312,3501146951,US +3501146952,3501146959,CA +3501146960,3501146975,GB +3501146976,3501181703,US 3501181704,3501181711,AU 3501181712,3501181727,KR 3501181728,3501181743,US @@ -119563,13 +121782,19 @@ 3507479109,3507479109,CA 3507479110,3507479154,US 3507479155,3507479155,CA -3507479156,3507482153,US +3507479156,3507479184,US +3507479185,3507479185,CA +3507479186,3507481766,US +3507481767,3507481767,CA +3507481768,3507482153,US 3507482154,3507482155,CA 3507482156,3507482197,US 3507482198,3507482198,CA 3507482199,3507482303,US 3507482304,3507482304,CA -3507482305,3507485103,US +3507482305,3507484047,US +3507484048,3507484063,CA +3507484064,3507485103,US 3507485104,3507485119,CA 3507485120,3507540015,US 3507540016,3507540031,IN @@ -121093,7 +123318,9 @@ 3512336384,3512369151,US 3512369152,3512385535,CA 3512385536,3512397823,US -3512397824,3512401919,CA +3512397824,3512399375,CA +3512399376,3512399383,US +3512399384,3512401919,CA 3512401920,3512418303,US 3512418304,3512451071,CA 3512451072,3512467455,PR @@ -121563,7 +123790,9 @@ 3514368000,3514433535,CA 3514433536,3514583479,US 3514583480,3514583487,PA -3514583488,3514587511,US +3514583488,3514583535,US +3514583536,3514583543,JE +3514583544,3514587511,US 3514587512,3514587519,PA 3514587520,3514589439,US 3514589440,3514589695,GT @@ -121611,10 +123840,11 @@ 3515456704,3515456767,JP 3515456768,3515596799,US 3515596800,3515613183,CA -3515613184,3515645951,US -3515678720,3515686911,US +3515613184,3515686911,US 3515686912,3515695103,CA 3515695104,3515711487,US +3515711488,3515731967,CA +3515731968,3515736063,US 3515744256,3515793351,US 3515793352,3515793359,MO 3515793360,3515867151,US @@ -121945,9 +124175,7 @@ 3517437136,3517437143,US 3517437144,3517437151,CA 3517437152,3517437175,US -3517437176,3517438015,CA -3517438016,3517438079,US -3517438080,3517438143,CA +3517437176,3517438143,CA 3517438144,3517438207,US 3517438208,3517438463,CA 3517438464,3517438527,US @@ -121987,9 +124215,7 @@ 3517446912,3517447167,US 3517447168,3517447727,CA 3517447728,3517447743,US -3517447744,3517447759,CA -3517447760,3517447767,US -3517447768,3517447783,CA +3517447744,3517447783,CA 3517447784,3517447791,US 3517447792,3517447847,CA 3517447848,3517447863,US @@ -122027,8 +124253,8 @@ 3517602304,3517602559,US 3517602560,3517602623,SE 3517602624,3517602815,US -3517602816,3517603327,SE -3517603328,3517603615,US +3517602816,3517603071,SE +3517603072,3517603615,US 3517603616,3517603647,DE 3517603648,3517603711,US 3517603712,3517603775,SE @@ -122078,8 +124304,12 @@ 3517979112,3517989487,US 3517989488,3517989503,CA 3517989504,3517989695,US -3517989696,3517989711,CA -3517989712,3518062591,US +3517989696,3517989727,CA +3517989728,3517990023,US +3517990024,3517990031,CA +3517990032,3517990911,US +3517990912,3517991423,CA +3517991424,3518062591,US 3518062592,3518066687,CA 3518066688,3518075455,US 3518075456,3518075519,AR @@ -122135,13 +124365,9 @@ 3518896232,3518896239,CN 3518896240,3518897119,US 3518897120,3518897127,IT -3518897128,3518898143,US -3518898144,3518898151,BR -3518898152,3518903191,US +3518897128,3518903191,US 3518903192,3518903199,JP -3518903200,3518903871,US -3518903872,3518903879,PK -3518903880,3518903927,US +3518903200,3518903927,US 3518903928,3518903935,RU 3518903936,3518904015,US 3518904016,3518904031,EG @@ -122324,8 +124550,8 @@ 3520020656,3520020663,US 3520020664,3520020719,CA 3520020720,3520020727,US -3520020728,3520020983,CA -3520020984,3520021015,US +3520020728,3520020991,CA +3520020992,3520021015,US 3520021016,3520021055,CA 3520021056,3520021071,US 3520021072,3520021183,CA @@ -122336,21 +124562,19 @@ 3520021384,3520021415,US 3520021416,3520021471,CA 3520021472,3520021479,US -3520021480,3520021487,CA -3520021488,3520021503,US +3520021480,3520021495,CA +3520021496,3520021503,US 3520021504,3520021687,CA 3520021688,3520021695,US 3520021696,3520021839,CA 3520021840,3520021847,US -3520021848,3520021863,CA -3520021864,3520021871,US -3520021872,3520021895,CA +3520021848,3520021895,CA 3520021896,3520021903,US 3520021904,3520021911,CA -3520021912,3520021935,US -3520021936,3520021943,CA -3520021944,3520021951,US -3520021952,3520021967,CA +3520021912,3520021919,US +3520021920,3520021927,CA +3520021928,3520021935,US +3520021936,3520021967,CA 3520021968,3520021975,US 3520021976,3520021983,CA 3520021984,3520021991,US @@ -122377,16 +124601,16 @@ 3520022936,3520023047,CA 3520023048,3520023063,US 3520023064,3520023087,CA -3520023088,3520023119,US +3520023088,3520023103,US +3520023104,3520023111,CA +3520023112,3520023119,US 3520023120,3520023127,CA 3520023128,3520023135,US 3520023136,3520023151,CA 3520023152,3520023159,US 3520023160,3520023383,CA 3520023384,3520023415,US -3520023416,3520023647,CA -3520023648,3520023655,US -3520023656,3520023879,CA +3520023416,3520023879,CA 3520023880,3520023887,US 3520023888,3520023911,CA 3520023912,3520023919,US @@ -122394,13 +124618,11 @@ 3520024216,3520024231,US 3520024232,3520024487,CA 3520024488,3520024495,US -3520024496,3520024799,CA -3520024800,3520024815,US +3520024496,3520024807,CA +3520024808,3520024815,US 3520024816,3520025191,CA 3520025192,3520025199,US -3520025200,3520025239,CA -3520025240,3520025247,US -3520025248,3520025295,CA +3520025200,3520025295,CA 3520025296,3520025303,US 3520025304,3520025343,CA 3520025344,3520025359,US @@ -122412,15 +124634,13 @@ 3520025624,3520025631,US 3520025632,3520025639,CA 3520025640,3520025647,US -3520025648,3520025767,CA -3520025768,3520025775,US -3520025776,3520025791,CA +3520025648,3520025791,CA 3520025792,3520025799,US 3520025800,3520026671,CA 3520026672,3520026679,US -3520026680,3520026719,CA -3520026720,3520026727,US -3520026728,3520026783,CA +3520026680,3520026703,CA +3520026704,3520026711,US +3520026712,3520026783,CA 3520026784,3520026791,US 3520026792,3520026895,CA 3520026896,3520026903,US @@ -122438,9 +124658,7 @@ 3520027440,3520027447,US 3520027448,3520027711,CA 3520027712,3520027727,US -3520027728,3520027783,CA -3520027784,3520027791,US -3520027792,3520027815,CA +3520027728,3520027815,CA 3520027816,3520027823,US 3520027824,3520027839,CA 3520027840,3520027887,US @@ -122458,8 +124676,8 @@ 3520028544,3520028551,US 3520028552,3520028655,CA 3520028656,3520028671,US -3520028672,3520028687,CA -3520028688,3520028711,US +3520028672,3520028695,CA +3520028696,3520028711,US 3520028712,3520029167,CA 3520029168,3520029175,IL 3520029176,3520030719,CA @@ -122582,7 +124800,9 @@ 3520075456,3520075471,CA 3520075472,3520075743,US 3520075744,3520075775,NL -3520075776,3520078911,US +3520075776,3520078031,US +3520078032,3520078047,BS +3520078048,3520078911,US 3520078912,3520078927,AU 3520078928,3520078943,EC 3520078944,3520081455,US @@ -122597,9 +124817,7 @@ 3520086960,3520086975,CA 3520086976,3520088447,US 3520088448,3520088463,ES -3520088464,3520092991,US -3520092992,3520093007,IL -3520093008,3520095455,US +3520088464,3520095455,US 3520095456,3520095471,NL 3520095472,3520097135,US 3520097136,3520097151,CA @@ -122764,7 +124982,7 @@ 3521933614,3521933621,US 3521933622,3521933629,IN 3521933630,3521933645,US -3521933646,3521933653,GB +3521933646,3521933653,PE 3521933654,3521933661,US 3521933662,3521933669,MA 3521933670,3521933725,US @@ -122784,7 +125002,9 @@ 3521934170,3521934177,EG 3521934178,3521934421,US 3521934422,3521934429,EG -3521934430,3521934477,US +3521934430,3521934437,US +3521934438,3521934445,IN +3521934446,3521934477,US 3521934478,3521934485,ID 3521934486,3521934509,US 3521934510,3521934517,IN @@ -122810,9 +125030,13 @@ 3521936394,3521936425,EG 3521936426,3521936669,US 3521936670,3521936677,PK -3521936678,3521936827,US +3521936678,3521936766,US +3521936767,3521936774,PE +3521936775,3521936827,US 3521936828,3521936860,EG -3521936861,3521936969,US +3521936861,3521936957,US +3521936958,3521936965,PE +3521936966,3521936969,US 3521936970,3521936977,PK 3521936978,3521936993,US 3521936994,3521937001,LK @@ -122850,8 +125074,7 @@ 3522131712,3522131743,GB 3522131744,3522132479,US 3522132480,3522132543,CO -3522132544,3522132575,NL -3522132576,3522132607,US +3522132544,3522132607,US 3522132608,3522132639,CA 3522132640,3522132671,BO 3522132672,3522132703,BR @@ -122879,7 +125102,9 @@ 3522854912,3522859999,CA 3522860000,3522860031,IN 3522860032,3522871295,CA -3522871296,3522937855,US +3522871296,3522902015,US +3522902016,3522903039,CA +3522903040,3522937855,US 3522937856,3522938367,GB 3522938368,3523215359,US 3523215360,3523223551,AU @@ -123206,7 +125431,6 @@ 3528884224,3528908799,TH 3528908800,3528912895,VN 3528912896,3528933375,AU -3528933376,3528949759,JP 3528949760,3528966143,CN 3528966144,3528974335,KR 3528974336,3528978431,JP @@ -123328,7 +125552,9 @@ 3556974592,3556982783,ES 3556982784,3556984623,DE 3556984624,3556984639,FR -3556984640,3556984655,DE +3556984640,3556984647,DE +3556984648,3556984651,FR +3556984652,3556984655,DE 3556984656,3556984663,FR 3556984664,3556984671,DE 3556984672,3556984719,FR @@ -123446,8 +125672,8 @@ 3557336192,3557336255,EU 3557336256,3557336319,BE 3557336320,3557336575,EU -3557336576,3557336655,BE -3557336656,3557336703,EU +3557336576,3557336663,BE +3557336664,3557336703,EU 3557336704,3557336831,BE 3557336832,3557338111,EU 3557338112,3557338367,BE @@ -123509,8 +125735,8 @@ 3557360560,3557360575,JE 3557360576,3557360680,GB 3557360681,3557360687,JE -3557360688,3557360847,GB -3557360848,3557360895,JE +3557360688,3557360863,GB +3557360864,3557360895,JE 3557360896,3557360927,GB 3557360928,3557360943,JE 3557360944,3557360959,GB @@ -123543,8 +125769,8 @@ 3557363672,3557363679,JE 3557363680,3557364103,GB 3557364104,3557364107,JE -3557364108,3557364119,GB -3557364120,3557364223,JE +3557364108,3557364135,GB +3557364136,3557364223,JE 3557364224,3557364479,GB 3557364480,3557364495,JE 3557364496,3557364527,GB @@ -123866,7 +126092,9 @@ 3558288640,3558288671,GB 3558288672,3558288687,DE 3558288688,3558288895,BE -3558288896,3558289111,FR +3558288896,3558289087,FR +3558289088,3558289103,GB +3558289104,3558289111,FR 3558289112,3558289119,GB 3558289120,3558289151,FR 3558289152,3558289407,GB @@ -123905,7 +126133,9 @@ 3558291280,3558291295,GB 3558291296,3558291455,CH 3558291456,3558291459,DE -3558291460,3558292223,GB +3558291460,3558291463,GB +3558291464,3558291471,DE +3558291472,3558292223,GB 3558292224,3558292287,SE 3558292288,3558292543,NL 3558292544,3558292607,GB @@ -124383,10 +126613,11 @@ 3559093216,3559093219,GB 3559093220,3559093239,BE 3559093240,3559093243,GB -3559093244,3559093567,BE +3559093244,3559093511,BE +3559093512,3559093567,GB 3559093568,3559093599,IT -3559093600,3559093647,BE -3559093648,3559093671,GB +3559093600,3559093663,BE +3559093664,3559093671,GB 3559093672,3559093687,BE 3559093688,3559093699,GB 3559093700,3559094019,BE @@ -124525,7 +126756,8 @@ 3559288832,3559289855,AZ 3559289856,3559292927,RU 3559292928,3559301119,JO -3559301120,3559309311,GB +3559301120,3559303167,DE +3559303168,3559309311,US 3559309312,3559317503,PL 3559317504,3559325695,FI 3559325696,3559333887,IT @@ -124554,9 +126786,11 @@ 3559490720,3559490751,NL 3559490752,3559490791,ES 3559490792,3559490799,BE -3559490800,3559491135,NL -3559491136,3559491199,ES -3559491200,3559491327,NL +3559490800,3559490815,NL +3559490816,3559491071,ES +3559491072,3559491135,NL +3559491136,3559491167,ES +3559491168,3559491327,NL 3559491328,3559491359,ES 3559491360,3559491439,NL 3559491440,3559491455,ES @@ -124698,13 +126932,19 @@ 3559902080,3559902175,UA 3559902176,3559902187,EE 3559902188,3559902191,UA -3559902192,3559902207,EE -3559902208,3559902975,UA -3559902976,3559903231,EE -3559903232,3559903487,UA -3559903488,3559904023,EE -3559904024,3559904255,UA -3559904256,3559904799,EE +3559902192,3559902215,EE +3559902216,3559902223,UA +3559902224,3559902239,EE +3559902240,3559902431,UA +3559902432,3559902463,EE +3559902464,3559902975,UA +3559902976,3559903295,EE +3559903296,3559903487,UA +3559903488,3559903999,EE +3559904000,3559904015,UA +3559904016,3559904023,EE +3559904024,3559904127,UA +3559904128,3559904799,EE 3559904800,3559905019,UA 3559905020,3559905031,EE 3559905032,3559905047,UA @@ -124725,7 +126965,8 @@ 3559905298,3559905299,UA 3559905300,3559905317,EE 3559905318,3559905319,LT -3559905320,3559905327,EE +3559905320,3559905323,UA +3559905324,3559905327,EE 3559905328,3559905535,UA 3559905536,3559905623,EE 3559905624,3559905631,UA @@ -124850,7 +127091,9 @@ 3560726528,3560734719,DK 3560734720,3560742911,DE 3560742912,3560751103,AT -3560751104,3560767487,DE +3560751104,3560761856,DE +3560761857,3560761857,A2 +3560761858,3560767487,DE 3560767488,3560832791,NL 3560832792,3560832799,BE 3560832800,3560833023,NL @@ -125580,8 +127823,8 @@ 3560943076,3560943079,ES 3560943080,3560943095,DE 3560943096,3560943099,PL -3560943100,3560943109,DE -3560943110,3560943111,ES +3560943100,3560943110,DE +3560943111,3560943111,ES 3560943112,3560943113,DE 3560943114,3560943115,ES 3560943116,3560943116,DK @@ -127262,9 +129505,7 @@ 3561938944,3561940991,IE 3561940992,3561942015,GB 3561942016,3561947135,IE -3561947136,3561963143,DE -3561963144,3561963151,AU -3561963152,3561963519,DE +3561947136,3561963519,DE 3561963520,3561971711,BE 3561971712,3561975807,CZ 3561975808,3561979903,UA @@ -127834,8 +130075,8 @@ 3563848384,3563848447,ES 3563848448,3563848575,NL 3563848576,3563848583,ES -3563848584,3563848655,NL -3563848656,3563848703,ES +3563848584,3563848671,NL +3563848672,3563848703,ES 3563848704,3563848979,NL 3563848980,3563848983,ES 3563848984,3563848987,NL @@ -127847,8 +130088,19 @@ 3563849216,3563849727,GB 3563849728,3563849999,NL 3563850000,3563850007,FI -3563850008,3563851023,NL -3563851024,3563851135,ES +3563850008,3563850015,NL +3563850016,3563850047,ES +3563850048,3563850111,NL +3563850112,3563850239,ES +3563850240,3563850751,GB +3563850752,3563850767,NL +3563850768,3563850783,ES +3563850784,3563850815,NL +3563850816,3563850831,ES +3563850832,3563850847,NL +3563850848,3563850879,ES +3563850880,3563851007,NL +3563851008,3563851135,ES 3563851136,3563851839,NL 3563851840,3563851903,ES 3563851904,3563852031,NL @@ -127917,7 +130169,9 @@ 3564068864,3564077055,ES 3564077056,3564093439,GB 3564093440,3564101631,UA -3564101632,3564109823,DE +3564101632,3564103743,DE +3564103744,3564103759,A2 +3564103760,3564109823,DE 3564109824,3564126207,SE 3564126208,3564128287,IT 3564128288,3564128303,BG @@ -128210,7 +130464,9 @@ 3564563712,3564563967,US 3564563968,3564564223,GB 3564564224,3564564239,US -3564564240,3564565231,GB +3564564240,3564564351,GB +3564564352,3564564415,US +3564564416,3564565231,GB 3564565232,3564565239,US 3564565240,3564565247,GB 3564565248,3564565279,US @@ -128262,7 +130518,9 @@ 3564572272,3564572287,DE 3564572288,3564572415,GB 3564572416,3564572543,DE -3564572544,3564576767,GB +3564572544,3564573695,GB +3564573696,3564574079,DE +3564574080,3564576767,GB 3564576768,3564584959,RU 3564584960,3564593151,SA 3564593152,3564601343,RU @@ -128320,8 +130578,8 @@ 3564736872,3564736887,GB 3564736888,3564736895,DE 3564736896,3564736903,GB -3564736904,3564736911,DE -3564736912,3564736959,GB +3564736904,3564736927,DE +3564736928,3564736959,GB 3564736960,3564736967,DE 3564736968,3564736975,GB 3564736976,3564736991,DE @@ -129347,9 +131605,7 @@ 3567388032,3567388159,GB 3567388160,3567388399,DE 3567388400,3567388415,GB -3567388416,3567388527,DE -3567388528,3567388543,GB -3567388544,3567388607,DE +3567388416,3567388607,DE 3567388608,3567388671,GB 3567388672,3567388927,CZ 3567388928,3567389183,DE @@ -129445,8 +131701,9 @@ 3567587328,3567591423,GB 3567591424,3567599615,IT 3567599616,3567615999,NL -3567616000,3567616511,BD -3567616512,3567616527,A2 +3567616000,3567616255,CG +3567616256,3567616263,CD +3567616264,3567616527,A2 3567616528,3567616535,GB 3567616536,3567616575,A2 3567616576,3567616583,GB @@ -129484,8 +131741,14 @@ 3567620960,3567620991,A2 3567620992,3567621055,NG 3567621056,3567621119,KE -3567621120,3567621375,GB -3567621376,3567621631,A2 +3567621120,3567621263,GB +3567621264,3567621279,CG +3567621280,3567621375,GB +3567621376,3567621391,A2 +3567621392,3567621399,CG +3567621400,3567621407,A2 +3567621408,3567621423,CG +3567621424,3567621631,A2 3567621632,3567621887,ID 3567621888,3567621895,TZ 3567621896,3567621903,KE @@ -129526,7 +131789,10 @@ 3567626240,3567627008,NG 3567627009,3567629311,A2 3567629312,3567630207,TJ -3567630208,3567647487,A2 +3567630208,3567635711,A2 +3567635712,3567635839,CG +3567635840,3567635967,CD +3567635968,3567647487,A2 3567647488,3567648767,GB 3567648768,3567665151,BE 3567665152,3567673343,ES @@ -129592,9 +131858,7 @@ 3568631808,3568697343,SE 3568697344,3568730111,PL 3568730112,3568746495,NL -3568746496,3568752895,FI -3568752896,3568752959,EE -3568752960,3568762879,FI +3568746496,3568762879,FI 3568762880,3568795647,AT 3568795648,3568803839,GB 3568803840,3568812031,IT @@ -129676,16 +131940,14 @@ 3569271912,3569271919,DE 3569271920,3569271935,BE 3569271936,3569271943,IT -3569271944,3569272063,BE -3569272064,3569272079,LU -3569272080,3569273167,BE +3569271944,3569273167,BE 3569273168,3569273183,LU 3569273184,3569273791,BE 3569273792,3569273823,LU 3569273824,3569274303,BE 3569274304,3569274311,LU -3569274312,3569274335,BE -3569274336,3569274383,LU +3569274312,3569274367,BE +3569274368,3569274383,LU 3569274384,3569274495,BE 3569274496,3569274559,LU 3569274560,3569274687,BE @@ -129967,7 +132229,9 @@ 3569830528,3569839187,IL 3569839188,3569839191,A2 3569839192,3569839359,IL -3569839360,3569839615,A2 +3569839360,3569839475,A2 +3569839476,3569839479,IL +3569839480,3569839615,A2 3569839616,3569846527,IL 3569846528,3569846783,A2 3569846784,3569851935,IL @@ -129982,7 +132246,9 @@ 3569873280,3569873407,A2 3569873408,3569876991,IL 3569876992,3569942527,RS -3569942528,3570073599,DE +3569942528,3570038463,DE +3570038464,3570038464,A2 +3570038465,3570073599,DE 3570073600,3570081791,NL 3570081792,3570106367,CH 3570106368,3570139135,PL @@ -130514,7 +132780,9 @@ 3571595744,3571646463,FI 3571646464,3571675679,DE 3571675680,3571675687,GB -3571675688,3571711999,DE +3571675688,3571710207,DE +3571710208,3571710463,GB +3571710464,3571711999,DE 3571712000,3571843071,GB 3571843072,3571974143,ES 3571974144,3571978239,RU @@ -130754,9 +133022,7 @@ 3575355232,3575355247,GB 3575355248,3575360199,ES 3575360200,3575360207,FR -3575360208,3575360319,ES -3575360320,3575360383,PT -3575360384,3575367111,ES +3575360208,3575367111,ES 3575367112,3575367119,DE 3575367120,3575372239,ES 3575372240,3575372247,PT @@ -131682,8 +133948,8 @@ 3576238512,3576238527,GB 3576238528,3576238559,FR 3576238560,3576238575,GB -3576238576,3576238607,FR -3576238608,3576238623,GB +3576238576,3576238615,FR +3576238616,3576238623,GB 3576238624,3576238639,FR 3576238640,3576238655,GB 3576238656,3576238863,FR @@ -131739,7 +134005,9 @@ 3576242336,3576242343,GB 3576242344,3576243967,FR 3576243968,3576243983,GB -3576243984,3576244127,FR +3576243984,3576244103,FR +3576244104,3576244111,GB +3576244112,3576244127,FR 3576244128,3576244143,GB 3576244144,3576246407,FR 3576246408,3576246463,GB @@ -131752,8 +134020,8 @@ 3576249368,3576249463,FR 3576249464,3576249471,GB 3576249472,3576249527,FR -3576249528,3576249535,GB -3576249536,3576249743,FR +3576249528,3576249567,GB +3576249568,3576249743,FR 3576249744,3576249791,GB 3576249792,3576249807,FR 3576249808,3576249823,GB @@ -131789,8 +134057,8 @@ 3576255152,3576255199,GB 3576255200,3576255215,FR 3576255216,3576255231,GB -3576255232,3576255239,FR -3576255240,3576255263,GB +3576255232,3576255247,FR +3576255248,3576255263,GB 3576255264,3576255375,FR 3576255376,3576255383,GB 3576255384,3576255407,FR @@ -131837,9 +134105,7 @@ 3576256896,3576256959,GB 3576256960,3576256991,FR 3576256992,3576257007,GB -3576257008,3576257079,FR -3576257080,3576257087,GB -3576257088,3576257103,FR +3576257008,3576257103,FR 3576257104,3576257111,GB 3576257112,3576257135,FR 3576257136,3576257151,GB @@ -131870,8 +134136,8 @@ 3576257872,3576257887,FR 3576257888,3576257903,GB 3576257904,3576257975,FR -3576257976,3576257999,GB -3576258000,3576258055,FR +3576257976,3576258015,GB +3576258016,3576258055,FR 3576258056,3576258063,GB 3576258064,3576258079,FR 3576258080,3576258095,GB @@ -131963,8 +134229,8 @@ 3576261080,3576261095,GB 3576261096,3576261103,FR 3576261104,3576261111,GB -3576261112,3576261383,FR -3576261384,3576261631,GB +3576261112,3576261391,FR +3576261392,3576261631,GB 3576261632,3576263463,FR 3576263464,3576263471,GB 3576263472,3576263503,FR @@ -132001,8 +134267,8 @@ 3576265288,3576265295,GB 3576265296,3576265303,FR 3576265304,3576265311,GB -3576265312,3576265327,FR -3576265328,3576265335,GB +3576265312,3576265319,FR +3576265320,3576265335,GB 3576265336,3576265343,FR 3576265344,3576265367,GB 3576265368,3576265375,FR @@ -132067,9 +134333,7 @@ 3576889344,3576954879,NL 3576954880,3576987647,NO 3576987648,3577001983,GB -3577001984,3577003615,NL -3577003616,3577003631,GB -3577003632,3577003647,NL +3577001984,3577003647,NL 3577003648,3577003711,GB 3577003712,3577003767,NL 3577003768,3577003771,GB @@ -132121,7 +134385,9 @@ 3577592432,3577592447,FR 3577592448,3577592743,DE 3577592744,3577592751,IE -3577592752,3577608743,DE +3577592752,3577607559,DE +3577607560,3577607567,NL +3577607568,3577608743,DE 3577608744,3577608751,ES 3577608752,3577610367,DE 3577610368,3577610495,GB @@ -132869,7 +135135,9 @@ 3582223088,3582223095,NL 3582223096,3582223967,SE 3582223968,3582223975,NO -3582223976,3582226599,SE +3582223976,3582224375,SE +3582224376,3582224379,FI +3582224380,3582226599,SE 3582226600,3582226607,FI 3582226608,3582230527,SE 3582230528,3582238719,BE @@ -133328,8 +135596,8 @@ 3583031920,3583032111,IT 3583032112,3583032127,GR 3583032128,3583032159,IT -3583032160,3583032191,FR -3583032192,3583032319,IT +3583032160,3583032255,FR +3583032256,3583032319,IT 3583032320,3583032575,FR 3583032576,3583032831,IT 3583032832,3583033087,SG @@ -133382,7 +135650,11 @@ 3583246336,3583254527,RU 3583254528,3583262719,GB 3583262720,3583270911,TR -3583270912,3583287295,DE +3583270912,3583283743,DE +3583283744,3583283747,A2 +3583283748,3583283867,DE +3583283868,3583283871,A2 +3583283872,3583287295,DE 3583287296,3583295487,RU 3583295488,3583303679,ES 3583303680,3583311871,NL @@ -133571,12 +135843,10 @@ 3583709448,3583709455,NA 3583709456,3583709503,UA 3583709504,3583709511,RU -3583709512,3583709575,UA -3583709576,3583709583,NA -3583709584,3583709607,UA +3583709512,3583709607,UA 3583709608,3583709615,RU 3583709616,3583709663,UA -3583709664,3583709671,NA +3583709664,3583709671,DE 3583709672,3583709699,UA 3583709700,3583709703,NA 3583709704,3583709727,UA @@ -133620,9 +135890,7 @@ 3583710744,3583710751,BE 3583710752,3583710855,UA 3583710856,3583710863,NA -3583710864,3583710871,UA -3583710872,3583710879,NA -3583710880,3583710887,UA +3583710864,3583710887,UA 3583710888,3583710895,NA 3583710896,3583710991,UA 3583710992,3583710999,NA @@ -133637,9 +135905,7 @@ 3583711360,3583711367,NA 3583711368,3583711375,UA 3583711376,3583711383,NA -3583711384,3583711631,UA -3583711632,3583711639,RU -3583711640,3583711695,UA +3583711384,3583711695,UA 3583711696,3583711703,NA 3583711704,3583711711,UA 3583711712,3583711719,NA @@ -133661,17 +135927,13 @@ 3583712456,3583712463,NA 3583712464,3583712471,UA 3583712472,3583712479,NA -3583712480,3583712655,UA -3583712656,3583712663,NA -3583712664,3583712775,UA +3583712480,3583712775,UA 3583712776,3583712783,NA 3583712784,3583712927,UA 3583712928,3583712943,NA 3583712944,3583713007,UA 3583713008,3583713015,RU -3583713016,3583713143,UA -3583713144,3583713151,NA -3583713152,3583713279,UA +3583713016,3583713279,UA 3583713280,3583721471,CZ 3583721472,3583729663,DE 3583729664,3583737855,TR @@ -133689,9 +135951,12 @@ 3583743616,3583743743,GB 3583743744,3583743975,EU 3583743976,3583743983,GB -3583743984,3583744067,EU +3583743984,3583744063,EU +3583744064,3583744067,FR 3583744068,3583744071,GB -3583744072,3583744103,EU +3583744072,3583744095,EU +3583744096,3583744099,DE +3583744100,3583744103,FR 3583744104,3583744111,GB 3583744112,3583744127,EU 3583744128,3583744255,GB @@ -133707,10 +135972,14 @@ 3583744840,3583744927,EU 3583744928,3583744959,DE 3583744960,3583744991,GB -3583744992,3583745279,EU +3583744992,3583744999,EU +3583745000,3583745003,NL +3583745004,3583745279,EU 3583745280,3583745535,GB 3583745536,3583745663,SE -3583745664,3583745799,EU +3583745664,3583745719,EU +3583745720,3583745723,FR +3583745724,3583745799,EU 3583745800,3583745807,NL 3583745808,3583745823,GB 3583745824,3583746047,EU @@ -134139,7 +136408,8 @@ 3585697536,3585697791,A2 3585697792,3585698047,NG 3585698048,3585698303,A2 -3585698304,3585698815,NG +3585698304,3585698559,US +3585698560,3585698815,A2 3585698816,3585699071,GB 3585699072,3585699583,A2 3585699584,3585699711,US @@ -134187,7 +136457,13 @@ 3585720320,3585728511,GB 3585728512,3585736703,SE 3585736704,3585744895,HR -3585744896,3585750335,FR +3585744896,3585749295,FR +3585749296,3585749311,ES +3585749312,3585750015,FR +3585750016,3585750031,GB +3585750032,3585750271,FR +3585750272,3585750287,ES +3585750288,3585750335,FR 3585750336,3585750351,GB 3585750352,3585750367,DE 3585750368,3585750383,ES @@ -134662,9 +136938,7 @@ 3587194880,3587211263,GB 3587211264,3587211531,AT 3587211532,3587211535,DE -3587211536,3587211663,AT -3587211664,3587211671,IT -3587211672,3587213911,AT +3587211536,3587213911,AT 3587213912,3587213919,UY 3587213920,3587219455,AT 3587219456,3587227647,RU @@ -134763,7 +137037,9 @@ 3587237376,3587237391,GB 3587237392,3587237399,NL 3587237400,3587237407,GB -3587237408,3587237471,NL +3587237408,3587237447,NL +3587237448,3587237455,GB +3587237456,3587237471,NL 3587237472,3587237487,GB 3587237488,3587237495,NL 3587237496,3587237519,GB @@ -134779,9 +137055,7 @@ 3587238608,3587238911,GB 3587238912,3587239303,NL 3587239304,3587239311,GB -3587239312,3587239703,NL -3587239704,3587239711,GB -3587239712,3587239903,NL +3587239312,3587239903,NL 3587239904,3587239935,GB 3587239936,3587239975,NL 3587239976,3587239983,GB @@ -134809,7 +137083,9 @@ 3587241344,3587241471,GB 3587241472,3587242095,NL 3587242096,3587242111,GB -3587242112,3587242463,NL +3587242112,3587242271,NL +3587242272,3587242287,GB +3587242288,3587242463,NL 3587242464,3587242471,GB 3587242472,3587243407,NL 3587243408,3587243415,GB @@ -135112,9 +137388,7 @@ 3588997120,3589013503,AT 3589013504,3589021695,ES 3589021696,3589029887,SA -3589029888,3589030183,NL -3589030184,3589030191,FR -3589030192,3589030303,NL +3589029888,3589030303,NL 3589030304,3589030335,FR 3589030336,3589034143,NL 3589034144,3589034159,ES @@ -135191,7 +137465,9 @@ 3589426176,3589426303,FR 3589426304,3589427199,GB 3589427200,3589427247,FR -3589427248,3589429247,GB +3589427248,3589428223,GB +3589428224,3589428255,CH +3589428256,3589429247,GB 3589429248,3589429503,FR 3589429504,3589430271,GB 3589430272,3589430543,FR @@ -135486,8 +137762,8 @@ 3589828608,3589828639,SE 3589828640,3589828671,NL 3589828672,3589828735,EU -3589828736,3589828863,NL -3589828864,3589829119,EU +3589828736,3589828991,NL +3589828992,3589829119,EU 3589829120,3589829183,GB 3589829184,3589829375,EU 3589829376,3589830655,GB @@ -135592,8 +137868,8 @@ 3590157824,3590157839,RS 3590157840,3590157855,SI 3590157856,3590157951,RS -3590157952,3590158015,SI -3590158016,3590158343,RS +3590157952,3590158079,SI +3590158080,3590158343,RS 3590158344,3590158407,SI 3590158408,3590158415,RS 3590158416,3590158431,SI @@ -135656,8 +137932,7 @@ 3590250752,3590251007,GR 3590251008,3590251263,IT 3590251264,3590251519,DE -3590251520,3590251551,ES -3590251552,3590251583,FR +3590251520,3590251583,FR 3590251584,3590251647,US 3590251648,3590251775,NL 3590251776,3590251839,FR @@ -135677,7 +137952,9 @@ 3590254136,3590254143,FR 3590254144,3590254271,US 3590254272,3590254463,FR -3590254464,3590254879,ES +3590254464,3590254591,ES +3590254592,3590254847,FR +3590254848,3590254879,ES 3590254880,3590254911,FR 3590254912,3590254975,ES 3590254976,3590255039,DE @@ -135721,12 +137998,12 @@ 3590308096,3590308103,IQ 3590308104,3590308119,A2 3590308120,3590308127,IQ -3590308128,3590308175,A2 +3590308128,3590308135,A2 +3590308136,3590308143,IQ +3590308144,3590308175,A2 3590308176,3590308191,IQ 3590308192,3590308199,A2 -3590308200,3590308215,IQ -3590308216,3590308223,A2 -3590308224,3590308238,IQ +3590308200,3590308238,IQ 3590308239,3590308247,A2 3590308248,3590308287,IQ 3590308288,3590308303,A2 @@ -135737,16 +138014,22 @@ 3590308368,3590308375,A2 3590308376,3590308391,IQ 3590308392,3590308399,A2 -3590308400,3590308439,IQ -3590308440,3590308447,A2 -3590308448,3590308455,IQ +3590308400,3590308407,IQ +3590308408,3590308415,A2 +3590308416,3590308455,IQ 3590308456,3590308463,A2 3590308464,3590308471,IQ -3590308472,3590308519,A2 +3590308472,3590308479,A2 +3590308480,3590308487,IQ +3590308488,3590308503,A2 +3590308504,3590308511,IQ +3590308512,3590308519,A2 3590308520,3590308527,IQ -3590308528,3590308567,A2 -3590308568,3590308575,IQ -3590308576,3590308607,A2 +3590308528,3590308551,A2 +3590308552,3590308559,IQ +3590308560,3590308567,A2 +3590308568,3590308583,IQ +3590308584,3590308607,A2 3590308608,3590308735,NG 3590308736,3590308767,A2 3590308768,3590308775,RW @@ -135793,9 +138076,7 @@ 3590309792,3590309799,A2 3590309800,3590309807,IQ 3590309808,3590309815,A2 -3590309816,3590309823,IQ -3590309824,3590309831,A2 -3590309832,3590309839,IQ +3590309816,3590309839,IQ 3590309840,3590309863,A2 3590309864,3590309871,IQ 3590309872,3590309879,A2 @@ -135803,8 +138084,7 @@ 3590309888,3590310143,A2 3590310144,3590310175,IQ 3590310176,3590310183,LR -3590310184,3590310207,IQ -3590310208,3590310215,SD +3590310184,3590310215,IQ 3590310216,3590310223,A2 3590310224,3590310255,IQ 3590310256,3590310263,A2 @@ -135832,10 +138112,10 @@ 3590311016,3590311023,IQ 3590311024,3590311031,A2 3590311032,3590311039,IQ -3590311040,3590311079,A2 -3590311080,3590311095,IQ -3590311096,3590311103,A2 -3590311104,3590311111,IQ +3590311040,3590311055,A2 +3590311056,3590311063,IQ +3590311064,3590311079,A2 +3590311080,3590311111,IQ 3590311112,3590311119,A2 3590311120,3590311135,IQ 3590311136,3590311143,A2 @@ -135855,7 +138135,7 @@ 3590311280,3590311287,NG 3590311288,3590311295,A2 3590311296,3590311303,LR -3590311304,3590311311,NG +3590311304,3590311311,A2 3590311312,3590311319,CD 3590311320,3590311327,CI 3590311328,3590311335,CD @@ -135999,7 +138279,7 @@ 3590323712,3590323903,A2 3590323904,3590323911,CD 3590323912,3590323967,A2 -3590323968,3590323975,SD +3590323968,3590323975,UG 3590323976,3590323983,A2 3590323984,3590323991,NG 3590323992,3590323999,A2 @@ -136128,9 +138408,7 @@ 3624380544,3624380551,CA 3624380552,3624380687,US 3624380688,3624380695,AU -3624380696,3624380735,US -3624380736,3624380743,VI -3624380744,3624380751,US +3624380696,3624380751,US 3624380752,3624380759,CA 3624380760,3624380767,CY 3624380768,3624380799,US @@ -136221,7 +138499,9 @@ 3624548040,3624548063,US 3624548064,3624548087,A2 3624548088,3624548095,US -3624548096,3624548679,A2 +3624548096,3624548351,A2 +3624548352,3624548359,US +3624548360,3624548679,A2 3624548680,3624548687,US 3624548688,3624548703,A2 3624548704,3624548711,US @@ -136373,9 +138653,7 @@ 3624908640,3624908687,CA 3624908688,3624908783,US 3624908784,3624908799,CA -3624908800,3624909375,US -3624909376,3624909407,VE -3624909408,3624909471,US +3624908800,3624909471,US 3624909472,3624909503,AU 3624909504,3624909567,US 3624909568,3624909822,MY @@ -139416,7 +141694,9 @@ 3637073168,3637073183,AD 3637073184,3637073215,US 3637073216,3637073231,CY -3637073232,3637073935,US +3637073232,3637073727,US +3637073728,3637073791,CY +3637073792,3637073935,US 3637073936,3637073959,AD 3637073960,3637074239,US 3637074240,3637074303,PA @@ -139454,7 +141734,8 @@ 3638198984,3638198991,ES 3638198992,3638199711,US 3638199712,3638199743,DE -3638199744,3638199815,US +3638199744,3638199807,US +3638199808,3638199815,CN 3638199816,3638199823,AU 3638199824,3638200007,US 3638200008,3638200015,JP @@ -139955,9 +142236,7 @@ 3641087696,3641087703,GB 3641087704,3641098191,DE 3641098192,3641098207,ES -3641098208,3641102607,DE -3641102608,3641102615,FR -3641102616,3641103719,DE +3641098208,3641103719,DE 3641103720,3641103727,HU 3641103728,3641106951,DE 3641106952,3641106959,CH @@ -140040,8 +142319,7 @@ 3641353840,3641353855,GB 3641353856,3641353983,A2 3641353984,3641354239,AF -3641354240,3641354255,GB -3641354256,3641354311,A2 +3641354240,3641354311,A2 3641354312,3641354319,GB 3641354320,3641354327,A2 3641354328,3641354335,GB @@ -140548,8 +142826,8 @@ 3642553162,3642553163,UA 3642553164,3642553175,LV 3642553176,3642553183,UA -3642553184,3642553215,LV -3642553216,3642553279,UA +3642553184,3642553223,LV +3642553224,3642553279,UA 3642553280,3642553343,LV 3642553344,3642553371,RU 3642553372,3642553379,UA @@ -140588,8 +142866,8 @@ 3642554208,3642554219,UA 3642554220,3642554223,RU 3642554224,3642554367,UA -3642554368,3642554559,LT -3642554560,3642554575,UA +3642554368,3642554567,LT +3642554568,3642554575,UA 3642554576,3642554623,LT 3642554624,3642554631,UA 3642554632,3642554671,LV @@ -140643,8 +142921,8 @@ 3642555432,3642555437,UA 3642555438,3642555439,PL 3642555440,3642555443,UA -3642555444,3642555471,PL -3642555472,3642555475,UA +3642555444,3642555455,PL +3642555456,3642555475,UA 3642555476,3642555493,PL 3642555494,3642555495,UA 3642555496,3642555503,PL @@ -140889,8 +143167,8 @@ 3645321216,3645325311,BA 3645325312,3645329407,IT 3645329408,3645333503,CH -3645333504,3645334031,DE -3645334032,3645334271,EU +3645333504,3645334039,DE +3645334040,3645334271,EU 3645334272,3645336927,DE 3645336928,3645336935,EU 3645336936,3645336991,DE @@ -140990,7 +143268,9 @@ 3645583360,3645587455,PL 3645587456,3645594711,SK 3645594712,3645594719,SR -3645594720,3645595647,SK +3645594720,3645594743,SK +3645594744,3645594751,SR +3645594752,3645595647,SK 3645595648,3645597751,SE 3645597752,3645597759,GB 3645597760,3645601471,SE @@ -141131,8 +143411,7 @@ 3645763691,3645763691,CH 3645763692,3645763692,DE 3645763693,3645763693,NL -3645763694,3645763694,IE -3645763695,3645763696,DE +3645763694,3645763696,DE 3645763697,3645763697,NL 3645763698,3645763698,LU 3645763699,3645763699,BE @@ -141511,8 +143790,8 @@ 3645764189,3645764189,FR 3645764190,3645764190,IT 3645764191,3645764191,GB -3645764192,3645764192,DE -3645764193,3645764194,ES +3645764192,3645764193,DE +3645764194,3645764194,ES 3645764195,3645764195,DE 3645764196,3645764196,IT 3645764197,3645764197,DE @@ -142073,9 +144352,11 @@ 3647972200,3647972259,GB 3647972260,3647972263,DE 3647972264,3647972351,GB -3647972352,3647973679,IT -3647973680,3647973695,DE -3647973696,3647973735,IT +3647972352,3647973623,IT +3647973624,3647973631,DE +3647973632,3647973679,IT +3647973680,3647973711,DE +3647973712,3647973735,IT 3647973736,3647973743,DE 3647973744,3647973783,IT 3647973784,3647973791,DE @@ -142113,11 +144394,21 @@ 3647978776,3647978783,DE 3647978784,3647978895,NL 3647978896,3647978911,DE -3647978912,3647978943,NL -3647978944,3647979007,DE +3647978912,3647978951,NL +3647978952,3647979007,DE 3647979008,3647979136,IT 3647979137,3647979519,DE -3647979520,3647980543,FR +3647979520,3647980215,FR +3647980216,3647980223,DE +3647980224,3647980239,FR +3647980240,3647980247,DE +3647980248,3647980255,FR +3647980256,3647980271,DE +3647980272,3647980343,FR +3647980344,3647980351,DE +3647980352,3647980495,FR +3647980496,3647980503,DE +3647980504,3647980543,FR 3647980544,3647981055,GB 3647981056,3647981567,IE 3647981568,3647982591,BE @@ -142131,17 +144422,17 @@ 3647986432,3647986687,FR 3647986688,3647986943,DE 3647986944,3647986951,ES -3647986952,3647987455,DE +3647986952,3647986975,DE +3647986976,3647986999,ES +3647987000,3647987455,DE 3647987456,3647987527,ES 3647987528,3647987543,DE -3647987544,3647987663,ES -3647987664,3647987679,DE -3647987680,3647987711,ES +3647987544,3647987711,ES 3647987712,3647988735,IT 3647988736,3647989759,BE 3647989760,3647995903,DE -3647995904,3648004095,RU -3648004096,3648004607,GB +3647995904,3648004223,RU +3648004224,3648004607,GB 3648004608,3648006271,RU 3648006272,3648006399,GB 3648006400,3648007167,RU @@ -142249,7 +144540,9 @@ 3648180992,3648181007,CY 3648181008,3648181023,DE 3648181024,3648181055,AT -3648181056,3648181215,DE +3648181056,3648181063,DE +3648181064,3648181071,DK +3648181072,3648181215,DE 3648181216,3648181231,RU 3648181232,3648181247,DE 3648181248,3648181279,IT @@ -142282,7 +144575,8 @@ 3648182824,3648182831,RU 3648182832,3648182847,DE 3648182848,3648182879,AT -3648182880,3648183551,DE +3648182880,3648182911,SG +3648182912,3648183551,DE 3648183552,3648183679,BR 3648183680,3648183871,DE 3648183872,3648183935,GB @@ -142298,7 +144592,8 @@ 3648225280,3648231263,DE 3648231264,3648231295,NL 3648231296,3648233471,DE -3648233472,3648237567,FR +3648233472,3648237311,FR +3648237312,3648237567,GB 3648237568,3648241663,RU 3648241664,3648245759,NL 3648245760,3648249855,RO @@ -142338,7 +144633,9 @@ 3648348160,3648352255,DE 3648352256,3648356351,RU 3648356352,3648360447,PL -3648360448,3648364543,FR +3648360448,3648362251,FR +3648362252,3648362255,GB +3648362256,3648364543,FR 3648364544,3648368639,CH 3648368640,3648372735,RU 3648372736,3648376831,LU @@ -142354,8 +144651,8 @@ 3648417808,3648417815,GB 3648417816,3648417831,AT 3648417832,3648417839,GB -3648417840,3648417879,AT -3648417880,3648417919,GB +3648417840,3648417855,AT +3648417856,3648417919,GB 3648417920,3648418047,AT 3648418048,3648418079,GB 3648418080,3648418247,ES @@ -142883,18 +145180,17 @@ 3651870720,3651874815,IT 3651874816,3651878911,PL 3651878912,3651883007,RU -3651883008,3651884031,BE -3651884032,3651885055,CD -3651885056,3651885567,BE -3651885568,3651885847,CD +3651883008,3651884031,CD +3651884032,3651884287,BE +3651884288,3651885847,CD 3651885848,3651885851,BE 3651885852,3651885855,CD 3651885856,3651885867,BE 3651885868,3651885875,CD 3651885876,3651885903,BE 3651885904,3651885927,CD -3651885928,3651886335,BE -3651886336,3651886347,CD +3651885928,3651886079,BE +3651886080,3651886347,CD 3651886348,3651886379,BE 3651886380,3651886383,CD 3651886384,3651886391,BE @@ -143264,13 +145560,9 @@ 3653409064,3653409071,CD 3653409072,3653409087,A2 3653409088,3653409095,CD -3653409096,3653409103,A2 -3653409104,3653409111,MW -3653409112,3653409127,A2 -3653409128,3653409143,CD -3653409144,3653409191,A2 -3653409192,3653409199,NG -3653409200,3653409223,A2 +3653409096,3653409127,A2 +3653409128,3653409135,CD +3653409136,3653409223,A2 3653409224,3653409231,NG 3653409232,3653409279,A2 3653409280,3653409311,TZ @@ -143403,7 +145695,6 @@ 3653586944,3653591039,DE 3653591040,3653595135,LU 3653595136,3653599231,RU -3653599232,3653603327,BG 3653603328,3653607423,CZ 3653607424,3653611519,PL 3653611520,3653615615,HU @@ -143420,8 +145711,8 @@ 3653660672,3653664767,CZ 3653664768,3653664895,DE 3653664896,3653665023,NL -3653665024,3653665039,DE -3653665040,3653668863,NL +3653665024,3653665071,DE +3653665072,3653668863,NL 3653668864,3653672959,SE 3653672960,3653681151,RU 3653681152,3653685247,ES @@ -143475,7 +145766,9 @@ 3654062080,3654066027,GB 3654066028,3654066031,MC 3654066032,3654287359,GB -3654287360,3654607871,SE +3654287360,3654607103,SE +3654607104,3654607359,DE +3654607360,3654607871,SE 3654607872,3654608127,NO 3654608128,3654608383,SE 3654608384,3654608895,PL @@ -143542,7 +145835,6 @@ 3663990784,3663991295,HK 3663991296,3663991551,MY 3663991552,3663991807,AU -3663991808,3663992063,JP 3663992064,3663992319,NZ 3663992320,3663992575,MY 3663992576,3663993599,NZ @@ -143585,6 +145877,8 @@ 3664008192,3664008447,MN 3664008448,3664008703,PK 3664008704,3664008959,MY +3664008960,3664009215,AU +3664009216,3664052223,CN 3664052224,3664084991,NZ 3664084992,3664117759,KR 3664117760,3664248831,HK @@ -143602,7 +145896,6 @@ 3669614592,3669616639,NZ 3669616640,3669618687,AU 3669618688,3669620735,BD -3669620736,3669622783,JP 3669622784,3669688319,SG 3669688320,3669753855,TW 3669753856,3669822719,HK @@ -143818,9 +146111,11 @@ 3742973952,3742982143,SG 3742982144,3742986239,ID 3742986240,3742988287,AU +3742988288,3742990335,VU 3742990336,3743006719,JP 3743006720,3743014911,TH 3743014912,3743016959,AU +3743016960,3743019007,SG 3743019008,3743022079,MY 3743022080,3743023103,SG 3743023104,3743027199,TW @@ -143835,6 +146130,7 @@ 3743106048,3743109119,JP 3743109120,3743113215,BD 3743113216,3743115263,AU +3743115264,3743117311,VN 3743117312,3743118335,BD 3743118336,3743119359,JP 3743119360,3743120383,IN @@ -143906,6 +146202,7 @@ 3755978752,3755986943,CN 3755986944,3755988991,JP 3755988992,3755990015,HK +3755990016,3755991039,SG 3755991040,3755999231,JP 3755999232,3757047807,IN 3757047808,3757834239,CN @@ -143921,3 +146218,5 @@ 3758092288,3758093311,HK 3758093312,3758094335,IN 3758094336,3758095359,AU +3758095360,3758095871,CN +3758095872,3758096127,SG diff --git a/src/config/torrc.bridge.in b/src/config/torrc.bridge.in new file mode 100644 index 0000000000..557b7adf46 --- /dev/null +++ b/src/config/torrc.bridge.in @@ -0,0 +1,171 @@ +## Configuration file for a typical Tor user +## Last updated 16 July 2009 for Tor 0.2.2.1-alpha. +## (May or may not work for much older or much newer versions of Tor.) +## +## Lines that begin with "## " try to explain what's going on. Lines +## that begin with just "#" are disabled commands: you can enable them +## by removing the "#" symbol. +## +## See 'man tor', or https://www.torproject.org/tor-manual.html, +## for more options you can use in this file. +## +## Tor will look for this file in various places based on your platform: +## https://wiki.torproject.org/noreply/TheOnionRouter/TorFAQ#torrc + + +## Replace this with "SocksPort 0" if you plan to run Tor only as a +## relay, and not make any local application connections yourself. +SocksPort 9050 # what port to open for local application connections +SocksListenAddress 127.0.0.1 # accept connections only from localhost +#SocksListenAddress 192.168.0.1:9100 # listen on this IP:port also + +## Entry policies to allow/deny SOCKS requests based on IP address. +## First entry that matches wins. If no SocksPolicy is set, we accept +## all (and only) requests from SocksListenAddress. +#SocksPolicy accept 192.168.0.0/16 +#SocksPolicy reject * + +## Logs go to stdout at level "notice" unless redirected by something +## else, like one of the below lines. You can have as many Log lines as +## you want. +## +## We advise using "notice" in most cases, since anything more verbose +## may provide sensitive information to an attacker who obtains the logs. +## +## Send all messages of level 'notice' or higher to @LOCALSTATEDIR@/log/tor/notices.log +#Log notice file @LOCALSTATEDIR@/log/tor/notices.log +## Send every possible message to @LOCALSTATEDIR@/log/tor/debug.log +#Log debug file @LOCALSTATEDIR@/log/tor/debug.log +## Use the system log instead of Tor's logfiles +#Log notice syslog +## To send all messages to stderr: +#Log debug stderr + +## Uncomment this to start the process in the background... or use +## --runasdaemon 1 on the command line. This is ignored on Windows; +## see the FAQ entry if you want Tor to run as an NT service. +#RunAsDaemon 1 + +## The directory for keeping all the keys/etc. By default, we store +## things in $HOME/.tor on Unix, and in Application Data\tor on Windows. +#DataDirectory @LOCALSTATEDIR@/lib/tor + +## The port on which Tor will listen for local connections from Tor +## controller applications, as documented in control-spec.txt. +#ControlPort 9051 +## If you enable the controlport, be sure to enable one of these +## authentication methods, to prevent attackers from accessing it. +#HashedControlPassword 16:872860B76453A77D60CA2BB8C1A7042072093276A3D701AD684053EC4C +#CookieAuthentication 1 + +############### This section is just for location-hidden services ### + +## Once you have configured a hidden service, you can look at the +## contents of the file ".../hidden_service/hostname" for the address +## to tell people. +## +## HiddenServicePort x y:z says to redirect requests on port x to the +## address y:z. + +#HiddenServiceDir @LOCALSTATEDIR@/lib/tor/hidden_service/ +#HiddenServicePort 80 127.0.0.1:80 + +#HiddenServiceDir @LOCALSTATEDIR@/lib/tor/other_hidden_service/ +#HiddenServicePort 80 127.0.0.1:80 +#HiddenServicePort 22 127.0.0.1:22 + +################ This section is just for relays ##################### +# +## See https://www.torproject.org/docs/tor-doc-relay for details. + +## Required: what port to advertise for incoming Tor connections. +ORPort 9001 +## If you want to listen on a port other than the one advertised +## in ORPort (e.g. to advertise 443 but bind to 9090), uncomment the +## line below too. You'll need to do ipchains or other port forwarding +## yourself to make this work. +#ORListenAddress 0.0.0.0:9090 + +## A handle for your relay, so people don't have to refer to it by key. +Nickname Unnamed + +## The IP address or full DNS name for your relay. Leave commented out +## and Tor will guess. +#Address noname.example.com + +## Define these to limit how much relayed traffic you will allow. Your +## own traffic is still unthrottled. Note that RelayBandwidthRate must +## be at least 20 KB. +#RelayBandwidthRate 100 KB # Throttle traffic to 100KB/s (800Kbps) +#RelayBandwidthBurst 200 KB # But allow bursts up to 200KB/s (1600Kbps) +RelayBandwidthBurst 10485760 +RelayBandwidthRate 5242880 + +## Use these to restrict the maximum traffic per day, week, or month. +## Note that this threshold applies to sent _and_ to received bytes, +## not to their sum: Setting "4 GB" may allow up to 8 GB +## total before hibernating. +## +## Set a maximum of 4 gigabytes each way per period. +#AccountingMax 4 GB +## Each period starts daily at midnight (AccountingMax is per day) +#AccountingStart day 00:00 +## Each period starts on the 3rd of the month at 15:00 (AccountingMax +## is per month) +#AccountingStart month 3 15:00 + +## Contact info to be published in the directory, so we can contact you +## if your relay is misconfigured or something else goes wrong. Google +## indexes this, so spammers might also collect it. +#ContactInfo Random Person <nobody AT example dot com> +## You might also include your PGP or GPG fingerprint if you have one: +#ContactInfo 1234D/FFFFFFFF Random Person <nobody AT example dot com> + +## Uncomment this to mirror directory information for others. Please do +## if you have enough bandwidth. +DirPort 9030 # what port to advertise for directory connections +## If you want to listen on a port other than the one advertised +## in DirPort (e.g. to advertise 80 but bind to 9091), uncomment the line +## below too. You'll need to do ipchains or other port forwarding yourself +## to make this work. +#DirListenAddress 0.0.0.0:9091 +## Uncomment to return an arbitrary blob of html on your DirPort. Now you +## can explain what Tor is if anybody wonders why your IP address is +## contacting them. See contrib/tor-exit-notice.html in Tor's source +## distribution for a sample. +#DirPortFrontPage @CONFDIR@/tor-exit-notice.html + +## Uncomment this if you run more than one Tor relay, and add the identity +## key fingerprint of each Tor relay you control, even if they're on +## different networks. You declare it here so Tor clients can avoid +## using more than one of your relays in a single circuit. See +## https://wiki.torproject.org/noreply/TheOnionRouter/TorFAQ#MultipleServers +#MyFamily $keyid,$keyid,... + +## A comma-separated list of exit policies. They're considered first +## to last, and the first match wins. If you want to _replace_ +## the default exit policy, end this with either a reject *:* or an +## accept *:*. Otherwise, you're _augmenting_ (prepending to) the +## default exit policy. Leave commented to just use the default, which is +## described in the man page or at +## https://www.torproject.org/documentation.html +## +## Look at https://www.torproject.org/faq-abuse.html#TypicalAbuses +## for issues you might encounter if you use the default exit policy. +## +## If certain IPs and ports are blocked externally, e.g. by your firewall, +## you should update your exit policy to reflect this -- otherwise Tor +## users will be told that those destinations are down. +## +#ExitPolicy accept *:6660-6667,reject *:* # allow irc ports but no more +#ExitPolicy accept *:119 # accept nntp as well as default exit policy +#ExitPolicy reject *:* # no exits allowed +# +## Bridge relays (or "bridges") are Tor relays that aren't listed in the +## main directory. Since there is no complete public list of them, even if an +## ISP is filtering connections to all the known Tor relays, they probably +## won't be able to block all the bridges. Also, websites won't treat you +## differently because they won't know you're running Tor. If you can +## be a real relay, please do; but if not, be a bridge! +BridgeRelay 1 +ExitPolicy reject *:* diff --git a/src/config/torrc.complete.in b/src/config/torrc.complete.in deleted file mode 100644 index 310458a5c0..0000000000 --- a/src/config/torrc.complete.in +++ /dev/null @@ -1,533 +0,0 @@ -# $Id$ -# Last updated on $Date$ -#################################################################### -## This config file is divided into four sections. They are: -## 1. Global Options (clients and servers) -## 2. Client Options Only -## 3. Server Options Only -## 4. Directory Server Options (for running your own Tor network) -## 5. Hidden Service Options (clients and servers) -## -## The conventions used are: -## double hash (##) is for summary text about the config option; -## single hash (#) is for the config option; and, -## the config option is always after the text. -#################################################################### - - -## Section 1: Global Options (clients and servers) - -## A token bucket limits the average incoming bandwidth on this node -## to the specified number of bytes per second. (Default: 2MB) -#BandwidthRate N bytes|KB|MB|GB|TB - -## Limit the maximum token bucket size (also known as the burst) to -## the given number of bytes. (Default: 5 MB) -#BandwidthBurst N bytes|KB|MB|GB|TB - -## If set, we will not advertise more than this amount of bandwidth -## for our BandwidthRate. Server operators who want to reduce the -## number of clients who ask to build circuits through them (since -## this is proportional to advertised bandwidth rate) can thus -## reduce the CPU demands on their server without impacting -## network performance. -#MaxAdvertisedBandwidth N bytes|KB|MB|GB|TB - -## If set, Tor will accept connections from the same machine -## (localhost only) on this port, and allow those connections to -## control the Tor process using the Tor Control Protocol -## (described in control-spec.txt). Note: unless you also specify -## one of HashedControlPassword or CookieAuthentication, setting -## this option will cause Tor to allow any process on the local -## host to control it. -#ControlPort Port - -## Don’t allow any connections on the control port except when the -## other process knows the password whose one-way hash is -## hashed_password. You can compute the hash of a password by -## running "tor --hash-password password". -#HashedControlPassword hashed_password - -## If this option is set to 1, don’t allow any connections on the -## control port except when the connecting process knows the -## contents of a file named "control_auth_cookie", which Tor will -## create in its data directory. This authentication method -## should only be used on systems with good filesystem security. -## (Default: 0) -#CookieAuthentication 0|1 - -## Store working data in DIR (Default: /usr/local/var/lib/tor) -#DataDirectory DIR - -## Every time the specified period elapses, Tor downloads a direc- -## tory. A directory contains a signed list of all known servers -## as well as their current liveness status. A value of "0 sec- -## onds" tells Tor to choose an appropriate default. -## (Default: 1 hour for clients, 20 minutes for servers) -#DirFetchPeriod N seconds|minutes|hours|days|weeks - -## Tor only trusts directories signed with one of these keys, and -## uses the given addresses to connect to the trusted directory -## servers. If no DirServer lines are specified, Tor uses the built-in -## defaults (moria1, moria2, tor26), so you can leave this alone unless -## you need to change it. -## -## WARNING! Changing these options will make your Tor behave -## differently from everyone else's, and hurt your anonymity. Even -## uncommenting these lines is a bad idea. They are the defaults now, -## but the defaults may change in the future, leaving you behind. -## -#DirServer moria1 v1 18.244.0.188:9031 FFCB 46DB 1339 DA84 674C 70D7 CB58 6434 C437 0441 -#DirServer moria2 v1 18.244.0.114:80 719B E45D E224 B607 C537 07D0 E214 3E2D 423E 74CF -#DirServer tor26 v1 86.59.21.38:80 847B 1F85 0344 D787 6491 A548 92F9 0493 4E4E B85D - -## On startup, setgid to this user. -#Group GID - -## Tor will make all its directory requests through this host:port -## (or host:80 if port is not specified), rather than connecting -## directly to any directory servers. -#HttpProxy host[:port] - -## If defined, Tor will use this username:password for Basic Http -## proxy authentication, as in RFC 2617. This is currently the -## only form of Http proxy authentication that Tor supports; feel -## free to submit a patch if you want it to support others. -#HttpProxyAuthenticator username:password - -## Tor will make all its OR (SSL) connections through this -## host:port (or host:443 if port is not specified), via HTTP CON- -## NECT rather than connecting directly to servers. You may want -## to set FascistFirewall to restrict the set of ports you might -## try to connect to, if your Https proxy only allows connecting -## to certain ports. -#HttpsProxy host[:port] - -## If defined, Tor will use this username:password for Basic Https -## proxy authentication, as in RFC 2617. This is currently the -## only form of Https proxy authentication that Tor supports; feel -## free to submit a patch if you want it to support others. -#HttpsProxyAuthenticator username:password - -## To keep firewalls from expiring connections, send a padding -## keepalive cell every NUM seconds on open connections that are -## in use. If the connection has no open circuits, it will instead -## be closed after NUM seconds of idleness. (Default: 5 minutes) -#KeepalivePeriod NUM - -## Send all messages between minSeverity and maxSeverity to the -## standard output stream, the standard error stream, or to the -## system log. (The "syslog" value is only supported on Unix.) -## Recognized severity levels are debug, info, notice, warn, and -## err. If only one severity level is given, all messages of that -## level or higher will be sent to the listed destination. -#Log minSeverity[-maxSeverity] stderr|stdout|syslog - -## As above, but send log messages to the listed filename. The -## "Log" option may appear more than once in a configuration file. -## Messages are sent to all the logs that match their severity -## level. -#Log minSeverity[-maxSeverity] file FILENAME - -## Maximum number of simultaneous sockets allowed. You probably -## don’t need to adjust this. (Default: 1024) -#MaxConn NUM - -## Make all outbound connections originate from the IP address -## specified. This is only useful when you have multiple network -## interfaces, and you want all of Tor’s outgoing connections to -## use a single one. -#OutboundBindAddress IP - -## On startup, write our PID to FILE. On clean shutdown, remove -## FILE. -#PIDFile FILE - -## If 1, Tor forks and daemonizes to the background. (Default: 0) -#RunAsDaemon 0|1 - -## If 1, Tor replaces potentially sensitive strings in the logs -## (e.g. addresses) with the string [scrubbed]. This way logs can -## still be useful, but they don’t leave behind personally identi- -## fying information about what sites a user might have visited. -## (Default: 1) -#SafeLogging 0|1 - -## Every time the specified period elapses, Tor downloads signed -## status information about the current state of known servers. A -## value of "0 seconds" tells Tor to choose an appropriate -## default. (Default: 30 minutes for clients, 15 minutes for -## servers) -#StatusFetchPeriod N seconds|minutes|hours|days|weeks - -## On startup, setuid to this user. -#User UID - -## If non-zero, try to use crypto hardware acceleration when -## available. (Default: 1) -#HardwareAccel 0|1 - - -## Section 2: Client Options Only - -## Where on our circuits should we allow Tor servers that the -## directory servers haven’t authenticated as "verified"? -## (Default: middle,rendezvous) -#AllowUnverifiedNodes entry|exit|middle|introduction|rendezvous|... - -## If set to 1, Tor will under no circumstances run as a server. -## The default is to run as a client unless ORPort is configured. -## (Usually, you don’t need to set this; Tor is pretty smart at -## figuring out whether you are reliable and high-bandwidth enough -## to be a useful server.) -## This option will likely be deprecated in the future; see the -## NoPublish option below. (Default: 0) -#ClientOnly 0|1 - -## A list of preferred nodes to use for the first hop in the -## circuit, if possible. -#EntryNodes nickname,nickname,... - -## A list of preferred nodes to use for the last hop in the -## circuit, if possible. -#ExitNodes nickname,nickname,... - -## A list of nodes to never use when building a circuit. -#ExcludeNodes nickname,nickname,... - -## If 1, Tor will never use any nodes besides those listed in -## "exitnodes" for the last hop of a circuit. -#StrictExitNodes 0|1 - -## If 1, Tor will never use any nodes besides those listed in -## "entrynodes" for the first hop of a circuit. -#StrictEntryNodes 0|1 - -## If 1, Tor will only create outgoing connections to ORs running -## on ports that your firewall allows (defaults to 80 and 443; see -## FirewallPorts). This will allow you to run Tor as a client -## behind a firewall with restrictive policies, but will not allow -## you to run as a server behind such a firewall. -#FascistFirewall 0|1 - -## A list of ports that your firewall allows you to connect to. -## Only used when FascistFirewall is set. (Default: 80, 443) -#FirewallPorts PORTS - -## A comma-separated list of IPs that your firewall allows you to -## connect to. Only used when FascistFirewall is set. The format -## is as for the addresses in ExitPolicy. -## For example, ’FirewallIPs 99.0.0.0/8, *:80’ means that your -## firewall allows connections to everything inside net 99, and -## to port 80 outside. -#FirewallIPs ADDR[/MASK][:PORT]... - -## A list of ports for services that tend to have long-running -## connections (e.g. chat and interactive shells). Circuits for -## streams that use these ports will contain only high-uptime -## nodes, to reduce the chance that a node will go down before the -## stream is finished. (Default: 21, 22, 706, 1863, 5050, 5190, -## 5222, 5223, 6667, 8300, 8888) -#LongLivedPorts PORTS - -## When a request for address arrives to Tor, it will rewrite it -## to newaddress before processing it. For example, if you always -## want connections to www.indymedia.org to exit via torserver -## (where torserver is the nickname of the server), -## use "MapAddress www.indymedia.org www.indymedia.org.torserver.exit". -#MapAddress address newaddress - -## Every NUM seconds consider whether to build a new circuit. -## (Default: 30 seconds) -#NewCircuitPeriod NUM - -## Feel free to reuse a circuit that was first used at most NUM -## seconds ago, but never attach a new stream to a circuit that is -## too old. (Default: 10 minutes) -#MaxCircuitDirtiness NUM - -## The named Tor servers constitute a "family" of similar or co- -## administered servers, so never use any two of them in the same -## circuit. Defining a NodeFamily is only needed when a server -## doesn’t list the family itself (with MyFamily). This option can -## be used multiple times. -#NodeFamily nickname,nickname,... - -## A list of preferred nodes to use for the rendezvous point, if -## possible. -#RendNodes nickname,nickname,... - -## A list of nodes to never use when choosing a rendezvous point. -#RendExcludeNodes nickname,nickname,... - -## Advertise this port to listen for connections from SOCKS-speak- -## ing applications. Set this to 0 if you don’t want to allow -## application connections. (Default: 9050) -#SOCKSPort PORT - -## Bind to this address to listen for connections from SOCKS- -## speaking applications. (Default: 127.0.0.1) You can also spec- -## ify a port (e.g. 192.168.0.1:9100). This directive can be spec- -## ified multiple times to bind to multiple addresses/ports. -#SOCKSBindAddress IP[:PORT] - -## Set an entrance policy for this server, to limit who can con- -## nect to the SOCKS ports. The policies have the same form as -## exit policies below. -#SOCKSPolicy policy,policy,... - -## For each value in the comma separated list, Tor will track -## recent connections to hosts that match this value and attempt -## to reuse the same exit node for each. If the value is prepended -## with a ’.’, it is treated as matching an entire domain. If one -## of the values is just a ’.’, it means match everything. This -## option is useful if you frequently connect to sites that will -## expire all your authentication cookies (ie log you out) if your -## IP address changes. Note that this option does have the disad- -## vantage of making it more clear that a given history is associ- -## ated with a single user. However, most people who would wish to -## observe this will observe it through cookies or other protocol- -## specific means anyhow. -#TrackHostExits host,.domain,... - -## Since exit servers go up and down, it is desirable to expire -## the association between host and exit server after NUM seconds. -## The default is 1800 seconds (30 minutes). -#TrackHostExitsExpire NUM - -## If this option is set to 1, we pick a few entry servers as our -## "helpers", and try to use only those fixed entry servers. This -## is desirable, because constantly changing servers increases the -## odds that an adversary who owns some servers will observe a -## fraction of your paths. (Defaults to 0; will eventually -## default to 1.) -#UseHelperNodes 0|1 - -## If UseHelperNodes is set to 1, we will try to pick a total of -## NUM helper nodes as entries for our circuits. (Defaults to 3.) -#NumHelperNodes NUM - - -## Section 3: Server Options Only - -## The IP or fqdn of this server (e.g. moria.mit.edu). You can -## leave this unset, and Tor will guess your IP. -#Address address - -## Administrative contact information for server. -#ContactInfo email_address - -## Set an exit policy for this server. Each policy is of the form -## "accept|reject ADDR[/MASK][:PORT]". If /MASK is omitted then -## this policy just applies to the host given. Instead of giving -## a host or network you can also use "*" to denote the universe -## (0.0.0.0/0). PORT can be a single port number, an interval of -## ports "FROM_PORT-TO_PORT", or "*". If PORT is omitted, that -## means "*". -## -## For example, "reject 127.0.0.1:*,reject 192.168.1.0/24:*,accept -## *:*" would reject any traffic destined for localhost and any -## 192.168.1.* address, but accept anything else. -## -## This directive can be specified multiple times so you don’t -## have to put it all on one line. -## -## See RFC 3330 for more details about internal and reserved IP -## address space. Policies are considered first to last, and the -## first match wins. If you want to _replace_ the default exit -## policy, end your exit policy with either a reject *:* or an -## accept *:*. Otherwise, you’re _augmenting_ (prepending to) the -## default exit policy. The default exit policy is: -## reject 0.0.0.0/8 -## reject 169.254.0.0/16 -## reject 127.0.0.0/8 -## reject 192.168.0.0/16 -## reject 10.0.0.0/8 -## reject 172.16.0.0/12 -## reject *:25 -## reject *:119 -## reject *:135-139 -## reject *:445 -## reject *:1214 -## reject *:4661-4666 -## reject *:6346-6429 -## reject *:6699 -## reject *:6881-6999 -## accept *:* -#ExitPolicy policy,policy,... - -## If you have more than this number of onionskins queued for -## decrypt, reject new ones. (Default: 100) -#MaxOnionsPending NUM - -## Declare that this Tor server is controlled or administered by a -## group or organization identical or similar to that of the other -## named servers. When two servers both declare that they are in -## the same ’family’, Tor clients will not use them in the same -## circuit. (Each server only needs to list the other servers in -## its family; it doesn’t need to list itself, but it won’t hurt.) -#MyFamily nickname,nickname,... - -## Set the server’s nickname to ’name’. -#Nickname name - -## If you set NoPublish 1, Tor will act as a server if you have an -## ORPort defined, but it will not publish its descriptor to the -## dirservers. This option is useful if you're testing out your -## server, or if you're using alternate dirservers (e.g. for other -## Tor networks such as Blossom). (Default: 0) -#NoPublish 0|1 - -## How many processes to use at once for decrypting onionskins. -## (Default: 1) -#NumCPUs num - -## Advertise this port to listen for connections from Tor clients -## and servers. -#ORPort PORT - -## Bind to this IP address to listen for connections from Tor -## clients and servers. If you specify a port, bind to this port -## rather than the one specified in ORPort. (Default: 0.0.0.0) -#ORBindAddress IP[:PORT] - -## Whenever an outgoing connection tries to connect to one of a -## given set of addresses, connect to target (an address:port -## pair) instead. The address pattern is given in the same format -## as for an exit policy. The address translation applies after -## exit policies are applied. Multiple RedirectExit options can -## be used: once any one has matched successfully, no subsequent -## rules are considered. You can specify that no redirection is -## to be performed on a given set of addresses by using the spe- -## cial target string "pass", which prevents subsequent rules from -## being considered. -#RedirectExit pattern target - -## When we get a SIGINT and we're a server, we begin shutting -## down: we close listeners and start refusing new circuits. After -## NUM seconds, we exit. If we get a second SIGINT, we exit imme- -## diately. (Default: 30 seconds) -#ShutdownWaitLengthNUM - -## Every time the specified period elapses, Tor uploads its server -## descriptors to the directory servers. This information is also -## uploaded whenever it changes. (Default: 20 minutes) -#DirPostPeriod N seconds|minutes|hours|days|weeks - -## A token bucket limits the average relayed bandwidth (server -## traffic only, not client traffic) on this node to the specified -## number of bytes per second. -#RelayBandwidthRate N bytes|KB|MB|GB|TB - -## Limit the maximum token bucket size (also known as the burst) for -## relayed traffic (server traffic only, not client traffic) to the -## given number of bytes. -#RelayBandwidthBurst N bytes|KB|MB|GB|TB - -## Never send more than the specified number of bytes in a given -## accounting period, or receive more than that number in the -## period. For example, with AccountingMax set to 1 GB, a server -## could send 900 MB and receive 800 MB and continue running. It -## will only hibernate once one of the two reaches 1 GB. When the -## number of bytes is exhausted, Tor will hibernate until some -## time in the next accounting period. To prevent all servers -## from waking at the same time, Tor will also wait until a random -## point in each period before waking up. If you have bandwidth -## cost issues, enabling hibernation is preferable to setting a -## low bandwidth, since it provides users with a collection of -## fast servers that are up some of the time, which is more useful -## than a set of slow servers that are always "available". -#AccountingMax N bytes|KB|MB|GB|TB - -## Specify how long accounting periods last. If month is given, -## each accounting period runs from the time HH:MM on the dayth -## day of one month to the same day and time of the next. (The -## day must be between 1 and 28.) If week is given, each account- -## ing period runs from the time HH:MM of the dayth day of one -## week to the same day and time of the next week, with Monday as -## day 1 and Sunday as day 7. If day is given, each accounting -## period runs from the time HH:MM each day to the same time on -## the next day. All times are local, and given in 24-hour time. -## (Defaults to "month 1 0:00".) -#AccountingStart day|week|month [day] HH:MM - - -## Section 4: Directory Server Options (for running your own Tor -## network) - -## When this option is set to 1, Tor operates as an authoritative -## directory server. Instead of caching the directory, it gener- -## ates its own list of good servers, signs it, and sends that to -## the clients. Unless the clients already have you listed as a -## trusted directory, you probably do not want to set this option. -## Please coordinate with the other admins at -## tor-ops@freehaven.net if you think you should be a directory. -#AuthoritativeDirectory 0|1 - -## Advertise the directory service on this port. -#DirPort PORT - -## Bind the directory service to this address. If you specify a -## port, bind to this port rather than the one specified in DirPort. -## (Default: 0.0.0.0) -#DirBindAddress IP[:PORT] - -## Set an entrance policy for this server, to limit who can con- -## nect to the directory ports. The policies have the same form -## as exit policies above. -#DirPolicy policy,policy,... - -## STRING is a command-separated list of Tor versions currently -## believed to be safe. The list is included in each directory, -## and nodes which pull down the directory learn whether they need -## to upgrade. This option can appear multiple times: the values -## from multiple lines are spliced together. -#RecommendedVersions STRING - - -## If set to 1, Tor will accept router descriptors with arbitrary -## "Address" elements. Otherwise, if the address is not an IP or -## is a private IP, it will reject the router descriptor. Defaults -## to 0. -#DirAllowPrivateAddresses 0|1 - -## If set to 1, Tor tries to build circuits through all of the -## servers it knows about, so it can tell which are up and which -## are down. This option is only useful for authoritative direc- -## tories, so you probably don't want to use it. -#RunTesting 0|1 - -## Section 5: Hidden Service Options (clients and servers) - -## Store data files for a hidden service in DIRECTORY. Every hid- -## den service must have a separate directory. You may use this -## option multiple times to specify multiple services. -#HiddenServiceDir DIRECTORY - -## Configure a virtual port VIRTPORT for a hidden service. You -## may use this option multiple times; each time applies to the -## service using the most recent hiddenservicedir. By default, -## this option maps the virtual port to the same port on -## 127.0.0.1. You may override the target port, address, or both -## by specifying a target of addr, port, or addr:port. -#HiddenServicePort VIRTPORT [TARGET] - -## If possible, use the specified nodes as introduction points for -## the hidden service. If this is left unset, Tor will be smart -## and pick some reasonable ones; most people can leave this unset. -#HiddenServiceNodes nickname,nickname,... - -## Do not use the specified nodes as introduction points for the -## hidden service. In normal use there is no reason to set this. -#HiddenServiceExcludeNodes nickname,nickname,... - -## Publish the given rendezvous service descriptor versions for the -## hidden service. -#HiddenServiceVersion 0,2 - -## Every time the specified period elapses, Tor uploads any ren- -## dezvous service descriptors to the directory servers. This -## information is also uploaded whenever it changes. -## (Default: 20 minutes) -#RendPostPeriod N seconds|minutes|hours|days|weeks -# diff --git a/src/config/torrc.sample.in b/src/config/torrc.sample.in index d0b1ee1591..f0c78ce5a9 100644 --- a/src/config/torrc.sample.in +++ b/src/config/torrc.sample.in @@ -1,5 +1,5 @@ ## Configuration file for a typical Tor user -## Last updated 12 April 2009 for Tor 0.2.1.14-rc. +## Last updated 16 July 2009 for Tor 0.2.2.1-alpha. ## (May or may not work for much older or much newer versions of Tor.) ## ## Lines that begin with "## " try to explain what's going on. Lines @@ -95,9 +95,22 @@ SocksListenAddress 127.0.0.1 # accept connections only from localhost ## Define these to limit how much relayed traffic you will allow. Your ## own traffic is still unthrottled. Note that RelayBandwidthRate must -## be at least 20 KBytes. -#RelayBandwidthRate 100 KBytes # Throttle traffic to 100KB/s (800Kbps) -#RelayBandwidthBurst 200 KBytes # But allow bursts up to 200KB/s (1600Kbps) +## be at least 20 KB. +#RelayBandwidthRate 100 KB # Throttle traffic to 100KB/s (800Kbps) +#RelayBandwidthBurst 200 KB # But allow bursts up to 200KB/s (1600Kbps) + +## Use these to restrict the maximum traffic per day, week, or month. +## Note that this threshold applies to sent _and_ to received bytes, +## not to their sum: Setting "4 GB" may allow up to 8 GB +## total before hibernating. +## +## Set a maximum of 4 gigabytes each way per period. +#AccountingMax 4 GB +## Each period starts daily at midnight (AccountingMax is per day) +#AccountingStart day 00:00 +## Each period starts on the 3rd of the month at 15:00 (AccountingMax +## is per month) +#AccountingStart month 3 15:00 ## Contact info to be published in the directory, so we can contact you ## if your relay is misconfigured or something else goes wrong. Google @@ -116,8 +129,9 @@ SocksListenAddress 127.0.0.1 # accept connections only from localhost #DirListenAddress 0.0.0.0:9091 ## Uncomment to return an arbitrary blob of html on your DirPort. Now you ## can explain what Tor is if anybody wonders why your IP address is -## contacting them. See contrib/tor-exit-notice.html for a sample. -#DirPortFrontPage /etc/tor/exit-notice.html +## contacting them. See contrib/tor-exit-notice.html in Tor's source +## distribution for a sample. +#DirPortFrontPage @CONFDIR@/tor-exit-notice.html ## Uncomment this if you run more than one Tor relay, and add the identity ## key fingerprint of each Tor relay you control, even if they're on diff --git a/src/or/Makefile.am b/src/or/Makefile.am index b56d496431..a9ac3cdee1 100644 --- a/src/or/Makefile.am +++ b/src/or/Makefile.am @@ -1,8 +1,5 @@ -TESTS = test - -noinst_PROGRAMS = test - bin_PROGRAMS = tor +noinst_LIBRARIES = libtor.a if BUILD_NT_SERVICES tor_platform_source=ntmain.c @@ -10,18 +7,30 @@ else tor_platform_source= endif -EXTRA_DIST=ntmain.c +EXTRA_DIST=ntmain.c or_sha1.i + +if USE_EXTERNAL_EVDNS +evdns_source= +else +evdns_source=eventdns.c +endif -tor_SOURCES = buffers.c circuitbuild.c circuitlist.c \ +libtor_a_SOURCES = buffers.c circuitbuild.c circuitlist.c \ circuituse.c command.c config.c \ connection.c connection_edge.c connection_or.c control.c \ cpuworker.c directory.c dirserv.c dirvote.c \ dns.c dnsserv.c geoip.c hibernate.c main.c $(tor_platform_source) \ + microdesc.c \ networkstatus.c onion.c policies.c \ reasons.c relay.c rendcommon.c rendclient.c rendmid.c \ rendservice.c rephist.c router.c routerlist.c routerparse.c \ - eventdns.c \ - tor_main.c + $(evdns_source) config_codedigest.c + +#libtor_a_LIBADD = ../common/libor.a ../common/libor-crypto.a \ +# ../common/libor-event.a + + +tor_SOURCES = tor_main.c AM_CPPFLAGS = -DSHARE_DATADIR="\"$(datadir)\"" \ -DLOCALSTATEDIR="\"$(localstatedir)\"" \ @@ -32,42 +41,48 @@ AM_CPPFLAGS = -DSHARE_DATADIR="\"$(datadir)\"" \ # matters a lot there, and is quite hard to debug if you forget to do it. tor_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ @TOR_LDFLAGS_libevent@ -tor_LDADD = ../common/libor.a ../common/libor-crypto.a \ - -lz @TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ -test_SOURCES = buffers.c circuitbuild.c circuitlist.c \ - circuituse.c command.c config.c \ - connection.c connection_edge.c connection_or.c control.c \ - cpuworker.c directory.c dirserv.c dirvote.c \ - dns.c dnsserv.c geoip.c hibernate.c main.c $(tor_platform_source) \ - networkstatus.c onion.c policies.c \ - reasons.c relay.c rendcommon.c rendclient.c rendmid.c \ - rendservice.c rephist.c router.c routerlist.c routerparse.c \ - eventdns.c \ - test_data.c test.c +tor_LDADD = ./libtor.a ../common/libor.a ../common/libor-crypto.a \ + ../common/libor-event.a \ + @TOR_ZLIB_LIBS@ -lm @TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ -test_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ \ - @TOR_LDFLAGS_libevent@ -test_LDADD = ../common/libor.a ../common/libor-crypto.a \ - -lz @TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ +noinst_HEADERS = buffers.h circuitbuild.h circuitlist.h circuituse.h \ + command.h config.h connection_edge.h connection.h connection_or.h \ + control.h cpuworker.h directory.h dirserv.h dirvote.h dns.h \ + dnsserv.h geoip.h hibernate.h main.h microdesc.h networkstatus.h \ + ntmain.h onion.h policies.h reasons.h relay.h rendclient.h \ + rendcommon.h rendmid.h rendservice.h rephist.h router.h routerlist.h \ + routerparse.h or.h eventdns.h eventdns_tor.h micro-revision.i -noinst_HEADERS = or.h eventdns.h eventdns_tor.h micro-revision.i +config_codedigest.o: or_sha1.i tor_main.o: micro-revision.i micro-revision.i: FORCE @rm -f micro-revision.tmp; \ - if test -d ../../.git && test -x "`which git 2>&1;true`"; then \ - HASH="`git rev-parse --short=16 HEAD`"; \ - echo \"$$HASH\" > micro-revision.tmp; \ - fi; \ + if test -d ../../.git && test -x "`which git 2>&1;true`"; then \ + HASH="`git rev-parse --short=16 HEAD`"; \ + echo \"$$HASH\" > micro-revision.tmp; \ + fi; \ if test ! -f micro-revision.tmp ; then \ if test ! -f micro-revision.i ; then \ echo '""' > micro-revision.i; \ fi; \ elif test ! -f micro-revision.i || \ - test x"`cat micro-revision.tmp`" != x"`cat micro-revision.i`"; then \ + test x"`cat micro-revision.tmp`" != x"`cat micro-revision.i`"; then \ mv micro-revision.tmp micro-revision.i; \ fi; true +or_sha1.i: $(tor_SOURCES) + if test "@SHA1SUM@" != none; then \ + @SHA1SUM@ $(tor_SOURCES) | @SED@ -n 's/^\(.*\)$$/"\1\\n"/p' > or_sha1.i; \ + elif test "@OPENSSL@" != none; then \ + @OPENSSL@ sha1 $(tor_SOURCES) | @SED@ -n 's/SHA1(\(.*\))= \(.*\)/"\2 \1\\n"/p' > or_sha1.i; \ + else \ + rm or_sha1.i; \ + touch or_sha1.i; \ + fi + +CLEANFILES = micro-revision.i + #Dummy target to ensure that micro-revision.i _always_ gets built. FORCE: diff --git a/src/or/buffers.c b/src/or/buffers.c index 710374638b..db926955b4 100644 --- a/src/or/buffers.c +++ b/src/or/buffers.c @@ -12,6 +12,14 @@ **/ #define BUFFERS_PRIVATE #include "or.h" +#include "buffers.h" +#include "config.h" +#include "connection_edge.h" +#include "connection_or.h" +#include "control.h" +#include "reasons.h" +#include "../common/util.h" +#include "../common/torlog.h" #ifdef HAVE_UNISTD_H #include <unistd.h> #endif @@ -145,10 +153,13 @@ get_freelist(size_t alloc) /** Deallocate a chunk or put it on a freelist */ static void -chunk_free(chunk_t *chunk) +chunk_free_unchecked(chunk_t *chunk) { - size_t alloc = CHUNK_ALLOC_SIZE(chunk->memlen); - chunk_freelist_t *freelist = get_freelist(alloc); + size_t alloc; + chunk_freelist_t *freelist; + + alloc = CHUNK_ALLOC_SIZE(chunk->memlen); + freelist = get_freelist(alloc); if (freelist && freelist->cur_length < freelist->max_length) { chunk->next = freelist->head; freelist->head = chunk; @@ -193,7 +204,7 @@ chunk_new_with_alloc_size(size_t alloc) } #else static void -chunk_free(chunk_t *chunk) +chunk_free_unchecked(chunk_t *chunk) { tor_free(chunk); } @@ -259,13 +270,22 @@ buf_shrink_freelists(int free_all) int n_to_free = free_all ? freelists[i].cur_length : (freelists[i].lowest_length - slack); int n_to_skip = freelists[i].cur_length - n_to_free; + int orig_length = freelists[i].cur_length; int orig_n_to_free = n_to_free, n_freed=0; int orig_n_to_skip = n_to_skip; int new_length = n_to_skip; chunk_t **chp = &freelists[i].head; chunk_t *chunk; while (n_to_skip) { - tor_assert((*chp)->next); + if (! (*chp)->next) { + log_warn(LD_BUG, "I wanted to skip %d chunks in the freelist for " + "%d-byte chunks, but only found %d. (Length %d)", + orig_n_to_skip, (int)freelists[i].alloc_size, + orig_n_to_skip-n_to_skip, freelists[i].cur_length); + assert_freelist_ok(&freelists[i]); + goto done; + } + // tor_assert((*chp)->next); chp = &(*chp)->next; --n_to_skip; } @@ -290,13 +310,15 @@ buf_shrink_freelists(int free_all) } // tor_assert(!n_to_free); freelists[i].cur_length = new_length; - log_info(LD_MM, "Cleaned freelist for %d-byte chunks: kept %d, " - "dropped %d.", - (int)freelists[i].alloc_size, orig_n_to_skip, orig_n_to_free); + log_info(LD_MM, "Cleaned freelist for %d-byte chunks: original " + "length %d, kept %d, dropped %d.", + (int)freelists[i].alloc_size, orig_length, + orig_n_to_skip, orig_n_to_free); } freelists[i].lowest_length = freelists[i].cur_length; assert_freelist_ok(&freelists[i]); } + done: enable_control_logging(); #else (void) free_all; @@ -404,7 +426,7 @@ buf_pullup(buf_t *buf, size_t bytes, int nulterminate) dest->next = src->next; if (buf->tail == src) buf->tail = dest; - chunk_free(src); + chunk_free_unchecked(src); } else { memcpy(CHUNK_WRITE_PTR(dest), src->data, n); dest->datalen += n; @@ -450,7 +472,7 @@ buf_remove_from_front(buf_t *buf, size_t n) buf->head = victim->next; if (buf->tail == victim) buf->tail = NULL; - chunk_free(victim); + chunk_free_unchecked(victim); } } check(); @@ -484,7 +506,7 @@ buf_clear(buf_t *buf) buf->datalen = 0; for (chunk = buf->head; chunk; chunk = next) { next = chunk->next; - chunk_free(chunk); + chunk_free_unchecked(chunk); } buf->head = buf->tail = NULL; } @@ -523,6 +545,8 @@ buf_slack(const buf_t *buf) void buf_free(buf_t *buf) { + if (!buf) + return; buf_clear(buf); buf->magic = 0xdeadbeef; tor_free(buf); @@ -642,12 +666,12 @@ read_to_chunk_tls(buf_t *buf, chunk_t *chunk, tor_tls_t *tls, * (because of EOF), set *<b>reached_eof</b> to 1 and return 0. Return -1 on * error; else return the number of bytes read. */ -/* XXXX021 indicate "read blocked" somehow? */ +/* XXXX023 indicate "read blocked" somehow? */ int read_to_buf(int s, size_t at_most, buf_t *buf, int *reached_eof, int *socket_error) { - /* XXXX021 It's stupid to overload the return values for these functions: + /* XXXX023 It's stupid to overload the return values for these functions: * "error status" and "number of bytes read" are not mutually exclusive. */ int r = 0; @@ -832,7 +856,7 @@ flush_chunk_tls(tor_tls_t *tls, buf_t *buf, chunk_t *chunk, int flush_buf(int s, buf_t *buf, size_t sz, size_t *buf_flushlen) { - /* XXXX021 It's stupid to overload the return values for these functions: + /* XXXX023 It's stupid to overload the return values for these functions: * "error status" and "number of bytes flushed" are not mutually exclusive. */ int r; @@ -1275,6 +1299,47 @@ fetch_from_buf_http(buf_t *buf, return 1; } +/** + * Wait this many seconds before warning the user about using SOCKS unsafely + * again (requires that WarnUnsafeSocks is turned on). */ +#define SOCKS_WARN_INTERVAL 5 + +/** Warn that the user application has made an unsafe socks request using + * protocol <b>socks_protocol</b> on port <b>port</b>. Don't warn more than + * once per SOCKS_WARN_INTERVAL, unless <b>safe_socks</b> is set. */ +static void +log_unsafe_socks_warning(int socks_protocol, const char *address, + uint16_t port, int safe_socks) +{ + static ratelim_t socks_ratelim = RATELIM_INIT(SOCKS_WARN_INTERVAL); + + or_options_t *options = get_options(); + char *m = NULL; + if (! options->WarnUnsafeSocks) + return; + if (safe_socks || (m = rate_limit_log(&socks_ratelim, approx_time()))) { + log_warn(LD_APP, + "Your application (using socks%d to port %d) is giving " + "Tor only an IP address. Applications that do DNS resolves " + "themselves may leak information. Consider using Socks4A " + "(e.g. via privoxy or socat) instead. For more information, " + "please see https://wiki.torproject.org/TheOnionRouter/" + "TorFAQ#SOCKSAndDNS.%s%s", + socks_protocol, + (int)port, + safe_socks ? " Rejecting." : "", + m ? m : ""); + tor_free(m); + } + control_event_client_status(LOG_WARN, + "DANGEROUS_SOCKS PROTOCOL=SOCKS%d ADDRESS=%s:%d", + socks_protocol, address, (int)port); +} + +/** Do not attempt to parse socks messages longer than this. This value is + * actually significantly higher than the longest possible socks message. */ +#define MAX_SOCKS_MESSAGE_LEN 512 + /** There is a (possibly incomplete) socks handshake on <b>buf</b>, of one * of the forms * - socks4: "socksheader username\\0" @@ -1313,14 +1378,10 @@ fetch_from_buf_socks(buf_t *buf, socks_request_t *req, char *next, *startaddr; struct in_addr in; - /* If the user connects with socks4 or the wrong variant of socks5, - * then log a warning to let him know that it might be unwise. */ - static int have_warned_about_unsafe_socks = 0; - if (buf->datalen < 2) /* version and another byte */ return 0; - buf_pullup(buf, 128, 0); + buf_pullup(buf, MAX_SOCKS_MESSAGE_LEN, 0); tor_assert(buf->head && buf->head->datalen >= 2); socksver = *buf->head->data; @@ -1396,21 +1457,8 @@ fetch_from_buf_socks(buf_t *buf, socks_request_t *req, req->port = ntohs(get_uint16(buf->head->data+4+addrlen)); buf_remove_from_front(buf, 6+addrlen); if (req->command != SOCKS_COMMAND_RESOLVE_PTR && - !addressmap_have_mapping(req->address,0) && - !have_warned_about_unsafe_socks) { - log_warn(LD_APP, - "Your application (using socks5 to port %d) is giving " - "Tor only an IP address. Applications that do DNS resolves " - "themselves may leak information. Consider using Socks4A " - "(e.g. via privoxy or socat) instead. For more information, " - "please see http://wiki.noreply.org/noreply/TheOnionRouter/" - "TorFAQ#SOCKSAndDNS.%s", req->port, - safe_socks ? " Rejecting." : ""); - /*have_warned_about_unsafe_socks = 1;*/ - /*(for now, warn every time)*/ - control_event_client_status(LOG_WARN, - "DANGEROUS_SOCKS PROTOCOL=SOCKS5 ADDRESS=%s:%d", - req->address, req->port); + !addressmap_have_mapping(req->address,0)) { + log_unsafe_socks_warning(5, req->address, req->port, safe_socks); if (safe_socks) return -1; } @@ -1475,8 +1523,8 @@ fetch_from_buf_socks(buf_t *buf, socks_request_t *req, return -1; } - req->port = ntohs(*(uint16_t*)(buf->head->data+2)); - destip = ntohl(*(uint32_t*)(buf->head->data+4)); + req->port = ntohs(get_uint16(buf->head->data+2)); + destip = ntohl(get_uint32(buf->head->data+4)); if ((!req->port && req->command!=SOCKS_COMMAND_RESOLVE) || !destip) { log_warn(LD_APP,"socks4: Port or DestIP is zero. Rejecting."); return -1; @@ -1491,7 +1539,8 @@ fetch_from_buf_socks(buf_t *buf, socks_request_t *req, return -1; } log_debug(LD_APP, - "socks4: successfully read destip (%s)", safe_str(tmpbuf)); + "socks4: successfully read destip (%s)", + safe_str_client(tmpbuf)); socks4_prot = socks4; } @@ -1509,20 +1558,9 @@ fetch_from_buf_socks(buf_t *buf, socks_request_t *req, startaddr = NULL; if (socks4_prot != socks4a && - !addressmap_have_mapping(tmpbuf,0) && - !have_warned_about_unsafe_socks) { - log_warn(LD_APP, - "Your application (using socks4 to port %d) is giving Tor " - "only an IP address. Applications that do DNS resolves " - "themselves may leak information. Consider using Socks4A " - "(e.g. via privoxy or socat) instead. For more information, " - "please see http://wiki.noreply.org/noreply/TheOnionRouter/" - "TorFAQ#SOCKSAndDNS.%s", req->port, - safe_socks ? " Rejecting." : ""); - /*have_warned_about_unsafe_socks = 1;*/ /*(for now, warn every time)*/ - control_event_client_status(LOG_WARN, - "DANGEROUS_SOCKS PROTOCOL=SOCKS4 ADDRESS=%s:%d", - tmpbuf, req->port); + !addressmap_have_mapping(tmpbuf,0)) { + log_unsafe_socks_warning(4, tmpbuf, req->port, safe_socks); + if (safe_socks) return -1; } @@ -1614,6 +1652,132 @@ fetch_from_buf_socks(buf_t *buf, socks_request_t *req, } } +/** Inspect a reply from SOCKS server stored in <b>buf</b> according + * to <b>state</b>, removing the protocol data upon success. Return 0 on + * incomplete response, 1 on success and -1 on error, in which case + * <b>reason</b> is set to a descriptive message (free() when finished + * with it). + * + * As a special case, 2 is returned when user/pass is required + * during SOCKS5 handshake and user/pass is configured. + */ +int +fetch_from_buf_socks_client(buf_t *buf, int state, char **reason) +{ + unsigned char *data; + size_t addrlen; + + if (buf->datalen < 2) + return 0; + + buf_pullup(buf, MAX_SOCKS_MESSAGE_LEN, 0); + tor_assert(buf->head && buf->head->datalen >= 2); + + data = (unsigned char *) buf->head->data; + + switch (state) { + case PROXY_SOCKS4_WANT_CONNECT_OK: + /* Wait for the complete response */ + if (buf->head->datalen < 8) + return 0; + + if (data[1] != 0x5a) { + *reason = tor_strdup(socks4_response_code_to_string(data[1])); + return -1; + } + + /* Success */ + buf_remove_from_front(buf, 8); + return 1; + + case PROXY_SOCKS5_WANT_AUTH_METHOD_NONE: + /* we don't have any credentials */ + if (data[1] != 0x00) { + *reason = tor_strdup("server doesn't support any of our " + "available authentication methods"); + return -1; + } + + log_info(LD_NET, "SOCKS 5 client: continuing without authentication"); + buf_clear(buf); + return 1; + + case PROXY_SOCKS5_WANT_AUTH_METHOD_RFC1929: + /* we have a username and password. return 1 if we can proceed without + * providing authentication, or 2 otherwise. */ + switch (data[1]) { + case 0x00: + log_info(LD_NET, "SOCKS 5 client: we have auth details but server " + "doesn't require authentication."); + buf_clear(buf); + return 1; + case 0x02: + log_info(LD_NET, "SOCKS 5 client: need authentication."); + buf_clear(buf); + return 2; + /* fall through */ + } + + *reason = tor_strdup("server doesn't support any of our available " + "authentication methods"); + return -1; + + case PROXY_SOCKS5_WANT_AUTH_RFC1929_OK: + /* handle server reply to rfc1929 authentication */ + if (data[1] != 0x00) { + *reason = tor_strdup("authentication failed"); + return -1; + } + + log_info(LD_NET, "SOCKS 5 client: authentication successful."); + buf_clear(buf); + return 1; + + case PROXY_SOCKS5_WANT_CONNECT_OK: + /* response is variable length. BND.ADDR, etc, isn't needed + * (don't bother with buf_pullup()), but make sure to eat all + * the data used */ + + /* wait for address type field to arrive */ + if (buf->datalen < 4) + return 0; + + switch (data[3]) { + case 0x01: /* ip4 */ + addrlen = 4; + break; + case 0x04: /* ip6 */ + addrlen = 16; + break; + case 0x03: /* fqdn (can this happen here?) */ + if (buf->datalen < 5) + return 0; + addrlen = 1 + data[4]; + break; + default: + *reason = tor_strdup("invalid response to connect request"); + return -1; + } + + /* wait for address and port */ + if (buf->datalen < 6 + addrlen) + return 0; + + if (data[1] != 0x00) { + *reason = tor_strdup(socks5_response_code_to_string(data[1])); + return -1; + } + + buf_remove_from_front(buf, 6 + addrlen); + return 1; + } + + /* shouldn't get here... */ + tor_assert(0); + + return -1; +} + /** Return 1 iff buf looks more like it has an (obsolete) v0 controller * command on it than any valid v1 controller command. */ int diff --git a/src/or/buffers.h b/src/or/buffers.h new file mode 100644 index 0000000000..e50b9ff6fb --- /dev/null +++ b/src/or/buffers.h @@ -0,0 +1,58 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file buffers.h + * \brief Header file for buffers.c. + **/ + +#ifndef _TOR_BUFFERS_H +#define _TOR_BUFFERS_H + +buf_t *buf_new(void); +buf_t *buf_new_with_capacity(size_t size); +void buf_free(buf_t *buf); +void buf_clear(buf_t *buf); +void buf_shrink(buf_t *buf); +void buf_shrink_freelists(int free_all); +void buf_dump_freelist_sizes(int severity); + +size_t buf_datalen(const buf_t *buf); +size_t buf_allocation(const buf_t *buf); +size_t buf_slack(const buf_t *buf); + +int read_to_buf(int s, size_t at_most, buf_t *buf, int *reached_eof, + int *socket_error); +int read_to_buf_tls(tor_tls_t *tls, size_t at_most, buf_t *buf); + +int flush_buf(int s, buf_t *buf, size_t sz, size_t *buf_flushlen); +int flush_buf_tls(tor_tls_t *tls, buf_t *buf, size_t sz, size_t *buf_flushlen); + +int write_to_buf(const char *string, size_t string_len, buf_t *buf); +int write_to_buf_zlib(buf_t *buf, tor_zlib_state_t *state, + const char *data, size_t data_len, int done); +int move_buf_to_buf(buf_t *buf_out, buf_t *buf_in, size_t *buf_flushlen); +int fetch_from_buf(char *string, size_t string_len, buf_t *buf); +int fetch_var_cell_from_buf(buf_t *buf, var_cell_t **out, int linkproto); +int fetch_from_buf_http(buf_t *buf, + char **headers_out, size_t max_headerlen, + char **body_out, size_t *body_used, size_t max_bodylen, + int force_complete); +int fetch_from_buf_socks(buf_t *buf, socks_request_t *req, + int log_sockstype, int safe_socks); +int fetch_from_buf_socks_client(buf_t *buf, int state, char **reason); +int fetch_from_buf_line(buf_t *buf, char *data_out, size_t *data_len); + +int peek_buf_has_control0_command(buf_t *buf); + +void assert_buf_ok(buf_t *buf); + +#ifdef BUFFERS_PRIVATE +int buf_find_string_offset(const buf_t *buf, const char *s, size_t n); +#endif + +#endif + diff --git a/src/or/circuitbuild.c b/src/or/circuitbuild.c index 208a9cb11b..a9986d309c 100644 --- a/src/or/circuitbuild.c +++ b/src/or/circuitbuild.c @@ -9,9 +9,46 @@ * \brief The actual details of building circuits. **/ +#define CIRCUIT_PRIVATE + #include "or.h" +#include "circuitbuild.h" +#include "circuitlist.h" +#include "circuituse.h" +#include "config.h" +#include "connection.h" +#include "connection_edge.h" +#include "connection_or.h" +#include "control.h" +#include "directory.h" +#include "main.h" +#include "networkstatus.h" +#include "onion.h" +#include "policies.h" +#include "relay.h" +#include "rephist.h" +#include "router.h" +#include "routerlist.h" +#include "routerparse.h" +#include "crypto.h" +#undef log +#include <math.h> + +#ifndef MIN +#define MIN(a,b) ((a)<(b)?(a):(b)) +#endif + +#define CBT_BIN_TO_MS(bin) ((bin)*CBT_BIN_WIDTH + (CBT_BIN_WIDTH/2)) /********* START VARIABLES **********/ +/** Global list of circuit build times */ +// XXXX023: Add this as a member for entry_guard_t instead of global? +// Then we could do per-guard statistics, as guards are likely to +// vary in their own latency. The downside of this is that guards +// can change frequently, so we'd be building a lot more circuits +// most likely. +/* XXXX023 Make this static; add accessor functions. */ +circuit_build_times_t circ_times; /** A global list of all circuits at this hop. */ extern circuit_t *global_circuitlist; @@ -47,6 +84,10 @@ static smartlist_t *entry_guards = NULL; * and those changes need to be flushed to disk. */ static int entry_guards_dirty = 0; +/** If set, we're running the unit tests: we should avoid clobbering + * our state file or accessing get_options() or get_or_state() */ +static int unit_tests = 0; + /********* END VARIABLES ************/ static int circuit_deliver_create_cell(circuit_t *circ, @@ -59,6 +100,1358 @@ static int onion_append_hop(crypt_path_t **head_ptr, extend_info_t *choice); static void entry_guards_changed(void); +/** + * This function decides if CBT learning should be disabled. It returns + * true if one or more of the following four conditions are met: + * + * 1. If the cbtdisabled consensus parameter is set. + * 2. If the torrc option LearnCircuitBuildTimeout is false. + * 3. If we are a directory authority + * 4. If we fail to write circuit build time history to our state file. + */ +static int +circuit_build_times_disabled(void) +{ + if (unit_tests) { + return 0; + } else { + int consensus_disabled = networkstatus_get_param(NULL, "cbtdisabled", + 0, 0, 1); + int config_disabled = !get_options()->LearnCircuitBuildTimeout; + int dirauth_disabled = get_options()->AuthoritativeDir; + int state_disabled = did_last_state_file_write_fail() ? 1 : 0; + + if (consensus_disabled || config_disabled || dirauth_disabled || + state_disabled) { + log_info(LD_CIRC, + "CircuitBuildTime learning is disabled. " + "Consensus=%d, Config=%d, AuthDir=%d, StateFile=%d", + consensus_disabled, config_disabled, dirauth_disabled, + state_disabled); + return 1; + } else { + return 0; + } + } +} + +/** + * Retrieve and bounds-check the cbtmaxtimeouts consensus paramter. + * + * Effect: When this many timeouts happen in the last 'cbtrecentcount' + * circuit attempts, the client should discard all of its history and + * begin learning a fresh timeout value. + */ +static int32_t +circuit_build_times_max_timeouts(void) +{ + return networkstatus_get_param(NULL, "cbtmaxtimeouts", + CBT_DEFAULT_MAX_RECENT_TIMEOUT_COUNT, + CBT_MIN_MAX_RECENT_TIMEOUT_COUNT, + CBT_MAX_MAX_RECENT_TIMEOUT_COUNT); +} + +/** + * Retrieve and bounds-check the cbtnummodes consensus paramter. + * + * Effect: This value governs how many modes to use in the weighted + * average calculation of Pareto parameter Xm. A value of 3 introduces + * some bias (2-5% of CDF) under ideal conditions, but allows for better + * performance in the event that a client chooses guard nodes of radically + * different performance characteristics. + */ +static int32_t +circuit_build_times_default_num_xm_modes(void) +{ + int32_t num = networkstatus_get_param(NULL, "cbtnummodes", + CBT_DEFAULT_NUM_XM_MODES, + CBT_MIN_NUM_XM_MODES, + CBT_MAX_NUM_XM_MODES); + return num; +} + +/** + * Retrieve and bounds-check the cbtmincircs consensus paramter. + * + * Effect: This is the minimum number of circuits to build before + * computing a timeout. + */ +static int32_t +circuit_build_times_min_circs_to_observe(void) +{ + int32_t num = networkstatus_get_param(NULL, "cbtmincircs", + CBT_DEFAULT_MIN_CIRCUITS_TO_OBSERVE, + CBT_MIN_MIN_CIRCUITS_TO_OBSERVE, + CBT_MAX_MIN_CIRCUITS_TO_OBSERVE); + return num; +} + +/** Return true iff <b>cbt</b> has recorded enough build times that we + * want to start acting on the timeout it implies. */ +int +circuit_build_times_enough_to_compute(circuit_build_times_t *cbt) +{ + return cbt->total_build_times >= circuit_build_times_min_circs_to_observe(); +} + +/** + * Retrieve and bounds-check the cbtquantile consensus paramter. + * + * Effect: This is the position on the quantile curve to use to set the + * timeout value. It is a percent (10-99). + */ +double +circuit_build_times_quantile_cutoff(void) +{ + int32_t num = networkstatus_get_param(NULL, "cbtquantile", + CBT_DEFAULT_QUANTILE_CUTOFF, + CBT_MIN_QUANTILE_CUTOFF, + CBT_MAX_QUANTILE_CUTOFF); + return num/100.0; +} + +int +circuit_build_times_get_bw_scale(networkstatus_t *ns) +{ + return networkstatus_get_param(ns, "bwweightscale", + BW_WEIGHT_SCALE, + BW_MIN_WEIGHT_SCALE, + BW_MAX_WEIGHT_SCALE); +} + +/** + * Retrieve and bounds-check the cbtclosequantile consensus paramter. + * + * Effect: This is the position on the quantile curve to use to set the + * timeout value to use to actually close circuits. It is a percent + * (0-99). + */ +static double +circuit_build_times_close_quantile(void) +{ + int32_t param; + /* Cast is safe - circuit_build_times_quantile_cutoff() is capped */ + int32_t min = (int)tor_lround(100*circuit_build_times_quantile_cutoff()); + param = networkstatus_get_param(NULL, "cbtclosequantile", + CBT_DEFAULT_CLOSE_QUANTILE, + CBT_MIN_CLOSE_QUANTILE, + CBT_MAX_CLOSE_QUANTILE); + if (param < min) { + log_warn(LD_DIR, "Consensus parameter cbtclosequantile is " + "too small, raising to %d", min); + param = min; + } + return param / 100.0; +} + +/** + * Retrieve and bounds-check the cbttestfreq consensus paramter. + * + * Effect: Describes how often in seconds to build a test circuit to + * gather timeout values. Only applies if less than 'cbtmincircs' + * have been recorded. + */ +static int32_t +circuit_build_times_test_frequency(void) +{ + int32_t num = networkstatus_get_param(NULL, "cbttestfreq", + CBT_DEFAULT_TEST_FREQUENCY, + CBT_MIN_TEST_FREQUENCY, + CBT_MAX_TEST_FREQUENCY); + return num; +} + +/** + * Retrieve and bounds-check the cbtmintimeout consensus paramter. + * + * Effect: This is the minimum allowed timeout value in milliseconds. + * The minimum is to prevent rounding to 0 (we only check once + * per second). + */ +static int32_t +circuit_build_times_min_timeout(void) +{ + int32_t num = networkstatus_get_param(NULL, "cbtmintimeout", + CBT_DEFAULT_TIMEOUT_MIN_VALUE, + CBT_MIN_TIMEOUT_MIN_VALUE, + CBT_MAX_TIMEOUT_MIN_VALUE); + return num; +} + +/** + * Retrieve and bounds-check the cbtinitialtimeout consensus paramter. + * + * Effect: This is the timeout value to use before computing a timeout, + * in milliseconds. + */ +int32_t +circuit_build_times_initial_timeout(void) +{ + int32_t min = circuit_build_times_min_timeout(); + int32_t param = networkstatus_get_param(NULL, "cbtinitialtimeout", + CBT_DEFAULT_TIMEOUT_INITIAL_VALUE, + CBT_MIN_TIMEOUT_INITIAL_VALUE, + CBT_MAX_TIMEOUT_INITIAL_VALUE); + if (param < min) { + log_warn(LD_DIR, "Consensus parameter cbtinitialtimeout is too small, " + "raising to %d", min); + param = min; + } + return param; +} + +/** + * Retrieve and bounds-check the cbtrecentcount consensus paramter. + * + * Effect: This is the number of circuit build times to keep track of + * for deciding if we hit cbtmaxtimeouts and need to reset our state + * and learn a new timeout. + */ +static int32_t +circuit_build_times_recent_circuit_count(networkstatus_t *ns) +{ + return networkstatus_get_param(ns, "cbtrecentcount", + CBT_DEFAULT_RECENT_CIRCUITS, + CBT_MIN_RECENT_CIRCUITS, + CBT_MAX_RECENT_CIRCUITS); +} + +/** + * This function is called when we get a consensus update. + * + * It checks to see if we have changed any consensus parameters + * that require reallocation or discard of previous stats. + */ +void +circuit_build_times_new_consensus_params(circuit_build_times_t *cbt, + networkstatus_t *ns) +{ + int32_t num = circuit_build_times_recent_circuit_count(ns); + + if (num > 0 && num != cbt->liveness.num_recent_circs) { + int8_t *recent_circs; + log_notice(LD_CIRC, "The Tor Directory Consensus has changed how many " + "circuits we must track to detect network failures from %d " + "to %d.", cbt->liveness.num_recent_circs, num); + + tor_assert(cbt->liveness.timeouts_after_firsthop); + + /* + * Technically this is a circular array that we are reallocating + * and memcopying. However, since it only consists of either 1s + * or 0s, and is only used in a statistical test to determine when + * we should discard our history after a sufficient number of 1's + * have been reached, it is fine if order is not preserved or + * elements are lost. + * + * cbtrecentcount should only be changing in cases of severe network + * distress anyway, so memory correctness here is paramount over + * doing acrobatics to preserve the array. + */ + recent_circs = tor_malloc_zero(sizeof(int8_t)*num); + memcpy(recent_circs, cbt->liveness.timeouts_after_firsthop, + sizeof(int8_t)*MIN(num, cbt->liveness.num_recent_circs)); + + // Adjust the index if it needs it. + if (num < cbt->liveness.num_recent_circs) { + cbt->liveness.after_firsthop_idx = MIN(num-1, + cbt->liveness.after_firsthop_idx); + } + + tor_free(cbt->liveness.timeouts_after_firsthop); + cbt->liveness.timeouts_after_firsthop = recent_circs; + cbt->liveness.num_recent_circs = num; + } +} + +/** Make a note that we're running unit tests (rather than running Tor + * itself), so we avoid clobbering our state file. */ +void +circuitbuild_running_unit_tests(void) +{ + unit_tests = 1; +} + +/** + * Return the initial default or configured timeout in milliseconds + */ +static double +circuit_build_times_get_initial_timeout(void) +{ + double timeout; + if (!unit_tests && get_options()->CircuitBuildTimeout) { + timeout = get_options()->CircuitBuildTimeout*1000; + if (timeout < circuit_build_times_min_timeout()) { + log_warn(LD_CIRC, "Config CircuitBuildTimeout too low. Setting to %ds", + circuit_build_times_min_timeout()/1000); + timeout = circuit_build_times_min_timeout(); + } + } else { + timeout = circuit_build_times_initial_timeout(); + } + return timeout; +} + +/** + * Reset the build time state. + * + * Leave estimated parameters, timeout and network liveness intact + * for future use. + */ +void +circuit_build_times_reset(circuit_build_times_t *cbt) +{ + memset(cbt->circuit_build_times, 0, sizeof(cbt->circuit_build_times)); + cbt->total_build_times = 0; + cbt->build_times_idx = 0; + cbt->have_computed_timeout = 0; +} + +/** + * Initialize the buildtimes structure for first use. + * + * Sets the initial timeout values based on either the config setting, + * the consensus param, or the default (CBT_DEFAULT_TIMEOUT_INITIAL_VALUE). + */ +void +circuit_build_times_init(circuit_build_times_t *cbt) +{ + memset(cbt, 0, sizeof(*cbt)); + cbt->liveness.num_recent_circs = + circuit_build_times_recent_circuit_count(NULL); + cbt->liveness.timeouts_after_firsthop = tor_malloc_zero(sizeof(int8_t)* + cbt->liveness.num_recent_circs); + cbt->close_ms = cbt->timeout_ms = circuit_build_times_get_initial_timeout(); + control_event_buildtimeout_set(cbt, BUILDTIMEOUT_SET_EVENT_RESET); +} + +#if 0 +/** + * Rewind our build time history by n positions. + */ +static void +circuit_build_times_rewind_history(circuit_build_times_t *cbt, int n) +{ + int i = 0; + + cbt->build_times_idx -= n; + cbt->build_times_idx %= CBT_NCIRCUITS_TO_OBSERVE; + + for (i = 0; i < n; i++) { + cbt->circuit_build_times[(i+cbt->build_times_idx) + %CBT_NCIRCUITS_TO_OBSERVE]=0; + } + + if (cbt->total_build_times > n) { + cbt->total_build_times -= n; + } else { + cbt->total_build_times = 0; + } + + log_info(LD_CIRC, + "Rewound history by %d places. Current index: %d. " + "Total: %d", n, cbt->build_times_idx, cbt->total_build_times); +} +#endif + +/** + * Add a new build time value <b>time</b> to the set of build times. Time + * units are milliseconds. + * + * circuit_build_times <b>cbt</b> is a circular array, so loop around when + * array is full. + */ +int +circuit_build_times_add_time(circuit_build_times_t *cbt, build_time_t time) +{ + if (time <= 0 || time > CBT_BUILD_TIME_MAX) { + log_warn(LD_BUG, "Circuit build time is too large (%u)." + "This is probably a bug.", time); + tor_fragile_assert(); + return -1; + } + + log_debug(LD_CIRC, "Adding circuit build time %u", time); + + cbt->circuit_build_times[cbt->build_times_idx] = time; + cbt->build_times_idx = (cbt->build_times_idx + 1) % CBT_NCIRCUITS_TO_OBSERVE; + if (cbt->total_build_times < CBT_NCIRCUITS_TO_OBSERVE) + cbt->total_build_times++; + + if ((cbt->total_build_times % CBT_SAVE_STATE_EVERY) == 0) { + /* Save state every n circuit builds */ + if (!unit_tests && !get_options()->AvoidDiskWrites) + or_state_mark_dirty(get_or_state(), 0); + } + + return 0; +} + +/** + * Return maximum circuit build time + */ +static build_time_t +circuit_build_times_max(circuit_build_times_t *cbt) +{ + int i = 0; + build_time_t max_build_time = 0; + for (i = 0; i < CBT_NCIRCUITS_TO_OBSERVE; i++) { + if (cbt->circuit_build_times[i] > max_build_time + && cbt->circuit_build_times[i] != CBT_BUILD_ABANDONED) + max_build_time = cbt->circuit_build_times[i]; + } + return max_build_time; +} + +#if 0 +/** Return minimum circuit build time */ +build_time_t +circuit_build_times_min(circuit_build_times_t *cbt) +{ + int i = 0; + build_time_t min_build_time = CBT_BUILD_TIME_MAX; + for (i = 0; i < CBT_NCIRCUITS_TO_OBSERVE; i++) { + if (cbt->circuit_build_times[i] && /* 0 <-> uninitialized */ + cbt->circuit_build_times[i] < min_build_time) + min_build_time = cbt->circuit_build_times[i]; + } + if (min_build_time == CBT_BUILD_TIME_MAX) { + log_warn(LD_CIRC, "No build times less than CBT_BUILD_TIME_MAX!"); + } + return min_build_time; +} +#endif + +/** + * Calculate and return a histogram for the set of build times. + * + * Returns an allocated array of histrogram bins representing + * the frequency of index*CBT_BIN_WIDTH millisecond + * build times. Also outputs the number of bins in nbins. + * + * The return value must be freed by the caller. + */ +static uint32_t * +circuit_build_times_create_histogram(circuit_build_times_t *cbt, + build_time_t *nbins) +{ + uint32_t *histogram; + build_time_t max_build_time = circuit_build_times_max(cbt); + int i, c; + + *nbins = 1 + (max_build_time / CBT_BIN_WIDTH); + histogram = tor_malloc_zero(*nbins * sizeof(build_time_t)); + + // calculate histogram + for (i = 0; i < CBT_NCIRCUITS_TO_OBSERVE; i++) { + if (cbt->circuit_build_times[i] == 0 + || cbt->circuit_build_times[i] == CBT_BUILD_ABANDONED) + continue; /* 0 <-> uninitialized */ + + c = (cbt->circuit_build_times[i] / CBT_BIN_WIDTH); + histogram[c]++; + } + + return histogram; +} + +/** + * Return the Pareto start-of-curve parameter Xm. + * + * Because we are not a true Pareto curve, we compute this as the + * weighted average of the N most frequent build time bins. N is either + * 1 if we don't have enough circuit build time data collected, or + * determined by the consensus parameter cbtnummodes (default 3). + */ +static build_time_t +circuit_build_times_get_xm(circuit_build_times_t *cbt) +{ + build_time_t i, nbins; + build_time_t *nth_max_bin; + int32_t bin_counts=0; + build_time_t ret = 0; + uint32_t *histogram = circuit_build_times_create_histogram(cbt, &nbins); + int n=0; + int num_modes = circuit_build_times_default_num_xm_modes(); + + tor_assert(nbins > 0); + tor_assert(num_modes > 0); + + // Only use one mode if < 1000 buildtimes. Not enough data + // for multiple. + if (cbt->total_build_times < CBT_NCIRCUITS_TO_OBSERVE) + num_modes = 1; + + nth_max_bin = (build_time_t*)tor_malloc_zero(num_modes*sizeof(build_time_t)); + + /* Determine the N most common build times */ + for (i = 0; i < nbins; i++) { + if (histogram[i] >= histogram[nth_max_bin[0]]) { + nth_max_bin[0] = i; + } + + for (n = 1; n < num_modes; n++) { + if (histogram[i] >= histogram[nth_max_bin[n]] && + (!histogram[nth_max_bin[n-1]] + || histogram[i] < histogram[nth_max_bin[n-1]])) { + nth_max_bin[n] = i; + } + } + } + + for (n = 0; n < num_modes; n++) { + bin_counts += histogram[nth_max_bin[n]]; + ret += CBT_BIN_TO_MS(nth_max_bin[n])*histogram[nth_max_bin[n]]; + log_info(LD_CIRC, "Xm mode #%d: %u %u", n, CBT_BIN_TO_MS(nth_max_bin[n]), + histogram[nth_max_bin[n]]); + } + + /* The following assert is safe, because we don't get called when we + * haven't observed at least CBT_MIN_MIN_CIRCUITS_TO_OBSERVE circuits. */ + tor_assert(bin_counts > 0); + + ret /= bin_counts; + tor_free(histogram); + tor_free(nth_max_bin); + + return ret; +} + +/** + * Output a histogram of current circuit build times to + * the or_state_t state structure. + */ +void +circuit_build_times_update_state(circuit_build_times_t *cbt, + or_state_t *state) +{ + uint32_t *histogram; + build_time_t i = 0; + build_time_t nbins = 0; + config_line_t **next, *line; + + histogram = circuit_build_times_create_histogram(cbt, &nbins); + // write to state + config_free_lines(state->BuildtimeHistogram); + next = &state->BuildtimeHistogram; + *next = NULL; + + state->TotalBuildTimes = cbt->total_build_times; + state->CircuitBuildAbandonedCount = 0; + + for (i = 0; i < CBT_NCIRCUITS_TO_OBSERVE; i++) { + if (cbt->circuit_build_times[i] == CBT_BUILD_ABANDONED) + state->CircuitBuildAbandonedCount++; + } + + for (i = 0; i < nbins; i++) { + // compress the histogram by skipping the blanks + if (histogram[i] == 0) continue; + *next = line = tor_malloc_zero(sizeof(config_line_t)); + line->key = tor_strdup("CircuitBuildTimeBin"); + line->value = tor_malloc(25); + tor_snprintf(line->value, 25, "%d %d", + CBT_BIN_TO_MS(i), histogram[i]); + next = &(line->next); + } + + if (!unit_tests) { + if (!get_options()->AvoidDiskWrites) + or_state_mark_dirty(get_or_state(), 0); + } + + tor_free(histogram); +} + +/** + * Shuffle the build times array. + * + * Adapted from http://en.wikipedia.org/wiki/Fisher-Yates_shuffle + */ +static void +circuit_build_times_shuffle_and_store_array(circuit_build_times_t *cbt, + build_time_t *raw_times, + uint32_t num_times) +{ + uint32_t n = num_times; + if (num_times > CBT_NCIRCUITS_TO_OBSERVE) { + log_notice(LD_CIRC, "The number of circuit times that this Tor version " + "uses to calculate build times is less than the number stored " + "in your state file. Decreasing the circuit time history from " + "%d to %d.", num_times, CBT_NCIRCUITS_TO_OBSERVE); + } + + /* This code can only be run on a compact array */ + while (n-- > 1) { + int k = crypto_rand_int(n + 1); /* 0 <= k <= n. */ + build_time_t tmp = raw_times[k]; + raw_times[k] = raw_times[n]; + raw_times[n] = tmp; + } + + /* Since the times are now shuffled, take a random CBT_NCIRCUITS_TO_OBSERVE + * subset (ie the first CBT_NCIRCUITS_TO_OBSERVE values) */ + for (n = 0; n < MIN(num_times, CBT_NCIRCUITS_TO_OBSERVE); n++) { + circuit_build_times_add_time(cbt, raw_times[n]); + } +} + +/** + * Filter old synthetic timeouts that were created before the + * new right-censored Pareto calculation was deployed. + * + * Once all clients before 0.2.1.13-alpha are gone, this code + * will be unused. + */ +static int +circuit_build_times_filter_timeouts(circuit_build_times_t *cbt) +{ + int num_filtered=0, i=0; + double timeout_rate = 0; + build_time_t max_timeout = 0; + + timeout_rate = circuit_build_times_timeout_rate(cbt); + max_timeout = (build_time_t)cbt->close_ms; + + for (i = 0; i < CBT_NCIRCUITS_TO_OBSERVE; i++) { + if (cbt->circuit_build_times[i] > max_timeout) { + build_time_t replaced = cbt->circuit_build_times[i]; + num_filtered++; + cbt->circuit_build_times[i] = CBT_BUILD_ABANDONED; + + log_debug(LD_CIRC, "Replaced timeout %d with %d", replaced, + cbt->circuit_build_times[i]); + } + } + + log_info(LD_CIRC, + "We had %d timeouts out of %d build times, " + "and filtered %d above the max of %u", + (int)(cbt->total_build_times*timeout_rate), + cbt->total_build_times, num_filtered, max_timeout); + + return num_filtered; +} + +/** + * Load histogram from <b>state</b>, shuffling the resulting array + * after we do so. Use this result to estimate parameters and + * calculate the timeout. + * + * Return -1 on error. + */ +int +circuit_build_times_parse_state(circuit_build_times_t *cbt, + or_state_t *state) +{ + int tot_values = 0; + uint32_t loaded_cnt = 0, N = 0; + config_line_t *line; + unsigned int i; + build_time_t *loaded_times; + int err = 0; + circuit_build_times_init(cbt); + + if (circuit_build_times_disabled()) { + return 0; + } + + /* build_time_t 0 means uninitialized */ + loaded_times = tor_malloc_zero(sizeof(build_time_t)*state->TotalBuildTimes); + + for (line = state->BuildtimeHistogram; line; line = line->next) { + smartlist_t *args = smartlist_create(); + smartlist_split_string(args, line->value, " ", + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); + if (smartlist_len(args) < 2) { + log_warn(LD_GENERAL, "Unable to parse circuit build times: " + "Too few arguments to CircuitBuildTime"); + err = 1; + SMARTLIST_FOREACH(args, char*, cp, tor_free(cp)); + smartlist_free(args); + break; + } else { + const char *ms_str = smartlist_get(args,0); + const char *count_str = smartlist_get(args,1); + uint32_t count, k; + build_time_t ms; + int ok; + ms = (build_time_t)tor_parse_ulong(ms_str, 0, 0, + CBT_BUILD_TIME_MAX, &ok, NULL); + if (!ok) { + log_warn(LD_GENERAL, "Unable to parse circuit build times: " + "Unparsable bin number"); + err = 1; + SMARTLIST_FOREACH(args, char*, cp, tor_free(cp)); + smartlist_free(args); + break; + } + count = (uint32_t)tor_parse_ulong(count_str, 0, 0, + UINT32_MAX, &ok, NULL); + if (!ok) { + log_warn(LD_GENERAL, "Unable to parse circuit build times: " + "Unparsable bin count"); + err = 1; + SMARTLIST_FOREACH(args, char*, cp, tor_free(cp)); + smartlist_free(args); + break; + } + + if (loaded_cnt+count+state->CircuitBuildAbandonedCount + > state->TotalBuildTimes) { + log_warn(LD_CIRC, + "Too many build times in state file. " + "Stopping short before %d", + loaded_cnt+count); + SMARTLIST_FOREACH(args, char*, cp, tor_free(cp)); + smartlist_free(args); + break; + } + + for (k = 0; k < count; k++) { + loaded_times[loaded_cnt++] = ms; + } + N++; + SMARTLIST_FOREACH(args, char*, cp, tor_free(cp)); + smartlist_free(args); + } + } + + log_info(LD_CIRC, + "Adding %d timeouts.", state->CircuitBuildAbandonedCount); + for (i=0; i < state->CircuitBuildAbandonedCount; i++) { + loaded_times[loaded_cnt++] = CBT_BUILD_ABANDONED; + } + + if (loaded_cnt != state->TotalBuildTimes) { + log_warn(LD_CIRC, + "Corrupt state file? Build times count mismatch. " + "Read %d times, but file says %d", loaded_cnt, + state->TotalBuildTimes); + err = 1; + circuit_build_times_reset(cbt); + goto done; + } + + circuit_build_times_shuffle_and_store_array(cbt, loaded_times, loaded_cnt); + + /* Verify that we didn't overwrite any indexes */ + for (i=0; i < CBT_NCIRCUITS_TO_OBSERVE; i++) { + if (!cbt->circuit_build_times[i]) + break; + tot_values++; + } + log_info(LD_CIRC, + "Loaded %d/%d values from %d lines in circuit time histogram", + tot_values, cbt->total_build_times, N); + + if (cbt->total_build_times != tot_values + || cbt->total_build_times > CBT_NCIRCUITS_TO_OBSERVE) { + log_warn(LD_CIRC, + "Corrupt state file? Shuffled build times mismatch. " + "Read %d times, but file says %d", tot_values, + state->TotalBuildTimes); + err = 1; + circuit_build_times_reset(cbt); + goto done; + } + + circuit_build_times_set_timeout(cbt); + + if (!state->CircuitBuildAbandonedCount && cbt->total_build_times) { + circuit_build_times_filter_timeouts(cbt); + } + + done: + tor_free(loaded_times); + return err ? -1 : 0; +} + +/** + * Estimates the Xm and Alpha parameters using + * http://en.wikipedia.org/wiki/Pareto_distribution#Parameter_estimation + * + * The notable difference is that we use mode instead of min to estimate Xm. + * This is because our distribution is frechet-like. We claim this is + * an acceptable approximation because we are only concerned with the + * accuracy of the CDF of the tail. + */ +int +circuit_build_times_update_alpha(circuit_build_times_t *cbt) +{ + build_time_t *x=cbt->circuit_build_times; + double a = 0; + int n=0,i=0,abandoned_count=0; + build_time_t max_time=0; + + /* http://en.wikipedia.org/wiki/Pareto_distribution#Parameter_estimation */ + /* We sort of cheat here and make our samples slightly more pareto-like + * and less frechet-like. */ + cbt->Xm = circuit_build_times_get_xm(cbt); + + tor_assert(cbt->Xm > 0); + + for (i=0; i< CBT_NCIRCUITS_TO_OBSERVE; i++) { + if (!x[i]) { + continue; + } + + if (x[i] < cbt->Xm) { + a += tor_mathlog(cbt->Xm); + } else if (x[i] == CBT_BUILD_ABANDONED) { + abandoned_count++; + } else { + a += tor_mathlog(x[i]); + if (x[i] > max_time) + max_time = x[i]; + } + n++; + } + + /* + * We are erring and asserting here because this can only happen + * in codepaths other than startup. The startup state parsing code + * performs this same check, and resets state if it hits it. If we + * hit it at runtime, something serious has gone wrong. + */ + if (n!=cbt->total_build_times) { + log_err(LD_CIRC, "Discrepancy in build times count: %d vs %d", n, + cbt->total_build_times); + } + tor_assert(n==cbt->total_build_times); + + if (max_time <= 0) { + /* This can happen if Xm is actually the *maximum* value in the set. + * It can also happen if we've abandoned every single circuit somehow. + * In either case, tell the caller not to compute a new build timeout. */ + log_warn(LD_BUG, + "Could not determine largest build time (%d). " + "Xm is %dms and we've abandoned %d out of %d circuits.", max_time, + cbt->Xm, abandoned_count, n); + return 0; + } + + a += abandoned_count*tor_mathlog(max_time); + + a -= n*tor_mathlog(cbt->Xm); + // Estimator comes from Eq #4 in: + // "Bayesian estimation based on trimmed samples from Pareto populations" + // by Arturo J. Fernández. We are right-censored only. + a = (n-abandoned_count)/a; + + cbt->alpha = a; + + return 1; +} + +/** + * This is the Pareto Quantile Function. It calculates the point x + * in the distribution such that F(x) = quantile (ie quantile*100% + * of the mass of the density function is below x on the curve). + * + * We use it to calculate the timeout and also to generate synthetic + * values of time for circuits that timeout before completion. + * + * See http://en.wikipedia.org/wiki/Quantile_function, + * http://en.wikipedia.org/wiki/Inverse_transform_sampling and + * http://en.wikipedia.org/wiki/Pareto_distribution#Generating_a_ + * random_sample_from_Pareto_distribution + * That's right. I'll cite wikipedia all day long. + * + * Return value is in milliseconds. + */ +double +circuit_build_times_calculate_timeout(circuit_build_times_t *cbt, + double quantile) +{ + double ret; + tor_assert(quantile >= 0); + tor_assert(1.0-quantile > 0); + tor_assert(cbt->Xm > 0); + + ret = cbt->Xm/pow(1.0-quantile,1.0/cbt->alpha); + if (ret > INT32_MAX) { + ret = INT32_MAX; + } + tor_assert(ret > 0); + return ret; +} + +/** Pareto CDF */ +double +circuit_build_times_cdf(circuit_build_times_t *cbt, double x) +{ + double ret; + tor_assert(cbt->Xm > 0); + ret = 1.0-pow(cbt->Xm/x,cbt->alpha); + tor_assert(0 <= ret && ret <= 1.0); + return ret; +} + +/** + * Generate a synthetic time using our distribution parameters. + * + * The return value will be within the [q_lo, q_hi) quantile points + * on the CDF. + */ +build_time_t +circuit_build_times_generate_sample(circuit_build_times_t *cbt, + double q_lo, double q_hi) +{ + double randval = crypto_rand_double(); + build_time_t ret; + double u; + + /* Generate between [q_lo, q_hi) */ + /*XXXX This is what nextafter is supposed to be for; we should use it on the + * platforms that support it. */ + q_hi -= 1.0/(INT32_MAX); + + tor_assert(q_lo >= 0); + tor_assert(q_hi < 1); + tor_assert(q_lo < q_hi); + + u = q_lo + (q_hi-q_lo)*randval; + + tor_assert(0 <= u && u < 1.0); + /* circuit_build_times_calculate_timeout returns <= INT32_MAX */ + ret = (build_time_t) + tor_lround(circuit_build_times_calculate_timeout(cbt, u)); + tor_assert(ret > 0); + return ret; +} + +/** + * Estimate an initial alpha parameter by solving the quantile + * function with a quantile point and a specific timeout value. + */ +void +circuit_build_times_initial_alpha(circuit_build_times_t *cbt, + double quantile, double timeout_ms) +{ + // Q(u) = Xm/((1-u)^(1/a)) + // Q(0.8) = Xm/((1-0.8))^(1/a)) = CircBuildTimeout + // CircBuildTimeout = Xm/((1-0.8))^(1/a)) + // CircBuildTimeout = Xm*((1-0.8))^(-1/a)) + // ln(CircBuildTimeout) = ln(Xm)+ln(((1-0.8)))*(-1/a) + // -ln(1-0.8)/(ln(CircBuildTimeout)-ln(Xm))=a + tor_assert(quantile >= 0); + tor_assert(cbt->Xm > 0); + cbt->alpha = tor_mathlog(1.0-quantile)/ + (tor_mathlog(cbt->Xm)-tor_mathlog(timeout_ms)); + tor_assert(cbt->alpha > 0); +} + +/** + * Returns true if we need circuits to be built + */ +int +circuit_build_times_needs_circuits(circuit_build_times_t *cbt) +{ + /* Return true if < MIN_CIRCUITS_TO_OBSERVE */ + return !circuit_build_times_enough_to_compute(cbt); +} + +/** + * Returns true if we should build a timeout test circuit + * right now. + */ +int +circuit_build_times_needs_circuits_now(circuit_build_times_t *cbt) +{ + return circuit_build_times_needs_circuits(cbt) && + approx_time()-cbt->last_circ_at > circuit_build_times_test_frequency(); +} + +/** + * Called to indicate that the network showed some signs of liveness, + * i.e. we received a cell. + * + * This is used by circuit_build_times_network_check_live() to decide + * if we should record the circuit build timeout or not. + * + * This function is called every time we receive a cell. Avoid + * syscalls, events, and other high-intensity work. + */ +void +circuit_build_times_network_is_live(circuit_build_times_t *cbt) +{ + time_t now = approx_time(); + if (cbt->liveness.nonlive_timeouts > 0) { + log_notice(LD_CIRC, + "Tor now sees network activity. Restoring circuit build " + "timeout recording. Network was down for %d seconds " + "during %d circuit attempts.", + (int)(now - cbt->liveness.network_last_live), + cbt->liveness.nonlive_timeouts); + } + cbt->liveness.network_last_live = now; + cbt->liveness.nonlive_timeouts = 0; +} + +/** + * Called to indicate that we completed a circuit. Because this circuit + * succeeded, it doesn't count as a timeout-after-the-first-hop. + * + * This is used by circuit_build_times_network_check_changed() to determine + * if we had too many recent timeouts and need to reset our learned timeout + * to something higher. + */ +void +circuit_build_times_network_circ_success(circuit_build_times_t *cbt) +{ + cbt->liveness.timeouts_after_firsthop[cbt->liveness.after_firsthop_idx] = 0; + cbt->liveness.after_firsthop_idx++; + cbt->liveness.after_firsthop_idx %= cbt->liveness.num_recent_circs; +} + +/** + * A circuit just timed out. If it failed after the first hop, record it + * in our history for later deciding if the network speed has changed. + * + * This is used by circuit_build_times_network_check_changed() to determine + * if we had too many recent timeouts and need to reset our learned timeout + * to something higher. + */ +static void +circuit_build_times_network_timeout(circuit_build_times_t *cbt, + int did_onehop) +{ + if (did_onehop) { + cbt->liveness.timeouts_after_firsthop[cbt->liveness.after_firsthop_idx]=1; + cbt->liveness.after_firsthop_idx++; + cbt->liveness.after_firsthop_idx %= cbt->liveness.num_recent_circs; + } +} + +/** + * A circuit was just forcibly closed. If there has been no recent network + * activity at all, but this circuit was launched back when we thought the + * network was live, increment the number of "nonlive" circuit timeouts. + * + * This is used by circuit_build_times_network_check_live() to decide + * if we should record the circuit build timeout or not. + */ +static void +circuit_build_times_network_close(circuit_build_times_t *cbt, + int did_onehop, time_t start_time) +{ + time_t now = time(NULL); + /* + * Check if this is a timeout that was for a circuit that spent its + * entire existence during a time where we have had no network activity. + */ + if (cbt->liveness.network_last_live < start_time) { + if (did_onehop) { + char last_live_buf[ISO_TIME_LEN+1]; + char start_time_buf[ISO_TIME_LEN+1]; + char now_buf[ISO_TIME_LEN+1]; + format_local_iso_time(last_live_buf, cbt->liveness.network_last_live); + format_local_iso_time(start_time_buf, start_time); + format_local_iso_time(now_buf, now); + log_warn(LD_BUG, + "Circuit somehow completed a hop while the network was " + "not live. Network was last live at %s, but circuit launched " + "at %s. It's now %s.", last_live_buf, start_time_buf, + now_buf); + } + cbt->liveness.nonlive_timeouts++; + if (cbt->liveness.nonlive_timeouts == 1) { + log_notice(LD_CIRC, + "Tor has not observed any network activity for the past %d " + "seconds. Disabling circuit build timeout recording.", + (int)(now - cbt->liveness.network_last_live)); + } else { + log_info(LD_CIRC, + "Got non-live timeout. Current count is: %d", + cbt->liveness.nonlive_timeouts); + } + } +} + +/** + * When the network is not live, we do not record circuit build times. + * + * The network is considered not live if there has been at least one + * circuit build that began and ended (had its close_ms measurement + * period expire) since we last received a cell. + * + * Also has the side effect of rewinding the circuit time history + * in the case of recent liveness changes. + */ +int +circuit_build_times_network_check_live(circuit_build_times_t *cbt) +{ + if (cbt->liveness.nonlive_timeouts > 0) { + return 0; + } + + return 1; +} + +/** + * Returns true if we have seen more than MAX_RECENT_TIMEOUT_COUNT of + * the past RECENT_CIRCUITS time out after the first hop. Used to detect + * if the network connection has changed significantly, and if so, + * resets our circuit build timeout to the default. + * + * Also resets the entire timeout history in this case and causes us + * to restart the process of building test circuits and estimating a + * new timeout. + */ +int +circuit_build_times_network_check_changed(circuit_build_times_t *cbt) +{ + int total_build_times = cbt->total_build_times; + int timeout_count=0; + int i; + + /* how many of our recent circuits made it to the first hop but then + * timed out? */ + for (i = 0; i < cbt->liveness.num_recent_circs; i++) { + timeout_count += cbt->liveness.timeouts_after_firsthop[i]; + } + + /* If 80% of our recent circuits are timing out after the first hop, + * we need to re-estimate a new initial alpha and timeout. */ + if (timeout_count < circuit_build_times_max_timeouts()) { + return 0; + } + + circuit_build_times_reset(cbt); + memset(cbt->liveness.timeouts_after_firsthop, 0, + sizeof(*cbt->liveness.timeouts_after_firsthop)* + cbt->liveness.num_recent_circs); + cbt->liveness.after_firsthop_idx = 0; + + /* Check to see if this has happened before. If so, double the timeout + * to give people on abysmally bad network connections a shot at access */ + if (cbt->timeout_ms >= circuit_build_times_get_initial_timeout()) { + if (cbt->timeout_ms > INT32_MAX/2 || cbt->close_ms > INT32_MAX/2) { + log_warn(LD_CIRC, "Insanely large circuit build timeout value. " + "(timeout = %lfmsec, close = %lfmsec)", + cbt->timeout_ms, cbt->close_ms); + } else { + cbt->timeout_ms *= 2; + cbt->close_ms *= 2; + } + } else { + cbt->close_ms = cbt->timeout_ms + = circuit_build_times_get_initial_timeout(); + } + + control_event_buildtimeout_set(cbt, BUILDTIMEOUT_SET_EVENT_RESET); + + log_notice(LD_CIRC, + "Your network connection speed appears to have changed. Resetting " + "timeout to %lds after %d timeouts and %d buildtimes.", + tor_lround(cbt->timeout_ms/1000), timeout_count, + total_build_times); + + return 1; +} + +/** + * Count the number of timeouts in a set of cbt data. + */ +double +circuit_build_times_timeout_rate(const circuit_build_times_t *cbt) +{ + int i=0,timeouts=0; + for (i = 0; i < CBT_NCIRCUITS_TO_OBSERVE; i++) { + if (cbt->circuit_build_times[i] >= cbt->timeout_ms) { + timeouts++; + } + } + + if (!cbt->total_build_times) + return 0; + + return ((double)timeouts)/cbt->total_build_times; +} + +/** + * Count the number of closed circuits in a set of cbt data. + */ +double +circuit_build_times_close_rate(const circuit_build_times_t *cbt) +{ + int i=0,closed=0; + for (i = 0; i < CBT_NCIRCUITS_TO_OBSERVE; i++) { + if (cbt->circuit_build_times[i] == CBT_BUILD_ABANDONED) { + closed++; + } + } + + if (!cbt->total_build_times) + return 0; + + return ((double)closed)/cbt->total_build_times; +} + +/** + * Store a timeout as a synthetic value. + * + * Returns true if the store was successful and we should possibly + * update our timeout estimate. + */ +int +circuit_build_times_count_close(circuit_build_times_t *cbt, + int did_onehop, + time_t start_time) +{ + if (circuit_build_times_disabled()) { + cbt->close_ms = cbt->timeout_ms + = circuit_build_times_get_initial_timeout(); + return 0; + } + + /* Record this force-close to help determine if the network is dead */ + circuit_build_times_network_close(cbt, did_onehop, start_time); + + /* Only count timeouts if network is live.. */ + if (!circuit_build_times_network_check_live(cbt)) { + return 0; + } + + circuit_build_times_add_time(cbt, CBT_BUILD_ABANDONED); + return 1; +} + +/** + * Update timeout counts to determine if we need to expire + * our build time history due to excessive timeouts. + * + * We do not record any actual time values at this stage; + * we are only interested in recording the fact that a timeout + * happened. We record the time values via + * circuit_build_times_count_close() and circuit_build_times_add_time(). + */ +void +circuit_build_times_count_timeout(circuit_build_times_t *cbt, + int did_onehop) +{ + if (circuit_build_times_disabled()) { + cbt->close_ms = cbt->timeout_ms + = circuit_build_times_get_initial_timeout(); + return; + } + + /* Register the fact that a timeout just occurred. */ + circuit_build_times_network_timeout(cbt, did_onehop); + + /* If there are a ton of timeouts, we should reset + * the circuit build timeout. */ + circuit_build_times_network_check_changed(cbt); +} + +/** + * Estimate a new timeout based on history and set our timeout + * variable accordingly. + */ +static int +circuit_build_times_set_timeout_worker(circuit_build_times_t *cbt) +{ + build_time_t max_time; + if (!circuit_build_times_enough_to_compute(cbt)) + return 0; + + if (!circuit_build_times_update_alpha(cbt)) + return 0; + + cbt->timeout_ms = circuit_build_times_calculate_timeout(cbt, + circuit_build_times_quantile_cutoff()); + + cbt->close_ms = circuit_build_times_calculate_timeout(cbt, + circuit_build_times_close_quantile()); + + max_time = circuit_build_times_max(cbt); + + /* Sometimes really fast guard nodes give us such a steep curve + * that this ends up being not that much greater than timeout_ms. + * Make it be at least 1 min to handle this case. */ + cbt->close_ms = MAX(cbt->close_ms, circuit_build_times_initial_timeout()); + + if (cbt->timeout_ms > max_time) { + log_notice(LD_CIRC, + "Circuit build timeout of %dms is beyond the maximum build " + "time we have ever observed. Capping it to %dms.", + (int)cbt->timeout_ms, max_time); + cbt->timeout_ms = max_time; + } + + if (max_time < INT32_MAX/2 && cbt->close_ms > 2*max_time) { + log_info(LD_CIRC, + "Circuit build measurement period of %dms is more than twice " + "the maximum build time we have ever observed. Capping it to " + "%dms.", (int)cbt->close_ms, 2*max_time); + cbt->close_ms = 2*max_time; + } + + cbt->have_computed_timeout = 1; + return 1; +} + +/** + * Exposed function to compute a new timeout. Dispatches events and + * also filters out extremely high timeout values. + */ +void +circuit_build_times_set_timeout(circuit_build_times_t *cbt) +{ + long prev_timeout = tor_lround(cbt->timeout_ms/1000); + double timeout_rate; + + if (!circuit_build_times_set_timeout_worker(cbt)) + return; + + if (cbt->timeout_ms < circuit_build_times_min_timeout()) { + log_warn(LD_CIRC, "Set buildtimeout to low value %lfms. Setting to %dms", + cbt->timeout_ms, circuit_build_times_min_timeout()); + cbt->timeout_ms = circuit_build_times_min_timeout(); + if (cbt->close_ms < cbt->timeout_ms) { + /* This shouldn't happen because of MAX() in timeout_worker above, + * but doing it just in case */ + cbt->close_ms = circuit_build_times_initial_timeout(); + } + } + + control_event_buildtimeout_set(cbt, BUILDTIMEOUT_SET_EVENT_COMPUTED); + + timeout_rate = circuit_build_times_timeout_rate(cbt); + + if (prev_timeout > tor_lround(cbt->timeout_ms/1000)) { + log_notice(LD_CIRC, + "Based on %d circuit times, it looks like we don't need to " + "wait so long for circuits to finish. We will now assume a " + "circuit is too slow to use after waiting %ld seconds.", + cbt->total_build_times, + tor_lround(cbt->timeout_ms/1000)); + log_info(LD_CIRC, + "Circuit timeout data: %lfms, %lfms, Xm: %d, a: %lf, r: %lf", + cbt->timeout_ms, cbt->close_ms, cbt->Xm, cbt->alpha, + timeout_rate); + } else if (prev_timeout < tor_lround(cbt->timeout_ms/1000)) { + log_notice(LD_CIRC, + "Based on %d circuit times, it looks like we need to wait " + "longer for circuits to finish. We will now assume a " + "circuit is too slow to use after waiting %ld seconds.", + cbt->total_build_times, + tor_lround(cbt->timeout_ms/1000)); + log_info(LD_CIRC, + "Circuit timeout data: %lfms, %lfms, Xm: %d, a: %lf, r: %lf", + cbt->timeout_ms, cbt->close_ms, cbt->Xm, cbt->alpha, + timeout_rate); + } else { + log_info(LD_CIRC, + "Set circuit build timeout to %lds (%lfms, %lfms, Xm: %d, a: %lf," + " r: %lf) based on %d circuit times", + tor_lround(cbt->timeout_ms/1000), + cbt->timeout_ms, cbt->close_ms, cbt->Xm, cbt->alpha, timeout_rate, + cbt->total_build_times); + } +} + /** Iterate over values of circ_id, starting from conn-\>next_circ_id, * and with the high bit specified by conn-\>circ_id_type, until we get * a circ_id that is not in use by any other circuit on that conn. @@ -112,21 +1505,21 @@ circuit_list_path_impl(origin_circuit_t *circ, int verbose, int verbose_names) crypt_path_t *hop; smartlist_t *elements; const char *states[] = {"closed", "waiting for keys", "open"}; - char buf[128]; char *s; elements = smartlist_create(); if (verbose) { const char *nickname = build_state_get_exit_nickname(circ->build_state); - tor_snprintf(buf, sizeof(buf), "%s%s circ (length %d%s%s):", + char *cp; + tor_asprintf(&cp, "%s%s circ (length %d%s%s):", circ->build_state->is_internal ? "internal" : "exit", circ->build_state->need_uptime ? " (high-uptime)" : "", circ->build_state->desired_path_len, - circ->_base.state == CIRCUIT_STATE_OPEN ? "" : ", exit ", + circ->_base.state == CIRCUIT_STATE_OPEN ? "" : ", last hop ", circ->_base.state == CIRCUIT_STATE_OPEN ? "" : (nickname?nickname:"*unnamed*")); - smartlist_add(elements, tor_strdup(buf)); + smartlist_add(elements, cp); } hop = circ->cpath; @@ -148,8 +1541,7 @@ circuit_list_path_impl(origin_circuit_t *circ, int verbose, int verbose_names) router_get_verbose_nickname(elt, ri); } else if ((rs = router_get_consensus_status_by_id(id))) { routerstatus_get_verbose_nickname(elt, rs); - } else if (hop->extend_info->nickname && - is_legal_nickname(hop->extend_info->nickname)) { + } else if (is_legal_nickname(hop->extend_info->nickname)) { elt[0] = '$'; base16_encode(elt+1, HEX_DIGEST_LEN+1, id, DIGEST_LEN); elt[HEX_DIGEST_LEN+1]= '~'; @@ -217,7 +1609,7 @@ void circuit_log_path(int severity, unsigned int domain, origin_circuit_t *circ) { char *s = circuit_list_path(circ,1); - log(severity,domain,"%s",s); + tor_log(severity,domain,"%s",s); tor_free(s); } @@ -269,7 +1661,7 @@ static int onion_populate_cpath(origin_circuit_t *circ) { int r; -again: + again: r = onion_extend_cpath(circ); if (r < 0) { log_info(LD_CIRC,"Generating cpath hop failed."); @@ -361,9 +1753,10 @@ circuit_handle_first_hop(origin_circuit_t *circ) if (!n_conn) { /* not currently connected in a useful way. */ - const char *name = firsthop->extend_info->nickname ? + const char *name = strlen(firsthop->extend_info->nickname) ? firsthop->extend_info->nickname : fmt_addr(&firsthop->extend_info->addr); - log_info(LD_CIRC, "Next router is %s: %s ", safe_str(name), msg?msg:"???"); + log_info(LD_CIRC, "Next router is %s: %s ", + safe_str_client(name), msg?msg:"???"); circ->_base.n_hop = extend_info_dup(firsthop->extend_info); if (should_launch) { @@ -506,7 +1899,8 @@ circuit_deliver_create_cell(circuit_t *circ, uint8_t cell_type, cell.circ_id = circ->n_circ_id; memcpy(cell.payload, payload, ONIONSKIN_CHALLENGE_LEN); - append_cell_to_circuit_queue(circ, circ->n_conn, &cell, CELL_DIRECTION_OUT); + append_cell_to_circuit_queue(circ, circ->n_conn, &cell, + CELL_DIRECTION_OUT, 0); if (CIRCUIT_IS_ORIGIN(circ)) { /* mark it so it gets better rate limiting treatment. */ @@ -536,7 +1930,7 @@ inform_testing_reachability(void) "CHECKING_REACHABILITY DIRADDRESS=%s:%d", me->address, me->dir_port); } - log(LOG_NOTICE, LD_OR, "Now checking whether ORPort %s:%d%s %s reachable... " + log_notice(LD_OR, "Now checking whether ORPort %s:%d%s %s reachable... " "(this may take up to %d minutes -- look for log " "messages indicating success)", me->address, me->or_port, @@ -569,6 +1963,18 @@ should_use_create_fast_for_circuit(origin_circuit_t *circ) return 1; } +/** Return true if <b>circ</b> is the type of circuit we want to count + * timeouts from. In particular, we want it to have not completed yet + * (already completing indicates we cannibalized it), and we want it to + * have exactly three hops. + */ +int +circuit_timeout_want_to_count_circ(origin_circuit_t *circ) +{ + return !circ->has_opened + && circ->build_state->desired_path_len == DEFAULT_ROUTE_LEN; +} + /** This is the backbone function for building circuits. * * If circ's first hop is closed, then we need to build a create @@ -642,15 +2048,43 @@ circuit_send_next_onion_skin(origin_circuit_t *circ) if (!hop) { /* done building the circuit. whew. */ circuit_set_state(TO_CIRCUIT(circ), CIRCUIT_STATE_OPEN); + if (circuit_timeout_want_to_count_circ(circ)) { + struct timeval end; + long timediff; + tor_gettimeofday(&end); + timediff = tv_mdiff(&circ->_base.timestamp_created, &end); + + /* + * If the circuit build time is much greater than we would have cut + * it off at, we probably had a suspend event along this codepath, + * and we should discard the value. + */ + if (timediff < 0 || timediff > 2*circ_times.close_ms+1000) { + log_notice(LD_CIRC, "Strange value for circuit build time: %ldmsec. " + "Assuming clock jump. Purpose %d (%s)", timediff, + circ->_base.purpose, + circuit_purpose_to_string(circ->_base.purpose)); + } else if (!circuit_build_times_disabled()) { + /* Only count circuit times if the network is live */ + if (circuit_build_times_network_check_live(&circ_times)) { + circuit_build_times_add_time(&circ_times, (build_time_t)timediff); + circuit_build_times_set_timeout(&circ_times); + } + + if (circ->_base.purpose != CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT) { + circuit_build_times_network_circ_success(&circ_times); + } + } + } log_info(LD_CIRC,"circuit built!"); circuit_reset_failure_count(0); if (circ->build_state->onehop_tunnel) control_event_bootstrap(BOOTSTRAP_STATUS_REQUESTING_STATUS, 0); - if (!has_completed_circuit && !circ->build_state->onehop_tunnel) { + if (!can_complete_circuit && !circ->build_state->onehop_tunnel) { or_options_t *options = get_options(); - has_completed_circuit=1; + can_complete_circuit=1; /* FFFF Log a count of known routers here */ - log(LOG_NOTICE, LD_GENERAL, + log_notice(LD_GENERAL, "Tor has successfully opened a circuit. " "Looks like client functionality is working."); control_event_bootstrap(BOOTSTRAP_STATUS_DONE, 0); @@ -662,6 +2096,10 @@ circuit_send_next_onion_skin(origin_circuit_t *circ) } circuit_rep_hist_note_result(circ); circuit_has_opened(circ); /* do other actions as necessary */ + + /* We're done with measurement circuits here. Just close them */ + if (circ->_base.purpose == CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT) + circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_FINISHED); return 0; } @@ -705,13 +2143,13 @@ void circuit_note_clock_jumped(int seconds_elapsed) { int severity = server_mode(get_options()) ? LOG_WARN : LOG_NOTICE; - log(severity, LD_GENERAL, "Your system clock just jumped %d seconds %s; " + tor_log(severity, LD_GENERAL, "Your system clock just jumped %d seconds %s; " "assuming established circuits no longer work.", seconds_elapsed >=0 ? seconds_elapsed : -seconds_elapsed, seconds_elapsed >=0 ? "forward" : "backward"); control_event_general_status(LOG_WARN, "CLOCK_JUMPED TIME=%d", seconds_elapsed); - has_completed_circuit=0; /* so it'll log when it works again */ + can_complete_circuit=0; /* so it'll log when it works again */ control_event_client_status(severity, "CIRCUIT_NOT_ESTABLISHED REASON=%s", "CLOCK_JUMPED"); circuit_mark_all_unused_circs(); @@ -944,10 +2382,9 @@ circuit_finish_handshake(origin_circuit_t *circ, uint8_t reply_type, return -END_CIRC_REASON_TORPROTOCOL; } - if (hop->dh_handshake_state) { - crypto_dh_free(hop->dh_handshake_state); /* don't need it anymore */ - hop->dh_handshake_state = NULL; - } + crypto_dh_free(hop->dh_handshake_state); /* don't need it anymore */ + hop->dh_handshake_state = NULL; + memset(hop->fast_handshake_state, 0, sizeof(hop->fast_handshake_state)); if (circuit_init_cpath_crypto(hop, keys, 0)<0) { @@ -1035,8 +2472,8 @@ onionskin_answer(or_circuit_t *circ, uint8_t cell_type, const char *payload, cell_type == CELL_CREATED ? ONIONSKIN_REPLY_LEN : DIGEST_LEN*2); log_debug(LD_CIRC,"init digest forward 0x%.8x, backward 0x%.8x.", - (unsigned int)*(uint32_t*)(keys), - (unsigned int)*(uint32_t*)(keys+20)); + (unsigned int)get_uint32(keys), + (unsigned int)get_uint32(keys+20)); if (circuit_init_cpath_crypto(tmp_cpath, keys, 0)<0) { log_warn(LD_BUG,"Circuit initialization failed"); tor_free(tmp_cpath); @@ -1057,7 +2494,7 @@ onionskin_answer(or_circuit_t *circ, uint8_t cell_type, const char *payload, circ->is_first_hop = (cell_type == CELL_CREATED_FAST); append_cell_to_circuit_queue(TO_CIRCUIT(circ), - circ->p_conn, &cell, CELL_DIRECTION_IN); + circ->p_conn, &cell, CELL_DIRECTION_IN, 0); log_debug(LD_CIRC,"Finished sending 'created' cell."); if (!is_local_addr(&circ->p_conn->_base.addr) && @@ -1086,7 +2523,7 @@ new_route_len(uint8_t purpose, extend_info_t *exit, tor_assert(routers); - routelen = 3; + routelen = DEFAULT_ROUTE_LEN; if (exit && purpose != CIRCUIT_PURPOSE_TESTING && purpose != CIRCUIT_PURPOSE_S_ESTABLISH_INTRO) @@ -1151,6 +2588,8 @@ circuit_all_predicted_ports_handled(time_t now, int *need_uptime, smartlist_t *LongLivedServices = get_options()->LongLivedPorts; tor_assert(need_uptime); tor_assert(need_capacity); + // Always predict need_capacity + *need_capacity = 1; enough = (smartlist_len(sl) == 0); for (i = 0; i < smartlist_len(sl); ++i) { port = smartlist_get(sl, i); @@ -1173,6 +2612,8 @@ router_handles_some_port(routerinfo_t *router, smartlist_t *needed_ports) for (i = 0; i < smartlist_len(needed_ports); ++i) { addr_policy_result_t r; + /* alignment issues aren't a worry for this dereference, since + needed_ports is explicitly a smartlist of uint16_t's */ port = *(uint16_t *)smartlist_get(needed_ports, i); tor_assert(port); r = compare_addr_to_addr_policy(0, port, router->exit_policy); @@ -1255,9 +2696,24 @@ choose_good_exit_server_general(routerlist_t *dir, int need_uptime, n_supported[i] = -1; continue; /* skip routers that are known to be down or bad exits */ } + + if (options->_ExcludeExitNodesUnion && + routerset_contains_router(options->_ExcludeExitNodesUnion, router)) { + n_supported[i] = -1; + continue; /* user asked us not to use it, no matter what */ + } + if (options->ExitNodes && + !routerset_contains_router(options->ExitNodes, router)) { + n_supported[i] = -1; + continue; /* not one of our chosen exit nodes */ + } + if (router_is_unreliable(router, need_uptime, need_capacity, 0)) { n_supported[i] = -1; - continue; /* skip routers that are not suitable */ + continue; /* skip routers that are not suitable. Don't worry if + * this makes us reject all the possible routers: if so, + * we'll retry later in this function with need_update and + * need_capacity set to 0. */ } if (!(router->is_valid || options->_AllowInvalid & ALLOW_INVALID_EXIT)) { /* if it's invalid and we don't want it */ @@ -1316,28 +2772,21 @@ choose_good_exit_server_general(routerlist_t *dir, int need_uptime, /* If any routers definitely support any pending connections, choose one * at random. */ if (best_support > 0) { - smartlist_t *supporting = smartlist_create(), *use = smartlist_create(); + smartlist_t *supporting = smartlist_create(); for (i = 0; i < smartlist_len(dir->routers); i++) if (n_supported[i] == best_support) smartlist_add(supporting, smartlist_get(dir->routers, i)); - routersets_get_disjunction(use, supporting, options->ExitNodes, - options->_ExcludeExitNodesUnion, 1); - if (smartlist_len(use) == 0 && !options->StrictExitNodes) { - routersets_get_disjunction(use, supporting, NULL, - options->_ExcludeExitNodesUnion, 1); - } - router = routerlist_sl_choose_by_bandwidth(use, WEIGHT_FOR_EXIT); - smartlist_free(use); + router = routerlist_sl_choose_by_bandwidth(supporting, WEIGHT_FOR_EXIT); smartlist_free(supporting); } else { /* Either there are no pending connections, or no routers even seem to * possibly support any of them. Choose a router at random that satisfies * at least one predicted exit port. */ - int try; - smartlist_t *needed_ports, *supporting, *use; + int attempt; + smartlist_t *needed_ports, *supporting; if (best_support == -1) { if (need_uptime || need_capacity) { @@ -1349,42 +2798,32 @@ choose_good_exit_server_general(routerlist_t *dir, int need_uptime, tor_free(n_supported); return choose_good_exit_server_general(dir, 0, 0); } - log_notice(LD_CIRC, "All routers are down or won't exit -- choosing a " - "doomed exit at random."); + log_notice(LD_CIRC, "All routers are down or won't exit%s -- " + "choosing a doomed exit at random.", + options->_ExcludeExitNodesUnion ? " or are Excluded" : ""); } supporting = smartlist_create(); - use = smartlist_create(); needed_ports = circuit_get_unhandled_ports(time(NULL)); - for (try = 0; try < 2; try++) { + for (attempt = 0; attempt < 2; attempt++) { /* try once to pick only from routers that satisfy a needed port, * then if there are none, pick from any that support exiting. */ for (i = 0; i < smartlist_len(dir->routers); i++) { router = smartlist_get(dir->routers, i); if (n_supported[i] != -1 && - (try || router_handles_some_port(router, needed_ports))) { + (attempt || router_handles_some_port(router, needed_ports))) { // log_fn(LOG_DEBUG,"Try %d: '%s' is a possibility.", // try, router->nickname); smartlist_add(supporting, router); } } - routersets_get_disjunction(use, supporting, options->ExitNodes, - options->_ExcludeExitNodesUnion, 1); - if (smartlist_len(use) == 0 && !options->StrictExitNodes) { - routersets_get_disjunction(use, supporting, NULL, - options->_ExcludeExitNodesUnion, 1); - } - /* XXX sometimes the above results in null, when the requested - * exit node is down. we should pick it anyway. */ - router = routerlist_sl_choose_by_bandwidth(use, WEIGHT_FOR_EXIT); + router = routerlist_sl_choose_by_bandwidth(supporting, WEIGHT_FOR_EXIT); if (router) break; smartlist_clear(supporting); - smartlist_clear(use); } SMARTLIST_FOREACH(needed_ports, uint16_t *, cp, tor_free(cp)); smartlist_free(needed_ports); - smartlist_free(use); smartlist_free(supporting); } @@ -1393,10 +2832,11 @@ choose_good_exit_server_general(routerlist_t *dir, int need_uptime, log_info(LD_CIRC, "Chose exit server '%s'", router->nickname); return router; } - if (options->StrictExitNodes) { + if (options->ExitNodes) { log_warn(LD_CIRC, - "No specified exit routers seem to be running, and " - "StrictExitNodes is set: can't choose an exit."); + "No specified %sexit routers seem to be running: " + "can't choose an exit.", + options->_ExcludeExitNodesUnion ? "non-excluded " : ""); } return NULL; } @@ -1427,15 +2867,13 @@ choose_good_exit_server(uint8_t purpose, routerlist_t *dir, if (options->_AllowInvalid & ALLOW_INVALID_MIDDLE) flags |= CRN_ALLOW_INVALID; if (is_internal) /* pick it like a middle hop */ - return router_choose_random_node(NULL, NULL, - options->ExcludeNodes, flags); + return router_choose_random_node(NULL, options->ExcludeNodes, flags); else return choose_good_exit_server_general(dir,need_uptime,need_capacity); case CIRCUIT_PURPOSE_C_ESTABLISH_REND: if (options->_AllowInvalid & ALLOW_INVALID_RENDEZVOUS) flags |= CRN_ALLOW_INVALID; - return router_choose_random_node(NULL, NULL, - options->ExcludeNodes, flags); + return router_choose_random_node(NULL, options->ExcludeNodes, flags); } log_warn(LD_BUG,"Unhandled purpose %d", purpose); tor_fragile_assert(); @@ -1450,7 +2888,6 @@ warn_if_last_router_excluded(origin_circuit_t *circ, const extend_info_t *exit) or_options_t *options = get_options(); routerset_t *rs = options->ExcludeNodes; const char *description; - int domain = LD_CIRC; uint8_t purpose = circ->_base.purpose; if (circ->build_state->onehop_tunnel) @@ -1463,13 +2900,14 @@ warn_if_last_router_excluded(origin_circuit_t *circ, const extend_info_t *exit) case CIRCUIT_PURPOSE_INTRO_POINT: case CIRCUIT_PURPOSE_REND_POINT_WAITING: case CIRCUIT_PURPOSE_REND_ESTABLISHED: - log_warn(LD_BUG, "Called on non-origin circuit (purpose %d)", - (int)purpose); + log_warn(LD_BUG, "Called on non-origin circuit (purpose %d, %s)", + (int)purpose, + circuit_purpose_to_string(purpose)); return; case CIRCUIT_PURPOSE_C_GENERAL: if (circ->build_state->is_internal) return; - description = "Requested exit node"; + description = "requested exit node"; rs = options->_ExcludeExitNodesUnion; break; case CIRCUIT_PURPOSE_C_INTRODUCING: @@ -1484,22 +2922,34 @@ warn_if_last_router_excluded(origin_circuit_t *circ, const extend_info_t *exit) case CIRCUIT_PURPOSE_C_REND_READY: case CIRCUIT_PURPOSE_C_REND_READY_INTRO_ACKED: case CIRCUIT_PURPOSE_C_REND_JOINED: - description = "Chosen rendezvous point"; - domain = LD_BUG; + description = "chosen rendezvous point"; break; case CIRCUIT_PURPOSE_CONTROLLER: rs = options->_ExcludeExitNodesUnion; - description = "Controller-selected circuit target"; + description = "controller-selected circuit target"; break; } if (routerset_contains_extendinfo(rs, exit)) { - log_fn(LOG_WARN, domain, "%s '%s' is in ExcludeNodes%s. Using anyway " - "(circuit purpose %d).", - description,exit->nickname, - rs==options->ExcludeNodes?"":" or ExcludeExitNodes", - (int)purpose); - circuit_log_path(LOG_WARN, domain, circ); + /* We should never get here if StrictNodes is set to 1. */ + if (options->StrictNodes) { + log_warn(LD_BUG, "Using %s '%s' which is listed in ExcludeNodes%s, " + "even though StrictNodes is set. Please report. " + "(Circuit purpose: %s)", + description, exit->nickname, + rs==options->ExcludeNodes?"":" or ExcludeExitNodes", + circuit_purpose_to_string(purpose)); + } else { + log_warn(LD_CIRC, "Using %s '%s' which is listed in " + "ExcludeNodes%s, because no better options were available. To " + "prevent this (and possibly break your Tor functionality), " + "set the StrictNodes configuration option. " + "(Circuit purpose: %s)", + description, exit->nickname, + rs==options->ExcludeNodes?"":" or ExcludeExitNodes", + circuit_purpose_to_string(purpose)); + } + circuit_log_path(LOG_WARN, LD_CIRC, circ); } return; @@ -1555,8 +3005,7 @@ circuit_append_new_exit(origin_circuit_t *circ, extend_info_t *exit) state = circ->build_state; tor_assert(state); - if (state->chosen_exit) - extend_info_free(state->chosen_exit); + extend_info_free(state->chosen_exit); state->chosen_exit = extend_info_dup(exit); ++circ->build_state->desired_path_len; @@ -1678,8 +3127,7 @@ choose_good_middle_server(uint8_t purpose, flags |= CRN_NEED_CAPACITY; if (options->_AllowInvalid & ALLOW_INVALID_MIDDLE) flags |= CRN_ALLOW_INVALID; - choice = router_choose_random_node(NULL, - excluded, options->ExcludeNodes, flags); + choice = router_choose_random_node(excluded, options->ExcludeNodes, flags); smartlist_free(excluded); return choice; } @@ -1743,11 +3191,7 @@ choose_good_entry_server(uint8_t purpose, cpath_build_state_t *state) if (options->_AllowInvalid & ALLOW_INVALID_ENTRY) flags |= CRN_ALLOW_INVALID; - choice = router_choose_random_node( - NULL, - excluded, - options->ExcludeNodes, - flags); + choice = router_choose_random_node(excluded, options->ExcludeNodes, flags); smartlist_free(excluded); return choice; } @@ -1868,9 +3312,9 @@ extend_info_from_router(routerinfo_t *r) void extend_info_free(extend_info_t *info) { - tor_assert(info); - if (info->onion_key) - crypto_free_pk_env(info->onion_key); + if (!info) + return; + crypto_free_pk_env(info->onion_key); tor_free(info); } @@ -1929,8 +3373,6 @@ entry_guard_set_status(entry_guard_t *e, routerinfo_t *ri, char buf[HEX_DIGEST_LEN+1]; int changed = 0; - tor_assert(options); - *reason = NULL; /* Do we want to mark this guard as bad? */ @@ -1993,35 +3435,58 @@ entry_is_time_to_retry(entry_guard_t *e, time_t now) * - Listed as either up or never yet contacted; * - Present in the routerlist; * - Listed as 'stable' or 'fast' by the current dirserver consensus, - * if demanded by <b>need_uptime</b> or <b>need_capacity</b>; - * (This check is currently redundant with the Guard flag, but in - * the future that might change. Best to leave it in for now.) + * if demanded by <b>need_uptime</b> or <b>need_capacity</b> + * (unless it's a configured EntryNode); * - Allowed by our current ReachableORAddresses config option; and - * - Currently thought to be reachable by us (unless assume_reachable + * - Currently thought to be reachable by us (unless <b>assume_reachable</b> * is true). + * + * If the answer is no, set *<b>msg</b> to an explanation of why. */ static INLINE routerinfo_t * entry_is_live(entry_guard_t *e, int need_uptime, int need_capacity, - int assume_reachable) + int assume_reachable, const char **msg) { routerinfo_t *r; - if (e->bad_since) + or_options_t *options = get_options(); + tor_assert(msg); + + if (e->bad_since) { + *msg = "bad"; return NULL; + } /* no good if it's unreachable, unless assume_unreachable or can_retry. */ if (!assume_reachable && !e->can_retry && - e->unreachable_since && !entry_is_time_to_retry(e, time(NULL))) + e->unreachable_since && !entry_is_time_to_retry(e, time(NULL))) { + *msg = "unreachable"; return NULL; + } r = router_get_by_digest(e->identity); - if (!r) + if (!r) { + *msg = "no descriptor"; return NULL; - if (get_options()->UseBridges && r->purpose != ROUTER_PURPOSE_BRIDGE) + } + if (get_options()->UseBridges && r->purpose != ROUTER_PURPOSE_BRIDGE) { + *msg = "not a bridge"; return NULL; - if (!get_options()->UseBridges && r->purpose != ROUTER_PURPOSE_GENERAL) + } + if (!get_options()->UseBridges && r->purpose != ROUTER_PURPOSE_GENERAL) { + *msg = "not general-purpose"; return NULL; - if (router_is_unreliable(r, need_uptime, need_capacity, 0)) + } + if (options->EntryNodes && + routerset_contains_router(options->EntryNodes, r)) { + /* they asked for it, they get it */ + need_uptime = need_capacity = 0; + } + if (router_is_unreliable(r, need_uptime, need_capacity, 0)) { + *msg = "not fast/stable"; return NULL; - if (!fascist_firewall_allows_or(r)) + } + if (!fascist_firewall_allows_or(r)) { + *msg = "unreachable by config"; return NULL; + } return r; } @@ -2030,11 +3495,12 @@ static int num_live_entry_guards(void) { int n = 0; + const char *msg; if (! entry_guards) return 0; SMARTLIST_FOREACH(entry_guards, entry_guard_t *, entry, { - if (entry_is_live(entry, 0, 1, 0)) + if (entry_is_live(entry, 0, 1, 0, &msg)) ++n; }); return n; @@ -2058,16 +3524,21 @@ static void log_entry_guards(int severity) { smartlist_t *elements = smartlist_create(); - char buf[1024]; char *s; SMARTLIST_FOREACH(entry_guards, entry_guard_t *, e, { - tor_snprintf(buf, sizeof(buf), "%s (%s%s)", - e->nickname, - entry_is_live(e, 0, 1, 0) ? "up " : "down ", - e->made_contact ? "made-contact" : "never-contacted"); - smartlist_add(elements, tor_strdup(buf)); + const char *msg = NULL; + char *cp; + if (entry_is_live(e, 0, 1, 0, &msg)) + tor_asprintf(&cp, "%s (up %s)", + e->nickname, + e->made_contact ? "made-contact" : "never-contacted"); + else + tor_asprintf(&cp, "%s (%s, %s)", + e->nickname, msg, + e->made_contact ? "made-contact" : "never-contacted"); + smartlist_add(elements, cp); }); s = smartlist_join_strings(elements, ",", 0, NULL); @@ -2091,12 +3562,13 @@ control_event_guard_deferred(void) **/ #if 0 int n = 0; + const char *msg; or_options_t *options = get_options(); if (!entry_guards) return; SMARTLIST_FOREACH(entry_guards, entry_guard_t *, entry, { - if (entry_is_live(entry, 0, 1, 0)) { + if (entry_is_live(entry, 0, 1, 0, &msg)) { if (n++ == options->NumEntryGuards) { control_event_guard(entry->nickname, entry->identity, "DEFERRED"); return; @@ -2158,9 +3630,8 @@ add_an_entry_guard(routerinfo_t *chosen, int reset_status) /** If the use of entry guards is configured, choose more entry guards * until we have enough in the list. */ static void -pick_entry_guards(void) +pick_entry_guards(or_options_t *options) { - or_options_t *options = get_options(); int changed = 0; tor_assert(entry_guards); @@ -2182,7 +3653,8 @@ pick_entry_guards(void) static void entry_guard_free(entry_guard_t *e) { - tor_assert(e); + if (!e) + return; tor_free(e->chosen_by_version); tor_free(e); } @@ -2191,10 +3663,9 @@ entry_guard_free(entry_guard_t *e) * or which was selected by a version of Tor that's known to select * entry guards badly. */ static int -remove_obsolete_entry_guards(void) +remove_obsolete_entry_guards(time_t now) { int changed = 0, i; - time_t now = time(NULL); for (i = 0; i < smartlist_len(entry_guards); ++i) { entry_guard_t *entry = smartlist_get(entry_guards, i); @@ -2254,11 +3725,10 @@ remove_obsolete_entry_guards(void) * long that we don't think they'll come up again. Return 1 if we * removed any, or 0 if we did nothing. */ static int -remove_dead_entry_guards(void) +remove_dead_entry_guards(time_t now) { char dbuf[HEX_DIGEST_LEN+1]; char tbuf[ISO_TIME_LEN+1]; - time_t now = time(NULL); int i; int changed = 0; @@ -2293,19 +3763,17 @@ remove_dead_entry_guards(void) * think that things are unlisted. */ void -entry_guards_compute_status(void) +entry_guards_compute_status(or_options_t *options, time_t now) { - time_t now; int changed = 0; int severity = LOG_DEBUG; - or_options_t *options; digestmap_t *reasons; + if (! entry_guards) return; - options = get_options(); - - now = time(NULL); + if (options->EntryNodes) /* reshuffle the entry guard list if needed */ + entry_nodes_should_be_added(); reasons = digestmap_new(); SMARTLIST_FOREACH_BEGIN(entry_guards, entry_guard_t *, entry) @@ -2322,7 +3790,7 @@ entry_guards_compute_status(void) } SMARTLIST_FOREACH_END(entry); - if (remove_dead_entry_guards()) + if (remove_dead_entry_guards(now)) changed = 1; severity = changed ? LOG_DEBUG : LOG_INFO; @@ -2330,13 +3798,16 @@ entry_guards_compute_status(void) if (changed) { SMARTLIST_FOREACH_BEGIN(entry_guards, entry_guard_t *, entry) { const char *reason = digestmap_get(reasons, entry->identity); - log_info(LD_CIRC, "Summary: Entry '%s' is %s, %s%s%s, and %s.", + const char *live_msg = ""; + routerinfo_t *r = entry_is_live(entry, 0, 1, 0, &live_msg); + log_info(LD_CIRC, "Summary: Entry '%s' is %s, %s%s%s, and %s%s.", entry->nickname, entry->unreachable_since ? "unreachable" : "reachable", entry->bad_since ? "unusable" : "usable", reason ? ", ": "", reason ? reason : "", - entry_is_live(entry, 0, 1, 0) ? "live" : "not live"); + r ? "live" : "not live / ", + r ? "" : live_msg); } SMARTLIST_FOREACH_END(entry); log_info(LD_CIRC, " (%d/%d entry guards are usable/new)", num_live_entry_guards(), smartlist_len(entry_guards)); @@ -2355,7 +3826,7 @@ entry_guards_compute_status(void) * If <b>mark_relay_status</b>, also call router_set_status() on this * relay. * - * XXX022 change succeeded and mark_relay_status into 'int flags'. + * XXX023 change succeeded and mark_relay_status into 'int flags'. */ int entry_guard_register_connect_status(const char *digest, int succeeded, @@ -2407,6 +3878,7 @@ entry_guard_register_connect_status(const char *digest, int succeeded, "Removing from the list. %d/%d entry guards usable/new.", entry->nickname, buf, num_live_entry_guards()-1, smartlist_len(entry_guards)-1); + control_event_guard(entry->nickname, entry->identity, "DROPPED"); entry_guard_free(entry); smartlist_del_keeporder(entry_guards, idx); log_entry_guards(LOG_INFO); @@ -2443,7 +3915,8 @@ entry_guard_register_connect_status(const char *digest, int succeeded, if (e == entry) break; if (e->made_contact) { - routerinfo_t *r = entry_is_live(e, 0, 1, 1); + const char *msg; + routerinfo_t *r = entry_is_live(e, 0, 1, 1, &msg); if (r && e->unreachable_since) { refuse_conn = 1; e->can_retry = 1; @@ -2474,16 +3947,16 @@ static int should_add_entry_nodes = 0; void entry_nodes_should_be_added(void) { - log_info(LD_CIRC, "New EntryNodes config option detected. Will use."); + log_info(LD_CIRC, "EntryNodes config option set. Putting configured " + "relays at the front of the entry guard list."); should_add_entry_nodes = 1; } /** Add all nodes in EntryNodes that aren't currently guard nodes to the list * of guard nodes, at the front. */ static void -entry_guards_prepend_from_config(void) +entry_guards_prepend_from_config(or_options_t *options) { - or_options_t *options = get_options(); smartlist_t *entry_routers, *entry_fps; smartlist_t *old_entry_guards_on_list, *old_entry_guards_not_on_list; tor_assert(entry_guards); @@ -2498,7 +3971,7 @@ entry_guards_prepend_from_config(void) return; } - if (options->EntryNodes) { + { char *string = routerset_to_string(options->EntryNodes); log_info(LD_CIRC,"Adding configured EntryNodes '%s'.", string); tor_free(string); @@ -2511,13 +3984,14 @@ entry_guards_prepend_from_config(void) /* Split entry guards into those on the list and those not. */ - /* XXXX022 Now that we allow countries and IP ranges in EntryNodes, this is + /* XXXX023 Now that we allow countries and IP ranges in EntryNodes, this is * potentially an enormous list. For now, we disable such values for * EntryNodes in options_validate(); really, this wants a better solution. * Perhaps we should do this calculation once whenever the list of routers * changes or the entrynodes setting changes. */ - routerset_get_all_routers(entry_routers, options->EntryNodes, 0); + routerset_get_all_routers(entry_routers, options->EntryNodes, + options->ExcludeNodes, 0); SMARTLIST_FOREACH(entry_routers, routerinfo_t *, ri, smartlist_add(entry_fps,ri->cache_info.identity_digest)); SMARTLIST_FOREACH(entry_guards, entry_guard_t *, e, { @@ -2542,13 +4016,10 @@ entry_guards_prepend_from_config(void) SMARTLIST_FOREACH(entry_routers, routerinfo_t *, ri, { add_an_entry_guard(ri, 0); }); - /* Finally, the remaining EntryNodes, unless we're strict */ - if (options->StrictEntryNodes) { - SMARTLIST_FOREACH(old_entry_guards_not_on_list, entry_guard_t *, e, - entry_guard_free(e)); - } else { - smartlist_add_all(entry_guards, old_entry_guards_not_on_list); - } + /* Finally, free the remaining previously configured guards that are not in + * EntryNodes. */ + SMARTLIST_FOREACH(old_entry_guards_not_on_list, entry_guard_t *, e, + entry_guard_free(e)); smartlist_free(entry_routers); smartlist_free(entry_fps); @@ -2557,16 +4028,18 @@ entry_guards_prepend_from_config(void) entry_guards_changed(); } -/** Return 1 if we're fine adding arbitrary routers out of the - * directory to our entry guard list. Else return 0. */ +/** Return 0 if we're fine adding arbitrary routers out of the + * directory to our entry guard list, or return 1 if we have a + * list already and we must stick to it. + */ int -entry_list_can_grow(or_options_t *options) +entry_list_is_constrained(or_options_t *options) { - if (options->StrictEntryNodes) - return 0; + if (options->EntryNodes) + return 1; if (options->UseBridges) - return 0; - return 1; + return 1; + return 0; } /** Pick a live (up and listed) entry guard from entry_guards. If @@ -2584,10 +4057,9 @@ choose_random_entry(cpath_build_state_t *state) routerinfo_t *r = NULL; int need_uptime = state ? state->need_uptime : 0; int need_capacity = state ? state->need_capacity : 0; - int consider_exit_family = 0; + int preferred_min, consider_exit_family = 0; if (chosen_exit) { - smartlist_add(exit_family, chosen_exit); routerlist_add_family(exit_family, chosen_exit); consider_exit_family = 1; } @@ -2596,38 +4068,66 @@ choose_random_entry(cpath_build_state_t *state) entry_guards = smartlist_create(); if (should_add_entry_nodes) - entry_guards_prepend_from_config(); + entry_guards_prepend_from_config(options); - if (entry_list_can_grow(options) && - (! entry_guards || - smartlist_len(entry_guards) < options->NumEntryGuards)) - pick_entry_guards(); + if (!entry_list_is_constrained(options) && + smartlist_len(entry_guards) < options->NumEntryGuards) + pick_entry_guards(options); retry: smartlist_clear(live_entry_guards); SMARTLIST_FOREACH(entry_guards, entry_guard_t *, entry, { - r = entry_is_live(entry, need_uptime, need_capacity, 0); - if (r && (!consider_exit_family || !smartlist_isin(exit_family, r))) { - smartlist_add(live_entry_guards, r); - if (!entry->made_contact) { - /* Always start with the first not-yet-contacted entry - * guard. Otherwise we might add several new ones, pick - * the second new one, and now we've expanded our entry - * guard list without needing to. */ - goto choose_and_finish; + const char *msg; + r = entry_is_live(entry, need_uptime, need_capacity, 0, &msg); + if (!r) + continue; /* down, no point */ + if (r == chosen_exit) + continue; /* don't pick the same node for entry and exit */ + if (consider_exit_family && smartlist_isin(exit_family, r)) + continue; /* avoid relays that are family members of our exit */ +#if 0 /* since EntryNodes is always strict now, this clause is moot */ + if (options->EntryNodes && + !routerset_contains_router(options->EntryNodes, r)) { + /* We've come to the end of our preferred entry nodes. */ + if (smartlist_len(live_entry_guards)) + goto choose_and_finish; /* only choose from the ones we like */ + if (options->StrictNodes) { + /* in theory this case should never happen, since + * entry_guards_prepend_from_config() drops unwanted relays */ + tor_fragile_assert(); + } else { + log_info(LD_CIRC, + "No relays from EntryNodes available. Using others."); } - if (smartlist_len(live_entry_guards) >= options->NumEntryGuards) - break; /* we have enough */ } +#endif + smartlist_add(live_entry_guards, r); + if (!entry->made_contact) { + /* Always start with the first not-yet-contacted entry + * guard. Otherwise we might add several new ones, pick + * the second new one, and now we've expanded our entry + * guard list without needing to. */ + goto choose_and_finish; + } + if (smartlist_len(live_entry_guards) >= options->NumEntryGuards) + break; /* we have enough */ }); - /* Try to have at least 2 choices available. This way we don't - * get stuck with a single live-but-crummy entry and just keep - * using him. - * (We might get 2 live-but-crummy entry guards, but so be it.) */ - if (smartlist_len(live_entry_guards) < 2) { - if (entry_list_can_grow(options)) { + if (entry_list_is_constrained(options)) { + /* If we prefer the entry nodes we've got, and we have at least + * one choice, that's great. Use it. */ + preferred_min = 1; + } else { + /* Try to have at least 2 choices available. This way we don't + * get stuck with a single live-but-crummy entry and just keep + * using him. + * (We might get 2 live-but-crummy entry guards, but so be it.) */ + preferred_min = 2; + } + + if (smartlist_len(live_entry_guards) < preferred_min) { + if (!entry_list_is_constrained(options)) { /* still no? try adding a new entry then */ /* XXX if guard doesn't imply fast and stable, then we need * to tell add_an_entry_guard below what we want, or it might @@ -2652,26 +4152,31 @@ choose_random_entry(cpath_build_state_t *state) need_capacity = 0; goto retry; } - if (!r && !entry_list_can_grow(options) && consider_exit_family) { - /* still no? if we're using bridges or have strictentrynodes - * set, and our chosen exit is in the same family as all our - * bridges/entry guards, then be flexible about families. */ +#if 0 + /* Removing this retry logic: if we only allow one exit, and it is in the + same family as all our entries, then we are just plain not going to win + here. */ + if (!r && entry_list_is_constrained(options) && consider_exit_family) { + /* still no? if we're using bridges, + * and our chosen exit is in the same family as all our + * bridges, then be flexible about families. */ consider_exit_family = 0; goto retry; } +#endif /* live_entry_guards may be empty below. Oh well, we tried. */ } choose_and_finish: - if (entry_list_can_grow(options)) { + if (entry_list_is_constrained(options)) { + /* We need to weight by bandwidth, because our bridges or entryguards + * were not already selected proportional to their bandwidth. */ + r = routerlist_sl_choose_by_bandwidth(live_entry_guards, WEIGHT_FOR_GUARD); + } else { /* We choose uniformly at random here, because choose_good_entry_server() * already weights its choices by bandwidth, so we don't want to * *double*-weight our guard selection. */ r = smartlist_choose(live_entry_guards); - } else { - /* We need to weight by bandwidth, because our bridges or entryguards - * were not already selected proportional to their bandwidth. */ - r = routerlist_sl_choose_by_bandwidth(live_entry_guards, WEIGHT_FOR_GUARD); } smartlist_free(live_entry_guards); smartlist_free(exit_family); @@ -2803,9 +4308,9 @@ entry_guards_parse_state(or_state_t *state, int set, char **msg) } entry_guards = new_entry_guards; entry_guards_dirty = 0; - /* XXX022 hand new_entry_guards to this func, and move it up a + /* XXX023 hand new_entry_guards to this func, and move it up a * few lines, so we don't have to re-dirty it */ - if (remove_obsolete_entry_guards()) + if (remove_obsolete_entry_guards(now)) entry_guards_dirty = 1; } digestmap_free(added_by, _tor_free); @@ -2904,9 +4409,11 @@ entry_guards_update_state(or_state_t *state) * */ int getinfo_helper_entry_guards(control_connection_t *conn, - const char *question, char **answer) + const char *question, char **answer, + const char **errmsg) { - int use_long_names = conn->use_long_names; + (void) conn; + (void) errmsg; if (!strcmp(question,"entry-guards") || !strcmp(question,"helper-nodes")) { @@ -2915,12 +4422,13 @@ getinfo_helper_entry_guards(control_connection_t *conn, char nbuf[MAX_VERBOSE_NICKNAME_LEN+1]; if (!entry_guards) entry_guards = smartlist_create(); - SMARTLIST_FOREACH(entry_guards, entry_guard_t *, e, - { + SMARTLIST_FOREACH_BEGIN(entry_guards, entry_guard_t *, e) { size_t len = MAX_VERBOSE_NICKNAME_LEN+ISO_TIME_LEN+32; char *c = tor_malloc(len); const char *status = NULL; time_t when = 0; + routerinfo_t *ri; + if (!e->made_contact) { status = "never-connected"; } else if (e->bad_since) { @@ -2929,19 +4437,17 @@ getinfo_helper_entry_guards(control_connection_t *conn, } else { status = "up"; } - if (use_long_names) { - routerinfo_t *ri = router_get_by_digest(e->identity); - if (ri) { - router_get_verbose_nickname(nbuf, ri); - } else { - nbuf[0] = '$'; - base16_encode(nbuf+1, sizeof(nbuf)-1, e->identity, DIGEST_LEN); - /* e->nickname field is not very reliable if we don't know about - * this router any longer; don't include it. */ - } + + ri = router_get_by_digest(e->identity); + if (ri) { + router_get_verbose_nickname(nbuf, ri); } else { - base16_encode(nbuf, sizeof(nbuf), e->identity, DIGEST_LEN); + nbuf[0] = '$'; + base16_encode(nbuf+1, sizeof(nbuf)-1, e->identity, DIGEST_LEN); + /* e->nickname field is not very reliable if we don't know about + * this router any longer; don't include it. */ } + if (when) { format_iso_time(tbuf, when); tor_snprintf(c, len, "%s %s %s\n", nbuf, status, tbuf); @@ -2949,7 +4455,7 @@ getinfo_helper_entry_guards(control_connection_t *conn, tor_snprintf(c, len, "%s %s\n", nbuf, status); } smartlist_add(sl, c); - }); + } SMARTLIST_FOREACH_END(e); *answer = smartlist_join_strings(sl, "", 0, NULL); SMARTLIST_FOREACH(sl, char *, c, tor_free(c)); smartlist_free(sl); @@ -2990,29 +4496,56 @@ clear_bridge_list(void) * (either by comparing keys if possible, else by comparing addr/port). * Else return NULL. */ static bridge_info_t * -routerinfo_get_configured_bridge(routerinfo_t *ri) +get_configured_bridge_by_addr_port_digest(tor_addr_t *addr, uint16_t port, + const char *digest) { if (!bridge_list) return NULL; SMARTLIST_FOREACH_BEGIN(bridge_list, bridge_info_t *, bridge) { if (tor_digest_is_zero(bridge->identity) && - tor_addr_eq_ipv4h(&bridge->addr, ri->addr) && - bridge->port == ri->or_port) + !tor_addr_compare(&bridge->addr, addr, CMP_EXACT) && + bridge->port == port) return bridge; - if (tor_memeq(bridge->identity, ri->cache_info.identity_digest, - DIGEST_LEN)) + if (tor_memeq(bridge->identity, digest, DIGEST_LEN)) return bridge; } SMARTLIST_FOREACH_END(bridge); return NULL; } +/** Wrapper around get_configured_bridge_by_addr_port_digest() to look + * it up via router descriptor <b>ri</b>. */ +static bridge_info_t * +get_configured_bridge_by_routerinfo(routerinfo_t *ri) +{ + tor_addr_t addr; + tor_addr_from_ipv4h(&addr, ri->addr); + return get_configured_bridge_by_addr_port_digest(&addr, + ri->or_port, ri->cache_info.identity_digest); +} + /** Return 1 if <b>ri</b> is one of our known bridges, else 0. */ int routerinfo_is_a_configured_bridge(routerinfo_t *ri) { - return routerinfo_get_configured_bridge(ri) ? 1 : 0; + return get_configured_bridge_by_routerinfo(ri) ? 1 : 0; +} + +/** We made a connection to a router at <b>addr</b>:<b>port</b> + * without knowing its digest. Its digest turned out to be <b>digest</b>. + * If it was a bridge, and we still don't know its digest, record it. + */ +void +learned_router_identity(tor_addr_t *addr, uint16_t port, const char *digest) +{ + bridge_info_t *bridge = + get_configured_bridge_by_addr_port_digest(addr, port, digest); + if (bridge && tor_digest_is_zero(bridge->identity)) { + memcpy(bridge->identity, digest, DIGEST_LEN); + log_notice(LD_DIR, "Learned fingerprint %s for bridge %s:%d", + hex_str(digest, DIGEST_LEN), fmt_addr(addr), port); + } } /** Remember a new bridge at <b>addr</b>:<b>port</b>. If <b>digest</b> @@ -3031,6 +4564,24 @@ bridge_add_from_config(const tor_addr_t *addr, uint16_t port, char *digest) smartlist_add(bridge_list, b); } +/** Return true iff <b>routerset</b> contains the bridge <b>bridge</b>. */ +static int +routerset_contains_bridge(const routerset_t *routerset, + const bridge_info_t *bridge) +{ + int result; + extend_info_t *extinfo; + tor_assert(bridge); + if (!routerset) + return 0; + + extinfo = extend_info_alloc( + NULL, bridge->identity, NULL, &bridge->addr, bridge->port); + result = routerset_contains_extendinfo(routerset, extinfo); + extend_info_free(extinfo); + return result; +} + /** If <b>digest</b> is one of our known bridges, return it. */ static bridge_info_t * find_bridge_by_digest(const char *digest) @@ -3043,12 +4594,12 @@ find_bridge_by_digest(const char *digest) return NULL; } -/** We need to ask <b>bridge</b> for its server descriptor. <b>address</b> - * is a helpful string describing this bridge. */ +/** We need to ask <b>bridge</b> for its server descriptor. */ static void launch_direct_bridge_descriptor_fetch(bridge_info_t *bridge) { char *address; + or_options_t *options = get_options(); if (connection_get_by_type_addr_port_purpose( CONN_TYPE_DIR, &bridge->addr, bridge->port, @@ -3056,6 +4607,13 @@ launch_direct_bridge_descriptor_fetch(bridge_info_t *bridge) return; /* it's already on the way */ address = tor_dup_addr(&bridge->addr); + if (routerset_contains_bridge(options->ExcludeNodes, bridge)) { + download_status_mark_impossible(&bridge->fetch_status); + log_warn(LD_APP, "Not using bridge at %s: it is in ExcludeNodes.", + safe_str_client(fmt_addr(&bridge->addr))); + return; + } + directory_initiate_command(address, &bridge->addr, bridge->port, 0, 0, /* does not matter */ @@ -3082,9 +4640,8 @@ retry_bridge_descriptor_fetch_directly(const char *digest) * descriptor, fetch a new copy of its descriptor -- either directly * from the bridge or via a bridge authority. */ void -fetch_bridge_descriptors(time_t now) +fetch_bridge_descriptors(or_options_t *options, time_t now) { - or_options_t *options = get_options(); int num_bridge_auths = get_n_authorities(BRIDGE_AUTHORITY); int ask_bridge_directly; int can_use_bridge_authority; @@ -3097,6 +4654,12 @@ fetch_bridge_descriptors(time_t now) if (!download_status_is_ready(&bridge->fetch_status, now, IMPOSSIBLE_TO_DOWNLOAD)) continue; /* don't bother, no need to retry yet */ + if (routerset_contains_bridge(options->ExcludeNodes, bridge)) { + download_status_mark_impossible(&bridge->fetch_status); + log_warn(LD_APP, "Not using bridge at %s: it is in ExcludeNodes.", + safe_str_client(fmt_addr(&bridge->addr))); + continue; + } /* schedule another fetch as if this one will fail, in case it does */ download_status_failed(&bridge->fetch_status, 0); @@ -3143,6 +4706,29 @@ fetch_bridge_descriptors(time_t now) SMARTLIST_FOREACH_END(bridge); } +/** If our <b>bridge</b> is configured to be a different address than + * the bridge gives in its routerinfo <b>ri</b>, rewrite the routerinfo + * we received to use the address we meant to use. Now we handle + * multihomed bridges better. + */ +static void +rewrite_routerinfo_address_for_bridge(bridge_info_t *bridge, routerinfo_t *ri) +{ + tor_addr_t addr; + tor_addr_from_ipv4h(&addr, ri->addr); + + if (!tor_addr_compare(&bridge->addr, &addr, CMP_EXACT) && + bridge->port == ri->or_port) + return; /* they match, so no need to do anything */ + + ri->addr = tor_addr_to_ipv4h(&bridge->addr); + tor_free(ri->address); + ri->address = tor_dup_ip(ri->addr); + ri->or_port = bridge->port; + log_info(LD_DIR, "Adjusted bridge '%s' to match configured address %s:%d.", + ri->nickname, ri->address, ri->or_port); +} + /** We just learned a descriptor for a bridge. See if that * digest is in our entry guard list, and add it if not. */ void @@ -3152,7 +4738,7 @@ learned_bridge_descriptor(routerinfo_t *ri, int from_cache) tor_assert(ri->purpose == ROUTER_PURPOSE_BRIDGE); if (get_options()->UseBridges) { int first = !any_bridge_descriptors_known(); - bridge_info_t *bridge = routerinfo_get_configured_bridge(ri); + bridge_info_t *bridge = get_configured_bridge_by_routerinfo(ri); time_t now = time(NULL); ri->is_running = 1; @@ -3161,6 +4747,8 @@ learned_bridge_descriptor(routerinfo_t *ri, int from_cache) if (!from_cache) download_status_reset(&bridge->fetch_status); + rewrite_routerinfo_address_for_bridge(bridge, ri); + add_an_entry_guard(ri, 1); log_notice(LD_DIR, "new bridge descriptor '%s' (%s)", ri->nickname, from_cache ? "cached" : "fresh"); @@ -3209,26 +4797,38 @@ any_pending_bridge_descriptor_fetches(void) return 0; } -/** Return 1 if we have at least one descriptor for a bridge and - * all descriptors we know are down. Else return 0. If <b>act</b> is - * 1, then mark the down bridges up; else just observe and report. */ +/** Return 1 if we have at least one descriptor for an entry guard + * (bridge or member of EntryNodes) and all descriptors we know are + * down. Else return 0. If <b>act</b> is 1, then mark the down guards + * up; else just observe and report. */ static int -bridges_retry_helper(int act) +entries_retry_helper(or_options_t *options, int act) { routerinfo_t *ri; int any_known = 0; int any_running = 0; + int purpose = options->UseBridges ? + ROUTER_PURPOSE_BRIDGE : ROUTER_PURPOSE_GENERAL; if (!entry_guards) entry_guards = smartlist_create(); SMARTLIST_FOREACH(entry_guards, entry_guard_t *, e, { ri = router_get_by_digest(e->identity); - if (ri && ri->purpose == ROUTER_PURPOSE_BRIDGE) { + if (ri && ri->purpose == purpose) { any_known = 1; if (ri->is_running) - any_running = 1; /* some bridge is both known and running */ - else if (act) { /* mark it for retry */ - ri->is_running = 1; + any_running = 1; /* some entry is both known and running */ + else if (act) { + /* Mark all current connections to this OR as unhealthy, since + * otherwise there could be one that started 30 seconds + * ago, and in 30 seconds it will time out, causing us to mark + * the node down and undermine the retry attempt. We mark even + * the established conns, since if the network just came back + * we'll want to attach circuits to fresh conns. */ + connection_or_set_bad_connections(ri->cache_info.identity_digest, 1); + + /* mark this entry node for retry */ + router_set_status(ri->cache_info.identity_digest, 1); e->can_retry = 1; e->bad_since = 0; } @@ -3239,19 +4839,21 @@ bridges_retry_helper(int act) return any_known && !any_running; } -/** Do we know any descriptors for our bridges, and are they all - * down? */ +/** Do we know any descriptors for our bridges / entrynodes, and are + * all the ones we have descriptors for down? */ int -bridges_known_but_down(void) +entries_known_but_down(or_options_t *options) { - return bridges_retry_helper(0); + tor_assert(entry_list_is_constrained(options)); + return entries_retry_helper(options, 0); } -/** Mark all down known bridges up. */ +/** Mark all down known bridges / entrynodes up. */ void -bridges_retry_all(void) +entries_retry_all(or_options_t *options) { - bridges_retry_helper(1); + tor_assert(entry_list_is_constrained(options)); + entries_retry_helper(options, 1); } /** Release all storage held by the list of entry guards and related diff --git a/src/or/circuitbuild.h b/src/or/circuitbuild.h new file mode 100644 index 0000000000..af24931878 --- /dev/null +++ b/src/or/circuitbuild.h @@ -0,0 +1,127 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file circuitbuild.h + * \brief Header file for circuitbuild.c. + **/ + +#ifndef _TOR_CIRCUITBUILD_H +#define _TOR_CIRCUITBUILD_H + +char *circuit_list_path(origin_circuit_t *circ, int verbose); +char *circuit_list_path_for_controller(origin_circuit_t *circ); +void circuit_log_path(int severity, unsigned int domain, + origin_circuit_t *circ); +void circuit_rep_hist_note_result(origin_circuit_t *circ); +origin_circuit_t *origin_circuit_init(uint8_t purpose, int flags); +origin_circuit_t *circuit_establish_circuit(uint8_t purpose, + extend_info_t *exit, + int flags); +int circuit_handle_first_hop(origin_circuit_t *circ); +void circuit_n_conn_done(or_connection_t *or_conn, int status); +int inform_testing_reachability(void); +int circuit_timeout_want_to_count_circ(origin_circuit_t *circ); +int circuit_send_next_onion_skin(origin_circuit_t *circ); +void circuit_note_clock_jumped(int seconds_elapsed); +int circuit_extend(cell_t *cell, circuit_t *circ); +int circuit_init_cpath_crypto(crypt_path_t *cpath, const char *key_data, + int reverse); +int circuit_finish_handshake(origin_circuit_t *circ, uint8_t cell_type, + const uint8_t *reply); +int circuit_truncated(origin_circuit_t *circ, crypt_path_t *layer); +int onionskin_answer(or_circuit_t *circ, uint8_t cell_type, + const char *payload, const char *keys); +int circuit_all_predicted_ports_handled(time_t now, int *need_uptime, + int *need_capacity); + +int circuit_append_new_exit(origin_circuit_t *circ, extend_info_t *info); +int circuit_extend_to_new_exit(origin_circuit_t *circ, extend_info_t *info); +void onion_append_to_cpath(crypt_path_t **head_ptr, crypt_path_t *new_hop); +extend_info_t *extend_info_alloc(const char *nickname, const char *digest, + crypto_pk_env_t *onion_key, + const tor_addr_t *addr, uint16_t port); +extend_info_t *extend_info_from_router(routerinfo_t *r); +extend_info_t *extend_info_dup(extend_info_t *info); +void extend_info_free(extend_info_t *info); +routerinfo_t *build_state_get_exit_router(cpath_build_state_t *state); +const char *build_state_get_exit_nickname(cpath_build_state_t *state); + +void entry_guards_compute_status(or_options_t *options, time_t now); +int entry_guard_register_connect_status(const char *digest, int succeeded, + int mark_relay_status, time_t now); +void entry_nodes_should_be_added(void); +int entry_list_is_constrained(or_options_t *options); +routerinfo_t *choose_random_entry(cpath_build_state_t *state); +int entry_guards_parse_state(or_state_t *state, int set, char **msg); +void entry_guards_update_state(or_state_t *state); +int getinfo_helper_entry_guards(control_connection_t *conn, + const char *question, char **answer, + const char **errmsg); + +void clear_bridge_list(void); +int routerinfo_is_a_configured_bridge(routerinfo_t *ri); +void +learned_router_identity(tor_addr_t *addr, uint16_t port, const char *digest); +void bridge_add_from_config(const tor_addr_t *addr, uint16_t port, + char *digest); +void retry_bridge_descriptor_fetch_directly(const char *digest); +void fetch_bridge_descriptors(or_options_t *options, time_t now); +void learned_bridge_descriptor(routerinfo_t *ri, int from_cache); +int any_bridge_descriptors_known(void); +int any_pending_bridge_descriptor_fetches(void); +int entries_known_but_down(or_options_t *options); +void entries_retry_all(or_options_t *options); + +void entry_guards_free_all(void); + +extern circuit_build_times_t circ_times; +int circuit_build_times_enough_to_compute(circuit_build_times_t *cbt); +void circuit_build_times_update_state(circuit_build_times_t *cbt, + or_state_t *state); +int circuit_build_times_parse_state(circuit_build_times_t *cbt, + or_state_t *state); +void circuit_build_times_count_timeout(circuit_build_times_t *cbt, + int did_onehop); +int circuit_build_times_count_close(circuit_build_times_t *cbt, + int did_onehop, time_t start_time); +void circuit_build_times_set_timeout(circuit_build_times_t *cbt); +int circuit_build_times_add_time(circuit_build_times_t *cbt, + build_time_t time); +int circuit_build_times_needs_circuits(circuit_build_times_t *cbt); + +int circuit_build_times_needs_circuits_now(circuit_build_times_t *cbt); +void circuit_build_times_init(circuit_build_times_t *cbt); +void circuit_build_times_new_consensus_params(circuit_build_times_t *cbt, + networkstatus_t *ns); +double circuit_build_times_timeout_rate(const circuit_build_times_t *cbt); +double circuit_build_times_close_rate(const circuit_build_times_t *cbt); + +#ifdef CIRCUIT_PRIVATE +double circuit_build_times_calculate_timeout(circuit_build_times_t *cbt, + double quantile); +build_time_t circuit_build_times_generate_sample(circuit_build_times_t *cbt, + double q_lo, double q_hi); +void circuit_build_times_initial_alpha(circuit_build_times_t *cbt, + double quantile, double time_ms); +int circuit_build_times_update_alpha(circuit_build_times_t *cbt); +double circuit_build_times_cdf(circuit_build_times_t *cbt, double x); +void circuitbuild_running_unit_tests(void); +void circuit_build_times_reset(circuit_build_times_t *cbt); + +/* Network liveness functions */ +int circuit_build_times_network_check_changed(circuit_build_times_t *cbt); +#endif + +/* Network liveness functions */ +void circuit_build_times_network_is_live(circuit_build_times_t *cbt); +int circuit_build_times_network_check_live(circuit_build_times_t *cbt); +void circuit_build_times_network_circ_success(circuit_build_times_t *cbt); + +int circuit_build_times_get_bw_scale(networkstatus_t *ns); + +#endif + diff --git a/src/or/circuitlist.c b/src/or/circuitlist.c index 9cf331ecf2..8ec46186d9 100644 --- a/src/or/circuitlist.c +++ b/src/or/circuitlist.c @@ -10,6 +10,21 @@ **/ #include "or.h" +#include "circuitbuild.h" +#include "circuitlist.h" +#include "circuituse.h" +#include "connection.h" +#include "config.h" +#include "connection_edge.h" +#include "connection_or.h" +#include "control.h" +#include "networkstatus.h" +#include "onion.h" +#include "relay.h" +#include "rendclient.h" +#include "rendcommon.h" +#include "rephist.h" +#include "routerlist.h" #include "ht.h" /********* START VARIABLES **********/ @@ -352,6 +367,8 @@ circuit_purpose_to_controller_string(uint8_t purpose) case CIRCUIT_PURPOSE_TESTING: return "TESTING"; + case CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT: + return "MEASURE_TIMEOUT"; case CIRCUIT_PURPOSE_CONTROLLER: return "CONTROLLER"; @@ -361,13 +378,71 @@ circuit_purpose_to_controller_string(uint8_t purpose) } } +/** Return a human-readable string for the circuit purpose <b>purpose</b>. */ +const char * +circuit_purpose_to_string(uint8_t purpose) +{ + static char buf[32]; + + switch (purpose) + { + case CIRCUIT_PURPOSE_OR: + return "Circuit at relay"; + case CIRCUIT_PURPOSE_INTRO_POINT: + return "Acting as intro point"; + case CIRCUIT_PURPOSE_REND_POINT_WAITING: + return "Acting as rendevous (pending)"; + case CIRCUIT_PURPOSE_REND_ESTABLISHED: + return "Acting as rendevous (established)"; + case CIRCUIT_PURPOSE_C_GENERAL: + return "General-purpose client"; + case CIRCUIT_PURPOSE_C_INTRODUCING: + return "Hidden service client: Connecting to intro point"; + case CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT: + return "Hidden service client: Waiting for ack from intro point"; + case CIRCUIT_PURPOSE_C_INTRODUCE_ACKED: + return "Hidden service client: Received ack from intro point"; + case CIRCUIT_PURPOSE_C_ESTABLISH_REND: + return "Hidden service client: Establishing rendezvous point"; + case CIRCUIT_PURPOSE_C_REND_READY: + return "Hidden service client: Pending rendezvous point"; + case CIRCUIT_PURPOSE_C_REND_READY_INTRO_ACKED: + return "Hidden service client: Pending rendezvous point (ack received)"; + case CIRCUIT_PURPOSE_C_REND_JOINED: + return "Hidden service client: Active rendezvous point"; + case CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT: + return "Measuring circuit timeout"; + + case CIRCUIT_PURPOSE_S_ESTABLISH_INTRO: + return "Hidden service: Establishing introduction point"; + case CIRCUIT_PURPOSE_S_INTRO: + return "Hidden service: Introduction point"; + case CIRCUIT_PURPOSE_S_CONNECT_REND: + return "Hidden service: Connecting to rendezvous point"; + case CIRCUIT_PURPOSE_S_REND_JOINED: + return "Hidden service: Active rendezvous point"; + + case CIRCUIT_PURPOSE_TESTING: + return "Testing circuit"; + + case CIRCUIT_PURPOSE_CONTROLLER: + return "Circuit made by controller"; + + default: + tor_snprintf(buf, sizeof(buf), "UNKNOWN_%d", (int)purpose); + return buf; + } +} + /** Pick a reasonable package_window to start out for our circuits. * Originally this was hard-coded at 1000, but now the consensus votes * on the answer. See proposal 168. */ int32_t circuit_initial_package_window(void) { - int32_t num = networkstatus_get_param(NULL, "circwindow", CIRCWINDOW_START); + int32_t num = networkstatus_get_param(NULL, "circwindow", CIRCWINDOW_START, + CIRCWINDOW_START_MIN, + CIRCWINDOW_START_MAX); /* If the consensus tells us a negative number, we'd assert. */ if (num < 0) num = CIRCWINDOW_START; @@ -379,11 +454,17 @@ circuit_initial_package_window(void) static void init_circuit_base(circuit_t *circ) { - circ->timestamp_created = time(NULL); + tor_gettimeofday(&circ->timestamp_created); circ->package_window = circuit_initial_package_window(); circ->deliver_window = CIRCWINDOW_START; + /* Initialize the cell_ewma_t structure */ + circ->n_cell_ewma.last_adjusted_tick = cell_ewma_get_tick(); + circ->n_cell_ewma.cell_count = 0.0; + circ->n_cell_ewma.heap_index = -1; + circ->n_cell_ewma.is_for_p_conn = 0; + circuit_add(circ); } @@ -408,6 +489,8 @@ origin_circuit_new(void) init_circuit_base(TO_CIRCUIT(circ)); + circ_times.last_circ_at = approx_time(); + return circ; } @@ -429,6 +512,16 @@ or_circuit_new(circid_t p_circ_id, or_connection_t *p_conn) init_circuit_base(TO_CIRCUIT(circ)); + /* Initialize the cell_ewma_t structure */ + + /* Initialize the cell counts to 0 */ + circ->p_cell_ewma.cell_count = 0.0; + circ->p_cell_ewma.last_adjusted_tick = cell_ewma_get_tick(); + circ->p_cell_ewma.is_for_p_conn = 1; + + /* It's not in any heap yet. */ + circ->p_cell_ewma.heap_index = -1; + return circ; } @@ -439,39 +532,37 @@ circuit_free(circuit_t *circ) { void *mem; size_t memlen; - tor_assert(circ); + if (!circ) + return; + if (CIRCUIT_IS_ORIGIN(circ)) { origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ); mem = ocirc; memlen = sizeof(origin_circuit_t); tor_assert(circ->magic == ORIGIN_CIRCUIT_MAGIC); if (ocirc->build_state) { - if (ocirc->build_state->chosen_exit) extend_info_free(ocirc->build_state->chosen_exit); - if (ocirc->build_state->pending_final_cpath) circuit_free_cpath_node(ocirc->build_state->pending_final_cpath); } tor_free(ocirc->build_state); circuit_free_cpath(ocirc->cpath); - if (ocirc->intro_key) - crypto_free_pk_env(ocirc->intro_key); - if (ocirc->rend_data) - rend_data_free(ocirc->rend_data); + + crypto_free_pk_env(ocirc->intro_key); + rend_data_free(ocirc->rend_data); } else { or_circuit_t *ocirc = TO_OR_CIRCUIT(circ); + /* Remember cell statistics for this circuit before deallocating. */ + if (get_options()->CellStatistics) + rep_hist_buffer_stats_add_circ(circ, time(NULL)); mem = ocirc; memlen = sizeof(or_circuit_t); tor_assert(circ->magic == OR_CIRCUIT_MAGIC); - if (ocirc->p_crypto) - crypto_free_cipher_env(ocirc->p_crypto); - if (ocirc->p_digest) - crypto_free_digest_env(ocirc->p_digest); - if (ocirc->n_crypto) - crypto_free_cipher_env(ocirc->n_crypto); - if (ocirc->n_digest) - crypto_free_digest_env(ocirc->n_digest); + crypto_free_cipher_env(ocirc->p_crypto); + crypto_free_digest_env(ocirc->p_digest); + crypto_free_cipher_env(ocirc->n_crypto); + crypto_free_digest_env(ocirc->n_digest); if (ocirc->rend_splice) { or_circuit_t *other = ocirc->rend_splice; @@ -487,8 +578,7 @@ circuit_free(circuit_t *circ) cell_queue_clear(ô->p_conn_cells); } - if (circ->n_hop) - extend_info_free(circ->n_hop); + extend_info_free(circ->n_hop); tor_free(circ->n_conn_onionskin); /* Remove from map. */ @@ -498,7 +588,7 @@ circuit_free(circuit_t *circ) * "active" checks will be violated. */ cell_queue_clear(&circ->n_conn_cells); - memset(circ, 0xAA, memlen); /* poison memory */ + memset(mem, 0xAA, memlen); /* poison memory */ tor_free(mem); } @@ -541,10 +631,10 @@ circuit_free_all(void) circuit_free(global_circuitlist); global_circuitlist = next; } - if (circuits_pending_or_conns) { - smartlist_free(circuits_pending_or_conns); - circuits_pending_or_conns = NULL; - } + + smartlist_free(circuits_pending_or_conns); + circuits_pending_or_conns = NULL; + HT_CLEAR(orconn_circid_map, &orconn_circid_circuit_map); } @@ -552,18 +642,15 @@ circuit_free_all(void) static void circuit_free_cpath_node(crypt_path_t *victim) { - if (victim->f_crypto) - crypto_free_cipher_env(victim->f_crypto); - if (victim->b_crypto) - crypto_free_cipher_env(victim->b_crypto); - if (victim->f_digest) - crypto_free_digest_env(victim->f_digest); - if (victim->b_digest) - crypto_free_digest_env(victim->b_digest); - if (victim->dh_handshake_state) - crypto_dh_free(victim->dh_handshake_state); - if (victim->extend_info) - extend_info_free(victim->extend_info); + if (!victim) + return; + + crypto_free_cipher_env(victim->f_crypto); + crypto_free_cipher_env(victim->b_crypto); + crypto_free_digest_env(victim->f_digest); + crypto_free_digest_env(victim->b_digest); + crypto_dh_free(victim->dh_handshake_state); + extend_info_free(victim->extend_info); memset(victim, 0xBB, sizeof(crypt_path_t)); /* poison memory */ tor_free(victim); @@ -577,9 +664,10 @@ circuit_dump_details(int severity, circuit_t *circ, int conn_array_index, const char *type, int this_circid, int other_circid) { log(severity, LD_CIRC, "Conn %d has %s circuit: circID %d (other side %d), " - "state %d (%s), born %d:", + "state %d (%s), born %ld:", conn_array_index, type, this_circid, other_circid, circ->state, - circuit_state_to_string(circ->state), (int)circ->timestamp_created); + circuit_state_to_string(circ->state), + (long)circ->timestamp_created.tv_sec); if (CIRCUIT_IS_ORIGIN(circ)) { /* circ starts at this node */ circuit_log_path(severity, LD_CIRC, TO_ORIGIN_CIRCUIT(circ)); } @@ -891,6 +979,11 @@ circuit_find_to_cannibalize(uint8_t purpose, extend_info_t *info, int need_uptime = (flags & CIRCLAUNCH_NEED_UPTIME) != 0; int need_capacity = (flags & CIRCLAUNCH_NEED_CAPACITY) != 0; int internal = (flags & CIRCLAUNCH_IS_INTERNAL) != 0; + or_options_t *options = get_options(); + + /* Make sure we're not trying to create a onehop circ by + * cannibalization. */ + tor_assert(!(flags & CIRCLAUNCH_ONEHOP_TUNNEL)); log_debug(LD_CIRC, "Hunting for a circ to cannibalize: purpose %d, uptime %d, " @@ -907,7 +1000,8 @@ circuit_find_to_cannibalize(uint8_t purpose, extend_info_t *info, if ((!need_uptime || circ->build_state->need_uptime) && (!need_capacity || circ->build_state->need_capacity) && (internal == circ->build_state->is_internal) && - circ->remaining_relay_early_cells) { + circ->remaining_relay_early_cells && + !circ->build_state->onehop_tunnel) { if (info) { /* need to make sure we don't duplicate hops */ crypt_path_t *hop = circ->cpath; @@ -924,6 +1018,19 @@ circuit_find_to_cannibalize(uint8_t purpose, extend_info_t *info, hop=hop->next; } while (hop!=circ->cpath); } + if (options->ExcludeNodes) { + /* Make sure no existing nodes in the circuit are excluded for + * general use. (This may be possible if StrictNodes is 0, and we + * thought we needed to use an otherwise excluded node for, say, a + * directory operation.) */ + crypt_path_t *hop = circ->cpath; + do { + if (routerset_contains_extendinfo(options->ExcludeNodes, + hop->extend_info)) + goto next; + hop = hop->next; + } while (hop != circ->cpath); + } if (!best || (best->build_state->need_uptime && !need_uptime)) best = circ; next: ; @@ -986,6 +1093,7 @@ circuit_mark_all_unused_circs(void) * This is useful for letting the user change pseudonyms, so new * streams will not be linkable to old streams. */ +/* XXX023 this is a bad name for what this function does */ void circuit_expire_all_dirty_circs(void) { @@ -996,6 +1104,8 @@ circuit_expire_all_dirty_circs(void) if (CIRCUIT_IS_ORIGIN(circ) && !circ->marked_for_close && circ->timestamp_dirty) + /* XXXX023 This is a screwed-up way to say "This is too dirty + * for new circuits. */ circ->timestamp_dirty -= options->MaxCircuitDirtiness; } } @@ -1083,14 +1193,16 @@ _circuit_mark_for_close(circuit_t *circ, int reason, int line, tor_assert(ocirc->rend_data); /* treat this like getting a nack from it */ log_info(LD_REND, "Failed intro circ %s to %s (awaiting ack). " - "Removing from descriptor.", - safe_str(ocirc->rend_data->onion_address), - safe_str(build_state_get_exit_nickname(ocirc->build_state))); + "Removing from descriptor.", + safe_str_client(ocirc->rend_data->onion_address), + safe_str_client(build_state_get_exit_nickname(ocirc->build_state))); rend_client_remove_intro_point(ocirc->build_state->chosen_exit, ocirc->rend_data); } - if (circ->n_conn) + if (circ->n_conn) { + circuit_clear_cell_queue(circ, circ->n_conn); connection_or_send_destroy(circ->n_circ_id, circ->n_conn, reason); + } if (! CIRCUIT_IS_ORIGIN(circ)) { or_circuit_t *or_circ = TO_OR_CIRCUIT(circ); @@ -1114,8 +1226,10 @@ _circuit_mark_for_close(circuit_t *circ, int reason, int line, conn->on_circuit = NULL; } - if (or_circ->p_conn) + if (or_circ->p_conn) { + circuit_clear_cell_queue(circ, or_circ->p_conn); connection_or_send_destroy(or_circ->p_circ_id, or_circ->p_conn, reason); + } } else { origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ); edge_connection_t *conn; @@ -1236,11 +1350,6 @@ assert_circuit_ok(const circuit_t *c) tor_assert(c == c2); } } -#if 0 /* false now that rendezvous exits are attached to p_streams */ - if (origin_circ) - for (conn = origin_circ->p_streams; conn; conn = conn->next_stream) - tor_assert(conn->_base.type == CONN_TYPE_AP); -#endif if (or_circ) for (conn = or_circ->n_streams; conn; conn = conn->next_stream) tor_assert(conn->_base.type == CONN_TYPE_EXIT); diff --git a/src/or/circuitlist.h b/src/or/circuitlist.h new file mode 100644 index 0000000000..7b01ca3ae2 --- /dev/null +++ b/src/or/circuitlist.h @@ -0,0 +1,61 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file circuitlist.h + * \brief Header file for circuitlist.c. + **/ + +#ifndef _TOR_CIRCUITLIST_H +#define _TOR_CIRCUITLIST_H + +circuit_t * _circuit_get_global_list(void); +const char *circuit_state_to_string(int state); +const char *circuit_purpose_to_controller_string(uint8_t purpose); +const char *circuit_purpose_to_string(uint8_t purpose); +void circuit_dump_by_conn(connection_t *conn, int severity); +void circuit_set_p_circid_orconn(or_circuit_t *circ, circid_t id, + or_connection_t *conn); +void circuit_set_n_circid_orconn(circuit_t *circ, circid_t id, + or_connection_t *conn); +void circuit_set_state(circuit_t *circ, uint8_t state); +void circuit_close_all_marked(void); +int32_t circuit_initial_package_window(void); +origin_circuit_t *origin_circuit_new(void); +or_circuit_t *or_circuit_new(circid_t p_circ_id, or_connection_t *p_conn); +circuit_t *circuit_get_by_circid_orconn(circid_t circ_id, + or_connection_t *conn); +int circuit_id_in_use_on_orconn(circid_t circ_id, or_connection_t *conn); +circuit_t *circuit_get_by_edge_conn(edge_connection_t *conn); +void circuit_unlink_all_from_or_conn(or_connection_t *conn, int reason); +origin_circuit_t *circuit_get_by_global_id(uint32_t id); +origin_circuit_t *circuit_get_by_rend_query_and_purpose(const char *rend_query, + uint8_t purpose); +origin_circuit_t *circuit_get_next_by_pk_and_purpose(origin_circuit_t *start, + const char *digest, uint8_t purpose); +or_circuit_t *circuit_get_rendezvous(const char *cookie); +or_circuit_t *circuit_get_intro_point(const char *digest); +origin_circuit_t *circuit_find_to_cannibalize(uint8_t purpose, + extend_info_t *info, int flags); +void circuit_mark_all_unused_circs(void); +void circuit_expire_all_dirty_circs(void); +void _circuit_mark_for_close(circuit_t *circ, int reason, + int line, const char *file); +int circuit_get_cpath_len(origin_circuit_t *circ); +crypt_path_t *circuit_get_cpath_hop(origin_circuit_t *circ, int hopnum); +void circuit_get_all_pending_on_or_conn(smartlist_t *out, + or_connection_t *or_conn); +int circuit_count_pending_on_or_conn(or_connection_t *or_conn); + +#define circuit_mark_for_close(c, reason) \ + _circuit_mark_for_close((c), (reason), __LINE__, _SHORT_FILE_) + +void assert_cpath_layer_ok(const crypt_path_t *cp); +void assert_circuit_ok(const circuit_t *c); +void circuit_free_all(void); + +#endif + diff --git a/src/or/circuituse.c b/src/or/circuituse.c index 41c1899c3a..9feb4e1c4c 100644 --- a/src/or/circuituse.c +++ b/src/or/circuituse.c @@ -10,6 +10,20 @@ **/ #include "or.h" +#include "circuitbuild.h" +#include "circuitlist.h" +#include "circuituse.h" +#include "config.h" +#include "connection.h" +#include "connection_edge.h" +#include "control.h" +#include "policies.h" +#include "rendclient.h" +#include "rendcommon.h" +#include "rendservice.h" +#include "rephist.h" +#include "router.h" +#include "routerlist.h" /********* START VARIABLES **********/ @@ -17,7 +31,7 @@ extern circuit_t *global_circuitlist; /* from circuitlist.c */ /********* END VARIABLES ************/ -static void circuit_expire_old_circuits_clientside(time_t now); +static void circuit_expire_old_circuits_clientside(void); static void circuit_increment_failure_count(void); /** Return 1 if <b>circ</b> could be returned by circuit_get_best(). @@ -148,10 +162,14 @@ circuit_is_better(circuit_t *a, circuit_t *b, uint8_t purpose) return 1; } else { if (a->timestamp_dirty || - a->timestamp_created > b->timestamp_created) + timercmp(&a->timestamp_created, &b->timestamp_created, >)) return 1; if (CIRCUIT_IS_ORIGIN(b) && TO_ORIGIN_CIRCUIT(b)->build_state->is_internal) + /* XXX023 what the heck is this internal thing doing here. I + * think we can get rid of it. circuit_is_acceptable() already + * makes sure that is_internal is exactly what we need it to + * be. -RD */ return 1; } break; @@ -190,7 +208,7 @@ circuit_get_best(edge_connection_t *conn, int must_be_open, uint8_t purpose, int need_uptime, int need_internal) { circuit_t *circ, *best=NULL; - time_t now = time(NULL); + struct timeval now; int intro_going_on_but_too_old = 0; tor_assert(conn); @@ -199,17 +217,16 @@ circuit_get_best(edge_connection_t *conn, int must_be_open, uint8_t purpose, purpose == CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT || purpose == CIRCUIT_PURPOSE_C_REND_JOINED); + tor_gettimeofday(&now); + for (circ=global_circuitlist;circ;circ = circ->next) { if (!circuit_is_acceptable(circ,conn,must_be_open,purpose, - need_uptime,need_internal,now)) + need_uptime,need_internal,now.tv_sec)) continue; -/* XXX022 make this 15 be a function of circuit finishing times we've - * seen lately, a la Fallon Chen's GSoC work -RD */ -#define REND_PARALLEL_INTRO_DELAY 15 if (purpose == CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT && - !must_be_open && circ->state != CIRCUIT_STATE_OPEN && - circ->timestamp_created + REND_PARALLEL_INTRO_DELAY < now) { + !must_be_open && circ->state != CIRCUIT_STATE_OPEN && + tv_mdiff(&now, &circ->timestamp_created) > circ_times.timeout_ms) { intro_going_on_but_too_old = 1; continue; } @@ -229,50 +246,73 @@ circuit_get_best(edge_connection_t *conn, int must_be_open, uint8_t purpose, return best ? TO_ORIGIN_CIRCUIT(best) : NULL; } +#if 0 /** Check whether, according to the policies in <b>options</b>, the * circuit <b>circ</b> makes sense. */ -/* XXXX currently only checks Exclude{Exit}Nodes. It should check more. */ +/* XXXX currently only checks Exclude{Exit}Nodes; it should check more. + * Also, it doesn't have the right definition of an exit circuit. Also, + * it's never called. */ int circuit_conforms_to_options(const origin_circuit_t *circ, const or_options_t *options) { const crypt_path_t *cpath, *cpath_next = NULL; - for (cpath = circ->cpath; cpath && cpath_next != circ->cpath; - cpath = cpath_next) { + /* first check if it includes any excluded nodes */ + for (cpath = circ->cpath; cpath_next != circ->cpath; cpath = cpath_next) { cpath_next = cpath->next; - if (routerset_contains_extendinfo(options->ExcludeNodes, cpath->extend_info)) return 0; + } - if (cpath->next == circ->cpath) { - /* This is apparently the exit node. */ + /* then consider the final hop */ + if (routerset_contains_extendinfo(options->ExcludeExitNodes, + circ->cpath->prev->extend_info)) + return 0; - if (routerset_contains_extendinfo(options->ExcludeExitNodes, - cpath->extend_info)) - return 0; - } - } return 1; } +#endif /** Close all circuits that start at us, aren't open, and were born * at least CircuitBuildTimeout seconds ago. */ void -circuit_expire_building(time_t now) +circuit_expire_building(void) { - circuit_t *victim, *circ = global_circuitlist; - time_t general_cutoff = now - get_options()->CircuitBuildTimeout; - time_t begindir_cutoff = now - get_options()->CircuitBuildTimeout/2; - time_t introcirc_cutoff = begindir_cutoff; + circuit_t *victim, *next_circ = global_circuitlist; + /* circ_times.timeout_ms and circ_times.close_ms are from + * circuit_build_times_get_initial_timeout() if we haven't computed + * custom timeouts yet */ + struct timeval general_cutoff, begindir_cutoff, fourhop_cutoff, + cannibalize_cutoff, close_cutoff, extremely_old_cutoff; + struct timeval now; + struct timeval introcirc_cutoff; cpath_build_state_t *build_state; - while (circ) { - time_t cutoff; - victim = circ; - circ = circ->next; + tor_gettimeofday(&now); +#define SET_CUTOFF(target, msec) do { \ + long ms = tor_lround(msec); \ + struct timeval diff; \ + diff.tv_sec = ms / 1000; \ + diff.tv_usec = (int)((ms % 1000) * 1000); \ + timersub(&now, &diff, &target); \ + } while (0) + + SET_CUTOFF(general_cutoff, circ_times.timeout_ms); + SET_CUTOFF(begindir_cutoff, circ_times.timeout_ms / 2.0); + SET_CUTOFF(fourhop_cutoff, circ_times.timeout_ms * (4/3.0)); + SET_CUTOFF(cannibalize_cutoff, circ_times.timeout_ms / 2.0); + SET_CUTOFF(close_cutoff, circ_times.close_ms); + SET_CUTOFF(extremely_old_cutoff, circ_times.close_ms*2 + 1000); + + introcirc_cutoff = begindir_cutoff; + + while (next_circ) { + struct timeval cutoff; + victim = next_circ; + next_circ = next_circ->next; if (!CIRCUIT_IS_ORIGIN(victim) || /* didn't originate here */ victim->marked_for_close) /* don't mess with marked circs */ continue; @@ -280,11 +320,19 @@ circuit_expire_building(time_t now) build_state = TO_ORIGIN_CIRCUIT(victim)->build_state; if (build_state && build_state->onehop_tunnel) cutoff = begindir_cutoff; + else if (build_state && build_state->desired_path_len == 4 + && !TO_ORIGIN_CIRCUIT(victim)->has_opened) + cutoff = fourhop_cutoff; + else if (TO_ORIGIN_CIRCUIT(victim)->has_opened) + cutoff = cannibalize_cutoff; else if (victim->purpose == CIRCUIT_PURPOSE_C_INTRODUCING) cutoff = introcirc_cutoff; + else if (victim->purpose == CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT) + cutoff = close_cutoff; else cutoff = general_cutoff; - if (victim->timestamp_created > cutoff) + + if (timercmp(&victim->timestamp_created, &cutoff, >)) continue; /* it's still young, leave it alone */ #if 0 @@ -330,7 +378,7 @@ circuit_expire_building(time_t now) * because that's set when they switch purposes */ if (TO_ORIGIN_CIRCUIT(victim)->rend_data || - victim->timestamp_dirty > cutoff) + victim->timestamp_dirty > cutoff.tv_sec) continue; break; case CIRCUIT_PURPOSE_C_REND_READY_INTRO_ACKED: @@ -339,10 +387,60 @@ circuit_expire_building(time_t now) * make an introduction attempt. so timestamp_dirty * will reflect the time since the last attempt. */ - if (victim->timestamp_dirty > cutoff) + if (victim->timestamp_dirty > cutoff.tv_sec) continue; break; } + } else { /* circuit not open, consider recording failure as timeout */ + int first_hop_succeeded = TO_ORIGIN_CIRCUIT(victim)->cpath && + TO_ORIGIN_CIRCUIT(victim)->cpath->state == CPATH_STATE_OPEN; + + if (TO_ORIGIN_CIRCUIT(victim)->p_streams != NULL) { + log_warn(LD_BUG, "Circuit %d (purpose %d, %s) has timed out, " + "yet has attached streams!", + TO_ORIGIN_CIRCUIT(victim)->global_identifier, + victim->purpose, + circuit_purpose_to_string(victim->purpose)); + tor_fragile_assert(); + continue; + } + + if (circuit_timeout_want_to_count_circ(TO_ORIGIN_CIRCUIT(victim)) && + circuit_build_times_enough_to_compute(&circ_times)) { + /* Circuits are allowed to last longer for measurement. + * Switch their purpose and wait. */ + if (victim->purpose != CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT) { + control_event_circuit_status(TO_ORIGIN_CIRCUIT(victim), + CIRC_EVENT_FAILED, + END_CIRC_REASON_TIMEOUT); + victim->purpose = CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT; + /* Record this failure to check for too many timeouts + * in a row. This function does not record a time value yet + * (we do that later); it only counts the fact that we did + * have a timeout. */ + circuit_build_times_count_timeout(&circ_times, + first_hop_succeeded); + continue; + } + + /* + * If the circuit build time is much greater than we would have cut + * it off at, we probably had a suspend event along this codepath, + * and we should discard the value. + */ + if (timercmp(&victim->timestamp_created, &extremely_old_cutoff, <)) { + log_notice(LD_CIRC, + "Extremely large value for circuit build timeout: %lds. " + "Assuming clock jump. Purpose %d (%s)", + (long)(now.tv_sec - victim->timestamp_created.tv_sec), + victim->purpose, + circuit_purpose_to_string(victim->purpose)); + } else if (circuit_build_times_count_close(&circ_times, + first_hop_succeeded, + victim->timestamp_created.tv_sec)) { + circuit_build_times_set_timeout(&circ_times); + } + } } if (victim->n_conn) @@ -357,7 +455,10 @@ circuit_expire_building(time_t now) circuit_state_to_string(victim->state), victim->purpose); circuit_log_path(LOG_INFO,LD_CIRC,TO_ORIGIN_CIRCUIT(victim)); - circuit_mark_for_close(victim, END_CIRC_REASON_TIMEOUT); + if (victim->purpose == CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT) + circuit_mark_for_close(victim, END_CIRC_REASON_MEASUREMENT_EXPIRED); + else + circuit_mark_for_close(victim, END_CIRC_REASON_TIMEOUT); } } @@ -431,11 +532,11 @@ circuit_stream_is_being_handled(edge_connection_t *conn, } /** Don't keep more than this many unused open circuits around. */ -#define MAX_UNUSED_OPEN_CIRCUITS 12 +#define MAX_UNUSED_OPEN_CIRCUITS 14 /** Figure out how many circuits we have open that are clean. Make * sure it's enough for all the upcoming behaviors we predict we'll have. - * But if we have too many, close the not-so-useful ones. + * But put an upper bound on the total number of circuits. */ static void circuit_predict_and_launch_new(void) @@ -517,6 +618,19 @@ circuit_predict_and_launch_new(void) circuit_launch_by_router(CIRCUIT_PURPOSE_C_GENERAL, NULL, flags); return; } + + /* Finally, check to see if we still need more circuits to learn + * a good build timeout. But if we're close to our max number we + * want, don't do another -- we want to leave a few slots open so + * we can still build circuits preemptively as needed. */ + if (num < MAX_UNUSED_OPEN_CIRCUITS-2 && + circuit_build_times_needs_circuits_now(&circ_times)) { + flags = CIRCLAUNCH_NEED_CAPACITY; + log_info(LD_CIRC, + "Have %d clean circs need another buildtime test circ.", num); + circuit_launch_by_router(CIRCUIT_PURPOSE_C_GENERAL, NULL, flags); + return; + } } /** Build a new test circuit every 5 minutes */ @@ -544,7 +658,7 @@ circuit_build_needed_circs(time_t now) time_to_new_circuit = now + options->NewCircuitPeriod; if (proxy_mode(get_options())) addressmap_clean(now); - circuit_expire_old_circuits_clientside(now); + circuit_expire_old_circuits_clientside(); #if 0 /* disable for now, until predict-and-launch-new can cull leftovers */ circ = circuit_get_youngest_clean_open(CIRCUIT_PURPOSE_C_GENERAL); @@ -624,37 +738,77 @@ circuit_detach_stream(circuit_t *circ, edge_connection_t *conn) tor_fragile_assert(); } +/** If we haven't yet decided on a good timeout value for circuit + * building, we close idles circuits aggressively so we can get more + * data points. */ +#define IDLE_TIMEOUT_WHILE_LEARNING (10*60) + /** Find each circuit that has been unused for too long, or dirty * for too long and has no streams on it: mark it for close. */ static void -circuit_expire_old_circuits_clientside(time_t now) +circuit_expire_old_circuits_clientside(void) { circuit_t *circ; - time_t cutoff = now - get_options()->CircuitIdleTimeout; + struct timeval cutoff, now; + + tor_gettimeofday(&now); + cutoff = now; + + if (circuit_build_times_needs_circuits(&circ_times)) { + /* Circuits should be shorter lived if we need more of them + * for learning a good build timeout */ + cutoff.tv_sec -= IDLE_TIMEOUT_WHILE_LEARNING; + } else { + cutoff.tv_sec -= get_options()->CircuitIdleTimeout; + } for (circ = global_circuitlist; circ; circ = circ->next) { - if (circ->marked_for_close || ! CIRCUIT_IS_ORIGIN(circ)) + if (circ->marked_for_close || !CIRCUIT_IS_ORIGIN(circ)) continue; /* If the circuit has been dirty for too long, and there are no streams * on it, mark it for close. */ if (circ->timestamp_dirty && - circ->timestamp_dirty + get_options()->MaxCircuitDirtiness < now && + circ->timestamp_dirty + get_options()->MaxCircuitDirtiness < + now.tv_sec && !TO_ORIGIN_CIRCUIT(circ)->p_streams /* nothing attached */ ) { - log_debug(LD_CIRC, "Closing n_circ_id %d (dirty %d secs ago, " + log_debug(LD_CIRC, "Closing n_circ_id %d (dirty %ld sec ago, " "purpose %d)", - circ->n_circ_id, (int)(now - circ->timestamp_dirty), + circ->n_circ_id, (long)(now.tv_sec - circ->timestamp_dirty), circ->purpose); circuit_mark_for_close(circ, END_CIRC_REASON_FINISHED); - } else if (!circ->timestamp_dirty && - circ->state == CIRCUIT_STATE_OPEN && - circ->purpose == CIRCUIT_PURPOSE_C_GENERAL) { - if (circ->timestamp_created < cutoff) { - log_debug(LD_CIRC, - "Closing circuit that has been unused for %d seconds.", - (int)(now - circ->timestamp_created)); - circuit_mark_for_close(circ, END_CIRC_REASON_FINISHED); + } else if (!circ->timestamp_dirty && circ->state == CIRCUIT_STATE_OPEN) { + if (timercmp(&circ->timestamp_created, &cutoff, <)) { + if (circ->purpose == CIRCUIT_PURPOSE_C_GENERAL || + circ->purpose == CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT || + circ->purpose == CIRCUIT_PURPOSE_S_ESTABLISH_INTRO || + circ->purpose == CIRCUIT_PURPOSE_TESTING || + (circ->purpose >= CIRCUIT_PURPOSE_C_INTRODUCING && + circ->purpose <= CIRCUIT_PURPOSE_C_REND_READY_INTRO_ACKED) || + circ->purpose == CIRCUIT_PURPOSE_S_CONNECT_REND) { + log_debug(LD_CIRC, + "Closing circuit that has been unused for %ld msec.", + tv_mdiff(&circ->timestamp_created, &now)); + circuit_mark_for_close(circ, END_CIRC_REASON_FINISHED); + } else if (!TO_ORIGIN_CIRCUIT(circ)->is_ancient) { + /* Server-side rend joined circuits can end up really old, because + * they are reused by clients for longer than normal. The client + * controls their lifespan. (They never become dirty, because + * connection_exit_begin_conn() never marks anything as dirty.) + * Similarly, server-side intro circuits last a long time. */ + if (circ->purpose != CIRCUIT_PURPOSE_S_REND_JOINED && + circ->purpose != CIRCUIT_PURPOSE_S_INTRO) { + log_notice(LD_CIRC, + "Ancient non-dirty circuit %d is still around after " + "%ld milliseconds. Purpose: %d (%s)", + TO_ORIGIN_CIRCUIT(circ)->global_identifier, + tv_mdiff(&circ->timestamp_created, &now), + circ->purpose, + circuit_purpose_to_string(circ->purpose)); + TO_ORIGIN_CIRCUIT(circ)->is_ancient = 1; + } + } } } } @@ -794,6 +948,11 @@ circuit_has_opened(origin_circuit_t *circ) { control_event_circuit_status(circ, CIRC_EVENT_BUILT, 0); + /* Remember that this circuit has finished building. Now if we start + * it building again later (e.g. by extending it), we will know not + * to consider its build time. */ + circ->has_opened = 1; + switch (TO_CIRCUIT(circ)->purpose) { case CIRCUIT_PURPOSE_C_ESTABLISH_REND: rend_client_rendcirc_has_opened(circ); @@ -846,15 +1005,29 @@ circuit_build_failed(origin_circuit_t *circ) * to blame, blame it. Also, avoid this relay for a while, and * fail any one-hop directory fetches destined for it. */ const char *n_conn_id = circ->cpath->extend_info->identity_digest; + int already_marked = 0; if (circ->_base.n_conn) { or_connection_t *n_conn = circ->_base.n_conn; + if (n_conn->is_bad_for_new_circs) { + /* We only want to blame this router when a fresh healthy + * connection fails. So don't mark this router as newly failed, + * since maybe this was just an old circuit attempt that's + * finally timing out now. Also, there's no need to blow away + * circuits/streams/etc, since the failure of an unhealthy conn + * doesn't tell us much about whether a healthy conn would + * succeed. */ + already_marked = 1; + } log_info(LD_OR, "Our circuit failed to get a response from the first hop " "(%s:%d). I'm going to try to rotate to a better connection.", n_conn->_base.address, n_conn->_base.port); n_conn->is_bad_for_new_circs = 1; + } else { + log_info(LD_OR, + "Our circuit died before the first hop with no connection"); } - if (n_conn_id) { + if (n_conn_id && !already_marked) { entry_guard_register_connect_status(n_conn_id, 0, 1, time(NULL)); /* if there are any one-hop streams waiting on this circuit, fail * them now so they can retry elsewhere. */ @@ -936,8 +1109,8 @@ circuit_launch_by_router(uint8_t purpose, if (exit) info = extend_info_from_router(exit); circ = circuit_launch_by_extend_info(purpose, info, flags); - if (info) - extend_info_free(info); + + extend_info_free(info); return circ; } @@ -969,13 +1142,14 @@ circuit_launch_by_extend_info(uint8_t purpose, * internal circs rather than exit circs? -RD */ circ = circuit_find_to_cannibalize(purpose, extend_info, flags); if (circ) { - log_info(LD_CIRC,"Cannibalizing circ '%s' for purpose %d", - build_state_get_exit_nickname(circ->build_state), purpose); + log_info(LD_CIRC,"Cannibalizing circ '%s' for purpose %d (%s)", + build_state_get_exit_nickname(circ->build_state), purpose, + circuit_purpose_to_string(purpose)); circ->_base.purpose = purpose; /* reset the birth date of this circ, else expire_building * will see it and think it's been trying to build since it * began. */ - circ->_base.timestamp_created = time(NULL); + tor_gettimeofday(&circ->_base.timestamp_created); switch (purpose) { case CIRCUIT_PURPOSE_C_ESTABLISH_REND: case CIRCUIT_PURPOSE_S_ESTABLISH_INTRO: @@ -1080,15 +1254,17 @@ circuit_get_open_circ_or_launch(edge_connection_t *conn, int severity = LOG_NOTICE; /* FFFF if this is a tunneled directory fetch, don't yell * as loudly. the user doesn't even know it's happening. */ - if (options->UseBridges && bridges_known_but_down()) { + if (entry_list_is_constrained(options) && + entries_known_but_down(options)) { log_fn(severity, LD_APP|LD_DIR, - "Application request when we're believed to be " - "offline. Optimistically trying known bridges again."); - bridges_retry_all(); + "Application request when we haven't used client functionality " + "lately. Optimistically trying known %s again.", + options->UseBridges ? "bridges" : "entrynodes"); + entries_retry_all(options); } else if (!options->UseBridges || any_bridge_descriptors_known()) { log_fn(severity, LD_APP|LD_DIR, - "Application request when we're believed to be " - "offline. Optimistically trying directory fetches again."); + "Application request when we haven't used client functionality " + "lately. Optimistically trying directory fetches again."); routerlist_retry_directory_downloads(time(NULL)); } } @@ -1111,17 +1287,19 @@ circuit_get_open_circ_or_launch(edge_connection_t *conn, need_uptime)) { log_notice(LD_APP, "No Tor server allows exit to %s:%d. Rejecting.", - safe_str(conn->socks_request->address), + safe_str_client(conn->socks_request->address), conn->socks_request->port); return -1; } } else { - /* XXXX022 Duplicates checks in connection_ap_handshake_attach_circuit */ + /* XXXX023 Duplicates checks in connection_ap_handshake_attach_circuit: + * refactor into a single function? */ routerinfo_t *router = router_get_by_nickname(conn->chosen_exit_name, 1); int opt = conn->chosen_exit_optional; if (router && !connection_ap_can_use_exit(conn, router)) { log_fn(opt ? LOG_INFO : LOG_WARN, LD_APP, - "Requested exit point '%s' would refuse request. %s.", + "Requested exit point '%s' is excluded or " + "would refuse request. %s.", conn->chosen_exit_name, opt ? "Trying others" : "Closing"); if (opt) { conn->chosen_exit_optional = 0; @@ -1152,19 +1330,14 @@ circuit_get_open_circ_or_launch(edge_connection_t *conn, if (!extend_info) { log_info(LD_REND, "No intro points for '%s': re-fetching service descriptor.", - safe_str(conn->rend_data->onion_address)); - /* Fetch both, v0 and v2 rend descriptors in parallel. Use whichever - * arrives first. Exception: When using client authorization, only - * fetch v2 descriptors.*/ + safe_str_client(conn->rend_data->onion_address)); rend_client_refetch_v2_renddesc(conn->rend_data); - if (conn->rend_data->auth_type == REND_NO_AUTH) - rend_client_refetch_renddesc(conn->rend_data->onion_address); conn->_base.state = AP_CONN_STATE_RENDDESC_WAIT; return 0; } log_info(LD_REND,"Chose '%s' as intro point for '%s'.", extend_info->nickname, - safe_str(conn->rend_data->onion_address)); + safe_str_client(conn->rend_data->onion_address)); } /* If we have specified a particular exit node for our @@ -1193,7 +1366,7 @@ circuit_get_open_circ_or_launch(edge_connection_t *conn, } if (tor_addr_from_str(&addr, conn->socks_request->address) < 0) { log_info(LD_DIR, "Broken address %s on tunnel conn. Closing.", - escaped_safe_str(conn->socks_request->address)); + escaped_safe_str_client(conn->socks_request->address)); return -1; } extend_info = extend_info_alloc(conn->chosen_exit_name+1, @@ -1235,10 +1408,20 @@ circuit_get_open_circ_or_launch(edge_connection_t *conn, flags); } - if (extend_info) - extend_info_free(extend_info); + extend_info_free(extend_info); - if (desired_circuit_purpose != CIRCUIT_PURPOSE_C_GENERAL) { + if (desired_circuit_purpose == CIRCUIT_PURPOSE_C_GENERAL) { + /* We just caused a circuit to get built because of this stream. + * If this stream has caused a _lot_ of circuits to be built, that's + * a bad sign: we should tell the user. */ + if (conn->num_circuits_launched < NUM_CIRCUITS_LAUNCHED_THRESHOLD && + ++conn->num_circuits_launched == NUM_CIRCUITS_LAUNCHED_THRESHOLD) + log_warn(LD_BUG, "The application request to %s:%d has launched " + "%d circuits without finding one it likes.", + escaped_safe_str_client(conn->socks_request->address), + conn->socks_request->port, + conn->num_circuits_launched); + } else { /* help predict this next time */ rep_hist_note_used_internal(time(NULL), need_uptime, 1); if (circ) { @@ -1418,7 +1601,7 @@ connection_ap_handshake_attach_circuit(edge_connection_t *conn) LOG_INFO : LOG_NOTICE; log_fn(severity, LD_APP, "Tried for %d seconds to get a connection to %s:%d. Giving up.", - conn_age, safe_str(conn->socks_request->address), + conn_age, safe_str_client(conn->socks_request->address), conn->socks_request->port); return -1; } @@ -1447,7 +1630,8 @@ connection_ap_handshake_attach_circuit(edge_connection_t *conn) } if (router && !connection_ap_can_use_exit(conn, router)) { log_fn(opt ? LOG_INFO : LOG_WARN, LD_APP, - "Requested exit point '%s' would refuse request. %s.", + "Requested exit point '%s' is excluded or " + "would refuse request. %s.", conn->chosen_exit_name, opt ? "Trying others" : "Closing"); if (opt) { conn->chosen_exit_optional = 0; @@ -1461,7 +1645,7 @@ connection_ap_handshake_attach_circuit(edge_connection_t *conn) /* find the circuit that we should use, if there is one. */ retval = circuit_get_open_circ_or_launch( conn, CIRCUIT_PURPOSE_C_GENERAL, &circ); - if (retval < 1) // XXX021 if we totally fail, this still returns 0 -RD + if (retval < 1) // XXX022 if we totally fail, this still returns 0 -RD return retval; log_debug(LD_APP|LD_CIRC, diff --git a/src/or/circuituse.h b/src/or/circuituse.h new file mode 100644 index 0000000000..9f393ab378 --- /dev/null +++ b/src/or/circuituse.h @@ -0,0 +1,55 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file circuituse.h + * \brief Header file for circuituse.c. + **/ + +#ifndef _TOR_CIRCUITUSE_H +#define _TOR_CIRCUITUSE_H + +void circuit_expire_building(void); +void circuit_remove_handled_ports(smartlist_t *needed_ports); +int circuit_stream_is_being_handled(edge_connection_t *conn, uint16_t port, + int min); +#if 0 +int circuit_conforms_to_options(const origin_circuit_t *circ, + const or_options_t *options); +#endif +void circuit_build_needed_circs(time_t now); +void circuit_detach_stream(circuit_t *circ, edge_connection_t *conn); + +void circuit_expire_old_circuits_serverside(time_t now); + +void reset_bandwidth_test(void); +int circuit_enough_testing_circs(void); + +void circuit_has_opened(origin_circuit_t *circ); +void circuit_build_failed(origin_circuit_t *circ); + +/** Flag to set when a circuit should have only a single hop. */ +#define CIRCLAUNCH_ONEHOP_TUNNEL (1<<0) +/** Flag to set when a circuit needs to be built of high-uptime nodes */ +#define CIRCLAUNCH_NEED_UPTIME (1<<1) +/** Flag to set when a circuit needs to be built of high-capacity nodes */ +#define CIRCLAUNCH_NEED_CAPACITY (1<<2) +/** Flag to set when the last hop of a circuit doesn't need to be an + * exit node. */ +#define CIRCLAUNCH_IS_INTERNAL (1<<3) +origin_circuit_t *circuit_launch_by_extend_info(uint8_t purpose, + extend_info_t *info, + int flags); +origin_circuit_t *circuit_launch_by_router(uint8_t purpose, + routerinfo_t *exit, int flags); +void circuit_reset_failure_count(int timeout); +int connection_ap_handshake_attach_chosen_circuit(edge_connection_t *conn, + origin_circuit_t *circ, + crypt_path_t *cpath); +int connection_ap_handshake_attach_circuit(edge_connection_t *conn); + +#endif + diff --git a/src/or/command.c b/src/or/command.c index 9269456805..00d9af33fa 100644 --- a/src/or/command.c +++ b/src/or/command.c @@ -16,6 +16,19 @@ */ #include "or.h" +#include "circuitbuild.h" +#include "circuitlist.h" +#include "command.h" +#include "connection.h" +#include "connection_or.h" +#include "config.h" +#include "control.h" +#include "cpuworker.h" +#include "hibernate.h" +#include "onion.h" +#include "relay.h" +#include "router.h" +#include "routerlist.h" /** How many CELL_PADDING cells have we received, ever? */ uint64_t stats_n_padding_cells_processed = 0; @@ -275,7 +288,14 @@ command_process_create_cell(cell_t *cell, or_connection_t *conn) /* hand it off to the cpuworkers, and then return. */ if (assign_onionskin_to_cpuworker(NULL, circ, onionskin) < 0) { - log_warn(LD_GENERAL,"Failed to hand off onionskin. Closing."); +#define WARN_HANDOFF_FAILURE_INTERVAL (6*60*60) + static ratelim_t handoff_warning = + RATELIM_INIT(WARN_HANDOFF_FAILURE_INTERVAL); + char *m; + if ((m = rate_limit_log(&handoff_warning, approx_time()))) { + log_warn(LD_GENERAL,"Failed to hand off onionskin. Closing.%s",m); + tor_free(m); + } circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_INTERNAL); return; } @@ -396,15 +416,18 @@ command_process_relay_cell(cell_t *cell, or_connection_t *conn) * gotten no more than MAX_RELAY_EARLY_CELLS_PER_CIRCUIT of them. */ if (cell->command == CELL_RELAY_EARLY) { if (direction == CELL_DIRECTION_IN) { - /* XXX Allow an unlimited number of inbound relay_early cells for - * now, for hidden service compatibility. See bug 1038. -RD */ + /* Allow an unlimited number of inbound relay_early cells, + * for hidden service compatibility. There isn't any way to make + * a long circuit through inbound relay_early cells anyway. See + * bug 1038. -RD */ } else { or_circuit_t *or_circ = TO_OR_CIRCUIT(circ); if (or_circ->remaining_relay_early_cells == 0) { log_fn(LOG_PROTOCOL_WARN, LD_OR, "Received too many RELAY_EARLY cells on circ %d from %s:%d." " Closing circuit.", - cell->circ_id, safe_str(conn->_base.address), conn->_base.port); + cell->circ_id, safe_str(conn->_base.address), + conn->_base.port); circuit_mark_for_close(circ, END_CIRC_REASON_TORPROTOCOL); return; } @@ -512,7 +535,8 @@ command_process_versions_cell(var_cell_t *cell, or_connection_t *conn) conn->handshake_state->received_versions = 1; log_info(LD_OR, "Negotiated version %d with %s:%d; sending NETINFO.", - highest_supported_version, safe_str(conn->_base.address), + highest_supported_version, + safe_str_client(conn->_base.address), conn->_base.port); tor_assert(conn->link_proto >= 2); @@ -627,8 +651,8 @@ command_process_netinfo_cell(cell_t *cell, or_connection_t *conn) else log_info(LD_OR, "Got good NETINFO cell from %s:%d; OR connection is now " "open, using protocol version %d", - safe_str(conn->_base.address), conn->_base.port, - (int)conn->link_proto); + safe_str_client(conn->_base.address), + conn->_base.port, (int)conn->link_proto); assert_connection_ok(TO_CONN(conn),time(NULL)); } diff --git a/src/or/command.h b/src/or/command.h new file mode 100644 index 0000000000..95b0f3a931 --- /dev/null +++ b/src/or/command.h @@ -0,0 +1,25 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file command.h + * \brief Header file for command.c. + **/ + +#ifndef _TOR_COMMAND_H +#define _TOR_COMMAND_H + +void command_process_cell(cell_t *cell, or_connection_t *conn); +void command_process_var_cell(var_cell_t *cell, or_connection_t *conn); + +extern uint64_t stats_n_padding_cells_processed; +extern uint64_t stats_n_create_cells_processed; +extern uint64_t stats_n_created_cells_processed; +extern uint64_t stats_n_relay_cells_processed; +extern uint64_t stats_n_destroy_cells_processed; + +#endif + diff --git a/src/or/config.c b/src/or/config.c index 9cd8149ce8..34208e85bf 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -12,6 +12,28 @@ #define CONFIG_PRIVATE #include "or.h" +#include "circuitbuild.h" +#include "circuitlist.h" +#include "config.h" +#include "connection.h" +#include "connection_edge.h" +#include "connection_or.h" +#include "control.h" +#include "cpuworker.h" +#include "dirserv.h" +#include "dirvote.h" +#include "dns.h" +#include "geoip.h" +#include "hibernate.h" +#include "main.h" +#include "networkstatus.h" +#include "policies.h" +#include "relay.h" +#include "rendclient.h" +#include "rendservice.h" +#include "rephist.h" +#include "router.h" +#include "routerlist.h" #ifdef MS_WINDOWS #include <shlobj.h> #endif @@ -61,11 +83,12 @@ static config_abbrev_t _option_abbrevs[] = { PLURAL(LongLivedPort), PLURAL(HiddenServiceNode), PLURAL(HiddenServiceExcludeNode), - PLURAL(NumCpu), + PLURAL(NumCPU), PLURAL(RendNode), PLURAL(RendExcludeNode), PLURAL(StrictEntryNode), PLURAL(StrictExitNode), + PLURAL(StrictNode), { "l", "Log", 1, 0}, { "AllowUnverifiedNodes", "AllowInvalidNodes", 0, 0}, { "AutomapHostSuffixes", "AutomapHostsSuffixes", 0, 0}, @@ -83,10 +106,12 @@ static config_abbrev_t _option_abbrevs[] = { { "NumEntryNodes", "NumEntryGuards", 0, 0}, { "ResolvConf", "ServerDNSResolvConfFile", 0, 1}, { "SearchDomains", "ServerDNSSearchDomains", 0, 1}, - { "ServerDNSAllowBrokenResolvConf", "ServerDNSAllowBrokenConfig", 0, 0 }, + { "ServerDNSAllowBrokenResolvConf", "ServerDNSAllowBrokenConfig", 0, 0}, { "PreferTunnelledDirConns", "PreferTunneledDirConns", 0, 0}, { "BridgeAuthoritativeDirectory", "BridgeAuthoritativeDir", 0, 0}, { "HashedControlPassword", "__HashedControlSessionPassword", 1, 0}, + { "StrictEntryNodes", "StrictNodes", 0, 1}, + { "StrictExitNodes", "StrictNodes", 0, 1}, { NULL, NULL, 0, 0}, }; @@ -134,6 +159,7 @@ static config_var_t _option_vars[] = { V(AccountingMax, MEMUNIT, "0 bytes"), V(AccountingStart, STRING, NULL), V(Address, STRING, NULL), + V(AllowDotExit, BOOL, "0"), V(AllowInvalidNodes, CSV, "middle,rendezvous"), V(AllowNonRFC953Hostnames, BOOL, "0"), V(AllowSingleHopCircuits, BOOL, "0"), @@ -162,10 +188,16 @@ static config_var_t _option_vars[] = { V(BridgePassword, STRING, NULL), V(BridgeRecordUsageByCountry, BOOL, "1"), V(BridgeRelay, BOOL, "0"), - V(CircuitBuildTimeout, INTERVAL, "1 minute"), + V(CellStatistics, BOOL, "0"), + V(LearnCircuitBuildTimeout, BOOL, "1"), + V(CircuitBuildTimeout, INTERVAL, "0"), V(CircuitIdleTimeout, INTERVAL, "1 hour"), + V(CircuitStreamTimeout, INTERVAL, "0"), + V(CircuitPriorityHalflife, DOUBLE, "-100.0"), /*negative:'Use default'*/ V(ClientDNSRejectInternalAddresses, BOOL,"1"), + V(ClientRejectInternalAddresses, BOOL, "1"), V(ClientOnly, BOOL, "0"), + V(ConsensusParams, STRING, NULL), V(ConnLimit, UINT, "1000"), V(ConstrainedSockets, BOOL, "0"), V(ConstrainedSockSize, MEMUNIT, "8192"), @@ -186,18 +218,19 @@ static config_var_t _option_vars[] = { V(DirPort, UINT, "0"), V(DirPortFrontPage, FILENAME, NULL), OBSOLETE("DirPostPeriod"), -#ifdef ENABLE_GEOIP_STATS - V(DirRecordUsageByCountry, BOOL, "0"), - V(DirRecordUsageGranularity, UINT, "4"), - V(DirRecordUsageRetainIPs, INTERVAL, "14 days"), - V(DirRecordUsageSaveInterval, INTERVAL, "6 hours"), -#endif + OBSOLETE("DirRecordUsageByCountry"), + OBSOLETE("DirRecordUsageGranularity"), + OBSOLETE("DirRecordUsageRetainIPs"), + OBSOLETE("DirRecordUsageSaveInterval"), + V(DirReqStatistics, BOOL, "0"), VAR("DirServer", LINELIST, DirServers, NULL), + V(DisableAllSwap, BOOL, "0"), V(DNSPort, UINT, "0"), V(DNSListenAddress, LINELIST, NULL), V(DownloadExtraInfo, BOOL, "0"), V(EnforceDistinctSubnets, BOOL, "1"), V(EntryNodes, ROUTERSET, NULL), + V(EntryStatistics, BOOL, "0"), V(TestingEstimatedDescriptorPropagationTime, INTERVAL, "10 minutes"), V(ExcludeNodes, ROUTERSET, NULL), V(ExcludeExitNodes, ROUTERSET, NULL), @@ -205,15 +238,24 @@ static config_var_t _option_vars[] = { V(ExitNodes, ROUTERSET, NULL), V(ExitPolicy, LINELIST, NULL), V(ExitPolicyRejectPrivate, BOOL, "1"), + V(ExitPortStatistics, BOOL, "0"), + V(ExtraInfoStatistics, BOOL, "0"), + +#if defined (WINCE) + V(FallbackNetworkstatusFile, FILENAME, "fallback-consensus"), +#else V(FallbackNetworkstatusFile, FILENAME, SHARE_DATADIR PATH_SEPARATOR "tor" PATH_SEPARATOR "fallback-consensus"), +#endif V(FascistFirewall, BOOL, "0"), V(FirewallPorts, CSV, ""), V(FastFirstHopPK, BOOL, "1"), V(FetchDirInfoEarly, BOOL, "0"), + V(FetchDirInfoExtraEarly, BOOL, "0"), V(FetchServerDescriptors, BOOL, "1"), V(FetchHidServDescriptors, BOOL, "1"), V(FetchUselessDescriptors, BOOL, "0"), + V(FetchV2Networkstatus, BOOL, "0"), #ifdef WIN32 V(GeoIPFile, FILENAME, "<default>"), #else @@ -222,6 +264,8 @@ static config_var_t _option_vars[] = { #endif OBSOLETE("Group"), V(HardwareAccel, BOOL, "0"), + V(AccelName, STRING, NULL), + V(AccelDir, FILENAME, NULL), V(HashedControlPassword, LINELIST, NULL), V(HidServDirectoryV2, BOOL, "1"), VAR("HiddenServiceDir", LINELIST_S, RendConfigLines, NULL), @@ -233,14 +277,19 @@ static config_var_t _option_vars[] = { VAR("HiddenServiceAuthorizeClient",LINELIST_S,RendConfigLines, NULL), V(HidServAuth, LINELIST, NULL), V(HSAuthoritativeDir, BOOL, "0"), - V(HSAuthorityRecordStats, BOOL, "0"), - V(HttpProxy, STRING, NULL), - V(HttpProxyAuthenticator, STRING, NULL), - V(HttpsProxy, STRING, NULL), - V(HttpsProxyAuthenticator, STRING, NULL), + OBSOLETE("HSAuthorityRecordStats"), + V(HTTPProxy, STRING, NULL), + V(HTTPProxyAuthenticator, STRING, NULL), + V(HTTPSProxy, STRING, NULL), + V(HTTPSProxyAuthenticator, STRING, NULL), + V(Socks4Proxy, STRING, NULL), + V(Socks5Proxy, STRING, NULL), + V(Socks5ProxyUsername, STRING, NULL), + V(Socks5ProxyPassword, STRING, NULL), OBSOLETE("IgnoreVersion"), V(KeepalivePeriod, INTERVAL, "5 minutes"), VAR("Log", LINELIST, Logs, NULL), + V(LogMessageDomains, BOOL, "0"), OBSOLETE("LinkPadding"), OBSOLETE("LogLevel"), OBSOLETE("LogFile"), @@ -254,17 +303,20 @@ static config_var_t _option_vars[] = { V(MyFamily, STRING, NULL), V(NewCircuitPeriod, INTERVAL, "30 seconds"), VAR("NamingAuthoritativeDirectory",BOOL, NamingAuthoritativeDir, "0"), - V(NatdListenAddress, LINELIST, NULL), - V(NatdPort, UINT, "0"), + V(NATDListenAddress, LINELIST, NULL), + V(NATDPort, UINT, "0"), V(Nickname, STRING, NULL), - V(NoPublish, BOOL, "0"), + V(WarnUnsafeSocks, BOOL, "1"), + OBSOLETE("NoPublish"), VAR("NodeFamily", LINELIST, NodeFamilies, NULL), - V(NumCpus, UINT, "1"), + V(NumCPUs, UINT, "1"), V(NumEntryGuards, UINT, "3"), V(ORListenAddress, LINELIST, NULL), V(ORPort, UINT, "0"), V(OutboundBindAddress, STRING, NULL), OBSOLETE("PathlenCoinWeight"), + V(PerConnBWBurst, MEMUNIT, "0"), + V(PerConnBWRate, MEMUNIT, "0"), V(PidFile, STRING, NULL), V(TestingTorNetwork, BOOL, "0"), V(PreferTunneledDirConns, BOOL, "1"), @@ -278,6 +330,7 @@ static config_var_t _option_vars[] = { V(RecommendedClientVersions, LINELIST, NULL), V(RecommendedServerVersions, LINELIST, NULL), OBSOLETE("RedirectExit"), + V(RefuseUnknownExits, STRING, "auto"), V(RejectPlaintextPorts, CSV, ""), V(RelayBandwidthBurst, MEMUNIT, "0"), V(RelayBandwidthRate, MEMUNIT, "0"), @@ -287,8 +340,9 @@ static config_var_t _option_vars[] = { V(RephistTrackTime, INTERVAL, "24 hours"), OBSOLETE("RouterFile"), V(RunAsDaemon, BOOL, "0"), - V(RunTesting, BOOL, "0"), - V(SafeLogging, BOOL, "1"), +// V(RunTesting, BOOL, "0"), + OBSOLETE("RunTesting"), // currently unused + V(SafeLogging, STRING, "1"), V(SafeSocks, BOOL, "0"), V(ServerDNSAllowBrokenConfig, BOOL, "1"), V(ServerDNSAllowNonRFC953Hostnames, BOOL,"0"), @@ -304,8 +358,7 @@ static config_var_t _option_vars[] = { V(SocksPort, UINT, "9050"), V(SocksTimeout, INTERVAL, "2 minutes"), OBSOLETE("StatusFetchPeriod"), - V(StrictEntryNodes, BOOL, "0"), - V(StrictExitNodes, BOOL, "0"), + V(StrictNodes, BOOL, "0"), OBSOLETE("SysLog"), V(TestSocks, BOOL, "0"), OBSOLETE("TestVia"), @@ -330,6 +383,7 @@ static config_var_t _option_vars[] = { V(V3AuthDistDelay, INTERVAL, "5 minutes"), V(V3AuthNIntervalsValid, UINT, "3"), V(V3AuthUseLegacyKey, BOOL, "0"), + V(V3BandwidthsFile, FILENAME, NULL), VAR("VersioningAuthoritativeDirectory",BOOL,VersioningAuthoritativeDir, "0"), V(VirtualAddrNetwork, STRING, "127.192.0.0/10"), V(WarnPlaintextPorts, CSV, "23,109,110,143"), @@ -340,6 +394,8 @@ static config_var_t _option_vars[] = { VAR("__HashedControlSessionPassword", LINELIST, HashedControlSessionPassword, NULL), V(MinUptimeHidServDirectoryV2, INTERVAL, "24 hours"), + V(_UsingTestNetworkDefaults, BOOL, "0"), + { NULL, CONFIG_TYPE_OBSOLETE, 0, NULL } }; @@ -353,6 +409,7 @@ static config_var_t testing_tor_network_defaults[] = { V(AuthDirMaxServersPerAddr, UINT, "0"), V(AuthDirMaxServersPerAuthAddr,UINT, "0"), V(ClientDNSRejectInternalAddresses, BOOL,"0"), + V(ClientRejectInternalAddresses, BOOL, "0"), V(ExitPolicyRejectPrivate, BOOL, "0"), V(V3AuthVotingInterval, INTERVAL, "5 minutes"), V(V3AuthVoteDelay, INTERVAL, "20 seconds"), @@ -362,6 +419,8 @@ static config_var_t testing_tor_network_defaults[] = { V(TestingV3AuthInitialDistDelay, INTERVAL, "20 seconds"), V(TestingAuthDirTimeToLearnReachability, INTERVAL, "0 minutes"), V(TestingEstimatedDescriptorPropagationTime, INTERVAL, "0 minutes"), + V(MinUptimeHidServDirectoryV2, INTERVAL, "0 minutes"), + V(_UsingTestNetworkDefaults, BOOL, "1"), { NULL, CONFIG_TYPE_OBSOLETE, 0, NULL } }; #undef VAR @@ -377,6 +436,9 @@ static config_var_t _state_vars[] = { V(AccountingExpectedUsage, MEMUNIT, NULL), V(AccountingIntervalStart, ISOTIME, NULL), V(AccountingSecondsActive, INTERVAL, NULL), + V(AccountingSecondsToReachSoftLimit,INTERVAL, NULL), + V(AccountingSoftLimitHitAt, ISOTIME, NULL), + V(AccountingBytesAtSoftLimit, MEMUNIT, NULL), VAR("EntryGuard", LINELIST_S, EntryGuards, NULL), VAR("EntryGuardDownSince", LINELIST_S, EntryGuards, NULL), @@ -387,15 +449,30 @@ static config_var_t _state_vars[] = { V(BWHistoryReadEnds, ISOTIME, NULL), V(BWHistoryReadInterval, UINT, "900"), V(BWHistoryReadValues, CSV, ""), + V(BWHistoryReadMaxima, CSV, ""), V(BWHistoryWriteEnds, ISOTIME, NULL), V(BWHistoryWriteInterval, UINT, "900"), V(BWHistoryWriteValues, CSV, ""), + V(BWHistoryWriteMaxima, CSV, ""), + V(BWHistoryDirReadEnds, ISOTIME, NULL), + V(BWHistoryDirReadInterval, UINT, "900"), + V(BWHistoryDirReadValues, CSV, ""), + V(BWHistoryDirReadMaxima, CSV, ""), + V(BWHistoryDirWriteEnds, ISOTIME, NULL), + V(BWHistoryDirWriteInterval, UINT, "900"), + V(BWHistoryDirWriteValues, CSV, ""), + V(BWHistoryDirWriteMaxima, CSV, ""), V(TorVersion, STRING, NULL), V(LastRotatedOnionKey, ISOTIME, NULL), V(LastWritten, ISOTIME, NULL), + V(TotalBuildTimes, UINT, NULL), + V(CircuitBuildAbandonedCount, UINT, "0"), + VAR("CircuitBuildTimeBin", LINELIST_S, BuildtimeHistogram, NULL), + VAR("BuildtimeHistogram", LINELIST_V, BuildtimeHistogram, NULL), + { NULL, CONFIG_TYPE_OBSOLETE, 0, NULL } }; @@ -410,213 +487,6 @@ typedef struct config_var_description_t { const char *description; } config_var_description_t; -/** Descriptions of the configuration options, to be displayed by online - * option browsers */ -/* XXXX022 did anybody want this? at all? If not, kill it.*/ -static config_var_description_t options_description[] = { - /* ==== general options */ - { "AvoidDiskWrites", "If non-zero, try to write to disk less frequently than" - " we would otherwise." }, - { "BandwidthRate", "A token bucket limits the average incoming bandwidth on " - "this node to the specified number of bytes per second." }, - { "BandwidthBurst", "Limit the maximum token buffer size (also known as " - "burst) to the given number of bytes." }, - { "ConnLimit", "Minimum number of simultaneous sockets we must have." }, - { "ConstrainedSockets", "Shrink tx and rx buffers for sockets to avoid " - "system limits on vservers and related environments. See man page for " - "more information regarding this option." }, - { "ConstrainedSockSize", "Limit socket buffers to this size when " - "ConstrainedSockets is enabled." }, - /* ControlListenAddress */ - { "ControlPort", "If set, Tor will accept connections from the same machine " - "(localhost only) on this port, and allow those connections to control " - "the Tor process using the Tor Control Protocol (described in " - "control-spec.txt).", }, - { "CookieAuthentication", "If this option is set to 1, don't allow any " - "connections to the control port except when the connecting process " - "can read a file that Tor creates in its data directory." }, - { "DataDirectory", "Store working data, state, keys, and caches here." }, - { "DirServer", "Tor only trusts directories signed with one of these " - "servers' keys. Used to override the standard list of directory " - "authorities." }, - /* { "FastFirstHopPK", "" }, */ - /* FetchServerDescriptors, FetchHidServDescriptors, - * FetchUselessDescriptors */ - { "HardwareAccel", "If set, Tor tries to use hardware crypto accelerators " - "when it can." }, - /* HashedControlPassword */ - { "HTTPProxy", "Force Tor to make all HTTP directory requests through this " - "host:port (or host:80 if port is not set)." }, - { "HTTPProxyAuthenticator", "A username:password pair to be used with " - "HTTPProxy." }, - { "HTTPSProxy", "Force Tor to make all TLS (SSL) connections through this " - "host:port (or host:80 if port is not set)." }, - { "HTTPSProxyAuthenticator", "A username:password pair to be used with " - "HTTPSProxy." }, - { "KeepalivePeriod", "Send a padding cell every N seconds to keep firewalls " - "from closing our connections while Tor is not in use." }, - { "Log", "Where to send logging messages. Format is " - "minSeverity[-maxSeverity] (stderr|stdout|syslog|file FILENAME)." }, - { "OutboundBindAddress", "Make all outbound connections originate from the " - "provided IP address (only useful for multiple network interfaces)." }, - { "PIDFile", "On startup, write our PID to this file. On clean shutdown, " - "remove the file." }, - { "PreferTunneledDirConns", "If non-zero, avoid directory servers that " - "don't support tunneled connections." }, - /* PreferTunneledDirConns */ - /* ProtocolWarnings */ - /* RephistTrackTime */ - { "RunAsDaemon", "If set, Tor forks and daemonizes to the background when " - "started. Unix only." }, - { "SafeLogging", "If set to 0, Tor logs potentially sensitive strings " - "rather than replacing them with the string [scrubbed]." }, - { "TunnelDirConns", "If non-zero, when a directory server we contact " - "supports it, we will build a one-hop circuit and make an encrypted " - "connection via its ORPort." }, - { "User", "On startup, setuid to this user." }, - - /* ==== client options */ - { "AllowInvalidNodes", "Where on our circuits should Tor allow servers " - "that the directory authorities haven't called \"valid\"?" }, - { "AllowNonRFC953Hostnames", "If set to 1, we don't automatically reject " - "hostnames for having invalid characters." }, - /* CircuitBuildTimeout, CircuitIdleTimeout */ - { "ClientOnly", "If set to 1, Tor will under no circumstances run as a " - "server, even if ORPort is enabled." }, - { "EntryNodes", "A list of preferred entry nodes to use for the first hop " - "in circuits, when possible." }, - /* { "EnforceDistinctSubnets" , "" }, */ - { "ExitNodes", "A list of preferred nodes to use for the last hop in " - "circuits, when possible." }, - { "ExcludeNodes", "A list of nodes never to use when building a circuit." }, - { "FascistFirewall", "If set, Tor will only create outgoing connections to " - "servers running on the ports listed in FirewallPorts." }, - { "FirewallPorts", "A list of ports that we can connect to. Only used " - "when FascistFirewall is set." }, - { "LongLivedPorts", "A list of ports for services that tend to require " - "high-uptime connections." }, - { "MapAddress", "Force Tor to treat all requests for one address as if " - "they were for another." }, - { "NewCircuitPeriod", "Force Tor to consider whether to build a new circuit " - "every NUM seconds." }, - { "MaxCircuitDirtiness", "Do not attach new streams to a circuit that has " - "been used more than this many seconds ago." }, - /* NatdPort, NatdListenAddress */ - { "NodeFamily", "A list of servers that constitute a 'family' and should " - "never be used in the same circuit." }, - { "NumEntryGuards", "How many entry guards should we keep at a time?" }, - /* PathlenCoinWeight */ - { "ReachableAddresses", "Addresses we can connect to, as IP/bits:port-port. " - "By default, we assume all addresses are reachable." }, - /* reachablediraddresses, reachableoraddresses. */ - /* SafeSOCKS */ - { "SOCKSPort", "The port where we listen for SOCKS connections from " - "applications." }, - { "SOCKSListenAddress", "Bind to this address to listen to connections from " - "SOCKS-speaking applications." }, - { "SOCKSPolicy", "Set an entry policy to limit which addresses can connect " - "to the SOCKSPort." }, - /* SocksTimeout */ - { "StrictExitNodes", "If set, Tor will fail to operate when none of the " - "configured ExitNodes can be used." }, - { "StrictEntryNodes", "If set, Tor will fail to operate when none of the " - "configured EntryNodes can be used." }, - /* TestSocks */ - { "TrackHostsExit", "Hosts and domains which should, if possible, be " - "accessed from the same exit node each time we connect to them." }, - { "TrackHostsExitExpire", "Time after which we forget which exit we were " - "using to connect to hosts in TrackHostsExit." }, - /* "TransPort", "TransListenAddress */ - { "UseEntryGuards", "Set to 0 if we want to pick from the whole set of " - "servers for the first position in each circuit, rather than picking a " - "set of 'Guards' to prevent profiling attacks." }, - - /* === server options */ - { "Address", "The advertised (external) address we should use." }, - /* Accounting* options. */ - /* AssumeReachable */ - { "ContactInfo", "Administrative contact information to advertise for this " - "server." }, - { "ExitPolicy", "Address/port ranges for which to accept or reject outgoing " - "connections on behalf of Tor users." }, - /* { "ExitPolicyRejectPrivate, "" }, */ - { "MaxAdvertisedBandwidth", "If set, we will not advertise more than this " - "amount of bandwidth for our bandwidth rate, regardless of how much " - "bandwidth we actually detect." }, - { "MaxOnionsPending", "Reject new attempts to extend circuits when we " - "already have this many pending." }, - { "MyFamily", "Declare a list of other servers as belonging to the same " - "family as this one, so that clients will not use two from the same " - "family in the same circuit." }, - { "Nickname", "Set the server nickname." }, - { "NoPublish", "{DEPRECATED}" }, - { "NumCPUs", "How many processes to use at once for public-key crypto." }, - { "ORPort", "Advertise this port to listen for connections from Tor clients " - "and servers." }, - { "ORListenAddress", "Bind to this address to listen for connections from " - "clients and servers, instead of the default 0.0.0.0:ORPort." }, - { "PublishServerDescriptor", "Set to 0 to keep the server from " - "uploading info to the directory authorities." }, - /* ServerDNS: DetectHijacking, ResolvConfFile, SearchDomains */ - { "ShutdownWaitLength", "Wait this long for clients to finish when " - "shutting down because of a SIGINT." }, - - /* === directory cache options */ - { "DirPort", "Serve directory information from this port, and act as a " - "directory cache." }, - { "DirPortFrontPage", "Serve a static html disclaimer on DirPort." }, - { "DirListenAddress", "Bind to this address to listen for connections from " - "clients and servers, instead of the default 0.0.0.0:DirPort." }, - { "DirPolicy", "Set a policy to limit who can connect to the directory " - "port." }, - - /* Authority options: AuthDirBadExit, AuthDirInvalid, AuthDirReject, - * AuthDirRejectUnlisted, AuthDirListBadExits, AuthoritativeDirectory, - * DirAllowPrivateAddresses, HSAuthoritativeDir, - * NamingAuthoritativeDirectory, RecommendedVersions, - * RecommendedClientVersions, RecommendedServerVersions, RendPostPeriod, - * RunTesting, V1AuthoritativeDirectory, VersioningAuthoritativeDirectory, */ - - /* Hidden service options: HiddenService: dir,excludenodes, nodes, - * options, port. PublishHidServDescriptor */ - - /* Nonpersistent options: __LeaveStreamsUnattached, __AllDirActionsPrivate */ - { NULL, NULL }, -}; - -/** Online description of state variables. */ -static config_var_description_t state_description[] = { - { "AccountingBytesReadInInterval", - "How many bytes have we read in this accounting period?" }, - { "AccountingBytesWrittenInInterval", - "How many bytes have we written in this accounting period?" }, - { "AccountingExpectedUsage", - "How many bytes did we expect to use per minute? (0 for no estimate.)" }, - { "AccountingIntervalStart", "When did this accounting period begin?" }, - { "AccountingSecondsActive", "How long have we been awake in this period?" }, - - { "BWHistoryReadEnds", "When does the last-recorded read-interval end?" }, - { "BWHistoryReadInterval", "How long is each read-interval (in seconds)?" }, - { "BWHistoryReadValues", "Number of bytes read in each interval." }, - { "BWHistoryWriteEnds", "When does the last-recorded write-interval end?" }, - { "BWHistoryWriteInterval", "How long is each write-interval (in seconds)?"}, - { "BWHistoryWriteValues", "Number of bytes written in each interval." }, - - { "EntryGuard", "One of the nodes we have chosen as a fixed entry" }, - { "EntryGuardDownSince", - "The last entry guard has been unreachable since this time." }, - { "EntryGuardUnlistedSince", - "The last entry guard has been unusable since this time." }, - - { "LastRotatedOnionKey", - "The last time at which we changed the medium-term private key used for " - "building circuits." }, - { "LastWritten", "When was this state file last regenerated?" }, - - { "TorVersion", "Which version of Tor generated this state file?" }, - { NULL, NULL }, -}; - /** Type of a callback to validate whether a given configuration is * well-formed and consistent. See options_trial_assign() for documentation * of arguments. */ @@ -635,8 +505,6 @@ typedef struct { config_var_t *vars; /**< List of variables we recognize, their default * values, and where we stick them in the structure. */ validate_fn_t validate_fn; /**< Function to validate config. */ - /** Documentation for configuration variables. */ - config_var_description_t *descriptions; /** If present, extra is a LINELIST variable for unrecognized * lines. Otherwise, unrecognized lines are an error. */ config_var_t *extra; @@ -700,20 +568,6 @@ static uint64_t config_parse_memunit(const char *s, int *ok); static int config_parse_interval(const char *s, int *ok); static void init_libevent(void); static int opt_streq(const char *s1, const char *s2); -/** Versions of libevent. */ -typedef enum { - /* Note: we compare these, so it's important that "old" precede everything, - * and that "other" come last. */ - LE_OLD=0, LE_10C, LE_10D, LE_10E, LE_11, LE_11A, LE_11B, LE_12, LE_12A, - LE_13, LE_13A, LE_13B, LE_13C, LE_13D, LE_13E, - LE_140, LE_141, LE_142, LE_143, LE_144, LE_145, LE_146, LE_147, LE_148, - LE_1499, - LE_OTHER -} le_version_t; -static le_version_t decode_libevent_version(const char *v, int *bincompat_out); -#if defined(HAVE_EVENT_GET_VERSION) && defined(HAVE_EVENT_GET_METHOD) -static void check_libevent_version(const char *m, int server); -#endif /** Magic value for or_options_t. */ #define OR_OPTIONS_MAGIC 9090909 @@ -726,7 +580,6 @@ static config_format_t options_format = { _option_abbrevs, _option_vars, (validate_fn_t)options_validate, - options_description, NULL }; @@ -747,7 +600,6 @@ static config_format_t state_format = { _state_abbrevs, _state_vars, (validate_fn_t)or_state_validate, - state_description, &state_extra_var, }; @@ -812,13 +664,13 @@ set_options(or_options_t *new_val, char **msg) "Acting on config options left us in a broken state. Dying."); exit(1); } - if (old_options) - config_free(&options_format, old_options); + + config_free(&options_format, old_options); return 0; } -extern const char tor_svn_revision[]; /* from tor_main.c */ +extern const char tor_git_revision[]; /* from tor_main.c */ /** The version of this Tor process, as parsed. */ static char *_version = NULL; @@ -828,10 +680,10 @@ const char * get_version(void) { if (_version == NULL) { - if (strlen(tor_svn_revision)) { - size_t len = strlen(VERSION)+strlen(tor_svn_revision)+16; + if (strlen(tor_git_revision)) { + size_t len = strlen(VERSION)+strlen(tor_git_revision)+16; _version = tor_malloc(len); - tor_snprintf(_version, len, "%s (git-%s)", VERSION, tor_svn_revision); + tor_snprintf(_version, len, "%s (git-%s)", VERSION, tor_git_revision); } else { _version = tor_strdup(VERSION); } @@ -844,8 +696,10 @@ get_version(void) static void or_options_free(or_options_t *options) { - if (options->_ExcludeExitNodesUnion) - routerset_free(options->_ExcludeExitNodesUnion); + if (!options) + return; + + routerset_free(options->_ExcludeExitNodesUnion); config_free(&options_format, options); } @@ -854,43 +708,72 @@ or_options_free(or_options_t *options) void config_free_all(void) { - if (global_options) { - or_options_free(global_options); - global_options = NULL; - } - if (global_state) { - config_free(&state_format, global_state); - global_state = NULL; - } - if (global_cmdline_options) { - config_free_lines(global_cmdline_options); - global_cmdline_options = NULL; - } + or_options_free(global_options); + global_options = NULL; + + config_free(&state_format, global_state); + global_state = NULL; + + config_free_lines(global_cmdline_options); + global_cmdline_options = NULL; + tor_free(torrc_fname); tor_free(_version); tor_free(global_dirfrontpagecontents); } -/** If options->SafeLogging is on, return a not very useful string, - * else return address. +/** Make <b>address</b> -- a piece of information related to our operation as + * a client -- safe to log according to the settings in options->SafeLogging, + * and return it. + * + * (We return "[scrubbed]" if SafeLogging is "1", and address otherwise.) + */ +const char * +safe_str_client(const char *address) +{ + tor_assert(address); + if (get_options()->_SafeLogging == SAFELOG_SCRUB_ALL) + return "[scrubbed]"; + else + return address; +} + +/** Make <b>address</b> -- a piece of information of unspecified sensitivity + * -- safe to log according to the settings in options->SafeLogging, and + * return it. + * + * (We return "[scrubbed]" if SafeLogging is anything besides "0", and address + * otherwise.) */ const char * safe_str(const char *address) { tor_assert(address); - if (get_options()->SafeLogging) + if (get_options()->_SafeLogging != SAFELOG_SCRUB_NONE) return "[scrubbed]"; else return address; } +/** Equivalent to escaped(safe_str_client(address)). See reentrancy note on + * escaped(): don't use this outside the main thread, or twice in the same + * log statement. */ +const char * +escaped_safe_str_client(const char *address) +{ + if (get_options()->_SafeLogging == SAFELOG_SCRUB_ALL) + return "[scrubbed]"; + else + return escaped(address); +} + /** Equivalent to escaped(safe_str(address)). See reentrancy note on * escaped(): don't use this outside the main thread, or twice in the same * log statement. */ const char * escaped_safe_str(const char *address) { - if (get_options()->SafeLogging) + if (get_options()->_SafeLogging != SAFELOG_SCRUB_NONE) return "[scrubbed]"; else return escaped(address); @@ -1089,10 +972,12 @@ options_act_reversible(or_options_t *old_options, char **msg) } /* Launch the listeners. (We do this before we setuid, so we can bind to - * ports under 1024.) */ - if (retry_all_listeners(replaced_listeners, new_listeners) < 0) { - *msg = tor_strdup("Failed to bind one of the listener ports."); - goto rollback; + * ports under 1024.) We don't want to rebind if we're hibernating. */ + if (!we_are_hibernating()) { + if (retry_all_listeners(replaced_listeners, new_listeners) < 0) { + *msg = tor_strdup("Failed to bind one of the listener ports."); + goto rollback; + } } } @@ -1106,6 +991,15 @@ options_act_reversible(or_options_t *old_options, char **msg) } #endif + /* Attempt to lock all current and future memory with mlockall() only once */ + if (options->DisableAllSwap) { + if (tor_mlockall() == -1) { + *msg = tor_strdup("DisableAllSwap failure. Do you have proper " + "permissions?"); + goto done; + } + } + /* Setuid/setgid as appropriate */ if (options->User) { if (switch_id(options->User) != 0) { @@ -1118,11 +1012,9 @@ options_act_reversible(or_options_t *old_options, char **msg) /* Ensure data directory is private; create if possible. */ if (check_private_dir(options->DataDirectory, running_tor ? CPD_CREATE : CPD_CHECK)<0) { - char buf[1024]; - int tmp = tor_snprintf(buf, sizeof(buf), + tor_asprintf(msg, "Couldn't access/create private data directory \"%s\"", options->DataDirectory); - *msg = tor_strdup(tmp >= 0 ? buf : "internal error"); goto done; /* No need to roll back, since you can't change the value. */ } @@ -1133,10 +1025,8 @@ options_act_reversible(or_options_t *old_options, char **msg) tor_snprintf(fn, len, "%s"PATH_SEPARATOR"cached-status", options->DataDirectory); if (check_private_dir(fn, running_tor ? CPD_CREATE : CPD_CHECK) < 0) { - char buf[1024]; - int tmp = tor_snprintf(buf, sizeof(buf), + tor_asprintf(msg, "Couldn't access/create private data directory \"%s\"", fn); - *msg = tor_strdup(tmp >= 0 ? buf : "internal error"); tor_free(fn); goto done; } @@ -1236,7 +1126,6 @@ get_effective_bwrate(or_options_t *options) bw = options->MaxAdvertisedBandwidth; if (options->RelayBandwidthRate > 0 && bw > options->RelayBandwidthRate) bw = options->RelayBandwidthRate; - /* ensure_bandwidth_cap() makes sure that this cast can't overflow. */ return (uint32_t)bw; } @@ -1314,14 +1203,14 @@ options_act(or_options_t *old_options) return 0; /* Finish backgrounding the process */ - if (running_tor && options->RunAsDaemon) { + if (options->RunAsDaemon) { /* We may be calling this for the n'th time (on SIGHUP), but it's safe. */ finish_daemon(options->DataDirectory); } /* Write our PID to the PID file. If we do not have write permissions we * will log a warning */ - if (running_tor && options->PidFile) + if (options->PidFile) write_pidfile(options->PidFile); /* Register addressmap directives */ @@ -1354,30 +1243,75 @@ options_act(or_options_t *old_options) if (accounting_is_enabled(options)) configure_accounting(time(NULL)); + /* parse RefuseUnknownExits tristate */ + if (!strcmp(options->RefuseUnknownExits, "0")) + options->RefuseUnknownExits_ = 0; + else if (!strcmp(options->RefuseUnknownExits, "1")) + options->RefuseUnknownExits_ = 1; + else if (!strcmp(options->RefuseUnknownExits, "auto")) + options->RefuseUnknownExits_ = -1; + else { + /* Should have caught this in options_validate */ + return -1; + } + + /* Change the cell EWMA settings */ + cell_ewma_set_scale_factor(options, networkstatus_get_latest_consensus()); + /* Check for transitions that need action. */ if (old_options) { - if (options->UseEntryGuards && !old_options->UseEntryGuards) { + if ((options->UseEntryGuards && !old_options->UseEntryGuards) || + !routerset_equal(old_options->ExcludeNodes,options->ExcludeNodes) || + !routerset_equal(old_options->ExcludeExitNodes, + options->ExcludeExitNodes) || + !routerset_equal(old_options->EntryNodes, options->EntryNodes) || + !routerset_equal(old_options->ExitNodes, options->ExitNodes) || + options->StrictNodes != old_options->StrictNodes) { log_info(LD_CIRC, - "Switching to entry guards; abandoning previous circuits"); + "Changed to using entry guards, or changed preferred or " + "excluded node lists. Abandoning previous circuits."); circuit_mark_all_unused_circs(); circuit_expire_all_dirty_circs(); + addressmap_clear_excluded_trackexithosts(options); } +/* How long should we delay counting bridge stats after becoming a bridge? + * We use this so we don't count people who used our bridge thinking it is + * a relay. If you change this, don't forget to change the log message + * below. It's 4 hours (the time it takes to stop being used by clients) + * plus some extra time for clock skew. */ +#define RELAY_BRIDGE_STATS_DELAY (6 * 60 * 60) + if (! bool_eq(options->BridgeRelay, old_options->BridgeRelay)) { - log_info(LD_GENERAL, "Bridge status changed. Forgetting GeoIP stats."); - geoip_remove_old_clients(time(NULL)+(2*60*60)); + int was_relay = 0; + if (options->BridgeRelay) { + time_t int_start = time(NULL); + if (old_options->ORPort == options->ORPort) { + int_start += RELAY_BRIDGE_STATS_DELAY; + was_relay = 1; + } + geoip_bridge_stats_init(int_start); + log_info(LD_CONFIG, "We are acting as a bridge now. Starting new " + "GeoIP stats interval%s.", was_relay ? " in 6 " + "hours from now" : ""); + } else { + geoip_bridge_stats_term(); + log_info(LD_GENERAL, "We are no longer acting as a bridge. " + "Forgetting GeoIP stats."); + } } if (options_transition_affects_workers(old_options, options)) { log_info(LD_GENERAL, "Worker-related options changed. Rotating workers."); + + if (init_keys() < 0) { + log_warn(LD_BUG,"Error initializing keys; exiting"); + return -1; + } if (server_mode(options) && !server_mode(old_options)) { - if (init_keys() < 0) { - log_warn(LD_BUG,"Error initializing keys; exiting"); - return -1; - } ip_address_changed(0); - if (has_completed_circuit || !any_predicted_circuits(time(NULL))) + if (can_complete_circuit || !any_predicted_circuits(time(NULL))) inform_testing_reachability(); } cpuworkers_rotate(); @@ -1390,6 +1324,10 @@ options_act(or_options_t *old_options) if (options->V3AuthoritativeDir && !old_options->V3AuthoritativeDir) init_keys(); + + if (options->PerConnBWRate != old_options->PerConnBWRate || + options->PerConnBWBurst != old_options->PerConnBWBurst) + connection_or_update_token_buckets(get_connection_array(), options); } /* Maybe load geoip file */ @@ -1398,7 +1336,7 @@ options_act(or_options_t *old_options) || !geoip_is_loaded())) { /* XXXX Don't use this "<default>" junk; make our filename options * understand prefixes somehow. -NM */ - /* XXXX021 Reload GeoIPFile on SIGHUP. -NM */ + /* XXXX023 Reload GeoIPFile on SIGHUP. -NM */ char *actual_fname = tor_strdup(options->GeoIPFile); #ifdef WIN32 if (!strcmp(actual_fname, "<default>")) { @@ -1412,17 +1350,68 @@ options_act(or_options_t *old_options) geoip_load_file(actual_fname, options); tor_free(actual_fname); } -#ifdef ENABLE_GEOIP_STATS - log_warn(LD_CONFIG, "We are configured to measure GeoIP statistics, but " - "the way these statistics are measured has changed " - "significantly in later versions of Tor. The results may not be " - "as expected if you are used to later versions. Be sure you " - "know what you are doing."); -#endif + + if (options->DirReqStatistics && !geoip_is_loaded()) { + /* Check if GeoIP database could be loaded. */ + log_warn(LD_CONFIG, "Configured to measure directory request " + "statistics, but no GeoIP database found!"); + return -1; + } + + if (options->EntryStatistics) { + if (should_record_bridge_info(options)) { + /* Don't allow measuring statistics on entry guards when configured + * as bridge. */ + log_warn(LD_CONFIG, "Bridges cannot be configured to measure " + "additional GeoIP statistics as entry guards."); + return -1; + } else if (!geoip_is_loaded()) { + /* Check if GeoIP database could be loaded. */ + log_warn(LD_CONFIG, "Configured to measure entry node statistics, " + "but no GeoIP database found!"); + return -1; + } + } + + if (options->CellStatistics || options->DirReqStatistics || + options->EntryStatistics || options->ExitPortStatistics) { + time_t now = time(NULL); + if ((!old_options || !old_options->CellStatistics) && + options->CellStatistics) + rep_hist_buffer_stats_init(now); + if ((!old_options || !old_options->DirReqStatistics) && + options->DirReqStatistics) + geoip_dirreq_stats_init(now); + if ((!old_options || !old_options->EntryStatistics) && + options->EntryStatistics) + geoip_entry_stats_init(now); + if ((!old_options || !old_options->ExitPortStatistics) && + options->ExitPortStatistics) + rep_hist_exit_stats_init(now); + if (!old_options) + log_notice(LD_CONFIG, "Configured to measure statistics. Look for " + "the *-stats files that will first be written to the " + "data directory in 24 hours from now."); + } + + if (old_options && old_options->CellStatistics && + !options->CellStatistics) + rep_hist_buffer_stats_term(); + if (old_options && old_options->DirReqStatistics && + !options->DirReqStatistics) + geoip_dirreq_stats_term(); + if (old_options && old_options->EntryStatistics && + !options->EntryStatistics) + geoip_entry_stats_term(); + if (old_options && old_options->ExitPortStatistics && + !options->ExitPortStatistics) + rep_hist_exit_stats_term(); + /* Check if we need to parse and add the EntryNodes config option. */ if (options->EntryNodes && (!old_options || - (!routerset_equal(old_options->EntryNodes,options->EntryNodes)))) + !routerset_equal(old_options->EntryNodes,options->EntryNodes) || + !routerset_equal(old_options->ExcludeNodes,options->ExcludeNodes))) entry_nodes_should_be_added(); /* Since our options changed, we might need to regenerate and upload our @@ -1491,7 +1480,10 @@ expand_abbrev(config_format_t *fmt, const char *option, int command_line, fmt->abbrevs[i].abbreviated, fmt->abbrevs[i].full); } - return fmt->abbrevs[i].full; + /* Keep going through the list in case we want to rewrite it more. + * (We could imagine recursing here, but I don't want to get the + * user into an infinite loop if we craft our list wrong.) */ + option = fmt->abbrevs[i].full; } } return option; @@ -1536,7 +1528,10 @@ config_get_commandlines(int argc, char **argv, config_line_t **result) *new = tor_malloc_zero(sizeof(config_line_t)); s = argv[i]; - while (*s == '-') + /* Each keyword may be prefixed with one or two dashes. */ + if (*s == '-') + s++; + if (*s == '-') s++; (*new)->key = tor_strdup(expand_abbrev(&options_format, s, 1, 1)); @@ -1628,19 +1623,6 @@ config_free_lines(config_line_t *front) } } -/** Return the description for a given configuration variable, or NULL if no - * description exists. */ -static const char * -config_find_description(config_format_t *fmt, const char *name) -{ - int i; - for (i=0; fmt->descriptions[i].name; ++i) { - if (!strcasecmp(name, fmt->descriptions[i].name)) - return fmt->descriptions[i].description; - } - return NULL; -} - /** If <b>key</b> is a configuration option, return the corresponding * config_var_t. Otherwise, if <b>key</b> is a non-standard abbreviation, * warn, and return the corresponding config_var_t. Otherwise return NULL. @@ -1671,6 +1653,16 @@ config_find_option(config_format_t *fmt, const char *key) return NULL; } +/** Return the number of option entries in <b>fmt</b>. */ +static int +config_count_options(config_format_t *fmt) +{ + int i; + for (i=0; fmt->vars[i].name; ++i) + ; + return i; +} + /* * Functions to assign config options. */ @@ -1684,8 +1676,7 @@ static int config_assign_value(config_format_t *fmt, or_options_t *options, config_line_t *c, char **msg) { - int i, r, ok; - char buf[1024]; + int i, ok; config_var_t *var; void *lvalue; @@ -1701,10 +1692,9 @@ config_assign_value(config_format_t *fmt, or_options_t *options, case CONFIG_TYPE_UINT: i = (int)tor_parse_long(c->value, 10, 0, INT_MAX, &ok, NULL); if (!ok) { - r = tor_snprintf(buf, sizeof(buf), + tor_asprintf(msg, "Int keyword '%s %s' is malformed or out of bounds.", c->key, c->value); - *msg = tor_strdup(r >= 0 ? buf : "internal error"); return -1; } *(int *)lvalue = i; @@ -1713,10 +1703,9 @@ config_assign_value(config_format_t *fmt, or_options_t *options, case CONFIG_TYPE_INTERVAL: { i = config_parse_interval(c->value, &ok); if (!ok) { - r = tor_snprintf(buf, sizeof(buf), + tor_asprintf(msg, "Interval '%s %s' is malformed or out of bounds.", c->key, c->value); - *msg = tor_strdup(r >= 0 ? buf : "internal error"); return -1; } *(int *)lvalue = i; @@ -1726,10 +1715,9 @@ config_assign_value(config_format_t *fmt, or_options_t *options, case CONFIG_TYPE_MEMUNIT: { uint64_t u64 = config_parse_memunit(c->value, &ok); if (!ok) { - r = tor_snprintf(buf, sizeof(buf), + tor_asprintf(msg, "Value '%s %s' is malformed or out of bounds.", c->key, c->value); - *msg = tor_strdup(r >= 0 ? buf : "internal error"); return -1; } *(uint64_t *)lvalue = u64; @@ -1739,10 +1727,9 @@ config_assign_value(config_format_t *fmt, or_options_t *options, case CONFIG_TYPE_BOOL: i = (int)tor_parse_long(c->value, 10, 0, 1, &ok, NULL); if (!ok) { - r = tor_snprintf(buf, sizeof(buf), + tor_asprintf(msg, "Boolean '%s %s' expects 0 or 1.", c->key, c->value); - *msg = tor_strdup(r >= 0 ? buf : "internal error"); return -1; } *(int *)lvalue = i; @@ -1760,9 +1747,8 @@ config_assign_value(config_format_t *fmt, or_options_t *options, case CONFIG_TYPE_ISOTIME: if (parse_iso_time(c->value, (time_t *)lvalue)) { - r = tor_snprintf(buf, sizeof(buf), + tor_asprintf(msg, "Invalid time '%s' for keyword '%s'", c->value, c->key); - *msg = tor_strdup(r >= 0 ? buf : "internal error"); return -1; } break; @@ -1773,9 +1759,8 @@ config_assign_value(config_format_t *fmt, or_options_t *options, } *(routerset_t**)lvalue = routerset_new(); if (routerset_parse(*(routerset_t**)lvalue, c->value, c->key)<0) { - tor_snprintf(buf, sizeof(buf), "Invalid exit list '%s' for option '%s'", + tor_asprintf(msg, "Invalid exit list '%s' for option '%s'", c->value, c->key); - *msg = tor_strdup(buf); return -1; } break; @@ -1800,9 +1785,8 @@ config_assign_value(config_format_t *fmt, or_options_t *options, log_warn(LD_CONFIG, "Skipping obsolete configuration option '%s'", c->key); break; case CONFIG_TYPE_LINELIST_V: - r = tor_snprintf(buf, sizeof(buf), + tor_asprintf(msg, "You may not provide a value for virtual option '%s'", c->key); - *msg = tor_strdup(r >= 0 ? buf : "internal error"); return -1; default: tor_assert(0); @@ -1823,7 +1807,7 @@ config_assign_value(config_format_t *fmt, or_options_t *options, static int config_assign_line(config_format_t *fmt, or_options_t *options, config_line_t *c, int use_defaults, - int clear_first, char **msg) + int clear_first, bitarray_t *options_seen, char **msg) { config_var_t *var; @@ -1838,13 +1822,12 @@ config_assign_line(config_format_t *fmt, or_options_t *options, config_line_append((config_line_t**)lvalue, c->key, c->value); return 0; } else { - char buf[1024]; - int tmp = tor_snprintf(buf, sizeof(buf), + tor_asprintf(msg, "Unknown option '%s'. Failing.", c->key); - *msg = tor_strdup(tmp >= 0 ? buf : "internal error"); return -1; } } + /* Put keyword into canonical case. */ if (strcmp(var->name, c->key)) { tor_free(c->key); @@ -1867,6 +1850,18 @@ config_assign_line(config_format_t *fmt, or_options_t *options, return 0; } + if (options_seen && (var->type != CONFIG_TYPE_LINELIST && + var->type != CONFIG_TYPE_LINELIST_S)) { + /* We're tracking which options we've seen, and this option is not + * supposed to occur more than once. */ + int var_index = (int)(var - fmt->vars); + if (bitarray_is_set(options_seen, var_index)) { + log_warn(LD_CONFIG, "Option '%s' used more than once; all but the last " + "value will be ignored.", var->name); + } + bitarray_set(options_seen, var_index); + } + if (config_assign_value(fmt, options, c, msg) < 0) return -2; return 0; @@ -1967,7 +1962,6 @@ get_assigned_option(config_format_t *fmt, void *options, { config_var_t *var; const void *value; - char buf[32]; config_line_t *result; tor_assert(options && key); @@ -2008,19 +2002,16 @@ get_assigned_option(config_format_t *fmt, void *options, case CONFIG_TYPE_UINT: /* This means every or_options_t uint or bool element * needs to be an int. Not, say, a uint16_t or char. */ - tor_snprintf(buf, sizeof(buf), "%d", *(int*)value); - result->value = tor_strdup(buf); + tor_asprintf(&result->value, "%d", *(int*)value); escape_val = 0; /* Can't need escape. */ break; case CONFIG_TYPE_MEMUNIT: - tor_snprintf(buf, sizeof(buf), U64_FORMAT, + tor_asprintf(&result->value, U64_FORMAT, U64_PRINTF_ARG(*(uint64_t*)value)); - result->value = tor_strdup(buf); escape_val = 0; /* Can't need escape. */ break; case CONFIG_TYPE_DOUBLE: - tor_snprintf(buf, sizeof(buf), "%f", *(double*)value); - result->value = tor_strdup(buf); + tor_asprintf(&result->value, "%f", *(double*)value); escape_val = 0; /* Can't need escape. */ break; case CONFIG_TYPE_BOOL: @@ -2139,6 +2130,8 @@ config_assign(config_format_t *fmt, void *options, config_line_t *list, int use_defaults, int clear_first, char **msg) { config_line_t *p; + bitarray_t *options_seen; + const int n_options = config_count_options(fmt); CHECK(fmt, options); @@ -2158,14 +2151,18 @@ config_assign(config_format_t *fmt, void *options, config_line_t *list, config_reset_line(fmt, options, p->key, use_defaults); } + options_seen = bitarray_init_zero(n_options); /* pass 3: assign. */ while (list) { int r; if ((r=config_assign_line(fmt, options, list, use_defaults, - clear_first, msg))) + clear_first, options_seen, msg))) { + bitarray_free(options_seen); return r; + } list = list->next; } + bitarray_free(options_seen); return 0; } @@ -2308,20 +2305,10 @@ list_torrc_options(void) smartlist_t *lines = smartlist_create(); for (i = 0; _option_vars[i].name; ++i) { config_var_t *var = &_option_vars[i]; - const char *desc; if (var->type == CONFIG_TYPE_OBSOLETE || var->type == CONFIG_TYPE_LINELIST_V) continue; - desc = config_find_description(&options_format, var->name); printf("%s\n", var->name); - if (desc) { - wrap_string(lines, desc, 76, " ", " "); - SMARTLIST_FOREACH(lines, char *, cp, { - printf("%s", cp); - tor_free(cp); - }); - smartlist_clear(lines); - } } smartlist_free(lines); } @@ -2340,7 +2327,7 @@ resolve_my_address(int warn_severity, or_options_t *options, uint32_t *addr_out, char **hostname_out) { struct in_addr in; - uint32_t addr; + uint32_t addr; /* host order */ char hostname[256]; int explicit_ip=1; int explicit_hostname=1; @@ -2370,8 +2357,8 @@ resolve_my_address(int warn_severity, or_options_t *options, if (tor_inet_aton(hostname, &in) == 0) { /* then we have to resolve it */ explicit_ip = 0; - if (tor_lookup_hostname(hostname, &addr)) { - uint32_t interface_ip; + if (tor_lookup_hostname(hostname, &addr)) { /* failed to resolve */ + uint32_t interface_ip; /* host order */ if (explicit_hostname) { log_fn(warn_severity, LD_CONFIG, @@ -2392,7 +2379,7 @@ resolve_my_address(int warn_severity, or_options_t *options, log_fn(notice_severity, LD_CONFIG, "Learned IP address '%s' for " "local interface. Using that.", tmpbuf); strlcpy(hostname, "<guessed from interfaces>", sizeof(hostname)); - } else { + } else { /* resolved hostname into addr */ in.s_addr = htonl(addr); if (!explicit_hostname && @@ -2495,7 +2482,7 @@ is_local_addr(const tor_addr_t *addr) if (get_options()->EnforceDistinctSubnets == 0) return 0; if (tor_addr_family(addr) == AF_INET) { - /*XXXX022 IP6 what corresponds to an /24? */ + /*XXXX023 IP6 what corresponds to an /24? */ uint32_t ip = tor_addr_to_ipv4h(addr); /* It's possible that this next check will hit before the first time @@ -2512,61 +2499,16 @@ is_local_addr(const tor_addr_t *addr) return 0; } -/** Called when we don't have a nickname set. Try to guess a good nickname - * based on the hostname, and return it in a newly allocated string. If we - * can't, return NULL and let the caller warn if it wants to. */ -static char * -get_default_nickname(void) -{ - static const char * const bad_default_nicknames[] = { - "localhost", - NULL, - }; - char localhostname[256]; - char *cp, *out, *outp; - int i; - - if (gethostname(localhostname, sizeof(localhostname)) < 0) - return NULL; - - /* Put it in lowercase; stop at the first dot. */ - if ((cp = strchr(localhostname, '.'))) - *cp = '\0'; - tor_strlower(localhostname); - - /* Strip invalid characters. */ - cp = localhostname; - out = outp = tor_malloc(strlen(localhostname) + 1); - while (*cp) { - if (strchr(LEGAL_NICKNAME_CHARACTERS, *cp)) - *outp++ = *cp++; - else - cp++; - } - *outp = '\0'; - - /* Enforce length. */ - if (strlen(out) > MAX_NICKNAME_LEN) - out[MAX_NICKNAME_LEN]='\0'; - - /* Check for dumb names. */ - for (i = 0; bad_default_nicknames[i]; ++i) { - if (!strcmp(out, bad_default_nicknames[i])) { - tor_free(out); - return NULL; - } - } - - return out; -} - /** Release storage held by <b>options</b>. */ static void config_free(config_format_t *fmt, void *options) { int i; - tor_assert(options); + if (!options) + return; + + tor_assert(fmt); for (i=0; fmt->vars[i].name; ++i) option_clear(fmt, options, &(fmt->vars[i])); @@ -2668,6 +2610,8 @@ is_listening_on_low_port(uint16_t port_option, const config_line_t *listen_options) { #ifdef MS_WINDOWS + (void) port_option; + (void) listen_options; return 0; /* No port is too low for windows. */ #else const config_line_t *l; @@ -2717,7 +2661,6 @@ config_dump(config_format_t *fmt, void *options, int minimal, config_line_t *line, *assigned; char *result; int i; - const char *desc; char *msg = NULL; defaults = config_alloc(fmt); @@ -2745,24 +2688,13 @@ config_dump(config_format_t *fmt, void *options, int minimal, option_is_same(fmt, options, defaults, fmt->vars[i].name)) comment_option = 1; - desc = config_find_description(fmt, fmt->vars[i].name); line = assigned = get_assigned_option(fmt, options, fmt->vars[i].name, 1); - if (line && desc) { - /* Only dump the description if there's something to describe. */ - wrap_string(elements, desc, 78, "# ", "# "); - } - for (; line; line = line->next) { - size_t len = strlen(line->key) + strlen(line->value) + 5; char *tmp; - tmp = tor_malloc(len); - if (tor_snprintf(tmp, len, "%s%s %s\n", - comment_option ? "# " : "", - line->key, line->value)<0) { - log_err(LD_BUG,"Internal error writing option value"); - tor_assert(0); - } + tor_asprintf(&tmp, "%s%s %s\n", + comment_option ? "# " : "", + line->key, line->value); smartlist_add(elements, tmp); } config_free_lines(assigned); @@ -2771,13 +2703,8 @@ config_dump(config_format_t *fmt, void *options, int minimal, if (fmt->extra) { line = *(config_line_t**)STRUCT_VAR_P(options, fmt->extra->var_offset); for (; line; line = line->next) { - size_t len = strlen(line->key) + strlen(line->value) + 3; char *tmp; - tmp = tor_malloc(len); - if (tor_snprintf(tmp, len, "%s %s\n", line->key, line->value)<0) { - log_err(LD_BUG,"Internal error writing option value"); - tor_assert(0); - } + tor_asprintf(&tmp, "%s %s\n", line->key, line->value); smartlist_add(elements, tmp); } } @@ -2793,7 +2720,7 @@ config_dump(config_format_t *fmt, void *options, int minimal, * the configuration in <b>options</b>. If <b>minimal</b> is true, do not * include options that are the same as Tor's defaults. */ -static char * +char * options_dump(or_options_t *options, int minimal) { return config_dump(&options_format, options, minimal, 0); @@ -2806,7 +2733,6 @@ static int validate_ports_csv(smartlist_t *sl, const char *name, char **msg) { int i; - char buf[1024]; tor_assert(name); if (!sl) @@ -2816,9 +2742,7 @@ validate_ports_csv(smartlist_t *sl, const char *name, char **msg) { i = atoi(cp); if (i < 1 || i > 65535) { - int r = tor_snprintf(buf, sizeof(buf), - "Port '%s' out of range in %s", cp, name); - *msg = tor_strdup(r >= 0 ? buf : "internal error"); + tor_asprintf(msg, "Port '%s' out of range in %s", cp, name); return -1; } }); @@ -2832,18 +2756,15 @@ validate_ports_csv(smartlist_t *sl, const char *name, char **msg) static int ensure_bandwidth_cap(uint64_t *value, const char *desc, char **msg) { - int r; - char buf[1024]; if (*value > ROUTER_MAX_DECLARED_BANDWIDTH) { /* This handles an understandable special case where somebody says "2gb" * whereas our actual maximum is 2gb-1 (INT_MAX) */ --*value; } if (*value > ROUTER_MAX_DECLARED_BANDWIDTH) { - r = tor_snprintf(buf, sizeof(buf), "%s ("U64_FORMAT") must be at most %d", - desc, U64_PRINTF_ARG(*value), - ROUTER_MAX_DECLARED_BANDWIDTH); - *msg = tor_strdup(r >= 0 ? buf : "internal error"); + tor_asprintf(msg, "%s ("U64_FORMAT") must be at most %d", + desc, U64_PRINTF_ARG(*value), + ROUTER_MAX_DECLARED_BANDWIDTH); return -1; } return 0; @@ -2896,15 +2817,14 @@ compute_publishserverdescriptor(or_options_t *options) /** Highest allowable value for RendPostPeriod. */ #define MAX_DIR_PERIOD (MIN_ONION_KEY_LIFETIME/2) -/** Lowest allowable value for CircuitBuildTimeout; values too low will - * increase network load because of failing connections being retried, and - * might prevent users from connecting to the network at all. */ -#define MIN_CIRCUIT_BUILD_TIMEOUT 30 - /** Lowest allowable value for MaxCircuitDirtiness; if this is too low, Tor * will generate too many circuits and potentially overload the network. */ #define MIN_MAX_CIRCUIT_DIRTINESS 10 +/** Lowest allowable value for CircuitStreamTimeout; if this is too low, Tor + * will generate too many circuits and potentially overload the network. */ +#define MIN_CIRCUIT_STREAM_TIMEOUT 10 + /** Return 0 if every setting in <b>options</b> is reasonable, and a * permissible transition from <b>old_options</b>. Else return -1. * Should have no side effects, except for normalizing the contents of @@ -2921,10 +2841,9 @@ static int options_validate(or_options_t *old_options, or_options_t *options, int from_setconf, char **msg) { - int i, r; + int i; config_line_t *cl; const char *uname = get_uname(); - char buf[1024]; #define REJECT(arg) \ STMT_BEGIN *msg = tor_strdup(arg); return -1; STMT_END #define COMPLAIN(arg) STMT_BEGIN log(LOG_WARN, LD_CONFIG, arg); STMT_END @@ -2941,7 +2860,7 @@ options_validate(or_options_t *old_options, or_options_t *options, !strcmpstart(uname, "Windows Me"))) { log(LOG_WARN, LD_CONFIG, "Tor is running as a server, but you are " "running %s; this probably won't work. See " - "http://wiki.noreply.org/noreply/TheOnionRouter/TorFAQ#ServerOS " + "https://wiki.torproject.org/TheOnionRouter/TorFAQ#ServerOS " "for details.", uname); } @@ -2960,8 +2879,8 @@ options_validate(or_options_t *old_options, or_options_t *options, if (options->TransPort == 0 && options->TransListenAddress != NULL) REJECT("TransPort must be defined if TransListenAddress is defined."); - if (options->NatdPort == 0 && options->NatdListenAddress != NULL) - REJECT("NatdPort must be defined if NatdListenAddress is defined."); + if (options->NATDPort == 0 && options->NATDListenAddress != NULL) + REJECT("NATDPort must be defined if NATDListenAddress is defined."); /* Don't gripe about SocksPort 0 with SocksListenAddress set; a standard * configuration does this. */ @@ -2980,8 +2899,8 @@ options_validate(or_options_t *old_options, or_options_t *options, old = old_options ? old_options->TransListenAddress : NULL; tp = "transparent proxy"; } else { - opt = options->NatdListenAddress; - old = old_options ? old_options->NatdListenAddress : NULL; + opt = options->NATDListenAddress; + old = old_options ? old_options->NATDListenAddress : NULL; tp = "natd proxy"; } @@ -3008,21 +2927,13 @@ options_validate(or_options_t *old_options, or_options_t *options, if (options->Nickname == NULL) { if (server_mode(options)) { - if (!(options->Nickname = get_default_nickname())) { - log_notice(LD_CONFIG, "Couldn't pick a nickname based on " - "our hostname; using %s instead.", UNNAMED_ROUTER_NICKNAME); options->Nickname = tor_strdup(UNNAMED_ROUTER_NICKNAME); - } else { - log_notice(LD_CONFIG, "Choosing default nickname '%s'", - options->Nickname); - } } } else { if (!is_legal_nickname(options->Nickname)) { - r = tor_snprintf(buf, sizeof(buf), + tor_asprintf(msg, "Nickname '%s' is wrong length or contains illegal characters.", options->Nickname); - *msg = tor_strdup(r >= 0 ? buf : "internal error"); return -1; } } @@ -3039,14 +2950,6 @@ options_validate(or_options_t *old_options, or_options_t *options, if (options_init_logs(options, 1)<0) /* Validate the log(s) */ REJECT("Failed to validate Log options. See logs for details."); - if (options->NoPublish) { - log(LOG_WARN, LD_CONFIG, - "NoPublish is obsolete. Use PublishServerDescriptor instead."); - SMARTLIST_FOREACH(options->PublishServerDescriptor, char *, s, - tor_free(s)); - smartlist_clear(options->PublishServerDescriptor); - } - if (authdir_mode(options)) { /* confirm that our address isn't broken, so we can complain now */ uint32_t tmp; @@ -3054,6 +2957,12 @@ options_validate(or_options_t *old_options, or_options_t *options, REJECT("Failed to resolve/guess local address. See logs for details."); } + if (strcmp(options->RefuseUnknownExits, "0") && + strcmp(options->RefuseUnknownExits, "1") && + strcmp(options->RefuseUnknownExits, "auto")) { + REJECT("RefuseUnknownExits must be 0, 1, or auto"); + } + #ifndef MS_WINDOWS if (options->RunAsDaemon && torrc_fname && path_is_relative(torrc_fname)) REJECT("Can't use a relative path to torrc when RunAsDaemon is set."); @@ -3068,14 +2977,14 @@ options_validate(or_options_t *old_options, or_options_t *options, if (options->TransPort < 0 || options->TransPort > 65535) REJECT("TransPort option out of bounds."); - if (options->NatdPort < 0 || options->NatdPort > 65535) - REJECT("NatdPort option out of bounds."); + if (options->NATDPort < 0 || options->NATDPort > 65535) + REJECT("NATDPort option out of bounds."); if (options->SocksPort == 0 && options->TransPort == 0 && - options->NatdPort == 0 && options->ORPort == 0 && + options->NATDPort == 0 && options->ORPort == 0 && options->DNSPort == 0 && !options->RendConfigLines) log(LOG_WARN, LD_CONFIG, - "SocksPort, TransPort, NatdPort, DNSPort, and ORPort are all " + "SocksPort, TransPort, NATDPort, DNSPort, and ORPort are all " "undefined, and there aren't any hidden services configured. " "Tor will still run, but probably won't do anything."); @@ -3109,19 +3018,11 @@ options_validate(or_options_t *old_options, or_options_t *options, routerset_union(options->_ExcludeExitNodesUnion,options->ExcludeNodes); } - if (options->StrictExitNodes && - (!options->ExitNodes) && - (!old_options || - (old_options->StrictExitNodes != options->StrictExitNodes) || - (!routerset_equal(old_options->ExitNodes,options->ExitNodes)))) - COMPLAIN("StrictExitNodes set, but no ExitNodes listed."); - - if (options->StrictEntryNodes && - (!options->EntryNodes) && - (!old_options || - (old_options->StrictEntryNodes != options->StrictEntryNodes) || - (!routerset_equal(old_options->EntryNodes,options->EntryNodes)))) - COMPLAIN("StrictEntryNodes set, but no EntryNodes listed."); + if (options->ExcludeNodes && options->StrictNodes) { + COMPLAIN("You have asked to exclude certain relays from all positions " + "in your circuits. Expect hidden services and other Tor " + "features to be broken in unpredictable ways."); + } if (options->EntryNodes && !routerset_is_list(options->EntryNodes)) { /* XXXX fix this; see entry_guards_prepend_from_config(). */ @@ -3159,6 +3060,10 @@ options_validate(or_options_t *old_options, or_options_t *options, options->V3AuthoritativeDir)) REJECT("AuthoritativeDir is set, but none of " "(Bridge/HS/V1/V2/V3)AuthoritativeDir is set."); + /* If we have a v3bandwidthsfile and it's broken, complain on startup */ + if (options->V3BandwidthsFile && !old_options) { + dirserv_read_measured_bandwidths(options->V3BandwidthsFile, NULL); + } } if (options->AuthoritativeDir && !options->DirPort) @@ -3170,15 +3075,18 @@ options_validate(or_options_t *old_options, or_options_t *options, if (options->AuthoritativeDir && options->ClientOnly) REJECT("Running as authoritative directory, but ClientOnly also set."); - if (options->HSAuthorityRecordStats && !options->HSAuthoritativeDir) - REJECT("HSAuthorityRecordStats is set but we're not running as " - "a hidden service authority."); + if (options->FetchDirInfoExtraEarly && !options->FetchDirInfoEarly) + REJECT("FetchDirInfoExtraEarly requires that you also set " + "FetchDirInfoEarly"); + + if (options->HSAuthoritativeDir && proxy_mode(options)) + REJECT("Running as authoritative v0 HS directory, but also configured " + "as a client."); if (options->ConnLimit <= 0) { - r = tor_snprintf(buf, sizeof(buf), + tor_asprintf(msg, "ConnLimit must be greater than 0, but was set to %d", options->ConnLimit); - *msg = tor_strdup(r >= 0 ? buf : "internal error"); return -1; } @@ -3283,6 +3191,12 @@ options_validate(or_options_t *old_options, or_options_t *options, REJECT("Servers must be able to freely connect to the rest " "of the Internet, so they must not set UseBridges."); + /* If both of these are set, we'll end up with funny behavior where we + * demand enough entrynodes be up and running else we won't build + * circuits, yet we never actually use them. */ + if (options->UseBridges && options->EntryNodes) + REJECT("You cannot set both UseBridges and EntryNodes."); + options->_AllowInvalid = 0; if (options->AllowInvalidNodes) { SMARTLIST_FOREACH(options->AllowInvalidNodes, const char *, cp, { @@ -3297,18 +3211,29 @@ options_validate(or_options_t *old_options, or_options_t *options, else if (!strcasecmp(cp, "rendezvous")) options->_AllowInvalid |= ALLOW_INVALID_RENDEZVOUS; else { - r = tor_snprintf(buf, sizeof(buf), + tor_asprintf(msg, "Unrecognized value '%s' in AllowInvalidNodes", cp); - *msg = tor_strdup(r >= 0 ? buf : "internal error"); return -1; } }); } + if (!options->SafeLogging || + !strcasecmp(options->SafeLogging, "0")) { + options->_SafeLogging = SAFELOG_SCRUB_NONE; + } else if (!strcasecmp(options->SafeLogging, "relay")) { + options->_SafeLogging = SAFELOG_SCRUB_RELAY; + } else if (!strcasecmp(options->SafeLogging, "1")) { + options->_SafeLogging = SAFELOG_SCRUB_ALL; + } else { + tor_asprintf(msg, + "Unrecognized value '%s' in SafeLogging", + escaped(options->SafeLogging)); + return -1; + } + if (compute_publishserverdescriptor(options) < 0) { - r = tor_snprintf(buf, sizeof(buf), - "Unrecognized value in PublishServerDescriptor"); - *msg = tor_strdup(r >= 0 ? buf : "internal error"); + tor_asprintf(msg, "Unrecognized value in PublishServerDescriptor"); return -1; } @@ -3321,6 +3246,12 @@ options_validate(or_options_t *old_options, or_options_t *options, "PublishServerDescriptor line."); } + if (options->BridgeRelay && options->DirPort) { + log_warn(LD_CONFIG, "Can't set a DirPort on a bridge relay; disabling " + "DirPort"); + options->DirPort = 0; + } + if (options->MinUptimeHidServDirectoryV2 < 0) { log_warn(LD_CONFIG, "MinUptimeHidServDirectoryV2 option must be at " "least 0 seconds. Changing to 0."); @@ -3328,29 +3259,30 @@ options_validate(or_options_t *old_options, or_options_t *options, } if (options->RendPostPeriod < MIN_REND_POST_PERIOD) { - log(LOG_WARN,LD_CONFIG,"RendPostPeriod option is too short; " - "raising to %d seconds.", MIN_REND_POST_PERIOD); + log_warn(LD_CONFIG, "RendPostPeriod option is too short; " + "raising to %d seconds.", MIN_REND_POST_PERIOD); options->RendPostPeriod = MIN_REND_POST_PERIOD; } if (options->RendPostPeriod > MAX_DIR_PERIOD) { - log(LOG_WARN, LD_CONFIG, "RendPostPeriod is too large; clipping to %ds.", - MAX_DIR_PERIOD); + log_warn(LD_CONFIG, "RendPostPeriod is too large; clipping to %ds.", + MAX_DIR_PERIOD); options->RendPostPeriod = MAX_DIR_PERIOD; } - if (options->CircuitBuildTimeout < MIN_CIRCUIT_BUILD_TIMEOUT) { - log(LOG_WARN, LD_CONFIG, "CircuitBuildTimeout option is too short; " - "raising to %d seconds.", MIN_CIRCUIT_BUILD_TIMEOUT); - options->CircuitBuildTimeout = MIN_CIRCUIT_BUILD_TIMEOUT; - } - if (options->MaxCircuitDirtiness < MIN_MAX_CIRCUIT_DIRTINESS) { - log(LOG_WARN, LD_CONFIG, "MaxCircuitDirtiness option is too short; " - "raising to %d seconds.", MIN_MAX_CIRCUIT_DIRTINESS); + log_warn(LD_CONFIG, "MaxCircuitDirtiness option is too short; " + "raising to %d seconds.", MIN_MAX_CIRCUIT_DIRTINESS); options->MaxCircuitDirtiness = MIN_MAX_CIRCUIT_DIRTINESS; } + if (options->CircuitStreamTimeout && + options->CircuitStreamTimeout < MIN_CIRCUIT_STREAM_TIMEOUT) { + log_warn(LD_CONFIG, "CircuitStreamTimeout option is too short; " + "raising to %d seconds.", MIN_CIRCUIT_STREAM_TIMEOUT); + options->CircuitStreamTimeout = MIN_CIRCUIT_STREAM_TIMEOUT; + } + if (options->KeepalivePeriod < 1) REJECT("KeepalivePeriod option must be positive."); @@ -3369,6 +3301,12 @@ options_validate(or_options_t *old_options, or_options_t *options, if (ensure_bandwidth_cap(&options->RelayBandwidthBurst, "RelayBandwidthBurst", msg) < 0) return -1; + if (ensure_bandwidth_cap(&options->PerConnBWRate, + "PerConnBWRate", msg) < 0) + return -1; + if (ensure_bandwidth_cap(&options->PerConnBWBurst, + "PerConnBWBurst", msg) < 0) + return -1; if (options->RelayBandwidthRate && !options->RelayBandwidthBurst) options->RelayBandwidthBurst = options->RelayBandwidthRate; @@ -3377,31 +3315,28 @@ options_validate(or_options_t *old_options, or_options_t *options, if (server_mode(options)) { if (options->BandwidthRate < ROUTER_REQUIRED_MIN_BANDWIDTH) { - r = tor_snprintf(buf, sizeof(buf), + tor_asprintf(msg, "BandwidthRate is set to %d bytes/second. " "For servers, it must be at least %d.", (int)options->BandwidthRate, ROUTER_REQUIRED_MIN_BANDWIDTH); - *msg = tor_strdup(r >= 0 ? buf : "internal error"); return -1; } else if (options->MaxAdvertisedBandwidth < ROUTER_REQUIRED_MIN_BANDWIDTH/2) { - r = tor_snprintf(buf, sizeof(buf), + tor_asprintf(msg, "MaxAdvertisedBandwidth is set to %d bytes/second. " "For servers, it must be at least %d.", (int)options->MaxAdvertisedBandwidth, ROUTER_REQUIRED_MIN_BANDWIDTH/2); - *msg = tor_strdup(r >= 0 ? buf : "internal error"); return -1; } if (options->RelayBandwidthRate && options->RelayBandwidthRate < ROUTER_REQUIRED_MIN_BANDWIDTH) { - r = tor_snprintf(buf, sizeof(buf), + tor_asprintf(msg, "RelayBandwidthRate is set to %d bytes/second. " "For servers, it must be at least %d.", (int)options->RelayBandwidthRate, ROUTER_REQUIRED_MIN_BANDWIDTH); - *msg = tor_strdup(r >= 0 ? buf : "internal error"); return -1; } } @@ -3423,34 +3358,73 @@ options_validate(or_options_t *old_options, or_options_t *options, if (accounting_parse_options(options, 1)<0) REJECT("Failed to parse accounting options. See logs for details."); - if (options->HttpProxy) { /* parse it now */ - if (parse_addr_port(LOG_WARN, options->HttpProxy, NULL, - &options->HttpProxyAddr, &options->HttpProxyPort) < 0) - REJECT("HttpProxy failed to parse or resolve. Please fix."); - if (options->HttpProxyPort == 0) { /* give it a default */ - options->HttpProxyPort = 80; + if (options->HTTPProxy) { /* parse it now */ + if (tor_addr_port_parse(options->HTTPProxy, + &options->HTTPProxyAddr, &options->HTTPProxyPort) < 0) + REJECT("HTTPProxy failed to parse or resolve. Please fix."); + if (options->HTTPProxyPort == 0) { /* give it a default */ + options->HTTPProxyPort = 80; } } - if (options->HttpProxyAuthenticator) { - if (strlen(options->HttpProxyAuthenticator) >= 48) - REJECT("HttpProxyAuthenticator is too long (>= 48 chars)."); + if (options->HTTPProxyAuthenticator) { + if (strlen(options->HTTPProxyAuthenticator) >= 48) + REJECT("HTTPProxyAuthenticator is too long (>= 48 chars)."); } - if (options->HttpsProxy) { /* parse it now */ - if (parse_addr_port(LOG_WARN, options->HttpsProxy, NULL, - &options->HttpsProxyAddr, &options->HttpsProxyPort) <0) - REJECT("HttpsProxy failed to parse or resolve. Please fix."); - if (options->HttpsProxyPort == 0) { /* give it a default */ - options->HttpsProxyPort = 443; + if (options->HTTPSProxy) { /* parse it now */ + if (tor_addr_port_parse(options->HTTPSProxy, + &options->HTTPSProxyAddr, &options->HTTPSProxyPort) <0) + REJECT("HTTPSProxy failed to parse or resolve. Please fix."); + if (options->HTTPSProxyPort == 0) { /* give it a default */ + options->HTTPSProxyPort = 443; } } - if (options->HttpsProxyAuthenticator) { - if (strlen(options->HttpsProxyAuthenticator) >= 48) - REJECT("HttpsProxyAuthenticator is too long (>= 48 chars)."); + if (options->HTTPSProxyAuthenticator) { + if (strlen(options->HTTPSProxyAuthenticator) >= 48) + REJECT("HTTPSProxyAuthenticator is too long (>= 48 chars)."); } + if (options->Socks4Proxy) { /* parse it now */ + if (tor_addr_port_parse(options->Socks4Proxy, + &options->Socks4ProxyAddr, + &options->Socks4ProxyPort) <0) + REJECT("Socks4Proxy failed to parse or resolve. Please fix."); + if (options->Socks4ProxyPort == 0) { /* give it a default */ + options->Socks4ProxyPort = 1080; + } + } + + if (options->Socks5Proxy) { /* parse it now */ + if (tor_addr_port_parse(options->Socks5Proxy, + &options->Socks5ProxyAddr, + &options->Socks5ProxyPort) <0) + REJECT("Socks5Proxy failed to parse or resolve. Please fix."); + if (options->Socks5ProxyPort == 0) { /* give it a default */ + options->Socks5ProxyPort = 1080; + } + } + + if (options->Socks4Proxy && options->Socks5Proxy) + REJECT("You cannot specify both Socks4Proxy and SOCKS5Proxy"); + + if (options->Socks5ProxyUsername) { + size_t len; + + len = strlen(options->Socks5ProxyUsername); + if (len < 1 || len > 255) + REJECT("Socks5ProxyUsername must be between 1 and 255 characters."); + + if (!options->Socks5ProxyPassword) + REJECT("Socks5ProxyPassword must be included with Socks5ProxyUsername."); + + len = strlen(options->Socks5ProxyPassword); + if (len < 1 || len > 255) + REJECT("Socks5ProxyPassword must be between 1 and 255 characters."); + } else if (options->Socks5ProxyPassword) + REJECT("Socks5ProxyPassword must be included with Socks5ProxyUsername."); + if (options->HashedControlPassword) { smartlist_t *sl = decode_hashed_passwords(options->HashedControlPassword); if (!sl) { @@ -3514,6 +3488,12 @@ options_validate(or_options_t *old_options, or_options_t *options, "upgrade your Tor controller as soon as possible."); } + if (options->CookieAuthFileGroupReadable && !options->CookieAuthFile) { + log_warn(LD_CONFIG, "CookieAuthFileGroupReadable is set, but will have " + "no effect: you must specify an explicit CookieAuthFile to " + "have it group-readable."); + } + if (options->UseEntryGuards && ! options->NumEntryGuards) REJECT("Cannot enable UseEntryGuards with NumEntryGuards set to 0"); @@ -3547,11 +3527,10 @@ options_validate(or_options_t *old_options, or_options_t *options, if (options->ConstrainedSockSize < MIN_CONSTRAINED_TCP_BUFFER || options->ConstrainedSockSize > MAX_CONSTRAINED_TCP_BUFFER || options->ConstrainedSockSize % 1024) { - r = tor_snprintf(buf, sizeof(buf), + tor_asprintf(msg, "ConstrainedSockSize is invalid. Must be a value between %d and %d " "in 1024 byte increments.", MIN_CONSTRAINED_TCP_BUFFER, MAX_CONSTRAINED_TCP_BUFFER); - *msg = tor_strdup(r >= 0 ? buf : "internal error"); return -1; } if (options->DirPort) { @@ -3599,6 +3578,12 @@ options_validate(or_options_t *old_options, or_options_t *options, if (options->PreferTunneledDirConns && !options->TunnelDirConns) REJECT("Must set TunnelDirConns if PreferTunneledDirConns is set."); + if ((options->Socks4Proxy || options->Socks5Proxy) && + !options->HTTPProxy && !options->PreferTunneledDirConns) + REJECT("When Socks4Proxy or Socks5Proxy is configured, " + "PreferTunneledDirConns and TunnelDirConns must both be " + "set to 1, or HTTPProxy must be configured."); + if (options->AutomapHostsSuffixes) { SMARTLIST_FOREACH(options->AutomapHostsSuffixes, char *, suf, { @@ -3613,12 +3598,19 @@ options_validate(or_options_t *old_options, or_options_t *options, "a non-default set of DirServers."); } - /*XXXX022 checking for defaults manually like this is a bit fragile.*/ + if (options->AllowSingleHopExits && !options->DirServers) { + COMPLAIN("You have set AllowSingleHopExits; now your relay will allow " + "others to make one-hop exits. However, since by default most " + "clients avoid relays that set this option, most clients will " + "ignore you."); + } + + /*XXXX023 checking for defaults manually like this is a bit fragile.*/ /* Keep changes to hard-coded values synchronous to man page and default * values table. */ if (options->TestingV3AuthInitialVotingInterval != 30*60 && - !options->TestingTorNetwork) { + !options->TestingTorNetwork && !options->_UsingTestNetworkDefaults) { REJECT("TestingV3AuthInitialVotingInterval may only be changed in testing " "Tor networks!"); } else if (options->TestingV3AuthInitialVotingInterval < MIN_VOTE_INTERVAL) { @@ -3629,7 +3621,8 @@ options_validate(or_options_t *old_options, or_options_t *options, } if (options->TestingV3AuthInitialVoteDelay != 5*60 && - !options->TestingTorNetwork) { + !options->TestingTorNetwork && !options->_UsingTestNetworkDefaults) { + REJECT("TestingV3AuthInitialVoteDelay may only be changed in testing " "Tor networks!"); } else if (options->TestingV3AuthInitialVoteDelay < MIN_VOTE_SECONDS) { @@ -3637,7 +3630,7 @@ options_validate(or_options_t *old_options, or_options_t *options, } if (options->TestingV3AuthInitialDistDelay != 5*60 && - !options->TestingTorNetwork) { + !options->TestingTorNetwork && !options->_UsingTestNetworkDefaults) { REJECT("TestingV3AuthInitialDistDelay may only be changed in testing " "Tor networks!"); } else if (options->TestingV3AuthInitialDistDelay < MIN_DIST_SECONDS) { @@ -3652,7 +3645,7 @@ options_validate(or_options_t *old_options, or_options_t *options, } if (options->TestingAuthDirTimeToLearnReachability != 30*60 && - !options->TestingTorNetwork) { + !options->TestingTorNetwork && !options->_UsingTestNetworkDefaults) { REJECT("TestingAuthDirTimeToLearnReachability may only be changed in " "testing Tor networks!"); } else if (options->TestingAuthDirTimeToLearnReachability < 0) { @@ -3662,7 +3655,7 @@ options_validate(or_options_t *old_options, or_options_t *options, } if (options->TestingEstimatedDescriptorPropagationTime != 10*60 && - !options->TestingTorNetwork) { + !options->TestingTorNetwork && !options->_UsingTestNetworkDefaults) { REJECT("TestingEstimatedDescriptorPropagationTime may only be changed in " "testing Tor networks!"); } else if (options->TestingEstimatedDescriptorPropagationTime < 0) { @@ -3678,6 +3671,26 @@ options_validate(or_options_t *old_options, or_options_t *options, "testing Tor network!"); } + if (options->AccelName && !options->HardwareAccel) + options->HardwareAccel = 1; + if (options->AccelDir && !options->AccelName) + REJECT("Can't use hardware crypto accelerator dir without engine name."); + + if (options->PublishServerDescriptor) + SMARTLIST_FOREACH(options->PublishServerDescriptor, const char *, pubdes, { + if (!strcmp(pubdes, "1") || !strcmp(pubdes, "0")) + if (smartlist_len(options->PublishServerDescriptor) > 1) { + COMPLAIN("You have passed a list of multiple arguments to the " + "PublishServerDescriptor option that includes 0 or 1. " + "0 or 1 should only be used as the sole argument. " + "This configuration will be rejected in a future release."); + break; + } + }); + + if (options->BridgeRelay == 1 && options->ORPort == 0) + REJECT("BridgeRelay is 1, ORPort is 0. This is an invalid combination."); + return 0; #undef REJECT #undef COMPLAIN @@ -3716,12 +3729,10 @@ options_transition_allowed(or_options_t *old, or_options_t *new_val, } if (strcmp(old->DataDirectory,new_val->DataDirectory)!=0) { - char buf[1024]; - int r = tor_snprintf(buf, sizeof(buf), + tor_asprintf(msg, "While Tor is running, changing DataDirectory " "(\"%s\"->\"%s\") is not allowed.", old->DataDirectory, new_val->DataDirectory); - *msg = tor_strdup(r >= 0 ? buf : "internal error"); return -1; } @@ -3730,19 +3741,22 @@ options_transition_allowed(or_options_t *old, or_options_t *new_val, return -1; } - if (!opt_streq(old->Group, new_val->Group)) { - *msg = tor_strdup("While Tor is running, changing Group is not allowed."); + if ((old->HardwareAccel != new_val->HardwareAccel) + || !opt_streq(old->AccelName, new_val->AccelName) + || !opt_streq(old->AccelDir, new_val->AccelDir)) { + *msg = tor_strdup("While Tor is running, changing OpenSSL hardware " + "acceleration engine is not allowed."); return -1; } - if (old->HardwareAccel != new_val->HardwareAccel) { - *msg = tor_strdup("While Tor is running, changing HardwareAccel is " - "not allowed."); + if (old->TestingTorNetwork != new_val->TestingTorNetwork) { + *msg = tor_strdup("While Tor is running, changing TestingTorNetwork " + "is not allowed."); return -1; } - if (old->TestingTorNetwork != new_val->TestingTorNetwork) { - *msg = tor_strdup("While Tor is running, changing TestingTorNetwork " + if (old->DisableAllSwap != new_val->DisableAllSwap) { + *msg = tor_strdup("While Tor is running, changing DisableAllSwap " "is not allowed."); return -1; } @@ -3757,13 +3771,15 @@ options_transition_affects_workers(or_options_t *old_options, or_options_t *new_options) { if (!opt_streq(old_options->DataDirectory, new_options->DataDirectory) || - old_options->NumCpus != new_options->NumCpus || + old_options->NumCPUs != new_options->NumCPUs || old_options->ORPort != new_options->ORPort || old_options->ServerDNSSearchDomains != new_options->ServerDNSSearchDomains || old_options->SafeLogging != new_options->SafeLogging || old_options->ClientOnly != new_options->ClientOnly || - !config_lines_eq(old_options->Logs, new_options->Logs)) + public_server_mode(old_options) != public_server_mode(new_options) || + !config_lines_eq(old_options->Logs, new_options->Logs) || + old_options->LogMessageDomains != new_options->LogMessageDomains) return 1; /* Check whether log options match. */ @@ -3789,7 +3805,6 @@ options_transition_affects_descriptor(or_options_t *old_options, old_options->ORPort != new_options->ORPort || old_options->DirPort != new_options->DirPort || old_options->ClientOnly != new_options->ClientOnly || - old_options->NoPublish != new_options->NoPublish || old_options->_PublishServerDescriptor != new_options->_PublishServerDescriptor || get_effective_bwrate(old_options) != get_effective_bwrate(new_options) || @@ -3812,6 +3827,7 @@ get_windows_conf_root(void) { static int is_set = 0; static char path[MAX_PATH+1]; + TCHAR tpath[MAX_PATH] = {0}; LPITEMIDLIST idl; IMalloc *m; @@ -3829,7 +3845,7 @@ get_windows_conf_root(void) #define APPDATA_PATH CSIDL_APPDATA #endif if (!SUCCEEDED(SHGetSpecialFolderLocation(NULL, APPDATA_PATH, &idl))) { - GetCurrentDirectory(MAX_PATH, path); + getcwd(path,MAX_PATH); is_set = 1; log_warn(LD_CONFIG, "I couldn't find your application data folder: are you " @@ -3838,8 +3854,15 @@ get_windows_conf_root(void) return path; } /* Convert the path from an "ID List" (whatever that is!) to a path. */ - result = SHGetPathFromIDList(idl, path); - /* Now we need to free the */ + result = SHGetPathFromIDList(idl, tpath); +#ifdef UNICODE + wcstombs(path,tpath,MAX_PATH); +#else + strlcpy(path,tpath,sizeof(path)); +#endif + + /* Now we need to free the memory that the path-idl was stored in. In + * typical Windows fashion, we can't just call 'free()' on it. */ SHGetMalloc(&m); if (m) { m->lpVtbl->Free(m, idl); @@ -3887,10 +3910,7 @@ check_nickname_list(const char *lst, const char *name, char **msg) SMARTLIST_FOREACH(sl, const char *, s, { if (!is_legal_nickname_or_hexdigest(s)) { - char buf[1024]; - int tmp = tor_snprintf(buf, sizeof(buf), - "Invalid nickname '%s' in %s line", s, name); - *msg = tor_strdup(tmp >= 0 ? buf : "internal error"); + tor_asprintf(msg, "Invalid nickname '%s' in %s line", s, name); r = -1; break; } @@ -3914,13 +3934,7 @@ find_torrc_filename(int argc, char **argv, log(LOG_WARN, LD_CONFIG, "Duplicate -f options on command line."); tor_free(fname); } -#ifdef MS_WINDOWS - /* XXX one day we might want to extend expand_filename to work - * under Windows as well. */ - fname = tor_strdup(argv[i+1]); -#else fname = expand_filename(argv[i+1]); -#endif *using_default_torrc = 0; ++i; } else if (!strcmp(argv[i],"--ignore-missing-torrc")) { @@ -4026,6 +4040,12 @@ options_init_from_torrc(int argc, char **argv) printf("Tor version %s.\n",get_version()); exit(0); } + if (argc > 1 && (!strcmp(argv[1],"--digests"))) { + printf("Tor version %s.\n",get_version()); + printf("%s", libor_get_digests()); + printf("%s", tor_get_digests()); + exit(0); + } /* Go through command-line variables */ if (!global_cmdline_options) { @@ -4193,12 +4213,9 @@ options_init_from_string(const char *cf, err: config_free(&options_format, newoptions); if (*msg) { - int len = (int)strlen(*msg)+256; - char *newmsg = tor_malloc(len); - - tor_snprintf(newmsg, len, "Failed to parse/validate config: %s", *msg); - tor_free(*msg); - *msg = newmsg; + char *old_msg = *msg; + tor_asprintf(msg, "Failed to parse/validate config: %s", old_msg); + tor_free(old_msg); } return err; } @@ -4318,11 +4335,13 @@ options_init_logs(or_options_t *options, int validate_only) if (smartlist_len(elts) == 2 && !strcasecmp(smartlist_get(elts,0), "file")) { if (!validate_only) { - if (add_file_log(severity, smartlist_get(elts, 1)) < 0) { + char *fname = expand_filename(smartlist_get(elts, 1)); + if (add_file_log(severity, fname) < 0) { log_warn(LD_CONFIG, "Couldn't open file for 'Log %s': %s", opt->value, strerror(errno)); ok = 0; } + tor_free(fname); } goto cleanup; } @@ -4338,6 +4357,9 @@ options_init_logs(or_options_t *options, int validate_only) } smartlist_free(elts); + if (ok && !validate_only) + logs_set_domain_logging(options->LogMessageDomains); + return ok?0:-1; } @@ -4586,7 +4608,7 @@ normalize_data_directory(or_options_t *options) } /** Check and normalize the value of options->DataDirectory; return 0 if it - * sane, -1 otherwise. */ + * is sane, -1 otherwise. */ static int validate_data_directory(or_options_t *options) { @@ -4618,7 +4640,6 @@ write_configuration_file(const char *fname, or_options_t *options) { char *old_val=NULL, *new_val=NULL, *new_conf=NULL; int rename_old = 0, r; - size_t len; tor_assert(fname); @@ -4645,9 +4666,7 @@ write_configuration_file(const char *fname, or_options_t *options) goto err; } - len = strlen(new_conf)+256; - new_val = tor_malloc(len); - tor_snprintf(new_val, len, "%s\n%s\n\n%s", + tor_asprintf(&new_val, "%s\n%s\n\n%s", GENERATED_FILE_PREFIX, GENERATED_FILE_COMMENT, new_conf); if (rename_old) { @@ -4697,22 +4716,19 @@ write_configuration_file(const char *fname, or_options_t *options) int options_save_current(void) { - if (torrc_fname) { - /* This fails if we can't write to our configuration file. - * - * If we try falling back to datadirectory or something, we have a better - * chance of saving the configuration, but a better chance of doing - * something the user never expected. Let's just warn instead. */ - return write_configuration_file(torrc_fname, get_options()); - } - return write_configuration_file(get_default_conf_file(), get_options()); + /* This fails if we can't write to our configuration file. + * + * If we try falling back to datadirectory or something, we have a better + * chance of saving the configuration, but a better chance of doing + * something the user never expected. */ + return write_configuration_file(get_torrc_fname(), get_options()); } /** Mapping from a unit name to a multiplier for converting that unit into a - * base unit. */ + * base unit. Used by config_parse_unit. */ struct unit_table_t { - const char *unit; - uint64_t multiplier; + const char *unit; /**< The name of the unit */ + uint64_t multiplier; /**< How many of the base unit appear in this unit */ }; /** Table to map the names of memory units to the number of bytes they @@ -4770,30 +4786,47 @@ static struct unit_table_t time_units[] = { static uint64_t config_parse_units(const char *val, struct unit_table_t *u, int *ok) { - uint64_t v; + uint64_t v = 0; + double d = 0; + int use_float = 0; char *cp; tor_assert(ok); v = tor_parse_uint64(val, 10, 0, UINT64_MAX, ok, &cp); - if (!*ok) - return 0; + if (!*ok || (cp && *cp == '.')) { + d = tor_parse_double(val, 0, UINT64_MAX, ok, &cp); + if (!*ok) + goto done; + use_float = 1; + } + if (!cp) { *ok = 1; - return v; + v = use_float ? DBL_TO_U64(d) : v; + goto done; } - while (TOR_ISSPACE(*cp)) - ++cp; + + cp = (char*) eat_whitespace(cp); + for ( ;u->unit;++u) { if (!strcasecmp(u->unit, cp)) { - v *= u->multiplier; + if (use_float) + v = u->multiplier * d; + else + v *= u->multiplier; *ok = 1; - return v; + goto done; } } log_warn(LD_CONFIG, "Unknown unit '%s'.", cp); *ok = 0; - return 0; + done: + + if (*ok) + return v; + else + return 0; } /** Parse a string in the format "number unit", where unit is a unit of @@ -4803,7 +4836,8 @@ config_parse_units(const char *val, struct unit_table_t *u, int *ok) static uint64_t config_parse_memunit(const char *s, int *ok) { - return config_parse_units(s, memory_units, ok); + uint64_t u = config_parse_units(s, memory_units, ok); + return u; } /** Parse a string in the format "number unit", where unit is a unit of time. @@ -4825,256 +4859,37 @@ config_parse_interval(const char *s, int *ok) return (int)r; } -/* This is what passes for version detection on OSX. We set - * MACOSX_KQUEUE_IS_BROKEN to true iff we're on a version of OSX before - * 10.4.0 (aka 1040). */ -#ifdef __APPLE__ -#ifdef __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ -#define MACOSX_KQUEUE_IS_BROKEN \ - (__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 1040) -#else -#define MACOSX_KQUEUE_IS_BROKEN 0 -#endif -#endif - /** * Initialize the libevent library. */ static void init_libevent(void) { + const char *badness=NULL; + configure_libevent_logging(); /* If the kernel complains that some method (say, epoll) doesn't * exist, we don't care about it, since libevent will cope. */ suppress_libevent_log_msg("Function not implemented"); -#ifdef __APPLE__ - if (MACOSX_KQUEUE_IS_BROKEN || - decode_libevent_version(event_get_version(), NULL) < LE_11B) { - setenv("EVENT_NOKQUEUE","1",1); - } -#endif - /* In libevent versions before 2.0, it's hard to keep binary compatibility - * between upgrades, and unpleasant to detect when the version we compiled - * against is unlike the version we have linked against. Here's how. */ -#if defined(_EVENT_VERSION) && defined(HAVE_EVENT_GET_VERSION) - /* We have a header-file version and a function-call version. Easy. */ - if (strcmp(_EVENT_VERSION, event_get_version())) { - int compat1 = -1, compat2 = -1; - int verybad, prettybad ; - decode_libevent_version(_EVENT_VERSION, &compat1); - decode_libevent_version(event_get_version(), &compat2); - verybad = compat1 != compat2; - prettybad = (compat1 == -1 || compat2 == -1) && compat1 != compat2; - - log(verybad ? LOG_WARN : (prettybad ? LOG_NOTICE : LOG_INFO), - LD_GENERAL, "We were compiled with headers from version %s " - "of Libevent, but we're using a Libevent library that says it's " - "version %s.", _EVENT_VERSION, event_get_version()); - if (verybad) - log_warn(LD_GENERAL, "This will almost certainly make Tor crash."); - else if (prettybad) - log_notice(LD_GENERAL, "If Tor crashes, this might be why."); - else - log_info(LD_GENERAL, "I think these versions are binary-compatible."); - } -#elif defined(HAVE_EVENT_GET_VERSION) - /* event_get_version but no _EVENT_VERSION. We might be in 1.4.0-beta or - earlier, where that's normal. To see whether we were compiled with an - earlier version, let's see whether the struct event defines MIN_HEAP_IDX. - */ -#ifdef HAVE_STRUCT_EVENT_MIN_HEAP_IDX - /* The header files are 1.4.0-beta or later. If the version is not - * 1.4.0-beta, we are incompatible. */ - { - if (strcmp(event_get_version(), "1.4.0-beta")) { - log_warn(LD_GENERAL, "It's a little hard to tell, but you seem to have " - "Libevent 1.4.0-beta header files, whereas you have linked " - "against Libevent %s. This will probably make Tor crash.", - event_get_version()); - } - } -#else - /* Our headers are 1.3e or earlier. If the library version is not 1.4.x or - later, we're probably fine. */ - { - const char *v = event_get_version(); - if ((v[0] == '1' && v[2] == '.' && v[3] > '3') || v[0] > '1') { - log_warn(LD_GENERAL, "It's a little hard to tell, but you seem to have " - "Libevent header file from 1.3e or earlier, whereas you have " - "linked against Libevent %s. This will probably make Tor " - "crash.", event_get_version()); - } - } -#endif + tor_check_libevent_header_compatibility(); -#elif defined(_EVENT_VERSION) -#warn "_EVENT_VERSION is defined but not get_event_version(): Libevent is odd." -#else - /* Your libevent is ancient. */ -#endif + tor_libevent_initialize(); - event_init(); suppress_libevent_log_msg(NULL); -#if defined(HAVE_EVENT_GET_VERSION) && defined(HAVE_EVENT_GET_METHOD) - /* Making this a NOTICE for now so we can link bugs to a libevent versions - * or methods better. */ - log(LOG_NOTICE, LD_GENERAL, - "Initialized libevent version %s using method %s. Good.", - event_get_version(), event_get_method()); - check_libevent_version(event_get_method(), get_options()->ORPort != 0); -#else - log(LOG_NOTICE, LD_GENERAL, - "Initialized old libevent (version 1.0b or earlier)."); - log(LOG_WARN, LD_GENERAL, - "You have a *VERY* old version of libevent. It is likely to be buggy; " - "please build Tor with a more recent version."); -#endif -} - -/** Table mapping return value of event_get_version() to le_version_t. */ -static const struct { - const char *name; le_version_t version; int bincompat; -} le_version_table[] = { - /* earlier versions don't have get_version. */ - { "1.0c", LE_10C, 1}, - { "1.0d", LE_10D, 1}, - { "1.0e", LE_10E, 1}, - { "1.1", LE_11, 1 }, - { "1.1a", LE_11A, 1 }, - { "1.1b", LE_11B, 1 }, - { "1.2", LE_12, 1 }, - { "1.2a", LE_12A, 1 }, - { "1.3", LE_13, 1 }, - { "1.3a", LE_13A, 1 }, - { "1.3b", LE_13B, 1 }, - { "1.3c", LE_13C, 1 }, - { "1.3d", LE_13D, 1 }, - { "1.3e", LE_13E, 1 }, - { "1.4.0-beta", LE_140, 2 }, - { "1.4.1-beta", LE_141, 2 }, - { "1.4.2-rc", LE_142, 2 }, - { "1.4.3-stable", LE_143, 2 }, - { "1.4.4-stable", LE_144, 2 }, - { "1.4.5-stable", LE_145, 2 }, - { "1.4.6-stable", LE_146, 2 }, - { "1.4.7-stable", LE_147, 2 }, - { "1.4.8-stable", LE_148, 2 }, - { "1.4.99-trunk", LE_1499, 3 }, - { NULL, LE_OTHER, 0 } -}; -/** Return the le_version_t for the current version of libevent. If the - * version is very new, return LE_OTHER. If the version is so old that it - * doesn't support event_get_version(), return LE_OLD. */ -static le_version_t -decode_libevent_version(const char *v, int *bincompat_out) -{ - int i; - for (i=0; le_version_table[i].name; ++i) { - if (!strcmp(le_version_table[i].name, v)) { - if (bincompat_out) - *bincompat_out = le_version_table[i].bincompat; - return le_version_table[i].version; - } - } - if (v[0] != '1' && bincompat_out) - *bincompat_out = 100; - else if (!strcmpstart(v, "1.4") && bincompat_out) - *bincompat_out = 2; - return LE_OTHER; -} - -#if defined(HAVE_EVENT_GET_VERSION) && defined(HAVE_EVENT_GET_METHOD) -/** - * Compare the given libevent method and version to a list of versions - * which are known not to work. Warn the user as appropriate. - */ -static void -check_libevent_version(const char *m, int server) -{ - int buggy = 0, iffy = 0, slow = 0, thread_unsafe = 0; - le_version_t version; - const char *v = event_get_version(); - const char *badness = NULL; - const char *sad_os = ""; - - version = decode_libevent_version(v, NULL); - - /* XXX Would it be worthwhile disabling the methods that we know - * are buggy, rather than just warning about them and then proceeding - * to use them? If so, we should probably not wrap this whole thing - * in HAVE_EVENT_GET_VERSION and HAVE_EVENT_GET_METHOD. -RD */ - /* XXXX The problem is that it's not trivial to get libevent to change it's - * method once it's initialized, and it's not trivial to tell what method it - * will use without initializing it. I guess we could preemptively disable - * buggy libevent modes based on the version _before_ initializing it, - * though, but then there's no good way (afaict) to warn "I would have used - * kqueue, but instead I'm using select." -NM */ - if (!strcmp(m, "kqueue")) { - if (version < LE_11B) - buggy = 1; - } else if (!strcmp(m, "epoll")) { - if (version < LE_11) - iffy = 1; - } else if (!strcmp(m, "poll")) { - if (version < LE_10E) - buggy = 1; - else if (version < LE_11) - slow = 1; - } else if (!strcmp(m, "select")) { - if (version < LE_11) - slow = 1; - } else if (!strcmp(m, "win32")) { - if (version < LE_11B) - buggy = 1; - } - - /* Libevent versions before 1.3b do very badly on operating systems with - * user-space threading implementations. */ -#if defined(__OpenBSD__) || defined(__FreeBSD__) || defined(__NetBSD__) - if (server && version < LE_13B) { - thread_unsafe = 1; - sad_os = "BSD variants"; - } -#elif defined(__APPLE__) || defined(__darwin__) - if (server && version < LE_13B) { - thread_unsafe = 1; - sad_os = "Mac OS X"; - } -#endif - - if (thread_unsafe) { - log(LOG_WARN, LD_GENERAL, - "Libevent version %s often crashes when running a Tor server with %s. " - "Please use the latest version of libevent (1.3b or later)",v,sad_os); - badness = "BROKEN"; - } else if (buggy) { - log(LOG_WARN, LD_GENERAL, - "There are serious bugs in using %s with libevent %s. " - "Please use the latest version of libevent.", m, v); - badness = "BROKEN"; - } else if (iffy) { - log(LOG_WARN, LD_GENERAL, - "There are minor bugs in using %s with libevent %s. " - "You may want to use the latest version of libevent.", m, v); - badness = "BUGGY"; - } else if (slow && server) { - log(LOG_WARN, LD_GENERAL, - "libevent %s can be very slow with %s. " - "When running a server, please use the latest version of libevent.", - v,m); - badness = "SLOW"; - } + tor_check_libevent_version(tor_libevent_get_method(), + get_options()->ORPort != 0, + &badness); if (badness) { + const char *v = tor_libevent_get_version_str(); + const char *m = tor_libevent_get_method(); control_event_general_status(LOG_WARN, "BAD_LIBEVENT VERSION=%s METHOD=%s BADNESS=%s RECOVERED=NO", v, m, badness); } - } -#endif /** Return the persistent state struct for this Tor. */ or_state_t * @@ -5156,22 +4971,61 @@ or_state_validate(or_state_t *old_state, or_state_t *state, } /** Replace the current persistent state with <b>new_state</b> */ -static void +static int or_state_set(or_state_t *new_state) { char *err = NULL; + int ret = 0; tor_assert(new_state); - if (global_state) - config_free(&state_format, global_state); + config_free(&state_format, global_state); global_state = new_state; if (entry_guards_parse_state(global_state, 1, &err)<0) { log_warn(LD_GENERAL,"%s",err); tor_free(err); + ret = -1; } if (rep_hist_load_state(global_state, &err)<0) { log_warn(LD_GENERAL,"Unparseable bandwidth history state: %s",err); tor_free(err); + ret = -1; } + if (circuit_build_times_parse_state(&circ_times, global_state) < 0) { + ret = -1; + } + return ret; +} + +/** + * Save a broken state file to a backup location. + */ +static void +or_state_save_broken(char *fname) +{ + int i; + file_status_t status; + size_t len = strlen(fname)+16; + char *fname2 = tor_malloc(len); + for (i = 0; i < 100; ++i) { + tor_snprintf(fname2, len, "%s.%d", fname, i); + status = file_status(fname2); + if (status == FN_NOENT) + break; + } + if (i == 100) { + log_warn(LD_BUG, "Unable to parse state in \"%s\"; too many saved bad " + "state files to move aside. Discarding the old state file.", + fname); + unlink(fname); + } else { + log_warn(LD_BUG, "Unable to parse state in \"%s\". Moving it aside " + "to \"%s\". This could be a bug in Tor; please tell " + "the developers.", fname, fname2); + if (rename(fname, fname2) < 0) { + log_warn(LD_BUG, "Weirdly, I couldn't even move the state aside. The " + "OS gave an error of %s", strerror(errno)); + } + } + tor_free(fname2); } /** Reload the persistent state from disk, generating a new state as needed. @@ -5233,31 +5087,8 @@ or_state_load(void) " This is a bug in Tor."); goto done; } else if (badstate && contents) { - int i; - file_status_t status; - size_t len = strlen(fname)+16; - char *fname2 = tor_malloc(len); - for (i = 0; i < 100; ++i) { - tor_snprintf(fname2, len, "%s.%d", fname, i); - status = file_status(fname2); - if (status == FN_NOENT) - break; - } - if (i == 100) { - log_warn(LD_BUG, "Unable to parse state in \"%s\"; too many saved bad " - "state files to move aside. Discarding the old state file.", - fname); - unlink(fname); - } else { - log_warn(LD_BUG, "Unable to parse state in \"%s\". Moving it aside " - "to \"%s\". This could be a bug in Tor; please tell " - "the developers.", fname, fname2); - if (rename(fname, fname2) < 0) { - log_warn(LD_BUG, "Weirdly, I couldn't even move the state aside. The " - "OS gave an error of %s", strerror(errno)); - } - } - tor_free(fname2); + or_state_save_broken(fname); + tor_free(contents); config_free(&state_format, new_state); @@ -5269,7 +5100,9 @@ or_state_load(void) } else { log_info(LD_GENERAL, "Initialized state"); } - or_state_set(new_state); + if (or_state_set(new_state) == -1) { + or_state_save_broken(fname); + } new_state = NULL; if (!contents) { global_state->next_write = 0; @@ -5286,13 +5119,32 @@ or_state_load(void) return r; } +/** Did the last time we tried to write the state file fail? If so, we + * should consider disabling such features as preemptive circuit generation + * to compute circuit-build-time. */ +static int last_state_file_write_failed = 0; + +/** Return whether the state file failed to write last time we tried. */ +int +did_last_state_file_write_fail(void) +{ + return last_state_file_write_failed; +} + +/** If writing the state to disk fails, try again after this many seconds. */ +#define STATE_WRITE_RETRY_INTERVAL 3600 + +/** If we're a relay, how often should we checkpoint our state file even + * if nothing else dirties it? This will checkpoint ongoing stats like + * bandwidth used, per-country user stats, etc. */ +#define STATE_RELAY_CHECKPOINT_INTERVAL (12*60*60) + /** Write the persistent state to disk. Return 0 for success, <0 on failure. */ int or_state_save(time_t now) { char *state, *contents; char tbuf[ISO_TIME_LEN+1]; - size_t len; char *fname; tor_assert(global_state); @@ -5304,20 +5156,18 @@ or_state_save(time_t now) * to avoid redundant writes. */ entry_guards_update_state(global_state); rep_hist_update_state(global_state); + circuit_build_times_update_state(&circ_times, global_state); if (accounting_is_enabled(get_options())) accounting_run_housekeeping(now); - global_state->LastWritten = time(NULL); + global_state->LastWritten = now; + tor_free(global_state->TorVersion); - len = strlen(get_version())+8; - global_state->TorVersion = tor_malloc(len); - tor_snprintf(global_state->TorVersion, len, "Tor %s", get_version()); + tor_asprintf(&global_state->TorVersion, "Tor %s", get_version()); state = config_dump(&state_format, global_state, 1, 0); - len = strlen(state)+256; - contents = tor_malloc(len); - format_local_iso_time(tbuf, time(NULL)); - tor_snprintf(contents, len, + format_local_iso_time(tbuf, now); + tor_asprintf(&contents, "# Tor state file last generated on %s local time\n" "# Other times below are in GMT\n" "# You *do not* need to edit this file.\n\n%s", @@ -5327,15 +5177,25 @@ or_state_save(time_t now) if (write_str_to_file(fname, contents, 0)<0) { log_warn(LD_FS, "Unable to write state to file \"%s\"; " "will try again later", fname); + last_state_file_write_failed = 1; tor_free(fname); tor_free(contents); + /* Try again after STATE_WRITE_RETRY_INTERVAL (or sooner, if the state + * changes sooner). */ + global_state->next_write = now + STATE_WRITE_RETRY_INTERVAL; return -1; } + + last_state_file_write_failed = 0; log_info(LD_GENERAL, "Saved state to \"%s\"", fname); tor_free(fname); tor_free(contents); - global_state->next_write = TIME_MAX; + if (server_mode(get_options())) + global_state->next_write = now + STATE_RELAY_CHECKPOINT_INTERVAL; + else + global_state->next_write = TIME_MAX; + return 0; } @@ -5362,18 +5222,18 @@ remove_file_if_very_old(const char *fname, time_t now) * types. */ int getinfo_helper_config(control_connection_t *conn, - const char *question, char **answer) + const char *question, char **answer, + const char **errmsg) { (void) conn; + (void) errmsg; if (!strcmp(question, "config/names")) { smartlist_t *sl = smartlist_create(); int i; for (i = 0; _option_vars[i].name; ++i) { config_var_t *var = &_option_vars[i]; - const char *type, *desc; + const char *type; char *line; - size_t len; - desc = config_find_description(&options_format, var->name); switch (var->type) { case CONFIG_TYPE_STRING: type = "String"; break; case CONFIG_TYPE_FILENAME: type = "Filename"; break; @@ -5394,14 +5254,7 @@ getinfo_helper_config(control_connection_t *conn, } if (!type) continue; - len = strlen(var->name)+strlen(type)+16; - if (desc) - len += strlen(desc); - line = tor_malloc(len); - if (desc) - tor_snprintf(line, len, "%s %s %s\n",var->name,type,desc); - else - tor_snprintf(line, len, "%s %s\n",var->name,type); + tor_asprintf(&line, "%s %s\n",var->name,type); smartlist_add(sl, line); } *answer = smartlist_join_strings(sl, "", 0, NULL); diff --git a/src/or/config.h b/src/or/config.h new file mode 100644 index 0000000000..78a67dddf5 --- /dev/null +++ b/src/or/config.h @@ -0,0 +1,79 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file config.h + * \brief Header file for config.c. + **/ + +#ifndef _TOR_CONFIG_H +#define _TOR_CONFIG_H + +const char *get_dirportfrontpage(void); +or_options_t *get_options(void); +int set_options(or_options_t *new_val, char **msg); +void config_free_all(void); +const char *safe_str_client(const char *address); +const char *safe_str(const char *address); +const char *escaped_safe_str_client(const char *address); +const char *escaped_safe_str(const char *address); +const char *get_version(void); + +int config_get_lines(const char *string, config_line_t **result); +void config_free_lines(config_line_t *front); +setopt_err_t options_trial_assign(config_line_t *list, int use_defaults, + int clear_first, char **msg); +int resolve_my_address(int warn_severity, or_options_t *options, + uint32_t *addr, char **hostname_out); +int is_local_addr(const tor_addr_t *addr) ATTR_PURE; +void options_init(or_options_t *options); +char *options_dump(or_options_t *options, int minimal); +int options_init_from_torrc(int argc, char **argv); +setopt_err_t options_init_from_string(const char *cf, + int command, const char *command_arg, char **msg); +int option_is_recognized(const char *key); +const char *option_get_canonical_name(const char *key); +config_line_t *option_get_assignment(or_options_t *options, + const char *key); +int options_save_current(void); +const char *get_torrc_fname(void); +char *options_get_datadir_fname2_suffix(or_options_t *options, + const char *sub1, const char *sub2, + const char *suffix); +#define get_datadir_fname2_suffix(sub1, sub2, suffix) \ + options_get_datadir_fname2_suffix(get_options(), (sub1), (sub2), (suffix)) +/** Return a newly allocated string containing datadir/sub1. See + * get_datadir_fname2_suffix. */ +#define get_datadir_fname(sub1) get_datadir_fname2_suffix((sub1), NULL, NULL) +/** Return a newly allocated string containing datadir/sub1/sub2. See + * get_datadir_fname2_suffix. */ +#define get_datadir_fname2(sub1,sub2) \ + get_datadir_fname2_suffix((sub1), (sub2), NULL) +/** Return a newly allocated string containing datadir/sub1suffix. See + * get_datadir_fname2_suffix. */ +#define get_datadir_fname_suffix(sub1, suffix) \ + get_datadir_fname2_suffix((sub1), NULL, (suffix)) + +or_state_t *get_or_state(void); +int did_last_state_file_write_fail(void); +int or_state_save(time_t now); + +int options_need_geoip_info(or_options_t *options, const char **reason_out); +int getinfo_helper_config(control_connection_t *conn, + const char *question, char **answer, + const char **errmsg); + +const char *tor_get_digests(void); +uint32_t get_effective_bwrate(or_options_t *options); +uint32_t get_effective_bwburst(or_options_t *options); + +#ifdef CONFIG_PRIVATE +/* Used only by config.c and test.c */ +or_options_t *options_new(void); +#endif + +#endif + diff --git a/src/or/config_codedigest.c b/src/or/config_codedigest.c new file mode 100644 index 0000000000..be9eaa331d --- /dev/null +++ b/src/or/config_codedigest.c @@ -0,0 +1,11 @@ + +const char *tor_get_digests(void); + +const char * +tor_get_digests(void) +{ + return "" +#include "or_sha1.i" + ; +} + diff --git a/src/or/connection.c b/src/or/connection.c index 4869a2439a..fc2097f9a9 100644 --- a/src/or/connection.c +++ b/src/or/connection.c @@ -11,6 +11,30 @@ **/ #include "or.h" +#include "buffers.h" +#include "circuitbuild.h" +#include "circuitlist.h" +#include "circuituse.h" +#include "config.h" +#include "connection.h" +#include "connection_edge.h" +#include "connection_or.h" +#include "control.h" +#include "cpuworker.h" +#include "directory.h" +#include "dirserv.h" +#include "dns.h" +#include "dnsserv.h" +#include "geoip.h" +#include "main.h" +#include "policies.h" +#include "reasons.h" +#include "relay.h" +#include "rendclient.h" +#include "rendcommon.h" +#include "rephist.h" +#include "router.h" +#include "routerparse.h" static connection_t *connection_create_listener( struct sockaddr *listensockaddr, @@ -21,17 +45,22 @@ static void connection_init(time_t now, connection_t *conn, int type, static int connection_init_accepted_conn(connection_t *conn, uint8_t listener_type); static int connection_handle_listener_read(connection_t *conn, int new_type); -static int connection_read_bucket_should_increase(or_connection_t *conn); +static int connection_bucket_should_increase(int bucket, + or_connection_t *conn); static int connection_finished_flushing(connection_t *conn); static int connection_flushed_some(connection_t *conn); static int connection_finished_connecting(connection_t *conn); static int connection_reached_eof(connection_t *conn); -static int connection_read_to_buf(connection_t *conn, int *max_to_read, +static int connection_read_to_buf(connection_t *conn, ssize_t *max_to_read, int *socket_error); static int connection_process_inbuf(connection_t *conn, int package_partial); static void client_check_address_changed(int sock); static void set_constrained_socket_buffers(int sock, int size); +static const char *connection_proxy_state_to_string(int state); +static int connection_read_https_proxy_response(connection_t *conn); +static void connection_send_socks5_connect(connection_t *conn); + /** The last IPv4 address that our network interface seemed to have been * binding to, in host order. We use this to detect when our IP changes. */ static uint32_t last_interface_ip = 0; @@ -92,8 +121,7 @@ conn_state_to_string(int type, int state) case CONN_TYPE_OR: switch (state) { case OR_CONN_STATE_CONNECTING: return "connect()ing"; - case OR_CONN_STATE_PROXY_FLUSHING: return "proxy flushing"; - case OR_CONN_STATE_PROXY_READING: return "proxy reading"; + case OR_CONN_STATE_PROXY_HANDSHAKING: return "handshaking (proxy)"; case OR_CONN_STATE_TLS_HANDSHAKING: return "handshaking (TLS)"; case OR_CONN_STATE_TLS_CLIENT_RENEGOTIATING: return "renegotiating (TLS)"; @@ -177,6 +205,9 @@ or_connection_new(int socket_family) or_conn->timestamp_last_added_nonpadding = time(NULL); or_conn->next_circ_id = crypto_rand_int(1<<15); + or_conn->active_circuit_pqueue = smartlist_create(); + or_conn->active_circuit_pqueue_last_recalibrated = cell_ewma_get_tick(); + return or_conn; } @@ -202,6 +233,7 @@ control_connection_new(int socket_family) tor_malloc_zero(sizeof(control_connection_t)); connection_init(time(NULL), TO_CONN(control_conn), CONN_TYPE_CONTROL, socket_family); + log_notice(LD_CONTROL, "New control connection opened."); return control_conn; } @@ -299,25 +331,6 @@ connection_link_connections(connection_t *conn_a, connection_t *conn_b) conn_b->linked_conn = conn_a; } -/** Tell libevent that we don't care about <b>conn</b> any more. */ -void -connection_unregister_events(connection_t *conn) -{ - if (conn->read_event) { - if (event_del(conn->read_event)) - log_warn(LD_BUG, "Error removing read event for %d", conn->s); - tor_free(conn->read_event); - } - if (conn->write_event) { - if (event_del(conn->write_event)) - log_warn(LD_BUG, "Error removing write event for %d", conn->s); - tor_free(conn->write_event); - } - if (conn->dns_server_port) { - dnsserv_close_listener(conn); - } -} - /** Deallocate memory used by <b>conn</b>. Deallocate its buffers if * necessary, close its socket if necessary, and mark the directory as dirty * if <b>conn</b> is an OR or OP connection. @@ -327,6 +340,9 @@ _connection_free(connection_t *conn) { void *mem; size_t memlen; + if (!conn) + return; + switch (conn->type) { case CONN_TYPE_OR: tor_assert(conn->magic == OR_CONNECTION_MAGIC); @@ -384,14 +400,11 @@ _connection_free(connection_t *conn) if (connection_speaks_cells(conn)) { or_connection_t *or_conn = TO_OR_CONN(conn); - if (or_conn->tls) { - tor_tls_free(or_conn->tls); - or_conn->tls = NULL; - } - if (or_conn->handshake_state) { - or_handshake_state_free(or_conn->handshake_state); - or_conn->handshake_state = NULL; - } + tor_tls_free(or_conn->tls); + or_conn->tls = NULL; + or_handshake_state_free(or_conn->handshake_state); + or_conn->handshake_state = NULL; + smartlist_free(or_conn->active_circuit_pqueue); tor_free(or_conn->nickname); } if (CONN_IS_EDGE(conn)) { @@ -401,8 +414,8 @@ _connection_free(connection_t *conn) memset(edge_conn->socks_request, 0xcc, sizeof(socks_request_t)); tor_free(edge_conn->socks_request); } - if (edge_conn->rend_data) - rend_data_free(edge_conn->rend_data); + + rend_data_free(edge_conn->rend_data); } if (conn->type == CONN_TYPE_CONTROL) { control_connection_t *control_conn = TO_CONTROL_CONN(conn); @@ -415,16 +428,15 @@ _connection_free(connection_t *conn) if (conn->type == CONN_TYPE_DIR) { dir_connection_t *dir_conn = TO_DIR_CONN(conn); tor_free(dir_conn->requested_resource); - if (dir_conn->zlib_state) - tor_zlib_free(dir_conn->zlib_state); + + tor_zlib_free(dir_conn->zlib_state); if (dir_conn->fingerprint_stack) { SMARTLIST_FOREACH(dir_conn->fingerprint_stack, char *, cp, tor_free(cp)); smartlist_free(dir_conn->fingerprint_stack); } - if (dir_conn->cached_dir) - cached_dir_decref(dir_conn->cached_dir); - if (dir_conn->rend_data) - rend_data_free(dir_conn->rend_data); + + cached_dir_decref(dir_conn->cached_dir); + rend_data_free(dir_conn->rend_data); } if (conn->s >= 0) { @@ -439,7 +451,7 @@ _connection_free(connection_t *conn) connection_or_remove_from_identity_map(TO_OR_CONN(conn)); } - memset(conn, 0xAA, memlen); /* poison memory */ + memset(mem, 0xCC, memlen); /* poison memory */ tor_free(mem); } @@ -448,7 +460,8 @@ _connection_free(connection_t *conn) void connection_free(connection_t *conn) { - tor_assert(conn); + if (!conn) + return; tor_assert(!connection_is_on_closeable_list(conn)); tor_assert(!connection_in_array(conn)); if (conn->linked_conn) { @@ -544,13 +557,6 @@ connection_about_to_close_connection(connection_t *conn) * failed: forget about this router, and maybe try again. */ connection_dir_request_failed(dir_conn); } - if (conn->purpose == DIR_PURPOSE_FETCH_RENDDESC && dir_conn->rend_data) { - /* Give it a try. However, there is no re-fetching for v0 rend - * descriptors; if the response is empty or the descriptor is - * unusable, close pending connections (unless a v2 request is - * still in progress). */ - rend_client_desc_trynow(dir_conn->rend_data->onion_address, 0); - } /* If we were trying to fetch a v2 rend desc and did not succeed, * retry as needed. (If a fetch is successful, the connection state * is changed to DIR_PURPOSE_HAS_FETCHED_RENDDESC to mark that @@ -573,7 +579,7 @@ connection_about_to_close_connection(connection_t *conn) or_options_t *options = get_options(); rep_hist_note_connect_failed(or_conn->identity_digest, now); entry_guard_register_connect_status(or_conn->identity_digest,0, - !options->HttpsProxy, now); + !options->HTTPSProxy, now); if (conn->state >= OR_CONN_STATE_TLS_HANDSHAKING) { int reason = tls_error_to_orconn_end_reason(or_conn->tls_error); control_event_or_conn_status(or_conn, OR_CONN_EVENT_FAILED, @@ -589,7 +595,7 @@ connection_about_to_close_connection(connection_t *conn) rep_hist_note_disconnect(or_conn->identity_digest, now); control_event_or_conn_status(or_conn, OR_CONN_EVENT_CLOSED, tls_error_to_orconn_end_reason(or_conn->tls_error)); - } else if (or_conn->identity_digest) { + } else if (!tor_digest_is_zero(or_conn->identity_digest)) { rep_hist_note_connection_died(or_conn->identity_digest, now); control_event_or_conn_status(or_conn, OR_CONN_EVENT_CLOSED, tls_error_to_orconn_end_reason(or_conn->tls_error)); @@ -833,16 +839,16 @@ static void warn_too_many_conns(void) { #define WARN_TOO_MANY_CONNS_INTERVAL (6*60*60) - static time_t last_warned = 0; - time_t now = time(NULL); - int n_conns = get_n_open_sockets(); - if (last_warned + WARN_TOO_MANY_CONNS_INTERVAL < now) { + static ratelim_t last_warned = RATELIM_INIT(WARN_TOO_MANY_CONNS_INTERVAL); + char *m; + if ((m = rate_limit_log(&last_warned, approx_time()))) { + int n_conns = get_n_open_sockets(); log_warn(LD_NET,"Failing because we have %d connections already. Please " - "raise your ulimit -n.", n_conns); - last_warned = now; + "raise your ulimit -n.%s", n_conns, m); + tor_free(m); + control_event_general_status(LOG_WARN, "TOO_MANY_CONNECTIONS CURRENT=%d", + n_conns); } - control_event_general_status(LOG_WARN, "TOO_MANY_CONNECTIONS CURRENT=%d", - n_conns); } /** Bind a new non-blocking socket listening to the socket described @@ -1172,7 +1178,8 @@ connection_handle_listener_read(connection_t *conn, int new_type) } if (connection_init_accepted_conn(newconn, conn->type) < 0) { - connection_mark_for_close(newconn); + if (! newconn->marked_for_close) + connection_mark_for_close(newconn); return 0; } return 0; @@ -1198,9 +1205,11 @@ connection_init_accepted_conn(connection_t *conn, uint8_t listener_type) conn->state = AP_CONN_STATE_SOCKS_WAIT; break; case CONN_TYPE_AP_TRANS_LISTENER: + TO_EDGE_CONN(conn)->is_transparent_ap = 1; conn->state = AP_CONN_STATE_CIRCUIT_WAIT; return connection_ap_process_transparent(TO_EDGE_CONN(conn)); case CONN_TYPE_AP_NATD_LISTENER: + TO_EDGE_CONN(conn)->is_transparent_ap = 1; conn->state = AP_CONN_STATE_NATD_WAIT; break; } @@ -1231,7 +1240,7 @@ connection_connect(connection_t *conn, const char *address, { int s, inprogress = 0; char addrbuf[256]; - struct sockaddr *dest_addr = (struct sockaddr*) addrbuf; + struct sockaddr *dest_addr; socklen_t dest_addr_len; or_options_t *options = get_options(); int protocol_family; @@ -1254,7 +1263,7 @@ connection_connect(connection_t *conn, const char *address, return -1; } - if (options->OutboundBindAddress) { + if (options->OutboundBindAddress && !tor_addr_is_loopback(addr)) { struct sockaddr_in ext_addr; memset(&ext_addr, 0, sizeof(ext_addr)); @@ -1285,7 +1294,8 @@ connection_connect(connection_t *conn, const char *address, dest_addr_len = tor_addr_to_sockaddr(addr, port, dest_addr, sizeof(addrbuf)); tor_assert(dest_addr_len > 0); - log_debug(LD_NET,"Connecting to %s:%u.",escaped_safe_str(address),port); + log_debug(LD_NET, "Connecting to %s:%u.", + escaped_safe_str_client(address), port); if (connect(s, dest_addr, dest_addr_len) < 0) { int e = tor_socket_errno(s); @@ -1293,7 +1303,8 @@ connection_connect(connection_t *conn, const char *address, /* yuck. kill it. */ *socket_error = e; log_info(LD_NET, - "connect() to %s:%u failed: %s",escaped_safe_str(address), + "connect() to %s:%u failed: %s", + escaped_safe_str_client(address), port, tor_socket_strerror(e)); tor_close_socket(s); return -1; @@ -1307,7 +1318,8 @@ connection_connect(connection_t *conn, const char *address, /* it succeeded. we're connected. */ log_fn(inprogress?LOG_DEBUG:LOG_INFO, LD_NET, - "Connection to %s:%u %s (sock %d).",escaped_safe_str(address), + "Connection to %s:%u %s (sock %d).", + escaped_safe_str_client(address), port, inprogress?"in progress":"established", s); conn->s = s; if (connection_add(conn) < 0) /* no space, forget it */ @@ -1315,6 +1327,353 @@ connection_connect(connection_t *conn, const char *address, return inprogress ? 0 : 1; } +/** Convert state number to string representation for logging purposes. + */ +static const char * +connection_proxy_state_to_string(int state) +{ + static const char *unknown = "???"; + static const char *states[] = { + "PROXY_NONE", + "PROXY_HTTPS_WANT_CONNECT_OK", + "PROXY_SOCKS4_WANT_CONNECT_OK", + "PROXY_SOCKS5_WANT_AUTH_METHOD_NONE", + "PROXY_SOCKS5_WANT_AUTH_METHOD_RFC1929", + "PROXY_SOCKS5_WANT_AUTH_RFC1929_OK", + "PROXY_SOCKS5_WANT_CONNECT_OK", + "PROXY_CONNECTED", + }; + + if (state < PROXY_NONE || state > PROXY_CONNECTED) + return unknown; + + return states[state]; +} + +/** Write a proxy request of <b>type</b> (socks4, socks5, https) to conn + * for conn->addr:conn->port, authenticating with the auth details given + * in the configuration (if available). SOCKS 5 and HTTP CONNECT proxies + * support authentication. + * + * Returns -1 if conn->addr is incompatible with the proxy protocol, and + * 0 otherwise. + * + * Use connection_read_proxy_handshake() to complete the handshake. + */ +int +connection_proxy_connect(connection_t *conn, int type) +{ + or_options_t *options; + + tor_assert(conn); + + options = get_options(); + + switch (type) { + case PROXY_CONNECT: { + char buf[1024]; + char *base64_authenticator=NULL; + const char *authenticator = options->HTTPSProxyAuthenticator; + + /* Send HTTP CONNECT and authentication (if available) in + * one request */ + + if (authenticator) { + base64_authenticator = alloc_http_authenticator(authenticator); + if (!base64_authenticator) + log_warn(LD_OR, "Encoding https authenticator failed"); + } + + if (base64_authenticator) { + tor_snprintf(buf, sizeof(buf), "CONNECT %s:%d HTTP/1.1\r\n" + "Proxy-Authorization: Basic %s\r\n\r\n", + fmt_addr(&conn->addr), + conn->port, base64_authenticator); + tor_free(base64_authenticator); + } else { + tor_snprintf(buf, sizeof(buf), "CONNECT %s:%d HTTP/1.0\r\n\r\n", + fmt_addr(&conn->addr), conn->port); + } + + connection_write_to_buf(buf, strlen(buf), conn); + conn->proxy_state = PROXY_HTTPS_WANT_CONNECT_OK; + break; + } + + case PROXY_SOCKS4: { + unsigned char buf[9]; + uint16_t portn; + uint32_t ip4addr; + + /* Send a SOCKS4 connect request with empty user id */ + + if (tor_addr_family(&conn->addr) != AF_INET) { + log_warn(LD_NET, "SOCKS4 client is incompatible with IPv6"); + return -1; + } + + ip4addr = tor_addr_to_ipv4n(&conn->addr); + portn = htons(conn->port); + + buf[0] = 4; /* version */ + buf[1] = SOCKS_COMMAND_CONNECT; /* command */ + memcpy(buf + 2, &portn, 2); /* port */ + memcpy(buf + 4, &ip4addr, 4); /* addr */ + buf[8] = 0; /* userid (empty) */ + + connection_write_to_buf((char *)buf, sizeof(buf), conn); + conn->proxy_state = PROXY_SOCKS4_WANT_CONNECT_OK; + break; + } + + case PROXY_SOCKS5: { + unsigned char buf[4]; /* fields: vers, num methods, method list */ + + /* Send a SOCKS5 greeting (connect request must wait) */ + + buf[0] = 5; /* version */ + + /* number of auth methods */ + if (options->Socks5ProxyUsername) { + buf[1] = 2; + buf[2] = 0x00; /* no authentication */ + buf[3] = 0x02; /* rfc1929 Username/Passwd auth */ + conn->proxy_state = PROXY_SOCKS5_WANT_AUTH_METHOD_RFC1929; + } else { + buf[1] = 1; + buf[2] = 0x00; /* no authentication */ + conn->proxy_state = PROXY_SOCKS5_WANT_AUTH_METHOD_NONE; + } + + connection_write_to_buf((char *)buf, 2 + buf[1], conn); + break; + } + + default: + log_err(LD_BUG, "Invalid proxy protocol, %d", type); + tor_fragile_assert(); + return -1; + } + + log_debug(LD_NET, "set state %s", + connection_proxy_state_to_string(conn->proxy_state)); + + return 0; +} + +/** Read conn's inbuf. If the http response from the proxy is all + * here, make sure it's good news, then return 1. If it's bad news, + * return -1. Else return 0 and hope for better luck next time. + */ +static int +connection_read_https_proxy_response(connection_t *conn) +{ + char *headers; + char *reason=NULL; + int status_code; + time_t date_header; + + switch (fetch_from_buf_http(conn->inbuf, + &headers, MAX_HEADERS_SIZE, + NULL, NULL, 10000, 0)) { + case -1: /* overflow */ + log_warn(LD_PROTOCOL, + "Your https proxy sent back an oversized response. Closing."); + return -1; + case 0: + log_info(LD_NET,"https proxy response not all here yet. Waiting."); + return 0; + /* case 1, fall through */ + } + + if (parse_http_response(headers, &status_code, &date_header, + NULL, &reason) < 0) { + log_warn(LD_NET, + "Unparseable headers from proxy (connecting to '%s'). Closing.", + conn->address); + tor_free(headers); + return -1; + } + if (!reason) reason = tor_strdup("[no reason given]"); + + if (status_code == 200) { + log_info(LD_NET, + "HTTPS connect to '%s' successful! (200 %s) Starting TLS.", + conn->address, escaped(reason)); + tor_free(reason); + return 1; + } + /* else, bad news on the status code */ + log_warn(LD_NET, + "The https proxy sent back an unexpected status code %d (%s). " + "Closing.", + status_code, escaped(reason)); + tor_free(reason); + return -1; +} + +/** Send SOCKS5 CONNECT command to <b>conn</b>, copying <b>conn->addr</b> + * and <b>conn->port</b> into the request. + */ +static void +connection_send_socks5_connect(connection_t *conn) +{ + unsigned char buf[1024]; + size_t reqsize = 6; + uint16_t port = htons(conn->port); + + buf[0] = 5; /* version */ + buf[1] = SOCKS_COMMAND_CONNECT; /* command */ + buf[2] = 0; /* reserved */ + + if (tor_addr_family(&conn->addr) == AF_INET) { + uint32_t addr = tor_addr_to_ipv4n(&conn->addr); + + buf[3] = 1; + reqsize += 4; + memcpy(buf + 4, &addr, 4); + memcpy(buf + 8, &port, 2); + } else { /* AF_INET6 */ + buf[3] = 4; + reqsize += 16; + memcpy(buf + 4, tor_addr_to_in6(&conn->addr), 16); + memcpy(buf + 20, &port, 2); + } + + connection_write_to_buf((char *)buf, reqsize, conn); + + conn->proxy_state = PROXY_SOCKS5_WANT_CONNECT_OK; +} + +/** Call this from connection_*_process_inbuf() to advance the proxy + * handshake. + * + * No matter what proxy protocol is used, if this function returns 1, the + * handshake is complete, and the data remaining on inbuf may contain the + * start of the communication with the requested server. + * + * Returns 0 if the current buffer contains an incomplete response, and -1 + * on error. + */ +int +connection_read_proxy_handshake(connection_t *conn) +{ + int ret = 0; + char *reason = NULL; + + log_debug(LD_NET, "enter state %s", + connection_proxy_state_to_string(conn->proxy_state)); + + switch (conn->proxy_state) { + case PROXY_HTTPS_WANT_CONNECT_OK: + ret = connection_read_https_proxy_response(conn); + if (ret == 1) + conn->proxy_state = PROXY_CONNECTED; + break; + + case PROXY_SOCKS4_WANT_CONNECT_OK: + ret = fetch_from_buf_socks_client(conn->inbuf, + conn->proxy_state, + &reason); + if (ret == 1) + conn->proxy_state = PROXY_CONNECTED; + break; + + case PROXY_SOCKS5_WANT_AUTH_METHOD_NONE: + ret = fetch_from_buf_socks_client(conn->inbuf, + conn->proxy_state, + &reason); + /* no auth needed, do connect */ + if (ret == 1) { + connection_send_socks5_connect(conn); + ret = 0; + } + break; + + case PROXY_SOCKS5_WANT_AUTH_METHOD_RFC1929: + ret = fetch_from_buf_socks_client(conn->inbuf, + conn->proxy_state, + &reason); + + /* send auth if needed, otherwise do connect */ + if (ret == 1) { + connection_send_socks5_connect(conn); + ret = 0; + } else if (ret == 2) { + unsigned char buf[1024]; + size_t reqsize, usize, psize; + const char *user, *pass; + + user = get_options()->Socks5ProxyUsername; + pass = get_options()->Socks5ProxyPassword; + tor_assert(user && pass); + + /* XXX len of user and pass must be <= 255 !!! */ + usize = strlen(user); + psize = strlen(pass); + tor_assert(usize <= 255 && psize <= 255); + reqsize = 3 + usize + psize; + + buf[0] = 1; /* negotiation version */ + buf[1] = usize; + memcpy(buf + 2, user, usize); + buf[2 + usize] = psize; + memcpy(buf + 3 + usize, pass, psize); + + connection_write_to_buf((char *)buf, reqsize, conn); + + conn->proxy_state = PROXY_SOCKS5_WANT_AUTH_RFC1929_OK; + ret = 0; + } + break; + + case PROXY_SOCKS5_WANT_AUTH_RFC1929_OK: + ret = fetch_from_buf_socks_client(conn->inbuf, + conn->proxy_state, + &reason); + /* send the connect request */ + if (ret == 1) { + connection_send_socks5_connect(conn); + ret = 0; + } + break; + + case PROXY_SOCKS5_WANT_CONNECT_OK: + ret = fetch_from_buf_socks_client(conn->inbuf, + conn->proxy_state, + &reason); + if (ret == 1) + conn->proxy_state = PROXY_CONNECTED; + break; + + default: + log_err(LD_BUG, "Invalid proxy_state for reading, %d", + conn->proxy_state); + tor_fragile_assert(); + ret = -1; + break; + } + + log_debug(LD_NET, "leaving state %s", + connection_proxy_state_to_string(conn->proxy_state)); + + if (ret < 0) { + if (reason) { + log_warn(LD_NET, "Proxy Client: unable to connect to %s:%d (%s)", + conn->address, conn->port, escaped(reason)); + tor_free(reason); + } else { + log_warn(LD_NET, "Proxy Client: unable to connect to %s:%d", + conn->address, conn->port); + } + } else if (ret == 1) { + log_info(LD_NET, "Proxy Client: connection to %s:%d successful", + conn->address, conn->port); + } + + return ret; +} + /** * Launch any configured listener connections of type <b>type</b>. (A * listener is configured if <b>port_option</b> is non-zero. If any @@ -1506,8 +1865,8 @@ retry_all_listeners(smartlist_t *replaced_conns, replaced_conns, new_conns, 0, AF_INET)<0) return -1; - if (retry_listeners(CONN_TYPE_AP_NATD_LISTENER, options->NatdListenAddress, - options->NatdPort, "127.0.0.1", + if (retry_listeners(CONN_TYPE_AP_NATD_LISTENER, options->NATDListenAddress, + options->NATDPort, "127.0.0.1", replaced_conns, new_conns, 0, AF_INET)<0) return -1; @@ -1643,6 +2002,7 @@ connection_bucket_write_limit(connection_t *conn, time_t now) int base = connection_speaks_cells(conn) ? CELL_NETWORK_SIZE : RELAY_PAYLOAD_SIZE; int priority = conn->type != CONN_TYPE_DIR; + int conn_bucket = (int)conn->outbuf_flushlen; int global_bucket = global_write_bucket; if (!connection_is_rate_limited(conn)) { @@ -1650,12 +2010,22 @@ connection_bucket_write_limit(connection_t *conn, time_t now) return conn->outbuf_flushlen; } + if (connection_speaks_cells(conn)) { + /* use the per-conn write limit if it's lower, but if it's less + * than zero just use zero */ + or_connection_t *or_conn = TO_OR_CONN(conn); + if (conn->state == OR_CONN_STATE_OPEN) + if (or_conn->write_bucket < conn_bucket) + conn_bucket = or_conn->write_bucket >= 0 ? + or_conn->write_bucket : 0; + } + if (connection_counts_as_relayed_traffic(conn, now) && global_relayed_write_bucket <= global_write_bucket) global_bucket = global_relayed_write_bucket; - return connection_bucket_round_robin(base, priority, global_bucket, - conn->outbuf_flushlen); + return connection_bucket_round_robin(base, priority, + global_bucket, conn_bucket); } /** Return 1 if the global write buckets are low enough that we @@ -1709,14 +2079,12 @@ global_write_bucket_low(connection_t *conn, size_t attempt, int priority) return 0; } -/** We just read num_read and wrote num_written onto conn. - * Decrement buckets appropriately. */ +/** We just read <b>num_read</b> and wrote <b>num_written</b> bytes + * onto <b>conn</b>. Decrement buckets appropriately. */ static void connection_buckets_decrement(connection_t *conn, time_t now, size_t num_read, size_t num_written) { - if (!connection_is_rate_limited(conn)) - return; /* local IPs are free */ if (num_written >= INT_MAX || num_read >= INT_MAX) { log_err(LD_BUG, "Value out of range. num_read=%lu, num_written=%lu, " "connection type=%s, state=%s", @@ -1728,10 +2096,24 @@ connection_buckets_decrement(connection_t *conn, time_t now, tor_fragile_assert(); } - if (num_read > 0) + /* Count bytes of answering direct and tunneled directory requests */ + if (conn->type == CONN_TYPE_DIR && conn->purpose == DIR_PURPOSE_SERVER) { + if (num_read > 0) + rep_hist_note_dir_bytes_read(num_read, now); + if (num_written > 0) + rep_hist_note_dir_bytes_written(num_written, now); + } + + if (!connection_is_rate_limited(conn)) + return; /* local IPs are free */ + if (num_read > 0) { rep_hist_note_bytes_read(num_read, now); - if (num_written > 0) + } + if (num_written > 0) { rep_hist_note_bytes_written(num_written, now); + } + if (conn->type == CONN_TYPE_EXIT) + rep_hist_note_exit_bytes(conn->port, num_written, num_read); if (connection_counts_as_relayed_traffic(conn, now)) { global_relayed_read_bucket -= (int)num_read; @@ -1739,8 +2121,10 @@ connection_buckets_decrement(connection_t *conn, time_t now, } global_read_bucket -= (int)num_read; global_write_bucket -= (int)num_written; - if (connection_speaks_cells(conn) && conn->state == OR_CONN_STATE_OPEN) + if (connection_speaks_cells(conn) && conn->state == OR_CONN_STATE_OPEN) { TO_OR_CONN(conn)->read_bucket -= (int)num_read; + TO_OR_CONN(conn)->write_bucket -= (int)num_written; + } } /** If we have exhausted our global buckets, or the buckets for conn, @@ -1779,12 +2163,10 @@ connection_consider_empty_write_buckets(connection_t *conn) } else if (connection_counts_as_relayed_traffic(conn, approx_time()) && global_relayed_write_bucket <= 0) { reason = "global relayed write bucket exhausted. Pausing."; -#if 0 } else if (connection_speaks_cells(conn) && conn->state == OR_CONN_STATE_OPEN && TO_OR_CONN(conn)->write_bucket <= 0) { reason = "connection write bucket exhausted. Pausing."; -#endif } else return; /* all good, no need to stop it */ @@ -1880,14 +2262,19 @@ connection_bucket_refill(int seconds_elapsed, time_t now) { if (connection_speaks_cells(conn)) { or_connection_t *or_conn = TO_OR_CONN(conn); - if (connection_read_bucket_should_increase(or_conn)) { + if (connection_bucket_should_increase(or_conn->read_bucket, or_conn)) { connection_bucket_refill_helper(&or_conn->read_bucket, or_conn->bandwidthrate, or_conn->bandwidthburst, seconds_elapsed, "or_conn->read_bucket"); - //log_fn(LOG_DEBUG,"Receiver bucket %d now %d.", i, - // conn->read_bucket); + } + if (connection_bucket_should_increase(or_conn->write_bucket, or_conn)) { + connection_bucket_refill_helper(&or_conn->write_bucket, + or_conn->bandwidthrate, + or_conn->bandwidthburst, + seconds_elapsed, + "or_conn->write_bucket"); } } @@ -1908,8 +2295,10 @@ connection_bucket_refill(int seconds_elapsed, time_t now) if (conn->write_blocked_on_bw == 1 && global_write_bucket > 0 /* and we're allowed to write */ && (!connection_counts_as_relayed_traffic(conn, now) || - global_relayed_write_bucket > 0)) { - /* even if we're relayed traffic */ + global_relayed_write_bucket > 0) /* even if it's relayed traffic */ + && (!connection_speaks_cells(conn) || + conn->state != OR_CONN_STATE_OPEN || + TO_OR_CONN(conn)->write_bucket > 0)) { LOG_FN_CONN(conn, (LOG_DEBUG,LD_NET, "waking up conn (fd %d) for write", conn->s)); conn->write_blocked_on_bw = 0; @@ -1918,17 +2307,17 @@ connection_bucket_refill(int seconds_elapsed, time_t now) }); } -/** Is the receiver bucket for connection <b>conn</b> low enough that we +/** Is the <b>bucket</b> for connection <b>conn</b> low enough that we * should add another pile of tokens to it? */ static int -connection_read_bucket_should_increase(or_connection_t *conn) +connection_bucket_should_increase(int bucket, or_connection_t *conn) { tor_assert(conn); if (conn->_base.state != OR_CONN_STATE_OPEN) return 0; /* only open connections play the rate limiting game */ - if (conn->read_bucket >= conn->bandwidthburst) + if (bucket >= conn->bandwidthburst) return 0; return 1; @@ -1946,10 +2335,10 @@ connection_read_bucket_should_increase(or_connection_t *conn) * Mark the connection and return -1 if you want to close it, else * return 0. */ -int -connection_handle_read(connection_t *conn) +static int +connection_handle_read_impl(connection_t *conn) { - int max_to_read=-1, try_to_read; + ssize_t max_to_read=-1, try_to_read; size_t before, n_read = 0; int socket_error = 0; @@ -1975,7 +2364,7 @@ connection_handle_read(connection_t *conn) return 0; } -loop_again: + loop_again: try_to_read = max_to_read; tor_assert(!conn->marked_for_close); @@ -2016,12 +2405,16 @@ loop_again: return -1; } if (conn->linked_conn) { - /* The other side's handle_write will never actually get called, so + /* The other side's handle_write() will never actually get called, so * we need to invoke the appropriate callbacks ourself. */ connection_t *linked = conn->linked_conn; if (n_read) { - /* Probably a no-op, but hey. */ + /* Probably a no-op, since linked conns typically don't count for + * bandwidth rate limiting. But do it anyway so we can keep stats + * accurately. Note that since we read the bytes from conn, and + * we're writing the bytes onto the linked connection, we count + * these as <i>written</i> bytes. */ connection_buckets_decrement(linked, approx_time(), 0, n_read); if (connection_flushed_some(linked) < 0) @@ -2033,7 +2426,7 @@ loop_again: if (!buf_datalen(linked->outbuf) && conn->active_on_link) connection_stop_reading_from_linked_conn(conn); } - /* If we hit the EOF, call connection_reached_eof. */ + /* If we hit the EOF, call connection_reached_eof(). */ if (!conn->marked_for_close && conn->inbuf_reached_eof && connection_reached_eof(conn) < 0) { @@ -2042,6 +2435,16 @@ loop_again: return 0; } +int +connection_handle_read(connection_t *conn) +{ + int res; + + tor_gettimeofday_cache_clear(); + res = connection_handle_read_impl(conn); + return res; +} + /** Pull in new bytes from conn-\>s or conn-\>linked_conn onto conn-\>inbuf, * either directly or via TLS. Reduce the token buckets by the number of bytes * read. @@ -2053,7 +2456,8 @@ loop_again: * Return -1 if we want to break conn, else return 0. */ static int -connection_read_to_buf(connection_t *conn, int *max_to_read, int *socket_error) +connection_read_to_buf(connection_t *conn, ssize_t *max_to_read, + int *socket_error) { int result; ssize_t at_most = *max_to_read; @@ -2075,7 +2479,7 @@ connection_read_to_buf(connection_t *conn, int *max_to_read, int *socket_error) } if (connection_speaks_cells(conn) && - conn->state > OR_CONN_STATE_PROXY_READING) { + conn->state > OR_CONN_STATE_PROXY_HANDSHAKING) { int pending; or_connection_t *or_conn = TO_OR_CONN(conn); size_t initial_size; @@ -2171,15 +2575,19 @@ connection_read_to_buf(connection_t *conn, int *max_to_read, int *socket_error) n_read = (size_t) result; } - if (n_read > 0) { /* change *max_to_read */ - /*XXXX021 check for overflow*/ - *max_to_read = (int)(at_most - n_read); - } + if (n_read > 0) { + /* change *max_to_read */ + *max_to_read = at_most - n_read; - if (conn->type == CONN_TYPE_AP) { - edge_connection_t *edge_conn = TO_EDGE_CONN(conn); - /*XXXX021 check for overflow*/ - edge_conn->n_read += (int)n_read; + /* Update edge_conn->n_read */ + if (conn->type == CONN_TYPE_AP) { + edge_connection_t *edge_conn = TO_EDGE_CONN(conn); + /* Check for overflow: */ + if (PREDICT_LIKELY(UINT32_MAX - edge_conn->n_read > n_read)) + edge_conn->n_read += (int)n_read; + else + edge_conn->n_read = UINT32_MAX; + } } connection_buckets_decrement(conn, approx_time(), n_read, n_written); @@ -2243,8 +2651,8 @@ connection_outbuf_too_full(connection_t *conn) * Mark the connection and return -1 if you want to close it, else * return 0. */ -int -connection_handle_write(connection_t *conn, int force) +static int +connection_handle_write_impl(connection_t *conn, int force) { int e; socklen_t len=(socklen_t)sizeof(e); @@ -2259,7 +2667,7 @@ connection_handle_write(connection_t *conn, int force) return 0; /* do nothing */ if (conn->in_flushed_some) { - log_warn(LD_BUG, "called recursively from inside conn->in_flushed_some()"); + log_warn(LD_BUG, "called recursively from inside conn->in_flushed_some"); return 0; } @@ -2303,7 +2711,7 @@ connection_handle_write(connection_t *conn, int force) : connection_bucket_write_limit(conn, now); if (connection_speaks_cells(conn) && - conn->state > OR_CONN_STATE_PROXY_READING) { + conn->state > OR_CONN_STATE_PROXY_HANDSHAKING) { or_connection_t *or_conn = TO_OR_CONN(conn); if (conn->state == OR_CONN_STATE_TLS_HANDSHAKING || conn->state == OR_CONN_STATE_TLS_CLIENT_RENEGOTIATING) { @@ -2322,6 +2730,13 @@ connection_handle_write(connection_t *conn, int force) /* else open, or closing */ result = flush_buf_tls(or_conn->tls, conn->outbuf, max_to_write, &conn->outbuf_flushlen); + + /* If we just flushed the last bytes, check if this tunneled dir + * request is done. */ + if (buf_datalen(conn->outbuf) == 0 && conn->dirreq_id) + geoip_change_dirreq_state(conn->dirreq_id, DIRREQ_TUNNELED, + DIRREQ_OR_CONN_BUFFER_FLUSHED); + switch (result) { CASE_TOR_TLS_ERROR_ANY: case TOR_TLS_CLOSE: @@ -2341,8 +2756,8 @@ connection_handle_write(connection_t *conn, int force) if (!connection_is_reading(conn)) { connection_stop_writing(conn); conn->write_blocked_on_bw = 1; - /* we'll start reading again when the next second arrives, - * and then also start writing again. + /* we'll start reading again when we get more tokens in our + * read bucket; then we'll start writing again too. */ } /* else no problem, we're already reading */ @@ -2371,10 +2786,13 @@ connection_handle_write(connection_t *conn, int force) n_written = (size_t) result; } - if (conn->type == CONN_TYPE_AP) { + if (n_written && conn->type == CONN_TYPE_AP) { edge_connection_t *edge_conn = TO_EDGE_CONN(conn); - /*XXXX021 check for overflow.*/ - edge_conn->n_written += (int)n_written; + /* Check for overflow: */ + if (PREDICT_LIKELY(UINT32_MAX - edge_conn->n_written > n_written)) + edge_conn->n_written += (int)n_written; + else + edge_conn->n_written = UINT32_MAX; } connection_buckets_decrement(conn, approx_time(), n_read, n_written); @@ -2404,6 +2822,15 @@ connection_handle_write(connection_t *conn, int force) return 0; } +int +connection_handle_write(connection_t *conn, int force) +{ + int res; + tor_gettimeofday_cache_clear(); + res = connection_handle_write_impl(conn, force); + return res; +} + /** OpenSSL TLS record size is 16383; this is close. The goal here is to * push data out as soon as we know there's enough for a TLS record, so * during periods of high load we won't read entire megabytes from @@ -2577,13 +3004,11 @@ connection_get_by_type_state(int type, int state) /** Return a connection of type <b>type</b> that has rendquery equal * to <b>rendquery</b>, and that is not marked for close. If state - * is non-zero, conn must be of that state too. If rendversion is - * nonnegative, conn must be fetching that rendversion, too. + * is non-zero, conn must be of that state too. */ connection_t * connection_get_by_type_state_rendquery(int type, int state, - const char *rendquery, - int rendversion) + const char *rendquery) { smartlist_t *conns = get_connection_array(); @@ -2598,8 +3023,6 @@ connection_get_by_type_state_rendquery(int type, int state, (!state || state == conn->state)) { if (type == CONN_TYPE_DIR && TO_DIR_CONN(conn)->rend_data && - (rendversion < 0 || - rendversion == TO_DIR_CONN(conn)->rend_data->rend_desc_version) && !rend_cmp_service_ids(rendquery, TO_DIR_CONN(conn)->rend_data->onion_address)) return conn; @@ -2717,10 +3140,10 @@ alloc_http_authenticator(const char *authenticator) static void client_check_address_changed(int sock) { - uint32_t iface_ip, ip_out; + uint32_t iface_ip, ip_out; /* host order */ struct sockaddr_in out_addr; socklen_t out_addr_len = (socklen_t) sizeof(out_addr); - uint32_t *ip; + uint32_t *ip; /* host order */ if (!last_interface_ip) get_interface_address(LOG_INFO, &last_interface_ip); @@ -2734,7 +3157,7 @@ client_check_address_changed(int sock) return; } - /* Okay. If we've used this address previously, we're okay. */ + /* If we've used this address previously, we're okay. */ ip_out = ntohl(out_addr.sin_addr.s_addr); SMARTLIST_FOREACH(outgoing_addrs, uint32_t*, ip_ptr, if (*ip_ptr == ip_out) return; @@ -2830,6 +3253,8 @@ connection_flushed_some(connection_t *conn) r = connection_dirserv_flushed_some(TO_DIR_CONN(conn)); } else if (conn->type == CONN_TYPE_OR) { r = connection_or_flushed_some(TO_OR_CONN(conn)); + } else if (CONN_IS_EDGE(conn)) { + r = connection_edge_flushed_some(TO_EDGE_CONN(conn)); } conn->in_flushed_some = 0; return r; @@ -3033,7 +3458,7 @@ assert_connection_ok(connection_t *conn, time_t now) } // tor_assert(conn->addr && conn->port); tor_assert(conn->address); - if (conn->state > OR_CONN_STATE_PROXY_READING) + if (conn->state > OR_CONN_STATE_PROXY_HANDSHAKING) tor_assert(or_conn->tls); } diff --git a/src/or/connection.h b/src/or/connection.h new file mode 100644 index 0000000000..576d3a63e1 --- /dev/null +++ b/src/or/connection.h @@ -0,0 +1,100 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file connection.h + * \brief Header file for connection.c. + **/ + +#ifndef _TOR_CONNECTION_H +#define _TOR_CONNECTION_H + +const char *conn_type_to_string(int type); +const char *conn_state_to_string(int type, int state); + +dir_connection_t *dir_connection_new(int socket_family); +or_connection_t *or_connection_new(int socket_family); +edge_connection_t *edge_connection_new(int type, int socket_family); +control_connection_t *control_connection_new(int socket_family); +connection_t *connection_new(int type, int socket_family); + +void connection_link_connections(connection_t *conn_a, connection_t *conn_b); +void connection_free(connection_t *conn); +void connection_free_all(void); +void connection_about_to_close_connection(connection_t *conn); +void connection_close_immediate(connection_t *conn); +void _connection_mark_for_close(connection_t *conn,int line, const char *file); + +#define connection_mark_for_close(c) \ + _connection_mark_for_close((c), __LINE__, _SHORT_FILE_) + +void connection_expire_held_open(void); + +int connection_connect(connection_t *conn, const char *address, + const tor_addr_t *addr, + uint16_t port, int *socket_error); + +int connection_proxy_connect(connection_t *conn, int type); +int connection_read_proxy_handshake(connection_t *conn); + +int retry_all_listeners(smartlist_t *replaced_conns, + smartlist_t *new_conns); + +ssize_t connection_bucket_write_limit(connection_t *conn, time_t now); +int global_write_bucket_low(connection_t *conn, size_t attempt, int priority); +void connection_bucket_init(void); +void connection_bucket_refill(int seconds_elapsed, time_t now); + +int connection_handle_read(connection_t *conn); + +int connection_fetch_from_buf(char *string, size_t len, connection_t *conn); + +int connection_wants_to_flush(connection_t *conn); +int connection_outbuf_too_full(connection_t *conn); +int connection_handle_write(connection_t *conn, int force); +void _connection_write_to_buf_impl(const char *string, size_t len, + connection_t *conn, int zlib); +static void connection_write_to_buf(const char *string, size_t len, + connection_t *conn); +static void connection_write_to_buf_zlib(const char *string, size_t len, + dir_connection_t *conn, int done); +static INLINE void +connection_write_to_buf(const char *string, size_t len, connection_t *conn) +{ + _connection_write_to_buf_impl(string, len, conn, 0); +} +static INLINE void +connection_write_to_buf_zlib(const char *string, size_t len, + dir_connection_t *conn, int done) +{ + _connection_write_to_buf_impl(string, len, TO_CONN(conn), done ? -1 : 1); +} + +connection_t *connection_get_by_global_id(uint64_t id); + +connection_t *connection_get_by_type(int type); +connection_t *connection_get_by_type_purpose(int type, int purpose); +connection_t *connection_get_by_type_addr_port_purpose(int type, + const tor_addr_t *addr, + uint16_t port, int purpose); +connection_t *connection_get_by_type_state(int type, int state); +connection_t *connection_get_by_type_state_rendquery(int type, int state, + const char *rendquery); + +#define connection_speaks_cells(conn) ((conn)->type == CONN_TYPE_OR) +int connection_is_listener(connection_t *conn); +int connection_state_is_open(connection_t *conn); +int connection_state_is_connecting(connection_t *conn); + +char *alloc_http_authenticator(const char *authenticator); + +void assert_connection_ok(connection_t *conn, time_t now); +int connection_or_nonopen_was_started_here(or_connection_t *conn); +void connection_dump_buffer_mem_stats(int severity); +void remove_file_if_very_old(const char *fname, time_t now); + +#endif + diff --git a/src/or/connection_edge.c b/src/or/connection_edge.c index 0ec2002afe..037920b688 100644 --- a/src/or/connection_edge.c +++ b/src/or/connection_edge.c @@ -10,6 +10,28 @@ **/ #include "or.h" +#include "buffers.h" +#include "circuitlist.h" +#include "circuituse.h" +#include "config.h" +#include "connection.h" +#include "connection_edge.h" +#include "connection_or.h" +#include "control.h" +#include "dns.h" +#include "dnsserv.h" +#include "dirserv.h" +#include "hibernate.h" +#include "main.h" +#include "policies.h" +#include "reasons.h" +#include "relay.h" +#include "rendclient.h" +#include "rendcommon.h" +#include "rendservice.h" +#include "rephist.h" +#include "router.h" +#include "routerlist.h" #ifdef HAVE_LINUX_TYPES_H #include <linux/types.h> @@ -125,7 +147,7 @@ connection_edge_process_inbuf(edge_connection_t *conn, int package_partial) return 0; case AP_CONN_STATE_OPEN: case EXIT_CONN_STATE_OPEN: - if (connection_edge_package_raw_inbuf(conn, package_partial) < 0) { + if (connection_edge_package_raw_inbuf(conn, package_partial, NULL) < 0) { /* (We already sent an end cell if possible) */ connection_mark_for_close(TO_CONN(conn)); return -1; @@ -279,6 +301,23 @@ connection_edge_end_errno(edge_connection_t *conn) return connection_edge_end(conn, reason); } +/** We just wrote some data to <b>conn</b>; act appropriately. + * + * (That is, if it's open, consider sending a stream-level sendme cell if we + * have just flushed enough.) + */ +int +connection_edge_flushed_some(edge_connection_t *conn) +{ + switch (conn->_base.state) { + case AP_CONN_STATE_OPEN: + case EXIT_CONN_STATE_OPEN: + connection_edge_consider_sending_sendme(conn); + break; + } + return 0; +} + /** Connection <b>conn</b> has finished writing and has no bytes left on * its outbuf. * @@ -330,11 +369,13 @@ connection_edge_finished_connecting(edge_connection_t *edge_conn) tor_assert(conn->state == EXIT_CONN_STATE_CONNECTING); log_info(LD_EXIT,"Exit connection to %s:%u (%s) established.", - escaped_safe_str(conn->address),conn->port, + escaped_safe_str(conn->address), conn->port, safe_str(fmt_addr(&conn->addr))); + rep_hist_note_exit_stream_opened(conn->port); + conn->state = EXIT_CONN_STATE_OPEN; - connection_watch_events(conn, EV_READ); /* stop writing, continue reading */ + connection_watch_events(conn, READ_EVENT); /* stop writing, keep reading */ if (connection_wants_to_flush(conn)) /* in case there are any queued relay * cells */ connection_start_writing(conn); @@ -375,13 +416,16 @@ connection_edge_finished_connecting(edge_connection_t *edge_conn) static int compute_retry_timeout(edge_connection_t *conn) { + int timeout = get_options()->CircuitStreamTimeout; + if (timeout) /* if our config options override the default, use them */ + return timeout; if (conn->num_socks_retries < 2) /* try 0 and try 1 */ return 10; return 15; } /** Find all general-purpose AP streams waiting for a response that sent their - * begin/resolve cell >=15 seconds ago. Detach from their current circuit, and + * begin/resolve cell too long ago. Detach from their current circuit, and * mark their current circuit as unsuitable for new streams. Then call * connection_ap_handshake_attach_circuit() to attach to a new circuit (if * available) or launch a new one. @@ -423,7 +467,8 @@ connection_ap_expire_beginning(void) log_fn(severity, LD_APP, "Tried for %d seconds to get a connection to %s:%d. " "Giving up. (%s)", - seconds_since_born, safe_str(conn->socks_request->address), + seconds_since_born, + safe_str_client(conn->socks_request->address), conn->socks_request->port, conn_state_to_string(CONN_TYPE_AP, conn->_base.state)); connection_mark_unattached_ap(conn, END_STREAM_REASON_TIMEOUT); @@ -440,7 +485,7 @@ connection_ap_expire_beginning(void) circ = circuit_get_by_edge_conn(conn); if (!circ) { /* it's vanished? */ log_info(LD_APP,"Conn is waiting (address %s), but lost its circ.", - safe_str(conn->socks_request->address)); + safe_str_client(conn->socks_request->address)); connection_mark_unattached_ap(conn, END_STREAM_REASON_TIMEOUT); continue; } @@ -450,7 +495,7 @@ connection_ap_expire_beginning(void) "Rend stream is %d seconds late. Giving up on address" " '%s.onion'.", seconds_idle, - safe_str(conn->socks_request->address)); + safe_str_client(conn->socks_request->address)); connection_edge_end(conn, END_STREAM_REASON_TIMEOUT); connection_mark_unattached_ap(conn, END_STREAM_REASON_TIMEOUT); } @@ -460,7 +505,8 @@ connection_ap_expire_beginning(void) log_fn(cutoff < 15 ? LOG_INFO : severity, LD_APP, "We tried for %d seconds to connect to '%s' using exit '%s'." " Retrying on a new circuit.", - seconds_idle, safe_str(conn->socks_request->address), + seconds_idle, + safe_str_client(conn->socks_request->address), conn->cpath_layer ? conn->cpath_layer->extend_info->nickname : "*unnamed*"); /* send an end down the circuit */ @@ -471,6 +517,7 @@ connection_ap_expire_beginning(void) /* kludge to make us not try this circuit again, yet to allow * current streams on it to survive if they can: make it * unattractive to use for new streams */ + /* XXXX023 this is a kludgy way to do this. */ tor_assert(circ->timestamp_dirty); circ->timestamp_dirty -= options->MaxCircuitDirtiness; /* give our stream another 'cutoff' seconds to try */ @@ -511,7 +558,7 @@ connection_ap_attach_pending(void) /** Tell any AP streams that are waiting for a one-hop tunnel to * <b>failed_digest</b> that they are going to fail. */ -/* XXX022 We should get rid of this function, and instead attach +/* XXX023 We should get rid of this function, and instead attach * one-hop streams to circ->p_streams so they get marked in * circuit_mark_for_close like normal p_streams. */ void @@ -577,8 +624,8 @@ circuit_discard_optional_exit_enclaves(extend_info_t *info) tor_assert(edge_conn->socks_request); if (edge_conn->chosen_exit_optional) { log_info(LD_APP, "Giving up on enclave exit '%s' for destination %s.", - safe_str(edge_conn->chosen_exit_name), - escaped_safe_str(edge_conn->socks_request->address)); + safe_str_client(edge_conn->chosen_exit_name), + escaped_safe_str_client(edge_conn->socks_request->address)); edge_conn->chosen_exit_optional = 0; tor_free(edge_conn->chosen_exit_name); /* clears it */ /* if this port is dangerous, warn or reject it now that we don't @@ -683,7 +730,11 @@ addressmap_init(void) static void addressmap_ent_free(void *_ent) { - addressmap_entry_t *ent = _ent; + addressmap_entry_t *ent; + if (!_ent) + return; + + ent = _ent; tor_free(ent->new_address); tor_free(ent); } @@ -692,7 +743,11 @@ addressmap_ent_free(void *_ent) static void addressmap_virtaddress_ent_free(void *_ent) { - virtaddress_entry_t *ent = _ent; + virtaddress_entry_t *ent; + if (!_ent) + return; + + ent = _ent; tor_free(ent->ipv4_address); tor_free(ent->hostname_address); tor_free(ent); @@ -744,7 +799,8 @@ clear_trackexithost_mappings(const char *exitname) tor_strlower(suffix); STRMAP_FOREACH_MODIFY(addressmap, address, addressmap_entry_t *, ent) { - if (ent->source == ADDRMAPSRC_TRACKEXIT && !strcmpend(address, suffix)) { + if (ent->source == ADDRMAPSRC_TRACKEXIT && + !strcmpend(ent->new_address, suffix)) { addressmap_ent_remove(address, ent); MAP_DEL_CURRENT(address); } @@ -753,6 +809,56 @@ clear_trackexithost_mappings(const char *exitname) tor_free(suffix); } +/** Remove all TRACKEXIT mappings from the addressmap for which the target + * host is unknown or no longer allowed. */ +void +addressmap_clear_excluded_trackexithosts(or_options_t *options) +{ + const routerset_t *allow_nodes = options->ExitNodes; + const routerset_t *exclude_nodes = options->_ExcludeExitNodesUnion; + + if (!addressmap) + return; + if (routerset_is_empty(allow_nodes)) + allow_nodes = NULL; + if (allow_nodes == NULL && routerset_is_empty(exclude_nodes)) + return; + + STRMAP_FOREACH_MODIFY(addressmap, address, addressmap_entry_t *, ent) { + size_t len; + const char *target = ent->new_address, *dot; + char *nodename; + routerinfo_t *ri; /* XXX023 Use node_t. */ + + if (strcmpend(target, ".exit")) { + /* Not a .exit mapping */ + continue; + } else if (ent->source != ADDRMAPSRC_TRACKEXIT) { + /* Not a trackexit mapping. */ + continue; + } + len = strlen(target); + if (len < 6) + continue; /* malformed. */ + dot = target + len - 6; /* dot now points to just before .exit */ + dot = strrchr(dot, '.'); /* dot now points to the . before .exit or NULL */ + if (!dot) { + nodename = tor_strndup(target, len-5); + } else { + nodename = tor_strndup(dot+1, strlen(dot+1)-5); + } + ri = router_get_by_nickname(nodename, 0); + tor_free(nodename); + if (!ri || + (allow_nodes && !routerset_contains_router(allow_nodes, ri)) || + routerset_contains_router(exclude_nodes, ri)) { + /* We don't know this one, or we want to be rid of it. */ + addressmap_ent_remove(address, ent); + MAP_DEL_CURRENT(address); + } + } STRMAP_FOREACH_END; +} + /** Remove all entries from the addressmap that were set via the * configuration file or the command line. */ void @@ -782,14 +888,11 @@ addressmap_clean(time_t now) void addressmap_free_all(void) { - if (addressmap) { - strmap_free(addressmap, addressmap_ent_free); - addressmap = NULL; - } - if (virtaddress_reversemap) { - strmap_free(virtaddress_reversemap, addressmap_virtaddress_ent_free); - virtaddress_reversemap = NULL; - } + strmap_free(addressmap, addressmap_ent_free); + addressmap = NULL; + + strmap_free(virtaddress_reversemap, addressmap_virtaddress_ent_free); + virtaddress_reversemap = NULL; } /** Look at address, and rewrite it until it doesn't want any @@ -816,9 +919,9 @@ addressmap_rewrite(char *address, size_t maxlen, time_t *expires_out) return (rewrites > 0); /* done, no rewrite needed */ } - cp = tor_strdup(escaped_safe_str(ent->new_address)); + cp = tor_strdup(escaped_safe_str_client(ent->new_address)); log_info(LD_APP, "Addressmap: rewriting %s to %s", - escaped_safe_str(address), cp); + escaped_safe_str_client(address), cp); if (ent->expires > 1 && ent->expires < expires) expires = ent->expires; tor_free(cp); @@ -826,7 +929,7 @@ addressmap_rewrite(char *address, size_t maxlen, time_t *expires_out) } log_warn(LD_CONFIG, "Loop detected: we've rewritten %s 16 times! Using it as-is.", - escaped_safe_str(address)); + escaped_safe_str_client(address)); /* it's fine to rewrite a rewrite, but don't loop forever */ if (expires_out) *expires_out = TIME_MAX; @@ -848,9 +951,9 @@ addressmap_rewrite_reverse(char *address, size_t maxlen, time_t *expires_out) tor_snprintf(s, len, "REVERSE[%s]", address); ent = strmap_get(addressmap, s); if (ent) { - cp = tor_strdup(escaped_safe_str(ent->new_address)); + cp = tor_strdup(escaped_safe_str_client(ent->new_address)); log_info(LD_APP, "Rewrote reverse lookup %s -> %s", - escaped_safe_str(s), cp); + escaped_safe_str_client(s), cp); tor_free(cp); strlcpy(address, ent->new_address, maxlen); r = 1; @@ -912,7 +1015,9 @@ addressmap_register(const char *address, char *new_address, time_t expires, if (expires > 1) { log_info(LD_APP,"Temporary addressmap ('%s' to '%s') not performed, " "since it's already mapped to '%s'", - safe_str(address), safe_str(new_address), safe_str(ent->new_address)); + safe_str_client(address), + safe_str_client(new_address), + safe_str_client(ent->new_address)); tor_free(new_address); return; } @@ -931,7 +1036,8 @@ addressmap_register(const char *address, char *new_address, time_t expires, ent->source = source; log_info(LD_CONFIG, "Addressmap: (re)mapped '%s' to '%s'", - safe_str(address), safe_str(ent->new_address)); + safe_str_client(address), + safe_str_client(ent->new_address)); control_event_address_mapped(address, ent->new_address, expires, NULL); } @@ -951,7 +1057,8 @@ client_dns_incr_failures(const char *address) if (ent->num_resolve_failures < SHORT_MAX) ++ent->num_resolve_failures; /* don't overflow */ log_info(LD_APP, "Address %s now has %d resolve failures.", - safe_str(address), ent->num_resolve_failures); + safe_str_client(address), + ent->num_resolve_failures); return ent->num_resolve_failures; } @@ -1247,8 +1354,10 @@ addressmap_register_virtual_address(int type, char *new_address) log_warn(LD_BUG, "Internal confusion: I thought that '%s' was mapped to by " "'%s', but '%s' really maps to '%s'. This is a harmless bug.", - safe_str(new_address), safe_str(*addrp), safe_str(*addrp), - ent?safe_str(ent->new_address):"(nothing)"); + safe_str_client(new_address), + safe_str_client(*addrp), + safe_str_client(*addrp), + ent?safe_str_client(ent->new_address):"(nothing)"); } tor_free(*addrp); @@ -1276,7 +1385,8 @@ addressmap_register_virtual_address(int type, char *new_address) (type == RESOLVED_TYPE_IPV4) ? vent->ipv4_address : vent->hostname_address)); log_info(LD_APP, "Map from %s to %s okay.", - safe_str(*addrp),safe_str(new_address)); + safe_str_client(*addrp), + safe_str_client(new_address)); } #endif @@ -1391,6 +1501,26 @@ consider_plaintext_ports(edge_connection_t *conn, uint16_t port) * different one? */ #define TRACKHOSTEXITS_RETRIES 5 +/** Call connection_ap_handshake_rewrite_and_attach() unless a controller + * asked us to leave streams unattached. Return 0 in that case. + * + * See connection_ap_handshake_rewrite_and_attach()'s + * documentation for arguments and return value. + */ +int +connection_ap_rewrite_and_attach_if_allowed(edge_connection_t *conn, + origin_circuit_t *circ, + crypt_path_t *cpath) +{ + or_options_t *options = get_options(); + + if (options->LeaveStreamsUnattached) { + conn->_base.state = AP_CONN_STATE_CONTROLLER_WAIT; + return 0; + } + return connection_ap_handshake_rewrite_and_attach(conn, circ, cpath); +} + /** Connection <b>conn</b> just finished its socks handshake, or the * controller asked us to take care of it. If <b>circ</b> is defined, * then that's where we'll want to attach it. Otherwise we have to @@ -1415,16 +1545,20 @@ connection_ap_handshake_rewrite_and_attach(edge_connection_t *conn, hostname_type_t addresstype; or_options_t *options = get_options(); struct in_addr addr_tmp; + /* We set this to true if this is an address we should automatically + * remap to a local address in VirtualAddrNetwork */ int automap = 0; char orig_address[MAX_SOCKS_ADDR_LEN]; time_t map_expires = TIME_MAX; + /* This will be set to true iff the address starts out as a non-.exit + address, and we remap it to one because of an entry in the addressmap. */ int remapped_to_exit = 0; time_t now = time(NULL); tor_strlower(socks->address); /* normalize it */ strlcpy(orig_address, socks->address, sizeof(orig_address)); log_debug(LD_APP,"Client asked for %s:%d", - safe_str(socks->address), + safe_str_client(socks->address), socks->port); if (socks->command == SOCKS_COMMAND_RESOLVE && @@ -1446,7 +1580,8 @@ connection_ap_handshake_rewrite_and_attach(edge_connection_t *conn, return -1; } log_info(LD_APP, "Automapping %s to %s", - escaped_safe_str(socks->address), safe_str(new_addr)); + escaped_safe_str_client(socks->address), + safe_str_client(new_addr)); strlcpy(socks->address, new_addr, sizeof(socks->address)); } } @@ -1503,7 +1638,7 @@ connection_ap_handshake_rewrite_and_attach(edge_connection_t *conn, * information. */ log_warn(LD_APP,"Missing mapping for virtual address '%s'. Refusing.", - socks->address); /* don't safe_str() this yet. */ + safe_str_client(socks->address)); connection_mark_unattached_ap(conn, END_STREAM_REASON_INTERNAL); return -1; } @@ -1511,11 +1646,12 @@ connection_ap_handshake_rewrite_and_attach(edge_connection_t *conn, /* Parse the address provided by SOCKS. Modify it in-place if it * specifies a hidden-service (.onion) or particular exit node (.exit). */ - addresstype = parse_extended_hostname(socks->address); + addresstype = parse_extended_hostname(socks->address, + remapped_to_exit || options->AllowDotExit); if (addresstype == BAD_HOSTNAME) { log_warn(LD_APP, "Invalid onion hostname %s; rejecting", - safe_str(socks->address)); + safe_str_client(socks->address)); control_event_client_status(LOG_WARN, "SOCKS_BAD_HOSTNAME HOSTNAME=%s", escaped(socks->address)); connection_mark_unattached_ap(conn, END_STREAM_REASON_TORPROTOCOL); @@ -1524,38 +1660,61 @@ connection_ap_handshake_rewrite_and_attach(edge_connection_t *conn, if (addresstype == EXIT_HOSTNAME) { /* foo.exit -- modify conn->chosen_exit_node to specify the exit - * node, and conn->address to hold only the address portion.*/ + * node, and conn->address to hold only the address portion. */ char *s = strrchr(socks->address,'.'); + + /* If StrictNodes is not set, then .exit overrides ExcludeNodes. */ + routerset_t *excludeset = options->StrictNodes ? + options->_ExcludeExitNodesUnion : options->ExcludeExitNodes; + /*XXX023 make this a node_t. */ + routerinfo_t *router; + tor_assert(!automap); if (s) { + /* The address was of the form "(stuff).(name).exit */ if (s[1] != '\0') { conn->chosen_exit_name = tor_strdup(s+1); + router = router_get_by_nickname(conn->chosen_exit_name, 1); if (remapped_to_exit) /* 5 tries before it expires the addressmap */ conn->chosen_exit_retries = TRACKHOSTEXITS_RETRIES; *s = 0; } else { + /* Oops, the address was (stuff)..exit. That's not okay. */ log_warn(LD_APP,"Malformed exit address '%s.exit'. Refusing.", - safe_str(socks->address)); + safe_str_client(socks->address)); control_event_client_status(LOG_WARN, "SOCKS_BAD_HOSTNAME HOSTNAME=%s", escaped(socks->address)); connection_mark_unattached_ap(conn, END_STREAM_REASON_TORPROTOCOL); return -1; } } else { - routerinfo_t *r; + /* It looks like they just asked for "foo.exit". */ conn->chosen_exit_name = tor_strdup(socks->address); - r = router_get_by_nickname(conn->chosen_exit_name, 1); - *socks->address = 0; - if (r) { - strlcpy(socks->address, r->address, sizeof(socks->address)); - } else { - log_warn(LD_APP, - "Unrecognized server in exit address '%s.exit'. Refusing.", - safe_str(socks->address)); - connection_mark_unattached_ap(conn, END_STREAM_REASON_TORPROTOCOL); - return -1; + router = router_get_by_nickname(conn->chosen_exit_name, 1); + if (router) { + *socks->address = 0; + strlcpy(socks->address, router->address, sizeof(socks->address)); } } + /* Now make sure that the chosen exit exists... */ + if (!router) { + log_warn(LD_APP, + "Unrecognized relay in exit address '%s.exit'. Refusing.", + safe_str_client(socks->address)); + connection_mark_unattached_ap(conn, END_STREAM_REASON_TORPROTOCOL); + return -1; + } + /* ...and make sure that it isn't excluded. */ + if (routerset_contains_router(excludeset, router)) { + log_warn(LD_APP, + "Excluded relay in exit address '%s.exit'. Refusing.", + safe_str_client(socks->address)); + connection_mark_unattached_ap(conn, END_STREAM_REASON_TORPROTOCOL); + return -1; + } + /* XXXX022-1090 Should we also allow foo.bar.exit if ExitNodes is set and + Bar is not listed in it? I say yes, but our revised manpage branch + implies no. */ } if (addresstype != ONION_HOSTNAME) { @@ -1565,7 +1724,7 @@ connection_ap_handshake_rewrite_and_attach(edge_connection_t *conn, escaped(socks->address)); log_warn(LD_APP, "Destination '%s' seems to be an invalid hostname. Failing.", - safe_str(socks->address)); + safe_str_client(socks->address)); connection_mark_unattached_ap(conn, END_STREAM_REASON_TORPROTOCOL); return -1; } @@ -1574,18 +1733,6 @@ connection_ap_handshake_rewrite_and_attach(edge_connection_t *conn, uint32_t answer; struct in_addr in; /* Reply to resolves immediately if we can. */ - if (strlen(socks->address) > RELAY_PAYLOAD_SIZE) { - log_warn(LD_APP,"Address to be resolved is too large. Failing."); - control_event_client_status(LOG_WARN, "SOCKS_BAD_HOSTNAME HOSTNAME=%s", - escaped(socks->address)); - connection_ap_handshake_socks_resolved(conn, - RESOLVED_TYPE_ERROR_TRANSIENT, - 0,NULL,-1,TIME_MAX); - connection_mark_unattached_ap(conn, - END_STREAM_REASON_SOCKSPROTOCOL | - END_STREAM_REASON_FLAG_ALREADY_SOCKS_REPLIED); - return -1; - } if (tor_inet_aton(socks->address, &in)) { /* see if it's an IP already */ /* leave it in network order */ answer = in.s_addr; @@ -1608,6 +1755,28 @@ connection_ap_handshake_rewrite_and_attach(edge_connection_t *conn, connection_mark_unattached_ap(conn, END_STREAM_REASON_TORPROTOCOL); return -1; } + if (options->ClientRejectInternalAddresses && + !conn->use_begindir && !conn->chosen_exit_name && !circ) { + tor_addr_t addr; + if (tor_addr_from_str(&addr, socks->address) >= 0 && + tor_addr_is_internal(&addr, 0)) { + /* If this is an explicit private address with no chosen exit node, + * then we really don't want to try to connect to it. That's + * probably an error. */ + if (conn->is_transparent_ap) { + log_warn(LD_NET, + "Rejecting request for anonymous connection to private " + "address %s on a TransPort or NATDPort. Possible loop " + "in your NAT rules?", safe_str_client(socks->address)); + } else { + log_warn(LD_NET, + "Rejecting SOCKS request for anonymous connection to " + "private address %s", safe_str_client(socks->address)); + } + connection_mark_unattached_ap(conn, END_STREAM_REASON_PRIVATE_ADDR); + return -1; + } + } if (!conn->use_begindir && !conn->chosen_exit_name && !circ) { /* see if we can find a suitable enclave exit */ @@ -1616,7 +1785,7 @@ connection_ap_handshake_rewrite_and_attach(edge_connection_t *conn, if (r) { log_info(LD_APP, "Redirecting address %s to exit at enclave router %s", - safe_str(socks->address), r->nickname); + safe_str_client(socks->address), r->nickname); /* use the hex digest, not nickname, in case there are two routers with this nickname */ conn->chosen_exit_name = @@ -1680,12 +1849,12 @@ connection_ap_handshake_rewrite_and_attach(edge_connection_t *conn, strlcpy(conn->rend_data->onion_address, socks->address, sizeof(conn->rend_data->onion_address)); log_info(LD_REND,"Got a hidden service request for ID '%s'", - safe_str(conn->rend_data->onion_address)); + safe_str_client(conn->rend_data->onion_address)); /* see if we already have it cached */ r = rend_cache_lookup_entry(conn->rend_data->onion_address, -1, &entry); if (r<0) { log_warn(LD_BUG,"Invalid service name '%s'", - safe_str(conn->rend_data->onion_address)); + safe_str_client(conn->rend_data->onion_address)); connection_mark_unattached_ap(conn, END_STREAM_REASON_TORPROTOCOL); return -1; } @@ -1707,32 +1876,15 @@ connection_ap_handshake_rewrite_and_attach(edge_connection_t *conn, if (r==0) { conn->_base.state = AP_CONN_STATE_RENDDESC_WAIT; log_info(LD_REND, "Unknown descriptor %s. Fetching.", - safe_str(conn->rend_data->onion_address)); - /* Fetch both, v0 and v2 rend descriptors in parallel. Use whichever - * arrives first. Exception: When using client authorization, only - * fetch v2 descriptors.*/ + safe_str_client(conn->rend_data->onion_address)); rend_client_refetch_v2_renddesc(conn->rend_data); - if (conn->rend_data->auth_type == REND_NO_AUTH) - rend_client_refetch_renddesc(conn->rend_data->onion_address); } else { /* r > 0 */ - if (now - entry->received < NUM_SECONDS_BEFORE_HS_REFETCH) { - conn->_base.state = AP_CONN_STATE_CIRCUIT_WAIT; - log_info(LD_REND, "Descriptor is here and fresh enough. Great."); - if (connection_ap_handshake_attach_circuit(conn) < 0) { - if (!conn->_base.marked_for_close) - connection_mark_unattached_ap(conn, END_STREAM_REASON_CANT_ATTACH); - return -1; - } - } else { - conn->_base.state = AP_CONN_STATE_RENDDESC_WAIT; - log_info(LD_REND, "Stale descriptor %s. Re-fetching.", - safe_str(conn->rend_data->onion_address)); - /* Fetch both, v0 and v2 rend descriptors in parallel. Use whichever - * arrives first. Exception: When using client authorization, only - * fetch v2 descriptors.*/ - rend_client_refetch_v2_renddesc(conn->rend_data); - if (conn->rend_data->auth_type == REND_NO_AUTH) - rend_client_refetch_renddesc(conn->rend_data->onion_address); + conn->_base.state = AP_CONN_STATE_CIRCUIT_WAIT; + log_info(LD_REND, "Descriptor is here. Great."); + if (connection_ap_handshake_attach_circuit(conn) < 0) { + if (!conn->_base.marked_for_close) + connection_mark_unattached_ap(conn, END_STREAM_REASON_CANT_ATTACH); + return -1; } } return 0; @@ -1920,24 +2072,12 @@ connection_ap_handshake_process_socks(edge_connection_t *conn) return -1; } /* else socks handshake is done, continue processing */ - if (hostname_is_noconnect_address(socks->address)) - { - control_event_stream_status(conn, STREAM_EVENT_NEW, 0); - control_event_stream_status(conn, STREAM_EVENT_CLOSED, 0); - connection_mark_unattached_ap(conn, END_STREAM_REASON_DONE); - return -1; - } - if (SOCKS_COMMAND_IS_CONNECT(socks->command)) control_event_stream_status(conn, STREAM_EVENT_NEW, 0); else control_event_stream_status(conn, STREAM_EVENT_NEW_RESOLVE, 0); - if (options->LeaveStreamsUnattached) { - conn->_base.state = AP_CONN_STATE_CONTROLLER_WAIT; - return 0; - } - return connection_ap_handshake_rewrite_and_attach(conn, NULL, NULL); + return connection_ap_rewrite_and_attach_if_allowed(conn, NULL, NULL); } /** connection_init_accepted_conn() found a new trans AP conn. @@ -1951,7 +2091,6 @@ int connection_ap_process_transparent(edge_connection_t *conn) { socks_request_t *socks; - or_options_t *options = get_options(); tor_assert(conn); tor_assert(conn->_base.type == CONN_TYPE_AP); @@ -1975,11 +2114,7 @@ connection_ap_process_transparent(edge_connection_t *conn) control_event_stream_status(conn, STREAM_EVENT_NEW, 0); - if (options->LeaveStreamsUnattached) { - conn->_base.state = AP_CONN_STATE_CONTROLLER_WAIT; - return 0; - } - return connection_ap_handshake_rewrite_and_attach(conn, NULL, NULL); + return connection_ap_rewrite_and_attach_if_allowed(conn, NULL, NULL); } /** connection_edge_process_inbuf() found a conn in state natd_wait. See if @@ -2000,7 +2135,6 @@ connection_ap_process_natd(edge_connection_t *conn) size_t tlen = 30; int err, port_ok; socks_request_t *socks; - or_options_t *options = get_options(); tor_assert(conn); tor_assert(conn->_base.type == CONN_TYPE_AP); @@ -2016,13 +2150,13 @@ connection_ap_process_natd(edge_connection_t *conn) if (err == 0) return 0; if (err < 0) { - log_warn(LD_APP,"Natd handshake failed (DEST too long). Closing"); + log_warn(LD_APP,"NATD handshake failed (DEST too long). Closing"); connection_mark_unattached_ap(conn, END_STREAM_REASON_INVALID_NATD_DEST); return -1; } if (strcmpstart(tmp_buf, "[DEST ")) { - log_warn(LD_APP,"Natd handshake was ill-formed; closing. The client " + log_warn(LD_APP,"NATD handshake was ill-formed; closing. The client " "said: %s", escaped(tmp_buf)); connection_mark_unattached_ap(conn, END_STREAM_REASON_INVALID_NATD_DEST); @@ -2031,7 +2165,7 @@ connection_ap_process_natd(edge_connection_t *conn) daddr = tbuf = &tmp_buf[0] + 6; /* after end of "[DEST " */ if (!(tbuf = strchr(tbuf, ' '))) { - log_warn(LD_APP,"Natd handshake was ill-formed; closing. The client " + log_warn(LD_APP,"NATD handshake was ill-formed; closing. The client " "said: %s", escaped(tmp_buf)); connection_mark_unattached_ap(conn, END_STREAM_REASON_INVALID_NATD_DEST); @@ -2045,7 +2179,7 @@ connection_ap_process_natd(edge_connection_t *conn) socks->port = (uint16_t) tor_parse_long(tbuf, 10, 1, 65535, &port_ok, &daddr); if (!port_ok) { - log_warn(LD_APP,"Natd handshake failed; port %s is ill-formed or out " + log_warn(LD_APP,"NATD handshake failed; port %s is ill-formed or out " "of range.", escaped(tbuf)); connection_mark_unattached_ap(conn, END_STREAM_REASON_INVALID_NATD_DEST); return -1; @@ -2056,13 +2190,9 @@ connection_ap_process_natd(edge_connection_t *conn) control_event_stream_status(conn, STREAM_EVENT_NEW, 0); - if (options->LeaveStreamsUnattached) { - conn->_base.state = AP_CONN_STATE_CONTROLLER_WAIT; - return 0; - } conn->_base.state = AP_CONN_STATE_CIRCUIT_WAIT; - return connection_ap_handshake_rewrite_and_attach(conn, NULL, NULL); + return connection_ap_rewrite_and_attach_if_allowed(conn, NULL, NULL); } /** Iterate over the two bytes of stream_id until we get one that is not @@ -2075,7 +2205,7 @@ get_unique_stream_id_by_circ(origin_circuit_t *circ) streamid_t test_stream_id; uint32_t attempts=0; -again: + again: test_stream_id = circ->next_stream_id++; if (++attempts > 1<<16) { /* Make sure we don't loop forever if all stream_id's are used. */ @@ -2112,8 +2242,14 @@ connection_ap_handshake_send_begin(edge_connection_t *ap_conn) ap_conn->stream_id = get_unique_stream_id_by_circ(circ); if (ap_conn->stream_id==0) { + /* XXXX023 Instead of closing this stream, we should make it get + * retried on another circuit. */ connection_mark_unattached_ap(ap_conn, END_STREAM_REASON_INTERNAL); - circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_RESOURCELIMIT); + + /* Mark this circuit "unusable for new streams". */ + /* XXXX023 this is a kludgy way to do this. */ + tor_assert(circ->_base.timestamp_dirty); + circ->_base.timestamp_dirty -= get_options()->MaxCircuitDirtiness; return -1; } @@ -2171,9 +2307,14 @@ connection_ap_handshake_send_resolve(edge_connection_t *ap_conn) ap_conn->stream_id = get_unique_stream_id_by_circ(circ); if (ap_conn->stream_id==0) { + /* XXXX023 Instead of closing this stream, we should make it get + * retried on another circuit. */ connection_mark_unattached_ap(ap_conn, END_STREAM_REASON_INTERNAL); - /*XXXX022 _close_ the circuit because it's full? That sounds dumb. */ - circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_RESOURCELIMIT); + + /* Mark this circuit "unusable for new streams". */ + /* XXXX023 this is a kludgy way to do this. */ + tor_assert(circ->_base.timestamp_dirty); + circ->_base.timestamp_dirty -= get_options()->MaxCircuitDirtiness; return -1; } @@ -2191,7 +2332,7 @@ connection_ap_handshake_send_resolve(edge_connection_t *ap_conn) r = tor_addr_parse_reverse_lookup_name(&addr, a, AF_INET, 1); if (r <= 0) { log_warn(LD_APP, "Rejecting ill-formed reverse lookup of %s", - safe_str(a)); + safe_str_client(a)); connection_mark_unattached_ap(ap_conn, END_STREAM_REASON_INTERNAL); return -1; } @@ -2199,7 +2340,7 @@ connection_ap_handshake_send_resolve(edge_connection_t *ap_conn) r = tor_addr_to_reverse_lookup_name(inaddr_buf, sizeof(inaddr_buf), &addr); if (r < 0) { log_warn(LD_BUG, "Couldn't generate reverse lookup hostname of %s", - safe_str(a)); + safe_str_client(a)); connection_mark_unattached_ap(ap_conn, END_STREAM_REASON_INTERNAL); return -1; } @@ -2209,12 +2350,6 @@ connection_ap_handshake_send_resolve(edge_connection_t *ap_conn) tor_assert(payload_len <= (int)sizeof(inaddr_buf)); } - if (payload_len > RELAY_PAYLOAD_SIZE) { - /* This should be impossible: we don't accept addresses this big. */ - connection_mark_unattached_ap(ap_conn, END_STREAM_REASON_INTERNAL); - return -1; - } - log_debug(LD_APP, "Sending relay cell to begin stream %d.", ap_conn->stream_id); @@ -2246,7 +2381,8 @@ connection_ap_make_link(char *address, uint16_t port, edge_connection_t *conn; log_info(LD_APP,"Making internal %s tunnel to %s:%d ...", - want_onehop ? "direct" : "anonymized" , safe_str(address),port); + want_onehop ? "direct" : "anonymized", + safe_str_client(address), port); conn = edge_connection_new(CONN_TYPE_AP, AF_INET); conn->_base.linked = 1; /* so that we can add it safely below. */ @@ -2336,7 +2472,7 @@ tell_controller_about_resolved_result(edge_connection_t *conn, * certain errors or for values that didn't come via DNS. <b>expires</b> is * a time when the answer expires, or -1 or TIME_MAX if there's a good TTL. **/ -/* XXXX022 the use of the ttl and expires fields is nutty. Let's make this +/* XXXX023 the use of the ttl and expires fields is nutty. Let's make this * interface and those that use it less ugly. */ void connection_ap_handshake_socks_resolved(edge_connection_t *conn, @@ -2512,6 +2648,7 @@ connection_exit_begin_conn(cell_t *cell, circuit_t *circ) char *address=NULL; uint16_t port; or_circuit_t *or_circ = NULL; + or_options_t *options = get_options(); assert_circuit_ok(circ); if (!CIRCUIT_IS_ORIGIN(circ)) @@ -2526,7 +2663,7 @@ connection_exit_begin_conn(cell_t *cell, circuit_t *circ) * that we have a stream connected to a circuit, and we don't connect to a * circuit until we have a pending/successful resolve. */ - if (!server_mode(get_options()) && + if (!server_mode(options) && circ->purpose != CIRCUIT_PURPOSE_S_REND_JOINED) { log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, "Relay begin cell at non-server. Closing."); @@ -2560,21 +2697,30 @@ connection_exit_begin_conn(cell_t *cell, circuit_t *circ) tor_free(address); return 0; } - if (or_circ && or_circ->is_first_hop && - !get_options()->AllowSingleHopExits) { + if (or_circ && or_circ->p_conn && !options->AllowSingleHopExits && + (or_circ->is_first_hop || + (!connection_or_digest_is_known_relay( + or_circ->p_conn->identity_digest) && + should_refuse_unknown_exits(options)))) { /* Don't let clients use us as a single-hop proxy, unless the user - * has explicitly allowed that in the config. It attracts attackers + * has explicitly allowed that in the config. It attracts attackers * and users who'd be better off with, well, single-hop proxies. */ log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, - "Attempt to open a stream on first hop of circuit. Closing."); + "Attempt by %s to open a stream %s. Closing.", + safe_str(or_circ->p_conn->_base.address), + or_circ->is_first_hop ? "on first hop of circuit" : + "from unknown relay"); relay_send_end_cell_from_edge(rh.stream_id, circ, - END_STREAM_REASON_TORPROTOCOL, NULL); + or_circ->is_first_hop ? + END_STREAM_REASON_TORPROTOCOL : + END_STREAM_REASON_MISC, + NULL); tor_free(address); return 0; } } else if (rh.command == RELAY_COMMAND_BEGIN_DIR) { - if (!directory_permits_begindir_requests(get_options()) || + if (!directory_permits_begindir_requests(options) || circ->purpose != CIRCUIT_PURPOSE_OR) { relay_send_end_cell_from_edge(rh.stream_id, circ, END_STREAM_REASON_NOTDIRECTORY, NULL); @@ -2600,6 +2746,11 @@ connection_exit_begin_conn(cell_t *cell, circuit_t *circ) log_debug(LD_EXIT,"Creating new exit connection."); n_stream = edge_connection_new(CONN_TYPE_EXIT, AF_INET); + + /* Remember the tunneled request ID in the new edge connection, so that + * we can measure download times. */ + TO_CONN(n_stream)->dirreq_id = circ->dirreq_id; + n_stream->_base.purpose = EXIT_PURPOSE_CONNECT; n_stream->stream_id = rh.stream_id; @@ -2630,7 +2781,7 @@ connection_exit_begin_conn(cell_t *cell, circuit_t *circ) log_debug(LD_REND,"Finished assigning addr/port"); n_stream->cpath_layer = origin_circ->cpath->prev; /* link it */ - /* add it into the linked list of n_streams on this circuit */ + /* add it into the linked list of p_streams on this circuit */ n_stream->next_stream = origin_circ->p_streams; n_stream->on_circuit = circ; origin_circ->p_streams = n_stream; @@ -2657,7 +2808,7 @@ connection_exit_begin_conn(cell_t *cell, circuit_t *circ) if (rh.command == RELAY_COMMAND_BEGIN_DIR) { tor_assert(or_circ); if (or_circ->p_conn && !tor_addr_is_null(&or_circ->p_conn->real_addr)) - tor_addr_assign(&n_stream->_base.addr, &or_circ->p_conn->real_addr); + tor_addr_copy(&n_stream->_base.addr, &or_circ->p_conn->real_addr); return connection_exit_connect_dir(n_stream); } @@ -2750,7 +2901,7 @@ connection_exit_connect(edge_connection_t *edge_conn) if (!connection_edge_is_rendezvous_stream(edge_conn) && router_compare_to_my_exit_policy(edge_conn)) { log_info(LD_EXIT,"%s:%d failed exit policy. Closing.", - escaped_safe_str(conn->address), conn->port); + escaped_safe_str_client(conn->address), conn->port); connection_edge_end(edge_conn, END_STREAM_REASON_EXITPOLICY); circuit_detach_stream(circuit_get_by_edge_conn(edge_conn), edge_conn); connection_free(conn); @@ -2762,17 +2913,17 @@ connection_exit_connect(edge_connection_t *edge_conn) log_debug(LD_EXIT,"about to try connecting"); switch (connection_connect(conn, conn->address, addr, port, &socket_error)) { - case -1: - /* XXX021 use socket_error below rather than trying to piece things - * together from the current errno, which may have been clobbered. */ - connection_edge_end_errno(edge_conn); + case -1: { + int reason = errno_to_stream_end_reason(socket_error); + connection_edge_end(edge_conn, reason); circuit_detach_stream(circuit_get_by_edge_conn(edge_conn), edge_conn); connection_free(conn); return; + } case 0: conn->state = EXIT_CONN_STATE_CONNECTING; - connection_watch_events(conn, EV_WRITE | EV_READ); + connection_watch_events(conn, READ_EVENT | WRITE_EVENT); /* writable indicates finish; * readable/error indicates broken link in windows-land. */ return; @@ -2785,7 +2936,7 @@ connection_exit_connect(edge_connection_t *edge_conn) log_warn(LD_BUG,"newly connected conn had data waiting!"); // connection_start_writing(conn); } - connection_watch_events(conn, EV_READ); + connection_watch_events(conn, READ_EVENT); /* also, deliver a 'connected' cell back through the circuit. */ if (connection_edge_is_rendezvous_stream(edge_conn)) { @@ -2832,13 +2983,17 @@ connection_exit_connect_dir(edge_connection_t *exitconn) dirconn = dir_connection_new(AF_INET); - tor_addr_assign(&dirconn->_base.addr, &exitconn->_base.addr); + tor_addr_copy(&dirconn->_base.addr, &exitconn->_base.addr); dirconn->_base.port = 0; dirconn->_base.address = tor_strdup(exitconn->_base.address); dirconn->_base.type = CONN_TYPE_DIR; dirconn->_base.purpose = DIR_PURPOSE_SERVER; dirconn->_base.state = DIR_CONN_STATE_SERVER_COMMAND_WAIT; + /* Note that the new dir conn belongs to the same tunneled request as + * the edge conn, so that we can measure download times. */ + TO_CONN(dirconn)->dirreq_id = TO_CONN(exitconn)->dirreq_id; + connection_link_connections(TO_CONN(dirconn), TO_CONN(exitconn)); if (connection_add(TO_CONN(exitconn))<0) { @@ -2893,6 +3048,8 @@ connection_edge_is_rendezvous_stream(edge_connection_t *conn) int connection_ap_can_use_exit(edge_connection_t *conn, routerinfo_t *exit) { + or_options_t *options = get_options(); + tor_assert(conn); tor_assert(conn->_base.type == CONN_TYPE_AP); tor_assert(conn->socks_request); @@ -2938,20 +3095,26 @@ connection_ap_can_use_exit(edge_connection_t *conn, routerinfo_t *exit) if (!conn->chosen_exit_name && policy_is_reject_star(exit->exit_policy)) return 0; } + if (options->_ExcludeExitNodesUnion && + routerset_contains_router(options->_ExcludeExitNodesUnion, exit)) { + /* Not a suitable exit. Refuse it. */ + return 0; + } + return 1; } /** If address is of the form "y.onion" with a well-formed handle y: * Put a NUL after y, lower-case it, and return ONION_HOSTNAME. * - * If address is of the form "y.exit": + * If address is of the form "y.exit" and <b>allowdotexit</b> is true: * Put a NUL after y and return EXIT_HOSTNAME. * * Otherwise: * Return NORMAL_HOSTNAME and change nothing. */ hostname_type_t -parse_extended_hostname(char *address) +parse_extended_hostname(char *address, int allowdotexit) { char *s; char query[REND_SERVICE_ID_LEN_BASE32+1]; @@ -2960,8 +3123,16 @@ parse_extended_hostname(char *address) if (!s) return NORMAL_HOSTNAME; /* no dot, thus normal */ if (!strcmp(s+1,"exit")) { - *s = 0; /* NUL-terminate it */ - return EXIT_HOSTNAME; /* .exit */ + if (allowdotexit) { + *s = 0; /* NUL-terminate it */ + return EXIT_HOSTNAME; /* .exit */ + } else { + log_warn(LD_APP, "The \".exit\" notation is disabled in Tor due to " + "security risks. Set AllowDotExit in your torrc to enable " + "it."); + /* FFFF send a controller event too to notify Vidalia users */ + return BAD_HOSTNAME; + } } if (strcmp(s+1,"onion")) return NORMAL_HOSTNAME; /* neither .exit nor .onion, thus normal */ @@ -2974,17 +3145,9 @@ parse_extended_hostname(char *address) if (rend_valid_service_id(query)) { return ONION_HOSTNAME; /* success */ } -failed: + failed: /* otherwise, return to previous state and return 0 */ *s = '.'; return BAD_HOSTNAME; } -/** Check if the address is of the form "y.noconnect" - */ -int -hostname_is_noconnect_address(const char *address) -{ - return ! strcasecmpend(address, ".noconnect"); -} - diff --git a/src/or/connection_edge.h b/src/or/connection_edge.h new file mode 100644 index 0000000000..70d0dd2713 --- /dev/null +++ b/src/or/connection_edge.h @@ -0,0 +1,101 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file connection_edge.h + * \brief Header file for connection_edge.c. + **/ + +#ifndef _TOR_CONNECTION_EDGE_H +#define _TOR_CONNECTION_EDGE_H + +#define connection_mark_unattached_ap(conn, endreason) \ + _connection_mark_unattached_ap((conn), (endreason), __LINE__, _SHORT_FILE_) + +void _connection_mark_unattached_ap(edge_connection_t *conn, int endreason, + int line, const char *file); +int connection_edge_reached_eof(edge_connection_t *conn); +int connection_edge_process_inbuf(edge_connection_t *conn, + int package_partial); +int connection_edge_destroy(circid_t circ_id, edge_connection_t *conn); +int connection_edge_end(edge_connection_t *conn, uint8_t reason); +int connection_edge_end_errno(edge_connection_t *conn); +int connection_edge_flushed_some(edge_connection_t *conn); +int connection_edge_finished_flushing(edge_connection_t *conn); +int connection_edge_finished_connecting(edge_connection_t *conn); + +int connection_ap_handshake_send_begin(edge_connection_t *ap_conn); +int connection_ap_handshake_send_resolve(edge_connection_t *ap_conn); + +edge_connection_t *connection_ap_make_link(char *address, uint16_t port, + const char *digest, + int use_begindir, int want_onehop); +void connection_ap_handshake_socks_reply(edge_connection_t *conn, char *reply, + size_t replylen, + int endreason); +void connection_ap_handshake_socks_resolved(edge_connection_t *conn, + int answer_type, + size_t answer_len, + const uint8_t *answer, + int ttl, + time_t expires); + +int connection_exit_begin_conn(cell_t *cell, circuit_t *circ); +int connection_exit_begin_resolve(cell_t *cell, or_circuit_t *circ); +void connection_exit_connect(edge_connection_t *conn); +int connection_edge_is_rendezvous_stream(edge_connection_t *conn); +int connection_ap_can_use_exit(edge_connection_t *conn, routerinfo_t *exit); +void connection_ap_expire_beginning(void); +void connection_ap_attach_pending(void); +void connection_ap_fail_onehop(const char *failed_digest, + cpath_build_state_t *build_state); +void circuit_discard_optional_exit_enclaves(extend_info_t *info); +int connection_ap_detach_retriable(edge_connection_t *conn, + origin_circuit_t *circ, + int reason); +int connection_ap_process_transparent(edge_connection_t *conn); + +int address_is_invalid_destination(const char *address, int client); + +void addressmap_init(void); +void addressmap_clear_excluded_trackexithosts(or_options_t *options); +void addressmap_clean(time_t now); +void addressmap_clear_configured(void); +void addressmap_clear_transient(void); +void addressmap_free_all(void); +int addressmap_rewrite(char *address, size_t maxlen, time_t *expires_out); +int addressmap_have_mapping(const char *address, int update_timeout); + +void addressmap_register(const char *address, char *new_address, + time_t expires, addressmap_entry_source_t source); +int parse_virtual_addr_network(const char *val, int validate_only, + char **msg); +int client_dns_incr_failures(const char *address); +void client_dns_clear_failures(const char *address); +void client_dns_set_addressmap(const char *address, uint32_t val, + const char *exitname, int ttl); +const char *addressmap_register_virtual_address(int type, char *new_address); +void addressmap_get_mappings(smartlist_t *sl, time_t min_expires, + time_t max_expires, int want_expiry); +int connection_ap_rewrite_and_attach_if_allowed(edge_connection_t *conn, + origin_circuit_t *circ, + crypt_path_t *cpath); +int connection_ap_handshake_rewrite_and_attach(edge_connection_t *conn, + origin_circuit_t *circ, + crypt_path_t *cpath); + +/** Possible return values for parse_extended_hostname. */ +typedef enum hostname_type_t { + NORMAL_HOSTNAME, ONION_HOSTNAME, EXIT_HOSTNAME, BAD_HOSTNAME +} hostname_type_t; +hostname_type_t parse_extended_hostname(char *address, int allowdotexit); + +#if defined(HAVE_NET_IF_H) && defined(HAVE_NET_PFVAR_H) +int get_pf_socket(void); +#endif + +#endif + diff --git a/src/or/connection_or.c b/src/or/connection_or.c index d402563fb5..4ce08af152 100644 --- a/src/or/connection_or.c +++ b/src/or/connection_or.c @@ -11,6 +11,22 @@ **/ #include "or.h" +#include "buffers.h" +#include "circuitbuild.h" +#include "command.h" +#include "config.h" +#include "connection.h" +#include "connection_or.h" +#include "control.h" +#include "dirserv.h" +#include "geoip.h" +#include "main.h" +#include "networkstatus.h" +#include "reasons.h" +#include "relay.h" +#include "rephist.h" +#include "router.h" +#include "routerlist.h" static int connection_tls_finish_handshake(or_connection_t *conn); static int connection_or_process_cells_from_inbuf(or_connection_t *conn); @@ -80,10 +96,8 @@ connection_or_clear_identity_map(void) } }); - if (orconn_identity_map) { - digestmap_free(orconn_identity_map, NULL); - orconn_identity_map = NULL; - } + digestmap_free(orconn_identity_map, NULL); + orconn_identity_map = NULL; } /** Change conn->identity_digest to digest, and add conn into @@ -133,7 +147,7 @@ void cell_pack(packed_cell_t *dst, const cell_t *src) { char *dest = dst->body; - *(uint16_t*)dest = htons(src->circ_id); + set_uint16(dest, htons(src->circ_id)); *(uint8_t*)(dest+2) = src->command; memcpy(dest+3, src->payload, CELL_PAYLOAD_SIZE); } @@ -144,7 +158,7 @@ cell_pack(packed_cell_t *dst, const cell_t *src) static void cell_unpack(cell_t *dest, const char *src) { - dest->circ_id = ntohs(*(uint16_t*)(src)); + dest->circ_id = ntohs(get_uint16(src)); dest->command = *(uint8_t*)(src+2); memcpy(dest->payload, src+3, CELL_PAYLOAD_SIZE); } @@ -187,66 +201,6 @@ connection_or_reached_eof(or_connection_t *conn) return 0; } -/** Read conn's inbuf. If the http response from the proxy is all - * here, make sure it's good news, and begin the tls handshake. If - * it's bad news, close the connection and return -1. Else return 0 - * and hope for better luck next time. - */ -static int -connection_or_read_proxy_response(or_connection_t *or_conn) -{ - char *headers; - char *reason=NULL; - int status_code; - time_t date_header; - connection_t *conn = TO_CONN(or_conn); - - switch (fetch_from_buf_http(conn->inbuf, - &headers, MAX_HEADERS_SIZE, - NULL, NULL, 10000, 0)) { - case -1: /* overflow */ - log_warn(LD_PROTOCOL, - "Your https proxy sent back an oversized response. Closing."); - return -1; - case 0: - log_info(LD_OR,"https proxy response not all here yet. Waiting."); - return 0; - /* case 1, fall through */ - } - - if (parse_http_response(headers, &status_code, &date_header, - NULL, &reason) < 0) { - log_warn(LD_OR, - "Unparseable headers from proxy (connecting to '%s'). Closing.", - conn->address); - tor_free(headers); - return -1; - } - if (!reason) reason = tor_strdup("[no reason given]"); - - if (status_code == 200) { - log_info(LD_OR, - "HTTPS connect to '%s' successful! (200 %s) Starting TLS.", - conn->address, escaped(reason)); - tor_free(reason); - if (connection_tls_start_handshake(or_conn, 0) < 0) { - /* TLS handshaking error of some kind. */ - connection_mark_for_close(conn); - - return -1; - } - return 0; - } - /* else, bad news on the status code */ - log_warn(LD_OR, - "The https proxy sent back an unexpected status code %d (%s). " - "Closing.", - status_code, escaped(reason)); - tor_free(reason); - connection_mark_for_close(conn); - return -1; -} - /** Handle any new bytes that have come in on connection <b>conn</b>. * If conn is in 'open' state, hand it to * connection_or_process_cells_from_inbuf() @@ -255,11 +209,24 @@ connection_or_read_proxy_response(or_connection_t *or_conn) int connection_or_process_inbuf(or_connection_t *conn) { + int ret; tor_assert(conn); switch (conn->_base.state) { - case OR_CONN_STATE_PROXY_READING: - return connection_or_read_proxy_response(conn); + case OR_CONN_STATE_PROXY_HANDSHAKING: + ret = connection_read_proxy_handshake(TO_CONN(conn)); + + /* start TLS after handshake completion, or deal with error */ + if (ret == 1) { + tor_assert(TO_CONN(conn)->proxy_state == PROXY_CONNECTED); + if (connection_tls_start_handshake(conn, 0) < 0) + ret = -1; + } + if (ret < 0) { + connection_mark_for_close(TO_CONN(conn)); + } + + return ret; case OR_CONN_STATE_OPEN: case OR_CONN_STATE_OR_HANDSHAKING: return connection_or_process_cells_from_inbuf(conn); @@ -285,8 +252,7 @@ connection_or_flushed_some(or_connection_t *conn) /* If we're under the low water mark, add cells until we're just over the * high water mark. */ if (datalen < OR_CONN_LOWWATER) { - ssize_t n = (OR_CONN_HIGHWATER - datalen + CELL_NETWORK_SIZE-1) - / CELL_NETWORK_SIZE; + ssize_t n = CEIL_DIV(OR_CONN_HIGHWATER - datalen, CELL_NETWORK_SIZE); time_t now = approx_time(); while (conn->active_circuits && n > 0) { int flushed; @@ -312,11 +278,7 @@ connection_or_finished_flushing(or_connection_t *conn) assert_connection_ok(TO_CONN(conn),0); switch (conn->_base.state) { - case OR_CONN_STATE_PROXY_FLUSHING: - log_debug(LD_OR,"finished sending CONNECT to proxy."); - conn->_base.state = OR_CONN_STATE_PROXY_READING; - connection_stop_writing(TO_CONN(conn)); - break; + case OR_CONN_STATE_PROXY_HANDSHAKING: case OR_CONN_STATE_OPEN: case OR_CONN_STATE_OR_HANDSHAKING: connection_stop_writing(TO_CONN(conn)); @@ -334,37 +296,34 @@ connection_or_finished_flushing(or_connection_t *conn) int connection_or_finished_connecting(or_connection_t *or_conn) { + int proxy_type; connection_t *conn; tor_assert(or_conn); conn = TO_CONN(or_conn); tor_assert(conn->state == OR_CONN_STATE_CONNECTING); - log_debug(LD_OR,"OR connect() to router at %s:%u finished.", + log_debug(LD_HANDSHAKE,"OR connect() to router at %s:%u finished.", conn->address,conn->port); control_event_bootstrap(BOOTSTRAP_STATUS_HANDSHAKE, 0); - if (get_options()->HttpsProxy) { - char buf[1024]; - char *base64_authenticator=NULL; - const char *authenticator = get_options()->HttpsProxyAuthenticator; + proxy_type = PROXY_NONE; - if (authenticator) { - base64_authenticator = alloc_http_authenticator(authenticator); - if (!base64_authenticator) - log_warn(LD_OR, "Encoding https authenticator failed"); - } - if (base64_authenticator) { - tor_snprintf(buf, sizeof(buf), "CONNECT %s:%d HTTP/1.1\r\n" - "Proxy-Authorization: Basic %s\r\n\r\n", - fmt_addr(&conn->addr), - conn->port, base64_authenticator); - tor_free(base64_authenticator); - } else { - tor_snprintf(buf, sizeof(buf), "CONNECT %s:%d HTTP/1.0\r\n\r\n", - fmt_addr(&conn->addr), conn->port); + if (get_options()->HTTPSProxy) + proxy_type = PROXY_CONNECT; + else if (get_options()->Socks4Proxy) + proxy_type = PROXY_SOCKS4; + else if (get_options()->Socks5Proxy) + proxy_type = PROXY_SOCKS5; + + if (proxy_type != PROXY_NONE) { + /* start proxy handshake */ + if (connection_proxy_connect(conn, proxy_type) < 0) { + connection_mark_for_close(conn); + return -1; } - connection_write_to_buf(buf, strlen(buf), conn); - conn->state = OR_CONN_STATE_PROXY_FLUSHING; + + connection_start_reading(conn); + conn->state = OR_CONN_STATE_PROXY_HANDSHAKING; return 0; } @@ -376,6 +335,78 @@ connection_or_finished_connecting(or_connection_t *or_conn) return 0; } +/** Return 1 if identity digest <b>id_digest</b> is known to be a + * currently or recently running relay. Otherwise return 0. */ +int +connection_or_digest_is_known_relay(const char *id_digest) +{ + if (router_get_consensus_status_by_id(id_digest)) + return 1; /* It's in the consensus: "yes" */ + if (router_get_by_digest(id_digest)) + return 1; /* Not in the consensus, but we have a descriptor for + * it. Probably it was in a recent consensus. "Yes". */ + return 0; +} + +/** Set the per-conn read and write limits for <b>conn</b>. If it's a known + * relay, we will rely on the global read and write buckets, so give it + * per-conn limits that are big enough they'll never matter. But if it's + * not a known relay, first check if we set PerConnBwRate/Burst, then + * check if the consensus sets them, else default to 'big enough'. + * + * If <b>reset</b> is true, set the bucket to be full. Otherwise, just + * clip the bucket if it happens to be <em>too</em> full. + */ +static void +connection_or_update_token_buckets_helper(or_connection_t *conn, int reset, + or_options_t *options) +{ + int rate, burst; /* per-connection rate limiting params */ + if (connection_or_digest_is_known_relay(conn->identity_digest)) { + /* It's in the consensus, or we have a descriptor for it meaning it + * was probably in a recent consensus. It's a recognized relay: + * give it full bandwidth. */ + rate = (int)options->BandwidthRate; + burst = (int)options->BandwidthBurst; + } else { + /* Not a recognized relay. Squeeze it down based on the suggested + * bandwidth parameters in the consensus, but allow local config + * options to override. */ + rate = options->PerConnBWRate ? (int)options->PerConnBWRate : + networkstatus_get_param(NULL, "perconnbwrate", + (int)options->BandwidthRate, 1, INT32_MAX); + burst = options->PerConnBWBurst ? (int)options->PerConnBWBurst : + networkstatus_get_param(NULL, "perconnbwburst", + (int)options->BandwidthBurst, 1, INT32_MAX); + } + + conn->bandwidthrate = rate; + conn->bandwidthburst = burst; + if (reset) { /* set up the token buckets to be full */ + conn->read_bucket = conn->write_bucket = burst; + return; + } + /* If the new token bucket is smaller, take out the extra tokens. + * (If it's larger, don't -- the buckets can grow to reach the cap.) */ + if (conn->read_bucket > burst) + conn->read_bucket = burst; + if (conn->write_bucket > burst) + conn->write_bucket = burst; +} + +/** Either our set of relays or our per-conn rate limits have changed. + * Go through all the OR connections and update their token buckets to make + * sure they don't exceed their maximum values. */ +void +connection_or_update_token_buckets(smartlist_t *conns, or_options_t *options) +{ + SMARTLIST_FOREACH(conns, connection_t *, conn, + { + if (connection_speaks_cells(conn)) + connection_or_update_token_buckets_helper(TO_OR_CONN(conn), 0, options); + }); +} + /** If we don't necessarily know the router we're connecting to, but we * have an addr/port/id_digest, then fill in as much as we can. Start * by checking to see if this describes a router we know. */ @@ -385,11 +416,9 @@ connection_or_init_conn_from_address(or_connection_t *conn, const char *id_digest, int started_here) { - or_options_t *options = get_options(); routerinfo_t *r = router_get_by_digest(id_digest); - conn->bandwidthrate = (int)options->BandwidthRate; - conn->read_bucket = conn->bandwidthburst = (int)options->BandwidthBurst; connection_or_set_identity_digest(conn, id_digest); + connection_or_update_token_buckets_helper(conn, 1, get_options()); conn->_base.port = port; tor_addr_copy(&conn->_base.addr, addr); @@ -442,7 +471,7 @@ connection_or_init_conn_from_address(or_connection_t *conn, * Requires that both input connections are open; not is_bad_for_new_circs, * and not impossibly non-canonical. * - * If </b>forgive_new_connections</b> is true, then we do not call + * If <b>forgive_new_connections</b> is true, then we do not call * <b>a</b>better than <b>b</b> simply because b has no circuits, * unless b is also relatively old. */ @@ -581,11 +610,24 @@ connection_or_get_for_extend(const char *digest, #define TIME_BEFORE_OR_CONN_IS_TOO_OLD (60*60*24*7) /** Given the head of the linked list for all the or_connections with a given - * identity, set elements of that list as is_bad_for_new_circs() as - * appropriate. Helper for connection_or_set_bad_connections(). + * identity, set elements of that list as is_bad_for_new_circs as + * appropriate. Helper for connection_or_set_bad_connections(). + * + * Specifically, we set the is_bad_for_new_circs flag on: + * - all connections if <b>force</b> is true. + * - all connections that are too old. + * - all open non-canonical connections for which a canonical connection + * exists to the same router. + * - all open canonical connections for which a 'better' canonical + * connection exists to the same router. + * - all open non-canonical connections for which a 'better' non-canonical + * connection exists to the same router at the same address. + * + * See connection_or_is_better() for our idea of what makes one OR connection + * better than another. */ static void -connection_or_group_set_badness(or_connection_t *head) +connection_or_group_set_badness(or_connection_t *head, int force) { or_connection_t *or_conn = NULL, *best = NULL; int n_old = 0, n_inprogress = 0, n_canonical = 0, n_other = 0; @@ -597,8 +639,9 @@ connection_or_group_set_badness(or_connection_t *head) if (or_conn->_base.marked_for_close || or_conn->is_bad_for_new_circs) continue; - if (or_conn->_base.timestamp_created + TIME_BEFORE_OR_CONN_IS_TOO_OLD - < now) { + if (force || + or_conn->_base.timestamp_created + TIME_BEFORE_OR_CONN_IS_TOO_OLD + < now) { log_info(LD_OR, "Marking OR conn to %s:%d as too old for new circuits " "(fd %d, %d secs old).", @@ -631,7 +674,7 @@ connection_or_group_set_badness(or_connection_t *head) /* We have at least one open canonical connection to this router, * and this one is open but not canonical. Mark it bad. */ log_info(LD_OR, - "Marking OR conn to %s:%d as too old for new circuits: " + "Marking OR conn to %s:%d as unsuitable for new circuits: " "(fd %d, %d secs old). It is not canonical, and we have " "another connection to that OR that is.", or_conn->_base.address, or_conn->_base.port, or_conn->_base.s, @@ -671,7 +714,7 @@ connection_or_group_set_badness(or_connection_t *head) even when we're being forgiving. */ if (best->is_canonical) { log_info(LD_OR, - "Marking OR conn to %s:%d as too old for new circuits: " + "Marking OR conn to %s:%d as unsuitable for new circuits: " "(fd %d, %d secs old). We have a better canonical one " "(fd %d; %d secs old).", or_conn->_base.address, or_conn->_base.port, or_conn->_base.s, @@ -681,9 +724,9 @@ connection_or_group_set_badness(or_connection_t *head) } else if (!tor_addr_compare(&or_conn->real_addr, &best->real_addr, CMP_EXACT)) { log_info(LD_OR, - "Marking OR conn to %s:%d as too old for new circuits: " - "(fd %d, %d secs old). We have a better one " - "(fd %d; %d secs old).", + "Marking OR conn to %s:%d as unsuitable for new circuits: " + "(fd %d, %d secs old). We have a better one with the " + "same address (fd %d; %d secs old).", or_conn->_base.address, or_conn->_base.port, or_conn->_base.s, (int)(now - or_conn->_base.timestamp_created), best->_base.s, (int)(now - best->_base.timestamp_created)); @@ -693,27 +736,20 @@ connection_or_group_set_badness(or_connection_t *head) } } -/** Go through all the OR connections, and set the is_bad_for_new_circs - * flag on: - * - all connections that are too old. - * - all open non-canonical connections for which a canonical connection - * exists to the same router. - * - all open canonical connections for which a 'better' canonical - * connection exists to the same router. - * - all open non-canonical connections for which a 'better' non-canonical - * connection exists to the same router at the same address. - * - * See connection_or_is_better() for our idea of what makes one OR connection - * better than another. +/** Go through all the OR connections (or if <b>digest</b> is non-NULL, just + * the OR connections with that digest), and set the is_bad_for_new_circs + * flag based on the rules in connection_or_group_set_badness() (or just + * always set it if <b>force</b> is true). */ void -connection_or_set_bad_connections(void) +connection_or_set_bad_connections(const char *digest, int force) { if (!orconn_identity_map) return; DIGESTMAP_FOREACH(orconn_identity_map, identity, or_connection_t *, conn) { - connection_or_group_set_badness(conn); + if (!digest || !memcmp(digest, conn->identity_digest, DIGEST_LEN)) + connection_or_group_set_badness(conn, force); } DIGESTMAP_FOREACH_END; } @@ -753,6 +789,7 @@ connection_or_connect(const tor_addr_t *_addr, uint16_t port, or_connection_t *conn; or_options_t *options = get_options(); int socket_error = 0; + int using_proxy = 0; tor_addr_t addr; tor_assert(_addr); @@ -771,19 +808,27 @@ connection_or_connect(const tor_addr_t *_addr, uint16_t port, conn->_base.state = OR_CONN_STATE_CONNECTING; control_event_or_conn_status(conn, OR_CONN_EVENT_LAUNCHED, 0); - if (options->HttpsProxy) { - /* we shouldn't connect directly. use the https proxy instead. */ - tor_addr_from_ipv4h(&addr, options->HttpsProxyAddr); - port = options->HttpsProxyPort; + /* use a proxy server if available */ + if (options->HTTPSProxy) { + using_proxy = 1; + tor_addr_copy(&addr, &options->HTTPSProxyAddr); + port = options->HTTPSProxyPort; + } else if (options->Socks4Proxy) { + using_proxy = 1; + tor_addr_copy(&addr, &options->Socks4ProxyAddr); + port = options->Socks4ProxyPort; + } else if (options->Socks5Proxy) { + using_proxy = 1; + tor_addr_copy(&addr, &options->Socks5ProxyAddr); + port = options->Socks5ProxyPort; } switch (connection_connect(TO_CONN(conn), conn->_base.address, &addr, port, &socket_error)) { case -1: /* If the connection failed immediately, and we're using - * an https proxy, our https proxy is down. Don't blame the - * Tor server. */ - if (!options->HttpsProxy) + * a proxy, our proxy is down. Don't blame the Tor server. */ + if (!using_proxy) entry_guard_register_connect_status(conn->identity_digest, 0, 1, time(NULL)); connection_or_connect_failed(conn, @@ -792,7 +837,7 @@ connection_or_connect(const tor_addr_t *_addr, uint16_t port, connection_free(TO_CONN(conn)); return NULL; case 0: - connection_watch_events(TO_CONN(conn), EV_READ | EV_WRITE); + connection_watch_events(TO_CONN(conn), READ_EVENT | WRITE_EVENT); /* writable indicates finish, readable indicates broken link, error indicates broken link on windows */ return conn; @@ -819,13 +864,14 @@ connection_tls_start_handshake(or_connection_t *conn, int receiving) { conn->_base.state = OR_CONN_STATE_TLS_HANDSHAKING; conn->tls = tor_tls_new(conn->_base.s, receiving); - tor_tls_set_logged_address(conn->tls, escaped_safe_str(conn->_base.address)); + tor_tls_set_logged_address(conn->tls, // XXX client and relay? + escaped_safe_str(conn->_base.address)); if (!conn->tls) { log_warn(LD_BUG,"tor_tls_new failed. Closing."); return -1; } connection_start_reading(TO_CONN(conn)); - log_debug(LD_OR,"starting TLS handshake on fd %d", conn->_base.s); + log_debug(LD_HANDSHAKE,"starting TLS handshake on fd %d", conn->_base.s); note_crypto_pk_op(receiving ? TLS_HANDSHAKE_S : TLS_HANDSHAKE_C); if (connection_tls_continue_handshake(conn) < 0) { @@ -932,23 +978,26 @@ connection_or_nonopen_was_started_here(or_connection_t *conn) * return -1 if he is lying, broken, or otherwise something is wrong. * * If we initiated this connection (<b>started_here</b> is true), make sure - * the other side sent sent a correctly formed certificate. If I initiated the + * the other side sent a correctly formed certificate. If I initiated the * connection, make sure it's the right guy. * * Otherwise (if we _didn't_ initiate this connection), it's okay for * the certificate to be weird or absent. * * If we return 0, and the certificate is as expected, write a hash of the - * identity key into digest_rcvd, which must have DIGEST_LEN space in it. (If - * we return -1 this buffer is undefined.) If the certificate is invalid - * or missing on an incoming connection, we return 0 and set digest_rcvd to - * DIGEST_LEN 0 bytes. + * identity key into <b>digest_rcvd_out</b>, which must have DIGEST_LEN + * space in it. + * If the certificate is invalid or missing on an incoming connection, + * we return 0 and set <b>digest_rcvd_out</b> to DIGEST_LEN NUL bytes. + * (If we return -1, the contents of this buffer are undefined.) * * As side effects, * 1) Set conn->circ_id_type according to tor-spec.txt. * 2) If we're an authdirserver and we initiated the connection: drop all * descriptors that claim to be on that IP/port but that aren't * this guy; and note that this guy is reachable. + * 3) If this is a bridge and we didn't configure its identity + * fingerprint, remember the keyid we just learned. */ static int connection_or_check_valid_tls_handshake(or_connection_t *conn, @@ -959,19 +1008,23 @@ connection_or_check_valid_tls_handshake(or_connection_t *conn, or_options_t *options = get_options(); int severity = server_mode(options) ? LOG_PROTOCOL_WARN : LOG_WARN; const char *safe_address = - started_here ? conn->_base.address : safe_str(conn->_base.address); + started_here ? conn->_base.address : + safe_str_client(conn->_base.address); const char *conn_type = started_here ? "outgoing" : "incoming"; + crypto_pk_env_t *our_identity = + started_here ? get_tlsclient_identity_key() : + get_server_identity_key(); int has_cert = 0, has_identity=0; check_no_tls_errors(); has_cert = tor_tls_peer_has_cert(conn->tls); if (started_here && !has_cert) { - log_info(LD_PROTOCOL,"Tried connecting to router at %s:%d, but it didn't " + log_info(LD_HANDSHAKE,"Tried connecting to router at %s:%d, but it didn't " "send a cert! Closing.", safe_address, conn->_base.port); return -1; } else if (!has_cert) { - log_debug(LD_PROTOCOL,"Got incoming connection with no certificate. " + log_debug(LD_HANDSHAKE,"Got incoming connection with no certificate. " "That's ok."); } check_no_tls_errors(); @@ -980,15 +1033,16 @@ connection_or_check_valid_tls_handshake(or_connection_t *conn, int v = tor_tls_verify(started_here?severity:LOG_INFO, conn->tls, &identity_rcvd); if (started_here && v<0) { - log_fn(severity,LD_OR,"Tried connecting to router at %s:%d: It" + log_fn(severity,LD_HANDSHAKE,"Tried connecting to router at %s:%d: It" " has a cert but it's invalid. Closing.", safe_address, conn->_base.port); return -1; } else if (v<0) { - log_info(LD_PROTOCOL,"Incoming connection gave us an invalid cert " + log_info(LD_HANDSHAKE,"Incoming connection gave us an invalid cert " "chain; ignoring."); } else { - log_debug(LD_OR,"The certificate seems to be valid on %s connection " + log_debug(LD_HANDSHAKE, + "The certificate seems to be valid on %s connection " "with %s:%d", conn_type, safe_address, conn->_base.port); } check_no_tls_errors(); @@ -997,7 +1051,7 @@ connection_or_check_valid_tls_handshake(or_connection_t *conn, if (identity_rcvd) { has_identity = 1; crypto_pk_get_digest(identity_rcvd, digest_rcvd_out); - if (crypto_pk_cmp_keys(get_identity_key(), identity_rcvd)<0) { + if (crypto_pk_cmp_keys(our_identity, identity_rcvd)<0) { conn->circ_id_type = CIRC_ID_TYPE_LOWER; } else { conn->circ_id_type = CIRC_ID_TYPE_HIGHER; @@ -1015,9 +1069,13 @@ connection_or_check_valid_tls_handshake(or_connection_t *conn, conn->nickname[0] = '$'; base16_encode(conn->nickname+1, HEX_DIGEST_LEN+1, conn->identity_digest, DIGEST_LEN); - log_info(LD_OR, "Connected to router %s at %s:%d without knowing " + log_info(LD_HANDSHAKE, "Connected to router %s at %s:%d without knowing " "its key. Hoping for the best.", conn->nickname, conn->_base.address, conn->_base.port); + /* if it's a bridge and we didn't know its identity fingerprint, now + * we do -- remember it for future attempts. */ + learned_router_identity(&conn->_base.addr, conn->_base.port, + digest_rcvd_out); } if (started_here) { @@ -1031,7 +1089,7 @@ connection_or_check_valid_tls_handshake(or_connection_t *conn, base16_encode(seen, sizeof(seen), digest_rcvd_out, DIGEST_LEN); base16_encode(expected, sizeof(expected), conn->identity_digest, DIGEST_LEN); - log_fn(severity, LD_OR, + log_fn(severity, LD_HANDSHAKE, "Tried connecting to router at %s:%d, but identity key was not " "as expected: wanted %s but got %s.", conn->_base.address, conn->_base.port, expected, seen); @@ -1044,9 +1102,6 @@ connection_or_check_valid_tls_handshake(or_connection_t *conn, as_advertised = 0; } if (authdir_mode_tests_reachability(options)) { - /* We initiated this connection to address:port. Drop all routers - * with the same address:port and a different key. - */ dirserv_orconn_tls_done(conn->_base.address, conn->_base.port, digest_rcvd_out, as_advertised); } @@ -1073,8 +1128,8 @@ connection_tls_finish_handshake(or_connection_t *conn) char digest_rcvd[DIGEST_LEN]; int started_here = connection_or_nonopen_was_started_here(conn); - log_debug(LD_OR,"tls handshake with %s done. verifying.", - safe_str(conn->_base.address)); + log_debug(LD_HANDSHAKE,"tls handshake with %s done. verifying.", + safe_str_client(conn->_base.address)); directory_set_dirty(); @@ -1082,6 +1137,8 @@ connection_tls_finish_handshake(or_connection_t *conn) digest_rcvd) < 0) return -1; + circuit_build_times_network_is_live(&circ_times); + if (tor_tls_used_v1_handshake(conn->tls)) { conn->link_proto = 1; if (!started_here) { @@ -1117,7 +1174,8 @@ connection_init_or_handshake_state(or_connection_t *conn, int started_here) void or_handshake_state_free(or_handshake_state_t *state) { - tor_assert(state); + if (!state) + return; memset(state, 0xBE, sizeof(or_handshake_state_t)); tor_free(state); } @@ -1134,6 +1192,7 @@ connection_or_set_state_open(or_connection_t *conn) control_event_or_conn_status(conn, OR_CONN_EVENT_CONNECTED, 0); if (started_here) { + circuit_build_times_network_is_live(&circ_times); rep_hist_note_connect_succeeded(conn->identity_digest, now); if (entry_guard_register_connect_status(conn->identity_digest, 1, 0, now) < 0) { @@ -1158,10 +1217,10 @@ connection_or_set_state_open(or_connection_t *conn) } } } - if (conn->handshake_state) { - or_handshake_state_free(conn->handshake_state); - conn->handshake_state = NULL; - } + + or_handshake_state_free(conn->handshake_state); + conn->handshake_state = NULL; + connection_start_reading(TO_CONN(conn)); circuit_n_conn_done(conn, 1); /* send the pending creates, if any. */ @@ -1235,6 +1294,7 @@ connection_or_process_cells_from_inbuf(or_connection_t *conn) if (connection_fetch_var_cell_from_buf(conn, &var_cell)) { if (!var_cell) return 0; /* not yet. */ + circuit_build_times_network_is_live(&circ_times); command_process_var_cell(var_cell, conn); var_cell_free(var_cell); } else { @@ -1244,6 +1304,7 @@ connection_or_process_cells_from_inbuf(or_connection_t *conn) available? */ return 0; /* not yet */ + circuit_build_times_network_is_live(&circ_times); connection_fetch_from_buf(buf, CELL_NETWORK_SIZE, TO_CONN(conn)); /* retrieve cell info from buf (create the host-order struct from the @@ -1274,10 +1335,6 @@ connection_or_send_destroy(circid_t circ_id, or_connection_t *conn, int reason) cell.payload[0] = (uint8_t) reason; log_debug(LD_OR,"Sending destroy (circID %d).", circ_id); - /* XXXX It's possible that under some circumstances, we want the destroy - * to take precedence over other data waiting on the circuit's cell queue. - */ - connection_or_write_cell_to_buf(&cell, conn); return 0; } @@ -1357,9 +1414,8 @@ connection_or_send_netinfo(or_connection_t *conn) len = append_address_to_payload(out, &my_addr); if (len < 0) return -1; - out += len; } else { - *out++ = 0; + *out = 0; } connection_or_write_cell_to_buf(&cell, conn); diff --git a/src/or/connection_or.h b/src/or/connection_or.h new file mode 100644 index 0000000000..70ef96a335 --- /dev/null +++ b/src/or/connection_or.h @@ -0,0 +1,57 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file connection_or.h + * \brief Header file for connection_or.c. + **/ + +#ifndef _TOR_CONNECTION_OR_H +#define _TOR_CONNECTION_OR_H + +void connection_or_remove_from_identity_map(or_connection_t *conn); +void connection_or_clear_identity_map(void); +or_connection_t *connection_or_get_for_extend(const char *digest, + const tor_addr_t *target_addr, + const char **msg_out, + int *launch_out); +void connection_or_set_bad_connections(const char *digest, int force); + +int connection_or_reached_eof(or_connection_t *conn); +int connection_or_process_inbuf(or_connection_t *conn); +int connection_or_flushed_some(or_connection_t *conn); +int connection_or_finished_flushing(or_connection_t *conn); +int connection_or_finished_connecting(or_connection_t *conn); +int connection_or_digest_is_known_relay(const char *id_digest); +void connection_or_update_token_buckets(smartlist_t *conns, + or_options_t *options); + +void connection_or_connect_failed(or_connection_t *conn, + int reason, const char *msg); +or_connection_t *connection_or_connect(const tor_addr_t *addr, uint16_t port, + const char *id_digest); + +int connection_tls_start_handshake(or_connection_t *conn, int receiving); +int connection_tls_continue_handshake(or_connection_t *conn); + +void or_handshake_state_free(or_handshake_state_t *state); +int connection_or_set_state_open(or_connection_t *conn); +void connection_or_write_cell_to_buf(const cell_t *cell, + or_connection_t *conn); +void connection_or_write_var_cell_to_buf(const var_cell_t *cell, + or_connection_t *conn); +int connection_or_send_destroy(circid_t circ_id, or_connection_t *conn, + int reason); +int connection_or_send_netinfo(or_connection_t *conn); +int is_or_protocol_version_known(uint16_t version); + +void cell_pack(packed_cell_t *dest, const cell_t *src); +void var_cell_pack_header(const var_cell_t *cell, char *hdr_out); +var_cell_t *var_cell_new(uint16_t payload_len); +void var_cell_free(var_cell_t *cell); + +#endif + diff --git a/src/or/control.c b/src/or/control.c index f91afafeee..c5362424ad 100644 --- a/src/or/control.c +++ b/src/or/control.c @@ -11,6 +11,26 @@ #define CONTROL_PRIVATE #include "or.h" +#include "buffers.h" +#include "circuitbuild.h" +#include "circuitlist.h" +#include "circuituse.h" +#include "config.h" +#include "connection.h" +#include "connection_edge.h" +#include "control.h" +#include "directory.h" +#include "dirserv.h" +#include "dnsserv.h" +#include "geoip.h" +#include "hibernate.h" +#include "main.h" +#include "networkstatus.h" +#include "policies.h" +#include "reasons.h" +#include "router.h" +#include "routerlist.h" +#include "routerparse.h" /** Yield true iff <b>s</b> is the state of a control_connection_t that has * finished authentication and is accepting commands. */ @@ -43,7 +63,8 @@ #define EVENT_STREAM_BANDWIDTH_USED 0x0014 #define EVENT_CLIENTS_SEEN 0x0015 #define EVENT_NEWCONSENSUS 0x0016 -#define _EVENT_MAX 0x0016 +#define EVENT_BUILDTIMEOUT_SET 0x0017 +#define _EVENT_MAX 0x0017 /* If _EVENT_MAX ever hits 0x0020, we need to make the mask wider. */ /** Bitfield: The bit 1<<e is set if <b>any</b> open control @@ -54,13 +75,9 @@ **/ typedef uint32_t event_mask_t; -/** An event mask of all the events that controller with the LONG_NAMES option - * set is interested in receiving. */ -static event_mask_t global_event_mask1long = 0; - -/** An event mask of all the events that controller with the SHORT_NAMES option - * set is interested in receiving. */ -static event_mask_t global_event_mask1short = 0; +/** An event mask of all the events that any controller is interested in + * receiving. */ +static event_mask_t global_event_mask = 0; /** True iff we have disabled log messages from being sent to the controller */ static int disable_log_messages = 0; @@ -68,13 +85,7 @@ static int disable_log_messages = 0; /** Macro: true if any control connection is interested in events of type * <b>e</b>. */ #define EVENT_IS_INTERESTING(e) \ - ((global_event_mask1long|global_event_mask1short) & (1<<(e))) -/** Macro: true if any control connection with the LONG_NAMES option is - * interested in events of type <b>e</b>. */ -#define EVENT_IS_INTERESTING1L(e) (global_event_mask1long & (1<<(e))) -/** Macro: true if any control connection with the SHORT_NAMES option is - * interested in events of type <b>e</b>. */ -#define EVENT_IS_INTERESTING1S(e) (global_event_mask1short & (1<<(e))) + (global_event_mask & (1<<(e))) /** If we're using cookie-type authentication, how long should our cookies be? */ @@ -95,25 +106,13 @@ static char authentication_cookie[AUTHENTICATION_COOKIE_LEN]; * of this so we can respond to getinfo status/bootstrap-phase queries. */ static char last_sent_bootstrap_message[BOOTSTRAP_MSG_LEN]; -/** Flag for event_format_t. Indicates that we should use the old - * name format of nickname|hexdigest - */ -#define SHORT_NAMES 1 -/** Flag for event_format_t. Indicates that we should use the new - * name format of $hexdigest[=~]nickname +/** Flag for event_format_t. Indicates that we should use the one standard + format. */ -#define LONG_NAMES 2 -#define ALL_NAMES (SHORT_NAMES|LONG_NAMES) -/** Flag for event_format_t. Indicates that we should use the new event - * format where extra event fields are allowed using a NAME=VAL format. */ -#define EXTENDED_FORMAT 4 -/** Flag for event_format_t. Indicates that we are using the old event format - * where extra fields aren't allowed. */ -#define NONEXTENDED_FORMAT 8 -#define ALL_FORMATS (EXTENDED_FORMAT|NONEXTENDED_FORMAT) +#define ALL_FORMATS 1 /** Bit field of flags to select how to format a controller event. Recognized - * flags are SHORT_NAMES, LONG_NAMES, EXTENDED_FORMAT, NONEXTENDED_FORMAT. */ + * flag is ALL_FORMATS. */ typedef int event_format_t; static void connection_printf_to_buf(control_connection_t *conn, @@ -123,9 +122,6 @@ static void send_control_done(control_connection_t *conn); static void send_control_event(uint16_t event, event_format_t which, const char *format, ...) CHECK_PRINTF(3,4); -static void send_control_event_extended(uint16_t event, event_format_t which, - const char *format, ...) - CHECK_PRINTF(3,4); static int handle_control_setconf(control_connection_t *conn, uint32_t len, char *body); static int handle_control_resetconf(control_connection_t *conn, uint32_t len, @@ -139,8 +135,6 @@ static int handle_control_setevents(control_connection_t *conn, uint32_t len, static int handle_control_authenticate(control_connection_t *conn, uint32_t len, const char *body); -static int handle_control_saveconf(control_connection_t *conn, uint32_t len, - const char *body); static int handle_control_signal(control_connection_t *conn, uint32_t len, const char *body); static int handle_control_mapaddress(control_connection_t *conn, uint32_t len, @@ -174,7 +168,7 @@ static int handle_control_usefeature(control_connection_t *conn, const char *body); static int write_stream_target_to_buf(edge_connection_t *conn, char *buf, size_t len); -static void orconn_target_get_name(int long_names, char *buf, size_t len, +static void orconn_target_get_name(char *buf, size_t len, or_connection_t *conn); static char *get_cookie_file(void); @@ -214,25 +208,19 @@ control_update_global_event_mask(void) { smartlist_t *conns = get_connection_array(); event_mask_t old_mask, new_mask; - old_mask = global_event_mask1short; - old_mask |= global_event_mask1long; + old_mask = global_event_mask; - global_event_mask1short = 0; - global_event_mask1long = 0; + global_event_mask = 0; SMARTLIST_FOREACH(conns, connection_t *, _conn, { if (_conn->type == CONN_TYPE_CONTROL && STATE_IS_OPEN(_conn->state)) { control_connection_t *conn = TO_CONTROL_CONN(_conn); - if (conn->use_long_names) - global_event_mask1long |= conn->event_mask; - else - global_event_mask1short |= conn->event_mask; + global_event_mask |= conn->event_mask; } }); - new_mask = global_event_mask1short; - new_mask |= global_event_mask1long; + new_mask = global_event_mask; /* Handle the aftermath. Set up the log callback to tell us only what * we want to hear...*/ @@ -312,7 +300,7 @@ connection_write_str_to_buf(const char *s, control_connection_t *conn) /** Given a <b>len</b>-character string in <b>data</b>, made of lines * terminated by CRLF, allocate a new string in *<b>out</b>, and copy the * contents of <b>data</b> into *<b>out</b>, adding a period before any period - * that that appears at the start of a line, and adding a period-CRLF line at + * that appears at the start of a line, and adding a period-CRLF line at * the end. Replace all LF characters sequences with CRLF. Return the number * of bytes in *<b>out</b>. */ @@ -542,28 +530,15 @@ send_control_event_string(uint16_t event, event_format_t which, const char *msg) { smartlist_t *conns = get_connection_array(); + (void)which; tor_assert(event >= _EVENT_MIN && event <= _EVENT_MAX); - SMARTLIST_FOREACH(conns, connection_t *, conn, - { + SMARTLIST_FOREACH_BEGIN(conns, connection_t *, conn) { if (conn->type == CONN_TYPE_CONTROL && !conn->marked_for_close && conn->state == CONTROL_CONN_STATE_OPEN) { control_connection_t *control_conn = TO_CONTROL_CONN(conn); - if (control_conn->use_long_names) { - if (!(which & LONG_NAMES)) - continue; - } else { - if (!(which & SHORT_NAMES)) - continue; - } - if (control_conn->use_extended_events) { - if (!(which & EXTENDED_FORMAT)) - continue; - } else { - if (!(which & NONEXTENDED_FORMAT)) - continue; - } + if (control_conn->event_mask & (1<<event)) { int is_err = 0; connection_write_to_buf(msg, strlen(msg), TO_CONN(control_conn)); @@ -579,7 +554,7 @@ send_control_event_string(uint16_t event, event_format_t which, connection_handle_write(TO_CONN(control_conn), 1); } } - }); + } SMARTLIST_FOREACH_END(conn); } /** Helper for send_control1_event and send_control1_event_extended: @@ -587,22 +562,17 @@ send_control_event_string(uint16_t event, event_format_t which, * <b>event</b>. The event's body is created by the printf-style format in * <b>format</b>, and other arguments as provided. * - * If <b>extended</b> is true, and the format contains a single '@' character, - * it will be replaced with a space and all text after that character will be - * sent only to controllers that have enabled extended events. - * * Currently the length of the message is limited to 1024 (including the * ending \\r\\n\\0). */ static void -send_control_event_impl(uint16_t event, event_format_t which, int extended, - const char *format, va_list ap) +send_control_event_impl(uint16_t event, event_format_t which, + const char *format, va_list ap) { /* This is just a little longer than the longest allowed log message */ #define SEND_CONTROL1_EVENT_BUFFERSIZE 10064 int r; char buf[SEND_CONTROL1_EVENT_BUFFERSIZE]; size_t len; - char *cp; r = tor_vsnprintf(buf, sizeof(buf), format, ap); if (r<0) { @@ -618,15 +588,7 @@ send_control_event_impl(uint16_t event, event_format_t which, int extended, buf[SEND_CONTROL1_EVENT_BUFFERSIZE-3] = '\r'; } - if (extended && (cp = strchr(buf, '@'))) { - which &= ~ALL_FORMATS; - *cp = ' '; - send_control_event_string(event, which|EXTENDED_FORMAT, buf); - memcpy(cp, "\r\n\0", 3); - send_control_event_string(event, which|NONEXTENDED_FORMAT, buf); - } else { - send_control_event_string(event, which|ALL_FORMATS, buf); - } + send_control_event_string(event, which|ALL_FORMATS, buf); } /** Send an event to all v1 controllers that are listening for code @@ -641,27 +603,7 @@ send_control_event(uint16_t event, event_format_t which, { va_list ap; va_start(ap, format); - send_control_event_impl(event, which, 0, format, ap); - va_end(ap); -} - -/** Send an event to all v1 controllers that are listening for code - * <b>event</b>. The event's body is created by the printf-style format in - * <b>format</b>, and other arguments as provided. - * - * If the format contains a single '@' character, it will be replaced with a - * space and all text after that character will be sent only to controllers - * that have enabled extended events. - * - * Currently the length of the message is limited to 1024 (including the - * ending \\n\\r\\0. */ -static void -send_control_event_extended(uint16_t event, event_format_t which, - const char *format, ...) -{ - va_list ap; - va_start(ap, format); - send_control_event_impl(event, which, 1, format, ap); + send_control_event_impl(event, which, format, ap); va_end(ap); } @@ -907,36 +849,37 @@ handle_control_loadconf(control_connection_t *conn, uint32_t len, retval = options_init_from_string(body, CMD_RUN_TOR, NULL, &errstring); - if (retval != SETOPT_OK) { + if (retval != SETOPT_OK) log_warn(LD_CONTROL, "Controller gave us config file that didn't validate: %s", errstring); - switch (retval) { - case SETOPT_ERR_PARSE: - msg = "552 Invalid config file"; - break; - case SETOPT_ERR_TRANSITION: - msg = "553 Transition not allowed"; - break; - case SETOPT_ERR_SETTING: - msg = "553 Unable to set option"; - break; - case SETOPT_ERR_MISC: - default: - msg = "550 Unable to load config"; - break; - case SETOPT_OK: - tor_fragile_assert(); - break; - } + + switch (retval) { + case SETOPT_ERR_PARSE: + msg = "552 Invalid config file"; + break; + case SETOPT_ERR_TRANSITION: + msg = "553 Transition not allowed"; + break; + case SETOPT_ERR_SETTING: + msg = "553 Unable to set option"; + break; + case SETOPT_ERR_MISC: + default: + msg = "550 Unable to load config"; + break; + case SETOPT_OK: + break; + } + if (msg) { if (errstring) connection_printf_to_buf(conn, "%s: %s\r\n", msg, errstring); else connection_printf_to_buf(conn, "%s\r\n", msg); - tor_free(errstring); - return 0; + } else { + send_control_done(conn); } - send_control_done(conn); + tor_free(errstring); return 0; } @@ -948,7 +891,6 @@ handle_control_setevents(control_connection_t *conn, uint32_t len, { uint16_t event_code; uint32_t event_mask = 0; - unsigned int extended = 0; smartlist_t *events = smartlist_create(); (void) len; @@ -958,7 +900,6 @@ handle_control_setevents(control_connection_t *conn, uint32_t len, SMARTLIST_FOREACH_BEGIN(events, const char *, ev) { if (!strcasecmp(ev, "EXTENDED")) { - extended = 1; continue; } else if (!strcasecmp(ev, "CIRC")) event_code = EVENT_CIRCUIT_STATUS; @@ -1002,6 +943,8 @@ handle_control_setevents(control_connection_t *conn, uint32_t len, event_code = EVENT_CLIENTS_SEEN; else if (!strcasecmp(ev, "NEWCONSENSUS")) event_code = EVENT_NEWCONSENSUS; + else if (!strcasecmp(ev, "BUILDTIMEOUT_SET")) + event_code = EVENT_BUILDTIMEOUT_SET; else { connection_printf_to_buf(conn, "552 Unrecognized event \"%s\"\r\n", ev); @@ -1016,8 +959,6 @@ handle_control_setevents(control_connection_t *conn, uint32_t len, smartlist_free(events); conn->event_mask = event_mask; - if (extended) - conn->use_extended_events = 1; control_update_global_event_mask(); send_control_done(conn); @@ -1281,7 +1222,9 @@ handle_control_signal(control_connection_t *conn, uint32_t len, /* Flush the "done" first if the signal might make us shut down. */ if (sig == SIGTERM || sig == SIGINT) connection_handle_write(TO_CONN(conn), 1); - control_signal_act(sig); + + process_signal(sig); + return 0; } @@ -1328,7 +1271,7 @@ handle_control_mapaddress(control_connection_t *conn, uint32_t len, smartlist_add(reply, ans); log_warn(LD_CONTROL, "Unable to allocate address for '%s' in MapAddress msg", - safe_str(line)); + safe_str_client(line)); } else { tor_snprintf(ans, anslen, "250-%s=%s", address, to); smartlist_add(reply, ans); @@ -1345,7 +1288,8 @@ handle_control_mapaddress(control_connection_t *conn, uint32_t len, "not of expected form 'foo=bar'.", line); smartlist_add(reply, ans); log_info(LD_CONTROL, "Skipping MapAddress '%s': wrong " - "number of items.", safe_str(line)); + "number of items.", + safe_str_client(line)); } SMARTLIST_FOREACH(elts, char *, cp, tor_free(cp)); smartlist_clear(elts); @@ -1374,13 +1318,15 @@ handle_control_mapaddress(control_connection_t *conn, uint32_t len, * trivial-to-implement questions. */ static int getinfo_helper_misc(control_connection_t *conn, const char *question, - char **answer) + char **answer, const char **errmsg) { (void) conn; if (!strcmp(question, "version")) { *answer = tor_strdup(get_version()); } else if (!strcmp(question, "config-file")) { *answer = tor_strdup(get_torrc_fname()); + } else if (!strcmp(question, "config-text")) { + *answer = options_dump(get_options(), 1); } else if (!strcmp(question, "info/names")) { *answer = list_getinfo_options(); } else if (!strcmp(question, "events/names")) { @@ -1392,15 +1338,19 @@ getinfo_helper_misc(control_connection_t *conn, const char *question, *answer = tor_strdup("VERBOSE_NAMES EXTENDED_EVENTS"); } else if (!strcmp(question, "address")) { uint32_t addr; - if (router_pick_published_address(get_options(), &addr) < 0) + if (router_pick_published_address(get_options(), &addr) < 0) { + *errmsg = "Address unknown"; return -1; + } *answer = tor_dup_ip(addr); } else if (!strcmp(question, "dir-usage")) { *answer = directory_dump_request_log(); } else if (!strcmp(question, "fingerprint")) { routerinfo_t *me = router_get_my_routerinfo(); - if (!me) + if (!me) { + *errmsg = "No routerdesc known; am I really a server?"; return -1; + } *answer = tor_malloc(HEX_DIGEST_LEN+1); base16_encode(*answer, HEX_DIGEST_LEN+1, me->cache_info.identity_digest, DIGEST_LEN); @@ -1461,8 +1411,10 @@ munge_extrainfo_into_routerinfo(const char *ri_body, signed_descriptor_t *ri, * directory information. */ static int getinfo_helper_dir(control_connection_t *control_conn, - const char *question, char **answer) + const char *question, char **answer, + const char **errmsg) { + (void) control_conn; if (!strcmpstart(question, "desc/id/")) { routerinfo_t *ri = router_get_by_hexdigest(question+strlen("desc/id/")); if (ri) { @@ -1537,6 +1489,7 @@ getinfo_helper_dir(control_connection_t *control_conn, log_warn(LD_CONTROL, "getinfo '%s': %s", question, msg); smartlist_free(descs); tor_free(url); + *errmsg = msg; return -1; } SMARTLIST_FOREACH(descs, signed_descriptor_t *, sd, @@ -1587,7 +1540,7 @@ getinfo_helper_dir(control_connection_t *control_conn, } } else if (!strcmp(question, "dir/status-vote/current/consensus")) { /* v3 */ if (directory_caches_dir_info(get_options())) { - const cached_dir_t *consensus = dirserv_get_consensus(); + const cached_dir_t *consensus = dirserv_get_consensus("ns"); if (consensus) *answer = tor_strdup(consensus->dir); } @@ -1598,10 +1551,8 @@ getinfo_helper_dir(control_connection_t *control_conn, } } else if (!strcmp(question, "network-status")) { /* v1 */ routerlist_t *routerlist = router_get_routerlist(); - int verbose = control_conn->use_long_names; if (!routerlist || !routerlist->routers || - list_server_status_v1(routerlist->routers, answer, - verbose ? 2 : 1) < 0) { + list_server_status_v1(routerlist->routers, answer, 1) < 0) { return -1; } } else if (!strcmpstart(question, "extra-info/digest/")) { @@ -1635,8 +1586,10 @@ getinfo_helper_dir(control_connection_t *control_conn, * current states of things we send events about. */ static int getinfo_helper_events(control_connection_t *control_conn, - const char *question, char **answer) + const char *question, char **answer, + const char **errmsg) { + (void) control_conn; if (!strcmp(question, "circuit-status")) { circuit_t *circ; smartlist_t *status = smartlist_create(); @@ -1647,10 +1600,9 @@ getinfo_helper_events(control_connection_t *control_conn, const char *purpose; if (! CIRCUIT_IS_ORIGIN(circ) || circ->marked_for_close) continue; - if (control_conn->use_long_names) - path = circuit_list_path_for_controller(TO_ORIGIN_CIRCUIT(circ)); - else - path = circuit_list_path(TO_ORIGIN_CIRCUIT(circ),0); + + path = circuit_list_path_for_controller(TO_ORIGIN_CIRCUIT(circ)); + if (circ->state == CIRCUIT_STATE_OPEN) state = "BUILT"; else if (strlen(path)) @@ -1728,8 +1680,7 @@ getinfo_helper_events(control_connection_t *control_conn, } else if (!strcmp(question, "orconn-status")) { smartlist_t *conns = get_connection_array(); smartlist_t *status = smartlist_create(); - SMARTLIST_FOREACH(conns, connection_t *, base_conn, - { + SMARTLIST_FOREACH_BEGIN(conns, connection_t *, base_conn) { const char *state; char *s; char name[128]; @@ -1744,29 +1695,19 @@ getinfo_helper_events(control_connection_t *control_conn, state = "LAUNCHED"; else state = "NEW"; - orconn_target_get_name(control_conn->use_long_names, name, sizeof(name), - conn); + orconn_target_get_name(name, sizeof(name), conn); slen = strlen(name)+strlen(state)+2; s = tor_malloc(slen+1); tor_snprintf(s, slen, "%s %s", name, state); smartlist_add(status, s); - }); + } SMARTLIST_FOREACH_END(base_conn); *answer = smartlist_join_strings(status, "\r\n", 0, NULL); SMARTLIST_FOREACH(status, char *, cp, tor_free(cp)); smartlist_free(status); - } else if (!strcmpstart(question, "addr-mappings/") || - !strcmpstart(question, "address-mappings/")) { + } else if (!strcmpstart(question, "address-mappings/")) { time_t min_e, max_e; smartlist_t *mappings; - int want_expiry = !strcmpstart(question, "address-mappings/"); - if (!strcmpstart(question, "addr-mappings/")) { - /* XXXX022 This has been deprecated since 0.2.0.3-alpha, and has - generated a warning since 0.2.1.10-alpha; remove late in 0.2.2.x. */ - log_warn(LD_CONTROL, "Controller used obsolete addr-mappings/ GETINFO " - "key; use address-mappings/ instead."); - } - question += strlen(want_expiry ? "address-mappings/" - : "addr-mappings/"); + question += strlen("address-mappings/"); if (!strcmp(question, "all")) { min_e = 0; max_e = TIME_MAX; } else if (!strcmp(question, "cache")) { @@ -1779,7 +1720,7 @@ getinfo_helper_events(control_connection_t *control_conn, return 0; } mappings = smartlist_create(); - addressmap_get_mappings(mappings, min_e, max_e, want_expiry); + addressmap_get_mappings(mappings, min_e, max_e, 1); *answer = smartlist_join_strings(mappings, "\r\n", 0, NULL); SMARTLIST_FOREACH(mappings, char *, cp, tor_free(cp)); smartlist_free(mappings); @@ -1787,7 +1728,7 @@ getinfo_helper_events(control_connection_t *control_conn, /* Note that status/ is not a catch-all for events; there's only supposed * to be a status GETINFO if there's a corresponding STATUS event. */ if (!strcmp(question, "status/circuit-established")) { - *answer = tor_strdup(has_completed_circuit ? "1" : "0"); + *answer = tor_strdup(can_complete_circuit ? "1" : "0"); } else if (!strcmp(question, "status/enough-dir-info")) { *answer = tor_strdup(router_have_minimum_dir_info() ? "1" : "0"); } else if (!strcmp(question, "status/good-server-descriptor") || @@ -1846,21 +1787,12 @@ getinfo_helper_events(control_connection_t *control_conn, "information", question); } } else if (!strcmp(question, "status/clients-seen")) { - char geoip_start[ISO_TIME_LEN+1]; - size_t answer_len; - char *geoip_summary = extrainfo_get_client_geoip_summary(time(NULL)); - - if (!geoip_summary) + char *bridge_stats = geoip_get_bridge_stats_controller(time(NULL)); + if (!bridge_stats) { + *errmsg = "No bridge-client stats available"; return -1; - - answer_len = strlen("TimeStarted=\"\" CountrySummary=") + - ISO_TIME_LEN + strlen(geoip_summary) + 1; - *answer = tor_malloc(answer_len); - format_iso_time(geoip_start, geoip_get_history_start()); - tor_snprintf(*answer, answer_len, - "TimeStarted=\"%s\" CountrySummary=%s", - geoip_start, geoip_summary); - tor_free(geoip_summary); + } + *answer = bridge_stats; } else { return 0; } @@ -1870,11 +1802,14 @@ getinfo_helper_events(control_connection_t *control_conn, /** Callback function for GETINFO: on a given control connection, try to * answer the question <b>q</b> and store the newly-allocated answer in - * *<b>a</b>. If there's no answer, or an error occurs, just don't set - * <b>a</b>. Return 0. + * *<b>a</b>. If an internal error occurs, return -1 and optionally set + * *<b>error_out</b> to point to an error message to be delivered to the + * controller. On success, _or if the key is not recognized_, return 0. Do not + * set <b>a</b> if the key is not recognized. */ typedef int (*getinfo_helper_t)(control_connection_t *, - const char *q, char **a); + const char *q, char **a, + const char **error_out); /** A single item for the GETINFO question-to-answer-function table. */ typedef struct getinfo_item_t { @@ -1894,6 +1829,8 @@ typedef struct getinfo_item_t { static const getinfo_item_t getinfo_items[] = { ITEM("version", misc, "The current version of Tor."), ITEM("config-file", misc, "Current location of the \"torrc\" file."), + ITEM("config-text", misc, + "Return the string that would be written by a saveconf command."), ITEM("accounting/bytes", accounting, "Number of bytes read/written so far in the accounting interval."), ITEM("accounting/bytes-left", accounting, @@ -1933,7 +1870,6 @@ static const getinfo_item_t getinfo_items[] = { PREFIX("ns/purpose/", networkstatus, "Brief summary of router status by purpose (v2 directory format)."), - PREFIX("unregistered-servers-", dirserv_unregistered, NULL), ITEM("network-status", dir, "Brief summary of router status (v1 directory format)"), ITEM("circuit-status", events, "List of current circuits originating here."), @@ -1945,14 +1881,6 @@ static const getinfo_item_t getinfo_items[] = { DOC("address-mappings/config", "Current address mappings from configuration."), DOC("address-mappings/control", "Current address mappings from controller."), - PREFIX("addr-mappings/", events, NULL), - DOC("addr-mappings/all", "Current address mappings without expiry times."), - DOC("addr-mappings/cache", - "Current cached DNS replies without expiry times."), - DOC("addr-mappings/config", - "Current address mappings from configuration without expiry times."), - DOC("addr-mappings/control", - "Current address mappings from controller without expiry times."), PREFIX("status/", events, NULL), DOC("status/circuit-established", "Whether we think client functionality is working."), @@ -1988,18 +1916,18 @@ static char * list_getinfo_options(void) { int i; - char buf[300]; + char *buf=NULL; smartlist_t *lines = smartlist_create(); char *ans; for (i = 0; getinfo_items[i].varname; ++i) { if (!getinfo_items[i].desc) continue; - tor_snprintf(buf, sizeof(buf), "%s%s -- %s\n", + tor_asprintf(&buf, "%s%s -- %s\n", getinfo_items[i].varname, getinfo_items[i].is_prefix ? "*" : "", getinfo_items[i].desc); - smartlist_add(lines, tor_strdup(buf)); + smartlist_add(lines, buf); } smartlist_sort_strings(lines); @@ -2016,7 +1944,8 @@ list_getinfo_options(void) * internal error. */ static int handle_getinfo_helper(control_connection_t *control_conn, - const char *question, char **answer) + const char *question, char **answer, + const char **err_out) { int i; *answer = NULL; /* unrecognized key by default */ @@ -2029,7 +1958,7 @@ handle_getinfo_helper(control_connection_t *control_conn, match = !strcmp(question, getinfo_items[i].varname); if (match) { tor_assert(getinfo_items[i].fn); - return getinfo_items[i].fn(control_conn, question, answer); + return getinfo_items[i].fn(control_conn, question, answer, err_out); } } @@ -2051,10 +1980,12 @@ handle_control_getinfo(control_connection_t *conn, uint32_t len, smartlist_split_string(questions, body, " ", SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); - SMARTLIST_FOREACH(questions, const char *, q, - { - if (handle_getinfo_helper(conn, q, &ans) < 0) { - connection_write_str_to_buf("551 Internal error\r\n", conn); + SMARTLIST_FOREACH_BEGIN(questions, const char *, q) { + const char *errmsg = NULL; + if (handle_getinfo_helper(conn, q, &ans, &errmsg) < 0) { + if (!errmsg) + errmsg = "Internal error"; + connection_printf_to_buf(conn, "551 %s\r\n", errmsg); goto done; } if (!ans) { @@ -2063,7 +1994,7 @@ handle_control_getinfo(control_connection_t *conn, uint32_t len, smartlist_add(answers, tor_strdup(q)); smartlist_add(answers, ans); } - }); + } SMARTLIST_FOREACH_END(q); if (smartlist_len(unrecognized)) { for (i=0; i < smartlist_len(unrecognized)-1; ++i) connection_printf_to_buf(conn, @@ -2108,12 +2039,12 @@ handle_control_getinfo(control_connection_t *conn, uint32_t len, static uint8_t circuit_purpose_from_string(const char *string) { - if (!strcmpstart(string, "purpose=")) + if (!strcasecmpstart(string, "purpose=")) string += strlen("purpose="); - if (!strcmp(string, "general")) + if (!strcasecmp(string, "general")) return CIRCUIT_PURPOSE_C_GENERAL; - else if (!strcmp(string, "controller")) + else if (!strcasecmp(string, "controller")) return CIRCUIT_PURPOSE_CONTROLLER; else return CIRCUIT_PURPOSE_UNKNOWN; @@ -2145,6 +2076,31 @@ getargs_helper(const char *command, control_connection_t *conn, return NULL; } +/** Helper. Return the first element of <b>sl</b> at index <b>start_at</b> or + * higher that starts with <b>prefix</b>, case-insensitive. Return NULL if no + * such element exists. */ +static const char * +find_element_starting_with(smartlist_t *sl, int start_at, const char *prefix) +{ + int i; + for (i = start_at; i < smartlist_len(sl); ++i) { + const char *elt = smartlist_get(sl, i); + if (!strcasecmpstart(elt, prefix)) + return elt; + } + return NULL; +} + +/** Helper. Return true iff s is an argument that we should treat as a + * key-value pair. */ +static int +is_keyval_pair(const char *s) +{ + /* An argument is a key-value pair if it has an =, and it isn't of the form + * $fingeprint=name */ + return strchr(s, '=') && s[0] != '$'; +} + /** Called when we get an EXTENDCIRCUIT message. Try to extend the listed * circuit, and report success or failure. */ static int @@ -2160,33 +2116,57 @@ handle_control_extendcircuit(control_connection_t *conn, uint32_t len, router_nicknames = smartlist_create(); - args = getargs_helper("EXTENDCIRCUIT", conn, body, 2, -1); + args = getargs_helper("EXTENDCIRCUIT", conn, body, 1, -1); if (!args) goto done; zero_circ = !strcmp("0", (char*)smartlist_get(args,0)); - if (!zero_circ && !(circ = get_circ(smartlist_get(args,0)))) { - connection_printf_to_buf(conn, "552 Unknown circuit \"%s\"\r\n", - (char*)smartlist_get(args, 0)); - } - smartlist_split_string(router_nicknames, smartlist_get(args,1), ",", 0, 0); - if (zero_circ && smartlist_len(args)>2) { - char *purp = smartlist_get(args,2); - intended_purpose = circuit_purpose_from_string(purp); - if (intended_purpose == CIRCUIT_PURPOSE_UNKNOWN) { - connection_printf_to_buf(conn, "552 Unknown purpose \"%s\"\r\n", purp); + if (zero_circ) { + const char *purp = find_element_starting_with(args, 1, "PURPOSE="); + + if (purp) { + intended_purpose = circuit_purpose_from_string(purp); + if (intended_purpose == CIRCUIT_PURPOSE_UNKNOWN) { + connection_printf_to_buf(conn, "552 Unknown purpose \"%s\"\r\n", purp); + SMARTLIST_FOREACH(args, char *, cp, tor_free(cp)); + smartlist_free(args); + goto done; + } + } + + if ((smartlist_len(args) == 1) || + (smartlist_len(args) >= 2 && is_keyval_pair(smartlist_get(args, 1)))) { + // "EXTENDCIRCUIT 0" || EXTENDCIRCUIT 0 foo=bar" + circ = circuit_launch_by_router(intended_purpose, NULL, + CIRCLAUNCH_NEED_CAPACITY); + if (!circ) { + connection_write_str_to_buf("551 Couldn't start circuit\r\n", conn); + } else { + connection_printf_to_buf(conn, "250 EXTENDED %lu\r\n", + (unsigned long)circ->global_identifier); + } SMARTLIST_FOREACH(args, char *, cp, tor_free(cp)); smartlist_free(args); goto done; } + // "EXTENDCIRCUIT 0 router1,router2" || + // "EXTENDCIRCUIT 0 router1,router2 PURPOSE=foo" } - SMARTLIST_FOREACH(args, char *, cp, tor_free(cp)); - smartlist_free(args); - if (!zero_circ && !circ) { + + if (!zero_circ && !(circ = get_circ(smartlist_get(args,0)))) { + connection_printf_to_buf(conn, "552 Unknown circuit \"%s\"\r\n", + (char*)smartlist_get(args, 0)); + SMARTLIST_FOREACH(args, char *, cp, tor_free(cp)); + smartlist_free(args); goto done; } + smartlist_split_string(router_nicknames, smartlist_get(args,1), ",", 0, 0); + + SMARTLIST_FOREACH(args, char *, cp, tor_free(cp)); + smartlist_free(args); + routers = smartlist_create(); SMARTLIST_FOREACH(router_nicknames, const char *, n, { @@ -2244,8 +2224,7 @@ handle_control_extendcircuit(control_connection_t *conn, uint32_t len, done: SMARTLIST_FOREACH(router_nicknames, char *, n, tor_free(n)); smartlist_free(router_nicknames); - if (routers) - smartlist_free(routers); + smartlist_free(routers); return 0; } @@ -2271,7 +2250,7 @@ handle_control_setcircuitpurpose(control_connection_t *conn, } { - char *purp = smartlist_get(args,1); + const char *purp = find_element_starting_with(args,1,"PURPOSE="); new_purpose = circuit_purpose_from_string(purp); if (new_purpose == CIRCUIT_PURPOSE_UNKNOWN) { connection_printf_to_buf(conn, "552 Unknown purpose \"%s\"\r\n", purp); @@ -2282,7 +2261,7 @@ handle_control_setcircuitpurpose(control_connection_t *conn, circ->_base.purpose = new_purpose; connection_write_str_to_buf("250 OK\r\n", conn); -done: + done: if (args) { SMARTLIST_FOREACH(args, char *, cp, tor_free(cp)); smartlist_free(args); @@ -2316,9 +2295,9 @@ handle_control_attachstream(control_connection_t *conn, uint32_t len, } else if (!zero_circ && !(circ = get_circ(smartlist_get(args, 1)))) { connection_printf_to_buf(conn, "552 Unknown circuit \"%s\"\r\n", (char*)smartlist_get(args, 1)); - } else if (circ && smartlist_len(args) > 2) { - char *hopstring = smartlist_get(args, 2); - if (!strcasecmpstart(hopstring, "HOP=")) { + } else if (circ) { + const char *hopstring = find_element_starting_with(args,2,"HOP="); + if (hopstring) { hopstring += strlen("HOP="); hop = (int) tor_parse_ulong(hopstring, 10, 0, INT_MAX, &hop_line_ok, NULL); @@ -2365,7 +2344,7 @@ handle_control_attachstream(control_connection_t *conn, uint32_t len, char* exit_digest; if (circ->build_state && circ->build_state->chosen_exit && - circ->build_state->chosen_exit->identity_digest) { + !tor_digest_is_zero(circ->build_state->chosen_exit->identity_digest)) { exit_digest = circ->build_state->chosen_exit->identity_digest; r = router_get_by_digest(exit_digest); } @@ -2426,9 +2405,9 @@ handle_control_postdescriptor(control_connection_t *conn, uint32_t len, } } else if (!strcasecmpstart(option, "cache=")) { option += strlen("cache="); - if (!strcmp(option, "no")) + if (!strcasecmp(option, "no")) cache = 0; - else if (!strcmp(option, "yes")) + else if (!strcasecmp(option, "yes")) cache = 1; else { connection_printf_to_buf(conn, "552 Unknown cache request \"%s\"\r\n", @@ -2610,17 +2589,17 @@ handle_control_resolve(control_connection_t *conn, uint32_t len, args = smartlist_create(); smartlist_split_string(args, body, " ", SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); - if (smartlist_len(args) && - !strcasecmp(smartlist_get(args, 0), "mode=reverse")) { - char *cp = smartlist_get(args, 0); - smartlist_del_keeporder(args, 0); - tor_free(cp); - is_reverse = 1; + { + const char *modearg = find_element_starting_with(args, 0, "mode="); + if (modearg && !strcasecmp(modearg, "mode=reverse")) + is_reverse = 1; } failed = smartlist_create(); SMARTLIST_FOREACH(args, const char *, arg, { - if (dnsserv_launch_request(arg, is_reverse)<0) - smartlist_add(failed, (char*)arg); + if (!is_keyval_pair(arg)) { + if (dnsserv_launch_request(arg, is_reverse)<0) + smartlist_add(failed, (char*)arg); + } }); send_control_done(conn); @@ -2710,7 +2689,6 @@ handle_control_usefeature(control_connection_t *conn, const char *body) { smartlist_t *args; - int verbose_names = 0, extended_events = 0; int bad = 0; (void) len; /* body is nul-terminated; it's safe to ignore the length */ args = smartlist_create(); @@ -2718,9 +2696,9 @@ handle_control_usefeature(control_connection_t *conn, SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); SMARTLIST_FOREACH(args, const char *, arg, { if (!strcasecmp(arg, "VERBOSE_NAMES")) - verbose_names = 1; + ; else if (!strcasecmp(arg, "EXTENDED_EVENTS")) - extended_events = 1; + ; else { connection_printf_to_buf(conn, "552 Unrecognized feature \"%s\"\r\n", arg); @@ -2730,12 +2708,6 @@ handle_control_usefeature(control_connection_t *conn, }); if (!bad) { - if (verbose_names) { - conn->use_long_names = 1; - control_update_global_event_mask(); - } - if (extended_events) - conn->use_extended_events = 1; send_control_done(conn); } @@ -2885,9 +2857,10 @@ connection_control_process_inbuf(control_connection_t *conn) && !TOR_ISSPACE(conn->incoming_cmd[cmd_len])) ++cmd_len; - data_len -= cmd_len; conn->incoming_cmd[cmd_len]='\0'; args = conn->incoming_cmd+cmd_len+1; + tor_assert(data_len>(size_t)cmd_len); + data_len -= (cmd_len+1); /* skip the command and NUL we added after it */ while (*args == ' ' || *args == '\t') { ++args; --data_len; @@ -3039,20 +3012,11 @@ control_event_circuit_status(origin_circuit_t *circ, circuit_status_event_t tp, tor_free(reason); } - if (EVENT_IS_INTERESTING1S(EVENT_CIRCUIT_STATUS)) { - char *path = circuit_list_path(circ,0); - const char *sp = strlen(path) ? " " : ""; - send_control_event_extended(EVENT_CIRCUIT_STATUS, SHORT_NAMES, - "650 CIRC %lu %s%s%s@%s\r\n", - (unsigned long)circ->global_identifier, - status, sp, path, extended_buf); - tor_free(path); - } - if (EVENT_IS_INTERESTING1L(EVENT_CIRCUIT_STATUS)) { + { char *vpath = circuit_list_path_for_controller(circ); const char *sp = strlen(vpath) ? " " : ""; - send_control_event_extended(EVENT_CIRCUIT_STATUS, LONG_NAMES, - "650 CIRC %lu %s%s%s@%s\r\n", + send_control_event(EVENT_CIRCUIT_STATUS, ALL_FORMATS, + "650 CIRC %lu %s%s%s %s\r\n", (unsigned long)circ->global_identifier, status, sp, vpath, extended_buf); tor_free(vpath); @@ -3131,26 +3095,26 @@ control_event_stream_status(edge_connection_t *conn, stream_status_event_t tp, char *r = NULL; if (!reason_str) { r = tor_malloc(16); - tor_snprintf(r, 16, "UNKNOWN_%d", reason_code); + tor_snprintf(r, 16, " UNKNOWN_%d", reason_code); reason_str = r; } if (reason_code & END_STREAM_REASON_FLAG_REMOTE) tor_snprintf(reason_buf, sizeof(reason_buf), - "REASON=END REMOTE_REASON=%s", reason_str); + " REASON=END REMOTE_REASON=%s", reason_str); else tor_snprintf(reason_buf, sizeof(reason_buf), - "REASON=%s", reason_str); + " REASON=%s", reason_str); tor_free(r); } else if (reason_code && tp == STREAM_EVENT_REMAP) { switch (reason_code) { case REMAP_STREAM_SOURCE_CACHE: - strlcpy(reason_buf, "SOURCE=CACHE", sizeof(reason_buf)); + strlcpy(reason_buf, " SOURCE=CACHE", sizeof(reason_buf)); break; case REMAP_STREAM_SOURCE_EXIT: - strlcpy(reason_buf, "SOURCE=EXIT", sizeof(reason_buf)); + strlcpy(reason_buf, " SOURCE=EXIT", sizeof(reason_buf)); break; default: - tor_snprintf(reason_buf, sizeof(reason_buf), "REASON=UNKNOWN_%d", + tor_snprintf(reason_buf, sizeof(reason_buf), " REASON=UNKNOWN_%d", reason_code); /* XXX do we want SOURCE=UNKNOWN_%d above instead? -RD */ break; @@ -3158,8 +3122,7 @@ control_event_stream_status(edge_connection_t *conn, stream_status_event_t tp, } if (tp == STREAM_EVENT_NEW) { - tor_snprintf(addrport_buf,sizeof(addrport_buf), "%sSOURCE_ADDR=%s:%d", - strlen(reason_buf) ? " " : "", + tor_snprintf(addrport_buf,sizeof(addrport_buf), " SOURCE_ADDR=%s:%d", TO_CONN(conn)->address, TO_CONN(conn)->port ); } else { addrport_buf[0] = '\0'; @@ -3188,8 +3151,8 @@ control_event_stream_status(edge_connection_t *conn, stream_status_event_t tp, circ = circuit_get_by_edge_conn(conn); if (circ && CIRCUIT_IS_ORIGIN(circ)) origin_circ = TO_ORIGIN_CIRCUIT(circ); - send_control_event_extended(EVENT_STREAM_STATUS, ALL_NAMES, - "650 STREAM "U64_FORMAT" %s %lu %s@%s%s%s\r\n", + send_control_event(EVENT_STREAM_STATUS, ALL_FORMATS, + "650 STREAM "U64_FORMAT" %s %lu %s%s%s%s\r\n", U64_PRINTF_ARG(conn->_base.global_identifier), status, origin_circ? (unsigned long)origin_circ->global_identifier : 0ul, @@ -3202,30 +3165,21 @@ control_event_stream_status(edge_connection_t *conn, stream_status_event_t tp, /** Figure out the best name for the target router of an OR connection * <b>conn</b>, and write it into the <b>len</b>-character buffer - * <b>name</b>. Use verbose names if <b>long_names</b> is set. */ + * <b>name</b>. */ static void -orconn_target_get_name(int long_names, - char *name, size_t len, or_connection_t *conn) -{ - if (! long_names) { - if (conn->nickname) - strlcpy(name, conn->nickname, len); - else - tor_snprintf(name, len, "%s:%d", - conn->_base.address, conn->_base.port); +orconn_target_get_name(char *name, size_t len, or_connection_t *conn) +{ + routerinfo_t *ri = router_get_by_digest(conn->identity_digest); + if (ri) { + tor_assert(len > MAX_VERBOSE_NICKNAME_LEN); + router_get_verbose_nickname(name, ri); + } else if (! tor_digest_is_zero(conn->identity_digest)) { + name[0] = '$'; + base16_encode(name+1, len-1, conn->identity_digest, + DIGEST_LEN); } else { - routerinfo_t *ri = router_get_by_digest(conn->identity_digest); - if (ri) { - tor_assert(len > MAX_VERBOSE_NICKNAME_LEN); - router_get_verbose_nickname(name, ri); - } else if (! tor_digest_is_zero(conn->identity_digest)) { - name[0] = '$'; - base16_encode(name+1, len-1, conn->identity_digest, - DIGEST_LEN); - } else { - tor_snprintf(name, len, "%s:%d", - conn->_base.address, conn->_base.port); - } + tor_snprintf(name, len, "%s:%d", + conn->_base.address, conn->_base.port); } } @@ -3264,24 +3218,13 @@ control_event_or_conn_status(or_connection_t *conn, or_conn_status_event_t tp, reason ? " " : "", ncircs); } - if (EVENT_IS_INTERESTING1S(EVENT_OR_CONN_STATUS)) { - orconn_target_get_name(0, name, sizeof(name), conn); - send_control_event_extended(EVENT_OR_CONN_STATUS, SHORT_NAMES, - "650 ORCONN %s %s@%s%s%s\r\n", - name, status, - reason ? "REASON=" : "", - orconn_end_reason_to_control_string(reason), - ncircs_buf); - } - if (EVENT_IS_INTERESTING1L(EVENT_OR_CONN_STATUS)) { - orconn_target_get_name(1, name, sizeof(name), conn); - send_control_event_extended(EVENT_OR_CONN_STATUS, LONG_NAMES, - "650 ORCONN %s %s@%s%s%s\r\n", - name, status, - reason ? "REASON=" : "", - orconn_end_reason_to_control_string(reason), - ncircs_buf); - } + orconn_target_get_name(name, sizeof(name), conn); + send_control_event(EVENT_OR_CONN_STATUS, ALL_FORMATS, + "650 ORCONN %s %s %s%s%s\r\n", + name, status, + reason ? "REASON=" : "", + orconn_end_reason_to_control_string(reason), + ncircs_buf); return 0; } @@ -3296,7 +3239,7 @@ control_event_stream_bandwidth(edge_connection_t *edge_conn) if (!edge_conn->n_read && !edge_conn->n_written) return 0; - send_control_event(EVENT_STREAM_BANDWIDTH_USED, ALL_NAMES, + send_control_event(EVENT_STREAM_BANDWIDTH_USED, ALL_FORMATS, "650 STREAM_BW "U64_FORMAT" %lu %lu\r\n", U64_PRINTF_ARG(edge_conn->_base.global_identifier), (unsigned long)edge_conn->n_read, @@ -3325,7 +3268,7 @@ control_event_stream_bandwidth_used(void) if (!edge_conn->n_read && !edge_conn->n_written) continue; - send_control_event(EVENT_STREAM_BANDWIDTH_USED, ALL_NAMES, + send_control_event(EVENT_STREAM_BANDWIDTH_USED, ALL_FORMATS, "650 STREAM_BW "U64_FORMAT" %lu %lu\r\n", U64_PRINTF_ARG(edge_conn->_base.global_identifier), (unsigned long)edge_conn->n_read, @@ -3345,7 +3288,7 @@ int control_event_bandwidth_used(uint32_t n_read, uint32_t n_written) { if (EVENT_IS_INTERESTING(EVENT_BANDWIDTH_USED)) { - send_control_event(EVENT_BANDWIDTH_USED, ALL_NAMES, + send_control_event(EVENT_BANDWIDTH_USED, ALL_FORMATS, "650 BW %lu %lu\r\n", (unsigned long)n_read, (unsigned long)n_written); @@ -3415,7 +3358,7 @@ control_event_logmsg(int severity, uint32_t domain, const char *msg) default: s = "UnknownLogSeverity"; break; } ++disable_log_messages; - send_control_event(event, ALL_NAMES, "650 %s %s\r\n", s, b?b:msg); + send_control_event(event, ALL_FORMATS, "650 %s %s\r\n", s, b?b:msg); --disable_log_messages; tor_free(b); } @@ -3428,31 +3371,12 @@ control_event_logmsg(int severity, uint32_t domain, const char *msg) int control_event_descriptors_changed(smartlist_t *routers) { - size_t len; char *msg; - smartlist_t *identities = NULL; - char buf[HEX_DIGEST_LEN+1]; if (!EVENT_IS_INTERESTING(EVENT_NEW_DESC)) return 0; - if (EVENT_IS_INTERESTING1S(EVENT_NEW_DESC)) { - identities = smartlist_create(); - SMARTLIST_FOREACH(routers, routerinfo_t *, r, - { - base16_encode(buf,sizeof(buf),r->cache_info.identity_digest,DIGEST_LEN); - smartlist_add(identities, tor_strdup(buf)); - }); - } - if (EVENT_IS_INTERESTING1S(EVENT_NEW_DESC)) { - char *ids = smartlist_join_strings(identities, " ", 0, &len); - size_t ids_len = strlen(ids)+32; - msg = tor_malloc(ids_len); - tor_snprintf(msg, ids_len, "650 NEWDESC %s\r\n", ids); - send_control_event_string(EVENT_NEW_DESC, SHORT_NAMES|ALL_FORMATS, msg); - tor_free(ids); - tor_free(msg); - } - if (EVENT_IS_INTERESTING1L(EVENT_NEW_DESC)) { + + { smartlist_t *names = smartlist_create(); char *ids; size_t names_len; @@ -3465,16 +3389,12 @@ control_event_descriptors_changed(smartlist_t *routers) names_len = strlen(ids)+32; msg = tor_malloc(names_len); tor_snprintf(msg, names_len, "650 NEWDESC %s\r\n", ids); - send_control_event_string(EVENT_NEW_DESC, LONG_NAMES|ALL_FORMATS, msg); + send_control_event_string(EVENT_NEW_DESC, ALL_FORMATS, msg); tor_free(ids); tor_free(msg); SMARTLIST_FOREACH(names, char *, cp, tor_free(cp)); smartlist_free(names); } - if (identities) { - SMARTLIST_FOREACH(identities, char *, cp, tor_free(cp)); - smartlist_free(identities); - } return 0; } @@ -3491,17 +3411,17 @@ control_event_address_mapped(const char *from, const char *to, time_t expires, return 0; if (expires < 3 || expires == TIME_MAX) - send_control_event_extended(EVENT_ADDRMAP, ALL_NAMES, - "650 ADDRMAP %s %s NEVER@%s\r\n", from, to, + send_control_event(EVENT_ADDRMAP, ALL_FORMATS, + "650 ADDRMAP %s %s NEVER %s\r\n", from, to, error?error:""); else { char buf[ISO_TIME_LEN+1]; char buf2[ISO_TIME_LEN+1]; format_local_iso_time(buf,expires); format_iso_time(buf2,expires); - send_control_event_extended(EVENT_ADDRMAP, ALL_NAMES, + send_control_event(EVENT_ADDRMAP, ALL_FORMATS, "650 ADDRMAP %s %s \"%s\"" - "@%s%sEXPIRES=\"%s\"\r\n", + " %s%sEXPIRES=\"%s\"\r\n", from, to, buf, error?error:"", error?" ":"", buf2); @@ -3541,9 +3461,9 @@ control_event_or_authdir_new_descriptor(const char *action, buf = tor_malloc(totallen); strlcpy(buf, firstline, totallen); strlcpy(buf+strlen(firstline), esc, totallen); - send_control_event_string(EVENT_AUTHDIR_NEWDESCS, ALL_NAMES|ALL_FORMATS, + send_control_event_string(EVENT_AUTHDIR_NEWDESCS, ALL_FORMATS, buf); - send_control_event_string(EVENT_AUTHDIR_NEWDESCS, ALL_NAMES|ALL_FORMATS, + send_control_event_string(EVENT_AUTHDIR_NEWDESCS, ALL_FORMATS, "650 OK\r\n"); tor_free(esc); tor_free(buf); @@ -3581,8 +3501,8 @@ control_event_networkstatus_changed_helper(smartlist_t *statuses, SMARTLIST_FOREACH(strs, char *, cp, tor_free(cp)); smartlist_free(strs); tor_free(s); - send_control_event_string(event, ALL_NAMES|ALL_FORMATS, esc); - send_control_event_string(event, ALL_NAMES|ALL_FORMATS, + send_control_event_string(event, ALL_FORMATS, esc); + send_control_event_string(event, ALL_FORMATS, "650 OK\r\n"); tor_free(esc); @@ -3608,6 +3528,55 @@ control_event_newconsensus(const networkstatus_t *consensus) consensus->routerstatus_list, EVENT_NEWCONSENSUS, "NEWCONSENSUS"); } +/** Called when we compute a new circuitbuildtimeout */ +int +control_event_buildtimeout_set(const circuit_build_times_t *cbt, + buildtimeout_set_event_t type) +{ + const char *type_string = NULL; + double qnt = circuit_build_times_quantile_cutoff(); + + if (!control_event_is_interesting(EVENT_BUILDTIMEOUT_SET)) + return 0; + + switch (type) { + case BUILDTIMEOUT_SET_EVENT_COMPUTED: + type_string = "COMPUTED"; + break; + case BUILDTIMEOUT_SET_EVENT_RESET: + type_string = "RESET"; + qnt = 1.0; + break; + case BUILDTIMEOUT_SET_EVENT_SUSPENDED: + type_string = "SUSPENDED"; + qnt = 1.0; + break; + case BUILDTIMEOUT_SET_EVENT_DISCARD: + type_string = "DISCARD"; + qnt = 1.0; + break; + case BUILDTIMEOUT_SET_EVENT_RESUME: + type_string = "RESUME"; + break; + default: + type_string = "UNKNOWN"; + break; + } + + send_control_event(EVENT_BUILDTIMEOUT_SET, ALL_FORMATS, + "650 BUILDTIMEOUT_SET %s TOTAL_TIMES=%lu " + "TIMEOUT_MS=%lu XM=%lu ALPHA=%lf CUTOFF_QUANTILE=%lf " + "TIMEOUT_RATE=%lf CLOSE_MS=%lu CLOSE_RATE=%lf\r\n", + type_string, (unsigned long)cbt->total_build_times, + (unsigned long)cbt->timeout_ms, + (unsigned long)cbt->Xm, cbt->alpha, qnt, + circuit_build_times_timeout_rate(cbt), + (unsigned long)cbt->close_ms, + circuit_build_times_close_rate(cbt)); + + return 0; +} + /** Called when a single local_routerstatus_t has changed: Sends an NS event * to any controller that cares. */ int @@ -3631,7 +3600,7 @@ control_event_networkstatus_changed_single(routerstatus_t *rs) int control_event_my_descriptor_changed(void) { - send_control_event(EVENT_DESCCHANGED, ALL_NAMES, "650 DESCCHANGED\r\n"); + send_control_event(EVENT_DESCCHANGED, ALL_FORMATS, "650 DESCCHANGED\r\n"); return 0; } @@ -3679,7 +3648,7 @@ control_event_status(int type, int severity, const char *format, va_list args) return -1; } - send_control_event_impl(type, ALL_NAMES|ALL_FORMATS, 0, format_buf, args); + send_control_event_impl(type, ALL_FORMATS, format_buf, args); return 0; } @@ -3743,7 +3712,7 @@ control_event_guard(const char *nickname, const char *digest, if (!EVENT_IS_INTERESTING(EVENT_GUARD)) return 0; - if (EVENT_IS_INTERESTING1L(EVENT_GUARD)) { + { char buf[MAX_VERBOSE_NICKNAME_LEN+1]; routerinfo_t *ri = router_get_by_digest(digest); if (ri) { @@ -3751,13 +3720,9 @@ control_event_guard(const char *nickname, const char *digest, } else { tor_snprintf(buf, sizeof(buf), "$%s~%s", hbuf, nickname); } - send_control_event(EVENT_GUARD, LONG_NAMES, + send_control_event(EVENT_GUARD, ALL_FORMATS, "650 GUARD ENTRY %s %s\r\n", buf, status); } - if (EVENT_IS_INTERESTING1S(EVENT_GUARD)) { - send_control_event(EVENT_GUARD, SHORT_NAMES, - "650 GUARD ENTRY $%s %s\r\n", hbuf, status); - } return 0; } @@ -3910,7 +3875,7 @@ static int bootstrap_problems = 0; * information and initial circuits. * * <b>status</b> is the new status, that is, what task we will be doing - * next. <b>percent</b> is zero if we just started this task, else it + * next. <b>progress</b> is zero if we just started this task, else it * represents progress on the task. */ void control_event_bootstrap(bootstrap_status_t status, int progress) @@ -3966,6 +3931,9 @@ control_event_bootstrap_problem(const char *warn, int reason) char buf[BOOTSTRAP_MSG_LEN]; const char *recommendation = "ignore"; + /* bootstrap_percent must not be in "undefined" state here. */ + tor_assert(status >= 0); + if (bootstrap_percent == 100) return; /* already bootstrapped; nothing to be done here. */ @@ -4008,10 +3976,9 @@ control_event_bootstrap_problem(const char *warn, int reason) * from recently. Send a copy to the controller in case it wants to * display it for the user. */ void -control_event_clients_seen(const char *timestarted, const char *countries) +control_event_clients_seen(const char *controller_str) { send_control_event(EVENT_CLIENTS_SEEN, 0, - "650 CLIENTS_SEEN TimeStarted=\"%s\" CountrySummary=%s\r\n", - timestarted, countries); + "650 CLIENTS_SEEN %s\r\n", controller_str); } diff --git a/src/or/control.h b/src/or/control.h new file mode 100644 index 0000000000..2ae96b4b8a --- /dev/null +++ b/src/or/control.h @@ -0,0 +1,85 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file control.h + * \brief Header file for control.c. + **/ + +#ifndef _TOR_CONTROL_H +#define _TOR_CONTROL_H + +void control_update_global_event_mask(void); +void control_adjust_event_log_severity(void); + +/** Log information about the connection <b>conn</b>, protecting it as with + * CONN_LOG_PROTECT. Example: + * + * LOG_FN_CONN(conn, (LOG_DEBUG, "Socket %d wants to write", conn->s)); + **/ +#define LOG_FN_CONN(conn, args) \ + CONN_LOG_PROTECT(conn, log_fn args) + +int connection_control_finished_flushing(control_connection_t *conn); +int connection_control_reached_eof(control_connection_t *conn); +int connection_control_process_inbuf(control_connection_t *conn); + +#define EVENT_AUTHDIR_NEWDESCS 0x000D +#define EVENT_NS 0x000F +int control_event_is_interesting(int event); + +int control_event_circuit_status(origin_circuit_t *circ, + circuit_status_event_t e, int reason); +int control_event_stream_status(edge_connection_t *conn, + stream_status_event_t e, + int reason); +int control_event_or_conn_status(or_connection_t *conn, + or_conn_status_event_t e, int reason); +int control_event_bandwidth_used(uint32_t n_read, uint32_t n_written); +int control_event_stream_bandwidth(edge_connection_t *edge_conn); +int control_event_stream_bandwidth_used(void); +void control_event_logmsg(int severity, unsigned int domain, const char *msg); +int control_event_descriptors_changed(smartlist_t *routers); +int control_event_address_mapped(const char *from, const char *to, + time_t expires, const char *error); +int control_event_or_authdir_new_descriptor(const char *action, + const char *desc, + size_t desclen, + const char *msg); +int control_event_my_descriptor_changed(void); +int control_event_networkstatus_changed(smartlist_t *statuses); + +int control_event_newconsensus(const networkstatus_t *consensus); +int control_event_networkstatus_changed_single(routerstatus_t *rs); +int control_event_general_status(int severity, const char *format, ...) + CHECK_PRINTF(2,3); +int control_event_client_status(int severity, const char *format, ...) + CHECK_PRINTF(2,3); +int control_event_server_status(int severity, const char *format, ...) + CHECK_PRINTF(2,3); +int control_event_guard(const char *nickname, const char *digest, + const char *status); +int control_event_buildtimeout_set(const circuit_build_times_t *cbt, + buildtimeout_set_event_t type); + +int init_cookie_authentication(int enabled); +smartlist_t *decode_hashed_passwords(config_line_t *passwords); +void disable_control_logging(void); +void enable_control_logging(void); + +void control_event_bootstrap(bootstrap_status_t status, int progress); +void control_event_bootstrap_problem(const char *warn, int reason); + +void control_event_clients_seen(const char *controller_str); + +#ifdef CONTROL_PRIVATE +/* Used only by control.c and test.c */ +size_t write_escaped_data(const char *data, size_t len, char **out); +size_t read_escaped_data(const char *data, size_t len, char **out); +#endif + +#endif + diff --git a/src/or/cpuworker.c b/src/or/cpuworker.c index 91af5f1d61..7cbc191333 100644 --- a/src/or/cpuworker.c +++ b/src/or/cpuworker.c @@ -13,6 +13,15 @@ **/ #include "or.h" +#include "buffers.h" +#include "circuitbuild.h" +#include "circuitlist.h" +#include "config.h" +#include "connection.h" +#include "cpuworker.h" +#include "main.h" +#include "onion.h" +#include "router.h" /** The maximum number of cpuworker processes we will keep around. */ #define MAX_CPUWORKERS 16 @@ -183,7 +192,7 @@ connection_cpu_process_inbuf(connection_t *conn) tor_assert(0); /* don't ask me to do handshakes yet */ } -done_processing: + done_processing: conn->state = CPUWORKER_STATE_IDLE; num_cpuworkers_busy--; if (conn->timestamp_created < last_rotation_time) { @@ -241,7 +250,7 @@ cpuworker_main(void *data) for (;;) { ssize_t r; - if ((r = recv(fd, &question_type, 1, 0)) != 1) { + if ((r = recv(fd, (void *)&question_type, 1, 0)) != 1) { // log_fn(LOG_ERR,"read type failed. Exiting."); if (r == 0) { log_info(LD_OR, @@ -358,7 +367,7 @@ spawn_cpuworker(void) static void spawn_enough_cpuworkers(void) { - int num_cpuworkers_needed = get_options()->NumCpus; + int num_cpuworkers_needed = get_options()->NumCPUs; if (num_cpuworkers_needed < MIN_CPUWORKERS) num_cpuworkers_needed = MIN_CPUWORKERS; diff --git a/src/or/cpuworker.h b/src/or/cpuworker.h new file mode 100644 index 0000000000..04e37ee459 --- /dev/null +++ b/src/or/cpuworker.h @@ -0,0 +1,25 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file cpuworker.h + * \brief Header file for cpuworker.c. + **/ + +#ifndef _TOR_CPUWORKER_H +#define _TOR_CPUWORKER_H + +void cpu_init(void); +void cpuworkers_rotate(void); +int connection_cpu_finished_flushing(connection_t *conn); +int connection_cpu_reached_eof(connection_t *conn); +int connection_cpu_process_inbuf(connection_t *conn); +int assign_onionskin_to_cpuworker(connection_t *cpuworker, + or_circuit_t *circ, + char *onionskin); + +#endif + diff --git a/src/or/directory.c b/src/or/directory.c index 01f33752ff..309d99745f 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -4,6 +4,26 @@ /* See LICENSE for licensing information */ #include "or.h" +#include "buffers.h" +#include "circuitbuild.h" +#include "config.h" +#include "connection.h" +#include "connection_edge.h" +#include "control.h" +#include "directory.h" +#include "dirserv.h" +#include "dirvote.h" +#include "geoip.h" +#include "main.h" +#include "networkstatus.h" +#include "policies.h" +#include "rendclient.h" +#include "rendcommon.h" +#include "rephist.h" +#include "router.h" +#include "routerlist.h" +#include "routerparse.h" + #if defined(EXPORTMALLINFO) && defined(HAVE_MALLOC_H) && defined(HAVE_MALLINFO) #ifndef OPENBSD #include <malloc.h> @@ -47,8 +67,10 @@ static void http_set_address_origin(const char *headers, connection_t *conn); static void connection_dir_download_networkstatus_failed( dir_connection_t *conn, int status_code); static void connection_dir_download_routerdesc_failed(dir_connection_t *conn); +static void connection_dir_bridge_routerdesc_failed(dir_connection_t *conn); static void connection_dir_download_cert_failed( dir_connection_t *conn, int status_code); +static void connection_dir_retry_bridges(smartlist_t *descs); static void dir_networkstatus_download_failed(smartlist_t *failed, int status_code); static void dir_routerdesc_download_failed(smartlist_t *failed, @@ -92,17 +114,19 @@ static void directory_initiate_command_rend(const char *address, #define ROUTERDESC_CACHE_LIFETIME (30*60) #define ROUTERDESC_BY_DIGEST_CACHE_LIFETIME (48*60*60) #define ROBOTS_CACHE_LIFETIME (24*60*60) +#define MICRODESC_CACHE_LIFETIME (48*60*60) /********* END VARIABLES ************/ -/** Return true iff the directory purpose 'purpose' must use an - * anonymous connection to a directory. */ +/** Return true iff the directory purpose <b>dir_purpose</b> (and if it's + * fetching descriptors, it's fetching them for <b>router_purpose</b>) + * must use an anonymous connection to a directory. */ static int purpose_needs_anonymity(uint8_t dir_purpose, uint8_t router_purpose) { if (get_options()->AllDirActionsPrivate) return 1; - if (router_purpose == ROUTER_PURPOSE_BRIDGE && has_completed_circuit) + if (router_purpose == ROUTER_PURPOSE_BRIDGE && can_complete_circuit) return 1; /* if no circuits yet, we may need this info to bootstrap. */ if (dir_purpose == DIR_PURPOSE_UPLOAD_DIR || dir_purpose == DIR_PURPOSE_UPLOAD_VOTE || @@ -229,10 +253,13 @@ directories_have_accepted_server_descriptor(void) } /** Start a connection to every suitable directory authority, using - * connection purpose 'purpose' and uploading the payload 'payload' - * (length 'payload_len'). The purpose should be one of + * connection purpose <b>dir_purpose</b> and uploading <b>payload</b> + * (of length <b>payload_len</b>). The dir_purpose should be one of * 'DIR_PURPOSE_UPLOAD_DIR' or 'DIR_PURPOSE_UPLOAD_RENDDESC'. * + * <b>router_purpose</b> describes the type of descriptor we're + * publishing, if we're publishing a descriptor -- e.g. general or bridge. + * * <b>type</b> specifies what sort of dir authorities (V1, V2, * HIDSERV, BRIDGE) we should upload to. * @@ -248,6 +275,7 @@ directory_post_to_dirservers(uint8_t dir_purpose, uint8_t router_purpose, const char *payload, size_t payload_len, size_t extrainfo_len) { + or_options_t *options = get_options(); int post_via_tor; smartlist_t *dirservers = router_get_trusted_dir_servers(); int found = 0; @@ -263,6 +291,16 @@ directory_post_to_dirservers(uint8_t dir_purpose, uint8_t router_purpose, if ((type & ds->type) == 0) continue; + if (options->ExcludeNodes && options->StrictNodes && + routerset_contains_routerstatus(options->ExcludeNodes, rs)) { + log_warn(LD_DIR, "Wanted to contact authority '%s' for %s, but " + "it's in our ExcludedNodes list and StrictNodes is set. " + "Skipping.", + ds->nickname, + dir_conn_purpose_to_string(dir_purpose)); + continue; + } + found = 1; /* at least one authority of this type was listed */ if (dir_purpose == DIR_PURPOSE_UPLOAD_DIR) ds->has_accepted_serverdesc = 0; @@ -315,6 +353,7 @@ directory_get_from_dirserver(uint8_t dir_purpose, uint8_t router_purpose, break; case DIR_PURPOSE_FETCH_V2_NETWORKSTATUS: type = V2_AUTHORITY; + prefer_authority = 1; /* Only v2 authorities have these anyway. */ break; case DIR_PURPOSE_FETCH_SERVERDESC: type = (router_purpose == ROUTER_PURPOSE_BRIDGE ? BRIDGE_AUTHORITY : @@ -348,7 +387,7 @@ directory_get_from_dirserver(uint8_t dir_purpose, uint8_t router_purpose, if (!get_via_tor) { if (options->UseBridges && type != BRIDGE_AUTHORITY) { /* want to ask a running bridge for which we have a descriptor. */ - /* XXX022 we assume that all of our bridges can answer any + /* XXX023 we assume that all of our bridges can answer any * possible directory question. This won't be true forever. -RD */ /* It certainly is not true with conditional consensus downloading, * so, for now, never assume the server supports that. */ @@ -472,12 +511,14 @@ directory_initiate_command_routerstatus_rend(routerstatus_t *status, time_t if_modified_since, const rend_data_t *rend_query) { + or_options_t *options = get_options(); routerinfo_t *router; char address_buf[INET_NTOA_BUF_LEN+1]; struct in_addr in; const char *address; tor_addr_t addr; router = router_get_by_digest(status->identity_digest); + if (!router && anonymized_connection) { log_info(LD_DIR, "Not sending anonymized request to directory '%s'; we " "don't have its router descriptor.", status->nickname); @@ -490,6 +531,17 @@ directory_initiate_command_routerstatus_rend(routerstatus_t *status, address = address_buf; } tor_addr_from_ipv4h(&addr, status->addr); + + if (options->ExcludeNodes && options->StrictNodes && + routerset_contains_routerstatus(options->ExcludeNodes, status)) { + log_warn(LD_DIR, "Wanted to contact directory mirror '%s' for %s, but " + "it's in our ExcludedNodes list and StrictNodes is set. " + "Skipping. This choice might make your Tor not work.", + status->nickname, + dir_conn_purpose_to_string(dir_purpose)); + return; + } + directory_initiate_command_rend(address, &addr, status->or_port, status->dir_port, status->version_supports_conditional_consensus, @@ -560,7 +612,7 @@ connection_dir_request_failed(dir_connection_t *conn) if (directory_conn_is_self_reachability_test(conn)) { return; /* this was a test fetch. don't retry. */ } - if (entry_list_can_grow(get_options())) + if (!entry_list_is_constrained(get_options())) router_set_status(conn->identity_digest, 0); /* don't try him again */ if (conn->_base.purpose == DIR_PURPOSE_FETCH_V2_NETWORKSTATUS) { log_info(LD_DIR, "Giving up on directory server at '%s'; retrying", @@ -570,6 +622,8 @@ connection_dir_request_failed(dir_connection_t *conn) conn->_base.purpose == DIR_PURPOSE_FETCH_EXTRAINFO) { log_info(LD_DIR, "Giving up on directory server at '%s'; retrying", conn->_base.address); + if (conn->router_purpose == ROUTER_PURPOSE_BRIDGE) + connection_dir_bridge_routerdesc_failed(conn); connection_dir_download_routerdesc_failed(conn); } else if (conn->_base.purpose == DIR_PURPOSE_FETCH_CONSENSUS) { networkstatus_consensus_download_failed(0); @@ -614,7 +668,7 @@ connection_dir_download_networkstatus_failed(dir_connection_t *conn, * failed, and possibly retry them later.*/ smartlist_t *failed = smartlist_create(); dir_split_resource_into_fingerprints(conn->requested_resource+3, - failed, NULL, 0, 0); + failed, NULL, 0); if (smartlist_len(failed)) { dir_networkstatus_download_failed(failed, status_code); SMARTLIST_FOREACH(failed, char *, cp, tor_free(cp)); @@ -623,6 +677,24 @@ connection_dir_download_networkstatus_failed(dir_connection_t *conn, } } +/** Helper: Attempt to fetch directly the descriptors of each bridge + * listed in <b>failed</b>. + */ +static void +connection_dir_retry_bridges(smartlist_t *descs) +{ + char digest[DIGEST_LEN]; + SMARTLIST_FOREACH(descs, const char *, cp, + { + if (base16_decode(digest, DIGEST_LEN, cp, strlen(cp))<0) { + log_warn(LD_BUG, "Malformed fingerprint in list: %s", + escaped(cp)); + continue; + } + retry_bridge_descriptor_fetch_directly(digest); + }); +} + /** Called when an attempt to download one or more router descriptors * or extra-info documents on connection <b>conn</b> failed. */ @@ -640,6 +712,33 @@ connection_dir_download_routerdesc_failed(dir_connection_t *conn) (void) conn; } +/** Called when an attempt to download a bridge's routerdesc from + * one of the authorities failed due to a network error. If + * possible attempt to download descriptors from the bridge directly. + */ +static void +connection_dir_bridge_routerdesc_failed(dir_connection_t *conn) +{ + smartlist_t *which = NULL; + + /* Requests for bridge descriptors are in the form 'fp/', so ignore + anything else. */ + if (!conn->requested_resource || strcmpstart(conn->requested_resource,"fp/")) + return; + + which = smartlist_create(); + dir_split_resource_into_fingerprints(conn->requested_resource + + strlen("fp/"), + which, NULL, 0); + + tor_assert(conn->_base.purpose != DIR_PURPOSE_FETCH_EXTRAINFO); + if (smartlist_len(which)) { + connection_dir_retry_bridges(which); + SMARTLIST_FOREACH(which, char *, cp, tor_free(cp)); + } + smartlist_free(which); +} + /** Called when an attempt to fetch a certificate fails. */ static void connection_dir_download_cert_failed(dir_connection_t *conn, int status) @@ -651,7 +750,7 @@ connection_dir_download_cert_failed(dir_connection_t *conn, int status) return; failed = smartlist_create(); dir_split_resource_into_fingerprints(conn->requested_resource+3, - failed, NULL, 1, 0); + failed, NULL, DSR_HEX); SMARTLIST_FOREACH(failed, char *, cp, { authority_cert_dl_failed(cp, status); @@ -667,7 +766,7 @@ connection_dir_download_cert_failed(dir_connection_t *conn, int status) * 1) If or_port is 0, or it's a direct conn and or_port is firewalled * or we're a dir mirror, no. * 2) If we prefer to avoid begindir conns, and we're not fetching or - * publishing a bridge relay descriptor, no. + * publishing a bridge relay descriptor, no. * 3) Else yes. */ static int @@ -746,6 +845,15 @@ directory_initiate_command_rend(const char *address, const tor_addr_t *_addr, log_debug(LD_DIR, "Initiating %s", dir_conn_purpose_to_string(dir_purpose)); + /* ensure that we don't make direct connections when a SOCKS server is + * configured. */ + if (!anonymized_connection && !use_begindir && !options->HTTPProxy && + (options->Socks4Proxy || options->Socks5Proxy)) { + log_warn(LD_DIR, "Cannot connect to a directory server through a " + "SOCKS proxy!"); + return; + } + conn = dir_connection_new(AF_INET); /* set up conn so it's got all the data we need to remember */ @@ -770,9 +878,9 @@ directory_initiate_command_rend(const char *address, const tor_addr_t *_addr, if (!anonymized_connection && !use_begindir) { /* then we want to connect to dirport directly */ - if (options->HttpProxy) { - tor_addr_from_ipv4h(&addr, options->HttpProxyAddr); - dir_port = options->HttpProxyPort; + if (options->HTTPProxy) { + tor_addr_copy(&addr, &options->HTTPProxyAddr); + dir_port = options->HTTPProxyPort; } switch (connection_connect(TO_CONN(conn), conn->_base.address, &addr, @@ -793,7 +901,7 @@ directory_initiate_command_rend(const char *address, const tor_addr_t *_addr, payload, payload_len, supports_conditional_consensus, if_modified_since); - connection_watch_events(TO_CONN(conn), EV_READ | EV_WRITE); + connection_watch_events(TO_CONN(conn), READ_EVENT | WRITE_EVENT); /* writable indicates finish, readable indicates broken link, error indicates broken link in windowsland. */ } @@ -832,7 +940,7 @@ directory_initiate_command_rend(const char *address, const tor_addr_t *_addr, payload, payload_len, supports_conditional_consensus, if_modified_since); - connection_watch_events(TO_CONN(conn), EV_READ | EV_WRITE); + connection_watch_events(TO_CONN(conn), READ_EVENT|WRITE_EVENT); connection_start_reading(TO_CONN(linked_conn)); } } @@ -880,7 +988,7 @@ directory_get_consensus_url(int supports_conditional_consensus) if (supports_conditional_consensus) { char *authority_id_list; - smartlist_t *authority_digets = smartlist_create(); + smartlist_t *authority_digests = smartlist_create(); SMARTLIST_FOREACH(router_get_trusted_dir_servers(), trusted_dir_server_t *, ds, @@ -892,10 +1000,10 @@ directory_get_consensus_url(int supports_conditional_consensus) hex = tor_malloc(2*CONDITIONAL_CONSENSUS_FPR_LEN+1); base16_encode(hex, 2*CONDITIONAL_CONSENSUS_FPR_LEN+1, ds->v3_identity_digest, CONDITIONAL_CONSENSUS_FPR_LEN); - smartlist_add(authority_digets, hex); + smartlist_add(authority_digests, hex); }); - smartlist_sort(authority_digets, _compare_strs); - authority_id_list = smartlist_join_strings(authority_digets, + smartlist_sort(authority_digests, _compare_strs); + authority_id_list = smartlist_join_strings(authority_digests, "+", 0, NULL); len = strlen(authority_id_list)+64; @@ -903,8 +1011,8 @@ directory_get_consensus_url(int supports_conditional_consensus) tor_snprintf(url, len, "/tor/status-vote/current/consensus/%s.z", authority_id_list); - SMARTLIST_FOREACH(authority_digets, char *, cp, tor_free(cp)); - smartlist_free(authority_digets); + SMARTLIST_FOREACH(authority_digests, char *, cp, tor_free(cp)); + smartlist_free(authority_digests); tor_free(authority_id_list); } else { url = tor_strdup("/tor/status-vote/current/consensus.z"); @@ -913,7 +1021,7 @@ directory_get_consensus_url(int supports_conditional_consensus) } /** Queue an appropriate HTTP command on conn-\>outbuf. The other args - * are as in directory_initiate_command. + * are as in directory_initiate_command(). */ static void directory_send_command(dir_connection_t *conn, @@ -956,9 +1064,9 @@ directory_send_command(dir_connection_t *conn, } /* come up with some proxy lines, if we're using one. */ - if (direct && get_options()->HttpProxy) { + if (direct && get_options()->HTTPProxy) { char *base64_authenticator=NULL; - const char *authenticator = get_options()->HttpProxyAuthenticator; + const char *authenticator = get_options()->HTTPProxyAuthenticator; tor_snprintf(proxystring, sizeof(proxystring),"http://%s", hoststring); if (authenticator) { @@ -1049,31 +1157,10 @@ directory_send_command(dir_connection_t *conn, httpcommand = "POST"; url = tor_strdup("/tor/post/consensus-signature"); break; - case DIR_PURPOSE_FETCH_RENDDESC: - tor_assert(resource); - tor_assert(!payload); - - /* this must be true or we wouldn't be doing the lookup */ - tor_assert(strlen(resource) <= REND_SERVICE_ID_LEN_BASE32); - /* This breaks the function abstraction. */ - conn->rend_data = tor_malloc_zero(sizeof(rend_data_t)); - strlcpy(conn->rend_data->onion_address, resource, - sizeof(conn->rend_data->onion_address)); - conn->rend_data->rend_desc_version = 0; - - httpcommand = "GET"; - /* Request the most recent versioned descriptor. */ - // (XXXX We were going to switch this to fetch rendezvous1 descriptors, - // but that never got testing, and it wasn't a good design.) - len = strlen(resource)+32; - url = tor_malloc(len); - tor_snprintf(url, len, "/tor/rendezvous/%s", resource); - break; case DIR_PURPOSE_FETCH_RENDDESC_V2: tor_assert(resource); tor_assert(strlen(resource) <= REND_DESC_ID_V2_LEN_BASE32); tor_assert(!payload); - conn->rend_data->rend_desc_version = 2; httpcommand = "GET"; len = strlen(resource) + 32; url = tor_malloc(len); @@ -1162,7 +1249,7 @@ parse_http_url(const char *headers, char **url) if (s-tmp >= 3 && !strcmpstart(tmp,"://")) { tmp = strchr(tmp+3, '/'); if (tmp && tmp < s) { - log_debug(LD_DIR,"Skipping over 'http[s]://hostname' string"); + log_debug(LD_DIR,"Skipping over 'http[s]://hostname/' string"); start = tmp; } } @@ -1479,7 +1566,7 @@ connection_dir_client_reached_eof(dir_connection_t *conn) } (void) skewed; /* skewed isn't used yet. */ - if (status_code == 503 && body_len < 16) { + if (status_code == 503) { routerstatus_t *rs; trusted_dir_server_t *ds; log_info(LD_DIR,"Received http status code %d (%s) from server " @@ -1493,12 +1580,6 @@ connection_dir_client_reached_eof(dir_connection_t *conn) tor_free(body); tor_free(headers); tor_free(reason); return -1; - } else if (status_code == 503) { - /* XXXX022 Remove this once every server with bug 539 is obsolete. */ - log_info(LD_DIR, "Server at '%s:%d' sent us a 503 response, but included " - "a body anyway. We'll pretend it gave us a 200.", - conn->_base.address, conn->_base.port); - status_code = 200; } plausible = body_is_plausible(body, body_len, conn->_base.purpose); @@ -1565,7 +1646,7 @@ connection_dir_client_reached_eof(dir_connection_t *conn) v2_networkstatus_source_t source; char *cp; log_info(LD_DIR,"Received networkstatus objects (size %d) from server " - "'%s:%d'",(int) body_len, conn->_base.address, conn->_base.port); + "'%s:%d'", (int)body_len, conn->_base.address, conn->_base.port); if (status_code != 200) { log_warn(LD_DIR, "Received http status code %d (%s) from server " @@ -1581,7 +1662,7 @@ connection_dir_client_reached_eof(dir_connection_t *conn) source = NS_FROM_DIR_BY_FP; which = smartlist_create(); dir_split_resource_into_fingerprints(conn->requested_resource+3, - which, NULL, 0, 0); + which, NULL, 0); } else if (conn->requested_resource && !strcmpstart(conn->requested_resource, "all")) { source = NS_FROM_DIR_ALL; @@ -1639,8 +1720,8 @@ connection_dir_client_reached_eof(dir_connection_t *conn) return -1; } log_info(LD_DIR,"Received consensus directory (size %d) from server " - "'%s:%d'",(int) body_len, conn->_base.address, conn->_base.port); - if ((r=networkstatus_set_current_consensus(body, 0))<0) { + "'%s:%d'", (int)body_len, conn->_base.address, conn->_base.port); + if ((r=networkstatus_set_current_consensus(body, "ns", 0))<0) { log_fn(r<-1?LOG_WARN:LOG_INFO, LD_DIR, "Unable to load consensus directory downloaded from " "server '%s:%d'. I'll try again soon.", @@ -1667,9 +1748,11 @@ connection_dir_client_reached_eof(dir_connection_t *conn) return -1; } log_info(LD_DIR,"Received authority certificates (size %d) from server " - "'%s:%d'",(int) body_len, conn->_base.address, conn->_base.port); + "'%s:%d'", (int)body_len, conn->_base.address, conn->_base.port); if (trusted_dirs_load_certs_from_string(body, 0, 1)<0) { log_warn(LD_DIR, "Unable to parse fetched certificates"); + /* if we fetched more than one and only some failed, the successful + * ones got flushed to disk so it's safe to call this on them */ connection_dir_download_cert_failed(conn, status_code); } else { directory_info_has_arrived(now, 0); @@ -1680,7 +1763,7 @@ connection_dir_client_reached_eof(dir_connection_t *conn) const char *msg; int st; log_info(LD_DIR,"Got votes (size %d) from server %s:%d", - (int) body_len, conn->_base.address, conn->_base.port); + (int)body_len, conn->_base.address, conn->_base.port); if (status_code != 200) { log_warn(LD_DIR, "Received http status code %d (%s) from server " @@ -1700,11 +1783,11 @@ connection_dir_client_reached_eof(dir_connection_t *conn) if (conn->_base.purpose == DIR_PURPOSE_FETCH_DETACHED_SIGNATURES) { const char *msg = NULL; log_info(LD_DIR,"Got detached signatures (size %d) from server %s:%d", - (int) body_len, conn->_base.address, conn->_base.port); + (int)body_len, conn->_base.address, conn->_base.port); if (status_code != 200) { log_warn(LD_DIR, - "Received http status code %d (%s) from server " - "'%s:%d' while fetching \"/tor/status-vote/consensus-signatures.z\".", + "Received http status code %d (%s) from server '%s:%d' while fetching " + "\"/tor/status-vote/next/consensus-signatures.z\".", status_code, escaped(reason), conn->_base.address, conn->_base.port); tor_free(body); tor_free(headers); tor_free(reason); @@ -1732,7 +1815,7 @@ connection_dir_client_reached_eof(dir_connection_t *conn) which = smartlist_create(); dir_split_resource_into_fingerprints(conn->requested_resource + (descriptor_digests ? 2 : 3), - which, NULL, 0, 0); + which, NULL, 0); n_asked_for = smartlist_len(which); } if (status_code != 200) { @@ -1814,7 +1897,7 @@ connection_dir_client_reached_eof(dir_connection_t *conn) ds->nickname); /* XXXX use this information; be sure to upload next one * sooner. -NM */ - /* XXXX021 On further thought, the task above implies that we're + /* XXXX023 On further thought, the task above implies that we're * basing our regenerate-descriptor time on when we uploaded the * last descriptor, not on the published time of the last * descriptor. If those are different, that's a bad thing to @@ -1920,7 +2003,7 @@ connection_dir_client_reached_eof(dir_connection_t *conn) /* Success, or at least there's a v2 descriptor already * present. Notify pending connections about this. */ conn->_base.purpose = DIR_PURPOSE_HAS_FETCHED_RENDDESC; - rend_client_desc_trynow(conn->rend_data->onion_address, -1); + rend_client_desc_trynow(conn->rend_data->onion_address); } break; case 404: @@ -1967,7 +2050,7 @@ connection_dir_client_reached_eof(dir_connection_t *conn) log_info(LD_REND, "Successfully fetched v2 rendezvous " "descriptor."); conn->_base.purpose = DIR_PURPOSE_HAS_FETCHED_RENDDESC; - rend_client_desc_trynow(conn->rend_data->onion_address, -1); + rend_client_desc_trynow(conn->rend_data->onion_address); break; } break; @@ -2010,12 +2093,6 @@ connection_dir_client_reached_eof(dir_connection_t *conn) "'%s:%d'. Malformed rendezvous descriptor?", escaped(reason), conn->_base.address, conn->_base.port); break; - case 503: - log_info(LD_REND,"http status 503 (%s) response from dirserver " - "'%s:%d'. Node is (currently) not acting as v2 hidden " - "service directory.", - escaped(reason), conn->_base.address, conn->_base.port); - break; default: log_warn(LD_REND,"http status %d (%s) response unexpected (server " "'%s:%d').", @@ -2322,7 +2399,7 @@ directory_dump_request_log(void) } #endif -/** Decide whether a client would accept the consensus we have +/** Decide whether a client would accept the consensus we have. * * Clients can say they only want a consensus if it's signed by more * than half the authorities in a list. They pass this list in @@ -2343,31 +2420,32 @@ client_likes_consensus(networkstatus_t *v, const char *want_url) int need_at_least; int have = 0; - dir_split_resource_into_fingerprints(want_url, want_authorities, NULL, 0, 0); + dir_split_resource_into_fingerprints(want_url, want_authorities, NULL, 0); need_at_least = smartlist_len(want_authorities)/2+1; - SMARTLIST_FOREACH(want_authorities, const char *, d, { + SMARTLIST_FOREACH_BEGIN(want_authorities, const char *, d) { char want_digest[DIGEST_LEN]; size_t want_len = strlen(d)/2; if (want_len > DIGEST_LEN) want_len = DIGEST_LEN; if (base16_decode(want_digest, DIGEST_LEN, d, want_len*2) < 0) { - log_warn(LD_DIR,"Failed to decode requested authority digest %s.", d); + log_fn(LOG_PROTOCOL_WARN, LD_DIR, + "Failed to decode requested authority digest %s.", d); continue; }; - SMARTLIST_FOREACH(v->voters, networkstatus_voter_info_t *, vi, { - if (vi->signature && - fast_memeq(vi->identity_digest, want_digest, want_len)) { + SMARTLIST_FOREACH_BEGIN(v->voters, networkstatus_voter_info_t *, vi) { + if (smartlist_len(vi->sigs) && + tor_memeq(vi->identity_digest, want_digest, want_len)) { have++; break; }; - }); + } SMARTLIST_FOREACH_END(vi); /* early exit, if we already have enough */ if (have >= need_at_least) break; - }); + } SMARTLIST_FOREACH_END(d); SMARTLIST_FOREACH(want_authorities, char *, d, tor_free(d)); smartlist_free(want_authorities); @@ -2514,9 +2592,12 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, /* v2 or v3 network status fetch. */ smartlist_t *dir_fps = smartlist_create(); int is_v3 = !strcmpstart(url, "/tor/status-vote"); + geoip_client_action_t act = + is_v3 ? GEOIP_CLIENT_NETWORKSTATUS : GEOIP_CLIENT_NETWORKSTATUS_V2; const char *request_type = NULL; const char *key = url + strlen("/tor/status/"); long lifetime = NETWORKSTATUS_CACHE_LIFETIME; + if (!is_v3) { dirserv_get_networkstatus_v2_fingerprints(dir_fps, key); if (!strcmpstart(key, "fp/")) @@ -2531,18 +2612,44 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, } else { networkstatus_t *v = networkstatus_get_latest_consensus(); time_t now = time(NULL); + const char *want_fps = NULL; + char *flavor = NULL; #define CONSENSUS_URL_PREFIX "/tor/status-vote/current/consensus/" - if (v && - !strcmpstart(url, CONSENSUS_URL_PREFIX) && - !client_likes_consensus(v, url + strlen(CONSENSUS_URL_PREFIX))) { + #define CONSENSUS_FLAVORED_PREFIX "/tor/status-vote/current/consensus-" + /* figure out the flavor if any, and who we wanted to sign the thing */ + if (!strcmpstart(url, CONSENSUS_FLAVORED_PREFIX)) { + const char *f, *cp; + f = url + strlen(CONSENSUS_FLAVORED_PREFIX); + cp = strchr(f, '/'); + if (cp) { + want_fps = cp+1; + flavor = tor_strndup(f, cp-f); + } else { + flavor = tor_strdup(f); + } + } else { + if (!strcmpstart(url, CONSENSUS_URL_PREFIX)) + want_fps = url+strlen(CONSENSUS_URL_PREFIX); + } + + /* XXXX MICRODESC NM NM should check document of correct flavor */ + if (v && want_fps && + !client_likes_consensus(v, want_fps)) { write_http_status_line(conn, 404, "Consensus not signed by sufficient " "number of requested authorities"); smartlist_free(dir_fps); + geoip_note_ns_response(act, GEOIP_REJECT_NOT_ENOUGH_SIGS); + tor_free(flavor); goto done; } - smartlist_add(dir_fps, tor_memdup("\0\0\0\0\0\0\0\0\0\0" - "\0\0\0\0\0\0\0\0\0\0", 20)); + { + char *fp = tor_malloc_zero(DIGEST_LEN); + if (flavor) + strlcpy(fp, flavor, DIGEST_LEN); + tor_free(flavor); + smartlist_add(dir_fps, fp); + } request_type = compressed?"v3.z":"v3"; lifetime = (v && v->fresh_until > now) ? v->fresh_until - now : 0; } @@ -2550,6 +2657,7 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, if (!smartlist_len(dir_fps)) { /* we failed to create/cache cp */ write_http_status_line(conn, 503, "Network status object unavailable"); smartlist_free(dir_fps); + geoip_note_ns_response(act, GEOIP_REJECT_UNAVAILABLE); goto done; } @@ -2557,11 +2665,13 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, write_http_status_line(conn, 404, "Not found"); SMARTLIST_FOREACH(dir_fps, char *, cp, tor_free(cp)); smartlist_free(dir_fps); + geoip_note_ns_response(act, GEOIP_REJECT_NOT_FOUND); goto done; } else if (!smartlist_len(dir_fps)) { write_http_status_line(conn, 304, "Not modified"); SMARTLIST_FOREACH(dir_fps, char *, cp, tor_free(cp)); smartlist_free(dir_fps); + geoip_note_ns_response(act, GEOIP_REJECT_NOT_MODIFIED); goto done; } @@ -2573,18 +2683,25 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, write_http_status_line(conn, 503, "Directory busy, try again later"); SMARTLIST_FOREACH(dir_fps, char *, fp, tor_free(fp)); smartlist_free(dir_fps); + geoip_note_ns_response(act, GEOIP_REJECT_BUSY); goto done; } -#ifdef ENABLE_GEOIP_STATS { - geoip_client_action_t act = - is_v3 ? GEOIP_CLIENT_NETWORKSTATUS : GEOIP_CLIENT_NETWORKSTATUS_V2; struct in_addr in; - if (tor_inet_aton((TO_CONN(conn))->address, &in)) + if (tor_inet_aton((TO_CONN(conn))->address, &in)) { geoip_note_client_seen(act, ntohl(in.s_addr), time(NULL)); + geoip_note_ns_response(act, GEOIP_SUCCESS); + /* Note that a request for a network status has started, so that we + * can measure the download time later on. */ + if (TO_CONN(conn)->dirreq_id) + geoip_start_dirreq(TO_CONN(conn)->dirreq_id, dlen, act, + DIRREQ_TUNNELED); + else + geoip_start_dirreq(TO_CONN(conn)->global_identifier, dlen, act, + DIRREQ_DIRECT); + } } -#endif // note_request(request_type,dlen); (void) request_type; @@ -2610,7 +2727,7 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, ssize_t estimated_len = 0; smartlist_t *items = smartlist_create(); smartlist_t *dir_items = smartlist_create(); - int lifetime = 60; /* XXXX022 should actually use vote intervals. */ + int lifetime = 60; /* XXXX023 should actually use vote intervals. */ url += strlen("/tor/status-vote/"); current = !strcmpstart(url, "current/"); url = strchr(url, '/'); @@ -2620,7 +2737,7 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, const char *item; tor_assert(!current); /* we handle current consensus specially above, * since it wants to be spooled. */ - if ((item = dirvote_get_pending_consensus())) + if ((item = dirvote_get_pending_consensus(FLAV_NS))) smartlist_add(items, (char*)item); } else if (!current && !strcmp(url, "consensus-signatures")) { /* XXXX the spec says that we should implement @@ -2646,7 +2763,8 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, flags = DGV_BY_ID | (current ? DGV_INCLUDE_PREVIOUS : DGV_INCLUDE_PENDING); } - dir_split_resource_into_fingerprints(url, fps, NULL, 1, 1); + dir_split_resource_into_fingerprints(url, fps, NULL, + DSR_HEX|DSR_SORT_UNIQ); SMARTLIST_FOREACH(fps, char *, fp, { if ((d = dirvote_get_vote(fp, flags))) smartlist_add(dir_items, (cached_dir_t*)d); @@ -2699,6 +2817,41 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, goto done; } + if (!strcmpstart(url, "/tor/micro/d/")) { + smartlist_t *fps = smartlist_create(); + + dir_split_resource_into_fingerprints(url+strlen("/tor/micro/d/"), + fps, NULL, + DSR_DIGEST256|DSR_BASE64|DSR_SORT_UNIQ); + + if (!dirserv_have_any_microdesc(fps)) { + write_http_status_line(conn, 404, "Not found"); + SMARTLIST_FOREACH(fps, char *, fp, tor_free(fp)); + smartlist_free(fps); + goto done; + } + dlen = dirserv_estimate_microdesc_size(fps, compressed); + if (global_write_bucket_low(TO_CONN(conn), dlen, 2)) { + log_info(LD_DIRSERV, + "Client asked for server descriptors, but we've been " + "writing too many bytes lately. Sending 503 Dir busy."); + write_http_status_line(conn, 503, "Directory busy, try again later"); + SMARTLIST_FOREACH(fps, char *, fp, tor_free(fp)); + smartlist_free(fps); + goto done; + } + + write_http_response_header(conn, -1, compressed, MICRODESC_CACHE_LIFETIME); + conn->dir_spool_src = DIR_SPOOL_MICRODESC; + conn->fingerprint_stack = fps; + + if (compressed) + conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD); + + connection_dirserv_flushed_some(conn); + goto done; + } + if (!strcmpstart(url,"/tor/server/") || (!options->BridgeAuthoritativeDir && !options->BridgeRelay && !strcmpstart(url,"/tor/extra/"))) { @@ -2780,7 +2933,8 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, } else if (!strcmpstart(url, "/tor/keys/fp/")) { smartlist_t *fps = smartlist_create(); dir_split_resource_into_fingerprints(url+strlen("/tor/keys/fp/"), - fps, NULL, 1, 1); + fps, NULL, + DSR_HEX|DSR_SORT_UNIQ); SMARTLIST_FOREACH(fps, char *, d, { authority_cert_t *c = authority_cert_get_newest_by_id(d); if (c) smartlist_add(certs, c); @@ -2790,7 +2944,8 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, } else if (!strcmpstart(url, "/tor/keys/sk/")) { smartlist_t *fps = smartlist_create(); dir_split_resource_into_fingerprints(url+strlen("/tor/keys/sk/"), - fps, NULL, 1, 1); + fps, NULL, + DSR_HEX|DSR_SORT_UNIQ); SMARTLIST_FOREACH(fps, char *, d, { authority_cert_t *c = authority_cert_get_by_sk_digest(d); if (c) smartlist_add(certs, c); @@ -2892,18 +3047,9 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, note_request("/tor/rendezvous?/", desc_len); /* need to send descp separately, because it may include NULs */ connection_write_to_buf(descp, desc_len, TO_CONN(conn)); - /* report successful fetch to statistic */ - if (options->HSAuthorityRecordStats) { - hs_usage_note_fetch_total(query, time(NULL)); - hs_usage_note_fetch_successful(query, time(NULL)); - } break; case 0: /* well-formed but not present */ write_http_status_line(conn, 404, "Not found"); - /* report (unsuccessful) fetch to statistic */ - if (options->HSAuthorityRecordStats) { - hs_usage_note_fetch_total(query, time(NULL)); - } break; case -1: /* not well-formed */ write_http_status_line(conn, 400, "Bad request"); @@ -3134,6 +3280,8 @@ directory_handle_command_post(dir_connection_t *conn, const char *headers, write_http_status_line(conn, status, "Vote stored"); } else { tor_assert(msg); + log_warn(LD_DIRSERV, "Rejected vote from %s (\"%s\").", + conn->_base.address, msg); write_http_status_line(conn, status, msg); } goto done; @@ -3180,8 +3328,8 @@ directory_handle_command(dir_connection_t *conn) &body, &body_len, MAX_DIR_UL_SIZE, 0)) { case -1: /* overflow */ log_warn(LD_DIRSERV, - "Invalid input from address '%s'. Closing.", - conn->_base.address); + "Request too large from address '%s' to DirPort. Closing.", + safe_str(conn->_base.address)); return -1; case 0: log_debug(LD_DIRSERV,"command not all here yet."); @@ -3217,6 +3365,16 @@ connection_dir_finished_flushing(dir_connection_t *conn) tor_assert(conn); tor_assert(conn->_base.type == CONN_TYPE_DIR); + /* Note that we have finished writing the directory response. For direct + * connections this means we're done, for tunneled connections its only + * an intermediate step. */ + if (TO_CONN(conn)->dirreq_id) + geoip_change_dirreq_state(TO_CONN(conn)->dirreq_id, DIRREQ_TUNNELED, + DIRREQ_FLUSHING_DIR_CONN_FINISHED); + else + geoip_change_dirreq_state(TO_CONN(conn)->global_identifier, + DIRREQ_DIRECT, + DIRREQ_FLUSHING_DIR_CONN_FINISHED); switch (conn->_base.state) { case DIR_CONN_STATE_CLIENT_SENDING: log_debug(LD_DIR,"client finished sending command."); @@ -3399,6 +3557,14 @@ download_status_reset(download_status_t *dls) dls->next_attempt_at = time(NULL) + schedule[0]; } +/** Return the number of failures on <b>dls</b> since the last success (if + * any). */ +int +download_status_get_n_failures(const download_status_t *dls) +{ + return dls->n_download_failures; +} + /** Called when one or more routerdesc (or extrainfo, if <b>was_extrainfo</b>) * fetches have failed (with uppercase fingerprints listed in <b>failed</b>, * either as descriptor digests or as identity digests based on @@ -3414,16 +3580,8 @@ dir_routerdesc_download_failed(smartlist_t *failed, int status_code, int server = directory_fetches_from_authorities(get_options()); if (!was_descriptor_digests) { if (router_purpose == ROUTER_PURPOSE_BRIDGE) { - tor_assert(!was_extrainfo); /* not supported yet */ - SMARTLIST_FOREACH(failed, const char *, cp, - { - if (base16_decode(digest, DIGEST_LEN, cp, strlen(cp))<0) { - log_warn(LD_BUG, "Malformed fingerprint in list: %s", - escaped(cp)); - continue; - } - retry_bridge_descriptor_fetch_directly(digest); - }); + tor_assert(!was_extrainfo); + connection_dir_retry_bridges(failed); } return; /* FFFF should implement for other-than-router-purpose someday */ } @@ -3516,19 +3674,37 @@ dir_split_resource_into_fingerprint_pairs(const char *res, /** Given a directory <b>resource</b> request, containing zero * or more strings separated by plus signs, followed optionally by ".z", store * the strings, in order, into <b>fp_out</b>. If <b>compressed_out</b> is - * non-NULL, set it to 1 if the resource ends in ".z", else set it to 0. If - * decode_hex is true, then delete all elements that aren't hex digests, and - * decode the rest. If sort_uniq is true, then sort the list and remove - * all duplicates. + * non-NULL, set it to 1 if the resource ends in ".z", else set it to 0. + * + * If (flags & DSR_HEX), then delete all elements that aren't hex digests, and + * decode the rest. If (flags & DSR_BASE64), then use "-" rather than "+" as + * a separator, delete all the elements that aren't base64-encoded digests, + * and decode the rest. If (flags & DSR_DIGEST256), these digests should be + * 256 bits long; else they should be 160. + * + * If (flags & DSR_SORT_UNIQ), then sort the list and remove all duplicates. */ int dir_split_resource_into_fingerprints(const char *resource, smartlist_t *fp_out, int *compressed_out, - int decode_hex, int sort_uniq) + int flags) { + const int decode_hex = flags & DSR_HEX; + const int decode_base64 = flags & DSR_BASE64; + const int digests_are_256 = flags & DSR_DIGEST256; + const int sort_uniq = flags & DSR_SORT_UNIQ; + + const int digest_len = digests_are_256 ? DIGEST256_LEN : DIGEST_LEN; + const int hex_digest_len = digests_are_256 ? + HEX_DIGEST256_LEN : HEX_DIGEST_LEN; + const int base64_digest_len = digests_are_256 ? + BASE64_DIGEST256_LEN : BASE64_DIGEST_LEN; smartlist_t *fp_tmp = smartlist_create(); + + tor_assert(!(decode_hex && decode_base64)); tor_assert(fp_out); - smartlist_split_string(fp_tmp, resource, "+", 0, 0); + + smartlist_split_string(fp_tmp, resource, decode_base64?"-":"+", 0, 0); if (compressed_out) *compressed_out = 0; if (smartlist_len(fp_tmp)) { @@ -3540,22 +3716,25 @@ dir_split_resource_into_fingerprints(const char *resource, *compressed_out = 1; } } - if (decode_hex) { + if (decode_hex || decode_base64) { + const size_t encoded_len = decode_hex ? hex_digest_len : base64_digest_len; int i; char *cp, *d = NULL; for (i = 0; i < smartlist_len(fp_tmp); ++i) { cp = smartlist_get(fp_tmp, i); - if (strlen(cp) != HEX_DIGEST_LEN) { + if (strlen(cp) != encoded_len) { log_info(LD_DIR, "Skipping digest %s with non-standard length.", escaped(cp)); smartlist_del_keeporder(fp_tmp, i--); goto again; } - d = tor_malloc_zero(DIGEST_LEN); - if (base16_decode(d, DIGEST_LEN, cp, HEX_DIGEST_LEN)<0) { - log_info(LD_DIR, "Skipping non-decodable digest %s", escaped(cp)); - smartlist_del_keeporder(fp_tmp, i--); - goto again; + d = tor_malloc_zero(digest_len); + if (decode_hex ? + (base16_decode(d, digest_len, cp, hex_digest_len)<0) : + (base64_decode(d, digest_len, cp, base64_digest_len)<0)) { + log_info(LD_DIR, "Skipping non-decodable digest %s", escaped(cp)); + smartlist_del_keeporder(fp_tmp, i--); + goto again; } smartlist_set(fp_tmp, i, d); d = NULL; @@ -3565,26 +3744,18 @@ dir_split_resource_into_fingerprints(const char *resource, } } if (sort_uniq) { - smartlist_t *fp_tmp2 = smartlist_create(); - int i; - if (decode_hex) - smartlist_sort_digests(fp_tmp); - else + if (decode_hex || decode_base64) { + if (digests_are_256) { + smartlist_sort_digests256(fp_tmp); + smartlist_uniq_digests256(fp_tmp); + } else { + smartlist_sort_digests(fp_tmp); + smartlist_uniq_digests(fp_tmp); + } + } else { smartlist_sort_strings(fp_tmp); - if (smartlist_len(fp_tmp)) - smartlist_add(fp_tmp2, smartlist_get(fp_tmp, 0)); - for (i = 1; i < smartlist_len(fp_tmp); ++i) { - char *cp = smartlist_get(fp_tmp, i); - char *last = smartlist_get(fp_tmp2, smartlist_len(fp_tmp2)-1); - - if ((decode_hex && fast_memcmp(cp, last, DIGEST_LEN)) - || (!decode_hex && strcasecmp(cp, last))) - smartlist_add(fp_tmp2, cp); - else - tor_free(cp); + smartlist_uniq_strings(fp_tmp); } - smartlist_free(fp_tmp); - fp_tmp = fp_tmp2; } smartlist_add_all(fp_out, fp_tmp); smartlist_free(fp_tmp); diff --git a/src/or/directory.h b/src/or/directory.h new file mode 100644 index 0000000000..94dfb17644 --- /dev/null +++ b/src/or/directory.h @@ -0,0 +1,110 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file directory.h + * \brief Header file for directory.c. + **/ + +#ifndef _TOR_DIRECTORY_H +#define _TOR_DIRECTORY_H + +int directories_have_accepted_server_descriptor(void); +char *authority_type_to_string(authority_type_t auth); +void directory_post_to_dirservers(uint8_t dir_purpose, uint8_t router_purpose, + authority_type_t type, const char *payload, + size_t payload_len, size_t extrainfo_len); +void directory_get_from_dirserver(uint8_t dir_purpose, uint8_t router_purpose, + const char *resource, + int pds_flags); +void directory_get_from_all_authorities(uint8_t dir_purpose, + uint8_t router_purpose, + const char *resource); +void directory_initiate_command_routerstatus(routerstatus_t *status, + uint8_t dir_purpose, + uint8_t router_purpose, + int anonymized_connection, + const char *resource, + const char *payload, + size_t payload_len, + time_t if_modified_since); +void directory_initiate_command_routerstatus_rend(routerstatus_t *status, + uint8_t dir_purpose, + uint8_t router_purpose, + int anonymized_connection, + const char *resource, + const char *payload, + size_t payload_len, + time_t if_modified_since, + const rend_data_t *rend_query); + +int parse_http_response(const char *headers, int *code, time_t *date, + compress_method_t *compression, char **response); + +int connection_dir_is_encrypted(dir_connection_t *conn); +int connection_dir_reached_eof(dir_connection_t *conn); +int connection_dir_process_inbuf(dir_connection_t *conn); +int connection_dir_finished_flushing(dir_connection_t *conn); +int connection_dir_finished_connecting(dir_connection_t *conn); +void connection_dir_request_failed(dir_connection_t *conn); +void directory_initiate_command(const char *address, const tor_addr_t *addr, + uint16_t or_port, uint16_t dir_port, + int supports_conditional_consensus, + int supports_begindir, const char *digest, + uint8_t dir_purpose, uint8_t router_purpose, + int anonymized_connection, + const char *resource, + const char *payload, size_t payload_len, + time_t if_modified_since); + +#define DSR_HEX (1<<0) +#define DSR_BASE64 (1<<1) +#define DSR_DIGEST256 (1<<2) +#define DSR_SORT_UNIQ (1<<3) +int dir_split_resource_into_fingerprints(const char *resource, + smartlist_t *fp_out, int *compressed_out, + int flags); + +int dir_split_resource_into_fingerprint_pairs(const char *res, + smartlist_t *pairs_out); +char *directory_dump_request_log(void); +void note_request(const char *key, size_t bytes); +int router_supports_extrainfo(const char *identity_digest, int is_authority); + +time_t download_status_increment_failure(download_status_t *dls, + int status_code, const char *item, + int server, time_t now); +/** Increment the failure count of the download_status_t <b>dls</b>, with + * the optional status code <b>sc</b>. */ +#define download_status_failed(dls, sc) \ + download_status_increment_failure((dls), (sc), NULL, \ + get_options()->DirPort, time(NULL)) + +void download_status_reset(download_status_t *dls); +static int download_status_is_ready(download_status_t *dls, time_t now, + int max_failures); +/** Return true iff, as of <b>now</b>, the resource tracked by <b>dls</b> is + * ready to get its download reattempted. */ +static INLINE int +download_status_is_ready(download_status_t *dls, time_t now, + int max_failures) +{ + return (dls->n_download_failures <= max_failures + && dls->next_attempt_at <= now); +} + +static void download_status_mark_impossible(download_status_t *dl); +/** Mark <b>dl</b> as never downloadable. */ +static INLINE void +download_status_mark_impossible(download_status_t *dl) +{ + dl->n_download_failures = IMPOSSIBLE_TO_DOWNLOAD; +} + +int download_status_get_n_failures(const download_status_t *dls); + +#endif + diff --git a/src/or/dirserv.c b/src/or/dirserv.c index e367cb1c3a..1f4489a7e0 100644 --- a/src/or/dirserv.c +++ b/src/or/dirserv.c @@ -5,6 +5,22 @@ #define DIRSERV_PRIVATE #include "or.h" +#include "buffers.h" +#include "config.h" +#include "connection.h" +#include "connection_or.h" +#include "control.h" +#include "directory.h" +#include "dirserv.h" +#include "dirvote.h" +#include "hibernate.h" +#include "microdesc.h" +#include "networkstatus.h" +#include "policies.h" +#include "rephist.h" +#include "router.h" +#include "routerlist.h" +#include "routerparse.h" /** * \file dirserv.c @@ -27,6 +43,8 @@ extern time_t time_of_process_start; /* from main.c */ +extern long stats_n_seconds_working; /* from main.c */ + /** Do we need to regenerate the v1 directory when someone asks for it? */ static time_t the_directory_is_dirty = 1; /** Do we need to regenerate the v1 runningrouters document when somebody @@ -41,7 +59,7 @@ static time_t the_v2_networkstatus_is_dirty = 1; static cached_dir_t *the_directory = NULL; /** For authoritative directories: the current (v1) network status. */ -static cached_dir_t the_runningrouters = { NULL, NULL, 0, 0, 0, -1 }; +static cached_dir_t the_runningrouters; static void directory_remove_invalid(void); static cached_dir_t *dirserv_regenerate_directory(void); @@ -63,13 +81,16 @@ static signed_descriptor_t *get_signed_descriptor_by_fp(const char *fp, time_t publish_cutoff); static int dirserv_add_extrainfo(extrainfo_t *ei, const char **msg); +/************** Measured Bandwidth parsing code ******/ +#define MAX_MEASUREMENT_AGE (3*24*60*60) /* 3 days */ + /************** Fingerprint handling code ************/ #define FP_NAMED 1 /**< Listed in fingerprint file. */ #define FP_INVALID 2 /**< Believed invalid. */ #define FP_REJECT 4 /**< We will not publish this router. */ #define FP_BADDIR 8 /**< We'll tell clients to avoid using this as a dir. */ -#define FP_BADEXIT 16 /**< We'll tell clients not to use this as an exit. */ +#define FP_BADEXIT 16 /**< We'll tell clients not to use this as an exit. */ #define FP_UNNAMED 32 /**< Another router has this name in fingerprint file. */ /** Encapsulate a nickname and an FP_* status; target of status_by_digest @@ -99,7 +120,7 @@ authdir_config_new(void) return list; } -/** Add the fingerprint <b>fp</b> for the nickname <b>nickname</b> to +/** Add the fingerprint <b>fp</b> for <b>nickname</b> to * the smartlist of fingerprint_entry_t's <b>list</b>. Return 0 if it's * new, or 1 if we replaced the old value. */ @@ -181,8 +202,7 @@ dirserv_add_own_fingerprint(const char *nickname, crypto_pk_env_t *pk) * file. The file format is line-based, with each non-blank holding one * nickname, some space, and a fingerprint for that nickname. On success, * replace the current fingerprint list with the new list and return 0. On - * failure, leave the current fingerprint list untouched, and - * return -1. */ + * failure, leave the current fingerprint list untouched, and return -1. */ int dirserv_load_fingerprint_file(void) { @@ -368,13 +388,19 @@ dirserv_get_status_impl(const char *id_digest, const char *nickname, strmap_size(fingerprint_list->fp_by_name), digestmap_size(fingerprint_list->status_by_digest)); - /* 0.1.1.17-rc was the first version that claimed to be stable, doesn't - * crash and drop circuits all the time, and is even vaguely compatible with - * the current network */ - if (platform && !tor_version_as_new_as(platform,"0.1.1.17-rc")) { + /* Tor 0.2.0.26-rc is the oldest version that currently caches the right + * directory information. Once more of them die off, we should raise this + * minimum. */ + if (platform && !tor_version_as_new_as(platform,"0.2.0.26-rc")) { if (msg) *msg = "Tor version is far too old to work."; return FP_REJECT; + } else if (platform && tor_version_as_new_as(platform,"0.2.1.3-alpha") + && !tor_version_as_new_as(platform, "0.2.1.19")) { + /* These versions mishandled RELAY_EARLY cells on rend circuits. */ + if (msg) + *msg = "Tor version is too buggy to work."; + return FP_REJECT; } result = dirserv_get_name_status(id_digest, nickname); @@ -520,7 +546,7 @@ authdir_wants_to_reject_router(routerinfo_t *ri, const char **msg, /* Okay. Now check whether the fingerprint is recognized. */ uint32_t status = dirserv_router_get_status(ri, msg); time_t now; - int severity = complain ? LOG_NOTICE : LOG_INFO; + int severity = (complain && ri->contact_info) ? LOG_NOTICE : LOG_INFO; tor_assert(msg); if (status & FP_REJECT) return -1; /* msg is already set. */ @@ -712,6 +738,10 @@ dirserv_add_descriptor(routerinfo_t *ri, const char **msg, const char *source) desc = tor_strndup(ri->cache_info.signed_descriptor_body, desclen); nickname = tor_strdup(ri->nickname); + /* Tell if we're about to need to launch a test if we add this. */ + ri->needs_retest_if_added = + dirserv_should_launch_reachability_test(ri, ri_old); + r = router_add_to_routerlist(ri, msg, 0, 0); if (!WRA_WAS_ADDED(r)) { /* unless the routerinfo was fine, just out-of-date */ @@ -726,7 +756,7 @@ dirserv_add_descriptor(routerinfo_t *ri, const char **msg, const char *source) changed = smartlist_create(); smartlist_add(changed, ri); - control_event_descriptors_changed(changed); + routerlist_descriptors_added(changed, 0); smartlist_free(changed); if (!*msg) { *msg = ri->is_valid ? "Descriptor for valid server accepted" : @@ -835,46 +865,6 @@ directory_remove_invalid(void) routerlist_assert_ok(rl); } -/** Write a list of unregistered descriptors into a newly allocated - * string and return it. Used by dirserv operators to keep track of - * fast nodes that haven't registered. - */ -int -getinfo_helper_dirserv_unregistered(control_connection_t *control_conn, - const char *question, char **answer_out) -{ - smartlist_t *answerlist; - char buf[1024]; - char *answer; - int min_bw = atoi(question); - routerlist_t *rl = router_get_routerlist(); - - (void) control_conn; - - if (strcmpstart(question, "unregistered-servers-")) - return 0; - question += strlen("unregistered-servers-"); - - answerlist = smartlist_create(); - SMARTLIST_FOREACH(rl->routers, routerinfo_t *, ent, { - uint32_t r = dirserv_router_get_status(ent, NULL); - if (router_get_advertised_bandwidth(ent) >= (size_t)min_bw && - !(r & FP_NAMED)) { - /* then log this one */ - tor_snprintf(buf, sizeof(buf), - "%s: BW %d on '%s'.", - ent->nickname, router_get_advertised_bandwidth(ent), - ent->platform ? ent->platform : ""); - smartlist_add(answerlist, tor_strdup(buf)); - } - }); - answer = smartlist_join_strings(answerlist, "\r\n", 0, NULL); - SMARTLIST_FOREACH(answerlist, char *, cp, tor_free(cp)); - smartlist_free(answerlist); - *answer_out = answer; - return 0; -} - /** Mark the directory as <b>dirty</b> -- when we're next asked for a * directory, we will rebuild it instead of reusing the most recently * generated one. @@ -933,28 +923,66 @@ list_single_server_status(routerinfo_t *desc, int is_live) return tor_strdup(buf); } +static INLINE int +running_long_enough_to_decide_unreachable(void) +{ + return time_of_process_start + + get_options()->TestingAuthDirTimeToLearnReachability < approx_time(); +} + /** Each server needs to have passed a reachability test no more * than this number of seconds ago, or he is listed as down in * the directory. */ #define REACHABLE_TIMEOUT (45*60) +/** If we tested a router and found it reachable _at least this long_ after it + * declared itself hibernating, it is probably done hibernating and we just + * missed a descriptor from it. */ +#define HIBERNATION_PUBLICATION_SKEW (60*60) + /** Treat a router as alive if * - It's me, and I'm not hibernating. * or - We've found it reachable recently. */ void dirserv_set_router_is_running(routerinfo_t *router, time_t now) { + /*XXXX023 This function is a mess. Separate out the part that calculates + whether it's reachable and the part that tells rephist that the router was + unreachable. + */ int answer; - if (router_is_me(router) && !we_are_hibernating()) + if (router_is_me(router)) { + /* We always know if we are down ourselves. */ + answer = ! we_are_hibernating(); + } else if (router->is_hibernating && + (router->cache_info.published_on + + HIBERNATION_PUBLICATION_SKEW) > router->last_reachable) { + /* A hibernating router is down unless we (somehow) had contact with it + * since it declared itself to be hibernating. */ + answer = 0; + } else if (get_options()->AssumeReachable) { + /* If AssumeReachable, everybody is up unless they say they are down! */ answer = 1; - else - answer = get_options()->AssumeReachable || - now < router->last_reachable + REACHABLE_TIMEOUT; + } else { + /* Otherwise, a router counts as up if we found it reachable in the last + REACHABLE_TIMEOUT seconds. */ + answer = (now < router->last_reachable + REACHABLE_TIMEOUT); + } + + if (!answer && running_long_enough_to_decide_unreachable()) { + /* Not considered reachable. tell rephist about that. - if (!answer) { - /* not considered reachable. tell rephist. */ - rep_hist_note_router_unreachable(router->cache_info.identity_digest, now); + Because we launch a reachability test for each router every + REACHABILITY_TEST_CYCLE_PERIOD seconds, then the router has probably + been down since at least that time after we last successfully reached + it. + */ + time_t when = now; + if (router->last_reachable && + router->last_reachable + REACHABILITY_TEST_CYCLE_PERIOD < now) + when = router->last_reachable + REACHABILITY_TEST_CYCLE_PERIOD; + rep_hist_note_router_unreachable(router->cache_info.identity_digest, when); } router->is_running = answer; @@ -965,7 +993,6 @@ dirserv_set_router_is_running(routerinfo_t *router, time_t now) * *<b>router_status_out</b>. Return 0 on success, -1 on failure. * * If for_controller is true, include the routers with very old descriptors. - * If for_controller is >1, use the verbose nickname format. */ int list_server_status_v1(smartlist_t *routers, char **router_status_out, @@ -985,23 +1012,22 @@ list_server_status_v1(smartlist_t *routers, char **router_status_out, rs_entries = smartlist_create(); - SMARTLIST_FOREACH(routers, routerinfo_t *, ri, - { + SMARTLIST_FOREACH_BEGIN(routers, routerinfo_t *, ri) { if (authdir) { /* Update router status in routerinfo_t. */ dirserv_set_router_is_running(ri, now); } - if (for_controller == 1 || ri->cache_info.published_on >= cutoff) - smartlist_add(rs_entries, list_single_server_status(ri, ri->is_running)); - else if (for_controller > 2) { + if (for_controller) { char name_buf[MAX_VERBOSE_NICKNAME_LEN+2]; char *cp = name_buf; if (!ri->is_running) *cp++ = '!'; router_get_verbose_nickname(cp, ri); smartlist_add(rs_entries, tor_strdup(name_buf)); + } else if (ri->cache_info.published_on >= cutoff) { + smartlist_add(rs_entries, list_single_server_status(ri, ri->is_running)); } - }); + } SMARTLIST_FOREACH_END(ri); *router_status_out = smartlist_join_strings(rs_entries, " ", 0, NULL); @@ -1119,7 +1145,8 @@ dirserv_dump_directory_to_string(char **dir_out, return -1; } note_crypto_pk_op(SIGN_DIR); - if (router_append_dirobj_signature(buf,buf_len,digest,private_key)<0) { + if (router_append_dirobj_signature(buf,buf_len,digest,DIGEST_LEN, + private_key)<0) { tor_free(buf); return -1; } @@ -1144,18 +1171,21 @@ directory_fetches_from_authorities(or_options_t *options) { routerinfo_t *me; uint32_t addr; + int refuseunknown; if (options->FetchDirInfoEarly) return 1; if (options->BridgeRelay == 1) return 0; if (server_mode(options) && router_pick_published_address(options, &addr)<0) return 1; /* we don't know our IP address; ask an authority. */ - if (options->DirPort == 0) + refuseunknown = ! router_my_exit_policy_is_reject_star() && + should_refuse_unknown_exits(options); + if (options->DirPort == 0 && !refuseunknown) return 0; if (!server_mode(options) || !advertised_server_mode()) return 0; me = router_get_my_routerinfo(); - if (!me || !me->dir_port) + if (!me || (!me->dir_port && !refuseunknown)) return 0; /* if dirport not advertised, return 0 too */ return 1; } @@ -1195,7 +1225,14 @@ directory_caches_v2_dir_info(or_options_t *options) int directory_caches_dir_info(or_options_t *options) { - return options->BridgeRelay != 0 || options->DirPort != 0; + if (options->BridgeRelay || options->DirPort) + return 1; + if (!server_mode(options) || !advertised_server_mode()) + return 0; + /* We need an up-to-date view of network info if we're going to try to + * block exit attempts from unknown relays. */ + return ! router_my_exit_policy_is_reject_star() && + should_refuse_unknown_exits(options); } /** Return 1 if we want to allow remote people to ask us directory @@ -1238,14 +1275,14 @@ directory_too_idle_to_fetch_descriptors(or_options_t *options, time_t now) static cached_dir_t *cached_directory = NULL; /** The v1 runningrouters document we'll serve (as a cache or as an authority) * if requested. */ -static cached_dir_t cached_runningrouters = { NULL, NULL, 0, 0, 0, -1 }; +static cached_dir_t cached_runningrouters; /** Used for other dirservers' v2 network statuses. Map from hexdigest to * cached_dir_t. */ static digestmap_t *cached_v2_networkstatus = NULL; -/** The v3 consensus network status that we're currently serving. */ -static cached_dir_t *cached_v3_networkstatus = NULL; +/** Map from flavor name to the v3 consensuses that we're currently serving. */ +static strmap_t *cached_consensuses = NULL; /** Possibly replace the contents of <b>d</b> with the value of * <b>directory</b> published on <b>when</b>, unless <b>when</b> is older than @@ -1319,7 +1356,11 @@ clear_cached_dir(cached_dir_t *d) static void _free_cached_dir(void *_d) { - cached_dir_t *d = (cached_dir_t *)_d; + cached_dir_t *d; + if (!_d) + return; + + d = (cached_dir_t *)_d; cached_dir_decref(d); } @@ -1413,17 +1454,26 @@ dirserv_set_cached_networkstatus_v2(const char *networkstatus, } } -/** Replace the v3 consensus networkstatus that we're serving with - * <b>networkstatus</b>, published at <b>published</b>. No validation is - * performed. */ +/** Replace the v3 consensus networkstatus of type <b>flavor_name</b> that + * we're serving with <b>networkstatus</b>, published at <b>published</b>. No + * validation is performed. */ void -dirserv_set_cached_networkstatus_v3(const char *networkstatus, - time_t published) +dirserv_set_cached_consensus_networkstatus(const char *networkstatus, + const char *flavor_name, + const digests_t *digests, + time_t published) { - if (cached_v3_networkstatus) - cached_dir_decref(cached_v3_networkstatus); - cached_v3_networkstatus = new_cached_dir( - tor_strdup(networkstatus), published); + cached_dir_t *new_networkstatus; + cached_dir_t *old_networkstatus; + if (!cached_consensuses) + cached_consensuses = strmap_new(); + + new_networkstatus = new_cached_dir(tor_strdup(networkstatus), published); + memcpy(&new_networkstatus->digests, digests, sizeof(digests_t)); + old_networkstatus = strmap_set(cached_consensuses, flavor_name, + new_networkstatus); + if (old_networkstatus) + cached_dir_decref(old_networkstatus); } /** Remove any v2 networkstatus from the directory cache that was published @@ -1520,7 +1570,8 @@ dirserv_regenerate_directory(void) { char *new_directory=NULL; - if (dirserv_dump_directory_to_string(&new_directory, get_identity_key())) { + if (dirserv_dump_directory_to_string(&new_directory, + get_server_identity_key())) { log_warn(LD_BUG, "Error creating directory."); tor_free(new_directory); return NULL; @@ -1550,7 +1601,7 @@ generate_runningrouters(void) char digest[DIGEST_LEN]; char published[ISO_TIME_LEN+1]; size_t len; - crypto_pk_env_t *private_key = get_identity_key(); + crypto_pk_env_t *private_key = get_server_identity_key(); char *identity_pkey; /* Identity key, DER64-encoded. */ size_t identity_pkey_len; @@ -1577,7 +1628,8 @@ generate_runningrouters(void) goto err; } note_crypto_pk_op(SIGN_DIR); - if (router_append_dirobj_signature(s, len, digest, private_key)<0) + if (router_append_dirobj_signature(s, len, digest, DIGEST_LEN, + private_key)<0) goto err; set_cached_dir(&the_runningrouters, s, time(NULL)); @@ -1605,9 +1657,11 @@ dirserv_get_runningrouters(void) /** Return the latest downloaded consensus networkstatus in encoded, signed, * optionally compressed format, suitable for sending to clients. */ cached_dir_t * -dirserv_get_consensus(void) +dirserv_get_consensus(const char *flavor_name) { - return cached_v3_networkstatus; + if (!cached_consensuses) + return NULL; + return strmap_get(cached_consensuses, flavor_name); } /** For authoritative directories: the current (v2) network status. */ @@ -1645,7 +1699,7 @@ should_generate_v2_networkstatus(void) #define TIME_KNOWN_TO_GUARANTEE_FAMILIAR (8*24*60*60) /** Similarly, every node with sufficient WFU is around enough to be a guard. */ -#define WFU_TO_GUARANTEE_GUARD (0.995) +#define WFU_TO_GUARANTEE_GUARD (0.98) /* Thresholds for server performance: set by * dirserv_compute_performance_thresholds, and used by @@ -1700,9 +1754,12 @@ dirserv_thinks_router_is_unreliable(time_t now, { if (need_uptime) { if (!enough_mtbf_info) { - /* XXX022 Once most authorities are on v3, we should change the rule from + /* XXX023 Once most authorities are on v3, we should change the rule from * "use uptime if we don't have mtbf data" to "don't advertise Stable on - * v3 if we don't have enough mtbf data." */ + * v3 if we don't have enough mtbf data." Or maybe not, since if we ever + * hit a point where we need to reset a lot of authorities at once, + * none of them would be in a position to declare Stable. + */ long uptime = real_uptime(router, now); if ((unsigned)uptime < stable_uptime && (unsigned)uptime < UPTIME_TO_GUARANTEE_STABLE) @@ -1725,7 +1782,7 @@ dirserv_thinks_router_is_unreliable(time_t now, /** Return true iff <b>router</b> should be assigned the "HSDir" flag. * Right now this means it advertises support for it, it has a high - * uptime, and it's currently considered Running. + * uptime, it has a DirPort open, and it's currently considered Running. * * This function needs to be called after router-\>is_running has * been set. @@ -1733,9 +1790,31 @@ dirserv_thinks_router_is_unreliable(time_t now, static int dirserv_thinks_router_is_hs_dir(routerinfo_t *router, time_t now) { - long uptime = real_uptime(router, now); - return (router->wants_to_be_hs_dir && + long uptime; + + /* If we haven't been running for at least + * get_options()->MinUptimeHidServDirectoryV2 seconds, we can't + * have accurate data telling us a relay has been up for at least + * that long. We also want to allow a bit of slack: Reachability + * tests aren't instant. If we haven't been running long enough, + * trust the relay. */ + + if (stats_n_seconds_working > + get_options()->MinUptimeHidServDirectoryV2 * 1.1) + uptime = MIN(rep_hist_get_uptime(router->cache_info.identity_digest, now), + real_uptime(router, now)); + else + uptime = real_uptime(router, now); + + /* XXX We shouldn't need to check dir_port, but we do because of + * bug 1693. In the future, once relays set wants_to_be_hs_dir + * correctly, we can revert to only checking dir_port if router's + * version is too old. */ + /* XXX Unfortunately, we need to keep checking dir_port until all + * *clients* suffering from bug 2722 are obsolete. The first version + * to fix the bug was 0.2.2.25-alpha. */ + return (router->wants_to_be_hs_dir && router->dir_port && uptime > get_options()->MinUptimeHidServDirectoryV2 && router->is_running); } @@ -1790,7 +1869,8 @@ dirserv_compute_performance_thresholds(routerlist_t *rl) if (router_is_active(ri, now)) { const char *id = ri->cache_info.identity_digest; uint32_t bw; - ri->is_exit = exit_policy_is_general_exit(ri->exit_policy); + ri->is_exit = (!router_exit_policy_rejects_all(ri) && + exit_policy_is_general_exit(ri->exit_policy)); uptimes[n_active] = (uint32_t)real_uptime(ri, now); mtbfs[n_active] = rep_hist_get_stability(id, now); tks [n_active] = rep_hist_get_weighted_time_known(id, now); @@ -1897,16 +1977,20 @@ version_from_platform(const char *platform) * which has at least <b>buf_len</b> free characters. Do NUL-termination. * Use the same format as in network-status documents. If <b>version</b> is * non-NULL, add a "v" line for the platform. Return 0 on success, -1 on - * failure. If <b>first_line_only</b> is true, don't include any flags - * or version line. + * failure. + * + * The format argument has three possible values: + * NS_V2 - Output an entry suitable for a V2 NS opinion document + * NS_V3_CONSENSUS - Output the first portion of a V3 NS consensus entry + * NS_V3_CONSENSUS_MICRODESC - Output the first portion of a V3 microdesc + * consensus entry. + * NS_V3_VOTE - Output a complete V3 NS vote + * NS_CONTROL_PORT - Output a NS document for the control port */ int routerstatus_format_entry(char *buf, size_t buf_len, routerstatus_t *rs, const char *version, - int first_line_only, int v2_format) -/* XXX: first_line_only and v2_format should probably be be both - * replaced by a single purpose parameter. - */ + routerstatus_format_type_t format) { int r; struct in_addr in; @@ -1925,10 +2009,11 @@ routerstatus_format_entry(char *buf, size_t buf_len, tor_inet_ntoa(&in, ipaddr, sizeof(ipaddr)); r = tor_snprintf(buf, buf_len, - "r %s %s %s %s %s %d %d\n", + "r %s %s %s%s%s %s %d %d\n", rs->nickname, identity64, - digest64, + (format==NS_V3_CONSENSUS_MICRODESC)?"":digest64, + (format==NS_V3_CONSENSUS_MICRODESC)?"":" ", published, ipaddr, (int)rs->or_port, @@ -1937,7 +2022,12 @@ routerstatus_format_entry(char *buf, size_t buf_len, log_warn(LD_BUG, "Not enough space in buffer."); return -1; } - if (first_line_only) + + /* TODO: Maybe we want to pass in what we need to build the rest of + * this here, instead of in the caller. Then we could use the + * networkstatus_type_t values, with an additional control port value + * added -MP */ + if (format == NS_V3_CONSENSUS || format == NS_V3_CONSENSUS_MICRODESC) return 0; cp = buf + strlen(buf); @@ -1974,62 +2064,87 @@ routerstatus_format_entry(char *buf, size_t buf_len, cp += strlen(cp); } - if (!v2_format) { + if (format != NS_V2) { routerinfo_t* desc = router_get_by_digest(rs->identity_digest); + uint32_t bw; + + if (format != NS_CONTROL_PORT) { + /* Blow up more or less nicely if we didn't get anything or not the + * thing we expected. + */ + if (!desc) { + char id[HEX_DIGEST_LEN+1]; + char dd[HEX_DIGEST_LEN+1]; + + base16_encode(id, sizeof(id), rs->identity_digest, DIGEST_LEN); + base16_encode(dd, sizeof(dd), rs->descriptor_digest, DIGEST_LEN); + log_warn(LD_BUG, "Cannot get any descriptor for %s " + "(wanted descriptor %s).", + id, dd); + return -1; + }; + + /* This assert can fire for the control port, because + * it can request NS documents before all descriptors + * have been fetched. */ + if (memcmp(desc->cache_info.signed_descriptor_digest, + rs->descriptor_digest, + DIGEST_LEN)) { + char rl_d[HEX_DIGEST_LEN+1]; + char rs_d[HEX_DIGEST_LEN+1]; + char id[HEX_DIGEST_LEN+1]; + + base16_encode(rl_d, sizeof(rl_d), + desc->cache_info.signed_descriptor_digest, DIGEST_LEN); + base16_encode(rs_d, sizeof(rs_d), rs->descriptor_digest, DIGEST_LEN); + base16_encode(id, sizeof(id), rs->identity_digest, DIGEST_LEN); + log_err(LD_BUG, "descriptor digest in routerlist does not match " + "the one in routerstatus: %s vs %s " + "(router %s)\n", + rl_d, rs_d, id); + + tor_assert(!memcmp(desc->cache_info.signed_descriptor_digest, + rs->descriptor_digest, + DIGEST_LEN)); + }; + } - /* Blow up more or less nicely if we didn't get anything or not the - * thing we expected. - */ - if (!desc) { - char id[HEX_DIGEST_LEN+1]; - char dd[HEX_DIGEST_LEN+1]; - - base16_encode(id, sizeof(id), rs->identity_digest, DIGEST_LEN); - base16_encode(dd, sizeof(dd), rs->descriptor_digest, DIGEST_LEN); - log_warn(LD_BUG, "Cannot get any descriptor for %s " - "(wanted descriptor %s).", - id, dd); - return -1; - }; - if (fast_memcmp(desc->cache_info.signed_descriptor_digest, - rs->descriptor_digest, - DIGEST_LEN)) { - char rl_d[HEX_DIGEST_LEN+1]; - char rs_d[HEX_DIGEST_LEN+1]; - char id[HEX_DIGEST_LEN+1]; - - base16_encode(rl_d, sizeof(rl_d), - desc->cache_info.signed_descriptor_digest, DIGEST_LEN); - base16_encode(rs_d, sizeof(rs_d), rs->descriptor_digest, DIGEST_LEN); - base16_encode(id, sizeof(id), rs->identity_digest, DIGEST_LEN); - log_err(LD_BUG, "descriptor digest in routerlist does not match " - "the one in routerstatus: %s vs %s " - "(router %s)\n", - rl_d, rs_d, id); - - tor_assert(fast_memeq(desc->cache_info.signed_descriptor_digest, - rs->descriptor_digest, - DIGEST_LEN)); - }; - + if (format == NS_CONTROL_PORT && rs->has_bandwidth) { + bw = rs->bandwidth; + } else { + tor_assert(desc); + bw = router_get_advertised_bandwidth_capped(desc) / 1000; + } r = tor_snprintf(cp, buf_len - (cp-buf), - "w Bandwidth=%d\n", - router_get_advertised_bandwidth_capped(desc) / 1024); + "w Bandwidth=%d\n", bw); + if (r<0) { log_warn(LD_BUG, "Not enough space in buffer."); return -1; } cp += strlen(cp); + if (format == NS_V3_VOTE && rs->has_measured_bw) { + *--cp = '\0'; /* Kill "\n" */ + r = tor_snprintf(cp, buf_len - (cp-buf), + " Measured=%d\n", rs->measured_bw); + if (r<0) { + log_warn(LD_BUG, "Not enough space in buffer for weight line."); + return -1; + } + cp += strlen(cp); + } - summary = policy_summarize(desc->exit_policy); - r = tor_snprintf(cp, buf_len - (cp-buf), "p %s\n", summary); - if (r<0) { - log_warn(LD_BUG, "Not enough space in buffer."); + if (desc) { + summary = policy_summarize(desc->exit_policy); + r = tor_snprintf(cp, buf_len - (cp-buf), "p %s\n", summary); + if (r<0) { + log_warn(LD_BUG, "Not enough space in buffer."); + tor_free(summary); + return -1; + } + cp += strlen(cp); tor_free(summary); - return -1; } - cp += strlen(cp); - tor_free(summary); } return 0; @@ -2132,9 +2247,7 @@ get_possible_sybil_list(const smartlist_t *routers) /** Extract status information from <b>ri</b> and from other authority * functions and store it in <b>rs</b>>. If <b>naming</b>, consider setting - * the named flag in <b>rs</b>. If not <b>exits_can_be_guards</b>, never mark - * an exit as a guard. If <b>listbadexits</b>, consider setting the badexit - * flag. + * the named flag in <b>rs</b>. * * We assume that ri-\>is_running has already been set, e.g. by * dirserv_set_router_is_running(ri, now); @@ -2142,8 +2255,8 @@ get_possible_sybil_list(const smartlist_t *routers) void set_routerstatus_from_routerinfo(routerstatus_t *rs, routerinfo_t *ri, time_t now, - int naming, int exits_can_be_guards, - int listbadexits, int listbaddirs) + int naming, int listbadexits, + int listbaddirs) { int unstable_version = !tor_version_as_new_as(ri->platform,"0.1.1.16-rc-cvs"); @@ -2172,11 +2285,10 @@ set_routerstatus_from_routerinfo(routerstatus_t *rs, rs->is_valid = ri->is_valid; if (rs->is_fast && - (!rs->is_exit || exits_can_be_guards) && (router_get_advertised_bandwidth(ri) >= BANDWIDTH_TO_GUARANTEE_GUARD || router_get_advertised_bandwidth(ri) >= - (exits_can_be_guards ? guard_bandwidth_including_exits : - guard_bandwidth_excluding_exits))) { + MIN(guard_bandwidth_including_exits, + guard_bandwidth_excluding_exits))) { long tk = rep_hist_get_weighted_time_known( ri->cache_info.identity_digest, now); double wfu = rep_hist_get_weighted_fractional_uptime( @@ -2232,6 +2344,177 @@ router_clear_status_flags(routerinfo_t *router) router->is_bad_exit = router->is_bad_directory = 0; } +/** + * Helper function to parse out a line in the measured bandwidth file + * into a measured_bw_line_t output structure. Returns -1 on failure + * or 0 on success. + */ +int +measured_bw_line_parse(measured_bw_line_t *out, const char *orig_line) +{ + char *line = tor_strdup(orig_line); + char *cp = line; + int got_bw = 0; + int got_node_id = 0; + char *strtok_state; /* lame sauce d'jour */ + cp = tor_strtok_r(cp, " \t", &strtok_state); + + if (!cp) { + log_warn(LD_DIRSERV, "Invalid line in bandwidth file: %s", + escaped(orig_line)); + tor_free(line); + return -1; + } + + if (orig_line[strlen(orig_line)-1] != '\n') { + log_warn(LD_DIRSERV, "Incomplete line in bandwidth file: %s", + escaped(orig_line)); + tor_free(line); + return -1; + } + + do { + if (strcmpstart(cp, "bw=") == 0) { + int parse_ok = 0; + char *endptr; + if (got_bw) { + log_warn(LD_DIRSERV, "Double bw= in bandwidth file line: %s", + escaped(orig_line)); + tor_free(line); + return -1; + } + cp+=strlen("bw="); + + out->bw = tor_parse_long(cp, 0, 0, LONG_MAX, &parse_ok, &endptr); + if (!parse_ok || (*endptr && !TOR_ISSPACE(*endptr))) { + log_warn(LD_DIRSERV, "Invalid bandwidth in bandwidth file line: %s", + escaped(orig_line)); + tor_free(line); + return -1; + } + got_bw=1; + } else if (strcmpstart(cp, "node_id=$") == 0) { + if (got_node_id) { + log_warn(LD_DIRSERV, "Double node_id= in bandwidth file line: %s", + escaped(orig_line)); + tor_free(line); + return -1; + } + cp+=strlen("node_id=$"); + + if (strlen(cp) != HEX_DIGEST_LEN || + base16_decode(out->node_id, DIGEST_LEN, cp, HEX_DIGEST_LEN)) { + log_warn(LD_DIRSERV, "Invalid node_id in bandwidth file line: %s", + escaped(orig_line)); + tor_free(line); + return -1; + } + strncpy(out->node_hex, cp, sizeof(out->node_hex)); + got_node_id=1; + } + } while ((cp = tor_strtok_r(NULL, " \t", &strtok_state))); + + if (got_bw && got_node_id) { + tor_free(line); + return 0; + } else { + log_warn(LD_DIRSERV, "Incomplete line in bandwidth file: %s", + escaped(orig_line)); + tor_free(line); + return -1; + } +} + +/** + * Helper function to apply a parsed measurement line to a list + * of bandwidth statuses. Returns true if a line is found, + * false otherwise. + */ +int +measured_bw_line_apply(measured_bw_line_t *parsed_line, + smartlist_t *routerstatuses) +{ + routerstatus_t *rs = NULL; + if (!routerstatuses) + return 0; + + rs = smartlist_bsearch(routerstatuses, parsed_line->node_id, + compare_digest_to_routerstatus_entry); + + if (rs) { + rs->has_measured_bw = 1; + rs->measured_bw = (uint32_t)parsed_line->bw; + } else { + log_info(LD_DIRSERV, "Node ID %s not found in routerstatus list", + parsed_line->node_hex); + } + + return rs != NULL; +} + +/** + * Read the measured bandwidth file and apply it to the list of + * routerstatuses. Returns -1 on error, 0 otherwise. + */ +int +dirserv_read_measured_bandwidths(const char *from_file, + smartlist_t *routerstatuses) +{ + char line[256]; + FILE *fp = fopen(from_file, "r"); + int applied_lines = 0; + time_t file_time; + int ok; + if (fp == NULL) { + log_warn(LD_CONFIG, "Can't open bandwidth file at configured location: %s", + from_file); + return -1; + } + + if (!fgets(line, sizeof(line), fp) + || !strlen(line) || line[strlen(line)-1] != '\n') { + log_warn(LD_DIRSERV, "Long or truncated time in bandwidth file: %s", + escaped(line)); + fclose(fp); + return -1; + } + + line[strlen(line)-1] = '\0'; + file_time = tor_parse_ulong(line, 10, 0, ULONG_MAX, &ok, NULL); + if (!ok) { + log_warn(LD_DIRSERV, "Non-integer time in bandwidth file: %s", + escaped(line)); + fclose(fp); + return -1; + } + + if ((time(NULL) - file_time) > MAX_MEASUREMENT_AGE) { + log_warn(LD_DIRSERV, "Bandwidth measurement file stale. Age: %u", + (unsigned)(time(NULL) - file_time)); + fclose(fp); + return -1; + } + + if (routerstatuses) + smartlist_sort(routerstatuses, compare_routerstatus_entries); + + while (!feof(fp)) { + measured_bw_line_t parsed_line; + if (fgets(line, sizeof(line), fp) && strlen(line)) { + if (measured_bw_line_parse(&parsed_line, line) != -1) { + if (measured_bw_line_apply(&parsed_line, routerstatuses) > 0) + applied_lines++; + } + } + } + + fclose(fp); + log_info(LD_DIRSERV, + "Bandwidth measurement file successfully read. " + "Applied %d measurements.", applied_lines); + return 0; +} + /** Return a new networkstatus_t* containing our current opinion. (For v3 * authorities) */ networkstatus_t * @@ -2249,22 +2532,18 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_env_t *private_key, int naming = options->NamingAuthoritativeDir; int listbadexits = options->AuthDirListBadExits; int listbaddirs = options->AuthDirListBadDirs; - int exits_can_be_guards; routerlist_t *rl = router_get_routerlist(); time_t now = time(NULL); time_t cutoff = now - ROUTER_MAX_AGE_TO_PUBLISH; networkstatus_voter_info_t *voter = NULL; vote_timing_t timing; digestmap_t *omit_as_sybil = NULL; - int vote_on_reachability = 1; + const int vote_on_reachability = running_long_enough_to_decide_unreachable(); + smartlist_t *microdescriptors = NULL; tor_assert(private_key); tor_assert(cert); - if (now - time_of_process_start < - options->TestingAuthDirTimeToLearnReachability) - vote_on_reachability = 0; - if (resolve_my_address(LOG_WARN, options, &addr, &hostname)<0) { log_warn(LD_NET, "Couldn't resolve my hostname"); return NULL; @@ -2299,27 +2578,24 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_env_t *private_key, dirserv_compute_performance_thresholds(rl); - /* XXXX We should take steps to keep this from oscillating if - * total_exit_bandwidth is close to total_bandwidth/3. */ - exits_can_be_guards = total_exit_bandwidth >= (total_bandwidth / 3); - routers = smartlist_create(); smartlist_add_all(routers, rl->routers); routers_sort_by_identity(routers); omit_as_sybil = get_possible_sybil_list(routers); routerstatuses = smartlist_create(); + microdescriptors = smartlist_create(); - SMARTLIST_FOREACH(routers, routerinfo_t *, ri, { + SMARTLIST_FOREACH_BEGIN(routers, routerinfo_t *, ri) { if (ri->cache_info.published_on >= cutoff) { routerstatus_t *rs; vote_routerstatus_t *vrs; + microdesc_t *md; vrs = tor_malloc_zero(sizeof(vote_routerstatus_t)); rs = &vrs->status; set_routerstatus_from_routerinfo(rs, ri, now, - naming, exits_can_be_guards, - listbadexits, listbaddirs); + naming, listbadexits, listbaddirs); if (digestmap_get(omit_as_sybil, ri->cache_info.identity_digest)) clear_status_flags_on_sybil(rs); @@ -2328,12 +2604,39 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_env_t *private_key, rs->is_running = 0; vrs->version = version_from_platform(ri->platform); + md = dirvote_create_microdescriptor(ri); + if (md) { + char buf[128]; + vote_microdesc_hash_t *h; + dirvote_format_microdesc_vote_line(buf, sizeof(buf), md); + h = tor_malloc(sizeof(vote_microdesc_hash_t)); + h->microdesc_hash_line = tor_strdup(buf); + h->next = NULL; + vrs->microdesc = h; + md->last_listed = now; + smartlist_add(microdescriptors, md); + } + smartlist_add(routerstatuses, vrs); } - }); + } SMARTLIST_FOREACH_END(ri); + + { + smartlist_t *added = + microdescs_add_list_to_cache(get_microdesc_cache(), + microdescriptors, SAVED_NOWHERE, 0); + smartlist_free(added); + smartlist_free(microdescriptors); + } + smartlist_free(routers); digestmap_free(omit_as_sybil, NULL); + if (options->V3BandwidthsFile) { + dirserv_read_measured_bandwidths(options->V3BandwidthsFile, + routerstatuses); + } + v3_out = tor_malloc_zero(sizeof(networkstatus_t)); v3_out->type = NS_TYPE_VOTE; @@ -2383,19 +2686,29 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_env_t *private_key, } smartlist_sort_strings(v3_out->known_flags); + if (options->ConsensusParams) { + v3_out->net_params = smartlist_create(); + smartlist_split_string(v3_out->net_params, + options->ConsensusParams, NULL, 0, 0); + smartlist_sort_strings(v3_out->net_params); + } + voter = tor_malloc_zero(sizeof(networkstatus_voter_info_t)); voter->nickname = tor_strdup(options->Nickname); memcpy(voter->identity_digest, identity_digest, DIGEST_LEN); + voter->sigs = smartlist_create(); voter->address = hostname; voter->addr = addr; voter->dir_port = options->DirPort; voter->or_port = options->ORPort; voter->contact = tor_strdup(contact); - memcpy(voter->signing_key_digest, signing_key_digest, DIGEST_LEN); if (options->V3AuthUseLegacyKey) { authority_cert_t *c = get_my_v3_legacy_cert(); if (c) { - crypto_pk_get_digest(c->identity_key, voter->legacy_id_digest); + if (crypto_pk_get_digest(c->identity_key, voter->legacy_id_digest)) { + log_warn(LD_BUG, "Unable to compute digest of legacy v3 identity key"); + memset(voter->legacy_id_digest, 0, DIGEST_LEN); + } } } @@ -2435,13 +2748,12 @@ generate_v2_networkstatus_opinion(void) int versioning = options->VersioningAuthoritativeDir; int listbaddirs = options->AuthDirListBadDirs; int listbadexits = options->AuthDirListBadExits; - int exits_can_be_guards; const char *contact; char *version_lines = NULL; smartlist_t *routers = NULL; digestmap_t *omit_as_sybil = NULL; - private_key = get_identity_key(); + private_key = get_server_identity_key(); if (resolve_my_address(LOG_WARN, options, &addr, &hostname)<0) { log_warn(LD_NET, "Couldn't resolve my hostname"); @@ -2515,10 +2827,6 @@ generate_v2_networkstatus_opinion(void) dirserv_compute_performance_thresholds(rl); - /* XXXX We should take steps to keep this from oscillating if - * total_exit_bandwidth is close to total_bandwidth/3. */ - exits_can_be_guards = total_exit_bandwidth >= (total_bandwidth / 3); - routers = smartlist_create(); smartlist_add_all(routers, rl->routers); routers_sort_by_identity(routers); @@ -2531,13 +2839,12 @@ generate_v2_networkstatus_opinion(void) char *version = version_from_platform(ri->platform); set_routerstatus_from_routerinfo(&rs, ri, now, - naming, exits_can_be_guards, - listbadexits, listbaddirs); + naming, listbadexits, listbaddirs); if (digestmap_get(omit_as_sybil, ri->cache_info.identity_digest)) clear_status_flags_on_sybil(&rs); - if (routerstatus_format_entry(outp, endp-outp, &rs, version, 0, 1)) { + if (routerstatus_format_entry(outp, endp-outp, &rs, version, NS_V2)) { log_warn(LD_BUG, "Unable to print router status."); tor_free(version); goto done; @@ -2559,7 +2866,8 @@ generate_v2_networkstatus_opinion(void) outp += strlen(outp); note_crypto_pk_op(SIGN_DIR); - if (router_append_dirobj_signature(outp,endp-outp,digest,private_key)<0) { + if (router_append_dirobj_signature(outp,endp-outp,digest,DIGEST_LEN, + private_key)<0) { log_warn(LD_BUG, "Unable to sign router status."); goto done; } @@ -2591,10 +2899,8 @@ generate_v2_networkstatus_opinion(void) tor_free(status); tor_free(hostname); tor_free(identity_pkey); - if (routers) - smartlist_free(routers); - if (omit_as_sybil) - digestmap_free(omit_as_sybil, NULL); + smartlist_free(routers); + digestmap_free(omit_as_sybil, NULL); return r; } @@ -2642,7 +2948,8 @@ dirserv_get_networkstatus_v2_fingerprints(smartlist_t *result, log_info(LD_DIRSERV, "Client requested 'all' network status objects; we have none."); } else if (!strcmpstart(key, "fp/")) { - dir_split_resource_into_fingerprints(key+3, result, NULL, 1, 1); + dir_split_resource_into_fingerprints(key+3, result, NULL, + DSR_HEX|DSR_SORT_UNIQ); } } @@ -2709,10 +3016,12 @@ dirserv_get_routerdesc_fingerprints(smartlist_t *fps_out, const char *key, } else if (!strcmpstart(key, "d/")) { by_id = 0; key += strlen("d/"); - dir_split_resource_into_fingerprints(key, fps_out, NULL, 1, 1); + dir_split_resource_into_fingerprints(key, fps_out, NULL, + DSR_HEX|DSR_SORT_UNIQ); } else if (!strcmpstart(key, "fp/")) { key += strlen("fp/"); - dir_split_resource_into_fingerprints(key, fps_out, NULL, 1, 1); + dir_split_resource_into_fingerprints(key, fps_out, NULL, + DSR_HEX|DSR_SORT_UNIQ); } else { *msg = "Key not recognized"; return -1; @@ -2777,7 +3086,8 @@ dirserv_get_routerdescs(smartlist_t *descs_out, const char *key, } else if (!strcmpstart(key, "/tor/server/d/")) { smartlist_t *digests = smartlist_create(); key += strlen("/tor/server/d/"); - dir_split_resource_into_fingerprints(key, digests, NULL, 1, 1); + dir_split_resource_into_fingerprints(key, digests, NULL, + DSR_HEX|DSR_SORT_UNIQ); SMARTLIST_FOREACH(digests, const char *, d, { signed_descriptor_t *sd = router_get_by_descriptor_digest(d); @@ -2790,7 +3100,8 @@ dirserv_get_routerdescs(smartlist_t *descs_out, const char *key, smartlist_t *digests = smartlist_create(); time_t cutoff = time(NULL) - ROUTER_MAX_AGE_TO_PUBLISH; key += strlen("/tor/server/fp/"); - dir_split_resource_into_fingerprints(key, digests, NULL, 1, 1); + dir_split_resource_into_fingerprints(key, digests, NULL, + DSR_HEX|DSR_SORT_UNIQ); SMARTLIST_FOREACH(digests, const char *, d, { if (router_digest_is_me(d)) { @@ -2841,34 +3152,82 @@ dirserv_orconn_tls_done(const char *address, tor_assert(address); tor_assert(digest_rcvd); - SMARTLIST_FOREACH(rl->routers, routerinfo_t *, ri, { + /* XXX023 Doing a loop like this is stupid. We should just look up the + * router by digest_rcvd, and see if address, orport, and as_advertised + * match up. -NM */ + SMARTLIST_FOREACH_BEGIN(rl->routers, routerinfo_t *, ri) { if (!strcasecmp(address, ri->address) && or_port == ri->or_port && as_advertised && fast_memeq(ri->cache_info.identity_digest, digest_rcvd, DIGEST_LEN)) { /* correct digest. mark this router reachable! */ if (!bridge_auth || ri->purpose == ROUTER_PURPOSE_BRIDGE) { - log_info(LD_DIRSERV, "Found router %s to be reachable. Yay.", - ri->nickname); - rep_hist_note_router_reachable(digest_rcvd, now); + tor_addr_t addr, *addrp=NULL; + log_info(LD_DIRSERV, "Found router %s to be reachable at %s:%d. Yay.", + ri->nickname, address, ri->or_port ); + if (tor_addr_from_str(&addr, ri->address) != -1) + addrp = &addr; + else + log_warn(LD_BUG, "Couldn't parse IP address \"%s\"", ri->address); + rep_hist_note_router_reachable(digest_rcvd, addrp, or_port, now); ri->last_reachable = now; } } - }); + } SMARTLIST_FOREACH_END(ri); /* FFFF Maybe we should reinstate the code that dumps routers with the same * addr/port but with nonmatching keys, but instead of dumping, we should * skip testing. */ } -/** Auth dir server only: if <b>try_all</b> is 1, launch connections to - * all known routers; else we want to load balance such that we only +/** Called when we, as an authority, receive a new router descriptor either as + * an upload or a download. Used to decide whether to relaunch reachability + * testing for the server. */ +int +dirserv_should_launch_reachability_test(routerinfo_t *ri, routerinfo_t *ri_old) +{ + if (!authdir_mode_handles_descs(get_options(), ri->purpose)) + return 0; + if (!ri_old) { + /* New router: Launch an immediate reachability test, so we will have an + * opinion soon in case we're generating a consensus soon */ + return 1; + } + if (ri_old->is_hibernating && !ri->is_hibernating) { + /* It just came out of hibernation; launch a reachability test */ + return 1; + } + if (! routers_have_same_or_addr(ri, ri_old)) { + /* Address or port changed; launch a reachability test */ + return 1; + } + return 0; +} + +/** Helper function for dirserv_test_reachability(). Start a TLS + * connection to <b>router</b>, and annotate it with when we started + * the test. */ +void +dirserv_single_reachability_test(time_t now, routerinfo_t *router) +{ + tor_addr_t router_addr; + log_debug(LD_OR,"Testing reachability of %s at %s:%u.", + router->nickname, router->address, router->or_port); + /* Remember when we started trying to determine reachability */ + if (!router->testing_since) + router->testing_since = now; + tor_addr_from_ipv4h(&router_addr, router->addr); + connection_or_connect(&router_addr, router->or_port, + router->cache_info.identity_digest); +} + +/** Auth dir server only: load balance such that we only * try a few connections per call. * * The load balancing is such that if we get called once every ten - * seconds, we will cycle through all the tests in 1280 seconds (a - * bit over 20 minutes). + * seconds, we will cycle through all the tests in + * REACHABILITY_TEST_CYCLE_PERIOD seconds (a bit over 20 minutes). */ void -dirserv_test_reachability(time_t now, int try_all) +dirserv_test_reachability(time_t now) { /* XXX decide what to do here; see or-talk thread "purging old router * information, revocation." -NM @@ -2885,38 +3244,33 @@ dirserv_test_reachability(time_t now, int try_all) SMARTLIST_FOREACH_BEGIN(rl->routers, routerinfo_t *, router) { const char *id_digest = router->cache_info.identity_digest; - tor_addr_t router_addr; if (router_is_me(router)) continue; if (bridge_auth && router->purpose != ROUTER_PURPOSE_BRIDGE) continue; /* bridge authorities only test reachability on bridges */ // if (router->cache_info.published_on > cutoff) // continue; - if (try_all || (((uint8_t)id_digest[0]) % 128) == ctr) { - log_debug(LD_OR,"Testing reachability of %s at %s:%u.", - router->nickname, router->address, router->or_port); - /* Remember when we started trying to determine reachability */ - if (!router->testing_since) - router->testing_since = now; - tor_addr_from_ipv4h(&router_addr, router->addr); - connection_or_connect(&router_addr, router->or_port, id_digest); + if ((((uint8_t)id_digest[0]) % REACHABILITY_MODULO_PER_TEST) == ctr) { + dirserv_single_reachability_test(now, router); } } SMARTLIST_FOREACH_END(router); - if (!try_all) /* increment ctr */ - ctr = (ctr + 1) % 128; + ctr = (ctr + 1) % REACHABILITY_MODULO_PER_TEST; /* increment ctr */ } -/** Given a fingerprint <b>fp</b> which is either set if we're looking - * for a v2 status, or zeroes if we're looking for a v3 status, return - * a pointer to the appropriate cached dir object, or NULL if there isn't - * one available. */ +/** Given a fingerprint <b>fp</b> which is either set if we're looking for a + * v2 status, or zeroes if we're looking for a v3 status, or a NUL-padded + * flavor name if we want a flavored v3 status, return a pointer to the + * appropriate cached dir object, or NULL if there isn't one available. */ static cached_dir_t * lookup_cached_dir_by_fp(const char *fp) { cached_dir_t *d = NULL; - if (tor_digest_is_zero(fp) && cached_v3_networkstatus) - d = cached_v3_networkstatus; - else if (router_digest_is_me(fp) && the_v2_networkstatus) + if (tor_digest_is_zero(fp) && cached_consensuses) + d = strmap_get(cached_consensuses, "ns"); + else if (memchr(fp, '\0', DIGEST_LEN) && cached_consensuses && + (d = strmap_get(cached_consensuses, fp))) { + /* this here interface is a nasty hack XXXX023 */; + } else if (router_digest_is_me(fp) && the_v2_networkstatus) d = the_v2_networkstatus; else if (cached_v2_networkstatus) d = digestmap_get(cached_v2_networkstatus, fp); @@ -3002,6 +3356,18 @@ dirserv_have_any_serverdesc(smartlist_t *fps, int spool_src) return 0; } +/** Return true iff any of the 256-bit elements in <b>fps</b> is the digest of + * a microdescriptor we have. */ +int +dirserv_have_any_microdesc(const smartlist_t *fps) +{ + microdesc_cache_t *cache = get_microdesc_cache(); + SMARTLIST_FOREACH(fps, const char *, fp, + if (microdesc_cache_lookup_by_digest256(cache, fp)) + return 1); + return 0; +} + /** Return an approximate estimate of the number of bytes that will * be needed to transmit the server descriptors (if is_serverdescs -- * they can be either d/ or fp/ queries) or networkstatus objects (if @@ -3033,6 +3399,17 @@ dirserv_estimate_data_size(smartlist_t *fps, int is_serverdescs, return result; } +/** Given a list of microdescriptor hashes, guess how many bytes will be + * needed to transmit them, and return the guess. */ +size_t +dirserv_estimate_microdesc_size(const smartlist_t *fps, int compressed) +{ + size_t result = smartlist_len(fps) * microdesc_average_size(NULL); + if (compressed) + result /= 2; + return result; +} + /** When we're spooling data onto our outbuf, add more whenever we dip * below this threshold. */ #define DIRSERV_BUFFER_MIN 16384 @@ -3096,6 +3473,8 @@ connection_dirserv_add_servers_to_outbuf(dir_connection_t *conn) #endif body = signed_descriptor_get_body(sd); if (conn->zlib_state) { + /* XXXX022 This 'last' business should actually happen on the last + * routerinfo, not on the last fingerprint. */ int last = ! smartlist_len(conn->fingerprint_stack); connection_write_to_buf_zlib(body, sd->signed_descriptor_len, conn, last); @@ -3119,6 +3498,44 @@ connection_dirserv_add_servers_to_outbuf(dir_connection_t *conn) return 0; } +/** Spooling helper: called when we're sending a bunch of microdescriptors, + * and the outbuf has become too empty. Pulls some entries from + * fingerprint_stack, and writes the corresponding microdescs onto outbuf. If + * we run out of entries, flushes the zlib state and sets the spool source to + * NONE. Returns 0 on success, negative on failure. + */ +static int +connection_dirserv_add_microdescs_to_outbuf(dir_connection_t *conn) +{ + microdesc_cache_t *cache = get_microdesc_cache(); + while (smartlist_len(conn->fingerprint_stack) && + buf_datalen(conn->_base.outbuf) < DIRSERV_BUFFER_MIN) { + char *fp256 = smartlist_pop_last(conn->fingerprint_stack); + microdesc_t *md = microdesc_cache_lookup_by_digest256(cache, fp256); + tor_free(fp256); + if (!md) + continue; + if (conn->zlib_state) { + /* XXXX022 This 'last' business should actually happen on the last + * routerinfo, not on the last fingerprint. */ + int last = !smartlist_len(conn->fingerprint_stack); + connection_write_to_buf_zlib(md->body, md->bodylen, conn, last); + if (last) { + tor_zlib_free(conn->zlib_state); + conn->zlib_state = NULL; + } + } else { + connection_write_to_buf(md->body, md->bodylen, TO_CONN(conn)); + } + } + if (!smartlist_len(conn->fingerprint_stack)) { + conn->dir_spool_src = DIR_SPOOL_NONE; + smartlist_free(conn->fingerprint_stack); + conn->fingerprint_stack = NULL; + } + return 0; +} + /** Spooling helper: Called when we're sending a directory or networkstatus, * and the outbuf has become too empty. Pulls some bytes from * <b>conn</b>-\>cached_dir-\>dir_z, uncompresses them if appropriate, and @@ -3201,8 +3618,7 @@ connection_dirserv_add_networkstatus_bytes_to_outbuf(dir_connection_t *conn) } } else { connection_dirserv_finish_spooling(conn); - if (conn->fingerprint_stack) - smartlist_free(conn->fingerprint_stack); + smartlist_free(conn->fingerprint_stack); conn->fingerprint_stack = NULL; return 0; } @@ -3226,6 +3642,8 @@ connection_dirserv_flushed_some(dir_connection_t *conn) case DIR_SPOOL_SERVER_BY_DIGEST: case DIR_SPOOL_SERVER_BY_FP: return connection_dirserv_add_servers_to_outbuf(conn); + case DIR_SPOOL_MICRODESC: + return connection_dirserv_add_microdescs_to_outbuf(conn); case DIR_SPOOL_CACHED_DIR: return connection_dirserv_add_dir_bytes_to_outbuf(conn); case DIR_SPOOL_NETWORKSTATUS: @@ -3247,10 +3665,10 @@ dirserv_free_all(void) cached_dir_decref(the_v2_networkstatus); cached_dir_decref(cached_directory); clear_cached_dir(&cached_runningrouters); - if (cached_v2_networkstatus) { - digestmap_free(cached_v2_networkstatus, _free_cached_dir); - cached_v2_networkstatus = NULL; - } - cached_dir_decref(cached_v3_networkstatus); + + digestmap_free(cached_v2_networkstatus, _free_cached_dir); + cached_v2_networkstatus = NULL; + strmap_free(cached_consensuses, _free_cached_dir); + cached_consensuses = NULL; } diff --git a/src/or/dirserv.h b/src/or/dirserv.h new file mode 100644 index 0000000000..569abfca2e --- /dev/null +++ b/src/or/dirserv.h @@ -0,0 +1,146 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file dirserv.h + * \brief Header file for dirserv.c. + **/ + +#ifndef _TOR_DIRSERV_H +#define _TOR_DIRSERV_H + +/** What fraction (1 over this number) of the relay ID space do we + * (as a directory authority) launch connections to at each reachability + * test? */ +#define REACHABILITY_MODULO_PER_TEST 128 + +/** How often (in seconds) do we launch reachability tests? */ +#define REACHABILITY_TEST_INTERVAL 10 + +/** How many seconds apart are the reachability tests for a given relay? */ +#define REACHABILITY_TEST_CYCLE_PERIOD \ + (REACHABILITY_TEST_INTERVAL*REACHABILITY_MODULO_PER_TEST) + +/** Maximum length of an exit policy summary. */ +#define MAX_EXITPOLICY_SUMMARY_LEN 1000 + +/** Maximum allowable length of a version line in a networkstatus. */ +#define MAX_V_LINE_LEN 128 +/** Length of "r Authority BadDirectory BadExit Exit Fast Guard HSDir Named + * Running Stable Unnamed V2Dir Valid\n". */ +#define MAX_FLAG_LINE_LEN 96 +/** Length of "w" line for weighting. Currently at most + * "w Bandwidth=<uint32t> Measured=<uint32t>\n" */ +#define MAX_WEIGHT_LINE_LEN (12+10+10+10+1) +/** Maximum length of an exit policy summary line. */ +#define MAX_POLICY_LINE_LEN (3+MAX_EXITPOLICY_SUMMARY_LEN) +/** Amount of space to allocate for each entry: r, s, and v lines. */ +#define RS_ENTRY_LEN \ + ( /* first line */ \ + MAX_NICKNAME_LEN+BASE64_DIGEST_LEN*2+ISO_TIME_LEN+INET_NTOA_BUF_LEN+ \ + 5*2 /* ports */ + 10 /* punctuation */ + \ + /* second line */ \ + MAX_FLAG_LINE_LEN + \ + /* weight line */ \ + MAX_WEIGHT_LINE_LEN + \ + /* p line. */ \ + MAX_POLICY_LINE_LEN + \ + /* v line. */ \ + MAX_V_LINE_LEN \ + ) + +#define UNNAMED_ROUTER_NICKNAME "Unnamed" + +int connection_dirserv_flushed_some(dir_connection_t *conn); + +int dirserv_add_own_fingerprint(const char *nickname, crypto_pk_env_t *pk); +int dirserv_load_fingerprint_file(void); +void dirserv_free_fingerprint_list(void); +const char *dirserv_get_nickname_by_digest(const char *digest); +enum was_router_added_t dirserv_add_multiple_descriptors( + const char *desc, uint8_t purpose, + const char *source, + const char **msg); +enum was_router_added_t dirserv_add_descriptor(routerinfo_t *ri, + const char **msg, + const char *source); +void dirserv_set_router_is_running(routerinfo_t *router, time_t now); +int list_server_status_v1(smartlist_t *routers, char **router_status_out, + int for_controller); +int dirserv_dump_directory_to_string(char **dir_out, + crypto_pk_env_t *private_key); + +int directory_fetches_from_authorities(or_options_t *options); +int directory_fetches_dir_info_early(or_options_t *options); +int directory_fetches_dir_info_later(or_options_t *options); +int directory_caches_v2_dir_info(or_options_t *options); +#define directory_caches_v1_dir_info(o) directory_caches_v2_dir_info(o) +int directory_caches_dir_info(or_options_t *options); +int directory_permits_begindir_requests(or_options_t *options); +int directory_permits_controller_requests(or_options_t *options); +int directory_too_idle_to_fetch_descriptors(or_options_t *options, time_t now); + +void directory_set_dirty(void); +cached_dir_t *dirserv_get_directory(void); +cached_dir_t *dirserv_get_runningrouters(void); +cached_dir_t *dirserv_get_consensus(const char *flavor_name); +void dirserv_set_cached_directory(const char *directory, time_t when, + int is_running_routers); +void dirserv_set_cached_networkstatus_v2(const char *directory, + const char *identity, + time_t published); +void dirserv_set_cached_consensus_networkstatus(const char *consensus, + const char *flavor_name, + const digests_t *digests, + time_t published); +void dirserv_clear_old_networkstatuses(time_t cutoff); +void dirserv_clear_old_v1_info(time_t now); +void dirserv_get_networkstatus_v2(smartlist_t *result, const char *key); +void dirserv_get_networkstatus_v2_fingerprints(smartlist_t *result, + const char *key); +int dirserv_get_routerdesc_fingerprints(smartlist_t *fps_out, const char *key, + const char **msg, + int for_unencrypted_conn, + int is_extrainfo); +int dirserv_get_routerdescs(smartlist_t *descs_out, const char *key, + const char **msg); +void dirserv_orconn_tls_done(const char *address, + uint16_t or_port, + const char *digest_rcvd, + int as_advertised); +int dirserv_should_launch_reachability_test(routerinfo_t *ri, + routerinfo_t *ri_old); +void dirserv_single_reachability_test(time_t now, routerinfo_t *router); +void dirserv_test_reachability(time_t now); +int authdir_wants_to_reject_router(routerinfo_t *ri, const char **msg, + int complain); +int dirserv_would_reject_router(routerstatus_t *rs); +int dirserv_remove_old_statuses(smartlist_t *fps, time_t cutoff); +int dirserv_have_any_serverdesc(smartlist_t *fps, int spool_src); +int dirserv_have_any_microdesc(const smartlist_t *fps); +size_t dirserv_estimate_data_size(smartlist_t *fps, int is_serverdescs, + int compressed); +size_t dirserv_estimate_microdesc_size(const smartlist_t *fps, int compressed); + +int routerstatus_format_entry(char *buf, size_t buf_len, + routerstatus_t *rs, const char *platform, + routerstatus_format_type_t format); +void dirserv_free_all(void); +void cached_dir_decref(cached_dir_t *d); +cached_dir_t *new_cached_dir(char *s, time_t published); + +#ifdef DIRSERV_PRIVATE +int measured_bw_line_parse(measured_bw_line_t *out, const char *line); + +int measured_bw_line_apply(measured_bw_line_t *parsed_line, + smartlist_t *routerstatuses); +#endif + +int dirserv_read_measured_bandwidths(const char *from_file, + smartlist_t *routerstatuses); + +#endif + diff --git a/src/or/dirvote.c b/src/or/dirvote.c index 9e763bdc91..a95cea4b7d 100644 --- a/src/or/dirvote.c +++ b/src/or/dirvote.c @@ -5,27 +5,72 @@ #define DIRVOTE_PRIVATE #include "or.h" +#include "config.h" +#include "directory.h" +#include "dirserv.h" +#include "dirvote.h" +#include "microdesc.h" +#include "networkstatus.h" +#include "policies.h" +#include "rephist.h" +#include "router.h" +#include "routerlist.h" +#include "routerparse.h" /** * \file dirvote.c * \brief Functions to compute directory consensus, and schedule voting. **/ -static int dirvote_add_signatures_to_pending_consensus( +/** A consensus that we have built and are appending signatures to. Once it's + * time to publish it, it will become an active consensus if it accumulates + * enough signatures. */ +typedef struct pending_consensus_t { + /** The body of the consensus that we're currently building. Once we + * have it built, it goes into dirserv.c */ + char *body; + /** The parsed in-progress consensus document. */ + networkstatus_t *consensus; +} pending_consensus_t; + +static int dirvote_add_signatures_to_all_pending_consensuses( const char *detached_signatures_body, const char **msg_out); +static int dirvote_add_signatures_to_pending_consensus( + pending_consensus_t *pc, + ns_detached_signatures_t *sigs, + const char **msg_out); static char *list_v3_auth_ids(void); static void dirvote_fetch_missing_votes(void); static void dirvote_fetch_missing_signatures(void); static int dirvote_perform_vote(void); static void dirvote_clear_votes(int all_votes); -static int dirvote_compute_consensus(void); +static int dirvote_compute_consensuses(void); static int dirvote_publish_consensus(void); +static char *make_consensus_method_list(int low, int high, const char *sep); + +/** The highest consensus method that we currently support. */ +#define MAX_SUPPORTED_CONSENSUS_METHOD 11 + +/** Lowest consensus method that contains a 'directory-footer' marker */ +#define MIN_METHOD_FOR_FOOTER 9 + +/** Lowest consensus method that contains bandwidth weights */ +#define MIN_METHOD_FOR_BW_WEIGHTS 9 + +/** Lowest consensus method that contains consensus params */ +#define MIN_METHOD_FOR_PARAMS 7 + +/** Lowest consensus method that generates microdescriptors */ +#define MIN_METHOD_FOR_MICRODESC 8 /* ===== * Voting * =====*/ +/* Overestimated. */ +#define MICRODESC_LINE_LEN 80 + /** Return a new string containing the string representation of the vote in * <b>v3_ns</b>, signed with our v3 signing key <b>private_signing_key</b>. * For v3 authorities. */ @@ -44,6 +89,7 @@ format_networkstatus_vote(crypto_pk_env_t *private_signing_key, uint32_t addr; routerlist_t *rl = router_get_routerlist(); char *version_lines = NULL; + int r; networkstatus_voter_info_t *voter; tor_assert(private_signing_key); @@ -70,20 +116,30 @@ format_networkstatus_vote(crypto_pk_env_t *private_signing_key, version_lines = tor_malloc(v_len); cp = version_lines; if (client_versions) { - tor_snprintf(cp, v_len-(cp-version_lines), + r = tor_snprintf(cp, v_len-(cp-version_lines), "client-versions %s\n", client_versions); + if (r < 0) { + log_err(LD_BUG, "Insufficient memory for client-versions line"); + tor_assert(0); + } cp += strlen(cp); } - if (server_versions) - tor_snprintf(cp, v_len-(cp-version_lines), + if (server_versions) { + r = tor_snprintf(cp, v_len-(cp-version_lines), "server-versions %s\n", server_versions); + if (r < 0) { + log_err(LD_BUG, "Insufficient memory for server-versions line"); + tor_assert(0); + } + } } else { version_lines = tor_strdup(""); } len = 8192; len += strlen(version_lines); - len += (RS_ENTRY_LEN)*smartlist_len(rl->routers); + len += (RS_ENTRY_LEN+MICRODESC_LINE_LEN)*smartlist_len(rl->routers); + len += strlen("\ndirectory-footer\n"); len += v3_ns->cert->cache_info.signed_descriptor_len; status = tor_malloc(len); @@ -93,17 +149,25 @@ format_networkstatus_vote(crypto_pk_env_t *private_signing_key, char fu[ISO_TIME_LEN+1]; char vu[ISO_TIME_LEN+1]; char *flags = smartlist_join_strings(v3_ns->known_flags, " ", 0, NULL); + char *params; authority_cert_t *cert = v3_ns->cert; + char *methods = + make_consensus_method_list(1, MAX_SUPPORTED_CONSENSUS_METHOD, " "); format_iso_time(published, v3_ns->published); format_iso_time(va, v3_ns->valid_after); format_iso_time(fu, v3_ns->fresh_until); format_iso_time(vu, v3_ns->valid_until); + if (v3_ns->net_params) + params = smartlist_join_strings(v3_ns->net_params, " ", 0, NULL); + else + params = tor_strdup(""); + tor_assert(cert); - tor_snprintf(status, len, + r = tor_snprintf(status, len, "network-status-version 3\n" "vote-status %s\n" - "consensus-methods 1 2 3 4 5\n" + "consensus-methods %s\n" "published %s\n" "valid-after %s\n" "fresh-until %s\n" @@ -111,24 +175,38 @@ format_networkstatus_vote(crypto_pk_env_t *private_signing_key, "voting-delay %d %d\n" "%s" /* versions */ "known-flags %s\n" + "params %s\n" "dir-source %s %s %s %s %d %d\n" "contact %s\n", v3_ns->type == NS_TYPE_VOTE ? "vote" : "opinion", + methods, published, va, fu, vu, v3_ns->vote_seconds, v3_ns->dist_seconds, version_lines, flags, + params, voter->nickname, fingerprint, voter->address, - ipaddr, voter->dir_port, voter->or_port, voter->contact); + ipaddr, voter->dir_port, voter->or_port, voter->contact); + if (r < 0) { + log_err(LD_BUG, "Insufficient memory for network status line"); + tor_assert(0); + } + + tor_free(params); tor_free(flags); + tor_free(methods); outp = status + strlen(status); endp = status + len; if (!tor_digest_is_zero(voter->legacy_id_digest)) { char fpbuf[HEX_DIGEST_LEN+1]; base16_encode(fpbuf, sizeof(fpbuf), voter->legacy_id_digest, DIGEST_LEN); - tor_snprintf(outp, endp-outp, "legacy-dir-key %s\n", fpbuf); + r = tor_snprintf(outp, endp-outp, "legacy-dir-key %s\n", fpbuf); + if (r < 0) { + log_err(LD_BUG, "Insufficient memory for legacy-dir-key line"); + tor_assert(0); + } outp += strlen(outp); } @@ -139,15 +217,32 @@ format_networkstatus_vote(crypto_pk_env_t *private_signing_key, outp += cert->cache_info.signed_descriptor_len; } - SMARTLIST_FOREACH(v3_ns->routerstatus_list, vote_routerstatus_t *, vrs, - { + SMARTLIST_FOREACH_BEGIN(v3_ns->routerstatus_list, vote_routerstatus_t *, + vrs) { + vote_microdesc_hash_t *h; if (routerstatus_format_entry(outp, endp-outp, &vrs->status, - vrs->version, 0, 0) < 0) { + vrs->version, NS_V3_VOTE) < 0) { log_warn(LD_BUG, "Unable to print router status."); goto err; } outp += strlen(outp); - }); + + for (h = vrs->microdesc; h; h = h->next) { + size_t mlen = strlen(h->microdesc_hash_line); + if (outp+mlen >= endp) { + log_warn(LD_BUG, "Can't fit microdesc line in vote."); + } + memcpy(outp, h->microdesc_hash_line, mlen+1); + outp += strlen(outp); + } + } SMARTLIST_FOREACH_END(vrs); + + r = tor_snprintf(outp, endp-outp, "directory-footer\n"); + if (r < 0) { + log_err(LD_BUG, "Insufficient memory for directory-footer line"); + tor_assert(0); + } + outp += strlen(outp); { char signing_key_fingerprint[FINGERPRINT_LEN+1]; @@ -170,10 +265,10 @@ format_networkstatus_vote(crypto_pk_env_t *private_signing_key, outp += strlen(outp); } - if (router_get_networkstatus_v3_hash(status, digest)<0) + if (router_get_networkstatus_v3_hash(status, digest, DIGEST_SHA1)<0) goto err; note_crypto_pk_op(SIGN_DIR); - if (router_append_dirobj_signature(outp,endp-outp,digest, + if (router_append_dirobj_signature(outp,endp-outp,digest, DIGEST_LEN, private_signing_key)<0) { log_warn(LD_BUG, "Unable to sign networkstatus vote."); goto err; @@ -216,6 +311,20 @@ get_voter(const networkstatus_t *vote) return smartlist_get(vote->voters, 0); } +/** Return the signature made by <b>voter</b> using the algorithm + * <b>alg</b>, or NULL if none is found. */ +document_signature_t * +voter_get_sig_by_algorithm(const networkstatus_voter_info_t *voter, + digest_algorithm_t alg) +{ + if (!voter->sigs) + return NULL; + SMARTLIST_FOREACH(voter->sigs, document_signature_t *, sig, + if (sig->alg == alg) + return sig); + return NULL; +} + /** Temporary structure used in constructing a list of dir-source entries * for a consensus. One of these is generated for every vote, and one more * for every legacy key in each vote. */ @@ -275,34 +384,8 @@ get_frequent_members(smartlist_t *out, smartlist_t *in, int min) /** Given a sorted list of strings <b>lst</b>, return the member that appears * most. Break ties in favor of later-occurring members. */ -static const char * -get_most_frequent_member(smartlist_t *lst) -{ - const char *most_frequent = NULL; - int most_frequent_count = 0; - - const char *cur = NULL; - int count = 0; - - SMARTLIST_FOREACH(lst, const char *, s, - { - if (cur && !strcmp(s, cur)) { - ++count; - } else { - if (count >= most_frequent_count) { - most_frequent = cur; - most_frequent_count = count; - } - cur = s; - count = 1; - } - }); - if (count >= most_frequent_count) { - most_frequent = cur; - most_frequent_count = count; - } - return most_frequent; -} +#define get_most_frequent_member(lst) \ + smartlist_get_most_frequent_string(lst) /** Return 0 if and only if <b>a</b> and <b>b</b> are routerstatuses * that come from the same routerinfo, with the same derived elements. @@ -344,7 +427,8 @@ _compare_vote_rs(const void **_a, const void **_b) * in favor of smaller descriptor digest. */ static vote_routerstatus_t * -compute_routerstatus_consensus(smartlist_t *votes) +compute_routerstatus_consensus(smartlist_t *votes, int consensus_method, + char *microdesc_digest256_out) { vote_routerstatus_t *most = NULL, *cur = NULL; int most_n = 0, cur_n = 0; @@ -360,9 +444,9 @@ compute_routerstatus_consensus(smartlist_t *votes) if (cur && !compare_vote_rs(cur, rs)) { ++cur_n; } else { - if (cur_n > most_n || - (cur && cur_n == most_n && - cur->status.published_on > most_published)) { + if (cur && (cur_n > most_n || + (cur_n == most_n && + cur->status.published_on > most_published))) { most = cur; most_n = cur_n; most_published = cur->status.published_on; @@ -380,18 +464,45 @@ compute_routerstatus_consensus(smartlist_t *votes) } tor_assert(most); + + if (consensus_method >= MIN_METHOD_FOR_MICRODESC && + microdesc_digest256_out) { + smartlist_t *digests = smartlist_create(); + const char *best_microdesc_digest; + SMARTLIST_FOREACH_BEGIN(votes, vote_routerstatus_t *, rs) { + char d[DIGEST256_LEN]; + if (compare_vote_rs(rs, most)) + continue; + if (!vote_routerstatus_find_microdesc_hash(d, rs, consensus_method, + DIGEST_SHA256)) + smartlist_add(digests, tor_memdup(d, sizeof(d))); + } SMARTLIST_FOREACH_END(rs); + smartlist_sort_digests256(digests); + best_microdesc_digest = smartlist_get_most_frequent_digest256(digests); + if (best_microdesc_digest) + memcpy(microdesc_digest256_out, best_microdesc_digest, DIGEST256_LEN); + SMARTLIST_FOREACH(digests, char *, cp, tor_free(cp)); + smartlist_free(digests); + } + return most; } -/** Given a list of strings in <b>lst</b>, set the DIGEST_LEN-byte digest at - * <b>digest_out</b> to the hash of the concatenation of those strings. */ +/** Given a list of strings in <b>lst</b>, set the <b>len_out</b>-byte digest + * at <b>digest_out</b> to the hash of the concatenation of those strings, + * computed with the algorithm <b>alg</b>. */ static void -hash_list_members(char *digest_out, smartlist_t *lst) +hash_list_members(char *digest_out, size_t len_out, + smartlist_t *lst, digest_algorithm_t alg) { - crypto_digest_env_t *d = crypto_new_digest_env(); + crypto_digest_env_t *d; + if (alg == DIGEST_SHA1) + d = crypto_new_digest_env(); + else + d = crypto_new_digest256_env(alg); SMARTLIST_FOREACH(lst, const char *, cp, crypto_digest_add_bytes(d, cp, strlen(cp))); - crypto_digest_get_digest(d, digest_out, DIGEST_LEN); + crypto_digest_get_digest(d, digest_out, len_out); crypto_free_digest_env(d); } @@ -455,7 +566,31 @@ compute_consensus_method(smartlist_t *votes) static int consensus_method_is_supported(int method) { - return (method >= 1) && (method <= 5); + return (method >= 1) && (method <= MAX_SUPPORTED_CONSENSUS_METHOD); +} + +/** Return a newly allocated string holding the numbers between low and high + * (inclusive) that are supported consensus methods. */ +static char * +make_consensus_method_list(int low, int high, const char *separator) +{ + char *list; + + char b[32]; + int i; + smartlist_t *lst; + lst = smartlist_create(); + for (i = low; i <= high; ++i) { + if (!consensus_method_is_supported(i)) + continue; + tor_snprintf(b, sizeof(b), "%d", i); + smartlist_add(lst, tor_strdup(b)); + } + list = smartlist_join_strings(lst, separator, 0, NULL); + tor_assert(list); + SMARTLIST_FOREACH(lst, char *, cp, tor_free(cp)); + smartlist_free(lst); + return list; } /** Helper: given <b>lst</b>, a list of version strings such that every @@ -475,6 +610,728 @@ compute_consensus_versions_list(smartlist_t *lst, int n_versioning) return result; } +/** Helper: given a list of valid networkstatus_t, return a new string + * containing the contents of the consensus network parameter set. + */ +/* private */ char * +dirvote_compute_params(smartlist_t *votes) +{ + int i; + int32_t *vals; + + int cur_param_len; + const char *cur_param; + const char *eq; + char *result; + + const int n_votes = smartlist_len(votes); + smartlist_t *output; + smartlist_t *param_list = smartlist_create(); + + /* We require that the parameter lists in the votes are well-formed: that + is, that their keywords are unique and sorted, and that their values are + between INT32_MIN and INT32_MAX inclusive. This should be guaranteed by + the parsing code. */ + + vals = tor_malloc(sizeof(int)*n_votes); + + SMARTLIST_FOREACH_BEGIN(votes, networkstatus_t *, v) { + if (!v->net_params) + continue; + smartlist_add_all(param_list, v->net_params); + } SMARTLIST_FOREACH_END(v); + + if (smartlist_len(param_list) == 0) { + tor_free(vals); + smartlist_free(param_list); + return NULL; + } + + smartlist_sort_strings(param_list); + i = 0; + cur_param = smartlist_get(param_list, 0); + eq = strchr(cur_param, '='); + tor_assert(eq); + cur_param_len = (int)(eq+1 - cur_param); + + output = smartlist_create(); + + SMARTLIST_FOREACH_BEGIN(param_list, const char *, param) { + const char *next_param; + int ok=0; + eq = strchr(param, '='); + tor_assert(i<n_votes); + vals[i++] = (int32_t) + tor_parse_long(eq+1, 10, INT32_MIN, INT32_MAX, &ok, NULL); + tor_assert(ok); + + if (param_sl_idx+1 == smartlist_len(param_list)) + next_param = NULL; + else + next_param = smartlist_get(param_list, param_sl_idx+1); + if (!next_param || strncmp(next_param, param, cur_param_len)) { + /* We've reached the end of a series. */ + int32_t median = median_int32(vals, i); + char *out_string = tor_malloc(64+cur_param_len); + memcpy(out_string, param, cur_param_len); + tor_snprintf(out_string+cur_param_len,64, "%ld", (long)median); + smartlist_add(output, out_string); + + i = 0; + if (next_param) { + eq = strchr(next_param, '='); + cur_param_len = (int)(eq+1 - next_param); + } + } + } SMARTLIST_FOREACH_END(param); + + result = smartlist_join_strings(output, " ", 0, NULL); + SMARTLIST_FOREACH(output, char *, cp, tor_free(cp)); + smartlist_free(output); + smartlist_free(param_list); + tor_free(vals); + return result; +} + +#define RANGE_CHECK(a,b,c,d,e,f,g,mx) \ + ((a) >= 0 && (a) <= (mx) && (b) >= 0 && (b) <= (mx) && \ + (c) >= 0 && (c) <= (mx) && (d) >= 0 && (d) <= (mx) && \ + (e) >= 0 && (e) <= (mx) && (f) >= 0 && (f) <= (mx) && \ + (g) >= 0 && (g) <= (mx)) + +#define CHECK_EQ(a, b, margin) \ + ((a)-(b) >= 0 ? (a)-(b) <= (margin) : (b)-(a) <= (margin)) + +typedef enum { + BW_WEIGHTS_NO_ERROR = 0, + BW_WEIGHTS_RANGE_ERROR = 1, + BW_WEIGHTS_SUMG_ERROR = 2, + BW_WEIGHTS_SUME_ERROR = 3, + BW_WEIGHTS_SUMD_ERROR = 4, + BW_WEIGHTS_BALANCE_MID_ERROR = 5, + BW_WEIGHTS_BALANCE_EG_ERROR = 6 +} bw_weights_error_t; + +/** + * Verify that any weightings satisfy the balanced formulas. + */ +static bw_weights_error_t +networkstatus_check_weights(int64_t Wgg, int64_t Wgd, int64_t Wmg, + int64_t Wme, int64_t Wmd, int64_t Wee, + int64_t Wed, int64_t scale, int64_t G, + int64_t M, int64_t E, int64_t D, int64_t T, + int64_t margin, int do_balance) { + bw_weights_error_t berr = BW_WEIGHTS_NO_ERROR; + + // Wed + Wmd + Wgd == 1 + if (!CHECK_EQ(Wed + Wmd + Wgd, scale, margin)) { + berr = BW_WEIGHTS_SUMD_ERROR; + goto out; + } + + // Wmg + Wgg == 1 + if (!CHECK_EQ(Wmg + Wgg, scale, margin)) { + berr = BW_WEIGHTS_SUMG_ERROR; + goto out; + } + + // Wme + Wee == 1 + if (!CHECK_EQ(Wme + Wee, scale, margin)) { + berr = BW_WEIGHTS_SUME_ERROR; + goto out; + } + + // Verify weights within range 0->1 + if (!RANGE_CHECK(Wgg, Wgd, Wmg, Wme, Wmd, Wed, Wee, scale)) { + berr = BW_WEIGHTS_RANGE_ERROR; + goto out; + } + + if (do_balance) { + // Wgg*G + Wgd*D == Wee*E + Wed*D, already scaled + if (!CHECK_EQ(Wgg*G + Wgd*D, Wee*E + Wed*D, (margin*T)/3)) { + berr = BW_WEIGHTS_BALANCE_EG_ERROR; + goto out; + } + + // Wgg*G + Wgd*D == M*scale + Wmd*D + Wme*E + Wmg*G, already scaled + if (!CHECK_EQ(Wgg*G + Wgd*D, M*scale + Wmd*D + Wme*E + Wmg*G, + (margin*T)/3)) { + berr = BW_WEIGHTS_BALANCE_MID_ERROR; + goto out; + } + } + + out: + if (berr) { + log_info(LD_DIR, + "Bw weight mismatch %d. G="I64_FORMAT" M="I64_FORMAT + " E="I64_FORMAT" D="I64_FORMAT" T="I64_FORMAT + " Wmd=%d Wme=%d Wmg=%d Wed=%d Wee=%d" + " Wgd=%d Wgg=%d Wme=%d Wmg=%d", + berr, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + (int)Wmd, (int)Wme, (int)Wmg, (int)Wed, (int)Wee, + (int)Wgd, (int)Wgg, (int)Wme, (int)Wmg); + } + + return berr; +} + +/** + * This function computes the bandwidth weights for consensus method 10. + * + * It returns true if weights could be computed, false otherwise. + */ +static int +networkstatus_compute_bw_weights_v10(smartlist_t *chunks, int64_t G, + int64_t M, int64_t E, int64_t D, + int64_t T, int64_t weight_scale) +{ + bw_weights_error_t berr = 0; + int64_t Wgg = -1, Wgd = -1; + int64_t Wmg = -1, Wme = -1, Wmd = -1; + int64_t Wed = -1, Wee = -1; + const char *casename; + char buf[512]; + int r; + + if (G <= 0 || M <= 0 || E <= 0 || D <= 0) { + log_warn(LD_DIR, "Consensus with empty bandwidth: " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT + " D="I64_FORMAT" T="I64_FORMAT, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); + return 0; + } + + /* + * Computed from cases in 3.4.3 of dir-spec.txt + * + * 1. Neither are scarce + * 2. Both Guard and Exit are scarce + * a. R+D <= S + * b. R+D > S + * 3. One of Guard or Exit is scarce + * a. S+D < T/3 + * b. S+D >= T/3 + */ + if (3*E >= T && 3*G >= T) { // E >= T/3 && G >= T/3 + /* Case 1: Neither are scarce. */ + casename = "Case 1 (Wgd=Wmd=Wed)"; + Wgd = weight_scale/3; + Wed = weight_scale/3; + Wmd = weight_scale/3; + Wee = (weight_scale*(E+G+M))/(3*E); + Wme = weight_scale - Wee; + Wmg = (weight_scale*(2*G-E-M))/(3*G); + Wgg = weight_scale - Wmg; + + berr = networkstatus_check_weights(Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed, + weight_scale, G, M, E, D, T, 10, 1); + + if (berr) { + log_warn(LD_DIR, + "Bw Weights error %d for %s v10. G="I64_FORMAT" M="I64_FORMAT + " E="I64_FORMAT" D="I64_FORMAT" T="I64_FORMAT + " Wmd=%d Wme=%d Wmg=%d Wed=%d Wee=%d" + " Wgd=%d Wgg=%d Wme=%d Wmg=%d weight_scale=%d", + berr, casename, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + (int)Wmd, (int)Wme, (int)Wmg, (int)Wed, (int)Wee, + (int)Wgd, (int)Wgg, (int)Wme, (int)Wmg, (int)weight_scale); + return 0; + } + } else if (3*E < T && 3*G < T) { // E < T/3 && G < T/3 + int64_t R = MIN(E, G); + int64_t S = MAX(E, G); + /* + * Case 2: Both Guards and Exits are scarce + * Balance D between E and G, depending upon + * D capacity and scarcity. + */ + if (R+D < S) { // Subcase a + Wgg = weight_scale; + Wee = weight_scale; + Wmg = 0; + Wme = 0; + Wmd = 0; + if (E < G) { + casename = "Case 2a (E scarce)"; + Wed = weight_scale; + Wgd = 0; + } else { /* E >= G */ + casename = "Case 2a (G scarce)"; + Wed = 0; + Wgd = weight_scale; + } + } else { // Subcase b: R+D >= S + casename = "Case 2b1 (Wgg=1, Wmd=Wgd)"; + Wee = (weight_scale*(E - G + M))/E; + Wed = (weight_scale*(D - 2*E + 4*G - 2*M))/(3*D); + Wme = (weight_scale*(G-M))/E; + Wmg = 0; + Wgg = weight_scale; + Wmd = (weight_scale - Wed)/2; + Wgd = (weight_scale - Wed)/2; + + berr = networkstatus_check_weights(Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed, + weight_scale, G, M, E, D, T, 10, 1); + + if (berr) { + casename = "Case 2b2 (Wgg=1, Wee=1)"; + Wgg = weight_scale; + Wee = weight_scale; + Wed = (weight_scale*(D - 2*E + G + M))/(3*D); + Wmd = (weight_scale*(D - 2*M + G + E))/(3*D); + Wme = 0; + Wmg = 0; + + if (Wmd < 0) { // Can happen if M > T/3 + casename = "Case 2b3 (Wmd=0)"; + Wmd = 0; + log_warn(LD_DIR, + "Too much Middle bandwidth on the network to calculate " + "balanced bandwidth-weights. Consider increasing the " + "number of Guard nodes by lowering the requirements."); + } + Wgd = weight_scale - Wed - Wmd; + berr = networkstatus_check_weights(Wgg, Wgd, Wmg, Wme, Wmd, Wee, + Wed, weight_scale, G, M, E, D, T, 10, 1); + } + if (berr != BW_WEIGHTS_NO_ERROR && + berr != BW_WEIGHTS_BALANCE_MID_ERROR) { + log_warn(LD_DIR, + "Bw Weights error %d for %s v10. G="I64_FORMAT" M="I64_FORMAT + " E="I64_FORMAT" D="I64_FORMAT" T="I64_FORMAT + " Wmd=%d Wme=%d Wmg=%d Wed=%d Wee=%d" + " Wgd=%d Wgg=%d Wme=%d Wmg=%d weight_scale=%d", + berr, casename, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + (int)Wmd, (int)Wme, (int)Wmg, (int)Wed, (int)Wee, + (int)Wgd, (int)Wgg, (int)Wme, (int)Wmg, (int)weight_scale); + return 0; + } + } + } else { // if (E < T/3 || G < T/3) { + int64_t S = MIN(E, G); + // Case 3: Exactly one of Guard or Exit is scarce + if (!(3*E < T || 3*G < T) || !(3*G >= T || 3*E >= T)) { + log_warn(LD_BUG, + "Bw-Weights Case 3 v10 but with G="I64_FORMAT" M=" + I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT" T="I64_FORMAT, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); + } + + if (3*(S+D) < T) { // Subcase a: S+D < T/3 + if (G < E) { + casename = "Case 3a (G scarce)"; + Wgg = Wgd = weight_scale; + Wmd = Wed = Wmg = 0; + // Minor subcase, if E is more scarce than M, + // keep its bandwidth in place. + if (E < M) Wme = 0; + else Wme = (weight_scale*(E-M))/(2*E); + Wee = weight_scale-Wme; + } else { // G >= E + casename = "Case 3a (E scarce)"; + Wee = Wed = weight_scale; + Wmd = Wgd = Wme = 0; + // Minor subcase, if G is more scarce than M, + // keep its bandwidth in place. + if (G < M) Wmg = 0; + else Wmg = (weight_scale*(G-M))/(2*G); + Wgg = weight_scale-Wmg; + } + } else { // Subcase b: S+D >= T/3 + // D != 0 because S+D >= T/3 + if (G < E) { + casename = "Case 3bg (G scarce, Wgg=1, Wmd == Wed)"; + Wgg = weight_scale; + Wgd = (weight_scale*(D - 2*G + E + M))/(3*D); + Wmg = 0; + Wee = (weight_scale*(E+M))/(2*E); + Wme = weight_scale - Wee; + Wmd = (weight_scale - Wgd)/2; + Wed = (weight_scale - Wgd)/2; + + berr = networkstatus_check_weights(Wgg, Wgd, Wmg, Wme, Wmd, Wee, + Wed, weight_scale, G, M, E, D, T, 10, 1); + } else { // G >= E + casename = "Case 3be (E scarce, Wee=1, Wmd == Wgd)"; + Wee = weight_scale; + Wed = (weight_scale*(D - 2*E + G + M))/(3*D); + Wme = 0; + Wgg = (weight_scale*(G+M))/(2*G); + Wmg = weight_scale - Wgg; + Wmd = (weight_scale - Wed)/2; + Wgd = (weight_scale - Wed)/2; + + berr = networkstatus_check_weights(Wgg, Wgd, Wmg, Wme, Wmd, Wee, + Wed, weight_scale, G, M, E, D, T, 10, 1); + } + if (berr) { + log_warn(LD_DIR, + "Bw Weights error %d for %s v10. G="I64_FORMAT" M="I64_FORMAT + " E="I64_FORMAT" D="I64_FORMAT" T="I64_FORMAT + " Wmd=%d Wme=%d Wmg=%d Wed=%d Wee=%d" + " Wgd=%d Wgg=%d Wme=%d Wmg=%d weight_scale=%d", + berr, casename, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + (int)Wmd, (int)Wme, (int)Wmg, (int)Wed, (int)Wee, + (int)Wgd, (int)Wgg, (int)Wme, (int)Wmg, (int)weight_scale); + return 0; + } + } + } + + /* We cast down the weights to 32 bit ints on the assumption that + * weight_scale is ~= 10000. We need to ensure a rogue authority + * doesn't break this assumption to rig our weights */ + tor_assert(0 < weight_scale && weight_scale < INT32_MAX); + + /* + * Provide Wgm=Wgg, Wmm=1, Wem=Wee, Weg=Wed. May later determine + * that middle nodes need different bandwidth weights for dirport traffic, + * or that weird exit policies need special weight, or that bridges + * need special weight. + * + * NOTE: This list is sorted. + */ + r = tor_snprintf(buf, sizeof(buf), + "bandwidth-weights Wbd=%d Wbe=%d Wbg=%d Wbm=%d " + "Wdb=%d " + "Web=%d Wed=%d Wee=%d Weg=%d Wem=%d " + "Wgb=%d Wgd=%d Wgg=%d Wgm=%d " + "Wmb=%d Wmd=%d Wme=%d Wmg=%d Wmm=%d\n", + (int)Wmd, (int)Wme, (int)Wmg, (int)weight_scale, + (int)weight_scale, + (int)weight_scale, (int)Wed, (int)Wee, (int)Wed, (int)Wee, + (int)weight_scale, (int)Wgd, (int)Wgg, (int)Wgg, + (int)weight_scale, (int)Wmd, (int)Wme, (int)Wmg, (int)weight_scale); + if (r<0) { + log_warn(LD_BUG, + "Not enough space in buffer for bandwidth-weights line."); + *buf = '\0'; + return 0; + } + smartlist_add(chunks, tor_strdup(buf)); + + log_notice(LD_CIRC, "Computed bandwidth weights for %s with v10: " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT, + casename, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); + return 1; +} +/** + * This function computes the bandwidth weights for consensus method 9. + * + * It has been obsoleted in favor of consensus method 10. + */ +static void +networkstatus_compute_bw_weights_v9(smartlist_t *chunks, int64_t G, int64_t M, + int64_t E, int64_t D, int64_t T, + int64_t weight_scale) +{ + int64_t Wgg = -1, Wgd = -1; + int64_t Wmg = -1, Wme = -1, Wmd = -1; + int64_t Wed = -1, Wee = -1; + const char *casename; + char buf[512]; + int r; + + if (G <= 0 || M <= 0 || E <= 0 || D <= 0) { + log_warn(LD_DIR, "Consensus with empty bandwidth: " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT + " D="I64_FORMAT" T="I64_FORMAT, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); + return; + } + + /* + * Computed from cases in 3.4.3 of dir-spec.txt + * + * 1. Neither are scarce + * 2. Both Guard and Exit are scarce + * a. R+D <= S + * b. R+D > S + * 3. One of Guard or Exit is scarce + * a. S+D < T/3 + * b. S+D >= T/3 + */ + if (3*E >= T && 3*G >= T) { // E >= T/3 && G >= T/3 + bw_weights_error_t berr = 0; + /* Case 1: Neither are scarce. + * + * Attempt to ensure that we have a large amount of exit bandwidth + * in the middle position. + */ + casename = "Case 1 (Wme*E = Wmd*D)"; + Wgg = (weight_scale*(D+E+G+M))/(3*G); + if (D==0) Wmd = 0; + else Wmd = (weight_scale*(2*D + 2*E - G - M))/(6*D); + Wme = (weight_scale*(2*D + 2*E - G - M))/(6*E); + Wee = (weight_scale*(-2*D + 4*E + G + M))/(6*E); + Wgd = 0; + Wmg = weight_scale - Wgg; + Wed = weight_scale - Wmd; + + berr = networkstatus_check_weights(Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed, + weight_scale, G, M, E, D, T, 10, 1); + + if (berr) { + log_warn(LD_DIR, "Bw Weights error %d for case %s. " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT + " D="I64_FORMAT" T="I64_FORMAT, + berr, casename, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); + } + } else if (3*E < T && 3*G < T) { // E < T/3 && G < T/3 + int64_t R = MIN(E, G); + int64_t S = MAX(E, G); + /* + * Case 2: Both Guards and Exits are scarce + * Balance D between E and G, depending upon + * D capacity and scarcity. + */ + if (R+D < S) { // Subcase a + Wgg = weight_scale; + Wee = weight_scale; + Wmg = 0; + Wme = 0; + Wmd = 0; + if (E < G) { + casename = "Case 2a (E scarce)"; + Wed = weight_scale; + Wgd = 0; + } else { /* E >= G */ + casename = "Case 2a (G scarce)"; + Wed = 0; + Wgd = weight_scale; + } + } else { // Subcase b: R+D > S + bw_weights_error_t berr = 0; + casename = "Case 2b (Wme*E == Wmd*D)"; + if (D != 0) { + Wgg = weight_scale; + Wgd = (weight_scale*(D + E - 2*G + M))/(3*D); // T/3 >= G (Ok) + Wmd = (weight_scale*(D + E + G - 2*M))/(6*D); // T/3 >= M + Wme = (weight_scale*(D + E + G - 2*M))/(6*E); + Wee = (weight_scale*(-D + 5*E - G + 2*M))/(6*E); // 2E+M >= T/3 + Wmg = 0; + Wed = weight_scale - Wgd - Wmd; + + berr = networkstatus_check_weights(Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed, + weight_scale, G, M, E, D, T, 10, 1); + } + + if (D == 0 || berr) { // Can happen if M > T/3 + casename = "Case 2b (E=G)"; + Wgg = weight_scale; + Wee = weight_scale; + Wmg = 0; + Wme = 0; + Wmd = 0; + if (D == 0) Wgd = 0; + else Wgd = (weight_scale*(D+E-G))/(2*D); + Wed = weight_scale - Wgd; + berr = networkstatus_check_weights(Wgg, Wgd, Wmg, Wme, Wmd, Wee, + Wed, weight_scale, G, M, E, D, T, 10, 1); + } + if (berr != BW_WEIGHTS_NO_ERROR && + berr != BW_WEIGHTS_BALANCE_MID_ERROR) { + log_warn(LD_DIR, "Bw Weights error %d for case %s. " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT + " D="I64_FORMAT" T="I64_FORMAT, + berr, casename, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); + } + } + } else { // if (E < T/3 || G < T/3) { + int64_t S = MIN(E, G); + // Case 3: Exactly one of Guard or Exit is scarce + if (!(3*E < T || 3*G < T) || !(3*G >= T || 3*E >= T)) { + log_warn(LD_BUG, + "Bw-Weights Case 3 but with G="I64_FORMAT" M=" + I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT" T="I64_FORMAT, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); + } + + if (3*(S+D) < T) { // Subcase a: S+D < T/3 + if (G < E) { + casename = "Case 3a (G scarce)"; + Wgg = Wgd = weight_scale; + Wmd = Wed = Wmg = 0; + // Minor subcase, if E is more scarce than M, + // keep its bandwidth in place. + if (E < M) Wme = 0; + else Wme = (weight_scale*(E-M))/(2*E); + Wee = weight_scale-Wme; + } else { // G >= E + casename = "Case 3a (E scarce)"; + Wee = Wed = weight_scale; + Wmd = Wgd = Wme = 0; + // Minor subcase, if G is more scarce than M, + // keep its bandwidth in place. + if (G < M) Wmg = 0; + else Wmg = (weight_scale*(G-M))/(2*G); + Wgg = weight_scale-Wmg; + } + } else { // Subcase b: S+D >= T/3 + bw_weights_error_t berr = 0; + // D != 0 because S+D >= T/3 + if (G < E) { + casename = "Case 3b (G scarce, Wme*E == Wmd*D)"; + Wgd = (weight_scale*(D + E - 2*G + M))/(3*D); + Wmd = (weight_scale*(D + E + G - 2*M))/(6*D); + Wme = (weight_scale*(D + E + G - 2*M))/(6*E); + Wee = (weight_scale*(-D + 5*E - G + 2*M))/(6*E); + Wgg = weight_scale; + Wmg = 0; + Wed = weight_scale - Wgd - Wmd; + + berr = networkstatus_check_weights(Wgg, Wgd, Wmg, Wme, Wmd, Wee, + Wed, weight_scale, G, M, E, D, T, 10, 1); + } else { // G >= E + casename = "Case 3b (E scarce, Wme*E == Wmd*D)"; + Wgg = (weight_scale*(D + E + G + M))/(3*G); + Wmd = (weight_scale*(2*D + 2*E - G - M))/(6*D); + Wme = (weight_scale*(2*D + 2*E - G - M))/(6*E); + Wee = (weight_scale*(-2*D + 4*E + G + M))/(6*E); + Wgd = 0; + Wmg = weight_scale - Wgg; + Wed = weight_scale - Wmd; + + berr = networkstatus_check_weights(Wgg, Wgd, Wmg, Wme, Wmd, Wee, + Wed, weight_scale, G, M, E, D, T, 10, 1); + } + if (berr) { + log_warn(LD_DIR, "Bw Weights error %d for case %s. " + "G="I64_FORMAT" M="I64_FORMAT + " E="I64_FORMAT" D="I64_FORMAT" T="I64_FORMAT, + berr, casename, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); + } + } + } + + /* We cast down the weights to 32 bit ints on the assumption that + * weight_scale is ~= 10000. We need to ensure a rogue authority + * doesn't break this assumption to rig our weights */ + tor_assert(0 < weight_scale && weight_scale < INT32_MAX); + + if (Wgg < 0 || Wgg > weight_scale) { + log_warn(LD_DIR, "Bw %s: Wgg="I64_FORMAT"! G="I64_FORMAT + " M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT, + casename, I64_PRINTF_ARG(Wgg), + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); + + Wgg = MAX(MIN(Wgg, weight_scale), 0); + } + if (Wgd < 0 || Wgd > weight_scale) { + log_warn(LD_DIR, "Bw %s: Wgd="I64_FORMAT"! G="I64_FORMAT + " M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT, + casename, I64_PRINTF_ARG(Wgd), + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); + Wgd = MAX(MIN(Wgd, weight_scale), 0); + } + if (Wmg < 0 || Wmg > weight_scale) { + log_warn(LD_DIR, "Bw %s: Wmg="I64_FORMAT"! G="I64_FORMAT + " M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT, + casename, I64_PRINTF_ARG(Wmg), + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); + Wmg = MAX(MIN(Wmg, weight_scale), 0); + } + if (Wme < 0 || Wme > weight_scale) { + log_warn(LD_DIR, "Bw %s: Wme="I64_FORMAT"! G="I64_FORMAT + " M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT, + casename, I64_PRINTF_ARG(Wme), + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); + Wme = MAX(MIN(Wme, weight_scale), 0); + } + if (Wmd < 0 || Wmd > weight_scale) { + log_warn(LD_DIR, "Bw %s: Wmd="I64_FORMAT"! G="I64_FORMAT + " M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT, + casename, I64_PRINTF_ARG(Wmd), + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); + Wmd = MAX(MIN(Wmd, weight_scale), 0); + } + if (Wee < 0 || Wee > weight_scale) { + log_warn(LD_DIR, "Bw %s: Wee="I64_FORMAT"! G="I64_FORMAT + " M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT, + casename, I64_PRINTF_ARG(Wee), + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); + Wee = MAX(MIN(Wee, weight_scale), 0); + } + if (Wed < 0 || Wed > weight_scale) { + log_warn(LD_DIR, "Bw %s: Wed="I64_FORMAT"! G="I64_FORMAT + " M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT, + casename, I64_PRINTF_ARG(Wed), + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); + Wed = MAX(MIN(Wed, weight_scale), 0); + } + + // Add consensus weight keywords + smartlist_add(chunks, tor_strdup("bandwidth-weights ")); + /* + * Provide Wgm=Wgg, Wmm=1, Wem=Wee, Weg=Wed. May later determine + * that middle nodes need different bandwidth weights for dirport traffic, + * or that weird exit policies need special weight, or that bridges + * need special weight. + * + * NOTE: This list is sorted. + */ + r = tor_snprintf(buf, sizeof(buf), + "Wbd=%d Wbe=%d Wbg=%d Wbm=%d " + "Wdb=%d " + "Web=%d Wed=%d Wee=%d Weg=%d Wem=%d " + "Wgb=%d Wgd=%d Wgg=%d Wgm=%d " + "Wmb=%d Wmd=%d Wme=%d Wmg=%d Wmm=%d\n", + (int)Wmd, (int)Wme, (int)Wmg, (int)weight_scale, + (int)weight_scale, + (int)weight_scale, (int)Wed, (int)Wee, (int)Wed, (int)Wee, + (int)weight_scale, (int)Wgd, (int)Wgg, (int)Wgg, + (int)weight_scale, (int)Wmd, (int)Wme, (int)Wmg, (int)weight_scale); + if (r<0) { + log_warn(LD_BUG, + "Not enough space in buffer for bandwidth-weights line."); + *buf = '\0'; + } + smartlist_add(chunks, tor_strdup(buf)); + log_notice(LD_CIRC, "Computed bandwidth weights for %s with v9: " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT, + casename, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); +} + /** Given a list of vote networkstatus_t in <b>votes</b>, our public * authority <b>identity_key</b>, our private authority <b>signing_key</b>, * and the number of <b>total_authorities</b> that we believe exist in our @@ -489,18 +1346,27 @@ networkstatus_compute_consensus(smartlist_t *votes, crypto_pk_env_t *identity_key, crypto_pk_env_t *signing_key, const char *legacy_id_key_digest, - crypto_pk_env_t *legacy_signing_key) + crypto_pk_env_t *legacy_signing_key, + consensus_flavor_t flavor) { smartlist_t *chunks; char *result = NULL; int consensus_method; - time_t valid_after, fresh_until, valid_until; int vote_seconds, dist_seconds; char *client_versions = NULL, *server_versions = NULL; smartlist_t *flags; + const char *flavor_name; + int64_t G=0, M=0, E=0, D=0, T=0; /* For bandwidth weights */ + const routerstatus_format_type_t rs_format = + flavor == FLAV_NS ? NS_V3_CONSENSUS : NS_V3_CONSENSUS_MICRODESC; + char *params = NULL; + int added_weights = 0; + tor_assert(flavor == FLAV_NS || flavor == FLAV_MICRODESC); tor_assert(total_authorities >= smartlist_len(votes)); + flavor_name = networkstatus_get_flavor_name(flavor); + if (!smartlist_len(votes)) { log_warn(LD_DIR, "Can't compute a consensus from no votes."); return NULL; @@ -593,7 +1459,7 @@ networkstatus_compute_consensus(smartlist_t *votes, chunks = smartlist_create(); { - char buf[1024]; + char *buf=NULL; char va_buf[ISO_TIME_LEN+1], fu_buf[ISO_TIME_LEN+1], vu_buf[ISO_TIME_LEN+1]; char *flaglist; @@ -602,16 +1468,20 @@ networkstatus_compute_consensus(smartlist_t *votes, format_iso_time(vu_buf, valid_until); flaglist = smartlist_join_strings(flags, " ", 0, NULL); - smartlist_add(chunks, tor_strdup("network-status-version 3\n" - "vote-status consensus\n")); + tor_asprintf(&buf, "network-status-version 3%s%s\n" + "vote-status consensus\n", + flavor == FLAV_NS ? "" : " ", + flavor == FLAV_NS ? "" : flavor_name); + + smartlist_add(chunks, buf); if (consensus_method >= 2) { - tor_snprintf(buf, sizeof(buf), "consensus-method %d\n", + tor_asprintf(&buf, "consensus-method %d\n", consensus_method); - smartlist_add(chunks, tor_strdup(buf)); + smartlist_add(chunks, buf); } - tor_snprintf(buf, sizeof(buf), + tor_asprintf(&buf, "valid-after %s\n" "fresh-until %s\n" "valid-until %s\n" @@ -622,18 +1492,26 @@ networkstatus_compute_consensus(smartlist_t *votes, va_buf, fu_buf, vu_buf, vote_seconds, dist_seconds, client_versions, server_versions, flaglist); - smartlist_add(chunks, tor_strdup(buf)); + smartlist_add(chunks, buf); tor_free(flaglist); } + if (consensus_method >= MIN_METHOD_FOR_PARAMS) { + params = dirvote_compute_params(votes); + if (params) { + smartlist_add(chunks, tor_strdup("params ")); + smartlist_add(chunks, params); + smartlist_add(chunks, tor_strdup("\n")); + } + } + /* Sort the votes. */ smartlist_sort(votes, _compare_votes_by_authority_id); /* Add the authority sections. */ { smartlist_t *dir_sources = smartlist_create(); - SMARTLIST_FOREACH(votes, networkstatus_t *, v, - { + SMARTLIST_FOREACH_BEGIN(votes, networkstatus_t *, v) { dir_src_ent_t *e = tor_malloc_zero(sizeof(dir_src_ent_t)); e->v = v; e->digest = get_voter(v)->identity_digest; @@ -647,18 +1525,17 @@ networkstatus_compute_consensus(smartlist_t *votes, e_legacy->is_legacy = 1; smartlist_add(dir_sources, e_legacy); } - }); + } SMARTLIST_FOREACH_END(v); smartlist_sort(dir_sources, _compare_dir_src_ents_by_authority_id); - SMARTLIST_FOREACH(dir_sources, const dir_src_ent_t *, e, - { - char buf[1024]; + SMARTLIST_FOREACH_BEGIN(dir_sources, const dir_src_ent_t *, e) { struct in_addr in; char ip[INET_NTOA_BUF_LEN]; char fingerprint[HEX_DIGEST_LEN+1]; char votedigest[HEX_DIGEST_LEN+1]; networkstatus_t *v = e->v; networkstatus_voter_info_t *voter = get_voter(v); + char *buf = NULL; if (e->is_legacy) tor_assert(consensus_method >= 2); @@ -669,22 +1546,22 @@ networkstatus_compute_consensus(smartlist_t *votes, base16_encode(votedigest, sizeof(votedigest), voter->vote_digest, DIGEST_LEN); - tor_snprintf(buf, sizeof(buf), + tor_asprintf(&buf, "dir-source %s%s %s %s %s %d %d\n", voter->nickname, e->is_legacy ? "-legacy" : "", fingerprint, voter->address, ip, voter->dir_port, voter->or_port); - smartlist_add(chunks, tor_strdup(buf)); + smartlist_add(chunks, buf); if (! e->is_legacy) { - tor_snprintf(buf, sizeof(buf), + tor_asprintf(&buf, "contact %s\n" "vote-digest %s\n", voter->contact, votedigest); - smartlist_add(chunks, tor_strdup(buf)); + smartlist_add(chunks, buf); } - }); + } SMARTLIST_FOREACH_END(e); SMARTLIST_FOREACH(dir_sources, dir_src_ent_t *, e, tor_free(e)); smartlist_free(dir_sources); } @@ -701,7 +1578,10 @@ networkstatus_compute_consensus(smartlist_t *votes, smartlist_t *versions = smartlist_create(); smartlist_t *exitsummaries = smartlist_create(); uint32_t *bandwidths = tor_malloc(sizeof(uint32_t) * smartlist_len(votes)); + uint32_t *measured_bws = tor_malloc(sizeof(uint32_t) * + smartlist_len(votes)); int num_bandwidths; + int num_mbws; int *n_voter_flags; /* n_voter_flags[j] is the number of flags that * votes[j] knows about. */ @@ -813,10 +1693,12 @@ networkstatus_compute_consensus(smartlist_t *votes, const char *chosen_name = NULL; int exitsummary_disagreement = 0; int is_named = 0, is_unnamed = 0, is_running = 0; + int is_guard = 0, is_exit = 0, is_bad_exit = 0; int naming_conflict = 0; int n_listing = 0; int i; - char buf[256]; + char *buf=NULL; + char microdesc_digest[DIGEST256_LEN]; /* Of the next-to-be-considered digest in each voter, which is first? */ SMARTLIST_FOREACH(votes, networkstatus_t *, v, { @@ -835,6 +1717,7 @@ networkstatus_compute_consensus(smartlist_t *votes, smartlist_clear(chosen_flags); smartlist_clear(versions); num_bandwidths = 0; + num_mbws = 0; /* Okay, go through all the entries for this digest. */ SMARTLIST_FOREACH_BEGIN(votes, networkstatus_t *, v) { @@ -868,6 +1751,9 @@ networkstatus_compute_consensus(smartlist_t *votes, } /* count bandwidths */ + if (rs->status.has_measured_bw) + measured_bws[num_mbws++] = rs->status.measured_bw; + if (rs->status.has_bandwidth) bandwidths[num_bandwidths++] = rs->status.bandwidth; } SMARTLIST_FOREACH_END(v); @@ -879,7 +1765,9 @@ networkstatus_compute_consensus(smartlist_t *votes, /* Figure out the most popular opinion of what the most recent * routerinfo and its contents are. */ - rs = compute_routerstatus_consensus(matching_descs); + memset(microdesc_digest, 0, sizeof(microdesc_digest)); + rs = compute_routerstatus_consensus(matching_descs, consensus_method, + microdesc_digest); /* Copy bits of that into rs_out. */ tor_assert(fast_memeq(lowest_id, rs->status.identity_digest, DIGEST_LEN)); memcpy(rs_out.identity_digest, lowest_id, DIGEST_LEN); @@ -925,8 +1813,14 @@ networkstatus_compute_consensus(smartlist_t *votes, } else { if (flag_counts[fl_sl_idx] > n_flag_voters[fl_sl_idx]/2) { smartlist_add(chosen_flags, (char*)fl); - if (!strcmp(fl, "Running")) + if (!strcmp(fl, "Exit")) + is_exit = 1; + else if (!strcmp(fl, "Guard")) + is_guard = 1; + else if (!strcmp(fl, "Running")) is_running = 1; + else if (!strcmp(fl, "BadExit")) + is_bad_exit = 1; } } }); @@ -945,11 +1839,36 @@ networkstatus_compute_consensus(smartlist_t *votes, } /* Pick a bandwidth */ - if (consensus_method >= 5 && num_bandwidths > 0) { + if (consensus_method >= 6 && num_mbws > 2) { + rs_out.has_bandwidth = 1; + rs_out.bandwidth = median_uint32(measured_bws, num_mbws); + } else if (consensus_method >= 5 && num_bandwidths > 0) { rs_out.has_bandwidth = 1; rs_out.bandwidth = median_uint32(bandwidths, num_bandwidths); } + /* Fix bug 2203: Do not count BadExit nodes as Exits for bw weights */ + if (consensus_method >= 11) { + is_exit = is_exit && !is_bad_exit; + } + + if (consensus_method >= MIN_METHOD_FOR_BW_WEIGHTS) { + if (rs_out.has_bandwidth) { + T += rs_out.bandwidth; + if (is_exit && is_guard) + D += rs_out.bandwidth; + else if (is_exit) + E += rs_out.bandwidth; + else if (is_guard) + G += rs_out.bandwidth; + else + M += rs_out.bandwidth; + } else { + log_warn(LD_BUG, "Missing consensus bandwidth for router %s", + rs_out.nickname); + } + } + /* Ok, we already picked a descriptor digest we want to list * previously. Now we want to use the exit policy summary from * that descriptor. If everybody plays nice all the voters who @@ -1034,11 +1953,23 @@ networkstatus_compute_consensus(smartlist_t *votes, } } - /* Okay!! Now we can write the descriptor... */ - /* First line goes into "buf". */ - routerstatus_format_entry(buf, sizeof(buf), &rs_out, NULL, 1, 0); - smartlist_add(chunks, tor_strdup(buf)); - /* Second line is all flags. The "\n" is missing. */ + { + char buf[4096]; + /* Okay!! Now we can write the descriptor... */ + /* First line goes into "buf". */ + routerstatus_format_entry(buf, sizeof(buf), &rs_out, NULL, + rs_format); + smartlist_add(chunks, tor_strdup(buf)); + } + /* Now an m line, if applicable. */ + if (flavor == FLAV_MICRODESC && + !tor_digest256_is_zero(microdesc_digest)) { + char m[BASE64_DIGEST256_LEN+1], *cp; + digest256_to_base64(m, microdesc_digest); + tor_asprintf(&cp, "m %s\n", m); + smartlist_add(chunks, cp); + } + /* Next line is all flags. The "\n" is missing. */ smartlist_add(chunks, smartlist_join_strings(chosen_flags, " ", 0, NULL)); /* Now the version line. */ @@ -1049,24 +1980,16 @@ networkstatus_compute_consensus(smartlist_t *votes, smartlist_add(chunks, tor_strdup("\n")); /* Now the weight line. */ if (rs_out.has_bandwidth) { - int r = tor_snprintf(buf, sizeof(buf), - "w Bandwidth=%d\n", rs_out.bandwidth); - if (r<0) { - log_warn(LD_BUG, "Not enough space in buffer for weight line."); - *buf = '\0'; - } - smartlist_add(chunks, tor_strdup(buf)); - }; + char *cp=NULL; + tor_asprintf(&cp, "w Bandwidth=%d\n", rs_out.bandwidth); + smartlist_add(chunks, cp); + } + /* Now the exitpolicy summary line. */ - if (rs_out.has_exitsummary) { - char buf[MAX_POLICY_LINE_LEN+1]; - int r = tor_snprintf(buf, sizeof(buf), "p %s\n", rs_out.exitsummary); - if (r<0) { - log_warn(LD_BUG, "Not enough space in buffer for exitpolicy line."); - *buf = '\0'; - } - smartlist_add(chunks, tor_strdup(buf)); - }; + if (rs_out.has_exitsummary && flavor == FLAV_NS) { + tor_asprintf(&buf, "p %s\n", rs_out.exitsummary); + smartlist_add(chunks, buf); + } /* And the loop is over and we move on to the next router */ } @@ -1087,34 +2010,97 @@ networkstatus_compute_consensus(smartlist_t *votes, smartlist_free(versions); smartlist_free(exitsummaries); tor_free(bandwidths); + tor_free(measured_bws); + } + + if (consensus_method >= MIN_METHOD_FOR_FOOTER) { + /* Starting with consensus method 9, we clearly mark the directory + * footer region */ + smartlist_add(chunks, tor_strdup("directory-footer\n")); + } + + if (consensus_method >= MIN_METHOD_FOR_BW_WEIGHTS) { + int64_t weight_scale = BW_WEIGHT_SCALE; + char *bw_weight_param = NULL; + + // Parse params, extract BW_WEIGHT_SCALE if present + // DO NOT use consensus_param_bw_weight_scale() in this code! + // The consensus is not formed yet! + if (params) { + if (strcmpstart(params, "bwweightscale=") == 0) + bw_weight_param = params; + else + bw_weight_param = strstr(params, " bwweightscale="); + } + + if (bw_weight_param) { + int ok=0; + char *eq = strchr(bw_weight_param, '='); + if (eq) { + weight_scale = tor_parse_long(eq+1, 10, INT32_MIN, INT32_MAX, &ok, + NULL); + if (!ok) { + log_warn(LD_DIR, "Bad element '%s' in bw weight param", + escaped(bw_weight_param)); + weight_scale = BW_WEIGHT_SCALE; + } + } else { + log_warn(LD_DIR, "Bad element '%s' in bw weight param", + escaped(bw_weight_param)); + weight_scale = BW_WEIGHT_SCALE; + } + } + + if (consensus_method < 10) { + networkstatus_compute_bw_weights_v9(chunks, G, M, E, D, T, weight_scale); + added_weights = 1; + } else { + added_weights = networkstatus_compute_bw_weights_v10(chunks, G, M, E, D, + T, weight_scale); + } } /* Add a signature. */ { - char digest[DIGEST_LEN]; + char digest[DIGEST256_LEN]; char fingerprint[HEX_DIGEST_LEN+1]; char signing_key_fingerprint[HEX_DIGEST_LEN+1]; + digest_algorithm_t digest_alg = + flavor == FLAV_NS ? DIGEST_SHA1 : DIGEST_SHA256; + size_t digest_len = + flavor == FLAV_NS ? DIGEST_LEN : DIGEST256_LEN; + const char *algname = crypto_digest_algorithm_get_name(digest_alg); + char *buf = NULL; + char sigbuf[4096]; - char buf[4096]; smartlist_add(chunks, tor_strdup("directory-signature ")); /* Compute the hash of the chunks. */ - hash_list_members(digest, chunks); + hash_list_members(digest, digest_len, chunks, digest_alg); /* Get the fingerprints */ crypto_pk_get_fingerprint(identity_key, fingerprint, 0); crypto_pk_get_fingerprint(signing_key, signing_key_fingerprint, 0); /* add the junk that will go at the end of the line. */ - tor_snprintf(buf, sizeof(buf), "%s %s\n", fingerprint, - signing_key_fingerprint); + if (flavor == FLAV_NS) { + tor_asprintf(&buf, "%s %s\n", fingerprint, + signing_key_fingerprint); + } else { + tor_asprintf(&buf, "%s %s %s\n", + algname, fingerprint, + signing_key_fingerprint); + } + smartlist_add(chunks, buf); /* And the signature. */ - if (router_append_dirobj_signature(buf, sizeof(buf), digest, + sigbuf[0] = '\0'; + if (router_append_dirobj_signature(sigbuf, sizeof(sigbuf), + digest, digest_len, signing_key)) { log_warn(LD_BUG, "Couldn't sign consensus networkstatus."); return NULL; /* This leaks, but it should never happen. */ } - smartlist_add(chunks, tor_strdup(buf)); + smartlist_add(chunks, tor_strdup(sigbuf)); if (legacy_id_key_digest && legacy_signing_key && consensus_method >= 3) { smartlist_add(chunks, tor_strdup("directory-signature ")); @@ -1122,14 +2108,23 @@ networkstatus_compute_consensus(smartlist_t *votes, legacy_id_key_digest, DIGEST_LEN); crypto_pk_get_fingerprint(legacy_signing_key, signing_key_fingerprint, 0); - tor_snprintf(buf, sizeof(buf), "%s %s\n", fingerprint, - signing_key_fingerprint); - if (router_append_dirobj_signature(buf, sizeof(buf), digest, + if (flavor == FLAV_NS) { + tor_asprintf(&buf, "%s %s\n", fingerprint, + signing_key_fingerprint); + } else { + tor_asprintf(&buf, "%s %s %s\n", + algname, fingerprint, + signing_key_fingerprint); + } + smartlist_add(chunks, buf); + sigbuf[0] = '\0'; + if (router_append_dirobj_signature(sigbuf, sizeof(sigbuf), + digest, digest_len, legacy_signing_key)) { log_warn(LD_BUG, "Couldn't sign consensus networkstatus."); return NULL; /* This leaks, but it should never happen. */ } - smartlist_add(chunks, tor_strdup(buf)); + smartlist_add(chunks, tor_strdup(sigbuf)); } } @@ -1146,11 +2141,15 @@ networkstatus_compute_consensus(smartlist_t *votes, networkstatus_t *c; if (!(c = networkstatus_parse_vote_from_string(result, NULL, NS_TYPE_CONSENSUS))) { - log_err(LD_BUG,"Generated a networkstatus consensus we couldn't " + log_err(LD_BUG, "Generated a networkstatus consensus we couldn't " "parse."); tor_free(result); return NULL; } + // Verify balancing parameters + if (consensus_method >= MIN_METHOD_FOR_BW_WEIGHTS && added_weights) { + networkstatus_verify_bw_weights(c); + } networkstatus_vote_free(c); } @@ -1171,10 +2170,14 @@ networkstatus_add_detached_signatures(networkstatus_t *target, const char **msg_out) { int r = 0; + const char *flavor; + smartlist_t *siglist; tor_assert(sigs); tor_assert(target); tor_assert(target->type == NS_TYPE_CONSENSUS); + flavor = networkstatus_get_flavor_name(target->flavor); + /* Do the times seem right? */ if (target->valid_after != sigs->valid_after) { *msg_out = "Valid-After times do not match " @@ -1191,79 +2194,179 @@ networkstatus_add_detached_signatures(networkstatus_t *target, "when adding detached signatures to consensus"; return -1; } - /* Are they the same consensus? */ - if (fast_memcmp(target->networkstatus_digest, sigs->networkstatus_digest, - DIGEST_LEN)) { - *msg_out = "Digest mismatch when adding detached signatures to consensus"; + siglist = strmap_get(sigs->signatures, flavor); + if (!siglist) { + *msg_out = "No signatures for given consensus flavor"; return -1; } - /* For each voter in src... */ - SMARTLIST_FOREACH_BEGIN(sigs->signatures, networkstatus_voter_info_t *, - src_voter) { - char voter_identity[HEX_DIGEST_LEN+1]; - networkstatus_voter_info_t *target_voter = - networkstatus_get_voter_by_id(target, src_voter->identity_digest); - authority_cert_t *cert = NULL; - - base16_encode(voter_identity, sizeof(voter_identity), - src_voter->identity_digest, DIGEST_LEN); - log_info(LD_DIR, "Looking at signature from %s", voter_identity); - /* If the target doesn't know about this voter, then forget it. */ - if (!target_voter) { - log_info(LD_DIR, "We do not know about %s", voter_identity); - continue; + /** Make sure all the digests we know match, and at least one matches. */ + { + digests_t *digests = strmap_get(sigs->digests, flavor); + int n_matches = 0; + digest_algorithm_t alg; + if (!digests) { + *msg_out = "No digests for given consensus flavor"; + return -1; + } + for (alg = DIGEST_SHA1; alg < N_DIGEST_ALGORITHMS; ++alg) { + if (!tor_mem_is_zero(digests->d[alg], DIGEST256_LEN)) { + if (!memcmp(target->digests.d[alg], digests->d[alg], DIGEST256_LEN)) { + ++n_matches; + } else { + *msg_out = "Mismatched digest."; + return -1; + } } + } + if (!n_matches) { + *msg_out = "No regognized digests for given consensus flavor"; + } + } - /* If the target already has a good signature from this voter, then skip - * this one. */ - if (target_voter->good_signature) { - log_info(LD_DIR, "We already have a good signature from %s", - voter_identity); - continue; - } + /* For each voter in src... */ + SMARTLIST_FOREACH_BEGIN(siglist, document_signature_t *, sig) { + char voter_identity[HEX_DIGEST_LEN+1]; + networkstatus_voter_info_t *target_voter = + networkstatus_get_voter_by_id(target, sig->identity_digest); + authority_cert_t *cert = NULL; + const char *algorithm; + document_signature_t *old_sig = NULL; + + algorithm = crypto_digest_algorithm_get_name(sig->alg); + + base16_encode(voter_identity, sizeof(voter_identity), + sig->identity_digest, DIGEST_LEN); + log_info(LD_DIR, "Looking at signature from %s using %s", voter_identity, + algorithm); + /* If the target doesn't know about this voter, then forget it. */ + if (!target_voter) { + log_info(LD_DIR, "We do not know any voter with ID %s", voter_identity); + continue; + } - /* Try checking the signature if we haven't already. */ - if (!src_voter->good_signature && !src_voter->bad_signature) { - cert = authority_cert_get_by_digests(src_voter->identity_digest, - src_voter->signing_key_digest); - if (cert) { - networkstatus_check_voter_signature(target, src_voter, cert); - } + old_sig = voter_get_sig_by_algorithm(target_voter, sig->alg); + + /* If the target already has a good signature from this voter, then skip + * this one. */ + if (old_sig && old_sig->good_signature) { + log_info(LD_DIR, "We already have a good signature from %s using %s", + voter_identity, algorithm); + continue; + } + + /* Try checking the signature if we haven't already. */ + if (!sig->good_signature && !sig->bad_signature) { + cert = authority_cert_get_by_digests(sig->identity_digest, + sig->signing_key_digest); + if (cert) + networkstatus_check_document_signature(target, sig, cert); + } + + /* If this signature is good, or we don't have any signature yet, + * then maybe add it. */ + if (sig->good_signature || !old_sig || old_sig->bad_signature) { + log_info(LD_DIR, "Adding signature from %s with %s", voter_identity, + algorithm); + ++r; + if (old_sig) { + smartlist_remove(target_voter->sigs, old_sig); + document_signature_free(old_sig); } + smartlist_add(target_voter->sigs, document_signature_dup(sig)); + } else { + log_info(LD_DIR, "Not adding signature from %s", voter_identity); + } + } SMARTLIST_FOREACH_END(sig); + + return r; +} + +/** Return a newly allocated string containing all the signatures on + * <b>consensus</b> by all voters. If <b>for_detached_signatures</b> is true, + * then the signatures will be put in a detached signatures document, so + * prefix any non-NS-flavored signatures with "additional-signature" rather + * than "directory-signature". */ +static char * +networkstatus_format_signatures(networkstatus_t *consensus, + int for_detached_signatures) +{ + smartlist_t *elements; + char buf[4096]; + char *result = NULL; + int n_sigs = 0; + const consensus_flavor_t flavor = consensus->flavor; + const char *flavor_name = networkstatus_get_flavor_name(flavor); + const char *keyword; + + if (for_detached_signatures && flavor != FLAV_NS) + keyword = "additional-signature"; + else + keyword = "directory-signature"; - /* If this signature is good, or we don't have any signature yet, - * then add it. */ - if (src_voter->good_signature || !target_voter->signature) { - log_info(LD_DIR, "Adding signature from %s", voter_identity); - ++r; - tor_free(target_voter->signature); - target_voter->signature = - tor_memdup(src_voter->signature, src_voter->signature_len); - memcpy(target_voter->signing_key_digest, src_voter->signing_key_digest, - DIGEST_LEN); - target_voter->signature_len = src_voter->signature_len; - target_voter->good_signature = src_voter->good_signature; - target_voter->bad_signature = src_voter->bad_signature; + elements = smartlist_create(); + + SMARTLIST_FOREACH_BEGIN(consensus->voters, networkstatus_voter_info_t *, v) { + SMARTLIST_FOREACH_BEGIN(v->sigs, document_signature_t *, sig) { + char sk[HEX_DIGEST_LEN+1]; + char id[HEX_DIGEST_LEN+1]; + if (!sig->signature || sig->bad_signature) + continue; + ++n_sigs; + base16_encode(sk, sizeof(sk), sig->signing_key_digest, DIGEST_LEN); + base16_encode(id, sizeof(id), sig->identity_digest, DIGEST_LEN); + if (flavor == FLAV_NS) { + tor_snprintf(buf, sizeof(buf), + "%s %s %s\n-----BEGIN SIGNATURE-----\n", + keyword, id, sk); } else { - log_info(LD_DIR, "Not adding signature from %s", voter_identity); + const char *digest_name = + crypto_digest_algorithm_get_name(sig->alg); + tor_snprintf(buf, sizeof(buf), + "%s%s%s %s %s %s\n-----BEGIN SIGNATURE-----\n", + keyword, + for_detached_signatures ? " " : "", + for_detached_signatures ? flavor_name : "", + digest_name, id, sk); } - } SMARTLIST_FOREACH_END(src_voter); + smartlist_add(elements, tor_strdup(buf)); + base64_encode(buf, sizeof(buf), sig->signature, sig->signature_len); + strlcat(buf, "-----END SIGNATURE-----\n", sizeof(buf)); + smartlist_add(elements, tor_strdup(buf)); + } SMARTLIST_FOREACH_END(sig); + } SMARTLIST_FOREACH_END(v); - return r; + result = smartlist_join_strings(elements, "", 0, NULL); + SMARTLIST_FOREACH(elements, char *, cp, tor_free(cp)); + smartlist_free(elements); + if (!n_sigs) + tor_free(result); + return result; } /** Return a newly allocated string holding the detached-signatures document - * corresponding to the signatures on <b>consensus</b>. */ + * corresponding to the signatures on <b>consensuses</b>, which must contain + * exactly one FLAV_NS consensus, and no more than one consensus for each + * other flavor. */ char * -networkstatus_get_detached_signatures(networkstatus_t *consensus) +networkstatus_get_detached_signatures(smartlist_t *consensuses) { smartlist_t *elements; char buf[4096]; - char *result = NULL; - int n_sigs = 0; - tor_assert(consensus); - tor_assert(consensus->type == NS_TYPE_CONSENSUS); + char *result = NULL, *sigs = NULL; + networkstatus_t *consensus_ns = NULL; + tor_assert(consensuses); + + SMARTLIST_FOREACH(consensuses, networkstatus_t *, ns, { + tor_assert(ns); + tor_assert(ns->type == NS_TYPE_CONSENSUS); + if (ns && ns->flavor == FLAV_NS) + consensus_ns = ns; + }); + if (!consensus_ns) { + log_warn(LD_BUG, "No NS consensus given."); + return NULL; + } elements = smartlist_create(); @@ -1272,10 +2375,11 @@ networkstatus_get_detached_signatures(networkstatus_t *consensus) vu_buf[ISO_TIME_LEN+1]; char d[HEX_DIGEST_LEN+1]; - base16_encode(d, sizeof(d), consensus->networkstatus_digest, DIGEST_LEN); - format_iso_time(va_buf, consensus->valid_after); - format_iso_time(fu_buf, consensus->fresh_until); - format_iso_time(vu_buf, consensus->valid_until); + base16_encode(d, sizeof(d), + consensus_ns->digests.d[DIGEST_SHA1], DIGEST_LEN); + format_iso_time(va_buf, consensus_ns->valid_after); + format_iso_time(fu_buf, consensus_ns->fresh_until); + format_iso_time(vu_buf, consensus_ns->valid_until); tor_snprintf(buf, sizeof(buf), "consensus-digest %s\n" @@ -1285,45 +2389,89 @@ networkstatus_get_detached_signatures(networkstatus_t *consensus) smartlist_add(elements, tor_strdup(buf)); } - SMARTLIST_FOREACH(consensus->voters, networkstatus_voter_info_t *, v, - { - char sk[HEX_DIGEST_LEN+1]; - char id[HEX_DIGEST_LEN+1]; - if (!v->signature || v->bad_signature) + /* Get all the digests for the non-FLAV_NS consensuses */ + SMARTLIST_FOREACH_BEGIN(consensuses, networkstatus_t *, ns) { + const char *flavor_name = networkstatus_get_flavor_name(ns->flavor); + int alg; + if (ns->flavor == FLAV_NS) + continue; + + /* start with SHA256; we don't include SHA1 for anything but the basic + * consensus. */ + for (alg = DIGEST_SHA256; alg < N_DIGEST_ALGORITHMS; ++alg) { + char d[HEX_DIGEST256_LEN+1]; + const char *alg_name = + crypto_digest_algorithm_get_name(alg); + if (tor_mem_is_zero(ns->digests.d[alg], DIGEST256_LEN)) continue; - ++n_sigs; - base16_encode(sk, sizeof(sk), v->signing_key_digest, DIGEST_LEN); - base16_encode(id, sizeof(id), v->identity_digest, DIGEST_LEN); - tor_snprintf(buf, sizeof(buf), - "directory-signature %s %s\n-----BEGIN SIGNATURE-----\n", - id, sk); + base16_encode(d, sizeof(d), ns->digests.d[alg], DIGEST256_LEN); + tor_snprintf(buf, sizeof(buf), "additional-digest %s %s %s\n", + flavor_name, alg_name, d); smartlist_add(elements, tor_strdup(buf)); - base64_encode(buf, sizeof(buf), v->signature, v->signature_len); - strlcat(buf, "-----END SIGNATURE-----\n", sizeof(buf)); - smartlist_add(elements, tor_strdup(buf)); - }); + } + } SMARTLIST_FOREACH_END(ns); + + /* Now get all the sigs for non-FLAV_NS consensuses */ + SMARTLIST_FOREACH_BEGIN(consensuses, networkstatus_t *, ns) { + char *sigs; + if (ns->flavor == FLAV_NS) + continue; + sigs = networkstatus_format_signatures(ns, 1); + if (!sigs) { + log_warn(LD_DIR, "Couldn't format signatures"); + goto err; + } + smartlist_add(elements, sigs); + } SMARTLIST_FOREACH_END(ns); - result = smartlist_join_strings(elements, "", 0, NULL); + /* Now add the FLAV_NS consensus signatrures. */ + sigs = networkstatus_format_signatures(consensus_ns, 1); + if (!sigs) + goto err; + smartlist_add(elements, sigs); + result = smartlist_join_strings(elements, "", 0, NULL); + err: SMARTLIST_FOREACH(elements, char *, cp, tor_free(cp)); smartlist_free(elements); - if (!n_sigs) - tor_free(result); return result; } +/** Return a newly allocated string holding a detached-signatures document for + * all of the in-progress consensuses in the <b>n_flavors</b>-element array at + * <b>pending</b>. */ +static char * +get_detached_signatures_from_pending_consensuses(pending_consensus_t *pending, + int n_flavors) +{ + int flav; + char *signatures; + smartlist_t *c = smartlist_create(); + for (flav = 0; flav < n_flavors; ++flav) { + if (pending[flav].consensus) + smartlist_add(c, pending[flav].consensus); + } + signatures = networkstatus_get_detached_signatures(c); + smartlist_free(c); + return signatures; +} + /** Release all storage held in <b>s</b>. */ void ns_detached_signatures_free(ns_detached_signatures_t *s) { + if (!s) + return; if (s->signatures) { - SMARTLIST_FOREACH(s->signatures, networkstatus_voter_info_t *, v, - { - tor_free(v->signature); - tor_free(v); - }); - smartlist_free(s->signatures); + STRMAP_FOREACH(s->signatures, flavor, smartlist_t *, sigs) { + SMARTLIST_FOREACH(sigs, document_signature_t *, sig, + document_signature_free(sig)); + smartlist_free(sigs); + } STRMAP_FOREACH_END; + strmap_free(s->signatures, NULL); + strmap_free(s->digests, _tor_free); } + tor_free(s); } @@ -1513,7 +2661,7 @@ dirvote_act(or_options_t *options, time_t now) if (voting_schedule.voting_ends < now && !voting_schedule.have_built_consensus) { log_notice(LD_DIR, "Time to compute a consensus."); - dirvote_compute_consensus(); + dirvote_compute_consensuses(); /* XXXX We will want to try again later if we haven't got enough * votes yet. Implement this if it turns out to ever happen. */ voting_schedule.have_built_consensus = 1; @@ -1550,14 +2698,13 @@ static smartlist_t *pending_vote_list = NULL; /** List of pending_vote_t for the previous vote. After we've used them to * build a consensus, the votes go here for the next period. */ static smartlist_t *previous_vote_list = NULL; -/** The body of the consensus that we're currently building. Once we - * have it built, it goes into dirserv.c */ -static char *pending_consensus_body = NULL; + +static pending_consensus_t pending_consensuses[N_CONSENSUS_FLAVORS]; + /** The detached signatures for the consensus that we're currently * building. */ static char *pending_consensus_signatures = NULL; -/** The parsed in-progress consensus document. */ -static networkstatus_t *pending_consensus = NULL; + /** List of ns_detached_signatures_t: hold signatures that get posted to us * before we have generated the consensus on our own. */ static smartlist_t *pending_consensus_signature_list = NULL; @@ -1651,15 +2798,39 @@ dirvote_fetch_missing_votes(void) static void dirvote_fetch_missing_signatures(void) { - if (!pending_consensus) + int need_any = 0; + int i; + for (i=0; i < N_CONSENSUS_FLAVORS; ++i) { + networkstatus_t *consensus = pending_consensuses[i].consensus; + if (!consensus || + networkstatus_check_consensus_signature(consensus, -1) == 1) { + /* We have no consensus, or we have one that's signed by everybody. */ + continue; + } + need_any = 1; + } + if (!need_any) return; - if (networkstatus_check_consensus_signature(pending_consensus, -1) == 1) - return; /* we have a signature from everybody. */ directory_get_from_all_authorities(DIR_PURPOSE_FETCH_DETACHED_SIGNATURES, 0, NULL); } +/** Release all storage held by pending consensuses (those waiting for + * signatures). */ +static void +dirvote_clear_pending_consensuses(void) +{ + int i; + for (i = 0; i < N_CONSENSUS_FLAVORS; ++i) { + pending_consensus_t *pc = &pending_consensuses[i]; + tor_free(pc->body); + + networkstatus_vote_free(pc->consensus); + pc->consensus = NULL; + } +} + /** Drop all currently pending votes, consensus, and detached signatures. */ static void dirvote_clear_votes(int all_votes) @@ -1697,12 +2868,8 @@ dirvote_clear_votes(int all_votes) tor_free(cp)); smartlist_clear(pending_consensus_signature_list); } - tor_free(pending_consensus_body); tor_free(pending_consensus_signatures); - if (pending_consensus) { - networkstatus_vote_free(pending_consensus); - pending_consensus = NULL; - } + dirvote_clear_pending_consensuses(); } /** Return a newly allocated string containing the hex-encoded v3 authority @@ -1760,7 +2927,13 @@ dirvote_add_vote(const char *vote_body, const char **msg_out, int *status_out) } tor_assert(smartlist_len(vote->voters) == 1); vi = get_voter(vote); - tor_assert(vi->good_signature == 1); + { + int any_sig_good = 0; + SMARTLIST_FOREACH(vi->sigs, document_signature_t *, sig, + if (sig->good_signature) + any_sig_good = 1); + tor_assert(any_sig_good); + } ds = trusteddirserver_get_by_v3_auth_digest(vi->identity_digest); if (!ds) { char *keys = list_v3_auth_ids(); @@ -1797,6 +2970,9 @@ dirvote_add_vote(const char *vote_body, const char **msg_out, int *status_out) goto err; } + /* Fetch any new router descriptors we just learned about */ + update_consensus_router_descriptor_downloads(time(NULL), 1, vote); + /* Now see whether we already have a vote from this authority. */ SMARTLIST_FOREACH(pending_vote_list, pending_vote_t *, v, { if (fast_memeq(v->vote->cert->cache_info.identity_digest, @@ -1805,7 +2981,8 @@ dirvote_add_vote(const char *vote_body, const char **msg_out, int *status_out) networkstatus_voter_info_t *vi_old = get_voter(v->vote); if (fast_memeq(vi_old->vote_digest, vi->vote_digest, DIGEST_LEN)) { /* Ah, it's the same vote. Not a problem. */ - log_info(LD_DIR, "Discarding a vote we already have."); + log_info(LD_DIR, "Discarding a vote we already have (from %s).", + vi->address); if (*status_out < 200) *status_out = 200; goto discard; @@ -1832,7 +3009,7 @@ dirvote_add_vote(const char *vote_body, const char **msg_out, int *status_out) goto err; } } - }); + }); pending_vote = tor_malloc_zero(sizeof(pending_vote_t)); pending_vote->vote_body = new_cached_dir(tor_strndup(vote_body, @@ -1856,8 +3033,7 @@ dirvote_add_vote(const char *vote_body, const char **msg_out, int *status_out) *status_out = 400; discard: - if (vote) - networkstatus_vote_free(vote); + networkstatus_vote_free(vote); if (end_of_vote && !strcmpstart(end_of_vote, "network-status-version ")) { vote_body = end_of_vote; @@ -1884,14 +3060,18 @@ dirvote_add_vote(const char *vote_body, const char **msg_out, int *status_out) * pending_consensus: it won't be ready to be published until we have * everybody else's signatures collected too. (V3 Authority only) */ static int -dirvote_compute_consensus(void) +dirvote_compute_consensuses(void) { /* Have we got enough votes to try? */ - int n_votes, n_voters; + int n_votes, n_voters, n_vote_running = 0; smartlist_t *votes = NULL, *votestrings = NULL; char *consensus_body = NULL, *signatures = NULL, *votefile; networkstatus_t *consensus = NULL; authority_cert_t *my_cert; + pending_consensus_t pending[N_CONSENSUS_FLAVORS]; + int flav; + + memset(pending, 0, sizeof(pending)); if (!pending_vote_list) pending_vote_list = smartlist_create(); @@ -1900,7 +3080,20 @@ dirvote_compute_consensus(void) n_votes = smartlist_len(pending_vote_list); if (n_votes <= n_voters/2) { log_warn(LD_DIR, "We don't have enough votes to generate a consensus: " - "%d of %d", n_votes, n_voters/2); + "%d of %d", n_votes, n_voters/2+1); + goto err; + } + tor_assert(pending_vote_list); + SMARTLIST_FOREACH(pending_vote_list, pending_vote_t *, v, { + if (smartlist_string_isin(v->vote->known_flags, "Running")) + n_vote_running++; + }); + if (!n_vote_running) { + /* See task 1066. */ + log_warn(LD_DIR, "Nobody has voted on the Running flag. Generating " + "and publishing a consensus without Running nodes " + "would make many clients stop working. Not " + "generating a consensus!"); goto err; } @@ -1931,47 +3124,71 @@ dirvote_compute_consensus(void) char legacy_dbuf[DIGEST_LEN]; crypto_pk_env_t *legacy_sign=NULL; char *legacy_id_digest = NULL; + int n_generated = 0; if (get_options()->V3AuthUseLegacyKey) { authority_cert_t *cert = get_my_v3_legacy_cert(); legacy_sign = get_my_v3_legacy_signing_key(); if (cert) { - crypto_pk_get_digest(cert->identity_key, legacy_dbuf); - legacy_id_digest = legacy_dbuf; + if (crypto_pk_get_digest(cert->identity_key, legacy_dbuf)) { + log_warn(LD_BUG, + "Unable to compute digest of legacy v3 identity key"); + } else { + legacy_id_digest = legacy_dbuf; + } } } - consensus_body = networkstatus_compute_consensus( + + for (flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) { + const char *flavor_name = networkstatus_get_flavor_name(flav); + consensus_body = networkstatus_compute_consensus( votes, n_voters, my_cert->identity_key, - get_my_v3_authority_signing_key(), legacy_id_digest, legacy_sign); - } - if (!consensus_body) { - log_warn(LD_DIR, "Couldn't generate a consensus at all!"); - goto err; - } - consensus = networkstatus_parse_vote_from_string(consensus_body, NULL, - NS_TYPE_CONSENSUS); - if (!consensus) { - log_warn(LD_DIR, "Couldn't parse consensus we generated!"); - goto err; + get_my_v3_authority_signing_key(), legacy_id_digest, legacy_sign, + flav); + + if (!consensus_body) { + log_warn(LD_DIR, "Couldn't generate a %s consensus at all!", + flavor_name); + continue; + } + consensus = networkstatus_parse_vote_from_string(consensus_body, NULL, + NS_TYPE_CONSENSUS); + if (!consensus) { + log_warn(LD_DIR, "Couldn't parse %s consensus we generated!", + flavor_name); + tor_free(consensus_body); + continue; + } + + /* 'Check' our own signature, to mark it valid. */ + networkstatus_check_consensus_signature(consensus, -1); + + pending[flav].body = consensus_body; + pending[flav].consensus = consensus; + n_generated++; + consensus_body = NULL; + consensus = NULL; + } + if (!n_generated) { + log_warn(LD_DIR, "Couldn't generate any consensus flavors at all."); + goto err; + } } - /* 'Check' our own signature, to mark it valid. */ - networkstatus_check_consensus_signature(consensus, -1); - signatures = networkstatus_get_detached_signatures(consensus); + signatures = get_detached_signatures_from_pending_consensuses( + pending, N_CONSENSUS_FLAVORS); + if (!signatures) { log_warn(LD_DIR, "Couldn't extract signatures."); goto err; } - tor_free(pending_consensus_body); - pending_consensus_body = consensus_body; + dirvote_clear_pending_consensuses(); + memcpy(pending_consensuses, pending, sizeof(pending)); + tor_free(pending_consensus_signatures); pending_consensus_signatures = signatures; - if (pending_consensus) - networkstatus_vote_free(pending_consensus); - pending_consensus = consensus; - if (pending_consensus_signature_list) { int n_sigs = 0; /* we may have gotten signatures for this consensus before we built @@ -1979,7 +3196,7 @@ dirvote_compute_consensus(void) SMARTLIST_FOREACH(pending_consensus_signature_list, char *, sig, { const char *msg = NULL; - int r = dirvote_add_signatures_to_pending_consensus(sig, &msg); + int r = dirvote_add_signatures_to_all_pending_consensuses(sig, &msg); if (r >= 0) n_sigs += r; else @@ -2003,10 +3220,10 @@ dirvote_compute_consensus(void) strlen(pending_consensus_signatures), 0); log_notice(LD_DIR, "Signature(s) posted."); + smartlist_free(votes); return 0; err: - if (votes) - smartlist_free(votes); + smartlist_free(votes); tor_free(consensus_body); tor_free(signatures); networkstatus_vote_free(consensus); @@ -2015,76 +3232,62 @@ dirvote_compute_consensus(void) } /** Helper: we just got the <b>detached_signatures_body</b> sent to us as - * signatures on the currently pending consensus. Add them to the consensus + * signatures on the currently pending consensus. Add them to <b>pc</b> * as appropriate. Return the number of signatures added. (?) */ static int dirvote_add_signatures_to_pending_consensus( - const char *detached_signatures_body, + pending_consensus_t *pc, + ns_detached_signatures_t *sigs, const char **msg_out) { - ns_detached_signatures_t *sigs = NULL; + const char *flavor_name; int r = -1; - tor_assert(detached_signatures_body); - tor_assert(msg_out); - /* Only call if we have a pending consensus right now. */ - tor_assert(pending_consensus); - tor_assert(pending_consensus_body); + tor_assert(pc->consensus); + tor_assert(pc->body); tor_assert(pending_consensus_signatures); + flavor_name = networkstatus_get_flavor_name(pc->consensus->flavor); *msg_out = NULL; - if (!(sigs = networkstatus_parse_detached_signatures( - detached_signatures_body, NULL))) { - *msg_out = "Couldn't parse detached signatures."; - goto err; + { + smartlist_t *sig_list = strmap_get(sigs->signatures, flavor_name); + log_info(LD_DIR, "Have %d signatures for adding to %s consensus.", + sig_list ? smartlist_len(sig_list) : 0, flavor_name); } - - log_info(LD_DIR, "Have %d signatures for adding to consensus.", - smartlist_len(sigs->signatures)); - r = networkstatus_add_detached_signatures(pending_consensus, - sigs, msg_out); + r = networkstatus_add_detached_signatures(pc->consensus, sigs, msg_out); log_info(LD_DIR,"Added %d signatures to consensus.", r); if (r >= 1) { - char *new_detached = - networkstatus_get_detached_signatures(pending_consensus); - const char *src; + char *new_signatures = + networkstatus_format_signatures(pc->consensus, 0); char *dst, *dst_end; size_t new_consensus_len; - if (!new_detached) { + if (!new_signatures) { *msg_out = "No signatures to add"; goto err; } new_consensus_len = - strlen(pending_consensus_body) + strlen(new_detached) + 1; - pending_consensus_body = tor_realloc(pending_consensus_body, - new_consensus_len); - dst_end = pending_consensus_body + new_consensus_len; - dst = strstr(pending_consensus_body, "directory-signature "); + strlen(pc->body) + strlen(new_signatures) + 1; + pc->body = tor_realloc(pc->body, new_consensus_len); + dst_end = pc->body + new_consensus_len; + dst = strstr(pc->body, "directory-signature "); tor_assert(dst); - src = strstr(new_detached, "directory-signature "); - tor_assert(src); - strlcpy(dst, src, dst_end-dst); + strlcpy(dst, new_signatures, dst_end-dst); /* We remove this block once it has failed to crash for a while. But * unless it shows up in profiles, we're probably better leaving it in, * just in case we break detached signature processing at some point. */ { - ns_detached_signatures_t *sigs = - networkstatus_parse_detached_signatures(new_detached, NULL); networkstatus_t *v = networkstatus_parse_vote_from_string( - pending_consensus_body, NULL, + pc->body, NULL, NS_TYPE_CONSENSUS); - tor_assert(sigs); - ns_detached_signatures_free(sigs); tor_assert(v); networkstatus_vote_free(v); } - tor_free(pending_consensus_signatures); - pending_consensus_signatures = new_detached; *msg_out = "Signatures added"; + tor_free(new_signatures); } else if (r == 0) { *msg_out = "Signatures ignored"; } else { @@ -2096,8 +3299,62 @@ dirvote_add_signatures_to_pending_consensus( if (!*msg_out) *msg_out = "Unrecognized error while adding detached signatures."; done: - if (sigs) - ns_detached_signatures_free(sigs); + return r; +} + +static int +dirvote_add_signatures_to_all_pending_consensuses( + const char *detached_signatures_body, + const char **msg_out) +{ + int r=0, i, n_added = 0, errors = 0; + ns_detached_signatures_t *sigs; + tor_assert(detached_signatures_body); + tor_assert(msg_out); + tor_assert(pending_consensus_signatures); + + if (!(sigs = networkstatus_parse_detached_signatures( + detached_signatures_body, NULL))) { + *msg_out = "Couldn't parse detached signatures."; + goto err; + } + + for (i = 0; i < N_CONSENSUS_FLAVORS; ++i) { + int res; + pending_consensus_t *pc = &pending_consensuses[i]; + if (!pc->consensus) + continue; + res = dirvote_add_signatures_to_pending_consensus(pc, sigs, msg_out); + if (res < 0) + errors++; + else + n_added += res; + } + + if (errors && !n_added) { + r = -1; + goto err; + } + + if (n_added && pending_consensuses[FLAV_NS].consensus) { + char *new_detached = + get_detached_signatures_from_pending_consensuses( + pending_consensuses, N_CONSENSUS_FLAVORS); + if (new_detached) { + tor_free(pending_consensus_signatures); + pending_consensus_signatures = new_detached; + } + } + + r = n_added; + goto done; + err: + if (!*msg_out) + *msg_out = "Unrecognized error while adding detached signatures."; + done: + ns_detached_signatures_free(sigs); + /* XXXX NM Check how return is used. We can now have an error *and* + signatures added. */ return r; } @@ -2110,10 +3367,10 @@ dirvote_add_signatures(const char *detached_signatures_body, const char *source, const char **msg) { - if (pending_consensus) { + if (pending_consensuses[FLAV_NS].consensus) { log_notice(LD_DIR, "Got a signature from %s. " "Adding it to the pending consensus.", source); - return dirvote_add_signatures_to_pending_consensus( + return dirvote_add_signatures_to_all_pending_consensuses( detached_signatures_body, msg); } else { log_notice(LD_DIR, "Got a signature from %s. " @@ -2132,17 +3389,25 @@ dirvote_add_signatures(const char *detached_signatures_body, static int dirvote_publish_consensus(void) { - /* Can we actually publish it yet? */ - if (!pending_consensus || - networkstatus_check_consensus_signature(pending_consensus, 1)<0) { - log_warn(LD_DIR, "Not enough info to publish pending consensus"); - return -1; - } + int i; + + /* Now remember all the other consensuses as if we were a directory cache. */ + for (i = 0; i < N_CONSENSUS_FLAVORS; ++i) { + pending_consensus_t *pending = &pending_consensuses[i]; + const char *name; + name = networkstatus_get_flavor_name(i); + tor_assert(name); + if (!pending->consensus || + networkstatus_check_consensus_signature(pending->consensus, 1)<0) { + log_warn(LD_DIR, "Not enough info to publish pending %s consensus",name); + continue; + } - if (networkstatus_set_current_consensus(pending_consensus_body, 0)) - log_warn(LD_DIR, "Error publishing consensus"); - else - log_notice(LD_DIR, "Consensus published."); + if (networkstatus_set_current_consensus(pending->body, name, 0)) + log_warn(LD_DIR, "Error publishing %s consensus", name); + else + log_notice(LD_DIR, "Published %s consensus", name); + } return 0; } @@ -2152,20 +3417,16 @@ void dirvote_free_all(void) { dirvote_clear_votes(1); - /* now empty as a result of clear_pending_votes. */ + /* now empty as a result of dirvote_clear_votes(). */ smartlist_free(pending_vote_list); pending_vote_list = NULL; smartlist_free(previous_vote_list); previous_vote_list = NULL; - tor_free(pending_consensus_body); + dirvote_clear_pending_consensuses(); tor_free(pending_consensus_signatures); - if (pending_consensus) { - networkstatus_vote_free(pending_consensus); - pending_consensus = NULL; - } if (pending_consensus_signature_list) { - /* now empty as a result of clear_pending_votes. */ + /* now empty as a result of dirvote_clear_votes(). */ smartlist_free(pending_consensus_signature_list); pending_consensus_signature_list = NULL; } @@ -2177,13 +3438,14 @@ dirvote_free_all(void) /** Return the body of the consensus that we're currently trying to build. */ const char * -dirvote_get_pending_consensus(void) +dirvote_get_pending_consensus(consensus_flavor_t flav) { - return pending_consensus_body; + tor_assert(((int)flav) >= 0 && flav < N_CONSENSUS_FLAVORS); + return pending_consensuses[flav].body; } /** Return the signatures that we know for the consensus that we're currently - * trying to build */ + * trying to build. */ const char * dirvote_get_pending_detached_signatures(void) { @@ -2229,15 +3491,147 @@ dirvote_get_vote(const char *fp, int flags) } else { if (pending_vote_list && include_pending) { SMARTLIST_FOREACH(pending_vote_list, pending_vote_t *, pv, - if (fast_memeq(pv->vote->networkstatus_digest, fp, DIGEST_LEN)) + if (fast_memeq(pv->vote->digests.d[DIGEST_SHA1], fp, DIGEST_LEN)) return pv->vote_body); } if (previous_vote_list && include_previous) { SMARTLIST_FOREACH(previous_vote_list, pending_vote_t *, pv, - if (fast_memeq(pv->vote->networkstatus_digest, fp, DIGEST_LEN)) + if (fast_memeq(pv->vote->digests.d[DIGEST_SHA1], fp, DIGEST_LEN)) return pv->vote_body); } } return NULL; } +/** Construct and return a new microdescriptor from a routerinfo <b>ri</b>. + * + * XXX Right now, there is only one way to generate microdescriptors from + * router descriptors. This may change in future consensus methods. If so, + * we'll need an internal way to remember which method we used, and ask for a + * particular method. + **/ +microdesc_t * +dirvote_create_microdescriptor(const routerinfo_t *ri) +{ + microdesc_t *result = NULL; + char *key = NULL, *summary = NULL, *family = NULL; + char buf[1024]; + size_t keylen; + char *out = buf, *end = buf+sizeof(buf); + + if (crypto_pk_write_public_key_to_string(ri->onion_pkey, &key, &keylen)<0) + goto done; + summary = policy_summarize(ri->exit_policy); + if (ri->declared_family) + family = smartlist_join_strings(ri->declared_family, " ", 0, NULL); + + if (tor_snprintf(out, end-out, "onion-key\n%s", key)<0) + goto done; + out += strlen(out); + if (family) { + if (tor_snprintf(out, end-out, "family %s\n", family)<0) + goto done; + out += strlen(out); + } + if (summary && strcmp(summary, "reject 1-65535")) { + if (tor_snprintf(out, end-out, "p %s\n", summary)<0) + goto done; + out += strlen(out); + } + *out = '\0'; /* Make sure it's nul-terminated. This should be a no-op */ + + { + smartlist_t *lst = microdescs_parse_from_string(buf, out, 0, 1); + if (smartlist_len(lst) != 1) { + log_warn(LD_DIR, "We generated a microdescriptor we couldn't parse."); + SMARTLIST_FOREACH(lst, microdesc_t *, md, microdesc_free(md)); + smartlist_free(lst); + goto done; + } + result = smartlist_get(lst, 0); + smartlist_free(lst); + } + + done: + tor_free(key); + tor_free(summary); + tor_free(family); + return result; +} + +/** Cached space-separated string to hold */ +static char *microdesc_consensus_methods = NULL; + +/** Format the appropriate vote line to describe the microdescriptor <b>md</b> + * in a consensus vote document. Write it into the <b>out_len</b>-byte buffer + * in <b>out</b>. Return -1 on failure and the number of characters written + * on success. */ +ssize_t +dirvote_format_microdesc_vote_line(char *out, size_t out_len, + const microdesc_t *md) +{ + char d64[BASE64_DIGEST256_LEN+1]; + if (!microdesc_consensus_methods) { + microdesc_consensus_methods = + make_consensus_method_list(MIN_METHOD_FOR_MICRODESC, + MAX_SUPPORTED_CONSENSUS_METHOD, + ","); + tor_assert(microdesc_consensus_methods); + } + if (digest256_to_base64(d64, md->digest)<0) + return -1; + + if (tor_snprintf(out, out_len, "m %s sha256=%s\n", + microdesc_consensus_methods, d64)<0) + return -1; + + return strlen(out); +} + +/** If <b>vrs</b> has a hash made for the consensus method <b>method</b> with + * the digest algorithm <b>alg</b>, decode it and copy it into + * <b>digest256_out</b> and return 0. Otherwise return -1. */ +int +vote_routerstatus_find_microdesc_hash(char *digest256_out, + const vote_routerstatus_t *vrs, + int method, + digest_algorithm_t alg) +{ + /* XXXX only returns the sha256 method. */ + const vote_microdesc_hash_t *h; + char mstr[64]; + size_t mlen; + char dstr[64]; + + tor_snprintf(mstr, sizeof(mstr), "%d", method); + mlen = strlen(mstr); + tor_snprintf(dstr, sizeof(dstr), " %s=", + crypto_digest_algorithm_get_name(alg)); + + for (h = vrs->microdesc; h; h = h->next) { + const char *cp = h->microdesc_hash_line; + size_t num_len; + /* cp looks like \d+(,\d+)* (digesttype=val )+ . Let's hunt for mstr in + * the first part. */ + while (1) { + num_len = strspn(cp, "1234567890"); + if (num_len == mlen && !memcmp(mstr, cp, mlen)) { + /* This is the line. */ + char buf[BASE64_DIGEST256_LEN+1]; + /* XXXX ignores extraneous stuff if the digest is too long. This + * seems harmless enough, right? */ + cp = strstr(cp, dstr); + if (!cp) + return -1; + cp += strlen(dstr); + strlcpy(buf, cp, sizeof(buf)); + return digest256_from_base64(digest256_out, buf); + } + if (num_len == 0 || cp[num_len] != ',') + break; + cp += num_len + 1; + } + } + return -1; +} + diff --git a/src/or/dirvote.h b/src/or/dirvote.h new file mode 100644 index 0000000000..67540a37fb --- /dev/null +++ b/src/or/dirvote.h @@ -0,0 +1,90 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file dirvote.h + * \brief Header file for dirvote.c. + **/ + +#ifndef _TOR_DIRVOTE_H +#define _TOR_DIRVOTE_H + +/** Lowest allowable value for VoteSeconds. */ +#define MIN_VOTE_SECONDS 20 +/** Lowest allowable value for DistSeconds. */ +#define MIN_DIST_SECONDS 20 +/** Smallest allowable voting interval. */ +#define MIN_VOTE_INTERVAL 300 + +void dirvote_free_all(void); + +/* vote manipulation */ +char *networkstatus_compute_consensus(smartlist_t *votes, + int total_authorities, + crypto_pk_env_t *identity_key, + crypto_pk_env_t *signing_key, + const char *legacy_identity_key_digest, + crypto_pk_env_t *legacy_signing_key, + consensus_flavor_t flavor); +int networkstatus_add_detached_signatures(networkstatus_t *target, + ns_detached_signatures_t *sigs, + const char **msg_out); +char *networkstatus_get_detached_signatures(smartlist_t *consensuses); +void ns_detached_signatures_free(ns_detached_signatures_t *s); + +/* cert manipulation */ +authority_cert_t *authority_cert_dup(authority_cert_t *cert); + +/* vote scheduling */ +void dirvote_get_preferred_voting_intervals(vote_timing_t *timing_out); +time_t dirvote_get_start_of_next_interval(time_t now, int interval); +void dirvote_recalculate_timing(or_options_t *options, time_t now); +void dirvote_act(or_options_t *options, time_t now); + +/* invoked on timers and by outside triggers. */ +struct pending_vote_t * dirvote_add_vote(const char *vote_body, + const char **msg_out, + int *status_out); +int dirvote_add_signatures(const char *detached_signatures_body, + const char *source, + const char **msg_out); + +/* Item access */ +const char *dirvote_get_pending_consensus(consensus_flavor_t flav); +const char *dirvote_get_pending_detached_signatures(void); +#define DGV_BY_ID 1 +#define DGV_INCLUDE_PENDING 2 +#define DGV_INCLUDE_PREVIOUS 4 +const cached_dir_t *dirvote_get_vote(const char *fp, int flags); +void set_routerstatus_from_routerinfo(routerstatus_t *rs, + routerinfo_t *ri, time_t now, + int naming, int listbadexits, + int listbaddirs); +void router_clear_status_flags(routerinfo_t *ri); +networkstatus_t * +dirserv_generate_networkstatus_vote_obj(crypto_pk_env_t *private_key, + authority_cert_t *cert); + +microdesc_t *dirvote_create_microdescriptor(const routerinfo_t *ri); +ssize_t dirvote_format_microdesc_vote_line(char *out, size_t out_len, + const microdesc_t *md); + +int vote_routerstatus_find_microdesc_hash(char *digest256_out, + const vote_routerstatus_t *vrs, + int method, + digest_algorithm_t alg); +document_signature_t *voter_get_sig_by_algorithm( + const networkstatus_voter_info_t *voter, + digest_algorithm_t alg); + +#ifdef DIRVOTE_PRIVATE +char *format_networkstatus_vote(crypto_pk_env_t *private_key, + networkstatus_t *v3_ns); +char *dirvote_compute_params(smartlist_t *votes); +#endif + +#endif + diff --git a/src/or/dns.c b/src/or/dns.c index 3bb9c34892..61c8f32c98 100644 --- a/src/or/dns.c +++ b/src/or/dns.c @@ -12,8 +12,70 @@ **/ #include "or.h" +#include "circuitlist.h" +#include "circuituse.h" +#include "config.h" +#include "connection.h" +#include "connection_edge.h" +#include "control.h" +#include "dns.h" +#include "main.h" +#include "policies.h" +#include "relay.h" +#include "router.h" #include "ht.h" +#ifdef HAVE_EVENT2_DNS_H +#include <event2/event.h> +#include <event2/dns.h> +#else +#include <event.h> #include "eventdns.h" +#ifndef HAVE_EVDNS_SET_DEFAULT_OUTGOING_BIND_ADDRESS +#define HAVE_EVDNS_SET_DEFAULT_OUTGOING_BIND_ADDRESS +#endif +#endif + +#ifndef HAVE_EVENT2_DNS_H +struct evdns_base; +struct evdns_request; +#define evdns_base_new(x,y) tor_malloc(1) +#define evdns_base_clear_nameservers_and_suspend(base) \ + evdns_clear_nameservers_and_suspend() +#define evdns_base_search_clear(base) evdns_search_clear() +#define evdns_base_set_default_outgoing_bind_address(base, a, len) \ + evdns_set_default_outgoing_bind_address((a),(len)) +#define evdns_base_resolv_conf_parse(base, options, fname) \ + evdns_resolv_conf_parse((options), (fname)) +#define evdns_base_count_nameservers(base) \ + evdns_count_nameservers() +#define evdns_base_resume(base) \ + evdns_resume() +#define evdns_base_config_windows_nameservers(base) \ + evdns_config_windows_nameservers() +#define evdns_base_set_option_(base, opt, val) \ + evdns_set_option((opt),(val),DNS_OPTIONS_ALL) +/* Note: our internal eventdns.c, plus Libevent 1.4, used a 1 return to + * signify failure to launch a resolve. Libevent 2.0 uses a -1 return to + * signify a failure on a resolve, though if we're on Libevent 2.0, we should + * have event2/dns.h and never hit these macros. Regardless, 0 is success. */ +#define evdns_base_resolve_ipv4(base, addr, options, cb, ptr) \ + ((evdns_resolve_ipv4((addr), (options), (cb), (ptr))!=0) \ + ? NULL : ((void*)1)) +#define evdns_base_resolve_reverse(base, addr, options, cb, ptr) \ + ((evdns_resolve_reverse((addr), (options), (cb), (ptr))!=0) \ + ? NULL : ((void*)1)) +#define evdns_base_resolve_reverse_ipv6(base, addr, options, cb, ptr) \ + ((evdns_resolve_reverse_ipv6((addr), (options), (cb), (ptr))!=0) \ + ? NULL : ((void*)1)) + +#elif defined(LIBEVENT_VERSION_NUMBER) && LIBEVENT_VERSION_NUMBER < 0x02000303 +#define evdns_base_set_option_(base, opt, val) \ + evdns_base_set_option((base), (opt),(val),DNS_OPTIONS_ALL) + +#else +#define evdns_base_set_option_ evdns_base_set_option + +#endif /** Longest hostname we're willing to resolve. */ #define MAX_ADDRESSLEN 256 @@ -28,6 +90,9 @@ #define DNS_RESOLVE_FAILED_PERMANENT 2 #define DNS_RESOLVE_SUCCEEDED 3 +/** Our evdns_base; this structure handles all our name lookups. */ +static struct evdns_base *the_evdns_base = NULL; + /** Have we currently configured nameservers with eventdns? */ static int nameservers_configured = 0; /** Did our most recent attempt to configure nameservers with eventdns fail? */ @@ -89,6 +154,8 @@ typedef struct cached_resolve_t { uint32_t ttl; /**< What TTL did the nameserver tell us? */ /** Connections that want to know when we get an answer for this resolve. */ pending_connection_t *pending_connections; + /** Position of this element in the heap*/ + int minheap_idx; } cached_resolve_t; static void purge_expired_resolves(time_t now); @@ -211,8 +278,16 @@ dns_reset(void) { or_options_t *options = get_options(); if (! server_mode(options)) { - evdns_clear_nameservers_and_suspend(); - evdns_search_clear(); + + if (!the_evdns_base) { + if (!(the_evdns_base = evdns_base_new(tor_libevent_get_base(), 0))) { + log_err(LD_BUG, "Couldn't create an evdns_base"); + return -1; + } + } + + evdns_base_clear_nameservers_and_suspend(the_evdns_base); + evdns_base_search_clear(the_evdns_base); nameservers_configured = 0; tor_free(resolv_conf_fname); resolv_conf_mtime = 0; @@ -262,6 +337,8 @@ dns_get_expiry_ttl(uint32_t ttl) static void _free_cached_resolve(cached_resolve_t *r) { + if (!r) + return; while (r->pending_connections) { pending_connection_t *victim = r->pending_connections; r->pending_connections = victim->next; @@ -303,6 +380,7 @@ set_expiry(cached_resolve_t *resolve, time_t expires) resolve->expire = expires; smartlist_pqueue_add(cached_resolve_pqueue, _compare_cached_resolves_by_expiry, + STRUCT_OFFSET(cached_resolve_t, minheap_idx), resolve); } @@ -325,8 +403,7 @@ dns_free_all(void) _free_cached_resolve(item); } HT_CLEAR(cache_map, &cache_root); - if (cached_resolve_pqueue) - smartlist_free(cached_resolve_pqueue); + smartlist_free(cached_resolve_pqueue); cached_resolve_pqueue = NULL; tor_free(resolv_conf_fname); } @@ -349,7 +426,8 @@ purge_expired_resolves(time_t now) if (resolve->expire > now) break; smartlist_pqueue_pop(cached_resolve_pqueue, - _compare_cached_resolves_by_expiry); + _compare_cached_resolves_by_expiry, + STRUCT_OFFSET(cached_resolve_t, minheap_idx)); if (resolve->state == CACHE_STATE_PENDING) { log_debug(LD_EXIT, @@ -393,7 +471,7 @@ purge_expired_resolves(time_t now) log_err(LD_BUG, "The expired resolve we purged didn't match any in" " the cache. Tried to purge %s (%p); instead got %s (%p).", resolve->address, (void*)resolve, - removed ? removed->address : "NULL", (void*)remove); + removed ? removed->address : "NULL", (void*)removed); } tor_assert(removed == resolve); } else { @@ -611,7 +689,7 @@ dns_resolve_impl(edge_connection_t *exitconn, int is_resolve, * know the answer. */ if (tor_addr_from_str(&addr, exitconn->_base.address) >= 0) { if (tor_addr_family(&addr) == AF_INET) { - tor_addr_assign(&exitconn->_base.addr, &addr); + tor_addr_copy(&exitconn->_base.addr, &addr); exitconn->address_ttl = DEFAULT_DNS_TTL; return 1; } else { @@ -711,6 +789,7 @@ dns_resolve_impl(edge_connection_t *exitconn, int is_resolve, resolve = tor_malloc_zero(sizeof(cached_resolve_t)); resolve->magic = CACHED_RESOLVE_MAGIC; resolve->state = CACHE_STATE_PENDING; + resolve->minheap_idx = -1; resolve->is_reverse = is_reverse; strlcpy(resolve->address, exitconn->_base.address, sizeof(resolve->address)); @@ -807,7 +886,8 @@ connection_dns_remove(edge_connection_t *conn) tor_free(pend); log_debug(LD_EXIT, "First connection (fd %d) no longer waiting " "for resolve of %s", - conn->_base.s, escaped_safe_str(conn->_base.address)); + conn->_base.s, + escaped_safe_str(conn->_base.address)); return; } else { for ( ; pend->next; pend = pend->next) { @@ -1108,6 +1188,14 @@ configure_nameservers(int force) conf_fname = "/etc/resolv.conf"; #endif + if (!the_evdns_base) { + if (!(the_evdns_base = evdns_base_new(tor_libevent_get_base(), 0))) { + log_err(LD_BUG, "Couldn't create an evdns_base"); + return -1; + } + } + +#ifdef HAVE_EVDNS_SET_DEFAULT_OUTGOING_BIND_ADDRESS if (options->OutboundBindAddress) { tor_addr_t addr; if (tor_addr_from_str(&addr, options->OutboundBindAddress) < 0) { @@ -1118,20 +1206,17 @@ configure_nameservers(int force) struct sockaddr_storage ss; socklen = tor_addr_to_sockaddr(&addr, 0, (struct sockaddr *)&ss, sizeof(ss)); - if (socklen < 0) { + if (socklen <= 0) { log_warn(LD_BUG, "Couldn't convert outbound bind address to sockaddr." " Ignoring."); } else { - evdns_set_default_outgoing_bind_address((struct sockaddr *)&ss, - socklen); + evdns_base_set_default_outgoing_bind_address(the_evdns_base, + (struct sockaddr *)&ss, + socklen); } } } - - if (options->ServerDNSRandomizeCase) - evdns_set_option("randomize-case:", "1", DNS_OPTIONS_ALL); - else - evdns_set_option("randomize-case:", "0", DNS_OPTIONS_ALL); +#endif evdns_set_log_fn(evdns_log_cb); if (conf_fname) { @@ -1146,16 +1231,17 @@ configure_nameservers(int force) return 0; } if (nameservers_configured) { - evdns_search_clear(); - evdns_clear_nameservers_and_suspend(); + evdns_base_search_clear(the_evdns_base); + evdns_base_clear_nameservers_and_suspend(the_evdns_base); } log_info(LD_EXIT, "Parsing resolver configuration in '%s'", conf_fname); - if ((r = evdns_resolv_conf_parse(DNS_OPTIONS_ALL, conf_fname))) { + if ((r = evdns_base_resolv_conf_parse(the_evdns_base, + DNS_OPTIONS_ALL, conf_fname))) { log_warn(LD_EXIT, "Unable to parse '%s', or no nameservers in '%s' (%d)", conf_fname, conf_fname, r); goto err; } - if (evdns_count_nameservers() == 0) { + if (evdns_base_count_nameservers(the_evdns_base) == 0) { log_warn(LD_EXIT, "Unable to find any nameservers in '%s'.", conf_fname); goto err; } @@ -1163,38 +1249,47 @@ configure_nameservers(int force) resolv_conf_fname = tor_strdup(conf_fname); resolv_conf_mtime = st.st_mtime; if (nameservers_configured) - evdns_resume(); + evdns_base_resume(the_evdns_base); } #ifdef MS_WINDOWS else { if (nameservers_configured) { - evdns_search_clear(); - evdns_clear_nameservers_and_suspend(); + evdns_base_search_clear(the_evdns_base); + evdns_base_clear_nameservers_and_suspend(the_evdns_base); } - if (evdns_config_windows_nameservers()) { + if (evdns_base_config_windows_nameservers(the_evdns_base)) { log_warn(LD_EXIT,"Could not config nameservers."); goto err; } - if (evdns_count_nameservers() == 0) { + if (evdns_base_count_nameservers(the_evdns_base) == 0) { log_warn(LD_EXIT, "Unable to find any platform nameservers in " "your Windows configuration."); goto err; } if (nameservers_configured) - evdns_resume(); + evdns_base_resume(the_evdns_base); tor_free(resolv_conf_fname); resolv_conf_mtime = 0; } #endif - if (evdns_count_nameservers() == 1) { - evdns_set_option("max-timeouts:", "16", DNS_OPTIONS_ALL); - evdns_set_option("timeout:", "10", DNS_OPTIONS_ALL); +#define SET(k,v) evdns_base_set_option_(the_evdns_base, (k), (v)) + + if (evdns_base_count_nameservers(the_evdns_base) == 1) { + SET("max-timeouts:", "16"); + SET("timeout:", "10"); } else { - evdns_set_option("max-timeouts:", "3", DNS_OPTIONS_ALL); - evdns_set_option("timeout:", "5", DNS_OPTIONS_ALL); + SET("max-timeouts:", "3"); + SET("timeout:", "5"); } + if (options->ServerDNSRandomizeCase) + SET("randomize-case:", "1"); + else + SET("randomize-case:", "0"); + +#undef SET + dns_servers_relaunch_checks(); nameservers_configured = 1; @@ -1292,6 +1387,7 @@ static int launch_resolve(edge_connection_t *exitconn) { char *addr = tor_strdup(exitconn->_base.address); + struct evdns_request *req = NULL; tor_addr_t a; int r; int options = get_options()->ServerDNSSearchDomains ? 0 @@ -1307,28 +1403,34 @@ launch_resolve(edge_connection_t *exitconn) r = tor_addr_parse_reverse_lookup_name( &a, exitconn->_base.address, AF_UNSPEC, 0); + + tor_assert(the_evdns_base); if (r == 0) { log_info(LD_EXIT, "Launching eventdns request for %s", escaped_safe_str(exitconn->_base.address)); - r = evdns_resolve_ipv4(exitconn->_base.address, options, - evdns_callback, addr); + req = evdns_base_resolve_ipv4(the_evdns_base, + exitconn->_base.address, options, + evdns_callback, addr); } else if (r == 1) { log_info(LD_EXIT, "Launching eventdns reverse request for %s", escaped_safe_str(exitconn->_base.address)); if (tor_addr_family(&a) == AF_INET) - r = evdns_resolve_reverse(tor_addr_to_in(&a), DNS_QUERY_NO_SEARCH, + req = evdns_base_resolve_reverse(the_evdns_base, + tor_addr_to_in(&a), DNS_QUERY_NO_SEARCH, evdns_callback, addr); else - r = evdns_resolve_reverse_ipv6(tor_addr_to_in6(&a), DNS_QUERY_NO_SEARCH, + req = evdns_base_resolve_reverse_ipv6(the_evdns_base, + tor_addr_to_in6(&a), DNS_QUERY_NO_SEARCH, evdns_callback, addr); } else if (r == -1) { log_warn(LD_BUG, "Somehow a malformed in-addr.arpa address reached here."); } - if (r) { - log_warn(LD_EXIT, "eventdns rejected address %s: error %d.", - escaped_safe_str(addr), r); - r = evdns_err_is_transient(r) ? -2 : -1; + r = 0; + if (!req) { + log_warn(LD_EXIT, "eventdns rejected address %s.", + escaped_safe_str(addr)); + r = -1; tor_free(addr); /* There is no evdns request in progress; stop * addr from getting leaked. */ } @@ -1449,8 +1551,8 @@ evdns_wildcard_check_callback(int result, char type, int count, int ttl, } log(dns_wildcard_one_notice_given ? LOG_INFO : LOG_NOTICE, LD_EXIT, "Your DNS provider gave an answer for \"%s\", which " - "is not supposed to exist. Apparently they are hijacking " - "DNS failures. Trying to correct for this. We've noticed %d " + "is not supposed to exist. Apparently they are hijacking " + "DNS failures. Trying to correct for this. We've noticed %d " "possibly bad address%s so far.", string_address, strmap_size(dns_wildcard_response_count), (strmap_size(dns_wildcard_response_count) == 1) ? "" : "es"); @@ -1466,17 +1568,20 @@ static void launch_wildcard_check(int min_len, int max_len, const char *suffix) { char *addr; - int r; + struct evdns_request *req; addr = crypto_random_hostname(min_len, max_len, "", suffix); log_info(LD_EXIT, "Testing whether our DNS server is hijacking nonexistent " "domains with request for bogus hostname \"%s\"", addr); - r = evdns_resolve_ipv4(/* This "addr" tells us which address to resolve */ + tor_assert(the_evdns_base); + req = evdns_base_resolve_ipv4( + the_evdns_base, + /* This "addr" tells us which address to resolve */ addr, DNS_QUERY_NO_SEARCH, evdns_wildcard_check_callback, /* This "addr" is an argument to the callback*/ addr); - if (r) { + if (!req) { /* There is no evdns request in progress; stop addr from getting leaked */ tor_free(addr); } @@ -1488,6 +1593,7 @@ static void launch_test_addresses(int fd, short event, void *args) { or_options_t *options = get_options(); + struct evdns_request *req; (void)fd; (void)event; (void)args; @@ -1499,14 +1605,19 @@ launch_test_addresses(int fd, short event, void *args) * be an exit server.*/ if (!options->ServerDNSTestAddresses) return; - SMARTLIST_FOREACH(options->ServerDNSTestAddresses, const char *, address, - { - int r = evdns_resolve_ipv4(address, DNS_QUERY_NO_SEARCH, evdns_callback, - tor_strdup(address)); - if (r) - log_info(LD_EXIT, "eventdns rejected test address %s: error %d", - escaped_safe_str(address), r); - }); + tor_assert(the_evdns_base); + SMARTLIST_FOREACH_BEGIN(options->ServerDNSTestAddresses, + const char *, address) { + char *a = tor_strdup(address); + req = evdns_base_resolve_ipv4(the_evdns_base, + address, DNS_QUERY_NO_SEARCH, evdns_callback, a); + + if (!req) { + log_info(LD_EXIT, "eventdns rejected test address %s", + escaped_safe_str(address)); + tor_free(a); + } + } SMARTLIST_FOREACH_END(address); } #define N_WILDCARD_CHECKS 2 @@ -1547,7 +1658,7 @@ dns_launch_wildcard_checks(void) void dns_launch_correctness_checks(void) { - static struct event launch_event; + static struct event *launch_event = NULL; struct timeval timeout; if (!get_options()->ServerDNSDetectHijacking) return; @@ -1555,10 +1666,12 @@ dns_launch_correctness_checks(void) /* Wait a while before launching requests for test addresses, so we can * get the results from checking for wildcarding. */ - evtimer_set(&launch_event, launch_test_addresses, NULL); + if (! launch_event) + launch_event = tor_evtimer_new(tor_libevent_get_base(), + launch_test_addresses, NULL); timeout.tv_sec = 30; timeout.tv_usec = 0; - if (evtimer_add(&launch_event, &timeout)<0) { + if (evtimer_add(launch_event, &timeout)<0) { log_warn(LD_BUG, "Couldn't add timer for checking for dns hijacking"); } } @@ -1574,10 +1687,9 @@ dns_seems_to_be_broken(void) void dns_reset_correctness_checks(void) { - if (dns_wildcard_response_count) { - strmap_free(dns_wildcard_response_count, _tor_free); - dns_wildcard_response_count = NULL; - } + strmap_free(dns_wildcard_response_count, _tor_free); + dns_wildcard_response_count = NULL; + n_wildcard_requests = 0; if (dns_wildcard_list) { @@ -1622,6 +1734,30 @@ assert_resolve_ok(cached_resolve_t *resolve) } } +/** Return the number of DNS cache entries as an int */ +static int +dns_cache_entry_count(void) +{ + return HT_SIZE(&cache_root); +} + +/** Log memory information about our internal DNS cache at level 'severity'. */ +void +dump_dns_mem_usage(int severity) +{ + /* This should never be larger than INT_MAX. */ + int hash_count = dns_cache_entry_count(); + size_t hash_mem = sizeof(struct cached_resolve_t) * hash_count; + hash_mem += HT_MEM_USAGE(&cache_root); + + /* Print out the count and estimated size of our &cache_root. It undercounts + hostnames in cached reverse resolves. + */ + log(severity, LD_MM, "Our DNS cache has %d entries.", hash_count); + log(severity, LD_MM, "Our DNS cache size is approximately %u bytes.", + (unsigned)hash_mem); +} + #ifdef DEBUG_DNS_CACHE /** Exit with an assertion if the DNS cache is corrupt. */ static void @@ -1642,7 +1778,8 @@ _assert_cache_ok(void) return; smartlist_pqueue_assert_ok(cached_resolve_pqueue, - _compare_cached_resolves_by_expiry); + _compare_cached_resolves_by_expiry, + STRUCT_OFFSET(cached_resolve_t, minheap_idx)); SMARTLIST_FOREACH(cached_resolve_pqueue, cached_resolve_t *, res, { diff --git a/src/or/dns.h b/src/or/dns.h new file mode 100644 index 0000000000..25ff86e2c6 --- /dev/null +++ b/src/or/dns.h @@ -0,0 +1,31 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file dns.h + * \brief Header file for dns.c. + **/ + +#ifndef _TOR_DNS_H +#define _TOR_DNS_H + +int dns_init(void); +int has_dns_init_failed(void); +void dns_free_all(void); +uint32_t dns_clip_ttl(uint32_t ttl); +int dns_reset(void); +void connection_dns_remove(edge_connection_t *conn); +void assert_connection_edge_not_dns_pending(edge_connection_t *conn); +void assert_all_pending_dns_resolves_ok(void); +void dns_cancel_pending_resolve(const char *question); +int dns_resolve(edge_connection_t *exitconn); +void dns_launch_correctness_checks(void); +int dns_seems_to_be_broken(void); +void dns_reset_correctness_checks(void); +void dump_dns_mem_usage(int severity); + +#endif + diff --git a/src/or/dnsserv.c b/src/or/dnsserv.c index f8b5c3cdd2..243b730cbf 100644 --- a/src/or/dnsserv.c +++ b/src/or/dnsserv.c @@ -9,7 +9,21 @@ **/ #include "or.h" +#include "dnsserv.h" +#include "config.h" +#include "connection.h" +#include "connection_edge.h" +#include "control.h" +#include "main.h" +#include "policies.h" +#ifdef HAVE_EVENT2_DNS_H +#include <event2/dns.h> +#include <event2/dns_compat.h> +/* XXXX023 this implies we want an improved evdns */ +#include <event2/dns_struct.h> +#else #include "eventdns.h" +#endif /** Helper function: called by evdns whenever the client sends a request to our * DNSPort. We need to eventually answer the request <b>req</b>. @@ -85,12 +99,7 @@ evdns_server_callback(struct evdns_server_request *req, void *_data) evdns_server_request_respond(req, DNS_ERR_NONE); return; } - if (q->type == EVDNS_TYPE_A) { - /* Refuse any attempt to resolve a noconnect address, right now. */ - if (hostname_is_noconnect_address(q->name)) { - err = DNS_ERR_REFUSED; - } - } else { + if (q->type != EVDNS_TYPE_A) { tor_assert(q->type == EVDNS_TYPE_PTR); } @@ -132,17 +141,18 @@ evdns_server_callback(struct evdns_server_request *req, void *_data) control_event_stream_status(conn, STREAM_EVENT_NEW, 0); - /* Now, throw the connection over to get rewritten (which will answer it - * immediately if it's in the cache, or completely bogus, or automapped), - * and then attached to a circuit. */ + /* Now, unless a controller asked us to leave streams unattached, + * throw the connection over to get rewritten (which will + * answer it immediately if it's in the cache, or completely bogus, or + * automapped), and then attached to a circuit. */ log_info(LD_APP, "Passing request for %s to rewrite_and_attach.", - escaped_safe_str(q->name)); + escaped_safe_str_client(q->name)); q_name = tor_strdup(q->name); /* q could be freed in rewrite_and_attach */ - connection_ap_handshake_rewrite_and_attach(conn, NULL, NULL); + connection_ap_rewrite_and_attach_if_allowed(conn, NULL, NULL); /* Now, the connection is marked if it was bad. */ - log_info(LD_APP, "Passed request for %s to rewrite_and_attach.", - escaped_safe_str(q_name)); + log_info(LD_APP, "Passed request for %s to rewrite_and_attach_if_allowed.", + escaped_safe_str_client(q_name)); tor_free(q_name); } @@ -177,17 +187,18 @@ dnsserv_launch_request(const char *name, int reverse) return -1; } - /* Now, throw the connection over to get rewritten (which will answer it - * immediately if it's in the cache, or completely bogus, or automapped), - * and then attached to a circuit. */ + /* Now, unless a controller asked us to leave streams unattached, + * throw the connection over to get rewritten (which will + * answer it immediately if it's in the cache, or completely bogus, or + * automapped), and then attached to a circuit. */ log_info(LD_APP, "Passing request for %s to rewrite_and_attach.", - escaped_safe_str(name)); + escaped_safe_str_client(name)); q_name = tor_strdup(name); /* q could be freed in rewrite_and_attach */ - connection_ap_handshake_rewrite_and_attach(conn, NULL, NULL); + connection_ap_rewrite_and_attach_if_allowed(conn, NULL, NULL); /* Now, the connection is marked if it was bad. */ - log_info(LD_APP, "Passed request for %s to rewrite_and_attach.", - escaped_safe_str(q_name)); + log_info(LD_APP, "Passed request for %s to rewrite_and_attach_if_allowed.", + escaped_safe_str_client(q_name)); tor_free(q_name); return 0; } @@ -269,7 +280,7 @@ dnsserv_resolved(edge_connection_t *conn, conn->socks_request->command == SOCKS_COMMAND_RESOLVE) { evdns_server_request_add_a_reply(req, name, - 1, (char*)answer, ttl); + 1, answer, ttl); } else if (answer_type == RESOLVED_TYPE_HOSTNAME && answer_len < 256 && conn->socks_request->command == SOCKS_COMMAND_RESOLVE_PTR) { @@ -298,8 +309,8 @@ dnsserv_configure_listener(connection_t *conn) tor_assert(conn->s >= 0); tor_assert(conn->type == CONN_TYPE_AP_DNS_LISTENER); - conn->dns_server_port = evdns_add_server_port(conn->s, 0, - evdns_server_callback, NULL); + conn->dns_server_port = + tor_evdns_add_server_port(conn->s, 0, evdns_server_callback, NULL); } /** Free the evdns server port for <b>conn</b>, which must be an diff --git a/src/or/dnsserv.h b/src/or/dnsserv.h new file mode 100644 index 0000000000..fcca868885 --- /dev/null +++ b/src/or/dnsserv.h @@ -0,0 +1,26 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file dnsserv.h + * \brief Header file for dnsserv.c. + **/ + +#ifndef _TOR_DNSSERV_H +#define _TOR_DNSSERV_H + +void dnsserv_configure_listener(connection_t *conn); +void dnsserv_close_listener(connection_t *conn); +void dnsserv_resolved(edge_connection_t *conn, + int answer_type, + size_t answer_len, + const char *answer, + int ttl); +void dnsserv_reject_request(edge_connection_t *conn); +int dnsserv_launch_request(const char *name, int is_reverse); + +#endif + diff --git a/src/or/eventdns.c b/src/or/eventdns.c index bef6062681..9e3b4263ae 100644 --- a/src/or/eventdns.c +++ b/src/or/eventdns.c @@ -31,6 +31,7 @@ */ #include "eventdns_tor.h" +#include "../common/util.h" #include <sys/types.h> /* #define NDEBUG */ @@ -89,6 +90,7 @@ #include <stdarg.h> #include "eventdns.h" + #ifdef WIN32 #include <windows.h> #include <winsock2.h> @@ -173,8 +175,6 @@ struct evdns_request { /* these objects are kept in a circular list */ struct evdns_request *next, *prev; - u16 timeout_event_deleted; /**< Debugging: where was timeout_event - * deleted? 0 for "it's added." */ struct event timeout_event; u16 trans_id; /* the transaction id */ @@ -214,8 +214,6 @@ struct nameserver { struct event event; /* these objects are kept in a circular list */ struct nameserver *next, *prev; - u16 timeout_event_deleted; /**< Debugging: where was timeout_event - * deleted? 0 for "it's added." */ struct event timeout_event; /* used to keep the timeout for */ /* when we next probe this server. */ /* Valid if state == 0 */ @@ -474,51 +472,10 @@ sockaddr_eq(const struct sockaddr *sa1, const struct sockaddr *sa2, return 1; } -/* for debugging bug 929. XXXX022 */ -static int -_add_timeout_event(u16 *lineno, struct event *ev, struct timeval *to) -{ - *lineno = 0; - return evtimer_add(ev, to); -} -#define add_timeout_event(s, to) \ - (_add_timeout_event(&(s)->timeout_event_deleted, &(s)->timeout_event, (to))) - -/* for debugging bug 929. XXXX022 */ -static int -_del_timeout_event(u16 *lineno, struct event *ev, int line) -{ - if (*lineno) { - log(EVDNS_LOG_DEBUG, - "Duplicate timeout event_del from line %d: first call " - "was at %d.", line, (int)*lineno); - return 0; - } else { - *lineno = (u16)line; - return event_del(ev); - } -} -#define del_timeout_event(s) \ - (_del_timeout_event(&(s)->timeout_event_deleted, &(s)->timeout_event, \ - __LINE__)) -/* For debugging bug 929/957. XXXX022 */ -static int -_del_timeout_event_if_set(u16 *lineno, struct event *ev, int line) -{ - if (*lineno == 0) { - log(EVDNS_LOG_DEBUG, - "Event that I thought was non-added as of line %d " - "was actually added on line %d", - line, (int)*lineno); - *lineno = line; - return event_del(ev); - } - return 0; -} -#define del_timeout_event_if_set(s) \ - _del_timeout_event_if_set(&(s)->timeout_event_deleted, \ - &(s)->timeout_event, \ - __LINE__) +#define add_timeout_event(s, to) \ + (event_add(&(s)->timeout_event, (to))) +#define del_timeout_event(s) \ + (event_del(&(s)->timeout_event)) /* This walks the list of inflight requests to find the */ /* one with a matching transaction id. Returns NULL on */ @@ -555,7 +512,7 @@ static void nameserver_probe_failed(struct nameserver *const ns) { const struct timeval * timeout; del_timeout_event(ns); - CLEAR(&ns->timeout_event); + if (ns->state == 1) { /* This can happen if the nameserver acts in a way which makes us mark */ /* it as bad and then starts sending good replies. */ @@ -567,8 +524,6 @@ nameserver_probe_failed(struct nameserver *const ns) { global_nameserver_timeouts_length - 1)]; ns->failed_times++; - del_timeout_event_if_set(ns); - evtimer_set(&ns->timeout_event, nameserver_prod_callback, ns); if (add_timeout_event(ns, (struct timeval *) timeout) < 0) { log(EVDNS_LOG_WARN, "Error from libevent when adding timer event for %s", @@ -597,8 +552,6 @@ nameserver_failed(struct nameserver *const ns, const char *msg) { ns->state = 0; ns->failed_times = 1; - del_timeout_event_if_set(ns); - evtimer_set(&ns->timeout_event, nameserver_prod_callback, ns); if (add_timeout_event(ns, (struct timeval *) &global_nameserver_timeouts[0]) < 0) { log(EVDNS_LOG_WARN, "Error from libevent when adding timer event for %s", @@ -634,7 +587,6 @@ nameserver_up(struct nameserver *const ns) { log(EVDNS_LOG_WARN, "Nameserver %s is back up", debug_ntop((struct sockaddr *)&ns->address)); del_timeout_event(ns); - CLEAR(&ns->timeout_event); ns->state = 1; ns->failed_times = 0; ns->timedout = 0; @@ -666,7 +618,6 @@ request_finished(struct evdns_request *const req, struct evdns_request **head) { log(EVDNS_LOG_DEBUG, "Removing timeout for request %lx", (unsigned long) req); del_timeout_event(req); - CLEAR(&req->timeout_event); search_request_finished(req); global_requests_inflight--; @@ -1294,7 +1245,8 @@ nameserver_read(struct nameserver *ns) { for (;;) { const int r = - (int)recvfrom(ns->socket, packet, (socklen_t)sizeof(packet), 0, + (int)recvfrom(ns->socket, (void*)packet, + (socklen_t)sizeof(packet), 0, sa, &addrlen); if (r < 0) { int err = last_error(ns->socket); @@ -1325,7 +1277,7 @@ server_port_read(struct evdns_server_port *s) { for (;;) { addrlen = (socklen_t)sizeof(struct sockaddr_storage); - r = recvfrom(s->socket, packet, sizeof(packet), 0, + r = recvfrom(s->socket, (void*)packet, sizeof(packet), 0, (struct sockaddr*) &addr, &addrlen); if (r < 0) { int err = last_error(s->socket); @@ -1342,8 +1294,8 @@ server_port_read(struct evdns_server_port *s) { static void server_port_flush(struct evdns_server_port *port) { - while (port->pending_replies) { - struct server_request *req = port->pending_replies; + struct server_request *req = port->pending_replies; + while (req) { ssize_t r = sendto(port->socket, req->response, req->response_len, 0, (struct sockaddr*) &req->addr, (socklen_t)req->addrlen); if (r < 0) { @@ -1355,6 +1307,9 @@ server_port_flush(struct evdns_server_port *port) if (server_request_free(req)) { /* we released the last reference to req->port. */ return; + } else { + assert(port->pending_replies != req); + req = port->pending_replies; } } @@ -1713,7 +1668,7 @@ evdns_server_request_add_reply(struct evdns_server_request *_req, int section, c /* exported function */ int -evdns_server_request_add_a_reply(struct evdns_server_request *req, const char *name, int n, void *addrs, int ttl) +evdns_server_request_add_a_reply(struct evdns_server_request *req, const char *name, int n, const void *addrs, int ttl) { return evdns_server_request_add_reply( req, EVDNS_ANSWER_SECTION, name, TYPE_A, CLASS_INET, @@ -1722,7 +1677,7 @@ evdns_server_request_add_a_reply(struct evdns_server_request *req, const char *n /* exported function */ int -evdns_server_request_add_aaaa_reply(struct evdns_server_request *req, const char *name, int n, void *addrs, int ttl) +evdns_server_request_add_aaaa_reply(struct evdns_server_request *req, const char *name, int n, const void *addrs, int ttl) { return evdns_server_request_add_reply( req, EVDNS_ANSWER_SECTION, name, TYPE_AAAA, CLASS_INET, @@ -2044,9 +1999,8 @@ evdns_request_timeout_callback(int fd, short events, void *arg) { /* retransmit it */ /* Stop waiting for the timeout. No need to do this in * request_finished; that one already deletes the timeout event. - * XXXX021 port this change to libevent. */ + * XXXX023 port this change to libevent. */ del_timeout_event(req); - CLEAR(&req->timeout_event); evdns_request_transmit(req); } } @@ -2059,7 +2013,8 @@ evdns_request_timeout_callback(int fd, short events, void *arg) { /* 2 other failure */ static int evdns_request_transmit_to(struct evdns_request *req, struct nameserver *server) { - const ssize_t r = send(server->socket, req->request, req->request_len, 0); + const ssize_t r = send(server->socket, (void*)req->request, + req->request_len, 0); if (r < 0) { int err = last_error(server->socket); if (error_is_eagain(err)) return 1; @@ -2109,8 +2064,7 @@ evdns_request_transmit(struct evdns_request *req) { /* transmitted; we need to check for timeout. */ log(EVDNS_LOG_DEBUG, "Setting timeout for request %lx", (unsigned long) req); - del_timeout_event_if_set(req); - evtimer_set(&req->timeout_event, evdns_request_timeout_callback, req); + if (add_timeout_event(req, &global_timeout) < 0) { log(EVDNS_LOG_WARN, "Error from libevent when adding timer for request %lx", @@ -2225,7 +2179,6 @@ evdns_clear_nameservers_and_suspend(void) (void) event_del(&server->event); CLEAR(&server->event); del_timeout_event(server); - CLEAR(&server->timeout_event); if (server->socket >= 0) CLOSE_SOCKET(server->socket); CLEAR(server); @@ -2243,7 +2196,6 @@ evdns_clear_nameservers_and_suspend(void) req->ns = NULL; /* ???? What to do about searches? */ del_timeout_event(req); - CLEAR(&req->timeout_event); req->trans_id = 0; req->transmit_me = 0; @@ -2292,6 +2244,21 @@ evdns_resume(void) } static int +sockaddr_is_loopback(const struct sockaddr *addr) +{ + static const char LOOPBACK_S6[16] = + "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\1"; + if (addr->sa_family == AF_INET) { + struct sockaddr_in *sin = (struct sockaddr_in *)addr; + return (ntohl(sin->sin_addr.s_addr) & 0xff000000) == 0x7f000000; + } else if (addr->sa_family == AF_INET6) { + struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)addr; + return !memcmp(sin6->sin6_addr.s6_addr, LOOPBACK_S6, 16); + } + return 0; +} + +static int _evdns_nameserver_add_impl(const struct sockaddr *address, socklen_t addrlen) { /* first check to see if we already have this nameserver */ @@ -2318,7 +2285,8 @@ _evdns_nameserver_add_impl(const struct sockaddr *address, if (!ns) return -1; memset(ns, 0, sizeof(struct nameserver)); - ns->timeout_event_deleted = __LINE__; + + evtimer_set(&ns->timeout_event, nameserver_prod_callback, ns); ns->socket = socket(PF_INET, SOCK_DGRAM, 0); if (ns->socket < 0) { err = 1; goto out1; } @@ -2331,7 +2299,8 @@ _evdns_nameserver_add_impl(const struct sockaddr *address, fcntl(ns->socket, F_SETFL, O_NONBLOCK); #endif - if (global_bind_addr_is_set) { + if (global_bind_addr_is_set && + !sockaddr_is_loopback((struct sockaddr*)&global_bind_address)) { if (bind(ns->socket, (struct sockaddr *)&global_bind_address, global_bind_addrlen) < 0) { log(EVDNS_LOG_DEBUG, "Couldn't bind to outgoing address."); @@ -2553,7 +2522,8 @@ request_new(int type, const char *name, int flags, } memset(req, 0, sizeof(struct evdns_request)); - req->timeout_event_deleted = __LINE__; + + evtimer_set(&req->timeout_event, evdns_request_timeout_callback, req); if (global_randomize_case) { unsigned i; @@ -2941,14 +2911,6 @@ evdns_resolv_set_defaults(int flags) { if (flags & DNS_OPTION_NAMESERVERS) evdns_nameserver_ip_add("127.0.0.1"); } -#ifndef HAVE_STRTOK_R -static char * -strtok_r(char *s, const char *delim, char **state) { - (void)state; - return strtok(s, delim); -} -#endif - /* helper version of atoi which returns -1 on error */ static int strtoint(const char *const str) { @@ -3025,9 +2987,9 @@ static void resolv_conf_parse_line(char *const start, int flags) { char *strtok_state; static const char *const delims = " \t"; -#define NEXT_TOKEN strtok_r(NULL, delims, &strtok_state) +#define NEXT_TOKEN tor_strtok_r(NULL, delims, &strtok_state) - char *const first_token = strtok_r(start, delims, &strtok_state); + char *const first_token = tor_strtok_r(start, delims, &strtok_state); if (!first_token) return; if (!strcmp(first_token, "nameserver") && (flags & DNS_OPTION_NAMESERVERS)) { @@ -3171,14 +3133,13 @@ load_nameservers_with_getnetworkparams(void) IP_ADDR_STRING *ns; GetNetworkParams_fn_t fn; - /* XXXX Possibly, we should hardcode the location of this DLL. */ - if (!(handle = LoadLibrary("iphlpapi.dll"))) { + if (!(handle = load_windows_system_library(TEXT("iphlpapi.dll")))) { log(EVDNS_LOG_WARN, "Could not open iphlpapi.dll"); /* right now status = 0, doesn't that mean "good" - mikec */ status = -1; goto done; } - if (!(fn = (GetNetworkParams_fn_t) GetProcAddress(handle, "GetNetworkParams"))) { + if (!(fn = (GetNetworkParams_fn_t) GetProcAddress(handle, TEXT("GetNetworkParams")))) { log(EVDNS_LOG_WARN, "Could not get address of function."); /* same as above */ status = -1; @@ -3241,9 +3202,10 @@ load_nameservers_with_getnetworkparams(void) } static int -config_nameserver_from_reg_key(HKEY key, const char *subkey) +config_nameserver_from_reg_key(HKEY key, const TCHAR *subkey) { char *buf; + char ansibuf[MAX_PATH] = {0}; DWORD bufsz = 0, type = 0; int status = 0; @@ -3255,24 +3217,30 @@ config_nameserver_from_reg_key(HKEY key, const char *subkey) if (RegQueryValueEx(key, subkey, 0, &type, (LPBYTE)buf, &bufsz) == ERROR_SUCCESS && bufsz > 1) { - status = evdns_nameserver_ip_add_line(buf); + wcstombs(ansibuf,(wchar_t*)buf,MAX_PATH);/*XXXX UNICODE */ + status = evdns_nameserver_ip_add_line(ansibuf); } mm_free(buf); return status; } -#define SERVICES_KEY "System\\CurrentControlSet\\Services\\" -#define WIN_NS_9X_KEY SERVICES_KEY "VxD\\MSTCP" -#define WIN_NS_NT_KEY SERVICES_KEY "Tcpip\\Parameters" +#define SERVICES_KEY TEXT("System\\CurrentControlSet\\Services\\") +#define WIN_NS_9X_KEY SERVICES_KEY TEXT("VxD\\MSTCP") +#define WIN_NS_NT_KEY SERVICES_KEY TEXT("Tcpip\\Parameters") static int load_nameservers_from_registry(void) { int found = 0; int r; + OSVERSIONINFO info; + memset(&info, 0, sizeof(info)); + info.dwOSVersionInfoSize = sizeof (info); + GetVersionEx(&info); + #define TRY(k, name) \ - if (!found && config_nameserver_from_reg_key(k,name) == 0) { \ + if (!found && config_nameserver_from_reg_key(k,TEXT(name)) == 0) { \ log(EVDNS_LOG_DEBUG,"Found nameservers in %s/%s",#k,name); \ found = 1; \ } else if (!found) { \ @@ -3280,7 +3248,7 @@ load_nameservers_from_registry(void) #k,#name); \ } - if (((int)GetVersion()) > 0) { /* NT */ + if (info.dwMajorVersion >= 5) { /* NT */ HKEY nt_key = 0, interfaces_key = 0; if (RegOpenKeyEx(HKEY_LOCAL_MACHINE, WIN_NS_NT_KEY, 0, @@ -3288,7 +3256,7 @@ load_nameservers_from_registry(void) log(EVDNS_LOG_DEBUG,"Couldn't open nt key, %d",(int)GetLastError()); return -1; } - r = RegOpenKeyEx(nt_key, "Interfaces", 0, + r = RegOpenKeyEx(nt_key, TEXT("Interfaces"), 0, KEY_QUERY_VALUE|KEY_ENUMERATE_SUB_KEYS, &interfaces_key); if (r != ERROR_SUCCESS) { @@ -3384,8 +3352,7 @@ evdns_shutdown(int fail_requests) if (server->socket >= 0) CLOSE_SOCKET(server->socket); (void) event_del(&server->event); - if (server->state == 0) - del_timeout_event(server); + del_timeout_event(server); CLEAR(server); mm_free(server); if (server_next == server_head) diff --git a/src/or/eventdns.h b/src/or/eventdns.h index bf3b64d08a..2fe4ac9371 100644 --- a/src/or/eventdns.h +++ b/src/or/eventdns.h @@ -323,8 +323,8 @@ struct evdns_server_port *evdns_add_server_port(int socket, int is_tcp, evdns_re void evdns_close_server_port(struct evdns_server_port *port); int evdns_server_request_add_reply(struct evdns_server_request *req, int section, const char *name, int type, int class, int ttl, int datalen, int is_name, const char *data); -int evdns_server_request_add_a_reply(struct evdns_server_request *req, const char *name, int n, void *addrs, int ttl); -int evdns_server_request_add_aaaa_reply(struct evdns_server_request *req, const char *name, int n, void *addrs, int ttl); +int evdns_server_request_add_a_reply(struct evdns_server_request *req, const char *name, int n, const void *addrs, int ttl); +int evdns_server_request_add_aaaa_reply(struct evdns_server_request *req, const char *name, int n, const void *addrs, int ttl); int evdns_server_request_add_ptr_reply(struct evdns_server_request *req, struct in_addr *in, const char *inaddr_name, const char *hostname, int ttl); int evdns_server_request_add_cname_reply(struct evdns_server_request *req, const char *name, const char *cname, int ttl); diff --git a/src/or/geoip.c b/src/or/geoip.c index 360b65f41c..5bb2410a75 100644 --- a/src/or/geoip.c +++ b/src/or/geoip.c @@ -3,15 +3,23 @@ /** * \file geoip.c - * \brief Functions related to maintaining an IP-to-country database and to - * summarizing client connections by country. + * \brief Functions related to maintaining an IP-to-country database; + * to summarizing client connections by country to entry guards, bridges, + * and directory servers; and for statistics on answering network status + * requests. */ #define GEOIP_PRIVATE #include "or.h" #include "ht.h" +#include "config.h" +#include "control.h" +#include "dnsserv.h" +#include "geoip.h" +#include "routerlist.h" static void clear_geoip_db(void); +static void init_geoip_countries(void); /** An entry from the GeoIP file: maps an IP range to a country. */ typedef struct geoip_entry_t { @@ -20,16 +28,11 @@ typedef struct geoip_entry_t { intptr_t country; /**< An index into geoip_countries */ } geoip_entry_t; -/** For how many periods should we remember per-country request history? */ -#define REQUEST_HIST_LEN 3 -/** How long are the periods for which we should remember request history? */ -#define REQUEST_HIST_PERIOD (8*60*60) - /** A per-country record for GeoIP request history. */ typedef struct geoip_country_t { char countrycode[3]; - uint32_t n_v2_ns_requests[REQUEST_HIST_LEN]; - uint32_t n_v3_ns_requests[REQUEST_HIST_LEN]; + uint32_t n_v2_ns_requests; + uint32_t n_v3_ns_requests; } geoip_country_t; /** A list of geoip_country_t */ @@ -42,7 +45,7 @@ static strmap_t *country_idxplus1_by_lc_code = NULL; static smartlist_t *geoip_entries = NULL; /** Return the index of the <b>country</b>'s entry in the GeoIP DB - * if it is a valid 2-letter country code, otherwise return zero. + * if it is a valid 2-letter country code, otherwise return -1. */ country_t geoip_get_country(const char *country) @@ -101,11 +104,11 @@ geoip_parse_entry(const char *line) { unsigned int low, high; char b[3]; - if (!geoip_countries) { - geoip_countries = smartlist_create(); + if (!geoip_countries) + init_geoip_countries(); + if (!geoip_entries) geoip_entries = smartlist_create(); - country_idxplus1_by_lc_code = strmap_new(); - } + while (TOR_ISSPACE(*line)) ++line; if (*line == '#') @@ -142,6 +145,7 @@ _geoip_compare_entries(const void **_a, const void **_b) static int _geoip_compare_key_to_entry(const void *_key, const void **_member) { + /* No alignment issue here, since _key really is a pointer to uint32_t */ const uint32_t addr = *(uint32_t *)_key; const geoip_entry_t *entry = *_member; if (addr < entry->ip_low) @@ -160,6 +164,24 @@ should_record_bridge_info(or_options_t *options) return options->BridgeRelay && options->BridgeRecordUsageByCountry; } +/** Set up a new list of geoip countries with no countries (yet) set in it, + * except for the unknown country. + */ +static void +init_geoip_countries(void) +{ + geoip_country_t *geoip_unresolved; + geoip_countries = smartlist_create(); + /* Add a geoip_country_t for requests that could not be resolved to a + * country as first element (index 0) to geoip_countries. */ + geoip_unresolved = tor_malloc_zero(sizeof(geoip_country_t)); + strlcpy(geoip_unresolved->countrycode, "??", + sizeof(geoip_unresolved->countrycode)); + smartlist_add(geoip_countries, geoip_unresolved); + country_idxplus1_by_lc_code = strmap_new(); + strmap_set_lc(country_idxplus1_by_lc_code, "??", (void*)(1)); +} + /** Clear the GeoIP database and reload it from the file * <b>filename</b>. Return 0 on success, -1 on failure. * @@ -185,16 +207,14 @@ geoip_load_file(const char *filename, or_options_t *options) filename, msg); return -1; } - if (!geoip_countries) { - geoip_countries = smartlist_create(); - country_idxplus1_by_lc_code = strmap_new(); - } + if (!geoip_countries) + init_geoip_countries(); if (geoip_entries) { SMARTLIST_FOREACH(geoip_entries, geoip_entry_t *, e, tor_free(e)); smartlist_free(geoip_entries); } geoip_entries = smartlist_create(); - log_notice(LD_GENERAL, "Parsing GEOIP file."); + log_notice(LD_GENERAL, "Parsing GEOIP file %s.", filename); while (!feof(f)) { char buf[512]; if (fgets(buf, (int)sizeof(buf), f) == NULL) @@ -215,9 +235,10 @@ geoip_load_file(const char *filename, or_options_t *options) } /** Given an IP address in host order, return a number representing the - * country to which that address belongs, or -1 for unknown. The return value - * will always be less than geoip_get_n_countries(). To decode it, - * call geoip_get_country_name(). + * country to which that address belongs, -1 for "No geoip information + * available", or 0 for the 'unknown country'. The return value will always + * be less than geoip_get_n_countries(). To decode it, call + * geoip_get_country_name(). */ int geoip_get_country_by_ip(uint32_t ipaddr) @@ -226,13 +247,15 @@ geoip_get_country_by_ip(uint32_t ipaddr) if (!geoip_entries) return -1; ent = smartlist_bsearch(geoip_entries, &ipaddr, _geoip_compare_key_to_entry); - return ent ? (int)ent->country : -1; + return ent ? (int)ent->country : 0; } /** Return the number of countries recognized by the GeoIP database. */ int geoip_get_n_countries(void) { + if (!geoip_countries) + init_geoip_countries(); return (int) smartlist_len(geoip_countries); } @@ -261,23 +284,22 @@ geoip_is_loaded(void) typedef struct clientmap_entry_t { HT_ENTRY(clientmap_entry_t) node; uint32_t ipaddr; - time_t last_seen; /* The last 2 bits of this value hold the client - * operation. */ + /** Time when we last saw this IP address, in MINUTES since the epoch. + * + * (This will run out of space around 4011 CE. If Tor is still in use around + * 4000 CE, please remember to add more bits to last_seen_in_minutes.) */ + unsigned int last_seen_in_minutes:30; + unsigned int action:2; } clientmap_entry_t; -#define ACTION_MASK 3 +/** Largest allowable value for last_seen_in_minutes. (It's a 30-bit field, + * so it can hold up to (1u<<30)-1, or 0x3fffffffu. + */ +#define MAX_LAST_SEEN_IN_MINUTES 0X3FFFFFFFu /** Map from client IP address to last time seen. */ static HT_HEAD(clientmap, clientmap_entry_t) client_history = HT_INITIALIZER(); -/** Time at which we started tracking client IP history. */ -static time_t client_history_starts = 0; - -/** When did the current period of checking per-country request history - * start? */ -static time_t current_request_period_starts = 0; -/** How many older request periods are we remembering? */ -static int n_old_request_periods = 0; /** Hashtable helper: compute a hash of a clientmap_entry_t. */ static INLINE unsigned @@ -289,7 +311,7 @@ clientmap_entry_hash(const clientmap_entry_t *a) static INLINE int clientmap_entries_eq(const clientmap_entry_t *a, const clientmap_entry_t *b) { - return a->ipaddr == b->ipaddr; + return a->ipaddr == b->ipaddr && a->action == b->action; } HT_PROTOTYPE(clientmap, clientmap_entry_t, node, clientmap_entry_hash, @@ -297,8 +319,87 @@ HT_PROTOTYPE(clientmap, clientmap_entry_t, node, clientmap_entry_hash, HT_GENERATE(clientmap, clientmap_entry_t, node, clientmap_entry_hash, clientmap_entries_eq, 0.6, malloc, realloc, free); +/** Clear history of connecting clients used by entry and bridge stats. */ +static void +client_history_clear(void) +{ + clientmap_entry_t **ent, **next, *this; + for (ent = HT_START(clientmap, &client_history); ent != NULL; + ent = next) { + if ((*ent)->action == GEOIP_CLIENT_CONNECT) { + this = *ent; + next = HT_NEXT_RMV(clientmap, &client_history, ent); + tor_free(this); + } else { + next = HT_NEXT(clientmap, &client_history, ent); + } + } +} + +/** How often do we update our estimate which share of v2 and v3 directory + * requests is sent to us? We could as well trigger updates of shares from + * network status updates, but that means adding a lot of calls into code + * that is independent from geoip stats (and keeping them up-to-date). We + * are perfectly fine with an approximation of 15-minute granularity. */ +#define REQUEST_SHARE_INTERVAL (15 * 60) + +/** When did we last determine which share of v2 and v3 directory requests + * is sent to us? */ +static time_t last_time_determined_shares = 0; + +/** Sum of products of v2 shares times the number of seconds for which we + * consider these shares as valid. */ +static double v2_share_times_seconds; + +/** Sum of products of v3 shares times the number of seconds for which we + * consider these shares as valid. */ +static double v3_share_times_seconds; + +/** Number of seconds we are determining v2 and v3 shares. */ +static int share_seconds; + +/** Try to determine which fraction of v2 and v3 directory requests aimed at + * caches will be sent to us at time <b>now</b> and store that value in + * order to take a mean value later on. */ +static void +geoip_determine_shares(time_t now) +{ + double v2_share = 0.0, v3_share = 0.0; + if (router_get_my_share_of_directory_requests(&v2_share, &v3_share) < 0) + return; + if (last_time_determined_shares) { + v2_share_times_seconds += v2_share * + ((double) (now - last_time_determined_shares)); + v3_share_times_seconds += v3_share * + ((double) (now - last_time_determined_shares)); + share_seconds += (int)(now - last_time_determined_shares); + } + last_time_determined_shares = now; +} + +/** Calculate which fraction of v2 and v3 directory requests aimed at caches + * have been sent to us since the last call of this function up to time + * <b>now</b>. Set *<b>v2_share_out</b> and *<b>v3_share_out</b> to the + * fractions of v2 and v3 protocol shares we expect to have seen. Reset + * counters afterwards. Return 0 on success, -1 on failure (e.g. when zero + * seconds have passed since the last call).*/ +static int +geoip_get_mean_shares(time_t now, double *v2_share_out, + double *v3_share_out) +{ + geoip_determine_shares(now); + if (!share_seconds) + return -1; + *v2_share_out = v2_share_times_seconds / ((double) share_seconds); + *v3_share_out = v3_share_times_seconds / ((double) share_seconds); + v2_share_times_seconds = v3_share_times_seconds = 0.0; + share_seconds = 0; + return 0; +} + /** Note that we've seen a client connect from the IP <b>addr</b> (host order) - * at time <b>now</b>. Ignored by all but bridges. */ + * at time <b>now</b>. Ignored by all but bridges and directories if + * configured accordingly. */ void geoip_note_client_seen(geoip_client_action_t action, uint32_t addr, time_t now) @@ -306,72 +407,46 @@ geoip_note_client_seen(geoip_client_action_t action, or_options_t *options = get_options(); clientmap_entry_t lookup, *ent; if (action == GEOIP_CLIENT_CONNECT) { - if (!(options->BridgeRelay && options->BridgeRecordUsageByCountry)) - return; - /* Did we recently switch from bridge to relay or back? */ - if (client_history_starts > now) + /* Only remember statistics as entry guard or as bridge. */ + if (!options->EntryStatistics && + (!(options->BridgeRelay && options->BridgeRecordUsageByCountry))) return; } else { -#ifndef ENABLE_GEOIP_STATS - return; -#else if (options->BridgeRelay || options->BridgeAuthoritativeDir || - !options->DirRecordUsageByCountry) + !options->DirReqStatistics) return; -#endif } - /* Rotate the current request period. */ - while (current_request_period_starts + REQUEST_HIST_PERIOD < now) { - if (!geoip_countries) - geoip_countries = smartlist_create(); - if (!current_request_period_starts) { - current_request_period_starts = now; - break; - } - SMARTLIST_FOREACH(geoip_countries, geoip_country_t *, c, { - memmove(&c->n_v2_ns_requests[0], &c->n_v2_ns_requests[1], - sizeof(uint32_t)*(REQUEST_HIST_LEN-1)); - memmove(&c->n_v3_ns_requests[0], &c->n_v3_ns_requests[1], - sizeof(uint32_t)*(REQUEST_HIST_LEN-1)); - c->n_v2_ns_requests[REQUEST_HIST_LEN-1] = 0; - c->n_v3_ns_requests[REQUEST_HIST_LEN-1] = 0; - }); - current_request_period_starts += REQUEST_HIST_PERIOD; - if (n_old_request_periods < REQUEST_HIST_LEN-1) - ++n_old_request_periods; - } - - /* We use the low 3 bits of the time to encode the action. Since we're - * potentially remembering tons of clients, we don't want to make - * clientmap_entry_t larger than it has to be. */ - now = (now & ~ACTION_MASK) | (((int)action) & ACTION_MASK); lookup.ipaddr = addr; + lookup.action = (int)action; ent = HT_FIND(clientmap, &client_history, &lookup); - if (ent) { - ent->last_seen = now; - } else { + if (! ent) { ent = tor_malloc_zero(sizeof(clientmap_entry_t)); ent->ipaddr = addr; - ent->last_seen = now; + ent->action = (int)action; HT_INSERT(clientmap, &client_history, ent); } + if (now / 60 <= (int)MAX_LAST_SEEN_IN_MINUTES && now >= 0) + ent->last_seen_in_minutes = (unsigned)(now/60); + else + ent->last_seen_in_minutes = 0; if (action == GEOIP_CLIENT_NETWORKSTATUS || action == GEOIP_CLIENT_NETWORKSTATUS_V2) { int country_idx = geoip_get_country_by_ip(addr); + if (country_idx < 0) + country_idx = 0; /** unresolved requests are stored at index 0. */ if (country_idx >= 0 && country_idx < smartlist_len(geoip_countries)) { geoip_country_t *country = smartlist_get(geoip_countries, country_idx); if (action == GEOIP_CLIENT_NETWORKSTATUS) - ++country->n_v3_ns_requests[REQUEST_HIST_LEN-1]; + ++country->n_v3_ns_requests; else - ++country->n_v2_ns_requests[REQUEST_HIST_LEN-1]; + ++country->n_v2_ns_requests; } - } - if (!client_history_starts) { - client_history_starts = now; - current_request_period_starts = now; + /* Periodically determine share of requests that we should see */ + if (last_time_determined_shares + REQUEST_SHARE_INTERVAL < now) + geoip_determine_shares(now); } } @@ -380,8 +455,8 @@ geoip_note_client_seen(geoip_client_action_t action, static int _remove_old_client_helper(struct clientmap_entry_t *ent, void *_cutoff) { - time_t cutoff = *(time_t*)_cutoff; - if (ent->last_seen < cutoff) { + time_t cutoff = *(time_t*)_cutoff / 60; + if (ent->last_seen_in_minutes < cutoff) { tor_free(ent); return 1; } else { @@ -389,18 +464,45 @@ _remove_old_client_helper(struct clientmap_entry_t *ent, void *_cutoff) } } -/** Forget about all clients that haven't connected since <b>cutoff</b>. - * If <b>cutoff</b> is in the future, clients won't be added to the history - * until this time is reached. This is useful to prevent relays that switch - * to bridges from reporting unbelievable numbers of clients. */ +/** Forget about all clients that haven't connected since <b>cutoff</b>. */ void geoip_remove_old_clients(time_t cutoff) { clientmap_HT_FOREACH_FN(&client_history, _remove_old_client_helper, &cutoff); - if (client_history_starts < cutoff) - client_history_starts = cutoff; +} + +/** How many responses are we giving to clients requesting v2 network + * statuses? */ +static uint32_t ns_v2_responses[GEOIP_NS_RESPONSE_NUM]; + +/** How many responses are we giving to clients requesting v3 network + * statuses? */ +static uint32_t ns_v3_responses[GEOIP_NS_RESPONSE_NUM]; + +/** Note that we've rejected a client's request for a v2 or v3 network + * status, encoded in <b>action</b> for reason <b>reason</b> at time + * <b>now</b>. */ +void +geoip_note_ns_response(geoip_client_action_t action, + geoip_ns_response_t response) +{ + static int arrays_initialized = 0; + if (!get_options()->DirReqStatistics) + return; + if (!arrays_initialized) { + memset(ns_v2_responses, 0, sizeof(ns_v2_responses)); + memset(ns_v3_responses, 0, sizeof(ns_v3_responses)); + arrays_initialized = 1; + } + tor_assert(action == GEOIP_CLIENT_NETWORKSTATUS || + action == GEOIP_CLIENT_NETWORKSTATUS_V2); + tor_assert(response < GEOIP_NS_RESPONSE_NUM); + if (action == GEOIP_CLIENT_NETWORKSTATUS) + ns_v3_responses[response]++; + else + ns_v2_responses[response]++; } /** Do not mention any country from which fewer than this number of IPs have @@ -414,13 +516,6 @@ geoip_remove_old_clients(time_t cutoff) * multiple of this value. */ #define IP_GRANULARITY 8 -/** Return the time at which we started recording geoip data. */ -time_t -geoip_get_history_start(void) -{ - return client_history_starts; -} - /** Helper type: used to sort per-country totals by value. */ typedef struct c_hist_t { char country[3]; /**< Two-letter country code. */ @@ -442,97 +537,320 @@ _c_hist_compare(const void **_a, const void **_b) return strcmp(a->country, b->country); } -/** How long do we have to have observed per-country request history before we - * are willing to talk about it? */ -#define GEOIP_MIN_OBSERVATION_TIME (12*60*60) +/** When there are incomplete directory requests at the end of a 24-hour + * period, consider those requests running for longer than this timeout as + * failed, the others as still running. */ +#define DIRREQ_TIMEOUT (10*60) -/** Return the lowest x such that x is at least <b>number</b>, and x modulo - * <b>divisor</b> == 0. */ -static INLINE unsigned -round_to_next_multiple_of(unsigned number, unsigned divisor) +/** Entry in a map from either conn->global_identifier for direct requests + * or a unique circuit identifier for tunneled requests to request time, + * response size, and completion time of a network status request. Used to + * measure download times of requests to derive average client + * bandwidths. */ +typedef struct dirreq_map_entry_t { + HT_ENTRY(dirreq_map_entry_t) node; + /** Unique identifier for this network status request; this is either the + * conn->global_identifier of the dir conn (direct request) or a new + * locally unique identifier of a circuit (tunneled request). This ID is + * only unique among other direct or tunneled requests, respectively. */ + uint64_t dirreq_id; + unsigned int state:3; /**< State of this directory request. */ + unsigned int type:1; /**< Is this a direct or a tunneled request? */ + unsigned int completed:1; /**< Is this request complete? */ + unsigned int action:2; /**< Is this a v2 or v3 request? */ + /** When did we receive the request and started sending the response? */ + struct timeval request_time; + size_t response_size; /**< What is the size of the response in bytes? */ + struct timeval completion_time; /**< When did the request succeed? */ +} dirreq_map_entry_t; + +/** Map of all directory requests asking for v2 or v3 network statuses in + * the current geoip-stats interval. Values are + * of type *<b>dirreq_map_entry_t</b>. */ +static HT_HEAD(dirreqmap, dirreq_map_entry_t) dirreq_map = + HT_INITIALIZER(); + +static int +dirreq_map_ent_eq(const dirreq_map_entry_t *a, + const dirreq_map_entry_t *b) { - number += divisor - 1; - number -= number % divisor; - return number; + return a->dirreq_id == b->dirreq_id && a->type == b->type; } -/** Return a newly allocated comma-separated string containing entries for all - * the countries from which we've seen enough clients connect. The entry - * format is cc=num where num is the number of IPs we've seen connecting from - * that country, and cc is a lowercased country code. Returns NULL if we don't - * want to export geoip data yet. */ -char * -geoip_get_client_history(time_t now, geoip_client_action_t action) +static unsigned +dirreq_map_ent_hash(const dirreq_map_entry_t *entry) +{ + unsigned u = (unsigned) entry->dirreq_id; + u += entry->type << 20; + return u; +} + +HT_PROTOTYPE(dirreqmap, dirreq_map_entry_t, node, dirreq_map_ent_hash, + dirreq_map_ent_eq); +HT_GENERATE(dirreqmap, dirreq_map_entry_t, node, dirreq_map_ent_hash, + dirreq_map_ent_eq, 0.6, malloc, realloc, free); + +/** Helper: Put <b>entry</b> into map of directory requests using + * <b>type</b> and <b>dirreq_id</b> as key parts. If there is + * already an entry for that key, print out a BUG warning and return. */ +static void +_dirreq_map_put(dirreq_map_entry_t *entry, dirreq_type_t type, + uint64_t dirreq_id) +{ + dirreq_map_entry_t *old_ent; + tor_assert(entry->type == type); + tor_assert(entry->dirreq_id == dirreq_id); + + /* XXXX we could switch this to HT_INSERT some time, since it seems that + * this bug doesn't happen. But since this function doesn't seem to be + * critical-path, it's sane to leave it alone. */ + old_ent = HT_REPLACE(dirreqmap, &dirreq_map, entry); + if (old_ent && old_ent != entry) { + log_warn(LD_BUG, "Error when putting directory request into local " + "map. There was already an entry for the same identifier."); + return; + } +} + +/** Helper: Look up and return an entry in the map of directory requests + * using <b>type</b> and <b>dirreq_id</b> as key parts. If there + * is no such entry, return NULL. */ +static dirreq_map_entry_t * +_dirreq_map_get(dirreq_type_t type, uint64_t dirreq_id) +{ + dirreq_map_entry_t lookup; + lookup.type = type; + lookup.dirreq_id = dirreq_id; + return HT_FIND(dirreqmap, &dirreq_map, &lookup); +} + +/** Note that an either direct or tunneled (see <b>type</b>) directory + * request for a network status with unique ID <b>dirreq_id</b> of size + * <b>response_size</b> and action <b>action</b> (either v2 or v3) has + * started. */ +void +geoip_start_dirreq(uint64_t dirreq_id, size_t response_size, + geoip_client_action_t action, dirreq_type_t type) +{ + dirreq_map_entry_t *ent; + if (!get_options()->DirReqStatistics) + return; + ent = tor_malloc_zero(sizeof(dirreq_map_entry_t)); + ent->dirreq_id = dirreq_id; + tor_gettimeofday(&ent->request_time); + ent->response_size = response_size; + ent->action = action; + ent->type = type; + _dirreq_map_put(ent, type, dirreq_id); +} + +/** Change the state of the either direct or tunneled (see <b>type</b>) + * directory request with <b>dirreq_id</b> to <b>new_state</b> and + * possibly mark it as completed. If no entry can be found for the given + * key parts (e.g., if this is a directory request that we are not + * measuring, or one that was started in the previous measurement period), + * or if the state cannot be advanced to <b>new_state</b>, do nothing. */ +void +geoip_change_dirreq_state(uint64_t dirreq_id, dirreq_type_t type, + dirreq_state_t new_state) +{ + dirreq_map_entry_t *ent; + if (!get_options()->DirReqStatistics) + return; + ent = _dirreq_map_get(type, dirreq_id); + if (!ent) + return; + if (new_state == DIRREQ_IS_FOR_NETWORK_STATUS) + return; + if (new_state - 1 != ent->state) + return; + ent->state = new_state; + if ((type == DIRREQ_DIRECT && + new_state == DIRREQ_FLUSHING_DIR_CONN_FINISHED) || + (type == DIRREQ_TUNNELED && + new_state == DIRREQ_OR_CONN_BUFFER_FLUSHED)) { + tor_gettimeofday(&ent->completion_time); + ent->completed = 1; + } +} + +/** Return a newly allocated comma-separated string containing statistics + * on network status downloads. The string contains the number of completed + * requests, timeouts, and still running requests as well as the download + * times by deciles and quartiles. Return NULL if we have not observed + * requests for long enough. */ +static char * +geoip_get_dirreq_history(geoip_client_action_t action, + dirreq_type_t type) { char *result = NULL; - if (!geoip_is_loaded()) + smartlist_t *dirreq_completed = NULL; + uint32_t complete = 0, timeouts = 0, running = 0; + int bufsize = 1024, written; + dirreq_map_entry_t **ptr, **next, *ent; + struct timeval now; + + tor_gettimeofday(&now); + if (action != GEOIP_CLIENT_NETWORKSTATUS && + action != GEOIP_CLIENT_NETWORKSTATUS_V2) return NULL; - if (client_history_starts < (now - GEOIP_MIN_OBSERVATION_TIME)) { - char buf[32]; - smartlist_t *chunks = NULL; - smartlist_t *entries = NULL; - int n_countries = geoip_get_n_countries(); - int i; - clientmap_entry_t **ent; - unsigned *counts = tor_malloc_zero(sizeof(unsigned)*n_countries); - unsigned total = 0; - unsigned granularity = IP_GRANULARITY; -#ifdef ENABLE_GEOIP_STATS - if (get_options()->DirRecordUsageByCountry) - granularity = get_options()->DirRecordUsageGranularity; -#endif - HT_FOREACH(ent, clientmap, &client_history) { - int country; - if (((*ent)->last_seen & ACTION_MASK) != (int)action) - continue; - country = geoip_get_country_by_ip((*ent)->ipaddr); - if (country < 0) - continue; - tor_assert(0 <= country && country < n_countries); - ++counts[country]; - ++total; - } - /* Don't record anything if we haven't seen enough IPs. */ - if (total < MIN_IPS_TO_NOTE_ANYTHING) - goto done; - /* Make a list of c_hist_t */ - entries = smartlist_create(); - for (i = 0; i < n_countries; ++i) { - unsigned c = counts[i]; - const char *countrycode; - c_hist_t *ent; - /* Only report a country if it has a minimum number of IPs. */ - if (c >= MIN_IPS_TO_NOTE_COUNTRY) { - c = round_to_next_multiple_of(c, granularity); - countrycode = geoip_get_country_name(i); - ent = tor_malloc(sizeof(c_hist_t)); - strlcpy(ent->country, countrycode, sizeof(ent->country)); - ent->total = c; - smartlist_add(entries, ent); + dirreq_completed = smartlist_create(); + for (ptr = HT_START(dirreqmap, &dirreq_map); ptr; ptr = next) { + ent = *ptr; + if (ent->action != action || ent->type != type) { + next = HT_NEXT(dirreqmap, &dirreq_map, ptr); + continue; + } else { + if (ent->completed) { + smartlist_add(dirreq_completed, ent); + complete++; + next = HT_NEXT_RMV(dirreqmap, &dirreq_map, ptr); + } else { + if (tv_mdiff(&ent->request_time, &now) / 1000 > DIRREQ_TIMEOUT) + timeouts++; + else + running++; + next = HT_NEXT_RMV(dirreqmap, &dirreq_map, ptr); + tor_free(ent); } } - /* Sort entries. Note that we must do this _AFTER_ rounding, or else - * the sort order could leak info. */ - smartlist_sort(entries, _c_hist_compare); - - /* Build the result. */ - chunks = smartlist_create(); - SMARTLIST_FOREACH(entries, c_hist_t *, ch, { - tor_snprintf(buf, sizeof(buf), "%s=%u", ch->country, ch->total); - smartlist_add(chunks, tor_strdup(buf)); - }); - result = smartlist_join_strings(chunks, ",", 0, NULL); - done: - tor_free(counts); - if (chunks) { - SMARTLIST_FOREACH(chunks, char *, c, tor_free(c)); - smartlist_free(chunks); - } - if (entries) { - SMARTLIST_FOREACH(entries, c_hist_t *, c, tor_free(c)); - smartlist_free(entries); + } +#define DIR_REQ_GRANULARITY 4 + complete = round_uint32_to_next_multiple_of(complete, + DIR_REQ_GRANULARITY); + timeouts = round_uint32_to_next_multiple_of(timeouts, + DIR_REQ_GRANULARITY); + running = round_uint32_to_next_multiple_of(running, + DIR_REQ_GRANULARITY); + result = tor_malloc_zero(bufsize); + written = tor_snprintf(result, bufsize, "complete=%u,timeout=%u," + "running=%u", complete, timeouts, running); + if (written < 0) { + tor_free(result); + goto done; + } + +#define MIN_DIR_REQ_RESPONSES 16 + if (complete >= MIN_DIR_REQ_RESPONSES) { + uint32_t *dltimes; + /* We may have rounded 'completed' up. Here we want to use the + * real value. */ + complete = smartlist_len(dirreq_completed); + dltimes = tor_malloc_zero(sizeof(uint32_t) * complete); + SMARTLIST_FOREACH_BEGIN(dirreq_completed, dirreq_map_entry_t *, ent) { + uint32_t bytes_per_second; + uint32_t time_diff = (uint32_t) tv_mdiff(&ent->request_time, + &ent->completion_time); + if (time_diff == 0) + time_diff = 1; /* Avoid DIV/0; "instant" answers are impossible + * by law of nature or something, but a milisecond + * is a bit greater than "instantly" */ + bytes_per_second = (uint32_t)(1000 * ent->response_size / time_diff); + dltimes[ent_sl_idx] = bytes_per_second; + } SMARTLIST_FOREACH_END(ent); + median_uint32(dltimes, complete); /* sorts as a side effect. */ + written = tor_snprintf(result + written, bufsize - written, + ",min=%u,d1=%u,d2=%u,q1=%u,d3=%u,d4=%u,md=%u," + "d6=%u,d7=%u,q3=%u,d8=%u,d9=%u,max=%u", + dltimes[0], + dltimes[1*complete/10-1], + dltimes[2*complete/10-1], + dltimes[1*complete/4-1], + dltimes[3*complete/10-1], + dltimes[4*complete/10-1], + dltimes[5*complete/10-1], + dltimes[6*complete/10-1], + dltimes[7*complete/10-1], + dltimes[3*complete/4-1], + dltimes[8*complete/10-1], + dltimes[9*complete/10-1], + dltimes[complete-1]); + if (written<0) + tor_free(result); + tor_free(dltimes); + } + done: + SMARTLIST_FOREACH(dirreq_completed, dirreq_map_entry_t *, ent, + tor_free(ent)); + smartlist_free(dirreq_completed); + return result; +} + +/** Return a newly allocated comma-separated string containing entries for + * all the countries from which we've seen enough clients connect as a + * bridge, directory server, or entry guard. The entry format is cc=num + * where num is the number of IPs we've seen connecting from that country, + * and cc is a lowercased country code. Returns NULL if we don't want + * to export geoip data yet. */ +char * +geoip_get_client_history(geoip_client_action_t action) +{ + char *result = NULL; + unsigned granularity = IP_GRANULARITY; + smartlist_t *chunks = NULL; + smartlist_t *entries = NULL; + int n_countries = geoip_get_n_countries(); + int i; + clientmap_entry_t **ent; + unsigned *counts = NULL; + unsigned total = 0; + + if (!geoip_is_loaded()) + return NULL; + + counts = tor_malloc_zero(sizeof(unsigned)*n_countries); + HT_FOREACH(ent, clientmap, &client_history) { + int country; + if ((*ent)->action != (int)action) + continue; + country = geoip_get_country_by_ip((*ent)->ipaddr); + if (country < 0) + country = 0; /** unresolved requests are stored at index 0. */ + tor_assert(0 <= country && country < n_countries); + ++counts[country]; + ++total; + } + /* Don't record anything if we haven't seen enough IPs. */ + if (total < MIN_IPS_TO_NOTE_ANYTHING) + goto done; + /* Make a list of c_hist_t */ + entries = smartlist_create(); + for (i = 0; i < n_countries; ++i) { + unsigned c = counts[i]; + const char *countrycode; + c_hist_t *ent; + /* Only report a country if it has a minimum number of IPs. */ + if (c >= MIN_IPS_TO_NOTE_COUNTRY) { + c = round_to_next_multiple_of(c, granularity); + countrycode = geoip_get_country_name(i); + ent = tor_malloc(sizeof(c_hist_t)); + strlcpy(ent->country, countrycode, sizeof(ent->country)); + ent->total = c; + smartlist_add(entries, ent); } } + /* Sort entries. Note that we must do this _AFTER_ rounding, or else + * the sort order could leak info. */ + smartlist_sort(entries, _c_hist_compare); + + /* Build the result. */ + chunks = smartlist_create(); + SMARTLIST_FOREACH(entries, c_hist_t *, ch, { + char *buf=NULL; + tor_asprintf(&buf, "%s=%u", ch->country, ch->total); + smartlist_add(chunks, buf); + }); + result = smartlist_join_strings(chunks, ",", 0, NULL); + done: + tor_free(counts); + if (chunks) { + SMARTLIST_FOREACH(chunks, char *, c, tor_free(c)); + smartlist_free(chunks); + } + if (entries) { + SMARTLIST_FOREACH(entries, c_hist_t *, c, tor_free(c)); + smartlist_free(entries); + } return result; } @@ -540,18 +858,12 @@ geoip_get_client_history(time_t now, geoip_client_action_t action) * for <b>action</b> in a format suitable for an extra-info document, or NULL * on failure. */ char * -geoip_get_request_history(time_t now, geoip_client_action_t action) +geoip_get_request_history(geoip_client_action_t action) { smartlist_t *entries, *strings; char *result; unsigned granularity = IP_GRANULARITY; -#ifdef ENABLE_GEOIP_STATS - if (get_options()->DirRecordUsageByCountry) - granularity = get_options()->DirRecordUsageGranularity; -#endif - if (client_history_starts >= (now - GEOIP_MIN_OBSERVATION_TIME)) - return NULL; if (action != GEOIP_CLIENT_NETWORKSTATUS && action != GEOIP_CLIENT_NETWORKSTATUS_V2) return NULL; @@ -560,13 +872,10 @@ geoip_get_request_history(time_t now, geoip_client_action_t action) entries = smartlist_create(); SMARTLIST_FOREACH(geoip_countries, geoip_country_t *, c, { - uint32_t *n = (action == GEOIP_CLIENT_NETWORKSTATUS) - ? c->n_v3_ns_requests : c->n_v2_ns_requests; uint32_t tot = 0; - int i; c_hist_t *ent; - for (i=0; i < REQUEST_HIST_LEN; ++i) - tot += n[i]; + tot = (action == GEOIP_CLIENT_NETWORKSTATUS) ? + c->n_v3_ns_requests : c->n_v2_ns_requests; if (!tot) continue; ent = tor_malloc_zero(sizeof(c_hist_t)); @@ -578,9 +887,9 @@ geoip_get_request_history(time_t now, geoip_client_action_t action) strings = smartlist_create(); SMARTLIST_FOREACH(entries, c_hist_t *, ent, { - char buf[32]; - tor_snprintf(buf, sizeof(buf), "%s=%u", ent->country, ent->total); - smartlist_add(strings, tor_strdup(buf)); + char *buf = NULL; + tor_asprintf(&buf, "%s=%u", ent->country, ent->total); + smartlist_add(strings, buf); }); result = smartlist_join_strings(strings, ",", 0, NULL); SMARTLIST_FOREACH(strings, char *, cp, tor_free(cp)); @@ -590,69 +899,450 @@ geoip_get_request_history(time_t now, geoip_client_action_t action) return result; } -/** Store all our geoip statistics into $DATADIR/geoip-stats. */ +/** Start time of directory request stats or 0 if we're not collecting + * directory request statistics. */ +static time_t start_of_dirreq_stats_interval; + +/** Initialize directory request stats. */ void -dump_geoip_stats(void) +geoip_dirreq_stats_init(time_t now) +{ + start_of_dirreq_stats_interval = now; +} + +/** Stop collecting directory request stats in a way that we can re-start + * doing so in geoip_dirreq_stats_init(). */ +void +geoip_dirreq_stats_term(void) +{ + SMARTLIST_FOREACH(geoip_countries, geoip_country_t *, c, { + c->n_v2_ns_requests = c->n_v3_ns_requests = 0; + }); + { + clientmap_entry_t **ent, **next, *this; + for (ent = HT_START(clientmap, &client_history); ent != NULL; + ent = next) { + if ((*ent)->action == GEOIP_CLIENT_NETWORKSTATUS || + (*ent)->action == GEOIP_CLIENT_NETWORKSTATUS_V2) { + this = *ent; + next = HT_NEXT_RMV(clientmap, &client_history, ent); + tor_free(this); + } else { + next = HT_NEXT(clientmap, &client_history, ent); + } + } + } + v2_share_times_seconds = v3_share_times_seconds = 0.0; + last_time_determined_shares = 0; + share_seconds = 0; + memset(ns_v2_responses, 0, sizeof(ns_v2_responses)); + memset(ns_v3_responses, 0, sizeof(ns_v3_responses)); + { + dirreq_map_entry_t **ent, **next, *this; + for (ent = HT_START(dirreqmap, &dirreq_map); ent != NULL; ent = next) { + this = *ent; + next = HT_NEXT_RMV(dirreqmap, &dirreq_map, ent); + tor_free(this); + } + } + start_of_dirreq_stats_interval = 0; +} + +/** Write dirreq statistics to $DATADIR/stats/dirreq-stats and return when + * we would next want to write. */ +time_t +geoip_dirreq_stats_write(time_t now) { -#ifdef ENABLE_GEOIP_STATS - time_t now = time(NULL); - time_t request_start; - char *filename = get_datadir_fname("geoip-stats"); + char *statsdir = NULL, *filename = NULL; char *data_v2 = NULL, *data_v3 = NULL; - char since[ISO_TIME_LEN+1], written[ISO_TIME_LEN+1]; + char written[ISO_TIME_LEN+1]; open_file_t *open_file = NULL; double v2_share = 0.0, v3_share = 0.0; FILE *out; + int i; - data_v2 = geoip_get_client_history(now, GEOIP_CLIENT_NETWORKSTATUS_V2); - data_v3 = geoip_get_client_history(now, GEOIP_CLIENT_NETWORKSTATUS); - format_iso_time(since, geoip_get_history_start()); + if (!start_of_dirreq_stats_interval) + return 0; /* Not initialized. */ + if (start_of_dirreq_stats_interval + WRITE_STATS_INTERVAL > now) + goto done; /* Not ready to write. */ + + /* Discard all items in the client history that are too old. */ + geoip_remove_old_clients(start_of_dirreq_stats_interval); + + statsdir = get_datadir_fname("stats"); + if (check_private_dir(statsdir, CPD_CREATE) < 0) + goto done; + filename = get_datadir_fname2("stats", "dirreq-stats"); + data_v2 = geoip_get_client_history(GEOIP_CLIENT_NETWORKSTATUS_V2); + data_v3 = geoip_get_client_history(GEOIP_CLIENT_NETWORKSTATUS); format_iso_time(written, now); - out = start_writing_to_stdio_file(filename, OPEN_FLAGS_REPLACE, + out = start_writing_to_stdio_file(filename, OPEN_FLAGS_APPEND, 0600, &open_file); if (!out) goto done; - if (fprintf(out, "written %s\nstarted-at %s\nns-ips %s\nns-v2-ips %s\n", - written, since, + if (fprintf(out, "dirreq-stats-end %s (%d s)\ndirreq-v3-ips %s\n" + "dirreq-v2-ips %s\n", written, + (unsigned) (now - start_of_dirreq_stats_interval), data_v3 ? data_v3 : "", data_v2 ? data_v2 : "") < 0) goto done; tor_free(data_v2); tor_free(data_v3); - request_start = current_request_period_starts - - (n_old_request_periods * REQUEST_HIST_PERIOD); - format_iso_time(since, request_start); - data_v2 = geoip_get_request_history(now, GEOIP_CLIENT_NETWORKSTATUS_V2); - data_v3 = geoip_get_request_history(now, GEOIP_CLIENT_NETWORKSTATUS); - if (fprintf(out, "requests-start %s\nn-ns-reqs %s\nn-v2-ns-reqs %s\n", - since, + data_v2 = geoip_get_request_history(GEOIP_CLIENT_NETWORKSTATUS_V2); + data_v3 = geoip_get_request_history(GEOIP_CLIENT_NETWORKSTATUS); + if (fprintf(out, "dirreq-v3-reqs %s\ndirreq-v2-reqs %s\n", data_v3 ? data_v3 : "", data_v2 ? data_v2 : "") < 0) goto done; - if (!router_get_my_share_of_directory_requests(&v2_share, &v3_share)) { - if (fprintf(out, "v2-ns-share %0.2lf%%\n", v2_share*100) < 0) + tor_free(data_v2); + tor_free(data_v3); + SMARTLIST_FOREACH(geoip_countries, geoip_country_t *, c, { + c->n_v2_ns_requests = c->n_v3_ns_requests = 0; + }); +#define RESPONSE_GRANULARITY 8 + for (i = 0; i < GEOIP_NS_RESPONSE_NUM; i++) { + ns_v2_responses[i] = round_uint32_to_next_multiple_of( + ns_v2_responses[i], RESPONSE_GRANULARITY); + ns_v3_responses[i] = round_uint32_to_next_multiple_of( + ns_v3_responses[i], RESPONSE_GRANULARITY); + } +#undef RESPONSE_GRANULARITY + if (fprintf(out, "dirreq-v3-resp ok=%u,not-enough-sigs=%u,unavailable=%u," + "not-found=%u,not-modified=%u,busy=%u\n", + ns_v3_responses[GEOIP_SUCCESS], + ns_v3_responses[GEOIP_REJECT_NOT_ENOUGH_SIGS], + ns_v3_responses[GEOIP_REJECT_UNAVAILABLE], + ns_v3_responses[GEOIP_REJECT_NOT_FOUND], + ns_v3_responses[GEOIP_REJECT_NOT_MODIFIED], + ns_v3_responses[GEOIP_REJECT_BUSY]) < 0) + goto done; + if (fprintf(out, "dirreq-v2-resp ok=%u,unavailable=%u," + "not-found=%u,not-modified=%u,busy=%u\n", + ns_v2_responses[GEOIP_SUCCESS], + ns_v2_responses[GEOIP_REJECT_UNAVAILABLE], + ns_v2_responses[GEOIP_REJECT_NOT_FOUND], + ns_v2_responses[GEOIP_REJECT_NOT_MODIFIED], + ns_v2_responses[GEOIP_REJECT_BUSY]) < 0) + goto done; + memset(ns_v2_responses, 0, sizeof(ns_v2_responses)); + memset(ns_v3_responses, 0, sizeof(ns_v3_responses)); + if (!geoip_get_mean_shares(now, &v2_share, &v3_share)) { + if (fprintf(out, "dirreq-v2-share %0.2lf%%\n", v2_share*100) < 0) goto done; - if (fprintf(out, "v3-ns-share %0.2lf%%\n", v3_share*100) < 0) + if (fprintf(out, "dirreq-v3-share %0.2lf%%\n", v3_share*100) < 0) goto done; } + data_v2 = geoip_get_dirreq_history(GEOIP_CLIENT_NETWORKSTATUS_V2, + DIRREQ_DIRECT); + data_v3 = geoip_get_dirreq_history(GEOIP_CLIENT_NETWORKSTATUS, + DIRREQ_DIRECT); + if (fprintf(out, "dirreq-v3-direct-dl %s\ndirreq-v2-direct-dl %s\n", + data_v3 ? data_v3 : "", data_v2 ? data_v2 : "") < 0) + goto done; + tor_free(data_v2); + tor_free(data_v3); + data_v2 = geoip_get_dirreq_history(GEOIP_CLIENT_NETWORKSTATUS_V2, + DIRREQ_TUNNELED); + data_v3 = geoip_get_dirreq_history(GEOIP_CLIENT_NETWORKSTATUS, + DIRREQ_TUNNELED); + if (fprintf(out, "dirreq-v3-tunneled-dl %s\ndirreq-v2-tunneled-dl %s\n", + data_v3 ? data_v3 : "", data_v2 ? data_v2 : "") < 0) + goto done; + finish_writing_to_file(open_file); open_file = NULL; + + start_of_dirreq_stats_interval = now; + done: if (open_file) abort_writing_to_file(open_file); tor_free(filename); + tor_free(statsdir); tor_free(data_v2); tor_free(data_v3); -#endif + return start_of_dirreq_stats_interval + WRITE_STATS_INTERVAL; +} + +/** Start time of bridge stats or 0 if we're not collecting bridge + * statistics. */ +static time_t start_of_bridge_stats_interval; + +/** Initialize bridge stats. */ +void +geoip_bridge_stats_init(time_t now) +{ + start_of_bridge_stats_interval = now; +} + +/** Stop collecting bridge stats in a way that we can re-start doing so in + * geoip_bridge_stats_init(). */ +void +geoip_bridge_stats_term(void) +{ + client_history_clear(); + start_of_bridge_stats_interval = 0; +} + +/** Validate a bridge statistics string as it would be written to a + * current extra-info descriptor. Return 1 if the string is valid and + * recent enough, or 0 otherwise. */ +static int +validate_bridge_stats(const char *stats_str, time_t now) +{ + char stats_end_str[ISO_TIME_LEN+1], stats_start_str[ISO_TIME_LEN+1], + *eos; + + const char *BRIDGE_STATS_END = "bridge-stats-end "; + const char *BRIDGE_IPS = "bridge-ips "; + const char *BRIDGE_IPS_EMPTY_LINE = "bridge-ips\n"; + const char *tmp; + time_t stats_end_time; + int seconds; + tor_assert(stats_str); + + /* Parse timestamp and number of seconds from + "bridge-stats-end YYYY-MM-DD HH:MM:SS (N s)" */ + tmp = find_str_at_start_of_line(stats_str, BRIDGE_STATS_END); + if (!tmp) + return 0; + tmp += strlen(BRIDGE_STATS_END); + + if (strlen(tmp) < ISO_TIME_LEN + 6) + return 0; + strlcpy(stats_end_str, tmp, sizeof(stats_end_str)); + if (parse_iso_time(stats_end_str, &stats_end_time) < 0) + return 0; + if (stats_end_time < now - (25*60*60) || + stats_end_time > now + (1*60*60)) + return 0; + seconds = (int)strtol(tmp + ISO_TIME_LEN + 2, &eos, 10); + if (!eos || seconds < 23*60*60) + return 0; + format_iso_time(stats_start_str, stats_end_time - seconds); + + /* Parse: "bridge-ips CC=N,CC=N,..." */ + tmp = find_str_at_start_of_line(stats_str, BRIDGE_IPS); + if (!tmp) { + /* Look if there is an empty "bridge-ips" line */ + tmp = find_str_at_start_of_line(stats_str, BRIDGE_IPS_EMPTY_LINE); + if (!tmp) + return 0; + } + + return 1; +} + +/** Most recent bridge statistics formatted to be written to extra-info + * descriptors. */ +static char *bridge_stats_extrainfo = NULL; + +/** Return a newly allocated string holding our bridge usage stats by country + * in a format suitable for inclusion in an extrainfo document. Return NULL on + * failure. */ +static char * +format_bridge_stats_extrainfo(time_t now) +{ + char *out = NULL, *data = NULL; + long duration = now - start_of_bridge_stats_interval; + char written[ISO_TIME_LEN+1]; + + if (duration < 0) + return NULL; + + format_iso_time(written, now); + data = geoip_get_client_history(GEOIP_CLIENT_CONNECT); + + tor_asprintf(&out, + "bridge-stats-end %s (%ld s)\n" + "bridge-ips %s\n", + written, duration, + data ? data : ""); + tor_free(data); + + return out; +} + +/** Return a newly allocated string holding our bridge usage stats by country + * in a format suitable for the answer to a controller request. Return NULL on + * failure. */ +static char * +format_bridge_stats_controller(time_t now) +{ + char *out = NULL, *data = NULL; + char started[ISO_TIME_LEN+1]; + (void) now; + + format_iso_time(started, start_of_bridge_stats_interval); + data = geoip_get_client_history(GEOIP_CLIENT_CONNECT); + + tor_asprintf(&out, + "TimeStarted=\"%s\" CountrySummary=%s", + started, data ? data : ""); + tor_free(data); + return out; +} + +/** Write bridge statistics to $DATADIR/stats/bridge-stats and return + * when we should next try to write statistics. */ +time_t +geoip_bridge_stats_write(time_t now) +{ + char *filename = NULL, *val = NULL, *statsdir = NULL; + + /* Check if 24 hours have passed since starting measurements. */ + if (now < start_of_bridge_stats_interval + WRITE_STATS_INTERVAL) + return start_of_bridge_stats_interval + WRITE_STATS_INTERVAL; + + /* Discard all items in the client history that are too old. */ + geoip_remove_old_clients(start_of_bridge_stats_interval); + + /* Generate formatted string */ + val = format_bridge_stats_extrainfo(now); + if (val == NULL) + goto done; + + /* Update the stored value. */ + tor_free(bridge_stats_extrainfo); + bridge_stats_extrainfo = val; + start_of_bridge_stats_interval = now; + + /* Write it to disk. */ + statsdir = get_datadir_fname("stats"); + if (check_private_dir(statsdir, CPD_CREATE) < 0) + goto done; + filename = get_datadir_fname2("stats", "bridge-stats"); + + write_str_to_file(filename, bridge_stats_extrainfo, 0); + + /* Tell the controller, "hey, there are clients!" */ + { + char *controller_str = format_bridge_stats_controller(now); + if (controller_str) + control_event_clients_seen(controller_str); + tor_free(controller_str); + } + done: + tor_free(filename); + tor_free(statsdir); + + return start_of_bridge_stats_interval + WRITE_STATS_INTERVAL; +} + +/** Try to load the most recent bridge statistics from disk, unless we + * have finished a measurement interval lately, and check whether they + * are still recent enough. */ +static void +load_bridge_stats(time_t now) +{ + char *fname, *contents; + if (bridge_stats_extrainfo) + return; + + fname = get_datadir_fname2("stats", "bridge-stats"); + contents = read_file_to_str(fname, RFTS_IGNORE_MISSING, NULL); + if (contents && validate_bridge_stats(contents, now)) + bridge_stats_extrainfo = contents; + + tor_free(fname); +} + +/** Return most recent bridge statistics for inclusion in extra-info + * descriptors, or NULL if we don't have recent bridge statistics. */ +const char * +geoip_get_bridge_stats_extrainfo(time_t now) +{ + load_bridge_stats(now); + return bridge_stats_extrainfo; +} + +/** Return a new string containing the recent bridge statistics to be returned + * to controller clients, or NULL if we don't have any bridge statistics. */ +char * +geoip_get_bridge_stats_controller(time_t now) +{ + return format_bridge_stats_controller(now); +} + +/** Start time of entry stats or 0 if we're not collecting entry + * statistics. */ +static time_t start_of_entry_stats_interval; + +/** Initialize entry stats. */ +void +geoip_entry_stats_init(time_t now) +{ + start_of_entry_stats_interval = now; +} + +/** Stop collecting entry stats in a way that we can re-start doing so in + * geoip_entry_stats_init(). */ +void +geoip_entry_stats_term(void) +{ + client_history_clear(); + start_of_entry_stats_interval = 0; +} + +/** Write entry statistics to $DATADIR/stats/entry-stats and return time + * when we would next want to write. */ +time_t +geoip_entry_stats_write(time_t now) +{ + char *statsdir = NULL, *filename = NULL; + char *data = NULL; + char written[ISO_TIME_LEN+1]; + open_file_t *open_file = NULL; + FILE *out; + + if (!start_of_entry_stats_interval) + return 0; /* Not initialized. */ + if (start_of_entry_stats_interval + WRITE_STATS_INTERVAL > now) + goto done; /* Not ready to write. */ + + /* Discard all items in the client history that are too old. */ + geoip_remove_old_clients(start_of_entry_stats_interval); + + statsdir = get_datadir_fname("stats"); + if (check_private_dir(statsdir, CPD_CREATE) < 0) + goto done; + filename = get_datadir_fname2("stats", "entry-stats"); + data = geoip_get_client_history(GEOIP_CLIENT_CONNECT); + format_iso_time(written, now); + out = start_writing_to_stdio_file(filename, OPEN_FLAGS_APPEND, + 0600, &open_file); + if (!out) + goto done; + if (fprintf(out, "entry-stats-end %s (%u s)\nentry-ips %s\n", + written, (unsigned) (now - start_of_entry_stats_interval), + data ? data : "") < 0) + goto done; + + start_of_entry_stats_interval = now; + + finish_writing_to_file(open_file); + open_file = NULL; + done: + if (open_file) + abort_writing_to_file(open_file); + tor_free(filename); + tor_free(statsdir); + tor_free(data); + return start_of_entry_stats_interval + WRITE_STATS_INTERVAL; } /** Helper used to implement GETINFO ip-to-country/... controller command. */ int getinfo_helper_geoip(control_connection_t *control_conn, - const char *question, char **answer) + const char *question, char **answer, + const char **errmsg) { (void)control_conn; - if (geoip_is_loaded() && !strcmpstart(question, "ip-to-country/")) { + if (!geoip_is_loaded()) { + *errmsg = "GeoIP data not loaded"; + return -1; + } + if (!strcmpstart(question, "ip-to-country/")) { int c; uint32_t ip; struct in_addr in; @@ -674,8 +1364,8 @@ clear_geoip_db(void) SMARTLIST_FOREACH(geoip_countries, geoip_country_t *, c, tor_free(c)); smartlist_free(geoip_countries); } - if (country_idxplus1_by_lc_code) - strmap_free(country_idxplus1_by_lc_code, NULL); + + strmap_free(country_idxplus1_by_lc_code, NULL); if (geoip_entries) { SMARTLIST_FOREACH(geoip_entries, geoip_entry_t *, ent, tor_free(ent)); smartlist_free(geoip_entries); @@ -689,13 +1379,24 @@ clear_geoip_db(void) void geoip_free_all(void) { - clientmap_entry_t **ent, **next, *this; - for (ent = HT_START(clientmap, &client_history); ent != NULL; ent = next) { - this = *ent; - next = HT_NEXT_RMV(clientmap, &client_history, ent); - tor_free(this); + { + clientmap_entry_t **ent, **next, *this; + for (ent = HT_START(clientmap, &client_history); ent != NULL; ent = next) { + this = *ent; + next = HT_NEXT_RMV(clientmap, &client_history, ent); + tor_free(this); + } + HT_CLEAR(clientmap, &client_history); + } + { + dirreq_map_entry_t **ent, **next, *this; + for (ent = HT_START(dirreqmap, &dirreq_map); ent != NULL; ent = next) { + this = *ent; + next = HT_NEXT_RMV(dirreqmap, &dirreq_map, ent); + tor_free(this); + } + HT_CLEAR(dirreqmap, &dirreq_map); } - HT_CLEAR(clientmap, &client_history); clear_geoip_db(); } diff --git a/src/or/geoip.h b/src/or/geoip.h new file mode 100644 index 0000000000..24f7c5b931 --- /dev/null +++ b/src/or/geoip.h @@ -0,0 +1,57 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file geoip.h + * \brief Header file for geoip.c. + **/ + +#ifndef _TOR_GEOIP_H +#define _TOR_GEOIP_H + +#ifdef GEOIP_PRIVATE +int geoip_parse_entry(const char *line); +#endif +int should_record_bridge_info(or_options_t *options); +int geoip_load_file(const char *filename, or_options_t *options); +int geoip_get_country_by_ip(uint32_t ipaddr); +int geoip_get_n_countries(void); +const char *geoip_get_country_name(country_t num); +int geoip_is_loaded(void); +country_t geoip_get_country(const char *countrycode); + +void geoip_note_client_seen(geoip_client_action_t action, + uint32_t addr, time_t now); +void geoip_remove_old_clients(time_t cutoff); + +void geoip_note_ns_response(geoip_client_action_t action, + geoip_ns_response_t response); +char *geoip_get_client_history(geoip_client_action_t action); +char *geoip_get_request_history(geoip_client_action_t action); +int getinfo_helper_geoip(control_connection_t *control_conn, + const char *question, char **answer, + const char **errmsg); +void geoip_free_all(void); + +void geoip_start_dirreq(uint64_t dirreq_id, size_t response_size, + geoip_client_action_t action, dirreq_type_t type); +void geoip_change_dirreq_state(uint64_t dirreq_id, dirreq_type_t type, + dirreq_state_t new_state); + +void geoip_dirreq_stats_init(time_t now); +time_t geoip_dirreq_stats_write(time_t now); +void geoip_dirreq_stats_term(void); +void geoip_entry_stats_init(time_t now); +time_t geoip_entry_stats_write(time_t now); +void geoip_entry_stats_term(void); +void geoip_bridge_stats_init(time_t now); +time_t geoip_bridge_stats_write(time_t now); +void geoip_bridge_stats_term(void); +const char *geoip_get_bridge_stats_extrainfo(time_t); +char *geoip_get_bridge_stats_controller(time_t); + +#endif + diff --git a/src/or/hibernate.c b/src/or/hibernate.c index 9ec63756fa..aebce4cc88 100644 --- a/src/or/hibernate.c +++ b/src/or/hibernate.c @@ -22,6 +22,12 @@ hibernating, phase 2: */ #include "or.h" +#include "config.h" +#include "connection.h" +#include "connection_edge.h" +#include "hibernate.h" +#include "main.h" +#include "router.h" /** Possible values of hibernate_state */ typedef enum { @@ -89,6 +95,13 @@ static uint64_t n_bytes_read_in_interval = 0; static uint64_t n_bytes_written_in_interval = 0; /** How many seconds have we been running this interval? */ static uint32_t n_seconds_active_in_interval = 0; +/** How many seconds were we active in this interval before we hit our soft + * limit? */ +static int n_seconds_to_hit_soft_limit = 0; +/** When in this interval was the soft limit hit. */ +static time_t soft_limit_hit_at = 0; +/** How many bytes had we read/written when we hit the soft limit? */ +static uint64_t n_bytes_at_soft_limit = 0; /** When did this accounting interval start? */ static time_t interval_start_time = 0; /** When will this accounting interval end? */ @@ -182,6 +195,9 @@ accounting_parse_options(or_options_t *options, int validate_only) case UNIT_DAY: d = 0; break; + /* Coverity dislikes unreachable default cases; some compilers warn on + * switch statements missing a case. Tell Coverity not to worry. */ + /* coverity[dead_error_begin] */ default: tor_assert(0); } @@ -332,29 +348,58 @@ start_of_accounting_period_after(time_t now) return edge_of_accounting_period_containing(now, 1); } +/** Return the length of the accounting period containing the time + * <b>now</b>. */ +static long +length_of_accounting_period_containing(time_t now) +{ + return edge_of_accounting_period_containing(now, 1) - + edge_of_accounting_period_containing(now, 0); +} + /** Initialize the accounting subsystem. */ void configure_accounting(time_t now) { + time_t s_now; /* Try to remember our recorded usage. */ if (!interval_start_time) read_bandwidth_usage(); /* If we fail, we'll leave values at zero, and * reset below.*/ - if (!interval_start_time || - start_of_accounting_period_after(interval_start_time) <= now) { - /* We didn't have recorded usage, or we don't have recorded usage - * for this interval. Start a new interval. */ + + s_now = start_of_accounting_period_containing(now); + + if (!interval_start_time) { + /* We didn't have recorded usage; Start a new interval. */ log_info(LD_ACCT, "Starting new accounting interval."); reset_accounting(now); - } else if (interval_start_time == - start_of_accounting_period_containing(interval_start_time)) { + } else if (s_now == interval_start_time) { log_info(LD_ACCT, "Continuing accounting interval."); /* We are in the interval we thought we were in. Do nothing.*/ interval_end_time = start_of_accounting_period_after(interval_start_time); } else { - log_warn(LD_ACCT, - "Mismatched accounting interval; starting a fresh one."); - reset_accounting(now); + long duration = + length_of_accounting_period_containing(interval_start_time); + double delta = ((double)(s_now - interval_start_time)) / duration; + if (-0.50 <= delta && delta <= 0.50) { + /* The start of the period is now a little later or earlier than we + * remembered. That's fine; we might lose some bytes we could otherwise + * have written, but better to err on the side of obeying people's + * accounting settings. */ + log_info(LD_ACCT, "Accounting interval moved by %.02f%%; " + "that's fine.", delta*100); + interval_end_time = start_of_accounting_period_after(now); + } else if (delta >= 0.99) { + /* This is the regular time-moved-forward case; don't be too noisy + * about it or people will complain */ + log_info(LD_ACCT, "Accounting interval elapsed; starting a new one"); + reset_accounting(now); + } else { + log_warn(LD_ACCT, + "Mismatched accounting interval: moved by %.02f%%. " + "Starting a fresh one.", delta*100); + reset_accounting(now); + } } accounting_set_wakeup_time(); } @@ -365,23 +410,42 @@ configure_accounting(time_t now) static void update_expected_bandwidth(void) { - uint64_t used, expected; - uint64_t max_configured = (get_options()->BandwidthRate * 60); - - if (n_seconds_active_in_interval < 1800) { + uint64_t expected; + or_options_t *options= get_options(); + uint64_t max_configured = (options->RelayBandwidthRate > 0 ? + options->RelayBandwidthRate : + options->BandwidthRate) * 60; + +#define MIN_TIME_FOR_MEASUREMENT (1800) + + if (soft_limit_hit_at > interval_start_time && n_bytes_at_soft_limit && + (soft_limit_hit_at - interval_start_time) > MIN_TIME_FOR_MEASUREMENT) { + /* If we hit our soft limit last time, only count the bytes up to that + * time. This is a better predictor of our actual bandwidth than + * considering the entirety of the last interval, since we likely started + * using bytes very slowly once we hit our soft limit. */ + expected = n_bytes_at_soft_limit / + (soft_limit_hit_at - interval_start_time); + expected /= 60; + } else if (n_seconds_active_in_interval >= MIN_TIME_FOR_MEASUREMENT) { + /* Otherwise, we either measured enough time in the last interval but + * never hit our soft limit, or we're using a state file from a Tor that + * doesn't know to store soft-limit info. Just take rate at which + * we were reading/writing in the last interval as our expected rate. + */ + uint64_t used = MAX(n_bytes_written_in_interval, + n_bytes_read_in_interval); + expected = used / (n_seconds_active_in_interval / 60); + } else { /* If we haven't gotten enough data last interval, set 'expected' * to 0. This will set our wakeup to the start of the interval. * Next interval, we'll choose our starting time based on how much * we sent this interval. */ expected = 0; - } else { - used = n_bytes_written_in_interval < n_bytes_read_in_interval ? - n_bytes_read_in_interval : n_bytes_written_in_interval; - expected = used / (n_seconds_active_in_interval / 60); - if (expected > max_configured) - expected = max_configured; } + if (expected > max_configured) + expected = max_configured; expected_bandwidth_usage = expected; } @@ -399,6 +463,9 @@ reset_accounting(time_t now) n_bytes_read_in_interval = 0; n_bytes_written_in_interval = 0; n_seconds_active_in_interval = 0; + n_bytes_at_soft_limit = 0; + soft_limit_hit_at = 0; + n_seconds_to_hit_soft_limit = 0; } /** Return true iff we should save our bandwidth usage to disk. */ @@ -449,35 +516,39 @@ accounting_run_housekeeping(time_t now) static void accounting_set_wakeup_time(void) { - char buf[ISO_TIME_LEN+1]; char digest[DIGEST_LEN]; crypto_digest_env_t *d_env; int time_in_interval; uint64_t time_to_exhaust_bw; int time_to_consider; - if (! identity_key_is_set()) { + if (! server_identity_key_is_set()) { if (init_keys() < 0) { log_err(LD_BUG, "Error initializing keys"); tor_assert(0); } } - format_iso_time(buf, interval_start_time); - crypto_pk_get_digest(get_identity_key(), digest); + if (server_identity_key_is_set()) { + char buf[ISO_TIME_LEN+1]; + format_iso_time(buf, interval_start_time); - d_env = crypto_new_digest_env(); - crypto_digest_add_bytes(d_env, buf, ISO_TIME_LEN); - crypto_digest_add_bytes(d_env, digest, DIGEST_LEN); - crypto_digest_get_digest(d_env, digest, DIGEST_LEN); - crypto_free_digest_env(d_env); + crypto_pk_get_digest(get_server_identity_key(), digest); + + d_env = crypto_new_digest_env(); + crypto_digest_add_bytes(d_env, buf, ISO_TIME_LEN); + crypto_digest_add_bytes(d_env, digest, DIGEST_LEN); + crypto_digest_get_digest(d_env, digest, DIGEST_LEN); + crypto_free_digest_env(d_env); + } else { + crypto_rand(digest, DIGEST_LEN); + } if (!expected_bandwidth_usage) { char buf1[ISO_TIME_LEN+1]; char buf2[ISO_TIME_LEN+1]; format_local_iso_time(buf1, interval_start_time); format_local_iso_time(buf2, interval_end_time); - time_to_exhaust_bw = GUESS_TIME_TO_USE_BANDWIDTH; interval_wakeup_time = interval_start_time; log_notice(LD_ACCT, @@ -492,8 +563,8 @@ accounting_set_wakeup_time(void) time_to_exhaust_bw = (get_options()->AccountingMax/expected_bandwidth_usage)*60; - if (time_to_exhaust_bw > TIME_MAX) { - time_to_exhaust_bw = TIME_MAX; + if (time_to_exhaust_bw > INT_MAX) { + time_to_exhaust_bw = INT_MAX; time_to_consider = 0; } else { time_to_consider = time_in_interval - (int)time_to_exhaust_bw; @@ -511,8 +582,6 @@ accounting_set_wakeup_time(void) * to be chosen than the last half. */ interval_wakeup_time = interval_start_time + (get_uint32(digest) % time_to_consider); - - format_iso_time(buf, interval_wakeup_time); } { @@ -559,6 +628,10 @@ accounting_record_bandwidth_usage(time_t now, or_state_t *state) state->AccountingSecondsActive = n_seconds_active_in_interval; state->AccountingExpectedUsage = expected_bandwidth_usage; + state->AccountingSecondsToReachSoftLimit = n_seconds_to_hit_soft_limit; + state->AccountingSoftLimitHitAt = soft_limit_hit_at; + state->AccountingBytesAtSoftLimit = n_bytes_at_soft_limit; + or_state_mark_dirty(state, now+(get_options()->AvoidDiskWrites ? 7200 : 60)); @@ -582,10 +655,6 @@ read_bandwidth_usage(void) if (!state) return -1; - /* Okay; it looks like the state file is more up-to-date than the - * bw_accounting file, or the bw_accounting file is nonexistent, - * or the bw_accounting file is corrupt. - */ log_info(LD_ACCT, "Reading bandwidth accounting data from state file"); n_bytes_read_in_interval = state->AccountingBytesReadInInterval; n_bytes_written_in_interval = state->AccountingBytesWrittenInInterval; @@ -593,6 +662,21 @@ read_bandwidth_usage(void) interval_start_time = state->AccountingIntervalStart; expected_bandwidth_usage = state->AccountingExpectedUsage; + /* Older versions of Tor (before 0.2.2.17-alpha or so) didn't generate these + * fields. If you switch back and forth, you might get an + * AccountingSoftLimitHitAt value from long before the most recent + * interval_start_time. If that's so, then ignore the softlimit-related + * values. */ + if (state->AccountingSoftLimitHitAt > interval_start_time) { + soft_limit_hit_at = state->AccountingSoftLimitHitAt; + n_bytes_at_soft_limit = state->AccountingBytesAtSoftLimit; + n_seconds_to_hit_soft_limit = state->AccountingSecondsToReachSoftLimit; + } else { + soft_limit_hit_at = 0; + n_bytes_at_soft_limit = 0; + n_seconds_to_hit_soft_limit = 0; + } + { char tbuf1[ISO_TIME_LEN+1]; char tbuf2[ISO_TIME_LEN+1]; @@ -632,8 +716,27 @@ hibernate_hard_limit_reached(void) static int hibernate_soft_limit_reached(void) { - uint64_t soft_limit = DBL_TO_U64(U64_TO_DBL(get_options()->AccountingMax) - * .95); + const uint64_t acct_max = get_options()->AccountingMax; +#define SOFT_LIM_PCT (.95) +#define SOFT_LIM_BYTES (500*1024*1024) +#define SOFT_LIM_MINUTES (3*60) + /* The 'soft limit' is a fair bit more complicated now than once it was. + * We want to stop accepting connections when ALL of the following are true: + * - We expect to use up the remaining bytes in under 3 hours + * - We have used up 95% of our bytes. + * - We have less than 500MB of bytes left. + */ + uint64_t soft_limit = DBL_TO_U64(U64_TO_DBL(acct_max) * SOFT_LIM_PCT); + if (acct_max > SOFT_LIM_BYTES && acct_max - SOFT_LIM_BYTES > soft_limit) { + soft_limit = acct_max - SOFT_LIM_BYTES; + } + if (expected_bandwidth_usage) { + const uint64_t expected_usage = + expected_bandwidth_usage * SOFT_LIM_MINUTES; + if (acct_max > expected_usage && acct_max - expected_usage > soft_limit) + soft_limit = acct_max - expected_usage; + } + if (!soft_limit) return 0; return n_bytes_read_in_interval >= soft_limit @@ -658,6 +761,14 @@ hibernate_begin(hibernate_state_t new_state, time_t now) exit(0); } + if (new_state == HIBERNATE_STATE_LOWBANDWIDTH && + hibernate_state == HIBERNATE_STATE_LIVE) { + soft_limit_hit_at = now; + n_seconds_to_hit_soft_limit = n_seconds_active_in_interval; + n_bytes_at_soft_limit = MAX(n_bytes_read_in_interval, + n_bytes_written_in_interval); + } + /* close listeners. leave control listener(s). */ while ((conn = connection_get_by_type(CONN_TYPE_OR_LISTENER)) || (conn = connection_get_by_type(CONN_TYPE_AP_LISTENER)) || @@ -673,7 +784,8 @@ hibernate_begin(hibernate_state_t new_state, time_t now) /* XXX upload rendezvous service descriptors with no intro points */ if (new_state == HIBERNATE_STATE_EXITING) { - log_notice(LD_GENERAL,"Interrupt: will shut down in %d seconds. Interrupt " + log_notice(LD_GENERAL,"Interrupt: we have stopped accepting new " + "connections, and will shut down in %d seconds. Interrupt " "again to exit now.", options->ShutdownWaitLength); shutdown_time = time(NULL) + options->ShutdownWaitLength; } else { /* soft limit reached */ @@ -830,7 +942,8 @@ consider_hibernation(time_t now) if (hibernate_state == HIBERNATE_STATE_LIVE) { if (hibernate_soft_limit_reached()) { log_notice(LD_ACCT, - "Bandwidth soft limit reached; commencing hibernation."); + "Bandwidth soft limit reached; commencing hibernation. " + "No new conncetions will be accepted"); hibernate_begin(HIBERNATE_STATE_LOWBANDWIDTH, now); } else if (accounting_enabled && now < interval_wakeup_time) { format_local_iso_time(buf,interval_wakeup_time); @@ -860,9 +973,11 @@ consider_hibernation(time_t now) * NULL. */ int getinfo_helper_accounting(control_connection_t *conn, - const char *question, char **answer) + const char *question, char **answer, + const char **errmsg) { (void) conn; + (void) errmsg; if (!strcmp(question, "accounting/enabled")) { *answer = tor_strdup(accounting_is_enabled(get_options()) ? "1" : "0"); } else if (!strcmp(question, "accounting/hibernating")) { diff --git a/src/or/hibernate.h b/src/or/hibernate.h new file mode 100644 index 0000000000..2aea0fab0c --- /dev/null +++ b/src/or/hibernate.h @@ -0,0 +1,29 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file hibernate.h + * \brief Header file for hibernate.c. + **/ + +#ifndef _TOR_HIBERNATE_H +#define _TOR_HIBERNATE_H + +int accounting_parse_options(or_options_t *options, int validate_only); +int accounting_is_enabled(or_options_t *options); +void configure_accounting(time_t now); +void accounting_run_housekeeping(time_t now); +void accounting_add_bytes(size_t n_read, size_t n_written, int seconds); +int accounting_record_bandwidth_usage(time_t now, or_state_t *state); +void hibernate_begin_shutdown(void); +int we_are_hibernating(void); +void consider_hibernation(time_t now); +int getinfo_helper_accounting(control_connection_t *conn, + const char *question, char **answer, + const char **errmsg); + +#endif + diff --git a/src/or/main.c b/src/or/main.c index e44fd49462..7bae59ce06 100644 --- a/src/or/main.c +++ b/src/or/main.c @@ -12,12 +12,50 @@ #define MAIN_PRIVATE #include "or.h" +#include "buffers.h" +#include "circuitbuild.h" +#include "circuitlist.h" +#include "circuituse.h" +#include "command.h" +#include "config.h" +#include "connection.h" +#include "connection_edge.h" +#include "connection_or.h" +#include "control.h" +#include "cpuworker.h" +#include "directory.h" +#include "dirserv.h" +#include "dirvote.h" +#include "dns.h" +#include "dnsserv.h" +#include "geoip.h" +#include "hibernate.h" +#include "main.h" +#include "microdesc.h" +#include "networkstatus.h" +#include "ntmain.h" +#include "onion.h" +#include "policies.h" +#include "relay.h" +#include "rendclient.h" +#include "rendcommon.h" +#include "rendservice.h" +#include "rephist.h" +#include "router.h" +#include "routerlist.h" +#include "routerparse.h" #ifdef USE_DMALLOC #include <dmalloc.h> #include <openssl/crypto.h> #endif #include "memarea.h" +#ifdef HAVE_EVENT2_EVENT_H +#include <event2/event.h> +#else +#include <event.h> +#endif + void evdns_shutdown(int); /********* PROTOTYPES **********/ @@ -26,8 +64,7 @@ static void dumpmemusage(int severity); static void dumpstats(int severity); /* log stats */ static void conn_read_callback(int fd, short event, void *_conn); static void conn_write_callback(int fd, short event, void *_conn); -static void signal_callback(int fd, short events, void *arg); -static void second_elapsed_callback(int fd, short event, void *args); +static void second_elapsed_callback(periodic_timer_t *timer, void *args); static int conn_close_if_marked(int i); static void connection_start_reading_from_linked_conn(connection_t *conn); static int connection_should_read_from_linked_conn(connection_t *conn); @@ -81,8 +118,12 @@ static smartlist_t *active_linked_connection_lst = NULL; static int called_loop_once = 0; /** We set this to 1 when we've opened a circuit, so we can print a log - * entry to inform the user that Tor is working. */ -int has_completed_circuit=0; + * entry to inform the user that Tor is working. We set it to 0 when + * we think the fact that we once opened a circuit doesn't mean we can do so + * any longer (a big time jump happened, when we notice our directory is + * heinously out-of-date, etc. + */ +int can_complete_circuit=0; /** How often do we check for router descriptors that we should download * when we have too little directory info? */ @@ -127,12 +168,10 @@ connection_add(connection_t *conn) smartlist_add(connection_array, conn); if (conn->s >= 0 || conn->linked) { - conn->read_event = tor_malloc_zero(sizeof(struct event)); - conn->write_event = tor_malloc_zero(sizeof(struct event)); - event_set(conn->read_event, conn->s, EV_READ|EV_PERSIST, - conn_read_callback, conn); - event_set(conn->write_event, conn->s, EV_WRITE|EV_PERSIST, - conn_write_callback, conn); + conn->read_event = tor_event_new(tor_libevent_get_base(), + conn->s, EV_READ|EV_PERSIST, conn_read_callback, conn); + conn->write_event = tor_event_new(tor_libevent_get_base(), + conn->s, EV_WRITE|EV_PERSIST, conn_write_callback, conn); } log_debug(LD_NET,"new conn type %s, socket %d, address %s, n_conns %d.", @@ -142,6 +181,25 @@ connection_add(connection_t *conn) return 0; } +/** Tell libevent that we don't care about <b>conn</b> any more. */ +void +connection_unregister_events(connection_t *conn) +{ + if (conn->read_event) { + if (event_del(conn->read_event)) + log_warn(LD_BUG, "Error removing read event for %d", conn->s); + tor_free(conn->read_event); + } + if (conn->write_event) { + if (event_del(conn->write_event)) + log_warn(LD_BUG, "Error removing write event for %d", conn->s); + tor_free(conn->write_event); + } + if (conn->dns_server_port) { + dnsserv_close_listener(conn); + } +} + /** Remove the connection from the global list, and remove the * corresponding poll entry. Calling this function will shift the last * connection (if any) into the position occupied by conn. @@ -246,17 +304,17 @@ get_connection_array(void) } /** Set the event mask on <b>conn</b> to <b>events</b>. (The event - * mask is a bitmask whose bits are EV_READ and EV_WRITE.) + * mask is a bitmask whose bits are READ_EVENT and WRITE_EVENT) */ void -connection_watch_events(connection_t *conn, short events) +connection_watch_events(connection_t *conn, watchable_events_t events) { - if (events & EV_READ) + if (events & READ_EVENT) connection_start_reading(conn); else connection_stop_reading(conn); - if (events & EV_WRITE) + if (events & WRITE_EVENT) connection_start_writing(conn); else connection_stop_writing(conn); @@ -393,11 +451,11 @@ connection_start_reading_from_linked_conn(connection_t *conn) smartlist_add(active_linked_connection_lst, conn); if (!called_loop_once) { /* This is the first event on the list; we won't be in LOOP_ONCE mode, - * so we need to make sure that the event_loop() actually exits at the - * end of its run through the current connections and - * lets us activate read events for linked connections. */ + * so we need to make sure that the event_base_loop() actually exits at + * the end of its run through the current connections and lets us + * activate read events for linked connections. */ struct timeval tv = { 0, 0 }; - event_loopexit(&tv); + tor_event_base_loopexit(tor_libevent_get_base(), &tv); } } else { tor_assert(smartlist_isin(active_linked_connection_lst, conn)); @@ -540,7 +598,7 @@ conn_close_if_marked(int i) log_info(LD_NET, "Conn (addr %s, fd %d, type %s, state %d) marked, but wants " "to flush %d bytes. (Marked at %s:%d)", - escaped_safe_str(conn->address), + escaped_safe_str_client(conn->address), conn->s, conn_type_to_string(conn->type), conn->state, (int)conn->outbuf_flushlen, conn->marked_for_close_file, conn->marked_for_close); @@ -593,8 +651,8 @@ conn_close_if_marked(int i) "something is wrong with theirs. " "(fd %d, type %s, state %d, marked at %s:%d).", (int)buf_datalen(conn->outbuf), - escaped_safe_str(conn->address), conn->s, - conn_type_to_string(conn->type), conn->state, + escaped_safe_str_client(conn->address), + conn->s, conn_type_to_string(conn->type), conn->state, conn->marked_for_close_file, conn->marked_for_close); } @@ -623,7 +681,7 @@ directory_all_unreachable(time_t now) log_notice(LD_NET, "Is your network connection down? " "Failing connection to '%s:%d'.", - safe_str(edge_conn->socks_request->address), + safe_str_client(edge_conn->socks_request->address), edge_conn->socks_request->port); connection_mark_unattached_ap(edge_conn, END_STREAM_REASON_NET_UNREACHABLE); @@ -651,7 +709,7 @@ directory_info_has_arrived(time_t now, int from_cache) /* if we have enough dir info, then update our guard status with * whatever we just learned. */ - entry_guards_compute_status(); + entry_guards_compute_status(options, now); /* Don't even bother trying to get extrainfo until the rest of our * directory info is up-to-date */ if (options->DownloadExtraInfo) @@ -659,7 +717,7 @@ directory_info_has_arrived(time_t now, int from_cache) } if (server_mode(options) && !we_are_hibernating() && !from_cache && - (has_completed_circuit || !any_predicted_circuits(now))) + (can_complete_circuit || !any_predicted_circuits(now))) consider_testing_reachability(1, 1); } @@ -722,6 +780,7 @@ run_connection_housekeeping(int i, time_t now) the connection or send a keepalive, depending. */ or_conn = TO_OR_CONN(conn); + tor_assert(conn->outbuf); if (or_conn->is_bad_for_new_circs && !or_conn->n_circuits) { /* It's bad for new circuits, and has no unmarked circuits on it: @@ -778,11 +837,18 @@ run_connection_housekeeping(int i, time_t now) } } -/** Honor a NEWNYM request: make future requests unlinkability to past +/** Honor a NEWNYM request: make future requests unlinkable to past * requests. */ static void signewnym_impl(time_t now) { + or_options_t *options = get_options(); + if (!proxy_mode(options)) { + log_info(LD_CONTROL, "Ignoring SIGNAL NEWNYM because client functionality " + "is disabled."); + return; + } + circuit_expire_all_dirty_circs(); addressmap_clear_transient(); rend_cache_purge(); @@ -806,16 +872,19 @@ run_scheduled_events(time_t now) static time_t time_to_try_getting_descriptors = 0; static time_t time_to_reset_descriptor_failures = 0; static time_t time_to_add_entropy = 0; - static time_t time_to_write_hs_statistics = 0; static time_t time_to_write_bridge_status_file = 0; static time_t time_to_downrate_stability = 0; static time_t time_to_save_stability = 0; static time_t time_to_clean_caches = 0; static time_t time_to_recheck_bandwidth = 0; static time_t time_to_check_for_expired_networkstatus = 0; - static time_t time_to_dump_geoip_stats = 0; + static time_t time_to_write_stats_files = 0; + static time_t time_to_write_bridge_stats = 0; + static time_t time_to_launch_reachability_tests = 0; + static int should_init_bridge_stats = 1; static time_t time_to_retry_dns_init = 0; or_options_t *options = get_options(); + int is_server = server_mode(options); int i; int have_dir_info; @@ -833,11 +902,14 @@ run_scheduled_events(time_t now) signewnym_impl(now); } + /* 0c. If we've deferred log messages for the controller, handle them now */ + flush_pending_log_callbacks(); + /** 1a. Every MIN_ONION_KEY_LIFETIME seconds, rotate the onion keys, * shut down and restart all cpuworkers, and update the directory if * necessary. */ - if (server_mode(options) && + if (is_server && get_onion_key_set_at()+MIN_ONION_KEY_LIFETIME < now) { log_info(LD_GENERAL,"Rotating onion key."); rotate_onion_key(); @@ -853,7 +925,7 @@ run_scheduled_events(time_t now) update_router_descriptor_downloads(now); update_extrainfo_downloads(now); if (options->UseBridges) - fetch_bridge_descriptors(now); + fetch_bridge_descriptors(options, now); if (router_have_minimum_dir_info()) time_to_try_getting_descriptors = now + LAZY_DESCRIPTOR_RETRY_INTERVAL; else @@ -871,7 +943,10 @@ run_scheduled_events(time_t now) last_rotated_x509_certificate = now; if (last_rotated_x509_certificate+MAX_SSL_KEY_LIFETIME < now) { log_info(LD_GENERAL,"Rotating tls context."); - if (tor_tls_context_new(get_identity_key(), MAX_SSL_KEY_LIFETIME) < 0) { + if (tor_tls_context_init(public_server_mode(options), + get_tlsclient_identity_key(), + is_server ? get_server_identity_key() : NULL, + MAX_SSL_KEY_LIFETIME) < 0) { log_warn(LD_BUG, "Error reinitializing TLS context"); /* XXX is it a bug here, that we just keep going? -RD */ } @@ -896,10 +971,12 @@ run_scheduled_events(time_t now) if (accounting_is_enabled(options)) accounting_run_housekeeping(now); - if (now % 10 == 0 && (authdir_mode_tests_reachability(options)) && - !we_are_hibernating()) { + if (time_to_launch_reachability_tests < now && + (authdir_mode_tests_reachability(options)) && + !we_are_hibernating()) { + time_to_launch_reachability_tests = now + REACHABILITY_TEST_INTERVAL; /* try to determine reachability of the other Tor relays */ - dirserv_test_reachability(now, 0); + dirserv_test_reachability(now); } /** 1d. Periodically, we discount older stability information so that new @@ -941,11 +1018,56 @@ run_scheduled_events(time_t now) time_to_check_for_expired_networkstatus = now + CHECK_EXPIRED_NS_INTERVAL; } - if (time_to_dump_geoip_stats < now) { -#define DUMP_GEOIP_STATS_INTERVAL (60*60); - if (time_to_dump_geoip_stats) - dump_geoip_stats(); - time_to_dump_geoip_stats = now + DUMP_GEOIP_STATS_INTERVAL; + /* 1g. Check whether we should write statistics to disk. + */ + if (time_to_write_stats_files < now) { +#define CHECK_WRITE_STATS_INTERVAL (60*60) + time_t next_time_to_write_stats_files = (time_to_write_stats_files > 0 ? + time_to_write_stats_files : now) + CHECK_WRITE_STATS_INTERVAL; + if (options->CellStatistics) { + time_t next_write = + rep_hist_buffer_stats_write(time_to_write_stats_files); + if (next_write && next_write < next_time_to_write_stats_files) + next_time_to_write_stats_files = next_write; + } + if (options->DirReqStatistics) { + time_t next_write = geoip_dirreq_stats_write(time_to_write_stats_files); + if (next_write && next_write < next_time_to_write_stats_files) + next_time_to_write_stats_files = next_write; + } + if (options->EntryStatistics) { + time_t next_write = geoip_entry_stats_write(time_to_write_stats_files); + if (next_write && next_write < next_time_to_write_stats_files) + next_time_to_write_stats_files = next_write; + } + if (options->ExitPortStatistics) { + time_t next_write = rep_hist_exit_stats_write(time_to_write_stats_files); + if (next_write && next_write < next_time_to_write_stats_files) + next_time_to_write_stats_files = next_write; + } + time_to_write_stats_files = next_time_to_write_stats_files; + } + + /* 1h. Check whether we should write bridge statistics to disk. + */ + if (should_record_bridge_info(options)) { + if (time_to_write_bridge_stats < now) { + if (should_init_bridge_stats) { + /* (Re-)initialize bridge statistics. */ + geoip_bridge_stats_init(now); + time_to_write_bridge_stats = now + WRITE_STATS_INTERVAL; + should_init_bridge_stats = 0; + } else { + /* Possibly write bridge statistics to disk and ask when to write + * them next time. */ + time_to_write_bridge_stats = geoip_bridge_stats_write( + time_to_write_bridge_stats); + } + } + } else if (!should_init_bridge_stats) { + /* Bridge mode was turned off. Ensure that stats are re-initialized + * next time bridge mode is turned on. */ + should_init_bridge_stats = 1; } /* Remove old information from rephist and the rend cache. */ @@ -953,6 +1075,8 @@ run_scheduled_events(time_t now) rep_history_clean(now - options->RephistTrackTime); rend_cache_clean(); rend_cache_clean_v2_descs_as_dir(); + if (authdir_mode_v3(options)) + microdesc_cache_rebuild(NULL, 0); #define CLEAN_CACHES_INTERVAL (30*60) time_to_clean_caches = now + CLEAN_CACHES_INTERVAL; } @@ -993,7 +1117,7 @@ run_scheduled_events(time_t now) /* also, check religiously for reachability, if it's within the first * 20 minutes of our uptime. */ if (server_mode(options) && - (has_completed_circuit || !any_predicted_circuits(now)) && + (can_complete_circuit || !any_predicted_circuits(now)) && !we_are_hibernating()) { if (stats_n_seconds_working < TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT) { consider_testing_reachability(1, dirport_reachability_count==0); @@ -1036,7 +1160,9 @@ run_scheduled_events(time_t now) * We do this before step 4, so it can try building more if * it's not comfortable with the number of available circuits. */ - circuit_expire_building(now); + /* XXXX022 If our circuit build timeout is much lower than a second, maybe + we should do this more often? */ + circuit_expire_building(); /** 3b. Also look at pending streams and prune the ones that 'began' * a long time ago but haven't gotten a 'connected' yet. @@ -1069,7 +1195,7 @@ run_scheduled_events(time_t now) circuit_expire_old_circuits_serverside(now); /** 5. We do housekeeping for each connection... */ - connection_or_set_bad_connections(); + connection_or_set_bad_connections(NULL, 0); for (i=0;i<smartlist_len(connection_array);i++) { run_connection_housekeeping(i, now); } @@ -1092,7 +1218,7 @@ run_scheduled_events(time_t now) circuit_close_all_marked(); /** 7. And upload service descriptors if necessary. */ - if (has_completed_circuit && !we_are_hibernating()) { + if (can_complete_circuit && !we_are_hibernating()) { rend_consider_services_upload(now); rend_consider_descriptor_republication(); } @@ -1109,7 +1235,7 @@ run_scheduled_events(time_t now) /** 9. and if we're a server, check whether our DNS is telling stories to * us. */ - if (server_mode(options) && time_to_check_for_correct_dns < now) { + if (is_server && time_to_check_for_correct_dns < now) { if (!time_to_check_for_correct_dns) { time_to_check_for_correct_dns = now + 60 + crypto_rand_int(120); } else { @@ -1119,12 +1245,6 @@ run_scheduled_events(time_t now) } } - /** 10. write hidden service usage statistic to disk */ - if (options->HSAuthorityRecordStats && time_to_write_hs_statistics < now) { - hs_usage_write_statistics_to_file(now); -#define WRITE_HSUSAGE_INTERVAL (30*60) - time_to_write_hs_statistics = now+WRITE_HSUSAGE_INTERVAL; - } /** 10b. write bridge networkstatus file to disk */ if (options->BridgeAuthoritativeDir && time_to_write_bridge_status_file < now) { @@ -1134,39 +1254,30 @@ run_scheduled_events(time_t now) } } -/** Libevent timer: used to invoke second_elapsed_callback() once per - * second. */ -static struct event *timeout_event = NULL; +/** Timer: used to invoke second_elapsed_callback() once per second. */ +static periodic_timer_t *second_timer = NULL; /** Number of libevent errors in the last second: we die if we get too many. */ static int n_libevent_errors = 0; /** Libevent callback: invoked once every second. */ static void -second_elapsed_callback(int fd, short event, void *args) +second_elapsed_callback(periodic_timer_t *timer, void *arg) { /* XXXX This could be sensibly refactored into multiple callbacks, and we * could use Libevent's timers for this rather than checking the current * time against a bunch of timeouts every second. */ - static struct timeval one_second; static time_t current_second = 0; time_t now; size_t bytes_written; size_t bytes_read; int seconds_elapsed; or_options_t *options = get_options(); - (void)fd; - (void)event; - (void)args; - if (!timeout_event) { - timeout_event = tor_malloc_zero(sizeof(struct event)); - evtimer_set(timeout_event, second_elapsed_callback, NULL); - one_second.tv_sec = 1; - one_second.tv_usec = 0; - } + (void)timer; + (void)arg; n_libevent_errors = 0; - /* log_fn(LOG_NOTICE, "Tick."); */ + /* log_notice(LD_GENERAL, "Tick."); */ now = time(NULL); update_approx_time(now); @@ -1189,7 +1300,7 @@ second_elapsed_callback(int fd, short event, void *args) if (server_mode(options) && !we_are_hibernating() && seconds_elapsed > 0 && - has_completed_circuit && + can_complete_circuit && stats_n_seconds_working / TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT != (stats_n_seconds_working+seconds_elapsed) / TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT) { @@ -1231,18 +1342,6 @@ second_elapsed_callback(int fd, short event, void *args) run_scheduled_events(now); current_second = now; /* remember which second it is, for next time */ - -#if 0 - if (current_second % 300 == 0) { - rep_history_clean(current_second - options->RephistTrackTime); - dumpmemusage(get_min_log_level()<LOG_INFO ? - get_min_log_level() : LOG_INFO); - } -#endif - - if (evtimer_add(timeout_event, &one_second)) - log_err(LD_NET, - "Error from libevent when setting one-second timeout event"); } #ifndef MS_WINDOWS @@ -1385,7 +1484,7 @@ do_main_loop(void) /* load the private keys, if we're supposed to have them, and set up the * TLS context. */ - if (! identity_key_is_set()) { + if (! client_identity_key_is_set()) { if (init_keys() < 0) { log_err(LD_BUG,"Error initializing keys; exiting"); return -1; @@ -1403,8 +1502,10 @@ do_main_loop(void) /* initialize the bootstrap status events to know we're starting up */ control_event_bootstrap(BOOTSTRAP_STATUS_STARTING, 0); - if (trusted_dirs_reload_certs()) - return -1; + if (trusted_dirs_reload_certs()) { + log_warn(LD_DIR, + "Couldn't load all cached v3 certificates. Starting anyway."); + } if (router_reload_v2_networkstatus()) { return -1; } @@ -1421,18 +1522,23 @@ do_main_loop(void) now = time(NULL); directory_info_has_arrived(now, 1); - if (authdir_mode_tests_reachability(get_options())) { - /* the directory is already here, run startup things */ - dirserv_test_reachability(now, 1); - } - if (server_mode(get_options())) { /* launch cpuworkers. Need to do this *after* we've read the onion key. */ cpu_init(); } /* set up once-a-second callback. */ - second_elapsed_callback(0,0,NULL); + if (! second_timer) { + struct timeval one_second; + one_second.tv_sec = 1; + one_second.tv_usec = 0; + + second_timer = periodic_timer_new(tor_libevent_get_base(), + &one_second, + second_elapsed_callback, + NULL); + tor_assert(second_timer); + } for (;;) { if (nt_service_is_stopping()) @@ -1451,20 +1557,16 @@ do_main_loop(void) /* poll until we have an event, or the second ends, or until we have * some active linked connections to trigger events for. */ - loop_result = event_loop(called_loop_once ? EVLOOP_ONCE : 0); + loop_result = event_base_loop(tor_libevent_get_base(), + called_loop_once ? EVLOOP_ONCE : 0); /* let catch() handle things like ^c, and otherwise don't worry about it */ if (loop_result < 0) { int e = tor_socket_errno(-1); /* let the program survive things like ^z */ if (e != EINTR && !ERRNO_IS_EINPROGRESS(e)) { -#ifdef HAVE_EVENT_GET_METHOD log_err(LD_NET,"libevent call with %s failed: %s [%d]", - event_get_method(), tor_socket_strerror(e), e); -#else - log_err(LD_NET,"libevent call failed: %s [%d]", - tor_socket_strerror(e), e); -#endif + tor_libevent_get_method(), tor_socket_strerror(e), e); return -1; #ifndef MS_WINDOWS } else if (e == EINVAL) { @@ -1485,46 +1587,7 @@ do_main_loop(void) } } -/** Used to implement the SIGNAL control command: if we accept - * <b>the_signal</b> as a remote pseudo-signal, act on it. */ -/* We don't re-use catch() here because: - * 1. We handle a different set of signals than those allowed in catch. - * 2. Platforms without signal() are unlikely to define SIGfoo. - * 3. The control spec is defined to use fixed numeric signal values - * which just happen to match the Unix values. - */ -void -control_signal_act(int the_signal) -{ - switch (the_signal) - { - case 1: - signal_callback(0,0,(void*)(uintptr_t)SIGHUP); - break; - case 2: - signal_callback(0,0,(void*)(uintptr_t)SIGINT); - break; - case 10: - signal_callback(0,0,(void*)(uintptr_t)SIGUSR1); - break; - case 12: - signal_callback(0,0,(void*)(uintptr_t)SIGUSR2); - break; - case 15: - signal_callback(0,0,(void*)(uintptr_t)SIGTERM); - break; - case SIGNEWNYM: - signal_callback(0,0,(void*)(uintptr_t)SIGNEWNYM); - break; - case SIGCLEARDNSCACHE: - signal_callback(0,0,(void*)(uintptr_t)SIGCLEARDNSCACHE); - break; - default: - log_warn(LD_BUG, "Unrecognized signal number %d.", the_signal); - break; - } -} - +#ifndef MS_WINDOWS /* Only called when we're willing to use signals */ /** Libevent callback: invoked when we get a signal. */ static void @@ -1533,6 +1596,15 @@ signal_callback(int fd, short events, void *arg) uintptr_t sig = (uintptr_t)arg; (void)fd; (void)events; + + process_signal(sig); +} +#endif + +/** Do the work of acting on a signal received in <b>sig</b> */ +void +process_signal(uintptr_t sig) +{ switch (sig) { case SIGTERM: @@ -1607,6 +1679,7 @@ dumpmemusage(int severity) U64_PRINTF_ARG(rephist_total_alloc), rephist_total_num); dump_routerlist_mem_usage(severity); dump_cell_pool_usage(severity); + dump_dns_mem_usage(severity); buf_dump_freelist_sizes(severity); tor_log_mallinfo(severity); } @@ -1633,7 +1706,8 @@ dumpstats(int severity) if (!connection_is_listener(conn)) { log(severity,LD_GENERAL, "Conn %d is to %s:%d.", i, - safe_str(conn->address), conn->port); + safe_str_client(conn->address), + conn->port); log(severity,LD_GENERAL, "Conn %d: %d bytes waiting on inbuf (len %d, last read %d secs ago)", i, @@ -1730,7 +1804,7 @@ handle_signals(int is_parent) { #ifndef MS_WINDOWS /* do signal stuff only on Unix */ int i; - static int signals[] = { + static const int signals[] = { SIGINT, /* do a controlled slow shutdown */ SIGTERM, /* to terminate now */ SIGPIPE, /* otherwise SIGPIPE kills us */ @@ -1742,12 +1816,13 @@ handle_signals(int is_parent) #endif SIGCHLD, /* handle dns/cpu workers that exit */ -1 }; - static struct event signal_events[16]; /* bigger than it has to be. */ + static struct event *signal_events[16]; /* bigger than it has to be. */ if (is_parent) { for (i = 0; signals[i] >= 0; ++i) { - signal_set(&signal_events[i], signals[i], signal_callback, - (void*)(uintptr_t)signals[i]); - if (signal_add(&signal_events[i], NULL)) + signal_events[i] = tor_evsignal_new( + tor_libevent_get_base(), signals[i], signal_callback, + (void*)(uintptr_t)signals[i]); + if (event_add(signal_events[i], NULL)) log_warn(LD_BUG, "Error from libevent when adding event for signal %d", signals[i]); } @@ -1836,7 +1911,9 @@ tor_init(int argc, char *argv[]) "and you probably shouldn't."); #endif - if (crypto_global_init(get_options()->HardwareAccel)) { + if (crypto_global_init(get_options()->HardwareAccel, + get_options()->AccelName, + get_options()->AccelDir)) { log_err(LD_BUG, "Unable to initialize OpenSSL. Exiting."); return -1; } @@ -1930,7 +2007,6 @@ tor_free_all(int postfork) rend_cache_free_all(); rend_service_authorization_free_all(); rep_hist_free_all(); - hs_usage_free_all(); dns_free_all(); clear_pending_onions(); circuit_free_all(); @@ -1938,6 +2014,7 @@ tor_free_all(int postfork) connection_free_all(); buf_shrink_freelists(1); memarea_clear_freelist(); + microdesc_free_all(); if (!postfork) { config_free_all(); router_free_all(); @@ -1948,13 +2025,11 @@ tor_free_all(int postfork) tor_tls_free_all(); } /* stuff in main.c */ - if (connection_array) - smartlist_free(connection_array); - if (closeable_connection_lst) - smartlist_free(closeable_connection_lst); - if (active_linked_connection_lst) - smartlist_free(active_linked_connection_lst); - tor_free(timeout_event); + + smartlist_free(connection_array); + smartlist_free(closeable_connection_lst); + smartlist_free(active_linked_connection_lst); + periodic_timer_free(second_timer); if (!postfork) { release_lockfile(); } @@ -2015,7 +2090,7 @@ do_list_fingerprint(void) log_err(LD_BUG,"Error initializing keys; can't display fingerprint"); return -1; } - if (!(k = get_identity_key())) { + if (!(k = get_server_identity_key())) { log_err(LD_GENERAL,"Error: missing identity key."); return -1; } @@ -2045,6 +2120,31 @@ do_hash_password(void) printf("16:%s\n",output); } +#if defined (WINCE) +int +find_flashcard_path(PWCHAR path, size_t size) +{ + WIN32_FIND_DATA d = {0}; + HANDLE h = NULL; + + if (!path) + return -1; + + h = FindFirstFlashCard(&d); + if (h == INVALID_HANDLE_VALUE) + return -1; + + if (wcslen(d.cFileName) == 0) { + FindClose(h); + return -1; + } + + wcsncpy(path,d.cFileName,size); + FindClose(h); + return 0; +} +#endif + /** Main entry point for the Tor process. Called from main(). */ /* This function is distinct from main() only so we can link main.c into * the unittest binary without conflicting with the unittests' main. */ @@ -2052,6 +2152,46 @@ int tor_main(int argc, char *argv[]) { int result = 0; +#if defined (WINCE) + WCHAR path [MAX_PATH] = {0}; + WCHAR fullpath [MAX_PATH] = {0}; + PWCHAR p = NULL; + FILE* redir = NULL; + FILE* redirdbg = NULL; + + // this is to facilitate debugging by opening + // a file on a folder shared by the wm emulator. + // if no flashcard (real or emulated) is present, + // log files will be written in the root folder + if (find_flashcard_path(path,MAX_PATH) == -1) + { + redir = _wfreopen( L"\\stdout.log", L"w", stdout ); + redirdbg = _wfreopen( L"\\stderr.log", L"w", stderr ); + } else { + swprintf(fullpath,L"\\%s\\tor",path); + CreateDirectory(fullpath,NULL); + + swprintf(fullpath,L"\\%s\\tor\\stdout.log",path); + redir = _wfreopen( fullpath, L"w", stdout ); + + swprintf(fullpath,L"\\%s\\tor\\stderr.log",path); + redirdbg = _wfreopen( fullpath, L"w", stderr ); + } +#endif + +#ifdef MS_WINDOWS + /* Call SetProcessDEPPolicy to permanently enable DEP. + The function will not resolve on earlier versions of Windows, + and failure is not dangerous. */ + HMODULE hMod = GetModuleHandleA("Kernel32.dll"); + if (hMod) { + typedef BOOL (WINAPI *PSETDEP)(DWORD); + PSETDEP setdeppolicy = (PSETDEP)GetProcAddress(hMod, + "SetProcessDEPPolicy"); + if (setdeppolicy) setdeppolicy(1); /* PROCESS_DEP_ENABLE */ + } +#endif + update_approx_time(time(NULL)); tor_threads_init(); init_logging(); diff --git a/src/or/main.h b/src/or/main.h new file mode 100644 index 0000000000..0551f7aaf9 --- /dev/null +++ b/src/or/main.h @@ -0,0 +1,69 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file main.h + * \brief Header file for main.c. + **/ + +#ifndef _TOR_MAIN_H +#define _TOR_MAIN_H + +extern int can_complete_circuit; + +int connection_add(connection_t *conn); +int connection_remove(connection_t *conn); +void connection_unregister_events(connection_t *conn); +int connection_in_array(connection_t *conn); +void add_connection_to_closeable_list(connection_t *conn); +int connection_is_on_closeable_list(connection_t *conn); + +smartlist_t *get_connection_array(void); + +/** Bitmask for events that we can turn on and off with + * connection_watch_events. */ +typedef enum watchable_events { + READ_EVENT=0x02, /**< We want to know when a connection is readable */ + WRITE_EVENT=0x04 /**< We want to know when a connection is writable */ +} watchable_events_t; +void connection_watch_events(connection_t *conn, watchable_events_t events); +int connection_is_reading(connection_t *conn); +void connection_stop_reading(connection_t *conn); +void connection_start_reading(connection_t *conn); + +int connection_is_writing(connection_t *conn); +void connection_stop_writing(connection_t *conn); +void connection_start_writing(connection_t *conn); + +void connection_stop_reading_from_linked_conn(connection_t *conn); + +void directory_all_unreachable(time_t now); +void directory_info_has_arrived(time_t now, int from_cache); + +void ip_address_changed(int at_interface); +void dns_servers_relaunch_checks(void); + +void handle_signals(int is_parent); +void process_signal(uintptr_t sig); + +int try_locking(or_options_t *options, int err_if_locked); +int have_lockfile(void); +void release_lockfile(void); + +void tor_cleanup(void); +void tor_free_all(int postfork); + +int tor_main(int argc, char *argv[]); + +#ifdef MAIN_PRIVATE +int do_main_loop(void); +int do_list_fingerprint(void); +void do_hash_password(void); +int tor_init(int argc, char **argv); +#endif + +#endif + diff --git a/src/or/microdesc.c b/src/or/microdesc.c new file mode 100644 index 0000000000..5740c40d5f --- /dev/null +++ b/src/or/microdesc.c @@ -0,0 +1,492 @@ +/* Copyright (c) 2009-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "or.h" +#include "config.h" +#include "microdesc.h" +#include "routerparse.h" + +/** A data structure to hold a bunch of cached microdescriptors. There are + * two active files in the cache: a "cache file" that we mmap, and a "journal + * file" that we append to. Periodically, we rebuild the cache file to hold + * only the microdescriptors that we want to keep */ +struct microdesc_cache_t { + /** Map from sha256-digest to microdesc_t for every microdesc_t in the + * cache. */ + HT_HEAD(microdesc_map, microdesc_t) map; + + /** Name of the cache file. */ + char *cache_fname; + /** Name of the journal file. */ + char *journal_fname; + /** Mmap'd contents of the cache file, or NULL if there is none. */ + tor_mmap_t *cache_content; + /** Number of bytes used in the journal file. */ + size_t journal_len; + /** Number of bytes in descriptors removed as too old. */ + size_t bytes_dropped; + + /** Total bytes of microdescriptor bodies we have added to this cache */ + uint64_t total_len_seen; + /** Total number of microdescriptors we have added to this cache */ + unsigned n_seen; +}; + +/** Helper: computes a hash of <b>md</b> to place it in a hash table. */ +static INLINE unsigned int +_microdesc_hash(microdesc_t *md) +{ + unsigned *d = (unsigned*)md->digest; +#if SIZEOF_INT == 4 + return d[0] ^ d[1] ^ d[2] ^ d[3] ^ d[4] ^ d[5] ^ d[6] ^ d[7]; +#else + return d[0] ^ d[1] ^ d[2] ^ d[3]; +#endif +} + +/** Helper: compares <b>a</b> and </b> for equality for hash-table purposes. */ +static INLINE int +_microdesc_eq(microdesc_t *a, microdesc_t *b) +{ + return !memcmp(a->digest, b->digest, DIGEST256_LEN); +} + +HT_PROTOTYPE(microdesc_map, microdesc_t, node, + _microdesc_hash, _microdesc_eq); +HT_GENERATE(microdesc_map, microdesc_t, node, + _microdesc_hash, _microdesc_eq, 0.6, + malloc, realloc, free); + +/** Write the body of <b>md</b> into <b>f</b>, with appropriate annotations. + * On success, return the total number of bytes written, and set + * *<b>annotation_len_out</b> to the number of bytes written as + * annotations. */ +static ssize_t +dump_microdescriptor(FILE *f, microdesc_t *md, size_t *annotation_len_out) +{ + ssize_t r = 0; + size_t written; + /* XXXX drops unkown annotations. */ + if (md->last_listed) { + char buf[ISO_TIME_LEN+1]; + char annotation[ISO_TIME_LEN+32]; + format_iso_time(buf, md->last_listed); + tor_snprintf(annotation, sizeof(annotation), "@last-listed %s\n", buf); + fputs(annotation, f); + r += strlen(annotation); + *annotation_len_out = r; + } else { + *annotation_len_out = 0; + } + + md->off = (off_t) ftell(f); + written = fwrite(md->body, 1, md->bodylen, f); + if (written != md->bodylen) { + log_warn(LD_DIR, + "Couldn't dump microdescriptor (wrote %lu out of %lu): %s", + (unsigned long)written, (unsigned long)md->bodylen, + strerror(ferror(f))); + return -1; + } + r += md->bodylen; + return r; +} + +/** Holds a pointer to the current microdesc_cache_t object, or NULL if no + * such object has been allocated. */ +static microdesc_cache_t *the_microdesc_cache = NULL; + +/** Return a pointer to the microdescriptor cache, loading it if necessary. */ +microdesc_cache_t * +get_microdesc_cache(void) +{ + if (PREDICT_UNLIKELY(the_microdesc_cache==NULL)) { + microdesc_cache_t *cache = tor_malloc_zero(sizeof(microdesc_cache_t)); + HT_INIT(microdesc_map, &cache->map); + cache->cache_fname = get_datadir_fname("cached-microdescs"); + cache->journal_fname = get_datadir_fname("cached-microdescs.new"); + microdesc_cache_reload(cache); + the_microdesc_cache = cache; + } + return the_microdesc_cache; +} + +/* There are three sources of microdescriptors: + 1) Generated by us while acting as a directory authority. + 2) Loaded from the cache on disk. + 3) Downloaded. +*/ + +/** Decode the microdescriptors from the string starting at <b>s</b> and + * ending at <b>eos</b>, and store them in <b>cache</b>. If <b>no-save</b>, + * mark them as non-writable to disk. If <b>where</b> is SAVED_IN_CACHE, + * leave their bodies as pointers to the mmap'd cache. If where is + * <b>SAVED_NOWHERE</b>, do not allow annotations. Return a list of the added + * microdescriptors. */ +smartlist_t * +microdescs_add_to_cache(microdesc_cache_t *cache, + const char *s, const char *eos, saved_location_t where, + int no_save) +{ + /*XXXX need an argument that sets last_listed as appropriate. */ + + smartlist_t *descriptors, *added; + const int allow_annotations = (where != SAVED_NOWHERE); + const int copy_body = (where != SAVED_IN_CACHE); + + descriptors = microdescs_parse_from_string(s, eos, + allow_annotations, + copy_body); + + added = microdescs_add_list_to_cache(cache, descriptors, where, no_save); + smartlist_free(descriptors); + return added; +} + +/* As microdescs_add_to_cache, but takes a list of micrdescriptors instead of + * a string to encode. Frees any members of <b>descriptors</b> that it does + * not add. */ +smartlist_t * +microdescs_add_list_to_cache(microdesc_cache_t *cache, + smartlist_t *descriptors, saved_location_t where, + int no_save) +{ + smartlist_t *added; + open_file_t *open_file = NULL; + FILE *f = NULL; + // int n_added = 0; + ssize_t size = 0; + + if (where == SAVED_NOWHERE && !no_save) { + f = start_writing_to_stdio_file(cache->journal_fname, + OPEN_FLAGS_APPEND|O_BINARY, + 0600, &open_file); + if (!f) { + log_warn(LD_DIR, "Couldn't append to journal in %s: %s", + cache->journal_fname, strerror(errno)); + return NULL; + } + } + + added = smartlist_create(); + SMARTLIST_FOREACH_BEGIN(descriptors, microdesc_t *, md) { + microdesc_t *md2; + md2 = HT_FIND(microdesc_map, &cache->map, md); + if (md2) { + /* We already had this one. */ + if (md2->last_listed < md->last_listed) + md2->last_listed = md->last_listed; + microdesc_free(md); + if (where != SAVED_NOWHERE) + cache->bytes_dropped += size; + continue; + } + + /* Okay, it's a new one. */ + if (f) { + size_t annotation_len; + size = dump_microdescriptor(f, md, &annotation_len); + if (size < 0) { + /* XXX handle errors from dump_microdescriptor() */ + /* log? return -1? die? coredump the universe? */ + continue; + } + md->saved_location = SAVED_IN_JOURNAL; + cache->journal_len += size; + } else { + md->saved_location = where; + } + + md->no_save = no_save; + + HT_INSERT(microdesc_map, &cache->map, md); + smartlist_add(added, md); + ++cache->n_seen; + cache->total_len_seen += md->bodylen; + } SMARTLIST_FOREACH_END(md); + + if (f) + finish_writing_to_file(open_file); /*XXX Check me.*/ + + return added; +} + +/** Remove every microdescriptor in <b>cache</b>. */ +void +microdesc_cache_clear(microdesc_cache_t *cache) +{ + microdesc_t **entry, **next; + for (entry = HT_START(microdesc_map, &cache->map); entry; entry = next) { + microdesc_t *md = *entry; + next = HT_NEXT_RMV(microdesc_map, &cache->map, entry); + microdesc_free(md); + } + HT_CLEAR(microdesc_map, &cache->map); + if (cache->cache_content) { + tor_munmap_file(cache->cache_content); + cache->cache_content = NULL; + } + cache->total_len_seen = 0; + cache->n_seen = 0; + cache->bytes_dropped = 0; +} + +/** Reload the contents of <b>cache</b> from disk. If it is empty, load it + * for the first time. Return 0 on success, -1 on failure. */ +int +microdesc_cache_reload(microdesc_cache_t *cache) +{ + struct stat st; + char *journal_content; + smartlist_t *added; + tor_mmap_t *mm; + int total = 0; + + microdesc_cache_clear(cache); + + mm = cache->cache_content = tor_mmap_file(cache->cache_fname); + if (mm) { + added = microdescs_add_to_cache(cache, mm->data, mm->data+mm->size, + SAVED_IN_CACHE, 0); + if (added) { + total += smartlist_len(added); + smartlist_free(added); + } + } + + journal_content = read_file_to_str(cache->journal_fname, + RFTS_IGNORE_MISSING, &st); + if (journal_content) { + cache->journal_len = (size_t) st.st_size; + added = microdescs_add_to_cache(cache, journal_content, + journal_content+st.st_size, + SAVED_IN_JOURNAL, 0); + if (added) { + total += smartlist_len(added); + smartlist_free(added); + } + tor_free(journal_content); + } + log_notice(LD_DIR, "Reloaded microdescriptor cache. Found %d descriptors.", + total); + + microdesc_cache_rebuild(cache, 0 /* don't force */); + + return 0; +} + +/** By default, we remove any microdescriptors that have gone at least this + * long without appearing in a current consensus. */ +#define TOLERATE_MICRODESC_AGE (7*24*60*60) + +/** Remove all microdescriptors from <b>cache</b> that haven't been listed for + * a long time. Does not rebuild the cache on disk. If <b>cutoff</b> is + * positive, specifically remove microdescriptors that have been unlisted + * since <b>cutoff</b>. If <b>force</b> is true, remove microdescriptors even + * if we have no current live microdescriptor consensus. + */ +void +microdesc_cache_clean(microdesc_cache_t *cache, time_t cutoff, int force) +{ + microdesc_t **mdp, *victim; + int dropped=0, kept=0; + size_t bytes_dropped = 0; + time_t now = time(NULL); + + (void) force; + /* In 0.2.2, we let this proceed unconditionally: only authorities have + * microdesc caches. */ + + if (cutoff <= 0) + cutoff = now - TOLERATE_MICRODESC_AGE; + + for (mdp = HT_START(microdesc_map, &cache->map); mdp != NULL; ) { + if ((*mdp)->last_listed < cutoff) { + ++dropped; + victim = *mdp; + mdp = HT_NEXT_RMV(microdesc_map, &cache->map, mdp); + bytes_dropped += victim->bodylen; + microdesc_free(victim); + } else { + ++kept; + mdp = HT_NEXT(microdesc_map, &cache->map, mdp); + } + } + + if (dropped) { + log_notice(LD_DIR, "Removed %d/%d microdescriptors as old.", + dropped,dropped+kept); + cache->bytes_dropped += bytes_dropped; + } +} + +static int +should_rebuild_md_cache(microdesc_cache_t *cache) +{ + const size_t old_len = + cache->cache_content ? cache->cache_content->size : 0; + const size_t journal_len = cache->journal_len; + const size_t dropped = cache->bytes_dropped; + + if (journal_len < 16384) + return 0; /* Don't bother, not enough has happened yet. */ + if (dropped > (journal_len + old_len) / 3) + return 1; /* We could save 1/3 or more of the currently used space. */ + if (journal_len > old_len / 2) + return 1; /* We should append to the regular file */ + + return 0; +} + +/** Regenerate the main cache file for <b>cache</b>, clear the journal file, + * and update every microdesc_t in the cache with pointers to its new + * location. If <b>force</b> is true, do this unconditionally. If + * <b>force</b> is false, do it only if we expect to save space on disk. */ +int +microdesc_cache_rebuild(microdesc_cache_t *cache, int force) +{ + open_file_t *open_file; + FILE *f; + microdesc_t **mdp; + smartlist_t *wrote; + ssize_t size; + off_t off = 0; + int orig_size, new_size; + + if (cache == NULL) { + cache = the_microdesc_cache; + if (cache == NULL) + return 0; + } + + /* Remove dead descriptors */ + microdesc_cache_clean(cache, 0/*cutoff*/, 0/*force*/); + + if (!force && !should_rebuild_md_cache(cache)) + return 0; + + log_info(LD_DIR, "Rebuilding the microdescriptor cache..."); + + orig_size = (int)(cache->cache_content ? cache->cache_content->size : 0); + orig_size += (int)cache->journal_len; + + f = start_writing_to_stdio_file(cache->cache_fname, + OPEN_FLAGS_REPLACE|O_BINARY, + 0600, &open_file); + if (!f) + return -1; + + wrote = smartlist_create(); + + HT_FOREACH(mdp, microdesc_map, &cache->map) { + microdesc_t *md = *mdp; + size_t annotation_len; + if (md->no_save) + continue; + + size = dump_microdescriptor(f, md, &annotation_len); + if (size < 0) { + /* XXX handle errors from dump_microdescriptor() */ + /* log? return -1? die? coredump the universe? */ + continue; + } + md->off = off + annotation_len; + off += size; + if (md->saved_location != SAVED_IN_CACHE) { + tor_free(md->body); + md->saved_location = SAVED_IN_CACHE; + } + smartlist_add(wrote, md); + } + + finish_writing_to_file(open_file); /*XXX Check me.*/ + + if (cache->cache_content) + tor_munmap_file(cache->cache_content); + cache->cache_content = tor_mmap_file(cache->cache_fname); + + if (!cache->cache_content && smartlist_len(wrote)) { + log_err(LD_DIR, "Couldn't map file that we just wrote to %s!", + cache->cache_fname); + smartlist_free(wrote); + return -1; + } + SMARTLIST_FOREACH_BEGIN(wrote, microdesc_t *, md) { + tor_assert(md->saved_location == SAVED_IN_CACHE); + md->body = (char*)cache->cache_content->data + md->off; + tor_assert(!memcmp(md->body, "onion-key", 9)); + } SMARTLIST_FOREACH_END(md); + + smartlist_free(wrote); + + write_str_to_file(cache->journal_fname, "", 1); + cache->journal_len = 0; + cache->bytes_dropped = 0; + + new_size = cache->cache_content ? (int)cache->cache_content->size : 0; + log_info(LD_DIR, "Done rebuilding microdesc cache. " + "Saved %d bytes; %d still used.", + orig_size-new_size, new_size); + + return 0; +} + +/** Deallocate a single microdescriptor. Note: the microdescriptor MUST have + * previously been removed from the cache if it had ever been inserted. */ +void +microdesc_free(microdesc_t *md) +{ + if (!md) + return; + /* Must be removed from hash table! */ + if (md->onion_pkey) + crypto_free_pk_env(md->onion_pkey); + if (md->body && md->saved_location != SAVED_IN_CACHE) + tor_free(md->body); + + if (md->family) { + SMARTLIST_FOREACH(md->family, char *, cp, tor_free(cp)); + smartlist_free(md->family); + } + tor_free(md->exitsummary); + + tor_free(md); +} + +/** Free all storage held in the microdesc.c module. */ +void +microdesc_free_all(void) +{ + if (the_microdesc_cache) { + microdesc_cache_clear(the_microdesc_cache); + tor_free(the_microdesc_cache->cache_fname); + tor_free(the_microdesc_cache->journal_fname); + tor_free(the_microdesc_cache); + } +} + +/** If there is a microdescriptor in <b>cache</b> whose sha256 digest is + * <b>d</b>, return it. Otherwise return NULL. */ +microdesc_t * +microdesc_cache_lookup_by_digest256(microdesc_cache_t *cache, const char *d) +{ + microdesc_t *md, search; + if (!cache) + cache = get_microdesc_cache(); + memcpy(search.digest, d, DIGEST256_LEN); + md = HT_FIND(microdesc_map, &cache->map, &search); + return md; +} + +/** Return the mean size of decriptors added to <b>cache</b> since it was last + * cleared. Used to estimate the size of large downloads. */ +size_t +microdesc_average_size(microdesc_cache_t *cache) +{ + if (!cache) + cache = get_microdesc_cache(); + if (!cache->n_seen) + return 512; + return (size_t)(cache->total_len_seen / cache->n_seen); +} + diff --git a/src/or/microdesc.h b/src/or/microdesc.h new file mode 100644 index 0000000000..77ce8536bc --- /dev/null +++ b/src/or/microdesc.h @@ -0,0 +1,38 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file microdesc.h + * \brief Header file for microdesc.c. + **/ + +#ifndef _TOR_MICRODESC_H +#define _TOR_MICRODESC_H + +microdesc_cache_t *get_microdesc_cache(void); + +smartlist_t *microdescs_add_to_cache(microdesc_cache_t *cache, + const char *s, const char *eos, saved_location_t where, + int no_save); +smartlist_t *microdescs_add_list_to_cache(microdesc_cache_t *cache, + smartlist_t *descriptors, saved_location_t where, + int no_save); + +void microdesc_cache_clean(microdesc_cache_t *cache, time_t cutoff, int force); +int microdesc_cache_rebuild(microdesc_cache_t *cache, int force); +int microdesc_cache_reload(microdesc_cache_t *cache); +void microdesc_cache_clear(microdesc_cache_t *cache); + +microdesc_t *microdesc_cache_lookup_by_digest256(microdesc_cache_t *cache, + const char *d); + +size_t microdesc_average_size(microdesc_cache_t *cache); + +void microdesc_free(microdesc_t *md); +void microdesc_free_all(void); + +#endif + diff --git a/src/or/networkstatus.c b/src/or/networkstatus.c index dcd8159aff..ff20566cd6 100644 --- a/src/or/networkstatus.c +++ b/src/or/networkstatus.c @@ -11,6 +11,20 @@ */ #include "or.h" +#include "circuitbuild.h" +#include "config.h" +#include "connection.h" +#include "connection_or.h" +#include "control.h" +#include "directory.h" +#include "dirserv.h" +#include "dirvote.h" +#include "main.h" +#include "networkstatus.h" +#include "relay.h" +#include "router.h" +#include "routerlist.h" +#include "routerparse.h" /* For tracking v2 networkstatus documents. Only caches do this now. */ @@ -35,16 +49,22 @@ static networkstatus_t *current_consensus = NULL; /** A v3 consensus networkstatus that we've received, but which we don't * have enough certificates to be happy about. */ -static networkstatus_t *consensus_waiting_for_certs = NULL; -/** The encoded version of consensus_waiting_for_certs. */ -static char *consensus_waiting_for_certs_body = NULL; -/** When did we set the current value of consensus_waiting_for_certs? If this - * is too recent, we shouldn't try to fetch a new consensus for a little while, - * to give ourselves time to get certificates for this one. */ -static time_t consensus_waiting_for_certs_set_at = 0; -/** Set to 1 if we've been holding on to consensus_waiting_for_certs so long - * that we should treat it as maybe being bad. */ -static int consensus_waiting_for_certs_dl_failed = 0; +typedef struct consensus_waiting_for_certs_t { + /** The consensus itself. */ + networkstatus_t *consensus; + /** The encoded version of the consensus, nul-terminated. */ + char *body; + /** When did we set the current value of consensus_waiting_for_certs? If + * this is too recent, we shouldn't try to fetch a new consensus for a + * little while, to give ourselves time to get certificates for this one. */ + time_t set_at; + /** Set to 1 if we've been holding on to it for so long we should maybe + * treat it as being bad. */ + int dl_failed; +} consensus_waiting_for_certs_t; + +static consensus_waiting_for_certs_t + consensus_waiting_for_certs[N_CONSENSUS_FLAVORS]; /** The last time we tried to download a networkstatus, or 0 for "never". We * use this to rate-limit download attempts for directory caches (including @@ -56,7 +76,7 @@ static time_t last_networkstatus_download_attempted = 0; * before the current consensus becomes invalid. */ static time_t time_to_download_next_consensus = 0; /** Download status for the current consensus networkstatus. */ -static download_status_t consensus_dl_status = { 0, 0, DL_SCHED_CONSENSUS }; +static download_status_t consensus_dl_status[N_CONSENSUS_FLAVORS]; /** True iff we have logged a warning about this OR's version being older than * listed by the authorities. */ @@ -89,6 +109,7 @@ networkstatus_reset_warnings(void) void networkstatus_reset_download_failures(void) { + int i; const smartlist_t *networkstatus_v2_list = networkstatus_get_v2_list(); SMARTLIST_FOREACH(networkstatus_v2_list, networkstatus_v2_t *, ns, SMARTLIST_FOREACH(ns->entries, routerstatus_t *, rs, @@ -97,7 +118,8 @@ networkstatus_reset_download_failures(void) rs->need_to_mirror = 1; }));; - download_status_reset(&consensus_dl_status); + for (i=0; i < N_CONSENSUS_FLAVORS; ++i) + download_status_reset(&consensus_dl_status[i]); if (v2_download_status_map) { digestmap_iter_t *iter; digestmap_t *map = v2_download_status_map; @@ -170,7 +192,7 @@ router_reload_v2_networkstatus(void) return 0; } -/** Read the cached v3 consensus networkstatus from the disk. */ +/** Read every cached v3 consensus networkstatus from the disk. */ int router_reload_consensus_networkstatus(void) { @@ -179,31 +201,46 @@ router_reload_consensus_networkstatus(void) struct stat st; or_options_t *options = get_options(); const unsigned int flags = NSSET_FROM_CACHE | NSSET_DONT_DOWNLOAD_CERTS; + int flav; /* FFFF Suppress warnings if cached consensus is bad? */ + for (flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) { + char buf[128]; + const char *flavor = networkstatus_get_flavor_name(flav); + if (flav == FLAV_NS) { + filename = get_datadir_fname("cached-consensus"); + } else { + tor_snprintf(buf, sizeof(buf), "cached-%s-consensus", flavor); + filename = get_datadir_fname(buf); + } + s = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL); + if (s) { + if (networkstatus_set_current_consensus(s, flavor, flags) < -1) { + log_warn(LD_FS, "Couldn't load consensus %s networkstatus from \"%s\"", + flavor, filename); + } + tor_free(s); + } + tor_free(filename); - filename = get_datadir_fname("cached-consensus"); - s = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL); - if (s) { - if (networkstatus_set_current_consensus(s, flags) < -1) { - log_warn(LD_FS, "Couldn't load consensus networkstatus from \"%s\"", - filename); + if (flav == FLAV_NS) { + filename = get_datadir_fname("unverified-consensus"); + } else { + tor_snprintf(buf, sizeof(buf), "unverified-%s-consensus", flavor); + filename = get_datadir_fname(buf); } - tor_free(s); - } - tor_free(filename); - filename = get_datadir_fname("unverified-consensus"); - s = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL); - if (s) { - if (networkstatus_set_current_consensus(s, + s = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL); + if (s) { + if (networkstatus_set_current_consensus(s, flavor, flags|NSSET_WAS_WAITING_FOR_CERTS)) { - log_info(LD_FS, "Couldn't load consensus networkstatus from \"%s\"", - filename); + log_info(LD_FS, "Couldn't load consensus %s networkstatus from \"%s\"", + flavor, filename); } - tor_free(s); + tor_free(s); + } + tor_free(filename); } - tor_free(filename); if (!current_consensus || (stat(options->FallbackNetworkstatusFile, &st)==0 && @@ -211,7 +248,7 @@ router_reload_consensus_networkstatus(void) s = read_file_to_str(options->FallbackNetworkstatusFile, RFTS_IGNORE_MISSING, NULL); if (s) { - if (networkstatus_set_current_consensus(s, + if (networkstatus_set_current_consensus(s, "ns", flags|NSSET_ACCEPT_OBSOLETE)) { log_info(LD_FS, "Couldn't load consensus networkstatus from \"%s\"", options->FallbackNetworkstatusFile); @@ -242,8 +279,16 @@ router_reload_consensus_networkstatus(void) static void vote_routerstatus_free(vote_routerstatus_t *rs) { + vote_microdesc_hash_t *h, *next; + if (!rs) + return; tor_free(rs->version); tor_free(rs->status.exitsummary); + for (h = rs->microdesc; h; h = next) { + tor_free(h->microdesc_hash_line); + next = h->next; + tor_free(h); + } tor_free(rs); } @@ -251,6 +296,8 @@ vote_routerstatus_free(vote_routerstatus_t *rs) void routerstatus_free(routerstatus_t *rs) { + if (!rs) + return; tor_free(rs->exitsummary); tor_free(rs); } @@ -259,6 +306,8 @@ routerstatus_free(routerstatus_t *rs) void networkstatus_v2_free(networkstatus_v2_t *ns) { + if (!ns) + return; tor_free(ns->source_address); tor_free(ns->contact); if (ns->signing_key) @@ -273,7 +322,25 @@ networkstatus_v2_free(networkstatus_v2_t *ns) tor_free(ns); } -/** Clear all storage held in <b>ns</b>. */ +/** Free all storage held in <b>sig</b> */ +void +document_signature_free(document_signature_t *sig) +{ + tor_free(sig->signature); + tor_free(sig); +} + +/** Return a newly allocated copy of <b>sig</b> */ +document_signature_t * +document_signature_dup(const document_signature_t *sig) +{ + document_signature_t *r = tor_memdup(sig, sizeof(document_signature_t)); + if (r->signature) + r->signature = tor_memdup(sig->signature, sig->signature_len); + return r; +} + +/** Free all storage held in <b>ns</b>. */ void networkstatus_vote_free(networkstatus_t *ns) { @@ -286,6 +353,10 @@ networkstatus_vote_free(networkstatus_t *ns) SMARTLIST_FOREACH(ns->known_flags, char *, c, tor_free(c)); smartlist_free(ns->known_flags); } + if (ns->weight_params) { + SMARTLIST_FOREACH(ns->weight_params, char *, c, tor_free(c)); + smartlist_free(ns->weight_params); + } if (ns->net_params) { SMARTLIST_FOREACH(ns->net_params, char *, c, tor_free(c)); smartlist_free(ns->net_params); @@ -295,18 +366,20 @@ networkstatus_vote_free(networkstatus_t *ns) smartlist_free(ns->supported_methods); } if (ns->voters) { - SMARTLIST_FOREACH(ns->voters, networkstatus_voter_info_t *, voter, - { + SMARTLIST_FOREACH_BEGIN(ns->voters, networkstatus_voter_info_t *, voter) { tor_free(voter->nickname); tor_free(voter->address); tor_free(voter->contact); - tor_free(voter->signature); + if (voter->sigs) { + SMARTLIST_FOREACH(voter->sigs, document_signature_t *, sig, + document_signature_free(sig)); + smartlist_free(voter->sigs); + } tor_free(voter); - }); + } SMARTLIST_FOREACH_END(voter); smartlist_free(ns->voters); } - if (ns->cert) - authority_cert_free(ns->cert); + authority_cert_free(ns->cert); if (ns->routerstatus_list) { if (ns->type == NS_TYPE_VOTE || ns->type == NS_TYPE_OPINION) { @@ -319,8 +392,8 @@ networkstatus_vote_free(networkstatus_t *ns) smartlist_free(ns->routerstatus_list); } - if (ns->desc_digest_map) - digestmap_free(ns->desc_digest_map, NULL); + + digestmap_free(ns->desc_digest_map, NULL); memset(ns, 11, sizeof(*ns)); tor_free(ns); @@ -341,35 +414,39 @@ networkstatus_get_voter_by_id(networkstatus_t *vote, return NULL; } -/** Check whether the signature on <b>voter</b> is correctly signed by - * the signing key of <b>cert</b>. Return -1 if <b>cert</b> doesn't match the +/** Check whether the signature <b>sig</b> is correctly signed with the + * signing key in <b>cert</b>. Return -1 if <b>cert</b> doesn't match the * signing key; otherwise set the good_signature or bad_signature flag on * <b>voter</b>, and return 0. */ -/* (private; exposed for testing.) */ int -networkstatus_check_voter_signature(networkstatus_t *consensus, - networkstatus_voter_info_t *voter, - authority_cert_t *cert) +networkstatus_check_document_signature(const networkstatus_t *consensus, + document_signature_t *sig, + const authority_cert_t *cert) { - char d[DIGEST_LEN]; + char key_digest[DIGEST_LEN]; + const int dlen = sig->alg == DIGEST_SHA1 ? DIGEST_LEN : DIGEST256_LEN; char *signed_digest; size_t signed_digest_len; - if (crypto_pk_get_digest(cert->signing_key, d)<0) + + if (crypto_pk_get_digest(cert->signing_key, key_digest)<0) return -1; - if (tor_memneq(voter->signing_key_digest, d, DIGEST_LEN)) + if (tor_memneq(sig->signing_key_digest, key_digest, DIGEST_LEN) || + tor_memneq(sig->identity_digest, cert->cache_info.identity_digest, + DIGEST_LEN)) return -1; + signed_digest_len = crypto_pk_keysize(cert->signing_key); signed_digest = tor_malloc(signed_digest_len); if (crypto_pk_public_checksig(cert->signing_key, signed_digest, signed_digest_len, - voter->signature, - voter->signature_len) != DIGEST_LEN || - tor_memneq(signed_digest, consensus->networkstatus_digest, DIGEST_LEN)) { + sig->signature, + sig->signature_len) < dlen || + tor_memneq(signed_digest, consensus->digests.d[sig->alg], dlen)) { log_warn(LD_DIR, "Got a bad signature on a networkstatus vote"); - voter->bad_signature = 1; + sig->bad_signature = 1; } else { - voter->good_signature = 1; + sig->good_signature = 1; } tor_free(signed_digest); return 0; @@ -388,7 +465,7 @@ networkstatus_check_consensus_signature(networkstatus_t *consensus, int warn) { int n_good = 0; - int n_missing_key = 0; + int n_missing_key = 0, n_dl_failed_key = 0; int n_bad = 0; int n_unknown = 0; int n_no_signature = 0; @@ -402,37 +479,62 @@ networkstatus_check_consensus_signature(networkstatus_t *consensus, tor_assert(consensus->type == NS_TYPE_CONSENSUS); - SMARTLIST_FOREACH(consensus->voters, networkstatus_voter_info_t *, voter, - { - if (!voter->good_signature && !voter->bad_signature && voter->signature) { - /* we can try to check the signature. */ - int is_v3_auth = trusteddirserver_get_by_v3_auth_digest( - voter->identity_digest) != NULL; - authority_cert_t *cert = - authority_cert_get_by_digests(voter->identity_digest, - voter->signing_key_digest); - if (!is_v3_auth) { - smartlist_add(unrecognized, voter); - ++n_unknown; - continue; - } else if (!cert || cert->expires < now) { - smartlist_add(need_certs_from, voter); - ++n_missing_key; - continue; - } - if (networkstatus_check_voter_signature(consensus, voter, cert) < 0) { - smartlist_add(need_certs_from, voter); - ++n_missing_key; - continue; + SMARTLIST_FOREACH_BEGIN(consensus->voters, networkstatus_voter_info_t *, + voter) { + int good_here = 0; + int bad_here = 0; + int unknown_here = 0; + int missing_key_here = 0, dl_failed_key_here = 0; + SMARTLIST_FOREACH_BEGIN(voter->sigs, document_signature_t *, sig) { + if (!sig->good_signature && !sig->bad_signature && + sig->signature) { + /* we can try to check the signature. */ + int is_v3_auth = trusteddirserver_get_by_v3_auth_digest( + sig->identity_digest) != NULL; + authority_cert_t *cert = + authority_cert_get_by_digests(sig->identity_digest, + sig->signing_key_digest); + tor_assert(!memcmp(sig->identity_digest, voter->identity_digest, + DIGEST_LEN)); + + if (!is_v3_auth) { + smartlist_add(unrecognized, voter); + ++unknown_here; + continue; + } else if (!cert || cert->expires < now) { + smartlist_add(need_certs_from, voter); + ++missing_key_here; + if (authority_cert_dl_looks_uncertain(sig->identity_digest)) + ++dl_failed_key_here; + continue; + } + if (networkstatus_check_document_signature(consensus, sig, cert) < 0) { + smartlist_add(need_certs_from, voter); + ++missing_key_here; + if (authority_cert_dl_looks_uncertain(sig->identity_digest)) + ++dl_failed_key_here; + continue; + } } - } - if (voter->good_signature) + if (sig->good_signature) + ++good_here; + else if (sig->bad_signature) + ++bad_here; + } SMARTLIST_FOREACH_END(sig); + if (good_here) ++n_good; - else if (voter->bad_signature) + else if (bad_here) ++n_bad; - else + else if (missing_key_here) { + ++n_missing_key; + if (dl_failed_key_here) + ++n_dl_failed_key; + } else if (unknown_here) { + ++n_unknown; + } else { ++n_no_signature; - }); + } + } SMARTLIST_FOREACH_END(voter); /* Now see whether we're missing any voters entirely. */ SMARTLIST_FOREACH(router_get_trusted_dir_servers(), @@ -443,39 +545,71 @@ networkstatus_check_consensus_signature(networkstatus_t *consensus, smartlist_add(missing_authorities, ds); }); - if (warn > 1 || (warn >= 0 && n_good < n_required)) + if (warn > 1 || (warn >= 0 && + (n_good + n_missing_key - n_dl_failed_key < n_required))) { severity = LOG_WARN; - else + } else { severity = LOG_INFO; + } if (warn >= 0) { SMARTLIST_FOREACH(unrecognized, networkstatus_voter_info_t *, voter, { - log_info(LD_DIR, "Consensus includes unrecognized authority '%s' " - "at %s:%d (contact %s; identity %s)", + log(severity, LD_DIR, "Consensus includes unrecognized authority " + "'%s' at %s:%d (contact %s; identity %s)", voter->nickname, voter->address, (int)voter->dir_port, voter->contact?voter->contact:"n/a", hex_str(voter->identity_digest, DIGEST_LEN)); }); SMARTLIST_FOREACH(need_certs_from, networkstatus_voter_info_t *, voter, { - log_info(LD_DIR, "Looks like we need to download a new certificate " - "from authority '%s' at %s:%d (contact %s; identity %s)", + log(severity, LD_DIR, "Looks like we need to download a new " + "certificate from authority '%s' at %s:%d (contact %s; " + "identity %s)", voter->nickname, voter->address, (int)voter->dir_port, voter->contact?voter->contact:"n/a", hex_str(voter->identity_digest, DIGEST_LEN)); }); SMARTLIST_FOREACH(missing_authorities, trusted_dir_server_t *, ds, { - log_info(LD_DIR, "Consensus does not include configured " + log(severity, LD_DIR, "Consensus does not include configured " "authority '%s' at %s:%d (identity %s)", ds->nickname, ds->address, (int)ds->dir_port, hex_str(ds->v3_identity_digest, DIGEST_LEN)); }); - log(severity, LD_DIR, - "%d unknown, %d missing key, %d good, %d bad, %d no signature, " - "%d required", n_unknown, n_missing_key, n_good, n_bad, - n_no_signature, n_required); + { + smartlist_t *sl = smartlist_create(); + char *cp; + tor_asprintf(&cp, "A consensus needs %d good signatures from recognized " + "authorities for us to accept it. This one has %d.", + n_required, n_good); + smartlist_add(sl,cp); + if (n_no_signature) { + tor_asprintf(&cp, "%d of the authorities we know didn't sign it.", + n_no_signature); + smartlist_add(sl,cp); + } + if (n_unknown) { + tor_asprintf(&cp, "It has %d signatures from authorities we don't " + "recognize.", n_unknown); + smartlist_add(sl,cp); + } + if (n_bad) { + tor_asprintf(&cp, "%d of the signatures on it didn't verify " + "correctly.", n_bad); + smartlist_add(sl,cp); + } + if (n_missing_key) { + tor_asprintf(&cp, "We were unable to check %d of the signatures, " + "because we were missing the keys.", n_missing_key); + smartlist_add(sl,cp); + } + cp = smartlist_join_strings(sl, " ", 0, NULL); + log(severity, LD_DIR, "%s", cp); + tor_free(cp); + SMARTLIST_FOREACH(sl, char *, c, tor_free(c)); + smartlist_free(sl); + } } smartlist_free(unrecognized); @@ -785,8 +919,8 @@ networkstatus_v2_list_clean(time_t now) /** Helper for bsearching a list of routerstatus_t pointers: compare a * digest in the key to the identity digest of a routerstatus_t. */ -static int -_compare_digest_to_routerstatus_entry(const void *_key, const void **_member) +int +compare_digest_to_routerstatus_entry(const void *_key, const void **_member) { const char *key = _key; const routerstatus_t *rs = *_member; @@ -799,7 +933,7 @@ routerstatus_t * networkstatus_v2_find_entry(networkstatus_v2_t *ns, const char *digest) { return smartlist_bsearch(ns->entries, digest, - _compare_digest_to_routerstatus_entry); + compare_digest_to_routerstatus_entry); } /** Return the entry in <b>ns</b> for the identity digest <b>digest</b>, or @@ -808,7 +942,7 @@ routerstatus_t * networkstatus_vote_find_entry(networkstatus_t *ns, const char *digest) { return smartlist_bsearch(ns->routerstatus_list, digest, - _compare_digest_to_routerstatus_entry); + compare_digest_to_routerstatus_entry); } /*XXXX make this static once functions are moved into this file. */ @@ -820,7 +954,7 @@ networkstatus_vote_find_entry_idx(networkstatus_t *ns, const char *digest, int *found_out) { return smartlist_bsearch_idx(ns->routerstatus_list, digest, - _compare_digest_to_routerstatus_entry, + compare_digest_to_routerstatus_entry, found_out); } @@ -873,7 +1007,7 @@ router_get_consensus_status_by_id(const char *digest) if (!current_consensus) return NULL; return smartlist_bsearch(current_consensus->routerstatus_list, digest, - _compare_digest_to_routerstatus_entry); + compare_digest_to_routerstatus_entry); } /** Given a nickname (possibly verbose, possibly a hexadecimal digest), return @@ -1077,28 +1211,30 @@ update_v2_networkstatus_cache_downloads(time_t now) static void update_consensus_networkstatus_downloads(time_t now) { - or_options_t *options = get_options(); + int i; if (!networkstatus_get_live_consensus(now)) time_to_download_next_consensus = now; /* No live consensus? Get one now!*/ if (time_to_download_next_consensus > now) return; /* Wait until the current consensus is older. */ - if (authdir_mode_v3(options)) - return; /* Authorities never fetch a consensus */ - if (!download_status_is_ready(&consensus_dl_status, now, + /* XXXXNM Microdescs: may need to download more types. */ + if (!download_status_is_ready(&consensus_dl_status[FLAV_NS], now, CONSENSUS_NETWORKSTATUS_MAX_DL_TRIES)) return; /* We failed downloading a consensus too recently. */ if (connection_get_by_type_purpose(CONN_TYPE_DIR, DIR_PURPOSE_FETCH_CONSENSUS)) return; /* There's an in-progress download.*/ - if (consensus_waiting_for_certs) { - /* XXXX make sure this doesn't delay sane downloads. */ - if (consensus_waiting_for_certs_set_at + DELAY_WHILE_FETCHING_CERTS > now) - return; /* We're still getting certs for this one. */ - else { - if (!consensus_waiting_for_certs_dl_failed) { - download_status_failed(&consensus_dl_status, 0); - consensus_waiting_for_certs_dl_failed=1; + for (i=0; i < N_CONSENSUS_FLAVORS; ++i) { + consensus_waiting_for_certs_t *waiting = &consensus_waiting_for_certs[i]; + if (waiting->consensus) { + /* XXXX make sure this doesn't delay sane downloads. */ + if (waiting->set_at + DELAY_WHILE_FETCHING_CERTS > now) + return; /* We're still getting certs for this one. */ + else { + if (!waiting->dl_failed) { + download_status_failed(&consensus_dl_status[FLAV_NS], 0); + waiting->dl_failed=1; + } } } } @@ -1114,7 +1250,8 @@ update_consensus_networkstatus_downloads(time_t now) void networkstatus_consensus_download_failed(int status_code) { - download_status_failed(&consensus_dl_status, status_code); + /* XXXXNM Microdescs: may need to handle more types. */ + download_status_failed(&consensus_dl_status[FLAV_NS], status_code); /* Retry immediately, if appropriate. */ update_consensus_networkstatus_downloads(time(NULL)); } @@ -1148,8 +1285,15 @@ update_consensus_networkstatus_fetch_time(time_t now) /* We want to cache the next one at some point after this one * is no longer fresh... */ start = c->fresh_until + min_sec_before_caching; - /* But only in the first half-interval after that. */ - dl_interval = interval/2; + /* Some clients may need the consensus sooner than others. */ + if (options->FetchDirInfoExtraEarly || authdir_mode_v3(options)) { + dl_interval = 60; + if (min_sec_before_caching + dl_interval > interval) + dl_interval = interval/2; + } else { + /* But only in the first half-interval after that. */ + dl_interval = interval/2; + } } else { /* We're an ordinary client or a bridge. Give all the caches enough * time to download the consensus. */ @@ -1168,7 +1312,7 @@ update_consensus_networkstatus_fetch_time(time_t now) } if (dl_interval < 1) dl_interval = 1; - /* We must not try to replace c while it's still the most valid: */ + /* We must not try to replace c while it's still fresh: */ tor_assert(c->fresh_until < start); /* We must download the next one before c is invalid: */ tor_assert(start+dl_interval < c->valid_until); @@ -1189,7 +1333,6 @@ update_consensus_networkstatus_fetch_time(time_t now) time_to_download_next_consensus = now; log_info(LD_DIR, "No live consensus; we should fetch one immediately."); } - } /** Return 1 if there's a reason we shouldn't try any directory @@ -1213,7 +1356,7 @@ update_networkstatus_downloads(time_t now) or_options_t *options = get_options(); if (should_delay_dir_fetches(options)) return; - if (directory_fetches_dir_info_early(options)) + if (authdir_mode_any_main(options) || options->FetchV2Networkstatus) update_v2_networkstatus_cache_downloads(now); update_consensus_networkstatus_downloads(now); update_certificate_downloads(now); @@ -1224,10 +1367,14 @@ update_networkstatus_downloads(time_t now) void update_certificate_downloads(time_t now) { - if (consensus_waiting_for_certs) - authority_certs_fetch_missing(consensus_waiting_for_certs, now); - else - authority_certs_fetch_missing(current_consensus, now); + int i; + for (i = 0; i < N_CONSENSUS_FLAVORS; ++i) { + if (consensus_waiting_for_certs[i].consensus) + authority_certs_fetch_missing(consensus_waiting_for_certs[i].consensus, + now); + } + + authority_certs_fetch_missing(current_consensus, now); } /** Return 1 if we have a consensus but we don't have enough certificates @@ -1235,7 +1382,8 @@ update_certificate_downloads(time_t now) int consensus_is_waiting_for_certs(void) { - return consensus_waiting_for_certs ? 1 : 0; + return consensus_waiting_for_certs[USABLE_CONSENSUS_FLAVOR].consensus + ? 1 : 0; } /** Return the network status with a given identity digest. */ @@ -1404,16 +1552,31 @@ networkstatus_copy_old_consensus_info(networkstatus_t *new_c, * user, and -2 for more serious problems. */ int -networkstatus_set_current_consensus(const char *consensus, unsigned flags) +networkstatus_set_current_consensus(const char *consensus, + const char *flavor, + unsigned flags) { - networkstatus_t *c; + networkstatus_t *c=NULL; int r, result = -1; time_t now = time(NULL); + or_options_t *options = get_options(); char *unverified_fname = NULL, *consensus_fname = NULL; + int flav = networkstatus_parse_flavor_name(flavor); const unsigned from_cache = flags & NSSET_FROM_CACHE; const unsigned was_waiting_for_certs = flags & NSSET_WAS_WAITING_FOR_CERTS; const unsigned dl_certs = !(flags & NSSET_DONT_DOWNLOAD_CERTS); const unsigned accept_obsolete = flags & NSSET_ACCEPT_OBSOLETE; + const unsigned require_flavor = flags & NSSET_REQUIRE_FLAVOR; + const digests_t *current_digests = NULL; + consensus_waiting_for_certs_t *waiting = NULL; + time_t current_valid_after = 0; + int free_consensus = 1; /* Free 'c' at the end of the function */ + + if (flav < 0) { + /* XXXX we don't handle unrecognized flavors yet. */ + log_warn(LD_BUG, "Unrecognized consensus flavor %s", flavor); + return -2; + } /* Make sure it's parseable. */ c = networkstatus_parse_vote_from_string(consensus, NULL, NS_TYPE_CONSENSUS); @@ -1423,33 +1586,69 @@ networkstatus_set_current_consensus(const char *consensus, unsigned flags) goto done; } + if ((int)c->flavor != flav) { + /* This wasn't the flavor we thought we were getting. */ + if (require_flavor) { + log_warn(LD_DIR, "Got consensus with unexpected flavor %s (wanted %s)", + networkstatus_get_flavor_name(c->flavor), flavor); + goto done; + } + flav = c->flavor; + flavor = networkstatus_get_flavor_name(flav); + } + + if (flav != USABLE_CONSENSUS_FLAVOR && + !directory_caches_dir_info(options)) { + /* This consensus is totally boring to us: we won't use it, and we won't + * serve it. Drop it. */ + goto done; + } + if (from_cache && !accept_obsolete && c->valid_until < now-OLD_ROUTER_DESC_MAX_AGE) { - /* XXX022 when we try to make fallbackconsensus work again, we should + /* XXXX If we try to make fallbackconsensus work again, we should * consider taking this out. Until then, believing obsolete consensuses * is causing more harm than good. See also bug 887. */ - log_info(LD_DIR, "Loaded an obsolete consensus. Discarding."); + log_info(LD_DIR, "Loaded an expired consensus. Discarding."); goto done; } - if (current_consensus && - tor_memeq(c->networkstatus_digest, current_consensus->networkstatus_digest, - DIGEST_LEN)) { + if (!strcmp(flavor, "ns")) { + consensus_fname = get_datadir_fname("cached-consensus"); + unverified_fname = get_datadir_fname("unverified-consensus"); + if (current_consensus) { + current_digests = ¤t_consensus->digests; + current_valid_after = current_consensus->valid_after; + } + } else { + cached_dir_t *cur; + char buf[128]; + tor_snprintf(buf, sizeof(buf), "cached-%s-consensus", flavor); + consensus_fname = get_datadir_fname(buf); + tor_snprintf(buf, sizeof(buf), "unverified-%s-consensus", flavor); + unverified_fname = get_datadir_fname(buf); + cur = dirserv_get_consensus(flavor); + if (cur) { + current_digests = &cur->digests; + current_valid_after = cur->published; + } + } + + if (current_digests && + tor_memeq(&c->digests, current_digests, sizeof(c->digests))) { /* We already have this one. That's a failure. */ - log_info(LD_DIR, "Got a consensus we already have"); + log_info(LD_DIR, "Got a %s consensus we already have", flavor); goto done; } - if (current_consensus && c->valid_after <= current_consensus->valid_after) { + if (current_valid_after && c->valid_after <= current_valid_after) { /* We have a newer one. There's no point in accepting this one, * even if it's great. */ - log_info(LD_DIR, "Got a consensus at least as old as the one we have"); + log_info(LD_DIR, "Got a %s consensus at least as old as the one we have", + flavor); goto done; } - consensus_fname = get_datadir_fname("cached-consensus"); - unverified_fname = get_datadir_fname("unverified-consensus"); - /* Make sure it's signed enough. */ if ((r=networkstatus_check_consensus_signature(c, 1))<0) { if (r == -1) { @@ -1458,16 +1657,16 @@ networkstatus_set_current_consensus(const char *consensus, unsigned flags) log_info(LD_DIR, "Not enough certificates to check networkstatus consensus"); } - if (!current_consensus || - c->valid_after > current_consensus->valid_after) { - if (consensus_waiting_for_certs) - networkstatus_vote_free(consensus_waiting_for_certs); - tor_free(consensus_waiting_for_certs_body); - consensus_waiting_for_certs = c; - c = NULL; /* Prevent free. */ - consensus_waiting_for_certs_body = tor_strdup(consensus); - consensus_waiting_for_certs_set_at = now; - consensus_waiting_for_certs_dl_failed = 0; + if (!current_valid_after || + c->valid_after > current_valid_after) { + waiting = &consensus_waiting_for_certs[flav]; + networkstatus_vote_free(waiting->consensus); + tor_free(waiting->body); + waiting->consensus = c; + free_consensus = 0; + waiting->body = tor_strdup(consensus); + waiting->set_at = now; + waiting->dl_failed = 0; if (!from_cache) { write_str_to_file(unverified_fname, consensus, 0); } @@ -1496,57 +1695,81 @@ networkstatus_set_current_consensus(const char *consensus, unsigned flags) } } - if (!from_cache) + if (!from_cache && flav == USABLE_CONSENSUS_FLAVOR) control_event_client_status(LOG_NOTICE, "CONSENSUS_ARRIVED"); /* Are we missing any certificates at all? */ if (r != 1 && dl_certs) authority_certs_fetch_missing(c, now); - notify_control_networkstatus_changed(current_consensus, c); + if (flav == USABLE_CONSENSUS_FLAVOR) { + notify_control_networkstatus_changed(current_consensus, c); - if (current_consensus) { - networkstatus_copy_old_consensus_info(c, current_consensus); - networkstatus_vote_free(current_consensus); + if (current_consensus) { + networkstatus_copy_old_consensus_info(c, current_consensus); + networkstatus_vote_free(current_consensus); + /* Defensive programming : we should set current_consensus very soon, + * but we're about to call some stuff in the meantime, and leaving this + * dangling pointer around has proven to be trouble. */ + current_consensus = NULL; + } } - if (consensus_waiting_for_certs && - consensus_waiting_for_certs->valid_after <= c->valid_after) { - networkstatus_vote_free(consensus_waiting_for_certs); - consensus_waiting_for_certs = NULL; - if (consensus != consensus_waiting_for_certs_body) - tor_free(consensus_waiting_for_certs_body); + waiting = &consensus_waiting_for_certs[flav]; + if (waiting->consensus && + waiting->consensus->valid_after <= c->valid_after) { + networkstatus_vote_free(waiting->consensus); + waiting->consensus = NULL; + if (consensus != waiting->body) + tor_free(waiting->body); else - consensus_waiting_for_certs_body = NULL; - consensus_waiting_for_certs_set_at = 0; - consensus_waiting_for_certs_dl_failed = 0; + waiting->body = NULL; + waiting->set_at = 0; + waiting->dl_failed = 0; unlink(unverified_fname); } /* Reset the failure count only if this consensus is actually valid. */ if (c->valid_after <= now && now <= c->valid_until) { - download_status_reset(&consensus_dl_status); + download_status_reset(&consensus_dl_status[flav]); } else { if (!from_cache) - download_status_failed(&consensus_dl_status, 0); + download_status_failed(&consensus_dl_status[flav], 0); } - current_consensus = c; - c = NULL; /* Prevent free. */ + if (flav == USABLE_CONSENSUS_FLAVOR) { + current_consensus = c; + free_consensus = 0; /* Prevent free. */ + + /* XXXXNM Microdescs: needs a non-ns variant. */ + update_consensus_networkstatus_fetch_time(now); + dirvote_recalculate_timing(options, now); + routerstatus_list_update_named_server_map(); + cell_ewma_set_scale_factor(options, current_consensus); - update_consensus_networkstatus_fetch_time(now); - dirvote_recalculate_timing(get_options(), now); - routerstatus_list_update_named_server_map(); + /* XXXX023 this call might be unnecessary here: can changing the + * current consensus really alter our view of any OR's rate limits? */ + connection_or_update_token_buckets(get_connection_array(), options); + + circuit_build_times_new_consensus_params(&circ_times, current_consensus); + } + + if (directory_caches_dir_info(options)) { + dirserv_set_cached_consensus_networkstatus(consensus, + flavor, + &c->digests, + c->valid_after); + } if (!from_cache) { write_str_to_file(consensus_fname, consensus, 0); } - if (directory_caches_dir_info(get_options())) - dirserv_set_cached_networkstatus_v3(consensus, - current_consensus->valid_after); +/** If a consensus appears more than this many seconds before its declared + * valid-after time, declare that our clock is skewed. */ +#define EARLY_CONSENSUS_NOTICE_SKEW 60 - if (ftime_definitely_before(now, current_consensus->valid_after)) { + if (now < current_consensus->valid_after - EARLY_CONSENSUS_NOTICE_SKEW) { char tbuf[ISO_TIME_LEN+1]; char dbuf[64]; long delta = now - current_consensus->valid_after; @@ -1564,7 +1787,7 @@ networkstatus_set_current_consensus(const char *consensus, unsigned flags) result = 0; done: - if (c) + if (free_consensus) networkstatus_vote_free(c); tor_free(consensus_fname); tor_free(unverified_fname); @@ -1576,13 +1799,17 @@ networkstatus_set_current_consensus(const char *consensus, unsigned flags) void networkstatus_note_certs_arrived(void) { - if (consensus_waiting_for_certs) { - if (networkstatus_check_consensus_signature( - consensus_waiting_for_certs, 0)>=0) { + int i; + for (i=0; i<N_CONSENSUS_FLAVORS; ++i) { + consensus_waiting_for_certs_t *waiting = &consensus_waiting_for_certs[i]; + if (!waiting->consensus) + continue; + if (networkstatus_check_consensus_signature(waiting->consensus, 0)>=0) { if (!networkstatus_set_current_consensus( - consensus_waiting_for_certs_body, + waiting->body, + networkstatus_get_flavor_name(i), NSSET_WAS_WAITING_FOR_CERTS)) { - tor_free(consensus_waiting_for_certs_body); + tor_free(waiting->body); } } } @@ -1668,10 +1895,8 @@ download_status_map_update_from_v2_networkstatus(void) v2_download_status_map = digestmap_new(); dl_status = digestmap_new(); - SMARTLIST_FOREACH(networkstatus_v2_list, networkstatus_v2_t *, ns, - { - SMARTLIST_FOREACH(ns->entries, routerstatus_t *, rs, - { + SMARTLIST_FOREACH_BEGIN(networkstatus_v2_list, networkstatus_v2_t *, ns) { + SMARTLIST_FOREACH_BEGIN(ns->entries, routerstatus_t *, rs) { const char *d = rs->descriptor_digest; download_status_t *s; if (digestmap_get(dl_status, d)) @@ -1680,8 +1905,8 @@ download_status_map_update_from_v2_networkstatus(void) s = tor_malloc_zero(sizeof(download_status_t)); } digestmap_set(dl_status, d, s); - }); - }); + } SMARTLIST_FOREACH_END(rs); + } SMARTLIST_FOREACH_END(ns); digestmap_free(v2_download_status_map, _tor_free); v2_download_status_map = dl_status; networkstatus_v2_list_has_changed = 0; @@ -1695,11 +1920,9 @@ routerstatus_list_update_named_server_map(void) if (!current_consensus) return; - if (named_server_map) - strmap_free(named_server_map, _tor_free); + strmap_free(named_server_map, _tor_free); named_server_map = strmap_new(); - if (unnamed_server_map) - strmap_free(unnamed_server_map, NULL); + strmap_free(unnamed_server_map, NULL); unnamed_server_map = strmap_new(); SMARTLIST_FOREACH(current_consensus->routerstatus_list, routerstatus_t *, rs, { @@ -1774,6 +1997,15 @@ routers_update_status_from_consensus_networkstatus(smartlist_t *routers, router->is_bad_directory = rs->is_bad_directory; router->is_bad_exit = rs->is_bad_exit; router->is_hs_dir = rs->is_hs_dir; + } else { + /* If we _are_ an authority, we should check whether this router + * is one that will cause us to need a reachability test. */ + routerinfo_t *old_router = + router_get_by_digest(router->cache_info.identity_digest); + if (old_router != router) { + router->needs_retest_if_added = + dirserv_should_launch_reachability_test(router, old_router); + } } if (router->is_running && ds) { download_status_reset(&ds->v2_ns_dl_status); @@ -1836,7 +2068,7 @@ char * networkstatus_getinfo_helper_single(routerstatus_t *rs) { char buf[RS_ENTRY_LEN+1]; - routerstatus_format_entry(buf, sizeof(buf), rs, NULL, 0, 1); + routerstatus_format_entry(buf, sizeof(buf), rs, NULL, NS_CONTROL_PORT); return tor_strdup(buf); } @@ -1873,7 +2105,7 @@ networkstatus_getinfo_by_purpose(const char *purpose_string, time_t now) if (bridge_auth && ri->purpose == ROUTER_PURPOSE_BRIDGE) dirserv_set_router_is_running(ri, now); /* then generate and write out status lines for each of them */ - set_routerstatus_from_routerinfo(&rs, ri, now, 0, 0, 0, 0); + set_routerstatus_from_routerinfo(&rs, ri, now, 0, 0, 0); smartlist_add(statuses, networkstatus_getinfo_helper_single(&rs)); }); @@ -1898,35 +2130,118 @@ networkstatus_dump_bridge_status_to_file(time_t now) tor_free(status); } +static int32_t +get_net_param_from_list(smartlist_t *net_params, const char *param_name, + int32_t default_val, int32_t min_val, int32_t max_val) +{ + int32_t res = default_val; + size_t name_len = strlen(param_name); + + tor_assert(max_val > min_val); + tor_assert(min_val <= default_val); + tor_assert(max_val >= default_val); + + SMARTLIST_FOREACH_BEGIN(net_params, const char *, p) { + if (!strcmpstart(p, param_name) && p[name_len] == '=') { + int ok=0; + long v = tor_parse_long(p+name_len+1, 10, INT32_MIN, + INT32_MAX, &ok, NULL); + if (ok) { + res = (int32_t) v; + break; + } + } + } SMARTLIST_FOREACH_END(p); + + if (res < min_val) { + log_warn(LD_DIR, "Consensus parameter %s is too small. Got %d, raising to " + "%d.", param_name, res, min_val); + res = min_val; + } else if (res > max_val) { + log_warn(LD_DIR, "Consensus parameter %s is too large. Got %d, capping to " + "%d.", param_name, res, max_val); + res = max_val; + } + + return res; +} + /** Return the value of a integer parameter from the networkstatus <b>ns</b> * whose name is <b>param_name</b>. If <b>ns</b> is NULL, try loading the * latest consensus ourselves. Return <b>default_val</b> if no latest - * consensus, or if it has no parameter called <b>param_name</b>. */ + * consensus, or if it has no parameter called <b>param_name</b>. + * Make sure the value parsed from the consensus is at least + * <b>min_val</b> and at most <b>max_val</b> and raise/cap the parsed value + * if necessary. */ int32_t networkstatus_get_param(networkstatus_t *ns, const char *param_name, - int32_t default_val) + int32_t default_val, int32_t min_val, int32_t max_val) { - size_t name_len; - if (!ns) /* if they pass in null, go find it ourselves */ ns = networkstatus_get_latest_consensus(); if (!ns || !ns->net_params) return default_val; - name_len = strlen(param_name); + return get_net_param_from_list(ns->net_params, param_name, + default_val, min_val, max_val); +} - SMARTLIST_FOREACH_BEGIN(ns->net_params, const char *, p) { - if (!strcmpstart(p, param_name) && p[name_len] == '=') { - int ok=0; - long v = tor_parse_long(p+name_len+1, 10, INT32_MIN, INT32_MAX, &ok, - NULL); - if (ok) - return (int32_t) v; - } - } SMARTLIST_FOREACH_END(p); +/** Return the value of a integer bw weight parameter from the networkstatus + * <b>ns</b> whose name is <b>weight_name</b>. If <b>ns</b> is NULL, try + * loading the latest consensus ourselves. Return <b>default_val</b> if no + * latest consensus, or if it has no parameter called <b>weight_name</b>. */ +int32_t +networkstatus_get_bw_weight(networkstatus_t *ns, const char *weight_name, + int32_t default_val) +{ + int32_t param; + int max; + if (!ns) /* if they pass in null, go find it ourselves */ + ns = networkstatus_get_latest_consensus(); + + if (!ns || !ns->weight_params) + return default_val; + + max = circuit_build_times_get_bw_scale(ns); + param = get_net_param_from_list(ns->weight_params, weight_name, + default_val, -1, + BW_MAX_WEIGHT_SCALE); + if (param > max) { + log_warn(LD_DIR, "Value of consensus weight %s was too large, capping " + "to %d", weight_name, max); + param = max; + } + return param; +} + +/** Return the name of the consensus flavor <b>flav</b> as used to identify + * the flavor in directory documents. */ +const char * +networkstatus_get_flavor_name(consensus_flavor_t flav) +{ + switch (flav) { + case FLAV_NS: + return "ns"; + case FLAV_MICRODESC: + return "microdesc"; + default: + tor_fragile_assert(); + return "??"; + } +} - return default_val; +/** Return the consensus_flavor_t value for the flavor called <b>flavname</b>, + * or -1 if the flavor is not recognized. */ +int +networkstatus_parse_flavor_name(const char *flavname) +{ + if (!strcmp(flavname, "ns")) + return FLAV_NS; + else if (!strcmp(flavname, "microdesc")) + return FLAV_MICRODESC; + else + return -1; } /** If <b>question</b> is a string beginning with "ns/" in a format the @@ -1935,7 +2250,8 @@ networkstatus_get_param(networkstatus_t *ns, const char *param_name, * ORs. Return 0 on success, -1 on unrecognized question format. */ int getinfo_helper_networkstatus(control_connection_t *conn, - const char *question, char **answer) + const char *question, char **answer, + const char **errmsg) { routerstatus_t *status; (void) conn; @@ -1959,8 +2275,10 @@ getinfo_helper_networkstatus(control_connection_t *conn, } else if (!strcmpstart(question, "ns/id/")) { char d[DIGEST_LEN]; - if (base16_decode(d, DIGEST_LEN, question+6, strlen(question+6))) + if (base16_decode(d, DIGEST_LEN, question+6, strlen(question+6))) { + *errmsg = "Data not decodeable as hex"; return -1; + } status = router_get_consensus_status_by_id(d); } else if (!strcmpstart(question, "ns/name/")) { status = router_get_consensus_status_by_nickname(question+8, 0); @@ -1968,7 +2286,7 @@ getinfo_helper_networkstatus(control_connection_t *conn, *answer = networkstatus_getinfo_by_purpose(question+11, time(NULL)); return *answer ? 0 : -1; } else { - return -1; + return 0; } if (status) @@ -1980,30 +2298,29 @@ getinfo_helper_networkstatus(control_connection_t *conn, void networkstatus_free_all(void) { + int i; if (networkstatus_v2_list) { SMARTLIST_FOREACH(networkstatus_v2_list, networkstatus_v2_t *, ns, networkstatus_v2_free(ns)); smartlist_free(networkstatus_v2_list); networkstatus_v2_list = NULL; } - if (v2_download_status_map) { - digestmap_free(v2_download_status_map, _tor_free); - v2_download_status_map = NULL; - } - if (current_consensus) { - networkstatus_vote_free(current_consensus); - current_consensus = NULL; - } - if (consensus_waiting_for_certs) { - networkstatus_vote_free(consensus_waiting_for_certs); - consensus_waiting_for_certs = NULL; - } - tor_free(consensus_waiting_for_certs_body); - if (named_server_map) { - strmap_free(named_server_map, _tor_free); - } - if (unnamed_server_map) { - strmap_free(unnamed_server_map, NULL); + + digestmap_free(v2_download_status_map, _tor_free); + v2_download_status_map = NULL; + networkstatus_vote_free(current_consensus); + current_consensus = NULL; + + for (i=0; i < N_CONSENSUS_FLAVORS; ++i) { + consensus_waiting_for_certs_t *waiting = &consensus_waiting_for_certs[i]; + if (waiting->consensus) { + networkstatus_vote_free(waiting->consensus); + waiting->consensus = NULL; + } + tor_free(waiting->body); } + + strmap_free(named_server_map, _tor_free); + strmap_free(unnamed_server_map, NULL); } diff --git a/src/or/networkstatus.h b/src/or/networkstatus.h new file mode 100644 index 0000000000..ec2e8f884d --- /dev/null +++ b/src/or/networkstatus.h @@ -0,0 +1,99 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file networkstatus.h + * \brief Header file for networkstatus.c. + **/ + +#ifndef _TOR_NETWORKSTATUS_H +#define _TOR_NETWORKSTATUS_H + +/** How old do we allow a v2 network-status to get before removing it + * completely? */ +#define MAX_NETWORKSTATUS_AGE (10*24*60*60) + +void networkstatus_reset_warnings(void); +void networkstatus_reset_download_failures(void); +int router_reload_v2_networkstatus(void); +int router_reload_consensus_networkstatus(void); +void routerstatus_free(routerstatus_t *rs); +void networkstatus_v2_free(networkstatus_v2_t *ns); +void networkstatus_vote_free(networkstatus_t *ns); +networkstatus_voter_info_t *networkstatus_get_voter_by_id( + networkstatus_t *vote, + const char *identity); +int networkstatus_check_consensus_signature(networkstatus_t *consensus, + int warn); +int networkstatus_check_document_signature(const networkstatus_t *consensus, + document_signature_t *sig, + const authority_cert_t *cert); +char *networkstatus_get_cache_filename(const char *identity_digest); +int router_set_networkstatus_v2(const char *s, time_t arrived_at, + v2_networkstatus_source_t source, + smartlist_t *requested_fingerprints); +void networkstatus_v2_list_clean(time_t now); +int compare_digest_to_routerstatus_entry(const void *_key, + const void **_member); +routerstatus_t *networkstatus_v2_find_entry(networkstatus_v2_t *ns, + const char *digest); +routerstatus_t *networkstatus_vote_find_entry(networkstatus_t *ns, + const char *digest); +int networkstatus_vote_find_entry_idx(networkstatus_t *ns, + const char *digest, int *found_out); +const smartlist_t *networkstatus_get_v2_list(void); +download_status_t *router_get_dl_status_by_descriptor_digest(const char *d); +routerstatus_t *router_get_consensus_status_by_id(const char *digest); +routerstatus_t *router_get_consensus_status_by_descriptor_digest( + const char *digest); +routerstatus_t *router_get_consensus_status_by_nickname(const char *nickname, + int warn_if_unnamed); +const char *networkstatus_get_router_digest_by_nickname(const char *nickname); +int networkstatus_nickname_is_unnamed(const char *nickname); +void networkstatus_consensus_download_failed(int status_code); +void update_consensus_networkstatus_fetch_time(time_t now); +int should_delay_dir_fetches(or_options_t *options); +void update_networkstatus_downloads(time_t now); +void update_certificate_downloads(time_t now); +int consensus_is_waiting_for_certs(void); +networkstatus_v2_t *networkstatus_v2_get_by_digest(const char *digest); +networkstatus_t *networkstatus_get_latest_consensus(void); +networkstatus_t *networkstatus_get_live_consensus(time_t now); +networkstatus_t *networkstatus_get_reasonably_live_consensus(time_t now); +#define NSSET_FROM_CACHE 1 +#define NSSET_WAS_WAITING_FOR_CERTS 2 +#define NSSET_DONT_DOWNLOAD_CERTS 4 +#define NSSET_ACCEPT_OBSOLETE 8 +#define NSSET_REQUIRE_FLAVOR 16 +int networkstatus_set_current_consensus(const char *consensus, + const char *flavor, + unsigned flags); +void networkstatus_note_certs_arrived(void); +void routers_update_all_from_networkstatus(time_t now, int dir_version); +void routers_update_status_from_consensus_networkstatus(smartlist_t *routers, + int reset_failures); +void signed_descs_update_status_from_consensus_networkstatus( + smartlist_t *descs); + +char *networkstatus_getinfo_helper_single(routerstatus_t *rs); +char *networkstatus_getinfo_by_purpose(const char *purpose_string, time_t now); +void networkstatus_dump_bridge_status_to_file(time_t now); +int32_t networkstatus_get_param(networkstatus_t *ns, const char *param_name, + int32_t default_val, int32_t min_val, + int32_t max_val); +int getinfo_helper_networkstatus(control_connection_t *conn, + const char *question, char **answer, + const char **errmsg); +int32_t networkstatus_get_bw_weight(networkstatus_t *ns, const char *weight, + int32_t default_val); +const char *networkstatus_get_flavor_name(consensus_flavor_t flav); +int networkstatus_parse_flavor_name(const char *flavname); +void document_signature_free(document_signature_t *sig); +document_signature_t *document_signature_dup(const document_signature_t *sig); +void networkstatus_free_all(void); + +#endif + diff --git a/src/or/ntmain.c b/src/or/ntmain.c index 4dcf9b3763..b2fee648cc 100644 --- a/src/or/ntmain.c +++ b/src/or/ntmain.c @@ -5,13 +5,22 @@ #define MAIN_PRIVATE #include "or.h" - -#include <tchar.h> -#define GENSRV_SERVICENAME TEXT("tor") -#define GENSRV_DISPLAYNAME TEXT("Tor Win32 Service") +#include "config.h" +#include "main.h" +#include "ntmain.h" + +#ifdef HAVE_EVENT2_EVENT_H +#include <event2/event.h> +#else +#include <event.h> +#endif + +#include <windows.h> +#define GENSRV_SERVICENAME "tor" +#define GENSRV_DISPLAYNAME "Tor Win32 Service" #define GENSRV_DESCRIPTION \ - TEXT("Provides an anonymous Internet communication system") -#define GENSRV_USERACCT TEXT("NT AUTHORITY\\LocalService") + "Provides an anonymous Internet communication system" +#define GENSRV_USERACCT "NT AUTHORITY\\LocalService" // Cheating: using the pre-defined error codes, tricks Windows into displaying // a semi-related human-readable error message if startup fails as @@ -27,7 +36,6 @@ static SERVICE_STATUS_HANDLE hStatus; * to the NT service functions. */ static char **backup_argv; static int backup_argc; -static char* nt_strerror(uint32_t errnum); static void nt_service_control(DWORD request); static void nt_service_body(int argc, char **argv); @@ -46,6 +54,11 @@ static int nt_service_cmd_stop(void); struct service_fns { int loaded; + /** @{ */ + /** Function pointers for Windows API functions related to service + * management. These are NULL, or they point to the . They're set by + * calling the LOAD macro below. */ + BOOL (WINAPI *ChangeServiceConfig2A_fn)( SC_HANDLE hService, DWORD dwInfoLevel, @@ -61,30 +74,30 @@ struct service_fns { SC_HANDLE (WINAPI *CreateServiceA_fn)( SC_HANDLE hSCManager, - LPCTSTR lpServiceName, - LPCTSTR lpDisplayName, + LPCSTR lpServiceName, + LPCSTR lpDisplayName, DWORD dwDesiredAccess, DWORD dwServiceType, DWORD dwStartType, DWORD dwErrorControl, - LPCTSTR lpBinaryPathName, - LPCTSTR lpLoadOrderGroup, + LPCSTR lpBinaryPathName, + LPCSTR lpLoadOrderGroup, LPDWORD lpdwTagId, - LPCTSTR lpDependencies, - LPCTSTR lpServiceStartName, - LPCTSTR lpPassword); + LPCSTR lpDependencies, + LPCSTR lpServiceStartName, + LPCSTR lpPassword); BOOL (WINAPI *DeleteService_fn)( SC_HANDLE hService); SC_HANDLE (WINAPI *OpenSCManagerA_fn)( - LPCTSTR lpMachineName, - LPCTSTR lpDatabaseName, + LPCSTR lpMachineName, + LPCSTR lpDatabaseName, DWORD dwDesiredAccess); SC_HANDLE (WINAPI *OpenServiceA_fn)( SC_HANDLE hSCManager, - LPCTSTR lpServiceName, + LPCSTR lpServiceName, DWORD dwDesiredAccess); BOOL (WINAPI *QueryServiceStatus_fn)( @@ -92,28 +105,29 @@ struct service_fns { LPSERVICE_STATUS lpServiceStatus); SERVICE_STATUS_HANDLE (WINAPI *RegisterServiceCtrlHandlerA_fn)( - LPCTSTR lpServiceName, + LPCSTR lpServiceName, LPHANDLER_FUNCTION lpHandlerProc); BOOL (WINAPI *SetServiceStatus_fn)(SERVICE_STATUS_HANDLE, LPSERVICE_STATUS); BOOL (WINAPI *StartServiceCtrlDispatcherA_fn)( - const SERVICE_TABLE_ENTRY* lpServiceTable); + const SERVICE_TABLE_ENTRYA* lpServiceTable); BOOL (WINAPI *StartServiceA_fn)( SC_HANDLE hService, DWORD dwNumServiceArgs, - LPCTSTR* lpServiceArgVectors); + LPCSTR* lpServiceArgVectors); BOOL (WINAPI *LookupAccountNameA_fn)( - LPCTSTR lpSystemName, - LPCTSTR lpAccountName, + LPCSTR lpSystemName, + LPCSTR lpAccountName, PSID Sid, LPDWORD cbSid, LPTSTR ReferencedDomainName, LPDWORD cchReferencedDomainName, PSID_NAME_USE peUse); + /** @} */ } service_fns = { 0, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, @@ -130,13 +144,16 @@ nt_service_loadlibrary(void) if (service_fns.loaded) return; - /* XXXX Possibly, we should hardcode the location of this DLL. */ - if (!(library = LoadLibrary("advapi32.dll"))) { + if (!(library = load_windows_system_library(TEXT("advapi32.dll")))) { log_err(LD_GENERAL, "Couldn't open advapi32.dll. Are you trying to use " "NT services on Windows 98? That doesn't work."); goto err; } +/* Helper macro: try to load a function named <b>f</b> from "library" into + * service_functions.<b>f</b>_fn. On failure, log an error message, and goto + * err. + */ #define LOAD(f) STMT_BEGIN \ if (!(fn = GetProcAddress(library, #f))) { \ log_err(LD_BUG, \ @@ -218,7 +235,7 @@ nt_service_control(DWORD request) log_notice(LD_GENERAL, "Got stop/shutdown request; shutting down cleanly."); service_status.dwCurrentState = SERVICE_STOP_PENDING; - event_loopexit(&exit_now); + event_base_loopexit(tor_libevent_get_base(), &exit_now); return; } service_fns.SetServiceStatus_fn(hStatus, &service_status); @@ -275,20 +292,20 @@ nt_service_body(int argc, char **argv) static void nt_service_main(void) { - SERVICE_TABLE_ENTRY table[2]; + SERVICE_TABLE_ENTRYA table[2]; DWORD result = 0; char *errmsg; nt_service_loadlibrary(); table[0].lpServiceName = (char*)GENSRV_SERVICENAME; - table[0].lpServiceProc = (LPSERVICE_MAIN_FUNCTION)nt_service_body; + table[0].lpServiceProc = (LPSERVICE_MAIN_FUNCTIONA)nt_service_body; table[1].lpServiceName = NULL; table[1].lpServiceProc = NULL; if (!service_fns.StartServiceCtrlDispatcherA_fn(table)) { result = GetLastError(); - errmsg = nt_strerror(result); + errmsg = format_win32_error(result); printf("Service error %d : %s\n", (int) result, errmsg); - LocalFree(errmsg); + tor_free(errmsg); if (result == ERROR_FAILED_SERVICE_CONTROLLER_CONNECT) { if (tor_init(backup_argc, backup_argv) < 0) return; @@ -323,9 +340,9 @@ nt_service_open_scm(void) nt_service_loadlibrary(); if ((hSCManager = service_fns.OpenSCManagerA_fn( NULL, NULL, SC_MANAGER_CREATE_SERVICE)) == NULL) { - errmsg = nt_strerror(GetLastError()); + errmsg = format_win32_error(GetLastError()); printf("OpenSCManager() failed : %s\n", errmsg); - LocalFree(errmsg); + tor_free(errmsg); } return hSCManager; } @@ -340,9 +357,9 @@ nt_service_open(SC_HANDLE hSCManager) nt_service_loadlibrary(); if ((hService = service_fns.OpenServiceA_fn(hSCManager, GENSRV_SERVICENAME, SERVICE_ALL_ACCESS)) == NULL) { - errmsg = nt_strerror(GetLastError()); + errmsg = format_win32_error(GetLastError()); printf("OpenService() failed : %s\n", errmsg); - LocalFree(errmsg); + tor_free(errmsg); } return hService; } @@ -374,14 +391,14 @@ nt_service_start(SC_HANDLE hService) printf("Service started successfully\n"); return 0; } else { - errmsg = nt_strerror(service_status.dwWin32ExitCode); + errmsg = format_win32_error(service_status.dwWin32ExitCode); printf("Service failed to start : %s\n", errmsg); - LocalFree(errmsg); + tor_free(errmsg); } } else { - errmsg = nt_strerror(GetLastError()); + errmsg = format_win32_error(GetLastError()); printf("StartService() failed : %s\n", errmsg); - LocalFree(errmsg); + tor_free(errmsg); } return -1; } @@ -418,14 +435,14 @@ nt_service_stop(SC_HANDLE hService) } else if (wait_time == MAX_SERVICE_WAIT_TIME) { printf("Service did not stop within %d seconds.\n", wait_time); } else { - errmsg = nt_strerror(GetLastError()); + errmsg = format_win32_error(GetLastError()); printf("QueryServiceStatus() failed : %s\n",errmsg); - LocalFree(errmsg); + tor_free(errmsg); } } else { - errmsg = nt_strerror(GetLastError()); + errmsg = format_win32_error(GetLastError()); printf("ControlService() failed : %s\n", errmsg); - LocalFree(errmsg); + tor_free(errmsg); } return -1; } @@ -439,6 +456,7 @@ static char * nt_service_command_line(int *using_default_torrc) { TCHAR tor_exe[MAX_PATH+1]; + char tor_exe_ascii[MAX_PATH+1]; char *command, *options=NULL; smartlist_t *sl; int i, cmdlen; @@ -464,18 +482,25 @@ nt_service_command_line(int *using_default_torrc) options = smartlist_join_strings(sl,"\" \"",0,NULL); smartlist_free(sl); +#ifdef UNICODE + wcstombs(tor_exe_ascii, tor_exe, sizeof(tor_exe_ascii)); +#else + strlcpy(tor_exe_ascii, tor_exe, sizeof(tor_exe_ascii)); +#endif + /* Allocate a string for the NT service command line */ - cmdlen = strlen(tor_exe) + (options?strlen(options):0) + 32; + cmdlen = strlen(tor_exe_ascii) + (options?strlen(options):0) + 32; command = tor_malloc(cmdlen); /* Format the service command */ if (options) { if (tor_snprintf(command, cmdlen, "\"%s\" --nt-service \"%s\"", - tor_exe, options)<0) { + tor_exe_ascii, options)<0) { tor_free(command); /* sets command to NULL. */ } } else { /* ! options */ - if (tor_snprintf(command, cmdlen, "\"%s\" --nt-service", tor_exe)<0) { + if (tor_snprintf(command, cmdlen, "\"%s\" --nt-service", + tor_exe_ascii)<0) { tor_free(command); /* sets command to NULL. */ } } @@ -500,10 +525,10 @@ nt_service_install(int argc, char **argv) SC_HANDLE hSCManager = NULL; SC_HANDLE hService = NULL; - SERVICE_DESCRIPTION sdBuff; + SERVICE_DESCRIPTIONA sdBuff; char *command; char *errmsg; - const char *user_acct = GENSRV_USERACCT; + const char *user_acct = NULL; const char *password = ""; int i; OSVERSIONINFOEX info; @@ -547,13 +572,12 @@ nt_service_install(int argc, char **argv) is_win2k_or_worse = 1; } - if (user_acct == GENSRV_USERACCT) { + if (!user_acct) { if (is_win2k_or_worse) { /* On Win2k, there is no LocalService account, so we actually need to * fall back on NULL (the system account). */ printf("Running on Win2K or earlier, so the LocalService account " "doesn't exist. Falling back to SYSTEM account.\n"); - user_acct = NULL; } else { /* Genericity is apparently _so_ last year in Redmond, where some * accounts are accounts that you can look up, and some accounts @@ -562,6 +586,7 @@ nt_service_install(int argc, char **argv) */ printf("Running on a Post-Win2K OS, so we'll assume that the " "LocalService account exists.\n"); + user_acct = GENSRV_USERACCT; } } else if (0 && service_fns.LookupAccountNameA_fn(NULL, // On this system user_acct, @@ -590,10 +615,10 @@ nt_service_install(int argc, char **argv) SERVICE_AUTO_START, SERVICE_ERROR_IGNORE, command, NULL, NULL, NULL, user_acct, password)) == NULL) { - errmsg = nt_strerror(GetLastError()); + errmsg = format_win32_error(GetLastError()); printf("CreateService() failed : %s\n", errmsg); service_fns.CloseServiceHandle_fn(hSCManager); - LocalFree(errmsg); + tor_free(errmsg); tor_free(command); return -1; } @@ -634,9 +659,9 @@ nt_service_remove(void) nt_service_stop(hService); if (service_fns.DeleteService_fn(hService) == FALSE) { - errmsg = nt_strerror(GetLastError()); + errmsg = format_win32_error(GetLastError()); printf("DeleteService() failed : %s\n", errmsg); - LocalFree(errmsg); + tor_free(errmsg); service_fns.CloseServiceHandle_fn(hService); service_fns.CloseServiceHandle_fn(hSCManager); return -1; @@ -693,20 +718,6 @@ nt_service_cmd_stop(void) return stop; } -/** Given a Win32 error code, this attempts to make Windows - * return a human-readable error message. The char* returned - * is allocated by Windows, but should be freed with LocalFree() - * when finished with it. */ -static char* -nt_strerror(uint32_t errnum) -{ - char *msgbuf; - FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM, - NULL, errnum, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), - (LPSTR)&msgbuf, 0, NULL); - return msgbuf; -} - int nt_service_parse_options(int argc, char **argv, int *should_exit) { diff --git a/src/or/ntmain.h b/src/or/ntmain.h new file mode 100644 index 0000000000..acd0e1d7eb --- /dev/null +++ b/src/or/ntmain.h @@ -0,0 +1,30 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file ntmain.h + * \brief Header file for ntmain.c. + **/ + +#ifndef _TOR_NTMAIN_H +#define _TOR_NTMAIN_H + +#ifdef MS_WINDOWS +#if !defined (WINCE) +#define NT_SERVICE +#endif +#endif + +#ifdef NT_SERVICE +int nt_service_parse_options(int argc, char **argv, int *should_exit); +int nt_service_is_stopping(void); +void nt_service_set_state(DWORD state); +#else +#define nt_service_is_stopping() 0 +#endif + +#endif + diff --git a/src/or/onion.c b/src/or/onion.c index 4c26debf1c..211d14c1e1 100644 --- a/src/or/onion.c +++ b/src/or/onion.c @@ -11,6 +11,10 @@ **/ #include "or.h" +#include "circuitlist.h" +#include "config.h" +#include "onion.h" +#include "rephist.h" /** Type for a linked list of circuits that are waiting for a free CPU worker * to process a waiting onion handshake. */ @@ -58,11 +62,18 @@ onion_pending_add(or_circuit_t *circ, char *onionskin) tor_assert(!ol_tail->next); if (ol_length >= get_options()->MaxOnionsPending) { - log_warn(LD_GENERAL, - "Your computer is too slow to handle this many circuit " - "creation requests! Please consider using the " - "MaxAdvertisedBandwidth config option or choosing a more " - "restricted exit policy."); +#define WARN_TOO_MANY_CIRC_CREATIONS_INTERVAL (60) + static ratelim_t last_warned = + RATELIM_INIT(WARN_TOO_MANY_CIRC_CREATIONS_INTERVAL); + char *m; + if ((m = rate_limit_log(&last_warned, approx_time()))) { + log_warn(LD_GENERAL, + "Your computer is too slow to handle this many circuit " + "creation requests! Please consider using the " + "MaxAdvertisedBandwidth config option or choosing a more " + "restricted exit policy.%s",m); + tor_free(m); + } tor_free(tmp); return -1; } @@ -248,6 +259,10 @@ onion_skin_server_handshake(const char *onion_skin, /*ONIONSKIN_CHALLENGE_LEN*/ } dh = crypto_dh_new(DH_TYPE_CIRCUIT); + if (!dh) { + log_warn(LD_BUG, "Couldn't allocate DH key"); + goto err; + } if (crypto_dh_get_public(dh, handshake_reply_out, DH_KEY_LEN)) { log_info(LD_GENERAL, "crypto_dh_get_public failed."); goto err; @@ -255,8 +270,9 @@ onion_skin_server_handshake(const char *onion_skin, /*ONIONSKIN_CHALLENGE_LEN*/ key_material_len = DIGEST_LEN+key_out_len; key_material = tor_malloc(key_material_len); - len = crypto_dh_compute_secret(dh, challenge, DH_KEY_LEN, - key_material, key_material_len); + len = crypto_dh_compute_secret(LOG_PROTOCOL_WARN, dh, challenge, + DH_KEY_LEN, key_material, + key_material_len); if (len < 0) { log_info(LD_GENERAL, "crypto_dh_compute_secret failed."); goto err; @@ -306,8 +322,9 @@ onion_skin_client_handshake(crypto_dh_env_t *handshake_state, key_material_len = DIGEST_LEN + key_out_len; key_material = tor_malloc(key_material_len); - len = crypto_dh_compute_secret(handshake_state, handshake_reply, DH_KEY_LEN, - key_material, key_material_len); + len = crypto_dh_compute_secret(LOG_PROTOCOL_WARN, handshake_state, + handshake_reply, DH_KEY_LEN, key_material, + key_material_len); if (len < 0) goto err; diff --git a/src/or/onion.h b/src/or/onion.h new file mode 100644 index 0000000000..7f603b8147 --- /dev/null +++ b/src/or/onion.h @@ -0,0 +1,48 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file onion.h + * \brief Header file for onion.c. + **/ + +#ifndef _TOR_ONION_H +#define _TOR_ONION_H + +int onion_pending_add(or_circuit_t *circ, char *onionskin); +or_circuit_t *onion_next_task(char **onionskin_out); +void onion_pending_remove(or_circuit_t *circ); + +int onion_skin_create(crypto_pk_env_t *router_key, + crypto_dh_env_t **handshake_state_out, + char *onion_skin_out); + +int onion_skin_server_handshake(const char *onion_skin, + crypto_pk_env_t *private_key, + crypto_pk_env_t *prev_private_key, + char *handshake_reply_out, + char *key_out, + size_t key_out_len); + +int onion_skin_client_handshake(crypto_dh_env_t *handshake_state, + const char *handshake_reply, + char *key_out, + size_t key_out_len); + +int fast_server_handshake(const uint8_t *key_in, + uint8_t *handshake_reply_out, + uint8_t *key_out, + size_t key_out_len); + +int fast_client_handshake(const uint8_t *handshake_state, + const uint8_t *handshake_reply_out, + uint8_t *key_out, + size_t key_out_len); + +void clear_pending_onions(void); + +#endif + diff --git a/src/or/or.h b/src/or/or.h index 897ad32a43..d667358eb0 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -20,9 +20,6 @@ #ifndef INSTRUMENT_DOWNLOADS #define INSTRUMENT_DOWNLOADS 1 #endif -#ifndef ENABLE_GEOIP_STATS -#define ENABLE_GEOIP_STATS 1 -#endif #endif #ifdef MS_WINDOWS @@ -62,6 +59,9 @@ #ifdef HAVE_SYS_STAT_H #include <sys/stat.h> #endif +#ifdef HAVE_NETINET_IN_H +#include <netinet/in.h> +#endif #ifdef HAVE_ARPA_INET_H #include <arpa/inet.h> #endif @@ -83,18 +83,15 @@ #define snprintf _snprintf #endif -#include "crypto.h" #include "tortls.h" -#include "log.h" -#include "compat.h" +#include "../common/torlog.h" #include "container.h" -#include "util.h" #include "torgzip.h" #include "address.h" +#include "compat_libevent.h" +#include "ht.h" -#include <event.h> - -/* These signals are defined to help control_signal_act work. +/* These signals are defined to help handle_control_signal work. */ #ifndef SIGHUP #define SIGHUP 1 @@ -161,7 +158,7 @@ #define MAX_DNS_TTL (3*60*60) /** How small can a TTL be before we stop believing it? Provides rudimentary * pinning. */ -#define MIN_DNS_TTL (60) +#define MIN_DNS_TTL 60 /** How often do we rotate onion keys? */ #define MIN_ONION_KEY_LIFETIME (7*24*60*60) @@ -221,6 +218,21 @@ typedef enum { /* !!!! If _CONN_TYPE_MAX is ever over 15, we must grow the type field in * connection_t. */ +/* Proxy client types */ +#define PROXY_NONE 0 +#define PROXY_CONNECT 1 +#define PROXY_SOCKS4 2 +#define PROXY_SOCKS5 3 + +/* Proxy client handshake states */ +#define PROXY_HTTPS_WANT_CONNECT_OK 1 +#define PROXY_SOCKS4_WANT_CONNECT_OK 2 +#define PROXY_SOCKS5_WANT_AUTH_METHOD_NONE 3 +#define PROXY_SOCKS5_WANT_AUTH_METHOD_RFC1929 4 +#define PROXY_SOCKS5_WANT_AUTH_RFC1929_OK 5 +#define PROXY_SOCKS5_WANT_CONNECT_OK 6 +#define PROXY_CONNECTED 7 + /** True iff <b>x</b> is an edge connection. */ #define CONN_IS_EDGE(x) \ ((x)->type == CONN_TYPE_EXIT || (x)->type == CONN_TYPE_AP) @@ -241,26 +253,24 @@ typedef enum { #define _OR_CONN_STATE_MIN 1 /** State for a connection to an OR: waiting for connect() to finish. */ #define OR_CONN_STATE_CONNECTING 1 -/** State for a connection to an OR: waiting for proxy command to flush. */ -#define OR_CONN_STATE_PROXY_FLUSHING 2 -/** State for a connection to an OR: waiting for proxy response. */ -#define OR_CONN_STATE_PROXY_READING 3 +/** State for a connection to an OR: waiting for proxy handshake to complete */ +#define OR_CONN_STATE_PROXY_HANDSHAKING 2 /** State for a connection to an OR or client: SSL is handshaking, not done * yet. */ -#define OR_CONN_STATE_TLS_HANDSHAKING 4 +#define OR_CONN_STATE_TLS_HANDSHAKING 3 /** State for a connection to an OR: We're doing a second SSL handshake for * renegotiation purposes. */ -#define OR_CONN_STATE_TLS_CLIENT_RENEGOTIATING 5 +#define OR_CONN_STATE_TLS_CLIENT_RENEGOTIATING 4 /** State for a connection at an OR: We're waiting for the client to * renegotiate. */ -#define OR_CONN_STATE_TLS_SERVER_RENEGOTIATING 6 +#define OR_CONN_STATE_TLS_SERVER_RENEGOTIATING 5 /** State for a connection to an OR: We're done with our SSL handshake, but we * haven't yet negotiated link protocol versions and sent a netinfo cell. */ -#define OR_CONN_STATE_OR_HANDSHAKING 7 +#define OR_CONN_STATE_OR_HANDSHAKING 6 /** State for a connection to an OR: Ready to send/receive cells. */ -#define OR_CONN_STATE_OPEN 8 -#define _OR_CONN_STATE_MAX 8 +#define OR_CONN_STATE_OPEN 7 +#define _OR_CONN_STATE_MAX 7 #define _EXIT_CONN_STATE_MIN 1 /** State for an exit connection: waiting for response from DNS farm. */ @@ -457,23 +467,23 @@ typedef enum { #define CIRCUIT_PURPOSE_C_REND_READY_INTRO_ACKED 11 /** Client-side circuit purpose: at Alice, rendezvous established. */ #define CIRCUIT_PURPOSE_C_REND_JOINED 12 - -#define _CIRCUIT_PURPOSE_C_MAX 12 - +/** This circuit is used for build time measurement only */ +#define CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT 13 +#define _CIRCUIT_PURPOSE_C_MAX 13 /** Hidden-service-side circuit purpose: at Bob, waiting for introductions. */ -#define CIRCUIT_PURPOSE_S_ESTABLISH_INTRO 13 +#define CIRCUIT_PURPOSE_S_ESTABLISH_INTRO 14 /** Hidden-service-side circuit purpose: at Bob, successfully established * intro. */ -#define CIRCUIT_PURPOSE_S_INTRO 14 +#define CIRCUIT_PURPOSE_S_INTRO 15 /** Hidden-service-side circuit purpose: at Bob, connecting to rend point. */ -#define CIRCUIT_PURPOSE_S_CONNECT_REND 15 +#define CIRCUIT_PURPOSE_S_CONNECT_REND 16 /** Hidden-service-side circuit purpose: at Bob, rendezvous established. */ -#define CIRCUIT_PURPOSE_S_REND_JOINED 16 +#define CIRCUIT_PURPOSE_S_REND_JOINED 17 /** A testing circuit; not meant to be used for actual traffic. */ -#define CIRCUIT_PURPOSE_TESTING 17 +#define CIRCUIT_PURPOSE_TESTING 18 /** A controller made this circuit and Tor should not use it. */ -#define CIRCUIT_PURPOSE_CONTROLLER 18 -#define _CIRCUIT_PURPOSE_MAX 18 +#define CIRCUIT_PURPOSE_CONTROLLER 19 +#define _CIRCUIT_PURPOSE_MAX 19 /** A catch-all for unrecognized purposes. Currently we don't expect * to make or see any circuits with this purpose. */ #define CIRCUIT_PURPOSE_UNKNOWN 255 @@ -483,7 +493,7 @@ typedef enum { #define CIRCUIT_PURPOSE_IS_ORIGIN(p) ((p)>_CIRCUIT_PURPOSE_OR_MAX) /** True iff the circuit purpose <b>p</b> is for a circuit that originated * here to serve as a client. (Hidden services don't count here.) */ -#define CIRCUIT_PURPOSE_IS_CLIENT(p) \ +#define CIRCUIT_PURPOSE_IS_CLIENT(p) \ ((p)> _CIRCUIT_PURPOSE_OR_MAX && \ (p)<=_CIRCUIT_PURPOSE_C_MAX) /** True iff the circuit_t <b>c</b> is actually an origin_circuit_t. */ @@ -573,6 +583,9 @@ typedef enum { /** This is a connection on the NATD port, and the destination IP:Port was * either ill-formed or out-of-range. */ #define END_STREAM_REASON_INVALID_NATD_DEST 261 +/** The target address is in a private network (like 127.0.0.1 or 10.0.0.1); + * you don't want to do that over a randomly chosen exit */ +#define END_STREAM_REASON_PRIVATE_ADDR 262 /** Bitwise-and this value with endreason to mask out all flags. */ #define END_STREAM_REASON_MASK 511 @@ -604,6 +617,10 @@ typedef enum { /* Negative reasons are internal: we never send them in a DESTROY or TRUNCATE * call; they only go to the controller for tracking */ +/** Our post-timeout circuit time measurement period expired. + * We must give up now */ +#define END_CIRC_REASON_MEASUREMENT_EXPIRED -3 + /** We couldn't build a path for this circuit. */ #define END_CIRC_REASON_NOPATH -2 /** Catch-all "other" reason for closing origin circuits. */ @@ -641,10 +658,6 @@ typedef enum { /** Length of a binary-encoded rendezvous service ID. */ #define REND_SERVICE_ID_LEN 10 -/** How long after we receive a hidden service descriptor do we consider - * it fresh? */ -#define NUM_SECONDS_BEFORE_HS_REFETCH (60*15) - /** Time period for which a v2 descriptor will be valid. */ #define REND_TIME_PERIOD_V2_DESC_VALIDITY (24*60*60) @@ -739,12 +752,6 @@ typedef struct rend_data_t { /** Rendezvous cookie used by both, client and service. */ char rend_cookie[REND_COOKIE_LEN]; - - /** Rendezvous descriptor version that is used by a service. Used to - * distinguish introduction and rendezvous points belonging to the same - * rendezvous service ID, but different descriptor versions. - */ - uint8_t rend_desc_version; } rend_data_t; /** Time interval for tracking possible replays of INTRODUCE2 cells. @@ -761,6 +768,8 @@ typedef enum { /** Initial value for both sides of a circuit transmission window when the * circuit is initialized. Measured in cells. */ #define CIRCWINDOW_START 1000 +#define CIRCWINDOW_START_MIN 100 +#define CIRCWINDOW_START_MAX 1000 /** Amount to increment a circuit window when we get a circuit SENDME. */ #define CIRCWINDOW_INCREMENT 100 /** Initial value on both sides of a stream transmission window when the @@ -839,9 +848,13 @@ typedef struct cell_t { /** Parsed variable-length onion routing cell. */ typedef struct var_cell_t { + /** Type of the cell: CELL_VERSIONS, etc. */ uint8_t command; + /** Circuit thich received the cell */ circid_t circ_id; + /** Number of bytes actually stored in <b>payload</b> */ uint16_t payload_len; + /** Payload of this cell */ uint8_t payload[1]; } var_cell_t; @@ -851,12 +864,28 @@ typedef struct packed_cell_t { char body[CELL_NETWORK_SIZE]; /**< Cell as packed for network. */ } packed_cell_t; +/** Number of cells added to a circuit queue including their insertion + * time on 10 millisecond detail; used for buffer statistics. */ +typedef struct insertion_time_elem_t { + struct insertion_time_elem_t *next; /**< Next element in queue. */ + uint32_t insertion_time; /**< When were cells inserted (in 10 ms steps + * starting at 0:00 of the current day)? */ + unsigned counter; /**< How many cells were inserted? */ +} insertion_time_elem_t; + +/** Queue of insertion times. */ +typedef struct insertion_time_queue_t { + struct insertion_time_elem_t *first; /**< First element in queue. */ + struct insertion_time_elem_t *last; /**< Last element in queue. */ +} insertion_time_queue_t; + /** A queue of cells on a circuit, waiting to be added to the * or_connection_t's outbuf. */ typedef struct cell_queue_t { packed_cell_t *head; /**< The first cell, or NULL if the queue is empty. */ packed_cell_t *tail; /**< The last cell, or NULL if the queue is empty. */ int n; /**< The number of cells in the queue. */ + insertion_time_queue_t *insertion_times; /**< Insertion times of cells. */ } cell_queue_t; /** Beginning of a RELAY cell payload. */ @@ -912,7 +941,7 @@ typedef struct connection_t { * again once the bandwidth throttler allows it? */ unsigned int write_blocked_on_bw:1; /**< Boolean: should we start writing * again once the bandwidth throttler allows - * reads? */ + * writes? */ unsigned int hold_open_until_flushed:1; /**< Despite this connection's being * marked for close, do we flush it * before closing it? */ @@ -937,8 +966,11 @@ typedef struct connection_t { * connection. */ unsigned int linked_conn_is_closed:1; - int s; /**< Our socket; -1 if this connection is closed, or has no - * socket. */ + /** CONNECT/SOCKS proxy client handshake state (for outgoing connections). */ + unsigned int proxy_state:4; + + /** Our socket; -1 if this connection is closed, or has no socket. */ + evutil_socket_t s; int conn_array_index; /**< Index into the global connection array. */ struct event *read_event; /**< Libevent event structure. */ struct event *write_event; /**< Libevent event structure. */ @@ -974,12 +1006,14 @@ typedef struct connection_t { /** Unique identifier for this connection on this Tor instance. */ uint64_t global_identifier; - /* XXXX022 move this field, and all the listener-only fields (just + /* XXXX023 move this field, and all the listener-only fields (just socket_family, I think), into a new listener_connection_t subtype. */ /** If the connection is a CONN_TYPE_AP_DNS_LISTENER, this field points * to the evdns_server_port is uses to listen to and answer connections. */ struct evdns_server_port *dns_server_port; + /** Unique ID for measuring tunneled network status requests. */ + uint64_t dirreq_id; } connection_t; /** Stores flags and information related to the portion of a v2 Tor OR @@ -1026,7 +1060,10 @@ typedef struct or_connection_t { * NETINFO cell listed the address we're connected to as recognized. */ unsigned int is_canonical:1; /** True iff this connection shouldn't get any new circs attached to it, - * because the connection is too old, or because there's a better one, etc. + * because the connection is too old, or because there's a better one. + * More generally, this flag is used to note an unhealthy connection; + * for example, if a bad connection fails we shouldn't assume that the + * router itself has a problem. */ unsigned int is_bad_for_new_circs:1; uint8_t link_proto; /**< What protocol version are we using? 0 for @@ -1041,12 +1078,13 @@ typedef struct or_connection_t { time_t timestamp_last_added_nonpadding; /** When did we last add a * non-padding cell to the outbuf? */ - /* bandwidth* and read_bucket only used by ORs in OPEN state: */ + /* bandwidth* and *_bucket only used by ORs in OPEN state: */ int bandwidthrate; /**< Bytes/s added to the bucket. (OPEN ORs only.) */ int bandwidthburst; /**< Max bucket size for this conn. (OPEN ORs only.) */ int read_bucket; /**< When this hits 0, stop receiving. Every second we * add 'bandwidthrate' to this, capping it at * bandwidthburst. (OPEN ORs only) */ + int write_bucket; /**< When this hits 0, stop writing. Like read_bucket. */ int n_circuits; /**< How many circuits use this connection as p_conn or * n_conn ? */ @@ -1054,6 +1092,17 @@ typedef struct or_connection_t { * free up on this connection's outbuf. Every time we pull cells from a * circuit, we advance this pointer to the next circuit in the ring. */ struct circuit_t *active_circuits; + /** Priority queue of cell_ewma_t for circuits with queued cells waiting for + * room to free up on this connection's outbuf. Kept in heap order + * according to EWMA. + * + * This is redundant with active_circuits; if we ever decide only to use the + * cell_ewma algorithm for choosing circuits, we can remove active_circuits. + */ + smartlist_t *active_circuit_pqueue; + /** The tick on which the cell_ewma_ts in active_circuit_pqueue last had + * their ewma values rescaled. */ + unsigned active_circuit_pqueue_last_recalibrated; struct or_connection_t *next_with_same_id; /**< Next connection with same * identity digest as this one. */ } or_connection_t; @@ -1102,6 +1151,13 @@ typedef struct edge_connection_t { * already retried several times. */ uint8_t num_socks_retries; +#define NUM_CIRCUITS_LAUNCHED_THRESHOLD 10 + /** Number of times we've launched a circuit to handle this stream. If + * it gets too high, that could indicate an inconsistency between our + * "launch a circuit to handle this stream" logic and our "attach our + * stream to one of the available circuits" logic. */ + unsigned int num_circuits_launched:4; + /** True iff this connection is for a DNS request only. */ unsigned int is_dns_request:1; @@ -1128,6 +1184,10 @@ typedef struct edge_connection_t { * zero, abandon the associated mapaddress. */ unsigned int chosen_exit_retries:3; + /** True iff this is an AP connection that came from a transparent or + * NATd connection */ + unsigned int is_transparent_ap:1; + /** If this is a DNSPort connection, this field holds the pending DNS * request that we're going to try to answer. */ struct evdns_server_request *dns_server_request; @@ -1150,7 +1210,8 @@ typedef struct dir_connection_t { enum { DIR_SPOOL_NONE=0, DIR_SPOOL_SERVER_BY_DIGEST, DIR_SPOOL_SERVER_BY_FP, DIR_SPOOL_EXTRA_BY_DIGEST, DIR_SPOOL_EXTRA_BY_FP, - DIR_SPOOL_CACHED_DIR, DIR_SPOOL_NETWORKSTATUS + DIR_SPOOL_CACHED_DIR, DIR_SPOOL_NETWORKSTATUS, + DIR_SPOOL_MICRODESC, /* NOTE: if we add another entry, add another bit. */ } dir_spool_src : 3; /** If we're fetching descriptors, what router purpose shall we assign * to them? */ @@ -1178,12 +1239,6 @@ typedef struct control_connection_t { uint32_t event_mask; /**< Bitfield: which events does this controller * care about? */ - unsigned int use_long_names:1; /**< True if we should use long nicknames - * on this (v1) connection. Only settable - * via v1 controllers. */ - /** For control connections only. If set, we send extended info with control - * events as appropriate. */ - unsigned int use_extended_events:1; /** True if we have sent a protocolinfo reply on this connection. */ unsigned int have_sent_protocolinfo:1; @@ -1267,6 +1322,7 @@ typedef struct cached_dir_t { size_t dir_len; /**< Length of <b>dir</b> (not counting its NUL). */ size_t dir_z_len; /**< Length of <b>dir_z</b>. */ time_t published; /**< When was this object published. */ + digests_t digests; /**< Digests of this object (networkstatus only) */ int refcnt; /**< Reference count for this cached_dir_t. */ } cached_dir_t; @@ -1427,6 +1483,9 @@ typedef struct { * directory according to the authorities. */ unsigned int policy_is_reject_star:1; /**< True iff the exit policy for this * router rejects everything. */ + /** True if, after we have added this router, we should re-launch + * tests for it. */ + unsigned int needs_retest_if_added:1; /** Tor can use this router for general positions in circuits. */ #define ROUTER_PURPOSE_GENERAL 0 @@ -1516,6 +1575,9 @@ typedef struct routerstatus_t { unsigned int has_bandwidth:1; /**< The vote/consensus had bw info */ unsigned int has_exitsummary:1; /**< The vote/consensus had exit summaries */ + unsigned int has_measured_bw:1; /**< The vote/consensus had a measured bw */ + + uint32_t measured_bw; /**< Measured bandwidth (capacity) of the router */ uint32_t bandwidth; /**< Bandwidth (capacity) of the router as reported in * the vote/consensus, in kilobytes/sec. */ @@ -1540,6 +1602,52 @@ typedef struct routerstatus_t { } routerstatus_t; +/** A microdescriptor is the smallest amount of information needed to build a + * circuit through a router. They are generated by the directory authorities, + * using information from the uploaded routerinfo documents. They are not + * self-signed, but are rather authenticated by having their hash in a signed + * networkstatus document. */ +typedef struct microdesc_t { + /** Hashtable node, used to look up the microdesc by its digest. */ + HT_ENTRY(microdesc_t) node; + + /* Cache information */ + + /** When was this microdescriptor last listed in a consensus document? + * Once a microdesc has been unlisted long enough, we can drop it. + */ + time_t last_listed; + /** Where is this microdescriptor currently stored? */ + saved_location_t saved_location : 3; + /** If true, do not attempt to cache this microdescriptor on disk. */ + unsigned int no_save : 1; + /** If saved_location == SAVED_IN_CACHE, this field holds the offset of the + * microdescriptor in the cache. */ + off_t off; + + /* The string containing the microdesc. */ + + /** A pointer to the encoded body of the microdescriptor. If the + * saved_location is SAVED_IN_CACHE, then the body is a pointer into an + * mmap'd region. Otherwise, it is a malloc'd string. The string might not + * be NUL-terminated; take the length from <b>bodylen</b>. */ + char *body; + /** The length of the microdescriptor in <b>body</b>. */ + size_t bodylen; + /** A SHA256-digest of the microdescriptor. */ + char digest[DIGEST256_LEN]; + + /* Fields in the microdescriptor. */ + + /** As routerinfo_t.onion_pkey */ + crypto_pk_env_t *onion_pkey; + /** As routerinfo_t.family */ + smartlist_t *family; + /** Encoded exit policy summary */ + char *exitsummary; /**< exit policy summary - + * XXX this probably should not stay a string. */ +} microdesc_t; + /** How many times will we try to download a router's descriptor before giving * up? */ #define MAX_ROUTERDESC_DOWNLOAD_FAILURES 8 @@ -1582,6 +1690,16 @@ typedef struct networkstatus_v2_t { * sorted by identity_digest. */ } networkstatus_v2_t; +/** Linked list of microdesc hash lines for a single router in a directory + * vote. + */ +typedef struct vote_microdesc_hash_t { + /** Next element in the list, or NULL. */ + struct vote_microdesc_hash_t *next; + /** The raw contents of the microdesc hash line, excluding the "m". */ + char *microdesc_hash_line; +} vote_microdesc_hash_t; + /** The claim about a single router, made in a vote. */ typedef struct vote_routerstatus_t { routerstatus_t status; /**< Underlying 'status' object for this router. @@ -1590,31 +1708,46 @@ typedef struct vote_routerstatus_t { * networkstatus_t.known_flags. */ char *version; /**< The version that the authority says this router is * running. */ + /** The hash or hashes that the authority claims this microdesc has. */ + vote_microdesc_hash_t *microdesc; } vote_routerstatus_t; +/** A signature of some document by an authority. */ +typedef struct document_signature_t { + /** Declared SHA-1 digest of this voter's identity key */ + char identity_digest[DIGEST_LEN]; + /** Declared SHA-1 digest of signing key used by this voter. */ + char signing_key_digest[DIGEST_LEN]; + /** Algorithm used to compute the digest of the document. */ + digest_algorithm_t alg; + /** Signature of the signed thing. */ + char *signature; + /** Length of <b>signature</b> */ + int signature_len; + unsigned int bad_signature : 1; /**< Set to true if we've tried to verify + * the sig, and we know it's bad. */ + unsigned int good_signature : 1; /**< Set to true if we've verified the sig + * as good. */ +} document_signature_t; + /** Information about a single voter in a vote or a consensus. */ typedef struct networkstatus_voter_info_t { + /** Declared SHA-1 digest of this voter's identity key */ + char identity_digest[DIGEST_LEN]; char *nickname; /**< Nickname of this voter */ - char identity_digest[DIGEST_LEN]; /**< Digest of this voter's identity key */ + /** Digest of this voter's "legacy" identity key, if any. In vote only; for + * consensuses, we treat legacy keys as additional signers. */ + char legacy_id_digest[DIGEST_LEN]; char *address; /**< Address of this voter, in string format. */ uint32_t addr; /**< Address of this voter, in IPv4, in host order. */ uint16_t dir_port; /**< Directory port of this voter */ uint16_t or_port; /**< OR port of this voter */ char *contact; /**< Contact information for this voter. */ char vote_digest[DIGEST_LEN]; /**< Digest of this voter's vote, as signed. */ - /** Digest of this voter's "legacy" identity key, if any. In vote only; for - * consensuses, we treat legacy keys as additional signers. */ - char legacy_id_digest[DIGEST_LEN]; /* Nothing from here on is signed. */ - char signing_key_digest[DIGEST_LEN]; /**< Declared digest of signing key - * used by this voter. */ - char *signature; /**< Signature from this voter. */ - int signature_len; /**< Length of <b>signature</b> */ - unsigned int bad_signature : 1; /**< Set to true if we've tried to verify - * the sig, and we know it's bad. */ - unsigned int good_signature : 1; /**< Set to true if we've verified the sig - * as good. */ + /** The signature of the document and the signature's status. */ + smartlist_t *sigs; } networkstatus_voter_info_t; /** Enumerates the possible seriousness values of a networkstatus document. */ @@ -1624,10 +1757,25 @@ typedef enum { NS_TYPE_OPINION, } networkstatus_type_t; +/** Enumerates recognized flavors of a consensus networkstatus document. All + * flavors of a consensus are generated from the same set of votes, but they + * present different types information to different versions of Tor. */ +typedef enum { + FLAV_NS = 0, + FLAV_MICRODESC = 1, +} consensus_flavor_t; + +/** Which consensus flavor do we actually want to use to build circuits? */ +#define USABLE_CONSENSUS_FLAVOR FLAV_NS + +/** How many different consensus flavors are there? */ +#define N_CONSENSUS_FLAVORS ((int)(FLAV_MICRODESC)+1) + /** A common structure to hold a v3 network status vote, or a v3 network * status consensus. */ typedef struct networkstatus_t { - networkstatus_type_t type; /**< Vote, consensus, or opinion? */ + networkstatus_type_t type : 8; /**< Vote, consensus, or opinion? */ + consensus_flavor_t flavor : 8; /**< If a consensus, what kind? */ time_t published; /**< Vote only: Time when vote was written. */ time_t valid_after; /**< Time after which this vote or consensus applies. */ time_t fresh_until; /**< Time before which this is the most recent vote or @@ -1659,6 +1807,10 @@ typedef struct networkstatus_t { * consensus, sorted by key. */ smartlist_t *net_params; + /** List of key=value strings for the bw weight parameters in the + * consensus. */ + smartlist_t *weight_params; + /** List of networkstatus_voter_info_t. For a vote, only one element * is included. For a consensus, one element is included for every voter * whose vote contributed to the consensus. */ @@ -1666,8 +1818,8 @@ typedef struct networkstatus_t { struct authority_cert_t *cert; /**< Vote only: the voter's certificate. */ - /** Digest of this document, as signed. */ - char networkstatus_digest[DIGEST_LEN]; + /** Digests of this document, as signed. */ + digests_t digests; /** List of router statuses, sorted by identity digest. For a vote, * the elements are vote_routerstatus_t; for a consensus, the elements @@ -1679,14 +1831,15 @@ typedef struct networkstatus_t { digestmap_t *desc_digest_map; } networkstatus_t; -/** A set of signatures for a networkstatus consensus. All fields are as for - * networkstatus_t. */ +/** A set of signatures for a networkstatus consensus. Unless otherwise + * noted, all fields are as for networkstatus_t. */ typedef struct ns_detached_signatures_t { time_t valid_after; time_t fresh_until; time_t valid_until; - char networkstatus_digest[DIGEST_LEN]; - smartlist_t *signatures; /* list of networkstatus_voter_info_t */ + strmap_t *digests; /**< Map from flavor name to digestset_t */ + strmap_t *signatures; /**< Map from flavor name to list of + * document_signature_t */ } ns_detached_signatures_t; /** Allowable types of desc_store_t. */ @@ -1891,6 +2044,29 @@ typedef struct { time_t expiry_time; } cpath_build_state_t; +/** + * The cell_ewma_t structure keeps track of how many cells a circuit has + * transferred recently. It keeps an EWMA (exponentially weighted moving + * average) of the number of cells flushed from the circuit queue onto a + * connection in connection_or_flush_from_first_active_circuit(). + */ +typedef struct { + /** The last 'tick' at which we recalibrated cell_count. + * + * A cell sent at exactly the start of this tick has weight 1.0. Cells sent + * since the start of this tick have weight greater than 1.0; ones sent + * earlier have less weight. */ + unsigned last_adjusted_tick; + /** The EWMA of the cell count. */ + double cell_count; + /** True iff this is the cell count for a circuit's previous + * connection. */ + unsigned int is_for_p_conn : 1; + /** The position of the circuit within the OR connection's priority + * queue. */ + int heap_index; +} cell_ewma_t; + #define ORIGIN_CIRCUIT_MAGIC 0x35315243u #define OR_CIRCUIT_MAGIC 0x98ABC04Fu @@ -1957,9 +2133,15 @@ typedef struct circuit_t { * length ONIONSKIN_CHALLENGE_LEN. */ char *n_conn_onionskin; - time_t timestamp_created; /**< When was this circuit created? */ - time_t timestamp_dirty; /**< When the circuit was first used, or 0 if the - * circuit is clean. */ + struct timeval timestamp_created; /**< When was the circuit created? */ + /** When the circuit was first used, or 0 if the circuit is clean. + * + * XXXX023 Note that some code will artifically adjust this value backward + * in time in order to indicate that a circuit shouldn't be used for new + * streams, but that it can stay alive as long as it has streams on it. + * That's a kludge we should fix. + */ + time_t timestamp_dirty; uint16_t marked_for_close; /**< Should we close this circuit at the end of * the main loop? (If true, holds the line number @@ -1976,6 +2158,14 @@ typedef struct circuit_t { * linked to an OR connection. */ struct circuit_t *prev_active_on_n_conn; struct circuit_t *next; /**< Next circuit in linked list of all circuits. */ + + /** Unique ID for measuring tunneled network status requests. */ + uint64_t dirreq_id; + + /** The EWMA count for the number of cells flushed from the + * n_conn_cells queue. Used to determine which circuit to flush from next. + */ + cell_ewma_t n_cell_ewma; } circuit_t; /** Largest number of relay_early cells that we can send on a given @@ -2008,6 +2198,13 @@ typedef struct origin_circuit_t { * to the specification? */ unsigned int remaining_relay_early_cells : 4; + /** Set if this circuit is insanely old and we already informed the user */ + unsigned int is_ancient : 1; + + /** Set if this circuit has already been opened. Used to detect + * cannibalized circuits. */ + unsigned int has_opened : 1; + /** What commands were sent over this circuit that decremented the * RELAY_EARLY counter? This is for debugging task 878. */ uint8_t relay_early_commands[MAX_RELAY_EARLY_CELLS_PER_CIRCUIT]; @@ -2098,6 +2295,19 @@ typedef struct or_circuit_t { /** True iff this circuit was made with a CREATE_FAST cell. */ unsigned int is_first_hop : 1; + + /** Number of cells that were removed from circuit queue; reset every + * time when writing buffer stats to disk. */ + uint32_t processed_cells; + + /** Total time in milliseconds that cells spent in both app-ward and + * exit-ward queues of this circuit; reset every time when writing + * buffer stats to disk. */ + uint64_t total_cell_waiting_time; + + /** The EWMA count for the number of cells flushed from the + * p_conn_cells queue. */ + cell_ewma_t p_cell_ewma; } or_circuit_t; /** Convert a circuit subtype to a circuit_t.*/ @@ -2157,6 +2367,9 @@ typedef struct { config_line_t *Logs; /**< New-style list of configuration lines * for logs */ + int LogMessageDomains; /**< Boolean: Should we log the domain(s) in which + * each log message occurs? */ + char *DebugLogFile; /**< Where to send verbose log messages. */ char *DataDirectory; /**< OR only: where to store long-term data. */ char *Nickname; /**< OR only: nickname of this onion router. */ @@ -2169,19 +2382,22 @@ typedef struct { routerset_t *EntryNodes;/**< Structure containing nicknames, digests, * country codes and IP address patterns of ORs to * consider as entry points. */ - int StrictExitNodes; /**< Boolean: When none of our ExitNodes are up, do we - * stop building circuits? */ - int StrictEntryNodes; /**< Boolean: When none of our EntryNodes are up, do we - * stop building circuits? */ + int StrictNodes; /**< Boolean: When none of our EntryNodes or ExitNodes + * are up, or we need to access a node in ExcludeNodes, + * do we just fail instead? */ routerset_t *ExcludeNodes;/**< Structure containing nicknames, digests, * country codes and IP address patterns of ORs - * not to use in circuits. */ + * not to use in circuits. But see StrictNodes + * above. */ routerset_t *ExcludeExitNodes;/**< Structure containing nicknames, digests, * country codes and IP address patterns of * ORs not to consider as exits. */ /** Union of ExcludeNodes and ExcludeExitNodes */ - struct routerset_t *_ExcludeExitNodesUnion; + routerset_t *_ExcludeExitNodesUnion; + + int DisableAllSwap; /**< Boolean: Attempt to call mlockall() on our + * process for all current and future memory. */ /** List of "entry", "middle", "exit", "introduction", "rendezvous". */ smartlist_t *AllowInvalidNodes; @@ -2197,7 +2413,7 @@ typedef struct { * connections. */ config_line_t *TransListenAddress; /** Addresses to bind for listening for transparent natd connections */ - config_line_t *NatdListenAddress; + config_line_t *NATDListenAddress; /** Addresses to bind for listening for SOCKS connections. */ config_line_t *DNSListenAddress; /** Addresses to bind for listening for OR connections. */ @@ -2221,7 +2437,7 @@ typedef struct { int SocksPort; /**< Port to listen on for SOCKS connections. */ /** Port to listen on for transparent pf/netfilter connections. */ int TransPort; - int NatdPort; /**< Port to listen on for transparent natd connections. */ + int NATDPort; /**< Port to listen on for transparent natd connections. */ int ControlPort; /**< Port to listen on for control connections. */ config_line_t *ControlSocket; /**< List of Unix Domain Sockets to listen on * for control connections. */ @@ -2237,8 +2453,6 @@ typedef struct { * for version 3 directories? */ int HSAuthoritativeDir; /**< Boolean: does this an authoritative directory * handle hidden service requests? */ - int HSAuthorityRecordStats; /**< Boolean: does this HS authoritative - * directory record statistics? */ int NamingAuthoritativeDir; /**< Boolean: is this an authoritative directory * that's willing to bind names? */ int VersioningAuthoritativeDir; /**< Boolean: is this an authoritative @@ -2267,8 +2481,6 @@ typedef struct { int AvoidDiskWrites; /**< Boolean: should we never cache things to disk? * Not used yet. */ int ClientOnly; /**< Boolean: should we never evolve into a server role? */ - /** Boolean: should we never publish a descriptor? Deprecated. */ - int NoPublish; /** To what authority types do we publish our descriptor? Choices are * "v1", "v2", "v3", "bridge", or "". */ smartlist_t *PublishServerDescriptor; @@ -2277,7 +2489,9 @@ typedef struct { /** Boolean: do we publish hidden service descriptors to the HS auths? */ int PublishHidServDescriptors; int FetchServerDescriptors; /**< Do we fetch server descriptors as normal? */ - int FetchHidServDescriptors; /** and hidden service descriptors? */ + int FetchHidServDescriptors; /**< and hidden service descriptors? */ + int FetchV2Networkstatus; /**< Do we fetch v2 networkstatus documents when + * we don't need to? */ int HidServDirectoryV2; /**< Do we participate in the HS DHT? */ int MinUptimeHidServDirectoryV2; /**< As directory authority, accept hidden @@ -2299,6 +2513,14 @@ typedef struct { int ConstrainedSockets; /**< Shrink xmit and recv socket buffers. */ uint64_t ConstrainedSockSize; /**< Size of constrained buffers. */ + /** Whether we should drop exit streams from Tors that we don't know are + * relays. One of "0" (never refuse), "1" (always refuse), or "auto" (do + * what the consensus says, defaulting to 'refuse' if the consensus says + * nothing). */ + char *RefuseUnknownExits; + /** Parsed version of RefuseUnknownExits. -1 for auto. */ + int RefuseUnknownExits_; + /** Application ports that require all nodes in circ to have sufficient * uptime. */ smartlist_t *LongLivedPorts; @@ -2328,10 +2550,18 @@ typedef struct { * connections alive? */ int SocksTimeout; /**< How long do we let a socks connection wait * unattached before we fail it? */ - int CircuitBuildTimeout; /**< Cull non-open circuits that were born - * at least this many seconds ago. */ + int LearnCircuitBuildTimeout; /**< If non-zero, we attempt to learn a value + * for CircuitBuildTimeout based on timeout + * history */ + int CircuitBuildTimeout; /**< Cull non-open circuits that were born at + * least this many seconds ago. Used until + * adaptive algorithm learns a new value. */ int CircuitIdleTimeout; /**< Cull open clean circuits that were born * at least this many seconds ago. */ + int CircuitStreamTimeout; /**< If non-zero, detach streams from circuits + * and try a new circuit if the stream has been + * waiting for this many seconds. If zero, use + * our default internal timeout schedule. */ int MaxOnionsPending; /**< How many circuit CREATE requests do we allow * to wait simultaneously before we start dropping * them? */ @@ -2349,24 +2579,36 @@ typedef struct { * willing to use for all relayed conns? */ uint64_t RelayBandwidthBurst; /**< How much bandwidth, at maximum, will we * use in a second for all relayed conns? */ - int NumCpus; /**< How many CPUs should we try to use? */ - int RunTesting; /**< If true, create testing circuits to measure how well the - * other ORs are running. */ + uint64_t PerConnBWRate; /**< Long-term bw on a single TLS conn, if set. */ + uint64_t PerConnBWBurst; /**< Allowed burst on a single TLS conn, if set. */ + int NumCPUs; /**< How many CPUs should we try to use? */ +//int RunTesting; /**< If true, create testing circuits to measure how well the +// * other ORs are running. */ config_line_t *RendConfigLines; /**< List of configuration lines * for rendezvous services. */ config_line_t *HidServAuth; /**< List of configuration lines for client-side * authorizations for hidden services */ char *ContactInfo; /**< Contact info to be published in the directory. */ - char *HttpProxy; /**< hostname[:port] to use as http proxy, if any. */ - uint32_t HttpProxyAddr; /**< Parsed IPv4 addr for http proxy, if any. */ - uint16_t HttpProxyPort; /**< Parsed port for http proxy, if any. */ - char *HttpProxyAuthenticator; /**< username:password string, if any. */ + char *HTTPProxy; /**< hostname[:port] to use as http proxy, if any. */ + tor_addr_t HTTPProxyAddr; /**< Parsed IPv4 addr for http proxy, if any. */ + uint16_t HTTPProxyPort; /**< Parsed port for http proxy, if any. */ + char *HTTPProxyAuthenticator; /**< username:password string, if any. */ + + char *HTTPSProxy; /**< hostname[:port] to use as https proxy, if any. */ + tor_addr_t HTTPSProxyAddr; /**< Parsed addr for https proxy, if any. */ + uint16_t HTTPSProxyPort; /**< Parsed port for https proxy, if any. */ + char *HTTPSProxyAuthenticator; /**< username:password string, if any. */ - char *HttpsProxy; /**< hostname[:port] to use as https proxy, if any. */ - uint32_t HttpsProxyAddr; /**< Parsed IPv4 addr for https proxy, if any. */ - uint16_t HttpsProxyPort; /**< Parsed port for https proxy, if any. */ - char *HttpsProxyAuthenticator; /**< username:password string, if any. */ + char *Socks4Proxy; /**< hostname:port to use as a SOCKS4 proxy, if any. */ + tor_addr_t Socks4ProxyAddr; /**< Derived from Socks4Proxy. */ + uint16_t Socks4ProxyPort; /**< Derived from Socks4Proxy. */ + + char *Socks5Proxy; /**< hostname:port to use as a SOCKS5 proxy, if any. */ + tor_addr_t Socks5ProxyAddr; /**< Derived from Sock5Proxy. */ + uint16_t Socks5ProxyPort; /**< Derived from Socks5Proxy. */ + char *Socks5ProxyUsername; /**< Username for SOCKS5 authentication, if any */ + char *Socks5ProxyPassword; /**< Password for SOCKS5 authentication, if any */ /** List of configuration lines for replacement directory authorities. * If you just want to replace one class of authority at a time, @@ -2429,8 +2671,13 @@ typedef struct { * or not (1)? */ int ShutdownWaitLength; /**< When we get a SIGINT and we're a server, how * long do we wait before exiting? */ - int SafeLogging; /**< Boolean: are we allowed to log sensitive strings - * such as addresses (0), or do we scrub them first (1)? */ + char *SafeLogging; /**< Contains "relay", "1", "0" (meaning no scrubbing). */ + + /* Derived from SafeLogging */ + enum { + SAFELOG_SCRUB_ALL, SAFELOG_SCRUB_RELAY, SAFELOG_SCRUB_NONE + } _SafeLogging; + int SafeSocks; /**< Boolean: should we outright refuse application * connections that use socks4 or socks5-with-local-dns? */ #define LOG_PROTOCOL_WARN (get_options()->ProtocolWarnings ? \ @@ -2441,6 +2688,8 @@ typedef struct { * log whether it was DNS-leaking or not? */ int HardwareAccel; /**< Boolean: Should we enable OpenSSL hardware * acceleration where available? */ + char *AccelName; /**< Optional hardware acceleration engine name. */ + char *AccelDir; /**< Optional hardware acceleration engine search dir. */ int UseEntryGuards; /**< Boolean: Do we try to enter from a smallish number * of fixed nodes? */ int NumEntryGuards; /**< How many entry guards do we try to establish? */ @@ -2451,6 +2700,9 @@ typedef struct { * means directly from the authorities) no matter our other config? */ int FetchDirInfoEarly; + /** Should we fetch our dir info at the start of the consensus period? */ + int FetchDirInfoExtraEarly; + char *VirtualAddrNetwork; /**< Address and mask to hand out for virtual * MAPADDRESS requests. */ int ServerDNSSearchDomains; /**< Boolean: If set, we don't force exit @@ -2499,11 +2751,42 @@ typedef struct { * exit allows it, we use it. */ int AllowSingleHopCircuits; + /** If true, we convert "www.google.com.foo.exit" addresses on the + * socks/trans/natd ports into "www.google.com" addresses that + * exit from the node "foo". Disabled by default since attacking + * websites and exit relays can use it to manipulate your path + * selection. */ + int AllowDotExit; + + /** If true, we will warn if a user gives us only an IP address + * instead of a hostname. */ + int WarnUnsafeSocks; + + /** If true, the user wants us to collect statistics on clients + * requesting network statuses from us as directory. */ + int DirReqStatistics; + + /** If true, the user wants us to collect statistics on port usage. */ + int ExitPortStatistics; + + /** If true, the user wants us to collect cell statistics. */ + int CellStatistics; + + /** If true, the user wants us to collect statistics as entry node. */ + int EntryStatistics; + + /** If true, include statistics file contents in extra-info documents. */ + int ExtraInfoStatistics; + /** If true, do not believe anybody who tells us that a domain resolves * to an internal address, or that an internal address has a PTR mapping. * Helps avoid some cross-site attacks. */ int ClientDNSRejectInternalAddresses; + /** If true, do not accept any requests to connect to internal addresses + * over randomly chosen exits. */ + int ClientRejectInternalAddresses; + /** The length of time that we think a consensus should be fresh. */ int V3AuthVotingInterval; /** The length of time we think it will take to distribute votes. */ @@ -2517,6 +2800,13 @@ typedef struct { * migration purposes? */ int V3AuthUseLegacyKey; + /** Location of bandwidth measurement file */ + char *V3BandwidthsFile; + + /** Authority only: key=value pairs that we add to our networkstatus + * consensus vote on the 'params' line. */ + char *ConsensusParams; + /** The length of time that we think an initial consensus should be fresh. * Only altered on testing networks. */ int TestingV3AuthInitialVotingInterval; @@ -2553,19 +2843,6 @@ typedef struct { * the bridge authority guess which countries have blocked access to us. */ int BridgeRecordUsageByCountry; -#ifdef ENABLE_GEOIP_STATS - /** If true, and Tor is built with GEOIP_STATS support, and we're a - * directory, record how many directory requests we get from each country. */ - int DirRecordUsageByCountry; - /** Round all GeoIP results to the next multiple of this value, to avoid - * leaking information. */ - int DirRecordUsageGranularity; - /** Time interval: purge geoip stats after this long. */ - int DirRecordUsageRetainIPs; - /** Time interval: Flush geoip data to disk this often. */ - int DirRecordUsageSaveInterval; -#endif - /** Optionally, a file with GeoIP data. */ char *GeoIPFile; @@ -2573,6 +2850,26 @@ typedef struct { * to make this false. */ int ReloadTorrcOnSIGHUP; + /* The main parameter for picking circuits within a connection. + * + * If this value is positive, when picking a cell to relay on a connection, + * we always relay from the circuit whose weighted cell count is lowest. + * Cells are weighted exponentially such that if one cell is sent + * 'CircuitPriorityHalflife' seconds before another, it counts for half as + * much. + * + * If this value is zero, we're disabling the cell-EWMA algorithm. + * + * If this value is negative, we're using the default approach + * according to either Tor or a parameter set in the consensus. + */ + double CircuitPriorityHalflife; + + /** Set to true if the TestingTorNetwork configuration option is set. + * This is used so that options_validate() has a chance to realize that + * the defaults have changed. */ + int _UsingTestNetworkDefaults; + } or_options_t; /** Persistent state for an onion router, as saved to disk. */ @@ -2591,6 +2888,9 @@ typedef struct { uint64_t AccountingBytesReadInInterval; uint64_t AccountingBytesWrittenInInterval; int AccountingSecondsActive; + int AccountingSecondsToReachSoftLimit; + time_t AccountingSoftLimitHitAt; + uint64_t AccountingBytesAtSoftLimit; uint64_t AccountingExpectedUsage; /** A list of Entry Guard-related configuration lines. */ @@ -2601,13 +2901,30 @@ typedef struct { * bandwidth usage. The "Interval" fields hold the granularity, in seconds, * of the entries of Values. The "Values" lists hold decimal string * representations of the number of bytes read or written in each - * interval. */ + * interval. The "Maxima" list holds decimal strings describing the highest + * rate achieved during the interval. + */ time_t BWHistoryReadEnds; int BWHistoryReadInterval; smartlist_t *BWHistoryReadValues; + smartlist_t *BWHistoryReadMaxima; time_t BWHistoryWriteEnds; int BWHistoryWriteInterval; smartlist_t *BWHistoryWriteValues; + smartlist_t *BWHistoryWriteMaxima; + time_t BWHistoryDirReadEnds; + int BWHistoryDirReadInterval; + smartlist_t *BWHistoryDirReadValues; + smartlist_t *BWHistoryDirReadMaxima; + time_t BWHistoryDirWriteEnds; + int BWHistoryDirWriteInterval; + smartlist_t *BWHistoryDirWriteValues; + smartlist_t *BWHistoryDirWriteMaxima; + + /** Build time histogram */ + config_line_t * BuildtimeHistogram; + unsigned int TotalBuildTimes; + unsigned int CircuitBuildAbandonedCount; /** What version of Tor wrote this state file? */ char *TorVersion; @@ -2669,209 +2986,143 @@ struct socks_request_t { /* all the function prototypes go here */ -/********************************* buffers.c ***************************/ - -buf_t *buf_new(void); -buf_t *buf_new_with_capacity(size_t size); -void buf_free(buf_t *buf); -void buf_clear(buf_t *buf); -void buf_shrink(buf_t *buf); -void buf_shrink_freelists(int free_all); -void buf_dump_freelist_sizes(int severity); - -size_t buf_datalen(const buf_t *buf); -size_t buf_allocation(const buf_t *buf); -size_t buf_slack(const buf_t *buf); -const char *_buf_peek_raw_buffer(const buf_t *buf); - -int read_to_buf(int s, size_t at_most, buf_t *buf, int *reached_eof, - int *socket_error); -int read_to_buf_tls(tor_tls_t *tls, size_t at_most, buf_t *buf); - -int flush_buf(int s, buf_t *buf, size_t sz, size_t *buf_flushlen); -int flush_buf_tls(tor_tls_t *tls, buf_t *buf, size_t sz, size_t *buf_flushlen); - -int write_to_buf(const char *string, size_t string_len, buf_t *buf); -int write_to_buf_zlib(buf_t *buf, tor_zlib_state_t *state, - const char *data, size_t data_len, int done); -int move_buf_to_buf(buf_t *buf_out, buf_t *buf_in, size_t *buf_flushlen); -int fetch_from_buf(char *string, size_t string_len, buf_t *buf); -int fetch_var_cell_from_buf(buf_t *buf, var_cell_t **out, int linkproto); -int fetch_from_buf_http(buf_t *buf, - char **headers_out, size_t max_headerlen, - char **body_out, size_t *body_used, size_t max_bodylen, - int force_complete); -int fetch_from_buf_socks(buf_t *buf, socks_request_t *req, - int log_sockstype, int safe_socks); -int fetch_from_buf_line(buf_t *buf, char *data_out, size_t *data_len); - -int peek_buf_has_control0_command(buf_t *buf); - -void assert_buf_ok(buf_t *buf); - -#ifdef BUFFERS_PRIVATE -int buf_find_string_offset(const buf_t *buf, const char *s, size_t n); -#endif - /********************************* circuitbuild.c **********************/ -char *circuit_list_path(origin_circuit_t *circ, int verbose); -char *circuit_list_path_for_controller(origin_circuit_t *circ); -void circuit_log_path(int severity, unsigned int domain, - origin_circuit_t *circ); -void circuit_rep_hist_note_result(origin_circuit_t *circ); -origin_circuit_t *origin_circuit_init(uint8_t purpose, int flags); -origin_circuit_t *circuit_establish_circuit(uint8_t purpose, - extend_info_t *exit, - int flags); -int circuit_handle_first_hop(origin_circuit_t *circ); -void circuit_n_conn_done(or_connection_t *or_conn, int status); -int inform_testing_reachability(void); -int circuit_send_next_onion_skin(origin_circuit_t *circ); -void circuit_note_clock_jumped(int seconds_elapsed); -int circuit_extend(cell_t *cell, circuit_t *circ); -int circuit_init_cpath_crypto(crypt_path_t *cpath, const char *key_data, - int reverse); -int circuit_finish_handshake(origin_circuit_t *circ, uint8_t cell_type, - const uint8_t *reply); -int circuit_truncated(origin_circuit_t *circ, crypt_path_t *layer); -int onionskin_answer(or_circuit_t *circ, uint8_t cell_type, - const char *payload, const char *keys); -int circuit_all_predicted_ports_handled(time_t now, int *need_uptime, - int *need_capacity); - -int circuit_append_new_exit(origin_circuit_t *circ, extend_info_t *info); -int circuit_extend_to_new_exit(origin_circuit_t *circ, extend_info_t *info); -void onion_append_to_cpath(crypt_path_t **head_ptr, crypt_path_t *new_hop); -extend_info_t *extend_info_alloc(const char *nickname, const char *digest, - crypto_pk_env_t *onion_key, - const tor_addr_t *addr, uint16_t port); -extend_info_t *extend_info_from_router(routerinfo_t *r); -extend_info_t *extend_info_dup(extend_info_t *info); -void extend_info_free(extend_info_t *info); -routerinfo_t *build_state_get_exit_router(cpath_build_state_t *state); -const char *build_state_get_exit_nickname(cpath_build_state_t *state); - -void entry_guards_compute_status(void); -int entry_guard_register_connect_status(const char *digest, int succeeded, - int mark_relay_status, time_t now); -void entry_nodes_should_be_added(void); -int entry_list_can_grow(or_options_t *options); -routerinfo_t *choose_random_entry(cpath_build_state_t *state); -int entry_guards_parse_state(or_state_t *state, int set, char **msg); -void entry_guards_update_state(or_state_t *state); -int getinfo_helper_entry_guards(control_connection_t *conn, - const char *question, char **answer); - -void clear_bridge_list(void); -int routerinfo_is_a_configured_bridge(routerinfo_t *ri); -void bridge_add_from_config(const tor_addr_t *addr, uint16_t port, - char *digest); -void retry_bridge_descriptor_fetch_directly(const char *digest); -void fetch_bridge_descriptors(time_t now); -void learned_bridge_descriptor(routerinfo_t *ri, int from_cache); -int any_bridge_descriptors_known(void); -int any_pending_bridge_descriptor_fetches(void); -int bridges_known_but_down(void); -void bridges_retry_all(void); - -void entry_guards_free_all(void); - -/********************************* circuitlist.c ***********************/ - -circuit_t * _circuit_get_global_list(void); -const char *circuit_state_to_string(int state); -const char *circuit_purpose_to_controller_string(uint8_t purpose); -void circuit_dump_by_conn(connection_t *conn, int severity); -void circuit_set_p_circid_orconn(or_circuit_t *circ, circid_t id, - or_connection_t *conn); -void circuit_set_n_circid_orconn(circuit_t *circ, circid_t id, - or_connection_t *conn); -void circuit_set_state(circuit_t *circ, uint8_t state); -void circuit_close_all_marked(void); -int32_t circuit_initial_package_window(void); -origin_circuit_t *origin_circuit_new(void); -or_circuit_t *or_circuit_new(circid_t p_circ_id, or_connection_t *p_conn); -circuit_t *circuit_get_by_circid_orconn(circid_t circ_id, - or_connection_t *conn); -int circuit_id_in_use_on_orconn(circid_t circ_id, or_connection_t *conn); -circuit_t *circuit_get_by_edge_conn(edge_connection_t *conn); -void circuit_unlink_all_from_or_conn(or_connection_t *conn, int reason); -origin_circuit_t *circuit_get_by_global_id(uint32_t id); -origin_circuit_t *circuit_get_by_rend_query_and_purpose(const char *rend_query, - uint8_t purpose); -origin_circuit_t *circuit_get_next_by_pk_and_purpose(origin_circuit_t *start, - const char *digest, uint8_t purpose); -or_circuit_t *circuit_get_rendezvous(const char *cookie); -or_circuit_t *circuit_get_intro_point(const char *digest); -origin_circuit_t *circuit_find_to_cannibalize(uint8_t purpose, - extend_info_t *info, int flags); -void circuit_mark_all_unused_circs(void); -void circuit_expire_all_dirty_circs(void); -void _circuit_mark_for_close(circuit_t *circ, int reason, - int line, const char *file); -int circuit_get_cpath_len(origin_circuit_t *circ); -crypt_path_t *circuit_get_cpath_hop(origin_circuit_t *circ, int hopnum); -void circuit_get_all_pending_on_or_conn(smartlist_t *out, - or_connection_t *or_conn); -int circuit_count_pending_on_or_conn(or_connection_t *or_conn); - -#define circuit_mark_for_close(c, reason) \ - _circuit_mark_for_close((c), (reason), __LINE__, _SHORT_FILE_) - -void assert_cpath_layer_ok(const crypt_path_t *cp); -void assert_circuit_ok(const circuit_t *c); -void circuit_free_all(void); - -/********************************* circuituse.c ************************/ - -void circuit_expire_building(time_t now); -void circuit_remove_handled_ports(smartlist_t *needed_ports); -int circuit_stream_is_being_handled(edge_connection_t *conn, uint16_t port, - int min); -int circuit_conforms_to_options(const origin_circuit_t *circ, - const or_options_t *options); -void circuit_build_needed_circs(time_t now); -void circuit_detach_stream(circuit_t *circ, edge_connection_t *conn); - -void circuit_expire_old_circuits_serverside(time_t now); - -void reset_bandwidth_test(void); -int circuit_enough_testing_circs(void); - -void circuit_has_opened(origin_circuit_t *circ); -void circuit_build_failed(origin_circuit_t *circ); - -/** Flag to set when a circuit should have only a single hop. */ -#define CIRCLAUNCH_ONEHOP_TUNNEL (1<<0) -/** Flag to set when a circuit needs to be built of high-uptime nodes */ -#define CIRCLAUNCH_NEED_UPTIME (1<<1) -/** Flag to set when a circuit needs to be built of high-capacity nodes */ -#define CIRCLAUNCH_NEED_CAPACITY (1<<2) -/** Flag to set when the last hop of a circuit doesn't need to be an - * exit node. */ -#define CIRCLAUNCH_IS_INTERNAL (1<<3) -origin_circuit_t *circuit_launch_by_extend_info(uint8_t purpose, - extend_info_t *info, - int flags); -origin_circuit_t *circuit_launch_by_router(uint8_t purpose, - routerinfo_t *exit, int flags); -void circuit_reset_failure_count(int timeout); -int connection_ap_handshake_attach_chosen_circuit(edge_connection_t *conn, - origin_circuit_t *circ, - crypt_path_t *cpath); -int connection_ap_handshake_attach_circuit(edge_connection_t *conn); - -/********************************* command.c ***************************/ - -void command_process_cell(cell_t *cell, or_connection_t *conn); -void command_process_var_cell(var_cell_t *cell, or_connection_t *conn); - -extern uint64_t stats_n_padding_cells_processed; -extern uint64_t stats_n_create_cells_processed; -extern uint64_t stats_n_created_cells_processed; -extern uint64_t stats_n_relay_cells_processed; -extern uint64_t stats_n_destroy_cells_processed; +/** How many hops does a general-purpose circuit have by default? */ +#define DEFAULT_ROUTE_LEN 3 + +/* Circuit Build Timeout "public" structures. */ + +/** Precision multiplier for the Bw weights */ +#define BW_WEIGHT_SCALE 10000 +#define BW_MIN_WEIGHT_SCALE 1 +#define BW_MAX_WEIGHT_SCALE INT32_MAX + +/** Total size of the circuit timeout history to accumulate. + * 1000 is approx 2.5 days worth of continual-use circuits. */ +#define CBT_NCIRCUITS_TO_OBSERVE 1000 + +/** Width of the histogram bins in milliseconds */ +#define CBT_BIN_WIDTH ((build_time_t)50) + +/** Number of modes to use in the weighted-avg computation of Xm */ +#define CBT_DEFAULT_NUM_XM_MODES 3 +#define CBT_MIN_NUM_XM_MODES 1 +#define CBT_MAX_NUM_XM_MODES 20 + +/** A build_time_t is milliseconds */ +typedef uint32_t build_time_t; + +/** + * CBT_BUILD_ABANDONED is our flag value to represent a force-closed + * circuit (Aka a 'right-censored' pareto value). + */ +#define CBT_BUILD_ABANDONED ((build_time_t)(INT32_MAX-1)) +#define CBT_BUILD_TIME_MAX ((build_time_t)(INT32_MAX)) + +/** Save state every 10 circuits */ +#define CBT_SAVE_STATE_EVERY 10 + +/* Circuit build times consensus parameters */ + +/** + * How long to wait before actually closing circuits that take too long to + * build in terms of CDF quantile. + */ +#define CBT_DEFAULT_CLOSE_QUANTILE 95 +#define CBT_MIN_CLOSE_QUANTILE CBT_MIN_QUANTILE_CUTOFF +#define CBT_MAX_CLOSE_QUANTILE CBT_MAX_QUANTILE_CUTOFF + +/** + * How many circuits count as recent when considering if the + * connection has gone gimpy or changed. + */ +#define CBT_DEFAULT_RECENT_CIRCUITS 20 +#define CBT_MIN_RECENT_CIRCUITS 3 +#define CBT_MAX_RECENT_CIRCUITS 1000 + +/** + * Maximum count of timeouts that finish the first hop in the past + * RECENT_CIRCUITS before calculating a new timeout. + * + * This tells us whether to abandon timeout history and set + * the timeout back to whatever circuit_build_times_get_initial_timeout() + * gives us. + */ +#define CBT_DEFAULT_MAX_RECENT_TIMEOUT_COUNT (CBT_DEFAULT_RECENT_CIRCUITS*9/10) +#define CBT_MIN_MAX_RECENT_TIMEOUT_COUNT 3 +#define CBT_MAX_MAX_RECENT_TIMEOUT_COUNT 10000 + +/** Minimum circuits before estimating a timeout */ +#define CBT_DEFAULT_MIN_CIRCUITS_TO_OBSERVE 100 +#define CBT_MIN_MIN_CIRCUITS_TO_OBSERVE 1 +#define CBT_MAX_MIN_CIRCUITS_TO_OBSERVE 10000 + +/** Cutoff percentile on the CDF for our timeout estimation. */ +#define CBT_DEFAULT_QUANTILE_CUTOFF 80 +#define CBT_MIN_QUANTILE_CUTOFF 10 +#define CBT_MAX_QUANTILE_CUTOFF 99 +double circuit_build_times_quantile_cutoff(void); + +/** How often in seconds should we build a test circuit */ +#define CBT_DEFAULT_TEST_FREQUENCY 60 +#define CBT_MIN_TEST_FREQUENCY 1 +#define CBT_MAX_TEST_FREQUENCY INT32_MAX + +/** Lowest allowable value for CircuitBuildTimeout in milliseconds */ +#define CBT_DEFAULT_TIMEOUT_MIN_VALUE (1500) +#define CBT_MIN_TIMEOUT_MIN_VALUE 500 +#define CBT_MAX_TIMEOUT_MIN_VALUE INT32_MAX + +/** Initial circuit build timeout in milliseconds */ +#define CBT_DEFAULT_TIMEOUT_INITIAL_VALUE (60*1000) +#define CBT_MIN_TIMEOUT_INITIAL_VALUE CBT_MIN_TIMEOUT_MIN_VALUE +#define CBT_MAX_TIMEOUT_INITIAL_VALUE INT32_MAX +int32_t circuit_build_times_initial_timeout(void); + +#if CBT_DEFAULT_MAX_RECENT_TIMEOUT_COUNT < CBT_MIN_MAX_RECENT_TIMEOUT_COUNT +#error "RECENT_CIRCUITS is set too low." +#endif + +/** Information about the state of our local network connection */ +typedef struct { + /** The timestamp we last completed a TLS handshake or received a cell */ + time_t network_last_live; + /** If the network is not live, how many timeouts has this caused? */ + int nonlive_timeouts; + /** Circular array of circuits that have made it to the first hop. Slot is + * 1 if circuit timed out, 0 if circuit succeeded */ + int8_t *timeouts_after_firsthop; + /** Number of elements allocated for the above array */ + int num_recent_circs; + /** Index into circular array. */ + int after_firsthop_idx; +} network_liveness_t; + +/** Structure for circuit build times history */ +typedef struct { + /** The circular array of recorded build times in milliseconds */ + build_time_t circuit_build_times[CBT_NCIRCUITS_TO_OBSERVE]; + /** Current index in the circuit_build_times circular array */ + int build_times_idx; + /** Total number of build times accumulated. Max CBT_NCIRCUITS_TO_OBSERVE */ + int total_build_times; + /** Information about the state of our local network connection */ + network_liveness_t liveness; + /** Last time we built a circuit. Used to decide to build new test circs */ + time_t last_circ_at; + /** "Minimum" value of our pareto distribution (actually mode) */ + build_time_t Xm; + /** alpha exponent for pareto dist. */ + double alpha; + /** Have we computed a timeout? */ + int have_computed_timeout; + /** The exact value for that timeout in milliseconds. Stored as a double + * to maintain precision from calculations to and from quantile value. */ + double timeout_ms; + /** How long we wait before actually closing the circuit. */ + double close_ms; +} circuit_build_times_t; /********************************* config.c ***************************/ @@ -2884,203 +3135,8 @@ typedef enum setopt_err_t { SETOPT_ERR_SETTING = -4, } setopt_err_t; -const char *get_dirportfrontpage(void); -or_options_t *get_options(void); -int set_options(or_options_t *new_val, char **msg); -void config_free_all(void); -const char *safe_str(const char *address); -const char *escaped_safe_str(const char *address); -const char *get_version(void); - -int config_get_lines(const char *string, config_line_t **result); -void config_free_lines(config_line_t *front); -setopt_err_t options_trial_assign(config_line_t *list, int use_defaults, - int clear_first, char **msg); -int resolve_my_address(int warn_severity, or_options_t *options, - uint32_t *addr, char **hostname_out); -int is_local_addr(const tor_addr_t *addr) ATTR_PURE; -void options_init(or_options_t *options); -int options_init_from_torrc(int argc, char **argv); -setopt_err_t options_init_from_string(const char *cf, - int command, const char *command_arg, char **msg); -int option_is_recognized(const char *key); -const char *option_get_canonical_name(const char *key); -config_line_t *option_get_assignment(or_options_t *options, - const char *key); -int options_save_current(void); -const char *get_torrc_fname(void); -char *options_get_datadir_fname2_suffix(or_options_t *options, - const char *sub1, const char *sub2, - const char *suffix); -#define get_datadir_fname2_suffix(sub1, sub2, suffix) \ - options_get_datadir_fname2_suffix(get_options(), (sub1), (sub2), (suffix)) -/** Return a newly allocated string containing datadir/sub1. See - * get_datadir_fname2_suffix. */ -#define get_datadir_fname(sub1) get_datadir_fname2_suffix((sub1), NULL, NULL) -/** Return a newly allocated string containing datadir/sub1/sub2. See - * get_datadir_fname2_suffix. */ -#define get_datadir_fname2(sub1,sub2) \ - get_datadir_fname2_suffix((sub1), (sub2), NULL) -/** Return a newly allocated string containing datadir/sub1suffix. See - * get_datadir_fname2_suffix. */ -#define get_datadir_fname_suffix(sub1, suffix) \ - get_datadir_fname2_suffix((sub1), NULL, (suffix)) - -or_state_t *get_or_state(void); -int or_state_save(time_t now); - -int options_need_geoip_info(or_options_t *options, const char **reason_out); -int getinfo_helper_config(control_connection_t *conn, - const char *question, char **answer); - -uint32_t get_effective_bwrate(or_options_t *options); -uint32_t get_effective_bwburst(or_options_t *options); - -#ifdef CONFIG_PRIVATE -/* Used only by config.c and test.c */ -or_options_t *options_new(void); -#endif - -/********************************* connection.c ***************************/ - -const char *conn_type_to_string(int type); -const char *conn_state_to_string(int type, int state); - -dir_connection_t *dir_connection_new(int socket_family); -or_connection_t *or_connection_new(int socket_family); -edge_connection_t *edge_connection_new(int type, int socket_family); -control_connection_t *control_connection_new(int socket_family); -connection_t *connection_new(int type, int socket_family); - -void connection_link_connections(connection_t *conn_a, connection_t *conn_b); -void connection_unregister_events(connection_t *conn); -void connection_free(connection_t *conn); -void connection_free_all(void); -void connection_about_to_close_connection(connection_t *conn); -void connection_close_immediate(connection_t *conn); -void _connection_mark_for_close(connection_t *conn,int line, const char *file); - -#define connection_mark_for_close(c) \ - _connection_mark_for_close((c), __LINE__, _SHORT_FILE_) - -void connection_expire_held_open(void); - -int connection_connect(connection_t *conn, const char *address, - const tor_addr_t *addr, - uint16_t port, int *socket_error); -int retry_all_listeners(smartlist_t *replaced_conns, - smartlist_t *new_conns); - -ssize_t connection_bucket_write_limit(connection_t *conn, time_t now); -int global_write_bucket_low(connection_t *conn, size_t attempt, int priority); -void connection_bucket_init(void); -void connection_bucket_refill(int seconds_elapsed, time_t now); - -int connection_handle_read(connection_t *conn); - -int connection_fetch_from_buf(char *string, size_t len, connection_t *conn); - -int connection_wants_to_flush(connection_t *conn); -int connection_outbuf_too_full(connection_t *conn); -int connection_handle_write(connection_t *conn, int force); -void _connection_write_to_buf_impl(const char *string, size_t len, - connection_t *conn, int zlib); -static void connection_write_to_buf(const char *string, size_t len, - connection_t *conn); -static void connection_write_to_buf_zlib(const char *string, size_t len, - dir_connection_t *conn, int done); -static INLINE void -connection_write_to_buf(const char *string, size_t len, connection_t *conn) -{ - _connection_write_to_buf_impl(string, len, conn, 0); -} -static INLINE void -connection_write_to_buf_zlib(const char *string, size_t len, - dir_connection_t *conn, int done) -{ - _connection_write_to_buf_impl(string, len, TO_CONN(conn), done ? -1 : 1); -} - -connection_t *connection_get_by_global_id(uint64_t id); - -connection_t *connection_get_by_type(int type); -connection_t *connection_get_by_type_purpose(int type, int purpose); -connection_t *connection_get_by_type_addr_port_purpose(int type, - const tor_addr_t *addr, - uint16_t port, int purpose); -connection_t *connection_get_by_type_state(int type, int state); -connection_t *connection_get_by_type_state_rendquery(int type, int state, - const char *rendquery, - int rendversion); - -#define connection_speaks_cells(conn) ((conn)->type == CONN_TYPE_OR) -int connection_is_listener(connection_t *conn); -int connection_state_is_open(connection_t *conn); -int connection_state_is_connecting(connection_t *conn); - -char *alloc_http_authenticator(const char *authenticator); - -void assert_connection_ok(connection_t *conn, time_t now); -int connection_or_nonopen_was_started_here(or_connection_t *conn); -void connection_dump_buffer_mem_stats(int severity); -void remove_file_if_very_old(const char *fname, time_t now); - /********************************* connection_edge.c *************************/ -#define connection_mark_unattached_ap(conn, endreason) \ - _connection_mark_unattached_ap((conn), (endreason), __LINE__, _SHORT_FILE_) - -void _connection_mark_unattached_ap(edge_connection_t *conn, int endreason, - int line, const char *file); -int connection_edge_reached_eof(edge_connection_t *conn); -int connection_edge_process_inbuf(edge_connection_t *conn, - int package_partial); -int connection_edge_destroy(circid_t circ_id, edge_connection_t *conn); -int connection_edge_end(edge_connection_t *conn, uint8_t reason); -int connection_edge_end_errno(edge_connection_t *conn); -int connection_edge_finished_flushing(edge_connection_t *conn); -int connection_edge_finished_connecting(edge_connection_t *conn); - -int connection_ap_handshake_send_begin(edge_connection_t *ap_conn); -int connection_ap_handshake_send_resolve(edge_connection_t *ap_conn); - -edge_connection_t *connection_ap_make_link(char *address, uint16_t port, - const char *digest, - int use_begindir, int want_onehop); -void connection_ap_handshake_socks_reply(edge_connection_t *conn, char *reply, - size_t replylen, - int endreason); -void connection_ap_handshake_socks_resolved(edge_connection_t *conn, - int answer_type, - size_t answer_len, - const uint8_t *answer, - int ttl, - time_t expires); - -int connection_exit_begin_conn(cell_t *cell, circuit_t *circ); -int connection_exit_begin_resolve(cell_t *cell, or_circuit_t *circ); -void connection_exit_connect(edge_connection_t *conn); -int connection_edge_is_rendezvous_stream(edge_connection_t *conn); -int connection_ap_can_use_exit(edge_connection_t *conn, routerinfo_t *exit); -void connection_ap_expire_beginning(void); -void connection_ap_attach_pending(void); -void connection_ap_fail_onehop(const char *failed_digest, - cpath_build_state_t *build_state); -void circuit_discard_optional_exit_enclaves(extend_info_t *info); -int connection_ap_detach_retriable(edge_connection_t *conn, - origin_circuit_t *circ, - int reason); -int connection_ap_process_transparent(edge_connection_t *conn); - -int address_is_invalid_destination(const char *address, int client); - -void addressmap_init(void); -void addressmap_clean(time_t now); -void addressmap_clear_configured(void); -void addressmap_clear_transient(void); -void addressmap_free_all(void); -int addressmap_rewrite(char *address, size_t maxlen, time_t *expires_out); -int addressmap_have_mapping(const char *address, int update_timeout); /** Enumerates possible origins of a client-side address mapping. */ typedef enum { /** We're remapping this address because the controller told us to. */ @@ -3095,75 +3151,6 @@ typedef enum { * Tor server that told us what its value was. */ ADDRMAPSRC_DNS, } addressmap_entry_source_t; -void addressmap_register(const char *address, char *new_address, - time_t expires, addressmap_entry_source_t source); -int parse_virtual_addr_network(const char *val, int validate_only, - char **msg); -int client_dns_incr_failures(const char *address); -void client_dns_clear_failures(const char *address); -void client_dns_set_addressmap(const char *address, uint32_t val, - const char *exitname, int ttl); -const char *addressmap_register_virtual_address(int type, char *new_address); -void addressmap_get_mappings(smartlist_t *sl, time_t min_expires, - time_t max_expires, int want_expiry); -int connection_ap_handshake_rewrite_and_attach(edge_connection_t *conn, - origin_circuit_t *circ, - crypt_path_t *cpath); -int hostname_is_noconnect_address(const char *address); - -/** Possible return values for parse_extended_hostname. */ -typedef enum hostname_type_t { - NORMAL_HOSTNAME, ONION_HOSTNAME, EXIT_HOSTNAME, BAD_HOSTNAME -} hostname_type_t; -hostname_type_t parse_extended_hostname(char *address); - -#if defined(HAVE_NET_IF_H) && defined(HAVE_NET_PFVAR_H) -int get_pf_socket(void); -#endif - -/********************************* connection_or.c ***************************/ - -void connection_or_remove_from_identity_map(or_connection_t *conn); -void connection_or_clear_identity_map(void); -or_connection_t *connection_or_get_for_extend(const char *digest, - const tor_addr_t *target_addr, - const char **msg_out, - int *launch_out); -void connection_or_set_bad_connections(void); - -int connection_or_reached_eof(or_connection_t *conn); -int connection_or_process_inbuf(or_connection_t *conn); -int connection_or_flushed_some(or_connection_t *conn); -int connection_or_finished_flushing(or_connection_t *conn); -int connection_or_finished_connecting(or_connection_t *conn); - -void connection_or_connect_failed(or_connection_t *conn, - int reason, const char *msg); -or_connection_t *connection_or_connect(const tor_addr_t *addr, uint16_t port, - const char *id_digest); - -int connection_tls_start_handshake(or_connection_t *conn, int receiving); -int connection_tls_continue_handshake(or_connection_t *conn); - -void or_handshake_state_free(or_handshake_state_t *state); -int connection_or_set_state_open(or_connection_t *conn); -void connection_or_write_cell_to_buf(const cell_t *cell, - or_connection_t *conn); -void connection_or_write_var_cell_to_buf(const var_cell_t *cell, - or_connection_t *conn); -int connection_or_send_destroy(circid_t circ_id, or_connection_t *conn, - int reason); -int connection_or_send_netinfo(or_connection_t *conn); -int connection_or_send_cert(or_connection_t *conn); -int connection_or_send_link_auth(or_connection_t *conn); -int connection_or_compute_link_auth_hmac(or_connection_t *conn, - char *hmac_out); -int is_or_protocol_version_known(uint16_t version); - -void cell_pack(packed_cell_t *dest, const cell_t *src); -void var_cell_pack_header(const var_cell_t *cell, char *hdr_out); -var_cell_t *var_cell_new(uint16_t payload_len); -void var_cell_free(var_cell_t *cell); /********************************* control.c ***************************/ @@ -3201,8 +3188,14 @@ typedef enum or_conn_status_event_t { OR_CONN_EVENT_NEW = 4, } or_conn_status_event_t; -void control_update_global_event_mask(void); -void control_adjust_event_log_severity(void); +/** Used to indicate the type of a buildtime event */ +typedef enum buildtimeout_set_event_t { + BUILDTIMEOUT_SET_EVENT_COMPUTED = 0, + BUILDTIMEOUT_SET_EVENT_RESET = 1, + BUILDTIMEOUT_SET_EVENT_SUSPENDED = 2, + BUILDTIMEOUT_SET_EVENT_DISCARD = 3, + BUILDTIMEOUT_SET_EVENT_RESUME = 4 +} buildtimeout_set_event_t; /** Execute the statement <b>stmt</b>, which may log events concerning the * connection <b>conn</b>. To prevent infinite loops, disable log messages @@ -3212,7 +3205,9 @@ void control_adjust_event_log_severity(void); */ #define CONN_LOG_PROTECT(conn, stmt) \ STMT_BEGIN \ - int _log_conn_is_control = (conn && conn->type == CONN_TYPE_CONTROL); \ + int _log_conn_is_control; \ + tor_assert(conn); \ + _log_conn_is_control = (conn->type == CONN_TYPE_CONTROL); \ if (_log_conn_is_control) \ disable_control_logging(); \ STMT_BEGIN stmt; STMT_END; \ @@ -3220,58 +3215,6 @@ void control_adjust_event_log_severity(void); enable_control_logging(); \ STMT_END -/** Log information about the connection <b>conn</b>, protecting it as with - * CONN_LOG_PROTECT. Example: - * - * LOG_FN_CONN(conn, (LOG_DEBUG, "Socket %d wants to write", conn->s)); - **/ -#define LOG_FN_CONN(conn, args) \ - CONN_LOG_PROTECT(conn, log_fn args) - -int connection_control_finished_flushing(control_connection_t *conn); -int connection_control_reached_eof(control_connection_t *conn); -int connection_control_process_inbuf(control_connection_t *conn); - -#define EVENT_AUTHDIR_NEWDESCS 0x000D -#define EVENT_NS 0x000F -int control_event_is_interesting(int event); - -int control_event_circuit_status(origin_circuit_t *circ, - circuit_status_event_t e, int reason); -int control_event_stream_status(edge_connection_t *conn, - stream_status_event_t e, - int reason); -int control_event_or_conn_status(or_connection_t *conn, - or_conn_status_event_t e, int reason); -int control_event_bandwidth_used(uint32_t n_read, uint32_t n_written); -int control_event_stream_bandwidth(edge_connection_t *edge_conn); -int control_event_stream_bandwidth_used(void); -void control_event_logmsg(int severity, unsigned int domain, const char *msg); -int control_event_descriptors_changed(smartlist_t *routers); -int control_event_address_mapped(const char *from, const char *to, - time_t expires, const char *error); -int control_event_or_authdir_new_descriptor(const char *action, - const char *desc, - size_t desclen, - const char *msg); -int control_event_my_descriptor_changed(void); -int control_event_networkstatus_changed(smartlist_t *statuses); -int control_event_newconsensus(const networkstatus_t *consensus); -int control_event_networkstatus_changed_single(routerstatus_t *rs); -int control_event_general_status(int severity, const char *format, ...) - CHECK_PRINTF(2,3); -int control_event_client_status(int severity, const char *format, ...) - CHECK_PRINTF(2,3); -int control_event_server_status(int severity, const char *format, ...) - CHECK_PRINTF(2,3); -int control_event_guard(const char *nickname, const char *digest, - const char *status); - -int init_cookie_authentication(int enabled); -smartlist_t *decode_hashed_passwords(config_line_t *passwords); -void disable_control_logging(void); -void enable_control_logging(void); - /** Enum describing various stages of bootstrapping, for use with controller * bootstrap status events. The values range from 0 to 100. */ typedef enum { @@ -3292,427 +3235,133 @@ typedef enum { BOOTSTRAP_STATUS_DONE=100 } bootstrap_status_t; -void control_event_bootstrap(bootstrap_status_t status, int progress); -void control_event_bootstrap_problem(const char *warn, int reason); - -void control_event_clients_seen(const char *timestarted, - const char *countries); - -#ifdef CONTROL_PRIVATE -/* Used only by control.c and test.c */ -size_t write_escaped_data(const char *data, size_t len, char **out); -size_t read_escaped_data(const char *data, size_t len, char **out); -#endif - -/********************************* cpuworker.c *****************************/ - -void cpu_init(void); -void cpuworkers_rotate(void); -int connection_cpu_finished_flushing(connection_t *conn); -int connection_cpu_reached_eof(connection_t *conn); -int connection_cpu_process_inbuf(connection_t *conn); -int assign_onionskin_to_cpuworker(connection_t *cpuworker, - or_circuit_t *circ, - char *onionskin); - /********************************* directory.c ***************************/ -int directories_have_accepted_server_descriptor(void); -char *authority_type_to_string(authority_type_t auth); -void directory_post_to_dirservers(uint8_t dir_purpose, uint8_t router_purpose, - authority_type_t type, const char *payload, - size_t payload_len, size_t extrainfo_len); -void directory_get_from_dirserver(uint8_t dir_purpose, uint8_t router_purpose, - const char *resource, - int pds_flags); -void directory_get_from_all_authorities(uint8_t dir_purpose, - uint8_t router_purpose, - const char *resource); -void directory_initiate_command_routerstatus(routerstatus_t *status, - uint8_t dir_purpose, - uint8_t router_purpose, - int anonymized_connection, - const char *resource, - const char *payload, - size_t payload_len, - time_t if_modified_since); -void directory_initiate_command_routerstatus_rend(routerstatus_t *status, - uint8_t dir_purpose, - uint8_t router_purpose, - int anonymized_connection, - const char *resource, - const char *payload, - size_t payload_len, - time_t if_modified_since, - const rend_data_t *rend_query); - -int parse_http_response(const char *headers, int *code, time_t *date, - compress_method_t *compression, char **response); - -int connection_dir_is_encrypted(dir_connection_t *conn); -int connection_dir_reached_eof(dir_connection_t *conn); -int connection_dir_process_inbuf(dir_connection_t *conn); -int connection_dir_finished_flushing(dir_connection_t *conn); -int connection_dir_finished_connecting(dir_connection_t *conn); -void connection_dir_request_failed(dir_connection_t *conn); -void directory_initiate_command(const char *address, const tor_addr_t *addr, - uint16_t or_port, uint16_t dir_port, - int supports_conditional_consensus, - int supports_begindir, const char *digest, - uint8_t dir_purpose, uint8_t router_purpose, - int anonymized_connection, - const char *resource, - const char *payload, size_t payload_len, - time_t if_modified_since); - -int dir_split_resource_into_fingerprints(const char *resource, - smartlist_t *fp_out, int *compresseed_out, - int decode_hex, int sort_uniq); /** A pair of digests created by dir_split_resource_info_fingerprint_pairs() */ typedef struct { char first[DIGEST_LEN]; char second[DIGEST_LEN]; } fp_pair_t; -int dir_split_resource_into_fingerprint_pairs(const char *res, - smartlist_t *pairs_out); -char *directory_dump_request_log(void); -void note_request(const char *key, size_t bytes); -int router_supports_extrainfo(const char *identity_digest, int is_authority); - -time_t download_status_increment_failure(download_status_t *dls, - int status_code, const char *item, - int server, time_t now); -/** Increment the failure count of the download_status_t <b>dls</b>, with - * the optional status code <b>sc</b>. */ -#define download_status_failed(dls, sc) \ - download_status_increment_failure((dls), (sc), NULL, \ - get_options()->DirPort, time(NULL)) - -void download_status_reset(download_status_t *dls); -static int download_status_is_ready(download_status_t *dls, time_t now, - int max_failures); -/** Return true iff, as of <b>now</b>, the resource tracked by <b>dls</b> is - * ready to get its download reattempted. */ -static INLINE int -download_status_is_ready(download_status_t *dls, time_t now, - int max_failures) -{ - return (dls->n_download_failures <= max_failures - && dls->next_attempt_at <= now); -} - -static void download_status_mark_impossible(download_status_t *dl); -/** Mark <b>dl</b> as never downloadable. */ -static INLINE void -download_status_mark_impossible(download_status_t *dl) -{ - dl->n_download_failures = IMPOSSIBLE_TO_DOWNLOAD; -} /********************************* dirserv.c ***************************/ -/** Maximum length of an exit policy summary. */ -#define MAX_EXITPOLICY_SUMMARY_LEN (1000) - -/** Maximum allowable length of a version line in a networkstatus. */ -#define MAX_V_LINE_LEN 128 -/** Length of "r Authority BadDirectory BadExit Exit Fast Guard HSDir Named - * Running Stable Unnamed V2Dir Valid\n". */ -#define MAX_FLAG_LINE_LEN 96 -/** Length of "w" line for weighting. Currently at most - * "w Bandwidth=<uint32t>\n" */ -#define MAX_WEIGHT_LINE_LEN (13+10) -/** Maximum length of an exit policy summary line. */ -#define MAX_POLICY_LINE_LEN (3+MAX_EXITPOLICY_SUMMARY_LEN) -/** Amount of space to allocate for each entry: r, s, and v lines. */ -#define RS_ENTRY_LEN \ - ( /* first line */ \ - MAX_NICKNAME_LEN+BASE64_DIGEST_LEN*2+ISO_TIME_LEN+INET_NTOA_BUF_LEN+ \ - 5*2 /* ports */ + 10 /* punctuation */ + \ - /* second line */ \ - MAX_FLAG_LINE_LEN + \ - /* weight line */ \ - MAX_WEIGHT_LINE_LEN + \ - /* p line. */ \ - MAX_POLICY_LINE_LEN + \ - /* v line. */ \ - MAX_V_LINE_LEN \ - ) -#define UNNAMED_ROUTER_NICKNAME "Unnamed" - -int connection_dirserv_flushed_some(dir_connection_t *conn); - -int dirserv_add_own_fingerprint(const char *nickname, crypto_pk_env_t *pk); -int dirserv_load_fingerprint_file(void); -void dirserv_free_fingerprint_list(void); -const char *dirserv_get_nickname_by_digest(const char *digest); -enum was_router_added_t dirserv_add_multiple_descriptors( - const char *desc, uint8_t purpose, - const char *source, - const char **msg); -enum was_router_added_t dirserv_add_descriptor(routerinfo_t *ri, - const char **msg, - const char *source); -int getinfo_helper_dirserv_unregistered(control_connection_t *conn, - const char *question, char **answer); -void dirserv_free_descriptors(void); -void dirserv_set_router_is_running(routerinfo_t *router, time_t now); -int list_server_status_v1(smartlist_t *routers, char **router_status_out, - int for_controller); -int dirserv_dump_directory_to_string(char **dir_out, - crypto_pk_env_t *private_key); - -int directory_fetches_from_authorities(or_options_t *options); -int directory_fetches_dir_info_early(or_options_t *options); -int directory_fetches_dir_info_later(or_options_t *options); -int directory_caches_v2_dir_info(or_options_t *options); -#define directory_caches_v1_dir_info(o) directory_caches_v2_dir_info(o) -int directory_caches_dir_info(or_options_t *options); -int directory_permits_begindir_requests(or_options_t *options); -int directory_permits_controller_requests(or_options_t *options); -int directory_too_idle_to_fetch_descriptors(or_options_t *options, time_t now); - -void directory_set_dirty(void); -cached_dir_t *dirserv_get_directory(void); -cached_dir_t *dirserv_get_runningrouters(void); -cached_dir_t *dirserv_get_consensus(void); -void dirserv_set_cached_directory(const char *directory, time_t when, - int is_running_routers); -void dirserv_set_cached_networkstatus_v2(const char *directory, - const char *identity, - time_t published); -void dirserv_set_cached_networkstatus_v3(const char *consensus, - time_t published); -void dirserv_clear_old_networkstatuses(time_t cutoff); -void dirserv_clear_old_v1_info(time_t now); -void dirserv_get_networkstatus_v2(smartlist_t *result, const char *key); -void dirserv_get_networkstatus_v2_fingerprints(smartlist_t *result, - const char *key); -int dirserv_get_routerdesc_fingerprints(smartlist_t *fps_out, const char *key, - const char **msg, - int for_unencrypted_conn, - int is_extrainfo); -int dirserv_get_routerdescs(smartlist_t *descs_out, const char *key, - const char **msg); -void dirserv_orconn_tls_done(const char *address, - uint16_t or_port, - const char *digest_rcvd, - int as_advertised); -void dirserv_test_reachability(time_t now, int try_all); -int authdir_wants_to_reject_router(routerinfo_t *ri, const char **msg, - int complain); -int dirserv_would_reject_router(routerstatus_t *rs); -int dirserv_remove_old_statuses(smartlist_t *fps, time_t cutoff); -int dirserv_have_any_serverdesc(smartlist_t *fps, int spool_src); -size_t dirserv_estimate_data_size(smartlist_t *fps, int is_serverdescs, - int compressed); -int routerstatus_format_entry(char *buf, size_t buf_len, - routerstatus_t *rs, const char *platform, - int first_line_only, int v2_format); -void dirserv_free_all(void); -void cached_dir_decref(cached_dir_t *d); -cached_dir_t *new_cached_dir(char *s, time_t published); -/********************************* dirvote.c ************************/ +/** An enum to describe what format we're generating a routerstatus line in. + */ +typedef enum { + /** For use in a v2 opinion */ + NS_V2, + /** For use in a consensus networkstatus document (ns flavor) */ + NS_V3_CONSENSUS, + /** For use in a vote networkstatus document */ + NS_V3_VOTE, + /** For passing to the controlport in response to a GETINFO request */ + NS_CONTROL_PORT, + /** For use in a consensus networkstatus document (microdesc flavor) */ + NS_V3_CONSENSUS_MICRODESC +} routerstatus_format_type_t; + +#ifdef DIRSERV_PRIVATE +typedef struct measured_bw_line_t { + char node_id[DIGEST_LEN]; + char node_hex[MAX_HEX_NICKNAME_LEN+1]; + long int bw; +} measured_bw_line_t; -/** Lowest allowable value for VoteSeconds. */ -#define MIN_VOTE_SECONDS 20 -/** Lowest allowable value for DistSeconds. */ -#define MIN_DIST_SECONDS 20 -/** Smallest allowable voting interval. */ -#define MIN_VOTE_INTERVAL 300 - -void dirvote_free_all(void); - -/* vote manipulation */ -char *networkstatus_compute_consensus(smartlist_t *votes, - int total_authorities, - crypto_pk_env_t *identity_key, - crypto_pk_env_t *signing_key, - const char *legacy_identity_key_digest, - crypto_pk_env_t *legacy_signing_key); -int networkstatus_add_detached_signatures(networkstatus_t *target, - ns_detached_signatures_t *sigs, - const char **msg_out); -char *networkstatus_get_detached_signatures(networkstatus_t *consensus); -void ns_detached_signatures_free(ns_detached_signatures_t *s); - -/* cert manipulation */ -authority_cert_t *authority_cert_dup(authority_cert_t *cert); +#endif + +/********************************* dirvote.c ************************/ /** Describes the schedule by which votes should be generated. */ typedef struct vote_timing_t { + /** Length in seconds between one consensus becoming valid and the next + * becoming valid. */ int vote_interval; + /** For how many intervals is a consensus valid? */ int n_intervals_valid; + /** Time in seconds allowed to propagate votes */ int vote_delay; + /** Time in seconds allowed to propagate signatures */ int dist_delay; } vote_timing_t; -/* vote scheduling */ -void dirvote_get_preferred_voting_intervals(vote_timing_t *timing_out); -time_t dirvote_get_start_of_next_interval(time_t now, int interval); -void dirvote_recalculate_timing(or_options_t *options, time_t now); -void dirvote_act(or_options_t *options, time_t now); - -/* invoked on timers and by outside triggers. */ -struct pending_vote_t * dirvote_add_vote(const char *vote_body, - const char **msg_out, - int *status_out); -int dirvote_add_signatures(const char *detached_signatures_body, - const char *source, - const char **msg_out); - -/* Item access */ -const char *dirvote_get_pending_consensus(void); -const char *dirvote_get_pending_detached_signatures(void); -#define DGV_BY_ID 1 -#define DGV_INCLUDE_PENDING 2 -#define DGV_INCLUDE_PREVIOUS 4 -const cached_dir_t *dirvote_get_vote(const char *fp, int flags); -void set_routerstatus_from_routerinfo(routerstatus_t *rs, - routerinfo_t *ri, time_t now, - int naming, int exits_can_be_guards, - int listbadexits, int listbaddirs); -void router_clear_status_flags(routerinfo_t *ri); -networkstatus_t * -dirserv_generate_networkstatus_vote_obj(crypto_pk_env_t *private_key, - authority_cert_t *cert); - -#ifdef DIRVOTE_PRIVATE -char *format_networkstatus_vote(crypto_pk_env_t *private_key, - networkstatus_t *v3_ns); -char *dirvote_compute_params(smartlist_t *votes); -#endif - -/********************************* dns.c ***************************/ - -int dns_init(void); -int has_dns_init_failed(void); -void dns_free_all(void); -uint32_t dns_clip_ttl(uint32_t ttl); -int dns_reset(void); -void connection_dns_remove(edge_connection_t *conn); -void assert_connection_edge_not_dns_pending(edge_connection_t *conn); -void assert_all_pending_dns_resolves_ok(void); -void dns_cancel_pending_resolve(const char *question); -int dns_resolve(edge_connection_t *exitconn); -void dns_launch_correctness_checks(void); -int dns_seems_to_be_broken(void); -void dns_reset_correctness_checks(void); - -/********************************* dnsserv.c ************************/ - -void dnsserv_configure_listener(connection_t *conn); -void dnsserv_close_listener(connection_t *conn); -void dnsserv_resolved(edge_connection_t *conn, - int answer_type, - size_t answer_len, - const char *answer, - int ttl); -void dnsserv_reject_request(edge_connection_t *conn); -int dnsserv_launch_request(const char *name, int is_reverse); /********************************* geoip.c **************************/ -#ifdef GEOIP_PRIVATE -int geoip_parse_entry(const char *line); -#endif -int should_record_bridge_info(or_options_t *options); -int geoip_load_file(const char *filename, or_options_t *options); -int geoip_get_country_by_ip(uint32_t ipaddr); -int geoip_get_n_countries(void); -const char *geoip_get_country_name(country_t num); -int geoip_is_loaded(void); -country_t geoip_get_country(const char *countrycode); +/** Round all GeoIP results to the next multiple of this value, to avoid + * leaking information. */ +#define DIR_RECORD_USAGE_GRANULARITY 8 +/** Time interval: Flush geoip data to disk this often. */ +#define DIR_ENTRY_RECORD_USAGE_RETAIN_IPS (24*60*60) +/** How long do we have to have observed per-country request history before + * we are willing to talk about it? */ +#define DIR_RECORD_USAGE_MIN_OBSERVATION_TIME (12*60*60) + /** Indicates an action that we might be noting geoip statistics on. * Note that if we're noticing CONNECT, we're a bridge, and if we're noticing * the others, we're not. */ typedef enum { - /** We've noticed a connection as a bridge relay. */ + /** We've noticed a connection as a bridge relay or entry guard. */ GEOIP_CLIENT_CONNECT = 0, /** We've served a networkstatus consensus as a directory server. */ GEOIP_CLIENT_NETWORKSTATUS = 1, /** We've served a v2 networkstatus consensus as a directory server. */ GEOIP_CLIENT_NETWORKSTATUS_V2 = 2, } geoip_client_action_t; -void geoip_note_client_seen(geoip_client_action_t action, - uint32_t addr, time_t now); -void geoip_remove_old_clients(time_t cutoff); -time_t geoip_get_history_start(void); -char *geoip_get_client_history(time_t now, geoip_client_action_t action); -char *geoip_get_request_history(time_t now, geoip_client_action_t action); -int getinfo_helper_geoip(control_connection_t *control_conn, - const char *question, char **answer); -void geoip_free_all(void); -void dump_geoip_stats(void); - -/********************************* hibernate.c **********************/ - -int accounting_parse_options(or_options_t *options, int validate_only); -int accounting_is_enabled(or_options_t *options); -void configure_accounting(time_t now); -void accounting_run_housekeeping(time_t now); -void accounting_add_bytes(size_t n_read, size_t n_written, int seconds); -int accounting_record_bandwidth_usage(time_t now, or_state_t *state); -void hibernate_begin_shutdown(void); -int we_are_hibernating(void); -void consider_hibernation(time_t now); -int getinfo_helper_accounting(control_connection_t *conn, - const char *question, char **answer); -void accounting_set_bandwidth_usage_from_state(or_state_t *state); - -/********************************* main.c ***************************/ - -extern int has_completed_circuit; - -int connection_add(connection_t *conn); -int connection_remove(connection_t *conn); -int connection_in_array(connection_t *conn); -void add_connection_to_closeable_list(connection_t *conn); -int connection_is_on_closeable_list(connection_t *conn); - -smartlist_t *get_connection_array(void); - -void connection_watch_events(connection_t *conn, short events); -int connection_is_reading(connection_t *conn); -void connection_stop_reading(connection_t *conn); -void connection_start_reading(connection_t *conn); - -int connection_is_writing(connection_t *conn); -void connection_stop_writing(connection_t *conn); -void connection_start_writing(connection_t *conn); - -void connection_stop_reading_from_linked_conn(connection_t *conn); - -void directory_all_unreachable(time_t now); -void directory_info_has_arrived(time_t now, int from_cache); - -void ip_address_changed(int at_interface); -void dns_servers_relaunch_checks(void); - -void control_signal_act(int the_signal); -void handle_signals(int is_parent); - -int try_locking(or_options_t *options, int err_if_locked); -int have_lockfile(void); -void release_lockfile(void); - -void tor_cleanup(void); -void tor_free_all(int postfork); - -int tor_main(int argc, char *argv[]); - -#ifdef MAIN_PRIVATE -int do_main_loop(void); -int do_list_fingerprint(void); -void do_hash_password(void); -int tor_init(int argc, char **argv); -#endif +/** Indicates either a positive reply or a reason for rejectng a network + * status request that will be included in geoip statistics. */ +typedef enum { + /** Request is answered successfully. */ + GEOIP_SUCCESS = 0, + /** V3 network status is not signed by a sufficient number of requested + * authorities. */ + GEOIP_REJECT_NOT_ENOUGH_SIGS = 1, + /** Requested network status object is unavailable. */ + GEOIP_REJECT_UNAVAILABLE = 2, + /** Requested network status not found. */ + GEOIP_REJECT_NOT_FOUND = 3, + /** Network status has not been modified since If-Modified-Since time. */ + GEOIP_REJECT_NOT_MODIFIED = 4, + /** Directory is busy. */ + GEOIP_REJECT_BUSY = 5, +} geoip_ns_response_t; +#define GEOIP_NS_RESPONSE_NUM 6 + +/** Directory requests that we are measuring can be either direct or + * tunneled. */ +typedef enum { + DIRREQ_DIRECT = 0, + DIRREQ_TUNNELED = 1, +} dirreq_type_t; -/********************************* networkstatus.c *********************/ +/** Possible states for either direct or tunneled directory requests that + * are relevant for determining network status download times. */ +typedef enum { + /** Found that the client requests a network status; applies to both + * direct and tunneled requests; initial state of a request that we are + * measuring. */ + DIRREQ_IS_FOR_NETWORK_STATUS = 0, + /** Finished writing a network status to the directory connection; + * applies to both direct and tunneled requests; completes a direct + * request. */ + DIRREQ_FLUSHING_DIR_CONN_FINISHED = 1, + /** END cell sent to circuit that initiated a tunneled request. */ + DIRREQ_END_CELL_SENT = 2, + /** Flushed last cell from queue of the circuit that initiated a + * tunneled request to the outbuf of the OR connection. */ + DIRREQ_CIRC_QUEUE_FLUSHED = 3, + /** Flushed last byte from buffer of the OR connection belonging to the + * circuit that initiated a tunneled request; completes a tunneled + * request. */ + DIRREQ_OR_CONN_BUFFER_FLUSHED = 4 +} dirreq_state_t; + +#define WRITE_STATS_INTERVAL (24*60*60) + +/********************************* microdesc.c *************************/ + +typedef struct microdesc_cache_t microdesc_cache_t; -/** How old do we allow a v2 network-status to get before removing it - * completely? */ -#define MAX_NETWORKSTATUS_AGE (10*24*60*60) +/********************************* networkstatus.c *********************/ /** Location where we found a v2 networkstatus. */ typedef enum { @@ -3733,127 +3382,8 @@ typedef enum version_status_t { VS_UNKNOWN, /**< We have no idea. */ } version_status_t; -void networkstatus_reset_warnings(void); -void networkstatus_reset_download_failures(void); -int router_reload_v2_networkstatus(void); -int router_reload_consensus_networkstatus(void); -void routerstatus_free(routerstatus_t *rs); -void networkstatus_v2_free(networkstatus_v2_t *ns); -void networkstatus_vote_free(networkstatus_t *ns); -networkstatus_voter_info_t *networkstatus_get_voter_by_id( - networkstatus_t *vote, - const char *identity); -int networkstatus_check_consensus_signature(networkstatus_t *consensus, - int warn); -int networkstatus_check_voter_signature(networkstatus_t *consensus, - networkstatus_voter_info_t *voter, - authority_cert_t *cert); -char *networkstatus_get_cache_filename(const char *identity_digest); -int router_set_networkstatus_v2(const char *s, time_t arrived_at, - v2_networkstatus_source_t source, - smartlist_t *requested_fingerprints); -void networkstatus_v2_list_clean(time_t now); -routerstatus_t *networkstatus_v2_find_entry(networkstatus_v2_t *ns, - const char *digest); -routerstatus_t *networkstatus_vote_find_entry(networkstatus_t *ns, - const char *digest); -int networkstatus_vote_find_entry_idx(networkstatus_t *ns, - const char *digest, int *found_out); -const smartlist_t *networkstatus_get_v2_list(void); -download_status_t *router_get_dl_status_by_descriptor_digest(const char *d); -routerstatus_t *router_get_consensus_status_by_id(const char *digest); -routerstatus_t *router_get_consensus_status_by_descriptor_digest( - const char *digest); -routerstatus_t *router_get_consensus_status_by_nickname(const char *nickname, - int warn_if_unnamed); -const char *networkstatus_get_router_digest_by_nickname(const char *nickname); -int networkstatus_nickname_is_unnamed(const char *nickname); -void networkstatus_consensus_download_failed(int status_code); -void update_consensus_networkstatus_fetch_time(time_t now); -int should_delay_dir_fetches(or_options_t *options); -void update_networkstatus_downloads(time_t now); -void update_certificate_downloads(time_t now); -int consensus_is_waiting_for_certs(void); -networkstatus_v2_t *networkstatus_v2_get_by_digest(const char *digest); -networkstatus_t *networkstatus_get_latest_consensus(void); -networkstatus_t *networkstatus_get_live_consensus(time_t now); -networkstatus_t *networkstatus_get_reasonably_live_consensus(time_t now); -#define NSSET_FROM_CACHE 1 -#define NSSET_WAS_WAITING_FOR_CERTS 2 -#define NSSET_DONT_DOWNLOAD_CERTS 4 -#define NSSET_ACCEPT_OBSOLETE 8 -int networkstatus_set_current_consensus(const char *consensus, unsigned flags); -void networkstatus_note_certs_arrived(void); -void routers_update_all_from_networkstatus(time_t now, int dir_version); -void routerstatus_list_update_from_consensus_networkstatus(time_t now); -void routers_update_status_from_consensus_networkstatus(smartlist_t *routers, - int reset_failures); -void signed_descs_update_status_from_consensus_networkstatus( - smartlist_t *descs); - -char *networkstatus_getinfo_helper_single(routerstatus_t *rs); -char *networkstatus_getinfo_by_purpose(const char *purpose_string, time_t now); -void networkstatus_dump_bridge_status_to_file(time_t now); -int32_t networkstatus_get_param(networkstatus_t *ns, const char *param_name, - int32_t default_val); -int getinfo_helper_networkstatus(control_connection_t *conn, - const char *question, char **answer); -void networkstatus_free_all(void); - -/********************************* ntmain.c ***************************/ -#ifdef MS_WINDOWS -#define NT_SERVICE -#endif - -#ifdef NT_SERVICE -int nt_service_parse_options(int argc, char **argv, int *should_exit); -int nt_service_is_stopping(void); -void nt_service_set_state(DWORD state); -#else -#define nt_service_is_stopping() (0) -#endif - -/********************************* onion.c ***************************/ - -int onion_pending_add(or_circuit_t *circ, char *onionskin); -or_circuit_t *onion_next_task(char **onionskin_out); -void onion_pending_remove(or_circuit_t *circ); - -int onion_skin_create(crypto_pk_env_t *router_key, - crypto_dh_env_t **handshake_state_out, - char *onion_skin_out); - -int onion_skin_server_handshake(const char *onion_skin, - crypto_pk_env_t *private_key, - crypto_pk_env_t *prev_private_key, - char *handshake_reply_out, - char *key_out, - size_t key_out_len); - -int onion_skin_client_handshake(crypto_dh_env_t *handshake_state, - const char *handshake_reply, - char *key_out, - size_t key_out_len); - -int fast_server_handshake(const uint8_t *key_in, - uint8_t *handshake_reply_out, - uint8_t *key_out, - size_t key_out_len); - -int fast_client_handshake(const uint8_t *handshake_state, - const uint8_t *handshake_reply_out, - uint8_t *key_out, - size_t key_out_len); - -void clear_pending_onions(void); - /********************************* policies.c ************************/ -/* (length of "accept 255.255.255.255/255.255.255.255:65535-65535\n" plus a - * NUL.) - */ -#define POLICY_BUF_LEN 52 - /** Outcome of applying an address policy to an address. */ typedef enum { /** The address was accepted */ @@ -3868,146 +3398,8 @@ typedef enum { ADDR_POLICY_PROBABLY_REJECTED=2 } addr_policy_result_t; -int firewall_is_fascist_or(void); -int fascist_firewall_allows_address_or(const tor_addr_t *addr, uint16_t port); -int fascist_firewall_allows_or(routerinfo_t *ri); -int fascist_firewall_allows_address_dir(const tor_addr_t *addr, uint16_t port); -int dir_policy_permits_address(const tor_addr_t *addr); -int socks_policy_permits_address(const tor_addr_t *addr); -int authdir_policy_permits_address(uint32_t addr, uint16_t port); -int authdir_policy_valid_address(uint32_t addr, uint16_t port); -int authdir_policy_baddir_address(uint32_t addr, uint16_t port); -int authdir_policy_badexit_address(uint32_t addr, uint16_t port); - -int validate_addr_policies(or_options_t *options, char **msg); -void policy_expand_private(smartlist_t **policy); -int policies_parse_from_options(or_options_t *options); - -addr_policy_t *addr_policy_get_canonical_entry(addr_policy_t *ent); -int cmp_addr_policies(smartlist_t *a, smartlist_t *b); -addr_policy_result_t compare_tor_addr_to_addr_policy(const tor_addr_t *addr, - uint16_t port, const smartlist_t *policy); -addr_policy_result_t compare_addr_to_addr_policy(uint32_t addr, - uint16_t port, const smartlist_t *policy); -int policies_parse_exit_policy(config_line_t *cfg, smartlist_t **dest, - int rejectprivate, const char *local_address); -void policies_set_router_exitpolicy_to_reject_all(routerinfo_t *exitrouter); -int exit_policy_is_general_exit(smartlist_t *policy); -int policy_is_reject_star(const smartlist_t *policy); -int getinfo_helper_policies(control_connection_t *conn, - const char *question, char **answer); -int policy_write_item(char *buf, size_t buflen, addr_policy_t *item, - int format_for_desc); - -void addr_policy_list_free(smartlist_t *p); -void addr_policy_free(addr_policy_t *p); -void policies_free_all(void); - -char *policy_summarize(smartlist_t *policy); - -/********************************* reasons.c ***************************/ - -const char *stream_end_reason_to_control_string(int reason); -const char *stream_end_reason_to_string(int reason); -socks5_reply_status_t stream_end_reason_to_socks5_response(int reason); -uint8_t errno_to_stream_end_reason(int e); - -const char *orconn_end_reason_to_control_string(int r); -int tls_error_to_orconn_end_reason(int e); -int errno_to_orconn_end_reason(int e); - -const char *circuit_end_reason_to_control_string(int reason); - -/********************************* relay.c ***************************/ - -extern uint64_t stats_n_relay_cells_relayed; -extern uint64_t stats_n_relay_cells_delivered; - -int circuit_receive_relay_cell(cell_t *cell, circuit_t *circ, - cell_direction_t cell_direction); - -void relay_header_pack(uint8_t *dest, const relay_header_t *src); -void relay_header_unpack(relay_header_t *dest, const uint8_t *src); -int relay_send_command_from_edge(streamid_t stream_id, circuit_t *circ, - uint8_t relay_command, const char *payload, - size_t payload_len, crypt_path_t *cpath_layer); -int connection_edge_send_command(edge_connection_t *fromconn, - uint8_t relay_command, const char *payload, - size_t payload_len); -int connection_edge_package_raw_inbuf(edge_connection_t *conn, - int package_partial); -void connection_edge_consider_sending_sendme(edge_connection_t *conn); - -extern uint64_t stats_n_data_cells_packaged; -extern uint64_t stats_n_data_bytes_packaged; -extern uint64_t stats_n_data_cells_received; -extern uint64_t stats_n_data_bytes_received; - -void init_cell_pool(void); -void free_cell_pool(void); -void clean_cell_pool(void); -void dump_cell_pool_usage(int severity); - -void cell_queue_clear(cell_queue_t *queue); -void cell_queue_append(cell_queue_t *queue, packed_cell_t *cell); -void cell_queue_append_packed_copy(cell_queue_t *queue, const cell_t *cell); - -void append_cell_to_circuit_queue(circuit_t *circ, or_connection_t *orconn, - cell_t *cell, cell_direction_t direction); -void connection_or_unlink_all_active_circs(or_connection_t *conn); -int connection_or_flush_from_first_active_circuit(or_connection_t *conn, - int max, time_t now); -void assert_active_circuits_ok(or_connection_t *orconn); -void make_circuit_inactive_on_conn(circuit_t *circ, or_connection_t *conn); -void make_circuit_active_on_conn(circuit_t *circ, or_connection_t *conn); - -int append_address_to_payload(uint8_t *payload_out, const tor_addr_t *addr); -const uint8_t *decode_address_from_payload(tor_addr_t *addr_out, - const uint8_t *payload, - int payload_len); - /********************************* rephist.c ***************************/ -void rep_hist_init(void); -void rep_hist_note_connect_failed(const char* nickname, time_t when); -void rep_hist_note_connect_succeeded(const char* nickname, time_t when); -void rep_hist_note_disconnect(const char* nickname, time_t when); -void rep_hist_note_connection_died(const char* nickname, time_t when); -void rep_hist_note_extend_succeeded(const char *from_name, - const char *to_name); -void rep_hist_note_extend_failed(const char *from_name, const char *to_name); -void rep_hist_dump_stats(time_t now, int severity); -void rep_hist_note_bytes_read(size_t num_bytes, time_t when); -void rep_hist_note_bytes_written(size_t num_bytes, time_t when); -int rep_hist_bandwidth_assess(void); -char *rep_hist_get_bandwidth_lines(int for_extrainfo); -void rep_hist_update_state(or_state_t *state); -int rep_hist_load_state(or_state_t *state, char **err); -void rep_history_clean(time_t before); - -void rep_hist_note_router_reachable(const char *id, time_t when); -void rep_hist_note_router_unreachable(const char *id, time_t when); -int rep_hist_record_mtbf_data(time_t now, int missing_means_down); -int rep_hist_load_mtbf_data(time_t now); - -time_t rep_hist_downrate_old_runs(time_t now); -double rep_hist_get_stability(const char *id, time_t when); -double rep_hist_get_weighted_fractional_uptime(const char *id, time_t when); -long rep_hist_get_weighted_time_known(const char *id, time_t when); -int rep_hist_have_measured_enough_stability(void); -const char *rep_hist_get_router_stability_doc(time_t now); - -void rep_hist_note_used_port(time_t now, uint16_t port); -smartlist_t *rep_hist_get_predicted_ports(time_t now); -void rep_hist_note_used_resolve(time_t now); -void rep_hist_note_used_internal(time_t now, int need_uptime, - int need_capacity); -int rep_hist_get_predicted_internal(time_t now, int *need_uptime, - int *need_capacity); - -int any_predicted_circuits(time_t now); -int rep_hist_circbuilding_dormant(time_t now); - /** Possible public/private key operations in Tor: used to keep track of where * we're spending our time. */ typedef enum { @@ -4017,49 +3409,6 @@ typedef enum { TLS_HANDSHAKE_C, TLS_HANDSHAKE_S, REND_CLIENT, REND_MID, REND_SERVER, } pk_op_t; -void note_crypto_pk_op(pk_op_t operation); -void dump_pk_ops(int severity); - -void rep_hist_free_all(void); - -/* for hidden service usage statistics */ -void hs_usage_note_publish_total(const char *service_id, time_t now); -void hs_usage_note_publish_novel(const char *service_id, time_t now); -void hs_usage_note_fetch_total(const char *service_id, time_t now); -void hs_usage_note_fetch_successful(const char *service_id, time_t now); -void hs_usage_write_statistics_to_file(time_t now); -void hs_usage_free_all(void); - -/********************************* rendclient.c ***************************/ - -void rend_client_introcirc_has_opened(origin_circuit_t *circ); -void rend_client_rendcirc_has_opened(origin_circuit_t *circ); -int rend_client_introduction_acked(origin_circuit_t *circ, - const uint8_t *request, - size_t request_len); -void rend_client_refetch_renddesc(const char *query); -void rend_client_refetch_v2_renddesc(const rend_data_t *rend_query); -void rend_client_cancel_descriptor_fetches(void); -int rend_client_remove_intro_point(extend_info_t *failed_intro, - const rend_data_t *rend_query); -int rend_client_rendezvous_acked(origin_circuit_t *circ, - const uint8_t *request, - size_t request_len); -int rend_client_receive_rendezvous(origin_circuit_t *circ, - const uint8_t *request, - size_t request_len); -void rend_client_desc_trynow(const char *query, int rend_version); - -extend_info_t *rend_client_get_random_intro(const rend_data_t *rend_query); - -int rend_client_send_introduction(origin_circuit_t *introcirc, - origin_circuit_t *rendcirc); -int rend_parse_service_authorization(or_options_t *options, - int validate_only); -rend_service_authorization_t *rend_client_lookup_service_authorization( - const char *onion_address); -void rend_service_authorization_free_all(void); -rend_data_t *rend_data_dup(const rend_data_t *request); /********************************* rendcommon.c ***************************/ @@ -4102,31 +3451,6 @@ typedef struct rend_service_descriptor_t { smartlist_t *successful_uploads; } rend_service_descriptor_t; -/** Free all storage associated with <b>data</b> */ -static INLINE void -rend_data_free(rend_data_t *data) -{ - tor_free(data); -} - -int rend_cmp_service_ids(const char *one, const char *two); - -void rend_process_relay_cell(circuit_t *circ, const crypt_path_t *layer_hint, - int command, size_t length, - const uint8_t *payload); - -void rend_service_descriptor_free(rend_service_descriptor_t *desc); -int rend_encode_service_descriptor(rend_service_descriptor_t *desc, - crypto_pk_env_t *key, - char **str_out, - size_t *len_out); -rend_service_descriptor_t *rend_parse_service_descriptor(const char *str, - size_t len); -int rend_get_service_id(crypto_pk_env_t *pk, char *out); -void rend_encoded_v2_service_descriptor_free( - rend_encoded_v2_service_descriptor_t *desc); -void rend_intro_point_free(rend_intro_point_t *intro); - /** A cached rendezvous descriptor. */ typedef struct rend_cache_entry_t { size_t len; /**< Length of <b>desc</b> */ @@ -4135,151 +3459,6 @@ typedef struct rend_cache_entry_t { rend_service_descriptor_t *parsed; /**< Parsed value of 'desc' */ } rend_cache_entry_t; -void rend_cache_init(void); -void rend_cache_clean(void); -void rend_cache_clean_v2_descs_as_dir(void); -void rend_cache_purge(void); -void rend_cache_free_all(void); -int rend_valid_service_id(const char *query); -int rend_cache_lookup_desc(const char *query, int version, const char **desc, - size_t *desc_len); -int rend_cache_lookup_entry(const char *query, int version, - rend_cache_entry_t **entry_out); -int rend_cache_lookup_v2_desc_as_dir(const char *query, const char **desc); -int rend_cache_store(const char *desc, size_t desc_len, int published); -int rend_cache_store_v2_desc_as_client(const char *desc, - const rend_data_t *rend_query); -int rend_cache_store_v2_desc_as_dir(const char *desc); -int rend_cache_size(void); -int rend_encode_v2_descriptors(smartlist_t *descs_out, - rend_service_descriptor_t *desc, time_t now, - uint8_t period, rend_auth_type_t auth_type, - crypto_pk_env_t *client_key, - smartlist_t *client_cookies); -int rend_compute_v2_desc_id(char *desc_id_out, const char *service_id, - const char *descriptor_cookie, - time_t now, uint8_t replica); -int rend_id_is_in_interval(const char *a, const char *b, const char *c); -void rend_get_descriptor_id_bytes(char *descriptor_id_out, - const char *service_id, - const char *secret_id_part); - -/********************************* rendservice.c ***************************/ - -int num_rend_services(void); -int rend_config_services(or_options_t *options, int validate_only); -int rend_service_load_keys(void); -void rend_services_init(void); -void rend_services_introduce(void); -void rend_consider_services_upload(time_t now); -void rend_hsdir_routers_changed(void); -void rend_consider_descriptor_republication(void); - -void rend_service_intro_has_opened(origin_circuit_t *circuit); -int rend_service_intro_established(origin_circuit_t *circuit, - const uint8_t *request, - size_t request_len); -void rend_service_rendezvous_has_opened(origin_circuit_t *circuit); -int rend_service_introduce(origin_circuit_t *circuit, const uint8_t *request, - size_t request_len); -void rend_service_relaunch_rendezvous(origin_circuit_t *oldcirc); -int rend_service_set_connection_addr_port(edge_connection_t *conn, - origin_circuit_t *circ); -void rend_service_dump_stats(int severity); -void rend_service_free_all(void); - -/********************************* rendmid.c *******************************/ -int rend_mid_establish_intro(or_circuit_t *circ, const uint8_t *request, - size_t request_len); -int rend_mid_introduce(or_circuit_t *circ, const uint8_t *request, - size_t request_len); -int rend_mid_establish_rendezvous(or_circuit_t *circ, const uint8_t *request, - size_t request_len); -int rend_mid_rendezvous(or_circuit_t *circ, const uint8_t *request, - size_t request_len); - -/********************************* router.c ***************************/ - -crypto_pk_env_t *get_onion_key(void); -time_t get_onion_key_set_at(void); -void set_identity_key(crypto_pk_env_t *k); -crypto_pk_env_t *get_identity_key(void); -int identity_key_is_set(void); -authority_cert_t *get_my_v3_authority_cert(void); -crypto_pk_env_t *get_my_v3_authority_signing_key(void); -authority_cert_t *get_my_v3_legacy_cert(void); -crypto_pk_env_t *get_my_v3_legacy_signing_key(void); -void dup_onion_keys(crypto_pk_env_t **key, crypto_pk_env_t **last); -void rotate_onion_key(void); -crypto_pk_env_t *init_key_from_file(const char *fname, int generate, - int severity); -void v3_authority_check_key_expiry(void); - -int init_keys(void); - -int check_whether_orport_reachable(void); -int check_whether_dirport_reachable(void); -void consider_testing_reachability(int test_or, int test_dir); -void router_orport_found_reachable(void); -void router_dirport_found_reachable(void); -void router_perform_bandwidth_test(int num_circs, time_t now); - -int authdir_mode(or_options_t *options); -int authdir_mode_v1(or_options_t *options); -int authdir_mode_v2(or_options_t *options); -int authdir_mode_v3(or_options_t *options); -int authdir_mode_any_main(or_options_t *options); -int authdir_mode_any_nonhidserv(or_options_t *options); -int authdir_mode_handles_descs(or_options_t *options, int purpose); -int authdir_mode_publishes_statuses(or_options_t *options); -int authdir_mode_tests_reachability(or_options_t *options); -int authdir_mode_bridge(or_options_t *options); - -int server_mode(or_options_t *options); -int advertised_server_mode(void); -int proxy_mode(or_options_t *options); -void consider_publishable_server(int force); - -void router_upload_dir_desc_to_dirservers(int force); -void mark_my_descriptor_dirty_if_older_than(time_t when); -void mark_my_descriptor_dirty(void); -void check_descriptor_bandwidth_changed(time_t now); -void check_descriptor_ipaddress_changed(time_t now); -void router_new_address_suggestion(const char *suggestion, - const dir_connection_t *d_conn); -int router_compare_to_my_exit_policy(edge_connection_t *conn); -routerinfo_t *router_get_my_routerinfo(void); -extrainfo_t *router_get_my_extrainfo(void); -const char *router_get_my_descriptor(void); -int router_digest_is_me(const char *digest); -int router_extrainfo_digest_is_me(const char *digest); -int router_is_me(routerinfo_t *router); -int router_fingerprint_is_me(const char *fp); -int router_pick_published_address(or_options_t *options, uint32_t *addr); -int router_rebuild_descriptor(int force); -int router_dump_router_to_string(char *s, size_t maxlen, routerinfo_t *router, - crypto_pk_env_t *ident_key); -int extrainfo_dump_to_string(char *s, size_t maxlen, extrainfo_t *extrainfo, - crypto_pk_env_t *ident_key); -char *extrainfo_get_client_geoip_summary(time_t); -int is_legal_nickname(const char *s); -int is_legal_nickname_or_hexdigest(const char *s); -int is_legal_hexdigest(const char *s); -void router_get_verbose_nickname(char *buf, const routerinfo_t *router); -void routerstatus_get_verbose_nickname(char *buf, - const routerstatus_t *router); -void router_reset_warnings(void); -void router_reset_reachability(void); -void router_free_all(void); - -const char *router_purpose_to_string(uint8_t p); -uint8_t router_purpose_from_string(const char *s); - -#ifdef ROUTER_PRIVATE -/* Used only by router.c and test.c */ -void get_platform_str(char *platform, size_t len); -#endif - /********************************* routerlist.c ***************************/ /** Represents information about a single trusted directory server. */ @@ -4319,22 +3498,7 @@ typedef struct trusted_dir_server_t { #define ROUTER_MAX_DECLARED_BANDWIDTH INT32_MAX -int get_n_authorities(authority_type_t type); -int trusted_dirs_reload_certs(void); -int trusted_dirs_load_certs_from_string(const char *contents, int from_store, - int flush); -void trusted_dirs_flush_certs_to_disk(void); -authority_cert_t *authority_cert_get_newest_by_id(const char *id_digest); -authority_cert_t *authority_cert_get_by_sk_digest(const char *sk_digest); -authority_cert_t *authority_cert_get_by_digests(const char *id_digest, - const char *sk_digest); -void authority_cert_get_all(smartlist_t *certs_out); -void authority_cert_dl_failed(const char *id_digest, int status); -void authority_certs_fetch_missing(networkstatus_t *status, time_t now); -int router_reload_router_list(void); -smartlist_t *router_get_trusted_dir_servers(void); - -/* Flags for pick_directory_server and pick_trusteddirserver. */ +/* Flags for pick_directory_server() and pick_trusteddirserver(). */ /** Flag to indicate that we should not automatically be willing to use * ourself to answer a directory request. * Passed to router_pick_directory_server (et al).*/ @@ -4363,34 +3527,13 @@ smartlist_t *router_get_trusted_dir_servers(void); */ #define PDS_NO_EXISTING_SERVERDESC_FETCH (1<<3) #define _PDS_PREFER_TUNNELED_DIR_CONNS (1<<16) -routerstatus_t *router_pick_directory_server(authority_type_t type, int flags); -trusted_dir_server_t *router_get_trusteddirserver_by_digest(const char *d); -trusted_dir_server_t *trusteddirserver_get_by_v3_auth_digest(const char *d); -routerstatus_t *router_pick_trusteddirserver(authority_type_t type, int flags); -int router_get_my_share_of_directory_requests(double *v2_share_out, - double *v3_share_out); -void router_reset_status_download_failures(void); -void routerlist_add_family(smartlist_t *sl, routerinfo_t *router); -int routers_in_same_family(routerinfo_t *r1, routerinfo_t *r2); -void add_nickname_list_to_smartlist(smartlist_t *sl, const char *list, - int must_be_running); -int router_nickname_is_in_list(routerinfo_t *router, const char *list); -routerinfo_t *routerlist_find_my_routerinfo(void); -routerinfo_t *router_find_exact_exit_enclave(const char *address, - uint16_t port); -int router_is_unreliable(routerinfo_t *router, int need_uptime, - int need_capacity, int need_guard); -uint32_t router_get_advertised_bandwidth(routerinfo_t *router); -uint32_t router_get_advertised_bandwidth_capped(routerinfo_t *router); /** Possible ways to weight routers when choosing one randomly. See * routerlist_sl_choose_by_bandwidth() for more information.*/ -typedef enum { - NO_WEIGHTING, WEIGHT_FOR_EXIT, WEIGHT_FOR_GUARD +typedef enum bandwidth_weight_rule_t { + NO_WEIGHTING, WEIGHT_FOR_EXIT, WEIGHT_FOR_MID, WEIGHT_FOR_GUARD, + WEIGHT_FOR_DIR } bandwidth_weight_rule_t; -routerinfo_t *routerlist_sl_choose_by_bandwidth(smartlist_t *sl, - bandwidth_weight_rule_t rule); -routerstatus_t *routerstatus_sl_choose_by_bandwidth(smartlist_t *sl); /** Flags to be passed to control router_choose_random_node() to indicate what * kind of nodes to pick according to what algorithm. */ @@ -4400,44 +3543,9 @@ typedef enum { CRN_NEED_GUARD = 1<<2, CRN_ALLOW_INVALID = 1<<3, /* XXXX not used, apparently. */ - CRN_STRICT_PREFERRED = 1<<4, - /* XXXX not used, apparently. */ CRN_WEIGHT_AS_EXIT = 1<<5 } router_crn_flags_t; -routerinfo_t *router_choose_random_node(const char *preferred, - smartlist_t *excludedsmartlist, - struct routerset_t *excludedset, - router_crn_flags_t flags); - -routerinfo_t *router_get_by_nickname(const char *nickname, - int warn_if_unnamed); -int router_digest_version_as_new_as(const char *digest, const char *cutoff); -int router_digest_is_trusted_dir_type(const char *digest, - authority_type_t type); -#define router_digest_is_trusted_dir(d) \ - router_digest_is_trusted_dir_type((d), NO_AUTHORITY) - -int router_addr_is_trusted_dir(uint32_t addr); -int hexdigest_to_digest(const char *hexdigest, char *digest); -routerinfo_t *router_get_by_hexdigest(const char *hexdigest); -routerinfo_t *router_get_by_digest(const char *digest); -signed_descriptor_t *router_get_by_descriptor_digest(const char *digest); -signed_descriptor_t *router_get_by_extrainfo_digest(const char *digest); -signed_descriptor_t *extrainfo_get_by_descriptor_digest(const char *digest); -const char *signed_descriptor_get_body(signed_descriptor_t *desc); -const char *signed_descriptor_get_annotations(signed_descriptor_t *desc); -routerlist_t *router_get_routerlist(void); -void routerinfo_free(routerinfo_t *router); -void extrainfo_free(extrainfo_t *extrainfo); -void routerlist_free(routerlist_t *rl); -void dump_routerlist_mem_usage(int severity); -void routerlist_remove(routerlist_t *rl, routerinfo_t *ri, int make_old, - time_t now); -void routerlist_free_all(void); -void routerlist_reset_warnings(void); -void router_set_status(const char *digest, int up); - /** Return value for router_add_to_routerlist() and dirserv_add_descriptor() */ typedef enum was_router_added_t { ROUTER_ADDED_SUCCESSFULLY = 1, @@ -4447,111 +3555,9 @@ typedef enum was_router_added_t { ROUTER_NOT_IN_CONSENSUS = -3, ROUTER_NOT_IN_CONSENSUS_OR_NETWORKSTATUS = -4, ROUTER_AUTHDIR_REJECTS = -5, + ROUTER_WAS_NOT_WANTED = -6 } was_router_added_t; -static int WRA_WAS_ADDED(was_router_added_t s); -static int WRA_WAS_OUTDATED(was_router_added_t s); -static int WRA_WAS_REJECTED(was_router_added_t s); -/** Return true iff the descriptor was added. It might still be necessary to - * check whether the descriptor generator should be notified. - */ -static INLINE int -WRA_WAS_ADDED(was_router_added_t s) { - return s == ROUTER_ADDED_SUCCESSFULLY || s == ROUTER_ADDED_NOTIFY_GENERATOR; -} -/** Return true iff the descriptor was not added because it was either: - * - not in the consensus - * - neither in the consensus nor in any networkstatus document - * - it was outdated. - */ -static INLINE int WRA_WAS_OUTDATED(was_router_added_t s) -{ - return (s == ROUTER_WAS_NOT_NEW || - s == ROUTER_NOT_IN_CONSENSUS || - s == ROUTER_NOT_IN_CONSENSUS_OR_NETWORKSTATUS); -} -/** Return true iff the descriptor rejected because it was malformed. */ -static INLINE int WRA_WAS_REJECTED(was_router_added_t s) -{ - return (s == ROUTER_AUTHDIR_REJECTS); -} -was_router_added_t router_add_to_routerlist(routerinfo_t *router, - const char **msg, - int from_cache, - int from_fetch); -was_router_added_t router_add_extrainfo_to_routerlist( - extrainfo_t *ei, const char **msg, - int from_cache, int from_fetch); -void routerlist_remove_old_routers(void); -int router_load_single_router(const char *s, uint8_t purpose, int cache, - const char **msg); -int router_load_routers_from_string(const char *s, const char *eos, - saved_location_t saved_location, - smartlist_t *requested_fingerprints, - int descriptor_digests, - const char *prepend_annotations); -void router_load_extrainfo_from_string(const char *s, const char *eos, - saved_location_t saved_location, - smartlist_t *requested_fingerprints, - int descriptor_digests); -void routerlist_retry_directory_downloads(time_t now); -int router_exit_policy_all_routers_reject(uint32_t addr, uint16_t port, - int need_uptime); -int router_exit_policy_rejects_all(routerinfo_t *router); -trusted_dir_server_t *add_trusted_dir_server(const char *nickname, - const char *address, - uint16_t dir_port, uint16_t or_port, - const char *digest, const char *v3_auth_digest, - authority_type_t type); -void authority_cert_free(authority_cert_t *cert); -void clear_trusted_dir_servers(void); -int any_trusted_dir_is_v1_authority(void); -void update_router_descriptor_downloads(time_t now); -void update_extrainfo_downloads(time_t now); -int router_have_minimum_dir_info(void); -void router_dir_info_changed(void); -const char *get_dir_info_status_string(void); -int count_loading_descriptors_progress(void); -void router_reset_descriptor_download_failures(void); -int router_differences_are_cosmetic(routerinfo_t *r1, routerinfo_t *r2); -int routerinfo_incompatible_with_extrainfo(routerinfo_t *ri, extrainfo_t *ei, - signed_descriptor_t *sd, - const char **msg); -void routerlist_assert_ok(routerlist_t *rl); -const char *esc_router_info(routerinfo_t *router); -void routers_sort_by_identity(smartlist_t *routers); - -routerset_t *routerset_new(void); -int routerset_parse(routerset_t *target, const char *s, - const char *description); -void routerset_union(routerset_t *target, const routerset_t *source); -int routerset_is_list(const routerset_t *set); -int routerset_needs_geoip(const routerset_t *set); -int routerset_contains_router(const routerset_t *set, routerinfo_t *ri); -int routerset_contains_routerstatus(const routerset_t *set, - routerstatus_t *rs); -int routerset_contains_extendinfo(const routerset_t *set, - const extend_info_t *ei); -void routerset_get_all_routers(smartlist_t *out, const routerset_t *routerset, - int running_only); -void routersets_get_disjunction(smartlist_t *target, const smartlist_t *source, - const routerset_t *include, - const routerset_t *exclude, int running_only); -void routerset_subtract_routers(smartlist_t *out, - const routerset_t *routerset); -char *routerset_to_string(const routerset_t *routerset); -void routerset_refresh_countries(routerset_t *target); -int routerset_equal(const routerset_t *old, const routerset_t *new); -void routerset_free(routerset_t *routerset); -void routerinfo_set_country(routerinfo_t *ri); -void routerlist_refresh_countries(void); -void refresh_all_country_info(void); - -int hid_serv_get_responsible_directories(smartlist_t *responsible_dirs, - const char *id); -int hid_serv_acting_as_directory(void); -int hid_serv_responsible_for_desc_id(const char *id); - /********************************* routerparse.c ************************/ #define MAX_STATUS_TAG_LEN 32 @@ -4570,71 +3576,10 @@ typedef struct tor_version_t { int patchlevel; char status_tag[MAX_STATUS_TAG_LEN]; int svn_revision; -} tor_version_t; -int router_get_router_hash(const char *s, size_t s_len, char *digest); -int router_get_dir_hash(const char *s, char *digest); -int router_get_runningrouters_hash(const char *s, char *digest); -int router_get_networkstatus_v2_hash(const char *s, char *digest); -int router_get_networkstatus_v3_hash(const char *s, char *digest); -int router_get_extrainfo_hash(const char *s, char *digest); -int router_append_dirobj_signature(char *buf, size_t buf_len, - const char *digest, - crypto_pk_env_t *private_key); -int router_parse_list_from_string(const char **s, const char *eos, - smartlist_t *dest, - saved_location_t saved_location, - int is_extrainfo, - int allow_annotations, - const char *prepend_annotations); -int router_parse_routerlist_from_directory(const char *s, - routerlist_t **dest, - crypto_pk_env_t *pkey, - int check_version, - int write_to_cache); -int router_parse_runningrouters(const char *str); -int router_parse_directory(const char *str); -routerinfo_t *router_parse_entry_from_string(const char *s, const char *end, - int cache_copy, - int allow_annotations, - const char *prepend_annotations); -extrainfo_t *extrainfo_parse_entry_from_string(const char *s, const char *end, - int cache_copy, struct digest_ri_map_t *routermap); -addr_policy_t *router_parse_addr_policy_item_from_string(const char *s, - int assume_action); -version_status_t tor_version_is_obsolete(const char *myversion, - const char *versionlist); -int tor_version_parse(const char *s, tor_version_t *out); -int tor_version_as_new_as(const char *platform, const char *cutoff); -int tor_version_compare(tor_version_t *a, tor_version_t *b); -void sort_version_list(smartlist_t *lst, int remove_duplicates); -void assert_addr_policy_ok(smartlist_t *t); -void dump_distinct_digest_count(int severity); - -networkstatus_v2_t *networkstatus_v2_parse_from_string(const char *s); -networkstatus_t *networkstatus_parse_vote_from_string(const char *s, - const char **eos_out, - networkstatus_type_t ns_type); -ns_detached_signatures_t *networkstatus_parse_detached_signatures( - const char *s, const char *eos); - -authority_cert_t *authority_cert_parse_from_string(const char *s, - const char **end_of_string); -int rend_parse_v2_service_descriptor(rend_service_descriptor_t **parsed_out, - char *desc_id_out, - char **intro_points_encrypted_out, - size_t *intro_points_encrypted_size_out, - size_t *encoded_size_out, - const char **next_out, const char *desc); -int rend_decrypt_introduction_points(char **ipos_decrypted, - size_t *ipos_decrypted_size, - const char *descriptor_cookie, - const char *ipos_encrypted, - size_t ipos_encrypted_size); -int rend_parse_introduction_points(rend_service_descriptor_t *parsed, - const char *intro_points_encoded, - size_t intro_points_encoded_size); -int rend_parse_client_keys(strmap_t *parsed_clients, const char *str); + int git_tag_len; + char git_tag[DIGEST_LEN]; +} tor_version_t; #endif diff --git a/src/or/policies.c b/src/or/policies.c index f8c36c784b..e48f42058f 100644 --- a/src/or/policies.c +++ b/src/or/policies.c @@ -9,6 +9,10 @@ **/ #include "or.h" +#include "config.h" +#include "dirserv.h" +#include "policies.h" +#include "routerparse.h" #include "ht.h" /** Policy that addresses for incoming SOCKS connections must match. */ @@ -344,7 +348,8 @@ validate_addr_policies(or_options_t *options, char **msg) *msg = NULL; if (policies_parse_exit_policy(options->ExitPolicy, &addr_policy, - options->ExitPolicyRejectPrivate, NULL)) + options->ExitPolicyRejectPrivate, NULL, + !options->BridgeRelay)) REJECT("Error in ExitPolicy entry."); /* The rest of these calls *append* to addr_policy. So don't actually @@ -375,14 +380,8 @@ validate_addr_policies(or_options_t *options, char **msg) if (parse_addr_policy(options->ReachableDirAddresses, &addr_policy, ADDR_POLICY_ACCEPT)) REJECT("Error in ReachableDirAddresses entry."); - if (parse_addr_policy(options->AuthDirReject, &addr_policy, - ADDR_POLICY_REJECT)) - REJECT("Error in AuthDirReject entry."); - if (parse_addr_policy(options->AuthDirInvalid, &addr_policy, - ADDR_POLICY_REJECT)) - REJECT("Error in AuthDirInvalid entry."); -err: + err: addr_policy_list_free(addr_policy); return *msg ? -1 : 0; #undef REJECT @@ -829,14 +828,16 @@ exit_policy_remove_redundancies(smartlist_t *dest) "reject *:6346-6429,reject *:6699,reject *:6881-6999,accept *:*" /** Parse the exit policy <b>cfg</b> into the linked list *<b>dest</b>. If - * cfg doesn't end in an absolute accept or reject, add the default exit + * cfg doesn't end in an absolute accept or reject and if + * <b>add_default_policy</b> is true, add the default exit * policy afterwards. If <b>rejectprivate</b> is true, prepend * "reject private:*" to the policy. Return -1 if we can't parse cfg, * else return 0. */ int policies_parse_exit_policy(config_line_t *cfg, smartlist_t **dest, - int rejectprivate, const char *local_address) + int rejectprivate, const char *local_address, + int add_default_policy) { if (rejectprivate) { append_exit_policy_string(dest, "reject private:*"); @@ -848,13 +849,23 @@ policies_parse_exit_policy(config_line_t *cfg, smartlist_t **dest, } if (parse_addr_policy(cfg, dest, -1)) return -1; - append_exit_policy_string(dest, DEFAULT_EXIT_POLICY); - + if (add_default_policy) + append_exit_policy_string(dest, DEFAULT_EXIT_POLICY); + else + append_exit_policy_string(dest, "reject *:*"); exit_policy_remove_redundancies(*dest); return 0; } +/** Add "reject *:*" to the end of the policy in *<b>dest</b>, allocating + * *<b>dest</b> as needed. */ +void +policies_exit_policy_append_reject_star(smartlist_t **dest) +{ + append_exit_policy_string(dest, "reject *:*"); +} + /** Replace the exit policy of <b>r</b> with reject *:*. */ void policies_set_router_exitpolicy_to_reject_all(routerinfo_t *r) @@ -866,6 +877,51 @@ policies_set_router_exitpolicy_to_reject_all(routerinfo_t *r) smartlist_add(r->exit_policy, item); } +/** Return 1 if there is at least one /8 subnet in <b>policy</b> that + * allows exiting to <b>port</b>. Otherwise, return 0. */ +static int +exit_policy_is_general_exit_helper(smartlist_t *policy, int port) +{ + uint32_t mask, ip, i; + /* Is this /8 rejected (1), or undecided (0)? */ + char subnet_status[256]; + + memset(subnet_status, 0, sizeof(subnet_status)); + SMARTLIST_FOREACH(policy, addr_policy_t *, p, { + if (tor_addr_family(&p->addr) != AF_INET) + continue; /* IPv4 only for now */ + if (p->prt_min > port || p->prt_max < port) + continue; /* Doesn't cover our port. */ + mask = 0; + tor_assert(p->maskbits <= 32); + + if (p->maskbits) + mask = UINT32_MAX<<(32-p->maskbits); + ip = tor_addr_to_ipv4h(&p->addr); + + /* Calculate the first and last subnet that this exit policy touches + * and set it as loop boundaries. */ + for (i = ((mask & ip)>>24); i <= (~((mask & ip) ^ mask)>>24); ++i) { + tor_addr_t addr; + if (subnet_status[i] != 0) + continue; /* We already reject some part of this /8 */ + tor_addr_from_ipv4h(&addr, i<<24); + if (tor_addr_is_internal(&addr, 0)) + continue; /* Local or non-routable addresses */ + if (p->policy_type == ADDR_POLICY_ACCEPT) { + if (p->maskbits > 8) + continue; /* Narrower than a /8. */ + /* We found an allowed subnet of at least size /8. Done + * for this port! */ + return 1; + } else if (p->policy_type == ADDR_POLICY_REJECT) { + subnet_status[i] = 1; + } + } + }); + return 0; +} + /** Return true iff <b>ri</b> is "useful as an exit node", meaning * it allows exit to at least one /8 address space for at least * two of ports 80, 443, and 6667. */ @@ -879,21 +935,7 @@ exit_policy_is_general_exit(smartlist_t *policy) return 0; for (i = 0; i < 3; ++i) { - SMARTLIST_FOREACH(policy, addr_policy_t *, p, { - if (tor_addr_family(&p->addr) != AF_INET) - continue; /* IPv4 only for now */ - if (p->prt_min > ports[i] || p->prt_max < ports[i]) - continue; /* Doesn't cover our port. */ - if (p->maskbits > 8) - continue; /* Narrower than a /8. */ - if (tor_addr_is_loopback(&p->addr)) - continue; /* 127.x or ::1. */ - /* We have a match that is at least a /8. */ - if (p->policy_type == ADDR_POLICY_ACCEPT) { - ++n_allowed; - break; /* stop considering this port */ - } - }); + n_allowed += exit_policy_is_general_exit_helper(policy, ports[i]); } return n_allowed >= 2; } @@ -1240,7 +1282,7 @@ policy_summarize(smartlist_t *policy) result = tor_malloc(final_size); tor_snprintf(result, final_size, "%s %s", prefix, shorter_str); -cleanup: + cleanup: /* cleanup */ SMARTLIST_FOREACH(summary, policy_summary_item_t *, s, tor_free(s)); smartlist_free(summary); @@ -1260,9 +1302,11 @@ cleanup: * about "exit-policy/..." */ int getinfo_helper_policies(control_connection_t *conn, - const char *question, char **answer) + const char *question, char **answer, + const char **errmsg) { (void) conn; + (void) errmsg; if (!strcmp(question, "exit-policy/default")) { *answer = tor_strdup(DEFAULT_EXIT_POLICY); } @@ -1273,7 +1317,8 @@ getinfo_helper_policies(control_connection_t *conn, void addr_policy_list_free(smartlist_t *lst) { - if (!lst) return; + if (!lst) + return; SMARTLIST_FOREACH(lst, addr_policy_t *, policy, addr_policy_free(policy)); smartlist_free(lst); } @@ -1282,19 +1327,20 @@ addr_policy_list_free(smartlist_t *lst) void addr_policy_free(addr_policy_t *p) { - if (p) { - if (--p->refcnt <= 0) { - if (p->is_canonical) { - policy_map_ent_t search, *found; - search.policy = p; - found = HT_REMOVE(policy_map, &policy_root, &search); - if (found) { - tor_assert(p == found->policy); - tor_free(found); - } + if (!p) + return; + + if (--p->refcnt <= 0) { + if (p->is_canonical) { + policy_map_ent_t search, *found; + search.policy = p; + found = HT_REMOVE(policy_map, &policy_root, &search); + if (found) { + tor_assert(p == found->policy); + tor_free(found); } - tor_free(p); } + tor_free(p); } } diff --git a/src/or/policies.h b/src/or/policies.h new file mode 100644 index 0000000000..b2947c67e7 --- /dev/null +++ b/src/or/policies.h @@ -0,0 +1,61 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file policies.h + * \brief Header file for policies.c. + **/ + +#ifndef _TOR_POLICIES_H +#define _TOR_POLICIES_H + +/* (length of "accept 255.255.255.255/255.255.255.255:65535-65535\n" plus a + * NUL.) + */ +#define POLICY_BUF_LEN 52 + +int firewall_is_fascist_or(void); +int fascist_firewall_allows_address_or(const tor_addr_t *addr, uint16_t port); +int fascist_firewall_allows_or(routerinfo_t *ri); +int fascist_firewall_allows_address_dir(const tor_addr_t *addr, uint16_t port); +int dir_policy_permits_address(const tor_addr_t *addr); +int socks_policy_permits_address(const tor_addr_t *addr); +int authdir_policy_permits_address(uint32_t addr, uint16_t port); +int authdir_policy_valid_address(uint32_t addr, uint16_t port); +int authdir_policy_baddir_address(uint32_t addr, uint16_t port); +int authdir_policy_badexit_address(uint32_t addr, uint16_t port); + +int validate_addr_policies(or_options_t *options, char **msg); +void policy_expand_private(smartlist_t **policy); +int policies_parse_from_options(or_options_t *options); + +addr_policy_t *addr_policy_get_canonical_entry(addr_policy_t *ent); +int cmp_addr_policies(smartlist_t *a, smartlist_t *b); +addr_policy_result_t compare_tor_addr_to_addr_policy(const tor_addr_t *addr, + uint16_t port, const smartlist_t *policy); +addr_policy_result_t compare_addr_to_addr_policy(uint32_t addr, + uint16_t port, const smartlist_t *policy); +int policies_parse_exit_policy(config_line_t *cfg, smartlist_t **dest, + int rejectprivate, const char *local_address, + int add_default_policy); +void policies_exit_policy_append_reject_star(smartlist_t **dest); +void policies_set_router_exitpolicy_to_reject_all(routerinfo_t *exitrouter); +int exit_policy_is_general_exit(smartlist_t *policy); +int policy_is_reject_star(const smartlist_t *policy); +int getinfo_helper_policies(control_connection_t *conn, + const char *question, char **answer, + const char **errmsg); +int policy_write_item(char *buf, size_t buflen, addr_policy_t *item, + int format_for_desc); + +void addr_policy_list_free(smartlist_t *p); +void addr_policy_free(addr_policy_t *p); +void policies_free_all(void); + +char *policy_summarize(smartlist_t *policy); + +#endif + diff --git a/src/or/reasons.c b/src/or/reasons.c index 38eb4078ca..319e6c055a 100644 --- a/src/or/reasons.c +++ b/src/or/reasons.c @@ -9,6 +9,8 @@ **/ #include "or.h" +#include "config.h" +#include "reasons.h" /***************************** Edge (stream) reasons **********************/ @@ -38,6 +40,8 @@ stream_end_reason_to_control_string(int reason) case END_STREAM_REASON_NET_UNREACHABLE: return "NET_UNREACHABLE"; case END_STREAM_REASON_SOCKSPROTOCOL: return "SOCKS_PROTOCOL"; + case END_STREAM_REASON_PRIVATE_ADDR: return "PRIVATE_ADDR"; + default: return NULL; } } @@ -123,6 +127,9 @@ stream_end_reason_to_socks5_response(int reason) return SOCKS5_NET_UNREACHABLE; case END_STREAM_REASON_SOCKSPROTOCOL: return SOCKS5_GENERAL_ERROR; + case END_STREAM_REASON_PRIVATE_ADDR: + return SOCKS5_GENERAL_ERROR; + default: log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, "Reason for ending (%d) not recognized; " @@ -167,13 +174,7 @@ errno_to_stream_end_reason(int e) S_CASE(ENETUNREACH): return END_STREAM_REASON_INTERNAL; S_CASE(EHOSTUNREACH): - /* XXXX022 - * The correct behavior is END_STREAM_REASON_NOROUTE, but older - * clients don't recognize it. So we're going to continue sending - * "MISC" until 0.2.1.27 or later is "well established". - */ - /* return END_STREAM_REASON_NOROUTE; */ - return END_STREAM_REASON_MISC; + return END_STREAM_REASON_NOROUTE; S_CASE(ECONNREFUSED): return END_STREAM_REASON_CONNECTREFUSED; S_CASE(ECONNRESET): @@ -332,9 +333,78 @@ circuit_end_reason_to_control_string(int reason) return "NOPATH"; case END_CIRC_REASON_NOSUCHSERVICE: return "NOSUCHSERVICE"; + case END_CIRC_REASON_MEASUREMENT_EXPIRED: + return "MEASUREMENT_EXPIRED"; default: log_warn(LD_BUG, "Unrecognized reason code %d", (int)reason); return NULL; } } +/** Return a string corresponding to a SOCKS4 reponse code. */ +const char * +socks4_response_code_to_string(uint8_t code) +{ + switch (code) { + case 0x5a: + return "connection accepted"; + case 0x5b: + return "server rejected connection"; + case 0x5c: + return "server cannot connect to identd on this client"; + case 0x5d: + return "user id does not match identd"; + default: + return "invalid SOCKS 4 response code"; + } +} + +/** Return a string corresponding to a SOCKS5 reponse code. */ +const char * +socks5_response_code_to_string(uint8_t code) +{ + switch (code) { + case 0x00: + return "connection accepted"; + case 0x01: + return "general SOCKS server failure"; + case 0x02: + return "connection not allowed by ruleset"; + case 0x03: + return "Network unreachable"; + case 0x04: + return "Host unreachable"; + case 0x05: + return "Connection refused"; + case 0x06: + return "TTL expired"; + case 0x07: + return "Command not supported"; + case 0x08: + return "Address type not supported"; + default: + return "unknown reason"; + } +} + +/** Return a string corresponding to a bandwidht_weight_rule_t */ +const char * +bandwidth_weight_rule_to_string(bandwidth_weight_rule_t rule) +{ + switch (rule) + { + case NO_WEIGHTING: + return "no weighting"; + case WEIGHT_FOR_EXIT: + return "weight as exit"; + case WEIGHT_FOR_MID: + return "weight as middle node"; + case WEIGHT_FOR_GUARD: + return "weight as guard"; + case WEIGHT_FOR_DIR: + return "weight as directory"; + default: + return "unknown rule"; + } +} + diff --git a/src/or/reasons.h b/src/or/reasons.h new file mode 100644 index 0000000000..01f9717948 --- /dev/null +++ b/src/or/reasons.h @@ -0,0 +1,31 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file reasons.h + * \brief Header file for reasons.c. + **/ + +#ifndef _TOR_REASONS_H +#define _TOR_REASONS_H + +const char *stream_end_reason_to_control_string(int reason); +const char *stream_end_reason_to_string(int reason); +socks5_reply_status_t stream_end_reason_to_socks5_response(int reason); +uint8_t errno_to_stream_end_reason(int e); + +const char *orconn_end_reason_to_control_string(int r); +int tls_error_to_orconn_end_reason(int e); +int errno_to_orconn_end_reason(int e); + +const char *circuit_end_reason_to_control_string(int reason); +const char *socks4_response_code_to_string(uint8_t code); +const char *socks5_response_code_to_string(uint8_t code); + +const char *bandwidth_weight_rule_to_string(enum bandwidth_weight_rule_t rule); + +#endif + diff --git a/src/or/relay.c b/src/or/relay.c index 59e2c5c969..9effae3036 100644 --- a/src/or/relay.c +++ b/src/or/relay.c @@ -10,8 +10,26 @@ * receiving from circuits, plus queuing on circuits. **/ +#include <math.h> #include "or.h" +#include "buffers.h" +#include "circuitbuild.h" +#include "circuitlist.h" +#include "config.h" +#include "connection.h" +#include "connection_edge.h" +#include "connection_or.h" +#include "control.h" +#include "geoip.h" +#include "main.h" #include "mempool.h" +#include "networkstatus.h" +#include "policies.h" +#include "reasons.h" +#include "relay.h" +#include "rendcommon.h" +#include "routerlist.h" +#include "routerparse.h" static int relay_crypt(circuit_t *circ, cell_t *cell, cell_direction_t cell_direction, @@ -20,20 +38,46 @@ static edge_connection_t *relay_lookup_conn(circuit_t *circ, cell_t *cell, cell_direction_t cell_direction, crypt_path_t *layer_hint); -static int -connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, - edge_connection_t *conn, - crypt_path_t *layer_hint); -static void -circuit_consider_sending_sendme(circuit_t *circ, crypt_path_t *layer_hint); +static int connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, + edge_connection_t *conn, + crypt_path_t *layer_hint); +static void circuit_consider_sending_sendme(circuit_t *circ, + crypt_path_t *layer_hint); +static void circuit_resume_edge_reading(circuit_t *circ, + crypt_path_t *layer_hint); +static int circuit_resume_edge_reading_helper(edge_connection_t *conn, + circuit_t *circ, + crypt_path_t *layer_hint); +static int circuit_consider_stop_edge_reading(circuit_t *circ, + crypt_path_t *layer_hint); +static int circuit_queue_streams_are_blocked(circuit_t *circ); + +/* XXXX023 move this all to compat_libevent */ +/** Cache the current hi-res time; the cache gets reset when libevent + * calls us. */ +static struct timeval cached_time_hires = {0, 0}; + +/** Stop reading on edge connections when we have this many cells + * waiting on the appropriate queue. */ +#define CELL_QUEUE_HIGHWATER_SIZE 256 +/** Start reading from edge connections again when we get down to this many + * cells. */ +#define CELL_QUEUE_LOWWATER_SIZE 64 + static void -circuit_resume_edge_reading(circuit_t *circ, crypt_path_t *layer_hint); -static int -circuit_resume_edge_reading_helper(edge_connection_t *conn, - circuit_t *circ, - crypt_path_t *layer_hint); -static int -circuit_consider_stop_edge_reading(circuit_t *circ, crypt_path_t *layer_hint); +tor_gettimeofday_cached(struct timeval *tv) +{ + if (cached_time_hires.tv_sec == 0) { + tor_gettimeofday(&cached_time_hires); + } + *tv = cached_time_hires; +} + +void +tor_gettimeofday_cache_clear(void) +{ + cached_time_hires.tv_sec = 0; +} /** Stats: how many relay cells have originated at this hop, or have * been relayed onward (not recognized at this hop)? @@ -230,7 +274,7 @@ circuit_receive_relay_cell(cell_t *cell, circuit_t *circ, * we might kill the circ before we relay * the cells. */ - append_cell_to_circuit_queue(circ, or_conn, cell, cell_direction); + append_cell_to_circuit_queue(circ, or_conn, cell, cell_direction, 0); return 0; } @@ -327,7 +371,7 @@ relay_crypt(circuit_t *circ, cell_t *cell, cell_direction_t cell_direction, static int circuit_package_relay_cell(cell_t *cell, circuit_t *circ, cell_direction_t cell_direction, - crypt_path_t *layer_hint) + crypt_path_t *layer_hint, streamid_t on_stream) { or_connection_t *conn; /* where to send the cell */ @@ -371,7 +415,7 @@ circuit_package_relay_cell(cell_t *cell, circuit_t *circ, } ++stats_n_relay_cells_relayed; - append_cell_to_circuit_queue(circ, conn, cell, cell_direction); + append_cell_to_circuit_queue(circ, conn, cell, cell_direction, on_stream); return 0; } @@ -496,7 +540,7 @@ relay_command_to_string(uint8_t command) * return 0. */ int -relay_send_command_from_edge(uint16_t stream_id, circuit_t *circ, +relay_send_command_from_edge(streamid_t stream_id, circuit_t *circ, uint8_t relay_command, const char *payload, size_t payload_len, crypt_path_t *cpath_layer) { @@ -531,6 +575,12 @@ relay_send_command_from_edge(uint16_t stream_id, circuit_t *circ, log_debug(LD_OR,"delivering %d cell %s.", relay_command, cell_direction == CELL_DIRECTION_OUT ? "forward" : "backward"); + /* If we are sending an END cell and this circuit is used for a tunneled + * directory request, advance its state. */ + if (relay_command == RELAY_COMMAND_END && circ->dirreq_id) + geoip_change_dirreq_state(circ->dirreq_id, DIRREQ_TUNNELED, + DIRREQ_END_CELL_SENT); + if (cell_direction == CELL_DIRECTION_OUT && circ->n_conn) { /* if we're using relaybandwidthrate, this conn wants priority */ circ->n_conn->client_used = approx_time(); @@ -540,17 +590,11 @@ relay_send_command_from_edge(uint16_t stream_id, circuit_t *circ, origin_circuit_t *origin_circ = TO_ORIGIN_CIRCUIT(circ); if (origin_circ->remaining_relay_early_cells > 0 && (relay_command == RELAY_COMMAND_EXTEND || - (cpath_layer != origin_circ->cpath && - !CIRCUIT_PURPOSE_IS_ESTABLISHED_REND(circ->purpose)))) { - /* If we've got any relay_early cells left, and we're sending - * an extend cell or (we're not talking to the first hop and we're - * not talking to a rendezvous circuit), use one of them. - * Don't worry about the conn protocol version: + cpath_layer != origin_circ->cpath)) { + /* If we've got any relay_early cells left and (we're sending + * an extend cell or we're not talking to the first hop), use + * one of them. Don't worry about the conn protocol version: * append_cell_to_circuit_queue will fix it up. */ - /* XXX For now, clients don't use RELAY_EARLY cells when sending - * relay cells on rendezvous circuits. See bug 1038. Eventually, - * we can take this behavior away in favor of having clients avoid - * rendezvous points running 0.2.1.3-alpha through 0.2.1.18. -RD */ cell.command = CELL_RELAY_EARLY; --origin_circ->remaining_relay_early_cells; log_debug(LD_OR, "Sending a RELAY_EARLY cell; %d remaining.", @@ -578,8 +622,8 @@ relay_send_command_from_edge(uint16_t stream_id, circuit_t *circ, } } - if (circuit_package_relay_cell(&cell, circ, cell_direction, cpath_layer) - < 0) { + if (circuit_package_relay_cell(&cell, circ, cell_direction, cpath_layer, + stream_id) < 0) { log_warn(LD_BUG,"circuit_package_relay_cell failed. Closing."); circuit_mark_for_close(circ, END_CIRC_REASON_INTERNAL); return -1; @@ -747,6 +791,8 @@ connection_ap_process_end_not_open( < MAX_RESOLVE_FAILURES) { /* We haven't retried too many times; reattach the connection. */ circuit_log_path(LOG_INFO,LD_APP,circ); + /* Mark this circuit "unusable for new streams". */ + /* XXXX023 this is a kludgy way to do this. */ tor_assert(circ->_base.timestamp_dirty); circ->_base.timestamp_dirty -= get_options()->MaxCircuitDirtiness; @@ -901,7 +947,7 @@ connection_edge_process_relay_cell_not_open( } /* handle anything that might have queued */ - if (connection_edge_package_raw_inbuf(conn, 1) < 0) { + if (connection_edge_package_raw_inbuf(conn, 1, NULL) < 0) { /* (We already sent an end cell if possible) */ connection_mark_for_close(TO_CONN(conn)); return 0; @@ -999,7 +1045,8 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, relay_header_unpack(&rh, cell->payload); // log_fn(LOG_DEBUG,"command %d stream %d", rh.command, rh.stream_id); num_seen++; - log_debug(domain, "Now seen %d relay cells here.", num_seen); + log_debug(domain, "Now seen %d relay cells here (command %d, stream %d).", + num_seen, rh.command, rh.stream_id); if (rh.length > RELAY_PAYLOAD_SIZE) { log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, @@ -1038,6 +1085,16 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, "Begin cell for known stream. Dropping."); return 0; } + if (rh.command == RELAY_COMMAND_BEGIN_DIR) { + /* Assign this circuit and its app-ward OR connection a unique ID, + * so that we can measure download times. The local edge and dir + * connection will be assigned the same ID when they are created + * and linked. */ + static uint64_t next_id = 0; + circ->dirreq_id = ++next_id; + TO_CONN(TO_OR_CIRCUIT(circ)->p_conn)->dirreq_id = circ->dirreq_id; + } + return connection_exit_begin_conn(cell, circ); case RELAY_COMMAND_DATA: ++stats_n_data_cells_received; @@ -1131,6 +1188,7 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, } if (circ->n_conn) { uint8_t trunc_reason = *(uint8_t*)(cell->payload + RELAY_HEADER_SIZE); + circuit_clear_cell_queue(circ, circ->n_conn); connection_or_send_destroy(circ->n_circ_id, circ->n_conn, trunc_reason); circuit_set_n_circid_orconn(circ, 0, NULL); @@ -1179,9 +1237,13 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, conn->package_window += STREAMWINDOW_INCREMENT; log_debug(domain,"stream-level sendme, packagewindow now %d.", conn->package_window); + if (circuit_queue_streams_are_blocked(circ)) { + /* Still waiting for queue to flush; don't touch conn */ + return 0; + } connection_start_reading(TO_CONN(conn)); /* handle whatever might still be on the inbuf */ - if (connection_edge_package_raw_inbuf(conn, 1) < 0) { + if (connection_edge_package_raw_inbuf(conn, 1, NULL) < 0) { /* (We already sent an end cell if possible) */ connection_mark_for_close(TO_CONN(conn)); return 0; @@ -1247,15 +1309,19 @@ uint64_t stats_n_data_cells_received = 0; * ever received were completely full of data. */ uint64_t stats_n_data_bytes_received = 0; -/** While conn->inbuf has an entire relay payload of bytes on it, - * and the appropriate package windows aren't empty, grab a cell - * and send it down the circuit. +/** If <b>conn</b> has an entire relay payload of bytes on its inbuf (or + * <b>package_partial</b> is true), and the appropriate package windows aren't + * empty, grab a cell and send it down the circuit. + * + * If *<b>max_cells</b> is given, package no more than max_cells. Decrement + * *<b>max_cells</b> by the number of cells packaged. * * Return -1 (and send a RELAY_COMMAND_END cell if necessary) if conn should * be marked for close, else return 0. */ int -connection_edge_package_raw_inbuf(edge_connection_t *conn, int package_partial) +connection_edge_package_raw_inbuf(edge_connection_t *conn, int package_partial, + int *max_cells) { size_t amount_to_process, length; char payload[CELL_PAYLOAD_SIZE]; @@ -1271,7 +1337,10 @@ connection_edge_package_raw_inbuf(edge_connection_t *conn, int package_partial) return 0; } -repeat_connection_edge_package_raw_inbuf: + if (max_cells && *max_cells <= 0) + return 0; + + repeat_connection_edge_package_raw_inbuf: circ = circuit_get_by_edge_conn(conn); if (!circ) { @@ -1332,12 +1401,19 @@ repeat_connection_edge_package_raw_inbuf: } log_debug(domain,"conn->package_window is now %d",conn->package_window); + if (max_cells) { + *max_cells -= 1; + if (*max_cells <= 0) + return 0; + } + /* handle more if there's more, or return 0 if there isn't */ goto repeat_connection_edge_package_raw_inbuf; } -/** Called when we've just received a relay data cell, or when - * we've just finished flushing all bytes to stream <b>conn</b>. +/** Called when we've just received a relay data cell, when + * we've just finished flushing all bytes to stream <b>conn</b>, + * or when we've flushed *some* bytes to the stream <b>conn</b>. * * If conn->outbuf is not too full, and our deliver window is * low, send back a suitable number of stream-level sendme cells. @@ -1379,7 +1455,10 @@ connection_edge_consider_sending_sendme(edge_connection_t *conn) static void circuit_resume_edge_reading(circuit_t *circ, crypt_path_t *layer_hint) { - + if (circuit_queue_streams_are_blocked(circ)) { + log_debug(layer_hint?LD_APP:LD_EXIT,"Too big queue, no resuming"); + return; + } log_debug(layer_hint?LD_APP:LD_EXIT,"resuming"); if (CIRCUIT_IS_ORIGIN(circ)) @@ -1395,31 +1474,136 @@ circuit_resume_edge_reading(circuit_t *circ, crypt_path_t *layer_hint) * of a linked list of edge streams that should each be considered. */ static int -circuit_resume_edge_reading_helper(edge_connection_t *conn, +circuit_resume_edge_reading_helper(edge_connection_t *first_conn, circuit_t *circ, crypt_path_t *layer_hint) { - for ( ; conn; conn=conn->next_stream) { - if (conn->_base.marked_for_close) + edge_connection_t *conn; + int n_packaging_streams, n_streams_left; + int packaged_this_round; + int cells_on_queue; + int cells_per_conn; + edge_connection_t *chosen_stream = NULL; + + /* How many cells do we have space for? It will be the minimum of + * the number needed to exhaust the package window, and the minimum + * needed to fill the cell queue. */ + int max_to_package = circ->package_window; + if (CIRCUIT_IS_ORIGIN(circ)) { + cells_on_queue = circ->n_conn_cells.n; + } else { + or_circuit_t *or_circ = TO_OR_CIRCUIT(circ); + cells_on_queue = or_circ->p_conn_cells.n; + } + if (CELL_QUEUE_HIGHWATER_SIZE - cells_on_queue < max_to_package) + max_to_package = CELL_QUEUE_HIGHWATER_SIZE - cells_on_queue; + + /* Once we used to start listening on the streams in the order they + * appeared in the linked list. That leads to starvation on the + * streams that appeared later on the list, since the first streams + * would always get to read first. Instead, we just pick a random + * stream on the list, and enable reading for streams starting at that + * point (and wrapping around as if the list were circular). It would + * probably be better to actually remember which streams we've + * serviced in the past, but this is simple and effective. */ + + /* Select a stream uniformly at random from the linked list. We + * don't need cryptographic randomness here. */ + { + int num_streams = 0; + for (conn = first_conn; conn; conn = conn->next_stream) { + num_streams++; + if ((tor_weak_random() % num_streams)==0) + chosen_stream = conn; + /* Invariant: chosen_stream has been chosen uniformly at random from + * among the first num_streams streams on first_conn. */ + } + } + + /* Count how many non-marked streams there are that have anything on + * their inbuf, and enable reading on all of the connections. */ + n_packaging_streams = 0; + /* Activate reading starting from the chosen stream */ + for (conn=chosen_stream; conn; conn = conn->next_stream) { + /* Start reading for the streams starting from here */ + if (conn->_base.marked_for_close || conn->package_window <= 0) continue; - if ((!layer_hint && conn->package_window > 0) || - (layer_hint && conn->package_window > 0 && - conn->cpath_layer == layer_hint)) { + if (!layer_hint || conn->cpath_layer == layer_hint) { connection_start_reading(TO_CONN(conn)); + + if (buf_datalen(conn->_base.inbuf) > 0) + ++n_packaging_streams; + } + } + /* Go back and do the ones we skipped, circular-style */ + for (conn = first_conn; conn != chosen_stream; conn = conn->next_stream) { + if (conn->_base.marked_for_close || conn->package_window <= 0) + continue; + if (!layer_hint || conn->cpath_layer == layer_hint) { + connection_start_reading(TO_CONN(conn)); + + if (buf_datalen(conn->_base.inbuf) > 0) + ++n_packaging_streams; + } + } + + if (n_packaging_streams == 0) /* avoid divide-by-zero */ + return 0; + + again: + + cells_per_conn = CEIL_DIV(max_to_package, n_packaging_streams); + + packaged_this_round = 0; + n_streams_left = 0; + + /* Iterate over all connections. Package up to cells_per_conn cells on + * each. Update packaged_this_round with the total number of cells + * packaged, and n_streams_left with the number that still have data to + * package. + */ + for (conn=first_conn; conn; conn=conn->next_stream) { + if (conn->_base.marked_for_close || conn->package_window <= 0) + continue; + if (!layer_hint || conn->cpath_layer == layer_hint) { + int n = cells_per_conn, r; /* handle whatever might still be on the inbuf */ - if (connection_edge_package_raw_inbuf(conn, 1)<0) { - /* (We already sent an end cell if possible) */ + r = connection_edge_package_raw_inbuf(conn, 1, &n); + + /* Note how many we packaged */ + packaged_this_round += (cells_per_conn-n); + + if (r<0) { + /* Problem while packaging. (We already sent an end cell if + * possible) */ connection_mark_for_close(TO_CONN(conn)); continue; } + /* If there's still data to read, we'll be coming back to this stream. */ + if (buf_datalen(conn->_base.inbuf)) + ++n_streams_left; + /* If the circuit won't accept any more data, return without looking * at any more of the streams. Any connections that should be stopped * have already been stopped by connection_edge_package_raw_inbuf. */ if (circuit_consider_stop_edge_reading(circ, layer_hint)) return -1; + /* XXXX should we also stop immediately if we fill up the cell queue? + * Probably. */ } } + + /* If we made progress, and we are willing to package more, and there are + * any streams left that want to package stuff... try again! + */ + if (packaged_this_round && packaged_this_round < max_to_package && + n_streams_left) { + max_to_package -= packaged_this_round; + n_packaging_streams = n_streams_left; + goto again; + } + return 0; } @@ -1488,13 +1672,6 @@ circuit_consider_sending_sendme(circuit_t *circ, crypt_path_t *layer_hint) } } -/** Stop reading on edge connections when we have this many cells - * waiting on the appropriate queue. */ -#define CELL_QUEUE_HIGHWATER_SIZE 256 -/** Start reading from edge connections again when we get down to this many - * cells. */ -#define CELL_QUEUE_LOWWATER_SIZE 64 - #ifdef ACTIVE_CIRCUITS_PARANOIA #define assert_active_circuits_ok_paranoid(conn) \ assert_active_circuits_ok(conn) @@ -1508,6 +1685,10 @@ static int total_cells_allocated = 0; /** A memory pool to allocate packed_cell_t objects. */ static mp_pool_t *cell_pool = NULL; +/** Memory pool to allocate insertion_time_elem_t objects used for cell + * statistics. */ +static mp_pool_t *it_pool = NULL; + /** Allocate structures to hold cells. */ void init_cell_pool(void) @@ -1516,7 +1697,8 @@ init_cell_pool(void) cell_pool = mp_pool_new(sizeof(packed_cell_t), 128*1024); } -/** Free all storage used to hold cells. */ +/** Free all storage used to hold cells (and insertion times if we measure + * cell statistics). */ void free_cell_pool(void) { @@ -1525,6 +1707,10 @@ free_cell_pool(void) mp_pool_destroy(cell_pool); cell_pool = NULL; } + if (it_pool) { + mp_pool_destroy(it_pool); + it_pool = NULL; + } } /** Free excess storage in cell pool. */ @@ -1537,7 +1723,7 @@ clean_cell_pool(void) /** Release storage held by <b>cell</b>. */ static INLINE void -packed_cell_free(packed_cell_t *cell) +packed_cell_free_unchecked(packed_cell_t *cell) { --total_cells_allocated; mp_pool_release(cell); @@ -1599,7 +1785,38 @@ cell_queue_append(cell_queue_t *queue, packed_cell_t *cell) void cell_queue_append_packed_copy(cell_queue_t *queue, const cell_t *cell) { - cell_queue_append(queue, packed_cell_copy(cell)); + packed_cell_t *copy = packed_cell_copy(cell); + /* Remember the time when this cell was put in the queue. */ + if (get_options()->CellStatistics) { + struct timeval now; + uint32_t added; + insertion_time_queue_t *it_queue = queue->insertion_times; + if (!it_pool) + it_pool = mp_pool_new(sizeof(insertion_time_elem_t), 1024); + tor_gettimeofday_cached(&now); +#define SECONDS_IN_A_DAY 86400L + added = (uint32_t)(((now.tv_sec % SECONDS_IN_A_DAY) * 100L) + + ((uint32_t)now.tv_usec / (uint32_t)10000L)); + if (!it_queue) { + it_queue = tor_malloc_zero(sizeof(insertion_time_queue_t)); + queue->insertion_times = it_queue; + } + if (it_queue->last && it_queue->last->insertion_time == added) { + it_queue->last->counter++; + } else { + insertion_time_elem_t *elem = mp_pool_get(it_pool); + elem->next = NULL; + elem->insertion_time = added; + elem->counter = 1; + if (it_queue->last) { + it_queue->last->next = elem; + it_queue->last = elem; + } else { + it_queue->first = it_queue->last = elem; + } + } + } + cell_queue_append(queue, copy); } /** Remove and free every cell in <b>queue</b>. */ @@ -1610,11 +1827,19 @@ cell_queue_clear(cell_queue_t *queue) cell = queue->head; while (cell) { next = cell->next; - packed_cell_free(cell); + packed_cell_free_unchecked(cell); cell = next; } queue->head = queue->tail = NULL; queue->n = 0; + if (queue->insertion_times) { + while (queue->insertion_times->first) { + insertion_time_elem_t *elem = queue->insertion_times->first; + queue->insertion_times->first = elem->next; + mp_pool_release(elem); + } + tor_free(queue->insertion_times); + } } /** Extract and return the cell at the head of <b>queue</b>; return NULL if @@ -1666,8 +1891,226 @@ prev_circ_on_conn_p(circuit_t *circ, or_connection_t *conn) } } +/** Helper for sorting cell_ewma_t values in their priority queue. */ +static int +compare_cell_ewma_counts(const void *p1, const void *p2) +{ + const cell_ewma_t *e1=p1, *e2=p2; + if (e1->cell_count < e2->cell_count) + return -1; + else if (e1->cell_count > e2->cell_count) + return 1; + else + return 0; +} + +/** Given a cell_ewma_t, return a pointer to the circuit containing it. */ +static circuit_t * +cell_ewma_to_circuit(cell_ewma_t *ewma) +{ + if (ewma->is_for_p_conn) { + /* This is an or_circuit_t's p_cell_ewma. */ + or_circuit_t *orcirc = SUBTYPE_P(ewma, or_circuit_t, p_cell_ewma); + return TO_CIRCUIT(orcirc); + } else { + /* This is some circuit's n_cell_ewma. */ + return SUBTYPE_P(ewma, circuit_t, n_cell_ewma); + } +} + +/* ==== Functions for scaling cell_ewma_t ==== + + When choosing which cells to relay first, we favor circuits that have been + quiet recently. This gives better latency on connections that aren't + pushing lots of data, and makes the network feel more interactive. + + Conceptually, we take an exponentially weighted mean average of the number + of cells a circuit has sent, and allow active circuits (those with cells to + relay) to send cells in reverse order of their exponentially-weighted mean + average (EWMA) cell count. [That is, a cell sent N seconds ago 'counts' + F^N times as much as a cell sent now, for 0<F<1.0, and we favor the + circuit that has sent the fewest cells] + + If 'double' had infinite precision, we could do this simply by counting a + cell sent at startup as having weight 1.0, and a cell sent N seconds later + as having weight F^-N. This way, we would never need to re-scale + any already-sent cells. + + To prevent double from overflowing, we could count a cell sent now as + having weight 1.0 and a cell sent N seconds ago as having weight F^N. + This, however, would mean we'd need to re-scale *ALL* old circuits every + time we wanted to send a cell. + + So as a compromise, we divide time into 'ticks' (currently, 10-second + increments) and say that a cell sent at the start of a current tick is + worth 1.0, a cell sent N seconds before the start of the current tick is + worth F^N, and a cell sent N seconds after the start of the current tick is + worth F^-N. This way we don't overflow, and we don't need to constantly + rescale. + */ + +/** How long does a tick last (seconds)? */ +#define EWMA_TICK_LEN 10 + +/** The default per-tick scale factor, if it hasn't been overridden by a + * consensus or a configuration setting. zero means "disabled". */ +#define EWMA_DEFAULT_HALFLIFE 0.0 + +/** Given a timeval <b>now</b>, compute the cell_ewma tick in which it occurs + * and the fraction of the tick that has elapsed between the start of the tick + * and <b>now</b>. Return the former and store the latter in + * *<b>remainder_out</b>. + * + * These tick values are not meant to be shared between Tor instances, or used + * for other purposes. */ +static unsigned +cell_ewma_tick_from_timeval(const struct timeval *now, + double *remainder_out) +{ + unsigned res = (unsigned) (now->tv_sec / EWMA_TICK_LEN); + /* rem */ + double rem = (now->tv_sec % EWMA_TICK_LEN) + + ((double)(now->tv_usec)) / 1.0e6; + *remainder_out = rem / EWMA_TICK_LEN; + return res; +} + +/** Compute and return the current cell_ewma tick. */ +unsigned +cell_ewma_get_tick(void) +{ + return ((unsigned)approx_time() / EWMA_TICK_LEN); +} + +/** The per-tick scale factor to be used when computing cell-count EWMA + * values. (A cell sent N ticks before the start of the current tick + * has value ewma_scale_factor ** N.) + */ +static double ewma_scale_factor = 0.1; +static int ewma_enabled = 0; + +#define EPSILON 0.00001 +#define LOG_ONEHALF -0.69314718055994529 + +/** Adjust the global cell scale factor based on <b>options</b> */ +void +cell_ewma_set_scale_factor(or_options_t *options, networkstatus_t *consensus) +{ + int32_t halflife_ms; + double halflife; + const char *source; + if (options && options->CircuitPriorityHalflife >= -EPSILON) { + halflife = options->CircuitPriorityHalflife; + source = "CircuitPriorityHalflife in configuration"; + } else if (consensus && (halflife_ms = networkstatus_get_param( + consensus, "CircuitPriorityHalflifeMsec", + -1, -1, INT32_MAX)) >= 0) { + halflife = ((double)halflife_ms)/1000.0; + source = "CircuitPriorityHalflifeMsec in consensus"; + } else { + halflife = EWMA_DEFAULT_HALFLIFE; + source = "Default value"; + } + + if (halflife <= EPSILON) { + /* The cell EWMA algorithm is disabled. */ + ewma_scale_factor = 0.1; + ewma_enabled = 0; + log_info(LD_OR, + "Disabled cell_ewma algorithm because of value in %s", + source); + } else { + /* convert halflife into halflife-per-tick. */ + halflife /= EWMA_TICK_LEN; + /* compute per-tick scale factor. */ + ewma_scale_factor = exp( LOG_ONEHALF / halflife ); + ewma_enabled = 1; + log_info(LD_OR, + "Enabled cell_ewma algorithm because of value in %s; " + "scale factor is %lf per %d seconds", + source, ewma_scale_factor, EWMA_TICK_LEN); + } +} + +/** Return the multiplier necessary to convert the value of a cell sent in + * 'from_tick' to one sent in 'to_tick'. */ +static INLINE double +get_scale_factor(unsigned from_tick, unsigned to_tick) +{ + /* This math can wrap around, but that's okay: unsigned overflow is + well-defined */ + int diff = (int)(to_tick - from_tick); + return pow(ewma_scale_factor, diff); +} + +/** Adjust the cell count of <b>ewma</b> so that it is scaled with respect to + * <b>cur_tick</b> */ +static void +scale_single_cell_ewma(cell_ewma_t *ewma, unsigned cur_tick) +{ + double factor = get_scale_factor(ewma->last_adjusted_tick, cur_tick); + ewma->cell_count *= factor; + ewma->last_adjusted_tick = cur_tick; +} + +/** Adjust the cell count of every active circuit on <b>conn</b> so + * that they are scaled with respect to <b>cur_tick</b> */ +static void +scale_active_circuits(or_connection_t *conn, unsigned cur_tick) +{ + + double factor = get_scale_factor( + conn->active_circuit_pqueue_last_recalibrated, + cur_tick); + /** Ordinarily it isn't okay to change the value of an element in a heap, + * but it's okay here, since we are preserving the order. */ + SMARTLIST_FOREACH(conn->active_circuit_pqueue, cell_ewma_t *, e, { + tor_assert(e->last_adjusted_tick == + conn->active_circuit_pqueue_last_recalibrated); + e->cell_count *= factor; + e->last_adjusted_tick = cur_tick; + }); + conn->active_circuit_pqueue_last_recalibrated = cur_tick; +} + +/** Rescale <b>ewma</b> to the same scale as <b>conn</b>, and add it to + * <b>conn</b>'s priority queue of active circuits */ +static void +add_cell_ewma_to_conn(or_connection_t *conn, cell_ewma_t *ewma) +{ + tor_assert(ewma->heap_index == -1); + scale_single_cell_ewma(ewma, + conn->active_circuit_pqueue_last_recalibrated); + + smartlist_pqueue_add(conn->active_circuit_pqueue, + compare_cell_ewma_counts, + STRUCT_OFFSET(cell_ewma_t, heap_index), + ewma); +} + +/** Remove <b>ewma</b> from <b>conn</b>'s priority queue of active circuits */ +static void +remove_cell_ewma_from_conn(or_connection_t *conn, cell_ewma_t *ewma) +{ + tor_assert(ewma->heap_index != -1); + smartlist_pqueue_remove(conn->active_circuit_pqueue, + compare_cell_ewma_counts, + STRUCT_OFFSET(cell_ewma_t, heap_index), + ewma); +} + +/** Remove and return the first cell_ewma_t from conn's priority queue of + * active circuits. Requires that the priority queue is nonempty. */ +static cell_ewma_t * +pop_first_cell_ewma_from_conn(or_connection_t *conn) +{ + return smartlist_pqueue_pop(conn->active_circuit_pqueue, + compare_cell_ewma_counts, + STRUCT_OFFSET(cell_ewma_t, heap_index)); +} + /** Add <b>circ</b> to the list of circuits with pending cells on - * <b>conn</b>. No effect if <b>circ</b> is already unlinked. */ + * <b>conn</b>. No effect if <b>circ</b> is already linked. */ void make_circuit_active_on_conn(circuit_t *circ, or_connection_t *conn) { @@ -1679,6 +2122,8 @@ make_circuit_active_on_conn(circuit_t *circ, or_connection_t *conn) return; } + assert_active_circuits_ok_paranoid(conn); + if (! conn->active_circuits) { conn->active_circuits = circ; *prevp = *nextp = circ; @@ -1690,10 +2135,19 @@ make_circuit_active_on_conn(circuit_t *circ, or_connection_t *conn) *prev_circ_on_conn_p(head, conn) = circ; *prevp = old_tail; } + + if (circ->n_conn == conn) { + add_cell_ewma_to_conn(conn, &circ->n_cell_ewma); + } else { + or_circuit_t *orcirc = TO_OR_CIRCUIT(circ); + tor_assert(conn == orcirc->p_conn); + add_cell_ewma_to_conn(conn, &orcirc->p_cell_ewma); + } + assert_active_circuits_ok_paranoid(conn); } -/** Remove <b>circ</b> to the list of circuits with pending cells on +/** Remove <b>circ</b> from the list of circuits with pending cells on * <b>conn</b>. No effect if <b>circ</b> is already unlinked. */ void make_circuit_inactive_on_conn(circuit_t *circ, or_connection_t *conn) @@ -1707,6 +2161,8 @@ make_circuit_inactive_on_conn(circuit_t *circ, or_connection_t *conn) return; } + assert_active_circuits_ok_paranoid(conn); + tor_assert(next && prev); tor_assert(*prev_circ_on_conn_p(next, conn) == circ); tor_assert(*next_circ_on_conn_p(prev, conn) == circ); @@ -1720,6 +2176,15 @@ make_circuit_inactive_on_conn(circuit_t *circ, or_connection_t *conn) conn->active_circuits = next; } *prevp = *nextp = NULL; + + if (circ->n_conn == conn) { + remove_cell_ewma_from_conn(conn, &circ->n_cell_ewma); + } else { + or_circuit_t *orcirc = TO_OR_CIRCUIT(circ); + tor_assert(conn == orcirc->p_conn); + remove_cell_ewma_from_conn(conn, &orcirc->p_cell_ewma); + } + assert_active_circuits_ok_paranoid(conn); } @@ -1739,16 +2204,27 @@ connection_or_unlink_all_active_circs(or_connection_t *orconn) cur = next; } while (cur != head); orconn->active_circuits = NULL; + + SMARTLIST_FOREACH(orconn->active_circuit_pqueue, cell_ewma_t *, e, + e->heap_index = -1); + smartlist_clear(orconn->active_circuit_pqueue); } /** Block (if <b>block</b> is true) or unblock (if <b>block</b> is false) * every edge connection that is using <b>circ</b> to write to <b>orconn</b>, - * and start or stop reading as appropriate. */ -static void + * and start or stop reading as appropriate. + * + * If <b>stream_id</b> is nonzero, block only the edge connection whose + * stream_id matches it. + * + * Returns the number of streams whose status we changed. + */ +static int set_streams_blocked_on_circ(circuit_t *circ, or_connection_t *orconn, - int block) + int block, streamid_t stream_id) { edge_connection_t *edge = NULL; + int n = 0; if (circ->n_conn == orconn) { circ->streams_blocked_on_n_conn = block; if (CIRCUIT_IS_ORIGIN(circ)) @@ -1761,7 +2237,13 @@ set_streams_blocked_on_circ(circuit_t *circ, or_connection_t *orconn, for (; edge; edge = edge->next_stream) { connection_t *conn = TO_CONN(edge); - edge->edge_blocked_on_circ = block; + if (stream_id && edge->stream_id != stream_id) + continue; + + if (edge->edge_blocked_on_circ != block) { + ++n; + edge->edge_blocked_on_circ = block; + } if (!conn->read_event) { /* This connection is a placeholder for something; probably a DNS @@ -1778,10 +2260,12 @@ set_streams_blocked_on_circ(circuit_t *circ, or_connection_t *orconn, connection_start_reading(conn); } } + + return n; } /** Pull as many cells as possible (but no more than <b>max</b>) from the - * queue of the first active circuit on <b>conn</b>, and write then to + * queue of the first active circuit on <b>conn</b>, and write them to * <b>conn</b>->outbuf. Return the number of cells written. Advance * the active circuit pointer to the next active circuit in the ring. */ int @@ -1792,9 +2276,35 @@ connection_or_flush_from_first_active_circuit(or_connection_t *conn, int max, cell_queue_t *queue; circuit_t *circ; int streams_blocked; + + /* The current (hi-res) time */ + struct timeval now_hires; + + /* The EWMA cell counter for the circuit we're flushing. */ + cell_ewma_t *cell_ewma = NULL; + double ewma_increment = -1; + circ = conn->active_circuits; if (!circ) return 0; assert_active_circuits_ok_paranoid(conn); + + /* See if we're doing the ewma circuit selection algorithm. */ + if (ewma_enabled) { + unsigned tick; + double fractional_tick; + tor_gettimeofday_cached(&now_hires); + tick = cell_ewma_tick_from_timeval(&now_hires, &fractional_tick); + + if (tick != conn->active_circuit_pqueue_last_recalibrated) { + scale_active_circuits(conn, tick); + } + + ewma_increment = pow(ewma_scale_factor, -fractional_tick); + + cell_ewma = smartlist_get(conn->active_circuit_pqueue, 0); + circ = cell_ewma_to_circuit(cell_ewma); + } + if (circ->n_conn == conn) { queue = &circ->n_conn_cells; streams_blocked = circ->streams_blocked_on_n_conn; @@ -1808,10 +2318,60 @@ connection_or_flush_from_first_active_circuit(or_connection_t *conn, int max, packed_cell_t *cell = cell_queue_pop(queue); tor_assert(*next_circ_on_conn_p(circ,conn)); + /* Calculate the exact time that this cell has spent in the queue. */ + if (get_options()->CellStatistics && !CIRCUIT_IS_ORIGIN(circ)) { + struct timeval now; + uint32_t flushed; + uint32_t cell_waiting_time; + insertion_time_queue_t *it_queue = queue->insertion_times; + tor_gettimeofday_cached(&now); + flushed = (uint32_t)((now.tv_sec % SECONDS_IN_A_DAY) * 100L + + (uint32_t)now.tv_usec / (uint32_t)10000L); + if (!it_queue || !it_queue->first) { + log_info(LD_GENERAL, "Cannot determine insertion time of cell. " + "Looks like the CellStatistics option was " + "recently enabled."); + } else { + or_circuit_t *orcirc = TO_OR_CIRCUIT(circ); + insertion_time_elem_t *elem = it_queue->first; + cell_waiting_time = + (uint32_t)((flushed * 10L + SECONDS_IN_A_DAY * 1000L - + elem->insertion_time * 10L) % + (SECONDS_IN_A_DAY * 1000L)); +#undef SECONDS_IN_A_DAY + elem->counter--; + if (elem->counter < 1) { + it_queue->first = elem->next; + if (elem == it_queue->last) + it_queue->last = NULL; + mp_pool_release(elem); + } + orcirc->total_cell_waiting_time += cell_waiting_time; + orcirc->processed_cells++; + } + } + + /* If we just flushed our queue and this circuit is used for a + * tunneled directory request, possibly advance its state. */ + if (queue->n == 0 && TO_CONN(conn)->dirreq_id) + geoip_change_dirreq_state(TO_CONN(conn)->dirreq_id, + DIRREQ_TUNNELED, + DIRREQ_CIRC_QUEUE_FLUSHED); + connection_write_to_buf(cell->body, CELL_NETWORK_SIZE, TO_CONN(conn)); - packed_cell_free(cell); + packed_cell_free_unchecked(cell); ++n_flushed; + if (cell_ewma) { + cell_ewma_t *tmp; + cell_ewma->cell_count += ewma_increment; + /* We pop and re-add the cell_ewma_t here, not above, since we need to + * re-add it immediately to keep the priority queue consistent with + * the linked-list implementation */ + tmp = pop_first_cell_ewma_from_conn(conn); + tor_assert(tmp == cell_ewma); + add_cell_ewma_to_conn(conn, cell_ewma); + } if (circ != conn->active_circuits) { /* If this happens, the current circuit just got made inactive by * a call in connection_write_to_buf(). That's nothing to worry about: @@ -1829,9 +2389,9 @@ connection_or_flush_from_first_active_circuit(or_connection_t *conn, int max, /* Is the cell queue low enough to unblock all the streams that are waiting * to write to this circuit? */ if (streams_blocked && queue->n <= CELL_QUEUE_LOWWATER_SIZE) - set_streams_blocked_on_circ(circ, conn, 0); /* unblock streams */ + set_streams_blocked_on_circ(circ, conn, 0, 0); /* unblock streams */ - /* Did we just ran out of cells on this queue? */ + /* Did we just run out of cells on this circuit's queue? */ if (queue->n == 0) { log_debug(LD_GENERAL, "Made a circuit inactive."); make_circuit_inactive_on_conn(circ, conn); @@ -1846,10 +2406,14 @@ connection_or_flush_from_first_active_circuit(or_connection_t *conn, int max, * transmitting in <b>direction</b>. */ void append_cell_to_circuit_queue(circuit_t *circ, or_connection_t *orconn, - cell_t *cell, cell_direction_t direction) + cell_t *cell, cell_direction_t direction, + streamid_t fromstream) { cell_queue_t *queue; int streams_blocked; + if (circ->marked_for_close) + return; + if (direction == CELL_DIRECTION_OUT) { queue = &circ->n_conn_cells; streams_blocked = circ->streams_blocked_on_n_conn; @@ -1868,7 +2432,12 @@ append_cell_to_circuit_queue(circuit_t *circ, or_connection_t *orconn, /* If we have too many cells on the circuit, we should stop reading from * the edge streams for a while. */ if (!streams_blocked && queue->n >= CELL_QUEUE_HIGHWATER_SIZE) - set_streams_blocked_on_circ(circ, orconn, 1); /* block streams */ + set_streams_blocked_on_circ(circ, orconn, 1, 0); /* block streams */ + + if (streams_blocked && fromstream) { + /* This edge connection is apparently not blocked; block it. */ + set_streams_blocked_on_circ(circ, orconn, 1, fromstream); + } if (queue->n == 1) { /* This was the first cell added to the queue. We need to make this @@ -1947,6 +2516,25 @@ decode_address_from_payload(tor_addr_t *addr_out, const uint8_t *payload, return payload + 2 + payload[1]; } +/** Remove all the cells queued on <b>circ</b> for <b>orconn</b>. */ +void +circuit_clear_cell_queue(circuit_t *circ, or_connection_t *orconn) +{ + cell_queue_t *queue; + if (circ->n_conn == orconn) { + queue = &circ->n_conn_cells; + } else { + or_circuit_t *orcirc = TO_OR_CIRCUIT(circ); + tor_assert(orcirc->p_conn == orconn); + queue = &orcirc->p_conn_cells; + } + + if (queue->n) + make_circuit_inactive_on_conn(circ,orconn); + + cell_queue_clear(queue); +} + /** Fail with an assert if the active circuits ring on <b>orconn</b> is * corrupt. */ void @@ -1954,16 +2542,44 @@ assert_active_circuits_ok(or_connection_t *orconn) { circuit_t *head = orconn->active_circuits; circuit_t *cur = head; + int n = 0; if (! head) return; do { circuit_t *next = *next_circ_on_conn_p(cur, orconn); circuit_t *prev = *prev_circ_on_conn_p(cur, orconn); + cell_ewma_t *ewma; tor_assert(next); tor_assert(prev); tor_assert(*next_circ_on_conn_p(prev, orconn) == cur); tor_assert(*prev_circ_on_conn_p(next, orconn) == cur); + if (orconn == cur->n_conn) { + ewma = &cur->n_cell_ewma; + tor_assert(!ewma->is_for_p_conn); + } else { + ewma = &TO_OR_CIRCUIT(cur)->p_cell_ewma; + tor_assert(ewma->is_for_p_conn); + } + tor_assert(ewma->heap_index != -1); + tor_assert(ewma == smartlist_get(orconn->active_circuit_pqueue, + ewma->heap_index)); + n++; cur = next; } while (cur != head); + + tor_assert(n == smartlist_len(orconn->active_circuit_pqueue)); +} + +/** Return 1 if we shouldn't restart reading on this circuit, even if + * we get a SENDME. Else return 0. +*/ +static int +circuit_queue_streams_are_blocked(circuit_t *circ) +{ + if (CIRCUIT_IS_ORIGIN(circ)) { + return circ->streams_blocked_on_n_conn; + } else { + return circ->streams_blocked_on_p_conn; + } } diff --git a/src/or/relay.h b/src/or/relay.h new file mode 100644 index 0000000000..f64752da5d --- /dev/null +++ b/src/or/relay.h @@ -0,0 +1,70 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file relay.h + * \brief Header file for relay.c. + **/ + +#ifndef _TOR_RELAY_H +#define _TOR_RELAY_H + +extern uint64_t stats_n_relay_cells_relayed; +extern uint64_t stats_n_relay_cells_delivered; + +int circuit_receive_relay_cell(cell_t *cell, circuit_t *circ, + cell_direction_t cell_direction); + +void relay_header_pack(uint8_t *dest, const relay_header_t *src); +void relay_header_unpack(relay_header_t *dest, const uint8_t *src); +int relay_send_command_from_edge(streamid_t stream_id, circuit_t *circ, + uint8_t relay_command, const char *payload, + size_t payload_len, crypt_path_t *cpath_layer); +int connection_edge_send_command(edge_connection_t *fromconn, + uint8_t relay_command, const char *payload, + size_t payload_len); +int connection_edge_package_raw_inbuf(edge_connection_t *conn, + int package_partial, + int *max_cells); +void connection_edge_consider_sending_sendme(edge_connection_t *conn); + +extern uint64_t stats_n_data_cells_packaged; +extern uint64_t stats_n_data_bytes_packaged; +extern uint64_t stats_n_data_cells_received; +extern uint64_t stats_n_data_bytes_received; + +void init_cell_pool(void); +void free_cell_pool(void); +void clean_cell_pool(void); +void dump_cell_pool_usage(int severity); + +void cell_queue_clear(cell_queue_t *queue); +void cell_queue_append(cell_queue_t *queue, packed_cell_t *cell); +void cell_queue_append_packed_copy(cell_queue_t *queue, const cell_t *cell); + +void append_cell_to_circuit_queue(circuit_t *circ, or_connection_t *orconn, + cell_t *cell, cell_direction_t direction, + streamid_t fromstream); +void connection_or_unlink_all_active_circs(or_connection_t *conn); +int connection_or_flush_from_first_active_circuit(or_connection_t *conn, + int max, time_t now); +void assert_active_circuits_ok(or_connection_t *orconn); +void make_circuit_inactive_on_conn(circuit_t *circ, or_connection_t *conn); +void make_circuit_active_on_conn(circuit_t *circ, or_connection_t *conn); + +int append_address_to_payload(uint8_t *payload_out, const tor_addr_t *addr); +const uint8_t *decode_address_from_payload(tor_addr_t *addr_out, + const uint8_t *payload, + int payload_len); +unsigned cell_ewma_get_tick(void); +void cell_ewma_set_scale_factor(or_options_t *options, + networkstatus_t *consensus); +void circuit_clear_cell_queue(circuit_t *circ, or_connection_t *orconn); + +void tor_gettimeofday_cache_clear(void); + +#endif + diff --git a/src/or/rendclient.c b/src/or/rendclient.c index 7bda70562b..77e11c2a07 100644 --- a/src/or/rendclient.c +++ b/src/or/rendclient.c @@ -8,6 +8,23 @@ **/ #include "or.h" +#include "circuitbuild.h" +#include "circuitlist.h" +#include "circuituse.h" +#include "config.h" +#include "connection.h" +#include "connection_edge.h" +#include "directory.h" +#include "main.h" +#include "relay.h" +#include "rendclient.h" +#include "rendcommon.h" +#include "rephist.h" +#include "routerlist.h" + +static extend_info_t *rend_client_get_random_intro_impl( + const rend_cache_entry_t *rend_query, + const int strict, const int warnings); /** Called when we've established a circuit to an introduction point: * send the introduction request. */ @@ -49,6 +66,50 @@ rend_client_send_establish_rendezvous(origin_circuit_t *circ) return 0; } +/** Extend the introduction circuit <b>circ</b> to another valid + * introduction point for the hidden service it is trying to connect + * to, or mark it and launch a new circuit if we can't extend it. + * Return 0 on success. Return -1 and mark the introduction + * circuit on failure. + * + * On failure, the caller is responsible for marking the associated + * rendezvous circuit for close. */ +static int +rend_client_reextend_intro_circuit(origin_circuit_t *circ) +{ + extend_info_t *extend_info; + int result; + extend_info = rend_client_get_random_intro(circ->rend_data); + if (!extend_info) { + log_warn(LD_REND, + "No usable introduction points left for %s. Closing.", + safe_str_client(circ->rend_data->onion_address)); + circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_INTERNAL); + return -1; + } + if (circ->remaining_relay_early_cells) { + log_info(LD_REND, + "Re-extending circ %d, this time to %s.", + circ->_base.n_circ_id, extend_info->nickname); + result = circuit_extend_to_new_exit(circ, extend_info); + } else { + log_info(LD_REND, + "Building a new introduction circuit, this time to %s.", + extend_info->nickname); + circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_FINISHED); + if (!circuit_launch_by_extend_info(CIRCUIT_PURPOSE_C_INTRODUCING, + extend_info, + CIRCLAUNCH_IS_INTERNAL)) { + log_warn(LD_REND, "Building introduction circuit failed."); + result = -1; + } else { + result = 0; + } + } + extend_info_free(extend_info); + return result; +} + /** Called when we're trying to connect an ap conn; sends an INTRODUCE1 cell * down introcirc if possible. */ @@ -63,7 +124,7 @@ rend_client_send_introduction(origin_circuit_t *introcirc, rend_cache_entry_t *entry; crypt_path_t *cpath; off_t dh_offset; - crypto_pk_env_t *intro_key; /* either Bob's public key or an intro key. */ + crypto_pk_env_t *intro_key = NULL; tor_assert(introcirc->_base.purpose == CIRCUIT_PURPOSE_C_INTRODUCING); tor_assert(rendcirc->_base.purpose == CIRCUIT_PURPOSE_C_REND_READY); @@ -77,19 +138,14 @@ rend_client_send_introduction(origin_circuit_t *introcirc, log_info(LD_REND, "query %s didn't have valid rend desc in cache. " "Refetching descriptor.", - safe_str(introcirc->rend_data->onion_address)); - /* Fetch both v0 and v2 rend descriptors in parallel. Use whichever - * arrives first. Exception: When using client authorization, only - * fetch v2 descriptors.*/ + safe_str_client(introcirc->rend_data->onion_address)); rend_client_refetch_v2_renddesc(introcirc->rend_data); - if (introcirc->rend_data->auth_type == REND_NO_AUTH) - rend_client_refetch_renddesc(introcirc->rend_data->onion_address); { connection_t *conn; while ((conn = connection_get_by_type_state_rendquery(CONN_TYPE_AP, AP_CONN_STATE_CIRCUIT_WAIT, - introcirc->rend_data->onion_address, -1))) { + introcirc->rend_data->onion_address))) { conn->state = AP_CONN_STATE_RENDDESC_WAIT; } } @@ -98,42 +154,27 @@ rend_client_send_introduction(origin_circuit_t *introcirc, } /* first 20 bytes of payload are the hash of Bob's pk */ - if (entry->parsed->version == 0) { /* un-versioned descriptor */ - intro_key = entry->parsed->pk; - } else { /* versioned descriptor */ - intro_key = NULL; - SMARTLIST_FOREACH(entry->parsed->intro_nodes, rend_intro_point_t *, - intro, { - if (tor_memeq(introcirc->build_state->chosen_exit->identity_digest, - intro->extend_info->identity_digest, DIGEST_LEN)) { - intro_key = intro->intro_key; - break; - } - }); - if (!intro_key) { - /** XXX This case probably means that the intro point vanished while - * we were building a circuit to it. In the future, we should find - * out how that happened and whether we should kill the circuits to - * removed intro points immediately. See task 1073. */ - int num_intro_points = smartlist_len(entry->parsed->intro_nodes); - if (rend_cache_lookup_entry(introcirc->rend_data->onion_address, - 0, &entry) > 0) { - log_info(LD_REND, "We have both a v0 and a v2 rend desc for this " - "service. The v2 desc doesn't contain the introduction " - "point (and key) to send an INTRODUCE1/2 cell to this " - "introduction point. Assuming the introduction point " - "is for v0 rend clients and using the service key " - "from the v0 desc instead. (This is probably a bug, " - "because we shouldn't even have both a v0 and a v2 " - "descriptor for the same service.)"); - /* See flyspray task 1024. */ - intro_key = entry->parsed->pk; - } else { - log_info(LD_REND, "Internal error: could not find intro key; we " - "only have a v2 rend desc with %d intro points.", - num_intro_points); - goto perm_err; - } + intro_key = NULL; + SMARTLIST_FOREACH(entry->parsed->intro_nodes, rend_intro_point_t *, + intro, { + if (tor_memeq(introcirc->build_state->chosen_exit->identity_digest, + intro->extend_info->identity_digest, DIGEST_LEN)) { + intro_key = intro->intro_key; + break; + } + }); + if (!intro_key) { + log_info(LD_REND, "Could not find intro key for %s at %s; we " + "have a v2 rend desc with %d intro points. " + "Trying a different intro point...", + safe_str_client(introcirc->rend_data->onion_address), + introcirc->build_state->chosen_exit->nickname, + smartlist_len(entry->parsed->intro_nodes)); + + if (rend_client_reextend_intro_circuit(introcirc)) { + goto perm_err; + } else { + return -1; } } if (crypto_pk_get_digest(intro_key, payload)<0) { @@ -236,8 +277,9 @@ rend_client_send_introduction(origin_circuit_t *introcirc, introcirc->_base.purpose = CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT; return 0; -perm_err: - circuit_mark_for_close(TO_CIRCUIT(introcirc), END_CIRC_REASON_INTERNAL); + perm_err: + if (!introcirc->_base.marked_for_close) + circuit_mark_for_close(TO_CIRCUIT(introcirc), END_CIRC_REASON_INTERNAL); circuit_mark_for_close(TO_CIRCUIT(rendcirc), END_CIRC_REASON_INTERNAL); return -2; } @@ -300,45 +342,16 @@ rend_client_introduction_acked(origin_circuit_t *circ, * points. If any remain, extend to a new one and try again. * If none remain, refetch the service descriptor. */ + log_info(LD_REND, "Got nack for %s from %s...", + safe_str_client(circ->rend_data->onion_address), + circ->build_state->chosen_exit->nickname); if (rend_client_remove_intro_point(circ->build_state->chosen_exit, circ->rend_data) > 0) { /* There are introduction points left. Re-extend the circuit to * another intro point and try again. */ - extend_info_t *extend_info; - int result; - extend_info = rend_client_get_random_intro(circ->rend_data); - if (!extend_info) { - log_warn(LD_REND, "No introduction points left for %s. Closing.", - escaped_safe_str(circ->rend_data->onion_address)); - circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_INTERNAL); - return -1; - } - if (circ->remaining_relay_early_cells) { - log_info(LD_REND, - "Got nack for %s from %s. Re-extending circ %d, " - "this time to %s.", - escaped_safe_str(circ->rend_data->onion_address), - circ->build_state->chosen_exit->nickname, - circ->_base.n_circ_id, extend_info->nickname); - result = circuit_extend_to_new_exit(circ, extend_info); - } else { - log_info(LD_REND, - "Got nack for %s from %s. Building a new introduction " - "circuit, this time to %s.", - escaped_safe_str(circ->rend_data->onion_address), - circ->build_state->chosen_exit->nickname, - extend_info->nickname); - circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_FINISHED); - if (!circuit_launch_by_extend_info(CIRCUIT_PURPOSE_C_INTRODUCING, - extend_info, - CIRCLAUNCH_IS_INTERNAL)) { - log_warn(LD_REND, "Building introduction circuit failed."); - result = -1; - } else { - result = 0; - } - } - extend_info_free(extend_info); + int result = rend_client_reextend_intro_circuit(circ); + /* XXXX If that call failed, should we close the rend circuit, + * too? */ return result; } } @@ -488,45 +501,21 @@ directory_get_from_hs_dir(const char *desc_id, const rend_data_t *rend_query) rend_query->onion_address, desc_id_base32, rend_query->auth_type, (rend_query->auth_type == REND_NO_AUTH ? "[none]" : - escaped_safe_str(descriptor_cookie_base64)), + escaped_safe_str_client(descriptor_cookie_base64)), hs_dir->nickname, hs_dir->dir_port); return 1; } -/** If we are not currently fetching a rendezvous service descriptor - * for the service ID <b>query</b>, start a directory connection to fetch a - * new one. - */ -void -rend_client_refetch_renddesc(const char *query) -{ - if (!get_options()->FetchHidServDescriptors) - return; - log_info(LD_REND, "Fetching rendezvous descriptor for service %s", - escaped_safe_str(query)); - if (connection_get_by_type_state_rendquery(CONN_TYPE_DIR, 0, query, 0)) { - log_info(LD_REND,"Would fetch a new renddesc here (for %s), but one is " - "already in progress.", escaped_safe_str(query)); - } else { - /* not one already; initiate a dir rend desc lookup */ - directory_get_from_dirserver(DIR_PURPOSE_FETCH_RENDDESC, - ROUTER_PURPOSE_GENERAL, query, - PDS_RETRY_IF_NO_SERVERS); - } -} - -/** Start a connection to a hidden service directory to fetch a v2 - * rendezvous service descriptor for the base32-encoded service ID - * <b>query</b>. - */ +/** Unless we already have a descriptor for <b>rend_query</b> with at least + * one (possibly) working introduction point in it, start a connection to a + * hidden service directory to fetch a v2 rendezvous service descriptor. */ void rend_client_refetch_v2_renddesc(const rend_data_t *rend_query) { char descriptor_id[DIGEST_LEN]; int replicas_left_to_try[REND_NUMBER_OF_NON_CONSECUTIVE_REPLICAS]; - int i, tries_left, r; + int i, tries_left; rend_cache_entry_t *e = NULL; - time_t now = time(NULL); tor_assert(rend_query); /* Are we configured to fetch descriptors? */ if (!get_options()->FetchHidServDescriptors) { @@ -535,15 +524,13 @@ rend_client_refetch_v2_renddesc(const rend_data_t *rend_query) return; } /* Before fetching, check if we already have the descriptor here. */ - r = rend_cache_lookup_entry(rend_query->onion_address, -1, &e); - if (r > 0 && now - e->received < NUM_SECONDS_BEFORE_HS_REFETCH) { + if (rend_cache_lookup_entry(rend_query->onion_address, -1, &e) > 0) { log_info(LD_REND, "We would fetch a v2 rendezvous descriptor, but we " - "already have a fresh copy of that descriptor here. " - "Not fetching."); + "already have that descriptor here. Not fetching."); return; } log_debug(LD_REND, "Fetching v2 rendezvous descriptor for service %s", - safe_str(rend_query->onion_address)); + safe_str_client(rend_query->onion_address)); /* Randomly iterate over the replicas until a descriptor can be fetched * from one of the consecutive nodes, or no options are left. */ tries_left = REND_NUMBER_OF_NON_CONSECUTIVE_REPLICAS; @@ -569,8 +556,8 @@ rend_client_refetch_v2_renddesc(const rend_data_t *rend_query) log_info(LD_REND, "Could not pick one of the responsible hidden " "service directories to fetch descriptors, because " "we already tried them all unsuccessfully."); - /* Close pending connections (unless a v0 request is still going on). */ - rend_client_desc_trynow(rend_query->onion_address, 2); + /* Close pending connections. */ + rend_client_desc_trynow(rend_query->onion_address); return; } @@ -600,9 +587,8 @@ rend_client_cancel_descriptor_fetches(void) "Marking for close dir conn fetching rendezvous " "descriptor for unknown service!"); } else { - log_debug(LD_REND, "Marking for close dir conn fetching v%d " + log_debug(LD_REND, "Marking for close dir conn fetching " "rendezvous descriptor for service %s", - (int)(rd->rend_desc_version), safe_str(rd->onion_address)); } connection_mark_for_close(conn); @@ -628,18 +614,13 @@ rend_client_remove_intro_point(extend_info_t *failed_intro, r = rend_cache_lookup_entry(rend_query->onion_address, -1, &ent); if (r<0) { log_warn(LD_BUG, "Malformed service ID %s.", - escaped_safe_str(rend_query->onion_address)); + escaped_safe_str_client(rend_query->onion_address)); return -1; } if (r==0) { log_info(LD_REND, "Unknown service %s. Re-fetching descriptor.", - escaped_safe_str(rend_query->onion_address)); - /* Fetch both, v0 and v2 rend descriptors in parallel. Use whichever - * arrives first. Exception: When using client authorization, only - * fetch v2 descriptors.*/ + escaped_safe_str_client(rend_query->onion_address)); rend_client_refetch_v2_renddesc(rend_query); - if (rend_query->auth_type == REND_NO_AUTH) - rend_client_refetch_renddesc(rend_query->onion_address); return 0; } @@ -653,21 +634,16 @@ rend_client_remove_intro_point(extend_info_t *failed_intro, } } - if (smartlist_len(ent->parsed->intro_nodes) == 0) { + if (! rend_client_any_intro_points_usable(ent)) { log_info(LD_REND, "No more intro points remain for %s. Re-fetching descriptor.", - escaped_safe_str(rend_query->onion_address)); - /* Fetch both, v0 and v2 rend descriptors in parallel. Use whichever - * arrives first. Exception: When using client authorization, only - * fetch v2 descriptors.*/ + escaped_safe_str_client(rend_query->onion_address)); rend_client_refetch_v2_renddesc(rend_query); - if (rend_query->auth_type == REND_NO_AUTH) - rend_client_refetch_renddesc(rend_query->onion_address); /* move all pending streams back to renddesc_wait */ while ((conn = connection_get_by_type_state_rendquery(CONN_TYPE_AP, AP_CONN_STATE_CIRCUIT_WAIT, - rend_query->onion_address, -1))) { + rend_query->onion_address))) { conn->state = AP_CONN_STATE_RENDDESC_WAIT; } @@ -675,7 +651,7 @@ rend_client_remove_intro_point(extend_info_t *failed_intro, } log_info(LD_REND,"%d options left for %s.", smartlist_len(ent->parsed->intro_nodes), - escaped_safe_str(rend_query->onion_address)); + escaped_safe_str_client(rend_query->onion_address)); return 1; } @@ -698,7 +674,7 @@ rend_client_rendezvous_acked(origin_circuit_t *circ, const uint8_t *request, log_info(LD_REND,"Got rendezvous ack. This circuit is now ready for " "rendezvous."); circ->_base.purpose = CIRCUIT_PURPOSE_C_REND_READY; - /* XXXX022 This is a pretty brute-force approach. It'd be better to + /* XXXX023 This is a pretty brute-force approach. It'd be better to * attach only the connections that are waiting on this circuit, rather * than trying to attach them all. See comments bug 743. */ /* If we already have the introduction circuit built, make sure we send @@ -737,7 +713,8 @@ rend_client_receive_rendezvous(origin_circuit_t *circ, const uint8_t *request, tor_assert(circ->build_state->pending_final_cpath); hop = circ->build_state->pending_final_cpath; tor_assert(hop->dh_handshake_state); - if (crypto_dh_compute_secret(hop->dh_handshake_state, (char*)request, + if (crypto_dh_compute_secret(LOG_PROTOCOL_WARN, + hop->dh_handshake_state, (char*)request, DH_KEY_LEN, keys, DIGEST_LEN+CPATH_KEY_MATERIAL_LEN)<0) { log_warn(LD_GENERAL, "Couldn't complete DH handshake."); @@ -767,7 +744,7 @@ rend_client_receive_rendezvous(origin_circuit_t *circ, const uint8_t *request, onion_append_to_cpath(&circ->cpath, hop); circ->build_state->pending_final_cpath = NULL; /* prevent double-free */ - /* XXXX022 This is a pretty brute-force approach. It'd be better to + /* XXXX023 This is a pretty brute-force approach. It'd be better to * attach only the connections that are waiting on this circuit, rather * than trying to attach them all. See comments bug 743. */ connection_ap_attach_pending(); @@ -779,24 +756,18 @@ rend_client_receive_rendezvous(origin_circuit_t *circ, const uint8_t *request, return -1; } -/** Find all the apconns in state AP_CONN_STATE_RENDDESC_WAIT that - * are waiting on query. If there's a working cache entry here - * with at least one intro point, move them to the next state. If - * <b>rend_version</b> is non-negative, fail connections that have - * requested <b>query</b> unless there are still descriptor fetch - * requests in progress for other descriptor versions than - * <b>rend_version</b>. - */ +/** Find all the apconns in state AP_CONN_STATE_RENDDESC_WAIT that are + * waiting on <b>query</b>. If there's a working cache entry here with at + * least one intro point, move them to the next state. */ void -rend_client_desc_trynow(const char *query, int rend_version) +rend_client_desc_trynow(const char *query) { edge_connection_t *conn; rend_cache_entry_t *entry; time_t now = time(NULL); smartlist_t *conns = get_connection_array(); - SMARTLIST_FOREACH(conns, connection_t *, _conn, - { + SMARTLIST_FOREACH_BEGIN(conns, connection_t *, _conn) { if (_conn->type != CONN_TYPE_AP || _conn->state != AP_CONN_STATE_RENDDESC_WAIT || _conn->marked_for_close) @@ -809,7 +780,7 @@ rend_client_desc_trynow(const char *query, int rend_version) assert_connection_ok(TO_CONN(conn), now); if (rend_cache_lookup_entry(conn->rend_data->onion_address, -1, &entry) == 1 && - smartlist_len(entry->parsed->intro_nodes) > 0) { + rend_client_any_intro_points_usable(entry)) { /* either this fetch worked, or it failed but there was a * valid entry from before which we should reuse */ log_info(LD_REND,"Rend desc is usable. Launching circuits."); @@ -828,17 +799,12 @@ rend_client_desc_trynow(const char *query, int rend_version) connection_mark_unattached_ap(conn, END_STREAM_REASON_CANT_ATTACH); } } else { /* 404, or fetch didn't get that far */ - /* Unless there are requests for another descriptor version pending, - * close the connection. */ - if (rend_version >= 0 && - !connection_get_by_type_state_rendquery(CONN_TYPE_DIR, 0, query, - rend_version == 0 ? 2 : 0)) { - log_notice(LD_REND,"Closing stream for '%s.onion': hidden service is " - "unavailable (try again later).", safe_str(query)); - connection_mark_unattached_ap(conn, END_STREAM_REASON_RESOLVEFAILED); - } + log_notice(LD_REND,"Closing stream for '%s.onion': hidden service is " + "unavailable (try again later).", + safe_str_client(query)); + connection_mark_unattached_ap(conn, END_STREAM_REASON_RESOLVEFAILED); } - }); + } SMARTLIST_FOREACH_END(_conn); } /** Return a newly allocated extend_info_t* for a randomly chosen introduction @@ -848,24 +814,63 @@ rend_client_desc_trynow(const char *query, int rend_version) extend_info_t * rend_client_get_random_intro(const rend_data_t *rend_query) { - int i; + extend_info_t *result; rend_cache_entry_t *entry; - rend_intro_point_t *intro; - routerinfo_t *router; if (rend_cache_lookup_entry(rend_query->onion_address, -1, &entry) < 1) { - log_warn(LD_REND, - "Query '%s' didn't have valid rend desc in cache. Failing.", - safe_str(rend_query->onion_address)); + log_warn(LD_REND, + "Query '%s' didn't have valid rend desc in cache. Failing.", + safe_str_client(rend_query->onion_address)); return NULL; } + /* See if we can get a node that complies with ExcludeNodes */ + if ((result = rend_client_get_random_intro_impl(entry, 1, 1))) + return result; + /* If not, and StrictNodes is not set, see if we can return any old node + */ + if (!get_options()->StrictNodes) + return rend_client_get_random_intro_impl(entry, 0, 1); + return NULL; +} + +/** As rend_client_get_random_intro, except assume that StrictNodes is set + * iff <b>strict</b> is true. If <b>warnings</b> is false, don't complain + * to the user when we're out of nodes, even if StrictNodes is true. + */ +static extend_info_t * +rend_client_get_random_intro_impl(const rend_cache_entry_t *entry, + const int strict, + const int warnings) +{ + int i; + + rend_intro_point_t *intro; + routerinfo_t *router; + or_options_t *options = get_options(); + smartlist_t *usable_nodes; + int n_excluded = 0; + + /* We'll keep a separate list of the usable nodes. If this becomes empty, + * no nodes are usable. */ + usable_nodes = smartlist_create(); + smartlist_add_all(usable_nodes, entry->parsed->intro_nodes); + again: - if (smartlist_len(entry->parsed->intro_nodes) == 0) + if (smartlist_len(usable_nodes) == 0) { + if (n_excluded && get_options()->StrictNodes && warnings) { + /* We only want to warn if StrictNodes is really set. Otherwise + * we're just about to retry anyways. + */ + log_warn(LD_REND, "All introduction points for hidden service are " + "at excluded relays, and StrictNodes is set. Skipping."); + } + smartlist_free(usable_nodes); return NULL; + } - i = crypto_rand_int(smartlist_len(entry->parsed->intro_nodes)); - intro = smartlist_get(entry->parsed->intro_nodes, i); + i = crypto_rand_int(smartlist_len(usable_nodes)); + intro = smartlist_get(usable_nodes, i); /* Do we need to look up the router or is the extend info complete? */ if (!intro->extend_info->onion_key) { if (tor_digest_is_zero(intro->extend_info->identity_digest)) @@ -875,16 +880,34 @@ rend_client_get_random_intro(const rend_data_t *rend_query) if (!router) { log_info(LD_REND, "Unknown router with nickname '%s'; trying another.", intro->extend_info->nickname); - rend_intro_point_free(intro); - smartlist_del(entry->parsed->intro_nodes, i); + smartlist_del(usable_nodes, i); goto again; } extend_info_free(intro->extend_info); intro->extend_info = extend_info_from_router(router); } + /* Check if we should refuse to talk to this router. */ + if (options->ExcludeNodes && strict && + routerset_contains_extendinfo(options->ExcludeNodes, + intro->extend_info)) { + n_excluded++; + smartlist_del(usable_nodes, i); + goto again; + } + + smartlist_free(usable_nodes); return extend_info_dup(intro->extend_info); } +/** Return true iff any introduction points still listed in <b>entry</b> are + * usable. */ +int +rend_client_any_intro_points_usable(const rend_cache_entry_t *entry) +{ + return rend_client_get_random_intro_impl( + entry, get_options()->StrictNodes, 0) != NULL; +} + /** Client-side authorizations for hidden services; map of onion address to * rend_service_authorization_t*. */ static strmap_t *auth_hid_servs = NULL; @@ -1009,8 +1032,7 @@ rend_parse_service_authorization(or_options_t *options, int validate_only) err: res = -1; done: - if (auth) - rend_service_authorization_free(auth); + rend_service_authorization_free(auth); SMARTLIST_FOREACH(sl, char *, c, tor_free(c);); smartlist_free(sl); if (!validate_only && res == 0) { diff --git a/src/or/rendclient.h b/src/or/rendclient.h new file mode 100644 index 0000000000..6910c1a97b --- /dev/null +++ b/src/or/rendclient.h @@ -0,0 +1,45 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file rendclient.h + * \brief Header file for rendclient.c. + **/ + +#ifndef _TOR_RENDCLIENT_H +#define _TOR_RENDCLIENT_H + +void rend_client_introcirc_has_opened(origin_circuit_t *circ); +void rend_client_rendcirc_has_opened(origin_circuit_t *circ); +int rend_client_introduction_acked(origin_circuit_t *circ, + const uint8_t *request, + size_t request_len); +void rend_client_refetch_v2_renddesc(const rend_data_t *rend_query); +void rend_client_cancel_descriptor_fetches(void); +int rend_client_remove_intro_point(extend_info_t *failed_intro, + const rend_data_t *rend_query); +int rend_client_rendezvous_acked(origin_circuit_t *circ, + const uint8_t *request, + size_t request_len); +int rend_client_receive_rendezvous(origin_circuit_t *circ, + const uint8_t *request, + size_t request_len); +void rend_client_desc_trynow(const char *query); + +extend_info_t *rend_client_get_random_intro(const rend_data_t *rend_query); +int rend_client_any_intro_points_usable(const rend_cache_entry_t *entry); + +int rend_client_send_introduction(origin_circuit_t *introcirc, + origin_circuit_t *rendcirc); +int rend_parse_service_authorization(or_options_t *options, + int validate_only); +rend_service_authorization_t *rend_client_lookup_service_authorization( + const char *onion_address); +void rend_service_authorization_free_all(void); +rend_data_t *rend_data_dup(const rend_data_t *request); + +#endif + diff --git a/src/or/rendcommon.c b/src/or/rendcommon.c index c83573b208..da33feccbc 100644 --- a/src/or/rendcommon.c +++ b/src/or/rendcommon.c @@ -9,6 +9,15 @@ **/ #include "or.h" +#include "circuitbuild.h" +#include "config.h" +#include "rendclient.h" +#include "rendcommon.h" +#include "rendmid.h" +#include "rendservice.h" +#include "rephist.h" +#include "routerlist.h" +#include "routerparse.h" /** Return 0 if one and two are the same service ids, else -1 or 1 */ int @@ -22,6 +31,8 @@ rend_cmp_service_ids(const char *one, const char *two) void rend_service_descriptor_free(rend_service_descriptor_t *desc) { + if (!desc) + return; if (desc->pk) crypto_free_pk_env(desc->pk); if (desc->intro_nodes) { @@ -125,7 +136,8 @@ rend_compute_v2_desc_id(char *desc_id_out, const char *service_id, if (!service_id || strlen(service_id) != REND_SERVICE_ID_LEN_BASE32) { log_warn(LD_REND, "Could not compute v2 descriptor ID: " - "Illegal service ID: %s", safe_str(service_id)); + "Illegal service ID: %s", + safe_str(service_id)); return -1; } if (replica >= REND_NUMBER_OF_NON_CONSECUTIVE_REPLICAS) { @@ -138,7 +150,7 @@ rend_compute_v2_desc_id(char *desc_id_out, const char *service_id, service_id, REND_SERVICE_ID_LEN_BASE32) < 0) { log_warn(LD_REND, "Could not compute v2 descriptor ID: " "Illegal characters in service ID: %s", - safe_str(service_id)); + safe_str_client(service_id)); return -1; } /* Calculate current time-period. */ @@ -403,8 +415,7 @@ rend_desc_v2_is_parsable(rend_encoded_v2_service_descriptor_t *desc) &test_intro_size, &test_encoded_size, &test_next, desc->desc_str); - if (test_parsed) - rend_service_descriptor_free(test_parsed); + rend_service_descriptor_free(test_parsed); tor_free(test_intro_content); return (res >= 0); } @@ -414,6 +425,8 @@ void rend_encoded_v2_service_descriptor_free( rend_encoded_v2_service_descriptor_t *desc) { + if (!desc) + return; tor_free(desc->desc_str); tor_free(desc); } @@ -422,10 +435,11 @@ rend_encoded_v2_service_descriptor_free( void rend_intro_point_free(rend_intro_point_t *intro) { - if (intro->extend_info) - extend_info_free(intro->extend_info); - if (intro->intro_key) - crypto_free_pk_env(intro->intro_key); + if (!intro) + return; + + extend_info_free(intro->extend_info); + crypto_free_pk_env(intro->intro_key); tor_free(intro); } @@ -618,7 +632,8 @@ rend_encode_v2_descriptors(smartlist_t *descs_out, } if (router_append_dirobj_signature(desc_str + written, desc_len - written, - desc_digest, service_key) < 0) { + desc_digest, DIGEST_LEN, + service_key) < 0) { log_warn(LD_BUG, "Couldn't sign desc."); rend_encoded_v2_service_descriptor_free(enc); goto err; @@ -655,63 +670,6 @@ rend_encode_v2_descriptors(smartlist_t *descs_out, return seconds_valid; } -/** Encode a service descriptor for <b>desc</b>, and sign it with - * <b>key</b>. Store the descriptor in *<b>str_out</b>, and set - * *<b>len_out</b> to its length. - */ -int -rend_encode_service_descriptor(rend_service_descriptor_t *desc, - crypto_pk_env_t *key, - char **str_out, size_t *len_out) -{ - char *cp; - char *end; - int i, r; - size_t asn1len; - size_t buflen = - PK_BYTES*2*(smartlist_len(desc->intro_nodes)+2);/*Too long, but ok*/ - cp = *str_out = tor_malloc(buflen); - end = cp + PK_BYTES*2*(smartlist_len(desc->intro_nodes)+1); - r = crypto_pk_asn1_encode(desc->pk, cp+2, end-(cp+2)); - if (r < 0) { - tor_free(*str_out); - return -1; - } - asn1len = r; - set_uint16(cp, htons((uint16_t)asn1len)); - cp += 2+asn1len; - set_uint32(cp, htonl((uint32_t)desc->timestamp)); - cp += 4; - set_uint16(cp, htons((uint16_t)smartlist_len(desc->intro_nodes))); - cp += 2; - for (i=0; i < smartlist_len(desc->intro_nodes); ++i) { - rend_intro_point_t *intro = smartlist_get(desc->intro_nodes, i); - char ipoint[HEX_DIGEST_LEN+2]; - const size_t ipoint_len = HEX_DIGEST_LEN+1; - ipoint[0] = '$'; - base16_encode(ipoint+1, HEX_DIGEST_LEN+1, - intro->extend_info->identity_digest, - DIGEST_LEN); - tor_assert(strlen(ipoint) == ipoint_len); - /* Assert that appending ipoint and its NUL won't over overrun the - * buffer. */ - tor_assert(cp + ipoint_len+1 < *str_out + buflen); - memcpy(cp, ipoint, ipoint_len+1); - cp += ipoint_len+1; - } - note_crypto_pk_op(REND_SERVER); - r = crypto_pk_private_sign_digest(key, - cp, buflen - (cp - *str_out), - *str_out, cp-*str_out); - if (r<0) { - tor_free(*str_out); - return -1; - } - cp += r; - *len_out = (size_t)(cp-*str_out); - return 0; -} - /** Parse a service descriptor at <b>str</b> (<b>len</b> bytes). On * success, return a newly alloced service_descriptor_t. On failure, * return NULL. @@ -828,22 +786,27 @@ rend_cache_init(void) /** Helper: free storage held by a single service descriptor cache entry. */ static void -_rend_cache_entry_free(void *p) +rend_cache_entry_free(rend_cache_entry_t *e) { - rend_cache_entry_t *e = p; + if (!e) + return; rend_service_descriptor_free(e->parsed); tor_free(e->desc); tor_free(e); } +static void +_rend_cache_entry_free(void *p) +{ + rend_cache_entry_free(p); +} + /** Free all storage held by the service descriptor cache. */ void rend_cache_free_all(void) { - if (rend_cache) - strmap_free(rend_cache, _rend_cache_entry_free); - if (rend_cache_v2_dir) - digestmap_free(rend_cache_v2_dir, _rend_cache_entry_free); + strmap_free(rend_cache, _rend_cache_entry_free); + digestmap_free(rend_cache_v2_dir, _rend_cache_entry_free); rend_cache = NULL; rend_cache_v2_dir = NULL; } @@ -864,7 +827,7 @@ rend_cache_clean(void) ent = (rend_cache_entry_t*)val; if (ent->parsed->timestamp < cutoff) { iter = strmap_iter_next_rmv(rend_cache, iter); - _rend_cache_entry_free(ent); + rend_cache_entry_free(ent); } else { iter = strmap_iter_next(rend_cache, iter); } @@ -900,9 +863,9 @@ rend_cache_clean_v2_descs_as_dir(void) char key_base32[REND_DESC_ID_V2_LEN_BASE32 + 1]; base32_encode(key_base32, sizeof(key_base32), key, DIGEST_LEN); log_info(LD_REND, "Removing descriptor with ID '%s' from cache", - safe_str(key_base32)); + safe_str_client(key_base32)); iter = digestmap_iter_next_rmv(rend_cache_v2_dir, iter); - _rend_cache_entry_free(ent); + rend_cache_entry_free(ent); } else { iter = digestmap_iter_next(rend_cache_v2_dir, iter); } @@ -978,6 +941,11 @@ rend_cache_lookup_entry(const char *query, int version, rend_cache_entry_t **e) } if (!*e) return 0; + tor_assert((*e)->parsed && (*e)->parsed->intro_nodes); + /* XXX023 hack for now, to return "not found" if there are no intro + * points remaining. See bug 997. */ + if (! rend_client_any_intro_points_usable(*e)) + return 0; return 1; } @@ -1056,7 +1024,6 @@ rend_cache_store(const char *desc, size_t desc_len, int published) char query[REND_SERVICE_ID_LEN_BASE32+1]; char key[REND_SERVICE_ID_LEN_BASE32+2]; /* 0<query>\0 */ time_t now; - or_options_t *options = get_options(); tor_assert(rend_cache); parsed = rend_parse_service_descriptor(desc,desc_len); if (!parsed) { @@ -1071,13 +1038,15 @@ rend_cache_store(const char *desc, size_t desc_len, int published) now = time(NULL); if (parsed->timestamp < now-REND_CACHE_MAX_AGE-REND_CACHE_MAX_SKEW) { log_fn(LOG_PROTOCOL_WARN, LD_REND, - "Service descriptor %s is too old.", safe_str(query)); + "Service descriptor %s is too old.", + safe_str_client(query)); rend_service_descriptor_free(parsed); return -2; } if (parsed->timestamp > now+REND_CACHE_MAX_SKEW) { log_fn(LOG_PROTOCOL_WARN, LD_REND, - "Service descriptor %s is too far in the future.", safe_str(query)); + "Service descriptor %s is too far in the future.", + safe_str_client(query)); rend_service_descriptor_free(parsed); return -2; } @@ -1085,25 +1054,22 @@ rend_cache_store(const char *desc, size_t desc_len, int published) tor_snprintf(key, sizeof(key), "2%s", query); if (!published && strmap_get_lc(rend_cache, key)) { log_info(LD_REND, "We already have a v2 descriptor for service %s.", - safe_str(query)); + safe_str_client(query)); rend_service_descriptor_free(parsed); return -1; } - /* report novel publication to statistics */ - if (published && options->HSAuthorityRecordStats) { - hs_usage_note_publish_total(query, now); - } tor_snprintf(key, sizeof(key), "0%s", query); e = (rend_cache_entry_t*) strmap_get_lc(rend_cache, key); if (e && e->parsed->timestamp > parsed->timestamp) { log_info(LD_REND,"We already have a newer service descriptor %s with the " - "same ID and version.", safe_str(query)); + "same ID and version.", + safe_str_client(query)); rend_service_descriptor_free(parsed); return 0; } if (e && e->len == desc_len && tor_memeq(desc,e->desc,desc_len)) { log_info(LD_REND,"We already have this service descriptor %s.", - safe_str(query)); + safe_str_client(query)); e->received = time(NULL); rend_service_descriptor_free(parsed); return 0; @@ -1111,10 +1077,6 @@ rend_cache_store(const char *desc, size_t desc_len, int published) if (!e) { e = tor_malloc_zero(sizeof(rend_cache_entry_t)); strmap_set_lc(rend_cache, key, e); - /* report novel publication to statistics */ - if (published && options->HSAuthorityRecordStats) { - hs_usage_note_publish_novel(query, now); - } } else { rend_service_descriptor_free(e->parsed); tor_free(e->desc); @@ -1126,7 +1088,7 @@ rend_cache_store(const char *desc, size_t desc_len, int published) memcpy(e->desc, desc, desc_len); log_debug(LD_REND,"Successfully stored rend desc '%s', len %d.", - safe_str(query), (int)desc_len); + safe_str_client(query), (int)desc_len); return 1; } @@ -1177,7 +1139,7 @@ rend_cache_store_v2_desc_as_dir(const char *desc) if (!hid_serv_responsible_for_desc_id(desc_id)) { log_info(LD_REND, "Service descriptor with desc ID %s is not in " "interval that we are responsible for.", - safe_str(desc_id_base32)); + safe_str_client(desc_id_base32)); goto skip; } /* Is descriptor too old? */ @@ -1306,7 +1268,8 @@ rend_cache_store_v2_desc_as_client(const char *desc, /* Decode/decrypt introduction points. */ if (intro_content) { if (rend_query->auth_type != REND_NO_AUTH && - rend_query->descriptor_cookie) { + !tor_mem_is_zero(rend_query->descriptor_cookie, + sizeof(rend_query->descriptor_cookie))) { char *ipos_decrypted = NULL; size_t ipos_decrypted_size; if (rend_decrypt_introduction_points(&ipos_decrypted, @@ -1341,14 +1304,14 @@ rend_cache_store_v2_desc_as_client(const char *desc, /* Is descriptor too old? */ if (parsed->timestamp < now - REND_CACHE_MAX_AGE-REND_CACHE_MAX_SKEW) { log_warn(LD_REND, "Service descriptor with service ID %s is too old.", - safe_str(service_id)); + safe_str_client(service_id)); retval = -2; goto err; } /* Is descriptor too far in the future? */ if (parsed->timestamp > now + REND_CACHE_MAX_SKEW) { log_warn(LD_REND, "Service descriptor with service ID %s is too far in " - "the future.", safe_str(service_id)); + "the future.", safe_str_client(service_id)); retval = -2; goto err; } @@ -1356,7 +1319,7 @@ rend_cache_store_v2_desc_as_client(const char *desc, tor_snprintf(key, sizeof(key), "0%s", service_id); if (strmap_get_lc(rend_cache, key)) { log_info(LD_REND, "We already have a v0 descriptor for service ID %s.", - safe_str(service_id)); + safe_str_client(service_id)); retval = -1; goto err; } @@ -1366,14 +1329,14 @@ rend_cache_store_v2_desc_as_client(const char *desc, if (e && e->parsed->timestamp > parsed->timestamp) { log_info(LD_REND, "We already have a newer service descriptor for " "service ID %s with the same desc ID and version.", - safe_str(service_id)); + safe_str_client(service_id)); retval = 0; goto err; } /* Do we already have this descriptor? */ if (e && !strcmp(desc, e->desc)) { log_info(LD_REND,"We already have this service descriptor %s.", - safe_str(service_id)); + safe_str_client(service_id)); e->received = time(NULL); retval = 0; goto err; @@ -1391,12 +1354,11 @@ rend_cache_store_v2_desc_as_client(const char *desc, strlcpy(e->desc, desc, encoded_size + 1); e->len = encoded_size; log_debug(LD_REND,"Successfully stored rend desc '%s', len %d.", - safe_str(service_id), (int)encoded_size); + safe_str_client(service_id), (int)encoded_size); return 1; err: - if (parsed) - rend_service_descriptor_free(parsed); + rend_service_descriptor_free(parsed); tor_free(intro_content); return retval; } diff --git a/src/or/rendcommon.h b/src/or/rendcommon.h new file mode 100644 index 0000000000..44b5227cf5 --- /dev/null +++ b/src/or/rendcommon.h @@ -0,0 +1,66 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file rendcommon.h + * \brief Header file for rendcommon.c. + **/ + +#ifndef _TOR_RENDCOMMON_H +#define _TOR_RENDCOMMON_H + +/** Free all storage associated with <b>data</b> */ +static INLINE void +rend_data_free(rend_data_t *data) +{ + tor_free(data); +} + +int rend_cmp_service_ids(const char *one, const char *two); + +void rend_process_relay_cell(circuit_t *circ, const crypt_path_t *layer_hint, + int command, size_t length, + const uint8_t *payload); + +void rend_service_descriptor_free(rend_service_descriptor_t *desc); +rend_service_descriptor_t *rend_parse_service_descriptor(const char *str, + size_t len); +int rend_get_service_id(crypto_pk_env_t *pk, char *out); +void rend_encoded_v2_service_descriptor_free( + rend_encoded_v2_service_descriptor_t *desc); +void rend_intro_point_free(rend_intro_point_t *intro); + +void rend_cache_init(void); +void rend_cache_clean(void); +void rend_cache_clean_v2_descs_as_dir(void); +void rend_cache_purge(void); +void rend_cache_free_all(void); +int rend_valid_service_id(const char *query); +int rend_cache_lookup_desc(const char *query, int version, const char **desc, + size_t *desc_len); +int rend_cache_lookup_entry(const char *query, int version, + rend_cache_entry_t **entry_out); +int rend_cache_lookup_v2_desc_as_dir(const char *query, const char **desc); +int rend_cache_store(const char *desc, size_t desc_len, int published); +int rend_cache_store_v2_desc_as_client(const char *desc, + const rend_data_t *rend_query); +int rend_cache_store_v2_desc_as_dir(const char *desc); +int rend_cache_size(void); +int rend_encode_v2_descriptors(smartlist_t *descs_out, + rend_service_descriptor_t *desc, time_t now, + uint8_t period, rend_auth_type_t auth_type, + crypto_pk_env_t *client_key, + smartlist_t *client_cookies); +int rend_compute_v2_desc_id(char *desc_id_out, const char *service_id, + const char *descriptor_cookie, + time_t now, uint8_t replica); +int rend_id_is_in_interval(const char *a, const char *b, const char *c); +void rend_get_descriptor_id_bytes(char *descriptor_id_out, + const char *service_id, + const char *secret_id_part); + +#endif + diff --git a/src/or/rendmid.c b/src/or/rendmid.c index d73f0a1ba9..04edd8e3e2 100644 --- a/src/or/rendmid.c +++ b/src/or/rendmid.c @@ -8,6 +8,11 @@ **/ #include "or.h" +#include "circuitlist.h" +#include "config.h" +#include "relay.h" +#include "rendmid.h" +#include "rephist.h" /** Respond to an ESTABLISH_INTRO cell by checking the signed data and * setting the circuit's purpose and service pk digest. diff --git a/src/or/rendmid.h b/src/or/rendmid.h new file mode 100644 index 0000000000..5ed87fd2b1 --- /dev/null +++ b/src/or/rendmid.h @@ -0,0 +1,25 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file rendmid.h + * \brief Header file for rendmid.c. + **/ + +#ifndef _TOR_RENDMID_H +#define _TOR_RENDMID_H + +int rend_mid_establish_intro(or_circuit_t *circ, const uint8_t *request, + size_t request_len); +int rend_mid_introduce(or_circuit_t *circ, const uint8_t *request, + size_t request_len); +int rend_mid_establish_rendezvous(or_circuit_t *circ, const uint8_t *request, + size_t request_len); +int rend_mid_rendezvous(or_circuit_t *circ, const uint8_t *request, + size_t request_len); + +#endif + diff --git a/src/or/rendservice.c b/src/or/rendservice.c index d1cc7f4f11..35e8b9057a 100644 --- a/src/or/rendservice.c +++ b/src/or/rendservice.c @@ -8,10 +8,23 @@ **/ #include "or.h" +#include "circuitbuild.h" +#include "circuitlist.h" +#include "circuituse.h" +#include "config.h" +#include "directory.h" +#include "networkstatus.h" +#include "rendclient.h" +#include "rendcommon.h" +#include "rendservice.h" +#include "router.h" +#include "relay.h" +#include "rephist.h" +#include "routerlist.h" +#include "routerparse.h" static origin_circuit_t *find_intro_circuit(rend_intro_point_t *intro, - const char *pk_digest, - int desc_version); + const char *pk_digest); /** Represents the mapping from a virtual port of a rendezvous service to * a real port on some IP. @@ -42,8 +55,6 @@ typedef struct rend_service_t { /* Fields specified in config file */ char *directory; /**< where in the filesystem it stores it */ smartlist_t *ports; /**< List of rend_service_port_config_t */ - int descriptor_version; /**< Rendezvous descriptor version that will be - * published. */ rend_auth_type_t auth_type; /**< Client authorization type or 0 if no client * authorization is performed. */ smartlist_t *clients; /**< List of rend_authorized_client_t's of @@ -58,7 +69,7 @@ typedef struct rend_service_t { * or are trying to establish. */ time_t intro_period_started; /**< Start of the current period to build * introduction points. */ - int n_intro_circuits_launched; /**< count of intro circuits we have + int n_intro_circuits_launched; /**< Count of intro circuits we have * established in this period. */ rend_service_descriptor_t *desc; /**< Current hidden service descriptor. */ time_t desc_is_dirty; /**< Time at which changes to the hidden service @@ -90,7 +101,8 @@ num_rend_services(void) static void rend_authorized_client_free(rend_authorized_client_t *client) { - if (!client) return; + if (!client) + return; if (client->client_key) crypto_free_pk_env(client->client_key); tor_free(client->client_name); @@ -109,7 +121,9 @@ rend_authorized_client_strmap_item_free(void *authorized_client) static void rend_service_free(rend_service_t *service) { - if (!service) return; + if (!service) + return; + tor_free(service->directory); SMARTLIST_FOREACH(service->ports, void*, p, tor_free(p)); smartlist_free(service->ports); @@ -120,15 +134,14 @@ rend_service_free(rend_service_t *service) rend_intro_point_free(intro);); smartlist_free(service->intro_nodes); } - if (service->desc) - rend_service_descriptor_free(service->desc); + + rend_service_descriptor_free(service->desc); if (service->clients) { SMARTLIST_FOREACH(service->clients, rend_authorized_client_t *, c, rend_authorized_client_free(c);); smartlist_free(service->clients); } - if (service->accepted_intros) - digestmap_free(service->accepted_intros, _tor_free); + digestmap_free(service->accepted_intros, _tor_free); tor_free(service); } @@ -137,9 +150,9 @@ rend_service_free(rend_service_t *service) void rend_service_free_all(void) { - if (!rend_service_list) { + if (!rend_service_list) return; - } + SMARTLIST_FOREACH(rend_service_list, rend_service_t*, ptr, rend_service_free(ptr)); smartlist_free(rend_service_list); @@ -156,36 +169,6 @@ rend_add_service(rend_service_t *service) service->intro_nodes = smartlist_create(); - /* If the service is configured to publish unversioned (v0) and versioned - * descriptors (v2 or higher), split it up into two separate services - * (unless it is configured to perform client authorization). */ - if (service->descriptor_version == -1) { - if (service->auth_type == REND_NO_AUTH) { - rend_service_t *v0_service = tor_malloc_zero(sizeof(rend_service_t)); - v0_service->directory = tor_strdup(service->directory); - v0_service->ports = smartlist_create(); - SMARTLIST_FOREACH(service->ports, rend_service_port_config_t *, p, { - rend_service_port_config_t *copy = - tor_malloc_zero(sizeof(rend_service_port_config_t)); - memcpy(copy, p, sizeof(rend_service_port_config_t)); - smartlist_add(v0_service->ports, copy); - }); - v0_service->intro_period_started = service->intro_period_started; - v0_service->descriptor_version = 0; /* Unversioned descriptor. */ - v0_service->auth_type = REND_NO_AUTH; - rend_add_service(v0_service); - } - - service->descriptor_version = 2; /* Versioned descriptor. */ - } - - if (service->auth_type != REND_NO_AUTH && !service->descriptor_version) { - log_warn(LD_CONFIG, "Hidden service with client authorization and " - "version 0 descriptors configured; ignoring."); - rend_service_free(service); - return; - } - if (service->auth_type != REND_NO_AUTH && smartlist_len(service->clients) == 0) { log_warn(LD_CONFIG, "Hidden service with client authorization but no " @@ -297,7 +280,7 @@ rend_config_services(or_options_t *options, int validate_only) for (line = options->RendConfigLines; line; line = line->next) { if (!strcasecmp(line->key, "HiddenServiceDir")) { - if (service) { + if (service) { /* register the one we just finished parsing */ if (validate_only) rend_service_free(service); else @@ -307,7 +290,6 @@ rend_config_services(or_options_t *options, int validate_only) service->directory = tor_strdup(line->value); service->ports = smartlist_create(); service->intro_period_started = time(NULL); - service->descriptor_version = -1; /**< All descriptor versions. */ continue; } if (!service) { @@ -433,35 +415,13 @@ rend_config_services(or_options_t *options, int validate_only) return -1; } } else { - smartlist_t *versions; - char *version_str; - int i, version, ver_ok=1, versions_bitmask = 0; tor_assert(!strcasecmp(line->key, "HiddenServiceVersion")); - versions = smartlist_create(); - smartlist_split_string(versions, line->value, ",", - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); - for (i = 0; i < smartlist_len(versions); i++) { - version_str = smartlist_get(versions, i); - if (strlen(version_str) != 1 || strspn(version_str, "02") != 1) { - log_warn(LD_CONFIG, - "HiddenServiceVersion can only be 0 and/or 2."); - SMARTLIST_FOREACH(versions, char *, cp, tor_free(cp)); - smartlist_free(versions); - rend_service_free(service); - return -1; - } - version = (int)tor_parse_long(version_str, 10, 0, INT_MAX, &ver_ok, - NULL); - if (!ver_ok) - continue; - versions_bitmask |= 1 << version; + if (strcmp(line->value, "2")) { + log_warn(LD_CONFIG, + "The only supported HiddenServiceVersion is 2."); + rend_service_free(service); + return -1; } - /* If exactly one version is set, change descriptor_version to that - * value; otherwise leave it at -1. */ - if (versions_bitmask == 1 << 0) service->descriptor_version = 0; - if (versions_bitmask == 1 << 2) service->descriptor_version = 2; - SMARTLIST_FOREACH(versions, char *, cp, tor_free(cp)); - smartlist_free(versions); } } if (service) { @@ -483,8 +443,7 @@ rend_config_services(or_options_t *options, int validate_only) * probably ok? */ SMARTLIST_FOREACH(rend_service_list, rend_service_t *, new, { SMARTLIST_FOREACH(old_service_list, rend_service_t *, old, { - if (!strcmp(old->directory, new->directory) && - old->descriptor_version == new->descriptor_version) { + if (!strcmp(old->directory, new->directory)) { smartlist_add_all(new->intro_nodes, old->intro_nodes); smartlist_clear(old->intro_nodes); smartlist_add(surviving_services, old); @@ -507,18 +466,16 @@ rend_config_services(or_options_t *options, int validate_only) tor_assert(oc->rend_data); SMARTLIST_FOREACH(surviving_services, rend_service_t *, ptr, { if (tor_memeq(ptr->pk_digest, oc->rend_data->rend_pk_digest, - DIGEST_LEN) && - ptr->descriptor_version == oc->rend_data->rend_desc_version) { + DIGEST_LEN)) { keep_it = 1; break; } }); if (keep_it) continue; - log_info(LD_REND, "Closing intro point %s for service %s version %d.", - safe_str(oc->build_state->chosen_exit->nickname), - oc->rend_data->onion_address, - oc->rend_data->rend_desc_version); + log_info(LD_REND, "Closing intro point %s for service %s.", + safe_str_client(oc->build_state->chosen_exit->nickname), + oc->rend_data->onion_address); circuit_mark_for_close(circ, END_CIRC_REASON_FINISHED); /* XXXX Is there another reason we should use here? */ } @@ -541,14 +498,13 @@ rend_service_update_descriptor(rend_service_t *service) rend_service_descriptor_t *d; origin_circuit_t *circ; int i; - if (service->desc) { - rend_service_descriptor_free(service->desc); - service->desc = NULL; - } + + rend_service_descriptor_free(service->desc); + service->desc = NULL; + d = service->desc = tor_malloc_zero(sizeof(rend_service_descriptor_t)); d->pk = crypto_pk_dup_key(service->private_key); d->timestamp = time(NULL); - d->version = service->descriptor_version; d->intro_nodes = smartlist_create(); /* Support intro protocols 2 and 3. */ d->protocols = (1 << 2) + (1 << 3); @@ -556,7 +512,7 @@ rend_service_update_descriptor(rend_service_t *service) for (i = 0; i < smartlist_len(service->intro_nodes); ++i) { rend_intro_point_t *intro_svc = smartlist_get(service->intro_nodes, i); rend_intro_point_t *intro_desc; - circ = find_intro_circuit(intro_svc, service->pk_digest, d->version); + circ = find_intro_circuit(intro_svc, service->pk_digest); if (!circ || circ->_base.purpose != CIRCUIT_PURPOSE_S_INTRO) continue; @@ -797,17 +753,15 @@ rend_service_load_keys(void) return r; } -/** Return the service whose public key has a digest of <b>digest</b> and - * which publishes the given descriptor <b>version</b>. Return NULL if no - * such service exists. +/** Return the service whose public key has a digest of <b>digest</b>, or + * NULL if no such service exists. */ static rend_service_t * -rend_service_get_by_pk_digest_and_version(const char* digest, - uint8_t version) +rend_service_get_by_pk_digest(const char* digest) { SMARTLIST_FOREACH(rend_service_list, rend_service_t*, s, - if (tor_memeq(s->pk_digest,digest,DIGEST_LEN) && - s->descriptor_version == version) return s); + if (tor_memeq(s->pk_digest,digest,DIGEST_LEN)) + return s); return NULL; } @@ -894,6 +848,7 @@ clean_accepted_intros(rend_service_t *service, time_t now) /** Respond to an INTRODUCE2 cell by launching a circuit to the chosen * rendezvous point. */ + /* XXX022 this function sure could use some organizing. -RD */ int rend_service_introduce(origin_circuit_t *circuit, const uint8_t *request, size_t request_len) @@ -921,6 +876,8 @@ rend_service_introduce(origin_circuit_t *circuit, const uint8_t *request, time_t now = time(NULL); char diffie_hellman_hash[DIGEST_LEN]; time_t *access_time; + or_options_t *options = get_options(); + tor_assert(circuit->rend_data); base32_encode(serviceid, REND_SERVICE_ID_LEN_BASE32+1, @@ -944,21 +901,16 @@ rend_service_introduce(origin_circuit_t *circuit, const uint8_t *request, } /* look up service depending on circuit. */ - service = rend_service_get_by_pk_digest_and_version( - circuit->rend_data->rend_pk_digest, - circuit->rend_data->rend_desc_version); + service = rend_service_get_by_pk_digest( + circuit->rend_data->rend_pk_digest); if (!service) { log_warn(LD_REND, "Got an INTRODUCE2 cell for an unrecognized service %s.", escaped(serviceid)); return -1; } - /* if descriptor version is 2, use intro key instead of service key. */ - if (circuit->rend_data->rend_desc_version == 0) { - intro_key = service->private_key; - } else { - intro_key = circuit->intro_key; - } + /* use intro key instead of service key. */ + intro_key = circuit->intro_key; /* first DIGEST_LEN bytes of request is intro or service pk digest */ crypto_pk_get_digest(intro_key, intro_key_digest); @@ -989,7 +941,7 @@ rend_service_introduce(origin_circuit_t *circuit, const uint8_t *request, len = r; if (*buf == 3) { /* Version 3 INTRODUCE2 cell. */ - time_t ts = 0, now = time(NULL); + time_t ts = 0; v3_shift = 1; auth_type = buf[1]; switch (auth_type) { @@ -1083,7 +1035,7 @@ rend_service_introduce(origin_circuit_t *circuit, const uint8_t *request, router = router_get_by_nickname(rp_nickname, 0); if (!router) { log_info(LD_REND, "Couldn't find router %s named in introduce2 cell.", - escaped_safe_str(rp_nickname)); + escaped_safe_str_client(rp_nickname)); /* XXXX Add a no-such-router reason? */ reason = END_CIRC_REASON_TORPROTOCOL; goto err; @@ -1098,6 +1050,15 @@ rend_service_introduce(origin_circuit_t *circuit, const uint8_t *request, goto err; } + /* Check if we'd refuse to talk to this router */ + if (options->ExcludeNodes && options->StrictNodes && + routerset_contains_extendinfo(options->ExcludeNodes, extend_info)) { + log_warn(LD_REND, "Client asked to rendezvous at a relay that we " + "exclude, and StrictNodes is set. Refusing service."); + reason = END_CIRC_REASON_INTERNAL; /* XXX might leak why we refused */ + goto err; + } + r_cookie = ptr; base16_encode(hexcookie,9,r_cookie,4); @@ -1158,7 +1119,8 @@ rend_service_introduce(origin_circuit_t *circuit, const uint8_t *request, reason = END_CIRC_REASON_INTERNAL; goto err; } - if (crypto_dh_compute_secret(dh, ptr+REND_COOKIE_LEN, DH_KEY_LEN, keys, + if (crypto_dh_compute_secret(LOG_PROTOCOL_WARN, dh, ptr+REND_COOKIE_LEN, + DH_KEY_LEN, keys, DIGEST_LEN+CPATH_KEY_MATERIAL_LEN)<0) { log_warn(LD_BUG, "Internal error: couldn't complete DH handshake"); reason = END_CIRC_REASON_INTERNAL; @@ -1168,7 +1130,7 @@ rend_service_introduce(origin_circuit_t *circuit, const uint8_t *request, circ_needs_uptime = rend_service_requires_uptime(service); /* help predict this next time */ - rep_hist_note_used_internal(time(NULL), circ_needs_uptime, 1); + rep_hist_note_used_internal(now, circ_needs_uptime, 1); /* Launch a circuit to alice's chosen rendezvous point. */ @@ -1184,14 +1146,16 @@ rend_service_introduce(origin_circuit_t *circuit, const uint8_t *request, if (!launched) { /* give up */ log_warn(LD_REND, "Giving up launching first hop of circuit to rendezvous " "point %s for service %s.", - escaped_safe_str(extend_info->nickname), serviceid); + escaped_safe_str_client(extend_info->nickname), + serviceid); reason = END_CIRC_REASON_CONNECTFAILED; goto err; } log_info(LD_REND, "Accepted intro; launching circuit to %s " "(cookie %s) for service %s.", - escaped_safe_str(extend_info->nickname), hexcookie, serviceid); + escaped_safe_str_client(extend_info->nickname), + hexcookie, serviceid); tor_assert(launched->build_state); /* Fill in the circuit's state. */ launched->rend_data = tor_malloc_zero(sizeof(rend_data_t)); @@ -1201,11 +1165,10 @@ rend_service_introduce(origin_circuit_t *circuit, const uint8_t *request, memcpy(launched->rend_data->rend_cookie, r_cookie, REND_COOKIE_LEN); strlcpy(launched->rend_data->onion_address, service->service_id, sizeof(launched->rend_data->onion_address)); - launched->rend_data->rend_desc_version = service->descriptor_version; launched->build_state->pending_final_cpath = cpath = tor_malloc_zero(sizeof(crypt_path_t)); cpath->magic = CRYPT_PATH_MAGIC; - launched->build_state->expiry_time = time(NULL) + MAX_REND_TIMEOUT; + launched->build_state->expiry_time = now + MAX_REND_TIMEOUT; cpath->dh_handshake_state = dh; dh = NULL; @@ -1289,7 +1252,7 @@ rend_service_launch_establish_intro(rend_service_t *service, log_info(LD_REND, "Launching circuit to introduction point %s for service %s", - escaped_safe_str(intro->extend_info->nickname), + escaped_safe_str_client(intro->extend_info->nickname), service->service_id); rep_hist_note_used_internal(time(NULL), 1, 0); @@ -1302,7 +1265,7 @@ rend_service_launch_establish_intro(rend_service_t *service, if (!launched) { log_info(LD_REND, "Can't launch circuit to establish introduction at %s.", - escaped_safe_str(intro->extend_info->nickname)); + escaped_safe_str_client(intro->extend_info->nickname)); return -1; } @@ -1325,18 +1288,16 @@ rend_service_launch_establish_intro(rend_service_t *service, strlcpy(launched->rend_data->onion_address, service->service_id, sizeof(launched->rend_data->onion_address)); memcpy(launched->rend_data->rend_pk_digest, service->pk_digest, DIGEST_LEN); - launched->rend_data->rend_desc_version = service->descriptor_version; - if (service->descriptor_version == 2) - launched->intro_key = crypto_pk_dup_key(intro->intro_key); + launched->intro_key = crypto_pk_dup_key(intro->intro_key); if (launched->_base.state == CIRCUIT_STATE_OPEN) rend_service_intro_has_opened(launched); return 0; } /** Return the number of introduction points that are or have been - * established for the given service address and rendezvous version. */ + * established for the given service address in <b>query</b>. */ static int -count_established_intro_points(const char *query, int rend_version) +count_established_intro_points(const char *query) { int num_ipos = 0; circuit_t *circ; @@ -1347,7 +1308,6 @@ count_established_intro_points(const char *query, int rend_version) circ->purpose == CIRCUIT_PURPOSE_S_INTRO)) { origin_circuit_t *oc = TO_ORIGIN_CIRCUIT(circ); if (oc->rend_data && - oc->rend_data->rend_desc_version == rend_version && !rend_cmp_service_ids(query, oc->rend_data->onion_address)) num_ipos++; } @@ -1377,9 +1337,8 @@ rend_service_intro_has_opened(origin_circuit_t *circuit) base32_encode(serviceid, REND_SERVICE_ID_LEN_BASE32+1, circuit->rend_data->rend_pk_digest, REND_SERVICE_ID_LEN); - service = rend_service_get_by_pk_digest_and_version( - circuit->rend_data->rend_pk_digest, - circuit->rend_data->rend_desc_version); + service = rend_service_get_by_pk_digest( + circuit->rend_data->rend_pk_digest); if (!service) { log_warn(LD_REND, "Unrecognized service ID %s on introduction circuit %d.", serviceid, circuit->_base.n_circ_id); @@ -1388,28 +1347,34 @@ rend_service_intro_has_opened(origin_circuit_t *circuit) } /* If we already have enough introduction circuits for this service, - * redefine this one as a general circuit. */ - if (count_established_intro_points(serviceid, - circuit->rend_data->rend_desc_version) > NUM_INTRO_POINTS) { - log_info(LD_CIRC|LD_REND, "We have just finished an introduction " - "circuit, but we already have enough. Redefining purpose to " - "general."); - TO_CIRCUIT(circuit)->purpose = CIRCUIT_PURPOSE_C_GENERAL; - circuit_has_opened(circuit); - return; + * redefine this one as a general circuit or close it, depending. */ + if (count_established_intro_points(serviceid) > NUM_INTRO_POINTS) { + or_options_t *options = get_options(); + if (options->ExcludeNodes) { + /* XXXX in some future version, we can test whether the transition is + allowed or not given the actual nodes in the circuit. But for now, + this case, we might as well close the thing. */ + log_info(LD_CIRC|LD_REND, "We have just finished an introduction " + "circuit, but we already have enough. Closing it."); + circuit_mark_for_close(TO_CIRCUIT(circuit), END_CIRC_REASON_NONE); + return; + } else { + tor_assert(circuit->build_state->is_internal); + log_info(LD_CIRC|LD_REND, "We have just finished an introduction " + "circuit, but we already have enough. Redefining purpose to " + "general; leaving as internal."); + TO_CIRCUIT(circuit)->purpose = CIRCUIT_PURPOSE_C_GENERAL; + circuit_has_opened(circuit); + return; + } } log_info(LD_REND, "Established circuit %d as introduction point for service %s", circuit->_base.n_circ_id, serviceid); - /* If the introduction point will not be used in an unversioned - * descriptor, use the intro key instead of the service key in - * ESTABLISH_INTRO. */ - if (service->descriptor_version == 0) - intro_key = service->private_key; - else - intro_key = circuit->intro_key; + /* Use the intro key instead of the service key in ESTABLISH_INTRO. */ + intro_key = circuit->intro_key; /* Build the payload for a RELAY_ESTABLISH_INTRO cell. */ r = crypto_pk_asn1_encode(intro_key, buf+2, RELAY_PAYLOAD_SIZE-2); @@ -1453,7 +1418,7 @@ rend_service_intro_has_opened(origin_circuit_t *circuit) /** Called when we get an INTRO_ESTABLISHED cell; mark the circuit as a * live introduction point, and note that the service descriptor is - * now out-of-date.*/ + * now out-of-date. */ int rend_service_intro_established(origin_circuit_t *circuit, const uint8_t *request, @@ -1470,9 +1435,8 @@ rend_service_intro_established(origin_circuit_t *circuit, goto err; } tor_assert(circuit->rend_data); - service = rend_service_get_by_pk_digest_and_version( - circuit->rend_data->rend_pk_digest, - circuit->rend_data->rend_desc_version); + service = rend_service_get_by_pk_digest( + circuit->rend_data->rend_pk_digest); if (!service) { log_warn(LD_REND, "Unknown service on introduction circuit %d.", circuit->_base.n_circ_id); @@ -1522,9 +1486,8 @@ rend_service_rendezvous_has_opened(origin_circuit_t *circuit) "cookie %s for service %s", circuit->_base.n_circ_id, hexcookie, serviceid); - service = rend_service_get_by_pk_digest_and_version( - circuit->rend_data->rend_pk_digest, - circuit->rend_data->rend_desc_version); + service = rend_service_get_by_pk_digest( + circuit->rend_data->rend_pk_digest); if (!service) { log_warn(LD_GENERAL, "Internal error: unrecognized service ID on " "introduction circuit."); @@ -1580,13 +1543,12 @@ rend_service_rendezvous_has_opened(origin_circuit_t *circuit) */ /** Return the (possibly non-open) introduction circuit ending at - * <b>intro</b> for the service whose public key is <b>pk_digest</b> and - * which publishes descriptor of version <b>desc_version</b>. Return - * NULL if no such service is found. + * <b>intro</b> for the service whose public key is <b>pk_digest</b>. + * (<b>desc_version</b> is ignored). Return NULL if no such service is + * found. */ static origin_circuit_t * -find_intro_circuit(rend_intro_point_t *intro, const char *pk_digest, - int desc_version) +find_intro_circuit(rend_intro_point_t *intro, const char *pk_digest) { origin_circuit_t *circ = NULL; @@ -1595,8 +1557,7 @@ find_intro_circuit(rend_intro_point_t *intro, const char *pk_digest, CIRCUIT_PURPOSE_S_INTRO))) { if (tor_memeq(circ->build_state->chosen_exit->identity_digest, intro->extend_info->identity_digest, DIGEST_LEN) && - circ->rend_data && - circ->rend_data->rend_desc_version == desc_version) { + circ->rend_data) { return circ; } } @@ -1606,8 +1567,7 @@ find_intro_circuit(rend_intro_point_t *intro, const char *pk_digest, CIRCUIT_PURPOSE_S_ESTABLISH_INTRO))) { if (tor_memeq(circ->build_state->chosen_exit->identity_digest, intro->extend_info->identity_digest, DIGEST_LEN) && - circ->rend_data && - circ->rend_data->rend_desc_version == desc_version) { + circ->rend_data) { return circ; } } @@ -1640,6 +1600,7 @@ directory_post_to_hs_dir(rend_service_descriptor_t *renddesc, } for (j = 0; j < smartlist_len(responsible_dirs); j++) { char desc_id_base32[REND_DESC_ID_V2_LEN_BASE32 + 1]; + char *hs_dir_ip; hs_dir = smartlist_get(responsible_dirs, j); if (smartlist_digest_isin(renddesc->successful_uploads, hs_dir->identity_digest)) @@ -1661,15 +1622,18 @@ directory_post_to_hs_dir(rend_service_descriptor_t *renddesc, strlen(desc->desc_str), 0); base32_encode(desc_id_base32, sizeof(desc_id_base32), desc->desc_id, DIGEST_LEN); + hs_dir_ip = tor_dup_ip(hs_dir->addr); log_info(LD_REND, "Sending publish request for v2 descriptor for " "service '%s' with descriptor ID '%s' with validity " "of %d seconds to hidden service directory '%s' on " - "port %d.", - safe_str(service_id), - safe_str(desc_id_base32), + "%s:%d.", + safe_str_client(service_id), + safe_str_client(desc_id_base32), seconds_valid, hs_dir->nickname, - hs_dir->dir_port); + hs_dir_ip, + hs_dir->or_port); + tor_free(hs_dir_ip); /* Remember successful upload to this router for next time. */ if (!smartlist_digest_isin(successful_uploads, hs_dir->identity_digest)) smartlist_add(successful_uploads, hs_dir->identity_digest); @@ -1699,9 +1663,8 @@ directory_post_to_hs_dir(rend_service_descriptor_t *renddesc, smartlist_free(successful_uploads); } -/** Encode and sign up-to-date v0 and/or v2 service descriptors for - * <b>service</b>, and upload it/them to all the dirservers/to the - * responsible hidden service directories. +/** Encode and sign an up-to-date service descriptor for <b>service</b>, + * and upload it/them to the responsible hidden service directories. */ static void upload_service_descriptor(rend_service_t *service) @@ -1713,35 +1676,8 @@ upload_service_descriptor(rend_service_t *service) rendpostperiod = get_options()->RendPostPeriod; - /* Upload unversioned (v0) descriptor? */ - if (service->descriptor_version == 0 && - get_options()->PublishHidServDescriptors) { - char *desc; - size_t desc_len; - /* Encode the descriptor. */ - if (rend_encode_service_descriptor(service->desc, - service->private_key, - &desc, &desc_len)<0) { - log_warn(LD_BUG, "Internal error: couldn't encode service descriptor; " - "not uploading."); - return; - } - - /* Post it to the dirservers */ - rend_get_service_id(service->desc->pk, serviceid); - log_info(LD_REND, "Sending publish request for hidden service %s", - serviceid); - directory_post_to_dirservers(DIR_PURPOSE_UPLOAD_RENDDESC, - ROUTER_PURPOSE_GENERAL, - HIDSERV_AUTHORITY, desc, desc_len, 0); - tor_free(desc); - service->next_upload_time = now + rendpostperiod; - uploaded = 1; - } - - /* Upload v2 descriptor? */ - if (service->descriptor_version == 2 && - get_options()->PublishHidServDescriptors) { + /* Upload descriptor? */ + if (get_options()->PublishHidServDescriptors) { networkstatus_t *c = networkstatus_get_latest_consensus(); if (c && smartlist_len(c->routerstatus_list) > 0) { int seconds_valid, i, j, num_descs; @@ -1880,8 +1816,7 @@ rend_services_introduce(void) for (j=0; j < smartlist_len(service->intro_nodes); ++j) { intro = smartlist_get(service->intro_nodes, j); router = router_get_by_digest(intro->extend_info->identity_digest); - if (!router || !find_intro_circuit(intro, service->pk_digest, - service->descriptor_version)) { + if (!router || !find_intro_circuit(intro, service->pk_digest)) { log_info(LD_REND,"Giving up on %s as intro point for %s.", intro->extend_info->nickname, service->service_id); if (service->desc) { @@ -1933,7 +1868,7 @@ rend_services_introduce(void) router_crn_flags_t flags = CRN_NEED_UPTIME; if (get_options()->_AllowInvalid & ALLOW_INVALID_INTRODUCTION) flags |= CRN_ALLOW_INVALID; - router = router_choose_random_node(NULL, intro_routers, + router = router_choose_random_node(intro_routers, options->ExcludeNodes, flags); if (!router) { log_warn(LD_REND, @@ -1945,10 +1880,8 @@ rend_services_introduce(void) smartlist_add(intro_routers, router); intro = tor_malloc_zero(sizeof(rend_intro_point_t)); intro->extend_info = extend_info_from_router(router); - if (service->descriptor_version == 2) { - intro->intro_key = crypto_new_pk_env(); - tor_assert(!crypto_pk_generate_key(intro->intro_key)); - } + intro->intro_key = crypto_new_pk_env(); + tor_assert(!crypto_pk_generate_key(intro->intro_key)); smartlist_add(service->intro_nodes, intro); log_info(LD_REND, "Picked router %s as an intro point for %s.", router->nickname, service->service_id); @@ -2041,8 +1974,7 @@ rend_consider_descriptor_republication(void) for (i=0; i < smartlist_len(rend_service_list); ++i) { service = smartlist_get(rend_service_list, i); - if (service->descriptor_version && service->desc && - !service->desc->all_uploads_performed) { + if (service->desc && !service->desc->all_uploads_performed) { /* If we failed in uploading a descriptor last time, try again *without* * updating the descriptor's contents. */ upload_service_descriptor(service); @@ -2068,10 +2000,9 @@ rend_service_dump_stats(int severity) service->directory); for (j=0; j < smartlist_len(service->intro_nodes); ++j) { intro = smartlist_get(service->intro_nodes, j); - safe_name = safe_str(intro->extend_info->nickname); + safe_name = safe_str_client(intro->extend_info->nickname); - circ = find_intro_circuit(intro, service->pk_digest, - service->descriptor_version); + circ = find_intro_circuit(intro, service->pk_digest); if (!circ) { log(severity, LD_GENERAL, " Intro point %d at %s: no circuit", j, safe_name); @@ -2102,9 +2033,8 @@ rend_service_set_connection_addr_port(edge_connection_t *conn, log_debug(LD_REND,"beginning to hunt for addr/port"); base32_encode(serviceid, REND_SERVICE_ID_LEN_BASE32+1, circ->rend_data->rend_pk_digest, REND_SERVICE_ID_LEN); - service = rend_service_get_by_pk_digest_and_version( - circ->rend_data->rend_pk_digest, - circ->rend_data->rend_desc_version); + service = rend_service_get_by_pk_digest( + circ->rend_data->rend_pk_digest); if (!service) { log_warn(LD_REND, "Couldn't find any service associated with pk %s on " "rendezvous circuit %d; closing.", diff --git a/src/or/rendservice.h b/src/or/rendservice.h new file mode 100644 index 0000000000..70389afe9a --- /dev/null +++ b/src/or/rendservice.h @@ -0,0 +1,37 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file rendservice.h + * \brief Header file for rendservice.c. + **/ + +#ifndef _TOR_RENDSERVICE_H +#define _TOR_RENDSERVICE_H + +int num_rend_services(void); +int rend_config_services(or_options_t *options, int validate_only); +int rend_service_load_keys(void); +void rend_services_introduce(void); +void rend_consider_services_upload(time_t now); +void rend_hsdir_routers_changed(void); +void rend_consider_descriptor_republication(void); + +void rend_service_intro_has_opened(origin_circuit_t *circuit); +int rend_service_intro_established(origin_circuit_t *circuit, + const uint8_t *request, + size_t request_len); +void rend_service_rendezvous_has_opened(origin_circuit_t *circuit); +int rend_service_introduce(origin_circuit_t *circuit, const uint8_t *request, + size_t request_len); +void rend_service_relaunch_rendezvous(origin_circuit_t *oldcirc); +int rend_service_set_connection_addr_port(edge_connection_t *conn, + origin_circuit_t *circ); +void rend_service_dump_stats(int severity); +void rend_service_free_all(void); + +#endif + diff --git a/src/or/rephist.c b/src/or/rephist.c index a3d8bcc986..8cddd2b5eb 100644 --- a/src/or/rephist.c +++ b/src/or/rephist.c @@ -6,15 +6,22 @@ * \file rephist.c * \brief Basic history and "reputation" functionality to remember * which servers have worked in the past, how much bandwidth we've - * been using, which ports we tend to want, and so on. + * been using, which ports we tend to want, and so on; further, + * exit port statistics and cell statistics. **/ #include "or.h" +#include "circuitlist.h" +#include "circuituse.h" +#include "config.h" +#include "networkstatus.h" +#include "rephist.h" +#include "router.h" +#include "routerlist.h" #include "ht.h" static void bw_arrays_init(void); static void predicted_ports_init(void); -static void hs_usage_init(void); /** Total number of bytes currently allocated in fields used by rephist.c. */ uint64_t rephist_total_alloc=0; @@ -67,6 +74,13 @@ typedef struct or_history_t { /** If nonzero, we have been unable to connect since this time. */ time_t down_since; + /** The address at which we most recently connected to this OR + * successfully. */ + tor_addr_t last_reached_addr; + + /** The port at which we most recently connected to this OR successfully */ + uint16_t last_reached_port; + /* === For MTBF tracking: */ /** Weighted sum total of all times that this router has been online. */ @@ -113,6 +127,7 @@ get_or_history(const char* id) rephist_total_num++; hist->link_history_map = digestmap_new(); hist->since = hist->changed = time(NULL); + tor_addr_make_unspec(&hist->last_reached_addr); digestmap_set(history_map, id, hist); } return hist; @@ -185,7 +200,6 @@ rep_hist_init(void) history_map = digestmap_new(); bw_arrays_init(); predicted_ports_init(); - hs_usage_init(); } /** Helper: note that we are no longer connected to the router with history @@ -284,13 +298,20 @@ rep_hist_note_connection_died(const char* id, time_t when) /** We have just decided that this router with identity digest <b>id</b> is * reachable, meaning we will give it a "Running" flag for the next while. */ void -rep_hist_note_router_reachable(const char *id, time_t when) +rep_hist_note_router_reachable(const char *id, const tor_addr_t *at_addr, + const uint16_t at_port, time_t when) { or_history_t *hist = get_or_history(id); int was_in_run = 1; char tbuf[ISO_TIME_LEN+1]; + int addr_changed, port_changed; tor_assert(hist); + tor_assert((!at_addr && !at_port) || (at_addr && at_port)); + + addr_changed = at_addr && + tor_addr_compare(at_addr, &hist->last_reached_addr, CMP_EXACT) != 0; + port_changed = at_port && at_port != hist->last_reached_port; if (!started_tracking_stability) started_tracking_stability = time(NULL); @@ -310,6 +331,27 @@ rep_hist_note_router_reachable(const char *id, time_t when) down_length = when - hist->start_of_downtime; hist->total_weighted_time += down_length; hist->start_of_downtime = 0; + } else if (addr_changed || port_changed) { + /* If we're reachable, but the address changed, treat this as some + * downtime. */ + int penalty = get_options()->TestingTorNetwork ? 240 : 3600; + networkstatus_t *ns; + + if ((ns = networkstatus_get_latest_consensus())) { + int fresh_interval = (int)(ns->fresh_until - ns->valid_after); + int live_interval = (int)(ns->valid_until - ns->valid_after); + /* on average, a descriptor addr change takes .5 intervals to make it + * into a consensus, and half a liveness period to make it to + * clients. */ + penalty = (int)(fresh_interval + live_interval) / 2; + } + format_local_iso_time(tbuf, hist->start_of_run); + log_info(LD_HIST,"Router %s still seems Running, but its address appears " + "to have changed since the last time it was reachable. I'm " + "going to treat it as having been down for %d seconds", + hex_str(id, DIGEST_LEN), penalty); + rep_hist_note_router_unreachable(id, when-penalty); + rep_hist_note_router_reachable(id, NULL, 0, when); } else { format_local_iso_time(tbuf, hist->start_of_run); if (was_in_run) @@ -319,6 +361,10 @@ rep_hist_note_router_reachable(const char *id, time_t when) log_info(LD_HIST,"Router %s is now Running; it was previously untracked", hex_str(id, DIGEST_LEN)); } + if (at_addr) + tor_addr_copy(&hist->last_reached_addr, at_addr); + if (at_port) + hist->last_reached_port = at_port; } /** We have just decided that this router is unreachable, meaning @@ -339,12 +385,20 @@ rep_hist_note_router_unreachable(const char *id, time_t when) long run_length = when - hist->start_of_run; format_local_iso_time(tbuf, hist->start_of_run); - hist->weighted_run_length += run_length; hist->total_run_weights += 1.0; hist->start_of_run = 0; - hist->weighted_uptime += run_length; - hist->total_weighted_time += run_length; + if (run_length < 0) { + unsigned long penalty = -run_length; +#define SUBTRACT_CLAMPED(var, penalty) \ + do { (var) = (var) < (penalty) ? 0 : (var) - (penalty); } while (0) + SUBTRACT_CLAMPED(hist->weighted_run_length, penalty); + SUBTRACT_CLAMPED(hist->weighted_uptime, penalty); + } else { + hist->weighted_run_length += run_length; + hist->weighted_uptime += run_length; + hist->total_weighted_time += run_length; + } was_running = 1; log_info(LD_HIST, "Router %s is now non-Running: it had previously been " "Running since %s. Its total weighted uptime is %lu/%lu.", @@ -417,7 +471,7 @@ rep_hist_downrate_old_runs(time_t now) static double get_stability(or_history_t *hist, time_t when) { - unsigned long total = hist->weighted_run_length; + long total = hist->weighted_run_length; double total_weights = hist->total_run_weights; if (hist->start_of_run) { @@ -453,8 +507,8 @@ get_total_weighted_time(or_history_t *hist, time_t when) static double get_weighted_fractional_uptime(or_history_t *hist, time_t when) { - unsigned long total = hist->total_weighted_time; - unsigned long up = hist->weighted_uptime; + long total = hist->total_weighted_time; + long up = hist->weighted_uptime; if (hist->start_of_run) { long run_length = (when - hist->start_of_run); @@ -474,6 +528,20 @@ get_weighted_fractional_uptime(or_history_t *hist, time_t when) return ((double) up) / total; } +/** Return how long the router whose identity digest is <b>id</b> has + * been reachable. Return 0 if the router is unknown or currently deemed + * unreachable. */ +long +rep_hist_get_uptime(const char *id, time_t when) +{ + or_history_t *hist = get_or_history(id); + if (!hist) + return 0; + if (!hist->start_of_run || when < hist->start_of_run) + return 0; + return when - hist->start_of_run; +} + /** Return an estimated MTBF for the router whose identity digest is * <b>id</b>. Return 0 if the router is unknown. */ double @@ -519,7 +587,7 @@ rep_hist_get_weighted_time_known(const char *id, time_t when) int rep_hist_have_measured_enough_stability(void) { - /* XXXX021 This doesn't do so well when we change our opinion + /* XXXX022 This doesn't do so well when we change our opinion * as to whether we're tracking router stability. */ return started_tracking_stability < time(NULL) - 4*60*60; } @@ -795,12 +863,12 @@ rep_hist_record_mtbf_data(time_t now, int missing_means_down) static char * rep_hist_format_router_status(or_history_t *hist, time_t now) { - char buf[1024]; char sor_buf[ISO_TIME_LEN+1]; char sod_buf[ISO_TIME_LEN+1]; double wfu; double mtbf; int up = 0, down = 0; + char *cp = NULL; if (hist->start_of_run) { format_iso_time(sor_buf, hist->start_of_run); @@ -813,7 +881,7 @@ rep_hist_format_router_status(or_history_t *hist, time_t now) wfu = get_weighted_fractional_uptime(hist, now); mtbf = get_stability(hist, now); - tor_snprintf(buf, sizeof(buf), + tor_asprintf(&cp, "%s%s%s" "%s%s%s" "wfu %0.3lf\n" @@ -831,8 +899,7 @@ rep_hist_format_router_status(or_history_t *hist, time_t now) hist->weighted_run_length, hist->total_run_weights ); - - return tor_strdup(buf); + return cp; } /** The last stability analysis document that we created, or NULL if we never @@ -925,10 +992,10 @@ find_next_with(smartlist_t *sl, int i, const char *prefix) return -1; } -/** How many bad times has parse_possibly_bad_iso_time parsed? */ +/** How many bad times has parse_possibly_bad_iso_time() parsed? */ static int n_bogus_times = 0; /** Parse the ISO-formatted time in <b>s</b> into *<b>time_out</b>, but - * rounds any pre-1970 date to Jan 1, 1970. */ + * round any pre-1970 date to Jan 1, 1970. */ static int parse_possibly_bad_iso_time(const char *s, time_t *time_out) { @@ -1161,6 +1228,8 @@ rep_hist_load_mtbf_data(time_t now) * totals? */ #define NUM_SECS_ROLLING_MEASURE 10 /** How large are the intervals for which we track and report bandwidth use? */ +/* XXXX Watch out! Before Tor 0.2.2.21-alpha, using any other value here would + * generate an unparseable state file. */ #define NUM_SECS_BW_SUM_INTERVAL (15*60) /** How far in the past do we remember and publish bandwidth use? */ #define NUM_SECS_BW_SUM_IS_VALID (24*60*60) @@ -1219,7 +1288,7 @@ commit_max(bw_array_t *b) b->total_in_period = 0; } -/** Shift the current observation time of 'b' forward by one second. */ +/** Shift the current observation time of <b>b</b> forward by one second. */ static INLINE void advance_obs(bw_array_t *b) { @@ -1249,14 +1318,18 @@ advance_obs(bw_array_t *b) static INLINE void add_obs(bw_array_t *b, time_t when, uint64_t n) { - /* Don't record data in the past. */ - if (when<b->cur_obs_time) - return; + if (when < b->cur_obs_time) + return; /* Don't record data in the past. */ + /* If we're currently adding observations for an earlier second than * 'when', advance b->cur_obs_time and b->cur_obs_idx by an - * appropriate number of seconds, and do all the other housekeeping */ - while (when>b->cur_obs_time) + * appropriate number of seconds, and do all the other housekeeping. */ + while (when > b->cur_obs_time) { + /* Doing this one second at a time is potentially inefficient, if we start + with a state file that is very old. Fortunately, it doesn't seem to + show up in profiles, so we can just ignore it for now. */ advance_obs(b); + } b->obs[b->cur_obs_idx] += n; b->total_in_period += n; @@ -1280,16 +1353,29 @@ bw_array_new(void) static bw_array_t *read_array = NULL; /** Recent history of bandwidth observations for write operations. */ static bw_array_t *write_array = NULL; - -/** Set up read_array and write_array. */ +/** Recent history of bandwidth observations for read operations for the + directory protocol. */ +static bw_array_t *dir_read_array = NULL; +/** Recent history of bandwidth observations for write operations for the + directory protocol. */ +static bw_array_t *dir_write_array = NULL; + +/** Set up [dir-]read_array and [dir-]write_array, freeing them if they + * already exist. */ static void bw_arrays_init(void) { + tor_free(read_array); + tor_free(write_array); + tor_free(dir_read_array); + tor_free(dir_write_array); read_array = bw_array_new(); write_array = bw_array_new(); + dir_read_array = bw_array_new(); + dir_write_array = bw_array_new(); } -/** We read <b>num_bytes</b> more bytes in second <b>when</b>. +/** Remember that we read <b>num_bytes</b> bytes in second <b>when</b>. * * Add num_bytes to the current running total for <b>when</b>. * @@ -1310,7 +1396,7 @@ rep_hist_note_bytes_written(size_t num_bytes, time_t when) add_obs(write_array, when, num_bytes); } -/** We wrote <b>num_bytes</b> more bytes in second <b>when</b>. +/** Remember that we wrote <b>num_bytes</b> bytes in second <b>when</b>. * (like rep_hist_note_bytes_written() above) */ void @@ -1320,6 +1406,24 @@ rep_hist_note_bytes_read(size_t num_bytes, time_t when) add_obs(read_array, when, num_bytes); } +/** Remember that we wrote <b>num_bytes</b> directory bytes in second + * <b>when</b>. (like rep_hist_note_bytes_written() above) + */ +void +rep_hist_note_dir_bytes_written(size_t num_bytes, time_t when) +{ + add_obs(dir_write_array, when, num_bytes); +} + +/** Remember that we read <b>num_bytes</b> directory bytes in second + * <b>when</b>. (like rep_hist_note_bytes_written() above) + */ +void +rep_hist_note_dir_bytes_read(size_t num_bytes, time_t when) +{ + add_obs(dir_read_array, when, num_bytes); +} + /** Helper: Return the largest value in b->maxima. (This is equal to the * most bandwidth used in any NUM_SECS_ROLLING_MEASURE period for the last * NUM_SECS_BW_SUM_IS_VALID seconds.) @@ -1355,14 +1459,14 @@ rep_hist_bandwidth_assess(void) return (int)(U64_TO_DBL(r)/NUM_SECS_ROLLING_MEASURE); } -/** Print the bandwidth history of b (either read_array or write_array) - * into the buffer pointed to by buf. The format is simply comma - * separated numbers, from oldest to newest. +/** Print the bandwidth history of b (either [dir-]read_array or + * [dir-]write_array) into the buffer pointed to by buf. The format is + * simply comma separated numbers, from oldest to newest. * * It returns the number of bytes written. */ static size_t -rep_hist_fill_bandwidth_history(char *buf, size_t len, bw_array_t *b) +rep_hist_fill_bandwidth_history(char *buf, size_t len, const bw_array_t *b) { char *cp = buf; int i, n; @@ -1407,151 +1511,254 @@ rep_hist_fill_bandwidth_history(char *buf, size_t len, bw_array_t *b) } /** Allocate and return lines for representing this server's bandwidth - * history in its descriptor. + * history in its descriptor. We publish these lines in our extra-info + * descriptor. */ char * -rep_hist_get_bandwidth_lines(int for_extrainfo) +rep_hist_get_bandwidth_lines(void) { char *buf, *cp; char t[ISO_TIME_LEN+1]; int r; - bw_array_t *b; + bw_array_t *b = NULL; + const char *desc = NULL; size_t len; - /* opt (read|write)-history yyyy-mm-dd HH:MM:SS (n s) n,n,n,n,n... */ - len = (60+20*NUM_TOTALS)*2; + /* opt [dirreq-](read|write)-history yyyy-mm-dd HH:MM:SS (n s) n,n,n... */ +/* The n,n,n part above. Largest representation of a uint64_t is 20 chars + * long, plus the comma. */ +#define MAX_HIST_VALUE_LEN 21*NUM_TOTALS + len = (67+MAX_HIST_VALUE_LEN)*4; buf = tor_malloc_zero(len); cp = buf; - for (r=0;r<2;++r) { - b = r?read_array:write_array; + for (r=0;r<4;++r) { + char tmp[MAX_HIST_VALUE_LEN]; + size_t slen; + switch (r) { + case 0: + b = write_array; + desc = "write-history"; + break; + case 1: + b = read_array; + desc = "read-history"; + break; + case 2: + b = dir_write_array; + desc = "dirreq-write-history"; + break; + case 3: + b = dir_read_array; + desc = "dirreq-read-history"; + break; + } tor_assert(b); + slen = rep_hist_fill_bandwidth_history(tmp, MAX_HIST_VALUE_LEN, b); + /* If we don't have anything to write, skip to the next entry. */ + if (slen == 0) + continue; format_iso_time(t, b->next_period-NUM_SECS_BW_SUM_INTERVAL); - tor_snprintf(cp, len-(cp-buf), "%s%s %s (%d s) ", - for_extrainfo ? "" : "opt ", - r ? "read-history" : "write-history", t, - NUM_SECS_BW_SUM_INTERVAL); + tor_snprintf(cp, len-(cp-buf), "%s %s (%d s) ", + desc, t, NUM_SECS_BW_SUM_INTERVAL); cp += strlen(cp); - cp += rep_hist_fill_bandwidth_history(cp, len-(cp-buf), b); + strlcat(cp, tmp, len-(cp-buf)); + cp += slen; strlcat(cp, "\n", len-(cp-buf)); ++cp; } return buf; } -/** Update <b>state</b> with the newest bandwidth history. */ +/** Write a single bw_array_t into the Values, Ends, Interval, and Maximum + * entries of an or_state_t. Done before writing out a new state file. */ +static void +rep_hist_update_bwhist_state_section(or_state_t *state, + const bw_array_t *b, + smartlist_t **s_values, + smartlist_t **s_maxima, + time_t *s_begins, + int *s_interval) +{ + char *cp; + int i,j; + uint64_t maxval; + + if (*s_values) { + SMARTLIST_FOREACH(*s_values, char *, val, tor_free(val)); + smartlist_free(*s_values); + } + if (*s_maxima) { + SMARTLIST_FOREACH(*s_maxima, char *, val, tor_free(val)); + smartlist_free(*s_maxima); + } + if (! server_mode(get_options())) { + /* Clients don't need to store bandwidth history persistently; + * force these values to the defaults. */ + /* FFFF we should pull the default out of config.c's state table, + * so we don't have two defaults. */ + if (*s_begins != 0 || *s_interval != 900) { + time_t now = time(NULL); + time_t save_at = get_options()->AvoidDiskWrites ? now+3600 : now+600; + or_state_mark_dirty(state, save_at); + } + *s_begins = 0; + *s_interval = 900; + *s_values = smartlist_create(); + *s_maxima = smartlist_create(); + return; + } + *s_begins = b->next_period; + *s_interval = NUM_SECS_BW_SUM_INTERVAL; + + *s_values = smartlist_create(); + *s_maxima = smartlist_create(); + /* Set i to first position in circular array */ + i = (b->num_maxes_set <= b->next_max_idx) ? 0 : b->next_max_idx; + for (j=0; j < b->num_maxes_set; ++j,++i) { + if (i >= NUM_TOTALS) + i = 0; + tor_asprintf(&cp, U64_FORMAT, U64_PRINTF_ARG(b->totals[i] & ~0x3ff)); + smartlist_add(*s_values, cp); + maxval = b->maxima[i] / NUM_SECS_ROLLING_MEASURE; + tor_asprintf(&cp, U64_FORMAT, U64_PRINTF_ARG(maxval & ~0x3ff)); + smartlist_add(*s_maxima, cp); + } + tor_asprintf(&cp, U64_FORMAT, U64_PRINTF_ARG(b->total_in_period & ~0x3ff)); + smartlist_add(*s_values, cp); + maxval = b->max_total / NUM_SECS_ROLLING_MEASURE; + tor_asprintf(&cp, U64_FORMAT, U64_PRINTF_ARG(maxval & ~0x3ff)); + smartlist_add(*s_maxima, cp); +} + +/** Update <b>state</b> with the newest bandwidth history. Done before + * writing out a new state file. */ void rep_hist_update_state(or_state_t *state) { - int len, r; - char *buf, *cp; - smartlist_t **s_values; - time_t *s_begins; - int *s_interval; - bw_array_t *b; - - len = 20*NUM_TOTALS+1; - buf = tor_malloc_zero(len); +#define UPDATE(arrname,st) \ + rep_hist_update_bwhist_state_section(state,\ + (arrname),\ + &state->BWHistory ## st ## Values, \ + &state->BWHistory ## st ## Maxima, \ + &state->BWHistory ## st ## Ends, \ + &state->BWHistory ## st ## Interval) - for (r=0;r<2;++r) { - b = r?read_array:write_array; - s_begins = r?&state->BWHistoryReadEnds :&state->BWHistoryWriteEnds; - s_interval= r?&state->BWHistoryReadInterval:&state->BWHistoryWriteInterval; - s_values = r?&state->BWHistoryReadValues :&state->BWHistoryWriteValues; + UPDATE(write_array, Write); + UPDATE(read_array, Read); + UPDATE(dir_write_array, DirWrite); + UPDATE(dir_read_array, DirRead); - if (*s_values) { - SMARTLIST_FOREACH(*s_values, char *, val, tor_free(val)); - smartlist_free(*s_values); - } - if (! server_mode(get_options())) { - /* Clients don't need to store bandwidth history persistently; - * force these values to the defaults. */ - /* FFFF we should pull the default out of config.c's state table, - * so we don't have two defaults. */ - if (*s_begins != 0 || *s_interval != 900) { - time_t now = time(NULL); - time_t save_at = get_options()->AvoidDiskWrites ? now+3600 : now+600; - or_state_mark_dirty(state, save_at); - } - *s_begins = 0; - *s_interval = 900; - *s_values = smartlist_create(); - continue; - } - *s_begins = b->next_period; - *s_interval = NUM_SECS_BW_SUM_INTERVAL; - cp = buf; - cp += rep_hist_fill_bandwidth_history(cp, len, b); - tor_snprintf(cp, len-(cp-buf), cp == buf ? U64_FORMAT : ","U64_FORMAT, - U64_PRINTF_ARG(b->total_in_period)); - *s_values = smartlist_create(); - if (server_mode(get_options())) - smartlist_split_string(*s_values, buf, ",", SPLIT_SKIP_SPACE, 0); - } - tor_free(buf); if (server_mode(get_options())) { - or_state_mark_dirty(get_or_state(), time(NULL)+(2*3600)); + or_state_mark_dirty(state, time(NULL)+(2*3600)); } +#undef UPDATE } -/** Set bandwidth history from our saved state. */ -int -rep_hist_load_state(or_state_t *state, char **err) +/** Load a single bw_array_t from its Values, Ends, Maxima, and Interval + * entries in an or_state_t. Done while reading the state file. */ +static int +rep_hist_load_bwhist_state_section(bw_array_t *b, + const smartlist_t *s_values, + const smartlist_t *s_maxima, + const time_t s_begins, + const int s_interval) { - time_t s_begins, start; time_t now = time(NULL); - uint64_t v; - int r,i,ok; - int all_ok = 1; - int s_interval; - smartlist_t *s_values; - bw_array_t *b; - - /* Assert they already have been malloced */ - tor_assert(read_array && write_array); + int retval = 0; + time_t start; - for (r=0;r<2;++r) { - b = r?read_array:write_array; - s_begins = r?state->BWHistoryReadEnds:state->BWHistoryWriteEnds; - s_interval = r?state->BWHistoryReadInterval:state->BWHistoryWriteInterval; - s_values = r?state->BWHistoryReadValues:state->BWHistoryWriteValues; - if (s_values && s_begins >= now - NUM_SECS_BW_SUM_INTERVAL*NUM_TOTALS) { - start = s_begins - s_interval*(smartlist_len(s_values)); - if (start > now) - continue; - b->cur_obs_time = start; - b->next_period = start + NUM_SECS_BW_SUM_INTERVAL; - SMARTLIST_FOREACH(s_values, char *, cp, { + uint64_t v, mv; + int i,ok,ok_m; + int have_maxima = (smartlist_len(s_values) == smartlist_len(s_maxima)); + + if (s_values && s_begins >= now - NUM_SECS_BW_SUM_INTERVAL*NUM_TOTALS) { + start = s_begins - s_interval*(smartlist_len(s_values)); + if (start > now) + return 0; + b->cur_obs_time = start; + b->next_period = start + NUM_SECS_BW_SUM_INTERVAL; + SMARTLIST_FOREACH_BEGIN(s_values, const char *, cp) { + const char *maxstr = NULL; v = tor_parse_uint64(cp, 10, 0, UINT64_MAX, &ok, NULL); + if (have_maxima) { + maxstr = smartlist_get(s_maxima, cp_sl_idx); + mv = tor_parse_uint64(maxstr, 10, 0, UINT64_MAX, &ok_m, NULL); + mv *= NUM_SECS_ROLLING_MEASURE; + } else { + /* No maxima known; guess average rate to be conservative. */ + mv = (v / s_interval) * NUM_SECS_ROLLING_MEASURE; + } if (!ok) { - all_ok=0; - log_notice(LD_HIST, "Could not parse '%s' into a number.'", cp); + retval = -1; + log_notice(LD_HIST, "Could not parse value '%s' into a number.'",cp); } + if (maxstr && !ok_m) { + retval = -1; + log_notice(LD_HIST, "Could not parse maximum '%s' into a number.'", + maxstr); + } + if (start < now) { - add_obs(b, start, v); - start += NUM_SECS_BW_SUM_INTERVAL; + time_t cur_start = start; + time_t actual_interval_len = s_interval; + uint64_t cur_val = 0; + /* Calculate the average per second. This is the best we can do + * because our state file doesn't have per-second resolution. */ + if (start + s_interval > now) + actual_interval_len = now - start; + cur_val = v / actual_interval_len; + /* This is potentially inefficient, but since we don't do it very + * often it should be ok. */ + while (cur_start < start + actual_interval_len) { + add_obs(b, cur_start, cur_val); + ++cur_start; + } + b->max_total = mv; + /* This will result in some fairly choppy history if s_interval + * is not the same as NUM_SECS_BW_SUM_INTERVAL. XXXX */ + start += actual_interval_len; } - }); - } + } SMARTLIST_FOREACH_END(cp); + } - /* Clean up maxima and observed */ - /* Do we really want to zero this for the purpose of max capacity? */ - for (i=0; i<NUM_SECS_ROLLING_MEASURE; ++i) { - b->obs[i] = 0; - } - b->total_obs = 0; - for (i=0; i<NUM_TOTALS; ++i) { - b->maxima[i] = 0; - } - b->max_total = 0; + /* Clean up maxima and observed */ + for (i=0; i<NUM_SECS_ROLLING_MEASURE; ++i) { + b->obs[i] = 0; } + b->total_obs = 0; + + return retval; +} + +/** Set bandwidth history from the state file we just loaded. */ +int +rep_hist_load_state(or_state_t *state, char **err) +{ + int all_ok = 1; + /* Assert they already have been malloced */ + tor_assert(read_array && write_array); + tor_assert(dir_read_array && dir_write_array); + +#define LOAD(arrname,st) \ + if (rep_hist_load_bwhist_state_section( \ + (arrname), \ + state->BWHistory ## st ## Values, \ + state->BWHistory ## st ## Maxima, \ + state->BWHistory ## st ## Ends, \ + state->BWHistory ## st ## Interval)<0) \ + all_ok = 0 + + LOAD(write_array, Write); + LOAD(read_array, Read); + LOAD(dir_write_array, DirWrite); + LOAD(dir_read_array, DirRead); + +#undef LOAD if (!all_ok) { *err = tor_strdup("Parsing of bandwidth history values failed"); /* and create fresh arrays */ - tor_free(read_array); - tor_free(write_array); - read_array = bw_array_new(); - write_array = bw_array_new(); + bw_arrays_init(); return -1; } return 0; @@ -1571,7 +1778,7 @@ static smartlist_t *predicted_ports_times=NULL; static void add_predicted_port(time_t now, uint16_t port) { - /* XXXX we could just use uintptr_t here, I think. */ + /* XXXX we could just use uintptr_t here, I think. -NM */ uint16_t *tmp_port = tor_malloc(sizeof(uint16_t)); time_t *tmp_time = tor_malloc(sizeof(time_t)); *tmp_port = port; @@ -1715,8 +1922,8 @@ rep_hist_get_predicted_internal(time_t now, int *need_uptime, return 0; /* too long ago */ if (predicted_internal_uptime_time + PREDICTED_CIRCS_RELEVANCE_TIME >= now) *need_uptime = 1; - if (predicted_internal_capacity_time + PREDICTED_CIRCS_RELEVANCE_TIME >= now) - *need_capacity = 1; + // Always predict that we need capacity. + *need_capacity = 1; return 1; } @@ -1845,564 +2052,530 @@ dump_pk_ops(int severity) pk_op_counts.n_rend_server_ops); } -/** Free all storage held by the OR/link history caches, by the - * bandwidth history arrays, or by the port history. */ +/*** Exit port statistics ***/ + +/* Some constants */ +/** To what multiple should byte numbers be rounded up? */ +#define EXIT_STATS_ROUND_UP_BYTES 1024 +/** To what multiple should stream counts be rounded up? */ +#define EXIT_STATS_ROUND_UP_STREAMS 4 +/** Number of TCP ports */ +#define EXIT_STATS_NUM_PORTS 65536 +/** Top n ports that will be included in exit stats. */ +#define EXIT_STATS_TOP_N_PORTS 10 + +/* The following data structures are arrays and no fancy smartlists or maps, + * so that all write operations can be done in constant time. This comes at + * the price of some memory (1.25 MB) and linear complexity when writing + * stats for measuring relays. */ +/** Number of bytes read in current period by exit port */ +static uint64_t *exit_bytes_read = NULL; +/** Number of bytes written in current period by exit port */ +static uint64_t *exit_bytes_written = NULL; +/** Number of streams opened in current period by exit port */ +static uint32_t *exit_streams = NULL; + +/** Start time of exit stats or 0 if we're not collecting exit stats. */ +static time_t start_of_exit_stats_interval; + +/** Initialize exit port stats. */ void -rep_hist_free_all(void) +rep_hist_exit_stats_init(time_t now) { - digestmap_free(history_map, free_or_history); - tor_free(read_array); - tor_free(write_array); - tor_free(last_stability_doc); - built_last_stability_doc_at = 0; - predicted_ports_free(); + start_of_exit_stats_interval = now; + exit_bytes_read = tor_malloc_zero(EXIT_STATS_NUM_PORTS * + sizeof(uint64_t)); + exit_bytes_written = tor_malloc_zero(EXIT_STATS_NUM_PORTS * + sizeof(uint64_t)); + exit_streams = tor_malloc_zero(EXIT_STATS_NUM_PORTS * + sizeof(uint32_t)); } -/****************** hidden service usage statistics ******************/ - -/** How large are the intervals for which we track and report hidden service - * use? */ -#define NUM_SECS_HS_USAGE_SUM_INTERVAL (15*60) -/** How far in the past do we remember and publish hidden service use? */ -#define NUM_SECS_HS_USAGE_SUM_IS_VALID (24*60*60) -/** How many hidden service usage intervals do we remember? (derived) */ -#define NUM_TOTALS_HS_USAGE (NUM_SECS_HS_USAGE_SUM_IS_VALID/ \ - NUM_SECS_HS_USAGE_SUM_INTERVAL) - -/** List element containing a service id and the count. */ -typedef struct hs_usage_list_elem_t { - /** Service id of this elem. */ - char service_id[REND_SERVICE_ID_LEN_BASE32+1]; - /** Number of occurrences for the given service id. */ - uint32_t count; - /* Pointer to next list elem */ - struct hs_usage_list_elem_t *next; -} hs_usage_list_elem_t; - -/** Ordered list that stores service ids and the number of observations. It is - * ordered by the number of occurrences in descending order. Its purpose is to - * calculate the frequency distribution when the period is over. */ -typedef struct hs_usage_list_t { - /* Pointer to the first element in the list. */ - hs_usage_list_elem_t *start; - /* Number of total occurrences for all list elements. */ - uint32_t total_count; - /* Number of service ids, i.e. number of list elements. */ - uint32_t total_service_ids; -} hs_usage_list_t; - -/** Tracks service-related observations in the current period and their - * history. */ -typedef struct hs_usage_service_related_observation_t { - /** Ordered list that stores service ids and the number of observations in - * the current period. It is ordered by the number of occurrences in - * descending order. Its purpose is to calculate the frequency distribution - * when the period is over. */ - hs_usage_list_t *list; - /** Circular arrays that store the history of observations. totals stores all - * observations, twenty (ten, five) the number of observations related to a - * service id being accounted for the top 20 (10, 5) percent of all - * observations. */ - uint32_t totals[NUM_TOTALS_HS_USAGE]; - uint32_t five[NUM_TOTALS_HS_USAGE]; - uint32_t ten[NUM_TOTALS_HS_USAGE]; - uint32_t twenty[NUM_TOTALS_HS_USAGE]; -} hs_usage_service_related_observation_t; - -/** Tracks the history of general period-related observations, i.e. those that - * cannot be related to a specific service id. */ -typedef struct hs_usage_general_period_related_observations_t { - /** Circular array that stores the history of observations. */ - uint32_t totals[NUM_TOTALS_HS_USAGE]; -} hs_usage_general_period_related_observations_t; - -/** Keeps information about the current observation period and its relation to - * the histories of observations. */ -typedef struct hs_usage_current_observation_period_t { - /** Where do we write the next history entry? */ - int next_idx; - /** How many values in history have been set ever? (upper bound!) */ - int num_set; - /** When did this period begin? */ - time_t start_of_current_period; - /** When does the next period begin? */ - time_t start_of_next_period; -} hs_usage_current_observation_period_t; - -/** Usage statistics for the current observation period. */ -static hs_usage_current_observation_period_t *current_period = NULL; - -/** Total number of descriptor publish requests in the current observation - * period. */ -static hs_usage_service_related_observation_t *publish_total = NULL; - -/** Number of descriptor publish requests for services that have not been - * seen before in the current observation period. */ -static hs_usage_service_related_observation_t *publish_novel = NULL; - -/** Total number of descriptor fetch requests in the current observation - * period. */ -static hs_usage_service_related_observation_t *fetch_total = NULL; - -/** Number of successful descriptor fetch requests in the current - * observation period. */ -static hs_usage_service_related_observation_t *fetch_successful = NULL; - -/** Number of descriptors stored in the current observation period. */ -static hs_usage_general_period_related_observations_t *descs = NULL; - -/** Creates an empty ordered list element. */ -static hs_usage_list_elem_t * -hs_usage_list_elem_new(void) +/** Reset counters for exit port statistics. */ +void +rep_hist_reset_exit_stats(time_t now) { - hs_usage_list_elem_t *e; - e = tor_malloc_zero(sizeof(hs_usage_list_elem_t)); - rephist_total_alloc += sizeof(hs_usage_list_elem_t); - e->count = 1; - e->next = NULL; - return e; + start_of_exit_stats_interval = now; + memset(exit_bytes_read, 0, EXIT_STATS_NUM_PORTS * sizeof(uint64_t)); + memset(exit_bytes_written, 0, EXIT_STATS_NUM_PORTS * sizeof(uint64_t)); + memset(exit_streams, 0, EXIT_STATS_NUM_PORTS * sizeof(uint32_t)); } -/** Creates an empty ordered list. */ -static hs_usage_list_t * -hs_usage_list_new(void) +/** Stop collecting exit port stats in a way that we can re-start doing + * so in rep_hist_exit_stats_init(). */ +void +rep_hist_exit_stats_term(void) { - hs_usage_list_t *l; - l = tor_malloc_zero(sizeof(hs_usage_list_t)); - rephist_total_alloc += sizeof(hs_usage_list_t); - l->start = NULL; - l->total_count = 0; - l->total_service_ids = 0; - return l; + start_of_exit_stats_interval = 0; + tor_free(exit_bytes_read); + tor_free(exit_bytes_written); + tor_free(exit_streams); } -/** Creates an empty structure for storing service-related observations. */ -static hs_usage_service_related_observation_t * -hs_usage_service_related_observation_new(void) +/** Helper for qsort: compare two ints. */ +static int +_compare_int(const void *x, const void *y) { - hs_usage_service_related_observation_t *h; - h = tor_malloc_zero(sizeof(hs_usage_service_related_observation_t)); - rephist_total_alloc += sizeof(hs_usage_service_related_observation_t); - h->list = hs_usage_list_new(); - return h; + return (*(int*)x - *(int*)y); } -/** Creates an empty structure for storing general period-related - * observations. */ -static hs_usage_general_period_related_observations_t * -hs_usage_general_period_related_observations_new(void) -{ - hs_usage_general_period_related_observations_t *p; - p = tor_malloc_zero(sizeof(hs_usage_general_period_related_observations_t)); - rephist_total_alloc+= sizeof(hs_usage_general_period_related_observations_t); - return p; -} +/** Return a newly allocated string containing the exit port statistics + * until <b>now</b>, or NULL if we're not collecting exit stats. */ +char * +rep_hist_format_exit_stats(time_t now) +{ + int i, j, top_elements = 0, cur_min_idx = 0, cur_port; + uint64_t top_bytes[EXIT_STATS_TOP_N_PORTS]; + int top_ports[EXIT_STATS_TOP_N_PORTS]; + uint64_t cur_bytes = 0, other_read = 0, other_written = 0, + total_read = 0, total_written = 0; + uint32_t total_streams = 0, other_streams = 0; + char *buf; + smartlist_t *written_strings, *read_strings, *streams_strings; + char *written_string, *read_string, *streams_string; + char t[ISO_TIME_LEN+1]; + char *result; -/** Creates an empty structure for storing period-specific information. */ -static hs_usage_current_observation_period_t * -hs_usage_current_observation_period_new(void) -{ - hs_usage_current_observation_period_t *c; - time_t now; - c = tor_malloc_zero(sizeof(hs_usage_current_observation_period_t)); - rephist_total_alloc += sizeof(hs_usage_current_observation_period_t); - now = time(NULL); - c->start_of_current_period = now; - c->start_of_next_period = now + NUM_SECS_HS_USAGE_SUM_INTERVAL; - return c; -} + if (!start_of_exit_stats_interval) + return NULL; /* Not initialized. */ -/** Initializes the structures for collecting hidden service usage data. */ -static void -hs_usage_init(void) -{ - current_period = hs_usage_current_observation_period_new(); - publish_total = hs_usage_service_related_observation_new(); - publish_novel = hs_usage_service_related_observation_new(); - fetch_total = hs_usage_service_related_observation_new(); - fetch_successful = hs_usage_service_related_observation_new(); - descs = hs_usage_general_period_related_observations_new(); -} + /* Go through all ports to find the n ports that saw most written and + * read bytes. + * + * Invariant: at the end of the loop for iteration i, + * total_read is the sum of all exit_bytes_read[0..i] + * total_written is the sum of all exit_bytes_written[0..i] + * total_stream is the sum of all exit_streams[0..i] + * + * top_elements = MAX(EXIT_STATS_TOP_N_PORTS, + * #{j | 0 <= j <= i && volume(i) > 0}) + * + * For all 0 <= j < top_elements, + * top_bytes[j] > 0 + * 0 <= top_ports[j] <= 65535 + * top_bytes[j] = volume(top_ports[j]) + * + * There is no j in 0..i and k in 0..top_elements such that: + * volume(j) > top_bytes[k] AND j is not in top_ports[0..top_elements] + * + * There is no j!=cur_min_idx in 0..top_elements such that: + * top_bytes[j] < top_bytes[cur_min_idx] + * + * where volume(x) == exit_bytes_read[x]+exit_bytes_written[x] + * + * Worst case: O(EXIT_STATS_NUM_PORTS * EXIT_STATS_TOP_N_PORTS) + */ + for (i = 1; i < EXIT_STATS_NUM_PORTS; i++) { + total_read += exit_bytes_read[i]; + total_written += exit_bytes_written[i]; + total_streams += exit_streams[i]; + cur_bytes = exit_bytes_read[i] + exit_bytes_written[i]; + if (cur_bytes == 0) { + continue; + } + if (top_elements < EXIT_STATS_TOP_N_PORTS) { + top_bytes[top_elements] = cur_bytes; + top_ports[top_elements++] = i; + } else if (cur_bytes > top_bytes[cur_min_idx]) { + top_bytes[cur_min_idx] = cur_bytes; + top_ports[cur_min_idx] = i; + } else { + continue; + } + cur_min_idx = 0; + for (j = 1; j < top_elements; j++) { + if (top_bytes[j] < top_bytes[cur_min_idx]) { + cur_min_idx = j; + } + } + } -/** Clears the given ordered list by resetting its attributes and releasing - * the memory allocated by its elements. */ -static void -hs_usage_list_clear(hs_usage_list_t *lst) -{ - /* walk through elements and free memory */ - hs_usage_list_elem_t *current = lst->start; - hs_usage_list_elem_t *tmp; - while (current != NULL) { - tmp = current->next; - rephist_total_alloc -= sizeof(hs_usage_list_elem_t); - tor_free(current); - current = tmp; + /* Add observations of top ports to smartlists. */ + written_strings = smartlist_create(); + read_strings = smartlist_create(); + streams_strings = smartlist_create(); + other_read = total_read; + other_written = total_written; + other_streams = total_streams; + /* Sort the ports; this puts them out of sync with top_bytes, but we + * won't be using top_bytes again anyway */ + qsort(top_ports, top_elements, sizeof(int), _compare_int); + for (j = 0; j < top_elements; j++) { + cur_port = top_ports[j]; + if (exit_bytes_written[cur_port] > 0) { + uint64_t num = round_uint64_to_next_multiple_of( + exit_bytes_written[cur_port], + EXIT_STATS_ROUND_UP_BYTES); + num /= 1024; + buf = NULL; + tor_asprintf(&buf, "%d="U64_FORMAT, cur_port, U64_PRINTF_ARG(num)); + smartlist_add(written_strings, buf); + other_written -= exit_bytes_written[cur_port]; + } + if (exit_bytes_read[cur_port] > 0) { + uint64_t num = round_uint64_to_next_multiple_of( + exit_bytes_read[cur_port], + EXIT_STATS_ROUND_UP_BYTES); + num /= 1024; + buf = NULL; + tor_asprintf(&buf, "%d="U64_FORMAT, cur_port, U64_PRINTF_ARG(num)); + smartlist_add(read_strings, buf); + other_read -= exit_bytes_read[cur_port]; + } + if (exit_streams[cur_port] > 0) { + uint32_t num = round_uint32_to_next_multiple_of( + exit_streams[cur_port], + EXIT_STATS_ROUND_UP_STREAMS); + buf = NULL; + tor_asprintf(&buf, "%d=%u", cur_port, num); + smartlist_add(streams_strings, buf); + other_streams -= exit_streams[cur_port]; + } } - /* reset attributes */ - lst->start = NULL; - lst->total_count = 0; - lst->total_service_ids = 0; - return; -} -/** Frees the memory used by the given list. */ -static void -hs_usage_list_free(hs_usage_list_t *lst) -{ - if (!lst) - return; - hs_usage_list_clear(lst); - rephist_total_alloc -= sizeof(hs_usage_list_t); - tor_free(lst); + /* Add observations of other ports in a single element. */ + other_written = round_uint64_to_next_multiple_of(other_written, + EXIT_STATS_ROUND_UP_BYTES); + other_written /= 1024; + buf = NULL; + tor_asprintf(&buf, "other="U64_FORMAT, U64_PRINTF_ARG(other_written)); + smartlist_add(written_strings, buf); + other_read = round_uint64_to_next_multiple_of(other_read, + EXIT_STATS_ROUND_UP_BYTES); + other_read /= 1024; + buf = NULL; + tor_asprintf(&buf, "other="U64_FORMAT, U64_PRINTF_ARG(other_read)); + smartlist_add(read_strings, buf); + other_streams = round_uint32_to_next_multiple_of(other_streams, + EXIT_STATS_ROUND_UP_STREAMS); + buf = NULL; + tor_asprintf(&buf, "other=%u", other_streams); + smartlist_add(streams_strings, buf); + + /* Join all observations in single strings. */ + written_string = smartlist_join_strings(written_strings, ",", 0, NULL); + read_string = smartlist_join_strings(read_strings, ",", 0, NULL); + streams_string = smartlist_join_strings(streams_strings, ",", 0, NULL); + SMARTLIST_FOREACH(written_strings, char *, cp, tor_free(cp)); + SMARTLIST_FOREACH(read_strings, char *, cp, tor_free(cp)); + SMARTLIST_FOREACH(streams_strings, char *, cp, tor_free(cp)); + smartlist_free(written_strings); + smartlist_free(read_strings); + smartlist_free(streams_strings); + + /* Put everything together. */ + format_iso_time(t, now); + tor_asprintf(&result, "exit-stats-end %s (%d s)\n" + "exit-kibibytes-written %s\n" + "exit-kibibytes-read %s\n" + "exit-streams-opened %s\n", + t, (unsigned) (now - start_of_exit_stats_interval), + written_string, + read_string, + streams_string); + tor_free(written_string); + tor_free(read_string); + tor_free(streams_string); + return result; } -/** Frees the memory used by the given service-related observations. */ -static void -hs_usage_service_related_observation_free( - hs_usage_service_related_observation_t *s) +/** If 24 hours have passed since the beginning of the current exit port + * stats period, write exit stats to $DATADIR/stats/exit-stats (possibly + * overwriting an existing file) and reset counters. Return when we would + * next want to write exit stats or 0 if we never want to write. */ +time_t +rep_hist_exit_stats_write(time_t now) { - if (!s) - return; - hs_usage_list_free(s->list); - rephist_total_alloc -= sizeof(hs_usage_service_related_observation_t); - tor_free(s); -} + char *statsdir = NULL, *filename = NULL, *str = NULL; -/** Frees the memory used by the given period-specific observations. */ -static void -hs_usage_general_period_related_observations_free( - hs_usage_general_period_related_observations_t *s) -{ - rephist_total_alloc-=sizeof(hs_usage_general_period_related_observations_t); - tor_free(s); -} + if (!start_of_exit_stats_interval) + return 0; /* Not initialized. */ + if (start_of_exit_stats_interval + WRITE_STATS_INTERVAL > now) + goto done; /* Not ready to write. */ -/** Frees the memory used by period-specific information. */ -static void -hs_usage_current_observation_period_free( - hs_usage_current_observation_period_t *s) -{ - rephist_total_alloc -= sizeof(hs_usage_current_observation_period_t); - tor_free(s); -} + log_info(LD_HIST, "Writing exit port statistics to disk."); -/** Frees all memory that was used for collecting hidden service usage data. */ -void -hs_usage_free_all(void) -{ - hs_usage_general_period_related_observations_free(descs); - descs = NULL; - hs_usage_service_related_observation_free(fetch_successful); - hs_usage_service_related_observation_free(fetch_total); - hs_usage_service_related_observation_free(publish_novel); - hs_usage_service_related_observation_free(publish_total); - fetch_successful = fetch_total = publish_novel = publish_total = NULL; - hs_usage_current_observation_period_free(current_period); - current_period = NULL; -} + /* Generate history string. */ + str = rep_hist_format_exit_stats(now); -/** Inserts a new occurrence for the given service id to the given ordered - * list. */ -static void -hs_usage_insert_value(hs_usage_list_t *lst, const char *service_id) -{ - /* search if there is already an elem with same service_id in list */ - hs_usage_list_elem_t *current = lst->start; - hs_usage_list_elem_t *previous = NULL; - while (current != NULL && strcasecmp(current->service_id,service_id)) { - previous = current; - current = current->next; - } - /* found an element with same service_id? */ - if (current == NULL) { - /* not found! append to end (which could also be the end of a zero-length - * list), don't need to sort (1 is smallest value). */ - /* create elem */ - hs_usage_list_elem_t *e = hs_usage_list_elem_new(); - /* update list attributes (one new elem, one new occurrence) */ - lst->total_count++; - lst->total_service_ids++; - /* copy service id to elem */ - strlcpy(e->service_id,service_id,sizeof(e->service_id)); - /* let either l->start or previously last elem point to new elem */ - if (lst->start == NULL) { - /* this is the first elem */ - lst->start = e; - } else { - /* there were elems in the list before */ - previous->next = e; - } - } else { - /* found! add occurrence to elem and consider resorting */ - /* update list attributes (no new elem, but one new occurrence) */ - lst->total_count++; - /* add occurrence to elem */ - current->count++; - /* is it another than the first list elem? and has previous elem fewer - * count than current? then we need to resort */ - if (previous != NULL && previous->count < current->count) { - /* yes! we need to resort */ - /* remove current elem first */ - previous->next = current->next; - /* can we prepend elem to all other elements? */ - if (lst->start->count <= current->count) { - /* yes! prepend elem */ - current->next = lst->start; - lst->start = current; - } else { - /* no! walk through list a second time and insert at correct place */ - hs_usage_list_elem_t *insert_current = lst->start->next; - hs_usage_list_elem_t *insert_previous = lst->start; - while (insert_current != NULL && - insert_current->count > current->count) { - insert_previous = insert_current; - insert_current = insert_current->next; - } - /* insert here */ - current->next = insert_current; - insert_previous->next = current; - } - } - } -} + /* Reset counters. */ + rep_hist_reset_exit_stats(now); -/** Writes the current service-related observations to the history array and - * clears the observations of the current period. */ -static void -hs_usage_write_service_related_observations_to_history( - hs_usage_current_observation_period_t *p, - hs_usage_service_related_observation_t *h) -{ - /* walk through the first 20 % of list elements and calculate frequency - * distributions */ - /* maximum indices for the three frequencies */ - int five_percent_idx = h->list->total_service_ids/20; - int ten_percent_idx = h->list->total_service_ids/10; - int twenty_percent_idx = h->list->total_service_ids/5; - /* temp values */ - uint32_t five_percent = 0; - uint32_t ten_percent = 0; - uint32_t twenty_percent = 0; - /* walk through list */ - hs_usage_list_elem_t *current = h->list->start; - int i=0; - while (current != NULL && i <= twenty_percent_idx) { - twenty_percent += current->count; - if (i <= ten_percent_idx) - ten_percent += current->count; - if (i <= five_percent_idx) - five_percent += current->count; - current = current->next; - i++; - } - /* copy frequencies */ - h->twenty[p->next_idx] = twenty_percent; - h->ten[p->next_idx] = ten_percent; - h->five[p->next_idx] = five_percent; - /* copy total number of observations */ - h->totals[p->next_idx] = h->list->total_count; - /* free memory of old list */ - hs_usage_list_clear(h->list); -} - -/** Advances to next observation period. */ -static void -hs_usage_advance_current_observation_period(void) -{ - /* aggregate observations to history, including frequency distribution - * arrays */ - hs_usage_write_service_related_observations_to_history( - current_period, publish_total); - hs_usage_write_service_related_observations_to_history( - current_period, publish_novel); - hs_usage_write_service_related_observations_to_history( - current_period, fetch_total); - hs_usage_write_service_related_observations_to_history( - current_period, fetch_successful); - /* write current number of descriptors to descs history */ - descs->totals[current_period->next_idx] = rend_cache_size(); - /* advance to next period */ - current_period->next_idx++; - if (current_period->next_idx == NUM_TOTALS_HS_USAGE) - current_period->next_idx = 0; - if (current_period->num_set < NUM_TOTALS_HS_USAGE) - ++current_period->num_set; - current_period->start_of_current_period=current_period->start_of_next_period; - current_period->start_of_next_period += NUM_SECS_HS_USAGE_SUM_INTERVAL; -} - -/** Checks if the current period is up to date, and if not, advances it. */ -static void -hs_usage_check_if_current_period_is_up_to_date(time_t now) -{ - while (now > current_period->start_of_next_period) { - hs_usage_advance_current_observation_period(); + /* Try to write to disk. */ + statsdir = get_datadir_fname("stats"); + if (check_private_dir(statsdir, CPD_CREATE) < 0) { + log_warn(LD_HIST, "Unable to create stats/ directory!"); + goto done; } -} + filename = get_datadir_fname2("stats", "exit-stats"); + if (write_str_to_file(filename, str, 0) < 0) + log_warn(LD_HIST, "Unable to write exit port statistics to disk!"); -/** Adds a service-related observation, maybe after advancing to next - * observation period. */ -static void -hs_usage_add_service_related_observation( - hs_usage_service_related_observation_t *h, - time_t now, - const char *service_id) -{ - if (now < current_period->start_of_current_period) { - /* don't record old data */ - return; - } - /* check if we are up-to-date */ - hs_usage_check_if_current_period_is_up_to_date(now); - /* add observation */ - hs_usage_insert_value(h->list, service_id); + done: + tor_free(str); + tor_free(statsdir); + tor_free(filename); + return start_of_exit_stats_interval + WRITE_STATS_INTERVAL; } -/** Adds the observation of storing a rendezvous service descriptor to our - * cache in our role as HS authoritative directory. */ +/** Note that we wrote <b>num_written</b> bytes and read <b>num_read</b> + * bytes to/from an exit connection to <b>port</b>. */ void -hs_usage_note_publish_total(const char *service_id, time_t now) +rep_hist_note_exit_bytes(uint16_t port, size_t num_written, + size_t num_read) { - hs_usage_add_service_related_observation(publish_total, now, service_id); + if (!start_of_exit_stats_interval) + return; /* Not initialized. */ + exit_bytes_written[port] += num_written; + exit_bytes_read[port] += num_read; + log_debug(LD_HIST, "Written %lu bytes and read %lu bytes to/from an " + "exit connection to port %d.", + (unsigned long)num_written, (unsigned long)num_read, port); } -/** Adds the observation of storing a novel rendezvous service descriptor to - * our cache in our role as HS authoritative directory. */ +/** Note that we opened an exit stream to <b>port</b>. */ void -hs_usage_note_publish_novel(const char *service_id, time_t now) +rep_hist_note_exit_stream_opened(uint16_t port) { - hs_usage_add_service_related_observation(publish_novel, now, service_id); + if (!start_of_exit_stats_interval) + return; /* Not initialized. */ + exit_streams[port]++; + log_debug(LD_HIST, "Opened exit stream to port %d", port); } -/** Adds the observation of being requested for a rendezvous service descriptor - * in our role as HS authoritative directory. */ +/*** cell statistics ***/ + +/** Start of the current buffer stats interval or 0 if we're not + * collecting buffer statistics. */ +static time_t start_of_buffer_stats_interval; + +/** Initialize buffer stats. */ void -hs_usage_note_fetch_total(const char *service_id, time_t now) +rep_hist_buffer_stats_init(time_t now) { - hs_usage_add_service_related_observation(fetch_total, now, service_id); + start_of_buffer_stats_interval = now; } -/** Adds the observation of being requested for a rendezvous service descriptor - * in our role as HS authoritative directory and being able to answer that - * request successfully. */ +/** Statistics from a single circuit. Collected when the circuit closes, or + * when we flush statistics to disk. */ +typedef struct circ_buffer_stats_t { + /** Average number of cells in the circuit's queue */ + double mean_num_cells_in_queue; + /** Average time a cell waits in the queue. */ + double mean_time_cells_in_queue; + /** Total number of cells sent over this circuit */ + uint32_t processed_cells; +} circ_buffer_stats_t; + +/** List of circ_buffer_stats_t. */ +static smartlist_t *circuits_for_buffer_stats = NULL; + +/** Remember cell statistics for circuit <b>circ</b> at time + * <b>end_of_interval</b> and reset cell counters in case the circuit + * remains open in the next measurement interval. */ void -hs_usage_note_fetch_successful(const char *service_id, time_t now) +rep_hist_buffer_stats_add_circ(circuit_t *circ, time_t end_of_interval) +{ + circ_buffer_stats_t *stat; + time_t start_of_interval; + int interval_length; + or_circuit_t *orcirc; + if (CIRCUIT_IS_ORIGIN(circ)) + return; + orcirc = TO_OR_CIRCUIT(circ); + if (!orcirc->processed_cells) + return; + if (!circuits_for_buffer_stats) + circuits_for_buffer_stats = smartlist_create(); + start_of_interval = circ->timestamp_created.tv_sec > + start_of_buffer_stats_interval ? + circ->timestamp_created.tv_sec : + start_of_buffer_stats_interval; + interval_length = (int) (end_of_interval - start_of_interval); + if (interval_length <= 0) + return; + stat = tor_malloc_zero(sizeof(circ_buffer_stats_t)); + stat->processed_cells = orcirc->processed_cells; + /* 1000.0 for s -> ms; 2.0 because of app-ward and exit-ward queues */ + stat->mean_num_cells_in_queue = interval_length == 0 ? 0.0 : + (double) orcirc->total_cell_waiting_time / + (double) interval_length / 1000.0 / 2.0; + stat->mean_time_cells_in_queue = + (double) orcirc->total_cell_waiting_time / + (double) orcirc->processed_cells; + smartlist_add(circuits_for_buffer_stats, stat); + orcirc->total_cell_waiting_time = 0; + orcirc->processed_cells = 0; +} + +/** Sorting helper: return -1, 1, or 0 based on comparison of two + * circ_buffer_stats_t */ +static int +_buffer_stats_compare_entries(const void **_a, const void **_b) { - hs_usage_add_service_related_observation(fetch_successful, now, service_id); + const circ_buffer_stats_t *a = *_a, *b = *_b; + if (a->processed_cells < b->processed_cells) + return 1; + else if (a->processed_cells > b->processed_cells) + return -1; + else + return 0; } -/** Writes the given circular array to a string. */ -static size_t -hs_usage_format_history(char *buf, size_t len, uint32_t *data) +/** Stop collecting cell stats in a way that we can re-start doing so in + * rep_hist_buffer_stats_init(). */ +void +rep_hist_buffer_stats_term(void) { - char *cp = buf; /* pointer where we are in the buffer */ - int i, n; - if (current_period->num_set <= current_period->next_idx) { - i = 0; /* not been through circular array */ - } else { - i = current_period->next_idx; - } - for (n = 0; n < current_period->num_set; ++n,++i) { - if (i >= NUM_TOTALS_HS_USAGE) - i -= NUM_TOTALS_HS_USAGE; - tor_assert(i < NUM_TOTALS_HS_USAGE); - if (n == (current_period->num_set-1)) - tor_snprintf(cp, len-(cp-buf), "%d", data[i]); - else - tor_snprintf(cp, len-(cp-buf), "%d,", data[i]); - cp += strlen(cp); - } - return cp-buf; + start_of_buffer_stats_interval = 0; + if (!circuits_for_buffer_stats) + circuits_for_buffer_stats = smartlist_create(); + SMARTLIST_FOREACH(circuits_for_buffer_stats, circ_buffer_stats_t *, + stat, tor_free(stat)); + smartlist_clear(circuits_for_buffer_stats); } -/** Writes the complete usage history as hidden service authoritative directory - * to a string. */ -static char * -hs_usage_format_statistics(void) +/** Write buffer statistics to $DATADIR/stats/buffer-stats and return when + * we would next want to write exit stats. */ +time_t +rep_hist_buffer_stats_write(time_t now) { - char *buf, *cp, *s = NULL; - char t[ISO_TIME_LEN+1]; - int r; - uint32_t *data = NULL; - size_t len; - len = (70+20*NUM_TOTALS_HS_USAGE)*11; - buf = tor_malloc_zero(len); - cp = buf; - for (r = 0; r < 11; ++r) { - switch (r) { - case 0: - s = (char*) "publish-total-history"; - data = publish_total->totals; - break; - case 1: - s = (char*) "publish-novel-history"; - data = publish_novel->totals; - break; - case 2: - s = (char*) "publish-top-5-percent-history"; - data = publish_total->five; - break; - case 3: - s = (char*) "publish-top-10-percent-history"; - data = publish_total->ten; - break; - case 4: - s = (char*) "publish-top-20-percent-history"; - data = publish_total->twenty; - break; - case 5: - s = (char*) "fetch-total-history"; - data = fetch_total->totals; - break; - case 6: - s = (char*) "fetch-successful-history"; - data = fetch_successful->totals; - break; - case 7: - s = (char*) "fetch-top-5-percent-history"; - data = fetch_total->five; - break; - case 8: - s = (char*) "fetch-top-10-percent-history"; - data = fetch_total->ten; - break; - case 9: - s = (char*) "fetch-top-20-percent-history"; - data = fetch_total->twenty; - break; - case 10: - s = (char*) "desc-total-history"; - data = descs->totals; - break; - } - format_iso_time(t, current_period->start_of_current_period); - tor_snprintf(cp, len-(cp-buf), "%s %s (%d s) ", s, t, - NUM_SECS_HS_USAGE_SUM_INTERVAL); - cp += strlen(cp); - cp += hs_usage_format_history(cp, len-(cp-buf), data); - strlcat(cp, "\n", len-(cp-buf)); - ++cp; + char *statsdir = NULL, *filename = NULL; + char written[ISO_TIME_LEN+1]; + open_file_t *open_file = NULL; + FILE *out; +#define SHARES 10 + int processed_cells[SHARES], circs_in_share[SHARES], + number_of_circuits, i; + double queued_cells[SHARES], time_in_queue[SHARES]; + smartlist_t *str_build = smartlist_create(); + char *str = NULL, *buf=NULL; + circuit_t *circ; + + if (!start_of_buffer_stats_interval) + return 0; /* Not initialized. */ + if (start_of_buffer_stats_interval + WRITE_STATS_INTERVAL > now) + goto done; /* Not ready to write */ + + /* add current circuits to stats */ + for (circ = _circuit_get_global_list(); circ; circ = circ->next) + rep_hist_buffer_stats_add_circ(circ, now); + /* calculate deciles */ + memset(processed_cells, 0, SHARES * sizeof(int)); + memset(circs_in_share, 0, SHARES * sizeof(int)); + memset(queued_cells, 0, SHARES * sizeof(double)); + memset(time_in_queue, 0, SHARES * sizeof(double)); + if (!circuits_for_buffer_stats) + circuits_for_buffer_stats = smartlist_create(); + smartlist_sort(circuits_for_buffer_stats, + _buffer_stats_compare_entries); + number_of_circuits = smartlist_len(circuits_for_buffer_stats); + if (number_of_circuits < 1) { + log_info(LD_HIST, "Attempt to write cell statistics to disk failed. " + "We haven't seen a single circuit to report about."); + goto done; } - return buf; + i = 0; + SMARTLIST_FOREACH_BEGIN(circuits_for_buffer_stats, + circ_buffer_stats_t *, stat) + { + int share = i++ * SHARES / number_of_circuits; + processed_cells[share] += stat->processed_cells; + queued_cells[share] += stat->mean_num_cells_in_queue; + time_in_queue[share] += stat->mean_time_cells_in_queue; + circs_in_share[share]++; + } + SMARTLIST_FOREACH_END(stat); + /* clear buffer stats history */ + SMARTLIST_FOREACH(circuits_for_buffer_stats, circ_buffer_stats_t *, + stat, tor_free(stat)); + smartlist_clear(circuits_for_buffer_stats); + /* write to file */ + statsdir = get_datadir_fname("stats"); + if (check_private_dir(statsdir, CPD_CREATE) < 0) + goto done; + filename = get_datadir_fname2("stats", "buffer-stats"); + out = start_writing_to_stdio_file(filename, OPEN_FLAGS_APPEND, + 0600, &open_file); + if (!out) + goto done; + format_iso_time(written, now); + if (fprintf(out, "cell-stats-end %s (%d s)\n", written, + (unsigned) (now - start_of_buffer_stats_interval)) < 0) + goto done; + for (i = 0; i < SHARES; i++) { + tor_asprintf(&buf,"%d", !circs_in_share[i] ? 0 : + processed_cells[i] / circs_in_share[i]); + smartlist_add(str_build, buf); + } + str = smartlist_join_strings(str_build, ",", 0, NULL); + if (fprintf(out, "cell-processed-cells %s\n", str) < 0) + goto done; + tor_free(str); + SMARTLIST_FOREACH(str_build, char *, c, tor_free(c)); + smartlist_clear(str_build); + for (i = 0; i < SHARES; i++) { + tor_asprintf(&buf, "%.2f", circs_in_share[i] == 0 ? 0.0 : + queued_cells[i] / (double) circs_in_share[i]); + smartlist_add(str_build, buf); + } + str = smartlist_join_strings(str_build, ",", 0, NULL); + if (fprintf(out, "cell-queued-cells %s\n", str) < 0) + goto done; + tor_free(str); + SMARTLIST_FOREACH(str_build, char *, c, tor_free(c)); + smartlist_clear(str_build); + for (i = 0; i < SHARES; i++) { + tor_asprintf(&buf, "%.0f", circs_in_share[i] == 0 ? 0.0 : + time_in_queue[i] / (double) circs_in_share[i]); + smartlist_add(str_build, buf); + } + str = smartlist_join_strings(str_build, ",", 0, NULL); + if (fprintf(out, "cell-time-in-queue %s\n", str) < 0) + goto done; + tor_free(str); + SMARTLIST_FOREACH(str_build, char *, c, tor_free(c)); + smartlist_free(str_build); + str_build = NULL; + if (fprintf(out, "cell-circuits-per-decile %d\n", + (number_of_circuits + SHARES - 1) / SHARES) < 0) + goto done; + finish_writing_to_file(open_file); + open_file = NULL; + start_of_buffer_stats_interval = now; + done: + if (open_file) + abort_writing_to_file(open_file); + tor_free(filename); + tor_free(statsdir); + if (str_build) { + SMARTLIST_FOREACH(str_build, char *, c, tor_free(c)); + smartlist_free(str_build); + } + tor_free(str); +#undef SHARES + return start_of_buffer_stats_interval + WRITE_STATS_INTERVAL; } -/** Write current statistics about hidden service usage to file. */ +/** Free all storage held by the OR/link history caches, by the + * bandwidth history arrays, by the port history, or by statistics . */ void -hs_usage_write_statistics_to_file(time_t now) +rep_hist_free_all(void) { - char *buf; - size_t len; - char *fname; - or_options_t *options = get_options(); - /* check if we are up-to-date */ - hs_usage_check_if_current_period_is_up_to_date(now); - buf = hs_usage_format_statistics(); - len = strlen(options->DataDirectory) + 16; - fname = tor_malloc(len); - tor_snprintf(fname, len, "%s"PATH_SEPARATOR"hsusage", - options->DataDirectory); - write_str_to_file(fname,buf,0); - tor_free(buf); - tor_free(fname); + digestmap_free(history_map, free_or_history); + tor_free(read_array); + tor_free(write_array); + tor_free(last_stability_doc); + tor_free(exit_bytes_read); + tor_free(exit_bytes_written); + tor_free(exit_streams); + built_last_stability_doc_at = 0; + predicted_ports_free(); + if (circuits_for_buffer_stats) { + SMARTLIST_FOREACH(circuits_for_buffer_stats, circ_buffer_stats_t *, s, + tor_free(s)); + smartlist_free(circuits_for_buffer_stats); + circuits_for_buffer_stats = NULL; + } } diff --git a/src/or/rephist.h b/src/or/rephist.h new file mode 100644 index 0000000000..b06a39ed59 --- /dev/null +++ b/src/or/rephist.h @@ -0,0 +1,82 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file rephist.h + * \brief Header file for rephist.c. + **/ + +#ifndef _TOR_REPHIST_H +#define _TOR_REPHIST_H + +void rep_hist_init(void); +void rep_hist_note_connect_failed(const char* nickname, time_t when); +void rep_hist_note_connect_succeeded(const char* nickname, time_t when); +void rep_hist_note_disconnect(const char* nickname, time_t when); +void rep_hist_note_connection_died(const char* nickname, time_t when); +void rep_hist_note_extend_succeeded(const char *from_name, + const char *to_name); +void rep_hist_note_extend_failed(const char *from_name, const char *to_name); +void rep_hist_dump_stats(time_t now, int severity); +void rep_hist_note_bytes_read(size_t num_bytes, time_t when); +void rep_hist_note_bytes_written(size_t num_bytes, time_t when); + +void rep_hist_note_dir_bytes_read(size_t num_bytes, time_t when); +void rep_hist_note_dir_bytes_written(size_t num_bytes, time_t when); + +int rep_hist_bandwidth_assess(void); +char *rep_hist_get_bandwidth_lines(void); +void rep_hist_update_state(or_state_t *state); +int rep_hist_load_state(or_state_t *state, char **err); +void rep_history_clean(time_t before); + +void rep_hist_note_router_reachable(const char *id, const tor_addr_t *at_addr, + uint16_t at_port, time_t when); +void rep_hist_note_router_unreachable(const char *id, time_t when); +int rep_hist_record_mtbf_data(time_t now, int missing_means_down); +int rep_hist_load_mtbf_data(time_t now); + +time_t rep_hist_downrate_old_runs(time_t now); +long rep_hist_get_uptime(const char *id, time_t when); +double rep_hist_get_stability(const char *id, time_t when); +double rep_hist_get_weighted_fractional_uptime(const char *id, time_t when); +long rep_hist_get_weighted_time_known(const char *id, time_t when); +int rep_hist_have_measured_enough_stability(void); +const char *rep_hist_get_router_stability_doc(time_t now); + +void rep_hist_note_used_port(time_t now, uint16_t port); +smartlist_t *rep_hist_get_predicted_ports(time_t now); +void rep_hist_note_used_resolve(time_t now); +void rep_hist_note_used_internal(time_t now, int need_uptime, + int need_capacity); +int rep_hist_get_predicted_internal(time_t now, int *need_uptime, + int *need_capacity); + +int any_predicted_circuits(time_t now); +int rep_hist_circbuilding_dormant(time_t now); + +void note_crypto_pk_op(pk_op_t operation); +void dump_pk_ops(int severity); + +void rep_hist_free_all(void); + +void rep_hist_exit_stats_init(time_t now); +void rep_hist_reset_exit_stats(time_t now); +void rep_hist_exit_stats_term(void); +char *rep_hist_format_exit_stats(time_t now); +time_t rep_hist_exit_stats_write(time_t now); +void rep_hist_note_exit_bytes(uint16_t port, size_t num_written, + size_t num_read); +void rep_hist_note_exit_stream_opened(uint16_t port); + +void rep_hist_buffer_stats_init(time_t now); +void rep_hist_buffer_stats_add_circ(circuit_t *circ, + time_t end_of_interval); +time_t rep_hist_buffer_stats_write(time_t now); +void rep_hist_buffer_stats_term(void); + +#endif + diff --git a/src/or/router.c b/src/or/router.c index cc600415f0..a7148ea1f7 100644 --- a/src/or/router.c +++ b/src/or/router.c @@ -7,6 +7,24 @@ #define ROUTER_PRIVATE #include "or.h" +#include "circuitlist.h" +#include "circuituse.h" +#include "config.h" +#include "connection.h" +#include "control.h" +#include "directory.h" +#include "dirserv.h" +#include "dns.h" +#include "geoip.h" +#include "hibernate.h" +#include "main.h" +#include "networkstatus.h" +#include "policies.h" +#include "relay.h" +#include "rephist.h" +#include "router.h" +#include "routerlist.h" +#include "routerparse.h" /** * \file router.c @@ -31,11 +49,15 @@ static crypto_pk_env_t *onionkey=NULL; /** Previous private onionskin decryption key: used to decode CREATE cells * generated by clients that have an older version of our descriptor. */ static crypto_pk_env_t *lastonionkey=NULL; -/** Private "identity key": used to sign directory info and TLS +/** Private server "identity key": used to sign directory info and TLS * certificates. Never changes. */ -static crypto_pk_env_t *identitykey=NULL; -/** Digest of identitykey. */ -static char identitykey_digest[DIGEST_LEN]; +static crypto_pk_env_t *server_identitykey=NULL; +/** Digest of server_identitykey. */ +static char server_identitykey_digest[DIGEST_LEN]; +/** Private client "identity key": used to sign bridges' and clients' + * outbound TLS certificates. Regenerated on startup and on IP address + * change. */ +static crypto_pk_env_t *client_identitykey=NULL; /** Signing key used for v3 directory material; only set for authorities. */ static crypto_pk_env_t *authority_signing_key = NULL; /** Key certificate to authenticate v3 directory material; only set for @@ -61,8 +83,7 @@ static void set_onion_key(crypto_pk_env_t *k) { tor_mutex_acquire(key_lock); - if (onionkey) - crypto_free_pk_env(onionkey); + crypto_free_pk_env(onionkey); onionkey = k; onionkey_set_at = time(NULL); tor_mutex_release(key_lock); @@ -106,32 +127,78 @@ get_onion_key_set_at(void) return onionkey_set_at; } -/** Set the current identity key to k. +/** Set the current server identity key to <b>k</b>. */ void -set_identity_key(crypto_pk_env_t *k) +set_server_identity_key(crypto_pk_env_t *k) { - if (identitykey) - crypto_free_pk_env(identitykey); - identitykey = k; - crypto_pk_get_digest(identitykey, identitykey_digest); + crypto_free_pk_env(server_identitykey); + server_identitykey = k; + crypto_pk_get_digest(server_identitykey, server_identitykey_digest); } -/** Returns the current identity key; requires that the identity key has been - * set. +/** Make sure that we have set up our identity keys to match or not match as + * appropriate, and die with an assertion if we have not. */ +static void +assert_identity_keys_ok(void) +{ + tor_assert(client_identitykey); + if (public_server_mode(get_options())) { + /* assert that we have set the client and server keys to be equal */ + tor_assert(server_identitykey); + tor_assert(0==crypto_pk_cmp_keys(client_identitykey, server_identitykey)); + } else { + /* assert that we have set the client and server keys to be unequal */ + if (server_identitykey) + tor_assert(0!=crypto_pk_cmp_keys(client_identitykey, + server_identitykey)); + } +} + +/** Returns the current server identity key; requires that the key has + * been set, and that we are running as a Tor server. */ crypto_pk_env_t * -get_identity_key(void) +get_server_identity_key(void) { - tor_assert(identitykey); - return identitykey; + tor_assert(server_identitykey); + tor_assert(server_mode(get_options())); + assert_identity_keys_ok(); + return server_identitykey; } -/** Return true iff the identity key has been set. */ +/** Return true iff the server identity key has been set. */ int -identity_key_is_set(void) +server_identity_key_is_set(void) +{ + return server_identitykey != NULL; +} + +/** Set the current client identity key to <b>k</b>. + */ +void +set_client_identity_key(crypto_pk_env_t *k) +{ + crypto_free_pk_env(client_identitykey); + client_identitykey = k; +} + +/** Returns the current client identity key for use on outgoing TLS + * connections; requires that the key has been set. + */ +crypto_pk_env_t * +get_tlsclient_identity_key(void) { - return identitykey != NULL; + tor_assert(client_identitykey); + assert_identity_keys_ok(); + return client_identitykey; +} + +/** Return true iff the client identity key has been set. */ +int +client_identity_key_is_set(void) +{ + return client_identitykey != NULL; } /** Return the key certificate for this v3 (voting) authority, or NULL @@ -201,8 +268,7 @@ rotate_onion_key(void) } log_info(LD_GENERAL, "Rotating onion key"); tor_mutex_acquire(key_lock); - if (lastonionkey) - crypto_free_pk_env(lastonionkey); + crypto_free_pk_env(lastonionkey); lastonionkey = onionkey; onionkey = prkey; now = time(NULL); @@ -331,10 +397,9 @@ load_authority_keyset(int legacy, crypto_pk_env_t **key_out, goto done; } - if (*key_out) - crypto_free_pk_env(*key_out); - if (*cert_out) - authority_cert_free(*cert_out); + crypto_free_pk_env(*key_out); + authority_cert_free(*cert_out); + *key_out = signing_key; *cert_out = parsed; r = 0; @@ -344,10 +409,8 @@ load_authority_keyset(int legacy, crypto_pk_env_t **key_out, done: tor_free(fname); tor_free(cert); - if (signing_key) - crypto_free_pk_env(signing_key); - if (parsed) - authority_cert_free(parsed); + crypto_free_pk_env(signing_key); + authority_cert_free(parsed); return r; } @@ -442,7 +505,9 @@ init_keys(void) key_lock = tor_mutex_new(); /* There are a couple of paths that put us here before */ - if (crypto_global_init(get_options()->HardwareAccel)) { + if (crypto_global_init(get_options()->HardwareAccel, + get_options()->AccelName, + get_options()->AccelDir)) { log_err(LD_BUG, "Unable to initialize OpenSSL. Exiting."); return -1; } @@ -456,9 +521,12 @@ init_keys(void) crypto_free_pk_env(prkey); return -1; } - set_identity_key(prkey); - /* Create a TLS context; default the client nickname to "client". */ - if (tor_tls_context_new(get_identity_key(), MAX_SSL_KEY_LIFETIME) < 0) { + set_client_identity_key(prkey); + /* Create a TLS context. */ + if (tor_tls_context_init(0, + get_tlsclient_identity_key(), + NULL, + MAX_SSL_KEY_LIFETIME) < 0) { log_err(LD_GENERAL,"Error creating TLS context for Tor client."); return -1; } @@ -493,13 +561,28 @@ init_keys(void) } } - /* 1. Read identity key. Make it if none is found. */ + /* 1b. Read identity key. Make it if none is found. */ keydir = get_datadir_fname2("keys", "secret_id_key"); log_info(LD_GENERAL,"Reading/making identity key \"%s\"...",keydir); prkey = init_key_from_file(keydir, 1, LOG_ERR); tor_free(keydir); if (!prkey) return -1; - set_identity_key(prkey); + set_server_identity_key(prkey); + + /* 1c. If we are configured as a bridge, generate a client key; + * otherwise, set the server identity key as our client identity + * key. */ + if (public_server_mode(options)) { + set_client_identity_key(crypto_pk_dup_key(prkey)); /* set above */ + } else { + if (!(prkey = crypto_new_pk_env())) + return -1; + if (crypto_pk_generate_key(prkey)) { + crypto_free_pk_env(prkey); + return -1; + } + set_client_identity_key(prkey); + } /* 2. Read onion key. Make it if none is found. */ keydir = get_datadir_fname2("keys", "secret_onion_key"); @@ -536,18 +619,22 @@ init_keys(void) tor_free(keydir); /* 3. Initialize link key and TLS context. */ - if (tor_tls_context_new(get_identity_key(), MAX_SSL_KEY_LIFETIME) < 0) { + if (tor_tls_context_init(public_server_mode(options), + get_tlsclient_identity_key(), + get_server_identity_key(), + MAX_SSL_KEY_LIFETIME) < 0) { log_err(LD_GENERAL,"Error initializing TLS context"); return -1; } /* 4. Build our router descriptor. */ /* Must be called after keys are initialized. */ mydesc = router_get_my_descriptor(); - if (authdir_mode(options)) { + if (authdir_mode_handles_descs(options, ROUTER_PURPOSE_GENERAL)) { const char *m = NULL; routerinfo_t *ri; /* We need to add our own fingerprint so it gets recognized. */ - if (dirserv_add_own_fingerprint(options->Nickname, get_identity_key())) { + if (dirserv_add_own_fingerprint(options->Nickname, + get_server_identity_key())) { log_err(LD_GENERAL,"Error adding own fingerprint to approved set"); return -1; } @@ -568,7 +655,8 @@ init_keys(void) /* 5. Dump fingerprint to 'fingerprint' */ keydir = get_datadir_fname("fingerprint"); log_info(LD_GENERAL,"Dumping fingerprint to \"%s\"...",keydir); - if (crypto_pk_get_fingerprint(get_identity_key(), fingerprint, 0)<0) { + if (crypto_pk_get_fingerprint(get_server_identity_key(), + fingerprint, 0) < 0) { log_err(LD_GENERAL,"Error computing fingerprint"); tor_free(keydir); return -1; @@ -606,7 +694,7 @@ init_keys(void) return -1; } /* 6b. [authdirserver only] add own key to approved directories. */ - crypto_pk_get_digest(get_identity_key(), digest); + crypto_pk_get_digest(get_server_identity_key(), digest); type = ((options->V1AuthoritativeDir ? V1_AUTHORITY : NO_AUTHORITY) | (options->V2AuthoritativeDir ? V2_AUTHORITY : NO_AUTHORITY) | (options->V3AuthoritativeDir ? V3_AUTHORITY : NO_AUTHORITY) | @@ -762,9 +850,29 @@ consider_testing_reachability(int test_or, int test_dir) routerinfo_t *me = router_get_my_routerinfo(); int orport_reachable = check_whether_orport_reachable(); tor_addr_t addr; + or_options_t *options = get_options(); if (!me) return; + if (routerset_contains_router(options->ExcludeNodes, me) && + options->StrictNodes) { + /* If we've excluded ourself, and StrictNodes is set, we can't test + * ourself. */ + if (test_or || test_dir) { +#define SELF_EXCLUDED_WARN_INTERVAL 3600 + static ratelim_t warning_limit=RATELIM_INIT(SELF_EXCLUDED_WARN_INTERVAL); + char *msg; + if ((msg = rate_limit_log(&warning_limit, approx_time()))) { + log_warn(LD_CIRC, "Can't peform self-tests for this relay: we have " + "listed ourself in ExcludeNodes, and StrictNodes is set. " + "We cannot learn whether we are usable, and will not " + "be able to advertise ourself.%s", msg); + tor_free(msg); + } + } + return; + } + if (test_or && (!orport_reachable || !circuit_enough_testing_circs())) { log_info(LD_CIRC, "Testing %s of my ORPort: %s:%d.", !orport_reachable ? "reachability" : "bandwidth", @@ -952,6 +1060,28 @@ server_mode(or_options_t *options) return (options->ORPort != 0 || options->ORListenAddress); } +/** Return true iff we are trying to be a non-bridge server. + */ +int +public_server_mode(or_options_t *options) +{ + if (!server_mode(options)) return 0; + return (!options->BridgeRelay); +} + +/** Return true iff the combination of options in <b>options</b> and parameters + * in the consensus mean that we don't want to allow exits from circuits + * we got from addresses not known to be servers. */ +int +should_refuse_unknown_exits(or_options_t *options) +{ + if (options->RefuseUnknownExits_ != -1) { + return options->RefuseUnknownExits_; + } else { + return networkstatus_get_param(NULL, "refuseunknownexits", 1, 0, 1); + } +} + /** Remember if we've advertised ourselves to the dirservers. */ static int server_is_advertised=0; @@ -977,10 +1107,10 @@ set_server_advertised(int s) int proxy_mode(or_options_t *options) { - return (options->SocksPort != 0 || options->SocksListenAddress || - options->TransPort != 0 || options->TransListenAddress || - options->NatdPort != 0 || options->NatdListenAddress || - options->DNSPort != 0 || options->DNSListenAddress); + return (options->SocksPort != 0 || + options->TransPort != 0 || + options->NATDPort != 0 || + options->DNSPort != 0); } /** Decide if we're a publishable server. We are a publishable server if: @@ -1114,12 +1244,24 @@ router_compare_to_my_exit_policy(edge_connection_t *conn) desc_routerinfo->exit_policy) != ADDR_POLICY_ACCEPTED; } +/** Return true iff my exit policy is reject *:*. Return -1 if we don't + * have a descriptor */ +int +router_my_exit_policy_is_reject_star(void) +{ + if (!router_get_my_routerinfo()) /* make sure desc_routerinfo exists */ + return -1; + + return desc_routerinfo->policy_is_reject_star; +} + /** Return true iff I'm a server and <b>digest</b> is equal to - * my identity digest. */ + * my server identity key digest. */ int router_digest_is_me(const char *digest) { - return identitykey && tor_memeq(identitykey_digest, digest, DIGEST_LEN); + return (server_identitykey && + tor_memeq(server_identitykey_digest, digest, DIGEST_LEN)); } /** Return true iff I'm a server and <b>digest</b> is equal to @@ -1209,6 +1351,8 @@ static int router_guess_address_from_dir_headers(uint32_t *guess); int router_pick_published_address(or_options_t *options, uint32_t *addr) { + char buf[INET_NTOA_BUF_LEN]; + struct in_addr a; if (resolve_my_address(LOG_INFO, options, addr, NULL) < 0) { log_info(LD_CONFIG, "Could not determine our address locally. " "Checking if directory headers provide any hints."); @@ -1218,6 +1362,9 @@ router_pick_published_address(or_options_t *options, uint32_t *addr) return -1; } } + a.s_addr = htonl(*addr); + tor_inet_ntoa(&a, buf, sizeof(buf)); + log_info(LD_CONFIG,"Success: chose address '%s'.", buf); return 0; } @@ -1240,7 +1387,7 @@ router_rebuild_descriptor(int force) if (router_pick_published_address(options, &addr) < 0) { /* Stop trying to rebuild our descriptor every second. We'll - * learn that it's time to try again when server_has_changed_ip() + * learn that it's time to try again when ip_address_changed() * marks it dirty. */ desc_clean_since = time(NULL); return -1; @@ -1256,7 +1403,7 @@ router_rebuild_descriptor(int force) ri->cache_info.published_on = time(NULL); ri->onion_pkey = crypto_pk_dup_key(get_onion_key()); /* must invoke from * main thread */ - ri->identity_pkey = crypto_pk_dup_key(get_identity_key()); + ri->identity_pkey = crypto_pk_dup_key(get_server_identity_key()); if (crypto_pk_get_digest(ri->identity_pkey, ri->cache_info.identity_digest)<0) { routerinfo_free(ri); @@ -1273,9 +1420,16 @@ router_rebuild_descriptor(int force) ri->bandwidthcapacity = hibernating ? 0 : rep_hist_bandwidth_assess(); - policies_parse_exit_policy(options->ExitPolicy, &ri->exit_policy, - options->ExitPolicyRejectPrivate, - ri->address); + if (dns_seems_to_be_broken() || has_dns_init_failed()) { + /* DNS is screwed up; don't claim to be an exit. */ + policies_exit_policy_append_reject_star(&ri->exit_policy); + } else { + policies_parse_exit_policy(options->ExitPolicy, &ri->exit_policy, + options->ExitPolicyRejectPrivate, + ri->address, !options->BridgeRelay); + } + ri->policy_is_reject_star = + policy_is_reject_star(ri->exit_policy); if (desc_routerinfo) { /* inherit values */ ri->is_valid = desc_routerinfo->is_valid; @@ -1346,26 +1500,30 @@ router_rebuild_descriptor(int force) ei->cache_info.published_on = ri->cache_info.published_on; memcpy(ei->cache_info.identity_digest, ri->cache_info.identity_digest, DIGEST_LEN); - ei->cache_info.signed_descriptor_body = tor_malloc(8192); - if (extrainfo_dump_to_string(ei->cache_info.signed_descriptor_body, 8192, - ei, get_identity_key()) < 0) { + if (extrainfo_dump_to_string(&ei->cache_info.signed_descriptor_body, + ei, get_server_identity_key()) < 0) { log_warn(LD_BUG, "Couldn't generate extra-info descriptor."); - routerinfo_free(ri); extrainfo_free(ei); - return -1; + ei = NULL; + } else { + ei->cache_info.signed_descriptor_len = + strlen(ei->cache_info.signed_descriptor_body); + router_get_extrainfo_hash(ei->cache_info.signed_descriptor_body, + ei->cache_info.signed_descriptor_digest); } - ei->cache_info.signed_descriptor_len = - strlen(ei->cache_info.signed_descriptor_body); - router_get_extrainfo_hash(ei->cache_info.signed_descriptor_body, - ei->cache_info.signed_descriptor_digest); /* Now finish the router descriptor. */ - memcpy(ri->cache_info.extra_info_digest, - ei->cache_info.signed_descriptor_digest, - DIGEST_LEN); + if (ei) { + memcpy(ri->cache_info.extra_info_digest, + ei->cache_info.signed_descriptor_digest, + DIGEST_LEN); + } else { + /* ri was allocated with tor_malloc_zero, so there is no need to + * zero ri->cache_info.extra_info_digest here. */ + } ri->cache_info.signed_descriptor_body = tor_malloc(8192); if (router_dump_router_to_string(ri->cache_info.signed_descriptor_body, 8192, - ri, get_identity_key())<0) { + ri, get_server_identity_key()) < 0) { log_warn(LD_BUG, "Couldn't generate router descriptor."); routerinfo_free(ri); extrainfo_free(ei); @@ -1380,7 +1538,7 @@ router_rebuild_descriptor(int force) /* Let bridges serve their own descriptors unencrypted, so they can * pass reachability testing. (If they want to be harder to notice, * they can always leave the DirPort off). */ - if (!options->BridgeRelay) + if (ei && !options->BridgeRelay) ei->cache_info.send_unencrypted = 1; router_get_router_hash(ri->cache_info.signed_descriptor_body, @@ -1389,13 +1547,13 @@ router_rebuild_descriptor(int force) routerinfo_set_country(ri); - tor_assert(! routerinfo_incompatible_with_extrainfo(ri, ei, NULL, NULL)); + if (ei) { + tor_assert(! routerinfo_incompatible_with_extrainfo(ri, ei, NULL, NULL)); + } - if (desc_routerinfo) - routerinfo_free(desc_routerinfo); + routerinfo_free(desc_routerinfo); desc_routerinfo = ri; - if (desc_extrainfo) - extrainfo_free(desc_extrainfo); + extrainfo_free(desc_extrainfo); desc_extrainfo = ei; desc_clean_since = time(NULL); @@ -1606,6 +1764,7 @@ router_dump_router_to_string(char *s, size_t maxlen, routerinfo_t *router, char digest[DIGEST_LEN]; char published[ISO_TIME_LEN+1]; char fingerprint[FINGERPRINT_LEN+1]; + int has_extra_info_digest; char extra_info_digest[HEX_DIGEST_LEN+1]; size_t onion_pkeylen, identity_pkeylen; size_t written; @@ -1634,7 +1793,7 @@ router_dump_router_to_string(char *s, size_t maxlen, routerinfo_t *router, return -1; } - /* PEM-encode the identity key key */ + /* PEM-encode the identity key */ if (crypto_pk_write_public_key_to_string(router->identity_pkey, &identity_pkey,&identity_pkeylen)<0) { log_warn(LD_BUG,"write identity_pkey to string failed!"); @@ -1656,8 +1815,13 @@ router_dump_router_to_string(char *s, size_t maxlen, routerinfo_t *router, family_line = tor_strdup(""); } - base16_encode(extra_info_digest, sizeof(extra_info_digest), - router->cache_info.extra_info_digest, DIGEST_LEN); + has_extra_info_digest = + ! tor_digest_is_zero(router->cache_info.extra_info_digest); + + if (has_extra_info_digest) { + base16_encode(extra_info_digest, sizeof(extra_info_digest), + router->cache_info.extra_info_digest, DIGEST_LEN); + } /* Generate the easy portion of the router descriptor. */ result = tor_snprintf(s, maxlen, @@ -1668,7 +1832,7 @@ router_dump_router_to_string(char *s, size_t maxlen, routerinfo_t *router, "opt fingerprint %s\n" "uptime %ld\n" "bandwidth %d %d %d\n" - "opt extra-info-digest %s\n%s" + "%s%s%s%s" "onion-key\n%s" "signing-key\n%s" "%s%s%s%s", @@ -1683,7 +1847,9 @@ router_dump_router_to_string(char *s, size_t maxlen, routerinfo_t *router, (int) router->bandwidthrate, (int) router->bandwidthburst, (int) router->bandwidthcapacity, - extra_info_digest, + has_extra_info_digest ? "opt extra-info-digest " : "", + has_extra_info_digest ? extra_info_digest : "", + has_extra_info_digest ? "\n" : "", options->DownloadExtraInfo ? "opt caches-extra-info\n" : "", onion_pkey, identity_pkey, family_line, @@ -1715,9 +1881,7 @@ router_dump_router_to_string(char *s, size_t maxlen, routerinfo_t *router, } /* Write the exit policy to the end of 's'. */ - if (dns_seems_to_be_broken() || has_dns_init_failed() || - !router->exit_policy || !smartlist_len(router->exit_policy)) { - /* DNS is screwed up; don't claim to be an exit. */ + if (!router->exit_policy || !smartlist_len(router->exit_policy)) { strlcat(s+written, "reject *:*\n", maxlen-written); written += strlen("reject *:*\n"); tmpe = NULL; @@ -1740,7 +1904,8 @@ router_dump_router_to_string(char *s, size_t maxlen, routerinfo_t *router, } } - if (written+256 > maxlen) { /* Not enough room for signature. */ + if (written + DIROBJ_MAX_SIG_LEN > maxlen) { + /* Not enough room for signature. */ log_warn(LD_BUG,"not enough room left in descriptor for signature!"); return -1; } @@ -1755,7 +1920,7 @@ router_dump_router_to_string(char *s, size_t maxlen, routerinfo_t *router, note_crypto_pk_op(SIGN_RTR); if (router_append_dirobj_signature(s+written,maxlen-written, - digest,ident_key)<0) { + digest,DIGEST_LEN,ident_key)<0) { log_warn(LD_BUG, "Couldn't sign router descriptor"); return -1; } @@ -1790,11 +1955,62 @@ router_dump_router_to_string(char *s, size_t maxlen, routerinfo_t *router, return (int)written+1; } -/** Write the contents of <b>extrainfo</b> to the <b>maxlen</b>-byte string - * <b>s</b>, signing them with <b>ident_key</b>. Return 0 on success, - * negative on failure. */ +/** Load the contents of <b>filename</b>, find the last line starting with + * <b>end_line</b>, ensure that its timestamp is not more than 25 hours in + * the past or more than 1 hour in the future with respect to <b>now</b>, + * and write the file contents starting with that line to *<b>out</b>. + * Return 1 for success, 0 if the file does not exist, or -1 if the file + * does not contain a line matching these criteria or other failure. */ +static int +load_stats_file(const char *filename, const char *end_line, time_t now, + char **out) +{ + int r = -1; + char *fname = get_datadir_fname(filename); + char *contents, *start = NULL, *tmp, timestr[ISO_TIME_LEN+1]; + time_t written; + switch (file_status(fname)) { + case FN_FILE: + /* X022 Find an alternative to reading the whole file to memory. */ + if ((contents = read_file_to_str(fname, 0, NULL))) { + tmp = strstr(contents, end_line); + /* Find last block starting with end_line */ + while (tmp) { + start = tmp; + tmp = strstr(tmp + 1, end_line); + } + if (!start) + goto notfound; + if (strlen(start) < strlen(end_line) + 1 + sizeof(timestr)) + goto notfound; + strlcpy(timestr, start + 1 + strlen(end_line), sizeof(timestr)); + if (parse_iso_time(timestr, &written) < 0) + goto notfound; + if (written < now - (25*60*60) || written > now + (1*60*60)) + goto notfound; + *out = tor_strdup(start); + r = 1; + } + notfound: + tor_free(contents); + break; + case FN_NOENT: + r = 0; + break; + case FN_ERROR: + case FN_DIR: + default: + break; + } + tor_free(fname); + return r; +} + +/** Write the contents of <b>extrainfo</b> and aggregated statistics to + * *<b>s_out</b>, signing them with <b>ident_key</b>. Return 0 on + * success, negative on failure. */ int -extrainfo_dump_to_string(char *s, size_t maxlen, extrainfo_t *extrainfo, +extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo, crypto_pk_env_t *ident_key) { or_options_t *options = get_options(); @@ -1803,87 +2019,128 @@ extrainfo_dump_to_string(char *s, size_t maxlen, extrainfo_t *extrainfo, char digest[DIGEST_LEN]; char *bandwidth_usage; int result; - size_t len; + static int write_stats_to_extrainfo = 1; + char sig[DIROBJ_MAX_SIG_LEN+1]; + char *s, *pre, *contents, *cp, *s_dup = NULL; + time_t now = time(NULL); + smartlist_t *chunks = smartlist_create(); + extrainfo_t *ei_tmp = NULL; base16_encode(identity, sizeof(identity), extrainfo->cache_info.identity_digest, DIGEST_LEN); format_iso_time(published, extrainfo->cache_info.published_on); - bandwidth_usage = rep_hist_get_bandwidth_lines(1); + bandwidth_usage = rep_hist_get_bandwidth_lines(); - result = tor_snprintf(s, maxlen, - "extra-info %s %s\n" - "published %s\n%s", - extrainfo->nickname, identity, - published, bandwidth_usage); + tor_asprintf(&pre, "extra-info %s %s\npublished %s\n%s", + extrainfo->nickname, identity, + published, bandwidth_usage); tor_free(bandwidth_usage); - if (result<0) - return -1; + smartlist_add(chunks, pre); + + if (options->ExtraInfoStatistics && write_stats_to_extrainfo) { + log_info(LD_GENERAL, "Adding stats to extra-info descriptor."); + if (options->DirReqStatistics && + load_stats_file("stats"PATH_SEPARATOR"dirreq-stats", + "dirreq-stats-end", now, &contents) > 0) { + smartlist_add(chunks, contents); + } + if (options->EntryStatistics && + load_stats_file("stats"PATH_SEPARATOR"entry-stats", + "entry-stats-end", now, &contents) > 0) { + smartlist_add(chunks, contents); + } + if (options->CellStatistics && + load_stats_file("stats"PATH_SEPARATOR"buffer-stats", + "cell-stats-end", now, &contents) > 0) { + smartlist_add(chunks, contents); + } + if (options->ExitPortStatistics && + load_stats_file("stats"PATH_SEPARATOR"exit-stats", + "exit-stats-end", now, &contents) > 0) { + smartlist_add(chunks, contents); + } + } - if (should_record_bridge_info(options)) { - char *geoip_summary = extrainfo_get_client_geoip_summary(time(NULL)); - if (geoip_summary) { - char geoip_start[ISO_TIME_LEN+1]; - format_iso_time(geoip_start, geoip_get_history_start()); - result = tor_snprintf(s+strlen(s), maxlen-strlen(s), - "geoip-start-time %s\n" - "geoip-client-origins %s\n", - geoip_start, geoip_summary); - control_event_clients_seen(geoip_start, geoip_summary); - tor_free(geoip_summary); - if (result<0) - return -1; + if (should_record_bridge_info(options) && write_stats_to_extrainfo) { + const char *bridge_stats = geoip_get_bridge_stats_extrainfo(now); + if (bridge_stats) { + smartlist_add(chunks, tor_strdup(bridge_stats)); } } - len = strlen(s); - strlcat(s+len, "router-signature\n", maxlen-len); - len += strlen(s+len); - if (router_get_extrainfo_hash(s, digest)<0) - return -1; - if (router_append_dirobj_signature(s+len, maxlen-len, digest, ident_key)<0) - return -1; + smartlist_add(chunks, tor_strdup("router-signature\n")); + s = smartlist_join_strings(chunks, "", 0, NULL); + + while (strlen(s) > MAX_EXTRAINFO_UPLOAD_SIZE - DIROBJ_MAX_SIG_LEN) { + /* So long as there are at least two chunks (one for the initial + * extra-info line and one for the router-signature), we can keep removing + * things. */ + if (smartlist_len(chunks) > 2) { + /* We remove the next-to-last element (remember, len-1 is the last + element), since we need to keep the router-signature element. */ + int idx = smartlist_len(chunks) - 2; + char *e = smartlist_get(chunks, idx); + smartlist_del_keeporder(chunks, idx); + log_warn(LD_GENERAL, "We just generated an extra-info descriptor " + "with statistics that exceeds the 50 KB " + "upload limit. Removing last added " + "statistics."); + tor_free(e); + tor_free(s); + s = smartlist_join_strings(chunks, "", 0, NULL); + } else { + log_warn(LD_BUG, "We just generated an extra-info descriptors that " + "exceeds the 50 KB upload limit."); + goto err; + } + } -#ifdef DEBUG_ROUTER_DUMP_ROUTER_TO_STRING - { - char *cp, *s_dup; - extrainfo_t *ei_tmp; - cp = s_dup = tor_strdup(s); - ei_tmp = extrainfo_parse_entry_from_string(cp, NULL, 1, NULL); - if (!ei_tmp) { - log_err(LD_BUG, - "We just generated an extrainfo descriptor we can't parse."); - log_err(LD_BUG, "Descriptor was: <<%s>>", s); - tor_free(s_dup); - return -1; + memset(sig, 0, sizeof(sig)); + if (router_get_extrainfo_hash(s, digest) < 0 || + router_append_dirobj_signature(sig, sizeof(sig), digest, DIGEST_LEN, + ident_key) < 0) { + log_warn(LD_BUG, "Could not append signature to extra-info " + "descriptor."); + goto err; + } + smartlist_add(chunks, tor_strdup(sig)); + tor_free(s); + s = smartlist_join_strings(chunks, "", 0, NULL); + + cp = s_dup = tor_strdup(s); + ei_tmp = extrainfo_parse_entry_from_string(cp, NULL, 1, NULL); + if (!ei_tmp) { + if (write_stats_to_extrainfo) { + log_warn(LD_GENERAL, "We just generated an extra-info descriptor " + "with statistics that we can't parse. Not " + "adding statistics to this or any future " + "extra-info descriptors."); + write_stats_to_extrainfo = 0; + result = extrainfo_dump_to_string(s_out, extrainfo, ident_key); + goto done; + } else { + log_warn(LD_BUG, "We just generated an extrainfo descriptor we " + "can't parse."); + goto err; } - tor_free(s_dup); - extrainfo_free(ei_tmp); } -#endif - return (int)strlen(s)+1; -} + *s_out = s; + s = NULL; /* prevent free */ + result = 0; + goto done; -/** Wrapper function for geoip_get_client_history(). It first discards - * any items in the client history that are too old -- it dumps anything - * more than 48 hours old, but it only considers whether to dump at most - * once per 48 hours, so we aren't too precise to an observer (see also - * r14780). - */ -char * -extrainfo_get_client_geoip_summary(time_t now) -{ - static time_t last_purged_at = 0; - int geoip_purge_interval = 48*60*60; -#ifdef ENABLE_GEOIP_STATS - if (get_options()->DirRecordUsageByCountry) - geoip_purge_interval = get_options()->DirRecordUsageRetainIPs; -#endif - if (now > last_purged_at+geoip_purge_interval) { - geoip_remove_old_clients(now-geoip_purge_interval); - last_purged_at = now; - } - return geoip_get_client_history(now, GEOIP_CLIENT_CONNECT); + err: + result = -1; + + done: + tor_free(s); + SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp)); + smartlist_free(chunks); + tor_free(s_dup); + extrainfo_free(ei_tmp); + + return result; } /** Return true iff <b>s</b> is a legally valid server nickname. */ @@ -2010,26 +2267,17 @@ router_purpose_from_string(const char *s) void router_free_all(void) { - if (onionkey) - crypto_free_pk_env(onionkey); - if (lastonionkey) - crypto_free_pk_env(lastonionkey); - if (identitykey) - crypto_free_pk_env(identitykey); - if (key_lock) - tor_mutex_free(key_lock); - if (desc_routerinfo) - routerinfo_free(desc_routerinfo); - if (desc_extrainfo) - extrainfo_free(desc_extrainfo); - if (authority_signing_key) - crypto_free_pk_env(authority_signing_key); - if (authority_key_certificate) - authority_cert_free(authority_key_certificate); - if (legacy_signing_key) - crypto_free_pk_env(legacy_signing_key); - if (legacy_key_certificate) - authority_cert_free(legacy_key_certificate); + crypto_free_pk_env(onionkey); + crypto_free_pk_env(lastonionkey); + crypto_free_pk_env(server_identitykey); + crypto_free_pk_env(client_identitykey); + tor_mutex_free(key_lock); + routerinfo_free(desc_routerinfo); + extrainfo_free(desc_extrainfo); + crypto_free_pk_env(authority_signing_key); + authority_cert_free(authority_key_certificate); + crypto_free_pk_env(legacy_signing_key); + authority_cert_free(legacy_key_certificate); if (warned_nonexistent_family) { SMARTLIST_FOREACH(warned_nonexistent_family, char *, cp, tor_free(cp)); diff --git a/src/or/router.h b/src/or/router.h new file mode 100644 index 0000000000..5e021f6fed --- /dev/null +++ b/src/or/router.h @@ -0,0 +1,101 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file router.h + * \brief Header file for router.c. + **/ + +#ifndef _TOR_ROUTER_H +#define _TOR_ROUTER_H + +crypto_pk_env_t *get_onion_key(void); +time_t get_onion_key_set_at(void); +void set_server_identity_key(crypto_pk_env_t *k); +crypto_pk_env_t *get_server_identity_key(void); +int server_identity_key_is_set(void); +void set_client_identity_key(crypto_pk_env_t *k); +crypto_pk_env_t *get_tlsclient_identity_key(void); +int client_identity_key_is_set(void); +authority_cert_t *get_my_v3_authority_cert(void); +crypto_pk_env_t *get_my_v3_authority_signing_key(void); +authority_cert_t *get_my_v3_legacy_cert(void); +crypto_pk_env_t *get_my_v3_legacy_signing_key(void); +void dup_onion_keys(crypto_pk_env_t **key, crypto_pk_env_t **last); +void rotate_onion_key(void); +crypto_pk_env_t *init_key_from_file(const char *fname, int generate, + int severity); +void v3_authority_check_key_expiry(void); + +int init_keys(void); + +int check_whether_orport_reachable(void); +int check_whether_dirport_reachable(void); +void consider_testing_reachability(int test_or, int test_dir); +void router_orport_found_reachable(void); +void router_dirport_found_reachable(void); +void router_perform_bandwidth_test(int num_circs, time_t now); + +int authdir_mode(or_options_t *options); +int authdir_mode_v1(or_options_t *options); +int authdir_mode_v2(or_options_t *options); +int authdir_mode_v3(or_options_t *options); +int authdir_mode_any_main(or_options_t *options); +int authdir_mode_any_nonhidserv(or_options_t *options); +int authdir_mode_handles_descs(or_options_t *options, int purpose); +int authdir_mode_publishes_statuses(or_options_t *options); +int authdir_mode_tests_reachability(or_options_t *options); +int authdir_mode_bridge(or_options_t *options); + +int server_mode(or_options_t *options); +int public_server_mode(or_options_t *options); +int advertised_server_mode(void); +int proxy_mode(or_options_t *options); +void consider_publishable_server(int force); +int should_refuse_unknown_exits(or_options_t *options); + +void router_upload_dir_desc_to_dirservers(int force); +void mark_my_descriptor_dirty_if_older_than(time_t when); +void mark_my_descriptor_dirty(void); +void check_descriptor_bandwidth_changed(time_t now); +void check_descriptor_ipaddress_changed(time_t now); +void router_new_address_suggestion(const char *suggestion, + const dir_connection_t *d_conn); +int router_compare_to_my_exit_policy(edge_connection_t *conn); +int router_my_exit_policy_is_reject_star(void); +routerinfo_t *router_get_my_routerinfo(void); +extrainfo_t *router_get_my_extrainfo(void); +const char *router_get_my_descriptor(void); +int router_digest_is_me(const char *digest); +int router_extrainfo_digest_is_me(const char *digest); +int router_is_me(routerinfo_t *router); +int router_fingerprint_is_me(const char *fp); +int router_pick_published_address(or_options_t *options, uint32_t *addr); +int router_rebuild_descriptor(int force); +int router_dump_router_to_string(char *s, size_t maxlen, routerinfo_t *router, + crypto_pk_env_t *ident_key); +int extrainfo_dump_to_string(char **s, extrainfo_t *extrainfo, + crypto_pk_env_t *ident_key); +int is_legal_nickname(const char *s); +int is_legal_nickname_or_hexdigest(const char *s); +int is_legal_hexdigest(const char *s); +void router_get_verbose_nickname(char *buf, const routerinfo_t *router); +void routerstatus_get_verbose_nickname(char *buf, + const routerstatus_t *router); +void router_reset_warnings(void); +void router_reset_reachability(void); +void router_free_all(void); + +const char *router_purpose_to_string(uint8_t p); +uint8_t router_purpose_from_string(const char *s); + +#ifdef ROUTER_PRIVATE +/* Used only by router.c and test.c */ +void get_platform_str(char *platform, size_t len); +#endif + +#endif + diff --git a/src/or/routerlist.c b/src/or/routerlist.c index 9f04620986..b0909eb730 100644 --- a/src/or/routerlist.c +++ b/src/or/routerlist.c @@ -12,6 +12,25 @@ **/ #include "or.h" +#include "circuitbuild.h" +#include "config.h" +#include "connection.h" +#include "control.h" +#include "directory.h" +#include "dirserv.h" +#include "dirvote.h" +#include "geoip.h" +#include "hibernate.h" +#include "main.h" +#include "networkstatus.h" +#include "policies.h" +#include "reasons.h" +#include "rendcommon.h" +#include "rendservice.h" +#include "rephist.h" +#include "router.h" +#include "routerlist.h" +#include "routerparse.h" // #define DEBUG_ROUTERLIST @@ -26,8 +45,8 @@ static void mark_all_trusteddirservers_up(void); static int router_nickname_matches(routerinfo_t *router, const char *nickname); static void trusted_dir_server_free(trusted_dir_server_t *ds); static void launch_router_descriptor_downloads(smartlist_t *downloadable, + routerstatus_t *source, time_t now); -static void update_consensus_router_descriptor_downloads(time_t now); static int signed_desc_digest_is_recognized(signed_descriptor_t *desc); static void update_router_have_minimum_dir_info(void); static const char *signed_descriptor_get_body_impl(signed_descriptor_t *desc, @@ -156,21 +175,24 @@ already_have_cert(authority_cert_t *cert) /** Load a bunch of new key certificates from the string <b>contents</b>. If * <b>from_store</b> is true, the certificates are from the cache, and we - * don't need to flush them to disk. If <b>from_store</b> is false, we need - * to flush any changed certificates to disk. Return 0 on success, -1 on - * failure. */ + * don't need to flush them to disk. If <b>flush</b> is true, we need + * to flush any changed certificates to disk now. Return 0 on success, -1 + * if any certs fail to parse. */ int trusted_dirs_load_certs_from_string(const char *contents, int from_store, int flush) { trusted_dir_server_t *ds; const char *s, *eos; + int failure_code = 0; for (s = contents; *s; s = eos) { authority_cert_t *cert = authority_cert_parse_from_string(s, &eos); cert_list_t *cl; - if (!cert) + if (!cert) { + failure_code = -1; break; + } ds = trusteddirserver_get_by_v3_auth_digest( cert->cache_info.identity_digest); log_debug(LD_DIR, "Parsed certificate for %s", @@ -181,15 +203,15 @@ trusted_dirs_load_certs_from_string(const char *contents, int from_store, log_info(LD_DIR, "Skipping %s certificate for %s that we " "already have.", from_store ? "cached" : "downloaded", - ds ? ds->nickname : "??"); + ds ? ds->nickname : "an old or new authority"); /* a duplicate on a download should be treated as a failure, since it * probably means we wanted a different secret key or we are trying to * replace an expired cert that has not in fact been updated. */ if (!from_store) { - log_warn(LD_DIR, "Got a certificate for %s that we already have. " - "Maybe they haven't updated it. Waiting for a while.", - ds ? ds->nickname : "??"); + log_warn(LD_DIR, "Got a certificate for %s, but we already have it. " + "Maybe they haven't updated it. Waiting for a while.", + ds ? ds->nickname : "an old or new authority"); authority_cert_dl_failed(cert->cache_info.identity_digest, 404); } @@ -224,7 +246,7 @@ trusted_dirs_load_certs_from_string(const char *contents, int from_store, ds->dir_port != cert->dir_port)) { char *a = tor_dup_ip(cert->addr); log_notice(LD_DIR, "Updating address for directory authority %s " - "from %s:%d to %s:%d based on in certificate.", + "from %s:%d to %s:%d based on certificate.", ds->nickname, ds->address, (int)ds->dir_port, a, cert->dir_port); tor_free(a); @@ -241,8 +263,11 @@ trusted_dirs_load_certs_from_string(const char *contents, int from_store, if (flush) trusted_dirs_flush_certs_to_disk(); + /* call this even if failure_code is <0, since some certs might have + * succeeded. */ networkstatus_note_certs_arrived(); - return 0; + + return failure_code; } /** Save all v3 key certificates to the cached-certs file. */ @@ -303,7 +328,7 @@ trusted_dirs_remove_old_certs(void) time_t cert_published; if (newest == cert) continue; - expired = ftime_definitely_after(now, cert->expires); + expired = now > cert->expires; cert_published = cert->cache_info.published_on; /* Store expired certs for 48 hours after a newer arrives; */ @@ -415,6 +440,23 @@ authority_cert_dl_failed(const char *id_digest, int status) download_status_failed(&cl->dl_status, status); } +/** Return true iff when we've been getting enough failures when trying to + * download the certificate with ID digest <b>id_digest</b> that we're willing + * to start bugging the user about it. */ +int +authority_cert_dl_looks_uncertain(const char *id_digest) +{ +#define N_AUTH_CERT_DL_FAILURES_TO_BUG_USER 2 + cert_list_t *cl; + int n_failures; + if (!trusted_dir_certs || + !(cl = digestmap_get(trusted_dir_certs, id_digest))) + return 0; + + n_failures = download_status_get_n_failures(&cl->dl_status); + return n_failures >= N_AUTH_CERT_DL_FAILURES_TO_BUG_USER; +} + /** How many times will we try to fetch a certificate before giving up? */ #define MAX_CERT_DL_FAILURES 8 @@ -442,17 +484,18 @@ authority_certs_fetch_missing(networkstatus_t *status, time_t now) list_pending_downloads(pending, DIR_PURPOSE_FETCH_CERTIFICATE, "fp/"); if (status) { - SMARTLIST_FOREACH(status->voters, networkstatus_voter_info_t *, voter, - { - if (tor_digest_is_zero(voter->signing_key_digest)) - continue; /* This authority never signed this consensus, so don't - * go looking for a cert with key digest 0000000000. */ - if (!cache && - !trusteddirserver_get_by_v3_auth_digest(voter->identity_digest)) - continue; /* We are not a cache, and we don't know this authority.*/ - cl = get_cert_list(voter->identity_digest); + SMARTLIST_FOREACH_BEGIN(status->voters, networkstatus_voter_info_t *, + voter) { + if (!smartlist_len(voter->sigs)) + continue; /* This authority never signed this consensus, so don't + * go looking for a cert with key digest 0000000000. */ + if (!cache && + !trusteddirserver_get_by_v3_auth_digest(voter->identity_digest)) + continue; /* We are not a cache, and we don't know this authority.*/ + cl = get_cert_list(voter->identity_digest); + SMARTLIST_FOREACH_BEGIN(voter->sigs, document_signature_t *, sig) { cert = authority_cert_get_by_digests(voter->identity_digest, - voter->signing_key_digest); + sig->signing_key_digest); if (cert) { if (now < cert->expires) download_status_reset(&cl->dl_status); @@ -463,37 +506,36 @@ authority_certs_fetch_missing(networkstatus_t *status, time_t now) !digestmap_get(pending, voter->identity_digest)) { log_notice(LD_DIR, "We're missing a certificate from authority " "with signing key %s: launching request.", - hex_str(voter->signing_key_digest, DIGEST_LEN)); - smartlist_add(missing_digests, voter->identity_digest); + hex_str(sig->signing_key_digest, DIGEST_LEN)); + smartlist_add(missing_digests, sig->identity_digest); } - }); + } SMARTLIST_FOREACH_END(sig); + } SMARTLIST_FOREACH_END(voter); } - SMARTLIST_FOREACH(trusted_dir_servers, trusted_dir_server_t *, ds, - { - int found = 0; - if (!(ds->type & V3_AUTHORITY)) - continue; - if (smartlist_digest_isin(missing_digests, ds->v3_identity_digest)) - continue; - cl = get_cert_list(ds->v3_identity_digest); - SMARTLIST_FOREACH(cl->certs, authority_cert_t *, cert, - { - if (!ftime_definitely_after(now, cert->expires)) { - /* It's not expired, and we weren't looking for something to - * verify a consensus with. Call it done. */ - download_status_reset(&cl->dl_status); - found = 1; - break; - } - }); - if (!found && - download_status_is_ready(&cl->dl_status, now,MAX_CERT_DL_FAILURES) && - !digestmap_get(pending, ds->v3_identity_digest)) { - log_notice(LD_DIR, "No current certificate known for authority %s; " - "launching request.", ds->nickname); - smartlist_add(missing_digests, ds->v3_identity_digest); + SMARTLIST_FOREACH_BEGIN(trusted_dir_servers, trusted_dir_server_t *, ds) { + int found = 0; + if (!(ds->type & V3_AUTHORITY)) + continue; + if (smartlist_digest_isin(missing_digests, ds->v3_identity_digest)) + continue; + cl = get_cert_list(ds->v3_identity_digest); + SMARTLIST_FOREACH(cl->certs, authority_cert_t *, cert, { + if (now < cert->expires) { + /* It's not expired, and we weren't looking for something to + * verify a consensus with. Call it done. */ + download_status_reset(&cl->dl_status); + found = 1; + break; } }); + if (!found && + download_status_is_ready(&cl->dl_status, now,MAX_CERT_DL_FAILURES) && + !digestmap_get(pending, ds->v3_identity_digest)) { + log_info(LD_DIR, "No current certificate known for authority %s; " + "launching request.", ds->nickname); + smartlist_add(missing_digests, ds->v3_identity_digest); + } + } SMARTLIST_FOREACH_END(ds); if (!smartlist_len(missing_digests)) { goto done; @@ -749,8 +791,7 @@ router_rebuild_store(int flags, desc_store_t *store) store->journal_len = 0; store->bytes_dropped = 0; done: - if (signed_descriptors) - smartlist_free(signed_descriptors); + smartlist_free(signed_descriptors); tor_free(fname); tor_free(fname_tmp); if (chunk_list) { @@ -968,8 +1009,9 @@ router_get_trusteddirserver_by_digest(const char *digest) return NULL; } -/** Return the trusted_dir_server_t for the directory authority whose identity - * key hashes to <b>digest</b>, or NULL if no such authority is known. +/** Return the trusted_dir_server_t for the directory authority whose + * v3 identity key hashes to <b>digest</b>, or NULL if no such authority + * is known. */ trusted_dir_server_t * trusteddirserver_get_by_v3_auth_digest(const char *digest) @@ -1028,6 +1070,7 @@ router_pick_trusteddirserver(authority_type_t type, int flags) static routerstatus_t * router_pick_directory_server_impl(authority_type_t type, int flags) { + or_options_t *options = get_options(); routerstatus_t *result; smartlist_t *direct, *tunnel; smartlist_t *trusted_direct, *trusted_tunnel; @@ -1037,10 +1080,13 @@ router_pick_directory_server_impl(authority_type_t type, int flags) int requireother = ! (flags & PDS_ALLOW_SELF); int fascistfirewall = ! (flags & PDS_IGNORE_FASCISTFIREWALL); int prefer_tunnel = (flags & _PDS_PREFER_TUNNELED_DIR_CONNS); + int try_excluding = 1, n_excluded = 0; if (!consensus) return NULL; + retry_without_exclude: + direct = smartlist_create(); tunnel = smartlist_create(); trusted_direct = smartlist_create(); @@ -1072,6 +1118,11 @@ router_pick_directory_server_impl(authority_type_t type, int flags) if ((type & EXTRAINFO_CACHE) && !router_supports_extrainfo(status->identity_digest, 0)) continue; + if (try_excluding && options->ExcludeNodes && + routerset_contains_routerstatus(options->ExcludeNodes, status)) { + ++n_excluded; + continue; + } /* XXXX IP6 proposal 118 */ tor_addr_from_ipv4h(&addr, status->addr); @@ -1089,9 +1140,10 @@ router_pick_directory_server_impl(authority_type_t type, int flags) } SMARTLIST_FOREACH_END(status); if (smartlist_len(tunnel)) { - result = routerstatus_sl_choose_by_bandwidth(tunnel); + result = routerstatus_sl_choose_by_bandwidth(tunnel, WEIGHT_FOR_DIR); } else if (smartlist_len(overloaded_tunnel)) { - result = routerstatus_sl_choose_by_bandwidth(overloaded_tunnel); + result = routerstatus_sl_choose_by_bandwidth(overloaded_tunnel, + WEIGHT_FOR_DIR); } else if (smartlist_len(trusted_tunnel)) { /* FFFF We don't distinguish between trusteds and overloaded trusteds * yet. Maybe one day we should. */ @@ -1099,9 +1151,10 @@ router_pick_directory_server_impl(authority_type_t type, int flags) * is a feature, but it could easily be a bug. -RD */ result = smartlist_choose(trusted_tunnel); } else if (smartlist_len(direct)) { - result = routerstatus_sl_choose_by_bandwidth(direct); + result = routerstatus_sl_choose_by_bandwidth(direct, WEIGHT_FOR_DIR); } else if (smartlist_len(overloaded_direct)) { - result = routerstatus_sl_choose_by_bandwidth(overloaded_direct); + result = routerstatus_sl_choose_by_bandwidth(overloaded_direct, + WEIGHT_FOR_DIR); } else { result = smartlist_choose(trusted_direct); } @@ -1111,6 +1164,15 @@ router_pick_directory_server_impl(authority_type_t type, int flags) smartlist_free(trusted_tunnel); smartlist_free(overloaded_direct); smartlist_free(overloaded_tunnel); + + if (result == NULL && try_excluding && !options->StrictNodes && n_excluded) { + /* If we got no result, and we are excluding nodes, and StrictNodes is + * not set, try again without excluding nodes. */ + try_excluding = 0; + n_excluded = 0; + goto retry_without_exclude; + } + return result; } @@ -1121,6 +1183,7 @@ static routerstatus_t * router_pick_trusteddirserver_impl(authority_type_t type, int flags, int *n_busy_out) { + or_options_t *options = get_options(); smartlist_t *direct, *tunnel; smartlist_t *overloaded_direct, *overloaded_tunnel; routerinfo_t *me = router_get_my_routerinfo(); @@ -1131,10 +1194,13 @@ router_pick_trusteddirserver_impl(authority_type_t type, int flags, const int prefer_tunnel = (flags & _PDS_PREFER_TUNNELED_DIR_CONNS); const int no_serverdesc_fetching =(flags & PDS_NO_EXISTING_SERVERDESC_FETCH); int n_busy = 0; + int try_excluding = 1, n_excluded = 0; if (!trusted_dir_servers) return NULL; + retry_without_exclude: + direct = smartlist_create(); tunnel = smartlist_create(); overloaded_direct = smartlist_create(); @@ -1153,6 +1219,12 @@ router_pick_trusteddirserver_impl(authority_type_t type, int flags, continue; if (requireother && me && router_digest_is_me(d->digest)) continue; + if (try_excluding && options->ExcludeNodes && + routerset_contains_routerstatus(options->ExcludeNodes, + &d->fake_status)) { + ++n_excluded; + continue; + } /* XXXX IP6 proposal 118 */ tor_addr_from_ipv4h(&addr, d->addr); @@ -1199,6 +1271,15 @@ router_pick_trusteddirserver_impl(authority_type_t type, int flags, smartlist_free(tunnel); smartlist_free(overloaded_direct); smartlist_free(overloaded_tunnel); + + if (result == NULL && try_excluding && !options->StrictNodes && n_excluded) { + /* If we got no result, and we are excluding nodes, and StrictNodes is + * not set, try again without excluding nodes. */ + try_excluding = 0; + n_excluded = 0; + goto retry_without_exclude; + } + return result; } @@ -1230,6 +1311,13 @@ mark_all_trusteddirservers_up(void) router_dir_info_changed(); } +/** Return true iff r1 and r2 have the same address and OR port. */ +int +routers_have_same_or_addr(const routerinfo_t *r1, const routerinfo_t *r2) +{ + return r1->addr == r2->addr && r1->or_port == r2->or_port; +} + /** Reset all internal variables used to count failed downloads of network * status objects. */ void @@ -1450,6 +1538,8 @@ routerlist_find_my_routerinfo(void) /** Find a router that's up, that has this IP address, and * that allows exit to this address:port, or return NULL if there * isn't a good one. + * Don't exit enclave to excluded relays -- it wouldn't actually + * hurt anything, but this way there are fewer confused users. */ routerinfo_t * router_find_exact_exit_enclave(const char *address, uint16_t port) @@ -1457,6 +1547,7 @@ router_find_exact_exit_enclave(const char *address, uint16_t port) uint32_t addr; struct in_addr in; tor_addr_t a; + or_options_t *options = get_options(); if (!tor_inet_aton(address, &in)) return NULL; /* it's not an IP already */ @@ -1469,7 +1560,8 @@ router_find_exact_exit_enclave(const char *address, uint16_t port) if (router->addr == addr && router->is_running && compare_tor_addr_to_addr_policy(&a, port, router->exit_policy) == - ADDR_POLICY_ACCEPTED) + ADDR_POLICY_ACCEPTED && + !routerset_contains_router(options->_ExcludeExitNodesUnion, router)) return router; }); return NULL; @@ -1521,6 +1613,29 @@ router_get_advertised_bandwidth_capped(routerinfo_t *router) return result; } +/** When weighting bridges, enforce these values as lower and upper + * bound for believable bandwidth, because there is no way for us + * to verify a bridge's bandwidth currently. */ +#define BRIDGE_MIN_BELIEVABLE_BANDWIDTH 20000 /* 20 kB/sec */ +#define BRIDGE_MAX_BELIEVABLE_BANDWIDTH 100000 /* 100 kB/sec */ + +/** Return the smaller of the router's configured BandwidthRate + * and its advertised capacity, making sure to stay within the + * interval between bridge-min-believe-bw and + * bridge-max-believe-bw. */ +static uint32_t +bridge_get_advertised_bandwidth_bounded(routerinfo_t *router) +{ + uint32_t result = router->bandwidthcapacity; + if (result > router->bandwidthrate) + result = router->bandwidthrate; + if (result > BRIDGE_MAX_BELIEVABLE_BANDWIDTH) + result = BRIDGE_MAX_BELIEVABLE_BANDWIDTH; + else if (result < BRIDGE_MIN_BELIEVABLE_BANDWIDTH) + result = BRIDGE_MIN_BELIEVABLE_BANDWIDTH; + return result; +} + /** Return bw*1000, unless bw*1000 would overflow, in which case return * INT32_MAX. */ static INLINE int32_t @@ -1531,6 +1646,218 @@ kb_to_bytes(uint32_t bw) /** Helper function: * choose a random element of smartlist <b>sl</b>, weighted by + * the advertised bandwidth of each element using the consensus + * bandwidth weights. + * + * If <b>statuses</b> is zero, then <b>sl</b> is a list of + * routerinfo_t's. Otherwise it's a list of routerstatus_t's. + * + * If <b>rule</b>==WEIGHT_FOR_EXIT. we're picking an exit node: consider all + * nodes' bandwidth equally regardless of their Exit status, since there may + * be some in the list because they exit to obscure ports. If + * <b>rule</b>==NO_WEIGHTING, we're picking a non-exit node: weight + * exit-node's bandwidth less depending on the smallness of the fraction of + * Exit-to-total bandwidth. If <b>rule</b>==WEIGHT_FOR_GUARD, we're picking a + * guard node: consider all guard's bandwidth equally. Otherwise, weight + * guards proportionally less. + */ +static void * +smartlist_choose_by_bandwidth_weights(smartlist_t *sl, + bandwidth_weight_rule_t rule, + int statuses) +{ + int64_t weight_scale; + int64_t rand_bw; + double Wg = -1, Wm = -1, We = -1, Wd = -1; + double Wgb = -1, Wmb = -1, Web = -1, Wdb = -1; + double weighted_bw = 0; + double *bandwidths; + double tmp = 0; + unsigned int i; + int have_unknown = 0; /* true iff sl contains element not in consensus. */ + + /* Can't choose exit and guard at same time */ + tor_assert(rule == NO_WEIGHTING || + rule == WEIGHT_FOR_EXIT || + rule == WEIGHT_FOR_GUARD || + rule == WEIGHT_FOR_MID || + rule == WEIGHT_FOR_DIR); + + if (smartlist_len(sl) == 0) { + log_info(LD_CIRC, + "Empty routerlist passed in to consensus weight node " + "selection for rule %s", + bandwidth_weight_rule_to_string(rule)); + return NULL; + } + + weight_scale = circuit_build_times_get_bw_scale(NULL); + + if (rule == WEIGHT_FOR_GUARD) { + Wg = networkstatus_get_bw_weight(NULL, "Wgg", -1); + Wm = networkstatus_get_bw_weight(NULL, "Wgm", -1); /* Bridges */ + We = 0; + Wd = networkstatus_get_bw_weight(NULL, "Wgd", -1); + + Wgb = networkstatus_get_bw_weight(NULL, "Wgb", -1); + Wmb = networkstatus_get_bw_weight(NULL, "Wmb", -1); + Web = networkstatus_get_bw_weight(NULL, "Web", -1); + Wdb = networkstatus_get_bw_weight(NULL, "Wdb", -1); + } else if (rule == WEIGHT_FOR_MID) { + Wg = networkstatus_get_bw_weight(NULL, "Wmg", -1); + Wm = networkstatus_get_bw_weight(NULL, "Wmm", -1); + We = networkstatus_get_bw_weight(NULL, "Wme", -1); + Wd = networkstatus_get_bw_weight(NULL, "Wmd", -1); + + Wgb = networkstatus_get_bw_weight(NULL, "Wgb", -1); + Wmb = networkstatus_get_bw_weight(NULL, "Wmb", -1); + Web = networkstatus_get_bw_weight(NULL, "Web", -1); + Wdb = networkstatus_get_bw_weight(NULL, "Wdb", -1); + } else if (rule == WEIGHT_FOR_EXIT) { + // Guards CAN be exits if they have weird exit policies + // They are d then I guess... + We = networkstatus_get_bw_weight(NULL, "Wee", -1); + Wm = networkstatus_get_bw_weight(NULL, "Wem", -1); /* Odd exit policies */ + Wd = networkstatus_get_bw_weight(NULL, "Wed", -1); + Wg = networkstatus_get_bw_weight(NULL, "Weg", -1); /* Odd exit policies */ + + Wgb = networkstatus_get_bw_weight(NULL, "Wgb", -1); + Wmb = networkstatus_get_bw_weight(NULL, "Wmb", -1); + Web = networkstatus_get_bw_weight(NULL, "Web", -1); + Wdb = networkstatus_get_bw_weight(NULL, "Wdb", -1); + } else if (rule == WEIGHT_FOR_DIR) { + We = networkstatus_get_bw_weight(NULL, "Wbe", -1); + Wm = networkstatus_get_bw_weight(NULL, "Wbm", -1); + Wd = networkstatus_get_bw_weight(NULL, "Wbd", -1); + Wg = networkstatus_get_bw_weight(NULL, "Wbg", -1); + + Wgb = Wmb = Web = Wdb = weight_scale; + } else if (rule == NO_WEIGHTING) { + Wg = Wm = We = Wd = weight_scale; + Wgb = Wmb = Web = Wdb = weight_scale; + } + + if (Wg < 0 || Wm < 0 || We < 0 || Wd < 0 || Wgb < 0 || Wmb < 0 || Wdb < 0 + || Web < 0) { + log_debug(LD_CIRC, + "Got negative bandwidth weights. Defaulting to old selection" + " algorithm."); + return NULL; // Use old algorithm. + } + + Wg /= weight_scale; + Wm /= weight_scale; + We /= weight_scale; + Wd /= weight_scale; + + Wgb /= weight_scale; + Wmb /= weight_scale; + Web /= weight_scale; + Wdb /= weight_scale; + + bandwidths = tor_malloc_zero(sizeof(double)*smartlist_len(sl)); + + // Cycle through smartlist and total the bandwidth. + for (i = 0; i < (unsigned)smartlist_len(sl); ++i) { + int is_exit = 0, is_guard = 0, is_dir = 0, this_bw = 0, is_me = 0; + double weight = 1; + if (statuses) { + routerstatus_t *status = smartlist_get(sl, i); + is_exit = status->is_exit && !status->is_bad_exit; + is_guard = status->is_possible_guard; + is_dir = (status->dir_port != 0); + if (!status->has_bandwidth) { + tor_free(bandwidths); + /* This should never happen, unless all the authorites downgrade + * to 0.2.0 or rogue routerstatuses get inserted into our consensus. */ + log_warn(LD_BUG, + "Consensus is not listing bandwidths. Defaulting back to " + "old router selection algorithm."); + return NULL; + } + this_bw = kb_to_bytes(status->bandwidth); + if (router_digest_is_me(status->identity_digest)) + is_me = 1; + } else { + routerstatus_t *rs; + routerinfo_t *router = smartlist_get(sl, i); + rs = router_get_consensus_status_by_id( + router->cache_info.identity_digest); + is_exit = router->is_exit && !router->is_bad_exit; + is_guard = router->is_possible_guard; + is_dir = (router->dir_port != 0); + if (rs && rs->has_bandwidth) { + this_bw = kb_to_bytes(rs->bandwidth); + } else { /* bridge or other descriptor not in our consensus */ + this_bw = bridge_get_advertised_bandwidth_bounded(router); + have_unknown = 1; + } + if (router_digest_is_me(router->cache_info.identity_digest)) + is_me = 1; + } + if (is_guard && is_exit) { + weight = (is_dir ? Wdb*Wd : Wd); + } else if (is_guard) { + weight = (is_dir ? Wgb*Wg : Wg); + } else if (is_exit) { + weight = (is_dir ? Web*We : We); + } else { // middle + weight = (is_dir ? Wmb*Wm : Wm); + } + + bandwidths[i] = weight*this_bw; + weighted_bw += weight*this_bw; + if (is_me) + sl_last_weighted_bw_of_me = weight*this_bw; + } + + /* XXXX023 this is a kludge to expose these values. */ + sl_last_total_weighted_bw = weighted_bw; + + log_debug(LD_CIRC, "Choosing node for rule %s based on weights " + "Wg=%lf Wm=%lf We=%lf Wd=%lf with total bw %lf", + bandwidth_weight_rule_to_string(rule), + Wg, Wm, We, Wd, weighted_bw); + + /* If there is no bandwidth, choose at random */ + if (DBL_TO_U64(weighted_bw) == 0) { + /* Don't warn when using bridges/relays not in the consensus */ + if (!have_unknown) + log_warn(LD_CIRC, + "Weighted bandwidth is %lf in node selection for rule %s", + weighted_bw, bandwidth_weight_rule_to_string(rule)); + tor_free(bandwidths); + return smartlist_choose(sl); + } + + rand_bw = crypto_rand_uint64(DBL_TO_U64(weighted_bw)); + rand_bw++; /* crypto_rand_uint64() counts from 0, and we need to count + * from 1 below. See bug 1203 for details. */ + + /* Last, count through sl until we get to the element we picked */ + tmp = 0.0; + for (i=0; i < (unsigned)smartlist_len(sl); i++) { + tmp += bandwidths[i]; + if (tmp >= rand_bw) + break; + } + + if (i == (unsigned)smartlist_len(sl)) { + /* This was once possible due to round-off error, but shouldn't be able + * to occur any longer. */ + tor_fragile_assert(); + --i; + log_warn(LD_BUG, "Round-off error in computing bandwidth had an effect on " + " which router we chose. Please tell the developers. " + "%lf " U64_FORMAT " %lf", tmp, U64_PRINTF_ARG(rand_bw), + weighted_bw); + } + tor_free(bandwidths); + return smartlist_get(sl, i); +} + +/** Helper function: + * choose a random element of smartlist <b>sl</b>, weighted by * the advertised bandwidth of each element. * * If <b>statuses</b> is zero, then <b>sl</b> is a list of @@ -1565,11 +1892,24 @@ smartlist_choose_by_bandwidth(smartlist_t *sl, bandwidth_weight_rule_t rule, bitarray_t *guard_bits; int me_idx = -1; + // This function does not support WEIGHT_FOR_DIR + // or WEIGHT_FOR_MID + if (rule == WEIGHT_FOR_DIR || rule == WEIGHT_FOR_MID) { + rule = NO_WEIGHTING; + } + /* Can't choose exit and guard at same time */ tor_assert(rule == NO_WEIGHTING || rule == WEIGHT_FOR_EXIT || rule == WEIGHT_FOR_GUARD); + if (smartlist_len(sl) == 0) { + log_info(LD_CIRC, + "Empty routerlist passed in to old node selection for rule %s", + bandwidth_weight_rule_to_string(rule)); + return NULL; + } + /* First count the total bandwidth weight, and make a list * of each value. <0 means "unknown; no routerinfo." We use the * bits of negative values to remember whether the router was fast (-x)&1 @@ -1594,7 +1934,7 @@ smartlist_choose_by_bandwidth(smartlist_t *sl, bandwidth_weight_rule_t rule, if (status->has_bandwidth) { this_bw = kb_to_bytes(status->bandwidth); } else { /* guess */ - /* XXX022 once consensuses always list bandwidths, we can take + /* XXX023 once consensuses always list bandwidths, we can take * this guessing business out. -RD */ is_known = 0; flags = status->is_fast ? 1 : 0; @@ -1613,14 +1953,14 @@ smartlist_choose_by_bandwidth(smartlist_t *sl, bandwidth_weight_rule_t rule, if (rs && rs->has_bandwidth) { this_bw = kb_to_bytes(rs->bandwidth); } else if (rs) { /* guess; don't trust the descriptor */ - /* XXX022 once consensuses always list bandwidths, we can take + /* XXX023 once consensuses always list bandwidths, we can take * this guessing business out. -RD */ is_known = 0; flags = router->is_fast ? 1 : 0; flags |= is_exit ? 2 : 0; flags |= is_guard ? 4 : 0; } else /* bridge or other descriptor not in our consensus */ - this_bw = router_get_advertised_bandwidth_capped(router); + this_bw = bridge_get_advertised_bandwidth_bounded(router); } if (is_exit) bitarray_set(exit_bits, i); @@ -1628,6 +1968,8 @@ smartlist_choose_by_bandwidth(smartlist_t *sl, bandwidth_weight_rule_t rule, bitarray_set(guard_bits, i); if (is_known) { bandwidths[i] = (int32_t) this_bw; // safe since MAX_BELIEVABLE<INT32_MAX + // XXX this is no longer true! We don't always cap the bw anymore. Can + // a consensus make us overflow?-sh tor_assert(bandwidths[i] >= 0); if (is_guard) total_guard_bw += this_bw; @@ -1691,12 +2033,12 @@ smartlist_choose_by_bandwidth(smartlist_t *sl, bandwidth_weight_rule_t rule, * For detailed derivation of this formula, see * http://archives.seul.org/or/dev/Jul-2007/msg00056.html */ - if (rule == WEIGHT_FOR_EXIT) + if (rule == WEIGHT_FOR_EXIT || !total_exit_bw) exit_weight = 1.0; else exit_weight = 1.0 - all_bw/(3.0*exit_bw); - if (rule == WEIGHT_FOR_GUARD) + if (rule == WEIGHT_FOR_GUARD || !total_guard_bw) guard_weight = 1.0; else guard_weight = 1.0 - all_bw/(3.0*guard_bw); @@ -1727,7 +2069,7 @@ smartlist_choose_by_bandwidth(smartlist_t *sl, bandwidth_weight_rule_t rule, } } - /* XXXX022 this is a kludge to expose these values. */ + /* XXXX023 this is a kludge to expose these values. */ sl_last_total_weighted_bw = total_bw; log_debug(LD_CIRC, "Total weighted bw = "U64_FORMAT @@ -1745,6 +2087,8 @@ smartlist_choose_by_bandwidth(smartlist_t *sl, bandwidth_weight_rule_t rule, /* Almost done: choose a random value from the bandwidth weights. */ rand_bw = crypto_rand_uint64(total_bw); + rand_bw++; /* crypto_rand_uint64() counts from 0, and we need to count + * from 1 below. See bug 1203 for details. */ /* Last, count through sl until we get to the element we picked */ tmp = 0; @@ -1788,26 +2132,34 @@ routerinfo_t * routerlist_sl_choose_by_bandwidth(smartlist_t *sl, bandwidth_weight_rule_t rule) { - return smartlist_choose_by_bandwidth(sl, rule, 0); + routerinfo_t *ret; + if ((ret = smartlist_choose_by_bandwidth_weights(sl, rule, 0))) { + return ret; + } else { + return smartlist_choose_by_bandwidth(sl, rule, 0); + } } /** Choose a random element of status list <b>sl</b>, weighted by * the advertised bandwidth of each status. */ routerstatus_t * -routerstatus_sl_choose_by_bandwidth(smartlist_t *sl) +routerstatus_sl_choose_by_bandwidth(smartlist_t *sl, + bandwidth_weight_rule_t rule) { /* We are choosing neither exit nor guard here. Weight accordingly. */ - return smartlist_choose_by_bandwidth(sl, NO_WEIGHTING, 1); + routerstatus_t *ret; + if ((ret = smartlist_choose_by_bandwidth_weights(sl, rule, 1))) { + return ret; + } else { + return smartlist_choose_by_bandwidth(sl, rule, 1); + } } -/** Return a random running router from the routerlist. If any node - * named in <b>preferred</b> is available, pick one of those. Never +/** Return a random running router from the routerlist. Never * pick a node whose routerinfo is in * <b>excludedsmartlist</b>, or whose routerinfo matches <b>excludedset</b>, - * even if they are the only nodes - * available. If <b>CRN_STRICT_PREFERRED</b> is set in flags, never pick - * any node besides those in <b>preferred</b>. + * even if they are the only nodes available. * If <b>CRN_NEED_UPTIME</b> is set in flags and any router has more than * a minimum uptime, return one of those. * If <b>CRN_NEED_CAPACITY</b> is set in flags, weight your choice by the @@ -1820,8 +2172,7 @@ routerstatus_sl_choose_by_bandwidth(smartlist_t *sl) * node (that is, possibly discounting exit nodes). */ routerinfo_t * -router_choose_random_node(const char *preferred, - smartlist_t *excludedsmartlist, +router_choose_random_node(smartlist_t *excludedsmartlist, routerset_t *excludedset, router_crn_flags_t flags) { @@ -1829,18 +2180,16 @@ router_choose_random_node(const char *preferred, const int need_capacity = (flags & CRN_NEED_CAPACITY) != 0; const int need_guard = (flags & CRN_NEED_GUARD) != 0; const int allow_invalid = (flags & CRN_ALLOW_INVALID) != 0; - const int strict = (flags & CRN_STRICT_PREFERRED) != 0; const int weight_for_exit = (flags & CRN_WEIGHT_AS_EXIT) != 0; - smartlist_t *sl, *excludednodes; + smartlist_t *sl=smartlist_create(), + *excludednodes=smartlist_create(); routerinfo_t *choice = NULL, *r; bandwidth_weight_rule_t rule; tor_assert(!(weight_for_exit && need_guard)); rule = weight_for_exit ? WEIGHT_FOR_EXIT : - (need_guard ? WEIGHT_FOR_GUARD : NO_WEIGHTING); - - excludednodes = smartlist_create(); + (need_guard ? WEIGHT_FOR_GUARD : WEIGHT_FOR_MID); /* Exclude relays that allow single hop exit circuits, if the user * wants to (such relays might be risky) */ @@ -1857,60 +2206,35 @@ router_choose_random_node(const char *preferred, routerlist_add_family(excludednodes, r); } - /* Try the preferred nodes first. Ignore need_uptime and need_capacity - * and need_guard, since the user explicitly asked for these nodes. */ - if (preferred) { - sl = smartlist_create(); - add_nickname_list_to_smartlist(sl,preferred,1); - smartlist_subtract(sl,excludednodes); - if (excludedsmartlist) - smartlist_subtract(sl,excludedsmartlist); - if (excludedset) - routerset_subtract_routers(sl,excludedset); - choice = smartlist_choose(sl); - smartlist_free(sl); - } - if (!choice && !strict) { - /* Then give up on our preferred choices: any node - * will do that has the required attributes. */ - sl = smartlist_create(); - router_add_running_routers_to_smartlist(sl, allow_invalid, - need_uptime, need_capacity, - need_guard); - smartlist_subtract(sl,excludednodes); - if (excludedsmartlist) - smartlist_subtract(sl,excludedsmartlist); - if (excludedset) - routerset_subtract_routers(sl,excludedset); - - if (need_capacity || need_guard) - choice = routerlist_sl_choose_by_bandwidth(sl, rule); - else - choice = smartlist_choose(sl); - - smartlist_free(sl); - if (!choice && (need_uptime || need_capacity || need_guard)) { - /* try once more -- recurse but with fewer restrictions. */ - log_info(LD_CIRC, - "We couldn't find any live%s%s%s routers; falling back " - "to list of all routers.", - need_capacity?", fast":"", - need_uptime?", stable":"", - need_guard?", guard":""); - flags &= ~ (CRN_NEED_UPTIME|CRN_NEED_CAPACITY|CRN_NEED_GUARD); - choice = router_choose_random_node( - NULL, excludedsmartlist, excludedset, flags); - } + router_add_running_routers_to_smartlist(sl, allow_invalid, + need_uptime, need_capacity, + need_guard); + smartlist_subtract(sl,excludednodes); + if (excludedsmartlist) + smartlist_subtract(sl,excludedsmartlist); + if (excludedset) + routerset_subtract_routers(sl,excludedset); + + // Always weight by bandwidth + choice = routerlist_sl_choose_by_bandwidth(sl, rule); + + smartlist_free(sl); + if (!choice && (need_uptime || need_capacity || need_guard)) { + /* try once more -- recurse but with fewer restrictions. */ + log_info(LD_CIRC, + "We couldn't find any live%s%s%s routers; falling back " + "to list of all routers.", + need_capacity?", fast":"", + need_uptime?", stable":"", + need_guard?", guard":""); + flags &= ~ (CRN_NEED_UPTIME|CRN_NEED_CAPACITY|CRN_NEED_GUARD); + choice = router_choose_random_node( + excludedsmartlist, excludedset, flags); } smartlist_free(excludednodes); if (!choice) { - if (strict) { - log_warn(LD_CIRC, "All preferred nodes were down when trying to choose " - "node, and the Strict[...]Nodes option is set. Failing."); - } else { - log_warn(LD_CIRC, - "No available nodes when trying to choose node. Failing."); - } + log_warn(LD_CIRC, + "No available nodes when trying to choose node. Failing."); } return choice; } @@ -2367,6 +2691,9 @@ extrainfo_free(extrainfo_t *extrainfo) static void signed_descriptor_free(signed_descriptor_t *sd) { + if (!sd) + return; + tor_free(sd->signed_descriptor_body); /* XXXX remove this once more bugs go away. */ @@ -2374,12 +2701,15 @@ signed_descriptor_free(signed_descriptor_t *sd) tor_free(sd); } -/** Extract a signed_descriptor_t from a routerinfo, and free the routerinfo. +/** Extract a signed_descriptor_t from a general routerinfo, and free the + * routerinfo. */ static signed_descriptor_t * signed_descriptor_from_routerinfo(routerinfo_t *ri) { - signed_descriptor_t *sd = tor_malloc_zero(sizeof(signed_descriptor_t)); + signed_descriptor_t *sd; + tor_assert(ri->purpose == ROUTER_PURPOSE_GENERAL); + sd = tor_malloc_zero(sizeof(signed_descriptor_t)); memcpy(sd, &(ri->cache_info), sizeof(signed_descriptor_t)); sd->routerlist_index = -1; ri->cache_info.signed_descriptor_body = NULL; @@ -2398,7 +2728,8 @@ _extrainfo_free(void *e) void routerlist_free(routerlist_t *rl) { - tor_assert(rl); + if (!rl) + return; rimap_free(rl->identity_map, NULL); sdmap_free(rl->desc_digest_map, NULL); sdmap_free(rl->desc_by_eid_map, NULL); @@ -2437,46 +2768,6 @@ dump_routerlist_mem_usage(int severity) "In %d old descriptors: "U64_FORMAT" bytes.", smartlist_len(routerlist->routers), U64_PRINTF_ARG(livedescs), smartlist_len(routerlist->old_routers), U64_PRINTF_ARG(olddescs)); - -#if 0 - { - const smartlist_t *networkstatus_v2_list = networkstatus_get_v2_list(); - networkstatus_t *consensus = networkstatus_get_latest_consensus(); - log(severity, LD_DIR, "Now let's look through old_descriptors!"); - SMARTLIST_FOREACH(routerlist->old_routers, signed_descriptor_t *, sd, { - int in_v2 = 0; - int in_v3 = 0; - char published[ISO_TIME_LEN+1]; - char last_valid_until[ISO_TIME_LEN+1]; - char last_served_at[ISO_TIME_LEN+1]; - char id[HEX_DIGEST_LEN+1]; - routerstatus_t *rs; - format_iso_time(published, sd->published_on); - format_iso_time(last_valid_until, sd->last_listed_as_valid_until); - format_iso_time(last_served_at, sd->last_served_at); - base16_encode(id, sizeof(id), sd->identity_digest, DIGEST_LEN); - SMARTLIST_FOREACH(networkstatus_v2_list, networkstatus_v2_t *, ns, - { - rs = networkstatus_v2_find_entry(ns, sd->identity_digest); - if (rs && tor_memeq(rs->descriptor_digest, - sd->signed_descriptor_digest, DIGEST_LEN)) { - in_v2 = 1; break; - } - }); - if (consensus) { - rs = networkstatus_vote_find_entry(consensus, sd->identity_digest); - if (rs && tor_memeq(rs->descriptor_digest, - sd->signed_descriptor_digest, DIGEST_LEN)) - in_v3 = 1; - } - log(severity, LD_DIR, - "Old descriptor for %s (published %s) %sin v2 ns, %sin v3 " - "consensus. Last valid until %s; last served at %s.", - id, published, in_v2 ? "" : "not ", in_v3 ? "" : "not ", - last_valid_until, last_served_at); - }); - } -#endif } /** Debugging helper: If <b>idx</b> is nonnegative, assert that <b>ri</b> is @@ -2509,6 +2800,7 @@ static void routerlist_insert(routerlist_t *rl, routerinfo_t *ri) { routerinfo_t *ri_old; + signed_descriptor_t *sd_old; { /* XXXX Remove if this slows us down. */ routerinfo_t *ri_generated = router_get_my_routerinfo(); @@ -2518,8 +2810,16 @@ routerlist_insert(routerlist_t *rl, routerinfo_t *ri) ri_old = rimap_set(rl->identity_map, ri->cache_info.identity_digest, ri); tor_assert(!ri_old); - sdmap_set(rl->desc_digest_map, ri->cache_info.signed_descriptor_digest, - &(ri->cache_info)); + + sd_old = sdmap_set(rl->desc_digest_map, + ri->cache_info.signed_descriptor_digest, + &(ri->cache_info)); + if (sd_old) { + rl->desc_store.bytes_dropped += sd_old->signed_descriptor_len; + sdmap_remove(rl->desc_by_eid_map, sd_old->extra_info_digest); + signed_descriptor_free(sd_old); + } + if (!tor_digest_is_zero(ri->cache_info.extra_info_digest)) sdmap_set(rl->desc_by_eid_map, ri->cache_info.extra_info_digest, &ri->cache_info); @@ -2738,6 +3038,7 @@ routerlist_replace(routerlist_t *rl, routerinfo_t *ri_old, routerinfo_t *ri_new) { int idx; + int same_descriptors; routerinfo_t *ri_tmp; extrainfo_t *ei_tmp; @@ -2782,8 +3083,15 @@ routerlist_replace(routerlist_t *rl, routerinfo_t *ri_old, &ri_new->cache_info); } + same_descriptors = ! memcmp(ri_old->cache_info.signed_descriptor_digest, + ri_new->cache_info.signed_descriptor_digest, + DIGEST_LEN); + if (should_cache_old_descriptors() && - ri_old->purpose == ROUTER_PURPOSE_GENERAL) { + ri_old->purpose == ROUTER_PURPOSE_GENERAL && + !same_descriptors) { + /* ri_old is going to become a signed_descriptor_t and go into + * old_routers */ signed_descriptor_t *sd = signed_descriptor_from_routerinfo(ri_old); smartlist_add(rl->old_routers, sd); sd->routerlist_index = smartlist_len(rl->old_routers)-1; @@ -2791,24 +3099,27 @@ routerlist_replace(routerlist_t *rl, routerinfo_t *ri_old, if (!tor_digest_is_zero(sd->extra_info_digest)) sdmap_set(rl->desc_by_eid_map, sd->extra_info_digest, sd); } else { - if (tor_memneq(ri_old->cache_info.signed_descriptor_digest, - ri_new->cache_info.signed_descriptor_digest, - DIGEST_LEN)) { - /* digests don't match; digestmap_set didn't replace */ + /* We're dropping ri_old. */ + if (!same_descriptors) { + /* digests don't match; The sdmap_set above didn't replace */ sdmap_remove(rl->desc_digest_map, ri_old->cache_info.signed_descriptor_digest); - } - ei_tmp = eimap_remove(rl->extra_info_map, - ri_old->cache_info.extra_info_digest); - if (ei_tmp) { - rl->extrainfo_store.bytes_dropped += - ei_tmp->cache_info.signed_descriptor_len; - extrainfo_free(ei_tmp); - } - if (!tor_digest_is_zero(ri_old->cache_info.extra_info_digest)) { - sdmap_remove(rl->desc_by_eid_map, - ri_old->cache_info.extra_info_digest); + if (memcmp(ri_old->cache_info.extra_info_digest, + ri_new->cache_info.extra_info_digest, DIGEST_LEN)) { + ei_tmp = eimap_remove(rl->extra_info_map, + ri_old->cache_info.extra_info_digest); + if (ei_tmp) { + rl->extrainfo_store.bytes_dropped += + ei_tmp->cache_info.signed_descriptor_len; + extrainfo_free(ei_tmp); + } + } + + if (!tor_digest_is_zero(ri_old->cache_info.extra_info_digest)) { + sdmap_remove(rl->desc_by_eid_map, + ri_old->cache_info.extra_info_digest); + } } rl->desc_store.bytes_dropped += ri_old->cache_info.signed_descriptor_len; routerinfo_free(ri_old); @@ -2846,8 +3157,7 @@ routerlist_reparse_old(routerlist_t *rl, signed_descriptor_t *sd) void routerlist_free_all(void) { - if (routerlist) - routerlist_free(routerlist); + routerlist_free(routerlist); routerlist = NULL; if (warned_nicknames) { SMARTLIST_FOREACH(warned_nicknames, char *, cp, tor_free(cp)); @@ -2940,7 +3250,8 @@ router_add_to_routerlist(routerinfo_t *router, const char **msg, int from_cache, int from_fetch) { const char *id_digest; - int authdir = authdir_mode_handles_descs(get_options(), router->purpose); + or_options_t *options = get_options(); + int authdir = authdir_mode_handles_descs(options, router->purpose); int authdir_believes_valid = 0; routerinfo_t *old_router; networkstatus_t *consensus = networkstatus_get_latest_consensus(); @@ -2954,15 +3265,33 @@ router_add_to_routerlist(routerinfo_t *router, const char **msg, id_digest = router->cache_info.identity_digest; + old_router = router_get_by_digest(id_digest); + /* Make sure that we haven't already got this exact descriptor. */ if (sdmap_get(routerlist->desc_digest_map, router->cache_info.signed_descriptor_digest)) { - log_info(LD_DIR, - "Dropping descriptor that we already have for router '%s'", - router->nickname); - *msg = "Router descriptor was not new."; - routerinfo_free(router); - return ROUTER_WAS_NOT_NEW; + /* If we have this descriptor already and the new descriptor is a bridge + * descriptor, replace it. If we had a bridge descriptor before and the + * new one is not a bridge descriptor, don't replace it. */ + + /* Only members of routerlist->identity_map can be bridges; we don't + * put bridges in old_routers. */ + const int was_bridge = old_router && + old_router->purpose == ROUTER_PURPOSE_BRIDGE; + + if (routerinfo_is_a_configured_bridge(router) && + router->purpose == ROUTER_PURPOSE_BRIDGE && + !was_bridge) { + log_info(LD_DIR, "Replacing non-bridge descriptor with bridge " + "descriptor for router '%s'", router->nickname); + } else { + log_info(LD_DIR, + "Dropping descriptor that we already have for router '%s'", + router->nickname); + *msg = "Router descriptor was not new."; + routerinfo_free(router); + return ROUTER_WAS_NOT_NEW; + } } if (authdir) { @@ -2999,15 +3328,14 @@ router_add_to_routerlist(routerinfo_t *router, const char **msg, SMARTLIST_FOREACH(networkstatus_v2_list, networkstatus_v2_t *, ns, { routerstatus_t *rs = - networkstatus_v2_find_entry(ns, router->cache_info.identity_digest); + networkstatus_v2_find_entry(ns, id_digest); if (rs && tor_memeq(rs->descriptor_digest, router->cache_info.signed_descriptor_digest, DIGEST_LEN)) rs->need_to_mirror = 0; }); if (consensus) { - routerstatus_t *rs = networkstatus_vote_find_entry(consensus, - router->cache_info.identity_digest); + routerstatus_t *rs = networkstatus_vote_find_entry(consensus, id_digest); if (rs && tor_memeq(rs->descriptor_digest, router->cache_info.signed_descriptor_digest, DIGEST_LEN)) { @@ -3028,14 +3356,26 @@ router_add_to_routerlist(routerinfo_t *router, const char **msg, return ROUTER_NOT_IN_CONSENSUS; } + /* If we're reading a bridge descriptor from our cache, and we don't + * recognize it as one of our currently configured bridges, drop the + * descriptor. Otherwise we could end up using it as one of our entry + * guards even if it isn't in our Bridge config lines. */ + if (router->purpose == ROUTER_PURPOSE_BRIDGE && from_cache && + !authdir_mode_bridge(options) && + !routerinfo_is_a_configured_bridge(router)) { + log_info(LD_DIR, "Dropping bridge descriptor for '%s' because we have " + "no bridge configured at that address.", router->nickname); + *msg = "Router descriptor was not a configured bridge."; + routerinfo_free(router); + return ROUTER_WAS_NOT_WANTED; + } + /* If we have a router with the same identity key, choose the newer one. */ - old_router = rimap_get(routerlist->identity_map, - router->cache_info.identity_digest); if (old_router) { if (!in_consensus && (router->cache_info.published_on <= old_router->cache_info.published_on)) { /* Same key, but old. This one is not listed in the consensus. */ - log_debug(LD_DIR, "Skipping not-new descriptor for router '%s'", + log_debug(LD_DIR, "Not-new descriptor for router '%s'", router->nickname); /* Only journal this desc if we'll be serving it. */ if (!from_cache && should_cache_old_descriptors()) @@ -3049,8 +3389,7 @@ router_add_to_routerlist(routerinfo_t *router, const char **msg, log_debug(LD_DIR, "Replacing entry for router '%s/%s' [%s]", router->nickname, old_router->nickname, hex_str(id_digest,DIGEST_LEN)); - if (router->addr == old_router->addr && - router->or_port == old_router->or_port) { + if (routers_have_same_or_addr(router, old_router)) { /* these carry over when the address and orport are unchanged. */ router->last_reachable = old_router->last_reachable; router->testing_since = old_router->testing_since; @@ -3078,9 +3417,10 @@ router_add_to_routerlist(routerinfo_t *router, const char **msg, /* We haven't seen a router with this identity before. Add it to the end of * the list. */ routerlist_insert(routerlist, router); - if (!from_cache) + if (!from_cache) { signed_desc_append_to_journal(&router->cache_info, &routerlist->desc_store); + } directory_set_dirty(); return ROUTER_ADDED_SUCCESSFULLY; } @@ -3096,7 +3436,7 @@ router_add_extrainfo_to_routerlist(extrainfo_t *ei, const char **msg, int inserted; (void)from_fetch; if (msg) *msg = NULL; - /*XXXX022 Do something with msg */ + /*XXXX023 Do something with msg */ inserted = extrainfo_insert(router_get_routerlist(), ei); @@ -3397,15 +3737,19 @@ routerlist_remove_old_routers(void) /** We just added a new set of descriptors. Take whatever extra steps * we need. */ -static void +void routerlist_descriptors_added(smartlist_t *sl, int from_cache) { tor_assert(sl); control_event_descriptors_changed(sl); - SMARTLIST_FOREACH(sl, routerinfo_t *, ri, + SMARTLIST_FOREACH_BEGIN(sl, routerinfo_t *, ri) { if (ri->purpose == ROUTER_PURPOSE_BRIDGE) learned_bridge_descriptor(ri, from_cache); - ); + if (ri->needs_retest_if_added) { + ri->needs_retest_if_added = 0; + dirserv_single_reachability_test(approx_time(), ri); + } + } SMARTLIST_FOREACH_END(ri); } /** @@ -3736,12 +4080,8 @@ add_trusted_dir_server(const char *nickname, const char *address, if (ent->or_port) ent->fake_status.version_supports_begindir = 1; -/* XX021 - wait until authorities are upgraded */ -#if 0 + ent->fake_status.version_supports_conditional_consensus = 1; -#else - ent->fake_status.version_supports_conditional_consensus = 0; -#endif smartlist_add(trusted_dir_servers, ent); router_dir_info_changed(); @@ -3756,10 +4096,8 @@ authority_cert_free(authority_cert_t *cert) return; tor_free(cert->cache_info.signed_descriptor_body); - if (cert->signing_key) - crypto_free_pk_env(cert->signing_key); - if (cert->identity_key) - crypto_free_pk_env(cert->identity_key); + crypto_free_pk_env(cert->signing_key); + crypto_free_pk_env(cert->identity_key); tor_free(cert); } @@ -3768,6 +4106,9 @@ authority_cert_free(authority_cert_t *cert) static void trusted_dir_server_free(trusted_dir_server_t *ds) { + if (!ds) + return; + tor_free(ds->nickname); tor_free(ds->description); tor_free(ds->address); @@ -3821,7 +4162,7 @@ list_pending_downloads(digestmap_t *result, const char *resource = TO_DIR_CONN(conn)->requested_resource; if (!strcmpstart(resource, prefix)) dir_split_resource_into_fingerprints(resource + p_len, - tmp, NULL, 1, 0); + tmp, NULL, DSR_HEX); } }); SMARTLIST_FOREACH(tmp, char *, d, @@ -3923,7 +4264,7 @@ client_would_use_router(routerstatus_t *rs, time_t now, or_options_t *options) * this number per server. */ #define MIN_DL_PER_REQUEST 4 /** To prevent a single screwy cache from confusing us by selective reply, - * try to split our requests into at least this this many requests. */ + * try to split our requests into at least this many requests. */ #define MIN_REQUESTS 3 /** If we want fewer than this many descriptors, wait until we * want more, or until MAX_CLIENT_INTERVAL_WITHOUT_REQUEST has @@ -3937,7 +4278,8 @@ client_would_use_router(routerstatus_t *rs, time_t now, or_options_t *options) * whether to delay fetching until we have more. If we don't want to delay, * launch one or more requests to the appropriate directory authorities. */ static void -launch_router_descriptor_downloads(smartlist_t *downloadable, time_t now) +launch_router_descriptor_downloads(smartlist_t *downloadable, + routerstatus_t *source, time_t now) { int should_delay = 0, n_downloadable; or_options_t *options = get_options(); @@ -3990,7 +4332,7 @@ launch_router_descriptor_downloads(smartlist_t *downloadable, time_t now) pds_flags |= PDS_NO_EXISTING_SERVERDESC_FETCH; } - n_per_request = (n_downloadable+MIN_REQUESTS-1) / MIN_REQUESTS; + n_per_request = CEIL_DIV(n_downloadable, MIN_REQUESTS); if (n_per_request > MAX_DL_PER_REQUEST) n_per_request = MAX_DL_PER_REQUEST; if (n_per_request < MIN_DL_PER_REQUEST) @@ -4003,11 +4345,11 @@ launch_router_descriptor_downloads(smartlist_t *downloadable, time_t now) log_info(LD_DIR, "Launching %d request%s for %d router%s, %d at a time", - (n_downloadable+n_per_request-1)/n_per_request, + CEIL_DIV(n_downloadable, n_per_request), req_plural, n_downloadable, rtr_plural, n_per_request); smartlist_sort_digests(downloadable); for (i=0; i < n_downloadable; i += n_per_request) { - initiate_descriptor_downloads(NULL, DIR_PURPOSE_FETCH_SERVERDESC, + initiate_descriptor_downloads(source, DIR_PURPOSE_FETCH_SERVERDESC, downloadable, i, i+n_per_request, pds_flags); } @@ -4163,18 +4505,18 @@ update_router_descriptor_cache_downloads_v2(time_t now) digestmap_free(map,NULL); } -/** For any descriptor that we want that's currently listed in the live - * consensus, download it as appropriate. */ -static void -update_consensus_router_descriptor_downloads(time_t now) +/** For any descriptor that we want that's currently listed in + * <b>consensus</b>, download it as appropriate. */ +void +update_consensus_router_descriptor_downloads(time_t now, int is_vote, + networkstatus_t *consensus) { or_options_t *options = get_options(); digestmap_t *map = NULL; smartlist_t *no_longer_old = smartlist_create(); smartlist_t *downloadable = smartlist_create(); + routerstatus_t *source = NULL; int authdir = authdir_mode(options); - networkstatus_t *consensus = - networkstatus_get_reasonably_live_consensus(now); int n_delayed=0, n_have=0, n_would_reject=0, n_wouldnt_use=0, n_inprogress=0, n_in_oldrouters=0; @@ -4183,10 +4525,24 @@ update_consensus_router_descriptor_downloads(time_t now) if (!consensus) goto done; + if (is_vote) { + /* where's it from, so we know whom to ask for descriptors */ + trusted_dir_server_t *ds; + networkstatus_voter_info_t *voter = smartlist_get(consensus->voters, 0); + tor_assert(voter); + ds = trusteddirserver_get_by_v3_auth_digest(voter->identity_digest); + if (ds) + source = &(ds->fake_status); + else + log_warn(LD_DIR, "couldn't lookup source from vote?"); + } + map = digestmap_new(); list_pending_descriptor_downloads(map, 0); - SMARTLIST_FOREACH(consensus->routerstatus_list, routerstatus_t *, rs, + SMARTLIST_FOREACH(consensus->routerstatus_list, void *, rsp, { + routerstatus_t *rs = + is_vote ? &(((vote_routerstatus_t *)rsp)->status) : rsp; signed_descriptor_t *sd; if ((sd = router_get_by_descriptor_digest(rs->descriptor_digest))) { routerinfo_t *ri; @@ -4221,6 +4577,18 @@ update_consensus_router_descriptor_downloads(time_t now) ++n_wouldnt_use; continue; /* We would never use it ourself. */ } + if (is_vote && source) { + char time_bufnew[ISO_TIME_LEN+1]; + char time_bufold[ISO_TIME_LEN+1]; + routerinfo_t *oldrouter = router_get_by_digest(rs->identity_digest); + format_iso_time(time_bufnew, rs->published_on); + if (oldrouter) + format_iso_time(time_bufold, oldrouter->cache_info.published_on); + log_info(LD_DIR, "Learned about %s (%s vs %s) from %s's vote (%s)", + rs->nickname, time_bufnew, + oldrouter ? time_bufold : "none", + source->nickname, oldrouter ? "known" : "unknown"); + } smartlist_add(downloadable, rs->descriptor_digest); }); @@ -4254,7 +4622,7 @@ update_consensus_router_descriptor_downloads(time_t now) smartlist_len(downloadable), n_delayed, n_have, n_in_oldrouters, n_would_reject, n_wouldnt_use, n_inprogress); - launch_router_descriptor_downloads(downloadable, now); + launch_router_descriptor_downloads(downloadable, source, now); digestmap_free(map, NULL); done: @@ -4264,7 +4632,7 @@ update_consensus_router_descriptor_downloads(time_t now) /** How often should we launch a server/authority request to be sure of getting * a guess for our IP? */ -/*XXXX021 this info should come from netinfo cells or something, or we should +/*XXXX023 this info should come from netinfo cells or something, or we should * do this only when we aren't seeing incoming data. see bug 652. */ #define DUMMY_DOWNLOAD_INTERVAL (20*60) @@ -4279,9 +4647,10 @@ update_router_descriptor_downloads(time_t now) if (directory_fetches_dir_info_early(options)) { update_router_descriptor_cache_downloads_v2(now); } - update_consensus_router_descriptor_downloads(now); + update_consensus_router_descriptor_downloads(now, 0, + networkstatus_get_reasonably_live_consensus(now)); - /* XXXX021 we could be smarter here; see notes on bug 652. */ + /* XXXX023 we could be smarter here; see notes on bug 652. */ /* If we're a server that doesn't have a configured address, we rely on * directory fetches to learn when our address changes. So if we haven't * tried to get any routerdescs in a long time, try a dummy fetch now. */ @@ -4416,16 +4785,21 @@ get_dir_info_status_string(void) /** Iterate over the servers listed in <b>consensus</b>, and count how many of * them seem like ones we'd use, and how many of <em>those</em> we have * descriptors for. Store the former in *<b>num_usable</b> and the latter in - * *<b>num_present</b>. */ + * *<b>num_present</b>. If <b>in_set</b> is non-NULL, only consider those + * routers in <b>in_set</b>. + */ static void count_usable_descriptors(int *num_present, int *num_usable, const networkstatus_t *consensus, - or_options_t *options, time_t now) + or_options_t *options, time_t now, + routerset_t *in_set) { *num_present = 0, *num_usable=0; SMARTLIST_FOREACH(consensus->routerstatus_list, routerstatus_t *, rs, { + if (in_set && ! routerset_contains_routerstatus(in_set, rs)) + continue; if (client_would_use_router(rs, now, options)) { ++*num_usable; /* the consensus says we want it. */ if (router_get_by_descriptor_digest(rs->descriptor_digest)) { @@ -4454,7 +4828,7 @@ count_loading_descriptors_progress(void) return 0; /* can't count descriptors if we have no list of them */ count_usable_descriptors(&num_present, &num_usable, - consensus, get_options(), now); + consensus, get_options(), now, NULL); if (num_usable == 0) return 0; /* don't div by 0 */ @@ -4498,22 +4872,39 @@ update_router_have_minimum_dir_info(void) goto done; } - count_usable_descriptors(&num_present, &num_usable, consensus, options, now); + count_usable_descriptors(&num_present, &num_usable, consensus, options, now, + NULL); if (num_present < num_usable/4) { tor_snprintf(dir_info_status, sizeof(dir_info_status), "We have only %d/%d usable descriptors.", num_present, num_usable); res = 0; control_event_bootstrap(BOOTSTRAP_STATUS_REQUESTING_DESCRIPTORS, 0); + goto done; } else if (num_present < 2) { tor_snprintf(dir_info_status, sizeof(dir_info_status), "Only %d descriptor%s here and believed reachable!", num_present, num_present ? "" : "s"); res = 0; - } else { - res = 1; + goto done; } + /* Check for entry nodes. */ + if (options->EntryNodes) { + count_usable_descriptors(&num_present, &num_usable, consensus, options, + now, options->EntryNodes); + + if (!num_usable || !num_present) { + tor_snprintf(dir_info_status, sizeof(dir_info_status), + "We have only %d/%d usable entry node descriptors.", + num_present, num_usable); + res = 0; + goto done; + } + } + + res = 1; + done: if (res && !have_min_dir_info) { log(LOG_NOTICE, LD_DIR, @@ -4526,6 +4917,13 @@ update_router_have_minimum_dir_info(void) log(quiet ? LOG_INFO : LOG_NOTICE, LD_DIR, "Our directory information is no longer up-to-date " "enough to build circuits: %s", dir_info_status); + + /* a) make us log when we next complete a circuit, so we know when Tor + * is back up and usable, and b) disable some activities that Tor + * should only do while circuits are working, like reachability tests + * and fetching bridge descriptors only over circuits. */ + can_complete_circuit = 0; + control_event_client_status(LOG_NOTICE, "NOT_ENOUGH_DIR_INFO"); } have_min_dir_info = res; @@ -4812,8 +5210,8 @@ esc_router_info(routerinfo_t *router) static char *info=NULL; char *esc_contact, *esc_platform; size_t len; - if (info) - tor_free(info); + tor_free(info); + if (!router) return NULL; /* we're exiting; just free the memory we use */ @@ -4948,9 +5346,8 @@ void routerset_refresh_countries(routerset_t *target) { int cc; - if (target->countries) { - bitarray_free(target->countries); - } + bitarray_free(target->countries); + if (!geoip_is_loaded()) { target->countries = NULL; target->n_countries = 0; @@ -5024,7 +5421,9 @@ routerset_parse(routerset_t *target, const char *s, const char *description) return r; } -/** DOCDOC */ +/** Called when we change a node set, or when we reload the geoip list: + * recompute all country info in all configuration node sets and in the + * routerlist. */ void refresh_all_country_info(void) { @@ -5075,7 +5474,7 @@ routerset_needs_geoip(const routerset_t *set) } /** Return true iff there are no entries in <b>set</b>. */ -static int +int routerset_is_empty(const routerset_t *set) { return !set || smartlist_len(set->list) == 0; @@ -5158,10 +5557,11 @@ routerset_contains_routerstatus(const routerset_t *set, routerstatus_t *rs) } /** Add every known routerinfo_t that is a member of <b>routerset</b> to - * <b>out</b>. If <b>running_only</b>, only add the running ones. */ + * <b>out</b>, but never add any that are part of <b>excludeset</b>. + * If <b>running_only</b>, only add the running ones. */ void routerset_get_all_routers(smartlist_t *out, const routerset_t *routerset, - int running_only) + const routerset_t *excludeset, int running_only) { tor_assert(out); if (!routerset || !routerset->list) @@ -5171,12 +5571,13 @@ routerset_get_all_routers(smartlist_t *out, const routerset_t *routerset, if (routerset_is_list(routerset)) { /* No routers are specified by type; all are given by name or digest. - * we can do a lookup in O(len(list)). */ + * we can do a lookup in O(len(routerset)). */ SMARTLIST_FOREACH(routerset->list, const char *, name, { routerinfo_t *router = router_get_by_nickname(name, 1); if (router) { if (!running_only || router->is_running) - smartlist_add(out, router); + if (!routerset_contains_router(excludeset, router)) + smartlist_add(out, router); } }); } else { @@ -5186,15 +5587,21 @@ routerset_get_all_routers(smartlist_t *out, const routerset_t *routerset, SMARTLIST_FOREACH(rl->routers, routerinfo_t *, router, { if (running_only && !router->is_running) continue; - if (routerset_contains_router(routerset, router)) + if (routerset_contains_router(routerset, router) && + !routerset_contains_router(excludeset, router)) smartlist_add(out, router); }); } } -/** Add to <b>target</b> every routerinfo_t from <b>source</b> that is in - * <b>include</b>, but not excluded in a more specific fashion by - * <b>exclude</b>. If <b>running_only</b>, only include running routers. +#if 0 +/** Add to <b>target</b> every routerinfo_t from <b>source</b> except: + * + * 1) Don't add it if <b>include</b> is non-empty and the relay isn't in + * <b>include</b>; and + * 2) Don't add it if <b>exclude</b> is non-empty and the relay is + * excluded in a more specific fashion by <b>exclude</b>. + * 3) If <b>running_only</b>, don't add non-running routers. */ void routersets_get_disjunction(smartlist_t *target, @@ -5218,6 +5625,7 @@ routersets_get_disjunction(smartlist_t *target, } }); } +#endif /** Remove every routerinfo_t from <b>lst</b> that is in <b>routerset</b>. */ void @@ -5249,10 +5657,15 @@ routerset_to_string(const routerset_t *set) int routerset_equal(const routerset_t *old, const routerset_t *new) { - if (old == NULL && new == NULL) + if (routerset_is_empty(old) && routerset_is_empty(new)) { + /* Two empty sets are equal */ return 1; - else if (old == NULL || new == NULL) + } else if (routerset_is_empty(old) || routerset_is_empty(new)) { + /* An empty set is equal to nothing else. */ return 0; + } + tor_assert(old != NULL); + tor_assert(new != NULL); if (smartlist_len(old->list) != smartlist_len(new->list)) return 0; @@ -5264,35 +5677,15 @@ routerset_equal(const routerset_t *old, const routerset_t *new) }); return 1; - -#if 0 - /* XXXX: This won't work if the names/digests are identical but in a - different order. Checking for exact equality would be heavy going, - is it worth it? -RH*/ - /* This code is totally bogus; sizeof doesn't work even remotely like this - * code seems to think. Let's revert to a string-based comparison for - * now. -NM*/ - if (sizeof(old->names) != sizeof(new->names)) - return 0; - - if (tor_memcmp(old->names,new->names,sizeof(new->names))) - return 0; - if (sizeof(old->digests) != sizeof(new->digests)) - return 0; - if (tor_memcmp(old->digests,new->digests,sizeof(new->digests))) - return 0; - if (sizeof(old->countries) != sizeof(new->countries)) - return 0; - if (tor_memcmp(old->countries,new->countries,sizeof(new->countries))) - return 0; - return 1; -#endif } /** Free all storage held in <b>routerset</b>. */ void routerset_free(routerset_t *routerset) { + if (!routerset) + return; + SMARTLIST_FOREACH(routerset->list, char *, cp, tor_free(cp)); smartlist_free(routerset->list); SMARTLIST_FOREACH(routerset->policies, addr_policy_t *, p, @@ -5303,8 +5696,7 @@ routerset_free(routerset_t *routerset) strmap_free(routerset->names, NULL); digestmap_free(routerset->digests, NULL); - if (routerset->countries) - bitarray_free(routerset->countries); + bitarray_free(routerset->countries); tor_free(routerset); } @@ -5335,7 +5727,6 @@ hid_serv_get_responsible_directories(smartlist_t *responsible_dirs, { int start, found, n_added = 0, i; networkstatus_t *c = networkstatus_get_latest_consensus(); - int use_begindir = get_options()->TunnelDirConns; if (!c || !smartlist_len(c->routerstatus_list)) { log_warn(LD_REND, "We don't have a consensus, so we can't perform v2 " "rendezvous operations."); @@ -5348,14 +5739,9 @@ hid_serv_get_responsible_directories(smartlist_t *responsible_dirs, do { routerstatus_t *r = smartlist_get(c->routerstatus_list, i); if (r->is_hs_dir) { - if (r->dir_port || use_begindir) - smartlist_add(responsible_dirs, r); - else - log_info(LD_REND, "Not adding router '%s' to list of responsible " - "hidden service directories, because we have no way of " - "reaching it.", r->nickname); + smartlist_add(responsible_dirs, r); if (++n_added == REND_NUMBER_OF_CONSECUTIVE_REPLICAS) - break; + return 0; } if (++i == smartlist_len(c->routerstatus_list)) i = 0; diff --git a/src/or/routerlist.h b/src/or/routerlist.h new file mode 100644 index 0000000000..fec18705b3 --- /dev/null +++ b/src/or/routerlist.h @@ -0,0 +1,200 @@ +/* Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file routerlist.h + * \brief Header file for routerlist.c. + **/ + +#ifndef _TOR_ROUTERLIST_H +#define _TOR_ROUTERLIST_H + +int get_n_authorities(authority_type_t type); +int trusted_dirs_reload_certs(void); +int trusted_dirs_load_certs_from_string(const char *contents, int from_store, + int flush); +void trusted_dirs_flush_certs_to_disk(void); +authority_cert_t *authority_cert_get_newest_by_id(const char *id_digest); +authority_cert_t *authority_cert_get_by_sk_digest(const char *sk_digest); +authority_cert_t *authority_cert_get_by_digests(const char *id_digest, + const char *sk_digest); +void authority_cert_get_all(smartlist_t *certs_out); +void authority_cert_dl_failed(const char *id_digest, int status); +void authority_certs_fetch_missing(networkstatus_t *status, time_t now); +int router_reload_router_list(void); +int authority_cert_dl_looks_uncertain(const char *id_digest); +smartlist_t *router_get_trusted_dir_servers(void); + +routerstatus_t *router_pick_directory_server(authority_type_t type, int flags); +trusted_dir_server_t *router_get_trusteddirserver_by_digest(const char *d); +trusted_dir_server_t *trusteddirserver_get_by_v3_auth_digest(const char *d); +routerstatus_t *router_pick_trusteddirserver(authority_type_t type, int flags); +int router_get_my_share_of_directory_requests(double *v2_share_out, + double *v3_share_out); +void router_reset_status_download_failures(void); +void routerlist_add_family(smartlist_t *sl, routerinfo_t *router); +int routers_in_same_family(routerinfo_t *r1, routerinfo_t *r2); +int routers_have_same_or_addr(const routerinfo_t *r1, const routerinfo_t *r2); +void add_nickname_list_to_smartlist(smartlist_t *sl, const char *list, + int must_be_running); +int router_nickname_is_in_list(routerinfo_t *router, const char *list); +routerinfo_t *routerlist_find_my_routerinfo(void); +routerinfo_t *router_find_exact_exit_enclave(const char *address, + uint16_t port); +int router_is_unreliable(routerinfo_t *router, int need_uptime, + int need_capacity, int need_guard); +uint32_t router_get_advertised_bandwidth(routerinfo_t *router); +uint32_t router_get_advertised_bandwidth_capped(routerinfo_t *router); + +routerinfo_t *routerlist_sl_choose_by_bandwidth(smartlist_t *sl, + bandwidth_weight_rule_t rule); +routerstatus_t *routerstatus_sl_choose_by_bandwidth(smartlist_t *sl, + bandwidth_weight_rule_t rule); + +routerinfo_t *router_choose_random_node(smartlist_t *excludedsmartlist, + struct routerset_t *excludedset, + router_crn_flags_t flags); + +routerinfo_t *router_get_by_nickname(const char *nickname, + int warn_if_unnamed); +int router_digest_version_as_new_as(const char *digest, const char *cutoff); +int router_digest_is_trusted_dir_type(const char *digest, + authority_type_t type); +#define router_digest_is_trusted_dir(d) \ + router_digest_is_trusted_dir_type((d), NO_AUTHORITY) + +int router_addr_is_trusted_dir(uint32_t addr); +int hexdigest_to_digest(const char *hexdigest, char *digest); +routerinfo_t *router_get_by_hexdigest(const char *hexdigest); +routerinfo_t *router_get_by_digest(const char *digest); +signed_descriptor_t *router_get_by_descriptor_digest(const char *digest); +signed_descriptor_t *router_get_by_extrainfo_digest(const char *digest); +signed_descriptor_t *extrainfo_get_by_descriptor_digest(const char *digest); +const char *signed_descriptor_get_body(signed_descriptor_t *desc); +const char *signed_descriptor_get_annotations(signed_descriptor_t *desc); +routerlist_t *router_get_routerlist(void); +void routerinfo_free(routerinfo_t *router); +void extrainfo_free(extrainfo_t *extrainfo); +void routerlist_free(routerlist_t *rl); +void dump_routerlist_mem_usage(int severity); +void routerlist_remove(routerlist_t *rl, routerinfo_t *ri, int make_old, + time_t now); +void routerlist_free_all(void); +void routerlist_reset_warnings(void); +void router_set_status(const char *digest, int up); + +static int WRA_WAS_ADDED(was_router_added_t s); +static int WRA_WAS_OUTDATED(was_router_added_t s); +static int WRA_WAS_REJECTED(was_router_added_t s); +/** Return true iff the descriptor was added. It might still be necessary to + * check whether the descriptor generator should be notified. + */ +static INLINE int +WRA_WAS_ADDED(was_router_added_t s) { + return s == ROUTER_ADDED_SUCCESSFULLY || s == ROUTER_ADDED_NOTIFY_GENERATOR; +} +/** Return true iff the descriptor was not added because it was either: + * - not in the consensus + * - neither in the consensus nor in any networkstatus document + * - it was outdated. + */ +static INLINE int WRA_WAS_OUTDATED(was_router_added_t s) +{ + return (s == ROUTER_WAS_NOT_NEW || + s == ROUTER_NOT_IN_CONSENSUS || + s == ROUTER_NOT_IN_CONSENSUS_OR_NETWORKSTATUS); +} +static INLINE int WRA_WAS_REJECTED(was_router_added_t s) +{ + return (s == ROUTER_AUTHDIR_REJECTS); +} +was_router_added_t router_add_to_routerlist(routerinfo_t *router, + const char **msg, + int from_cache, + int from_fetch); +was_router_added_t router_add_extrainfo_to_routerlist( + extrainfo_t *ei, const char **msg, + int from_cache, int from_fetch); +void routerlist_descriptors_added(smartlist_t *sl, int from_cache); +void routerlist_remove_old_routers(void); +int router_load_single_router(const char *s, uint8_t purpose, int cache, + const char **msg); +int router_load_routers_from_string(const char *s, const char *eos, + saved_location_t saved_location, + smartlist_t *requested_fingerprints, + int descriptor_digests, + const char *prepend_annotations); +void router_load_extrainfo_from_string(const char *s, const char *eos, + saved_location_t saved_location, + smartlist_t *requested_fingerprints, + int descriptor_digests); + +void routerlist_retry_directory_downloads(time_t now); +int router_exit_policy_all_routers_reject(uint32_t addr, uint16_t port, + int need_uptime); +int router_exit_policy_rejects_all(routerinfo_t *router); +trusted_dir_server_t *add_trusted_dir_server(const char *nickname, + const char *address, + uint16_t dir_port, uint16_t or_port, + const char *digest, const char *v3_auth_digest, + authority_type_t type); +void authority_cert_free(authority_cert_t *cert); +void clear_trusted_dir_servers(void); +int any_trusted_dir_is_v1_authority(void); +void update_consensus_router_descriptor_downloads(time_t now, int is_vote, + networkstatus_t *consensus); +void update_router_descriptor_downloads(time_t now); +void update_extrainfo_downloads(time_t now); +int router_have_minimum_dir_info(void); +void router_dir_info_changed(void); +const char *get_dir_info_status_string(void); +int count_loading_descriptors_progress(void); +void router_reset_descriptor_download_failures(void); +int router_differences_are_cosmetic(routerinfo_t *r1, routerinfo_t *r2); +int routerinfo_incompatible_with_extrainfo(routerinfo_t *ri, extrainfo_t *ei, + signed_descriptor_t *sd, + const char **msg); + +void routerlist_assert_ok(routerlist_t *rl); +const char *esc_router_info(routerinfo_t *router); +void routers_sort_by_identity(smartlist_t *routers); + +routerset_t *routerset_new(void); +int routerset_parse(routerset_t *target, const char *s, + const char *description); +void routerset_union(routerset_t *target, const routerset_t *source); +int routerset_is_list(const routerset_t *set); +int routerset_needs_geoip(const routerset_t *set); +int routerset_is_empty(const routerset_t *set); +int routerset_contains_router(const routerset_t *set, routerinfo_t *ri); +int routerset_contains_routerstatus(const routerset_t *set, + routerstatus_t *rs); +int routerset_contains_extendinfo(const routerset_t *set, + const extend_info_t *ei); +void routerset_get_all_routers(smartlist_t *out, const routerset_t *routerset, + const routerset_t *excludeset, + int running_only); +#if 0 +void routersets_get_disjunction(smartlist_t *target, const smartlist_t *source, + const routerset_t *include, + const routerset_t *exclude, int running_only); +#endif +void routerset_subtract_routers(smartlist_t *out, + const routerset_t *routerset); +char *routerset_to_string(const routerset_t *routerset); +void routerset_refresh_countries(routerset_t *target); +int routerset_equal(const routerset_t *old, const routerset_t *new); +void routerset_free(routerset_t *routerset); +void routerinfo_set_country(routerinfo_t *ri); +void routerlist_refresh_countries(void); +void refresh_all_country_info(void); + +int hid_serv_get_responsible_directories(smartlist_t *responsible_dirs, + const char *id); +int hid_serv_acting_as_directory(void); +int hid_serv_responsible_for_desc_id(const char *id); + +#endif + diff --git a/src/or/routerparse.c b/src/or/routerparse.c index 7ff0e2c3ce..3b669ad459 100644 --- a/src/or/routerparse.c +++ b/src/or/routerparse.c @@ -10,7 +10,21 @@ **/ #include "or.h" +#include "config.h" +#include "circuitbuild.h" +#include "dirserv.h" +#include "dirvote.h" +#include "policies.h" +#include "rendcommon.h" +#include "router.h" +#include "routerlist.h" #include "memarea.h" +#include "microdesc.h" +#include "networkstatus.h" +#include "rephist.h" +#include "routerparse.h" +#undef log +#include <math.h> /****************************************************************************/ @@ -55,6 +69,7 @@ typedef enum { K_S, K_V, K_W, + K_M, K_EVENTDNS, K_EXTRA_INFO, K_EXTRA_INFO_DIGEST, @@ -62,6 +77,31 @@ typedef enum { K_HIDDEN_SERVICE_DIR, K_ALLOW_SINGLE_HOP_EXITS, + K_DIRREQ_END, + K_DIRREQ_V2_IPS, + K_DIRREQ_V3_IPS, + K_DIRREQ_V2_REQS, + K_DIRREQ_V3_REQS, + K_DIRREQ_V2_SHARE, + K_DIRREQ_V3_SHARE, + K_DIRREQ_V2_RESP, + K_DIRREQ_V3_RESP, + K_DIRREQ_V2_DIR, + K_DIRREQ_V3_DIR, + K_DIRREQ_V2_TUN, + K_DIRREQ_V3_TUN, + K_ENTRY_END, + K_ENTRY_IPS, + K_CELL_END, + K_CELL_PROCESSED, + K_CELL_QUEUED, + K_CELL_TIME, + K_CELL_CIRCS, + K_EXIT_END, + K_EXIT_WRITTEN, + K_EXIT_READ, + K_EXIT_OPENED, + K_DIR_KEY_CERTIFICATE_VERSION, K_DIR_IDENTITY_KEY, K_DIR_KEY_PUBLISHED, @@ -78,13 +118,18 @@ typedef enum { K_KNOWN_FLAGS, K_PARAMS, + K_BW_WEIGHTS, K_VOTE_DIGEST, K_CONSENSUS_DIGEST, + K_ADDITIONAL_DIGEST, + K_ADDITIONAL_SIGNATURE, K_CONSENSUS_METHODS, K_CONSENSUS_METHOD, K_LEGACY_DIR_KEY, + K_DIRECTORY_FOOTER, A_PURPOSE, + A_LAST_LISTED, _A_UNKNOWN, R_RENDEZVOUS_SERVICE_DESCRIPTOR, @@ -122,7 +167,7 @@ typedef enum { * type. * * This structure is only allocated in memareas; do not allocate it on - * the heap, or token_free() won't work. + * the heap, or token_clear() won't work. */ typedef struct directory_token_t { directory_keyword tp; /**< Type of the token. */ @@ -258,6 +303,31 @@ static token_rule_t extrainfo_token_table[] = { T0N("opt", K_OPT, CONCAT_ARGS, OBJ_OK ), T01("read-history", K_READ_HISTORY, ARGS, NO_OBJ ), T01("write-history", K_WRITE_HISTORY, ARGS, NO_OBJ ), + T01("dirreq-stats-end", K_DIRREQ_END, ARGS, NO_OBJ ), + T01("dirreq-v2-ips", K_DIRREQ_V2_IPS, ARGS, NO_OBJ ), + T01("dirreq-v3-ips", K_DIRREQ_V3_IPS, ARGS, NO_OBJ ), + T01("dirreq-v2-reqs", K_DIRREQ_V2_REQS, ARGS, NO_OBJ ), + T01("dirreq-v3-reqs", K_DIRREQ_V3_REQS, ARGS, NO_OBJ ), + T01("dirreq-v2-share", K_DIRREQ_V2_SHARE, ARGS, NO_OBJ ), + T01("dirreq-v3-share", K_DIRREQ_V3_SHARE, ARGS, NO_OBJ ), + T01("dirreq-v2-resp", K_DIRREQ_V2_RESP, ARGS, NO_OBJ ), + T01("dirreq-v3-resp", K_DIRREQ_V3_RESP, ARGS, NO_OBJ ), + T01("dirreq-v2-direct-dl", K_DIRREQ_V2_DIR, ARGS, NO_OBJ ), + T01("dirreq-v3-direct-dl", K_DIRREQ_V3_DIR, ARGS, NO_OBJ ), + T01("dirreq-v2-tunneled-dl", K_DIRREQ_V2_TUN, ARGS, NO_OBJ ), + T01("dirreq-v3-tunneled-dl", K_DIRREQ_V3_TUN, ARGS, NO_OBJ ), + T01("entry-stats-end", K_ENTRY_END, ARGS, NO_OBJ ), + T01("entry-ips", K_ENTRY_IPS, ARGS, NO_OBJ ), + T01("cell-stats-end", K_CELL_END, ARGS, NO_OBJ ), + T01("cell-processed-cells", K_CELL_PROCESSED, ARGS, NO_OBJ ), + T01("cell-queued-cells", K_CELL_QUEUED, ARGS, NO_OBJ ), + T01("cell-time-in-queue", K_CELL_TIME, ARGS, NO_OBJ ), + T01("cell-circuits-per-decile", K_CELL_CIRCS, ARGS, NO_OBJ ), + T01("exit-stats-end", K_EXIT_END, ARGS, NO_OBJ ), + T01("exit-kibibytes-written", K_EXIT_WRITTEN, ARGS, NO_OBJ ), + T01("exit-kibibytes-read", K_EXIT_READ, ARGS, NO_OBJ ), + T01("exit-streams-opened", K_EXIT_OPENED, ARGS, NO_OBJ ), + T1_START( "extra-info", K_EXTRA_INFO, GE(2), NO_OBJ ), END_OF_TABLE @@ -267,10 +337,11 @@ static token_rule_t extrainfo_token_table[] = { * documents. */ static token_rule_t rtrstatus_token_table[] = { T01("p", K_P, CONCAT_ARGS, NO_OBJ ), - T1( "r", K_R, GE(8), NO_OBJ ), + T1( "r", K_R, GE(7), NO_OBJ ), T1( "s", K_S, ARGS, NO_OBJ ), T01("v", K_V, CONCAT_ARGS, NO_OBJ ), T01("w", K_W, ARGS, NO_OBJ ), + T0N("m", K_M, CONCAT_ARGS, NO_OBJ ), T0N("opt", K_OPT, CONCAT_ARGS, OBJ_OK ), END_OF_TABLE }; @@ -375,7 +446,7 @@ static token_rule_t client_keys_token_table[] = { /** List of tokens allowed in V3 networkstatus votes. */ static token_rule_t networkstatus_token_table[] = { - T1("network-status-version", K_NETWORK_STATUS_VERSION, + T1_START("network-status-version", K_NETWORK_STATUS_VERSION, GE(1), NO_OBJ ), T1("vote-status", K_VOTE_STATUS, GE(1), NO_OBJ ), T1("published", K_PUBLISHED, CONCAT_ARGS, NO_OBJ ), @@ -403,7 +474,7 @@ static token_rule_t networkstatus_token_table[] = { /** List of tokens allowed in V3 networkstatus consensuses. */ static token_rule_t networkstatus_consensus_token_table[] = { - T1("network-status-version", K_NETWORK_STATUS_VERSION, + T1_START("network-status-version", K_NETWORK_STATUS_VERSION, GE(1), NO_OBJ ), T1("vote-status", K_VOTE_STATUS, GE(1), NO_OBJ ), T1("valid-after", K_VALID_AFTER, CONCAT_ARGS, NO_OBJ ), @@ -430,17 +501,29 @@ static token_rule_t networkstatus_consensus_token_table[] = { /** List of tokens allowable in the footer of v1/v2 directory/networkstatus * footers. */ static token_rule_t networkstatus_vote_footer_token_table[] = { - T( "directory-signature", K_DIRECTORY_SIGNATURE, GE(2), NEED_OBJ ), + T01("directory-footer", K_DIRECTORY_FOOTER, NO_ARGS, NO_OBJ ), + T01("bandwidth-weights", K_BW_WEIGHTS, ARGS, NO_OBJ ), + T( "directory-signature", K_DIRECTORY_SIGNATURE, GE(2), NEED_OBJ ), END_OF_TABLE }; /** List of tokens allowable in detached networkstatus signature documents. */ static token_rule_t networkstatus_detached_signature_token_table[] = { T1_START("consensus-digest", K_CONSENSUS_DIGEST, GE(1), NO_OBJ ), + T("additional-digest", K_ADDITIONAL_DIGEST,GE(3), NO_OBJ ), T1("valid-after", K_VALID_AFTER, CONCAT_ARGS, NO_OBJ ), T1("fresh-until", K_FRESH_UNTIL, CONCAT_ARGS, NO_OBJ ), T1("valid-until", K_VALID_UNTIL, CONCAT_ARGS, NO_OBJ ), - T1N("directory-signature", K_DIRECTORY_SIGNATURE, GE(2), NEED_OBJ ), + T("additional-signature", K_ADDITIONAL_SIGNATURE, GE(4), NEED_OBJ ), + T1N("directory-signature", K_DIRECTORY_SIGNATURE, GE(2), NEED_OBJ ), + END_OF_TABLE +}; + +static token_rule_t microdesc_token_table[] = { + T1_START("onion-key", K_ONION_KEY, NO_ARGS, NEED_KEY_1024), + T01("family", K_FAMILY, ARGS, NO_OBJ ), + T01("p", K_P, CONCAT_ARGS, NO_OBJ ), + A01("@last-listed", A_LAST_LISTED, CONCAT_ARGS, NO_OBJ ), END_OF_TABLE }; @@ -453,9 +536,13 @@ static addr_policy_t *router_parse_addr_policy_private(directory_token_t *tok); static int router_get_hash_impl(const char *s, size_t s_len, char *digest, const char *start_str, const char *end_str, - char end_char); - -static void token_free(directory_token_t *tok); + char end_char, + digest_algorithm_t alg); +static int router_get_hashes_impl(const char *s, size_t s_len, + digests_t *digests, + const char *start_str, const char *end_str, + char end_char); +static void token_clear(directory_token_t *tok); static smartlist_t *find_all_exitpolicy(smartlist_t *s); static directory_token_t *_find_by_keyword(smartlist_t *s, directory_keyword keyword, @@ -479,6 +566,7 @@ static directory_token_t *get_next_token(memarea_t *area, #define CST_CHECK_AUTHORITY (1<<0) #define CST_NO_CHECK_OBJTYPE (1<<1) static int check_signature_token(const char *digest, + ssize_t digest_len, directory_token_t *tok, crypto_pk_env_t *pkey, int flags, @@ -499,6 +587,34 @@ static int tor_version_same_series(tor_version_t *a, tor_version_t *b); #define DUMP_AREA(a,name) STMT_NIL #endif +/** Last time we dumped a descriptor to disk. */ +static time_t last_desc_dumped = 0; + +/** For debugging purposes, dump unparseable descriptor *<b>desc</b> of + * type *<b>type</b> to file $DATADIR/unparseable-desc. Do not write more + * than one descriptor to disk per minute. If there is already such a + * file in the data directory, overwrite it. */ +static void +dump_desc(const char *desc, const char *type) +{ + time_t now = time(NULL); + tor_assert(desc); + tor_assert(type); + if (!last_desc_dumped || last_desc_dumped + 60 < now) { + char *debugfile = get_datadir_fname("unparseable-desc"); + size_t filelen = 50 + strlen(type) + strlen(desc); + char *content = tor_malloc_zero(filelen); + tor_snprintf(content, filelen, "Unable to parse descriptor of type " + "%s:\n%s", type, desc); + write_str_to_file(debugfile, content, 0); + log_info(LD_DIR, "Unable to parse descriptor of type %s. See file " + "unparseable-desc in data directory for details.", type); + tor_free(content); + tor_free(debugfile); + last_desc_dumped = now; + } +} + /** Set <b>digest</b> to the SHA-1 digest of the hash of the directory in * <b>s</b>. Return 0 on success, -1 on failure. */ @@ -506,7 +622,8 @@ int router_get_dir_hash(const char *s, char *digest) { return router_get_hash_impl(s, strlen(s), digest, - "signed-directory","\ndirectory-signature",'\n'); + "signed-directory","\ndirectory-signature",'\n', + DIGEST_SHA1); } /** Set <b>digest</b> to the SHA-1 digest of the hash of the first router in @@ -516,7 +633,8 @@ int router_get_router_hash(const char *s, size_t s_len, char *digest) { return router_get_hash_impl(s, s_len, digest, - "router ","\nrouter-signature", '\n'); + "router ","\nrouter-signature", '\n', + DIGEST_SHA1); } /** Set <b>digest</b> to the SHA-1 digest of the hash of the running-routers @@ -526,7 +644,8 @@ int router_get_runningrouters_hash(const char *s, char *digest) { return router_get_hash_impl(s, strlen(s), digest, - "network-status","\ndirectory-signature", '\n'); + "network-status","\ndirectory-signature", '\n', + DIGEST_SHA1); } /** Set <b>digest</b> to the SHA-1 digest of the hash of the network-status @@ -536,18 +655,31 @@ router_get_networkstatus_v2_hash(const char *s, char *digest) { return router_get_hash_impl(s, strlen(s), digest, "network-status-version","\ndirectory-signature", - '\n'); + '\n', + DIGEST_SHA1); +} + +/** Set <b>digests</b> to all the digests of the consensus document in + * <b>s</b> */ +int +router_get_networkstatus_v3_hashes(const char *s, digests_t *digests) +{ + return router_get_hashes_impl(s,strlen(s),digests, + "network-status-version", + "\ndirectory-signature", + ' '); } /** Set <b>digest</b> to the SHA-1 digest of the hash of the network-status * string in <b>s</b>. Return 0 on success, -1 on failure. */ int -router_get_networkstatus_v3_hash(const char *s, char *digest) +router_get_networkstatus_v3_hash(const char *s, char *digest, + digest_algorithm_t alg) { return router_get_hash_impl(s, strlen(s), digest, "network-status-version", "\ndirectory-signature", - ' '); + ' ', alg); } /** Set <b>digest</b> to the SHA-1 digest of the hash of the extrainfo @@ -556,7 +688,7 @@ int router_get_extrainfo_hash(const char *s, char *digest) { return router_get_hash_impl(s, strlen(s), digest, "extra-info", - "\nrouter-signature",'\n'); + "\nrouter-signature",'\n', DIGEST_SHA1); } /** Helper: used to generate signatures for routers, directories and @@ -568,16 +700,17 @@ router_get_extrainfo_hash(const char *s, char *digest) */ int router_append_dirobj_signature(char *buf, size_t buf_len, const char *digest, - crypto_pk_env_t *private_key) + size_t digest_len, crypto_pk_env_t *private_key) { char *signature; size_t i, keysize; + int siglen; keysize = crypto_pk_keysize(private_key); signature = tor_malloc(keysize); - if (crypto_pk_private_sign(private_key, signature, keysize, - digest, DIGEST_LEN) < 0) { - + siglen = crypto_pk_private_sign(private_key, signature, keysize, + digest, digest_len); + if (siglen < 0) { log_warn(LD_BUG,"Couldn't sign digest."); goto err; } @@ -585,7 +718,7 @@ router_append_dirobj_signature(char *buf, size_t buf_len, const char *digest, goto truncated; i = strlen(buf); - if (base64_encode(buf+i, buf_len-i, signature, 128) < 0) { + if (base64_encode(buf+i, buf_len-i, signature, siglen) < 0) { log_warn(LD_BUG,"couldn't base64-encode signature"); goto err; } @@ -692,7 +825,7 @@ router_parse_directory(const char *str) char digest[DIGEST_LEN]; time_t published_on; int r; - const char *end, *cp; + const char *end, *cp, *str_dup = str; smartlist_t *tokens = NULL; crypto_pk_env_t *declared_key = NULL; memarea_t *area = memarea_new(); @@ -728,11 +861,11 @@ router_parse_directory(const char *str) } declared_key = find_dir_signing_key(str, str+strlen(str)); note_crypto_pk_op(VERIFY_DIR); - if (check_signature_token(digest, tok, declared_key, + if (check_signature_token(digest, DIGEST_LEN, tok, declared_key, CST_CHECK_AUTHORITY, "directory")<0) goto err; - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_clear(tokens); memarea_clear(area); @@ -765,11 +898,12 @@ router_parse_directory(const char *str) r = 0; goto done; err: + dump_desc(str_dup, "v1 directory"); r = -1; done: if (declared_key) crypto_free_pk_env(declared_key); if (tokens) { - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_free(tokens); } if (area) { @@ -791,7 +925,7 @@ router_parse_runningrouters(const char *str) int r = -1; crypto_pk_env_t *declared_key = NULL; smartlist_t *tokens = NULL; - const char *eos = str + strlen(str); + const char *eos = str + strlen(str), *str_dup = str; memarea_t *area = NULL; if (router_get_runningrouters_hash(str, digest)) { @@ -820,7 +954,7 @@ router_parse_runningrouters(const char *str) } declared_key = find_dir_signing_key(str, eos); note_crypto_pk_op(VERIFY_DIR); - if (check_signature_token(digest, tok, declared_key, + if (check_signature_token(digest, DIGEST_LEN, tok, declared_key, CST_CHECK_AUTHORITY, "running-routers") < 0) goto err; @@ -832,9 +966,10 @@ router_parse_runningrouters(const char *str) r = 0; err: + dump_desc(str_dup, "v1 running-routers"); if (declared_key) crypto_free_pk_env(declared_key); if (tokens) { - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_free(tokens); } if (area) { @@ -884,7 +1019,7 @@ find_dir_signing_key(const char *str, const char *eos) } done: - if (tok) token_free(tok); + if (tok) token_clear(tok); if (area) { DUMP_AREA(area, "dir-signing-key token"); memarea_drop_all(area); @@ -920,6 +1055,7 @@ dir_signing_key_is_trusted(crypto_pk_env_t *key) */ static int check_signature_token(const char *digest, + ssize_t digest_len, directory_token_t *tok, crypto_pk_env_t *pkey, int flags, @@ -952,14 +1088,14 @@ check_signature_token(const char *digest, signed_digest = tor_malloc(keysize); if (crypto_pk_public_checksig(pkey, signed_digest, keysize, tok->object_body, tok->object_size) - != DIGEST_LEN) { + < digest_len) { log_warn(LD_DIR, "Error reading %s: invalid signature.", doctype); tor_free(signed_digest); return -1; } // log_debug(LD_DIR,"Signed %s hash starts %s", doctype, // hex_str(signed_digest,4)); - if (tor_memneq(digest, signed_digest, DIGEST_LEN)) { + if (tor_memneq(digest, signed_digest, digest_len)) { log_warn(LD_DIR, "Error reading %s: signature does not match.", doctype); tor_free(signed_digest); return -1; @@ -1145,7 +1281,7 @@ router_parse_entry_from_string(const char *s, const char *end, smartlist_t *tokens = NULL, *exit_policy_tokens = NULL; directory_token_t *tok; struct in_addr in; - const char *start_of_annotations, *cp; + const char *start_of_annotations, *cp, *s_dup = s; size_t prepend_len = prepend_annotations ? strlen(prepend_annotations) : 0; int ok = 1; memarea_t *area = NULL; @@ -1429,7 +1565,7 @@ router_parse_entry_from_string(const char *s, const char *end, verified_digests = digestmap_new(); digestmap_set(verified_digests, signed_digest, (void*)(uintptr_t)1); #endif - if (check_signature_token(digest, tok, router->identity_pkey, 0, + if (check_signature_token(digest, DIGEST_LEN, tok, router->identity_pkey, 0, "router descriptor") < 0) goto err; @@ -1447,16 +1583,15 @@ router_parse_entry_from_string(const char *s, const char *end, goto done; err: + dump_desc(s_dup, "router descriptor"); routerinfo_free(router); router = NULL; done: if (tokens) { - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_free(tokens); } - if (exit_policy_tokens) { - smartlist_free(exit_policy_tokens); - } + smartlist_free(exit_policy_tokens); if (area) { DUMP_AREA(area, "routerinfo"); memarea_drop_all(area); @@ -1481,6 +1616,7 @@ extrainfo_parse_entry_from_string(const char *s, const char *end, crypto_pk_env_t *key = NULL; routerinfo_t *router = NULL; memarea_t *area = NULL; + const char *s_dup = s; if (!end) { end = s + strlen(s); @@ -1555,7 +1691,8 @@ extrainfo_parse_entry_from_string(const char *s, const char *end, if (key) { note_crypto_pk_op(VERIFY_RTR); - if (check_signature_token(digest, tok, key, 0, "extra-info") < 0) + if (check_signature_token(digest, DIGEST_LEN, tok, key, 0, + "extra-info") < 0) goto err; if (router) @@ -1569,12 +1706,12 @@ extrainfo_parse_entry_from_string(const char *s, const char *end, goto done; err: - if (extrainfo) - extrainfo_free(extrainfo); + dump_desc(s_dup, "extra-info descriptor"); + extrainfo_free(extrainfo); extrainfo = NULL; done: if (tokens) { - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_free(tokens); } if (area) { @@ -1602,6 +1739,7 @@ authority_cert_parse_from_string(const char *s, const char **end_of_string) size_t len; int found; memarea_t *area = NULL; + const char *s_dup = s; s = eat_whitespace(s); eos = strstr(s, "\ndir-key-certification"); @@ -1632,7 +1770,7 @@ authority_cert_parse_from_string(const char *s, const char **end_of_string) goto err; } if (router_get_hash_impl(s, strlen(s), digest, "dir-key-certificate-version", - "\ndir-key-certification", '\n') < 0) + "\ndir-key-certification", '\n', DIGEST_SHA1) < 0) goto err; tok = smartlist_get(tokens, 0); if (tok->tp != K_DIR_KEY_CERTIFICATE_VERSION || strcmp(tok->args[0], "3")) { @@ -1680,7 +1818,7 @@ authority_cert_parse_from_string(const char *s, const char **end_of_string) struct in_addr in; char *address = NULL; tor_assert(tok->n_args); - /* XXX021 use tor_addr_port_parse() below instead. -RD */ + /* XXX023 use tor_addr_port_parse() below instead. -RD */ if (parse_addr_port(LOG_WARN, tok->args[0], &address, NULL, &cert->dir_port)<0 || tor_inet_aton(address, &in) == 0) { @@ -1725,7 +1863,7 @@ authority_cert_parse_from_string(const char *s, const char **end_of_string) } } if (!found) { - if (check_signature_token(digest, tok, cert->identity_key, 0, + if (check_signature_token(digest, DIGEST_LEN, tok, cert->identity_key, 0, "key certificate")) { goto err; } @@ -1734,6 +1872,7 @@ authority_cert_parse_from_string(const char *s, const char **end_of_string) /* XXXX Once all authorities generate cross-certified certificates, * make this field mandatory. */ if (check_signature_token(cert->cache_info.identity_digest, + DIGEST_LEN, tok, cert->signing_key, CST_NO_CHECK_OBJTYPE, @@ -1753,7 +1892,7 @@ authority_cert_parse_from_string(const char *s, const char **end_of_string) if (end_of_string) { *end_of_string = eat_whitespace(eos); } - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_free(tokens); if (area) { DUMP_AREA(area, "authority cert"); @@ -1761,8 +1900,9 @@ authority_cert_parse_from_string(const char *s, const char **end_of_string) } return cert; err: + dump_desc(s_dup, "authority cert"); authority_cert_free(cert); - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_free(tokens); if (area) { DUMP_AREA(area, "authority cert"); @@ -1773,23 +1913,28 @@ authority_cert_parse_from_string(const char *s, const char **end_of_string) /** Helper: given a string <b>s</b>, return the start of the next router-status * object (starting with "r " at the start of a line). If none is found, - * return the start of the next directory signature. If none is found, return - * the end of the string. */ + * return the start of the directory footer, or the next directory signature. + * If none is found, return the end of the string. */ static INLINE const char * find_start_of_next_routerstatus(const char *s) { - const char *eos = strstr(s, "\nr "); - if (eos) { - const char *eos2 = tor_memstr(s, eos-s, "\ndirectory-signature"); - if (eos2 && eos2 < eos) - return eos2; - else - return eos+1; - } else { - if ((eos = strstr(s, "\ndirectory-signature"))) - return eos+1; - return s + strlen(s); - } + const char *eos, *footer, *sig; + if ((eos = strstr(s, "\nr "))) + ++eos; + else + eos = s + strlen(s); + + footer = tor_memstr(s, eos-s, "\ndirectory-footer"); + sig = tor_memstr(s, eos-s, "\ndirectory-signature"); + + if (footer && sig) + return MIN(footer, sig) + 1; + else if (footer) + return footer+1; + else if (sig) + return sig+1; + else + return eos; } /** Given a string at *<b>s</b>, containing a routerstatus object, and an @@ -1803,22 +1948,29 @@ find_start_of_next_routerstatus(const char *s) * If <b>consensus_method</b> is nonzero, this routerstatus is part of a * consensus, and we should parse it according to the method used to * make that consensus. + * + * Parse according to the syntax used by the consensus flavor <b>flav</b>. **/ static routerstatus_t * routerstatus_parse_entry_from_string(memarea_t *area, const char **s, smartlist_t *tokens, networkstatus_t *vote, vote_routerstatus_t *vote_rs, - int consensus_method) + int consensus_method, + consensus_flavor_t flav) { - const char *eos; + const char *eos, *s_dup = *s; routerstatus_t *rs = NULL; directory_token_t *tok; char timebuf[ISO_TIME_LEN+1]; struct in_addr in; + int offset = 0; tor_assert(tokens); tor_assert(bool_eq(vote, vote_rs)); + if (!consensus_method) + flav = FLAV_NS; + eos = find_start_of_next_routerstatus(*s); if (tokenize_string(area,*s, eos, tokens, rtrstatus_token_table,0)) { @@ -1830,7 +1982,15 @@ routerstatus_parse_entry_from_string(memarea_t *area, goto err; } tok = find_by_keyword(tokens, K_R); - tor_assert(tok->n_args >= 8); + tor_assert(tok->n_args >= 7); + if (flav == FLAV_NS) { + if (tok->n_args < 8) { + log_warn(LD_DIR, "Too few arguments to r"); + goto err; + } + } else { + offset = -1; + } if (vote_rs) { rs = &vote_rs->status; } else { @@ -1851,29 +2011,34 @@ routerstatus_parse_entry_from_string(memarea_t *area, goto err; } - if (digest_from_base64(rs->descriptor_digest, tok->args[2])) { - log_warn(LD_DIR, "Error decoding descriptor digest %s", - escaped(tok->args[2])); - goto err; + if (flav == FLAV_NS) { + if (digest_from_base64(rs->descriptor_digest, tok->args[2])) { + log_warn(LD_DIR, "Error decoding descriptor digest %s", + escaped(tok->args[2])); + goto err; + } } if (tor_snprintf(timebuf, sizeof(timebuf), "%s %s", - tok->args[3], tok->args[4]) < 0 || + tok->args[3+offset], tok->args[4+offset]) < 0 || parse_iso_time(timebuf, &rs->published_on)<0) { - log_warn(LD_DIR, "Error parsing time '%s %s'", - tok->args[3], tok->args[4]); + log_warn(LD_DIR, "Error parsing time '%s %s' [%d %d]", + tok->args[3+offset], tok->args[4+offset], + offset, (int)flav); goto err; } - if (tor_inet_aton(tok->args[5], &in) == 0) { + if (tor_inet_aton(tok->args[5+offset], &in) == 0) { log_warn(LD_DIR, "Error parsing router address in network-status %s", - escaped(tok->args[5])); + escaped(tok->args[5+offset])); goto err; } rs->addr = ntohl(in.s_addr); - rs->or_port =(uint16_t) tor_parse_long(tok->args[6],10,0,65535,NULL,NULL); - rs->dir_port = (uint16_t) tor_parse_long(tok->args[7],10,0,65535,NULL,NULL); + rs->or_port = (uint16_t) tor_parse_long(tok->args[6+offset], + 10,0,65535,NULL,NULL); + rs->dir_port = (uint16_t) tor_parse_long(tok->args[7+offset], + 10,0,65535,NULL,NULL); tok = find_opt_by_keyword(tokens, K_S); if (tok && vote) { @@ -1959,6 +2124,17 @@ routerstatus_parse_entry_from_string(memarea_t *area, goto err; } rs->has_bandwidth = 1; + } else if (!strcmpstart(tok->args[i], "Measured=")) { + int ok; + rs->measured_bw = + (uint32_t)tor_parse_ulong(strchr(tok->args[i], '=')+1, + 10, 0, UINT32_MAX, &ok, NULL); + if (!ok) { + log_warn(LD_DIR, "Invalid Measured Bandwidth %s", + escaped(tok->args[i])); + goto err; + } + rs->has_measured_bw = 1; } } } @@ -1980,16 +2156,29 @@ routerstatus_parse_entry_from_string(memarea_t *area, rs->has_exitsummary = 1; } + if (vote_rs) { + SMARTLIST_FOREACH_BEGIN(tokens, directory_token_t *, t) { + if (t->tp == K_M && t->n_args) { + vote_microdesc_hash_t *line = + tor_malloc(sizeof(vote_microdesc_hash_t)); + line->next = vote_rs->microdesc; + line->microdesc_hash_line = tor_strdup(t->args[0]); + vote_rs->microdesc = line; + } + } SMARTLIST_FOREACH_END(t); + } + if (!strcasecmp(rs->nickname, UNNAMED_ROUTER_NICKNAME)) rs->is_named = 0; goto done; err: + dump_desc(s_dup, "routerstatus entry"); if (rs && !vote_rs) routerstatus_free(rs); rs = NULL; done: - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_clear(tokens); if (area) { DUMP_AREA(area, "routerstatus entry"); @@ -2001,8 +2190,8 @@ routerstatus_parse_entry_from_string(memarea_t *area, } /** Helper to sort a smartlist of pointers to routerstatus_t */ -static int -_compare_routerstatus_entries(const void **_a, const void **_b) +int +compare_routerstatus_entries(const void **_a, const void **_b) { const routerstatus_t *a = *_a, *b = *_b; return fast_memcmp(a->identity_digest, b->identity_digest, DIGEST_LEN); @@ -2026,7 +2215,7 @@ _free_duplicate_routerstatus_entry(void *e) networkstatus_v2_t * networkstatus_v2_parse_from_string(const char *s) { - const char *eos; + const char *eos, *s_dup = s; smartlist_t *tokens = smartlist_create(); smartlist_t *footer_tokens = smartlist_create(); networkstatus_v2_t *ns = NULL; @@ -2140,17 +2329,17 @@ networkstatus_v2_parse_from_string(const char *s) ns->entries = smartlist_create(); s = eos; - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_clear(tokens); memarea_clear(area); while (!strcmpstart(s, "r ")) { routerstatus_t *rs; if ((rs = routerstatus_parse_entry_from_string(area, &s, tokens, - NULL, NULL, 0))) + NULL, NULL, 0, 0))) smartlist_add(ns->entries, rs); } - smartlist_sort(ns->entries, _compare_routerstatus_entries); - smartlist_uniq(ns->entries, _compare_routerstatus_entries, + smartlist_sort(ns->entries, compare_routerstatus_entries); + smartlist_uniq(ns->entries, compare_routerstatus_entries, _free_duplicate_routerstatus_entry); if (tokenize_string(area,s, NULL, footer_tokens, dir_footer_token_table,0)) { @@ -2169,19 +2358,19 @@ networkstatus_v2_parse_from_string(const char *s) } note_crypto_pk_op(VERIFY_DIR); - if (check_signature_token(ns_digest, tok, ns->signing_key, 0, + if (check_signature_token(ns_digest, DIGEST_LEN, tok, ns->signing_key, 0, "network-status") < 0) goto err; goto done; err: - if (ns) - networkstatus_v2_free(ns); + dump_desc(s_dup, "v2 networkstatus"); + networkstatus_v2_free(ns); ns = NULL; done: - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_free(tokens); - SMARTLIST_FOREACH(footer_tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(footer_tokens, directory_token_t *, t, token_clear(t)); smartlist_free(footer_tokens); if (area) { DUMP_AREA(area, "v2 networkstatus"); @@ -2190,6 +2379,395 @@ networkstatus_v2_parse_from_string(const char *s) return ns; } +/** Verify the bandwidth weights of a network status document */ +int +networkstatus_verify_bw_weights(networkstatus_t *ns) +{ + int64_t weight_scale; + int64_t G=0, M=0, E=0, D=0, T=0; + double Wgg, Wgm, Wgd, Wmg, Wmm, Wme, Wmd, Weg, Wem, Wee, Wed; + double Gtotal=0, Mtotal=0, Etotal=0; + const char *casename = NULL; + int valid = 1; + + weight_scale = circuit_build_times_get_bw_scale(ns); + Wgg = networkstatus_get_bw_weight(ns, "Wgg", -1); + Wgm = networkstatus_get_bw_weight(ns, "Wgm", -1); + Wgd = networkstatus_get_bw_weight(ns, "Wgd", -1); + Wmg = networkstatus_get_bw_weight(ns, "Wmg", -1); + Wmm = networkstatus_get_bw_weight(ns, "Wmm", -1); + Wme = networkstatus_get_bw_weight(ns, "Wme", -1); + Wmd = networkstatus_get_bw_weight(ns, "Wmd", -1); + Weg = networkstatus_get_bw_weight(ns, "Weg", -1); + Wem = networkstatus_get_bw_weight(ns, "Wem", -1); + Wee = networkstatus_get_bw_weight(ns, "Wee", -1); + Wed = networkstatus_get_bw_weight(ns, "Wed", -1); + + if (Wgg<0 || Wgm<0 || Wgd<0 || Wmg<0 || Wmm<0 || Wme<0 || Wmd<0 || Weg<0 + || Wem<0 || Wee<0 || Wed<0) { + log_warn(LD_BUG, "No bandwidth weights produced in consensus!"); + return 0; + } + + // First, sanity check basic summing properties that hold for all cases + // We use > 1 as the check for these because they are computed as integers. + // Sometimes there are rounding errors. + if (fabs(Wmm - weight_scale) > 1) { + log_warn(LD_BUG, "Wmm=%lf != "I64_FORMAT, + Wmm, I64_PRINTF_ARG(weight_scale)); + valid = 0; + } + + if (fabs(Wem - Wee) > 1) { + log_warn(LD_BUG, "Wem=%lf != Wee=%lf", Wem, Wee); + valid = 0; + } + + if (fabs(Wgm - Wgg) > 1) { + log_warn(LD_BUG, "Wgm=%lf != Wgg=%lf", Wgm, Wgg); + valid = 0; + } + + if (fabs(Weg - Wed) > 1) { + log_warn(LD_BUG, "Wed=%lf != Weg=%lf", Wed, Weg); + valid = 0; + } + + if (fabs(Wgg + Wmg - weight_scale) > 0.001*weight_scale) { + log_warn(LD_BUG, "Wgg=%lf != "I64_FORMAT" - Wmg=%lf", Wgg, + I64_PRINTF_ARG(weight_scale), Wmg); + valid = 0; + } + + if (fabs(Wee + Wme - weight_scale) > 0.001*weight_scale) { + log_warn(LD_BUG, "Wee=%lf != "I64_FORMAT" - Wme=%lf", Wee, + I64_PRINTF_ARG(weight_scale), Wme); + valid = 0; + } + + if (fabs(Wgd + Wmd + Wed - weight_scale) > 0.001*weight_scale) { + log_warn(LD_BUG, "Wgd=%lf + Wmd=%lf + Wed=%lf != "I64_FORMAT, + Wgd, Wmd, Wed, I64_PRINTF_ARG(weight_scale)); + valid = 0; + } + + Wgg /= weight_scale; + Wgm /= weight_scale; + Wgd /= weight_scale; + + Wmg /= weight_scale; + Wmm /= weight_scale; + Wme /= weight_scale; + Wmd /= weight_scale; + + Weg /= weight_scale; + Wem /= weight_scale; + Wee /= weight_scale; + Wed /= weight_scale; + + // Then, gather G, M, E, D, T to determine case + SMARTLIST_FOREACH_BEGIN(ns->routerstatus_list, routerstatus_t *, rs) { + if (rs->has_bandwidth) { + T += rs->bandwidth; + if (rs->is_exit && rs->is_possible_guard) { + D += rs->bandwidth; + Gtotal += Wgd*rs->bandwidth; + Mtotal += Wmd*rs->bandwidth; + Etotal += Wed*rs->bandwidth; + } else if (rs->is_exit) { + E += rs->bandwidth; + Mtotal += Wme*rs->bandwidth; + Etotal += Wee*rs->bandwidth; + } else if (rs->is_possible_guard) { + G += rs->bandwidth; + Gtotal += Wgg*rs->bandwidth; + Mtotal += Wmg*rs->bandwidth; + } else { + M += rs->bandwidth; + Mtotal += Wmm*rs->bandwidth; + } + } else { + log_warn(LD_BUG, "Missing consensus bandwidth for router %s", + rs->nickname); + } + } SMARTLIST_FOREACH_END(rs); + + // Finally, check equality conditions depending upon case 1, 2 or 3 + // Full equality cases: 1, 3b + // Partial equality cases: 2b (E=G), 3a (M=E) + // Fully unknown: 2a + if (3*E >= T && 3*G >= T) { + // Case 1: Neither are scarce + casename = "Case 1"; + if (fabs(Etotal-Mtotal) > 0.01*MAX(Etotal,Mtotal)) { + log_warn(LD_DIR, + "Bw Weight Failure for %s: Etotal %lf != Mtotal %lf. " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT". " + "Wgg=%lf Wgd=%lf Wmg=%lf Wme=%lf Wmd=%lf Wee=%lf Wed=%lf", + casename, Etotal, Mtotal, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed); + valid = 0; + } + if (fabs(Etotal-Gtotal) > 0.01*MAX(Etotal,Gtotal)) { + log_warn(LD_DIR, + "Bw Weight Failure for %s: Etotal %lf != Gtotal %lf. " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT". " + "Wgg=%lf Wgd=%lf Wmg=%lf Wme=%lf Wmd=%lf Wee=%lf Wed=%lf", + casename, Etotal, Gtotal, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed); + valid = 0; + } + if (fabs(Gtotal-Mtotal) > 0.01*MAX(Gtotal,Mtotal)) { + log_warn(LD_DIR, + "Bw Weight Failure for %s: Mtotal %lf != Gtotal %lf. " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT". " + "Wgg=%lf Wgd=%lf Wmg=%lf Wme=%lf Wmd=%lf Wee=%lf Wed=%lf", + casename, Mtotal, Gtotal, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed); + valid = 0; + } + } else if (3*E < T && 3*G < T) { + int64_t R = MIN(E, G); + int64_t S = MAX(E, G); + /* + * Case 2: Both Guards and Exits are scarce + * Balance D between E and G, depending upon + * D capacity and scarcity. Devote no extra + * bandwidth to middle nodes. + */ + if (R+D < S) { // Subcase a + double Rtotal, Stotal; + if (E < G) { + Rtotal = Etotal; + Stotal = Gtotal; + } else { + Rtotal = Gtotal; + Stotal = Etotal; + } + casename = "Case 2a"; + // Rtotal < Stotal + if (Rtotal > Stotal) { + log_warn(LD_DIR, + "Bw Weight Failure for %s: Rtotal %lf > Stotal %lf. " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT". " + "Wgg=%lf Wgd=%lf Wmg=%lf Wme=%lf Wmd=%lf Wee=%lf Wed=%lf", + casename, Rtotal, Stotal, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed); + valid = 0; + } + // Rtotal < T/3 + if (3*Rtotal > T) { + log_warn(LD_DIR, + "Bw Weight Failure for %s: 3*Rtotal %lf > T " + I64_FORMAT". G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT + " D="I64_FORMAT" T="I64_FORMAT". " + "Wgg=%lf Wgd=%lf Wmg=%lf Wme=%lf Wmd=%lf Wee=%lf Wed=%lf", + casename, Rtotal*3, I64_PRINTF_ARG(T), + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed); + valid = 0; + } + // Stotal < T/3 + if (3*Stotal > T) { + log_warn(LD_DIR, + "Bw Weight Failure for %s: 3*Stotal %lf > T " + I64_FORMAT". G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT + " D="I64_FORMAT" T="I64_FORMAT". " + "Wgg=%lf Wgd=%lf Wmg=%lf Wme=%lf Wmd=%lf Wee=%lf Wed=%lf", + casename, Stotal*3, I64_PRINTF_ARG(T), + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed); + valid = 0; + } + // Mtotal > T/3 + if (3*Mtotal < T) { + log_warn(LD_DIR, + "Bw Weight Failure for %s: 3*Mtotal %lf < T " + I64_FORMAT". " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT". " + "Wgg=%lf Wgd=%lf Wmg=%lf Wme=%lf Wmd=%lf Wee=%lf Wed=%lf", + casename, Mtotal*3, I64_PRINTF_ARG(T), + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed); + valid = 0; + } + } else { // Subcase b: R+D > S + casename = "Case 2b"; + + /* Check the rare-M redirect case. */ + if (D != 0 && 3*M < T) { + casename = "Case 2b (balanced)"; + if (fabs(Etotal-Mtotal) > 0.01*MAX(Etotal,Mtotal)) { + log_warn(LD_DIR, + "Bw Weight Failure for %s: Etotal %lf != Mtotal %lf. " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT". " + "Wgg=%lf Wgd=%lf Wmg=%lf Wme=%lf Wmd=%lf Wee=%lf Wed=%lf", + casename, Etotal, Mtotal, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed); + valid = 0; + } + if (fabs(Etotal-Gtotal) > 0.01*MAX(Etotal,Gtotal)) { + log_warn(LD_DIR, + "Bw Weight Failure for %s: Etotal %lf != Gtotal %lf. " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT". " + "Wgg=%lf Wgd=%lf Wmg=%lf Wme=%lf Wmd=%lf Wee=%lf Wed=%lf", + casename, Etotal, Gtotal, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed); + valid = 0; + } + if (fabs(Gtotal-Mtotal) > 0.01*MAX(Gtotal,Mtotal)) { + log_warn(LD_DIR, + "Bw Weight Failure for %s: Mtotal %lf != Gtotal %lf. " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT". " + "Wgg=%lf Wgd=%lf Wmg=%lf Wme=%lf Wmd=%lf Wee=%lf Wed=%lf", + casename, Mtotal, Gtotal, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed); + valid = 0; + } + } else { + if (fabs(Etotal-Gtotal) > 0.01*MAX(Etotal,Gtotal)) { + log_warn(LD_DIR, + "Bw Weight Failure for %s: Etotal %lf != Gtotal %lf. " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT". " + "Wgg=%lf Wgd=%lf Wmg=%lf Wme=%lf Wmd=%lf Wee=%lf Wed=%lf", + casename, Etotal, Gtotal, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed); + valid = 0; + } + } + } + } else { // if (E < T/3 || G < T/3) { + int64_t S = MIN(E, G); + int64_t NS = MAX(E, G); + if (3*(S+D) < T) { // Subcase a: + double Stotal; + double NStotal; + if (G < E) { + casename = "Case 3a (G scarce)"; + Stotal = Gtotal; + NStotal = Etotal; + } else { // if (G >= E) { + casename = "Case 3a (E scarce)"; + NStotal = Gtotal; + Stotal = Etotal; + } + // Stotal < T/3 + if (3*Stotal > T) { + log_warn(LD_DIR, + "Bw Weight Failure for %s: 3*Stotal %lf > T " + I64_FORMAT". G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT + " D="I64_FORMAT" T="I64_FORMAT". " + "Wgg=%lf Wgd=%lf Wmg=%lf Wme=%lf Wmd=%lf Wee=%lf Wed=%lf", + casename, Stotal*3, I64_PRINTF_ARG(T), + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed); + valid = 0; + } + if (NS >= M) { + if (fabs(NStotal-Mtotal) > 0.01*MAX(NStotal,Mtotal)) { + log_warn(LD_DIR, + "Bw Weight Failure for %s: NStotal %lf != Mtotal %lf. " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT". " + "Wgg=%lf Wgd=%lf Wmg=%lf Wme=%lf Wmd=%lf Wee=%lf Wed=%lf", + casename, NStotal, Mtotal, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed); + valid = 0; + } + } else { + // if NS < M, NStotal > T/3 because only one of G or E is scarce + if (3*NStotal < T) { + log_warn(LD_DIR, + "Bw Weight Failure for %s: 3*NStotal %lf < T " + I64_FORMAT". G="I64_FORMAT" M="I64_FORMAT + " E="I64_FORMAT" D="I64_FORMAT" T="I64_FORMAT". " + "Wgg=%lf Wgd=%lf Wmg=%lf Wme=%lf Wmd=%lf Wee=%lf Wed=%lf", + casename, NStotal*3, I64_PRINTF_ARG(T), + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed); + valid = 0; + } + } + } else { // Subcase b: S+D >= T/3 + casename = "Case 3b"; + if (fabs(Etotal-Mtotal) > 0.01*MAX(Etotal,Mtotal)) { + log_warn(LD_DIR, + "Bw Weight Failure for %s: Etotal %lf != Mtotal %lf. " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT". " + "Wgg=%lf Wgd=%lf Wmg=%lf Wme=%lf Wmd=%lf Wee=%lf Wed=%lf", + casename, Etotal, Mtotal, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed); + valid = 0; + } + if (fabs(Etotal-Gtotal) > 0.01*MAX(Etotal,Gtotal)) { + log_warn(LD_DIR, + "Bw Weight Failure for %s: Etotal %lf != Gtotal %lf. " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT". " + "Wgg=%lf Wgd=%lf Wmg=%lf Wme=%lf Wmd=%lf Wee=%lf Wed=%lf", + casename, Etotal, Gtotal, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed); + valid = 0; + } + if (fabs(Gtotal-Mtotal) > 0.01*MAX(Gtotal,Mtotal)) { + log_warn(LD_DIR, + "Bw Weight Failure for %s: Mtotal %lf != Gtotal %lf. " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT". " + "Wgg=%lf Wgd=%lf Wmg=%lf Wme=%lf Wmd=%lf Wee=%lf Wed=%lf", + casename, Mtotal, Gtotal, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed); + valid = 0; + } + } + } + + if (valid) + log_notice(LD_DIR, "Bandwidth-weight %s is verified and valid.", + casename); + + return valid; +} + /** Parse a v3 networkstatus vote, opinion, or consensus (depending on * ns_type), from <b>s</b>, and return the result. Return NULL on failure. */ networkstatus_t * @@ -2200,19 +2778,21 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, smartlist_t *rs_tokens = NULL, *footer_tokens = NULL; networkstatus_voter_info_t *voter = NULL; networkstatus_t *ns = NULL; - char ns_digest[DIGEST_LEN]; - const char *cert, *end_of_header, *end_of_footer; + digests_t ns_digests; + const char *cert, *end_of_header, *end_of_footer, *s_dup = s; directory_token_t *tok; int ok; struct in_addr in; int i, inorder, n_signatures = 0; memarea_t *area = NULL, *rs_area = NULL; + consensus_flavor_t flav = FLAV_NS; + tor_assert(s); if (eos_out) *eos_out = NULL; - if (router_get_networkstatus_v3_hash(s, ns_digest)) { + if (router_get_networkstatus_v3_hashes(s, &ns_digests)) { log_warn(LD_DIR, "Unable to compute digest of network-status"); goto err; } @@ -2228,7 +2808,23 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, } ns = tor_malloc_zero(sizeof(networkstatus_t)); - memcpy(ns->networkstatus_digest, ns_digest, DIGEST_LEN); + memcpy(&ns->digests, &ns_digests, sizeof(ns_digests)); + + tok = find_by_keyword(tokens, K_NETWORK_STATUS_VERSION); + tor_assert(tok); + if (tok->n_args > 1) { + int flavor = networkstatus_parse_flavor_name(tok->args[1]); + if (flavor < 0) { + log_warn(LD_DIR, "Can't parse document with unknown flavor %s", + escaped(tok->args[2])); + goto err; + } + ns->flavor = flav = flavor; + } + if (flav != FLAV_NS && ns_type != NS_TYPE_CONSENSUS) { + log_warn(LD_DIR, "Flavor found on non-consensus networkstatus."); + goto err; + } if (ns_type != NS_TYPE_CONSENSUS) { const char *end_of_cert = NULL; @@ -2382,8 +2978,9 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, if (voter) smartlist_add(ns->voters, voter); voter = tor_malloc_zero(sizeof(networkstatus_voter_info_t)); + voter->sigs = smartlist_create(); if (ns->type != NS_TYPE_CONSENSUS) - memcpy(voter->vote_digest, ns_digest, DIGEST_LEN); + memcpy(voter->vote_digest, ns_digests.d[DIGEST_SHA1], DIGEST_LEN); voter->nickname = tor_strdup(tok->args[0]); if (strlen(tok->args[1]) != HEX_DIGEST_LEN || @@ -2475,7 +3072,7 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, if (ns->type != NS_TYPE_CONSENSUS) { vote_routerstatus_t *rs = tor_malloc_zero(sizeof(vote_routerstatus_t)); if (routerstatus_parse_entry_from_string(rs_area, &s, rs_tokens, ns, - rs, 0)) + rs, 0, 0)) smartlist_add(ns->routerstatus_list, rs); else { tor_free(rs->version); @@ -2485,7 +3082,8 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, routerstatus_t *rs; if ((rs = routerstatus_parse_entry_from_string(rs_area, &s, rs_tokens, NULL, NULL, - ns->consensus_method))) + ns->consensus_method, + flav))) smartlist_add(ns->routerstatus_list, rs); } } @@ -2519,14 +3117,73 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, goto err; } - SMARTLIST_FOREACH(footer_tokens, directory_token_t *, _tok, { + int found_sig = 0; + SMARTLIST_FOREACH_BEGIN(footer_tokens, directory_token_t *, _tok) { + tok = _tok; + if (tok->tp == K_DIRECTORY_SIGNATURE) + found_sig = 1; + else if (found_sig) { + log_warn(LD_DIR, "Extraneous token after first directory-signature"); + goto err; + } + } SMARTLIST_FOREACH_END(_tok); + } + + if ((tok = find_opt_by_keyword(footer_tokens, K_DIRECTORY_FOOTER))) { + if (tok != smartlist_get(footer_tokens, 0)) { + log_warn(LD_DIR, "Misplaced directory-footer token"); + goto err; + } + } + + tok = find_opt_by_keyword(footer_tokens, K_BW_WEIGHTS); + if (tok) { + ns->weight_params = smartlist_create(); + for (i = 0; i < tok->n_args; ++i) { + int ok=0; + char *eq = strchr(tok->args[i], '='); + if (!eq) { + log_warn(LD_DIR, "Bad element '%s' in weight params", + escaped(tok->args[i])); + goto err; + } + tor_parse_long(eq+1, 10, INT32_MIN, INT32_MAX, &ok, NULL); + if (!ok) { + log_warn(LD_DIR, "Bad element '%s' in params", escaped(tok->args[i])); + goto err; + } + smartlist_add(ns->weight_params, tor_strdup(tok->args[i])); + } + } + + SMARTLIST_FOREACH_BEGIN(footer_tokens, directory_token_t *, _tok) { char declared_identity[DIGEST_LEN]; networkstatus_voter_info_t *v; + document_signature_t *sig; + const char *id_hexdigest = NULL; + const char *sk_hexdigest = NULL; + digest_algorithm_t alg = DIGEST_SHA1; tok = _tok; if (tok->tp != K_DIRECTORY_SIGNATURE) continue; tor_assert(tok->n_args >= 2); + if (tok->n_args == 2) { + id_hexdigest = tok->args[0]; + sk_hexdigest = tok->args[1]; + } else { + const char *algname = tok->args[0]; + int a; + id_hexdigest = tok->args[1]; + sk_hexdigest = tok->args[2]; + a = crypto_digest_algorithm_parse_name(algname); + if (a<0) { + log_warn(LD_DIR, "Unknown digest algorithm %s; skipping", + escaped(algname)); + continue; + } + alg = a; + } if (!tok->object_type || strcmp(tok->object_type, "SIGNATURE") || @@ -2535,11 +3192,11 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, goto err; } - if (strlen(tok->args[0]) != HEX_DIGEST_LEN || + if (strlen(id_hexdigest) != HEX_DIGEST_LEN || base16_decode(declared_identity, sizeof(declared_identity), - tok->args[0], HEX_DIGEST_LEN) < 0) { + id_hexdigest, HEX_DIGEST_LEN) < 0) { log_warn(LD_DIR, "Error decoding declared identity %s in " - "network-status vote.", escaped(tok->args[0])); + "network-status vote.", escaped(id_hexdigest)); goto err; } if (!(v = networkstatus_get_voter_by_id(ns, declared_identity))) { @@ -2547,11 +3204,15 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, "any declared directory source."); goto err; } - if (strlen(tok->args[1]) != HEX_DIGEST_LEN || - base16_decode(v->signing_key_digest, sizeof(v->signing_key_digest), - tok->args[1], HEX_DIGEST_LEN) < 0) { - log_warn(LD_DIR, "Error decoding declared digest %s in " - "network-status vote.", escaped(tok->args[1])); + sig = tor_malloc_zero(sizeof(document_signature_t)); + memcpy(sig->identity_digest, v->identity_digest, DIGEST_LEN); + sig->alg = alg; + if (strlen(sk_hexdigest) != HEX_DIGEST_LEN || + base16_decode(sig->signing_key_digest, sizeof(sig->signing_key_digest), + sk_hexdigest, HEX_DIGEST_LEN) < 0) { + log_warn(LD_DIR, "Error decoding declared signing key digest %s in " + "network-status vote.", escaped(sk_hexdigest)); + tor_free(sig); goto err; } @@ -2560,35 +3221,49 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, DIGEST_LEN)) { log_warn(LD_DIR, "Digest mismatch between declared and actual on " "network-status vote."); + tor_free(sig); goto err; } } + if (voter_get_sig_by_algorithm(v, sig->alg)) { + /* We already parsed a vote with this algorithm from this voter. Use the + first one. */ + log_fn(LOG_PROTOCOL_WARN, LD_DIR, "We received a networkstatus " + "that contains two votes from the same voter with the same " + "algorithm. Ignoring the second vote."); + tor_free(sig); + continue; + } + if (ns->type != NS_TYPE_CONSENSUS) { - if (check_signature_token(ns_digest, tok, ns->cert->signing_key, 0, - "network-status vote")) + if (check_signature_token(ns_digests.d[DIGEST_SHA1], DIGEST_LEN, + tok, ns->cert->signing_key, 0, + "network-status vote")) { + tor_free(sig); goto err; - v->good_signature = 1; + } + sig->good_signature = 1; } else { - if (tok->object_size >= INT_MAX || tok->object_size >= SIZE_T_CEILING) + if (tok->object_size >= INT_MAX || tok->object_size >= SIZE_T_CEILING) { + tor_free(sig); goto err; - /* We already parsed a vote from this voter. Use the first one. */ - if (v->signature) { - log_fn(LOG_PROTOCOL_WARN, LD_DIR, "We received a networkstatus " - "that contains two votes from the same voter. Ignoring " - "the second vote."); - continue; } - - v->signature = tor_memdup(tok->object_body, tok->object_size); - v->signature_len = (int) tok->object_size; + sig->signature = tor_memdup(tok->object_body, tok->object_size); + sig->signature_len = (int) tok->object_size; } + smartlist_add(v->sigs, sig); + ++n_signatures; - }); + } SMARTLIST_FOREACH_END(_tok); if (! n_signatures) { log_warn(LD_DIR, "No signatures on networkstatus vote."); goto err; + } else if (ns->type == NS_TYPE_VOTE && n_signatures != 1) { + log_warn(LD_DIR, "Received more than one signature on a " + "network-status vote."); + goto err; } if (eos_out) @@ -2596,27 +3271,31 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, goto done; err: - if (ns) - networkstatus_vote_free(ns); + dump_desc(s_dup, "v3 networkstatus"); + networkstatus_vote_free(ns); ns = NULL; done: if (tokens) { - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_free(tokens); } if (voter) { + if (voter->sigs) { + SMARTLIST_FOREACH(voter->sigs, document_signature_t *, sig, + document_signature_free(sig)); + smartlist_free(voter->sigs); + } tor_free(voter->nickname); tor_free(voter->address); tor_free(voter->contact); - tor_free(voter->signature); tor_free(voter); } if (rs_tokens) { - SMARTLIST_FOREACH(rs_tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(rs_tokens, directory_token_t *, t, token_clear(t)); smartlist_free(rs_tokens); } if (footer_tokens) { - SMARTLIST_FOREACH(footer_tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(footer_tokens, directory_token_t *, t, token_clear(t)); smartlist_free(footer_tokens); } if (area) { @@ -2629,6 +3308,35 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, return ns; } +/** Return the digests_t that holds the digests of the + * <b>flavor_name</b>-flavored networkstatus according to the detached + * signatures document <b>sigs</b>, allocating a new digests_t as neeeded. */ +static digests_t * +detached_get_digests(ns_detached_signatures_t *sigs, const char *flavor_name) +{ + digests_t *d = strmap_get(sigs->digests, flavor_name); + if (!d) { + d = tor_malloc_zero(sizeof(digests_t)); + strmap_set(sigs->digests, flavor_name, d); + } + return d; +} + +/** Return the list of signatures of the <b>flavor_name</b>-flavored + * networkstatus according to the detached signatures document <b>sigs</b>, + * allocating a new digests_t as neeeded. */ +static smartlist_t * +detached_get_signatures(ns_detached_signatures_t *sigs, + const char *flavor_name) +{ + smartlist_t *sl = strmap_get(sigs->signatures, flavor_name); + if (!sl) { + sl = smartlist_create(); + strmap_set(sigs->signatures, flavor_name, sl); + } + return sl; +} + /** Parse a detached v3 networkstatus signature document between <b>s</b> and * <b>eos</b> and return the result. Return -1 on failure. */ ns_detached_signatures_t * @@ -2638,10 +3346,13 @@ networkstatus_parse_detached_signatures(const char *s, const char *eos) * networkstatus_parse_vote_from_string(). */ directory_token_t *tok; memarea_t *area = NULL; + digests_t *digests; smartlist_t *tokens = smartlist_create(); ns_detached_signatures_t *sigs = tor_malloc_zero(sizeof(ns_detached_signatures_t)); + sigs->digests = strmap_new(); + sigs->signatures = strmap_new(); if (!eos) eos = s + strlen(s); @@ -2653,18 +3364,57 @@ networkstatus_parse_detached_signatures(const char *s, const char *eos) goto err; } - tok = find_by_keyword(tokens, K_CONSENSUS_DIGEST); - if (strlen(tok->args[0]) != HEX_DIGEST_LEN) { - log_warn(LD_DIR, "Wrong length on consensus-digest in detached " - "networkstatus signatures"); - goto err; - } - if (base16_decode(sigs->networkstatus_digest, DIGEST_LEN, - tok->args[0], strlen(tok->args[0])) < 0) { - log_warn(LD_DIR, "Bad encoding on on consensus-digest in detached " - "networkstatus signatures"); - goto err; - } + /* Grab all the digest-like tokens. */ + SMARTLIST_FOREACH_BEGIN(tokens, directory_token_t *, _tok) { + const char *algname; + digest_algorithm_t alg; + const char *flavor; + const char *hexdigest; + size_t expected_length; + + tok = _tok; + + if (tok->tp == K_CONSENSUS_DIGEST) { + algname = "sha1"; + alg = DIGEST_SHA1; + flavor = "ns"; + hexdigest = tok->args[0]; + } else if (tok->tp == K_ADDITIONAL_DIGEST) { + int a = crypto_digest_algorithm_parse_name(tok->args[1]); + if (a<0) { + log_warn(LD_DIR, "Unrecognized algorithm name %s", tok->args[0]); + continue; + } + alg = (digest_algorithm_t) a; + flavor = tok->args[0]; + algname = tok->args[1]; + hexdigest = tok->args[2]; + } else { + continue; + } + + expected_length = + (alg == DIGEST_SHA1) ? HEX_DIGEST_LEN : HEX_DIGEST256_LEN; + + if (strlen(hexdigest) != expected_length) { + log_warn(LD_DIR, "Wrong length on consensus-digest in detached " + "networkstatus signatures"); + goto err; + } + digests = detached_get_digests(sigs, flavor); + tor_assert(digests); + if (!tor_mem_is_zero(digests->d[alg], DIGEST256_LEN)) { + log_warn(LD_DIR, "Multiple digests for %s with %s on detached " + "signatures document", flavor, algname); + continue; + } + if (base16_decode(digests->d[alg], DIGEST256_LEN, + hexdigest, strlen(hexdigest)) < 0) { + log_warn(LD_DIR, "Bad encoding on consensus-digest in detached " + "networkstatus signatures"); + goto err; + } + } SMARTLIST_FOREACH_END(_tok); tok = find_by_keyword(tokens, K_VALID_AFTER); if (parse_iso_time(tok->args[0], &sigs->valid_after)) { @@ -2684,57 +3434,102 @@ networkstatus_parse_detached_signatures(const char *s, const char *eos) goto err; } - sigs->signatures = smartlist_create(); - SMARTLIST_FOREACH(tokens, directory_token_t *, _tok, - { - char id_digest[DIGEST_LEN]; - char sk_digest[DIGEST_LEN]; - networkstatus_voter_info_t *voter; + SMARTLIST_FOREACH_BEGIN(tokens, directory_token_t *, _tok) { + const char *id_hexdigest; + const char *sk_hexdigest; + const char *algname; + const char *flavor; + digest_algorithm_t alg; + + char id_digest[DIGEST_LEN]; + char sk_digest[DIGEST_LEN]; + smartlist_t *siglist; + document_signature_t *sig; + int is_duplicate; - tok = _tok; - if (tok->tp != K_DIRECTORY_SIGNATURE) - continue; + tok = _tok; + if (tok->tp == K_DIRECTORY_SIGNATURE) { tor_assert(tok->n_args >= 2); + flavor = "ns"; + algname = "sha1"; + id_hexdigest = tok->args[0]; + sk_hexdigest = tok->args[1]; + } else if (tok->tp == K_ADDITIONAL_SIGNATURE) { + tor_assert(tok->n_args >= 4); + flavor = tok->args[0]; + algname = tok->args[1]; + id_hexdigest = tok->args[2]; + sk_hexdigest = tok->args[3]; + } else { + continue; + } - if (!tok->object_type || - strcmp(tok->object_type, "SIGNATURE") || - tok->object_size < 128 || tok->object_size > 512) { - log_warn(LD_DIR, "Bad object type or length on directory-signature"); - goto err; + { + int a = crypto_digest_algorithm_parse_name(algname); + if (a<0) { + log_warn(LD_DIR, "Unrecognized algorithm name %s", algname); + continue; } + alg = (digest_algorithm_t) a; + } - if (strlen(tok->args[0]) != HEX_DIGEST_LEN || - base16_decode(id_digest, sizeof(id_digest), - tok->args[0], HEX_DIGEST_LEN) < 0) { - log_warn(LD_DIR, "Error decoding declared identity %s in " - "network-status vote.", escaped(tok->args[0])); - goto err; - } - if (strlen(tok->args[1]) != HEX_DIGEST_LEN || - base16_decode(sk_digest, sizeof(sk_digest), - tok->args[1], HEX_DIGEST_LEN) < 0) { - log_warn(LD_DIR, "Error decoding declared digest %s in " - "network-status vote.", escaped(tok->args[1])); - goto err; - } + if (!tok->object_type || + strcmp(tok->object_type, "SIGNATURE") || + tok->object_size < 128 || tok->object_size > 512) { + log_warn(LD_DIR, "Bad object type or length on directory-signature"); + goto err; + } - voter = tor_malloc_zero(sizeof(networkstatus_voter_info_t)); - memcpy(voter->identity_digest, id_digest, DIGEST_LEN); - memcpy(voter->signing_key_digest, sk_digest, DIGEST_LEN); - if (tok->object_size >= INT_MAX || tok->object_size >= SIZE_T_CEILING) - goto err; - voter->signature = tor_memdup(tok->object_body, tok->object_size); - voter->signature_len = (int) tok->object_size; + if (strlen(id_hexdigest) != HEX_DIGEST_LEN || + base16_decode(id_digest, sizeof(id_digest), + id_hexdigest, HEX_DIGEST_LEN) < 0) { + log_warn(LD_DIR, "Error decoding declared identity %s in " + "network-status vote.", escaped(id_hexdigest)); + goto err; + } + if (strlen(sk_hexdigest) != HEX_DIGEST_LEN || + base16_decode(sk_digest, sizeof(sk_digest), + sk_hexdigest, HEX_DIGEST_LEN) < 0) { + log_warn(LD_DIR, "Error decoding declared signing key digest %s in " + "network-status vote.", escaped(sk_hexdigest)); + goto err; + } - smartlist_add(sigs->signatures, voter); + siglist = detached_get_signatures(sigs, flavor); + is_duplicate = 0; + SMARTLIST_FOREACH(siglist, document_signature_t *, s, { + if (s->alg == alg && + !memcmp(id_digest, s->identity_digest, DIGEST_LEN) && + !memcmp(sk_digest, s->signing_key_digest, DIGEST_LEN)) { + is_duplicate = 1; + } }); + if (is_duplicate) { + log_warn(LD_DIR, "Two signatures with identical keys and algorithm " + "found."); + continue; + } + + sig = tor_malloc_zero(sizeof(document_signature_t)); + sig->alg = alg; + memcpy(sig->identity_digest, id_digest, DIGEST_LEN); + memcpy(sig->signing_key_digest, sk_digest, DIGEST_LEN); + if (tok->object_size >= INT_MAX || tok->object_size >= SIZE_T_CEILING) { + tor_free(sig); + goto err; + } + sig->signature = tor_memdup(tok->object_body, tok->object_size); + sig->signature_len = (int) tok->object_size; + + smartlist_add(siglist, sig); + } SMARTLIST_FOREACH_END(_tok); goto done; err: ns_detached_signatures_free(sigs); sigs = NULL; done: - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_free(tokens); if (area) { DUMP_AREA(area, "detached signatures"); @@ -2790,7 +3585,7 @@ router_parse_addr_policy_item_from_string(const char *s, int assume_action) err: r = NULL; done: - token_free(tok); + token_clear(tok); if (area) { DUMP_AREA(area, "policy item"); memarea_drop_all(area); @@ -2913,9 +3708,8 @@ assert_addr_policy_ok(smartlist_t *lst) /** Free all resources allocated for <b>tok</b> */ static void -token_free(directory_token_t *tok) +token_clear(directory_token_t *tok) { - tor_assert(tok); if (tok->key) crypto_free_pk_env(tok->key); } @@ -2927,7 +3721,7 @@ token_free(directory_token_t *tok) #define RET_ERR(msg) \ STMT_BEGIN \ - if (tok) token_free(tok); \ + if (tok) token_clear(tok); \ tok = ALLOC_ZERO(sizeof(directory_token_t)); \ tok->tp = _ERR; \ tok->error = STRDUP(msg); \ @@ -3223,7 +4017,7 @@ tokenize_string(memarea_t *area, tok = get_next_token(area, s, end, table); if (tok->tp == _ERR) { log_warn(LD_DIR, "parse error: %s", tok->error); - token_free(tok); + token_clear(tok); return -1; } ++counts[tok->tp]; @@ -3337,17 +4131,11 @@ find_all_exitpolicy(smartlist_t *s) return out; } -/** Compute the SHA-1 digest of the substring of <b>s</b> taken from the first - * occurrence of <b>start_str</b> through the first instance of c after the - * first subsequent occurrence of <b>end_str</b>; store the 20-byte result in - * <b>digest</b>; return 0 on success. - * - * If no such substring exists, return -1. - */ static int -router_get_hash_impl(const char *s, size_t s_len, char *digest, +router_get_hash_impl_helper(const char *s, size_t s_len, const char *start_str, - const char *end_str, char end_c) + const char *end_str, char end_c, + const char **start_out, const char **end_out) { const char *start, *end; start = tor_memstr(s, s_len, start_str); @@ -3374,14 +4162,215 @@ router_get_hash_impl(const char *s, size_t s_len, char *digest, } ++end; - if (crypto_digest(digest, start, end-start)) { - log_warn(LD_BUG,"couldn't compute digest"); + *start_out = start; + *end_out = end; + return 0; +} + +/** Compute the digest of the substring of <b>s</b> taken from the first + * occurrence of <b>start_str</b> through the first instance of c after the + * first subsequent occurrence of <b>end_str</b>; store the 20-byte result in + * <b>digest</b>; return 0 on success. + * + * If no such substring exists, return -1. + */ +static int +router_get_hash_impl(const char *s, size_t s_len, char *digest, + const char *start_str, + const char *end_str, char end_c, + digest_algorithm_t alg) +{ + const char *start=NULL, *end=NULL; + if (router_get_hash_impl_helper(s,s_len,start_str,end_str,end_c, + &start,&end)<0) return -1; + + if (alg == DIGEST_SHA1) { + if (crypto_digest(digest, start, end-start)) { + log_warn(LD_BUG,"couldn't compute digest"); + return -1; + } + } else { + if (crypto_digest256(digest, start, end-start, alg)) { + log_warn(LD_BUG,"couldn't compute digest"); + return -1; + } } return 0; } +/** As router_get_hash_impl, but compute all hashes. */ +static int +router_get_hashes_impl(const char *s, size_t s_len, digests_t *digests, + const char *start_str, + const char *end_str, char end_c) +{ + const char *start=NULL, *end=NULL; + if (router_get_hash_impl_helper(s,s_len,start_str,end_str,end_c, + &start,&end)<0) + return -1; + + if (crypto_digest_all(digests, start, end-start)) { + log_warn(LD_BUG,"couldn't compute digests"); + return -1; + } + + return 0; +} + +/** Assuming that s starts with a microdesc, return the start of the + * *NEXT* one. Return NULL on "not found." */ +static const char * +find_start_of_next_microdesc(const char *s, const char *eos) +{ + int started_with_annotations; + s = eat_whitespace_eos(s, eos); + if (!s) + return NULL; + +#define CHECK_LENGTH() STMT_BEGIN \ + if (s+32 > eos) \ + return NULL; \ + STMT_END + +#define NEXT_LINE() STMT_BEGIN \ + s = memchr(s, '\n', eos-s); \ + if (!s || s+1 >= eos) \ + return NULL; \ + s++; \ + STMT_END + + CHECK_LENGTH(); + + started_with_annotations = (*s == '@'); + + if (started_with_annotations) { + /* Start by advancing to the first non-annotation line. */ + while (*s == '@') + NEXT_LINE(); + } + CHECK_LENGTH(); + + /* Now we should be pointed at an onion-key line. If we are, then skip + * it. */ + if (!strcmpstart(s, "onion-key")) + NEXT_LINE(); + + /* Okay, now we're pointed at the first line of the microdescriptor which is + not an annotation or onion-key. The next line that _is_ an annotation or + onion-key is the start of the next microdescriptor. */ + while (s+32 < eos) { + if (*s == '@' || !strcmpstart(s, "onion-key")) + return s; + NEXT_LINE(); + } + return NULL; + +#undef CHECK_LENGTH +#undef NEXT_LINE +} + +/** Parse as many microdescriptors as are found from the string starting at + * <b>s</b> and ending at <b>eos</b>. If allow_annotations is set, read any + * annotations we recognize and ignore ones we don't. If <b>copy_body</b> is + * true, then strdup the bodies of the microdescriptors. Return all newly + * parsed microdescriptors in a newly allocated smartlist_t. */ +smartlist_t * +microdescs_parse_from_string(const char *s, const char *eos, + int allow_annotations, int copy_body) +{ + smartlist_t *tokens; + smartlist_t *result; + microdesc_t *md = NULL; + memarea_t *area; + const char *start = s; + const char *start_of_next_microdesc; + int flags = allow_annotations ? TS_ANNOTATIONS_OK : 0; + + directory_token_t *tok; + + if (!eos) + eos = s + strlen(s); + + s = eat_whitespace_eos(s, eos); + area = memarea_new(); + result = smartlist_create(); + tokens = smartlist_create(); + + while (s < eos) { + start_of_next_microdesc = find_start_of_next_microdesc(s, eos); + if (!start_of_next_microdesc) + start_of_next_microdesc = eos; + + if (tokenize_string(area, s, start_of_next_microdesc, tokens, + microdesc_token_table, flags)) { + log_warn(LD_DIR, "Unparseable microdescriptor"); + goto next; + } + + md = tor_malloc_zero(sizeof(microdesc_t)); + { + const char *cp = tor_memstr(s, start_of_next_microdesc-s, + "onion-key"); + tor_assert(cp); + + md->bodylen = start_of_next_microdesc - cp; + if (copy_body) + md->body = tor_strndup(cp, md->bodylen); + else + md->body = (char*)cp; + md->off = cp - start; + } + + if ((tok = find_opt_by_keyword(tokens, A_LAST_LISTED))) { + if (parse_iso_time(tok->args[0], &md->last_listed)) { + log_warn(LD_DIR, "Bad last-listed time in microdescriptor"); + goto next; + } + } + + tok = find_by_keyword(tokens, K_ONION_KEY); + md->onion_pkey = tok->key; + tok->key = NULL; + + if ((tok = find_opt_by_keyword(tokens, K_FAMILY))) { + int i; + md->family = smartlist_create(); + for (i=0;i<tok->n_args;++i) { + if (!is_legal_nickname_or_hexdigest(tok->args[i])) { + log_warn(LD_DIR, "Illegal nickname %s in family line", + escaped(tok->args[i])); + goto next; + } + smartlist_add(md->family, tor_strdup(tok->args[i])); + } + } + + if ((tok = find_opt_by_keyword(tokens, K_P))) { + md->exitsummary = tor_strdup(tok->args[0]); + } + + crypto_digest256(md->digest, md->body, md->bodylen, DIGEST_SHA256); + + smartlist_add(result, md); + + md = NULL; + next: + microdesc_free(md); + md = NULL; + + memarea_clear(area); + smartlist_clear(tokens); + s = start_of_next_microdesc; + } + + memarea_drop_all(area); + smartlist_free(tokens); + + return result; +} + /** Parse the Tor version of the platform string <b>platform</b>, * and compare it to the version in <b>cutoff</b>. Return 1 if * the router is at least as new as the cutoff, else return 0. @@ -3406,7 +4395,7 @@ tor_version_as_new_as(const char *platform, const char *cutoff) if (!*start) return 0; s = (char *)find_whitespace(start); /* also finds '\0', which is fine */ s2 = (char*)eat_whitespace(s); - if (!strcmpstart(s2, "(r")) + if (!strcmpstart(s2, "(r") || !strcmpstart(s2, "(git-")) s = (char*)find_whitespace(s2); if ((size_t)(s-start+1) >= sizeof(tmp)) /* too big, no */ @@ -3502,6 +4491,23 @@ tor_version_parse(const char *s, tor_version_t *out) if (!strcmpstart(cp, "(r")) { cp += 2; out->svn_revision = (int) strtol(cp,&eos,10); + } else if (!strcmpstart(cp, "(git-")) { + char *close_paren = strchr(cp, ')'); + int hexlen; + char digest[DIGEST_LEN]; + if (! close_paren) + return -1; + cp += 5; + if (close_paren-cp > HEX_DIGEST_LEN) + return -1; + hexlen = (int)(close_paren-cp); + memset(digest, 0, sizeof(digest)); + if ( hexlen == 0 || (hexlen % 2) == 1) + return -1; + if (base16_decode(digest, hexlen/2, cp, hexlen)) + return -1; + memcpy(out->git_tag, digest, hexlen/2); + out->git_tag_len = hexlen/2; } return 0; @@ -3527,8 +4533,14 @@ tor_version_compare(tor_version_t *a, tor_version_t *b) return i; else if ((i = strcmp(a->status_tag, b->status_tag))) return i; + else if ((i = a->svn_revision - b->svn_revision)) + return i; + else if ((i = a->git_tag_len - b->git_tag_len)) + return i; + else if (a->git_tag_len) + return memcmp(a->git_tag, b->git_tag, a->git_tag_len); else - return a->svn_revision - b->svn_revision; + return 0; } /** Return true iff versions <b>a</b> and <b>b</b> belong to the same series. @@ -3617,7 +4629,7 @@ rend_parse_v2_service_descriptor(rend_service_descriptor_t **parsed_out, /* Compute descriptor hash for later validation. */ if (router_get_hash_impl(desc, strlen(desc), desc_hash, "rendezvous-service-descriptor ", - "\nsignature", '\n') < 0) { + "\nsignature", '\n', DIGEST_SHA1) < 0) { log_warn(LD_REND, "Couldn't compute descriptor hash."); goto err; } @@ -3628,12 +4640,12 @@ rend_parse_v2_service_descriptor(rend_service_descriptor_t **parsed_out, else eos = eos + 1; /* Check length. */ - if (strlen(desc) > REND_DESC_MAX_SIZE) { + if (eos-desc > REND_DESC_MAX_SIZE) { /* XXX023 If we are parsing this descriptor as a server, this * should be a protocol warning. */ - log_warn(LD_REND, "Descriptor length is %i which exceeds " - "maximum rendezvous descriptor size of %i bytes.", - (int)strlen(desc), REND_DESC_MAX_SIZE); + log_warn(LD_REND, "Descriptor length is %d which exceeds " + "maximum rendezvous descriptor size of %d bytes.", + (int)(eos-desc), REND_DESC_MAX_SIZE); goto err; } /* Tokenize descriptor. */ @@ -3738,7 +4750,7 @@ rend_parse_v2_service_descriptor(rend_service_descriptor_t **parsed_out, /* Parse and verify signature. */ tok = find_by_keyword(tokens, R_SIGNATURE); note_crypto_pk_op(VERIFY_RTR); - if (check_signature_token(desc_hash, tok, result->pk, 0, + if (check_signature_token(desc_hash, DIGEST_LEN, tok, result->pk, 0, "v2 rendezvous service descriptor") < 0) goto err; /* Verify that descriptor ID belongs to public key and secret ID part. */ @@ -3752,12 +4764,11 @@ rend_parse_v2_service_descriptor(rend_service_descriptor_t **parsed_out, } goto done; err: - if (result) - rend_service_descriptor_free(result); + rend_service_descriptor_free(result); result = NULL; done: if (tokens) { - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_free(tokens); } if (area) @@ -3915,7 +4926,7 @@ rend_parse_introduction_points(rend_service_descriptor_t *parsed, eos = eos+1; tor_assert(eos <= intro_points_encoded+intro_points_encoded_size); /* Free tokens and clear token list. */ - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_clear(tokens); memarea_clear(area); /* Tokenize string. */ @@ -3988,7 +4999,7 @@ rend_parse_introduction_points(rend_service_descriptor_t *parsed, done: /* Free tokens and clear token list. */ - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_free(tokens); if (area) memarea_drop_all(area); @@ -4027,7 +5038,7 @@ rend_parse_client_keys(strmap_t *parsed_clients, const char *ckstr) else eos = eos + 1; /* Free tokens and clear token list. */ - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_clear(tokens); memarea_clear(area); /* Tokenize string. */ @@ -4099,7 +5110,7 @@ rend_parse_client_keys(strmap_t *parsed_clients, const char *ckstr) result = -1; done: /* Free tokens and clear token list. */ - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_free(tokens); if (area) memarea_drop_all(area); diff --git a/src/or/routerparse.h b/src/or/routerparse.h new file mode 100644 index 0000000000..8b8cde25f6 --- /dev/null +++ b/src/or/routerparse.h @@ -0,0 +1,86 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file routerparse.h + * \brief Header file for routerparse.c. + **/ + +#ifndef _TOR_ROUTERPARSE_H +#define _TOR_ROUTERPARSE_H + +int router_get_router_hash(const char *s, size_t s_len, char *digest); +int router_get_dir_hash(const char *s, char *digest); +int router_get_runningrouters_hash(const char *s, char *digest); +int router_get_networkstatus_v2_hash(const char *s, char *digest); +int router_get_networkstatus_v3_hash(const char *s, char *digest, + digest_algorithm_t algorithm); +int router_get_networkstatus_v3_hashes(const char *s, digests_t *digests); +int router_get_extrainfo_hash(const char *s, char *digest); +#define DIROBJ_MAX_SIG_LEN 256 +int router_append_dirobj_signature(char *buf, size_t buf_len, + const char *digest, + size_t digest_len, + crypto_pk_env_t *private_key); +int router_parse_list_from_string(const char **s, const char *eos, + smartlist_t *dest, + saved_location_t saved_location, + int is_extrainfo, + int allow_annotations, + const char *prepend_annotations); +int router_parse_runningrouters(const char *str); +int router_parse_directory(const char *str); + +routerinfo_t *router_parse_entry_from_string(const char *s, const char *end, + int cache_copy, + int allow_annotations, + const char *prepend_annotations); +extrainfo_t *extrainfo_parse_entry_from_string(const char *s, const char *end, + int cache_copy, struct digest_ri_map_t *routermap); +addr_policy_t *router_parse_addr_policy_item_from_string(const char *s, + int assume_action); +version_status_t tor_version_is_obsolete(const char *myversion, + const char *versionlist); +int tor_version_parse(const char *s, tor_version_t *out); +int tor_version_as_new_as(const char *platform, const char *cutoff); +int tor_version_compare(tor_version_t *a, tor_version_t *b); +void sort_version_list(smartlist_t *lst, int remove_duplicates); +void assert_addr_policy_ok(smartlist_t *t); +void dump_distinct_digest_count(int severity); + +int compare_routerstatus_entries(const void **_a, const void **_b); +networkstatus_v2_t *networkstatus_v2_parse_from_string(const char *s); +int networkstatus_verify_bw_weights(networkstatus_t *ns); +networkstatus_t *networkstatus_parse_vote_from_string(const char *s, + const char **eos_out, + networkstatus_type_t ns_type); +ns_detached_signatures_t *networkstatus_parse_detached_signatures( + const char *s, const char *eos); + +smartlist_t *microdescs_parse_from_string(const char *s, const char *eos, + int allow_annotations, + int copy_body); + +authority_cert_t *authority_cert_parse_from_string(const char *s, + const char **end_of_string); +int rend_parse_v2_service_descriptor(rend_service_descriptor_t **parsed_out, + char *desc_id_out, + char **intro_points_encrypted_out, + size_t *intro_points_encrypted_size_out, + size_t *encoded_size_out, + const char **next_out, const char *desc); +int rend_decrypt_introduction_points(char **ipos_decrypted, + size_t *ipos_decrypted_size, + const char *descriptor_cookie, + const char *ipos_encrypted, + size_t ipos_encrypted_size); +int rend_parse_introduction_points(rend_service_descriptor_t *parsed, + const char *intro_points_encoded, + size_t intro_points_encoded_size); +int rend_parse_client_keys(strmap_t *parsed_clients, const char *str); + +#endif + diff --git a/src/or/test.c b/src/or/test.c deleted file mode 100644 index dc71db4e1d..0000000000 --- a/src/or/test.c +++ /dev/null @@ -1,4973 +0,0 @@ -/* Copyright (c) 2001-2004, Roger Dingledine. - * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2011, The Tor Project, Inc. */ -/* See LICENSE for licensing information */ - -/* Ordinarily defined in tor_main.c; this bit is just here to provide one - * since we're not linking to tor_main.c */ -const char tor_svn_revision[] = ""; - -/** - * \file test.c - * \brief Unit tests for many pieces of the lower level Tor modules. - **/ - -#include "orconfig.h" - -#include <stdio.h> -#ifdef HAVE_FCNTL_H -#include <fcntl.h> -#endif - -#ifdef MS_WINDOWS -/* For mkdir() */ -#include <direct.h> -#else -#include <dirent.h> -#endif - -/* These macros pull in declarations for some functions and structures that - * are typically file-private. */ -#define BUFFERS_PRIVATE -#define CONFIG_PRIVATE -#define CONTROL_PRIVATE -#define CRYPTO_PRIVATE -#define DIRSERV_PRIVATE -#define DIRVOTE_PRIVATE -#define GEOIP_PRIVATE -#define MEMPOOL_PRIVATE -#define ROUTER_PRIVATE - -#include "or.h" -#include "test.h" -#include "torgzip.h" -#include "mempool.h" -#include "memarea.h" -#include "di_ops.h" - -#ifdef USE_DMALLOC -#include <dmalloc.h> -#include <openssl/crypto.h> -#endif - -/** Set to true if any unit test has failed. Mostly, this is set by the macros - * in test.h */ -int have_failed = 0; - -/** Temporary directory (set up by setup_directory) under which we store all - * our files during testing. */ -static char temp_dir[256]; - -/** Select and create the temporary directory we'll use to run our unit tests. - * Store it in <b>temp_dir</b>. Exit immediately if we can't create it. - * idempotent. */ -static void -setup_directory(void) -{ - static int is_setup = 0; - int r; - if (is_setup) return; - -#ifdef MS_WINDOWS - // XXXX - tor_snprintf(temp_dir, sizeof(temp_dir), - "c:\\windows\\temp\\tor_test_%d", (int)getpid()); - r = mkdir(temp_dir); -#else - tor_snprintf(temp_dir, sizeof(temp_dir), "/tmp/tor_test_%d", (int) getpid()); - r = mkdir(temp_dir, 0700); -#endif - if (r) { - fprintf(stderr, "Can't create directory %s:", temp_dir); - perror(""); - exit(1); - } - is_setup = 1; -} - -/** Return a filename relative to our testing temporary directory */ -static const char * -get_fname(const char *name) -{ - static char buf[1024]; - setup_directory(); - tor_snprintf(buf,sizeof(buf),"%s/%s",temp_dir,name); - return buf; -} - -/** Remove all files stored under the temporary directory, and the directory - * itself. */ -static void -remove_directory(void) -{ - smartlist_t *elements = tor_listdir(temp_dir); - if (elements) { - SMARTLIST_FOREACH(elements, const char *, cp, - { - size_t len = strlen(cp)+strlen(temp_dir)+16; - char *tmp = tor_malloc(len); - tor_snprintf(tmp, len, "%s"PATH_SEPARATOR"%s", temp_dir, cp); - unlink(tmp); - tor_free(tmp); - }); - SMARTLIST_FOREACH(elements, char *, cp, tor_free(cp)); - smartlist_free(elements); - } - rmdir(temp_dir); -} - -/** Define this if unit tests spend too much time generating public keys*/ -#undef CACHE_GENERATED_KEYS - -static crypto_pk_env_t *pregen_keys[5] = {NULL, NULL, NULL, NULL, NULL}; -#define N_PREGEN_KEYS ((int)(sizeof(pregen_keys)/sizeof(pregen_keys[0]))) - -/** Generate and return a new keypair for use in unit tests. If we're using - * the key cache optimization, we might reuse keys: we only guarantee that - * keys made with distinct values for <b>idx</b> are different. The value of - * <b>idx</b> must be at least 0, and less than N_PREGEN_KEYS. */ -static crypto_pk_env_t * -pk_generate(int idx) -{ -#ifdef CACHE_GENERATED_KEYS - tor_assert(idx < N_PREGEN_KEYS); - if (! pregen_keys[idx]) { - pregen_keys[idx] = crypto_new_pk_env(); - tor_assert(!crypto_pk_generate_key(pregen_keys[idx])); - } - return crypto_pk_dup_key(pregen_keys[idx]); -#else - crypto_pk_env_t *result; - (void) idx; - result = crypto_new_pk_env(); - tor_assert(!crypto_pk_generate_key(result)); - return result; -#endif -} - -/** Free all storage used for the cached key optimization. */ -static void -free_pregenerated_keys(void) -{ - unsigned idx; - for (idx = 0; idx < N_PREGEN_KEYS; ++idx) { - if (pregen_keys[idx]) { - crypto_free_pk_env(pregen_keys[idx]); - pregen_keys[idx] = NULL; - } - } -} - -/** Run unit tests for buffers.c */ -static void -test_buffers(void) -{ - char str[256]; - char str2[256]; - - buf_t *buf = NULL, *buf2 = NULL; - const char *cp; - - int j; - size_t r; - - /**** - * buf_new - ****/ - if (!(buf = buf_new())) - test_fail(); - - //test_eq(buf_capacity(buf), 4096); - test_eq(buf_datalen(buf), 0); - - /**** - * General pointer frobbing - */ - for (j=0;j<256;++j) { - str[j] = (char)j; - } - write_to_buf(str, 256, buf); - write_to_buf(str, 256, buf); - test_eq(buf_datalen(buf), 512); - fetch_from_buf(str2, 200, buf); - test_memeq(str, str2, 200); - test_eq(buf_datalen(buf), 312); - memset(str2, 0, sizeof(str2)); - - fetch_from_buf(str2, 256, buf); - test_memeq(str+200, str2, 56); - test_memeq(str, str2+56, 200); - test_eq(buf_datalen(buf), 56); - memset(str2, 0, sizeof(str2)); - /* Okay, now we should be 512 bytes into the 4096-byte buffer. If we add - * another 3584 bytes, we hit the end. */ - for (j=0;j<15;++j) { - write_to_buf(str, 256, buf); - } - assert_buf_ok(buf); - test_eq(buf_datalen(buf), 3896); - fetch_from_buf(str2, 56, buf); - test_eq(buf_datalen(buf), 3840); - test_memeq(str+200, str2, 56); - for (j=0;j<15;++j) { - memset(str2, 0, sizeof(str2)); - fetch_from_buf(str2, 256, buf); - test_memeq(str, str2, 256); - } - test_eq(buf_datalen(buf), 0); - buf_free(buf); - buf = NULL; - - /* Okay, now make sure growing can work. */ - buf = buf_new_with_capacity(16); - //test_eq(buf_capacity(buf), 16); - write_to_buf(str+1, 255, buf); - //test_eq(buf_capacity(buf), 256); - fetch_from_buf(str2, 254, buf); - test_memeq(str+1, str2, 254); - //test_eq(buf_capacity(buf), 256); - assert_buf_ok(buf); - write_to_buf(str, 32, buf); - //test_eq(buf_capacity(buf), 256); - assert_buf_ok(buf); - write_to_buf(str, 256, buf); - assert_buf_ok(buf); - //test_eq(buf_capacity(buf), 512); - test_eq(buf_datalen(buf), 33+256); - fetch_from_buf(str2, 33, buf); - test_eq(*str2, str[255]); - - test_memeq(str2+1, str, 32); - //test_eq(buf_capacity(buf), 512); - test_eq(buf_datalen(buf), 256); - fetch_from_buf(str2, 256, buf); - test_memeq(str, str2, 256); - - /* now try shrinking: case 1. */ - buf_free(buf); - buf = buf_new_with_capacity(33668); - for (j=0;j<67;++j) { - write_to_buf(str,255, buf); - } - //test_eq(buf_capacity(buf), 33668); - test_eq(buf_datalen(buf), 17085); - for (j=0; j < 40; ++j) { - fetch_from_buf(str2, 255,buf); - test_memeq(str2, str, 255); - } - - /* now try shrinking: case 2. */ - buf_free(buf); - buf = buf_new_with_capacity(33668); - for (j=0;j<67;++j) { - write_to_buf(str,255, buf); - } - for (j=0; j < 20; ++j) { - fetch_from_buf(str2, 255,buf); - test_memeq(str2, str, 255); - } - for (j=0;j<80;++j) { - write_to_buf(str,255, buf); - } - //test_eq(buf_capacity(buf),33668); - for (j=0; j < 120; ++j) { - fetch_from_buf(str2, 255,buf); - test_memeq(str2, str, 255); - } - - /* Move from buf to buf. */ - buf_free(buf); - buf = buf_new_with_capacity(4096); - buf2 = buf_new_with_capacity(4096); - for (j=0;j<100;++j) - write_to_buf(str, 255, buf); - test_eq(buf_datalen(buf), 25500); - for (j=0;j<100;++j) { - r = 10; - move_buf_to_buf(buf2, buf, &r); - test_eq(r, 0); - } - test_eq(buf_datalen(buf), 24500); - test_eq(buf_datalen(buf2), 1000); - for (j=0;j<3;++j) { - fetch_from_buf(str2, 255, buf2); - test_memeq(str2, str, 255); - } - r = 8192; /*big move*/ - move_buf_to_buf(buf2, buf, &r); - test_eq(r, 0); - r = 30000; /* incomplete move */ - move_buf_to_buf(buf2, buf, &r); - test_eq(r, 13692); - for (j=0;j<97;++j) { - fetch_from_buf(str2, 255, buf2); - test_memeq(str2, str, 255); - } - buf_free(buf); - buf_free(buf2); - buf = buf2 = NULL; - - buf = buf_new_with_capacity(5); - cp = "Testing. This is a moderately long Testing string."; - for (j = 0; cp[j]; j++) - write_to_buf(cp+j, 1, buf); - test_eq(0, buf_find_string_offset(buf, "Testing", 7)); - test_eq(1, buf_find_string_offset(buf, "esting", 6)); - test_eq(1, buf_find_string_offset(buf, "est", 3)); - test_eq(39, buf_find_string_offset(buf, "ing str", 7)); - test_eq(35, buf_find_string_offset(buf, "Testing str", 11)); - test_eq(32, buf_find_string_offset(buf, "ng ", 3)); - test_eq(43, buf_find_string_offset(buf, "string.", 7)); - test_eq(-1, buf_find_string_offset(buf, "shrdlu", 6)); - test_eq(-1, buf_find_string_offset(buf, "Testing thing", 13)); - test_eq(-1, buf_find_string_offset(buf, "ngx", 3)); - buf_free(buf); - buf = NULL; - -#if 0 - { - int s; - int eof; - int i; - buf_t *buf2; - /**** - * read_to_buf - ****/ - s = open(get_fname("data"), O_WRONLY|O_CREAT|O_TRUNC, 0600); - write(s, str, 256); - close(s); - - s = open(get_fname("data"), O_RDONLY, 0); - eof = 0; - errno = 0; /* XXXX */ - i = read_to_buf(s, 10, buf, &eof); - printf("%s\n", strerror(errno)); - test_eq(i, 10); - test_eq(eof, 0); - //test_eq(buf_capacity(buf), 4096); - test_eq(buf_datalen(buf), 10); - - test_memeq(str, (char*)_buf_peek_raw_buffer(buf), 10); - - /* Test reading 0 bytes. */ - i = read_to_buf(s, 0, buf, &eof); - //test_eq(buf_capacity(buf), 512*1024); - test_eq(buf_datalen(buf), 10); - test_eq(eof, 0); - test_eq(i, 0); - - /* Now test when buffer is filled exactly. */ - buf2 = buf_new_with_capacity(6); - i = read_to_buf(s, 6, buf2, &eof); - //test_eq(buf_capacity(buf2), 6); - test_eq(buf_datalen(buf2), 6); - test_eq(eof, 0); - test_eq(i, 6); - test_memeq(str+10, (char*)_buf_peek_raw_buffer(buf2), 6); - buf_free(buf2); - buf2 = NULL; - - /* Now test when buffer is filled with more data to read. */ - buf2 = buf_new_with_capacity(32); - i = read_to_buf(s, 128, buf2, &eof); - //test_eq(buf_capacity(buf2), 128); - test_eq(buf_datalen(buf2), 32); - test_eq(eof, 0); - test_eq(i, 32); - buf_free(buf2); - buf2 = NULL; - - /* Now read to eof. */ - test_assert(buf_capacity(buf) > 256); - i = read_to_buf(s, 1024, buf, &eof); - test_eq(i, (256-32-10-6)); - test_eq(buf_capacity(buf), MAX_BUF_SIZE); - test_eq(buf_datalen(buf), 256-6-32); - test_memeq(str, (char*)_buf_peek_raw_buffer(buf), 10); /* XXX Check rest. */ - test_eq(eof, 0); - - i = read_to_buf(s, 1024, buf, &eof); - test_eq(i, 0); - test_eq(buf_capacity(buf), MAX_BUF_SIZE); - test_eq(buf_datalen(buf), 256-6-32); - test_eq(eof, 1); - } -#endif - - done: - if (buf) - buf_free(buf); - if (buf2) - buf_free(buf2); -} - -/** Run unit tests for Diffie-Hellman functionality. */ -static void -test_crypto_dh(void) -{ - crypto_dh_env_t *dh1 = crypto_dh_new(DH_TYPE_CIRCUIT); - crypto_dh_env_t *dh2 = crypto_dh_new(DH_TYPE_CIRCUIT); - char p1[DH_BYTES]; - char p2[DH_BYTES]; - char s1[DH_BYTES]; - char s2[DH_BYTES]; - ssize_t s1len, s2len; - - test_eq(crypto_dh_get_bytes(dh1), DH_BYTES); - test_eq(crypto_dh_get_bytes(dh2), DH_BYTES); - - memset(p1, 0, DH_BYTES); - memset(p2, 0, DH_BYTES); - test_memeq(p1, p2, DH_BYTES); - test_assert(! crypto_dh_get_public(dh1, p1, DH_BYTES)); - test_memneq(p1, p2, DH_BYTES); - test_assert(! crypto_dh_get_public(dh2, p2, DH_BYTES)); - test_memneq(p1, p2, DH_BYTES); - - memset(s1, 0, DH_BYTES); - memset(s2, 0xFF, DH_BYTES); - s1len = crypto_dh_compute_secret(dh1, p2, DH_BYTES, s1, 50); - s2len = crypto_dh_compute_secret(dh2, p1, DH_BYTES, s2, 50); - test_assert(s1len > 0); - test_eq(s1len, s2len); - test_memeq(s1, s2, s1len); - - { - /* XXXX Now fabricate some bad values and make sure they get caught, - * Check 0, 1, N-1, >= N, etc. - */ - } - - done: - crypto_dh_free(dh1); - crypto_dh_free(dh2); -} - -/** Run unit tests for our random number generation function and its wrappers. - */ -static void -test_crypto_rng(void) -{ - int i, j, allok; - char data1[100], data2[100]; - - /* Try out RNG. */ - test_assert(! crypto_seed_rng(0)); - crypto_rand(data1, 100); - crypto_rand(data2, 100); - test_memneq(data1,data2,100); - allok = 1; - for (i = 0; i < 100; ++i) { - uint64_t big; - char *host; - j = crypto_rand_int(100); - if (i < 0 || i >= 100) - allok = 0; - big = crypto_rand_uint64(U64_LITERAL(1)<<40); - if (big >= (U64_LITERAL(1)<<40)) - allok = 0; - big = crypto_rand_uint64(U64_LITERAL(5)); - if (big >= 5) - allok = 0; - host = crypto_random_hostname(3,8,"www.",".onion"); - if (strcmpstart(host,"www.") || - strcmpend(host,".onion") || - strlen(host) < 13 || - strlen(host) > 18) - allok = 0; - tor_free(host); - } - test_assert(allok); - done: - ; -} - -/** Run unit tests for our AES functionality */ -static void -test_crypto_aes(void) -{ - char *data1 = NULL, *data2 = NULL, *data3 = NULL; - crypto_cipher_env_t *env1 = NULL, *env2 = NULL; - int i, j; - - data1 = tor_malloc(1024); - data2 = tor_malloc(1024); - data3 = tor_malloc(1024); - - /* Now, test encryption and decryption with stream cipher. */ - data1[0]='\0'; - for (i = 1023; i>0; i -= 35) - strncat(data1, "Now is the time for all good onions", i); - - memset(data2, 0, 1024); - memset(data3, 0, 1024); - env1 = crypto_new_cipher_env(); - test_neq(env1, 0); - env2 = crypto_new_cipher_env(); - test_neq(env2, 0); - j = crypto_cipher_generate_key(env1); - crypto_cipher_set_key(env2, crypto_cipher_get_key(env1)); - crypto_cipher_encrypt_init_cipher(env1); - crypto_cipher_decrypt_init_cipher(env2); - - /* Try encrypting 512 chars. */ - crypto_cipher_encrypt(env1, data2, data1, 512); - crypto_cipher_decrypt(env2, data3, data2, 512); - test_memeq(data1, data3, 512); - test_memneq(data1, data2, 512); - - /* Now encrypt 1 at a time, and get 1 at a time. */ - for (j = 512; j < 560; ++j) { - crypto_cipher_encrypt(env1, data2+j, data1+j, 1); - } - for (j = 512; j < 560; ++j) { - crypto_cipher_decrypt(env2, data3+j, data2+j, 1); - } - test_memeq(data1, data3, 560); - /* Now encrypt 3 at a time, and get 5 at a time. */ - for (j = 560; j < 1024-5; j += 3) { - crypto_cipher_encrypt(env1, data2+j, data1+j, 3); - } - for (j = 560; j < 1024-5; j += 5) { - crypto_cipher_decrypt(env2, data3+j, data2+j, 5); - } - test_memeq(data1, data3, 1024-5); - /* Now make sure that when we encrypt with different chunk sizes, we get - the same results. */ - crypto_free_cipher_env(env2); - env2 = NULL; - - memset(data3, 0, 1024); - env2 = crypto_new_cipher_env(); - test_neq(env2, 0); - crypto_cipher_set_key(env2, crypto_cipher_get_key(env1)); - crypto_cipher_encrypt_init_cipher(env2); - for (j = 0; j < 1024-16; j += 17) { - crypto_cipher_encrypt(env2, data3+j, data1+j, 17); - } - for (j= 0; j < 1024-16; ++j) { - if (data2[j] != data3[j]) { - printf("%d: %d\t%d\n", j, (int) data2[j], (int) data3[j]); - } - } - test_memeq(data2, data3, 1024-16); - crypto_free_cipher_env(env1); - env1 = NULL; - crypto_free_cipher_env(env2); - env2 = NULL; - - /* NIST test vector for aes. */ - env1 = crypto_new_cipher_env(); /* IV starts at 0 */ - crypto_cipher_set_key(env1, "\x80\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00"); - crypto_cipher_encrypt_init_cipher(env1); - crypto_cipher_encrypt(env1, data1, - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00", 16); - test_memeq_hex(data1, "0EDD33D3C621E546455BD8BA1418BEC8"); - - /* Now test rollover. All these values are originally from a python - * script. */ - crypto_cipher_set_iv(env1, "\x00\x00\x00\x00\x00\x00\x00\x00" - "\xff\xff\xff\xff\xff\xff\xff\xff"); - memset(data2, 0, 1024); - crypto_cipher_encrypt(env1, data1, data2, 32); - test_memeq_hex(data1, "335fe6da56f843199066c14a00a40231" - "cdd0b917dbc7186908a6bfb5ffd574d3"); - - crypto_cipher_set_iv(env1, "\x00\x00\x00\x00\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff"); - memset(data2, 0, 1024); - crypto_cipher_encrypt(env1, data1, data2, 32); - test_memeq_hex(data1, "e627c6423fa2d77832a02b2794094b73" - "3e63c721df790d2c6469cc1953a3ffac"); - - crypto_cipher_set_iv(env1, "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff"); - memset(data2, 0, 1024); - crypto_cipher_encrypt(env1, data1, data2, 32); - test_memeq_hex(data1, "2aed2bff0de54f9328efd070bf48f70a" - "0EDD33D3C621E546455BD8BA1418BEC8"); - - /* Now check rollover on inplace cipher. */ - crypto_cipher_set_iv(env1, "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff"); - crypto_cipher_crypt_inplace(env1, data2, 64); - test_memeq_hex(data2, "2aed2bff0de54f9328efd070bf48f70a" - "0EDD33D3C621E546455BD8BA1418BEC8" - "93e2c5243d6839eac58503919192f7ae" - "1908e67cafa08d508816659c2e693191"); - crypto_cipher_set_iv(env1, "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff"); - crypto_cipher_crypt_inplace(env1, data2, 64); - test_assert(tor_mem_is_zero(data2, 64)); - - done: - if (env1) - crypto_free_cipher_env(env1); - if (env2) - crypto_free_cipher_env(env2); - tor_free(data1); - tor_free(data2); - tor_free(data3); -} - -/** Run unit tests for our SHA-1 functionality */ -static void -test_crypto_sha(void) -{ - crypto_digest_env_t *d1 = NULL, *d2 = NULL; - int i; - char key[80]; - char digest[20]; - char data[50]; - char d_out1[DIGEST_LEN], d_out2[DIGEST_LEN]; - - /* Test SHA-1 with a test vector from the specification. */ - i = crypto_digest(data, "abc", 3); - test_memeq_hex(data, "A9993E364706816ABA3E25717850C26C9CD0D89D"); - - /* Test HMAC-SHA-1 with test cases from RFC2202. */ - - /* Case 1. */ - memset(key, 0x0b, 20); - crypto_hmac_sha1(digest, key, 20, "Hi There", 8); - test_streq(hex_str(digest, 20), - "B617318655057264E28BC0B6FB378C8EF146BE00"); - /* Case 2. */ - crypto_hmac_sha1(digest, "Jefe", 4, "what do ya want for nothing?", 28); - test_streq(hex_str(digest, 20), - "EFFCDF6AE5EB2FA2D27416D5F184DF9C259A7C79"); - - /* Case 4. */ - base16_decode(key, 25, - "0102030405060708090a0b0c0d0e0f10111213141516171819", 50); - memset(data, 0xcd, 50); - crypto_hmac_sha1(digest, key, 25, data, 50); - test_streq(hex_str(digest, 20), - "4C9007F4026250C6BC8414F9BF50C86C2D7235DA"); - - /* Case . */ - memset(key, 0xaa, 80); - crypto_hmac_sha1(digest, key, 80, - "Test Using Larger Than Block-Size Key - Hash Key First", - 54); - test_streq(hex_str(digest, 20), - "AA4AE5E15272D00E95705637CE8A3B55ED402112"); - - /* Incremental digest code. */ - d1 = crypto_new_digest_env(); - test_assert(d1); - crypto_digest_add_bytes(d1, "abcdef", 6); - d2 = crypto_digest_dup(d1); - test_assert(d2); - crypto_digest_add_bytes(d2, "ghijkl", 6); - crypto_digest_get_digest(d2, d_out1, sizeof(d_out1)); - crypto_digest(d_out2, "abcdefghijkl", 12); - test_memeq(d_out1, d_out2, DIGEST_LEN); - crypto_digest_assign(d2, d1); - crypto_digest_add_bytes(d2, "mno", 3); - crypto_digest_get_digest(d2, d_out1, sizeof(d_out1)); - crypto_digest(d_out2, "abcdefmno", 9); - test_memeq(d_out1, d_out2, DIGEST_LEN); - crypto_digest_get_digest(d1, d_out1, sizeof(d_out1)); - crypto_digest(d_out2, "abcdef", 6); - test_memeq(d_out1, d_out2, DIGEST_LEN); - - done: - if (d1) - crypto_free_digest_env(d1); - if (d2) - crypto_free_digest_env(d2); -} - -/** Run unit tests for our public key crypto functions */ -static void -test_crypto_pk(void) -{ - crypto_pk_env_t *pk1 = NULL, *pk2 = NULL; - char *encoded = NULL; - char data1[1024], data2[1024], data3[1024]; - size_t size; - int i, j, p, len; - - /* Public-key ciphers */ - pk1 = pk_generate(0); - pk2 = crypto_new_pk_env(); - test_assert(pk1 && pk2); - test_assert(! crypto_pk_write_public_key_to_string(pk1, &encoded, &size)); - test_assert(! crypto_pk_read_public_key_from_string(pk2, encoded, size)); - test_eq(0, crypto_pk_cmp_keys(pk1, pk2)); - - test_eq(128, crypto_pk_keysize(pk1)); - test_eq(128, crypto_pk_keysize(pk2)); - - test_eq(128, crypto_pk_public_encrypt(pk2, data1, sizeof(data1), - "Hello whirled.", 15, - PK_PKCS1_OAEP_PADDING)); - test_eq(128, crypto_pk_public_encrypt(pk1, data2, sizeof(data2), - "Hello whirled.", 15, - PK_PKCS1_OAEP_PADDING)); - /* oaep padding should make encryption not match */ - test_memneq(data1, data2, 128); - test_eq(15, crypto_pk_private_decrypt(pk1, data3, sizeof(data3), data1, 128, - PK_PKCS1_OAEP_PADDING,1)); - test_streq(data3, "Hello whirled."); - memset(data3, 0, 1024); - test_eq(15, crypto_pk_private_decrypt(pk1, data3, sizeof(data3), data2, 128, - PK_PKCS1_OAEP_PADDING,1)); - test_streq(data3, "Hello whirled."); - /* Can't decrypt with public key. */ - test_eq(-1, crypto_pk_private_decrypt(pk2, data3, sizeof(data3), data2, 128, - PK_PKCS1_OAEP_PADDING,1)); - /* Try again with bad padding */ - memcpy(data2+1, "XYZZY", 5); /* This has fails ~ once-in-2^40 */ - test_eq(-1, crypto_pk_private_decrypt(pk1, data3, sizeof(data3), data2, 128, - PK_PKCS1_OAEP_PADDING,1)); - - /* File operations: save and load private key */ - test_assert(! crypto_pk_write_private_key_to_filename(pk1, - get_fname("pkey1"))); - /* failing case for read: can't read. */ - test_assert(crypto_pk_read_private_key_from_filename(pk2, - get_fname("xyzzy")) < 0); - write_str_to_file(get_fname("xyzzy"), "foobar", 6); - /* Failing case for read: no key. */ - test_assert(crypto_pk_read_private_key_from_filename(pk2, - get_fname("xyzzy")) < 0); - test_assert(! crypto_pk_read_private_key_from_filename(pk2, - get_fname("pkey1"))); - test_eq(15, crypto_pk_private_decrypt(pk2, data3, sizeof(data3), data1, 128, - PK_PKCS1_OAEP_PADDING,1)); - - /* Now try signing. */ - strlcpy(data1, "Ossifrage", 1024); - test_eq(128, crypto_pk_private_sign(pk1, data2, sizeof(data2), data1, 10)); - test_eq(10, crypto_pk_public_checksig(pk1, data3, sizeof(data3), data2, 128)); - test_streq(data3, "Ossifrage"); - /* Try signing digests. */ - test_eq(128, crypto_pk_private_sign_digest(pk1, data2, sizeof(data2), - data1, 10)); - test_eq(20, crypto_pk_public_checksig(pk1, data3, sizeof(data1), data2, 128)); - test_eq(0, crypto_pk_public_checksig_digest(pk1, data1, - 10, data2, 128)); - test_eq(-1, crypto_pk_public_checksig_digest(pk1, data1, - 11, data2, 128)); - /*XXXX test failed signing*/ - - /* Try encoding */ - crypto_free_pk_env(pk2); - pk2 = NULL; - i = crypto_pk_asn1_encode(pk1, data1, 1024); - test_assert(i>0); - pk2 = crypto_pk_asn1_decode(data1, i); - test_assert(crypto_pk_cmp_keys(pk1,pk2) == 0); - - /* Try with hybrid encryption wrappers. */ - crypto_rand(data1, 1024); - for (i = 0; i < 3; ++i) { - for (j = 85; j < 140; ++j) { - memset(data2,0,1024); - memset(data3,0,1024); - if (i == 0 && j < 129) - continue; - p = (i==0)?PK_NO_PADDING: - (i==1)?PK_PKCS1_PADDING:PK_PKCS1_OAEP_PADDING; - len = crypto_pk_public_hybrid_encrypt(pk1,data2,sizeof(data2), - data1,j,p,0); - test_assert(len>=0); - len = crypto_pk_private_hybrid_decrypt(pk1,data3,sizeof(data3), - data2,len,p,1); - test_eq(len,j); - test_memeq(data1,data3,j); - } - } - - /* Try copy_full */ - crypto_free_pk_env(pk2); - pk2 = crypto_pk_copy_full(pk1); - test_assert(pk2 != NULL); - test_neq_ptr(pk1, pk2); - test_assert(crypto_pk_cmp_keys(pk1,pk2) == 0); - - done: - if (pk1) - crypto_free_pk_env(pk1); - if (pk2) - crypto_free_pk_env(pk2); - tor_free(encoded); -} - -/** Run unit tests for misc crypto functionality. */ -static void -test_crypto(void) -{ - char *data1 = NULL, *data2 = NULL, *data3 = NULL; - int i, j, idx; - - data1 = tor_malloc(1024); - data2 = tor_malloc(1024); - data3 = tor_malloc(1024); - test_assert(data1 && data2 && data3); - - /* Base64 tests */ - memset(data1, 6, 1024); - for (idx = 0; idx < 10; ++idx) { - i = base64_encode(data2, 1024, data1, idx); - test_assert(i >= 0); - j = base64_decode(data3, 1024, data2, i); - test_eq(j,idx); - test_memeq(data3, data1, idx); - } - - strlcpy(data1, "Test string that contains 35 chars.", 1024); - strlcat(data1, " 2nd string that contains 35 chars.", 1024); - - i = base64_encode(data2, 1024, data1, 71); - j = base64_decode(data3, 1024, data2, i); - test_eq(j, 71); - test_streq(data3, data1); - test_assert(data2[i] == '\0'); - - crypto_rand(data1, DIGEST_LEN); - memset(data2, 100, 1024); - digest_to_base64(data2, data1); - test_eq(BASE64_DIGEST_LEN, strlen(data2)); - test_eq(100, data2[BASE64_DIGEST_LEN+2]); - memset(data3, 99, 1024); - test_eq(digest_from_base64(data3, data2), 0); - test_memeq(data1, data3, DIGEST_LEN); - test_eq(99, data3[DIGEST_LEN+1]); - - test_assert(digest_from_base64(data3, "###") < 0); - - /* Base32 tests */ - strlcpy(data1, "5chrs", 1024); - /* bit pattern is: [35 63 68 72 73] -> - * [00110101 01100011 01101000 01110010 01110011] - * By 5s: [00110 10101 10001 10110 10000 11100 10011 10011] - */ - base32_encode(data2, 9, data1, 5); - test_streq(data2, "gvrwq4tt"); - - strlcpy(data1, "\xFF\xF5\x6D\x44\xAE\x0D\x5C\xC9\x62\xC4", 1024); - base32_encode(data2, 30, data1, 10); - test_streq(data2, "772w2rfobvomsywe"); - - /* Base16 tests */ - strlcpy(data1, "6chrs\xff", 1024); - base16_encode(data2, 13, data1, 6); - test_streq(data2, "3663687273FF"); - - strlcpy(data1, "f0d678affc000100", 1024); - i = base16_decode(data2, 8, data1, 16); - test_eq(i,0); - test_memeq(data2, "\xf0\xd6\x78\xaf\xfc\x00\x01\x00",8); - - /* now try some failing base16 decodes */ - test_eq(-1, base16_decode(data2, 8, data1, 15)); /* odd input len */ - test_eq(-1, base16_decode(data2, 7, data1, 16)); /* dest too short */ - strlcpy(data1, "f0dz!8affc000100", 1024); - test_eq(-1, base16_decode(data2, 8, data1, 16)); - - tor_free(data1); - tor_free(data2); - tor_free(data3); - - /* Add spaces to fingerprint */ - { - data1 = tor_strdup("ABCD1234ABCD56780000ABCD1234ABCD56780000"); - test_eq(strlen(data1), 40); - data2 = tor_malloc(FINGERPRINT_LEN+1); - add_spaces_to_fp(data2, FINGERPRINT_LEN+1, data1); - test_streq(data2, "ABCD 1234 ABCD 5678 0000 ABCD 1234 ABCD 5678 0000"); - tor_free(data1); - tor_free(data2); - } - - /* Check fingerprint */ - { - test_assert(crypto_pk_check_fingerprint_syntax( - "ABCD 1234 ABCD 5678 0000 ABCD 1234 ABCD 5678 0000")); - test_assert(!crypto_pk_check_fingerprint_syntax( - "ABCD 1234 ABCD 5678 0000 ABCD 1234 ABCD 5678 000")); - test_assert(!crypto_pk_check_fingerprint_syntax( - "ABCD 1234 ABCD 5678 0000 ABCD 1234 ABCD 5678 00000")); - test_assert(!crypto_pk_check_fingerprint_syntax( - "ABCD 1234 ABCD 5678 0000 ABCD1234 ABCD 5678 0000")); - test_assert(!crypto_pk_check_fingerprint_syntax( - "ABCD 1234 ABCD 5678 0000 ABCD1234 ABCD 5678 00000")); - test_assert(!crypto_pk_check_fingerprint_syntax( - "ACD 1234 ABCD 5678 0000 ABCD 1234 ABCD 5678 00000")); - } - - done: - tor_free(data1); - tor_free(data2); - tor_free(data3); -} - -/** Run unit tests for our secret-to-key passphrase hashing functionality. */ -static void -test_crypto_s2k(void) -{ - char buf[29]; - char buf2[29]; - char *buf3 = NULL; - int i; - - memset(buf, 0, sizeof(buf)); - memset(buf2, 0, sizeof(buf2)); - buf3 = tor_malloc(65536); - memset(buf3, 0, 65536); - - secret_to_key(buf+9, 20, "", 0, buf); - crypto_digest(buf2+9, buf3, 1024); - test_memeq(buf, buf2, 29); - - memcpy(buf,"vrbacrda",8); - memcpy(buf2,"vrbacrda",8); - buf[8] = 96; - buf2[8] = 96; - secret_to_key(buf+9, 20, "12345678", 8, buf); - for (i = 0; i < 65536; i += 16) { - memcpy(buf3+i, "vrbacrda12345678", 16); - } - crypto_digest(buf2+9, buf3, 65536); - test_memeq(buf, buf2, 29); - - done: - tor_free(buf3); -} - -/** Helper: return a tristate based on comparing the strings in *<b>a</b> and - * *<b>b</b>. */ -static int -_compare_strs(const void **a, const void **b) -{ - const char *s1 = *a, *s2 = *b; - return strcmp(s1, s2); -} - -/** Helper: return a tristate based on comparing the strings in *<b>a</b> and - * *<b>b</b>, excluding a's first character, and ignoring case. */ -static int -_compare_without_first_ch(const void *a, const void **b) -{ - const char *s1 = a, *s2 = *b; - return strcasecmp(s1+1, s2); -} - -/** Test basic utility functionality. */ -static void -test_util(void) -{ - struct timeval start, end; - struct tm a_time; - char timestr[RFC1123_TIME_LEN+1]; - char buf[1024]; - time_t t_res; - int i; - uint32_t u32; - uint16_t u16; - char *cp, *k, *v; - const char *str; - - start.tv_sec = 5; - start.tv_usec = 5000; - - end.tv_sec = 5; - end.tv_usec = 5000; - - test_eq(0L, tv_udiff(&start, &end)); - - end.tv_usec = 7000; - - test_eq(2000L, tv_udiff(&start, &end)); - - end.tv_sec = 6; - - test_eq(1002000L, tv_udiff(&start, &end)); - - end.tv_usec = 0; - - test_eq(995000L, tv_udiff(&start, &end)); - - end.tv_sec = 4; - - test_eq(-1005000L, tv_udiff(&start, &end)); - - end.tv_usec = 999990; - start.tv_sec = 1; - start.tv_usec = 500; - - /* The test values here are confirmed to be correct on a platform - * with a working timegm. */ - a_time.tm_year = 2003-1900; - a_time.tm_mon = 7; - a_time.tm_mday = 30; - a_time.tm_hour = 6; - a_time.tm_min = 14; - a_time.tm_sec = 55; - test_eq((time_t) 1062224095UL, tor_timegm(&a_time)); - a_time.tm_year = 2004-1900; /* Try a leap year, after feb. */ - test_eq((time_t) 1093846495UL, tor_timegm(&a_time)); - a_time.tm_mon = 1; /* Try a leap year, in feb. */ - a_time.tm_mday = 10; - test_eq((time_t) 1076393695UL, tor_timegm(&a_time)); - - format_rfc1123_time(timestr, 0); - test_streq("Thu, 01 Jan 1970 00:00:00 GMT", timestr); - format_rfc1123_time(timestr, (time_t)1091580502UL); - test_streq("Wed, 04 Aug 2004 00:48:22 GMT", timestr); - - t_res = 0; - i = parse_rfc1123_time(timestr, &t_res); - test_eq(i,0); - test_eq(t_res, (time_t)1091580502UL); - test_eq(-1, parse_rfc1123_time("Wed, zz Aug 2004 99-99x99 GMT", &t_res)); - tor_gettimeofday(&start); - - /* Tests for corner cases of strl operations */ - test_eq(5, strlcpy(buf, "Hello", 0)); - strlcpy(buf, "Hello", sizeof(buf)); - test_eq(10, strlcat(buf, "Hello", 5)); - - /* Test tor_strstrip() */ - strlcpy(buf, "Testing 1 2 3", sizeof(buf)); - tor_strstrip(buf, ",!"); - test_streq(buf, "Testing 1 2 3"); - strlcpy(buf, "!Testing 1 2 3?", sizeof(buf)); - tor_strstrip(buf, "!? "); - test_streq(buf, "Testing123"); - - /* Test parse_addr_port */ - cp = NULL; u32 = 3; u16 = 3; - test_assert(!parse_addr_port(LOG_WARN, "1.2.3.4", &cp, &u32, &u16)); - test_streq(cp, "1.2.3.4"); - test_eq(u32, 0x01020304u); - test_eq(u16, 0); - tor_free(cp); - test_assert(!parse_addr_port(LOG_WARN, "4.3.2.1:99", &cp, &u32, &u16)); - test_streq(cp, "4.3.2.1"); - test_eq(u32, 0x04030201u); - test_eq(u16, 99); - tor_free(cp); - test_assert(!parse_addr_port(LOG_WARN, "nonexistent.address:4040", - &cp, NULL, &u16)); - test_streq(cp, "nonexistent.address"); - test_eq(u16, 4040); - tor_free(cp); - test_assert(!parse_addr_port(LOG_WARN, "localhost:9999", &cp, &u32, &u16)); - test_streq(cp, "localhost"); - test_eq(u32, 0x7f000001u); - test_eq(u16, 9999); - tor_free(cp); - u32 = 3; - test_assert(!parse_addr_port(LOG_WARN, "localhost", NULL, &u32, &u16)); - test_eq(cp, NULL); - test_eq(u32, 0x7f000001u); - test_eq(u16, 0); - tor_free(cp); - test_eq(0, addr_mask_get_bits(0x0u)); - test_eq(32, addr_mask_get_bits(0xFFFFFFFFu)); - test_eq(16, addr_mask_get_bits(0xFFFF0000u)); - test_eq(31, addr_mask_get_bits(0xFFFFFFFEu)); - test_eq(1, addr_mask_get_bits(0x80000000u)); - - /* Test tor_parse_long. */ - test_eq(10L, tor_parse_long("10",10,0,100,NULL,NULL)); - test_eq(0L, tor_parse_long("10",10,50,100,NULL,NULL)); - test_eq(-50L, tor_parse_long("-50",10,-100,100,NULL,NULL)); - - /* Test tor_parse_ulong */ - test_eq(10UL, tor_parse_ulong("10",10,0,100,NULL,NULL)); - test_eq(0UL, tor_parse_ulong("10",10,50,100,NULL,NULL)); - - /* Test tor_parse_uint64. */ - test_assert(U64_LITERAL(10) == tor_parse_uint64("10 x",10,0,100, &i, &cp)); - test_assert(i == 1); - test_streq(cp, " x"); - test_assert(U64_LITERAL(12345678901) == - tor_parse_uint64("12345678901",10,0,UINT64_MAX, &i, &cp)); - test_assert(i == 1); - test_streq(cp, ""); - test_assert(U64_LITERAL(0) == - tor_parse_uint64("12345678901",10,500,INT32_MAX, &i, &cp)); - test_assert(i == 0); - - /* Test failing snprintf cases */ - test_eq(-1, tor_snprintf(buf, 0, "Foo")); - test_eq(-1, tor_snprintf(buf, 2, "Foo")); - - /* Test printf with uint64 */ - tor_snprintf(buf, sizeof(buf), "x!"U64_FORMAT"!x", - U64_PRINTF_ARG(U64_LITERAL(12345678901))); - test_streq(buf, "x!12345678901!x"); - - /* Test parse_config_line_from_str */ - strlcpy(buf, "k v\n" " key value with spaces \n" "keykey val\n" - "k2\n" - "k3 \n" "\n" " \n" "#comment\n" - "k4#a\n" "k5#abc\n" "k6 val #with comment\n" - "kseven \"a quoted 'string\"\n" - "k8 \"a \\x71uoted\\n\\\"str\\\\ing\\t\\001\\01\\1\\\"\"\n" - , sizeof(buf)); - str = buf; - - str = parse_config_line_from_str(str, &k, &v); - test_streq(k, "k"); - test_streq(v, "v"); - tor_free(k); tor_free(v); - test_assert(!strcmpstart(str, "key value with")); - - str = parse_config_line_from_str(str, &k, &v); - test_streq(k, "key"); - test_streq(v, "value with spaces"); - tor_free(k); tor_free(v); - test_assert(!strcmpstart(str, "keykey")); - - str = parse_config_line_from_str(str, &k, &v); - test_streq(k, "keykey"); - test_streq(v, "val"); - tor_free(k); tor_free(v); - test_assert(!strcmpstart(str, "k2\n")); - - str = parse_config_line_from_str(str, &k, &v); - test_streq(k, "k2"); - test_streq(v, ""); - tor_free(k); tor_free(v); - test_assert(!strcmpstart(str, "k3 \n")); - - str = parse_config_line_from_str(str, &k, &v); - test_streq(k, "k3"); - test_streq(v, ""); - tor_free(k); tor_free(v); - test_assert(!strcmpstart(str, "#comment")); - - str = parse_config_line_from_str(str, &k, &v); - test_streq(k, "k4"); - test_streq(v, ""); - tor_free(k); tor_free(v); - test_assert(!strcmpstart(str, "k5#abc")); - - str = parse_config_line_from_str(str, &k, &v); - test_streq(k, "k5"); - test_streq(v, ""); - tor_free(k); tor_free(v); - test_assert(!strcmpstart(str, "k6")); - - str = parse_config_line_from_str(str, &k, &v); - test_streq(k, "k6"); - test_streq(v, "val"); - tor_free(k); tor_free(v); - test_assert(!strcmpstart(str, "kseven")); - - str = parse_config_line_from_str(str, &k, &v); - test_streq(k, "kseven"); - test_streq(v, "a quoted \'string"); - tor_free(k); tor_free(v); - test_assert(!strcmpstart(str, "k8 ")); - - str = parse_config_line_from_str(str, &k, &v); - test_streq(k, "k8"); - test_streq(v, "a quoted\n\"str\\ing\t\x01\x01\x01\""); - tor_free(k); tor_free(v); - test_streq(str, ""); - - /* Test for strcmpstart and strcmpend. */ - test_assert(strcmpstart("abcdef", "abcdef")==0); - test_assert(strcmpstart("abcdef", "abc")==0); - test_assert(strcmpstart("abcdef", "abd")<0); - test_assert(strcmpstart("abcdef", "abb")>0); - test_assert(strcmpstart("ab", "abb")<0); - - test_assert(strcmpend("abcdef", "abcdef")==0); - test_assert(strcmpend("abcdef", "def")==0); - test_assert(strcmpend("abcdef", "deg")<0); - test_assert(strcmpend("abcdef", "dee")>0); - test_assert(strcmpend("ab", "abb")<0); - - test_assert(strcasecmpend("AbcDEF", "abcdef")==0); - test_assert(strcasecmpend("abcdef", "dEF")==0); - test_assert(strcasecmpend("abcDEf", "deg")<0); - test_assert(strcasecmpend("abcdef", "DEE")>0); - test_assert(strcasecmpend("ab", "abB")<0); - - /* Test mem_is_zero */ - memset(buf,0,128); - buf[128] = 'x'; - test_assert(tor_digest_is_zero(buf)); - test_assert(tor_mem_is_zero(buf, 10)); - test_assert(tor_mem_is_zero(buf, 20)); - test_assert(tor_mem_is_zero(buf, 128)); - test_assert(!tor_mem_is_zero(buf, 129)); - buf[60] = (char)255; - test_assert(!tor_mem_is_zero(buf, 128)); - buf[0] = (char)1; - test_assert(!tor_mem_is_zero(buf, 10)); - - /* Test inet_ntop */ - { - char tmpbuf[TOR_ADDR_BUF_LEN]; - const char *ip = "176.192.208.224"; - struct in_addr in; - tor_inet_pton(AF_INET, ip, &in); - tor_inet_ntop(AF_INET, &in, tmpbuf, sizeof(tmpbuf)); - test_streq(tmpbuf, ip); - } - - /* Test 'escaped' */ - test_streq("\"\"", escaped("")); - test_streq("\"abcd\"", escaped("abcd")); - test_streq("\"\\\\\\n\\r\\t\\\"\\'\"", escaped("\\\n\r\t\"\'")); - test_streq("\"z\\001abc\\277d\"", escaped("z\001abc\277d")); - test_assert(NULL == escaped(NULL)); - - /* Test strndup and memdup */ - { - const char *s = "abcdefghijklmnopqrstuvwxyz"; - cp = tor_strndup(s, 30); - test_streq(cp, s); /* same string, */ - test_neq(cp, s); /* but different pointers. */ - tor_free(cp); - - cp = tor_strndup(s, 5); - test_streq(cp, "abcde"); - tor_free(cp); - - s = "a\0b\0c\0d\0e\0"; - cp = tor_memdup(s,10); - test_memeq(cp, s, 10); /* same ram, */ - test_neq(cp, s); /* but different pointers. */ - tor_free(cp); - } - - /* Test str-foo functions */ - cp = tor_strdup("abcdef"); - test_assert(tor_strisnonupper(cp)); - cp[3] = 'D'; - test_assert(!tor_strisnonupper(cp)); - tor_strupper(cp); - test_streq(cp, "ABCDEF"); - test_assert(tor_strisprint(cp)); - cp[3] = 3; - test_assert(!tor_strisprint(cp)); - tor_free(cp); - - /* Test eat_whitespace. */ - { - const char *s = " \n a"; - test_eq_ptr(eat_whitespace(s), s+4); - s = "abcd"; - test_eq_ptr(eat_whitespace(s), s); - s = "#xyz\nab"; - test_eq_ptr(eat_whitespace(s), s+5); - } - - /* Test memmem and memstr */ - { - const char *haystack = "abcde"; - tor_assert(!tor_memmem(haystack, 5, "ef", 2)); - test_eq_ptr(tor_memmem(haystack, 5, "cd", 2), haystack + 2); - test_eq_ptr(tor_memmem(haystack, 5, "cde", 3), haystack + 2); - haystack = "ababcad"; - test_eq_ptr(tor_memmem(haystack, 7, "abc", 3), haystack + 2); - test_eq_ptr(tor_memstr(haystack, 7, "abc"), haystack + 2); - test_assert(!tor_memstr(haystack, 7, "fe")); - test_assert(!tor_memstr(haystack, 7, "longerthantheoriginal")); - } - - /* Test wrap_string */ - { - smartlist_t *sl = smartlist_create(); - wrap_string(sl, "This is a test of string wrapping functionality: woot.", - 10, "", ""); - cp = smartlist_join_strings(sl, "", 0, NULL); - test_streq(cp, - "This is a\ntest of\nstring\nwrapping\nfunctional\nity: woot.\n"); - tor_free(cp); - SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); - smartlist_clear(sl); - - wrap_string(sl, "This is a test of string wrapping functionality: woot.", - 16, "### ", "# "); - cp = smartlist_join_strings(sl, "", 0, NULL); - test_streq(cp, - "### This is a\n# test of string\n# wrapping\n# functionality:\n" - "# woot.\n"); - - tor_free(cp); - SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); - smartlist_free(sl); - } - - tor_gettimeofday(&start); - /* now make sure time works. */ - tor_gettimeofday(&end); - /* We might've timewarped a little. */ - test_assert(tv_udiff(&start, &end) >= -5000); - - /* Test tor_log2(). */ - test_eq(tor_log2(64), 6); - test_eq(tor_log2(65), 6); - test_eq(tor_log2(63), 5); - test_eq(tor_log2(1), 0); - test_eq(tor_log2(2), 1); - test_eq(tor_log2(3), 1); - test_eq(tor_log2(4), 2); - test_eq(tor_log2(5), 2); - test_eq(tor_log2(U64_LITERAL(40000000000000000)), 55); - test_eq(tor_log2(UINT64_MAX), 63); - - /* Test round_to_power_of_2 */ - test_eq(round_to_power_of_2(120), 128); - test_eq(round_to_power_of_2(128), 128); - test_eq(round_to_power_of_2(130), 128); - test_eq(round_to_power_of_2(U64_LITERAL(40000000000000000)), - U64_LITERAL(1)<<55); - test_eq(round_to_power_of_2(0), 2); - - done: - ; -} - -static void -test_util_di_ops(void) -{ -#define LT -1 -#define GT 1 -#define EQ 0 - const struct { - const char *a; int want_sign; const char *b; - } examples[] = { - { "Foo", EQ, "Foo" }, - { "foo", GT, "bar", }, - { "foobar", EQ ,"foobar" }, - { "foobar", LT, "foobaw" }, - { "foobar", GT, "f00bar" }, - { "foobar", GT, "boobar" }, - { "", EQ, "" }, - { NULL, 0, NULL }, - }; - - int i; - - for (i = 0; examples[i].a; ++i) { - size_t len = strlen(examples[i].a); - int eq1, eq2, neq1, neq2, cmp1, cmp2; - test_eq(len, strlen(examples[i].b)); - /* We do all of the operations, with operands in both orders. */ - eq1 = tor_memeq(examples[i].a, examples[i].b, len); - eq2 = tor_memeq(examples[i].b, examples[i].a, len); - neq1 = tor_memneq(examples[i].a, examples[i].b, len); - neq2 = tor_memneq(examples[i].b, examples[i].a, len); - cmp1 = tor_memcmp(examples[i].a, examples[i].b, len); - cmp2 = tor_memcmp(examples[i].b, examples[i].a, len); - - /* Check for correctness of cmp1 */ - if (cmp1 < 0 && examples[i].want_sign != LT) - test_fail(); - else if (cmp1 > 0 && examples[i].want_sign != GT) - test_fail(); - else if (cmp1 == 0 && examples[i].want_sign != EQ) - test_fail(); - - /* Check for consistency of everything else with cmp1 */ - test_eq(eq1, eq2); - test_eq(neq1, neq2); - test_eq(cmp1, -cmp2); - test_eq(eq1, cmp1 == 0); - test_eq(neq1, !eq1); - } - - done: - ; -} - -/** Helper: assert that IPv6 addresses <b>a</b> and <b>b</b> are the same. On - * failure, reports an error, describing the addresses as <b>e1</b> and - * <b>e2</b>, and reporting the line number as <b>line</b>. */ -static void -_test_eq_ip6(struct in6_addr *a, struct in6_addr *b, const char *e1, - const char *e2, int line) -{ - int i; - int ok = 1; - for (i = 0; i < 16; ++i) { - if (a->s6_addr[i] != b->s6_addr[i]) { - ok = 0; - break; - } - } - if (ok) { - printf("."); fflush(stdout); - } else { - char buf1[128], *cp1; - char buf2[128], *cp2; - have_failed = 1; - cp1 = buf1; cp2 = buf2; - for (i=0; i<16; ++i) { - tor_snprintf(cp1, sizeof(buf1)-(cp1-buf1), "%02x", a->s6_addr[i]); - tor_snprintf(cp2, sizeof(buf2)-(cp2-buf2), "%02x", b->s6_addr[i]); - cp1 += 2; cp2 += 2; - if ((i%2)==1 && i != 15) { - *cp1++ = ':'; - *cp2++ = ':'; - } - } - *cp1 = *cp2 = '\0'; - printf("Line %d: assertion failed: (%s == %s)\n" - " %s != %s\n", line, e1, e2, buf1, buf2); - fflush(stdout); - } -} - -/** Helper: Assert that two strings both decode as IPv6 addresses with - * tor_inet_pton(), and both decode to the same address. */ -#define test_pton6_same(a,b) STMT_BEGIN \ - test_eq(tor_inet_pton(AF_INET6, a, &a1), 1); \ - test_eq(tor_inet_pton(AF_INET6, b, &a2), 1); \ - _test_eq_ip6(&a1,&a2,#a,#b,__LINE__); \ - STMT_END - -/** Helper: Assert that <b>a</b> is recognized as a bad IPv6 address by - * tor_inet_pton(). */ -#define test_pton6_bad(a) \ - test_eq(0, tor_inet_pton(AF_INET6, a, &a1)) - -/** Helper: assert that <b>a</b>, when parsed by tor_inet_pton() and displayed - * with tor_inet_ntop(), yields <b>b</b>. Also assert that <b>b</b> parses to - * the same value as <b>a</b>. */ -#define test_ntop6_reduces(a,b) STMT_BEGIN \ - test_eq(tor_inet_pton(AF_INET6, a, &a1), 1); \ - test_streq(tor_inet_ntop(AF_INET6, &a1, buf, sizeof(buf)), b); \ - test_eq(tor_inet_pton(AF_INET6, b, &a2), 1); \ - _test_eq_ip6(&a1, &a2, a, b, __LINE__); \ - STMT_END - -/** Helper: assert that <b>a</b> parses by tor_inet_pton() into a address that - * passes tor_addr_is_internal() with <b>for_listening</b>. */ -#define test_internal_ip(a,for_listening) STMT_BEGIN \ - test_eq(tor_inet_pton(AF_INET6, a, &t1.addr.in6_addr), 1); \ - t1.family = AF_INET6; \ - if (!tor_addr_is_internal(&t1, for_listening)) \ - test_fail_msg( a "was not internal."); \ - STMT_END - -/** Helper: assert that <b>a</b> parses by tor_inet_pton() into a address that - * does not pass tor_addr_is_internal() with <b>for_listening</b>. */ -#define test_external_ip(a,for_listening) STMT_BEGIN \ - test_eq(tor_inet_pton(AF_INET6, a, &t1.addr.in6_addr), 1); \ - t1.family = AF_INET6; \ - if (tor_addr_is_internal(&t1, for_listening)) \ - test_fail_msg(a "was not external."); \ - STMT_END - -/** Helper: Assert that <b>a</b> and <b>b</b>, when parsed by - * tor_inet_pton(), give addresses that compare in the order defined by - * <b>op</b> with tor_addr_compare(). */ -#define test_addr_compare(a, op, b) STMT_BEGIN \ - test_eq(tor_inet_pton(AF_INET6, a, &t1.addr.in6_addr), 1); \ - test_eq(tor_inet_pton(AF_INET6, b, &t2.addr.in6_addr), 1); \ - t1.family = t2.family = AF_INET6; \ - r = tor_addr_compare(&t1,&t2,CMP_SEMANTIC); \ - if (!(r op 0)) \ - test_fail_msg("failed: tor_addr_compare("a","b") "#op" 0"); \ - STMT_END - -/** Helper: Assert that <b>a</b> and <b>b</b>, when parsed by - * tor_inet_pton(), give addresses that compare in the order defined by - * <b>op</b> with tor_addr_compare_masked() with <b>m</b> masked. */ -#define test_addr_compare_masked(a, op, b, m) STMT_BEGIN \ - test_eq(tor_inet_pton(AF_INET6, a, &t1.addr.in6_addr), 1); \ - test_eq(tor_inet_pton(AF_INET6, b, &t2.addr.in6_addr), 1); \ - t1.family = t2.family = AF_INET6; \ - r = tor_addr_compare_masked(&t1,&t2,m,CMP_SEMANTIC); \ - if (!(r op 0)) \ - test_fail_msg("failed: tor_addr_compare_masked("a","b","#m") "#op" 0"); \ - STMT_END - -/** Helper: assert that <b>xx</b> is parseable as a masked IPv6 address with - * ports by tor_parse_mask_addr_ports(), with family <b>f</b>, IP address - * as 4 32-bit words <b>ip1...ip4</b>, mask bits as <b>mm</b>, and port range - * as <b>pt1..pt2</b>. */ -#define test_addr_mask_ports_parse(xx, f, ip1, ip2, ip3, ip4, mm, pt1, pt2) \ - STMT_BEGIN \ - test_eq(tor_addr_parse_mask_ports(xx, &t1, &mask, &port1, &port2), f); \ - p1=tor_inet_ntop(AF_INET6, &t1.addr.in6_addr, bug, sizeof(bug)); \ - test_eq(htonl(ip1), tor_addr_to_in6_addr32(&t1)[0]); \ - test_eq(htonl(ip2), tor_addr_to_in6_addr32(&t1)[1]); \ - test_eq(htonl(ip3), tor_addr_to_in6_addr32(&t1)[2]); \ - test_eq(htonl(ip4), tor_addr_to_in6_addr32(&t1)[3]); \ - test_eq(mask, mm); \ - test_eq(port1, pt1); \ - test_eq(port2, pt2); \ - STMT_END - -/** Run unit tests for IPv6 encoding/decoding/manipulation functions. */ -static void -test_util_ip6_helpers(void) -{ - char buf[TOR_ADDR_BUF_LEN], bug[TOR_ADDR_BUF_LEN]; - struct in6_addr a1, a2; - tor_addr_t t1, t2; - int r, i; - uint16_t port1, port2; - maskbits_t mask; - const char *p1; - struct sockaddr_storage sa_storage; - struct sockaddr_in *sin; - struct sockaddr_in6 *sin6; - - // struct in_addr b1, b2; - /* Test tor_inet_ntop and tor_inet_pton: IPv6 */ - - /* ==== Converting to and from sockaddr_t. */ - sin = (struct sockaddr_in *)&sa_storage; - sin->sin_family = AF_INET; - sin->sin_port = 9090; - sin->sin_addr.s_addr = htonl(0x7f7f0102); /*127.127.1.2*/ - tor_addr_from_sockaddr(&t1, (struct sockaddr *)sin, NULL); - test_eq(tor_addr_family(&t1), AF_INET); - test_eq(tor_addr_to_ipv4h(&t1), 0x7f7f0102); - - memset(&sa_storage, 0, sizeof(sa_storage)); - test_eq(sizeof(struct sockaddr_in), - tor_addr_to_sockaddr(&t1, 1234, (struct sockaddr *)&sa_storage, - sizeof(sa_storage))); - test_eq(1234, ntohs(sin->sin_port)); - test_eq(0x7f7f0102, ntohl(sin->sin_addr.s_addr)); - - memset(&sa_storage, 0, sizeof(sa_storage)); - sin6 = (struct sockaddr_in6 *)&sa_storage; - sin6->sin6_family = AF_INET6; - sin6->sin6_port = htons(7070); - sin6->sin6_addr.s6_addr[0] = 128; - tor_addr_from_sockaddr(&t1, (struct sockaddr *)sin6, NULL); - test_eq(tor_addr_family(&t1), AF_INET6); - p1 = tor_addr_to_str(buf, &t1, sizeof(buf), 0); - test_streq(p1, "8000::"); - - memset(&sa_storage, 0, sizeof(sa_storage)); - test_eq(sizeof(struct sockaddr_in6), - tor_addr_to_sockaddr(&t1, 9999, (struct sockaddr *)&sa_storage, - sizeof(sa_storage))); - test_eq(AF_INET6, sin6->sin6_family); - test_eq(9999, ntohs(sin6->sin6_port)); - test_eq(0x80000000, ntohl(S6_ADDR32(sin6->sin6_addr)[0])); - - /* ==== tor_addr_lookup: static cases. (Can't test dns without knowing we - * have a good resolver. */ - test_eq(0, tor_addr_lookup("127.128.129.130", AF_UNSPEC, &t1)); - test_eq(AF_INET, tor_addr_family(&t1)); - test_eq(tor_addr_to_ipv4h(&t1), 0x7f808182); - - test_eq(0, tor_addr_lookup("9000::5", AF_UNSPEC, &t1)); - test_eq(AF_INET6, tor_addr_family(&t1)); - test_eq(0x90, tor_addr_to_in6_addr8(&t1)[0]); - test_assert(tor_mem_is_zero((char*)tor_addr_to_in6_addr8(&t1)+1, 14)); - test_eq(0x05, tor_addr_to_in6_addr8(&t1)[15]); - - /* === Test pton: valid af_inet6 */ - /* Simple, valid parsing. */ - r = tor_inet_pton(AF_INET6, - "0102:0304:0506:0708:090A:0B0C:0D0E:0F10", &a1); - test_assert(r==1); - for (i=0;i<16;++i) { test_eq(i+1, (int)a1.s6_addr[i]); } - /* ipv4 ending. */ - test_pton6_same("0102:0304:0506:0708:090A:0B0C:0D0E:0F10", - "0102:0304:0506:0708:090A:0B0C:13.14.15.16"); - /* shortened words. */ - test_pton6_same("0001:0099:BEEF:0000:0123:FFFF:0001:0001", - "1:99:BEEF:0:0123:FFFF:1:1"); - /* zeros at the beginning */ - test_pton6_same("0000:0000:0000:0000:0009:C0A8:0001:0001", - "::9:c0a8:1:1"); - test_pton6_same("0000:0000:0000:0000:0009:C0A8:0001:0001", - "::9:c0a8:0.1.0.1"); - /* zeros in the middle. */ - test_pton6_same("fe80:0000:0000:0000:0202:1111:0001:0001", - "fe80::202:1111:1:1"); - /* zeros at the end. */ - test_pton6_same("1000:0001:0000:0007:0000:0000:0000:0000", - "1000:1:0:7::"); - - /* === Test ntop: af_inet6 */ - test_ntop6_reduces("0:0:0:0:0:0:0:0", "::"); - - test_ntop6_reduces("0001:0099:BEEF:0006:0123:FFFF:0001:0001", - "1:99:beef:6:123:ffff:1:1"); - - //test_ntop6_reduces("0:0:0:0:0:0:c0a8:0101", "::192.168.1.1"); - test_ntop6_reduces("0:0:0:0:0:ffff:c0a8:0101", "::ffff:192.168.1.1"); - test_ntop6_reduces("002:0:0000:0:3::4", "2::3:0:0:4"); - test_ntop6_reduces("0:0::1:0:3", "::1:0:3"); - test_ntop6_reduces("008:0::0", "8::"); - test_ntop6_reduces("0:0:0:0:0:ffff::1", "::ffff:0.0.0.1"); - test_ntop6_reduces("abcd:0:0:0:0:0:7f00::", "abcd::7f00:0"); - test_ntop6_reduces("0000:0000:0000:0000:0009:C0A8:0001:0001", - "::9:c0a8:1:1"); - test_ntop6_reduces("fe80:0000:0000:0000:0202:1111:0001:0001", - "fe80::202:1111:1:1"); - test_ntop6_reduces("1000:0001:0000:0007:0000:0000:0000:0000", - "1000:1:0:7::"); - - /* === Test pton: invalid in6. */ - test_pton6_bad("foobar."); - test_pton6_bad("55555::"); - test_pton6_bad("9:-60::"); - test_pton6_bad("1:2:33333:4:0002:3::"); - //test_pton6_bad("1:2:3333:4:00002:3::");// BAD, but glibc doesn't say so. - test_pton6_bad("1:2:3333:4:fish:3::"); - test_pton6_bad("1:2:3:4:5:6:7:8:9"); - test_pton6_bad("1:2:3:4:5:6:7"); - test_pton6_bad("1:2:3:4:5:6:1.2.3.4.5"); - test_pton6_bad("1:2:3:4:5:6:1.2.3"); - test_pton6_bad("::1.2.3"); - test_pton6_bad("::1.2.3.4.5"); - test_pton6_bad("99"); - test_pton6_bad(""); - test_pton6_bad("1::2::3:4"); - test_pton6_bad("a:::b:c"); - test_pton6_bad(":::a:b:c"); - test_pton6_bad("a:b:c:::"); - - /* test internal checking */ - test_external_ip("fbff:ffff::2:7", 0); - test_internal_ip("fc01::2:7", 0); - test_internal_ip("fdff:ffff::f:f", 0); - test_external_ip("fe00::3:f", 0); - - test_external_ip("fe7f:ffff::2:7", 0); - test_internal_ip("fe80::2:7", 0); - test_internal_ip("febf:ffff::f:f", 0); - - test_internal_ip("fec0::2:7:7", 0); - test_internal_ip("feff:ffff::e:7:7", 0); - test_external_ip("ff00::e:7:7", 0); - - test_internal_ip("::", 0); - test_internal_ip("::1", 0); - test_internal_ip("::1", 1); - test_internal_ip("::", 0); - test_external_ip("::", 1); - test_external_ip("::2", 0); - test_external_ip("2001::", 0); - test_external_ip("ffff::", 0); - - test_external_ip("::ffff:0.0.0.0", 1); - test_internal_ip("::ffff:0.0.0.0", 0); - test_internal_ip("::ffff:0.255.255.255", 0); - test_external_ip("::ffff:1.0.0.0", 0); - - test_external_ip("::ffff:9.255.255.255", 0); - test_internal_ip("::ffff:10.0.0.0", 0); - test_internal_ip("::ffff:10.255.255.255", 0); - test_external_ip("::ffff:11.0.0.0", 0); - - test_external_ip("::ffff:126.255.255.255", 0); - test_internal_ip("::ffff:127.0.0.0", 0); - test_internal_ip("::ffff:127.255.255.255", 0); - test_external_ip("::ffff:128.0.0.0", 0); - - test_external_ip("::ffff:172.15.255.255", 0); - test_internal_ip("::ffff:172.16.0.0", 0); - test_internal_ip("::ffff:172.31.255.255", 0); - test_external_ip("::ffff:172.32.0.0", 0); - - test_external_ip("::ffff:192.167.255.255", 0); - test_internal_ip("::ffff:192.168.0.0", 0); - test_internal_ip("::ffff:192.168.255.255", 0); - test_external_ip("::ffff:192.169.0.0", 0); - - test_external_ip("::ffff:169.253.255.255", 0); - test_internal_ip("::ffff:169.254.0.0", 0); - test_internal_ip("::ffff:169.254.255.255", 0); - test_external_ip("::ffff:169.255.0.0", 0); - test_assert(is_internal_IP(0x7f000001, 0)); - - /* tor_addr_compare(tor_addr_t x2) */ - test_addr_compare("ffff::", ==, "ffff::0"); - test_addr_compare("0::3:2:1", <, "0::ffff:0.3.2.1"); - test_addr_compare("0::2:2:1", <, "0::ffff:0.3.2.1"); - test_addr_compare("0::ffff:0.3.2.1", >, "0::0:0:0"); - test_addr_compare("0::ffff:5.2.2.1", <, "::ffff:6.0.0.0"); /* XXXX wrong. */ - tor_addr_parse_mask_ports("[::ffff:2.3.4.5]", &t1, NULL, NULL, NULL); - tor_addr_parse_mask_ports("2.3.4.5", &t2, NULL, NULL, NULL); - test_assert(tor_addr_compare(&t1, &t2, CMP_SEMANTIC) == 0); - tor_addr_parse_mask_ports("[::ffff:2.3.4.4]", &t1, NULL, NULL, NULL); - tor_addr_parse_mask_ports("2.3.4.5", &t2, NULL, NULL, NULL); - test_assert(tor_addr_compare(&t1, &t2, CMP_SEMANTIC) < 0); - - /* test compare_masked */ - test_addr_compare_masked("ffff::", ==, "ffff::0", 128); - test_addr_compare_masked("ffff::", ==, "ffff::0", 64); - test_addr_compare_masked("0::2:2:1", <, "0::8000:2:1", 81); - test_addr_compare_masked("0::2:2:1", ==, "0::8000:2:1", 80); - - /* Test decorated addr_to_string. */ - test_eq(AF_INET6, tor_addr_from_str(&t1, "[123:45:6789::5005:11]")); - p1 = tor_addr_to_str(buf, &t1, sizeof(buf), 1); - test_streq(p1, "[123:45:6789::5005:11]"); - test_eq(AF_INET, tor_addr_from_str(&t1, "18.0.0.1")); - p1 = tor_addr_to_str(buf, &t1, sizeof(buf), 1); - test_streq(p1, "18.0.0.1"); - - /* Test tor_addr_parse_reverse_lookup_name */ - i = tor_addr_parse_reverse_lookup_name(&t1, "Foobar.baz", AF_UNSPEC, 0); - test_eq(0, i); - i = tor_addr_parse_reverse_lookup_name(&t1, "Foobar.baz", AF_UNSPEC, 1); - test_eq(0, i); - i = tor_addr_parse_reverse_lookup_name(&t1, "1.0.168.192.in-addr.arpa", - AF_UNSPEC, 1); - test_eq(1, i); - test_eq(tor_addr_family(&t1), AF_INET); - p1 = tor_addr_to_str(buf, &t1, sizeof(buf), 1); - test_streq(p1, "192.168.0.1"); - i = tor_addr_parse_reverse_lookup_name(&t1, "192.168.0.99", AF_UNSPEC, 0); - test_eq(0, i); - i = tor_addr_parse_reverse_lookup_name(&t1, "192.168.0.99", AF_UNSPEC, 1); - test_eq(1, i); - p1 = tor_addr_to_str(buf, &t1, sizeof(buf), 1); - test_streq(p1, "192.168.0.99"); - memset(&t1, 0, sizeof(t1)); - i = tor_addr_parse_reverse_lookup_name(&t1, - "0.1.2.3.4.5.6.7.8.9.a.b.c.d.e.f." - "f.e.e.b.1.e.b.e.e.f.f.e.e.e.d.9." - "ip6.ARPA", - AF_UNSPEC, 0); - test_eq(1, i); - p1 = tor_addr_to_str(buf, &t1, sizeof(buf), 1); - test_streq(p1, "[9dee:effe:ebe1:beef:fedc:ba98:7654:3210]"); - /* Failing cases. */ - i = tor_addr_parse_reverse_lookup_name(&t1, - "6.7.8.9.a.b.c.d.e.f." - "f.e.e.b.1.e.b.e.e.f.f.e.e.e.d.9." - "ip6.ARPA", - AF_UNSPEC, 0); - test_eq(i, -1); - i = tor_addr_parse_reverse_lookup_name(&t1, - "6.7.8.9.a.b.c.d.e.f.a.b.c.d.e.f.0." - "f.e.e.b.1.e.b.e.e.f.f.e.e.e.d.9." - "ip6.ARPA", - AF_UNSPEC, 0); - test_eq(i, -1); - i = tor_addr_parse_reverse_lookup_name(&t1, - "6.7.8.9.a.b.c.d.e.f.X.0.0.0.0.9." - "f.e.e.b.1.e.b.e.e.f.f.e.e.e.d.9." - "ip6.ARPA", - AF_UNSPEC, 0); - test_eq(i, -1); - i = tor_addr_parse_reverse_lookup_name(&t1, "32.1.1.in-addr.arpa", - AF_UNSPEC, 0); - test_eq(i, -1); - i = tor_addr_parse_reverse_lookup_name(&t1, ".in-addr.arpa", - AF_UNSPEC, 0); - test_eq(i, -1); - i = tor_addr_parse_reverse_lookup_name(&t1, "1.2.3.4.5.in-addr.arpa", - AF_UNSPEC, 0); - test_eq(i, -1); - i = tor_addr_parse_reverse_lookup_name(&t1, "1.2.3.4.5.in-addr.arpa", - AF_INET6, 0); - test_eq(i, -1); - i = tor_addr_parse_reverse_lookup_name(&t1, - "6.7.8.9.a.b.c.d.e.f.a.b.c.d.e.0." - "f.e.e.b.1.e.b.e.e.f.f.e.e.e.d.9." - "ip6.ARPA", - AF_INET, 0); - test_eq(i, -1); - - /* test tor_addr_parse_mask_ports */ - test_addr_mask_ports_parse("[::f]/17:47-95", AF_INET6, - 0, 0, 0, 0x0000000f, 17, 47, 95); - //test_addr_parse("[::fefe:4.1.1.7/120]:999-1000"); - //test_addr_parse_check("::fefe:401:107", 120, 999, 1000); - test_addr_mask_ports_parse("[::ffff:4.1.1.7]/120:443", AF_INET6, - 0, 0, 0x0000ffff, 0x04010107, 120, 443, 443); - test_addr_mask_ports_parse("[abcd:2::44a:0]:2-65000", AF_INET6, - 0xabcd0002, 0, 0, 0x044a0000, 128, 2, 65000); - - r=tor_addr_parse_mask_ports("[fefef::]/112", &t1, NULL, NULL, NULL); - test_assert(r == -1); - r=tor_addr_parse_mask_ports("efef::/112", &t1, NULL, NULL, NULL); - test_assert(r == -1); - r=tor_addr_parse_mask_ports("[f:f:f:f:f:f:f:f::]", &t1, NULL, NULL, NULL); - test_assert(r == -1); - r=tor_addr_parse_mask_ports("[::f:f:f:f:f:f:f:f]", &t1, NULL, NULL, NULL); - test_assert(r == -1); - r=tor_addr_parse_mask_ports("[f:f:f:f:f:f:f:f:f]", &t1, NULL, NULL, NULL); - test_assert(r == -1); - /* Test for V4-mapped address with mask < 96. (arguably not valid) */ - r=tor_addr_parse_mask_ports("[::ffff:1.1.2.2/33]", &t1, &mask, NULL, NULL); - test_assert(r == -1); - r=tor_addr_parse_mask_ports("1.1.2.2/33", &t1, &mask, NULL, NULL); - test_assert(r == -1); - r=tor_addr_parse_mask_ports("1.1.2.2/31", &t1, &mask, NULL, NULL); - test_assert(r == AF_INET); - r=tor_addr_parse_mask_ports("[efef::]/112", &t1, &mask, &port1, &port2); - test_assert(r == AF_INET6); - test_assert(port1 == 1); - test_assert(port2 == 65535); - - /* make sure inet address lengths >= max */ - test_assert(INET_NTOA_BUF_LEN >= sizeof("255.255.255.255")); - test_assert(TOR_ADDR_BUF_LEN >= - sizeof("ffff:ffff:ffff:ffff:ffff:ffff:255.255.255.255")); - - test_assert(sizeof(tor_addr_t) >= sizeof(struct in6_addr)); - - /* get interface addresses */ - r = get_interface_address6(LOG_DEBUG, AF_INET, &t1); - i = get_interface_address6(LOG_DEBUG, AF_INET6, &t2); -#if 0 - tor_inet_ntop(AF_INET, &t1.sa.sin_addr, buf, sizeof(buf)); - printf("\nv4 address: %s (family=%i)", buf, IN_FAMILY(&t1)); - tor_inet_ntop(AF_INET6, &t2.sa6.sin6_addr, buf, sizeof(buf)); - printf("\nv6 address: %s (family=%i)", buf, IN_FAMILY(&t2)); -#endif - - done: - ; -} - -/** Run unit tests for basic dynamic-sized array functionality. */ -static void -test_util_smartlist_basic(void) -{ - smartlist_t *sl; - - /* XXXX test sort_digests, uniq_strings, uniq_digests */ - - /* Test smartlist add, del_keeporder, insert, get. */ - sl = smartlist_create(); - smartlist_add(sl, (void*)1); - smartlist_add(sl, (void*)2); - smartlist_add(sl, (void*)3); - smartlist_add(sl, (void*)4); - smartlist_del_keeporder(sl, 1); - smartlist_insert(sl, 1, (void*)22); - smartlist_insert(sl, 0, (void*)0); - smartlist_insert(sl, 5, (void*)555); - test_eq_ptr((void*)0, smartlist_get(sl,0)); - test_eq_ptr((void*)1, smartlist_get(sl,1)); - test_eq_ptr((void*)22, smartlist_get(sl,2)); - test_eq_ptr((void*)3, smartlist_get(sl,3)); - test_eq_ptr((void*)4, smartlist_get(sl,4)); - test_eq_ptr((void*)555, smartlist_get(sl,5)); - /* Try deleting in the middle. */ - smartlist_del(sl, 1); - test_eq_ptr((void*)555, smartlist_get(sl, 1)); - /* Try deleting at the end. */ - smartlist_del(sl, 4); - test_eq(4, smartlist_len(sl)); - - /* test isin. */ - test_assert(smartlist_isin(sl, (void*)3)); - test_assert(!smartlist_isin(sl, (void*)99)); - - done: - smartlist_free(sl); -} - -/** Run unit tests for smartlist-of-strings functionality. */ -static void -test_util_smartlist_strings(void) -{ - smartlist_t *sl = smartlist_create(); - char *cp=NULL, *cp_alloc=NULL; - size_t sz; - - /* Test split and join */ - test_eq(0, smartlist_len(sl)); - smartlist_split_string(sl, "abc", ":", 0, 0); - test_eq(1, smartlist_len(sl)); - test_streq("abc", smartlist_get(sl, 0)); - smartlist_split_string(sl, "a::bc::", "::", 0, 0); - test_eq(4, smartlist_len(sl)); - test_streq("a", smartlist_get(sl, 1)); - test_streq("bc", smartlist_get(sl, 2)); - test_streq("", smartlist_get(sl, 3)); - cp_alloc = smartlist_join_strings(sl, "", 0, NULL); - test_streq(cp_alloc, "abcabc"); - tor_free(cp_alloc); - cp_alloc = smartlist_join_strings(sl, "!", 0, NULL); - test_streq(cp_alloc, "abc!a!bc!"); - tor_free(cp_alloc); - cp_alloc = smartlist_join_strings(sl, "XY", 0, NULL); - test_streq(cp_alloc, "abcXYaXYbcXY"); - tor_free(cp_alloc); - cp_alloc = smartlist_join_strings(sl, "XY", 1, NULL); - test_streq(cp_alloc, "abcXYaXYbcXYXY"); - tor_free(cp_alloc); - cp_alloc = smartlist_join_strings(sl, "", 1, NULL); - test_streq(cp_alloc, "abcabc"); - tor_free(cp_alloc); - - smartlist_split_string(sl, "/def/ /ghijk", "/", 0, 0); - test_eq(8, smartlist_len(sl)); - test_streq("", smartlist_get(sl, 4)); - test_streq("def", smartlist_get(sl, 5)); - test_streq(" ", smartlist_get(sl, 6)); - test_streq("ghijk", smartlist_get(sl, 7)); - SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); - smartlist_clear(sl); - - smartlist_split_string(sl, "a,bbd,cdef", ",", SPLIT_SKIP_SPACE, 0); - test_eq(3, smartlist_len(sl)); - test_streq("a", smartlist_get(sl,0)); - test_streq("bbd", smartlist_get(sl,1)); - test_streq("cdef", smartlist_get(sl,2)); - smartlist_split_string(sl, " z <> zhasd <> <> bnud<> ", "<>", - SPLIT_SKIP_SPACE, 0); - test_eq(8, smartlist_len(sl)); - test_streq("z", smartlist_get(sl,3)); - test_streq("zhasd", smartlist_get(sl,4)); - test_streq("", smartlist_get(sl,5)); - test_streq("bnud", smartlist_get(sl,6)); - test_streq("", smartlist_get(sl,7)); - - SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); - smartlist_clear(sl); - - smartlist_split_string(sl, " ab\tc \td ef ", NULL, - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); - test_eq(4, smartlist_len(sl)); - test_streq("ab", smartlist_get(sl,0)); - test_streq("c", smartlist_get(sl,1)); - test_streq("d", smartlist_get(sl,2)); - test_streq("ef", smartlist_get(sl,3)); - smartlist_split_string(sl, "ghi\tj", NULL, - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); - test_eq(6, smartlist_len(sl)); - test_streq("ghi", smartlist_get(sl,4)); - test_streq("j", smartlist_get(sl,5)); - - SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); - smartlist_clear(sl); - - cp_alloc = smartlist_join_strings(sl, "XY", 0, NULL); - test_streq(cp_alloc, ""); - tor_free(cp_alloc); - cp_alloc = smartlist_join_strings(sl, "XY", 1, NULL); - test_streq(cp_alloc, "XY"); - tor_free(cp_alloc); - - smartlist_split_string(sl, " z <> zhasd <> <> bnud<> ", "<>", - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); - test_eq(3, smartlist_len(sl)); - test_streq("z", smartlist_get(sl, 0)); - test_streq("zhasd", smartlist_get(sl, 1)); - test_streq("bnud", smartlist_get(sl, 2)); - smartlist_split_string(sl, " z <> zhasd <> <> bnud<> ", "<>", - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 2); - test_eq(5, smartlist_len(sl)); - test_streq("z", smartlist_get(sl, 3)); - test_streq("zhasd <> <> bnud<>", smartlist_get(sl, 4)); - SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); - smartlist_clear(sl); - - smartlist_split_string(sl, "abcd\n", "\n", - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); - test_eq(1, smartlist_len(sl)); - test_streq("abcd", smartlist_get(sl, 0)); - smartlist_split_string(sl, "efgh", "\n", - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); - test_eq(2, smartlist_len(sl)); - test_streq("efgh", smartlist_get(sl, 1)); - - SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); - smartlist_clear(sl); - - /* Test swapping, shuffling, and sorting. */ - smartlist_split_string(sl, "the,onion,router,by,arma,and,nickm", ",", 0, 0); - test_eq(7, smartlist_len(sl)); - smartlist_sort(sl, _compare_strs); - cp_alloc = smartlist_join_strings(sl, ",", 0, NULL); - test_streq(cp_alloc,"and,arma,by,nickm,onion,router,the"); - tor_free(cp_alloc); - smartlist_swap(sl, 1, 5); - cp_alloc = smartlist_join_strings(sl, ",", 0, NULL); - test_streq(cp_alloc,"and,router,by,nickm,onion,arma,the"); - tor_free(cp_alloc); - smartlist_shuffle(sl); - test_eq(7, smartlist_len(sl)); - test_assert(smartlist_string_isin(sl, "and")); - test_assert(smartlist_string_isin(sl, "router")); - test_assert(smartlist_string_isin(sl, "by")); - test_assert(smartlist_string_isin(sl, "nickm")); - test_assert(smartlist_string_isin(sl, "onion")); - test_assert(smartlist_string_isin(sl, "arma")); - test_assert(smartlist_string_isin(sl, "the")); - - /* Test bsearch. */ - smartlist_sort(sl, _compare_strs); - test_streq("nickm", smartlist_bsearch(sl, "zNicKM", - _compare_without_first_ch)); - test_streq("and", smartlist_bsearch(sl, " AND", _compare_without_first_ch)); - test_eq_ptr(NULL, smartlist_bsearch(sl, " ANz", _compare_without_first_ch)); - - /* Test bsearch_idx */ - { - int f; - test_eq(0, smartlist_bsearch_idx(sl," aaa",_compare_without_first_ch,&f)); - test_eq(f, 0); - test_eq(0, smartlist_bsearch_idx(sl," and",_compare_without_first_ch,&f)); - test_eq(f, 1); - test_eq(1, smartlist_bsearch_idx(sl," arm",_compare_without_first_ch,&f)); - test_eq(f, 0); - test_eq(1, smartlist_bsearch_idx(sl," arma",_compare_without_first_ch,&f)); - test_eq(f, 1); - test_eq(2, smartlist_bsearch_idx(sl," armb",_compare_without_first_ch,&f)); - test_eq(f, 0); - test_eq(7, smartlist_bsearch_idx(sl," zzzz",_compare_without_first_ch,&f)); - test_eq(f, 0); - } - - /* Test reverse() and pop_last() */ - smartlist_reverse(sl); - cp_alloc = smartlist_join_strings(sl, ",", 0, NULL); - test_streq(cp_alloc,"the,router,onion,nickm,by,arma,and"); - tor_free(cp_alloc); - cp_alloc = smartlist_pop_last(sl); - test_streq(cp_alloc, "and"); - tor_free(cp_alloc); - test_eq(smartlist_len(sl), 6); - SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); - smartlist_clear(sl); - cp_alloc = smartlist_pop_last(sl); - test_eq(cp_alloc, NULL); - - /* Test uniq() */ - smartlist_split_string(sl, - "50,noon,radar,a,man,a,plan,a,canal,panama,radar,noon,50", - ",", 0, 0); - smartlist_sort(sl, _compare_strs); - smartlist_uniq(sl, _compare_strs, _tor_free); - cp_alloc = smartlist_join_strings(sl, ",", 0, NULL); - test_streq(cp_alloc, "50,a,canal,man,noon,panama,plan,radar"); - tor_free(cp_alloc); - - /* Test string_isin and isin_case and num_isin */ - test_assert(smartlist_string_isin(sl, "noon")); - test_assert(!smartlist_string_isin(sl, "noonoon")); - test_assert(smartlist_string_isin_case(sl, "nOOn")); - test_assert(!smartlist_string_isin_case(sl, "nooNooN")); - test_assert(smartlist_string_num_isin(sl, 50)); - test_assert(!smartlist_string_num_isin(sl, 60)); - - /* Test smartlist_choose */ - { - int i; - int allsame = 1; - int allin = 1; - void *first = smartlist_choose(sl); - test_assert(smartlist_isin(sl, first)); - for (i = 0; i < 100; ++i) { - void *second = smartlist_choose(sl); - if (second != first) - allsame = 0; - if (!smartlist_isin(sl, second)) - allin = 0; - } - test_assert(!allsame); - test_assert(allin); - } - SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); - smartlist_clear(sl); - - /* Test string_remove and remove and join_strings2 */ - smartlist_split_string(sl, - "Some say the Earth will end in ice and some in fire", - " ", 0, 0); - cp = smartlist_get(sl, 4); - test_streq(cp, "will"); - smartlist_add(sl, cp); - smartlist_remove(sl, cp); - tor_free(cp); - cp_alloc = smartlist_join_strings(sl, ",", 0, NULL); - test_streq(cp_alloc, "Some,say,the,Earth,fire,end,in,ice,and,some,in"); - tor_free(cp_alloc); - smartlist_string_remove(sl, "in"); - cp_alloc = smartlist_join_strings2(sl, "+XX", 1, 0, &sz); - test_streq(cp_alloc, "Some+say+the+Earth+fire+end+some+ice+and"); - test_eq((int)sz, 40); - - done: - - SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); - smartlist_free(sl); - tor_free(cp_alloc); -} - -/** Run unit tests for smartlist set manipulation functions. */ -static void -test_util_smartlist_overlap(void) -{ - smartlist_t *sl = smartlist_create(); - smartlist_t *ints = smartlist_create(); - smartlist_t *odds = smartlist_create(); - smartlist_t *evens = smartlist_create(); - smartlist_t *primes = smartlist_create(); - int i; - for (i=1; i < 10; i += 2) - smartlist_add(odds, (void*)(uintptr_t)i); - for (i=0; i < 10; i += 2) - smartlist_add(evens, (void*)(uintptr_t)i); - - /* add_all */ - smartlist_add_all(ints, odds); - smartlist_add_all(ints, evens); - test_eq(smartlist_len(ints), 10); - - smartlist_add(primes, (void*)2); - smartlist_add(primes, (void*)3); - smartlist_add(primes, (void*)5); - smartlist_add(primes, (void*)7); - - /* overlap */ - test_assert(smartlist_overlap(ints, odds)); - test_assert(smartlist_overlap(odds, primes)); - test_assert(smartlist_overlap(evens, primes)); - test_assert(!smartlist_overlap(odds, evens)); - - /* intersect */ - smartlist_add_all(sl, odds); - smartlist_intersect(sl, primes); - test_eq(smartlist_len(sl), 3); - test_assert(smartlist_isin(sl, (void*)3)); - test_assert(smartlist_isin(sl, (void*)5)); - test_assert(smartlist_isin(sl, (void*)7)); - - /* subtract */ - smartlist_add_all(sl, primes); - smartlist_subtract(sl, odds); - test_eq(smartlist_len(sl), 1); - test_assert(smartlist_isin(sl, (void*)2)); - - done: - smartlist_free(odds); - smartlist_free(evens); - smartlist_free(ints); - smartlist_free(primes); - smartlist_free(sl); -} - -/** Run unit tests for smartlist-of-digests functions. */ -static void -test_util_smartlist_digests(void) -{ - smartlist_t *sl = smartlist_create(); - - /* digest_isin. */ - smartlist_add(sl, tor_memdup("AAAAAAAAAAAAAAAAAAAA", DIGEST_LEN)); - smartlist_add(sl, tor_memdup("\00090AAB2AAAAaasdAAAAA", DIGEST_LEN)); - smartlist_add(sl, tor_memdup("\00090AAB2AAAAaasdAAAAA", DIGEST_LEN)); - test_eq(0, smartlist_digest_isin(NULL, "AAAAAAAAAAAAAAAAAAAA")); - test_assert(smartlist_digest_isin(sl, "AAAAAAAAAAAAAAAAAAAA")); - test_assert(smartlist_digest_isin(sl, "\00090AAB2AAAAaasdAAAAA")); - test_eq(0, smartlist_digest_isin(sl, "\00090AAB2AAABaasdAAAAA")); - - /* sort digests */ - smartlist_sort_digests(sl); - test_memeq(smartlist_get(sl, 0), "\00090AAB2AAAAaasdAAAAA", DIGEST_LEN); - test_memeq(smartlist_get(sl, 1), "\00090AAB2AAAAaasdAAAAA", DIGEST_LEN); - test_memeq(smartlist_get(sl, 2), "AAAAAAAAAAAAAAAAAAAA", DIGEST_LEN); - test_eq(3, smartlist_len(sl)); - - /* uniq_digests */ - smartlist_uniq_digests(sl); - test_eq(2, smartlist_len(sl)); - test_memeq(smartlist_get(sl, 0), "\00090AAB2AAAAaasdAAAAA", DIGEST_LEN); - test_memeq(smartlist_get(sl, 1), "AAAAAAAAAAAAAAAAAAAA", DIGEST_LEN); - - done: - SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); - smartlist_free(sl); -} - -/** Run unit tests for concatenate-a-smartlist-of-strings functions. */ -static void -test_util_smartlist_join(void) -{ - smartlist_t *sl = smartlist_create(); - smartlist_t *sl2 = smartlist_create(), *sl3 = smartlist_create(), - *sl4 = smartlist_create(); - char *joined=NULL; - /* unique, sorted. */ - smartlist_split_string(sl, - "Abashments Ambush Anchorman Bacon Banks Borscht " - "Bunks Inhumane Insurance Knish Know Manners " - "Maraschinos Stamina Sunbonnets Unicorns Wombats", - " ", 0, 0); - /* non-unique, sorted. */ - smartlist_split_string(sl2, - "Ambush Anchorman Anchorman Anemias Anemias Bacon " - "Crossbowmen Inhumane Insurance Knish Know Manners " - "Manners Maraschinos Wombats Wombats Work", - " ", 0, 0); - SMARTLIST_FOREACH_JOIN(sl, char *, cp1, - sl2, char *, cp2, - strcmp(cp1,cp2), - smartlist_add(sl3, cp2)) { - test_streq(cp1, cp2); - smartlist_add(sl4, cp1); - } SMARTLIST_FOREACH_JOIN_END(cp1, cp2); - - SMARTLIST_FOREACH(sl3, const char *, cp, - test_assert(smartlist_isin(sl2, cp) && - !smartlist_string_isin(sl, cp))); - SMARTLIST_FOREACH(sl4, const char *, cp, - test_assert(smartlist_isin(sl, cp) && - smartlist_string_isin(sl2, cp))); - joined = smartlist_join_strings(sl3, ",", 0, NULL); - test_streq(joined, "Anemias,Anemias,Crossbowmen,Work"); - tor_free(joined); - joined = smartlist_join_strings(sl4, ",", 0, NULL); - test_streq(joined, "Ambush,Anchorman,Anchorman,Bacon,Inhumane,Insurance," - "Knish,Know,Manners,Manners,Maraschinos,Wombats,Wombats"); - tor_free(joined); - - done: - smartlist_free(sl4); - smartlist_free(sl3); - SMARTLIST_FOREACH(sl2, char *, cp, tor_free(cp)); - smartlist_free(sl2); - SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); - smartlist_free(sl); - tor_free(joined); -} - -/** Run unit tests for bitarray code */ -static void -test_util_bitarray(void) -{ - bitarray_t *ba = NULL; - int i, j, ok=1; - - ba = bitarray_init_zero(1); - test_assert(ba); - test_assert(! bitarray_is_set(ba, 0)); - bitarray_set(ba, 0); - test_assert(bitarray_is_set(ba, 0)); - bitarray_clear(ba, 0); - test_assert(! bitarray_is_set(ba, 0)); - bitarray_free(ba); - - ba = bitarray_init_zero(1023); - for (i = 1; i < 64; ) { - for (j = 0; j < 1023; ++j) { - if (j % i) - bitarray_set(ba, j); - else - bitarray_clear(ba, j); - } - for (j = 0; j < 1023; ++j) { - if (!bool_eq(bitarray_is_set(ba, j), j%i)) - ok = 0; - } - test_assert(ok); - if (i < 7) - ++i; - else if (i == 28) - i = 32; - else - i += 7; - } - - done: - if (ba) - bitarray_free(ba); -} - -/** Run unit tests for digest set code (implemented as a hashtable or as a - * bloom filter) */ -static void -test_util_digestset(void) -{ - smartlist_t *included = smartlist_create(); - char d[DIGEST_LEN]; - int i; - int ok = 1; - int false_positives = 0; - digestset_t *set = NULL; - - for (i = 0; i < 1000; ++i) { - crypto_rand(d, DIGEST_LEN); - smartlist_add(included, tor_memdup(d, DIGEST_LEN)); - } - set = digestset_new(1000); - SMARTLIST_FOREACH(included, const char *, cp, - if (digestset_isin(set, cp)) - ok = 0); - test_assert(ok); - SMARTLIST_FOREACH(included, const char *, cp, - digestset_add(set, cp)); - SMARTLIST_FOREACH(included, const char *, cp, - if (!digestset_isin(set, cp)) - ok = 0); - test_assert(ok); - for (i = 0; i < 1000; ++i) { - crypto_rand(d, DIGEST_LEN); - if (digestset_isin(set, d)) - ++false_positives; - } - test_assert(false_positives < 50); /* Should be far lower. */ - - done: - if (set) - digestset_free(set); - SMARTLIST_FOREACH(included, char *, cp, tor_free(cp)); - smartlist_free(included); -} - -/** mutex for thread test to stop the threads hitting data at the same time. */ -static tor_mutex_t *_thread_test_mutex = NULL; -/** mutexes for the thread test to make sure that the threads have to - * interleave somewhat. */ -static tor_mutex_t *_thread_test_start1 = NULL, - *_thread_test_start2 = NULL; -/** Shared strmap for the thread test. */ -static strmap_t *_thread_test_strmap = NULL; -/** The name of thread1 for the thread test */ -static char *_thread1_name = NULL; -/** The name of thread2 for the thread test */ -static char *_thread2_name = NULL; - -static void _thread_test_func(void* _s) ATTR_NORETURN; - -/** How many iterations have the threads in the unit test run? */ -static int t1_count = 0, t2_count = 0; - -/** Helper function for threading unit tests: This function runs in a - * subthread. It grabs its own mutex (start1 or start2) to make sure that it - * should start, then it repeatedly alters _test_thread_strmap protected by - * _thread_test_mutex. */ -static void -_thread_test_func(void* _s) -{ - char *s = _s; - int i, *count; - tor_mutex_t *m; - char buf[64]; - char **cp; - if (!strcmp(s, "thread 1")) { - m = _thread_test_start1; - cp = &_thread1_name; - count = &t1_count; - } else { - m = _thread_test_start2; - cp = &_thread2_name; - count = &t2_count; - } - tor_mutex_acquire(m); - - tor_snprintf(buf, sizeof(buf), "%lu", tor_get_thread_id()); - *cp = tor_strdup(buf); - - for (i=0; i<10000; ++i) { - tor_mutex_acquire(_thread_test_mutex); - strmap_set(_thread_test_strmap, "last to run", *cp); - ++*count; - tor_mutex_release(_thread_test_mutex); - } - tor_mutex_acquire(_thread_test_mutex); - strmap_set(_thread_test_strmap, s, *cp); - tor_mutex_release(_thread_test_mutex); - - tor_mutex_release(m); - - spawn_exit(); -} - -/** Run unit tests for threading logic. */ -static void -test_util_threads(void) -{ - char *s1 = NULL, *s2 = NULL; - int done = 0, timedout = 0; - time_t started; -#ifndef MS_WINDOWS - struct timeval tv; - tv.tv_sec=0; - tv.tv_usec=10; -#endif -#ifndef TOR_IS_MULTITHREADED - /* Skip this test if we aren't threading. We should be threading most - * everywhere by now. */ - if (1) - return; -#endif - _thread_test_mutex = tor_mutex_new(); - _thread_test_start1 = tor_mutex_new(); - _thread_test_start2 = tor_mutex_new(); - _thread_test_strmap = strmap_new(); - s1 = tor_strdup("thread 1"); - s2 = tor_strdup("thread 2"); - tor_mutex_acquire(_thread_test_start1); - tor_mutex_acquire(_thread_test_start2); - spawn_func(_thread_test_func, s1); - spawn_func(_thread_test_func, s2); - tor_mutex_release(_thread_test_start2); - tor_mutex_release(_thread_test_start1); - started = time(NULL); - while (!done) { - tor_mutex_acquire(_thread_test_mutex); - strmap_assert_ok(_thread_test_strmap); - if (strmap_get(_thread_test_strmap, "thread 1") && - strmap_get(_thread_test_strmap, "thread 2")) { - done = 1; - } else if (time(NULL) > started + 25) { - timedout = done = 1; - } - tor_mutex_release(_thread_test_mutex); -#ifndef MS_WINDOWS - /* Prevent the main thread from starving the worker threads. */ - select(0, NULL, NULL, NULL, &tv); -#endif - } - - tor_mutex_acquire(_thread_test_start1); - tor_mutex_release(_thread_test_start1); - tor_mutex_acquire(_thread_test_start2); - tor_mutex_release(_thread_test_start2); - - tor_mutex_free(_thread_test_mutex); - - if (timedout) { - printf("\nTimed out: %d %d", t1_count, t2_count); - test_assert(strmap_get(_thread_test_strmap, "thread 1")); - test_assert(strmap_get(_thread_test_strmap, "thread 2")); - test_assert(!timedout); - } - - /* different thread IDs. */ - test_assert(strcmp(strmap_get(_thread_test_strmap, "thread 1"), - strmap_get(_thread_test_strmap, "thread 2"))); - test_assert(!strcmp(strmap_get(_thread_test_strmap, "thread 1"), - strmap_get(_thread_test_strmap, "last to run")) || - !strcmp(strmap_get(_thread_test_strmap, "thread 2"), - strmap_get(_thread_test_strmap, "last to run"))); - - done: - tor_free(s1); - tor_free(s2); - tor_free(_thread1_name); - tor_free(_thread2_name); - if (_thread_test_strmap) - strmap_free(_thread_test_strmap, NULL); - if (_thread_test_start1) - tor_mutex_free(_thread_test_start1); - if (_thread_test_start2) - tor_mutex_free(_thread_test_start2); -} - -/** Helper: return a tristate based on comparing two strings. */ -static int -_compare_strings_for_pqueue(const void *s1, const void *s2) -{ - return strcmp((const char*)s1, (const char*)s2); -} - -/** Run unit tests for heap-based priority queue functions. */ -static void -test_util_pqueue(void) -{ - smartlist_t *sl = smartlist_create(); - int (*cmp)(const void *, const void*); -#define OK() smartlist_pqueue_assert_ok(sl, cmp) - - cmp = _compare_strings_for_pqueue; - - smartlist_pqueue_add(sl, cmp, (char*)"cows"); - smartlist_pqueue_add(sl, cmp, (char*)"zebras"); - smartlist_pqueue_add(sl, cmp, (char*)"fish"); - smartlist_pqueue_add(sl, cmp, (char*)"frogs"); - smartlist_pqueue_add(sl, cmp, (char*)"apples"); - smartlist_pqueue_add(sl, cmp, (char*)"squid"); - smartlist_pqueue_add(sl, cmp, (char*)"daschunds"); - smartlist_pqueue_add(sl, cmp, (char*)"eggplants"); - smartlist_pqueue_add(sl, cmp, (char*)"weissbier"); - smartlist_pqueue_add(sl, cmp, (char*)"lobsters"); - smartlist_pqueue_add(sl, cmp, (char*)"roquefort"); - - OK(); - - test_eq(smartlist_len(sl), 11); - test_streq(smartlist_get(sl, 0), "apples"); - test_streq(smartlist_pqueue_pop(sl, cmp), "apples"); - test_eq(smartlist_len(sl), 10); - OK(); - test_streq(smartlist_pqueue_pop(sl, cmp), "cows"); - test_streq(smartlist_pqueue_pop(sl, cmp), "daschunds"); - smartlist_pqueue_add(sl, cmp, (char*)"chinchillas"); - OK(); - smartlist_pqueue_add(sl, cmp, (char*)"fireflies"); - OK(); - test_streq(smartlist_pqueue_pop(sl, cmp), "chinchillas"); - test_streq(smartlist_pqueue_pop(sl, cmp), "eggplants"); - test_streq(smartlist_pqueue_pop(sl, cmp), "fireflies"); - OK(); - test_streq(smartlist_pqueue_pop(sl, cmp), "fish"); - test_streq(smartlist_pqueue_pop(sl, cmp), "frogs"); - test_streq(smartlist_pqueue_pop(sl, cmp), "lobsters"); - test_streq(smartlist_pqueue_pop(sl, cmp), "roquefort"); - OK(); - test_eq(smartlist_len(sl), 3); - test_streq(smartlist_pqueue_pop(sl, cmp), "squid"); - test_streq(smartlist_pqueue_pop(sl, cmp), "weissbier"); - test_streq(smartlist_pqueue_pop(sl, cmp), "zebras"); - test_eq(smartlist_len(sl), 0); - OK(); -#undef OK - - done: - - smartlist_free(sl); -} - -/** Run unit tests for compression functions */ -static void -test_util_gzip(void) -{ - char *buf1=NULL, *buf2=NULL, *buf3=NULL, *cp1, *cp2; - const char *ccp2; - size_t len1, len2; - tor_zlib_state_t *state = NULL; - - buf1 = tor_strdup("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZAAAAAAAAAAAAAAAAAAAZ"); - test_assert(detect_compression_method(buf1, strlen(buf1)) == UNKNOWN_METHOD); - if (is_gzip_supported()) { - test_assert(!tor_gzip_compress(&buf2, &len1, buf1, strlen(buf1)+1, - GZIP_METHOD)); - test_assert(buf2); - test_assert(!memcmp(buf2, "\037\213", 2)); /* Gzip magic. */ - test_assert(detect_compression_method(buf2, len1) == GZIP_METHOD); - - test_assert(!tor_gzip_uncompress(&buf3, &len2, buf2, len1, - GZIP_METHOD, 1, LOG_INFO)); - test_assert(buf3); - test_streq(buf1,buf3); - - tor_free(buf2); - tor_free(buf3); - } - - test_assert(!tor_gzip_compress(&buf2, &len1, buf1, strlen(buf1)+1, - ZLIB_METHOD)); - test_assert(buf2); - test_assert(!memcmp(buf2, "\x78\xDA", 2)); /* deflate magic. */ - test_assert(detect_compression_method(buf2, len1) == ZLIB_METHOD); - - test_assert(!tor_gzip_uncompress(&buf3, &len2, buf2, len1, - ZLIB_METHOD, 1, LOG_INFO)); - test_assert(buf3); - test_streq(buf1,buf3); - - /* Check whether we can uncompress concatenated, compressed strings. */ - tor_free(buf3); - buf2 = tor_realloc(buf2, len1*2); - memcpy(buf2+len1, buf2, len1); - test_assert(!tor_gzip_uncompress(&buf3, &len2, buf2, len1*2, - ZLIB_METHOD, 1, LOG_INFO)); - test_eq(len2, (strlen(buf1)+1)*2); - test_memeq(buf3, - "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZAAAAAAAAAAAAAAAAAAAZ\0" - "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZAAAAAAAAAAAAAAAAAAAZ\0", - (strlen(buf1)+1)*2); - - tor_free(buf1); - tor_free(buf2); - tor_free(buf3); - - /* Check whether we can uncompress partial strings. */ - buf1 = - tor_strdup("String with low redundancy that won't be compressed much."); - test_assert(!tor_gzip_compress(&buf2, &len1, buf1, strlen(buf1)+1, - ZLIB_METHOD)); - tor_assert(len1>16); - /* when we allow an incomplete string, we should succeed.*/ - tor_assert(!tor_gzip_uncompress(&buf3, &len2, buf2, len1-16, - ZLIB_METHOD, 0, LOG_INFO)); - buf3[len2]='\0'; - tor_assert(len2 > 5); - tor_assert(!strcmpstart(buf1, buf3)); - - /* when we demand a complete string, this must fail. */ - tor_free(buf3); - tor_assert(tor_gzip_uncompress(&buf3, &len2, buf2, len1-16, - ZLIB_METHOD, 1, LOG_INFO)); - tor_assert(!buf3); - - /* Now, try streaming compression. */ - tor_free(buf1); - tor_free(buf2); - tor_free(buf3); - state = tor_zlib_new(1, ZLIB_METHOD); - tor_assert(state); - cp1 = buf1 = tor_malloc(1024); - len1 = 1024; - ccp2 = "ABCDEFGHIJABCDEFGHIJ"; - len2 = 21; - test_assert(tor_zlib_process(state, &cp1, &len1, &ccp2, &len2, 0) - == TOR_ZLIB_OK); - test_eq(len2, 0); /* Make sure we compressed it all. */ - test_assert(cp1 > buf1); - - len2 = 0; - cp2 = cp1; - test_assert(tor_zlib_process(state, &cp1, &len1, &ccp2, &len2, 1) - == TOR_ZLIB_DONE); - test_eq(len2, 0); - test_assert(cp1 > cp2); /* Make sure we really added something. */ - - tor_assert(!tor_gzip_uncompress(&buf3, &len2, buf1, 1024-len1, - ZLIB_METHOD, 1, LOG_WARN)); - test_streq(buf3, "ABCDEFGHIJABCDEFGHIJ"); /*Make sure it compressed right.*/ - - done: - if (state) - tor_zlib_free(state); - tor_free(buf2); - tor_free(buf3); - tor_free(buf1); -} - -/** Run unit tests for string-to-void* map functions */ -static void -test_util_strmap(void) -{ - strmap_t *map; - strmap_iter_t *iter; - const char *k; - void *v; - char *visited = NULL; - smartlist_t *found_keys = NULL; - - map = strmap_new(); - test_assert(map); - test_eq(strmap_size(map), 0); - test_assert(strmap_isempty(map)); - v = strmap_set(map, "K1", (void*)99); - test_eq(v, NULL); - test_assert(!strmap_isempty(map)); - v = strmap_set(map, "K2", (void*)101); - test_eq(v, NULL); - v = strmap_set(map, "K1", (void*)100); - test_eq(v, (void*)99); - test_eq_ptr(strmap_get(map,"K1"), (void*)100); - test_eq_ptr(strmap_get(map,"K2"), (void*)101); - test_eq_ptr(strmap_get(map,"K-not-there"), NULL); - strmap_assert_ok(map); - - v = strmap_remove(map,"K2"); - strmap_assert_ok(map); - test_eq_ptr(v, (void*)101); - test_eq_ptr(strmap_get(map,"K2"), NULL); - test_eq_ptr(strmap_remove(map,"K2"), NULL); - - strmap_set(map, "K2", (void*)101); - strmap_set(map, "K3", (void*)102); - strmap_set(map, "K4", (void*)103); - test_eq(strmap_size(map), 4); - strmap_assert_ok(map); - strmap_set(map, "K5", (void*)104); - strmap_set(map, "K6", (void*)105); - strmap_assert_ok(map); - - /* Test iterator. */ - iter = strmap_iter_init(map); - found_keys = smartlist_create(); - while (!strmap_iter_done(iter)) { - strmap_iter_get(iter,&k,&v); - smartlist_add(found_keys, tor_strdup(k)); - test_eq_ptr(v, strmap_get(map, k)); - - if (!strcmp(k, "K2")) { - iter = strmap_iter_next_rmv(map,iter); - } else { - iter = strmap_iter_next(map,iter); - } - } - - /* Make sure we removed K2, but not the others. */ - test_eq_ptr(strmap_get(map, "K2"), NULL); - test_eq_ptr(strmap_get(map, "K5"), (void*)104); - /* Make sure we visited everyone once */ - smartlist_sort_strings(found_keys); - visited = smartlist_join_strings(found_keys, ":", 0, NULL); - test_streq(visited, "K1:K2:K3:K4:K5:K6"); - - strmap_assert_ok(map); - /* Clean up after ourselves. */ - strmap_free(map, NULL); - map = NULL; - - /* Now try some lc functions. */ - map = strmap_new(); - strmap_set_lc(map,"Ab.C", (void*)1); - test_eq_ptr(strmap_get(map,"ab.c"), (void*)1); - strmap_assert_ok(map); - test_eq_ptr(strmap_get_lc(map,"AB.C"), (void*)1); - test_eq_ptr(strmap_get(map,"AB.C"), NULL); - test_eq_ptr(strmap_remove_lc(map,"aB.C"), (void*)1); - strmap_assert_ok(map); - test_eq_ptr(strmap_get_lc(map,"AB.C"), NULL); - - done: - if (map) - strmap_free(map,NULL); - if (found_keys) { - SMARTLIST_FOREACH(found_keys, char *, cp, tor_free(cp)); - smartlist_free(found_keys); - } - tor_free(visited); -} - -/** Run unit tests for mmap() wrapper functionality. */ -static void -test_util_mmap(void) -{ - char *fname1 = tor_strdup(get_fname("mapped_1")); - char *fname2 = tor_strdup(get_fname("mapped_2")); - char *fname3 = tor_strdup(get_fname("mapped_3")); - const size_t buflen = 17000; - char *buf = tor_malloc(17000); - tor_mmap_t *mapping = NULL; - - crypto_rand(buf, buflen); - - mapping = tor_mmap_file(fname1); - test_assert(! mapping); - - write_str_to_file(fname1, "Short file.", 1); - write_bytes_to_file(fname2, buf, buflen, 1); - write_bytes_to_file(fname3, buf, 16384, 1); - - mapping = tor_mmap_file(fname1); - test_assert(mapping); - test_eq(mapping->size, strlen("Short file.")); - test_streq(mapping->data, "Short file."); -#ifdef MS_WINDOWS - tor_munmap_file(mapping); - mapping = NULL; - test_assert(unlink(fname1) == 0); -#else - /* make sure we can unlink. */ - test_assert(unlink(fname1) == 0); - test_streq(mapping->data, "Short file."); - tor_munmap_file(mapping); - mapping = NULL; -#endif - - /* Now a zero-length file. */ - write_str_to_file(fname1, "", 1); - mapping = tor_mmap_file(fname1); - test_eq(mapping, NULL); - test_eq(ERANGE, errno); - unlink(fname1); - - /* Make sure that we fail to map a no-longer-existent file. */ - mapping = tor_mmap_file(fname1); - test_assert(mapping == NULL); - - /* Now try a big file that stretches across a few pages and isn't aligned */ - mapping = tor_mmap_file(fname2); - test_assert(mapping); - test_eq(mapping->size, buflen); - test_memeq(mapping->data, buf, buflen); - tor_munmap_file(mapping); - mapping = NULL; - - /* Now try a big aligned file. */ - mapping = tor_mmap_file(fname3); - test_assert(mapping); - test_eq(mapping->size, 16384); - test_memeq(mapping->data, buf, 16384); - tor_munmap_file(mapping); - mapping = NULL; - - done: - unlink(fname1); - unlink(fname2); - unlink(fname3); - - tor_free(fname1); - tor_free(fname2); - tor_free(fname3); - tor_free(buf); - - if (mapping) - tor_munmap_file(mapping); -} - -/** Run unit tests for escaping/unescaping data for use by controllers. */ -static void -test_util_control_formats(void) -{ - char *out = NULL; - const char *inp = - "..This is a test\r\nof the emergency \nbroadcast\r\n..system.\r\nZ.\r\n"; - size_t sz; - - sz = read_escaped_data(inp, strlen(inp), &out); - test_streq(out, - ".This is a test\nof the emergency \nbroadcast\n.system.\nZ.\n"); - test_eq(sz, strlen(out)); - - done: - tor_free(out); -} - -static void -test_util_sscanf(void) -{ - unsigned u1, u2, u3; - char s1[10], s2[10], s3[10], ch; - int r; - - r = tor_sscanf("hello world", "hello world"); /* String match: success */ - test_eq(r, 0); - r = tor_sscanf("hello world 3", "hello worlb %u", &u1); /* String fail */ - test_eq(r, 0); - r = tor_sscanf("12345", "%u", &u1); /* Simple number */ - test_eq(r, 1); - test_eq(u1, 12345u); - r = tor_sscanf("", "%u", &u1); /* absent number */ - test_eq(r, 0); - r = tor_sscanf("A", "%u", &u1); /* bogus number */ - test_eq(r, 0); - r = tor_sscanf("4294967295", "%u", &u1); /* UINT32_MAX should work. */ - test_eq(r, 1); - test_eq(u1, 4294967295u); - r = tor_sscanf("4294967296", "%u", &u1); /* Always say -1 at 32 bits. */ - test_eq(r, 0); - r = tor_sscanf("123456", "%2u%u", &u1, &u2); /* Width */ - test_eq(r, 2); - test_eq(u1, 12u); - test_eq(u2, 3456u); - r = tor_sscanf("!12:3:456", "!%2u:%2u:%3u", &u1, &u2, &u3); /* separators */ - test_eq(r, 3); - test_eq(u1, 12u); - test_eq(u2, 3u); - test_eq(u3, 456u); - r = tor_sscanf("12:3:045", "%2u:%2u:%3u", &u1, &u2, &u3); /* 0s */ - test_eq(r, 3); - test_eq(u1, 12u); - test_eq(u2, 3u); - test_eq(u3, 45u); - /* %u does not match space.*/ - r = tor_sscanf("12:3: 45", "%2u:%2u:%3u", &u1, &u2, &u3); - test_eq(r, 2); - /* %u does not match negative numbers. */ - r = tor_sscanf("12:3:-4", "%2u:%2u:%3u", &u1, &u2, &u3); - test_eq(r, 2); - /* Arbitrary amounts of 0-padding are okay */ - r = tor_sscanf("12:03:000000000000000099", "%2u:%2u:%u", &u1, &u2, &u3); - test_eq(r, 3); - test_eq(u1, 12u); - test_eq(u2, 3u); - test_eq(u3, 99u); - - r = tor_sscanf("99% fresh", "%3u%% fresh", &u1); /* percents are scannable.*/ - test_eq(r, 1); - test_eq(u1, 99); - - r = tor_sscanf("hello", "%s", s1); /* %s needs a number. */ - test_eq(r, -1); - - r = tor_sscanf("hello", "%3s%7s", s1, s2); /* %s matches characters. */ - test_eq(r, 2); - test_streq(s1, "hel"); - test_streq(s2, "lo"); - r = tor_sscanf("WD40", "%2s%u", s3, &u1); /* %s%u */ - test_eq(r, 2); - test_streq(s3, "WD"); - test_eq(u1, 40); - r = tor_sscanf("76trombones", "%6u%9s", &u1, s1); /* %u%s */ - test_eq(r, 2); - test_eq(u1, 76); - test_streq(s1, "trombones"); - r = tor_sscanf("hello world", "%9s %9s", s1, s2); /* %s doesn't eat space. */ - test_eq(r, 2); - test_streq(s1, "hello"); - test_streq(s2, "world"); - r = tor_sscanf("hi", "%9s%9s%3s", s1, s2, s3); /* %s can be empty. */ - test_eq(r, 3); - test_streq(s1, "hi"); - test_streq(s2, ""); - test_streq(s3, ""); - - r = tor_sscanf("1.2.3", "%u.%u.%u%c", &u1, &u2, &u3, &ch); - test_eq(r, 3); - r = tor_sscanf("1.2.3 foobar", "%u.%u.%u%c", &u1, &u2, &u3, &ch); - test_eq(r, 4); - - done: - ; -} - -/** Run unit tests for the onion handshake code. */ -static void -test_onion_handshake(void) -{ - /* client-side */ - crypto_dh_env_t *c_dh = NULL; - char c_buf[ONIONSKIN_CHALLENGE_LEN]; - char c_keys[40]; - - /* server-side */ - char s_buf[ONIONSKIN_REPLY_LEN]; - char s_keys[40]; - - /* shared */ - crypto_pk_env_t *pk = NULL; - - pk = pk_generate(0); - - /* client handshake 1. */ - memset(c_buf, 0, ONIONSKIN_CHALLENGE_LEN); - test_assert(! onion_skin_create(pk, &c_dh, c_buf)); - - /* server handshake */ - memset(s_buf, 0, ONIONSKIN_REPLY_LEN); - memset(s_keys, 0, 40); - test_assert(! onion_skin_server_handshake(c_buf, pk, NULL, - s_buf, s_keys, 40)); - - /* client handshake 2 */ - memset(c_keys, 0, 40); - test_assert(! onion_skin_client_handshake(c_dh, s_buf, c_keys, 40)); - - if (memcmp(c_keys, s_keys, 40)) { - puts("Aiiiie"); - exit(1); - } - test_memeq(c_keys, s_keys, 40); - memset(s_buf, 0, 40); - test_memneq(c_keys, s_buf, 40); - - done: - if (c_dh) - crypto_dh_free(c_dh); - if (pk) - crypto_free_pk_env(pk); -} - -/** Run unit tests for router descriptor generation logic. */ -static void -test_dir_format(void) -{ - char buf[8192], buf2[8192]; - char platform[256]; - char fingerprint[FINGERPRINT_LEN+1]; - char *pk1_str = NULL, *pk2_str = NULL, *pk3_str = NULL, *cp; - size_t pk1_str_len, pk2_str_len, pk3_str_len; - routerinfo_t *r1=NULL, *r2=NULL; - crypto_pk_env_t *pk1 = NULL, *pk2 = NULL, *pk3 = NULL; - routerinfo_t *rp1 = NULL; - addr_policy_t *ex1, *ex2; - routerlist_t *dir1 = NULL, *dir2 = NULL; - tor_version_t ver1; - - pk1 = pk_generate(0); - pk2 = pk_generate(1); - pk3 = pk_generate(2); - - test_assert( is_legal_nickname("a")); - test_assert(!is_legal_nickname("")); - test_assert(!is_legal_nickname("abcdefghijklmnopqrst")); /* 20 chars */ - test_assert(!is_legal_nickname("hyphen-")); /* bad char */ - test_assert( is_legal_nickname("abcdefghijklmnopqrs")); /* 19 chars */ - test_assert(!is_legal_nickname("$AAAAAAAA01234AAAAAAAAAAAAAAAAAAAAAAAAAAA")); - /* valid */ - test_assert( is_legal_nickname_or_hexdigest( - "$AAAAAAAA01234AAAAAAAAAAAAAAAAAAAAAAAAAAA")); - test_assert( is_legal_nickname_or_hexdigest( - "$AAAAAAAA01234AAAAAAAAAAAAAAAAAAAAAAAAAAA=fred")); - test_assert( is_legal_nickname_or_hexdigest( - "$AAAAAAAA01234AAAAAAAAAAAAAAAAAAAAAAAAAAA~fred")); - /* too short */ - test_assert(!is_legal_nickname_or_hexdigest( - "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")); - /* illegal char */ - test_assert(!is_legal_nickname_or_hexdigest( - "$AAAAAAzAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")); - /* hex part too long */ - test_assert(!is_legal_nickname_or_hexdigest( - "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")); - test_assert(!is_legal_nickname_or_hexdigest( - "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=fred")); - /* Bad nickname */ - test_assert(!is_legal_nickname_or_hexdigest( - "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=")); - test_assert(!is_legal_nickname_or_hexdigest( - "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~")); - test_assert(!is_legal_nickname_or_hexdigest( - "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~hyphen-")); - test_assert(!is_legal_nickname_or_hexdigest( - "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~" - "abcdefghijklmnoppqrst")); - /* Bad extra char. */ - test_assert(!is_legal_nickname_or_hexdigest( - "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA!")); - test_assert(is_legal_nickname_or_hexdigest("xyzzy")); - test_assert(is_legal_nickname_or_hexdigest("abcdefghijklmnopqrs")); - test_assert(!is_legal_nickname_or_hexdigest("abcdefghijklmnopqrst")); - - get_platform_str(platform, sizeof(platform)); - r1 = tor_malloc_zero(sizeof(routerinfo_t)); - r1->address = tor_strdup("18.244.0.1"); - r1->addr = 0xc0a80001u; /* 192.168.0.1 */ - r1->cache_info.published_on = 0; - r1->or_port = 9000; - r1->dir_port = 9003; - r1->onion_pkey = crypto_pk_dup_key(pk1); - r1->identity_pkey = crypto_pk_dup_key(pk2); - r1->bandwidthrate = 1000; - r1->bandwidthburst = 5000; - r1->bandwidthcapacity = 10000; - r1->exit_policy = NULL; - r1->nickname = tor_strdup("Magri"); - r1->platform = tor_strdup(platform); - - ex1 = tor_malloc_zero(sizeof(addr_policy_t)); - ex2 = tor_malloc_zero(sizeof(addr_policy_t)); - ex1->policy_type = ADDR_POLICY_ACCEPT; - tor_addr_from_ipv4h(&ex1->addr, 0); - ex1->maskbits = 0; - ex1->prt_min = ex1->prt_max = 80; - ex2->policy_type = ADDR_POLICY_REJECT; - tor_addr_from_ipv4h(&ex2->addr, 18<<24); - ex2->maskbits = 8; - ex2->prt_min = ex2->prt_max = 24; - r2 = tor_malloc_zero(sizeof(routerinfo_t)); - r2->address = tor_strdup("1.1.1.1"); - r2->addr = 0x0a030201u; /* 10.3.2.1 */ - r2->platform = tor_strdup(platform); - r2->cache_info.published_on = 5; - r2->or_port = 9005; - r2->dir_port = 0; - r2->onion_pkey = crypto_pk_dup_key(pk2); - r2->identity_pkey = crypto_pk_dup_key(pk1); - r2->bandwidthrate = r2->bandwidthburst = r2->bandwidthcapacity = 3000; - r2->exit_policy = smartlist_create(); - smartlist_add(r2->exit_policy, ex2); - smartlist_add(r2->exit_policy, ex1); - r2->nickname = tor_strdup("Fred"); - - test_assert(!crypto_pk_write_public_key_to_string(pk1, &pk1_str, - &pk1_str_len)); - test_assert(!crypto_pk_write_public_key_to_string(pk2 , &pk2_str, - &pk2_str_len)); - test_assert(!crypto_pk_write_public_key_to_string(pk3 , &pk3_str, - &pk3_str_len)); - - memset(buf, 0, 2048); - test_assert(router_dump_router_to_string(buf, 2048, r1, pk2)>0); - - strlcpy(buf2, "router Magri 18.244.0.1 9000 0 9003\n" - "platform Tor "VERSION" on ", sizeof(buf2)); - strlcat(buf2, get_uname(), sizeof(buf2)); - strlcat(buf2, "\n" - "opt protocols Link 1 2 Circuit 1\n" - "published 1970-01-01 00:00:00\n" - "opt fingerprint ", sizeof(buf2)); - test_assert(!crypto_pk_get_fingerprint(pk2, fingerprint, 1)); - strlcat(buf2, fingerprint, sizeof(buf2)); - strlcat(buf2, "\nuptime 0\n" - /* XXX the "0" above is hard-coded, but even if we made it reflect - * uptime, that still wouldn't make it right, because the two - * descriptors might be made on different seconds... hm. */ - "bandwidth 1000 5000 10000\n" - "opt extra-info-digest 0000000000000000000000000000000000000000\n" - "onion-key\n", sizeof(buf2)); - strlcat(buf2, pk1_str, sizeof(buf2)); - strlcat(buf2, "signing-key\n", sizeof(buf2)); - strlcat(buf2, pk2_str, sizeof(buf2)); - strlcat(buf2, "opt hidden-service-dir\n", sizeof(buf2)); - strlcat(buf2, "reject *:*\nrouter-signature\n", sizeof(buf2)); - buf[strlen(buf2)] = '\0'; /* Don't compare the sig; it's never the same - * twice */ - - test_streq(buf, buf2); - - test_assert(router_dump_router_to_string(buf, 2048, r1, pk2)>0); - cp = buf; - rp1 = router_parse_entry_from_string((const char*)cp,NULL,1,0,NULL); - test_assert(rp1); - test_streq(rp1->address, r1->address); - test_eq(rp1->or_port, r1->or_port); - //test_eq(rp1->dir_port, r1->dir_port); - test_eq(rp1->bandwidthrate, r1->bandwidthrate); - test_eq(rp1->bandwidthburst, r1->bandwidthburst); - test_eq(rp1->bandwidthcapacity, r1->bandwidthcapacity); - test_assert(crypto_pk_cmp_keys(rp1->onion_pkey, pk1) == 0); - test_assert(crypto_pk_cmp_keys(rp1->identity_pkey, pk2) == 0); - //test_assert(rp1->exit_policy == NULL); - -#if 0 - /* XXX Once we have exit policies, test this again. XXX */ - strlcpy(buf2, "router tor.tor.tor 9005 0 0 3000\n", sizeof(buf2)); - strlcat(buf2, pk2_str, sizeof(buf2)); - strlcat(buf2, "signing-key\n", sizeof(buf2)); - strlcat(buf2, pk1_str, sizeof(buf2)); - strlcat(buf2, "accept *:80\nreject 18.*:24\n\n", sizeof(buf2)); - test_assert(router_dump_router_to_string(buf, 2048, &r2, pk2)>0); - test_streq(buf, buf2); - - cp = buf; - rp2 = router_parse_entry_from_string(&cp,1); - test_assert(rp2); - test_streq(rp2->address, r2.address); - test_eq(rp2->or_port, r2.or_port); - test_eq(rp2->dir_port, r2.dir_port); - test_eq(rp2->bandwidth, r2.bandwidth); - test_assert(crypto_pk_cmp_keys(rp2->onion_pkey, pk2) == 0); - test_assert(crypto_pk_cmp_keys(rp2->identity_pkey, pk1) == 0); - test_eq(rp2->exit_policy->policy_type, EXIT_POLICY_ACCEPT); - test_streq(rp2->exit_policy->string, "accept *:80"); - test_streq(rp2->exit_policy->address, "*"); - test_streq(rp2->exit_policy->port, "80"); - test_eq(rp2->exit_policy->next->policy_type, EXIT_POLICY_REJECT); - test_streq(rp2->exit_policy->next->string, "reject 18.*:24"); - test_streq(rp2->exit_policy->next->address, "18.*"); - test_streq(rp2->exit_policy->next->port, "24"); - test_assert(rp2->exit_policy->next->next == NULL); - - /* Okay, now for the directories. */ - { - fingerprint_list = smartlist_create(); - crypto_pk_get_fingerprint(pk2, buf, 1); - add_fingerprint_to_dir("Magri", buf, fingerprint_list); - crypto_pk_get_fingerprint(pk1, buf, 1); - add_fingerprint_to_dir("Fred", buf, fingerprint_list); - } - - { - char d[DIGEST_LEN]; - const char *m; - /* XXXX NM re-enable. */ - /* Make sure routers aren't too far in the past any more. */ - r1->cache_info.published_on = time(NULL); - r2->cache_info.published_on = time(NULL)-3*60*60; - test_assert(router_dump_router_to_string(buf, 2048, r1, pk2)>0); - test_eq(dirserv_add_descriptor(buf,&m,""), ROUTER_ADDED_NOTIFY_GENERATOR); - test_assert(router_dump_router_to_string(buf, 2048, r2, pk1)>0); - test_eq(dirserv_add_descriptor(buf,&m,""), ROUTER_ADDED_NOTIFY_GENERATOR); - get_options()->Nickname = tor_strdup("DirServer"); - test_assert(!dirserv_dump_directory_to_string(&cp,pk3, 0)); - crypto_pk_get_digest(pk3, d); - test_assert(!router_parse_directory(cp)); - test_eq(2, smartlist_len(dir1->routers)); - tor_free(cp); - } -#endif - dirserv_free_fingerprint_list(); - - /* Try out version parsing functionality */ - test_eq(0, tor_version_parse("0.3.4pre2-cvs", &ver1)); - test_eq(0, ver1.major); - test_eq(3, ver1.minor); - test_eq(4, ver1.micro); - test_eq(VER_PRE, ver1.status); - test_eq(2, ver1.patchlevel); - test_eq(0, tor_version_parse("0.3.4rc1", &ver1)); - test_eq(0, ver1.major); - test_eq(3, ver1.minor); - test_eq(4, ver1.micro); - test_eq(VER_RC, ver1.status); - test_eq(1, ver1.patchlevel); - test_eq(0, tor_version_parse("1.3.4", &ver1)); - test_eq(1, ver1.major); - test_eq(3, ver1.minor); - test_eq(4, ver1.micro); - test_eq(VER_RELEASE, ver1.status); - test_eq(0, ver1.patchlevel); - test_eq(0, tor_version_parse("1.3.4.999", &ver1)); - test_eq(1, ver1.major); - test_eq(3, ver1.minor); - test_eq(4, ver1.micro); - test_eq(VER_RELEASE, ver1.status); - test_eq(999, ver1.patchlevel); - test_eq(0, tor_version_parse("0.1.2.4-alpha", &ver1)); - test_eq(0, ver1.major); - test_eq(1, ver1.minor); - test_eq(2, ver1.micro); - test_eq(4, ver1.patchlevel); - test_eq(VER_RELEASE, ver1.status); - test_streq("alpha", ver1.status_tag); - test_eq(0, tor_version_parse("0.1.2.4", &ver1)); - test_eq(0, ver1.major); - test_eq(1, ver1.minor); - test_eq(2, ver1.micro); - test_eq(4, ver1.patchlevel); - test_eq(VER_RELEASE, ver1.status); - test_streq("", ver1.status_tag); - -#define test_eq_vs(vs1, vs2) test_eq_type(version_status_t, "%d", (vs1), (vs2)) -#define test_v_i_o(val, ver, lst) \ - test_eq_vs(val, tor_version_is_obsolete(ver, lst)) - - /* make sure tor_version_is_obsolete() works */ - test_v_i_o(VS_OLD, "0.0.1", "Tor 0.0.2"); - test_v_i_o(VS_OLD, "0.0.1", "0.0.2, Tor 0.0.3"); - test_v_i_o(VS_OLD, "0.0.1", "0.0.2,Tor 0.0.3"); - test_v_i_o(VS_OLD, "0.0.1","0.0.3,BetterTor 0.0.1"); - test_v_i_o(VS_RECOMMENDED, "0.0.2", "Tor 0.0.2,Tor 0.0.3"); - test_v_i_o(VS_NEW_IN_SERIES, "0.0.2", "Tor 0.0.2pre1,Tor 0.0.3"); - test_v_i_o(VS_OLD, "0.0.2", "Tor 0.0.2.1,Tor 0.0.3"); - test_v_i_o(VS_NEW, "0.1.0", "Tor 0.0.2,Tor 0.0.3"); - test_v_i_o(VS_RECOMMENDED, "0.0.7rc2", "0.0.7,Tor 0.0.7rc2,Tor 0.0.8"); - test_v_i_o(VS_OLD, "0.0.5.0", "0.0.5.1-cvs"); - test_v_i_o(VS_NEW_IN_SERIES, "0.0.5.1-cvs", "0.0.5, 0.0.6"); - /* Not on list, but newer than any in same series. */ - test_v_i_o(VS_NEW_IN_SERIES, "0.1.0.3", - "Tor 0.1.0.2,Tor 0.0.9.5,Tor 0.1.1.0"); - /* Series newer than any on list. */ - test_v_i_o(VS_NEW, "0.1.2.3", "Tor 0.1.0.2,Tor 0.0.9.5,Tor 0.1.1.0"); - /* Series older than any on list. */ - test_v_i_o(VS_OLD, "0.0.1.3", "Tor 0.1.0.2,Tor 0.0.9.5,Tor 0.1.1.0"); - /* Not on list, not newer than any on same series. */ - test_v_i_o(VS_UNRECOMMENDED, "0.1.0.1", - "Tor 0.1.0.2,Tor 0.0.9.5,Tor 0.1.1.0"); - /* On list, not newer than any on same series. */ - test_v_i_o(VS_UNRECOMMENDED, - "0.1.0.1", "Tor 0.1.0.2,Tor 0.0.9.5,Tor 0.1.1.0"); - test_eq(0, tor_version_as_new_as("Tor 0.0.5", "0.0.9pre1-cvs")); - test_eq(1, tor_version_as_new_as( - "Tor 0.0.8 on Darwin 64-121-192-100.c3-0." - "sfpo-ubr1.sfrn-sfpo.ca.cable.rcn.com Power Macintosh", - "0.0.8rc2")); - test_eq(0, tor_version_as_new_as( - "Tor 0.0.8 on Darwin 64-121-192-100.c3-0." - "sfpo-ubr1.sfrn-sfpo.ca.cable.rcn.com Power Macintosh", "0.0.8.2")); - - /* Now try svn revisions. */ - test_eq(1, tor_version_as_new_as("Tor 0.2.1.0-dev (r100)", - "Tor 0.2.1.0-dev (r99)")); - test_eq(1, tor_version_as_new_as("Tor 0.2.1.0-dev (r100) on Banana Jr", - "Tor 0.2.1.0-dev (r99) on Hal 9000")); - test_eq(1, tor_version_as_new_as("Tor 0.2.1.0-dev (r100)", - "Tor 0.2.1.0-dev on Colossus")); - test_eq(0, tor_version_as_new_as("Tor 0.2.1.0-dev (r99)", - "Tor 0.2.1.0-dev (r100)")); - test_eq(0, tor_version_as_new_as("Tor 0.2.1.0-dev (r99) on MCP", - "Tor 0.2.1.0-dev (r100) on AM")); - test_eq(0, tor_version_as_new_as("Tor 0.2.1.0-dev", - "Tor 0.2.1.0-dev (r99)")); - test_eq(1, tor_version_as_new_as("Tor 0.2.1.1", - "Tor 0.2.1.0-dev (r99)")); - done: - if (r1) - routerinfo_free(r1); - if (r2) - routerinfo_free(r2); - - tor_free(pk1_str); - tor_free(pk2_str); - tor_free(pk3_str); - if (pk1) crypto_free_pk_env(pk1); - if (pk2) crypto_free_pk_env(pk2); - if (pk3) crypto_free_pk_env(pk3); - if (rp1) routerinfo_free(rp1); - tor_free(dir1); /* XXXX And more !*/ - tor_free(dir2); /* And more !*/ -} - -/** Run unit tests for misc directory functions. */ -static void -test_dirutil(void) -{ - smartlist_t *sl = smartlist_create(); - fp_pair_t *pair; - - dir_split_resource_into_fingerprint_pairs( - /* Two pairs, out of order, with one duplicate. */ - "73656372657420646174612E0000000000FFFFFF-" - "557365204145532d32353620696e73746561642e+" - "73656372657420646174612E0000000000FFFFFF-" - "557365204145532d32353620696e73746561642e+" - "48657861646563696d616c2069736e277420736f-" - "676f6f6420666f7220686964696e6720796f7572.z", sl); - - test_eq(smartlist_len(sl), 2); - pair = smartlist_get(sl, 0); - test_memeq(pair->first, "Hexadecimal isn't so", DIGEST_LEN); - test_memeq(pair->second, "good for hiding your", DIGEST_LEN); - pair = smartlist_get(sl, 1); - test_memeq(pair->first, "secret data.\0\0\0\0\0\xff\xff\xff", DIGEST_LEN); - test_memeq(pair->second, "Use AES-256 instead.", DIGEST_LEN); - - done: - SMARTLIST_FOREACH(sl, fp_pair_t *, pair, tor_free(pair)); - smartlist_free(sl); -} - -extern const char AUTHORITY_CERT_1[]; -extern const char AUTHORITY_SIGNKEY_1[]; -extern const char AUTHORITY_CERT_2[]; -extern const char AUTHORITY_SIGNKEY_2[]; -extern const char AUTHORITY_CERT_3[]; -extern const char AUTHORITY_SIGNKEY_3[]; - -/** Helper: Test that two networkstatus_voter_info_t do in fact represent the - * same voting authority, and that they do in fact have all the same - * information. */ -static void -test_same_voter(networkstatus_voter_info_t *v1, - networkstatus_voter_info_t *v2) -{ - test_streq(v1->nickname, v2->nickname); - test_memeq(v1->identity_digest, v2->identity_digest, DIGEST_LEN); - test_streq(v1->address, v2->address); - test_eq(v1->addr, v2->addr); - test_eq(v1->dir_port, v2->dir_port); - test_eq(v1->or_port, v2->or_port); - test_streq(v1->contact, v2->contact); - test_memeq(v1->vote_digest, v2->vote_digest, DIGEST_LEN); - done: - ; -} - -/** Run unit tests for getting the median of a list. */ -static void -test_util_order_functions(void) -{ - int lst[25], n = 0; - // int a=12,b=24,c=25,d=60,e=77; - -#define median() median_int(lst, n) - - lst[n++] = 12; - test_eq(12, median()); /* 12 */ - lst[n++] = 77; - //smartlist_shuffle(sl); - test_eq(12, median()); /* 12, 77 */ - lst[n++] = 77; - //smartlist_shuffle(sl); - test_eq(77, median()); /* 12, 77, 77 */ - lst[n++] = 24; - test_eq(24, median()); /* 12,24,77,77 */ - lst[n++] = 60; - lst[n++] = 12; - lst[n++] = 25; - //smartlist_shuffle(sl); - test_eq(25, median()); /* 12,12,24,25,60,77,77 */ -#undef median - - done: - ; -} - -/** Helper: Make a new routerinfo containing the right information for a - * given vote_routerstatus_t. */ -static routerinfo_t * -generate_ri_from_rs(const vote_routerstatus_t *vrs) -{ - routerinfo_t *r; - const routerstatus_t *rs = &vrs->status; - static time_t published = 0; - - r = tor_malloc_zero(sizeof(routerinfo_t)); - memcpy(r->cache_info.identity_digest, rs->identity_digest, DIGEST_LEN); - memcpy(r->cache_info.signed_descriptor_digest, rs->descriptor_digest, - DIGEST_LEN); - r->cache_info.do_not_cache = 1; - r->cache_info.routerlist_index = -1; - r->cache_info.signed_descriptor_body = - tor_strdup("123456789012345678901234567890123"); - r->cache_info.signed_descriptor_len = - strlen(r->cache_info.signed_descriptor_body); - r->exit_policy = smartlist_create(); - r->cache_info.published_on = ++published + time(NULL); - return r; -} - -/** Run unit tests for generating and parsing V3 consensus networkstatus - * documents. */ -static void -test_v3_networkstatus(void) -{ - authority_cert_t *cert1=NULL, *cert2=NULL, *cert3=NULL; - crypto_pk_env_t *sign_skey_1=NULL, *sign_skey_2=NULL, *sign_skey_3=NULL; - crypto_pk_env_t *sign_skey_leg1=NULL; - const char *msg=NULL; - - time_t now = time(NULL); - networkstatus_voter_info_t *voter; - networkstatus_t *vote=NULL, *v1=NULL, *v2=NULL, *v3=NULL, *con=NULL; - vote_routerstatus_t *vrs; - routerstatus_t *rs; - char *v1_text=NULL, *v2_text=NULL, *v3_text=NULL, *consensus_text=NULL, *cp; - smartlist_t *votes = smartlist_create(); - - /* For generating the two other consensuses. */ - char *detached_text1=NULL, *detached_text2=NULL; - char *consensus_text2=NULL, *consensus_text3=NULL; - networkstatus_t *con2=NULL, *con3=NULL; - ns_detached_signatures_t *dsig1=NULL, *dsig2=NULL; - - /* Parse certificates and keys. */ - cert1 = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL); - test_assert(cert1); - test_assert(cert1->is_cross_certified); - cert2 = authority_cert_parse_from_string(AUTHORITY_CERT_2, NULL); - test_assert(cert2); - cert3 = authority_cert_parse_from_string(AUTHORITY_CERT_3, NULL); - test_assert(cert3); - sign_skey_1 = crypto_new_pk_env(); - sign_skey_2 = crypto_new_pk_env(); - sign_skey_3 = crypto_new_pk_env(); - sign_skey_leg1 = pk_generate(4); - - test_assert(!crypto_pk_read_private_key_from_string(sign_skey_1, - AUTHORITY_SIGNKEY_1,-1)); - test_assert(!crypto_pk_read_private_key_from_string(sign_skey_2, - AUTHORITY_SIGNKEY_2,-1)); - test_assert(!crypto_pk_read_private_key_from_string(sign_skey_3, - AUTHORITY_SIGNKEY_3,-1)); - - test_assert(!crypto_pk_cmp_keys(sign_skey_1, cert1->signing_key)); - test_assert(!crypto_pk_cmp_keys(sign_skey_2, cert2->signing_key)); - - /* - * Set up a vote; generate it; try to parse it. - */ - vote = tor_malloc_zero(sizeof(networkstatus_t)); - vote->type = NS_TYPE_VOTE; - vote->published = now; - vote->valid_after = now+1000; - vote->fresh_until = now+2000; - vote->valid_until = now+3000; - vote->vote_seconds = 100; - vote->dist_seconds = 200; - vote->supported_methods = smartlist_create(); - smartlist_split_string(vote->supported_methods, "1 2 3", NULL, 0, -1); - vote->client_versions = tor_strdup("0.1.2.14,0.1.2.15"); - vote->server_versions = tor_strdup("0.1.2.14,0.1.2.15,0.1.2.16"); - vote->known_flags = smartlist_create(); - smartlist_split_string(vote->known_flags, - "Authority Exit Fast Guard Running Stable V2Dir Valid", - 0, SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); - vote->voters = smartlist_create(); - voter = tor_malloc_zero(sizeof(networkstatus_voter_info_t)); - voter->nickname = tor_strdup("Voter1"); - voter->address = tor_strdup("1.2.3.4"); - voter->addr = 0x01020304; - voter->dir_port = 80; - voter->or_port = 9000; - voter->contact = tor_strdup("voter@example.com"); - crypto_pk_get_digest(cert1->identity_key, voter->identity_digest); - smartlist_add(vote->voters, voter); - vote->cert = authority_cert_dup(cert1); - vote->routerstatus_list = smartlist_create(); - /* add the first routerstatus. */ - vrs = tor_malloc_zero(sizeof(vote_routerstatus_t)); - rs = &vrs->status; - vrs->version = tor_strdup("0.1.2.14"); - rs->published_on = now-1500; - strlcpy(rs->nickname, "router2", sizeof(rs->nickname)); - memset(rs->identity_digest, 3, DIGEST_LEN); - memset(rs->descriptor_digest, 78, DIGEST_LEN); - rs->addr = 0x99008801; - rs->or_port = 443; - rs->dir_port = 8000; - /* all flags but running cleared */ - rs->is_running = 1; - smartlist_add(vote->routerstatus_list, vrs); - test_assert(router_add_to_routerlist(generate_ri_from_rs(vrs), &msg,0,0)>=0); - - /* add the second routerstatus. */ - vrs = tor_malloc_zero(sizeof(vote_routerstatus_t)); - rs = &vrs->status; - vrs->version = tor_strdup("0.2.0.5"); - rs->published_on = now-1000; - strlcpy(rs->nickname, "router1", sizeof(rs->nickname)); - memset(rs->identity_digest, 5, DIGEST_LEN); - memset(rs->descriptor_digest, 77, DIGEST_LEN); - rs->addr = 0x99009901; - rs->or_port = 443; - rs->dir_port = 0; - rs->is_exit = rs->is_stable = rs->is_fast = rs->is_running = - rs->is_valid = rs->is_v2_dir = rs->is_possible_guard = 1; - smartlist_add(vote->routerstatus_list, vrs); - test_assert(router_add_to_routerlist(generate_ri_from_rs(vrs), &msg,0,0)>=0); - - /* add the third routerstatus. */ - vrs = tor_malloc_zero(sizeof(vote_routerstatus_t)); - rs = &vrs->status; - vrs->version = tor_strdup("0.1.0.3"); - rs->published_on = now-1000; - strlcpy(rs->nickname, "router3", sizeof(rs->nickname)); - memset(rs->identity_digest, 33, DIGEST_LEN); - memset(rs->descriptor_digest, 79, DIGEST_LEN); - rs->addr = 0xAA009901; - rs->or_port = 400; - rs->dir_port = 9999; - rs->is_authority = rs->is_exit = rs->is_stable = rs->is_fast = - rs->is_running = rs->is_valid = rs->is_v2_dir = rs->is_possible_guard = 1; - smartlist_add(vote->routerstatus_list, vrs); - test_assert(router_add_to_routerlist(generate_ri_from_rs(vrs), &msg,0,0)>=0); - - /* add a fourth routerstatus that is not running. */ - vrs = tor_malloc_zero(sizeof(vote_routerstatus_t)); - rs = &vrs->status; - vrs->version = tor_strdup("0.1.6.3"); - rs->published_on = now-1000; - strlcpy(rs->nickname, "router4", sizeof(rs->nickname)); - memset(rs->identity_digest, 34, DIGEST_LEN); - memset(rs->descriptor_digest, 48, DIGEST_LEN); - rs->addr = 0xC0000203; - rs->or_port = 500; - rs->dir_port = 1999; - /* Running flag (and others) cleared */ - smartlist_add(vote->routerstatus_list, vrs); - test_assert(router_add_to_routerlist(generate_ri_from_rs(vrs), &msg,0,0)>=0); - - /* dump the vote and try to parse it. */ - v1_text = format_networkstatus_vote(sign_skey_1, vote); - test_assert(v1_text); - v1 = networkstatus_parse_vote_from_string(v1_text, NULL, NS_TYPE_VOTE); - test_assert(v1); - - /* Make sure the parsed thing was right. */ - test_eq(v1->type, NS_TYPE_VOTE); - test_eq(v1->published, vote->published); - test_eq(v1->valid_after, vote->valid_after); - test_eq(v1->fresh_until, vote->fresh_until); - test_eq(v1->valid_until, vote->valid_until); - test_eq(v1->vote_seconds, vote->vote_seconds); - test_eq(v1->dist_seconds, vote->dist_seconds); - test_streq(v1->client_versions, vote->client_versions); - test_streq(v1->server_versions, vote->server_versions); - test_assert(v1->voters && smartlist_len(v1->voters)); - voter = smartlist_get(v1->voters, 0); - test_streq(voter->nickname, "Voter1"); - test_streq(voter->address, "1.2.3.4"); - test_eq(voter->addr, 0x01020304); - test_eq(voter->dir_port, 80); - test_eq(voter->or_port, 9000); - test_streq(voter->contact, "voter@example.com"); - test_assert(v1->cert); - test_assert(!crypto_pk_cmp_keys(sign_skey_1, v1->cert->signing_key)); - cp = smartlist_join_strings(v1->known_flags, ":", 0, NULL); - test_streq(cp, "Authority:Exit:Fast:Guard:Running:Stable:V2Dir:Valid"); - tor_free(cp); - test_eq(smartlist_len(v1->routerstatus_list), 4); - /* Check the first routerstatus. */ - vrs = smartlist_get(v1->routerstatus_list, 0); - rs = &vrs->status; - test_streq(vrs->version, "0.1.2.14"); - test_eq(rs->published_on, now-1500); - test_streq(rs->nickname, "router2"); - test_memeq(rs->identity_digest, - "\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3", - DIGEST_LEN); - test_memeq(rs->descriptor_digest, "NNNNNNNNNNNNNNNNNNNN", DIGEST_LEN); - test_eq(rs->addr, 0x99008801); - test_eq(rs->or_port, 443); - test_eq(rs->dir_port, 8000); - test_eq(vrs->flags, U64_LITERAL(16)); // no flags except "running" - /* Check the second routerstatus. */ - vrs = smartlist_get(v1->routerstatus_list, 1); - rs = &vrs->status; - test_streq(vrs->version, "0.2.0.5"); - test_eq(rs->published_on, now-1000); - test_streq(rs->nickname, "router1"); - test_memeq(rs->identity_digest, - "\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5", - DIGEST_LEN); - test_memeq(rs->descriptor_digest, "MMMMMMMMMMMMMMMMMMMM", DIGEST_LEN); - test_eq(rs->addr, 0x99009901); - test_eq(rs->or_port, 443); - test_eq(rs->dir_port, 0); - test_eq(vrs->flags, U64_LITERAL(254)); // all flags except "authority." - - /* Generate second vote. It disagrees on some of the times, - * and doesn't list versions, and knows some crazy flags */ - vote->published = now+1; - vote->fresh_until = now+3005; - vote->dist_seconds = 300; - authority_cert_free(vote->cert); - vote->cert = authority_cert_dup(cert2); - tor_free(vote->client_versions); - tor_free(vote->server_versions); - voter = smartlist_get(vote->voters, 0); - tor_free(voter->nickname); - tor_free(voter->address); - voter->nickname = tor_strdup("Voter2"); - voter->address = tor_strdup("2.3.4.5"); - voter->addr = 0x02030405; - crypto_pk_get_digest(cert2->identity_key, voter->identity_digest); - smartlist_add(vote->known_flags, tor_strdup("MadeOfCheese")); - smartlist_add(vote->known_flags, tor_strdup("MadeOfTin")); - smartlist_sort_strings(vote->known_flags); - vrs = smartlist_get(vote->routerstatus_list, 2); - smartlist_del_keeporder(vote->routerstatus_list, 2); - tor_free(vrs->version); - tor_free(vrs); - vrs = smartlist_get(vote->routerstatus_list, 0); - vrs->status.is_fast = 1; - /* generate and parse. */ - v2_text = format_networkstatus_vote(sign_skey_2, vote); - test_assert(v2_text); - v2 = networkstatus_parse_vote_from_string(v2_text, NULL, NS_TYPE_VOTE); - test_assert(v2); - /* Check that flags come out right.*/ - cp = smartlist_join_strings(v2->known_flags, ":", 0, NULL); - test_streq(cp, "Authority:Exit:Fast:Guard:MadeOfCheese:MadeOfTin:" - "Running:Stable:V2Dir:Valid"); - tor_free(cp); - vrs = smartlist_get(v2->routerstatus_list, 1); - /* 1023 - authority(1) - madeofcheese(16) - madeoftin(32) */ - test_eq(vrs->flags, U64_LITERAL(974)); - - /* Generate the third vote. */ - vote->published = now; - vote->fresh_until = now+2003; - vote->dist_seconds = 250; - authority_cert_free(vote->cert); - vote->cert = authority_cert_dup(cert3); - smartlist_add(vote->supported_methods, tor_strdup("4")); - vote->client_versions = tor_strdup("0.1.2.14,0.1.2.17"); - vote->server_versions = tor_strdup("0.1.2.10,0.1.2.15,0.1.2.16"); - voter = smartlist_get(vote->voters, 0); - tor_free(voter->nickname); - tor_free(voter->address); - voter->nickname = tor_strdup("Voter3"); - voter->address = tor_strdup("3.4.5.6"); - voter->addr = 0x03040506; - crypto_pk_get_digest(cert3->identity_key, voter->identity_digest); - /* This one has a legacy id. */ - memset(voter->legacy_id_digest, (int)'A', DIGEST_LEN); - vrs = smartlist_get(vote->routerstatus_list, 0); - smartlist_del_keeporder(vote->routerstatus_list, 0); - tor_free(vrs->version); - tor_free(vrs); - vrs = smartlist_get(vote->routerstatus_list, 0); - memset(vrs->status.descriptor_digest, (int)'Z', DIGEST_LEN); - test_assert(router_add_to_routerlist(generate_ri_from_rs(vrs), &msg,0,0)>=0); - - v3_text = format_networkstatus_vote(sign_skey_3, vote); - test_assert(v3_text); - - v3 = networkstatus_parse_vote_from_string(v3_text, NULL, NS_TYPE_VOTE); - test_assert(v3); - - /* Compute a consensus as voter 3. */ - smartlist_add(votes, v3); - smartlist_add(votes, v1); - smartlist_add(votes, v2); - consensus_text = networkstatus_compute_consensus(votes, 3, - cert3->identity_key, - sign_skey_3, - "AAAAAAAAAAAAAAAAAAAA", - sign_skey_leg1); - test_assert(consensus_text); - con = networkstatus_parse_vote_from_string(consensus_text, NULL, - NS_TYPE_CONSENSUS); - test_assert(con); - //log_notice(LD_GENERAL, "<<%s>>\n<<%s>>\n<<%s>>\n", - // v1_text, v2_text, v3_text); - - /* Check consensus contents. */ - test_assert(con->type == NS_TYPE_CONSENSUS); - test_eq(con->published, 0); /* this field only appears in votes. */ - test_eq(con->valid_after, now+1000); - test_eq(con->fresh_until, now+2003); /* median */ - test_eq(con->valid_until, now+3000); - test_eq(con->vote_seconds, 100); - test_eq(con->dist_seconds, 250); /* median */ - test_streq(con->client_versions, "0.1.2.14"); - test_streq(con->server_versions, "0.1.2.15,0.1.2.16"); - cp = smartlist_join_strings(v2->known_flags, ":", 0, NULL); - test_streq(cp, "Authority:Exit:Fast:Guard:MadeOfCheese:MadeOfTin:" - "Running:Stable:V2Dir:Valid"); - tor_free(cp); - test_eq(4, smartlist_len(con->voters)); /*3 voters, 1 legacy key.*/ - /* The voter id digests should be in this order. */ - test_assert(memcmp(cert2->cache_info.identity_digest, - cert1->cache_info.identity_digest,DIGEST_LEN)<0); - test_assert(memcmp(cert1->cache_info.identity_digest, - cert3->cache_info.identity_digest,DIGEST_LEN)<0); - test_same_voter(smartlist_get(con->voters, 1), - smartlist_get(v2->voters, 0)); - test_same_voter(smartlist_get(con->voters, 2), - smartlist_get(v1->voters, 0)); - test_same_voter(smartlist_get(con->voters, 3), - smartlist_get(v3->voters, 0)); - - test_assert(!con->cert); - test_eq(2, smartlist_len(con->routerstatus_list)); - /* There should be two listed routers: one with identity 3, one with - * identity 5. */ - /* This one showed up in 2 digests. */ - rs = smartlist_get(con->routerstatus_list, 0); - test_memeq(rs->identity_digest, - "\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3", - DIGEST_LEN); - test_memeq(rs->descriptor_digest, "NNNNNNNNNNNNNNNNNNNN", DIGEST_LEN); - test_assert(!rs->is_authority); - test_assert(!rs->is_exit); - test_assert(!rs->is_fast); - test_assert(!rs->is_possible_guard); - test_assert(!rs->is_stable); - test_assert(rs->is_running); /* If it wasn't running it wouldn't be here */ - test_assert(!rs->is_v2_dir); - test_assert(!rs->is_valid); - test_assert(!rs->is_named); - /* XXXX check version */ - - rs = smartlist_get(con->routerstatus_list, 1); - /* This one showed up in 3 digests. Twice with ID 'M', once with 'Z'. */ - test_memeq(rs->identity_digest, - "\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5", - DIGEST_LEN); - test_streq(rs->nickname, "router1"); - test_memeq(rs->descriptor_digest, "MMMMMMMMMMMMMMMMMMMM", DIGEST_LEN); - test_eq(rs->published_on, now-1000); - test_eq(rs->addr, 0x99009901); - test_eq(rs->or_port, 443); - test_eq(rs->dir_port, 0); - test_assert(!rs->is_authority); - test_assert(rs->is_exit); - test_assert(rs->is_fast); - test_assert(rs->is_possible_guard); - test_assert(rs->is_stable); - test_assert(rs->is_running); - test_assert(rs->is_v2_dir); - test_assert(rs->is_valid); - test_assert(!rs->is_named); - /* XXXX check version */ - // x231 - // x213 - - /* Check signatures. the first voter is a pseudo-entry with a legacy key. - * The second one hasn't signed. The fourth one has signed: validate it. */ - voter = smartlist_get(con->voters, 1); - test_assert(!voter->signature); - test_assert(!voter->good_signature); - test_assert(!voter->bad_signature); - - voter = smartlist_get(con->voters, 3); - test_assert(voter->signature); - test_assert(!voter->good_signature); - test_assert(!voter->bad_signature); - test_assert(!networkstatus_check_voter_signature(con, - smartlist_get(con->voters, 3), - cert3)); - test_assert(voter->signature); - test_assert(voter->good_signature); - test_assert(!voter->bad_signature); - - { - const char *msg=NULL; - /* Compute the other two signed consensuses. */ - smartlist_shuffle(votes); - consensus_text2 = networkstatus_compute_consensus(votes, 3, - cert2->identity_key, - sign_skey_2, NULL,NULL); - smartlist_shuffle(votes); - consensus_text3 = networkstatus_compute_consensus(votes, 3, - cert1->identity_key, - sign_skey_1, NULL,NULL); - test_assert(consensus_text2); - test_assert(consensus_text3); - con2 = networkstatus_parse_vote_from_string(consensus_text2, NULL, - NS_TYPE_CONSENSUS); - con3 = networkstatus_parse_vote_from_string(consensus_text3, NULL, - NS_TYPE_CONSENSUS); - test_assert(con2); - test_assert(con3); - - /* All three should have the same digest. */ - test_memeq(con->networkstatus_digest, con2->networkstatus_digest, - DIGEST_LEN); - test_memeq(con->networkstatus_digest, con3->networkstatus_digest, - DIGEST_LEN); - - /* Extract a detached signature from con3. */ - detached_text1 = networkstatus_get_detached_signatures(con3); - tor_assert(detached_text1); - /* Try to parse it. */ - dsig1 = networkstatus_parse_detached_signatures(detached_text1, NULL); - tor_assert(dsig1); - - /* Are parsed values as expected? */ - test_eq(dsig1->valid_after, con3->valid_after); - test_eq(dsig1->fresh_until, con3->fresh_until); - test_eq(dsig1->valid_until, con3->valid_until); - test_memeq(dsig1->networkstatus_digest, con3->networkstatus_digest, - DIGEST_LEN); - test_eq(1, smartlist_len(dsig1->signatures)); - voter = smartlist_get(dsig1->signatures, 0); - test_memeq(voter->identity_digest, cert1->cache_info.identity_digest, - DIGEST_LEN); - - /* Try adding it to con2. */ - detached_text2 = networkstatus_get_detached_signatures(con2); - test_eq(1, networkstatus_add_detached_signatures(con2, dsig1, &msg)); - tor_free(detached_text2); - detached_text2 = networkstatus_get_detached_signatures(con2); - //printf("\n<%s>\n", detached_text2); - dsig2 = networkstatus_parse_detached_signatures(detached_text2, NULL); - test_assert(dsig2); - /* - printf("\n"); - SMARTLIST_FOREACH(dsig2->signatures, networkstatus_voter_info_t *, vi, { - char hd[64]; - base16_encode(hd, sizeof(hd), vi->identity_digest, DIGEST_LEN); - printf("%s\n", hd); - }); - */ - test_eq(2, smartlist_len(dsig2->signatures)); - - /* Try adding to con2 twice; verify that nothing changes. */ - test_eq(0, networkstatus_add_detached_signatures(con2, dsig1, &msg)); - - /* Add to con. */ - test_eq(2, networkstatus_add_detached_signatures(con, dsig2, &msg)); - /* Check signatures */ - test_assert(!networkstatus_check_voter_signature(con, - smartlist_get(con->voters, 1), - cert2)); - test_assert(!networkstatus_check_voter_signature(con, - smartlist_get(con->voters, 2), - cert1)); - - } - - done: - smartlist_free(votes); - tor_free(v1_text); - tor_free(v2_text); - tor_free(v3_text); - tor_free(consensus_text); - - if (vote) - networkstatus_vote_free(vote); - if (v1) - networkstatus_vote_free(v1); - if (v2) - networkstatus_vote_free(v2); - if (v3) - networkstatus_vote_free(v3); - if (con) - networkstatus_vote_free(con); - if (sign_skey_1) - crypto_free_pk_env(sign_skey_1); - if (sign_skey_2) - crypto_free_pk_env(sign_skey_2); - if (sign_skey_3) - crypto_free_pk_env(sign_skey_3); - if (sign_skey_leg1) - crypto_free_pk_env(sign_skey_leg1); - if (cert1) - authority_cert_free(cert1); - if (cert2) - authority_cert_free(cert2); - if (cert3) - authority_cert_free(cert3); - - tor_free(consensus_text2); - tor_free(consensus_text3); - tor_free(detached_text1); - tor_free(detached_text2); - if (con2) - networkstatus_vote_free(con2); - if (con3) - networkstatus_vote_free(con3); - if (dsig1) - ns_detached_signatures_free(dsig1); - if (dsig2) - ns_detached_signatures_free(dsig2); -} - -/** Helper: Parse the exit policy string in <b>policy_str</b>, and make sure - * that policies_summarize() produces the string <b>expected_summary</b> from - * it. */ -static void -test_policy_summary_helper(const char *policy_str, - const char *expected_summary) -{ - config_line_t line; - smartlist_t *policy = smartlist_create(); - char *summary = NULL; - int r; - - line.key = (char*)"foo"; - line.value = (char *)policy_str; - line.next = NULL; - - r = policies_parse_exit_policy(&line, &policy, 0, NULL); - test_eq(r, 0); - summary = policy_summarize(policy); - - test_assert(summary != NULL); - test_streq(summary, expected_summary); - - done: - tor_free(summary); - if (policy) - addr_policy_list_free(policy); -} - -/** Run unit tests for generating summary lines of exit policies */ -static void -test_policies(void) -{ - int i; - smartlist_t *policy = NULL, *policy2 = NULL; - addr_policy_t *p; - tor_addr_t tar; - config_line_t line; - smartlist_t *sm = NULL; - char *policy_str = NULL; - - policy = smartlist_create(); - - p = router_parse_addr_policy_item_from_string("reject 192.168.0.0/16:*",-1); - test_assert(p != NULL); - test_eq(ADDR_POLICY_REJECT, p->policy_type); - tor_addr_from_ipv4h(&tar, 0xc0a80000u); - test_eq(0, tor_addr_compare(&p->addr, &tar, CMP_EXACT)); - test_eq(16, p->maskbits); - test_eq(1, p->prt_min); - test_eq(65535, p->prt_max); - - smartlist_add(policy, p); - - test_assert(ADDR_POLICY_ACCEPTED == - compare_addr_to_addr_policy(0x01020304u, 2, policy)); - test_assert(ADDR_POLICY_PROBABLY_ACCEPTED == - compare_addr_to_addr_policy(0, 2, policy)); - test_assert(ADDR_POLICY_REJECTED == - compare_addr_to_addr_policy(0xc0a80102, 2, policy)); - - policy2 = NULL; - test_assert(0 == policies_parse_exit_policy(NULL, &policy2, 1, NULL)); - test_assert(policy2); - - test_assert(!exit_policy_is_general_exit(policy)); - test_assert(exit_policy_is_general_exit(policy2)); - test_assert(!exit_policy_is_general_exit(NULL)); - - test_assert(cmp_addr_policies(policy, policy2)); - test_assert(cmp_addr_policies(policy, NULL)); - test_assert(!cmp_addr_policies(policy2, policy2)); - test_assert(!cmp_addr_policies(NULL, NULL)); - - test_assert(!policy_is_reject_star(policy2)); - test_assert(policy_is_reject_star(policy)); - test_assert(policy_is_reject_star(NULL)); - - addr_policy_list_free(policy); - policy = NULL; - - /* make sure compacting logic works. */ - policy = NULL; - line.key = (char*)"foo"; - line.value = (char*)"accept *:80,reject private:*,reject *:*"; - line.next = NULL; - test_assert(0 == policies_parse_exit_policy(&line, &policy, 0, NULL)); - test_assert(policy); - //test_streq(policy->string, "accept *:80"); - //test_streq(policy->next->string, "reject *:*"); - test_eq(smartlist_len(policy), 2); - - /* test policy summaries */ - /* check if we properly ignore private IP addresses */ - test_policy_summary_helper("reject 192.168.0.0/16:*," - "reject 0.0.0.0/8:*," - "reject 10.0.0.0/8:*," - "accept *:10-30," - "accept *:90," - "reject *:*", - "accept 10-30,90"); - /* check all accept policies, and proper counting of rejects */ - test_policy_summary_helper("reject 11.0.0.0/9:80," - "reject 12.0.0.0/9:80," - "reject 13.0.0.0/9:80," - "reject 14.0.0.0/9:80," - "accept *:*", "accept 1-65535"); - test_policy_summary_helper("reject 11.0.0.0/9:80," - "reject 12.0.0.0/9:80," - "reject 13.0.0.0/9:80," - "reject 14.0.0.0/9:80," - "reject 15.0.0.0:81," - "accept *:*", "accept 1-65535"); - test_policy_summary_helper("reject 11.0.0.0/9:80," - "reject 12.0.0.0/9:80," - "reject 13.0.0.0/9:80," - "reject 14.0.0.0/9:80," - "reject 15.0.0.0:80," - "accept *:*", - "reject 80"); - /* no exits */ - test_policy_summary_helper("accept 11.0.0.0/9:80," - "reject *:*", - "reject 1-65535"); - /* port merging */ - test_policy_summary_helper("accept *:80," - "accept *:81," - "accept *:100-110," - "accept *:111," - "reject *:*", - "accept 80-81,100-111"); - /* border ports */ - test_policy_summary_helper("accept *:1," - "accept *:3," - "accept *:65535," - "reject *:*", - "accept 1,3,65535"); - /* holes */ - test_policy_summary_helper("accept *:1," - "accept *:3," - "accept *:5," - "accept *:7," - "reject *:*", - "accept 1,3,5,7"); - test_policy_summary_helper("reject *:1," - "reject *:3," - "reject *:5," - "reject *:7," - "accept *:*", - "reject 1,3,5,7"); - - /* truncation ports */ - sm = smartlist_create(); - for (i=1; i<2000; i+=2) { - char buf[POLICY_BUF_LEN]; - tor_snprintf(buf, sizeof(buf), "reject *:%d", i); - smartlist_add(sm, tor_strdup(buf)); - } - smartlist_add(sm, tor_strdup("accept *:*")); - policy_str = smartlist_join_strings(sm, ",", 0, NULL); - test_policy_summary_helper( policy_str, - "accept 2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40,42,44," - "46,48,50,52,54,56,58,60,62,64,66,68,70,72,74,76,78,80,82,84,86,88,90," - "92,94,96,98,100,102,104,106,108,110,112,114,116,118,120,122,124,126,128," - "130,132,134,136,138,140,142,144,146,148,150,152,154,156,158,160,162,164," - "166,168,170,172,174,176,178,180,182,184,186,188,190,192,194,196,198,200," - "202,204,206,208,210,212,214,216,218,220,222,224,226,228,230,232,234,236," - "238,240,242,244,246,248,250,252,254,256,258,260,262,264,266,268,270,272," - "274,276,278,280,282,284,286,288,290,292,294,296,298,300,302,304,306,308," - "310,312,314,316,318,320,322,324,326,328,330,332,334,336,338,340,342,344," - "346,348,350,352,354,356,358,360,362,364,366,368,370,372,374,376,378,380," - "382,384,386,388,390,392,394,396,398,400,402,404,406,408,410,412,414,416," - "418,420,422,424,426,428,430,432,434,436,438,440,442,444,446,448,450,452," - "454,456,458,460,462,464,466,468,470,472,474,476,478,480,482,484,486,488," - "490,492,494,496,498,500,502,504,506,508,510,512,514,516,518,520,522"); - - done: - if (policy) - addr_policy_list_free(policy); - if (policy2) - addr_policy_list_free(policy2); - tor_free(policy_str); - if (sm) { - SMARTLIST_FOREACH(sm, char *, s, tor_free(s)); - smartlist_free(sm); - } -} - -/** Run unit tests for basic rendezvous functions. */ -static void -test_rend_fns(void) -{ - char address1[] = "fooaddress.onion"; - char address2[] = "aaaaaaaaaaaaaaaa.onion"; - char address3[] = "fooaddress.exit"; - char address4[] = "www.torproject.org"; - rend_service_descriptor_t *d1 = - tor_malloc_zero(sizeof(rend_service_descriptor_t)); - rend_service_descriptor_t *d2 = NULL; - char *encoded = NULL; - size_t len; - time_t now; - int i; - crypto_pk_env_t *pk1 = pk_generate(0), *pk2 = pk_generate(1); - - /* Test unversioned (v0) descriptor */ - d1->pk = crypto_pk_dup_key(pk1); - now = time(NULL); - d1->timestamp = now; - d1->version = 0; - d1->intro_nodes = smartlist_create(); - for (i = 0; i < 3; i++) { - rend_intro_point_t *intro = tor_malloc_zero(sizeof(rend_intro_point_t)); - intro->extend_info = tor_malloc_zero(sizeof(extend_info_t)); - crypto_rand(intro->extend_info->identity_digest, DIGEST_LEN); - intro->extend_info->nickname[0] = '$'; - base16_encode(intro->extend_info->nickname+1, HEX_DIGEST_LEN+1, - intro->extend_info->identity_digest, DIGEST_LEN); - smartlist_add(d1->intro_nodes, intro); - } - test_assert(! rend_encode_service_descriptor(d1, pk1, &encoded, &len)); - d2 = rend_parse_service_descriptor(encoded, len); - test_assert(d2); - - test_assert(!crypto_pk_cmp_keys(d1->pk, d2->pk)); - test_eq(d2->timestamp, now); - test_eq(d2->version, 0); - test_eq(d2->protocols, 1<<2); - test_eq(smartlist_len(d2->intro_nodes), 3); - for (i = 0; i < 3; i++) { - rend_intro_point_t *intro1 = smartlist_get(d1->intro_nodes, i); - rend_intro_point_t *intro2 = smartlist_get(d2->intro_nodes, i); - test_streq(intro1->extend_info->nickname, - intro2->extend_info->nickname); - } - - test_assert(BAD_HOSTNAME == parse_extended_hostname(address1)); - test_assert(ONION_HOSTNAME == parse_extended_hostname(address2)); - test_assert(EXIT_HOSTNAME == parse_extended_hostname(address3)); - test_assert(NORMAL_HOSTNAME == parse_extended_hostname(address4)); - - crypto_free_pk_env(pk1); - crypto_free_pk_env(pk2); - pk1 = pk2 = NULL; - rend_service_descriptor_free(d1); - rend_service_descriptor_free(d2); - d1 = d2 = NULL; - - done: - if (pk1) - crypto_free_pk_env(pk1); - if (pk2) - crypto_free_pk_env(pk2); - if (d1) - rend_service_descriptor_free(d1); - if (d2) - rend_service_descriptor_free(d2); - tor_free(encoded); -} - -/** Run AES performance benchmarks. */ -static void -bench_aes(void) -{ - int len, i; - char *b1, *b2; - crypto_cipher_env_t *c; - struct timeval start, end; - const int iters = 100000; - uint64_t nsec; - c = crypto_new_cipher_env(); - crypto_cipher_generate_key(c); - crypto_cipher_encrypt_init_cipher(c); - for (len = 1; len <= 8192; len *= 2) { - b1 = tor_malloc_zero(len); - b2 = tor_malloc_zero(len); - tor_gettimeofday(&start); - for (i = 0; i < iters; ++i) { - crypto_cipher_encrypt(c, b1, b2, len); - } - tor_gettimeofday(&end); - tor_free(b1); - tor_free(b2); - nsec = (uint64_t) tv_udiff(&start,&end); - nsec *= 1000; - nsec /= (iters*len); - printf("%d bytes: "U64_FORMAT" nsec per byte\n", len, - U64_PRINTF_ARG(nsec)); - } - crypto_free_cipher_env(c); -} - -/** Run digestmap_t performance benchmarks. */ -static void -bench_dmap(void) -{ - smartlist_t *sl = smartlist_create(); - smartlist_t *sl2 = smartlist_create(); - struct timeval start, end, pt2, pt3, pt4; - const int iters = 10000; - const int elts = 4000; - const int fpostests = 1000000; - char d[20]; - int i,n=0, fp = 0; - digestmap_t *dm = digestmap_new(); - digestset_t *ds = digestset_new(elts); - - for (i = 0; i < elts; ++i) { - crypto_rand(d, 20); - smartlist_add(sl, tor_memdup(d, 20)); - } - for (i = 0; i < elts; ++i) { - crypto_rand(d, 20); - smartlist_add(sl2, tor_memdup(d, 20)); - } - printf("nbits=%d\n", ds->mask+1); - - tor_gettimeofday(&start); - for (i = 0; i < iters; ++i) { - SMARTLIST_FOREACH(sl, const char *, cp, digestmap_set(dm, cp, (void*)1)); - } - tor_gettimeofday(&pt2); - for (i = 0; i < iters; ++i) { - SMARTLIST_FOREACH(sl, const char *, cp, digestmap_get(dm, cp)); - SMARTLIST_FOREACH(sl2, const char *, cp, digestmap_get(dm, cp)); - } - tor_gettimeofday(&pt3); - for (i = 0; i < iters; ++i) { - SMARTLIST_FOREACH(sl, const char *, cp, digestset_add(ds, cp)); - } - tor_gettimeofday(&pt4); - for (i = 0; i < iters; ++i) { - SMARTLIST_FOREACH(sl, const char *, cp, n += digestset_isin(ds, cp)); - SMARTLIST_FOREACH(sl2, const char *, cp, n += digestset_isin(ds, cp)); - } - tor_gettimeofday(&end); - - for (i = 0; i < fpostests; ++i) { - crypto_rand(d, 20); - if (digestset_isin(ds, d)) ++fp; - } - - printf("%ld\n",(unsigned long)tv_udiff(&start, &pt2)); - printf("%ld\n",(unsigned long)tv_udiff(&pt2, &pt3)); - printf("%ld\n",(unsigned long)tv_udiff(&pt3, &pt4)); - printf("%ld\n",(unsigned long)tv_udiff(&pt4, &end)); - printf("-- %d\n", n); - printf("++ %f\n", fp/(double)fpostests); - digestmap_free(dm, NULL); - digestset_free(ds); - SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); - SMARTLIST_FOREACH(sl2, char *, cp, tor_free(cp)); - smartlist_free(sl); - smartlist_free(sl2); -} - -/** Run unittests for memory pool allocator */ -static void -test_util_mempool(void) -{ - mp_pool_t *pool = NULL; - smartlist_t *allocated = NULL; - int i; - - pool = mp_pool_new(1, 100); - test_assert(pool); - test_assert(pool->new_chunk_capacity >= 100); - test_assert(pool->item_alloc_size >= sizeof(void*)+1); - mp_pool_destroy(pool); - pool = NULL; - - pool = mp_pool_new(241, 2500); - test_assert(pool); - test_assert(pool->new_chunk_capacity >= 10); - test_assert(pool->item_alloc_size >= sizeof(void*)+241); - test_eq(pool->item_alloc_size & 0x03, 0); - test_assert(pool->new_chunk_capacity < 60); - - allocated = smartlist_create(); - for (i = 0; i < 20000; ++i) { - if (smartlist_len(allocated) < 20 || crypto_rand_int(2)) { - void *m = mp_pool_get(pool); - memset(m, 0x09, 241); - smartlist_add(allocated, m); - //printf("%d: %p\n", i, m); - //mp_pool_assert_ok(pool); - } else { - int idx = crypto_rand_int(smartlist_len(allocated)); - void *m = smartlist_get(allocated, idx); - //printf("%d: free %p\n", i, m); - smartlist_del(allocated, idx); - mp_pool_release(m); - //mp_pool_assert_ok(pool); - } - if (crypto_rand_int(777)==0) - mp_pool_clean(pool, 1, 1); - - if (i % 777) - mp_pool_assert_ok(pool); - } - - done: - if (allocated) { - SMARTLIST_FOREACH(allocated, void *, m, mp_pool_release(m)); - mp_pool_assert_ok(pool); - mp_pool_clean(pool, 0, 0); - mp_pool_assert_ok(pool); - smartlist_free(allocated); - } - - if (pool) - mp_pool_destroy(pool); -} - -/** Run unittests for memory area allocator */ -static void -test_util_memarea(void) -{ - memarea_t *area = memarea_new(); - char *p1, *p2, *p3, *p1_orig; - void *malloced_ptr = NULL; - int i; - - test_assert(area); - - p1_orig = p1 = memarea_alloc(area,64); - p2 = memarea_alloc_zero(area,52); - p3 = memarea_alloc(area,11); - - test_assert(memarea_owns_ptr(area, p1)); - test_assert(memarea_owns_ptr(area, p2)); - test_assert(memarea_owns_ptr(area, p3)); - /* Make sure we left enough space. */ - test_assert(p1+64 <= p2); - test_assert(p2+52 <= p3); - /* Make sure we aligned. */ - test_eq(((uintptr_t)p1) % sizeof(void*), 0); - test_eq(((uintptr_t)p2) % sizeof(void*), 0); - test_eq(((uintptr_t)p3) % sizeof(void*), 0); - test_assert(!memarea_owns_ptr(area, p3+8192)); - test_assert(!memarea_owns_ptr(area, p3+30)); - test_assert(tor_mem_is_zero(p2, 52)); - /* Make sure we don't overalign. */ - p1 = memarea_alloc(area, 1); - p2 = memarea_alloc(area, 1); - test_eq(p1+sizeof(void*), p2); - { - malloced_ptr = tor_malloc(64); - test_assert(!memarea_owns_ptr(area, malloced_ptr)); - tor_free(malloced_ptr); - } - - /* memarea_memdup */ - { - malloced_ptr = tor_malloc(64); - crypto_rand((char*)malloced_ptr, 64); - p1 = memarea_memdup(area, malloced_ptr, 64); - test_assert(p1 != malloced_ptr); - test_memeq(p1, malloced_ptr, 64); - tor_free(malloced_ptr); - } - - /* memarea_strdup. */ - p1 = memarea_strdup(area,""); - p2 = memarea_strdup(area, "abcd"); - test_assert(p1); - test_assert(p2); - test_streq(p1, ""); - test_streq(p2, "abcd"); - - /* memarea_strndup. */ - { - const char *s = "Ad ogni porta batte la morte e grida: il nome!"; - /* (From Turandot, act 3.) */ - size_t len = strlen(s); - p1 = memarea_strndup(area, s, 1000); - p2 = memarea_strndup(area, s, 10); - test_streq(p1, s); - test_assert(p2 >= p1 + len + 1); - test_memeq(s, p2, 10); - test_eq(p2[10], '\0'); - p3 = memarea_strndup(area, s, len); - test_streq(p3, s); - p3 = memarea_strndup(area, s, len-1); - test_memeq(s, p3, len-1); - test_eq(p3[len-1], '\0'); - } - - memarea_clear(area); - p1 = memarea_alloc(area, 1); - test_eq(p1, p1_orig); - memarea_clear(area); - - /* Check for running over an area's size. */ - for (i = 0; i < 512; ++i) { - p1 = memarea_alloc(area, crypto_rand_int(5)+1); - test_assert(memarea_owns_ptr(area, p1)); - } - memarea_assert_ok(area); - /* Make sure we can allocate a too-big object. */ - p1 = memarea_alloc_zero(area, 9000); - p2 = memarea_alloc_zero(area, 16); - test_assert(memarea_owns_ptr(area, p1)); - test_assert(memarea_owns_ptr(area, p2)); - - done: - memarea_drop_all(area); - tor_free(malloced_ptr); -} - -/** Run unit tests for utility functions to get file names relative to - * the data directory. */ -static void -test_util_datadir(void) -{ - char buf[1024]; - char *f = NULL; - - f = get_datadir_fname(NULL); - test_streq(f, temp_dir); - tor_free(f); - f = get_datadir_fname("state"); - tor_snprintf(buf, sizeof(buf), "%s"PATH_SEPARATOR"state", temp_dir); - test_streq(f, buf); - tor_free(f); - f = get_datadir_fname2("cache", "thingy"); - tor_snprintf(buf, sizeof(buf), - "%s"PATH_SEPARATOR"cache"PATH_SEPARATOR"thingy", temp_dir); - test_streq(f, buf); - tor_free(f); - f = get_datadir_fname2_suffix("cache", "thingy", ".foo"); - tor_snprintf(buf, sizeof(buf), - "%s"PATH_SEPARATOR"cache"PATH_SEPARATOR"thingy.foo", temp_dir); - test_streq(f, buf); - tor_free(f); - f = get_datadir_fname_suffix("cache", ".foo"); - tor_snprintf(buf, sizeof(buf), "%s"PATH_SEPARATOR"cache.foo", - temp_dir); - test_streq(f, buf); - - done: - tor_free(f); -} - -/** Test AES-CTR encryption and decryption with IV. */ -static void -test_crypto_aes_iv(void) -{ - crypto_cipher_env_t *cipher; - char *plain, *encrypted1, *encrypted2, *decrypted1, *decrypted2; - char plain_1[1], plain_15[15], plain_16[16], plain_17[17]; - char key1[16], key2[16]; - ssize_t encrypted_size, decrypted_size; - - plain = tor_malloc(4095); - encrypted1 = tor_malloc(4095 + 1 + 16); - encrypted2 = tor_malloc(4095 + 1 + 16); - decrypted1 = tor_malloc(4095 + 1); - decrypted2 = tor_malloc(4095 + 1); - - crypto_rand(plain, 4095); - crypto_rand(key1, 16); - crypto_rand(key2, 16); - crypto_rand(plain_1, 1); - crypto_rand(plain_15, 15); - crypto_rand(plain_16, 16); - crypto_rand(plain_17, 17); - key1[0] = key2[0] + 128; /* Make sure that contents are different. */ - /* Encrypt and decrypt with the same key. */ - cipher = crypto_create_init_cipher(key1, 1); - encrypted_size = crypto_cipher_encrypt_with_iv(cipher, encrypted1, 16 + 4095, - plain, 4095); - crypto_free_cipher_env(cipher); - cipher = NULL; - test_eq(encrypted_size, 16 + 4095); - tor_assert(encrypted_size > 0); /* This is obviously true, since 4111 is - * greater than 0, but its truth is not - * obvious to all analysis tools. */ - cipher = crypto_create_init_cipher(key1, 0); - decrypted_size = crypto_cipher_decrypt_with_iv(cipher, decrypted1, 4095, - encrypted1, encrypted_size); - crypto_free_cipher_env(cipher); - cipher = NULL; - test_eq(decrypted_size, 4095); - tor_assert(decrypted_size > 0); - test_memeq(plain, decrypted1, 4095); - /* Encrypt a second time (with a new random initialization vector). */ - cipher = crypto_create_init_cipher(key1, 1); - encrypted_size = crypto_cipher_encrypt_with_iv(cipher, encrypted2, 16 + 4095, - plain, 4095); - crypto_free_cipher_env(cipher); - cipher = NULL; - test_eq(encrypted_size, 16 + 4095); - tor_assert(encrypted_size > 0); - cipher = crypto_create_init_cipher(key1, 0); - decrypted_size = crypto_cipher_decrypt_with_iv(cipher, decrypted2, 4095, - encrypted2, encrypted_size); - crypto_free_cipher_env(cipher); - cipher = NULL; - test_eq(decrypted_size, 4095); - tor_assert(decrypted_size > 0); - test_memeq(plain, decrypted2, 4095); - test_memneq(encrypted1, encrypted2, encrypted_size); - /* Decrypt with the wrong key. */ - cipher = crypto_create_init_cipher(key2, 0); - decrypted_size = crypto_cipher_decrypt_with_iv(cipher, decrypted2, 4095, - encrypted1, encrypted_size); - crypto_free_cipher_env(cipher); - cipher = NULL; - test_memneq(plain, decrypted2, encrypted_size); - /* Alter the initialization vector. */ - encrypted1[0] += 42; - cipher = crypto_create_init_cipher(key1, 0); - decrypted_size = crypto_cipher_decrypt_with_iv(cipher, decrypted1, 4095, - encrypted1, encrypted_size); - crypto_free_cipher_env(cipher); - cipher = NULL; - test_memneq(plain, decrypted2, 4095); - /* Special length case: 1. */ - cipher = crypto_create_init_cipher(key1, 1); - encrypted_size = crypto_cipher_encrypt_with_iv(cipher, encrypted1, 16 + 1, - plain_1, 1); - crypto_free_cipher_env(cipher); - cipher = NULL; - test_eq(encrypted_size, 16 + 1); - tor_assert(encrypted_size > 0); - cipher = crypto_create_init_cipher(key1, 0); - decrypted_size = crypto_cipher_decrypt_with_iv(cipher, decrypted1, 1, - encrypted1, encrypted_size); - crypto_free_cipher_env(cipher); - cipher = NULL; - test_eq(decrypted_size, 1); - tor_assert(decrypted_size > 0); - test_memeq(plain_1, decrypted1, 1); - /* Special length case: 15. */ - cipher = crypto_create_init_cipher(key1, 1); - encrypted_size = crypto_cipher_encrypt_with_iv(cipher, encrypted1, 16 + 15, - plain_15, 15); - crypto_free_cipher_env(cipher); - cipher = NULL; - test_eq(encrypted_size, 16 + 15); - tor_assert(encrypted_size > 0); - cipher = crypto_create_init_cipher(key1, 0); - decrypted_size = crypto_cipher_decrypt_with_iv(cipher, decrypted1, 15, - encrypted1, encrypted_size); - crypto_free_cipher_env(cipher); - cipher = NULL; - test_eq(decrypted_size, 15); - tor_assert(decrypted_size > 0); - test_memeq(plain_15, decrypted1, 15); - /* Special length case: 16. */ - cipher = crypto_create_init_cipher(key1, 1); - encrypted_size = crypto_cipher_encrypt_with_iv(cipher, encrypted1, 16 + 16, - plain_16, 16); - crypto_free_cipher_env(cipher); - cipher = NULL; - test_eq(encrypted_size, 16 + 16); - tor_assert(encrypted_size > 0); - cipher = crypto_create_init_cipher(key1, 0); - decrypted_size = crypto_cipher_decrypt_with_iv(cipher, decrypted1, 16, - encrypted1, encrypted_size); - crypto_free_cipher_env(cipher); - cipher = NULL; - test_eq(decrypted_size, 16); - tor_assert(decrypted_size > 0); - test_memeq(plain_16, decrypted1, 16); - /* Special length case: 17. */ - cipher = crypto_create_init_cipher(key1, 1); - encrypted_size = crypto_cipher_encrypt_with_iv(cipher, encrypted1, 16 + 17, - plain_17, 17); - crypto_free_cipher_env(cipher); - cipher = NULL; - test_eq(encrypted_size, 16 + 17); - tor_assert(encrypted_size > 0); - cipher = crypto_create_init_cipher(key1, 0); - decrypted_size = crypto_cipher_decrypt_with_iv(cipher, decrypted1, 17, - encrypted1, encrypted_size); - test_eq(decrypted_size, 17); - tor_assert(decrypted_size > 0); - test_memeq(plain_17, decrypted1, 17); - - done: - /* Free memory. */ - tor_free(plain); - tor_free(encrypted1); - tor_free(encrypted2); - tor_free(decrypted1); - tor_free(decrypted2); - if (cipher) - crypto_free_cipher_env(cipher); -} - -/** Test base32 decoding. */ -static void -test_crypto_base32_decode(void) -{ - char plain[60], encoded[96 + 1], decoded[60]; - int res; - crypto_rand(plain, 60); - /* Encode and decode a random string. */ - base32_encode(encoded, 96 + 1, plain, 60); - res = base32_decode(decoded, 60, encoded, 96); - test_eq(res, 0); - test_memeq(plain, decoded, 60); - /* Encode, uppercase, and decode a random string. */ - base32_encode(encoded, 96 + 1, plain, 60); - tor_strupper(encoded); - res = base32_decode(decoded, 60, encoded, 96); - test_eq(res, 0); - test_memeq(plain, decoded, 60); - /* Change encoded string and decode. */ - if (encoded[0] == 'A' || encoded[0] == 'a') - encoded[0] = 'B'; - else - encoded[0] = 'A'; - res = base32_decode(decoded, 60, encoded, 96); - test_eq(res, 0); - test_memneq(plain, decoded, 60); - /* Bad encodings. */ - encoded[0] = '!'; - res = base32_decode(decoded, 60, encoded, 96); - test_assert(res < 0); - - done: - ; -} - -/** Test encoding and parsing of v2 rendezvous service descriptors. */ -static void -test_rend_fns_v2(void) -{ - rend_service_descriptor_t *generated = NULL, *parsed = NULL; - char service_id[DIGEST_LEN]; - char service_id_base32[REND_SERVICE_ID_LEN_BASE32+1]; - const char *next_desc; - smartlist_t *descs = smartlist_create(); - char computed_desc_id[DIGEST_LEN]; - char parsed_desc_id[DIGEST_LEN]; - crypto_pk_env_t *pk1 = NULL, *pk2 = NULL; - time_t now; - char *intro_points_encrypted = NULL; - size_t intro_points_size; - size_t encoded_size; - int i; - pk1 = pk_generate(0); - pk2 = pk_generate(1); - generated = tor_malloc_zero(sizeof(rend_service_descriptor_t)); - generated->pk = crypto_pk_dup_key(pk1); - crypto_pk_get_digest(generated->pk, service_id); - base32_encode(service_id_base32, REND_SERVICE_ID_LEN_BASE32+1, - service_id, REND_SERVICE_ID_LEN); - now = time(NULL); - generated->timestamp = now; - generated->version = 2; - generated->protocols = 42; - generated->intro_nodes = smartlist_create(); - - for (i = 0; i < 3; i++) { - rend_intro_point_t *intro = tor_malloc_zero(sizeof(rend_intro_point_t)); - crypto_pk_env_t *okey = pk_generate(2 + i); - intro->extend_info = tor_malloc_zero(sizeof(extend_info_t)); - intro->extend_info->onion_key = okey; - crypto_pk_get_digest(intro->extend_info->onion_key, - intro->extend_info->identity_digest); - //crypto_rand(info->identity_digest, DIGEST_LEN); /* Would this work? */ - intro->extend_info->nickname[0] = '$'; - base16_encode(intro->extend_info->nickname + 1, - sizeof(intro->extend_info->nickname) - 1, - intro->extend_info->identity_digest, DIGEST_LEN); - /* Does not cover all IP addresses. */ - tor_addr_from_ipv4h(&intro->extend_info->addr, crypto_rand_int(65536)); - intro->extend_info->port = crypto_rand_int(65536); - intro->intro_key = crypto_pk_dup_key(pk2); - smartlist_add(generated->intro_nodes, intro); - } - test_assert(rend_encode_v2_descriptors(descs, generated, now, 0, - REND_NO_AUTH, NULL, NULL) > 0); - test_assert(rend_compute_v2_desc_id(computed_desc_id, service_id_base32, - NULL, now, 0) == 0); - test_memeq(((rend_encoded_v2_service_descriptor_t *) - smartlist_get(descs, 0))->desc_id, computed_desc_id, DIGEST_LEN); - test_assert(rend_parse_v2_service_descriptor(&parsed, parsed_desc_id, - &intro_points_encrypted, - &intro_points_size, - &encoded_size, - &next_desc, - ((rend_encoded_v2_service_descriptor_t *) - smartlist_get(descs, 0))->desc_str) == 0); - test_assert(parsed); - test_memeq(((rend_encoded_v2_service_descriptor_t *) - smartlist_get(descs, 0))->desc_id, parsed_desc_id, DIGEST_LEN); - test_eq(rend_parse_introduction_points(parsed, intro_points_encrypted, - intro_points_size), 3); - test_assert(!crypto_pk_cmp_keys(generated->pk, parsed->pk)); - test_eq(parsed->timestamp, now); - test_eq(parsed->version, 2); - test_eq(parsed->protocols, 42); - test_eq(smartlist_len(parsed->intro_nodes), 3); - for (i = 0; i < smartlist_len(parsed->intro_nodes); i++) { - rend_intro_point_t *par_intro = smartlist_get(parsed->intro_nodes, i), - *gen_intro = smartlist_get(generated->intro_nodes, i); - extend_info_t *par_info = par_intro->extend_info; - extend_info_t *gen_info = gen_intro->extend_info; - test_assert(!crypto_pk_cmp_keys(gen_info->onion_key, par_info->onion_key)); - test_memeq(gen_info->identity_digest, par_info->identity_digest, - DIGEST_LEN); - test_streq(gen_info->nickname, par_info->nickname); - test_assert(tor_addr_eq(&gen_info->addr, &par_info->addr)); - test_eq(gen_info->port, par_info->port); - } - - rend_service_descriptor_free(parsed); - rend_service_descriptor_free(generated); - parsed = generated = NULL; - - done: - if (descs) { - for (i = 0; i < smartlist_len(descs); i++) - rend_encoded_v2_service_descriptor_free(smartlist_get(descs, i)); - smartlist_free(descs); - } - if (parsed) - rend_service_descriptor_free(parsed); - if (generated) - rend_service_descriptor_free(generated); - if (pk1) - crypto_free_pk_env(pk1); - if (pk2) - crypto_free_pk_env(pk2); - tor_free(intro_points_encrypted); -} - -/** Run unit tests for GeoIP code. */ -static void -test_geoip(void) -{ - int i, j; - time_t now = time(NULL); - char *s = NULL; - - /* Populate the DB a bit. Add these in order, since we can't do the final - * 'sort' step. These aren't very good IP addresses, but they're perfectly - * fine uint32_t values. */ - test_eq(0, geoip_parse_entry("10,50,AB")); - test_eq(0, geoip_parse_entry("52,90,XY")); - test_eq(0, geoip_parse_entry("95,100,AB")); - test_eq(0, geoip_parse_entry("\"105\",\"140\",\"ZZ\"")); - test_eq(0, geoip_parse_entry("\"150\",\"190\",\"XY\"")); - test_eq(0, geoip_parse_entry("\"200\",\"250\",\"AB\"")); - - /* We should have 3 countries: ab, xy, zz. */ - test_eq(3, geoip_get_n_countries()); - /* Make sure that country ID actually works. */ -#define NAMEFOR(x) geoip_get_country_name(geoip_get_country_by_ip(x)) - test_streq("ab", NAMEFOR(32)); - test_streq("??", NAMEFOR(5)); - test_streq("??", NAMEFOR(51)); - test_streq("xy", NAMEFOR(150)); - test_streq("xy", NAMEFOR(190)); - test_streq("??", NAMEFOR(2000)); -#undef NAMEFOR - - get_options()->BridgeRelay = 1; - get_options()->BridgeRecordUsageByCountry = 1; - /* Put 9 observations in AB... */ - for (i=32; i < 40; ++i) - geoip_note_client_seen(GEOIP_CLIENT_CONNECT, i, now-7200); - geoip_note_client_seen(GEOIP_CLIENT_CONNECT, 225, now-7200); - /* and 3 observations in XY, several times. */ - for (j=0; j < 10; ++j) - for (i=52; i < 55; ++i) - geoip_note_client_seen(GEOIP_CLIENT_CONNECT, i, now-3600); - /* and 17 observations in ZZ... */ - for (i=110; i < 127; ++i) - geoip_note_client_seen(GEOIP_CLIENT_CONNECT, i, now); - s = geoip_get_client_history(now+5*24*60*60, GEOIP_CLIENT_CONNECT); - test_assert(s); - test_streq("zz=24,ab=16,xy=8", s); - tor_free(s); - - /* Now clear out all the AB observations. */ - geoip_remove_old_clients(now-6000); - s = geoip_get_client_history(now+5*24*60*60, GEOIP_CLIENT_CONNECT); - test_assert(s); - test_streq("zz=24,xy=8", s); - - done: - tor_free(s); -} - -/** For test_array. Declare an CLI-invocable off-by-default function in the - * unit tests, with function name and user-visible name <b>x</b>*/ -#define DISABLED(x) { #x, x, 0, 0, 0 } -/** For test_array. Declare an CLI-invocable unit test function, with function - * name test_<b>x</b>(), and user-visible name <b>x</b> */ -#define ENT(x) { #x, test_ ## x, 0, 0, 1 } -/** For test_array. Declare an CLI-invocable unit test function, with function - * name test_<b>x</b>_<b>y</b>(), and user-visible name - * <b>x</b>/<b>y</b>. This function will be treated as a subentry of <b>x</b>, - * so that invoking <b>x</b> from the CLI invokes this test too. */ -#define SUBENT(x,y) { #x "/" #y, test_ ## x ## _ ## y, 1, 0, 1 } - -/** An array of functions and information for all the unit tests we can run. */ -static struct { - const char *test_name; /**< How does the user refer to this test from the - * command line? */ - void (*test_fn)(void); /**< What function is called to run this test? */ - int is_subent; /**< Is this a subentry of a bigger set of related tests? */ - int selected; /**< Are we planning to run this one? */ - int is_default; /**< If the user doesn't say what tests they want, do they - * get this function by default? */ -} test_array[] = { - ENT(buffers), - ENT(crypto), - SUBENT(crypto, rng), - SUBENT(crypto, aes), - SUBENT(crypto, sha), - SUBENT(crypto, pk), - SUBENT(crypto, dh), - SUBENT(crypto, s2k), - SUBENT(crypto, aes_iv), - SUBENT(crypto, base32_decode), - ENT(util), - SUBENT(util, di_ops), - SUBENT(util, ip6_helpers), - SUBENT(util, gzip), - SUBENT(util, datadir), - SUBENT(util, smartlist_basic), - SUBENT(util, smartlist_strings), - SUBENT(util, smartlist_overlap), - SUBENT(util, smartlist_digests), - SUBENT(util, smartlist_join), - SUBENT(util, bitarray), - SUBENT(util, digestset), - SUBENT(util, mempool), - SUBENT(util, memarea), - SUBENT(util, strmap), - SUBENT(util, control_formats), - SUBENT(util, pqueue), - SUBENT(util, mmap), - SUBENT(util, threads), - SUBENT(util, order_functions), - SUBENT(util, sscanf), - ENT(onion_handshake), - ENT(dir_format), - ENT(dirutil), - ENT(v3_networkstatus), - ENT(policies), - ENT(rend_fns), - SUBENT(rend_fns, v2), - ENT(geoip), - - DISABLED(bench_aes), - DISABLED(bench_dmap), - { NULL, NULL, 0, 0, 0 }, -}; - -static void syntax(void) ATTR_NORETURN; - -/** Print a syntax usage message, and exit.*/ -static void -syntax(void) -{ - int i; - printf("Syntax:\n" - " test [-v|--verbose] [--warn|--notice|--info|--debug]\n" - " [testname...]\n" - "Recognized tests are:\n"); - for (i = 0; test_array[i].test_name; ++i) { - printf(" %s\n", test_array[i].test_name); - } - - exit(0); -} - -/** Main entry point for unit test code: parse the command line, and run - * some unit tests. */ -int -main(int c, char**v) -{ - or_options_t *options; - char *errmsg = NULL; - int i; - int verbose = 0, any_selected = 0; - int loglevel = LOG_ERR; - -#ifdef USE_DMALLOC - { - int r = CRYPTO_set_mem_ex_functions(_tor_malloc, _tor_realloc, _tor_free); - tor_assert(r); - } -#endif - - update_approx_time(time(NULL)); - options = options_new(); - tor_threads_init(); - init_logging(); - - for (i = 1; i < c; ++i) { - if (!strcmp(v[i], "-v") || !strcmp(v[i], "--verbose")) - verbose++; - else if (!strcmp(v[i], "--warn")) - loglevel = LOG_WARN; - else if (!strcmp(v[i], "--notice")) - loglevel = LOG_NOTICE; - else if (!strcmp(v[i], "--info")) - loglevel = LOG_INFO; - else if (!strcmp(v[i], "--debug")) - loglevel = LOG_DEBUG; - else if (!strcmp(v[i], "--help") || !strcmp(v[i], "-h") || v[i][0] == '-') - syntax(); - else { - int j, found=0; - for (j = 0; test_array[j].test_name; ++j) { - if (!strcmp(v[i], test_array[j].test_name) || - (test_array[j].is_subent && - !strcmpstart(test_array[j].test_name, v[i]) && - test_array[j].test_name[strlen(v[i])] == '/') || - (v[i][0] == '=' && !strcmp(v[i]+1, test_array[j].test_name))) { - test_array[j].selected = 1; - any_selected = 1; - found = 1; - } - } - if (!found) { - printf("Unknown test: %s\n", v[i]); - syntax(); - } - } - } - - if (!any_selected) { - for (i = 0; test_array[i].test_name; ++i) { - test_array[i].selected = test_array[i].is_default; - } - } - - { - log_severity_list_t s; - memset(&s, 0, sizeof(s)); - set_log_severity_config(loglevel, LOG_ERR, &s); - add_stream_log(&s, "", fileno(stdout)); - } - - options->command = CMD_RUN_UNITTESTS; - crypto_global_init(0); - rep_hist_init(); - network_init(); - setup_directory(); - options_init(options); - options->DataDirectory = tor_strdup(temp_dir); - if (set_options(options, &errmsg) < 0) { - printf("Failed to set initial options: %s\n", errmsg); - tor_free(errmsg); - return 1; - } - - crypto_seed_rng(1); - - atexit(remove_directory); - - printf("Running Tor unit tests on %s\n", get_uname()); - - for (i = 0; test_array[i].test_name; ++i) { - if (!test_array[i].selected) - continue; - if (!test_array[i].is_subent) { - printf("\n============================== %s\n",test_array[i].test_name); - } else if (test_array[i].is_subent && verbose) { - printf("\n%s", test_array[i].test_name); - } - test_array[i].test_fn(); - } - puts(""); - - free_pregenerated_keys(); -#ifdef USE_DMALLOC - tor_free_all(0); - dmalloc_log_unfreed(); -#endif - - if (have_failed) - return 1; - else - return 0; -} - diff --git a/src/or/tor_main.c b/src/or/tor_main.c index 8fd3804b29..1ce14ab768 100644 --- a/src/or/tor_main.c +++ b/src/or/tor_main.c @@ -7,7 +7,7 @@ * built from. This string is generated by a bit of shell kludging int * src/or/Makefile.am, and is usually right. */ -const char tor_svn_revision[] = +const char tor_git_revision[] = #ifndef _MSC_VER #include "micro-revision.i" #endif diff --git a/src/test/Makefile.am b/src/test/Makefile.am new file mode 100644 index 0000000000..546fa2f4b7 --- /dev/null +++ b/src/test/Makefile.am @@ -0,0 +1,30 @@ +TESTS = test + +noinst_PROGRAMS = test + +AM_CPPFLAGS = -DSHARE_DATADIR="\"$(datadir)\"" \ + -DLOCALSTATEDIR="\"$(localstatedir)\"" \ + -DBINDIR="\"$(bindir)\"" \ + -I"$(top_srcdir)/src/or" + +# -L flags need to go in LDFLAGS. -l flags need to go in LDADD. +# This seems to matter nowhere but on windows, but I assure you that it +# matters a lot there, and is quite hard to debug if you forget to do it. + +test_SOURCES = \ + test_data.c \ + test.c \ + test_addr.c \ + test_crypto.c \ + test_dir.c \ + test_containers.c \ + test_util.c \ + tinytest.c + +test_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ \ + @TOR_LDFLAGS_libevent@ +test_LDADD = ../or/libtor.a ../common/libor.a ../common/libor-crypto.a \ + ../common/libor-event.a \ + @TOR_ZLIB_LIBS@ -lm @TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ + +noinst_HEADERS = tinytest.h tinytest_macros.h test.h diff --git a/src/test/test.c b/src/test/test.c new file mode 100644 index 0000000000..9b24a99b58 --- /dev/null +++ b/src/test/test.c @@ -0,0 +1,1292 @@ +/* Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/* Ordinarily defined in tor_main.c; this bit is just here to provide one + * since we're not linking to tor_main.c */ +const char tor_git_revision[] = ""; + +/** + * \file test.c + * \brief Unit tests for many pieces of the lower level Tor modules. + **/ + +#include "orconfig.h" + +#include <stdio.h> +#ifdef HAVE_FCNTL_H +#include <fcntl.h> +#endif + +#ifdef MS_WINDOWS +/* For mkdir() */ +#include <direct.h> +#else +#include <dirent.h> +#endif + +/* These macros pull in declarations for some functions and structures that + * are typically file-private. */ +#define BUFFERS_PRIVATE +#define CONFIG_PRIVATE +#define GEOIP_PRIVATE +#define ROUTER_PRIVATE +#define CIRCUIT_PRIVATE + +/* + * Linux doesn't provide lround in math.h by default, but mac os does... + * It's best just to leave math.h out of the picture entirely. + */ +//#include <math.h> +long int lround(double x); +double fabs(double x); + +#include "or.h" +#include "buffers.h" +#include "circuitbuild.h" +#include "config.h" +#include "connection_edge.h" +#include "geoip.h" +#include "rendcommon.h" +#include "test.h" +#include "torgzip.h" +#include "mempool.h" +#include "memarea.h" +#include "onion.h" +#include "policies.h" +#include "rephist.h" +#include "routerparse.h" + +#ifdef USE_DMALLOC +#include <dmalloc.h> +#include <openssl/crypto.h> +#include "main.h" +#endif + +/** Set to true if any unit test has failed. Mostly, this is set by the macros + * in test.h */ +int have_failed = 0; + +/** Temporary directory (set up by setup_directory) under which we store all + * our files during testing. */ +static char temp_dir[256]; +static pid_t temp_dir_setup_in_pid = 0; + +/** Select and create the temporary directory we'll use to run our unit tests. + * Store it in <b>temp_dir</b>. Exit immediately if we can't create it. + * idempotent. */ +static void +setup_directory(void) +{ + static int is_setup = 0; + int r; + if (is_setup) return; + +#ifdef MS_WINDOWS + { + char buf[MAX_PATH]; + const char *tmp = buf; + /* If this fails, we're probably screwed anyway */ + if (!GetTempPath(sizeof(buf),buf)) + tmp = "c:\\windows\\temp"; + tor_snprintf(temp_dir, sizeof(temp_dir), + "%s\\tor_test_%d", tmp, (int)getpid()); + r = mkdir(temp_dir); + } +#else + tor_snprintf(temp_dir, sizeof(temp_dir), "/tmp/tor_test_%d", (int) getpid()); + r = mkdir(temp_dir, 0700); +#endif + if (r) { + fprintf(stderr, "Can't create directory %s:", temp_dir); + perror(""); + exit(1); + } + is_setup = 1; + temp_dir_setup_in_pid = getpid(); +} + +/** Return a filename relative to our testing temporary directory */ +const char * +get_fname(const char *name) +{ + static char buf[1024]; + setup_directory(); + if (!name) + return temp_dir; + tor_snprintf(buf,sizeof(buf),"%s/%s",temp_dir,name); + return buf; +} + +/** Remove all files stored under the temporary directory, and the directory + * itself. Called by atexit(). */ +static void +remove_directory(void) +{ + smartlist_t *elements; + if (getpid() != temp_dir_setup_in_pid) { + /* Only clean out the tempdir when the main process is exiting. */ + return; + } + elements = tor_listdir(temp_dir); + if (elements) { + SMARTLIST_FOREACH(elements, const char *, cp, + { + size_t len = strlen(cp)+strlen(temp_dir)+16; + char *tmp = tor_malloc(len); + tor_snprintf(tmp, len, "%s"PATH_SEPARATOR"%s", temp_dir, cp); + unlink(tmp); + tor_free(tmp); + }); + SMARTLIST_FOREACH(elements, char *, cp, tor_free(cp)); + smartlist_free(elements); + } + rmdir(temp_dir); +} + +/** Define this if unit tests spend too much time generating public keys*/ +#undef CACHE_GENERATED_KEYS + +static crypto_pk_env_t *pregen_keys[5] = {NULL, NULL, NULL, NULL, NULL}; +#define N_PREGEN_KEYS ((int)(sizeof(pregen_keys)/sizeof(pregen_keys[0]))) + +/** Generate and return a new keypair for use in unit tests. If we're using + * the key cache optimization, we might reuse keys: we only guarantee that + * keys made with distinct values for <b>idx</b> are different. The value of + * <b>idx</b> must be at least 0, and less than N_PREGEN_KEYS. */ +crypto_pk_env_t * +pk_generate(int idx) +{ +#ifdef CACHE_GENERATED_KEYS + tor_assert(idx < N_PREGEN_KEYS); + if (! pregen_keys[idx]) { + pregen_keys[idx] = crypto_new_pk_env(); + tor_assert(!crypto_pk_generate_key(pregen_keys[idx])); + } + return crypto_pk_dup_key(pregen_keys[idx]); +#else + crypto_pk_env_t *result; + (void) idx; + result = crypto_new_pk_env(); + tor_assert(!crypto_pk_generate_key(result)); + return result; +#endif +} + +/** Free all storage used for the cached key optimization. */ +static void +free_pregenerated_keys(void) +{ + unsigned idx; + for (idx = 0; idx < N_PREGEN_KEYS; ++idx) { + if (pregen_keys[idx]) { + crypto_free_pk_env(pregen_keys[idx]); + pregen_keys[idx] = NULL; + } + } +} + +/** Run unit tests for buffers.c */ +static void +test_buffers(void) +{ + char str[256]; + char str2[256]; + + buf_t *buf = NULL, *buf2 = NULL; + const char *cp; + + int j; + size_t r; + + /**** + * buf_new + ****/ + if (!(buf = buf_new())) + test_fail(); + + //test_eq(buf_capacity(buf), 4096); + test_eq(buf_datalen(buf), 0); + + /**** + * General pointer frobbing + */ + for (j=0;j<256;++j) { + str[j] = (char)j; + } + write_to_buf(str, 256, buf); + write_to_buf(str, 256, buf); + test_eq(buf_datalen(buf), 512); + fetch_from_buf(str2, 200, buf); + test_memeq(str, str2, 200); + test_eq(buf_datalen(buf), 312); + memset(str2, 0, sizeof(str2)); + + fetch_from_buf(str2, 256, buf); + test_memeq(str+200, str2, 56); + test_memeq(str, str2+56, 200); + test_eq(buf_datalen(buf), 56); + memset(str2, 0, sizeof(str2)); + /* Okay, now we should be 512 bytes into the 4096-byte buffer. If we add + * another 3584 bytes, we hit the end. */ + for (j=0;j<15;++j) { + write_to_buf(str, 256, buf); + } + assert_buf_ok(buf); + test_eq(buf_datalen(buf), 3896); + fetch_from_buf(str2, 56, buf); + test_eq(buf_datalen(buf), 3840); + test_memeq(str+200, str2, 56); + for (j=0;j<15;++j) { + memset(str2, 0, sizeof(str2)); + fetch_from_buf(str2, 256, buf); + test_memeq(str, str2, 256); + } + test_eq(buf_datalen(buf), 0); + buf_free(buf); + buf = NULL; + + /* Okay, now make sure growing can work. */ + buf = buf_new_with_capacity(16); + //test_eq(buf_capacity(buf), 16); + write_to_buf(str+1, 255, buf); + //test_eq(buf_capacity(buf), 256); + fetch_from_buf(str2, 254, buf); + test_memeq(str+1, str2, 254); + //test_eq(buf_capacity(buf), 256); + assert_buf_ok(buf); + write_to_buf(str, 32, buf); + //test_eq(buf_capacity(buf), 256); + assert_buf_ok(buf); + write_to_buf(str, 256, buf); + assert_buf_ok(buf); + //test_eq(buf_capacity(buf), 512); + test_eq(buf_datalen(buf), 33+256); + fetch_from_buf(str2, 33, buf); + test_eq(*str2, str[255]); + + test_memeq(str2+1, str, 32); + //test_eq(buf_capacity(buf), 512); + test_eq(buf_datalen(buf), 256); + fetch_from_buf(str2, 256, buf); + test_memeq(str, str2, 256); + + /* now try shrinking: case 1. */ + buf_free(buf); + buf = buf_new_with_capacity(33668); + for (j=0;j<67;++j) { + write_to_buf(str,255, buf); + } + //test_eq(buf_capacity(buf), 33668); + test_eq(buf_datalen(buf), 17085); + for (j=0; j < 40; ++j) { + fetch_from_buf(str2, 255,buf); + test_memeq(str2, str, 255); + } + + /* now try shrinking: case 2. */ + buf_free(buf); + buf = buf_new_with_capacity(33668); + for (j=0;j<67;++j) { + write_to_buf(str,255, buf); + } + for (j=0; j < 20; ++j) { + fetch_from_buf(str2, 255,buf); + test_memeq(str2, str, 255); + } + for (j=0;j<80;++j) { + write_to_buf(str,255, buf); + } + //test_eq(buf_capacity(buf),33668); + for (j=0; j < 120; ++j) { + fetch_from_buf(str2, 255,buf); + test_memeq(str2, str, 255); + } + + /* Move from buf to buf. */ + buf_free(buf); + buf = buf_new_with_capacity(4096); + buf2 = buf_new_with_capacity(4096); + for (j=0;j<100;++j) + write_to_buf(str, 255, buf); + test_eq(buf_datalen(buf), 25500); + for (j=0;j<100;++j) { + r = 10; + move_buf_to_buf(buf2, buf, &r); + test_eq(r, 0); + } + test_eq(buf_datalen(buf), 24500); + test_eq(buf_datalen(buf2), 1000); + for (j=0;j<3;++j) { + fetch_from_buf(str2, 255, buf2); + test_memeq(str2, str, 255); + } + r = 8192; /*big move*/ + move_buf_to_buf(buf2, buf, &r); + test_eq(r, 0); + r = 30000; /* incomplete move */ + move_buf_to_buf(buf2, buf, &r); + test_eq(r, 13692); + for (j=0;j<97;++j) { + fetch_from_buf(str2, 255, buf2); + test_memeq(str2, str, 255); + } + buf_free(buf); + buf_free(buf2); + buf = buf2 = NULL; + + buf = buf_new_with_capacity(5); + cp = "Testing. This is a moderately long Testing string."; + for (j = 0; cp[j]; j++) + write_to_buf(cp+j, 1, buf); + test_eq(0, buf_find_string_offset(buf, "Testing", 7)); + test_eq(1, buf_find_string_offset(buf, "esting", 6)); + test_eq(1, buf_find_string_offset(buf, "est", 3)); + test_eq(39, buf_find_string_offset(buf, "ing str", 7)); + test_eq(35, buf_find_string_offset(buf, "Testing str", 11)); + test_eq(32, buf_find_string_offset(buf, "ng ", 3)); + test_eq(43, buf_find_string_offset(buf, "string.", 7)); + test_eq(-1, buf_find_string_offset(buf, "shrdlu", 6)); + test_eq(-1, buf_find_string_offset(buf, "Testing thing", 13)); + test_eq(-1, buf_find_string_offset(buf, "ngx", 3)); + buf_free(buf); + buf = NULL; + + done: + if (buf) + buf_free(buf); + if (buf2) + buf_free(buf2); +} + +/** Run unit tests for the onion handshake code. */ +static void +test_onion_handshake(void) +{ + /* client-side */ + crypto_dh_env_t *c_dh = NULL; + char c_buf[ONIONSKIN_CHALLENGE_LEN]; + char c_keys[40]; + + /* server-side */ + char s_buf[ONIONSKIN_REPLY_LEN]; + char s_keys[40]; + + /* shared */ + crypto_pk_env_t *pk = NULL; + + pk = pk_generate(0); + + /* client handshake 1. */ + memset(c_buf, 0, ONIONSKIN_CHALLENGE_LEN); + test_assert(! onion_skin_create(pk, &c_dh, c_buf)); + + /* server handshake */ + memset(s_buf, 0, ONIONSKIN_REPLY_LEN); + memset(s_keys, 0, 40); + test_assert(! onion_skin_server_handshake(c_buf, pk, NULL, + s_buf, s_keys, 40)); + + /* client handshake 2 */ + memset(c_keys, 0, 40); + test_assert(! onion_skin_client_handshake(c_dh, s_buf, c_keys, 40)); + + if (memcmp(c_keys, s_keys, 40)) { + puts("Aiiiie"); + exit(1); + } + test_memeq(c_keys, s_keys, 40); + memset(s_buf, 0, 40); + test_memneq(c_keys, s_buf, 40); + + done: + if (c_dh) + crypto_dh_free(c_dh); + if (pk) + crypto_free_pk_env(pk); +} + +static void +test_circuit_timeout(void) +{ + /* Plan: + * 1. Generate 1000 samples + * 2. Estimate parameters + * 3. If difference, repeat + * 4. Save state + * 5. load state + * 6. Estimate parameters + * 7. compare differences + */ + circuit_build_times_t initial; + circuit_build_times_t estimate; + circuit_build_times_t final; + double timeout1, timeout2; + or_state_t state; + int i, runs; + double close_ms; + circuit_build_times_init(&initial); + circuit_build_times_init(&estimate); + circuit_build_times_init(&final); + + memset(&state, 0, sizeof(or_state_t)); + + circuitbuild_running_unit_tests(); +#define timeout0 (build_time_t)(30*1000.0) + initial.Xm = 3000; + circuit_build_times_initial_alpha(&initial, + CBT_DEFAULT_QUANTILE_CUTOFF/100.0, + timeout0); + close_ms = MAX(circuit_build_times_calculate_timeout(&initial, + CBT_DEFAULT_CLOSE_QUANTILE/100.0), + CBT_DEFAULT_TIMEOUT_INITIAL_VALUE); + do { + for (i=0; i < CBT_DEFAULT_MIN_CIRCUITS_TO_OBSERVE; i++) { + build_time_t sample = circuit_build_times_generate_sample(&initial,0,1); + + if (sample > close_ms) { + circuit_build_times_add_time(&estimate, CBT_BUILD_ABANDONED); + } else { + circuit_build_times_add_time(&estimate, sample); + } + } + circuit_build_times_update_alpha(&estimate); + timeout1 = circuit_build_times_calculate_timeout(&estimate, + CBT_DEFAULT_QUANTILE_CUTOFF/100.0); + circuit_build_times_set_timeout(&estimate); + log_notice(LD_CIRC, "Timeout1 is %lf, Xm is %d", timeout1, estimate.Xm); + /* 2% error */ + } while (fabs(circuit_build_times_cdf(&initial, timeout0) - + circuit_build_times_cdf(&initial, timeout1)) > 0.02); + + test_assert(estimate.total_build_times <= CBT_NCIRCUITS_TO_OBSERVE); + + circuit_build_times_update_state(&estimate, &state); + test_assert(circuit_build_times_parse_state(&final, &state) == 0); + + circuit_build_times_update_alpha(&final); + timeout2 = circuit_build_times_calculate_timeout(&final, + CBT_DEFAULT_QUANTILE_CUTOFF/100.0); + + circuit_build_times_set_timeout(&final); + log_notice(LD_CIRC, "Timeout2 is %lf, Xm is %d", timeout2, final.Xm); + + /* 5% here because some accuracy is lost due to histogram conversion */ + test_assert(fabs(circuit_build_times_cdf(&initial, timeout0) - + circuit_build_times_cdf(&initial, timeout2)) < 0.05); + + for (runs = 0; runs < 50; runs++) { + int build_times_idx = 0; + int total_build_times = 0; + + final.close_ms = final.timeout_ms = CBT_DEFAULT_TIMEOUT_INITIAL_VALUE; + estimate.close_ms = estimate.timeout_ms + = CBT_DEFAULT_TIMEOUT_INITIAL_VALUE; + + for (i = 0; i < CBT_DEFAULT_RECENT_CIRCUITS*2; i++) { + circuit_build_times_network_circ_success(&estimate); + circuit_build_times_add_time(&estimate, + circuit_build_times_generate_sample(&estimate, 0, + CBT_DEFAULT_QUANTILE_CUTOFF/100.0)); + + circuit_build_times_network_circ_success(&estimate); + circuit_build_times_add_time(&final, + circuit_build_times_generate_sample(&final, 0, + CBT_DEFAULT_QUANTILE_CUTOFF/100.0)); + } + + test_assert(!circuit_build_times_network_check_changed(&estimate)); + test_assert(!circuit_build_times_network_check_changed(&final)); + + /* Reset liveness to be non-live */ + final.liveness.network_last_live = 0; + estimate.liveness.network_last_live = 0; + + build_times_idx = estimate.build_times_idx; + total_build_times = estimate.total_build_times; + + test_assert(circuit_build_times_network_check_live(&estimate)); + test_assert(circuit_build_times_network_check_live(&final)); + + circuit_build_times_count_close(&estimate, 0, + (time_t)(approx_time()-estimate.close_ms/1000.0-1)); + circuit_build_times_count_close(&final, 0, + (time_t)(approx_time()-final.close_ms/1000.0-1)); + + test_assert(!circuit_build_times_network_check_live(&estimate)); + test_assert(!circuit_build_times_network_check_live(&final)); + + log_info(LD_CIRC, "idx: %d %d, tot: %d %d", + build_times_idx, estimate.build_times_idx, + total_build_times, estimate.total_build_times); + + /* Check rollback index. Should match top of loop. */ + test_assert(build_times_idx == estimate.build_times_idx); + // This can fail if estimate.total_build_times == 1000, because + // in that case, rewind actually causes us to lose timeouts + if (total_build_times != CBT_NCIRCUITS_TO_OBSERVE) + test_assert(total_build_times == estimate.total_build_times); + + /* Now simulate that the network has become live and we need + * a change */ + circuit_build_times_network_is_live(&estimate); + circuit_build_times_network_is_live(&final); + + for (i = 0; i < CBT_DEFAULT_MAX_RECENT_TIMEOUT_COUNT; i++) { + circuit_build_times_count_timeout(&estimate, 1); + + if (i < CBT_DEFAULT_MAX_RECENT_TIMEOUT_COUNT-1) { + circuit_build_times_count_timeout(&final, 1); + } + } + + test_assert(estimate.liveness.after_firsthop_idx == 0); + test_assert(final.liveness.after_firsthop_idx == + CBT_DEFAULT_MAX_RECENT_TIMEOUT_COUNT-1); + + test_assert(circuit_build_times_network_check_live(&estimate)); + test_assert(circuit_build_times_network_check_live(&final)); + + circuit_build_times_count_timeout(&final, 1); + } + + done: + return; +} + +/** Helper: Parse the exit policy string in <b>policy_str</b>, and make sure + * that policies_summarize() produces the string <b>expected_summary</b> from + * it. */ +static void +test_policy_summary_helper(const char *policy_str, + const char *expected_summary) +{ + config_line_t line; + smartlist_t *policy = smartlist_create(); + char *summary = NULL; + int r; + + line.key = (char*)"foo"; + line.value = (char *)policy_str; + line.next = NULL; + + r = policies_parse_exit_policy(&line, &policy, 0, NULL, 1); + test_eq(r, 0); + summary = policy_summarize(policy); + + test_assert(summary != NULL); + test_streq(summary, expected_summary); + + done: + tor_free(summary); + if (policy) + addr_policy_list_free(policy); +} + +/** Run unit tests for generating summary lines of exit policies */ +static void +test_policies(void) +{ + int i; + smartlist_t *policy = NULL, *policy2 = NULL, *policy3 = NULL, + *policy4 = NULL, *policy5 = NULL, *policy6 = NULL, + *policy7 = NULL; + addr_policy_t *p; + tor_addr_t tar; + config_line_t line; + smartlist_t *sm = NULL; + char *policy_str = NULL; + + policy = smartlist_create(); + + p = router_parse_addr_policy_item_from_string("reject 192.168.0.0/16:*",-1); + test_assert(p != NULL); + test_eq(ADDR_POLICY_REJECT, p->policy_type); + tor_addr_from_ipv4h(&tar, 0xc0a80000u); + test_eq(0, tor_addr_compare(&p->addr, &tar, CMP_EXACT)); + test_eq(16, p->maskbits); + test_eq(1, p->prt_min); + test_eq(65535, p->prt_max); + + smartlist_add(policy, p); + + test_assert(ADDR_POLICY_ACCEPTED == + compare_addr_to_addr_policy(0x01020304u, 2, policy)); + test_assert(ADDR_POLICY_PROBABLY_ACCEPTED == + compare_addr_to_addr_policy(0, 2, policy)); + test_assert(ADDR_POLICY_REJECTED == + compare_addr_to_addr_policy(0xc0a80102, 2, policy)); + + test_assert(0 == policies_parse_exit_policy(NULL, &policy2, 1, NULL, 1)); + test_assert(policy2); + + policy3 = smartlist_create(); + p = router_parse_addr_policy_item_from_string("reject *:*",-1); + test_assert(p != NULL); + smartlist_add(policy3, p); + p = router_parse_addr_policy_item_from_string("accept *:*",-1); + test_assert(p != NULL); + smartlist_add(policy3, p); + + policy4 = smartlist_create(); + p = router_parse_addr_policy_item_from_string("accept *:443",-1); + test_assert(p != NULL); + smartlist_add(policy4, p); + p = router_parse_addr_policy_item_from_string("accept *:443",-1); + test_assert(p != NULL); + smartlist_add(policy4, p); + + policy5 = smartlist_create(); + p = router_parse_addr_policy_item_from_string("reject 0.0.0.0/8:*",-1); + test_assert(p != NULL); + smartlist_add(policy5, p); + p = router_parse_addr_policy_item_from_string("reject 169.254.0.0/16:*",-1); + test_assert(p != NULL); + smartlist_add(policy5, p); + p = router_parse_addr_policy_item_from_string("reject 127.0.0.0/8:*",-1); + test_assert(p != NULL); + smartlist_add(policy5, p); + p = router_parse_addr_policy_item_from_string("reject 192.168.0.0/16:*",-1); + test_assert(p != NULL); + smartlist_add(policy5, p); + p = router_parse_addr_policy_item_from_string("reject 10.0.0.0/8:*",-1); + test_assert(p != NULL); + smartlist_add(policy5, p); + p = router_parse_addr_policy_item_from_string("reject 172.16.0.0/12:*",-1); + test_assert(p != NULL); + smartlist_add(policy5, p); + p = router_parse_addr_policy_item_from_string("reject 80.190.250.90:*",-1); + test_assert(p != NULL); + smartlist_add(policy5, p); + p = router_parse_addr_policy_item_from_string("reject *:1-65534",-1); + test_assert(p != NULL); + smartlist_add(policy5, p); + p = router_parse_addr_policy_item_from_string("reject *:65535",-1); + test_assert(p != NULL); + smartlist_add(policy5, p); + p = router_parse_addr_policy_item_from_string("accept *:1-65535",-1); + test_assert(p != NULL); + smartlist_add(policy5, p); + + policy6 = smartlist_create(); + p = router_parse_addr_policy_item_from_string("accept 43.3.0.0/9:*",-1); + test_assert(p != NULL); + smartlist_add(policy6, p); + + policy7 = smartlist_create(); + p = router_parse_addr_policy_item_from_string("accept 0.0.0.0/8:*",-1); + test_assert(p != NULL); + smartlist_add(policy7, p); + + test_assert(!exit_policy_is_general_exit(policy)); + test_assert(exit_policy_is_general_exit(policy2)); + test_assert(!exit_policy_is_general_exit(NULL)); + test_assert(!exit_policy_is_general_exit(policy3)); + test_assert(!exit_policy_is_general_exit(policy4)); + test_assert(!exit_policy_is_general_exit(policy5)); + test_assert(!exit_policy_is_general_exit(policy6)); + test_assert(!exit_policy_is_general_exit(policy7)); + + test_assert(cmp_addr_policies(policy, policy2)); + test_assert(cmp_addr_policies(policy, NULL)); + test_assert(!cmp_addr_policies(policy2, policy2)); + test_assert(!cmp_addr_policies(NULL, NULL)); + + test_assert(!policy_is_reject_star(policy2)); + test_assert(policy_is_reject_star(policy)); + test_assert(policy_is_reject_star(NULL)); + + addr_policy_list_free(policy); + policy = NULL; + + /* make sure compacting logic works. */ + policy = NULL; + line.key = (char*)"foo"; + line.value = (char*)"accept *:80,reject private:*,reject *:*"; + line.next = NULL; + test_assert(0 == policies_parse_exit_policy(&line, &policy, 0, NULL, 1)); + test_assert(policy); + //test_streq(policy->string, "accept *:80"); + //test_streq(policy->next->string, "reject *:*"); + test_eq(smartlist_len(policy), 2); + + /* test policy summaries */ + /* check if we properly ignore private IP addresses */ + test_policy_summary_helper("reject 192.168.0.0/16:*," + "reject 0.0.0.0/8:*," + "reject 10.0.0.0/8:*," + "accept *:10-30," + "accept *:90," + "reject *:*", + "accept 10-30,90"); + /* check all accept policies, and proper counting of rejects */ + test_policy_summary_helper("reject 11.0.0.0/9:80," + "reject 12.0.0.0/9:80," + "reject 13.0.0.0/9:80," + "reject 14.0.0.0/9:80," + "accept *:*", "accept 1-65535"); + test_policy_summary_helper("reject 11.0.0.0/9:80," + "reject 12.0.0.0/9:80," + "reject 13.0.0.0/9:80," + "reject 14.0.0.0/9:80," + "reject 15.0.0.0:81," + "accept *:*", "accept 1-65535"); + test_policy_summary_helper("reject 11.0.0.0/9:80," + "reject 12.0.0.0/9:80," + "reject 13.0.0.0/9:80," + "reject 14.0.0.0/9:80," + "reject 15.0.0.0:80," + "accept *:*", + "reject 80"); + /* no exits */ + test_policy_summary_helper("accept 11.0.0.0/9:80," + "reject *:*", + "reject 1-65535"); + /* port merging */ + test_policy_summary_helper("accept *:80," + "accept *:81," + "accept *:100-110," + "accept *:111," + "reject *:*", + "accept 80-81,100-111"); + /* border ports */ + test_policy_summary_helper("accept *:1," + "accept *:3," + "accept *:65535," + "reject *:*", + "accept 1,3,65535"); + /* holes */ + test_policy_summary_helper("accept *:1," + "accept *:3," + "accept *:5," + "accept *:7," + "reject *:*", + "accept 1,3,5,7"); + test_policy_summary_helper("reject *:1," + "reject *:3," + "reject *:5," + "reject *:7," + "accept *:*", + "reject 1,3,5,7"); + + /* truncation ports */ + sm = smartlist_create(); + for (i=1; i<2000; i+=2) { + char buf[POLICY_BUF_LEN]; + tor_snprintf(buf, sizeof(buf), "reject *:%d", i); + smartlist_add(sm, tor_strdup(buf)); + } + smartlist_add(sm, tor_strdup("accept *:*")); + policy_str = smartlist_join_strings(sm, ",", 0, NULL); + test_policy_summary_helper( policy_str, + "accept 2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40,42,44," + "46,48,50,52,54,56,58,60,62,64,66,68,70,72,74,76,78,80,82,84,86,88,90," + "92,94,96,98,100,102,104,106,108,110,112,114,116,118,120,122,124,126,128," + "130,132,134,136,138,140,142,144,146,148,150,152,154,156,158,160,162,164," + "166,168,170,172,174,176,178,180,182,184,186,188,190,192,194,196,198,200," + "202,204,206,208,210,212,214,216,218,220,222,224,226,228,230,232,234,236," + "238,240,242,244,246,248,250,252,254,256,258,260,262,264,266,268,270,272," + "274,276,278,280,282,284,286,288,290,292,294,296,298,300,302,304,306,308," + "310,312,314,316,318,320,322,324,326,328,330,332,334,336,338,340,342,344," + "346,348,350,352,354,356,358,360,362,364,366,368,370,372,374,376,378,380," + "382,384,386,388,390,392,394,396,398,400,402,404,406,408,410,412,414,416," + "418,420,422,424,426,428,430,432,434,436,438,440,442,444,446,448,450,452," + "454,456,458,460,462,464,466,468,470,472,474,476,478,480,482,484,486,488," + "490,492,494,496,498,500,502,504,506,508,510,512,514,516,518,520,522"); + + done: + addr_policy_list_free(policy); + addr_policy_list_free(policy2); + addr_policy_list_free(policy3); + addr_policy_list_free(policy4); + addr_policy_list_free(policy5); + addr_policy_list_free(policy6); + addr_policy_list_free(policy7); + tor_free(policy_str); + if (sm) { + SMARTLIST_FOREACH(sm, char *, s, tor_free(s)); + smartlist_free(sm); + } +} + +/** Run AES performance benchmarks. */ +static void +bench_aes(void) +{ + int len, i; + char *b1, *b2; + crypto_cipher_env_t *c; + struct timeval start, end; + const int iters = 100000; + uint64_t nsec; + c = crypto_new_cipher_env(); + crypto_cipher_generate_key(c); + crypto_cipher_encrypt_init_cipher(c); + for (len = 1; len <= 8192; len *= 2) { + b1 = tor_malloc_zero(len); + b2 = tor_malloc_zero(len); + tor_gettimeofday(&start); + for (i = 0; i < iters; ++i) { + crypto_cipher_encrypt(c, b1, b2, len); + } + tor_gettimeofday(&end); + tor_free(b1); + tor_free(b2); + nsec = (uint64_t) tv_udiff(&start,&end); + nsec *= 1000; + nsec /= (iters*len); + printf("%d bytes: "U64_FORMAT" nsec per byte\n", len, + U64_PRINTF_ARG(nsec)); + } + crypto_free_cipher_env(c); +} + +/** Run digestmap_t performance benchmarks. */ +static void +bench_dmap(void) +{ + smartlist_t *sl = smartlist_create(); + smartlist_t *sl2 = smartlist_create(); + struct timeval start, end, pt2, pt3, pt4; + const int iters = 10000; + const int elts = 4000; + const int fpostests = 1000000; + char d[20]; + int i,n=0, fp = 0; + digestmap_t *dm = digestmap_new(); + digestset_t *ds = digestset_new(elts); + + for (i = 0; i < elts; ++i) { + crypto_rand(d, 20); + smartlist_add(sl, tor_memdup(d, 20)); + } + for (i = 0; i < elts; ++i) { + crypto_rand(d, 20); + smartlist_add(sl2, tor_memdup(d, 20)); + } + printf("nbits=%d\n", ds->mask+1); + + tor_gettimeofday(&start); + for (i = 0; i < iters; ++i) { + SMARTLIST_FOREACH(sl, const char *, cp, digestmap_set(dm, cp, (void*)1)); + } + tor_gettimeofday(&pt2); + for (i = 0; i < iters; ++i) { + SMARTLIST_FOREACH(sl, const char *, cp, digestmap_get(dm, cp)); + SMARTLIST_FOREACH(sl2, const char *, cp, digestmap_get(dm, cp)); + } + tor_gettimeofday(&pt3); + for (i = 0; i < iters; ++i) { + SMARTLIST_FOREACH(sl, const char *, cp, digestset_add(ds, cp)); + } + tor_gettimeofday(&pt4); + for (i = 0; i < iters; ++i) { + SMARTLIST_FOREACH(sl, const char *, cp, n += digestset_isin(ds, cp)); + SMARTLIST_FOREACH(sl2, const char *, cp, n += digestset_isin(ds, cp)); + } + tor_gettimeofday(&end); + + for (i = 0; i < fpostests; ++i) { + crypto_rand(d, 20); + if (digestset_isin(ds, d)) ++fp; + } + + printf("%ld\n",(unsigned long)tv_udiff(&start, &pt2)); + printf("%ld\n",(unsigned long)tv_udiff(&pt2, &pt3)); + printf("%ld\n",(unsigned long)tv_udiff(&pt3, &pt4)); + printf("%ld\n",(unsigned long)tv_udiff(&pt4, &end)); + printf("-- %d\n", n); + printf("++ %f\n", fp/(double)fpostests); + digestmap_free(dm, NULL); + digestset_free(ds); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + SMARTLIST_FOREACH(sl2, char *, cp, tor_free(cp)); + smartlist_free(sl); + smartlist_free(sl2); +} + +/** Test encoding and parsing of rendezvous service descriptors. */ +static void +test_rend_fns(void) +{ + rend_service_descriptor_t *generated = NULL, *parsed = NULL; + char service_id[DIGEST_LEN]; + char service_id_base32[REND_SERVICE_ID_LEN_BASE32+1]; + const char *next_desc; + smartlist_t *descs = smartlist_create(); + char computed_desc_id[DIGEST_LEN]; + char parsed_desc_id[DIGEST_LEN]; + crypto_pk_env_t *pk1 = NULL, *pk2 = NULL; + time_t now; + char *intro_points_encrypted = NULL; + size_t intro_points_size; + size_t encoded_size; + int i; + char address1[] = "fooaddress.onion"; + char address2[] = "aaaaaaaaaaaaaaaa.onion"; + char address3[] = "fooaddress.exit"; + char address4[] = "www.torproject.org"; + + test_assert(BAD_HOSTNAME == parse_extended_hostname(address1, 1)); + test_assert(ONION_HOSTNAME == parse_extended_hostname(address2, 1)); + test_assert(EXIT_HOSTNAME == parse_extended_hostname(address3, 1)); + test_assert(NORMAL_HOSTNAME == parse_extended_hostname(address4, 1)); + + pk1 = pk_generate(0); + pk2 = pk_generate(1); + generated = tor_malloc_zero(sizeof(rend_service_descriptor_t)); + generated->pk = crypto_pk_dup_key(pk1); + crypto_pk_get_digest(generated->pk, service_id); + base32_encode(service_id_base32, REND_SERVICE_ID_LEN_BASE32+1, + service_id, REND_SERVICE_ID_LEN); + now = time(NULL); + generated->timestamp = now; + generated->version = 2; + generated->protocols = 42; + generated->intro_nodes = smartlist_create(); + + for (i = 0; i < 3; i++) { + rend_intro_point_t *intro = tor_malloc_zero(sizeof(rend_intro_point_t)); + crypto_pk_env_t *okey = pk_generate(2 + i); + intro->extend_info = tor_malloc_zero(sizeof(extend_info_t)); + intro->extend_info->onion_key = okey; + crypto_pk_get_digest(intro->extend_info->onion_key, + intro->extend_info->identity_digest); + //crypto_rand(info->identity_digest, DIGEST_LEN); /* Would this work? */ + intro->extend_info->nickname[0] = '$'; + base16_encode(intro->extend_info->nickname + 1, + sizeof(intro->extend_info->nickname) - 1, + intro->extend_info->identity_digest, DIGEST_LEN); + /* Does not cover all IP addresses. */ + tor_addr_from_ipv4h(&intro->extend_info->addr, crypto_rand_int(65536)); + intro->extend_info->port = 1 + crypto_rand_int(65535); + intro->intro_key = crypto_pk_dup_key(pk2); + smartlist_add(generated->intro_nodes, intro); + } + test_assert(rend_encode_v2_descriptors(descs, generated, now, 0, + REND_NO_AUTH, NULL, NULL) > 0); + test_assert(rend_compute_v2_desc_id(computed_desc_id, service_id_base32, + NULL, now, 0) == 0); + test_memeq(((rend_encoded_v2_service_descriptor_t *) + smartlist_get(descs, 0))->desc_id, computed_desc_id, DIGEST_LEN); + test_assert(rend_parse_v2_service_descriptor(&parsed, parsed_desc_id, + &intro_points_encrypted, + &intro_points_size, + &encoded_size, + &next_desc, + ((rend_encoded_v2_service_descriptor_t *) + smartlist_get(descs, 0))->desc_str) == 0); + test_assert(parsed); + test_memeq(((rend_encoded_v2_service_descriptor_t *) + smartlist_get(descs, 0))->desc_id, parsed_desc_id, DIGEST_LEN); + test_eq(rend_parse_introduction_points(parsed, intro_points_encrypted, + intro_points_size), 3); + test_assert(!crypto_pk_cmp_keys(generated->pk, parsed->pk)); + test_eq(parsed->timestamp, now); + test_eq(parsed->version, 2); + test_eq(parsed->protocols, 42); + test_eq(smartlist_len(parsed->intro_nodes), 3); + for (i = 0; i < smartlist_len(parsed->intro_nodes); i++) { + rend_intro_point_t *par_intro = smartlist_get(parsed->intro_nodes, i), + *gen_intro = smartlist_get(generated->intro_nodes, i); + extend_info_t *par_info = par_intro->extend_info; + extend_info_t *gen_info = gen_intro->extend_info; + test_assert(!crypto_pk_cmp_keys(gen_info->onion_key, par_info->onion_key)); + test_memeq(gen_info->identity_digest, par_info->identity_digest, + DIGEST_LEN); + test_streq(gen_info->nickname, par_info->nickname); + test_assert(tor_addr_eq(&gen_info->addr, &par_info->addr)); + test_eq(gen_info->port, par_info->port); + } + + rend_service_descriptor_free(parsed); + rend_service_descriptor_free(generated); + parsed = generated = NULL; + + done: + if (descs) { + for (i = 0; i < smartlist_len(descs); i++) + rend_encoded_v2_service_descriptor_free(smartlist_get(descs, i)); + smartlist_free(descs); + } + if (parsed) + rend_service_descriptor_free(parsed); + if (generated) + rend_service_descriptor_free(generated); + if (pk1) + crypto_free_pk_env(pk1); + if (pk2) + crypto_free_pk_env(pk2); + tor_free(intro_points_encrypted); +} + +/** Run unit tests for GeoIP code. */ +static void +test_geoip(void) +{ + int i, j; + time_t now = time(NULL); + char *s = NULL; + + /* Populate the DB a bit. Add these in order, since we can't do the final + * 'sort' step. These aren't very good IP addresses, but they're perfectly + * fine uint32_t values. */ + test_eq(0, geoip_parse_entry("10,50,AB")); + test_eq(0, geoip_parse_entry("52,90,XY")); + test_eq(0, geoip_parse_entry("95,100,AB")); + test_eq(0, geoip_parse_entry("\"105\",\"140\",\"ZZ\"")); + test_eq(0, geoip_parse_entry("\"150\",\"190\",\"XY\"")); + test_eq(0, geoip_parse_entry("\"200\",\"250\",\"AB\"")); + + /* We should have 4 countries: ??, ab, xy, zz. */ + test_eq(4, geoip_get_n_countries()); + /* Make sure that country ID actually works. */ +#define NAMEFOR(x) geoip_get_country_name(geoip_get_country_by_ip(x)) + test_streq("??", NAMEFOR(3)); + test_eq(0, geoip_get_country_by_ip(3)); + test_streq("ab", NAMEFOR(32)); + test_streq("??", NAMEFOR(5)); + test_streq("??", NAMEFOR(51)); + test_streq("xy", NAMEFOR(150)); + test_streq("xy", NAMEFOR(190)); + test_streq("??", NAMEFOR(2000)); +#undef NAMEFOR + + get_options()->BridgeRelay = 1; + get_options()->BridgeRecordUsageByCountry = 1; + /* Put 9 observations in AB... */ + for (i=32; i < 40; ++i) + geoip_note_client_seen(GEOIP_CLIENT_CONNECT, i, now-7200); + geoip_note_client_seen(GEOIP_CLIENT_CONNECT, 225, now-7200); + /* and 3 observations in XY, several times. */ + for (j=0; j < 10; ++j) + for (i=52; i < 55; ++i) + geoip_note_client_seen(GEOIP_CLIENT_CONNECT, i, now-3600); + /* and 17 observations in ZZ... */ + for (i=110; i < 127; ++i) + geoip_note_client_seen(GEOIP_CLIENT_CONNECT, i, now); + s = geoip_get_client_history(GEOIP_CLIENT_CONNECT); + test_assert(s); + test_streq("zz=24,ab=16,xy=8", s); + tor_free(s); + + /* Now clear out all the AB observations. */ + geoip_remove_old_clients(now-6000); + s = geoip_get_client_history(GEOIP_CLIENT_CONNECT); + test_assert(s); + test_streq("zz=24,xy=8", s); + + done: + tor_free(s); +} + +/** Run unit tests for stats code. */ +static void +test_stats(void) +{ + time_t now = 1281533250; /* 2010-08-11 13:27:30 UTC */ + char *s = NULL; + int i; + + /* We shouldn't collect exit stats without initializing them. */ + rep_hist_note_exit_stream_opened(80); + rep_hist_note_exit_bytes(80, 100, 10000); + s = rep_hist_format_exit_stats(now + 86400); + test_assert(!s); + + /* Initialize stats, note some streams and bytes, and generate history + * string. */ + rep_hist_exit_stats_init(now); + rep_hist_note_exit_stream_opened(80); + rep_hist_note_exit_bytes(80, 100, 10000); + rep_hist_note_exit_stream_opened(443); + rep_hist_note_exit_bytes(443, 100, 10000); + rep_hist_note_exit_bytes(443, 100, 10000); + s = rep_hist_format_exit_stats(now + 86400); + test_streq("exit-stats-end 2010-08-12 13:27:30 (86400 s)\n" + "exit-kibibytes-written 80=1,443=1,other=0\n" + "exit-kibibytes-read 80=10,443=20,other=0\n" + "exit-streams-opened 80=4,443=4,other=0\n", s); + tor_free(s); + + /* Add a few bytes on 10 more ports and ensure that only the top 10 + * ports are contained in the history string. */ + for (i = 50; i < 60; i++) { + rep_hist_note_exit_bytes(i, i, i); + rep_hist_note_exit_stream_opened(i); + } + s = rep_hist_format_exit_stats(now + 86400); + test_streq("exit-stats-end 2010-08-12 13:27:30 (86400 s)\n" + "exit-kibibytes-written 52=1,53=1,54=1,55=1,56=1,57=1,58=1," + "59=1,80=1,443=1,other=1\n" + "exit-kibibytes-read 52=1,53=1,54=1,55=1,56=1,57=1,58=1," + "59=1,80=10,443=20,other=1\n" + "exit-streams-opened 52=4,53=4,54=4,55=4,56=4,57=4,58=4," + "59=4,80=4,443=4,other=4\n", s); + tor_free(s); + + /* Stop collecting stats, add some bytes, and ensure we don't generate + * a history string. */ + rep_hist_exit_stats_term(); + rep_hist_note_exit_bytes(80, 100, 10000); + s = rep_hist_format_exit_stats(now + 86400); + test_assert(!s); + + /* Re-start stats, add some bytes, reset stats, and see what history we + * get when observing no streams or bytes at all. */ + rep_hist_exit_stats_init(now); + rep_hist_note_exit_stream_opened(80); + rep_hist_note_exit_bytes(80, 100, 10000); + rep_hist_reset_exit_stats(now); + s = rep_hist_format_exit_stats(now + 86400); + test_streq("exit-stats-end 2010-08-12 13:27:30 (86400 s)\n" + "exit-kibibytes-written other=0\n" + "exit-kibibytes-read other=0\n" + "exit-streams-opened other=0\n", s); + + done: + tor_free(s); +} + +static void * +legacy_test_setup(const struct testcase_t *testcase) +{ + return testcase->setup_data; +} + +void +legacy_test_helper(void *data) +{ + void (*fn)(void) = data; + fn(); +} + +static int +legacy_test_cleanup(const struct testcase_t *testcase, void *ptr) +{ + (void)ptr; + (void)testcase; + return 1; +} + +const struct testcase_setup_t legacy_setup = { + legacy_test_setup, legacy_test_cleanup +}; + +#define ENT(name) \ + { #name, legacy_test_helper, 0, &legacy_setup, test_ ## name } +#define SUBENT(group, name) \ + { #group "_" #name, legacy_test_helper, 0, &legacy_setup, \ + test_ ## group ## _ ## name } +#define DISABLED(name) \ + { #name, legacy_test_helper, TT_SKIP, &legacy_setup, name } +#define FORK(name) \ + { #name, legacy_test_helper, TT_FORK, &legacy_setup, test_ ## name } + +static struct testcase_t test_array[] = { + ENT(buffers), + ENT(onion_handshake), + ENT(circuit_timeout), + ENT(policies), + ENT(rend_fns), + ENT(geoip), + FORK(stats), + + DISABLED(bench_aes), + DISABLED(bench_dmap), + END_OF_TESTCASES +}; + +extern struct testcase_t addr_tests[]; +extern struct testcase_t crypto_tests[]; +extern struct testcase_t container_tests[]; +extern struct testcase_t util_tests[]; +extern struct testcase_t dir_tests[]; + +static struct testgroup_t testgroups[] = { + { "", test_array }, + { "addr/", addr_tests }, + { "crypto/", crypto_tests }, + { "container/", container_tests }, + { "util/", util_tests }, + { "dir/", dir_tests }, + END_OF_GROUPS +}; + +/** Main entry point for unit test code: parse the command line, and run + * some unit tests. */ +int +main(int c, const char **v) +{ + or_options_t *options; + char *errmsg = NULL; + int i, i_out; + int loglevel = LOG_ERR; + +#ifdef USE_DMALLOC + { + int r = CRYPTO_set_mem_ex_functions(_tor_malloc, _tor_realloc, _tor_free); + tor_assert(r); + } +#endif + + update_approx_time(time(NULL)); + options = options_new(); + tor_threads_init(); + init_logging(); + + for (i_out = i = 1; i < c; ++i) { + if (!strcmp(v[i], "--warn")) { + loglevel = LOG_WARN; + } else if (!strcmp(v[i], "--notice")) { + loglevel = LOG_NOTICE; + } else if (!strcmp(v[i], "--info")) { + loglevel = LOG_INFO; + } else if (!strcmp(v[i], "--debug")) { + loglevel = LOG_DEBUG; + } else { + v[i_out++] = v[i]; + } + } + c = i_out; + + { + log_severity_list_t s; + memset(&s, 0, sizeof(s)); + set_log_severity_config(loglevel, LOG_ERR, &s); + add_stream_log(&s, "", fileno(stdout)); + } + + options->command = CMD_RUN_UNITTESTS; + crypto_global_init(0, NULL, NULL); + rep_hist_init(); + network_init(); + setup_directory(); + options_init(options); + options->DataDirectory = tor_strdup(temp_dir); + options->EntryStatistics = 1; + if (set_options(options, &errmsg) < 0) { + printf("Failed to set initial options: %s\n", errmsg); + tor_free(errmsg); + return 1; + } + + crypto_seed_rng(1); + + atexit(remove_directory); + + have_failed = (tinytest_main(c, v, testgroups) != 0); + + free_pregenerated_keys(); +#ifdef USE_DMALLOC + tor_free_all(0); + dmalloc_log_unfreed(); +#endif + + if (have_failed) + return 1; + else + return 0; +} + diff --git a/src/test/test.h b/src/test/test.h new file mode 100644 index 0000000000..f7ae46ce6d --- /dev/null +++ b/src/test/test.h @@ -0,0 +1,75 @@ +/* Copyright (c) 2001-2003, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef _TOR_TEST_H +#define _TOR_TEST_H + +/** + * \file test.h + * \brief Macros and functions used by unit tests. + */ + +#include "compat.h" +#include "tinytest.h" +#define TT_EXIT_TEST_FUNCTION STMT_BEGIN goto done; STMT_END +#include "tinytest_macros.h" + +#ifdef __GNUC__ +#define PRETTY_FUNCTION __PRETTY_FUNCTION__ +#else +#define PRETTY_FUNCTION "" +#endif + +#define test_fail_msg(msg) TT_DIE((msg)) + +#define test_fail() test_fail_msg("Assertion failed.") + +#define test_assert(expr) tt_assert(expr) + +#define test_eq(expr1, expr2) tt_int_op((expr1), ==, (expr2)) +#define test_eq_ptr(expr1, expr2) tt_ptr_op((expr1), ==, (expr2)) +#define test_neq(expr1, expr2) tt_int_op((expr1), !=, (expr2)) +#define test_neq_ptr(expr1, expr2) tt_ptr_op((expr1), !=, (expr2)) +#define test_streq(expr1, expr2) tt_str_op((expr1), ==, (expr2)) +#define test_strneq(expr1, expr2) tt_str_op((expr1), !=, (expr2)) +#define test_streq(expr1, expr2) tt_str_op((expr1), ==, (expr2)) + +#define test_mem_op(expr1, op, expr2, len) \ + tt_assert_test_fmt_type(expr1,expr2,#expr1" "#op" "#expr2, \ + const char *, \ + (memcmp(_val1, _val2, len) op 0), \ + char *, "%s", \ + { size_t printlen = (len)*2+1; \ + _print = tor_malloc(printlen); \ + base16_encode(_print, printlen, _value, \ + (len)); }, \ + { tor_free(_print); } \ + ); + +#define test_memeq(expr1, expr2, len) test_mem_op((expr1), ==, (expr2), len) +#define test_memneq(expr1, expr2, len) test_mem_op((expr1), !=, (expr2), len) + +/* As test_mem_op, but decodes 'hex' before comparing. There must be a + * local char* variable called mem_op_hex_tmp for this to work. */ +#define test_mem_op_hex(expr1, op, hex) \ + STMT_BEGIN \ + size_t length = strlen(hex); \ + tor_free(mem_op_hex_tmp); \ + mem_op_hex_tmp = tor_malloc(length/2); \ + tor_assert((length&1)==0); \ + base16_decode(mem_op_hex_tmp, length/2, hex, length); \ + test_mem_op(expr1, op, mem_op_hex_tmp, length/2); \ + STMT_END + +#define test_memeq_hex(expr1, hex) test_mem_op_hex(expr1, ==, hex) + +const char *get_fname(const char *name); +crypto_pk_env_t *pk_generate(int idx); + +void legacy_test_helper(void *data); +extern const struct testcase_setup_t legacy_setup; + +#endif + diff --git a/src/test/test_addr.c b/src/test/test_addr.c new file mode 100644 index 0000000000..6db4ee2483 --- /dev/null +++ b/src/test/test_addr.c @@ -0,0 +1,497 @@ +/* Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "orconfig.h" +#include "or.h" +#include "test.h" + +static void +test_addr_basic(void) +{ + uint32_t u32; + uint16_t u16; + char *cp; + + /* Test parse_addr_port */ + cp = NULL; u32 = 3; u16 = 3; + test_assert(!parse_addr_port(LOG_WARN, "1.2.3.4", &cp, &u32, &u16)); + test_streq(cp, "1.2.3.4"); + test_eq(u32, 0x01020304u); + test_eq(u16, 0); + tor_free(cp); + test_assert(!parse_addr_port(LOG_WARN, "4.3.2.1:99", &cp, &u32, &u16)); + test_streq(cp, "4.3.2.1"); + test_eq(u32, 0x04030201u); + test_eq(u16, 99); + tor_free(cp); + test_assert(!parse_addr_port(LOG_WARN, "nonexistent.address:4040", + &cp, NULL, &u16)); + test_streq(cp, "nonexistent.address"); + test_eq(u16, 4040); + tor_free(cp); + test_assert(!parse_addr_port(LOG_WARN, "localhost:9999", &cp, &u32, &u16)); + test_streq(cp, "localhost"); + test_eq(u32, 0x7f000001u); + test_eq(u16, 9999); + tor_free(cp); + u32 = 3; + test_assert(!parse_addr_port(LOG_WARN, "localhost", NULL, &u32, &u16)); + test_eq(cp, NULL); + test_eq(u32, 0x7f000001u); + test_eq(u16, 0); + tor_free(cp); + test_eq(0, addr_mask_get_bits(0x0u)); + test_eq(32, addr_mask_get_bits(0xFFFFFFFFu)); + test_eq(16, addr_mask_get_bits(0xFFFF0000u)); + test_eq(31, addr_mask_get_bits(0xFFFFFFFEu)); + test_eq(1, addr_mask_get_bits(0x80000000u)); + + /* Test inet_ntop */ + { + char tmpbuf[TOR_ADDR_BUF_LEN]; + const char *ip = "176.192.208.224"; + struct in_addr in; + tor_inet_pton(AF_INET, ip, &in); + tor_inet_ntop(AF_INET, &in, tmpbuf, sizeof(tmpbuf)); + test_streq(tmpbuf, ip); + } + + done: + ; +} + +#define _test_op_ip6(a,op,b,e1,e2) \ + STMT_BEGIN \ + tt_assert_test_fmt_type(a,b,e1" "#op" "e2,struct in6_addr*, \ + (memcmp(_val1->s6_addr, _val2->s6_addr, 16) op 0), \ + char *, "%s", \ + { int i; char *cp; \ + cp = _print = tor_malloc(64); \ + for (i=0;i<16;++i) { \ + tor_snprintf(cp, 3,"%02x", (unsigned)_value->s6_addr[i]);\ + cp += 2; \ + if (i != 15) *cp++ = ':'; \ + } \ + }, { tor_free(_print); } \ + ); \ + STMT_END + +/** Helper: Assert that two strings both decode as IPv6 addresses with + * tor_inet_pton(), and both decode to the same address. */ +#define test_pton6_same(a,b) STMT_BEGIN \ + test_eq(tor_inet_pton(AF_INET6, a, &a1), 1); \ + test_eq(tor_inet_pton(AF_INET6, b, &a2), 1); \ + _test_op_ip6(&a1,==,&a2,#a,#b); \ + STMT_END + +/** Helper: Assert that <b>a</b> is recognized as a bad IPv6 address by + * tor_inet_pton(). */ +#define test_pton6_bad(a) \ + test_eq(0, tor_inet_pton(AF_INET6, a, &a1)) + +/** Helper: assert that <b>a</b>, when parsed by tor_inet_pton() and displayed + * with tor_inet_ntop(), yields <b>b</b>. Also assert that <b>b</b> parses to + * the same value as <b>a</b>. */ +#define test_ntop6_reduces(a,b) STMT_BEGIN \ + test_eq(tor_inet_pton(AF_INET6, a, &a1), 1); \ + test_streq(tor_inet_ntop(AF_INET6, &a1, buf, sizeof(buf)), b); \ + test_eq(tor_inet_pton(AF_INET6, b, &a2), 1); \ + _test_op_ip6(&a1, ==, &a2, a, b); \ + STMT_END + +/** Helper: assert that <b>a</b> parses by tor_inet_pton() into a address that + * passes tor_addr_is_internal() with <b>for_listening</b>. */ +#define test_internal_ip(a,for_listening) STMT_BEGIN \ + test_eq(tor_inet_pton(AF_INET6, a, &t1.addr.in6_addr), 1); \ + t1.family = AF_INET6; \ + if (!tor_addr_is_internal(&t1, for_listening)) \ + test_fail_msg( a "was not internal."); \ + STMT_END + +/** Helper: assert that <b>a</b> parses by tor_inet_pton() into a address that + * does not pass tor_addr_is_internal() with <b>for_listening</b>. */ +#define test_external_ip(a,for_listening) STMT_BEGIN \ + test_eq(tor_inet_pton(AF_INET6, a, &t1.addr.in6_addr), 1); \ + t1.family = AF_INET6; \ + if (tor_addr_is_internal(&t1, for_listening)) \ + test_fail_msg(a "was not external."); \ + STMT_END + +/** Helper: Assert that <b>a</b> and <b>b</b>, when parsed by + * tor_inet_pton(), give addresses that compare in the order defined by + * <b>op</b> with tor_addr_compare(). */ +#define test_addr_compare(a, op, b) STMT_BEGIN \ + test_eq(tor_inet_pton(AF_INET6, a, &t1.addr.in6_addr), 1); \ + test_eq(tor_inet_pton(AF_INET6, b, &t2.addr.in6_addr), 1); \ + t1.family = t2.family = AF_INET6; \ + r = tor_addr_compare(&t1,&t2,CMP_SEMANTIC); \ + if (!(r op 0)) \ + test_fail_msg("failed: tor_addr_compare("a","b") "#op" 0"); \ + STMT_END + +/** Helper: Assert that <b>a</b> and <b>b</b>, when parsed by + * tor_inet_pton(), give addresses that compare in the order defined by + * <b>op</b> with tor_addr_compare_masked() with <b>m</b> masked. */ +#define test_addr_compare_masked(a, op, b, m) STMT_BEGIN \ + test_eq(tor_inet_pton(AF_INET6, a, &t1.addr.in6_addr), 1); \ + test_eq(tor_inet_pton(AF_INET6, b, &t2.addr.in6_addr), 1); \ + t1.family = t2.family = AF_INET6; \ + r = tor_addr_compare_masked(&t1,&t2,m,CMP_SEMANTIC); \ + if (!(r op 0)) \ + test_fail_msg("failed: tor_addr_compare_masked("a","b","#m") "#op" 0"); \ + STMT_END + +/** Helper: assert that <b>xx</b> is parseable as a masked IPv6 address with + * ports by tor_parse_mask_addr_ports(), with family <b>f</b>, IP address + * as 4 32-bit words <b>ip1...ip4</b>, mask bits as <b>mm</b>, and port range + * as <b>pt1..pt2</b>. */ +#define test_addr_mask_ports_parse(xx, f, ip1, ip2, ip3, ip4, mm, pt1, pt2) \ + STMT_BEGIN \ + test_eq(tor_addr_parse_mask_ports(xx, &t1, &mask, &port1, &port2), f); \ + p1=tor_inet_ntop(AF_INET6, &t1.addr.in6_addr, bug, sizeof(bug)); \ + test_eq(htonl(ip1), tor_addr_to_in6_addr32(&t1)[0]); \ + test_eq(htonl(ip2), tor_addr_to_in6_addr32(&t1)[1]); \ + test_eq(htonl(ip3), tor_addr_to_in6_addr32(&t1)[2]); \ + test_eq(htonl(ip4), tor_addr_to_in6_addr32(&t1)[3]); \ + test_eq(mask, mm); \ + test_eq(port1, pt1); \ + test_eq(port2, pt2); \ + STMT_END + +/** Run unit tests for IPv6 encoding/decoding/manipulation functions. */ +static void +test_addr_ip6_helpers(void) +{ + char buf[TOR_ADDR_BUF_LEN], bug[TOR_ADDR_BUF_LEN]; + struct in6_addr a1, a2; + tor_addr_t t1, t2; + int r, i; + uint16_t port1, port2; + maskbits_t mask; + const char *p1; + struct sockaddr_storage sa_storage; + struct sockaddr_in *sin; + struct sockaddr_in6 *sin6; + + // struct in_addr b1, b2; + /* Test tor_inet_ntop and tor_inet_pton: IPv6 */ + + /* ==== Converting to and from sockaddr_t. */ + sin = (struct sockaddr_in *)&sa_storage; + sin->sin_family = AF_INET; + sin->sin_port = 9090; + sin->sin_addr.s_addr = htonl(0x7f7f0102); /*127.127.1.2*/ + tor_addr_from_sockaddr(&t1, (struct sockaddr *)sin, NULL); + test_eq(tor_addr_family(&t1), AF_INET); + test_eq(tor_addr_to_ipv4h(&t1), 0x7f7f0102); + + memset(&sa_storage, 0, sizeof(sa_storage)); + test_eq(sizeof(struct sockaddr_in), + tor_addr_to_sockaddr(&t1, 1234, (struct sockaddr *)&sa_storage, + sizeof(sa_storage))); + test_eq(1234, ntohs(sin->sin_port)); + test_eq(0x7f7f0102, ntohl(sin->sin_addr.s_addr)); + + memset(&sa_storage, 0, sizeof(sa_storage)); + sin6 = (struct sockaddr_in6 *)&sa_storage; + sin6->sin6_family = AF_INET6; + sin6->sin6_port = htons(7070); + sin6->sin6_addr.s6_addr[0] = 128; + tor_addr_from_sockaddr(&t1, (struct sockaddr *)sin6, NULL); + test_eq(tor_addr_family(&t1), AF_INET6); + p1 = tor_addr_to_str(buf, &t1, sizeof(buf), 0); + test_streq(p1, "8000::"); + + memset(&sa_storage, 0, sizeof(sa_storage)); + test_eq(sizeof(struct sockaddr_in6), + tor_addr_to_sockaddr(&t1, 9999, (struct sockaddr *)&sa_storage, + sizeof(sa_storage))); + test_eq(AF_INET6, sin6->sin6_family); + test_eq(9999, ntohs(sin6->sin6_port)); + test_eq(0x80000000, ntohl(S6_ADDR32(sin6->sin6_addr)[0])); + + /* ==== tor_addr_lookup: static cases. (Can't test dns without knowing we + * have a good resolver. */ + test_eq(0, tor_addr_lookup("127.128.129.130", AF_UNSPEC, &t1)); + test_eq(AF_INET, tor_addr_family(&t1)); + test_eq(tor_addr_to_ipv4h(&t1), 0x7f808182); + + test_eq(0, tor_addr_lookup("9000::5", AF_UNSPEC, &t1)); + test_eq(AF_INET6, tor_addr_family(&t1)); + test_eq(0x90, tor_addr_to_in6_addr8(&t1)[0]); + test_assert(tor_mem_is_zero((char*)tor_addr_to_in6_addr8(&t1)+1, 14)); + test_eq(0x05, tor_addr_to_in6_addr8(&t1)[15]); + + /* === Test pton: valid af_inet6 */ + /* Simple, valid parsing. */ + r = tor_inet_pton(AF_INET6, + "0102:0304:0506:0708:090A:0B0C:0D0E:0F10", &a1); + test_assert(r==1); + for (i=0;i<16;++i) { test_eq(i+1, (int)a1.s6_addr[i]); } + /* ipv4 ending. */ + test_pton6_same("0102:0304:0506:0708:090A:0B0C:0D0E:0F10", + "0102:0304:0506:0708:090A:0B0C:13.14.15.16"); + /* shortened words. */ + test_pton6_same("0001:0099:BEEF:0000:0123:FFFF:0001:0001", + "1:99:BEEF:0:0123:FFFF:1:1"); + /* zeros at the beginning */ + test_pton6_same("0000:0000:0000:0000:0009:C0A8:0001:0001", + "::9:c0a8:1:1"); + test_pton6_same("0000:0000:0000:0000:0009:C0A8:0001:0001", + "::9:c0a8:0.1.0.1"); + /* zeros in the middle. */ + test_pton6_same("fe80:0000:0000:0000:0202:1111:0001:0001", + "fe80::202:1111:1:1"); + /* zeros at the end. */ + test_pton6_same("1000:0001:0000:0007:0000:0000:0000:0000", + "1000:1:0:7::"); + + /* === Test ntop: af_inet6 */ + test_ntop6_reduces("0:0:0:0:0:0:0:0", "::"); + + test_ntop6_reduces("0001:0099:BEEF:0006:0123:FFFF:0001:0001", + "1:99:beef:6:123:ffff:1:1"); + + //test_ntop6_reduces("0:0:0:0:0:0:c0a8:0101", "::192.168.1.1"); + test_ntop6_reduces("0:0:0:0:0:ffff:c0a8:0101", "::ffff:192.168.1.1"); + test_ntop6_reduces("002:0:0000:0:3::4", "2::3:0:0:4"); + test_ntop6_reduces("0:0::1:0:3", "::1:0:3"); + test_ntop6_reduces("008:0::0", "8::"); + test_ntop6_reduces("0:0:0:0:0:ffff::1", "::ffff:0.0.0.1"); + test_ntop6_reduces("abcd:0:0:0:0:0:7f00::", "abcd::7f00:0"); + test_ntop6_reduces("0000:0000:0000:0000:0009:C0A8:0001:0001", + "::9:c0a8:1:1"); + test_ntop6_reduces("fe80:0000:0000:0000:0202:1111:0001:0001", + "fe80::202:1111:1:1"); + test_ntop6_reduces("1000:0001:0000:0007:0000:0000:0000:0000", + "1000:1:0:7::"); + + /* === Test pton: invalid in6. */ + test_pton6_bad("foobar."); + test_pton6_bad("55555::"); + test_pton6_bad("9:-60::"); + test_pton6_bad("1:2:33333:4:0002:3::"); + //test_pton6_bad("1:2:3333:4:00002:3::");// BAD, but glibc doesn't say so. + test_pton6_bad("1:2:3333:4:fish:3::"); + test_pton6_bad("1:2:3:4:5:6:7:8:9"); + test_pton6_bad("1:2:3:4:5:6:7"); + test_pton6_bad("1:2:3:4:5:6:1.2.3.4.5"); + test_pton6_bad("1:2:3:4:5:6:1.2.3"); + test_pton6_bad("::1.2.3"); + test_pton6_bad("::1.2.3.4.5"); + test_pton6_bad("99"); + test_pton6_bad(""); + test_pton6_bad("1::2::3:4"); + test_pton6_bad("a:::b:c"); + test_pton6_bad(":::a:b:c"); + test_pton6_bad("a:b:c:::"); + + /* test internal checking */ + test_external_ip("fbff:ffff::2:7", 0); + test_internal_ip("fc01::2:7", 0); + test_internal_ip("fdff:ffff::f:f", 0); + test_external_ip("fe00::3:f", 0); + + test_external_ip("fe7f:ffff::2:7", 0); + test_internal_ip("fe80::2:7", 0); + test_internal_ip("febf:ffff::f:f", 0); + + test_internal_ip("fec0::2:7:7", 0); + test_internal_ip("feff:ffff::e:7:7", 0); + test_external_ip("ff00::e:7:7", 0); + + test_internal_ip("::", 0); + test_internal_ip("::1", 0); + test_internal_ip("::1", 1); + test_internal_ip("::", 0); + test_external_ip("::", 1); + test_external_ip("::2", 0); + test_external_ip("2001::", 0); + test_external_ip("ffff::", 0); + + test_external_ip("::ffff:0.0.0.0", 1); + test_internal_ip("::ffff:0.0.0.0", 0); + test_internal_ip("::ffff:0.255.255.255", 0); + test_external_ip("::ffff:1.0.0.0", 0); + + test_external_ip("::ffff:9.255.255.255", 0); + test_internal_ip("::ffff:10.0.0.0", 0); + test_internal_ip("::ffff:10.255.255.255", 0); + test_external_ip("::ffff:11.0.0.0", 0); + + test_external_ip("::ffff:126.255.255.255", 0); + test_internal_ip("::ffff:127.0.0.0", 0); + test_internal_ip("::ffff:127.255.255.255", 0); + test_external_ip("::ffff:128.0.0.0", 0); + + test_external_ip("::ffff:172.15.255.255", 0); + test_internal_ip("::ffff:172.16.0.0", 0); + test_internal_ip("::ffff:172.31.255.255", 0); + test_external_ip("::ffff:172.32.0.0", 0); + + test_external_ip("::ffff:192.167.255.255", 0); + test_internal_ip("::ffff:192.168.0.0", 0); + test_internal_ip("::ffff:192.168.255.255", 0); + test_external_ip("::ffff:192.169.0.0", 0); + + test_external_ip("::ffff:169.253.255.255", 0); + test_internal_ip("::ffff:169.254.0.0", 0); + test_internal_ip("::ffff:169.254.255.255", 0); + test_external_ip("::ffff:169.255.0.0", 0); + test_assert(is_internal_IP(0x7f000001, 0)); + + /* tor_addr_compare(tor_addr_t x2) */ + test_addr_compare("ffff::", ==, "ffff::0"); + test_addr_compare("0::3:2:1", <, "0::ffff:0.3.2.1"); + test_addr_compare("0::2:2:1", <, "0::ffff:0.3.2.1"); + test_addr_compare("0::ffff:0.3.2.1", >, "0::0:0:0"); + test_addr_compare("0::ffff:5.2.2.1", <, "::ffff:6.0.0.0"); /* XXXX wrong. */ + tor_addr_parse_mask_ports("[::ffff:2.3.4.5]", &t1, NULL, NULL, NULL); + tor_addr_parse_mask_ports("2.3.4.5", &t2, NULL, NULL, NULL); + test_assert(tor_addr_compare(&t1, &t2, CMP_SEMANTIC) == 0); + tor_addr_parse_mask_ports("[::ffff:2.3.4.4]", &t1, NULL, NULL, NULL); + tor_addr_parse_mask_ports("2.3.4.5", &t2, NULL, NULL, NULL); + test_assert(tor_addr_compare(&t1, &t2, CMP_SEMANTIC) < 0); + + /* test compare_masked */ + test_addr_compare_masked("ffff::", ==, "ffff::0", 128); + test_addr_compare_masked("ffff::", ==, "ffff::0", 64); + test_addr_compare_masked("0::2:2:1", <, "0::8000:2:1", 81); + test_addr_compare_masked("0::2:2:1", ==, "0::8000:2:1", 80); + + /* Test decorated addr_to_string. */ + test_eq(AF_INET6, tor_addr_from_str(&t1, "[123:45:6789::5005:11]")); + p1 = tor_addr_to_str(buf, &t1, sizeof(buf), 1); + test_streq(p1, "[123:45:6789::5005:11]"); + test_eq(AF_INET, tor_addr_from_str(&t1, "18.0.0.1")); + p1 = tor_addr_to_str(buf, &t1, sizeof(buf), 1); + test_streq(p1, "18.0.0.1"); + + /* Test tor_addr_parse_reverse_lookup_name */ + i = tor_addr_parse_reverse_lookup_name(&t1, "Foobar.baz", AF_UNSPEC, 0); + test_eq(0, i); + i = tor_addr_parse_reverse_lookup_name(&t1, "Foobar.baz", AF_UNSPEC, 1); + test_eq(0, i); + i = tor_addr_parse_reverse_lookup_name(&t1, "1.0.168.192.in-addr.arpa", + AF_UNSPEC, 1); + test_eq(1, i); + test_eq(tor_addr_family(&t1), AF_INET); + p1 = tor_addr_to_str(buf, &t1, sizeof(buf), 1); + test_streq(p1, "192.168.0.1"); + i = tor_addr_parse_reverse_lookup_name(&t1, "192.168.0.99", AF_UNSPEC, 0); + test_eq(0, i); + i = tor_addr_parse_reverse_lookup_name(&t1, "192.168.0.99", AF_UNSPEC, 1); + test_eq(1, i); + p1 = tor_addr_to_str(buf, &t1, sizeof(buf), 1); + test_streq(p1, "192.168.0.99"); + memset(&t1, 0, sizeof(t1)); + i = tor_addr_parse_reverse_lookup_name(&t1, + "0.1.2.3.4.5.6.7.8.9.a.b.c.d.e.f." + "f.e.e.b.1.e.b.e.e.f.f.e.e.e.d.9." + "ip6.ARPA", + AF_UNSPEC, 0); + test_eq(1, i); + p1 = tor_addr_to_str(buf, &t1, sizeof(buf), 1); + test_streq(p1, "[9dee:effe:ebe1:beef:fedc:ba98:7654:3210]"); + /* Failing cases. */ + i = tor_addr_parse_reverse_lookup_name(&t1, + "6.7.8.9.a.b.c.d.e.f." + "f.e.e.b.1.e.b.e.e.f.f.e.e.e.d.9." + "ip6.ARPA", + AF_UNSPEC, 0); + test_eq(i, -1); + i = tor_addr_parse_reverse_lookup_name(&t1, + "6.7.8.9.a.b.c.d.e.f.a.b.c.d.e.f.0." + "f.e.e.b.1.e.b.e.e.f.f.e.e.e.d.9." + "ip6.ARPA", + AF_UNSPEC, 0); + test_eq(i, -1); + i = tor_addr_parse_reverse_lookup_name(&t1, + "6.7.8.9.a.b.c.d.e.f.X.0.0.0.0.9." + "f.e.e.b.1.e.b.e.e.f.f.e.e.e.d.9." + "ip6.ARPA", + AF_UNSPEC, 0); + test_eq(i, -1); + i = tor_addr_parse_reverse_lookup_name(&t1, "32.1.1.in-addr.arpa", + AF_UNSPEC, 0); + test_eq(i, -1); + i = tor_addr_parse_reverse_lookup_name(&t1, ".in-addr.arpa", + AF_UNSPEC, 0); + test_eq(i, -1); + i = tor_addr_parse_reverse_lookup_name(&t1, "1.2.3.4.5.in-addr.arpa", + AF_UNSPEC, 0); + test_eq(i, -1); + i = tor_addr_parse_reverse_lookup_name(&t1, "1.2.3.4.5.in-addr.arpa", + AF_INET6, 0); + test_eq(i, -1); + i = tor_addr_parse_reverse_lookup_name(&t1, + "6.7.8.9.a.b.c.d.e.f.a.b.c.d.e.0." + "f.e.e.b.1.e.b.e.e.f.f.e.e.e.d.9." + "ip6.ARPA", + AF_INET, 0); + test_eq(i, -1); + + /* test tor_addr_parse_mask_ports */ + test_addr_mask_ports_parse("[::f]/17:47-95", AF_INET6, + 0, 0, 0, 0x0000000f, 17, 47, 95); + //test_addr_parse("[::fefe:4.1.1.7/120]:999-1000"); + //test_addr_parse_check("::fefe:401:107", 120, 999, 1000); + test_addr_mask_ports_parse("[::ffff:4.1.1.7]/120:443", AF_INET6, + 0, 0, 0x0000ffff, 0x04010107, 120, 443, 443); + test_addr_mask_ports_parse("[abcd:2::44a:0]:2-65000", AF_INET6, + 0xabcd0002, 0, 0, 0x044a0000, 128, 2, 65000); + + r=tor_addr_parse_mask_ports("[fefef::]/112", &t1, NULL, NULL, NULL); + test_assert(r == -1); + r=tor_addr_parse_mask_ports("efef::/112", &t1, NULL, NULL, NULL); + test_assert(r == -1); + r=tor_addr_parse_mask_ports("[f:f:f:f:f:f:f:f::]", &t1, NULL, NULL, NULL); + test_assert(r == -1); + r=tor_addr_parse_mask_ports("[::f:f:f:f:f:f:f:f]", &t1, NULL, NULL, NULL); + test_assert(r == -1); + r=tor_addr_parse_mask_ports("[f:f:f:f:f:f:f:f:f]", &t1, NULL, NULL, NULL); + test_assert(r == -1); + /* Test for V4-mapped address with mask < 96. (arguably not valid) */ + r=tor_addr_parse_mask_ports("[::ffff:1.1.2.2/33]", &t1, &mask, NULL, NULL); + test_assert(r == -1); + r=tor_addr_parse_mask_ports("1.1.2.2/33", &t1, &mask, NULL, NULL); + test_assert(r == -1); + r=tor_addr_parse_mask_ports("1.1.2.2/31", &t1, &mask, NULL, NULL); + test_assert(r == AF_INET); + r=tor_addr_parse_mask_ports("[efef::]/112", &t1, &mask, &port1, &port2); + test_assert(r == AF_INET6); + test_assert(port1 == 1); + test_assert(port2 == 65535); + + /* make sure inet address lengths >= max */ + test_assert(INET_NTOA_BUF_LEN >= sizeof("255.255.255.255")); + test_assert(TOR_ADDR_BUF_LEN >= + sizeof("ffff:ffff:ffff:ffff:ffff:ffff:255.255.255.255")); + + test_assert(sizeof(tor_addr_t) >= sizeof(struct in6_addr)); + + /* get interface addresses */ + r = get_interface_address6(LOG_DEBUG, AF_INET, &t1); + i = get_interface_address6(LOG_DEBUG, AF_INET6, &t2); +#if 0 + tor_inet_ntop(AF_INET, &t1.sa.sin_addr, buf, sizeof(buf)); + printf("\nv4 address: %s (family=%d)", buf, IN_FAMILY(&t1)); + tor_inet_ntop(AF_INET6, &t2.sa6.sin6_addr, buf, sizeof(buf)); + printf("\nv6 address: %s (family=%d)", buf, IN_FAMILY(&t2)); +#endif + + done: + ; +} + +#define ADDR_LEGACY(name) \ + { #name, legacy_test_helper, 0, &legacy_setup, test_addr_ ## name } + +struct testcase_t addr_tests[] = { + ADDR_LEGACY(basic), + ADDR_LEGACY(ip6_helpers), + END_OF_TESTCASES +}; + diff --git a/src/test/test_containers.c b/src/test/test_containers.c new file mode 100644 index 0000000000..af9fb1c5c9 --- /dev/null +++ b/src/test/test_containers.c @@ -0,0 +1,765 @@ +/* Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "orconfig.h" +#include "or.h" +#include "test.h" + +/** Helper: return a tristate based on comparing the strings in *<b>a</b> and + * *<b>b</b>. */ +static int +_compare_strs(const void **a, const void **b) +{ + const char *s1 = *a, *s2 = *b; + return strcmp(s1, s2); +} + +/** Helper: return a tristate based on comparing the strings in *<b>a</b> and + * *<b>b</b>, excluding a's first character, and ignoring case. */ +static int +_compare_without_first_ch(const void *a, const void **b) +{ + const char *s1 = a, *s2 = *b; + return strcasecmp(s1+1, s2); +} + +/** Run unit tests for basic dynamic-sized array functionality. */ +static void +test_container_smartlist_basic(void) +{ + smartlist_t *sl; + + /* XXXX test sort_digests, uniq_strings, uniq_digests */ + + /* Test smartlist add, del_keeporder, insert, get. */ + sl = smartlist_create(); + smartlist_add(sl, (void*)1); + smartlist_add(sl, (void*)2); + smartlist_add(sl, (void*)3); + smartlist_add(sl, (void*)4); + smartlist_del_keeporder(sl, 1); + smartlist_insert(sl, 1, (void*)22); + smartlist_insert(sl, 0, (void*)0); + smartlist_insert(sl, 5, (void*)555); + test_eq_ptr((void*)0, smartlist_get(sl,0)); + test_eq_ptr((void*)1, smartlist_get(sl,1)); + test_eq_ptr((void*)22, smartlist_get(sl,2)); + test_eq_ptr((void*)3, smartlist_get(sl,3)); + test_eq_ptr((void*)4, smartlist_get(sl,4)); + test_eq_ptr((void*)555, smartlist_get(sl,5)); + /* Try deleting in the middle. */ + smartlist_del(sl, 1); + test_eq_ptr((void*)555, smartlist_get(sl, 1)); + /* Try deleting at the end. */ + smartlist_del(sl, 4); + test_eq(4, smartlist_len(sl)); + + /* test isin. */ + test_assert(smartlist_isin(sl, (void*)3)); + test_assert(!smartlist_isin(sl, (void*)99)); + + done: + smartlist_free(sl); +} + +/** Run unit tests for smartlist-of-strings functionality. */ +static void +test_container_smartlist_strings(void) +{ + smartlist_t *sl = smartlist_create(); + char *cp=NULL, *cp_alloc=NULL; + size_t sz; + + /* Test split and join */ + test_eq(0, smartlist_len(sl)); + smartlist_split_string(sl, "abc", ":", 0, 0); + test_eq(1, smartlist_len(sl)); + test_streq("abc", smartlist_get(sl, 0)); + smartlist_split_string(sl, "a::bc::", "::", 0, 0); + test_eq(4, smartlist_len(sl)); + test_streq("a", smartlist_get(sl, 1)); + test_streq("bc", smartlist_get(sl, 2)); + test_streq("", smartlist_get(sl, 3)); + cp_alloc = smartlist_join_strings(sl, "", 0, NULL); + test_streq(cp_alloc, "abcabc"); + tor_free(cp_alloc); + cp_alloc = smartlist_join_strings(sl, "!", 0, NULL); + test_streq(cp_alloc, "abc!a!bc!"); + tor_free(cp_alloc); + cp_alloc = smartlist_join_strings(sl, "XY", 0, NULL); + test_streq(cp_alloc, "abcXYaXYbcXY"); + tor_free(cp_alloc); + cp_alloc = smartlist_join_strings(sl, "XY", 1, NULL); + test_streq(cp_alloc, "abcXYaXYbcXYXY"); + tor_free(cp_alloc); + cp_alloc = smartlist_join_strings(sl, "", 1, NULL); + test_streq(cp_alloc, "abcabc"); + tor_free(cp_alloc); + + smartlist_split_string(sl, "/def/ /ghijk", "/", 0, 0); + test_eq(8, smartlist_len(sl)); + test_streq("", smartlist_get(sl, 4)); + test_streq("def", smartlist_get(sl, 5)); + test_streq(" ", smartlist_get(sl, 6)); + test_streq("ghijk", smartlist_get(sl, 7)); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + smartlist_split_string(sl, "a,bbd,cdef", ",", SPLIT_SKIP_SPACE, 0); + test_eq(3, smartlist_len(sl)); + test_streq("a", smartlist_get(sl,0)); + test_streq("bbd", smartlist_get(sl,1)); + test_streq("cdef", smartlist_get(sl,2)); + smartlist_split_string(sl, " z <> zhasd <> <> bnud<> ", "<>", + SPLIT_SKIP_SPACE, 0); + test_eq(8, smartlist_len(sl)); + test_streq("z", smartlist_get(sl,3)); + test_streq("zhasd", smartlist_get(sl,4)); + test_streq("", smartlist_get(sl,5)); + test_streq("bnud", smartlist_get(sl,6)); + test_streq("", smartlist_get(sl,7)); + + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + smartlist_split_string(sl, " ab\tc \td ef ", NULL, + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); + test_eq(4, smartlist_len(sl)); + test_streq("ab", smartlist_get(sl,0)); + test_streq("c", smartlist_get(sl,1)); + test_streq("d", smartlist_get(sl,2)); + test_streq("ef", smartlist_get(sl,3)); + smartlist_split_string(sl, "ghi\tj", NULL, + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); + test_eq(6, smartlist_len(sl)); + test_streq("ghi", smartlist_get(sl,4)); + test_streq("j", smartlist_get(sl,5)); + + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + cp_alloc = smartlist_join_strings(sl, "XY", 0, NULL); + test_streq(cp_alloc, ""); + tor_free(cp_alloc); + cp_alloc = smartlist_join_strings(sl, "XY", 1, NULL); + test_streq(cp_alloc, "XY"); + tor_free(cp_alloc); + + smartlist_split_string(sl, " z <> zhasd <> <> bnud<> ", "<>", + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); + test_eq(3, smartlist_len(sl)); + test_streq("z", smartlist_get(sl, 0)); + test_streq("zhasd", smartlist_get(sl, 1)); + test_streq("bnud", smartlist_get(sl, 2)); + smartlist_split_string(sl, " z <> zhasd <> <> bnud<> ", "<>", + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 2); + test_eq(5, smartlist_len(sl)); + test_streq("z", smartlist_get(sl, 3)); + test_streq("zhasd <> <> bnud<>", smartlist_get(sl, 4)); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + smartlist_split_string(sl, "abcd\n", "\n", + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); + test_eq(1, smartlist_len(sl)); + test_streq("abcd", smartlist_get(sl, 0)); + smartlist_split_string(sl, "efgh", "\n", + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); + test_eq(2, smartlist_len(sl)); + test_streq("efgh", smartlist_get(sl, 1)); + + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + /* Test swapping, shuffling, and sorting. */ + smartlist_split_string(sl, "the,onion,router,by,arma,and,nickm", ",", 0, 0); + test_eq(7, smartlist_len(sl)); + smartlist_sort(sl, _compare_strs); + cp_alloc = smartlist_join_strings(sl, ",", 0, NULL); + test_streq(cp_alloc,"and,arma,by,nickm,onion,router,the"); + tor_free(cp_alloc); + smartlist_swap(sl, 1, 5); + cp_alloc = smartlist_join_strings(sl, ",", 0, NULL); + test_streq(cp_alloc,"and,router,by,nickm,onion,arma,the"); + tor_free(cp_alloc); + smartlist_shuffle(sl); + test_eq(7, smartlist_len(sl)); + test_assert(smartlist_string_isin(sl, "and")); + test_assert(smartlist_string_isin(sl, "router")); + test_assert(smartlist_string_isin(sl, "by")); + test_assert(smartlist_string_isin(sl, "nickm")); + test_assert(smartlist_string_isin(sl, "onion")); + test_assert(smartlist_string_isin(sl, "arma")); + test_assert(smartlist_string_isin(sl, "the")); + + /* Test bsearch. */ + smartlist_sort(sl, _compare_strs); + test_streq("nickm", smartlist_bsearch(sl, "zNicKM", + _compare_without_first_ch)); + test_streq("and", smartlist_bsearch(sl, " AND", _compare_without_first_ch)); + test_eq_ptr(NULL, smartlist_bsearch(sl, " ANz", _compare_without_first_ch)); + + /* Test bsearch_idx */ + { + int f; + test_eq(0, smartlist_bsearch_idx(sl," aaa",_compare_without_first_ch,&f)); + test_eq(f, 0); + test_eq(0, smartlist_bsearch_idx(sl," and",_compare_without_first_ch,&f)); + test_eq(f, 1); + test_eq(1, smartlist_bsearch_idx(sl," arm",_compare_without_first_ch,&f)); + test_eq(f, 0); + test_eq(1, smartlist_bsearch_idx(sl," arma",_compare_without_first_ch,&f)); + test_eq(f, 1); + test_eq(2, smartlist_bsearch_idx(sl," armb",_compare_without_first_ch,&f)); + test_eq(f, 0); + test_eq(7, smartlist_bsearch_idx(sl," zzzz",_compare_without_first_ch,&f)); + test_eq(f, 0); + } + + /* Test reverse() and pop_last() */ + smartlist_reverse(sl); + cp_alloc = smartlist_join_strings(sl, ",", 0, NULL); + test_streq(cp_alloc,"the,router,onion,nickm,by,arma,and"); + tor_free(cp_alloc); + cp_alloc = smartlist_pop_last(sl); + test_streq(cp_alloc, "and"); + tor_free(cp_alloc); + test_eq(smartlist_len(sl), 6); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + cp_alloc = smartlist_pop_last(sl); + test_eq(cp_alloc, NULL); + + /* Test uniq() */ + smartlist_split_string(sl, + "50,noon,radar,a,man,a,plan,a,canal,panama,radar,noon,50", + ",", 0, 0); + smartlist_sort(sl, _compare_strs); + smartlist_uniq(sl, _compare_strs, _tor_free); + cp_alloc = smartlist_join_strings(sl, ",", 0, NULL); + test_streq(cp_alloc, "50,a,canal,man,noon,panama,plan,radar"); + tor_free(cp_alloc); + + /* Test string_isin and isin_case and num_isin */ + test_assert(smartlist_string_isin(sl, "noon")); + test_assert(!smartlist_string_isin(sl, "noonoon")); + test_assert(smartlist_string_isin_case(sl, "nOOn")); + test_assert(!smartlist_string_isin_case(sl, "nooNooN")); + test_assert(smartlist_string_num_isin(sl, 50)); + test_assert(!smartlist_string_num_isin(sl, 60)); + + /* Test smartlist_choose */ + { + int i; + int allsame = 1; + int allin = 1; + void *first = smartlist_choose(sl); + test_assert(smartlist_isin(sl, first)); + for (i = 0; i < 100; ++i) { + void *second = smartlist_choose(sl); + if (second != first) + allsame = 0; + if (!smartlist_isin(sl, second)) + allin = 0; + } + test_assert(!allsame); + test_assert(allin); + } + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + /* Test string_remove and remove and join_strings2 */ + smartlist_split_string(sl, + "Some say the Earth will end in ice and some in fire", + " ", 0, 0); + cp = smartlist_get(sl, 4); + test_streq(cp, "will"); + smartlist_add(sl, cp); + smartlist_remove(sl, cp); + tor_free(cp); + cp_alloc = smartlist_join_strings(sl, ",", 0, NULL); + test_streq(cp_alloc, "Some,say,the,Earth,fire,end,in,ice,and,some,in"); + tor_free(cp_alloc); + smartlist_string_remove(sl, "in"); + cp_alloc = smartlist_join_strings2(sl, "+XX", 1, 0, &sz); + test_streq(cp_alloc, "Some+say+the+Earth+fire+end+some+ice+and"); + test_eq((int)sz, 40); + + done: + + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_free(sl); + tor_free(cp_alloc); +} + +/** Run unit tests for smartlist set manipulation functions. */ +static void +test_container_smartlist_overlap(void) +{ + smartlist_t *sl = smartlist_create(); + smartlist_t *ints = smartlist_create(); + smartlist_t *odds = smartlist_create(); + smartlist_t *evens = smartlist_create(); + smartlist_t *primes = smartlist_create(); + int i; + for (i=1; i < 10; i += 2) + smartlist_add(odds, (void*)(uintptr_t)i); + for (i=0; i < 10; i += 2) + smartlist_add(evens, (void*)(uintptr_t)i); + + /* add_all */ + smartlist_add_all(ints, odds); + smartlist_add_all(ints, evens); + test_eq(smartlist_len(ints), 10); + + smartlist_add(primes, (void*)2); + smartlist_add(primes, (void*)3); + smartlist_add(primes, (void*)5); + smartlist_add(primes, (void*)7); + + /* overlap */ + test_assert(smartlist_overlap(ints, odds)); + test_assert(smartlist_overlap(odds, primes)); + test_assert(smartlist_overlap(evens, primes)); + test_assert(!smartlist_overlap(odds, evens)); + + /* intersect */ + smartlist_add_all(sl, odds); + smartlist_intersect(sl, primes); + test_eq(smartlist_len(sl), 3); + test_assert(smartlist_isin(sl, (void*)3)); + test_assert(smartlist_isin(sl, (void*)5)); + test_assert(smartlist_isin(sl, (void*)7)); + + /* subtract */ + smartlist_add_all(sl, primes); + smartlist_subtract(sl, odds); + test_eq(smartlist_len(sl), 1); + test_assert(smartlist_isin(sl, (void*)2)); + + done: + smartlist_free(odds); + smartlist_free(evens); + smartlist_free(ints); + smartlist_free(primes); + smartlist_free(sl); +} + +/** Run unit tests for smartlist-of-digests functions. */ +static void +test_container_smartlist_digests(void) +{ + smartlist_t *sl = smartlist_create(); + + /* digest_isin. */ + smartlist_add(sl, tor_memdup("AAAAAAAAAAAAAAAAAAAA", DIGEST_LEN)); + smartlist_add(sl, tor_memdup("\00090AAB2AAAAaasdAAAAA", DIGEST_LEN)); + smartlist_add(sl, tor_memdup("\00090AAB2AAAAaasdAAAAA", DIGEST_LEN)); + test_eq(0, smartlist_digest_isin(NULL, "AAAAAAAAAAAAAAAAAAAA")); + test_assert(smartlist_digest_isin(sl, "AAAAAAAAAAAAAAAAAAAA")); + test_assert(smartlist_digest_isin(sl, "\00090AAB2AAAAaasdAAAAA")); + test_eq(0, smartlist_digest_isin(sl, "\00090AAB2AAABaasdAAAAA")); + + /* sort digests */ + smartlist_sort_digests(sl); + test_memeq(smartlist_get(sl, 0), "\00090AAB2AAAAaasdAAAAA", DIGEST_LEN); + test_memeq(smartlist_get(sl, 1), "\00090AAB2AAAAaasdAAAAA", DIGEST_LEN); + test_memeq(smartlist_get(sl, 2), "AAAAAAAAAAAAAAAAAAAA", DIGEST_LEN); + test_eq(3, smartlist_len(sl)); + + /* uniq_digests */ + smartlist_uniq_digests(sl); + test_eq(2, smartlist_len(sl)); + test_memeq(smartlist_get(sl, 0), "\00090AAB2AAAAaasdAAAAA", DIGEST_LEN); + test_memeq(smartlist_get(sl, 1), "AAAAAAAAAAAAAAAAAAAA", DIGEST_LEN); + + done: + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_free(sl); +} + +/** Run unit tests for concatenate-a-smartlist-of-strings functions. */ +static void +test_container_smartlist_join(void) +{ + smartlist_t *sl = smartlist_create(); + smartlist_t *sl2 = smartlist_create(), *sl3 = smartlist_create(), + *sl4 = smartlist_create(); + char *joined=NULL; + /* unique, sorted. */ + smartlist_split_string(sl, + "Abashments Ambush Anchorman Bacon Banks Borscht " + "Bunks Inhumane Insurance Knish Know Manners " + "Maraschinos Stamina Sunbonnets Unicorns Wombats", + " ", 0, 0); + /* non-unique, sorted. */ + smartlist_split_string(sl2, + "Ambush Anchorman Anchorman Anemias Anemias Bacon " + "Crossbowmen Inhumane Insurance Knish Know Manners " + "Manners Maraschinos Wombats Wombats Work", + " ", 0, 0); + SMARTLIST_FOREACH_JOIN(sl, char *, cp1, + sl2, char *, cp2, + strcmp(cp1,cp2), + smartlist_add(sl3, cp2)) { + test_streq(cp1, cp2); + smartlist_add(sl4, cp1); + } SMARTLIST_FOREACH_JOIN_END(cp1, cp2); + + SMARTLIST_FOREACH(sl3, const char *, cp, + test_assert(smartlist_isin(sl2, cp) && + !smartlist_string_isin(sl, cp))); + SMARTLIST_FOREACH(sl4, const char *, cp, + test_assert(smartlist_isin(sl, cp) && + smartlist_string_isin(sl2, cp))); + joined = smartlist_join_strings(sl3, ",", 0, NULL); + test_streq(joined, "Anemias,Anemias,Crossbowmen,Work"); + tor_free(joined); + joined = smartlist_join_strings(sl4, ",", 0, NULL); + test_streq(joined, "Ambush,Anchorman,Anchorman,Bacon,Inhumane,Insurance," + "Knish,Know,Manners,Manners,Maraschinos,Wombats,Wombats"); + tor_free(joined); + + done: + smartlist_free(sl4); + smartlist_free(sl3); + SMARTLIST_FOREACH(sl2, char *, cp, tor_free(cp)); + smartlist_free(sl2); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_free(sl); + tor_free(joined); +} + +/** Run unit tests for bitarray code */ +static void +test_container_bitarray(void) +{ + bitarray_t *ba = NULL; + int i, j, ok=1; + + ba = bitarray_init_zero(1); + test_assert(ba); + test_assert(! bitarray_is_set(ba, 0)); + bitarray_set(ba, 0); + test_assert(bitarray_is_set(ba, 0)); + bitarray_clear(ba, 0); + test_assert(! bitarray_is_set(ba, 0)); + bitarray_free(ba); + + ba = bitarray_init_zero(1023); + for (i = 1; i < 64; ) { + for (j = 0; j < 1023; ++j) { + if (j % i) + bitarray_set(ba, j); + else + bitarray_clear(ba, j); + } + for (j = 0; j < 1023; ++j) { + if (!bool_eq(bitarray_is_set(ba, j), j%i)) + ok = 0; + } + test_assert(ok); + if (i < 7) + ++i; + else if (i == 28) + i = 32; + else + i += 7; + } + + done: + if (ba) + bitarray_free(ba); +} + +/** Run unit tests for digest set code (implemented as a hashtable or as a + * bloom filter) */ +static void +test_container_digestset(void) +{ + smartlist_t *included = smartlist_create(); + char d[DIGEST_LEN]; + int i; + int ok = 1; + int false_positives = 0; + digestset_t *set = NULL; + + for (i = 0; i < 1000; ++i) { + crypto_rand(d, DIGEST_LEN); + smartlist_add(included, tor_memdup(d, DIGEST_LEN)); + } + set = digestset_new(1000); + SMARTLIST_FOREACH(included, const char *, cp, + if (digestset_isin(set, cp)) + ok = 0); + test_assert(ok); + SMARTLIST_FOREACH(included, const char *, cp, + digestset_add(set, cp)); + SMARTLIST_FOREACH(included, const char *, cp, + if (!digestset_isin(set, cp)) + ok = 0); + test_assert(ok); + for (i = 0; i < 1000; ++i) { + crypto_rand(d, DIGEST_LEN); + if (digestset_isin(set, d)) + ++false_positives; + } + test_assert(false_positives < 50); /* Should be far lower. */ + + done: + if (set) + digestset_free(set); + SMARTLIST_FOREACH(included, char *, cp, tor_free(cp)); + smartlist_free(included); +} + +typedef struct pq_entry_t { + const char *val; + int idx; +} pq_entry_t; + +/** Helper: return a tristate based on comparing two pq_entry_t values. */ +static int +_compare_strings_for_pqueue(const void *p1, const void *p2) +{ + const pq_entry_t *e1=p1, *e2=p2; + return strcmp(e1->val, e2->val); +} + +/** Run unit tests for heap-based priority queue functions. */ +static void +test_container_pqueue(void) +{ + smartlist_t *sl = smartlist_create(); + int (*cmp)(const void *, const void*); + const int offset = STRUCT_OFFSET(pq_entry_t, idx); +#define ENTRY(s) pq_entry_t s = { #s, -1 } + ENTRY(cows); + ENTRY(zebras); + ENTRY(fish); + ENTRY(frogs); + ENTRY(apples); + ENTRY(squid); + ENTRY(daschunds); + ENTRY(eggplants); + ENTRY(weissbier); + ENTRY(lobsters); + ENTRY(roquefort); + ENTRY(chinchillas); + ENTRY(fireflies); + +#define OK() smartlist_pqueue_assert_ok(sl, cmp, offset) + + cmp = _compare_strings_for_pqueue; + smartlist_pqueue_add(sl, cmp, offset, &cows); + smartlist_pqueue_add(sl, cmp, offset, &zebras); + smartlist_pqueue_add(sl, cmp, offset, &fish); + smartlist_pqueue_add(sl, cmp, offset, &frogs); + smartlist_pqueue_add(sl, cmp, offset, &apples); + smartlist_pqueue_add(sl, cmp, offset, &squid); + smartlist_pqueue_add(sl, cmp, offset, &daschunds); + smartlist_pqueue_add(sl, cmp, offset, &eggplants); + smartlist_pqueue_add(sl, cmp, offset, &weissbier); + smartlist_pqueue_add(sl, cmp, offset, &lobsters); + smartlist_pqueue_add(sl, cmp, offset, &roquefort); + + OK(); + + test_eq(smartlist_len(sl), 11); + test_eq_ptr(smartlist_get(sl, 0), &apples); + test_eq_ptr(smartlist_pqueue_pop(sl, cmp, offset), &apples); + test_eq(smartlist_len(sl), 10); + OK(); + test_eq_ptr(smartlist_pqueue_pop(sl, cmp, offset), &cows); + test_eq_ptr(smartlist_pqueue_pop(sl, cmp, offset), &daschunds); + smartlist_pqueue_add(sl, cmp, offset, &chinchillas); + OK(); + smartlist_pqueue_add(sl, cmp, offset, &fireflies); + OK(); + test_eq_ptr(smartlist_pqueue_pop(sl, cmp, offset), &chinchillas); + test_eq_ptr(smartlist_pqueue_pop(sl, cmp, offset), &eggplants); + test_eq_ptr(smartlist_pqueue_pop(sl, cmp, offset), &fireflies); + OK(); + test_eq_ptr(smartlist_pqueue_pop(sl, cmp, offset), &fish); + test_eq_ptr(smartlist_pqueue_pop(sl, cmp, offset), &frogs); + test_eq_ptr(smartlist_pqueue_pop(sl, cmp, offset), &lobsters); + test_eq_ptr(smartlist_pqueue_pop(sl, cmp, offset), &roquefort); + OK(); + test_eq(smartlist_len(sl), 3); + test_eq_ptr(smartlist_pqueue_pop(sl, cmp, offset), &squid); + test_eq_ptr(smartlist_pqueue_pop(sl, cmp, offset), &weissbier); + test_eq_ptr(smartlist_pqueue_pop(sl, cmp, offset), &zebras); + test_eq(smartlist_len(sl), 0); + OK(); + + /* Now test remove. */ + smartlist_pqueue_add(sl, cmp, offset, &cows); + smartlist_pqueue_add(sl, cmp, offset, &fish); + smartlist_pqueue_add(sl, cmp, offset, &frogs); + smartlist_pqueue_add(sl, cmp, offset, &apples); + smartlist_pqueue_add(sl, cmp, offset, &squid); + smartlist_pqueue_add(sl, cmp, offset, &zebras); + test_eq(smartlist_len(sl), 6); + OK(); + smartlist_pqueue_remove(sl, cmp, offset, &zebras); + test_eq(smartlist_len(sl), 5); + OK(); + smartlist_pqueue_remove(sl, cmp, offset, &cows); + test_eq(smartlist_len(sl), 4); + OK(); + smartlist_pqueue_remove(sl, cmp, offset, &apples); + test_eq(smartlist_len(sl), 3); + OK(); + test_eq_ptr(smartlist_pqueue_pop(sl, cmp, offset), &fish); + test_eq_ptr(smartlist_pqueue_pop(sl, cmp, offset), &frogs); + test_eq_ptr(smartlist_pqueue_pop(sl, cmp, offset), &squid); + test_eq(smartlist_len(sl), 0); + OK(); + +#undef OK + + done: + + smartlist_free(sl); +} + +/** Run unit tests for string-to-void* map functions */ +static void +test_container_strmap(void) +{ + strmap_t *map; + strmap_iter_t *iter; + const char *k; + void *v; + char *visited = NULL; + smartlist_t *found_keys = NULL; + + map = strmap_new(); + test_assert(map); + test_eq(strmap_size(map), 0); + test_assert(strmap_isempty(map)); + v = strmap_set(map, "K1", (void*)99); + test_eq(v, NULL); + test_assert(!strmap_isempty(map)); + v = strmap_set(map, "K2", (void*)101); + test_eq(v, NULL); + v = strmap_set(map, "K1", (void*)100); + test_eq(v, (void*)99); + test_eq_ptr(strmap_get(map,"K1"), (void*)100); + test_eq_ptr(strmap_get(map,"K2"), (void*)101); + test_eq_ptr(strmap_get(map,"K-not-there"), NULL); + strmap_assert_ok(map); + + v = strmap_remove(map,"K2"); + strmap_assert_ok(map); + test_eq_ptr(v, (void*)101); + test_eq_ptr(strmap_get(map,"K2"), NULL); + test_eq_ptr(strmap_remove(map,"K2"), NULL); + + strmap_set(map, "K2", (void*)101); + strmap_set(map, "K3", (void*)102); + strmap_set(map, "K4", (void*)103); + test_eq(strmap_size(map), 4); + strmap_assert_ok(map); + strmap_set(map, "K5", (void*)104); + strmap_set(map, "K6", (void*)105); + strmap_assert_ok(map); + + /* Test iterator. */ + iter = strmap_iter_init(map); + found_keys = smartlist_create(); + while (!strmap_iter_done(iter)) { + strmap_iter_get(iter,&k,&v); + smartlist_add(found_keys, tor_strdup(k)); + test_eq_ptr(v, strmap_get(map, k)); + + if (!strcmp(k, "K2")) { + iter = strmap_iter_next_rmv(map,iter); + } else { + iter = strmap_iter_next(map,iter); + } + } + + /* Make sure we removed K2, but not the others. */ + test_eq_ptr(strmap_get(map, "K2"), NULL); + test_eq_ptr(strmap_get(map, "K5"), (void*)104); + /* Make sure we visited everyone once */ + smartlist_sort_strings(found_keys); + visited = smartlist_join_strings(found_keys, ":", 0, NULL); + test_streq(visited, "K1:K2:K3:K4:K5:K6"); + + strmap_assert_ok(map); + /* Clean up after ourselves. */ + strmap_free(map, NULL); + map = NULL; + + /* Now try some lc functions. */ + map = strmap_new(); + strmap_set_lc(map,"Ab.C", (void*)1); + test_eq_ptr(strmap_get(map,"ab.c"), (void*)1); + strmap_assert_ok(map); + test_eq_ptr(strmap_get_lc(map,"AB.C"), (void*)1); + test_eq_ptr(strmap_get(map,"AB.C"), NULL); + test_eq_ptr(strmap_remove_lc(map,"aB.C"), (void*)1); + strmap_assert_ok(map); + test_eq_ptr(strmap_get_lc(map,"AB.C"), NULL); + + done: + if (map) + strmap_free(map,NULL); + if (found_keys) { + SMARTLIST_FOREACH(found_keys, char *, cp, tor_free(cp)); + smartlist_free(found_keys); + } + tor_free(visited); +} + +/** Run unit tests for getting the median of a list. */ +static void +test_container_order_functions(void) +{ + int lst[25], n = 0; + // int a=12,b=24,c=25,d=60,e=77; + +#define median() median_int(lst, n) + + lst[n++] = 12; + test_eq(12, median()); /* 12 */ + lst[n++] = 77; + //smartlist_shuffle(sl); + test_eq(12, median()); /* 12, 77 */ + lst[n++] = 77; + //smartlist_shuffle(sl); + test_eq(77, median()); /* 12, 77, 77 */ + lst[n++] = 24; + test_eq(24, median()); /* 12,24,77,77 */ + lst[n++] = 60; + lst[n++] = 12; + lst[n++] = 25; + //smartlist_shuffle(sl); + test_eq(25, median()); /* 12,12,24,25,60,77,77 */ +#undef median + + done: + ; +} + +#define CONTAINER_LEGACY(name) \ + { #name, legacy_test_helper, 0, &legacy_setup, test_container_ ## name } + +struct testcase_t container_tests[] = { + CONTAINER_LEGACY(smartlist_basic), + CONTAINER_LEGACY(smartlist_strings), + CONTAINER_LEGACY(smartlist_overlap), + CONTAINER_LEGACY(smartlist_digests), + CONTAINER_LEGACY(smartlist_join), + CONTAINER_LEGACY(bitarray), + CONTAINER_LEGACY(digestset), + CONTAINER_LEGACY(strmap), + CONTAINER_LEGACY(pqueue), + CONTAINER_LEGACY(order_functions), + END_OF_TESTCASES +}; + diff --git a/src/test/test_crypto.c b/src/test/test_crypto.c new file mode 100644 index 0000000000..781081a4ad --- /dev/null +++ b/src/test/test_crypto.c @@ -0,0 +1,796 @@ +/* Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "orconfig.h" +#define CRYPTO_PRIVATE +#include "or.h" +#include "test.h" + +/** Run unit tests for Diffie-Hellman functionality. */ +static void +test_crypto_dh(void) +{ + crypto_dh_env_t *dh1 = crypto_dh_new(DH_TYPE_CIRCUIT); + crypto_dh_env_t *dh2 = crypto_dh_new(DH_TYPE_CIRCUIT); + char p1[DH_BYTES]; + char p2[DH_BYTES]; + char s1[DH_BYTES]; + char s2[DH_BYTES]; + ssize_t s1len, s2len; + + test_eq(crypto_dh_get_bytes(dh1), DH_BYTES); + test_eq(crypto_dh_get_bytes(dh2), DH_BYTES); + + memset(p1, 0, DH_BYTES); + memset(p2, 0, DH_BYTES); + test_memeq(p1, p2, DH_BYTES); + test_assert(! crypto_dh_get_public(dh1, p1, DH_BYTES)); + test_memneq(p1, p2, DH_BYTES); + test_assert(! crypto_dh_get_public(dh2, p2, DH_BYTES)); + test_memneq(p1, p2, DH_BYTES); + + memset(s1, 0, DH_BYTES); + memset(s2, 0xFF, DH_BYTES); + s1len = crypto_dh_compute_secret(LOG_WARN, dh1, p2, DH_BYTES, s1, 50); + s2len = crypto_dh_compute_secret(LOG_WARN, dh2, p1, DH_BYTES, s2, 50); + test_assert(s1len > 0); + test_eq(s1len, s2len); + test_memeq(s1, s2, s1len); + + { + /* XXXX Now fabricate some bad values and make sure they get caught, + * Check 0, 1, N-1, >= N, etc. + */ + } + + done: + crypto_dh_free(dh1); + crypto_dh_free(dh2); +} + +/** Run unit tests for our random number generation function and its wrappers. + */ +static void +test_crypto_rng(void) +{ + int i, j, allok; + char data1[100], data2[100]; + double d; + + /* Try out RNG. */ + test_assert(! crypto_seed_rng(0)); + crypto_rand(data1, 100); + crypto_rand(data2, 100); + test_memneq(data1,data2,100); + allok = 1; + for (i = 0; i < 100; ++i) { + uint64_t big; + char *host; + j = crypto_rand_int(100); + if (i < 0 || i >= 100) + allok = 0; + big = crypto_rand_uint64(U64_LITERAL(1)<<40); + if (big >= (U64_LITERAL(1)<<40)) + allok = 0; + big = crypto_rand_uint64(U64_LITERAL(5)); + if (big >= 5) + allok = 0; + d = crypto_rand_double(); + test_assert(d >= 0); + test_assert(d < 1.0); + host = crypto_random_hostname(3,8,"www.",".onion"); + if (strcmpstart(host,"www.") || + strcmpend(host,".onion") || + strlen(host) < 13 || + strlen(host) > 18) + allok = 0; + tor_free(host); + } + test_assert(allok); + done: + ; +} + +/** Run unit tests for our AES functionality */ +static void +test_crypto_aes(void) +{ + char *data1 = NULL, *data2 = NULL, *data3 = NULL; + crypto_cipher_env_t *env1 = NULL, *env2 = NULL; + int i, j; + char *mem_op_hex_tmp=NULL; + + data1 = tor_malloc(1024); + data2 = tor_malloc(1024); + data3 = tor_malloc(1024); + + /* Now, test encryption and decryption with stream cipher. */ + data1[0]='\0'; + for (i = 1023; i>0; i -= 35) + strncat(data1, "Now is the time for all good onions", i); + + memset(data2, 0, 1024); + memset(data3, 0, 1024); + env1 = crypto_new_cipher_env(); + test_neq(env1, 0); + env2 = crypto_new_cipher_env(); + test_neq(env2, 0); + j = crypto_cipher_generate_key(env1); + crypto_cipher_set_key(env2, crypto_cipher_get_key(env1)); + crypto_cipher_encrypt_init_cipher(env1); + crypto_cipher_decrypt_init_cipher(env2); + + /* Try encrypting 512 chars. */ + crypto_cipher_encrypt(env1, data2, data1, 512); + crypto_cipher_decrypt(env2, data3, data2, 512); + test_memeq(data1, data3, 512); + test_memneq(data1, data2, 512); + + /* Now encrypt 1 at a time, and get 1 at a time. */ + for (j = 512; j < 560; ++j) { + crypto_cipher_encrypt(env1, data2+j, data1+j, 1); + } + for (j = 512; j < 560; ++j) { + crypto_cipher_decrypt(env2, data3+j, data2+j, 1); + } + test_memeq(data1, data3, 560); + /* Now encrypt 3 at a time, and get 5 at a time. */ + for (j = 560; j < 1024-5; j += 3) { + crypto_cipher_encrypt(env1, data2+j, data1+j, 3); + } + for (j = 560; j < 1024-5; j += 5) { + crypto_cipher_decrypt(env2, data3+j, data2+j, 5); + } + test_memeq(data1, data3, 1024-5); + /* Now make sure that when we encrypt with different chunk sizes, we get + the same results. */ + crypto_free_cipher_env(env2); + env2 = NULL; + + memset(data3, 0, 1024); + env2 = crypto_new_cipher_env(); + test_neq(env2, 0); + crypto_cipher_set_key(env2, crypto_cipher_get_key(env1)); + crypto_cipher_encrypt_init_cipher(env2); + for (j = 0; j < 1024-16; j += 17) { + crypto_cipher_encrypt(env2, data3+j, data1+j, 17); + } + for (j= 0; j < 1024-16; ++j) { + if (data2[j] != data3[j]) { + printf("%d: %d\t%d\n", j, (int) data2[j], (int) data3[j]); + } + } + test_memeq(data2, data3, 1024-16); + crypto_free_cipher_env(env1); + env1 = NULL; + crypto_free_cipher_env(env2); + env2 = NULL; + + /* NIST test vector for aes. */ + env1 = crypto_new_cipher_env(); /* IV starts at 0 */ + crypto_cipher_set_key(env1, "\x80\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00"); + crypto_cipher_encrypt_init_cipher(env1); + crypto_cipher_encrypt(env1, data1, + "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00", 16); + test_memeq_hex(data1, "0EDD33D3C621E546455BD8BA1418BEC8"); + + /* Now test rollover. All these values are originally from a python + * script. */ + crypto_cipher_set_iv(env1, "\x00\x00\x00\x00\x00\x00\x00\x00" + "\xff\xff\xff\xff\xff\xff\xff\xff"); + memset(data2, 0, 1024); + crypto_cipher_encrypt(env1, data1, data2, 32); + test_memeq_hex(data1, "335fe6da56f843199066c14a00a40231" + "cdd0b917dbc7186908a6bfb5ffd574d3"); + + crypto_cipher_set_iv(env1, "\x00\x00\x00\x00\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff"); + memset(data2, 0, 1024); + crypto_cipher_encrypt(env1, data1, data2, 32); + test_memeq_hex(data1, "e627c6423fa2d77832a02b2794094b73" + "3e63c721df790d2c6469cc1953a3ffac"); + + crypto_cipher_set_iv(env1, "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff"); + memset(data2, 0, 1024); + crypto_cipher_encrypt(env1, data1, data2, 32); + test_memeq_hex(data1, "2aed2bff0de54f9328efd070bf48f70a" + "0EDD33D3C621E546455BD8BA1418BEC8"); + + /* Now check rollover on inplace cipher. */ + crypto_cipher_set_iv(env1, "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff"); + crypto_cipher_crypt_inplace(env1, data2, 64); + test_memeq_hex(data2, "2aed2bff0de54f9328efd070bf48f70a" + "0EDD33D3C621E546455BD8BA1418BEC8" + "93e2c5243d6839eac58503919192f7ae" + "1908e67cafa08d508816659c2e693191"); + crypto_cipher_set_iv(env1, "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff"); + crypto_cipher_crypt_inplace(env1, data2, 64); + test_assert(tor_mem_is_zero(data2, 64)); + + done: + tor_free(mem_op_hex_tmp); + if (env1) + crypto_free_cipher_env(env1); + if (env2) + crypto_free_cipher_env(env2); + tor_free(data1); + tor_free(data2); + tor_free(data3); +} + +/** Run unit tests for our SHA-1 functionality */ +static void +test_crypto_sha(void) +{ + crypto_digest_env_t *d1 = NULL, *d2 = NULL; + int i; + char key[80]; + char digest[32]; + char data[50]; + char d_out1[DIGEST_LEN], d_out2[DIGEST256_LEN]; + char *mem_op_hex_tmp=NULL; + + /* Test SHA-1 with a test vector from the specification. */ + i = crypto_digest(data, "abc", 3); + test_memeq_hex(data, "A9993E364706816ABA3E25717850C26C9CD0D89D"); + + /* Test SHA-256 with a test vector from the specification. */ + i = crypto_digest256(data, "abc", 3, DIGEST_SHA256); + test_memeq_hex(data, "BA7816BF8F01CFEA414140DE5DAE2223B00361A3" + "96177A9CB410FF61F20015AD"); + + /* Test HMAC-SHA-1 with test cases from RFC2202. */ + + /* Case 1. */ + memset(key, 0x0b, 20); + crypto_hmac_sha1(digest, key, 20, "Hi There", 8); + test_streq(hex_str(digest, 20), + "B617318655057264E28BC0B6FB378C8EF146BE00"); + /* Case 2. */ + crypto_hmac_sha1(digest, "Jefe", 4, "what do ya want for nothing?", 28); + test_streq(hex_str(digest, 20), + "EFFCDF6AE5EB2FA2D27416D5F184DF9C259A7C79"); + + /* Case 4. */ + base16_decode(key, 25, + "0102030405060708090a0b0c0d0e0f10111213141516171819", 50); + memset(data, 0xcd, 50); + crypto_hmac_sha1(digest, key, 25, data, 50); + test_streq(hex_str(digest, 20), + "4C9007F4026250C6BC8414F9BF50C86C2D7235DA"); + + /* Case 5. */ + memset(key, 0xaa, 80); + crypto_hmac_sha1(digest, key, 80, + "Test Using Larger Than Block-Size Key - Hash Key First", + 54); + test_streq(hex_str(digest, 20), + "AA4AE5E15272D00E95705637CE8A3B55ED402112"); + + /* Incremental digest code. */ + d1 = crypto_new_digest_env(); + test_assert(d1); + crypto_digest_add_bytes(d1, "abcdef", 6); + d2 = crypto_digest_dup(d1); + test_assert(d2); + crypto_digest_add_bytes(d2, "ghijkl", 6); + crypto_digest_get_digest(d2, d_out1, sizeof(d_out1)); + crypto_digest(d_out2, "abcdefghijkl", 12); + test_memeq(d_out1, d_out2, DIGEST_LEN); + crypto_digest_assign(d2, d1); + crypto_digest_add_bytes(d2, "mno", 3); + crypto_digest_get_digest(d2, d_out1, sizeof(d_out1)); + crypto_digest(d_out2, "abcdefmno", 9); + test_memeq(d_out1, d_out2, DIGEST_LEN); + crypto_digest_get_digest(d1, d_out1, sizeof(d_out1)); + crypto_digest(d_out2, "abcdef", 6); + test_memeq(d_out1, d_out2, DIGEST_LEN); + crypto_free_digest_env(d1); + crypto_free_digest_env(d2); + + /* Incremental digest code with sha256 */ + d1 = crypto_new_digest256_env(DIGEST_SHA256); + test_assert(d1); + crypto_digest_add_bytes(d1, "abcdef", 6); + d2 = crypto_digest_dup(d1); + test_assert(d2); + crypto_digest_add_bytes(d2, "ghijkl", 6); + crypto_digest_get_digest(d2, d_out1, sizeof(d_out1)); + crypto_digest256(d_out2, "abcdefghijkl", 12, DIGEST_SHA256); + test_memeq(d_out1, d_out2, DIGEST_LEN); + crypto_digest_assign(d2, d1); + crypto_digest_add_bytes(d2, "mno", 3); + crypto_digest_get_digest(d2, d_out1, sizeof(d_out1)); + crypto_digest256(d_out2, "abcdefmno", 9, DIGEST_SHA256); + test_memeq(d_out1, d_out2, DIGEST_LEN); + crypto_digest_get_digest(d1, d_out1, sizeof(d_out1)); + crypto_digest256(d_out2, "abcdef", 6, DIGEST_SHA256); + test_memeq(d_out1, d_out2, DIGEST_LEN); + + done: + if (d1) + crypto_free_digest_env(d1); + if (d2) + crypto_free_digest_env(d2); + tor_free(mem_op_hex_tmp); +} + +/** Run unit tests for our public key crypto functions */ +static void +test_crypto_pk(void) +{ + crypto_pk_env_t *pk1 = NULL, *pk2 = NULL; + char *encoded = NULL; + char data1[1024], data2[1024], data3[1024]; + size_t size; + int i, j, p, len; + + /* Public-key ciphers */ + pk1 = pk_generate(0); + pk2 = crypto_new_pk_env(); + test_assert(pk1 && pk2); + test_assert(! crypto_pk_write_public_key_to_string(pk1, &encoded, &size)); + test_assert(! crypto_pk_read_public_key_from_string(pk2, encoded, size)); + test_eq(0, crypto_pk_cmp_keys(pk1, pk2)); + + test_eq(128, crypto_pk_keysize(pk1)); + test_eq(128, crypto_pk_keysize(pk2)); + + test_eq(128, crypto_pk_public_encrypt(pk2, data1, sizeof(data1), + "Hello whirled.", 15, + PK_PKCS1_OAEP_PADDING)); + test_eq(128, crypto_pk_public_encrypt(pk1, data2, sizeof(data1), + "Hello whirled.", 15, + PK_PKCS1_OAEP_PADDING)); + /* oaep padding should make encryption not match */ + test_memneq(data1, data2, 128); + test_eq(15, crypto_pk_private_decrypt(pk1, data3, sizeof(data3), data1, 128, + PK_PKCS1_OAEP_PADDING,1)); + test_streq(data3, "Hello whirled."); + memset(data3, 0, 1024); + test_eq(15, crypto_pk_private_decrypt(pk1, data3, sizeof(data3), data2, 128, + PK_PKCS1_OAEP_PADDING,1)); + test_streq(data3, "Hello whirled."); + /* Can't decrypt with public key. */ + test_eq(-1, crypto_pk_private_decrypt(pk2, data3, sizeof(data3), data2, 128, + PK_PKCS1_OAEP_PADDING,1)); + /* Try again with bad padding */ + memcpy(data2+1, "XYZZY", 5); /* This has fails ~ once-in-2^40 */ + test_eq(-1, crypto_pk_private_decrypt(pk1, data3, sizeof(data3), data2, 128, + PK_PKCS1_OAEP_PADDING,1)); + + /* File operations: save and load private key */ + test_assert(! crypto_pk_write_private_key_to_filename(pk1, + get_fname("pkey1"))); + /* failing case for read: can't read. */ + test_assert(crypto_pk_read_private_key_from_filename(pk2, + get_fname("xyzzy")) < 0); + write_str_to_file(get_fname("xyzzy"), "foobar", 6); + /* Failing case for read: no key. */ + test_assert(crypto_pk_read_private_key_from_filename(pk2, + get_fname("xyzzy")) < 0); + test_assert(! crypto_pk_read_private_key_from_filename(pk2, + get_fname("pkey1"))); + test_eq(15, crypto_pk_private_decrypt(pk2, data3, sizeof(data3), data1, 128, + PK_PKCS1_OAEP_PADDING,1)); + + /* Now try signing. */ + strlcpy(data1, "Ossifrage", 1024); + test_eq(128, crypto_pk_private_sign(pk1, data2, sizeof(data2), data1, 10)); + test_eq(10, + crypto_pk_public_checksig(pk1, data3, sizeof(data3), data2, 128)); + test_streq(data3, "Ossifrage"); + /* Try signing digests. */ + test_eq(128, crypto_pk_private_sign_digest(pk1, data2, sizeof(data2), + data1, 10)); + test_eq(20, + crypto_pk_public_checksig(pk1, data3, sizeof(data3), data2, 128)); + test_eq(0, crypto_pk_public_checksig_digest(pk1, data1, 10, data2, 128)); + test_eq(-1, crypto_pk_public_checksig_digest(pk1, data1, 11, data2, 128)); + + /*XXXX test failed signing*/ + + /* Try encoding */ + crypto_free_pk_env(pk2); + pk2 = NULL; + i = crypto_pk_asn1_encode(pk1, data1, 1024); + test_assert(i>0); + pk2 = crypto_pk_asn1_decode(data1, i); + test_assert(crypto_pk_cmp_keys(pk1,pk2) == 0); + + /* Try with hybrid encryption wrappers. */ + crypto_rand(data1, 1024); + for (i = 0; i < 3; ++i) { + for (j = 85; j < 140; ++j) { + memset(data2,0,1024); + memset(data3,0,1024); + if (i == 0 && j < 129) + continue; + p = (i==0)?PK_NO_PADDING: + (i==1)?PK_PKCS1_PADDING:PK_PKCS1_OAEP_PADDING; + len = crypto_pk_public_hybrid_encrypt(pk1,data2,sizeof(data2), + data1,j,p,0); + test_assert(len>=0); + len = crypto_pk_private_hybrid_decrypt(pk1,data3,sizeof(data3), + data2,len,p,1); + test_eq(len,j); + test_memeq(data1,data3,j); + } + } + + /* Try copy_full */ + crypto_free_pk_env(pk2); + pk2 = crypto_pk_copy_full(pk1); + test_assert(pk2 != NULL); + test_neq_ptr(pk1, pk2); + test_assert(crypto_pk_cmp_keys(pk1,pk2) == 0); + + done: + if (pk1) + crypto_free_pk_env(pk1); + if (pk2) + crypto_free_pk_env(pk2); + tor_free(encoded); +} + +/** Run unit tests for misc crypto formatting functionality (base64, base32, + * fingerprints, etc) */ +static void +test_crypto_formats(void) +{ + char *data1 = NULL, *data2 = NULL, *data3 = NULL; + int i, j, idx; + + data1 = tor_malloc(1024); + data2 = tor_malloc(1024); + data3 = tor_malloc(1024); + test_assert(data1 && data2 && data3); + + /* Base64 tests */ + memset(data1, 6, 1024); + for (idx = 0; idx < 10; ++idx) { + i = base64_encode(data2, 1024, data1, idx); + test_assert(i >= 0); + j = base64_decode(data3, 1024, data2, i); + test_eq(j,idx); + test_memeq(data3, data1, idx); + } + + strlcpy(data1, "Test string that contains 35 chars.", 1024); + strlcat(data1, " 2nd string that contains 35 chars.", 1024); + + i = base64_encode(data2, 1024, data1, 71); + test_assert(i >= 0); + j = base64_decode(data3, 1024, data2, i); + test_eq(j, 71); + test_streq(data3, data1); + test_assert(data2[i] == '\0'); + + crypto_rand(data1, DIGEST_LEN); + memset(data2, 100, 1024); + digest_to_base64(data2, data1); + test_eq(BASE64_DIGEST_LEN, strlen(data2)); + test_eq(100, data2[BASE64_DIGEST_LEN+2]); + memset(data3, 99, 1024); + test_eq(digest_from_base64(data3, data2), 0); + test_memeq(data1, data3, DIGEST_LEN); + test_eq(99, data3[DIGEST_LEN+1]); + + test_assert(digest_from_base64(data3, "###") < 0); + + /* Encoding SHA256 */ + crypto_rand(data2, DIGEST256_LEN); + memset(data2, 100, 1024); + digest256_to_base64(data2, data1); + test_eq(BASE64_DIGEST256_LEN, strlen(data2)); + test_eq(100, data2[BASE64_DIGEST256_LEN+2]); + memset(data3, 99, 1024); + test_eq(digest256_from_base64(data3, data2), 0); + test_memeq(data1, data3, DIGEST256_LEN); + test_eq(99, data3[DIGEST256_LEN+1]); + + /* Base32 tests */ + strlcpy(data1, "5chrs", 1024); + /* bit pattern is: [35 63 68 72 73] -> + * [00110101 01100011 01101000 01110010 01110011] + * By 5s: [00110 10101 10001 10110 10000 11100 10011 10011] + */ + base32_encode(data2, 9, data1, 5); + test_streq(data2, "gvrwq4tt"); + + strlcpy(data1, "\xFF\xF5\x6D\x44\xAE\x0D\x5C\xC9\x62\xC4", 1024); + base32_encode(data2, 30, data1, 10); + test_streq(data2, "772w2rfobvomsywe"); + + /* Base16 tests */ + strlcpy(data1, "6chrs\xff", 1024); + base16_encode(data2, 13, data1, 6); + test_streq(data2, "3663687273FF"); + + strlcpy(data1, "f0d678affc000100", 1024); + i = base16_decode(data2, 8, data1, 16); + test_eq(i,0); + test_memeq(data2, "\xf0\xd6\x78\xaf\xfc\x00\x01\x00",8); + + /* now try some failing base16 decodes */ + test_eq(-1, base16_decode(data2, 8, data1, 15)); /* odd input len */ + test_eq(-1, base16_decode(data2, 7, data1, 16)); /* dest too short */ + strlcpy(data1, "f0dz!8affc000100", 1024); + test_eq(-1, base16_decode(data2, 8, data1, 16)); + + tor_free(data1); + tor_free(data2); + tor_free(data3); + + /* Add spaces to fingerprint */ + { + data1 = tor_strdup("ABCD1234ABCD56780000ABCD1234ABCD56780000"); + test_eq(strlen(data1), 40); + data2 = tor_malloc(FINGERPRINT_LEN+1); + add_spaces_to_fp(data2, FINGERPRINT_LEN+1, data1); + test_streq(data2, "ABCD 1234 ABCD 5678 0000 ABCD 1234 ABCD 5678 0000"); + tor_free(data1); + tor_free(data2); + } + + /* Check fingerprint */ + { + test_assert(crypto_pk_check_fingerprint_syntax( + "ABCD 1234 ABCD 5678 0000 ABCD 1234 ABCD 5678 0000")); + test_assert(!crypto_pk_check_fingerprint_syntax( + "ABCD 1234 ABCD 5678 0000 ABCD 1234 ABCD 5678 000")); + test_assert(!crypto_pk_check_fingerprint_syntax( + "ABCD 1234 ABCD 5678 0000 ABCD 1234 ABCD 5678 00000")); + test_assert(!crypto_pk_check_fingerprint_syntax( + "ABCD 1234 ABCD 5678 0000 ABCD1234 ABCD 5678 0000")); + test_assert(!crypto_pk_check_fingerprint_syntax( + "ABCD 1234 ABCD 5678 0000 ABCD1234 ABCD 5678 00000")); + test_assert(!crypto_pk_check_fingerprint_syntax( + "ACD 1234 ABCD 5678 0000 ABCD 1234 ABCD 5678 00000")); + } + + done: + tor_free(data1); + tor_free(data2); + tor_free(data3); +} + +/** Run unit tests for our secret-to-key passphrase hashing functionality. */ +static void +test_crypto_s2k(void) +{ + char buf[29]; + char buf2[29]; + char *buf3 = NULL; + int i; + + memset(buf, 0, sizeof(buf)); + memset(buf2, 0, sizeof(buf2)); + buf3 = tor_malloc(65536); + memset(buf3, 0, 65536); + + secret_to_key(buf+9, 20, "", 0, buf); + crypto_digest(buf2+9, buf3, 1024); + test_memeq(buf, buf2, 29); + + memcpy(buf,"vrbacrda",8); + memcpy(buf2,"vrbacrda",8); + buf[8] = 96; + buf2[8] = 96; + secret_to_key(buf+9, 20, "12345678", 8, buf); + for (i = 0; i < 65536; i += 16) { + memcpy(buf3+i, "vrbacrda12345678", 16); + } + crypto_digest(buf2+9, buf3, 65536); + test_memeq(buf, buf2, 29); + + done: + tor_free(buf3); +} + +/** Test AES-CTR encryption and decryption with IV. */ +static void +test_crypto_aes_iv(void) +{ + crypto_cipher_env_t *cipher; + char *plain, *encrypted1, *encrypted2, *decrypted1, *decrypted2; + char plain_1[1], plain_15[15], plain_16[16], plain_17[17]; + char key1[16], key2[16]; + ssize_t encrypted_size, decrypted_size; + + plain = tor_malloc(4095); + encrypted1 = tor_malloc(4095 + 1 + 16); + encrypted2 = tor_malloc(4095 + 1 + 16); + decrypted1 = tor_malloc(4095 + 1); + decrypted2 = tor_malloc(4095 + 1); + + crypto_rand(plain, 4095); + crypto_rand(key1, 16); + crypto_rand(key2, 16); + crypto_rand(plain_1, 1); + crypto_rand(plain_15, 15); + crypto_rand(plain_16, 16); + crypto_rand(plain_17, 17); + key1[0] = key2[0] + 128; /* Make sure that contents are different. */ + /* Encrypt and decrypt with the same key. */ + cipher = crypto_create_init_cipher(key1, 1); + encrypted_size = crypto_cipher_encrypt_with_iv(cipher, encrypted1, 16 + 4095, + plain, 4095); + crypto_free_cipher_env(cipher); + cipher = NULL; + test_eq(encrypted_size, 16 + 4095); + tor_assert(encrypted_size > 0); /* This is obviously true, since 4111 is + * greater than 0, but its truth is not + * obvious to all analysis tools. */ + cipher = crypto_create_init_cipher(key1, 0); + decrypted_size = crypto_cipher_decrypt_with_iv(cipher, decrypted1, 4095, + encrypted1, encrypted_size); + crypto_free_cipher_env(cipher); + cipher = NULL; + test_eq(decrypted_size, 4095); + tor_assert(decrypted_size > 0); + test_memeq(plain, decrypted1, 4095); + /* Encrypt a second time (with a new random initialization vector). */ + cipher = crypto_create_init_cipher(key1, 1); + encrypted_size = crypto_cipher_encrypt_with_iv(cipher, encrypted2, 16 + 4095, + plain, 4095); + crypto_free_cipher_env(cipher); + cipher = NULL; + test_eq(encrypted_size, 16 + 4095); + tor_assert(encrypted_size > 0); + cipher = crypto_create_init_cipher(key1, 0); + decrypted_size = crypto_cipher_decrypt_with_iv(cipher, decrypted2, 4095, + encrypted2, encrypted_size); + crypto_free_cipher_env(cipher); + cipher = NULL; + test_eq(decrypted_size, 4095); + tor_assert(decrypted_size > 0); + test_memeq(plain, decrypted2, 4095); + test_memneq(encrypted1, encrypted2, encrypted_size); + /* Decrypt with the wrong key. */ + cipher = crypto_create_init_cipher(key2, 0); + decrypted_size = crypto_cipher_decrypt_with_iv(cipher, decrypted2, 4095, + encrypted1, encrypted_size); + crypto_free_cipher_env(cipher); + cipher = NULL; + test_memneq(plain, decrypted2, encrypted_size); + /* Alter the initialization vector. */ + encrypted1[0] += 42; + cipher = crypto_create_init_cipher(key1, 0); + decrypted_size = crypto_cipher_decrypt_with_iv(cipher, decrypted1, 4095, + encrypted1, encrypted_size); + crypto_free_cipher_env(cipher); + cipher = NULL; + test_memneq(plain, decrypted2, 4095); + /* Special length case: 1. */ + cipher = crypto_create_init_cipher(key1, 1); + encrypted_size = crypto_cipher_encrypt_with_iv(cipher, encrypted1, 16 + 1, + plain_1, 1); + crypto_free_cipher_env(cipher); + cipher = NULL; + test_eq(encrypted_size, 16 + 1); + tor_assert(encrypted_size > 0); + cipher = crypto_create_init_cipher(key1, 0); + decrypted_size = crypto_cipher_decrypt_with_iv(cipher, decrypted1, 1, + encrypted1, encrypted_size); + crypto_free_cipher_env(cipher); + cipher = NULL; + test_eq(decrypted_size, 1); + tor_assert(decrypted_size > 0); + test_memeq(plain_1, decrypted1, 1); + /* Special length case: 15. */ + cipher = crypto_create_init_cipher(key1, 1); + encrypted_size = crypto_cipher_encrypt_with_iv(cipher, encrypted1, 16 + 15, + plain_15, 15); + crypto_free_cipher_env(cipher); + cipher = NULL; + test_eq(encrypted_size, 16 + 15); + tor_assert(encrypted_size > 0); + cipher = crypto_create_init_cipher(key1, 0); + decrypted_size = crypto_cipher_decrypt_with_iv(cipher, decrypted1, 15, + encrypted1, encrypted_size); + crypto_free_cipher_env(cipher); + cipher = NULL; + test_eq(decrypted_size, 15); + tor_assert(decrypted_size > 0); + test_memeq(plain_15, decrypted1, 15); + /* Special length case: 16. */ + cipher = crypto_create_init_cipher(key1, 1); + encrypted_size = crypto_cipher_encrypt_with_iv(cipher, encrypted1, 16 + 16, + plain_16, 16); + crypto_free_cipher_env(cipher); + cipher = NULL; + test_eq(encrypted_size, 16 + 16); + tor_assert(encrypted_size > 0); + cipher = crypto_create_init_cipher(key1, 0); + decrypted_size = crypto_cipher_decrypt_with_iv(cipher, decrypted1, 16, + encrypted1, encrypted_size); + crypto_free_cipher_env(cipher); + cipher = NULL; + test_eq(decrypted_size, 16); + tor_assert(decrypted_size > 0); + test_memeq(plain_16, decrypted1, 16); + /* Special length case: 17. */ + cipher = crypto_create_init_cipher(key1, 1); + encrypted_size = crypto_cipher_encrypt_with_iv(cipher, encrypted1, 16 + 17, + plain_17, 17); + crypto_free_cipher_env(cipher); + cipher = NULL; + test_eq(encrypted_size, 16 + 17); + tor_assert(encrypted_size > 0); + cipher = crypto_create_init_cipher(key1, 0); + decrypted_size = crypto_cipher_decrypt_with_iv(cipher, decrypted1, 17, + encrypted1, encrypted_size); + test_eq(decrypted_size, 17); + tor_assert(decrypted_size > 0); + test_memeq(plain_17, decrypted1, 17); + + done: + /* Free memory. */ + tor_free(plain); + tor_free(encrypted1); + tor_free(encrypted2); + tor_free(decrypted1); + tor_free(decrypted2); + if (cipher) + crypto_free_cipher_env(cipher); +} + +/** Test base32 decoding. */ +static void +test_crypto_base32_decode(void) +{ + char plain[60], encoded[96 + 1], decoded[60]; + int res; + crypto_rand(plain, 60); + /* Encode and decode a random string. */ + base32_encode(encoded, 96 + 1, plain, 60); + res = base32_decode(decoded, 60, encoded, 96); + test_eq(res, 0); + test_memeq(plain, decoded, 60); + /* Encode, uppercase, and decode a random string. */ + base32_encode(encoded, 96 + 1, plain, 60); + tor_strupper(encoded); + res = base32_decode(decoded, 60, encoded, 96); + test_eq(res, 0); + test_memeq(plain, decoded, 60); + /* Change encoded string and decode. */ + if (encoded[0] == 'A' || encoded[0] == 'a') + encoded[0] = 'B'; + else + encoded[0] = 'A'; + res = base32_decode(decoded, 60, encoded, 96); + test_eq(res, 0); + test_memneq(plain, decoded, 60); + /* Bad encodings. */ + encoded[0] = '!'; + res = base32_decode(decoded, 60, encoded, 96); + test_assert(res < 0); + + done: + ; +} + +#define CRYPTO_LEGACY(name) \ + { #name, legacy_test_helper, 0, &legacy_setup, test_crypto_ ## name } + +struct testcase_t crypto_tests[] = { + CRYPTO_LEGACY(formats), + CRYPTO_LEGACY(rng), + CRYPTO_LEGACY(aes), + CRYPTO_LEGACY(sha), + CRYPTO_LEGACY(pk), + CRYPTO_LEGACY(dh), + CRYPTO_LEGACY(s2k), + CRYPTO_LEGACY(aes_iv), + CRYPTO_LEGACY(base32_decode), + END_OF_TESTCASES +}; + diff --git a/src/or/test_data.c b/src/test/test_data.c index fc85857615..fc85857615 100644 --- a/src/or/test_data.c +++ b/src/test/test_data.c diff --git a/src/test/test_dir.c b/src/test/test_dir.c new file mode 100644 index 0000000000..8fd94289a9 --- /dev/null +++ b/src/test/test_dir.c @@ -0,0 +1,1320 @@ +/* Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "orconfig.h" +#define DIRSERV_PRIVATE +#define DIRVOTE_PRIVATE +#define ROUTER_PRIVATE +#include "or.h" +#include "directory.h" +#include "dirserv.h" +#include "dirvote.h" +#include "networkstatus.h" +#include "router.h" +#include "routerlist.h" +#include "routerparse.h" +#include "test.h" + +static void +test_dir_nicknames(void) +{ + test_assert( is_legal_nickname("a")); + test_assert(!is_legal_nickname("")); + test_assert(!is_legal_nickname("abcdefghijklmnopqrst")); /* 20 chars */ + test_assert(!is_legal_nickname("hyphen-")); /* bad char */ + test_assert( is_legal_nickname("abcdefghijklmnopqrs")); /* 19 chars */ + test_assert(!is_legal_nickname("$AAAAAAAA01234AAAAAAAAAAAAAAAAAAAAAAAAAAA")); + /* valid */ + test_assert( is_legal_nickname_or_hexdigest( + "$AAAAAAAA01234AAAAAAAAAAAAAAAAAAAAAAAAAAA")); + test_assert( is_legal_nickname_or_hexdigest( + "$AAAAAAAA01234AAAAAAAAAAAAAAAAAAAAAAAAAAA=fred")); + test_assert( is_legal_nickname_or_hexdigest( + "$AAAAAAAA01234AAAAAAAAAAAAAAAAAAAAAAAAAAA~fred")); + /* too short */ + test_assert(!is_legal_nickname_or_hexdigest( + "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")); + /* illegal char */ + test_assert(!is_legal_nickname_or_hexdigest( + "$AAAAAAzAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")); + /* hex part too long */ + test_assert(!is_legal_nickname_or_hexdigest( + "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")); + test_assert(!is_legal_nickname_or_hexdigest( + "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=fred")); + /* Bad nickname */ + test_assert(!is_legal_nickname_or_hexdigest( + "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=")); + test_assert(!is_legal_nickname_or_hexdigest( + "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~")); + test_assert(!is_legal_nickname_or_hexdigest( + "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~hyphen-")); + test_assert(!is_legal_nickname_or_hexdigest( + "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~" + "abcdefghijklmnoppqrst")); + /* Bad extra char. */ + test_assert(!is_legal_nickname_or_hexdigest( + "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA!")); + test_assert(is_legal_nickname_or_hexdigest("xyzzy")); + test_assert(is_legal_nickname_or_hexdigest("abcdefghijklmnopqrs")); + test_assert(!is_legal_nickname_or_hexdigest("abcdefghijklmnopqrst")); + done: + ; +} + +/** Run unit tests for router descriptor generation logic. */ +static void +test_dir_formats(void) +{ + char buf[8192], buf2[8192]; + char platform[256]; + char fingerprint[FINGERPRINT_LEN+1]; + char *pk1_str = NULL, *pk2_str = NULL, *pk3_str = NULL, *cp; + size_t pk1_str_len, pk2_str_len, pk3_str_len; + routerinfo_t *r1=NULL, *r2=NULL; + crypto_pk_env_t *pk1 = NULL, *pk2 = NULL, *pk3 = NULL; + routerinfo_t *rp1 = NULL; + addr_policy_t *ex1, *ex2; + routerlist_t *dir1 = NULL, *dir2 = NULL; + + pk1 = pk_generate(0); + pk2 = pk_generate(1); + pk3 = pk_generate(2); + + test_assert(pk1 && pk2 && pk3); + + get_platform_str(platform, sizeof(platform)); + r1 = tor_malloc_zero(sizeof(routerinfo_t)); + r1->address = tor_strdup("18.244.0.1"); + r1->addr = 0xc0a80001u; /* 192.168.0.1 */ + r1->cache_info.published_on = 0; + r1->or_port = 9000; + r1->dir_port = 9003; + r1->onion_pkey = crypto_pk_dup_key(pk1); + r1->identity_pkey = crypto_pk_dup_key(pk2); + r1->bandwidthrate = 1000; + r1->bandwidthburst = 5000; + r1->bandwidthcapacity = 10000; + r1->exit_policy = NULL; + r1->nickname = tor_strdup("Magri"); + r1->platform = tor_strdup(platform); + + ex1 = tor_malloc_zero(sizeof(addr_policy_t)); + ex2 = tor_malloc_zero(sizeof(addr_policy_t)); + ex1->policy_type = ADDR_POLICY_ACCEPT; + tor_addr_from_ipv4h(&ex1->addr, 0); + ex1->maskbits = 0; + ex1->prt_min = ex1->prt_max = 80; + ex2->policy_type = ADDR_POLICY_REJECT; + tor_addr_from_ipv4h(&ex2->addr, 18<<24); + ex2->maskbits = 8; + ex2->prt_min = ex2->prt_max = 24; + r2 = tor_malloc_zero(sizeof(routerinfo_t)); + r2->address = tor_strdup("1.1.1.1"); + r2->addr = 0x0a030201u; /* 10.3.2.1 */ + r2->platform = tor_strdup(platform); + r2->cache_info.published_on = 5; + r2->or_port = 9005; + r2->dir_port = 0; + r2->onion_pkey = crypto_pk_dup_key(pk2); + r2->identity_pkey = crypto_pk_dup_key(pk1); + r2->bandwidthrate = r2->bandwidthburst = r2->bandwidthcapacity = 3000; + r2->exit_policy = smartlist_create(); + smartlist_add(r2->exit_policy, ex2); + smartlist_add(r2->exit_policy, ex1); + r2->nickname = tor_strdup("Fred"); + + test_assert(!crypto_pk_write_public_key_to_string(pk1, &pk1_str, + &pk1_str_len)); + test_assert(!crypto_pk_write_public_key_to_string(pk2 , &pk2_str, + &pk2_str_len)); + test_assert(!crypto_pk_write_public_key_to_string(pk3 , &pk3_str, + &pk3_str_len)); + + memset(buf, 0, 2048); + test_assert(router_dump_router_to_string(buf, 2048, r1, pk2)>0); + + strlcpy(buf2, "router Magri 18.244.0.1 9000 0 9003\n" + "platform Tor "VERSION" on ", sizeof(buf2)); + strlcat(buf2, get_uname(), sizeof(buf2)); + strlcat(buf2, "\n" + "opt protocols Link 1 2 Circuit 1\n" + "published 1970-01-01 00:00:00\n" + "opt fingerprint ", sizeof(buf2)); + test_assert(!crypto_pk_get_fingerprint(pk2, fingerprint, 1)); + strlcat(buf2, fingerprint, sizeof(buf2)); + strlcat(buf2, "\nuptime 0\n" + /* XXX the "0" above is hard-coded, but even if we made it reflect + * uptime, that still wouldn't make it right, because the two + * descriptors might be made on different seconds... hm. */ + "bandwidth 1000 5000 10000\n" + "onion-key\n", sizeof(buf2)); + strlcat(buf2, pk1_str, sizeof(buf2)); + strlcat(buf2, "signing-key\n", sizeof(buf2)); + strlcat(buf2, pk2_str, sizeof(buf2)); + strlcat(buf2, "opt hidden-service-dir\n", sizeof(buf2)); + strlcat(buf2, "reject *:*\nrouter-signature\n", sizeof(buf2)); + buf[strlen(buf2)] = '\0'; /* Don't compare the sig; it's never the same + * twice */ + + test_streq(buf, buf2); + + test_assert(router_dump_router_to_string(buf, 2048, r1, pk2)>0); + cp = buf; + rp1 = router_parse_entry_from_string((const char*)cp,NULL,1,0,NULL); + test_assert(rp1); + test_streq(rp1->address, r1->address); + test_eq(rp1->or_port, r1->or_port); + //test_eq(rp1->dir_port, r1->dir_port); + test_eq(rp1->bandwidthrate, r1->bandwidthrate); + test_eq(rp1->bandwidthburst, r1->bandwidthburst); + test_eq(rp1->bandwidthcapacity, r1->bandwidthcapacity); + test_assert(crypto_pk_cmp_keys(rp1->onion_pkey, pk1) == 0); + test_assert(crypto_pk_cmp_keys(rp1->identity_pkey, pk2) == 0); + //test_assert(rp1->exit_policy == NULL); + +#if 0 + /* XXX Once we have exit policies, test this again. XXX */ + strlcpy(buf2, "router tor.tor.tor 9005 0 0 3000\n", sizeof(buf2)); + strlcat(buf2, pk2_str, sizeof(buf2)); + strlcat(buf2, "signing-key\n", sizeof(buf2)); + strlcat(buf2, pk1_str, sizeof(buf2)); + strlcat(buf2, "accept *:80\nreject 18.*:24\n\n", sizeof(buf2)); + test_assert(router_dump_router_to_string(buf, 2048, &r2, pk2)>0); + test_streq(buf, buf2); + + cp = buf; + rp2 = router_parse_entry_from_string(&cp,1); + test_assert(rp2); + test_streq(rp2->address, r2.address); + test_eq(rp2->or_port, r2.or_port); + test_eq(rp2->dir_port, r2.dir_port); + test_eq(rp2->bandwidth, r2.bandwidth); + test_assert(crypto_pk_cmp_keys(rp2->onion_pkey, pk2) == 0); + test_assert(crypto_pk_cmp_keys(rp2->identity_pkey, pk1) == 0); + test_eq(rp2->exit_policy->policy_type, EXIT_POLICY_ACCEPT); + test_streq(rp2->exit_policy->string, "accept *:80"); + test_streq(rp2->exit_policy->address, "*"); + test_streq(rp2->exit_policy->port, "80"); + test_eq(rp2->exit_policy->next->policy_type, EXIT_POLICY_REJECT); + test_streq(rp2->exit_policy->next->string, "reject 18.*:24"); + test_streq(rp2->exit_policy->next->address, "18.*"); + test_streq(rp2->exit_policy->next->port, "24"); + test_assert(rp2->exit_policy->next->next == NULL); + + /* Okay, now for the directories. */ + { + fingerprint_list = smartlist_create(); + crypto_pk_get_fingerprint(pk2, buf, 1); + add_fingerprint_to_dir("Magri", buf, fingerprint_list); + crypto_pk_get_fingerprint(pk1, buf, 1); + add_fingerprint_to_dir("Fred", buf, fingerprint_list); + } + + { + char d[DIGEST_LEN]; + const char *m; + /* XXXX NM re-enable. */ + /* Make sure routers aren't too far in the past any more. */ + r1->cache_info.published_on = time(NULL); + r2->cache_info.published_on = time(NULL)-3*60*60; + test_assert(router_dump_router_to_string(buf, 2048, r1, pk2)>0); + test_eq(dirserv_add_descriptor(buf,&m,""), ROUTER_ADDED_NOTIFY_GENERATOR); + test_assert(router_dump_router_to_string(buf, 2048, r2, pk1)>0); + test_eq(dirserv_add_descriptor(buf,&m,""), ROUTER_ADDED_NOTIFY_GENERATOR); + get_options()->Nickname = tor_strdup("DirServer"); + test_assert(!dirserv_dump_directory_to_string(&cp,pk3, 0)); + crypto_pk_get_digest(pk3, d); + test_assert(!router_parse_directory(cp)); + test_eq(2, smartlist_len(dir1->routers)); + tor_free(cp); + } +#endif + dirserv_free_fingerprint_list(); + + done: + if (r1) + routerinfo_free(r1); + if (r2) + routerinfo_free(r2); + + tor_free(pk1_str); + tor_free(pk2_str); + tor_free(pk3_str); + if (pk1) crypto_free_pk_env(pk1); + if (pk2) crypto_free_pk_env(pk2); + if (pk3) crypto_free_pk_env(pk3); + if (rp1) routerinfo_free(rp1); + tor_free(dir1); /* XXXX And more !*/ + tor_free(dir2); /* And more !*/ +} + +static void +test_dir_versions(void) +{ + tor_version_t ver1; + + /* Try out version parsing functionality */ + test_eq(0, tor_version_parse("0.3.4pre2-cvs", &ver1)); + test_eq(0, ver1.major); + test_eq(3, ver1.minor); + test_eq(4, ver1.micro); + test_eq(VER_PRE, ver1.status); + test_eq(2, ver1.patchlevel); + test_eq(0, tor_version_parse("0.3.4rc1", &ver1)); + test_eq(0, ver1.major); + test_eq(3, ver1.minor); + test_eq(4, ver1.micro); + test_eq(VER_RC, ver1.status); + test_eq(1, ver1.patchlevel); + test_eq(0, tor_version_parse("1.3.4", &ver1)); + test_eq(1, ver1.major); + test_eq(3, ver1.minor); + test_eq(4, ver1.micro); + test_eq(VER_RELEASE, ver1.status); + test_eq(0, ver1.patchlevel); + test_eq(0, tor_version_parse("1.3.4.999", &ver1)); + test_eq(1, ver1.major); + test_eq(3, ver1.minor); + test_eq(4, ver1.micro); + test_eq(VER_RELEASE, ver1.status); + test_eq(999, ver1.patchlevel); + test_eq(0, tor_version_parse("0.1.2.4-alpha", &ver1)); + test_eq(0, ver1.major); + test_eq(1, ver1.minor); + test_eq(2, ver1.micro); + test_eq(4, ver1.patchlevel); + test_eq(VER_RELEASE, ver1.status); + test_streq("alpha", ver1.status_tag); + test_eq(0, tor_version_parse("0.1.2.4", &ver1)); + test_eq(0, ver1.major); + test_eq(1, ver1.minor); + test_eq(2, ver1.micro); + test_eq(4, ver1.patchlevel); + test_eq(VER_RELEASE, ver1.status); + test_streq("", ver1.status_tag); + +#define tt_versionstatus_op(vs1, op, vs2) \ + tt_assert_test_type(vs1,vs2,#vs1" "#op" "#vs2,version_status_t, \ + (_val1 op _val2),"%d") +#define test_v_i_o(val, ver, lst) \ + tt_versionstatus_op(val, ==, tor_version_is_obsolete(ver, lst)) + + /* make sure tor_version_is_obsolete() works */ + test_v_i_o(VS_OLD, "0.0.1", "Tor 0.0.2"); + test_v_i_o(VS_OLD, "0.0.1", "0.0.2, Tor 0.0.3"); + test_v_i_o(VS_OLD, "0.0.1", "0.0.2,Tor 0.0.3"); + test_v_i_o(VS_OLD, "0.0.1","0.0.3,BetterTor 0.0.1"); + test_v_i_o(VS_RECOMMENDED, "0.0.2", "Tor 0.0.2,Tor 0.0.3"); + test_v_i_o(VS_NEW_IN_SERIES, "0.0.2", "Tor 0.0.2pre1,Tor 0.0.3"); + test_v_i_o(VS_OLD, "0.0.2", "Tor 0.0.2.1,Tor 0.0.3"); + test_v_i_o(VS_NEW, "0.1.0", "Tor 0.0.2,Tor 0.0.3"); + test_v_i_o(VS_RECOMMENDED, "0.0.7rc2", "0.0.7,Tor 0.0.7rc2,Tor 0.0.8"); + test_v_i_o(VS_OLD, "0.0.5.0", "0.0.5.1-cvs"); + test_v_i_o(VS_NEW_IN_SERIES, "0.0.5.1-cvs", "0.0.5, 0.0.6"); + /* Not on list, but newer than any in same series. */ + test_v_i_o(VS_NEW_IN_SERIES, "0.1.0.3", + "Tor 0.1.0.2,Tor 0.0.9.5,Tor 0.1.1.0"); + /* Series newer than any on list. */ + test_v_i_o(VS_NEW, "0.1.2.3", "Tor 0.1.0.2,Tor 0.0.9.5,Tor 0.1.1.0"); + /* Series older than any on list. */ + test_v_i_o(VS_OLD, "0.0.1.3", "Tor 0.1.0.2,Tor 0.0.9.5,Tor 0.1.1.0"); + /* Not on list, not newer than any on same series. */ + test_v_i_o(VS_UNRECOMMENDED, "0.1.0.1", + "Tor 0.1.0.2,Tor 0.0.9.5,Tor 0.1.1.0"); + /* On list, not newer than any on same series. */ + test_v_i_o(VS_UNRECOMMENDED, + "0.1.0.1", "Tor 0.1.0.2,Tor 0.0.9.5,Tor 0.1.1.0"); + test_eq(0, tor_version_as_new_as("Tor 0.0.5", "0.0.9pre1-cvs")); + test_eq(1, tor_version_as_new_as( + "Tor 0.0.8 on Darwin 64-121-192-100.c3-0." + "sfpo-ubr1.sfrn-sfpo.ca.cable.rcn.com Power Macintosh", + "0.0.8rc2")); + test_eq(0, tor_version_as_new_as( + "Tor 0.0.8 on Darwin 64-121-192-100.c3-0." + "sfpo-ubr1.sfrn-sfpo.ca.cable.rcn.com Power Macintosh", "0.0.8.2")); + + /* Now try svn revisions. */ + test_eq(1, tor_version_as_new_as("Tor 0.2.1.0-dev (r100)", + "Tor 0.2.1.0-dev (r99)")); + test_eq(1, tor_version_as_new_as("Tor 0.2.1.0-dev (r100) on Banana Jr", + "Tor 0.2.1.0-dev (r99) on Hal 9000")); + test_eq(1, tor_version_as_new_as("Tor 0.2.1.0-dev (r100)", + "Tor 0.2.1.0-dev on Colossus")); + test_eq(0, tor_version_as_new_as("Tor 0.2.1.0-dev (r99)", + "Tor 0.2.1.0-dev (r100)")); + test_eq(0, tor_version_as_new_as("Tor 0.2.1.0-dev (r99) on MCP", + "Tor 0.2.1.0-dev (r100) on AM")); + test_eq(0, tor_version_as_new_as("Tor 0.2.1.0-dev", + "Tor 0.2.1.0-dev (r99)")); + test_eq(1, tor_version_as_new_as("Tor 0.2.1.1", + "Tor 0.2.1.0-dev (r99)")); + + /* Now try git revisions */ + test_eq(0, tor_version_parse("0.5.6.7 (git-ff00ff)", &ver1)); + test_eq(0, ver1.major); + test_eq(5, ver1.minor); + test_eq(6, ver1.micro); + test_eq(7, ver1.patchlevel); + test_eq(3, ver1.git_tag_len); + test_memeq(ver1.git_tag, "\xff\x00\xff", 3); + test_eq(-1, tor_version_parse("0.5.6.7 (git-ff00xx)", &ver1)); + test_eq(-1, tor_version_parse("0.5.6.7 (git-ff00fff)", &ver1)); + test_eq(0, tor_version_parse("0.5.6.7 (git ff00fff)", &ver1)); + done: + ; +} + +/** Run unit tests for directory fp_pair functions. */ +static void +test_dir_fp_pairs(void) +{ + smartlist_t *sl = smartlist_create(); + fp_pair_t *pair; + + dir_split_resource_into_fingerprint_pairs( + /* Two pairs, out of order, with one duplicate. */ + "73656372657420646174612E0000000000FFFFFF-" + "557365204145532d32353620696e73746561642e+" + "73656372657420646174612E0000000000FFFFFF-" + "557365204145532d32353620696e73746561642e+" + "48657861646563696d616c2069736e277420736f-" + "676f6f6420666f7220686964696e6720796f7572.z", sl); + + test_eq(smartlist_len(sl), 2); + pair = smartlist_get(sl, 0); + test_memeq(pair->first, "Hexadecimal isn't so", DIGEST_LEN); + test_memeq(pair->second, "good for hiding your", DIGEST_LEN); + pair = smartlist_get(sl, 1); + test_memeq(pair->first, "secret data.\0\0\0\0\0\xff\xff\xff", DIGEST_LEN); + test_memeq(pair->second, "Use AES-256 instead.", DIGEST_LEN); + + done: + SMARTLIST_FOREACH(sl, fp_pair_t *, pair, tor_free(pair)); + smartlist_free(sl); +} + +static void +test_dir_split_fps(void *testdata) +{ + smartlist_t *sl = smartlist_create(); + char *mem_op_hex_tmp = NULL; + (void)testdata; + + /* Some example hex fingerprints and their base64 equivalents */ +#define HEX1 "Fe0daff89127389bc67558691231234551193EEE" +#define HEX2 "Deadbeef99999991111119999911111111f00ba4" +#define HEX3 "b33ff00db33ff00db33ff00db33ff00db33ff00d" +#define HEX256_1 \ + "f3f3f3f3fbbbbf3f3f3f3fbbbf3f3f3f3fbbbbf3f3f3f3fbbbf3f3f3f3fbbbbf" +#define HEX256_2 \ + "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccCCc" +#define HEX256_3 \ + "0123456789ABCdef0123456789ABCdef0123456789ABCdef0123456789ABCdef" +#define B64_1 "/g2v+JEnOJvGdVhpEjEjRVEZPu4" +#define B64_2 "3q2+75mZmZERERmZmRERERHwC6Q" +#define B64_3 "sz/wDbM/8A2zP/ANsz/wDbM/8A0" +#define B64_256_1 "8/Pz8/u7vz8/Pz+7vz8/Pz+7u/Pz8/P7u/Pz8/P7u78" +#define B64_256_2 "zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMw" +#define B64_256_3 "ASNFZ4mrze8BI0VniavN7wEjRWeJq83vASNFZ4mrze8" + + /* no flags set */ + dir_split_resource_into_fingerprints("A+C+B", sl, NULL, 0); + tt_int_op(smartlist_len(sl), ==, 3); + tt_str_op(smartlist_get(sl, 0), ==, "A"); + tt_str_op(smartlist_get(sl, 1), ==, "C"); + tt_str_op(smartlist_get(sl, 2), ==, "B"); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + /* uniq strings. */ + dir_split_resource_into_fingerprints("A+C+B+A+B+B", sl, NULL, DSR_SORT_UNIQ); + tt_int_op(smartlist_len(sl), ==, 3); + tt_str_op(smartlist_get(sl, 0), ==, "A"); + tt_str_op(smartlist_get(sl, 1), ==, "B"); + tt_str_op(smartlist_get(sl, 2), ==, "C"); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + /* Decode hex. */ + dir_split_resource_into_fingerprints(HEX1"+"HEX2, sl, NULL, DSR_HEX); + tt_int_op(smartlist_len(sl), ==, 2); + test_mem_op_hex(smartlist_get(sl, 0), ==, HEX1); + test_mem_op_hex(smartlist_get(sl, 1), ==, HEX2); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + /* decode hex and drop weirdness. */ + dir_split_resource_into_fingerprints(HEX1"+bogus+"HEX2"+"HEX256_1, + sl, NULL, DSR_HEX); + tt_int_op(smartlist_len(sl), ==, 2); + test_mem_op_hex(smartlist_get(sl, 0), ==, HEX1); + test_mem_op_hex(smartlist_get(sl, 1), ==, HEX2); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + /* Decode long hex */ + dir_split_resource_into_fingerprints(HEX256_1"+"HEX256_2"+"HEX2"+"HEX256_3, + sl, NULL, DSR_HEX|DSR_DIGEST256); + tt_int_op(smartlist_len(sl), ==, 3); + test_mem_op_hex(smartlist_get(sl, 0), ==, HEX256_1); + test_mem_op_hex(smartlist_get(sl, 1), ==, HEX256_2); + test_mem_op_hex(smartlist_get(sl, 2), ==, HEX256_3); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + /* Decode hex and sort. */ + dir_split_resource_into_fingerprints(HEX1"+"HEX2"+"HEX3"+"HEX2, + sl, NULL, DSR_HEX|DSR_SORT_UNIQ); + tt_int_op(smartlist_len(sl), ==, 3); + test_mem_op_hex(smartlist_get(sl, 0), ==, HEX3); + test_mem_op_hex(smartlist_get(sl, 1), ==, HEX2); + test_mem_op_hex(smartlist_get(sl, 2), ==, HEX1); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + /* Decode long hex and sort */ + dir_split_resource_into_fingerprints(HEX256_1"+"HEX256_2"+"HEX256_3 + "+"HEX256_1, + sl, NULL, + DSR_HEX|DSR_DIGEST256|DSR_SORT_UNIQ); + tt_int_op(smartlist_len(sl), ==, 3); + test_mem_op_hex(smartlist_get(sl, 0), ==, HEX256_3); + test_mem_op_hex(smartlist_get(sl, 1), ==, HEX256_2); + test_mem_op_hex(smartlist_get(sl, 2), ==, HEX256_1); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + /* Decode base64 */ + dir_split_resource_into_fingerprints(B64_1"-"B64_2, sl, NULL, DSR_BASE64); + tt_int_op(smartlist_len(sl), ==, 2); + test_mem_op_hex(smartlist_get(sl, 0), ==, HEX1); + test_mem_op_hex(smartlist_get(sl, 1), ==, HEX2); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + /* Decode long base64 */ + dir_split_resource_into_fingerprints(B64_256_1"-"B64_256_2, + sl, NULL, DSR_BASE64|DSR_DIGEST256); + tt_int_op(smartlist_len(sl), ==, 2); + test_mem_op_hex(smartlist_get(sl, 0), ==, HEX256_1); + test_mem_op_hex(smartlist_get(sl, 1), ==, HEX256_2); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + dir_split_resource_into_fingerprints(B64_256_1, + sl, NULL, DSR_BASE64|DSR_DIGEST256); + tt_int_op(smartlist_len(sl), ==, 1); + test_mem_op_hex(smartlist_get(sl, 0), ==, HEX256_1); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + done: + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_free(sl); + tor_free(mem_op_hex_tmp); +} + +static void +test_dir_measured_bw(void) +{ + measured_bw_line_t mbwl; + int i; + const char *lines_pass[] = { + "node_id=$557365204145532d32353620696e73746561642e bw=1024\n", + "node_id=$557365204145532d32353620696e73746561642e\t bw=1024 \n", + " node_id=$557365204145532d32353620696e73746561642e bw=1024\n", + "\tnoise\tnode_id=$557365204145532d32353620696e73746561642e " + "bw=1024 junk=007\n", + "misc=junk node_id=$557365204145532d32353620696e73746561642e " + "bw=1024 junk=007\n", + "end" + }; + const char *lines_fail[] = { + /* Test possible python stupidity on input */ + "node_id=None bw=1024\n", + "node_id=$None bw=1024\n", + "node_id=$557365204145532d32353620696e73746561642e bw=None\n", + "node_id=$557365204145532d32353620696e73746561642e bw=1024.0\n", + "node_id=$557365204145532d32353620696e73746561642e bw=.1024\n", + "node_id=$557365204145532d32353620696e73746561642e bw=1.024\n", + "node_id=$557365204145532d32353620696e73746561642e bw=1024 bw=0\n", + "node_id=$557365204145532d32353620696e73746561642e bw=1024 bw=None\n", + "node_id=$557365204145532d32353620696e73746561642e bw=-1024\n", + /* Test incomplete writes due to race conditions, partial copies, etc */ + "node_i", + "node_i\n", + "node_id=", + "node_id=\n", + "node_id=$557365204145532d32353620696e73746561642e bw=", + "node_id=$557365204145532d32353620696e73746561642e bw=1024", + "node_id=$557365204145532d32353620696e73746561642e bw=\n", + "node_id=$557365204145532d32353620696e7374", + "node_id=$557365204145532d32353620696e7374\n", + "", + "\n", + " \n ", + " \n\n", + /* Test assorted noise */ + " node_id= ", + "node_id==$557365204145532d32353620696e73746561642e bw==1024\n", + "node_id=$55736520414552d32353620696e73746561642e bw=1024\n", + "node_id=557365204145532d32353620696e73746561642e bw=1024\n", + "node_id= $557365204145532d32353620696e73746561642e bw=0.23\n", + "end" + }; + + for (i = 0; strcmp(lines_fail[i], "end"); i++) { + //fprintf(stderr, "Testing: %s\n", lines_fail[i]); + test_assert(measured_bw_line_parse(&mbwl, lines_fail[i]) == -1); + } + + for (i = 0; strcmp(lines_pass[i], "end"); i++) { + //fprintf(stderr, "Testing: %s %d\n", lines_pass[i], TOR_ISSPACE('\n')); + test_assert(measured_bw_line_parse(&mbwl, lines_pass[i]) == 0); + test_assert(mbwl.bw == 1024); + test_assert(strcmp(mbwl.node_hex, + "557365204145532d32353620696e73746561642e") == 0); + } + + done: + return; +} + +static void +test_dir_param_voting(void) +{ + networkstatus_t vote1, vote2, vote3, vote4; + smartlist_t *votes = smartlist_create(); + char *res = NULL; + + /* dirvote_compute_params only looks at the net_params field of the votes, + so that's all we need to set. + */ + memset(&vote1, 0, sizeof(vote1)); + memset(&vote2, 0, sizeof(vote2)); + memset(&vote3, 0, sizeof(vote3)); + memset(&vote4, 0, sizeof(vote4)); + vote1.net_params = smartlist_create(); + vote2.net_params = smartlist_create(); + vote3.net_params = smartlist_create(); + vote4.net_params = smartlist_create(); + smartlist_split_string(vote1.net_params, + "ab=90 abcd=20 cw=50 x-yz=-99", NULL, 0, 0); + smartlist_split_string(vote2.net_params, + "ab=27 cw=5 x-yz=88", NULL, 0, 0); + smartlist_split_string(vote3.net_params, + "abcd=20 c=60 cw=500 x-yz=-9 zzzzz=101", NULL, 0, 0); + smartlist_split_string(vote4.net_params, + "ab=900 abcd=200 c=1 cw=51 x-yz=100", NULL, 0, 0); + test_eq(100, networkstatus_get_param(&vote4, "x-yz", 50, 0, 300)); + test_eq(222, networkstatus_get_param(&vote4, "foobar", 222, 0, 300)); + test_eq(80, networkstatus_get_param(&vote4, "ab", 12, 0, 80)); + test_eq(-8, networkstatus_get_param(&vote4, "ab", -12, -100, -8)); + test_eq(0, networkstatus_get_param(&vote4, "foobar", 0, -100, 8)); + + smartlist_add(votes, &vote1); + smartlist_add(votes, &vote2); + smartlist_add(votes, &vote3); + smartlist_add(votes, &vote4); + + res = dirvote_compute_params(votes); + test_streq(res, + "ab=90 abcd=20 c=1 cw=50 x-yz=-9 zzzzz=101"); + + done: + tor_free(res); + SMARTLIST_FOREACH(vote1.net_params, char *, cp, tor_free(cp)); + SMARTLIST_FOREACH(vote2.net_params, char *, cp, tor_free(cp)); + SMARTLIST_FOREACH(vote3.net_params, char *, cp, tor_free(cp)); + SMARTLIST_FOREACH(vote4.net_params, char *, cp, tor_free(cp)); + smartlist_free(vote1.net_params); + smartlist_free(vote2.net_params); + smartlist_free(vote3.net_params); + smartlist_free(vote4.net_params); + smartlist_free(votes); + + return; +} + +extern const char AUTHORITY_CERT_1[]; +extern const char AUTHORITY_SIGNKEY_1[]; +extern const char AUTHORITY_CERT_2[]; +extern const char AUTHORITY_SIGNKEY_2[]; +extern const char AUTHORITY_CERT_3[]; +extern const char AUTHORITY_SIGNKEY_3[]; + +/** Helper: Test that two networkstatus_voter_info_t do in fact represent the + * same voting authority, and that they do in fact have all the same + * information. */ +static void +test_same_voter(networkstatus_voter_info_t *v1, + networkstatus_voter_info_t *v2) +{ + test_streq(v1->nickname, v2->nickname); + test_memeq(v1->identity_digest, v2->identity_digest, DIGEST_LEN); + test_streq(v1->address, v2->address); + test_eq(v1->addr, v2->addr); + test_eq(v1->dir_port, v2->dir_port); + test_eq(v1->or_port, v2->or_port); + test_streq(v1->contact, v2->contact); + test_memeq(v1->vote_digest, v2->vote_digest, DIGEST_LEN); + done: + ; +} + +/** Helper: Make a new routerinfo containing the right information for a + * given vote_routerstatus_t. */ +static routerinfo_t * +generate_ri_from_rs(const vote_routerstatus_t *vrs) +{ + routerinfo_t *r; + const routerstatus_t *rs = &vrs->status; + static time_t published = 0; + + r = tor_malloc_zero(sizeof(routerinfo_t)); + memcpy(r->cache_info.identity_digest, rs->identity_digest, DIGEST_LEN); + memcpy(r->cache_info.signed_descriptor_digest, rs->descriptor_digest, + DIGEST_LEN); + r->cache_info.do_not_cache = 1; + r->cache_info.routerlist_index = -1; + r->cache_info.signed_descriptor_body = + tor_strdup("123456789012345678901234567890123"); + r->cache_info.signed_descriptor_len = + strlen(r->cache_info.signed_descriptor_body); + r->exit_policy = smartlist_create(); + r->cache_info.published_on = ++published + time(NULL); + return r; +} + +/** Helper: get a detached signatures document for one or two + * consensuses. */ +static char * +get_detached_sigs(networkstatus_t *ns, networkstatus_t *ns2) +{ + char *r; + smartlist_t *sl; + tor_assert(ns && ns->flavor == FLAV_NS); + sl = smartlist_create(); + smartlist_add(sl,ns); + if (ns2) + smartlist_add(sl,ns2); + r = networkstatus_get_detached_signatures(sl); + smartlist_free(sl); + return r; +} + +/** Run unit tests for generating and parsing V3 consensus networkstatus + * documents. */ +static void +test_dir_v3_networkstatus(void) +{ + authority_cert_t *cert1=NULL, *cert2=NULL, *cert3=NULL; + crypto_pk_env_t *sign_skey_1=NULL, *sign_skey_2=NULL, *sign_skey_3=NULL; + crypto_pk_env_t *sign_skey_leg1=NULL; + const char *msg=NULL; + + time_t now = time(NULL); + networkstatus_voter_info_t *voter; + document_signature_t *sig; + networkstatus_t *vote=NULL, *v1=NULL, *v2=NULL, *v3=NULL, *con=NULL, + *con_md=NULL; + vote_routerstatus_t *vrs; + routerstatus_t *rs; + char *v1_text=NULL, *v2_text=NULL, *v3_text=NULL, *consensus_text=NULL, *cp; + smartlist_t *votes = smartlist_create(); + + /* For generating the two other consensuses. */ + char *detached_text1=NULL, *detached_text2=NULL; + char *consensus_text2=NULL, *consensus_text3=NULL; + char *consensus_text_md2=NULL, *consensus_text_md3=NULL; + char *consensus_text_md=NULL; + networkstatus_t *con2=NULL, *con_md2=NULL, *con3=NULL, *con_md3=NULL; + ns_detached_signatures_t *dsig1=NULL, *dsig2=NULL; + + /* Parse certificates and keys. */ + cert1 = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL); + test_assert(cert1); + test_assert(cert1->is_cross_certified); + cert2 = authority_cert_parse_from_string(AUTHORITY_CERT_2, NULL); + test_assert(cert2); + cert3 = authority_cert_parse_from_string(AUTHORITY_CERT_3, NULL); + test_assert(cert3); + sign_skey_1 = crypto_new_pk_env(); + sign_skey_2 = crypto_new_pk_env(); + sign_skey_3 = crypto_new_pk_env(); + sign_skey_leg1 = pk_generate(4); + + test_assert(!crypto_pk_read_private_key_from_string(sign_skey_1, + AUTHORITY_SIGNKEY_1, -1)); + test_assert(!crypto_pk_read_private_key_from_string(sign_skey_2, + AUTHORITY_SIGNKEY_2, -1)); + test_assert(!crypto_pk_read_private_key_from_string(sign_skey_3, + AUTHORITY_SIGNKEY_3, -1)); + + test_assert(!crypto_pk_cmp_keys(sign_skey_1, cert1->signing_key)); + test_assert(!crypto_pk_cmp_keys(sign_skey_2, cert2->signing_key)); + + /* + * Set up a vote; generate it; try to parse it. + */ + vote = tor_malloc_zero(sizeof(networkstatus_t)); + vote->type = NS_TYPE_VOTE; + vote->published = now; + vote->valid_after = now+1000; + vote->fresh_until = now+2000; + vote->valid_until = now+3000; + vote->vote_seconds = 100; + vote->dist_seconds = 200; + vote->supported_methods = smartlist_create(); + smartlist_split_string(vote->supported_methods, "1 2 3", NULL, 0, -1); + vote->client_versions = tor_strdup("0.1.2.14,0.1.2.15"); + vote->server_versions = tor_strdup("0.1.2.14,0.1.2.15,0.1.2.16"); + vote->known_flags = smartlist_create(); + smartlist_split_string(vote->known_flags, + "Authority Exit Fast Guard Running Stable V2Dir Valid", + 0, SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); + vote->voters = smartlist_create(); + voter = tor_malloc_zero(sizeof(networkstatus_voter_info_t)); + voter->nickname = tor_strdup("Voter1"); + voter->address = tor_strdup("1.2.3.4"); + voter->addr = 0x01020304; + voter->dir_port = 80; + voter->or_port = 9000; + voter->contact = tor_strdup("voter@example.com"); + crypto_pk_get_digest(cert1->identity_key, voter->identity_digest); + smartlist_add(vote->voters, voter); + vote->cert = authority_cert_dup(cert1); + vote->net_params = smartlist_create(); + smartlist_split_string(vote->net_params, "circuitwindow=101 foo=990", + NULL, 0, 0); + vote->routerstatus_list = smartlist_create(); + /* add the first routerstatus. */ + vrs = tor_malloc_zero(sizeof(vote_routerstatus_t)); + rs = &vrs->status; + vrs->version = tor_strdup("0.1.2.14"); + rs->published_on = now-1500; + strlcpy(rs->nickname, "router2", sizeof(rs->nickname)); + memset(rs->identity_digest, 3, DIGEST_LEN); + memset(rs->descriptor_digest, 78, DIGEST_LEN); + rs->addr = 0x99008801; + rs->or_port = 443; + rs->dir_port = 8000; + /* all flags but running cleared */ + rs->is_running = 1; + smartlist_add(vote->routerstatus_list, vrs); + test_assert(router_add_to_routerlist(generate_ri_from_rs(vrs), &msg,0,0)>=0); + + /* add the second routerstatus. */ + vrs = tor_malloc_zero(sizeof(vote_routerstatus_t)); + rs = &vrs->status; + vrs->version = tor_strdup("0.2.0.5"); + rs->published_on = now-1000; + strlcpy(rs->nickname, "router1", sizeof(rs->nickname)); + memset(rs->identity_digest, 5, DIGEST_LEN); + memset(rs->descriptor_digest, 77, DIGEST_LEN); + rs->addr = 0x99009901; + rs->or_port = 443; + rs->dir_port = 0; + rs->is_exit = rs->is_stable = rs->is_fast = rs->is_running = + rs->is_valid = rs->is_v2_dir = rs->is_possible_guard = 1; + smartlist_add(vote->routerstatus_list, vrs); + test_assert(router_add_to_routerlist(generate_ri_from_rs(vrs), &msg,0,0)>=0); + + /* add the third routerstatus. */ + vrs = tor_malloc_zero(sizeof(vote_routerstatus_t)); + rs = &vrs->status; + vrs->version = tor_strdup("0.1.0.3"); + rs->published_on = now-1000; + strlcpy(rs->nickname, "router3", sizeof(rs->nickname)); + memset(rs->identity_digest, 33, DIGEST_LEN); + memset(rs->descriptor_digest, 79, DIGEST_LEN); + rs->addr = 0xAA009901; + rs->or_port = 400; + rs->dir_port = 9999; + rs->is_authority = rs->is_exit = rs->is_stable = rs->is_fast = + rs->is_running = rs->is_valid = rs->is_v2_dir = rs->is_possible_guard = 1; + smartlist_add(vote->routerstatus_list, vrs); + test_assert(router_add_to_routerlist(generate_ri_from_rs(vrs), &msg,0,0)>=0); + + /* add a fourth routerstatus that is not running. */ + vrs = tor_malloc_zero(sizeof(vote_routerstatus_t)); + rs = &vrs->status; + vrs->version = tor_strdup("0.1.6.3"); + rs->published_on = now-1000; + strlcpy(rs->nickname, "router4", sizeof(rs->nickname)); + memset(rs->identity_digest, 34, DIGEST_LEN); + memset(rs->descriptor_digest, 47, DIGEST_LEN); + rs->addr = 0xC0000203; + rs->or_port = 500; + rs->dir_port = 1999; + /* Running flag (and others) cleared */ + smartlist_add(vote->routerstatus_list, vrs); + test_assert(router_add_to_routerlist(generate_ri_from_rs(vrs), &msg,0,0)>=0); + + /* dump the vote and try to parse it. */ + v1_text = format_networkstatus_vote(sign_skey_1, vote); + test_assert(v1_text); + v1 = networkstatus_parse_vote_from_string(v1_text, NULL, NS_TYPE_VOTE); + test_assert(v1); + + /* Make sure the parsed thing was right. */ + test_eq(v1->type, NS_TYPE_VOTE); + test_eq(v1->published, vote->published); + test_eq(v1->valid_after, vote->valid_after); + test_eq(v1->fresh_until, vote->fresh_until); + test_eq(v1->valid_until, vote->valid_until); + test_eq(v1->vote_seconds, vote->vote_seconds); + test_eq(v1->dist_seconds, vote->dist_seconds); + test_streq(v1->client_versions, vote->client_versions); + test_streq(v1->server_versions, vote->server_versions); + test_assert(v1->voters && smartlist_len(v1->voters)); + voter = smartlist_get(v1->voters, 0); + test_streq(voter->nickname, "Voter1"); + test_streq(voter->address, "1.2.3.4"); + test_eq(voter->addr, 0x01020304); + test_eq(voter->dir_port, 80); + test_eq(voter->or_port, 9000); + test_streq(voter->contact, "voter@example.com"); + test_assert(v1->cert); + test_assert(!crypto_pk_cmp_keys(sign_skey_1, v1->cert->signing_key)); + cp = smartlist_join_strings(v1->known_flags, ":", 0, NULL); + test_streq(cp, "Authority:Exit:Fast:Guard:Running:Stable:V2Dir:Valid"); + tor_free(cp); + test_eq(smartlist_len(v1->routerstatus_list), 4); + /* Check the first routerstatus. */ + vrs = smartlist_get(v1->routerstatus_list, 0); + rs = &vrs->status; + test_streq(vrs->version, "0.1.2.14"); + test_eq(rs->published_on, now-1500); + test_streq(rs->nickname, "router2"); + test_memeq(rs->identity_digest, + "\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3", + DIGEST_LEN); + test_memeq(rs->descriptor_digest, "NNNNNNNNNNNNNNNNNNNN", DIGEST_LEN); + test_eq(rs->addr, 0x99008801); + test_eq(rs->or_port, 443); + test_eq(rs->dir_port, 8000); + test_eq(vrs->flags, U64_LITERAL(16)); // no flags except "running" + /* Check the second routerstatus. */ + vrs = smartlist_get(v1->routerstatus_list, 1); + rs = &vrs->status; + test_streq(vrs->version, "0.2.0.5"); + test_eq(rs->published_on, now-1000); + test_streq(rs->nickname, "router1"); + test_memeq(rs->identity_digest, + "\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5", + DIGEST_LEN); + test_memeq(rs->descriptor_digest, "MMMMMMMMMMMMMMMMMMMM", DIGEST_LEN); + test_eq(rs->addr, 0x99009901); + test_eq(rs->or_port, 443); + test_eq(rs->dir_port, 0); + test_eq(vrs->flags, U64_LITERAL(254)); // all flags except "authority." + + { + measured_bw_line_t mbw; + memset(mbw.node_id, 33, sizeof(mbw.node_id)); + mbw.bw = 1024; + test_assert(measured_bw_line_apply(&mbw, + v1->routerstatus_list) == 1); + vrs = smartlist_get(v1->routerstatus_list, 2); + test_assert(vrs->status.has_measured_bw && + vrs->status.measured_bw == 1024); + } + + /* Generate second vote. It disagrees on some of the times, + * and doesn't list versions, and knows some crazy flags */ + vote->published = now+1; + vote->fresh_until = now+3005; + vote->dist_seconds = 300; + authority_cert_free(vote->cert); + vote->cert = authority_cert_dup(cert2); + vote->net_params = smartlist_create(); + smartlist_split_string(vote->net_params, "bar=2000000000 circuitwindow=20", + NULL, 0, 0); + tor_free(vote->client_versions); + tor_free(vote->server_versions); + voter = smartlist_get(vote->voters, 0); + tor_free(voter->nickname); + tor_free(voter->address); + voter->nickname = tor_strdup("Voter2"); + voter->address = tor_strdup("2.3.4.5"); + voter->addr = 0x02030405; + crypto_pk_get_digest(cert2->identity_key, voter->identity_digest); + smartlist_add(vote->known_flags, tor_strdup("MadeOfCheese")); + smartlist_add(vote->known_flags, tor_strdup("MadeOfTin")); + smartlist_sort_strings(vote->known_flags); + vrs = smartlist_get(vote->routerstatus_list, 2); + smartlist_del_keeporder(vote->routerstatus_list, 2); + tor_free(vrs->version); + tor_free(vrs); + vrs = smartlist_get(vote->routerstatus_list, 0); + vrs->status.is_fast = 1; + /* generate and parse. */ + v2_text = format_networkstatus_vote(sign_skey_2, vote); + test_assert(v2_text); + v2 = networkstatus_parse_vote_from_string(v2_text, NULL, NS_TYPE_VOTE); + test_assert(v2); + /* Check that flags come out right.*/ + cp = smartlist_join_strings(v2->known_flags, ":", 0, NULL); + test_streq(cp, "Authority:Exit:Fast:Guard:MadeOfCheese:MadeOfTin:" + "Running:Stable:V2Dir:Valid"); + tor_free(cp); + vrs = smartlist_get(v2->routerstatus_list, 1); + /* 1023 - authority(1) - madeofcheese(16) - madeoftin(32) */ + test_eq(vrs->flags, U64_LITERAL(974)); + + /* Generate the third vote. */ + vote->published = now; + vote->fresh_until = now+2003; + vote->dist_seconds = 250; + authority_cert_free(vote->cert); + vote->cert = authority_cert_dup(cert3); + vote->net_params = smartlist_create(); + smartlist_split_string(vote->net_params, "circuitwindow=80 foo=660", + NULL, 0, 0); + smartlist_add(vote->supported_methods, tor_strdup("4")); + vote->client_versions = tor_strdup("0.1.2.14,0.1.2.17"); + vote->server_versions = tor_strdup("0.1.2.10,0.1.2.15,0.1.2.16"); + voter = smartlist_get(vote->voters, 0); + tor_free(voter->nickname); + tor_free(voter->address); + voter->nickname = tor_strdup("Voter3"); + voter->address = tor_strdup("3.4.5.6"); + voter->addr = 0x03040506; + crypto_pk_get_digest(cert3->identity_key, voter->identity_digest); + /* This one has a legacy id. */ + memset(voter->legacy_id_digest, (int)'A', DIGEST_LEN); + vrs = smartlist_get(vote->routerstatus_list, 0); + smartlist_del_keeporder(vote->routerstatus_list, 0); + tor_free(vrs->version); + tor_free(vrs); + vrs = smartlist_get(vote->routerstatus_list, 0); + memset(vrs->status.descriptor_digest, (int)'Z', DIGEST_LEN); + test_assert(router_add_to_routerlist(generate_ri_from_rs(vrs), &msg,0,0)>=0); + + v3_text = format_networkstatus_vote(sign_skey_3, vote); + test_assert(v3_text); + + v3 = networkstatus_parse_vote_from_string(v3_text, NULL, NS_TYPE_VOTE); + test_assert(v3); + + /* Compute a consensus as voter 3. */ + smartlist_add(votes, v3); + smartlist_add(votes, v1); + smartlist_add(votes, v2); + consensus_text = networkstatus_compute_consensus(votes, 3, + cert3->identity_key, + sign_skey_3, + "AAAAAAAAAAAAAAAAAAAA", + sign_skey_leg1, + FLAV_NS); + test_assert(consensus_text); + con = networkstatus_parse_vote_from_string(consensus_text, NULL, + NS_TYPE_CONSENSUS); + test_assert(con); + //log_notice(LD_GENERAL, "<<%s>>\n<<%s>>\n<<%s>>\n", + // v1_text, v2_text, v3_text); + consensus_text_md = networkstatus_compute_consensus(votes, 3, + cert3->identity_key, + sign_skey_3, + "AAAAAAAAAAAAAAAAAAAA", + sign_skey_leg1, + FLAV_MICRODESC); + test_assert(consensus_text_md); + con_md = networkstatus_parse_vote_from_string(consensus_text_md, NULL, + NS_TYPE_CONSENSUS); + test_assert(con_md); + test_eq(con_md->flavor, FLAV_MICRODESC); + + /* Check consensus contents. */ + test_assert(con->type == NS_TYPE_CONSENSUS); + test_eq(con->published, 0); /* this field only appears in votes. */ + test_eq(con->valid_after, now+1000); + test_eq(con->fresh_until, now+2003); /* median */ + test_eq(con->valid_until, now+3000); + test_eq(con->vote_seconds, 100); + test_eq(con->dist_seconds, 250); /* median */ + test_streq(con->client_versions, "0.1.2.14"); + test_streq(con->server_versions, "0.1.2.15,0.1.2.16"); + cp = smartlist_join_strings(v2->known_flags, ":", 0, NULL); + test_streq(cp, "Authority:Exit:Fast:Guard:MadeOfCheese:MadeOfTin:" + "Running:Stable:V2Dir:Valid"); + tor_free(cp); + cp = smartlist_join_strings(con->net_params, ":", 0, NULL); + test_streq(cp, "bar=2000000000:circuitwindow=80:foo=660"); + tor_free(cp); + + test_eq(4, smartlist_len(con->voters)); /*3 voters, 1 legacy key.*/ + /* The voter id digests should be in this order. */ + test_assert(memcmp(cert2->cache_info.identity_digest, + cert1->cache_info.identity_digest,DIGEST_LEN)<0); + test_assert(memcmp(cert1->cache_info.identity_digest, + cert3->cache_info.identity_digest,DIGEST_LEN)<0); + test_same_voter(smartlist_get(con->voters, 1), + smartlist_get(v2->voters, 0)); + test_same_voter(smartlist_get(con->voters, 2), + smartlist_get(v1->voters, 0)); + test_same_voter(smartlist_get(con->voters, 3), + smartlist_get(v3->voters, 0)); + + test_assert(!con->cert); + test_eq(2, smartlist_len(con->routerstatus_list)); + /* There should be two listed routers: one with identity 3, one with + * identity 5. */ + /* This one showed up in 2 digests. */ + rs = smartlist_get(con->routerstatus_list, 0); + test_memeq(rs->identity_digest, + "\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3", + DIGEST_LEN); + test_memeq(rs->descriptor_digest, "NNNNNNNNNNNNNNNNNNNN", DIGEST_LEN); + test_assert(!rs->is_authority); + test_assert(!rs->is_exit); + test_assert(!rs->is_fast); + test_assert(!rs->is_possible_guard); + test_assert(!rs->is_stable); + test_assert(rs->is_running); /* If it wasn't running it wouldn't be here */ + test_assert(!rs->is_v2_dir); + test_assert(!rs->is_valid); + test_assert(!rs->is_named); + /* XXXX check version */ + + rs = smartlist_get(con->routerstatus_list, 1); + /* This one showed up in 3 digests. Twice with ID 'M', once with 'Z'. */ + test_memeq(rs->identity_digest, + "\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5", + DIGEST_LEN); + test_streq(rs->nickname, "router1"); + test_memeq(rs->descriptor_digest, "MMMMMMMMMMMMMMMMMMMM", DIGEST_LEN); + test_eq(rs->published_on, now-1000); + test_eq(rs->addr, 0x99009901); + test_eq(rs->or_port, 443); + test_eq(rs->dir_port, 0); + test_assert(!rs->is_authority); + test_assert(rs->is_exit); + test_assert(rs->is_fast); + test_assert(rs->is_possible_guard); + test_assert(rs->is_stable); + test_assert(rs->is_running); + test_assert(rs->is_v2_dir); + test_assert(rs->is_valid); + test_assert(!rs->is_named); + /* XXXX check version */ + + /* Check signatures. the first voter is a pseudo-entry with a legacy key. + * The second one hasn't signed. The fourth one has signed: validate it. */ + voter = smartlist_get(con->voters, 1); + test_eq(smartlist_len(voter->sigs), 0); + + voter = smartlist_get(con->voters, 3); + test_eq(smartlist_len(voter->sigs), 1); + sig = smartlist_get(voter->sigs, 0); + test_assert(sig->signature); + test_assert(!sig->good_signature); + test_assert(!sig->bad_signature); + + test_assert(!networkstatus_check_document_signature(con, sig, cert3)); + test_assert(sig->signature); + test_assert(sig->good_signature); + test_assert(!sig->bad_signature); + + { + const char *msg=NULL; + /* Compute the other two signed consensuses. */ + smartlist_shuffle(votes); + consensus_text2 = networkstatus_compute_consensus(votes, 3, + cert2->identity_key, + sign_skey_2, NULL,NULL, + FLAV_NS); + consensus_text_md2 = networkstatus_compute_consensus(votes, 3, + cert2->identity_key, + sign_skey_2, NULL,NULL, + FLAV_MICRODESC); + smartlist_shuffle(votes); + consensus_text3 = networkstatus_compute_consensus(votes, 3, + cert1->identity_key, + sign_skey_1, NULL,NULL, + FLAV_NS); + consensus_text_md3 = networkstatus_compute_consensus(votes, 3, + cert1->identity_key, + sign_skey_1, NULL,NULL, + FLAV_MICRODESC); + test_assert(consensus_text2); + test_assert(consensus_text3); + test_assert(consensus_text_md2); + test_assert(consensus_text_md3); + con2 = networkstatus_parse_vote_from_string(consensus_text2, NULL, + NS_TYPE_CONSENSUS); + con3 = networkstatus_parse_vote_from_string(consensus_text3, NULL, + NS_TYPE_CONSENSUS); + con_md2 = networkstatus_parse_vote_from_string(consensus_text_md2, NULL, + NS_TYPE_CONSENSUS); + con_md3 = networkstatus_parse_vote_from_string(consensus_text_md3, NULL, + NS_TYPE_CONSENSUS); + test_assert(con2); + test_assert(con3); + test_assert(con_md2); + test_assert(con_md3); + + /* All three should have the same digest. */ + test_memeq(&con->digests, &con2->digests, sizeof(digests_t)); + test_memeq(&con->digests, &con3->digests, sizeof(digests_t)); + + test_memeq(&con_md->digests, &con_md2->digests, sizeof(digests_t)); + test_memeq(&con_md->digests, &con_md3->digests, sizeof(digests_t)); + + /* Extract a detached signature from con3. */ + detached_text1 = get_detached_sigs(con3, con_md3); + tor_assert(detached_text1); + /* Try to parse it. */ + dsig1 = networkstatus_parse_detached_signatures(detached_text1, NULL); + tor_assert(dsig1); + + /* Are parsed values as expected? */ + test_eq(dsig1->valid_after, con3->valid_after); + test_eq(dsig1->fresh_until, con3->fresh_until); + test_eq(dsig1->valid_until, con3->valid_until); + { + digests_t *dsig_digests = strmap_get(dsig1->digests, "ns"); + test_assert(dsig_digests); + test_memeq(dsig_digests->d[DIGEST_SHA1], con3->digests.d[DIGEST_SHA1], + DIGEST_LEN); + dsig_digests = strmap_get(dsig1->digests, "microdesc"); + test_assert(dsig_digests); + test_memeq(dsig_digests->d[DIGEST_SHA256], + con_md3->digests.d[DIGEST_SHA256], + DIGEST256_LEN); + } + { + smartlist_t *dsig_signatures = strmap_get(dsig1->signatures, "ns"); + test_assert(dsig_signatures); + test_eq(1, smartlist_len(dsig_signatures)); + sig = smartlist_get(dsig_signatures, 0); + test_memeq(sig->identity_digest, cert1->cache_info.identity_digest, + DIGEST_LEN); + test_eq(sig->alg, DIGEST_SHA1); + + dsig_signatures = strmap_get(dsig1->signatures, "microdesc"); + test_assert(dsig_signatures); + test_eq(1, smartlist_len(dsig_signatures)); + sig = smartlist_get(dsig_signatures, 0); + test_memeq(sig->identity_digest, cert1->cache_info.identity_digest, + DIGEST_LEN); + test_eq(sig->alg, DIGEST_SHA256); + } + + /* Try adding it to con2. */ + detached_text2 = get_detached_sigs(con2,con_md2); + test_eq(1, networkstatus_add_detached_signatures(con2, dsig1, &msg)); + tor_free(detached_text2); + test_eq(1, networkstatus_add_detached_signatures(con_md2, dsig1, &msg)); + tor_free(detached_text2); + detached_text2 = get_detached_sigs(con2,con_md2); + //printf("\n<%s>\n", detached_text2); + dsig2 = networkstatus_parse_detached_signatures(detached_text2, NULL); + test_assert(dsig2); + /* + printf("\n"); + SMARTLIST_FOREACH(dsig2->signatures, networkstatus_voter_info_t *, vi, { + char hd[64]; + base16_encode(hd, sizeof(hd), vi->identity_digest, DIGEST_LEN); + printf("%s\n", hd); + }); + */ + test_eq(2, + smartlist_len((smartlist_t*)strmap_get(dsig2->signatures, "ns"))); + test_eq(2, + smartlist_len((smartlist_t*)strmap_get(dsig2->signatures, + "microdesc"))); + + /* Try adding to con2 twice; verify that nothing changes. */ + test_eq(0, networkstatus_add_detached_signatures(con2, dsig1, &msg)); + + /* Add to con. */ + test_eq(2, networkstatus_add_detached_signatures(con, dsig2, &msg)); + /* Check signatures */ + voter = smartlist_get(con->voters, 1); + sig = smartlist_get(voter->sigs, 0); + test_assert(sig); + test_assert(!networkstatus_check_document_signature(con, sig, cert2)); + voter = smartlist_get(con->voters, 2); + sig = smartlist_get(voter->sigs, 0); + test_assert(sig); + test_assert(!networkstatus_check_document_signature(con, sig, cert1)); + } + + done: + smartlist_free(votes); + tor_free(v1_text); + tor_free(v2_text); + tor_free(v3_text); + tor_free(consensus_text); + tor_free(consensus_text_md); + + if (vote) + networkstatus_vote_free(vote); + if (v1) + networkstatus_vote_free(v1); + if (v2) + networkstatus_vote_free(v2); + if (v3) + networkstatus_vote_free(v3); + if (con) + networkstatus_vote_free(con); + if (con_md) + networkstatus_vote_free(con_md); + if (sign_skey_1) + crypto_free_pk_env(sign_skey_1); + if (sign_skey_2) + crypto_free_pk_env(sign_skey_2); + if (sign_skey_3) + crypto_free_pk_env(sign_skey_3); + if (sign_skey_leg1) + crypto_free_pk_env(sign_skey_leg1); + if (cert1) + authority_cert_free(cert1); + if (cert2) + authority_cert_free(cert2); + if (cert3) + authority_cert_free(cert3); + + tor_free(consensus_text2); + tor_free(consensus_text3); + tor_free(consensus_text_md2); + tor_free(consensus_text_md3); + tor_free(detached_text1); + tor_free(detached_text2); + if (con2) + networkstatus_vote_free(con2); + if (con3) + networkstatus_vote_free(con3); + if (con_md2) + networkstatus_vote_free(con_md2); + if (con_md3) + networkstatus_vote_free(con_md3); + if (dsig1) + ns_detached_signatures_free(dsig1); + if (dsig2) + ns_detached_signatures_free(dsig2); +} + +#define DIR_LEGACY(name) \ + { #name, legacy_test_helper, 0, &legacy_setup, test_dir_ ## name } + +#define DIR(name) \ + { #name, test_dir_##name, 0, NULL, NULL } + +struct testcase_t dir_tests[] = { + DIR_LEGACY(nicknames), + DIR_LEGACY(formats), + DIR_LEGACY(versions), + DIR_LEGACY(fp_pairs), + DIR(split_fps), + DIR_LEGACY(measured_bw), + DIR_LEGACY(param_voting), + DIR_LEGACY(v3_networkstatus), + END_OF_TESTCASES +}; + diff --git a/src/test/test_util.c b/src/test/test_util.c new file mode 100644 index 0000000000..b1fafc84b2 --- /dev/null +++ b/src/test/test_util.c @@ -0,0 +1,1240 @@ +/* Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "orconfig.h" +#define CONTROL_PRIVATE +#define MEMPOOL_PRIVATE +#include "or.h" +#include "config.h" +#include "control.h" +#include "test.h" +#include "mempool.h" +#include "memarea.h" + +static void +test_util_time(void) +{ + struct timeval start, end; + struct tm a_time; + char timestr[RFC1123_TIME_LEN+1]; + time_t t_res; + int i; + + start.tv_sec = 5; + start.tv_usec = 5000; + + end.tv_sec = 5; + end.tv_usec = 5000; + + test_eq(0L, tv_udiff(&start, &end)); + + end.tv_usec = 7000; + + test_eq(2000L, tv_udiff(&start, &end)); + + end.tv_sec = 6; + + test_eq(1002000L, tv_udiff(&start, &end)); + + end.tv_usec = 0; + + test_eq(995000L, tv_udiff(&start, &end)); + + end.tv_sec = 4; + + test_eq(-1005000L, tv_udiff(&start, &end)); + + end.tv_usec = 999990; + start.tv_sec = 1; + start.tv_usec = 500; + + /* The test values here are confirmed to be correct on a platform + * with a working timegm. */ + a_time.tm_year = 2003-1900; + a_time.tm_mon = 7; + a_time.tm_mday = 30; + a_time.tm_hour = 6; + a_time.tm_min = 14; + a_time.tm_sec = 55; + test_eq((time_t) 1062224095UL, tor_timegm(&a_time)); + a_time.tm_year = 2004-1900; /* Try a leap year, after feb. */ + test_eq((time_t) 1093846495UL, tor_timegm(&a_time)); + a_time.tm_mon = 1; /* Try a leap year, in feb. */ + a_time.tm_mday = 10; + test_eq((time_t) 1076393695UL, tor_timegm(&a_time)); + + format_rfc1123_time(timestr, 0); + test_streq("Thu, 01 Jan 1970 00:00:00 GMT", timestr); + format_rfc1123_time(timestr, (time_t)1091580502UL); + test_streq("Wed, 04 Aug 2004 00:48:22 GMT", timestr); + + t_res = 0; + i = parse_rfc1123_time(timestr, &t_res); + test_eq(i,0); + test_eq(t_res, (time_t)1091580502UL); + test_eq(-1, parse_rfc1123_time("Wed, zz Aug 2004 99-99x99 GMT", &t_res)); + + tor_gettimeofday(&start); + /* now make sure time works. */ + tor_gettimeofday(&end); + /* We might've timewarped a little. */ + tt_int_op(tv_udiff(&start, &end), >=, -5000); + + done: + ; +} + +static void +test_util_config_line(void) +{ + char buf[1024]; + char *k=NULL, *v=NULL; + const char *str; + + /* Test parse_config_line_from_str */ + strlcpy(buf, "k v\n" " key value with spaces \n" "keykey val\n" + "k2\n" + "k3 \n" "\n" " \n" "#comment\n" + "k4#a\n" "k5#abc\n" "k6 val #with comment\n" + "kseven \"a quoted 'string\"\n" + "k8 \"a \\x71uoted\\n\\\"str\\\\ing\\t\\001\\01\\1\\\"\"\n" + "k9 a line that\\\n spans two lines.\n\n" + "k10 more than\\\n one contin\\\nuation\n" + "k11 \\\ncontinuation at the start\n" + "k12 line with a\\\n#comment\n embedded\n" + "k13\\\ncontinuation at the very start\n" + "k14 a line that has a comment and # ends with a slash \\\n" + "k15 this should be the next new line\n" + "k16 a line that has a comment and # ends without a slash \n" + "k17 this should be the next new line\n" + , sizeof(buf)); + str = buf; + + str = parse_config_line_from_str(str, &k, &v); + test_streq(k, "k"); + test_streq(v, "v"); + tor_free(k); tor_free(v); + test_assert(!strcmpstart(str, "key value with")); + + str = parse_config_line_from_str(str, &k, &v); + test_streq(k, "key"); + test_streq(v, "value with spaces"); + tor_free(k); tor_free(v); + test_assert(!strcmpstart(str, "keykey")); + + str = parse_config_line_from_str(str, &k, &v); + test_streq(k, "keykey"); + test_streq(v, "val"); + tor_free(k); tor_free(v); + test_assert(!strcmpstart(str, "k2\n")); + + str = parse_config_line_from_str(str, &k, &v); + test_streq(k, "k2"); + test_streq(v, ""); + tor_free(k); tor_free(v); + test_assert(!strcmpstart(str, "k3 \n")); + + str = parse_config_line_from_str(str, &k, &v); + test_streq(k, "k3"); + test_streq(v, ""); + tor_free(k); tor_free(v); + test_assert(!strcmpstart(str, "#comment")); + + str = parse_config_line_from_str(str, &k, &v); + test_streq(k, "k4"); + test_streq(v, ""); + tor_free(k); tor_free(v); + test_assert(!strcmpstart(str, "k5#abc")); + + str = parse_config_line_from_str(str, &k, &v); + test_streq(k, "k5"); + test_streq(v, ""); + tor_free(k); tor_free(v); + test_assert(!strcmpstart(str, "k6")); + + str = parse_config_line_from_str(str, &k, &v); + test_streq(k, "k6"); + test_streq(v, "val"); + tor_free(k); tor_free(v); + test_assert(!strcmpstart(str, "kseven")); + + str = parse_config_line_from_str(str, &k, &v); + test_streq(k, "kseven"); + test_streq(v, "a quoted \'string"); + tor_free(k); tor_free(v); + test_assert(!strcmpstart(str, "k8 ")); + + str = parse_config_line_from_str(str, &k, &v); + test_streq(k, "k8"); + test_streq(v, "a quoted\n\"str\\ing\t\x01\x01\x01\""); + tor_free(k); tor_free(v); + + str = parse_config_line_from_str(str, &k, &v); + test_streq(k, "k9"); + test_streq(v, "a line that spans two lines."); + tor_free(k); tor_free(v); + + str = parse_config_line_from_str(str, &k, &v); + test_streq(k, "k10"); + test_streq(v, "more than one continuation"); + tor_free(k); tor_free(v); + + str = parse_config_line_from_str(str, &k, &v); + test_streq(k, "k11"); + test_streq(v, "continuation at the start"); + tor_free(k); tor_free(v); + + str = parse_config_line_from_str(str, &k, &v); + test_streq(k, "k12"); + test_streq(v, "line with a embedded"); + tor_free(k); tor_free(v); + + str = parse_config_line_from_str(str, &k, &v); + test_streq(k, "k13"); + test_streq(v, "continuation at the very start"); + tor_free(k); tor_free(v); + + str = parse_config_line_from_str(str, &k, &v); + test_streq(k, "k14"); + test_streq(v, "a line that has a comment and" ); + tor_free(k); tor_free(v); + + str = parse_config_line_from_str(str, &k, &v); + test_streq(k, "k15"); + test_streq(v, "this should be the next new line"); + tor_free(k); tor_free(v); + + str = parse_config_line_from_str(str, &k, &v); + test_streq(k, "k16"); + test_streq(v, "a line that has a comment and" ); + tor_free(k); tor_free(v); + + str = parse_config_line_from_str(str, &k, &v); + test_streq(k, "k17"); + test_streq(v, "this should be the next new line"); + tor_free(k); tor_free(v); + + test_streq(str, ""); + + done: + tor_free(k); + tor_free(v); +} + +/** Test basic string functionality. */ +static void +test_util_strmisc(void) +{ + char buf[1024]; + int i; + char *cp; + + /* Tests for corner cases of strl operations */ + test_eq(5, strlcpy(buf, "Hello", 0)); + strlcpy(buf, "Hello", sizeof(buf)); + test_eq(10, strlcat(buf, "Hello", 5)); + + /* Test tor_strstrip() */ + strlcpy(buf, "Testing 1 2 3", sizeof(buf)); + tor_strstrip(buf, ",!"); + test_streq(buf, "Testing 1 2 3"); + strlcpy(buf, "!Testing 1 2 3?", sizeof(buf)); + tor_strstrip(buf, "!? "); + test_streq(buf, "Testing123"); + + /* Test tor_parse_long. */ + test_eq(10L, tor_parse_long("10",10,0,100,NULL,NULL)); + test_eq(0L, tor_parse_long("10",10,50,100,NULL,NULL)); + test_eq(-50L, tor_parse_long("-50",10,-100,100,NULL,NULL)); + + /* Test tor_parse_ulong */ + test_eq(10UL, tor_parse_ulong("10",10,0,100,NULL,NULL)); + test_eq(0UL, tor_parse_ulong("10",10,50,100,NULL,NULL)); + + /* Test tor_parse_uint64. */ + test_assert(U64_LITERAL(10) == tor_parse_uint64("10 x",10,0,100, &i, &cp)); + test_assert(i == 1); + test_streq(cp, " x"); + test_assert(U64_LITERAL(12345678901) == + tor_parse_uint64("12345678901",10,0,UINT64_MAX, &i, &cp)); + test_assert(i == 1); + test_streq(cp, ""); + test_assert(U64_LITERAL(0) == + tor_parse_uint64("12345678901",10,500,INT32_MAX, &i, &cp)); + test_assert(i == 0); + + { + /* Test tor_parse_double. */ + double d = tor_parse_double("10", 0, UINT64_MAX,&i,NULL); + test_assert(i == 1); + test_assert(DBL_TO_U64(d) == 10); + d = tor_parse_double("0", 0, UINT64_MAX,&i,NULL); + test_assert(i == 1); + test_assert(DBL_TO_U64(d) == 0); + d = tor_parse_double(" ", 0, UINT64_MAX,&i,NULL); + test_assert(i == 0); + d = tor_parse_double(".0a", 0, UINT64_MAX,&i,NULL); + test_assert(i == 0); + d = tor_parse_double(".0a", 0, UINT64_MAX,&i,&cp); + test_assert(i == 1); + d = tor_parse_double("-.0", 0, UINT64_MAX,&i,NULL); + test_assert(i == 1); + } + + /* Test failing snprintf cases */ + test_eq(-1, tor_snprintf(buf, 0, "Foo")); + test_eq(-1, tor_snprintf(buf, 2, "Foo")); + + /* Test printf with uint64 */ + tor_snprintf(buf, sizeof(buf), "x!"U64_FORMAT"!x", + U64_PRINTF_ARG(U64_LITERAL(12345678901))); + test_streq(buf, "x!12345678901!x"); + + /* Test for strcmpstart and strcmpend. */ + test_assert(strcmpstart("abcdef", "abcdef")==0); + test_assert(strcmpstart("abcdef", "abc")==0); + test_assert(strcmpstart("abcdef", "abd")<0); + test_assert(strcmpstart("abcdef", "abb")>0); + test_assert(strcmpstart("ab", "abb")<0); + + test_assert(strcmpend("abcdef", "abcdef")==0); + test_assert(strcmpend("abcdef", "def")==0); + test_assert(strcmpend("abcdef", "deg")<0); + test_assert(strcmpend("abcdef", "dee")>0); + test_assert(strcmpend("ab", "abb")<0); + + test_assert(strcasecmpend("AbcDEF", "abcdef")==0); + test_assert(strcasecmpend("abcdef", "dEF")==0); + test_assert(strcasecmpend("abcDEf", "deg")<0); + test_assert(strcasecmpend("abcdef", "DEE")>0); + test_assert(strcasecmpend("ab", "abB")<0); + + /* Test mem_is_zero */ + memset(buf,0,128); + buf[128] = 'x'; + test_assert(tor_digest_is_zero(buf)); + test_assert(tor_mem_is_zero(buf, 10)); + test_assert(tor_mem_is_zero(buf, 20)); + test_assert(tor_mem_is_zero(buf, 128)); + test_assert(!tor_mem_is_zero(buf, 129)); + buf[60] = (char)255; + test_assert(!tor_mem_is_zero(buf, 128)); + buf[0] = (char)1; + test_assert(!tor_mem_is_zero(buf, 10)); + + /* Test 'escaped' */ + test_streq("\"\"", escaped("")); + test_streq("\"abcd\"", escaped("abcd")); + test_streq("\"\\\\\\n\\r\\t\\\"\\'\"", escaped("\\\n\r\t\"\'")); + test_streq("\"z\\001abc\\277d\"", escaped("z\001abc\277d")); + test_assert(NULL == escaped(NULL)); + + /* Test strndup and memdup */ + { + const char *s = "abcdefghijklmnopqrstuvwxyz"; + cp = tor_strndup(s, 30); + test_streq(cp, s); /* same string, */ + test_neq(cp, s); /* but different pointers. */ + tor_free(cp); + + cp = tor_strndup(s, 5); + test_streq(cp, "abcde"); + tor_free(cp); + + s = "a\0b\0c\0d\0e\0"; + cp = tor_memdup(s,10); + test_memeq(cp, s, 10); /* same ram, */ + test_neq(cp, s); /* but different pointers. */ + tor_free(cp); + } + + /* Test str-foo functions */ + cp = tor_strdup("abcdef"); + test_assert(tor_strisnonupper(cp)); + cp[3] = 'D'; + test_assert(!tor_strisnonupper(cp)); + tor_strupper(cp); + test_streq(cp, "ABCDEF"); + test_assert(tor_strisprint(cp)); + cp[3] = 3; + test_assert(!tor_strisprint(cp)); + tor_free(cp); + + /* Test eat_whitespace. */ + { + const char *s = " \n a"; + test_eq_ptr(eat_whitespace(s), s+4); + s = "abcd"; + test_eq_ptr(eat_whitespace(s), s); + s = "#xyz\nab"; + test_eq_ptr(eat_whitespace(s), s+5); + } + + /* Test memmem and memstr */ + { + const char *haystack = "abcde"; + tor_assert(!tor_memmem(haystack, 5, "ef", 2)); + test_eq_ptr(tor_memmem(haystack, 5, "cd", 2), haystack + 2); + test_eq_ptr(tor_memmem(haystack, 5, "cde", 3), haystack + 2); + haystack = "ababcad"; + test_eq_ptr(tor_memmem(haystack, 7, "abc", 3), haystack + 2); + test_eq_ptr(tor_memstr(haystack, 7, "abc"), haystack + 2); + test_assert(!tor_memstr(haystack, 7, "fe")); + test_assert(!tor_memstr(haystack, 7, "longerthantheoriginal")); + } + + /* Test wrap_string */ + { + smartlist_t *sl = smartlist_create(); + wrap_string(sl, "This is a test of string wrapping functionality: woot.", + 10, "", ""); + cp = smartlist_join_strings(sl, "", 0, NULL); + test_streq(cp, + "This is a\ntest of\nstring\nwrapping\nfunctional\nity: woot.\n"); + tor_free(cp); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + wrap_string(sl, "This is a test of string wrapping functionality: woot.", + 16, "### ", "# "); + cp = smartlist_join_strings(sl, "", 0, NULL); + test_streq(cp, + "### This is a\n# test of string\n# wrapping\n# functionality:\n" + "# woot.\n"); + + tor_free(cp); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_free(sl); + } + done: + ; +} + +static void +test_util_pow2(void) +{ + /* Test tor_log2(). */ + test_eq(tor_log2(64), 6); + test_eq(tor_log2(65), 6); + test_eq(tor_log2(63), 5); + test_eq(tor_log2(1), 0); + test_eq(tor_log2(2), 1); + test_eq(tor_log2(3), 1); + test_eq(tor_log2(4), 2); + test_eq(tor_log2(5), 2); + test_eq(tor_log2(U64_LITERAL(40000000000000000)), 55); + test_eq(tor_log2(UINT64_MAX), 63); + + /* Test round_to_power_of_2 */ + test_eq(round_to_power_of_2(120), 128); + test_eq(round_to_power_of_2(128), 128); + test_eq(round_to_power_of_2(130), 128); + test_eq(round_to_power_of_2(U64_LITERAL(40000000000000000)), + U64_LITERAL(1)<<55); + test_eq(round_to_power_of_2(0), 2); + + done: + ; +} + +/** mutex for thread test to stop the threads hitting data at the same time. */ +static tor_mutex_t *_thread_test_mutex = NULL; +/** mutexes for the thread test to make sure that the threads have to + * interleave somewhat. */ +static tor_mutex_t *_thread_test_start1 = NULL, + *_thread_test_start2 = NULL; +/** Shared strmap for the thread test. */ +static strmap_t *_thread_test_strmap = NULL; +/** The name of thread1 for the thread test */ +static char *_thread1_name = NULL; +/** The name of thread2 for the thread test */ +static char *_thread2_name = NULL; + +static void _thread_test_func(void* _s) ATTR_NORETURN; + +/** How many iterations have the threads in the unit test run? */ +static int t1_count = 0, t2_count = 0; + +/** Helper function for threading unit tests: This function runs in a + * subthread. It grabs its own mutex (start1 or start2) to make sure that it + * should start, then it repeatedly alters _test_thread_strmap protected by + * _thread_test_mutex. */ +static void +_thread_test_func(void* _s) +{ + char *s = _s; + int i, *count; + tor_mutex_t *m; + char buf[64]; + char **cp; + if (!strcmp(s, "thread 1")) { + m = _thread_test_start1; + cp = &_thread1_name; + count = &t1_count; + } else { + m = _thread_test_start2; + cp = &_thread2_name; + count = &t2_count; + } + + tor_snprintf(buf, sizeof(buf), "%lu", tor_get_thread_id()); + *cp = tor_strdup(buf); + + tor_mutex_acquire(m); + + for (i=0; i<10000; ++i) { + tor_mutex_acquire(_thread_test_mutex); + strmap_set(_thread_test_strmap, "last to run", *cp); + ++*count; + tor_mutex_release(_thread_test_mutex); + } + tor_mutex_acquire(_thread_test_mutex); + strmap_set(_thread_test_strmap, s, *cp); + tor_mutex_release(_thread_test_mutex); + + tor_mutex_release(m); + + spawn_exit(); +} + +/** Run unit tests for threading logic. */ +static void +test_util_threads(void) +{ + char *s1 = NULL, *s2 = NULL; + int done = 0, timedout = 0; + time_t started; +#ifndef MS_WINDOWS + struct timeval tv; + tv.tv_sec=0; + tv.tv_usec=10; +#endif +#ifndef TOR_IS_MULTITHREADED + /* Skip this test if we aren't threading. We should be threading most + * everywhere by now. */ + if (1) + return; +#endif + _thread_test_mutex = tor_mutex_new(); + _thread_test_start1 = tor_mutex_new(); + _thread_test_start2 = tor_mutex_new(); + _thread_test_strmap = strmap_new(); + s1 = tor_strdup("thread 1"); + s2 = tor_strdup("thread 2"); + tor_mutex_acquire(_thread_test_start1); + tor_mutex_acquire(_thread_test_start2); + spawn_func(_thread_test_func, s1); + spawn_func(_thread_test_func, s2); + tor_mutex_release(_thread_test_start2); + tor_mutex_release(_thread_test_start1); + started = time(NULL); + while (!done) { + tor_mutex_acquire(_thread_test_mutex); + strmap_assert_ok(_thread_test_strmap); + if (strmap_get(_thread_test_strmap, "thread 1") && + strmap_get(_thread_test_strmap, "thread 2")) { + done = 1; + } else if (time(NULL) > started + 25) { + timedout = done = 1; + } + tor_mutex_release(_thread_test_mutex); +#ifndef MS_WINDOWS + /* Prevent the main thread from starving the worker threads. */ + select(0, NULL, NULL, NULL, &tv); +#endif + } + tor_mutex_acquire(_thread_test_start1); + tor_mutex_release(_thread_test_start1); + tor_mutex_acquire(_thread_test_start2); + tor_mutex_release(_thread_test_start2); + + tor_mutex_free(_thread_test_mutex); + + if (timedout) { + printf("\nTimed out: %d %d", t1_count, t2_count); + test_assert(strmap_get(_thread_test_strmap, "thread 1")); + test_assert(strmap_get(_thread_test_strmap, "thread 2")); + test_assert(!timedout); + } + + /* different thread IDs. */ + test_assert(strcmp(strmap_get(_thread_test_strmap, "thread 1"), + strmap_get(_thread_test_strmap, "thread 2"))); + test_assert(!strcmp(strmap_get(_thread_test_strmap, "thread 1"), + strmap_get(_thread_test_strmap, "last to run")) || + !strcmp(strmap_get(_thread_test_strmap, "thread 2"), + strmap_get(_thread_test_strmap, "last to run"))); + + done: + tor_free(s1); + tor_free(s2); + tor_free(_thread1_name); + tor_free(_thread2_name); + if (_thread_test_strmap) + strmap_free(_thread_test_strmap, NULL); + if (_thread_test_start1) + tor_mutex_free(_thread_test_start1); + if (_thread_test_start2) + tor_mutex_free(_thread_test_start2); +} + +/** Run unit tests for compression functions */ +static void +test_util_gzip(void) +{ + char *buf1=NULL, *buf2=NULL, *buf3=NULL, *cp1, *cp2; + const char *ccp2; + size_t len1, len2; + tor_zlib_state_t *state = NULL; + + buf1 = tor_strdup("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZAAAAAAAAAAAAAAAAAAAZ"); + test_assert(detect_compression_method(buf1, strlen(buf1)) == UNKNOWN_METHOD); + if (is_gzip_supported()) { + test_assert(!tor_gzip_compress(&buf2, &len1, buf1, strlen(buf1)+1, + GZIP_METHOD)); + test_assert(buf2); + test_assert(!memcmp(buf2, "\037\213", 2)); /* Gzip magic. */ + test_assert(detect_compression_method(buf2, len1) == GZIP_METHOD); + + test_assert(!tor_gzip_uncompress(&buf3, &len2, buf2, len1, + GZIP_METHOD, 1, LOG_INFO)); + test_assert(buf3); + test_streq(buf1,buf3); + + tor_free(buf2); + tor_free(buf3); + } + + test_assert(!tor_gzip_compress(&buf2, &len1, buf1, strlen(buf1)+1, + ZLIB_METHOD)); + test_assert(buf2); + test_assert(!memcmp(buf2, "\x78\xDA", 2)); /* deflate magic. */ + test_assert(detect_compression_method(buf2, len1) == ZLIB_METHOD); + + test_assert(!tor_gzip_uncompress(&buf3, &len2, buf2, len1, + ZLIB_METHOD, 1, LOG_INFO)); + test_assert(buf3); + test_streq(buf1,buf3); + + /* Check whether we can uncompress concatenated, compressed strings. */ + tor_free(buf3); + buf2 = tor_realloc(buf2, len1*2); + memcpy(buf2+len1, buf2, len1); + test_assert(!tor_gzip_uncompress(&buf3, &len2, buf2, len1*2, + ZLIB_METHOD, 1, LOG_INFO)); + test_eq(len2, (strlen(buf1)+1)*2); + test_memeq(buf3, + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZAAAAAAAAAAAAAAAAAAAZ\0" + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZAAAAAAAAAAAAAAAAAAAZ\0", + (strlen(buf1)+1)*2); + + tor_free(buf1); + tor_free(buf2); + tor_free(buf3); + + /* Check whether we can uncompress partial strings. */ + buf1 = + tor_strdup("String with low redundancy that won't be compressed much."); + test_assert(!tor_gzip_compress(&buf2, &len1, buf1, strlen(buf1)+1, + ZLIB_METHOD)); + tor_assert(len1>16); + /* when we allow an incomplete string, we should succeed.*/ + tor_assert(!tor_gzip_uncompress(&buf3, &len2, buf2, len1-16, + ZLIB_METHOD, 0, LOG_INFO)); + buf3[len2]='\0'; + tor_assert(len2 > 5); + tor_assert(!strcmpstart(buf1, buf3)); + + /* when we demand a complete string, this must fail. */ + tor_free(buf3); + tor_assert(tor_gzip_uncompress(&buf3, &len2, buf2, len1-16, + ZLIB_METHOD, 1, LOG_INFO)); + tor_assert(!buf3); + + /* Now, try streaming compression. */ + tor_free(buf1); + tor_free(buf2); + tor_free(buf3); + state = tor_zlib_new(1, ZLIB_METHOD); + tor_assert(state); + cp1 = buf1 = tor_malloc(1024); + len1 = 1024; + ccp2 = "ABCDEFGHIJABCDEFGHIJ"; + len2 = 21; + test_assert(tor_zlib_process(state, &cp1, &len1, &ccp2, &len2, 0) + == TOR_ZLIB_OK); + test_eq(len2, 0); /* Make sure we compressed it all. */ + test_assert(cp1 > buf1); + + len2 = 0; + cp2 = cp1; + test_assert(tor_zlib_process(state, &cp1, &len1, &ccp2, &len2, 1) + == TOR_ZLIB_DONE); + test_eq(len2, 0); + test_assert(cp1 > cp2); /* Make sure we really added something. */ + + tor_assert(!tor_gzip_uncompress(&buf3, &len2, buf1, 1024-len1, + ZLIB_METHOD, 1, LOG_WARN)); + test_streq(buf3, "ABCDEFGHIJABCDEFGHIJ"); /*Make sure it compressed right.*/ + + done: + if (state) + tor_zlib_free(state); + tor_free(buf2); + tor_free(buf3); + tor_free(buf1); +} + +/** Run unit tests for mmap() wrapper functionality. */ +static void +test_util_mmap(void) +{ + char *fname1 = tor_strdup(get_fname("mapped_1")); + char *fname2 = tor_strdup(get_fname("mapped_2")); + char *fname3 = tor_strdup(get_fname("mapped_3")); + const size_t buflen = 17000; + char *buf = tor_malloc(17000); + tor_mmap_t *mapping = NULL; + + crypto_rand(buf, buflen); + + mapping = tor_mmap_file(fname1); + test_assert(! mapping); + + write_str_to_file(fname1, "Short file.", 1); + write_bytes_to_file(fname2, buf, buflen, 1); + write_bytes_to_file(fname3, buf, 16384, 1); + + mapping = tor_mmap_file(fname1); + test_assert(mapping); + test_eq(mapping->size, strlen("Short file.")); + test_streq(mapping->data, "Short file."); +#ifdef MS_WINDOWS + tor_munmap_file(mapping); + mapping = NULL; + test_assert(unlink(fname1) == 0); +#else + /* make sure we can unlink. */ + test_assert(unlink(fname1) == 0); + test_streq(mapping->data, "Short file."); + tor_munmap_file(mapping); + mapping = NULL; +#endif + + /* Now a zero-length file. */ + write_str_to_file(fname1, "", 1); + mapping = tor_mmap_file(fname1); + test_eq(mapping, NULL); + test_eq(ERANGE, errno); + unlink(fname1); + + /* Make sure that we fail to map a no-longer-existent file. */ + mapping = tor_mmap_file(fname1); + test_assert(mapping == NULL); + + /* Now try a big file that stretches across a few pages and isn't aligned */ + mapping = tor_mmap_file(fname2); + test_assert(mapping); + test_eq(mapping->size, buflen); + test_memeq(mapping->data, buf, buflen); + tor_munmap_file(mapping); + mapping = NULL; + + /* Now try a big aligned file. */ + mapping = tor_mmap_file(fname3); + test_assert(mapping); + test_eq(mapping->size, 16384); + test_memeq(mapping->data, buf, 16384); + tor_munmap_file(mapping); + mapping = NULL; + + done: + unlink(fname1); + unlink(fname2); + unlink(fname3); + + tor_free(fname1); + tor_free(fname2); + tor_free(fname3); + tor_free(buf); + + if (mapping) + tor_munmap_file(mapping); +} + +/** Run unit tests for escaping/unescaping data for use by controllers. */ +static void +test_util_control_formats(void) +{ + char *out = NULL; + const char *inp = + "..This is a test\r\nof the emergency \nbroadcast\r\n..system.\r\nZ.\r\n"; + size_t sz; + + sz = read_escaped_data(inp, strlen(inp), &out); + test_streq(out, + ".This is a test\nof the emergency \nbroadcast\n.system.\nZ.\n"); + test_eq(sz, strlen(out)); + + done: + tor_free(out); +} + +static void +test_util_sscanf(void) +{ + unsigned u1, u2, u3; + char s1[10], s2[10], s3[10], ch; + int r; + + r = tor_sscanf("hello world", "hello world"); /* String match: success */ + test_eq(r, 0); + r = tor_sscanf("hello world 3", "hello worlb %u", &u1); /* String fail */ + test_eq(r, 0); + r = tor_sscanf("12345", "%u", &u1); /* Simple number */ + test_eq(r, 1); + test_eq(u1, 12345u); + r = tor_sscanf("", "%u", &u1); /* absent number */ + test_eq(r, 0); + r = tor_sscanf("A", "%u", &u1); /* bogus number */ + test_eq(r, 0); + r = tor_sscanf("4294967295", "%u", &u1); /* UINT32_MAX should work. */ + test_eq(r, 1); + test_eq(u1, 4294967295u); + r = tor_sscanf("4294967296", "%u", &u1); /* Always say -1 at 32 bits. */ + test_eq(r, 0); + r = tor_sscanf("123456", "%2u%u", &u1, &u2); /* Width */ + test_eq(r, 2); + test_eq(u1, 12u); + test_eq(u2, 3456u); + r = tor_sscanf("!12:3:456", "!%2u:%2u:%3u", &u1, &u2, &u3); /* separators */ + test_eq(r, 3); + test_eq(u1, 12u); + test_eq(u2, 3u); + test_eq(u3, 456u); + r = tor_sscanf("12:3:045", "%2u:%2u:%3u", &u1, &u2, &u3); /* 0s */ + test_eq(r, 3); + test_eq(u1, 12u); + test_eq(u2, 3u); + test_eq(u3, 45u); + /* %u does not match space.*/ + r = tor_sscanf("12:3: 45", "%2u:%2u:%3u", &u1, &u2, &u3); + test_eq(r, 2); + /* %u does not match negative numbers. */ + r = tor_sscanf("12:3:-4", "%2u:%2u:%3u", &u1, &u2, &u3); + test_eq(r, 2); + /* Arbitrary amounts of 0-padding are okay */ + r = tor_sscanf("12:03:000000000000000099", "%2u:%2u:%u", &u1, &u2, &u3); + test_eq(r, 3); + test_eq(u1, 12u); + test_eq(u2, 3u); + test_eq(u3, 99u); + + r = tor_sscanf("99% fresh", "%3u%% fresh", &u1); /* percents are scannable.*/ + test_eq(r, 1); + test_eq(u1, 99); + + r = tor_sscanf("hello", "%s", s1); /* %s needs a number. */ + test_eq(r, -1); + + r = tor_sscanf("hello", "%3s%7s", s1, s2); /* %s matches characters. */ + test_eq(r, 2); + test_streq(s1, "hel"); + test_streq(s2, "lo"); + r = tor_sscanf("WD40", "%2s%u", s3, &u1); /* %s%u */ + test_eq(r, 2); + test_streq(s3, "WD"); + test_eq(u1, 40); + r = tor_sscanf("76trombones", "%6u%9s", &u1, s1); /* %u%s */ + test_eq(r, 2); + test_eq(u1, 76); + test_streq(s1, "trombones"); + r = tor_sscanf("hello world", "%9s %9s", s1, s2); /* %s doesn't eat space. */ + test_eq(r, 2); + test_streq(s1, "hello"); + test_streq(s2, "world"); + r = tor_sscanf("hi", "%9s%9s%3s", s1, s2, s3); /* %s can be empty. */ + test_eq(r, 3); + test_streq(s1, "hi"); + test_streq(s2, ""); + test_streq(s3, ""); + + r = tor_sscanf("1.2.3", "%u.%u.%u%c", &u1, &u2, &u3, &ch); + test_eq(r, 3); + r = tor_sscanf("1.2.3 foobar", "%u.%u.%u%c", &u1, &u2, &u3, &ch); + test_eq(r, 4); + + done: + ; +} + +/** Run unittests for memory pool allocator */ +static void +test_util_mempool(void) +{ + mp_pool_t *pool = NULL; + smartlist_t *allocated = NULL; + int i; + + pool = mp_pool_new(1, 100); + test_assert(pool); + test_assert(pool->new_chunk_capacity >= 100); + test_assert(pool->item_alloc_size >= sizeof(void*)+1); + mp_pool_destroy(pool); + pool = NULL; + + pool = mp_pool_new(241, 2500); + test_assert(pool); + test_assert(pool->new_chunk_capacity >= 10); + test_assert(pool->item_alloc_size >= sizeof(void*)+241); + test_eq(pool->item_alloc_size & 0x03, 0); + test_assert(pool->new_chunk_capacity < 60); + + allocated = smartlist_create(); + for (i = 0; i < 20000; ++i) { + if (smartlist_len(allocated) < 20 || crypto_rand_int(2)) { + void *m = mp_pool_get(pool); + memset(m, 0x09, 241); + smartlist_add(allocated, m); + //printf("%d: %p\n", i, m); + //mp_pool_assert_ok(pool); + } else { + int idx = crypto_rand_int(smartlist_len(allocated)); + void *m = smartlist_get(allocated, idx); + //printf("%d: free %p\n", i, m); + smartlist_del(allocated, idx); + mp_pool_release(m); + //mp_pool_assert_ok(pool); + } + if (crypto_rand_int(777)==0) + mp_pool_clean(pool, 1, 1); + + if (i % 777) + mp_pool_assert_ok(pool); + } + + done: + if (allocated) { + SMARTLIST_FOREACH(allocated, void *, m, mp_pool_release(m)); + mp_pool_assert_ok(pool); + mp_pool_clean(pool, 0, 0); + mp_pool_assert_ok(pool); + smartlist_free(allocated); + } + + if (pool) + mp_pool_destroy(pool); +} + +/** Run unittests for memory area allocator */ +static void +test_util_memarea(void) +{ + memarea_t *area = memarea_new(); + char *p1, *p2, *p3, *p1_orig; + void *malloced_ptr = NULL; + int i; + + test_assert(area); + + p1_orig = p1 = memarea_alloc(area,64); + p2 = memarea_alloc_zero(area,52); + p3 = memarea_alloc(area,11); + + test_assert(memarea_owns_ptr(area, p1)); + test_assert(memarea_owns_ptr(area, p2)); + test_assert(memarea_owns_ptr(area, p3)); + /* Make sure we left enough space. */ + test_assert(p1+64 <= p2); + test_assert(p2+52 <= p3); + /* Make sure we aligned. */ + test_eq(((uintptr_t)p1) % sizeof(void*), 0); + test_eq(((uintptr_t)p2) % sizeof(void*), 0); + test_eq(((uintptr_t)p3) % sizeof(void*), 0); + test_assert(!memarea_owns_ptr(area, p3+8192)); + test_assert(!memarea_owns_ptr(area, p3+30)); + test_assert(tor_mem_is_zero(p2, 52)); + /* Make sure we don't overalign. */ + p1 = memarea_alloc(area, 1); + p2 = memarea_alloc(area, 1); + test_eq(p1+sizeof(void*), p2); + { + malloced_ptr = tor_malloc(64); + test_assert(!memarea_owns_ptr(area, malloced_ptr)); + tor_free(malloced_ptr); + } + + /* memarea_memdup */ + { + malloced_ptr = tor_malloc(64); + crypto_rand((char*)malloced_ptr, 64); + p1 = memarea_memdup(area, malloced_ptr, 64); + test_assert(p1 != malloced_ptr); + test_memeq(p1, malloced_ptr, 64); + tor_free(malloced_ptr); + } + + /* memarea_strdup. */ + p1 = memarea_strdup(area,""); + p2 = memarea_strdup(area, "abcd"); + test_assert(p1); + test_assert(p2); + test_streq(p1, ""); + test_streq(p2, "abcd"); + + /* memarea_strndup. */ + { + const char *s = "Ad ogni porta batte la morte e grida: il nome!"; + /* (From Turandot, act 3.) */ + size_t len = strlen(s); + p1 = memarea_strndup(area, s, 1000); + p2 = memarea_strndup(area, s, 10); + test_streq(p1, s); + test_assert(p2 >= p1 + len + 1); + test_memeq(s, p2, 10); + test_eq(p2[10], '\0'); + p3 = memarea_strndup(area, s, len); + test_streq(p3, s); + p3 = memarea_strndup(area, s, len-1); + test_memeq(s, p3, len-1); + test_eq(p3[len-1], '\0'); + } + + memarea_clear(area); + p1 = memarea_alloc(area, 1); + test_eq(p1, p1_orig); + memarea_clear(area); + + /* Check for running over an area's size. */ + for (i = 0; i < 512; ++i) { + p1 = memarea_alloc(area, crypto_rand_int(5)+1); + test_assert(memarea_owns_ptr(area, p1)); + } + memarea_assert_ok(area); + /* Make sure we can allocate a too-big object. */ + p1 = memarea_alloc_zero(area, 9000); + p2 = memarea_alloc_zero(area, 16); + test_assert(memarea_owns_ptr(area, p1)); + test_assert(memarea_owns_ptr(area, p2)); + + done: + memarea_drop_all(area); + tor_free(malloced_ptr); +} + +/** Run unit tests for utility functions to get file names relative to + * the data directory. */ +static void +test_util_datadir(void) +{ + char buf[1024]; + char *f = NULL; + char *temp_dir = NULL; + + temp_dir = get_datadir_fname(NULL); + f = get_datadir_fname("state"); + tor_snprintf(buf, sizeof(buf), "%s"PATH_SEPARATOR"state", temp_dir); + test_streq(f, buf); + tor_free(f); + f = get_datadir_fname2("cache", "thingy"); + tor_snprintf(buf, sizeof(buf), + "%s"PATH_SEPARATOR"cache"PATH_SEPARATOR"thingy", temp_dir); + test_streq(f, buf); + tor_free(f); + f = get_datadir_fname2_suffix("cache", "thingy", ".foo"); + tor_snprintf(buf, sizeof(buf), + "%s"PATH_SEPARATOR"cache"PATH_SEPARATOR"thingy.foo", temp_dir); + test_streq(f, buf); + tor_free(f); + f = get_datadir_fname_suffix("cache", ".foo"); + tor_snprintf(buf, sizeof(buf), "%s"PATH_SEPARATOR"cache.foo", + temp_dir); + test_streq(f, buf); + + done: + tor_free(f); + tor_free(temp_dir); +} + +static void +test_util_strtok(void) +{ + char buf[128]; + char buf2[128]; + char *cp1, *cp2; + strlcpy(buf, "Graved on the dark in gestures of descent", sizeof(buf)); + strlcpy(buf2, "they.seemed;their!own;most.perfect;monument", sizeof(buf2)); + /* -- "Year's End", Richard Wilbur */ + + test_streq("Graved", tor_strtok_r_impl(buf, " ", &cp1)); + test_streq("they", tor_strtok_r_impl(buf2, ".!..;!", &cp2)); +#define S1() tor_strtok_r_impl(NULL, " ", &cp1) +#define S2() tor_strtok_r_impl(NULL, ".!..;!", &cp2) + test_streq("on", S1()); + test_streq("the", S1()); + test_streq("dark", S1()); + test_streq("seemed", S2()); + test_streq("their", S2()); + test_streq("own", S2()); + test_streq("in", S1()); + test_streq("gestures", S1()); + test_streq("of", S1()); + test_streq("most", S2()); + test_streq("perfect", S2()); + test_streq("descent", S1()); + test_streq("monument", S2()); + test_assert(NULL == S1()); + test_assert(NULL == S2()); + done: + ; +} + +static void +test_util_find_str_at_start_of_line(void *ptr) +{ + const char *long_string = + "hello world. hello world. hello hello. howdy.\n" + "hello hello world\n"; + + (void)ptr; + + /* not-found case. */ + tt_assert(! find_str_at_start_of_line(long_string, "fred")); + + /* not-found case where haystack doesn't end with \n */ + tt_assert(! find_str_at_start_of_line("foobar\nbaz", "fred")); + + /* start-of-string case */ + tt_assert(long_string == + find_str_at_start_of_line(long_string, "hello world.")); + + /* start-of-line case */ + tt_assert(strchr(long_string,'\n')+1 == + find_str_at_start_of_line(long_string, "hello hello")); + done: + ; +} + +static void +test_util_asprintf(void *ptr) +{ +#define LOREMIPSUM \ + "Lorem ipsum dolor sit amet, consectetur adipisicing elit" + char *cp=NULL, *cp2=NULL; + int r; + (void)ptr; + + /* empty string. */ + r = tor_asprintf(&cp, "%s", ""); + tt_assert(cp); + tt_int_op(r, ==, strlen(cp)); + tt_str_op(cp, ==, ""); + + /* Short string with some printing in it. */ + r = tor_asprintf(&cp2, "First=%d, Second=%d", 101, 202); + tt_assert(cp2); + tt_int_op(r, ==, strlen(cp2)); + tt_str_op(cp2, ==, "First=101, Second=202"); + tt_assert(cp != cp2); + tor_free(cp); + tor_free(cp2); + + /* Glass-box test: a string exactly 128 characters long. */ + r = tor_asprintf(&cp, "Lorem1: %sLorem2: %s", LOREMIPSUM, LOREMIPSUM); + tt_assert(cp); + tt_int_op(r, ==, 128); + tt_assert(cp[128] == '\0'); + tt_str_op(cp, ==, + "Lorem1: "LOREMIPSUM"Lorem2: "LOREMIPSUM); + tor_free(cp); + + /* String longer than 128 characters */ + r = tor_asprintf(&cp, "1: %s 2: %s 3: %s", + LOREMIPSUM, LOREMIPSUM, LOREMIPSUM); + tt_assert(cp); + tt_int_op(r, ==, strlen(cp)); + tt_str_op(cp, ==, "1: "LOREMIPSUM" 2: "LOREMIPSUM" 3: "LOREMIPSUM); + + done: + tor_free(cp); + tor_free(cp2); +} + +static void +test_util_listdir(void *ptr) +{ + smartlist_t *dir_contents = NULL; + char *fname1=NULL, *fname2=NULL, *dirname=NULL; + (void)ptr; + + fname1 = tor_strdup(get_fname("hopscotch")); + fname2 = tor_strdup(get_fname("mumblety-peg")); + dirname = tor_strdup(get_fname(NULL)); + + tt_int_op(write_str_to_file(fname1, "X\n", 0), ==, 0); + tt_int_op(write_str_to_file(fname2, "Y\n", 0), ==, 0); + + dir_contents = tor_listdir(dirname); + tt_assert(dir_contents); + /* make sure that each filename is listed. */ + tt_assert(smartlist_string_isin_case(dir_contents, "hopscotch")); + tt_assert(smartlist_string_isin_case(dir_contents, "mumblety-peg")); + + tt_assert(!smartlist_string_isin(dir_contents, ".")); + tt_assert(!smartlist_string_isin(dir_contents, "..")); + + done: + tor_free(fname1); + tor_free(fname2); + tor_free(dirname); + if (dir_contents) { + SMARTLIST_FOREACH(dir_contents, char *, cp, tor_free(cp)); + smartlist_free(dir_contents); + } +} + +#ifdef MS_WINDOWS +static void +test_util_load_win_lib(void *ptr) +{ + HANDLE h = load_windows_system_library("advapi32.dll"); + (void) ptr; + + tt_assert(h); + done: + if (h) + CloseHandle(h); +} +#endif + +#define UTIL_LEGACY(name) \ + { #name, legacy_test_helper, 0, &legacy_setup, test_util_ ## name } + +#define UTIL_TEST(name, flags) \ + { #name, test_util_ ## name, flags, NULL, NULL } + +struct testcase_t util_tests[] = { + UTIL_LEGACY(time), + UTIL_LEGACY(config_line), + UTIL_LEGACY(strmisc), + UTIL_LEGACY(pow2), + UTIL_LEGACY(gzip), + UTIL_LEGACY(datadir), + UTIL_LEGACY(mempool), + UTIL_LEGACY(memarea), + UTIL_LEGACY(control_formats), + UTIL_LEGACY(mmap), + UTIL_LEGACY(threads), + UTIL_LEGACY(sscanf), + UTIL_LEGACY(strtok), + UTIL_TEST(find_str_at_start_of_line, 0), + UTIL_TEST(asprintf, 0), + UTIL_TEST(listdir, 0), +#ifdef MS_WINDOWS + UTIL_TEST(load_win_lib, 0), +#endif + END_OF_TESTCASES +}; + diff --git a/src/test/tinytest.c b/src/test/tinytest.c new file mode 100644 index 0000000000..11ffc2fe56 --- /dev/null +++ b/src/test/tinytest.c @@ -0,0 +1,380 @@ +/* tinytest.c -- Copyright 2009-2010 Nick Mathewson + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <assert.h> + +#ifdef WIN32 +#include <windows.h> +#else +#include <sys/types.h> +#include <sys/wait.h> +#include <unistd.h> +#endif + +#ifndef __GNUC__ +#define __attribute__(x) +#endif + +#ifdef TINYTEST_LOCAL +#include "tinytest_local.h" +#endif +#include "tinytest.h" +#include "tinytest_macros.h" + +#define LONGEST_TEST_NAME 16384 + +static int in_tinytest_main = 0; /**< true if we're in tinytest_main().*/ +static int n_ok = 0; /**< Number of tests that have passed */ +static int n_bad = 0; /**< Number of tests that have failed. */ +static int n_skipped = 0; /**< Number of tests that have been skipped. */ + +static int opt_forked = 0; /**< True iff we're called from inside a win32 fork*/ +static int opt_nofork = 0; /**< Suppress calls to fork() for debugging. */ +static int opt_verbosity = 1; /**< -==quiet,0==terse,1==normal,2==verbose */ +const char *verbosity_flag = ""; + +enum outcome { SKIP=2, OK=1, FAIL=0 }; +static enum outcome cur_test_outcome = 0; +const char *cur_test_prefix = NULL; /**< prefix of the current test group */ +/** Name of the current test, if we haven't logged is yet. Used for --quiet */ +const char *cur_test_name = NULL; + +#ifdef WIN32 +/** Pointer to argv[0] for win32. */ +static const char *commandname = NULL; +#endif + +static void usage(struct testgroup_t *groups, int list_groups) + __attribute__((noreturn)); + +static enum outcome +_testcase_run_bare(const struct testcase_t *testcase) +{ + void *env = NULL; + int outcome; + if (testcase->setup) { + env = testcase->setup->setup_fn(testcase); + if (!env) + return FAIL; + else if (env == (void*)TT_SKIP) + return SKIP; + } + + cur_test_outcome = OK; + testcase->fn(env); + outcome = cur_test_outcome; + + if (testcase->setup) { + if (testcase->setup->cleanup_fn(testcase, env) == 0) + outcome = FAIL; + } + + return outcome; +} + +#define MAGIC_EXITCODE 42 + +static enum outcome +_testcase_run_forked(const struct testgroup_t *group, + const struct testcase_t *testcase) +{ +#ifdef WIN32 + /* Fork? On Win32? How primitive! We'll do what the smart kids do: + we'll invoke our own exe (whose name we recall from the command + line) with a command line that tells it to run just the test we + want, and this time without forking. + + (No, threads aren't an option. The whole point of forking is to + share no state between tests.) + */ + int ok; + char buffer[LONGEST_TEST_NAME+256]; + STARTUPINFOA si; + PROCESS_INFORMATION info; + DWORD exitcode; + + if (!in_tinytest_main) { + printf("\nERROR. On Windows, _testcase_run_forked must be" + " called from within tinytest_main.\n"); + abort(); + } + if (opt_verbosity>0) + printf("[forking] "); + + snprintf(buffer, sizeof(buffer), "%s --RUNNING-FORKED %s %s%s", + commandname, verbosity_flag, group->prefix, testcase->name); + + memset(&si, 0, sizeof(si)); + memset(&info, 0, sizeof(info)); + si.cb = sizeof(si); + + ok = CreateProcessA(commandname, buffer, NULL, NULL, 0, + 0, NULL, NULL, &si, &info); + if (!ok) { + printf("CreateProcess failed!\n"); + return 0; + } + WaitForSingleObject(info.hProcess, INFINITE); + GetExitCodeProcess(info.hProcess, &exitcode); + CloseHandle(info.hProcess); + CloseHandle(info.hThread); + if (exitcode == 0) + return OK; + else if (exitcode == MAGIC_EXITCODE) + return SKIP; + else + return FAIL; +#else + int outcome_pipe[2]; + pid_t pid; + (void)group; + + if (pipe(outcome_pipe)) + perror("opening pipe"); + + if (opt_verbosity>0) + printf("[forking] "); + pid = fork(); + if (!pid) { + /* child. */ + int test_r, write_r; + char b[1]; + close(outcome_pipe[0]); + test_r = _testcase_run_bare(testcase); + assert(0<=(int)test_r && (int)test_r<=2); + b[0] = "NYS"[test_r]; + write_r = (int)write(outcome_pipe[1], b, 1); + if (write_r != 1) { + perror("write outcome to pipe"); + exit(1); + } + exit(0); + } else { + /* parent */ + int status, r; + char b[1]; + /* Close this now, so that if the other side closes it, + * our read fails. */ + close(outcome_pipe[1]); + r = (int)read(outcome_pipe[0], b, 1); + if (r == 0) { + printf("[Lost connection!] "); + return 0; + } else if (r != 1) { + perror("read outcome from pipe"); + } + waitpid(pid, &status, 0); + close(outcome_pipe[0]); + return b[0]=='Y' ? OK : (b[0]=='S' ? SKIP : FAIL); + } +#endif +} + +int +testcase_run_one(const struct testgroup_t *group, + const struct testcase_t *testcase) +{ + enum outcome outcome; + + if (testcase->flags & TT_SKIP) { + if (opt_verbosity>0) + printf("%s%s: SKIPPED\n", + group->prefix, testcase->name); + ++n_skipped; + return SKIP; + } + + if (opt_verbosity>0 && !opt_forked) { + printf("%s%s: ", group->prefix, testcase->name); + } else { + if (opt_verbosity==0) printf("."); + cur_test_prefix = group->prefix; + cur_test_name = testcase->name; + } + + if ((testcase->flags & TT_FORK) && !(opt_forked||opt_nofork)) { + outcome = _testcase_run_forked(group, testcase); + } else { + outcome = _testcase_run_bare(testcase); + } + + if (outcome == OK) { + ++n_ok; + if (opt_verbosity>0 && !opt_forked) + puts(opt_verbosity==1?"OK":""); + } else if (outcome == SKIP) { + ++n_skipped; + if (opt_verbosity>0 && !opt_forked) + puts("SKIPPED"); + } else { + ++n_bad; + if (!opt_forked) + printf("\n [%s FAILED]\n", testcase->name); + } + + if (opt_forked) { + exit(outcome==OK ? 0 : (outcome==SKIP?MAGIC_EXITCODE : 1)); + } else { + return (int)outcome; + } +} + +int +_tinytest_set_flag(struct testgroup_t *groups, const char *arg, unsigned long flag) +{ + int i, j; + size_t length = LONGEST_TEST_NAME; + char fullname[LONGEST_TEST_NAME]; + int found=0; + if (strstr(arg, "..")) + length = strstr(arg,"..")-arg; + for (i=0; groups[i].prefix; ++i) { + for (j=0; groups[i].cases[j].name; ++j) { + snprintf(fullname, sizeof(fullname), "%s%s", + groups[i].prefix, groups[i].cases[j].name); + if (!flag) /* Hack! */ + printf(" %s\n", fullname); + if (!strncmp(fullname, arg, length)) { + groups[i].cases[j].flags |= flag; + ++found; + } + } + } + return found; +} + +static void +usage(struct testgroup_t *groups, int list_groups) +{ + puts("Options are: [--verbose|--quiet|--terse] [--no-fork]"); + puts(" Specify tests by name, or using a prefix ending with '..'"); + puts(" To skip a test, list give its name prefixed with a colon."); + puts(" Use --list-tests for a list of tests."); + if (list_groups) { + puts("Known tests are:"); + _tinytest_set_flag(groups, "..", 0); + } + exit(0); +} + +int +tinytest_main(int c, const char **v, struct testgroup_t *groups) +{ + int i, j, n=0; + +#ifdef WIN32 + commandname = v[0]; +#endif + for (i=1; i<c; ++i) { + if (v[i][0] == '-') { + if (!strcmp(v[i], "--RUNNING-FORKED")) { + opt_forked = 1; + } else if (!strcmp(v[i], "--no-fork")) { + opt_nofork = 1; + } else if (!strcmp(v[i], "--quiet")) { + opt_verbosity = -1; + verbosity_flag = "--quiet"; + } else if (!strcmp(v[i], "--verbose")) { + opt_verbosity = 2; + verbosity_flag = "--verbose"; + } else if (!strcmp(v[i], "--terse")) { + opt_verbosity = 0; + verbosity_flag = "--terse"; + } else if (!strcmp(v[i], "--help")) { + usage(groups, 0); + } else if (!strcmp(v[i], "--list-tests")) { + usage(groups, 1); + } else { + printf("Unknown option %s. Try --help\n",v[i]); + return -1; + } + } else { + const char *test = v[i]; + int flag = _TT_ENABLED; + if (test[0] == ':') { + ++test; + flag = TT_SKIP; + } else { + ++n; + } + if (!_tinytest_set_flag(groups, test, flag)) { + printf("No such test as %s!\n", v[i]); + return -1; + } + } + } + if (!n) + _tinytest_set_flag(groups, "..", _TT_ENABLED); + + setvbuf(stdout, NULL, _IONBF, 0); + + ++in_tinytest_main; + for (i=0; groups[i].prefix; ++i) + for (j=0; groups[i].cases[j].name; ++j) + if (groups[i].cases[j].flags & _TT_ENABLED) + testcase_run_one(&groups[i], + &groups[i].cases[j]); + + --in_tinytest_main; + + if (opt_verbosity==0) + puts(""); + + if (n_bad) + printf("%d/%d TESTS FAILED. (%d skipped)\n", n_bad, + n_bad+n_ok,n_skipped); + else if (opt_verbosity >= 1) + printf("%d tests ok. (%d skipped)\n", n_ok, n_skipped); + + return (n_bad == 0) ? 0 : 1; +} + +int +_tinytest_get_verbosity(void) +{ + return opt_verbosity; +} + +void +_tinytest_set_test_failed(void) +{ + if (opt_verbosity <= 0 && cur_test_name) { + if (opt_verbosity==0) puts(""); + printf("%s%s: ", cur_test_prefix, cur_test_name); + cur_test_name = NULL; + } + cur_test_outcome = 0; +} + +void +_tinytest_set_test_skipped(void) +{ + if (cur_test_outcome==OK) + cur_test_outcome = SKIP; +} + diff --git a/src/test/tinytest.h b/src/test/tinytest.h new file mode 100644 index 0000000000..cbe28b7f51 --- /dev/null +++ b/src/test/tinytest.h @@ -0,0 +1,87 @@ +/* tinytest.h -- Copyright 2009-2010 Nick Mathewson + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _TINYTEST_H +#define _TINYTEST_H + +/** Flag for a test that needs to run in a subprocess. */ +#define TT_FORK (1<<0) +/** Runtime flag for a test we've decided to skip. */ +#define TT_SKIP (1<<1) +/** Internal runtime flag for a test we've decided to run. */ +#define _TT_ENABLED (1<<2) +/** If you add your own flags, make them start at this point. */ +#define TT_FIRST_USER_FLAG (1<<3) + +typedef void (*testcase_fn)(void *); + +struct testcase_t; + +/** Functions to initialize/teardown a structure for a testcase. */ +struct testcase_setup_t { + /** Return a new structure for use by a given testcase. */ + void *(*setup_fn)(const struct testcase_t *); + /** Clean/free a structure from setup_fn. Return 1 if ok, 0 on err. */ + int (*cleanup_fn)(const struct testcase_t *, void *); +}; + +/** A single test-case that you can run. */ +struct testcase_t { + const char *name; /**< An identifier for this case. */ + testcase_fn fn; /**< The function to run to implement this case. */ + unsigned long flags; /**< Bitfield of TT_* flags. */ + const struct testcase_setup_t *setup; /**< Optional setup/cleanup fns*/ + void *setup_data; /**< Extra data usable by setup function */ +}; +#define END_OF_TESTCASES { NULL, NULL, 0, NULL, NULL } + +/** A group of tests that are selectable together. */ +struct testgroup_t { + const char *prefix; /**< Prefix to prepend to testnames. */ + struct testcase_t *cases; /** Array, ending with END_OF_TESTCASES */ +}; +#define END_OF_GROUPS { NULL, NULL} + +/** Implementation: called from a test to indicate failure, before logging. */ +void _tinytest_set_test_failed(void); +/** Implementation: called from a test to indicate that we're skipping. */ +void _tinytest_set_test_skipped(void); +/** Implementation: return 0 for quiet, 1 for normal, 2 for loud. */ +int _tinytest_get_verbosity(void); +/** Implementation: Set a flag on tests matching a name; returns number + * of tests that matched. */ +int _tinytest_set_flag(struct testgroup_t *, const char *, unsigned long); + +/** Set all tests in 'groups' matching the name 'named' to be skipped. */ +#define tinytest_skip(groups, named) \ + _tinytest_set_flag(groups, named, TT_SKIP) + +/** Run a single testcase in a single group. */ +int testcase_run_one(const struct testgroup_t *,const struct testcase_t *); +/** Run a set of testcases from an END_OF_GROUPS-terminated array of groups, + as selected from the command line. */ +int tinytest_main(int argc, const char **argv, struct testgroup_t *groups); + +#endif diff --git a/src/test/tinytest_demo.c b/src/test/tinytest_demo.c new file mode 100644 index 0000000000..bd33cc37fa --- /dev/null +++ b/src/test/tinytest_demo.c @@ -0,0 +1,215 @@ +/* tinytest_demo.c -- Copyright 2009 Nick Mathewson + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + + +/* Welcome to the example file for tinytest! I'll show you how to set up + * some simple and not-so-simple testcases. */ + +/* Make sure you include these headers. */ +#include "tinytest.h" +#include "tinytest_macros.h" + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <errno.h> + +/* ============================================================ */ + +/* First, let's see if strcmp is working. (All your test cases should be + * functions declared to take a single void * as) an argument. */ +void +test_strcmp(void *data) +{ + (void)data; /* This testcase takes no data. */ + + /* Let's make sure the empty string is equal to itself */ + if (strcmp("","")) { + /* This macro tells tinytest to stop the current test + * and go straight to the "end" label. */ + tt_abort_msg("The empty string was not equal to itself"); + } + + /* Pretty often, calling tt_abort_msg to indicate failure is more + heavy-weight than you want. Instead, just say: */ + tt_assert(strcmp("testcase", "testcase") == 0); + + /* Occasionally, you don't want to stop the current testcase just + because a single assertion has failed. In that case, use + tt_want: */ + tt_want(strcmp("tinytest", "testcase") > 0); + + /* You can use the tt_*_op family of macros to compare values and to + fail unless they have the relationship you want. They produce + more useful output than tt_assert, since they display the actual + values of the failing things. + + Fail unless strcmp("abc, "abc") == 0 */ + tt_int_op(strcmp("abc", "abc"), ==, 0); + + /* Fail unless strcmp("abc, "abcd") is less than 0 */ + tt_int_op(strcmp("abc", "abcd"), < , 0); + + /* Incidentally, there's a test_str_op that uses strcmp internally. */ + tt_str_op("abc", <, "abcd"); + + + /* Every test-case function needs to finish with an "end:" + label and (optionally) code to clean up local variables. */ + end: + ; +} + +/* ============================================================ */ + +/* Now let's mess with setup and teardown functions! These are handy if + you have a bunch of tests that all need a similar environment, and you + want to reconstruct that environment freshly for each one. */ + +/* First you declare a type to hold the environment info, and functions to + set it up and tear it down. */ +struct data_buffer { + /* We're just going to have couple of character buffer. Using + setup/teardown functions is probably overkill for this case. + + You could also do file descriptors, complicated handles, temporary + files, etc. */ + char buffer1[512]; + char buffer2[512]; +}; +/* The setup function needs to take a const struct testcase_t and return + void* */ +void * +setup_data_buffer(const struct testcase_t *testcase) +{ + struct data_buffer *db = malloc(sizeof(struct data_buffer)); + + /* If you had a complicated set of setup rules, you might behave + differently here depending on testcase->flags or + testcase->setup_data or even or testcase->name. */ + + /* Returning a NULL here would mean that we couldn't set up for this + test, so we don't need to test db for null. */ + return db; +} +/* The clean function deallocates storage carefully and returns true on + success. */ +int +clean_data_buffer(const struct testcase_t *testcase, void *ptr) +{ + struct data_buffer *db = ptr; + + if (db) { + free(db); + return 1; + } + return 0; +} +/* Finally, declare a testcase_setup_t with these functions. */ +struct testcase_setup_t data_buffer_setup = { + setup_data_buffer, clean_data_buffer +}; + + +/* Now let's write our test. */ +void +test_memcpy(void *ptr) +{ + /* This time, we use the argument. */ + struct data_buffer *db = ptr; + + /* We'll also introduce a local variable that might need cleaning up. */ + char *mem = NULL; + + /* Let's make sure that memcpy does what we'd like. */ + strcpy(db->buffer1, "String 0"); + memcpy(db->buffer2, db->buffer1, sizeof(db->buffer1)); + tt_str_op(db->buffer1, ==, db->buffer2); + + /* Now we've allocated memory that's referenced by a local variable. + The end block of the function will clean it up. */ + mem = strdup("Hello world."); + tt_assert(mem); + + /* Another rather trivial test. */ + tt_str_op(db->buffer1, !=, mem); + + end: + /* This time our end block has something to do. */ + if (mem) + free(mem); +} + +/* ============================================================ */ + +/* Now we need to make sure that our tests get invoked. First, you take + a bunch of related tests and put them into an array of struct testcase_t. +*/ + +struct testcase_t demo_tests[] = { + /* Here's a really simple test: it has a name you can refer to it + with, and a function to invoke it. */ + { "strcmp", test_strcmp, }, + + /* The second test has a flag, "TT_FORK", to make it run in a + subprocess, and a pointer to the testcase_setup_t that configures + its environment. */ + { "memcpy", test_memcpy, TT_FORK, &data_buffer_setup }, + + /* The array has to end with END_OF_TESTCASES. */ + END_OF_TESTCASES +}; + +/* Next, we make an array of testgroups. This is mandatory. Unlike more + heavy-duty testing frameworks, groups can't nest. */ +struct testgroup_t groups[] = { + + /* Every group has a 'prefix', and an array of tests. That's it. */ + { "demo/", demo_tests }, + + END_OF_GROUPS +}; + + +int +main(int c, const char **v) +{ + /* Finally, just call tinytest_main(). It lets you specify verbose + or quiet output with --verbose and --quiet. You can list + specific tests: + + tinytest-demo demo/memcpy + + or use a ..-wildcard to select multiple tests with a common + prefix: + + tinytest-demo demo/.. + + If you list no tests, you get them all by default, so that + "tinytest-demo" and "tinytest-demo .." mean the same thing. + + */ + return tinytest_main(c, v, groups); +} diff --git a/src/test/tinytest_macros.h b/src/test/tinytest_macros.h new file mode 100644 index 0000000000..a7fa64a824 --- /dev/null +++ b/src/test/tinytest_macros.h @@ -0,0 +1,167 @@ +/* tinytest_macros.h -- Copyright 2009-2010 Nick Mathewson + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _TINYTEST_MACROS_H +#define _TINYTEST_MACROS_H + +/* Helpers for defining statement-like macros */ +#define TT_STMT_BEGIN do { +#define TT_STMT_END } while (0) + +/* Redefine this if your test functions want to abort with something besides + * "goto end;" */ +#ifndef TT_EXIT_TEST_FUNCTION +#define TT_EXIT_TEST_FUNCTION TT_STMT_BEGIN goto end; TT_STMT_END +#endif + +/* Redefine this if you want to note success/failure in some different way. */ +#ifndef TT_DECLARE +#define TT_DECLARE(prefix, args) \ + TT_STMT_BEGIN \ + printf("\n %s %s:%d: ",prefix,__FILE__,__LINE__); \ + printf args ; \ + TT_STMT_END +#endif + +/* Announce a failure. Args are parenthesized printf args. */ +#define TT_GRIPE(args) TT_DECLARE("FAIL", args) + +/* Announce a non-failure if we're verbose. */ +#define TT_BLATHER(args) \ + TT_STMT_BEGIN \ + if (_tinytest_get_verbosity()>1) TT_DECLARE(" OK", args); \ + TT_STMT_END + +#define TT_DIE(args) \ + TT_STMT_BEGIN \ + _tinytest_set_test_failed(); \ + TT_GRIPE(args); \ + TT_EXIT_TEST_FUNCTION; \ + TT_STMT_END + +#define TT_FAIL(args) \ + TT_STMT_BEGIN \ + _tinytest_set_test_failed(); \ + TT_GRIPE(args); \ + TT_STMT_END + +/* Fail and abort the current test for the reason in msg */ +#define tt_abort_printf(msg) TT_DIE(msg) +#define tt_abort_perror(op) TT_DIE(("%s: %s [%d]",(op),strerror(errno), errno)) +#define tt_abort_msg(msg) TT_DIE(("%s", msg)) +#define tt_abort() TT_DIE(("%s", "(Failed.)")) + +/* Fail but do not abort the current test for the reason in msg. */ +#define tt_fail_printf(msg) TT_FAIL(msg) +#define tt_fail_perror(op) TT_FAIL(("%s: %s [%d]",(op),strerror(errno), errno)) +#define tt_fail_msg(msg) TT_FAIL(("%s", msg)) +#define tt_fail() TT_FAIL(("%s", "(Failed.)")) + +/* End the current test, and indicate we are skipping it. */ +#define tt_skip() \ + TT_STMT_BEGIN \ + _tinytest_set_test_skipped(); \ + TT_EXIT_TEST_FUNCTION; \ + TT_STMT_END + +#define _tt_want(b, msg, fail) \ + TT_STMT_BEGIN \ + if (!(b)) { \ + _tinytest_set_test_failed(); \ + TT_GRIPE((msg)); \ + fail; \ + } else { \ + TT_BLATHER((msg)); \ + } \ + TT_STMT_END + +/* Assert b, but do not stop the test if b fails. Log msg on failure. */ +#define tt_want_msg(b, msg) \ + _tt_want(b, msg, ); + +/* Assert b and stop the test if b fails. Log msg on failure. */ +#define tt_assert_msg(b, msg) \ + _tt_want(b, msg, TT_EXIT_TEST_FUNCTION); + +/* Assert b, but do not stop the test if b fails. */ +#define tt_want(b) tt_want_msg( (b), "want("#b")") +/* Assert b, and stop the test if b fails. */ +#define tt_assert(b) tt_assert_msg((b), "assert("#b")") + +#define tt_assert_test_fmt_type(a,b,str_test,type,test,printf_type,printf_fmt, \ + setup_block,cleanup_block) \ + TT_STMT_BEGIN \ + type _val1 = (type)(a); \ + type _val2 = (type)(b); \ + int _tt_status = (test); \ + if (!_tt_status || _tinytest_get_verbosity()>1) { \ + printf_type _print; \ + printf_type _print1; \ + printf_type _print2; \ + type _value = _val1; \ + setup_block; \ + _print1 = _print; \ + _value = _val2; \ + setup_block; \ + _print2 = _print; \ + TT_DECLARE(_tt_status?" OK":"FAIL", \ + ("assert(%s): "printf_fmt" vs "printf_fmt, \ + str_test, _print1, _print2)); \ + _print = _print1; \ + cleanup_block; \ + _print = _print2; \ + cleanup_block; \ + if (!_tt_status) { \ + _tinytest_set_test_failed(); \ + TT_EXIT_TEST_FUNCTION; \ + } \ + } \ + TT_STMT_END + +#define tt_assert_test_type(a,b,str_test,type,test,fmt) \ + tt_assert_test_fmt_type(a,b,str_test,type,test,type,fmt, \ + {_print=_value;},{}) + +/* Helper: assert that a op b, when cast to type. Format the values with + * printf format fmt on failure. */ +#define tt_assert_op_type(a,op,b,type,fmt) \ + tt_assert_test_type(a,b,#a" "#op" "#b,type,(_val1 op _val2),fmt) + +#define tt_int_op(a,op,b) \ + tt_assert_test_type(a,b,#a" "#op" "#b,long,(_val1 op _val2),"%ld") + +#define tt_uint_op(a,op,b) \ + tt_assert_test_type(a,b,#a" "#op" "#b,unsigned long, \ + (_val1 op _val2),"%lu") + +#define tt_ptr_op(a,op,b) \ + tt_assert_test_type(a,b,#a" "#op" "#b,void*, \ + (_val1 op _val2),"%p") + +#define tt_str_op(a,op,b) \ + tt_assert_test_type(a,b,#a" "#op" "#b,const char *, \ + (strcmp(_val1,_val2) op 0),"<%s>") + +#endif diff --git a/src/tools/Makefile.am b/src/tools/Makefile.am index 41786e4378..1bb5076849 100644 --- a/src/tools/Makefile.am +++ b/src/tools/Makefile.am @@ -3,16 +3,16 @@ noinst_PROGRAMS = tor-checkkey tor_resolve_SOURCES = tor-resolve.c tor_resolve_LDFLAGS = @TOR_LDFLAGS_libevent@ -tor_resolve_LDADD = ../common/libor.a @TOR_LIBEVENT_LIBS@ @TOR_LIB_WS32@ +tor_resolve_LDADD = ../common/libor.a -lm @TOR_LIBEVENT_LIBS@ @TOR_LIB_WS32@ tor_gencert_SOURCES = tor-gencert.c tor_gencert_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ \ @TOR_LDFLAGS_libevent@ tor_gencert_LDADD = ../common/libor.a ../common/libor-crypto.a \ - -lz @TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ + -lm @TOR_ZLIB_LIBS@ @TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ tor_checkkey_SOURCES = tor-checkkey.c tor_checkkey_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ \ @TOR_LDFLAGS_libevent@ tor_checkkey_LDADD = ../common/libor.a ../common/libor-crypto.a \ - -lz @TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ + -lm @TOR_ZLIB_LIBS@ @TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ diff --git a/src/tools/tor-checkkey.c b/src/tools/tor-checkkey.c index b29b52d8db..94c8cbd44c 100644 --- a/src/tools/tor-checkkey.c +++ b/src/tools/tor-checkkey.c @@ -6,19 +6,21 @@ #include <stdio.h> #include <stdlib.h> #include "crypto.h" -#include "log.h" -#include "util.h" +#include "torlog.h" +#include "../common/util.h" #include "compat.h" #include <openssl/bn.h> #include <openssl/rsa.h> -int main(int c, char **v) +int +main(int c, char **v) { crypto_pk_env_t *env; char *str; RSA *rsa; int wantdigest=0; int fname_idx; + char *fname=NULL; init_logging(); if (c < 2) { @@ -29,7 +31,7 @@ int main(int c, char **v) return 1; } - if (crypto_global_init(0)) { + if (crypto_global_init(0, NULL, NULL)) { fprintf(stderr, "Couldn't initialize crypto library.\n"); return 1; } @@ -46,7 +48,9 @@ int main(int c, char **v) fname_idx = 1; } - str = read_file_to_str(v[fname_idx], 0, NULL); + fname = expand_filename(v[fname_idx]); + str = read_file_to_str(fname, 0, NULL); + tor_free(fname); if (!str) { fprintf(stderr, "Couldn't read %s\n", v[fname_idx]); return 1; diff --git a/src/tools/tor-gencert.c b/src/tools/tor-gencert.c index e6b09963bd..a04eddafc7 100644 --- a/src/tools/tor-gencert.c +++ b/src/tools/tor-gencert.c @@ -13,6 +13,7 @@ #include <openssl/evp.h> #include <openssl/pem.h> +#include <openssl/rsa.h> #include <openssl/objects.h> #include <openssl/obj_mac.h> #include <openssl/err.h> @@ -27,8 +28,8 @@ #define CRYPTO_PRIVATE #include "compat.h" -#include "util.h" -#include "log.h" +#include "../common/util.h" +#include "../common/torlog.h" #include "crypto.h" #include "address.h" @@ -63,7 +64,6 @@ show_help(void) "[-c certificate_file]\n" " [-m lifetime_in_months] [-a address:port] " "[--passphrase-fd <fd>]\n"); - } /* XXXX copied from crypto.c */ @@ -218,6 +218,20 @@ parse_commandline(int argc, char **argv) return 0; } +static RSA * +generate_key(int bits) +{ + RSA *rsa = NULL; + crypto_pk_env_t *env = crypto_new_pk_env(); + if (crypto_pk_generate_key_with_bits(env,bits)<0) + goto done; + rsa = _crypto_pk_env_get_rsa(env); + rsa = RSAPrivateKey_dup(rsa); + done: + crypto_free_pk_env(env); + return rsa; +} + /** Try to read the identity key from <b>identity_key_file</b>. If no such * file exists and create_identity_key is set, make a new identity key and * store it. Return 0 on success, nonzero on failure. @@ -238,7 +252,7 @@ load_identity_key(void) } log_notice(LD_GENERAL, "Generating %d-bit RSA identity key.", IDENTITY_KEY_BITS); - if (!(key = RSA_generate_key(IDENTITY_KEY_BITS, 65537, NULL, NULL))) { + if (!(key = generate_key(IDENTITY_KEY_BITS))) { log_err(LD_GENERAL, "Couldn't generate identity key."); crypto_log_errors(LOG_ERR, "Generating identity key"); return 1; @@ -323,7 +337,7 @@ generate_signing_key(void) RSA *key; log_notice(LD_GENERAL, "Generating %d-bit RSA signing key.", SIGNING_KEY_BITS); - if (!(key = RSA_generate_key(SIGNING_KEY_BITS, 65537, NULL, NULL))) { + if (!(key = generate_key(SIGNING_KEY_BITS))) { log_err(LD_GENERAL, "Couldn't generate signing key."); crypto_log_errors(LOG_ERR, "Generating signing key"); return 1; @@ -393,7 +407,6 @@ get_fingerprint(EVP_PKEY *pkey, char *out) return r; } - /** Set <b>out</b> to the hex-encoded fingerprint of <b>pkey</b>. */ static int get_digest(EVP_PKEY *pkey, char *out) @@ -487,7 +500,6 @@ generate_certificate(void) return 0; } - /** Entry point to tor-gencert */ int main(int argc, char **argv) @@ -496,7 +508,7 @@ main(int argc, char **argv) init_logging(); /* Don't bother using acceleration. */ - if (crypto_global_init(0)) { + if (crypto_global_init(0, NULL, NULL)) { fprintf(stderr, "Couldn't initialize crypto library.\n"); return 1; } diff --git a/src/tools/tor-resolve.c b/src/tools/tor-resolve.c index 4d9d57a31c..12349d9d12 100644 --- a/src/tools/tor-resolve.c +++ b/src/tools/tor-resolve.c @@ -6,9 +6,9 @@ #include "orconfig.h" #include "compat.h" -#include "util.h" +#include "../common/util.h" #include "address.h" -#include "log.h" +#include "../common/torlog.h" #include <stdio.h> #include <stdlib.h> @@ -148,7 +148,7 @@ parse_socks4a_resolve_response(const char *hostname, static const char * socks5_reason_to_string(char reason) { - switch(reason) { + switch (reason) { case SOCKS5_SUCCEEDED: return "succeeded"; case SOCKS5_GENERAL_ERROR: @@ -251,7 +251,7 @@ do_resolve(const char *hostname, uint32_t sockshost, uint16_t socksport, } if (parse_socks4a_resolve_response(hostname, reply_buf, RESPONSE_LEN_4, - result_addr)<0){ + result_addr)<0) { return -1; } } else { diff --git a/src/win32/orconfig.h b/src/win32/orconfig.h index 65804fe19d..b3cd1db50b 100644 --- a/src/win32/orconfig.h +++ b/src/win32/orconfig.h @@ -88,11 +88,18 @@ #define HAVE_STRING_H /* Define to 1 if you have the `strlcat' function. */ +#if defined (WINCE) +#define HAVE_STRLCAT +#else #undef HAVE_STRLCAT +#endif /* Define to 1 if you have the `strlcpy' function. */ +#if defined (WINCE) +#define HAVE_STRLCPY +#else #undef HAVE_STRLCPY - +#endif /* Define to 1 if you have the `strptime' function. */ #undef HAVE_STRPTIME @@ -226,6 +233,5 @@ #define USING_TWOS_COMPLEMENT /* Version number of package */ -#define VERSION "0.2.1.25" - +#define VERSION "0.2.2.19-alpha" |