diff options
Diffstat (limited to 'src')
125 files changed, 32489 insertions, 15478 deletions
diff --git a/src/Makefile.am b/src/Makefile.am index ae647b2d61..fa2dd560a6 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -1,5 +1,5 @@ # leave in dependency order, since common must be built first -SUBDIRS = common or tools win32 config -DIST_SUBDIRS = common or tools win32 config +SUBDIRS = common or test tools win32 config +DIST_SUBDIRS = common or test tools win32 config diff --git a/src/common/Makefile.am b/src/common/Makefile.am index 105c413343..b1e03cd710 100644 --- a/src/common/Makefile.am +++ b/src/common/Makefile.am @@ -1,5 +1,7 @@ -noinst_LIBRARIES = libor.a libor-crypto.a +noinst_LIBRARIES = libor.a libor-crypto.a libor-event.a + +EXTRA_DIST = common_sha1.i sha256.c #CFLAGS = -Wall -Wpointer-arith -O2 @@ -10,7 +12,21 @@ libor_extra_source= endif libor_a_SOURCES = address.c log.c util.c compat.c container.c mempool.c \ - memarea.c $(libor_extra_source) + memarea.c util_codedigest.c $(libor_extra_source) libor_crypto_a_SOURCES = crypto.c aes.c tortls.c torgzip.c +libor_event_a_SOURCES = compat_libevent.c + +noinst_HEADERS = address.h torlog.h crypto.h util.h compat.h aes.h torint.h tortls.h strlcpy.c strlcat.c torgzip.h container.h ht.h mempool.h memarea.h ciphers.inc compat_libevent.h tortls_states.h + +common_sha1.i: $(libor_SOURCES) $(libor_crypto_a_SOURCES) $(noinst_HEADERS) + if test "@SHA1SUM@" != none; then \ + @SHA1SUM@ $(libor_SOURCES) $(libor_crypto_a_SOURCES) $(noinst_HEADERS) | @SED@ -n 's/^\(.*\)$$/"\1\\n"/p' > common_sha1.i; \ + elif test "@OPENSSL@" != none; then \ + @OPENSSL@ sha1 $(libor_SOURCES) $(libor_crypto_a_SOURCES) $(noinst_HEADERS) | @SED@ -n 's/SHA1(\(.*\))= \(.*\)/"\2 \1\\n"/p' > common_sha1.i; \ + else \ + rm common_sha1.i; \ + touch common_sha1.i; \ + fi -noinst_HEADERS = address.h log.h crypto.h test.h util.h compat.h aes.h torint.h tortls.h strlcpy.c strlcat.c torgzip.h container.h ht.h mempool.h memarea.h ciphers.inc +util_codedigest.o: common_sha1.i +crypto.o: sha256.c diff --git a/src/common/address.c b/src/common/address.c index 82a709601f..aff517ca51 100644 --- a/src/common/address.c +++ b/src/common/address.c @@ -12,7 +12,7 @@ #include "compat.h" #include "util.h" #include "address.h" -#include "log.h" +#include "torlog.h" #ifdef MS_WINDOWS #include <process.h> @@ -50,11 +50,13 @@ #include <assert.h> /** Convert the tor_addr_t in <b>a</b>, with port in <b>port</b>, into a - * socklen object in *<b>sa_out</b> of object size <b>len</b>. If not enough - * room is free, or on error, return -1. Else return the length of the - * sockaddr. */ -/* XXXX021 This returns socklen_t. socklen_t is sometimes unsigned. This - * function claims to return -1 sometimes. Problematic! */ + * sockaddr object in *<b>sa_out</b> of object size <b>len</b>. If not enough + * room is available in sa_out, or on error, return 0. On success, return + * the length of the sockaddr. + * + * Interface note: ordinarily, we return -1 for error. We can't do that here, + * since socklen_t is unsigned on some platforms. + **/ socklen_t tor_addr_to_sockaddr(const tor_addr_t *a, uint16_t port, @@ -65,7 +67,7 @@ tor_addr_to_sockaddr(const tor_addr_t *a, if (family == AF_INET) { struct sockaddr_in *sin; if (len < (int)sizeof(struct sockaddr_in)) - return -1; + return 0; sin = (struct sockaddr_in *)sa_out; memset(sin, 0, sizeof(struct sockaddr_in)); #ifdef HAVE_STRUCT_SOCKADDR_IN_SIN_LEN @@ -78,7 +80,7 @@ tor_addr_to_sockaddr(const tor_addr_t *a, } else if (family == AF_INET6) { struct sockaddr_in6 *sin6; if (len < (int)sizeof(struct sockaddr_in6)) - return -1; + return 0; sin6 = (struct sockaddr_in6 *)sa_out; memset(sin6, 0, sizeof(struct sockaddr_in6)); #ifdef HAVE_STRUCT_SOCKADDR_IN6_SIN6_LEN @@ -89,7 +91,7 @@ tor_addr_to_sockaddr(const tor_addr_t *a, memcpy(&sin6->sin6_addr, tor_addr_to_in6(a), sizeof(struct in6_addr)); return sizeof(struct sockaddr_in6); } else { - return -1; + return 0; } } @@ -508,7 +510,8 @@ tor_addr_parse_mask_ports(const char *s, tor_addr_t *addr_out, tor_assert(s); tor_assert(addr_out); - /* IP, [], /mask, ports */ + /** Longest possible length for an address, mask, and port-range combination. + * Includes IP, [], /mask, :, ports */ #define MAX_ADDRESS_LENGTH (TOR_ADDR_BUF_LEN+2+(1+INET_NTOA_BUF_LEN)+12+1) if (strlen(s) > MAX_ADDRESS_LENGTH) { @@ -601,7 +604,7 @@ tor_addr_parse_mask_ports(const char *s, tor_addr_t *addr_out, if (family == AF_INET6 && v4map) { if (bits > 32 && bits < 96) { /* Crazy */ log_warn(LD_GENERAL, - "Bad mask bits %i for V4-mapped V6 address; rejecting.", + "Bad mask bits %d for V4-mapped V6 address; rejecting.", bits); goto err; } @@ -726,7 +729,7 @@ tor_addr_is_loopback(const tor_addr_t *addr) } /** Set <b>dest</b> to equal the IPv4 address in <b>v4addr</b> (given in - * network order. */ + * network order). */ void tor_addr_from_ipv4n(tor_addr_t *dest, uint32_t v4addr) { @@ -760,6 +763,8 @@ tor_addr_from_in6(tor_addr_t *dest, const struct in6_addr *in6) void tor_addr_copy(tor_addr_t *dest, const tor_addr_t *src) { + if (src == dest) + return; tor_assert(src); tor_assert(dest); memcpy(dest, src, sizeof(tor_addr_t)); @@ -786,7 +791,7 @@ tor_addr_compare(const tor_addr_t *addr1, const tor_addr_t *addr2, * Reduce over-specific masks (>128 for ipv6, >32 for ipv4) to 128 or 32. * * The mask is interpreted relative to <b>addr1</b>, so that if a is - * ::ffff:1.2.3.4, and b is 3.4.5.6, + * \::ffff:1.2.3.4, and b is 3.4.5.6, * tor_addr_compare_masked(a,b,100,CMP_SEMANTIC) is the same as * -tor_addr_compare_masked(b,a,4,CMP_SEMANTIC). * @@ -798,6 +803,8 @@ int tor_addr_compare_masked(const tor_addr_t *addr1, const tor_addr_t *addr2, maskbits_t mbits, tor_addr_comparison_t how) { + /** Helper: Evaluates to -1 if a is less than b, 0 if a equals b, or 1 if a + * is greater than b. May evaluate a and b more than once. */ #define TRISTATE(a,b) (((a)<(b))?-1: (((a)==(b))?0:1)) sa_family_t family1, family2, v_family1, v_family2; @@ -912,13 +919,6 @@ tor_dup_addr(const tor_addr_t *addr) return tor_strdup(buf); } -/** Copy the address in <b>src</b> to <b>dest</b> */ -void -tor_addr_assign(tor_addr_t *dest, const tor_addr_t *src) -{ - memcpy(dest, src, sizeof(tor_addr_t)); -} - /** Return a string representing the address <b>addr</b>. This string is * statically allocated, and must not be freed. Each call to * <b>fmt_addr</b> invalidates the last result of the function. This @@ -1030,27 +1030,28 @@ get_interface_address6(int severity, sa_family_t family, tor_addr_t *addr) { int sock=-1, r=-1; struct sockaddr_storage my_addr, target_addr; - socklen_t my_addr_len; + socklen_t addr_len; tor_assert(addr); memset(addr, 0, sizeof(tor_addr_t)); memset(&target_addr, 0, sizeof(target_addr)); - my_addr_len = (socklen_t)sizeof(my_addr); - /* Use the "discard" service port */ - ((struct sockaddr_in*)&target_addr)->sin_port = 9; /* Don't worry: no packets are sent. We just need to use a real address * on the actual Internet. */ if (family == AF_INET6) { struct sockaddr_in6 *sin6 = (struct sockaddr_in6*)&target_addr; + /* Use the "discard" service port */ + sin6->sin6_port = htons(9); sock = tor_open_socket(PF_INET6,SOCK_DGRAM,IPPROTO_UDP); - my_addr_len = (socklen_t)sizeof(struct sockaddr_in6); + addr_len = (socklen_t)sizeof(struct sockaddr_in6); sin6->sin6_family = AF_INET6; S6_ADDR16(sin6->sin6_addr)[0] = htons(0x2002); /* 2002:: */ } else if (family == AF_INET) { struct sockaddr_in *sin = (struct sockaddr_in*)&target_addr; + /* Use the "discard" service port */ + sin->sin_port = htons(9); sock = tor_open_socket(PF_INET,SOCK_DGRAM,IPPROTO_UDP); - my_addr_len = (socklen_t)sizeof(struct sockaddr_in); + addr_len = (socklen_t)sizeof(struct sockaddr_in); sin->sin_family = AF_INET; sin->sin_addr.s_addr = htonl(0x12000001); /* 18.0.0.1 */ } else { @@ -1063,14 +1064,13 @@ get_interface_address6(int severity, sa_family_t family, tor_addr_t *addr) goto err; } - if (connect(sock,(struct sockaddr *)&target_addr, - (socklen_t)sizeof(target_addr))<0) { + if (connect(sock,(struct sockaddr *)&target_addr, addr_len) < 0) { int e = tor_socket_errno(sock); log_fn(severity, LD_NET, "connect() failed: %s", tor_socket_strerror(e)); goto err; } - if (getsockname(sock,(struct sockaddr*)&my_addr, &my_addr_len)) { + if (getsockname(sock,(struct sockaddr*)&my_addr, &addr_len)) { int e = tor_socket_errno(sock); log_fn(severity, LD_NET, "getsockname() to determine interface failed: %s", tor_socket_strerror(e)); @@ -1087,7 +1087,7 @@ get_interface_address6(int severity, sa_family_t family, tor_addr_t *addr) /* ====== * IPv4 helpers - * XXXX022 IPv6 deprecate some of these. + * XXXX023 IPv6 deprecate some of these. */ /** Return true iff <b>ip</b> (in host order) is an IP reserved to localhost, diff --git a/src/common/address.h b/src/common/address.h index cb2438791f..d05a3de2e7 100644 --- a/src/common/address.h +++ b/src/common/address.h @@ -53,8 +53,20 @@ tor_addr_to_in6(const tor_addr_t *a) return a->family == AF_INET6 ? &a->addr.in6_addr : NULL; } +/** Given an IPv6 address <b>x</b>, yield it as an array of uint8_t. + * + * Requires that <b>x</b> is actually an IPv6 address. + */ #define tor_addr_to_in6_addr8(x) tor_addr_to_in6(x)->s6_addr +/** Given an IPv6 address <b>x</b>, yield it as an array of uint16_t. + * + * Requires that <b>x</b> is actually an IPv6 address. + */ #define tor_addr_to_in6_addr16(x) S6_ADDR16(*tor_addr_to_in6(x)) +/** Given an IPv6 address <b>x</b>, yield it as an array of uint32_t. + * + * Requires that <b>x</b> is actually an IPv6 address. + */ #define tor_addr_to_in6_addr32(x) S6_ADDR32(*tor_addr_to_in6(x)) /** Return an IPv4 address in network order for <b>a</b>, or 0 if @@ -71,7 +83,7 @@ tor_addr_to_ipv4h(const tor_addr_t *a) { return ntohl(tor_addr_to_ipv4n(a)); } -/* Given an IPv6 address, return its mapped IPv4 address in host order, or +/** Given an IPv6 address, return its mapped IPv4 address in host order, or * 0 if <b>a</b> is not an IPv6 address. * * (Does not check whether the address is really a mapped address */ @@ -102,12 +114,17 @@ tor_addr_eq_ipv4h(const tor_addr_t *a, uint32_t u) return a->family == AF_INET ? (tor_addr_to_ipv4h(a) == u) : 0; } -#define TOR_ADDR_BUF_LEN 48 /* [ffff:ffff:ffff:ffff:ffff:ffff:255.255.255.255] - */ +/** Length of a buffer that you need to allocate to be sure you can encode + * any tor_addr_t. + * + * This allows enough space for + * "[ffff:ffff:ffff:ffff:ffff:ffff:255.255.255.255]", + * plus a terminating NUL. + */ +#define TOR_ADDR_BUF_LEN 48 int tor_addr_lookup(const char *name, uint16_t family, tor_addr_t *addr_out); char *tor_dup_addr(const tor_addr_t *addr) ATTR_MALLOC; -void tor_addr_assign(tor_addr_t *dest, const tor_addr_t *src); const char *fmt_addr(const tor_addr_t *addr); int get_interface_address6(int severity, sa_family_t family, tor_addr_t *addr); @@ -155,6 +172,7 @@ void tor_addr_from_ipv4n(tor_addr_t *dest, uint32_t v4addr); #define tor_addr_from_ipv4h(dest, v4addr) \ tor_addr_from_ipv4n((dest), htonl(v4addr)) void tor_addr_from_ipv6_bytes(tor_addr_t *dest, const char *bytes); +/** Set <b>dest</b> to the IPv4 address incoded in <b>in</b>. */ #define tor_addr_from_in(dest, in) \ tor_addr_from_ipv4n((dest), (in)->s_addr); void tor_addr_from_in6(tor_addr_t *dest, const struct in6_addr *in6); diff --git a/src/common/aes.c b/src/common/aes.c index 7e332b0502..c2fdeb594a 100644 --- a/src/common/aes.c +++ b/src/common/aes.c @@ -21,7 +21,7 @@ #include "compat.h" #include "aes.h" #include "util.h" -#include "log.h" +#include "torlog.h" /* We have 3 strategies for getting AES: Via OpenSSL's AES_encrypt function, * via OpenSSL's EVP_EncryptUpdate function, or via the built-in AES @@ -100,6 +100,9 @@ /* Figure out which AES optimizations to use. */ #ifdef USE_BUILTIN_AES +/** If this is defined, we take advantage of the fact that AES treats its + * input as a set of 4 32-bit words, so that there is no need to encode and + * decode the 128-bit counter before every block encryption */ # define USE_RIJNDAEL_COUNTER_OPTIMIZATION # if 0 && (defined(__powerpc__) || defined(__powerpc64__)) /* XXXX do more experimentation before concluding this is actually @@ -263,7 +266,8 @@ aes_set_key(aes_cnt_cipher_t *cipher, const char *key, int key_bits) void aes_free_cipher(aes_cnt_cipher_t *cipher) { - tor_assert(cipher); + if (!cipher) + return; #ifdef USE_OPENSSL_EVP EVP_CIPHER_CTX_cleanup(&cipher->key); #endif diff --git a/src/common/compat.c b/src/common/compat.c index 333ef5b0f0..3644bd9996 100644 --- a/src/common/compat.c +++ b/src/common/compat.c @@ -15,6 +15,8 @@ /* This is required on rh7 to make strptime not complain. * We also need it to make memmem get defined (where available) */ +/* XXXX023 We should just use AC_USE_SYSTEM_EXTENSIONS in our autoconf, + * and get this (and other important stuff!) automatically */ #define _GNU_SOURCE #include "compat.h" @@ -88,7 +90,7 @@ #include <sys/prctl.h> #endif -#include "log.h" +#include "torlog.h" #include "util.h" #include "container.h" #include "address.h" @@ -170,12 +172,17 @@ tor_munmap_file(tor_mmap_t *handle) tor_mmap_t * tor_mmap_file(const char *filename) { + TCHAR tfilename[MAX_PATH]= {0}; tor_mmap_t *res = tor_malloc_zero(sizeof(tor_mmap_t)); int empty = 0; res->file_handle = INVALID_HANDLE_VALUE; res->mmap_handle = NULL; - - res->file_handle = CreateFile(filename, +#ifdef UNICODE + mbstowcs(tfilename,filename,MAX_PATH); +#else + strlcpy(tfilename,filename,MAX_PATH); +#endif + res->file_handle = CreateFile(tfilename, GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, @@ -308,6 +315,100 @@ tor_vsnprintf(char *str, size_t size, const char *format, va_list args) return r; } +/** + * Portable asprintf implementation. Does a printf() into a newly malloc'd + * string. Sets *<b>strp</b> to this string, and returns its length (not + * including the terminating NUL character). + * + * You can treat this function as if its implementation were something like + <pre> + char buf[_INFINITY_]; + tor_snprintf(buf, sizeof(buf), fmt, args); + *strp = tor_strdup(buf); + return strlen(*strp): + </pre> + * Where _INFINITY_ is an imaginary constant so big that any string can fit + * into it. + */ +int +tor_asprintf(char **strp, const char *fmt, ...) +{ + int r; + va_list args; + va_start(args, fmt); + r = tor_vasprintf(strp, fmt, args); + va_end(args); + if (!*strp || r < 0) { + log_err(LD_BUG, "Internal error in asprintf"); + tor_assert(0); + } + return r; +} + +/** + * Portable vasprintf implementation. Does a printf() into a newly malloc'd + * string. Differs from regular vasprintf in the same ways that + * tor_asprintf() differs from regular asprintf. + */ +int +tor_vasprintf(char **strp, const char *fmt, va_list args) +{ + /* use a temporary variable in case *strp is in args. */ + char *strp_tmp=NULL; +#ifdef HAVE_VASPRINTF + /* If the platform gives us one, use it. */ + int r = vasprintf(&strp_tmp, fmt, args); + if (r < 0) + *strp = NULL; + else + *strp = strp_tmp; + return r; +#elif defined(_MSC_VER) + /* On Windows, _vsnprintf won't tell us the length of the string if it + * overflows, so we need to use _vcsprintf to tell how much to allocate */ + int len, r; + char *res; + len = _vscprintf(fmt, args); + if (len < 0) { + *strp = NULL; + return -1; + } + strp_tmp = tor_malloc(len + 1); + r = _vsnprintf(strp_tmp, len+1, fmt, args); + if (r != len) { + tor_free(strp_tmp); + *strp = NULL; + return -1; + } + *strp = strp_tmp; + return len; +#else + /* Everywhere else, we have a decent vsnprintf that tells us how many + * characters we need. We give it a try on a short buffer first, since + * it might be nice to avoid the second vsnprintf call. + */ + char buf[128]; + int len, r; + va_list tmp_args; + va_copy(tmp_args, args); + len = vsnprintf(buf, sizeof(buf), fmt, tmp_args); + va_end(tmp_args); + if (len < (int)sizeof(buf)) { + *strp = tor_strdup(buf); + return len; + } + strp_tmp = tor_malloc(len+1); + r = vsnprintf(strp_tmp, len+1, fmt, args); + if (r != len) { + tor_free(strp_tmp); + *strp = NULL; + return -1; + } + *strp = strp_tmp; + return len; +#endif +} + /** Given <b>hlen</b> bytes at <b>haystack</b> and <b>nlen</b> bytes at * <b>needle</b>, return a pointer to the first occurrence of the needle * within the haystack, or NULL if there is no such occurrence. @@ -399,6 +500,37 @@ const char TOR_TOLOWER_TABLE[256] = { 240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255, }; +/** Implementation of strtok_r for platforms whose coders haven't figured out + * how to write one. Hey guys! You can use this code here for free! */ +char * +tor_strtok_r_impl(char *str, const char *sep, char **lasts) +{ + char *cp, *start; + if (str) + start = cp = *lasts = str; + else if (!*lasts) + return NULL; + else + start = cp = *lasts; + + tor_assert(*sep); + if (sep[1]) { + while (*cp && !strchr(sep, *cp)) + ++cp; + } else { + tor_assert(strlen(sep) == 1); + cp = strchr(cp, *sep); + } + + if (!cp || !*cp) { + *lasts = NULL; + } else { + *cp++ = '\0'; + *lasts = cp; + } + return start; +} + #ifdef MS_WINDOWS /** Take a filename and return a pointer to its final element. This * function is called on __FILE__ to fix a MSVC nit where __FILE__ @@ -450,8 +582,8 @@ get_uint32(const void *cp) return v; } /** - * Read a 32-bit value beginning at <b>cp</b>. Equivalent to - * *(uint32_t*)(cp), but will not cause segfaults on platforms that forbid + * Read a 64-bit value beginning at <b>cp</b>. Equivalent to + * *(uint64_t*)(cp), but will not cause segfaults on platforms that forbid * unaligned memory access. */ uint64_t @@ -530,7 +662,9 @@ touch_file(const char *fname) /** Represents a lockfile on which we hold the lock. */ struct tor_lockfile_t { + /** Name of the file */ char *filename; + /** File descriptor used to hold the file open */ int fd; }; @@ -546,7 +680,10 @@ struct tor_lockfile_t { * * (Implementation note: because we need to fall back to fcntl on some * platforms, these locks are per-process, not per-thread. If you want - * to do in-process locking, use tor_mutex_t like a normal person.) + * to do in-process locking, use tor_mutex_t like a normal person. + * On Windows, when <b>blocking</b> is true, the maximum time that + * is actually waited is 10 seconds, after which NULL is returned + * and <b>locked_out</b> is set to 1.) */ tor_lockfile_t * tor_lockfile_lock(const char *filename, int blocking, int *locked_out) @@ -566,7 +703,7 @@ tor_lockfile_lock(const char *filename, int blocking, int *locked_out) #ifdef WIN32 _lseek(fd, 0, SEEK_SET); if (_locking(fd, blocking ? _LK_LOCK : _LK_NBLCK, 1) < 0) { - if (errno != EDEADLOCK) + if (errno != EACCES && errno != EDEADLOCK) log_warn(LD_FS,"Couldn't lock \"%s\": %s", filename, strerror(errno)); else *locked_out = 1; @@ -633,7 +770,8 @@ tor_lockfile_unlock(tor_lockfile_t *lockfile) tor_free(lockfile); } -/* Some old versions of Unix didn't define constants for these values, +/** @{ */ +/** Some old versions of Unix didn't define constants for these values, * and instead expect you to say 0, 1, or 2. */ #ifndef SEEK_CUR #define SEEK_CUR 1 @@ -641,6 +779,7 @@ tor_lockfile_unlock(tor_lockfile_t *lockfile) #ifndef SEEK_END #define SEEK_END 2 #endif +/** @} */ /** Return the position of <b>fd</b> with respect to the start of the file. */ off_t @@ -680,6 +819,7 @@ static int n_sockets_open = 0; /** Mutex to protect open_sockets, max_socket, and n_sockets_open. */ static tor_mutex_t *socket_accounting_mutex = NULL; +/** Helper: acquire the socket accounting lock. */ static INLINE void socket_accounting_lock(void) { @@ -688,6 +828,7 @@ socket_accounting_lock(void) tor_mutex_acquire(socket_accounting_mutex); } +/** Helper: release the socket accounting lock. */ static INLINE void socket_accounting_unlock(void) { @@ -746,6 +887,7 @@ tor_close_socket(int s) return r; } +/** @{ */ #ifdef DEBUG_SOCKET_COUNTING /** Helper: if DEBUG_SOCKET_COUNTING is enabled, remember that <b>s</b> is * now an open socket. */ @@ -770,6 +912,7 @@ mark_socket_open(int s) #else #define mark_socket_open(s) STMT_NIL #endif +/** @} */ /** As socket(), but counts the number of open sockets. */ int @@ -957,6 +1100,8 @@ tor_socketpair(int family, int type, int protocol, int fd[2]) #endif } +/** Number of extra file descriptors to keep in reserve beyond those that we + * tell Tor it's allowed to use. */ #define ULIMIT_BUFFER 32 /* keep 32 extra fd's beyond _ConnLimit */ /** Learn the maximum allowed number of file descriptors. (Some systems @@ -979,9 +1124,6 @@ set_max_file_descriptors(rlim_t limit, int *max_out) #if defined(CYGWIN) || defined(__CYGWIN__) const char *platform = "Cygwin"; const unsigned long MAX_CONNECTIONS = 3200; -#elif defined(IPHONE) - const char *platform = "iPhone"; - const unsigned long MAX_CONNECTIONS = 9999; #elif defined(MS_WINDOWS) const char *platform = "Windows"; const unsigned long MAX_CONNECTIONS = 15000; @@ -1072,6 +1214,7 @@ set_max_file_descriptors(rlim_t limit, int *max_out) static int log_credential_status(void) { +/** Log level to use when describing non-error UID/GID status. */ #define CREDENTIAL_LOG_LEVEL LOG_INFO /* Real, effective and saved UIDs */ uid_t ruid, euid, suid; @@ -1553,6 +1696,30 @@ tor_lookup_hostname(const char *name, uint32_t *addr) return -1; } +/** Initialize the insecure libc RNG. */ +void +tor_init_weak_random(unsigned seed) +{ +#ifdef MS_WINDOWS + srand(seed); +#else + srandom(seed); +#endif +} + +/** Return a randomly chosen value in the range 0..TOR_RAND_MAX. This + * entropy will not be cryptographically strong; do not rely on it + * for anything an adversary should not be able to predict. */ +long +tor_weak_random(void) +{ +#ifdef MS_WINDOWS + return rand(); +#else + return random(); +#endif +} + /** Hold the result of our call to <b>uname</b>. */ static char uname_result[256]; /** True iff uname_result is set. */ @@ -1578,13 +1745,14 @@ get_uname(void) #ifdef MS_WINDOWS OSVERSIONINFOEX info; int i; - unsigned int leftover_mask; const char *plat = NULL; const char *extra = NULL; + char acsd[MAX_PATH] = {0}; static struct { unsigned major; unsigned minor; const char *version; } win_version_table[] = { - { 6, 0, "Windows \"Longhorn\"" }, + { 6, 1, "Windows 7" }, + { 6, 0, "Windows Vista" }, { 5, 2, "Windows Server 2003" }, { 5, 1, "Windows XP" }, { 5, 0, "Windows 2000" }, @@ -1595,25 +1763,6 @@ get_uname(void) { 3, 51, "Windows NT 3.51" }, { 0, 0, NULL } }; -#ifdef VER_SUITE_BACKOFFICE - static struct { - unsigned int mask; const char *str; - } win_mask_table[] = { - { VER_SUITE_BACKOFFICE, " {backoffice}" }, - { VER_SUITE_BLADE, " {\"blade\" (2003, web edition)}" }, - { VER_SUITE_DATACENTER, " {datacenter}" }, - { VER_SUITE_ENTERPRISE, " {enterprise}" }, - { VER_SUITE_EMBEDDEDNT, " {embedded}" }, - { VER_SUITE_PERSONAL, " {personal}" }, - { VER_SUITE_SINGLEUSERTS, - " {terminal services, single user}" }, - { VER_SUITE_SMALLBUSINESS, " {small business}" }, - { VER_SUITE_SMALLBUSINESS_RESTRICTED, - " {small business, restricted}" }, - { VER_SUITE_TERMINAL, " {terminal services}" }, - { 0, NULL }, - }; -#endif memset(&info, 0, sizeof(info)); info.dwOSVersionInfoSize = sizeof(info); if (! GetVersionEx((LPOSVERSIONINFO)&info)) { @@ -1622,14 +1771,19 @@ get_uname(void) uname_result_is_set = 1; return uname_result; } +#ifdef UNICODE + wcstombs(acsd, info.szCSDVersion, MAX_PATH); +#else + strlcpy(acsd, info.szCSDVersion, sizeof(acsd)); +#endif if (info.dwMajorVersion == 4 && info.dwMinorVersion == 0) { if (info.dwPlatformId == VER_PLATFORM_WIN32_NT) plat = "Windows NT 4.0"; else plat = "Windows 95"; - if (info.szCSDVersion[1] == 'B') + if (acsd[1] == 'B') extra = "OSR2 (B)"; - else if (info.szCSDVersion[1] == 'C') + else if (acsd[1] == 'C') extra = "OSR2 (C)"; } else { for (i=0; win_version_table[i].major>0; ++i) { @@ -1641,29 +1795,30 @@ get_uname(void) } } if (plat && !strcmp(plat, "Windows 98")) { - if (info.szCSDVersion[1] == 'A') + if (acsd[1] == 'A') extra = "SE (A)"; - else if (info.szCSDVersion[1] == 'B') + else if (acsd[1] == 'B') extra = "SE (B)"; } if (plat) { if (!extra) - extra = info.szCSDVersion; + extra = acsd; tor_snprintf(uname_result, sizeof(uname_result), "%s %s", plat, extra); } else { if (info.dwMajorVersion > 6 || - (info.dwMajorVersion==6 && info.dwMinorVersion>0)) + (info.dwMajorVersion==6 && info.dwMinorVersion>1)) tor_snprintf(uname_result, sizeof(uname_result), "Very recent version of Windows [major=%d,minor=%d] %s", (int)info.dwMajorVersion,(int)info.dwMinorVersion, - info.szCSDVersion); + acsd); else tor_snprintf(uname_result, sizeof(uname_result), "Unrecognized version of Windows [major=%d,minor=%d] %s", (int)info.dwMajorVersion,(int)info.dwMinorVersion, - info.szCSDVersion); + acsd); } +#if !defined (WINCE) #ifdef VER_SUITE_BACKOFFICE if (info.wProductType == VER_NT_DOMAIN_CONTROLLER) { strlcat(uname_result, " [domain controller]", sizeof(uname_result)); @@ -1672,18 +1827,7 @@ get_uname(void) } else if (info.wProductType == VER_NT_WORKSTATION) { strlcat(uname_result, " [workstation]", sizeof(uname_result)); } - leftover_mask = info.wSuiteMask; - for (i = 0; win_mask_table[i].mask; ++i) { - if (info.wSuiteMask & win_mask_table[i].mask) { - strlcat(uname_result, win_mask_table[i].str, sizeof(uname_result)); - leftover_mask &= ~win_mask_table[i].mask; - } - } - if (leftover_mask) { - size_t len = strlen(uname_result); - tor_snprintf(uname_result+len, sizeof(uname_result)-len, - " {0x%x}", info.wSuiteMask); - } +#endif #endif #else strlcpy(uname_result, "Unknown platform", sizeof(uname_result)); @@ -1793,7 +1937,6 @@ spawn_exit(void) * call _exit, not exit, from child processes. */ _exit(0); #endif - } /** Set *timeval to the current time of day. On error, log and terminate. @@ -1813,8 +1956,15 @@ tor_gettimeofday(struct timeval *timeval) uint64_t ft_64; FILETIME ft_ft; } ft; +#if defined (WINCE) + /* wince do not have GetSystemTimeAsFileTime */ + SYSTEMTIME stime; + GetSystemTime(&stime); + SystemTimeToFileTime(&stime,&ft.ft_ft); +#else /* number of 100-nsec units since Jan 1, 1601 */ GetSystemTimeAsFileTime(&ft.ft_ft); +#endif if (ft.ft_64 < EPOCH_BIAS) { log_err(LD_GENERAL,"System time is before 1970; failing."); exit(1); @@ -1846,8 +1996,88 @@ tor_gettimeofday(struct timeval *timeval) #define TIME_FNS_NEED_LOCKS #endif -#ifndef HAVE_LOCALTIME_R -#ifdef TIME_FNS_NEED_LOCKS +static struct tm * +correct_tm(int islocal, const time_t *timep, struct tm *resultbuf, + struct tm *r) +{ + const char *outcome; + + if (PREDICT_LIKELY(r)) { + if (r->tm_year > 8099) { /* We can't strftime dates after 9999 CE. */ + r->tm_year = 8099; + r->tm_mon = 11; + r->tm_mday = 31; + r->tm_yday = 365; + r->tm_hour = 23; + r->tm_min = 59; + r->tm_sec = 59; + } + return r; + } + + /* If we get here, gmtime or localtime returned NULL. It might have done + * this because of overrun or underrun, or it might have done it because of + * some other weird issue. */ + if (timep) { + if (*timep < 0) { + r = resultbuf; + r->tm_year = 70; /* 1970 CE */ + r->tm_mon = 0; + r->tm_mday = 1; + r->tm_yday = 1; + r->tm_hour = 0; + r->tm_min = 0 ; + r->tm_sec = 0; + outcome = "Rounding up to 1970"; + goto done; + } else if (*timep >= INT32_MAX) { + /* Rounding down to INT32_MAX isn't so great, but keep in mind that we + * only do it if gmtime/localtime tells us NULL. */ + r = resultbuf; + r->tm_year = 137; /* 2037 CE */ + r->tm_mon = 11; + r->tm_mday = 31; + r->tm_yday = 365; + r->tm_hour = 23; + r->tm_min = 59; + r->tm_sec = 59; + outcome = "Rounding down to 2037"; + goto done; + } + } + + /* If we get here, then gmtime/localtime failed without getting an extreme + * value for *timep */ + + tor_fragile_assert(); + r = resultbuf; + memset(resultbuf, 0, sizeof(struct tm)); + outcome="can't recover"; + done: + log_warn(LD_BUG, "%s("I64_FORMAT") failed with error %s: %s", + islocal?"localtime":"gmtime", + timep?I64_PRINTF_ARG(*timep):0, + strerror(errno), + outcome); + return r; +} + + +/** @{ */ +/** As localtime_r, but defined for platforms that don't have it: + * + * Convert *<b>timep</b> to a struct tm in local time, and store the value in + * *<b>result</b>. Return the result on success, or NULL on failure. + */ +#ifdef HAVE_LOCALTIME_R +struct tm * +tor_localtime_r(const time_t *timep, struct tm *result) +{ + struct tm *r; + r = localtime_r(timep, result); + return correct_tm(1, timep, result, r); +} +#elif defined(TIME_FNS_NEED_LOCKS) struct tm * tor_localtime_r(const time_t *timep, struct tm *result) { @@ -1857,9 +2087,10 @@ tor_localtime_r(const time_t *timep, struct tm *result) tor_assert(result); tor_mutex_acquire(m); r = localtime(timep); - memcpy(result, r, sizeof(struct tm)); + if (r) + memcpy(result, r, sizeof(struct tm)); tor_mutex_release(m); - return result; + return correct_tm(1, timep, result, r); } #else struct tm * @@ -1868,14 +2099,28 @@ tor_localtime_r(const time_t *timep, struct tm *result) struct tm *r; tor_assert(result); r = localtime(timep); - memcpy(result, r, sizeof(struct tm)); - return result; + if (r) + memcpy(result, r, sizeof(struct tm)); + return correct_tm(1, timep, result, r); } #endif -#endif +/** @} */ -#ifndef HAVE_GMTIME_R -#ifdef TIME_FNS_NEED_LOCKS +/** @{ */ +/** As gmtimee_r, but defined for platforms that don't have it: + * + * Convert *<b>timep</b> to a struct tm in UTC, and store the value in + * *<b>result</b>. Return the result on success, or NULL on failure. + */ +#ifdef HAVE_GMTIME_R +struct tm * +tor_gmtime_r(const time_t *timep, struct tm *result) +{ + struct tm *r; + r = gmtime_r(timep, result); + return correct_tm(0, timep, result, r); +} +#elif defined(TIME_FNS_NEED_LOCKS) struct tm * tor_gmtime_r(const time_t *timep, struct tm *result) { @@ -1885,9 +2130,10 @@ tor_gmtime_r(const time_t *timep, struct tm *result) tor_assert(result); tor_mutex_acquire(m); r = gmtime(timep); - memcpy(result, r, sizeof(struct tm)); + if (r) + memcpy(result, r, sizeof(struct tm)); tor_mutex_release(m); - return result; + return correct_tm(0, timep, result, r); } #else struct tm * @@ -1896,11 +2142,11 @@ tor_gmtime_r(const time_t *timep, struct tm *result) struct tm *r; tor_assert(result); r = gmtime(timep); - memcpy(result, r, sizeof(struct tm)); - return result; + if (r) + memcpy(result, r, sizeof(struct tm)); + return correct_tm(0, timep, result, r); } #endif -#endif #if defined(USE_WIN32_THREADS) void @@ -2014,6 +2260,8 @@ tor_mutex_new(void) void tor_mutex_free(tor_mutex_t *m) { + if (!m) + return; tor_mutex_uninit(m); tor_free(m); } @@ -2041,7 +2289,8 @@ tor_cond_new(void) void tor_cond_free(tor_cond_t *cond) { - tor_assert(cond); + if (!cond) + return; if (pthread_cond_destroy(&cond->cond)) { log_warn(LD_GENERAL,"Error freeing condition: %s", strerror(errno)); return; @@ -2098,7 +2347,8 @@ tor_cond_new(void) void tor_cond_free(tor_cond_t *cond) { - tor_assert(cond); + if (!cond) + return; DeleteCriticalSection(&cond->mutex); /* XXXX notify? */ smartlist_free(cond->events); @@ -2174,6 +2424,89 @@ tor_threads_init(void) } #endif +#if defined(HAVE_MLOCKALL) && HAVE_DECL_MLOCKALL && defined(RLIMIT_MEMLOCK) +/** Attempt to raise the current and max rlimit to infinity for our process. + * This only needs to be done once and can probably only be done when we have + * not already dropped privileges. + */ +static int +tor_set_max_memlock(void) +{ + /* Future consideration for Windows is probably SetProcessWorkingSetSize + * This is similar to setting the memory rlimit of RLIMIT_MEMLOCK + * http://msdn.microsoft.com/en-us/library/ms686234(VS.85).aspx + */ + + struct rlimit limit; + + /* RLIM_INFINITY is -1 on some platforms. */ + limit.rlim_cur = RLIM_INFINITY; + limit.rlim_max = RLIM_INFINITY; + + if (setrlimit(RLIMIT_MEMLOCK, &limit) == -1) { + if (errno == EPERM) { + log_warn(LD_GENERAL, "You appear to lack permissions to change memory " + "limits. Are you root?"); + } + log_warn(LD_GENERAL, "Unable to raise RLIMIT_MEMLOCK: %s", + strerror(errno)); + return -1; + } + + return 0; +} +#endif + +/** Attempt to lock all current and all future memory pages. + * This should only be called once and while we're privileged. + * Like mlockall() we return 0 when we're successful and -1 when we're not. + * Unlike mlockall() we return 1 if we've already attempted to lock memory. + */ +int +tor_mlockall(void) +{ + static int memory_lock_attempted = 0; + + if (memory_lock_attempted) { + return 1; + } + + memory_lock_attempted = 1; + + /* + * Future consideration for Windows may be VirtualLock + * VirtualLock appears to implement mlock() but not mlockall() + * + * http://msdn.microsoft.com/en-us/library/aa366895(VS.85).aspx + */ + +#if defined(HAVE_MLOCKALL) && HAVE_DECL_MLOCKALL && defined(RLIMIT_MEMLOCK) + if (tor_set_max_memlock() == 0) { + log_debug(LD_GENERAL, "RLIMIT_MEMLOCK is now set to RLIM_INFINITY."); + } + + if (mlockall(MCL_CURRENT|MCL_FUTURE) == 0) { + log_info(LD_GENERAL, "Insecure OS paging is effectively disabled."); + return 0; + } else { + if (errno == ENOSYS) { + /* Apple - it's 2009! I'm looking at you. Grrr. */ + log_notice(LD_GENERAL, "It appears that mlockall() is not available on " + "your platform."); + } else if (errno == EPERM) { + log_notice(LD_GENERAL, "It appears that you lack the permissions to " + "lock memory. Are you root?"); + } + log_notice(LD_GENERAL, "Unable to lock all current and future memory " + "pages: %s", strerror(errno)); + return -1; + } +#else + log_warn(LD_GENERAL, "Unable to lock memory pages. mlockall() unsupported?"); + return -1; +#endif +} + /** Identity of the "main" thread */ static unsigned long main_thread_id = -1; @@ -2325,20 +2658,26 @@ network_init(void) char * format_win32_error(DWORD err) { - LPVOID str = NULL; + TCHAR *str = NULL; char *result; /* Somebody once decided that this interface was better than strerror(). */ - FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | + FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, NULL, err, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), - (LPTSTR) &str, + (LPVOID)&str, 0, NULL); if (str) { - result = tor_strdup((char*)str); +#ifdef UNICODE + char abuf[1024] = {0}; + wcstombs(abuf,str,1024); + result = tor_strdup(abuf); +#else + result = tor_strdup(str); +#endif LocalFree(str); /* LocalFree != free() */ } else { result = tor_strdup("<unformattable error>"); diff --git a/src/common/compat.h b/src/common/compat.h index c7ed32b0d8..af795ffba9 100644 --- a/src/common/compat.h +++ b/src/common/compat.h @@ -51,6 +51,22 @@ #include <netinet6/in6.h> #endif +#if defined (WINCE) +#include <fcntl.h> +#include <io.h> +#include <math.h> +#include <projects.h> +#define snprintf _snprintf +/* this is not exported as W .... */ +#define SHGetPathFromIDListW SHGetPathFromIDList +/* wcecompat has vasprintf */ +#define HAVE_VASPRINTF +/* no service here */ +#ifdef NT_SERVICE +#undef NT_SERVICE +#endif +#endif // WINCE + #ifndef NULL_REP_IS_ZERO_BYTES #error "It seems your platform does not represent NULL as zero. We can't cope." #endif @@ -177,8 +193,8 @@ extern INLINE double U64_TO_DBL(uint64_t x) { /* ===== String compatibility */ #ifdef MS_WINDOWS /* Windows names string functions differently from most other platforms. */ -#define strncasecmp strnicmp -#define strcasecmp stricmp +#define strncasecmp _strnicmp +#define strcasecmp _stricmp #endif #ifndef HAVE_STRLCAT size_t strlcat(char *dst, const char *src, size_t siz) ATTR_NONNULL((1,2)); @@ -196,18 +212,26 @@ size_t strlcpy(char *dst, const char *src, size_t siz) ATTR_NONNULL((1,2)); #define U64_SCANF_ARG(a) (a) /** Expands to a literal uint64_t-typed constant for the value <b>n</b>. */ #define U64_LITERAL(n) (n ## ui64) +#define I64_PRINTF_ARG(a) (a) +#define I64_SCANF_ARG(a) (a) +#define I64_LITERAL(n) (n ## i64) #else #define U64_PRINTF_ARG(a) ((long long unsigned int)(a)) #define U64_SCANF_ARG(a) ((long long unsigned int*)(a)) #define U64_LITERAL(n) (n ## llu) +#define I64_PRINTF_ARG(a) ((long long signed int)(a)) +#define I64_SCANF_ARG(a) ((long long signed int*)(a)) +#define I64_LITERAL(n) (n ## ll) #endif #if defined(_MSC_VER) || defined(__MINGW32__) || defined(__MINGW64__) /** The formatting string used to put a uint64_t value in a printf() or * scanf() function. See also U64_PRINTF_ARG and U64_SCANF_ARG. */ #define U64_FORMAT "%I64u" +#define I64_FORMAT "%I64d" #else #define U64_FORMAT "%llu" +#define I64_FORMAT "%lld" #endif /** Represents an mmaped file. Allocated via tor_mmap_file; freed with @@ -235,6 +259,10 @@ int tor_snprintf(char *str, size_t size, const char *format, ...) int tor_vsnprintf(char *str, size_t size, const char *format, va_list args) ATTR_NONNULL((1,3)); +int tor_asprintf(char **strp, const char *fmt, ...) + CHECK_PRINTF(2,3); +int tor_vasprintf(char **strp, const char *fmt, va_list args); + const void *tor_memmem(const void *haystack, size_t hlen, const void *needle, size_t nlen) ATTR_PURE ATTR_NONNULL((1,3)); static const void *tor_memstr(const void *haystack, size_t hlen, @@ -267,6 +295,13 @@ extern const char TOR_TOLOWER_TABLE[]; #define TOR_TOLOWER(c) (TOR_TOLOWER_TABLE[(uint8_t)c]) #define TOR_TOUPPER(c) (TOR_TOUPPER_TABLE[(uint8_t)c]) +char *tor_strtok_r_impl(char *str, const char *sep, char **lasts); +#ifdef HAVE_STRTOK_R +#define tor_strtok_r(str, sep, lasts) strtok_r(str, sep, lasts) +#else +#define tor_strtok_r(str, sep, lasts) tor_strtok_r_impl(str, sep, lasts) +#endif + #ifdef MS_WINDOWS #define _SHORT_FILE_ (tor_fix_source_file(__FILE__)) const char *tor_fix_source_file(const char *fname); @@ -277,6 +312,7 @@ const char *tor_fix_source_file(const char *fname); /* ===== Time compatibility */ #if !defined(HAVE_GETTIMEOFDAY) && !defined(HAVE_STRUCT_TIMEVAL_TV_SEC) +/** Implementation of timeval for platforms that don't have it. */ struct timeval { time_t tv_sec; unsigned int tv_usec; @@ -285,16 +321,49 @@ struct timeval { void tor_gettimeofday(struct timeval *timeval); -#ifdef HAVE_LOCALTIME_R -#define tor_localtime_r localtime_r -#else struct tm *tor_localtime_r(const time_t *timep, struct tm *result); -#endif - -#ifdef HAVE_GMTIME_R -#define tor_gmtime_r gmtime_r -#else struct tm *tor_gmtime_r(const time_t *timep, struct tm *result); + +#ifndef timeradd +/** Replacement for timeradd on platforms that do not have it: sets tvout to + * the sum of tv1 and tv2. */ +#define timeradd(tv1,tv2,tvout) \ + do { \ + (tvout)->tv_sec = (tv1)->tv_sec + (tv2)->tv_sec; \ + (tvout)->tv_usec = (tv2)->tv_usec + (tv2)->tv_usec; \ + if ((tvout)->tv_usec >= 1000000) { \ + (tvout)->tv_usec -= 1000000; \ + (tvout)->tv_sec++; \ + } \ + } while (0) +#endif + +#ifndef timersub +/** Replacement for timersub on platforms that do not have it: sets tvout to + * tv1 minus tv2. */ +#define timersub(tv1,tv2,tvout) \ + do { \ + (tvout)->tv_sec = (tv1)->tv_sec - (tv2)->tv_sec; \ + (tvout)->tv_usec = (tv2)->tv_usec - (tv2)->tv_usec; \ + if ((tvout)->tv_usec < 0) { \ + (tvout)->tv_usec += 1000000; \ + (tvout)->tv_sec--; \ + } \ + } while (0) +#endif + +#ifndef timercmp +/** Replacement for timersub on platforms that do not have it: returns true + * iff the relational operator "op" makes the expression tv1 op tv2 true. + * + * Note that while this definition should work for all boolean opeators, some + * platforms' native timercmp definitions do not support >=, <=, or ==. So + * don't use those. + */ +#define timercmp(tv1,tv2,op) \ + (((tv1)->tv_sec == (tv2)->tv_sec) ? \ + ((tv1)->tv_usec op (tv2)->tv_usec) : \ + ((tv1)->tv_sec op (tv2)->tv_sec)) #endif /* ===== File compatibility */ @@ -329,9 +398,9 @@ int get_n_open_sockets(void); #define tor_socket_send(s, buf, len, flags) send(s, buf, len, flags) #define tor_socket_recv(s, buf, len, flags) recv(s, buf, len, flags) -/* Define struct in6_addr on platforms that do not have it. Generally, - * these platforms are ones without IPv6 support, but we want to have - * a working in6_addr there anyway, so we can use it to parse IPv6 +/** Implementatino of struct in6_addr for platforms that do not have it. + * Generally, these platforms are ones without IPv6 support, but we want to + * have a working in6_addr there anyway, so we can use it to parse IPv6 * addresses. */ #if !defined(HAVE_STRUCT_IN6_ADDR) struct in6_addr @@ -347,9 +416,10 @@ struct in6_addr }; #endif +/** @{ */ +/** Many BSD variants seem not to define these. */ #if defined(__APPLE__) || defined(__darwin__) || defined(__FreeBSD__) \ || defined(__NetBSD__) || defined(__OpenBSD__) -/* Many BSD variants seem not to define these. */ #ifndef s6_addr16 #define s6_addr16 __u6_addr.__u6_addr16 #endif @@ -357,12 +427,15 @@ struct in6_addr #define s6_addr32 __u6_addr.__u6_addr32 #endif #endif +/** @} */ #ifndef HAVE_SA_FAMILY_T typedef uint16_t sa_family_t; #endif -/* Apparently, MS and Solaris don't define s6_addr16 or s6_addr32. */ +/** @{ */ +/** Apparently, MS and Solaris don't define s6_addr16 or s6_addr32; these + * macros get you a pointer to s6_addr32 or local equivalent. */ #ifdef HAVE_STRUCT_IN6_ADDR_S6_ADDR32 #define S6_ADDR32(x) ((uint32_t*)(x).s6_addr32) #else @@ -373,9 +446,10 @@ typedef uint16_t sa_family_t; #else #define S6_ADDR16(x) ((uint16_t*)((char*)&(x).s6_addr)) #endif +/** @} */ -/* Define struct sockaddr_in6 on platforms that do not have it. See notes - * on struct in6_addr. */ +/** Implementation of struct sockaddr_in6 on platforms that do not have + * it. See notes on struct in6_addr. */ #if !defined(HAVE_STRUCT_SOCKADDR_IN6) struct sockaddr_in6 { sa_family_t sin6_family; @@ -445,6 +519,11 @@ typedef enum { SOCKS5_ADDRESS_TYPE_NOT_SUPPORTED = 0x08, } socks5_reply_status_t; +/* ===== Insecure rng */ +void tor_init_weak_random(unsigned seed); +long tor_weak_random(void); +#define TOR_RAND_MAX (RAND_MAX) + /* ===== OS compatibility */ const char *get_uname(void); @@ -494,14 +573,20 @@ void spawn_exit(void) ATTR_NORETURN; /** A generic lock structure for multithreaded builds. */ typedef struct tor_mutex_t { #if defined(USE_WIN32_THREADS) + /** Windows-only: on windows, we implement locks with CRITICAL_SECTIONS. */ CRITICAL_SECTION mutex; #elif defined(USE_PTHREADS) + /** Pthreads-only: with pthreads, we implement locks with + * pthread_mutex_t. */ pthread_mutex_t mutex; #else + /** No-threads only: Dummy variable so that tor_mutex_t takes up space. */ int _unused; #endif } tor_mutex_t; +int tor_mlockall(void); + #ifdef TOR_IS_MULTITHREADED tor_mutex_t *tor_mutex_new(void); void tor_mutex_init(tor_mutex_t *m); @@ -536,6 +621,19 @@ void tor_cond_signal_all(tor_cond_t *cond); #endif #endif +/** Macros for MIN/MAX. Never use these when the arguments could have + * side-effects. + * {With GCC extensions we could probably define a safer MIN/MAX. But + * depending on that safety would be dangerous, since not every platform + * has it.} + **/ +#ifndef MAX +#define MAX(a,b) ( ((a)<(b)) ? (b) : (a) ) +#endif +#ifndef MIN +#define MIN(a,b) ( ((a)>(b)) ? (b) : (a) ) +#endif + /* Platform-specific helpers. */ #ifdef MS_WINDOWS char *format_win32_error(DWORD err); diff --git a/src/common/compat_libevent.c b/src/common/compat_libevent.c new file mode 100644 index 0000000000..3ad9be145d --- /dev/null +++ b/src/common/compat_libevent.c @@ -0,0 +1,554 @@ +/* Copyright (c) 2009-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file compat_libevent.c + * \brief Wrappers to handle porting between different versions of libevent. + * + * In an ideal world, we'd just use Libevent 2.0 from now on. But as of June + * 2009, Libevent 2.0 is still in alpha, and we will have old versions of + * Libevent for the forseeable future. + **/ + +#include "orconfig.h" +#include "compat.h" +#include "compat_libevent.h" + +#include "util.h" +#include "torlog.h" + +#ifdef HAVE_EVENT2_EVENT_H +#include <event2/event.h> +#else +#include <event.h> +#endif + +/** A number representing a version of Libevent. + + This is a 4-byte number, with the first three bytes representing the + major, minor, and patchlevel respectively of the library. The fourth + byte is unused. + + This is equivalent to the format of LIBEVENT_VERSION_NUMBER on Libevent + 2.0.1 or later. For versions of Libevent before 1.4.0, which followed the + format of "1.0, 1.0a, 1.0b", we define 1.0 to be equivalent to 1.0.0, 1.0a + to be equivalent to 1.0.1, and so on. +*/ +typedef uint32_t le_version_t; + +/** @{ */ +/** Macros: returns the number of a libevent version as a le_version_t */ +#define V(major, minor, patch) \ + (((major) << 24) | ((minor) << 16) | ((patch) << 8)) +#define V_OLD(major, minor, patch) \ + V((major), (minor), (patch)-'a'+1) +/** @} */ + +/** Represetns a version of libevent so old we can't figure out what version + * it is. */ +#define LE_OLD V(0,0,0) +/** Represents a version of libevent so weird we can't figure out what version + * it it. */ +#define LE_OTHER V(0,0,99) + +static le_version_t tor_get_libevent_version(const char **v_out); + +#ifdef HAVE_EVENT_SET_LOG_CALLBACK +/** A string which, if it appears in a libevent log, should be ignored. */ +static const char *suppress_msg = NULL; +/** Callback function passed to event_set_log() so we can intercept + * log messages from libevent. */ +static void +libevent_logging_callback(int severity, const char *msg) +{ + char buf[1024]; + size_t n; + if (suppress_msg && strstr(msg, suppress_msg)) + return; + n = strlcpy(buf, msg, sizeof(buf)); + if (n && n < sizeof(buf) && buf[n-1] == '\n') { + buf[n-1] = '\0'; + } + switch (severity) { + case _EVENT_LOG_DEBUG: + log(LOG_DEBUG, LD_NOCB|LD_NET, "Message from libevent: %s", buf); + break; + case _EVENT_LOG_MSG: + log(LOG_INFO, LD_NOCB|LD_NET, "Message from libevent: %s", buf); + break; + case _EVENT_LOG_WARN: + log(LOG_WARN, LD_NOCB|LD_GENERAL, "Warning from libevent: %s", buf); + break; + case _EVENT_LOG_ERR: + log(LOG_ERR, LD_NOCB|LD_GENERAL, "Error from libevent: %s", buf); + break; + default: + log(LOG_WARN, LD_NOCB|LD_GENERAL, "Message [%d] from libevent: %s", + severity, buf); + break; + } +} +/** Set hook to intercept log messages from libevent. */ +void +configure_libevent_logging(void) +{ + event_set_log_callback(libevent_logging_callback); +} +/** Ignore any libevent log message that contains <b>msg</b>. */ +void +suppress_libevent_log_msg(const char *msg) +{ + suppress_msg = msg; +} +#else +void +configure_libevent_logging(void) +{ +} +void +suppress_libevent_log_msg(const char *msg) +{ + (void)msg; +} +#endif + +#ifndef HAVE_EVENT2_EVENT_H +/** Work-alike replacement for event_new() on pre-Libevent-2.0 systems. */ +struct event * +tor_event_new(struct event_base *base, int sock, short what, + void (*cb)(int, short, void *), void *arg) +{ + struct event *e = tor_malloc_zero(sizeof(struct event)); + event_set(e, sock, what, cb, arg); + if (! base) + base = tor_libevent_get_base(); + event_base_set(base, e); + return e; +} +/** Work-alike replacement for evtimer_new() on pre-Libevent-2.0 systems. */ +struct event * +tor_evtimer_new(struct event_base *base, + void (*cb)(int, short, void *), void *arg) +{ + return tor_event_new(base, -1, 0, cb, arg); +} +/** Work-alike replacement for evsignal_new() on pre-Libevent-2.0 systems. */ +struct event * +tor_evsignal_new(struct event_base * base, int sig, + void (*cb)(int, short, void *), void *arg) +{ + return tor_event_new(base, sig, EV_SIGNAL|EV_PERSIST, cb, arg); +} +/** Work-alike replacement for event_free() on pre-Libevent-2.0 systems. */ +void +tor_event_free(struct event *ev) +{ + event_del(ev); + tor_free(ev); +} +#endif + +/** Global event base for use by the main thread. */ +struct event_base *the_event_base = NULL; + +/* This is what passes for version detection on OSX. We set + * MACOSX_KQUEUE_IS_BROKEN to true iff we're on a version of OSX before + * 10.4.0 (aka 1040). */ +#ifdef __APPLE__ +#ifdef __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ +#define MACOSX_KQUEUE_IS_BROKEN \ + (__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 1040) +#else +#define MACOSX_KQUEUE_IS_BROKEN 0 +#endif +#endif + +/** Initialize the Libevent library and set up the event base. */ +void +tor_libevent_initialize(void) +{ + tor_assert(the_event_base == NULL); + +#ifdef __APPLE__ + if (MACOSX_KQUEUE_IS_BROKEN || + tor_get_libevent_version(NULL) < V_OLD(1,1,'b')) { + setenv("EVENT_NOKQUEUE","1",1); + } +#endif + +#ifdef HAVE_EVENT2_EVENT_H + the_event_base = event_base_new(); +#else + the_event_base = event_init(); +#endif + +#if defined(HAVE_EVENT_GET_VERSION) && defined(HAVE_EVENT_GET_METHOD) + /* Making this a NOTICE for now so we can link bugs to a libevent versions + * or methods better. */ + log(LOG_NOTICE, LD_GENERAL, + "Initialized libevent version %s using method %s. Good.", + event_get_version(), tor_libevent_get_method()); +#else + log(LOG_NOTICE, LD_GENERAL, + "Initialized old libevent (version 1.0b or earlier)."); + log(LOG_WARN, LD_GENERAL, + "You have a *VERY* old version of libevent. It is likely to be buggy; " + "please build Tor with a more recent version."); +#endif +} + +/** Return the current Libevent event base that we're set up to use. */ +struct event_base * +tor_libevent_get_base(void) +{ + return the_event_base; +} + +#ifndef HAVE_EVENT_BASE_LOOPEXIT +/** Replacement for event_base_loopexit on some very old versions of Libevent + * that we are not yet brave enough to deprecate. */ +int +tor_event_base_loopexit(struct event_base *base, struct timeval *tv) +{ + tor_assert(base == the_event_base); + return event_loopexit(tv); +} +#endif + +/** Return the name of the Libevent backend we're using. */ +const char * +tor_libevent_get_method(void) +{ +#ifdef HAVE_EVENT2_EVENT_H + return event_base_get_method(the_event_base); +#elif defined(HAVE_EVENT_GET_METHOD) + return event_get_method(); +#else + return "<unknown>"; +#endif +} + +/** Return the le_version_t for the version of libevent specified in the + * string <b>v</b>. If the version is very new or uses an unrecognized + * version, format, return LE_OTHER. */ +static le_version_t +tor_decode_libevent_version(const char *v) +{ + unsigned major, minor, patchlevel; + char c, e, extra; + int fields; + + /* Try the new preferred "1.4.11-stable" format. + * Also accept "1.4.14b-stable". */ + fields = sscanf(v, "%u.%u.%u%c%c", &major, &minor, &patchlevel, &c, &e); + if (fields == 3 || + ((fields == 4 || fields == 5 ) && (c == '-' || c == '_')) || + (fields == 5 && TOR_ISALPHA(c) && (e == '-' || e == '_'))) { + return V(major,minor,patchlevel); + } + + /* Try the old "1.3e" format. */ + fields = sscanf(v, "%u.%u%c%c", &major, &minor, &c, &extra); + if (fields == 3 && TOR_ISALPHA(c)) { + return V_OLD(major, minor, c); + } else if (fields == 2) { + return V(major, minor, 0); + } + + return LE_OTHER; +} + +/** Return an integer representing the binary interface of a Libevent library. + * Two different versions with different numbers are sure not to be binary + * compatible. Two different versions with the same numbers have a decent + * chance of binary compatibility.*/ +static int +le_versions_compatibility(le_version_t v) +{ + if (v == LE_OTHER) + return 0; + if (v < V_OLD(1,0,'c')) + return 1; + else if (v < V(1,4,0)) + return 2; + else if (v < V(1,4,99)) + return 3; + else if (v < V(2,0,1)) + return 4; + else /* Everything 2.0 and later should be compatible. */ + return 5; +} + +/** Return the version number of the currently running version of Libevent. + * See le_version_t for info on the format. + */ +static le_version_t +tor_get_libevent_version(const char **v_out) +{ + const char *v; + le_version_t r; +#if defined(HAVE_EVENT_GET_VERSION_NUMBER) + v = event_get_version(); + r = event_get_version_number(); +#elif defined (HAVE_EVENT_GET_VERSION) + v = event_get_version(); + r = tor_decode_libevent_version(v); +#else + v = "pre-1.0c"; + r = LE_OLD; +#endif + if (v_out) + *v_out = v; + return r; +} + +/** Return a string representation of the version of the currently running + * version of Libevent. */ +const char * +tor_libevent_get_version_str(void) +{ +#ifdef HAVE_EVENT_GET_VERSION + return event_get_version(); +#else + return "pre-1.0c"; +#endif +} + +/** + * Compare the current Libevent method and version to a list of versions + * which are known not to work. Warn the user as appropriate. + */ +void +tor_check_libevent_version(const char *m, int server, + const char **badness_out) +{ + int buggy = 0, iffy = 0, slow = 0, thread_unsafe = 0; + le_version_t version; + const char *v = NULL; + const char *badness = NULL; + const char *sad_os = ""; + + version = tor_get_libevent_version(&v); + + /* It would be better to disable known-buggy methods rather than warning + * about them. But the problem is that with older versions of Libevent, + * it's not trivial to get them to change their methods once they're + * initialized... and with newer versions of Libevent, they aren't actually + * broken. But we should revisit this if we ever find a post-1.4 version + * of Libevent where we need to disable a given method. */ + if (!strcmp(m, "kqueue")) { + if (version < V_OLD(1,1,'b')) + buggy = 1; + } else if (!strcmp(m, "epoll")) { + if (version < V(1,1,0)) + iffy = 1; + } else if (!strcmp(m, "poll")) { + if (version < V_OLD(1,0,'e')) + buggy = 1; + if (version < V(1,1,0)) + slow = 1; + } else if (!strcmp(m, "select")) { + if (version < V(1,1,0)) + slow = 1; + } else if (!strcmp(m, "win32")) { + if (version < V_OLD(1,1,'b')) + buggy = 1; + } + + /* Libevent versions before 1.3b do very badly on operating systems with + * user-space threading implementations. */ +#if defined(__OpenBSD__) || defined(__FreeBSD__) || defined(__NetBSD__) + if (server && version < V_OLD(1,3,'b')) { + thread_unsafe = 1; + sad_os = "BSD variants"; + } +#elif defined(__APPLE__) || defined(__darwin__) + if (server && version < V_OLD(1,3,'b')) { + thread_unsafe = 1; + sad_os = "Mac OS X"; + } +#endif + + if (thread_unsafe) { + log(LOG_WARN, LD_GENERAL, + "Libevent version %s often crashes when running a Tor server with %s. " + "Please use the latest version of libevent (1.3b or later)",v,sad_os); + badness = "BROKEN"; + } else if (buggy) { + log(LOG_WARN, LD_GENERAL, + "There are serious bugs in using %s with libevent %s. " + "Please use the latest version of libevent.", m, v); + badness = "BROKEN"; + } else if (iffy) { + log(LOG_WARN, LD_GENERAL, + "There are minor bugs in using %s with libevent %s. " + "You may want to use the latest version of libevent.", m, v); + badness = "BUGGY"; + } else if (slow && server) { + log(LOG_WARN, LD_GENERAL, + "libevent %s can be very slow with %s. " + "When running a server, please use the latest version of libevent.", + v,m); + badness = "SLOW"; + } + + *badness_out = badness; +} + +#if defined(LIBEVENT_VERSION) +#define HEADER_VERSION LIBEVENT_VERSION +#elif defined(_EVENT_VERSION) +#define HEADER_VERSION _EVENT_VERSION +#endif + +/** See whether the headers we were built against differ from the library we + * linked against so much that we're likely to crash. If so, warn the + * user. */ +void +tor_check_libevent_header_compatibility(void) +{ + (void) le_versions_compatibility; + (void) tor_decode_libevent_version; + + /* In libevent versions before 2.0, it's hard to keep binary compatibility + * between upgrades, and unpleasant to detect when the version we compiled + * against is unlike the version we have linked against. Here's how. */ +#if defined(HEADER_VERSION) && defined(HAVE_EVENT_GET_VERSION) + /* We have a header-file version and a function-call version. Easy. */ + if (strcmp(HEADER_VERSION, event_get_version())) { + le_version_t v1, v2; + int compat1 = -1, compat2 = -1; + int verybad; + v1 = tor_decode_libevent_version(HEADER_VERSION); + v2 = tor_decode_libevent_version(event_get_version()); + compat1 = le_versions_compatibility(v1); + compat2 = le_versions_compatibility(v2); + + verybad = compat1 != compat2; + + log(verybad ? LOG_WARN : LOG_NOTICE, + LD_GENERAL, "We were compiled with headers from version %s " + "of Libevent, but we're using a Libevent library that says it's " + "version %s.", HEADER_VERSION, event_get_version()); + if (verybad) + log_warn(LD_GENERAL, "This will almost certainly make Tor crash."); + else + log_info(LD_GENERAL, "I think these versions are binary-compatible."); + } +#elif defined(HAVE_EVENT_GET_VERSION) + /* event_get_version but no _EVENT_VERSION. We might be in 1.4.0-beta or + earlier, where that's normal. To see whether we were compiled with an + earlier version, let's see whether the struct event defines MIN_HEAP_IDX. + */ +#ifdef HAVE_STRUCT_EVENT_MIN_HEAP_IDX + /* The header files are 1.4.0-beta or later. If the version is not + * 1.4.0-beta, we are incompatible. */ + { + if (strcmp(event_get_version(), "1.4.0-beta")) { + log_warn(LD_GENERAL, "It's a little hard to tell, but you seem to have " + "Libevent 1.4.0-beta header files, whereas you have linked " + "against Libevent %s. This will probably make Tor crash.", + event_get_version()); + } + } +#else + /* Our headers are 1.3e or earlier. If the library version is not 1.4.x or + later, we're probably fine. */ + { + const char *v = event_get_version(); + if ((v[0] == '1' && v[2] == '.' && v[3] > '3') || v[0] > '1') { + log_warn(LD_GENERAL, "It's a little hard to tell, but you seem to have " + "Libevent header file from 1.3e or earlier, whereas you have " + "linked against Libevent %s. This will probably make Tor " + "crash.", event_get_version()); + } + } +#endif + +#elif defined(HEADER_VERSION) +#warn "_EVENT_VERSION is defined but not get_event_version(): Libevent is odd." +#else + /* Your libevent is ancient. */ +#endif +} + +/* + If possible, we're going to try to use Libevent's periodic timer support, + since it does a pretty good job of making sure that periodic events get + called exactly M seconds apart, rather than starting each one exactly M + seconds after the time that the last one was run. + */ +#ifdef HAVE_EVENT2_EVENT_H +#define HAVE_PERIODIC +#define PERIODIC_FLAGS EV_PERSIST +#else +#define PERIODIC_FLAGS 0 +#endif + +/** Represents a timer that's run every N microseconds by Libevent. */ +struct periodic_timer_t { + /** Underlying event used to implement this periodic event. */ + struct event *ev; + /** The callback we'll be invoking whenever the event triggers */ + void (*cb)(struct periodic_timer_t *, void *); + /** User-supplied data for the callback */ + void *data; +#ifndef HAVE_PERIODIC + /** If Libevent doesn't know how to invoke events every N microseconds, + * we'll need to remember the timeout interval here. */ + struct timeval tv; +#endif +}; + +/** Libevent callback to implement a periodic event. */ +static void +periodic_timer_cb(evutil_socket_t fd, short what, void *arg) +{ + periodic_timer_t *timer = arg; + (void) what; + (void) fd; +#ifndef HAVE_PERIODIC + /** reschedule the event as needed. */ + event_add(timer->ev, &timer->tv); +#endif + timer->cb(timer, timer->data); +} + +/** Create and schedule a new timer that will run every <b>tv</b> in + * the event loop of <b>base</b>. When the timer fires, it will + * run the timer in <b>cb</b> with the user-supplied data in <b>data</b>. */ +periodic_timer_t * +periodic_timer_new(struct event_base *base, + const struct timeval *tv, + void (*cb)(periodic_timer_t *timer, void *data), + void *data) +{ + periodic_timer_t *timer; + tor_assert(base); + tor_assert(tv); + tor_assert(cb); + timer = tor_malloc_zero(sizeof(periodic_timer_t)); + if (!(timer->ev = tor_event_new(base, -1, PERIODIC_FLAGS, + periodic_timer_cb, timer))) { + tor_free(timer); + return NULL; + } + timer->cb = cb; + timer->data = data; +#ifndef HAVE_PERIODIC + memcpy(&timer->tv, tv, sizeof(struct timeval)); +#endif + event_add(timer->ev, (struct timeval *)tv); /*drop const for old libevent*/ + return timer; +} + +/** Stop and free a periodic timer */ +void +periodic_timer_free(periodic_timer_t *timer) +{ + if (!timer) + return; + tor_event_free(timer->ev); + tor_free(timer); +} + diff --git a/src/common/compat_libevent.h b/src/common/compat_libevent.h new file mode 100644 index 0000000000..fdf5e0a18f --- /dev/null +++ b/src/common/compat_libevent.h @@ -0,0 +1,65 @@ +/* Copyright (c) 2009, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef _TOR_COMPAT_LIBEVENT_H +#define _TOR_COMPAT_LIBEVENT_H + +#include "orconfig.h" + +struct event; +struct event_base; + +#ifdef HAVE_EVENT2_EVENT_H +#include <event2/util.h> +#else +#define evutil_socket_t int +#endif + +void configure_libevent_logging(void); +void suppress_libevent_log_msg(const char *msg); + +#ifdef HAVE_EVENT2_EVENT_H +#define tor_event_new event_new +#define tor_evtimer_new evtimer_new +#define tor_evsignal_new evsignal_new +#define tor_event_free event_free +#define tor_evdns_add_server_port(sock, tcp, cb, data) \ + evdns_add_server_port_with_base(tor_libevent_get_base(), \ + (sock),(tcp),(cb),(data)); + +#else +struct event *tor_event_new(struct event_base * base, evutil_socket_t sock, + short what, void (*cb)(evutil_socket_t, short, void *), void *arg); +struct event *tor_evtimer_new(struct event_base * base, + void (*cb)(evutil_socket_t, short, void *), void *arg); +struct event *tor_evsignal_new(struct event_base * base, int sig, + void (*cb)(evutil_socket_t, short, void *), void *arg); +void tor_event_free(struct event *ev); +#define tor_evdns_add_server_port evdns_add_server_port +#endif + +typedef struct periodic_timer_t periodic_timer_t; + +periodic_timer_t *periodic_timer_new(struct event_base *base, + const struct timeval *tv, + void (*cb)(periodic_timer_t *timer, void *data), + void *data); +void periodic_timer_free(periodic_timer_t *); + +#ifdef HAVE_EVENT_BASE_LOOPEXIT +#define tor_event_base_loopexit event_base_loopexit +#else +struct timeval; +int tor_event_base_loopexit(struct event_base *base, struct timeval *tv); +#endif + +void tor_libevent_initialize(void); +struct event_base *tor_libevent_get_base(void); +const char *tor_libevent_get_method(void); +void tor_check_libevent_version(const char *m, int server, + const char **badness_out); +void tor_check_libevent_header_compatibility(void); +const char *tor_libevent_get_version_str(void); + +#endif + diff --git a/src/common/container.c b/src/common/container.c index 977d604071..7208d36803 100644 --- a/src/common/container.c +++ b/src/common/container.c @@ -13,7 +13,7 @@ #include "compat.h" #include "util.h" -#include "log.h" +#include "torlog.h" #include "container.h" #include "crypto.h" @@ -44,7 +44,8 @@ smartlist_create(void) void smartlist_free(smartlist_t *sl) { - tor_assert(sl != NULL); + if (!sl) + return; tor_free(sl->list); tor_free(sl); } @@ -459,6 +460,42 @@ smartlist_sort(smartlist_t *sl, int (*compare)(const void **a, const void **b)) (int (*)(const void *,const void*))compare); } +/** Given a smartlist <b>sl</b> sorted with the function <b>compare</b>, + * return the most frequent member in the list. Break ties in favor of + * later elements. If the list is empty, return NULL. + */ +void * +smartlist_get_most_frequent(const smartlist_t *sl, + int (*compare)(const void **a, const void **b)) +{ + const void *most_frequent = NULL; + int most_frequent_count = 0; + + const void *cur = NULL; + int i, count=0; + + if (!sl->num_used) + return NULL; + for (i = 0; i < sl->num_used; ++i) { + const void *item = sl->list[i]; + if (cur && 0 == compare(&cur, &item)) { + ++count; + } else { + if (cur && count >= most_frequent_count) { + most_frequent = cur; + most_frequent_count = count; + } + cur = item; + count = 1; + } + } + if (cur && count >= most_frequent_count) { + most_frequent = cur; + most_frequent_count = count; + } + return (void*)most_frequent; +} + /** Given a sorted smartlist <b>sl</b> and the comparison function used to * sort it, remove all duplicate members. If free_fn is provided, calls * free_fn on each duplicate. Otherwise, just removes them. Preserves order. @@ -550,6 +587,13 @@ smartlist_sort_strings(smartlist_t *sl) smartlist_sort(sl, _compare_string_ptrs); } +/** Return the most frequent string in the sorted list <b>sl</b> */ +char * +smartlist_get_most_frequent_string(smartlist_t *sl) +{ + return smartlist_get_most_frequent(sl, _compare_string_ptrs); +} + /** Remove duplicate strings from a sorted list, and free them with tor_free(). */ void @@ -561,16 +605,70 @@ smartlist_uniq_strings(smartlist_t *sl) /* Heap-based priority queue implementation for O(lg N) insert and remove. * Recall that the heap property is that, for every index I, h[I] < * H[LEFT_CHILD[I]] and h[I] < H[RIGHT_CHILD[I]]. + * + * For us to remove items other than the topmost item, each item must store + * its own index within the heap. When calling the pqueue functions, tell + * them about the offset of the field that stores the index within the item. + * + * Example: + * + * typedef struct timer_t { + * struct timeval tv; + * int heap_index; + * } timer_t; + * + * static int compare(const void *p1, const void *p2) { + * const timer_t *t1 = p1, *t2 = p2; + * if (t1->tv.tv_sec < t2->tv.tv_sec) { + * return -1; + * } else if (t1->tv.tv_sec > t2->tv.tv_sec) { + * return 1; + * } else { + * return t1->tv.tv_usec - t2->tv_usec; + * } + * } + * + * void timer_heap_insert(smartlist_t *heap, timer_t *timer) { + * smartlist_pqueue_add(heap, compare, STRUCT_OFFSET(timer_t, heap_index), + * timer); + * } + * + * void timer_heap_pop(smartlist_t *heap) { + * return smartlist_pqueue_pop(heap, compare, + * STRUCT_OFFSET(timer_t, heap_index)); + * } */ -/* For a 1-indexed array, we would use LEFT_CHILD[x] = 2*x and RIGHT_CHILD[x] - * = 2*x + 1. But this is C, so we have to adjust a little. */ +/** @{ */ +/** Functions to manipulate heap indices to find a node's parent and children. + * + * For a 1-indexed array, we would use LEFT_CHILD[x] = 2*x and RIGHT_CHILD[x] + * = 2*x + 1. But this is C, so we have to adjust a little. */ //#define LEFT_CHILD(i) ( ((i)+1)*2 - 1) //#define RIGHT_CHILD(i) ( ((i)+1)*2 ) //#define PARENT(i) ( ((i)+1)/2 - 1) #define LEFT_CHILD(i) ( 2*(i) + 1 ) #define RIGHT_CHILD(i) ( 2*(i) + 2 ) #define PARENT(i) ( ((i)-1) / 2 ) +/** }@ */ + +/** @{ */ +/** Helper macros for heaps: Given a local variable <b>idx_field_offset</b> + * set to the offset of an integer index within the heap element structure, + * IDX_OF_ITEM(p) gives you the index of p, and IDXP(p) gives you a pointer to + * where p's index is stored. Given additionally a local smartlist <b>sl</b>, + * UPDATE_IDX(i) sets the index of the element at <b>i</b> to the correct + * value (that is, to <b>i</b>). + */ +#define IDXP(p) ((int*)STRUCT_VAR_P(p, idx_field_offset)) + +#define UPDATE_IDX(i) do { \ + void *updated = sl->list[i]; \ + *IDXP(updated) = i; \ + } while (0) + +#define IDX_OF_ITEM(p) (*IDXP(p)) +/** @} */ /** Helper. <b>sl</b> may have at most one violation of the heap property: * the item at <b>idx</b> may be greater than one or both of its children. @@ -578,6 +676,7 @@ smartlist_uniq_strings(smartlist_t *sl) static INLINE void smartlist_heapify(smartlist_t *sl, int (*compare)(const void *a, const void *b), + int idx_field_offset, int idx) { while (1) { @@ -600,21 +699,28 @@ smartlist_heapify(smartlist_t *sl, void *tmp = sl->list[idx]; sl->list[idx] = sl->list[best_idx]; sl->list[best_idx] = tmp; + UPDATE_IDX(idx); + UPDATE_IDX(best_idx); idx = best_idx; } } } -/** Insert <b>item</b> into the heap stored in <b>sl</b>, where order - * is determined by <b>compare</b>. */ +/** Insert <b>item</b> into the heap stored in <b>sl</b>, where order is + * determined by <b>compare</b> and the offset of the item in the heap is + * stored in an int-typed field at position <b>idx_field_offset</b> within + * item. + */ void smartlist_pqueue_add(smartlist_t *sl, int (*compare)(const void *a, const void *b), + int idx_field_offset, void *item) { int idx; smartlist_add(sl,item); + UPDATE_IDX(sl->num_used-1); for (idx = sl->num_used - 1; idx; ) { int parent = PARENT(idx); @@ -622,6 +728,8 @@ smartlist_pqueue_add(smartlist_t *sl, void *tmp = sl->list[parent]; sl->list[parent] = sl->list[idx]; sl->list[idx] = tmp; + UPDATE_IDX(parent); + UPDATE_IDX(idx); idx = parent; } else { return; @@ -630,32 +738,63 @@ smartlist_pqueue_add(smartlist_t *sl, } /** Remove and return the top-priority item from the heap stored in <b>sl</b>, - * where order is determined by <b>compare</b>. <b>sl</b> must not be - * empty. */ + * where order is determined by <b>compare</b> and the item's position is + * stored at position <b>idx_field_offset</b> within the item. <b>sl</b> must + * not be empty. */ void * smartlist_pqueue_pop(smartlist_t *sl, - int (*compare)(const void *a, const void *b)) + int (*compare)(const void *a, const void *b), + int idx_field_offset) { void *top; tor_assert(sl->num_used); top = sl->list[0]; + *IDXP(top)=-1; if (--sl->num_used) { sl->list[0] = sl->list[sl->num_used]; - smartlist_heapify(sl, compare, 0); + UPDATE_IDX(0); + smartlist_heapify(sl, compare, idx_field_offset, 0); } return top; } +/** Remove the item <b>item</b> from the heap stored in <b>sl</b>, + * where order is determined by <b>compare</b> and the item's position is + * stored at position <b>idx_field_offset</b> within the item. <b>sl</b> must + * not be empty. */ +void +smartlist_pqueue_remove(smartlist_t *sl, + int (*compare)(const void *a, const void *b), + int idx_field_offset, + void *item) +{ + int idx = IDX_OF_ITEM(item); + tor_assert(idx >= 0); + tor_assert(sl->list[idx] == item); + --sl->num_used; + *IDXP(item) = -1; + if (idx == sl->num_used) { + return; + } else { + sl->list[idx] = sl->list[sl->num_used]; + UPDATE_IDX(idx); + smartlist_heapify(sl, compare, idx_field_offset, idx); + } +} + /** Assert that the heap property is correctly maintained by the heap stored * in <b>sl</b>, where order is determined by <b>compare</b>. */ void smartlist_pqueue_assert_ok(smartlist_t *sl, - int (*compare)(const void *a, const void *b)) + int (*compare)(const void *a, const void *b), + int idx_field_offset) { int i; - for (i = sl->num_used - 1; i > 0; --i) { - tor_assert(compare(sl->list[PARENT(i)], sl->list[i]) <= 0); + for (i = sl->num_used - 1; i >= 0; --i) { + if (i>0) + tor_assert(compare(sl->list[PARENT(i)], sl->list[i]) <= 0); + tor_assert(IDX_OF_ITEM(sl->list[i]) == i); } } @@ -681,6 +820,37 @@ smartlist_uniq_digests(smartlist_t *sl) smartlist_uniq(sl, _compare_digests, _tor_free); } +/** Helper: compare two DIGEST256_LEN digests. */ +static int +_compare_digests256(const void **_a, const void **_b) +{ + return memcmp((const char*)*_a, (const char*)*_b, DIGEST256_LEN); +} + +/** Sort the list of DIGEST256_LEN-byte digests into ascending order. */ +void +smartlist_sort_digests256(smartlist_t *sl) +{ + smartlist_sort(sl, _compare_digests256); +} + +/** Return the most frequent member of the sorted list of DIGEST256_LEN + * digests in <b>sl</b> */ +char * +smartlist_get_most_frequent_digest256(smartlist_t *sl) +{ + return smartlist_get_most_frequent(sl, _compare_digests256); +} + +/** Remove duplicate 256-bit digests from a sorted list, and free them with + * tor_free(). + */ +void +smartlist_uniq_digests256(smartlist_t *sl) +{ + smartlist_uniq(sl, _compare_digests256, _tor_free); +} + /** Helper: Declare an entry type and a map type to implement a mapping using * ht.h. The map type will be called <b>maptype</b>. The key part of each * entry is declared using the C declaration <b>keydecl</b>. All functions @@ -1113,6 +1283,9 @@ void strmap_free(strmap_t *map, void (*free_val)(void*)) { strmap_entry_t **ent, **next, *this; + if (!map) + return; + for (ent = HT_START(strmap_impl, &map->head); ent != NULL; ent = next) { this = *ent; next = HT_NEXT_RMV(strmap_impl, &map->head, ent); @@ -1134,6 +1307,8 @@ void digestmap_free(digestmap_t *map, void (*free_val)(void*)) { digestmap_entry_t **ent, **next, *this; + if (!map) + return; for (ent = HT_START(digestmap_impl, &map->head); ent != NULL; ent = next) { this = *ent; next = HT_NEXT_RMV(digestmap_impl, &map->head, ent); @@ -1220,6 +1395,7 @@ IMPLEMENT_ORDER_FUNC(find_nth_int, int) IMPLEMENT_ORDER_FUNC(find_nth_time, time_t) IMPLEMENT_ORDER_FUNC(find_nth_double, double) IMPLEMENT_ORDER_FUNC(find_nth_uint32, uint32_t) +IMPLEMENT_ORDER_FUNC(find_nth_int32, int32_t) IMPLEMENT_ORDER_FUNC(find_nth_long, long) /** Return a newly allocated digestset_t, optimized to hold a total of @@ -1248,6 +1424,8 @@ digestset_new(int max_elements) void digestset_free(digestset_t *set) { + if (!set) + return; bitarray_free(set->ba); tor_free(set); } diff --git a/src/common/container.h b/src/common/container.h index 2b9f964f76..8a3a405273 100644 --- a/src/common/container.h +++ b/src/common/container.h @@ -15,6 +15,7 @@ * and macros defined here. **/ typedef struct smartlist_t { + /** @{ */ /** <b>list</b> has enough capacity to store exactly <b>capacity</b> elements * before it needs to be resized. Only the first <b>num_used</b> (\<= * capacity) elements point to valid data. @@ -22,6 +23,7 @@ typedef struct smartlist_t { void **list; int num_used; int capacity; + /** @} */ } smartlist_t; smartlist_t *smartlist_create(void); @@ -93,13 +95,22 @@ void smartlist_del_keeporder(smartlist_t *sl, int idx); void smartlist_insert(smartlist_t *sl, int idx, void *val); void smartlist_sort(smartlist_t *sl, int (*compare)(const void **a, const void **b)); +void *smartlist_get_most_frequent(const smartlist_t *sl, + int (*compare)(const void **a, const void **b)); void smartlist_uniq(smartlist_t *sl, int (*compare)(const void **a, const void **b), void (*free_fn)(void *elt)); + void smartlist_sort_strings(smartlist_t *sl); void smartlist_sort_digests(smartlist_t *sl); +void smartlist_sort_digests256(smartlist_t *sl); + +char *smartlist_get_most_frequent_string(smartlist_t *sl); +char *smartlist_get_most_frequent_digest256(smartlist_t *sl); + void smartlist_uniq_strings(smartlist_t *sl); void smartlist_uniq_digests(smartlist_t *sl); +void smartlist_uniq_digests256(smartlist_t *sl); void *smartlist_bsearch(smartlist_t *sl, const void *key, int (*compare)(const void *key, const void **member)) ATTR_PURE; @@ -109,11 +120,18 @@ int smartlist_bsearch_idx(const smartlist_t *sl, const void *key, void smartlist_pqueue_add(smartlist_t *sl, int (*compare)(const void *a, const void *b), + int idx_field_offset, void *item); void *smartlist_pqueue_pop(smartlist_t *sl, - int (*compare)(const void *a, const void *b)); + int (*compare)(const void *a, const void *b), + int idx_field_offset); +void smartlist_pqueue_remove(smartlist_t *sl, + int (*compare)(const void *a, const void *b), + int idx_field_offset, + void *item); void smartlist_pqueue_assert_ok(smartlist_t *sl, - int (*compare)(const void *a, const void *b)); + int (*compare)(const void *a, const void *b), + int idx_field_offset); #define SPLIT_SKIP_SPACE 0x01 #define SPLIT_IGNORE_BLANK 0x02 @@ -579,9 +597,9 @@ bitarray_is_set(bitarray_t *b, int bit) /** A set of digests, implemented as a Bloom filter. */ typedef struct { - int mask; /* One less than the number of bits in <b>ba</b>; always one less + int mask; /**< One less than the number of bits in <b>ba</b>; always one less * than a power of two. */ - bitarray_t *ba; /* A bit array to implement the Bloom filter. */ + bitarray_t *ba; /**< A bit array to implement the Bloom filter. */ } digestset_t; #define BIT(n) ((n) & set->mask) @@ -627,6 +645,7 @@ void digestset_free(digestset_t* set); int find_nth_int(int *array, int n_elements, int nth); time_t find_nth_time(time_t *array, int n_elements, int nth); double find_nth_double(double *array, int n_elements, int nth); +int32_t find_nth_int32(int32_t *array, int n_elements, int nth); uint32_t find_nth_uint32(uint32_t *array, int n_elements, int nth); long find_nth_long(long *array, int n_elements, int nth); static INLINE int @@ -649,6 +668,11 @@ median_uint32(uint32_t *array, int n_elements) { return find_nth_uint32(array, n_elements, (n_elements-1)/2); } +static INLINE int32_t +median_int32(int32_t *array, int n_elements) +{ + return find_nth_int32(array, n_elements, (n_elements-1)/2); +} static INLINE long median_long(long *array, int n_elements) { diff --git a/src/common/crypto.c b/src/common/crypto.c index 48c8dea08f..8d17a3daee 100644 --- a/src/common/crypto.c +++ b/src/common/crypto.c @@ -27,6 +27,7 @@ #include <openssl/rsa.h> #include <openssl/pem.h> #include <openssl/evp.h> +#include <openssl/engine.h> #include <openssl/rand.h> #include <openssl/opensslv.h> #include <openssl/bn.h> @@ -49,9 +50,9 @@ #define CRYPTO_PRIVATE #include "crypto.h" -#include "log.h" +#include "../common/torlog.h" #include "aes.h" -#include "util.h" +#include "../common/util.h" #include "container.h" #include "compat.h" @@ -61,6 +62,35 @@ #include <openssl/engine.h> +#ifdef ANDROID +/* Android's OpenSSL seems to have removed all of its Engine support. */ +#define DISABLE_ENGINES +#endif + +#if OPENSSL_VERSION_NUMBER < 0x00908000l +/** @{ */ +/** On OpenSSL versions before 0.9.8, there is no working SHA256 + * implementation, so we use Tom St Denis's nice speedy one, slightly adapted + * to our needs. These macros make it usable by us. */ +#define SHA256_CTX sha256_state +#define SHA256_Init sha256_init +#define SHA256_Update sha256_process +#define LTC_ARGCHK(x) tor_assert(x) +/** @} */ +#include "sha256.c" +#define SHA256_Final(a,b) sha256_done(b,a) + +static unsigned char * +SHA256(const unsigned char *m, size_t len, unsigned char *d) +{ + SHA256_CTX ctx; + SHA256_Init(&ctx); + SHA256_Update(&ctx, m, len); + SHA256_Final(d, &ctx); + return d; +} +#endif + /** Macro: is k a valid RSA public or private key? */ #define PUBLIC_KEY_OK(k) ((k) && (k)->key && (k)->key->n) /** Macro: is k a valid RSA private key? */ @@ -76,25 +106,26 @@ static int _n_openssl_mutexes = 0; /** A public key, or a public/private key-pair. */ struct crypto_pk_env_t { - int refs; /* reference counting so we don't have to copy keys */ - RSA *key; + int refs; /**< reference count, so we don't have to copy keys */ + RSA *key; /**< The key itself */ }; /** Key and stream information for a stream cipher. */ struct crypto_cipher_env_t { - char key[CIPHER_KEY_LEN]; - aes_cnt_cipher_t *cipher; + char key[CIPHER_KEY_LEN]; /**< The raw key. */ + aes_cnt_cipher_t *cipher; /**< The key in format usable for counter-mode AES + * encryption */ }; /** A structure to hold the first half (x, g^x) of a Diffie-Hellman handshake * while we're waiting for the second.*/ struct crypto_dh_env_t { - DH *dh; + DH *dh; /**< The openssl DH object */ }; static int setup_openssl_threading(void); -static int tor_check_dh_key(BIGNUM *bn); +static int tor_check_dh_key(int severity, BIGNUM *bn); /** Return the number of bytes added by padding method <b>padding</b>. */ @@ -151,6 +182,7 @@ crypto_log_errors(int severity, const char *doing) } } +#ifndef DISABLE_ENGINES /** Log any OpenSSL engines we're using at NOTICE. */ static void log_engine(const char *fn, ENGINE *e) @@ -165,37 +197,82 @@ log_engine(const char *fn, ENGINE *e) log(LOG_INFO, LD_CRYPTO, "Using default implementation for %s", fn); } } +#endif + +#ifndef DISABLE_ENGINES +/** Try to load an engine in a shared library via fully qualified path. + */ +static ENGINE * +try_load_engine(const char *path, const char *engine) +{ + ENGINE *e = ENGINE_by_id("dynamic"); + if (e) { + if (!ENGINE_ctrl_cmd_string(e, "ID", engine, 0) || + !ENGINE_ctrl_cmd_string(e, "DIR_LOAD", "2", 0) || + !ENGINE_ctrl_cmd_string(e, "DIR_ADD", path, 0) || + !ENGINE_ctrl_cmd_string(e, "LOAD", NULL, 0)) { + ENGINE_free(e); + e = NULL; + } + } + return e; +} +#endif /** Initialize the crypto library. Return 0 on success, -1 on failure. */ int -crypto_global_init(int useAccel) +crypto_global_init(int useAccel, const char *accelName, const char *accelDir) { if (!_crypto_global_initialized) { ERR_load_crypto_strings(); OpenSSL_add_all_algorithms(); _crypto_global_initialized = 1; setup_openssl_threading(); - /* XXX the below is a bug, since we can't know if we're supposed - * to be using hardware acceleration or not. we should arrange - * for this function to be called before init_keys. But make it - * not complain loudly, at least until we make acceleration work. */ - if (useAccel < 0) { - log_info(LD_CRYPTO, "Initializing OpenSSL via tor_tls_init()."); - } if (useAccel > 0) { +#ifdef DISABLE_ENGINES + (void)accelName; + (void)accelDir; + log_warn(LD_CRYPTO, "No OpenSSL hardware acceleration support enabled."); +#else + ENGINE *e = NULL; + log_info(LD_CRYPTO, "Initializing OpenSSL engine support."); ENGINE_load_builtin_engines(); - if (!ENGINE_register_all_complete()) - return -1; - - /* XXXX make sure this isn't leaking. */ + ENGINE_register_all_complete(); + + if (accelName) { + if (accelDir) { + log_info(LD_CRYPTO, "Trying to load dynamic OpenSSL engine \"%s\"" + " via path \"%s\".", accelName, accelDir); + e = try_load_engine(accelName, accelDir); + } else { + log_info(LD_CRYPTO, "Initializing dynamic OpenSSL engine \"%s\"" + " acceleration support.", accelName); + e = ENGINE_by_id(accelName); + } + if (!e) { + log_warn(LD_CRYPTO, "Unable to load dynamic OpenSSL engine \"%s\".", + accelName); + } else { + log_info(LD_CRYPTO, "Loaded dynamic OpenSSL engine \"%s\".", + accelName); + } + } + if (e) { + log_info(LD_CRYPTO, "Loaded OpenSSL hardware acceleration engine," + " setting default ciphers."); + ENGINE_set_default(e, ENGINE_METHOD_ALL); + } log_engine("RSA", ENGINE_get_default_RSA()); log_engine("DH", ENGINE_get_default_DH()); log_engine("RAND", ENGINE_get_default_RAND()); log_engine("SHA1", ENGINE_get_digest_engine(NID_sha1)); log_engine("3DES", ENGINE_get_cipher_engine(NID_des_ede3_ecb)); log_engine("AES", ENGINE_get_cipher_engine(NID_aes_128_ecb)); +#endif + } else { + log_info(LD_CRYPTO, "NOT using OpenSSL engine support."); } return crypto_seed_rng(1); } @@ -217,7 +294,11 @@ crypto_global_cleanup(void) EVP_cleanup(); ERR_remove_state(0); ERR_free_strings(); + +#ifndef DISABLE_ENGINES ENGINE_cleanup(); +#endif + CONF_modules_unload(1); CRYPTO_cleanup_all_ex_data(); #ifdef TOR_IS_MULTITHREADED @@ -248,18 +329,8 @@ _crypto_new_pk_env_rsa(RSA *rsa) return env; } -/** used by tortls.c: wrap the RSA from an evp_pkey in a crypto_pk_env_t. - * returns NULL if this isn't an RSA key. */ -crypto_pk_env_t * -_crypto_new_pk_env_evp_pkey(EVP_PKEY *pkey) -{ - RSA *rsa; - if (!(rsa = EVP_PKEY_get1_RSA(pkey))) - return NULL; - return _crypto_new_pk_env_rsa(rsa); -} - -/** Helper, used by tor-checkkey.c. Return the RSA from a crypto_pk_env_t. */ +/** Helper, used by tor-checkkey.c and tor-gencert.c. Return the RSA from a + * crypto_pk_env_t. */ RSA * _crypto_pk_env_get_rsa(crypto_pk_env_t *env) { @@ -311,7 +382,7 @@ crypto_new_pk_env(void) RSA *rsa; rsa = RSA_new(); - if (!rsa) return NULL; + tor_assert(rsa); return _crypto_new_pk_env_rsa(rsa); } @@ -321,10 +392,12 @@ crypto_new_pk_env(void) void crypto_free_pk_env(crypto_pk_env_t *env) { - tor_assert(env); + if (!env) + return; if (--env->refs > 0) return; + tor_assert(env->refs == 0); if (env->key) RSA_free(env->key); @@ -347,10 +420,7 @@ crypto_create_init_cipher(const char *key, int encrypt_mode) return NULL; } - if (crypto_cipher_set_key(crypto, key)) { - crypto_log_errors(LOG_WARN, "setting symmetric key"); - goto error; - } + crypto_cipher_set_key(crypto, key); if (encrypt_mode) r = crypto_cipher_encrypt_init_cipher(crypto); @@ -384,7 +454,8 @@ crypto_new_cipher_env(void) void crypto_free_cipher_env(crypto_cipher_env_t *env) { - tor_assert(env); + if (!env) + return; tor_assert(env->cipher); aes_free_cipher(env->cipher); @@ -394,11 +465,11 @@ crypto_free_cipher_env(crypto_cipher_env_t *env) /* public key crypto */ -/** Generate a new public/private keypair in <b>env</b>. Return 0 on - * success, -1 on failure. +/** Generate a <b>bits</b>-bit new public/private keypair in <b>env</b>. + * Return 0 on success, -1 on failure. */ int -crypto_pk_generate_key(crypto_pk_env_t *env) +crypto_pk_generate_key_with_bits(crypto_pk_env_t *env, int bits) { tor_assert(env); @@ -406,7 +477,7 @@ crypto_pk_generate_key(crypto_pk_env_t *env) RSA_free(env->key); #if OPENSSL_VERSION_NUMBER < 0x00908000l /* In OpenSSL 0.9.7, RSA_generate_key is all we have. */ - env->key = RSA_generate_key(PK_BYTES*8,65537, NULL, NULL); + env->key = RSA_generate_key(bits, 65537, NULL, NULL); #else /* In OpenSSL 0.9.8, RSA_generate_key is deprecated. */ { @@ -419,7 +490,7 @@ crypto_pk_generate_key(crypto_pk_env_t *env) r = RSA_new(); if (!r) goto done; - if (RSA_generate_key_ex(r, PK_BYTES*8, e, NULL) == -1) + if (RSA_generate_key_ex(r, bits, e, NULL) == -1) goto done; env->key = r; @@ -452,10 +523,12 @@ crypto_pk_read_private_key_from_string(crypto_pk_env_t *env, tor_assert(env); tor_assert(s); - tor_assert(len < INT_MAX && len < SIZE_T_CEILING); + tor_assert(len < INT_MAX && len < SSIZE_T_CEILING); /* Create a read-only memory BIO, backed by the string 's' */ b = BIO_new_mem_buf((char*)s, (int)len); + if (!b) + return -1; if (env->key) RSA_free(env->key); @@ -516,6 +589,8 @@ crypto_pk_write_key_to_string_impl(crypto_pk_env_t *env, char **dest, tor_assert(dest); b = BIO_new(BIO_s_mem()); /* Create a memory BIO */ + if (!b) + return -1; /* Now you can treat b as if it were a file. Just use the * PEM_*_bio_* functions instead of the non-bio variants. @@ -583,6 +658,8 @@ crypto_pk_read_public_key_from_string(crypto_pk_env_t *env, const char *src, tor_assert(len<INT_MAX); b = BIO_new(BIO_s_mem()); /* Create a memory BIO */ + if (!b) + return -1; BIO_write(b, src, (int)len); @@ -705,14 +782,25 @@ crypto_pk_env_t * crypto_pk_copy_full(crypto_pk_env_t *env) { RSA *new_key; + int privatekey = 0; tor_assert(env); tor_assert(env->key); if (PRIVATE_KEY_OK(env)) { new_key = RSAPrivateKey_dup(env->key); + privatekey = 1; } else { new_key = RSAPublicKey_dup(env->key); } + if (!new_key) { + log_err(LD_CRYPTO, "Unable to duplicate a %s key: openssl failed.", + privatekey?"private":"public"); + crypto_log_errors(LOG_ERR, + privatekey ? "Duplicating a private key" : + "Duplicating a public key"); + tor_fragile_assert(); + return NULL; + } return _crypto_new_pk_env_rsa(new_key); } @@ -1213,19 +1301,14 @@ crypto_cipher_generate_key(crypto_cipher_env_t *env) /** Set the symmetric key for the cipher in <b>env</b> to the first * CIPHER_KEY_LEN bytes of <b>key</b>. Does not initialize the cipher. - * Return 0 on success, -1 on failure. */ -int +void crypto_cipher_set_key(crypto_cipher_env_t *env, const char *key) { tor_assert(env); tor_assert(key); - if (!env->key) - return -1; - memcpy(env->key, key, CIPHER_KEY_LEN); - return 0; } /** Generate an initialization vector for our AES-CTR cipher; store it @@ -1390,7 +1473,7 @@ crypto_cipher_decrypt_with_iv(crypto_cipher_env_t *cipher, /* SHA-1 */ -/** Compute the SHA1 digest of <b>len</b> bytes in data stored in +/** Compute the SHA1 digest of the <b>len</b> bytes on data stored in * <b>m</b>. Write the DIGEST_LEN byte result into <b>digest</b>. * Return 0 on success, -1 on failure. */ @@ -1402,19 +1485,97 @@ crypto_digest(char *digest, const char *m, size_t len) return (SHA1((const unsigned char*)m,len,(unsigned char*)digest) == NULL); } +/** Compute a 256-bit digest of <b>len</b> bytes in data stored in <b>m</b>, + * using the algorithm <b>algorithm</b>. Write the DIGEST_LEN256-byte result + * into <b>digest</b>. Return 0 on success, -1 on failure. */ +int +crypto_digest256(char *digest, const char *m, size_t len, + digest_algorithm_t algorithm) +{ + tor_assert(m); + tor_assert(digest); + tor_assert(algorithm == DIGEST_SHA256); + return (SHA256((const unsigned char*)m,len,(unsigned char*)digest) == NULL); +} + +/** Set the digests_t in <b>ds_out</b> to contain every digest on the + * <b>len</b> bytes in <b>m</b> that we know how to compute. Return 0 on + * success, -1 on failure. */ +int +crypto_digest_all(digests_t *ds_out, const char *m, size_t len) +{ + digest_algorithm_t i; + tor_assert(ds_out); + memset(ds_out, 0, sizeof(*ds_out)); + if (crypto_digest(ds_out->d[DIGEST_SHA1], m, len) < 0) + return -1; + for (i = DIGEST_SHA256; i < N_DIGEST_ALGORITHMS; ++i) { + if (crypto_digest256(ds_out->d[i], m, len, i) < 0) + return -1; + } + return 0; +} + +/** Return the name of an algorithm, as used in directory documents. */ +const char * +crypto_digest_algorithm_get_name(digest_algorithm_t alg) +{ + switch (alg) { + case DIGEST_SHA1: + return "sha1"; + case DIGEST_SHA256: + return "sha256"; + default: + tor_fragile_assert(); + return "??unknown_digest??"; + } +} + +/** Given the name of a digest algorithm, return its integer value, or -1 if + * the name is not recognized. */ +int +crypto_digest_algorithm_parse_name(const char *name) +{ + if (!strcmp(name, "sha1")) + return DIGEST_SHA1; + else if (!strcmp(name, "sha256")) + return DIGEST_SHA256; + else + return -1; +} + /** Intermediate information about the digest of a stream of data. */ struct crypto_digest_env_t { - SHA_CTX d; + union { + SHA_CTX sha1; /**< state for SHA1 */ + SHA256_CTX sha2; /**< state for SHA256 */ + } d; /**< State for the digest we're using. Only one member of the + * union is usable, depending on the value of <b>algorithm</b>. */ + digest_algorithm_t algorithm : 8; /**< Which algorithm is in use? */ }; -/** Allocate and return a new digest object. +/** Allocate and return a new digest object to compute SHA1 digests. */ crypto_digest_env_t * crypto_new_digest_env(void) { crypto_digest_env_t *r; r = tor_malloc(sizeof(crypto_digest_env_t)); - SHA1_Init(&r->d); + SHA1_Init(&r->d.sha1); + r->algorithm = DIGEST_SHA1; + return r; +} + +/** Allocate and return a new digest object to compute 256-bit digests + * using <b>algorithm</b>. */ +crypto_digest_env_t * +crypto_new_digest256_env(digest_algorithm_t algorithm) +{ + crypto_digest_env_t *r; + tor_assert(algorithm == DIGEST_SHA256); + r = tor_malloc(sizeof(crypto_digest_env_t)); + SHA256_Init(&r->d.sha2); + r->algorithm = algorithm; return r; } @@ -1423,6 +1584,8 @@ crypto_new_digest_env(void) void crypto_free_digest_env(crypto_digest_env_t *digest) { + if (!digest) + return; memset(digest, 0, sizeof(crypto_digest_env_t)); tor_free(digest); } @@ -1435,30 +1598,51 @@ crypto_digest_add_bytes(crypto_digest_env_t *digest, const char *data, { tor_assert(digest); tor_assert(data); - /* Using the SHA1_*() calls directly means we don't support doing - * SHA1 in hardware. But so far the delay of getting the question + /* Using the SHA*_*() calls directly means we don't support doing + * SHA in hardware. But so far the delay of getting the question * to the hardware, and hearing the answer, is likely higher than * just doing it ourselves. Hashes are fast. */ - SHA1_Update(&digest->d, (void*)data, len); + switch (digest->algorithm) { + case DIGEST_SHA1: + SHA1_Update(&digest->d.sha1, (void*)data, len); + break; + case DIGEST_SHA256: + SHA256_Update(&digest->d.sha2, (void*)data, len); + break; + default: + tor_fragile_assert(); + break; + } } /** Compute the hash of the data that has been passed to the digest * object; write the first out_len bytes of the result to <b>out</b>. - * <b>out_len</b> must be \<= DIGEST_LEN. + * <b>out_len</b> must be \<= DIGEST256_LEN. */ void crypto_digest_get_digest(crypto_digest_env_t *digest, char *out, size_t out_len) { - unsigned char r[DIGEST_LEN]; - SHA_CTX tmpctx; + unsigned char r[DIGEST256_LEN]; + crypto_digest_env_t tmpenv; tor_assert(digest); tor_assert(out); - tor_assert(out_len <= DIGEST_LEN); - /* memcpy into a temporary ctx, since SHA1_Final clears the context */ - memcpy(&tmpctx, &digest->d, sizeof(SHA_CTX)); - SHA1_Final(r, &tmpctx); + /* memcpy into a temporary ctx, since SHA*_Final clears the context */ + memcpy(&tmpenv, digest, sizeof(crypto_digest_env_t)); + switch (digest->algorithm) { + case DIGEST_SHA1: + tor_assert(out_len <= DIGEST_LEN); + SHA1_Final(r, &tmpenv.d.sha1); + break; + case DIGEST_SHA256: + tor_assert(out_len <= DIGEST256_LEN); + SHA256_Final(r, &tmpenv.d.sha2); + break; + default: + tor_fragile_assert(); + break; + } memcpy(out, r, out_len); memset(r, 0, sizeof(r)); } @@ -1557,6 +1741,10 @@ init_dh_param(void) dh_param_g = g; } +/** Number of bits to use when choosing the x or y value in a Diffie-Hellman + * handshake. Since we exponentiate by this value, choosing a smaller one + * lets our handhake go faster. + */ #define DH_PRIVATE_KEY_BITS 320 /** Allocate and return a new DH object for a key exchange. @@ -1616,7 +1804,7 @@ crypto_dh_generate_public(crypto_dh_env_t *dh) crypto_log_errors(LOG_WARN, "generating DH key"); return -1; } - if (tor_check_dh_key(dh->dh->pub_key)<0) { + if (tor_check_dh_key(LOG_WARN, dh->dh->pub_key)<0) { log_warn(LD_CRYPTO, "Weird! Our own DH key was invalid. I guess once-in-" "the-universe chances really do happen. Trying again."); /* Free and clear the keys, so OpenSSL will actually try again. */ @@ -1663,7 +1851,7 @@ crypto_dh_get_public(crypto_dh_env_t *dh, char *pubkey, size_t pubkey_len) * See http://www.cl.cam.ac.uk/ftp/users/rja14/psandqs.ps.gz for some tips. */ static int -tor_check_dh_key(BIGNUM *bn) +tor_check_dh_key(int severity, BIGNUM *bn) { BIGNUM *x; char *s; @@ -1674,13 +1862,13 @@ tor_check_dh_key(BIGNUM *bn) init_dh_param(); BN_set_word(x, 1); if (BN_cmp(bn,x)<=0) { - log_warn(LD_CRYPTO, "DH key must be at least 2."); + log_fn(severity, LD_CRYPTO, "DH key must be at least 2."); goto err; } BN_copy(x,dh_param_p); BN_sub_word(x, 1); if (BN_cmp(bn,x)>=0) { - log_warn(LD_CRYPTO, "DH key must be at most p-2."); + log_fn(severity, LD_CRYPTO, "DH key must be at most p-2."); goto err; } BN_free(x); @@ -1688,7 +1876,7 @@ tor_check_dh_key(BIGNUM *bn) err: BN_free(x); s = BN_bn2hex(bn); - log_warn(LD_CRYPTO, "Rejecting insecure DH key [%s]", s); + log_fn(severity, LD_CRYPTO, "Rejecting insecure DH key [%s]", s); OPENSSL_free(s); return -1; } @@ -1706,7 +1894,7 @@ tor_check_dh_key(BIGNUM *bn) * where || is concatenation.) */ ssize_t -crypto_dh_compute_secret(crypto_dh_env_t *dh, +crypto_dh_compute_secret(int severity, crypto_dh_env_t *dh, const char *pubkey, size_t pubkey_len, char *secret_out, size_t secret_bytes_out) { @@ -1721,9 +1909,9 @@ crypto_dh_compute_secret(crypto_dh_env_t *dh, if (!(pubkey_bn = BN_bin2bn((const unsigned char*)pubkey, (int)pubkey_len, NULL))) goto error; - if (tor_check_dh_key(pubkey_bn)<0) { + if (tor_check_dh_key(severity, pubkey_bn)<0) { /* Check for invalid public keys. */ - log_warn(LD_CRYPTO,"Rejected invalid g^x"); + log_fn(severity, LD_CRYPTO,"Rejected invalid g^x"); goto error; } secret_tmp_len = crypto_dh_get_bytes(dh); @@ -1799,7 +1987,8 @@ crypto_expand_key_material(const char *key_in, size_t key_in_len, void crypto_dh_free(crypto_dh_env_t *dh) { - tor_assert(dh); + if (!dh) + return; tor_assert(dh->dh); DH_free(dh->dh); tor_free(dh); @@ -1807,15 +1996,22 @@ crypto_dh_free(crypto_dh_env_t *dh) /* random numbers */ -/* This is how much entropy OpenSSL likes to add right now, so maybe it will +/** How many bytes of entropy we add at once. + * + * This is how much entropy OpenSSL likes to add right now, so maybe it will * work for us too. */ #define ADD_ENTROPY 32 -/* Use RAND_poll if OpenSSL is 0.9.6 release or later. (The "f" means - "release".) */ +/** True iff we should use OpenSSL's RAND_poll function to add entropy to its + * pool. + * + * Use RAND_poll if OpenSSL is 0.9.6 release or later. (The "f" means + *"release".) */ #define HAVE_RAND_POLL (OPENSSL_VERSION_NUMBER >= 0x0090600fl) -/* Versions of OpenSSL prior to 0.9.7k and 0.9.8c had a bug where RAND_poll +/** True iff it's safe to use RAND_poll after setup. + * + * Versions of OpenSSL prior to 0.9.7k and 0.9.8c had a bug where RAND_poll * would allocate an fd_set on the stack, open a new file, and try to FD_SET * that fd without checking whether it fit in the fd_set. Thus, if the * system has not just been started up, it is unsafe to call */ @@ -1824,6 +2020,15 @@ crypto_dh_free(crypto_dh_env_t *dh) OPENSSL_VERSION_NUMBER <= 0x00907fffl) || \ (OPENSSL_VERSION_NUMBER >= 0x0090803fl)) +/** Set the seed of the weak RNG to a random value. */ +static void +seed_weak_rng(void) +{ + unsigned seed; + crypto_rand((void*)&seed, sizeof(seed)); + tor_init_weak_random(seed); +} + /** Seed OpenSSL's random number generator with bytes from the operating * system. <b>startup</b> should be true iff we have just started Tor and * have not yet allocated a bunch of fds. Return 0 on success, -1 on failure. @@ -1831,14 +2036,15 @@ crypto_dh_free(crypto_dh_env_t *dh) int crypto_seed_rng(int startup) { - char buf[ADD_ENTROPY]; int rand_poll_status = 0; /* local variables */ #ifdef MS_WINDOWS + unsigned char buf[ADD_ENTROPY]; static int provider_set = 0; static HCRYPTPROV provider; #else + char buf[ADD_ENTROPY]; static const char *filenames[] = { "/dev/srandom", "/dev/urandom", "/dev/random", NULL }; @@ -1874,6 +2080,7 @@ crypto_seed_rng(int startup) } RAND_seed(buf, sizeof(buf)); memset(buf, 0, sizeof(buf)); + seed_weak_rng(); return 0; #else for (i = 0; filenames[i]; ++i) { @@ -1890,6 +2097,7 @@ crypto_seed_rng(int startup) } RAND_seed(buf, (int)sizeof(buf)); memset(buf, 0, sizeof(buf)); + seed_weak_rng(); return 0; } @@ -1957,6 +2165,26 @@ crypto_rand_uint64(uint64_t max) } } +/** Return a pseudorandom double d, chosen uniformly from the range + * 0.0 <= d < 1.0. + */ +double +crypto_rand_double(void) +{ + /* We just use an unsigned int here; we don't really care about getting + * more than 32 bits of resolution */ + unsigned int uint; + crypto_rand((char*)&uint, sizeof(uint)); +#if SIZEOF_INT == 4 +#define UINT_MAX_AS_DOUBLE 4294967296.0 +#elif SIZEOF_INT == 8 +#define UINT_MAX_AS_DOUBLE 1.8446744073709552e+19 +#else +#error SIZEOF_INT is neither 4 nor 8 +#endif + return ((double)uint) / UINT_MAX_AS_DOUBLE; +} + /** Generate and return a new random hostname starting with <b>prefix</b>, * ending with <b>suffix</b>, and containing no less than * <b>min_rand_len</b> and no more than <b>max_rand_len</b> random base32 @@ -2046,9 +2274,12 @@ base64_encode(char *dest, size_t destlen, const char *src, size_t srclen) return ret; } +/** @{ */ +/** Special values used for the base64_decode_table */ #define X 255 #define SP 64 #define PAD 65 +/** @} */ /** Internal table mapping byte values to what they represent in base64. * Numbers 0..63 are 6-bit integers. SPs are spaces, and should be * skipped. Xs are invalid and must not appear in base64. PAD indicates @@ -2217,15 +2448,54 @@ digest_from_base64(char *digest, const char *d64) #endif } +/** Base-64 encode DIGEST256_LINE bytes from <b>digest</b>, remove the + * trailing = and newline characters, and store the nul-terminated result in + * the first BASE64_DIGEST256_LEN+1 bytes of <b>d64</b>. */ +int +digest256_to_base64(char *d64, const char *digest) +{ + char buf[256]; + base64_encode(buf, sizeof(buf), digest, DIGEST256_LEN); + buf[BASE64_DIGEST256_LEN] = '\0'; + memcpy(d64, buf, BASE64_DIGEST256_LEN+1); + return 0; +} + +/** Given a base-64 encoded, nul-terminated digest in <b>d64</b> (without + * trailing newline or = characters), decode it and store the result in the + * first DIGEST256_LEN bytes at <b>digest</b>. */ +int +digest256_from_base64(char *digest, const char *d64) +{ +#ifdef USE_OPENSSL_BASE64 + char buf_in[BASE64_DIGEST256_LEN+3]; + char buf[256]; + if (strlen(d64) != BASE64_DIGEST256_LEN) + return -1; + memcpy(buf_in, d64, BASE64_DIGEST256_LEN); + memcpy(buf_in+BASE64_DIGEST256_LEN, "=\n\0", 3); + if (base64_decode(buf, sizeof(buf), buf_in, strlen(buf_in)) != DIGEST256_LEN) + return -1; + memcpy(digest, buf, DIGEST256_LEN); + return 0; +#else + if (base64_decode(digest, DIGEST256_LEN, d64, strlen(d64)) == DIGEST256_LEN) + return 0; + else + return -1; +#endif +} + /** Implements base32 encoding as in rfc3548. Limitation: Requires * that srclen*8 is a multiple of 5. */ void base32_encode(char *dest, size_t destlen, const char *src, size_t srclen) { - unsigned int i, bit, v, u; - size_t nbits = srclen * 8; + unsigned int i, v, u; + size_t nbits = srclen * 8, bit; + tor_assert(srclen < SIZE_T_CEILING/8); tor_assert((nbits%5) == 0); /* We need an even multiple of 5 bits. */ tor_assert((nbits/5)+1 <= destlen); /* We need enough space. */ tor_assert(destlen < SIZE_T_CEILING); @@ -2249,11 +2519,12 @@ base32_decode(char *dest, size_t destlen, const char *src, size_t srclen) { /* XXXX we might want to rewrite this along the lines of base64_decode, if * it ever shows up in the profile. */ - unsigned int i, j, bit; - size_t nbits; + unsigned int i; + size_t nbits, j, bit; char *tmp; nbits = srclen * 5; + tor_assert(srclen < SIZE_T_CEILING / 5); tor_assert((nbits%8) == 0); /* We need an even multiple of 8 bits. */ tor_assert((nbits/8) <= destlen); /* We need enough space. */ tor_assert(destlen < SIZE_T_CEILING); @@ -2412,6 +2683,7 @@ _openssl_dynlock_destroy_cb(struct CRYPTO_dynlock_value *v, tor_free(v); } +/** @{ */ /** Helper: Construct mutexes, and set callbacks to help OpenSSL handle being * multithreaded. */ static int @@ -2437,4 +2709,5 @@ setup_openssl_threading(void) return 0; } #endif +/** @} */ diff --git a/src/common/crypto.h b/src/common/crypto.h index 576c03dc30..05185f3f18 100644 --- a/src/common/crypto.h +++ b/src/common/crypto.h @@ -18,6 +18,9 @@ /** Length of the output of our message digest. */ #define DIGEST_LEN 20 +/** Length of the output of our second (improved) message digests. (For now + * this is just sha256, but any it can be any other 256-byte digest). */ +#define DIGEST256_LEN 32 /** Length of our symmetric cipher's keys. */ #define CIPHER_KEY_LEN 16 /** Length of our symmetric cipher's IV. */ @@ -27,9 +30,12 @@ /** Length of our DH keys. */ #define DH_BYTES (1024/8) -/** Length of a message digest when encoded in base64 with trailing = signs - * removed. */ +/** Length of a sha1 message digest when encoded in base64 with trailing = + * signs removed. */ #define BASE64_DIGEST_LEN 27 +/** Length of a sha256 message digest when encoded in base64 with trailing = + * signs removed. */ +#define BASE64_DIGEST256_LEN 43 /** Constants used to indicate no padding for public-key encryption */ #define PK_NO_PADDING 60000 @@ -48,6 +54,26 @@ #define FINGERPRINT_LEN 49 /** Length of hex encoding of SHA1 digest, not including final NUL. */ #define HEX_DIGEST_LEN 40 +/** Length of hex encoding of SHA256 digest, not including final NUL. */ +#define HEX_DIGEST256_LEN 64 + +typedef enum { + DIGEST_SHA1 = 0, + DIGEST_SHA256 = 1, +} digest_algorithm_t; +#define N_DIGEST_ALGORITHMS (DIGEST_SHA256+1) + +/** A set of all the digests we know how to compute, taken on a single + * string. Any digests that are shorter than 256 bits are right-padded + * with 0 bits. + * + * Note that this representation wastes 12 bytes for the SHA1 case, so + * don't use it for anything where we need to allocate a whole bunch at + * once. + **/ +typedef struct { + char d[N_DIGEST_ALGORITHMS][DIGEST256_LEN]; +} digests_t; typedef struct crypto_pk_env_t crypto_pk_env_t; typedef struct crypto_cipher_env_t crypto_cipher_env_t; @@ -55,7 +81,9 @@ typedef struct crypto_digest_env_t crypto_digest_env_t; typedef struct crypto_dh_env_t crypto_dh_env_t; /* global state */ -int crypto_global_init(int hardwareAccel); +int crypto_global_init(int hardwareAccel, + const char *accelName, + const char *accelPath); void crypto_thread_cleanup(void); int crypto_global_cleanup(void); @@ -71,7 +99,9 @@ crypto_cipher_env_t *crypto_new_cipher_env(void); void crypto_free_cipher_env(crypto_cipher_env_t *env); /* public key crypto */ -int crypto_pk_generate_key(crypto_pk_env_t *env); +int crypto_pk_generate_key_with_bits(crypto_pk_env_t *env, int bits); +#define crypto_pk_generate_key(env) \ + crypto_pk_generate_key_with_bits((env), (PK_BYTES*8)) int crypto_pk_read_private_key_from_filename(crypto_pk_env_t *env, const char *keyfile); @@ -123,7 +153,7 @@ int crypto_pk_check_fingerprint_syntax(const char *s); /* symmetric crypto */ int crypto_cipher_generate_key(crypto_cipher_env_t *env); -int crypto_cipher_set_key(crypto_cipher_env_t *env, const char *key); +void crypto_cipher_set_key(crypto_cipher_env_t *env, const char *key); void crypto_cipher_generate_iv(char *iv_out); int crypto_cipher_set_iv(crypto_cipher_env_t *env, const char *iv); const char *crypto_cipher_get_key(crypto_cipher_env_t *env); @@ -143,9 +173,15 @@ int crypto_cipher_decrypt_with_iv(crypto_cipher_env_t *env, char *to, size_t tolen, const char *from, size_t fromlen); -/* SHA-1 */ +/* SHA-1 and other digests. */ int crypto_digest(char *digest, const char *m, size_t len); +int crypto_digest256(char *digest, const char *m, size_t len, + digest_algorithm_t algorithm); +int crypto_digest_all(digests_t *ds_out, const char *m, size_t len); +const char *crypto_digest_algorithm_get_name(digest_algorithm_t alg); +int crypto_digest_algorithm_parse_name(const char *name); crypto_digest_env_t *crypto_new_digest_env(void); +crypto_digest_env_t *crypto_new_digest256_env(digest_algorithm_t algorithm); void crypto_free_digest_env(crypto_digest_env_t *digest); void crypto_digest_add_bytes(crypto_digest_env_t *digest, const char *data, size_t len); @@ -167,7 +203,7 @@ int crypto_dh_get_bytes(crypto_dh_env_t *dh); int crypto_dh_generate_public(crypto_dh_env_t *dh); int crypto_dh_get_public(crypto_dh_env_t *dh, char *pubkey_out, size_t pubkey_out_len); -ssize_t crypto_dh_compute_secret(crypto_dh_env_t *dh, +ssize_t crypto_dh_compute_secret(int severity, crypto_dh_env_t *dh, const char *pubkey, size_t pubkey_len, char *secret_out, size_t secret_out_len); void crypto_dh_free(crypto_dh_env_t *dh); @@ -179,6 +215,7 @@ int crypto_seed_rng(int startup); int crypto_rand(char *to, size_t n); int crypto_rand_int(unsigned int max); uint64_t crypto_rand_uint64(uint64_t max); +double crypto_rand_double(void); char *crypto_random_hostname(int min_rand_len, int max_rand_len, const char *prefix, const char *suffix); @@ -196,6 +233,8 @@ int base32_decode(char *dest, size_t destlen, const char *src, size_t srclen); int digest_to_base64(char *d64, const char *digest); int digest_from_base64(char *digest, const char *d64); +int digest256_to_base64(char *d64, const char *digest); +int digest256_from_base64(char *digest, const char *d64); /** Length of RFC2440-style S2K specifier: the first 8 bytes are a salt, the * 9th describes how much iteration to do. */ @@ -210,7 +249,6 @@ struct evp_pkey_st; struct dh_st; struct rsa_st *_crypto_pk_env_get_rsa(crypto_pk_env_t *env); crypto_pk_env_t *_crypto_new_pk_env_rsa(struct rsa_st *rsa); -crypto_pk_env_t *_crypto_new_pk_env_evp_pkey(struct evp_pkey_st *pkey); struct evp_pkey_st *_crypto_pk_env_get_evp_pkey(crypto_pk_env_t *env, int private); struct dh_st *_crypto_dh_env_get_dh(crypto_dh_env_t *dh); diff --git a/src/common/ht.h b/src/common/ht.h index 033cd89ab0..0850c07092 100644 --- a/src/common/ht.h +++ b/src/common/ht.h @@ -42,6 +42,10 @@ #define HT_SIZE(head) \ ((head)->hth_n_entries) +/* Return memory usage for a hashtable (not counting the entries themselves) */ +#define HT_MEM_USAGE(head) \ + (sizeof(*head) + (head)->hth_table_length * sizeof(void*)) + #define HT_FIND(name, head, elm) name##_HT_FIND((head), (elm)) #define HT_INSERT(name, head, elm) name##_HT_INSERT((head), (elm)) #define HT_REPLACE(name, head, elm) name##_HT_REPLACE((head), (elm)) diff --git a/src/common/log.c b/src/common/log.c index fe93bd7f57..d14563c885 100644 --- a/src/common/log.c +++ b/src/common/log.c @@ -33,13 +33,15 @@ #include "compat.h" #include "util.h" #define LOG_PRIVATE -#include "log.h" +#include "torlog.h" #include "container.h" -#include <event.h> - +/** @{ */ +/** The string we stick at the end of a log message when it is too long, + * and its length. */ #define TRUNCATED_STR "[...truncated]" #define TRUNCATED_STR_LEN 14 +/** @} */ /** Information for a single logfile; only used in log.c */ typedef struct logfile_t { @@ -83,7 +85,7 @@ should_log_function_name(log_domain_mask_t domain, int severity) /* All debugging messages occur in interesting places. */ return 1; case LOG_NOTICE: - case LOG_WARN: + case LOG_WARN: case LOG_ERR: /* We care about places where bugs occur. */ return (domain == LD_BUG); @@ -99,15 +101,31 @@ static int log_mutex_initialized = 0; /** Linked list of logfile_t. */ static logfile_t *logfiles = NULL; +/** Boolean: do we report logging domains? */ +static int log_domains_are_logged = 0; + #ifdef HAVE_SYSLOG_H /** The number of open syslog log handlers that we have. When this reaches 0, * we can close our connection to the syslog facility. */ static int syslog_count = 0; #endif +/** Represents a log message that we are going to send to callback-driven + * loggers once we can do so in a non-reentrant way. */ +typedef struct pending_cb_message_t { + int severity; /**< The severity of the message */ + log_domain_mask_t domain; /**< The domain of the message */ + char *msg; /**< The content of the message */ +} pending_cb_message_t; + +/** Log messages waiting to be replayed onto callback-based logs */ +static smartlist_t *pending_cb_messages = NULL; + +/** Lock the log_mutex to prevent others from changing the logfile_t list */ #define LOCK_LOGS() STMT_BEGIN \ tor_mutex_acquire(&log_mutex); \ STMT_END +/** Unlock the log_mutex */ #define UNLOCK_LOGS() STMT_BEGIN tor_mutex_release(&log_mutex); STMT_END /** What's the lowest log level anybody cares about? Checking this lets us @@ -117,6 +135,9 @@ int _log_global_min_severity = LOG_NOTICE; static void delete_log(logfile_t *victim); static void close_log(logfile_t *victim); +static char *domain_to_string(log_domain_mask_t domain, + char *buf, size_t buflen); + /** Name of the application: used to generate the message we write at the * start of each new log. */ static char *appname = NULL; @@ -208,13 +229,34 @@ format_msg(char *buf, size_t buf_len, size_t n; int r; char *end_of_prefix; + char *buf_end; - assert(buf_len >= 2); /* prevent integer underflow */ + assert(buf_len >= 16); /* prevent integer underflow and general stupidity */ buf_len -= 2; /* subtract 2 characters so we have room for \n\0 */ + buf_end = buf+buf_len; /* point *after* the last char we can write to */ n = _log_prefix(buf, buf_len, severity); end_of_prefix = buf+n; + if (log_domains_are_logged) { + char *cp = buf+n; + if (cp == buf_end) goto format_msg_no_room_for_domains; + *cp++ = '{'; + if (cp == buf_end) goto format_msg_no_room_for_domains; + cp = domain_to_string(domain, cp, (buf+buf_len-cp)); + if (cp == buf_end) goto format_msg_no_room_for_domains; + *cp++ = '}'; + if (cp == buf_end) goto format_msg_no_room_for_domains; + *cp++ = ' '; + if (cp == buf_end) goto format_msg_no_room_for_domains; + end_of_prefix = cp; + n = cp-buf; + format_msg_no_room_for_domains: + /* This will leave end_of_prefix and n unchanged, and thus cause + * whatever log domain string we had written to be clobbered. */ + ; + } + if (funcname && should_log_function_name(domain, severity)) { r = tor_snprintf(buf+n, buf_len-n, "%s(): ", funcname); if (r<0) @@ -263,6 +305,7 @@ logv(int severity, log_domain_mask_t domain, const char *funcname, int formatted = 0; logfile_t *lf; char *end_of_prefix=NULL; + int callbacks_deferred = 0; /* Call assert, not tor_assert, since tor_assert calls log on failure. */ assert(format); @@ -270,6 +313,10 @@ logv(int severity, log_domain_mask_t domain, const char *funcname, * interesting and hard to diagnose effects */ assert(severity >= LOG_ERR && severity <= LOG_DEBUG); LOCK_LOGS(); + + if ((! (domain & LD_NOCB)) && smartlist_len(pending_cb_messages)) + flush_pending_log_callbacks(); + lf = logfiles; while (lf) { if (! (lf->severities->masks[SEVERITY_MASK_IDX(severity)] & domain)) { @@ -280,10 +327,6 @@ logv(int severity, log_domain_mask_t domain, const char *funcname, lf = lf->next; continue; } - if (lf->callback && (domain & LD_NOCB)) { - lf = lf->next; - continue; - } if (lf->seems_dead) { lf = lf->next; continue; @@ -295,6 +338,7 @@ logv(int severity, log_domain_mask_t domain, const char *funcname, &msg_len); formatted = 1; } + if (lf->is_syslog) { #ifdef HAVE_SYSLOG_H char *m = end_of_prefix; @@ -318,7 +362,19 @@ logv(int severity, log_domain_mask_t domain, const char *funcname, lf = lf->next; continue; } else if (lf->callback) { - lf->callback(severity, domain, end_of_prefix); + if (domain & LD_NOCB) { + if (!callbacks_deferred && pending_cb_messages) { + pending_cb_message_t *msg = tor_malloc(sizeof(pending_cb_message_t)); + msg->severity = severity; + msg->domain = domain; + msg->msg = tor_strdup(end_of_prefix); + smartlist_add(pending_cb_messages, msg); + + callbacks_deferred = 1; + } + } else { + lf->callback(severity, domain, end_of_prefix); + } lf = lf->next; continue; } @@ -332,9 +388,12 @@ logv(int severity, log_domain_mask_t domain, const char *funcname, UNLOCK_LOGS(); } -/** Output a message to the log. */ +/** Output a message to the log. It gets logged to all logfiles that + * care about messages with <b>severity</b> in <b>domain</b>. The content + * is formatted printf style basedc on <b>format</b> and extra arguments. + * */ void -_log(int severity, log_domain_mask_t domain, const char *format, ...) +tor_log(int severity, log_domain_mask_t domain, const char *format, ...) { va_list ap; if (severity > _log_global_min_severity) @@ -346,6 +405,9 @@ _log(int severity, log_domain_mask_t domain, const char *format, ...) /** Output a message to the log, prefixed with a function name <b>fn</b>. */ #ifdef __GNUC__ +/** GCC-based implementation of the log_fn backend, used when we have + * variadic macros. All arguments are as for log_fn, except for + * <b>fn</b>, which is the name of the calling functions. */ void _log_fn(int severity, log_domain_mask_t domain, const char *fn, const char *format, ...) @@ -358,6 +420,11 @@ _log_fn(int severity, log_domain_mask_t domain, const char *fn, va_end(ap); } #else +/** @{ */ +/** Variant implementation of log_fn, log_debug, log_info,... for C compilers + * without variadic macros. In this case, the calling function sets + * _log_fn_function_name to the name of the function, then invokes the + * appropriate _log_fn, _log_debug, etc. */ const char *_log_fn_function_name=NULL; void _log_fn(int severity, log_domain_mask_t domain, const char *format, ...) @@ -426,12 +493,15 @@ _log_err(log_domain_mask_t domain, const char *format, ...) va_end(ap); _log_fn_function_name = NULL; } +/** @} */ #endif /** Free all storage held by <b>victim</b>. */ static void log_free(logfile_t *victim) { + if (!victim) + return; tor_free(victim->severities); tor_free(victim->filename); tor_free(victim); @@ -442,9 +512,12 @@ void logs_free_all(void) { logfile_t *victim, *next; + smartlist_t *messages; LOCK_LOGS(); next = logfiles; logfiles = NULL; + messages = pending_cb_messages; + pending_cb_messages = NULL; UNLOCK_LOGS(); while (next) { victim = next; @@ -454,6 +527,12 @@ logs_free_all(void) } tor_free(appname); + SMARTLIST_FOREACH(messages, pending_cb_message_t *, msg, { + tor_free(msg->msg); + tor_free(msg); + }); + smartlist_free(messages); + /* We _could_ destroy the log mutex here, but that would screw up any logs * that happened between here and the end of execution. */ } @@ -539,8 +618,7 @@ add_stream_log_impl(const log_severity_list_t *severity, * to <b>fd</b>. Steals a reference to <b>severity</b>; the caller must * not use it after calling this function. */ void -add_stream_log(const log_severity_list_t *severity, - const char *name, int fd) +add_stream_log(const log_severity_list_t *severity, const char *name, int fd) { LOCK_LOGS(); add_stream_log_impl(severity, name, fd); @@ -555,6 +633,18 @@ init_logging(void) tor_mutex_init(&log_mutex); log_mutex_initialized = 1; } + if (pending_cb_messages == NULL) + pending_cb_messages = smartlist_create(); +} + +/** Set whether we report logging domains as a part of our log messages. + */ +void +logs_set_domain_logging(int enabled) +{ + LOCK_LOGS(); + log_domains_are_logged = enabled; + UNLOCK_LOGS(); } /** Add a log handler to receive messages during startup (before the real @@ -613,6 +703,48 @@ change_callback_log_severity(int loglevelMin, int loglevelMax, UNLOCK_LOGS(); } +/** If there are any log messages that were genered with LD_NOCB waiting to + * be sent to callback-based loggers, send them now. */ +void +flush_pending_log_callbacks(void) +{ + logfile_t *lf; + smartlist_t *messages, *messages_tmp; + + LOCK_LOGS(); + if (0 == smartlist_len(pending_cb_messages)) { + UNLOCK_LOGS(); + return; + } + + messages = pending_cb_messages; + pending_cb_messages = smartlist_create(); + do { + SMARTLIST_FOREACH_BEGIN(messages, pending_cb_message_t *, msg) { + const int severity = msg->severity; + const int domain = msg->domain; + for (lf = logfiles; lf; lf = lf->next) { + if (! lf->callback || lf->seems_dead || + ! (lf->severities->masks[SEVERITY_MASK_IDX(severity)] & domain)) { + continue; + } + lf->callback(severity, domain, msg->msg); + } + tor_free(msg->msg); + tor_free(msg); + } SMARTLIST_FOREACH_END(msg); + smartlist_clear(messages); + + messages_tmp = pending_cb_messages; + pending_cb_messages = messages; + messages = messages_tmp; + } while (smartlist_len(messages)); + + smartlist_free(messages); + + UNLOCK_LOGS(); +} + /** Close any log handlers added by add_temp_log() or marked by * mark_logs_temp(). */ void @@ -682,13 +814,11 @@ add_file_log(const log_severity_list_t *severity, const char *filename) logfiles->needs_close = 1; lf = logfiles; _log_global_min_severity = get_min_log_level(); - UNLOCK_LOGS(); if (log_tor_version(lf, 0) < 0) { - LOCK_LOGS(); delete_log(lf); - UNLOCK_LOGS(); } + UNLOCK_LOGS(); return 0; } @@ -709,7 +839,6 @@ add_syslog_log(const log_severity_list_t *severity) lf->fd = -1; lf->severities = tor_memdup(severity, sizeof(log_severity_list_t)); lf->filename = tor_strdup("<syslog>"); - lf->is_syslog = 1; LOCK_LOGS(); @@ -751,7 +880,7 @@ log_level_to_string(int level) static const char *domain_list[] = { "GENERAL", "CRYPTO", "NET", "CONFIG", "FS", "PROTOCOL", "MM", "HTTP", "APP", "CONTROL", "CIRC", "REND", "BUG", "DIR", "DIRSERV", - "OR", "EDGE", "ACCT", "HIST", NULL + "OR", "EDGE", "ACCT", "HIST", "HANDSHAKE", NULL }; /** Return a bitmask for the log domain for which <b>domain</b> is the name, @@ -766,18 +895,41 @@ parse_log_domain(const char *domain) } return 0; } -#if 0 -/** Translate a bitmask of log domains to a string, or NULL if the bitmask - * is undecodable. */ -static const char * -domain_to_string(log_domain_mask_t domain) + +/** Translate a bitmask of log domains to a string. */ +static char * +domain_to_string(log_domain_mask_t domain, char *buf, size_t buflen) { - int bit = tor_log2(domain); - if ((bit == 0 && domain == 0) || bit >= N_LOGGING_DOMAINS) - return NULL; - return domain_list[bit]; + char *cp = buf; + char *eos = buf+buflen; + + buf[0] = '\0'; + if (! domain) + return buf; + while (1) { + const char *d; + int bit = tor_log2(domain); + size_t n; + if (bit >= N_LOGGING_DOMAINS) { + tor_snprintf(buf, buflen, "<BUG:Unknown domain %lx>", (long)domain); + return buf+strlen(buf); + } + d = domain_list[bit]; + n = strlcpy(cp, d, eos-cp); + if (n >= buflen) { + tor_snprintf(buf, buflen, "<BUG:Truncating domain %lx>", (long)domain); + return buf+strlen(buf); + } + cp += n; + domain &= ~(1<<bit); + + if (domain == 0 || (eos-cp) < 2) + return cp; + + memcpy(cp, ",", 2); /*Nul-terminated ,"*/ + cp++; + } } -#endif /** Parse a log severity pattern in *<b>cfg_ptr</b>. Advance cfg_ptr after * the end of the severityPattern. Set the value of <b>severity_out</b> to @@ -853,7 +1005,10 @@ parse_log_severity_config(const char **cfg_ptr, smartlist_free(domains_list); if (err) return -1; - domains &= ~neg_domains; + if (domains == 0 && neg_domains) + domains = ~neg_domains; + else + domains &= ~neg_domains; cfg = eat_whitespace(closebracket+1); } else { ++got_an_unqualified_range; @@ -929,65 +1084,6 @@ switch_logs_debug(void) UNLOCK_LOGS(); } -#ifdef HAVE_EVENT_SET_LOG_CALLBACK -/** A string which, if it appears in a libevent log, should be ignored. */ -static const char *suppress_msg = NULL; -/** Callback function passed to event_set_log() so we can intercept - * log messages from libevent. */ -static void -libevent_logging_callback(int severity, const char *msg) -{ - char buf[1024]; - size_t n; - if (suppress_msg && strstr(msg, suppress_msg)) - return; - n = strlcpy(buf, msg, sizeof(buf)); - if (n && n < sizeof(buf) && buf[n-1] == '\n') { - buf[n-1] = '\0'; - } - switch (severity) { - case _EVENT_LOG_DEBUG: - log(LOG_DEBUG, LD_NOCB|LD_NET, "Message from libevent: %s", buf); - break; - case _EVENT_LOG_MSG: - log(LOG_INFO, LD_NOCB|LD_NET, "Message from libevent: %s", buf); - break; - case _EVENT_LOG_WARN: - log(LOG_WARN, LD_NOCB|LD_GENERAL, "Warning from libevent: %s", buf); - break; - case _EVENT_LOG_ERR: - log(LOG_ERR, LD_NOCB|LD_GENERAL, "Error from libevent: %s", buf); - break; - default: - log(LOG_WARN, LD_NOCB|LD_GENERAL, "Message [%d] from libevent: %s", - severity, buf); - break; - } -} -/** Set hook to intercept log messages from libevent. */ -void -configure_libevent_logging(void) -{ - event_set_log_callback(libevent_logging_callback); -} -/** Ignore any libevent log message that contains <b>msg</b>. */ -void -suppress_libevent_log_msg(const char *msg) -{ - suppress_msg = msg; -} -#else -void -configure_libevent_logging(void) -{ -} -void -suppress_libevent_log_msg(const char *msg) -{ - (void)msg; -} -#endif - #if 0 static void dump_log_info(logfile_t *lf) diff --git a/src/common/memarea.c b/src/common/memarea.c index ac26c5fd90..a6b8c4ee9c 100644 --- a/src/common/memarea.c +++ b/src/common/memarea.c @@ -11,7 +11,11 @@ #include "memarea.h" #include "util.h" #include "compat.h" -#include "log.h" +#include "torlog.h" + +/** If true, we try to detect any attempts to write beyond the length of a + * memarea. */ +#define USE_SENTINELS /** All returned pointers should be aligned to the nearest multiple of this * value. */ @@ -25,13 +29,39 @@ #error "void* is neither 4 nor 8 bytes long. I don't know how to align stuff." #endif +#ifdef USE_SENTINELS +/** Magic value that we stick at the end of a memarea so we can make sure + * there are no run-off-the-end bugs. */ +#define SENTINEL_VAL 0x90806622u +/** How many bytes per area do we devote to the sentinel? */ +#define SENTINEL_LEN sizeof(uint32_t) +/** Given a mem_area_chunk_t with SENTINEL_LEN extra bytes allocated at the + * end, set those bytes. */ +#define SET_SENTINEL(chunk) \ + STMT_BEGIN \ + set_uint32( &(chunk)->u.mem[chunk->mem_size], SENTINEL_VAL ); \ + STMT_END +/** Assert that the sentinel on a memarea is set correctly. */ +#define CHECK_SENTINEL(chunk) \ + STMT_BEGIN \ + uint32_t sent_val = get_uint32(&(chunk)->u.mem[chunk->mem_size]); \ + tor_assert(sent_val == SENTINEL_VAL); \ + STMT_END +#else +#define SENTINEL_LEN 0 +#define SET_SENTINEL(chunk) STMT_NIL +#define CHECK_SENTINEL(chunk) STMT_NIL +#endif + /** Increment <b>ptr</b> until it is aligned to MEMAREA_ALIGN. */ static INLINE void * realign_pointer(void *ptr) { uintptr_t x = (uintptr_t)ptr; x = (x+MEMAREA_ALIGN_MASK) & ~MEMAREA_ALIGN_MASK; - tor_assert(((void*)x) >= ptr); // XXXX021 remove this once bug 930 is solved + /* Reinstate this if bug 930 ever reappears + tor_assert(((void*)x) >= ptr); + */ return (void*)x; } @@ -51,8 +81,11 @@ typedef struct memarea_chunk_t { } u; } memarea_chunk_t; +/** How many bytes are needed for overhead before we get to the memory part + * of a chunk? */ #define CHUNK_HEADER_SIZE STRUCT_OFFSET(memarea_chunk_t, u) +/** What's the smallest that we'll allocate a chunk? */ #define CHUNK_SIZE 4096 /** A memarea_t is an allocation region for a set of small memory requests @@ -79,15 +112,20 @@ alloc_chunk(size_t sz, int freelist_ok) freelist = res->next_chunk; res->next_chunk = NULL; --freelist_len; + CHECK_SENTINEL(res); return res; } else { size_t chunk_size = freelist_ok ? CHUNK_SIZE : sz; - memarea_chunk_t *res = tor_malloc_roundup(&chunk_size); + memarea_chunk_t *res; + chunk_size += SENTINEL_LEN; + res = tor_malloc_roundup(&chunk_size); res->next_chunk = NULL; - res->mem_size = chunk_size - CHUNK_HEADER_SIZE; + res->mem_size = chunk_size - CHUNK_HEADER_SIZE - SENTINEL_LEN; res->next_mem = res->u.mem; - tor_assert(res->next_mem+res->mem_size == ((char*)res)+chunk_size); + tor_assert(res->next_mem+res->mem_size+SENTINEL_LEN == + ((char*)res)+chunk_size); tor_assert(realign_pointer(res->next_mem) == res->next_mem); + SET_SENTINEL(res); return res; } } @@ -95,8 +133,9 @@ alloc_chunk(size_t sz, int freelist_ok) /** Release <b>chunk</b> from a memarea, either by adding it to the freelist * or by freeing it if the freelist is already too big. */ static void -chunk_free(memarea_chunk_t *chunk) +chunk_free_unchecked(memarea_chunk_t *chunk) { + CHECK_SENTINEL(chunk); if (freelist_len < MAX_FREELIST_LEN) { ++freelist_len; chunk->next_chunk = freelist; @@ -124,7 +163,7 @@ memarea_drop_all(memarea_t *area) memarea_chunk_t *chunk, *next; for (chunk = area->first; chunk; chunk = next) { next = chunk->next_chunk; - chunk_free(chunk); + chunk_free_unchecked(chunk); } area->first = NULL; /*fail fast on */ tor_free(area); @@ -140,7 +179,7 @@ memarea_clear(memarea_t *area) if (area->first->next_chunk) { for (chunk = area->first->next_chunk; chunk; chunk = next) { next = chunk->next_chunk; - chunk_free(chunk); + chunk_free_unchecked(chunk); } area->first->next_chunk = NULL; } @@ -183,6 +222,7 @@ memarea_alloc(memarea_t *area, size_t sz) memarea_chunk_t *chunk = area->first; char *result; tor_assert(chunk); + CHECK_SENTINEL(chunk); tor_assert(sz < SIZE_T_CEILING); if (sz == 0) sz = 1; @@ -203,9 +243,10 @@ memarea_alloc(memarea_t *area, size_t sz) } result = chunk->next_mem; chunk->next_mem = chunk->next_mem + sz; - // XXXX021 remove these once bug 930 is solved. + /* Reinstate these if bug 930 ever comes back tor_assert(chunk->next_mem >= chunk->u.mem); tor_assert(chunk->next_mem <= chunk->u.mem+chunk->mem_size); + */ chunk->next_mem = realign_pointer(chunk->next_mem); return result; } @@ -261,6 +302,7 @@ memarea_get_stats(memarea_t *area, size_t *allocated_out, size_t *used_out) size_t a = 0, u = 0; memarea_chunk_t *chunk; for (chunk = area->first; chunk; chunk = chunk->next_chunk) { + CHECK_SENTINEL(chunk); a += CHUNK_HEADER_SIZE + chunk->mem_size; tor_assert(chunk->next_mem >= chunk->u.mem); u += CHUNK_HEADER_SIZE + (chunk->next_mem - chunk->u.mem); @@ -277,6 +319,7 @@ memarea_assert_ok(memarea_t *area) tor_assert(area->first); for (chunk = area->first; chunk; chunk = chunk->next_chunk) { + CHECK_SENTINEL(chunk); tor_assert(chunk->next_mem >= chunk->u.mem); tor_assert(chunk->next_mem <= (char*) realign_pointer(chunk->u.mem+chunk->mem_size)); diff --git a/src/common/mempool.c b/src/common/mempool.c index 9538a05489..c444923189 100644 --- a/src/common/mempool.c +++ b/src/common/mempool.c @@ -65,7 +65,7 @@ #include "orconfig.h" #include "util.h" #include "compat.h" -#include "log.h" +#include "torlog.h" #define ALLOC(x) tor_malloc(x) #define FREE(x) tor_free(x) #define ASSERT(x) tor_assert(x) diff --git a/src/common/sha256.c b/src/common/sha256.c new file mode 100644 index 0000000000..258b7e062a --- /dev/null +++ b/src/common/sha256.c @@ -0,0 +1,331 @@ +/* Copyright (c) 2009-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ +/* This SHA256 implementation is adapted from the public domain one in + LibTomCrypt, version 1.6. Tor uses it on platforms where OpenSSL doesn't + have a SHA256. */ + + +typedef struct sha256_state { + uint64_t length; + uint32_t state[8], curlen; + unsigned char buf[64]; +} sha256_state; + +#define CRYPT_OK 0 +#define CRYPT_NOP -1 +#define CRYPT_INVALID_ARG -2 + +#define LOAD32H(x,y) STMT_BEGIN x = ntohl(get_uint32((const char*)y)); STMT_END +#define STORE32H(x,y) STMT_BEGIN set_uint32((char*)y, htonl(x)); STMT_END +#define STORE64H(x,y) STMT_BEGIN \ + set_uint32((char*)y, htonl((uint32_t)((x)>>32))); \ + set_uint32(((char*)y)+4, htonl((uint32_t)((x)&0xffffffff))); \ + STMT_END +#define RORc(x, y) ( ((((unsigned long)(x)&0xFFFFFFFFUL)>>(unsigned long)((y)&31)) | ((unsigned long)(x)<<(unsigned long)(32-((y)&31)))) & 0xFFFFFFFFUL) +#ifndef MIN + #define MIN(x, y) ( ((x)<(y))?(x):(y) ) +#endif + + +/* LibTomCrypt, modular cryptographic library -- Tom St Denis + * + * LibTomCrypt is a library that provides various cryptographic + * algorithms in a highly modular and flexible manner. + * + * The library is free for all purposes without any express + * guarantee it works. + * + * Tom St Denis, tomstdenis@gmail.com, http://libtomcrypt.com + */ + +/** + @file sha256.c + SHA256 by Tom St Denis +*/ + + +#ifdef LTC_SMALL_CODE +/* the K array */ +static const uint32_t K[64] = { + 0x428a2f98UL, 0x71374491UL, 0xb5c0fbcfUL, 0xe9b5dba5UL, 0x3956c25bUL, + 0x59f111f1UL, 0x923f82a4UL, 0xab1c5ed5UL, 0xd807aa98UL, 0x12835b01UL, + 0x243185beUL, 0x550c7dc3UL, 0x72be5d74UL, 0x80deb1feUL, 0x9bdc06a7UL, + 0xc19bf174UL, 0xe49b69c1UL, 0xefbe4786UL, 0x0fc19dc6UL, 0x240ca1ccUL, + 0x2de92c6fUL, 0x4a7484aaUL, 0x5cb0a9dcUL, 0x76f988daUL, 0x983e5152UL, + 0xa831c66dUL, 0xb00327c8UL, 0xbf597fc7UL, 0xc6e00bf3UL, 0xd5a79147UL, + 0x06ca6351UL, 0x14292967UL, 0x27b70a85UL, 0x2e1b2138UL, 0x4d2c6dfcUL, + 0x53380d13UL, 0x650a7354UL, 0x766a0abbUL, 0x81c2c92eUL, 0x92722c85UL, + 0xa2bfe8a1UL, 0xa81a664bUL, 0xc24b8b70UL, 0xc76c51a3UL, 0xd192e819UL, + 0xd6990624UL, 0xf40e3585UL, 0x106aa070UL, 0x19a4c116UL, 0x1e376c08UL, + 0x2748774cUL, 0x34b0bcb5UL, 0x391c0cb3UL, 0x4ed8aa4aUL, 0x5b9cca4fUL, + 0x682e6ff3UL, 0x748f82eeUL, 0x78a5636fUL, 0x84c87814UL, 0x8cc70208UL, + 0x90befffaUL, 0xa4506cebUL, 0xbef9a3f7UL, 0xc67178f2UL +}; +#endif + +/* Various logical functions */ +#define Ch(x,y,z) (z ^ (x & (y ^ z))) +#define Maj(x,y,z) (((x | y) & z) | (x & y)) +#define S(x, n) RORc((x),(n)) +#define R(x, n) (((x)&0xFFFFFFFFUL)>>(n)) +#define Sigma0(x) (S(x, 2) ^ S(x, 13) ^ S(x, 22)) +#define Sigma1(x) (S(x, 6) ^ S(x, 11) ^ S(x, 25)) +#define Gamma0(x) (S(x, 7) ^ S(x, 18) ^ R(x, 3)) +#define Gamma1(x) (S(x, 17) ^ S(x, 19) ^ R(x, 10)) + +/* compress 512-bits */ +#ifdef LTC_CLEAN_STACK +static int _sha256_compress(sha256_state * md, unsigned char *buf) +#else +static int sha256_compress(sha256_state * md, unsigned char *buf) +#endif +{ + uint32_t S[8], W[64], t0, t1; +#ifdef LTC_SMALL_CODE + uint32_t t; +#endif + int i; + + /* copy state into S */ + for (i = 0; i < 8; i++) { + S[i] = md->state[i]; + } + + /* copy the state into 512-bits into W[0..15] */ + for (i = 0; i < 16; i++) { + LOAD32H(W[i], buf + (4*i)); + } + + /* fill W[16..63] */ + for (i = 16; i < 64; i++) { + W[i] = Gamma1(W[i - 2]) + W[i - 7] + Gamma0(W[i - 15]) + W[i - 16]; + } + + /* Compress */ +#ifdef LTC_SMALL_CODE +#define RND(a,b,c,d,e,f,g,h,i) \ + t0 = h + Sigma1(e) + Ch(e, f, g) + K[i] + W[i]; \ + t1 = Sigma0(a) + Maj(a, b, c); \ + d += t0; \ + h = t0 + t1; + + for (i = 0; i < 64; ++i) { + RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],i); + t = S[7]; S[7] = S[6]; S[6] = S[5]; S[5] = S[4]; + S[4] = S[3]; S[3] = S[2]; S[2] = S[1]; S[1] = S[0]; S[0] = t; + } +#else +#define RND(a,b,c,d,e,f,g,h,i,ki) \ + t0 = h + Sigma1(e) + Ch(e, f, g) + ki + W[i]; \ + t1 = Sigma0(a) + Maj(a, b, c); \ + d += t0; \ + h = t0 + t1; + + RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],0,0x428a2f98); + RND(S[7],S[0],S[1],S[2],S[3],S[4],S[5],S[6],1,0x71374491); + RND(S[6],S[7],S[0],S[1],S[2],S[3],S[4],S[5],2,0xb5c0fbcf); + RND(S[5],S[6],S[7],S[0],S[1],S[2],S[3],S[4],3,0xe9b5dba5); + RND(S[4],S[5],S[6],S[7],S[0],S[1],S[2],S[3],4,0x3956c25b); + RND(S[3],S[4],S[5],S[6],S[7],S[0],S[1],S[2],5,0x59f111f1); + RND(S[2],S[3],S[4],S[5],S[6],S[7],S[0],S[1],6,0x923f82a4); + RND(S[1],S[2],S[3],S[4],S[5],S[6],S[7],S[0],7,0xab1c5ed5); + RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],8,0xd807aa98); + RND(S[7],S[0],S[1],S[2],S[3],S[4],S[5],S[6],9,0x12835b01); + RND(S[6],S[7],S[0],S[1],S[2],S[3],S[4],S[5],10,0x243185be); + RND(S[5],S[6],S[7],S[0],S[1],S[2],S[3],S[4],11,0x550c7dc3); + RND(S[4],S[5],S[6],S[7],S[0],S[1],S[2],S[3],12,0x72be5d74); + RND(S[3],S[4],S[5],S[6],S[7],S[0],S[1],S[2],13,0x80deb1fe); + RND(S[2],S[3],S[4],S[5],S[6],S[7],S[0],S[1],14,0x9bdc06a7); + RND(S[1],S[2],S[3],S[4],S[5],S[6],S[7],S[0],15,0xc19bf174); + RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],16,0xe49b69c1); + RND(S[7],S[0],S[1],S[2],S[3],S[4],S[5],S[6],17,0xefbe4786); + RND(S[6],S[7],S[0],S[1],S[2],S[3],S[4],S[5],18,0x0fc19dc6); + RND(S[5],S[6],S[7],S[0],S[1],S[2],S[3],S[4],19,0x240ca1cc); + RND(S[4],S[5],S[6],S[7],S[0],S[1],S[2],S[3],20,0x2de92c6f); + RND(S[3],S[4],S[5],S[6],S[7],S[0],S[1],S[2],21,0x4a7484aa); + RND(S[2],S[3],S[4],S[5],S[6],S[7],S[0],S[1],22,0x5cb0a9dc); + RND(S[1],S[2],S[3],S[4],S[5],S[6],S[7],S[0],23,0x76f988da); + RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],24,0x983e5152); + RND(S[7],S[0],S[1],S[2],S[3],S[4],S[5],S[6],25,0xa831c66d); + RND(S[6],S[7],S[0],S[1],S[2],S[3],S[4],S[5],26,0xb00327c8); + RND(S[5],S[6],S[7],S[0],S[1],S[2],S[3],S[4],27,0xbf597fc7); + RND(S[4],S[5],S[6],S[7],S[0],S[1],S[2],S[3],28,0xc6e00bf3); + RND(S[3],S[4],S[5],S[6],S[7],S[0],S[1],S[2],29,0xd5a79147); + RND(S[2],S[3],S[4],S[5],S[6],S[7],S[0],S[1],30,0x06ca6351); + RND(S[1],S[2],S[3],S[4],S[5],S[6],S[7],S[0],31,0x14292967); + RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],32,0x27b70a85); + RND(S[7],S[0],S[1],S[2],S[3],S[4],S[5],S[6],33,0x2e1b2138); + RND(S[6],S[7],S[0],S[1],S[2],S[3],S[4],S[5],34,0x4d2c6dfc); + RND(S[5],S[6],S[7],S[0],S[1],S[2],S[3],S[4],35,0x53380d13); + RND(S[4],S[5],S[6],S[7],S[0],S[1],S[2],S[3],36,0x650a7354); + RND(S[3],S[4],S[5],S[6],S[7],S[0],S[1],S[2],37,0x766a0abb); + RND(S[2],S[3],S[4],S[5],S[6],S[7],S[0],S[1],38,0x81c2c92e); + RND(S[1],S[2],S[3],S[4],S[5],S[6],S[7],S[0],39,0x92722c85); + RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],40,0xa2bfe8a1); + RND(S[7],S[0],S[1],S[2],S[3],S[4],S[5],S[6],41,0xa81a664b); + RND(S[6],S[7],S[0],S[1],S[2],S[3],S[4],S[5],42,0xc24b8b70); + RND(S[5],S[6],S[7],S[0],S[1],S[2],S[3],S[4],43,0xc76c51a3); + RND(S[4],S[5],S[6],S[7],S[0],S[1],S[2],S[3],44,0xd192e819); + RND(S[3],S[4],S[5],S[6],S[7],S[0],S[1],S[2],45,0xd6990624); + RND(S[2],S[3],S[4],S[5],S[6],S[7],S[0],S[1],46,0xf40e3585); + RND(S[1],S[2],S[3],S[4],S[5],S[6],S[7],S[0],47,0x106aa070); + RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],48,0x19a4c116); + RND(S[7],S[0],S[1],S[2],S[3],S[4],S[5],S[6],49,0x1e376c08); + RND(S[6],S[7],S[0],S[1],S[2],S[3],S[4],S[5],50,0x2748774c); + RND(S[5],S[6],S[7],S[0],S[1],S[2],S[3],S[4],51,0x34b0bcb5); + RND(S[4],S[5],S[6],S[7],S[0],S[1],S[2],S[3],52,0x391c0cb3); + RND(S[3],S[4],S[5],S[6],S[7],S[0],S[1],S[2],53,0x4ed8aa4a); + RND(S[2],S[3],S[4],S[5],S[6],S[7],S[0],S[1],54,0x5b9cca4f); + RND(S[1],S[2],S[3],S[4],S[5],S[6],S[7],S[0],55,0x682e6ff3); + RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],56,0x748f82ee); + RND(S[7],S[0],S[1],S[2],S[3],S[4],S[5],S[6],57,0x78a5636f); + RND(S[6],S[7],S[0],S[1],S[2],S[3],S[4],S[5],58,0x84c87814); + RND(S[5],S[6],S[7],S[0],S[1],S[2],S[3],S[4],59,0x8cc70208); + RND(S[4],S[5],S[6],S[7],S[0],S[1],S[2],S[3],60,0x90befffa); + RND(S[3],S[4],S[5],S[6],S[7],S[0],S[1],S[2],61,0xa4506ceb); + RND(S[2],S[3],S[4],S[5],S[6],S[7],S[0],S[1],62,0xbef9a3f7); + RND(S[1],S[2],S[3],S[4],S[5],S[6],S[7],S[0],63,0xc67178f2); + +#undef RND + +#endif + + /* feedback */ + for (i = 0; i < 8; i++) { + md->state[i] = md->state[i] + S[i]; + } + return CRYPT_OK; +} + +#ifdef LTC_CLEAN_STACK +static int sha256_compress(sha256_state * md, unsigned char *buf) +{ + int err; + err = _sha256_compress(md, buf); + burn_stack(sizeof(uint32_t) * 74); + return err; +} +#endif + +/** + Initialize the hash state + @param md The hash state you wish to initialize + @return CRYPT_OK if successful +*/ +static int sha256_init(sha256_state * md) +{ + LTC_ARGCHK(md != NULL); + + md->curlen = 0; + md->length = 0; + md->state[0] = 0x6A09E667UL; + md->state[1] = 0xBB67AE85UL; + md->state[2] = 0x3C6EF372UL; + md->state[3] = 0xA54FF53AUL; + md->state[4] = 0x510E527FUL; + md->state[5] = 0x9B05688CUL; + md->state[6] = 0x1F83D9ABUL; + md->state[7] = 0x5BE0CD19UL; + return CRYPT_OK; +} + +/** + Process a block of memory though the hash + @param md The hash state + @param in The data to hash + @param inlen The length of the data (octets) + @return CRYPT_OK if successful +*/ +static int sha256_process (sha256_state * md, const unsigned char *in, unsigned long inlen) +{ + unsigned long n; + int err; + LTC_ARGCHK(md != NULL); + LTC_ARGCHK(in != NULL); + if (md->curlen > sizeof(md->buf)) { + return CRYPT_INVALID_ARG; + } + while (inlen > 0) { + if (md->curlen == 0 && inlen >= 64) { + if ((err = sha256_compress (md, (unsigned char *)in)) != CRYPT_OK) { + return err; + } + md->length += 64 * 8; + in += 64; + inlen -= 64; + } else { + n = MIN(inlen, (64 - md->curlen)); + memcpy(md->buf + md->curlen, in, (size_t)n); + md->curlen += n; + in += n; + inlen -= n; + if (md->curlen == 64) { + if ((err = sha256_compress (md, md->buf)) != CRYPT_OK) { + return err; + } + md->length += 8*64; + md->curlen = 0; + } + } + } + return CRYPT_OK; +} + +/** + Terminate the hash to get the digest + @param md The hash state + @param out [out] The destination of the hash (32 bytes) + @return CRYPT_OK if successful +*/ +static int sha256_done(sha256_state * md, unsigned char *out) +{ + int i; + + LTC_ARGCHK(md != NULL); + LTC_ARGCHK(out != NULL); + + if (md->curlen >= sizeof(md->buf)) { + return CRYPT_INVALID_ARG; + } + + + /* increase the length of the message */ + md->length += md->curlen * 8; + + /* append the '1' bit */ + md->buf[md->curlen++] = (unsigned char)0x80; + + /* if the length is currently above 56 bytes we append zeros + * then compress. Then we can fall back to padding zeros and length + * encoding like normal. + */ + if (md->curlen > 56) { + while (md->curlen < 64) { + md->buf[md->curlen++] = (unsigned char)0; + } + sha256_compress(md, md->buf); + md->curlen = 0; + } + + /* pad upto 56 bytes of zeroes */ + while (md->curlen < 56) { + md->buf[md->curlen++] = (unsigned char)0; + } + + /* store length */ + STORE64H(md->length, md->buf+56); + sha256_compress(md, md->buf); + + /* copy output */ + for (i = 0; i < 8; i++) { + STORE32H(md->state[i], out+(4*i)); + } +#ifdef LTC_CLEAN_STACK + zeromem(md, sizeof(sha256_state)); +#endif + return CRYPT_OK; +} + +/* $Source: /cvs/libtom/libtomcrypt/src/hashes/sha2/sha256.c,v $ */ +/* $Revision: 1.9 $ */ +/* $Date: 2006/11/01 09:28:17 $ */ diff --git a/src/common/test.h b/src/common/test.h deleted file mode 100644 index c5995d1454..0000000000 --- a/src/common/test.h +++ /dev/null @@ -1,184 +0,0 @@ -/* Copyright (c) 2001-2003, Roger Dingledine. - * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2011, The Tor Project, Inc. */ -/* See LICENSE for licensing information */ - -#ifndef _TOR_TEST_H -#define _TOR_TEST_H - -/** - * \file test.h - * \brief Macros used by unit tests. - */ - -#include "compat.h" - -#ifdef __GNUC__ -#define PRETTY_FUNCTION __PRETTY_FUNCTION__ -#else -#define PRETTY_FUNCTION "" -#endif - -#define test_fail_msg(msg) \ - STMT_BEGIN \ - have_failed = 1; \ - printf("\nFile %s: line %d (%s): %s", \ - _SHORT_FILE_, \ - __LINE__, \ - PRETTY_FUNCTION, \ - msg); \ - goto done; \ - STMT_END - -#define test_fail() test_fail_msg("Assertion failed.") - -#define test_assert(expr) \ - STMT_BEGIN \ - if (expr) { printf("."); fflush(stdout); } else { \ - have_failed = 1; \ - printf("\nFile %s: line %d (%s): assertion failed: (%s)\n", \ - _SHORT_FILE_, \ - __LINE__, \ - PRETTY_FUNCTION, \ - #expr); \ - goto done; \ - } STMT_END - -#define test_eq_type(tp, fmt, expr1, expr2) \ - STMT_BEGIN \ - tp _test_v1=(tp)(expr1); \ - tp _test_v2=(tp)(expr2); \ - if (_test_v1==_test_v2) { printf("."); fflush(stdout); } else { \ - have_failed = 1; \ - printf("\nFile %s: line %d (%s): Assertion failed: (%s==%s)\n" \ - " "fmt "!="fmt"\n", \ - _SHORT_FILE_, \ - __LINE__, \ - PRETTY_FUNCTION, \ - #expr1, #expr2, \ - _test_v1, _test_v2); \ - goto done; \ - } STMT_END - -#define test_eq(expr1, expr2) \ - test_eq_type(long, "%ld", expr1, expr2) - -#define test_eq_ptr(expr1, expr2) \ - test_eq_type(void*, "%p", expr1, expr2) - -#define test_neq_type(tp, fmt, expr1, expr2) \ - STMT_BEGIN \ - tp _test_v1=(tp)(expr1); \ - tp _test_v2=(tp)(expr2); \ - if (_test_v1!=_test_v2) { printf("."); fflush(stdout); } else { \ - have_failed = 1; \ - printf("\nFile %s: line %d (%s): Assertion failed: (%s!=%s)\n" \ - " ("fmt" == "fmt")\n", \ - _SHORT_FILE_, \ - __LINE__, \ - PRETTY_FUNCTION, \ - #expr1, #expr2, \ - _test_v1, _test_v2); \ - goto done; \ - } STMT_END - -#define test_neq(expr1, expr2) \ - test_neq_type(long, "%ld", expr1, expr2) - -#define test_neq_ptr(expr1, expr2) \ - test_neq_type(void *, "%p", expr1, expr2) - -#define test_streq(expr1, expr2) \ - STMT_BEGIN \ - const char *_test_v1=(expr1), *_test_v2=(expr2); \ - if (!strcmp(_test_v1,_test_v2)) { printf("."); fflush(stdout); } else { \ - have_failed = 1; \ - printf("\nFile %s: line %d (%s): Assertion failed: (%s==%s)\n"\ - " (\"%s\" != \"%s\")\n", \ - _SHORT_FILE_, \ - __LINE__, \ - PRETTY_FUNCTION, \ - #expr1, #expr2, \ - _test_v1, _test_v2); \ - goto done; \ - } STMT_END - -#define test_strneq(expr1, expr2) \ - STMT_BEGIN \ - const char *_test_v1=(expr1), *_test_v2=(expr2); \ - if (strcmp(_test_v1,_test_v2)) { printf("."); fflush(stdout); } else { \ - have_failed = 1; \ - printf("\nFile %s: line %d (%s): Assertion failed: (%s!=%s)\n"\ - " (\"%s\" == \"%s\")\n", \ - _SHORT_FILE_, \ - __LINE__, \ - PRETTY_FUNCTION, \ - #expr1, #expr2, \ - _test_v1, _test_v2); \ - goto done; \ - } STMT_END - -#define test_memeq(expr1, expr2, len) \ - STMT_BEGIN \ - const void *_test_v1=(expr1), *_test_v2=(expr2); \ - char *mem1, *mem2; \ - if (!memcmp(_test_v1,_test_v2,(len))) { \ - printf("."); fflush(stdout); } else { \ - have_failed = 1; \ - mem1 = tor_malloc(len*2+1); \ - mem2 = tor_malloc(len*2+1); \ - base16_encode(mem1, len*2+1, _test_v1, len); \ - base16_encode(mem2, len*2+1, _test_v2, len); \ - printf("\nFile %s: line %d (%s): Assertion failed: (%s==%s)\n" \ - " %s != %s\n", \ - _SHORT_FILE_, \ - __LINE__, \ - PRETTY_FUNCTION, \ - #expr1, #expr2, mem1, mem2); \ - tor_free(mem1); \ - tor_free(mem2); \ - goto done; \ - } STMT_END - -#define test_memeq_hex(expr1, hex) \ - STMT_BEGIN \ - const char *_test_v1 = (char*)(expr1); \ - const char *_test_v2 = (hex); \ - size_t _len_v2 = strlen(_test_v2); \ - char *_mem2 = tor_malloc(_len_v2/2); \ - tor_assert((_len_v2 & 1) == 0); \ - base16_decode(_mem2, _len_v2/2, _test_v2, _len_v2); \ - if (!memcmp(_mem2, _test_v1, _len_v2/2)) { \ - printf("."); fflush(stdout); } else { \ - char *_mem1 = tor_malloc(_len_v2+1); \ - base16_encode(_mem1, _len_v2+1, _test_v1, _len_v2/2); \ - printf("\nFile %s: line %d (%s): Assertion failed: (%s==%s)\n" \ - " %s != %s\n", \ - _SHORT_FILE_, \ - __LINE__, \ - PRETTY_FUNCTION, \ - #expr1, _test_v2, _mem1, _test_v2); \ - tor_free(_mem1); \ - tor_free(_mem2); \ - goto done; \ - } \ - tor_free(_mem2); \ - STMT_END - -#define test_memneq(expr1, expr2, len) \ - STMT_BEGIN \ - void *_test_v1=(expr1), *_test_v2=(expr2); \ - if (memcmp(_test_v1,_test_v2,(len))) { \ - printf("."); fflush(stdout); \ - } else { \ - have_failed = 1; \ - printf("\nFile %s: line %d (%s): Assertion failed: (%s!=%s)\n", \ - _SHORT_FILE_, \ - __LINE__, \ - PRETTY_FUNCTION, \ - #expr1, #expr2); \ - goto done; \ - } STMT_END - -#endif - diff --git a/src/common/torgzip.c b/src/common/torgzip.c index 249151cc9b..e9079e0ba5 100644 --- a/src/common/torgzip.c +++ b/src/common/torgzip.c @@ -13,20 +13,42 @@ #include <stdlib.h> #include <stdio.h> #include <assert.h> -#ifdef _MSC_VER -#include "..\..\contrib\zlib\zlib.h" -#else -#include <zlib.h> -#endif #include <string.h> #ifdef HAVE_NETINET_IN_H #include <netinet/in.h> #endif +#include "torint.h" #include "util.h" -#include "log.h" +#include "torlog.h" #include "torgzip.h" +/* zlib 1.2.4 and 1.2.5 do some "clever" things with macros. Instead of + saying "(defined(FOO) ? FOO : 0)" they like to say "FOO-0", on the theory + that nobody will care if the compile outputs a no-such-identifier warning. + + Sorry, but we like -Werror over here, so I guess we need to define these. + I hope that zlib 1.2.6 doesn't break these too. +*/ +#ifndef _LARGEFILE64_SOURCE +#define _LARGEFILE64_SOURCE 0 +#endif +#ifndef _LFS64_LARGEFILE +#define _LFS64_LARGEFILE 0 +#endif +#ifndef _FILE_OFFSET_BITS +#define _FILE_OFFSET_BITS 0 +#endif +#ifndef off64_t +#define off64_t int64_t +#endif + +#ifdef _MSC_VER +#include "..\..\contrib\zlib\zlib.h" +#else +#include <zlib.h> +#endif + /** Set to 1 if zlib is a version that supports gzip; set to 0 if it doesn't; * set to -1 if we haven't checked yet. */ static int gzip_is_supported = -1; @@ -57,6 +79,7 @@ method_bits(compress_method_t method) return method == GZIP_METHOD ? 15+16 : 15; } +/** @{ */ /* These macros define the maximum allowable compression factor. Anything of * size greater than CHECK_FOR_COMPRESSION_BOMB_AFTER is not allowed to * have an uncompression factor (uncompressed size:compressed size ratio) of @@ -72,6 +95,7 @@ method_bits(compress_method_t method) */ #define MAX_UNCOMPRESSION_FACTOR 25 #define CHECK_FOR_COMPRESSION_BOMB_AFTER (1024*64) +/** @} */ /** Return true if uncompressing an input of size <b>in_size</b> to an input * of size at least <b>size_out</b> looks like a compression bomb. */ @@ -198,9 +222,7 @@ tor_gzip_compress(char **out, size_t *out_len, deflateEnd(stream); tor_free(stream); } - if (*out) { - tor_free(*out); - } + tor_free(*out); return -1; } @@ -369,12 +391,12 @@ detect_compression_method(const char *in, size_t in_len) /** Internal state for an incremental zlib compression/decompression. The * body of this struct is not exposed. */ struct tor_zlib_state_t { - struct z_stream_s stream; - int compress; + struct z_stream_s stream; /**< The zlib stream */ + int compress; /**< True if we are compressing; false if we are inflating */ - /* Number of bytes read so far. Used to detect zlib bombs. */ + /** Number of bytes read so far. Used to detect zlib bombs. */ size_t input_so_far; - /* Number of bytes written so far. Used to detect zlib bombs. */ + /** Number of bytes written so far. Used to detect zlib bombs. */ size_t output_so_far; }; @@ -479,7 +501,8 @@ tor_zlib_process(tor_zlib_state_t *state, void tor_zlib_free(tor_zlib_state_t *state) { - tor_assert(state); + if (!state) + return; if (state->compress) deflateEnd(&state->stream); diff --git a/src/common/torint.h b/src/common/torint.h index 2a9fba6fcf..0b5c29adc0 100644 --- a/src/common/torint.h +++ b/src/common/torint.h @@ -117,11 +117,10 @@ typedef unsigned int uint32_t; #ifndef INT32_MAX #define INT32_MAX 0x7fffffff #endif -#endif - #ifndef INT32_MIN #define INT32_MIN (-2147483647-1) #endif +#endif #if (SIZEOF_LONG == 4) #ifndef HAVE_INT32_T @@ -330,8 +329,10 @@ typedef uint32_t uintptr_t; #endif #endif -/* Any size_t larger than this amount is likely to be an underflow. */ -#define SIZE_T_CEILING (SSIZE_T_MAX-16) +/** Any ssize_t larger than this amount is likely to be an underflow. */ +#define SSIZE_T_CEILING ((ssize_t)(SSIZE_T_MAX-16)) +/** Any size_t larger than this amount is likely to be an underflow. */ +#define SIZE_T_CEILING ((size_t)(SSIZE_T_MAX-16)) #endif /* __TORINT_H */ diff --git a/src/common/log.h b/src/common/torlog.h index 2520ed1633..000e32ddab 100644 --- a/src/common/log.h +++ b/src/common/torlog.h @@ -5,7 +5,7 @@ /* See LICENSE for licensing information */ /** - * \file log.h + * \file torlog.h * * \brief Headers for log.c **/ @@ -90,14 +90,16 @@ #define LD_ACCT (1u<<17) /** Router history */ #define LD_HIST (1u<<18) - +/** OR handshaking */ +#define LD_HANDSHAKE (1u<<19) /** Number of logging domains in the code. */ -#define N_LOGGING_DOMAINS 19 +#define N_LOGGING_DOMAINS 20 -/** This log message is not safe to send to a callback-based logger. - * Used as a flag, not a log domain. */ +/** This log message is not safe to send to a callback-based logger + * immediately. Used as a flag, not a log domain. */ #define LD_NOCB (1u<<31) +/** Mask of zero or more log domains, OR'd together. */ typedef uint32_t log_domain_mask_t; /** Configures which severities are logged for each logging domain for a given @@ -131,6 +133,7 @@ int add_file_log(const log_severity_list_t *severity, const char *filename); int add_syslog_log(const log_severity_list_t *severity); #endif int add_callback_log(const log_severity_list_t *severity, log_callback cb); +void logs_set_domain_logging(int enabled); int get_min_log_level(void); void switch_logs_debug(void); void logs_free_all(void); @@ -138,16 +141,15 @@ void add_temp_log(int min_severity); void close_temp_logs(void); void rollback_log_changes(void); void mark_logs_temp(void); -void configure_libevent_logging(void); -void suppress_libevent_log_msg(const char *msg); void change_callback_log_severity(int loglevelMin, int loglevelMax, log_callback cb); +void flush_pending_log_callbacks(void); void log_set_application_name(const char *name); /* Outputs a message to stdout */ -void _log(int severity, log_domain_mask_t domain, const char *format, ...) +void tor_log(int severity, log_domain_mask_t domain, const char *format, ...) CHECK_PRINTF(3,4); -#define log _log /* hack it so we don't conflict with log() as much */ +#define log tor_log /* hack it so we don't conflict with log() as much */ #ifdef __GNUC__ extern int _log_global_min_severity; diff --git a/src/common/tortls.c b/src/common/tortls.c index 7735618ea2..10f4440cb4 100644 --- a/src/common/tortls.c +++ b/src/common/tortls.c @@ -16,6 +16,10 @@ #include "orconfig.h" +#if defined (WINCE) +#include <WinSock2.h> +#endif + #include <assert.h> #ifdef MS_WINDOWS /*wrkard for dtls1.h >= 0.9.8m of "#include <winsock.h>"*/ #define WIN32_WINNT 0x400 @@ -45,7 +49,7 @@ #include "crypto.h" #include "tortls.h" #include "util.h" -#include "log.h" +#include "torlog.h" #include "container.h" #include "ht.h" #include <string.h> @@ -82,7 +86,9 @@ static int use_unsafe_renegotiation_op = 0; * SSL3_FLAGS_ALLOW_UNSAFE_LEGACY_RENEGOTIATION? */ static int use_unsafe_renegotiation_flag = 0; -/** Structure holding the TLS state for a single connection. */ +/** Holds a SSL_CTX object and related state used to configure TLS + * connections. + */ typedef struct tor_tls_context_t { int refcnt; SSL_CTX *ctx; @@ -184,10 +190,16 @@ static X509* tor_tls_create_certificate(crypto_pk_env_t *rsa, const char *cname_sign, unsigned int lifetime); static void tor_tls_unblock_renegotiation(tor_tls_t *tls); - -/** Global tls context. We keep it here because nobody else needs to - * touch it. */ -static tor_tls_context_t *global_tls_context = NULL; +static int tor_tls_context_init_one(tor_tls_context_t **ppcontext, + crypto_pk_env_t *identity, + unsigned int key_lifetime); +static tor_tls_context_t *tor_tls_context_new(crypto_pk_env_t *identity, + unsigned int key_lifetime); + +/** Global TLS contexts. We keep them here because nobody else needs + * to touch them. */ +static tor_tls_context_t *server_tls_context = NULL; +static tor_tls_context_t *client_tls_context = NULL; /** True iff tor_tls_init() has been called. */ static int tls_library_is_initialized = 0; @@ -195,30 +207,51 @@ static int tls_library_is_initialized = 0; #define _TOR_TLS_SYSCALL (_MIN_TOR_TLS_ERROR_VAL - 2) #define _TOR_TLS_ZERORETURN (_MIN_TOR_TLS_ERROR_VAL - 1) +#include "tortls_states.h" + +/** Return the symbolic name of an OpenSSL state. */ +static const char * +ssl_state_to_string(int ssl_state) +{ + static char buf[40]; + int i; + for (i = 0; state_map[i].name; ++i) { + if (state_map[i].state == ssl_state) + return state_map[i].name; + } + tor_snprintf(buf, sizeof(buf), "Unknown state %d", ssl_state); + return buf; +} + /** Log all pending tls errors at level <b>severity</b>. Use * <b>doing</b> to describe our current activities. */ static void -tls_log_errors(tor_tls_t *tls, int severity, const char *doing) +tls_log_errors(tor_tls_t *tls, int severity, int domain, const char *doing) { + const char *state = NULL; + int st; unsigned long err; const char *msg, *lib, *func, *addr; addr = tls ? tls->address : NULL; + st = (tls && tls->ssl) ? tls->ssl->state : -1; while ((err = ERR_get_error()) != 0) { msg = (const char*)ERR_reason_error_string(err); lib = (const char*)ERR_lib_error_string(err); func = (const char*)ERR_func_error_string(err); + if (!state) + state = (st>=0)?ssl_state_to_string(st):"---"; if (!msg) msg = "(null)"; if (!lib) lib = "(null)"; if (!func) func = "(null)"; if (doing) { - log(severity, LD_NET, "TLS error while %s%s%s: %s (in %s:%s)", + log(severity, domain, "TLS error while %s%s%s: %s (in %s:%s:%s)", doing, addr?" with ":"", addr?addr:"", - msg, lib, func); + msg, lib, func, state); } else { - log(severity, LD_NET, "TLS error%s%s: %s (in %s:%s)", + log(severity, domain, "TLS error%s%s: %s (in %s:%s:%s)", addr?" with ":"", addr?addr:"", - msg, lib, func); + msg, lib, func, state); } } } @@ -294,7 +327,7 @@ tor_tls_err_to_string(int err) */ static int tor_tls_get_error(tor_tls_t *tls, int r, int extra, - const char *doing, int severity) + const char *doing, int severity, int domain) { int err = SSL_get_error(tls->ssl, r); int tor_error = TOR_TLS_ERROR_MISC; @@ -309,25 +342,28 @@ tor_tls_get_error(tor_tls_t *tls, int r, int extra, if (extra&CATCH_SYSCALL) return _TOR_TLS_SYSCALL; if (r == 0) { - log(severity, LD_NET, "TLS error: unexpected close while %s", doing); + log(severity, LD_NET, "TLS error: unexpected close while %s (%s)", + doing, ssl_state_to_string(tls->ssl->state)); tor_error = TOR_TLS_ERROR_IO; } else { int e = tor_socket_errno(tls->socket); log(severity, LD_NET, - "TLS error: <syscall error while %s> (errno=%d: %s)", - doing, e, tor_socket_strerror(e)); + "TLS error: <syscall error while %s> (errno=%d: %s; state=%s)", + doing, e, tor_socket_strerror(e), + ssl_state_to_string(tls->ssl->state)); tor_error = tor_errno_to_tls_error(e); } - tls_log_errors(tls, severity, doing); + tls_log_errors(tls, severity, domain, doing); return tor_error; case SSL_ERROR_ZERO_RETURN: if (extra&CATCH_ZERO) return _TOR_TLS_ZERORETURN; - log(severity, LD_NET, "TLS connection closed while %s", doing); - tls_log_errors(tls, severity, doing); + log(severity, LD_NET, "TLS connection closed while %s in state %s", + doing, ssl_state_to_string(tls->ssl->state)); + tls_log_errors(tls, severity, domain, doing); return TOR_TLS_CLOSE; default: - tls_log_errors(tls, severity, doing); + tls_log_errors(tls, severity, domain, doing); return TOR_TLS_ERROR_MISC; } } @@ -341,11 +377,10 @@ tor_tls_init(void) long version; SSL_library_init(); SSL_load_error_strings(); - crypto_global_init(-1); version = SSLeay(); - /* OpenSSL 0.9.8l introdeced SSL3_FLAGS_ALLOW_UNSAGE_LEGACY_RENEGOTIATION + /* OpenSSL 0.9.8l introduced SSL3_FLAGS_ALLOW_UNSAFE_LEGACY_RENEGOTIATION * here, but without thinking too hard about it: it turns out that the * flag in question needed to be set at the last minute, and that it * conflicted with an existing flag number that had already been added @@ -364,8 +399,8 @@ tor_tls_init(void) * leave their headers out of sync with their libraries. * * Yes, it _is_ almost as if the OpenSSL developers decided that no - * program should be allowed to use renegotiation its first passed an - * test of intelligence and determination. + * program should be allowed to use renegotiation unless it first passed + * a test of intelligence and determination. */ if (version >= 0x009080c0L && version < 0x009080d0L) { log_notice(LD_GENERAL, "OpenSSL %s looks like version 0.9.8l; " @@ -400,9 +435,15 @@ tor_tls_init(void) void tor_tls_free_all(void) { - if (global_tls_context) { - tor_tls_context_decref(global_tls_context); - global_tls_context = NULL; + if (server_tls_context) { + tor_tls_context_t *ctx = server_tls_context; + server_tls_context = NULL; + tor_tls_context_decref(ctx); + } + if (client_tls_context) { + tor_tls_context_t *ctx = client_tls_context; + client_tls_context = NULL; + tor_tls_context_decref(ctx); } if (!HT_EMPTY(&tlsmap_root)) { log_warn(LD_MM, "Still have entries in the tlsmap at shutdown."); @@ -511,7 +552,7 @@ tor_tls_create_certificate(crypto_pk_env_t *rsa, x509 = NULL; } done: - tls_log_errors(NULL, LOG_WARN, "generating certificate"); + tls_log_errors(NULL, LOG_WARN, LD_NET, "generating certificate"); if (sign_pkey) EVP_PKEY_free(sign_pkey); if (pkey) @@ -528,9 +569,9 @@ tor_tls_create_certificate(crypto_pk_env_t *rsa, (TLS1_TXT_DHE_RSA_WITH_AES_256_SHA ":" \ TLS1_TXT_DHE_RSA_WITH_AES_128_SHA ":" \ SSL3_TXT_EDH_RSA_DES_192_CBC3_SHA) -/* Note: for setting up your own private testing network with link crypto - * disabled, set the cipher lists to your cipher list to - * SSL3_TXT_RSA_NULL_SHA. If you do this, you won't be able to communicate +/* Note: to set up your own private testing network with link crypto + * disabled, set your Tors' cipher list to + * (SSL3_TXT_RSA_NULL_SHA). If you do this, you won't be able to communicate * with any of the "real" Tors, though. */ #ifdef V2_HANDSHAKE_CLIENT @@ -589,15 +630,97 @@ tor_tls_context_incref(tor_tls_context_t *ctx) ++ctx->refcnt; } -/** Create a new TLS context for use with Tor TLS handshakes. - * <b>identity</b> should be set to the identity key used to sign the - * certificate, and <b>nickname</b> set to the nickname to use. +/** Create new global client and server TLS contexts. + * + * If <b>server_identity</b> is NULL, this will not generate a server + * TLS context. If <b>is_public_server</b> is non-zero, this will use + * the same TLS context for incoming and outgoing connections, and + * ignore <b>client_identity</b>. */ +int +tor_tls_context_init(int is_public_server, + crypto_pk_env_t *client_identity, + crypto_pk_env_t *server_identity, + unsigned int key_lifetime) +{ + int rv1 = 0; + int rv2 = 0; + + if (is_public_server) { + tor_tls_context_t *new_ctx; + tor_tls_context_t *old_ctx; + + tor_assert(server_identity != NULL); + + rv1 = tor_tls_context_init_one(&server_tls_context, + server_identity, + key_lifetime); + + if (rv1 >= 0) { + new_ctx = server_tls_context; + tor_tls_context_incref(new_ctx); + old_ctx = client_tls_context; + client_tls_context = new_ctx; + + if (old_ctx != NULL) { + tor_tls_context_decref(old_ctx); + } + } + } else { + if (server_identity != NULL) { + rv1 = tor_tls_context_init_one(&server_tls_context, + server_identity, + key_lifetime); + } else { + tor_tls_context_t *old_ctx = server_tls_context; + server_tls_context = NULL; + + if (old_ctx != NULL) { + tor_tls_context_decref(old_ctx); + } + } + + rv2 = tor_tls_context_init_one(&client_tls_context, + client_identity, + key_lifetime); + } + + return MIN(rv1, rv2); +} + +/** Create a new global TLS context. * * You can call this function multiple times. Each time you call it, * it generates new certificates; all new connections will use * the new SSL context. */ -int +static int +tor_tls_context_init_one(tor_tls_context_t **ppcontext, + crypto_pk_env_t *identity, + unsigned int key_lifetime) +{ + tor_tls_context_t *new_ctx = tor_tls_context_new(identity, + key_lifetime); + tor_tls_context_t *old_ctx = *ppcontext; + + if (new_ctx != NULL) { + *ppcontext = new_ctx; + + /* Free the old context if one existed. */ + if (old_ctx != NULL) { + /* This is safe even if there are open connections: we reference- + * count tor_tls_context_t objects. */ + tor_tls_context_decref(old_ctx); + } + } + + return ((new_ctx != NULL) ? 0 : -1); +} + +/** Create a new TLS context for use with Tor TLS handshakes. + * <b>identity</b> should be set to the identity key used to sign the + * certificate. + */ +static tor_tls_context_t * tor_tls_context_new(crypto_pk_env_t *identity, unsigned int key_lifetime) { crypto_pk_env_t *rsa = NULL; @@ -685,6 +808,7 @@ tor_tls_context_new(crypto_pk_env_t *identity, unsigned int key_lifetime) goto error; { crypto_dh_env_t *dh = crypto_dh_new(DH_TYPE_TLS); + tor_assert(dh); SSL_CTX_set_tmp_dh(result->ctx, _crypto_dh_env_get_dh(dh)); crypto_dh_free(dh); } @@ -692,21 +816,15 @@ tor_tls_context_new(crypto_pk_env_t *identity, unsigned int key_lifetime) always_accept_verify_cb); /* let us realloc bufs that we're writing from */ SSL_CTX_set_mode(result->ctx, SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER); - /* Free the old context if one exists. */ - if (global_tls_context) { - /* This is safe even if there are open connections: OpenSSL does - * reference counting with SSL and SSL_CTX objects. */ - tor_tls_context_decref(global_tls_context); - } - global_tls_context = result; + if (rsa) crypto_free_pk_env(rsa); tor_free(nickname); tor_free(nn2); - return 0; + return result; error: - tls_log_errors(NULL, LOG_WARN, "creating TLS context"); + tls_log_errors(NULL, LOG_WARN, LD_NET, "creating TLS context"); tor_free(nickname); tor_free(nn2); if (pkey) @@ -719,7 +837,7 @@ tor_tls_context_new(crypto_pk_env_t *identity, unsigned int key_lifetime) X509_free(cert); if (idcert) X509_free(idcert); - return -1; + return NULL; } #ifdef V2_HANDSHAKE_SERVER @@ -734,11 +852,11 @@ tor_tls_client_is_using_v2_ciphers(const SSL *ssl, const char *address) /* If we reached this point, we just got a client hello. See if there is * a cipher list. */ if (!(session = SSL_get_session((SSL *)ssl))) { - log_warn(LD_NET, "No session on TLS?"); + log_info(LD_NET, "No session on TLS?"); return 0; } if (!session->ciphers) { - log_warn(LD_NET, "No ciphers on session"); + log_info(LD_NET, "No ciphers on session"); return 0; } /* Now we need to see if there are any ciphers whose presence means we're @@ -750,8 +868,7 @@ tor_tls_client_is_using_v2_ciphers(const SSL *ssl, const char *address) strcmp(ciphername, TLS1_TXT_DHE_RSA_WITH_AES_256_SHA) && strcmp(ciphername, SSL3_TXT_EDH_RSA_DES_192_CBC3_SHA) && strcmp(ciphername, "(NONE)")) { - /* XXXX should be ld_debug */ - log_info(LD_NET, "Got a non-version-1 cipher called '%s'", ciphername); + log_debug(LD_NET, "Got a non-version-1 cipher called '%s'", ciphername); // return 1; goto dump_list; } @@ -767,8 +884,8 @@ tor_tls_client_is_using_v2_ciphers(const SSL *ssl, const char *address) smartlist_add(elts, (char*)ciphername); } s = smartlist_join_strings(elts, ":", 0, NULL); - log_info(LD_NET, "Got a non-version-1 cipher list from %s. It is: '%s'", - address, s); + log_debug(LD_NET, "Got a non-version-1 cipher list from %s. It is: '%s'", + address, s); tor_free(s); smartlist_free(elts); } @@ -899,10 +1016,12 @@ tor_tls_new(int sock, int isServer) { BIO *bio = NULL; tor_tls_t *result = tor_malloc_zero(sizeof(tor_tls_t)); + tor_tls_context_t *context = isServer ? server_tls_context : + client_tls_context; - tor_assert(global_tls_context); /* make sure somebody made it first */ - if (!(result->ssl = SSL_new(global_tls_context->ctx))) { - tls_log_errors(NULL, LOG_WARN, "generating TLS context"); + tor_assert(context); /* make sure somebody made it first */ + if (!(result->ssl = SSL_new(context->ctx))) { + tls_log_errors(NULL, LOG_WARN, LD_NET, "creating SSL object"); tor_free(result); return NULL; } @@ -918,7 +1037,7 @@ tor_tls_new(int sock, int isServer) if (!SSL_set_cipher_list(result->ssl, isServer ? SERVER_CIPHER_LIST : CLIENT_CIPHER_LIST)) { - tls_log_errors(NULL, LOG_WARN, "setting ciphers"); + tls_log_errors(NULL, LOG_WARN, LD_NET, "setting ciphers"); #ifdef SSL_set_tlsext_host_name SSL_set_tlsext_host_name(result->ssl, NULL); #endif @@ -931,7 +1050,7 @@ tor_tls_new(int sock, int isServer) result->socket = sock; bio = BIO_new_socket(sock, BIO_NOCLOSE); if (! bio) { - tls_log_errors(NULL, LOG_WARN, "opening BIO"); + tls_log_errors(NULL, LOG_WARN, LD_NET, "opening BIO"); #ifdef SSL_set_tlsext_host_name SSL_set_tlsext_host_name(result->ssl, NULL); #endif @@ -941,8 +1060,8 @@ tor_tls_new(int sock, int isServer) } HT_INSERT(tlsmap, &tlsmap_root, result); SSL_set_bio(result->ssl, bio, bio); - tor_tls_context_incref(global_tls_context); - result->context = global_tls_context; + tor_tls_context_incref(context); + result->context = context; result->state = TOR_TLS_ST_HANDSHAKE; result->isServer = isServer; result->wantwrite_n = 0; @@ -959,7 +1078,7 @@ tor_tls_new(int sock, int isServer) #endif /* Not expected to get called. */ - tls_log_errors(NULL, LOG_WARN, "generating TLS context"); + tls_log_errors(NULL, LOG_WARN, LD_NET, "creating tor_tls_t object"); return result; } @@ -1038,7 +1157,9 @@ void tor_tls_free(tor_tls_t *tls) { tor_tls_t *removed; - tor_assert(tls && tls->ssl); + if (!tls) + return; + tor_assert(tls->ssl); removed = HT_REMOVE(tlsmap, &tlsmap_root, tls); if (!removed) { log_warn(LD_BUG, "Freeing a TLS that was not in the ssl->tls map."); @@ -1081,7 +1202,7 @@ tor_tls_read(tor_tls_t *tls, char *cp, size_t len) #endif return r; } - err = tor_tls_get_error(tls, r, CATCH_ZERO, "reading", LOG_DEBUG); + err = tor_tls_get_error(tls, r, CATCH_ZERO, "reading", LOG_DEBUG, LD_NET); if (err == _TOR_TLS_ZERORETURN || err == TOR_TLS_CLOSE) { log_debug(LD_NET,"read returned r=%d; TLS is closed",r); tls->state = TOR_TLS_ST_CLOSED; @@ -1117,7 +1238,7 @@ tor_tls_write(tor_tls_t *tls, const char *cp, size_t n) tls->wantwrite_n = 0; } r = SSL_write(tls->ssl, cp, (int)n); - err = tor_tls_get_error(tls, r, 0, "writing", LOG_INFO); + err = tor_tls_get_error(tls, r, 0, "writing", LOG_INFO, LD_NET); if (err == TOR_TLS_DONE) { return r; } @@ -1135,21 +1256,30 @@ int tor_tls_handshake(tor_tls_t *tls) { int r; + int oldstate; tor_assert(tls); tor_assert(tls->ssl); tor_assert(tls->state == TOR_TLS_ST_HANDSHAKE); check_no_tls_errors(); + oldstate = tls->ssl->state; if (tls->isServer) { + log_debug(LD_HANDSHAKE, "About to call SSL_accept on %p (%s)", tls, + ssl_state_to_string(tls->ssl->state)); r = SSL_accept(tls->ssl); } else { + log_debug(LD_HANDSHAKE, "About to call SSL_connect on %p (%s)", tls, + ssl_state_to_string(tls->ssl->state)); r = SSL_connect(tls->ssl); } + if (oldstate != tls->ssl->state) + log_debug(LD_HANDSHAKE, "After call, %p was in state %s", + tls, ssl_state_to_string(tls->ssl->state)); /* We need to call this here and not earlier, since OpenSSL has a penchant * for clearing its flags when you say accept or connect. */ tor_tls_unblock_renegotiation(tls); - r = tor_tls_get_error(tls,r,0, "handshaking", LOG_INFO); + r = tor_tls_get_error(tls,r,0, "handshaking", LOG_INFO, LD_HANDSHAKE); if (ERR_peek_error() != 0) { - tls_log_errors(tls, tls->isServer ? LOG_INFO : LOG_WARN, + tls_log_errors(tls, tls->isServer ? LOG_INFO : LOG_WARN, LD_HANDSHAKE, "handshaking"); return TOR_TLS_ERROR_MISC; } @@ -1170,7 +1300,8 @@ tor_tls_handshake(tor_tls_t *tls) " get set. Fixing that."); } tls->wasV2Handshake = 1; - log_debug(LD_NET, "Completed V2 TLS handshake with client; waiting " + log_debug(LD_HANDSHAKE, + "Completed V2 TLS handshake with client; waiting " "for renegotiation."); } else { tls->wasV2Handshake = 0; @@ -1182,10 +1313,13 @@ tor_tls_handshake(tor_tls_t *tls) X509 *cert = SSL_get_peer_certificate(tls->ssl); STACK_OF(X509) *chain = SSL_get_peer_cert_chain(tls->ssl); int n_certs = sk_X509_num(chain); - if (n_certs > 1 || (n_certs == 1 && cert != sk_X509_value(chain, 0))) + if (n_certs > 1 || (n_certs == 1 && cert != sk_X509_value(chain, 0))) { + log_debug(LD_HANDSHAKE, "Server sent back multiple certificates; it " + "looks like a v1 handshake on %p", tls); tls->wasV2Handshake = 0; - else { - log_debug(LD_NET, "Server sent back a single certificate; looks like " + } else { + log_debug(LD_HANDSHAKE, + "Server sent back a single certificate; looks like " "a v2 handshake on %p.", tls); tls->wasV2Handshake = 1; } @@ -1193,7 +1327,7 @@ tor_tls_handshake(tor_tls_t *tls) X509_free(cert); #endif if (SSL_set_cipher_list(tls->ssl, SERVER_CIPHER_LIST) == 0) { - tls_log_errors(NULL, LOG_WARN, "re-setting ciphers"); + tls_log_errors(NULL, LOG_WARN, LD_HANDSHAKE, "re-setting ciphers"); r = TOR_TLS_ERROR_MISC; } } @@ -1216,7 +1350,8 @@ tor_tls_renegotiate(tor_tls_t *tls) if (tls->state != TOR_TLS_ST_RENEGOTIATE) { int r = SSL_renegotiate(tls->ssl); if (r <= 0) { - return tor_tls_get_error(tls, r, 0, "renegotiating", LOG_WARN); + return tor_tls_get_error(tls, r, 0, "renegotiating", LOG_WARN, + LD_HANDSHAKE); } tls->state = TOR_TLS_ST_RENEGOTIATE; } @@ -1225,7 +1360,8 @@ tor_tls_renegotiate(tor_tls_t *tls) tls->state = TOR_TLS_ST_OPEN; return TOR_TLS_DONE; } else - return tor_tls_get_error(tls, r, 0, "renegotiating handshake", LOG_INFO); + return tor_tls_get_error(tls, r, 0, "renegotiating handshake", LOG_INFO, + LD_HANDSHAKE); } /** Shut down an open tls connection <b>tls</b>. When finished, returns @@ -1249,7 +1385,7 @@ tor_tls_shutdown(tor_tls_t *tls) r = SSL_read(tls->ssl, buf, 128); } while (r>0); err = tor_tls_get_error(tls, r, CATCH_ZERO, "reading to shut down", - LOG_INFO); + LOG_INFO, LD_NET); if (err == _TOR_TLS_ZERORETURN) { tls->state = TOR_TLS_ST_GOTCLOSE; /* fall through... */ @@ -1265,7 +1401,7 @@ tor_tls_shutdown(tor_tls_t *tls) return TOR_TLS_DONE; } err = tor_tls_get_error(tls, r, CATCH_SYSCALL|CATCH_ZERO, "shutting down", - LOG_INFO); + LOG_INFO, LD_NET); if (err == _TOR_TLS_SYSCALL) { /* The underlying TCP connection closed while we were shutting down. */ tls->state = TOR_TLS_ST_CLOSED; @@ -1297,7 +1433,7 @@ tor_tls_peer_has_cert(tor_tls_t *tls) { X509 *cert; cert = SSL_get_peer_certificate(tls->ssl); - tls_log_errors(tls, LOG_WARN, "getting peer certificate"); + tls_log_errors(tls, LOG_WARN, LD_HANDSHAKE, "getting peer certificate"); if (!cert) return 0; X509_free(cert); @@ -1324,7 +1460,7 @@ log_cert_lifetime(X509 *cert, const char *problem) log_warn(LD_GENERAL, "Couldn't allocate BIO!"); goto end; } if (!(ASN1_TIME_print(bio, X509_get_notBefore(cert)))) { - tls_log_errors(NULL, LOG_WARN, "printing certificate lifetime"); + tls_log_errors(NULL, LOG_WARN, LD_NET, "printing certificate lifetime"); goto end; } BIO_get_mem_ptr(bio, &buf); @@ -1332,7 +1468,7 @@ log_cert_lifetime(X509 *cert, const char *problem) (void)BIO_reset(bio); if (!(ASN1_TIME_print(bio, X509_get_notAfter(cert)))) { - tls_log_errors(NULL, LOG_WARN, "printing certificate lifetime"); + tls_log_errors(NULL, LOG_WARN, LD_NET, "printing certificate lifetime"); goto end; } BIO_get_mem_ptr(bio, &buf); @@ -1346,13 +1482,11 @@ log_cert_lifetime(X509 *cert, const char *problem) end: /* Not expected to get invoked */ - tls_log_errors(NULL, LOG_WARN, "getting certificate lifetime"); + tls_log_errors(NULL, LOG_WARN, LD_NET, "getting certificate lifetime"); if (bio) BIO_free(bio); - if (s1) - tor_free(s1); - if (s2) - tor_free(s2); + tor_free(s1); + tor_free(s2); } /** Helper function: try to extract a link certificate and an identity @@ -1420,7 +1554,7 @@ tor_tls_verify(int severity, tor_tls_t *tls, crypto_pk_env_t **identity_key) if (!(id_pkey = X509_get_pubkey(id_cert)) || X509_verify(cert, id_pkey) <= 0) { log_fn(severity,LD_PROTOCOL,"X509_verify on cert and pkey returned <= 0"); - tls_log_errors(tls, severity,"verifying certificate"); + tls_log_errors(tls, severity, LD_HANDSHAKE, "verifying certificate"); goto done; } @@ -1439,7 +1573,7 @@ tor_tls_verify(int severity, tor_tls_t *tls, crypto_pk_env_t **identity_key) /* This should never get invoked, but let's make sure in case OpenSSL * acts unexpectedly. */ - tls_log_errors(tls, LOG_WARN, "finishing tor_tls_verify"); + tls_log_errors(tls, LOG_WARN, LD_HANDSHAKE, "finishing tor_tls_verify"); return r; } @@ -1478,7 +1612,7 @@ tor_tls_check_lifetime(tor_tls_t *tls, int tolerance) if (cert) X509_free(cert); /* Not expected to get invoked */ - tls_log_errors(tls, LOG_WARN, "checking certificate lifetime"); + tls_log_errors(tls, LOG_WARN, LD_NET, "checking certificate lifetime"); return r; } @@ -1546,7 +1680,7 @@ _check_no_tls_errors(const char *fname, int line) return; log(LOG_WARN, LD_CRYPTO, "Unhandled OpenSSL errors found at %s:%d: ", tor_fix_source_file(fname), line); - tls_log_errors(NULL, LOG_WARN, NULL); + tls_log_errors(NULL, LOG_WARN, LD_NET, NULL); } /** Return true iff the initial TLS connection at <b>tls</b> did not use a v2 diff --git a/src/common/tortls.h b/src/common/tortls.h index 9644b87f2c..55fee81aea 100644 --- a/src/common/tortls.h +++ b/src/common/tortls.h @@ -50,7 +50,10 @@ typedef struct tor_tls_t tor_tls_t; const char *tor_tls_err_to_string(int err); void tor_tls_free_all(void); -int tor_tls_context_new(crypto_pk_env_t *rsa, unsigned int key_lifetime); +int tor_tls_context_init(int is_public_server, + crypto_pk_env_t *client_identity, + crypto_pk_env_t *server_identity, + unsigned int key_lifetime); tor_tls_t *tor_tls_new(int sock, int is_server); void tor_tls_set_logged_address(tor_tls_t *tls, const char *address); void tor_tls_set_renegotiate_callback(tor_tls_t *tls, diff --git a/src/common/tortls_states.h b/src/common/tortls_states.h new file mode 100644 index 0000000000..dcff2479f6 --- /dev/null +++ b/src/common/tortls_states.h @@ -0,0 +1,414 @@ +/* Copyright (c) 2003, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/* Helper file: included only in tortls.c */ + +#ifndef _TORTLS_STATES_H +#define _TORTLS_STATES_H + +/* The main body of this file was mechanically generated with this + perl script: + + my %keys = (); + for $fn (@ARGV) { + open(F, $fn); + while (<F>) { + next unless /^#define ((?:SSL|DTLS)\w*_ST_\w*)/; + $keys{$1} = 1; + } + close(F); + } + for $k (sort keys %keys) { + print "#ifdef $k\n S($k),\n#endif\n" + } +*/ + +/** Mapping from allowed value of SSL.state to the name of C macro for that + * state. Used for debugging an openssl connection. */ +static const struct { int state; const char *name; } state_map[] = { +#define S(state) { state, #state } +#ifdef DTLS1_ST_CR_HELLO_VERIFY_REQUEST_A + S(DTLS1_ST_CR_HELLO_VERIFY_REQUEST_A), +#endif +#ifdef DTLS1_ST_CR_HELLO_VERIFY_REQUEST_B + S(DTLS1_ST_CR_HELLO_VERIFY_REQUEST_B), +#endif +#ifdef DTLS1_ST_SW_HELLO_VERIFY_REQUEST_A + S(DTLS1_ST_SW_HELLO_VERIFY_REQUEST_A), +#endif +#ifdef DTLS1_ST_SW_HELLO_VERIFY_REQUEST_B + S(DTLS1_ST_SW_HELLO_VERIFY_REQUEST_B), +#endif +#ifdef SSL23_ST_CR_SRVR_HELLO_A + S(SSL23_ST_CR_SRVR_HELLO_A), +#endif +#ifdef SSL23_ST_CR_SRVR_HELLO_B + S(SSL23_ST_CR_SRVR_HELLO_B), +#endif +#ifdef SSL23_ST_CW_CLNT_HELLO_A + S(SSL23_ST_CW_CLNT_HELLO_A), +#endif +#ifdef SSL23_ST_CW_CLNT_HELLO_B + S(SSL23_ST_CW_CLNT_HELLO_B), +#endif +#ifdef SSL23_ST_SR_CLNT_HELLO_A + S(SSL23_ST_SR_CLNT_HELLO_A), +#endif +#ifdef SSL23_ST_SR_CLNT_HELLO_B + S(SSL23_ST_SR_CLNT_HELLO_B), +#endif +#ifdef SSL2_ST_CLIENT_START_ENCRYPTION + S(SSL2_ST_CLIENT_START_ENCRYPTION), +#endif +#ifdef SSL2_ST_GET_CLIENT_FINISHED_A + S(SSL2_ST_GET_CLIENT_FINISHED_A), +#endif +#ifdef SSL2_ST_GET_CLIENT_FINISHED_B + S(SSL2_ST_GET_CLIENT_FINISHED_B), +#endif +#ifdef SSL2_ST_GET_CLIENT_HELLO_A + S(SSL2_ST_GET_CLIENT_HELLO_A), +#endif +#ifdef SSL2_ST_GET_CLIENT_HELLO_B + S(SSL2_ST_GET_CLIENT_HELLO_B), +#endif +#ifdef SSL2_ST_GET_CLIENT_HELLO_C + S(SSL2_ST_GET_CLIENT_HELLO_C), +#endif +#ifdef SSL2_ST_GET_CLIENT_MASTER_KEY_A + S(SSL2_ST_GET_CLIENT_MASTER_KEY_A), +#endif +#ifdef SSL2_ST_GET_CLIENT_MASTER_KEY_B + S(SSL2_ST_GET_CLIENT_MASTER_KEY_B), +#endif +#ifdef SSL2_ST_GET_SERVER_FINISHED_A + S(SSL2_ST_GET_SERVER_FINISHED_A), +#endif +#ifdef SSL2_ST_GET_SERVER_FINISHED_B + S(SSL2_ST_GET_SERVER_FINISHED_B), +#endif +#ifdef SSL2_ST_GET_SERVER_HELLO_A + S(SSL2_ST_GET_SERVER_HELLO_A), +#endif +#ifdef SSL2_ST_GET_SERVER_HELLO_B + S(SSL2_ST_GET_SERVER_HELLO_B), +#endif +#ifdef SSL2_ST_GET_SERVER_VERIFY_A + S(SSL2_ST_GET_SERVER_VERIFY_A), +#endif +#ifdef SSL2_ST_GET_SERVER_VERIFY_B + S(SSL2_ST_GET_SERVER_VERIFY_B), +#endif +#ifdef SSL2_ST_SEND_CLIENT_CERTIFICATE_A + S(SSL2_ST_SEND_CLIENT_CERTIFICATE_A), +#endif +#ifdef SSL2_ST_SEND_CLIENT_CERTIFICATE_B + S(SSL2_ST_SEND_CLIENT_CERTIFICATE_B), +#endif +#ifdef SSL2_ST_SEND_CLIENT_CERTIFICATE_C + S(SSL2_ST_SEND_CLIENT_CERTIFICATE_C), +#endif +#ifdef SSL2_ST_SEND_CLIENT_CERTIFICATE_D + S(SSL2_ST_SEND_CLIENT_CERTIFICATE_D), +#endif +#ifdef SSL2_ST_SEND_CLIENT_FINISHED_A + S(SSL2_ST_SEND_CLIENT_FINISHED_A), +#endif +#ifdef SSL2_ST_SEND_CLIENT_FINISHED_B + S(SSL2_ST_SEND_CLIENT_FINISHED_B), +#endif +#ifdef SSL2_ST_SEND_CLIENT_HELLO_A + S(SSL2_ST_SEND_CLIENT_HELLO_A), +#endif +#ifdef SSL2_ST_SEND_CLIENT_HELLO_B + S(SSL2_ST_SEND_CLIENT_HELLO_B), +#endif +#ifdef SSL2_ST_SEND_CLIENT_MASTER_KEY_A + S(SSL2_ST_SEND_CLIENT_MASTER_KEY_A), +#endif +#ifdef SSL2_ST_SEND_CLIENT_MASTER_KEY_B + S(SSL2_ST_SEND_CLIENT_MASTER_KEY_B), +#endif +#ifdef SSL2_ST_SEND_REQUEST_CERTIFICATE_A + S(SSL2_ST_SEND_REQUEST_CERTIFICATE_A), +#endif +#ifdef SSL2_ST_SEND_REQUEST_CERTIFICATE_B + S(SSL2_ST_SEND_REQUEST_CERTIFICATE_B), +#endif +#ifdef SSL2_ST_SEND_REQUEST_CERTIFICATE_C + S(SSL2_ST_SEND_REQUEST_CERTIFICATE_C), +#endif +#ifdef SSL2_ST_SEND_REQUEST_CERTIFICATE_D + S(SSL2_ST_SEND_REQUEST_CERTIFICATE_D), +#endif +#ifdef SSL2_ST_SEND_SERVER_FINISHED_A + S(SSL2_ST_SEND_SERVER_FINISHED_A), +#endif +#ifdef SSL2_ST_SEND_SERVER_FINISHED_B + S(SSL2_ST_SEND_SERVER_FINISHED_B), +#endif +#ifdef SSL2_ST_SEND_SERVER_HELLO_A + S(SSL2_ST_SEND_SERVER_HELLO_A), +#endif +#ifdef SSL2_ST_SEND_SERVER_HELLO_B + S(SSL2_ST_SEND_SERVER_HELLO_B), +#endif +#ifdef SSL2_ST_SEND_SERVER_VERIFY_A + S(SSL2_ST_SEND_SERVER_VERIFY_A), +#endif +#ifdef SSL2_ST_SEND_SERVER_VERIFY_B + S(SSL2_ST_SEND_SERVER_VERIFY_B), +#endif +#ifdef SSL2_ST_SEND_SERVER_VERIFY_C + S(SSL2_ST_SEND_SERVER_VERIFY_C), +#endif +#ifdef SSL2_ST_SERVER_START_ENCRYPTION + S(SSL2_ST_SERVER_START_ENCRYPTION), +#endif +#ifdef SSL2_ST_X509_GET_CLIENT_CERTIFICATE + S(SSL2_ST_X509_GET_CLIENT_CERTIFICATE), +#endif +#ifdef SSL2_ST_X509_GET_SERVER_CERTIFICATE + S(SSL2_ST_X509_GET_SERVER_CERTIFICATE), +#endif +#ifdef SSL3_ST_CR_CERT_A + S(SSL3_ST_CR_CERT_A), +#endif +#ifdef SSL3_ST_CR_CERT_B + S(SSL3_ST_CR_CERT_B), +#endif +#ifdef SSL3_ST_CR_CERT_REQ_A + S(SSL3_ST_CR_CERT_REQ_A), +#endif +#ifdef SSL3_ST_CR_CERT_REQ_B + S(SSL3_ST_CR_CERT_REQ_B), +#endif +#ifdef SSL3_ST_CR_CERT_STATUS_A + S(SSL3_ST_CR_CERT_STATUS_A), +#endif +#ifdef SSL3_ST_CR_CERT_STATUS_B + S(SSL3_ST_CR_CERT_STATUS_B), +#endif +#ifdef SSL3_ST_CR_CHANGE_A + S(SSL3_ST_CR_CHANGE_A), +#endif +#ifdef SSL3_ST_CR_CHANGE_B + S(SSL3_ST_CR_CHANGE_B), +#endif +#ifdef SSL3_ST_CR_FINISHED_A + S(SSL3_ST_CR_FINISHED_A), +#endif +#ifdef SSL3_ST_CR_FINISHED_B + S(SSL3_ST_CR_FINISHED_B), +#endif +#ifdef SSL3_ST_CR_KEY_EXCH_A + S(SSL3_ST_CR_KEY_EXCH_A), +#endif +#ifdef SSL3_ST_CR_KEY_EXCH_B + S(SSL3_ST_CR_KEY_EXCH_B), +#endif +#ifdef SSL3_ST_CR_SESSION_TICKET_A + S(SSL3_ST_CR_SESSION_TICKET_A), +#endif +#ifdef SSL3_ST_CR_SESSION_TICKET_B + S(SSL3_ST_CR_SESSION_TICKET_B), +#endif +#ifdef SSL3_ST_CR_SRVR_DONE_A + S(SSL3_ST_CR_SRVR_DONE_A), +#endif +#ifdef SSL3_ST_CR_SRVR_DONE_B + S(SSL3_ST_CR_SRVR_DONE_B), +#endif +#ifdef SSL3_ST_CR_SRVR_HELLO_A + S(SSL3_ST_CR_SRVR_HELLO_A), +#endif +#ifdef SSL3_ST_CR_SRVR_HELLO_B + S(SSL3_ST_CR_SRVR_HELLO_B), +#endif +#ifdef SSL3_ST_CW_CERT_A + S(SSL3_ST_CW_CERT_A), +#endif +#ifdef SSL3_ST_CW_CERT_B + S(SSL3_ST_CW_CERT_B), +#endif +#ifdef SSL3_ST_CW_CERT_C + S(SSL3_ST_CW_CERT_C), +#endif +#ifdef SSL3_ST_CW_CERT_D + S(SSL3_ST_CW_CERT_D), +#endif +#ifdef SSL3_ST_CW_CERT_VRFY_A + S(SSL3_ST_CW_CERT_VRFY_A), +#endif +#ifdef SSL3_ST_CW_CERT_VRFY_B + S(SSL3_ST_CW_CERT_VRFY_B), +#endif +#ifdef SSL3_ST_CW_CHANGE_A + S(SSL3_ST_CW_CHANGE_A), +#endif +#ifdef SSL3_ST_CW_CHANGE_B + S(SSL3_ST_CW_CHANGE_B), +#endif +#ifdef SSL3_ST_CW_CLNT_HELLO_A + S(SSL3_ST_CW_CLNT_HELLO_A), +#endif +#ifdef SSL3_ST_CW_CLNT_HELLO_B + S(SSL3_ST_CW_CLNT_HELLO_B), +#endif +#ifdef SSL3_ST_CW_FINISHED_A + S(SSL3_ST_CW_FINISHED_A), +#endif +#ifdef SSL3_ST_CW_FINISHED_B + S(SSL3_ST_CW_FINISHED_B), +#endif +#ifdef SSL3_ST_CW_FLUSH + S(SSL3_ST_CW_FLUSH), +#endif +#ifdef SSL3_ST_CW_KEY_EXCH_A + S(SSL3_ST_CW_KEY_EXCH_A), +#endif +#ifdef SSL3_ST_CW_KEY_EXCH_B + S(SSL3_ST_CW_KEY_EXCH_B), +#endif +#ifdef SSL3_ST_SR_CERT_A + S(SSL3_ST_SR_CERT_A), +#endif +#ifdef SSL3_ST_SR_CERT_B + S(SSL3_ST_SR_CERT_B), +#endif +#ifdef SSL3_ST_SR_CERT_VRFY_A + S(SSL3_ST_SR_CERT_VRFY_A), +#endif +#ifdef SSL3_ST_SR_CERT_VRFY_B + S(SSL3_ST_SR_CERT_VRFY_B), +#endif +#ifdef SSL3_ST_SR_CHANGE_A + S(SSL3_ST_SR_CHANGE_A), +#endif +#ifdef SSL3_ST_SR_CHANGE_B + S(SSL3_ST_SR_CHANGE_B), +#endif +#ifdef SSL3_ST_SR_CLNT_HELLO_A + S(SSL3_ST_SR_CLNT_HELLO_A), +#endif +#ifdef SSL3_ST_SR_CLNT_HELLO_B + S(SSL3_ST_SR_CLNT_HELLO_B), +#endif +#ifdef SSL3_ST_SR_CLNT_HELLO_C + S(SSL3_ST_SR_CLNT_HELLO_C), +#endif +#ifdef SSL3_ST_SR_FINISHED_A + S(SSL3_ST_SR_FINISHED_A), +#endif +#ifdef SSL3_ST_SR_FINISHED_B + S(SSL3_ST_SR_FINISHED_B), +#endif +#ifdef SSL3_ST_SR_KEY_EXCH_A + S(SSL3_ST_SR_KEY_EXCH_A), +#endif +#ifdef SSL3_ST_SR_KEY_EXCH_B + S(SSL3_ST_SR_KEY_EXCH_B), +#endif +#ifdef SSL3_ST_SW_CERT_A + S(SSL3_ST_SW_CERT_A), +#endif +#ifdef SSL3_ST_SW_CERT_B + S(SSL3_ST_SW_CERT_B), +#endif +#ifdef SSL3_ST_SW_CERT_REQ_A + S(SSL3_ST_SW_CERT_REQ_A), +#endif +#ifdef SSL3_ST_SW_CERT_REQ_B + S(SSL3_ST_SW_CERT_REQ_B), +#endif +#ifdef SSL3_ST_SW_CERT_STATUS_A + S(SSL3_ST_SW_CERT_STATUS_A), +#endif +#ifdef SSL3_ST_SW_CERT_STATUS_B + S(SSL3_ST_SW_CERT_STATUS_B), +#endif +#ifdef SSL3_ST_SW_CHANGE_A + S(SSL3_ST_SW_CHANGE_A), +#endif +#ifdef SSL3_ST_SW_CHANGE_B + S(SSL3_ST_SW_CHANGE_B), +#endif +#ifdef SSL3_ST_SW_FINISHED_A + S(SSL3_ST_SW_FINISHED_A), +#endif +#ifdef SSL3_ST_SW_FINISHED_B + S(SSL3_ST_SW_FINISHED_B), +#endif +#ifdef SSL3_ST_SW_FLUSH + S(SSL3_ST_SW_FLUSH), +#endif +#ifdef SSL3_ST_SW_HELLO_REQ_A + S(SSL3_ST_SW_HELLO_REQ_A), +#endif +#ifdef SSL3_ST_SW_HELLO_REQ_B + S(SSL3_ST_SW_HELLO_REQ_B), +#endif +#ifdef SSL3_ST_SW_HELLO_REQ_C + S(SSL3_ST_SW_HELLO_REQ_C), +#endif +#ifdef SSL3_ST_SW_KEY_EXCH_A + S(SSL3_ST_SW_KEY_EXCH_A), +#endif +#ifdef SSL3_ST_SW_KEY_EXCH_B + S(SSL3_ST_SW_KEY_EXCH_B), +#endif +#ifdef SSL3_ST_SW_SESSION_TICKET_A + S(SSL3_ST_SW_SESSION_TICKET_A), +#endif +#ifdef SSL3_ST_SW_SESSION_TICKET_B + S(SSL3_ST_SW_SESSION_TICKET_B), +#endif +#ifdef SSL3_ST_SW_SRVR_DONE_A + S(SSL3_ST_SW_SRVR_DONE_A), +#endif +#ifdef SSL3_ST_SW_SRVR_DONE_B + S(SSL3_ST_SW_SRVR_DONE_B), +#endif +#ifdef SSL3_ST_SW_SRVR_HELLO_A + S(SSL3_ST_SW_SRVR_HELLO_A), +#endif +#ifdef SSL3_ST_SW_SRVR_HELLO_B + S(SSL3_ST_SW_SRVR_HELLO_B), +#endif +#ifdef SSL_ST_ACCEPT + S(SSL_ST_ACCEPT), +#endif +#ifdef SSL_ST_BEFORE + S(SSL_ST_BEFORE), +#endif +#ifdef SSL_ST_CONNECT + S(SSL_ST_CONNECT), +#endif +#ifdef SSL_ST_INIT + S(SSL_ST_INIT), +#endif +#ifdef SSL_ST_MASK + S(SSL_ST_MASK), +#endif +#ifdef SSL_ST_OK + S(SSL_ST_OK), +#endif +#ifdef SSL_ST_READ_BODY + S(SSL_ST_READ_BODY), +#endif +#ifdef SSL_ST_READ_DONE + S(SSL_ST_READ_DONE), +#endif +#ifdef SSL_ST_READ_HEADER + S(SSL_ST_READ_HEADER), +#endif +#ifdef SSL_ST_RENEGOTIATE + S(SSL_ST_RENEGOTIATE), +#endif + { 0, NULL } +}; + +#endif + diff --git a/src/common/util.c b/src/common/util.c index f206d00c49..38c0ad05e6 100644 --- a/src/common/util.c +++ b/src/common/util.c @@ -15,7 +15,8 @@ #include "orconfig.h" #include "util.h" -#include "log.h" +#include "torlog.h" +#undef log #include "crypto.h" #include "torint.h" #include "container.h" @@ -25,11 +26,17 @@ #include <io.h> #include <direct.h> #include <process.h> +#include <tchar.h> #else #include <dirent.h> #include <pwd.h> #endif +/* math.h needs this on Linux */ +#ifndef __USE_ISOC99 +#define __USE_ISOC99 1 +#endif +#include <math.h> #include <stdlib.h> #include <stdio.h> #include <string.h> @@ -287,7 +294,7 @@ tor_log_mallinfo(int severity) struct mallinfo mi; memset(&mi, 0, sizeof(mi)); mi = mallinfo(); - log(severity, LD_MM, + tor_log(severity, LD_MM, "mallinfo() said: arena=%d, ordblks=%d, smblks=%d, hblks=%d, " "hblkhd=%d, usmblks=%d, fsmblks=%d, uordblks=%d, fordblks=%d, " "keepcost=%d", @@ -310,6 +317,25 @@ tor_log_mallinfo(int severity) * Math * ===== */ +/** + * Returns the natural logarithm of d base 2. We define this wrapper here so + * as to make it easier not to conflict with Tor's log() macro. + */ +double +tor_mathlog(double d) +{ + return log(d); +} + +/** Return the long integer closest to d. We define this wrapper here so + * that not all users of math.h need to use the right incancations to get + * the c99 functions. */ +long +tor_lround(double d) +{ + return lround(d); +} + /** Returns floor(log2(u64)). If u64 is 0, (incorrectly) returns 0. */ int tor_log2(uint64_t u64) @@ -354,6 +380,36 @@ round_to_power_of_2(uint64_t u64) return low; } +/** Return the lowest x such that x is at least <b>number</b>, and x modulo + * <b>divisor</b> == 0. */ +unsigned +round_to_next_multiple_of(unsigned number, unsigned divisor) +{ + number += divisor - 1; + number -= number % divisor; + return number; +} + +/** Return the lowest x such that x is at least <b>number</b>, and x modulo + * <b>divisor</b> == 0. */ +uint32_t +round_uint32_to_next_multiple_of(uint32_t number, uint32_t divisor) +{ + number += divisor - 1; + number -= number % divisor; + return number; +} + +/** Return the lowest x such that x is at least <b>number</b>, and x modulo + * <b>divisor</b> == 0. */ +uint64_t +round_uint64_to_next_multiple_of(uint64_t number, uint64_t divisor) +{ + number += divisor - 1; + number -= number % divisor; + return number; +} + /* ===== * String manipulation * ===== */ @@ -636,6 +692,29 @@ find_whitespace_eos(const char *s, const char *eos) return s; } +/** Return the first occurrence of <b>needle</b> in <b>haystack</b> that + * occurs at the start of a line (that is, at the beginning of <b>haystack</b> + * or immediately after a newline). Return NULL if no such string is found. + */ +const char * +find_str_at_start_of_line(const char *haystack, const char *needle) +{ + size_t needle_len = strlen(needle); + + do { + if (!strncmp(haystack, needle, needle_len)) + return haystack; + + haystack = strchr(haystack, '\n'); + if (!haystack) + return NULL; + else + ++haystack; + } while (*haystack); + + return NULL; +} + /** Return true iff the 'len' bytes at 'mem' are all zero. */ int tor_mem_is_zero(const char *mem, size_t len) @@ -663,6 +742,13 @@ tor_digest_is_zero(const char *digest) return tor_mem_is_zero(digest, DIGEST_LEN); } +/** Return true iff the DIGEST256_LEN bytes in digest are all zero. */ +int +tor_digest256_is_zero(const char *digest) +{ + return tor_mem_is_zero(digest, DIGEST256_LEN); +} + /* Helper: common code to check whether the result of a strtol or strtoul or * strtoll is correct. */ #define CHECK_STRTOX_RESULT() \ @@ -714,7 +800,18 @@ tor_parse_ulong(const char *s, int base, unsigned long min, CHECK_STRTOX_RESULT(); } -/** As tor_parse_log, but return a unit64_t. Only base 10 is guaranteed to +/** As tor_parse_long(), but return a double. */ +double +tor_parse_double(const char *s, double min, double max, int *ok, char **next) +{ + char *endptr; + double r; + + r = strtod(s, &endptr); + CHECK_STRTOX_RESULT(); +} + +/** As tor_parse_long, but return a uint64_t. Only base 10 is guaranteed to * work for now. */ uint64_t tor_parse_uint64(const char *s, int base, uint64_t min, @@ -853,6 +950,9 @@ esc_for_log(const char *s) case '\\': case '\"': case '\'': + case '\r': + case '\n': + case '\t': len += 2; break; default: @@ -914,8 +1014,7 @@ const char * escaped(const char *s) { static char *_escaped_val = NULL; - if (_escaped_val) - tor_free(_escaped_val); + tor_free(_escaped_val); if (s) _escaped_val = esc_for_log(s); @@ -1002,6 +1101,42 @@ wrap_string(smartlist_t *out, const char *string, size_t width, * Time * ===== */ +/** + * Converts struct timeval to a double value. + * Preserves microsecond precision, but just barely. + * Error is approx +/- 0.1 usec when dealing with epoch values. + */ +double +tv_to_double(const struct timeval *tv) +{ + double conv = tv->tv_sec; + conv += tv->tv_usec/1000000.0; + return conv; +} + +/** + * Converts timeval to milliseconds. + */ +int64_t +tv_to_msec(const struct timeval *tv) +{ + int64_t conv = ((int64_t)tv->tv_sec)*1000L; + /* Round ghetto-style */ + conv += ((int64_t)tv->tv_usec+500)/1000L; + return conv; +} + +/** + * Converts timeval to microseconds. + */ +int64_t +tv_to_usec(const struct timeval *tv) +{ + int64_t conv = ((int64_t)tv->tv_sec)*1000000L; + conv += tv->tv_usec; + return conv; +} + /** Return the number of microseconds elapsed between *start and *end. */ long @@ -1011,7 +1146,8 @@ tv_udiff(const struct timeval *start, const struct timeval *end) long secdiff = end->tv_sec - start->tv_sec; if (labs(secdiff+1) > LONG_MAX/1000000) { - log_warn(LD_GENERAL, "comparing times too far apart."); + log_warn(LD_GENERAL, "comparing times on microsecond detail too far " + "apart: %ld seconds", secdiff); return LONG_MAX; } @@ -1019,6 +1155,26 @@ tv_udiff(const struct timeval *start, const struct timeval *end) return udiff; } +/** Return the number of milliseconds elapsed between *start and *end. + */ +long +tv_mdiff(const struct timeval *start, const struct timeval *end) +{ + long mdiff; + long secdiff = end->tv_sec - start->tv_sec; + + if (labs(secdiff+1) > LONG_MAX/1000) { + log_warn(LD_GENERAL, "comparing times on millisecond detail too far " + "apart: %ld seconds", secdiff); + return LONG_MAX; + } + + /* Subtract and round */ + mdiff = secdiff*1000L + + ((long)end->tv_usec - (long)start->tv_usec + 500L) / 1000L; + return mdiff; +} + /** Yield true iff <b>y</b> is a leap-year. */ #define IS_LEAPYEAR(y) (!(y % 4) && ((y % 100) || !(y % 400))) /** Helper: Return the number of leap-days between Jan 1, y1 and Jan 1, y2. */ @@ -1094,7 +1250,7 @@ format_rfc1123_time(char *buf, time_t t) memcpy(buf+8, MONTH_NAMES[tm.tm_mon], 3); } -/** Parse the the RFC1123 encoding of some time (in GMT) from <b>buf</b>, +/** Parse the RFC1123 encoding of some time (in GMT) from <b>buf</b>, * and store the result in *<b>t</b>. * * Return 0 on success, -1 on failure. @@ -1355,80 +1511,46 @@ update_approx_time(time_t now) #endif /* ===== - * Fuzzy time - * XXXX022 Use this consistently or rip most of it out. + * Rate limiting * ===== */ -/* In a perfect world, everybody would run NTP, and NTP would be perfect, so - * if we wanted to know "Is the current time before time X?" we could just say - * "time(NULL) < X". - * - * But unfortunately, many users are running Tor in an imperfect world, on - * even more imperfect computers. Hence, we need to track time oddly. We - * model the user's computer as being "skewed" from accurate time by - * -<b>ftime_skew</b> seconds, such that our best guess of the current time is - * time(NULL)+ftime_skew. We also assume that our measurements of time may - * have up to <b>ftime_slop</b> seconds of inaccuracy; IOW, our window of - * estimate for the current time is now + ftime_skew +/- ftime_slop. - */ -/** Our current estimate of our skew, such that we think the current time is - * closest to time(NULL)+ftime_skew. */ -static int ftime_skew = 0; -/** Tolerance during time comparisons, in seconds. */ -static int ftime_slop = 60; -/** Set the largest amount of sloppiness we'll allow in fuzzy time - * comparisons. */ -void -ftime_set_maximum_sloppiness(int seconds) -{ - tor_assert(seconds >= 0); - ftime_slop = seconds; -} -/** Set the amount by which we believe our system clock to differ from - * real time. */ -void -ftime_set_estimated_skew(int seconds) -{ - ftime_skew = seconds; -} -#if 0 -void -ftime_get_window(time_t now, ftime_t *ft_out) -{ - ft_out->earliest = now + ftime_skew - ftime_slop; - ft_out->latest = now + ftime_skew + ftime_slop; -} -#endif -/** Return true iff we think that <b>now</b> might be after <b>when</b>. */ -int -ftime_maybe_after(time_t now, time_t when) -{ - /* It may be after when iff the latest possible current time is after when */ - return (now + ftime_skew + ftime_slop) >= when; -} -/** Return true iff we think that <b>now</b> might be before <b>when</b>. */ -int -ftime_maybe_before(time_t now, time_t when) -{ - /* It may be before when iff the earliest possible current time is before */ - return (now + ftime_skew - ftime_slop) < when; -} -/** Return true if we think that <b>now</b> is definitely after <b>when</b>. */ -int -ftime_definitely_after(time_t now, time_t when) +/** If the rate-limiter <b>lim</b> is ready at <b>now</b>, return the number + * of calls to rate_limit_is_ready (including this one!) since the last time + * rate_limit_is_ready returned nonzero. Otherwise return 0. */ +static int +rate_limit_is_ready(ratelim_t *lim, time_t now) { - /* It is definitely after when if the earliest time it could be is still - * after when. */ - return (now + ftime_skew - ftime_slop) >= when; + if (lim->rate + lim->last_allowed <= now) { + int res = lim->n_calls_since_last_time + 1; + lim->last_allowed = now; + lim->n_calls_since_last_time = 0; + return res; + } else { + ++lim->n_calls_since_last_time; + return 0; + } } -/** Return true if we think that <b>now</b> is definitely before <b>when</b>. - */ -int -ftime_definitely_before(time_t now, time_t when) + +/** If the rate-limiter <b>lim</b> is ready at <b>now</b>, return a newly + * allocated string indicating how many messages were suppressed, suitable to + * append to a log message. Otherwise return NULL. */ +char * +rate_limit_log(ratelim_t *lim, time_t now) { - /* It is definitely before when if the latest time it could be is still - * before when. */ - return (now + ftime_skew + ftime_slop) < when; + int n; + if ((n = rate_limit_is_ready(lim, now))) { + if (n == 1) { + return tor_strdup(""); + } else { + char *cp=NULL; + tor_asprintf(&cp, + " [%d similar message(s) suppressed in last %d seconds]", + n-1, lim->rate); + return cp; + } + } else { + return NULL; + } } /* ===== @@ -1555,22 +1677,22 @@ check_private_dir(const char *dirname, cpd_check_t check) tor_free(f); if (r) { if (errno != ENOENT) { - log(LOG_WARN, LD_FS, "Directory %s cannot be read: %s", dirname, - strerror(errno)); + log_warn(LD_FS, "Directory %s cannot be read: %s", dirname, + strerror(errno)); return -1; } if (check == CPD_NONE) { - log(LOG_WARN, LD_FS, "Directory %s does not exist.", dirname); + log_warn(LD_FS, "Directory %s does not exist.", dirname); return -1; } else if (check == CPD_CREATE) { log_info(LD_GENERAL, "Creating directory %s", dirname); -#ifdef MS_WINDOWS +#if defined (MS_WINDOWS) && !defined (WINCE) r = mkdir(dirname); #else r = mkdir(dirname, 0700); #endif if (r) { - log(LOG_WARN, LD_FS, "Error creating directory %s: %s", dirname, + log_warn(LD_FS, "Error creating directory %s: %s", dirname, strerror(errno)); return -1; } @@ -1580,7 +1702,7 @@ check_private_dir(const char *dirname, cpd_check_t check) return 0; } if (!(st.st_mode & S_IFDIR)) { - log(LOG_WARN, LD_FS, "%s is not a directory", dirname); + log_warn(LD_FS, "%s is not a directory", dirname); return -1; } #ifndef MS_WINDOWS @@ -1593,7 +1715,7 @@ check_private_dir(const char *dirname, cpd_check_t check) pw = getpwuid(st.st_uid); - log(LOG_WARN, LD_FS, "%s is not owned by this user (%s, %d) but by " + log_warn(LD_FS, "%s is not owned by this user (%s, %d) but by " "%s (%d). Perhaps you are running Tor as the wrong user?", dirname, process_ownername, (int)getuid(), pw ? pw->pw_name : "<unknown>", (int)st.st_uid); @@ -1602,9 +1724,9 @@ check_private_dir(const char *dirname, cpd_check_t check) return -1; } if (st.st_mode & 0077) { - log(LOG_WARN, LD_FS, "Fixing permissions on directory %s", dirname); + log_warn(LD_FS, "Fixing permissions on directory %s", dirname); if (chmod(dirname, 0700)) { - log(LOG_WARN, LD_FS, "Could not chmod directory %s: %s", dirname, + log_warn(LD_FS, "Could not chmod directory %s: %s", dirname, strerror(errno)); return -1; } else { @@ -1635,12 +1757,13 @@ write_str_to_file(const char *fname, const char *str, int bin) } /** Represents a file that we're writing to, with support for atomic commit: - * we can write into a a temporary file, and either remove the file on + * we can write into a temporary file, and either remove the file on * failure, or replace the original file on success. */ struct open_file_t { char *tempname; /**< Name of the temporary file. */ char *filename; /**< Name of the original file. */ - int rename_on_close; /**< Are we using the temporary file or not? */ + unsigned rename_on_close:1; /**< Are we using the temporary file or not? */ + unsigned binary:1; /**< Did we open in binary mode? */ int fd; /**< fd for the open file. */ FILE *stdio_file; /**< stdio wrapper for <b>fd</b>. */ }; @@ -1688,7 +1811,7 @@ start_writing_to_file(const char *fname, int open_flags, int mode, } else { open_name = new_file->tempname = tor_malloc(tempname_len); if (tor_snprintf(new_file->tempname, tempname_len, "%s.tmp", fname)<0) { - log(LOG_WARN, LD_GENERAL, "Failed to generate filename"); + log_warn(LD_GENERAL, "Failed to generate filename"); goto err; } /* We always replace an existing temporary file if there is one. */ @@ -1696,9 +1819,12 @@ start_writing_to_file(const char *fname, int open_flags, int mode, open_flags &= ~O_EXCL; new_file->rename_on_close = 1; } + if (open_flags & O_BINARY) + new_file->binary = 1; - if ((new_file->fd = open(open_name, open_flags, mode)) < 0) { - log(LOG_WARN, LD_FS, "Couldn't open \"%s\" (%s) for writing: %s", + new_file->fd = open(open_name, open_flags, mode); + if (new_file->fd < 0) { + log_warn(LD_FS, "Couldn't open \"%s\" (%s) for writing: %s", open_name, fname, strerror(errno)); goto err; } @@ -1734,7 +1860,8 @@ fdopen_file(open_file_t *file_data) if (file_data->stdio_file) return file_data->stdio_file; tor_assert(file_data->fd >= 0); - if (!(file_data->stdio_file = fdopen(file_data->fd, "a"))) { + if (!(file_data->stdio_file = fdopen(file_data->fd, + file_data->binary?"ab":"a"))) { log_warn(LD_FS, "Couldn't fdopen \"%s\" [%d]: %s", file_data->filename, file_data->fd, strerror(errno)); } @@ -1834,7 +1961,7 @@ write_chunks_to_file_impl(const char *fname, const smartlist_t *chunks, { result = write_all(fd, chunk->bytes, chunk->len, 0); if (result < 0) { - log(LOG_WARN, LD_FS, "Error writing to \"%s\": %s", fname, + log_warn(LD_FS, "Error writing to \"%s\": %s", fname, strerror(errno)); goto err; } @@ -2090,7 +2217,40 @@ unescape_string(const char *s, char **result, size_t *size_out) const char * parse_config_line_from_str(const char *line, char **key_out, char **value_out) { + /* I believe the file format here is supposed to be: + FILE = (EMPTYLINE | LINE)* (EMPTYLASTLINE | LASTLINE)? + + EMPTYLASTLINE = SPACE* | COMMENT + EMPTYLINE = EMPTYLASTLINE NL + SPACE = ' ' | '\r' | '\t' + COMMENT = '#' NOT-NL* + NOT-NL = Any character except '\n' + NL = '\n' + + LASTLINE = SPACE* KEY SPACE* VALUES + LINE = LASTLINE NL + KEY = KEYCHAR+ + KEYCHAR = Any character except ' ', '\r', '\n', '\t', '#', "\" + + VALUES = QUOTEDVALUE | NORMALVALUE + QUOTEDVALUE = QUOTE QVITEM* QUOTE EOLSPACE? + QUOTE = '"' + QVCHAR = KEYCHAR | ESC ('n' | 't' | 'r' | '"' | ESC |'\'' | OCTAL | HEX) + ESC = "\\" + OCTAL = ODIGIT (ODIGIT ODIGIT?)? + HEX = ('x' | 'X') HEXDIGIT HEXDIGIT + ODIGIT = '0' .. '7' + HEXDIGIT = '0'..'9' | 'a' .. 'f' | 'A' .. 'F' + EOLSPACE = SPACE* COMMENT? + + NORMALVALUE = (VALCHAR | ESC ESC_IGNORE | CONTINUATION)* EOLSPACE? + VALCHAR = Any character except ESC, '#', and '\n' + ESC_IGNORE = Any character except '#' or '\n' + CONTINUATION = ESC NL ( COMMENT NL )* + */ + const char *key, *val, *cp; + int continuation = 0; tor_assert(key_out); tor_assert(value_out); @@ -2114,9 +2274,10 @@ parse_config_line_from_str(const char *line, char **key_out, char **value_out) return line; } - /* Skip until the next space. */ + /* Skip until the next space or \ followed by newline. */ key = line; - while (*line && !TOR_ISSPACE(*line) && *line != '#') + while (*line && !TOR_ISSPACE(*line) && *line != '#' && + ! (line[0] == '\\' && line[1] == '\n')) ++line; *key_out = tor_strndup(key, line-key); @@ -2127,7 +2288,7 @@ parse_config_line_from_str(const char *line, char **key_out, char **value_out) val = line; /* Find the end of the line. */ - if (*line == '\"') { + if (*line == '\"') { // XXX No continuation handling is done here if (!(line = unescape_string(line, value_out, NULL))) return NULL; while (*line == ' ' || *line == '\t') @@ -2135,18 +2296,53 @@ parse_config_line_from_str(const char *line, char **key_out, char **value_out) if (*line && *line != '#' && *line != '\n') return NULL; } else { - while (*line && *line != '\n' && *line != '#') - ++line; + /* Look for the end of the line. */ + while (*line && *line != '\n' && (*line != '#' || continuation)) { + if (*line == '\\' && line[1] == '\n') { + continuation = 1; + line += 2; + } else if (*line == '#') { + do { + ++line; + } while (*line && *line != '\n'); + if (*line == '\n') + ++line; + } else { + ++line; + } + } + if (*line == '\n') { cp = line++; } else { cp = line; } + /* Now back cp up to be the last nonspace character */ while (cp>val && TOR_ISSPACE(*(cp-1))) --cp; tor_assert(cp >= val); + + /* Now copy out and decode the value. */ *value_out = tor_strndup(val, cp-val); + if (continuation) { + char *v_out, *v_in; + v_out = v_in = *value_out; + while (*v_in) { + if (*v_in == '#') { + do { + ++v_in; + } while (*v_in && *v_in != '\n'); + if (*v_in == '\n') + ++v_in; + } else if (v_in[0] == '\\' && v_in[1] == '\n') { + v_in += 2; + } else { + *v_out++ = *v_in++; + } + } + *v_out = '\0'; + } } if (*line == '#') { @@ -2165,19 +2361,22 @@ char * expand_filename(const char *filename) { tor_assert(filename); +#ifdef MS_WINDOWS + return tor_strdup(filename); +#else if (*filename == '~') { - size_t len; - char *home, *result; + char *home, *result=NULL; const char *rest; if (filename[1] == '/' || filename[1] == '\0') { home = getenv("HOME"); if (!home) { log_warn(LD_CONFIG, "Couldn't find $HOME environment variable while " - "expanding \"%s\"", filename); - return NULL; + "expanding \"%s\"; defaulting to \"\".", filename); + home = tor_strdup(""); + } else { + home = tor_strdup(home); } - home = tor_strdup(home); rest = strlen(filename)>=2?(filename+2):""; } else { #ifdef HAVE_PWD_H @@ -2204,21 +2403,19 @@ expand_filename(const char *filename) if (strlen(home)>1 && !strcmpend(home,PATH_SEPARATOR)) { home[strlen(home)-1] = '\0'; } - /* Plus one for /, plus one for NUL. - * Round up to 16 in case we can't do math. */ - len = strlen(home)+strlen(rest)+16; - result = tor_malloc(len); - tor_snprintf(result,len,"%s"PATH_SEPARATOR"%s",home,rest); + tor_asprintf(&result,"%s"PATH_SEPARATOR"%s",home,rest); tor_free(home); return result; } else { return tor_strdup(filename); } +#endif } #define MAX_SCANF_WIDTH 9999 -/** DOCDOC */ +/** Helper: given an ASCII-encoded decimal digit, return its numeric value. + * NOTE: requires that its input be in-bounds. */ static int digit_to_num(char d) { @@ -2227,7 +2424,10 @@ digit_to_num(char d) return num; } -/** DOCDOC */ +/** Helper: Read an unsigned int from *<b>bufp</b> of up to <b>width</b> + * characters. (Handle arbitrary width if <b>width</b> is less than 0.) On + * success, store the result in <b>out</b>, advance bufp to the next + * character, and return 0. On failure, return -1. */ static int scan_unsigned(const char **bufp, unsigned *out, int width) { @@ -2254,7 +2454,9 @@ scan_unsigned(const char **bufp, unsigned *out, int width) return 0; } -/** DOCDOC */ +/** Helper: copy up to <b>width</b> non-space characters from <b>bufp</b> to + * <b>out</b>. Make sure <b>out</b> is nul-terminated. Advance <b>bufp</b> + * to the next non-space character or the EOS. */ static int scan_string(const char **bufp, char *out, int width) { @@ -2343,7 +2545,12 @@ tor_vsscanf(const char *buf, const char *pattern, va_list ap) * and store the results in the corresponding argument fields. Differs from * sscanf in that it: Only handles %u and %Ns. Does not handle arbitrarily * long widths. %u does not consume any space. Is locale-independent. - * Returns -1 on malformed patterns. */ + * Returns -1 on malformed patterns. + * + * (As with other locale-independent functions, we need this to parse data that + * is in ASCII without worrying that the C library's locale-handling will make + * miscellaneous characters look like numbers, spaces, and so on.) + */ int tor_sscanf(const char *buf, const char *pattern, ...) { @@ -2364,20 +2571,32 @@ tor_listdir(const char *dirname) smartlist_t *result; #ifdef MS_WINDOWS char *pattern; + TCHAR tpattern[MAX_PATH] = {0}; + char name[MAX_PATH] = {0}; HANDLE handle; WIN32_FIND_DATA findData; size_t pattern_len = strlen(dirname)+16; pattern = tor_malloc(pattern_len); tor_snprintf(pattern, pattern_len, "%s\\*", dirname); - if (INVALID_HANDLE_VALUE == (handle = FindFirstFile(pattern, &findData))) { +#ifdef UNICODE + mbstowcs(tpattern,pattern,MAX_PATH); +#else + strlcpy(tpattern, pattern, MAX_PATH); +#endif + if (INVALID_HANDLE_VALUE == (handle = FindFirstFile(tpattern, &findData))) { tor_free(pattern); return NULL; } result = smartlist_create(); while (1) { - if (strcmp(findData.cFileName, ".") && - strcmp(findData.cFileName, "..")) { - smartlist_add(result, tor_strdup(findData.cFileName)); +#ifdef UNICODE + wcstombs(name,findData.cFileName,MAX_PATH); +#else + strlcpy(name,findData.cFileName,sizeof(name)); +#endif + if (strcmp(name, ".") && + strcmp(name, "..")) { + smartlist_add(result, tor_strdup(name)); } if (!FindNextFile(handle, &findData)) { DWORD err; @@ -2576,3 +2795,18 @@ write_pidfile(char *filename) } } +#ifdef MS_WINDOWS +HANDLE +load_windows_system_library(const TCHAR *library_name) +{ + TCHAR path[MAX_PATH]; + unsigned n; + n = GetSystemDirectory(path, MAX_PATH); + if (n == 0 || n + _tcslen(library_name) + 2 >= MAX_PATH) + return 0; + _tcscat(path, TEXT("\\")); + _tcscat(path, library_name); + return LoadLibrary(path); +} +#endif + diff --git a/src/common/util.h b/src/common/util.h index fcea9c5b88..6b54856743 100644 --- a/src/common/util.h +++ b/src/common/util.h @@ -43,7 +43,7 @@ * stderr. */ #define tor_assert(expr) STMT_BEGIN \ if (PREDICT_UNLIKELY(!(expr))) { \ - log(LOG_ERR, LD_BUG, "%s:%d: %s: Assertion %s failed; aborting.", \ + log_err(LD_BUG, "%s:%d: %s: Assertion %s failed; aborting.", \ _SHORT_FILE_, __LINE__, __func__, #expr); \ fprintf(stderr,"%s:%d %s: Assertion %s failed; aborting.\n", \ _SHORT_FILE_, __LINE__, __func__, #expr); \ @@ -152,8 +152,18 @@ void tor_log_mallinfo(int severity); #define bool_neq(a,b) (!(a)!=!(b)) /* Math functions */ +double tor_mathlog(double d) ATTR_CONST; +long tor_lround(double d) ATTR_CONST; int tor_log2(uint64_t u64) ATTR_CONST; uint64_t round_to_power_of_2(uint64_t u64); +unsigned round_to_next_multiple_of(unsigned number, unsigned divisor); +uint32_t round_uint32_to_next_multiple_of(uint32_t number, uint32_t divisor); +uint64_t round_uint64_to_next_multiple_of(uint64_t number, uint64_t divisor); + +/* Compute the CEIL of <b>a</b> divided by <b>b</b>, for nonnegative <b>a</b> + * and positive <b>b</b>. Works on integer types only. Not defined if a+b can + * overflow. */ +#define CEIL_DIV(a,b) (((a)+(b)-1)/(b)) /* String manipulation */ @@ -179,6 +189,8 @@ long tor_parse_long(const char *s, int base, long min, long max, int *ok, char **next); unsigned long tor_parse_ulong(const char *s, int base, unsigned long min, unsigned long max, int *ok, char **next); +double tor_parse_double(const char *s, double min, double max, int *ok, + char **next); uint64_t tor_parse_uint64(const char *s, int base, uint64_t min, uint64_t max, int *ok, char **next); const char *hex_str(const char *from, size_t fromlen) ATTR_NONNULL((1)); @@ -188,8 +200,11 @@ const char *eat_whitespace_no_nl(const char *s) ATTR_PURE; const char *eat_whitespace_eos_no_nl(const char *s, const char *eos) ATTR_PURE; const char *find_whitespace(const char *s) ATTR_PURE; const char *find_whitespace_eos(const char *s, const char *eos) ATTR_PURE; +const char *find_str_at_start_of_line(const char *haystack, const char *needle) + ATTR_PURE; int tor_mem_is_zero(const char *mem, size_t len) ATTR_PURE; int tor_digest_is_zero(const char *digest) ATTR_PURE; +int tor_digest256_is_zero(const char *digest) ATTR_PURE; char *esc_for_log(const char *string) ATTR_MALLOC; const char *escaped(const char *string); struct smartlist_t; @@ -207,7 +222,11 @@ void base16_encode(char *dest, size_t destlen, const char *src, size_t srclen); int base16_decode(char *dest, size_t destlen, const char *src, size_t srclen); /* Time helpers */ +double tv_to_double(const struct timeval *tv); +int64_t tv_to_msec(const struct timeval *tv); +int64_t tv_to_usec(const struct timeval *tv); long tv_udiff(const struct timeval *start, const struct timeval *end); +long tv_mdiff(const struct timeval *start, const struct timeval *end); time_t tor_timegm(struct tm *tm); #define RFC1123_TIME_LEN 29 void format_rfc1123_time(char *buf, time_t t); @@ -228,15 +247,32 @@ time_t approx_time(void); void update_approx_time(time_t now); #endif -/* Fuzzy time. */ -void ftime_set_maximum_sloppiness(int seconds); -void ftime_set_estimated_skew(int seconds); -/* typedef struct ftime_t { time_t earliest; time_t latest; } ftime_t; */ -/* void ftime_get_window(time_t now, ftime_t *ft_out); */ -int ftime_maybe_after(time_t now, time_t when); -int ftime_maybe_before(time_t now, time_t when); -int ftime_definitely_after(time_t now, time_t when); -int ftime_definitely_before(time_t now, time_t when); +/* Rate-limiter */ + +/** A ratelim_t remembers how often an event is occurring, and how often + * it's allowed to occur. Typical usage is something like: + * + <pre> + if (possibly_very_frequent_event()) { + const int INTERVAL = 300; + static ratelim_t warning_limit = RATELIM_INIT(INTERVAL); + char *m; + if ((m = rate_limit_log(&warning_limit, approx_time()))) { + log_warn(LD_GENERAL, "The event occurred!%s", m); + tor_free(m); + } + } + </pre> + */ +typedef struct ratelim_t { + int rate; + time_t last_allowed; + int n_calls_since_last_time; +} ratelim_t; + +#define RATELIM_INIT(r) { (r), 0, 0 } + +char *rate_limit_log(ratelim_t *lim, time_t now); /* File helpers */ ssize_t write_all(int fd, const char *buf, size_t count, int isSocket); @@ -294,5 +330,11 @@ void start_daemon(void); void finish_daemon(const char *desired_cwd); void write_pidfile(char *filename); +#ifdef MS_WINDOWS +HANDLE load_windows_system_library(const TCHAR *library_name); +#endif + +const char *libor_get_digests(void); + #endif diff --git a/src/common/util_codedigest.c b/src/common/util_codedigest.c new file mode 100644 index 0000000000..88fe508b92 --- /dev/null +++ b/src/common/util_codedigest.c @@ -0,0 +1,11 @@ + +#include "util.h" + +const char * +libor_get_digests(void) +{ + return "" +#include "common_sha1.i" + ; +} + diff --git a/src/config/geoip b/src/config/geoip index c177b19a9a..39aceddd04 100644 --- a/src/config/geoip +++ b/src/config/geoip @@ -1,7 +1,12 @@ -# Last updated based on March 1 2011 Maxmind GeoLite Country +# Last updated based on April 1 2011 Maxmind GeoLite Country # wget http://geolite.maxmind.com/download/geoip/database/GeoIPCountryCSV.zip # cut -d, -f3-5 < GeoIPCountryWhois.csv|sed 's/"//g' > geoip -16777216,17301503,AU +16777216,16908287,AU +16908800,16909055,CN +16909056,16909311,AU +17039360,17039615,AU +17040384,17040639,CN +17170432,17301503,IN 17367040,17432575,MY 17435136,17435391,AU 17498112,17563647,KR @@ -35,12 +40,13 @@ 28573696,28966911,CN 28966912,29032447,IN 29097984,29884415,CN -29884416,29885439,AU 29949952,30015487,KR 30015488,30408703,CN 30408704,33554431,KR 33554432,34603007,FR -34603008,34620415,EU +34603008,34604543,EU +34604544,34605055,DE +34605056,34620415,EU 34620416,34620927,SE 34620928,34621439,IT 34621440,34636799,EU @@ -61,18 +67,20 @@ 34866176,34867199,IT 34867200,34880511,EU 34880512,34881535,DE -34881536,35127295,EU +34881536,34910975,EU +34910976,34911231,ES +34911232,34911743,DE +34911744,34911999,AT +34912000,34912255,GB +34912256,34913279,DE +34913280,34930687,EU +34930688,34938879,DE +34938880,35127295,EU 35127296,35651583,GB 35651584,36700159,IT 36700160,36962303,AE 36962304,37224447,IL -37224448,37265407,UA -37265408,37268479,CZ -37268480,37289983,UA -37289984,37298175,RU -37298176,37355519,UA -37355520,37421055,RU -37421056,37486591,UA +37224448,37486591,UA 37486592,37748735,RU 37748736,38273023,SE 38273024,38797311,KZ @@ -130,6 +138,7 @@ 72348928,72349055,US 72349056,72349119,BM 72349120,83886079,US +83886080,88080383,EU 100663296,121195295,US 121195296,121195327,IT 121195328,134693119,US @@ -656,7 +665,8 @@ 243531776,243662847,JP 243662848,243793919,CN 243793920,243859455,HK -243859456,243924991,AU +243859456,243916799,AU +243916800,243924991,JP 243924992,243990527,KR 243990528,244318207,IN 244318208,245366783,CN @@ -795,10 +805,9 @@ 411779072,411828223,PR 411828224,411893759,US 411893760,411975679,CA -411975680,411992063,US -412024832,412221439,US +411975680,412221439,US 412221440,412229631,CA -412286976,412483583,US +412254208,412483583,US 412483584,412549119,CA 412549120,412614655,US 412614656,412647423,CL @@ -868,6 +877,7 @@ 452988928,452997119,JP 452997120,453001215,IN 453001216,453009407,AU +453009408,453017599,BD 453017600,453050367,JP 453050368,453115903,KR 453115904,453246975,VN @@ -890,6 +900,7 @@ 456265728,456269823,ID 456269824,456271871,HK 456273920,456286207,AU +456286208,456294399,JP 456294400,456327167,CN 456327168,456523775,TW 456523776,456540159,SG @@ -915,6 +926,7 @@ 459300864,459309055,SG 459309056,459325439,KR 459325440,459333631,AU +459333632,459341823,TW 459341824,459407359,IN 459407360,459456511,JP 459456512,459460607,HK @@ -946,6 +958,7 @@ 459997184,460062719,IN 460062720,460128255,PH 460128256,460136447,IN +460136448,460144639,CN 460144640,460152831,PH 460152832,460154879,JP 460154880,460155903,SG @@ -1016,10 +1029,11 @@ 460939264,460940287,NZ 460940288,460941311,IN 460941312,460942335,AU +460942336,460943359,MY 460943360,460945407,AU 460947456,460980223,JP 460980224,460981247,NC -460982272,460983295,JP +460981248,460983295,JP 460983296,460984319,HK 460984320,460988415,PG 460988416,460994559,JP @@ -1045,7 +1059,7 @@ 461111296,461127679,IN 461127680,461131775,PH 461131776,461135871,ID -461135872,461139967,AU +461135872,461144063,AU 461144064,461209599,KR 461209600,461225983,SG 461225984,461227007,WF @@ -1094,11 +1108,30 @@ 468713472,469237759,TW 469237760,469499903,IN 469499904,469565439,NZ +469565440,469598207,AU +469598208,469630975,JP 469630976,469696511,TH 469696512,469712895,PK 469712896,469729279,KR 469729280,469762047,IN 469762048,520093695,US +521142272,521404415,DE +521404416,521535487,NL +521535488,521537535,GB +521537536,521539583,RU +521539584,521541631,BE +521541632,521543679,DE +521543680,521545727,RU +521545728,521547775,GB +521547776,521551871,AL +521551872,521553919,DE +521555968,521558015,NL +521558016,521560063,CY +521560064,521562111,PL +521564160,521566207,GR +521566208,521568255,LU +521568256,521601023,PT +521601024,521666559,RU 521666560,521668607,GB 521668608,521670655,CH 521670656,521672703,HU @@ -1127,18 +1160,98 @@ 521719808,521721855,UA 521721856,521723903,GB 521723904,521725951,SA +521725952,521727999,IR +521728000,521732095,FR +521732096,521736191,GB +521736192,521738239,DK +521738240,521740287,RU +521740288,521742335,IT +521742336,521746431,DE +521746432,521748479,NL +521748480,521750527,SE +521750528,521752575,PS +521752576,521754623,FR +521754624,521756671,IR +521756672,521758719,NL +521758720,521760767,IR +521760768,521762815,IL +521762816,521764863,CH +521764864,521766911,IR +521766912,521768959,IQ +521768960,521771007,FI +521771008,521773055,DE +521773056,521775103,GB +521775104,521777151,TR +521777152,521779199,ES +521779200,521783295,GB +521783296,521785343,RU +521785344,521787391,GB +521787392,521789439,NO +521789440,521791487,RU +521791488,521793535,IR +521793536,521795583,RU +521795584,521797631,PL +521797632,521928703,IT 521928704,521945087,RU 521945088,521953279,GB 521953280,521961471,RU 521961472,521969663,CZ 521969664,521977855,UA 521977856,521986047,RU +521986048,521994239,UA +521994240,522002431,KG +522002432,522010623,IR +522010624,522018815,AE +522018816,522027007,FR +522027008,522059775,RU +522059776,522125311,CZ +522125312,522133503,MD +522133504,522135551,NL +522135552,522137599,SE +522137600,522141695,CH +522141696,522143743,RU +522143744,522145791,CZ +522145792,522147839,DK +522147840,522149887,ES +522149888,522158079,UA +522158080,522166271,BE +522166272,522168319,SE +522168320,522170367,RU +522170368,522174463,YE +522174464,522178559,RU +522178560,522180607,DE +522180608,522182655,KZ +522182656,522190847,CZ 522190848,522715135,FR -522788864,522790911,RU +522780672,522782719,RU +522782720,522784767,UA +522784768,522786815,BG +522786816,522788863,PL +522788864,522797055,RU +522797056,522801151,UA +522801152,522803199,PL +522803200,522805247,UA +522805248,522807295,RU +522807296,522811391,UA +522811392,522813439,RU +522813440,522815487,UA +522815488,522819583,PL +522819584,522821631,KG +522821632,522823679,RU +522823680,522827775,PL +522827776,522831871,RU +522846208,522854399,PL +522854400,522858495,RU +522858496,522866687,UA +522866688,522870783,LV +522870784,522874879,RU +522874880,522878975,UA 522878976,522887167,RO 522887168,522895359,UA 522895360,522911743,RU -522911744,522944511,UA +522911744,522960895,UA +522960896,522969087,RU +522969088,522977279,UA 522977280,522981375,RU 522981376,522985471,IT 522985472,522989567,CH @@ -1147,25 +1260,176 @@ 522997760,523001855,RU 523001856,523005951,LT 523005952,523010047,IT +523010048,523014143,IL +523014144,523018239,ES +523018240,523022335,IT +523022336,523026431,RU +523026432,523030527,TR +523030528,523034623,FR +523034624,523038719,SE +523038720,523042815,RU +523042816,523075583,NO +523075584,523108351,HR +523108352,523173887,HU +523173888,523182079,BA +523182080,523190271,IR +523190272,523192319,FI +523192320,523194367,ES +523194368,523196415,DE +523196416,523198463,AE +523198464,523202559,CZ +523202560,523223039,RU +523223040,523225087,AM +523225088,523227135,SE +523227136,523229183,RO +523229184,523231231,GB +523231232,523239423,DE 523239424,523763711,GB 523763712,524025855,IR 524025856,524287999,PL 524288000,528482303,GB +528482304,528490495,PL +528490496,528498687,RU +528498688,528515071,UA +528515072,528523263,RU +528523264,528531455,UA +528531456,528539647,RU +528539648,528547839,UA +528547840,528564223,RU +528564224,528580607,UA +528580608,528588799,RU +528588800,528596991,UA +528596992,528605183,RU +528875520,528879615,RU +529006592,529268735,NL +529268736,529530879,TR 529530880,529596415,UA 529596416,529661951,TR 529661952,529727487,GE 529727488,529793023,HR -529793024,529825791,RU -529825792,529858559,CZ +529793024,529826815,RU +529826816,529827839,GB +529827840,529828863,RO +529828864,529829887,RU +529829888,529830399,UA +529830400,529830911,RU +529830912,529831935,DE +529831936,529832959,UA +529832960,529835007,RU +529835008,529836031,KG +529836032,529836543,UA +529836544,529837055,IL +529837056,529839103,RU +529839104,529839615,UA +529839616,529840127,CY +529840128,529842175,UA +529842176,529843199,IN +529843200,529844735,RU +529844736,529845247,UA +529845248,529846271,UZ +529846272,529846783,UA +529846784,529849343,RU +529849344,529855487,CZ +529855488,529857535,RU +529857536,529858559,UA 529858560,529924095,NL 529924096,529989631,DE 529989632,530055167,NL 530055168,530120703,GR 530120704,530186239,CY +530186240,530251775,IL +530251776,530317311,TR +530317312,530579455,IT 530579456,530710527,NL 530710528,530841599,RU 530841600,530972671,CH +530972672,531103743,SA +531103744,531169279,IL +531169280,531177471,KZ +531177472,531183615,GB +531183616,531185663,NL +531185664,531193855,TR +531193856,531194111,IE +531194112,531195903,GB +531195904,531197951,IT +531197952,531199999,GB +531200000,531202047,RU +531202048,531234815,CZ +531234816,531236863,IE +531236864,531238911,FR +531238912,531240959,LV +531240960,531243007,SE +531243008,531245055,RU +531245056,531247103,IT +531247104,531251199,IR +531251200,531259391,SK +531259392,531261439,ES +531261440,531263487,DE +531263488,531265535,RU +531265536,531267583,GB +531267584,531275775,UA +531275776,531277823,LT +531277824,531279871,RU +531281920,531283967,RU +531283968,531292159,DE +531292160,531333119,AZ +531333120,531335167,NL +531335168,531337215,IT +531341312,531349503,KZ +531628032,531660799,TR +531660800,531693567,BA +531693568,531695615,RU +531695616,531697663,LT +531697664,531699711,GB +531699712,531701759,CZ +531703808,531705855,IT +531705856,531707903,ES +531707904,531709951,GR +531709952,531718143,RU +531718144,531720191,ES +531720192,531722239,RU +531722240,531724287,IT +531724288,531726335,DE +531726336,531759103,TR +531759104,531890175,PL +531890176,532021247,RU +532021248,532152319,PL 532152320,532168703,DE +532168704,532185087,NL +532185088,532201471,IR +532201472,532221951,RU +532221952,532223999,IT +532224000,532226047,NO +532226048,532234239,GB +532234240,532242431,DK +532242432,532244479,IT +532244480,532246527,DE +532246528,532250623,BA +532250624,532283391,GB +532283392,532291583,TR +532291584,532293631,IE +532293632,532295679,IT +532295680,532297727,KG +532297728,532303871,RU +532303872,532305919,PL +532305920,532307967,NO +532307968,532310015,RU +532310016,532312063,GR +532312064,532314111,JE +532314112,532316159,DE +532316160,532324351,RU +532324352,532328447,NL +532328448,532330495,PS +532330496,532332543,CZ +532332544,532340735,PL +532340736,532348927,GB +532348928,532365311,PL +532365312,532373503,DE +532373504,532375551,RU +532375552,532377599,IT +532377600,532381695,DE +532381696,532414463,NL +532414464,532676607,IT 536870912,539624567,US 539624568,539624575,IE 539624576,539624703,US @@ -1266,7 +1530,9 @@ 540814016,540814079,US 540814080,540814271,TW 540814272,540814279,DE -540814280,540814511,US +540814280,540814327,US +540814328,540814335,IN +540814336,540814511,US 540814512,540814519,SG 540814520,540814719,US 540814720,540814735,SG @@ -1305,11 +1571,16 @@ 543691008,543844351,US 543844352,543844607,CH 543844608,603979775,US -603979776,604110847,CN +603980800,603981823,NP +603981824,603983871,CN +603996160,604110847,CN 604110848,604241919,JP 604241920,604504063,CN 604504064,605028351,JP 605028352,606404607,CN +606412800,606413823,HK +606420992,606437375,ID +606437376,606470143,KH 606470144,606601215,KR 606601216,607387647,CN 607387648,607649791,JP @@ -1321,6 +1592,7 @@ 620232704,620494847,CN 620494848,620625919,NP 620625920,620691455,CN +620756992,624951295,EU 637534208,644067391,US 644067392,644067455,CA 644067456,644835071,US @@ -1330,6 +1602,11 @@ 644840448,645225471,US 645225472,645225503,CA 645225504,654311423,US +654311680,654311935,AU +654376960,654442495,TW +655360000,656408575,KR +656408576,658505727,PK +658505728,660602879,CN 671088640,687865855,US 687865856,689963007,ZA 689963008,691011583,EG @@ -1490,6 +1767,13 @@ 693038080,693039103,ZW 693039104,693040127,GA 693040128,693041151,ZA +693041152,693042175,MW +693042176,693043199,NG +693043200,693044223,ZW +693044224,693045247,NG +693045248,693046271,KE +693046272,693047295,ZW +693047296,693048319,ZA 693101568,693102591,KE 693102592,693103615,CD 693103616,693104639,GN @@ -1899,38 +2183,72 @@ 703594496,704118783,ZA 704118784,704380927,MA 704380928,704643071,LY +704644096,704645119,BD +704650240,704651263,MY +704659456,704675839,TW +704675840,704708607,CN +704723968,704724991,MY +704724992,704741375,VN +704741376,704774143,CN 704774144,704905215,HK 704905216,705167359,CN 705167360,707788799,KR 707788800,708575231,CN 708575232,708706303,SG +708706304,708739071,CN +708751360,708752383,ID +708755456,708771839,AU 708771840,708837375,CN 708837376,709885951,TW 709885952,710017023,CN 710017024,710082559,KR -710148096,710934527,CN -711000064,711065599,CN +710082560,710098943,JP +710104064,710105087,PK +710115328,710934527,CN +710934528,710950911,VN +710961152,710962175,TH +710967296,711065599,CN 711065600,711131135,HK +711160832,711161855,BD +711163904,711196671,JP 711196672,711458815,CN 711458816,711983103,IN 711983104,712507391,VN -712507392,712769535,CN +712507392,712703999,CN +712712192,712713215,JP +712736768,712769535,CN 712769536,713031679,JP 713031680,714080255,CN 714080256,714604543,JP -714997760,721420287,CN +714604544,714866687,MY +714874880,714875903,MY +714899456,716898303,CN +716930048,716931071,JP +716931072,716963839,CN +716963840,717225983,MY +717225984,717750271,CN +717750272,717815807,HK +717848576,717881343,PK +717881344,720371711,CN +720404480,720437247,CN +720437248,720502783,AU +720502784,721420287,CN 721420288,738197503,JP 738197504,771751935,US 771751936,771817471,RU 771817472,771948543,TR 771948544,772014079,RU -772014080,772051223,DE +772014080,772050575,DE +772050576,772050583,TR +772050584,772051223,DE 772051224,772051231,CH 772051232,772054815,DE 772054816,772054847,ES 772054848,772057727,DE 772057728,772057735,IT -772057736,772145151,DE +772057736,772076095,DE +772076096,772076103,IT +772076104,772145151,DE 772145152,772210687,ES 772210688,772276223,IE 772276224,772341759,RU @@ -1966,7 +2284,9 @@ 772837376,772839423,KZ 772839424,772841471,CH 772841472,772843519,GB -772843520,772845567,IT +772843520,772844287,IT +772844288,772844543,US +772844544,772845567,IT 772845568,772847615,RU 772847616,772849663,GB 772849664,772851711,BG @@ -2043,7 +2363,10 @@ 772986880,772988927,GB 772988928,772990975,FR 772990976,772993023,IT -772993024,772995071,DE +772993024,772994559,DE +772994560,772994815,NL +772994816,772994943,US +772994944,772995071,DE 772995072,772997119,IR 772997120,772999167,BE 772999168,773001215,SI @@ -2081,7 +2404,9 @@ 773063168,773063424,TR 773063425,773063436,US 773063437,773064447,TR -773064448,773065215,US +773064448,773064703,US +773064704,773064959,TR +773064960,773065215,US 773065216,773066751,TR 773066752,773070847,AT 773070848,773074943,DE @@ -2108,11 +2433,15 @@ 773140480,773144575,CY 773144576,773148671,RU 773148672,773152767,IR -773152768,773156863,SE +773152768,773153791,SE +773153792,773154815,US +773154816,773156863,SE 773156864,773165055,FR 773165056,773168127,NL 773168128,773168383,US -773168384,773168511,NL +773168384,773168399,NL +773168400,773168415,US +773168416,773168511,NL 773168512,773168639,US 773168640,773169023,NL 773169024,773169151,US @@ -2138,7 +2467,9 @@ 773234688,773238783,PL 773238784,773242879,NL 773242880,773246975,FR -773246976,773251071,EE +773246976,773247231,RU +773247232,773247999,EE +773248000,773251071,US 773251072,773255167,AZ 773255168,773259263,RU 773259264,773263359,SI @@ -2273,7 +2604,11 @@ 773793792,773795839,FR 773795840,773797887,NL 773797888,773799935,ES -773799936,773801983,CY +773799936,773800447,GB +773800448,773800703,CY +773800704,773800959,GB +773800960,773801727,CY +773801728,773801983,GB 773801984,773804031,IQ 773804032,773806079,GB 773806080,773808127,BE @@ -2512,22 +2847,44 @@ 774162071,774162075,EC 774162076,774162080,SH 774162081,774162085,ZM -774162086,774168575,VA +774162086,774162090,KH +774162091,774162095,VN +774162096,774162100,FM +774162101,774162105,US +774162106,774162110,LY +774162111,774162115,SO +774162116,774162120,FI +774162121,774162125,DZ +774162126,774162130,CD +774162131,774162135,MR +774162136,774162140,CM +774162141,774162145,SV +774162146,774162150,HN +774162151,774162155,US +774162156,774162160,UY +774162161,774162165,LV +774162166,774162170,MD +774162171,774162173,LB +774162174,774162176,VA +774162177,774162181,LK +774162182,774162186,IE +774162187,774168575,VA 774168576,774176767,PL 774176768,774184959,IT 774184960,774193151,GB 774193152,774209535,ES 774209536,774217727,RU 774217728,774217759,PH -774217760,774217791,RU +774217760,774217791,GB 774217792,774217823,US 774217824,774217855,AE 774217856,774217919,US -774217920,774217927,CN +774217920,774217927,GB 774217928,774217935,LK 774217936,774217983,CN -774217984,774218239,GB -774218240,774218495,US +774217984,774218047,CA +774218048,774218111,GB +774218112,774218495,US 774218496,774218503,SG 774218504,774218519,CN 774218520,774218527,PK @@ -2563,54 +2920,65 @@ 774219288,774219295,CY 774219296,774219327,GB 774219328,774219335,TW -774219336,774219343,GB -774219344,774219351,CN -774219352,774219359,GB +774219336,774219343,EG +774219344,774219359,CN 774219360,774219367,PT -774219368,774219391,GB +774219368,774219375,IN +774219376,774219383,GB +774219384,774219391,IN 774219392,774219399,US -774219400,774219407,GB -774219408,774219415,CN -774219416,774219423,GB +774219400,774219407,IN +774219408,774219423,CN 774219424,774219431,IN -774219432,774219439,GB +774219432,774219439,HU 774219440,774219447,CN -774219448,774219455,GB -774219456,774219463,CN -774219464,774219471,GB -774219472,774219479,CN -774219480,774219487,GB +774219448,774219455,ES +774219456,774219479,CN +774219480,774219487,HU 774219488,774219495,CN -774219496,774219503,GB +774219496,774219503,EG 774219504,774219511,CN -774219512,774219519,GB -774219520,774219543,CN -774219544,774219679,GB +774219512,774219519,SG +774219520,774219551,CN +774219552,774219559,PS +774219560,774219575,CN +774219576,774219583,GR +774219584,774219591,TR +774219592,774219599,US +774219600,774219607,CN +774219608,774219679,GB 774219680,774219687,GR -774219688,774219695,UA +774219688,774219695,GB 774219696,774219711,US 774219712,774219727,GB 774219728,774219743,RU -774219744,774219759,GB +774219744,774219751,GB +774219752,774219759,CN 774219760,774219775,RU 774219776,774219839,CN -774219840,774219903,GB +774219840,774219847,UA +774219848,774219855,CN +774219856,774219871,GB +774219872,774219887,CN +774219888,774219895,RU +774219896,774219903,LK 774219904,774219967,US -774219968,774220031,GB +774219968,774220031,CN 774220032,774220287,US 774220288,774220543,CN -774220544,774220799,US -774220800,774221055,GB +774220544,774221055,GB 774221056,774221311,US -774221312,774221567,GB +774221312,774221567,CN 774221568,774221823,US 774221824,774222463,GB 774222464,774222495,CN -774222496,774222847,GB +774222496,774222511,FI +774222512,774222527,CN +774222528,774222847,GB 774222848,774222863,UA 774222864,774222879,TR 774222880,774222911,US -774222912,774222943,CN +774222912,774222943,MA 774222944,774222951,GR 774222952,774222967,CN 774222968,774222975,GB @@ -2630,7 +2998,7 @@ 774223280,774223287,GB 774223288,774223295,CN 774223296,774223303,TR -774223304,774223311,GB +774223304,774223311,CN 774223312,774223319,IN 774223320,774223327,ID 774223328,774223335,PK @@ -2674,8 +3042,7 @@ 774224512,774224767,GB 774224768,774224831,CN 774224832,774224847,UA -774224848,774224863,RU -774224864,774224967,GB +774224848,774224967,GB 774224968,774224975,UA 774224976,774224983,TR 774224984,774224991,CN @@ -2689,14 +3056,15 @@ 774225344,774225351,CN 774225352,774225359,LT 774225360,774225375,UA -774225376,774225391,RU +774225376,774225391,GB 774225392,774225415,UA 774225416,774225423,CN 774225424,774225431,TR -774225432,774225511,GB +774225432,774225439,DE +774225440,774225447,CN +774225448,774225511,GB 774225512,774225519,TR -774225520,774225535,GB -774225536,774225567,CN +774225520,774225567,GB 774225568,774225599,RU 774225600,774225615,PK 774225616,774225647,TR @@ -2705,15 +3073,12 @@ 774225672,774225679,IT 774225680,774225687,GB 774225688,774225695,CA -774225696,774225703,RU -774225704,774225711,GB -774225712,774225719,PK -774225720,774225727,GB +774225696,774225727,GB 774225728,774225743,LT 774225744,774225775,GB 774225776,774225791,US 774225792,774225807,UA -774225808,774225823,RU +774225808,774225823,GB 774225824,774225855,US 774225856,774225919,IN 774225920,774234111,UA @@ -2845,7 +3210,39 @@ 778500096,778502143,RO 778502144,778567679,GR 778567680,778633215,TR -778633216,778698751,FR +778633216,778638975,FR +778638976,778639103,CZ +778639104,778639231,LT +778639232,778639359,FI +778639360,778639615,ES +778639616,778639871,PL +778639872,778640127,DE +778640128,778640383,GB +778640384,778640639,IT +778640640,778640895,PT +778640896,778641151,NL +778641152,778641407,IE +778641408,778665983,FR +778665984,778666015,BE +778666016,778666031,FR +778666032,778666047,PL +778666048,778666063,FR +778666064,778666095,PL +778666096,778666099,DE +778666100,778666103,ES +778666104,778666107,GB +778666108,778666111,IE +778666112,778666119,FR +778666120,778666123,GB +778666124,778666127,FR +778666128,778666143,PL +778666144,778666151,FR +778666152,778666155,GB +778666156,778666159,ES +778666160,778666175,PL +778666176,778666207,FR +778666208,778666239,PL +778666240,778698751,FR 778698752,778764287,TR 778764288,778829823,HU 778829824,778895359,RO @@ -3070,7 +3467,11 @@ 782254080,782270463,UA 782270464,782305791,RU 782305792,782306303,RO -782306304,782319615,RU +782306304,782310399,RU +782310400,782310527,RO +782310528,782310655,RU +782310656,782310911,RO +782310912,782319615,RU 782319616,782335999,ME 782336000,782352383,RU 782352384,782368767,SY @@ -3092,12 +3493,14 @@ 782647296,782663679,RU 782663680,782664191,GB 782664192,782664447,DK -782664448,782664704,FR +782664448,782664703,LU +782664704,782664704,GB 782664705,782665471,NL 782665472,782680063,LU 782680064,782696447,RU 782696448,782712831,DE 782712832,782729215,RU +782729216,782745599,DE 782745600,782761983,CZ 782761984,783024127,PL 783024128,783040511,RU @@ -3225,7 +3628,8 @@ 783800320,783802367,LV 783802368,783804415,FR 783804416,783806463,IT -783806464,783810559,NL +783806464,783810303,NL +783810304,783810559,BE 783810560,783843327,SA 783843328,783876095,BH 783876096,783908863,UA @@ -3245,8 +3649,8 @@ 784334848,784465919,FR 784465920,784596991,SE 784596992,784728063,TR -784728064,784760831,GR -784760832,784859135,CY +784728064,784763903,GR +784763904,784859135,CY 784859136,785121279,UA 785121280,785252351,PL 785252352,785383423,AT @@ -3343,7 +3747,9 @@ 786747392,786751487,SK 786751488,786755583,PL 786755584,786759679,IT -786759680,786771967,DE +786759680,786771711,DE +786771712,786771775,AT +786771776,786771967,DE 786771968,786776063,UA 786776064,786780159,RU 786780160,786784255,IT @@ -3410,6 +3816,7 @@ 787021824,787038207,BG 787038208,787054591,GB 787054592,787070975,IT +787070976,787087359,RU 787087360,787095551,TR 787095552,787103743,EU 787103744,787111935,HR @@ -3539,9 +3946,9 @@ 788234240,788242431,SA 788242432,788250623,IR 788250624,788258815,KG -788258816,788259839,DE -788259840,788260351,NL -788260352,788267007,DE +788258816,788259583,DE +788259584,788260863,NL +788260864,788267007,DE 788267008,788271103,SE 788271104,788275199,DE 788275200,788279295,AL @@ -3623,7 +4030,8 @@ 788502528,788504575,IE 788504576,788506623,FR 788506624,788508671,CH -788508672,788510719,AT +788508672,788509183,FR +788509184,788510719,AT 788510720,788512767,ES 788512768,788514815,FI 788514816,788516863,FR @@ -3635,7 +4043,6 @@ 788527104,788529151,GB 788529152,805306367,CA 805306368,822083583,US -822083840,822084095,AU 822084608,822085631,ID 822085632,822087679,AU 822087680,822089727,JP @@ -3652,6 +4059,7 @@ 823132160,824180735,KR 824180736,825229311,IN 825229312,825360383,TH +825360384,825361407,KR 825361408,825363455,ID 825363456,825364479,MY 825364480,825376767,KR @@ -3661,7 +4069,6 @@ 825419776,825420799,TH 825420800,825421823,MY 825421824,825425919,NZ -825425920,825491455,AU 825491456,825753599,CN 825753600,826277887,KR 826277888,828375039,CN @@ -3669,7 +4076,7 @@ 829423616,830210047,CN 830210048,830341119,MY 830341120,830406655,NP -830406656,830474239,AU +830472192,830474239,AU 830474240,830475263,SG 830475264,830476287,AU 830476288,830480383,JP @@ -3752,30 +4159,48 @@ 838336512,838467583,IN 838467584,838729727,JP 838729728,838795263,KR +838795264,838860799,AU 838860800,838999039,US 838999040,838999295,CA -838999296,839010559,US +838999296,839010175,US +839010176,839010207,CA +839010208,839010559,US 839010560,839010815,CA 839010816,839017983,US 839017984,839018239,CA 839018240,839023359,US 839023360,839023615,CA -839023616,839061503,US +839023616,839025663,US +839025664,839026175,CA +839026176,839061503,US 839061504,839061759,CA 839061760,839062271,US 839062272,839064063,CA -839064064,839113215,US +839064064,839069183,US +839069184,839074303,CA +839074304,839095807,US +839095808,839096063,CA +839096064,839101695,US +839101696,839102207,CA +839102208,839103999,US +839104000,839104255,CA +839104256,839110143,US +839110144,839110399,CA +839110400,839112447,US +839112448,839112703,CA +839112704,839113215,US 839113216,839113471,CA 839113472,840269823,US 840269824,840273919,CA 840273920,840278015,US 840278016,840282111,CA 840282112,840294399,US -840302592,843055103,US +840294400,840298495,CA +840298496,843055103,US 843055104,843644927,CA 843644928,844890111,US 844890112,844988415,CA -844988416,845021183,US +844988416,845217791,US 847249408,855638015,US 855638016,872415231,GB 872415232,889192447,US @@ -4461,9 +4886,7 @@ 1041704480,1041704487,GB 1041704488,1041704551,FR 1041704552,1041704575,GB -1041704576,1041704671,FR -1041704672,1041704687,GB -1041704688,1041704751,FR +1041704576,1041704751,FR 1041704752,1041704759,GB 1041704760,1041704775,FR 1041704776,1041704783,GB @@ -4473,8 +4896,8 @@ 1041704848,1041704863,GB 1041704864,1041704879,FR 1041704880,1041704887,GB -1041704888,1041704943,FR -1041704944,1041704959,GB +1041704888,1041704927,FR +1041704928,1041704959,GB 1041704960,1041705231,FR 1041705232,1041705239,GB 1041705240,1041705255,FR @@ -4606,8 +5029,8 @@ 1041710016,1041710047,FR 1041710048,1041710055,GB 1041710056,1041710343,FR -1041710344,1041710359,GB -1041710360,1041710391,FR +1041710344,1041710367,GB +1041710368,1041710391,FR 1041710392,1041710407,GB 1041710408,1041710431,FR 1041710432,1041710455,GB @@ -4637,7 +5060,9 @@ 1041711944,1041711951,GB 1041711952,1041712415,FR 1041712416,1041712431,GB -1041712432,1041713095,FR +1041712432,1041712975,FR +1041712976,1041712983,GB +1041712984,1041713095,FR 1041713096,1041713103,GB 1041713104,1041713127,FR 1041713128,1041713135,GB @@ -4655,10 +5080,10 @@ 1041714096,1041714103,GB 1041714104,1041714111,FR 1041714112,1041714175,GB -1041714176,1041714487,FR -1041714488,1041714495,GB -1041714496,1041714631,FR -1041714632,1041714687,GB +1041714176,1041714631,FR +1041714632,1041714639,GB +1041714640,1041714655,FR +1041714656,1041714687,GB 1041714688,1041714775,FR 1041714776,1041714783,GB 1041714784,1041714815,FR @@ -4669,8 +5094,8 @@ 1041714912,1041714919,GB 1041714920,1041714999,FR 1041715000,1041715015,GB -1041715016,1041715055,FR -1041715056,1041715071,GB +1041715016,1041715063,FR +1041715064,1041715071,GB 1041715072,1041715079,FR 1041715080,1041715087,GB 1041715088,1041715095,FR @@ -4729,7 +5154,9 @@ 1041717200,1041717247,GB 1041717248,1041717503,FR 1041717504,1041717511,IT -1041717512,1041717687,FR +1041717512,1041717631,FR +1041717632,1041717639,GB +1041717640,1041717687,FR 1041717688,1041717695,GB 1041717696,1041718112,FR 1041718113,1041718119,GB @@ -4787,8 +5214,8 @@ 1041719920,1041719935,GB 1041719936,1041720015,FR 1041720016,1041720031,GB -1041720032,1041720335,FR -1041720336,1041720351,GB +1041720032,1041720343,FR +1041720344,1041720351,GB 1041720352,1041720551,FR 1041720552,1041720559,GB 1041720560,1041720567,FR @@ -4847,7 +5274,9 @@ 1041723888,1041723903,GB 1041723904,1041724455,FR 1041724456,1041724463,GB -1041724464,1041724535,FR +1041724464,1041724511,FR +1041724512,1041724519,GB +1041724520,1041724535,FR 1041724536,1041724543,GB 1041724544,1041724671,FR 1041724672,1041724927,GB @@ -4857,7 +5286,9 @@ 1041725120,1041725167,GB 1041725168,1041725175,FR 1041725176,1041725183,GB -1041725184,1041725207,FR +1041725184,1041725191,FR +1041725192,1041725199,GB +1041725200,1041725207,FR 1041725208,1041725231,GB 1041725232,1041725255,FR 1041725256,1041725279,GB @@ -4945,8 +5376,8 @@ 1041730240,1041730247,GB 1041730248,1041730639,FR 1041730640,1041730655,GB -1041730656,1041731983,FR -1041731984,1041732031,GB +1041730656,1041731975,FR +1041731976,1041732031,GB 1041732032,1041732055,FR 1041732056,1041732063,GB 1041732064,1041732103,FR @@ -4981,7 +5412,9 @@ 1041735000,1041735039,GB 1041735040,1041735055,FR 1041735056,1041735087,GB -1041735088,1041735119,FR +1041735088,1041735095,FR +1041735096,1041735103,GB +1041735104,1041735119,FR 1041735120,1041735167,GB 1041735168,1041735431,FR 1041735432,1041735495,GB @@ -4991,7 +5424,9 @@ 1041735568,1041735583,GB 1041735584,1041735615,FR 1041735616,1041735663,GB -1041735664,1041736375,FR +1041735664,1041736247,FR +1041736248,1041736255,GB +1041736256,1041736375,FR 1041736376,1041736383,GB 1041736384,1041736423,FR 1041736424,1041736463,GB @@ -5063,16 +5498,16 @@ 1041738584,1041738591,GB 1041738592,1041738599,FR 1041738600,1041738615,GB -1041738616,1041738703,FR +1041738616,1041738663,FR +1041738664,1041738671,GB +1041738672,1041738703,FR 1041738704,1041738711,GB 1041738712,1041738735,FR 1041738736,1041738751,GB 1041738752,1041739079,FR 1041739080,1041739087,GB 1041739088,1041739167,FR -1041739168,1041739175,GB -1041739176,1041739183,FR -1041739184,1041739207,GB +1041739168,1041739207,GB 1041739208,1041739215,FR 1041739216,1041739231,GB 1041739232,1041739295,FR @@ -5258,8 +5693,8 @@ 1041749776,1041749783,GB 1041749784,1041749807,FR 1041749808,1041749815,GB -1041749816,1041749847,FR -1041749848,1041749855,GB +1041749816,1041749839,FR +1041749840,1041749855,GB 1041749856,1041749967,FR 1041749968,1041749983,GB 1041749984,1041749999,FR @@ -5272,7 +5707,9 @@ 1041750744,1041750751,GB 1041750752,1041750775,FR 1041750776,1041750783,GB -1041750784,1041751567,FR +1041750784,1041751351,FR +1041751352,1041751359,GB +1041751360,1041751567,FR 1041751568,1041751575,GB 1041751576,1041751695,FR 1041751696,1041751719,GB @@ -5518,7 +5955,11 @@ 1042894144,1042894175,SE 1042894176,1042894335,NL 1042894336,1042894355,FR -1042894356,1042894431,NL +1042894356,1042894359,NL +1042894360,1042894367,GB +1042894368,1042894399,NL +1042894400,1042894415,GB +1042894416,1042894431,NL 1042894432,1042894463,PT 1042894464,1042894559,NL 1042894560,1042894591,DE @@ -5575,24 +6016,28 @@ 1043466320,1043466327,GB 1043466328,1043466335,NL 1043466336,1043466351,GB -1043466352,1043466431,NL +1043466352,1043466391,NL +1043466392,1043466399,GB +1043466400,1043466431,NL 1043466432,1043466447,GB 1043466448,1043466455,NL 1043466456,1043466463,GB 1043466464,1043466495,NL 1043466496,1043466503,GB 1043466504,1043466511,NL -1043466512,1043466519,GB -1043466520,1043466559,NL +1043466512,1043466527,GB +1043466528,1043466559,NL 1043466560,1043466575,GB 1043466576,1043466583,NL 1043466584,1043466607,GB -1043466608,1043466887,NL +1043466608,1043466623,NL +1043466624,1043466639,GB +1043466640,1043466671,NL +1043466672,1043466687,GB +1043466688,1043466887,NL 1043466888,1043466895,GB 1043466896,1043466911,NL -1043466912,1043466927,GB -1043466928,1043466943,NL -1043466944,1043467039,GB +1043466912,1043467039,GB 1043467040,1043467071,NL 1043467072,1043467087,GB 1043467088,1043467103,NL @@ -6363,8 +6808,8 @@ 1044932832,1044932839,GB 1044932840,1044932847,BE 1044932848,1044932863,GB -1044932864,1044932939,BE -1044932940,1044932947,GB +1044932864,1044932935,BE +1044932936,1044932947,GB 1044932948,1044932951,BE 1044932952,1044932959,GB 1044932960,1044933007,BE @@ -6554,8 +6999,7 @@ 1045018560,1045018623,GB 1045018624,1045018751,ES 1045018752,1045020159,GB -1045020160,1045020655,ES -1045020656,1045020671,GB +1045020160,1045020671,ES 1045020672,1045037055,NO 1045037056,1045119231,GR 1045119232,1045119743,AL @@ -6567,7 +7011,7 @@ 1045154592,1045154623,DE 1045154624,1045154655,GB 1045154656,1045154687,DE -1045154688,1045154719,NL +1045154688,1045154719,US 1045154720,1045154751,DE 1045154752,1045154783,SE 1045154784,1045155071,DE @@ -6576,9 +7020,7 @@ 1045160960,1045160991,TR 1045160992,1045161023,DE 1045161024,1045161055,AT -1045161056,1045161151,DE -1045161152,1045161183,PT -1045161184,1045168127,DE +1045161056,1045168127,DE 1045168128,1045233663,RU 1045233664,1045241855,GB 1045241856,1045250047,IT @@ -6601,27 +7043,11 @@ 1045446656,1045446911,HU 1045446912,1045447167,SK 1045447168,1045447231,HU -1045447232,1045447239,SK -1045447240,1045447263,HU -1045447264,1045447295,SK +1045447232,1045447295,SK 1045447296,1045447311,HU -1045447312,1045447423,SK -1045447424,1045447431,HU -1045447432,1045447439,SK -1045447440,1045447447,HU -1045447448,1045447471,SK -1045447472,1045447487,HU -1045447488,1045447519,SK +1045447312,1045447519,SK 1045447520,1045447551,HU -1045447552,1045448031,SK -1045448032,1045448039,HU -1045448040,1045448055,SK -1045448056,1045448063,HU -1045448064,1045448159,SK -1045448160,1045448191,HU -1045448192,1045448239,SK -1045448240,1045448247,HU -1045448248,1045448255,SK +1045447552,1045448255,SK 1045448256,1045448263,HU 1045448264,1045448703,SK 1045448704,1045448767,HU @@ -7133,8 +7559,8 @@ 1046535488,1046535551,DE 1046535552,1046535619,GB 1046535620,1046535623,DE -1046535624,1046535631,GB -1046535632,1046535935,DE +1046535624,1046535679,GB +1046535680,1046535935,DE 1046535936,1046536023,GB 1046536024,1046536039,DE 1046536040,1046536063,GB @@ -7155,9 +7581,7 @@ 1046536320,1046536351,DE 1046536352,1046536355,GB 1046536356,1046536359,DE -1046536360,1046536423,GB -1046536424,1046536431,DE -1046536432,1046536607,GB +1046536360,1046536607,GB 1046536608,1046536667,DE 1046536668,1046536703,GB 1046536704,1046537023,DE @@ -7184,8 +7608,8 @@ 1046541696,1046541727,GB 1046541728,1046541759,DE 1046541760,1046541767,GB -1046541768,1046541783,DE -1046541784,1046541807,GB +1046541768,1046541775,DE +1046541776,1046541807,GB 1046541808,1046543103,DE 1046543104,1046543263,GB 1046543264,1046543295,DE @@ -7209,7 +7633,9 @@ 1046757376,1046765567,IT 1046765568,1046773759,ES 1046773760,1046781951,FR -1046781952,1046798335,GB +1046781952,1046790143,GB +1046790144,1046791167,US +1046791168,1046798335,GB 1046798336,1046806527,HU 1046806528,1046814719,IT 1046814720,1046822911,SE @@ -7302,6 +7728,12 @@ 1046898688,1046898943,BE 1046898944,1046899167,EU 1046899168,1046904831,BE +1046904832,1046908927,SK +1046908928,1046910975,SE +1046910976,1046913023,IT +1046913024,1046921215,NO +1046921216,1046929407,DE +1046929408,1046937599,FR 1046937600,1047003135,GR 1047003136,1047068671,DE 1047068672,1047085055,RU @@ -7317,7 +7749,9 @@ 1047265280,1047273471,NL 1047273472,1047281663,DE 1047281664,1047289855,TR -1047289856,1047298047,RS +1047289856,1047295351,RS +1047295352,1047295359,HU +1047295360,1047298047,RS 1047298048,1047300095,CH 1047300096,1047300415,GB 1047300416,1047300479,CH @@ -7418,9 +7852,7 @@ 1047566872,1047566879,CH 1047566880,1047566891,DE 1047566892,1047566903,CH -1047566904,1047566911,DE -1047566912,1047566915,CH -1047566916,1047566947,DE +1047566904,1047566947,DE 1047566948,1047566951,AT 1047566952,1047566963,DE 1047566964,1047566967,AT @@ -7445,7 +7877,9 @@ 1047567160,1047567163,DE 1047567164,1047567167,AT 1047567168,1047567175,CH -1047567176,1047567211,DE +1047567176,1047567183,DE +1047567184,1047567191,CH +1047567192,1047567211,DE 1047567212,1047567227,CH 1047567228,1047567239,DE 1047567240,1047567243,CH @@ -7454,11 +7888,11 @@ 1047567256,1047567279,DE 1047567280,1047567287,CH 1047567288,1047567307,DE -1047567308,1047567315,CH +1047567308,1047567311,CH +1047567312,1047567315,DE 1047567316,1047567319,AT -1047567320,1047567359,DE -1047567360,1047567375,ES -1047567376,1047567447,DE +1047567320,1047567327,CH +1047567328,1047567447,DE 1047567448,1047567451,CH 1047567452,1047567455,AT 1047567456,1047567459,CH @@ -7607,9 +8041,7 @@ 1048604688,1048604743,UA 1048604744,1048604759,EE 1048604760,1048604927,UA -1048604928,1048604965,LT -1048604966,1048604967,UA -1048604968,1048605079,LT +1048604928,1048605079,LT 1048605080,1048607231,UA 1048607232,1048607247,EE 1048607248,1048607487,UA @@ -8615,9 +9047,7 @@ 1050505248,1050505279,CA 1050505280,1050507503,DE 1050507504,1050507519,A2 -1050507520,1050515647,DE -1050515648,1050515663,DK -1050515664,1050522407,DE +1050507520,1050522407,DE 1050522408,1050522415,CH 1050522416,1050619503,DE 1050619504,1050619511,CH @@ -8773,39 +9203,21 @@ 1051566080,1051574271,IT 1051574272,1051574623,GB 1051574624,1051574655,GM -1051574656,1051575999,GB -1051576000,1051576015,IT -1051576016,1051577503,GB -1051577504,1051577519,NL -1051577520,1051577535,GB +1051574656,1051577535,GB 1051577536,1051577567,NL -1051577568,1051577575,BE -1051577576,1051577583,GB -1051577584,1051577587,FI -1051577588,1051577591,GB -1051577592,1051577599,NL -1051577600,1051577855,GB +1051577568,1051577855,GB 1051577856,1051578111,NL 1051578112,1051578143,GB 1051578144,1051578175,NL -1051578176,1051578239,GB -1051578240,1051578263,FR -1051578264,1051578271,GB -1051578272,1051578279,ES -1051578280,1051578287,GR -1051578288,1051578295,GB -1051578296,1051578303,PL +1051578176,1051578247,GB +1051578248,1051578255,FR +1051578256,1051578303,GB 1051578304,1051578335,NL -1051578336,1051578343,DE -1051578344,1051578351,NL -1051578352,1051578363,GB -1051578364,1051578367,DE -1051578368,1051580415,SZ -1051580416,1051584767,GB -1051584768,1051585535,MG -1051585536,1051585599,GB -1051585600,1051585663,MG -1051585664,1051590655,GB +1051578336,1051580535,GB +1051580536,1051580543,SZ +1051580544,1051584207,GB +1051584208,1051584223,BE +1051584224,1051590655,GB 1051590656,1051702527,ES 1051702528,1051702783,US 1051702784,1051721727,ES @@ -8823,8 +9235,10 @@ 1051852800,1051918335,AT 1051918336,1051919359,PL 1051919360,1051920383,AT -1051920384,1051922431,PL -1051922432,1051949823,AT +1051920384,1051920895,PL +1051920896,1051934719,AT +1051934720,1051938815,DE +1051938816,1051949823,AT 1051949824,1051949951,NL 1051949952,1051983871,AT 1051983872,1051986687,EU @@ -9115,7 +9529,9 @@ 1052065792,1052082175,SE 1052082176,1052090367,DE 1052090368,1052098559,PL -1052098560,1052116991,SE +1052098560,1052099471,SE +1052099472,1052099479,NO +1052099480,1052116991,SE 1052116992,1052119039,RU 1052119040,1052121087,RO 1052121088,1052129279,RU @@ -10080,8 +10496,8 @@ 1053312928,1053312959,DK 1053312960,1053313023,EU 1053313024,1053313343,GB -1053313344,1053313439,EU -1053313440,1053313599,GB +1053313344,1053313407,EU +1053313408,1053313599,GB 1053313600,1053313663,EU 1053313664,1053313671,GB 1053313672,1053313679,EU @@ -10124,8 +10540,8 @@ 1053327360,1053327871,EU 1053327872,1053328383,ZA 1053328384,1053328639,EU -1053328640,1053328799,ZA -1053328800,1053328895,EU +1053328640,1053328863,ZA +1053328864,1053328895,EU 1053328896,1053329087,ZA 1053329088,1053329119,EU 1053329120,1053329135,ZA @@ -10134,8 +10550,8 @@ 1053329152,1053329183,EU 1053329184,1053329279,ZA 1053329280,1053329407,EU -1053329408,1053329439,ES -1053329440,1053329503,EU +1053329408,1053329471,ES +1053329472,1053329503,EU 1053329504,1053329527,ES 1053329528,1053329631,EU 1053329632,1053329663,ES @@ -10289,7 +10705,9 @@ 1053824024,1053824127,EU 1053824128,1053824255,NL 1053824256,1053825023,EU -1053825024,1053826047,ES +1053825024,1053825791,ES +1053825792,1053825919,GB +1053825920,1053826047,ES 1053826048,1053826815,EU 1053826816,1053827327,DE 1053827328,1053827583,BE @@ -10974,7 +11392,7 @@ 1066332160,1066336255,CA 1066336256,1066344447,US 1066352640,1066369023,JM -1066401792,1066584063,US +1066369024,1066584063,US 1066584064,1066586111,PE 1066586112,1066604927,US 1066604928,1066604959,CA @@ -11475,9 +11893,7 @@ 1074675312,1074675327,NO 1074675328,1074675679,US 1074675680,1074675687,GB -1074675688,1074677759,US -1074677760,1074678271,BG -1074678272,1074680623,US +1074675688,1074680623,US 1074680624,1074680639,CA 1074680640,1074680703,US 1074680704,1074680719,IL @@ -11702,8 +12118,8 @@ 1075972352,1075973887,US 1075973888,1075974143,CA 1075974144,1075975167,US -1075975168,1075975999,CA -1075976000,1075976063,US +1075975168,1075976031,CA +1075976032,1075976063,US 1075976064,1075976511,CA 1075976512,1075976543,US 1075976544,1075976647,CA @@ -11730,7 +12146,7 @@ 1075982592,1075982599,US 1075982600,1075982607,NZ 1075982608,1075982655,CA -1075982656,1075982671,US +1075982656,1075982671,MX 1075982672,1075982687,CA 1075982688,1075982703,US 1075982704,1075982751,CA @@ -11745,27 +12161,14 @@ 1075983232,1075984383,CA 1075984384,1075984823,US 1075984824,1075984831,CA -1075984832,1075985415,US -1075985416,1075985423,CR -1075985424,1075985919,US -1075985920,1075986431,CA -1075986432,1075986687,US -1075986688,1075986943,CA -1075986944,1075987199,US -1075987200,1075987455,CA -1075987456,1075987711,US -1075987712,1075987951,CA -1075987952,1075987967,US -1075987968,1075988287,CA -1075988288,1075988319,US -1075988320,1075988479,CA -1075988480,1075988735,US -1075988736,1075988991,CA -1075988992,1075989119,US -1075989120,1075989239,CA -1075989240,1075989247,US +1075984832,1075985919,US +1075985920,1075989055,CA +1075989056,1075989119,US +1075989120,1075989247,CA 1075989248,1075989311,FI -1075989312,1075989359,CA +1075989312,1075989327,CA +1075989328,1075989335,US +1075989336,1075989359,CA 1075989360,1075989375,US 1075989376,1075989455,CA 1075989456,1075989471,US @@ -11780,11 +12183,16 @@ 1075992320,1075992447,GB 1075992448,1075993087,US 1075993088,1075993103,CA -1075993104,1075995007,US +1075993104,1075994607,US +1075994608,1075994623,BE +1075994624,1075995007,CA 1075995008,1075995023,VG -1075995024,1075995047,US -1075995048,1075995055,CA -1075995056,1075995103,US +1075995024,1075995039,US +1075995040,1075995055,CA +1075995056,1075995071,US +1075995072,1075995079,MX +1075995080,1075995087,CA +1075995088,1075995103,US 1075995104,1075995263,CA 1075995264,1075995295,US 1075995296,1075995326,CA @@ -11872,9 +12280,7 @@ 1076007168,1076007183,US 1076007184,1076007247,CA 1076007248,1076007263,US -1076007264,1076007679,CA -1076007680,1076007935,US -1076007936,1076007947,CA +1076007264,1076007947,CA 1076007948,1076007951,US 1076007952,1076007967,CA 1076007968,1076007983,US @@ -11888,9 +12294,7 @@ 1076009696,1076009711,US 1076009712,1076009887,CA 1076009888,1076009919,US -1076009920,1076010687,CA -1076010688,1076010695,US -1076010696,1076011007,CA +1076009920,1076011007,CA 1076011008,1076012236,US 1076012237,1076012237,IN 1076012238,1076013094,US @@ -11916,9 +12320,7 @@ 1076027776,1076027791,US 1076027792,1076027911,CA 1076027912,1076027919,US -1076027920,1076027927,CA -1076027928,1076027935,US -1076027936,1076028143,CA +1076027920,1076028143,CA 1076028144,1076028151,US 1076028152,1076028479,CA 1076028480,1076028543,GB @@ -11930,7 +12332,9 @@ 1076029224,1076029231,DM 1076029232,1076029239,CA 1076029240,1076029247,US -1076029248,1076029415,CA +1076029248,1076029311,CA +1076029312,1076029319,GB +1076029320,1076029415,CA 1076029416,1076029422,US 1076029423,1076029430,CA 1076029431,1076029439,US @@ -12942,10 +13346,8 @@ 1077859952,1077859967,US 1077859968,1077860095,IT 1077860096,1077861119,US -1077861120,1077861631,IT -1077861632,1077862911,US -1077862912,1077863167,IT -1077863168,1077863207,US +1077861120,1077861375,IT +1077861376,1077863207,US 1077863208,1077863215,NG 1077863216,1077864703,US 1077864704,1077864719,IT @@ -13055,7 +13457,292 @@ 1077936623,1077936630,US 1077936631,1077936634,PE 1077936635,1077936638,CA -1077936639,1077938702,US +1077936639,1077936645,US +1077936646,1077936649,HK +1077936650,1077936695,US +1077936696,1077936699,OM +1077936700,1077936759,US +1077936760,1077936763,GB +1077936764,1077936771,US +1077936772,1077936775,IT +1077936776,1077936818,US +1077936819,1077936822,RS +1077936823,1077936834,US +1077936835,1077936838,VE +1077936839,1077936846,US +1077936847,1077936850,PK +1077936851,1077936854,US +1077936855,1077936859,IN +1077936860,1077936891,US +1077936892,1077936892,OM +1077936893,1077936913,US +1077936914,1077936917,ES +1077936918,1077936921,ID +1077936922,1077936945,US +1077936946,1077936949,GB +1077936950,1077936957,US +1077936958,1077936961,MA +1077936962,1077936965,TR +1077936966,1077936969,SE +1077936970,1077936981,US +1077936982,1077936985,AR +1077936986,1077936989,ES +1077936990,1077937005,US +1077937006,1077937007,CA +1077937008,1077937015,US +1077937016,1077937019,CA +1077937020,1077937023,GB +1077937024,1077937027,DE +1077937028,1077937047,US +1077937048,1077937051,BZ +1077937052,1077937067,US +1077937068,1077937071,GB +1077937072,1077937079,US +1077937080,1077937083,SE +1077937084,1077937087,US +1077937088,1077937088,BE +1077937089,1077937089,CA +1077937090,1077937093,IT +1077937094,1077937105,US +1077937106,1077937109,CA +1077937110,1077937129,US +1077937130,1077937133,CA +1077937134,1077937137,NG +1077937138,1077937145,US +1077937146,1077937149,CA +1077937150,1077937157,US +1077937158,1077937161,TR +1077937162,1077937165,NI +1077937166,1077937180,US +1077937181,1077937184,SA +1077937185,1077937196,US +1077937197,1077937200,LK +1077937201,1077937204,GB +1077937205,1077937208,AT +1077937209,1077937212,US +1077937213,1077937214,CA +1077937215,1077937222,US +1077937223,1077937226,CA +1077937227,1077937230,JM +1077937231,1077937235,US +1077937236,1077937239,JO +1077937240,1077937260,US +1077937261,1077937269,EE +1077937270,1077937273,LV +1077937274,1077937289,US +1077937290,1077937293,GF +1077937294,1077937297,US +1077937298,1077937301,GB +1077937302,1077937305,US +1077937306,1077937309,ES +1077937310,1077937335,US +1077937336,1077937349,GB +1077937350,1077937353,CA +1077937354,1077937357,US +1077937358,1077937361,CA +1077937362,1077937365,US +1077937366,1077937369,AR +1077937370,1077937394,CA +1077937395,1077937398,CN +1077937399,1077937402,MX +1077937403,1077937417,US +1077937418,1077937421,IN +1077937422,1077937429,US +1077937430,1077937433,TH +1077937434,1077937437,GB +1077937438,1077937441,CA +1077937442,1077937445,US +1077937446,1077937447,ES +1077937448,1077937449,US +1077937450,1077937453,GR +1077937454,1077937457,SA +1077937458,1077937461,GB +1077937462,1077937474,US +1077937475,1077937478,DK +1077937479,1077937482,MX +1077937483,1077937484,GB +1077937485,1077937485,LK +1077937486,1077937502,US +1077937503,1077937506,GB +1077937507,1077937510,CH +1077937511,1077937514,CR +1077937515,1077937531,US +1077937532,1077937535,IN +1077937536,1077937539,ES +1077937540,1077937547,AT +1077937548,1077937571,US +1077937572,1077937575,CY +1077937576,1077937577,GB +1077937578,1077937597,US +1077937598,1077937601,IN +1077937602,1077937609,US +1077937610,1077937613,NZ +1077937614,1077937617,US +1077937618,1077937621,IN +1077937622,1077937625,GB +1077937626,1077937629,IT +1077937630,1077937678,US +1077937679,1077937682,ZA +1077937683,1077937686,US +1077937687,1077937690,IN +1077937691,1077937702,US +1077937703,1077937706,PK +1077937707,1077937710,RS +1077937711,1077937714,AN +1077937715,1077937718,US +1077937719,1077937722,IN +1077937723,1077937726,AU +1077937727,1077937730,SG +1077937731,1077937734,US +1077937735,1077937738,BA +1077937739,1077937742,IL +1077937743,1077937750,US +1077937751,1077937754,AU +1077937755,1077937758,US +1077937759,1077937762,NL +1077937763,1077937766,US +1077937767,1077937770,RU +1077937771,1077937782,US +1077937783,1077937786,GR +1077937787,1077937790,GB +1077937791,1077937802,US +1077937803,1077937806,BE +1077937807,1077937822,US +1077937823,1077937826,GB +1077937827,1077937846,US +1077937847,1077937850,TH +1077937851,1077937854,US +1077937855,1077937866,TH +1077937867,1077937870,GB +1077937871,1077937878,US +1077937879,1077937882,GB +1077937883,1077937886,US +1077937887,1077937890,GR +1077937891,1077937902,US +1077937903,1077937906,IN +1077937907,1077937933,US +1077937934,1077937937,IT +1077937938,1077937941,US +1077937942,1077937945,NO +1077937946,1077937949,MY +1077937950,1077937953,CL +1077937954,1077937963,US +1077937964,1077937967,NO +1077937968,1077937975,US +1077937976,1077937979,TH +1077937980,1077937983,CN +1077937984,1077938003,US +1077938004,1077938007,TH +1077938008,1077938011,US +1077938012,1077938021,GB +1077938022,1077938025,US +1077938026,1077938029,FR +1077938030,1077938033,US +1077938034,1077938037,SA +1077938038,1077938041,AU +1077938042,1077938045,US +1077938046,1077938049,RU +1077938050,1077938053,US +1077938054,1077938057,AU +1077938058,1077938073,US +1077938074,1077938077,CN +1077938078,1077938081,US +1077938082,1077938085,IN +1077938086,1077938089,US +1077938090,1077938093,CA +1077938094,1077938121,US +1077938122,1077938125,DZ +1077938126,1077938129,IN +1077938130,1077938133,ID +1077938134,1077938137,CA +1077938138,1077938141,GB +1077938142,1077938145,CA +1077938146,1077938149,US +1077938150,1077938153,IN +1077938154,1077938157,AR +1077938158,1077938185,US +1077938186,1077938189,PH +1077938190,1077938201,US +1077938202,1077938205,CA +1077938206,1077938209,AR +1077938210,1077938221,US +1077938222,1077938225,IN +1077938226,1077938233,US +1077938234,1077938237,AR +1077938238,1077938258,US +1077938259,1077938262,PE +1077938263,1077938266,CA +1077938267,1077938269,GB +1077938270,1077938277,US +1077938278,1077938281,GB +1077938282,1077938283,DE +1077938284,1077938284,CH +1077938285,1077938288,CA +1077938289,1077938296,GB +1077938297,1077938301,US +1077938302,1077938305,GB +1077938306,1077938309,US +1077938310,1077938313,TH +1077938314,1077938321,US +1077938322,1077938325,GB +1077938326,1077938329,PK +1077938330,1077938333,US +1077938334,1077938337,MX +1077938338,1077938345,US +1077938346,1077938349,SA +1077938350,1077938357,US +1077938358,1077938361,ID +1077938362,1077938365,US +1077938366,1077938372,IN +1077938373,1077938376,DK +1077938377,1077938380,IL +1077938381,1077938384,LT +1077938385,1077938396,US +1077938397,1077938400,CA +1077938401,1077938404,US +1077938405,1077938408,IT +1077938409,1077938412,US +1077938413,1077938416,PT +1077938417,1077938426,US +1077938427,1077938430,SE +1077938431,1077938437,US +1077938438,1077938441,PE +1077938442,1077938445,US +1077938446,1077938446,BE +1077938447,1077938490,US +1077938491,1077938494,MT +1077938495,1077938502,US +1077938503,1077938506,ES +1077938507,1077938510,DK +1077938511,1077938529,US +1077938530,1077938533,NZ +1077938534,1077938537,CA +1077938538,1077938541,ES +1077938542,1077938545,CY +1077938546,1077938551,US +1077938552,1077938552,ES +1077938553,1077938570,US +1077938571,1077938571,NL +1077938572,1077938572,US +1077938573,1077938576,CO +1077938577,1077938580,US +1077938581,1077938584,CA +1077938585,1077938588,US +1077938589,1077938592,MX +1077938593,1077938596,US +1077938597,1077938600,DO +1077938601,1077938630,US +1077938631,1077938644,IN +1077938645,1077938648,US +1077938649,1077938652,CA +1077938653,1077938656,CN +1077938657,1077938660,RO +1077938661,1077938664,TR +1077938665,1077938668,CA +1077938669,1077938672,US +1077938673,1077938676,GB +1077938677,1077938680,IN +1077938681,1077938702,US 1077938703,1077938706,ID 1077938707,1077938734,GB 1077938735,1077938735,PK @@ -13064,14 +13751,13 @@ 1077938750,1077938750,NL 1077938751,1077938765,US 1077938766,1077938769,CN -1077938770,1077938773,US -1077938774,1077938777,GB +1077938770,1077938777,US 1077938778,1077938781,RO 1077938782,1077938785,US 1077938786,1077938789,GR 1077938790,1077938793,CA 1077938794,1077938797,IE -1077938798,1077938801,AR +1077938798,1077938801,SG 1077938802,1077938805,US 1077938806,1077938809,DK 1077938810,1077938813,HK @@ -13111,8 +13797,7 @@ 1077939061,1077939064,IE 1077939065,1077939065,TH 1077939066,1077939069,CA -1077939070,1077939074,TH -1077939075,1077939078,US +1077939070,1077939078,US 1077939079,1077939082,IN 1077939083,1077939086,CA 1077939087,1077939090,ES @@ -13131,7 +13816,12 @@ 1077939211,1077939218,US 1077939219,1077939219,NL 1077939220,1077939223,CA -1077939224,1077939304,US +1077939224,1077939240,US +1077939241,1077939244,NL +1077939245,1077939251,US +1077939252,1077939255,BR +1077939256,1077939259,FR +1077939260,1077939304,US 1077939305,1077939308,GB 1077939309,1077939321,US 1077939322,1077939329,CA @@ -13150,11 +13840,179 @@ 1077939396,1077939399,SE 1077939400,1077939415,US 1077939416,1077939423,BE -1077939424,1077960751,US +1077939424,1077939491,US +1077939492,1077939495,GB +1077939496,1077939523,US +1077939524,1077939533,CA +1077939534,1077939569,US +1077939570,1077939574,OM +1077939575,1077939625,US +1077939626,1077939630,UY +1077939631,1077939634,GB +1077939635,1077939644,US +1077939645,1077939648,GR +1077939649,1077939652,IE +1077939653,1077939725,US +1077939726,1077939727,DE +1077939728,1077939790,US +1077939791,1077939794,PE +1077939795,1077939823,US +1077939824,1077939827,BR +1077939828,1077939837,PT +1077939838,1077939895,US +1077939896,1077939899,GB +1077939900,1077939973,US +1077939974,1077939977,CL +1077939978,1077939981,US +1077939982,1077939985,MU +1077939986,1077939997,US +1077939998,1077940001,GB +1077940002,1077940005,EG +1077940006,1077940072,US +1077940073,1077940076,BE +1077940077,1077940285,US +1077940286,1077940289,AU +1077940290,1077940378,US +1077940379,1077940382,PK +1077940383,1077942394,US +1077942395,1077942398,CA +1077942399,1077942493,US +1077942494,1077942497,MX +1077942498,1077942501,VN +1077942502,1077942505,IN +1077942506,1077942509,GB +1077942510,1077942513,AE +1077942514,1077942572,US +1077942573,1077942576,CA +1077942577,1077942600,US +1077942601,1077942604,PE +1077942605,1077942620,US +1077942621,1077942624,BD +1077942625,1077942746,US +1077942747,1077942750,GB +1077942751,1077943045,US +1077943046,1077943053,FI +1077943054,1077943109,US +1077943110,1077943110,CN +1077943111,1077943187,US +1077943188,1077943191,CA +1077943192,1077943313,US +1077943314,1077943314,BE +1077943315,1077943378,US +1077943379,1077943379,CZ +1077943380,1077943406,US +1077943407,1077943410,AU +1077943411,1077943420,US +1077943421,1077943421,CN +1077943422,1077943425,US +1077943426,1077943426,CN +1077943427,1077943430,MY +1077943431,1077943503,US +1077943504,1077943507,CA +1077943508,1077943523,US +1077943524,1077943527,AU +1077943528,1077943531,US +1077943532,1077943535,AU +1077943536,1077943539,ID +1077943540,1077943540,US +1077943541,1077943544,CA +1077943545,1077943574,US +1077943575,1077943575,BE +1077943576,1077943576,CZ +1077943577,1077943634,US +1077943635,1077943638,ZA +1077943639,1077943674,US +1077943675,1077943675,CN +1077943676,1077943789,US +1077943790,1077943790,AU +1077943791,1077943794,US +1077943795,1077943796,SG +1077943797,1077943825,US +1077943826,1077943829,HK +1077943830,1077943845,US +1077943846,1077943849,AU +1077943850,1077943853,UA +1077943854,1077943867,US +1077943868,1077943868,CN +1077943869,1077943876,US +1077943877,1077943880,CA +1077943881,1077943896,US +1077943897,1077943897,CN +1077943898,1077943901,US +1077943902,1077943905,AU +1077943906,1077943921,US +1077943922,1077943925,AU +1077943926,1077943939,US +1077943940,1077943940,BE +1077943941,1077943944,US +1077943945,1077943948,KH +1077943949,1077943964,US +1077943965,1077943968,IN +1077943969,1077943970,US +1077943971,1077943972,CN +1077943973,1077943976,PK +1077943977,1077943984,AU +1077943985,1077943992,US +1077943993,1077943996,CA +1077943997,1077944000,AR +1077944001,1077944013,US +1077944014,1077944017,IN +1077944018,1077944025,US +1077944026,1077944028,CN +1077944029,1077944058,US +1077944059,1077944062,CA +1077944063,1077944081,US +1077944082,1077944085,CA +1077944086,1077944097,US +1077944098,1077944101,AU +1077944102,1077944127,US +1077944128,1077944128,BE +1077944129,1077944136,US +1077944137,1077944137,BE +1077944138,1077944141,PL +1077944142,1077944173,US +1077944174,1077944177,CA +1077944178,1077944181,US +1077944182,1077944182,CN +1077944183,1077944186,US +1077944187,1077944190,PH +1077944191,1077944198,US +1077944199,1077944202,CA +1077944203,1077944210,CN +1077944211,1077944218,US +1077944219,1077944219,BE +1077944220,1077944223,US +1077944224,1077944227,CA +1077944228,1077944231,US +1077944232,1077944235,JP +1077944236,1077944243,US +1077944244,1077944247,AU +1077944248,1077944255,US +1077944256,1077944259,TH +1077944260,1077944263,PY +1077944264,1077944271,US +1077944272,1077944275,CA +1077944276,1077944279,US +1077944280,1077944283,TH +1077944284,1077944284,US +1077944285,1077944286,RO +1077944287,1077944295,US +1077944296,1077944299,CZ +1077944300,1077944308,US +1077944309,1077944312,CN +1077944313,1077944316,US +1077944317,1077944317,CZ +1077944318,1077960703,US +1077960704,1077960711,CA +1077960712,1077960727,US +1077960728,1077960735,SE +1077960736,1077960751,US 1077960752,1077960759,CA 1077960760,1077965855,US 1077965856,1077965887,BA -1077965888,1077968831,US +1077965888,1077965911,US +1077965912,1077965919,CA +1077965920,1077968831,US 1077968832,1077968839,CA 1077968840,1077977087,US 1077977088,1077985279,CA @@ -13217,7 +14075,9 @@ 1078283648,1078283679,US 1078283680,1078283687,CA 1078283688,1078283695,US -1078283696,1078283751,CA +1078283696,1078283735,CA +1078283736,1078283743,US +1078283744,1078283751,CA 1078283752,1078283775,US 1078283776,1078283999,CA 1078284000,1078284007,US @@ -13562,9 +14422,7 @@ 1078798688,1078798703,AU 1078798704,1078798911,US 1078798912,1078798927,IT -1078798928,1078799039,US -1078799040,1078799055,IT -1078799056,1078799071,US +1078798928,1078799071,US 1078799072,1078799087,CA 1078799088,1078799263,US 1078799264,1078799295,MX @@ -13708,7 +14566,9 @@ 1079394208,1079394239,CA 1079394240,1079394303,US 1079394304,1079395327,EC -1079395328,1079396351,CA +1079395328,1079395839,CA +1079395840,1079396095,US +1079396096,1079396351,CA 1079396352,1079397375,MP 1079397376,1079397631,MH 1079397632,1079399583,CA @@ -13804,7 +14664,9 @@ 1079857888,1079857903,NZ 1079857904,1079861247,US 1079861248,1079865343,CA -1079865344,1079944295,US +1079865344,1079943999,US +1079944000,1079944031,AU +1079944032,1079944295,US 1079944296,1079944303,IL 1079944304,1079946271,US 1079946272,1079946303,IE @@ -14042,9 +14904,7 @@ 1081397760,1081398783,CL 1081398784,1081399295,US 1081399296,1081401343,CL -1081401344,1081401375,AR -1081401376,1081401383,US -1081401384,1081401391,AR +1081401344,1081401391,AR 1081401392,1081401415,US 1081401416,1081401471,AR 1081401472,1081401567,US @@ -14136,7 +14996,29 @@ 1081483248,1081483255,US 1081483256,1081483263,CA 1081483264,1081487359,US -1081487360,1081491455,A2 +1081487360,1081487375,A2 +1081487376,1081487383,US +1081487384,1081487391,A2 +1081487392,1081487407,US +1081487408,1081487567,A2 +1081487568,1081487583,US +1081487584,1081487919,A2 +1081487920,1081487927,US +1081487928,1081487959,A2 +1081487960,1081487967,US +1081487968,1081488055,A2 +1081488056,1081488079,US +1081488080,1081488087,A2 +1081488088,1081488095,US +1081488096,1081488135,A2 +1081488136,1081488151,US +1081488152,1081488199,A2 +1081488200,1081488215,US +1081488216,1081488223,A2 +1081488224,1081488247,US +1081488248,1081488255,A2 +1081488256,1081488287,US +1081488288,1081491455,A2 1081491456,1081565183,US 1081565184,1081573375,CA 1081573376,1081581615,US @@ -14284,9 +15166,7 @@ 1081912576,1081912639,DE 1081912640,1081927135,US 1081927136,1081927143,GB -1081927144,1081955327,US -1081955328,1081955583,CA -1081955584,1081955839,US +1081927144,1081955839,US 1081955840,1081956095,CA 1081956096,1081966871,US 1081966872,1081966879,AS @@ -14355,7 +15235,9 @@ 1082885344,1082885855,US 1082885856,1082885887,CA 1082885888,1082945535,US -1082945536,1082949631,CA +1082945536,1082948943,CA +1082948944,1082948959,US +1082948960,1082949631,CA 1082949632,1082952703,US 1082952704,1082952959,CA 1082952960,1082982399,US @@ -14503,7 +15385,9 @@ 1085685670,1085685685,CA 1085685686,1085698199,US 1085698200,1085698207,AS -1085698208,1085849599,US +1085698208,1085823775,US +1085823776,1085823807,DE +1085823808,1085849599,US 1085849600,1085857791,CA 1085857792,1085915135,US 1085915136,1085923327,PR @@ -14529,11 +15413,7 @@ 1086025448,1086025455,CA 1086025456,1086027391,US 1086027392,1086027407,IT -1086027408,1086027423,US -1086027424,1086027455,VN -1086027456,1086028031,US -1086028032,1086028287,PA -1086028288,1086028663,US +1086027408,1086028663,US 1086028664,1086028671,BR 1086028672,1086028751,US 1086028752,1086028759,BR @@ -14673,9 +15553,7 @@ 1088101024,1088101031,PR 1088101032,1088108351,US 1088108352,1088108359,PR -1088108360,1088117199,US -1088117200,1088117207,PR -1088117208,1088120527,US +1088108360,1088120527,US 1088120528,1088120535,PR 1088120536,1088146151,US 1088146152,1088146159,PR @@ -15159,7 +16037,9 @@ 1093113480,1093113487,CA 1093113488,1093113503,US 1093113504,1093113535,CA -1093113536,1093113727,US +1093113536,1093113639,US +1093113640,1093113647,CA +1093113648,1093113727,US 1093113728,1093113743,CA 1093113744,1093113759,US 1093113760,1093113783,CA @@ -16205,8 +17085,7 @@ 1102019584,1102019711,IN 1102019712,1102389247,US 1102389248,1102393343,CA -1102393344,1102397439,US -1102413824,1102446591,US +1102393344,1102446591,US 1102446592,1102448383,HN 1102448384,1102449151,US 1102449152,1102449407,SV @@ -16791,7 +17670,9 @@ 1110887424,1110887679,IE 1110887680,1110929407,US 1110929408,1110933503,BM -1110933504,1111195647,US +1110933504,1110940751,US +1110940752,1110940767,A1 +1110940768,1111195647,US 1111195648,1111212031,CA 1111212032,1111228415,US 1111228416,1111244799,AR @@ -16882,7 +17763,9 @@ 1114170112,1114505215,US 1114505216,1114507263,CA 1114507264,1114507295,US -1114507296,1114507423,CA +1114507296,1114507327,CA +1114507328,1114507391,US +1114507392,1114507423,CA 1114507424,1114507431,GB 1114507432,1114511359,CA 1114511360,1114511871,US @@ -16901,9 +17784,7 @@ 1114533376,1114533887,ZA 1114533888,1114537983,AO 1114537984,1114550271,CA -1114550272,1114627071,US -1114627072,1114628095,NZ -1114628096,1114653951,US +1114550272,1114653951,US 1114653952,1114653983,MO 1114653984,1114657927,US 1114657928,1114657935,LA @@ -16954,13 +17835,9 @@ 1115111936,1115112063,PR 1115112064,1115112079,US 1115112080,1115112087,ES -1115112088,1115112159,US -1115112160,1115112191,SV -1115112192,1115112575,US +1115112088,1115112575,US 1115112576,1115112607,IN -1115112608,1115113167,US -1115113168,1115113175,SV -1115113176,1115113471,US +1115112608,1115113471,US 1115113472,1115114495,HN 1115114496,1115114751,MX 1115114752,1115115007,GT @@ -17090,9 +17967,7 @@ 1116214160,1116214167,PR 1116214168,1116217151,US 1116217152,1116217159,PR -1116217160,1116217439,US -1116217440,1116217447,PR -1116217448,1116218527,US +1116217160,1116218527,US 1116218528,1116218535,PR 1116218536,1116220231,US 1116220232,1116220239,PR @@ -17260,21 +18135,17 @@ 1117163024,1117163031,CA 1117163032,1117169503,US 1117169504,1117169535,CA -1117169536,1117178287,US +1117169536,1117171071,US +1117171072,1117171103,CA +1117171104,1117178287,US 1117178288,1117178303,GB -1117178304,1117182271,US -1117182272,1117182335,CA -1117182336,1117189951,US +1117178304,1117189951,US 1117189952,1117190015,KR 1117190016,1117195991,US 1117195992,1117195999,A2 1117196000,1117198591,US 1117198592,1117198847,CA -1117198848,1117206599,US -1117206600,1117206623,FR -1117206624,1117253247,US -1117253248,1117253255,IL -1117253256,1117274111,US +1117198848,1117274111,US 1117274112,1117282303,CA 1117282304,1117299615,US 1117299616,1117299623,GB @@ -17585,7 +18456,9 @@ 1118564608,1118564639,NO 1118564640,1118564655,US 1118564656,1118564671,IN -1118564672,1118789783,US +1118564672,1118564895,US +1118564896,1118564927,AU +1118564928,1118789783,US 1118789784,1118789791,BB 1118789792,1118790351,US 1118790352,1118790367,GB @@ -17771,7 +18644,7 @@ 1118970112,1118970119,KE 1118970120,1118970239,A2 1118970240,1118970367,NG -1118970368,1118970623,GY +1118970368,1118970623,GB 1118970624,1118970751,SA 1118970752,1118970879,GY 1118970880,1118971520,SR @@ -17779,7 +18652,7 @@ 1118971648,1118971903,GY 1118971904,1118972159,A2 1118972160,1118972415,VE -1118972416,1118972671,EC +1118972416,1118972671,GB 1118972672,1118972927,SA 1118972928,1118972943,NI 1118972944,1118972959,VE @@ -17802,7 +18675,7 @@ 1118973160,1118973167,NI 1118973168,1118973183,A2 1118973184,1118973439,VE -1118973440,1118973695,GY +1118973440,1118973695,HK 1118973696,1118973951,NG 1118973952,1118974015,GY 1118974016,1118974047,A2 @@ -17856,8 +18729,8 @@ 1118977552,1118977575,A2 1118977576,1118977583,SA 1118977584,1118977631,A2 -1118977632,1118977663,NG -1118977664,1118977839,A2 +1118977632,1118977639,US +1118977640,1118977839,A2 1118977840,1118977847,LB 1118977848,1118977879,A2 1118977880,1118977887,NI @@ -17977,7 +18850,7 @@ 1118984896,1118984943,SA 1118984944,1118984951,LB 1118984952,1118984959,SA -1118984960,1118985215,VE +1118984960,1118985215,LB 1118985216,1118985983,A2 1118985984,1118986111,NG 1118986112,1118986239,A2 @@ -18060,7 +18933,9 @@ 1119171208,1119171215,CA 1119171216,1119171383,US 1119171384,1119171391,CA -1119171392,1119172895,US +1119171392,1119172199,US +1119172200,1119172207,CA +1119172208,1119172895,US 1119172896,1119172991,MX 1119172992,1119173007,US 1119173008,1119173095,MX @@ -18161,9 +19036,7 @@ 1120151696,1120151711,TW 1120151712,1120152159,CA 1120152160,1120152191,US -1120152192,1120152319,CA -1120152320,1120152575,US -1120152576,1120153599,CA +1120152192,1120153599,CA 1120153600,1120272383,US 1120272384,1120280575,CA 1120280576,1120280831,US @@ -19000,8 +19873,8 @@ 1121717432,1121717439,JP 1121717440,1121717759,US 1121717760,1121718015,IT -1121718016,1121721695,US -1121721696,1121721727,CN +1121718016,1121721703,US +1121721704,1121721727,CN 1121721728,1121878015,US 1121878016,1121910783,CA 1121910784,1122074623,US @@ -19010,11 +19883,19 @@ 1122088192,1122091007,CA 1122091008,1122092623,US 1122092624,1122092631,CA -1122092632,1122092799,US +1122092632,1122092687,US +1122092688,1122092695,GB +1122092696,1122092799,US 1122092800,1122093055,CA 1122093056,1122093807,US 1122093808,1122093815,HK -1122093816,1122103471,US +1122093816,1122101071,US +1122101072,1122101079,CA +1122101080,1122101111,US +1122101112,1122101119,CA +1122101120,1122101431,US +1122101432,1122101439,SG +1122101440,1122103471,US 1122103472,1122103479,BE 1122103480,1122125996,US 1122125997,1122126004,GB @@ -19071,9 +19952,7 @@ 1122493440,1122494207,PR 1122494208,1122495231,US 1122495232,1122495999,PR -1122496000,1122497407,US -1122497408,1122497471,SV -1122497472,1122497535,US +1122496000,1122497535,US 1122497536,1122498047,PR 1122498048,1122498559,US 1122498560,1122498815,PR @@ -19458,7 +20337,9 @@ 1125541392,1125541407,US 1125541408,1125541487,CA 1125541488,1125541503,US -1125541504,1125543543,CA +1125541504,1125541543,CA +1125541544,1125541551,CN +1125541552,1125543543,CA 1125543544,1125543551,US 1125543552,1125543935,CA 1125543936,1125545983,US @@ -21054,7 +21935,9 @@ 1138662274,1138662309,EG 1138662310,1138662333,US 1138662334,1138662353,GB -1138662354,1138662409,US +1138662354,1138662369,US +1138662370,1138662393,GB +1138662394,1138662409,US 1138662410,1138662473,EG 1138662474,1138662505,GB 1138662506,1138662537,US @@ -21112,7 +21995,9 @@ 1139167744,1139169023,PR 1139169024,1139169279,US 1139169280,1139170047,PR -1139170048,1139179519,US +1139170048,1139171327,US +1139171328,1139175423,GT +1139175424,1139179519,US 1139179520,1139188223,CA 1139188224,1139188479,GB 1139188480,1139188735,CA @@ -21183,11 +22068,7 @@ 1145333864,1145333871,EG 1145333872,1145333879,US 1145333880,1145333903,CN -1145333904,1145333911,US -1145333912,1145333919,CN -1145333920,1145334055,US -1145334056,1145334063,CN -1145334064,1145334143,US +1145333904,1145334143,US 1145334144,1145334167,CN 1145334168,1145334175,US 1145334176,1145334223,CN @@ -21196,8 +22077,8 @@ 1145334240,1145335103,US 1145335104,1145335135,NL 1145335136,1145335167,US -1145335168,1145335191,CN -1145335192,1145335215,US +1145335168,1145335183,CN +1145335184,1145335215,US 1145335216,1145335223,CN 1145335224,1145376767,US 1145376768,1145380863,CA @@ -21670,7 +22551,9 @@ 1158946640,1158946647,IL 1158946648,1158946711,US 1158946712,1158946719,HK -1158946720,1158947087,US +1158946720,1158946771,US +1158946772,1158946772,MX +1158946773,1158947087,US 1158947088,1158947095,ZA 1158947096,1158947391,US 1158947392,1158947399,CA @@ -21705,7 +22588,9 @@ 1158949408,1158949415,ZA 1158949416,1158949775,US 1158949776,1158949783,GB -1158949784,1158950319,US +1158949784,1158950105,US +1158950106,1158950107,CA +1158950108,1158950319,US 1158950320,1158950327,CA 1158950328,1158950335,US 1158950336,1158950351,CA @@ -21738,10 +22623,14 @@ 1158952816,1158952823,CA 1158952824,1158952895,US 1158952896,1158952903,IL -1158952904,1158954815,US +1158952904,1158953503,US +1158953504,1158953511,NL +1158953512,1158954815,US 1158954816,1158954823,GB 1158954824,1158954831,CA -1158954832,1158955039,US +1158954832,1158955015,US +1158955016,1158955023,CA +1158955024,1158955039,US 1158955040,1158955047,GB 1158955048,1158956063,US 1158956064,1158956079,NL @@ -21791,9 +22680,7 @@ 1158961256,1158961263,ES 1158961264,1158961775,US 1158961776,1158961783,CA -1158961784,1158961823,US -1158961824,1158961855,BD -1158961856,1158962175,US +1158961784,1158962175,US 1158962176,1158962183,GB 1158962184,1158962287,US 1158962288,1158962295,ZA @@ -21815,7 +22702,9 @@ 1158963728,1158963735,GB 1158963736,1158964103,US 1158964104,1158964111,IN -1158964112,1158964255,US +1158964112,1158964189,US +1158964190,1158964191,CA +1158964192,1158964255,US 1158964256,1158964263,IN 1158964264,1158964287,US 1158964288,1158964319,CA @@ -22504,15 +23393,20 @@ 1161629240,1161629247,PL 1161629248,1161629263,US 1161629264,1161629271,PL -1161629272,1161629335,US -1161629336,1161629343,NL +1161629272,1161629343,US 1161629344,1161629375,FI 1161629376,1161629383,CA -1161629384,1161629519,US +1161629384,1161629415,US +1161629416,1161629423,HR +1161629424,1161629439,MY +1161629440,1161629519,US 1161629520,1161629527,GB 1161629528,1161629535,MY 1161629536,1161629567,GB -1161629568,1161629695,US +1161629568,1161629583,US +1161629584,1161629599,GB +1161629600,1161629615,HR +1161629616,1161629695,US 1161629696,1161629951,PL 1161629952,1161630335,US 1161630336,1161630343,PL @@ -22827,8 +23721,8 @@ 1161795980,1161795983,CA 1161795984,1161818111,US 1161818112,1161822207,CA -1161822208,1161832799,US -1161832800,1161832831,BR +1161822208,1161832767,US +1161832768,1161832831,BR 1161832832,1161833759,US 1161833760,1161833791,FR 1161833792,1161833855,US @@ -22885,7 +23779,9 @@ 1161891528,1161891535,CA 1161891536,1161891543,US 1161891544,1161891551,CA -1161891552,1161901647,US +1161891552,1161900863,US +1161900864,1161900895,A1 +1161900896,1161901647,US 1161901648,1161901655,VN 1161901656,1161925631,US 1161925632,1161925887,EC @@ -22981,7 +23877,8 @@ 1162811648,1162812159,US 1162812160,1162812415,DO 1162812416,1162813439,BS -1162813440,1162821631,US +1162813440,1162817535,AN +1162817536,1162821631,GT 1162821632,1162823679,PR 1162823680,1162825727,GT 1162825728,1162827775,US @@ -22991,7 +23888,12 @@ 1162836224,1162836735,AN 1162836736,1162836991,US 1162836992,1162838015,BS -1162838016,1162870783,US +1162838016,1162840063,PR +1162840064,1162853375,US +1162853376,1162854399,PR +1162854400,1162858495,JM +1162858496,1162862591,HN +1162862592,1162870783,US 1162870784,1162871295,BB 1162871296,1162879999,LC 1162880000,1162882559,DM @@ -23121,7 +24023,9 @@ 1163407360,1163411455,CA 1163411456,1163468799,US 1163468800,1163472895,PR -1163472896,1163526143,US +1163472896,1163477695,US +1163477696,1163477727,JP +1163477728,1163526143,US 1163526144,1163526463,CA 1163526464,1163526911,US 1163526912,1163527007,CA @@ -23315,9 +24219,7 @@ 1163558029,1163560191,US 1163560192,1163560447,CA 1163560448,1163560959,US -1163560960,1163561727,CA -1163561728,1163561983,FI -1163561984,1163562063,CA +1163560960,1163562063,CA 1163562064,1163562079,US 1163562080,1163562119,CA 1163562120,1163562127,SG @@ -23613,11 +24515,14 @@ 1170461056,1170461439,CO 1170461440,1170461695,US 1170461696,1170462719,IL -1170462720,1170466767,US +1170462720,1170465791,US +1170465792,1170466047,CY +1170466048,1170466767,US 1170466768,1170466783,MQ 1170466784,1170466815,US 1170466816,1170467839,IL -1170467840,1170472959,US +1170467840,1170468351,DO +1170468352,1170472959,US 1170472960,1170481151,CA 1170481152,1170489343,US 1170489344,1170497535,CA @@ -23627,44 +24532,114 @@ 1170538496,1170538751,KH 1170538752,1170539329,US 1170539330,1170539330,CN -1170539331,1170539331,US +1170539331,1170539331,TR 1170539332,1170539332,CA -1170539333,1170539590,US +1170539333,1170539460,US +1170539461,1170539461,PK +1170539462,1170539522,US +1170539523,1170539523,AU +1170539524,1170539589,US +1170539590,1170539590,SA 1170539591,1170539591,CN -1170539592,1170539652,US +1170539592,1170539592,AU +1170539593,1170539650,US +1170539651,1170539651,TR +1170539652,1170539652,US 1170539653,1170539653,GB -1170539654,1170539714,US +1170539654,1170539712,US +1170539713,1170539713,CN +1170539714,1170539714,US 1170539715,1170539715,PK -1170539716,1170539969,US +1170539716,1170539777,US +1170539778,1170539778,CN +1170539779,1170539780,US +1170539781,1170539781,KR +1170539782,1170539842,US +1170539843,1170539843,MY +1170539844,1170539910,US +1170539911,1170539911,KH +1170539912,1170539969,US 1170539970,1170539970,KH -1170539971,1170540038,US +1170539971,1170539971,TR +1170539972,1170540036,US +1170540037,1170540037,TR +1170540038,1170540038,US 1170540039,1170540039,PH -1170540040,1170540097,US +1170540040,1170540096,US +1170540097,1170540097,CN 1170540098,1170540098,TR -1170540099,1170540231,US +1170540099,1170540105,US +1170540106,1170540106,CN +1170540107,1170540163,US +1170540164,1170540164,RO +1170540165,1170540228,US +1170540229,1170540229,TR +1170540230,1170540230,US +1170540231,1170540231,KR 1170540232,1170540232,CN -1170540233,1170540289,US -1170540290,1170540290,CN -1170540291,1170544199,US +1170540233,1170540288,US +1170540289,1170540290,CN +1170540291,1170540294,US +1170540295,1170540295,CN +1170540296,1170540296,TR +1170540297,1170540360,US +1170540361,1170540361,CA +1170540362,1170540369,US +1170540370,1170540370,CN +1170540371,1170540371,US +1170540372,1170540372,CN +1170540373,1170540416,US +1170540417,1170540417,CN +1170540418,1170540480,US +1170540481,1170540481,TR +1170540482,1170540492,US +1170540493,1170540493,TR +1170540494,1170544127,US +1170544128,1170544191,CN +1170544192,1170544199,US 1170544200,1170544207,TR 1170544208,1170544255,US 1170544256,1170544383,IN 1170544384,1170544387,US 1170544388,1170544391,BD -1170544392,1170544431,US +1170544392,1170544395,TR +1170544396,1170544431,US 1170544432,1170544439,CN 1170544440,1170544443,AU 1170544444,1170544511,US -1170544512,1170544575,SE -1170544576,1170553343,US -1170553344,1170553407,CN +1170544512,1170544543,TR +1170544544,1170544831,US +1170544832,1170544959,CN +1170544960,1170545407,US +1170545408,1170545439,TR +1170545440,1170552959,US +1170552960,1170553023,CN +1170553024,1170553279,US +1170553280,1170553407,CN 1170553408,1170553431,US 1170553432,1170553435,TR 1170553436,1170553471,US 1170553472,1170553535,PK -1170553536,1170553855,US +1170553536,1170553599,US +1170553600,1170553603,TR +1170553604,1170553615,US +1170553616,1170553619,SA +1170553620,1170553623,TR +1170553624,1170553627,US +1170553628,1170553631,TR +1170553632,1170553635,US +1170553636,1170553643,SA +1170553644,1170553647,TR +1170553648,1170553651,US +1170553652,1170553655,TR +1170553656,1170553663,US +1170553664,1170553667,SA +1170553668,1170553707,US +1170553708,1170553711,TR +1170553712,1170553855,US 1170553856,1170553983,MY -1170553984,1170554367,US +1170553984,1170554047,CN +1170554048,1170554367,US 1170554368,1170554431,CN 1170554432,1170554463,US 1170554464,1170554479,CN @@ -23721,8 +24696,8 @@ 1176620976,1176620991,CA 1176620992,1176620999,US 1176621000,1176621567,CA -1176621568,1176623103,US -1176623104,1176623359,CA +1176621568,1176622591,US +1176622592,1176623359,CA 1176623360,1176623487,US 1176623488,1176623567,CA 1176623568,1176623583,US @@ -23740,8 +24715,7 @@ 1176629464,1176629503,US 1176629504,1176629519,CA 1176629520,1176629535,NZ -1176629536,1176629567,US -1176629568,1176629631,CA +1176629536,1176629631,CA 1176629632,1176629727,US 1176629728,1176631295,CA 1176631296,1176631551,NL @@ -24032,7 +25006,9 @@ 1177164928,1177164943,CA 1177164944,1177165311,US 1177165312,1177165823,CA -1177165824,1177175199,US +1177165824,1177168223,US +1177168224,1177168231,IE +1177168232,1177175199,US 1177175200,1177175231,CZ 1177175232,1177182527,US 1177182528,1177182591,CA @@ -24448,37 +25424,75 @@ 1208337920,1208338175,A2 1208338176,1208516607,US 1208516608,1208516623,MX -1208516624,1208516671,US -1208516672,1208516735,TR +1208516624,1208516639,US +1208516640,1208516735,TR 1208516736,1208516799,ID -1208516800,1208517119,US +1208516800,1208517010,US +1208517011,1208517011,TR +1208517012,1208517012,GB +1208517013,1208517119,US 1208517120,1208517375,KH -1208517376,1208517631,US +1208517376,1208517503,US +1208517504,1208517567,CN +1208517568,1208517631,US 1208517632,1208517887,KH -1208517888,1208518243,US +1208517888,1208518015,US +1208518016,1208518079,CN +1208518080,1208518143,US +1208518144,1208518207,CN +1208518208,1208518243,US 1208518244,1208518244,CN 1208518245,1208518246,US 1208518247,1208518247,KH -1208518248,1208518263,US +1208518248,1208518261,US +1208518262,1208518262,TR +1208518263,1208518263,CN 1208518264,1208518264,GB -1208518265,1208518335,US +1208518265,1208518299,US +1208518300,1208518303,BD +1208518304,1208518335,US 1208518336,1208518399,TR 1208518400,1208519167,US 1208519168,1208519423,CN -1208519424,1208521983,US +1208519424,1208519551,TR +1208519552,1208521983,US 1208521984,1208522239,CN -1208522240,1208523991,US +1208522240,1208522271,US +1208522272,1208522303,TR +1208522304,1208522367,US +1208522368,1208522431,CN +1208522432,1208523991,US 1208523992,1208523995,TR -1208523996,1208524063,US +1208523996,1208523999,US +1208524000,1208524031,TR +1208524032,1208524047,US +1208524048,1208524055,TR +1208524056,1208524063,US 1208524064,1208524079,CN -1208524080,1208524159,US +1208524080,1208524083,US +1208524084,1208524087,KR +1208524088,1208524159,US 1208524160,1208524223,ID 1208524224,1208524287,US 1208524288,1208524351,PK 1208524352,1208524415,CN -1208524416,1208527871,US +1208524416,1208524863,US +1208524864,1208524927,PK +1208524928,1208525823,US +1208525824,1208526079,KH +1208526080,1208526335,US +1208526336,1208526591,CN +1208526592,1208527871,US 1208527872,1208528127,VN -1208528128,1208586364,US +1208528128,1208531007,US +1208531008,1208531071,CN +1208531072,1208531455,US +1208531456,1208531583,CN +1208531584,1208531711,US +1208531712,1208531967,PK +1208531968,1208532287,US +1208532288,1208532351,PK +1208532352,1208586364,US 1208586365,1208586372,MA 1208586373,1208586419,US 1208586420,1208586427,AE @@ -24773,32 +25787,81 @@ 1209190912,1209191167,MX 1209191168,1209270579,US 1209270580,1209270583,ID -1209270584,1209271067,US +1209270584,1209270607,US +1209270608,1209270611,KR +1209270612,1209270615,CN +1209270616,1209270783,US +1209270784,1209270787,TR +1209270788,1209271067,US 1209271068,1209271071,VN -1209271072,1209271371,US +1209271072,1209271083,US +1209271084,1209271087,BR +1209271088,1209271127,US +1209271128,1209271131,SA +1209271132,1209271187,US +1209271188,1209271191,TR +1209271192,1209271367,US +1209271368,1209271371,TR 1209271372,1209271375,PE 1209271376,1209271435,US 1209271436,1209271439,CN -1209271440,1209271463,US -1209271464,1209271467,IN -1209271468,1209271495,US +1209271440,1209271451,US +1209271452,1209271455,CN +1209271456,1209271463,US +1209271464,1209271467,PK +1209271468,1209271487,US +1209271488,1209271495,CN 1209271496,1209271499,CA 1209271500,1209271503,MX -1209271504,1209271579,US +1209271504,1209271543,US +1209271544,1209271551,CN +1209271552,1209271571,US +1209271572,1209271575,TR +1209271576,1209271579,US 1209271580,1209271583,BD 1209271584,1209271607,US 1209271608,1209271611,KH -1209271612,1209271639,US +1209271612,1209271615,CA +1209271616,1209271639,US 1209271640,1209271643,CZ -1209271644,1209271691,US +1209271644,1209271659,US +1209271660,1209271663,PK +1209271664,1209271691,US 1209271692,1209271695,CA -1209271696,1209271967,US +1209271696,1209271703,US +1209271704,1209271707,CN +1209271708,1209271719,US +1209271720,1209271723,CN +1209271724,1209271727,US +1209271728,1209271731,PK +1209271732,1209271735,CN +1209271736,1209271847,US +1209271848,1209271855,IN +1209271856,1209271867,US +1209271868,1209271871,BR +1209271872,1209271951,US +1209271952,1209271955,KH +1209271956,1209271963,US +1209271964,1209271967,TR 1209271968,1209271971,VN 1209271972,1209271975,TR 1209271976,1209271979,CA -1209271980,1209272127,US +1209271980,1209271983,ZA +1209271984,1209271999,US +1209272000,1209272007,TR +1209272008,1209272043,US +1209272044,1209272047,TR +1209272048,1209272079,US +1209272080,1209272095,TR +1209272096,1209272127,US 1209272128,1209272159,CN -1209272160,1209272383,US +1209272160,1209272175,US +1209272176,1209272191,BR +1209272192,1209272223,TR +1209272224,1209272239,US +1209272240,1209272255,TR +1209272256,1209272287,CN +1209272288,1209272383,US 1209272384,1209272399,CA 1209272400,1209272407,US 1209272408,1209272415,PE @@ -24808,52 +25871,123 @@ 1209272512,1209272575,CN 1209272576,1209272583,US 1209272584,1209272591,PH -1209272592,1209272655,US +1209272592,1209272607,US +1209272608,1209272623,TR +1209272624,1209272655,US 1209272656,1209272671,TR -1209272672,1209273555,US +1209272672,1209272719,US +1209272720,1209272767,CN +1209272768,1209272783,TR +1209272784,1209272847,US +1209272848,1209272863,TR +1209272864,1209272895,US +1209272896,1209272959,CN +1209272960,1209273023,BR +1209273024,1209273055,US +1209273056,1209273071,TR +1209273072,1209273119,US +1209273120,1209273135,BR +1209273136,1209273151,TR +1209273152,1209273207,US +1209273208,1209273215,CA +1209273216,1209273279,US +1209273280,1209273311,BR +1209273312,1209273359,US +1209273360,1209273375,TR +1209273376,1209273555,US 1209273556,1209273559,CA 1209273560,1209273647,US 1209273648,1209273655,CN -1209273656,1209273663,US +1209273656,1209273663,AU 1209273664,1209273671,BD -1209273672,1209274031,US +1209273672,1209273999,US +1209274000,1209274015,TR +1209274016,1209274031,US 1209274032,1209274039,TR -1209274040,1209274115,US +1209274040,1209274047,US +1209274048,1209274111,CN +1209274112,1209274115,US 1209274116,1209274119,CZ 1209274120,1209274135,US 1209274136,1209274151,TR -1209274152,1209274815,US +1209274152,1209274367,US +1209274368,1209274495,TR +1209274496,1209274527,CN +1209274528,1209274543,US +1209274544,1209274559,TR +1209274560,1209274815,US 1209274816,1209274879,CA 1209274880,1209274911,CZ 1209274912,1209274927,US 1209274928,1209274943,TR 1209274944,1209274959,US 1209274960,1209274967,PH -1209274968,1209275906,US +1209274968,1209275679,US +1209275680,1209275695,TR +1209275696,1209275791,US +1209275792,1209275799,TR +1209275800,1209275823,US +1209275824,1209275827,CN +1209275828,1209275906,US 1209275907,1209275908,KH -1209275909,1209275928,US +1209275909,1209275913,US +1209275914,1209275914,KH +1209275915,1209275917,US +1209275918,1209275918,KH +1209275919,1209275924,US +1209275925,1209275925,TR +1209275926,1209275928,US 1209275929,1209275929,MY 1209275930,1209275930,US 1209275931,1209275931,AU 1209275932,1209275932,CN -1209275933,1209276351,US +1209275933,1209275999,US +1209276000,1209276031,MY +1209276032,1209276063,BR +1209276064,1209276351,US 1209276352,1209276415,CN -1209276416,1209276799,US -1209276800,1209276807,BD +1209276416,1209276671,US +1209276672,1209276703,CN +1209276704,1209276799,US +1209276800,1209276803,SA +1209276804,1209276807,BD 1209276808,1209276903,US 1209276904,1209276911,CN 1209276912,1209276927,TR 1209276928,1209277103,US 1209277104,1209277119,TR -1209277120,1209277155,US +1209277120,1209277143,US +1209277144,1209277147,TR +1209277148,1209277154,US +1209277155,1209277155,TR 1209277156,1209277156,SE -1209277157,1209277159,US +1209277157,1209277157,US +1209277158,1209277158,IN +1209277159,1209277159,US 1209277160,1209277160,CN -1209277161,1209277575,US +1209277161,1209277166,US +1209277167,1209277167,CN +1209277168,1209277168,AU +1209277169,1209277215,US +1209277216,1209277231,BR +1209277232,1209277247,US +1209277248,1209277263,BR +1209277264,1209277279,US +1209277280,1209277283,CA +1209277284,1209277575,US 1209277576,1209277583,BD -1209277584,1209278207,US +1209277584,1209278083,US +1209278084,1209278087,CN +1209278088,1209278095,AU +1209278096,1209278207,US 1209278208,1209278463,IE -1209278464,1209278911,US +1209278464,1209278523,US +1209278524,1209278527,CN +1209278528,1209278591,US +1209278592,1209278719,CN +1209278720,1209278827,US +1209278828,1209278831,CN +1209278832,1209278911,US 1209278912,1209278919,TR 1209278920,1209278975,US 1209278976,1209279103,CA @@ -24861,44 +25995,103 @@ 1209279120,1209279135,CA 1209279136,1209279167,US 1209279168,1209279231,VN -1209279232,1209279871,US +1209279232,1209279295,BR +1209279296,1209279855,US +1209279856,1209279871,TR 1209279872,1209279935,MY -1209279936,1209280191,US +1209279936,1209279951,GB +1209279952,1209279967,US +1209279968,1209279999,CN +1209280000,1209280015,TR +1209280016,1209280027,US +1209280028,1209280031,SA +1209280032,1209280047,IN +1209280048,1209280191,US 1209280192,1209280255,VN 1209280256,1209280495,US 1209280496,1209280511,CA -1209280512,1209280623,US +1209280512,1209280515,US +1209280516,1209280519,TR +1209280520,1209280527,AU +1209280528,1209280623,US 1209280624,1209280639,CN -1209280640,1209281247,US +1209280640,1209280895,US +1209280896,1209280959,BR +1209280960,1209281007,US +1209281008,1209281023,TR +1209281024,1209281151,US +1209281152,1209281231,CN +1209281232,1209281247,US 1209281248,1209281279,CA 1209281280,1209281535,KH -1209281536,1209281983,US +1209281536,1209281791,IN +1209281792,1209281871,US +1209281872,1209281887,TR +1209281888,1209281983,US 1209281984,1209282047,TR -1209282048,1209283751,US +1209282048,1209282063,US +1209282064,1209282067,SA +1209282068,1209282559,US +1209282560,1209282623,CN +1209282624,1209283487,US +1209283488,1209283503,TR +1209283504,1209283543,US +1209283544,1209283547,SA +1209283548,1209283583,US +1209283584,1209283587,SA +1209283588,1209283751,US 1209283752,1209283759,BD -1209283760,1209283807,US +1209283760,1209283775,US +1209283776,1209283807,ZA 1209283808,1209283839,CA 1209283840,1209283871,US 1209283872,1209283875,TR -1209283876,1209284223,US +1209283876,1209283887,US +1209283888,1209283903,BR +1209283904,1209283915,US +1209283916,1209283919,KR +1209283920,1209284159,US +1209284160,1209284223,TR 1209284224,1209284351,IE -1209284352,1209284799,US -1209284800,1209284815,TR -1209284816,1209284855,US +1209284352,1209284399,US +1209284400,1209284415,TR +1209284416,1209284431,US +1209284432,1209284447,TR +1209284448,1209284607,US +1209284608,1209284671,TR +1209284672,1209284735,CN +1209284736,1209284767,US +1209284768,1209284799,MY +1209284800,1209284831,TR +1209284832,1209284839,US +1209284840,1209284843,TR +1209284844,1209284855,US 1209284856,1209284863,TR -1209284864,1209284999,US +1209284864,1209284993,US +1209284994,1209284994,TR +1209284995,1209284995,US +1209284996,1209284996,TR +1209284997,1209284997,US +1209284998,1209284998,CN +1209284999,1209284999,US 1209285000,1209285000,CA -1209285001,1209285003,US +1209285001,1209285002,US +1209285003,1209285003,CN 1209285004,1209285004,CA -1209285005,1209285017,US +1209285005,1209285009,US +1209285010,1209285010,CA +1209285011,1209285017,US 1209285018,1209285018,CA 1209285019,1209285021,US 1209285022,1209285022,CA 1209285023,1209285063,US 1209285064,1209285071,TR -1209285072,1209285183,US +1209285072,1209285151,US +1209285152,1209285183,RO 1209285184,1209285247,TR -1209285248,1209347839,US +1209285248,1209286143,US +1209286144,1209286399,KH +1209286400,1209347839,US 1209347840,1209348095,GB 1209348096,1209357215,US 1209357216,1209357231,SI @@ -25692,7 +26885,9 @@ 1211038064,1211038079,TT 1211038080,1211038719,US 1211038720,1211038975,CY -1211038976,1211039087,US +1211038976,1211039007,US +1211039008,1211039023,VG +1211039024,1211039087,US 1211039088,1211039103,RU 1211039104,1211039503,US 1211039504,1211039519,A2 @@ -25706,7 +26901,8 @@ 1211303936,1211304159,CA 1211304160,1211304223,US 1211304224,1211304255,CA -1211304256,1211304319,US +1211304256,1211304287,US +1211304288,1211304319,CA 1211304320,1211304351,NL 1211304352,1211304703,CA 1211304704,1211304767,GB @@ -26020,21 +27216,33 @@ 1249010688,1249011711,CA 1249011712,1249019903,US 1249019904,1249020927,CA -1249020928,1249027135,US +1249020928,1249026703,US +1249026704,1249026711,ZA +1249026712,1249026719,CA +1249026720,1249026767,US +1249026768,1249026775,CA +1249026776,1249027127,US +1249027128,1249027135,ZA 1249027136,1249027143,AU 1249027144,1249027175,US 1249027176,1249027183,IN -1249027184,1249027391,US +1249027184,1249027271,US +1249027272,1249027279,GB +1249027280,1249027391,US 1249027392,1249027407,PH 1249027408,1249027503,US 1249027504,1249027519,CA 1249027520,1249027551,US 1249027552,1249027559,CA 1249027560,1249027647,US -1249027648,1249027655,AU -1249027656,1249027823,US +1249027648,1249027655,GB +1249027656,1249027719,US +1249027720,1249027727,GR +1249027728,1249027823,US 1249027824,1249027839,CA -1249027840,1249027919,US +1249027840,1249027895,US +1249027896,1249027903,PL +1249027904,1249027919,US 1249027920,1249027935,AU 1249027936,1249029119,US 1249029120,1249030143,CA @@ -26245,9 +27453,7 @@ 1254704384,1254704639,PH 1254704640,1254704903,US 1254704904,1254704911,PH -1254704912,1254708927,US -1254708928,1254708959,SG -1254708960,1254713359,US +1254704912,1254713359,US 1254713360,1254713407,CA 1254713408,1254725119,US 1254725120,1254725151,GB @@ -26791,9 +27997,7 @@ 1255042752,1255042783,CA 1255042784,1255047167,US 1255047168,1255055359,CA -1255055360,1255055487,US -1255055488,1255055503,CA -1255055504,1255056063,US +1255055360,1255056063,US 1255056064,1255056095,NL 1255056096,1255057887,US 1255057888,1255057919,NL @@ -27447,7 +28651,9 @@ 1279979008,1279979263,IN 1279979264,1279979327,US 1279979328,1279979391,IN -1279979392,1279979559,US +1279979392,1279979424,US +1279979425,1279979425,IN +1279979426,1279979559,US 1279979560,1279979575,CA 1279979576,1279979583,US 1279979584,1279980063,CA @@ -27856,12 +29062,12 @@ 1296466384,1296466399,NO 1296466400,1296466415,AO 1296466416,1296466431,NG -1296466432,1296466439,TZ -1296466440,1296466495,NO +1296466432,1296466447,TZ +1296466448,1296466495,NO 1296466496,1296466543,GN 1296466544,1296466559,NO -1296466560,1296466583,NG -1296466584,1296466623,NO +1296466560,1296466575,NG +1296466576,1296466623,NO 1296466624,1296466631,MR 1296466632,1296466639,NO 1296466640,1296466655,NG @@ -27873,9 +29079,7 @@ 1296466760,1296466767,SL 1296466768,1296466775,BF 1296466776,1296466783,GN -1296466784,1296467711,NO -1296467712,1296467967,NG -1296467968,1296468479,NO +1296466784,1296468479,NO 1296468480,1296469247,NG 1296469248,1296472351,NO 1296472352,1296472383,NG @@ -27923,6 +29127,7 @@ 1296613376,1296615423,IT 1296615424,1296617471,GB 1296617472,1296619519,ES +1296619520,1296621567,DE 1296621568,1296623615,SE 1296623616,1296625663,BE 1296625664,1296629759,RU @@ -28037,6 +29242,7 @@ 1296756736,1296758783,FR 1296758784,1296760831,RS 1296760832,1296762879,GB +1296762880,1296764927,RU 1296764928,1296769023,AT 1296769024,1296771071,TR 1296771072,1296773119,LT @@ -28278,7 +29484,9 @@ 1299016960,1299017215,NL 1299017216,1299017727,RU 1299017728,1299021823,BE -1299021824,1299024559,CH +1299021824,1299023879,CH +1299023880,1299023887,DE +1299023888,1299024559,CH 1299024560,1299024575,DE 1299024576,1299026111,CH 1299026112,1299026127,ES @@ -28311,7 +29519,9 @@ 1305477120,1305739263,ES 1305739264,1306001407,DK 1306001408,1306132479,RU -1306132480,1306263551,SE +1306132480,1306198015,SE +1306198016,1306206207,LV +1306206208,1306263551,SE 1306263552,1306271743,KE 1306271744,1306279935,RU 1306279936,1306286079,IT @@ -28471,6 +29681,7 @@ 1307701248,1307709439,RU 1307709440,1307713535,AL 1307713536,1307717631,IT +1307717632,1307721727,GE 1307721728,1307725823,AT 1307725824,1307729919,CZ 1307729920,1307734015,RU @@ -28492,8 +29703,8 @@ 1307756912,1307757063,GB 1307757064,1307757071,US 1307757072,1307757183,GB -1307757184,1307757247,US -1307757248,1307758591,GB +1307757184,1307757279,US +1307757280,1307758591,GB 1307758592,1307762687,SM 1307762688,1307766783,PL 1307766784,1307770879,GB @@ -28646,7 +29857,9 @@ 1310214144,1310216191,UA 1310216192,1310218239,IT 1310218240,1310220287,NO -1310220288,1310222335,NL +1310220288,1310221311,NL +1310221312,1310221567,MK +1310221568,1310222335,NL 1310222336,1310224383,RS 1310224384,1310226431,GB 1310226432,1310228479,BE @@ -28968,7 +30181,7 @@ 1311367888,1311367895,DE 1311367896,1311367903,NG 1311367904,1311367927,A2 -1311367928,1311367935,IR +1311367928,1311367935,DE 1311367936,1311367967,BJ 1311367968,1311367999,ET 1311368000,1311368015,A2 @@ -29188,22 +30401,7 @@ 1315885056,1315889151,CZ 1315889152,1315893247,DE 1315893248,1315897343,RU -1315897344,1315897855,IR -1315897856,1315897863,AE -1315897864,1315897951,IR -1315897952,1315897991,AE -1315897992,1315897999,IR -1315898000,1315898015,AE -1315898016,1315898031,IR -1315898032,1315898111,AE -1315898112,1315900671,IR -1315900672,1315900735,AE -1315900736,1315900799,IR -1315900800,1315900927,AE -1315900928,1315901223,IR -1315901224,1315901247,AE -1315901248,1315901279,IR -1315901280,1315901439,AE +1315897344,1315901439,IR 1315901440,1315905535,UA 1315905536,1315907583,BA 1315907584,1315908095,SI @@ -29363,8 +30561,8 @@ 1317667776,1317667783,US 1317667784,1317667799,NG 1317667800,1317667807,A2 -1317667808,1317667823,NG -1317667824,1317668095,A2 +1317667808,1317667815,NG +1317667816,1317668095,A2 1317668096,1317668103,GH 1317668104,1317668111,ZM 1317668112,1317668143,A2 @@ -29494,8 +30692,8 @@ 1317670648,1317670655,NG 1317670656,1317670663,AO 1317670664,1317670679,A2 -1317670680,1317670703,NG -1317670704,1317670711,A2 +1317670680,1317670695,NG +1317670696,1317670711,A2 1317670712,1317670719,GH 1317670720,1317670727,A2 1317670728,1317670735,GH @@ -29726,14 +30924,10 @@ 1317675056,1317675063,NG 1317675064,1317675071,AO 1317675072,1317675095,NG -1317675096,1317675103,A2 -1317675104,1317675111,CD -1317675112,1317675183,A2 +1317675096,1317675183,A2 1317675184,1317675199,CM 1317675200,1317675215,NG -1317675216,1317675231,A2 -1317675232,1317675239,LR -1317675240,1317675247,A2 +1317675216,1317675247,A2 1317675248,1317675255,AO 1317675256,1317675271,A2 1317675272,1317675279,NG @@ -29795,8 +30989,7 @@ 1317675928,1317675943,A2 1317675944,1317675951,GB 1317675952,1317675975,NG -1317675976,1317675983,A2 -1317675984,1317675991,LR +1317675976,1317675991,A2 1317675992,1317675999,NG 1317676000,1317676007,GN 1317676008,1317676015,GH @@ -29804,7 +30997,8 @@ 1317676024,1317676031,GQ 1317676032,1317676039,NG 1317676040,1317676047,CD -1317676048,1317676063,A2 +1317676048,1317676055,A2 +1317676056,1317676063,NG 1317676064,1317676071,CM 1317676072,1317676079,A2 1317676080,1317676087,AO @@ -29821,15 +31015,15 @@ 1317676168,1317676175,ML 1317676176,1317676191,A2 1317676192,1317676207,SD -1317676208,1317676223,A2 -1317676224,1317676231,AO +1317676208,1317676231,A2 1317676232,1317676239,NG 1317676240,1317676247,SD 1317676248,1317676255,NG 1317676256,1317676263,CD 1317676264,1317676271,NG 1317676272,1317676287,A2 -1317676288,1317676551,LR +1317676288,1317676543,LR +1317676544,1317676551,A2 1317676552,1317676559,AO 1317676560,1317676567,NG 1317676568,1317676575,LR @@ -29846,8 +31040,7 @@ 1317676688,1317676695,LR 1317676696,1317676711,A2 1317676712,1317676719,NG -1317676720,1317676727,GQ -1317676728,1317676751,A2 +1317676720,1317676751,A2 1317676752,1317676759,ML 1317676760,1317676767,LR 1317676768,1317676775,A2 @@ -29879,7 +31072,9 @@ 1317677080,1317677087,SD 1317677088,1317677095,A2 1317677096,1317677103,BF -1317677104,1317677143,A2 +1317677104,1317677127,A2 +1317677128,1317677135,NG +1317677136,1317677143,A2 1317677144,1317677151,SD 1317677152,1317677191,A2 1317677192,1317677199,SD @@ -29923,8 +31118,7 @@ 1317677840,1317677847,LR 1317677848,1317677855,NG 1317677856,1317677863,BF -1317677864,1317677871,CM -1317677872,1317677895,A2 +1317677864,1317677895,A2 1317677896,1317677903,GN 1317677904,1317678007,A2 1317678008,1317678015,NG @@ -29934,7 +31128,9 @@ 1317678080,1317678095,NG 1317678096,1317678103,GQ 1317678104,1317678111,BE -1317678112,1317678143,A2 +1317678112,1317678127,A2 +1317678128,1317678135,SD +1317678136,1317678143,NG 1317678144,1317678151,CD 1317678152,1317678159,NG 1317678160,1317678167,GQ @@ -29943,8 +31139,7 @@ 1317678184,1317678191,NG 1317678192,1317678199,A2 1317678200,1317678207,CD -1317678208,1317678215,NG -1317678216,1317678231,A2 +1317678208,1317678231,A2 1317678232,1317678239,BF 1317678240,1317678271,A2 1317678272,1317678287,NG @@ -29966,15 +31161,16 @@ 1317678456,1317678463,CD 1317678464,1317678471,NG 1317678472,1317678479,AO -1317678480,1317678487,A2 -1317678488,1317678495,NG -1317678496,1317678519,A2 -1317678520,1317678527,NG +1317678480,1317678487,CD +1317678488,1317678495,ML +1317678496,1317678511,A2 +1317678512,1317678527,NG 1317678528,1317678543,A2 1317678544,1317678551,CD 1317678552,1317678559,A2 1317678560,1317678567,NG -1317678568,1317679615,A2 +1317678568,1317678575,CD +1317678576,1317679615,A2 1317679616,1317679631,AO 1317679632,1317679639,A2 1317679640,1317679647,ZW @@ -29987,14 +31183,13 @@ 1317679704,1317679719,NG 1317679720,1317679727,A2 1317679728,1317679735,GN -1317679736,1317679743,A2 +1317679736,1317679743,NG 1317679744,1317679751,AO 1317679752,1317679767,NG 1317679768,1317679775,A2 1317679776,1317679799,NG 1317679800,1317679831,A2 -1317679832,1317679839,CD -1317679840,1317679847,NG +1317679832,1317679847,NG 1317679848,1317679855,A2 1317679856,1317679863,NG 1317679864,1317683199,A2 @@ -30004,16 +31199,13 @@ 1317683864,1317683871,FR 1317683872,1317685503,DE 1317685504,1317685631,BE -1317685632,1317686271,DE -1317686272,1317686287,NL -1317686288,1317686335,DE +1317685632,1317686335,DE 1317686336,1317686399,NL 1317686400,1317695743,DE 1317695744,1317695999,CH 1317696000,1317698559,DE 1317698560,1317698687,NL -1317698688,1317699071,DE -1317699072,1317699583,CN +1317698688,1317699583,DE 1317699584,1317715967,RU 1317715968,1317732351,SA 1317732352,1317748735,HU @@ -30067,7 +31259,7 @@ 1317843136,1317843199,CY 1317843200,1317843391,US 1317843392,1317843423,NL -1317843424,1317843455,PA +1317843424,1317843455,PR 1317843456,1317843519,NL 1317843520,1317843583,GI 1317843584,1317843967,NL @@ -30083,8 +31275,8 @@ 1317965048,1317965055,DE 1317965056,1317978111,AT 1317978112,1317994495,RU -1317994496,1317996031,NL -1317996032,1317996095,DE +1317994496,1317995519,NL +1317995520,1317996095,DE 1317996096,1317996287,NL 1317996288,1317997567,DE 1317997568,1317998591,NL @@ -30100,8 +31292,8 @@ 1318008000,1318008031,NL 1318008032,1318009423,DE 1318009424,1318009471,NL -1318009472,1318010495,DE -1318010496,1318010879,NL +1318009472,1318010511,DE +1318010512,1318010879,NL 1318010880,1318027263,DK 1318027264,1318043647,IE 1318043648,1318584319,GB @@ -30140,10 +31332,10 @@ 1318708624,1318708627,GB 1318708628,1318708639,FR 1318708640,1318708647,GB -1318708648,1318708679,FR -1318708680,1318708687,GB -1318708688,1318708703,FR -1318708704,1318708731,GB +1318708648,1318708683,FR +1318708684,1318708687,GB +1318708688,1318708723,FR +1318708724,1318708731,GB 1318708732,1318708735,FR 1318708736,1318708991,GB 1318708992,1318711647,FR @@ -30209,15 +31401,14 @@ 1319069777,1319069823,CN 1319069824,1319070217,DE 1319070218,1319070254,CN -1319070255,1319070271,GB +1319070255,1319070271,DE 1319070272,1319070335,PT 1319070336,1319070463,RU 1319070464,1319070975,DE 1319070976,1319071103,PL 1319071104,1319071231,RU 1319071232,1319071487,PL -1319071488,1319072767,DE -1319072768,1319073023,IT +1319071488,1319073023,DE 1319073024,1319073279,TR 1319073280,1319074303,DE 1319074304,1319074559,HK @@ -30244,7 +31435,9 @@ 1331757056,1331822591,IE 1331822592,1331824639,IT 1331824640,1331826687,RU -1331826688,1331828735,FR +1331826688,1331828607,FR +1331828608,1331828615,BE +1331828616,1331828735,FR 1331828736,1331830783,LT 1331830784,1331831335,GB 1331831336,1331831343,IE @@ -30318,7 +31511,9 @@ 1331938840,1331938847,IE 1331938848,1331938879,GB 1331938880,1331938895,NG -1331938896,1331939327,GB +1331938896,1331938943,GB +1331938944,1331938958,US +1331938959,1331939327,GB 1331939328,1331941375,BE 1331941376,1331943423,ES 1331943424,1331945471,RU @@ -30506,7 +31701,9 @@ 1334710272,1334714367,RU 1334714368,1334718463,DE 1334718464,1334722559,KZ -1334722560,1334726655,NL +1334722560,1334725631,NL +1334725632,1334726143,SE +1334726144,1334726655,NL 1334726656,1334729983,RU 1334729984,1334730431,KZ 1334730432,1334730439,RU @@ -30521,10 +31718,8 @@ 1334730544,1334730579,KZ 1334730580,1334730583,RU 1334730584,1334730667,KZ -1334730668,1334730687,RU -1334730688,1334730699,KZ -1334730700,1334730703,RU -1334730704,1334730751,KZ +1334730668,1334730679,RU +1334730680,1334730751,KZ 1334730752,1334734847,RU 1334734848,1334738943,LT 1334738944,1334743039,CH @@ -30725,7 +31920,9 @@ 1346371584,1346375679,FI 1346375680,1346379775,RU 1346379776,1346383871,ES -1346383872,1346387967,GB +1346383872,1346386719,GB +1346386720,1346386727,SE +1346386728,1346387967,GB 1346387968,1346392063,HU 1346392064,1346396159,UA 1346396160,1346400255,RU @@ -30763,16 +31960,12 @@ 1346500640,1346500735,IM 1346500736,1346500767,GB 1346500768,1346500863,IM -1346500864,1346501119,GB -1346501120,1346501135,IM -1346501136,1346501231,GB +1346500864,1346501231,GB 1346501232,1346501247,IM 1346501248,1346501343,GB 1346501344,1346501375,IM 1346501376,1346501695,GB -1346501696,1346501727,IM -1346501728,1346501743,GB -1346501744,1346501759,IM +1346501696,1346501759,IM 1346501760,1346501795,GB 1346501796,1346501799,IM 1346501800,1346501807,GB @@ -30862,15 +32055,20 @@ 1346703432,1346703435,US 1346703436,1346703483,GB 1346703484,1346703487,US -1346703488,1346703871,GB +1346703488,1346703503,GB +1346703504,1346703511,US +1346703512,1346703871,GB 1346703872,1346704127,LI 1346704128,1346704383,GB 1346704384,1346704447,US 1346704448,1346704479,GB 1346704480,1346704511,US -1346704512,1346704639,GB +1346704512,1346704575,GB +1346704576,1346704639,US 1346704640,1346704767,LI -1346704768,1346707455,GB +1346704768,1346705247,GB +1346705248,1346705279,US +1346705280,1346707455,GB 1346707456,1346711551,DE 1346711552,1346715647,AZ 1346715648,1346723839,NL @@ -31026,7 +32224,9 @@ 1347229312,1347229343,AT 1347229344,1347229895,DE 1347229896,1347229903,GR -1347229904,1347231071,DE +1347229904,1347230975,DE +1347230976,1347230983,ES +1347230984,1347231071,DE 1347231072,1347231087,AT 1347231088,1347231095,GR 1347231096,1347231743,DE @@ -31179,7 +32379,9 @@ 1347386560,1347386567,RU 1347386568,1347386751,EE 1347386752,1347386815,MY -1347386816,1347387215,EE +1347386816,1347387011,EE +1347387012,1347387015,DE +1347387016,1347387215,EE 1347387216,1347387219,DE 1347387220,1347387224,EE 1347387225,1347387228,DE @@ -31213,11 +32415,7 @@ 1347440640,1347444735,ES 1347444736,1347452927,RU 1347452928,1347461119,BG -1347461120,1347461671,AL -1347461672,1347461679,RS -1347461680,1347462023,AL -1347462024,1347462031,RS -1347462032,1347462579,AL +1347461120,1347462579,AL 1347462580,1347462583,RS 1347462584,1347462591,AL 1347462592,1347462607,RS @@ -31330,7 +32528,7 @@ 1347754936,1347754943,RS 1347754944,1347754951,CY 1347754952,1347754959,TN -1347754960,1347754967,MA +1347754960,1347754967,CY 1347754968,1347754983,GR 1347754984,1347754991,CY 1347754992,1347755112,GR @@ -31392,8 +32590,8 @@ 1347856096,1347856239,DE 1347856240,1347856255,EU 1347856256,1347856383,DE -1347856384,1347856391,EU -1347856392,1347856431,DE +1347856384,1347856399,EU +1347856400,1347856431,DE 1347856432,1347856447,EU 1347856448,1347856639,DE 1347856640,1347856895,AT @@ -31797,7 +32995,11 @@ 1352149428,1352149451,FR 1352149452,1352149455,DE 1352149456,1352149487,FR -1352149488,1352277535,DE +1352149488,1352149743,DE +1352149744,1352149751,AT +1352149752,1352149783,DE +1352149784,1352149791,SE +1352149792,1352277535,DE 1352277536,1352277567,IT 1352277568,1352287399,DE 1352287400,1352287407,NL @@ -31808,7 +33010,9 @@ 1352404608,1352445687,DE 1352445688,1352445703,NL 1352445704,1352663039,DE -1352663040,1353187327,DK +1352663040,1353182591,DK +1353182592,1353182599,DE +1353182600,1353187327,DK 1353187328,1353252991,GB 1353252992,1353253023,FR 1353253024,1353253663,GB @@ -31825,7 +33029,7 @@ 1353257984,1353258247,SE 1353258248,1353258255,GB 1353258256,1353258263,SE -1353258264,1353258271,DK +1353258264,1353258271,NO 1353258272,1353258303,SE 1353258304,1353258367,FI 1353258368,1353258415,SE @@ -31848,7 +33052,9 @@ 1353266976,1353267455,GB 1353267456,1353268223,IE 1353268224,1353268479,GB -1353268480,1353269247,BE +1353268480,1353268575,BE +1353268576,1353268583,GB +1353268584,1353269247,BE 1353269248,1353270527,GB 1353270528,1353270783,IE 1353270784,1353271295,GB @@ -31892,12 +33098,16 @@ 1353280120,1353280127,GB 1353280128,1353280143,IT 1353280144,1353280151,GB -1353280152,1353280663,IT +1353280152,1353280287,IT +1353280288,1353280295,GB +1353280296,1353280663,IT 1353280664,1353280679,GB 1353280680,1353281023,IT 1353281024,1353281535,BE 1353281536,1353282047,GB -1353282048,1353282559,IT +1353282048,1353282159,IT +1353282160,1353282167,GB +1353282168,1353282559,IT 1353282560,1353283071,GB 1353283072,1353283327,IT 1353283328,1353287327,GB @@ -31994,7 +33204,9 @@ 1353310336,1353310463,GB 1353310464,1353310479,ES 1353310480,1353310487,GB -1353310488,1353310599,ES +1353310488,1353310559,ES +1353310560,1353310575,GB +1353310576,1353310599,ES 1353310600,1353310607,DE 1353310608,1353310719,ES 1353310720,1353311175,IT @@ -32075,7 +33287,9 @@ 1354664760,1354664767,IT 1354664768,1354665215,DE 1354665216,1354665247,ES -1354665248,1354665943,DE +1354665248,1354665923,DE +1354665924,1354665927,AT +1354665928,1354665943,DE 1354665944,1354665951,IT 1354665952,1354666639,DE 1354666640,1354666650,FI @@ -32194,7 +33408,7 @@ 1357321008,1357321015,GB 1357321016,1357321023,DK 1357321024,1357321087,KE -1357321088,1357321215,EU +1357321088,1357321215,HK 1357321216,1357321471,GB 1357321472,1357321503,ES 1357321504,1357321727,EU @@ -32281,7 +33495,7 @@ 1357366528,1357366591,BE 1357366592,1357366655,FR 1357366656,1357366719,PT -1357366720,1357366783,EU +1357366720,1357366783,GB 1357366784,1357366799,MD 1357366800,1357366815,ES 1357366816,1357366847,FR @@ -32359,7 +33573,8 @@ 1357876344,1357876383,EU 1357876384,1357876391,RU 1357876392,1357876399,PL -1357876400,1357876415,EU +1357876400,1357876407,DE +1357876408,1357876415,EU 1357876416,1357876479,PL 1357876480,1357876495,SE 1357876496,1357876543,EU @@ -32368,7 +33583,15 @@ 1357876928,1357877335,DE 1357877336,1357877343,EU 1357877344,1357877759,DE -1357877760,1357878783,EU +1357877760,1357878271,EU +1357878272,1357878335,CH +1357878336,1357878399,CZ +1357878400,1357878463,NL +1357878464,1357878527,NO +1357878528,1357878591,AT +1357878592,1357878655,PL +1357878656,1357878719,DE +1357878720,1357878783,EU 1357878784,1357879807,DE 1357879808,1357879871,RU 1357879872,1357879935,DE @@ -32391,12 +33614,14 @@ 1357883584,1357883727,FR 1357883728,1357883759,EU 1357883760,1357883775,FR -1357883776,1357883839,EU +1357883776,1357883807,RU +1357883808,1357883839,EU 1357883840,1357883903,FR 1357883904,1357883999,EU 1357884000,1357884031,FR 1357884032,1357884095,RU -1357884096,1357884415,EU +1357884096,1357884159,EU +1357884160,1357884415,GB 1357884416,1357884423,FR 1357884424,1357884427,BE 1357884428,1357884431,EU @@ -32417,9 +33642,7 @@ 1357885952,1357886463,FR 1357886464,1357887487,US 1357887488,1357888511,FR -1357888512,1357888703,DE -1357888704,1357888767,EU -1357888768,1357889023,DE +1357888512,1357889023,DE 1357889024,1357889535,GB 1357889536,1357890815,PL 1357890816,1357890827,NL @@ -32435,10 +33658,20 @@ 1357891584,1357891647,EU 1357891648,1357891679,NL 1357891680,1357891711,GB -1357891712,1357891839,EU +1357891712,1357891839,FI 1357891840,1357892095,GB -1357892096,1357892223,IE -1357892224,1357892351,EU +1357892096,1357892111,DE +1357892112,1357892127,ES +1357892128,1357892143,FR +1357892144,1357892159,IT +1357892160,1357892175,GB +1357892176,1357892183,AR +1357892184,1357892191,AU +1357892192,1357892199,BR +1357892200,1357892207,NL +1357892208,1357892215,SE +1357892216,1357892223,US +1357892224,1357892351,FI 1357892352,1357893119,NL 1357893120,1357893375,SE 1357893376,1357893407,EU @@ -32478,7 +33711,8 @@ 1357903872,1357904383,EU 1357904384,1357904895,DE 1357904896,1357905407,GB -1357905408,1357905919,EU +1357905408,1357905663,FR +1357905664,1357905919,NL 1357905920,1357910015,LT 1357910016,1357914111,GR 1357914112,1357922303,RU @@ -32551,9 +33785,7 @@ 1358020608,1358028799,DE 1358028800,1358032895,IT 1358032896,1358036991,GE -1358036992,1358039039,IR -1358039040,1358040831,SK -1358040832,1358041087,IR +1358036992,1358041087,IR 1358041088,1358045183,FI 1358045184,1358049279,SK 1358049280,1358061567,RU @@ -32605,14 +33837,22 @@ 1358223360,1358223631,GB 1358223632,1358223663,DE 1358223664,1358223671,GB -1358223672,1358223719,DE +1358223672,1358223687,DE +1358223688,1358223695,GB +1358223696,1358223719,DE 1358223720,1358223783,GB -1358223784,1358223843,DE +1358223784,1358223791,DE +1358223792,1358223799,GB +1358223800,1358223843,DE 1358223844,1358223871,GB 1358223872,1358223887,NL -1358223888,1358223911,DE +1358223888,1358223895,DE +1358223896,1358223903,NL +1358223904,1358223911,DE 1358223912,1358223927,NL -1358223928,1358223975,DE +1358223928,1358223935,DE +1358223936,1358223939,NL +1358223940,1358223975,DE 1358223976,1358223991,NL 1358223992,1358223999,DE 1358224000,1358224079,NL @@ -32621,14 +33861,21 @@ 1358224520,1358224611,DE 1358224612,1358224651,BE 1358224652,1358224655,DE -1358224656,1358224719,BE -1358224720,1358224959,DE +1358224656,1358224727,BE +1358224728,1358224959,DE 1358224960,1358225127,IT -1358225128,1358225407,DE +1358225128,1358225135,DE +1358225136,1358225143,IT +1358225144,1358225151,DE +1358225152,1358225183,IT +1358225184,1358225191,DE +1358225192,1358225199,IT +1358225200,1358225407,DE 1358225408,1358229503,RU 1358229504,1358230111,DE 1358230112,1358230115,AT -1358230116,1358230383,DE +1358230116,1358230119,ES +1358230120,1358230383,DE 1358230384,1358230391,TR 1358230392,1358230399,CH 1358230400,1358232863,DE @@ -32834,8 +34081,10 @@ 1358668368,1358668463,PT 1358668464,1358668479,ES 1358668480,1358668495,GB -1358668496,1358668551,PT -1358668552,1358668575,GB +1358668496,1358668535,PT +1358668536,1358668543,GB +1358668544,1358668559,PT +1358668560,1358668575,GB 1358668576,1358668799,PT 1358668800,1358668807,GB 1358668808,1358668927,PT @@ -32854,13 +34103,17 @@ 1358670016,1358670023,GB 1358670024,1358670183,PT 1358670184,1358670191,GB -1358670192,1358671415,PT +1358670192,1358670943,PT +1358670944,1358670959,GB +1358670960,1358671415,PT 1358671416,1358671423,GB 1358671424,1358671431,PT 1358671432,1358671439,GB 1358671440,1358671839,PT 1358671840,1358671855,ES -1358671856,1358671967,PT +1358671856,1358671871,PT +1358671872,1358671903,GB +1358671904,1358671967,PT 1358671968,1358671983,GB 1358671984,1358672479,PT 1358672480,1358672511,GB @@ -32868,7 +34121,8 @@ 1358672688,1358672703,GB 1358672704,1358673535,PT 1358673536,1358673567,US -1358673568,1358673663,PT +1358673568,1358673599,GB +1358673600,1358673663,PT 1358673664,1358675967,GB 1358675968,1358676735,SE 1358676736,1358677759,DK @@ -33141,7 +34395,6 @@ 1359003648,1359020031,CZ 1359020032,1359036415,FR 1359036416,1359052799,GB -1359052800,1359069183,SE 1359069184,1359101951,RU 1359101952,1359118335,GB 1359118336,1359118655,DE @@ -33152,7 +34405,9 @@ 1359120384,1359121407,DK 1359121408,1359123583,DE 1359123584,1359123711,SE -1359123712,1359134303,DE +1359123712,1359133311,DE +1359133312,1359133439,AT +1359133440,1359134303,DE 1359134304,1359134319,NL 1359134320,1359134719,DE 1359134720,1359150079,CZ @@ -33211,8 +34466,8 @@ 1359467488,1359467495,DE 1359467496,1359467647,US 1359467648,1359467775,DE -1359467776,1359467815,US -1359467816,1359467903,DE +1359467776,1359467823,US +1359467824,1359467903,DE 1359467904,1359468063,US 1359468064,1359468575,DE 1359468576,1359468583,SG @@ -33491,7 +34746,9 @@ 1360736256,1360740351,RU 1360740352,1360743679,GB 1360743680,1360743935,FR -1360743936,1360752639,GB +1360743936,1360749759,GB +1360749760,1360749791,IM +1360749792,1360752639,GB 1360752640,1360756735,DE 1360756736,1360760831,IT 1360760832,1360764927,RU @@ -33536,14 +34793,12 @@ 1360891648,1360891903,RU 1360891904,1360892427,CH 1360892428,1360892431,IT -1360892432,1360892479,CH -1360892480,1360892487,FR -1360892488,1360892495,CH -1360892496,1360892502,GB -1360892503,1360892763,CH -1360892764,1360892767,IT -1360892768,1360892795,CH -1360892796,1360892799,IT +1360892432,1360892487,CH +1360892488,1360892495,IT +1360892496,1360892751,CH +1360892752,1360892767,IT +1360892768,1360892791,CH +1360892792,1360892799,IT 1360892800,1360892807,CH 1360892808,1360892815,MR 1360892816,1360892927,CH @@ -33566,9 +34821,7 @@ 1360905392,1360905407,GB 1360905408,1360906687,IT 1360906688,1360906719,GB -1360906720,1360907679,IT -1360907680,1360907711,GB -1360907712,1360909391,IT +1360906720,1360909391,IT 1360909392,1360909407,GB 1360909408,1360910495,IT 1360910496,1360910511,GB @@ -34130,7 +35383,8 @@ 1364971520,1364975615,CZ 1364975616,1364979711,BJ 1364979712,1364982783,GB -1364982784,1364983551,CH +1364982784,1364983295,CH +1364983296,1364983551,US 1364983552,1364983807,GB 1364983808,1364991999,DE 1364992000,1364996095,IT @@ -34561,7 +35815,8 @@ 1371799552,1371865087,RO 1371865088,1371930623,FI 1371930624,1371996159,LV -1371996160,1371997183,A2 +1371996160,1371996415,NG +1371996416,1371997183,A2 1371997184,1371997439,KE 1371997440,1371998207,NG 1371998208,1371998463,CD @@ -34885,8 +36140,7 @@ 1382187008,1382191871,ES 1382191872,1382192127,VE 1382192128,1382203391,ES -1382203392,1382204479,NL -1382204480,1382205439,GB +1382203392,1382205439,GB 1382205440,1382205695,EU 1382205696,1382205951,DE 1382205952,1382207743,EU @@ -34906,25 +36160,13 @@ 1382212240,1382212607,EU 1382212608,1382212863,FR 1382212864,1382213631,EU -1382213632,1382214079,NL -1382214080,1382214111,GB -1382214112,1382214879,NL -1382214880,1382214895,GB -1382214896,1382216831,NL -1382216832,1382216895,GB -1382216896,1382216959,NL -1382216960,1382217215,GB -1382217216,1382218399,NL -1382218400,1382218447,GB -1382218448,1382218455,NL -1382218456,1382218495,GB -1382218496,1382218623,NL -1382218624,1382218751,GB -1382218752,1382218815,NL -1382218816,1382218911,GB -1382218912,1382218959,NL -1382218960,1382218991,GB -1382218992,1382219775,NL +1382213632,1382215679,GB +1382215680,1382216447,NL +1382216448,1382217727,GB +1382217728,1382217983,NL +1382217984,1382219007,GB +1382219008,1382219519,NL +1382219520,1382219775,GB 1382219776,1382223327,SE 1382223328,1382223359,FI 1382223360,1382223551,SE @@ -35360,7 +36602,6 @@ 1385480192,1385488383,NL 1385488384,1385496575,GB 1385496576,1385504767,SI -1385504768,1385512959,GB 1385512960,1385521151,DE 1385521152,1385529343,AT 1385529344,1385537535,RU @@ -35489,7 +36730,15 @@ 1387790336,1388314623,IT 1388314624,1388322815,AT 1388322816,1388331007,SI -1388331008,1388339199,NL +1388331008,1388338176,NL +1388338177,1388338178,FR +1388338179,1388338180,GB +1388338181,1388338182,DE +1388338183,1388338186,NL +1388338187,1388338188,IT +1388338189,1388338190,NL +1388338191,1388338192,PL +1388338193,1388339199,NL 1388339200,1388347391,GB 1388347392,1388363775,DK 1388363776,1388371967,DE @@ -35616,16 +36865,17 @@ 1388699648,1388707839,SE 1388707840,1388708863,RU 1388708864,1388709119,LT -1388709120,1388711679,RU +1388709120,1388709375,IQ +1388709376,1388709887,RU +1388709888,1388710143,LB +1388710144,1388711679,RU 1388711680,1388711935,IQ 1388711936,1388712191,RU 1388712192,1388712703,LT 1388712704,1388713215,LB 1388713216,1388713727,RU 1388713728,1388713983,LT -1388713984,1388714239,LB -1388714240,1388714495,LT -1388714496,1388714751,RU +1388713984,1388714751,RU 1388714752,1388715007,IQ 1388715008,1388715519,LT 1388715520,1388715775,LV @@ -35702,7 +36952,9 @@ 1388743444,1388743451,GB 1388743452,1388743555,IE 1388743556,1388743559,GB -1388743560,1388743919,IE +1388743560,1388743571,IE +1388743572,1388743583,GB +1388743584,1388743919,IE 1388743920,1388743935,GB 1388743936,1388744087,IE 1388744088,1388744095,GB @@ -35772,7 +37024,11 @@ 1388781568,1388789759,ES 1388789760,1388791519,NL 1388791520,1388791527,GB -1388791528,1388796559,NL +1388791528,1388794943,NL +1388794944,1388794959,GB +1388794960,1388795343,NL +1388795344,1388795359,GB +1388795360,1388796559,NL 1388796560,1388796575,GB 1388796576,1388796679,NL 1388796680,1388796687,GB @@ -35830,7 +37086,9 @@ 1389204800,1389204863,IQ 1389204864,1389204879,A2 1389204880,1389204895,IQ -1389204896,1389205631,A2 +1389204896,1389205247,A2 +1389205248,1389205503,AF +1389205504,1389205631,A2 1389205632,1389205759,DE 1389205760,1389205791,A2 1389205792,1389205935,IR @@ -36217,14 +37475,16 @@ 1389561408,1389561471,DE 1389561472,1389561599,IT 1389561600,1389562367,DE -1389562368,1389562623,IT -1389562624,1389562879,DE -1389562880,1389563135,IT +1389562368,1389563135,IT 1389563136,1389563391,DE 1389563392,1389563647,IT 1389563648,1389563903,DE 1389563904,1389564159,IT -1389564160,1389576191,DE +1389564160,1389564415,DE +1389564416,1389564671,IT +1389564672,1389565183,DE +1389565184,1389565695,IT +1389565696,1389576191,DE 1389576192,1389592575,GB 1389592576,1389608959,GE 1389608960,1389625343,DK @@ -36236,8 +37496,9 @@ 1389707264,1389723647,IT 1389723648,1389756415,ES 1389756416,1389772799,SE -1389772800,1389779455,SI -1389779456,1389780991,RS +1389772800,1389778431,SI +1389778432,1389778943,US +1389778944,1389780991,RS 1389780992,1389782527,HR 1389782528,1389782543,SI 1389782544,1389782559,HR @@ -36359,8 +37620,8 @@ 1400111104,1400373247,NL 1400373248,1400705279,DE 1400705280,1400706047,EU -1400706048,1400707071,DE -1400707072,1400709119,EU +1400706048,1400708095,DE +1400708096,1400709119,EU 1400709120,1400710143,DE 1400710144,1400710399,EU 1400710400,1400712191,DE @@ -36574,7 +37835,8 @@ 1401745936,1401745959,SE 1401745960,1401745967,US 1401745968,1401745971,MY -1401745972,1401745983,SE +1401745972,1401745975,NL +1401745976,1401745983,SE 1401745984,1401745999,NL 1401746000,1401746015,ES 1401746016,1401746019,GB @@ -36706,7 +37968,9 @@ 1401962496,1401978879,PL 1401978880,1401995263,PT 1401995264,1402011647,CH -1402011648,1402028031,SE +1402011648,1402027823,SE +1402027824,1402027831,GB +1402027832,1402028031,SE 1402028032,1402044415,FR 1402044416,1402060799,PL 1402060800,1402068159,FI @@ -36960,7 +38224,8 @@ 1404803072,1404804095,LV 1404804096,1404805119,SE 1404805120,1404813311,AT -1404813312,1404829695,SE +1404813312,1404815615,EE +1404815616,1404829695,SE 1404829696,1404870655,RU 1404870656,1404872703,LT 1404872704,1404874751,SE @@ -37073,9 +38338,7 @@ 1406935040,1406951423,RU 1406951424,1406959615,PL 1406959616,1406964287,DE -1406964288,1406964319,NL -1406964320,1406964327,DE -1406964328,1406964351,NL +1406964288,1406964351,NL 1406964352,1406964735,DE 1406964736,1406964927,US 1406964928,1406967295,DE @@ -37162,7 +38425,8 @@ 1407518016,1407518031,ZA 1407518032,1407518039,A2 1407518040,1407518047,CD -1407518048,1407518063,NG +1407518048,1407518055,A2 +1407518056,1407518063,NG 1407518064,1407518079,A2 1407518080,1407518111,LR 1407518112,1407518127,NG @@ -37309,7 +38573,7 @@ 1407520744,1407520751,NG 1407520752,1407520759,A2 1407520760,1407520767,CD -1407520768,1407520775,ZW +1407520768,1407520775,TZ 1407520776,1407520783,CD 1407520784,1407520791,SD 1407520792,1407520799,ZA @@ -37401,8 +38665,8 @@ 1407522584,1407522591,GL 1407522592,1407522599,A2 1407522600,1407522607,ZM -1407522608,1407522615,A2 -1407522616,1407522631,NG +1407522608,1407522623,A2 +1407522624,1407522631,NG 1407522632,1407522639,MW 1407522640,1407522647,GQ 1407522648,1407522655,NG @@ -37427,7 +38691,9 @@ 1407522816,1407522823,CD 1407522824,1407522831,NG 1407522832,1407522839,MZ -1407522840,1407522871,A2 +1407522840,1407522855,A2 +1407522856,1407522863,ZW +1407522864,1407522871,A2 1407522872,1407522879,IQ 1407522880,1407522903,A2 1407522904,1407522911,KE @@ -37449,7 +38715,7 @@ 1407523136,1407523143,BJ 1407523144,1407523159,A2 1407523160,1407523167,LR -1407523168,1407523183,UG +1407523168,1407523183,A2 1407523184,1407523191,NG 1407523192,1407523199,A2 1407523200,1407523207,NG @@ -37485,7 +38751,8 @@ 1407524640,1407524647,A2 1407524648,1407524655,CD 1407524656,1407524663,NG -1407524664,1407524679,A2 +1407524664,1407524671,CD +1407524672,1407524679,A2 1407524680,1407524687,LR 1407524688,1407524703,MZ 1407524704,1407524727,A2 @@ -37502,8 +38769,7 @@ 1407524856,1407524863,CD 1407524864,1407524895,A2 1407524896,1407524903,LR -1407524904,1407524911,NG -1407524912,1407524935,A2 +1407524904,1407524935,A2 1407524936,1407524943,GR 1407524944,1407524951,A2 1407524952,1407524959,CM @@ -37551,7 +38817,8 @@ 1407525400,1407525407,KE 1407525408,1407525471,A2 1407525472,1407525479,UG -1407525480,1407525543,A2 +1407525480,1407525487,CD +1407525488,1407525543,A2 1407525544,1407525551,KE 1407525552,1407525583,A2 1407525584,1407525591,GL @@ -37677,9 +38944,9 @@ 1407535776,1407535783,GN 1407535784,1407535815,A2 1407535816,1407535823,NG -1407535824,1407535839,GB -1407535840,1407535847,NG -1407535848,1407535855,A2 +1407535824,1407535831,GB +1407535832,1407535847,A2 +1407535848,1407535855,CD 1407535856,1407535863,NG 1407535864,1407536127,A2 1407536128,1407536639,GA @@ -37708,15 +38975,15 @@ 1407537184,1407537191,NG 1407537192,1407537207,A2 1407537208,1407537215,BE -1407537216,1407537231,A2 -1407537232,1407537255,NG +1407537216,1407537239,A2 +1407537240,1407537255,NG 1407537256,1407537287,A2 1407537288,1407537295,NG 1407537296,1407537303,UG 1407537304,1407537319,A2 1407537320,1407537327,NG -1407537328,1407537335,A2 -1407537336,1407537351,NG +1407537328,1407537343,A2 +1407537344,1407537351,NG 1407537352,1407537367,A2 1407537368,1407537383,NG 1407537384,1407537399,A2 @@ -37841,7 +39108,8 @@ 1407539768,1407539775,GR 1407539776,1407539783,A2 1407539784,1407539791,GR -1407539792,1407539831,A2 +1407539792,1407539799,NG +1407539800,1407539831,A2 1407539832,1407539839,NG 1407539840,1407539863,A2 1407539864,1407539879,NG @@ -37965,8 +39233,10 @@ 1407549440,1407582207,RU 1407582208,1407614975,PL 1407614976,1407680511,ES -1407680512,1407680575,FR -1407680576,1407681279,GB +1407680512,1407680591,FR +1407680592,1407680607,GB +1407680608,1407680623,FR +1407680624,1407681279,GB 1407681280,1407681287,ES 1407681288,1407681343,GB 1407681344,1407681407,ES @@ -38451,15 +39721,18 @@ 1410748416,1410752511,DE 1410752512,1410752703,AQ 1410752704,1410752763,DE -1410752764,1410752785,AQ -1410752786,1410752799,DE +1410752764,1410752787,AQ +1410752788,1410752799,DE 1410752800,1410752827,AQ -1410752828,1410753023,DE +1410752828,1410752831,DE +1410752832,1410752835,AQ +1410752836,1410753023,DE 1410753024,1410753031,AQ 1410753032,1410753055,BS 1410753056,1410753143,AQ 1410753144,1410753151,SC -1410753152,1410753159,DE +1410753152,1410753155,AQ +1410753156,1410753159,DE 1410753160,1410753167,AQ 1410753168,1410753303,DE 1410753304,1410753327,AQ @@ -38483,16 +39756,12 @@ 1410756736,1410756863,DE 1410756864,1410757119,AQ 1410757120,1410758655,DE -1410758656,1410760191,AQ -1410760192,1410760455,DE +1410758656,1410759679,AQ +1410759680,1410760455,DE 1410760456,1410760471,AQ -1410760472,1410760479,SC -1410760480,1410760487,DE +1410760472,1410760487,DE 1410760488,1410760503,AQ -1410760504,1410760683,DE -1410760684,1410760687,AQ -1410760688,1410760699,DE -1410760700,1410760703,AQ +1410760504,1410760703,DE 1410760704,1410768895,RU 1410768896,1410777087,GB 1410777088,1410785279,RU @@ -38613,7 +39882,9 @@ 1411999800,1411999807,SI 1411999808,1411999847,BA 1411999848,1411999871,SI -1411999872,1412000767,BA +1411999872,1411999887,BA +1411999888,1411999999,SI +1412000000,1412000767,BA 1412000768,1412000783,SI 1412000784,1412000791,BA 1412000792,1412000863,SI @@ -38860,7 +40131,7 @@ 1424595456,1424595527,IT 1424595528,1424595535,GB 1424595536,1424595575,IT -1424595576,1424595583,ES +1424595576,1424595583,GB 1424595584,1424595819,IT 1424595820,1424595823,GB 1424595824,1424595831,IT @@ -38868,9 +40139,7 @@ 1424595840,1424595903,IT 1424595904,1424595935,GB 1424595936,1424596479,IT -1424596480,1424596563,FR -1424596564,1424596567,GB -1424596568,1424596991,FR +1424596480,1424596991,FR 1424596992,1424597255,CZ 1424597256,1424597263,GB 1424597264,1424597311,CZ @@ -38893,7 +40162,9 @@ 1424599876,1424599879,GB 1424599880,1424599887,HU 1424599888,1424599903,GB -1424599904,1424599951,HU +1424599904,1424599935,HU +1424599936,1424599943,GB +1424599944,1424599951,HU 1424599952,1424599999,GB 1424600000,1424600031,HU 1424600032,1424600063,GB @@ -38963,7 +40234,9 @@ 1424608128,1424608279,GB 1424608280,1424608383,FR 1424608384,1424608399,ES -1424608400,1424608511,FR +1424608400,1424608407,FR +1424608408,1424608415,GB +1424608416,1424608511,FR 1424608512,1424608687,ES 1424608688,1424608695,GB 1424608696,1424609023,ES @@ -39013,8 +40286,7 @@ 1424616192,1424616447,IT 1424616448,1424616483,ES 1424616484,1424616487,GB -1424616488,1424616607,ES -1424616608,1424616615,PT +1424616488,1424616615,ES 1424616616,1424616623,GB 1424616624,1424616655,ES 1424616656,1424616659,GB @@ -39184,9 +40456,7 @@ 1425448960,1425451007,CZ 1425451008,1425457855,GB 1425457856,1425457919,US -1425457920,1425458271,GB -1425458272,1425458303,US -1425458304,1425459199,GB +1425457920,1425459199,GB 1425459200,1425460735,IM 1425460736,1425460767,GB 1425460768,1425460863,IM @@ -39490,7 +40760,9 @@ 1426731008,1426735103,HR 1426735104,1426751487,BE 1426751488,1426767871,FR -1426767872,1426784255,DE +1426767872,1426778991,DE +1426778992,1426778999,IT +1426779000,1426784255,DE 1426784256,1426800639,SE 1426800640,1426817023,PL 1426817024,1426833407,BG @@ -39528,7 +40800,9 @@ 1426915328,1426931711,AT 1426931712,1426948095,CZ 1426948096,1426964479,DE -1426964480,1426967455,GB +1426964480,1426967287,GB +1426967288,1426967295,ES +1426967296,1426967455,GB 1426967456,1426967463,ES 1426967464,1426967567,GB 1426967568,1426967575,ES @@ -39568,11 +40842,7 @@ 1427032736,1427032743,AT 1427032744,1427032959,DE 1427032960,1427032967,NL -1427032968,1427033247,DE -1427033248,1427033263,MZ -1427033264,1427033287,DE -1427033288,1427033295,SE -1427033296,1427035391,DE +1427032968,1427035391,DE 1427035392,1427035519,GB 1427035520,1427046399,DE 1427046400,1427062783,IR @@ -39621,9 +40891,13 @@ 1427742720,1427742751,IO 1427742752,1427742879,DE 1427742880,1427742911,TR -1427742912,1427743039,DE +1427742912,1427742975,DE +1427742976,1427743007,RU +1427743008,1427743039,US 1427743040,1427743071,BG -1427743072,1427743231,DE +1427743072,1427743103,CY +1427743104,1427743199,DE +1427743200,1427743231,TR 1427743232,1427743263,GR 1427743264,1427743327,DE 1427743328,1427743359,NL @@ -39650,7 +40924,8 @@ 1427744320,1427744351,DE 1427744352,1427744383,UA 1427744384,1427744415,PL -1427744416,1427744639,DE +1427744416,1427744447,UA +1427744448,1427744639,DE 1427744640,1427744671,TR 1427744672,1427744703,DE 1427744704,1427744735,CA @@ -39659,14 +40934,15 @@ 1427744864,1427744927,TR 1427744928,1427744991,DE 1427744992,1427745023,DK -1427745024,1427745087,DE -1427745088,1427745119,RU -1427745120,1427745151,DE +1427745024,1427745055,BM +1427745056,1427745151,DE 1427745152,1427745183,RO 1427745184,1427745215,CH 1427745216,1427745247,DE 1427745248,1427745279,TR -1427745280,1427745503,DE +1427745280,1427745439,DE +1427745440,1427745471,US +1427745472,1427745503,DE 1427745504,1427745535,RU 1427745536,1427745567,CN 1427745568,1427745599,RU @@ -39683,15 +40959,16 @@ 1427746112,1427746143,DE 1427746144,1427746175,GR 1427746176,1427746239,DE -1427746240,1427746271,NL +1427746240,1427746271,US 1427746272,1427747839,DE 1427747840,1427747871,FI -1427747872,1427747903,IN -1427747904,1427747967,DE +1427747872,1427747967,DE 1427747968,1427747999,BG 1427748000,1427748063,DE -1427748064,1427748095,NL -1427748096,1427748447,DE +1427748064,1427748095,US +1427748096,1427748255,DE +1427748256,1427748287,US +1427748288,1427748447,DE 1427748448,1427748479,RU 1427748480,1427748511,TR 1427748512,1427748543,DE @@ -39710,7 +40987,7 @@ 1427750016,1427750079,DE 1427750080,1427750111,BG 1427750112,1427750239,DE -1427750240,1427750271,NL +1427750240,1427750271,US 1427750272,1427750303,DE 1427750304,1427750335,GB 1427750336,1427751167,DE @@ -39718,8 +40995,7 @@ 1427751424,1427759935,DE 1427759936,1427759967,HR 1427759968,1427760191,DE -1427760192,1427760223,NL -1427760224,1427760255,US +1427760192,1427760255,US 1427760256,1427760319,BR 1427760320,1427760351,TR 1427760352,1427760383,RU @@ -39737,9 +41013,10 @@ 1427761120,1427761183,DE 1427761184,1427761215,IT 1427761216,1427761247,TR -1427761248,1427761279,BR +1427761248,1427761279,DE 1427761280,1427761311,RU -1427761312,1427761375,DE +1427761312,1427761343,DE +1427761344,1427761375,CY 1427761376,1427761407,TW 1427761408,1427761503,DE 1427761504,1427761535,PL @@ -39748,9 +41025,7 @@ 1427761600,1427761631,EG 1427761632,1427767295,DE 1427767296,1427800063,BE -1427800064,1427823615,RU -1427823616,1427824639,KG -1427824640,1427832831,RU +1427800064,1427832831,RU 1427832832,1427865599,BE 1427865600,1427898367,DK 1427898368,1427914751,RU @@ -39785,17 +41060,25 @@ 1428142080,1428143263,DE 1428143264,1428143279,CH 1428143280,1428144127,DE -1428144128,1428147595,FR -1428147596,1428147711,GB +1428144128,1428147343,FR +1428147344,1428147351,GB +1428147352,1428147595,FR +1428147596,1428147615,GB +1428147616,1428147639,FR +1428147640,1428147711,GB 1428147712,1428147855,FR 1428147856,1428147871,US 1428147872,1428148175,FR 1428148176,1428148191,CH 1428148192,1428150487,FR 1428150488,1428150495,GB -1428150496,1428150735,FR +1428150496,1428150663,FR +1428150664,1428150687,GB +1428150688,1428150735,FR 1428150736,1428150751,GB -1428150752,1428151007,FR +1428150752,1428150863,FR +1428150864,1428150911,GB +1428150912,1428151007,FR 1428151008,1428151039,GB 1428151040,1428151231,FR 1428151232,1428151295,CH @@ -39834,9 +41117,7 @@ 1431865632,1431865647,ES 1431865648,1431865663,PT 1431865664,1431865679,ES -1431865680,1431865711,PT -1431865712,1431865719,ES -1431865720,1431866447,PT +1431865680,1431866447,PT 1431866448,1431866455,NL 1431866456,1431866479,PT 1431866480,1431866495,ES @@ -39858,7 +41139,9 @@ 1431979480,1431979487,GB 1431979488,1431980151,NL 1431980152,1431980159,US -1431980160,1431980719,NL +1431980160,1431980687,NL +1431980688,1431980695,DE +1431980696,1431980719,NL 1431980720,1431980727,SA 1431980728,1431986175,NL 1431986176,1431994367,RU @@ -39914,8 +41197,8 @@ 1432322048,1432338431,RU 1432338432,1432346623,FR 1432346624,1433403391,TR -1433403392,1433406239,ES -1433406240,1433406271,US +1433403392,1433406255,ES +1433406256,1433406271,US 1433406272,1433406431,ES 1433406432,1433406447,US 1433406448,1433407487,ES @@ -39926,7 +41209,9 @@ 1433411328,1433411343,ES 1433411344,1433411375,TR 1433411376,1433411407,US -1433411408,1433411455,TR +1433411408,1433411423,TR +1433411424,1433411439,ES +1433411440,1433411455,TR 1433411456,1433411519,US 1433411520,1433411583,ES 1433411584,1433419775,RU @@ -40074,8 +41359,8 @@ 1434583040,1434615807,BG 1434615808,1434648575,IL 1434648576,1434681343,FI -1434681344,1434681855,DE -1434681856,1434682111,MY +1434681344,1434681983,DE +1434681984,1434682111,NL 1434682112,1434682303,DE 1434682304,1434682367,NL 1434682368,1434683119,DE @@ -40088,7 +41373,11 @@ 1434686720,1434687231,NL 1434687232,1434687359,DE 1434687360,1434687487,NL -1434687488,1434688207,DE +1434687488,1434687647,DE +1434687648,1434687679,NL +1434687680,1434688031,DE +1434688032,1434688063,NL +1434688064,1434688207,DE 1434688208,1434688223,NL 1434688224,1434689887,DE 1434689888,1434689919,NL @@ -40387,8 +41676,8 @@ 1439457280,1439457791,GB 1439457792,1439462911,RO 1439462912,1439463167,GB -1439463168,1439463679,RO -1439463680,1439463935,GB +1439463168,1439463423,RO +1439463424,1439463935,GB 1439463936,1439464447,RO 1439464448,1439464703,GB 1439464704,1439466495,RO @@ -40467,7 +41756,9 @@ 1441300480,1441308671,LT 1441308672,1441316863,SE 1441316864,1441325055,MT -1441325056,1441333247,PL +1441325056,1441325759,PL +1441325760,1441325775,US +1441325776,1441333247,PL 1441333248,1441349631,DE 1441349632,1441357823,RU 1441357824,1441366015,AT @@ -41727,9 +43018,15 @@ 1449711360,1449711615,GB 1449711616,1449717759,RO 1449717760,1449718783,GB -1449718784,1449767423,RO +1449718784,1449719039,RO +1449719040,1449719807,GB +1449719808,1449756671,RO +1449756672,1449758719,GB +1449758720,1449767423,RO 1449767424,1449767679,GB -1449767680,1449773055,RO +1449767680,1449771775,RO +1449771776,1449772031,GB +1449772032,1449773055,RO 1449773056,1449774079,GB 1449774080,1449776383,RO 1449776384,1449776639,GB @@ -41742,9 +43039,7 @@ 1449784320,1449785343,RO 1449785344,1449786367,GB 1449786368,1449787391,RO -1449787392,1449790207,GB -1449790208,1449790719,RO -1449790720,1449791487,GB +1449787392,1449791487,GB 1449791488,1449795327,RO 1449795328,1449795583,GB 1449795584,1449808639,RO @@ -41753,9 +43048,12 @@ 1449823232,1449826303,GB 1449826304,1449827839,RO 1449827840,1449828095,GB -1449828096,1449840639,RO +1449828096,1449830655,RO +1449830656,1449830911,GB +1449830912,1449840639,RO 1449840640,1449852927,MD -1449852928,1449859071,RO +1449852928,1449854975,GB +1449854976,1449859071,RO 1449859072,1449859583,GB 1449859584,1449859839,RO 1449859840,1449860095,GB @@ -41763,11 +43061,9 @@ 1449869312,1449870335,GB 1449870336,1449870847,RO 1449870848,1449871615,GB -1449871616,1449879807,RO -1449879808,1449880063,GB -1449880064,1449880319,RO -1449880320,1449881087,GB -1449881088,1449895935,RO +1449871616,1449879551,RO +1449879552,1449881599,GB +1449881600,1449895935,RO 1449895936,1449896191,GB 1449896192,1449896703,RO 1449896704,1449896959,GB @@ -41776,8 +43072,8 @@ 1449899264,1449899519,RO 1449899520,1449899775,GB 1449899776,1449904127,RO -1449904128,1449908223,GB -1449908224,1449918463,RO +1449904128,1449910271,GB +1449910272,1449918463,RO 1449918464,1449951231,JO 1449951232,1449983999,TR 1449984000,1449992191,NL @@ -41906,9 +43202,7 @@ 1467340960,1467340991,TR 1467340992,1467344639,GB 1467344640,1467344895,ZA -1467344896,1467347903,GB -1467347904,1467347935,SE -1467347936,1467351039,GB +1467344896,1467351039,GB 1467351040,1467367423,NO 1467367424,1467367615,DE 1467367616,1467367647,SE @@ -41953,9 +43247,7 @@ 1467466032,1467466111,NL 1467466112,1467466143,DE 1467466144,1467466175,NL -1467466176,1467466191,DE -1467466192,1467466207,NL -1467466208,1467466215,DE +1467466176,1467466215,DE 1467466216,1467466223,VG 1467466224,1467466239,NL 1467466240,1467466271,DE @@ -41983,7 +43275,9 @@ 1472259256,1472264799,IE 1472264800,1472264807,GB 1472264808,1472266239,IE -1472266240,1472331775,DE +1472266240,1472304610,DE +1472304611,1472304611,A2 +1472304612,1472331775,DE 1472331776,1472397311,GB 1472397312,1472462847,NL 1472462848,1472528383,PT @@ -42047,6 +43341,7 @@ 1475133440,1475135487,CZ 1475135488,1475137535,CH 1475137536,1475139583,SE +1475139584,1475141631,ES 1475141632,1475143679,FI 1475143680,1475145727,JO 1475145728,1475147775,ES @@ -42074,7 +43369,9 @@ 1475181536,1475181551,NL 1475181552,1475181855,DE 1475181856,1475181887,LU -1475181888,1475184639,DE +1475181888,1475182079,DE +1475182080,1475182335,RO +1475182336,1475184639,DE 1475184640,1475186687,RU 1475188736,1475190783,SE 1475190784,1475192831,GB @@ -42199,7 +43496,8 @@ 1475307520,1475309567,RU 1475309568,1475311615,LT 1475311616,1475313663,FI -1475313664,1475317759,DE +1475313664,1475315711,DE +1475315712,1475317759,US 1475317760,1475319807,RU 1475319808,1475321855,BG 1475321856,1475323903,RO @@ -42231,7 +43529,9 @@ 1475543040,1475559423,RO 1475559424,1475562751,GR 1475562752,1475563007,BG -1475563008,1475575807,GR +1475563008,1475571711,GR +1475571712,1475572735,BG +1475572736,1475575807,GR 1475575808,1475592191,AT 1475592192,1475608575,GB 1475608576,1475624959,RU @@ -42279,14 +43579,10 @@ 1475639528,1475639543,JE 1475639544,1475639595,GB 1475639596,1475639599,JE -1475639600,1475639727,GB -1475639728,1475639735,JE -1475639736,1475639807,GB +1475639600,1475639807,GB 1475639808,1475641343,JE 1475641344,1475657727,UA -1475657728,1475664263,SK -1475664264,1475664271,UA -1475664272,1475674111,SK +1475657728,1475674111,SK 1475674112,1475690495,DE 1475690496,1475706879,CH 1475706880,1475723263,RU @@ -42680,7 +43976,13 @@ 1489731584,1489764351,BG 1489764352,1489797119,RU 1489797120,1489829887,KZ -1489829888,1489862655,DE +1489829888,1489855519,DE +1489855520,1489855535,PL +1489855536,1489855543,DE +1489855544,1489855551,PL +1489855552,1489855999,DE +1489856000,1489856031,PL +1489856032,1489862655,DE 1489862656,1489928191,RU 1489928192,1489960959,SE 1489960960,1489993727,HR @@ -42983,49 +44285,40 @@ 1495334912,1495335935,MD 1495335936,1495339007,RO 1495339008,1495343103,MD -1495343104,1495345151,GB -1495345152,1495347199,RO +1495343104,1495346175,GB +1495346176,1495347199,RO 1495347200,1495349247,GB 1495349248,1495399935,RO 1495399936,1495400191,MD -1495400192,1495513087,RO +1495400192,1495429119,RO +1495429120,1495431167,GB +1495431168,1495513087,RO 1495513088,1495515135,GB -1495515136,1495615487,RO -1495615488,1495616511,GB -1495616512,1495623679,RO +1495515136,1495623679,RO 1495623680,1495623935,MD -1495623936,1495669759,RO -1495669760,1495670783,GB -1495670784,1495752703,RO +1495623936,1495752703,RO 1495752704,1495754751,ES 1495754752,1495801855,RO 1495801856,1495802879,GB 1495802880,1495875583,RO 1495875584,1495891967,MD -1495891968,1495896063,RO -1495896064,1495898111,GB -1495898112,1495927295,RO +1495891968,1495927295,RO 1495927296,1495927551,AE 1495927552,1495937023,RO 1495937024,1495937535,ES 1495937536,1495986175,RO 1495986176,1495988223,GB -1495988224,1495992319,RO -1495992320,1495994367,GB -1495994368,1496074239,RO -1496074240,1496075263,GB -1496075264,1496078335,RO +1495988224,1495990271,RO +1495990272,1495994367,GB +1495994368,1496078335,RO 1496078336,1496079359,MD -1496079360,1496080383,RO -1496080384,1496081407,GB -1496081408,1496119295,RO -1496119296,1496121343,GB +1496079360,1496086527,RO +1496086528,1496088575,GB +1496088576,1496121343,RO 1496121344,1496122367,MD 1496122368,1496178943,RO 1496178944,1496179199,AE -1496179200,1496195071,RO -1496195072,1496197119,GB -1496197120,1496317951,RO +1496179200,1496317951,RO 1496317952,1497366527,DE 1497366528,1498415103,PL 1498415104,1499463679,FR @@ -43090,9 +44383,7 @@ 1500174336,1500176383,DE 1500176384,1500178431,GB 1500178432,1500180479,KZ -1500180480,1500180619,NL -1500180620,1500181503,BE -1500181504,1500182527,NL +1500180480,1500182527,BE 1500182528,1500184575,RU 1500184576,1500184831,GB 1500184832,1500184895,IM @@ -43265,8 +44556,8 @@ 1502981120,1502981887,US 1502981888,1502982143,NL 1502982144,1502982911,DE -1502982912,1502983679,FR -1502983680,1502986255,DE +1502982912,1502983423,FR +1502983424,1502986255,DE 1502986256,1502986495,TR 1502986496,1502986511,DE 1502986512,1502986751,PL @@ -43335,15 +44626,17 @@ 1503897304,1503897311,BE 1503897312,1503897367,DE 1503897368,1503897375,BE -1503897376,1503897391,GR -1503897392,1503897407,DE +1503897376,1503897383,GR +1503897384,1503897407,DE 1503897408,1503897415,IT 1503897416,1503897431,DE 1503897432,1503897439,IT 1503897440,1503897463,DE 1503897464,1503897471,GR 1503897472,1503897479,AT -1503897480,1503897583,DE +1503897480,1503897519,DE +1503897520,1503897527,IT +1503897528,1503897583,DE 1503897584,1503897591,GB 1503897592,1503898119,DE 1503898120,1503898135,TH @@ -43357,9 +44650,7 @@ 1503898216,1503898239,DE 1503898240,1503898303,IT 1503898304,1503898311,BE -1503898312,1503898327,DE -1503898328,1503898335,AT -1503898336,1503898351,DE +1503898312,1503898351,DE 1503898352,1503898359,CH 1503898360,1503898415,DE 1503898416,1503898423,BE @@ -43368,8 +44659,7 @@ 1503898504,1503898511,GR 1503898512,1503898567,DE 1503898568,1503898575,NL -1503898576,1503898583,GR -1503898584,1503898599,DE +1503898576,1503898599,DE 1503898600,1503898607,IT 1503898608,1503898615,GR 1503898616,1503898623,SE @@ -43383,16 +44673,22 @@ 1503898816,1503898831,GB 1503898832,1503898847,DE 1503898848,1503898855,MX -1503898856,1503898863,HU -1503898864,1503898887,DE +1503898856,1503898887,DE 1503898888,1503898895,IT 1503898896,1503898935,DE 1503898936,1503898943,IT -1503898944,1503898959,DE -1503898960,1503898967,IT -1503898968,1503898991,DE +1503898944,1503898991,DE 1503898992,1503898999,GB -1503899000,1503908351,DE +1503899000,1503899007,AT +1503899008,1503899063,DE +1503899064,1503899079,AT +1503899080,1503899119,DE +1503899120,1503899127,NL +1503899128,1503899143,DE +1503899144,1503899151,IT +1503899152,1503899159,DE +1503899160,1503899167,GB +1503899168,1503908351,DE 1503908352,1503909375,IT 1503909376,1503920127,DE 1503920128,1503985663,HR @@ -43509,10 +44805,12 @@ 1505336832,1505336863,GB 1505336864,1505336864,IE 1505336865,1505336879,GB -1505336880,1505337023,IE +1505336880,1505336959,IE +1505336960,1505336975,GB +1505336976,1505337023,IE 1505337024,1505337055,GB -1505337056,1505337087,IE -1505337088,1505337215,GB +1505337056,1505337071,IE +1505337072,1505337215,GB 1505337216,1505337343,IE 1505337344,1505345535,FR 1505345536,1505353727,MK @@ -43762,8 +45060,8 @@ 1506445338,1506445343,GB 1506445344,1506445519,FR 1506445520,1506445527,GB -1506445528,1506445535,FR -1506445536,1506445567,GB +1506445528,1506445566,FR +1506445567,1506445567,GB 1506445568,1506445703,FR 1506445704,1506445711,NL 1506445712,1506445719,GB @@ -43790,12 +45088,16 @@ 1506448736,1506448895,GB 1506448896,1506449159,BE 1506449160,1506449171,GB -1506449172,1506449407,BE +1506449172,1506449247,BE +1506449248,1506449279,GB +1506449280,1506449407,BE 1506449408,1506449663,NL 1506449664,1506449919,SK 1506449920,1506449927,CH 1506449928,1506449935,GB -1506449936,1506450431,CH +1506449936,1506449999,CH +1506450000,1506450015,GB +1506450016,1506450431,CH 1506450432,1506450767,CZ 1506450768,1506450847,GB 1506450848,1506450863,CZ @@ -43808,11 +45110,13 @@ 1506451040,1506451055,NO 1506451056,1506451059,DK 1506451060,1506451199,GB -1506451200,1506451775,ES -1506451776,1506451807,GB +1506451200,1506451791,ES +1506451792,1506451807,GB 1506451808,1506451895,ES 1506451896,1506451903,GB -1506451904,1506452223,ES +1506451904,1506452127,ES +1506452128,1506452159,GB +1506452160,1506452223,ES 1506452224,1506452479,GB 1506452480,1506452735,US 1506452736,1506452751,RO @@ -43872,7 +45176,9 @@ 1506461864,1506461887,GB 1506461888,1506462207,FR 1506462208,1506462463,ES -1506462464,1506462607,FR +1506462464,1506462527,FR +1506462528,1506462583,GB +1506462584,1506462607,FR 1506462608,1506462623,A2 1506462624,1506462719,FR 1506462720,1506463231,IT @@ -43890,8 +45196,8 @@ 1506467584,1506468351,IT 1506468352,1506468607,GB 1506468608,1506468863,TZ -1506468864,1506469039,IT -1506469040,1506469055,GB +1506468864,1506469047,IT +1506469048,1506469055,GB 1506469056,1506470143,IT 1506470144,1506470399,GB 1506470400,1506470655,DE @@ -43901,7 +45207,9 @@ 1506471976,1506471979,GB 1506471980,1506471999,NL 1506472000,1506472031,BE -1506472032,1506472191,NL +1506472032,1506472159,NL +1506472160,1506472175,GB +1506472176,1506472191,NL 1506472192,1506472447,IT 1506472448,1506472703,GB 1506472704,1506473215,IT @@ -43912,7 +45220,8 @@ 1506474496,1506474751,FR 1506474752,1506475519,IT 1506475520,1506475551,AT -1506475552,1506476031,GB +1506475552,1506475775,GB +1506475776,1506476031,DE 1506476032,1506508799,KW 1506508800,1506541567,CZ 1506541568,1506574335,RU @@ -43990,8 +45299,8 @@ 1507664384,1507664767,DE 1507664768,1507664895,US 1507664896,1507665407,GR -1507665408,1507665663,IT -1507665664,1507666431,US +1507665408,1507665919,IT +1507665920,1507666431,US 1507666432,1507666559,GR 1507666560,1507666591,FR 1507666592,1507666639,GR @@ -44056,7 +45365,9 @@ 1508646912,1508646927,DK 1508646928,1508646935,SE 1508646936,1508647039,DK -1508647040,1508648447,SE +1508647040,1508647679,SE +1508647680,1508647807,DK +1508647808,1508648447,SE 1508648448,1508648703,DK 1508648704,1508650751,SE 1508650752,1508650847,DK @@ -44523,9 +45834,9 @@ 1533505536,1533507583,DE 1533507584,1533509631,UA 1533509632,1533511679,GB -1533511680,1533512383,FR -1533512384,1533512447,DE -1533512448,1533513215,FR +1533511680,1533513023,FR +1533513024,1533513087,ES +1533513088,1533513215,FR 1533513216,1533513471,GB 1533513472,1533513727,DE 1533513728,1533515775,KW @@ -44573,9 +45884,10 @@ 1534714064,1534714079,ES 1534714080,1534714095,FR 1534714096,1534714111,BE -1534714112,1534714119,GB -1534714120,1534714123,BE -1534714124,1534714127,NL +1534714112,1534714115,ES +1534714116,1534714119,DE +1534714120,1534714123,FI +1534714124,1534714127,ES 1534714128,1534714143,BE 1534714144,1534714159,GB 1534714160,1534714259,FR @@ -44634,7 +45946,7 @@ 1534715144,1534715167,FR 1534715168,1534715183,NL 1534715184,1534715199,FR -1534715200,1534715203,GB +1534715200,1534715203,IT 1534715204,1534715207,FR 1534715208,1534715215,BE 1534715216,1534715263,FR @@ -44653,7 +45965,8 @@ 1534715372,1534715375,GB 1534715376,1534715391,ES 1534715392,1534715407,PL -1534715408,1534715423,DE +1534715408,1534715419,DE +1534715420,1534715423,IT 1534715424,1534715439,PL 1534715440,1534715447,ES 1534715448,1534715451,FR @@ -44723,7 +46036,7 @@ 1534716704,1534716735,ES 1534716736,1534716751,FR 1534716752,1534716759,PL -1534716760,1534716763,FR +1534716760,1534716763,DE 1534716764,1534716767,ES 1534716768,1534716823,FR 1534716824,1534716831,GB @@ -44753,17 +46066,20 @@ 1534717324,1534717343,FR 1534717344,1534717359,GB 1534717360,1534717375,PL -1534717376,1534717503,FR -1534717504,1534717535,DE +1534717376,1534717519,FR +1534717520,1534717535,DE 1534717536,1534717551,PL 1534717552,1534717567,CZ 1534717568,1534717583,IE -1534717584,1534717647,FR -1534717648,1534717655,BE -1534717656,1534717659,ES +1534717584,1534717587,CZ +1534717588,1534717591,NL +1534717592,1534717599,IT +1534717600,1534717647,FR +1534717648,1534717659,PL 1534717660,1534717663,FR 1534717664,1534717679,GB -1534717680,1534717711,ES +1534717680,1534717695,ES +1534717696,1534717711,IE 1534717712,1534717727,FR 1534717728,1534717731,GB 1534717732,1534717735,FR @@ -44771,8 +46087,9 @@ 1534717740,1534717743,DE 1534717744,1534717751,FR 1534717752,1534717759,BE -1534717760,1534717775,FR -1534717776,1534717791,NL +1534717760,1534717783,FR +1534717784,1534717787,GB +1534717788,1534717791,DE 1534717792,1534717823,FR 1534717824,1534717855,BE 1534717856,1534717871,NL @@ -44878,8 +46195,10 @@ 1534719984,1534720003,FR 1534720004,1534720007,ES 1534720008,1534720015,PL -1534720016,1534720031,GB -1534720032,1534720047,DE +1534720016,1534720023,ES +1534720024,1534720027,PL +1534720028,1534720031,ES +1534720032,1534720047,FR 1534720048,1534720063,ES 1534720064,1534720079,IT 1534720080,1534720095,GB @@ -44921,8 +46240,7 @@ 1534720672,1534720739,FR 1534720740,1534720743,PL 1534720744,1534720747,NL -1534720748,1534720751,FR -1534720752,1534720767,GB +1534720748,1534720767,FR 1534720768,1534720783,ES 1534720784,1534720791,PL 1534720792,1534720795,IT @@ -44956,7 +46274,7 @@ 1534721116,1534721119,BE 1534721120,1534721135,IT 1534721136,1534721159,FR -1534721160,1534721163,GB +1534721160,1534721163,ES 1534721164,1534721183,FR 1534721184,1534721191,LT 1534721192,1534721195,FR @@ -45203,7 +46521,9 @@ 1536659760,1536659775,BI 1536659776,1536659791,DE 1536659792,1536659823,SD -1536659824,1536659991,DE +1536659824,1536659839,DE +1536659840,1536659903,LB +1536659904,1536659991,DE 1536659992,1536659999,EG 1536660000,1536660007,SO 1536660008,1536660015,DE @@ -45215,11 +46535,10 @@ 1536660736,1536660991,SA 1536660992,1536661247,DE 1536661248,1536661759,GQ -1536661760,1536662015,BJ -1536662016,1536662271,DE +1536661760,1536662015,DE +1536662016,1536662271,LB 1536662272,1536662303,SO -1536662304,1536662319,DE -1536662320,1536662335,LB +1536662304,1536662335,LB 1536662336,1536662367,DE 1536662368,1536662399,SO 1536662400,1536662415,DE @@ -45257,7 +46576,17 @@ 1538859008,1538875391,RU 1538875392,1538883583,RS 1538883584,1538891775,BE -1538891776,1538898175,DE +1538891776,1538897671,DE +1538897672,1538897679,IT +1538897680,1538897687,ES +1538897688,1538897695,RU +1538897696,1538897703,PL +1538897704,1538897711,GR +1538897712,1538897719,PT +1538897720,1538897727,GB +1538897728,1538897735,NL +1538897736,1538897919,FR +1538897920,1538898175,DE 1538898176,1538899967,FR 1538899968,1538904031,SI 1538904032,1538904039,BH @@ -45280,11 +46609,7 @@ 1538981888,1538990079,BY 1538990080,1538998271,CZ 1538998272,1539006463,AD -1539006464,1539006719,AL -1539006720,1539006735,RS -1539006736,1539008511,AL -1539008512,1539009023,RS -1539009024,1539014655,AL +1539006464,1539014655,AL 1539014656,1539022847,RS 1539022848,1539031039,LT 1539031040,1539039231,IT @@ -45414,7 +46739,6 @@ 1539314688,1539315711,RU 1539315712,1539316735,UA 1539316736,1539317759,SE -1539317760,1539318783,RU 1539318784,1539319807,NL 1539319808,1539320831,DE 1539320832,1539321855,UA @@ -45693,7 +47017,8 @@ 1539536384,1539536895,NL 1539536896,1539537407,UA 1539537408,1539537919,GB -1539537920,1539540479,RU +1539537920,1539539455,RU +1539539968,1539540479,RU 1539540480,1539540991,PL 1539540992,1539541503,RU 1539541504,1539542015,TR @@ -45769,9 +47094,7 @@ 1539589120,1539590143,GB 1539590144,1539591167,RU 1539591168,1539592191,UZ -1539592192,1539593215,UA -1539593216,1539594239,IL -1539594240,1539598335,UA +1539592192,1539598335,UA 1539598336,1539599359,NL 1539599360,1539600383,PL 1539600384,1539601407,HU @@ -45883,7 +47206,7 @@ 1539709184,1539709439,AT 1539709440,1539709695,BE 1539709696,1539709951,NL -1539709952,1539710207,DE +1539709952,1539710207,TR 1539710208,1539710463,GB 1539710464,1539710719,FR 1539710720,1539710975,UA @@ -46090,7 +47413,6 @@ 1539764992,1539765247,BE 1539765248,1539765503,GB 1539765504,1539766015,PL -1539766016,1539766271,AM 1539766272,1539766527,PL 1539766528,1539766783,UA 1539766784,1539767039,DE @@ -46436,6 +47758,7 @@ 1539948544,1539949567,RO 1539949568,1539950591,MD 1539950592,1539951103,KZ +1539951104,1539951615,UA 1539951616,1539953663,RU 1539953664,1539954687,UA 1539954688,1539956735,RO @@ -46565,7 +47888,6 @@ 1540118528,1540119551,PL 1540119552,1540120575,UA 1540120576,1540124671,RU -1540124672,1540125695,NO 1540125696,1540126719,FR 1540126720,1540127743,UA 1540127744,1540128767,GB @@ -47413,7 +48735,7 @@ 1540589568,1540593663,RU 1540593664,1540594687,GB 1540594688,1540595711,IT -1540595712,1540596735,UZ +1540595712,1540596735,KZ 1540596736,1540597759,FR 1540597760,1540598783,SE 1540598784,1540600831,UA @@ -47728,6 +49050,7 @@ 1540701952,1540702207,GB 1540702208,1540702463,PL 1540702464,1540702719,PT +1540702720,1540702975,DK 1540702976,1540703231,RU 1540703232,1540703487,MH 1540703488,1540703743,NL @@ -47780,6 +49103,7 @@ 1540717056,1540717311,CZ 1540717312,1540717823,PL 1540717824,1540718079,UA +1540718080,1540718335,RU 1540718592,1540718847,CH 1540718848,1540719103,IT 1540719104,1540719359,RU @@ -47868,6 +49192,7 @@ 1540742912,1540743167,RU 1540743168,1540743423,NO 1540743424,1540743679,RU +1540743680,1540743935,PL 1540743936,1540744191,CH 1540744192,1540744447,GR 1540744448,1540744703,UA @@ -48261,7 +49586,7 @@ 1540962816,1540963839,UA 1540963840,1540964351,RU 1540964352,1540964863,IR -1540964864,1540965375,UA +1540964864,1540965887,UA 1540965888,1540966399,RU 1540966400,1540966911,GB 1540966912,1540967935,RU @@ -49088,6 +50413,85 @@ 1541385216,1541385471,NL 1541385472,1541385727,RU 1541385728,1541385983,SI +1541385984,1541386239,PL +1541386240,1541386495,RU +1541386496,1541386751,UA +1541386752,1541387007,RU +1541387008,1541387263,CH +1541387264,1541387519,UA +1541387520,1541387775,RU +1541387776,1541388031,SI +1541388032,1541388287,RO +1541388288,1541388543,PL +1541388544,1541388799,RO +1541388800,1541389055,CH +1541389056,1541389311,NL +1541389312,1541389567,RU +1541389568,1541389823,SI +1541389824,1541390079,RO +1541390080,1541390335,US +1541390336,1541390591,DK +1541390592,1541390847,SI +1541390848,1541391103,RU +1541391104,1541391359,BE +1541391360,1541391615,PL +1541391616,1541391871,SI +1541391872,1541392127,DE +1541392128,1541392383,RU +1541392384,1541392639,RO +1541392640,1541392895,TR +1541392896,1541393151,DE +1541393152,1541393407,GB +1541393408,1541393663,DE +1541393664,1541393919,UA +1541393920,1541394175,ES +1541394176,1541394431,FR +1541394432,1541394687,CY +1541394688,1541394943,SK +1541394944,1541395199,SA +1541395200,1541395455,DE +1541395456,1541395711,PL +1541395712,1541395967,NO +1541395968,1541396223,UA +1541396224,1541396479,LB +1541396480,1541396735,UA +1541396736,1541396991,RU +1541396992,1541397247,GB +1541397248,1541397503,FR +1541397504,1541397759,NL +1541397760,1541398015,PL +1541398016,1541398271,HU +1541398272,1541398527,UA +1541398528,1541398783,RU +1541398784,1541399039,PL +1541399040,1541399295,RU +1541399296,1541399551,UA +1541399552,1541399807,DE +1541399808,1541400063,RU +1541400064,1541400319,SI +1541400320,1541400831,SE +1541400832,1541401087,NL +1541401088,1541401343,UA +1541401344,1541401599,DK +1541401600,1541401855,LV +1541401856,1541402111,HR +1541402112,1541402367,SI +1541402368,1541402623,RU +1541402624,1541402879,FR +1541402880,1541403135,PL +1541403136,1541403391,RU +1541403392,1541403647,EE +1541403648,1541403903,RU +1541403904,1541404159,GE +1541404160,1541404415,RU +1541404416,1541404671,DE +1541404672,1541404927,AT +1541404928,1541405183,HR +1541405184,1541405439,PL +1541405440,1541405951,RU +1541405952,1541406207,FR +1541406208,1541406463,RU +1541406464,1541406719,AT 1541406720,1541407231,LV 1541407232,1541407743,UA 1541407744,1541408255,FI @@ -49171,6 +50575,32 @@ 1541453824,1541454335,PL 1541454336,1541454847,GB 1541454848,1541455359,EE +1541455360,1541455871,GB +1541455872,1541456383,PL +1541456384,1541456895,RU +1541456896,1541457919,PL +1541457920,1541458943,UA +1541458944,1541459455,RO +1541459456,1541460479,RU +1541460480,1541460991,DK +1541460992,1541461503,RO +1541461504,1541462527,PL +1541462528,1541463039,RU +1541463040,1541463551,PL +1541463552,1541464063,UA +1541464064,1541464575,PL +1541464576,1541465087,DE +1541465088,1541465599,RU +1541465600,1541466111,FR +1541466112,1541467135,RU +1541467136,1541467647,PL +1541467648,1541468159,SI +1541468160,1541468671,ES +1541469184,1541469695,SE +1541469696,1541470207,LV +1541470208,1541470719,NL +1541470720,1541471231,RS +1541471232,1541472255,UA 1541472256,1541473279,GB 1541473280,1541474303,UA 1541474304,1541475327,PL @@ -49216,6 +50646,8 @@ 1541534720,1541535743,IT 1541535744,1541536767,US 1541536768,1541537791,RU +1541537792,1541538303,GB +1541538304,1541538815,RO 1543503872,1545601023,GB 1545601024,1545673167,SE 1545673168,1545673175,FI @@ -49350,9 +50782,7 @@ 1546649600,1546665983,SA 1546665984,1546673823,GB 1546673824,1546673839,RS -1546673840,1546677559,GB -1546677560,1546677561,IN -1546677562,1546678863,GB +1546673840,1546678863,GB 1546678864,1546678871,RS 1546678872,1546680831,GB 1546680832,1546680895,RS @@ -49587,17 +51017,19 @@ 1551548416,1551551487,DE 1551551488,1551556607,EU 1551556608,1551558655,FR -1551558656,1551564799,EU +1551558656,1551560703,EU +1551560704,1551561727,DE +1551561728,1551564799,EU 1551564800,1551572991,FR 1551572992,1551576063,NL 1551576064,1551577087,EU 1551577088,1551580159,NL -1551580160,1551630335,EU +1551580160,1551604479,EU +1551604480,1551604735,SE +1551604736,1551630335,EU 1551630336,1551892479,RU 1551892480,1556086783,FR -1556086784,1556486729,DE -1556486730,1556486730,A2 -1556486731,1556486924,DE +1556086784,1556486924,DE 1556486925,1556486925,A2 1556486926,1558708223,DE 1558708224,1559232511,GB @@ -49692,6 +51124,7 @@ 1566081024,1566085119,RU 1566085120,1566089215,NL 1566089216,1566097407,RU +1566097408,1566101503,HU 1566101504,1566105599,RU 1566105600,1566109695,DE 1566109696,1566111231,CH @@ -49984,7 +51417,9 @@ 1568309248,1568342015,RO 1568342016,1568374783,BG 1568374784,1568440319,RU -1568440320,1568473087,NO +1568440320,1568443919,NO +1568443920,1568443927,SE +1568443928,1568473087,NO 1568473088,1568505855,BY 1568505856,1568538623,NL 1568538624,1568555007,IR @@ -50122,7 +51557,8 @@ 1570667296,1570667327,GB 1570667328,1570667335,SE 1570667336,1570667339,GB -1570667340,1570667455,SE +1570667340,1570667343,NL +1570667344,1570667455,SE 1570667456,1570667471,FR 1570667472,1570667487,SE 1570667488,1570667503,FR @@ -50171,9 +51607,8 @@ 1571426304,1571428351,CZ 1571428352,1571428863,UA 1571428864,1571429119,KZ -1571429120,1571429375,CZ -1571429376,1571429631,RU -1571429632,1571432447,UA +1571429120,1571429375,RU +1571429376,1571432447,UA 1571432448,1571434495,CZ 1571434496,1571435519,UA 1571435520,1571438591,CZ @@ -50359,6 +51794,7 @@ 1572542464,1572544511,IT 1572544512,1572546559,IQ 1572546560,1572548607,FR +1572548608,1572550655,NL 1572550656,1572552703,DE 1572552704,1572554751,TR 1572554752,1572556799,CH @@ -50368,7 +51804,9 @@ 1572562944,1572564991,CZ 1572564992,1572567039,DE 1572567040,1572569087,RU -1572569088,1572571135,NL +1572569088,1572569343,NL +1572569344,1572569599,GB +1572569600,1572571135,NL 1572571136,1572573183,RO 1572573184,1572574975,GG 1572574976,1572575167,GB @@ -50430,7 +51868,11 @@ 1572668464,1572668471,SA 1572668472,1572668639,GB 1572668640,1572668647,SA -1572668648,1572669439,GB +1572668648,1572668655,GB +1572668656,1572668671,CY +1572668672,1572668703,GB +1572668704,1572668711,CY +1572668712,1572669439,GB 1572669440,1572673535,RU 1572673536,1572675583,AT 1572675584,1572677631,ES @@ -50507,7 +51949,9 @@ 1572800512,1572804607,RU 1572804608,1572808703,GB 1572808704,1572810751,FR -1572810752,1572812799,DE +1572810752,1572812031,DE +1572812032,1572812543,RO +1572812544,1572812799,DE 1572812800,1572814847,RU 1572814848,1572816895,KW 1572816896,1572818943,RU @@ -50545,7 +51989,9 @@ 1572854286,1572854286,CH 1572854287,1572854287,LU 1572854288,1572854527,BE -1572854528,1572855295,EU +1572854528,1572855039,EU +1572855040,1572855071,BE +1572855072,1572855295,EU 1572855296,1572855551,CH 1572855552,1572855552,EU 1572855553,1572855558,BE @@ -50670,13 +52116,12 @@ 1578591696,1578591699,DE 1578591700,1578591703,FR 1578591704,1578591707,DE -1578591708,1578591711,CZ +1578591708,1578591711,FR 1578591712,1578591743,PL 1578591744,1578591759,ES 1578591760,1578591791,FR 1578591792,1578591807,PL -1578591808,1578591823,DE -1578591824,1578591887,FR +1578591808,1578591887,FR 1578591888,1578591891,DE 1578591892,1578591899,FR 1578591900,1578591903,GB @@ -50733,8 +52178,9 @@ 1578592688,1578592695,DE 1578592696,1578592719,ES 1578592720,1578592735,CH -1578592736,1578592751,FR -1578592752,1578592783,DE +1578592736,1578592743,PL +1578592744,1578592747,PT +1578592748,1578592783,DE 1578592784,1578592799,FR 1578592800,1578592815,GB 1578592816,1578592831,FR @@ -50817,7 +52263,8 @@ 1578594112,1578594127,PL 1578594128,1578594159,FR 1578594160,1578594163,PT -1578594164,1578594171,GB +1578594164,1578594167,ES +1578594168,1578594171,DE 1578594172,1578594191,FR 1578594192,1578594207,IT 1578594208,1578594211,DE @@ -51104,7 +52551,8 @@ 1578991616,1579024383,KW 1579024384,1579057151,GB 1579057152,1579089919,LV -1579089920,1579090575,NL +1579089920,1579090175,GB +1579090176,1579090575,NL 1579090576,1579090687,GB 1579090688,1579090943,NL 1579090944,1579091759,GB @@ -51206,7 +52654,9 @@ 1583681536,1583685631,PL 1583685632,1583689727,DE 1583689728,1583693823,IT -1583693824,1583697919,RU +1583693824,1583693871,RU +1583693872,1583693879,A2 +1583693880,1583697919,RU 1583697920,1583702015,TR 1583702016,1583706111,RU 1583706112,1583710207,GI @@ -51247,8 +52697,10 @@ 1583804416,1583808511,GB 1583808512,1583812607,MD 1583812608,1583813171,NL -1583813172,1583813175,US -1583813176,1583813663,NL +1583813172,1583813183,US +1583813184,1583813199,NL +1583813200,1583813215,RU +1583813216,1583813663,NL 1583813664,1583813671,US 1583813672,1583813679,NL 1583813680,1583813683,GB @@ -51258,7 +52710,9 @@ 1583813736,1583813743,DE 1583813744,1583813808,NL 1583813809,1583813815,US -1583813816,1583815167,NL +1583813816,1583814655,NL +1583814656,1583814783,US +1583814784,1583815167,NL 1583815168,1583815215,US 1583815216,1583815343,NL 1583815344,1583815351,US @@ -51488,7 +52942,9 @@ 1586216960,1586225151,BG 1586225152,1586233343,RU 1586233344,1586241535,FR -1586241536,1586249727,SE +1586241536,1586243779,SE +1586243780,1586243787,NO +1586243788,1586249727,SE 1586249728,1586257919,SI 1586257920,1586266111,EU 1586266112,1586274303,RU @@ -51665,8 +53121,12 @@ 1590149120,1590151167,CZ 1590151168,1590153215,TR 1590153216,1590157311,RU -1590157312,1590159359,IE -1590159360,1590161407,GB +1590157312,1590157471,IE +1590157472,1590157823,GB +1590157824,1590157951,IE +1590157952,1590158335,GB +1590158336,1590158463,IE +1590158464,1590161407,GB 1590161408,1590163455,DE 1590163456,1590165503,NL 1590165504,1590176831,AE @@ -51692,15 +53152,15 @@ 1592054272,1592054527,AE 1592054528,1592054783,NL 1592054784,1592055295,AE -1592055296,1592056959,NL -1592056960,1592057023,SG -1592057024,1592057855,NL +1592055296,1592057855,NL 1592057856,1592061951,RS 1592061952,1592066047,RU 1592066048,1592067583,US 1592067584,1592067711,NO -1592067712,1592069295,CY -1592069296,1592069375,RU +1592067712,1592069119,CY +1592069120,1592069135,RU +1592069136,1592069247,CY +1592069248,1592069375,RU 1592069376,1592069407,US 1592069408,1592069631,CY 1592069632,1592069759,NL @@ -51754,7 +53214,6 @@ 1592281088,1592283135,UA 1592283136,1592285183,GR 1592285184,1592287231,RU -1592287232,1592289279,RO 1592289280,1592291327,RU 1592291328,1592293375,NL 1592293376,1592295423,LT @@ -51961,7 +53420,9 @@ 1600442368,1600446463,DE 1600446464,1600453119,EU 1600453120,1600453631,IT -1600453632,1600520191,EU +1600453632,1600456703,EU +1600456704,1600457727,DE +1600457728,1600520191,EU 1600520192,1600651263,SK 1600651264,1600684031,GE 1600684032,1600749567,RU @@ -51985,7 +53446,8 @@ 1601699840,1602224127,ES 1602224128,1602226175,NL 1602226176,1602228223,GB -1602228224,1602230271,DE +1602228224,1602230270,FR +1602230271,1602230271,DE 1602230272,1602232319,DK 1602232320,1602234367,CH 1602234368,1602234591,FR @@ -52018,7 +53480,11 @@ 1602260992,1602263039,FR 1602263040,1602265087,ES 1602265088,1602267135,RU -1602267136,1602269183,RO +1602267136,1602267199,DE +1602267200,1602267263,FR +1602267264,1602267327,GB +1602267328,1602267391,ES +1602267392,1602269183,RO 1602269184,1602271231,RU 1602271232,1602273279,MK 1602273280,1602273535,FR @@ -52044,7 +53510,7 @@ 1602298368,1602298431,MT 1602298432,1602298879,IL 1602298880,1602299391,MT -1602299392,1602299903,GG +1602299392,1602299903,GB 1602299904,1602301951,DK 1602301952,1602303999,DE 1602304000,1602306047,RU @@ -52131,8 +53597,8 @@ 1602451456,1602453503,DE 1602453504,1602455551,SK 1602455552,1602456015,FR -1602456016,1602456023,ES -1602456024,1602456175,FR +1602456016,1602456031,ES +1602456032,1602456175,FR 1602456176,1602456183,ES 1602456184,1602457599,FR 1602457600,1602459647,RU @@ -52172,7 +53638,9 @@ 1603067904,1603071999,GB 1603072000,1603076095,CZ 1603076096,1603080191,RU -1603080192,1603084287,DE +1603080192,1603083007,DE +1603083008,1603083263,UA +1603083264,1603084287,DE 1603084288,1603088383,IT 1603088384,1603092479,LB 1603092480,1603100671,NO @@ -52250,11 +53718,17 @@ 1603223952,1603224319,FR 1603224320,1603224335,GB 1603224336,1603224351,BE -1603224352,1603224575,GB +1603224352,1603224367,NL +1603224368,1603224383,DE +1603224384,1603224399,CH +1603224400,1603224415,IT +1603224416,1603224431,ES +1603224432,1603224447,US +1603224448,1603224575,GB 1603224576,1603224775,FR 1603224776,1603224783,GB -1603224784,1603224831,FR -1603224832,1603225087,GB +1603224784,1603224895,FR +1603224896,1603225087,GB 1603225088,1603225599,FR 1603225600,1603225607,ES 1603225608,1603225615,GB @@ -52263,8 +53737,8 @@ 1603225632,1603225639,US 1603225640,1603225647,CN 1603225648,1603225855,FR -1603225856,1603226623,GB -1603226624,1603227647,FR +1603225856,1603226111,GB +1603226112,1603227647,FR 1603227648,1603231743,AT 1603231744,1603235839,IT 1603235840,1603239935,RU @@ -52350,11 +53824,7 @@ 1604891968,1604892159,RU 1604892160,1604892927,DE 1604892928,1604893183,HK -1604893184,1604893311,DE -1604893312,1604893375,TR -1604893376,1604893439,DE -1604893440,1604893695,TR -1604893696,1604893951,DE +1604893184,1604893951,DE 1604893952,1604894463,TR 1604894464,1604894719,DE 1604894720,1604895487,CZ @@ -52398,23 +53868,35 @@ 1605107712,1605115903,GB 1605115904,1605124095,RU 1605124096,1605124607,US -1605124608,1605124671,GB -1605124672,1605124735,US -1605124736,1605125263,GB +1605124608,1605124639,GB +1605124640,1605124735,US +1605124736,1605124863,GB +1605124864,1605124895,US +1605124896,1605124927,GB +1605124928,1605124959,US +1605124960,1605125119,GB +1605125120,1605125247,US +1605125248,1605125263,GB 1605125264,1605125279,US 1605125280,1605125335,GB 1605125336,1605125343,US -1605125344,1605125439,GB -1605125440,1605125887,US +1605125344,1605125375,GB +1605125376,1605125887,US 1605125888,1605126143,GB 1605126144,1605127679,US -1605127680,1605130239,GB +1605127680,1605127935,GB +1605127936,1605128703,US +1605128704,1605129215,GB +1605129216,1605129727,US +1605129728,1605130239,GB 1605130240,1605130271,US 1605130272,1605130295,GB -1605130296,1605130303,US -1605130304,1605130495,GB +1605130296,1605130335,US +1605130336,1605130495,GB 1605130496,1605131007,US -1605131008,1605132287,GB +1605131008,1605131263,GB +1605131264,1605132031,US +1605132032,1605132287,GB 1605132288,1605148671,RU 1605148672,1605156863,PT 1605156864,1605165055,JO @@ -52617,10 +54099,12 @@ 1613557008,1613557015,US 1613557016,1613557087,CA 1613557088,1613557095,US -1613557096,1613565951,CA +1613557096,1613558359,CA +1613558360,1613558375,NL +1613558376,1613565951,CA 1613565952,1613570047,US 1613570048,1613574143,CA -1613574144,1613602815,US +1613574144,1613606911,US 1613606912,1613607167,CA 1613607168,1613607423,US 1613607424,1613607679,GB @@ -53402,7 +54886,6 @@ 1674051584,1674575871,US 1674575872,1677721599,CA 1677721600,1681915903,US -1694499072,1694499327,AU 1694499840,1694500863,ID 1694500864,1694507007,JP 1694507008,1694515199,IN @@ -53423,7 +54906,8 @@ 1694760960,1695023103,CN 1695023104,1695547391,TW 1695547392,1697775615,CN -1697775616,1697779711,JP +1697775616,1697776639,ID +1697776640,1697779711,JP 1697779712,1697783807,ID 1697783808,1697791999,JP 1697792000,1697808383,PK @@ -53431,6 +54915,7 @@ 1697841152,1697906687,TH 1697906688,1697972223,CN 1697972224,1697988607,VN +1697988608,1697996799,KR 1697996800,1698004991,JP 1698004992,1698037759,AU 1698037760,1698103295,CN @@ -53441,15 +54926,18 @@ 1699611648,1699612671,JP 1699612672,1699614719,LA 1699614720,1699618815,PH +1699618816,1699627007,CN 1699627008,1699643391,SG 1699643392,1699676159,HK 1699676160,1699741695,KR 1699741696,1700790271,CN -1700790272,1700823039,VN +1700793344,1700794367,VN +1700798464,1700806655,JP +1700806656,1700823039,VN 1700823040,1700855807,CN -1700855808,1700921343,AU 1700921344,1700986879,NZ 1700986880,1701003263,VN +1701003264,1701011455,MY 1701019648,1701052415,GU 1701052416,1701117951,NZ 1701117952,1701134335,NC @@ -53461,10 +54949,12 @@ 1701576704,1701707775,TH 1701707776,1701724159,JP 1701736448,1701737471,NZ -1701773312,1701838847,JP +1701740544,1701838847,JP 1701838848,1702363135,AU -1702363136,1702821887,CN -1702821888,1702889471,AU +1702363136,1702887423,CN +1702888448,1702889471,AU +1702903808,1702920191,ID +1702920192,1702952959,JP 1702952960,1703411711,CN 1703411712,1703673855,TW 1703673856,1703935999,JP @@ -53472,14 +54962,38 @@ 1704984576,1707081727,AU 1707081728,1707737087,CN 1707737088,1707802623,KR +1707802624,1707835391,JP 1707845632,1707846655,ID +1707851776,1707868159,JP 1707868160,1708130303,CN 1708130304,1709178879,IN 1709178880,1709834239,CN +1709834240,1709850623,SG +1709852672,1709853695,AU +1709867008,1709899775,AU 1709899776,1709965311,KR 1709965312,1710882815,CN 1710882816,1710948351,KR -1711013888,1711276031,CN +1710949376,1710950399,NP +1710964736,1711210495,CN +1711210496,1711276031,ID +1778385152,1778385407,AU +1778417664,1778450431,TH +1778450432,1778515967,TW +1778515968,1779040255,CN +1779073024,1779105791,SG +1779105792,1781727231,CN +1781727232,1781792767,IN +1782054912,1782579199,CN +1782972416,1783103487,AU +1783234560,1783365631,CN +1783627776,1784676351,CN +1784676352,1785200639,KR +1785724928,1786249215,CN +1786773504,1790967807,JP +1790967808,1793064959,IN +1793064960,1794113535,CN +1794113536,1795162111,KR 1795162112,1815822335,US 1815822336,1815826431,CA 1815826432,1815871487,US @@ -53845,7 +55359,9 @@ 1835892736,1835909119,GB 1835909120,1835913215,RS 1835913216,1835917311,RU -1835917312,1835925503,GB +1835917312,1835920127,GB +1835920128,1835920143,PT +1835920144,1835925503,GB 1835925504,1835933695,LV 1835933696,1835941887,RU 1835941888,1835950079,UA @@ -54093,8 +55609,8 @@ 1842206720,1842208767,SE 1842208768,1842210815,GB 1842210816,1842212863,LT -1842212864,1842214911,CZ -1842214912,1842216959,RU +1842212864,1842214655,CZ +1842214656,1842216959,RU 1842216960,1842225151,RO 1842225152,1842233343,UA 1842233344,1842241535,RO @@ -54198,8 +55714,8 @@ 1844011008,1844013055,IT 1844013056,1844015103,PL 1844015104,1844017151,GB -1844017152,1844019199,IT -1844019200,1844027391,DE +1844017152,1844021247,IT +1844021248,1844027391,DE 1844027392,1844029439,CZ 1844029440,1844031487,RU 1844031488,1844031743,EU @@ -54207,8 +55723,8 @@ 1844031776,1844031999,EU 1844032000,1844032031,GB 1844032032,1844032255,EU -1844032256,1844032767,CH -1844032768,1844033271,EU +1844032256,1844032831,CH +1844032832,1844033271,EU 1844033272,1844033535,CH 1844033536,1844035583,RS 1844035584,1844037631,MD @@ -54226,7 +55742,8 @@ 1844068352,1844072447,GB 1844072448,1844076543,ES 1844076544,1844078591,IE -1844078592,1844080639,DE +1844078592,1844080127,DE +1844080128,1844080639,PL 1844080640,1844082687,GE 1844082688,1844084735,DE 1844084736,1844086783,RU @@ -54263,10 +55780,7 @@ 1844148224,1844150271,GB 1844150272,1844152319,ES 1844152320,1844154367,DE -1844154368,1844154879,GB -1844154880,1844155135,FR -1844155136,1844155391,US -1844155392,1844156415,GB +1844154368,1844156415,GB 1844156416,1844158463,IT 1844158464,1844160511,RU 1844160512,1844162559,SE @@ -54280,9 +55794,14 @@ 1844169600,1844169647,US 1844169648,1844169655,AF 1844169656,1844169727,SE -1844169728,1844169759,DE -1844169760,1844169943,US -1844169944,1844170751,DE +1844169728,1844169767,DE +1844169768,1844169951,US +1844169952,1844169983,DE +1844169984,1844169987,AF +1844169988,1844169991,DE +1844169992,1844169995,AF +1844169996,1844169999,TM +1844170000,1844170751,DE 1844170752,1844174847,RU 1844174848,1844178943,DE 1844178944,1844180991,EE @@ -54322,9 +55841,11 @@ 1844215808,1844219903,BE 1844219904,1844220031,A2 1844220032,1844220191,IQ -1844220192,1844220415,DE +1844220192,1844220287,DE +1844220288,1844220415,JO 1844220416,1844220431,A2 -1844220432,1844223999,DE +1844220432,1844223743,DE +1844223744,1844223999,A2 1844224000,1844228095,GB 1844228096,1844228479,DK 1844228480,1844228511,SE @@ -54437,6 +55958,7 @@ 1847787520,1847803903,KR 1847803904,1847807999,VN 1847808000,1847810047,ID +1847810048,1847812095,FJ 1847812096,1847853055,KR 1847853056,1848115199,PK 1848115200,1848377343,CN @@ -54448,6 +55970,7 @@ 1848401920,1848406015,PH 1848406016,1848410111,NP 1848410112,1848414207,PH +1848414208,1848418303,CN 1848418304,1848420351,AU 1848420352,1848422399,ID 1848422400,1848424447,JP @@ -54457,6 +55980,7 @@ 1848705024,1848770559,TH 1848770560,1848774655,JP 1848774656,1848776703,AU +1848776704,1848778751,JP 1848778752,1848786943,IN 1848786944,1848791039,JP 1848791040,1848793087,ID @@ -54536,6 +56060,7 @@ 1856856064,1856864255,CN 1856864256,1856872447,AU 1856872448,1856876543,NZ +1856876544,1856880639,IN 1856880640,1856888831,CN 1856888832,1856892927,AU 1856892928,1856897023,JP @@ -54560,6 +56085,7 @@ 1860746240,1860747263,PK 1860747264,1860759551,JP 1860759552,1860761599,AU +1860761600,1860763647,IN 1860763648,1860829183,JP 1860829184,1860960255,IN 1860960256,1861091327,CN @@ -54570,6 +56096,7 @@ 1866563584,1866579967,KR 1866579968,1866588159,JP 1866588160,1866592255,NZ +1866592256,1866596351,VN 1866596352,1866661887,CN 1866661888,1866670079,AU 1866670080,1866674175,MY @@ -54629,9 +56156,11 @@ 1870045184,1870049279,AU 1870049280,1870053375,IN 1870053376,1870055423,AU +1870055424,1870057471,CN 1870057472,1870058495,AU 1870058496,1870059519,IN 1870059520,1870065663,AU +1870065664,1870069759,IN 1870069760,1870077951,JP 1870077952,1870086143,NP 1870086144,1870110719,CN @@ -54644,7 +56173,7 @@ 1870497792,1870499839,IN 1870499840,1870501887,JP 1870501888,1870503935,AF -1870503936,1870508031,AU +1870503936,1870512127,AU 1870512128,1870528511,IN 1870528512,1873281023,CN 1873281024,1873412095,JP @@ -54727,8 +56256,10 @@ 1885995008,1886191615,KR 1886191616,1886195711,PH 1886195712,1886197759,ID +1886197760,1886199807,JP 1886199808,1886207999,KR -1886208000,1886216191,NZ +1886208000,1886214143,NZ +1886214144,1886216191,VN 1886216192,1886224383,IN 1886224384,1886257151,CN 1886257152,1886322687,IN @@ -54777,6 +56308,7 @@ 1892155392,1892941823,PH 1892941824,1893015551,KR 1893015552,1893019647,AU +1893019648,1893023743,TW 1893023744,1893027839,HK 1893027840,1893031935,VN 1893031936,1893040127,HK @@ -54799,6 +56331,7 @@ 1896605696,1896606719,AU 1896606720,1896607743,MY 1896607744,1896609791,VU +1896609792,1896611839,SG 1896611840,1897070591,CN 1897070592,1897136127,IN 1897136128,1897140223,AU @@ -54849,6 +56382,7 @@ 1897922560,1898708991,JP 1898708992,1899233279,CN 1899233280,1899237375,AU +1899237376,1899241471,JP 1899241472,1899249663,VN 1899249664,1899266047,MO 1899266048,1899267071,AU @@ -54861,6 +56395,7 @@ 1899274240,1899282431,CN 1899282432,1899290623,KR 1899290624,1899294719,AU +1899294720,1899298815,JP 1899298816,1899364351,TH 1899364352,1899724799,CN 1899724800,1899741183,KR @@ -54892,8 +56427,9 @@ 1908539392,1908670463,CN 1908670464,1908735999,TW 1908736000,1908740095,AU -1908744192,1908748287,JP +1908740096,1908748287,JP 1908748288,1908750335,PK +1908750336,1908752383,JP 1908752384,1908753407,NZ 1908753408,1908754431,PF 1908754432,1908756479,PH @@ -54954,6 +56490,7 @@ 1914652672,1914654719,AU 1914654720,1914658815,JP 1914658816,1914660863,AU +1914660864,1914662911,JP 1914662912,1914667007,KR 1914667008,1914683391,IN 1914683392,1914687487,AU @@ -54983,6 +56520,7 @@ 1919815680,1919817727,ID 1919817728,1919819775,KR 1919819776,1919821823,NZ +1919821824,1919823871,ID 1919823872,1919827967,JP 1919827968,1919844351,CN 1919844352,1919877119,KR @@ -55009,12 +56547,14 @@ 1921253376,1921318911,CN 1921318912,1921384447,MY 1921384448,1921388543,NZ +1921388544,1921392639,PG 1921392640,1921400831,JP 1921400832,1921404927,ID 1921404928,1921406975,HK 1921406976,1921409023,BD 1921409024,1921425407,JP -1921425408,1921433599,NZ +1921425408,1921431551,NZ +1921431552,1921433599,KH 1921433600,1921449983,JP 1921449984,1921515519,CN 1921515520,1921646591,TW @@ -55029,6 +56569,7 @@ 1921863680,1921865727,BD 1921865728,1921867775,ID 1921867776,1921871871,AU +1921871872,1921875967,NZ 1921875968,1921892351,CN 1921892352,1921896447,AU 1921896448,1921898495,SG @@ -55080,6 +56621,7 @@ 1932001280,1932132351,KR 1932132352,1932148735,AU 1932148736,1932152831,PK +1932152832,1932156927,TW 1932156928,1932161023,JP 1932161024,1932163071,TW 1932163072,1932165119,PH @@ -55118,6 +56660,7 @@ 1934966784,1934974975,ID 1934974976,1934983167,JP 1934983168,1934985215,AU +1934985216,1934987263,JP 1934987264,1934991359,TW 1934991360,1934999551,KR 1934999552,1935015935,CN @@ -55128,9 +56671,11 @@ 1935933440,1936457727,CN 1936457728,1937244159,IN 1937244160,1937506303,CN -1937506304,1937508351,AU +1937506304,1937510399,AU 1937510400,1937514495,CN 1937514496,1937516543,AU +1937516544,1937518591,IN +1937518592,1937522687,JP 1937522688,1937530879,ID 1937530880,1937532927,US 1937532928,1937534463,IN @@ -55140,7 +56685,7 @@ 1937539072,1937637375,JP 1937637376,1937670143,HK 1937670144,1937672191,NZ -1937674240,1937678335,JP +1937672192,1937678335,JP 1937678336,1937686527,NC 1937686528,1937702911,KR 1937702912,1937768447,BD @@ -55148,7 +56693,7 @@ 1938030592,1938292735,MY 1938292736,1938948095,KR 1938948096,1938964479,JP -1938964480,1938968575,AU +1938964480,1938972671,AU 1938972672,1938976767,MY 1938976768,1938978815,SG 1938978816,1938980863,VN @@ -55883,7 +57428,7 @@ 2013048832,2013052927,AU 2013052928,2013061119,IN 2013061120,2013065215,PG -2013069312,2014314495,CN +2013065216,2014314495,CN 2014314496,2014838783,AU 2014838784,2015100927,CN 2015100928,2015166463,PH @@ -55908,7 +57453,7 @@ 2016550912,2016555007,SG 2016555008,2016559103,MY 2016559104,2016583679,KR -2016583680,2016585727,JP +2016583680,2016587775,JP 2016587776,2016589823,BD 2016589824,2016591871,VN 2016591872,2016673791,JP @@ -55930,12 +57475,15 @@ 2019035136,2019037183,CN 2019037184,2019041279,JP 2019041280,2019045375,IN -2019049472,2019082239,AU +2019045376,2019049471,JP +2019049472,2019078143,AU +2019078144,2019082239,IN 2019082240,2019098623,HK 2019098624,2019115007,PH 2019115008,2019117055,US 2019117056,2019119103,IN 2019119104,2019121151,NZ +2019121152,2019123199,ID 2019123200,2019131391,NP 2019131392,2019164159,JP 2019164160,2019360767,CN @@ -55944,6 +57492,7 @@ 2019557376,2021654527,TW 2021654528,2022178815,CN 2022178816,2022180863,NZ +2022180864,2022182911,JP 2022182912,2022184959,ID 2022184960,2022187007,KH 2022187008,2022187071,US @@ -57700,7 +59249,7 @@ 2304573440,2304638975,NO 2304638976,2304704511,CA 2304704512,2304770047,US -2304770048,2304835583,SE +2304770048,2304835583,FI 2304835584,2305097727,US 2305097728,2305163263,GB 2305163264,2305359871,US @@ -57899,8 +59448,9 @@ 2331770880,2331836415,AU 2331836416,2331901951,GB 2331901952,2331967487,US -2332033024,2332098559,AU +2332033024,2332098559,ID 2332098560,2332622847,DE +2332622848,2332688383,CN 2332688384,2332753919,NL 2332753920,2333868031,DE 2333933568,2334064639,DE @@ -57919,7 +59469,7 @@ 2335899648,2335965183,AU 2335965184,2336161791,US 2336161792,2336227327,NL -2336227328,2336358399,US +2336292864,2336358399,US 2336358400,2336423935,FI 2336423936,2336882687,US 2336882688,2336948223,FI @@ -57957,6 +59507,7 @@ 2338455552,2338521087,NO 2338521088,2338586623,US 2338586624,2338652159,FR +2338652160,2338717695,JP 2338717696,2338783231,US 2338783232,2338848767,CA 2338848768,2338914303,US @@ -57967,6 +59518,7 @@ 2340159488,2340225023,FR 2340225024,2340421631,US 2340421632,2340487167,IT +2340487168,2340552703,CN 2340552704,2340618239,AU 2340618240,2340683775,US 2340683776,2340749311,AU @@ -57979,14 +59531,18 @@ 2341470208,2341535743,US 2341535744,2341601279,NO 2341601280,2341732351,US +2341732352,2341797887,CN 2341797888,2341863423,GB +2341863424,2341928959,KR 2341928960,2341994495,US 2341994496,2342060031,JP 2342060032,2342125567,GB 2342125568,2342191103,JP +2342191104,2342256639,CN 2342256640,2342322175,NL 2342322176,2342387711,FI 2342387712,2342453247,FR +2342453248,2342518783,CN 2342518784,2342584319,FR 2342584320,2342649855,US 2342649856,2342715391,NL @@ -57997,10 +59553,12 @@ 2342977536,2343043071,US 2343043072,2343108607,AU 2343108608,2343174143,US +2343174144,2343239679,CN 2343239680,2343370751,US 2343370752,2343436287,CA 2343436288,2343501823,DE 2343501824,2343567359,TW +2343567360,2343632895,CN 2343632896,2343698431,US 2343698432,2343763967,NL 2343763968,2343829503,TR @@ -58009,19 +59567,32 @@ 2343925760,2343934975,US 2343934976,2343935999,IL 2343936000,2344026111,US +2344026112,2344091647,CN 2344091648,2344157183,GB 2344157184,2344222719,US +2344222720,2344288255,CN 2344288256,2344353791,US 2344353792,2344419327,AU +2344419328,2344484863,CN +2344484864,2344550399,PK 2344550400,2344615935,EU +2344615936,2344878079,ID +2344878080,2346188799,CN +2346188800,2346450943,AU +2346450944,2346582015,CN 2346582016,2346647551,GB 2346647552,2346713087,TW +2346713088,2346778623,CN 2346778624,2346844159,US +2346844160,2346975231,CN +2346975232,2347040767,ID 2347040768,2347106303,US 2347106304,2347171839,AU -2347171840,2348810239,US -2348810240,2348875775,AU -2348875776,2353856511,US +2347171840,2348744703,US +2348744704,2348875775,ID +2348875776,2353725439,US +2353725440,2353790975,CN +2353790976,2353856511,US 2353856512,2353922047,FR 2353922048,2353987583,AT 2353987584,2354053119,AU @@ -58045,8 +59616,11 @@ 2355691520,2355757055,IT 2355757056,2355953663,US 2355953664,2357919743,TW -2357919744,2358509567,US +2357919744,2358181887,US +2358181888,2358247423,CN +2358247424,2358509567,US 2358509568,2358575103,MX +2358575104,2358640639,TH 2358640640,2358706175,SE 2358706176,2358771711,FI 2358771712,2359230463,US @@ -58072,9 +59646,19 @@ 2361982976,2362114047,US 2362114048,2362179583,IE 2362179584,2362245119,GB -2362441728,2363883519,US +2362245120,2362441727,CN +2362441728,2362572799,US +2362572800,2362638335,CN +2362638336,2362769407,US +2362769408,2362834943,ID +2362834944,2363490303,US +2363490304,2363555839,CN +2363555840,2363883519,US 2363883520,2363949055,CA -2363949056,2364538879,US +2363949056,2364342271,US +2364342272,2364407807,CN +2364407808,2364538879,US +2364538880,2364604415,CN 2364604416,2364671487,US 2364671488,2364671743,CN 2364671744,2364675839,US @@ -58082,9 +59666,15 @@ 2364676096,2364676863,US 2364676864,2364677119,GB 2364677120,2364735487,US -2364801024,2365390847,US +2364735488,2364801023,CN +2364801024,2364932095,US +2364932096,2364997631,CN +2364997632,2365128703,US +2365128704,2365259775,CN +2365259776,2365390847,US 2365390848,2365456383,AU 2365456384,2365521919,US +2365521920,2365587455,CN 2365587456,2366111743,DE 2366177280,2366308351,DE 2366308352,2366373887,GB @@ -58295,14 +59885,17 @@ 2415656960,2415722495,GB 2415722496,2415788031,JP 2415788032,2415853567,US -2415919104,2415984639,AU +2415919104,2415984639,CN 2415984640,2416050175,JP 2416050176,2416115711,NL 2416115712,2416181247,US 2416181248,2416246783,FI 2416246784,2416312319,US 2416312320,2416377855,AU -2416443392,2416967679,US +2416377856,2416443391,CN +2416443392,2416705535,US +2416705536,2416771071,CN +2416771072,2416967679,US 2416967680,2417033215,IN 2417033216,2417229823,US 2417229824,2417295359,ES @@ -58320,6 +59913,7 @@ 2418737152,2418802687,NL 2418802688,2418868223,EU 2418868224,2419326975,US +2419326976,2419392511,CN 2419392512,2419458047,AU 2419458048,2419523583,NL 2419523584,2419589119,AU @@ -58359,6 +59953,7 @@ 2423783424,2423848959,FJ 2423848960,2423914495,US 2423914496,2423980031,TR +2423980032,2424045567,CN 2424045568,2424111103,GB 2424111104,2424242175,US 2424242176,2424307711,NO @@ -58394,7 +59989,8 @@ 2430205952,2430271487,AU 2430271488,2432172031,US 2432172032,2432237567,BE -2432237568,2432696319,US +2432237568,2432630783,US +2432630784,2432696319,CN 2432696320,2433810431,NL 2433810432,2433875967,GB 2433875968,2436300799,NL @@ -58833,17 +60429,23 @@ 2516254720,2516320255,FR 2516320256,2516451327,US 2516451328,2516516863,GB -2516582400,2516647935,AU +2516582400,2516647935,CN 2516647936,2523201535,JP 2523201536,2523267071,AU 2523267072,2523529215,US 2523529216,2523594751,NO -2523660288,2524971007,US +2523660288,2524119039,US +2524119040,2524184575,CN +2524184576,2524315647,TW +2524315648,2524512255,US +2524512256,2524643327,CN +2524643328,2524971007,US 2524971008,2525036543,ES 2525036544,2525102079,EU 2525102080,2525233151,US 2525233152,2525298687,SE 2525298688,2525626367,US +2525626368,2525757439,CN 2525757440,2525822975,GR 2525822976,2526085119,US 2526085120,2526216191,IT @@ -58879,7 +60481,9 @@ 2530607104,2530672639,ES 2530672640,2530803711,US 2530803712,2530869247,IT -2530869248,2531459071,US +2530869248,2531196927,US +2531196928,2531262463,CN +2531262464,2531459071,US 2531459072,2531524607,SE 2531524608,2531590143,US 2531590144,2531655679,AU @@ -58895,6 +60499,7 @@ 2533031936,2533097471,BE 2533097472,2533228543,US 2533228544,2533294079,PL +2533294080,2533359615,CN 2533359616,2539978751,IT 2539978752,2540240895,US 2540240896,2540306431,FI @@ -58955,7 +60560,6 @@ 2555641856,2555707391,BR 2555707392,2555903999,US 2555904000,2555969535,CH -2555969536,2555969791,FR 2556035072,2556100607,NO 2556100608,2556166143,AU 2556166144,2556231679,BR @@ -58994,9 +60598,10 @@ 2561671168,2564947967,US 2564947968,2565013503,SG 2565013504,2565210111,US -2566914048,2566979583,AU +2566914048,2566979583,CN 2566979584,2567045119,FI 2567045120,2567110655,US +2567110656,2567176191,CN 2567176192,2567241727,US 2567241728,2567307263,SI 2567307264,2567897087,US @@ -59006,7 +60611,9 @@ 2568093696,2568159231,US 2568159232,2568224767,PL 2568224768,2568290303,SG -2568290304,2569601023,US +2568290304,2569142271,US +2569142272,2569404415,CN +2569404416,2569601023,US 2569601024,2569666559,CA 2569666560,2569797631,US 2569797632,2569863167,NO @@ -59022,7 +60629,10 @@ 2572944896,2572945919,EU 2572945920,2572946175,DE 2572946176,2573008895,EU -2573008896,2573533183,DE +2573008896,2573402111,DE +2573402112,2573467647,CN +2573467648,2573533183,DE +2573533184,2573598719,CN 2573598720,2573926399,US 2573926400,2573991935,AU 2573991936,2574123007,CH @@ -59032,15 +60642,16 @@ 2574315008,2574315263,NO 2574315264,2574319615,SE 2574319616,2574647295,US -2575302656,2583691263,JP +2574647296,2574778367,CN +2574778368,2583691263,JP 2583691264,2584215551,US 2584215552,2584281087,GB 2584281088,2584346623,US 2584346624,2584412159,KR 2584412160,2584477695,CA 2584477696,2584608767,US -2584608768,2584805375,CH -2584805376,2585001983,US +2584608768,2584739839,CH +2584739840,2585001983,US 2585001984,2585067519,CA 2585067520,2585788415,US 2585788416,2585853951,GB @@ -59202,8 +60813,10 @@ 2627010560,2627076095,NZ 2627076096,2627141631,NL 2627141632,2627469311,US -2634022912,2634088447,AU -2634088448,2635399167,JP +2634022912,2634088447,CN +2634088448,2635202559,JP +2635202560,2635268095,CN +2635268096,2635399167,JP 2635399168,2635530239,US 2635530240,2635595775,FR 2635595776,2635661311,FI @@ -59213,6 +60826,7 @@ 2635988992,2636120063,US 2636120064,2637561855,ID 2637561856,2638020607,US +2638020608,2638086143,CN 2638086144,2638151679,US 2638151680,2639462399,JP 2639462400,2639593471,GB @@ -59232,6 +60846,7 @@ 2640576512,2640642047,EC 2640642048,2641952767,JP 2641952768,2642018303,US +2642018304,2642083839,CN 2642083840,2642149375,US 2642149376,2642214911,FI 2642214912,2642280447,SE @@ -59247,8 +60862,11 @@ 2643197952,2643263487,GB 2643263488,2643460095,US 2643460096,2643525631,FI -2643525632,2644180991,US +2643525632,2643722239,US +2643722240,2643787775,CN +2643787776,2644180991,US 2644180992,2644246527,AU +2644246528,2644312063,CN 2644312064,2644377599,IS 2644377600,2644443135,PL 2644443136,2644508671,FR @@ -59303,6 +60921,7 @@ 2650406912,2650603519,US 2650603520,2650669055,CO 2650669056,2650734591,US +2650734592,2650800127,CN 2650800128,2653159423,US 2653159424,2653421567,NO 2653421568,2653487103,AU @@ -59513,7 +61132,12 @@ 2678652928,2678718463,US 2678718464,2678783999,GB 2678784000,2678849535,NO -2678849536,2678915071,US +2678849536,2678885375,US +2678885376,2678885631,GB +2678885632,2678885887,DE +2678885888,2678886143,US +2678886144,2678886399,NL +2678886400,2678915071,US 2678915072,2678980607,FR 2678980608,2679046143,US 2679046144,2679111679,CA @@ -59830,7 +61454,7 @@ 2727018496,2727084031,CA 2727084032,2727346175,US 2727346176,2727608319,AU -2734686208,2734751743,AU +2734686208,2734751743,CN 2734751744,2734817279,GB 2734817280,2734882815,US 2734882816,2734948351,SE @@ -59856,10 +61480,13 @@ 2742681600,2742747135,IN 2742747136,2742779903,CA 2742812672,2742878207,US +2742878208,2742943743,CN 2742943744,2743009279,US 2743009280,2743140351,AU 2743140352,2743205887,US -2743205888,2744516607,JP +2743205888,2743992319,JP +2743992320,2744057855,CN +2744057856,2744516607,JP 2744516608,2744647679,US 2744647680,2744713215,KR 2744713216,2744844287,US @@ -59882,13 +61509,16 @@ 2746089472,2746155007,NO 2746155008,2746220543,NL 2746220544,2746286079,US +2746286080,2746351615,CN 2746351616,2746417151,CR +2746417152,2746482687,CN 2746482688,2746548223,KR 2746548224,2747072511,US 2747072512,2747138047,AU 2747138048,2747465727,US 2747465728,2748055551,ZA -2748055552,2748317695,US +2748055552,2748121087,CN +2748121088,2748317695,US 2748317696,2749628415,JP 2749628416,2749890559,US 2749890560,2750021631,AU @@ -60078,13 +61708,11 @@ 2780926328,2780926335,US 2780926336,2780926367,GB 2780926368,2780926823,US -2780926824,2780926847,GB -2780926848,2780926975,US +2780926824,2780926879,GB +2780926880,2780926975,US 2780926976,2780927487,GB 2780927488,2780927743,US -2780927744,2780927999,GB -2780928000,2780928031,US -2780928032,2780928127,GB +2780927744,2780928127,GB 2780928128,2780928383,US 2780928384,2780928415,GB 2780928416,2780928447,US @@ -60456,13 +62084,14 @@ 2848522240,2848587775,AU 2848587776,2848653311,ZA 2848653312,2848980991,US -2848980992,2849177855,AU +2848980992,2849177855,KR 2849177856,2849178111,CN -2849178112,2849178367,KR -2849178368,2849178623,AU +2849178112,2849178623,KR 2849178624,2849178879,ID 2849178880,2849179135,VN -2849179136,2849964031,AU +2849179136,2849898495,KR +2849898496,2849898751,AU +2849898752,2849964031,KR 2849964032,2850029567,PH 2850029568,2851995647,US 2852126720,2853306367,US @@ -60505,6 +62134,10 @@ 2867855360,2868117503,US 2868379648,2868772863,US 2868838400,2868903935,BE +2868903936,2869035007,SG +2869035008,2869166079,JP +2869166080,2869428223,TH +2869428224,2869952511,CN 2869952512,2870018047,FR 2870018048,2870083583,DE 2870083584,2870149119,FR @@ -60523,9 +62156,17 @@ 2870935552,2871001087,HU 2871001088,2871066623,SE 2871066624,2871083007,CH -2873098240,2874146815,US +2871132160,2872049663,CN +2872049664,2873098239,IN +2873098240,2873884671,US +2873884672,2874146815,IN +2874146816,2875195391,CN +2875195392,2875719679,TH +2875719680,2877292543,CN 2877292544,2882469887,US -2882469888,2882535423,AU +2882469888,2882535423,SG +2882535424,2883583999,CN +2883584000,2885681151,VN 2894069760,2894921727,US 2894921728,2895118335,GB 2895118336,2895301887,US @@ -60554,7 +62195,19 @@ 2902458368,2902462463,A1 2902462464,2902470936,US 2902470937,2902470938,LK -2902470939,2902507519,US +2902470939,2902471468,US +2902471469,2902471470,AU +2902471471,2902471895,US +2902471896,2902471897,AU +2902471898,2902471960,US +2902471961,2902471962,AU +2902471963,2902472037,US +2902472038,2902472039,AU +2902472040,2902472834,US +2902472835,2902472836,BD +2902472837,2902492927,US +2902492928,2902493183,IN +2902493184,2902507519,US 2902507520,2902515711,CA 2902515712,2904555519,US 2904555520,2904817663,CA @@ -60596,7 +62249,15 @@ 2913992704,2914516991,CA 2914516992,2915250175,US 2915250176,2915254271,CA -2915254272,2915765279,US +2915254272,2915274751,US +2915274752,2915274815,PL +2915274816,2915275007,US +2915275008,2915275071,PL +2915275072,2915275263,US +2915275264,2915275327,PL +2915275328,2915516671,US +2915516672,2915516927,CA +2915516928,2915765279,US 2915765280,2915765287,IN 2915765288,2915765343,US 2915765344,2915765351,NZ @@ -60884,7 +62545,11 @@ 2915811136,2915811199,IN 2915811200,2915958783,US 2915958784,2916024319,CA -2916024320,2916163583,US +2916024320,2916118223,US +2916118224,2916118231,LK +2916118232,2916121087,US +2916121088,2916121343,GB +2916121344,2916163583,US 2916163584,2916171775,CA 2916171776,2916184063,US 2916184064,2916196351,CA @@ -60900,7 +62565,9 @@ 2916434560,2916434591,CA 2916434592,2916434623,US 2916434624,2916434655,CA -2916434656,2916437503,US +2916434656,2916436735,US +2916436736,2916436743,CA +2916436744,2916437503,US 2916437504,2916437567,CA 2916437568,2916440143,US 2916440144,2916440159,CA @@ -60908,20 +62575,27 @@ 2916440176,2916440191,CA 2916440192,2916440223,US 2916440224,2916440239,CA -2916440240,2916441119,US +2916440240,2916440287,US +2916440288,2916440295,CA +2916440296,2916441119,US 2916441120,2916441151,CA -2916441152,2916441343,US +2916441152,2916441191,US +2916441192,2916441199,CA +2916441200,2916441343,US 2916441344,2916442111,CA 2916442112,2916442623,US 2916442624,2916442879,CA 2916442880,2916443614,US 2916443615,2916443647,CA -2916443648,2916444927,US +2916443648,2916444351,US +2916444352,2916444359,CA +2916444360,2916444927,US 2916444928,2916445951,CA 2916445952,2916449279,US 2916449280,2916450303,CA 2916450304,2916515839,US -2916548608,2916581375,US +2916515840,2916519935,CA +2916524032,2916581375,US 2916581376,2916614143,PR 2916614144,2917167679,US 2917167680,2917167743,BR @@ -60932,9 +62606,12 @@ 2917167904,2917167935,TR 2917167936,2917167967,US 2917167968,2917167999,GB -2917168000,2917168095,US +2917168000,2917168031,UA +2917168032,2917168095,US 2917168096,2917168127,NZ -2917168128,2917168223,US +2917168128,2917168159,US +2917168160,2917168191,NL +2917168192,2917168223,US 2917168224,2917168255,BR 2917168256,2917168319,US 2917168320,2917168351,AR @@ -60946,13 +62623,15 @@ 2917168608,2917168639,BR 2917168640,2917169663,US 2917169664,2917169695,DE -2917169696,2917170015,US +2917169696,2917169727,NL +2917169728,2917170015,US 2917170016,2917170047,UA 2917170048,2917170079,BR -2917170080,2917170111,US +2917170080,2917170111,NL 2917170112,2917170143,RU -2917170144,2917170207,US -2917170208,2917170239,NL +2917170144,2917170175,US +2917170176,2917170207,NL +2917170208,2917170239,US 2917170240,2917170271,RU 2917170272,2917170303,BR 2917170304,2917193025,US @@ -60962,8 +62641,10 @@ 2917194159,2917194201,US 2917194202,2917194206,SG 2917194207,2917195775,US -2917195776,2917196031,CA -2917196032,2917199871,A2 +2917195776,2917196159,CA +2917196160,2917196287,A2 +2917196288,2917196415,CA +2917196416,2917199871,A2 2917199872,2917203967,CA 2917203968,2917210335,US 2917210336,2917210367,CA @@ -61013,9 +62694,19 @@ 2917718528,2917719039,CA 2917719040,2917722367,US 2917722368,2917722623,CA -2917722624,2917842175,US +2917722624,2917802367,US +2917802368,2917802375,CA +2917802376,2917804935,US +2917804936,2917804943,CA +2917804944,2917815575,US +2917815576,2917815583,RO +2917815584,2917834751,US +2917834752,2917835007,GB +2917835008,2917842175,US 2917842176,2917842431,CA -2917842432,2918014975,US +2917842432,2917858047,US +2917858048,2917858303,BE +2917858304,2918014975,US 2918014976,2918023167,CA 2918023168,2918043647,US 2918043648,2918047743,CA @@ -61025,7 +62716,9 @@ 2918154240,2918170623,CA 2918170624,2918199679,US 2918199680,2918199743,CA -2918199744,2918232063,US +2918199744,2918200287,US +2918200288,2918200303,CA +2918200304,2918232063,US 2918232064,2918236159,CA 2918236160,2918260735,US 2918260736,2918264831,CA @@ -61043,7 +62736,9 @@ 2918375424,2918391807,US 2918391808,2918395903,CA 2918395904,2918404095,US -2918404096,2918408191,PR +2918404096,2918406911,PR +2918406912,2918407423,US +2918407424,2918408191,PR 2918408192,2918416383,US 2918416384,2918420479,CA 2918420480,2918432767,US @@ -61053,16 +62748,45 @@ 2918473728,2918477823,US 2918477824,2918481919,CA 2918481920,2918514943,US -2918514944,2918515071,CA -2918515072,2918533119,US +2918514944,2918515079,CA +2918515080,2918532111,US +2918532112,2918532127,CN +2918532128,2918532143,US +2918532144,2918532151,PK +2918532152,2918533119,US 2918533120,2918533127,CN -2918533128,2918533167,US -2918533168,2918533183,CN -2918533184,2918533247,US -2918533248,2918533375,CA -2918533376,2918534911,US +2918533128,2918533143,US +2918533144,2918533151,CN +2918533152,2918533159,US +2918533160,2918533175,CN +2918533176,2918533191,US +2918533192,2918533199,AU +2918533200,2918533215,CN +2918533216,2918534687,US +2918534688,2918534695,CN +2918534696,2918534703,US +2918534704,2918534807,CN +2918534808,2918534815,US +2918534816,2918534887,CN +2918534888,2918534911,US 2918534912,2918534943,KR -2918534944,2918580223,US +2918534944,2918534983,CN +2918534984,2918534991,ID +2918534992,2918535199,US +2918535200,2918535231,CN +2918535232,2918535423,US +2918535424,2918535679,CN +2918535680,2918535807,US +2918535808,2918536191,CN +2918536192,2918536703,US +2918536704,2918536959,CN +2918536960,2918537215,US +2918537216,2918537615,CN +2918537616,2918537623,US +2918537624,2918537727,CN +2918537728,2918539007,US +2918539008,2918539263,CN +2918539264,2918580223,US 2918580224,2918588415,CA 2918588416,2918596607,US 2918596608,2918604799,CA @@ -61081,13 +62805,17 @@ 2918829696,2918829823,DE 2918829824,2918830079,CN 2918830080,2918834175,CA -2918834176,2918838271,US -2918838272,2918840319,AU -2918840320,2918842367,US +2918834176,2918842367,US 2918842368,2918875135,CA 2918875136,2918973439,US 2918973440,2918989823,CA -2919006208,2919063551,US +2918989824,2919010303,US +2919010304,2919010559,CA +2919010560,2919020543,US +2919020544,2919020575,CA +2919020576,2919020591,US +2919020592,2919020655,CA +2919020656,2919063551,US 2919063552,2919064063,MX 2919064064,2919067655,CN 2919067656,2919067695,US @@ -61223,7 +62951,8 @@ 2919069048,2919069055,US 2919069056,2919069063,CN 2919069064,2919069071,AF -2919069072,2919069087,US +2919069072,2919069079,CN +2919069080,2919069087,US 2919069088,2919069095,IN 2919069096,2919069103,BZ 2919069104,2919069111,US @@ -61248,7 +62977,8 @@ 2919069296,2919069303,BR 2919069304,2919069327,US 2919069328,2919069335,BR -2919069336,2919069359,US +2919069336,2919069343,TR +2919069344,2919069359,US 2919069360,2919069367,CN 2919069368,2919069375,AU 2919069376,2919069391,US @@ -61257,16 +62987,25 @@ 2919069408,2919069415,VN 2919069416,2919069439,US 2919069440,2919069447,CA -2919069448,2919069471,US +2919069448,2919069455,US +2919069456,2919069463,CN +2919069464,2919069471,US 2919069472,2919069479,CA -2919069480,2919069527,US +2919069480,2919069487,CN +2919069488,2919069495,US +2919069496,2919069503,CN +2919069504,2919069511,AR +2919069512,2919069527,US 2919069528,2919069535,TR 2919069536,2919069543,US 2919069544,2919069559,CN 2919069560,2919069567,AE 2919069568,2919069575,VN 2919069576,2919069583,CN -2919069584,2919069631,US +2919069584,2919069607,US +2919069608,2919069615,CN +2919069616,2919069623,QA +2919069624,2919069631,PK 2919069632,2919069639,TW 2919069640,2919069663,US 2919069664,2919069671,CA @@ -61294,7 +63033,75 @@ 2919070944,2919070951,BR 2919070952,2919071143,US 2919071144,2919071151,CN -2919071152,2919174143,US +2919071152,2919170327,US +2919170328,2919170335,PE +2919170336,2919170351,US +2919170352,2919170359,PE +2919170360,2919170367,AU +2919170368,2919170383,BR +2919170384,2919170391,AU +2919170392,2919170399,IN +2919170400,2919170407,TR +2919170408,2919170415,ES +2919170416,2919170431,US +2919170432,2919170439,JP +2919170440,2919170447,IN +2919170448,2919170463,TH +2919170464,2919170487,US +2919170488,2919170495,NL +2919170496,2919170527,US +2919170528,2919170535,BR +2919170536,2919170551,US +2919170552,2919170559,FR +2919170560,2919170575,US +2919170576,2919170583,FR +2919170584,2919170591,US +2919170592,2919170599,CN +2919170600,2919170607,JP +2919170608,2919170615,US +2919170616,2919170623,CN +2919170624,2919170639,US +2919170640,2919170655,JP +2919170656,2919170663,US +2919170664,2919170671,PH +2919170672,2919170679,JP +2919170680,2919170695,US +2919170696,2919170703,CN +2919170704,2919170711,MX +2919170712,2919170727,US +2919170728,2919170735,IN +2919170736,2919170751,US +2919170752,2919170759,GB +2919170760,2919170767,BR +2919170768,2919170775,AR +2919170776,2919170799,US +2919170800,2919170815,SG +2919170816,2919170863,US +2919170864,2919170871,AU +2919170872,2919170879,CH +2919170880,2919170903,US +2919170904,2919170911,GR +2919170912,2919170919,BR +2919170920,2919170935,US +2919170936,2919170943,PH +2919170944,2919170967,US +2919170968,2919170975,NZ +2919170976,2919171047,US +2919171048,2919171055,BR +2919171056,2919171071,US +2919171072,2919171079,IN +2919171080,2919171087,CL +2919171088,2919171095,TR +2919171096,2919171103,US +2919171104,2919171111,PL +2919171112,2919171119,US +2919171120,2919171127,CL +2919171128,2919171135,BR +2919171136,2919171143,AU +2919171144,2919171151,TR +2919171152,2919171159,US +2919171160,2919171167,AU +2919171168,2919174143,US 2919174144,2919178239,CA 2919178240,2919186431,US 2919186432,2919190527,CA @@ -61330,29 +63137,38 @@ 2925002752,2925527039,CA 2925527040,2926575615,US 2926575616,2927099903,CA -2927099904,2928173103,US +2927099904,2927112031,US +2927112032,2927112039,CH +2927112040,2927152175,US +2927152176,2927152183,CH +2927152184,2927154847,US +2927154848,2927154855,CH +2927154856,2927911183,US +2927911184,2927911191,CH +2927911192,2928173103,US 2928173104,2928173119,PH 2928173120,2928173151,US 2928173152,2928173159,RU -2928173160,2928173343,US +2928173160,2928173223,US +2928173224,2928173231,JP +2928173232,2928173343,US 2928173344,2928173351,NL -2928173352,2928173551,US +2928173352,2928173519,US +2928173520,2928173527,JP +2928173528,2928173551,US 2928173552,2928173559,CA 2928173560,2928173679,US 2928173680,2928173695,JP 2928173696,2928173711,US 2928173712,2928173727,JP -2928173728,2928173775,US -2928173776,2928173783,IE -2928173784,2928173855,US -2928173856,2928173871,AU -2928173872,2928173903,US +2928173728,2928173903,US 2928173904,2928173919,NZ 2928173920,2928173959,US 2928173960,2928173967,CA 2928173968,2928174087,US 2928174088,2928174103,CA -2928174104,2928174223,US +2928174104,2928174215,US +2928174216,2928174223,MQ 2928174224,2928174231,CA 2928174232,2928174479,US 2928174480,2928174487,SE @@ -62356,7 +64172,9 @@ 2987433348,2987433359,KZ 2987433360,2987433375,RU 2987433376,2987433379,KZ -2987433380,2987433407,RU +2987433380,2987433383,RU +2987433384,2987433391,KZ +2987433392,2987433407,RU 2987433408,2987433471,KZ 2987433472,2987433983,RU 2987433984,2987438079,FR @@ -62431,8 +64249,7 @@ 2987655168,2987658367,DE 2987658368,2987658383,AT 2987658384,2987659263,DE -2987659264,2987660799,BA -2987660800,2987661311,SI +2987659264,2987661311,BA 2987661312,2987663359,GB 2987663360,2987665407,IT 2987665408,2987667455,RU @@ -62496,7 +64313,8 @@ 2987788288,2987788543,GB 2987788544,2987788799,DE 2987788800,2987789055,US -2987789056,2987790335,TR +2987789056,2987789311,GB +2987789312,2987790335,TR 2987790336,2987792383,GB 2987792384,2987794431,CH 2987794432,2987796479,IQ @@ -62571,7 +64389,9 @@ 2988441896,2988441903,PL 2988441904,2988441911,IT 2988441912,2988441919,ES -2988441920,2988441951,FR +2988441920,2988441939,FR +2988441940,2988441943,ES +2988441944,2988441951,FR 2988441952,2988441967,PL 2988441968,2988441971,NL 2988441972,2988441975,CH @@ -62599,7 +64419,10 @@ 2988442652,2988442655,GB 2988442656,2988442671,PL 2988442672,2988442675,GB -2988442676,2988442895,FR +2988442676,2988442687,FR +2988442688,2988442695,PL +2988442696,2988442703,PT +2988442704,2988442895,FR 2988442896,2988442899,PL 2988442900,2988442903,FR 2988442904,2988442907,IT @@ -62611,7 +64434,7 @@ 2988442928,2988442975,FR 2988442976,2988443007,GB 2988443008,2988443023,PT -2988443024,2988443027,NL +2988443024,2988443027,CZ 2988443028,2988443031,BE 2988443032,2988443035,GB 2988443036,2988443039,BE @@ -62662,8 +64485,8 @@ 2988444800,2988444927,ES 2988444928,2988444931,PL 2988444932,2988444943,DE -2988444944,2988444959,PL -2988444960,2988444999,FR +2988444944,2988444967,PL +2988444968,2988444999,FR 2988445000,2988445007,PL 2988445008,2988445023,FR 2988445024,2988445027,PL @@ -62672,8 +64495,8 @@ 2988445036,2988445039,PL 2988445040,2988445119,FR 2988445120,2988445127,ES -2988445128,2988445151,FR -2988445152,2988445183,IT +2988445128,2988445167,FR +2988445168,2988445183,ES 2988445184,2988445951,DE 2988445952,2988445967,FR 2988445968,2988445983,GB @@ -62698,10 +64521,8 @@ 2988447924,2988447927,IT 2988447928,2988447935,PL 2988447936,2988447943,DE -2988447944,2988447947,NL -2988447948,2988447951,DE -2988447952,2988447955,GB -2988447956,2988447959,BE +2988447944,2988447947,GB +2988447948,2988447959,FR 2988447960,2988447967,IT 2988447968,2988447999,GB 2988448000,2988448127,DE @@ -62769,8 +64590,10 @@ 2988458076,2988458111,PL 2988458112,2988458255,FR 2988458256,2988458271,IT -2988458272,2988458287,PL -2988458288,2988458295,FR +2988458272,2988458275,GB +2988458276,2988458279,FI +2988458280,2988458283,ES +2988458284,2988458295,FR 2988458296,2988458299,GB 2988458300,2988458319,ES 2988458320,2988458323,CZ @@ -62815,7 +64638,7 @@ 2988459236,2988459239,PL 2988459240,2988459243,FR 2988459244,2988459247,ES -2988459248,2988459251,FR +2988459248,2988459251,LT 2988459252,2988459255,BE 2988459256,2988459259,FR 2988459260,2988459263,NL @@ -62859,8 +64682,7 @@ 2988460016,2988460031,FR 2988460032,2988460063,DE 2988460064,2988460095,FR -2988460096,2988460103,ES -2988460104,2988460107,PL +2988460096,2988460107,PL 2988460108,2988460111,DE 2988460112,2988460119,FR 2988460120,2988460123,DE @@ -62869,8 +64691,7 @@ 2988460132,2988460135,CZ 2988460136,2988460143,FR 2988460144,2988460147,GB -2988460148,2988460151,FR -2988460152,2988460159,ES +2988460148,2988460159,ES 2988460160,2988460191,GB 2988460192,2988460195,FR 2988460196,2988460199,CZ @@ -62887,8 +64708,9 @@ 2988460288,2988460323,FR 2988460324,2988460327,GB 2988460328,2988460335,PL -2988460336,2988460351,DE -2988460352,2988460367,FR +2988460336,2988460339,FR +2988460340,2988460343,GB +2988460344,2988460367,FR 2988460368,2988460375,PL 2988460376,2988460543,FR 2988460544,2988460547,GB @@ -62915,8 +64737,8 @@ 2988461088,2988461103,PL 2988461104,2988461119,FR 2988461120,2988461151,NL -2988461152,2988461247,GB -2988461248,2988461255,FR +2988461152,2988461183,GB +2988461184,2988461255,FR 2988461256,2988461259,IT 2988461260,2988461263,PT 2988461264,2988461279,PL @@ -62981,11 +64803,15 @@ 2988461884,2988461887,IT 2988461888,2988461903,FR 2988461904,2988461911,ES -2988461912,2988461935,FR +2988461912,2988461915,IT +2988461916,2988461919,NL +2988461920,2988461935,FR 2988461936,2988461951,GB 2988461952,2988462087,PL 2988462088,2988462095,DE -2988462096,2988462127,FR +2988462096,2988462119,FR +2988462120,2988462123,PL +2988462124,2988462127,GB 2988462128,2988462131,ES 2988462132,2988462143,FR 2988462144,2988462151,PL @@ -63003,7 +64829,8 @@ 2988462312,2988462319,FR 2988462320,2988462323,IT 2988462324,2988462327,DE -2988462328,2988462335,FR +2988462328,2988462331,GB +2988462332,2988462335,FR 2988462336,2988462463,ES 2988462464,2988462495,GB 2988462496,2988462527,DE @@ -63096,7 +64923,7 @@ 2988464276,2988464279,PL 2988464280,2988464283,FR 2988464284,2988464287,PL -2988464288,2988464303,ES +2988464288,2988464303,FR 2988464304,2988464307,IT 2988464308,2988464351,FR 2988464352,2988464355,DE @@ -63202,8 +65029,9 @@ 2988483088,2988483091,FR 2988483092,2988483095,BE 2988483096,2988483103,FR -2988483104,2988483111,ES -2988483112,2988483115,FR +2988483104,2988483107,IE +2988483108,2988483111,PL +2988483112,2988483115,ES 2988483116,2988483119,PL 2988483120,2988483135,GB 2988483136,2988483151,FR @@ -63218,7 +65046,9 @@ 2988483284,2988483295,DE 2988483296,2988483299,FR 2988483300,2988483303,NL -2988483304,2988483319,FR +2988483304,2988483311,FR +2988483312,2988483315,GB +2988483316,2988483319,NL 2988483320,2988483327,PL 2988483328,2988483335,BE 2988483336,2988483343,DE @@ -63226,17 +65056,20 @@ 2988483352,2988483359,FR 2988483360,2988483367,ES 2988483368,2988483375,PL -2988483376,2988483383,GB +2988483376,2988483383,ES 2988483384,2988483427,FR 2988483428,2988483431,BE 2988483432,2988483447,FR 2988483448,2988483455,DE 2988483456,2988483711,FR -2988483712,2988483743,PL +2988483712,2988483727,PL +2988483728,2988483735,FR +2988483736,2988483739,DE +2988483740,2988483743,FR 2988483744,2988483767,GB 2988483768,2988483775,FR 2988483776,2988483871,PL -2988483872,2988483879,DE +2988483872,2988483879,FR 2988483880,2988483887,ES 2988483888,2988483895,GB 2988483896,2988483903,DE @@ -63272,7 +65105,36 @@ 2988484240,2988484243,DE 2988484244,2988484287,FR 2988484288,2988484351,PL -2988484352,2988485887,FR +2988484352,2988484383,GB +2988484384,2988484399,PL +2988484400,2988484403,PT +2988484404,2988484427,PL +2988484428,2988484431,GB +2988484432,2988484439,PL +2988484440,2988484443,ES +2988484444,2988484447,FR +2988484448,2988484463,SN +2988484464,2988484471,FR +2988484472,2988484475,PL +2988484476,2988484479,FR +2988484480,2988484511,PL +2988484512,2988484543,IT +2988484544,2988484591,FR +2988484592,2988484607,ES +2988484608,2988485599,FR +2988485600,2988485607,PL +2988485608,2988485611,GB +2988485612,2988485615,PL +2988485616,2988485631,GB +2988485632,2988485663,ES +2988485664,2988485671,PL +2988485672,2988485675,ES +2988485676,2988485683,FR +2988485684,2988485687,PL +2988485688,2988485691,GB +2988485692,2988485839,FR +2988485840,2988485855,BE +2988485856,2988485887,FR 2988485888,2988485903,PL 2988485904,2988485911,GB 2988485912,2988485951,FR @@ -63283,7 +65145,7 @@ 2988485984,2988485999,FR 2988486000,2988486015,PL 2988486016,2988486031,FR -2988486032,2988486047,ES +2988486032,2988486047,PL 2988486048,2988486063,GB 2988486064,2988486067,FR 2988486068,2988486071,ES @@ -63295,7 +65157,9 @@ 2988486112,2988486127,PL 2988486128,2988486159,NL 2988486160,2988486175,IE -2988486176,2988486191,GB +2988486176,2988486179,FR +2988486180,2988486183,DE +2988486184,2988486191,IE 2988486192,2988486195,IT 2988486196,2988486199,DE 2988486200,2988486207,PT @@ -63317,7 +65181,8 @@ 2988486348,2988486351,ES 2988486352,2988486399,FR 2988486400,2988486783,GB -2988486784,2988486795,FR +2988486784,2988486787,DE +2988486788,2988486795,FR 2988486796,2988486799,PL 2988486800,2988486807,FR 2988486808,2988486811,DE @@ -63479,7 +65344,7 @@ 2988507168,2988507183,DE 2988507184,2988507199,FR 2988507200,2988507203,GB -2988507204,2988507207,PL +2988507204,2988507207,DE 2988507208,2988507211,ES 2988507212,2988507215,PL 2988507216,2988507231,FR @@ -63564,7 +65429,8 @@ 2988508072,2988508079,FR 2988508080,2988508095,ES 2988508096,2988508103,FR -2988508104,2988508127,GB +2988508104,2988508111,GB +2988508112,2988508127,BE 2988508128,2988508135,PL 2988508136,2988508143,GB 2988508144,2988508151,FR @@ -63573,7 +65439,9 @@ 2988508208,2988508215,PL 2988508216,2988508219,FR 2988508220,2988508223,DE -2988508224,2988508287,FR +2988508224,2988508263,FR +2988508264,2988508271,ES +2988508272,2988508287,FR 2988508288,2988508303,PL 2988508304,2988508307,PT 2988508308,2988508315,FR @@ -63604,7 +65472,7 @@ 2988508856,2988508871,FR 2988508872,2988508875,PL 2988508876,2988508879,PT -2988508880,2988508895,ES +2988508880,2988508895,DE 2988508896,2988508911,PL 2988508912,2988508927,FR 2988508928,2988508943,GB @@ -63684,7 +65552,9 @@ 2988509960,2988509963,IT 2988509964,2988509967,GB 2988509968,2988509983,ES -2988509984,2988509995,FR +2988509984,2988509987,DE +2988509988,2988509991,CZ +2988509992,2988509995,FR 2988509996,2988509999,DE 2988510000,2988510015,PL 2988510016,2988510023,ES @@ -63766,7 +65636,7 @@ 2988511776,2988511807,GB 2988511808,2988511823,ES 2988511824,2988511831,DE -2988511832,2988511835,ES +2988511832,2988511835,GB 2988511836,2988511855,FR 2988511856,2988511871,GB 2988511872,2988511887,PL @@ -63809,7 +65679,7 @@ 2988512316,2988512319,DE 2988512320,2988512335,FR 2988512336,2988512339,PL -2988512340,2988512343,ES +2988512340,2988512343,DE 2988512344,2988512351,FR 2988512352,2988512383,IT 2988512384,2988512415,FR @@ -63849,8 +65719,8 @@ 2988512840,2988512847,DE 2988512848,2988512851,ES 2988512852,2988512855,DE -2988512856,2988512879,FR -2988512880,2988512895,DE +2988512856,2988512883,FR +2988512884,2988512895,DE 2988512896,2988512899,PL 2988512900,2988512903,GB 2988512904,2988512907,ES @@ -63892,14 +65762,14 @@ 2988513308,2988513311,PL 2988513312,2988513327,PT 2988513328,2988513331,PL -2988513332,2988513335,FR -2988513336,2988513343,GB +2988513332,2988513343,FR 2988513344,2988513351,ES 2988513352,2988513359,IT 2988513360,2988513375,GB 2988513376,2988513379,FR 2988513380,2988513383,PT -2988513384,2988513407,GB +2988513384,2988513391,GB +2988513392,2988513407,FR 2988513408,2988513471,PL 2988513472,2988513503,FR 2988513504,2988513507,CH @@ -63933,8 +65803,12 @@ 2988513900,2988513903,PL 2988513904,2988513919,IT 2988513920,2988513951,FR -2988513952,2988513983,PT -2988513984,2988514015,GB +2988513952,2988513967,PT +2988513968,2988513975,FI +2988513976,2988513983,IT +2988513984,2988513991,GB +2988513992,2988513999,IE +2988514000,2988514015,FR 2988514016,2988514019,PT 2988514020,2988514023,NL 2988514024,2988514027,PT @@ -63978,8 +65852,8 @@ 2988514400,2988514431,GB 2988514432,2988514447,FR 2988514448,2988514463,ES -2988514464,2988514527,FR -2988514528,2988514559,DE +2988514464,2988514543,FR +2988514544,2988514559,PL 2988514560,2988514623,FR 2988514624,2988514655,DE 2988514656,2988514671,FR @@ -63995,13 +65869,13 @@ 2988514752,2988514815,FI 2988514816,2988514823,FR 2988514824,2988514827,PL -2988514828,2988514831,DE +2988514828,2988514831,FR 2988514832,2988514839,PL 2988514840,2988514879,ES 2988514880,2988514943,PL 2988514944,2988514959,ES 2988514960,2988514975,PL -2988514976,2988514979,BE +2988514976,2988514979,FR 2988514980,2988514983,PL 2988514984,2988514991,IT 2988514992,2988514995,FR @@ -64036,8 +65910,15 @@ 2988524164,2988524167,FR 2988524168,2988524175,ES 2988524176,2988524191,FR -2988524192,2988524227,DE -2988524228,2988524231,IE +2988524192,2988524195,FI +2988524196,2988524199,DE +2988524200,2988524203,IE +2988524204,2988524211,FR +2988524212,2988524215,FI +2988524216,2988524219,FR +2988524220,2988524223,ES +2988524224,2988524227,DE +2988524228,2988524231,PL 2988524232,2988524239,ES 2988524240,2988524255,FR 2988524256,2988524259,LT @@ -64093,7 +65974,7 @@ 2988526580,2988526583,ES 2988526584,2988526591,DE 2988526592,2988526607,FR -2988526608,2988526615,GB +2988526608,2988526615,DE 2988526616,2988526687,FR 2988526688,2988526703,ES 2988526704,2988526711,FR @@ -64121,7 +66002,236 @@ 2988527192,2988527195,DE 2988527196,2988527199,FR 2988527200,2988527207,NL -2988527208,2988535807,FR +2988527208,2988527211,DE +2988527212,2988527391,FR +2988527392,2988527399,PL +2988527400,2988527407,DE +2988527408,2988527411,FR +2988527412,2988527415,PL +2988527416,2988527423,GB +2988527424,2988527431,FR +2988527432,2988527439,IT +2988527440,2988527451,PL +2988527452,2988527455,PT +2988527456,2988527459,IT +2988527460,2988527475,FR +2988527476,2988527479,ES +2988527480,2988527487,FR +2988527488,2988527503,GB +2988527504,2988527523,FR +2988527524,2988527531,DE +2988527532,2988527551,FR +2988527552,2988527583,PL +2988527584,2988527591,GB +2988527592,2988527595,IT +2988527596,2988527599,DE +2988527600,2988527603,NL +2988527604,2988527607,FR +2988527608,2988527611,DE +2988527612,2988527615,PT +2988527616,2988528127,DE +2988528128,2988528159,GB +2988528160,2988528175,ES +2988528176,2988528179,CH +2988528180,2988528183,IT +2988528184,2988528187,PL +2988528188,2988528191,GB +2988528192,2988528207,IE +2988528208,2988528215,FR +2988528216,2988528219,IE +2988528220,2988528223,DE +2988528224,2988528231,CH +2988528232,2988528235,GB +2988528236,2988528239,FR +2988528240,2988528255,GB +2988528256,2988528383,FR +2988528384,2988528399,GB +2988528400,2988528415,FR +2988528416,2988528423,GB +2988528424,2988528431,NL +2988528432,2988528451,CH +2988528452,2988528455,DE +2988528456,2988528459,ES +2988528460,2988528463,NL +2988528464,2988528467,DE +2988528468,2988528471,GB +2988528472,2988528475,ES +2988528476,2988528479,DE +2988528480,2988528483,NL +2988528484,2988528487,ES +2988528488,2988528495,FR +2988528496,2988528511,IT +2988528512,2988528671,FR +2988528672,2988528703,DE +2988528704,2988528767,FR +2988528768,2988528771,DE +2988528772,2988528783,FR +2988528784,2988528787,ES +2988528788,2988528791,FR +2988528792,2988528795,PL +2988528796,2988528799,GB +2988528800,2988528863,FR +2988528864,2988528867,DE +2988528868,2988528875,FR +2988528876,2988528879,GB +2988528880,2988528895,ES +2988528896,2988528911,FR +2988528912,2988528919,ES +2988528920,2988528923,CH +2988528924,2988528927,GB +2988528928,2988528959,FR +2988528960,2988528991,FI +2988528992,2988529007,LT +2988529008,2988529023,FI +2988529024,2988529031,PL +2988529032,2988529047,FR +2988529048,2988529051,PL +2988529052,2988529055,PT +2988529056,2988529119,FR +2988529120,2988529151,NL +2988529152,2988529159,IT +2988529160,2988529163,IE +2988529164,2988529167,DE +2988529168,2988529171,IE +2988529172,2988529175,PT +2988529176,2988529179,ES +2988529180,2988529199,FR +2988529200,2988529207,PL +2988529208,2988529247,FR +2988529248,2988529251,GB +2988529252,2988529255,PL +2988529256,2988529263,NL +2988529264,2988529311,FR +2988529312,2988529315,PL +2988529316,2988529319,FR +2988529320,2988529323,FI +2988529324,2988529331,DE +2988529332,2988529335,FR +2988529336,2988529339,GB +2988529340,2988529343,DE +2988529344,2988529351,IE +2988529352,2988529359,GB +2988529360,2988529375,IE +2988529376,2988529383,GB +2988529384,2988529387,DE +2988529388,2988529391,FR +2988529392,2988529407,PL +2988529408,2988529423,FR +2988529424,2988529431,DE +2988529432,2988529439,FR +2988529440,2988529455,PL +2988529456,2988529535,FR +2988529536,2988529567,FI +2988529568,2988529599,GB +2988529600,2988529607,IT +2988529608,2988529631,FR +2988529632,2988529647,LT +2988529648,2988529663,FR +2988529664,2988529679,PL +2988529680,2988529703,FR +2988529704,2988529707,GB +2988529708,2988529759,FR +2988529760,2988529767,PL +2988529768,2988529771,FR +2988529772,2988529775,DE +2988529776,2988529783,ES +2988529784,2988529791,CH +2988529792,2988529823,FR +2988529824,2988529855,ES +2988529856,2988529887,FR +2988529888,2988529891,DE +2988529892,2988529895,PL +2988529896,2988529899,GB +2988529900,2988529903,FI +2988529904,2988529919,BE +2988529920,2988529935,FR +2988529936,2988529939,CH +2988529940,2988529943,GB +2988529944,2988529951,NL +2988529952,2988529955,GB +2988529956,2988529959,DE +2988529960,2988529967,GB +2988529968,2988529983,NL +2988529984,2988529999,GB +2988530000,2988530003,FR +2988530004,2988530007,GB +2988530008,2988530039,FR +2988530040,2988530043,ES +2988530044,2988530047,FR +2988530048,2988530063,GB +2988530064,2988530079,DE +2988530080,2988530095,FR +2988530096,2988530099,DE +2988530100,2988530103,FR +2988530104,2988530111,GB +2988530112,2988530191,FR +2988530192,2988530207,NL +2988530208,2988530239,IE +2988530240,2988530271,FR +2988530272,2988530303,IE +2988530304,2988530367,FR +2988530368,2988530371,ES +2988530372,2988530379,FR +2988530380,2988530383,CH +2988530384,2988530391,FR +2988530392,2988530399,IE +2988530400,2988530403,ES +2988530404,2988530415,FR +2988530416,2988530419,PT +2988530420,2988530423,PL +2988530424,2988530431,ES +2988530432,2988530687,DE +2988530688,2988530695,IT +2988530696,2988530703,PL +2988530704,2988530735,FR +2988530736,2988530739,IE +2988530740,2988530743,NL +2988530744,2988530751,PL +2988530752,2988530847,FR +2988530848,2988530851,DE +2988530852,2988530855,ES +2988530856,2988530863,PT +2988530864,2988530867,PL +2988530868,2988530871,FR +2988530872,2988530879,IT +2988530880,2988530887,PL +2988530888,2988530895,DE +2988530896,2988530975,FR +2988530976,2988531007,PL +2988531008,2988531015,DE +2988531016,2988531019,NL +2988531020,2988531023,IE +2988531024,2988531027,ES +2988531028,2988531031,FR +2988531032,2988531039,PL +2988531040,2988531047,FR +2988531048,2988531051,GB +2988531052,2988531055,PL +2988531056,2988531071,FR +2988531072,2988531075,DE +2988531076,2988531079,GB +2988531080,2988531151,FR +2988531152,2988531167,PL +2988531168,2988531175,NL +2988531176,2988531183,GB +2988531184,2988531187,ES +2988531188,2988531191,IE +2988531192,2988531199,FR +2988531200,2988531247,ES +2988531248,2988531259,PL +2988531260,2988531263,GB +2988531264,2988531295,FR +2988531296,2988531311,IE +2988531312,2988531319,FR +2988531320,2988531323,PT +2988531324,2988531327,GB +2988531328,2988531391,FR +2988531392,2988531399,PL +2988531400,2988531403,DE +2988531404,2988531423,PL +2988531424,2988531451,FR +2988531452,2988531455,CH +2988531456,2988535807,FR 2988535808,2988537855,ES 2988537856,2988539935,FR 2988539936,2988539967,GB @@ -64172,7 +66282,7 @@ 2988540544,2988540607,PL 2988540608,2988540639,FR 2988540640,2988540647,PL -2988540648,2988540651,GB +2988540648,2988540651,FR 2988540652,2988540667,PL 2988540668,2988540671,IT 2988540672,2988540683,PL @@ -64230,13 +66340,12 @@ 2988541464,2988541467,ES 2988541468,2988541503,FR 2988541504,2988541519,CH -2988541520,2988541535,FR -2988541536,2988541539,GB +2988541520,2988541539,FR 2988541540,2988541543,PT 2988541544,2988541547,FR 2988541548,2988541551,ES -2988541552,2988541583,FR -2988541584,2988541587,ES +2988541552,2988541567,FR +2988541568,2988541587,ES 2988541588,2988541603,FR 2988541604,2988541607,NL 2988541608,2988541611,GB @@ -64306,7 +66415,8 @@ 2988542336,2988542367,FR 2988542368,2988542399,CZ 2988542400,2988542407,FR -2988542408,2988542415,PL +2988542408,2988542411,LT +2988542412,2988542415,PL 2988542416,2988542431,FR 2988542432,2988542439,PL 2988542440,2988542443,ES @@ -64373,7 +66483,7 @@ 2988543300,2988543303,FR 2988543304,2988543307,ES 2988543308,2988543311,PL -2988543312,2988543315,DE +2988543312,2988543315,FR 2988543316,2988543319,IE 2988543320,2988543323,FI 2988543324,2988543327,PL @@ -64408,8 +66518,14 @@ 2988543940,2988543947,PL 2988543948,2988543959,BE 2988543960,2988543967,PL -2988543968,2988543999,FR -2988544000,2988544127,CZ +2988543968,2988544015,FR +2988544016,2988544023,GB +2988544024,2988544031,IE +2988544032,2988544039,FR +2988544040,2988544043,DE +2988544044,2988544047,GB +2988544048,2988544063,FR +2988544064,2988544127,DE 2988544128,2988544159,GB 2988544160,2988544163,FR 2988544164,2988544167,PL @@ -64419,7 +66535,8 @@ 2988544184,2988544191,DE 2988544192,2988544227,ES 2988544228,2988544271,FR -2988544272,2988544279,BE +2988544272,2988544275,DE +2988544276,2988544279,FR 2988544280,2988544283,GB 2988544284,2988544287,DE 2988544288,2988544291,PL @@ -64441,16 +66558,24 @@ 2988544384,2988544447,GB 2988544448,2988544463,FR 2988544464,2988544479,ES -2988544480,2988544495,PL -2988544496,2988544511,IE +2988544480,2988544511,PL 2988544512,2988544527,FR 2988544528,2988544535,ES 2988544536,2988544539,PL 2988544540,2988544543,LT 2988544544,2988544639,ES -2988544640,2988544735,DE -2988544736,2988544751,PL -2988544752,2988544799,FR +2988544640,2988544647,PL +2988544648,2988544655,FR +2988544656,2988544659,ES +2988544660,2988544663,FR +2988544664,2988544667,DE +2988544668,2988544671,ES +2988544672,2988544687,GB +2988544688,2988544703,PL +2988544704,2988544719,IT +2988544720,2988544723,FR +2988544724,2988544727,DE +2988544728,2988544799,FR 2988544800,2988544831,FI 2988544832,2988544863,FR 2988544864,2988544895,PL @@ -64498,8 +66623,8 @@ 2988545376,2988545383,DE 2988545384,2988545387,ES 2988545388,2988545391,FR -2988545392,2988545399,GB -2988545400,2988545407,FR +2988545392,2988545395,DE +2988545396,2988545407,FR 2988545408,2988545439,IE 2988545440,2988545471,FR 2988545472,2988545503,ES @@ -64513,7 +66638,7 @@ 2988545552,2988545555,PL 2988545556,2988545559,IT 2988545560,2988545563,FR -2988545564,2988545567,ES +2988545564,2988545567,DE 2988545568,2988545571,NL 2988545572,2988545575,DE 2988545576,2988545579,FR @@ -64540,7 +66665,7 @@ 2988545944,2988545967,NL 2988545968,2988545971,IT 2988545972,2988545979,FR -2988545980,2988545983,GB +2988545980,2988545983,DE 2988545984,2988545987,FR 2988545988,2988545991,PL 2988545992,2988545995,LT @@ -64575,8 +66700,10 @@ 2988546328,2988546335,ES 2988546336,2988546351,PT 2988546352,2988546367,ES -2988546368,2988546431,FR -2988546432,2988546463,DE +2988546368,2988546439,FR +2988546440,2988546443,DE +2988546444,2988546447,FR +2988546448,2988546463,ES 2988546464,2988546527,FR 2988546528,2988546535,BE 2988546536,2988546539,IE @@ -64591,16 +66718,15 @@ 2988546600,2988546603,PT 2988546604,2988546615,GB 2988546616,2988546619,FI -2988546620,2988546623,FR +2988546620,2988546623,IT 2988546624,2988546687,FI 2988546688,2988546695,PL 2988546696,2988546699,FR 2988546700,2988546703,IT -2988546704,2988546727,FR -2988546728,2988546735,ES -2988546736,2988546751,FR +2988546704,2988546751,FR 2988546752,2988546783,ES -2988546784,2988546819,FR +2988546784,2988546815,FR +2988546816,2988546819,PT 2988546820,2988546823,NL 2988546824,2988546831,FR 2988546832,2988546835,DE @@ -64611,7 +66737,8 @@ 2988546948,2988546951,DE 2988546952,2988546955,FR 2988546956,2988546959,GB -2988546960,2988546967,DE +2988546960,2988546963,CH +2988546964,2988546967,IT 2988546968,2988546971,FR 2988546972,2988546975,BE 2988546976,2988546991,FI @@ -64642,13 +66769,11 @@ 2988547136,2988547167,IT 2988547168,2988547199,BE 2988547200,2988547207,IE -2988547208,2988547211,FR -2988547212,2988547215,DE -2988547216,2988547219,IT -2988547220,2988547223,FR -2988547224,2988547227,ES -2988547228,2988547231,PT -2988547232,2988547239,FR +2988547208,2988547211,ES +2988547212,2988547215,GB +2988547216,2988547223,FR +2988547224,2988547227,GB +2988547228,2988547239,FR 2988547240,2988547247,PL 2988547248,2988547255,NL 2988547256,2988547263,FR @@ -64667,7 +66792,7 @@ 2988547448,2988547451,FR 2988547452,2988547455,PT 2988547456,2988547471,DE -2988547472,2988547475,NL +2988547472,2988547475,LT 2988547476,2988547479,GB 2988547480,2988547487,FR 2988547488,2988547519,ES @@ -64729,7 +66854,9 @@ 2988556888,2988556895,FR 2988556896,2988556911,PL 2988556912,2988556919,NL -2988556920,2988556959,PL +2988556920,2988556927,PL +2988556928,2988556943,FR +2988556944,2988556959,PL 2988556960,2988556963,ES 2988556964,2988556967,FR 2988556968,2988556975,BE @@ -64745,8 +66872,11 @@ 2988557072,2988557075,PL 2988557076,2988557079,FR 2988557080,2988557087,FI -2988557088,2988557119,DE -2988557120,2988557151,NL +2988557088,2988557095,CH +2988557096,2988557103,ES +2988557104,2988557111,FR +2988557112,2988557115,LT +2988557116,2988557151,NL 2988557152,2988557215,FR 2988557216,2988557247,BE 2988557248,2988557251,FR @@ -64804,7 +66934,7 @@ 2988558104,2988558119,FR 2988558120,2988558123,GB 2988558124,2988558127,FR -2988558128,2988558131,ES +2988558128,2988558131,DE 2988558132,2988558135,PL 2988558136,2988558139,GB 2988558140,2988558143,DE @@ -64817,7 +66947,7 @@ 2988558528,2988558591,PL 2988558592,2988558655,GB 2988558656,2988558727,FR -2988558728,2988558731,DE +2988558728,2988558731,IT 2988558732,2988558735,PT 2988558736,2988558767,FR 2988558768,2988558783,CH @@ -64869,9 +66999,7 @@ 2988560384,2988560447,FR 2988560448,2988560451,DE 2988560452,2988560455,PL -2988560456,2988560463,FR -2988560464,2988560479,PL -2988560480,2988560575,FR +2988560456,2988560575,FR 2988560576,2988560591,ES 2988560592,2988560595,NL 2988560596,2988560599,IT @@ -64998,7 +67126,7 @@ 2988563076,2988563079,GB 2988563080,2988563083,DE 2988563084,2988563087,FR -2988563088,2988563103,NL +2988563088,2988563103,PL 2988563104,2988563135,FR 2988563136,2988563151,PL 2988563152,2988563199,FR @@ -65054,9 +67182,7 @@ 2990518016,2990518079,IT 2990518080,2990525247,DE 2990525248,2990525311,ES -2990525312,2990534655,DE -2990534656,2990534783,EG -2990534784,2990535935,DE +2990525312,2990535935,DE 2990535936,2990535967,PL 2990535968,2990538751,DE 2990538752,2991063039,RU @@ -65070,7 +67196,9 @@ 2991144960,2991161343,SA 2991161344,2991177727,FR 2991177728,2991178751,A1 -2991178752,2991180799,SE +2991178752,2991179263,SE +2991179264,2991179327,A1 +2991179328,2991180799,SE 2991180800,2991182847,A1 2991182848,2991185919,SE 2991185920,2991185951,GB @@ -65343,17 +67471,14 @@ 2996998528,2996998655,UA 2996998656,2996998719,DE 2996998720,2996998911,RU -2996998912,2996999199,DE -2996999200,2996999231,AU -2996999232,2996999359,DE +2996998912,2996999359,DE 2996999360,2996999423,IL 2996999424,2996999551,IN 2996999552,2996999679,DE 2996999680,2996999935,GB 2996999936,2996999951,DE 2996999952,2996999967,LT -2996999968,2996999999,AU -2997000000,2997000447,DE +2996999968,2997000447,DE 2997000448,2997000703,RU 2997000704,2997000831,DE 2997000832,2997000959,CA @@ -65405,18 +67530,19 @@ 2997019136,2997019391,TR 2997019392,2997019647,GB 2997019648,2997019679,DE -2997019680,2997019711,AU +2997019680,2997019711,BR 2997019712,2997019903,US 2997019904,2997020415,DE 2997020416,2997020671,CA 2997020672,2997020719,DE 2997020720,2997020735,SE -2997020736,2997021183,DE +2997020736,2997020799,DE +2997020800,2997020863,US +2997020864,2997021183,DE 2997021184,2997021695,GB -2997021696,2997022015,DE -2997022016,2997022143,GB -2997022144,2997022207,IN -2997022208,2997022239,DE +2997021696,2997022079,DE +2997022080,2997022143,GB +2997022144,2997022239,DE 2997022240,2997022303,NL 2997022304,2997022367,SG 2997022368,2997022751,DE @@ -65647,14 +67773,25 @@ 3001815040,3001819135,RU 3001819136,3001823231,IR 3001823232,3001827327,GE -3001827328,3001831423,SE +3001827328,3001827647,SE +3001827648,3001827743,GB +3001827744,3001827839,SE +3001827840,3001827871,GB +3001827872,3001830399,SE +3001830400,3001830431,GB +3001830432,3001830655,SE +3001830656,3001830687,GB +3001830688,3001830911,SE +3001830912,3001830943,GB +3001830944,3001830975,US +3001830976,3001831423,SE 3001831424,3001835519,BA 3001835520,3001839615,RU 3001839616,3001843711,ES -3001843712,3001845759,RU -3001845760,3001846271,GB -3001846272,3001847295,RU -3001847296,3001851903,GB +3001843712,3001846271,RU +3001846272,3001846783,GB +3001846784,3001847807,RU +3001847808,3001851903,GB 3001851904,3001855999,IT 3001856000,3001859071,NL 3001859072,3001860095,MD @@ -65718,13 +67855,13 @@ 3001975728,3001975735,GB 3001975736,3001975743,MY 3001975744,3001975751,CN -3001975752,3001975759,GB +3001975752,3001975759,UA 3001975760,3001975767,US 3001975768,3001975783,CN 3001975784,3001975791,US 3001975792,3001975799,IT 3001975800,3001975807,US -3001975808,3001975815,GB +3001975808,3001975815,UA 3001975816,3001975823,US 3001975824,3001975831,UA 3001975832,3001975847,GB @@ -65775,11 +67912,9 @@ 3001977600,3001977855,DK 3001977856,3001977983,GB 3001977984,3001978015,CN -3001978016,3001978047,RU -3001978048,3001978111,US -3001978112,3001978239,GB -3001978240,3001978303,CN -3001978304,3001978367,GB +3001978016,3001978031,GB +3001978032,3001978047,RU +3001978048,3001978367,GB 3001978368,3001978495,US 3001978496,3001978527,RU 3001978528,3001978559,UA @@ -65806,9 +67941,7 @@ 3002019840,3002020223,NL 3002020224,3002020287,IL 3002020288,3002020303,US -3002020304,3002022527,NL -3002022528,3002022655,BZ -3002022656,3002023935,NL +3002020304,3002023935,NL 3002023936,3002028031,DE 3002028032,3002036223,GB 3002036224,3002040319,BG @@ -65870,7 +68003,9 @@ 3002662912,3002664959,DE 3002664960,3002667007,ES 3002667008,3002669055,GE -3002669056,3002671103,DE +3002669056,3002669199,DE +3002669200,3002669207,CH +3002669208,3002671103,DE 3002671104,3002673151,LV 3002673152,3002675199,GB 3002675200,3002677247,DK @@ -65931,8 +68066,8 @@ 3002798080,3002800127,FR 3002800128,3002802175,NL 3002802176,3002804223,GB -3002804224,3002806271,TR -3002806272,3002808319,DE +3002804224,3002806015,TR +3002806016,3002808319,DE 3002808320,3002810367,FI 3002810368,3002812415,AT 3002812416,3002814463,IT @@ -66056,6 +68191,7 @@ 3028484096,3028500479,KR 3028500480,3028516863,JP 3028516864,3028518911,AU +3028518912,3028520959,JP 3028520960,3028521983,ID 3028521984,3028523007,LA 3028523008,3028525055,JP @@ -66187,11 +68323,13 @@ 3034456064,3034464255,JP 3034464256,3034466303,NZ 3034466304,3034472447,JP -3034472448,3034480639,IN +3034472448,3034478591,IN +3034478592,3034480639,JP 3034480640,3034482687,SG 3034482688,3034484735,AF 3034484736,3034488831,TH 3034488832,3034492927,AU +3034492928,3034497023,JP 3034497024,3034499071,AU 3034499072,3034500095,HK 3034500096,3034501119,TW @@ -66297,6 +68435,7 @@ 3056992256,3057025023,CN 3057025024,3057033215,PH 3057033216,3057037311,NZ +3057037312,3057041407,JP 3057041408,3057049599,MY 3057049600,3057050623,AU 3057050624,3057051647,SG @@ -66407,6 +68546,7 @@ 3075473408,3075571711,VN 3075571712,3075575807,FJ 3075575808,3075577855,AU +3075577856,3075579903,ID 3075579904,3075581951,MY 3075581952,3075582975,MN 3075582976,3075583999,TH @@ -66461,7 +68601,11 @@ 3082289152,3087007743,CN 3087007744,3091202047,US 3091202048,3091726335,CA -3091726336,3093168127,US +3091726336,3091955711,US +3091955712,3091959807,CA +3091959808,3091976191,US +3091976192,3091980287,CA +3091980288,3093168127,US 3093168128,3093200895,CA 3093233664,3093237759,PR 3093237760,3093266431,US @@ -66471,9 +68615,7 @@ 3093941104,3093941119,CN 3093941120,3093941191,US 3093941192,3093941199,CN -3093941200,3093942271,US -3093942272,3093942527,CA -3093942528,3093954759,US +3093941200,3093954759,US 3093954760,3093954767,AR 3093954768,3093954975,US 3093954976,3093954983,AU @@ -66613,9 +68755,14 @@ 3098214400,3098263551,US 3098263552,3098271743,CA 3098271744,3098275839,US -3098279936,3098476543,US +3098279936,3098281503,US +3098281504,3098281511,PA +3098281512,3098281711,US +3098281712,3098281719,PA +3098281720,3098476543,US 3098476544,3098492927,CA 3098492928,3103784959,US +3103784960,3107979263,EU 3120562176,3120594943,CO 3120594944,3120599039,AR 3120599040,3120601087,EC @@ -66660,6 +68807,7 @@ 3123707904,3123970047,UY 3124232192,3124772863,AR 3124789248,3124822015,CR +3124854784,3124887551,CL 3124887552,3124953087,EC 3125018624,3125149695,EC 3125280768,3125542911,PA @@ -66671,7 +68819,7 @@ 3127246848,3127377919,CO 3127377920,3127640063,CL 3127640064,3127902207,AR -3127902208,3128164351,CO +3127902208,3128426495,CO 3128426496,3128492031,DO 3128492032,3128524799,CO 3128557568,3128950783,AR @@ -66696,6 +68844,7 @@ 3133079552,3133145087,AR 3133145088,3145727999,BR 3145728000,3154116607,MX +3154116608,3154149375,NO 3154182144,3154247679,DE 3154247680,3154313215,RS 3154313216,3154378751,TR @@ -66724,7 +68873,9 @@ 3156791440,3156791455,NL 3156791456,3156802271,DE 3156802272,3156802303,US -3156802304,3156803583,DE +3156802304,3156803327,DE +3156803328,3156803359,CH +3156803360,3156803583,DE 3156803584,3156869119,TR 3156869120,3156934655,LU 3156934656,3157065727,RU @@ -66783,7 +68934,9 @@ 3158395152,3158395159,DE 3158395160,3158395263,AT 3158395264,3158395295,DE -3158395296,3158395647,AT +3158395296,3158395431,AT +3158395432,3158395439,DE +3158395440,3158395647,AT 3158395648,3158395663,DE 3158395664,3158396927,AT 3158396928,3158398975,IT @@ -66831,13 +68984,9 @@ 3158886176,3158886207,BZ 3158886208,3158886271,DE 3158886272,3158886335,RO -3158886336,3158886399,DE -3158886400,3158886655,TR -3158886656,3158887167,DE +3158886336,3158887167,DE 3158887168,3158887423,CA -3158887424,3158889215,DE -3158889216,3158889727,TR -3158889728,3158889983,DE +3158887424,3158889983,DE 3158889984,3158890239,HK 3158890240,3158891263,DE 3158891264,3158891519,CA @@ -66854,8 +69003,7 @@ 3158892672,3158892799,RU 3158892800,3158893567,DE 3158893568,3158893823,GB -3158893824,3158894079,TR -3158894080,3158895167,DE +3158893824,3158895167,DE 3158895168,3158895231,RU 3158895232,3158895359,TR 3158895360,3158895423,DE @@ -67180,8 +69328,7 @@ 3163161888,3163161919,US 3163161920,3163162111,DE 3163162112,3163162143,TR -3163162144,3163162175,DE -3163162176,3163162207,US +3163162144,3163162207,DE 3163162208,3163162239,TR 3163162240,3163162271,DE 3163162272,3163162303,NL @@ -67210,14 +69357,14 @@ 3163165824,3163166175,DE 3163166176,3163166207,US 3163166208,3163166367,DE -3163166368,3163166399,NL +3163166368,3163166399,US 3163166400,3163166495,DE 3163166496,3163166527,RO 3163166528,3163166559,DE 3163166560,3163166591,HU 3163166592,3163167775,DE 3163167776,3163167807,GR -3163167808,3163167839,NL +3163167808,3163167839,US 3163167840,3163167967,DE 3163167968,3163167999,TR 3163168000,3163168031,BG @@ -67231,7 +69378,7 @@ 3163168416,3163168511,DE 3163168512,3163168543,TR 3163168544,3163168575,DE -3163168576,3163168607,NL +3163168576,3163168607,US 3163168608,3163168671,DE 3163168672,3163168703,US 3163168704,3163169823,DE @@ -67248,26 +69395,26 @@ 3163170208,3163170239,RU 3163170240,3163170303,DE 3163170304,3163170335,IT -3163170336,3163170463,DE -3163170464,3163170495,GB -3163170496,3163170527,BR +3163170336,3163170495,DE +3163170496,3163170527,BG 3163170528,3163170591,DE 3163170592,3163170623,BR 3163170624,3163170655,GR -3163170656,3163170783,DE +3163170656,3163170751,DE +3163170752,3163170783,CH 3163170784,3163170815,US 3163170816,3163171871,DE 3163171872,3163171903,RO -3163171904,3163171935,DE +3163171904,3163171935,BZ 3163171936,3163171967,BR 3163171968,3163172127,DE -3163172128,3163172159,NL +3163172128,3163172159,US 3163172160,3163172319,DE 3163172320,3163172351,NL 3163172352,3163172383,GB 3163172384,3163172447,DE 3163172448,3163172479,TR -3163172480,3163172511,DE +3163172480,3163172511,US 3163172512,3163172543,RU 3163172544,3163172607,DE 3163172608,3163172639,DK @@ -67281,16 +69428,21 @@ 3163174272,3163174303,BG 3163174304,3163174335,IL 3163174336,3163174367,RU -3163174368,3163174591,DE +3163174368,3163174495,DE +3163174496,3163174527,US +3163174528,3163174591,DE 3163174592,3163174623,SE 3163174624,3163174655,GB 3163174656,3163174719,DE 3163174720,3163174751,NL 3163174752,3163174783,US -3163174784,3163176063,DE +3163174784,3163174815,DE +3163174816,3163174847,CA +3163174848,3163174879,TR +3163174880,3163176063,DE 3163176064,3163176095,US 3163176096,3163176127,DE -3163176128,3163176159,NL +3163176128,3163176159,US 3163176160,3163176255,DE 3163176256,3163176287,UG 3163176288,3163176479,DE @@ -67480,7 +69632,8 @@ 3164951876,3164951879,IT 3164951880,3164951887,DE 3164951888,3164951895,ES -3164951896,3164951907,FR +3164951896,3164951903,FR +3164951904,3164951907,DE 3164951908,3164951911,PL 3164951912,3164951915,ES 3164951916,3164951919,IE @@ -67590,8 +69743,35 @@ 3164959928,3164959931,GB 3164959932,3164959935,ES 3164959936,3164959999,FR -3164960000,3164960255,DE -3164960256,3164960511,FR +3164960000,3164960259,DE +3164960260,3164960263,FR +3164960264,3164960267,LT +3164960268,3164960271,NL +3164960272,3164960295,FR +3164960296,3164960299,IT +3164960300,3164960303,GB +3164960304,3164960319,FR +3164960320,3164960323,ES +3164960324,3164960327,IT +3164960328,3164960331,CZ +3164960332,3164960335,LT +3164960336,3164960339,PL +3164960340,3164960359,FR +3164960360,3164960367,PL +3164960368,3164960383,FR +3164960384,3164960387,DE +3164960388,3164960391,PL +3164960392,3164960395,GB +3164960396,3164960399,DE +3164960400,3164960415,PL +3164960416,3164960435,FR +3164960436,3164960443,PT +3164960444,3164960447,IT +3164960448,3164960463,PL +3164960464,3164960467,NL +3164960468,3164960471,GB +3164960472,3164960479,FR +3164960480,3164960511,GB 3164960512,3164960575,PL 3164960576,3164960591,DE 3164960592,3164960599,ES @@ -67606,7 +69786,7 @@ 3164960676,3164960679,GB 3164960680,3164960699,PL 3164960700,3164960703,ES -3164960704,3164960719,GB +3164960704,3164960719,FR 3164960720,3164960723,CH 3164960724,3164960727,FR 3164960728,3164960735,GB @@ -67624,12 +69804,11 @@ 3164961024,3164961151,FR 3164961152,3164961167,DE 3164961168,3164961175,PL -3164961176,3164961247,FR -3164961248,3164961279,NL +3164961176,3164961279,FR 3164961280,3164961311,DE 3164961312,3164961319,FR 3164961320,3164961327,PL -3164961328,3164961331,ES +3164961328,3164961331,CZ 3164961332,3164961363,FR 3164961364,3164961367,ES 3164961368,3164961375,FR @@ -67639,7 +69818,9 @@ 3164961400,3164961403,PL 3164961404,3164961471,FR 3164961472,3164961503,DE -3164961504,3164961551,ES +3164961504,3164961519,GB +3164961520,3164961535,FR +3164961536,3164961551,ES 3164961552,3164961559,FR 3164961560,3164961563,DE 3164961564,3164961583,FR @@ -67754,8 +69935,9 @@ 3164969020,3164969023,BE 3164969024,3164969055,PL 3164969056,3164969071,CZ -3164969072,3164969087,GB -3164969088,3164969095,ES +3164969072,3164969079,FR +3164969080,3164969083,PL +3164969084,3164969095,ES 3164969096,3164969099,PL 3164969100,3164969103,DE 3164969104,3164969135,FR @@ -67840,7 +70022,7 @@ 3164971072,3164971135,DE 3164971136,3164971263,GB 3164971264,3164971455,DE -3164971456,3164971459,NL +3164971456,3164971459,FR 3164971460,3164971463,IT 3164971464,3164971467,CH 3164971468,3164971471,FR @@ -67922,7 +70104,7 @@ 3164976160,3164976191,GB 3164976192,3164976215,FR 3164976216,3164976223,PL -3164976224,3164976231,NL +3164976224,3164976231,FR 3164976232,3164976239,ES 3164976240,3164976255,IE 3164976256,3164976259,GB @@ -67936,7 +70118,9 @@ 3164976308,3164976311,DE 3164976312,3164976315,GB 3164976316,3164976319,FR -3164976320,3164976335,DE +3164976320,3164976327,DE +3164976328,3164976331,FR +3164976332,3164976335,PL 3164976336,3164976343,FR 3164976344,3164976347,PL 3164976348,3164976351,ES @@ -67971,7 +70155,7 @@ 3164977760,3164977775,NL 3164977776,3164977791,PL 3164977792,3164977807,FR -3164977808,3164977823,NL +3164977808,3164977823,DE 3164977824,3164977839,GB 3164977840,3164977883,FR 3164977884,3164977887,GB @@ -67981,7 +70165,7 @@ 3164978048,3164978067,FR 3164978068,3164978079,ES 3164978080,3164978111,IT -3164978112,3164978127,ES +3164978112,3164978127,GB 3164978128,3164978143,FR 3164978144,3164978147,DE 3164978148,3164978151,ES @@ -68042,12 +70226,20 @@ 3166650368,3166654463,UA 3166654464,3166658559,RU 3166658560,3166662655,UA -3166668800,3166670847,RO +3166662656,3166666751,RU +3166666752,3166670847,RO 3166670848,3166672895,UA 3166672896,3166674943,GB 3166674944,3166679039,RU 3166679040,3166681087,IR -3166681088,3166683135,RU +3166681088,3166685183,RU +3166685184,3166687231,PL +3166687232,3166689279,MD +3166689280,3166691327,PL +3166691328,3166693375,RU +3166693376,3166695423,UA +3166695424,3166697471,RU +3166697472,3166699519,RO 3166699520,3166961663,DE 3166961664,3167223807,SI 3167223808,3167594831,NL @@ -68057,9 +70249,7 @@ 3167940096,3167940351,CY 3167940352,3168015871,RO 3168015872,3168016127,GB -3168016128,3168069631,RO -3168069632,3168075775,GB -3168075776,3168092159,RO +3168016128,3168092159,RO 3168092160,3168096255,GB 3168096256,3168100351,MD 3168100352,3168104447,RO @@ -68129,7 +70319,9 @@ 3169648640,3169714175,MD 3169714176,3169779711,FI 3169779712,3169845247,UA -3169845248,3169867775,RO +3169845248,3169863679,RO +3169863680,3169864703,MD +3169864704,3169867775,RO 3169867776,3169868031,DE 3169868032,3169976319,RO 3169976320,3170111487,RU @@ -68164,6 +70356,7 @@ 3170762752,3170795519,RU 3170795520,3170828287,BG 3170828288,3170861055,RU +3170861056,3170893823,RS 3170893824,3179282431,BR 3179282432,3179282943,UY 3179282944,3179283199,MX @@ -68216,8 +70409,9 @@ 3188097024,3188105215,DO 3188105216,3188113407,CO 3188113408,3188117503,HN +3188117504,3188121599,AR 3188121600,3188125695,TT -3188129792,3188146175,AR +3188125696,3188146175,AR 3188146176,3188170751,CO 3188170752,3188174847,CR 3188178944,3188187135,CR @@ -68255,6 +70449,7 @@ 3188580352,3188597759,AR 3188597760,3188598783,PA 3188598784,3188600831,AR +3188600832,3188604927,CL 3188604928,3188606207,AR 3188606208,3188606463,CL 3188606464,3188609279,AR @@ -68266,12 +70461,14 @@ 3188612352,3188621311,AR 3188621312,3188625407,GT 3188625408,3188627455,AR +3188627456,3188628479,CR 3188629504,3188637695,AR 3188637696,3188645887,PA 3188645888,3188662271,CO 3188670464,3188674559,HN 3188674560,3188686847,AR 3188686848,3188690943,EC +3188690944,3188695039,CU 3188695040,3188703231,VE 3188703232,3188981759,AR 3188981760,3189178367,CL @@ -68284,7 +70481,9 @@ 3190554624,3190816767,CL 3190816768,3191078911,AR 3191078912,3191087103,CO -3191087104,3191089151,AR +3191087104,3191091199,AR +3191091200,3191093247,BO +3191093248,3191095295,AR 3191095296,3191099391,EC 3191103488,3191107583,CO 3191111680,3191128063,PY @@ -68308,6 +70507,7 @@ 3191436288,3191438335,GT 3191438336,3191439359,SV 3191439360,3191455743,EC +3191455744,3191472127,AR 3191472128,3191603199,TT 3191603200,3191608319,MX 3191608320,3191608831,CO @@ -68337,11 +70537,12 @@ 3193307136,3193438207,SV 3193438208,3193569279,AN 3193569280,3193634815,CO +3193634816,3193700351,CL 3193700352,3193716735,HN 3193733120,3193765887,AR 3193765888,3193798655,TT 3193798656,3193806847,CO -3193815040,3193819135,AR +3193815040,3193823231,AR 3193823232,3193827327,CL 3193831424,3193864191,DO 3193864192,3193872383,EC @@ -68464,7 +70665,7 @@ 3195240448,3195256831,HT 3195256832,3195265023,AR 3195273216,3195535359,PE -3195535360,3195539455,SV +3195535360,3195543551,SV 3195543552,3195547647,PE 3195551744,3195559935,EC 3195559936,3195564031,AR @@ -68486,7 +70687,7 @@ 3195731968,3195736063,EC 3195740160,3195744255,PA 3195748352,3195752447,CL -3195756544,3195760639,AR +3195756544,3195764735,AR 3195764736,3195768831,CR 3195772928,3195777023,VE 3195781120,3195797503,PA @@ -68510,6 +70711,7 @@ 3196092416,3196125183,PY 3196125184,3196190719,BO 3196190720,3196207103,HN +3196207104,3196223487,CO 3196223488,3196321791,EC 3196321792,3196583935,UY 3196583936,3196690687,AR @@ -68525,6 +70727,7 @@ 3197566976,3197599743,CL 3197632512,3197698047,EC 3197698048,3197730815,VE +3197730816,3197763583,CL 3197763584,3197894655,EC 3197894656,3198156799,CO 3198156800,3198484479,CL @@ -68576,14 +70779,14 @@ 3202220032,3202351103,PE 3202351104,3202875391,AR 3202875392,3203399679,PE -3203399680,3203432447,CO +3203399680,3203465215,CO 3203465216,3203476479,CR 3203476480,3203476735,CO 3203476736,3203476991,CR 3203476992,3203477375,CO 3203477376,3203530751,CR 3203530752,3203538943,NI -3203538944,3203596287,CO +3203538944,3203661823,CO 3203661824,3203923967,AR 3203923968,3204448255,CO 3221225472,3221560319,US @@ -69911,7 +72114,9 @@ 3227806496,3227806527,GB 3227806528,3227806719,FI 3227806720,3227807743,US -3227807744,3227813375,FI +3227807744,3227809023,FI +3227809024,3227809279,IN +3227809280,3227813375,FI 3227813376,3227813631,US 3227813632,3227813887,MY 3227813888,3227815167,GB @@ -71286,7 +73491,38 @@ 3231776768,3231793151,US 3231793152,3231793663,BE 3231809536,3231810047,NZ +3231842304,3231843327,RU +3231843328,3231844351,NO +3231844352,3231845375,RU +3231845376,3231846399,ES 3231846400,3231846655,RO +3231846656,3231846911,PT +3231846912,3231847167,GB +3231847168,3231847423,UA +3231847424,3231848447,RU +3231848448,3231849471,ES +3231849472,3231850495,RO +3231850496,3231851519,UA +3231851520,3231853567,RU +3231853568,3231855615,PL +3231855616,3231856639,RS +3231856640,3231857663,RU +3231857664,3231858687,PL +3231858688,3231859711,RU +3231859712,3231860735,FR +3231860736,3231861759,SA +3231861760,3231863807,UA +3231863808,3231864831,DE +3231864832,3231865855,RU +3231865856,3231866879,PL +3231866880,3231867903,CZ +3231867904,3231868927,RU +3231868928,3231869951,LV +3231869952,3231873023,UA +3231873024,3231875071,RU +3231875072,3231876095,PL +3231876096,3231877119,UA +3231906816,3231907839,RU 3231907840,3231916031,US 3231916032,3231948799,FI 3231973376,3232038911,AT @@ -71301,21 +73537,26 @@ 3232095232,3232096255,GB 3232096256,3232104447,SE 3232104448,3232107519,DE -3232107520,3232108543,EU +3232107520,3232108543,RU 3232108544,3232129023,DE -3232129024,3232131071,EU +3232129024,3232130047,RO +3232130048,3232131071,UA 3232131072,3232133119,DE -3232133120,3232133631,EU +3232133120,3232133631,UA 3232133632,3232134143,DE -3232134144,3232141823,EU +3232134144,3232137215,EU +3232137216,3232139263,UA +3232139264,3232140287,GB +3232140288,3232141311,FR +3232141312,3232141823,UA 3232141824,3232156159,DE -3232156160,3232156671,EU +3232156160,3232156671,PL 3232156672,3232159743,DE -3232159744,3232160767,EU +3232159744,3232160767,PL 3232160768,3232163839,DE -3232163840,3232165887,EU +3232163840,3232165887,RU 3232165888,3232169727,DE -3232169728,3232169983,EU +3232169728,3232169983,PL 3232169984,3232235519,IT 3232301056,3232309247,US 3232309248,3232313343,SG @@ -72205,7 +74446,28 @@ 3238461440,3238502399,DE 3238502400,3238504447,RU 3238504448,3238526975,DE -3238526976,3238592511,CH +3238526976,3238527231,RU +3238527232,3238542591,CH +3238542592,3238542847,PL +3238542848,3238545919,CH +3238545920,3238546431,RU +3238546432,3238546943,CH +3238546944,3238547455,UA +3238547456,3238548991,CH +3238548992,3238549503,CZ +3238549504,3238559231,CH +3238559232,3238559487,SE +3238559488,3238562559,CH +3238562560,3238562815,IR +3238562816,3238573567,CH +3238573568,3238574079,PL +3238574080,3238578943,CH +3238578944,3238579199,RU +3238579200,3238589951,CH +3238589952,3238590207,LT +3238590208,3238590975,CH +3238590976,3238591231,SA +3238591232,3238592511,CH 3238592512,3238593023,GB 3238593024,3238594303,EU 3238594304,3238594559,GB @@ -72250,6 +74512,7 @@ 3238630656,3238632959,GB 3238632960,3238633215,UA 3238633216,3238653951,DK +3238657024,3238657535,AT 3238657536,3238657791,DK 3238657792,3238658047,AT 3238658048,3238675455,SE @@ -72740,9 +75003,11 @@ 3239762176,3239762431,RU 3239762432,3239762687,DK 3239762688,3239762943,RO +3239762944,3239763199,SI 3239763200,3239763455,SE 3239763456,3239763967,GB 3239763968,3239772159,DE +3239772160,3239772415,NL 3239772416,3239772671,GB 3239772672,3239773183,DK 3239773184,3239773439,FR @@ -72765,6 +75030,7 @@ 3239788544,3239789055,EU 3239789568,3239790079,FR 3239790080,3239790591,RO +3239790592,3239791103,LV 3239791104,3239792127,CH 3239792128,3239792639,FR 3239792640,3239793151,UA @@ -73064,7 +75330,6 @@ 3240207872,3240208383,RU 3240208384,3240208895,FR 3240208896,3240209407,GB -3240209408,3240209919,UA 3240209920,3240210943,PL 3240210944,3240211455,GB 3240211456,3240211967,NL @@ -73171,7 +75436,6 @@ 3240281600,3240281727,JO 3240281728,3240281855,PL 3240281856,3240281983,NL -3240281984,3240282111,PL 3240282112,3240282239,DE 3240282240,3240282367,UA 3240282368,3240282495,RO @@ -73966,11 +76230,12 @@ 3244823296,3244823551,FR 3244823552,3244823807,GE 3244823808,3244824063,RO +3244824064,3244824319,IR 3244824320,3244824575,SI 3244824576,3244824831,RU 3244824832,3244825087,GB 3244825088,3244825343,DE -3244825600,3244826111,RU +3244825344,3244826111,RU 3244826112,3244826367,RO 3244826368,3244826623,CH 3244826624,3244826879,DE @@ -74288,7 +76553,6 @@ 3244913408,3244913663,SI 3244913664,3244913919,DK 3244913920,3244914431,RU -3244914432,3244914687,UA 3244914688,3244914943,GB 3244914944,3244915199,UA 3244915200,3244915455,PL @@ -74323,7 +76587,8 @@ 3244922624,3244922879,BG 3244922880,3244923135,UA 3244923136,3244923391,AT -3244923392,3244923903,LV +3244923392,3244923647,EU +3244923648,3244923903,LV 3244923904,3244924159,RU 3244924160,3244924927,CZ 3244924928,3244925183,FI @@ -74366,7 +76631,6 @@ 3244935168,3244935423,NL 3244935424,3244935679,UA 3244935680,3244935935,PL -3244935936,3244936191,FR 3244936192,3244936959,AT 3244936960,3244937215,KZ 3244937216,3244937471,TR @@ -74661,6 +76925,7 @@ 3245168896,3245169151,RU 3245169152,3245169407,IT 3245169408,3245169919,PL +3245169920,3245170175,PT 3245170176,3245170431,GB 3245170432,3245170687,CH 3245170688,3245171711,DE @@ -74724,6 +76989,7 @@ 3245207552,3245208063,UA 3245208064,3245208575,DE 3245208576,3245209087,PL +3245209088,3245209599,VG 3245209600,3245210111,UA 3245210112,3245210623,FR 3245210624,3245211135,HU @@ -75415,6 +77681,8 @@ 3247347968,3247348223,HU 3247348224,3247348991,FI 3247348992,3247349247,DE +3247349248,3247349503,FR +3247349504,3247349759,UA 3247349760,3247353855,SE 3247353856,3247362047,FI 3247362048,3247362303,RO @@ -75621,7 +77889,9 @@ 3248791040,3248791295,BE 3248791296,3248791551,DE 3248791552,3248792487,GB -3248792488,3248796607,EU +3248792488,3248792495,EU +3248792496,3248792511,GB +3248792512,3248796607,EU 3248796608,3248796863,GB 3248796864,3248798975,EU 3248798976,3248799231,GB @@ -75766,6 +78036,7 @@ 3249136128,3249137151,RU 3249137152,3249137663,FR 3249137664,3249138175,PL +3249138176,3249138687,RU 3249138688,3249139199,GB 3249139200,3249139711,RU 3249139712,3249140223,UA @@ -75895,7 +78166,9 @@ 3249931008,3249931263,GB 3249931264,3249932031,SE 3249932032,3249934335,GB -3249934336,3250061311,SE +3249934336,3249968127,SE +3249968128,3249969151,FR +3249969152,3250061311,SE 3250061312,3250061635,FI 3250061636,3250061639,AX 3250061640,3250083643,FI @@ -76018,6 +78291,7 @@ 3250416128,3250417663,DE 3250417664,3250418175,IT 3250418176,3250418687,DE +3250418688,3250419199,PL 3250419200,3250419711,NO 3250419712,3250420223,RU 3250420224,3250420735,IR @@ -76093,7 +78367,7 @@ 3250692096,3250692351,NO 3250692352,3250692607,NL 3250692608,3250693375,BG -3250693376,3250694399,IE +3250693376,3250693631,IE 3250694400,3250694655,GB 3250694656,3250694911,SK 3250694912,3250695167,NL @@ -76115,7 +78389,6 @@ 3250722304,3250724863,GB 3250724864,3250733055,MA 3250733056,3250741247,DZ -3250741248,3250749439,EU 3250749440,3250749695,GH 3250749696,3250749951,IT 3250749952,3250750463,RO @@ -76183,6 +78456,7 @@ 3251122176,3251122687,CH 3251122688,3251123199,PL 3251123200,3251123711,FR +3251123712,3251124223,RU 3251124224,3251124735,NL 3251124736,3251125247,FR 3251125248,3251125759,BE @@ -76346,7 +78620,6 @@ 3251182080,3251182591,FR 3251182592,3251183103,IT 3251183104,3251183615,RU -3251183616,3251183871,NO 3251183872,3251184127,CH 3251184128,3251184383,PL 3251184384,3251184639,DK @@ -76401,6 +78674,7 @@ 3251209216,3251210239,GB 3251210240,3251211263,PL 3251211264,3251212287,UA +3251212288,3251212415,PL 3251212416,3251212671,FI 3251212672,3251212799,SE 3251212800,3251212927,FR @@ -76657,7 +78931,8 @@ 3252318720,3252318975,GP 3252318976,3252319231,PL 3252319232,3252319743,AT -3252319744,3252322303,GR +3252319744,3252321791,GR +3252321792,3252322303,PL 3252322304,3252323327,NO 3252323328,3252324351,PL 3252324352,3252326399,RU @@ -76758,7 +79033,8 @@ 3252408640,3252408671,CF 3252408672,3252408703,SO 3252408704,3252408735,NG -3252408736,3252408767,LT +3252408736,3252408751,SO +3252408752,3252408767,LT 3252408768,3252408783,NG 3252408784,3252408799,LT 3252408800,3252408831,SO @@ -76841,9 +79117,13 @@ 3252414656,3252414991,LT 3252414992,3252414999,IQ 3252415000,3252415015,LT -3252415016,3252415127,IQ +3252415016,3252415095,IQ +3252415096,3252415103,LT +3252415104,3252415127,IQ 3252415128,3252415135,LT -3252415136,3252415223,IQ +3252415136,3252415159,IQ +3252415160,3252415167,BE +3252415168,3252415223,IQ 3252415224,3252415487,LT 3252415488,3252415743,IQ 3252415744,3252415775,GB @@ -76855,8 +79135,8 @@ 3252416960,3252417023,GN 3252417024,3252417279,LT 3252417280,3252417287,IQ -3252417288,3252417439,AF -3252417440,3252417791,LT +3252417288,3252417455,AF +3252417456,3252417791,LT 3252417792,3252417855,NG 3252417856,3252417919,LT 3252417920,3252417935,MW @@ -76893,9 +79173,7 @@ 3252419920,3252419927,IQ 3252419928,3252419935,LT 3252419936,3252419943,IQ -3252419944,3252419999,LT -3252420000,3252420007,IQ -3252420008,3252420031,LT +3252419944,3252420031,LT 3252420032,3252420055,IQ 3252420056,3252420063,LT 3252420064,3252420071,IQ @@ -76904,9 +79182,7 @@ 3252420120,3252420143,IQ 3252420144,3252420191,LT 3252420192,3252420223,IQ -3252420224,3252420239,LT -3252420240,3252420247,IQ -3252420248,3252420263,LT +3252420224,3252420263,LT 3252420264,3252420271,IQ 3252420272,3252420351,LT 3252420352,3252420415,IQ @@ -77107,8 +79383,7 @@ 3252465952,3252465983,GH 3252465984,3252466047,LT 3252466048,3252466175,GH -3252466176,3252473343,LT -3252473344,3252473855,CD +3252466176,3252473855,LT 3252473856,3252474879,SO 3252474880,3252475903,LT 3252475904,3252476415,NG @@ -77308,8 +79583,7 @@ 3253230592,3253230847,BY 3253230848,3253247999,RU 3253248000,3253248255,DE -3253248256,3253248511,UZ -3253248512,3253265407,RU +3253248256,3253265407,RU 3253265408,3253265919,AM 3253265920,3253270527,RU 3253270528,3253271551,BY @@ -77654,11 +79928,11 @@ 3254510564,3254510847,NE 3254510848,3254521855,FR 3254521856,3254522367,GB -3254522368,3254523665,FR -3254523666,3254523666,UG -3254523667,3254523667,CH -3254523668,3254523668,RO -3254523669,3254550527,FR +3254522368,3254523739,FR +3254523740,3254523740,UG +3254523741,3254523741,CH +3254523742,3254523742,RO +3254523743,3254550527,FR 3254550528,3254550783,US 3254550784,3254551039,IR 3254551040,3254551295,US @@ -78115,7 +80389,9 @@ 3255255752,3255255759,DK 3255255760,3255256319,BE 3255256320,3255256575,LU -3255256576,3255259327,BE +3255256576,3255259199,BE +3255259200,3255259215,LU +3255259216,3255259327,BE 3255259328,3255259359,LU 3255259360,3255259919,BE 3255259920,3255259927,LU @@ -78133,11 +80409,15 @@ 3255262800,3255262815,LU 3255262816,3255263295,BE 3255263296,3255263327,US -3255263328,3255265007,BE +3255263328,3255264543,BE +3255264544,3255264575,LU +3255264576,3255265007,BE 3255265008,3255265023,LU 3255265024,3255270431,BE 3255270432,3255270463,FR -3255270464,3255273855,BE +3255270464,3255271807,BE +3255271808,3255271839,LU +3255271840,3255273855,BE 3255273856,3255273887,LU 3255273888,3255274207,BE 3255274208,3255274239,LU @@ -78288,7 +80568,8 @@ 3255374848,3255375359,CY 3255375360,3255375871,RO 3255375872,3255376383,PL -3255376384,3255377407,GB +3255376384,3255376895,UA +3255376896,3255377407,GB 3255377408,3255377919,KZ 3255377920,3255378431,GB 3255378432,3255378943,CH @@ -79852,7 +82133,9 @@ 3258109952,3258110207,DK 3258110208,3258111487,CZ 3258111488,3258111743,PL -3258111744,3258121215,CZ +3258111744,3258118399,CZ +3258118400,3258118655,UA +3258118656,3258121215,CZ 3258121216,3258121471,PL 3258121472,3258121727,AT 3258121728,3258121983,UA @@ -80427,7 +82710,9 @@ 3261472768,3261503487,RO 3261503488,3261503935,MD 3261503936,3261530111,RO -3261530112,3261570303,SE +3261530112,3261531903,SE +3261531904,3261532159,GB +3261532160,3261570303,SE 3261570304,3261570559,IT 3261570560,3261595647,SE 3261595648,3261661183,NL @@ -80471,6 +82756,7 @@ 3261767680,3261775871,RS 3261775872,3261776383,PL 3261776384,3261777407,RU +3261777408,3261777663,IR 3261777664,3261777919,KZ 3261777920,3261778431,PL 3261778432,3261778943,RU @@ -80797,8 +83083,7 @@ 3262444032,3262444543,AT 3262444544,3262445055,UA 3262445056,3262445567,RO -3262445568,3262446079,PL -3262446592,3262447103,PL +3262445568,3262447103,PL 3262447104,3262447615,GB 3262447616,3262460415,PT 3262460416,3262460543,UA @@ -83626,7 +85911,9 @@ 3263168512,3263430655,GB 3263430656,3263436799,SE 3263436800,3263437311,GB -3263437312,3263477759,SE +3263437312,3263461631,SE +3263461632,3263461887,AE +3263461888,3263477759,SE 3263477760,3263478015,JP 3263478016,3263478271,AU 3263478272,3263496191,SE @@ -83783,6 +86070,7 @@ 3264015360,3264015615,DK 3264015616,3264015871,UA 3264015872,3264016127,PT +3264016128,3264016383,UA 3264016384,3264016639,SE 3264016640,3264016895,PL 3264016896,3264017151,GB @@ -84001,8 +86289,8 @@ 3264612480,3264612511,GB 3264612512,3264612591,FR 3264612592,3264613119,GB -3264613120,3264614399,FR -3264614400,3264614431,GB +3264613120,3264614143,FR +3264614144,3264614431,GB 3264614432,3264614447,FR 3264614448,3264614463,GB 3264614464,3264614527,FR @@ -84040,7 +86328,9 @@ 3264620544,3264620927,FR 3264620928,3264621567,GB 3264621568,3264621823,FR -3264621824,3264626687,GB +3264621824,3264622847,GB +3264622848,3264623103,FR +3264623104,3264626687,GB 3264626688,3264627711,EE 3264627712,3264628735,GR 3264628736,3264630783,UA @@ -84084,6 +86374,7 @@ 3264665600,3264666111,RO 3264666112,3264666623,IL 3264666624,3264667135,UA +3264667136,3264667647,RU 3264667648,3264668159,BE 3264668160,3264669183,DE 3264669184,3264669695,PL @@ -84179,12 +86470,12 @@ 3264846592,3264846719,US 3264846720,3264846847,RU 3264846848,3264846911,DK -3264846912,3264846975,SK 3264846976,3264847103,GB 3264847104,3264847135,FR 3264847136,3264847167,ES 3264847168,3264847199,IE 3264847200,3264847231,NO +3264847232,3264847263,CH 3264847264,3264847295,LI 3264847360,3264847487,FR 3264847488,3264847615,RU @@ -84309,7 +86600,9 @@ 3265467520,3265467523,ES 3265467524,3265527055,DE 3265527056,3265527059,ES -3265527060,3265527807,DE +3265527060,3265527063,DE +3265527064,3265527071,SE +3265527072,3265527807,DE 3265527808,3265569279,GB 3265569280,3265569791,AE 3265569792,3265582335,GB @@ -84337,6 +86630,7 @@ 3265604096,3265604351,SA 3265604352,3265604607,FR 3265604608,3265604863,CH +3265604864,3265605119,RU 3265605120,3265605375,UA 3265605376,3265605631,CZ 3265605632,3265605887,PL @@ -84727,9 +87021,7 @@ 3267626240,3267626495,BE 3267626496,3267626751,ES 3267626752,3267627007,RU -3267627008,3267627167,DE -3267627168,3267627199,EU -3267627200,3267627343,DE +3267627008,3267627343,DE 3267627344,3267627359,EU 3267627360,3267627423,DE 3267627424,3267627431,EU @@ -85152,8 +87444,8 @@ 3268224768,3268225023,US 3268225024,3268226367,EU 3268226368,3268226663,GB -3268226664,3268226687,EU -3268226688,3268226815,GB +3268226664,3268226671,EU +3268226672,3268226815,GB 3268226816,3268227327,EU 3268227328,3268227391,GB 3268227392,3268227519,EU @@ -85535,8 +87827,8 @@ 3269279488,3269279671,EU 3269279672,3269279679,CH 3269279680,3269279743,EU -3269279744,3269279999,GB -3269280000,3269280767,NL +3269279744,3269280127,GB +3269280128,3269280767,NL 3269280768,3269281023,GB 3269281024,3269281279,DE 3269281280,3269282047,EU @@ -85637,7 +87929,9 @@ 3269320704,3269321727,GB 3269321728,3269322239,EU 3269322240,3269322495,DE -3269322496,3269322679,EU +3269322496,3269322655,EU +3269322656,3269322671,DE +3269322672,3269322679,EU 3269322680,3269322683,DE 3269322684,3269322751,EU 3269322752,3269323263,DE @@ -85945,6 +88239,7 @@ 3270979072,3270979327,PL 3270979328,3270979583,AT 3270979584,3270979839,LV +3270979840,3270980095,GB 3270980096,3270980351,AM 3270980352,3270980607,RU 3270980608,3270980863,MD @@ -86149,7 +88444,7 @@ 3271692288,3271694847,EU 3271694848,3271695103,GB 3271695104,3271696383,EU -3271696384,3271698431,DE +3271696384,3271698431,GB 3271698432,3271704575,EU 3271704576,3271712767,RU 3271712768,3271716863,LV @@ -87163,7 +89458,7 @@ 3273023488,3273024511,PL 3273024512,3273025535,NL 3273025536,3273026559,RU -3273027584,3273028607,UA +3273026560,3273028607,UA 3273028608,3273029631,PL 3273030656,3273031679,PL 3273031680,3273032191,GB @@ -87195,9 +89490,7 @@ 3273051968,3273052039,IT 3273052040,3273052047,NL 3273052048,3273052063,GR -3273052064,3273052087,IT -3273052088,3273052111,GR -3273052112,3273052415,IT +3273052064,3273052415,IT 3273052416,3273052671,GB 3273052672,3273052927,FR 3273052928,3273053183,DE @@ -87281,8 +89574,8 @@ 3273326984,3273326987,DE 3273326988,3273326991,EU 3273326992,3273327047,DE -3273327048,3273327279,EU -3273327280,3273327287,DE +3273327048,3273327263,EU +3273327264,3273327287,DE 3273327288,3273327359,EU 3273327360,3273327423,IE 3273327424,3273327511,GB @@ -87434,7 +89727,8 @@ 3273372672,3273372927,GB 3273372928,3273373183,NL 3273373184,3273373455,GB -3273373456,3273373695,DE +3273373456,3273373567,DE +3273373568,3273373695,GB 3273373696,3273375231,EU 3273375232,3273375551,DE 3273375552,3273375743,EU @@ -87527,6 +89821,7 @@ 3273438208,3273438719,IL 3273438720,3273439231,PL 3273439232,3273439743,RO +3273439744,3273440255,DE 3273440256,3273440767,RO 3273440768,3273441279,AT 3273441280,3273441599,GB @@ -88319,10 +90614,8 @@ 3275423752,3275423775,EU 3275423776,3275423807,GB 3275423808,3275423871,EU -3275423872,3275424647,GB -3275424648,3275424655,EU -3275424656,3275424671,GB -3275424672,3275425791,EU +3275423872,3275424711,GB +3275424712,3275425791,EU 3275425792,3275427271,GB 3275427272,3275427279,EU 3275427280,3275427615,GB @@ -88330,7 +90623,9 @@ 3275427840,3275428375,GB 3275428376,3275428383,EU 3275428384,3275428399,GB -3275428400,3275429887,EU +3275428400,3275428415,EU +3275428416,3275428447,GB +3275428448,3275429887,EU 3275429888,3275430143,GB 3275430144,3275430271,EU 3275430272,3275430399,GB @@ -88341,8 +90636,8 @@ 3275432704,3275433983,EU 3275433984,3275437567,GB 3275437568,3275438079,EU -3275438080,3275438615,GB -3275438616,3275440127,EU +3275438080,3275438623,GB +3275438624,3275440127,EU 3275440128,3275440639,GB 3275440640,3275442175,EU 3275442176,3275442719,GB @@ -88350,7 +90645,9 @@ 3275444224,3275444735,GB 3275444736,3275446271,EU 3275446272,3275447063,GB -3275447064,3275448319,EU +3275447064,3275447071,EU +3275447072,3275447087,GB +3275447088,3275448319,EU 3275448320,3275449519,GB 3275449520,3275449527,FR 3275449528,3275450879,GB @@ -88359,9 +90656,7 @@ 3275451264,3275452415,EU 3275452416,3275454127,GB 3275454128,3275454143,EU -3275454144,3275454415,GB -3275454416,3275454463,EU -3275454464,3275457023,GB +3275454144,3275457023,GB 3275457024,3275457791,FK 3275457792,3275458559,GB 3275458560,3275460095,IE @@ -88369,8 +90664,8 @@ 3275460608,3275460863,HK 3275460864,3275463523,GB 3275463524,3275463527,EU -3275463528,3275463551,GB -3275463552,3275463679,EU +3275463528,3275463583,GB +3275463584,3275463679,EU 3275463680,3275464031,GB 3275464032,3275464047,IE 3275464048,3275468655,GB @@ -88412,6 +90707,7 @@ 3275510624,3275510655,EE 3275510656,3275510687,FR 3275510720,3275510751,GB +3275510752,3275510783,NL 3275510784,3275510911,RU 3275510912,3275511167,GB 3275511168,3275511295,LV @@ -88650,6 +90946,7 @@ 3275628032,3275628543,IL 3275628544,3275629567,RU 3275629568,3275630079,UA +3275630080,3275630591,RU 3275630592,3275631103,PL 3275631104,3275631615,FR 3275631616,3275632127,DE @@ -88695,6 +90992,7 @@ 3275789312,3275789823,UA 3275789824,3275790335,PL 3275790336,3275790847,MD +3275790848,3275791359,UA 3275791360,3275791871,RU 3275791872,3275792383,PL 3275792384,3275792895,BG @@ -88780,6 +91078,7 @@ 3275914496,3275914751,BG 3275914752,3275915007,RU 3275915008,3275915263,PT +3275915264,3275915775,SA 3275915776,3275916287,SE 3275916288,3275916799,US 3275916800,3275917311,NL @@ -88843,9 +91142,13 @@ 3276014256,3276014263,GB 3276014264,3276014471,FR 3276014472,3276014495,GB -3276014496,3276014887,FR +3276014496,3276014775,FR +3276014776,3276014783,GB +3276014784,3276014887,FR 3276014888,3276014895,IE -3276014896,3276015103,FR +3276014896,3276014943,FR +3276014944,3276014951,GB +3276014952,3276015103,FR 3276015104,3276015119,GB 3276015120,3276015199,FR 3276015200,3276015231,GB @@ -88853,11 +91156,13 @@ 3276015248,3276015263,GB 3276015264,3276015295,FR 3276015296,3276015327,GB -3276015328,3276015415,FR +3276015328,3276015383,FR +3276015384,3276015391,GB +3276015392,3276015415,FR 3276015416,3276015423,GB 3276015424,3276015559,FR -3276015560,3276015583,GB -3276015584,3276015879,FR +3276015560,3276015599,GB +3276015600,3276015879,FR 3276015880,3276015887,GB 3276015888,3276016391,FR 3276016392,3276016399,GB @@ -88924,8 +91229,8 @@ 3276019544,3276019559,FR 3276019560,3276019599,GB 3276019600,3276019639,FR -3276019640,3276019647,GB -3276019648,3276019679,FR +3276019640,3276019663,GB +3276019664,3276019679,FR 3276019680,3276019687,GB 3276019688,3276019695,FR 3276019696,3276019703,GB @@ -88937,7 +91242,9 @@ 3276019864,3276019887,GB 3276019888,3276019919,FR 3276019920,3276019927,GB -3276019928,3276020407,FR +3276019928,3276019959,FR +3276019960,3276019967,GB +3276019968,3276020407,FR 3276020408,3276020415,GB 3276020416,3276020503,FR 3276020504,3276020511,GB @@ -88994,8 +91301,8 @@ 3276023952,3276023959,FR 3276023960,3276023967,GB 3276023968,3276024719,FR -3276024720,3276024831,GB -3276024832,3276025111,FR +3276024720,3276024767,GB +3276024768,3276025111,FR 3276025112,3276025135,GB 3276025136,3276025151,FR 3276025152,3276025159,GB @@ -89175,17 +91482,9 @@ 3276033880,3276033887,GB 3276033888,3276033927,FR 3276033928,3276033951,GB -3276033952,3276036111,FR -3276036112,3276036119,GB -3276036120,3276036191,FR -3276036192,3276036207,GB -3276036208,3276036223,FR -3276036224,3276036255,GB -3276036256,3276036271,FR -3276036272,3276036287,GB -3276036288,3276036303,FR -3276036304,3276036335,GB -3276036336,3276036343,FR +3276033952,3276036271,FR +3276036272,3276036279,DE +3276036280,3276036343,FR 3276036344,3276036351,GB 3276036352,3276036383,FR 3276036384,3276036415,GB @@ -89225,9 +91524,7 @@ 3276039552,3276039567,GB 3276039568,3276039647,FR 3276039648,3276039663,GB -3276039664,3276039943,FR -3276039944,3276039951,GB -3276039952,3276039967,FR +3276039664,3276039967,FR 3276039968,3276040031,GB 3276040032,3276040063,FR 3276040064,3276040095,GB @@ -89424,6 +91721,7 @@ 3276133376,3276134399,GB 3276134400,3276135423,RU 3276135424,3276136447,UA +3276136448,3276137471,NL 3276137472,3276138495,PL 3276138496,3276139519,NL 3276139520,3276140543,IL @@ -91597,9 +93895,7 @@ 3276480096,3276480111,EU 3276480112,3276480159,FR 3276480160,3276480191,EU -3276480192,3276480295,FR -3276480296,3276480303,EU -3276480304,3276480319,FR +3276480192,3276480319,FR 3276480320,3276480351,EU 3276480352,3276480375,FR 3276480376,3276480399,EU @@ -91643,9 +93939,7 @@ 3276492128,3276492143,EU 3276492144,3276492151,GB 3276492152,3276492287,EU -3276492288,3276492487,GB -3276492488,3276492495,EU -3276492496,3276492831,GB +3276492288,3276492831,GB 3276492832,3276492839,EU 3276492840,3276493215,GB 3276493216,3276493247,EU @@ -91672,9 +93966,7 @@ 3276497664,3276497919,DE 3276497920,3276498047,GB 3276498048,3276498303,EU -3276498304,3276498311,GB -3276498312,3276498319,EU -3276498320,3276498431,GB +3276498304,3276498431,GB 3276498432,3276499199,DE 3276499200,3276499455,EU 3276499456,3276499503,DE @@ -91733,8 +94025,8 @@ 3276508928,3276509183,EU 3276509184,3276510207,IT 3276510208,3276510719,EU -3276510720,3276510735,IT -3276510736,3276510783,EU +3276510720,3276510751,IT +3276510752,3276510783,EU 3276510784,3276510815,IT 3276510816,3276510847,EU 3276510848,3276510911,GB @@ -91809,7 +94101,9 @@ 3276520832,3276520927,SE 3276520928,3276520943,EU 3276520944,3276521215,SE -3276521216,3276521983,EU +3276521216,3276521471,EU +3276521472,3276521479,RO +3276521480,3276521983,EU 3276521984,3276523519,NL 3276523520,3276523791,EU 3276523792,3276523911,NO @@ -92388,7 +94682,6 @@ 3276799872,3276799999,NL 3276800000,3276824575,GB 3276824576,3276832767,EE -3276832768,3276834815,GB 3276836864,3276837887,RU 3276837888,3276838911,EU 3276838912,3276840959,FR @@ -92441,7 +94734,9 @@ 3276875776,3276876031,GB 3276876032,3276876287,DK 3276876288,3276876415,NL -3276876416,3276876799,GB +3276876416,3276876431,GB +3276876432,3276876479,NL +3276876480,3276876799,GB 3276876800,3276876823,AT 3276876824,3276877303,GB 3276877304,3276877311,AT @@ -92451,18 +94746,18 @@ 3276877560,3276877567,AT 3276877568,3276877815,GB 3276877816,3276877823,AT -3276877824,3276877831,BG -3276877832,3276878079,GB +3276877824,3276878079,GB 3276878080,3276878335,BG -3276878336,3276878847,GB +3276878336,3276878591,GB +3276878592,3276878847,FR 3276878848,3276879359,ES 3276879360,3276879423,TR 3276879424,3276879615,GB 3276879616,3276879871,TR -3276879872,3276880639,DK -3276880640,3276880895,GB +3276879872,3276880895,DK 3276880896,3276881151,DE -3276881152,3276881407,GB +3276881152,3276881279,GB +3276881280,3276881407,DE 3276881408,3276881919,FR 3276881920,3276883391,IT 3276883392,3276883455,GB @@ -92474,8 +94769,8 @@ 3276884992,3276886015,GB 3276886016,3276886271,RO 3276886272,3276886527,GB -3276886528,3276886911,DE -3276886912,3276887071,GB +3276886528,3276886943,DE +3276886944,3276887071,GB 3276887072,3276888063,DE 3276888064,3276888575,GB 3276888576,3276888831,AT @@ -92493,13 +94788,14 @@ 3276893696,3276894207,GB 3276894208,3276895999,IT 3276896000,3276896255,CZ -3276896256,3276896831,BE +3276896256,3276896767,BE +3276896768,3276896831,GB 3276896832,3276896847,SE 3276896848,3276896895,GB 3276896896,3276896927,BE 3276896928,3276897023,GB -3276897024,3276897279,BE -3276897280,3276898047,GB +3276897024,3276897663,BE +3276897664,3276898047,GB 3276898048,3276898303,CZ 3276898304,3276898775,CH 3276898776,3276898783,GB @@ -92507,9 +94803,7 @@ 3276900040,3276900047,GB 3276900048,3276900351,CH 3276900352,3276900607,GB -3276900608,3276900855,CH -3276900856,3276900863,GB -3276900864,3276901391,CH +3276900608,3276901391,CH 3276901392,3276901407,GB 3276901408,3276901439,CH 3276901440,3276901631,GB @@ -92523,8 +94817,8 @@ 3276902584,3276902615,GB 3276902616,3276902639,SE 3276902640,3276902655,GB -3276902656,3276903311,SE -3276903312,3276903423,GB +3276902656,3276903319,SE +3276903320,3276903423,GB 3276903424,3276903935,SE 3276903936,3276904191,GB 3276904192,3276904447,SE @@ -92546,14 +94840,12 @@ 3276907552,3276907775,GB 3276907776,3276908159,NL 3276908160,3276908287,GB -3276908288,3276908543,NL -3276908544,3276909567,GB +3276908288,3276908799,NL +3276908800,3276909567,GB 3276909568,3276910591,NL 3276910592,3276910967,IT 3276910968,3276910975,GB -3276910976,3276912255,IT -3276912256,3276912287,GB -3276912288,3276912615,IT +3276910976,3276912615,IT 3276912616,3276912623,GB 3276912624,3276913183,IT 3276913184,3276913215,GB @@ -92565,8 +94857,10 @@ 3276914688,3276914943,ES 3276914944,3276915199,GB 3276915200,3276915567,ES -3276915568,3276915583,GB -3276915584,3276916175,ES +3276915568,3276915583,NL +3276915584,3276916079,ES +3276916080,3276916095,GB +3276916096,3276916175,ES 3276916176,3276916183,GB 3276916184,3276917231,ES 3276917232,3276917247,GB @@ -92596,8 +94890,8 @@ 3276923432,3276923439,GB 3276923440,3276924071,FR 3276924072,3276924079,GB -3276924080,3276926735,FR -3276926736,3276926783,GB +3276924080,3276926751,FR +3276926752,3276926783,GB 3276926784,3276926847,FR 3276926848,3276931071,GB 3276931072,3276939263,KZ @@ -92941,7 +95235,7 @@ 3277823744,3277823999,UA 3277824000,3277824255,TR 3277824256,3277824511,SI -3277824768,3277825023,AT +3277824512,3277825023,AT 3277825024,3277825279,PL 3277825280,3277825535,UA 3277825536,3277826047,RO @@ -93067,7 +95361,7 @@ 3278769152,3278769663,FR 3278769664,3278770175,DE 3278770176,3278770687,DK -3278770688,3278771199,DE +3278770688,3278771711,DE 3278771712,3278772223,NL 3278772224,3278772735,BG 3278772736,3278773247,DE @@ -95446,9 +97740,7 @@ 3278945833,3278945833,GB 3278945834,3278945834,NL 3278945835,3278945835,CH -3278945836,3278945838,DE -3278945839,3278945839,CZ -3278945840,3278945840,DE +3278945836,3278945840,DE 3278945841,3278945841,CH 3278945842,3278945844,DE 3278945845,3278945845,ES @@ -96084,7 +98376,7 @@ 3280583424,3280583679,RU 3280583680,3280583935,PS 3280583936,3280584191,PL -3280584192,3280584447,RO +3280584192,3280584447,KW 3280584448,3280584703,DE 3280584704,3280585215,UA 3280585216,3280585727,DK @@ -96188,9 +98480,7 @@ 3280796672,3280797695,CZ 3280797696,3280810783,CH 3280810784,3280810799,DE -3280810800,3280844292,CH -3280844293,3280844295,GB -3280844296,3280863231,CH +3280810800,3280863231,CH 3280863232,3280928767,TR 3280928768,3280928768,GB 3280928769,3280928769,DE @@ -97672,6 +99962,7 @@ 3281343232,3281343487,FI 3281343488,3281343743,GB 3281343744,3281343999,TR +3281344000,3281344255,FR 3281344256,3281344511,RU 3281344512,3281344767,UA 3281344768,3281345023,RU @@ -97843,22 +100134,15 @@ 3282206720,3282223103,CH 3282223104,3282231295,BE 3282231296,3282239487,DE -3282239488,3282242223,SE -3282242224,3282242227,NO -3282242228,3282284543,SE -3282284544,3282284551,FI -3282284552,3282284567,SE -3282284568,3282284575,FR -3282284576,3282284579,FI +3282239488,3282284567,SE +3282284568,3282284579,FI 3282284580,3282284591,SE -3282284592,3282284595,FR +3282284592,3282284595,FI 3282284596,3282284599,SE -3282284600,3282284603,FR +3282284600,3282284603,FI 3282284604,3282284735,SE 3282284736,3282284783,FI -3282284784,3282286975,SE -3282286976,3282287007,FI -3282287008,3282287615,SE +3282284784,3282287615,SE 3282287616,3282287759,FI 3282287760,3282305023,SE 3282305024,3282370559,PL @@ -98784,6 +101068,7 @@ 3284041984,3284042239,SI 3284042240,3284042495,GB 3284042496,3284042751,AT +3284042752,3284043007,RU 3284043008,3284043263,DE 3284043264,3284043519,EE 3284043520,3284043775,DE @@ -98909,6 +101194,7 @@ 3284127232,3284127743,NL 3284127744,3284128255,RU 3284128256,3284128767,GB +3284128768,3284129279,SE 3284129280,3284129791,UA 3284129792,3284130303,DE 3284130304,3284130815,GB @@ -99116,7 +101402,6 @@ 3285388288,3285389311,CH 3285389312,3285390335,FI 3285390336,3285391359,RU -3285391360,3285392383,AM 3285392384,3285393407,RU 3285393408,3285394431,AT 3285394432,3285396479,RU @@ -99163,11 +101448,7 @@ 3285453056,3285453119,EU 3285453120,3285453415,GB 3285453416,3285453423,EU -3285453424,3285454047,GB -3285454048,3285454079,EU -3285454080,3285454175,GB -3285454176,3285454207,EU -3285454208,3285454847,GB +3285453424,3285454847,GB 3285454848,3285455695,DE 3285455696,3285455711,EU 3285455712,3285455743,DE @@ -99184,7 +101465,8 @@ 3285456352,3285456383,EU 3285456384,3285456639,DE 3285456640,3285456703,DK -3285456704,3285456735,EU +3285456704,3285456711,DE +3285456712,3285456735,EU 3285456736,3285456743,DK 3285456744,3285456751,EU 3285456752,3285456767,DK @@ -99204,17 +101486,15 @@ 3285457664,3285457759,IT 3285457760,3285457791,EU 3285457792,3285457919,IT -3285457920,3285457951,GB -3285457952,3285457983,EU -3285457984,3285458095,GB -3285458096,3285458103,EU -3285458104,3285458111,GB +3285457920,3285457967,GB +3285457968,3285457983,EU +3285457984,3285458111,GB 3285458112,3285458175,EU 3285458176,3285458943,GB 3285458944,3285458975,DK 3285458976,3285459007,EU -3285459008,3285459103,DK -3285459104,3285459119,EU +3285459008,3285459079,DK +3285459080,3285459119,EU 3285459120,3285459327,DK 3285459328,3285459391,EU 3285459392,3285459535,DK @@ -99318,9 +101598,7 @@ 3285466624,3285466879,AT 3285466880,3285466895,BG 3285466896,3285466911,EU -3285466912,3285466943,BG -3285466944,3285466951,EU -3285466952,3285466959,BG +3285466912,3285466959,BG 3285466960,3285466983,EU 3285466984,3285467007,BG 3285467008,3285467015,EU @@ -99418,8 +101696,8 @@ 3285477024,3285477087,AT 3285477088,3285477151,EU 3285477152,3285477319,IT -3285477320,3285477335,EU -3285477336,3285477343,IT +3285477320,3285477327,EU +3285477328,3285477343,IT 3285477344,3285477359,EU 3285477360,3285477375,IT 3285477376,3285477631,FR @@ -99481,8 +101759,8 @@ 3285487616,3285487679,NL 3285487680,3285487743,EU 3285487744,3285488127,NL -3285488128,3285488895,EU -3285488896,3285489663,GB +3285488128,3285488639,EU +3285488640,3285489663,GB 3285489664,3285490175,EU 3285490176,3285490319,RU 3285490320,3285490335,EU @@ -99530,9 +101808,7 @@ 3285496528,3285496543,EU 3285496544,3285496575,FR 3285496576,3285496607,EU -3285496608,3285496815,DE -3285496816,3285496831,EU -3285496832,3285497855,DE +3285496608,3285497855,DE 3285497856,3285497887,EU 3285497888,3285498031,DE 3285498032,3285498047,EU @@ -100507,7 +102783,8 @@ 3285916712,3285916719,CY 3285916720,3285916735,ES 3285916736,3285916799,GB -3285916800,3285917183,EU +3285916800,3285916927,EU +3285916928,3285917183,GB 3285917184,3285917439,ES 3285917440,3285917695,YE 3285917696,3285918719,EU @@ -100517,10 +102794,9 @@ 3285919488,3285919743,UA 3285919744,3285921791,QA 3285921792,3285921823,CZ -3285921824,3285921983,GB -3285921984,3285922111,EU +3285921824,3285922111,GB 3285922112,3285922175,FR -3285922176,3285922183,EU +3285922176,3285922183,GB 3285922184,3285922191,RU 3285922192,3285922207,DE 3285922208,3285922239,ES @@ -100539,7 +102815,8 @@ 3285926408,3285926415,ES 3285926416,3285926431,GB 3285926432,3285926463,CH -3285926464,3285926847,EU +3285926464,3285926591,GB +3285926592,3285926847,EU 3285926848,3285926911,GB 3285926912,3285927423,DE 3285927424,3285927679,GB @@ -100551,8 +102828,8 @@ 3285928224,3285928255,DE 3285928256,3285928271,GB 3285928272,3285928287,DE -3285928288,3285928303,GB -3285928304,3285928447,EU +3285928288,3285928319,GB +3285928320,3285928447,EU 3285928448,3285928959,ES 3285928960,3285929983,EU 3285929984,3285930495,GB @@ -100601,8 +102878,7 @@ 3285942784,3285943039,ES 3285943040,3285943295,SE 3285943296,3285943551,ES -3285943552,3285943559,GB -3285943560,3285943567,EU +3285943552,3285943567,GB 3285943568,3285943575,ES 3285943576,3285943583,DE 3285943584,3285943615,GB @@ -101154,6 +103430,7 @@ 3287450624,3287451135,SE 3287451136,3287451647,PL 3287451648,3287452159,ES +3287452160,3287452671,PL 3287452672,3287453183,CH 3287453184,3287454207,RU 3287454208,3287454719,SE @@ -101295,7 +103572,7 @@ 3287664896,3287665151,GB 3287665152,3287665407,NO 3287665408,3287665919,PL -3287666176,3287666431,RU +3287665920,3287666431,RU 3287666432,3287666687,DK 3287666688,3287666943,SI 3287666944,3287667199,CZ @@ -101308,6 +103585,7 @@ 3287668736,3287668991,SI 3287668992,3287669247,PL 3287669248,3287669503,LI +3287669504,3287669759,MD 3287669760,3287670015,RO 3287670016,3287670271,UA 3287670272,3287670527,PL @@ -101501,7 +103779,6 @@ 3287950080,3287950335,DE 3287950336,3287950591,UA 3287950592,3287950847,PL -3287950848,3287951103,DE 3287951104,3287951359,HU 3287951360,3287951615,FR 3287951616,3287951871,CH @@ -101823,7 +104100,10 @@ 3291203584,3291203839,ZA 3291203840,3291204095,EG 3291204096,3291204351,KE -3291204352,3291204607,ZA +3291204352,3291204863,ZA +3291204864,3291205119,KE +3291205120,3291205631,GH +3291205632,3291205887,ZA 3291217920,3291230207,ZA 3291230208,3291234303,GH 3291234304,3291242495,ZA @@ -102001,7 +104281,9 @@ 3302760448,3302768639,ZA 3302768640,3302776831,NG 3302776832,3302785023,ZW -3302801408,3302817791,NG +3302801408,3302805503,NG +3302805504,3302809599,MW +3302809600,3302817791,NG 3302817792,3302883327,EG 3302883328,3302948863,MA 3302948864,3302949119,MU @@ -102022,8 +104304,10 @@ 3302953728,3302953983,AO 3302953984,3302954239,SD 3302954240,3302954495,KE +3302954496,3302955007,ZA +3302955008,3302955263,LS 3305111552,3307208703,TN -3311403008,3311927295,ZA +3309305856,3313500159,ZA 3313500160,3313762303,MA 3313762304,3314024447,EG 3314024448,3314286591,KE @@ -102035,16 +104319,19 @@ 3318218752,3318743039,DZ 3319529472,3319537663,ZM 3319537664,3319545855,UG +3319652352,3319660543,ZW 3319660544,3319791615,EG 3319791616,3320053759,MU 3320578048,3320643583,ZA 3320643584,3320709119,KE +3321724928,3321757695,MA 3321757696,3321790463,KE 3321790464,3321806847,LS 3321806848,3321823231,SD 3321823232,3321839615,NG 3321839616,3321855999,GH -3321886720,3321887743,MU +3321856000,3321860095,CV +3321860096,3321864191,ZA 3321954304,3321970687,US 3322019840,3322023935,US 3322023936,3322028031,CL @@ -102472,7 +104759,32 @@ 3330778880,3330779135,GB 3330779136,3330791423,US 3330791424,3330791679,CA -3330791680,3331102463,US +3330791680,3330883583,US +3330883584,3330884351,NL +3330884352,3330884863,US +3330884864,3330885119,GB +3330885120,3330888063,US +3330888064,3330888127,NL +3330888128,3330888191,US +3330888192,3330888703,AU +3330888704,3330888959,US +3330888960,3330889215,CH +3330889216,3330889471,GB +3330889472,3330890239,JP +3330890240,3330890751,BE +3330890752,3330891263,GB +3330891264,3330892287,US +3330892288,3330892543,FR +3330892544,3330893567,US +3330893568,3330893823,NL +3330893824,3330894079,CA +3330894080,3330894591,GB +3330894592,3330894847,DE +3330894848,3330897919,US +3330897920,3330898175,CA +3330898176,3330898431,DE +3330898432,3330898943,FR +3330898944,3331102463,US 3331102464,3331102719,CA 3331102720,3331194879,US 3331194880,3331260415,AU @@ -102943,7 +105255,9 @@ 3341778944,3341807615,US 3341807616,3341808639,CA 3341808640,3341828095,US -3341844480,3341863935,US +3341844480,3341854551,US +3341854552,3341854559,SG +3341854560,3341863935,US 3341863936,3341864959,AG 3341864960,3341867007,US 3341867008,3341869055,CA @@ -102962,7 +105276,9 @@ 3342526464,3342528511,CA 3342528512,3342548991,US 3342565376,3342567423,CA -3342567424,3342579711,US +3342567424,3342569471,US +3342569472,3342569727,CR +3342569728,3342579711,US 3342579712,3342581759,CA 3342581760,3342595071,US 3342595072,3342595839,CA @@ -103049,8 +105365,22 @@ 3345007616,3345008639,CA 3345008640,3345010687,US 3345010688,3345011711,CA -3345011712,3345367039,US -3345416192,3345418239,US +3345011712,3345301503,US +3345317888,3345327103,US +3345327104,3345328127,CA +3345328128,3345333247,US +3345333248,3345334271,MF +3345334272,3345339391,US +3345339392,3345340415,CA +3345340416,3345367039,US +3345383424,3345384447,CA +3345384448,3345390591,US +3345390592,3345391615,CA +3345391616,3345398783,US +3345398784,3345399807,BS +3345399808,3345401855,PR +3345401856,3345403903,CA +3345403904,3345418239,US 3345418240,3345419519,NL 3345419520,3345422847,US 3345423360,3345424383,TC @@ -103809,11 +106139,13 @@ 3356380928,3356381183,DO 3356381184,3356381439,CL 3356381440,3356381695,PA +3356381696,3356382207,UY 3356382208,3356389375,CL 3356389376,3356389887,CO 3356389888,3356390399,CL 3356390400,3356390655,AR 3356390656,3356391167,CL +3356391168,3356391423,PA 3356391424,3356393471,CO 3356393472,3356413183,CL 3356413184,3356413951,CO @@ -103827,11 +106159,14 @@ 3356499968,3356508159,MX 3356508160,3356508671,AR 3356508672,3356509183,CR +3356509184,3356510207,VE 3356510208,3356514303,AR 3356514304,3356514559,US +3356514560,3356514815,AR 3356516352,3356520447,AR 3356520448,3356521471,CL 3356521472,3356521727,AR +3356521728,3356522495,CO 3356522496,3356557311,AR 3356557312,3356819455,BR 3356819456,3356884991,CO @@ -104615,6 +106950,7 @@ 3363569664,3363577855,PA 3363577856,3363586047,CL 3363586048,3363594239,BZ +3363594240,3363598335,AR 3363602432,3363610623,AN 3363610624,3363614719,CO 3363618816,3363622911,UY @@ -104633,6 +106969,11 @@ 3363733504,3363831807,AR 3363831808,3378511871,BR 3378511872,3383754751,MX +3384279040,3384410111,CO +3384541184,3384672255,CL +3384672256,3384688639,HN +3384688640,3384705023,CO +3384705024,3384737791,PA 3384737792,3385851903,CR 3385851904,3386114047,VE 3386114048,3386245119,AR @@ -105810,8 +108151,7 @@ 3393859328,3393859583,JP 3393859584,3393860095,AU 3393860096,3393860607,HK -3393860608,3393860863,AU -3393860864,3393861631,IN +3393860608,3393861631,IN 3393861632,3393861887,VN 3393861888,3393862143,AU 3393862144,3393862655,VN @@ -107411,11 +109751,16 @@ 3406434304,3406436351,MY 3406436352,3406445055,AU 3406445056,3406445311,MY -3406445312,3406542847,AU +3406445312,3406512383,AU +3406512384,3406512639,IN +3406512640,3406514687,AU +3406514688,3406514943,TH +3406514944,3406542847,AU 3406542848,3406543103,SG 3406543104,3406565887,AU 3406565888,3406566143,PH 3406566144,3406566399,AU +3406566400,3406566911,ID 3406566912,3406591487,AU 3406591488,3406591743,NP 3406591744,3406617599,AU @@ -107424,7 +109769,9 @@ 3406618112,3406618623,IN 3406618624,3406625023,AU 3406625024,3406625279,NF -3406625280,3406669823,AU +3406625280,3406637055,AU +3406637056,3406637311,IN +3406637312,3406669823,AU 3406669824,3406670847,IN 3406670848,3406696959,AU 3406696960,3406697215,IN @@ -107436,16 +109783,29 @@ 3406739200,3406739455,ID 3406739456,3406746623,AU 3406746624,3406746879,HK -3406746880,3406865663,AU +3406746880,3406832127,AU +3406832128,3406832383,PK +3406832384,3406865663,AU 3406865664,3406865919,IN -3406865920,3406946815,AU +3406865920,3406884607,AU +3406884608,3406884863,HK +3406884864,3406894335,AU +3406894336,3406894591,ID +3406894592,3406896895,AU +3406896896,3406897151,IN +3406897152,3406938623,AU +3406938624,3406938879,IN +3406938880,3406946815,AU 3406946816,3406947071,KR 3406947072,3406950399,AU 3406950400,3406951423,NF -3406951424,3406961151,AU +3406951424,3406952703,AU +3406952704,3406952959,PH +3406952960,3406961151,AU 3406961152,3406961407,IN 3406961408,3406967295,AU 3406967296,3406967551,CN +3406967552,3406967807,IN 3406967808,3406989567,AU 3406989568,3406989823,IN 3406989824,3407020287,AU @@ -107499,11 +109859,17 @@ 3407159232,3407159239,NZ 3407159240,3407159263,AU 3407159264,3407159295,NZ -3407159296,3407243263,AU +3407159296,3407161599,AU +3407161856,3407162367,TH +3407162368,3407170047,AU +3407170048,3407170559,ID +3407170560,3407243263,AU 3407243264,3407243775,HK 3407243776,3407268863,AU 3407268864,3407269119,US -3407269120,3407367167,AU +3407269120,3407360511,AU +3407360512,3407361023,ID +3407361024,3407367167,AU 3407367168,3407367679,ID 3407367680,3407367935,AU 3407368192,3407369983,AU @@ -107512,13 +109878,17 @@ 3407498496,3407498751,PK 3407498752,3407524607,AU 3407524608,3407524863,NZ -3407524864,3407602943,AU +3407524864,3407545855,AU +3407545856,3407546367,ID +3407546368,3407602943,AU 3407602944,3407603199,JP 3407603200,3407604479,AU 3407604480,3407604735,IN 3407604736,3407608715,AU 3407608716,3407608736,JP -3407608737,3407682047,AU +3407608737,3407642623,AU +3407642624,3407643135,TH +3407643136,3407682047,AU 3407682048,3407682559,ID 3407682560,3407732223,AU 3407732224,3407732479,HK @@ -107532,7 +109902,9 @@ 3407801344,3407801855,ID 3407801856,3407805951,AU 3407805952,3407806463,ID -3407806464,3407814655,AU +3407806464,3407808511,AU +3407808512,3407809023,IN +3407809024,3407814655,AU 3407814656,3407815167,HK 3407815168,3407828991,AU 3407828992,3407829503,US @@ -109250,7 +111622,6 @@ 3419512832,3419516927,AU 3419516928,3419517951,JP 3419517952,3419518975,VN -3419518976,3419519999,TW 3419520000,3419520767,ID 3419520768,3419521023,TH 3419521024,3419529215,AU @@ -109523,7 +111894,9 @@ 3423161480,3423161487,HK 3423161488,3423161613,US 3423161614,3423161621,CA -3423161622,3423182847,US +3423161622,3423162303,US +3423162304,3423162311,GB +3423162312,3423182847,US 3423182848,3423183199,CA 3423183200,3423183231,BM 3423183232,3423183263,CA @@ -109563,7 +111936,7 @@ 3423236096,3423238143,JM 3423238144,3423258623,US 3423258624,3423260671,CA -3423262720,3423264831,US +3423260672,3423264831,US 3423264832,3423264863,NG 3423264864,3423265247,US 3423265248,3423265263,EC @@ -109761,7 +112134,9 @@ 3423365776,3423365783,CA 3423365784,3423365903,US 3423365904,3423365911,AE -3423365912,3423366223,US +3423365912,3423366127,US +3423366128,3423366135,SA +3423366136,3423366223,US 3423366224,3423366239,AU 3423366240,3423366295,US 3423366296,3423366303,CA @@ -109771,7 +112146,9 @@ 3423366480,3423366495,IT 3423366496,3423366703,US 3423366704,3423366711,GB -3423366712,3423366991,US +3423366712,3423366855,US +3423366856,3423366871,BR +3423366872,3423366991,US 3423366992,3423366999,CA 3423367000,3423367223,US 3423367224,3423367231,AU @@ -109833,8 +112210,8 @@ 3423401184,3423401191,RU 3423401192,3423402943,US 3423402944,3423402951,RU -3423402952,3423412223,US -3423414272,3423416319,US +3423402952,3423416319,US +3423416320,3423417343,CA 3423417344,3423417470,US 3423417471,3423417480,AU 3423417481,3423462655,US @@ -109878,8 +112255,7 @@ 3423544512,3423544543,CA 3423544544,3423544551,US 3423544552,3423545343,CA -3423545344,3423549439,US -3423551488,3423554183,US +3423545344,3423554183,US 3423554184,3423554191,CA 3423554192,3423554271,US 3423554272,3423554279,GB @@ -109960,7 +112336,9 @@ 3423653888,3423705599,US 3423705600,3423705855,CA 3423705856,3423797247,US -3423797248,3423827711,CA +3423797248,3423823359,CA +3423823360,3423823871,US +3423823872,3423827711,CA 3423827712,3423827967,US 3423827968,3423848447,CA 3423848448,3423849471,KN @@ -110002,7 +112380,12 @@ 3426013184,3426013439,IL 3426013440,3426387967,US 3426387968,3426388991,MX -3426388992,3426646015,US +3426388992,3426617855,US +3426618368,3426618687,US +3426618688,3426618703,NZ +3426618704,3426618911,US +3426618912,3426618943,NZ +3426618944,3426646015,US 3426646016,3426647039,CA 3426647040,3426680831,US 3426680832,3426680959,KN @@ -110063,8 +112446,7 @@ 3427750912,3427751423,US 3427751424,3427751519,BE 3427751520,3427752447,US -3427752448,3427752703,NL -3427752704,3427752959,GB +3427752448,3427752959,GB 3427752960,3427753215,US 3427753216,3427753471,JP 3427753472,3427753727,FI @@ -110235,7 +112617,9 @@ 3428587776,3428588287,CA 3428588288,3428588543,US 3428588544,3428589055,CA -3428589056,3428591623,US +3428589056,3428589311,US +3428589312,3428589567,CA +3428589568,3428591623,US 3428591624,3428591663,CA 3428591664,3428591679,US 3428591680,3428591695,CA @@ -110339,8 +112723,8 @@ 3428607744,3428607871,CA 3428607872,3428607999,US 3428608000,3428609023,CA -3428609024,3428610303,US -3428610304,3428611071,CA +3428609024,3428610047,US +3428610048,3428611071,CA 3428611072,3428611583,US 3428611584,3428611839,CA 3428611840,3428612607,US @@ -110356,8 +112740,8 @@ 3428623616,3428623871,US 3428623872,3428624639,CA 3428624640,3428625407,US -3428625408,3428627199,CA -3428627200,3428630015,US +3428625408,3428628223,CA +3428628224,3428630015,US 3428630016,3428634623,CA 3428634624,3428634879,US 3428634880,3428635135,CA @@ -110435,7 +112819,9 @@ 3429411584,3429411839,DE 3429411840,3429500927,US 3429500928,3429502975,A2 -3429502976,3429775359,US +3429502976,3429774599,US +3429774600,3429774607,BD +3429774608,3429775359,US 3429775360,3429777407,TC 3429777408,3429892095,US 3429892096,3429957631,CA @@ -110498,7 +112884,7 @@ 3430807296,3430807551,CA 3430807552,3430809087,US 3430809088,3430809343,CA -3430809344,3430812159,US +3430809344,3430812671,US 3430812672,3430813183,MX 3430813184,3430842367,US 3430842368,3430842879,DO @@ -110508,12 +112894,18 @@ 3431114496,3431114751,CA 3431114752,3431468031,US 3431468032,3431469055,CA -3431469056,3431596031,US -3431596032,3431613439,CA +3431469056,3431596287,US +3431596288,3431602687,CA +3431602688,3431602943,US +3431602944,3431613439,CA 3431613440,3431613695,US -3431613696,3431621631,CA -3431621632,3431622143,US -3431622400,3431624703,CA +3431613696,3431620095,CA +3431620096,3431620351,US +3431620352,3431621375,CA +3431621376,3431622399,US +3431622400,3431622655,CA +3431622656,3431622911,US +3431622912,3431624703,CA 3431624704,3431624959,US 3431624960,3431638783,CA 3431638784,3431639039,US @@ -110662,7 +113054,9 @@ 3435271424,3435271679,CA 3435271680,3435507711,US 3435507712,3435511807,CA -3435511808,3436249343,US +3435511808,3435518463,US +3435518464,3435518527,GI +3435518528,3436249343,US 3436249344,3436255743,CA 3436255744,3436256255,US 3436256256,3436282367,CA @@ -110709,7 +113103,9 @@ 3437332480,3437332735,US 3437332736,3437334015,CA 3437334016,3437334271,US -3437334272,3437336063,CA +3437334272,3437334783,CA +3437334784,3437335551,US +3437335552,3437336063,CA 3437336064,3437336319,US 3437336320,3437341695,CA 3437341696,3437341951,US @@ -110794,7 +113190,9 @@ 3437756160,3437756415,IE 3437756416,3437772799,US 3437772800,3437776895,CA -3437776896,3437789863,US +3437776896,3437789847,US +3437789848,3437789855,IN +3437789856,3437789863,US 3437789864,3437789871,AU 3437789872,3437790143,US 3437790144,3437790159,CA @@ -112707,7 +115105,9 @@ 3461514752,3461516287,CA 3461516288,3461516543,IL 3461516544,3461517055,CA -3461517056,3461554175,US +3461517056,3461532607,US +3461532608,3461532623,CA +3461532624,3461554175,US 3461554176,3461556223,CA 3461558272,3461597887,US 3461597888,3461597951,CO @@ -112882,7 +115282,9 @@ 3464185760,3464185791,PR 3464185792,3464190463,US 3464190464,3464190719,CA -3464190720,3464191799,US +3464190720,3464191775,US +3464191776,3464191783,ES +3464191784,3464191799,US 3464191800,3464191807,AF 3464191808,3464191815,US 3464191816,3464191823,CA @@ -113441,9 +115843,7 @@ 3468628512,3468628607,US 3468628608,3468628927,CA 3468628928,3468628991,US -3468628992,3468629311,CA -3468629312,3468629319,US -3468629320,3468631583,CA +3468628992,3468631583,CA 3468631584,3468631615,US 3468631616,3468631679,CA 3468631680,3468631695,US @@ -113967,7 +116367,8 @@ 3470188522,3470188541,HK 3470188542,3470192639,US 3470192640,3470196735,CA -3470196736,3470360631,US +3470196736,3470360623,US +3470360624,3470360631,SG 3470360632,3470360639,CA 3470360640,3470360687,US 3470360688,3470360695,AF @@ -113992,7 +116393,8 @@ 3470362320,3470362335,AF 3470362336,3470362455,US 3470362456,3470362471,CA -3470362472,3470362495,US +3470362472,3470362487,US +3470362488,3470362495,ES 3470362496,3470362503,CA 3470362504,3470362559,US 3470362560,3470362623,CA @@ -114262,11 +116664,7 @@ 3470754304,3470754559,PE 3470754560,3470755839,US 3470755840,3470756095,ZA -3470756096,3470757887,US -3470757888,3470760959,KN -3470760960,3470761215,US -3470761216,3470761727,KN -3470761728,3470761983,US +3470756096,3470761983,US 3470761984,3470762751,CA 3470762752,3470762759,NL 3470762760,3470767871,CA @@ -114372,17 +116770,7 @@ 3474548224,3474548479,JP 3474548480,3474623599,US 3474623600,3474623615,CA -3474623616,3474841599,US -3474841600,3474843071,KN -3474843072,3474843135,US -3474843136,3474844927,KN -3474844928,3474845055,US -3474845056,3474845439,KN -3474845440,3474845567,US -3474845568,3474846463,KN -3474846464,3474846591,US -3474846592,3474849791,KN -3474849792,3475029927,US +3474623616,3475029927,US 3475029928,3475029935,CA 3475029936,3475112191,US 3475112192,3475113215,CA @@ -114503,7 +116891,11 @@ 3476725424,3476725431,GB 3476725432,3476732373,US 3476732374,3476732377,MX -3476732378,3476881407,US +3476732378,3476733603,US +3476733604,3476733604,MX +3476733605,3476733913,US +3476733914,3476733914,MX +3476733915,3476881407,US 3476881408,3476946943,CA 3476946944,3477161503,US 3477161504,3477161519,TN @@ -115626,7 +118018,9 @@ 3492646624,3492646639,IL 3492646640,3492669695,US 3492669696,3492671487,PA -3492671488,3492795775,US +3492671488,3492730415,US +3492730416,3492730431,CA +3492730432,3492795775,US 3492795776,3492795903,AR 3492795904,3492827391,US 3492827392,3492827423,CA @@ -115674,7 +118068,8 @@ 3493901024,3493901031,HK 3493901032,3493901311,US 3493901312,3493901567,AE -3493901568,3493901599,IL +3493901568,3493901583,US +3493901584,3493901599,VG 3493901600,3493901759,US 3493901760,3493901767,AE 3493901768,3493901791,US @@ -115690,7 +118085,9 @@ 3493914368,3493916671,US 3493918720,3493936127,US 3493936128,3493937151,CA -3493937152,3493982207,US +3493937152,3493980159,US +3493980160,3493981183,CA +3493981184,3493982207,US 3493982208,3493984255,CA 3493984256,3493986303,US 3493986304,3493986623,CA @@ -115729,7 +118126,11 @@ 3494101430,3494101437,GB 3494101438,3494102623,US 3494102624,3494102639,SB -3494102640,3494102735,US +3494102640,3494102687,US +3494102688,3494102701,PE +3494102702,3494102703,US +3494102704,3494102711,CA +3494102712,3494102735,US 3494102736,3494102743,VI 3494102744,3494109925,US 3494109926,3494109942,CA @@ -115957,7 +118358,7 @@ 3494438032,3494438063,GP 3494438064,3494438071,KN 3494438072,3494438143,GP -3494438144,3494438399,VC +3494438144,3494438399,DM 3494438400,3494438655,KN 3494438656,3494438911,DM 3494438912,3494439935,CA @@ -116139,15 +118540,14 @@ 3494861096,3494862639,US 3494862640,3494862647,JP 3494862648,3494862847,US -3494862848,3494863615,VC -3494863616,3494863871,DM +3494862848,3494863871,DM 3494863872,3494866943,US 3494866944,3494867967,CA 3494867968,3494893567,US 3494893568,3494894591,CA 3494894592,3494917119,US -3494917120,3494918143,CA -3494918144,3494928383,US +3494917120,3494917631,CA +3494917632,3494928383,US 3494928384,3494930431,CA 3494930432,3494935746,US 3494935747,3494935747,GB @@ -116215,7 +118615,9 @@ 3495159808,3495159815,ES 3495159816,3495159839,US 3495159840,3495159847,BR -3495159848,3495159895,US +3495159848,3495159871,US +3495159872,3495159879,AE +3495159880,3495159895,US 3495159896,3495159903,IN 3495159904,3495160111,US 3495160112,3495160119,NZ @@ -116223,7 +118625,12 @@ 3495160304,3495160319,TR 3495160320,3495160367,US 3495160368,3495160383,ZA -3495160384,3495160535,US +3495160384,3495160447,US +3495160448,3495160455,BR +3495160456,3495160463,US +3495160464,3495160479,TR +3495160480,3495160511,CA +3495160512,3495160535,US 3495160536,3495160543,BR 3495160544,3495160639,US 3495160640,3495160647,CA @@ -116231,7 +118638,9 @@ 3495160664,3495160671,NL 3495160672,3495160863,US 3495160864,3495160895,TR -3495160896,3495161151,US +3495160896,3495161055,US +3495161056,3495161087,TR +3495161088,3495161151,US 3495161152,3495161183,TR 3495161184,3495161599,US 3495161600,3495161855,BR @@ -116377,7 +118786,17 @@ 3495476224,3495477247,CA 3495477248,3495478271,US 3495478272,3495479295,CA -3495479296,3495505919,US +3495479296,3495485599,US +3495485600,3495485615,GB +3495485616,3495485631,US +3495485632,3495485663,VG +3495485664,3495486079,US +3495486080,3495486143,PA +3495486144,3495486207,US +3495486208,3495486239,AI +3495486240,3495486335,US +3495486336,3495486399,VG +3495486400,3495505919,US 3495505920,3495507967,CA 3495507968,3495515135,US 3495515136,3495516159,CA @@ -116607,8 +119026,7 @@ 3496887808,3496887935,AU 3496887936,3496887975,US 3496887976,3496887983,AU -3496887984,3496887999,US -3496888000,3496888063,IN +3496887984,3496888063,US 3496888064,3496888095,AU 3496888096,3496888103,PT 3496888104,3496888111,GB @@ -117111,7 +119529,9 @@ 3506058816,3506058823,AF 3506058824,3506135263,US 3506135264,3506135295,GB -3506135296,3506177967,US +3506135296,3506161975,US +3506161976,3506161983,AF +3506161984,3506177967,US 3506177968,3506177983,DE 3506177984,3506192639,US 3506192640,3506192895,A2 @@ -118452,7 +120872,7 @@ 3510935552,3510943743,CA 3510943744,3511129087,US 3511129088,3511131135,CA -3511131136,3511132159,US +3511131136,3511140351,US 3511140352,3511156735,CA 3511156736,3511331199,US 3511331200,3511331231,CA @@ -118525,9 +120945,7 @@ 3512020096,3512020223,ES 3512020224,3512020479,US 3512020480,3512020511,GB -3512020512,3512020543,US -3512020544,3512020607,CA -3512020608,3512021503,US +3512020512,3512021503,US 3512021504,3512021759,GB 3512021760,3512022335,US 3512022336,3512022399,ES @@ -120910,11 +123328,16 @@ 3556974592,3556982783,ES 3556982784,3556984623,DE 3556984624,3556984639,FR -3556984640,3556985663,DE +3556984640,3556984655,DE +3556984656,3556984663,FR +3556984664,3556984671,DE +3556984672,3556984719,FR +3556984720,3556985663,DE 3556985664,3556985671,HU 3556985672,3556990975,DE 3556990976,3556999167,UA 3556999168,3557007359,RU +3557007360,3557015551,IT 3557015552,3557023743,DE 3557023744,3557023879,BE 3557023880,3557023903,GB @@ -120934,9 +123357,7 @@ 3557028032,3557028063,GB 3557028064,3557028095,BE 3557028096,3557028351,GB -3557028352,3557028543,BE -3557028544,3557028607,GB -3557028608,3557028735,BE +3557028352,3557028735,BE 3557028736,3557028799,GB 3557028800,3557028911,BE 3557028912,3557028927,GB @@ -121013,11 +123434,15 @@ 3557302272,3557310463,UA 3557310464,3557326847,ES 3557326848,3557335039,DE -3557335040,3557335391,BE +3557335040,3557335295,BE +3557335296,3557335327,EU +3557335328,3557335391,BE 3557335392,3557335455,EU 3557335456,3557335951,BE 3557335952,3557335967,LU -3557335968,3557336191,BE +3557335968,3557336063,BE +3557336064,3557336127,EU +3557336128,3557336191,BE 3557336192,3557336255,EU 3557336256,3557336319,BE 3557336320,3557336575,EU @@ -121084,8 +123509,8 @@ 3557360560,3557360575,JE 3557360576,3557360680,GB 3557360681,3557360687,JE -3557360688,3557360831,GB -3557360832,3557360895,JE +3557360688,3557360847,GB +3557360848,3557360895,JE 3557360896,3557360927,GB 3557360928,3557360943,JE 3557360944,3557360959,GB @@ -121116,7 +123541,11 @@ 3557363656,3557363663,JE 3557363664,3557363671,GB 3557363672,3557363679,JE -3557363680,3557364479,GB +3557363680,3557364103,GB +3557364104,3557364107,JE +3557364108,3557364119,GB +3557364120,3557364223,JE +3557364224,3557364479,GB 3557364480,3557364495,JE 3557364496,3557364527,GB 3557364528,3557364559,JE @@ -121238,7 +123667,9 @@ 3557851136,3557859327,HU 3557859328,3557859839,SE 3557859840,3557860095,NO -3557860096,3557860607,SE +3557860096,3557860159,SE +3557860160,3557860167,DK +3557860168,3557860607,SE 3557860608,3557860623,FI 3557860624,3557860831,SE 3557860832,3557860847,FI @@ -121427,7 +123858,9 @@ 3558288320,3558288383,CH 3558288384,3558288423,US 3558288424,3558288447,GB -3558288448,3558288483,US +3558288448,3558288471,US +3558288472,3558288479,GB +3558288480,3558288483,US 3558288484,3558288487,GB 3558288488,3558288639,US 3558288640,3558288671,GB @@ -121906,9 +124339,11 @@ 3559092160,3559092160,GB 3559092161,3559092222,BE 3559092223,3559092223,GB -3559092224,3559092295,BE -3559092296,3559092303,GB -3559092304,3559092399,BE +3559092224,3559092287,BE +3559092288,3559092303,GB +3559092304,3559092311,BE +3559092312,3559092319,GB +3559092320,3559092399,BE 3559092400,3559092407,GB 3559092408,3559092735,BE 3559092736,3559092739,GB @@ -121950,7 +124385,9 @@ 3559093240,3559093243,GB 3559093244,3559093567,BE 3559093568,3559093599,IT -3559093600,3559093687,BE +3559093600,3559093647,BE +3559093648,3559093671,GB +3559093672,3559093687,BE 3559093688,3559093699,GB 3559093700,3559094019,BE 3559094020,3559094023,GB @@ -122288,11 +124725,13 @@ 3559905298,3559905299,UA 3559905300,3559905317,EE 3559905318,3559905319,LT -3559905320,3559905323,EE -3559905324,3559905535,UA +3559905320,3559905327,EE +3559905328,3559905535,UA 3559905536,3559905623,EE 3559905624,3559905631,UA -3559905632,3559905903,EE +3559905632,3559905695,EE +3559905696,3559905703,UA +3559905704,3559905903,EE 3559905904,3559905911,UA 3559905912,3559906257,EE 3559906258,3559906259,UA @@ -124421,7 +126860,11 @@ 3560951124,3560951127,GB 3560951128,3560951131,SE 3560951132,3560951135,ES -3560951136,3560951195,SE +3560951136,3560951143,SE +3560951144,3560951147,BR +3560951148,3560951155,SE +3560951156,3560951159,US +3560951160,3560951195,SE 3560951196,3560951199,DK 3560951200,3560951203,SE 3560951204,3560951207,AE @@ -124493,13 +126936,13 @@ 3560952236,3560952239,PT 3560952240,3560952255,SE 3560952256,3560952259,PL -3560952260,3560952431,SE +3560952260,3560952295,SE +3560952296,3560952299,NL +3560952300,3560952431,SE 3560952432,3560952435,DK 3560952436,3560953103,SE 3560953104,3560953119,JP -3560953120,3560954239,SE -3560954240,3560954367,AX -3560954368,3560955903,SE +3560953120,3560955903,SE 3560955904,3560964095,BE 3560964096,3560996863,NL 3560996864,3561005055,GB @@ -124641,7 +127084,9 @@ 3561562112,3561570303,ES 3561570304,3561578495,AM 3561578496,3561586687,RU -3561586688,3561594879,BG +3561586688,3561590591,BG +3561590592,3561590623,CY +3561590624,3561594879,BG 3561594880,3561594935,SE 3561594936,3561594939,DK 3561594940,3561594975,SE @@ -125387,9 +127832,7 @@ 3563848320,3563848327,ES 3563848328,3563848383,NL 3563848384,3563848447,ES -3563848448,3563848495,NL -3563848496,3563848511,ES -3563848512,3563848575,NL +3563848448,3563848575,NL 3563848576,3563848583,ES 3563848584,3563848655,NL 3563848656,3563848703,ES @@ -125748,9 +128191,7 @@ 3564560512,3564560527,US 3564560528,3564560543,GB 3564560544,3564560607,US -3564560608,3564560927,GB -3564560928,3564560959,US -3564560960,3564561071,GB +3564560608,3564561071,GB 3564561072,3564561087,US 3564561088,3564561151,GB 3564561152,3564561407,US @@ -125759,8 +128200,7 @@ 3564561672,3564561679,GB 3564561680,3564561687,US 3564561688,3564561695,GB -3564561696,3564561791,US -3564561792,3564561919,GB +3564561696,3564561919,US 3564561920,3564562431,CH 3564562432,3564562687,GB 3564562688,3564562831,US @@ -125775,8 +128215,12 @@ 3564565240,3564565247,GB 3564565248,3564565279,US 3564565280,3564565327,GB -3564565328,3564565407,US -3564565408,3564566047,GB +3564565328,3564565439,US +3564565440,3564565567,GB +3564565568,3564565631,US +3564565632,3564565663,GB +3564565664,3564565679,US +3564565680,3564566047,GB 3564566048,3564566079,US 3564566080,3564566143,GB 3564566144,3564566175,US @@ -125792,8 +128236,8 @@ 3564567184,3564567199,US 3564567200,3564567247,GB 3564567248,3564567255,US -3564567256,3564567295,GB -3564567296,3564567551,US +3564567256,3564567263,GB +3564567264,3564567551,US 3564567552,3564567807,GB 3564567808,3564567935,US 3564567936,3564568039,GB @@ -125816,7 +128260,9 @@ 3564571592,3564571599,FR 3564571600,3564572271,GB 3564572272,3564572287,DE -3564572288,3564576767,GB +3564572288,3564572415,GB +3564572416,3564572543,DE +3564572544,3564576767,GB 3564576768,3564584959,RU 3564584960,3564593151,SA 3564593152,3564601343,RU @@ -126368,9 +128814,9 @@ 3565093392,3565093399,NO 3565093400,3565096759,GB 3565096760,3565096767,IE -3565096768,3565099647,GB -3565099648,3565099651,US -3565099652,3565101055,GB +3565096768,3565097151,GB +3565097152,3565097167,TZ +3565097168,3565101055,GB 3565101056,3565109247,DE 3565109248,3565117439,PL 3565117440,3565125631,TR @@ -126715,9 +129161,7 @@ 3566739456,3566747647,RU 3566747648,3566764031,GR 3566764032,3566796799,AT -3566796800,3566803455,NL -3566803456,3566803711,A2 -3566803712,3566816511,NL +3566796800,3566816511,NL 3566816512,3566816767,US 3566816768,3566862335,NL 3566862336,3566895103,TR @@ -126971,21 +129415,13 @@ 3567403008,3567419391,IT 3567419392,3567427583,SA 3567427584,3567435775,SE -3567435776,3567436543,GB -3567436544,3567436595,IN -3567436596,3567436599,GB -3567436600,3567436603,IN -3567436604,3567436799,GB -3567436800,3567436895,IN -3567436896,3567441375,GB +3567435776,3567441375,GB 3567441376,3567441407,NL 3567441408,3567444223,GB 3567444224,3567444479,TZ 3567444480,3567445951,GB 3567445952,3567445983,NL -3567445984,3567452031,GB -3567452032,3567452047,GR -3567452048,3567452159,GB +3567445984,3567452159,GB 3567452160,3567507455,ES 3567507456,3567509503,NL 3567509504,3567512575,ES @@ -127246,9 +129682,7 @@ 3569273168,3569273183,LU 3569273184,3569273791,BE 3569273792,3569273823,LU -3569273824,3569273935,BE -3569273936,3569273951,LI -3569273952,3569274303,BE +3569273824,3569274303,BE 3569274304,3569274311,LU 3569274312,3569274335,BE 3569274336,3569274383,LU @@ -127548,9 +129982,7 @@ 3569873280,3569873407,A2 3569873408,3569876991,IL 3569876992,3569942527,RS -3569942528,3570064759,DE -3570064760,3570064767,A2 -3570064768,3570073599,DE +3569942528,3570073599,DE 3570073600,3570081791,NL 3570081792,3570106367,CH 3570106368,3570139135,PL @@ -127585,9 +130017,7 @@ 3570617344,3570617855,GB 3570617856,3570622463,DE 3570622464,3570630655,GB -3570630656,3570633343,DE -3570633344,3570633471,CH -3570633472,3570663423,DE +3570630656,3570663423,DE 3570663424,3570728959,GB 3570728960,3570729983,FI 3570729984,3570731007,SE @@ -128134,9 +130564,7 @@ 3572629504,3572695039,DK 3572695040,3572704951,CH 3572704952,3572704959,DE -3572704960,3572705479,CH -3572705480,3572705487,NL -3572705488,3572715519,CH +3572704960,3572715519,CH 3572715520,3572715775,GB 3572715776,3572760575,CH 3572760576,3572826111,PL @@ -128248,21 +130676,9 @@ 3574348544,3574398975,EU 3574398976,3574464511,PT 3574464512,3574530047,TR -3574530048,3574563679,SE -3574563680,3574563695,FI -3574563696,3574563727,SE -3574563728,3574563775,FI -3574563776,3574563807,SE +3574530048,3574563807,SE 3574563808,3574563823,FI -3574563824,3574563839,SE -3574563840,3574564863,FI -3574564864,3574565887,SE -3574565888,3574565951,FI -3574565952,3574566207,SE -3574566208,3574566271,FI -3574566272,3574566447,SE -3574566448,3574566463,FI -3574566464,3574594559,SE +3574563824,3574594559,SE 3574594560,3574595583,GB 3574595584,3574596351,FR 3574596352,3574596607,MQ @@ -129259,8 +131675,8 @@ 3576237936,3576238143,FR 3576238144,3576238159,GB 3576238160,3576238305,FR -3576238306,3576238343,GB -3576238344,3576238447,FR +3576238306,3576238335,GB +3576238336,3576238447,FR 3576238448,3576238463,GB 3576238464,3576238511,FR 3576238512,3576238527,GB @@ -129322,13 +131738,11 @@ 3576242328,3576242335,FR 3576242336,3576242343,GB 3576242344,3576243967,FR -3576243968,3576244015,GB -3576244016,3576244031,FR -3576244032,3576244047,GB -3576244048,3576244127,FR +3576243968,3576243983,GB +3576243984,3576244127,FR 3576244128,3576244143,GB -3576244144,3576246399,FR -3576246400,3576246463,GB +3576244144,3576246407,FR +3576246408,3576246463,GB 3576246464,3576246656,FR 3576246657,3576246727,GB 3576246728,3576246743,FR @@ -129479,16 +131893,16 @@ 3576258672,3576258687,GB 3576258688,3576258703,FR 3576258704,3576258783,GB -3576258784,3576258815,FR -3576258816,3576258847,GB +3576258784,3576258823,FR +3576258824,3576258847,GB 3576258848,3576258863,FR 3576258864,3576258895,GB 3576258896,3576258943,FR 3576258944,3576259007,GB 3576259008,3576259023,FR 3576259024,3576259039,GB -3576259040,3576259071,FR -3576259072,3576259087,GB +3576259040,3576259079,FR +3576259080,3576259087,GB 3576259088,3576259103,FR 3576259104,3576259199,GB 3576259200,3576259231,FR @@ -129549,11 +131963,9 @@ 3576261080,3576261095,GB 3576261096,3576261103,FR 3576261104,3576261111,GB -3576261112,3576261375,FR -3576261376,3576261631,GB -3576261632,3576261927,FR -3576261928,3576261935,GB -3576261936,3576263463,FR +3576261112,3576261383,FR +3576261384,3576261631,GB +3576261632,3576263463,FR 3576263464,3576263471,GB 3576263472,3576263503,FR 3576263504,3576263527,GB @@ -129655,8 +132067,10 @@ 3576889344,3576954879,NL 3576954880,3576987647,NO 3576987648,3577001983,GB -3577001984,3577003583,NL -3577003584,3577003711,GB +3577001984,3577003615,NL +3577003616,3577003631,GB +3577003632,3577003647,NL +3577003648,3577003711,GB 3577003712,3577003767,NL 3577003768,3577003771,GB 3577003772,3577004031,NL @@ -129699,7 +132113,9 @@ 3577567712,3577567719,IT 3577567720,3577571391,DE 3577571392,3577571399,DK -3577571400,3577577231,DE +3577571400,3577572391,DE +3577572392,3577572399,GB +3577572400,3577577231,DE 3577577232,3577577247,US 3577577248,3577592431,DE 3577592432,3577592447,FR @@ -129925,9 +132341,7 @@ 3578208128,3578208191,PK 3578208192,3578234759,DE 3578234760,3578234767,US -3578234768,3578236791,DE -3578236792,3578236799,PL -3578236800,3578265599,DE +3578234768,3578265599,DE 3578265600,3578331135,GB 3578331136,3578339327,PL 3578339328,3578347519,ES @@ -129978,8 +132392,7 @@ 3579193712,3579193727,BE 3579193728,3579193815,NL 3579193816,3579193823,ES -3579193824,3579193839,GB -3579193840,3579193855,NL +3579193824,3579193855,NL 3579193856,3579194103,GB 3579194104,3579194111,US 3579194112,3579197055,GB @@ -130834,6 +133247,7 @@ 3582590976,3582599167,FR 3582599168,3582607359,DE 3582607360,3582615551,RU +3582615552,3582623743,GB 3582623744,3582631935,FI 3582631936,3582635007,NO 3582635008,3582635263,SE @@ -130911,8 +133325,8 @@ 3583031848,3583031895,GR 3583031896,3583031903,SG 3583031904,3583031919,LB -3583031920,3583032095,IT -3583032096,3583032127,GR +3583031920,3583032111,IT +3583032112,3583032127,GR 3583032128,3583032159,IT 3583032160,3583032191,FR 3583032192,3583032319,IT @@ -131159,12 +133573,9 @@ 3583709504,3583709511,RU 3583709512,3583709575,UA 3583709576,3583709583,NA -3583709584,3583709599,UA -3583709600,3583709607,NA +3583709584,3583709607,UA 3583709608,3583709615,RU -3583709616,3583709631,UA -3583709632,3583709639,NA -3583709640,3583709663,UA +3583709616,3583709663,UA 3583709664,3583709671,NA 3583709672,3583709699,UA 3583709700,3583709703,NA @@ -131174,9 +133585,7 @@ 3583709760,3583709767,NA 3583709768,3583709791,UA 3583709792,3583709799,NA -3583709800,3583709823,UA -3583709824,3583709831,NA -3583709832,3583709839,UA +3583709800,3583709839,UA 3583709840,3583709863,NA 3583709864,3583709871,RU 3583709872,3583709879,NA @@ -131222,9 +133631,7 @@ 3583711016,3583711023,NA 3583711024,3583711247,UA 3583711248,3583711255,NA -3583711256,3583711311,UA -3583711312,3583711319,NA -3583711320,3583711327,UA +3583711256,3583711327,UA 3583711328,3583711335,NA 3583711336,3583711359,UA 3583711360,3583711367,NA @@ -131247,8 +133654,8 @@ 3583712024,3583712119,UA 3583712120,3583712127,NA 3583712128,3583712159,UA -3583712160,3583712199,NA -3583712200,3583712311,UA +3583712160,3583712191,NA +3583712192,3583712311,UA 3583712312,3583712319,NA 3583712320,3583712455,UA 3583712456,3583712463,NA @@ -131256,21 +133663,15 @@ 3583712472,3583712479,NA 3583712480,3583712655,UA 3583712656,3583712663,NA -3583712664,3583712735,UA -3583712736,3583712743,NA -3583712744,3583712775,UA +3583712664,3583712775,UA 3583712776,3583712783,NA -3583712784,3583712831,UA -3583712832,3583712847,AQ -3583712848,3583712927,UA +3583712784,3583712927,UA 3583712928,3583712943,NA 3583712944,3583713007,UA 3583713008,3583713015,RU 3583713016,3583713143,UA 3583713144,3583713151,NA -3583713152,3583713183,UA -3583713184,3583713191,NA -3583713192,3583713279,UA +3583713152,3583713279,UA 3583713280,3583721471,CZ 3583721472,3583729663,DE 3583729664,3583737855,TR @@ -131290,11 +133691,14 @@ 3583743976,3583743983,GB 3583743984,3583744067,EU 3583744068,3583744071,GB -3583744072,3583744127,EU +3583744072,3583744103,EU +3583744104,3583744111,GB +3583744112,3583744127,EU 3583744128,3583744255,GB 3583744256,3583744287,EU 3583744288,3583744303,GB -3583744304,3583744319,EU +3583744304,3583744311,EU +3583744312,3583744319,DE 3583744320,3583744447,GB 3583744448,3583744511,EU 3583744512,3583744767,GB @@ -131331,8 +133735,8 @@ 3583852544,3583853055,SG 3583853056,3583854591,FI 3583854592,3583854726,GB -3583854727,3583854735,FI -3583854736,3583854871,GB +3583854727,3583854727,FI +3583854728,3583854871,GB 3583854872,3583854879,FI 3583854880,3583855103,GB 3583855104,3583855167,US @@ -131345,8 +133749,8 @@ 3583855248,3583855311,FI 3583855312,3583855327,US 3583855328,3583856383,FI -3583856384,3583856447,SG -3583856448,3583856615,FI +3583856384,3583856463,SG +3583856464,3583856615,FI 3583856616,3583856639,SG 3583856640,3583860735,FI 3583860736,3583868927,HU @@ -131596,9 +134000,7 @@ 3584974848,3584983039,FR 3584983040,3584988255,US 3584988256,3584988287,UA -3584988288,3584988543,US -3584988544,3584988559,UA -3584988560,3584988575,US +3584988288,3584988575,US 3584988576,3584988591,UA 3584988592,3584988623,US 3584988624,3584988639,UA @@ -131939,8 +134341,8 @@ 3586258944,3586259455,BE 3586259456,3586269247,NL 3586269248,3586269263,ES -3586269264,3586269343,NL -3586269344,3586269415,ES +3586269264,3586269375,NL +3586269376,3586269415,ES 3586269416,3586269423,NL 3586269424,3586269439,ES 3586269440,3586269471,NL @@ -131965,7 +134367,8 @@ 3586272384,3586272799,ES 3586272800,3586272807,IT 3586272808,3586272815,ES -3586272816,3586272831,IT +3586272816,3586272823,IT +3586272824,3586272831,ES 3586272832,3586272895,NL 3586272896,3586272959,IT 3586272960,3586272991,ES @@ -131990,7 +134393,9 @@ 3586441216,3586457599,DE 3586457600,3586473983,NL 3586473984,3586473987,SK -3586473988,3586476287,HU +3586473988,3586476031,HU +3586476032,3586476063,TR +3586476064,3586476287,HU 3586476288,3586476295,SK 3586476296,3586476311,HU 3586476312,3586476319,AT @@ -132001,7 +134406,7 @@ 3586476432,3586476439,SK 3586476440,3586478079,HU 3586478080,3586478591,SK -3586478592,3586478847,TR +3586478592,3586478847,HU 3586478848,3586479103,SK 3586479104,3586479359,TR 3586479360,3586490367,HU @@ -132141,9 +134546,11 @@ 3586910560,3586910567,NL 3586910568,3586910575,CY 3586910576,3586910587,NL -3586910588,3586910655,DE -3586910656,3586910703,NL -3586910704,3586910711,DE +3586910588,3586910591,DE +3586910592,3586910623,NL +3586910624,3586910655,DE +3586910656,3586910687,NL +3586910688,3586910711,DE 3586910712,3586910719,NL 3586910720,3586910991,DE 3586910992,3586911007,BE @@ -132372,13 +134779,13 @@ 3587238608,3587238911,GB 3587238912,3587239303,NL 3587239304,3587239311,GB -3587239312,3587239695,NL -3587239696,3587239711,GB +3587239312,3587239703,NL +3587239704,3587239711,GB 3587239712,3587239903,NL 3587239904,3587239935,GB -3587239936,3587240023,NL -3587240024,3587240031,GB -3587240032,3587240087,NL +3587239936,3587239975,NL +3587239976,3587239983,GB +3587239984,3587240087,NL 3587240088,3587240095,GB 3587240096,3587240103,NL 3587240104,3587240107,GB @@ -132400,13 +134807,13 @@ 3587241264,3587241271,GB 3587241272,3587241343,NL 3587241344,3587241471,GB -3587241472,3587242055,NL -3587242056,3587242063,GB -3587242064,3587242095,NL +3587241472,3587242095,NL 3587242096,3587242111,GB 3587242112,3587242463,NL 3587242464,3587242471,GB -3587242472,3587243935,NL +3587242472,3587243407,NL +3587243408,3587243415,GB +3587243416,3587243935,NL 3587243936,3587243943,GB 3587243944,3587243967,NL 3587243968,3587243975,GB @@ -132664,9 +135071,7 @@ 3588588568,3588588575,IT 3588588576,3588590175,ES 3588590176,3588590207,BE -3588590208,3588594751,ES -3588594752,3588594759,DE -3588594760,3588598607,ES +3588590208,3588598607,ES 3588598608,3588598615,IT 3588598616,3588603903,ES 3588603904,3588620287,SI @@ -132776,7 +135181,7 @@ 3589373952,3589375495,DE 3589375496,3589375503,CI 3589375504,3589390335,DE -3589390336,3589406719,RU +3589390336,3589423103,RU 3589423104,3589423615,IT 3589423616,3589424127,CH 3589424128,3589424639,SE @@ -133153,7 +135558,9 @@ 3590155808,3590155815,SI 3590155816,3590155823,HR 3590155824,3590155871,SI -3590155872,3590156431,HR +3590155872,3590156359,HR +3590156360,3590156367,SI +3590156368,3590156431,HR 3590156432,3590156447,SI 3590156448,3590156543,HR 3590156544,3590156575,SI @@ -133317,7 +135724,9 @@ 3590308128,3590308175,A2 3590308176,3590308191,IQ 3590308192,3590308199,A2 -3590308200,3590308238,IQ +3590308200,3590308215,IQ +3590308216,3590308223,A2 +3590308224,3590308238,IQ 3590308239,3590308247,A2 3590308248,3590308287,IQ 3590308288,3590308303,A2 @@ -134021,20 +136430,20 @@ 3624914272,3624914303,CA 3624914304,3624914335,US 3624914336,3624914367,CA -3624914368,3624914687,US +3624914368,3624914399,IN +3624914400,3624914687,US 3624914688,3624914943,BE -3624914944,3624915199,US -3624915200,3624915455,CA -3624915456,3624915903,US +3624914944,3624915071,CA +3624915072,3624915455,US +3624915456,3624915487,CA +3624915488,3624915903,US 3624915904,3624915935,HK -3624915936,3624916159,US -3624916160,3624916191,MY -3624916192,3624916223,US +3624915936,3624916223,US 3624916224,3624916735,CA 3624916736,3624917247,US 3624917248,3624917503,AU 3624917504,3624917919,US -3624917920,3624917951,CH +3624917920,3624917951,CA 3624917952,3624917983,US 3624917984,3624918015,CA 3624918016,3624918527,US @@ -134072,8 +136481,7 @@ 3624922336,3624922367,FR 3624922368,3624922495,IN 3624922496,3624922623,BE -3624922624,3624922655,US -3624922656,3624922687,CA +3624922624,3624922687,CA 3624922688,3624922783,US 3624922784,3624922815,CN 3624922816,3624922847,FR @@ -134084,18 +136492,23 @@ 3624923392,3624923519,CA 3624923520,3624923647,US 3624923648,3624923775,CA -3624923776,3624924095,US +3624923776,3624923903,US +3624923904,3624923935,CA +3624923936,3624923967,US +3624923968,3624923999,IN +3624924000,3624924095,US 3624924096,3624924127,CA 3624924128,3624924287,US -3624924288,3624924671,CA -3624924672,3624924895,US +3624924288,3624924703,CA +3624924704,3624924895,US 3624924896,3624924927,CA 3624924928,3624925054,US 3624925055,3624925695,CA -3624925696,3624926591,US +3624925696,3624926463,US +3624926464,3624926527,CA +3624926528,3624926591,US 3624926592,3624926655,IN -3624926656,3624926719,US -3624926720,3624926975,CA +3624926656,3624926975,CA 3624926976,3624927039,US 3624927040,3624927071,HK 3624927072,3624927167,US @@ -134108,7 +136521,9 @@ 3624990464,3624991223,CA 3624991224,3624991231,US 3624991232,3624992767,CA -3624992768,3625042327,US +3624992768,3625002351,US +3625002352,3625002367,CA +3625002368,3625042327,US 3625042328,3625042335,IT 3625042336,3625058303,US 3625058304,3625091071,CA @@ -134693,8 +137108,7 @@ 3626385242,3626385245,US 3626385246,3626385249,ID 3626385250,3626385253,CN -3626385254,3626385257,US -3626385258,3626385261,PK +3626385254,3626385261,US 3626385262,3626385265,GR 3626385266,3626385321,US 3626385322,3626385325,FR @@ -134776,7 +137190,9 @@ 3626529808,3626529863,US 3626529864,3626529919,CA 3626529920,3626529927,US -3626529928,3626530039,CA +3626529928,3626529951,CA +3626529952,3626529959,US +3626529960,3626530039,CA 3626530040,3626530055,US 3626530056,3626530151,CA 3626530152,3626530159,US @@ -135152,20 +137568,18 @@ 3629201488,3629201515,US 3629201516,3629201539,CA 3629201540,3629201543,US -3629201544,3629201591,CA -3629201592,3629201599,US +3629201544,3629201551,CA +3629201552,3629201559,US +3629201560,3629201563,CA +3629201564,3629201599,US 3629201600,3629201615,CA 3629201616,3629201623,US -3629201624,3629201639,CA -3629201640,3629201687,US -3629201688,3629201707,CA -3629201708,3629201743,US +3629201624,3629201627,CA +3629201628,3629201743,US 3629201744,3629201747,CA 3629201748,3629201751,US 3629201752,3629201755,CA -3629201756,3629201763,US -3629201764,3629201767,CA -3629201768,3629201775,US +3629201756,3629201775,US 3629201776,3629201791,CA 3629201792,3629201839,US 3629201840,3629201855,CA @@ -135178,8 +137592,8 @@ 3629201984,3629201991,US 3629201992,3629201999,CA 3629202000,3629202003,US -3629202004,3629202011,CA -3629202012,3629202015,US +3629202004,3629202007,CA +3629202008,3629202015,US 3629202016,3629202047,CA 3629202048,3629202175,US 3629202176,3629202203,CA @@ -135187,13 +137601,9 @@ 3629202220,3629202239,CA 3629202240,3629202263,US 3629202264,3629202271,CA -3629202272,3629202279,US -3629202280,3629202287,CA -3629202288,3629202311,US +3629202272,3629202311,US 3629202312,3629202367,CA -3629202368,3629202399,US -3629202400,3629202407,CA -3629202408,3629202427,US +3629202368,3629202427,US 3629202428,3629203199,CA 3629203200,3629318143,US 3629318144,3629326335,CA @@ -135235,7 +137645,9 @@ 3629839104,3629839359,CA 3629839360,3629841951,US 3629841952,3629841983,CA -3629841984,3629875775,US +3629841984,3629859551,US +3629859552,3629859583,CA +3629859584,3629875775,US 3629875776,3629875807,CA 3629875808,3629876255,US 3629876256,3629876263,CA @@ -135765,7 +138177,27 @@ 3631058592,3631059229,US 3631059230,3631059239,NO 3631059240,3631112191,US -3631112192,3631116543,BB +3631112192,3631112385,BB +3631112386,3631112386,VC +3631112387,3631112387,BB +3631112388,3631112391,VC +3631112392,3631112393,BB +3631112394,3631112398,VC +3631112399,3631112399,BB +3631112400,3631112401,VC +3631112402,3631112404,BB +3631112405,3631112405,VC +3631112406,3631112417,BB +3631112418,3631112418,VC +3631112419,3631112425,BB +3631112426,3631112427,VC +3631112428,3631112429,BB +3631112430,3631112430,VC +3631112431,3631112433,BB +3631112434,3631112434,VC +3631112435,3631112439,BB +3631112440,3631112440,VC +3631112441,3631116543,BB 3631116544,3631117567,GD 3631117568,3631117823,BB 3631117824,3631118079,GD @@ -135927,7 +138359,7 @@ 3632481184,3632481199,TH 3632481200,3632481279,US 3632481280,3632481287,GB -3632481288,3632481295,PH +3632481288,3632481295,US 3632481296,3632481311,TH 3632481312,3632481391,US 3632481392,3632481399,GB @@ -136045,9 +138477,7 @@ 3632485264,3632485271,IT 3632485272,3632485279,US 3632485280,3632485311,TH -3632485312,3632485359,US -3632485360,3632485367,TH -3632485368,3632485391,US +3632485312,3632485391,US 3632485392,3632485399,SR 3632485400,3632485431,US 3632485432,3632485439,CA @@ -136343,7 +138773,9 @@ 3632902144,3632971775,US 3632971776,3632972031,CA 3632972032,3632972063,US -3632972064,3632988159,CA +3632972064,3632973087,CA +3632973088,3632973119,US +3632973120,3632988159,CA 3632988160,3633029119,US 3633029120,3633029631,PY 3633029632,3633030143,NI @@ -136493,7 +138925,9 @@ 3633553312,3633553343,A2 3633553344,3633776399,US 3633776400,3633776415,CN -3633776416,3633815551,US +3633776416,3633776463,US +3633776464,3633776479,CN +3633776480,3633815551,US 3633815552,3633816079,CA 3633816080,3633816095,US 3633816096,3633816119,CA @@ -136511,7 +138945,9 @@ 3633816256,3633816263,NG 3633816264,3633816279,CA 3633816280,3633816311,US -3633816312,3633816351,CA +3633816312,3633816319,CA +3633816320,3633816327,ZA +3633816328,3633816351,CA 3633816352,3633816383,IN 3633816384,3633816391,NG 3633816392,3633816399,US @@ -136543,7 +138979,9 @@ 3633822304,3633822327,CA 3633822328,3633827839,US 3633827840,3633828095,GB -3633828096,3633881087,US +3633828096,3633874431,US +3633874432,3633874687,GB +3633874688,3633881087,US 3633881088,3633885183,AN 3633885184,3633889279,US 3633889280,3633893375,CA @@ -136645,8 +139083,7 @@ 3635532288,3635532303,ES 3635532304,3635532831,US 3635532832,3635532863,VE -3635532864,3635532927,SV -3635532928,3635532999,US +3635532864,3635532999,US 3635533000,3635533007,NO 3635533008,3635533535,US 3635533536,3635533551,IN @@ -136689,9 +139126,7 @@ 3636151008,3636151023,BS 3636151024,3636151031,CA 3636151032,3636151039,US -3636151040,3636151167,CA -3636151168,3636151199,US -3636151200,3636151455,CA +3636151040,3636151455,CA 3636151456,3636151479,US 3636151480,3636151488,CA 3636151489,3636151535,US @@ -136726,9 +139161,7 @@ 3636154991,3636155071,US 3636155072,3636155391,CA 3636155392,3636155407,US -3636155408,3636155423,CA -3636155424,3636155439,US -3636155440,3636155455,CA +3636155408,3636155455,CA 3636155456,3636155519,US 3636155520,3636155775,CA 3636155776,3636155839,US @@ -136736,9 +139169,7 @@ 3636156160,3636156191,US 3636156192,3636156255,CA 3636156256,3636156256,US -3636156257,3636156271,CA -3636156272,3636156279,US -3636156280,3636156415,CA +3636156257,3636156415,CA 3636156416,3636156927,US 3636156928,3636157063,CA 3636157064,3636157167,US @@ -137013,7 +139444,9 @@ 3637667519,3637669887,CA 3637669888,3637706751,US 3637706752,3637739519,CA -3637739520,3638165503,US +3637739520,3637827327,US +3637827328,3637827583,VI +3637827584,3638165503,US 3638165504,3638181887,CA 3638181888,3638198951,US 3638198952,3638198959,BM @@ -137100,7 +139533,9 @@ 3638249216,3638249471,GB 3638249472,3638249751,US 3638249752,3638249791,GB -3638249792,3638250535,US +3638249792,3638249983,US +3638249984,3638249991,GB +3638249992,3638250535,US 3638250536,3638250543,GB 3638250544,3638250559,US 3638250560,3638250623,GB @@ -137130,7 +139565,8 @@ 3638399488,3638399615,CH 3638399616,3638399743,US 3638399744,3638399999,CH -3638400000,3638509295,US +3638400000,3638400063,CA +3638400064,3638509295,US 3638509296,3638509311,GB 3638509312,3638509567,US 3638509568,3638526911,CA @@ -137383,7 +139819,9 @@ 3640007312,3640007319,HK 3640007320,3640007359,US 3640007360,3640007367,SO -3640007368,3640013767,US +3640007368,3640013095,US +3640013096,3640013103,CA +3640013104,3640013767,US 3640013768,3640013775,GI 3640013776,3640023079,US 3640023080,3640023087,EG @@ -137393,14 +139831,16 @@ 3640023272,3640023279,HK 3640023280,3640027415,US 3640027416,3640027423,GB -3640027424,3640028279,US -3640028280,3640028287,AF -3640028288,3640028295,US +3640027424,3640028207,US +3640028208,3640028215,GB +3640028216,3640028295,US 3640028296,3640028303,CA 3640028304,3640028311,GB 3640028312,3640028335,US 3640028336,3640028343,CA -3640028344,3640028591,US +3640028344,3640028447,US +3640028448,3640028455,GB +3640028456,3640028591,US 3640028592,3640028599,CA 3640028600,3640057855,US 3640057856,3640066047,CA @@ -137591,8 +140031,7 @@ 3641353184,3641353215,NG 3641353216,3641353231,GB 3641353232,3641353247,IQ -3641353248,3641353263,GB -3641353264,3641353727,A2 +3641353248,3641353727,A2 3641353728,3641353759,NG 3641353760,3641353775,GB 3641353776,3641353807,NG @@ -137625,12 +140064,12 @@ 3641357880,3641357887,A2 3641357888,3641357927,GB 3641357928,3641358335,A2 -3641358336,3641359359,GB +3641358336,3641358591,GB +3641358592,3641358847,A2 +3641358848,3641359359,GB 3641359360,3641359615,IQ 3641359616,3641359639,GB -3641359640,3641359647,A2 -3641359648,3641359664,GB -3641359665,3641359871,A2 +3641359640,3641359871,A2 3641359872,3641360383,GB 3641360384,3641368575,RO 3641368576,3641372671,GB @@ -137725,7 +140164,10 @@ 3641680128,3641681151,GB 3641681152,3641681407,SE 3641681408,3641681663,FR -3641681664,3641683967,EU +3641681664,3641681791,DK +3641681792,3641681919,FR +3641681920,3641682431,GB +3641682432,3641683967,EU 3641683968,3641688063,KZ 3641688064,3641692159,RU 3641692160,3641696255,IT @@ -137815,11 +140257,10 @@ 3641937920,3641942015,IT 3641942016,3641950207,DE 3641950208,3641954303,FR -3641954304,3641957887,MD -3641957888,3641957951,GB -3641957952,3641958143,MD -3641958144,3641958207,GB -3641958208,3641958399,MD +3641954304,3641956863,MD +3641956864,3641957631,GB +3641957632,3641957887,MD +3641957888,3641958399,GB 3641958400,3641960699,BE 3641960700,3641960703,NL 3641960704,3641961727,BE @@ -137852,7 +140293,9 @@ 3642029057,3642029311,NG 3642029312,3642029312,A2 3642029313,3642029567,GB -3642029568,3642031359,NG +3642029568,3642030591,NG +3642030592,3642030847,GN +3642030848,3642031359,NG 3642031360,3642031615,GB 3642031616,3642031616,A2 3642031617,3642031743,SL @@ -137925,7 +140368,17 @@ 3642113792,3642113823,BE 3642113824,3642113855,NL 3642113856,3642113919,BE -3642113920,3642118143,NL +3642113920,3642114591,NL +3642114592,3642114815,NO +3642114816,3642114831,NL +3642114832,3642115095,NO +3642115096,3642115103,NL +3642115104,3642116095,NO +3642116096,3642117119,NL +3642117120,3642117375,NO +3642117376,3642117631,NL +3642117632,3642117887,NO +3642117888,3642118143,NL 3642118144,3642122239,GB 3642122240,3642126335,ES 3642126336,3642130431,IL @@ -138087,7 +140540,9 @@ 3642552688,3642552831,UA 3642552832,3642552847,EE 3642552848,3642553095,UA -3642553096,3642553139,LV +3642553096,3642553099,LV +3642553100,3642553103,UA +3642553104,3642553139,LV 3642553140,3642553143,UA 3642553144,3642553161,LV 3642553162,3642553163,UA @@ -138150,7 +140605,8 @@ 3642554912,3642554919,NL 3642554920,3642554931,DE 3642554932,3642554951,UA -3642554952,3642554959,GE +3642554952,3642554955,GE +3642554956,3642554959,UA 3642554960,3642554963,DE 3642554964,3642554967,UA 3642554968,3642554971,LV @@ -138260,7 +140716,9 @@ 3642736640,3642740735,CZ 3642740736,3642744831,DE 3642744832,3642753023,TR -3642753024,3643801599,GB +3642753024,3642830671,GB +3642830672,3642830687,A2 +3642830688,3643801599,GB 3643801600,3644063743,DE 3644063744,3644325887,EG 3644325888,3644588031,IT @@ -139608,9 +142066,7 @@ 3647969280,3647969791,IT 3647969792,3647970303,BE 3647970304,3647971327,FR -3647971328,3647971591,GB -3647971592,3647971599,DE -3647971600,3647971671,GB +3647971328,3647971671,GB 3647971672,3647971679,DE 3647971680,3647972191,GB 3647972192,3647972199,DE @@ -139643,11 +142099,9 @@ 3647976792,3647976799,DE 3647976800,3647976927,BE 3647976928,3647976935,DE -3647976936,3647977079,BE -3647977080,3647977087,DE -3647977088,3647977119,BE -3647977120,3647977135,DE -3647977136,3647977151,BE +3647976936,3647977119,BE +3647977120,3647977143,DE +3647977144,3647977151,BE 3647977152,3647977215,DE 3647977216,3647977471,BE 3647977472,3647977791,GB @@ -139659,8 +142113,8 @@ 3647978776,3647978783,DE 3647978784,3647978895,NL 3647978896,3647978911,DE -3647978912,3647978935,NL -3647978936,3647979007,DE +3647978912,3647978943,NL +3647978944,3647979007,DE 3647979008,3647979136,IT 3647979137,3647979519,DE 3647979520,3647980543,FR @@ -139675,11 +142129,17 @@ 3647985920,3647985935,BE 3647985936,3647986431,ES 3647986432,3647986687,FR -3647986688,3647987455,DE -3647987456,3647987471,ES -3647987472,3647987479,DE -3647987480,3647987519,ES -3647987520,3647995903,DE +3647986688,3647986943,DE +3647986944,3647986951,ES +3647986952,3647987455,DE +3647987456,3647987527,ES +3647987528,3647987543,DE +3647987544,3647987663,ES +3647987664,3647987679,DE +3647987680,3647987711,ES +3647987712,3647988735,IT +3647988736,3647989759,BE +3647989760,3647995903,DE 3647995904,3648004095,RU 3648004096,3648004607,GB 3648004608,3648006271,RU @@ -139786,8 +142246,8 @@ 3648180752,3648180767,DE 3648180768,3648180799,AT 3648180800,3648180991,DE -3648180992,3648180999,CY -3648181000,3648181023,DE +3648180992,3648181007,CY +3648181008,3648181023,DE 3648181024,3648181055,AT 3648181056,3648181215,DE 3648181216,3648181231,RU @@ -139806,7 +142266,7 @@ 3648181648,3648181887,DE 3648181888,3648181903,AT 3648181904,3648182143,DE -3648182144,3648182159,NL +3648182144,3648182159,BR 3648182160,3648182175,DE 3648182176,3648182207,RU 3648182208,3648182271,GB @@ -140152,7 +142612,9 @@ 3650465792,3650469887,RU 3650469888,3650478079,NL 3650478080,3650482175,AT -3650482176,3650486271,PL +3650482176,3650484639,PL +3650484640,3650484671,US +3650484672,3650486271,PL 3650486272,3650502655,RU 3650502656,3650510847,SE 3650510848,3650519039,GB @@ -140421,15 +142883,17 @@ 3651870720,3651874815,IT 3651874816,3651878911,PL 3651878912,3651883007,RU -3651883008,3651885823,BE -3651885824,3651885847,CD +3651883008,3651884031,BE +3651884032,3651885055,CD +3651885056,3651885567,BE +3651885568,3651885847,CD 3651885848,3651885851,BE 3651885852,3651885855,CD 3651885856,3651885867,BE 3651885868,3651885875,CD 3651885876,3651885903,BE -3651885904,3651885919,CD -3651885920,3651886335,BE +3651885904,3651885927,CD +3651885928,3651886335,BE 3651886336,3651886347,CD 3651886348,3651886379,BE 3651886380,3651886383,CD @@ -140448,8 +142912,8 @@ 3651886472,3651886479,BE 3651886480,3651886495,CD 3651886496,3651886511,BE -3651886512,3651886527,CD -3651886528,3651886591,BE +3651886512,3651886543,CD +3651886544,3651886591,BE 3651886592,3651886847,CD 3651886848,3651887103,BE 3651887104,3651891199,GB @@ -140536,9 +143000,9 @@ 3651942816,3651944447,DE 3651944448,3651948543,ES 3651948544,3651952639,FO -3651952640,3651954175,IR -3651954176,3651954431,AE -3651954432,3651958271,IR +3651952640,3651953663,IR +3651953664,3651954687,AE +3651954688,3651958271,IR 3651958272,3651958783,AE 3651958784,3651959039,IR 3651959040,3651960831,AE @@ -140855,7 +143319,8 @@ 3653439488,3653443583,FR 3653443584,3653444351,DE 3653444352,3653444383,IT -3653444384,3653447679,DE +3653444384,3653447167,DE +3653447168,3653447679,NL 3653447680,3653451775,LV 3653451776,3653464063,RU 3653464064,3653468159,NL @@ -140953,7 +143418,10 @@ 3653652480,3653656575,RU 3653656576,3653660671,GB 3653660672,3653664767,CZ -3653664768,3653668863,NL +3653664768,3653664895,DE +3653664896,3653665023,NL +3653665024,3653665039,DE +3653665040,3653668863,NL 3653668864,3653672959,SE 3653672960,3653681151,RU 3653681152,3653685247,ES @@ -141009,7 +143477,8 @@ 3654066032,3654287359,GB 3654287360,3654607871,SE 3654607872,3654608127,NO -3654608128,3654608895,SE +3654608128,3654608383,SE +3654608384,3654608895,PL 3654608896,3654609919,NO 3654609920,3654613007,SE 3654613008,3654613015,RU @@ -141115,6 +143584,7 @@ 3664007168,3664008191,AU 3664008192,3664008447,MN 3664008448,3664008703,PK +3664008704,3664008959,MY 3664052224,3664084991,NZ 3664084992,3664117759,KR 3664117760,3664248831,HK @@ -141360,11 +143830,13 @@ 3743039488,3743055871,TW 3743055872,3743088639,KR 3743088640,3743096831,AU +3743096832,3743105023,TW 3743105024,3743106047,AU -3743107072,3743109119,JP +3743106048,3743109119,JP 3743109120,3743113215,BD 3743113216,3743115263,AU 3743117312,3743118335,BD +3743118336,3743119359,JP 3743119360,3743120383,IN 3743120384,3743121407,JP 3743121408,3743125503,MY @@ -141372,16 +143844,19 @@ 3743129600,3743130623,HK 3743130624,3743133695,SG 3743133696,3743134719,AU +3743134720,3743135743,JP 3743135744,3743136767,CN 3743136768,3743137791,MY 3743137792,3743154175,TH 3743154176,3743186943,MY 3743186944,3743219711,KR 3743219712,3743252479,JP -3743252480,3743260671,NC +3743252480,3743264767,NC +3743264768,3743268863,JP 3743268864,3743277055,IN 3743277056,3743281151,PK 3743281152,3743282175,AU +3743282176,3743283199,JP 3743283200,3743284223,HK 3743284224,3743285247,PK 3743285248,3743416319,IN @@ -141390,7 +143865,8 @@ 3749838848,3749839871,SG 3749839872,3749840895,IN 3749840896,3749841919,PH -3749841920,3749847039,AU +3749841920,3749846015,AU +3749846016,3749847039,IN 3749847040,3749855231,HK 3749855232,3749969919,KR 3749969920,3750232063,JP @@ -141401,9 +143877,11 @@ 3752133632,3752134655,ID 3752134656,3752136703,TW 3752136704,3752137727,NZ +3752137728,3752138751,JP 3752138752,3752140799,IN 3752140800,3752148991,JP 3752148992,3752153087,NZ +3752153088,3752157183,JP 3752157184,3752165375,AU 3752165376,3752198143,KR 3752198144,3752329215,CN @@ -141439,6 +143917,7 @@ 3758080000,3758088191,KR 3758088192,3758090239,ID 3758090240,3758091263,AU +3758091264,3758092287,CN 3758092288,3758093311,HK 3758093312,3758094335,IN 3758094336,3758095359,AU diff --git a/src/config/torrc.bridge.in b/src/config/torrc.bridge.in new file mode 100644 index 0000000000..557b7adf46 --- /dev/null +++ b/src/config/torrc.bridge.in @@ -0,0 +1,171 @@ +## Configuration file for a typical Tor user +## Last updated 16 July 2009 for Tor 0.2.2.1-alpha. +## (May or may not work for much older or much newer versions of Tor.) +## +## Lines that begin with "## " try to explain what's going on. Lines +## that begin with just "#" are disabled commands: you can enable them +## by removing the "#" symbol. +## +## See 'man tor', or https://www.torproject.org/tor-manual.html, +## for more options you can use in this file. +## +## Tor will look for this file in various places based on your platform: +## https://wiki.torproject.org/noreply/TheOnionRouter/TorFAQ#torrc + + +## Replace this with "SocksPort 0" if you plan to run Tor only as a +## relay, and not make any local application connections yourself. +SocksPort 9050 # what port to open for local application connections +SocksListenAddress 127.0.0.1 # accept connections only from localhost +#SocksListenAddress 192.168.0.1:9100 # listen on this IP:port also + +## Entry policies to allow/deny SOCKS requests based on IP address. +## First entry that matches wins. If no SocksPolicy is set, we accept +## all (and only) requests from SocksListenAddress. +#SocksPolicy accept 192.168.0.0/16 +#SocksPolicy reject * + +## Logs go to stdout at level "notice" unless redirected by something +## else, like one of the below lines. You can have as many Log lines as +## you want. +## +## We advise using "notice" in most cases, since anything more verbose +## may provide sensitive information to an attacker who obtains the logs. +## +## Send all messages of level 'notice' or higher to @LOCALSTATEDIR@/log/tor/notices.log +#Log notice file @LOCALSTATEDIR@/log/tor/notices.log +## Send every possible message to @LOCALSTATEDIR@/log/tor/debug.log +#Log debug file @LOCALSTATEDIR@/log/tor/debug.log +## Use the system log instead of Tor's logfiles +#Log notice syslog +## To send all messages to stderr: +#Log debug stderr + +## Uncomment this to start the process in the background... or use +## --runasdaemon 1 on the command line. This is ignored on Windows; +## see the FAQ entry if you want Tor to run as an NT service. +#RunAsDaemon 1 + +## The directory for keeping all the keys/etc. By default, we store +## things in $HOME/.tor on Unix, and in Application Data\tor on Windows. +#DataDirectory @LOCALSTATEDIR@/lib/tor + +## The port on which Tor will listen for local connections from Tor +## controller applications, as documented in control-spec.txt. +#ControlPort 9051 +## If you enable the controlport, be sure to enable one of these +## authentication methods, to prevent attackers from accessing it. +#HashedControlPassword 16:872860B76453A77D60CA2BB8C1A7042072093276A3D701AD684053EC4C +#CookieAuthentication 1 + +############### This section is just for location-hidden services ### + +## Once you have configured a hidden service, you can look at the +## contents of the file ".../hidden_service/hostname" for the address +## to tell people. +## +## HiddenServicePort x y:z says to redirect requests on port x to the +## address y:z. + +#HiddenServiceDir @LOCALSTATEDIR@/lib/tor/hidden_service/ +#HiddenServicePort 80 127.0.0.1:80 + +#HiddenServiceDir @LOCALSTATEDIR@/lib/tor/other_hidden_service/ +#HiddenServicePort 80 127.0.0.1:80 +#HiddenServicePort 22 127.0.0.1:22 + +################ This section is just for relays ##################### +# +## See https://www.torproject.org/docs/tor-doc-relay for details. + +## Required: what port to advertise for incoming Tor connections. +ORPort 9001 +## If you want to listen on a port other than the one advertised +## in ORPort (e.g. to advertise 443 but bind to 9090), uncomment the +## line below too. You'll need to do ipchains or other port forwarding +## yourself to make this work. +#ORListenAddress 0.0.0.0:9090 + +## A handle for your relay, so people don't have to refer to it by key. +Nickname Unnamed + +## The IP address or full DNS name for your relay. Leave commented out +## and Tor will guess. +#Address noname.example.com + +## Define these to limit how much relayed traffic you will allow. Your +## own traffic is still unthrottled. Note that RelayBandwidthRate must +## be at least 20 KB. +#RelayBandwidthRate 100 KB # Throttle traffic to 100KB/s (800Kbps) +#RelayBandwidthBurst 200 KB # But allow bursts up to 200KB/s (1600Kbps) +RelayBandwidthBurst 10485760 +RelayBandwidthRate 5242880 + +## Use these to restrict the maximum traffic per day, week, or month. +## Note that this threshold applies to sent _and_ to received bytes, +## not to their sum: Setting "4 GB" may allow up to 8 GB +## total before hibernating. +## +## Set a maximum of 4 gigabytes each way per period. +#AccountingMax 4 GB +## Each period starts daily at midnight (AccountingMax is per day) +#AccountingStart day 00:00 +## Each period starts on the 3rd of the month at 15:00 (AccountingMax +## is per month) +#AccountingStart month 3 15:00 + +## Contact info to be published in the directory, so we can contact you +## if your relay is misconfigured or something else goes wrong. Google +## indexes this, so spammers might also collect it. +#ContactInfo Random Person <nobody AT example dot com> +## You might also include your PGP or GPG fingerprint if you have one: +#ContactInfo 1234D/FFFFFFFF Random Person <nobody AT example dot com> + +## Uncomment this to mirror directory information for others. Please do +## if you have enough bandwidth. +DirPort 9030 # what port to advertise for directory connections +## If you want to listen on a port other than the one advertised +## in DirPort (e.g. to advertise 80 but bind to 9091), uncomment the line +## below too. You'll need to do ipchains or other port forwarding yourself +## to make this work. +#DirListenAddress 0.0.0.0:9091 +## Uncomment to return an arbitrary blob of html on your DirPort. Now you +## can explain what Tor is if anybody wonders why your IP address is +## contacting them. See contrib/tor-exit-notice.html in Tor's source +## distribution for a sample. +#DirPortFrontPage @CONFDIR@/tor-exit-notice.html + +## Uncomment this if you run more than one Tor relay, and add the identity +## key fingerprint of each Tor relay you control, even if they're on +## different networks. You declare it here so Tor clients can avoid +## using more than one of your relays in a single circuit. See +## https://wiki.torproject.org/noreply/TheOnionRouter/TorFAQ#MultipleServers +#MyFamily $keyid,$keyid,... + +## A comma-separated list of exit policies. They're considered first +## to last, and the first match wins. If you want to _replace_ +## the default exit policy, end this with either a reject *:* or an +## accept *:*. Otherwise, you're _augmenting_ (prepending to) the +## default exit policy. Leave commented to just use the default, which is +## described in the man page or at +## https://www.torproject.org/documentation.html +## +## Look at https://www.torproject.org/faq-abuse.html#TypicalAbuses +## for issues you might encounter if you use the default exit policy. +## +## If certain IPs and ports are blocked externally, e.g. by your firewall, +## you should update your exit policy to reflect this -- otherwise Tor +## users will be told that those destinations are down. +## +#ExitPolicy accept *:6660-6667,reject *:* # allow irc ports but no more +#ExitPolicy accept *:119 # accept nntp as well as default exit policy +#ExitPolicy reject *:* # no exits allowed +# +## Bridge relays (or "bridges") are Tor relays that aren't listed in the +## main directory. Since there is no complete public list of them, even if an +## ISP is filtering connections to all the known Tor relays, they probably +## won't be able to block all the bridges. Also, websites won't treat you +## differently because they won't know you're running Tor. If you can +## be a real relay, please do; but if not, be a bridge! +BridgeRelay 1 +ExitPolicy reject *:* diff --git a/src/config/torrc.complete.in b/src/config/torrc.complete.in deleted file mode 100644 index 310458a5c0..0000000000 --- a/src/config/torrc.complete.in +++ /dev/null @@ -1,533 +0,0 @@ -# $Id$ -# Last updated on $Date$ -#################################################################### -## This config file is divided into four sections. They are: -## 1. Global Options (clients and servers) -## 2. Client Options Only -## 3. Server Options Only -## 4. Directory Server Options (for running your own Tor network) -## 5. Hidden Service Options (clients and servers) -## -## The conventions used are: -## double hash (##) is for summary text about the config option; -## single hash (#) is for the config option; and, -## the config option is always after the text. -#################################################################### - - -## Section 1: Global Options (clients and servers) - -## A token bucket limits the average incoming bandwidth on this node -## to the specified number of bytes per second. (Default: 2MB) -#BandwidthRate N bytes|KB|MB|GB|TB - -## Limit the maximum token bucket size (also known as the burst) to -## the given number of bytes. (Default: 5 MB) -#BandwidthBurst N bytes|KB|MB|GB|TB - -## If set, we will not advertise more than this amount of bandwidth -## for our BandwidthRate. Server operators who want to reduce the -## number of clients who ask to build circuits through them (since -## this is proportional to advertised bandwidth rate) can thus -## reduce the CPU demands on their server without impacting -## network performance. -#MaxAdvertisedBandwidth N bytes|KB|MB|GB|TB - -## If set, Tor will accept connections from the same machine -## (localhost only) on this port, and allow those connections to -## control the Tor process using the Tor Control Protocol -## (described in control-spec.txt). Note: unless you also specify -## one of HashedControlPassword or CookieAuthentication, setting -## this option will cause Tor to allow any process on the local -## host to control it. -#ControlPort Port - -## Don’t allow any connections on the control port except when the -## other process knows the password whose one-way hash is -## hashed_password. You can compute the hash of a password by -## running "tor --hash-password password". -#HashedControlPassword hashed_password - -## If this option is set to 1, don’t allow any connections on the -## control port except when the connecting process knows the -## contents of a file named "control_auth_cookie", which Tor will -## create in its data directory. This authentication method -## should only be used on systems with good filesystem security. -## (Default: 0) -#CookieAuthentication 0|1 - -## Store working data in DIR (Default: /usr/local/var/lib/tor) -#DataDirectory DIR - -## Every time the specified period elapses, Tor downloads a direc- -## tory. A directory contains a signed list of all known servers -## as well as their current liveness status. A value of "0 sec- -## onds" tells Tor to choose an appropriate default. -## (Default: 1 hour for clients, 20 minutes for servers) -#DirFetchPeriod N seconds|minutes|hours|days|weeks - -## Tor only trusts directories signed with one of these keys, and -## uses the given addresses to connect to the trusted directory -## servers. If no DirServer lines are specified, Tor uses the built-in -## defaults (moria1, moria2, tor26), so you can leave this alone unless -## you need to change it. -## -## WARNING! Changing these options will make your Tor behave -## differently from everyone else's, and hurt your anonymity. Even -## uncommenting these lines is a bad idea. They are the defaults now, -## but the defaults may change in the future, leaving you behind. -## -#DirServer moria1 v1 18.244.0.188:9031 FFCB 46DB 1339 DA84 674C 70D7 CB58 6434 C437 0441 -#DirServer moria2 v1 18.244.0.114:80 719B E45D E224 B607 C537 07D0 E214 3E2D 423E 74CF -#DirServer tor26 v1 86.59.21.38:80 847B 1F85 0344 D787 6491 A548 92F9 0493 4E4E B85D - -## On startup, setgid to this user. -#Group GID - -## Tor will make all its directory requests through this host:port -## (or host:80 if port is not specified), rather than connecting -## directly to any directory servers. -#HttpProxy host[:port] - -## If defined, Tor will use this username:password for Basic Http -## proxy authentication, as in RFC 2617. This is currently the -## only form of Http proxy authentication that Tor supports; feel -## free to submit a patch if you want it to support others. -#HttpProxyAuthenticator username:password - -## Tor will make all its OR (SSL) connections through this -## host:port (or host:443 if port is not specified), via HTTP CON- -## NECT rather than connecting directly to servers. You may want -## to set FascistFirewall to restrict the set of ports you might -## try to connect to, if your Https proxy only allows connecting -## to certain ports. -#HttpsProxy host[:port] - -## If defined, Tor will use this username:password for Basic Https -## proxy authentication, as in RFC 2617. This is currently the -## only form of Https proxy authentication that Tor supports; feel -## free to submit a patch if you want it to support others. -#HttpsProxyAuthenticator username:password - -## To keep firewalls from expiring connections, send a padding -## keepalive cell every NUM seconds on open connections that are -## in use. If the connection has no open circuits, it will instead -## be closed after NUM seconds of idleness. (Default: 5 minutes) -#KeepalivePeriod NUM - -## Send all messages between minSeverity and maxSeverity to the -## standard output stream, the standard error stream, or to the -## system log. (The "syslog" value is only supported on Unix.) -## Recognized severity levels are debug, info, notice, warn, and -## err. If only one severity level is given, all messages of that -## level or higher will be sent to the listed destination. -#Log minSeverity[-maxSeverity] stderr|stdout|syslog - -## As above, but send log messages to the listed filename. The -## "Log" option may appear more than once in a configuration file. -## Messages are sent to all the logs that match their severity -## level. -#Log minSeverity[-maxSeverity] file FILENAME - -## Maximum number of simultaneous sockets allowed. You probably -## don’t need to adjust this. (Default: 1024) -#MaxConn NUM - -## Make all outbound connections originate from the IP address -## specified. This is only useful when you have multiple network -## interfaces, and you want all of Tor’s outgoing connections to -## use a single one. -#OutboundBindAddress IP - -## On startup, write our PID to FILE. On clean shutdown, remove -## FILE. -#PIDFile FILE - -## If 1, Tor forks and daemonizes to the background. (Default: 0) -#RunAsDaemon 0|1 - -## If 1, Tor replaces potentially sensitive strings in the logs -## (e.g. addresses) with the string [scrubbed]. This way logs can -## still be useful, but they don’t leave behind personally identi- -## fying information about what sites a user might have visited. -## (Default: 1) -#SafeLogging 0|1 - -## Every time the specified period elapses, Tor downloads signed -## status information about the current state of known servers. A -## value of "0 seconds" tells Tor to choose an appropriate -## default. (Default: 30 minutes for clients, 15 minutes for -## servers) -#StatusFetchPeriod N seconds|minutes|hours|days|weeks - -## On startup, setuid to this user. -#User UID - -## If non-zero, try to use crypto hardware acceleration when -## available. (Default: 1) -#HardwareAccel 0|1 - - -## Section 2: Client Options Only - -## Where on our circuits should we allow Tor servers that the -## directory servers haven’t authenticated as "verified"? -## (Default: middle,rendezvous) -#AllowUnverifiedNodes entry|exit|middle|introduction|rendezvous|... - -## If set to 1, Tor will under no circumstances run as a server. -## The default is to run as a client unless ORPort is configured. -## (Usually, you don’t need to set this; Tor is pretty smart at -## figuring out whether you are reliable and high-bandwidth enough -## to be a useful server.) -## This option will likely be deprecated in the future; see the -## NoPublish option below. (Default: 0) -#ClientOnly 0|1 - -## A list of preferred nodes to use for the first hop in the -## circuit, if possible. -#EntryNodes nickname,nickname,... - -## A list of preferred nodes to use for the last hop in the -## circuit, if possible. -#ExitNodes nickname,nickname,... - -## A list of nodes to never use when building a circuit. -#ExcludeNodes nickname,nickname,... - -## If 1, Tor will never use any nodes besides those listed in -## "exitnodes" for the last hop of a circuit. -#StrictExitNodes 0|1 - -## If 1, Tor will never use any nodes besides those listed in -## "entrynodes" for the first hop of a circuit. -#StrictEntryNodes 0|1 - -## If 1, Tor will only create outgoing connections to ORs running -## on ports that your firewall allows (defaults to 80 and 443; see -## FirewallPorts). This will allow you to run Tor as a client -## behind a firewall with restrictive policies, but will not allow -## you to run as a server behind such a firewall. -#FascistFirewall 0|1 - -## A list of ports that your firewall allows you to connect to. -## Only used when FascistFirewall is set. (Default: 80, 443) -#FirewallPorts PORTS - -## A comma-separated list of IPs that your firewall allows you to -## connect to. Only used when FascistFirewall is set. The format -## is as for the addresses in ExitPolicy. -## For example, ’FirewallIPs 99.0.0.0/8, *:80’ means that your -## firewall allows connections to everything inside net 99, and -## to port 80 outside. -#FirewallIPs ADDR[/MASK][:PORT]... - -## A list of ports for services that tend to have long-running -## connections (e.g. chat and interactive shells). Circuits for -## streams that use these ports will contain only high-uptime -## nodes, to reduce the chance that a node will go down before the -## stream is finished. (Default: 21, 22, 706, 1863, 5050, 5190, -## 5222, 5223, 6667, 8300, 8888) -#LongLivedPorts PORTS - -## When a request for address arrives to Tor, it will rewrite it -## to newaddress before processing it. For example, if you always -## want connections to www.indymedia.org to exit via torserver -## (where torserver is the nickname of the server), -## use "MapAddress www.indymedia.org www.indymedia.org.torserver.exit". -#MapAddress address newaddress - -## Every NUM seconds consider whether to build a new circuit. -## (Default: 30 seconds) -#NewCircuitPeriod NUM - -## Feel free to reuse a circuit that was first used at most NUM -## seconds ago, but never attach a new stream to a circuit that is -## too old. (Default: 10 minutes) -#MaxCircuitDirtiness NUM - -## The named Tor servers constitute a "family" of similar or co- -## administered servers, so never use any two of them in the same -## circuit. Defining a NodeFamily is only needed when a server -## doesn’t list the family itself (with MyFamily). This option can -## be used multiple times. -#NodeFamily nickname,nickname,... - -## A list of preferred nodes to use for the rendezvous point, if -## possible. -#RendNodes nickname,nickname,... - -## A list of nodes to never use when choosing a rendezvous point. -#RendExcludeNodes nickname,nickname,... - -## Advertise this port to listen for connections from SOCKS-speak- -## ing applications. Set this to 0 if you don’t want to allow -## application connections. (Default: 9050) -#SOCKSPort PORT - -## Bind to this address to listen for connections from SOCKS- -## speaking applications. (Default: 127.0.0.1) You can also spec- -## ify a port (e.g. 192.168.0.1:9100). This directive can be spec- -## ified multiple times to bind to multiple addresses/ports. -#SOCKSBindAddress IP[:PORT] - -## Set an entrance policy for this server, to limit who can con- -## nect to the SOCKS ports. The policies have the same form as -## exit policies below. -#SOCKSPolicy policy,policy,... - -## For each value in the comma separated list, Tor will track -## recent connections to hosts that match this value and attempt -## to reuse the same exit node for each. If the value is prepended -## with a ’.’, it is treated as matching an entire domain. If one -## of the values is just a ’.’, it means match everything. This -## option is useful if you frequently connect to sites that will -## expire all your authentication cookies (ie log you out) if your -## IP address changes. Note that this option does have the disad- -## vantage of making it more clear that a given history is associ- -## ated with a single user. However, most people who would wish to -## observe this will observe it through cookies or other protocol- -## specific means anyhow. -#TrackHostExits host,.domain,... - -## Since exit servers go up and down, it is desirable to expire -## the association between host and exit server after NUM seconds. -## The default is 1800 seconds (30 minutes). -#TrackHostExitsExpire NUM - -## If this option is set to 1, we pick a few entry servers as our -## "helpers", and try to use only those fixed entry servers. This -## is desirable, because constantly changing servers increases the -## odds that an adversary who owns some servers will observe a -## fraction of your paths. (Defaults to 0; will eventually -## default to 1.) -#UseHelperNodes 0|1 - -## If UseHelperNodes is set to 1, we will try to pick a total of -## NUM helper nodes as entries for our circuits. (Defaults to 3.) -#NumHelperNodes NUM - - -## Section 3: Server Options Only - -## The IP or fqdn of this server (e.g. moria.mit.edu). You can -## leave this unset, and Tor will guess your IP. -#Address address - -## Administrative contact information for server. -#ContactInfo email_address - -## Set an exit policy for this server. Each policy is of the form -## "accept|reject ADDR[/MASK][:PORT]". If /MASK is omitted then -## this policy just applies to the host given. Instead of giving -## a host or network you can also use "*" to denote the universe -## (0.0.0.0/0). PORT can be a single port number, an interval of -## ports "FROM_PORT-TO_PORT", or "*". If PORT is omitted, that -## means "*". -## -## For example, "reject 127.0.0.1:*,reject 192.168.1.0/24:*,accept -## *:*" would reject any traffic destined for localhost and any -## 192.168.1.* address, but accept anything else. -## -## This directive can be specified multiple times so you don’t -## have to put it all on one line. -## -## See RFC 3330 for more details about internal and reserved IP -## address space. Policies are considered first to last, and the -## first match wins. If you want to _replace_ the default exit -## policy, end your exit policy with either a reject *:* or an -## accept *:*. Otherwise, you’re _augmenting_ (prepending to) the -## default exit policy. The default exit policy is: -## reject 0.0.0.0/8 -## reject 169.254.0.0/16 -## reject 127.0.0.0/8 -## reject 192.168.0.0/16 -## reject 10.0.0.0/8 -## reject 172.16.0.0/12 -## reject *:25 -## reject *:119 -## reject *:135-139 -## reject *:445 -## reject *:1214 -## reject *:4661-4666 -## reject *:6346-6429 -## reject *:6699 -## reject *:6881-6999 -## accept *:* -#ExitPolicy policy,policy,... - -## If you have more than this number of onionskins queued for -## decrypt, reject new ones. (Default: 100) -#MaxOnionsPending NUM - -## Declare that this Tor server is controlled or administered by a -## group or organization identical or similar to that of the other -## named servers. When two servers both declare that they are in -## the same ’family’, Tor clients will not use them in the same -## circuit. (Each server only needs to list the other servers in -## its family; it doesn’t need to list itself, but it won’t hurt.) -#MyFamily nickname,nickname,... - -## Set the server’s nickname to ’name’. -#Nickname name - -## If you set NoPublish 1, Tor will act as a server if you have an -## ORPort defined, but it will not publish its descriptor to the -## dirservers. This option is useful if you're testing out your -## server, or if you're using alternate dirservers (e.g. for other -## Tor networks such as Blossom). (Default: 0) -#NoPublish 0|1 - -## How many processes to use at once for decrypting onionskins. -## (Default: 1) -#NumCPUs num - -## Advertise this port to listen for connections from Tor clients -## and servers. -#ORPort PORT - -## Bind to this IP address to listen for connections from Tor -## clients and servers. If you specify a port, bind to this port -## rather than the one specified in ORPort. (Default: 0.0.0.0) -#ORBindAddress IP[:PORT] - -## Whenever an outgoing connection tries to connect to one of a -## given set of addresses, connect to target (an address:port -## pair) instead. The address pattern is given in the same format -## as for an exit policy. The address translation applies after -## exit policies are applied. Multiple RedirectExit options can -## be used: once any one has matched successfully, no subsequent -## rules are considered. You can specify that no redirection is -## to be performed on a given set of addresses by using the spe- -## cial target string "pass", which prevents subsequent rules from -## being considered. -#RedirectExit pattern target - -## When we get a SIGINT and we're a server, we begin shutting -## down: we close listeners and start refusing new circuits. After -## NUM seconds, we exit. If we get a second SIGINT, we exit imme- -## diately. (Default: 30 seconds) -#ShutdownWaitLengthNUM - -## Every time the specified period elapses, Tor uploads its server -## descriptors to the directory servers. This information is also -## uploaded whenever it changes. (Default: 20 minutes) -#DirPostPeriod N seconds|minutes|hours|days|weeks - -## A token bucket limits the average relayed bandwidth (server -## traffic only, not client traffic) on this node to the specified -## number of bytes per second. -#RelayBandwidthRate N bytes|KB|MB|GB|TB - -## Limit the maximum token bucket size (also known as the burst) for -## relayed traffic (server traffic only, not client traffic) to the -## given number of bytes. -#RelayBandwidthBurst N bytes|KB|MB|GB|TB - -## Never send more than the specified number of bytes in a given -## accounting period, or receive more than that number in the -## period. For example, with AccountingMax set to 1 GB, a server -## could send 900 MB and receive 800 MB and continue running. It -## will only hibernate once one of the two reaches 1 GB. When the -## number of bytes is exhausted, Tor will hibernate until some -## time in the next accounting period. To prevent all servers -## from waking at the same time, Tor will also wait until a random -## point in each period before waking up. If you have bandwidth -## cost issues, enabling hibernation is preferable to setting a -## low bandwidth, since it provides users with a collection of -## fast servers that are up some of the time, which is more useful -## than a set of slow servers that are always "available". -#AccountingMax N bytes|KB|MB|GB|TB - -## Specify how long accounting periods last. If month is given, -## each accounting period runs from the time HH:MM on the dayth -## day of one month to the same day and time of the next. (The -## day must be between 1 and 28.) If week is given, each account- -## ing period runs from the time HH:MM of the dayth day of one -## week to the same day and time of the next week, with Monday as -## day 1 and Sunday as day 7. If day is given, each accounting -## period runs from the time HH:MM each day to the same time on -## the next day. All times are local, and given in 24-hour time. -## (Defaults to "month 1 0:00".) -#AccountingStart day|week|month [day] HH:MM - - -## Section 4: Directory Server Options (for running your own Tor -## network) - -## When this option is set to 1, Tor operates as an authoritative -## directory server. Instead of caching the directory, it gener- -## ates its own list of good servers, signs it, and sends that to -## the clients. Unless the clients already have you listed as a -## trusted directory, you probably do not want to set this option. -## Please coordinate with the other admins at -## tor-ops@freehaven.net if you think you should be a directory. -#AuthoritativeDirectory 0|1 - -## Advertise the directory service on this port. -#DirPort PORT - -## Bind the directory service to this address. If you specify a -## port, bind to this port rather than the one specified in DirPort. -## (Default: 0.0.0.0) -#DirBindAddress IP[:PORT] - -## Set an entrance policy for this server, to limit who can con- -## nect to the directory ports. The policies have the same form -## as exit policies above. -#DirPolicy policy,policy,... - -## STRING is a command-separated list of Tor versions currently -## believed to be safe. The list is included in each directory, -## and nodes which pull down the directory learn whether they need -## to upgrade. This option can appear multiple times: the values -## from multiple lines are spliced together. -#RecommendedVersions STRING - - -## If set to 1, Tor will accept router descriptors with arbitrary -## "Address" elements. Otherwise, if the address is not an IP or -## is a private IP, it will reject the router descriptor. Defaults -## to 0. -#DirAllowPrivateAddresses 0|1 - -## If set to 1, Tor tries to build circuits through all of the -## servers it knows about, so it can tell which are up and which -## are down. This option is only useful for authoritative direc- -## tories, so you probably don't want to use it. -#RunTesting 0|1 - -## Section 5: Hidden Service Options (clients and servers) - -## Store data files for a hidden service in DIRECTORY. Every hid- -## den service must have a separate directory. You may use this -## option multiple times to specify multiple services. -#HiddenServiceDir DIRECTORY - -## Configure a virtual port VIRTPORT for a hidden service. You -## may use this option multiple times; each time applies to the -## service using the most recent hiddenservicedir. By default, -## this option maps the virtual port to the same port on -## 127.0.0.1. You may override the target port, address, or both -## by specifying a target of addr, port, or addr:port. -#HiddenServicePort VIRTPORT [TARGET] - -## If possible, use the specified nodes as introduction points for -## the hidden service. If this is left unset, Tor will be smart -## and pick some reasonable ones; most people can leave this unset. -#HiddenServiceNodes nickname,nickname,... - -## Do not use the specified nodes as introduction points for the -## hidden service. In normal use there is no reason to set this. -#HiddenServiceExcludeNodes nickname,nickname,... - -## Publish the given rendezvous service descriptor versions for the -## hidden service. -#HiddenServiceVersion 0,2 - -## Every time the specified period elapses, Tor uploads any ren- -## dezvous service descriptors to the directory servers. This -## information is also uploaded whenever it changes. -## (Default: 20 minutes) -#RendPostPeriod N seconds|minutes|hours|days|weeks -# diff --git a/src/config/torrc.sample.in b/src/config/torrc.sample.in index d0b1ee1591..f0c78ce5a9 100644 --- a/src/config/torrc.sample.in +++ b/src/config/torrc.sample.in @@ -1,5 +1,5 @@ ## Configuration file for a typical Tor user -## Last updated 12 April 2009 for Tor 0.2.1.14-rc. +## Last updated 16 July 2009 for Tor 0.2.2.1-alpha. ## (May or may not work for much older or much newer versions of Tor.) ## ## Lines that begin with "## " try to explain what's going on. Lines @@ -95,9 +95,22 @@ SocksListenAddress 127.0.0.1 # accept connections only from localhost ## Define these to limit how much relayed traffic you will allow. Your ## own traffic is still unthrottled. Note that RelayBandwidthRate must -## be at least 20 KBytes. -#RelayBandwidthRate 100 KBytes # Throttle traffic to 100KB/s (800Kbps) -#RelayBandwidthBurst 200 KBytes # But allow bursts up to 200KB/s (1600Kbps) +## be at least 20 KB. +#RelayBandwidthRate 100 KB # Throttle traffic to 100KB/s (800Kbps) +#RelayBandwidthBurst 200 KB # But allow bursts up to 200KB/s (1600Kbps) + +## Use these to restrict the maximum traffic per day, week, or month. +## Note that this threshold applies to sent _and_ to received bytes, +## not to their sum: Setting "4 GB" may allow up to 8 GB +## total before hibernating. +## +## Set a maximum of 4 gigabytes each way per period. +#AccountingMax 4 GB +## Each period starts daily at midnight (AccountingMax is per day) +#AccountingStart day 00:00 +## Each period starts on the 3rd of the month at 15:00 (AccountingMax +## is per month) +#AccountingStart month 3 15:00 ## Contact info to be published in the directory, so we can contact you ## if your relay is misconfigured or something else goes wrong. Google @@ -116,8 +129,9 @@ SocksListenAddress 127.0.0.1 # accept connections only from localhost #DirListenAddress 0.0.0.0:9091 ## Uncomment to return an arbitrary blob of html on your DirPort. Now you ## can explain what Tor is if anybody wonders why your IP address is -## contacting them. See contrib/tor-exit-notice.html for a sample. -#DirPortFrontPage /etc/tor/exit-notice.html +## contacting them. See contrib/tor-exit-notice.html in Tor's source +## distribution for a sample. +#DirPortFrontPage @CONFDIR@/tor-exit-notice.html ## Uncomment this if you run more than one Tor relay, and add the identity ## key fingerprint of each Tor relay you control, even if they're on diff --git a/src/or/Makefile.am b/src/or/Makefile.am index ad2476ff15..a9ac3cdee1 100644 --- a/src/or/Makefile.am +++ b/src/or/Makefile.am @@ -1,8 +1,5 @@ -TESTS = test - -noinst_PROGRAMS = test - bin_PROGRAMS = tor +noinst_LIBRARIES = libtor.a if BUILD_NT_SERVICES tor_platform_source=ntmain.c @@ -10,18 +7,30 @@ else tor_platform_source= endif -EXTRA_DIST=ntmain.c +EXTRA_DIST=ntmain.c or_sha1.i + +if USE_EXTERNAL_EVDNS +evdns_source= +else +evdns_source=eventdns.c +endif -tor_SOURCES = buffers.c circuitbuild.c circuitlist.c \ +libtor_a_SOURCES = buffers.c circuitbuild.c circuitlist.c \ circuituse.c command.c config.c \ connection.c connection_edge.c connection_or.c control.c \ cpuworker.c directory.c dirserv.c dirvote.c \ dns.c dnsserv.c geoip.c hibernate.c main.c $(tor_platform_source) \ + microdesc.c \ networkstatus.c onion.c policies.c \ reasons.c relay.c rendcommon.c rendclient.c rendmid.c \ rendservice.c rephist.c router.c routerlist.c routerparse.c \ - eventdns.c \ - tor_main.c + $(evdns_source) config_codedigest.c + +#libtor_a_LIBADD = ../common/libor.a ../common/libor-crypto.a \ +# ../common/libor-event.a + + +tor_SOURCES = tor_main.c AM_CPPFLAGS = -DSHARE_DATADIR="\"$(datadir)\"" \ -DLOCALSTATEDIR="\"$(localstatedir)\"" \ @@ -32,76 +41,48 @@ AM_CPPFLAGS = -DSHARE_DATADIR="\"$(datadir)\"" \ # matters a lot there, and is quite hard to debug if you forget to do it. tor_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ @TOR_LDFLAGS_libevent@ -tor_LDADD = ../common/libor.a ../common/libor-crypto.a \ - -lz @TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ -test_SOURCES = buffers.c circuitbuild.c circuitlist.c \ - circuituse.c command.c config.c \ - connection.c connection_edge.c connection_or.c control.c \ - cpuworker.c directory.c dirserv.c dirvote.c \ - dns.c dnsserv.c geoip.c hibernate.c main.c $(tor_platform_source) \ - networkstatus.c onion.c policies.c \ - reasons.c relay.c rendcommon.c rendclient.c rendmid.c \ - rendservice.c rephist.c router.c routerlist.c routerparse.c \ - eventdns.c \ - test_data.c test.c +tor_LDADD = ./libtor.a ../common/libor.a ../common/libor-crypto.a \ + ../common/libor-event.a \ + @TOR_ZLIB_LIBS@ -lm @TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ -test_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ \ - @TOR_LDFLAGS_libevent@ -test_LDADD = ../common/libor.a ../common/libor-crypto.a \ - -lz @TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ +noinst_HEADERS = buffers.h circuitbuild.h circuitlist.h circuituse.h \ + command.h config.h connection_edge.h connection.h connection_or.h \ + control.h cpuworker.h directory.h dirserv.h dirvote.h dns.h \ + dnsserv.h geoip.h hibernate.h main.h microdesc.h networkstatus.h \ + ntmain.h onion.h policies.h reasons.h relay.h rendclient.h \ + rendcommon.h rendmid.h rendservice.h rephist.h router.h routerlist.h \ + routerparse.h or.h eventdns.h eventdns_tor.h micro-revision.i -noinst_HEADERS = or.h eventdns.h eventdns_tor.h micro-revision.i +config_codedigest.o: or_sha1.i tor_main.o: micro-revision.i micro-revision.i: FORCE - @svkdir=$$SVKROOT; \ - if test "x$$svkdir" = x ; then \ - svkdir=$$HOME/.svk; \ - fi; \ - if test -d ../../.git && test -x "`which git 2>&1;true`" ; then \ - if test -d ../../.git/svn && test -x "`which git-svn 2>&1;true`" ; then \ - git-svn info ../../README | \ - sed -n 's/^Revision: \([0-9][0-9]*\).*/"\1"/p' \ - > micro-revision.tmp \ - || true; \ - fi; \ - elif test -d ../../.svn && test -x "`which svn 2>&1;true`" ; then \ - svn info ../.. | \ - sed -n 's/^Revision: \([0-9][0-9]*\).*/"\1"/p' > micro-revision.tmp \ - || true; \ - elif test -x "`which svk 2>&1;true`" && test -d $$svkdir/local; then \ - location=../..; \ - rev=x; \ - while test x$$rev = xx; do \ - x=`svk info $$location | \ - sed -n 's/^Mirrored From:.*, Rev\. \([0-9][0-9]*\)/\1/p'`; \ - if test x$$x != x; then \ - rev=$$x; \ - break; \ - else \ - loc=`svk info $$location | \ - sed -n 's/^Copied From: \(.*\), Rev\. [0-9][0-9]*/\1/p' | \ - head -1`; \ - if test x$$loc = x; then \ - break; \ - else \ - location=/$$loc; \ - fi; \ - fi; \ - done; \ - if test x$$rev != xx; then \ - echo \"$$rev\" > micro-revision.tmp; \ - fi; \ - fi; \ - if test ! -f micro-revision.tmp ; then \ - if test ! -f micro-revision.i ; then \ - echo '""' > micro-revision.i; \ - fi; \ - elif test ! -f micro-revision.i || \ + @rm -f micro-revision.tmp; \ + if test -d ../../.git && test -x "`which git 2>&1;true`"; then \ + HASH="`git rev-parse --short=16 HEAD`"; \ + echo \"$$HASH\" > micro-revision.tmp; \ + fi; \ + if test ! -f micro-revision.tmp ; then \ + if test ! -f micro-revision.i ; then \ + echo '""' > micro-revision.i; \ + fi; \ + elif test ! -f micro-revision.i || \ test x"`cat micro-revision.tmp`" != x"`cat micro-revision.i`"; then \ - mv micro-revision.tmp micro-revision.i; \ + mv micro-revision.tmp micro-revision.i; \ fi; true +or_sha1.i: $(tor_SOURCES) + if test "@SHA1SUM@" != none; then \ + @SHA1SUM@ $(tor_SOURCES) | @SED@ -n 's/^\(.*\)$$/"\1\\n"/p' > or_sha1.i; \ + elif test "@OPENSSL@" != none; then \ + @OPENSSL@ sha1 $(tor_SOURCES) | @SED@ -n 's/SHA1(\(.*\))= \(.*\)/"\2 \1\\n"/p' > or_sha1.i; \ + else \ + rm or_sha1.i; \ + touch or_sha1.i; \ + fi + +CLEANFILES = micro-revision.i + #Dummy target to ensure that micro-revision.i _always_ gets built. FORCE: diff --git a/src/or/buffers.c b/src/or/buffers.c index 710374638b..db926955b4 100644 --- a/src/or/buffers.c +++ b/src/or/buffers.c @@ -12,6 +12,14 @@ **/ #define BUFFERS_PRIVATE #include "or.h" +#include "buffers.h" +#include "config.h" +#include "connection_edge.h" +#include "connection_or.h" +#include "control.h" +#include "reasons.h" +#include "../common/util.h" +#include "../common/torlog.h" #ifdef HAVE_UNISTD_H #include <unistd.h> #endif @@ -145,10 +153,13 @@ get_freelist(size_t alloc) /** Deallocate a chunk or put it on a freelist */ static void -chunk_free(chunk_t *chunk) +chunk_free_unchecked(chunk_t *chunk) { - size_t alloc = CHUNK_ALLOC_SIZE(chunk->memlen); - chunk_freelist_t *freelist = get_freelist(alloc); + size_t alloc; + chunk_freelist_t *freelist; + + alloc = CHUNK_ALLOC_SIZE(chunk->memlen); + freelist = get_freelist(alloc); if (freelist && freelist->cur_length < freelist->max_length) { chunk->next = freelist->head; freelist->head = chunk; @@ -193,7 +204,7 @@ chunk_new_with_alloc_size(size_t alloc) } #else static void -chunk_free(chunk_t *chunk) +chunk_free_unchecked(chunk_t *chunk) { tor_free(chunk); } @@ -259,13 +270,22 @@ buf_shrink_freelists(int free_all) int n_to_free = free_all ? freelists[i].cur_length : (freelists[i].lowest_length - slack); int n_to_skip = freelists[i].cur_length - n_to_free; + int orig_length = freelists[i].cur_length; int orig_n_to_free = n_to_free, n_freed=0; int orig_n_to_skip = n_to_skip; int new_length = n_to_skip; chunk_t **chp = &freelists[i].head; chunk_t *chunk; while (n_to_skip) { - tor_assert((*chp)->next); + if (! (*chp)->next) { + log_warn(LD_BUG, "I wanted to skip %d chunks in the freelist for " + "%d-byte chunks, but only found %d. (Length %d)", + orig_n_to_skip, (int)freelists[i].alloc_size, + orig_n_to_skip-n_to_skip, freelists[i].cur_length); + assert_freelist_ok(&freelists[i]); + goto done; + } + // tor_assert((*chp)->next); chp = &(*chp)->next; --n_to_skip; } @@ -290,13 +310,15 @@ buf_shrink_freelists(int free_all) } // tor_assert(!n_to_free); freelists[i].cur_length = new_length; - log_info(LD_MM, "Cleaned freelist for %d-byte chunks: kept %d, " - "dropped %d.", - (int)freelists[i].alloc_size, orig_n_to_skip, orig_n_to_free); + log_info(LD_MM, "Cleaned freelist for %d-byte chunks: original " + "length %d, kept %d, dropped %d.", + (int)freelists[i].alloc_size, orig_length, + orig_n_to_skip, orig_n_to_free); } freelists[i].lowest_length = freelists[i].cur_length; assert_freelist_ok(&freelists[i]); } + done: enable_control_logging(); #else (void) free_all; @@ -404,7 +426,7 @@ buf_pullup(buf_t *buf, size_t bytes, int nulterminate) dest->next = src->next; if (buf->tail == src) buf->tail = dest; - chunk_free(src); + chunk_free_unchecked(src); } else { memcpy(CHUNK_WRITE_PTR(dest), src->data, n); dest->datalen += n; @@ -450,7 +472,7 @@ buf_remove_from_front(buf_t *buf, size_t n) buf->head = victim->next; if (buf->tail == victim) buf->tail = NULL; - chunk_free(victim); + chunk_free_unchecked(victim); } } check(); @@ -484,7 +506,7 @@ buf_clear(buf_t *buf) buf->datalen = 0; for (chunk = buf->head; chunk; chunk = next) { next = chunk->next; - chunk_free(chunk); + chunk_free_unchecked(chunk); } buf->head = buf->tail = NULL; } @@ -523,6 +545,8 @@ buf_slack(const buf_t *buf) void buf_free(buf_t *buf) { + if (!buf) + return; buf_clear(buf); buf->magic = 0xdeadbeef; tor_free(buf); @@ -642,12 +666,12 @@ read_to_chunk_tls(buf_t *buf, chunk_t *chunk, tor_tls_t *tls, * (because of EOF), set *<b>reached_eof</b> to 1 and return 0. Return -1 on * error; else return the number of bytes read. */ -/* XXXX021 indicate "read blocked" somehow? */ +/* XXXX023 indicate "read blocked" somehow? */ int read_to_buf(int s, size_t at_most, buf_t *buf, int *reached_eof, int *socket_error) { - /* XXXX021 It's stupid to overload the return values for these functions: + /* XXXX023 It's stupid to overload the return values for these functions: * "error status" and "number of bytes read" are not mutually exclusive. */ int r = 0; @@ -832,7 +856,7 @@ flush_chunk_tls(tor_tls_t *tls, buf_t *buf, chunk_t *chunk, int flush_buf(int s, buf_t *buf, size_t sz, size_t *buf_flushlen) { - /* XXXX021 It's stupid to overload the return values for these functions: + /* XXXX023 It's stupid to overload the return values for these functions: * "error status" and "number of bytes flushed" are not mutually exclusive. */ int r; @@ -1275,6 +1299,47 @@ fetch_from_buf_http(buf_t *buf, return 1; } +/** + * Wait this many seconds before warning the user about using SOCKS unsafely + * again (requires that WarnUnsafeSocks is turned on). */ +#define SOCKS_WARN_INTERVAL 5 + +/** Warn that the user application has made an unsafe socks request using + * protocol <b>socks_protocol</b> on port <b>port</b>. Don't warn more than + * once per SOCKS_WARN_INTERVAL, unless <b>safe_socks</b> is set. */ +static void +log_unsafe_socks_warning(int socks_protocol, const char *address, + uint16_t port, int safe_socks) +{ + static ratelim_t socks_ratelim = RATELIM_INIT(SOCKS_WARN_INTERVAL); + + or_options_t *options = get_options(); + char *m = NULL; + if (! options->WarnUnsafeSocks) + return; + if (safe_socks || (m = rate_limit_log(&socks_ratelim, approx_time()))) { + log_warn(LD_APP, + "Your application (using socks%d to port %d) is giving " + "Tor only an IP address. Applications that do DNS resolves " + "themselves may leak information. Consider using Socks4A " + "(e.g. via privoxy or socat) instead. For more information, " + "please see https://wiki.torproject.org/TheOnionRouter/" + "TorFAQ#SOCKSAndDNS.%s%s", + socks_protocol, + (int)port, + safe_socks ? " Rejecting." : "", + m ? m : ""); + tor_free(m); + } + control_event_client_status(LOG_WARN, + "DANGEROUS_SOCKS PROTOCOL=SOCKS%d ADDRESS=%s:%d", + socks_protocol, address, (int)port); +} + +/** Do not attempt to parse socks messages longer than this. This value is + * actually significantly higher than the longest possible socks message. */ +#define MAX_SOCKS_MESSAGE_LEN 512 + /** There is a (possibly incomplete) socks handshake on <b>buf</b>, of one * of the forms * - socks4: "socksheader username\\0" @@ -1313,14 +1378,10 @@ fetch_from_buf_socks(buf_t *buf, socks_request_t *req, char *next, *startaddr; struct in_addr in; - /* If the user connects with socks4 or the wrong variant of socks5, - * then log a warning to let him know that it might be unwise. */ - static int have_warned_about_unsafe_socks = 0; - if (buf->datalen < 2) /* version and another byte */ return 0; - buf_pullup(buf, 128, 0); + buf_pullup(buf, MAX_SOCKS_MESSAGE_LEN, 0); tor_assert(buf->head && buf->head->datalen >= 2); socksver = *buf->head->data; @@ -1396,21 +1457,8 @@ fetch_from_buf_socks(buf_t *buf, socks_request_t *req, req->port = ntohs(get_uint16(buf->head->data+4+addrlen)); buf_remove_from_front(buf, 6+addrlen); if (req->command != SOCKS_COMMAND_RESOLVE_PTR && - !addressmap_have_mapping(req->address,0) && - !have_warned_about_unsafe_socks) { - log_warn(LD_APP, - "Your application (using socks5 to port %d) is giving " - "Tor only an IP address. Applications that do DNS resolves " - "themselves may leak information. Consider using Socks4A " - "(e.g. via privoxy or socat) instead. For more information, " - "please see http://wiki.noreply.org/noreply/TheOnionRouter/" - "TorFAQ#SOCKSAndDNS.%s", req->port, - safe_socks ? " Rejecting." : ""); - /*have_warned_about_unsafe_socks = 1;*/ - /*(for now, warn every time)*/ - control_event_client_status(LOG_WARN, - "DANGEROUS_SOCKS PROTOCOL=SOCKS5 ADDRESS=%s:%d", - req->address, req->port); + !addressmap_have_mapping(req->address,0)) { + log_unsafe_socks_warning(5, req->address, req->port, safe_socks); if (safe_socks) return -1; } @@ -1475,8 +1523,8 @@ fetch_from_buf_socks(buf_t *buf, socks_request_t *req, return -1; } - req->port = ntohs(*(uint16_t*)(buf->head->data+2)); - destip = ntohl(*(uint32_t*)(buf->head->data+4)); + req->port = ntohs(get_uint16(buf->head->data+2)); + destip = ntohl(get_uint32(buf->head->data+4)); if ((!req->port && req->command!=SOCKS_COMMAND_RESOLVE) || !destip) { log_warn(LD_APP,"socks4: Port or DestIP is zero. Rejecting."); return -1; @@ -1491,7 +1539,8 @@ fetch_from_buf_socks(buf_t *buf, socks_request_t *req, return -1; } log_debug(LD_APP, - "socks4: successfully read destip (%s)", safe_str(tmpbuf)); + "socks4: successfully read destip (%s)", + safe_str_client(tmpbuf)); socks4_prot = socks4; } @@ -1509,20 +1558,9 @@ fetch_from_buf_socks(buf_t *buf, socks_request_t *req, startaddr = NULL; if (socks4_prot != socks4a && - !addressmap_have_mapping(tmpbuf,0) && - !have_warned_about_unsafe_socks) { - log_warn(LD_APP, - "Your application (using socks4 to port %d) is giving Tor " - "only an IP address. Applications that do DNS resolves " - "themselves may leak information. Consider using Socks4A " - "(e.g. via privoxy or socat) instead. For more information, " - "please see http://wiki.noreply.org/noreply/TheOnionRouter/" - "TorFAQ#SOCKSAndDNS.%s", req->port, - safe_socks ? " Rejecting." : ""); - /*have_warned_about_unsafe_socks = 1;*/ /*(for now, warn every time)*/ - control_event_client_status(LOG_WARN, - "DANGEROUS_SOCKS PROTOCOL=SOCKS4 ADDRESS=%s:%d", - tmpbuf, req->port); + !addressmap_have_mapping(tmpbuf,0)) { + log_unsafe_socks_warning(4, tmpbuf, req->port, safe_socks); + if (safe_socks) return -1; } @@ -1614,6 +1652,132 @@ fetch_from_buf_socks(buf_t *buf, socks_request_t *req, } } +/** Inspect a reply from SOCKS server stored in <b>buf</b> according + * to <b>state</b>, removing the protocol data upon success. Return 0 on + * incomplete response, 1 on success and -1 on error, in which case + * <b>reason</b> is set to a descriptive message (free() when finished + * with it). + * + * As a special case, 2 is returned when user/pass is required + * during SOCKS5 handshake and user/pass is configured. + */ +int +fetch_from_buf_socks_client(buf_t *buf, int state, char **reason) +{ + unsigned char *data; + size_t addrlen; + + if (buf->datalen < 2) + return 0; + + buf_pullup(buf, MAX_SOCKS_MESSAGE_LEN, 0); + tor_assert(buf->head && buf->head->datalen >= 2); + + data = (unsigned char *) buf->head->data; + + switch (state) { + case PROXY_SOCKS4_WANT_CONNECT_OK: + /* Wait for the complete response */ + if (buf->head->datalen < 8) + return 0; + + if (data[1] != 0x5a) { + *reason = tor_strdup(socks4_response_code_to_string(data[1])); + return -1; + } + + /* Success */ + buf_remove_from_front(buf, 8); + return 1; + + case PROXY_SOCKS5_WANT_AUTH_METHOD_NONE: + /* we don't have any credentials */ + if (data[1] != 0x00) { + *reason = tor_strdup("server doesn't support any of our " + "available authentication methods"); + return -1; + } + + log_info(LD_NET, "SOCKS 5 client: continuing without authentication"); + buf_clear(buf); + return 1; + + case PROXY_SOCKS5_WANT_AUTH_METHOD_RFC1929: + /* we have a username and password. return 1 if we can proceed without + * providing authentication, or 2 otherwise. */ + switch (data[1]) { + case 0x00: + log_info(LD_NET, "SOCKS 5 client: we have auth details but server " + "doesn't require authentication."); + buf_clear(buf); + return 1; + case 0x02: + log_info(LD_NET, "SOCKS 5 client: need authentication."); + buf_clear(buf); + return 2; + /* fall through */ + } + + *reason = tor_strdup("server doesn't support any of our available " + "authentication methods"); + return -1; + + case PROXY_SOCKS5_WANT_AUTH_RFC1929_OK: + /* handle server reply to rfc1929 authentication */ + if (data[1] != 0x00) { + *reason = tor_strdup("authentication failed"); + return -1; + } + + log_info(LD_NET, "SOCKS 5 client: authentication successful."); + buf_clear(buf); + return 1; + + case PROXY_SOCKS5_WANT_CONNECT_OK: + /* response is variable length. BND.ADDR, etc, isn't needed + * (don't bother with buf_pullup()), but make sure to eat all + * the data used */ + + /* wait for address type field to arrive */ + if (buf->datalen < 4) + return 0; + + switch (data[3]) { + case 0x01: /* ip4 */ + addrlen = 4; + break; + case 0x04: /* ip6 */ + addrlen = 16; + break; + case 0x03: /* fqdn (can this happen here?) */ + if (buf->datalen < 5) + return 0; + addrlen = 1 + data[4]; + break; + default: + *reason = tor_strdup("invalid response to connect request"); + return -1; + } + + /* wait for address and port */ + if (buf->datalen < 6 + addrlen) + return 0; + + if (data[1] != 0x00) { + *reason = tor_strdup(socks5_response_code_to_string(data[1])); + return -1; + } + + buf_remove_from_front(buf, 6 + addrlen); + return 1; + } + + /* shouldn't get here... */ + tor_assert(0); + + return -1; +} + /** Return 1 iff buf looks more like it has an (obsolete) v0 controller * command on it than any valid v1 controller command. */ int diff --git a/src/or/buffers.h b/src/or/buffers.h new file mode 100644 index 0000000000..e50b9ff6fb --- /dev/null +++ b/src/or/buffers.h @@ -0,0 +1,58 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file buffers.h + * \brief Header file for buffers.c. + **/ + +#ifndef _TOR_BUFFERS_H +#define _TOR_BUFFERS_H + +buf_t *buf_new(void); +buf_t *buf_new_with_capacity(size_t size); +void buf_free(buf_t *buf); +void buf_clear(buf_t *buf); +void buf_shrink(buf_t *buf); +void buf_shrink_freelists(int free_all); +void buf_dump_freelist_sizes(int severity); + +size_t buf_datalen(const buf_t *buf); +size_t buf_allocation(const buf_t *buf); +size_t buf_slack(const buf_t *buf); + +int read_to_buf(int s, size_t at_most, buf_t *buf, int *reached_eof, + int *socket_error); +int read_to_buf_tls(tor_tls_t *tls, size_t at_most, buf_t *buf); + +int flush_buf(int s, buf_t *buf, size_t sz, size_t *buf_flushlen); +int flush_buf_tls(tor_tls_t *tls, buf_t *buf, size_t sz, size_t *buf_flushlen); + +int write_to_buf(const char *string, size_t string_len, buf_t *buf); +int write_to_buf_zlib(buf_t *buf, tor_zlib_state_t *state, + const char *data, size_t data_len, int done); +int move_buf_to_buf(buf_t *buf_out, buf_t *buf_in, size_t *buf_flushlen); +int fetch_from_buf(char *string, size_t string_len, buf_t *buf); +int fetch_var_cell_from_buf(buf_t *buf, var_cell_t **out, int linkproto); +int fetch_from_buf_http(buf_t *buf, + char **headers_out, size_t max_headerlen, + char **body_out, size_t *body_used, size_t max_bodylen, + int force_complete); +int fetch_from_buf_socks(buf_t *buf, socks_request_t *req, + int log_sockstype, int safe_socks); +int fetch_from_buf_socks_client(buf_t *buf, int state, char **reason); +int fetch_from_buf_line(buf_t *buf, char *data_out, size_t *data_len); + +int peek_buf_has_control0_command(buf_t *buf); + +void assert_buf_ok(buf_t *buf); + +#ifdef BUFFERS_PRIVATE +int buf_find_string_offset(const buf_t *buf, const char *s, size_t n); +#endif + +#endif + diff --git a/src/or/circuitbuild.c b/src/or/circuitbuild.c index 76713e676f..fe94264555 100644 --- a/src/or/circuitbuild.c +++ b/src/or/circuitbuild.c @@ -9,9 +9,46 @@ * \brief The actual details of building circuits. **/ +#define CIRCUIT_PRIVATE + #include "or.h" +#include "circuitbuild.h" +#include "circuitlist.h" +#include "circuituse.h" +#include "config.h" +#include "connection.h" +#include "connection_edge.h" +#include "connection_or.h" +#include "control.h" +#include "directory.h" +#include "main.h" +#include "networkstatus.h" +#include "onion.h" +#include "policies.h" +#include "relay.h" +#include "rephist.h" +#include "router.h" +#include "routerlist.h" +#include "routerparse.h" +#include "crypto.h" +#undef log +#include <math.h> + +#ifndef MIN +#define MIN(a,b) ((a)<(b)?(a):(b)) +#endif + +#define CBT_BIN_TO_MS(bin) ((bin)*CBT_BIN_WIDTH + (CBT_BIN_WIDTH/2)) /********* START VARIABLES **********/ +/** Global list of circuit build times */ +// XXXX023: Add this as a member for entry_guard_t instead of global? +// Then we could do per-guard statistics, as guards are likely to +// vary in their own latency. The downside of this is that guards +// can change frequently, so we'd be building a lot more circuits +// most likely. +/* XXXX023 Make this static; add accessor functions. */ +circuit_build_times_t circ_times; /** A global list of all circuits at this hop. */ extern circuit_t *global_circuitlist; @@ -47,6 +84,10 @@ static smartlist_t *entry_guards = NULL; * and those changes need to be flushed to disk. */ static int entry_guards_dirty = 0; +/** If set, we're running the unit tests: we should avoid clobbering + * our state file or accessing get_options() or get_or_state() */ +static int unit_tests = 0; + /********* END VARIABLES ************/ static int circuit_deliver_create_cell(circuit_t *circ, @@ -59,6 +100,1348 @@ static int onion_append_hop(crypt_path_t **head_ptr, extend_info_t *choice); static void entry_guards_changed(void); +/** + * This function decides if CBT learning should be disabled. It returns + * true if one or more of the following four conditions are met: + * + * 1. If the cbtdisabled consensus parameter is set. + * 2. If the torrc option LearnCircuitBuildTimeout is false. + * 3. If we are a directory authority + * 4. If we fail to write circuit build time history to our state file. + */ +static int +circuit_build_times_disabled(void) +{ + if (unit_tests) { + return 0; + } else { + int consensus_disabled = networkstatus_get_param(NULL, "cbtdisabled", + 0, 0, 1); + int config_disabled = !get_options()->LearnCircuitBuildTimeout; + int dirauth_disabled = get_options()->AuthoritativeDir; + int state_disabled = (get_or_state()->LastWritten == -1); + + if (consensus_disabled || config_disabled || dirauth_disabled || + state_disabled) { + log_info(LD_CIRC, + "CircuitBuildTime learning is disabled. " + "Consensus=%d, Config=%d, AuthDir=%d, StateFile=%d", + consensus_disabled, config_disabled, dirauth_disabled, + state_disabled); + return 1; + } else { + return 0; + } + } +} + +/** + * Retrieve and bounds-check the cbtmaxtimeouts consensus paramter. + * + * Effect: When this many timeouts happen in the last 'cbtrecentcount' + * circuit attempts, the client should discard all of its history and + * begin learning a fresh timeout value. + */ +static int32_t +circuit_build_times_max_timeouts(void) +{ + return networkstatus_get_param(NULL, "cbtmaxtimeouts", + CBT_DEFAULT_MAX_RECENT_TIMEOUT_COUNT, + CBT_MIN_MAX_RECENT_TIMEOUT_COUNT, + CBT_MAX_MAX_RECENT_TIMEOUT_COUNT); +} + +/** + * Retrieve and bounds-check the cbtnummodes consensus paramter. + * + * Effect: This value governs how many modes to use in the weighted + * average calculation of Pareto parameter Xm. A value of 3 introduces + * some bias (2-5% of CDF) under ideal conditions, but allows for better + * performance in the event that a client chooses guard nodes of radically + * different performance characteristics. + */ +static int32_t +circuit_build_times_default_num_xm_modes(void) +{ + int32_t num = networkstatus_get_param(NULL, "cbtnummodes", + CBT_DEFAULT_NUM_XM_MODES, + CBT_MIN_NUM_XM_MODES, + CBT_MAX_NUM_XM_MODES); + return num; +} + +/** + * Retrieve and bounds-check the cbtmincircs consensus paramter. + * + * Effect: This is the minimum number of circuits to build before + * computing a timeout. + */ +static int32_t +circuit_build_times_min_circs_to_observe(void) +{ + int32_t num = networkstatus_get_param(NULL, "cbtmincircs", + CBT_DEFAULT_MIN_CIRCUITS_TO_OBSERVE, + CBT_MIN_MIN_CIRCUITS_TO_OBSERVE, + CBT_MAX_MIN_CIRCUITS_TO_OBSERVE); + return num; +} + +/** Return true iff <b>cbt</b> has recorded enough build times that we + * want to start acting on the timeout it implies. */ +int +circuit_build_times_enough_to_compute(circuit_build_times_t *cbt) +{ + return cbt->total_build_times >= circuit_build_times_min_circs_to_observe(); +} + +/** + * Retrieve and bounds-check the cbtquantile consensus paramter. + * + * Effect: This is the position on the quantile curve to use to set the + * timeout value. It is a percent (10-99). + */ +double +circuit_build_times_quantile_cutoff(void) +{ + int32_t num = networkstatus_get_param(NULL, "cbtquantile", + CBT_DEFAULT_QUANTILE_CUTOFF, + CBT_MIN_QUANTILE_CUTOFF, + CBT_MAX_QUANTILE_CUTOFF); + return num/100.0; +} + +int +circuit_build_times_get_bw_scale(networkstatus_t *ns) +{ + return networkstatus_get_param(ns, "bwweightscale", + BW_WEIGHT_SCALE, + BW_MIN_WEIGHT_SCALE, + BW_MAX_WEIGHT_SCALE); +} + +/** + * Retrieve and bounds-check the cbtclosequantile consensus paramter. + * + * Effect: This is the position on the quantile curve to use to set the + * timeout value to use to actually close circuits. It is a percent + * (0-99). + */ +static double +circuit_build_times_close_quantile(void) +{ + int32_t param; + /* Cast is safe - circuit_build_times_quantile_cutoff() is capped */ + int32_t min = (int)tor_lround(100*circuit_build_times_quantile_cutoff()); + param = networkstatus_get_param(NULL, "cbtclosequantile", + CBT_DEFAULT_CLOSE_QUANTILE, + CBT_MIN_CLOSE_QUANTILE, + CBT_MAX_CLOSE_QUANTILE); + if (param < min) { + log_warn(LD_DIR, "Consensus parameter cbtclosequantile is " + "too small, raising to %d", min); + param = min; + } + return param / 100.0; +} + +/** + * Retrieve and bounds-check the cbttestfreq consensus paramter. + * + * Effect: Describes how often in seconds to build a test circuit to + * gather timeout values. Only applies if less than 'cbtmincircs' + * have been recorded. + */ +static int32_t +circuit_build_times_test_frequency(void) +{ + int32_t num = networkstatus_get_param(NULL, "cbttestfreq", + CBT_DEFAULT_TEST_FREQUENCY, + CBT_MIN_TEST_FREQUENCY, + CBT_MAX_TEST_FREQUENCY); + return num; +} + +/** + * Retrieve and bounds-check the cbtmintimeout consensus paramter. + * + * Effect: This is the minimum allowed timeout value in milliseconds. + * The minimum is to prevent rounding to 0 (we only check once + * per second). + */ +static int32_t +circuit_build_times_min_timeout(void) +{ + int32_t num = networkstatus_get_param(NULL, "cbtmintimeout", + CBT_DEFAULT_TIMEOUT_MIN_VALUE, + CBT_MIN_TIMEOUT_MIN_VALUE, + CBT_MAX_TIMEOUT_MIN_VALUE); + return num; +} + +/** + * Retrieve and bounds-check the cbtinitialtimeout consensus paramter. + * + * Effect: This is the timeout value to use before computing a timeout, + * in milliseconds. + */ +int32_t +circuit_build_times_initial_timeout(void) +{ + int32_t min = circuit_build_times_min_timeout(); + int32_t param = networkstatus_get_param(NULL, "cbtinitialtimeout", + CBT_DEFAULT_TIMEOUT_INITIAL_VALUE, + CBT_MIN_TIMEOUT_INITIAL_VALUE, + CBT_MAX_TIMEOUT_INITIAL_VALUE); + if (param < min) { + log_warn(LD_DIR, "Consensus parameter cbtinitialtimeout is too small, " + "raising to %d", min); + param = min; + } + return param; +} + +/** + * Retrieve and bounds-check the cbtrecentcount consensus paramter. + * + * Effect: This is the number of circuit build times to keep track of + * for deciding if we hit cbtmaxtimeouts and need to reset our state + * and learn a new timeout. + */ +static int32_t +circuit_build_times_recent_circuit_count(networkstatus_t *ns) +{ + return networkstatus_get_param(ns, "cbtrecentcount", + CBT_DEFAULT_RECENT_CIRCUITS, + CBT_MIN_RECENT_CIRCUITS, + CBT_MAX_RECENT_CIRCUITS); +} + +/** + * This function is called when we get a consensus update. + * + * It checks to see if we have changed any consensus parameters + * that require reallocation or discard of previous stats. + */ +void +circuit_build_times_new_consensus_params(circuit_build_times_t *cbt, + networkstatus_t *ns) +{ + int32_t num = circuit_build_times_recent_circuit_count(ns); + + if (num > 0 && num != cbt->liveness.num_recent_circs) { + int8_t *recent_circs; + log_notice(LD_CIRC, "The Tor Directory Consensus has changed how many " + "circuits we must track to detect network failures from %d " + "to %d.", cbt->liveness.num_recent_circs, num); + + tor_assert(cbt->liveness.timeouts_after_firsthop); + + /* + * Technically this is a circular array that we are reallocating + * and memcopying. However, since it only consists of either 1s + * or 0s, and is only used in a statistical test to determine when + * we should discard our history after a sufficient number of 1's + * have been reached, it is fine if order is not preserved or + * elements are lost. + * + * cbtrecentcount should only be changing in cases of severe network + * distress anyway, so memory correctness here is paramount over + * doing acrobatics to preserve the array. + */ + recent_circs = tor_malloc_zero(sizeof(int8_t)*num); + memcpy(recent_circs, cbt->liveness.timeouts_after_firsthop, + sizeof(int8_t)*MIN(num, cbt->liveness.num_recent_circs)); + + // Adjust the index if it needs it. + if (num < cbt->liveness.num_recent_circs) { + cbt->liveness.after_firsthop_idx = MIN(num-1, + cbt->liveness.after_firsthop_idx); + } + + tor_free(cbt->liveness.timeouts_after_firsthop); + cbt->liveness.timeouts_after_firsthop = recent_circs; + cbt->liveness.num_recent_circs = num; + } +} + +/** Make a note that we're running unit tests (rather than running Tor + * itself), so we avoid clobbering our state file. */ +void +circuitbuild_running_unit_tests(void) +{ + unit_tests = 1; +} + +/** + * Return the initial default or configured timeout in milliseconds + */ +static double +circuit_build_times_get_initial_timeout(void) +{ + double timeout; + if (!unit_tests && get_options()->CircuitBuildTimeout) { + timeout = get_options()->CircuitBuildTimeout*1000; + if (timeout < circuit_build_times_min_timeout()) { + log_warn(LD_CIRC, "Config CircuitBuildTimeout too low. Setting to %ds", + circuit_build_times_min_timeout()/1000); + timeout = circuit_build_times_min_timeout(); + } + } else { + timeout = circuit_build_times_initial_timeout(); + } + return timeout; +} + +/** + * Reset the build time state. + * + * Leave estimated parameters, timeout and network liveness intact + * for future use. + */ +void +circuit_build_times_reset(circuit_build_times_t *cbt) +{ + memset(cbt->circuit_build_times, 0, sizeof(cbt->circuit_build_times)); + cbt->total_build_times = 0; + cbt->build_times_idx = 0; + cbt->have_computed_timeout = 0; +} + +/** + * Initialize the buildtimes structure for first use. + * + * Sets the initial timeout values based on either the config setting, + * the consensus param, or the default (CBT_DEFAULT_TIMEOUT_INITIAL_VALUE). + */ +void +circuit_build_times_init(circuit_build_times_t *cbt) +{ + memset(cbt, 0, sizeof(*cbt)); + cbt->liveness.num_recent_circs = + circuit_build_times_recent_circuit_count(NULL); + cbt->liveness.timeouts_after_firsthop = tor_malloc_zero(sizeof(int8_t)* + cbt->liveness.num_recent_circs); + cbt->close_ms = cbt->timeout_ms = circuit_build_times_get_initial_timeout(); + control_event_buildtimeout_set(cbt, BUILDTIMEOUT_SET_EVENT_RESET); +} + +#if 0 +/** + * Rewind our build time history by n positions. + */ +static void +circuit_build_times_rewind_history(circuit_build_times_t *cbt, int n) +{ + int i = 0; + + cbt->build_times_idx -= n; + cbt->build_times_idx %= CBT_NCIRCUITS_TO_OBSERVE; + + for (i = 0; i < n; i++) { + cbt->circuit_build_times[(i+cbt->build_times_idx) + %CBT_NCIRCUITS_TO_OBSERVE]=0; + } + + if (cbt->total_build_times > n) { + cbt->total_build_times -= n; + } else { + cbt->total_build_times = 0; + } + + log_info(LD_CIRC, + "Rewound history by %d places. Current index: %d. " + "Total: %d", n, cbt->build_times_idx, cbt->total_build_times); +} +#endif + +/** + * Add a new build time value <b>time</b> to the set of build times. Time + * units are milliseconds. + * + * circuit_build_times <b>cbt</b> is a circular array, so loop around when + * array is full. + */ +int +circuit_build_times_add_time(circuit_build_times_t *cbt, build_time_t time) +{ + if (time <= 0 || time > CBT_BUILD_TIME_MAX) { + log_warn(LD_BUG, "Circuit build time is too large (%u)." + "This is probably a bug.", time); + tor_fragile_assert(); + return -1; + } + + log_debug(LD_CIRC, "Adding circuit build time %u", time); + + cbt->circuit_build_times[cbt->build_times_idx] = time; + cbt->build_times_idx = (cbt->build_times_idx + 1) % CBT_NCIRCUITS_TO_OBSERVE; + if (cbt->total_build_times < CBT_NCIRCUITS_TO_OBSERVE) + cbt->total_build_times++; + + if ((cbt->total_build_times % CBT_SAVE_STATE_EVERY) == 0) { + /* Save state every n circuit builds */ + if (!unit_tests && !get_options()->AvoidDiskWrites) + or_state_mark_dirty(get_or_state(), 0); + } + + return 0; +} + +/** + * Return maximum circuit build time + */ +static build_time_t +circuit_build_times_max(circuit_build_times_t *cbt) +{ + int i = 0; + build_time_t max_build_time = 0; + for (i = 0; i < CBT_NCIRCUITS_TO_OBSERVE; i++) { + if (cbt->circuit_build_times[i] > max_build_time + && cbt->circuit_build_times[i] != CBT_BUILD_ABANDONED) + max_build_time = cbt->circuit_build_times[i]; + } + return max_build_time; +} + +#if 0 +/** Return minimum circuit build time */ +build_time_t +circuit_build_times_min(circuit_build_times_t *cbt) +{ + int i = 0; + build_time_t min_build_time = CBT_BUILD_TIME_MAX; + for (i = 0; i < CBT_NCIRCUITS_TO_OBSERVE; i++) { + if (cbt->circuit_build_times[i] && /* 0 <-> uninitialized */ + cbt->circuit_build_times[i] < min_build_time) + min_build_time = cbt->circuit_build_times[i]; + } + if (min_build_time == CBT_BUILD_TIME_MAX) { + log_warn(LD_CIRC, "No build times less than CBT_BUILD_TIME_MAX!"); + } + return min_build_time; +} +#endif + +/** + * Calculate and return a histogram for the set of build times. + * + * Returns an allocated array of histrogram bins representing + * the frequency of index*CBT_BIN_WIDTH millisecond + * build times. Also outputs the number of bins in nbins. + * + * The return value must be freed by the caller. + */ +static uint32_t * +circuit_build_times_create_histogram(circuit_build_times_t *cbt, + build_time_t *nbins) +{ + uint32_t *histogram; + build_time_t max_build_time = circuit_build_times_max(cbt); + int i, c; + + *nbins = 1 + (max_build_time / CBT_BIN_WIDTH); + histogram = tor_malloc_zero(*nbins * sizeof(build_time_t)); + + // calculate histogram + for (i = 0; i < CBT_NCIRCUITS_TO_OBSERVE; i++) { + if (cbt->circuit_build_times[i] == 0 + || cbt->circuit_build_times[i] == CBT_BUILD_ABANDONED) + continue; /* 0 <-> uninitialized */ + + c = (cbt->circuit_build_times[i] / CBT_BIN_WIDTH); + histogram[c]++; + } + + return histogram; +} + +/** + * Return the Pareto start-of-curve parameter Xm. + * + * Because we are not a true Pareto curve, we compute this as the + * weighted average of the N=3 most frequent build time bins. + */ +static build_time_t +circuit_build_times_get_xm(circuit_build_times_t *cbt) +{ + build_time_t i, nbins; + build_time_t *nth_max_bin; + int32_t bin_counts=0; + build_time_t ret = 0; + uint32_t *histogram = circuit_build_times_create_histogram(cbt, &nbins); + int n=0; + int num_modes = circuit_build_times_default_num_xm_modes(); + + // Only use one mode if < 1000 buildtimes. Not enough data + // for multiple. + if (cbt->total_build_times < CBT_NCIRCUITS_TO_OBSERVE) + num_modes = 1; + + nth_max_bin = (build_time_t*)tor_malloc_zero(num_modes*sizeof(build_time_t)); + + for (i = 0; i < nbins; i++) { + if (histogram[i] >= histogram[nth_max_bin[0]]) { + nth_max_bin[0] = i; + } + + for (n = 1; n < num_modes; n++) { + if (histogram[i] >= histogram[nth_max_bin[n]] && + (!histogram[nth_max_bin[n-1]] + || histogram[i] < histogram[nth_max_bin[n-1]])) { + nth_max_bin[n] = i; + } + } + } + + for (n = 0; n < num_modes; n++) { + bin_counts += histogram[nth_max_bin[n]]; + ret += CBT_BIN_TO_MS(nth_max_bin[n])*histogram[nth_max_bin[n]]; + log_info(LD_CIRC, "Xm mode #%d: %u %u", n, CBT_BIN_TO_MS(nth_max_bin[n]), + histogram[nth_max_bin[n]]); + } + + ret /= bin_counts; + tor_free(histogram); + tor_free(nth_max_bin); + + return ret; +} + +/** + * Output a histogram of current circuit build times to + * the or_state_t state structure. + */ +void +circuit_build_times_update_state(circuit_build_times_t *cbt, + or_state_t *state) +{ + uint32_t *histogram; + build_time_t i = 0; + build_time_t nbins = 0; + config_line_t **next, *line; + + histogram = circuit_build_times_create_histogram(cbt, &nbins); + // write to state + config_free_lines(state->BuildtimeHistogram); + next = &state->BuildtimeHistogram; + *next = NULL; + + state->TotalBuildTimes = cbt->total_build_times; + state->CircuitBuildAbandonedCount = 0; + + for (i = 0; i < CBT_NCIRCUITS_TO_OBSERVE; i++) { + if (cbt->circuit_build_times[i] == CBT_BUILD_ABANDONED) + state->CircuitBuildAbandonedCount++; + } + + for (i = 0; i < nbins; i++) { + // compress the histogram by skipping the blanks + if (histogram[i] == 0) continue; + *next = line = tor_malloc_zero(sizeof(config_line_t)); + line->key = tor_strdup("CircuitBuildTimeBin"); + line->value = tor_malloc(25); + tor_snprintf(line->value, 25, "%d %d", + CBT_BIN_TO_MS(i), histogram[i]); + next = &(line->next); + } + + if (!unit_tests) { + if (!get_options()->AvoidDiskWrites) + or_state_mark_dirty(get_or_state(), 0); + } + + tor_free(histogram); +} + +/** + * Shuffle the build times array. + * + * Adapted from http://en.wikipedia.org/wiki/Fisher-Yates_shuffle + */ +static void +circuit_build_times_shuffle_and_store_array(circuit_build_times_t *cbt, + build_time_t *raw_times, + uint32_t num_times) +{ + uint32_t n = num_times; + if (num_times > CBT_NCIRCUITS_TO_OBSERVE) { + log_notice(LD_CIRC, "The number of circuit times that this Tor version " + "uses to calculate build times is less than the number stored " + "in your state file. Decreasing the circuit time history from " + "%d to %d.", num_times, CBT_NCIRCUITS_TO_OBSERVE); + } + + /* This code can only be run on a compact array */ + while (n-- > 1) { + int k = crypto_rand_int(n + 1); /* 0 <= k <= n. */ + build_time_t tmp = raw_times[k]; + raw_times[k] = raw_times[n]; + raw_times[n] = tmp; + } + + /* Since the times are now shuffled, take a random CBT_NCIRCUITS_TO_OBSERVE + * subset (ie the first CBT_NCIRCUITS_TO_OBSERVE values) */ + for (n = 0; n < MIN(num_times, CBT_NCIRCUITS_TO_OBSERVE); n++) { + circuit_build_times_add_time(cbt, raw_times[n]); + } +} + +/** + * Filter old synthetic timeouts that were created before the + * new right-censored Pareto calculation was deployed. + * + * Once all clients before 0.2.1.13-alpha are gone, this code + * will be unused. + */ +static int +circuit_build_times_filter_timeouts(circuit_build_times_t *cbt) +{ + int num_filtered=0, i=0; + double timeout_rate = 0; + build_time_t max_timeout = 0; + + timeout_rate = circuit_build_times_timeout_rate(cbt); + max_timeout = (build_time_t)cbt->close_ms; + + for (i = 0; i < CBT_NCIRCUITS_TO_OBSERVE; i++) { + if (cbt->circuit_build_times[i] > max_timeout) { + build_time_t replaced = cbt->circuit_build_times[i]; + num_filtered++; + cbt->circuit_build_times[i] = CBT_BUILD_ABANDONED; + + log_debug(LD_CIRC, "Replaced timeout %d with %d", replaced, + cbt->circuit_build_times[i]); + } + } + + log_info(LD_CIRC, + "We had %d timeouts out of %d build times, " + "and filtered %d above the max of %u", + (int)(cbt->total_build_times*timeout_rate), + cbt->total_build_times, num_filtered, max_timeout); + + return num_filtered; +} + +/** + * Load histogram from <b>state</b>, shuffling the resulting array + * after we do so. Use this result to estimate parameters and + * calculate the timeout. + * + * Return -1 on error. + */ +int +circuit_build_times_parse_state(circuit_build_times_t *cbt, + or_state_t *state) +{ + int tot_values = 0; + uint32_t loaded_cnt = 0, N = 0; + config_line_t *line; + unsigned int i; + build_time_t *loaded_times; + int err = 0; + circuit_build_times_init(cbt); + + if (circuit_build_times_disabled()) { + return 0; + } + + /* build_time_t 0 means uninitialized */ + loaded_times = tor_malloc_zero(sizeof(build_time_t)*state->TotalBuildTimes); + + for (line = state->BuildtimeHistogram; line; line = line->next) { + smartlist_t *args = smartlist_create(); + smartlist_split_string(args, line->value, " ", + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); + if (smartlist_len(args) < 2) { + log_warn(LD_GENERAL, "Unable to parse circuit build times: " + "Too few arguments to CircuitBuildTime"); + err = 1; + SMARTLIST_FOREACH(args, char*, cp, tor_free(cp)); + smartlist_free(args); + break; + } else { + const char *ms_str = smartlist_get(args,0); + const char *count_str = smartlist_get(args,1); + uint32_t count, k; + build_time_t ms; + int ok; + ms = (build_time_t)tor_parse_ulong(ms_str, 0, 0, + CBT_BUILD_TIME_MAX, &ok, NULL); + if (!ok) { + log_warn(LD_GENERAL, "Unable to parse circuit build times: " + "Unparsable bin number"); + err = 1; + SMARTLIST_FOREACH(args, char*, cp, tor_free(cp)); + smartlist_free(args); + break; + } + count = (uint32_t)tor_parse_ulong(count_str, 0, 0, + UINT32_MAX, &ok, NULL); + if (!ok) { + log_warn(LD_GENERAL, "Unable to parse circuit build times: " + "Unparsable bin count"); + err = 1; + SMARTLIST_FOREACH(args, char*, cp, tor_free(cp)); + smartlist_free(args); + break; + } + + if (loaded_cnt+count+state->CircuitBuildAbandonedCount + > state->TotalBuildTimes) { + log_warn(LD_CIRC, + "Too many build times in state file. " + "Stopping short before %d", + loaded_cnt+count); + SMARTLIST_FOREACH(args, char*, cp, tor_free(cp)); + smartlist_free(args); + break; + } + + for (k = 0; k < count; k++) { + loaded_times[loaded_cnt++] = ms; + } + N++; + SMARTLIST_FOREACH(args, char*, cp, tor_free(cp)); + smartlist_free(args); + } + } + + log_info(LD_CIRC, + "Adding %d timeouts.", state->CircuitBuildAbandonedCount); + for (i=0; i < state->CircuitBuildAbandonedCount; i++) { + loaded_times[loaded_cnt++] = CBT_BUILD_ABANDONED; + } + + if (loaded_cnt != state->TotalBuildTimes) { + log_warn(LD_CIRC, + "Corrupt state file? Build times count mismatch. " + "Read %d times, but file says %d", loaded_cnt, + state->TotalBuildTimes); + err = 1; + circuit_build_times_reset(cbt); + goto done; + } + + circuit_build_times_shuffle_and_store_array(cbt, loaded_times, loaded_cnt); + + /* Verify that we didn't overwrite any indexes */ + for (i=0; i < CBT_NCIRCUITS_TO_OBSERVE; i++) { + if (!cbt->circuit_build_times[i]) + break; + tot_values++; + } + log_info(LD_CIRC, + "Loaded %d/%d values from %d lines in circuit time histogram", + tot_values, cbt->total_build_times, N); + + if (cbt->total_build_times != tot_values + || cbt->total_build_times > CBT_NCIRCUITS_TO_OBSERVE) { + log_warn(LD_CIRC, + "Corrupt state file? Shuffled build times mismatch. " + "Read %d times, but file says %d", tot_values, + state->TotalBuildTimes); + err = 1; + circuit_build_times_reset(cbt); + goto done; + } + + circuit_build_times_set_timeout(cbt); + + if (!state->CircuitBuildAbandonedCount && cbt->total_build_times) { + circuit_build_times_filter_timeouts(cbt); + } + + done: + tor_free(loaded_times); + return err ? -1 : 0; +} + +/** + * Estimates the Xm and Alpha parameters using + * http://en.wikipedia.org/wiki/Pareto_distribution#Parameter_estimation + * + * The notable difference is that we use mode instead of min to estimate Xm. + * This is because our distribution is frechet-like. We claim this is + * an acceptable approximation because we are only concerned with the + * accuracy of the CDF of the tail. + */ +int +circuit_build_times_update_alpha(circuit_build_times_t *cbt) +{ + build_time_t *x=cbt->circuit_build_times; + double a = 0; + int n=0,i=0,abandoned_count=0; + build_time_t max_time=0; + + /* http://en.wikipedia.org/wiki/Pareto_distribution#Parameter_estimation */ + /* We sort of cheat here and make our samples slightly more pareto-like + * and less frechet-like. */ + cbt->Xm = circuit_build_times_get_xm(cbt); + + tor_assert(cbt->Xm > 0); + + for (i=0; i< CBT_NCIRCUITS_TO_OBSERVE; i++) { + if (!x[i]) { + continue; + } + + if (x[i] < cbt->Xm) { + a += tor_mathlog(cbt->Xm); + } else if (x[i] == CBT_BUILD_ABANDONED) { + abandoned_count++; + } else { + a += tor_mathlog(x[i]); + if (x[i] > max_time) + max_time = x[i]; + } + n++; + } + + /* + * We are erring and asserting here because this can only happen + * in codepaths other than startup. The startup state parsing code + * performs this same check, and resets state if it hits it. If we + * hit it at runtime, something serious has gone wrong. + */ + if (n!=cbt->total_build_times) { + log_err(LD_CIRC, "Discrepancy in build times count: %d vs %d", n, + cbt->total_build_times); + } + tor_assert(n==cbt->total_build_times); + + if (max_time <= 0) { + /* This can happen if Xm is actually the *maximum* value in the set. + * It can also happen if we've abandoned every single circuit somehow. + * In either case, tell the caller not to compute a new build timeout. */ + log_warn(LD_BUG, + "Could not determine largest build time (%d). " + "Xm is %dms and we've abandoned %d out of %d circuits.", max_time, + cbt->Xm, abandoned_count, n); + return 0; + } + + a += abandoned_count*tor_mathlog(max_time); + + a -= n*tor_mathlog(cbt->Xm); + // Estimator comes from Eq #4 in: + // "Bayesian estimation based on trimmed samples from Pareto populations" + // by Arturo J. Fernández. We are right-censored only. + a = (n-abandoned_count)/a; + + cbt->alpha = a; + + return 1; +} + +/** + * This is the Pareto Quantile Function. It calculates the point x + * in the distribution such that F(x) = quantile (ie quantile*100% + * of the mass of the density function is below x on the curve). + * + * We use it to calculate the timeout and also to generate synthetic + * values of time for circuits that timeout before completion. + * + * See http://en.wikipedia.org/wiki/Quantile_function, + * http://en.wikipedia.org/wiki/Inverse_transform_sampling and + * http://en.wikipedia.org/wiki/Pareto_distribution#Generating_a_ + * random_sample_from_Pareto_distribution + * That's right. I'll cite wikipedia all day long. + * + * Return value is in milliseconds. + */ +double +circuit_build_times_calculate_timeout(circuit_build_times_t *cbt, + double quantile) +{ + double ret; + tor_assert(quantile >= 0); + tor_assert(1.0-quantile > 0); + tor_assert(cbt->Xm > 0); + + ret = cbt->Xm/pow(1.0-quantile,1.0/cbt->alpha); + if (ret > INT32_MAX) { + ret = INT32_MAX; + } + tor_assert(ret > 0); + return ret; +} + +/** Pareto CDF */ +double +circuit_build_times_cdf(circuit_build_times_t *cbt, double x) +{ + double ret; + tor_assert(cbt->Xm > 0); + ret = 1.0-pow(cbt->Xm/x,cbt->alpha); + tor_assert(0 <= ret && ret <= 1.0); + return ret; +} + +/** + * Generate a synthetic time using our distribution parameters. + * + * The return value will be within the [q_lo, q_hi) quantile points + * on the CDF. + */ +build_time_t +circuit_build_times_generate_sample(circuit_build_times_t *cbt, + double q_lo, double q_hi) +{ + double randval = crypto_rand_double(); + build_time_t ret; + double u; + + /* Generate between [q_lo, q_hi) */ + /*XXXX This is what nextafter is supposed to be for; we should use it on the + * platforms that support it. */ + q_hi -= 1.0/(INT32_MAX); + + tor_assert(q_lo >= 0); + tor_assert(q_hi < 1); + tor_assert(q_lo < q_hi); + + u = q_lo + (q_hi-q_lo)*randval; + + tor_assert(0 <= u && u < 1.0); + /* circuit_build_times_calculate_timeout returns <= INT32_MAX */ + ret = (build_time_t) + tor_lround(circuit_build_times_calculate_timeout(cbt, u)); + tor_assert(ret > 0); + return ret; +} + +/** + * Estimate an initial alpha parameter by solving the quantile + * function with a quantile point and a specific timeout value. + */ +void +circuit_build_times_initial_alpha(circuit_build_times_t *cbt, + double quantile, double timeout_ms) +{ + // Q(u) = Xm/((1-u)^(1/a)) + // Q(0.8) = Xm/((1-0.8))^(1/a)) = CircBuildTimeout + // CircBuildTimeout = Xm/((1-0.8))^(1/a)) + // CircBuildTimeout = Xm*((1-0.8))^(-1/a)) + // ln(CircBuildTimeout) = ln(Xm)+ln(((1-0.8)))*(-1/a) + // -ln(1-0.8)/(ln(CircBuildTimeout)-ln(Xm))=a + tor_assert(quantile >= 0); + tor_assert(cbt->Xm > 0); + cbt->alpha = tor_mathlog(1.0-quantile)/ + (tor_mathlog(cbt->Xm)-tor_mathlog(timeout_ms)); + tor_assert(cbt->alpha > 0); +} + +/** + * Returns true if we need circuits to be built + */ +int +circuit_build_times_needs_circuits(circuit_build_times_t *cbt) +{ + /* Return true if < MIN_CIRCUITS_TO_OBSERVE */ + return !circuit_build_times_enough_to_compute(cbt); +} + +/** + * Returns true if we should build a timeout test circuit + * right now. + */ +int +circuit_build_times_needs_circuits_now(circuit_build_times_t *cbt) +{ + return circuit_build_times_needs_circuits(cbt) && + approx_time()-cbt->last_circ_at > circuit_build_times_test_frequency(); +} + +/** + * Called to indicate that the network showed some signs of liveness, + * i.e. we received a cell. + * + * This is used by circuit_build_times_network_check_live() to decide + * if we should record the circuit build timeout or not. + * + * This function is called every time we receive a cell. Avoid + * syscalls, events, and other high-intensity work. + */ +void +circuit_build_times_network_is_live(circuit_build_times_t *cbt) +{ + time_t now = approx_time(); + if (cbt->liveness.nonlive_timeouts > 0) { + log_notice(LD_CIRC, + "Tor now sees network activity. Restoring circuit build " + "timeout recording. Network was down for %d seconds " + "during %d circuit attempts.", + (int)(now - cbt->liveness.network_last_live), + cbt->liveness.nonlive_timeouts); + } + cbt->liveness.network_last_live = now; + cbt->liveness.nonlive_timeouts = 0; +} + +/** + * Called to indicate that we completed a circuit. Because this circuit + * succeeded, it doesn't count as a timeout-after-the-first-hop. + * + * This is used by circuit_build_times_network_check_changed() to determine + * if we had too many recent timeouts and need to reset our learned timeout + * to something higher. + */ +void +circuit_build_times_network_circ_success(circuit_build_times_t *cbt) +{ + cbt->liveness.timeouts_after_firsthop[cbt->liveness.after_firsthop_idx] = 0; + cbt->liveness.after_firsthop_idx++; + cbt->liveness.after_firsthop_idx %= cbt->liveness.num_recent_circs; +} + +/** + * A circuit just timed out. If it failed after the first hop, record it + * in our history for later deciding if the network speed has changed. + * + * This is used by circuit_build_times_network_check_changed() to determine + * if we had too many recent timeouts and need to reset our learned timeout + * to something higher. + */ +static void +circuit_build_times_network_timeout(circuit_build_times_t *cbt, + int did_onehop) +{ + if (did_onehop) { + cbt->liveness.timeouts_after_firsthop[cbt->liveness.after_firsthop_idx]=1; + cbt->liveness.after_firsthop_idx++; + cbt->liveness.after_firsthop_idx %= cbt->liveness.num_recent_circs; + } +} + +/** + * A circuit was just forcibly closed. If there has been no recent network + * activity at all, but this circuit was launched back when we thought the + * network was live, increment the number of "nonlive" circuit timeouts. + * + * This is used by circuit_build_times_network_check_live() to decide + * if we should record the circuit build timeout or not. + */ +static void +circuit_build_times_network_close(circuit_build_times_t *cbt, + int did_onehop, time_t start_time) +{ + time_t now = time(NULL); + /* + * Check if this is a timeout that was for a circuit that spent its + * entire existence during a time where we have had no network activity. + */ + if (cbt->liveness.network_last_live < start_time) { + if (did_onehop) { + char last_live_buf[ISO_TIME_LEN+1]; + char start_time_buf[ISO_TIME_LEN+1]; + char now_buf[ISO_TIME_LEN+1]; + format_local_iso_time(last_live_buf, cbt->liveness.network_last_live); + format_local_iso_time(start_time_buf, start_time); + format_local_iso_time(now_buf, now); + log_warn(LD_BUG, + "Circuit somehow completed a hop while the network was " + "not live. Network was last live at %s, but circuit launched " + "at %s. It's now %s.", last_live_buf, start_time_buf, + now_buf); + } + cbt->liveness.nonlive_timeouts++; + if (cbt->liveness.nonlive_timeouts == 1) { + log_notice(LD_CIRC, + "Tor has not observed any network activity for the past %d " + "seconds. Disabling circuit build timeout recording.", + (int)(now - cbt->liveness.network_last_live)); + } else { + log_info(LD_CIRC, + "Got non-live timeout. Current count is: %d", + cbt->liveness.nonlive_timeouts); + } + } +} + +/** + * When the network is not live, we do not record circuit build times. + * + * The network is considered not live if there has been at least one + * circuit build that began and ended (had its close_ms measurement + * period expire) since we last received a cell. + * + * Also has the side effect of rewinding the circuit time history + * in the case of recent liveness changes. + */ +int +circuit_build_times_network_check_live(circuit_build_times_t *cbt) +{ + if (cbt->liveness.nonlive_timeouts > 0) { + return 0; + } + + return 1; +} + +/** + * Returns true if we have seen more than MAX_RECENT_TIMEOUT_COUNT of + * the past RECENT_CIRCUITS time out after the first hop. Used to detect + * if the network connection has changed significantly, and if so, + * resets our circuit build timeout to the default. + * + * Also resets the entire timeout history in this case and causes us + * to restart the process of building test circuits and estimating a + * new timeout. + */ +int +circuit_build_times_network_check_changed(circuit_build_times_t *cbt) +{ + int total_build_times = cbt->total_build_times; + int timeout_count=0; + int i; + + /* how many of our recent circuits made it to the first hop but then + * timed out? */ + for (i = 0; i < cbt->liveness.num_recent_circs; i++) { + timeout_count += cbt->liveness.timeouts_after_firsthop[i]; + } + + /* If 80% of our recent circuits are timing out after the first hop, + * we need to re-estimate a new initial alpha and timeout. */ + if (timeout_count < circuit_build_times_max_timeouts()) { + return 0; + } + + circuit_build_times_reset(cbt); + memset(cbt->liveness.timeouts_after_firsthop, 0, + sizeof(*cbt->liveness.timeouts_after_firsthop)* + cbt->liveness.num_recent_circs); + cbt->liveness.after_firsthop_idx = 0; + + /* Check to see if this has happened before. If so, double the timeout + * to give people on abysmally bad network connections a shot at access */ + if (cbt->timeout_ms >= circuit_build_times_get_initial_timeout()) { + if (cbt->timeout_ms > INT32_MAX/2 || cbt->close_ms > INT32_MAX/2) { + log_warn(LD_CIRC, "Insanely large circuit build timeout value. " + "(timeout = %lfmsec, close = %lfmsec)", + cbt->timeout_ms, cbt->close_ms); + } else { + cbt->timeout_ms *= 2; + cbt->close_ms *= 2; + } + } else { + cbt->close_ms = cbt->timeout_ms + = circuit_build_times_get_initial_timeout(); + } + + control_event_buildtimeout_set(cbt, BUILDTIMEOUT_SET_EVENT_RESET); + + log_notice(LD_CIRC, + "Your network connection speed appears to have changed. Resetting " + "timeout to %lds after %d timeouts and %d buildtimes.", + tor_lround(cbt->timeout_ms/1000), timeout_count, + total_build_times); + + return 1; +} + +/** + * Count the number of timeouts in a set of cbt data. + */ +double +circuit_build_times_timeout_rate(const circuit_build_times_t *cbt) +{ + int i=0,timeouts=0; + for (i = 0; i < CBT_NCIRCUITS_TO_OBSERVE; i++) { + if (cbt->circuit_build_times[i] >= cbt->timeout_ms) { + timeouts++; + } + } + + if (!cbt->total_build_times) + return 0; + + return ((double)timeouts)/cbt->total_build_times; +} + +/** + * Count the number of closed circuits in a set of cbt data. + */ +double +circuit_build_times_close_rate(const circuit_build_times_t *cbt) +{ + int i=0,closed=0; + for (i = 0; i < CBT_NCIRCUITS_TO_OBSERVE; i++) { + if (cbt->circuit_build_times[i] == CBT_BUILD_ABANDONED) { + closed++; + } + } + + if (!cbt->total_build_times) + return 0; + + return ((double)closed)/cbt->total_build_times; +} + +/** + * Store a timeout as a synthetic value. + * + * Returns true if the store was successful and we should possibly + * update our timeout estimate. + */ +int +circuit_build_times_count_close(circuit_build_times_t *cbt, + int did_onehop, + time_t start_time) +{ + if (circuit_build_times_disabled()) { + cbt->close_ms = cbt->timeout_ms + = circuit_build_times_get_initial_timeout(); + return 0; + } + + /* Record this force-close to help determine if the network is dead */ + circuit_build_times_network_close(cbt, did_onehop, start_time); + + /* Only count timeouts if network is live.. */ + if (!circuit_build_times_network_check_live(cbt)) { + return 0; + } + + circuit_build_times_add_time(cbt, CBT_BUILD_ABANDONED); + return 1; +} + +/** + * Update timeout counts to determine if we need to expire + * our build time history due to excessive timeouts. + * + * We do not record any actual time values at this stage; + * we are only interested in recording the fact that a timeout + * happened. We record the time values via + * circuit_build_times_count_close() and circuit_build_times_add_time(). + */ +void +circuit_build_times_count_timeout(circuit_build_times_t *cbt, + int did_onehop) +{ + if (circuit_build_times_disabled()) { + cbt->close_ms = cbt->timeout_ms + = circuit_build_times_get_initial_timeout(); + return; + } + + /* Register the fact that a timeout just occurred. */ + circuit_build_times_network_timeout(cbt, did_onehop); + + /* If there are a ton of timeouts, we should reset + * the circuit build timeout. */ + circuit_build_times_network_check_changed(cbt); +} + +/** + * Estimate a new timeout based on history and set our timeout + * variable accordingly. + */ +static int +circuit_build_times_set_timeout_worker(circuit_build_times_t *cbt) +{ + build_time_t max_time; + if (!circuit_build_times_enough_to_compute(cbt)) + return 0; + + if (!circuit_build_times_update_alpha(cbt)) + return 0; + + cbt->timeout_ms = circuit_build_times_calculate_timeout(cbt, + circuit_build_times_quantile_cutoff()); + + cbt->close_ms = circuit_build_times_calculate_timeout(cbt, + circuit_build_times_close_quantile()); + + max_time = circuit_build_times_max(cbt); + + /* Sometimes really fast guard nodes give us such a steep curve + * that this ends up being not that much greater than timeout_ms. + * Make it be at least 1 min to handle this case. */ + cbt->close_ms = MAX(cbt->close_ms, circuit_build_times_initial_timeout()); + + if (cbt->timeout_ms > max_time) { + log_notice(LD_CIRC, + "Circuit build timeout of %dms is beyond the maximum build " + "time we have ever observed. Capping it to %dms.", + (int)cbt->timeout_ms, max_time); + cbt->timeout_ms = max_time; + } + + if (max_time < INT32_MAX/2 && cbt->close_ms > 2*max_time) { + log_info(LD_CIRC, + "Circuit build measurement period of %dms is more than twice " + "the maximum build time we have ever observed. Capping it to " + "%dms.", (int)cbt->close_ms, 2*max_time); + cbt->close_ms = 2*max_time; + } + + cbt->have_computed_timeout = 1; + return 1; +} + +/** + * Exposed function to compute a new timeout. Dispatches events and + * also filters out extremely high timeout values. + */ +void +circuit_build_times_set_timeout(circuit_build_times_t *cbt) +{ + long prev_timeout = tor_lround(cbt->timeout_ms/1000); + double timeout_rate; + + if (!circuit_build_times_set_timeout_worker(cbt)) + return; + + if (cbt->timeout_ms < circuit_build_times_min_timeout()) { + log_warn(LD_CIRC, "Set buildtimeout to low value %lfms. Setting to %dms", + cbt->timeout_ms, circuit_build_times_min_timeout()); + cbt->timeout_ms = circuit_build_times_min_timeout(); + if (cbt->close_ms < cbt->timeout_ms) { + /* This shouldn't happen because of MAX() in timeout_worker above, + * but doing it just in case */ + cbt->close_ms = circuit_build_times_initial_timeout(); + } + } + + control_event_buildtimeout_set(cbt, BUILDTIMEOUT_SET_EVENT_COMPUTED); + + timeout_rate = circuit_build_times_timeout_rate(cbt); + + if (prev_timeout > tor_lround(cbt->timeout_ms/1000)) { + log_notice(LD_CIRC, + "Based on %d circuit times, it looks like we don't need to " + "wait so long for circuits to finish. We will now assume a " + "circuit is too slow to use after waiting %ld seconds.", + cbt->total_build_times, + tor_lround(cbt->timeout_ms/1000)); + log_info(LD_CIRC, + "Circuit timeout data: %lfms, %lfms, Xm: %d, a: %lf, r: %lf", + cbt->timeout_ms, cbt->close_ms, cbt->Xm, cbt->alpha, + timeout_rate); + } else if (prev_timeout < tor_lround(cbt->timeout_ms/1000)) { + log_notice(LD_CIRC, + "Based on %d circuit times, it looks like we need to wait " + "longer for circuits to finish. We will now assume a " + "circuit is too slow to use after waiting %ld seconds.", + cbt->total_build_times, + tor_lround(cbt->timeout_ms/1000)); + log_info(LD_CIRC, + "Circuit timeout data: %lfms, %lfms, Xm: %d, a: %lf, r: %lf", + cbt->timeout_ms, cbt->close_ms, cbt->Xm, cbt->alpha, + timeout_rate); + } else { + log_info(LD_CIRC, + "Set circuit build timeout to %lds (%lfms, %lfms, Xm: %d, a: %lf," + " r: %lf) based on %d circuit times", + tor_lround(cbt->timeout_ms/1000), + cbt->timeout_ms, cbt->close_ms, cbt->Xm, cbt->alpha, timeout_rate, + cbt->total_build_times); + } +} + /** Iterate over values of circ_id, starting from conn-\>next_circ_id, * and with the high bit specified by conn-\>circ_id_type, until we get * a circ_id that is not in use by any other circuit on that conn. @@ -112,21 +1495,21 @@ circuit_list_path_impl(origin_circuit_t *circ, int verbose, int verbose_names) crypt_path_t *hop; smartlist_t *elements; const char *states[] = {"closed", "waiting for keys", "open"}; - char buf[128]; char *s; elements = smartlist_create(); if (verbose) { const char *nickname = build_state_get_exit_nickname(circ->build_state); - tor_snprintf(buf, sizeof(buf), "%s%s circ (length %d%s%s):", + char *cp; + tor_asprintf(&cp, "%s%s circ (length %d%s%s):", circ->build_state->is_internal ? "internal" : "exit", circ->build_state->need_uptime ? " (high-uptime)" : "", circ->build_state->desired_path_len, circ->_base.state == CIRCUIT_STATE_OPEN ? "" : ", exit ", circ->_base.state == CIRCUIT_STATE_OPEN ? "" : (nickname?nickname:"*unnamed*")); - smartlist_add(elements, tor_strdup(buf)); + smartlist_add(elements, cp); } hop = circ->cpath; @@ -148,8 +1531,7 @@ circuit_list_path_impl(origin_circuit_t *circ, int verbose, int verbose_names) router_get_verbose_nickname(elt, ri); } else if ((rs = router_get_consensus_status_by_id(id))) { routerstatus_get_verbose_nickname(elt, rs); - } else if (hop->extend_info->nickname && - is_legal_nickname(hop->extend_info->nickname)) { + } else if (is_legal_nickname(hop->extend_info->nickname)) { elt[0] = '$'; base16_encode(elt+1, HEX_DIGEST_LEN+1, id, DIGEST_LEN); elt[HEX_DIGEST_LEN+1]= '~'; @@ -217,7 +1599,7 @@ void circuit_log_path(int severity, unsigned int domain, origin_circuit_t *circ) { char *s = circuit_list_path(circ,1); - log(severity,domain,"%s",s); + tor_log(severity,domain,"%s",s); tor_free(s); } @@ -269,7 +1651,7 @@ static int onion_populate_cpath(origin_circuit_t *circ) { int r; -again: + again: r = onion_extend_cpath(circ); if (r < 0) { log_info(LD_CIRC,"Generating cpath hop failed."); @@ -361,9 +1743,10 @@ circuit_handle_first_hop(origin_circuit_t *circ) if (!n_conn) { /* not currently connected in a useful way. */ - const char *name = firsthop->extend_info->nickname ? + const char *name = strlen(firsthop->extend_info->nickname) ? firsthop->extend_info->nickname : fmt_addr(&firsthop->extend_info->addr); - log_info(LD_CIRC, "Next router is %s: %s ", safe_str(name), msg?msg:"???"); + log_info(LD_CIRC, "Next router is %s: %s ", + safe_str_client(name), msg?msg:"???"); circ->_base.n_hop = extend_info_dup(firsthop->extend_info); if (should_launch) { @@ -506,7 +1889,8 @@ circuit_deliver_create_cell(circuit_t *circ, uint8_t cell_type, cell.circ_id = circ->n_circ_id; memcpy(cell.payload, payload, ONIONSKIN_CHALLENGE_LEN); - append_cell_to_circuit_queue(circ, circ->n_conn, &cell, CELL_DIRECTION_OUT); + append_cell_to_circuit_queue(circ, circ->n_conn, &cell, + CELL_DIRECTION_OUT, 0); if (CIRCUIT_IS_ORIGIN(circ)) { /* mark it so it gets better rate limiting treatment. */ @@ -536,7 +1920,7 @@ inform_testing_reachability(void) "CHECKING_REACHABILITY DIRADDRESS=%s:%d", me->address, me->dir_port); } - log(LOG_NOTICE, LD_OR, "Now checking whether ORPort %s:%d%s %s reachable... " + log_notice(LD_OR, "Now checking whether ORPort %s:%d%s %s reachable... " "(this may take up to %d minutes -- look for log " "messages indicating success)", me->address, me->or_port, @@ -569,6 +1953,18 @@ should_use_create_fast_for_circuit(origin_circuit_t *circ) return 1; } +/** Return true if <b>circ</b> is the type of circuit we want to count + * timeouts from. In particular, we want it to have not completed yet + * (already completing indicates we cannibalized it), and we want it to + * have exactly three hops. + */ +int +circuit_timeout_want_to_count_circ(origin_circuit_t *circ) +{ + return !circ->has_opened + && circ->build_state->desired_path_len == DEFAULT_ROUTE_LEN; +} + /** This is the backbone function for building circuits. * * If circ's first hop is closed, then we need to build a create @@ -642,15 +2038,43 @@ circuit_send_next_onion_skin(origin_circuit_t *circ) if (!hop) { /* done building the circuit. whew. */ circuit_set_state(TO_CIRCUIT(circ), CIRCUIT_STATE_OPEN); + if (circuit_timeout_want_to_count_circ(circ)) { + struct timeval end; + long timediff; + tor_gettimeofday(&end); + timediff = tv_mdiff(&circ->_base.timestamp_created, &end); + + /* + * If the circuit build time is much greater than we would have cut + * it off at, we probably had a suspend event along this codepath, + * and we should discard the value. + */ + if (timediff < 0 || timediff > 2*circ_times.close_ms+1000) { + log_notice(LD_CIRC, "Strange value for circuit build time: %ldmsec. " + "Assuming clock jump. Purpose %d (%s)", timediff, + circ->_base.purpose, + circuit_purpose_to_string(circ->_base.purpose)); + } else if (!circuit_build_times_disabled()) { + /* Only count circuit times if the network is live */ + if (circuit_build_times_network_check_live(&circ_times)) { + circuit_build_times_add_time(&circ_times, (build_time_t)timediff); + circuit_build_times_set_timeout(&circ_times); + } + + if (circ->_base.purpose != CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT) { + circuit_build_times_network_circ_success(&circ_times); + } + } + } log_info(LD_CIRC,"circuit built!"); circuit_reset_failure_count(0); if (circ->build_state->onehop_tunnel) control_event_bootstrap(BOOTSTRAP_STATUS_REQUESTING_STATUS, 0); - if (!has_completed_circuit && !circ->build_state->onehop_tunnel) { + if (!can_complete_circuit && !circ->build_state->onehop_tunnel) { or_options_t *options = get_options(); - has_completed_circuit=1; + can_complete_circuit=1; /* FFFF Log a count of known routers here */ - log(LOG_NOTICE, LD_GENERAL, + log_notice(LD_GENERAL, "Tor has successfully opened a circuit. " "Looks like client functionality is working."); control_event_bootstrap(BOOTSTRAP_STATUS_DONE, 0); @@ -662,6 +2086,10 @@ circuit_send_next_onion_skin(origin_circuit_t *circ) } circuit_rep_hist_note_result(circ); circuit_has_opened(circ); /* do other actions as necessary */ + + /* We're done with measurement circuits here. Just close them */ + if (circ->_base.purpose == CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT) + circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_FINISHED); return 0; } @@ -705,13 +2133,13 @@ void circuit_note_clock_jumped(int seconds_elapsed) { int severity = server_mode(get_options()) ? LOG_WARN : LOG_NOTICE; - log(severity, LD_GENERAL, "Your system clock just jumped %d seconds %s; " + tor_log(severity, LD_GENERAL, "Your system clock just jumped %d seconds %s; " "assuming established circuits no longer work.", seconds_elapsed >=0 ? seconds_elapsed : -seconds_elapsed, seconds_elapsed >=0 ? "forward" : "backward"); control_event_general_status(LOG_WARN, "CLOCK_JUMPED TIME=%d", seconds_elapsed); - has_completed_circuit=0; /* so it'll log when it works again */ + can_complete_circuit=0; /* so it'll log when it works again */ control_event_client_status(severity, "CIRCUIT_NOT_ESTABLISHED REASON=%s", "CLOCK_JUMPED"); circuit_mark_all_unused_circs(); @@ -944,10 +2372,9 @@ circuit_finish_handshake(origin_circuit_t *circ, uint8_t reply_type, return -END_CIRC_REASON_TORPROTOCOL; } - if (hop->dh_handshake_state) { - crypto_dh_free(hop->dh_handshake_state); /* don't need it anymore */ - hop->dh_handshake_state = NULL; - } + crypto_dh_free(hop->dh_handshake_state); /* don't need it anymore */ + hop->dh_handshake_state = NULL; + memset(hop->fast_handshake_state, 0, sizeof(hop->fast_handshake_state)); if (circuit_init_cpath_crypto(hop, keys, 0)<0) { @@ -1035,8 +2462,8 @@ onionskin_answer(or_circuit_t *circ, uint8_t cell_type, const char *payload, cell_type == CELL_CREATED ? ONIONSKIN_REPLY_LEN : DIGEST_LEN*2); log_debug(LD_CIRC,"init digest forward 0x%.8x, backward 0x%.8x.", - (unsigned int)*(uint32_t*)(keys), - (unsigned int)*(uint32_t*)(keys+20)); + (unsigned int)get_uint32(keys), + (unsigned int)get_uint32(keys+20)); if (circuit_init_cpath_crypto(tmp_cpath, keys, 0)<0) { log_warn(LD_BUG,"Circuit initialization failed"); tor_free(tmp_cpath); @@ -1057,7 +2484,7 @@ onionskin_answer(or_circuit_t *circ, uint8_t cell_type, const char *payload, circ->is_first_hop = (cell_type == CELL_CREATED_FAST); append_cell_to_circuit_queue(TO_CIRCUIT(circ), - circ->p_conn, &cell, CELL_DIRECTION_IN); + circ->p_conn, &cell, CELL_DIRECTION_IN, 0); log_debug(LD_CIRC,"Finished sending 'created' cell."); if (!is_local_addr(&circ->p_conn->_base.addr) && @@ -1086,7 +2513,7 @@ new_route_len(uint8_t purpose, extend_info_t *exit, tor_assert(routers); - routelen = 3; + routelen = DEFAULT_ROUTE_LEN; if (exit && purpose != CIRCUIT_PURPOSE_TESTING && purpose != CIRCUIT_PURPOSE_S_ESTABLISH_INTRO) @@ -1151,6 +2578,8 @@ circuit_all_predicted_ports_handled(time_t now, int *need_uptime, smartlist_t *LongLivedServices = get_options()->LongLivedPorts; tor_assert(need_uptime); tor_assert(need_capacity); + // Always predict need_capacity + *need_capacity = 1; enough = (smartlist_len(sl) == 0); for (i = 0; i < smartlist_len(sl); ++i) { port = smartlist_get(sl, i); @@ -1173,6 +2602,8 @@ router_handles_some_port(routerinfo_t *router, smartlist_t *needed_ports) for (i = 0; i < smartlist_len(needed_ports); ++i) { addr_policy_result_t r; + /* alignment issues aren't a worry for this dereference, since + needed_ports is explicitly a smartlist of uint16_t's */ port = *(uint16_t *)smartlist_get(needed_ports, i); tor_assert(port); r = compare_addr_to_addr_policy(0, port, router->exit_policy); @@ -1255,9 +2686,24 @@ choose_good_exit_server_general(routerlist_t *dir, int need_uptime, n_supported[i] = -1; continue; /* skip routers that are known to be down or bad exits */ } + + if (options->_ExcludeExitNodesUnion && + routerset_contains_router(options->_ExcludeExitNodesUnion, router)) { + n_supported[i] = -1; + continue; /* user asked us not to use it, no matter what */ + } + if (options->ExitNodes && + !routerset_contains_router(options->ExitNodes, router)) { + n_supported[i] = -1; + continue; /* not one of our chosen exit nodes */ + } + if (router_is_unreliable(router, need_uptime, need_capacity, 0)) { n_supported[i] = -1; - continue; /* skip routers that are not suitable */ + continue; /* skip routers that are not suitable. Don't worry if + * this makes us reject all the possible routers: if so, + * we'll retry later in this function with need_update and + * need_capacity set to 0. */ } if (!(router->is_valid || options->_AllowInvalid & ALLOW_INVALID_EXIT)) { /* if it's invalid and we don't want it */ @@ -1316,28 +2762,21 @@ choose_good_exit_server_general(routerlist_t *dir, int need_uptime, /* If any routers definitely support any pending connections, choose one * at random. */ if (best_support > 0) { - smartlist_t *supporting = smartlist_create(), *use = smartlist_create(); + smartlist_t *supporting = smartlist_create(); for (i = 0; i < smartlist_len(dir->routers); i++) if (n_supported[i] == best_support) smartlist_add(supporting, smartlist_get(dir->routers, i)); - routersets_get_disjunction(use, supporting, options->ExitNodes, - options->_ExcludeExitNodesUnion, 1); - if (smartlist_len(use) == 0 && !options->StrictExitNodes) { - routersets_get_disjunction(use, supporting, NULL, - options->_ExcludeExitNodesUnion, 1); - } - router = routerlist_sl_choose_by_bandwidth(use, WEIGHT_FOR_EXIT); - smartlist_free(use); + router = routerlist_sl_choose_by_bandwidth(supporting, WEIGHT_FOR_EXIT); smartlist_free(supporting); } else { /* Either there are no pending connections, or no routers even seem to * possibly support any of them. Choose a router at random that satisfies * at least one predicted exit port. */ - int try; - smartlist_t *needed_ports, *supporting, *use; + int attempt; + smartlist_t *needed_ports, *supporting; if (best_support == -1) { if (need_uptime || need_capacity) { @@ -1349,42 +2788,32 @@ choose_good_exit_server_general(routerlist_t *dir, int need_uptime, tor_free(n_supported); return choose_good_exit_server_general(dir, 0, 0); } - log_notice(LD_CIRC, "All routers are down or won't exit -- choosing a " - "doomed exit at random."); + log_notice(LD_CIRC, "All routers are down or won't exit%s -- " + "choosing a doomed exit at random.", + options->_ExcludeExitNodesUnion ? " or are Excluded" : ""); } supporting = smartlist_create(); - use = smartlist_create(); needed_ports = circuit_get_unhandled_ports(time(NULL)); - for (try = 0; try < 2; try++) { + for (attempt = 0; attempt < 2; attempt++) { /* try once to pick only from routers that satisfy a needed port, * then if there are none, pick from any that support exiting. */ for (i = 0; i < smartlist_len(dir->routers); i++) { router = smartlist_get(dir->routers, i); if (n_supported[i] != -1 && - (try || router_handles_some_port(router, needed_ports))) { + (attempt || router_handles_some_port(router, needed_ports))) { // log_fn(LOG_DEBUG,"Try %d: '%s' is a possibility.", // try, router->nickname); smartlist_add(supporting, router); } } - routersets_get_disjunction(use, supporting, options->ExitNodes, - options->_ExcludeExitNodesUnion, 1); - if (smartlist_len(use) == 0 && !options->StrictExitNodes) { - routersets_get_disjunction(use, supporting, NULL, - options->_ExcludeExitNodesUnion, 1); - } - /* XXX sometimes the above results in null, when the requested - * exit node is down. we should pick it anyway. */ - router = routerlist_sl_choose_by_bandwidth(use, WEIGHT_FOR_EXIT); + router = routerlist_sl_choose_by_bandwidth(supporting, WEIGHT_FOR_EXIT); if (router) break; smartlist_clear(supporting); - smartlist_clear(use); } SMARTLIST_FOREACH(needed_ports, uint16_t *, cp, tor_free(cp)); smartlist_free(needed_ports); - smartlist_free(use); smartlist_free(supporting); } @@ -1393,10 +2822,11 @@ choose_good_exit_server_general(routerlist_t *dir, int need_uptime, log_info(LD_CIRC, "Chose exit server '%s'", router->nickname); return router; } - if (options->StrictExitNodes) { + if (options->ExitNodes) { log_warn(LD_CIRC, - "No specified exit routers seem to be running, and " - "StrictExitNodes is set: can't choose an exit."); + "No specified %sexit routers seem to be running: " + "can't choose an exit.", + options->_ExcludeExitNodesUnion ? "non-excluded " : ""); } return NULL; } @@ -1427,15 +2857,13 @@ choose_good_exit_server(uint8_t purpose, routerlist_t *dir, if (options->_AllowInvalid & ALLOW_INVALID_MIDDLE) flags |= CRN_ALLOW_INVALID; if (is_internal) /* pick it like a middle hop */ - return router_choose_random_node(NULL, NULL, - options->ExcludeNodes, flags); + return router_choose_random_node(NULL, options->ExcludeNodes, flags); else return choose_good_exit_server_general(dir,need_uptime,need_capacity); case CIRCUIT_PURPOSE_C_ESTABLISH_REND: if (options->_AllowInvalid & ALLOW_INVALID_RENDEZVOUS) flags |= CRN_ALLOW_INVALID; - return router_choose_random_node(NULL, NULL, - options->ExcludeNodes, flags); + return router_choose_random_node(NULL, options->ExcludeNodes, flags); } log_warn(LD_BUG,"Unhandled purpose %d", purpose); tor_fragile_assert(); @@ -1450,7 +2878,6 @@ warn_if_last_router_excluded(origin_circuit_t *circ, const extend_info_t *exit) or_options_t *options = get_options(); routerset_t *rs = options->ExcludeNodes; const char *description; - int domain = LD_CIRC; uint8_t purpose = circ->_base.purpose; if (circ->build_state->onehop_tunnel) @@ -1463,13 +2890,14 @@ warn_if_last_router_excluded(origin_circuit_t *circ, const extend_info_t *exit) case CIRCUIT_PURPOSE_INTRO_POINT: case CIRCUIT_PURPOSE_REND_POINT_WAITING: case CIRCUIT_PURPOSE_REND_ESTABLISHED: - log_warn(LD_BUG, "Called on non-origin circuit (purpose %d)", - (int)purpose); + log_warn(LD_BUG, "Called on non-origin circuit (purpose %d, %s)", + (int)purpose, + circuit_purpose_to_string(purpose)); return; case CIRCUIT_PURPOSE_C_GENERAL: if (circ->build_state->is_internal) return; - description = "Requested exit node"; + description = "requested exit node"; rs = options->_ExcludeExitNodesUnion; break; case CIRCUIT_PURPOSE_C_INTRODUCING: @@ -1484,22 +2912,34 @@ warn_if_last_router_excluded(origin_circuit_t *circ, const extend_info_t *exit) case CIRCUIT_PURPOSE_C_REND_READY: case CIRCUIT_PURPOSE_C_REND_READY_INTRO_ACKED: case CIRCUIT_PURPOSE_C_REND_JOINED: - description = "Chosen rendezvous point"; - domain = LD_BUG; + description = "chosen rendezvous point"; break; case CIRCUIT_PURPOSE_CONTROLLER: rs = options->_ExcludeExitNodesUnion; - description = "Controller-selected circuit target"; + description = "controller-selected circuit target"; break; } if (routerset_contains_extendinfo(rs, exit)) { - log_fn(LOG_WARN, domain, "%s '%s' is in ExcludeNodes%s. Using anyway " - "(circuit purpose %d).", - description,exit->nickname, - rs==options->ExcludeNodes?"":" or ExcludeExitNodes", - (int)purpose); - circuit_log_path(LOG_WARN, domain, circ); + /* We should never get here if StrictNodes is set to 1. */ + if (options->StrictNodes) { + log_warn(LD_BUG, "Using %s '%s' which is listed in ExcludeNodes%s, " + "even though StrictNodes is set. Please report. " + "(Circuit purpose: %s)", + description, exit->nickname, + rs==options->ExcludeNodes?"":" or ExcludeExitNodes", + circuit_purpose_to_string(purpose)); + } else { + log_warn(LD_CIRC, "Using %s '%s' which is listed in " + "ExcludeNodes%s, because no better options were available. To " + "prevent this (and possibly break your Tor functionality), " + "set the StrictNodes configuration option. " + "(Circuit purpose: %s)", + description, exit->nickname, + rs==options->ExcludeNodes?"":" or ExcludeExitNodes", + circuit_purpose_to_string(purpose)); + } + circuit_log_path(LOG_WARN, LD_CIRC, circ); } return; @@ -1555,8 +2995,7 @@ circuit_append_new_exit(origin_circuit_t *circ, extend_info_t *exit) state = circ->build_state; tor_assert(state); - if (state->chosen_exit) - extend_info_free(state->chosen_exit); + extend_info_free(state->chosen_exit); state->chosen_exit = extend_info_dup(exit); ++circ->build_state->desired_path_len; @@ -1678,8 +3117,7 @@ choose_good_middle_server(uint8_t purpose, flags |= CRN_NEED_CAPACITY; if (options->_AllowInvalid & ALLOW_INVALID_MIDDLE) flags |= CRN_ALLOW_INVALID; - choice = router_choose_random_node(NULL, - excluded, options->ExcludeNodes, flags); + choice = router_choose_random_node(excluded, options->ExcludeNodes, flags); smartlist_free(excluded); return choice; } @@ -1743,11 +3181,7 @@ choose_good_entry_server(uint8_t purpose, cpath_build_state_t *state) if (options->_AllowInvalid & ALLOW_INVALID_ENTRY) flags |= CRN_ALLOW_INVALID; - choice = router_choose_random_node( - NULL, - excluded, - options->ExcludeNodes, - flags); + choice = router_choose_random_node(excluded, options->ExcludeNodes, flags); smartlist_free(excluded); return choice; } @@ -1868,9 +3302,9 @@ extend_info_from_router(routerinfo_t *r) void extend_info_free(extend_info_t *info) { - tor_assert(info); - if (info->onion_key) - crypto_free_pk_env(info->onion_key); + if (!info) + return; + crypto_free_pk_env(info->onion_key); tor_free(info); } @@ -1929,8 +3363,6 @@ entry_guard_set_status(entry_guard_t *e, routerinfo_t *ri, char buf[HEX_DIGEST_LEN+1]; int changed = 0; - tor_assert(options); - *reason = NULL; /* Do we want to mark this guard as bad? */ @@ -1993,35 +3425,58 @@ entry_is_time_to_retry(entry_guard_t *e, time_t now) * - Listed as either up or never yet contacted; * - Present in the routerlist; * - Listed as 'stable' or 'fast' by the current dirserver consensus, - * if demanded by <b>need_uptime</b> or <b>need_capacity</b>; - * (This check is currently redundant with the Guard flag, but in - * the future that might change. Best to leave it in for now.) + * if demanded by <b>need_uptime</b> or <b>need_capacity</b> + * (unless it's a configured EntryNode); * - Allowed by our current ReachableORAddresses config option; and - * - Currently thought to be reachable by us (unless assume_reachable + * - Currently thought to be reachable by us (unless <b>assume_reachable</b> * is true). + * + * If the answer is no, set *<b>msg</b> to an explanation of why. */ static INLINE routerinfo_t * entry_is_live(entry_guard_t *e, int need_uptime, int need_capacity, - int assume_reachable) + int assume_reachable, const char **msg) { routerinfo_t *r; - if (e->bad_since) + or_options_t *options = get_options(); + tor_assert(msg); + + if (e->bad_since) { + *msg = "bad"; return NULL; + } /* no good if it's unreachable, unless assume_unreachable or can_retry. */ if (!assume_reachable && !e->can_retry && - e->unreachable_since && !entry_is_time_to_retry(e, time(NULL))) + e->unreachable_since && !entry_is_time_to_retry(e, time(NULL))) { + *msg = "unreachable"; return NULL; + } r = router_get_by_digest(e->identity); - if (!r) + if (!r) { + *msg = "no descriptor"; return NULL; - if (get_options()->UseBridges && r->purpose != ROUTER_PURPOSE_BRIDGE) + } + if (get_options()->UseBridges && r->purpose != ROUTER_PURPOSE_BRIDGE) { + *msg = "not a bridge"; return NULL; - if (!get_options()->UseBridges && r->purpose != ROUTER_PURPOSE_GENERAL) + } + if (!get_options()->UseBridges && r->purpose != ROUTER_PURPOSE_GENERAL) { + *msg = "not general-purpose"; return NULL; - if (router_is_unreliable(r, need_uptime, need_capacity, 0)) + } + if (options->EntryNodes && + routerset_contains_router(options->EntryNodes, r)) { + /* they asked for it, they get it */ + need_uptime = need_capacity = 0; + } + if (router_is_unreliable(r, need_uptime, need_capacity, 0)) { + *msg = "not fast/stable"; return NULL; - if (!fascist_firewall_allows_or(r)) + } + if (!fascist_firewall_allows_or(r)) { + *msg = "unreachable by config"; return NULL; + } return r; } @@ -2030,11 +3485,12 @@ static int num_live_entry_guards(void) { int n = 0; + const char *msg; if (! entry_guards) return 0; SMARTLIST_FOREACH(entry_guards, entry_guard_t *, entry, { - if (entry_is_live(entry, 0, 1, 0)) + if (entry_is_live(entry, 0, 1, 0, &msg)) ++n; }); return n; @@ -2058,16 +3514,21 @@ static void log_entry_guards(int severity) { smartlist_t *elements = smartlist_create(); - char buf[1024]; char *s; SMARTLIST_FOREACH(entry_guards, entry_guard_t *, e, { - tor_snprintf(buf, sizeof(buf), "%s (%s%s)", - e->nickname, - entry_is_live(e, 0, 1, 0) ? "up " : "down ", - e->made_contact ? "made-contact" : "never-contacted"); - smartlist_add(elements, tor_strdup(buf)); + const char *msg = NULL; + char *cp; + if (entry_is_live(e, 0, 1, 0, &msg)) + tor_asprintf(&cp, "%s (up %s)", + e->nickname, + e->made_contact ? "made-contact" : "never-contacted"); + else + tor_asprintf(&cp, "%s (%s, %s)", + e->nickname, msg, + e->made_contact ? "made-contact" : "never-contacted"); + smartlist_add(elements, cp); }); s = smartlist_join_strings(elements, ",", 0, NULL); @@ -2091,12 +3552,13 @@ control_event_guard_deferred(void) **/ #if 0 int n = 0; + const char *msg; or_options_t *options = get_options(); if (!entry_guards) return; SMARTLIST_FOREACH(entry_guards, entry_guard_t *, entry, { - if (entry_is_live(entry, 0, 1, 0)) { + if (entry_is_live(entry, 0, 1, 0, &msg)) { if (n++ == options->NumEntryGuards) { control_event_guard(entry->nickname, entry->identity, "DEFERRED"); return; @@ -2158,9 +3620,8 @@ add_an_entry_guard(routerinfo_t *chosen, int reset_status) /** If the use of entry guards is configured, choose more entry guards * until we have enough in the list. */ static void -pick_entry_guards(void) +pick_entry_guards(or_options_t *options) { - or_options_t *options = get_options(); int changed = 0; tor_assert(entry_guards); @@ -2182,7 +3643,8 @@ pick_entry_guards(void) static void entry_guard_free(entry_guard_t *e) { - tor_assert(e); + if (!e) + return; tor_free(e->chosen_by_version); tor_free(e); } @@ -2191,10 +3653,9 @@ entry_guard_free(entry_guard_t *e) * or which was selected by a version of Tor that's known to select * entry guards badly. */ static int -remove_obsolete_entry_guards(void) +remove_obsolete_entry_guards(time_t now) { int changed = 0, i; - time_t now = time(NULL); for (i = 0; i < smartlist_len(entry_guards); ++i) { entry_guard_t *entry = smartlist_get(entry_guards, i); @@ -2254,11 +3715,10 @@ remove_obsolete_entry_guards(void) * long that we don't think they'll come up again. Return 1 if we * removed any, or 0 if we did nothing. */ static int -remove_dead_entry_guards(void) +remove_dead_entry_guards(time_t now) { char dbuf[HEX_DIGEST_LEN+1]; char tbuf[ISO_TIME_LEN+1]; - time_t now = time(NULL); int i; int changed = 0; @@ -2293,19 +3753,17 @@ remove_dead_entry_guards(void) * think that things are unlisted. */ void -entry_guards_compute_status(void) +entry_guards_compute_status(or_options_t *options, time_t now) { - time_t now; int changed = 0; int severity = LOG_DEBUG; - or_options_t *options; digestmap_t *reasons; + if (! entry_guards) return; - options = get_options(); - - now = time(NULL); + if (options->EntryNodes) /* reshuffle the entry guard list if needed */ + entry_nodes_should_be_added(); reasons = digestmap_new(); SMARTLIST_FOREACH_BEGIN(entry_guards, entry_guard_t *, entry) @@ -2322,7 +3780,7 @@ entry_guards_compute_status(void) } SMARTLIST_FOREACH_END(entry); - if (remove_dead_entry_guards()) + if (remove_dead_entry_guards(now)) changed = 1; severity = changed ? LOG_DEBUG : LOG_INFO; @@ -2330,13 +3788,16 @@ entry_guards_compute_status(void) if (changed) { SMARTLIST_FOREACH_BEGIN(entry_guards, entry_guard_t *, entry) { const char *reason = digestmap_get(reasons, entry->identity); - log_info(LD_CIRC, "Summary: Entry '%s' is %s, %s%s%s, and %s.", + const char *live_msg = ""; + routerinfo_t *r = entry_is_live(entry, 0, 1, 0, &live_msg); + log_info(LD_CIRC, "Summary: Entry '%s' is %s, %s%s%s, and %s%s.", entry->nickname, entry->unreachable_since ? "unreachable" : "reachable", entry->bad_since ? "unusable" : "usable", reason ? ", ": "", reason ? reason : "", - entry_is_live(entry, 0, 1, 0) ? "live" : "not live"); + r ? "live" : "not live / ", + r ? "" : live_msg); } SMARTLIST_FOREACH_END(entry); log_info(LD_CIRC, " (%d/%d entry guards are usable/new)", num_live_entry_guards(), smartlist_len(entry_guards)); @@ -2355,7 +3816,7 @@ entry_guards_compute_status(void) * If <b>mark_relay_status</b>, also call router_set_status() on this * relay. * - * XXX022 change succeeded and mark_relay_status into 'int flags'. + * XXX023 change succeeded and mark_relay_status into 'int flags'. */ int entry_guard_register_connect_status(const char *digest, int succeeded, @@ -2407,6 +3868,7 @@ entry_guard_register_connect_status(const char *digest, int succeeded, "Removing from the list. %d/%d entry guards usable/new.", entry->nickname, buf, num_live_entry_guards()-1, smartlist_len(entry_guards)-1); + control_event_guard(entry->nickname, entry->identity, "DROPPED"); entry_guard_free(entry); smartlist_del_keeporder(entry_guards, idx); log_entry_guards(LOG_INFO); @@ -2443,7 +3905,8 @@ entry_guard_register_connect_status(const char *digest, int succeeded, if (e == entry) break; if (e->made_contact) { - routerinfo_t *r = entry_is_live(e, 0, 1, 1); + const char *msg; + routerinfo_t *r = entry_is_live(e, 0, 1, 1, &msg); if (r && e->unreachable_since) { refuse_conn = 1; e->can_retry = 1; @@ -2474,16 +3937,16 @@ static int should_add_entry_nodes = 0; void entry_nodes_should_be_added(void) { - log_info(LD_CIRC, "New EntryNodes config option detected. Will use."); + log_info(LD_CIRC, "EntryNodes config option set. Putting configured " + "relays at the front of the entry guard list."); should_add_entry_nodes = 1; } /** Add all nodes in EntryNodes that aren't currently guard nodes to the list * of guard nodes, at the front. */ static void -entry_guards_prepend_from_config(void) +entry_guards_prepend_from_config(or_options_t *options) { - or_options_t *options = get_options(); smartlist_t *entry_routers, *entry_fps; smartlist_t *old_entry_guards_on_list, *old_entry_guards_not_on_list; tor_assert(entry_guards); @@ -2498,7 +3961,7 @@ entry_guards_prepend_from_config(void) return; } - if (options->EntryNodes) { + { char *string = routerset_to_string(options->EntryNodes); log_info(LD_CIRC,"Adding configured EntryNodes '%s'.", string); tor_free(string); @@ -2511,13 +3974,14 @@ entry_guards_prepend_from_config(void) /* Split entry guards into those on the list and those not. */ - /* XXXX022 Now that we allow countries and IP ranges in EntryNodes, this is + /* XXXX023 Now that we allow countries and IP ranges in EntryNodes, this is * potentially an enormous list. For now, we disable such values for * EntryNodes in options_validate(); really, this wants a better solution. * Perhaps we should do this calculation once whenever the list of routers * changes or the entrynodes setting changes. */ - routerset_get_all_routers(entry_routers, options->EntryNodes, 0); + routerset_get_all_routers(entry_routers, options->EntryNodes, + options->ExcludeNodes, 0); SMARTLIST_FOREACH(entry_routers, routerinfo_t *, ri, smartlist_add(entry_fps,ri->cache_info.identity_digest)); SMARTLIST_FOREACH(entry_guards, entry_guard_t *, e, { @@ -2542,13 +4006,10 @@ entry_guards_prepend_from_config(void) SMARTLIST_FOREACH(entry_routers, routerinfo_t *, ri, { add_an_entry_guard(ri, 0); }); - /* Finally, the remaining EntryNodes, unless we're strict */ - if (options->StrictEntryNodes) { - SMARTLIST_FOREACH(old_entry_guards_not_on_list, entry_guard_t *, e, - entry_guard_free(e)); - } else { - smartlist_add_all(entry_guards, old_entry_guards_not_on_list); - } + /* Finally, free the remaining previously configured guards that are not in + * EntryNodes. */ + SMARTLIST_FOREACH(old_entry_guards_not_on_list, entry_guard_t *, e, + entry_guard_free(e)); smartlist_free(entry_routers); smartlist_free(entry_fps); @@ -2557,16 +4018,18 @@ entry_guards_prepend_from_config(void) entry_guards_changed(); } -/** Return 1 if we're fine adding arbitrary routers out of the - * directory to our entry guard list. Else return 0. */ +/** Return 0 if we're fine adding arbitrary routers out of the + * directory to our entry guard list, or return 1 if we have a + * list already and we must stick to it. + */ int -entry_list_can_grow(or_options_t *options) +entry_list_is_constrained(or_options_t *options) { - if (options->StrictEntryNodes) - return 0; + if (options->EntryNodes) + return 1; if (options->UseBridges) - return 0; - return 1; + return 1; + return 0; } /** Pick a live (up and listed) entry guard from entry_guards. If @@ -2584,10 +4047,9 @@ choose_random_entry(cpath_build_state_t *state) routerinfo_t *r = NULL; int need_uptime = state ? state->need_uptime : 0; int need_capacity = state ? state->need_capacity : 0; - int consider_exit_family = 0; + int preferred_min, consider_exit_family = 0; if (chosen_exit) { - smartlist_add(exit_family, chosen_exit); routerlist_add_family(exit_family, chosen_exit); consider_exit_family = 1; } @@ -2596,38 +4058,66 @@ choose_random_entry(cpath_build_state_t *state) entry_guards = smartlist_create(); if (should_add_entry_nodes) - entry_guards_prepend_from_config(); + entry_guards_prepend_from_config(options); - if (entry_list_can_grow(options) && - (! entry_guards || - smartlist_len(entry_guards) < options->NumEntryGuards)) - pick_entry_guards(); + if (!entry_list_is_constrained(options) && + smartlist_len(entry_guards) < options->NumEntryGuards) + pick_entry_guards(options); retry: smartlist_clear(live_entry_guards); SMARTLIST_FOREACH(entry_guards, entry_guard_t *, entry, { - r = entry_is_live(entry, need_uptime, need_capacity, 0); - if (r && (!consider_exit_family || !smartlist_isin(exit_family, r))) { - smartlist_add(live_entry_guards, r); - if (!entry->made_contact) { - /* Always start with the first not-yet-contacted entry - * guard. Otherwise we might add several new ones, pick - * the second new one, and now we've expanded our entry - * guard list without needing to. */ - goto choose_and_finish; + const char *msg; + r = entry_is_live(entry, need_uptime, need_capacity, 0, &msg); + if (!r) + continue; /* down, no point */ + if (r == chosen_exit) + continue; /* don't pick the same node for entry and exit */ + if (consider_exit_family && smartlist_isin(exit_family, r)) + continue; /* avoid relays that are family members of our exit */ +#if 0 /* since EntryNodes is always strict now, this clause is moot */ + if (options->EntryNodes && + !routerset_contains_router(options->EntryNodes, r)) { + /* We've come to the end of our preferred entry nodes. */ + if (smartlist_len(live_entry_guards)) + goto choose_and_finish; /* only choose from the ones we like */ + if (options->StrictNodes) { + /* in theory this case should never happen, since + * entry_guards_prepend_from_config() drops unwanted relays */ + tor_fragile_assert(); + } else { + log_info(LD_CIRC, + "No relays from EntryNodes available. Using others."); } - if (smartlist_len(live_entry_guards) >= options->NumEntryGuards) - break; /* we have enough */ } +#endif + smartlist_add(live_entry_guards, r); + if (!entry->made_contact) { + /* Always start with the first not-yet-contacted entry + * guard. Otherwise we might add several new ones, pick + * the second new one, and now we've expanded our entry + * guard list without needing to. */ + goto choose_and_finish; + } + if (smartlist_len(live_entry_guards) >= options->NumEntryGuards) + break; /* we have enough */ }); - /* Try to have at least 2 choices available. This way we don't - * get stuck with a single live-but-crummy entry and just keep - * using him. - * (We might get 2 live-but-crummy entry guards, but so be it.) */ - if (smartlist_len(live_entry_guards) < 2) { - if (entry_list_can_grow(options)) { + if (entry_list_is_constrained(options)) { + /* If we prefer the entry nodes we've got, and we have at least + * one choice, that's great. Use it. */ + preferred_min = 1; + } else { + /* Try to have at least 2 choices available. This way we don't + * get stuck with a single live-but-crummy entry and just keep + * using him. + * (We might get 2 live-but-crummy entry guards, but so be it.) */ + preferred_min = 2; + } + + if (smartlist_len(live_entry_guards) < preferred_min) { + if (!entry_list_is_constrained(options)) { /* still no? try adding a new entry then */ /* XXX if guard doesn't imply fast and stable, then we need * to tell add_an_entry_guard below what we want, or it might @@ -2652,26 +4142,31 @@ choose_random_entry(cpath_build_state_t *state) need_capacity = 0; goto retry; } - if (!r && !entry_list_can_grow(options) && consider_exit_family) { - /* still no? if we're using bridges or have strictentrynodes - * set, and our chosen exit is in the same family as all our - * bridges/entry guards, then be flexible about families. */ +#if 0 + /* Removing this retry logic: if we only allow one exit, and it is in the + same family as all our entries, then we are just plain not going to win + here. */ + if (!r && entry_list_is_constrained(options) && consider_exit_family) { + /* still no? if we're using bridges, + * and our chosen exit is in the same family as all our + * bridges, then be flexible about families. */ consider_exit_family = 0; goto retry; } +#endif /* live_entry_guards may be empty below. Oh well, we tried. */ } choose_and_finish: - if (entry_list_can_grow(options)) { + if (entry_list_is_constrained(options)) { + /* We need to weight by bandwidth, because our bridges or entryguards + * were not already selected proportional to their bandwidth. */ + r = routerlist_sl_choose_by_bandwidth(live_entry_guards, WEIGHT_FOR_GUARD); + } else { /* We choose uniformly at random here, because choose_good_entry_server() * already weights its choices by bandwidth, so we don't want to * *double*-weight our guard selection. */ r = smartlist_choose(live_entry_guards); - } else { - /* We need to weight by bandwidth, because our bridges or entryguards - * were not already selected proportional to their bandwidth. */ - r = routerlist_sl_choose_by_bandwidth(live_entry_guards, WEIGHT_FOR_GUARD); } smartlist_free(live_entry_guards); smartlist_free(exit_family); @@ -2803,9 +4298,9 @@ entry_guards_parse_state(or_state_t *state, int set, char **msg) } entry_guards = new_entry_guards; entry_guards_dirty = 0; - /* XXX022 hand new_entry_guards to this func, and move it up a + /* XXX023 hand new_entry_guards to this func, and move it up a * few lines, so we don't have to re-dirty it */ - if (remove_obsolete_entry_guards()) + if (remove_obsolete_entry_guards(now)) entry_guards_dirty = 1; } digestmap_free(added_by, _tor_free); @@ -2904,9 +4399,11 @@ entry_guards_update_state(or_state_t *state) * */ int getinfo_helper_entry_guards(control_connection_t *conn, - const char *question, char **answer) + const char *question, char **answer, + const char **errmsg) { - int use_long_names = conn->use_long_names; + (void) conn; + (void) errmsg; if (!strcmp(question,"entry-guards") || !strcmp(question,"helper-nodes")) { @@ -2915,12 +4412,13 @@ getinfo_helper_entry_guards(control_connection_t *conn, char nbuf[MAX_VERBOSE_NICKNAME_LEN+1]; if (!entry_guards) entry_guards = smartlist_create(); - SMARTLIST_FOREACH(entry_guards, entry_guard_t *, e, - { + SMARTLIST_FOREACH_BEGIN(entry_guards, entry_guard_t *, e) { size_t len = MAX_VERBOSE_NICKNAME_LEN+ISO_TIME_LEN+32; char *c = tor_malloc(len); const char *status = NULL; time_t when = 0; + routerinfo_t *ri; + if (!e->made_contact) { status = "never-connected"; } else if (e->bad_since) { @@ -2929,19 +4427,17 @@ getinfo_helper_entry_guards(control_connection_t *conn, } else { status = "up"; } - if (use_long_names) { - routerinfo_t *ri = router_get_by_digest(e->identity); - if (ri) { - router_get_verbose_nickname(nbuf, ri); - } else { - nbuf[0] = '$'; - base16_encode(nbuf+1, sizeof(nbuf)-1, e->identity, DIGEST_LEN); - /* e->nickname field is not very reliable if we don't know about - * this router any longer; don't include it. */ - } + + ri = router_get_by_digest(e->identity); + if (ri) { + router_get_verbose_nickname(nbuf, ri); } else { - base16_encode(nbuf, sizeof(nbuf), e->identity, DIGEST_LEN); + nbuf[0] = '$'; + base16_encode(nbuf+1, sizeof(nbuf)-1, e->identity, DIGEST_LEN); + /* e->nickname field is not very reliable if we don't know about + * this router any longer; don't include it. */ } + if (when) { format_iso_time(tbuf, when); tor_snprintf(c, len, "%s %s %s\n", nbuf, status, tbuf); @@ -2949,7 +4445,7 @@ getinfo_helper_entry_guards(control_connection_t *conn, tor_snprintf(c, len, "%s %s\n", nbuf, status); } smartlist_add(sl, c); - }); + } SMARTLIST_FOREACH_END(e); *answer = smartlist_join_strings(sl, "", 0, NULL); SMARTLIST_FOREACH(sl, char *, c, tor_free(c)); smartlist_free(sl); @@ -2990,29 +4486,56 @@ clear_bridge_list(void) * (either by comparing keys if possible, else by comparing addr/port). * Else return NULL. */ static bridge_info_t * -routerinfo_get_configured_bridge(routerinfo_t *ri) +get_configured_bridge_by_addr_port_digest(tor_addr_t *addr, uint16_t port, + const char *digest) { if (!bridge_list) return NULL; SMARTLIST_FOREACH_BEGIN(bridge_list, bridge_info_t *, bridge) { if (tor_digest_is_zero(bridge->identity) && - tor_addr_eq_ipv4h(&bridge->addr, ri->addr) && - bridge->port == ri->or_port) + !tor_addr_compare(&bridge->addr, addr, CMP_EXACT) && + bridge->port == port) return bridge; - if (!memcmp(bridge->identity, ri->cache_info.identity_digest, - DIGEST_LEN)) + if (!memcmp(bridge->identity, digest, DIGEST_LEN)) return bridge; } SMARTLIST_FOREACH_END(bridge); return NULL; } +/** Wrapper around get_configured_bridge_by_addr_port_digest() to look + * it up via router descriptor <b>ri</b>. */ +static bridge_info_t * +get_configured_bridge_by_routerinfo(routerinfo_t *ri) +{ + tor_addr_t addr; + tor_addr_from_ipv4h(&addr, ri->addr); + return get_configured_bridge_by_addr_port_digest(&addr, + ri->or_port, ri->cache_info.identity_digest); +} + /** Return 1 if <b>ri</b> is one of our known bridges, else 0. */ int routerinfo_is_a_configured_bridge(routerinfo_t *ri) { - return routerinfo_get_configured_bridge(ri) ? 1 : 0; + return get_configured_bridge_by_routerinfo(ri) ? 1 : 0; +} + +/** We made a connection to a router at <b>addr</b>:<b>port</b> + * without knowing its digest. Its digest turned out to be <b>digest</b>. + * If it was a bridge, and we still don't know its digest, record it. + */ +void +learned_router_identity(tor_addr_t *addr, uint16_t port, const char *digest) +{ + bridge_info_t *bridge = + get_configured_bridge_by_addr_port_digest(addr, port, digest); + if (bridge && tor_digest_is_zero(bridge->identity)) { + memcpy(bridge->identity, digest, DIGEST_LEN); + log_notice(LD_DIR, "Learned fingerprint %s for bridge %s:%d", + hex_str(digest, DIGEST_LEN), fmt_addr(addr), port); + } } /** Remember a new bridge at <b>addr</b>:<b>port</b>. If <b>digest</b> @@ -3031,6 +4554,24 @@ bridge_add_from_config(const tor_addr_t *addr, uint16_t port, char *digest) smartlist_add(bridge_list, b); } +/** Return true iff <b>routerset</b> contains the bridge <b>bridge</b>. */ +static int +routerset_contains_bridge(const routerset_t *routerset, + const bridge_info_t *bridge) +{ + int result; + extend_info_t *extinfo; + tor_assert(bridge); + if (!routerset) + return 0; + + extinfo = extend_info_alloc( + NULL, bridge->identity, NULL, &bridge->addr, bridge->port); + result = routerset_contains_extendinfo(routerset, extinfo); + extend_info_free(extinfo); + return result; +} + /** If <b>digest</b> is one of our known bridges, return it. */ static bridge_info_t * find_bridge_by_digest(const char *digest) @@ -3043,12 +4584,12 @@ find_bridge_by_digest(const char *digest) return NULL; } -/** We need to ask <b>bridge</b> for its server descriptor. <b>address</b> - * is a helpful string describing this bridge. */ +/** We need to ask <b>bridge</b> for its server descriptor. */ static void launch_direct_bridge_descriptor_fetch(bridge_info_t *bridge) { char *address; + or_options_t *options = get_options(); if (connection_get_by_type_addr_port_purpose( CONN_TYPE_DIR, &bridge->addr, bridge->port, @@ -3056,6 +4597,13 @@ launch_direct_bridge_descriptor_fetch(bridge_info_t *bridge) return; /* it's already on the way */ address = tor_dup_addr(&bridge->addr); + if (routerset_contains_bridge(options->ExcludeNodes, bridge)) { + download_status_mark_impossible(&bridge->fetch_status); + log_warn(LD_APP, "Not using bridge at %s: it is in ExcludeNodes.", + safe_str_client(fmt_addr(&bridge->addr))); + return; + } + directory_initiate_command(address, &bridge->addr, bridge->port, 0, 0, /* does not matter */ @@ -3082,9 +4630,8 @@ retry_bridge_descriptor_fetch_directly(const char *digest) * descriptor, fetch a new copy of its descriptor -- either directly * from the bridge or via a bridge authority. */ void -fetch_bridge_descriptors(time_t now) +fetch_bridge_descriptors(or_options_t *options, time_t now) { - or_options_t *options = get_options(); int num_bridge_auths = get_n_authorities(BRIDGE_AUTHORITY); int ask_bridge_directly; int can_use_bridge_authority; @@ -3097,6 +4644,12 @@ fetch_bridge_descriptors(time_t now) if (!download_status_is_ready(&bridge->fetch_status, now, IMPOSSIBLE_TO_DOWNLOAD)) continue; /* don't bother, no need to retry yet */ + if (routerset_contains_bridge(options->ExcludeNodes, bridge)) { + download_status_mark_impossible(&bridge->fetch_status); + log_warn(LD_APP, "Not using bridge at %s: it is in ExcludeNodes.", + safe_str_client(fmt_addr(&bridge->addr))); + continue; + } /* schedule another fetch as if this one will fail, in case it does */ download_status_failed(&bridge->fetch_status, 0); @@ -3143,6 +4696,29 @@ fetch_bridge_descriptors(time_t now) SMARTLIST_FOREACH_END(bridge); } +/** If our <b>bridge</b> is configured to be a different address than + * the bridge gives in its routerinfo <b>ri</b>, rewrite the routerinfo + * we received to use the address we meant to use. Now we handle + * multihomed bridges better. + */ +static void +rewrite_routerinfo_address_for_bridge(bridge_info_t *bridge, routerinfo_t *ri) +{ + tor_addr_t addr; + tor_addr_from_ipv4h(&addr, ri->addr); + + if (!tor_addr_compare(&bridge->addr, &addr, CMP_EXACT) && + bridge->port == ri->or_port) + return; /* they match, so no need to do anything */ + + ri->addr = tor_addr_to_ipv4h(&bridge->addr); + tor_free(ri->address); + ri->address = tor_dup_ip(ri->addr); + ri->or_port = bridge->port; + log_info(LD_DIR, "Adjusted bridge '%s' to match configured address %s:%d.", + ri->nickname, ri->address, ri->or_port); +} + /** We just learned a descriptor for a bridge. See if that * digest is in our entry guard list, and add it if not. */ void @@ -3152,7 +4728,7 @@ learned_bridge_descriptor(routerinfo_t *ri, int from_cache) tor_assert(ri->purpose == ROUTER_PURPOSE_BRIDGE); if (get_options()->UseBridges) { int first = !any_bridge_descriptors_known(); - bridge_info_t *bridge = routerinfo_get_configured_bridge(ri); + bridge_info_t *bridge = get_configured_bridge_by_routerinfo(ri); time_t now = time(NULL); ri->is_running = 1; @@ -3161,6 +4737,8 @@ learned_bridge_descriptor(routerinfo_t *ri, int from_cache) if (!from_cache) download_status_reset(&bridge->fetch_status); + rewrite_routerinfo_address_for_bridge(bridge, ri); + add_an_entry_guard(ri, 1); log_notice(LD_DIR, "new bridge descriptor '%s' (%s)", ri->nickname, from_cache ? "cached" : "fresh"); @@ -3209,26 +4787,38 @@ any_pending_bridge_descriptor_fetches(void) return 0; } -/** Return 1 if we have at least one descriptor for a bridge and - * all descriptors we know are down. Else return 0. If <b>act</b> is - * 1, then mark the down bridges up; else just observe and report. */ +/** Return 1 if we have at least one descriptor for an entry guard + * (bridge or member of EntryNodes) and all descriptors we know are + * down. Else return 0. If <b>act</b> is 1, then mark the down guards + * up; else just observe and report. */ static int -bridges_retry_helper(int act) +entries_retry_helper(or_options_t *options, int act) { routerinfo_t *ri; int any_known = 0; int any_running = 0; + int purpose = options->UseBridges ? + ROUTER_PURPOSE_BRIDGE : ROUTER_PURPOSE_GENERAL; if (!entry_guards) entry_guards = smartlist_create(); SMARTLIST_FOREACH(entry_guards, entry_guard_t *, e, { ri = router_get_by_digest(e->identity); - if (ri && ri->purpose == ROUTER_PURPOSE_BRIDGE) { + if (ri && ri->purpose == purpose) { any_known = 1; if (ri->is_running) - any_running = 1; /* some bridge is both known and running */ - else if (act) { /* mark it for retry */ - ri->is_running = 1; + any_running = 1; /* some entry is both known and running */ + else if (act) { + /* Mark all current connections to this OR as unhealthy, since + * otherwise there could be one that started 30 seconds + * ago, and in 30 seconds it will time out, causing us to mark + * the node down and undermine the retry attempt. We mark even + * the established conns, since if the network just came back + * we'll want to attach circuits to fresh conns. */ + connection_or_set_bad_connections(ri->cache_info.identity_digest, 1); + + /* mark this entry node for retry */ + router_set_status(ri->cache_info.identity_digest, 1); e->can_retry = 1; e->bad_since = 0; } @@ -3239,19 +4829,21 @@ bridges_retry_helper(int act) return any_known && !any_running; } -/** Do we know any descriptors for our bridges, and are they all - * down? */ +/** Do we know any descriptors for our bridges / entrynodes, and are + * all the ones we have descriptors for down? */ int -bridges_known_but_down(void) +entries_known_but_down(or_options_t *options) { - return bridges_retry_helper(0); + tor_assert(entry_list_is_constrained(options)); + return entries_retry_helper(options, 0); } -/** Mark all down known bridges up. */ +/** Mark all down known bridges / entrynodes up. */ void -bridges_retry_all(void) +entries_retry_all(or_options_t *options) { - bridges_retry_helper(1); + tor_assert(entry_list_is_constrained(options)); + entries_retry_helper(options, 1); } /** Release all storage held by the list of entry guards and related diff --git a/src/or/circuitbuild.h b/src/or/circuitbuild.h new file mode 100644 index 0000000000..af24931878 --- /dev/null +++ b/src/or/circuitbuild.h @@ -0,0 +1,127 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file circuitbuild.h + * \brief Header file for circuitbuild.c. + **/ + +#ifndef _TOR_CIRCUITBUILD_H +#define _TOR_CIRCUITBUILD_H + +char *circuit_list_path(origin_circuit_t *circ, int verbose); +char *circuit_list_path_for_controller(origin_circuit_t *circ); +void circuit_log_path(int severity, unsigned int domain, + origin_circuit_t *circ); +void circuit_rep_hist_note_result(origin_circuit_t *circ); +origin_circuit_t *origin_circuit_init(uint8_t purpose, int flags); +origin_circuit_t *circuit_establish_circuit(uint8_t purpose, + extend_info_t *exit, + int flags); +int circuit_handle_first_hop(origin_circuit_t *circ); +void circuit_n_conn_done(or_connection_t *or_conn, int status); +int inform_testing_reachability(void); +int circuit_timeout_want_to_count_circ(origin_circuit_t *circ); +int circuit_send_next_onion_skin(origin_circuit_t *circ); +void circuit_note_clock_jumped(int seconds_elapsed); +int circuit_extend(cell_t *cell, circuit_t *circ); +int circuit_init_cpath_crypto(crypt_path_t *cpath, const char *key_data, + int reverse); +int circuit_finish_handshake(origin_circuit_t *circ, uint8_t cell_type, + const uint8_t *reply); +int circuit_truncated(origin_circuit_t *circ, crypt_path_t *layer); +int onionskin_answer(or_circuit_t *circ, uint8_t cell_type, + const char *payload, const char *keys); +int circuit_all_predicted_ports_handled(time_t now, int *need_uptime, + int *need_capacity); + +int circuit_append_new_exit(origin_circuit_t *circ, extend_info_t *info); +int circuit_extend_to_new_exit(origin_circuit_t *circ, extend_info_t *info); +void onion_append_to_cpath(crypt_path_t **head_ptr, crypt_path_t *new_hop); +extend_info_t *extend_info_alloc(const char *nickname, const char *digest, + crypto_pk_env_t *onion_key, + const tor_addr_t *addr, uint16_t port); +extend_info_t *extend_info_from_router(routerinfo_t *r); +extend_info_t *extend_info_dup(extend_info_t *info); +void extend_info_free(extend_info_t *info); +routerinfo_t *build_state_get_exit_router(cpath_build_state_t *state); +const char *build_state_get_exit_nickname(cpath_build_state_t *state); + +void entry_guards_compute_status(or_options_t *options, time_t now); +int entry_guard_register_connect_status(const char *digest, int succeeded, + int mark_relay_status, time_t now); +void entry_nodes_should_be_added(void); +int entry_list_is_constrained(or_options_t *options); +routerinfo_t *choose_random_entry(cpath_build_state_t *state); +int entry_guards_parse_state(or_state_t *state, int set, char **msg); +void entry_guards_update_state(or_state_t *state); +int getinfo_helper_entry_guards(control_connection_t *conn, + const char *question, char **answer, + const char **errmsg); + +void clear_bridge_list(void); +int routerinfo_is_a_configured_bridge(routerinfo_t *ri); +void +learned_router_identity(tor_addr_t *addr, uint16_t port, const char *digest); +void bridge_add_from_config(const tor_addr_t *addr, uint16_t port, + char *digest); +void retry_bridge_descriptor_fetch_directly(const char *digest); +void fetch_bridge_descriptors(or_options_t *options, time_t now); +void learned_bridge_descriptor(routerinfo_t *ri, int from_cache); +int any_bridge_descriptors_known(void); +int any_pending_bridge_descriptor_fetches(void); +int entries_known_but_down(or_options_t *options); +void entries_retry_all(or_options_t *options); + +void entry_guards_free_all(void); + +extern circuit_build_times_t circ_times; +int circuit_build_times_enough_to_compute(circuit_build_times_t *cbt); +void circuit_build_times_update_state(circuit_build_times_t *cbt, + or_state_t *state); +int circuit_build_times_parse_state(circuit_build_times_t *cbt, + or_state_t *state); +void circuit_build_times_count_timeout(circuit_build_times_t *cbt, + int did_onehop); +int circuit_build_times_count_close(circuit_build_times_t *cbt, + int did_onehop, time_t start_time); +void circuit_build_times_set_timeout(circuit_build_times_t *cbt); +int circuit_build_times_add_time(circuit_build_times_t *cbt, + build_time_t time); +int circuit_build_times_needs_circuits(circuit_build_times_t *cbt); + +int circuit_build_times_needs_circuits_now(circuit_build_times_t *cbt); +void circuit_build_times_init(circuit_build_times_t *cbt); +void circuit_build_times_new_consensus_params(circuit_build_times_t *cbt, + networkstatus_t *ns); +double circuit_build_times_timeout_rate(const circuit_build_times_t *cbt); +double circuit_build_times_close_rate(const circuit_build_times_t *cbt); + +#ifdef CIRCUIT_PRIVATE +double circuit_build_times_calculate_timeout(circuit_build_times_t *cbt, + double quantile); +build_time_t circuit_build_times_generate_sample(circuit_build_times_t *cbt, + double q_lo, double q_hi); +void circuit_build_times_initial_alpha(circuit_build_times_t *cbt, + double quantile, double time_ms); +int circuit_build_times_update_alpha(circuit_build_times_t *cbt); +double circuit_build_times_cdf(circuit_build_times_t *cbt, double x); +void circuitbuild_running_unit_tests(void); +void circuit_build_times_reset(circuit_build_times_t *cbt); + +/* Network liveness functions */ +int circuit_build_times_network_check_changed(circuit_build_times_t *cbt); +#endif + +/* Network liveness functions */ +void circuit_build_times_network_is_live(circuit_build_times_t *cbt); +int circuit_build_times_network_check_live(circuit_build_times_t *cbt); +void circuit_build_times_network_circ_success(circuit_build_times_t *cbt); + +int circuit_build_times_get_bw_scale(networkstatus_t *ns); + +#endif + diff --git a/src/or/circuitlist.c b/src/or/circuitlist.c index 524856875d..33dc8f09ad 100644 --- a/src/or/circuitlist.c +++ b/src/or/circuitlist.c @@ -10,6 +10,21 @@ **/ #include "or.h" +#include "circuitbuild.h" +#include "circuitlist.h" +#include "circuituse.h" +#include "connection.h" +#include "config.h" +#include "connection_edge.h" +#include "connection_or.h" +#include "control.h" +#include "networkstatus.h" +#include "onion.h" +#include "relay.h" +#include "rendclient.h" +#include "rendcommon.h" +#include "rephist.h" +#include "routerlist.h" #include "ht.h" /********* START VARIABLES **********/ @@ -352,6 +367,8 @@ circuit_purpose_to_controller_string(uint8_t purpose) case CIRCUIT_PURPOSE_TESTING: return "TESTING"; + case CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT: + return "MEASURE_TIMEOUT"; case CIRCUIT_PURPOSE_CONTROLLER: return "CONTROLLER"; @@ -361,13 +378,71 @@ circuit_purpose_to_controller_string(uint8_t purpose) } } +/** Return a human-readable string for the circuit purpose <b>purpose</b>. */ +const char * +circuit_purpose_to_string(uint8_t purpose) +{ + static char buf[32]; + + switch (purpose) + { + case CIRCUIT_PURPOSE_OR: + return "Circuit at relay"; + case CIRCUIT_PURPOSE_INTRO_POINT: + return "Acting as intro point"; + case CIRCUIT_PURPOSE_REND_POINT_WAITING: + return "Acting as rendevous (pending)"; + case CIRCUIT_PURPOSE_REND_ESTABLISHED: + return "Acting as rendevous (established)"; + case CIRCUIT_PURPOSE_C_GENERAL: + return "General-purpose client"; + case CIRCUIT_PURPOSE_C_INTRODUCING: + return "Hidden service client: Connecting to intro point"; + case CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT: + return "Hidden service client: Waiting for ack from intro point"; + case CIRCUIT_PURPOSE_C_INTRODUCE_ACKED: + return "Hidden service client: Received ack from intro point"; + case CIRCUIT_PURPOSE_C_ESTABLISH_REND: + return "Hidden service client: Establishing rendezvous point"; + case CIRCUIT_PURPOSE_C_REND_READY: + return "Hidden service client: Pending rendezvous point"; + case CIRCUIT_PURPOSE_C_REND_READY_INTRO_ACKED: + return "Hidden service client: Pending rendezvous point (ack received)"; + case CIRCUIT_PURPOSE_C_REND_JOINED: + return "Hidden service client: Active rendezvous point"; + case CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT: + return "Measuring circuit timeout"; + + case CIRCUIT_PURPOSE_S_ESTABLISH_INTRO: + return "Hidden service: Establishing introduction point"; + case CIRCUIT_PURPOSE_S_INTRO: + return "Hidden service: Introduction point"; + case CIRCUIT_PURPOSE_S_CONNECT_REND: + return "Hidden service: Connecting to rendezvous point"; + case CIRCUIT_PURPOSE_S_REND_JOINED: + return "Hidden service: Active rendezvous point"; + + case CIRCUIT_PURPOSE_TESTING: + return "Testing circuit"; + + case CIRCUIT_PURPOSE_CONTROLLER: + return "Circuit made by controller"; + + default: + tor_snprintf(buf, sizeof(buf), "UNKNOWN_%d", (int)purpose); + return buf; + } +} + /** Pick a reasonable package_window to start out for our circuits. * Originally this was hard-coded at 1000, but now the consensus votes * on the answer. See proposal 168. */ int32_t circuit_initial_package_window(void) { - int32_t num = networkstatus_get_param(NULL, "circwindow", CIRCWINDOW_START); + int32_t num = networkstatus_get_param(NULL, "circwindow", CIRCWINDOW_START, + CIRCWINDOW_START_MIN, + CIRCWINDOW_START_MAX); /* If the consensus tells us a negative number, we'd assert. */ if (num < 0) num = CIRCWINDOW_START; @@ -379,11 +454,17 @@ circuit_initial_package_window(void) static void init_circuit_base(circuit_t *circ) { - circ->timestamp_created = time(NULL); + tor_gettimeofday(&circ->timestamp_created); circ->package_window = circuit_initial_package_window(); circ->deliver_window = CIRCWINDOW_START; + /* Initialize the cell_ewma_t structure */ + circ->n_cell_ewma.last_adjusted_tick = cell_ewma_get_tick(); + circ->n_cell_ewma.cell_count = 0.0; + circ->n_cell_ewma.heap_index = -1; + circ->n_cell_ewma.is_for_p_conn = 0; + circuit_add(circ); } @@ -408,6 +489,8 @@ origin_circuit_new(void) init_circuit_base(TO_CIRCUIT(circ)); + circ_times.last_circ_at = approx_time(); + return circ; } @@ -429,6 +512,16 @@ or_circuit_new(circid_t p_circ_id, or_connection_t *p_conn) init_circuit_base(TO_CIRCUIT(circ)); + /* Initialize the cell_ewma_t structure */ + + /* Initialize the cell counts to 0 */ + circ->p_cell_ewma.cell_count = 0.0; + circ->p_cell_ewma.last_adjusted_tick = cell_ewma_get_tick(); + circ->p_cell_ewma.is_for_p_conn = 1; + + /* It's not in any heap yet. */ + circ->p_cell_ewma.heap_index = -1; + return circ; } @@ -439,39 +532,37 @@ circuit_free(circuit_t *circ) { void *mem; size_t memlen; - tor_assert(circ); + if (!circ) + return; + if (CIRCUIT_IS_ORIGIN(circ)) { origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ); mem = ocirc; memlen = sizeof(origin_circuit_t); tor_assert(circ->magic == ORIGIN_CIRCUIT_MAGIC); if (ocirc->build_state) { - if (ocirc->build_state->chosen_exit) extend_info_free(ocirc->build_state->chosen_exit); - if (ocirc->build_state->pending_final_cpath) circuit_free_cpath_node(ocirc->build_state->pending_final_cpath); } tor_free(ocirc->build_state); circuit_free_cpath(ocirc->cpath); - if (ocirc->intro_key) - crypto_free_pk_env(ocirc->intro_key); - if (ocirc->rend_data) - rend_data_free(ocirc->rend_data); + + crypto_free_pk_env(ocirc->intro_key); + rend_data_free(ocirc->rend_data); } else { or_circuit_t *ocirc = TO_OR_CIRCUIT(circ); + /* Remember cell statistics for this circuit before deallocating. */ + if (get_options()->CellStatistics) + rep_hist_buffer_stats_add_circ(circ, time(NULL)); mem = ocirc; memlen = sizeof(or_circuit_t); tor_assert(circ->magic == OR_CIRCUIT_MAGIC); - if (ocirc->p_crypto) - crypto_free_cipher_env(ocirc->p_crypto); - if (ocirc->p_digest) - crypto_free_digest_env(ocirc->p_digest); - if (ocirc->n_crypto) - crypto_free_cipher_env(ocirc->n_crypto); - if (ocirc->n_digest) - crypto_free_digest_env(ocirc->n_digest); + crypto_free_cipher_env(ocirc->p_crypto); + crypto_free_digest_env(ocirc->p_digest); + crypto_free_cipher_env(ocirc->n_crypto); + crypto_free_digest_env(ocirc->n_digest); if (ocirc->rend_splice) { or_circuit_t *other = ocirc->rend_splice; @@ -487,8 +578,7 @@ circuit_free(circuit_t *circ) cell_queue_clear(ô->p_conn_cells); } - if (circ->n_hop) - extend_info_free(circ->n_hop); + extend_info_free(circ->n_hop); tor_free(circ->n_conn_onionskin); /* Remove from map. */ @@ -498,7 +588,7 @@ circuit_free(circuit_t *circ) * "active" checks will be violated. */ cell_queue_clear(&circ->n_conn_cells); - memset(circ, 0xAA, memlen); /* poison memory */ + memset(mem, 0xAA, memlen); /* poison memory */ tor_free(mem); } @@ -541,10 +631,10 @@ circuit_free_all(void) circuit_free(global_circuitlist); global_circuitlist = next; } - if (circuits_pending_or_conns) { - smartlist_free(circuits_pending_or_conns); - circuits_pending_or_conns = NULL; - } + + smartlist_free(circuits_pending_or_conns); + circuits_pending_or_conns = NULL; + HT_CLEAR(orconn_circid_map, &orconn_circid_circuit_map); } @@ -552,18 +642,15 @@ circuit_free_all(void) static void circuit_free_cpath_node(crypt_path_t *victim) { - if (victim->f_crypto) - crypto_free_cipher_env(victim->f_crypto); - if (victim->b_crypto) - crypto_free_cipher_env(victim->b_crypto); - if (victim->f_digest) - crypto_free_digest_env(victim->f_digest); - if (victim->b_digest) - crypto_free_digest_env(victim->b_digest); - if (victim->dh_handshake_state) - crypto_dh_free(victim->dh_handshake_state); - if (victim->extend_info) - extend_info_free(victim->extend_info); + if (!victim) + return; + + crypto_free_cipher_env(victim->f_crypto); + crypto_free_cipher_env(victim->b_crypto); + crypto_free_digest_env(victim->f_digest); + crypto_free_digest_env(victim->b_digest); + crypto_dh_free(victim->dh_handshake_state); + extend_info_free(victim->extend_info); memset(victim, 0xBB, sizeof(crypt_path_t)); /* poison memory */ tor_free(victim); @@ -577,9 +664,10 @@ circuit_dump_details(int severity, circuit_t *circ, int conn_array_index, const char *type, int this_circid, int other_circid) { log(severity, LD_CIRC, "Conn %d has %s circuit: circID %d (other side %d), " - "state %d (%s), born %d:", + "state %d (%s), born %ld:", conn_array_index, type, this_circid, other_circid, circ->state, - circuit_state_to_string(circ->state), (int)circ->timestamp_created); + circuit_state_to_string(circ->state), + (long)circ->timestamp_created.tv_sec); if (CIRCUIT_IS_ORIGIN(circ)) { /* circ starts at this node */ circuit_log_path(severity, LD_CIRC, TO_ORIGIN_CIRCUIT(circ)); } @@ -891,6 +979,11 @@ circuit_find_to_cannibalize(uint8_t purpose, extend_info_t *info, int need_uptime = (flags & CIRCLAUNCH_NEED_UPTIME) != 0; int need_capacity = (flags & CIRCLAUNCH_NEED_CAPACITY) != 0; int internal = (flags & CIRCLAUNCH_IS_INTERNAL) != 0; + or_options_t *options = get_options(); + + /* Make sure we're not trying to create a onehop circ by + * cannibalization. */ + tor_assert(!(flags & CIRCLAUNCH_ONEHOP_TUNNEL)); log_debug(LD_CIRC, "Hunting for a circ to cannibalize: purpose %d, uptime %d, " @@ -907,7 +1000,8 @@ circuit_find_to_cannibalize(uint8_t purpose, extend_info_t *info, if ((!need_uptime || circ->build_state->need_uptime) && (!need_capacity || circ->build_state->need_capacity) && (internal == circ->build_state->is_internal) && - circ->remaining_relay_early_cells) { + circ->remaining_relay_early_cells && + !circ->build_state->onehop_tunnel) { if (info) { /* need to make sure we don't duplicate hops */ crypt_path_t *hop = circ->cpath; @@ -924,6 +1018,19 @@ circuit_find_to_cannibalize(uint8_t purpose, extend_info_t *info, hop=hop->next; } while (hop!=circ->cpath); } + if (options->ExcludeNodes) { + /* Make sure no existing nodes in the circuit are excluded for + * general use. (This may be possible if StrictNodes is 0, and we + * thought we needed to use an otherwise excluded node for, say, a + * directory operation.) */ + crypt_path_t *hop = circ->cpath; + do { + if (routerset_contains_extendinfo(options->ExcludeNodes, + hop->extend_info)) + goto next; + hop = hop->next; + } while (hop != circ->cpath); + } if (!best || (best->build_state->need_uptime && !need_uptime)) best = circ; next: ; @@ -986,6 +1093,7 @@ circuit_mark_all_unused_circs(void) * This is useful for letting the user change pseudonyms, so new * streams will not be linkable to old streams. */ +/* XXX023 this is a bad name for what this function does */ void circuit_expire_all_dirty_circs(void) { @@ -996,6 +1104,8 @@ circuit_expire_all_dirty_circs(void) if (CIRCUIT_IS_ORIGIN(circ) && !circ->marked_for_close && circ->timestamp_dirty) + /* XXXX023 This is a screwed-up way to say "This is too dirty + * for new circuits. */ circ->timestamp_dirty -= options->MaxCircuitDirtiness; } } @@ -1083,14 +1193,16 @@ _circuit_mark_for_close(circuit_t *circ, int reason, int line, tor_assert(ocirc->rend_data); /* treat this like getting a nack from it */ log_info(LD_REND, "Failed intro circ %s to %s (awaiting ack). " - "Removing from descriptor.", - safe_str(ocirc->rend_data->onion_address), - safe_str(build_state_get_exit_nickname(ocirc->build_state))); + "Removing from descriptor.", + safe_str_client(ocirc->rend_data->onion_address), + safe_str_client(build_state_get_exit_nickname(ocirc->build_state))); rend_client_remove_intro_point(ocirc->build_state->chosen_exit, ocirc->rend_data); } - if (circ->n_conn) + if (circ->n_conn) { + circuit_clear_cell_queue(circ, circ->n_conn); connection_or_send_destroy(circ->n_circ_id, circ->n_conn, reason); + } if (! CIRCUIT_IS_ORIGIN(circ)) { or_circuit_t *or_circ = TO_OR_CIRCUIT(circ); @@ -1114,8 +1226,10 @@ _circuit_mark_for_close(circuit_t *circ, int reason, int line, conn->on_circuit = NULL; } - if (or_circ->p_conn) + if (or_circ->p_conn) { + circuit_clear_cell_queue(circ, or_circ->p_conn); connection_or_send_destroy(or_circ->p_circ_id, or_circ->p_conn, reason); + } } else { origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ); edge_connection_t *conn; @@ -1236,11 +1350,6 @@ assert_circuit_ok(const circuit_t *c) tor_assert(c == c2); } } -#if 0 /* false now that rendezvous exits are attached to p_streams */ - if (origin_circ) - for (conn = origin_circ->p_streams; conn; conn = conn->next_stream) - tor_assert(conn->_base.type == CONN_TYPE_AP); -#endif if (or_circ) for (conn = or_circ->n_streams; conn; conn = conn->next_stream) tor_assert(conn->_base.type == CONN_TYPE_EXIT); diff --git a/src/or/circuitlist.h b/src/or/circuitlist.h new file mode 100644 index 0000000000..7b01ca3ae2 --- /dev/null +++ b/src/or/circuitlist.h @@ -0,0 +1,61 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file circuitlist.h + * \brief Header file for circuitlist.c. + **/ + +#ifndef _TOR_CIRCUITLIST_H +#define _TOR_CIRCUITLIST_H + +circuit_t * _circuit_get_global_list(void); +const char *circuit_state_to_string(int state); +const char *circuit_purpose_to_controller_string(uint8_t purpose); +const char *circuit_purpose_to_string(uint8_t purpose); +void circuit_dump_by_conn(connection_t *conn, int severity); +void circuit_set_p_circid_orconn(or_circuit_t *circ, circid_t id, + or_connection_t *conn); +void circuit_set_n_circid_orconn(circuit_t *circ, circid_t id, + or_connection_t *conn); +void circuit_set_state(circuit_t *circ, uint8_t state); +void circuit_close_all_marked(void); +int32_t circuit_initial_package_window(void); +origin_circuit_t *origin_circuit_new(void); +or_circuit_t *or_circuit_new(circid_t p_circ_id, or_connection_t *p_conn); +circuit_t *circuit_get_by_circid_orconn(circid_t circ_id, + or_connection_t *conn); +int circuit_id_in_use_on_orconn(circid_t circ_id, or_connection_t *conn); +circuit_t *circuit_get_by_edge_conn(edge_connection_t *conn); +void circuit_unlink_all_from_or_conn(or_connection_t *conn, int reason); +origin_circuit_t *circuit_get_by_global_id(uint32_t id); +origin_circuit_t *circuit_get_by_rend_query_and_purpose(const char *rend_query, + uint8_t purpose); +origin_circuit_t *circuit_get_next_by_pk_and_purpose(origin_circuit_t *start, + const char *digest, uint8_t purpose); +or_circuit_t *circuit_get_rendezvous(const char *cookie); +or_circuit_t *circuit_get_intro_point(const char *digest); +origin_circuit_t *circuit_find_to_cannibalize(uint8_t purpose, + extend_info_t *info, int flags); +void circuit_mark_all_unused_circs(void); +void circuit_expire_all_dirty_circs(void); +void _circuit_mark_for_close(circuit_t *circ, int reason, + int line, const char *file); +int circuit_get_cpath_len(origin_circuit_t *circ); +crypt_path_t *circuit_get_cpath_hop(origin_circuit_t *circ, int hopnum); +void circuit_get_all_pending_on_or_conn(smartlist_t *out, + or_connection_t *or_conn); +int circuit_count_pending_on_or_conn(or_connection_t *or_conn); + +#define circuit_mark_for_close(c, reason) \ + _circuit_mark_for_close((c), (reason), __LINE__, _SHORT_FILE_) + +void assert_cpath_layer_ok(const crypt_path_t *cp); +void assert_circuit_ok(const circuit_t *c); +void circuit_free_all(void); + +#endif + diff --git a/src/or/circuituse.c b/src/or/circuituse.c index 996c99cef1..fbe2e459a5 100644 --- a/src/or/circuituse.c +++ b/src/or/circuituse.c @@ -10,6 +10,20 @@ **/ #include "or.h" +#include "circuitbuild.h" +#include "circuitlist.h" +#include "circuituse.h" +#include "config.h" +#include "connection.h" +#include "connection_edge.h" +#include "control.h" +#include "policies.h" +#include "rendclient.h" +#include "rendcommon.h" +#include "rendservice.h" +#include "rephist.h" +#include "router.h" +#include "routerlist.h" /********* START VARIABLES **********/ @@ -17,7 +31,7 @@ extern circuit_t *global_circuitlist; /* from circuitlist.c */ /********* END VARIABLES ************/ -static void circuit_expire_old_circuits_clientside(time_t now); +static void circuit_expire_old_circuits_clientside(void); static void circuit_increment_failure_count(void); /** Return 1 if <b>circ</b> could be returned by circuit_get_best(). @@ -148,10 +162,14 @@ circuit_is_better(circuit_t *a, circuit_t *b, uint8_t purpose) return 1; } else { if (a->timestamp_dirty || - a->timestamp_created > b->timestamp_created) + timercmp(&a->timestamp_created, &b->timestamp_created, >)) return 1; if (CIRCUIT_IS_ORIGIN(b) && TO_ORIGIN_CIRCUIT(b)->build_state->is_internal) + /* XXX023 what the heck is this internal thing doing here. I + * think we can get rid of it. circuit_is_acceptable() already + * makes sure that is_internal is exactly what we need it to + * be. -RD */ return 1; } break; @@ -190,7 +208,7 @@ circuit_get_best(edge_connection_t *conn, int must_be_open, uint8_t purpose, int need_uptime, int need_internal) { circuit_t *circ, *best=NULL; - time_t now = time(NULL); + struct timeval now; int intro_going_on_but_too_old = 0; tor_assert(conn); @@ -199,17 +217,16 @@ circuit_get_best(edge_connection_t *conn, int must_be_open, uint8_t purpose, purpose == CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT || purpose == CIRCUIT_PURPOSE_C_REND_JOINED); + tor_gettimeofday(&now); + for (circ=global_circuitlist;circ;circ = circ->next) { if (!circuit_is_acceptable(circ,conn,must_be_open,purpose, - need_uptime,need_internal,now)) + need_uptime,need_internal,now.tv_sec)) continue; -/* XXX022 make this 15 be a function of circuit finishing times we've - * seen lately, a la Fallon Chen's GSoC work -RD */ -#define REND_PARALLEL_INTRO_DELAY 15 if (purpose == CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT && - !must_be_open && circ->state != CIRCUIT_STATE_OPEN && - circ->timestamp_created + REND_PARALLEL_INTRO_DELAY < now) { + !must_be_open && circ->state != CIRCUIT_STATE_OPEN && + tv_mdiff(&now, &circ->timestamp_created) > circ_times.timeout_ms) { intro_going_on_but_too_old = 1; continue; } @@ -229,50 +246,73 @@ circuit_get_best(edge_connection_t *conn, int must_be_open, uint8_t purpose, return best ? TO_ORIGIN_CIRCUIT(best) : NULL; } +#if 0 /** Check whether, according to the policies in <b>options</b>, the * circuit <b>circ</b> makes sense. */ -/* XXXX currently only checks Exclude{Exit}Nodes. It should check more. */ +/* XXXX currently only checks Exclude{Exit}Nodes; it should check more. + * Also, it doesn't have the right definition of an exit circuit. Also, + * it's never called. */ int circuit_conforms_to_options(const origin_circuit_t *circ, const or_options_t *options) { const crypt_path_t *cpath, *cpath_next = NULL; - for (cpath = circ->cpath; cpath && cpath_next != circ->cpath; - cpath = cpath_next) { + /* first check if it includes any excluded nodes */ + for (cpath = circ->cpath; cpath_next != circ->cpath; cpath = cpath_next) { cpath_next = cpath->next; - if (routerset_contains_extendinfo(options->ExcludeNodes, cpath->extend_info)) return 0; + } - if (cpath->next == circ->cpath) { - /* This is apparently the exit node. */ + /* then consider the final hop */ + if (routerset_contains_extendinfo(options->ExcludeExitNodes, + circ->cpath->prev->extend_info)) + return 0; - if (routerset_contains_extendinfo(options->ExcludeExitNodes, - cpath->extend_info)) - return 0; - } - } return 1; } +#endif /** Close all circuits that start at us, aren't open, and were born * at least CircuitBuildTimeout seconds ago. */ void -circuit_expire_building(time_t now) +circuit_expire_building(void) { - circuit_t *victim, *circ = global_circuitlist; - time_t general_cutoff = now - get_options()->CircuitBuildTimeout; - time_t begindir_cutoff = now - get_options()->CircuitBuildTimeout/2; - time_t introcirc_cutoff = begindir_cutoff; + circuit_t *victim, *next_circ = global_circuitlist; + /* circ_times.timeout_ms and circ_times.close_ms are from + * circuit_build_times_get_initial_timeout() if we haven't computed + * custom timeouts yet */ + struct timeval general_cutoff, begindir_cutoff, fourhop_cutoff, + cannibalize_cutoff, close_cutoff, extremely_old_cutoff; + struct timeval now; + struct timeval introcirc_cutoff; cpath_build_state_t *build_state; - while (circ) { - time_t cutoff; - victim = circ; - circ = circ->next; + tor_gettimeofday(&now); +#define SET_CUTOFF(target, msec) do { \ + long ms = tor_lround(msec); \ + struct timeval diff; \ + diff.tv_sec = ms / 1000; \ + diff.tv_usec = (int)((ms % 1000) * 1000); \ + timersub(&now, &diff, &target); \ + } while (0) + + SET_CUTOFF(general_cutoff, circ_times.timeout_ms); + SET_CUTOFF(begindir_cutoff, circ_times.timeout_ms / 2.0); + SET_CUTOFF(fourhop_cutoff, circ_times.timeout_ms * (4/3.0)); + SET_CUTOFF(cannibalize_cutoff, circ_times.timeout_ms / 2.0); + SET_CUTOFF(close_cutoff, circ_times.close_ms); + SET_CUTOFF(extremely_old_cutoff, circ_times.close_ms*2 + 1000); + + introcirc_cutoff = begindir_cutoff; + + while (next_circ) { + struct timeval cutoff; + victim = next_circ; + next_circ = next_circ->next; if (!CIRCUIT_IS_ORIGIN(victim) || /* didn't originate here */ victim->marked_for_close) /* don't mess with marked circs */ continue; @@ -280,11 +320,19 @@ circuit_expire_building(time_t now) build_state = TO_ORIGIN_CIRCUIT(victim)->build_state; if (build_state && build_state->onehop_tunnel) cutoff = begindir_cutoff; + else if (build_state && build_state->desired_path_len == 4 + && !TO_ORIGIN_CIRCUIT(victim)->has_opened) + cutoff = fourhop_cutoff; + else if (TO_ORIGIN_CIRCUIT(victim)->has_opened) + cutoff = cannibalize_cutoff; else if (victim->purpose == CIRCUIT_PURPOSE_C_INTRODUCING) cutoff = introcirc_cutoff; + else if (victim->purpose == CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT) + cutoff = close_cutoff; else cutoff = general_cutoff; - if (victim->timestamp_created > cutoff) + + if (timercmp(&victim->timestamp_created, &cutoff, >)) continue; /* it's still young, leave it alone */ #if 0 @@ -330,7 +378,7 @@ circuit_expire_building(time_t now) * because that's set when they switch purposes */ if (TO_ORIGIN_CIRCUIT(victim)->rend_data || - victim->timestamp_dirty > cutoff) + victim->timestamp_dirty > cutoff.tv_sec) continue; break; case CIRCUIT_PURPOSE_C_REND_READY_INTRO_ACKED: @@ -339,10 +387,60 @@ circuit_expire_building(time_t now) * make an introduction attempt. so timestamp_dirty * will reflect the time since the last attempt. */ - if (victim->timestamp_dirty > cutoff) + if (victim->timestamp_dirty > cutoff.tv_sec) continue; break; } + } else { /* circuit not open, consider recording failure as timeout */ + int first_hop_succeeded = TO_ORIGIN_CIRCUIT(victim)->cpath && + TO_ORIGIN_CIRCUIT(victim)->cpath->state == CPATH_STATE_OPEN; + + if (TO_ORIGIN_CIRCUIT(victim)->p_streams != NULL) { + log_warn(LD_BUG, "Circuit %d (purpose %d, %s) has timed out, " + "yet has attached streams!", + TO_ORIGIN_CIRCUIT(victim)->global_identifier, + victim->purpose, + circuit_purpose_to_string(victim->purpose)); + tor_fragile_assert(); + continue; + } + + if (circuit_timeout_want_to_count_circ(TO_ORIGIN_CIRCUIT(victim)) && + circuit_build_times_enough_to_compute(&circ_times)) { + /* Circuits are allowed to last longer for measurement. + * Switch their purpose and wait. */ + if (victim->purpose != CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT) { + control_event_circuit_status(TO_ORIGIN_CIRCUIT(victim), + CIRC_EVENT_FAILED, + END_CIRC_REASON_TIMEOUT); + victim->purpose = CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT; + /* Record this failure to check for too many timeouts + * in a row. This function does not record a time value yet + * (we do that later); it only counts the fact that we did + * have a timeout. */ + circuit_build_times_count_timeout(&circ_times, + first_hop_succeeded); + continue; + } + + /* + * If the circuit build time is much greater than we would have cut + * it off at, we probably had a suspend event along this codepath, + * and we should discard the value. + */ + if (timercmp(&victim->timestamp_created, &extremely_old_cutoff, <)) { + log_notice(LD_CIRC, + "Extremely large value for circuit build timeout: %lds. " + "Assuming clock jump. Purpose %d (%s)", + (long)(now.tv_sec - victim->timestamp_created.tv_sec), + victim->purpose, + circuit_purpose_to_string(victim->purpose)); + } else if (circuit_build_times_count_close(&circ_times, + first_hop_succeeded, + victim->timestamp_created.tv_sec)) { + circuit_build_times_set_timeout(&circ_times); + } + } } if (victim->n_conn) @@ -357,7 +455,10 @@ circuit_expire_building(time_t now) circuit_state_to_string(victim->state), victim->purpose); circuit_log_path(LOG_INFO,LD_CIRC,TO_ORIGIN_CIRCUIT(victim)); - circuit_mark_for_close(victim, END_CIRC_REASON_TIMEOUT); + if (victim->purpose == CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT) + circuit_mark_for_close(victim, END_CIRC_REASON_MEASUREMENT_EXPIRED); + else + circuit_mark_for_close(victim, END_CIRC_REASON_TIMEOUT); } } @@ -431,11 +532,11 @@ circuit_stream_is_being_handled(edge_connection_t *conn, } /** Don't keep more than this many unused open circuits around. */ -#define MAX_UNUSED_OPEN_CIRCUITS 12 +#define MAX_UNUSED_OPEN_CIRCUITS 14 /** Figure out how many circuits we have open that are clean. Make * sure it's enough for all the upcoming behaviors we predict we'll have. - * But if we have too many, close the not-so-useful ones. + * But put an upper bound on the total number of circuits. */ static void circuit_predict_and_launch_new(void) @@ -517,6 +618,19 @@ circuit_predict_and_launch_new(void) circuit_launch_by_router(CIRCUIT_PURPOSE_C_GENERAL, NULL, flags); return; } + + /* Finally, check to see if we still need more circuits to learn + * a good build timeout. But if we're close to our max number we + * want, don't do another -- we want to leave a few slots open so + * we can still build circuits preemptively as needed. */ + if (num < MAX_UNUSED_OPEN_CIRCUITS-2 && + circuit_build_times_needs_circuits_now(&circ_times)) { + flags = CIRCLAUNCH_NEED_CAPACITY; + log_info(LD_CIRC, + "Have %d clean circs need another buildtime test circ.", num); + circuit_launch_by_router(CIRCUIT_PURPOSE_C_GENERAL, NULL, flags); + return; + } } /** Build a new test circuit every 5 minutes */ @@ -544,7 +658,7 @@ circuit_build_needed_circs(time_t now) time_to_new_circuit = now + options->NewCircuitPeriod; if (proxy_mode(get_options())) addressmap_clean(now); - circuit_expire_old_circuits_clientside(now); + circuit_expire_old_circuits_clientside(); #if 0 /* disable for now, until predict-and-launch-new can cull leftovers */ circ = circuit_get_youngest_clean_open(CIRCUIT_PURPOSE_C_GENERAL); @@ -624,37 +738,77 @@ circuit_detach_stream(circuit_t *circ, edge_connection_t *conn) tor_fragile_assert(); } +/** If we haven't yet decided on a good timeout value for circuit + * building, we close idles circuits aggressively so we can get more + * data points. */ +#define IDLE_TIMEOUT_WHILE_LEARNING (10*60) + /** Find each circuit that has been unused for too long, or dirty * for too long and has no streams on it: mark it for close. */ static void -circuit_expire_old_circuits_clientside(time_t now) +circuit_expire_old_circuits_clientside(void) { circuit_t *circ; - time_t cutoff = now - get_options()->CircuitIdleTimeout; + struct timeval cutoff, now; + + tor_gettimeofday(&now); + cutoff = now; + + if (circuit_build_times_needs_circuits(&circ_times)) { + /* Circuits should be shorter lived if we need more of them + * for learning a good build timeout */ + cutoff.tv_sec -= IDLE_TIMEOUT_WHILE_LEARNING; + } else { + cutoff.tv_sec -= get_options()->CircuitIdleTimeout; + } for (circ = global_circuitlist; circ; circ = circ->next) { - if (circ->marked_for_close || ! CIRCUIT_IS_ORIGIN(circ)) + if (circ->marked_for_close || !CIRCUIT_IS_ORIGIN(circ)) continue; /* If the circuit has been dirty for too long, and there are no streams * on it, mark it for close. */ if (circ->timestamp_dirty && - circ->timestamp_dirty + get_options()->MaxCircuitDirtiness < now && + circ->timestamp_dirty + get_options()->MaxCircuitDirtiness < + now.tv_sec && !TO_ORIGIN_CIRCUIT(circ)->p_streams /* nothing attached */ ) { - log_debug(LD_CIRC, "Closing n_circ_id %d (dirty %d secs ago, " + log_debug(LD_CIRC, "Closing n_circ_id %d (dirty %ld sec ago, " "purpose %d)", - circ->n_circ_id, (int)(now - circ->timestamp_dirty), + circ->n_circ_id, (long)(now.tv_sec - circ->timestamp_dirty), circ->purpose); circuit_mark_for_close(circ, END_CIRC_REASON_FINISHED); - } else if (!circ->timestamp_dirty && - circ->state == CIRCUIT_STATE_OPEN && - circ->purpose == CIRCUIT_PURPOSE_C_GENERAL) { - if (circ->timestamp_created < cutoff) { - log_debug(LD_CIRC, - "Closing circuit that has been unused for %d seconds.", - (int)(now - circ->timestamp_created)); - circuit_mark_for_close(circ, END_CIRC_REASON_FINISHED); + } else if (!circ->timestamp_dirty && circ->state == CIRCUIT_STATE_OPEN) { + if (timercmp(&circ->timestamp_created, &cutoff, <)) { + if (circ->purpose == CIRCUIT_PURPOSE_C_GENERAL || + circ->purpose == CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT || + circ->purpose == CIRCUIT_PURPOSE_S_ESTABLISH_INTRO || + circ->purpose == CIRCUIT_PURPOSE_TESTING || + (circ->purpose >= CIRCUIT_PURPOSE_C_INTRODUCING && + circ->purpose <= CIRCUIT_PURPOSE_C_REND_READY_INTRO_ACKED) || + circ->purpose == CIRCUIT_PURPOSE_S_CONNECT_REND) { + log_debug(LD_CIRC, + "Closing circuit that has been unused for %ld msec.", + tv_mdiff(&circ->timestamp_created, &now)); + circuit_mark_for_close(circ, END_CIRC_REASON_FINISHED); + } else if (!TO_ORIGIN_CIRCUIT(circ)->is_ancient) { + /* Server-side rend joined circuits can end up really old, because + * they are reused by clients for longer than normal. The client + * controls their lifespan. (They never become dirty, because + * connection_exit_begin_conn() never marks anything as dirty.) + * Similarly, server-side intro circuits last a long time. */ + if (circ->purpose != CIRCUIT_PURPOSE_S_REND_JOINED && + circ->purpose != CIRCUIT_PURPOSE_S_INTRO) { + log_notice(LD_CIRC, + "Ancient non-dirty circuit %d is still around after " + "%ld milliseconds. Purpose: %d (%s)", + TO_ORIGIN_CIRCUIT(circ)->global_identifier, + tv_mdiff(&circ->timestamp_created, &now), + circ->purpose, + circuit_purpose_to_string(circ->purpose)); + TO_ORIGIN_CIRCUIT(circ)->is_ancient = 1; + } + } } } } @@ -794,6 +948,11 @@ circuit_has_opened(origin_circuit_t *circ) { control_event_circuit_status(circ, CIRC_EVENT_BUILT, 0); + /* Remember that this circuit has finished building. Now if we start + * it building again later (e.g. by extending it), we will know not + * to consider its build time. */ + circ->has_opened = 1; + switch (TO_CIRCUIT(circ)->purpose) { case CIRCUIT_PURPOSE_C_ESTABLISH_REND: rend_client_rendcirc_has_opened(circ); @@ -846,15 +1005,29 @@ circuit_build_failed(origin_circuit_t *circ) * to blame, blame it. Also, avoid this relay for a while, and * fail any one-hop directory fetches destined for it. */ const char *n_conn_id = circ->cpath->extend_info->identity_digest; + int already_marked = 0; if (circ->_base.n_conn) { or_connection_t *n_conn = circ->_base.n_conn; + if (n_conn->is_bad_for_new_circs) { + /* We only want to blame this router when a fresh healthy + * connection fails. So don't mark this router as newly failed, + * since maybe this was just an old circuit attempt that's + * finally timing out now. Also, there's no need to blow away + * circuits/streams/etc, since the failure of an unhealthy conn + * doesn't tell us much about whether a healthy conn would + * succeed. */ + already_marked = 1; + } log_info(LD_OR, "Our circuit failed to get a response from the first hop " "(%s:%d). I'm going to try to rotate to a better connection.", n_conn->_base.address, n_conn->_base.port); n_conn->is_bad_for_new_circs = 1; + } else { + log_info(LD_OR, + "Our circuit died before the first hop with no connection"); } - if (n_conn_id) { + if (n_conn_id && !already_marked) { entry_guard_register_connect_status(n_conn_id, 0, 1, time(NULL)); /* if there are any one-hop streams waiting on this circuit, fail * them now so they can retry elsewhere. */ @@ -936,8 +1109,8 @@ circuit_launch_by_router(uint8_t purpose, if (exit) info = extend_info_from_router(exit); circ = circuit_launch_by_extend_info(purpose, info, flags); - if (info) - extend_info_free(info); + + extend_info_free(info); return circ; } @@ -969,13 +1142,14 @@ circuit_launch_by_extend_info(uint8_t purpose, * internal circs rather than exit circs? -RD */ circ = circuit_find_to_cannibalize(purpose, extend_info, flags); if (circ) { - log_info(LD_CIRC,"Cannibalizing circ '%s' for purpose %d", - build_state_get_exit_nickname(circ->build_state), purpose); + log_info(LD_CIRC,"Cannibalizing circ '%s' for purpose %d (%s)", + build_state_get_exit_nickname(circ->build_state), purpose, + circuit_purpose_to_string(purpose)); circ->_base.purpose = purpose; /* reset the birth date of this circ, else expire_building * will see it and think it's been trying to build since it * began. */ - circ->_base.timestamp_created = time(NULL); + tor_gettimeofday(&circ->_base.timestamp_created); switch (purpose) { case CIRCUIT_PURPOSE_C_ESTABLISH_REND: case CIRCUIT_PURPOSE_S_ESTABLISH_INTRO: @@ -1080,15 +1254,17 @@ circuit_get_open_circ_or_launch(edge_connection_t *conn, int severity = LOG_NOTICE; /* FFFF if this is a tunneled directory fetch, don't yell * as loudly. the user doesn't even know it's happening. */ - if (options->UseBridges && bridges_known_but_down()) { + if (entry_list_is_constrained(options) && + entries_known_but_down(options)) { log_fn(severity, LD_APP|LD_DIR, - "Application request when we're believed to be " - "offline. Optimistically trying known bridges again."); - bridges_retry_all(); + "Application request when we haven't used client functionality " + "lately. Optimistically trying known %s again.", + options->UseBridges ? "bridges" : "entrynodes"); + entries_retry_all(options); } else if (!options->UseBridges || any_bridge_descriptors_known()) { log_fn(severity, LD_APP|LD_DIR, - "Application request when we're believed to be " - "offline. Optimistically trying directory fetches again."); + "Application request when we haven't used client functionality " + "lately. Optimistically trying directory fetches again."); routerlist_retry_directory_downloads(time(NULL)); } } @@ -1111,17 +1287,19 @@ circuit_get_open_circ_or_launch(edge_connection_t *conn, need_uptime)) { log_notice(LD_APP, "No Tor server allows exit to %s:%d. Rejecting.", - safe_str(conn->socks_request->address), + safe_str_client(conn->socks_request->address), conn->socks_request->port); return -1; } } else { - /* XXXX022 Duplicates checks in connection_ap_handshake_attach_circuit */ + /* XXXX023 Duplicates checks in connection_ap_handshake_attach_circuit: + * refactor into a single function? */ routerinfo_t *router = router_get_by_nickname(conn->chosen_exit_name, 1); int opt = conn->chosen_exit_optional; if (router && !connection_ap_can_use_exit(conn, router)) { log_fn(opt ? LOG_INFO : LOG_WARN, LD_APP, - "Requested exit point '%s' would refuse request. %s.", + "Requested exit point '%s' is excluded or " + "would refuse request. %s.", conn->chosen_exit_name, opt ? "Trying others" : "Closing"); if (opt) { conn->chosen_exit_optional = 0; @@ -1152,19 +1330,14 @@ circuit_get_open_circ_or_launch(edge_connection_t *conn, if (!extend_info) { log_info(LD_REND, "No intro points for '%s': re-fetching service descriptor.", - safe_str(conn->rend_data->onion_address)); - /* Fetch both, v0 and v2 rend descriptors in parallel. Use whichever - * arrives first. Exception: When using client authorization, only - * fetch v2 descriptors.*/ + safe_str_client(conn->rend_data->onion_address)); rend_client_refetch_v2_renddesc(conn->rend_data); - if (conn->rend_data->auth_type == REND_NO_AUTH) - rend_client_refetch_renddesc(conn->rend_data->onion_address); conn->_base.state = AP_CONN_STATE_RENDDESC_WAIT; return 0; } log_info(LD_REND,"Chose '%s' as intro point for '%s'.", extend_info->nickname, - safe_str(conn->rend_data->onion_address)); + safe_str_client(conn->rend_data->onion_address)); } /* If we have specified a particular exit node for our @@ -1193,7 +1366,7 @@ circuit_get_open_circ_or_launch(edge_connection_t *conn, } if (tor_addr_from_str(&addr, conn->socks_request->address) < 0) { log_info(LD_DIR, "Broken address %s on tunnel conn. Closing.", - escaped_safe_str(conn->socks_request->address)); + escaped_safe_str_client(conn->socks_request->address)); return -1; } extend_info = extend_info_alloc(conn->chosen_exit_name+1, @@ -1235,10 +1408,20 @@ circuit_get_open_circ_or_launch(edge_connection_t *conn, flags); } - if (extend_info) - extend_info_free(extend_info); + extend_info_free(extend_info); - if (desired_circuit_purpose != CIRCUIT_PURPOSE_C_GENERAL) { + if (desired_circuit_purpose == CIRCUIT_PURPOSE_C_GENERAL) { + /* We just caused a circuit to get built because of this stream. + * If this stream has caused a _lot_ of circuits to be built, that's + * a bad sign: we should tell the user. */ + if (conn->num_circuits_launched < NUM_CIRCUITS_LAUNCHED_THRESHOLD && + ++conn->num_circuits_launched == NUM_CIRCUITS_LAUNCHED_THRESHOLD) + log_warn(LD_BUG, "The application request to %s:%d has launched " + "%d circuits without finding one it likes.", + escaped_safe_str_client(conn->socks_request->address), + conn->socks_request->port, + conn->num_circuits_launched); + } else { /* help predict this next time */ rep_hist_note_used_internal(time(NULL), need_uptime, 1); if (circ) { @@ -1418,7 +1601,7 @@ connection_ap_handshake_attach_circuit(edge_connection_t *conn) LOG_INFO : LOG_NOTICE; log_fn(severity, LD_APP, "Tried for %d seconds to get a connection to %s:%d. Giving up.", - conn_age, safe_str(conn->socks_request->address), + conn_age, safe_str_client(conn->socks_request->address), conn->socks_request->port); return -1; } @@ -1447,7 +1630,8 @@ connection_ap_handshake_attach_circuit(edge_connection_t *conn) } if (router && !connection_ap_can_use_exit(conn, router)) { log_fn(opt ? LOG_INFO : LOG_WARN, LD_APP, - "Requested exit point '%s' would refuse request. %s.", + "Requested exit point '%s' is excluded or " + "would refuse request. %s.", conn->chosen_exit_name, opt ? "Trying others" : "Closing"); if (opt) { conn->chosen_exit_optional = 0; @@ -1461,7 +1645,7 @@ connection_ap_handshake_attach_circuit(edge_connection_t *conn) /* find the circuit that we should use, if there is one. */ retval = circuit_get_open_circ_or_launch( conn, CIRCUIT_PURPOSE_C_GENERAL, &circ); - if (retval < 1) // XXX021 if we totally fail, this still returns 0 -RD + if (retval < 1) // XXX022 if we totally fail, this still returns 0 -RD return retval; log_debug(LD_APP|LD_CIRC, @@ -1560,14 +1744,21 @@ connection_ap_handshake_attach_circuit(edge_connection_t *conn) "introduction. (stream %d sec old)", introcirc->_base.n_circ_id, rendcirc->_base.n_circ_id, conn_age); - if (rend_client_send_introduction(introcirc, rendcirc) < 0) { + switch (rend_client_send_introduction(introcirc, rendcirc)) { + case 0: /* success */ + rendcirc->_base.timestamp_dirty = time(NULL); + introcirc->_base.timestamp_dirty = time(NULL); + assert_circuit_ok(TO_CIRCUIT(rendcirc)); + assert_circuit_ok(TO_CIRCUIT(introcirc)); + return 0; + case -1: /* transient error */ + return 0; + case -2: /* permanent error */ + return -1; + default: /* oops */ + tor_fragile_assert(); return -1; } - rendcirc->_base.timestamp_dirty = time(NULL); - introcirc->_base.timestamp_dirty = time(NULL); - assert_circuit_ok(TO_CIRCUIT(rendcirc)); - assert_circuit_ok(TO_CIRCUIT(introcirc)); - return 0; } } diff --git a/src/or/circuituse.h b/src/or/circuituse.h new file mode 100644 index 0000000000..9f393ab378 --- /dev/null +++ b/src/or/circuituse.h @@ -0,0 +1,55 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file circuituse.h + * \brief Header file for circuituse.c. + **/ + +#ifndef _TOR_CIRCUITUSE_H +#define _TOR_CIRCUITUSE_H + +void circuit_expire_building(void); +void circuit_remove_handled_ports(smartlist_t *needed_ports); +int circuit_stream_is_being_handled(edge_connection_t *conn, uint16_t port, + int min); +#if 0 +int circuit_conforms_to_options(const origin_circuit_t *circ, + const or_options_t *options); +#endif +void circuit_build_needed_circs(time_t now); +void circuit_detach_stream(circuit_t *circ, edge_connection_t *conn); + +void circuit_expire_old_circuits_serverside(time_t now); + +void reset_bandwidth_test(void); +int circuit_enough_testing_circs(void); + +void circuit_has_opened(origin_circuit_t *circ); +void circuit_build_failed(origin_circuit_t *circ); + +/** Flag to set when a circuit should have only a single hop. */ +#define CIRCLAUNCH_ONEHOP_TUNNEL (1<<0) +/** Flag to set when a circuit needs to be built of high-uptime nodes */ +#define CIRCLAUNCH_NEED_UPTIME (1<<1) +/** Flag to set when a circuit needs to be built of high-capacity nodes */ +#define CIRCLAUNCH_NEED_CAPACITY (1<<2) +/** Flag to set when the last hop of a circuit doesn't need to be an + * exit node. */ +#define CIRCLAUNCH_IS_INTERNAL (1<<3) +origin_circuit_t *circuit_launch_by_extend_info(uint8_t purpose, + extend_info_t *info, + int flags); +origin_circuit_t *circuit_launch_by_router(uint8_t purpose, + routerinfo_t *exit, int flags); +void circuit_reset_failure_count(int timeout); +int connection_ap_handshake_attach_chosen_circuit(edge_connection_t *conn, + origin_circuit_t *circ, + crypt_path_t *cpath); +int connection_ap_handshake_attach_circuit(edge_connection_t *conn); + +#endif + diff --git a/src/or/command.c b/src/or/command.c index 9269456805..00d9af33fa 100644 --- a/src/or/command.c +++ b/src/or/command.c @@ -16,6 +16,19 @@ */ #include "or.h" +#include "circuitbuild.h" +#include "circuitlist.h" +#include "command.h" +#include "connection.h" +#include "connection_or.h" +#include "config.h" +#include "control.h" +#include "cpuworker.h" +#include "hibernate.h" +#include "onion.h" +#include "relay.h" +#include "router.h" +#include "routerlist.h" /** How many CELL_PADDING cells have we received, ever? */ uint64_t stats_n_padding_cells_processed = 0; @@ -275,7 +288,14 @@ command_process_create_cell(cell_t *cell, or_connection_t *conn) /* hand it off to the cpuworkers, and then return. */ if (assign_onionskin_to_cpuworker(NULL, circ, onionskin) < 0) { - log_warn(LD_GENERAL,"Failed to hand off onionskin. Closing."); +#define WARN_HANDOFF_FAILURE_INTERVAL (6*60*60) + static ratelim_t handoff_warning = + RATELIM_INIT(WARN_HANDOFF_FAILURE_INTERVAL); + char *m; + if ((m = rate_limit_log(&handoff_warning, approx_time()))) { + log_warn(LD_GENERAL,"Failed to hand off onionskin. Closing.%s",m); + tor_free(m); + } circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_INTERNAL); return; } @@ -396,15 +416,18 @@ command_process_relay_cell(cell_t *cell, or_connection_t *conn) * gotten no more than MAX_RELAY_EARLY_CELLS_PER_CIRCUIT of them. */ if (cell->command == CELL_RELAY_EARLY) { if (direction == CELL_DIRECTION_IN) { - /* XXX Allow an unlimited number of inbound relay_early cells for - * now, for hidden service compatibility. See bug 1038. -RD */ + /* Allow an unlimited number of inbound relay_early cells, + * for hidden service compatibility. There isn't any way to make + * a long circuit through inbound relay_early cells anyway. See + * bug 1038. -RD */ } else { or_circuit_t *or_circ = TO_OR_CIRCUIT(circ); if (or_circ->remaining_relay_early_cells == 0) { log_fn(LOG_PROTOCOL_WARN, LD_OR, "Received too many RELAY_EARLY cells on circ %d from %s:%d." " Closing circuit.", - cell->circ_id, safe_str(conn->_base.address), conn->_base.port); + cell->circ_id, safe_str(conn->_base.address), + conn->_base.port); circuit_mark_for_close(circ, END_CIRC_REASON_TORPROTOCOL); return; } @@ -512,7 +535,8 @@ command_process_versions_cell(var_cell_t *cell, or_connection_t *conn) conn->handshake_state->received_versions = 1; log_info(LD_OR, "Negotiated version %d with %s:%d; sending NETINFO.", - highest_supported_version, safe_str(conn->_base.address), + highest_supported_version, + safe_str_client(conn->_base.address), conn->_base.port); tor_assert(conn->link_proto >= 2); @@ -627,8 +651,8 @@ command_process_netinfo_cell(cell_t *cell, or_connection_t *conn) else log_info(LD_OR, "Got good NETINFO cell from %s:%d; OR connection is now " "open, using protocol version %d", - safe_str(conn->_base.address), conn->_base.port, - (int)conn->link_proto); + safe_str_client(conn->_base.address), + conn->_base.port, (int)conn->link_proto); assert_connection_ok(TO_CONN(conn),time(NULL)); } diff --git a/src/or/command.h b/src/or/command.h new file mode 100644 index 0000000000..95b0f3a931 --- /dev/null +++ b/src/or/command.h @@ -0,0 +1,25 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file command.h + * \brief Header file for command.c. + **/ + +#ifndef _TOR_COMMAND_H +#define _TOR_COMMAND_H + +void command_process_cell(cell_t *cell, or_connection_t *conn); +void command_process_var_cell(var_cell_t *cell, or_connection_t *conn); + +extern uint64_t stats_n_padding_cells_processed; +extern uint64_t stats_n_create_cells_processed; +extern uint64_t stats_n_created_cells_processed; +extern uint64_t stats_n_relay_cells_processed; +extern uint64_t stats_n_destroy_cells_processed; + +#endif + diff --git a/src/or/config.c b/src/or/config.c index 209a92d159..9384b3a68a 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -12,6 +12,28 @@ #define CONFIG_PRIVATE #include "or.h" +#include "circuitbuild.h" +#include "circuitlist.h" +#include "config.h" +#include "connection.h" +#include "connection_edge.h" +#include "connection_or.h" +#include "control.h" +#include "cpuworker.h" +#include "dirserv.h" +#include "dirvote.h" +#include "dns.h" +#include "geoip.h" +#include "hibernate.h" +#include "main.h" +#include "networkstatus.h" +#include "policies.h" +#include "relay.h" +#include "rendclient.h" +#include "rendservice.h" +#include "rephist.h" +#include "router.h" +#include "routerlist.h" #ifdef MS_WINDOWS #include <shlobj.h> #endif @@ -61,11 +83,12 @@ static config_abbrev_t _option_abbrevs[] = { PLURAL(LongLivedPort), PLURAL(HiddenServiceNode), PLURAL(HiddenServiceExcludeNode), - PLURAL(NumCpu), + PLURAL(NumCPU), PLURAL(RendNode), PLURAL(RendExcludeNode), PLURAL(StrictEntryNode), PLURAL(StrictExitNode), + PLURAL(StrictNode), { "l", "Log", 1, 0}, { "AllowUnverifiedNodes", "AllowInvalidNodes", 0, 0}, { "AutomapHostSuffixes", "AutomapHostsSuffixes", 0, 0}, @@ -83,10 +106,12 @@ static config_abbrev_t _option_abbrevs[] = { { "NumEntryNodes", "NumEntryGuards", 0, 0}, { "ResolvConf", "ServerDNSResolvConfFile", 0, 1}, { "SearchDomains", "ServerDNSSearchDomains", 0, 1}, - { "ServerDNSAllowBrokenResolvConf", "ServerDNSAllowBrokenConfig", 0, 0 }, + { "ServerDNSAllowBrokenResolvConf", "ServerDNSAllowBrokenConfig", 0, 0}, { "PreferTunnelledDirConns", "PreferTunneledDirConns", 0, 0}, { "BridgeAuthoritativeDirectory", "BridgeAuthoritativeDir", 0, 0}, { "HashedControlPassword", "__HashedControlSessionPassword", 1, 0}, + { "StrictEntryNodes", "StrictNodes", 0, 1}, + { "StrictExitNodes", "StrictNodes", 0, 1}, { NULL, NULL, 0, 0}, }; @@ -134,6 +159,7 @@ static config_var_t _option_vars[] = { V(AccountingMax, MEMUNIT, "0 bytes"), V(AccountingStart, STRING, NULL), V(Address, STRING, NULL), + V(AllowDotExit, BOOL, "0"), V(AllowInvalidNodes, CSV, "middle,rendezvous"), V(AllowNonRFC953Hostnames, BOOL, "0"), V(AllowSingleHopCircuits, BOOL, "0"), @@ -162,10 +188,16 @@ static config_var_t _option_vars[] = { V(BridgePassword, STRING, NULL), V(BridgeRecordUsageByCountry, BOOL, "1"), V(BridgeRelay, BOOL, "0"), - V(CircuitBuildTimeout, INTERVAL, "1 minute"), + V(CellStatistics, BOOL, "0"), + V(LearnCircuitBuildTimeout, BOOL, "1"), + V(CircuitBuildTimeout, INTERVAL, "0"), V(CircuitIdleTimeout, INTERVAL, "1 hour"), + V(CircuitStreamTimeout, INTERVAL, "0"), + V(CircuitPriorityHalflife, DOUBLE, "-100.0"), /*negative:'Use default'*/ V(ClientDNSRejectInternalAddresses, BOOL,"1"), + V(ClientRejectInternalAddresses, BOOL, "1"), V(ClientOnly, BOOL, "0"), + V(ConsensusParams, STRING, NULL), V(ConnLimit, UINT, "1000"), V(ConstrainedSockets, BOOL, "0"), V(ConstrainedSockSize, MEMUNIT, "8192"), @@ -186,18 +218,19 @@ static config_var_t _option_vars[] = { V(DirPort, UINT, "0"), V(DirPortFrontPage, FILENAME, NULL), OBSOLETE("DirPostPeriod"), -#ifdef ENABLE_GEOIP_STATS - V(DirRecordUsageByCountry, BOOL, "0"), - V(DirRecordUsageGranularity, UINT, "4"), - V(DirRecordUsageRetainIPs, INTERVAL, "14 days"), - V(DirRecordUsageSaveInterval, INTERVAL, "6 hours"), -#endif + OBSOLETE("DirRecordUsageByCountry"), + OBSOLETE("DirRecordUsageGranularity"), + OBSOLETE("DirRecordUsageRetainIPs"), + OBSOLETE("DirRecordUsageSaveInterval"), + V(DirReqStatistics, BOOL, "0"), VAR("DirServer", LINELIST, DirServers, NULL), + V(DisableAllSwap, BOOL, "0"), V(DNSPort, UINT, "0"), V(DNSListenAddress, LINELIST, NULL), V(DownloadExtraInfo, BOOL, "0"), V(EnforceDistinctSubnets, BOOL, "1"), V(EntryNodes, ROUTERSET, NULL), + V(EntryStatistics, BOOL, "0"), V(TestingEstimatedDescriptorPropagationTime, INTERVAL, "10 minutes"), V(ExcludeNodes, ROUTERSET, NULL), V(ExcludeExitNodes, ROUTERSET, NULL), @@ -205,12 +238,20 @@ static config_var_t _option_vars[] = { V(ExitNodes, ROUTERSET, NULL), V(ExitPolicy, LINELIST, NULL), V(ExitPolicyRejectPrivate, BOOL, "1"), + V(ExitPortStatistics, BOOL, "0"), + V(ExtraInfoStatistics, BOOL, "0"), + +#if defined (WINCE) + V(FallbackNetworkstatusFile, FILENAME, "fallback-consensus"), +#else V(FallbackNetworkstatusFile, FILENAME, SHARE_DATADIR PATH_SEPARATOR "tor" PATH_SEPARATOR "fallback-consensus"), +#endif V(FascistFirewall, BOOL, "0"), V(FirewallPorts, CSV, ""), V(FastFirstHopPK, BOOL, "1"), V(FetchDirInfoEarly, BOOL, "0"), + V(FetchDirInfoExtraEarly, BOOL, "0"), V(FetchServerDescriptors, BOOL, "1"), V(FetchHidServDescriptors, BOOL, "1"), V(FetchUselessDescriptors, BOOL, "0"), @@ -222,6 +263,8 @@ static config_var_t _option_vars[] = { #endif OBSOLETE("Group"), V(HardwareAccel, BOOL, "0"), + V(AccelName, STRING, NULL), + V(AccelDir, FILENAME, NULL), V(HashedControlPassword, LINELIST, NULL), V(HidServDirectoryV2, BOOL, "1"), VAR("HiddenServiceDir", LINELIST_S, RendConfigLines, NULL), @@ -233,14 +276,19 @@ static config_var_t _option_vars[] = { VAR("HiddenServiceAuthorizeClient",LINELIST_S,RendConfigLines, NULL), V(HidServAuth, LINELIST, NULL), V(HSAuthoritativeDir, BOOL, "0"), - V(HSAuthorityRecordStats, BOOL, "0"), - V(HttpProxy, STRING, NULL), - V(HttpProxyAuthenticator, STRING, NULL), - V(HttpsProxy, STRING, NULL), - V(HttpsProxyAuthenticator, STRING, NULL), + OBSOLETE("HSAuthorityRecordStats"), + V(HTTPProxy, STRING, NULL), + V(HTTPProxyAuthenticator, STRING, NULL), + V(HTTPSProxy, STRING, NULL), + V(HTTPSProxyAuthenticator, STRING, NULL), + V(Socks4Proxy, STRING, NULL), + V(Socks5Proxy, STRING, NULL), + V(Socks5ProxyUsername, STRING, NULL), + V(Socks5ProxyPassword, STRING, NULL), OBSOLETE("IgnoreVersion"), V(KeepalivePeriod, INTERVAL, "5 minutes"), VAR("Log", LINELIST, Logs, NULL), + V(LogMessageDomains, BOOL, "0"), OBSOLETE("LinkPadding"), OBSOLETE("LogLevel"), OBSOLETE("LogFile"), @@ -254,17 +302,20 @@ static config_var_t _option_vars[] = { V(MyFamily, STRING, NULL), V(NewCircuitPeriod, INTERVAL, "30 seconds"), VAR("NamingAuthoritativeDirectory",BOOL, NamingAuthoritativeDir, "0"), - V(NatdListenAddress, LINELIST, NULL), - V(NatdPort, UINT, "0"), + V(NATDListenAddress, LINELIST, NULL), + V(NATDPort, UINT, "0"), V(Nickname, STRING, NULL), - V(NoPublish, BOOL, "0"), + V(WarnUnsafeSocks, BOOL, "1"), + OBSOLETE("NoPublish"), VAR("NodeFamily", LINELIST, NodeFamilies, NULL), - V(NumCpus, UINT, "1"), + V(NumCPUs, UINT, "1"), V(NumEntryGuards, UINT, "3"), V(ORListenAddress, LINELIST, NULL), V(ORPort, UINT, "0"), V(OutboundBindAddress, STRING, NULL), OBSOLETE("PathlenCoinWeight"), + V(PerConnBWBurst, MEMUNIT, "0"), + V(PerConnBWRate, MEMUNIT, "0"), V(PidFile, STRING, NULL), V(TestingTorNetwork, BOOL, "0"), V(PreferTunneledDirConns, BOOL, "1"), @@ -278,6 +329,7 @@ static config_var_t _option_vars[] = { V(RecommendedClientVersions, LINELIST, NULL), V(RecommendedServerVersions, LINELIST, NULL), OBSOLETE("RedirectExit"), + V(RefuseUnknownExits, STRING, "auto"), V(RejectPlaintextPorts, CSV, ""), V(RelayBandwidthBurst, MEMUNIT, "0"), V(RelayBandwidthRate, MEMUNIT, "0"), @@ -287,8 +339,9 @@ static config_var_t _option_vars[] = { V(RephistTrackTime, INTERVAL, "24 hours"), OBSOLETE("RouterFile"), V(RunAsDaemon, BOOL, "0"), - V(RunTesting, BOOL, "0"), - V(SafeLogging, BOOL, "1"), +// V(RunTesting, BOOL, "0"), + OBSOLETE("RunTesting"), // currently unused + V(SafeLogging, STRING, "1"), V(SafeSocks, BOOL, "0"), V(ServerDNSAllowBrokenConfig, BOOL, "1"), V(ServerDNSAllowNonRFC953Hostnames, BOOL,"0"), @@ -304,8 +357,7 @@ static config_var_t _option_vars[] = { V(SocksPort, UINT, "9050"), V(SocksTimeout, INTERVAL, "2 minutes"), OBSOLETE("StatusFetchPeriod"), - V(StrictEntryNodes, BOOL, "0"), - V(StrictExitNodes, BOOL, "0"), + V(StrictNodes, BOOL, "0"), OBSOLETE("SysLog"), V(TestSocks, BOOL, "0"), OBSOLETE("TestVia"), @@ -330,6 +382,7 @@ static config_var_t _option_vars[] = { V(V3AuthDistDelay, INTERVAL, "5 minutes"), V(V3AuthNIntervalsValid, UINT, "3"), V(V3AuthUseLegacyKey, BOOL, "0"), + V(V3BandwidthsFile, FILENAME, NULL), VAR("VersioningAuthoritativeDirectory",BOOL,VersioningAuthoritativeDir, "0"), V(VirtualAddrNetwork, STRING, "127.192.0.0/10"), V(WarnPlaintextPorts, CSV, "23,109,110,143"), @@ -340,6 +393,8 @@ static config_var_t _option_vars[] = { VAR("__HashedControlSessionPassword", LINELIST, HashedControlSessionPassword, NULL), V(MinUptimeHidServDirectoryV2, INTERVAL, "24 hours"), + V(_UsingTestNetworkDefaults, BOOL, "0"), + { NULL, CONFIG_TYPE_OBSOLETE, 0, NULL } }; @@ -353,6 +408,7 @@ static config_var_t testing_tor_network_defaults[] = { V(AuthDirMaxServersPerAddr, UINT, "0"), V(AuthDirMaxServersPerAuthAddr,UINT, "0"), V(ClientDNSRejectInternalAddresses, BOOL,"0"), + V(ClientRejectInternalAddresses, BOOL, "0"), V(ExitPolicyRejectPrivate, BOOL, "0"), V(V3AuthVotingInterval, INTERVAL, "5 minutes"), V(V3AuthVoteDelay, INTERVAL, "20 seconds"), @@ -362,6 +418,8 @@ static config_var_t testing_tor_network_defaults[] = { V(TestingV3AuthInitialDistDelay, INTERVAL, "20 seconds"), V(TestingAuthDirTimeToLearnReachability, INTERVAL, "0 minutes"), V(TestingEstimatedDescriptorPropagationTime, INTERVAL, "0 minutes"), + V(MinUptimeHidServDirectoryV2, INTERVAL, "0 minutes"), + V(_UsingTestNetworkDefaults, BOOL, "1"), { NULL, CONFIG_TYPE_OBSOLETE, 0, NULL } }; #undef VAR @@ -377,6 +435,9 @@ static config_var_t _state_vars[] = { V(AccountingExpectedUsage, MEMUNIT, NULL), V(AccountingIntervalStart, ISOTIME, NULL), V(AccountingSecondsActive, INTERVAL, NULL), + V(AccountingSecondsToReachSoftLimit,INTERVAL, NULL), + V(AccountingSoftLimitHitAt, ISOTIME, NULL), + V(AccountingBytesAtSoftLimit, MEMUNIT, NULL), VAR("EntryGuard", LINELIST_S, EntryGuards, NULL), VAR("EntryGuardDownSince", LINELIST_S, EntryGuards, NULL), @@ -387,15 +448,30 @@ static config_var_t _state_vars[] = { V(BWHistoryReadEnds, ISOTIME, NULL), V(BWHistoryReadInterval, UINT, "900"), V(BWHistoryReadValues, CSV, ""), + V(BWHistoryReadMaxima, CSV, ""), V(BWHistoryWriteEnds, ISOTIME, NULL), V(BWHistoryWriteInterval, UINT, "900"), V(BWHistoryWriteValues, CSV, ""), + V(BWHistoryWriteMaxima, CSV, ""), + V(BWHistoryDirReadEnds, ISOTIME, NULL), + V(BWHistoryDirReadInterval, UINT, "900"), + V(BWHistoryDirReadValues, CSV, ""), + V(BWHistoryDirReadMaxima, CSV, ""), + V(BWHistoryDirWriteEnds, ISOTIME, NULL), + V(BWHistoryDirWriteInterval, UINT, "900"), + V(BWHistoryDirWriteValues, CSV, ""), + V(BWHistoryDirWriteMaxima, CSV, ""), V(TorVersion, STRING, NULL), V(LastRotatedOnionKey, ISOTIME, NULL), V(LastWritten, ISOTIME, NULL), + V(TotalBuildTimes, UINT, NULL), + V(CircuitBuildAbandonedCount, UINT, "0"), + VAR("CircuitBuildTimeBin", LINELIST_S, BuildtimeHistogram, NULL), + VAR("BuildtimeHistogram", LINELIST_V, BuildtimeHistogram, NULL), + { NULL, CONFIG_TYPE_OBSOLETE, 0, NULL } }; @@ -410,213 +486,6 @@ typedef struct config_var_description_t { const char *description; } config_var_description_t; -/** Descriptions of the configuration options, to be displayed by online - * option browsers */ -/* XXXX022 did anybody want this? at all? If not, kill it.*/ -static config_var_description_t options_description[] = { - /* ==== general options */ - { "AvoidDiskWrites", "If non-zero, try to write to disk less frequently than" - " we would otherwise." }, - { "BandwidthRate", "A token bucket limits the average incoming bandwidth on " - "this node to the specified number of bytes per second." }, - { "BandwidthBurst", "Limit the maximum token buffer size (also known as " - "burst) to the given number of bytes." }, - { "ConnLimit", "Minimum number of simultaneous sockets we must have." }, - { "ConstrainedSockets", "Shrink tx and rx buffers for sockets to avoid " - "system limits on vservers and related environments. See man page for " - "more information regarding this option." }, - { "ConstrainedSockSize", "Limit socket buffers to this size when " - "ConstrainedSockets is enabled." }, - /* ControlListenAddress */ - { "ControlPort", "If set, Tor will accept connections from the same machine " - "(localhost only) on this port, and allow those connections to control " - "the Tor process using the Tor Control Protocol (described in " - "control-spec.txt).", }, - { "CookieAuthentication", "If this option is set to 1, don't allow any " - "connections to the control port except when the connecting process " - "can read a file that Tor creates in its data directory." }, - { "DataDirectory", "Store working data, state, keys, and caches here." }, - { "DirServer", "Tor only trusts directories signed with one of these " - "servers' keys. Used to override the standard list of directory " - "authorities." }, - /* { "FastFirstHopPK", "" }, */ - /* FetchServerDescriptors, FetchHidServDescriptors, - * FetchUselessDescriptors */ - { "HardwareAccel", "If set, Tor tries to use hardware crypto accelerators " - "when it can." }, - /* HashedControlPassword */ - { "HTTPProxy", "Force Tor to make all HTTP directory requests through this " - "host:port (or host:80 if port is not set)." }, - { "HTTPProxyAuthenticator", "A username:password pair to be used with " - "HTTPProxy." }, - { "HTTPSProxy", "Force Tor to make all TLS (SSL) connections through this " - "host:port (or host:80 if port is not set)." }, - { "HTTPSProxyAuthenticator", "A username:password pair to be used with " - "HTTPSProxy." }, - { "KeepalivePeriod", "Send a padding cell every N seconds to keep firewalls " - "from closing our connections while Tor is not in use." }, - { "Log", "Where to send logging messages. Format is " - "minSeverity[-maxSeverity] (stderr|stdout|syslog|file FILENAME)." }, - { "OutboundBindAddress", "Make all outbound connections originate from the " - "provided IP address (only useful for multiple network interfaces)." }, - { "PIDFile", "On startup, write our PID to this file. On clean shutdown, " - "remove the file." }, - { "PreferTunneledDirConns", "If non-zero, avoid directory servers that " - "don't support tunneled connections." }, - /* PreferTunneledDirConns */ - /* ProtocolWarnings */ - /* RephistTrackTime */ - { "RunAsDaemon", "If set, Tor forks and daemonizes to the background when " - "started. Unix only." }, - { "SafeLogging", "If set to 0, Tor logs potentially sensitive strings " - "rather than replacing them with the string [scrubbed]." }, - { "TunnelDirConns", "If non-zero, when a directory server we contact " - "supports it, we will build a one-hop circuit and make an encrypted " - "connection via its ORPort." }, - { "User", "On startup, setuid to this user." }, - - /* ==== client options */ - { "AllowInvalidNodes", "Where on our circuits should Tor allow servers " - "that the directory authorities haven't called \"valid\"?" }, - { "AllowNonRFC953Hostnames", "If set to 1, we don't automatically reject " - "hostnames for having invalid characters." }, - /* CircuitBuildTimeout, CircuitIdleTimeout */ - { "ClientOnly", "If set to 1, Tor will under no circumstances run as a " - "server, even if ORPort is enabled." }, - { "EntryNodes", "A list of preferred entry nodes to use for the first hop " - "in circuits, when possible." }, - /* { "EnforceDistinctSubnets" , "" }, */ - { "ExitNodes", "A list of preferred nodes to use for the last hop in " - "circuits, when possible." }, - { "ExcludeNodes", "A list of nodes never to use when building a circuit." }, - { "FascistFirewall", "If set, Tor will only create outgoing connections to " - "servers running on the ports listed in FirewallPorts." }, - { "FirewallPorts", "A list of ports that we can connect to. Only used " - "when FascistFirewall is set." }, - { "LongLivedPorts", "A list of ports for services that tend to require " - "high-uptime connections." }, - { "MapAddress", "Force Tor to treat all requests for one address as if " - "they were for another." }, - { "NewCircuitPeriod", "Force Tor to consider whether to build a new circuit " - "every NUM seconds." }, - { "MaxCircuitDirtiness", "Do not attach new streams to a circuit that has " - "been used more than this many seconds ago." }, - /* NatdPort, NatdListenAddress */ - { "NodeFamily", "A list of servers that constitute a 'family' and should " - "never be used in the same circuit." }, - { "NumEntryGuards", "How many entry guards should we keep at a time?" }, - /* PathlenCoinWeight */ - { "ReachableAddresses", "Addresses we can connect to, as IP/bits:port-port. " - "By default, we assume all addresses are reachable." }, - /* reachablediraddresses, reachableoraddresses. */ - /* SafeSOCKS */ - { "SOCKSPort", "The port where we listen for SOCKS connections from " - "applications." }, - { "SOCKSListenAddress", "Bind to this address to listen to connections from " - "SOCKS-speaking applications." }, - { "SOCKSPolicy", "Set an entry policy to limit which addresses can connect " - "to the SOCKSPort." }, - /* SocksTimeout */ - { "StrictExitNodes", "If set, Tor will fail to operate when none of the " - "configured ExitNodes can be used." }, - { "StrictEntryNodes", "If set, Tor will fail to operate when none of the " - "configured EntryNodes can be used." }, - /* TestSocks */ - { "TrackHostsExit", "Hosts and domains which should, if possible, be " - "accessed from the same exit node each time we connect to them." }, - { "TrackHostsExitExpire", "Time after which we forget which exit we were " - "using to connect to hosts in TrackHostsExit." }, - /* "TransPort", "TransListenAddress */ - { "UseEntryGuards", "Set to 0 if we want to pick from the whole set of " - "servers for the first position in each circuit, rather than picking a " - "set of 'Guards' to prevent profiling attacks." }, - - /* === server options */ - { "Address", "The advertised (external) address we should use." }, - /* Accounting* options. */ - /* AssumeReachable */ - { "ContactInfo", "Administrative contact information to advertise for this " - "server." }, - { "ExitPolicy", "Address/port ranges for which to accept or reject outgoing " - "connections on behalf of Tor users." }, - /* { "ExitPolicyRejectPrivate, "" }, */ - { "MaxAdvertisedBandwidth", "If set, we will not advertise more than this " - "amount of bandwidth for our bandwidth rate, regardless of how much " - "bandwidth we actually detect." }, - { "MaxOnionsPending", "Reject new attempts to extend circuits when we " - "already have this many pending." }, - { "MyFamily", "Declare a list of other servers as belonging to the same " - "family as this one, so that clients will not use two from the same " - "family in the same circuit." }, - { "Nickname", "Set the server nickname." }, - { "NoPublish", "{DEPRECATED}" }, - { "NumCPUs", "How many processes to use at once for public-key crypto." }, - { "ORPort", "Advertise this port to listen for connections from Tor clients " - "and servers." }, - { "ORListenAddress", "Bind to this address to listen for connections from " - "clients and servers, instead of the default 0.0.0.0:ORPort." }, - { "PublishServerDescriptor", "Set to 0 to keep the server from " - "uploading info to the directory authorities." }, - /* ServerDNS: DetectHijacking, ResolvConfFile, SearchDomains */ - { "ShutdownWaitLength", "Wait this long for clients to finish when " - "shutting down because of a SIGINT." }, - - /* === directory cache options */ - { "DirPort", "Serve directory information from this port, and act as a " - "directory cache." }, - { "DirPortFrontPage", "Serve a static html disclaimer on DirPort." }, - { "DirListenAddress", "Bind to this address to listen for connections from " - "clients and servers, instead of the default 0.0.0.0:DirPort." }, - { "DirPolicy", "Set a policy to limit who can connect to the directory " - "port." }, - - /* Authority options: AuthDirBadExit, AuthDirInvalid, AuthDirReject, - * AuthDirRejectUnlisted, AuthDirListBadExits, AuthoritativeDirectory, - * DirAllowPrivateAddresses, HSAuthoritativeDir, - * NamingAuthoritativeDirectory, RecommendedVersions, - * RecommendedClientVersions, RecommendedServerVersions, RendPostPeriod, - * RunTesting, V1AuthoritativeDirectory, VersioningAuthoritativeDirectory, */ - - /* Hidden service options: HiddenService: dir,excludenodes, nodes, - * options, port. PublishHidServDescriptor */ - - /* Nonpersistent options: __LeaveStreamsUnattached, __AllDirActionsPrivate */ - { NULL, NULL }, -}; - -/** Online description of state variables. */ -static config_var_description_t state_description[] = { - { "AccountingBytesReadInInterval", - "How many bytes have we read in this accounting period?" }, - { "AccountingBytesWrittenInInterval", - "How many bytes have we written in this accounting period?" }, - { "AccountingExpectedUsage", - "How many bytes did we expect to use per minute? (0 for no estimate.)" }, - { "AccountingIntervalStart", "When did this accounting period begin?" }, - { "AccountingSecondsActive", "How long have we been awake in this period?" }, - - { "BWHistoryReadEnds", "When does the last-recorded read-interval end?" }, - { "BWHistoryReadInterval", "How long is each read-interval (in seconds)?" }, - { "BWHistoryReadValues", "Number of bytes read in each interval." }, - { "BWHistoryWriteEnds", "When does the last-recorded write-interval end?" }, - { "BWHistoryWriteInterval", "How long is each write-interval (in seconds)?"}, - { "BWHistoryWriteValues", "Number of bytes written in each interval." }, - - { "EntryGuard", "One of the nodes we have chosen as a fixed entry" }, - { "EntryGuardDownSince", - "The last entry guard has been unreachable since this time." }, - { "EntryGuardUnlistedSince", - "The last entry guard has been unusable since this time." }, - - { "LastRotatedOnionKey", - "The last time at which we changed the medium-term private key used for " - "building circuits." }, - { "LastWritten", "When was this state file last regenerated?" }, - - { "TorVersion", "Which version of Tor generated this state file?" }, - { NULL, NULL }, -}; - /** Type of a callback to validate whether a given configuration is * well-formed and consistent. See options_trial_assign() for documentation * of arguments. */ @@ -635,8 +504,6 @@ typedef struct { config_var_t *vars; /**< List of variables we recognize, their default * values, and where we stick them in the structure. */ validate_fn_t validate_fn; /**< Function to validate config. */ - /** Documentation for configuration variables. */ - config_var_description_t *descriptions; /** If present, extra is a LINELIST variable for unrecognized * lines. Otherwise, unrecognized lines are an error. */ config_var_t *extra; @@ -700,20 +567,6 @@ static uint64_t config_parse_memunit(const char *s, int *ok); static int config_parse_interval(const char *s, int *ok); static void init_libevent(void); static int opt_streq(const char *s1, const char *s2); -/** Versions of libevent. */ -typedef enum { - /* Note: we compare these, so it's important that "old" precede everything, - * and that "other" come last. */ - LE_OLD=0, LE_10C, LE_10D, LE_10E, LE_11, LE_11A, LE_11B, LE_12, LE_12A, - LE_13, LE_13A, LE_13B, LE_13C, LE_13D, LE_13E, - LE_140, LE_141, LE_142, LE_143, LE_144, LE_145, LE_146, LE_147, LE_148, - LE_1499, - LE_OTHER -} le_version_t; -static le_version_t decode_libevent_version(const char *v, int *bincompat_out); -#if defined(HAVE_EVENT_GET_VERSION) && defined(HAVE_EVENT_GET_METHOD) -static void check_libevent_version(const char *m, int server); -#endif /** Magic value for or_options_t. */ #define OR_OPTIONS_MAGIC 9090909 @@ -726,7 +579,6 @@ static config_format_t options_format = { _option_abbrevs, _option_vars, (validate_fn_t)options_validate, - options_description, NULL }; @@ -747,7 +599,6 @@ static config_format_t state_format = { _state_abbrevs, _state_vars, (validate_fn_t)or_state_validate, - state_description, &state_extra_var, }; @@ -812,13 +663,13 @@ set_options(or_options_t *new_val, char **msg) "Acting on config options left us in a broken state. Dying."); exit(1); } - if (old_options) - config_free(&options_format, old_options); + + config_free(&options_format, old_options); return 0; } -extern const char tor_svn_revision[]; /* from tor_main.c */ +extern const char tor_git_revision[]; /* from tor_main.c */ /** The version of this Tor process, as parsed. */ static char *_version = NULL; @@ -827,7 +678,16 @@ static char *_version = NULL; const char * get_version(void) { - return VERSION; + if (_version == NULL) { + if (strlen(tor_git_revision)) { + size_t len = strlen(VERSION)+strlen(tor_git_revision)+16; + _version = tor_malloc(len); + tor_snprintf(_version, len, "%s (git-%s)", VERSION, tor_git_revision); + } else { + _version = tor_strdup(VERSION); + } + } + return _version; } /** Release additional memory allocated in options @@ -835,8 +695,10 @@ get_version(void) static void or_options_free(or_options_t *options) { - if (options->_ExcludeExitNodesUnion) - routerset_free(options->_ExcludeExitNodesUnion); + if (!options) + return; + + routerset_free(options->_ExcludeExitNodesUnion); config_free(&options_format, options); } @@ -845,43 +707,72 @@ or_options_free(or_options_t *options) void config_free_all(void) { - if (global_options) { - or_options_free(global_options); - global_options = NULL; - } - if (global_state) { - config_free(&state_format, global_state); - global_state = NULL; - } - if (global_cmdline_options) { - config_free_lines(global_cmdline_options); - global_cmdline_options = NULL; - } + or_options_free(global_options); + global_options = NULL; + + config_free(&state_format, global_state); + global_state = NULL; + + config_free_lines(global_cmdline_options); + global_cmdline_options = NULL; + tor_free(torrc_fname); tor_free(_version); tor_free(global_dirfrontpagecontents); } -/** If options->SafeLogging is on, return a not very useful string, - * else return address. +/** Make <b>address</b> -- a piece of information related to our operation as + * a client -- safe to log according to the settings in options->SafeLogging, + * and return it. + * + * (We return "[scrubbed]" if SafeLogging is "1", and address otherwise.) + */ +const char * +safe_str_client(const char *address) +{ + tor_assert(address); + if (get_options()->_SafeLogging == SAFELOG_SCRUB_ALL) + return "[scrubbed]"; + else + return address; +} + +/** Make <b>address</b> -- a piece of information of unspecified sensitivity + * -- safe to log according to the settings in options->SafeLogging, and + * return it. + * + * (We return "[scrubbed]" if SafeLogging is anything besides "0", and address + * otherwise.) */ const char * safe_str(const char *address) { tor_assert(address); - if (get_options()->SafeLogging) + if (get_options()->_SafeLogging != SAFELOG_SCRUB_NONE) return "[scrubbed]"; else return address; } +/** Equivalent to escaped(safe_str_client(address)). See reentrancy note on + * escaped(): don't use this outside the main thread, or twice in the same + * log statement. */ +const char * +escaped_safe_str_client(const char *address) +{ + if (get_options()->_SafeLogging == SAFELOG_SCRUB_ALL) + return "[scrubbed]"; + else + return escaped(address); +} + /** Equivalent to escaped(safe_str(address)). See reentrancy note on * escaped(): don't use this outside the main thread, or twice in the same * log statement. */ const char * escaped_safe_str(const char *address) { - if (get_options()->SafeLogging) + if (get_options()->_SafeLogging != SAFELOG_SCRUB_NONE) return "[scrubbed]"; else return escaped(address); @@ -1080,10 +971,12 @@ options_act_reversible(or_options_t *old_options, char **msg) } /* Launch the listeners. (We do this before we setuid, so we can bind to - * ports under 1024.) */ - if (retry_all_listeners(replaced_listeners, new_listeners) < 0) { - *msg = tor_strdup("Failed to bind one of the listener ports."); - goto rollback; + * ports under 1024.) We don't want to rebind if we're hibernating. */ + if (!we_are_hibernating()) { + if (retry_all_listeners(replaced_listeners, new_listeners) < 0) { + *msg = tor_strdup("Failed to bind one of the listener ports."); + goto rollback; + } } } @@ -1097,6 +990,15 @@ options_act_reversible(or_options_t *old_options, char **msg) } #endif + /* Attempt to lock all current and future memory with mlockall() only once */ + if (options->DisableAllSwap) { + if (tor_mlockall() == -1) { + *msg = tor_strdup("DisableAllSwap failure. Do you have proper " + "permissions?"); + goto done; + } + } + /* Setuid/setgid as appropriate */ if (options->User) { if (switch_id(options->User) != 0) { @@ -1109,11 +1011,9 @@ options_act_reversible(or_options_t *old_options, char **msg) /* Ensure data directory is private; create if possible. */ if (check_private_dir(options->DataDirectory, running_tor ? CPD_CREATE : CPD_CHECK)<0) { - char buf[1024]; - int tmp = tor_snprintf(buf, sizeof(buf), + tor_asprintf(msg, "Couldn't access/create private data directory \"%s\"", options->DataDirectory); - *msg = tor_strdup(tmp >= 0 ? buf : "internal error"); goto done; /* No need to roll back, since you can't change the value. */ } @@ -1124,10 +1024,8 @@ options_act_reversible(or_options_t *old_options, char **msg) tor_snprintf(fn, len, "%s"PATH_SEPARATOR"cached-status", options->DataDirectory); if (check_private_dir(fn, running_tor ? CPD_CREATE : CPD_CHECK) < 0) { - char buf[1024]; - int tmp = tor_snprintf(buf, sizeof(buf), + tor_asprintf(msg, "Couldn't access/create private data directory \"%s\"", fn); - *msg = tor_strdup(tmp >= 0 ? buf : "internal error"); tor_free(fn); goto done; } @@ -1227,7 +1125,6 @@ get_effective_bwrate(or_options_t *options) bw = options->MaxAdvertisedBandwidth; if (options->RelayBandwidthRate > 0 && bw > options->RelayBandwidthRate) bw = options->RelayBandwidthRate; - /* ensure_bandwidth_cap() makes sure that this cast can't overflow. */ return (uint32_t)bw; } @@ -1305,14 +1202,14 @@ options_act(or_options_t *old_options) return 0; /* Finish backgrounding the process */ - if (running_tor && options->RunAsDaemon) { + if (options->RunAsDaemon) { /* We may be calling this for the n'th time (on SIGHUP), but it's safe. */ finish_daemon(options->DataDirectory); } /* Write our PID to the PID file. If we do not have write permissions we * will log a warning */ - if (running_tor && options->PidFile) + if (options->PidFile) write_pidfile(options->PidFile); /* Register addressmap directives */ @@ -1345,30 +1242,75 @@ options_act(or_options_t *old_options) if (accounting_is_enabled(options)) configure_accounting(time(NULL)); + /* parse RefuseUnknownExits tristate */ + if (!strcmp(options->RefuseUnknownExits, "0")) + options->RefuseUnknownExits_ = 0; + else if (!strcmp(options->RefuseUnknownExits, "1")) + options->RefuseUnknownExits_ = 1; + else if (!strcmp(options->RefuseUnknownExits, "auto")) + options->RefuseUnknownExits_ = -1; + else { + /* Should have caught this in options_validate */ + return -1; + } + + /* Change the cell EWMA settings */ + cell_ewma_set_scale_factor(options, networkstatus_get_latest_consensus()); + /* Check for transitions that need action. */ if (old_options) { - if (options->UseEntryGuards && !old_options->UseEntryGuards) { + if ((options->UseEntryGuards && !old_options->UseEntryGuards) || + !routerset_equal(old_options->ExcludeNodes,options->ExcludeNodes) || + !routerset_equal(old_options->ExcludeExitNodes, + options->ExcludeExitNodes) || + !routerset_equal(old_options->EntryNodes, options->EntryNodes) || + !routerset_equal(old_options->ExitNodes, options->ExitNodes) || + options->StrictNodes != old_options->StrictNodes) { log_info(LD_CIRC, - "Switching to entry guards; abandoning previous circuits"); + "Changed to using entry guards, or changed preferred or " + "excluded node lists. Abandoning previous circuits."); circuit_mark_all_unused_circs(); circuit_expire_all_dirty_circs(); + addressmap_clear_excluded_trackexithosts(options); } +/* How long should we delay counting bridge stats after becoming a bridge? + * We use this so we don't count people who used our bridge thinking it is + * a relay. If you change this, don't forget to change the log message + * below. It's 4 hours (the time it takes to stop being used by clients) + * plus some extra time for clock skew. */ +#define RELAY_BRIDGE_STATS_DELAY (6 * 60 * 60) + if (! bool_eq(options->BridgeRelay, old_options->BridgeRelay)) { - log_info(LD_GENERAL, "Bridge status changed. Forgetting GeoIP stats."); - geoip_remove_old_clients(time(NULL)+(2*60*60)); + int was_relay = 0; + if (options->BridgeRelay) { + time_t int_start = time(NULL); + if (old_options->ORPort == options->ORPort) { + int_start += RELAY_BRIDGE_STATS_DELAY; + was_relay = 1; + } + geoip_bridge_stats_init(int_start); + log_info(LD_CONFIG, "We are acting as a bridge now. Starting new " + "GeoIP stats interval%s.", was_relay ? " in 6 " + "hours from now" : ""); + } else { + geoip_bridge_stats_term(); + log_info(LD_GENERAL, "We are no longer acting as a bridge. " + "Forgetting GeoIP stats."); + } } if (options_transition_affects_workers(old_options, options)) { log_info(LD_GENERAL, "Worker-related options changed. Rotating workers."); + + if (init_keys() < 0) { + log_warn(LD_BUG,"Error initializing keys; exiting"); + return -1; + } if (server_mode(options) && !server_mode(old_options)) { - if (init_keys() < 0) { - log_warn(LD_BUG,"Error initializing keys; exiting"); - return -1; - } ip_address_changed(0); - if (has_completed_circuit || !any_predicted_circuits(time(NULL))) + if (can_complete_circuit || !any_predicted_circuits(time(NULL))) inform_testing_reachability(); } cpuworkers_rotate(); @@ -1381,6 +1323,10 @@ options_act(or_options_t *old_options) if (options->V3AuthoritativeDir && !old_options->V3AuthoritativeDir) init_keys(); + + if (options->PerConnBWRate != old_options->PerConnBWRate || + options->PerConnBWBurst != old_options->PerConnBWBurst) + connection_or_update_token_buckets(get_connection_array(), options); } /* Maybe load geoip file */ @@ -1389,7 +1335,7 @@ options_act(or_options_t *old_options) || !geoip_is_loaded())) { /* XXXX Don't use this "<default>" junk; make our filename options * understand prefixes somehow. -NM */ - /* XXXX021 Reload GeoIPFile on SIGHUP. -NM */ + /* XXXX023 Reload GeoIPFile on SIGHUP. -NM */ char *actual_fname = tor_strdup(options->GeoIPFile); #ifdef WIN32 if (!strcmp(actual_fname, "<default>")) { @@ -1403,17 +1349,68 @@ options_act(or_options_t *old_options) geoip_load_file(actual_fname, options); tor_free(actual_fname); } -#ifdef ENABLE_GEOIP_STATS - log_warn(LD_CONFIG, "We are configured to measure GeoIP statistics, but " - "the way these statistics are measured has changed " - "significantly in later versions of Tor. The results may not be " - "as expected if you are used to later versions. Be sure you " - "know what you are doing."); -#endif + + if (options->DirReqStatistics && !geoip_is_loaded()) { + /* Check if GeoIP database could be loaded. */ + log_warn(LD_CONFIG, "Configured to measure directory request " + "statistics, but no GeoIP database found!"); + return -1; + } + + if (options->EntryStatistics) { + if (should_record_bridge_info(options)) { + /* Don't allow measuring statistics on entry guards when configured + * as bridge. */ + log_warn(LD_CONFIG, "Bridges cannot be configured to measure " + "additional GeoIP statistics as entry guards."); + return -1; + } else if (!geoip_is_loaded()) { + /* Check if GeoIP database could be loaded. */ + log_warn(LD_CONFIG, "Configured to measure entry node statistics, " + "but no GeoIP database found!"); + return -1; + } + } + + if (options->CellStatistics || options->DirReqStatistics || + options->EntryStatistics || options->ExitPortStatistics) { + time_t now = time(NULL); + if ((!old_options || !old_options->CellStatistics) && + options->CellStatistics) + rep_hist_buffer_stats_init(now); + if ((!old_options || !old_options->DirReqStatistics) && + options->DirReqStatistics) + geoip_dirreq_stats_init(now); + if ((!old_options || !old_options->EntryStatistics) && + options->EntryStatistics) + geoip_entry_stats_init(now); + if ((!old_options || !old_options->ExitPortStatistics) && + options->ExitPortStatistics) + rep_hist_exit_stats_init(now); + if (!old_options) + log_notice(LD_CONFIG, "Configured to measure statistics. Look for " + "the *-stats files that will first be written to the " + "data directory in 24 hours from now."); + } + + if (old_options && old_options->CellStatistics && + !options->CellStatistics) + rep_hist_buffer_stats_term(); + if (old_options && old_options->DirReqStatistics && + !options->DirReqStatistics) + geoip_dirreq_stats_term(); + if (old_options && old_options->EntryStatistics && + !options->EntryStatistics) + geoip_entry_stats_term(); + if (old_options && old_options->ExitPortStatistics && + !options->ExitPortStatistics) + rep_hist_exit_stats_term(); + /* Check if we need to parse and add the EntryNodes config option. */ if (options->EntryNodes && (!old_options || - (!routerset_equal(old_options->EntryNodes,options->EntryNodes)))) + !routerset_equal(old_options->EntryNodes,options->EntryNodes) || + !routerset_equal(old_options->ExcludeNodes,options->ExcludeNodes))) entry_nodes_should_be_added(); /* Since our options changed, we might need to regenerate and upload our @@ -1482,7 +1479,10 @@ expand_abbrev(config_format_t *fmt, const char *option, int command_line, fmt->abbrevs[i].abbreviated, fmt->abbrevs[i].full); } - return fmt->abbrevs[i].full; + /* Keep going through the list in case we want to rewrite it more. + * (We could imagine recursing here, but I don't want to get the + * user into an infinite loop if we craft our list wrong.) */ + option = fmt->abbrevs[i].full; } } return option; @@ -1527,7 +1527,10 @@ config_get_commandlines(int argc, char **argv, config_line_t **result) *new = tor_malloc_zero(sizeof(config_line_t)); s = argv[i]; - while (*s == '-') + /* Each keyword may be prefixed with one or two dashes. */ + if (*s == '-') + s++; + if (*s == '-') s++; (*new)->key = tor_strdup(expand_abbrev(&options_format, s, 1, 1)); @@ -1619,19 +1622,6 @@ config_free_lines(config_line_t *front) } } -/** Return the description for a given configuration variable, or NULL if no - * description exists. */ -static const char * -config_find_description(config_format_t *fmt, const char *name) -{ - int i; - for (i=0; fmt->descriptions[i].name; ++i) { - if (!strcasecmp(name, fmt->descriptions[i].name)) - return fmt->descriptions[i].description; - } - return NULL; -} - /** If <b>key</b> is a configuration option, return the corresponding * config_var_t. Otherwise, if <b>key</b> is a non-standard abbreviation, * warn, and return the corresponding config_var_t. Otherwise return NULL. @@ -1662,6 +1652,16 @@ config_find_option(config_format_t *fmt, const char *key) return NULL; } +/** Return the number of option entries in <b>fmt</b>. */ +static int +config_count_options(config_format_t *fmt) +{ + int i; + for (i=0; fmt->vars[i].name; ++i) + ; + return i; +} + /* * Functions to assign config options. */ @@ -1675,8 +1675,7 @@ static int config_assign_value(config_format_t *fmt, or_options_t *options, config_line_t *c, char **msg) { - int i, r, ok; - char buf[1024]; + int i, ok; config_var_t *var; void *lvalue; @@ -1692,10 +1691,9 @@ config_assign_value(config_format_t *fmt, or_options_t *options, case CONFIG_TYPE_UINT: i = (int)tor_parse_long(c->value, 10, 0, INT_MAX, &ok, NULL); if (!ok) { - r = tor_snprintf(buf, sizeof(buf), + tor_asprintf(msg, "Int keyword '%s %s' is malformed or out of bounds.", c->key, c->value); - *msg = tor_strdup(r >= 0 ? buf : "internal error"); return -1; } *(int *)lvalue = i; @@ -1704,10 +1702,9 @@ config_assign_value(config_format_t *fmt, or_options_t *options, case CONFIG_TYPE_INTERVAL: { i = config_parse_interval(c->value, &ok); if (!ok) { - r = tor_snprintf(buf, sizeof(buf), + tor_asprintf(msg, "Interval '%s %s' is malformed or out of bounds.", c->key, c->value); - *msg = tor_strdup(r >= 0 ? buf : "internal error"); return -1; } *(int *)lvalue = i; @@ -1717,10 +1714,9 @@ config_assign_value(config_format_t *fmt, or_options_t *options, case CONFIG_TYPE_MEMUNIT: { uint64_t u64 = config_parse_memunit(c->value, &ok); if (!ok) { - r = tor_snprintf(buf, sizeof(buf), + tor_asprintf(msg, "Value '%s %s' is malformed or out of bounds.", c->key, c->value); - *msg = tor_strdup(r >= 0 ? buf : "internal error"); return -1; } *(uint64_t *)lvalue = u64; @@ -1730,10 +1726,9 @@ config_assign_value(config_format_t *fmt, or_options_t *options, case CONFIG_TYPE_BOOL: i = (int)tor_parse_long(c->value, 10, 0, 1, &ok, NULL); if (!ok) { - r = tor_snprintf(buf, sizeof(buf), + tor_asprintf(msg, "Boolean '%s %s' expects 0 or 1.", c->key, c->value); - *msg = tor_strdup(r >= 0 ? buf : "internal error"); return -1; } *(int *)lvalue = i; @@ -1751,9 +1746,8 @@ config_assign_value(config_format_t *fmt, or_options_t *options, case CONFIG_TYPE_ISOTIME: if (parse_iso_time(c->value, (time_t *)lvalue)) { - r = tor_snprintf(buf, sizeof(buf), + tor_asprintf(msg, "Invalid time '%s' for keyword '%s'", c->value, c->key); - *msg = tor_strdup(r >= 0 ? buf : "internal error"); return -1; } break; @@ -1764,9 +1758,8 @@ config_assign_value(config_format_t *fmt, or_options_t *options, } *(routerset_t**)lvalue = routerset_new(); if (routerset_parse(*(routerset_t**)lvalue, c->value, c->key)<0) { - tor_snprintf(buf, sizeof(buf), "Invalid exit list '%s' for option '%s'", + tor_asprintf(msg, "Invalid exit list '%s' for option '%s'", c->value, c->key); - *msg = tor_strdup(buf); return -1; } break; @@ -1791,9 +1784,8 @@ config_assign_value(config_format_t *fmt, or_options_t *options, log_warn(LD_CONFIG, "Skipping obsolete configuration option '%s'", c->key); break; case CONFIG_TYPE_LINELIST_V: - r = tor_snprintf(buf, sizeof(buf), + tor_asprintf(msg, "You may not provide a value for virtual option '%s'", c->key); - *msg = tor_strdup(r >= 0 ? buf : "internal error"); return -1; default: tor_assert(0); @@ -1814,7 +1806,7 @@ config_assign_value(config_format_t *fmt, or_options_t *options, static int config_assign_line(config_format_t *fmt, or_options_t *options, config_line_t *c, int use_defaults, - int clear_first, char **msg) + int clear_first, bitarray_t *options_seen, char **msg) { config_var_t *var; @@ -1829,13 +1821,12 @@ config_assign_line(config_format_t *fmt, or_options_t *options, config_line_append((config_line_t**)lvalue, c->key, c->value); return 0; } else { - char buf[1024]; - int tmp = tor_snprintf(buf, sizeof(buf), + tor_asprintf(msg, "Unknown option '%s'. Failing.", c->key); - *msg = tor_strdup(tmp >= 0 ? buf : "internal error"); return -1; } } + /* Put keyword into canonical case. */ if (strcmp(var->name, c->key)) { tor_free(c->key); @@ -1858,6 +1849,18 @@ config_assign_line(config_format_t *fmt, or_options_t *options, return 0; } + if (options_seen && (var->type != CONFIG_TYPE_LINELIST && + var->type != CONFIG_TYPE_LINELIST_S)) { + /* We're tracking which options we've seen, and this option is not + * supposed to occur more than once. */ + int var_index = (int)(var - fmt->vars); + if (bitarray_is_set(options_seen, var_index)) { + log_warn(LD_CONFIG, "Option '%s' used more than once; all but the last " + "value will be ignored.", var->name); + } + bitarray_set(options_seen, var_index); + } + if (config_assign_value(fmt, options, c, msg) < 0) return -2; return 0; @@ -1958,7 +1961,6 @@ get_assigned_option(config_format_t *fmt, void *options, { config_var_t *var; const void *value; - char buf[32]; config_line_t *result; tor_assert(options && key); @@ -1999,19 +2001,16 @@ get_assigned_option(config_format_t *fmt, void *options, case CONFIG_TYPE_UINT: /* This means every or_options_t uint or bool element * needs to be an int. Not, say, a uint16_t or char. */ - tor_snprintf(buf, sizeof(buf), "%d", *(int*)value); - result->value = tor_strdup(buf); + tor_asprintf(&result->value, "%d", *(int*)value); escape_val = 0; /* Can't need escape. */ break; case CONFIG_TYPE_MEMUNIT: - tor_snprintf(buf, sizeof(buf), U64_FORMAT, + tor_asprintf(&result->value, U64_FORMAT, U64_PRINTF_ARG(*(uint64_t*)value)); - result->value = tor_strdup(buf); escape_val = 0; /* Can't need escape. */ break; case CONFIG_TYPE_DOUBLE: - tor_snprintf(buf, sizeof(buf), "%f", *(double*)value); - result->value = tor_strdup(buf); + tor_asprintf(&result->value, "%f", *(double*)value); escape_val = 0; /* Can't need escape. */ break; case CONFIG_TYPE_BOOL: @@ -2130,6 +2129,8 @@ config_assign(config_format_t *fmt, void *options, config_line_t *list, int use_defaults, int clear_first, char **msg) { config_line_t *p; + bitarray_t *options_seen; + const int n_options = config_count_options(fmt); CHECK(fmt, options); @@ -2149,14 +2150,18 @@ config_assign(config_format_t *fmt, void *options, config_line_t *list, config_reset_line(fmt, options, p->key, use_defaults); } + options_seen = bitarray_init_zero(n_options); /* pass 3: assign. */ while (list) { int r; if ((r=config_assign_line(fmt, options, list, use_defaults, - clear_first, msg))) + clear_first, options_seen, msg))) { + bitarray_free(options_seen); return r; + } list = list->next; } + bitarray_free(options_seen); return 0; } @@ -2299,20 +2304,10 @@ list_torrc_options(void) smartlist_t *lines = smartlist_create(); for (i = 0; _option_vars[i].name; ++i) { config_var_t *var = &_option_vars[i]; - const char *desc; if (var->type == CONFIG_TYPE_OBSOLETE || var->type == CONFIG_TYPE_LINELIST_V) continue; - desc = config_find_description(&options_format, var->name); printf("%s\n", var->name); - if (desc) { - wrap_string(lines, desc, 76, " ", " "); - SMARTLIST_FOREACH(lines, char *, cp, { - printf("%s", cp); - tor_free(cp); - }); - smartlist_clear(lines); - } } smartlist_free(lines); } @@ -2331,7 +2326,7 @@ resolve_my_address(int warn_severity, or_options_t *options, uint32_t *addr_out, char **hostname_out) { struct in_addr in; - uint32_t addr; + uint32_t addr; /* host order */ char hostname[256]; int explicit_ip=1; int explicit_hostname=1; @@ -2361,8 +2356,8 @@ resolve_my_address(int warn_severity, or_options_t *options, if (tor_inet_aton(hostname, &in) == 0) { /* then we have to resolve it */ explicit_ip = 0; - if (tor_lookup_hostname(hostname, &addr)) { - uint32_t interface_ip; + if (tor_lookup_hostname(hostname, &addr)) { /* failed to resolve */ + uint32_t interface_ip; /* host order */ if (explicit_hostname) { log_fn(warn_severity, LD_CONFIG, @@ -2383,7 +2378,7 @@ resolve_my_address(int warn_severity, or_options_t *options, log_fn(notice_severity, LD_CONFIG, "Learned IP address '%s' for " "local interface. Using that.", tmpbuf); strlcpy(hostname, "<guessed from interfaces>", sizeof(hostname)); - } else { + } else { /* resolved hostname into addr */ in.s_addr = htonl(addr); if (!explicit_hostname && @@ -2486,7 +2481,7 @@ is_local_addr(const tor_addr_t *addr) if (get_options()->EnforceDistinctSubnets == 0) return 0; if (tor_addr_family(addr) == AF_INET) { - /*XXXX022 IP6 what corresponds to an /24? */ + /*XXXX023 IP6 what corresponds to an /24? */ uint32_t ip = tor_addr_to_ipv4h(addr); /* It's possible that this next check will hit before the first time @@ -2503,61 +2498,16 @@ is_local_addr(const tor_addr_t *addr) return 0; } -/** Called when we don't have a nickname set. Try to guess a good nickname - * based on the hostname, and return it in a newly allocated string. If we - * can't, return NULL and let the caller warn if it wants to. */ -static char * -get_default_nickname(void) -{ - static const char * const bad_default_nicknames[] = { - "localhost", - NULL, - }; - char localhostname[256]; - char *cp, *out, *outp; - int i; - - if (gethostname(localhostname, sizeof(localhostname)) < 0) - return NULL; - - /* Put it in lowercase; stop at the first dot. */ - if ((cp = strchr(localhostname, '.'))) - *cp = '\0'; - tor_strlower(localhostname); - - /* Strip invalid characters. */ - cp = localhostname; - out = outp = tor_malloc(strlen(localhostname) + 1); - while (*cp) { - if (strchr(LEGAL_NICKNAME_CHARACTERS, *cp)) - *outp++ = *cp++; - else - cp++; - } - *outp = '\0'; - - /* Enforce length. */ - if (strlen(out) > MAX_NICKNAME_LEN) - out[MAX_NICKNAME_LEN]='\0'; - - /* Check for dumb names. */ - for (i = 0; bad_default_nicknames[i]; ++i) { - if (!strcmp(out, bad_default_nicknames[i])) { - tor_free(out); - return NULL; - } - } - - return out; -} - /** Release storage held by <b>options</b>. */ static void config_free(config_format_t *fmt, void *options) { int i; - tor_assert(options); + if (!options) + return; + + tor_assert(fmt); for (i=0; fmt->vars[i].name; ++i) option_clear(fmt, options, &(fmt->vars[i])); @@ -2659,6 +2609,8 @@ is_listening_on_low_port(uint16_t port_option, const config_line_t *listen_options) { #ifdef MS_WINDOWS + (void) port_option; + (void) listen_options; return 0; /* No port is too low for windows. */ #else const config_line_t *l; @@ -2708,7 +2660,6 @@ config_dump(config_format_t *fmt, void *options, int minimal, config_line_t *line, *assigned; char *result; int i; - const char *desc; char *msg = NULL; defaults = config_alloc(fmt); @@ -2736,24 +2687,13 @@ config_dump(config_format_t *fmt, void *options, int minimal, option_is_same(fmt, options, defaults, fmt->vars[i].name)) comment_option = 1; - desc = config_find_description(fmt, fmt->vars[i].name); line = assigned = get_assigned_option(fmt, options, fmt->vars[i].name, 1); - if (line && desc) { - /* Only dump the description if there's something to describe. */ - wrap_string(elements, desc, 78, "# ", "# "); - } - for (; line; line = line->next) { - size_t len = strlen(line->key) + strlen(line->value) + 5; char *tmp; - tmp = tor_malloc(len); - if (tor_snprintf(tmp, len, "%s%s %s\n", - comment_option ? "# " : "", - line->key, line->value)<0) { - log_err(LD_BUG,"Internal error writing option value"); - tor_assert(0); - } + tor_asprintf(&tmp, "%s%s %s\n", + comment_option ? "# " : "", + line->key, line->value); smartlist_add(elements, tmp); } config_free_lines(assigned); @@ -2762,13 +2702,8 @@ config_dump(config_format_t *fmt, void *options, int minimal, if (fmt->extra) { line = *(config_line_t**)STRUCT_VAR_P(options, fmt->extra->var_offset); for (; line; line = line->next) { - size_t len = strlen(line->key) + strlen(line->value) + 3; char *tmp; - tmp = tor_malloc(len); - if (tor_snprintf(tmp, len, "%s %s\n", line->key, line->value)<0) { - log_err(LD_BUG,"Internal error writing option value"); - tor_assert(0); - } + tor_asprintf(&tmp, "%s %s\n", line->key, line->value); smartlist_add(elements, tmp); } } @@ -2784,7 +2719,7 @@ config_dump(config_format_t *fmt, void *options, int minimal, * the configuration in <b>options</b>. If <b>minimal</b> is true, do not * include options that are the same as Tor's defaults. */ -static char * +char * options_dump(or_options_t *options, int minimal) { return config_dump(&options_format, options, minimal, 0); @@ -2797,7 +2732,6 @@ static int validate_ports_csv(smartlist_t *sl, const char *name, char **msg) { int i; - char buf[1024]; tor_assert(name); if (!sl) @@ -2807,9 +2741,7 @@ validate_ports_csv(smartlist_t *sl, const char *name, char **msg) { i = atoi(cp); if (i < 1 || i > 65535) { - int r = tor_snprintf(buf, sizeof(buf), - "Port '%s' out of range in %s", cp, name); - *msg = tor_strdup(r >= 0 ? buf : "internal error"); + tor_asprintf(msg, "Port '%s' out of range in %s", cp, name); return -1; } }); @@ -2823,18 +2755,15 @@ validate_ports_csv(smartlist_t *sl, const char *name, char **msg) static int ensure_bandwidth_cap(uint64_t *value, const char *desc, char **msg) { - int r; - char buf[1024]; if (*value > ROUTER_MAX_DECLARED_BANDWIDTH) { /* This handles an understandable special case where somebody says "2gb" * whereas our actual maximum is 2gb-1 (INT_MAX) */ --*value; } if (*value > ROUTER_MAX_DECLARED_BANDWIDTH) { - r = tor_snprintf(buf, sizeof(buf), "%s ("U64_FORMAT") must be at most %d", - desc, U64_PRINTF_ARG(*value), - ROUTER_MAX_DECLARED_BANDWIDTH); - *msg = tor_strdup(r >= 0 ? buf : "internal error"); + tor_asprintf(msg, "%s ("U64_FORMAT") must be at most %d", + desc, U64_PRINTF_ARG(*value), + ROUTER_MAX_DECLARED_BANDWIDTH); return -1; } return 0; @@ -2887,15 +2816,14 @@ compute_publishserverdescriptor(or_options_t *options) /** Highest allowable value for RendPostPeriod. */ #define MAX_DIR_PERIOD (MIN_ONION_KEY_LIFETIME/2) -/** Lowest allowable value for CircuitBuildTimeout; values too low will - * increase network load because of failing connections being retried, and - * might prevent users from connecting to the network at all. */ -#define MIN_CIRCUIT_BUILD_TIMEOUT 30 - /** Lowest allowable value for MaxCircuitDirtiness; if this is too low, Tor * will generate too many circuits and potentially overload the network. */ #define MIN_MAX_CIRCUIT_DIRTINESS 10 +/** Lowest allowable value for CircuitStreamTimeout; if this is too low, Tor + * will generate too many circuits and potentially overload the network. */ +#define MIN_CIRCUIT_STREAM_TIMEOUT 10 + /** Return 0 if every setting in <b>options</b> is reasonable, and a * permissible transition from <b>old_options</b>. Else return -1. * Should have no side effects, except for normalizing the contents of @@ -2912,10 +2840,9 @@ static int options_validate(or_options_t *old_options, or_options_t *options, int from_setconf, char **msg) { - int i, r; + int i; config_line_t *cl; const char *uname = get_uname(); - char buf[1024]; #define REJECT(arg) \ STMT_BEGIN *msg = tor_strdup(arg); return -1; STMT_END #define COMPLAIN(arg) STMT_BEGIN log(LOG_WARN, LD_CONFIG, arg); STMT_END @@ -2932,7 +2859,7 @@ options_validate(or_options_t *old_options, or_options_t *options, !strcmpstart(uname, "Windows Me"))) { log(LOG_WARN, LD_CONFIG, "Tor is running as a server, but you are " "running %s; this probably won't work. See " - "http://wiki.noreply.org/noreply/TheOnionRouter/TorFAQ#ServerOS " + "https://wiki.torproject.org/TheOnionRouter/TorFAQ#ServerOS " "for details.", uname); } @@ -2951,8 +2878,8 @@ options_validate(or_options_t *old_options, or_options_t *options, if (options->TransPort == 0 && options->TransListenAddress != NULL) REJECT("TransPort must be defined if TransListenAddress is defined."); - if (options->NatdPort == 0 && options->NatdListenAddress != NULL) - REJECT("NatdPort must be defined if NatdListenAddress is defined."); + if (options->NATDPort == 0 && options->NATDListenAddress != NULL) + REJECT("NATDPort must be defined if NATDListenAddress is defined."); /* Don't gripe about SocksPort 0 with SocksListenAddress set; a standard * configuration does this. */ @@ -2971,8 +2898,8 @@ options_validate(or_options_t *old_options, or_options_t *options, old = old_options ? old_options->TransListenAddress : NULL; tp = "transparent proxy"; } else { - opt = options->NatdListenAddress; - old = old_options ? old_options->NatdListenAddress : NULL; + opt = options->NATDListenAddress; + old = old_options ? old_options->NATDListenAddress : NULL; tp = "natd proxy"; } @@ -2999,21 +2926,13 @@ options_validate(or_options_t *old_options, or_options_t *options, if (options->Nickname == NULL) { if (server_mode(options)) { - if (!(options->Nickname = get_default_nickname())) { - log_notice(LD_CONFIG, "Couldn't pick a nickname based on " - "our hostname; using %s instead.", UNNAMED_ROUTER_NICKNAME); options->Nickname = tor_strdup(UNNAMED_ROUTER_NICKNAME); - } else { - log_notice(LD_CONFIG, "Choosing default nickname '%s'", - options->Nickname); - } } } else { if (!is_legal_nickname(options->Nickname)) { - r = tor_snprintf(buf, sizeof(buf), + tor_asprintf(msg, "Nickname '%s' is wrong length or contains illegal characters.", options->Nickname); - *msg = tor_strdup(r >= 0 ? buf : "internal error"); return -1; } } @@ -3030,14 +2949,6 @@ options_validate(or_options_t *old_options, or_options_t *options, if (options_init_logs(options, 1)<0) /* Validate the log(s) */ REJECT("Failed to validate Log options. See logs for details."); - if (options->NoPublish) { - log(LOG_WARN, LD_CONFIG, - "NoPublish is obsolete. Use PublishServerDescriptor instead."); - SMARTLIST_FOREACH(options->PublishServerDescriptor, char *, s, - tor_free(s)); - smartlist_clear(options->PublishServerDescriptor); - } - if (authdir_mode(options)) { /* confirm that our address isn't broken, so we can complain now */ uint32_t tmp; @@ -3045,6 +2956,12 @@ options_validate(or_options_t *old_options, or_options_t *options, REJECT("Failed to resolve/guess local address. See logs for details."); } + if (strcmp(options->RefuseUnknownExits, "0") && + strcmp(options->RefuseUnknownExits, "1") && + strcmp(options->RefuseUnknownExits, "auto")) { + REJECT("RefuseUnknownExits must be 0, 1, or auto"); + } + #ifndef MS_WINDOWS if (options->RunAsDaemon && torrc_fname && path_is_relative(torrc_fname)) REJECT("Can't use a relative path to torrc when RunAsDaemon is set."); @@ -3059,14 +2976,14 @@ options_validate(or_options_t *old_options, or_options_t *options, if (options->TransPort < 0 || options->TransPort > 65535) REJECT("TransPort option out of bounds."); - if (options->NatdPort < 0 || options->NatdPort > 65535) - REJECT("NatdPort option out of bounds."); + if (options->NATDPort < 0 || options->NATDPort > 65535) + REJECT("NATDPort option out of bounds."); if (options->SocksPort == 0 && options->TransPort == 0 && - options->NatdPort == 0 && options->ORPort == 0 && + options->NATDPort == 0 && options->ORPort == 0 && options->DNSPort == 0 && !options->RendConfigLines) log(LOG_WARN, LD_CONFIG, - "SocksPort, TransPort, NatdPort, DNSPort, and ORPort are all " + "SocksPort, TransPort, NATDPort, DNSPort, and ORPort are all " "undefined, and there aren't any hidden services configured. " "Tor will still run, but probably won't do anything."); @@ -3100,19 +3017,11 @@ options_validate(or_options_t *old_options, or_options_t *options, routerset_union(options->_ExcludeExitNodesUnion,options->ExcludeNodes); } - if (options->StrictExitNodes && - (!options->ExitNodes) && - (!old_options || - (old_options->StrictExitNodes != options->StrictExitNodes) || - (!routerset_equal(old_options->ExitNodes,options->ExitNodes)))) - COMPLAIN("StrictExitNodes set, but no ExitNodes listed."); - - if (options->StrictEntryNodes && - (!options->EntryNodes) && - (!old_options || - (old_options->StrictEntryNodes != options->StrictEntryNodes) || - (!routerset_equal(old_options->EntryNodes,options->EntryNodes)))) - COMPLAIN("StrictEntryNodes set, but no EntryNodes listed."); + if (options->ExcludeNodes && options->StrictNodes) { + COMPLAIN("You have asked to exclude certain relays from all positions " + "in your circuits. Expect hidden services and other Tor " + "features to be broken in unpredictable ways."); + } if (options->EntryNodes && !routerset_is_list(options->EntryNodes)) { /* XXXX fix this; see entry_guards_prepend_from_config(). */ @@ -3150,6 +3059,10 @@ options_validate(or_options_t *old_options, or_options_t *options, options->V3AuthoritativeDir)) REJECT("AuthoritativeDir is set, but none of " "(Bridge/HS/V1/V2/V3)AuthoritativeDir is set."); + /* If we have a v3bandwidthsfile and it's broken, complain on startup */ + if (options->V3BandwidthsFile && !old_options) { + dirserv_read_measured_bandwidths(options->V3BandwidthsFile, NULL); + } } if (options->AuthoritativeDir && !options->DirPort) @@ -3161,15 +3074,18 @@ options_validate(or_options_t *old_options, or_options_t *options, if (options->AuthoritativeDir && options->ClientOnly) REJECT("Running as authoritative directory, but ClientOnly also set."); - if (options->HSAuthorityRecordStats && !options->HSAuthoritativeDir) - REJECT("HSAuthorityRecordStats is set but we're not running as " - "a hidden service authority."); + if (options->FetchDirInfoExtraEarly && !options->FetchDirInfoEarly) + REJECT("FetchDirInfoExtraEarly requires that you also set " + "FetchDirInfoEarly"); + + if (options->HSAuthoritativeDir && proxy_mode(options)) + REJECT("Running as authoritative v0 HS directory, but also configured " + "as a client."); if (options->ConnLimit <= 0) { - r = tor_snprintf(buf, sizeof(buf), + tor_asprintf(msg, "ConnLimit must be greater than 0, but was set to %d", options->ConnLimit); - *msg = tor_strdup(r >= 0 ? buf : "internal error"); return -1; } @@ -3274,6 +3190,12 @@ options_validate(or_options_t *old_options, or_options_t *options, REJECT("Servers must be able to freely connect to the rest " "of the Internet, so they must not set UseBridges."); + /* If both of these are set, we'll end up with funny behavior where we + * demand enough entrynodes be up and running else we won't build + * circuits, yet we never actually use them. */ + if (options->UseBridges && options->EntryNodes) + REJECT("You cannot set both UseBridges and EntryNodes."); + options->_AllowInvalid = 0; if (options->AllowInvalidNodes) { SMARTLIST_FOREACH(options->AllowInvalidNodes, const char *, cp, { @@ -3288,18 +3210,29 @@ options_validate(or_options_t *old_options, or_options_t *options, else if (!strcasecmp(cp, "rendezvous")) options->_AllowInvalid |= ALLOW_INVALID_RENDEZVOUS; else { - r = tor_snprintf(buf, sizeof(buf), + tor_asprintf(msg, "Unrecognized value '%s' in AllowInvalidNodes", cp); - *msg = tor_strdup(r >= 0 ? buf : "internal error"); return -1; } }); } + if (!options->SafeLogging || + !strcasecmp(options->SafeLogging, "0")) { + options->_SafeLogging = SAFELOG_SCRUB_NONE; + } else if (!strcasecmp(options->SafeLogging, "relay")) { + options->_SafeLogging = SAFELOG_SCRUB_RELAY; + } else if (!strcasecmp(options->SafeLogging, "1")) { + options->_SafeLogging = SAFELOG_SCRUB_ALL; + } else { + tor_asprintf(msg, + "Unrecognized value '%s' in SafeLogging", + escaped(options->SafeLogging)); + return -1; + } + if (compute_publishserverdescriptor(options) < 0) { - r = tor_snprintf(buf, sizeof(buf), - "Unrecognized value in PublishServerDescriptor"); - *msg = tor_strdup(r >= 0 ? buf : "internal error"); + tor_asprintf(msg, "Unrecognized value in PublishServerDescriptor"); return -1; } @@ -3312,6 +3245,12 @@ options_validate(or_options_t *old_options, or_options_t *options, "PublishServerDescriptor line."); } + if (options->BridgeRelay && options->DirPort) { + log_warn(LD_CONFIG, "Can't set a DirPort on a bridge relay; disabling " + "DirPort"); + options->DirPort = 0; + } + if (options->MinUptimeHidServDirectoryV2 < 0) { log_warn(LD_CONFIG, "MinUptimeHidServDirectoryV2 option must be at " "least 0 seconds. Changing to 0."); @@ -3319,29 +3258,30 @@ options_validate(or_options_t *old_options, or_options_t *options, } if (options->RendPostPeriod < MIN_REND_POST_PERIOD) { - log(LOG_WARN,LD_CONFIG,"RendPostPeriod option is too short; " - "raising to %d seconds.", MIN_REND_POST_PERIOD); + log_warn(LD_CONFIG, "RendPostPeriod option is too short; " + "raising to %d seconds.", MIN_REND_POST_PERIOD); options->RendPostPeriod = MIN_REND_POST_PERIOD; } if (options->RendPostPeriod > MAX_DIR_PERIOD) { - log(LOG_WARN, LD_CONFIG, "RendPostPeriod is too large; clipping to %ds.", - MAX_DIR_PERIOD); + log_warn(LD_CONFIG, "RendPostPeriod is too large; clipping to %ds.", + MAX_DIR_PERIOD); options->RendPostPeriod = MAX_DIR_PERIOD; } - if (options->CircuitBuildTimeout < MIN_CIRCUIT_BUILD_TIMEOUT) { - log(LOG_WARN, LD_CONFIG, "CircuitBuildTimeout option is too short; " - "raising to %d seconds.", MIN_CIRCUIT_BUILD_TIMEOUT); - options->CircuitBuildTimeout = MIN_CIRCUIT_BUILD_TIMEOUT; - } - if (options->MaxCircuitDirtiness < MIN_MAX_CIRCUIT_DIRTINESS) { - log(LOG_WARN, LD_CONFIG, "MaxCircuitDirtiness option is too short; " - "raising to %d seconds.", MIN_MAX_CIRCUIT_DIRTINESS); + log_warn(LD_CONFIG, "MaxCircuitDirtiness option is too short; " + "raising to %d seconds.", MIN_MAX_CIRCUIT_DIRTINESS); options->MaxCircuitDirtiness = MIN_MAX_CIRCUIT_DIRTINESS; } + if (options->CircuitStreamTimeout && + options->CircuitStreamTimeout < MIN_CIRCUIT_STREAM_TIMEOUT) { + log_warn(LD_CONFIG, "CircuitStreamTimeout option is too short; " + "raising to %d seconds.", MIN_CIRCUIT_STREAM_TIMEOUT); + options->CircuitStreamTimeout = MIN_CIRCUIT_STREAM_TIMEOUT; + } + if (options->KeepalivePeriod < 1) REJECT("KeepalivePeriod option must be positive."); @@ -3360,6 +3300,12 @@ options_validate(or_options_t *old_options, or_options_t *options, if (ensure_bandwidth_cap(&options->RelayBandwidthBurst, "RelayBandwidthBurst", msg) < 0) return -1; + if (ensure_bandwidth_cap(&options->PerConnBWRate, + "PerConnBWRate", msg) < 0) + return -1; + if (ensure_bandwidth_cap(&options->PerConnBWBurst, + "PerConnBWBurst", msg) < 0) + return -1; if (options->RelayBandwidthRate && !options->RelayBandwidthBurst) options->RelayBandwidthBurst = options->RelayBandwidthRate; @@ -3368,31 +3314,28 @@ options_validate(or_options_t *old_options, or_options_t *options, if (server_mode(options)) { if (options->BandwidthRate < ROUTER_REQUIRED_MIN_BANDWIDTH) { - r = tor_snprintf(buf, sizeof(buf), + tor_asprintf(msg, "BandwidthRate is set to %d bytes/second. " "For servers, it must be at least %d.", (int)options->BandwidthRate, ROUTER_REQUIRED_MIN_BANDWIDTH); - *msg = tor_strdup(r >= 0 ? buf : "internal error"); return -1; } else if (options->MaxAdvertisedBandwidth < ROUTER_REQUIRED_MIN_BANDWIDTH/2) { - r = tor_snprintf(buf, sizeof(buf), + tor_asprintf(msg, "MaxAdvertisedBandwidth is set to %d bytes/second. " "For servers, it must be at least %d.", (int)options->MaxAdvertisedBandwidth, ROUTER_REQUIRED_MIN_BANDWIDTH/2); - *msg = tor_strdup(r >= 0 ? buf : "internal error"); return -1; } if (options->RelayBandwidthRate && options->RelayBandwidthRate < ROUTER_REQUIRED_MIN_BANDWIDTH) { - r = tor_snprintf(buf, sizeof(buf), + tor_asprintf(msg, "RelayBandwidthRate is set to %d bytes/second. " "For servers, it must be at least %d.", (int)options->RelayBandwidthRate, ROUTER_REQUIRED_MIN_BANDWIDTH); - *msg = tor_strdup(r >= 0 ? buf : "internal error"); return -1; } } @@ -3414,34 +3357,73 @@ options_validate(or_options_t *old_options, or_options_t *options, if (accounting_parse_options(options, 1)<0) REJECT("Failed to parse accounting options. See logs for details."); - if (options->HttpProxy) { /* parse it now */ - if (parse_addr_port(LOG_WARN, options->HttpProxy, NULL, - &options->HttpProxyAddr, &options->HttpProxyPort) < 0) - REJECT("HttpProxy failed to parse or resolve. Please fix."); - if (options->HttpProxyPort == 0) { /* give it a default */ - options->HttpProxyPort = 80; + if (options->HTTPProxy) { /* parse it now */ + if (tor_addr_port_parse(options->HTTPProxy, + &options->HTTPProxyAddr, &options->HTTPProxyPort) < 0) + REJECT("HTTPProxy failed to parse or resolve. Please fix."); + if (options->HTTPProxyPort == 0) { /* give it a default */ + options->HTTPProxyPort = 80; } } - if (options->HttpProxyAuthenticator) { - if (strlen(options->HttpProxyAuthenticator) >= 48) - REJECT("HttpProxyAuthenticator is too long (>= 48 chars)."); + if (options->HTTPProxyAuthenticator) { + if (strlen(options->HTTPProxyAuthenticator) >= 48) + REJECT("HTTPProxyAuthenticator is too long (>= 48 chars)."); } - if (options->HttpsProxy) { /* parse it now */ - if (parse_addr_port(LOG_WARN, options->HttpsProxy, NULL, - &options->HttpsProxyAddr, &options->HttpsProxyPort) <0) - REJECT("HttpsProxy failed to parse or resolve. Please fix."); - if (options->HttpsProxyPort == 0) { /* give it a default */ - options->HttpsProxyPort = 443; + if (options->HTTPSProxy) { /* parse it now */ + if (tor_addr_port_parse(options->HTTPSProxy, + &options->HTTPSProxyAddr, &options->HTTPSProxyPort) <0) + REJECT("HTTPSProxy failed to parse or resolve. Please fix."); + if (options->HTTPSProxyPort == 0) { /* give it a default */ + options->HTTPSProxyPort = 443; } } - if (options->HttpsProxyAuthenticator) { - if (strlen(options->HttpsProxyAuthenticator) >= 48) - REJECT("HttpsProxyAuthenticator is too long (>= 48 chars)."); + if (options->HTTPSProxyAuthenticator) { + if (strlen(options->HTTPSProxyAuthenticator) >= 48) + REJECT("HTTPSProxyAuthenticator is too long (>= 48 chars)."); + } + + if (options->Socks4Proxy) { /* parse it now */ + if (tor_addr_port_parse(options->Socks4Proxy, + &options->Socks4ProxyAddr, + &options->Socks4ProxyPort) <0) + REJECT("Socks4Proxy failed to parse or resolve. Please fix."); + if (options->Socks4ProxyPort == 0) { /* give it a default */ + options->Socks4ProxyPort = 1080; + } } + if (options->Socks5Proxy) { /* parse it now */ + if (tor_addr_port_parse(options->Socks5Proxy, + &options->Socks5ProxyAddr, + &options->Socks5ProxyPort) <0) + REJECT("Socks5Proxy failed to parse or resolve. Please fix."); + if (options->Socks5ProxyPort == 0) { /* give it a default */ + options->Socks5ProxyPort = 1080; + } + } + + if (options->Socks4Proxy && options->Socks5Proxy) + REJECT("You cannot specify both Socks4Proxy and SOCKS5Proxy"); + + if (options->Socks5ProxyUsername) { + size_t len; + + len = strlen(options->Socks5ProxyUsername); + if (len < 1 || len > 255) + REJECT("Socks5ProxyUsername must be between 1 and 255 characters."); + + if (!options->Socks5ProxyPassword) + REJECT("Socks5ProxyPassword must be included with Socks5ProxyUsername."); + + len = strlen(options->Socks5ProxyPassword); + if (len < 1 || len > 255) + REJECT("Socks5ProxyPassword must be between 1 and 255 characters."); + } else if (options->Socks5ProxyPassword) + REJECT("Socks5ProxyPassword must be included with Socks5ProxyUsername."); + if (options->HashedControlPassword) { smartlist_t *sl = decode_hashed_passwords(options->HashedControlPassword); if (!sl) { @@ -3505,6 +3487,13 @@ options_validate(or_options_t *old_options, or_options_t *options, "upgrade your Tor controller as soon as possible."); } + if (options->CookieAuthFileGroupReadable && !options->CookieAuthFile) { + log_warn(LD_CONFIG, "You set the CookieAuthFileGroupReadable but did " + "not configure a the path for the cookie file via " + "CookieAuthFile. This means your cookie will not be group " + "readable."); + } + if (options->UseEntryGuards && ! options->NumEntryGuards) REJECT("Cannot enable UseEntryGuards with NumEntryGuards set to 0"); @@ -3538,11 +3527,10 @@ options_validate(or_options_t *old_options, or_options_t *options, if (options->ConstrainedSockSize < MIN_CONSTRAINED_TCP_BUFFER || options->ConstrainedSockSize > MAX_CONSTRAINED_TCP_BUFFER || options->ConstrainedSockSize % 1024) { - r = tor_snprintf(buf, sizeof(buf), + tor_asprintf(msg, "ConstrainedSockSize is invalid. Must be a value between %d and %d " "in 1024 byte increments.", MIN_CONSTRAINED_TCP_BUFFER, MAX_CONSTRAINED_TCP_BUFFER); - *msg = tor_strdup(r >= 0 ? buf : "internal error"); return -1; } if (options->DirPort) { @@ -3590,6 +3578,12 @@ options_validate(or_options_t *old_options, or_options_t *options, if (options->PreferTunneledDirConns && !options->TunnelDirConns) REJECT("Must set TunnelDirConns if PreferTunneledDirConns is set."); + if ((options->Socks4Proxy || options->Socks5Proxy) && + !options->HTTPProxy && !options->PreferTunneledDirConns) + REJECT("When Socks4Proxy or Socks5Proxy is configured, " + "PreferTunneledDirConns and TunnelDirConns must both be " + "set to 1, or HTTPProxy must be configured."); + if (options->AutomapHostsSuffixes) { SMARTLIST_FOREACH(options->AutomapHostsSuffixes, char *, suf, { @@ -3604,12 +3598,19 @@ options_validate(or_options_t *old_options, or_options_t *options, "a non-default set of DirServers."); } - /*XXXX022 checking for defaults manually like this is a bit fragile.*/ + if (options->AllowSingleHopExits && !options->DirServers) { + COMPLAIN("You have set AllowSingleHopExits; now your relay will allow " + "others to make one-hop exits. However, since by default most " + "clients avoid relays that set this option, most clients will " + "ignore you."); + } + + /*XXXX023 checking for defaults manually like this is a bit fragile.*/ /* Keep changes to hard-coded values synchronous to man page and default * values table. */ if (options->TestingV3AuthInitialVotingInterval != 30*60 && - !options->TestingTorNetwork) { + !options->TestingTorNetwork && !options->_UsingTestNetworkDefaults) { REJECT("TestingV3AuthInitialVotingInterval may only be changed in testing " "Tor networks!"); } else if (options->TestingV3AuthInitialVotingInterval < MIN_VOTE_INTERVAL) { @@ -3620,7 +3621,8 @@ options_validate(or_options_t *old_options, or_options_t *options, } if (options->TestingV3AuthInitialVoteDelay != 5*60 && - !options->TestingTorNetwork) { + !options->TestingTorNetwork && !options->_UsingTestNetworkDefaults) { + REJECT("TestingV3AuthInitialVoteDelay may only be changed in testing " "Tor networks!"); } else if (options->TestingV3AuthInitialVoteDelay < MIN_VOTE_SECONDS) { @@ -3628,7 +3630,7 @@ options_validate(or_options_t *old_options, or_options_t *options, } if (options->TestingV3AuthInitialDistDelay != 5*60 && - !options->TestingTorNetwork) { + !options->TestingTorNetwork && !options->_UsingTestNetworkDefaults) { REJECT("TestingV3AuthInitialDistDelay may only be changed in testing " "Tor networks!"); } else if (options->TestingV3AuthInitialDistDelay < MIN_DIST_SECONDS) { @@ -3643,7 +3645,7 @@ options_validate(or_options_t *old_options, or_options_t *options, } if (options->TestingAuthDirTimeToLearnReachability != 30*60 && - !options->TestingTorNetwork) { + !options->TestingTorNetwork && !options->_UsingTestNetworkDefaults) { REJECT("TestingAuthDirTimeToLearnReachability may only be changed in " "testing Tor networks!"); } else if (options->TestingAuthDirTimeToLearnReachability < 0) { @@ -3653,7 +3655,7 @@ options_validate(or_options_t *old_options, or_options_t *options, } if (options->TestingEstimatedDescriptorPropagationTime != 10*60 && - !options->TestingTorNetwork) { + !options->TestingTorNetwork && !options->_UsingTestNetworkDefaults) { REJECT("TestingEstimatedDescriptorPropagationTime may only be changed in " "testing Tor networks!"); } else if (options->TestingEstimatedDescriptorPropagationTime < 0) { @@ -3669,6 +3671,26 @@ options_validate(or_options_t *old_options, or_options_t *options, "testing Tor network!"); } + if (options->AccelName && !options->HardwareAccel) + options->HardwareAccel = 1; + if (options->AccelDir && !options->AccelName) + REJECT("Can't use hardware crypto accelerator dir without engine name."); + + if (options->PublishServerDescriptor) + SMARTLIST_FOREACH(options->PublishServerDescriptor, const char *, pubdes, { + if (!strcmp(pubdes, "1") || !strcmp(pubdes, "0")) + if (smartlist_len(options->PublishServerDescriptor) > 1) { + COMPLAIN("You have passed a list of multiple arguments to the " + "PublishServerDescriptor option that includes 0 or 1. " + "0 or 1 should only be used as the sole argument. " + "This configuration will be rejected in a future release."); + break; + } + }); + + if (options->BridgeRelay == 1 && options->ORPort == 0) + REJECT("BridgeRelay is 1, ORPort is 0. This is an invalid combination."); + return 0; #undef REJECT #undef COMPLAIN @@ -3707,12 +3729,10 @@ options_transition_allowed(or_options_t *old, or_options_t *new_val, } if (strcmp(old->DataDirectory,new_val->DataDirectory)!=0) { - char buf[1024]; - int r = tor_snprintf(buf, sizeof(buf), + tor_asprintf(msg, "While Tor is running, changing DataDirectory " "(\"%s\"->\"%s\") is not allowed.", old->DataDirectory, new_val->DataDirectory); - *msg = tor_strdup(r >= 0 ? buf : "internal error"); return -1; } @@ -3721,19 +3741,22 @@ options_transition_allowed(or_options_t *old, or_options_t *new_val, return -1; } - if (!opt_streq(old->Group, new_val->Group)) { - *msg = tor_strdup("While Tor is running, changing Group is not allowed."); + if ((old->HardwareAccel != new_val->HardwareAccel) + || !opt_streq(old->AccelName, new_val->AccelName) + || !opt_streq(old->AccelDir, new_val->AccelDir)) { + *msg = tor_strdup("While Tor is running, changing OpenSSL hardware " + "acceleration engine is not allowed."); return -1; } - if (old->HardwareAccel != new_val->HardwareAccel) { - *msg = tor_strdup("While Tor is running, changing HardwareAccel is " - "not allowed."); + if (old->TestingTorNetwork != new_val->TestingTorNetwork) { + *msg = tor_strdup("While Tor is running, changing TestingTorNetwork " + "is not allowed."); return -1; } - if (old->TestingTorNetwork != new_val->TestingTorNetwork) { - *msg = tor_strdup("While Tor is running, changing TestingTorNetwork " + if (old->DisableAllSwap != new_val->DisableAllSwap) { + *msg = tor_strdup("While Tor is running, changing DisableAllSwap " "is not allowed."); return -1; } @@ -3748,13 +3771,15 @@ options_transition_affects_workers(or_options_t *old_options, or_options_t *new_options) { if (!opt_streq(old_options->DataDirectory, new_options->DataDirectory) || - old_options->NumCpus != new_options->NumCpus || + old_options->NumCPUs != new_options->NumCPUs || old_options->ORPort != new_options->ORPort || old_options->ServerDNSSearchDomains != new_options->ServerDNSSearchDomains || old_options->SafeLogging != new_options->SafeLogging || old_options->ClientOnly != new_options->ClientOnly || - !config_lines_eq(old_options->Logs, new_options->Logs)) + public_server_mode(old_options) != public_server_mode(new_options) || + !config_lines_eq(old_options->Logs, new_options->Logs) || + old_options->LogMessageDomains != new_options->LogMessageDomains) return 1; /* Check whether log options match. */ @@ -3780,7 +3805,6 @@ options_transition_affects_descriptor(or_options_t *old_options, old_options->ORPort != new_options->ORPort || old_options->DirPort != new_options->DirPort || old_options->ClientOnly != new_options->ClientOnly || - old_options->NoPublish != new_options->NoPublish || old_options->_PublishServerDescriptor != new_options->_PublishServerDescriptor || get_effective_bwrate(old_options) != get_effective_bwrate(new_options) || @@ -3803,6 +3827,7 @@ get_windows_conf_root(void) { static int is_set = 0; static char path[MAX_PATH+1]; + TCHAR tpath[MAX_PATH] = {0}; LPITEMIDLIST idl; IMalloc *m; @@ -3820,7 +3845,7 @@ get_windows_conf_root(void) #define APPDATA_PATH CSIDL_APPDATA #endif if (!SUCCEEDED(SHGetSpecialFolderLocation(NULL, APPDATA_PATH, &idl))) { - GetCurrentDirectory(MAX_PATH, path); + getcwd(path,MAX_PATH); is_set = 1; log_warn(LD_CONFIG, "I couldn't find your application data folder: are you " @@ -3829,8 +3854,15 @@ get_windows_conf_root(void) return path; } /* Convert the path from an "ID List" (whatever that is!) to a path. */ - result = SHGetPathFromIDList(idl, path); - /* Now we need to free the */ + result = SHGetPathFromIDList(idl, tpath); +#ifdef UNICODE + wcstombs(path,tpath,MAX_PATH); +#else + strlcpy(path,tpath,sizeof(path)); +#endif + + /* Now we need to free the memory that the path-idl was stored in. In + * typical Windows fashion, we can't just call 'free()' on it. */ SHGetMalloc(&m); if (m) { m->lpVtbl->Free(m, idl); @@ -3878,10 +3910,7 @@ check_nickname_list(const char *lst, const char *name, char **msg) SMARTLIST_FOREACH(sl, const char *, s, { if (!is_legal_nickname_or_hexdigest(s)) { - char buf[1024]; - int tmp = tor_snprintf(buf, sizeof(buf), - "Invalid nickname '%s' in %s line", s, name); - *msg = tor_strdup(tmp >= 0 ? buf : "internal error"); + tor_asprintf(msg, "Invalid nickname '%s' in %s line", s, name); r = -1; break; } @@ -3905,13 +3934,7 @@ find_torrc_filename(int argc, char **argv, log(LOG_WARN, LD_CONFIG, "Duplicate -f options on command line."); tor_free(fname); } -#ifdef MS_WINDOWS - /* XXX one day we might want to extend expand_filename to work - * under Windows as well. */ - fname = tor_strdup(argv[i+1]); -#else fname = expand_filename(argv[i+1]); -#endif *using_default_torrc = 0; ++i; } else if (!strcmp(argv[i],"--ignore-missing-torrc")) { @@ -4017,6 +4040,12 @@ options_init_from_torrc(int argc, char **argv) printf("Tor version %s.\n",get_version()); exit(0); } + if (argc > 1 && (!strcmp(argv[1],"--digests"))) { + printf("Tor version %s.\n",get_version()); + printf("%s", libor_get_digests()); + printf("%s", tor_get_digests()); + exit(0); + } /* Go through command-line variables */ if (!global_cmdline_options) { @@ -4184,12 +4213,9 @@ options_init_from_string(const char *cf, err: config_free(&options_format, newoptions); if (*msg) { - int len = (int)strlen(*msg)+256; - char *newmsg = tor_malloc(len); - - tor_snprintf(newmsg, len, "Failed to parse/validate config: %s", *msg); - tor_free(*msg); - *msg = newmsg; + char *old_msg = *msg; + tor_asprintf(msg, "Failed to parse/validate config: %s", old_msg); + tor_free(old_msg); } return err; } @@ -4309,11 +4335,13 @@ options_init_logs(or_options_t *options, int validate_only) if (smartlist_len(elts) == 2 && !strcasecmp(smartlist_get(elts,0), "file")) { if (!validate_only) { - if (add_file_log(severity, smartlist_get(elts, 1)) < 0) { + char *fname = expand_filename(smartlist_get(elts, 1)); + if (add_file_log(severity, fname) < 0) { log_warn(LD_CONFIG, "Couldn't open file for 'Log %s': %s", opt->value, strerror(errno)); ok = 0; } + tor_free(fname); } goto cleanup; } @@ -4329,6 +4357,9 @@ options_init_logs(or_options_t *options, int validate_only) } smartlist_free(elts); + if (ok && !validate_only) + logs_set_domain_logging(options->LogMessageDomains); + return ok?0:-1; } @@ -4577,7 +4608,7 @@ normalize_data_directory(or_options_t *options) } /** Check and normalize the value of options->DataDirectory; return 0 if it - * sane, -1 otherwise. */ + * is sane, -1 otherwise. */ static int validate_data_directory(or_options_t *options) { @@ -4609,7 +4640,6 @@ write_configuration_file(const char *fname, or_options_t *options) { char *old_val=NULL, *new_val=NULL, *new_conf=NULL; int rename_old = 0, r; - size_t len; tor_assert(fname); @@ -4636,9 +4666,7 @@ write_configuration_file(const char *fname, or_options_t *options) goto err; } - len = strlen(new_conf)+256; - new_val = tor_malloc(len); - tor_snprintf(new_val, len, "%s\n%s\n\n%s", + tor_asprintf(&new_val, "%s\n%s\n\n%s", GENERATED_FILE_PREFIX, GENERATED_FILE_COMMENT, new_conf); if (rename_old) { @@ -4688,22 +4716,19 @@ write_configuration_file(const char *fname, or_options_t *options) int options_save_current(void) { - if (torrc_fname) { - /* This fails if we can't write to our configuration file. - * - * If we try falling back to datadirectory or something, we have a better - * chance of saving the configuration, but a better chance of doing - * something the user never expected. Let's just warn instead. */ - return write_configuration_file(torrc_fname, get_options()); - } - return write_configuration_file(get_default_conf_file(), get_options()); + /* This fails if we can't write to our configuration file. + * + * If we try falling back to datadirectory or something, we have a better + * chance of saving the configuration, but a better chance of doing + * something the user never expected. */ + return write_configuration_file(get_torrc_fname(), get_options()); } /** Mapping from a unit name to a multiplier for converting that unit into a - * base unit. */ + * base unit. Used by config_parse_unit. */ struct unit_table_t { - const char *unit; - uint64_t multiplier; + const char *unit; /**< The name of the unit */ + uint64_t multiplier; /**< How many of the base unit appear in this unit */ }; /** Table to map the names of memory units to the number of bytes they @@ -4761,30 +4786,47 @@ static struct unit_table_t time_units[] = { static uint64_t config_parse_units(const char *val, struct unit_table_t *u, int *ok) { - uint64_t v; + uint64_t v = 0; + double d = 0; + int use_float = 0; char *cp; tor_assert(ok); v = tor_parse_uint64(val, 10, 0, UINT64_MAX, ok, &cp); - if (!*ok) - return 0; + if (!*ok || (cp && *cp == '.')) { + d = tor_parse_double(val, 0, UINT64_MAX, ok, &cp); + if (!*ok) + goto done; + use_float = 1; + } + if (!cp) { *ok = 1; - return v; + v = use_float ? DBL_TO_U64(d) : v; + goto done; } - while (TOR_ISSPACE(*cp)) - ++cp; + + cp = (char*) eat_whitespace(cp); + for ( ;u->unit;++u) { if (!strcasecmp(u->unit, cp)) { - v *= u->multiplier; + if (use_float) + v = u->multiplier * d; + else + v *= u->multiplier; *ok = 1; - return v; + goto done; } } log_warn(LD_CONFIG, "Unknown unit '%s'.", cp); *ok = 0; - return 0; + done: + + if (*ok) + return v; + else + return 0; } /** Parse a string in the format "number unit", where unit is a unit of @@ -4794,7 +4836,8 @@ config_parse_units(const char *val, struct unit_table_t *u, int *ok) static uint64_t config_parse_memunit(const char *s, int *ok) { - return config_parse_units(s, memory_units, ok); + uint64_t u = config_parse_units(s, memory_units, ok); + return u; } /** Parse a string in the format "number unit", where unit is a unit of time. @@ -4816,256 +4859,37 @@ config_parse_interval(const char *s, int *ok) return (int)r; } -/* This is what passes for version detection on OSX. We set - * MACOSX_KQUEUE_IS_BROKEN to true iff we're on a version of OSX before - * 10.4.0 (aka 1040). */ -#ifdef __APPLE__ -#ifdef __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ -#define MACOSX_KQUEUE_IS_BROKEN \ - (__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 1040) -#else -#define MACOSX_KQUEUE_IS_BROKEN 0 -#endif -#endif - /** * Initialize the libevent library. */ static void init_libevent(void) { + const char *badness=NULL; + configure_libevent_logging(); /* If the kernel complains that some method (say, epoll) doesn't * exist, we don't care about it, since libevent will cope. */ suppress_libevent_log_msg("Function not implemented"); -#ifdef __APPLE__ - if (MACOSX_KQUEUE_IS_BROKEN || - decode_libevent_version(event_get_version(), NULL) < LE_11B) { - setenv("EVENT_NOKQUEUE","1",1); - } -#endif - /* In libevent versions before 2.0, it's hard to keep binary compatibility - * between upgrades, and unpleasant to detect when the version we compiled - * against is unlike the version we have linked against. Here's how. */ -#if defined(_EVENT_VERSION) && defined(HAVE_EVENT_GET_VERSION) - /* We have a header-file version and a function-call version. Easy. */ - if (strcmp(_EVENT_VERSION, event_get_version())) { - int compat1 = -1, compat2 = -1; - int verybad, prettybad ; - decode_libevent_version(_EVENT_VERSION, &compat1); - decode_libevent_version(event_get_version(), &compat2); - verybad = compat1 != compat2; - prettybad = (compat1 == -1 || compat2 == -1) && compat1 != compat2; - - log(verybad ? LOG_WARN : (prettybad ? LOG_NOTICE : LOG_INFO), - LD_GENERAL, "We were compiled with headers from version %s " - "of Libevent, but we're using a Libevent library that says it's " - "version %s.", _EVENT_VERSION, event_get_version()); - if (verybad) - log_warn(LD_GENERAL, "This will almost certainly make Tor crash."); - else if (prettybad) - log_notice(LD_GENERAL, "If Tor crashes, this might be why."); - else - log_info(LD_GENERAL, "I think these versions are binary-compatible."); - } -#elif defined(HAVE_EVENT_GET_VERSION) - /* event_get_version but no _EVENT_VERSION. We might be in 1.4.0-beta or - earlier, where that's normal. To see whether we were compiled with an - earlier version, let's see whether the struct event defines MIN_HEAP_IDX. - */ -#ifdef HAVE_STRUCT_EVENT_MIN_HEAP_IDX - /* The header files are 1.4.0-beta or later. If the version is not - * 1.4.0-beta, we are incompatible. */ - { - if (strcmp(event_get_version(), "1.4.0-beta")) { - log_warn(LD_GENERAL, "It's a little hard to tell, but you seem to have " - "Libevent 1.4.0-beta header files, whereas you have linked " - "against Libevent %s. This will probably make Tor crash.", - event_get_version()); - } - } -#else - /* Our headers are 1.3e or earlier. If the library version is not 1.4.x or - later, we're probably fine. */ - { - const char *v = event_get_version(); - if ((v[0] == '1' && v[2] == '.' && v[3] > '3') || v[0] > '1') { - log_warn(LD_GENERAL, "It's a little hard to tell, but you seem to have " - "Libevent header file from 1.3e or earlier, whereas you have " - "linked against Libevent %s. This will probably make Tor " - "crash.", event_get_version()); - } - } -#endif + tor_check_libevent_header_compatibility(); -#elif defined(_EVENT_VERSION) -#warn "_EVENT_VERSION is defined but not get_event_version(): Libevent is odd." -#else - /* Your libevent is ancient. */ -#endif + tor_libevent_initialize(); - event_init(); suppress_libevent_log_msg(NULL); -#if defined(HAVE_EVENT_GET_VERSION) && defined(HAVE_EVENT_GET_METHOD) - /* Making this a NOTICE for now so we can link bugs to a libevent versions - * or methods better. */ - log(LOG_NOTICE, LD_GENERAL, - "Initialized libevent version %s using method %s. Good.", - event_get_version(), event_get_method()); - check_libevent_version(event_get_method(), get_options()->ORPort != 0); -#else - log(LOG_NOTICE, LD_GENERAL, - "Initialized old libevent (version 1.0b or earlier)."); - log(LOG_WARN, LD_GENERAL, - "You have a *VERY* old version of libevent. It is likely to be buggy; " - "please build Tor with a more recent version."); -#endif -} - -/** Table mapping return value of event_get_version() to le_version_t. */ -static const struct { - const char *name; le_version_t version; int bincompat; -} le_version_table[] = { - /* earlier versions don't have get_version. */ - { "1.0c", LE_10C, 1}, - { "1.0d", LE_10D, 1}, - { "1.0e", LE_10E, 1}, - { "1.1", LE_11, 1 }, - { "1.1a", LE_11A, 1 }, - { "1.1b", LE_11B, 1 }, - { "1.2", LE_12, 1 }, - { "1.2a", LE_12A, 1 }, - { "1.3", LE_13, 1 }, - { "1.3a", LE_13A, 1 }, - { "1.3b", LE_13B, 1 }, - { "1.3c", LE_13C, 1 }, - { "1.3d", LE_13D, 1 }, - { "1.3e", LE_13E, 1 }, - { "1.4.0-beta", LE_140, 2 }, - { "1.4.1-beta", LE_141, 2 }, - { "1.4.2-rc", LE_142, 2 }, - { "1.4.3-stable", LE_143, 2 }, - { "1.4.4-stable", LE_144, 2 }, - { "1.4.5-stable", LE_145, 2 }, - { "1.4.6-stable", LE_146, 2 }, - { "1.4.7-stable", LE_147, 2 }, - { "1.4.8-stable", LE_148, 2 }, - { "1.4.99-trunk", LE_1499, 3 }, - { NULL, LE_OTHER, 0 } -}; -/** Return the le_version_t for the current version of libevent. If the - * version is very new, return LE_OTHER. If the version is so old that it - * doesn't support event_get_version(), return LE_OLD. */ -static le_version_t -decode_libevent_version(const char *v, int *bincompat_out) -{ - int i; - for (i=0; le_version_table[i].name; ++i) { - if (!strcmp(le_version_table[i].name, v)) { - if (bincompat_out) - *bincompat_out = le_version_table[i].bincompat; - return le_version_table[i].version; - } - } - if (v[0] != '1' && bincompat_out) - *bincompat_out = 100; - else if (!strcmpstart(v, "1.4") && bincompat_out) - *bincompat_out = 2; - return LE_OTHER; -} - -#if defined(HAVE_EVENT_GET_VERSION) && defined(HAVE_EVENT_GET_METHOD) -/** - * Compare the given libevent method and version to a list of versions - * which are known not to work. Warn the user as appropriate. - */ -static void -check_libevent_version(const char *m, int server) -{ - int buggy = 0, iffy = 0, slow = 0, thread_unsafe = 0; - le_version_t version; - const char *v = event_get_version(); - const char *badness = NULL; - const char *sad_os = ""; - - version = decode_libevent_version(v, NULL); - - /* XXX Would it be worthwhile disabling the methods that we know - * are buggy, rather than just warning about them and then proceeding - * to use them? If so, we should probably not wrap this whole thing - * in HAVE_EVENT_GET_VERSION and HAVE_EVENT_GET_METHOD. -RD */ - /* XXXX The problem is that it's not trivial to get libevent to change it's - * method once it's initialized, and it's not trivial to tell what method it - * will use without initializing it. I guess we could preemptively disable - * buggy libevent modes based on the version _before_ initializing it, - * though, but then there's no good way (afaict) to warn "I would have used - * kqueue, but instead I'm using select." -NM */ - if (!strcmp(m, "kqueue")) { - if (version < LE_11B) - buggy = 1; - } else if (!strcmp(m, "epoll")) { - if (version < LE_11) - iffy = 1; - } else if (!strcmp(m, "poll")) { - if (version < LE_10E) - buggy = 1; - else if (version < LE_11) - slow = 1; - } else if (!strcmp(m, "select")) { - if (version < LE_11) - slow = 1; - } else if (!strcmp(m, "win32")) { - if (version < LE_11B) - buggy = 1; - } - - /* Libevent versions before 1.3b do very badly on operating systems with - * user-space threading implementations. */ -#if defined(__OpenBSD__) || defined(__FreeBSD__) || defined(__NetBSD__) - if (server && version < LE_13B) { - thread_unsafe = 1; - sad_os = "BSD variants"; - } -#elif defined(__APPLE__) || defined(__darwin__) - if (server && version < LE_13B) { - thread_unsafe = 1; - sad_os = "Mac OS X"; - } -#endif - - if (thread_unsafe) { - log(LOG_WARN, LD_GENERAL, - "Libevent version %s often crashes when running a Tor server with %s. " - "Please use the latest version of libevent (1.3b or later)",v,sad_os); - badness = "BROKEN"; - } else if (buggy) { - log(LOG_WARN, LD_GENERAL, - "There are serious bugs in using %s with libevent %s. " - "Please use the latest version of libevent.", m, v); - badness = "BROKEN"; - } else if (iffy) { - log(LOG_WARN, LD_GENERAL, - "There are minor bugs in using %s with libevent %s. " - "You may want to use the latest version of libevent.", m, v); - badness = "BUGGY"; - } else if (slow && server) { - log(LOG_WARN, LD_GENERAL, - "libevent %s can be very slow with %s. " - "When running a server, please use the latest version of libevent.", - v,m); - badness = "SLOW"; - } + tor_check_libevent_version(tor_libevent_get_method(), + get_options()->ORPort != 0, + &badness); if (badness) { + const char *v = tor_libevent_get_version_str(); + const char *m = tor_libevent_get_method(); control_event_general_status(LOG_WARN, "BAD_LIBEVENT VERSION=%s METHOD=%s BADNESS=%s RECOVERED=NO", v, m, badness); } - } -#endif /** Return the persistent state struct for this Tor. */ or_state_t * @@ -5147,22 +4971,61 @@ or_state_validate(or_state_t *old_state, or_state_t *state, } /** Replace the current persistent state with <b>new_state</b> */ -static void +static int or_state_set(or_state_t *new_state) { char *err = NULL; + int ret = 0; tor_assert(new_state); - if (global_state) - config_free(&state_format, global_state); + config_free(&state_format, global_state); global_state = new_state; if (entry_guards_parse_state(global_state, 1, &err)<0) { log_warn(LD_GENERAL,"%s",err); tor_free(err); + ret = -1; } if (rep_hist_load_state(global_state, &err)<0) { log_warn(LD_GENERAL,"Unparseable bandwidth history state: %s",err); tor_free(err); + ret = -1; + } + if (circuit_build_times_parse_state(&circ_times, global_state) < 0) { + ret = -1; + } + return ret; +} + +/** + * Save a broken state file to a backup location. + */ +static void +or_state_save_broken(char *fname) +{ + int i; + file_status_t status; + size_t len = strlen(fname)+16; + char *fname2 = tor_malloc(len); + for (i = 0; i < 100; ++i) { + tor_snprintf(fname2, len, "%s.%d", fname, i); + status = file_status(fname2); + if (status == FN_NOENT) + break; + } + if (i == 100) { + log_warn(LD_BUG, "Unable to parse state in \"%s\"; too many saved bad " + "state files to move aside. Discarding the old state file.", + fname); + unlink(fname); + } else { + log_warn(LD_BUG, "Unable to parse state in \"%s\". Moving it aside " + "to \"%s\". This could be a bug in Tor; please tell " + "the developers.", fname, fname2); + if (rename(fname, fname2) < 0) { + log_warn(LD_BUG, "Weirdly, I couldn't even move the state aside. The " + "OS gave an error of %s", strerror(errno)); + } } + tor_free(fname2); } /** Reload the persistent state from disk, generating a new state as needed. @@ -5224,31 +5087,8 @@ or_state_load(void) " This is a bug in Tor."); goto done; } else if (badstate && contents) { - int i; - file_status_t status; - size_t len = strlen(fname)+16; - char *fname2 = tor_malloc(len); - for (i = 0; i < 100; ++i) { - tor_snprintf(fname2, len, "%s.%d", fname, i); - status = file_status(fname2); - if (status == FN_NOENT) - break; - } - if (i == 100) { - log_warn(LD_BUG, "Unable to parse state in \"%s\"; too many saved bad " - "state files to move aside. Discarding the old state file.", - fname); - unlink(fname); - } else { - log_warn(LD_BUG, "Unable to parse state in \"%s\". Moving it aside " - "to \"%s\". This could be a bug in Tor; please tell " - "the developers.", fname, fname2); - if (rename(fname, fname2) < 0) { - log_warn(LD_BUG, "Weirdly, I couldn't even move the state aside. The " - "OS gave an error of %s", strerror(errno)); - } - } - tor_free(fname2); + or_state_save_broken(fname); + tor_free(contents); config_free(&state_format, new_state); @@ -5260,7 +5100,9 @@ or_state_load(void) } else { log_info(LD_GENERAL, "Initialized state"); } - or_state_set(new_state); + if (or_state_set(new_state) == -1) { + or_state_save_broken(fname); + } new_state = NULL; if (!contents) { global_state->next_write = 0; @@ -5277,13 +5119,15 @@ or_state_load(void) return r; } +/** If writing the state to disk fails, try again after this many seconds. */ +#define STATE_WRITE_RETRY_INTERVAL 3600 + /** Write the persistent state to disk. Return 0 for success, <0 on failure. */ int or_state_save(time_t now) { char *state, *contents; char tbuf[ISO_TIME_LEN+1]; - size_t len; char *fname; tor_assert(global_state); @@ -5295,20 +5139,16 @@ or_state_save(time_t now) * to avoid redundant writes. */ entry_guards_update_state(global_state); rep_hist_update_state(global_state); + circuit_build_times_update_state(&circ_times, global_state); if (accounting_is_enabled(get_options())) accounting_run_housekeeping(now); - global_state->LastWritten = time(NULL); tor_free(global_state->TorVersion); - len = strlen(get_version())+8; - global_state->TorVersion = tor_malloc(len); - tor_snprintf(global_state->TorVersion, len, "Tor %s", get_version()); + tor_asprintf(&global_state->TorVersion, "Tor %s", get_version()); state = config_dump(&state_format, global_state, 1, 0); - len = strlen(state)+256; - contents = tor_malloc(len); format_local_iso_time(tbuf, time(NULL)); - tor_snprintf(contents, len, + tor_asprintf(&contents, "# Tor state file last generated on %s local time\n" "# Other times below are in GMT\n" "# You *do not* need to edit this file.\n\n%s", @@ -5318,10 +5158,16 @@ or_state_save(time_t now) if (write_str_to_file(fname, contents, 0)<0) { log_warn(LD_FS, "Unable to write state to file \"%s\"; " "will try again later", fname); + global_state->LastWritten = -1; tor_free(fname); tor_free(contents); + /* Try again after STATE_WRITE_RETRY_INTERVAL (or sooner, if the state + * changes sooner). */ + global_state->next_write = now + STATE_WRITE_RETRY_INTERVAL; return -1; } + + global_state->LastWritten = time(NULL); log_info(LD_GENERAL, "Saved state to \"%s\"", fname); tor_free(fname); tor_free(contents); @@ -5353,18 +5199,18 @@ remove_file_if_very_old(const char *fname, time_t now) * types. */ int getinfo_helper_config(control_connection_t *conn, - const char *question, char **answer) + const char *question, char **answer, + const char **errmsg) { (void) conn; + (void) errmsg; if (!strcmp(question, "config/names")) { smartlist_t *sl = smartlist_create(); int i; for (i = 0; _option_vars[i].name; ++i) { config_var_t *var = &_option_vars[i]; - const char *type, *desc; + const char *type; char *line; - size_t len; - desc = config_find_description(&options_format, var->name); switch (var->type) { case CONFIG_TYPE_STRING: type = "String"; break; case CONFIG_TYPE_FILENAME: type = "Filename"; break; @@ -5385,14 +5231,7 @@ getinfo_helper_config(control_connection_t *conn, } if (!type) continue; - len = strlen(var->name)+strlen(type)+16; - if (desc) - len += strlen(desc); - line = tor_malloc(len); - if (desc) - tor_snprintf(line, len, "%s %s %s\n",var->name,type,desc); - else - tor_snprintf(line, len, "%s %s\n",var->name,type); + tor_asprintf(&line, "%s %s\n",var->name,type); smartlist_add(sl, line); } *answer = smartlist_join_strings(sl, "", 0, NULL); diff --git a/src/or/config.h b/src/or/config.h new file mode 100644 index 0000000000..defda35e0b --- /dev/null +++ b/src/or/config.h @@ -0,0 +1,78 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file config.h + * \brief Header file for config.c. + **/ + +#ifndef _TOR_CONFIG_H +#define _TOR_CONFIG_H + +const char *get_dirportfrontpage(void); +or_options_t *get_options(void); +int set_options(or_options_t *new_val, char **msg); +void config_free_all(void); +const char *safe_str_client(const char *address); +const char *safe_str(const char *address); +const char *escaped_safe_str_client(const char *address); +const char *escaped_safe_str(const char *address); +const char *get_version(void); + +int config_get_lines(const char *string, config_line_t **result); +void config_free_lines(config_line_t *front); +setopt_err_t options_trial_assign(config_line_t *list, int use_defaults, + int clear_first, char **msg); +int resolve_my_address(int warn_severity, or_options_t *options, + uint32_t *addr, char **hostname_out); +int is_local_addr(const tor_addr_t *addr) ATTR_PURE; +void options_init(or_options_t *options); +char *options_dump(or_options_t *options, int minimal); +int options_init_from_torrc(int argc, char **argv); +setopt_err_t options_init_from_string(const char *cf, + int command, const char *command_arg, char **msg); +int option_is_recognized(const char *key); +const char *option_get_canonical_name(const char *key); +config_line_t *option_get_assignment(or_options_t *options, + const char *key); +int options_save_current(void); +const char *get_torrc_fname(void); +char *options_get_datadir_fname2_suffix(or_options_t *options, + const char *sub1, const char *sub2, + const char *suffix); +#define get_datadir_fname2_suffix(sub1, sub2, suffix) \ + options_get_datadir_fname2_suffix(get_options(), (sub1), (sub2), (suffix)) +/** Return a newly allocated string containing datadir/sub1. See + * get_datadir_fname2_suffix. */ +#define get_datadir_fname(sub1) get_datadir_fname2_suffix((sub1), NULL, NULL) +/** Return a newly allocated string containing datadir/sub1/sub2. See + * get_datadir_fname2_suffix. */ +#define get_datadir_fname2(sub1,sub2) \ + get_datadir_fname2_suffix((sub1), (sub2), NULL) +/** Return a newly allocated string containing datadir/sub1suffix. See + * get_datadir_fname2_suffix. */ +#define get_datadir_fname_suffix(sub1, suffix) \ + get_datadir_fname2_suffix((sub1), NULL, (suffix)) + +or_state_t *get_or_state(void); +int or_state_save(time_t now); + +int options_need_geoip_info(or_options_t *options, const char **reason_out); +int getinfo_helper_config(control_connection_t *conn, + const char *question, char **answer, + const char **errmsg); + +const char *tor_get_digests(void); +uint32_t get_effective_bwrate(or_options_t *options); +uint32_t get_effective_bwburst(or_options_t *options); + +#ifdef CONFIG_PRIVATE +/* Used only by config.c and test.c */ +or_options_t *options_new(void); +#endif + +#endif + diff --git a/src/or/config_codedigest.c b/src/or/config_codedigest.c new file mode 100644 index 0000000000..be9eaa331d --- /dev/null +++ b/src/or/config_codedigest.c @@ -0,0 +1,11 @@ + +const char *tor_get_digests(void); + +const char * +tor_get_digests(void) +{ + return "" +#include "or_sha1.i" + ; +} + diff --git a/src/or/connection.c b/src/or/connection.c index 4869a2439a..6e7bbd5bad 100644 --- a/src/or/connection.c +++ b/src/or/connection.c @@ -11,6 +11,30 @@ **/ #include "or.h" +#include "buffers.h" +#include "circuitbuild.h" +#include "circuitlist.h" +#include "circuituse.h" +#include "config.h" +#include "connection.h" +#include "connection_edge.h" +#include "connection_or.h" +#include "control.h" +#include "cpuworker.h" +#include "directory.h" +#include "dirserv.h" +#include "dns.h" +#include "dnsserv.h" +#include "geoip.h" +#include "main.h" +#include "policies.h" +#include "reasons.h" +#include "relay.h" +#include "rendclient.h" +#include "rendcommon.h" +#include "rephist.h" +#include "router.h" +#include "routerparse.h" static connection_t *connection_create_listener( struct sockaddr *listensockaddr, @@ -21,17 +45,22 @@ static void connection_init(time_t now, connection_t *conn, int type, static int connection_init_accepted_conn(connection_t *conn, uint8_t listener_type); static int connection_handle_listener_read(connection_t *conn, int new_type); -static int connection_read_bucket_should_increase(or_connection_t *conn); +static int connection_bucket_should_increase(int bucket, + or_connection_t *conn); static int connection_finished_flushing(connection_t *conn); static int connection_flushed_some(connection_t *conn); static int connection_finished_connecting(connection_t *conn); static int connection_reached_eof(connection_t *conn); -static int connection_read_to_buf(connection_t *conn, int *max_to_read, +static int connection_read_to_buf(connection_t *conn, ssize_t *max_to_read, int *socket_error); static int connection_process_inbuf(connection_t *conn, int package_partial); static void client_check_address_changed(int sock); static void set_constrained_socket_buffers(int sock, int size); +static const char *connection_proxy_state_to_string(int state); +static int connection_read_https_proxy_response(connection_t *conn); +static void connection_send_socks5_connect(connection_t *conn); + /** The last IPv4 address that our network interface seemed to have been * binding to, in host order. We use this to detect when our IP changes. */ static uint32_t last_interface_ip = 0; @@ -92,8 +121,7 @@ conn_state_to_string(int type, int state) case CONN_TYPE_OR: switch (state) { case OR_CONN_STATE_CONNECTING: return "connect()ing"; - case OR_CONN_STATE_PROXY_FLUSHING: return "proxy flushing"; - case OR_CONN_STATE_PROXY_READING: return "proxy reading"; + case OR_CONN_STATE_PROXY_HANDSHAKING: return "handshaking (proxy)"; case OR_CONN_STATE_TLS_HANDSHAKING: return "handshaking (TLS)"; case OR_CONN_STATE_TLS_CLIENT_RENEGOTIATING: return "renegotiating (TLS)"; @@ -177,6 +205,9 @@ or_connection_new(int socket_family) or_conn->timestamp_last_added_nonpadding = time(NULL); or_conn->next_circ_id = crypto_rand_int(1<<15); + or_conn->active_circuit_pqueue = smartlist_create(); + or_conn->active_circuit_pqueue_last_recalibrated = cell_ewma_get_tick(); + return or_conn; } @@ -202,6 +233,7 @@ control_connection_new(int socket_family) tor_malloc_zero(sizeof(control_connection_t)); connection_init(time(NULL), TO_CONN(control_conn), CONN_TYPE_CONTROL, socket_family); + log_notice(LD_CONTROL, "New control connection opened."); return control_conn; } @@ -299,25 +331,6 @@ connection_link_connections(connection_t *conn_a, connection_t *conn_b) conn_b->linked_conn = conn_a; } -/** Tell libevent that we don't care about <b>conn</b> any more. */ -void -connection_unregister_events(connection_t *conn) -{ - if (conn->read_event) { - if (event_del(conn->read_event)) - log_warn(LD_BUG, "Error removing read event for %d", conn->s); - tor_free(conn->read_event); - } - if (conn->write_event) { - if (event_del(conn->write_event)) - log_warn(LD_BUG, "Error removing write event for %d", conn->s); - tor_free(conn->write_event); - } - if (conn->dns_server_port) { - dnsserv_close_listener(conn); - } -} - /** Deallocate memory used by <b>conn</b>. Deallocate its buffers if * necessary, close its socket if necessary, and mark the directory as dirty * if <b>conn</b> is an OR or OP connection. @@ -327,6 +340,9 @@ _connection_free(connection_t *conn) { void *mem; size_t memlen; + if (!conn) + return; + switch (conn->type) { case CONN_TYPE_OR: tor_assert(conn->magic == OR_CONNECTION_MAGIC); @@ -384,14 +400,11 @@ _connection_free(connection_t *conn) if (connection_speaks_cells(conn)) { or_connection_t *or_conn = TO_OR_CONN(conn); - if (or_conn->tls) { - tor_tls_free(or_conn->tls); - or_conn->tls = NULL; - } - if (or_conn->handshake_state) { - or_handshake_state_free(or_conn->handshake_state); - or_conn->handshake_state = NULL; - } + tor_tls_free(or_conn->tls); + or_conn->tls = NULL; + or_handshake_state_free(or_conn->handshake_state); + or_conn->handshake_state = NULL; + smartlist_free(or_conn->active_circuit_pqueue); tor_free(or_conn->nickname); } if (CONN_IS_EDGE(conn)) { @@ -401,8 +414,8 @@ _connection_free(connection_t *conn) memset(edge_conn->socks_request, 0xcc, sizeof(socks_request_t)); tor_free(edge_conn->socks_request); } - if (edge_conn->rend_data) - rend_data_free(edge_conn->rend_data); + + rend_data_free(edge_conn->rend_data); } if (conn->type == CONN_TYPE_CONTROL) { control_connection_t *control_conn = TO_CONTROL_CONN(conn); @@ -415,16 +428,15 @@ _connection_free(connection_t *conn) if (conn->type == CONN_TYPE_DIR) { dir_connection_t *dir_conn = TO_DIR_CONN(conn); tor_free(dir_conn->requested_resource); - if (dir_conn->zlib_state) - tor_zlib_free(dir_conn->zlib_state); + + tor_zlib_free(dir_conn->zlib_state); if (dir_conn->fingerprint_stack) { SMARTLIST_FOREACH(dir_conn->fingerprint_stack, char *, cp, tor_free(cp)); smartlist_free(dir_conn->fingerprint_stack); } - if (dir_conn->cached_dir) - cached_dir_decref(dir_conn->cached_dir); - if (dir_conn->rend_data) - rend_data_free(dir_conn->rend_data); + + cached_dir_decref(dir_conn->cached_dir); + rend_data_free(dir_conn->rend_data); } if (conn->s >= 0) { @@ -439,7 +451,7 @@ _connection_free(connection_t *conn) connection_or_remove_from_identity_map(TO_OR_CONN(conn)); } - memset(conn, 0xAA, memlen); /* poison memory */ + memset(mem, 0xCC, memlen); /* poison memory */ tor_free(mem); } @@ -448,7 +460,8 @@ _connection_free(connection_t *conn) void connection_free(connection_t *conn) { - tor_assert(conn); + if (!conn) + return; tor_assert(!connection_is_on_closeable_list(conn)); tor_assert(!connection_in_array(conn)); if (conn->linked_conn) { @@ -544,13 +557,6 @@ connection_about_to_close_connection(connection_t *conn) * failed: forget about this router, and maybe try again. */ connection_dir_request_failed(dir_conn); } - if (conn->purpose == DIR_PURPOSE_FETCH_RENDDESC && dir_conn->rend_data) { - /* Give it a try. However, there is no re-fetching for v0 rend - * descriptors; if the response is empty or the descriptor is - * unusable, close pending connections (unless a v2 request is - * still in progress). */ - rend_client_desc_trynow(dir_conn->rend_data->onion_address, 0); - } /* If we were trying to fetch a v2 rend desc and did not succeed, * retry as needed. (If a fetch is successful, the connection state * is changed to DIR_PURPOSE_HAS_FETCHED_RENDDESC to mark that @@ -573,7 +579,7 @@ connection_about_to_close_connection(connection_t *conn) or_options_t *options = get_options(); rep_hist_note_connect_failed(or_conn->identity_digest, now); entry_guard_register_connect_status(or_conn->identity_digest,0, - !options->HttpsProxy, now); + !options->HTTPSProxy, now); if (conn->state >= OR_CONN_STATE_TLS_HANDSHAKING) { int reason = tls_error_to_orconn_end_reason(or_conn->tls_error); control_event_or_conn_status(or_conn, OR_CONN_EVENT_FAILED, @@ -589,7 +595,7 @@ connection_about_to_close_connection(connection_t *conn) rep_hist_note_disconnect(or_conn->identity_digest, now); control_event_or_conn_status(or_conn, OR_CONN_EVENT_CLOSED, tls_error_to_orconn_end_reason(or_conn->tls_error)); - } else if (or_conn->identity_digest) { + } else if (!tor_digest_is_zero(or_conn->identity_digest)) { rep_hist_note_connection_died(or_conn->identity_digest, now); control_event_or_conn_status(or_conn, OR_CONN_EVENT_CLOSED, tls_error_to_orconn_end_reason(or_conn->tls_error)); @@ -833,16 +839,16 @@ static void warn_too_many_conns(void) { #define WARN_TOO_MANY_CONNS_INTERVAL (6*60*60) - static time_t last_warned = 0; - time_t now = time(NULL); - int n_conns = get_n_open_sockets(); - if (last_warned + WARN_TOO_MANY_CONNS_INTERVAL < now) { + static ratelim_t last_warned = RATELIM_INIT(WARN_TOO_MANY_CONNS_INTERVAL); + char *m; + if ((m = rate_limit_log(&last_warned, approx_time()))) { + int n_conns = get_n_open_sockets(); log_warn(LD_NET,"Failing because we have %d connections already. Please " - "raise your ulimit -n.", n_conns); - last_warned = now; + "raise your ulimit -n.%s", n_conns, m); + tor_free(m); + control_event_general_status(LOG_WARN, "TOO_MANY_CONNECTIONS CURRENT=%d", + n_conns); } - control_event_general_status(LOG_WARN, "TOO_MANY_CONNECTIONS CURRENT=%d", - n_conns); } /** Bind a new non-blocking socket listening to the socket described @@ -1172,7 +1178,8 @@ connection_handle_listener_read(connection_t *conn, int new_type) } if (connection_init_accepted_conn(newconn, conn->type) < 0) { - connection_mark_for_close(newconn); + if (! newconn->marked_for_close) + connection_mark_for_close(newconn); return 0; } return 0; @@ -1198,9 +1205,11 @@ connection_init_accepted_conn(connection_t *conn, uint8_t listener_type) conn->state = AP_CONN_STATE_SOCKS_WAIT; break; case CONN_TYPE_AP_TRANS_LISTENER: + TO_EDGE_CONN(conn)->is_transparent_ap = 1; conn->state = AP_CONN_STATE_CIRCUIT_WAIT; return connection_ap_process_transparent(TO_EDGE_CONN(conn)); case CONN_TYPE_AP_NATD_LISTENER: + TO_EDGE_CONN(conn)->is_transparent_ap = 1; conn->state = AP_CONN_STATE_NATD_WAIT; break; } @@ -1254,7 +1263,7 @@ connection_connect(connection_t *conn, const char *address, return -1; } - if (options->OutboundBindAddress) { + if (options->OutboundBindAddress && !tor_addr_is_loopback(addr)) { struct sockaddr_in ext_addr; memset(&ext_addr, 0, sizeof(ext_addr)); @@ -1285,7 +1294,8 @@ connection_connect(connection_t *conn, const char *address, dest_addr_len = tor_addr_to_sockaddr(addr, port, dest_addr, sizeof(addrbuf)); tor_assert(dest_addr_len > 0); - log_debug(LD_NET,"Connecting to %s:%u.",escaped_safe_str(address),port); + log_debug(LD_NET, "Connecting to %s:%u.", + escaped_safe_str_client(address), port); if (connect(s, dest_addr, dest_addr_len) < 0) { int e = tor_socket_errno(s); @@ -1293,7 +1303,8 @@ connection_connect(connection_t *conn, const char *address, /* yuck. kill it. */ *socket_error = e; log_info(LD_NET, - "connect() to %s:%u failed: %s",escaped_safe_str(address), + "connect() to %s:%u failed: %s", + escaped_safe_str_client(address), port, tor_socket_strerror(e)); tor_close_socket(s); return -1; @@ -1307,7 +1318,8 @@ connection_connect(connection_t *conn, const char *address, /* it succeeded. we're connected. */ log_fn(inprogress?LOG_DEBUG:LOG_INFO, LD_NET, - "Connection to %s:%u %s (sock %d).",escaped_safe_str(address), + "Connection to %s:%u %s (sock %d).", + escaped_safe_str_client(address), port, inprogress?"in progress":"established", s); conn->s = s; if (connection_add(conn) < 0) /* no space, forget it */ @@ -1315,6 +1327,353 @@ connection_connect(connection_t *conn, const char *address, return inprogress ? 0 : 1; } +/** Convert state number to string representation for logging purposes. + */ +static const char * +connection_proxy_state_to_string(int state) +{ + static const char *unknown = "???"; + static const char *states[] = { + "PROXY_NONE", + "PROXY_HTTPS_WANT_CONNECT_OK", + "PROXY_SOCKS4_WANT_CONNECT_OK", + "PROXY_SOCKS5_WANT_AUTH_METHOD_NONE", + "PROXY_SOCKS5_WANT_AUTH_METHOD_RFC1929", + "PROXY_SOCKS5_WANT_AUTH_RFC1929_OK", + "PROXY_SOCKS5_WANT_CONNECT_OK", + "PROXY_CONNECTED", + }; + + if (state < PROXY_NONE || state > PROXY_CONNECTED) + return unknown; + + return states[state]; +} + +/** Write a proxy request of <b>type</b> (socks4, socks5, https) to conn + * for conn->addr:conn->port, authenticating with the auth details given + * in the configuration (if available). SOCKS 5 and HTTP CONNECT proxies + * support authentication. + * + * Returns -1 if conn->addr is incompatible with the proxy protocol, and + * 0 otherwise. + * + * Use connection_read_proxy_handshake() to complete the handshake. + */ +int +connection_proxy_connect(connection_t *conn, int type) +{ + or_options_t *options; + + tor_assert(conn); + + options = get_options(); + + switch (type) { + case PROXY_CONNECT: { + char buf[1024]; + char *base64_authenticator=NULL; + const char *authenticator = options->HTTPSProxyAuthenticator; + + /* Send HTTP CONNECT and authentication (if available) in + * one request */ + + if (authenticator) { + base64_authenticator = alloc_http_authenticator(authenticator); + if (!base64_authenticator) + log_warn(LD_OR, "Encoding https authenticator failed"); + } + + if (base64_authenticator) { + tor_snprintf(buf, sizeof(buf), "CONNECT %s:%d HTTP/1.1\r\n" + "Proxy-Authorization: Basic %s\r\n\r\n", + fmt_addr(&conn->addr), + conn->port, base64_authenticator); + tor_free(base64_authenticator); + } else { + tor_snprintf(buf, sizeof(buf), "CONNECT %s:%d HTTP/1.0\r\n\r\n", + fmt_addr(&conn->addr), conn->port); + } + + connection_write_to_buf(buf, strlen(buf), conn); + conn->proxy_state = PROXY_HTTPS_WANT_CONNECT_OK; + break; + } + + case PROXY_SOCKS4: { + unsigned char buf[9]; + uint16_t portn; + uint32_t ip4addr; + + /* Send a SOCKS4 connect request with empty user id */ + + if (tor_addr_family(&conn->addr) != AF_INET) { + log_warn(LD_NET, "SOCKS4 client is incompatible with IPv6"); + return -1; + } + + ip4addr = tor_addr_to_ipv4n(&conn->addr); + portn = htons(conn->port); + + buf[0] = 4; /* version */ + buf[1] = SOCKS_COMMAND_CONNECT; /* command */ + memcpy(buf + 2, &portn, 2); /* port */ + memcpy(buf + 4, &ip4addr, 4); /* addr */ + buf[8] = 0; /* userid (empty) */ + + connection_write_to_buf((char *)buf, sizeof(buf), conn); + conn->proxy_state = PROXY_SOCKS4_WANT_CONNECT_OK; + break; + } + + case PROXY_SOCKS5: { + unsigned char buf[4]; /* fields: vers, num methods, method list */ + + /* Send a SOCKS5 greeting (connect request must wait) */ + + buf[0] = 5; /* version */ + + /* number of auth methods */ + if (options->Socks5ProxyUsername) { + buf[1] = 2; + buf[2] = 0x00; /* no authentication */ + buf[3] = 0x02; /* rfc1929 Username/Passwd auth */ + conn->proxy_state = PROXY_SOCKS5_WANT_AUTH_METHOD_RFC1929; + } else { + buf[1] = 1; + buf[2] = 0x00; /* no authentication */ + conn->proxy_state = PROXY_SOCKS5_WANT_AUTH_METHOD_NONE; + } + + connection_write_to_buf((char *)buf, 2 + buf[1], conn); + break; + } + + default: + log_err(LD_BUG, "Invalid proxy protocol, %d", type); + tor_fragile_assert(); + return -1; + } + + log_debug(LD_NET, "set state %s", + connection_proxy_state_to_string(conn->proxy_state)); + + return 0; +} + +/** Read conn's inbuf. If the http response from the proxy is all + * here, make sure it's good news, then return 1. If it's bad news, + * return -1. Else return 0 and hope for better luck next time. + */ +static int +connection_read_https_proxy_response(connection_t *conn) +{ + char *headers; + char *reason=NULL; + int status_code; + time_t date_header; + + switch (fetch_from_buf_http(conn->inbuf, + &headers, MAX_HEADERS_SIZE, + NULL, NULL, 10000, 0)) { + case -1: /* overflow */ + log_warn(LD_PROTOCOL, + "Your https proxy sent back an oversized response. Closing."); + return -1; + case 0: + log_info(LD_NET,"https proxy response not all here yet. Waiting."); + return 0; + /* case 1, fall through */ + } + + if (parse_http_response(headers, &status_code, &date_header, + NULL, &reason) < 0) { + log_warn(LD_NET, + "Unparseable headers from proxy (connecting to '%s'). Closing.", + conn->address); + tor_free(headers); + return -1; + } + if (!reason) reason = tor_strdup("[no reason given]"); + + if (status_code == 200) { + log_info(LD_NET, + "HTTPS connect to '%s' successful! (200 %s) Starting TLS.", + conn->address, escaped(reason)); + tor_free(reason); + return 1; + } + /* else, bad news on the status code */ + log_warn(LD_NET, + "The https proxy sent back an unexpected status code %d (%s). " + "Closing.", + status_code, escaped(reason)); + tor_free(reason); + return -1; +} + +/** Send SOCKS5 CONNECT command to <b>conn</b>, copying <b>conn->addr</b> + * and <b>conn->port</b> into the request. + */ +static void +connection_send_socks5_connect(connection_t *conn) +{ + unsigned char buf[1024]; + size_t reqsize = 6; + uint16_t port = htons(conn->port); + + buf[0] = 5; /* version */ + buf[1] = SOCKS_COMMAND_CONNECT; /* command */ + buf[2] = 0; /* reserved */ + + if (tor_addr_family(&conn->addr) == AF_INET) { + uint32_t addr = tor_addr_to_ipv4n(&conn->addr); + + buf[3] = 1; + reqsize += 4; + memcpy(buf + 4, &addr, 4); + memcpy(buf + 8, &port, 2); + } else { /* AF_INET6 */ + buf[3] = 4; + reqsize += 16; + memcpy(buf + 4, tor_addr_to_in6(&conn->addr), 16); + memcpy(buf + 20, &port, 2); + } + + connection_write_to_buf((char *)buf, reqsize, conn); + + conn->proxy_state = PROXY_SOCKS5_WANT_CONNECT_OK; +} + +/** Call this from connection_*_process_inbuf() to advance the proxy + * handshake. + * + * No matter what proxy protocol is used, if this function returns 1, the + * handshake is complete, and the data remaining on inbuf may contain the + * start of the communication with the requested server. + * + * Returns 0 if the current buffer contains an incomplete response, and -1 + * on error. + */ +int +connection_read_proxy_handshake(connection_t *conn) +{ + int ret = 0; + char *reason = NULL; + + log_debug(LD_NET, "enter state %s", + connection_proxy_state_to_string(conn->proxy_state)); + + switch (conn->proxy_state) { + case PROXY_HTTPS_WANT_CONNECT_OK: + ret = connection_read_https_proxy_response(conn); + if (ret == 1) + conn->proxy_state = PROXY_CONNECTED; + break; + + case PROXY_SOCKS4_WANT_CONNECT_OK: + ret = fetch_from_buf_socks_client(conn->inbuf, + conn->proxy_state, + &reason); + if (ret == 1) + conn->proxy_state = PROXY_CONNECTED; + break; + + case PROXY_SOCKS5_WANT_AUTH_METHOD_NONE: + ret = fetch_from_buf_socks_client(conn->inbuf, + conn->proxy_state, + &reason); + /* no auth needed, do connect */ + if (ret == 1) { + connection_send_socks5_connect(conn); + ret = 0; + } + break; + + case PROXY_SOCKS5_WANT_AUTH_METHOD_RFC1929: + ret = fetch_from_buf_socks_client(conn->inbuf, + conn->proxy_state, + &reason); + + /* send auth if needed, otherwise do connect */ + if (ret == 1) { + connection_send_socks5_connect(conn); + ret = 0; + } else if (ret == 2) { + unsigned char buf[1024]; + size_t reqsize, usize, psize; + const char *user, *pass; + + user = get_options()->Socks5ProxyUsername; + pass = get_options()->Socks5ProxyPassword; + tor_assert(user && pass); + + /* XXX len of user and pass must be <= 255 !!! */ + usize = strlen(user); + psize = strlen(pass); + tor_assert(usize <= 255 && psize <= 255); + reqsize = 3 + usize + psize; + + buf[0] = 1; /* negotiation version */ + buf[1] = usize; + memcpy(buf + 2, user, usize); + buf[2 + usize] = psize; + memcpy(buf + 3 + usize, pass, psize); + + connection_write_to_buf((char *)buf, reqsize, conn); + + conn->proxy_state = PROXY_SOCKS5_WANT_AUTH_RFC1929_OK; + ret = 0; + } + break; + + case PROXY_SOCKS5_WANT_AUTH_RFC1929_OK: + ret = fetch_from_buf_socks_client(conn->inbuf, + conn->proxy_state, + &reason); + /* send the connect request */ + if (ret == 1) { + connection_send_socks5_connect(conn); + ret = 0; + } + break; + + case PROXY_SOCKS5_WANT_CONNECT_OK: + ret = fetch_from_buf_socks_client(conn->inbuf, + conn->proxy_state, + &reason); + if (ret == 1) + conn->proxy_state = PROXY_CONNECTED; + break; + + default: + log_err(LD_BUG, "Invalid proxy_state for reading, %d", + conn->proxy_state); + tor_fragile_assert(); + ret = -1; + break; + } + + log_debug(LD_NET, "leaving state %s", + connection_proxy_state_to_string(conn->proxy_state)); + + if (ret < 0) { + if (reason) { + log_warn(LD_NET, "Proxy Client: unable to connect to %s:%d (%s)", + conn->address, conn->port, escaped(reason)); + tor_free(reason); + } else { + log_warn(LD_NET, "Proxy Client: unable to connect to %s:%d", + conn->address, conn->port); + } + } else if (ret == 1) { + log_info(LD_NET, "Proxy Client: connection to %s:%d successful", + conn->address, conn->port); + } + + return ret; +} + /** * Launch any configured listener connections of type <b>type</b>. (A * listener is configured if <b>port_option</b> is non-zero. If any @@ -1506,8 +1865,8 @@ retry_all_listeners(smartlist_t *replaced_conns, replaced_conns, new_conns, 0, AF_INET)<0) return -1; - if (retry_listeners(CONN_TYPE_AP_NATD_LISTENER, options->NatdListenAddress, - options->NatdPort, "127.0.0.1", + if (retry_listeners(CONN_TYPE_AP_NATD_LISTENER, options->NATDListenAddress, + options->NATDPort, "127.0.0.1", replaced_conns, new_conns, 0, AF_INET)<0) return -1; @@ -1643,6 +2002,7 @@ connection_bucket_write_limit(connection_t *conn, time_t now) int base = connection_speaks_cells(conn) ? CELL_NETWORK_SIZE : RELAY_PAYLOAD_SIZE; int priority = conn->type != CONN_TYPE_DIR; + int conn_bucket = (int)conn->outbuf_flushlen; int global_bucket = global_write_bucket; if (!connection_is_rate_limited(conn)) { @@ -1650,12 +2010,22 @@ connection_bucket_write_limit(connection_t *conn, time_t now) return conn->outbuf_flushlen; } + if (connection_speaks_cells(conn)) { + /* use the per-conn write limit if it's lower, but if it's less + * than zero just use zero */ + or_connection_t *or_conn = TO_OR_CONN(conn); + if (conn->state == OR_CONN_STATE_OPEN) + if (or_conn->write_bucket < conn_bucket) + conn_bucket = or_conn->write_bucket >= 0 ? + or_conn->write_bucket : 0; + } + if (connection_counts_as_relayed_traffic(conn, now) && global_relayed_write_bucket <= global_write_bucket) global_bucket = global_relayed_write_bucket; - return connection_bucket_round_robin(base, priority, global_bucket, - conn->outbuf_flushlen); + return connection_bucket_round_robin(base, priority, + global_bucket, conn_bucket); } /** Return 1 if the global write buckets are low enough that we @@ -1709,14 +2079,12 @@ global_write_bucket_low(connection_t *conn, size_t attempt, int priority) return 0; } -/** We just read num_read and wrote num_written onto conn. - * Decrement buckets appropriately. */ +/** We just read <b>num_read</b> and wrote <b>num_written</b> bytes + * onto <b>conn</b>. Decrement buckets appropriately. */ static void connection_buckets_decrement(connection_t *conn, time_t now, size_t num_read, size_t num_written) { - if (!connection_is_rate_limited(conn)) - return; /* local IPs are free */ if (num_written >= INT_MAX || num_read >= INT_MAX) { log_err(LD_BUG, "Value out of range. num_read=%lu, num_written=%lu, " "connection type=%s, state=%s", @@ -1728,10 +2096,24 @@ connection_buckets_decrement(connection_t *conn, time_t now, tor_fragile_assert(); } - if (num_read > 0) + /* Count bytes of answering direct and tunneled directory requests */ + if (conn->type == CONN_TYPE_DIR && conn->purpose == DIR_PURPOSE_SERVER) { + if (num_read > 0) + rep_hist_note_dir_bytes_read(num_read, now); + if (num_written > 0) + rep_hist_note_dir_bytes_written(num_written, now); + } + + if (!connection_is_rate_limited(conn)) + return; /* local IPs are free */ + if (num_read > 0) { rep_hist_note_bytes_read(num_read, now); - if (num_written > 0) + } + if (num_written > 0) { rep_hist_note_bytes_written(num_written, now); + } + if (conn->type == CONN_TYPE_EXIT) + rep_hist_note_exit_bytes(conn->port, num_written, num_read); if (connection_counts_as_relayed_traffic(conn, now)) { global_relayed_read_bucket -= (int)num_read; @@ -1739,8 +2121,10 @@ connection_buckets_decrement(connection_t *conn, time_t now, } global_read_bucket -= (int)num_read; global_write_bucket -= (int)num_written; - if (connection_speaks_cells(conn) && conn->state == OR_CONN_STATE_OPEN) + if (connection_speaks_cells(conn) && conn->state == OR_CONN_STATE_OPEN) { TO_OR_CONN(conn)->read_bucket -= (int)num_read; + TO_OR_CONN(conn)->write_bucket -= (int)num_written; + } } /** If we have exhausted our global buckets, or the buckets for conn, @@ -1779,12 +2163,10 @@ connection_consider_empty_write_buckets(connection_t *conn) } else if (connection_counts_as_relayed_traffic(conn, approx_time()) && global_relayed_write_bucket <= 0) { reason = "global relayed write bucket exhausted. Pausing."; -#if 0 } else if (connection_speaks_cells(conn) && conn->state == OR_CONN_STATE_OPEN && TO_OR_CONN(conn)->write_bucket <= 0) { reason = "connection write bucket exhausted. Pausing."; -#endif } else return; /* all good, no need to stop it */ @@ -1880,14 +2262,19 @@ connection_bucket_refill(int seconds_elapsed, time_t now) { if (connection_speaks_cells(conn)) { or_connection_t *or_conn = TO_OR_CONN(conn); - if (connection_read_bucket_should_increase(or_conn)) { + if (connection_bucket_should_increase(or_conn->read_bucket, or_conn)) { connection_bucket_refill_helper(&or_conn->read_bucket, or_conn->bandwidthrate, or_conn->bandwidthburst, seconds_elapsed, "or_conn->read_bucket"); - //log_fn(LOG_DEBUG,"Receiver bucket %d now %d.", i, - // conn->read_bucket); + } + if (connection_bucket_should_increase(or_conn->write_bucket, or_conn)) { + connection_bucket_refill_helper(&or_conn->write_bucket, + or_conn->bandwidthrate, + or_conn->bandwidthburst, + seconds_elapsed, + "or_conn->write_bucket"); } } @@ -1908,8 +2295,10 @@ connection_bucket_refill(int seconds_elapsed, time_t now) if (conn->write_blocked_on_bw == 1 && global_write_bucket > 0 /* and we're allowed to write */ && (!connection_counts_as_relayed_traffic(conn, now) || - global_relayed_write_bucket > 0)) { - /* even if we're relayed traffic */ + global_relayed_write_bucket > 0) /* even if it's relayed traffic */ + && (!connection_speaks_cells(conn) || + conn->state != OR_CONN_STATE_OPEN || + TO_OR_CONN(conn)->write_bucket > 0)) { LOG_FN_CONN(conn, (LOG_DEBUG,LD_NET, "waking up conn (fd %d) for write", conn->s)); conn->write_blocked_on_bw = 0; @@ -1918,17 +2307,17 @@ connection_bucket_refill(int seconds_elapsed, time_t now) }); } -/** Is the receiver bucket for connection <b>conn</b> low enough that we +/** Is the <b>bucket</b> for connection <b>conn</b> low enough that we * should add another pile of tokens to it? */ static int -connection_read_bucket_should_increase(or_connection_t *conn) +connection_bucket_should_increase(int bucket, or_connection_t *conn) { tor_assert(conn); if (conn->_base.state != OR_CONN_STATE_OPEN) return 0; /* only open connections play the rate limiting game */ - if (conn->read_bucket >= conn->bandwidthburst) + if (bucket >= conn->bandwidthburst) return 0; return 1; @@ -1946,10 +2335,10 @@ connection_read_bucket_should_increase(or_connection_t *conn) * Mark the connection and return -1 if you want to close it, else * return 0. */ -int -connection_handle_read(connection_t *conn) +static int +connection_handle_read_impl(connection_t *conn) { - int max_to_read=-1, try_to_read; + ssize_t max_to_read=-1, try_to_read; size_t before, n_read = 0; int socket_error = 0; @@ -1975,7 +2364,7 @@ connection_handle_read(connection_t *conn) return 0; } -loop_again: + loop_again: try_to_read = max_to_read; tor_assert(!conn->marked_for_close); @@ -2016,12 +2405,16 @@ loop_again: return -1; } if (conn->linked_conn) { - /* The other side's handle_write will never actually get called, so + /* The other side's handle_write() will never actually get called, so * we need to invoke the appropriate callbacks ourself. */ connection_t *linked = conn->linked_conn; if (n_read) { - /* Probably a no-op, but hey. */ + /* Probably a no-op, since linked conns typically don't count for + * bandwidth rate limiting. But do it anyway so we can keep stats + * accurately. Note that since we read the bytes from conn, and + * we're writing the bytes onto the linked connection, we count + * these as <i>written</i> bytes. */ connection_buckets_decrement(linked, approx_time(), 0, n_read); if (connection_flushed_some(linked) < 0) @@ -2033,7 +2426,7 @@ loop_again: if (!buf_datalen(linked->outbuf) && conn->active_on_link) connection_stop_reading_from_linked_conn(conn); } - /* If we hit the EOF, call connection_reached_eof. */ + /* If we hit the EOF, call connection_reached_eof(). */ if (!conn->marked_for_close && conn->inbuf_reached_eof && connection_reached_eof(conn) < 0) { @@ -2042,6 +2435,16 @@ loop_again: return 0; } +int +connection_handle_read(connection_t *conn) +{ + int res; + + tor_gettimeofday_cache_clear(); + res = connection_handle_read_impl(conn); + return res; +} + /** Pull in new bytes from conn-\>s or conn-\>linked_conn onto conn-\>inbuf, * either directly or via TLS. Reduce the token buckets by the number of bytes * read. @@ -2053,7 +2456,8 @@ loop_again: * Return -1 if we want to break conn, else return 0. */ static int -connection_read_to_buf(connection_t *conn, int *max_to_read, int *socket_error) +connection_read_to_buf(connection_t *conn, ssize_t *max_to_read, + int *socket_error) { int result; ssize_t at_most = *max_to_read; @@ -2075,7 +2479,7 @@ connection_read_to_buf(connection_t *conn, int *max_to_read, int *socket_error) } if (connection_speaks_cells(conn) && - conn->state > OR_CONN_STATE_PROXY_READING) { + conn->state > OR_CONN_STATE_PROXY_HANDSHAKING) { int pending; or_connection_t *or_conn = TO_OR_CONN(conn); size_t initial_size; @@ -2171,15 +2575,19 @@ connection_read_to_buf(connection_t *conn, int *max_to_read, int *socket_error) n_read = (size_t) result; } - if (n_read > 0) { /* change *max_to_read */ - /*XXXX021 check for overflow*/ - *max_to_read = (int)(at_most - n_read); - } + if (n_read > 0) { + /* change *max_to_read */ + *max_to_read = at_most - n_read; - if (conn->type == CONN_TYPE_AP) { - edge_connection_t *edge_conn = TO_EDGE_CONN(conn); - /*XXXX021 check for overflow*/ - edge_conn->n_read += (int)n_read; + /* Update edge_conn->n_read */ + if (conn->type == CONN_TYPE_AP) { + edge_connection_t *edge_conn = TO_EDGE_CONN(conn); + /* Check for overflow: */ + if (PREDICT_LIKELY(UINT32_MAX - edge_conn->n_read > n_read)) + edge_conn->n_read += (int)n_read; + else + edge_conn->n_read = UINT32_MAX; + } } connection_buckets_decrement(conn, approx_time(), n_read, n_written); @@ -2243,8 +2651,8 @@ connection_outbuf_too_full(connection_t *conn) * Mark the connection and return -1 if you want to close it, else * return 0. */ -int -connection_handle_write(connection_t *conn, int force) +static int +connection_handle_write_impl(connection_t *conn, int force) { int e; socklen_t len=(socklen_t)sizeof(e); @@ -2259,7 +2667,7 @@ connection_handle_write(connection_t *conn, int force) return 0; /* do nothing */ if (conn->in_flushed_some) { - log_warn(LD_BUG, "called recursively from inside conn->in_flushed_some()"); + log_warn(LD_BUG, "called recursively from inside conn->in_flushed_some"); return 0; } @@ -2303,7 +2711,7 @@ connection_handle_write(connection_t *conn, int force) : connection_bucket_write_limit(conn, now); if (connection_speaks_cells(conn) && - conn->state > OR_CONN_STATE_PROXY_READING) { + conn->state > OR_CONN_STATE_PROXY_HANDSHAKING) { or_connection_t *or_conn = TO_OR_CONN(conn); if (conn->state == OR_CONN_STATE_TLS_HANDSHAKING || conn->state == OR_CONN_STATE_TLS_CLIENT_RENEGOTIATING) { @@ -2322,6 +2730,13 @@ connection_handle_write(connection_t *conn, int force) /* else open, or closing */ result = flush_buf_tls(or_conn->tls, conn->outbuf, max_to_write, &conn->outbuf_flushlen); + + /* If we just flushed the last bytes, check if this tunneled dir + * request is done. */ + if (buf_datalen(conn->outbuf) == 0 && conn->dirreq_id) + geoip_change_dirreq_state(conn->dirreq_id, DIRREQ_TUNNELED, + DIRREQ_OR_CONN_BUFFER_FLUSHED); + switch (result) { CASE_TOR_TLS_ERROR_ANY: case TOR_TLS_CLOSE: @@ -2341,8 +2756,8 @@ connection_handle_write(connection_t *conn, int force) if (!connection_is_reading(conn)) { connection_stop_writing(conn); conn->write_blocked_on_bw = 1; - /* we'll start reading again when the next second arrives, - * and then also start writing again. + /* we'll start reading again when we get more tokens in our + * read bucket; then we'll start writing again too. */ } /* else no problem, we're already reading */ @@ -2371,10 +2786,13 @@ connection_handle_write(connection_t *conn, int force) n_written = (size_t) result; } - if (conn->type == CONN_TYPE_AP) { + if (n_written && conn->type == CONN_TYPE_AP) { edge_connection_t *edge_conn = TO_EDGE_CONN(conn); - /*XXXX021 check for overflow.*/ - edge_conn->n_written += (int)n_written; + /* Check for overflow: */ + if (PREDICT_LIKELY(UINT32_MAX - edge_conn->n_written > n_written)) + edge_conn->n_written += (int)n_written; + else + edge_conn->n_written = UINT32_MAX; } connection_buckets_decrement(conn, approx_time(), n_read, n_written); @@ -2404,6 +2822,15 @@ connection_handle_write(connection_t *conn, int force) return 0; } +int +connection_handle_write(connection_t *conn, int force) +{ + int res; + tor_gettimeofday_cache_clear(); + res = connection_handle_write_impl(conn, force); + return res; +} + /** OpenSSL TLS record size is 16383; this is close. The goal here is to * push data out as soon as we know there's enough for a TLS record, so * during periods of high load we won't read entire megabytes from @@ -2577,13 +3004,11 @@ connection_get_by_type_state(int type, int state) /** Return a connection of type <b>type</b> that has rendquery equal * to <b>rendquery</b>, and that is not marked for close. If state - * is non-zero, conn must be of that state too. If rendversion is - * nonnegative, conn must be fetching that rendversion, too. + * is non-zero, conn must be of that state too. */ connection_t * connection_get_by_type_state_rendquery(int type, int state, - const char *rendquery, - int rendversion) + const char *rendquery) { smartlist_t *conns = get_connection_array(); @@ -2598,8 +3023,6 @@ connection_get_by_type_state_rendquery(int type, int state, (!state || state == conn->state)) { if (type == CONN_TYPE_DIR && TO_DIR_CONN(conn)->rend_data && - (rendversion < 0 || - rendversion == TO_DIR_CONN(conn)->rend_data->rend_desc_version) && !rend_cmp_service_ids(rendquery, TO_DIR_CONN(conn)->rend_data->onion_address)) return conn; @@ -2717,10 +3140,10 @@ alloc_http_authenticator(const char *authenticator) static void client_check_address_changed(int sock) { - uint32_t iface_ip, ip_out; + uint32_t iface_ip, ip_out; /* host order */ struct sockaddr_in out_addr; socklen_t out_addr_len = (socklen_t) sizeof(out_addr); - uint32_t *ip; + uint32_t *ip; /* host order */ if (!last_interface_ip) get_interface_address(LOG_INFO, &last_interface_ip); @@ -2734,7 +3157,7 @@ client_check_address_changed(int sock) return; } - /* Okay. If we've used this address previously, we're okay. */ + /* If we've used this address previously, we're okay. */ ip_out = ntohl(out_addr.sin_addr.s_addr); SMARTLIST_FOREACH(outgoing_addrs, uint32_t*, ip_ptr, if (*ip_ptr == ip_out) return; @@ -2830,6 +3253,8 @@ connection_flushed_some(connection_t *conn) r = connection_dirserv_flushed_some(TO_DIR_CONN(conn)); } else if (conn->type == CONN_TYPE_OR) { r = connection_or_flushed_some(TO_OR_CONN(conn)); + } else if (CONN_IS_EDGE(conn)) { + r = connection_edge_flushed_some(TO_EDGE_CONN(conn)); } conn->in_flushed_some = 0; return r; @@ -3033,7 +3458,7 @@ assert_connection_ok(connection_t *conn, time_t now) } // tor_assert(conn->addr && conn->port); tor_assert(conn->address); - if (conn->state > OR_CONN_STATE_PROXY_READING) + if (conn->state > OR_CONN_STATE_PROXY_HANDSHAKING) tor_assert(or_conn->tls); } diff --git a/src/or/connection.h b/src/or/connection.h new file mode 100644 index 0000000000..576d3a63e1 --- /dev/null +++ b/src/or/connection.h @@ -0,0 +1,100 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file connection.h + * \brief Header file for connection.c. + **/ + +#ifndef _TOR_CONNECTION_H +#define _TOR_CONNECTION_H + +const char *conn_type_to_string(int type); +const char *conn_state_to_string(int type, int state); + +dir_connection_t *dir_connection_new(int socket_family); +or_connection_t *or_connection_new(int socket_family); +edge_connection_t *edge_connection_new(int type, int socket_family); +control_connection_t *control_connection_new(int socket_family); +connection_t *connection_new(int type, int socket_family); + +void connection_link_connections(connection_t *conn_a, connection_t *conn_b); +void connection_free(connection_t *conn); +void connection_free_all(void); +void connection_about_to_close_connection(connection_t *conn); +void connection_close_immediate(connection_t *conn); +void _connection_mark_for_close(connection_t *conn,int line, const char *file); + +#define connection_mark_for_close(c) \ + _connection_mark_for_close((c), __LINE__, _SHORT_FILE_) + +void connection_expire_held_open(void); + +int connection_connect(connection_t *conn, const char *address, + const tor_addr_t *addr, + uint16_t port, int *socket_error); + +int connection_proxy_connect(connection_t *conn, int type); +int connection_read_proxy_handshake(connection_t *conn); + +int retry_all_listeners(smartlist_t *replaced_conns, + smartlist_t *new_conns); + +ssize_t connection_bucket_write_limit(connection_t *conn, time_t now); +int global_write_bucket_low(connection_t *conn, size_t attempt, int priority); +void connection_bucket_init(void); +void connection_bucket_refill(int seconds_elapsed, time_t now); + +int connection_handle_read(connection_t *conn); + +int connection_fetch_from_buf(char *string, size_t len, connection_t *conn); + +int connection_wants_to_flush(connection_t *conn); +int connection_outbuf_too_full(connection_t *conn); +int connection_handle_write(connection_t *conn, int force); +void _connection_write_to_buf_impl(const char *string, size_t len, + connection_t *conn, int zlib); +static void connection_write_to_buf(const char *string, size_t len, + connection_t *conn); +static void connection_write_to_buf_zlib(const char *string, size_t len, + dir_connection_t *conn, int done); +static INLINE void +connection_write_to_buf(const char *string, size_t len, connection_t *conn) +{ + _connection_write_to_buf_impl(string, len, conn, 0); +} +static INLINE void +connection_write_to_buf_zlib(const char *string, size_t len, + dir_connection_t *conn, int done) +{ + _connection_write_to_buf_impl(string, len, TO_CONN(conn), done ? -1 : 1); +} + +connection_t *connection_get_by_global_id(uint64_t id); + +connection_t *connection_get_by_type(int type); +connection_t *connection_get_by_type_purpose(int type, int purpose); +connection_t *connection_get_by_type_addr_port_purpose(int type, + const tor_addr_t *addr, + uint16_t port, int purpose); +connection_t *connection_get_by_type_state(int type, int state); +connection_t *connection_get_by_type_state_rendquery(int type, int state, + const char *rendquery); + +#define connection_speaks_cells(conn) ((conn)->type == CONN_TYPE_OR) +int connection_is_listener(connection_t *conn); +int connection_state_is_open(connection_t *conn); +int connection_state_is_connecting(connection_t *conn); + +char *alloc_http_authenticator(const char *authenticator); + +void assert_connection_ok(connection_t *conn, time_t now); +int connection_or_nonopen_was_started_here(or_connection_t *conn); +void connection_dump_buffer_mem_stats(int severity); +void remove_file_if_very_old(const char *fname, time_t now); + +#endif + diff --git a/src/or/connection_edge.c b/src/or/connection_edge.c index 5609c1df40..2c1196c0cd 100644 --- a/src/or/connection_edge.c +++ b/src/or/connection_edge.c @@ -10,6 +10,28 @@ **/ #include "or.h" +#include "buffers.h" +#include "circuitlist.h" +#include "circuituse.h" +#include "config.h" +#include "connection.h" +#include "connection_edge.h" +#include "connection_or.h" +#include "control.h" +#include "dns.h" +#include "dnsserv.h" +#include "dirserv.h" +#include "hibernate.h" +#include "main.h" +#include "policies.h" +#include "reasons.h" +#include "relay.h" +#include "rendclient.h" +#include "rendcommon.h" +#include "rendservice.h" +#include "rephist.h" +#include "router.h" +#include "routerlist.h" #ifdef HAVE_LINUX_TYPES_H #include <linux/types.h> @@ -125,7 +147,7 @@ connection_edge_process_inbuf(edge_connection_t *conn, int package_partial) return 0; case AP_CONN_STATE_OPEN: case EXIT_CONN_STATE_OPEN: - if (connection_edge_package_raw_inbuf(conn, package_partial) < 0) { + if (connection_edge_package_raw_inbuf(conn, package_partial, NULL) < 0) { /* (We already sent an end cell if possible) */ connection_mark_for_close(TO_CONN(conn)); return -1; @@ -279,6 +301,23 @@ connection_edge_end_errno(edge_connection_t *conn) return connection_edge_end(conn, reason); } +/** We just wrote some data to <b>conn</b>; act appropriately. + * + * (That is, if it's open, consider sending a stream-level sendme cell if we + * have just flushed enough.) + */ +int +connection_edge_flushed_some(edge_connection_t *conn) +{ + switch (conn->_base.state) { + case AP_CONN_STATE_OPEN: + case EXIT_CONN_STATE_OPEN: + connection_edge_consider_sending_sendme(conn); + break; + } + return 0; +} + /** Connection <b>conn</b> has finished writing and has no bytes left on * its outbuf. * @@ -330,11 +369,13 @@ connection_edge_finished_connecting(edge_connection_t *edge_conn) tor_assert(conn->state == EXIT_CONN_STATE_CONNECTING); log_info(LD_EXIT,"Exit connection to %s:%u (%s) established.", - escaped_safe_str(conn->address),conn->port, + escaped_safe_str(conn->address), conn->port, safe_str(fmt_addr(&conn->addr))); + rep_hist_note_exit_stream_opened(conn->port); + conn->state = EXIT_CONN_STATE_OPEN; - connection_watch_events(conn, EV_READ); /* stop writing, continue reading */ + connection_watch_events(conn, READ_EVENT); /* stop writing, keep reading */ if (connection_wants_to_flush(conn)) /* in case there are any queued relay * cells */ connection_start_writing(conn); @@ -375,13 +416,16 @@ connection_edge_finished_connecting(edge_connection_t *edge_conn) static int compute_retry_timeout(edge_connection_t *conn) { + int timeout = get_options()->CircuitStreamTimeout; + if (timeout) /* if our config options override the default, use them */ + return timeout; if (conn->num_socks_retries < 2) /* try 0 and try 1 */ return 10; return 15; } /** Find all general-purpose AP streams waiting for a response that sent their - * begin/resolve cell >=15 seconds ago. Detach from their current circuit, and + * begin/resolve cell too long ago. Detach from their current circuit, and * mark their current circuit as unsuitable for new streams. Then call * connection_ap_handshake_attach_circuit() to attach to a new circuit (if * available) or launch a new one. @@ -423,7 +467,8 @@ connection_ap_expire_beginning(void) log_fn(severity, LD_APP, "Tried for %d seconds to get a connection to %s:%d. " "Giving up. (%s)", - seconds_since_born, safe_str(conn->socks_request->address), + seconds_since_born, + safe_str_client(conn->socks_request->address), conn->socks_request->port, conn_state_to_string(CONN_TYPE_AP, conn->_base.state)); connection_mark_unattached_ap(conn, END_STREAM_REASON_TIMEOUT); @@ -440,7 +485,7 @@ connection_ap_expire_beginning(void) circ = circuit_get_by_edge_conn(conn); if (!circ) { /* it's vanished? */ log_info(LD_APP,"Conn is waiting (address %s), but lost its circ.", - safe_str(conn->socks_request->address)); + safe_str_client(conn->socks_request->address)); connection_mark_unattached_ap(conn, END_STREAM_REASON_TIMEOUT); continue; } @@ -450,7 +495,7 @@ connection_ap_expire_beginning(void) "Rend stream is %d seconds late. Giving up on address" " '%s.onion'.", seconds_idle, - safe_str(conn->socks_request->address)); + safe_str_client(conn->socks_request->address)); connection_edge_end(conn, END_STREAM_REASON_TIMEOUT); connection_mark_unattached_ap(conn, END_STREAM_REASON_TIMEOUT); } @@ -460,7 +505,8 @@ connection_ap_expire_beginning(void) log_fn(cutoff < 15 ? LOG_INFO : severity, LD_APP, "We tried for %d seconds to connect to '%s' using exit '%s'." " Retrying on a new circuit.", - seconds_idle, safe_str(conn->socks_request->address), + seconds_idle, + safe_str_client(conn->socks_request->address), conn->cpath_layer ? conn->cpath_layer->extend_info->nickname : "*unnamed*"); /* send an end down the circuit */ @@ -471,6 +517,7 @@ connection_ap_expire_beginning(void) /* kludge to make us not try this circuit again, yet to allow * current streams on it to survive if they can: make it * unattractive to use for new streams */ + /* XXXX023 this is a kludgy way to do this. */ tor_assert(circ->timestamp_dirty); circ->timestamp_dirty -= options->MaxCircuitDirtiness; /* give our stream another 'cutoff' seconds to try */ @@ -511,7 +558,7 @@ connection_ap_attach_pending(void) /** Tell any AP streams that are waiting for a one-hop tunnel to * <b>failed_digest</b> that they are going to fail. */ -/* XXX022 We should get rid of this function, and instead attach +/* XXX023 We should get rid of this function, and instead attach * one-hop streams to circ->p_streams so they get marked in * circuit_mark_for_close like normal p_streams. */ void @@ -577,8 +624,8 @@ circuit_discard_optional_exit_enclaves(extend_info_t *info) tor_assert(edge_conn->socks_request); if (edge_conn->chosen_exit_optional) { log_info(LD_APP, "Giving up on enclave exit '%s' for destination %s.", - safe_str(edge_conn->chosen_exit_name), - escaped_safe_str(edge_conn->socks_request->address)); + safe_str_client(edge_conn->chosen_exit_name), + escaped_safe_str_client(edge_conn->socks_request->address)); edge_conn->chosen_exit_optional = 0; tor_free(edge_conn->chosen_exit_name); /* clears it */ /* if this port is dangerous, warn or reject it now that we don't @@ -683,7 +730,11 @@ addressmap_init(void) static void addressmap_ent_free(void *_ent) { - addressmap_entry_t *ent = _ent; + addressmap_entry_t *ent; + if (!_ent) + return; + + ent = _ent; tor_free(ent->new_address); tor_free(ent); } @@ -692,7 +743,11 @@ addressmap_ent_free(void *_ent) static void addressmap_virtaddress_ent_free(void *_ent) { - virtaddress_entry_t *ent = _ent; + virtaddress_entry_t *ent; + if (!_ent) + return; + + ent = _ent; tor_free(ent->ipv4_address); tor_free(ent->hostname_address); tor_free(ent); @@ -744,7 +799,8 @@ clear_trackexithost_mappings(const char *exitname) tor_strlower(suffix); STRMAP_FOREACH_MODIFY(addressmap, address, addressmap_entry_t *, ent) { - if (ent->source == ADDRMAPSRC_TRACKEXIT && !strcmpend(address, suffix)) { + if (ent->source == ADDRMAPSRC_TRACKEXIT && + !strcmpend(ent->new_address, suffix)) { addressmap_ent_remove(address, ent); MAP_DEL_CURRENT(address); } @@ -753,6 +809,56 @@ clear_trackexithost_mappings(const char *exitname) tor_free(suffix); } +/** Remove all TRACKEXIT mappings from the addressmap for which the target + * host is unknown or no longer allowed. */ +void +addressmap_clear_excluded_trackexithosts(or_options_t *options) +{ + const routerset_t *allow_nodes = options->ExitNodes; + const routerset_t *exclude_nodes = options->_ExcludeExitNodesUnion; + + if (!addressmap) + return; + if (routerset_is_empty(allow_nodes)) + allow_nodes = NULL; + if (allow_nodes == NULL && routerset_is_empty(exclude_nodes)) + return; + + STRMAP_FOREACH_MODIFY(addressmap, address, addressmap_entry_t *, ent) { + size_t len; + const char *target = ent->new_address, *dot; + char *nodename; + routerinfo_t *ri; /* XXX023 Use node_t. */ + + if (strcmpend(target, ".exit")) { + /* Not a .exit mapping */ + continue; + } else if (ent->source != ADDRMAPSRC_TRACKEXIT) { + /* Not a trackexit mapping. */ + continue; + } + len = strlen(target); + if (len < 6) + continue; /* malformed. */ + dot = target + len - 6; /* dot now points to just before .exit */ + dot = strrchr(dot, '.'); /* dot now points to the . before .exit or NULL */ + if (!dot) { + nodename = tor_strndup(target, len-5); + } else { + nodename = tor_strndup(dot+1, strlen(dot+1)-5); + } + ri = router_get_by_nickname(nodename, 0); + tor_free(nodename); + if (!ri || + (allow_nodes && !routerset_contains_router(allow_nodes, ri)) || + routerset_contains_router(exclude_nodes, ri)) { + /* We don't know this one, or we want to be rid of it. */ + addressmap_ent_remove(address, ent); + MAP_DEL_CURRENT(address); + } + } STRMAP_FOREACH_END; +} + /** Remove all entries from the addressmap that were set via the * configuration file or the command line. */ void @@ -782,14 +888,11 @@ addressmap_clean(time_t now) void addressmap_free_all(void) { - if (addressmap) { - strmap_free(addressmap, addressmap_ent_free); - addressmap = NULL; - } - if (virtaddress_reversemap) { - strmap_free(virtaddress_reversemap, addressmap_virtaddress_ent_free); - virtaddress_reversemap = NULL; - } + strmap_free(addressmap, addressmap_ent_free); + addressmap = NULL; + + strmap_free(virtaddress_reversemap, addressmap_virtaddress_ent_free); + virtaddress_reversemap = NULL; } /** Look at address, and rewrite it until it doesn't want any @@ -816,9 +919,9 @@ addressmap_rewrite(char *address, size_t maxlen, time_t *expires_out) return (rewrites > 0); /* done, no rewrite needed */ } - cp = tor_strdup(escaped_safe_str(ent->new_address)); + cp = tor_strdup(escaped_safe_str_client(ent->new_address)); log_info(LD_APP, "Addressmap: rewriting %s to %s", - escaped_safe_str(address), cp); + escaped_safe_str_client(address), cp); if (ent->expires > 1 && ent->expires < expires) expires = ent->expires; tor_free(cp); @@ -826,7 +929,7 @@ addressmap_rewrite(char *address, size_t maxlen, time_t *expires_out) } log_warn(LD_CONFIG, "Loop detected: we've rewritten %s 16 times! Using it as-is.", - escaped_safe_str(address)); + escaped_safe_str_client(address)); /* it's fine to rewrite a rewrite, but don't loop forever */ if (expires_out) *expires_out = TIME_MAX; @@ -848,9 +951,9 @@ addressmap_rewrite_reverse(char *address, size_t maxlen, time_t *expires_out) tor_snprintf(s, len, "REVERSE[%s]", address); ent = strmap_get(addressmap, s); if (ent) { - cp = tor_strdup(escaped_safe_str(ent->new_address)); + cp = tor_strdup(escaped_safe_str_client(ent->new_address)); log_info(LD_APP, "Rewrote reverse lookup %s -> %s", - escaped_safe_str(s), cp); + escaped_safe_str_client(s), cp); tor_free(cp); strlcpy(address, ent->new_address, maxlen); r = 1; @@ -912,7 +1015,9 @@ addressmap_register(const char *address, char *new_address, time_t expires, if (expires > 1) { log_info(LD_APP,"Temporary addressmap ('%s' to '%s') not performed, " "since it's already mapped to '%s'", - safe_str(address), safe_str(new_address), safe_str(ent->new_address)); + safe_str_client(address), + safe_str_client(new_address), + safe_str_client(ent->new_address)); tor_free(new_address); return; } @@ -931,7 +1036,8 @@ addressmap_register(const char *address, char *new_address, time_t expires, ent->source = source; log_info(LD_CONFIG, "Addressmap: (re)mapped '%s' to '%s'", - safe_str(address), safe_str(ent->new_address)); + safe_str_client(address), + safe_str_client(ent->new_address)); control_event_address_mapped(address, ent->new_address, expires, NULL); } @@ -951,7 +1057,8 @@ client_dns_incr_failures(const char *address) if (ent->num_resolve_failures < SHORT_MAX) ++ent->num_resolve_failures; /* don't overflow */ log_info(LD_APP, "Address %s now has %d resolve failures.", - safe_str(address), ent->num_resolve_failures); + safe_str_client(address), + ent->num_resolve_failures); return ent->num_resolve_failures; } @@ -1247,8 +1354,10 @@ addressmap_register_virtual_address(int type, char *new_address) log_warn(LD_BUG, "Internal confusion: I thought that '%s' was mapped to by " "'%s', but '%s' really maps to '%s'. This is a harmless bug.", - safe_str(new_address), safe_str(*addrp), safe_str(*addrp), - ent?safe_str(ent->new_address):"(nothing)"); + safe_str_client(new_address), + safe_str_client(*addrp), + safe_str_client(*addrp), + ent?safe_str_client(ent->new_address):"(nothing)"); } tor_free(*addrp); @@ -1276,7 +1385,8 @@ addressmap_register_virtual_address(int type, char *new_address) (type == RESOLVED_TYPE_IPV4) ? vent->ipv4_address : vent->hostname_address)); log_info(LD_APP, "Map from %s to %s okay.", - safe_str(*addrp),safe_str(new_address)); + safe_str_client(*addrp), + safe_str_client(new_address)); } #endif @@ -1391,6 +1501,26 @@ consider_plaintext_ports(edge_connection_t *conn, uint16_t port) * different one? */ #define TRACKHOSTEXITS_RETRIES 5 +/** Call connection_ap_handshake_rewrite_and_attach() unless a controller + * asked us to leave streams unattached. Return 0 in that case. + * + * See connection_ap_handshake_rewrite_and_attach()'s + * documentation for arguments and return value. + */ +int +connection_ap_rewrite_and_attach_if_allowed(edge_connection_t *conn, + origin_circuit_t *circ, + crypt_path_t *cpath) +{ + or_options_t *options = get_options(); + + if (options->LeaveStreamsUnattached) { + conn->_base.state = AP_CONN_STATE_CONTROLLER_WAIT; + return 0; + } + return connection_ap_handshake_rewrite_and_attach(conn, circ, cpath); +} + /** Connection <b>conn</b> just finished its socks handshake, or the * controller asked us to take care of it. If <b>circ</b> is defined, * then that's where we'll want to attach it. Otherwise we have to @@ -1415,16 +1545,20 @@ connection_ap_handshake_rewrite_and_attach(edge_connection_t *conn, hostname_type_t addresstype; or_options_t *options = get_options(); struct in_addr addr_tmp; + /* We set this to true if this is an address we should automatically + * remap to a local address in VirtualAddrNetwork */ int automap = 0; char orig_address[MAX_SOCKS_ADDR_LEN]; time_t map_expires = TIME_MAX; + /* This will be set to true iff the address starts out as a non-.exit + address, and we remap it to one because of an entry in the addressmap. */ int remapped_to_exit = 0; time_t now = time(NULL); tor_strlower(socks->address); /* normalize it */ strlcpy(orig_address, socks->address, sizeof(orig_address)); log_debug(LD_APP,"Client asked for %s:%d", - safe_str(socks->address), + safe_str_client(socks->address), socks->port); if (socks->command == SOCKS_COMMAND_RESOLVE && @@ -1446,7 +1580,8 @@ connection_ap_handshake_rewrite_and_attach(edge_connection_t *conn, return -1; } log_info(LD_APP, "Automapping %s to %s", - escaped_safe_str(socks->address), safe_str(new_addr)); + escaped_safe_str_client(socks->address), + safe_str_client(new_addr)); strlcpy(socks->address, new_addr, sizeof(socks->address)); } } @@ -1503,7 +1638,7 @@ connection_ap_handshake_rewrite_and_attach(edge_connection_t *conn, * information. */ log_warn(LD_APP,"Missing mapping for virtual address '%s'. Refusing.", - socks->address); /* don't safe_str() this yet. */ + safe_str_client(socks->address)); connection_mark_unattached_ap(conn, END_STREAM_REASON_INTERNAL); return -1; } @@ -1511,11 +1646,12 @@ connection_ap_handshake_rewrite_and_attach(edge_connection_t *conn, /* Parse the address provided by SOCKS. Modify it in-place if it * specifies a hidden-service (.onion) or particular exit node (.exit). */ - addresstype = parse_extended_hostname(socks->address); + addresstype = parse_extended_hostname(socks->address, + remapped_to_exit || options->AllowDotExit); if (addresstype == BAD_HOSTNAME) { log_warn(LD_APP, "Invalid onion hostname %s; rejecting", - safe_str(socks->address)); + safe_str_client(socks->address)); control_event_client_status(LOG_WARN, "SOCKS_BAD_HOSTNAME HOSTNAME=%s", escaped(socks->address)); connection_mark_unattached_ap(conn, END_STREAM_REASON_TORPROTOCOL); @@ -1524,38 +1660,61 @@ connection_ap_handshake_rewrite_and_attach(edge_connection_t *conn, if (addresstype == EXIT_HOSTNAME) { /* foo.exit -- modify conn->chosen_exit_node to specify the exit - * node, and conn->address to hold only the address portion.*/ + * node, and conn->address to hold only the address portion. */ char *s = strrchr(socks->address,'.'); + + /* If StrictNodes is not set, then .exit overrides ExcludeNodes. */ + routerset_t *excludeset = options->StrictNodes ? + options->_ExcludeExitNodesUnion : options->ExcludeExitNodes; + /*XXX023 make this a node_t. */ + routerinfo_t *router; + tor_assert(!automap); if (s) { + /* The address was of the form "(stuff).(name).exit */ if (s[1] != '\0') { conn->chosen_exit_name = tor_strdup(s+1); + router = router_get_by_nickname(conn->chosen_exit_name, 1); if (remapped_to_exit) /* 5 tries before it expires the addressmap */ conn->chosen_exit_retries = TRACKHOSTEXITS_RETRIES; *s = 0; } else { + /* Oops, the address was (stuff)..exit. That's not okay. */ log_warn(LD_APP,"Malformed exit address '%s.exit'. Refusing.", - safe_str(socks->address)); + safe_str_client(socks->address)); control_event_client_status(LOG_WARN, "SOCKS_BAD_HOSTNAME HOSTNAME=%s", escaped(socks->address)); connection_mark_unattached_ap(conn, END_STREAM_REASON_TORPROTOCOL); return -1; } } else { - routerinfo_t *r; + /* It looks like they just asked for "foo.exit". */ conn->chosen_exit_name = tor_strdup(socks->address); - r = router_get_by_nickname(conn->chosen_exit_name, 1); - *socks->address = 0; - if (r) { - strlcpy(socks->address, r->address, sizeof(socks->address)); - } else { - log_warn(LD_APP, - "Unrecognized server in exit address '%s.exit'. Refusing.", - safe_str(socks->address)); - connection_mark_unattached_ap(conn, END_STREAM_REASON_TORPROTOCOL); - return -1; + router = router_get_by_nickname(conn->chosen_exit_name, 1); + if (router) { + *socks->address = 0; + strlcpy(socks->address, router->address, sizeof(socks->address)); } } + /* Now make sure that the chosen exit exists... */ + if (!router) { + log_warn(LD_APP, + "Unrecognized relay in exit address '%s.exit'. Refusing.", + safe_str_client(socks->address)); + connection_mark_unattached_ap(conn, END_STREAM_REASON_TORPROTOCOL); + return -1; + } + /* ...and make sure that it isn't excluded. */ + if (routerset_contains_router(excludeset, router)) { + log_warn(LD_APP, + "Excluded relay in exit address '%s.exit'. Refusing.", + safe_str_client(socks->address)); + connection_mark_unattached_ap(conn, END_STREAM_REASON_TORPROTOCOL); + return -1; + } + /* XXXX022-1090 Should we also allow foo.bar.exit if ExitNodes is set and + Bar is not listed in it? I say yes, but our revised manpage branch + implies no. */ } if (addresstype != ONION_HOSTNAME) { @@ -1565,7 +1724,7 @@ connection_ap_handshake_rewrite_and_attach(edge_connection_t *conn, escaped(socks->address)); log_warn(LD_APP, "Destination '%s' seems to be an invalid hostname. Failing.", - safe_str(socks->address)); + safe_str_client(socks->address)); connection_mark_unattached_ap(conn, END_STREAM_REASON_TORPROTOCOL); return -1; } @@ -1574,18 +1733,6 @@ connection_ap_handshake_rewrite_and_attach(edge_connection_t *conn, uint32_t answer; struct in_addr in; /* Reply to resolves immediately if we can. */ - if (strlen(socks->address) > RELAY_PAYLOAD_SIZE) { - log_warn(LD_APP,"Address to be resolved is too large. Failing."); - control_event_client_status(LOG_WARN, "SOCKS_BAD_HOSTNAME HOSTNAME=%s", - escaped(socks->address)); - connection_ap_handshake_socks_resolved(conn, - RESOLVED_TYPE_ERROR_TRANSIENT, - 0,NULL,-1,TIME_MAX); - connection_mark_unattached_ap(conn, - END_STREAM_REASON_SOCKSPROTOCOL | - END_STREAM_REASON_FLAG_ALREADY_SOCKS_REPLIED); - return -1; - } if (tor_inet_aton(socks->address, &in)) { /* see if it's an IP already */ /* leave it in network order */ answer = in.s_addr; @@ -1608,6 +1755,28 @@ connection_ap_handshake_rewrite_and_attach(edge_connection_t *conn, connection_mark_unattached_ap(conn, END_STREAM_REASON_TORPROTOCOL); return -1; } + if (options->ClientRejectInternalAddresses && + !conn->use_begindir && !conn->chosen_exit_name && !circ) { + tor_addr_t addr; + if (tor_addr_from_str(&addr, socks->address) >= 0 && + tor_addr_is_internal(&addr, 0)) { + /* If this is an explicit private address with no chosen exit node, + * then we really don't want to try to connect to it. That's + * probably an error. */ + if (conn->is_transparent_ap) { + log_warn(LD_NET, + "Rejecting request for anonymous connection to private " + "address %s on a TransPort or NATDPort. Possible loop " + "in your NAT rules?", safe_str_client(socks->address)); + } else { + log_warn(LD_NET, + "Rejecting SOCKS request for anonymous connection to " + "private address %s", safe_str_client(socks->address)); + } + connection_mark_unattached_ap(conn, END_STREAM_REASON_PRIVATE_ADDR); + return -1; + } + } if (!conn->use_begindir && !conn->chosen_exit_name && !circ) { /* see if we can find a suitable enclave exit */ @@ -1616,7 +1785,7 @@ connection_ap_handshake_rewrite_and_attach(edge_connection_t *conn, if (r) { log_info(LD_APP, "Redirecting address %s to exit at enclave router %s", - safe_str(socks->address), r->nickname); + safe_str_client(socks->address), r->nickname); /* use the hex digest, not nickname, in case there are two routers with this nickname */ conn->chosen_exit_name = @@ -1680,12 +1849,12 @@ connection_ap_handshake_rewrite_and_attach(edge_connection_t *conn, strlcpy(conn->rend_data->onion_address, socks->address, sizeof(conn->rend_data->onion_address)); log_info(LD_REND,"Got a hidden service request for ID '%s'", - safe_str(conn->rend_data->onion_address)); + safe_str_client(conn->rend_data->onion_address)); /* see if we already have it cached */ r = rend_cache_lookup_entry(conn->rend_data->onion_address, -1, &entry); if (r<0) { log_warn(LD_BUG,"Invalid service name '%s'", - safe_str(conn->rend_data->onion_address)); + safe_str_client(conn->rend_data->onion_address)); connection_mark_unattached_ap(conn, END_STREAM_REASON_TORPROTOCOL); return -1; } @@ -1707,32 +1876,15 @@ connection_ap_handshake_rewrite_and_attach(edge_connection_t *conn, if (r==0) { conn->_base.state = AP_CONN_STATE_RENDDESC_WAIT; log_info(LD_REND, "Unknown descriptor %s. Fetching.", - safe_str(conn->rend_data->onion_address)); - /* Fetch both, v0 and v2 rend descriptors in parallel. Use whichever - * arrives first. Exception: When using client authorization, only - * fetch v2 descriptors.*/ + safe_str_client(conn->rend_data->onion_address)); rend_client_refetch_v2_renddesc(conn->rend_data); - if (conn->rend_data->auth_type == REND_NO_AUTH) - rend_client_refetch_renddesc(conn->rend_data->onion_address); } else { /* r > 0 */ - if (now - entry->received < NUM_SECONDS_BEFORE_HS_REFETCH) { - conn->_base.state = AP_CONN_STATE_CIRCUIT_WAIT; - log_info(LD_REND, "Descriptor is here and fresh enough. Great."); - if (connection_ap_handshake_attach_circuit(conn) < 0) { - if (!conn->_base.marked_for_close) - connection_mark_unattached_ap(conn, END_STREAM_REASON_CANT_ATTACH); - return -1; - } - } else { - conn->_base.state = AP_CONN_STATE_RENDDESC_WAIT; - log_info(LD_REND, "Stale descriptor %s. Re-fetching.", - safe_str(conn->rend_data->onion_address)); - /* Fetch both, v0 and v2 rend descriptors in parallel. Use whichever - * arrives first. Exception: When using client authorization, only - * fetch v2 descriptors.*/ - rend_client_refetch_v2_renddesc(conn->rend_data); - if (conn->rend_data->auth_type == REND_NO_AUTH) - rend_client_refetch_renddesc(conn->rend_data->onion_address); + conn->_base.state = AP_CONN_STATE_CIRCUIT_WAIT; + log_info(LD_REND, "Descriptor is here. Great."); + if (connection_ap_handshake_attach_circuit(conn) < 0) { + if (!conn->_base.marked_for_close) + connection_mark_unattached_ap(conn, END_STREAM_REASON_CANT_ATTACH); + return -1; } } return 0; @@ -1920,24 +2072,12 @@ connection_ap_handshake_process_socks(edge_connection_t *conn) return -1; } /* else socks handshake is done, continue processing */ - if (hostname_is_noconnect_address(socks->address)) - { - control_event_stream_status(conn, STREAM_EVENT_NEW, 0); - control_event_stream_status(conn, STREAM_EVENT_CLOSED, 0); - connection_mark_unattached_ap(conn, END_STREAM_REASON_DONE); - return -1; - } - if (SOCKS_COMMAND_IS_CONNECT(socks->command)) control_event_stream_status(conn, STREAM_EVENT_NEW, 0); else control_event_stream_status(conn, STREAM_EVENT_NEW_RESOLVE, 0); - if (options->LeaveStreamsUnattached) { - conn->_base.state = AP_CONN_STATE_CONTROLLER_WAIT; - return 0; - } - return connection_ap_handshake_rewrite_and_attach(conn, NULL, NULL); + return connection_ap_rewrite_and_attach_if_allowed(conn, NULL, NULL); } /** connection_init_accepted_conn() found a new trans AP conn. @@ -1951,7 +2091,6 @@ int connection_ap_process_transparent(edge_connection_t *conn) { socks_request_t *socks; - or_options_t *options = get_options(); tor_assert(conn); tor_assert(conn->_base.type == CONN_TYPE_AP); @@ -1975,11 +2114,7 @@ connection_ap_process_transparent(edge_connection_t *conn) control_event_stream_status(conn, STREAM_EVENT_NEW, 0); - if (options->LeaveStreamsUnattached) { - conn->_base.state = AP_CONN_STATE_CONTROLLER_WAIT; - return 0; - } - return connection_ap_handshake_rewrite_and_attach(conn, NULL, NULL); + return connection_ap_rewrite_and_attach_if_allowed(conn, NULL, NULL); } /** connection_edge_process_inbuf() found a conn in state natd_wait. See if @@ -2000,7 +2135,6 @@ connection_ap_process_natd(edge_connection_t *conn) size_t tlen = 30; int err, port_ok; socks_request_t *socks; - or_options_t *options = get_options(); tor_assert(conn); tor_assert(conn->_base.type == CONN_TYPE_AP); @@ -2016,13 +2150,13 @@ connection_ap_process_natd(edge_connection_t *conn) if (err == 0) return 0; if (err < 0) { - log_warn(LD_APP,"Natd handshake failed (DEST too long). Closing"); + log_warn(LD_APP,"NATD handshake failed (DEST too long). Closing"); connection_mark_unattached_ap(conn, END_STREAM_REASON_INVALID_NATD_DEST); return -1; } if (strcmpstart(tmp_buf, "[DEST ")) { - log_warn(LD_APP,"Natd handshake was ill-formed; closing. The client " + log_warn(LD_APP,"NATD handshake was ill-formed; closing. The client " "said: %s", escaped(tmp_buf)); connection_mark_unattached_ap(conn, END_STREAM_REASON_INVALID_NATD_DEST); @@ -2031,7 +2165,7 @@ connection_ap_process_natd(edge_connection_t *conn) daddr = tbuf = &tmp_buf[0] + 6; /* after end of "[DEST " */ if (!(tbuf = strchr(tbuf, ' '))) { - log_warn(LD_APP,"Natd handshake was ill-formed; closing. The client " + log_warn(LD_APP,"NATD handshake was ill-formed; closing. The client " "said: %s", escaped(tmp_buf)); connection_mark_unattached_ap(conn, END_STREAM_REASON_INVALID_NATD_DEST); @@ -2045,7 +2179,7 @@ connection_ap_process_natd(edge_connection_t *conn) socks->port = (uint16_t) tor_parse_long(tbuf, 10, 1, 65535, &port_ok, &daddr); if (!port_ok) { - log_warn(LD_APP,"Natd handshake failed; port %s is ill-formed or out " + log_warn(LD_APP,"NATD handshake failed; port %s is ill-formed or out " "of range.", escaped(tbuf)); connection_mark_unattached_ap(conn, END_STREAM_REASON_INVALID_NATD_DEST); return -1; @@ -2056,13 +2190,9 @@ connection_ap_process_natd(edge_connection_t *conn) control_event_stream_status(conn, STREAM_EVENT_NEW, 0); - if (options->LeaveStreamsUnattached) { - conn->_base.state = AP_CONN_STATE_CONTROLLER_WAIT; - return 0; - } conn->_base.state = AP_CONN_STATE_CIRCUIT_WAIT; - return connection_ap_handshake_rewrite_and_attach(conn, NULL, NULL); + return connection_ap_rewrite_and_attach_if_allowed(conn, NULL, NULL); } /** Iterate over the two bytes of stream_id until we get one that is not @@ -2075,7 +2205,7 @@ get_unique_stream_id_by_circ(origin_circuit_t *circ) streamid_t test_stream_id; uint32_t attempts=0; -again: + again: test_stream_id = circ->next_stream_id++; if (++attempts > 1<<16) { /* Make sure we don't loop forever if all stream_id's are used. */ @@ -2112,8 +2242,14 @@ connection_ap_handshake_send_begin(edge_connection_t *ap_conn) ap_conn->stream_id = get_unique_stream_id_by_circ(circ); if (ap_conn->stream_id==0) { + /* XXXX023 Instead of closing this stream, we should make it get + * retried on another circuit. */ connection_mark_unattached_ap(ap_conn, END_STREAM_REASON_INTERNAL); - circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_RESOURCELIMIT); + + /* Mark this circuit "unusable for new streams". */ + /* XXXX023 this is a kludgy way to do this. */ + tor_assert(circ->_base.timestamp_dirty); + circ->_base.timestamp_dirty -= get_options()->MaxCircuitDirtiness; return -1; } @@ -2171,9 +2307,14 @@ connection_ap_handshake_send_resolve(edge_connection_t *ap_conn) ap_conn->stream_id = get_unique_stream_id_by_circ(circ); if (ap_conn->stream_id==0) { + /* XXXX023 Instead of closing this stream, we should make it get + * retried on another circuit. */ connection_mark_unattached_ap(ap_conn, END_STREAM_REASON_INTERNAL); - /*XXXX022 _close_ the circuit because it's full? That sounds dumb. */ - circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_RESOURCELIMIT); + + /* Mark this circuit "unusable for new streams". */ + /* XXXX023 this is a kludgy way to do this. */ + tor_assert(circ->_base.timestamp_dirty); + circ->_base.timestamp_dirty -= get_options()->MaxCircuitDirtiness; return -1; } @@ -2191,7 +2332,7 @@ connection_ap_handshake_send_resolve(edge_connection_t *ap_conn) r = tor_addr_parse_reverse_lookup_name(&addr, a, AF_INET, 1); if (r <= 0) { log_warn(LD_APP, "Rejecting ill-formed reverse lookup of %s", - safe_str(a)); + safe_str_client(a)); connection_mark_unattached_ap(ap_conn, END_STREAM_REASON_INTERNAL); return -1; } @@ -2199,7 +2340,7 @@ connection_ap_handshake_send_resolve(edge_connection_t *ap_conn) r = tor_addr_to_reverse_lookup_name(inaddr_buf, sizeof(inaddr_buf), &addr); if (r < 0) { log_warn(LD_BUG, "Couldn't generate reverse lookup hostname of %s", - safe_str(a)); + safe_str_client(a)); connection_mark_unattached_ap(ap_conn, END_STREAM_REASON_INTERNAL); return -1; } @@ -2209,12 +2350,6 @@ connection_ap_handshake_send_resolve(edge_connection_t *ap_conn) tor_assert(payload_len <= (int)sizeof(inaddr_buf)); } - if (payload_len > RELAY_PAYLOAD_SIZE) { - /* This should be impossible: we don't accept addresses this big. */ - connection_mark_unattached_ap(ap_conn, END_STREAM_REASON_INTERNAL); - return -1; - } - log_debug(LD_APP, "Sending relay cell to begin stream %d.", ap_conn->stream_id); @@ -2246,7 +2381,8 @@ connection_ap_make_link(char *address, uint16_t port, edge_connection_t *conn; log_info(LD_APP,"Making internal %s tunnel to %s:%d ...", - want_onehop ? "direct" : "anonymized" , safe_str(address),port); + want_onehop ? "direct" : "anonymized", + safe_str_client(address), port); conn = edge_connection_new(CONN_TYPE_AP, AF_INET); conn->_base.linked = 1; /* so that we can add it safely below. */ @@ -2336,7 +2472,7 @@ tell_controller_about_resolved_result(edge_connection_t *conn, * certain errors or for values that didn't come via DNS. <b>expires</b> is * a time when the answer expires, or -1 or TIME_MAX if there's a good TTL. **/ -/* XXXX022 the use of the ttl and expires fields is nutty. Let's make this +/* XXXX023 the use of the ttl and expires fields is nutty. Let's make this * interface and those that use it less ugly. */ void connection_ap_handshake_socks_resolved(edge_connection_t *conn, @@ -2512,6 +2648,7 @@ connection_exit_begin_conn(cell_t *cell, circuit_t *circ) char *address=NULL; uint16_t port; or_circuit_t *or_circ = NULL; + or_options_t *options = get_options(); assert_circuit_ok(circ); if (!CIRCUIT_IS_ORIGIN(circ)) @@ -2526,7 +2663,7 @@ connection_exit_begin_conn(cell_t *cell, circuit_t *circ) * that we have a stream connected to a circuit, and we don't connect to a * circuit until we have a pending/successful resolve. */ - if (!server_mode(get_options()) && + if (!server_mode(options) && circ->purpose != CIRCUIT_PURPOSE_S_REND_JOINED) { log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, "Relay begin cell at non-server. Closing."); @@ -2560,21 +2697,30 @@ connection_exit_begin_conn(cell_t *cell, circuit_t *circ) tor_free(address); return 0; } - if (or_circ && or_circ->is_first_hop && - !get_options()->AllowSingleHopExits) { + if (or_circ && or_circ->p_conn && !options->AllowSingleHopExits && + (or_circ->is_first_hop || + (!connection_or_digest_is_known_relay( + or_circ->p_conn->identity_digest) && + should_refuse_unknown_exits(options)))) { /* Don't let clients use us as a single-hop proxy, unless the user - * has explicitly allowed that in the config. It attracts attackers + * has explicitly allowed that in the config. It attracts attackers * and users who'd be better off with, well, single-hop proxies. */ log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, - "Attempt to open a stream on first hop of circuit. Closing."); + "Attempt by %s to open a stream %s. Closing.", + safe_str(or_circ->p_conn->_base.address), + or_circ->is_first_hop ? "on first hop of circuit" : + "from unknown relay"); relay_send_end_cell_from_edge(rh.stream_id, circ, - END_STREAM_REASON_TORPROTOCOL, NULL); + or_circ->is_first_hop ? + END_STREAM_REASON_TORPROTOCOL : + END_STREAM_REASON_MISC, + NULL); tor_free(address); return 0; } } else if (rh.command == RELAY_COMMAND_BEGIN_DIR) { - if (!directory_permits_begindir_requests(get_options()) || + if (!directory_permits_begindir_requests(options) || circ->purpose != CIRCUIT_PURPOSE_OR) { relay_send_end_cell_from_edge(rh.stream_id, circ, END_STREAM_REASON_NOTDIRECTORY, NULL); @@ -2600,6 +2746,11 @@ connection_exit_begin_conn(cell_t *cell, circuit_t *circ) log_debug(LD_EXIT,"Creating new exit connection."); n_stream = edge_connection_new(CONN_TYPE_EXIT, AF_INET); + + /* Remember the tunneled request ID in the new edge connection, so that + * we can measure download times. */ + TO_CONN(n_stream)->dirreq_id = circ->dirreq_id; + n_stream->_base.purpose = EXIT_PURPOSE_CONNECT; n_stream->stream_id = rh.stream_id; @@ -2630,7 +2781,7 @@ connection_exit_begin_conn(cell_t *cell, circuit_t *circ) log_debug(LD_REND,"Finished assigning addr/port"); n_stream->cpath_layer = origin_circ->cpath->prev; /* link it */ - /* add it into the linked list of n_streams on this circuit */ + /* add it into the linked list of p_streams on this circuit */ n_stream->next_stream = origin_circ->p_streams; n_stream->on_circuit = circ; origin_circ->p_streams = n_stream; @@ -2657,7 +2808,7 @@ connection_exit_begin_conn(cell_t *cell, circuit_t *circ) if (rh.command == RELAY_COMMAND_BEGIN_DIR) { tor_assert(or_circ); if (or_circ->p_conn && !tor_addr_is_null(&or_circ->p_conn->real_addr)) - tor_addr_assign(&n_stream->_base.addr, &or_circ->p_conn->real_addr); + tor_addr_copy(&n_stream->_base.addr, &or_circ->p_conn->real_addr); return connection_exit_connect_dir(n_stream); } @@ -2750,7 +2901,7 @@ connection_exit_connect(edge_connection_t *edge_conn) if (!connection_edge_is_rendezvous_stream(edge_conn) && router_compare_to_my_exit_policy(edge_conn)) { log_info(LD_EXIT,"%s:%d failed exit policy. Closing.", - escaped_safe_str(conn->address), conn->port); + escaped_safe_str_client(conn->address), conn->port); connection_edge_end(edge_conn, END_STREAM_REASON_EXITPOLICY); circuit_detach_stream(circuit_get_by_edge_conn(edge_conn), edge_conn); connection_free(conn); @@ -2762,17 +2913,17 @@ connection_exit_connect(edge_connection_t *edge_conn) log_debug(LD_EXIT,"about to try connecting"); switch (connection_connect(conn, conn->address, addr, port, &socket_error)) { - case -1: - /* XXX021 use socket_error below rather than trying to piece things - * together from the current errno, which may have been clobbered. */ - connection_edge_end_errno(edge_conn); + case -1: { + int reason = errno_to_stream_end_reason(socket_error); + connection_edge_end(edge_conn, reason); circuit_detach_stream(circuit_get_by_edge_conn(edge_conn), edge_conn); connection_free(conn); return; + } case 0: conn->state = EXIT_CONN_STATE_CONNECTING; - connection_watch_events(conn, EV_WRITE | EV_READ); + connection_watch_events(conn, READ_EVENT | WRITE_EVENT); /* writable indicates finish; * readable/error indicates broken link in windows-land. */ return; @@ -2785,7 +2936,7 @@ connection_exit_connect(edge_connection_t *edge_conn) log_warn(LD_BUG,"newly connected conn had data waiting!"); // connection_start_writing(conn); } - connection_watch_events(conn, EV_READ); + connection_watch_events(conn, READ_EVENT); /* also, deliver a 'connected' cell back through the circuit. */ if (connection_edge_is_rendezvous_stream(edge_conn)) { @@ -2832,13 +2983,17 @@ connection_exit_connect_dir(edge_connection_t *exitconn) dirconn = dir_connection_new(AF_INET); - tor_addr_assign(&dirconn->_base.addr, &exitconn->_base.addr); + tor_addr_copy(&dirconn->_base.addr, &exitconn->_base.addr); dirconn->_base.port = 0; dirconn->_base.address = tor_strdup(exitconn->_base.address); dirconn->_base.type = CONN_TYPE_DIR; dirconn->_base.purpose = DIR_PURPOSE_SERVER; dirconn->_base.state = DIR_CONN_STATE_SERVER_COMMAND_WAIT; + /* Note that the new dir conn belongs to the same tunneled request as + * the edge conn, so that we can measure download times. */ + TO_CONN(dirconn)->dirreq_id = TO_CONN(exitconn)->dirreq_id; + connection_link_connections(TO_CONN(dirconn), TO_CONN(exitconn)); if (connection_add(TO_CONN(exitconn))<0) { @@ -2893,6 +3048,8 @@ connection_edge_is_rendezvous_stream(edge_connection_t *conn) int connection_ap_can_use_exit(edge_connection_t *conn, routerinfo_t *exit) { + or_options_t *options = get_options(); + tor_assert(conn); tor_assert(conn->_base.type == CONN_TYPE_AP); tor_assert(conn->socks_request); @@ -2938,20 +3095,26 @@ connection_ap_can_use_exit(edge_connection_t *conn, routerinfo_t *exit) if (!conn->chosen_exit_name && policy_is_reject_star(exit->exit_policy)) return 0; } + if (options->_ExcludeExitNodesUnion && + routerset_contains_router(options->_ExcludeExitNodesUnion, exit)) { + /* Not a suitable exit. Refuse it. */ + return 0; + } + return 1; } /** If address is of the form "y.onion" with a well-formed handle y: * Put a NUL after y, lower-case it, and return ONION_HOSTNAME. * - * If address is of the form "y.exit": + * If address is of the form "y.exit" and <b>allowdotexit</b> is true: * Put a NUL after y and return EXIT_HOSTNAME. * * Otherwise: * Return NORMAL_HOSTNAME and change nothing. */ hostname_type_t -parse_extended_hostname(char *address) +parse_extended_hostname(char *address, int allowdotexit) { char *s; char query[REND_SERVICE_ID_LEN_BASE32+1]; @@ -2960,8 +3123,16 @@ parse_extended_hostname(char *address) if (!s) return NORMAL_HOSTNAME; /* no dot, thus normal */ if (!strcmp(s+1,"exit")) { - *s = 0; /* NUL-terminate it */ - return EXIT_HOSTNAME; /* .exit */ + if (allowdotexit) { + *s = 0; /* NUL-terminate it */ + return EXIT_HOSTNAME; /* .exit */ + } else { + log_warn(LD_APP, "The \".exit\" notation is disabled in Tor due to " + "security risks. Set AllowDotExit in your torrc to enable " + "it."); + /* FFFF send a controller event too to notify Vidalia users */ + return BAD_HOSTNAME; + } } if (strcmp(s+1,"onion")) return NORMAL_HOSTNAME; /* neither .exit nor .onion, thus normal */ @@ -2974,17 +3145,9 @@ parse_extended_hostname(char *address) if (rend_valid_service_id(query)) { return ONION_HOSTNAME; /* success */ } -failed: + failed: /* otherwise, return to previous state and return 0 */ *s = '.'; return BAD_HOSTNAME; } -/** Check if the address is of the form "y.noconnect" - */ -int -hostname_is_noconnect_address(const char *address) -{ - return ! strcasecmpend(address, ".noconnect"); -} - diff --git a/src/or/connection_edge.h b/src/or/connection_edge.h new file mode 100644 index 0000000000..70d0dd2713 --- /dev/null +++ b/src/or/connection_edge.h @@ -0,0 +1,101 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file connection_edge.h + * \brief Header file for connection_edge.c. + **/ + +#ifndef _TOR_CONNECTION_EDGE_H +#define _TOR_CONNECTION_EDGE_H + +#define connection_mark_unattached_ap(conn, endreason) \ + _connection_mark_unattached_ap((conn), (endreason), __LINE__, _SHORT_FILE_) + +void _connection_mark_unattached_ap(edge_connection_t *conn, int endreason, + int line, const char *file); +int connection_edge_reached_eof(edge_connection_t *conn); +int connection_edge_process_inbuf(edge_connection_t *conn, + int package_partial); +int connection_edge_destroy(circid_t circ_id, edge_connection_t *conn); +int connection_edge_end(edge_connection_t *conn, uint8_t reason); +int connection_edge_end_errno(edge_connection_t *conn); +int connection_edge_flushed_some(edge_connection_t *conn); +int connection_edge_finished_flushing(edge_connection_t *conn); +int connection_edge_finished_connecting(edge_connection_t *conn); + +int connection_ap_handshake_send_begin(edge_connection_t *ap_conn); +int connection_ap_handshake_send_resolve(edge_connection_t *ap_conn); + +edge_connection_t *connection_ap_make_link(char *address, uint16_t port, + const char *digest, + int use_begindir, int want_onehop); +void connection_ap_handshake_socks_reply(edge_connection_t *conn, char *reply, + size_t replylen, + int endreason); +void connection_ap_handshake_socks_resolved(edge_connection_t *conn, + int answer_type, + size_t answer_len, + const uint8_t *answer, + int ttl, + time_t expires); + +int connection_exit_begin_conn(cell_t *cell, circuit_t *circ); +int connection_exit_begin_resolve(cell_t *cell, or_circuit_t *circ); +void connection_exit_connect(edge_connection_t *conn); +int connection_edge_is_rendezvous_stream(edge_connection_t *conn); +int connection_ap_can_use_exit(edge_connection_t *conn, routerinfo_t *exit); +void connection_ap_expire_beginning(void); +void connection_ap_attach_pending(void); +void connection_ap_fail_onehop(const char *failed_digest, + cpath_build_state_t *build_state); +void circuit_discard_optional_exit_enclaves(extend_info_t *info); +int connection_ap_detach_retriable(edge_connection_t *conn, + origin_circuit_t *circ, + int reason); +int connection_ap_process_transparent(edge_connection_t *conn); + +int address_is_invalid_destination(const char *address, int client); + +void addressmap_init(void); +void addressmap_clear_excluded_trackexithosts(or_options_t *options); +void addressmap_clean(time_t now); +void addressmap_clear_configured(void); +void addressmap_clear_transient(void); +void addressmap_free_all(void); +int addressmap_rewrite(char *address, size_t maxlen, time_t *expires_out); +int addressmap_have_mapping(const char *address, int update_timeout); + +void addressmap_register(const char *address, char *new_address, + time_t expires, addressmap_entry_source_t source); +int parse_virtual_addr_network(const char *val, int validate_only, + char **msg); +int client_dns_incr_failures(const char *address); +void client_dns_clear_failures(const char *address); +void client_dns_set_addressmap(const char *address, uint32_t val, + const char *exitname, int ttl); +const char *addressmap_register_virtual_address(int type, char *new_address); +void addressmap_get_mappings(smartlist_t *sl, time_t min_expires, + time_t max_expires, int want_expiry); +int connection_ap_rewrite_and_attach_if_allowed(edge_connection_t *conn, + origin_circuit_t *circ, + crypt_path_t *cpath); +int connection_ap_handshake_rewrite_and_attach(edge_connection_t *conn, + origin_circuit_t *circ, + crypt_path_t *cpath); + +/** Possible return values for parse_extended_hostname. */ +typedef enum hostname_type_t { + NORMAL_HOSTNAME, ONION_HOSTNAME, EXIT_HOSTNAME, BAD_HOSTNAME +} hostname_type_t; +hostname_type_t parse_extended_hostname(char *address, int allowdotexit); + +#if defined(HAVE_NET_IF_H) && defined(HAVE_NET_PFVAR_H) +int get_pf_socket(void); +#endif + +#endif + diff --git a/src/or/connection_or.c b/src/or/connection_or.c index e70edc7c8f..4b932ec51e 100644 --- a/src/or/connection_or.c +++ b/src/or/connection_or.c @@ -11,6 +11,22 @@ **/ #include "or.h" +#include "buffers.h" +#include "circuitbuild.h" +#include "command.h" +#include "config.h" +#include "connection.h" +#include "connection_or.h" +#include "control.h" +#include "dirserv.h" +#include "geoip.h" +#include "main.h" +#include "networkstatus.h" +#include "reasons.h" +#include "relay.h" +#include "rephist.h" +#include "router.h" +#include "routerlist.h" static int connection_tls_finish_handshake(or_connection_t *conn); static int connection_or_process_cells_from_inbuf(or_connection_t *conn); @@ -80,10 +96,8 @@ connection_or_clear_identity_map(void) } }); - if (orconn_identity_map) { - digestmap_free(orconn_identity_map, NULL); - orconn_identity_map = NULL; - } + digestmap_free(orconn_identity_map, NULL); + orconn_identity_map = NULL; } /** Change conn->identity_digest to digest, and add conn into @@ -133,7 +147,7 @@ void cell_pack(packed_cell_t *dst, const cell_t *src) { char *dest = dst->body; - *(uint16_t*)dest = htons(src->circ_id); + set_uint16(dest, htons(src->circ_id)); *(uint8_t*)(dest+2) = src->command; memcpy(dest+3, src->payload, CELL_PAYLOAD_SIZE); } @@ -144,7 +158,7 @@ cell_pack(packed_cell_t *dst, const cell_t *src) static void cell_unpack(cell_t *dest, const char *src) { - dest->circ_id = ntohs(*(uint16_t*)(src)); + dest->circ_id = ntohs(get_uint16(src)); dest->command = *(uint8_t*)(src+2); memcpy(dest->payload, src+3, CELL_PAYLOAD_SIZE); } @@ -187,66 +201,6 @@ connection_or_reached_eof(or_connection_t *conn) return 0; } -/** Read conn's inbuf. If the http response from the proxy is all - * here, make sure it's good news, and begin the tls handshake. If - * it's bad news, close the connection and return -1. Else return 0 - * and hope for better luck next time. - */ -static int -connection_or_read_proxy_response(or_connection_t *or_conn) -{ - char *headers; - char *reason=NULL; - int status_code; - time_t date_header; - connection_t *conn = TO_CONN(or_conn); - - switch (fetch_from_buf_http(conn->inbuf, - &headers, MAX_HEADERS_SIZE, - NULL, NULL, 10000, 0)) { - case -1: /* overflow */ - log_warn(LD_PROTOCOL, - "Your https proxy sent back an oversized response. Closing."); - return -1; - case 0: - log_info(LD_OR,"https proxy response not all here yet. Waiting."); - return 0; - /* case 1, fall through */ - } - - if (parse_http_response(headers, &status_code, &date_header, - NULL, &reason) < 0) { - log_warn(LD_OR, - "Unparseable headers from proxy (connecting to '%s'). Closing.", - conn->address); - tor_free(headers); - return -1; - } - if (!reason) reason = tor_strdup("[no reason given]"); - - if (status_code == 200) { - log_info(LD_OR, - "HTTPS connect to '%s' successful! (200 %s) Starting TLS.", - conn->address, escaped(reason)); - tor_free(reason); - if (connection_tls_start_handshake(or_conn, 0) < 0) { - /* TLS handshaking error of some kind. */ - connection_mark_for_close(conn); - - return -1; - } - return 0; - } - /* else, bad news on the status code */ - log_warn(LD_OR, - "The https proxy sent back an unexpected status code %d (%s). " - "Closing.", - status_code, escaped(reason)); - tor_free(reason); - connection_mark_for_close(conn); - return -1; -} - /** Handle any new bytes that have come in on connection <b>conn</b>. * If conn is in 'open' state, hand it to * connection_or_process_cells_from_inbuf() @@ -255,11 +209,24 @@ connection_or_read_proxy_response(or_connection_t *or_conn) int connection_or_process_inbuf(or_connection_t *conn) { + int ret; tor_assert(conn); switch (conn->_base.state) { - case OR_CONN_STATE_PROXY_READING: - return connection_or_read_proxy_response(conn); + case OR_CONN_STATE_PROXY_HANDSHAKING: + ret = connection_read_proxy_handshake(TO_CONN(conn)); + + /* start TLS after handshake completion, or deal with error */ + if (ret == 1) { + tor_assert(TO_CONN(conn)->proxy_state == PROXY_CONNECTED); + if (connection_tls_start_handshake(conn, 0) < 0) + ret = -1; + } + if (ret < 0) { + connection_mark_for_close(TO_CONN(conn)); + } + + return ret; case OR_CONN_STATE_OPEN: case OR_CONN_STATE_OR_HANDSHAKING: return connection_or_process_cells_from_inbuf(conn); @@ -285,8 +252,7 @@ connection_or_flushed_some(or_connection_t *conn) /* If we're under the low water mark, add cells until we're just over the * high water mark. */ if (datalen < OR_CONN_LOWWATER) { - ssize_t n = (OR_CONN_HIGHWATER - datalen + CELL_NETWORK_SIZE-1) - / CELL_NETWORK_SIZE; + ssize_t n = CEIL_DIV(OR_CONN_HIGHWATER - datalen, CELL_NETWORK_SIZE); time_t now = approx_time(); while (conn->active_circuits && n > 0) { int flushed; @@ -312,11 +278,7 @@ connection_or_finished_flushing(or_connection_t *conn) assert_connection_ok(TO_CONN(conn),0); switch (conn->_base.state) { - case OR_CONN_STATE_PROXY_FLUSHING: - log_debug(LD_OR,"finished sending CONNECT to proxy."); - conn->_base.state = OR_CONN_STATE_PROXY_READING; - connection_stop_writing(TO_CONN(conn)); - break; + case OR_CONN_STATE_PROXY_HANDSHAKING: case OR_CONN_STATE_OPEN: case OR_CONN_STATE_OR_HANDSHAKING: connection_stop_writing(TO_CONN(conn)); @@ -334,37 +296,34 @@ connection_or_finished_flushing(or_connection_t *conn) int connection_or_finished_connecting(or_connection_t *or_conn) { + int proxy_type; connection_t *conn; tor_assert(or_conn); conn = TO_CONN(or_conn); tor_assert(conn->state == OR_CONN_STATE_CONNECTING); - log_debug(LD_OR,"OR connect() to router at %s:%u finished.", + log_debug(LD_HANDSHAKE,"OR connect() to router at %s:%u finished.", conn->address,conn->port); control_event_bootstrap(BOOTSTRAP_STATUS_HANDSHAKE, 0); - if (get_options()->HttpsProxy) { - char buf[1024]; - char *base64_authenticator=NULL; - const char *authenticator = get_options()->HttpsProxyAuthenticator; + proxy_type = PROXY_NONE; - if (authenticator) { - base64_authenticator = alloc_http_authenticator(authenticator); - if (!base64_authenticator) - log_warn(LD_OR, "Encoding https authenticator failed"); - } - if (base64_authenticator) { - tor_snprintf(buf, sizeof(buf), "CONNECT %s:%d HTTP/1.1\r\n" - "Proxy-Authorization: Basic %s\r\n\r\n", - fmt_addr(&conn->addr), - conn->port, base64_authenticator); - tor_free(base64_authenticator); - } else { - tor_snprintf(buf, sizeof(buf), "CONNECT %s:%d HTTP/1.0\r\n\r\n", - fmt_addr(&conn->addr), conn->port); + if (get_options()->HTTPSProxy) + proxy_type = PROXY_CONNECT; + else if (get_options()->Socks4Proxy) + proxy_type = PROXY_SOCKS4; + else if (get_options()->Socks5Proxy) + proxy_type = PROXY_SOCKS5; + + if (proxy_type != PROXY_NONE) { + /* start proxy handshake */ + if (connection_proxy_connect(conn, proxy_type) < 0) { + connection_mark_for_close(conn); + return -1; } - connection_write_to_buf(buf, strlen(buf), conn); - conn->state = OR_CONN_STATE_PROXY_FLUSHING; + + connection_start_reading(conn); + conn->state = OR_CONN_STATE_PROXY_HANDSHAKING; return 0; } @@ -376,6 +335,78 @@ connection_or_finished_connecting(or_connection_t *or_conn) return 0; } +/** Return 1 if identity digest <b>id_digest</b> is known to be a + * currently or recently running relay. Otherwise return 0. */ +int +connection_or_digest_is_known_relay(const char *id_digest) +{ + if (router_get_consensus_status_by_id(id_digest)) + return 1; /* It's in the consensus: "yes" */ + if (router_get_by_digest(id_digest)) + return 1; /* Not in the consensus, but we have a descriptor for + * it. Probably it was in a recent consensus. "Yes". */ + return 0; +} + +/** Set the per-conn read and write limits for <b>conn</b>. If it's a known + * relay, we will rely on the global read and write buckets, so give it + * per-conn limits that are big enough they'll never matter. But if it's + * not a known relay, first check if we set PerConnBwRate/Burst, then + * check if the consensus sets them, else default to 'big enough'. + * + * If <b>reset</b> is true, set the bucket to be full. Otherwise, just + * clip the bucket if it happens to be <em>too</em> full. + */ +static void +connection_or_update_token_buckets_helper(or_connection_t *conn, int reset, + or_options_t *options) +{ + int rate, burst; /* per-connection rate limiting params */ + if (connection_or_digest_is_known_relay(conn->identity_digest)) { + /* It's in the consensus, or we have a descriptor for it meaning it + * was probably in a recent consensus. It's a recognized relay: + * give it full bandwidth. */ + rate = (int)options->BandwidthRate; + burst = (int)options->BandwidthBurst; + } else { + /* Not a recognized relay. Squeeze it down based on the suggested + * bandwidth parameters in the consensus, but allow local config + * options to override. */ + rate = options->PerConnBWRate ? (int)options->PerConnBWRate : + networkstatus_get_param(NULL, "perconnbwrate", + (int)options->BandwidthRate, 1, INT32_MAX); + burst = options->PerConnBWBurst ? (int)options->PerConnBWBurst : + networkstatus_get_param(NULL, "perconnbwburst", + (int)options->BandwidthBurst, 1, INT32_MAX); + } + + conn->bandwidthrate = rate; + conn->bandwidthburst = burst; + if (reset) { /* set up the token buckets to be full */ + conn->read_bucket = conn->write_bucket = burst; + return; + } + /* If the new token bucket is smaller, take out the extra tokens. + * (If it's larger, don't -- the buckets can grow to reach the cap.) */ + if (conn->read_bucket > burst) + conn->read_bucket = burst; + if (conn->write_bucket > burst) + conn->write_bucket = burst; +} + +/** Either our set of relays or our per-conn rate limits have changed. + * Go through all the OR connections and update their token buckets to make + * sure they don't exceed their maximum values. */ +void +connection_or_update_token_buckets(smartlist_t *conns, or_options_t *options) +{ + SMARTLIST_FOREACH(conns, connection_t *, conn, + { + if (connection_speaks_cells(conn)) + connection_or_update_token_buckets_helper(TO_OR_CONN(conn), 0, options); + }); +} + /** If we don't necessarily know the router we're connecting to, but we * have an addr/port/id_digest, then fill in as much as we can. Start * by checking to see if this describes a router we know. */ @@ -385,11 +416,9 @@ connection_or_init_conn_from_address(or_connection_t *conn, const char *id_digest, int started_here) { - or_options_t *options = get_options(); routerinfo_t *r = router_get_by_digest(id_digest); - conn->bandwidthrate = (int)options->BandwidthRate; - conn->read_bucket = conn->bandwidthburst = (int)options->BandwidthBurst; connection_or_set_identity_digest(conn, id_digest); + connection_or_update_token_buckets_helper(conn, 1, get_options()); conn->_base.port = port; tor_addr_copy(&conn->_base.addr, addr); @@ -442,7 +471,7 @@ connection_or_init_conn_from_address(or_connection_t *conn, * Requires that both input connections are open; not is_bad_for_new_circs, * and not impossibly non-canonical. * - * If </b>forgive_new_connections</b> is true, then we do not call + * If <b>forgive_new_connections</b> is true, then we do not call * <b>a</b>better than <b>b</b> simply because b has no circuits, * unless b is also relatively old. */ @@ -581,11 +610,24 @@ connection_or_get_for_extend(const char *digest, #define TIME_BEFORE_OR_CONN_IS_TOO_OLD (60*60*24*7) /** Given the head of the linked list for all the or_connections with a given - * identity, set elements of that list as is_bad_for_new_circs() as - * appropriate. Helper for connection_or_set_bad_connections(). + * identity, set elements of that list as is_bad_for_new_circs as + * appropriate. Helper for connection_or_set_bad_connections(). + * + * Specifically, we set the is_bad_for_new_circs flag on: + * - all connections if <b>force</b> is true. + * - all connections that are too old. + * - all open non-canonical connections for which a canonical connection + * exists to the same router. + * - all open canonical connections for which a 'better' canonical + * connection exists to the same router. + * - all open non-canonical connections for which a 'better' non-canonical + * connection exists to the same router at the same address. + * + * See connection_or_is_better() for our idea of what makes one OR connection + * better than another. */ static void -connection_or_group_set_badness(or_connection_t *head) +connection_or_group_set_badness(or_connection_t *head, int force) { or_connection_t *or_conn = NULL, *best = NULL; int n_old = 0, n_inprogress = 0, n_canonical = 0, n_other = 0; @@ -597,8 +639,9 @@ connection_or_group_set_badness(or_connection_t *head) if (or_conn->_base.marked_for_close || or_conn->is_bad_for_new_circs) continue; - if (or_conn->_base.timestamp_created + TIME_BEFORE_OR_CONN_IS_TOO_OLD - < now) { + if (force || + or_conn->_base.timestamp_created + TIME_BEFORE_OR_CONN_IS_TOO_OLD + < now) { log_info(LD_OR, "Marking OR conn to %s:%d as too old for new circuits " "(fd %d, %d secs old).", @@ -631,7 +674,7 @@ connection_or_group_set_badness(or_connection_t *head) /* We have at least one open canonical connection to this router, * and this one is open but not canonical. Mark it bad. */ log_info(LD_OR, - "Marking OR conn to %s:%d as too old for new circuits: " + "Marking OR conn to %s:%d as unsuitable for new circuits: " "(fd %d, %d secs old). It is not canonical, and we have " "another connection to that OR that is.", or_conn->_base.address, or_conn->_base.port, or_conn->_base.s, @@ -671,7 +714,7 @@ connection_or_group_set_badness(or_connection_t *head) even when we're being forgiving. */ if (best->is_canonical) { log_info(LD_OR, - "Marking OR conn to %s:%d as too old for new circuits: " + "Marking OR conn to %s:%d as unsuitable for new circuits: " "(fd %d, %d secs old). We have a better canonical one " "(fd %d; %d secs old).", or_conn->_base.address, or_conn->_base.port, or_conn->_base.s, @@ -681,9 +724,9 @@ connection_or_group_set_badness(or_connection_t *head) } else if (!tor_addr_compare(&or_conn->real_addr, &best->real_addr, CMP_EXACT)) { log_info(LD_OR, - "Marking OR conn to %s:%d as too old for new circuits: " - "(fd %d, %d secs old). We have a better one " - "(fd %d; %d secs old).", + "Marking OR conn to %s:%d as unsuitable for new circuits: " + "(fd %d, %d secs old). We have a better one with the " + "same address (fd %d; %d secs old).", or_conn->_base.address, or_conn->_base.port, or_conn->_base.s, (int)(now - or_conn->_base.timestamp_created), best->_base.s, (int)(now - best->_base.timestamp_created)); @@ -693,27 +736,20 @@ connection_or_group_set_badness(or_connection_t *head) } } -/** Go through all the OR connections, and set the is_bad_for_new_circs - * flag on: - * - all connections that are too old. - * - all open non-canonical connections for which a canonical connection - * exists to the same router. - * - all open canonical connections for which a 'better' canonical - * connection exists to the same router. - * - all open non-canonical connections for which a 'better' non-canonical - * connection exists to the same router at the same address. - * - * See connection_or_is_better() for our idea of what makes one OR connection - * better than another. +/** Go through all the OR connections (or if <b>digest</b> is non-NULL, just + * the OR connections with that digest), and set the is_bad_for_new_circs + * flag based on the rules in connection_or_group_set_badness() (or just + * always set it if <b>force</b> is true). */ void -connection_or_set_bad_connections(void) +connection_or_set_bad_connections(const char *digest, int force) { if (!orconn_identity_map) return; DIGESTMAP_FOREACH(orconn_identity_map, identity, or_connection_t *, conn) { - connection_or_group_set_badness(conn); + if (!digest || !memcmp(digest, conn->identity_digest, DIGEST_LEN)) + connection_or_group_set_badness(conn, force); } DIGESTMAP_FOREACH_END; } @@ -753,6 +789,7 @@ connection_or_connect(const tor_addr_t *_addr, uint16_t port, or_connection_t *conn; or_options_t *options = get_options(); int socket_error = 0; + int using_proxy = 0; tor_addr_t addr; tor_assert(_addr); @@ -771,19 +808,27 @@ connection_or_connect(const tor_addr_t *_addr, uint16_t port, conn->_base.state = OR_CONN_STATE_CONNECTING; control_event_or_conn_status(conn, OR_CONN_EVENT_LAUNCHED, 0); - if (options->HttpsProxy) { - /* we shouldn't connect directly. use the https proxy instead. */ - tor_addr_from_ipv4h(&addr, options->HttpsProxyAddr); - port = options->HttpsProxyPort; + /* use a proxy server if available */ + if (options->HTTPSProxy) { + using_proxy = 1; + tor_addr_copy(&addr, &options->HTTPSProxyAddr); + port = options->HTTPSProxyPort; + } else if (options->Socks4Proxy) { + using_proxy = 1; + tor_addr_copy(&addr, &options->Socks4ProxyAddr); + port = options->Socks4ProxyPort; + } else if (options->Socks5Proxy) { + using_proxy = 1; + tor_addr_copy(&addr, &options->Socks5ProxyAddr); + port = options->Socks5ProxyPort; } switch (connection_connect(TO_CONN(conn), conn->_base.address, &addr, port, &socket_error)) { case -1: /* If the connection failed immediately, and we're using - * an https proxy, our https proxy is down. Don't blame the - * Tor server. */ - if (!options->HttpsProxy) + * a proxy, our proxy is down. Don't blame the Tor server. */ + if (!using_proxy) entry_guard_register_connect_status(conn->identity_digest, 0, 1, time(NULL)); connection_or_connect_failed(conn, @@ -792,7 +837,7 @@ connection_or_connect(const tor_addr_t *_addr, uint16_t port, connection_free(TO_CONN(conn)); return NULL; case 0: - connection_watch_events(TO_CONN(conn), EV_READ | EV_WRITE); + connection_watch_events(TO_CONN(conn), READ_EVENT | WRITE_EVENT); /* writable indicates finish, readable indicates broken link, error indicates broken link on windows */ return conn; @@ -819,13 +864,14 @@ connection_tls_start_handshake(or_connection_t *conn, int receiving) { conn->_base.state = OR_CONN_STATE_TLS_HANDSHAKING; conn->tls = tor_tls_new(conn->_base.s, receiving); - tor_tls_set_logged_address(conn->tls, escaped_safe_str(conn->_base.address)); + tor_tls_set_logged_address(conn->tls, // XXX client and relay? + escaped_safe_str(conn->_base.address)); if (!conn->tls) { log_warn(LD_BUG,"tor_tls_new failed. Closing."); return -1; } connection_start_reading(TO_CONN(conn)); - log_debug(LD_OR,"starting TLS handshake on fd %d", conn->_base.s); + log_debug(LD_HANDSHAKE,"starting TLS handshake on fd %d", conn->_base.s); note_crypto_pk_op(receiving ? TLS_HANDSHAKE_S : TLS_HANDSHAKE_C); if (connection_tls_continue_handshake(conn) < 0) { @@ -932,23 +978,26 @@ connection_or_nonopen_was_started_here(or_connection_t *conn) * return -1 if he is lying, broken, or otherwise something is wrong. * * If we initiated this connection (<b>started_here</b> is true), make sure - * the other side sent sent a correctly formed certificate. If I initiated the + * the other side sent a correctly formed certificate. If I initiated the * connection, make sure it's the right guy. * * Otherwise (if we _didn't_ initiate this connection), it's okay for * the certificate to be weird or absent. * * If we return 0, and the certificate is as expected, write a hash of the - * identity key into digest_rcvd, which must have DIGEST_LEN space in it. (If - * we return -1 this buffer is undefined.) If the certificate is invalid - * or missing on an incoming connection, we return 0 and set digest_rcvd to - * DIGEST_LEN 0 bytes. + * identity key into <b>digest_rcvd_out</b>, which must have DIGEST_LEN + * space in it. + * If the certificate is invalid or missing on an incoming connection, + * we return 0 and set <b>digest_rcvd_out</b> to DIGEST_LEN NUL bytes. + * (If we return -1, the contents of this buffer are undefined.) * * As side effects, * 1) Set conn->circ_id_type according to tor-spec.txt. * 2) If we're an authdirserver and we initiated the connection: drop all * descriptors that claim to be on that IP/port but that aren't * this guy; and note that this guy is reachable. + * 3) If this is a bridge and we didn't configure its identity + * fingerprint, remember the keyid we just learned. */ static int connection_or_check_valid_tls_handshake(or_connection_t *conn, @@ -959,19 +1008,23 @@ connection_or_check_valid_tls_handshake(or_connection_t *conn, or_options_t *options = get_options(); int severity = server_mode(options) ? LOG_PROTOCOL_WARN : LOG_WARN; const char *safe_address = - started_here ? conn->_base.address : safe_str(conn->_base.address); + started_here ? conn->_base.address : + safe_str_client(conn->_base.address); const char *conn_type = started_here ? "outgoing" : "incoming"; + crypto_pk_env_t *our_identity = + started_here ? get_tlsclient_identity_key() : + get_server_identity_key(); int has_cert = 0, has_identity=0; check_no_tls_errors(); has_cert = tor_tls_peer_has_cert(conn->tls); if (started_here && !has_cert) { - log_info(LD_PROTOCOL,"Tried connecting to router at %s:%d, but it didn't " + log_info(LD_HANDSHAKE,"Tried connecting to router at %s:%d, but it didn't " "send a cert! Closing.", safe_address, conn->_base.port); return -1; } else if (!has_cert) { - log_debug(LD_PROTOCOL,"Got incoming connection with no certificate. " + log_debug(LD_HANDSHAKE,"Got incoming connection with no certificate. " "That's ok."); } check_no_tls_errors(); @@ -980,15 +1033,16 @@ connection_or_check_valid_tls_handshake(or_connection_t *conn, int v = tor_tls_verify(started_here?severity:LOG_INFO, conn->tls, &identity_rcvd); if (started_here && v<0) { - log_fn(severity,LD_OR,"Tried connecting to router at %s:%d: It" + log_fn(severity,LD_HANDSHAKE,"Tried connecting to router at %s:%d: It" " has a cert but it's invalid. Closing.", safe_address, conn->_base.port); return -1; } else if (v<0) { - log_info(LD_PROTOCOL,"Incoming connection gave us an invalid cert " + log_info(LD_HANDSHAKE,"Incoming connection gave us an invalid cert " "chain; ignoring."); } else { - log_debug(LD_OR,"The certificate seems to be valid on %s connection " + log_debug(LD_HANDSHAKE, + "The certificate seems to be valid on %s connection " "with %s:%d", conn_type, safe_address, conn->_base.port); } check_no_tls_errors(); @@ -997,7 +1051,7 @@ connection_or_check_valid_tls_handshake(or_connection_t *conn, if (identity_rcvd) { has_identity = 1; crypto_pk_get_digest(identity_rcvd, digest_rcvd_out); - if (crypto_pk_cmp_keys(get_identity_key(), identity_rcvd)<0) { + if (crypto_pk_cmp_keys(our_identity, identity_rcvd)<0) { conn->circ_id_type = CIRC_ID_TYPE_LOWER; } else { conn->circ_id_type = CIRC_ID_TYPE_HIGHER; @@ -1015,9 +1069,13 @@ connection_or_check_valid_tls_handshake(or_connection_t *conn, conn->nickname[0] = '$'; base16_encode(conn->nickname+1, HEX_DIGEST_LEN+1, conn->identity_digest, DIGEST_LEN); - log_info(LD_OR, "Connected to router %s at %s:%d without knowing " + log_info(LD_HANDSHAKE, "Connected to router %s at %s:%d without knowing " "its key. Hoping for the best.", conn->nickname, conn->_base.address, conn->_base.port); + /* if it's a bridge and we didn't know its identity fingerprint, now + * we do -- remember it for future attempts. */ + learned_router_identity(&conn->_base.addr, conn->_base.port, + digest_rcvd_out); } if (started_here) { @@ -1031,7 +1089,7 @@ connection_or_check_valid_tls_handshake(or_connection_t *conn, base16_encode(seen, sizeof(seen), digest_rcvd_out, DIGEST_LEN); base16_encode(expected, sizeof(expected), conn->identity_digest, DIGEST_LEN); - log_fn(severity, LD_OR, + log_fn(severity, LD_HANDSHAKE, "Tried connecting to router at %s:%d, but identity key was not " "as expected: wanted %s but got %s.", conn->_base.address, conn->_base.port, expected, seen); @@ -1044,9 +1102,6 @@ connection_or_check_valid_tls_handshake(or_connection_t *conn, as_advertised = 0; } if (authdir_mode_tests_reachability(options)) { - /* We initiated this connection to address:port. Drop all routers - * with the same address:port and a different key. - */ dirserv_orconn_tls_done(conn->_base.address, conn->_base.port, digest_rcvd_out, as_advertised); } @@ -1073,8 +1128,8 @@ connection_tls_finish_handshake(or_connection_t *conn) char digest_rcvd[DIGEST_LEN]; int started_here = connection_or_nonopen_was_started_here(conn); - log_debug(LD_OR,"tls handshake with %s done. verifying.", - safe_str(conn->_base.address)); + log_debug(LD_HANDSHAKE,"tls handshake with %s done. verifying.", + safe_str_client(conn->_base.address)); directory_set_dirty(); @@ -1082,6 +1137,8 @@ connection_tls_finish_handshake(or_connection_t *conn) digest_rcvd) < 0) return -1; + circuit_build_times_network_is_live(&circ_times); + if (tor_tls_used_v1_handshake(conn->tls)) { conn->link_proto = 1; if (!started_here) { @@ -1117,7 +1174,8 @@ connection_init_or_handshake_state(or_connection_t *conn, int started_here) void or_handshake_state_free(or_handshake_state_t *state) { - tor_assert(state); + if (!state) + return; memset(state, 0xBE, sizeof(or_handshake_state_t)); tor_free(state); } @@ -1134,6 +1192,7 @@ connection_or_set_state_open(or_connection_t *conn) control_event_or_conn_status(conn, OR_CONN_EVENT_CONNECTED, 0); if (started_here) { + circuit_build_times_network_is_live(&circ_times); rep_hist_note_connect_succeeded(conn->identity_digest, now); if (entry_guard_register_connect_status(conn->identity_digest, 1, 0, now) < 0) { @@ -1158,10 +1217,10 @@ connection_or_set_state_open(or_connection_t *conn) } } } - if (conn->handshake_state) { - or_handshake_state_free(conn->handshake_state); - conn->handshake_state = NULL; - } + + or_handshake_state_free(conn->handshake_state); + conn->handshake_state = NULL; + connection_start_reading(TO_CONN(conn)); circuit_n_conn_done(conn, 1); /* send the pending creates, if any. */ @@ -1235,6 +1294,7 @@ connection_or_process_cells_from_inbuf(or_connection_t *conn) if (connection_fetch_var_cell_from_buf(conn, &var_cell)) { if (!var_cell) return 0; /* not yet. */ + circuit_build_times_network_is_live(&circ_times); command_process_var_cell(var_cell, conn); var_cell_free(var_cell); } else { @@ -1244,6 +1304,7 @@ connection_or_process_cells_from_inbuf(or_connection_t *conn) available? */ return 0; /* not yet */ + circuit_build_times_network_is_live(&circ_times); connection_fetch_from_buf(buf, CELL_NETWORK_SIZE, TO_CONN(conn)); /* retrieve cell info from buf (create the host-order struct from the @@ -1274,10 +1335,6 @@ connection_or_send_destroy(circid_t circ_id, or_connection_t *conn, int reason) cell.payload[0] = (uint8_t) reason; log_debug(LD_OR,"Sending destroy (circID %d).", circ_id); - /* XXXX It's possible that under some circumstances, we want the destroy - * to take precedence over other data waiting on the circuit's cell queue. - */ - connection_or_write_cell_to_buf(&cell, conn); return 0; } diff --git a/src/or/connection_or.h b/src/or/connection_or.h new file mode 100644 index 0000000000..70ef96a335 --- /dev/null +++ b/src/or/connection_or.h @@ -0,0 +1,57 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file connection_or.h + * \brief Header file for connection_or.c. + **/ + +#ifndef _TOR_CONNECTION_OR_H +#define _TOR_CONNECTION_OR_H + +void connection_or_remove_from_identity_map(or_connection_t *conn); +void connection_or_clear_identity_map(void); +or_connection_t *connection_or_get_for_extend(const char *digest, + const tor_addr_t *target_addr, + const char **msg_out, + int *launch_out); +void connection_or_set_bad_connections(const char *digest, int force); + +int connection_or_reached_eof(or_connection_t *conn); +int connection_or_process_inbuf(or_connection_t *conn); +int connection_or_flushed_some(or_connection_t *conn); +int connection_or_finished_flushing(or_connection_t *conn); +int connection_or_finished_connecting(or_connection_t *conn); +int connection_or_digest_is_known_relay(const char *id_digest); +void connection_or_update_token_buckets(smartlist_t *conns, + or_options_t *options); + +void connection_or_connect_failed(or_connection_t *conn, + int reason, const char *msg); +or_connection_t *connection_or_connect(const tor_addr_t *addr, uint16_t port, + const char *id_digest); + +int connection_tls_start_handshake(or_connection_t *conn, int receiving); +int connection_tls_continue_handshake(or_connection_t *conn); + +void or_handshake_state_free(or_handshake_state_t *state); +int connection_or_set_state_open(or_connection_t *conn); +void connection_or_write_cell_to_buf(const cell_t *cell, + or_connection_t *conn); +void connection_or_write_var_cell_to_buf(const var_cell_t *cell, + or_connection_t *conn); +int connection_or_send_destroy(circid_t circ_id, or_connection_t *conn, + int reason); +int connection_or_send_netinfo(or_connection_t *conn); +int is_or_protocol_version_known(uint16_t version); + +void cell_pack(packed_cell_t *dest, const cell_t *src); +void var_cell_pack_header(const var_cell_t *cell, char *hdr_out); +var_cell_t *var_cell_new(uint16_t payload_len); +void var_cell_free(var_cell_t *cell); + +#endif + diff --git a/src/or/control.c b/src/or/control.c index eae6170368..9f7739a402 100644 --- a/src/or/control.c +++ b/src/or/control.c @@ -11,6 +11,26 @@ #define CONTROL_PRIVATE #include "or.h" +#include "buffers.h" +#include "circuitbuild.h" +#include "circuitlist.h" +#include "circuituse.h" +#include "config.h" +#include "connection.h" +#include "connection_edge.h" +#include "control.h" +#include "directory.h" +#include "dirserv.h" +#include "dnsserv.h" +#include "geoip.h" +#include "hibernate.h" +#include "main.h" +#include "networkstatus.h" +#include "policies.h" +#include "reasons.h" +#include "router.h" +#include "routerlist.h" +#include "routerparse.h" /** Yield true iff <b>s</b> is the state of a control_connection_t that has * finished authentication and is accepting commands. */ @@ -43,7 +63,8 @@ #define EVENT_STREAM_BANDWIDTH_USED 0x0014 #define EVENT_CLIENTS_SEEN 0x0015 #define EVENT_NEWCONSENSUS 0x0016 -#define _EVENT_MAX 0x0016 +#define EVENT_BUILDTIMEOUT_SET 0x0017 +#define _EVENT_MAX 0x0017 /* If _EVENT_MAX ever hits 0x0020, we need to make the mask wider. */ /** Bitfield: The bit 1<<e is set if <b>any</b> open control @@ -54,13 +75,9 @@ **/ typedef uint32_t event_mask_t; -/** An event mask of all the events that controller with the LONG_NAMES option - * set is interested in receiving. */ -static event_mask_t global_event_mask1long = 0; - -/** An event mask of all the events that controller with the SHORT_NAMES option - * set is interested in receiving. */ -static event_mask_t global_event_mask1short = 0; +/** An event mask of all the events that any controller is interested in + * receiving. */ +static event_mask_t global_event_mask = 0; /** True iff we have disabled log messages from being sent to the controller */ static int disable_log_messages = 0; @@ -68,13 +85,7 @@ static int disable_log_messages = 0; /** Macro: true if any control connection is interested in events of type * <b>e</b>. */ #define EVENT_IS_INTERESTING(e) \ - ((global_event_mask1long|global_event_mask1short) & (1<<(e))) -/** Macro: true if any control connection with the LONG_NAMES option is - * interested in events of type <b>e</b>. */ -#define EVENT_IS_INTERESTING1L(e) (global_event_mask1long & (1<<(e))) -/** Macro: true if any control connection with the SHORT_NAMES option is - * interested in events of type <b>e</b>. */ -#define EVENT_IS_INTERESTING1S(e) (global_event_mask1short & (1<<(e))) + (global_event_mask & (1<<(e))) /** If we're using cookie-type authentication, how long should our cookies be? */ @@ -95,25 +106,13 @@ static char authentication_cookie[AUTHENTICATION_COOKIE_LEN]; * of this so we can respond to getinfo status/bootstrap-phase queries. */ static char last_sent_bootstrap_message[BOOTSTRAP_MSG_LEN]; -/** Flag for event_format_t. Indicates that we should use the old - * name format of nickname|hexdigest - */ -#define SHORT_NAMES 1 -/** Flag for event_format_t. Indicates that we should use the new - * name format of $hexdigest[=~]nickname +/** Flag for event_format_t. Indicates that we should use the one standard + format. */ -#define LONG_NAMES 2 -#define ALL_NAMES (SHORT_NAMES|LONG_NAMES) -/** Flag for event_format_t. Indicates that we should use the new event - * format where extra event fields are allowed using a NAME=VAL format. */ -#define EXTENDED_FORMAT 4 -/** Flag for event_format_t. Indicates that we are using the old event format - * where extra fields aren't allowed. */ -#define NONEXTENDED_FORMAT 8 -#define ALL_FORMATS (EXTENDED_FORMAT|NONEXTENDED_FORMAT) +#define ALL_FORMATS 1 /** Bit field of flags to select how to format a controller event. Recognized - * flags are SHORT_NAMES, LONG_NAMES, EXTENDED_FORMAT, NONEXTENDED_FORMAT. */ + * flag is ALL_FORMATS. */ typedef int event_format_t; static void connection_printf_to_buf(control_connection_t *conn, @@ -123,9 +122,6 @@ static void send_control_done(control_connection_t *conn); static void send_control_event(uint16_t event, event_format_t which, const char *format, ...) CHECK_PRINTF(3,4); -static void send_control_event_extended(uint16_t event, event_format_t which, - const char *format, ...) - CHECK_PRINTF(3,4); static int handle_control_setconf(control_connection_t *conn, uint32_t len, char *body); static int handle_control_resetconf(control_connection_t *conn, uint32_t len, @@ -139,8 +135,6 @@ static int handle_control_setevents(control_connection_t *conn, uint32_t len, static int handle_control_authenticate(control_connection_t *conn, uint32_t len, const char *body); -static int handle_control_saveconf(control_connection_t *conn, uint32_t len, - const char *body); static int handle_control_signal(control_connection_t *conn, uint32_t len, const char *body); static int handle_control_mapaddress(control_connection_t *conn, uint32_t len, @@ -174,7 +168,7 @@ static int handle_control_usefeature(control_connection_t *conn, const char *body); static int write_stream_target_to_buf(edge_connection_t *conn, char *buf, size_t len); -static void orconn_target_get_name(int long_names, char *buf, size_t len, +static void orconn_target_get_name(char *buf, size_t len, or_connection_t *conn); static char *get_cookie_file(void); @@ -214,25 +208,19 @@ control_update_global_event_mask(void) { smartlist_t *conns = get_connection_array(); event_mask_t old_mask, new_mask; - old_mask = global_event_mask1short; - old_mask |= global_event_mask1long; + old_mask = global_event_mask; - global_event_mask1short = 0; - global_event_mask1long = 0; + global_event_mask = 0; SMARTLIST_FOREACH(conns, connection_t *, _conn, { if (_conn->type == CONN_TYPE_CONTROL && STATE_IS_OPEN(_conn->state)) { control_connection_t *conn = TO_CONTROL_CONN(_conn); - if (conn->use_long_names) - global_event_mask1long |= conn->event_mask; - else - global_event_mask1short |= conn->event_mask; + global_event_mask |= conn->event_mask; } }); - new_mask = global_event_mask1short; - new_mask |= global_event_mask1long; + new_mask = global_event_mask; /* Handle the aftermath. Set up the log callback to tell us only what * we want to hear...*/ @@ -312,7 +300,7 @@ connection_write_str_to_buf(const char *s, control_connection_t *conn) /** Given a <b>len</b>-character string in <b>data</b>, made of lines * terminated by CRLF, allocate a new string in *<b>out</b>, and copy the * contents of <b>data</b> into *<b>out</b>, adding a period before any period - * that that appears at the start of a line, and adding a period-CRLF line at + * that appears at the start of a line, and adding a period-CRLF line at * the end. Replace all LF characters sequences with CRLF. Return the number * of bytes in *<b>out</b>. */ @@ -542,28 +530,15 @@ send_control_event_string(uint16_t event, event_format_t which, const char *msg) { smartlist_t *conns = get_connection_array(); + (void)which; tor_assert(event >= _EVENT_MIN && event <= _EVENT_MAX); - SMARTLIST_FOREACH(conns, connection_t *, conn, - { + SMARTLIST_FOREACH_BEGIN(conns, connection_t *, conn) { if (conn->type == CONN_TYPE_CONTROL && !conn->marked_for_close && conn->state == CONTROL_CONN_STATE_OPEN) { control_connection_t *control_conn = TO_CONTROL_CONN(conn); - if (control_conn->use_long_names) { - if (!(which & LONG_NAMES)) - continue; - } else { - if (!(which & SHORT_NAMES)) - continue; - } - if (control_conn->use_extended_events) { - if (!(which & EXTENDED_FORMAT)) - continue; - } else { - if (!(which & NONEXTENDED_FORMAT)) - continue; - } + if (control_conn->event_mask & (1<<event)) { int is_err = 0; connection_write_to_buf(msg, strlen(msg), TO_CONN(control_conn)); @@ -579,7 +554,7 @@ send_control_event_string(uint16_t event, event_format_t which, connection_handle_write(TO_CONN(control_conn), 1); } } - }); + } SMARTLIST_FOREACH_END(conn); } /** Helper for send_control1_event and send_control1_event_extended: @@ -587,22 +562,17 @@ send_control_event_string(uint16_t event, event_format_t which, * <b>event</b>. The event's body is created by the printf-style format in * <b>format</b>, and other arguments as provided. * - * If <b>extended</b> is true, and the format contains a single '@' character, - * it will be replaced with a space and all text after that character will be - * sent only to controllers that have enabled extended events. - * * Currently the length of the message is limited to 1024 (including the * ending \\r\\n\\0). */ static void -send_control_event_impl(uint16_t event, event_format_t which, int extended, - const char *format, va_list ap) +send_control_event_impl(uint16_t event, event_format_t which, + const char *format, va_list ap) { /* This is just a little longer than the longest allowed log message */ #define SEND_CONTROL1_EVENT_BUFFERSIZE 10064 int r; char buf[SEND_CONTROL1_EVENT_BUFFERSIZE]; size_t len; - char *cp; r = tor_vsnprintf(buf, sizeof(buf), format, ap); if (r<0) { @@ -618,15 +588,7 @@ send_control_event_impl(uint16_t event, event_format_t which, int extended, buf[SEND_CONTROL1_EVENT_BUFFERSIZE-3] = '\r'; } - if (extended && (cp = strchr(buf, '@'))) { - which &= ~ALL_FORMATS; - *cp = ' '; - send_control_event_string(event, which|EXTENDED_FORMAT, buf); - memcpy(cp, "\r\n\0", 3); - send_control_event_string(event, which|NONEXTENDED_FORMAT, buf); - } else { - send_control_event_string(event, which|ALL_FORMATS, buf); - } + send_control_event_string(event, which|ALL_FORMATS, buf); } /** Send an event to all v1 controllers that are listening for code @@ -641,27 +603,7 @@ send_control_event(uint16_t event, event_format_t which, { va_list ap; va_start(ap, format); - send_control_event_impl(event, which, 0, format, ap); - va_end(ap); -} - -/** Send an event to all v1 controllers that are listening for code - * <b>event</b>. The event's body is created by the printf-style format in - * <b>format</b>, and other arguments as provided. - * - * If the format contains a single '@' character, it will be replaced with a - * space and all text after that character will be sent only to controllers - * that have enabled extended events. - * - * Currently the length of the message is limited to 1024 (including the - * ending \\n\\r\\0. */ -static void -send_control_event_extended(uint16_t event, event_format_t which, - const char *format, ...) -{ - va_list ap; - va_start(ap, format); - send_control_event_impl(event, which, 1, format, ap); + send_control_event_impl(event, which, format, ap); va_end(ap); } @@ -907,36 +849,37 @@ handle_control_loadconf(control_connection_t *conn, uint32_t len, retval = options_init_from_string(body, CMD_RUN_TOR, NULL, &errstring); - if (retval != SETOPT_OK) { + if (retval != SETOPT_OK) log_warn(LD_CONTROL, "Controller gave us config file that didn't validate: %s", errstring); - switch (retval) { - case SETOPT_ERR_PARSE: - msg = "552 Invalid config file"; - break; - case SETOPT_ERR_TRANSITION: - msg = "553 Transition not allowed"; - break; - case SETOPT_ERR_SETTING: - msg = "553 Unable to set option"; - break; - case SETOPT_ERR_MISC: - default: - msg = "550 Unable to load config"; - break; - case SETOPT_OK: - tor_fragile_assert(); - break; - } + + switch (retval) { + case SETOPT_ERR_PARSE: + msg = "552 Invalid config file"; + break; + case SETOPT_ERR_TRANSITION: + msg = "553 Transition not allowed"; + break; + case SETOPT_ERR_SETTING: + msg = "553 Unable to set option"; + break; + case SETOPT_ERR_MISC: + default: + msg = "550 Unable to load config"; + break; + case SETOPT_OK: + break; + } + if (msg) { if (errstring) connection_printf_to_buf(conn, "%s: %s\r\n", msg, errstring); else connection_printf_to_buf(conn, "%s\r\n", msg); - tor_free(errstring); - return 0; + } else { + send_control_done(conn); } - send_control_done(conn); + tor_free(errstring); return 0; } @@ -948,7 +891,6 @@ handle_control_setevents(control_connection_t *conn, uint32_t len, { uint16_t event_code; uint32_t event_mask = 0; - unsigned int extended = 0; smartlist_t *events = smartlist_create(); (void) len; @@ -958,7 +900,6 @@ handle_control_setevents(control_connection_t *conn, uint32_t len, SMARTLIST_FOREACH_BEGIN(events, const char *, ev) { if (!strcasecmp(ev, "EXTENDED")) { - extended = 1; continue; } else if (!strcasecmp(ev, "CIRC")) event_code = EVENT_CIRCUIT_STATUS; @@ -1002,6 +943,8 @@ handle_control_setevents(control_connection_t *conn, uint32_t len, event_code = EVENT_CLIENTS_SEEN; else if (!strcasecmp(ev, "NEWCONSENSUS")) event_code = EVENT_NEWCONSENSUS; + else if (!strcasecmp(ev, "BUILDTIMEOUT_SET")) + event_code = EVENT_BUILDTIMEOUT_SET; else { connection_printf_to_buf(conn, "552 Unrecognized event \"%s\"\r\n", ev); @@ -1016,8 +959,6 @@ handle_control_setevents(control_connection_t *conn, uint32_t len, smartlist_free(events); conn->event_mask = event_mask; - if (extended) - conn->use_extended_events = 1; control_update_global_event_mask(); send_control_done(conn); @@ -1281,7 +1222,9 @@ handle_control_signal(control_connection_t *conn, uint32_t len, /* Flush the "done" first if the signal might make us shut down. */ if (sig == SIGTERM || sig == SIGINT) connection_handle_write(TO_CONN(conn), 1); - control_signal_act(sig); + + process_signal(sig); + return 0; } @@ -1328,7 +1271,7 @@ handle_control_mapaddress(control_connection_t *conn, uint32_t len, smartlist_add(reply, ans); log_warn(LD_CONTROL, "Unable to allocate address for '%s' in MapAddress msg", - safe_str(line)); + safe_str_client(line)); } else { tor_snprintf(ans, anslen, "250-%s=%s", address, to); smartlist_add(reply, ans); @@ -1345,7 +1288,8 @@ handle_control_mapaddress(control_connection_t *conn, uint32_t len, "not of expected form 'foo=bar'.", line); smartlist_add(reply, ans); log_info(LD_CONTROL, "Skipping MapAddress '%s': wrong " - "number of items.", safe_str(line)); + "number of items.", + safe_str_client(line)); } SMARTLIST_FOREACH(elts, char *, cp, tor_free(cp)); smartlist_clear(elts); @@ -1374,13 +1318,15 @@ handle_control_mapaddress(control_connection_t *conn, uint32_t len, * trivial-to-implement questions. */ static int getinfo_helper_misc(control_connection_t *conn, const char *question, - char **answer) + char **answer, const char **errmsg) { (void) conn; if (!strcmp(question, "version")) { *answer = tor_strdup(get_version()); } else if (!strcmp(question, "config-file")) { *answer = tor_strdup(get_torrc_fname()); + } else if (!strcmp(question, "config-text")) { + *answer = options_dump(get_options(), 1); } else if (!strcmp(question, "info/names")) { *answer = list_getinfo_options(); } else if (!strcmp(question, "events/names")) { @@ -1392,15 +1338,19 @@ getinfo_helper_misc(control_connection_t *conn, const char *question, *answer = tor_strdup("VERBOSE_NAMES EXTENDED_EVENTS"); } else if (!strcmp(question, "address")) { uint32_t addr; - if (router_pick_published_address(get_options(), &addr) < 0) + if (router_pick_published_address(get_options(), &addr) < 0) { + *errmsg = "Address unknown"; return -1; + } *answer = tor_dup_ip(addr); } else if (!strcmp(question, "dir-usage")) { *answer = directory_dump_request_log(); } else if (!strcmp(question, "fingerprint")) { routerinfo_t *me = router_get_my_routerinfo(); - if (!me) + if (!me) { + *errmsg = "No routerdesc known; am I really a server?"; return -1; + } *answer = tor_malloc(HEX_DIGEST_LEN+1); base16_encode(*answer, HEX_DIGEST_LEN+1, me->cache_info.identity_digest, DIGEST_LEN); @@ -1461,8 +1411,10 @@ munge_extrainfo_into_routerinfo(const char *ri_body, signed_descriptor_t *ri, * directory information. */ static int getinfo_helper_dir(control_connection_t *control_conn, - const char *question, char **answer) + const char *question, char **answer, + const char **errmsg) { + (void) control_conn; if (!strcmpstart(question, "desc/id/")) { routerinfo_t *ri = router_get_by_hexdigest(question+strlen("desc/id/")); if (ri) { @@ -1537,6 +1489,7 @@ getinfo_helper_dir(control_connection_t *control_conn, log_warn(LD_CONTROL, "getinfo '%s': %s", question, msg); smartlist_free(descs); tor_free(url); + *errmsg = msg; return -1; } SMARTLIST_FOREACH(descs, signed_descriptor_t *, sd, @@ -1587,7 +1540,7 @@ getinfo_helper_dir(control_connection_t *control_conn, } } else if (!strcmp(question, "dir/status-vote/current/consensus")) { /* v3 */ if (directory_caches_dir_info(get_options())) { - const cached_dir_t *consensus = dirserv_get_consensus(); + const cached_dir_t *consensus = dirserv_get_consensus("ns"); if (consensus) *answer = tor_strdup(consensus->dir); } @@ -1598,10 +1551,8 @@ getinfo_helper_dir(control_connection_t *control_conn, } } else if (!strcmp(question, "network-status")) { /* v1 */ routerlist_t *routerlist = router_get_routerlist(); - int verbose = control_conn->use_long_names; if (!routerlist || !routerlist->routers || - list_server_status_v1(routerlist->routers, answer, - verbose ? 2 : 1) < 0) { + list_server_status_v1(routerlist->routers, answer, 1) < 0) { return -1; } } else if (!strcmpstart(question, "extra-info/digest/")) { @@ -1635,8 +1586,10 @@ getinfo_helper_dir(control_connection_t *control_conn, * current states of things we send events about. */ static int getinfo_helper_events(control_connection_t *control_conn, - const char *question, char **answer) + const char *question, char **answer, + const char **errmsg) { + (void) control_conn; if (!strcmp(question, "circuit-status")) { circuit_t *circ; smartlist_t *status = smartlist_create(); @@ -1647,10 +1600,9 @@ getinfo_helper_events(control_connection_t *control_conn, const char *purpose; if (! CIRCUIT_IS_ORIGIN(circ) || circ->marked_for_close) continue; - if (control_conn->use_long_names) - path = circuit_list_path_for_controller(TO_ORIGIN_CIRCUIT(circ)); - else - path = circuit_list_path(TO_ORIGIN_CIRCUIT(circ),0); + + path = circuit_list_path_for_controller(TO_ORIGIN_CIRCUIT(circ)); + if (circ->state == CIRCUIT_STATE_OPEN) state = "BUILT"; else if (strlen(path)) @@ -1728,8 +1680,7 @@ getinfo_helper_events(control_connection_t *control_conn, } else if (!strcmp(question, "orconn-status")) { smartlist_t *conns = get_connection_array(); smartlist_t *status = smartlist_create(); - SMARTLIST_FOREACH(conns, connection_t *, base_conn, - { + SMARTLIST_FOREACH_BEGIN(conns, connection_t *, base_conn) { const char *state; char *s; char name[128]; @@ -1744,29 +1695,19 @@ getinfo_helper_events(control_connection_t *control_conn, state = "LAUNCHED"; else state = "NEW"; - orconn_target_get_name(control_conn->use_long_names, name, sizeof(name), - conn); + orconn_target_get_name(name, sizeof(name), conn); slen = strlen(name)+strlen(state)+2; s = tor_malloc(slen+1); tor_snprintf(s, slen, "%s %s", name, state); smartlist_add(status, s); - }); + } SMARTLIST_FOREACH_END(base_conn); *answer = smartlist_join_strings(status, "\r\n", 0, NULL); SMARTLIST_FOREACH(status, char *, cp, tor_free(cp)); smartlist_free(status); - } else if (!strcmpstart(question, "addr-mappings/") || - !strcmpstart(question, "address-mappings/")) { + } else if (!strcmpstart(question, "address-mappings/")) { time_t min_e, max_e; smartlist_t *mappings; - int want_expiry = !strcmpstart(question, "address-mappings/"); - if (!strcmpstart(question, "addr-mappings/")) { - /* XXXX022 This has been deprecated since 0.2.0.3-alpha, and has - generated a warning since 0.2.1.10-alpha; remove late in 0.2.2.x. */ - log_warn(LD_CONTROL, "Controller used obsolete addr-mappings/ GETINFO " - "key; use address-mappings/ instead."); - } - question += strlen(want_expiry ? "address-mappings/" - : "addr-mappings/"); + question += strlen("address-mappings/"); if (!strcmp(question, "all")) { min_e = 0; max_e = TIME_MAX; } else if (!strcmp(question, "cache")) { @@ -1779,7 +1720,7 @@ getinfo_helper_events(control_connection_t *control_conn, return 0; } mappings = smartlist_create(); - addressmap_get_mappings(mappings, min_e, max_e, want_expiry); + addressmap_get_mappings(mappings, min_e, max_e, 1); *answer = smartlist_join_strings(mappings, "\r\n", 0, NULL); SMARTLIST_FOREACH(mappings, char *, cp, tor_free(cp)); smartlist_free(mappings); @@ -1787,7 +1728,7 @@ getinfo_helper_events(control_connection_t *control_conn, /* Note that status/ is not a catch-all for events; there's only supposed * to be a status GETINFO if there's a corresponding STATUS event. */ if (!strcmp(question, "status/circuit-established")) { - *answer = tor_strdup(has_completed_circuit ? "1" : "0"); + *answer = tor_strdup(can_complete_circuit ? "1" : "0"); } else if (!strcmp(question, "status/enough-dir-info")) { *answer = tor_strdup(router_have_minimum_dir_info() ? "1" : "0"); } else if (!strcmp(question, "status/good-server-descriptor") || @@ -1846,21 +1787,12 @@ getinfo_helper_events(control_connection_t *control_conn, "information", question); } } else if (!strcmp(question, "status/clients-seen")) { - char geoip_start[ISO_TIME_LEN+1]; - size_t answer_len; - char *geoip_summary = extrainfo_get_client_geoip_summary(time(NULL)); - - if (!geoip_summary) + char *bridge_stats = geoip_get_bridge_stats_controller(time(NULL)); + if (!bridge_stats) { + *errmsg = "No bridge-client stats available"; return -1; - - answer_len = strlen("TimeStarted=\"\" CountrySummary=") + - ISO_TIME_LEN + strlen(geoip_summary) + 1; - *answer = tor_malloc(answer_len); - format_iso_time(geoip_start, geoip_get_history_start()); - tor_snprintf(*answer, answer_len, - "TimeStarted=\"%s\" CountrySummary=%s", - geoip_start, geoip_summary); - tor_free(geoip_summary); + } + *answer = bridge_stats; } else { return 0; } @@ -1870,11 +1802,14 @@ getinfo_helper_events(control_connection_t *control_conn, /** Callback function for GETINFO: on a given control connection, try to * answer the question <b>q</b> and store the newly-allocated answer in - * *<b>a</b>. If there's no answer, or an error occurs, just don't set - * <b>a</b>. Return 0. + * *<b>a</b>. If an internal error occurs, return -1 and optionally set + * *<b>error_out</b> to point to an error message to be delivered to the + * controller. On success, _or if the key is not recognized_, return 0. Do not + * set <b>a</b> if the key is not recognized. */ typedef int (*getinfo_helper_t)(control_connection_t *, - const char *q, char **a); + const char *q, char **a, + const char **error_out); /** A single item for the GETINFO question-to-answer-function table. */ typedef struct getinfo_item_t { @@ -1894,6 +1829,8 @@ typedef struct getinfo_item_t { static const getinfo_item_t getinfo_items[] = { ITEM("version", misc, "The current version of Tor."), ITEM("config-file", misc, "Current location of the \"torrc\" file."), + ITEM("config-text", misc, + "Return the string that would be written by a saveconf command."), ITEM("accounting/bytes", accounting, "Number of bytes read/written so far in the accounting interval."), ITEM("accounting/bytes-left", accounting, @@ -1933,7 +1870,6 @@ static const getinfo_item_t getinfo_items[] = { PREFIX("ns/purpose/", networkstatus, "Brief summary of router status by purpose (v2 directory format)."), - PREFIX("unregistered-servers-", dirserv_unregistered, NULL), ITEM("network-status", dir, "Brief summary of router status (v1 directory format)"), ITEM("circuit-status", events, "List of current circuits originating here."), @@ -1945,14 +1881,6 @@ static const getinfo_item_t getinfo_items[] = { DOC("address-mappings/config", "Current address mappings from configuration."), DOC("address-mappings/control", "Current address mappings from controller."), - PREFIX("addr-mappings/", events, NULL), - DOC("addr-mappings/all", "Current address mappings without expiry times."), - DOC("addr-mappings/cache", - "Current cached DNS replies without expiry times."), - DOC("addr-mappings/config", - "Current address mappings from configuration without expiry times."), - DOC("addr-mappings/control", - "Current address mappings from controller without expiry times."), PREFIX("status/", events, NULL), DOC("status/circuit-established", "Whether we think client functionality is working."), @@ -1988,18 +1916,18 @@ static char * list_getinfo_options(void) { int i; - char buf[300]; + char *buf=NULL; smartlist_t *lines = smartlist_create(); char *ans; for (i = 0; getinfo_items[i].varname; ++i) { if (!getinfo_items[i].desc) continue; - tor_snprintf(buf, sizeof(buf), "%s%s -- %s\n", + tor_asprintf(&buf, "%s%s -- %s\n", getinfo_items[i].varname, getinfo_items[i].is_prefix ? "*" : "", getinfo_items[i].desc); - smartlist_add(lines, tor_strdup(buf)); + smartlist_add(lines, buf); } smartlist_sort_strings(lines); @@ -2016,7 +1944,8 @@ list_getinfo_options(void) * internal error. */ static int handle_getinfo_helper(control_connection_t *control_conn, - const char *question, char **answer) + const char *question, char **answer, + const char **err_out) { int i; *answer = NULL; /* unrecognized key by default */ @@ -2029,7 +1958,7 @@ handle_getinfo_helper(control_connection_t *control_conn, match = !strcmp(question, getinfo_items[i].varname); if (match) { tor_assert(getinfo_items[i].fn); - return getinfo_items[i].fn(control_conn, question, answer); + return getinfo_items[i].fn(control_conn, question, answer, err_out); } } @@ -2051,10 +1980,12 @@ handle_control_getinfo(control_connection_t *conn, uint32_t len, smartlist_split_string(questions, body, " ", SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); - SMARTLIST_FOREACH(questions, const char *, q, - { - if (handle_getinfo_helper(conn, q, &ans) < 0) { - connection_write_str_to_buf("551 Internal error\r\n", conn); + SMARTLIST_FOREACH_BEGIN(questions, const char *, q) { + const char *errmsg = NULL; + if (handle_getinfo_helper(conn, q, &ans, &errmsg) < 0) { + if (!errmsg) + errmsg = "Internal error"; + connection_printf_to_buf(conn, "551 %s\r\n", errmsg); goto done; } if (!ans) { @@ -2063,7 +1994,7 @@ handle_control_getinfo(control_connection_t *conn, uint32_t len, smartlist_add(answers, tor_strdup(q)); smartlist_add(answers, ans); } - }); + } SMARTLIST_FOREACH_END(q); if (smartlist_len(unrecognized)) { for (i=0; i < smartlist_len(unrecognized)-1; ++i) connection_printf_to_buf(conn, @@ -2108,12 +2039,12 @@ handle_control_getinfo(control_connection_t *conn, uint32_t len, static uint8_t circuit_purpose_from_string(const char *string) { - if (!strcmpstart(string, "purpose=")) + if (!strcasecmpstart(string, "purpose=")) string += strlen("purpose="); - if (!strcmp(string, "general")) + if (!strcasecmp(string, "general")) return CIRCUIT_PURPOSE_C_GENERAL; - else if (!strcmp(string, "controller")) + else if (!strcasecmp(string, "controller")) return CIRCUIT_PURPOSE_CONTROLLER; else return CIRCUIT_PURPOSE_UNKNOWN; @@ -2145,6 +2076,31 @@ getargs_helper(const char *command, control_connection_t *conn, return NULL; } +/** Helper. Return the first element of <b>sl</b> at index <b>start_at</b> or + * higher that starts with <b>prefix</b>, case-insensitive. Return NULL if no + * such element exists. */ +static const char * +find_element_starting_with(smartlist_t *sl, int start_at, const char *prefix) +{ + int i; + for (i = start_at; i < smartlist_len(sl); ++i) { + const char *elt = smartlist_get(sl, i); + if (!strcasecmpstart(elt, prefix)) + return elt; + } + return NULL; +} + +/** Helper. Return true iff s is an argument that we should treat as a + * key-value pair. */ +static int +is_keyval_pair(const char *s) +{ + /* An argument is a key-value pair if it has an =, and it isn't of the form + * $fingeprint=name */ + return strchr(s, '=') && s[0] != '$'; +} + /** Called when we get an EXTENDCIRCUIT message. Try to extend the listed * circuit, and report success or failure. */ static int @@ -2160,33 +2116,57 @@ handle_control_extendcircuit(control_connection_t *conn, uint32_t len, router_nicknames = smartlist_create(); - args = getargs_helper("EXTENDCIRCUIT", conn, body, 2, -1); + args = getargs_helper("EXTENDCIRCUIT", conn, body, 1, -1); if (!args) goto done; zero_circ = !strcmp("0", (char*)smartlist_get(args,0)); - if (!zero_circ && !(circ = get_circ(smartlist_get(args,0)))) { - connection_printf_to_buf(conn, "552 Unknown circuit \"%s\"\r\n", - (char*)smartlist_get(args, 0)); - } - smartlist_split_string(router_nicknames, smartlist_get(args,1), ",", 0, 0); - if (zero_circ && smartlist_len(args)>2) { - char *purp = smartlist_get(args,2); - intended_purpose = circuit_purpose_from_string(purp); - if (intended_purpose == CIRCUIT_PURPOSE_UNKNOWN) { - connection_printf_to_buf(conn, "552 Unknown purpose \"%s\"\r\n", purp); + if (zero_circ) { + const char *purp = find_element_starting_with(args, 1, "PURPOSE="); + + if (purp) { + intended_purpose = circuit_purpose_from_string(purp); + if (intended_purpose == CIRCUIT_PURPOSE_UNKNOWN) { + connection_printf_to_buf(conn, "552 Unknown purpose \"%s\"\r\n", purp); + SMARTLIST_FOREACH(args, char *, cp, tor_free(cp)); + smartlist_free(args); + goto done; + } + } + + if ((smartlist_len(args) == 1) || + (smartlist_len(args) >= 2 && is_keyval_pair(smartlist_get(args, 1)))) { + // "EXTENDCIRCUIT 0" || EXTENDCIRCUIT 0 foo=bar" + circ = circuit_launch_by_router(intended_purpose, NULL, + CIRCLAUNCH_NEED_CAPACITY); + if (!circ) { + connection_write_str_to_buf("551 Couldn't start circuit\r\n", conn); + } else { + connection_printf_to_buf(conn, "250 EXTENDED %lu\r\n", + (unsigned long)circ->global_identifier); + } SMARTLIST_FOREACH(args, char *, cp, tor_free(cp)); smartlist_free(args); goto done; } + // "EXTENDCIRCUIT 0 router1,router2" || + // "EXTENDCIRCUIT 0 router1,router2 PURPOSE=foo" } - SMARTLIST_FOREACH(args, char *, cp, tor_free(cp)); - smartlist_free(args); - if (!zero_circ && !circ) { + + if (!zero_circ && !(circ = get_circ(smartlist_get(args,0)))) { + connection_printf_to_buf(conn, "552 Unknown circuit \"%s\"\r\n", + (char*)smartlist_get(args, 0)); + SMARTLIST_FOREACH(args, char *, cp, tor_free(cp)); + smartlist_free(args); goto done; } + smartlist_split_string(router_nicknames, smartlist_get(args,1), ",", 0, 0); + + SMARTLIST_FOREACH(args, char *, cp, tor_free(cp)); + smartlist_free(args); + routers = smartlist_create(); SMARTLIST_FOREACH(router_nicknames, const char *, n, { @@ -2244,8 +2224,7 @@ handle_control_extendcircuit(control_connection_t *conn, uint32_t len, done: SMARTLIST_FOREACH(router_nicknames, char *, n, tor_free(n)); smartlist_free(router_nicknames); - if (routers) - smartlist_free(routers); + smartlist_free(routers); return 0; } @@ -2271,7 +2250,7 @@ handle_control_setcircuitpurpose(control_connection_t *conn, } { - char *purp = smartlist_get(args,1); + const char *purp = find_element_starting_with(args,1,"PURPOSE="); new_purpose = circuit_purpose_from_string(purp); if (new_purpose == CIRCUIT_PURPOSE_UNKNOWN) { connection_printf_to_buf(conn, "552 Unknown purpose \"%s\"\r\n", purp); @@ -2282,7 +2261,7 @@ handle_control_setcircuitpurpose(control_connection_t *conn, circ->_base.purpose = new_purpose; connection_write_str_to_buf("250 OK\r\n", conn); -done: + done: if (args) { SMARTLIST_FOREACH(args, char *, cp, tor_free(cp)); smartlist_free(args); @@ -2316,9 +2295,9 @@ handle_control_attachstream(control_connection_t *conn, uint32_t len, } else if (!zero_circ && !(circ = get_circ(smartlist_get(args, 1)))) { connection_printf_to_buf(conn, "552 Unknown circuit \"%s\"\r\n", (char*)smartlist_get(args, 1)); - } else if (circ && smartlist_len(args) > 2) { - char *hopstring = smartlist_get(args, 2); - if (!strcasecmpstart(hopstring, "HOP=")) { + } else if (circ) { + const char *hopstring = find_element_starting_with(args,2,"HOP="); + if (hopstring) { hopstring += strlen("HOP="); hop = (int) tor_parse_ulong(hopstring, 10, 0, INT_MAX, &hop_line_ok, NULL); @@ -2365,7 +2344,7 @@ handle_control_attachstream(control_connection_t *conn, uint32_t len, char* exit_digest; if (circ->build_state && circ->build_state->chosen_exit && - circ->build_state->chosen_exit->identity_digest) { + !tor_digest_is_zero(circ->build_state->chosen_exit->identity_digest)) { exit_digest = circ->build_state->chosen_exit->identity_digest; r = router_get_by_digest(exit_digest); } @@ -2426,9 +2405,9 @@ handle_control_postdescriptor(control_connection_t *conn, uint32_t len, } } else if (!strcasecmpstart(option, "cache=")) { option += strlen("cache="); - if (!strcmp(option, "no")) + if (!strcasecmp(option, "no")) cache = 0; - else if (!strcmp(option, "yes")) + else if (!strcasecmp(option, "yes")) cache = 1; else { connection_printf_to_buf(conn, "552 Unknown cache request \"%s\"\r\n", @@ -2610,17 +2589,17 @@ handle_control_resolve(control_connection_t *conn, uint32_t len, args = smartlist_create(); smartlist_split_string(args, body, " ", SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); - if (smartlist_len(args) && - !strcasecmp(smartlist_get(args, 0), "mode=reverse")) { - char *cp = smartlist_get(args, 0); - smartlist_del_keeporder(args, 0); - tor_free(cp); - is_reverse = 1; + { + const char *modearg = find_element_starting_with(args, 0, "mode="); + if (modearg && !strcasecmp(modearg, "mode=reverse")) + is_reverse = 1; } failed = smartlist_create(); SMARTLIST_FOREACH(args, const char *, arg, { - if (dnsserv_launch_request(arg, is_reverse)<0) - smartlist_add(failed, (char*)arg); + if (!is_keyval_pair(arg)) { + if (dnsserv_launch_request(arg, is_reverse)<0) + smartlist_add(failed, (char*)arg); + } }); send_control_done(conn); @@ -2710,7 +2689,6 @@ handle_control_usefeature(control_connection_t *conn, const char *body) { smartlist_t *args; - int verbose_names = 0, extended_events = 0; int bad = 0; (void) len; /* body is nul-terminated; it's safe to ignore the length */ args = smartlist_create(); @@ -2718,9 +2696,9 @@ handle_control_usefeature(control_connection_t *conn, SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); SMARTLIST_FOREACH(args, const char *, arg, { if (!strcasecmp(arg, "VERBOSE_NAMES")) - verbose_names = 1; + ; else if (!strcasecmp(arg, "EXTENDED_EVENTS")) - extended_events = 1; + ; else { connection_printf_to_buf(conn, "552 Unrecognized feature \"%s\"\r\n", arg); @@ -2730,12 +2708,6 @@ handle_control_usefeature(control_connection_t *conn, }); if (!bad) { - if (verbose_names) { - conn->use_long_names = 1; - control_update_global_event_mask(); - } - if (extended_events) - conn->use_extended_events = 1; send_control_done(conn); } @@ -2885,9 +2857,10 @@ connection_control_process_inbuf(control_connection_t *conn) && !TOR_ISSPACE(conn->incoming_cmd[cmd_len])) ++cmd_len; - data_len -= cmd_len; conn->incoming_cmd[cmd_len]='\0'; args = conn->incoming_cmd+cmd_len+1; + tor_assert(data_len>(size_t)cmd_len); + data_len -= (cmd_len+1); /* skip the command and NUL we added after it */ while (*args == ' ' || *args == '\t') { ++args; --data_len; @@ -3039,20 +3012,11 @@ control_event_circuit_status(origin_circuit_t *circ, circuit_status_event_t tp, tor_free(reason); } - if (EVENT_IS_INTERESTING1S(EVENT_CIRCUIT_STATUS)) { - char *path = circuit_list_path(circ,0); - const char *sp = strlen(path) ? " " : ""; - send_control_event_extended(EVENT_CIRCUIT_STATUS, SHORT_NAMES, - "650 CIRC %lu %s%s%s@%s\r\n", - (unsigned long)circ->global_identifier, - status, sp, path, extended_buf); - tor_free(path); - } - if (EVENT_IS_INTERESTING1L(EVENT_CIRCUIT_STATUS)) { + { char *vpath = circuit_list_path_for_controller(circ); const char *sp = strlen(vpath) ? " " : ""; - send_control_event_extended(EVENT_CIRCUIT_STATUS, LONG_NAMES, - "650 CIRC %lu %s%s%s@%s\r\n", + send_control_event(EVENT_CIRCUIT_STATUS, ALL_FORMATS, + "650 CIRC %lu %s%s%s %s\r\n", (unsigned long)circ->global_identifier, status, sp, vpath, extended_buf); tor_free(vpath); @@ -3131,26 +3095,26 @@ control_event_stream_status(edge_connection_t *conn, stream_status_event_t tp, char *r = NULL; if (!reason_str) { r = tor_malloc(16); - tor_snprintf(r, 16, "UNKNOWN_%d", reason_code); + tor_snprintf(r, 16, " UNKNOWN_%d", reason_code); reason_str = r; } if (reason_code & END_STREAM_REASON_FLAG_REMOTE) tor_snprintf(reason_buf, sizeof(reason_buf), - "REASON=END REMOTE_REASON=%s", reason_str); + " REASON=END REMOTE_REASON=%s", reason_str); else tor_snprintf(reason_buf, sizeof(reason_buf), - "REASON=%s", reason_str); + " REASON=%s", reason_str); tor_free(r); } else if (reason_code && tp == STREAM_EVENT_REMAP) { switch (reason_code) { case REMAP_STREAM_SOURCE_CACHE: - strlcpy(reason_buf, "SOURCE=CACHE", sizeof(reason_buf)); + strlcpy(reason_buf, " SOURCE=CACHE", sizeof(reason_buf)); break; case REMAP_STREAM_SOURCE_EXIT: - strlcpy(reason_buf, "SOURCE=EXIT", sizeof(reason_buf)); + strlcpy(reason_buf, " SOURCE=EXIT", sizeof(reason_buf)); break; default: - tor_snprintf(reason_buf, sizeof(reason_buf), "REASON=UNKNOWN_%d", + tor_snprintf(reason_buf, sizeof(reason_buf), " REASON=UNKNOWN_%d", reason_code); /* XXX do we want SOURCE=UNKNOWN_%d above instead? -RD */ break; @@ -3158,8 +3122,7 @@ control_event_stream_status(edge_connection_t *conn, stream_status_event_t tp, } if (tp == STREAM_EVENT_NEW) { - tor_snprintf(addrport_buf,sizeof(addrport_buf), "%sSOURCE_ADDR=%s:%d", - strlen(reason_buf) ? " " : "", + tor_snprintf(addrport_buf,sizeof(addrport_buf), " SOURCE_ADDR=%s:%d", TO_CONN(conn)->address, TO_CONN(conn)->port ); } else { addrport_buf[0] = '\0'; @@ -3188,8 +3151,8 @@ control_event_stream_status(edge_connection_t *conn, stream_status_event_t tp, circ = circuit_get_by_edge_conn(conn); if (circ && CIRCUIT_IS_ORIGIN(circ)) origin_circ = TO_ORIGIN_CIRCUIT(circ); - send_control_event_extended(EVENT_STREAM_STATUS, ALL_NAMES, - "650 STREAM "U64_FORMAT" %s %lu %s@%s%s%s\r\n", + send_control_event(EVENT_STREAM_STATUS, ALL_FORMATS, + "650 STREAM "U64_FORMAT" %s %lu %s%s%s%s\r\n", U64_PRINTF_ARG(conn->_base.global_identifier), status, origin_circ? (unsigned long)origin_circ->global_identifier : 0ul, @@ -3202,30 +3165,21 @@ control_event_stream_status(edge_connection_t *conn, stream_status_event_t tp, /** Figure out the best name for the target router of an OR connection * <b>conn</b>, and write it into the <b>len</b>-character buffer - * <b>name</b>. Use verbose names if <b>long_names</b> is set. */ + * <b>name</b>. */ static void -orconn_target_get_name(int long_names, - char *name, size_t len, or_connection_t *conn) -{ - if (! long_names) { - if (conn->nickname) - strlcpy(name, conn->nickname, len); - else - tor_snprintf(name, len, "%s:%d", - conn->_base.address, conn->_base.port); +orconn_target_get_name(char *name, size_t len, or_connection_t *conn) +{ + routerinfo_t *ri = router_get_by_digest(conn->identity_digest); + if (ri) { + tor_assert(len > MAX_VERBOSE_NICKNAME_LEN); + router_get_verbose_nickname(name, ri); + } else if (! tor_digest_is_zero(conn->identity_digest)) { + name[0] = '$'; + base16_encode(name+1, len-1, conn->identity_digest, + DIGEST_LEN); } else { - routerinfo_t *ri = router_get_by_digest(conn->identity_digest); - if (ri) { - tor_assert(len > MAX_VERBOSE_NICKNAME_LEN); - router_get_verbose_nickname(name, ri); - } else if (! tor_digest_is_zero(conn->identity_digest)) { - name[0] = '$'; - base16_encode(name+1, len-1, conn->identity_digest, - DIGEST_LEN); - } else { - tor_snprintf(name, len, "%s:%d", - conn->_base.address, conn->_base.port); - } + tor_snprintf(name, len, "%s:%d", + conn->_base.address, conn->_base.port); } } @@ -3264,24 +3218,13 @@ control_event_or_conn_status(or_connection_t *conn, or_conn_status_event_t tp, reason ? " " : "", ncircs); } - if (EVENT_IS_INTERESTING1S(EVENT_OR_CONN_STATUS)) { - orconn_target_get_name(0, name, sizeof(name), conn); - send_control_event_extended(EVENT_OR_CONN_STATUS, SHORT_NAMES, - "650 ORCONN %s %s@%s%s%s\r\n", - name, status, - reason ? "REASON=" : "", - orconn_end_reason_to_control_string(reason), - ncircs_buf); - } - if (EVENT_IS_INTERESTING1L(EVENT_OR_CONN_STATUS)) { - orconn_target_get_name(1, name, sizeof(name), conn); - send_control_event_extended(EVENT_OR_CONN_STATUS, LONG_NAMES, - "650 ORCONN %s %s@%s%s%s\r\n", - name, status, - reason ? "REASON=" : "", - orconn_end_reason_to_control_string(reason), - ncircs_buf); - } + orconn_target_get_name(name, sizeof(name), conn); + send_control_event(EVENT_OR_CONN_STATUS, ALL_FORMATS, + "650 ORCONN %s %s %s%s%s\r\n", + name, status, + reason ? "REASON=" : "", + orconn_end_reason_to_control_string(reason), + ncircs_buf); return 0; } @@ -3296,7 +3239,7 @@ control_event_stream_bandwidth(edge_connection_t *edge_conn) if (!edge_conn->n_read && !edge_conn->n_written) return 0; - send_control_event(EVENT_STREAM_BANDWIDTH_USED, ALL_NAMES, + send_control_event(EVENT_STREAM_BANDWIDTH_USED, ALL_FORMATS, "650 STREAM_BW "U64_FORMAT" %lu %lu\r\n", U64_PRINTF_ARG(edge_conn->_base.global_identifier), (unsigned long)edge_conn->n_read, @@ -3325,7 +3268,7 @@ control_event_stream_bandwidth_used(void) if (!edge_conn->n_read && !edge_conn->n_written) continue; - send_control_event(EVENT_STREAM_BANDWIDTH_USED, ALL_NAMES, + send_control_event(EVENT_STREAM_BANDWIDTH_USED, ALL_FORMATS, "650 STREAM_BW "U64_FORMAT" %lu %lu\r\n", U64_PRINTF_ARG(edge_conn->_base.global_identifier), (unsigned long)edge_conn->n_read, @@ -3345,7 +3288,7 @@ int control_event_bandwidth_used(uint32_t n_read, uint32_t n_written) { if (EVENT_IS_INTERESTING(EVENT_BANDWIDTH_USED)) { - send_control_event(EVENT_BANDWIDTH_USED, ALL_NAMES, + send_control_event(EVENT_BANDWIDTH_USED, ALL_FORMATS, "650 BW %lu %lu\r\n", (unsigned long)n_read, (unsigned long)n_written); @@ -3415,7 +3358,7 @@ control_event_logmsg(int severity, uint32_t domain, const char *msg) default: s = "UnknownLogSeverity"; break; } ++disable_log_messages; - send_control_event(event, ALL_NAMES, "650 %s %s\r\n", s, b?b:msg); + send_control_event(event, ALL_FORMATS, "650 %s %s\r\n", s, b?b:msg); --disable_log_messages; tor_free(b); } @@ -3428,31 +3371,12 @@ control_event_logmsg(int severity, uint32_t domain, const char *msg) int control_event_descriptors_changed(smartlist_t *routers) { - size_t len; char *msg; - smartlist_t *identities = NULL; - char buf[HEX_DIGEST_LEN+1]; if (!EVENT_IS_INTERESTING(EVENT_NEW_DESC)) return 0; - if (EVENT_IS_INTERESTING1S(EVENT_NEW_DESC)) { - identities = smartlist_create(); - SMARTLIST_FOREACH(routers, routerinfo_t *, r, - { - base16_encode(buf,sizeof(buf),r->cache_info.identity_digest,DIGEST_LEN); - smartlist_add(identities, tor_strdup(buf)); - }); - } - if (EVENT_IS_INTERESTING1S(EVENT_NEW_DESC)) { - char *ids = smartlist_join_strings(identities, " ", 0, &len); - size_t ids_len = strlen(ids)+32; - msg = tor_malloc(ids_len); - tor_snprintf(msg, ids_len, "650 NEWDESC %s\r\n", ids); - send_control_event_string(EVENT_NEW_DESC, SHORT_NAMES|ALL_FORMATS, msg); - tor_free(ids); - tor_free(msg); - } - if (EVENT_IS_INTERESTING1L(EVENT_NEW_DESC)) { + + { smartlist_t *names = smartlist_create(); char *ids; size_t names_len; @@ -3465,16 +3389,12 @@ control_event_descriptors_changed(smartlist_t *routers) names_len = strlen(ids)+32; msg = tor_malloc(names_len); tor_snprintf(msg, names_len, "650 NEWDESC %s\r\n", ids); - send_control_event_string(EVENT_NEW_DESC, LONG_NAMES|ALL_FORMATS, msg); + send_control_event_string(EVENT_NEW_DESC, ALL_FORMATS, msg); tor_free(ids); tor_free(msg); SMARTLIST_FOREACH(names, char *, cp, tor_free(cp)); smartlist_free(names); } - if (identities) { - SMARTLIST_FOREACH(identities, char *, cp, tor_free(cp)); - smartlist_free(identities); - } return 0; } @@ -3491,17 +3411,17 @@ control_event_address_mapped(const char *from, const char *to, time_t expires, return 0; if (expires < 3 || expires == TIME_MAX) - send_control_event_extended(EVENT_ADDRMAP, ALL_NAMES, - "650 ADDRMAP %s %s NEVER@%s\r\n", from, to, + send_control_event(EVENT_ADDRMAP, ALL_FORMATS, + "650 ADDRMAP %s %s NEVER %s\r\n", from, to, error?error:""); else { char buf[ISO_TIME_LEN+1]; char buf2[ISO_TIME_LEN+1]; format_local_iso_time(buf,expires); format_iso_time(buf2,expires); - send_control_event_extended(EVENT_ADDRMAP, ALL_NAMES, + send_control_event(EVENT_ADDRMAP, ALL_FORMATS, "650 ADDRMAP %s %s \"%s\"" - "@%s%sEXPIRES=\"%s\"\r\n", + " %s%sEXPIRES=\"%s\"\r\n", from, to, buf, error?error:"", error?" ":"", buf2); @@ -3541,9 +3461,9 @@ control_event_or_authdir_new_descriptor(const char *action, buf = tor_malloc(totallen); strlcpy(buf, firstline, totallen); strlcpy(buf+strlen(firstline), esc, totallen); - send_control_event_string(EVENT_AUTHDIR_NEWDESCS, ALL_NAMES|ALL_FORMATS, + send_control_event_string(EVENT_AUTHDIR_NEWDESCS, ALL_FORMATS, buf); - send_control_event_string(EVENT_AUTHDIR_NEWDESCS, ALL_NAMES|ALL_FORMATS, + send_control_event_string(EVENT_AUTHDIR_NEWDESCS, ALL_FORMATS, "650 OK\r\n"); tor_free(esc); tor_free(buf); @@ -3581,8 +3501,8 @@ control_event_networkstatus_changed_helper(smartlist_t *statuses, SMARTLIST_FOREACH(strs, char *, cp, tor_free(cp)); smartlist_free(strs); tor_free(s); - send_control_event_string(event, ALL_NAMES|ALL_FORMATS, esc); - send_control_event_string(event, ALL_NAMES|ALL_FORMATS, + send_control_event_string(event, ALL_FORMATS, esc); + send_control_event_string(event, ALL_FORMATS, "650 OK\r\n"); tor_free(esc); @@ -3608,6 +3528,55 @@ control_event_newconsensus(const networkstatus_t *consensus) consensus->routerstatus_list, EVENT_NEWCONSENSUS, "NEWCONSENSUS"); } +/** Called when we compute a new circuitbuildtimeout */ +int +control_event_buildtimeout_set(const circuit_build_times_t *cbt, + buildtimeout_set_event_t type) +{ + const char *type_string = NULL; + double qnt = circuit_build_times_quantile_cutoff(); + + if (!control_event_is_interesting(EVENT_BUILDTIMEOUT_SET)) + return 0; + + switch (type) { + case BUILDTIMEOUT_SET_EVENT_COMPUTED: + type_string = "COMPUTED"; + break; + case BUILDTIMEOUT_SET_EVENT_RESET: + type_string = "RESET"; + qnt = 1.0; + break; + case BUILDTIMEOUT_SET_EVENT_SUSPENDED: + type_string = "SUSPENDED"; + qnt = 1.0; + break; + case BUILDTIMEOUT_SET_EVENT_DISCARD: + type_string = "DISCARD"; + qnt = 1.0; + break; + case BUILDTIMEOUT_SET_EVENT_RESUME: + type_string = "RESUME"; + break; + default: + type_string = "UNKNOWN"; + break; + } + + send_control_event(EVENT_BUILDTIMEOUT_SET, ALL_FORMATS, + "650 BUILDTIMEOUT_SET %s TOTAL_TIMES=%lu " + "TIMEOUT_MS=%lu XM=%lu ALPHA=%lf CUTOFF_QUANTILE=%lf " + "TIMEOUT_RATE=%lf CLOSE_MS=%lu CLOSE_RATE=%lf\r\n", + type_string, (unsigned long)cbt->total_build_times, + (unsigned long)cbt->timeout_ms, + (unsigned long)cbt->Xm, cbt->alpha, qnt, + circuit_build_times_timeout_rate(cbt), + (unsigned long)cbt->close_ms, + circuit_build_times_close_rate(cbt)); + + return 0; +} + /** Called when a single local_routerstatus_t has changed: Sends an NS event * to any controller that cares. */ int @@ -3631,7 +3600,7 @@ control_event_networkstatus_changed_single(routerstatus_t *rs) int control_event_my_descriptor_changed(void) { - send_control_event(EVENT_DESCCHANGED, ALL_NAMES, "650 DESCCHANGED\r\n"); + send_control_event(EVENT_DESCCHANGED, ALL_FORMATS, "650 DESCCHANGED\r\n"); return 0; } @@ -3679,7 +3648,7 @@ control_event_status(int type, int severity, const char *format, va_list args) return -1; } - send_control_event_impl(type, ALL_NAMES|ALL_FORMATS, 0, format_buf, args); + send_control_event_impl(type, ALL_FORMATS, format_buf, args); return 0; } @@ -3743,7 +3712,7 @@ control_event_guard(const char *nickname, const char *digest, if (!EVENT_IS_INTERESTING(EVENT_GUARD)) return 0; - if (EVENT_IS_INTERESTING1L(EVENT_GUARD)) { + { char buf[MAX_VERBOSE_NICKNAME_LEN+1]; routerinfo_t *ri = router_get_by_digest(digest); if (ri) { @@ -3751,13 +3720,9 @@ control_event_guard(const char *nickname, const char *digest, } else { tor_snprintf(buf, sizeof(buf), "$%s~%s", hbuf, nickname); } - send_control_event(EVENT_GUARD, LONG_NAMES, + send_control_event(EVENT_GUARD, ALL_FORMATS, "650 GUARD ENTRY %s %s\r\n", buf, status); } - if (EVENT_IS_INTERESTING1S(EVENT_GUARD)) { - send_control_event(EVENT_GUARD, SHORT_NAMES, - "650 GUARD ENTRY $%s %s\r\n", hbuf, status); - } return 0; } @@ -4008,10 +3973,9 @@ control_event_bootstrap_problem(const char *warn, int reason) * from recently. Send a copy to the controller in case it wants to * display it for the user. */ void -control_event_clients_seen(const char *timestarted, const char *countries) +control_event_clients_seen(const char *controller_str) { send_control_event(EVENT_CLIENTS_SEEN, 0, - "650 CLIENTS_SEEN TimeStarted=\"%s\" CountrySummary=%s\r\n", - timestarted, countries); + "650 CLIENTS_SEEN %s\r\n", controller_str); } diff --git a/src/or/control.h b/src/or/control.h new file mode 100644 index 0000000000..2ae96b4b8a --- /dev/null +++ b/src/or/control.h @@ -0,0 +1,85 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file control.h + * \brief Header file for control.c. + **/ + +#ifndef _TOR_CONTROL_H +#define _TOR_CONTROL_H + +void control_update_global_event_mask(void); +void control_adjust_event_log_severity(void); + +/** Log information about the connection <b>conn</b>, protecting it as with + * CONN_LOG_PROTECT. Example: + * + * LOG_FN_CONN(conn, (LOG_DEBUG, "Socket %d wants to write", conn->s)); + **/ +#define LOG_FN_CONN(conn, args) \ + CONN_LOG_PROTECT(conn, log_fn args) + +int connection_control_finished_flushing(control_connection_t *conn); +int connection_control_reached_eof(control_connection_t *conn); +int connection_control_process_inbuf(control_connection_t *conn); + +#define EVENT_AUTHDIR_NEWDESCS 0x000D +#define EVENT_NS 0x000F +int control_event_is_interesting(int event); + +int control_event_circuit_status(origin_circuit_t *circ, + circuit_status_event_t e, int reason); +int control_event_stream_status(edge_connection_t *conn, + stream_status_event_t e, + int reason); +int control_event_or_conn_status(or_connection_t *conn, + or_conn_status_event_t e, int reason); +int control_event_bandwidth_used(uint32_t n_read, uint32_t n_written); +int control_event_stream_bandwidth(edge_connection_t *edge_conn); +int control_event_stream_bandwidth_used(void); +void control_event_logmsg(int severity, unsigned int domain, const char *msg); +int control_event_descriptors_changed(smartlist_t *routers); +int control_event_address_mapped(const char *from, const char *to, + time_t expires, const char *error); +int control_event_or_authdir_new_descriptor(const char *action, + const char *desc, + size_t desclen, + const char *msg); +int control_event_my_descriptor_changed(void); +int control_event_networkstatus_changed(smartlist_t *statuses); + +int control_event_newconsensus(const networkstatus_t *consensus); +int control_event_networkstatus_changed_single(routerstatus_t *rs); +int control_event_general_status(int severity, const char *format, ...) + CHECK_PRINTF(2,3); +int control_event_client_status(int severity, const char *format, ...) + CHECK_PRINTF(2,3); +int control_event_server_status(int severity, const char *format, ...) + CHECK_PRINTF(2,3); +int control_event_guard(const char *nickname, const char *digest, + const char *status); +int control_event_buildtimeout_set(const circuit_build_times_t *cbt, + buildtimeout_set_event_t type); + +int init_cookie_authentication(int enabled); +smartlist_t *decode_hashed_passwords(config_line_t *passwords); +void disable_control_logging(void); +void enable_control_logging(void); + +void control_event_bootstrap(bootstrap_status_t status, int progress); +void control_event_bootstrap_problem(const char *warn, int reason); + +void control_event_clients_seen(const char *controller_str); + +#ifdef CONTROL_PRIVATE +/* Used only by control.c and test.c */ +size_t write_escaped_data(const char *data, size_t len, char **out); +size_t read_escaped_data(const char *data, size_t len, char **out); +#endif + +#endif + diff --git a/src/or/cpuworker.c b/src/or/cpuworker.c index 91af5f1d61..7cbc191333 100644 --- a/src/or/cpuworker.c +++ b/src/or/cpuworker.c @@ -13,6 +13,15 @@ **/ #include "or.h" +#include "buffers.h" +#include "circuitbuild.h" +#include "circuitlist.h" +#include "config.h" +#include "connection.h" +#include "cpuworker.h" +#include "main.h" +#include "onion.h" +#include "router.h" /** The maximum number of cpuworker processes we will keep around. */ #define MAX_CPUWORKERS 16 @@ -183,7 +192,7 @@ connection_cpu_process_inbuf(connection_t *conn) tor_assert(0); /* don't ask me to do handshakes yet */ } -done_processing: + done_processing: conn->state = CPUWORKER_STATE_IDLE; num_cpuworkers_busy--; if (conn->timestamp_created < last_rotation_time) { @@ -241,7 +250,7 @@ cpuworker_main(void *data) for (;;) { ssize_t r; - if ((r = recv(fd, &question_type, 1, 0)) != 1) { + if ((r = recv(fd, (void *)&question_type, 1, 0)) != 1) { // log_fn(LOG_ERR,"read type failed. Exiting."); if (r == 0) { log_info(LD_OR, @@ -358,7 +367,7 @@ spawn_cpuworker(void) static void spawn_enough_cpuworkers(void) { - int num_cpuworkers_needed = get_options()->NumCpus; + int num_cpuworkers_needed = get_options()->NumCPUs; if (num_cpuworkers_needed < MIN_CPUWORKERS) num_cpuworkers_needed = MIN_CPUWORKERS; diff --git a/src/or/cpuworker.h b/src/or/cpuworker.h new file mode 100644 index 0000000000..04e37ee459 --- /dev/null +++ b/src/or/cpuworker.h @@ -0,0 +1,25 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file cpuworker.h + * \brief Header file for cpuworker.c. + **/ + +#ifndef _TOR_CPUWORKER_H +#define _TOR_CPUWORKER_H + +void cpu_init(void); +void cpuworkers_rotate(void); +int connection_cpu_finished_flushing(connection_t *conn); +int connection_cpu_reached_eof(connection_t *conn); +int connection_cpu_process_inbuf(connection_t *conn); +int assign_onionskin_to_cpuworker(connection_t *cpuworker, + or_circuit_t *circ, + char *onionskin); + +#endif + diff --git a/src/or/directory.c b/src/or/directory.c index 7150fce407..0c095fe871 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -4,6 +4,26 @@ /* See LICENSE for licensing information */ #include "or.h" +#include "buffers.h" +#include "circuitbuild.h" +#include "config.h" +#include "connection.h" +#include "connection_edge.h" +#include "control.h" +#include "directory.h" +#include "dirserv.h" +#include "dirvote.h" +#include "geoip.h" +#include "main.h" +#include "networkstatus.h" +#include "policies.h" +#include "rendclient.h" +#include "rendcommon.h" +#include "rephist.h" +#include "router.h" +#include "routerlist.h" +#include "routerparse.h" + #if defined(EXPORTMALLINFO) && defined(HAVE_MALLOC_H) && defined(HAVE_MALLINFO) #ifndef OPENBSD #include <malloc.h> @@ -47,8 +67,10 @@ static void http_set_address_origin(const char *headers, connection_t *conn); static void connection_dir_download_networkstatus_failed( dir_connection_t *conn, int status_code); static void connection_dir_download_routerdesc_failed(dir_connection_t *conn); +static void connection_dir_bridge_routerdesc_failed(dir_connection_t *conn); static void connection_dir_download_cert_failed( dir_connection_t *conn, int status_code); +static void connection_dir_retry_bridges(smartlist_t *descs); static void dir_networkstatus_download_failed(smartlist_t *failed, int status_code); static void dir_routerdesc_download_failed(smartlist_t *failed, @@ -92,17 +114,19 @@ static void directory_initiate_command_rend(const char *address, #define ROUTERDESC_CACHE_LIFETIME (30*60) #define ROUTERDESC_BY_DIGEST_CACHE_LIFETIME (48*60*60) #define ROBOTS_CACHE_LIFETIME (24*60*60) +#define MICRODESC_CACHE_LIFETIME (48*60*60) /********* END VARIABLES ************/ -/** Return true iff the directory purpose 'purpose' must use an - * anonymous connection to a directory. */ +/** Return true iff the directory purpose <b>dir_purpose</b> (and if it's + * fetching descriptors, it's fetching them for <b>router_purpose</b>) + * must use an anonymous connection to a directory. */ static int purpose_needs_anonymity(uint8_t dir_purpose, uint8_t router_purpose) { if (get_options()->AllDirActionsPrivate) return 1; - if (router_purpose == ROUTER_PURPOSE_BRIDGE && has_completed_circuit) + if (router_purpose == ROUTER_PURPOSE_BRIDGE && can_complete_circuit) return 1; /* if no circuits yet, we may need this info to bootstrap. */ if (dir_purpose == DIR_PURPOSE_UPLOAD_DIR || dir_purpose == DIR_PURPOSE_UPLOAD_VOTE || @@ -229,10 +253,13 @@ directories_have_accepted_server_descriptor(void) } /** Start a connection to every suitable directory authority, using - * connection purpose 'purpose' and uploading the payload 'payload' - * (length 'payload_len'). The purpose should be one of + * connection purpose <b>dir_purpose</b> and uploading <b>payload</b> + * (of length <b>payload_len</b>). The dir_purpose should be one of * 'DIR_PURPOSE_UPLOAD_DIR' or 'DIR_PURPOSE_UPLOAD_RENDDESC'. * + * <b>router_purpose</b> describes the type of descriptor we're + * publishing, if we're publishing a descriptor -- e.g. general or bridge. + * * <b>type</b> specifies what sort of dir authorities (V1, V2, * HIDSERV, BRIDGE) we should upload to. * @@ -248,6 +275,7 @@ directory_post_to_dirservers(uint8_t dir_purpose, uint8_t router_purpose, const char *payload, size_t payload_len, size_t extrainfo_len) { + or_options_t *options = get_options(); int post_via_tor; smartlist_t *dirservers = router_get_trusted_dir_servers(); int found = 0; @@ -263,6 +291,16 @@ directory_post_to_dirservers(uint8_t dir_purpose, uint8_t router_purpose, if ((type & ds->type) == 0) continue; + if (options->ExcludeNodes && options->StrictNodes && + routerset_contains_routerstatus(options->ExcludeNodes, rs)) { + log_warn(LD_DIR, "Wanted to contact authority '%s' for %s, but " + "it's in our ExcludedNodes list and StrictNodes is set. " + "Skipping.", + ds->nickname, + dir_conn_purpose_to_string(dir_purpose)); + continue; + } + found = 1; /* at least one authority of this type was listed */ if (dir_purpose == DIR_PURPOSE_UPLOAD_DIR) ds->has_accepted_serverdesc = 0; @@ -348,7 +386,7 @@ directory_get_from_dirserver(uint8_t dir_purpose, uint8_t router_purpose, if (!get_via_tor) { if (options->UseBridges && type != BRIDGE_AUTHORITY) { /* want to ask a running bridge for which we have a descriptor. */ - /* XXX022 we assume that all of our bridges can answer any + /* XXX023 we assume that all of our bridges can answer any * possible directory question. This won't be true forever. -RD */ /* It certainly is not true with conditional consensus downloading, * so, for now, never assume the server supports that. */ @@ -472,12 +510,14 @@ directory_initiate_command_routerstatus_rend(routerstatus_t *status, time_t if_modified_since, const rend_data_t *rend_query) { + or_options_t *options = get_options(); routerinfo_t *router; char address_buf[INET_NTOA_BUF_LEN+1]; struct in_addr in; const char *address; tor_addr_t addr; router = router_get_by_digest(status->identity_digest); + if (!router && anonymized_connection) { log_info(LD_DIR, "Not sending anonymized request to directory '%s'; we " "don't have its router descriptor.", status->nickname); @@ -490,6 +530,17 @@ directory_initiate_command_routerstatus_rend(routerstatus_t *status, address = address_buf; } tor_addr_from_ipv4h(&addr, status->addr); + + if (options->ExcludeNodes && options->StrictNodes && + routerset_contains_routerstatus(options->ExcludeNodes, status)) { + log_warn(LD_DIR, "Wanted to contact directory mirror '%s' for %s, but " + "it's in our ExcludedNodes list and StrictNodes is set. " + "Skipping. This choice might make your Tor not work.", + status->nickname, + dir_conn_purpose_to_string(dir_purpose)); + return; + } + directory_initiate_command_rend(address, &addr, status->or_port, status->dir_port, status->version_supports_conditional_consensus, @@ -560,7 +611,7 @@ connection_dir_request_failed(dir_connection_t *conn) if (directory_conn_is_self_reachability_test(conn)) { return; /* this was a test fetch. don't retry. */ } - if (entry_list_can_grow(get_options())) + if (!entry_list_is_constrained(get_options())) router_set_status(conn->identity_digest, 0); /* don't try him again */ if (conn->_base.purpose == DIR_PURPOSE_FETCH_V2_NETWORKSTATUS) { log_info(LD_DIR, "Giving up on directory server at '%s'; retrying", @@ -570,6 +621,8 @@ connection_dir_request_failed(dir_connection_t *conn) conn->_base.purpose == DIR_PURPOSE_FETCH_EXTRAINFO) { log_info(LD_DIR, "Giving up on directory server at '%s'; retrying", conn->_base.address); + if (conn->router_purpose == ROUTER_PURPOSE_BRIDGE) + connection_dir_bridge_routerdesc_failed(conn); connection_dir_download_routerdesc_failed(conn); } else if (conn->_base.purpose == DIR_PURPOSE_FETCH_CONSENSUS) { networkstatus_consensus_download_failed(0); @@ -614,7 +667,7 @@ connection_dir_download_networkstatus_failed(dir_connection_t *conn, * failed, and possibly retry them later.*/ smartlist_t *failed = smartlist_create(); dir_split_resource_into_fingerprints(conn->requested_resource+3, - failed, NULL, 0, 0); + failed, NULL, 0); if (smartlist_len(failed)) { dir_networkstatus_download_failed(failed, status_code); SMARTLIST_FOREACH(failed, char *, cp, tor_free(cp)); @@ -623,6 +676,24 @@ connection_dir_download_networkstatus_failed(dir_connection_t *conn, } } +/** Helper: Attempt to fetch directly the descriptors of each bridge + * listed in <b>failed</b>. + */ +static void +connection_dir_retry_bridges(smartlist_t *descs) +{ + char digest[DIGEST_LEN]; + SMARTLIST_FOREACH(descs, const char *, cp, + { + if (base16_decode(digest, DIGEST_LEN, cp, strlen(cp))<0) { + log_warn(LD_BUG, "Malformed fingerprint in list: %s", + escaped(cp)); + continue; + } + retry_bridge_descriptor_fetch_directly(digest); + }); +} + /** Called when an attempt to download one or more router descriptors * or extra-info documents on connection <b>conn</b> failed. */ @@ -640,6 +711,33 @@ connection_dir_download_routerdesc_failed(dir_connection_t *conn) (void) conn; } +/** Called when an attempt to download a bridge's routerdesc from + * one of the authorities failed due to a network error. If + * possible attempt to download descriptors from the bridge directly. + */ +static void +connection_dir_bridge_routerdesc_failed(dir_connection_t *conn) +{ + smartlist_t *which = NULL; + + /* Requests for bridge descriptors are in the form 'fp/', so ignore + anything else. */ + if (!conn->requested_resource || strcmpstart(conn->requested_resource,"fp/")) + return; + + which = smartlist_create(); + dir_split_resource_into_fingerprints(conn->requested_resource + + strlen("fp/"), + which, NULL, 0); + + tor_assert(conn->_base.purpose != DIR_PURPOSE_FETCH_EXTRAINFO); + if (smartlist_len(which)) { + connection_dir_retry_bridges(which); + SMARTLIST_FOREACH(which, char *, cp, tor_free(cp)); + } + smartlist_free(which); +} + /** Called when an attempt to fetch a certificate fails. */ static void connection_dir_download_cert_failed(dir_connection_t *conn, int status) @@ -651,7 +749,7 @@ connection_dir_download_cert_failed(dir_connection_t *conn, int status) return; failed = smartlist_create(); dir_split_resource_into_fingerprints(conn->requested_resource+3, - failed, NULL, 1, 0); + failed, NULL, DSR_HEX); SMARTLIST_FOREACH(failed, char *, cp, { authority_cert_dl_failed(cp, status); @@ -667,7 +765,7 @@ connection_dir_download_cert_failed(dir_connection_t *conn, int status) * 1) If or_port is 0, or it's a direct conn and or_port is firewalled * or we're a dir mirror, no. * 2) If we prefer to avoid begindir conns, and we're not fetching or - * publishing a bridge relay descriptor, no. + * publishing a bridge relay descriptor, no. * 3) Else yes. */ static int @@ -746,6 +844,15 @@ directory_initiate_command_rend(const char *address, const tor_addr_t *_addr, log_debug(LD_DIR, "Initiating %s", dir_conn_purpose_to_string(dir_purpose)); + /* ensure that we don't make direct connections when a SOCKS server is + * configured. */ + if (!anonymized_connection && !use_begindir && !options->HTTPProxy && + (options->Socks4Proxy || options->Socks5Proxy)) { + log_warn(LD_DIR, "Cannot connect to a directory server through a " + "SOCKS proxy!"); + return; + } + conn = dir_connection_new(AF_INET); /* set up conn so it's got all the data we need to remember */ @@ -770,9 +877,9 @@ directory_initiate_command_rend(const char *address, const tor_addr_t *_addr, if (!anonymized_connection && !use_begindir) { /* then we want to connect to dirport directly */ - if (options->HttpProxy) { - tor_addr_from_ipv4h(&addr, options->HttpProxyAddr); - dir_port = options->HttpProxyPort; + if (options->HTTPProxy) { + tor_addr_copy(&addr, &options->HTTPProxyAddr); + dir_port = options->HTTPProxyPort; } switch (connection_connect(TO_CONN(conn), conn->_base.address, &addr, @@ -793,7 +900,7 @@ directory_initiate_command_rend(const char *address, const tor_addr_t *_addr, payload, payload_len, supports_conditional_consensus, if_modified_since); - connection_watch_events(TO_CONN(conn), EV_READ | EV_WRITE); + connection_watch_events(TO_CONN(conn), READ_EVENT | WRITE_EVENT); /* writable indicates finish, readable indicates broken link, error indicates broken link in windowsland. */ } @@ -832,7 +939,7 @@ directory_initiate_command_rend(const char *address, const tor_addr_t *_addr, payload, payload_len, supports_conditional_consensus, if_modified_since); - connection_watch_events(TO_CONN(conn), EV_READ | EV_WRITE); + connection_watch_events(TO_CONN(conn), READ_EVENT|WRITE_EVENT); connection_start_reading(TO_CONN(linked_conn)); } } @@ -880,7 +987,7 @@ directory_get_consensus_url(int supports_conditional_consensus) if (supports_conditional_consensus) { char *authority_id_list; - smartlist_t *authority_digets = smartlist_create(); + smartlist_t *authority_digests = smartlist_create(); SMARTLIST_FOREACH(router_get_trusted_dir_servers(), trusted_dir_server_t *, ds, @@ -892,10 +999,10 @@ directory_get_consensus_url(int supports_conditional_consensus) hex = tor_malloc(2*CONDITIONAL_CONSENSUS_FPR_LEN+1); base16_encode(hex, 2*CONDITIONAL_CONSENSUS_FPR_LEN+1, ds->v3_identity_digest, CONDITIONAL_CONSENSUS_FPR_LEN); - smartlist_add(authority_digets, hex); + smartlist_add(authority_digests, hex); }); - smartlist_sort(authority_digets, _compare_strs); - authority_id_list = smartlist_join_strings(authority_digets, + smartlist_sort(authority_digests, _compare_strs); + authority_id_list = smartlist_join_strings(authority_digests, "+", 0, NULL); len = strlen(authority_id_list)+64; @@ -903,8 +1010,8 @@ directory_get_consensus_url(int supports_conditional_consensus) tor_snprintf(url, len, "/tor/status-vote/current/consensus/%s.z", authority_id_list); - SMARTLIST_FOREACH(authority_digets, char *, cp, tor_free(cp)); - smartlist_free(authority_digets); + SMARTLIST_FOREACH(authority_digests, char *, cp, tor_free(cp)); + smartlist_free(authority_digests); tor_free(authority_id_list); } else { url = tor_strdup("/tor/status-vote/current/consensus.z"); @@ -913,7 +1020,7 @@ directory_get_consensus_url(int supports_conditional_consensus) } /** Queue an appropriate HTTP command on conn-\>outbuf. The other args - * are as in directory_initiate_command. + * are as in directory_initiate_command(). */ static void directory_send_command(dir_connection_t *conn, @@ -956,9 +1063,9 @@ directory_send_command(dir_connection_t *conn, } /* come up with some proxy lines, if we're using one. */ - if (direct && get_options()->HttpProxy) { + if (direct && get_options()->HTTPProxy) { char *base64_authenticator=NULL; - const char *authenticator = get_options()->HttpProxyAuthenticator; + const char *authenticator = get_options()->HTTPProxyAuthenticator; tor_snprintf(proxystring, sizeof(proxystring),"http://%s", hoststring); if (authenticator) { @@ -1049,31 +1156,10 @@ directory_send_command(dir_connection_t *conn, httpcommand = "POST"; url = tor_strdup("/tor/post/consensus-signature"); break; - case DIR_PURPOSE_FETCH_RENDDESC: - tor_assert(resource); - tor_assert(!payload); - - /* this must be true or we wouldn't be doing the lookup */ - tor_assert(strlen(resource) <= REND_SERVICE_ID_LEN_BASE32); - /* This breaks the function abstraction. */ - conn->rend_data = tor_malloc_zero(sizeof(rend_data_t)); - strlcpy(conn->rend_data->onion_address, resource, - sizeof(conn->rend_data->onion_address)); - conn->rend_data->rend_desc_version = 0; - - httpcommand = "GET"; - /* Request the most recent versioned descriptor. */ - // (XXXX We were going to switch this to fetch rendezvous1 descriptors, - // but that never got testing, and it wasn't a good design.) - len = strlen(resource)+32; - url = tor_malloc(len); - tor_snprintf(url, len, "/tor/rendezvous/%s", resource); - break; case DIR_PURPOSE_FETCH_RENDDESC_V2: tor_assert(resource); tor_assert(strlen(resource) <= REND_DESC_ID_V2_LEN_BASE32); tor_assert(!payload); - conn->rend_data->rend_desc_version = 2; httpcommand = "GET"; len = strlen(resource) + 32; url = tor_malloc(len); @@ -1162,7 +1248,7 @@ parse_http_url(const char *headers, char **url) if (s-tmp >= 3 && !strcmpstart(tmp,"://")) { tmp = strchr(tmp+3, '/'); if (tmp && tmp < s) { - log_debug(LD_DIR,"Skipping over 'http[s]://hostname' string"); + log_debug(LD_DIR,"Skipping over 'http[s]://hostname/' string"); start = tmp; } } @@ -1479,7 +1565,7 @@ connection_dir_client_reached_eof(dir_connection_t *conn) } (void) skewed; /* skewed isn't used yet. */ - if (status_code == 503 && body_len < 16) { + if (status_code == 503) { routerstatus_t *rs; trusted_dir_server_t *ds; log_info(LD_DIR,"Received http status code %d (%s) from server " @@ -1493,12 +1579,6 @@ connection_dir_client_reached_eof(dir_connection_t *conn) tor_free(body); tor_free(headers); tor_free(reason); return -1; - } else if (status_code == 503) { - /* XXXX022 Remove this once every server with bug 539 is obsolete. */ - log_info(LD_DIR, "Server at '%s:%d' sent us a 503 response, but included " - "a body anyway. We'll pretend it gave us a 200.", - conn->_base.address, conn->_base.port); - status_code = 200; } plausible = body_is_plausible(body, body_len, conn->_base.purpose); @@ -1565,7 +1645,7 @@ connection_dir_client_reached_eof(dir_connection_t *conn) v2_networkstatus_source_t source; char *cp; log_info(LD_DIR,"Received networkstatus objects (size %d) from server " - "'%s:%d'",(int) body_len, conn->_base.address, conn->_base.port); + "'%s:%d'", (int)body_len, conn->_base.address, conn->_base.port); if (status_code != 200) { log_warn(LD_DIR, "Received http status code %d (%s) from server " @@ -1581,7 +1661,7 @@ connection_dir_client_reached_eof(dir_connection_t *conn) source = NS_FROM_DIR_BY_FP; which = smartlist_create(); dir_split_resource_into_fingerprints(conn->requested_resource+3, - which, NULL, 0, 0); + which, NULL, 0); } else if (conn->requested_resource && !strcmpstart(conn->requested_resource, "all")) { source = NS_FROM_DIR_ALL; @@ -1639,8 +1719,8 @@ connection_dir_client_reached_eof(dir_connection_t *conn) return -1; } log_info(LD_DIR,"Received consensus directory (size %d) from server " - "'%s:%d'",(int) body_len, conn->_base.address, conn->_base.port); - if ((r=networkstatus_set_current_consensus(body, 0))<0) { + "'%s:%d'", (int)body_len, conn->_base.address, conn->_base.port); + if ((r=networkstatus_set_current_consensus(body, "ns", 0))<0) { log_fn(r<-1?LOG_WARN:LOG_INFO, LD_DIR, "Unable to load consensus directory downloaded from " "server '%s:%d'. I'll try again soon.", @@ -1667,9 +1747,11 @@ connection_dir_client_reached_eof(dir_connection_t *conn) return -1; } log_info(LD_DIR,"Received authority certificates (size %d) from server " - "'%s:%d'",(int) body_len, conn->_base.address, conn->_base.port); + "'%s:%d'", (int)body_len, conn->_base.address, conn->_base.port); if (trusted_dirs_load_certs_from_string(body, 0, 1)<0) { log_warn(LD_DIR, "Unable to parse fetched certificates"); + /* if we fetched more than one and only some failed, the successful + * ones got flushed to disk so it's safe to call this on them */ connection_dir_download_cert_failed(conn, status_code); } else { directory_info_has_arrived(now, 0); @@ -1680,7 +1762,7 @@ connection_dir_client_reached_eof(dir_connection_t *conn) const char *msg; int st; log_info(LD_DIR,"Got votes (size %d) from server %s:%d", - (int) body_len, conn->_base.address, conn->_base.port); + (int)body_len, conn->_base.address, conn->_base.port); if (status_code != 200) { log_warn(LD_DIR, "Received http status code %d (%s) from server " @@ -1700,11 +1782,11 @@ connection_dir_client_reached_eof(dir_connection_t *conn) if (conn->_base.purpose == DIR_PURPOSE_FETCH_DETACHED_SIGNATURES) { const char *msg = NULL; log_info(LD_DIR,"Got detached signatures (size %d) from server %s:%d", - (int) body_len, conn->_base.address, conn->_base.port); + (int)body_len, conn->_base.address, conn->_base.port); if (status_code != 200) { log_warn(LD_DIR, - "Received http status code %d (%s) from server " - "'%s:%d' while fetching \"/tor/status-vote/consensus-signatures.z\".", + "Received http status code %d (%s) from server '%s:%d' while fetching " + "\"/tor/status-vote/next/consensus-signatures.z\".", status_code, escaped(reason), conn->_base.address, conn->_base.port); tor_free(body); tor_free(headers); tor_free(reason); @@ -1732,7 +1814,7 @@ connection_dir_client_reached_eof(dir_connection_t *conn) which = smartlist_create(); dir_split_resource_into_fingerprints(conn->requested_resource + (descriptor_digests ? 2 : 3), - which, NULL, 0, 0); + which, NULL, 0); n_asked_for = smartlist_len(which); } if (status_code != 200) { @@ -1814,7 +1896,7 @@ connection_dir_client_reached_eof(dir_connection_t *conn) ds->nickname); /* XXXX use this information; be sure to upload next one * sooner. -NM */ - /* XXXX021 On further thought, the task above implies that we're + /* XXXX023 On further thought, the task above implies that we're * basing our regenerate-descriptor time on when we uploaded the * last descriptor, not on the published time of the last * descriptor. If those are different, that's a bad thing to @@ -1920,7 +2002,7 @@ connection_dir_client_reached_eof(dir_connection_t *conn) /* Success, or at least there's a v2 descriptor already * present. Notify pending connections about this. */ conn->_base.purpose = DIR_PURPOSE_HAS_FETCHED_RENDDESC; - rend_client_desc_trynow(conn->rend_data->onion_address, -1); + rend_client_desc_trynow(conn->rend_data->onion_address); } break; case 404: @@ -1967,7 +2049,7 @@ connection_dir_client_reached_eof(dir_connection_t *conn) log_info(LD_REND, "Successfully fetched v2 rendezvous " "descriptor."); conn->_base.purpose = DIR_PURPOSE_HAS_FETCHED_RENDDESC; - rend_client_desc_trynow(conn->rend_data->onion_address, -1); + rend_client_desc_trynow(conn->rend_data->onion_address); break; } break; @@ -2010,12 +2092,6 @@ connection_dir_client_reached_eof(dir_connection_t *conn) "'%s:%d'. Malformed rendezvous descriptor?", escaped(reason), conn->_base.address, conn->_base.port); break; - case 503: - log_info(LD_REND,"http status 503 (%s) response from dirserver " - "'%s:%d'. Node is (currently) not acting as v2 hidden " - "service directory.", - escaped(reason), conn->_base.address, conn->_base.port); - break; default: log_warn(LD_REND,"http status %d (%s) response unexpected (server " "'%s:%d').", @@ -2322,7 +2398,7 @@ directory_dump_request_log(void) } #endif -/** Decide whether a client would accept the consensus we have +/** Decide whether a client would accept the consensus we have. * * Clients can say they only want a consensus if it's signed by more * than half the authorities in a list. They pass this list in @@ -2343,31 +2419,32 @@ client_likes_consensus(networkstatus_t *v, const char *want_url) int need_at_least; int have = 0; - dir_split_resource_into_fingerprints(want_url, want_authorities, NULL, 0, 0); + dir_split_resource_into_fingerprints(want_url, want_authorities, NULL, 0); need_at_least = smartlist_len(want_authorities)/2+1; - SMARTLIST_FOREACH(want_authorities, const char *, d, { + SMARTLIST_FOREACH_BEGIN(want_authorities, const char *, d) { char want_digest[DIGEST_LEN]; size_t want_len = strlen(d)/2; if (want_len > DIGEST_LEN) want_len = DIGEST_LEN; if (base16_decode(want_digest, DIGEST_LEN, d, want_len*2) < 0) { - log_warn(LD_DIR,"Failed to decode requested authority digest %s.", d); + log_fn(LOG_PROTOCOL_WARN, LD_DIR, + "Failed to decode requested authority digest %s.", d); continue; }; - SMARTLIST_FOREACH(v->voters, networkstatus_voter_info_t *, vi, { - if (vi->signature && + SMARTLIST_FOREACH_BEGIN(v->voters, networkstatus_voter_info_t *, vi) { + if (smartlist_len(vi->sigs) && !memcmp(vi->identity_digest, want_digest, want_len)) { have++; break; }; - }); + } SMARTLIST_FOREACH_END(vi); /* early exit, if we already have enough */ if (have >= need_at_least) break; - }); + } SMARTLIST_FOREACH_END(d); SMARTLIST_FOREACH(want_authorities, char *, d, tor_free(d)); smartlist_free(want_authorities); @@ -2514,9 +2591,12 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, /* v2 or v3 network status fetch. */ smartlist_t *dir_fps = smartlist_create(); int is_v3 = !strcmpstart(url, "/tor/status-vote"); + geoip_client_action_t act = + is_v3 ? GEOIP_CLIENT_NETWORKSTATUS : GEOIP_CLIENT_NETWORKSTATUS_V2; const char *request_type = NULL; const char *key = url + strlen("/tor/status/"); long lifetime = NETWORKSTATUS_CACHE_LIFETIME; + if (!is_v3) { dirserv_get_networkstatus_v2_fingerprints(dir_fps, key); if (!strcmpstart(key, "fp/")) @@ -2531,18 +2611,44 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, } else { networkstatus_t *v = networkstatus_get_latest_consensus(); time_t now = time(NULL); + const char *want_fps = NULL; + char *flavor = NULL; #define CONSENSUS_URL_PREFIX "/tor/status-vote/current/consensus/" - if (v && - !strcmpstart(url, CONSENSUS_URL_PREFIX) && - !client_likes_consensus(v, url + strlen(CONSENSUS_URL_PREFIX))) { + #define CONSENSUS_FLAVORED_PREFIX "/tor/status-vote/current/consensus-" + /* figure out the flavor if any, and who we wanted to sign the thing */ + if (!strcmpstart(url, CONSENSUS_FLAVORED_PREFIX)) { + const char *f, *cp; + f = url + strlen(CONSENSUS_FLAVORED_PREFIX); + cp = strchr(f, '/'); + if (cp) { + want_fps = cp+1; + flavor = tor_strndup(f, cp-f); + } else { + flavor = tor_strdup(f); + } + } else { + if (!strcmpstart(url, CONSENSUS_URL_PREFIX)) + want_fps = url+strlen(CONSENSUS_URL_PREFIX); + } + + /* XXXX MICRODESC NM NM should check document of correct flavor */ + if (v && want_fps && + !client_likes_consensus(v, want_fps)) { write_http_status_line(conn, 404, "Consensus not signed by sufficient " "number of requested authorities"); smartlist_free(dir_fps); + geoip_note_ns_response(act, GEOIP_REJECT_NOT_ENOUGH_SIGS); + tor_free(flavor); goto done; } - smartlist_add(dir_fps, tor_memdup("\0\0\0\0\0\0\0\0\0\0" - "\0\0\0\0\0\0\0\0\0\0", 20)); + { + char *fp = tor_malloc_zero(DIGEST_LEN); + if (flavor) + strlcpy(fp, flavor, DIGEST_LEN); + tor_free(flavor); + smartlist_add(dir_fps, fp); + } request_type = compressed?"v3.z":"v3"; lifetime = (v && v->fresh_until > now) ? v->fresh_until - now : 0; } @@ -2550,6 +2656,7 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, if (!smartlist_len(dir_fps)) { /* we failed to create/cache cp */ write_http_status_line(conn, 503, "Network status object unavailable"); smartlist_free(dir_fps); + geoip_note_ns_response(act, GEOIP_REJECT_UNAVAILABLE); goto done; } @@ -2557,11 +2664,13 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, write_http_status_line(conn, 404, "Not found"); SMARTLIST_FOREACH(dir_fps, char *, cp, tor_free(cp)); smartlist_free(dir_fps); + geoip_note_ns_response(act, GEOIP_REJECT_NOT_FOUND); goto done; } else if (!smartlist_len(dir_fps)) { write_http_status_line(conn, 304, "Not modified"); SMARTLIST_FOREACH(dir_fps, char *, cp, tor_free(cp)); smartlist_free(dir_fps); + geoip_note_ns_response(act, GEOIP_REJECT_NOT_MODIFIED); goto done; } @@ -2573,18 +2682,25 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, write_http_status_line(conn, 503, "Directory busy, try again later"); SMARTLIST_FOREACH(dir_fps, char *, fp, tor_free(fp)); smartlist_free(dir_fps); + geoip_note_ns_response(act, GEOIP_REJECT_BUSY); goto done; } -#ifdef ENABLE_GEOIP_STATS { - geoip_client_action_t act = - is_v3 ? GEOIP_CLIENT_NETWORKSTATUS : GEOIP_CLIENT_NETWORKSTATUS_V2; struct in_addr in; - if (tor_inet_aton((TO_CONN(conn))->address, &in)) + if (tor_inet_aton((TO_CONN(conn))->address, &in)) { geoip_note_client_seen(act, ntohl(in.s_addr), time(NULL)); + geoip_note_ns_response(act, GEOIP_SUCCESS); + /* Note that a request for a network status has started, so that we + * can measure the download time later on. */ + if (TO_CONN(conn)->dirreq_id) + geoip_start_dirreq(TO_CONN(conn)->dirreq_id, dlen, act, + DIRREQ_TUNNELED); + else + geoip_start_dirreq(TO_CONN(conn)->global_identifier, dlen, act, + DIRREQ_DIRECT); + } } -#endif // note_request(request_type,dlen); (void) request_type; @@ -2610,7 +2726,7 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, ssize_t estimated_len = 0; smartlist_t *items = smartlist_create(); smartlist_t *dir_items = smartlist_create(); - int lifetime = 60; /* XXXX022 should actually use vote intervals. */ + int lifetime = 60; /* XXXX023 should actually use vote intervals. */ url += strlen("/tor/status-vote/"); current = !strcmpstart(url, "current/"); url = strchr(url, '/'); @@ -2620,7 +2736,7 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, const char *item; tor_assert(!current); /* we handle current consensus specially above, * since it wants to be spooled. */ - if ((item = dirvote_get_pending_consensus())) + if ((item = dirvote_get_pending_consensus(FLAV_NS))) smartlist_add(items, (char*)item); } else if (!current && !strcmp(url, "consensus-signatures")) { /* XXXX the spec says that we should implement @@ -2646,7 +2762,8 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, flags = DGV_BY_ID | (current ? DGV_INCLUDE_PREVIOUS : DGV_INCLUDE_PENDING); } - dir_split_resource_into_fingerprints(url, fps, NULL, 1, 1); + dir_split_resource_into_fingerprints(url, fps, NULL, + DSR_HEX|DSR_SORT_UNIQ); SMARTLIST_FOREACH(fps, char *, fp, { if ((d = dirvote_get_vote(fp, flags))) smartlist_add(dir_items, (cached_dir_t*)d); @@ -2699,6 +2816,41 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, goto done; } + if (!strcmpstart(url, "/tor/micro/d/")) { + smartlist_t *fps = smartlist_create(); + + dir_split_resource_into_fingerprints(url+strlen("/tor/micro/d/"), + fps, NULL, + DSR_DIGEST256|DSR_BASE64|DSR_SORT_UNIQ); + + if (!dirserv_have_any_microdesc(fps)) { + write_http_status_line(conn, 404, "Not found"); + SMARTLIST_FOREACH(fps, char *, fp, tor_free(fp)); + smartlist_free(fps); + goto done; + } + dlen = dirserv_estimate_microdesc_size(fps, compressed); + if (global_write_bucket_low(TO_CONN(conn), dlen, 2)) { + log_info(LD_DIRSERV, + "Client asked for server descriptors, but we've been " + "writing too many bytes lately. Sending 503 Dir busy."); + write_http_status_line(conn, 503, "Directory busy, try again later"); + SMARTLIST_FOREACH(fps, char *, fp, tor_free(fp)); + smartlist_free(fps); + goto done; + } + + write_http_response_header(conn, -1, compressed, MICRODESC_CACHE_LIFETIME); + conn->dir_spool_src = DIR_SPOOL_MICRODESC; + conn->fingerprint_stack = fps; + + if (compressed) + conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD); + + connection_dirserv_flushed_some(conn); + goto done; + } + if (!strcmpstart(url,"/tor/server/") || (!options->BridgeAuthoritativeDir && !options->BridgeRelay && !strcmpstart(url,"/tor/extra/"))) { @@ -2780,7 +2932,8 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, } else if (!strcmpstart(url, "/tor/keys/fp/")) { smartlist_t *fps = smartlist_create(); dir_split_resource_into_fingerprints(url+strlen("/tor/keys/fp/"), - fps, NULL, 1, 1); + fps, NULL, + DSR_HEX|DSR_SORT_UNIQ); SMARTLIST_FOREACH(fps, char *, d, { authority_cert_t *c = authority_cert_get_newest_by_id(d); if (c) smartlist_add(certs, c); @@ -2790,7 +2943,8 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, } else if (!strcmpstart(url, "/tor/keys/sk/")) { smartlist_t *fps = smartlist_create(); dir_split_resource_into_fingerprints(url+strlen("/tor/keys/sk/"), - fps, NULL, 1, 1); + fps, NULL, + DSR_HEX|DSR_SORT_UNIQ); SMARTLIST_FOREACH(fps, char *, d, { authority_cert_t *c = authority_cert_get_by_sk_digest(d); if (c) smartlist_add(certs, c); @@ -2892,18 +3046,9 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, note_request("/tor/rendezvous?/", desc_len); /* need to send descp separately, because it may include NULs */ connection_write_to_buf(descp, desc_len, TO_CONN(conn)); - /* report successful fetch to statistic */ - if (options->HSAuthorityRecordStats) { - hs_usage_note_fetch_total(query, time(NULL)); - hs_usage_note_fetch_successful(query, time(NULL)); - } break; case 0: /* well-formed but not present */ write_http_status_line(conn, 404, "Not found"); - /* report (unsuccessful) fetch to statistic */ - if (options->HSAuthorityRecordStats) { - hs_usage_note_fetch_total(query, time(NULL)); - } break; case -1: /* not well-formed */ write_http_status_line(conn, 400, "Bad request"); @@ -3134,6 +3279,8 @@ directory_handle_command_post(dir_connection_t *conn, const char *headers, write_http_status_line(conn, status, "Vote stored"); } else { tor_assert(msg); + log_warn(LD_DIRSERV, "Rejected vote from %s (\"%s\").", + conn->_base.address, msg); write_http_status_line(conn, status, msg); } goto done; @@ -3180,8 +3327,8 @@ directory_handle_command(dir_connection_t *conn) &body, &body_len, MAX_DIR_UL_SIZE, 0)) { case -1: /* overflow */ log_warn(LD_DIRSERV, - "Invalid input from address '%s'. Closing.", - conn->_base.address); + "Request too large from address '%s' to DirPort. Closing.", + safe_str(conn->_base.address)); return -1; case 0: log_debug(LD_DIRSERV,"command not all here yet."); @@ -3217,6 +3364,16 @@ connection_dir_finished_flushing(dir_connection_t *conn) tor_assert(conn); tor_assert(conn->_base.type == CONN_TYPE_DIR); + /* Note that we have finished writing the directory response. For direct + * connections this means we're done, for tunneled connections its only + * an intermediate step. */ + if (TO_CONN(conn)->dirreq_id) + geoip_change_dirreq_state(TO_CONN(conn)->dirreq_id, DIRREQ_TUNNELED, + DIRREQ_FLUSHING_DIR_CONN_FINISHED); + else + geoip_change_dirreq_state(TO_CONN(conn)->global_identifier, + DIRREQ_DIRECT, + DIRREQ_FLUSHING_DIR_CONN_FINISHED); switch (conn->_base.state) { case DIR_CONN_STATE_CLIENT_SENDING: log_debug(LD_DIR,"client finished sending command."); @@ -3399,6 +3556,14 @@ download_status_reset(download_status_t *dls) dls->next_attempt_at = time(NULL) + schedule[0]; } +/** Return the number of failures on <b>dls</b> since the last success (if + * any). */ +int +download_status_get_n_failures(const download_status_t *dls) +{ + return dls->n_download_failures; +} + /** Called when one or more routerdesc (or extrainfo, if <b>was_extrainfo</b>) * fetches have failed (with uppercase fingerprints listed in <b>failed</b>, * either as descriptor digests or as identity digests based on @@ -3414,16 +3579,8 @@ dir_routerdesc_download_failed(smartlist_t *failed, int status_code, int server = directory_fetches_from_authorities(get_options()); if (!was_descriptor_digests) { if (router_purpose == ROUTER_PURPOSE_BRIDGE) { - tor_assert(!was_extrainfo); /* not supported yet */ - SMARTLIST_FOREACH(failed, const char *, cp, - { - if (base16_decode(digest, DIGEST_LEN, cp, strlen(cp))<0) { - log_warn(LD_BUG, "Malformed fingerprint in list: %s", - escaped(cp)); - continue; - } - retry_bridge_descriptor_fetch_directly(digest); - }); + tor_assert(!was_extrainfo); + connection_dir_retry_bridges(failed); } return; /* FFFF should implement for other-than-router-purpose someday */ } @@ -3516,19 +3673,37 @@ dir_split_resource_into_fingerprint_pairs(const char *res, /** Given a directory <b>resource</b> request, containing zero * or more strings separated by plus signs, followed optionally by ".z", store * the strings, in order, into <b>fp_out</b>. If <b>compressed_out</b> is - * non-NULL, set it to 1 if the resource ends in ".z", else set it to 0. If - * decode_hex is true, then delete all elements that aren't hex digests, and - * decode the rest. If sort_uniq is true, then sort the list and remove - * all duplicates. + * non-NULL, set it to 1 if the resource ends in ".z", else set it to 0. + * + * If (flags & DSR_HEX), then delete all elements that aren't hex digests, and + * decode the rest. If (flags & DSR_BASE64), then use "-" rather than "+" as + * a separator, delete all the elements that aren't base64-encoded digests, + * and decode the rest. If (flags & DSR_DIGEST256), these digests should be + * 256 bits long; else they should be 160. + * + * If (flags & DSR_SORT_UNIQ), then sort the list and remove all duplicates. */ int dir_split_resource_into_fingerprints(const char *resource, smartlist_t *fp_out, int *compressed_out, - int decode_hex, int sort_uniq) + int flags) { + const int decode_hex = flags & DSR_HEX; + const int decode_base64 = flags & DSR_BASE64; + const int digests_are_256 = flags & DSR_DIGEST256; + const int sort_uniq = flags & DSR_SORT_UNIQ; + + const int digest_len = digests_are_256 ? DIGEST256_LEN : DIGEST_LEN; + const int hex_digest_len = digests_are_256 ? + HEX_DIGEST256_LEN : HEX_DIGEST_LEN; + const int base64_digest_len = digests_are_256 ? + BASE64_DIGEST256_LEN : BASE64_DIGEST_LEN; smartlist_t *fp_tmp = smartlist_create(); + + tor_assert(!(decode_hex && decode_base64)); tor_assert(fp_out); - smartlist_split_string(fp_tmp, resource, "+", 0, 0); + + smartlist_split_string(fp_tmp, resource, decode_base64?"-":"+", 0, 0); if (compressed_out) *compressed_out = 0; if (smartlist_len(fp_tmp)) { @@ -3540,22 +3715,25 @@ dir_split_resource_into_fingerprints(const char *resource, *compressed_out = 1; } } - if (decode_hex) { + if (decode_hex || decode_base64) { + const size_t encoded_len = decode_hex ? hex_digest_len : base64_digest_len; int i; char *cp, *d = NULL; for (i = 0; i < smartlist_len(fp_tmp); ++i) { cp = smartlist_get(fp_tmp, i); - if (strlen(cp) != HEX_DIGEST_LEN) { + if (strlen(cp) != encoded_len) { log_info(LD_DIR, "Skipping digest %s with non-standard length.", escaped(cp)); smartlist_del_keeporder(fp_tmp, i--); goto again; } - d = tor_malloc_zero(DIGEST_LEN); - if (base16_decode(d, DIGEST_LEN, cp, HEX_DIGEST_LEN)<0) { - log_info(LD_DIR, "Skipping non-decodable digest %s", escaped(cp)); - smartlist_del_keeporder(fp_tmp, i--); - goto again; + d = tor_malloc_zero(digest_len); + if (decode_hex ? + (base16_decode(d, digest_len, cp, hex_digest_len)<0) : + (base64_decode(d, digest_len, cp, base64_digest_len)<0)) { + log_info(LD_DIR, "Skipping non-decodable digest %s", escaped(cp)); + smartlist_del_keeporder(fp_tmp, i--); + goto again; } smartlist_set(fp_tmp, i, d); d = NULL; @@ -3565,26 +3743,18 @@ dir_split_resource_into_fingerprints(const char *resource, } } if (sort_uniq) { - smartlist_t *fp_tmp2 = smartlist_create(); - int i; - if (decode_hex) - smartlist_sort_digests(fp_tmp); - else + if (decode_hex || decode_base64) { + if (digests_are_256) { + smartlist_sort_digests256(fp_tmp); + smartlist_uniq_digests256(fp_tmp); + } else { + smartlist_sort_digests(fp_tmp); + smartlist_uniq_digests(fp_tmp); + } + } else { smartlist_sort_strings(fp_tmp); - if (smartlist_len(fp_tmp)) - smartlist_add(fp_tmp2, smartlist_get(fp_tmp, 0)); - for (i = 1; i < smartlist_len(fp_tmp); ++i) { - char *cp = smartlist_get(fp_tmp, i); - char *last = smartlist_get(fp_tmp2, smartlist_len(fp_tmp2)-1); - - if ((decode_hex && memcmp(cp, last, DIGEST_LEN)) - || (!decode_hex && strcasecmp(cp, last))) - smartlist_add(fp_tmp2, cp); - else - tor_free(cp); + smartlist_uniq_strings(fp_tmp); } - smartlist_free(fp_tmp); - fp_tmp = fp_tmp2; } smartlist_add_all(fp_out, fp_tmp); smartlist_free(fp_tmp); diff --git a/src/or/directory.h b/src/or/directory.h new file mode 100644 index 0000000000..94dfb17644 --- /dev/null +++ b/src/or/directory.h @@ -0,0 +1,110 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file directory.h + * \brief Header file for directory.c. + **/ + +#ifndef _TOR_DIRECTORY_H +#define _TOR_DIRECTORY_H + +int directories_have_accepted_server_descriptor(void); +char *authority_type_to_string(authority_type_t auth); +void directory_post_to_dirservers(uint8_t dir_purpose, uint8_t router_purpose, + authority_type_t type, const char *payload, + size_t payload_len, size_t extrainfo_len); +void directory_get_from_dirserver(uint8_t dir_purpose, uint8_t router_purpose, + const char *resource, + int pds_flags); +void directory_get_from_all_authorities(uint8_t dir_purpose, + uint8_t router_purpose, + const char *resource); +void directory_initiate_command_routerstatus(routerstatus_t *status, + uint8_t dir_purpose, + uint8_t router_purpose, + int anonymized_connection, + const char *resource, + const char *payload, + size_t payload_len, + time_t if_modified_since); +void directory_initiate_command_routerstatus_rend(routerstatus_t *status, + uint8_t dir_purpose, + uint8_t router_purpose, + int anonymized_connection, + const char *resource, + const char *payload, + size_t payload_len, + time_t if_modified_since, + const rend_data_t *rend_query); + +int parse_http_response(const char *headers, int *code, time_t *date, + compress_method_t *compression, char **response); + +int connection_dir_is_encrypted(dir_connection_t *conn); +int connection_dir_reached_eof(dir_connection_t *conn); +int connection_dir_process_inbuf(dir_connection_t *conn); +int connection_dir_finished_flushing(dir_connection_t *conn); +int connection_dir_finished_connecting(dir_connection_t *conn); +void connection_dir_request_failed(dir_connection_t *conn); +void directory_initiate_command(const char *address, const tor_addr_t *addr, + uint16_t or_port, uint16_t dir_port, + int supports_conditional_consensus, + int supports_begindir, const char *digest, + uint8_t dir_purpose, uint8_t router_purpose, + int anonymized_connection, + const char *resource, + const char *payload, size_t payload_len, + time_t if_modified_since); + +#define DSR_HEX (1<<0) +#define DSR_BASE64 (1<<1) +#define DSR_DIGEST256 (1<<2) +#define DSR_SORT_UNIQ (1<<3) +int dir_split_resource_into_fingerprints(const char *resource, + smartlist_t *fp_out, int *compressed_out, + int flags); + +int dir_split_resource_into_fingerprint_pairs(const char *res, + smartlist_t *pairs_out); +char *directory_dump_request_log(void); +void note_request(const char *key, size_t bytes); +int router_supports_extrainfo(const char *identity_digest, int is_authority); + +time_t download_status_increment_failure(download_status_t *dls, + int status_code, const char *item, + int server, time_t now); +/** Increment the failure count of the download_status_t <b>dls</b>, with + * the optional status code <b>sc</b>. */ +#define download_status_failed(dls, sc) \ + download_status_increment_failure((dls), (sc), NULL, \ + get_options()->DirPort, time(NULL)) + +void download_status_reset(download_status_t *dls); +static int download_status_is_ready(download_status_t *dls, time_t now, + int max_failures); +/** Return true iff, as of <b>now</b>, the resource tracked by <b>dls</b> is + * ready to get its download reattempted. */ +static INLINE int +download_status_is_ready(download_status_t *dls, time_t now, + int max_failures) +{ + return (dls->n_download_failures <= max_failures + && dls->next_attempt_at <= now); +} + +static void download_status_mark_impossible(download_status_t *dl); +/** Mark <b>dl</b> as never downloadable. */ +static INLINE void +download_status_mark_impossible(download_status_t *dl) +{ + dl->n_download_failures = IMPOSSIBLE_TO_DOWNLOAD; +} + +int download_status_get_n_failures(const download_status_t *dls); + +#endif + diff --git a/src/or/dirserv.c b/src/or/dirserv.c index 7db6c19a33..525524c42a 100644 --- a/src/or/dirserv.c +++ b/src/or/dirserv.c @@ -5,6 +5,22 @@ #define DIRSERV_PRIVATE #include "or.h" +#include "buffers.h" +#include "config.h" +#include "connection.h" +#include "connection_or.h" +#include "control.h" +#include "directory.h" +#include "dirserv.h" +#include "dirvote.h" +#include "hibernate.h" +#include "microdesc.h" +#include "networkstatus.h" +#include "policies.h" +#include "rephist.h" +#include "router.h" +#include "routerlist.h" +#include "routerparse.h" /** * \file dirserv.c @@ -27,6 +43,8 @@ extern time_t time_of_process_start; /* from main.c */ +extern long stats_n_seconds_working; /* from main.c */ + /** Do we need to regenerate the v1 directory when someone asks for it? */ static time_t the_directory_is_dirty = 1; /** Do we need to regenerate the v1 runningrouters document when somebody @@ -41,7 +59,7 @@ static time_t the_v2_networkstatus_is_dirty = 1; static cached_dir_t *the_directory = NULL; /** For authoritative directories: the current (v1) network status. */ -static cached_dir_t the_runningrouters = { NULL, NULL, 0, 0, 0, -1 }; +static cached_dir_t the_runningrouters; static void directory_remove_invalid(void); static cached_dir_t *dirserv_regenerate_directory(void); @@ -63,13 +81,16 @@ static signed_descriptor_t *get_signed_descriptor_by_fp(const char *fp, time_t publish_cutoff); static int dirserv_add_extrainfo(extrainfo_t *ei, const char **msg); +/************** Measured Bandwidth parsing code ******/ +#define MAX_MEASUREMENT_AGE (3*24*60*60) /* 3 days */ + /************** Fingerprint handling code ************/ #define FP_NAMED 1 /**< Listed in fingerprint file. */ #define FP_INVALID 2 /**< Believed invalid. */ #define FP_REJECT 4 /**< We will not publish this router. */ #define FP_BADDIR 8 /**< We'll tell clients to avoid using this as a dir. */ -#define FP_BADEXIT 16 /**< We'll tell clients not to use this as an exit. */ +#define FP_BADEXIT 16 /**< We'll tell clients not to use this as an exit. */ #define FP_UNNAMED 32 /**< Another router has this name in fingerprint file. */ /** Encapsulate a nickname and an FP_* status; target of status_by_digest @@ -99,7 +120,7 @@ authdir_config_new(void) return list; } -/** Add the fingerprint <b>fp</b> for the nickname <b>nickname</b> to +/** Add the fingerprint <b>fp</b> for <b>nickname</b> to * the smartlist of fingerprint_entry_t's <b>list</b>. Return 0 if it's * new, or 1 if we replaced the old value. */ @@ -181,8 +202,7 @@ dirserv_add_own_fingerprint(const char *nickname, crypto_pk_env_t *pk) * file. The file format is line-based, with each non-blank holding one * nickname, some space, and a fingerprint for that nickname. On success, * replace the current fingerprint list with the new list and return 0. On - * failure, leave the current fingerprint list untouched, and - * return -1. */ + * failure, leave the current fingerprint list untouched, and return -1. */ int dirserv_load_fingerprint_file(void) { @@ -368,13 +388,19 @@ dirserv_get_status_impl(const char *id_digest, const char *nickname, strmap_size(fingerprint_list->fp_by_name), digestmap_size(fingerprint_list->status_by_digest)); - /* 0.1.1.17-rc was the first version that claimed to be stable, doesn't - * crash and drop circuits all the time, and is even vaguely compatible with - * the current network */ - if (platform && !tor_version_as_new_as(platform,"0.1.1.17-rc")) { + /* Tor 0.2.0.26-rc is the oldest version that currently caches the right + * directory information. Once more of them die off, we should raise this + * minimum. */ + if (platform && !tor_version_as_new_as(platform,"0.2.0.26-rc")) { if (msg) *msg = "Tor version is far too old to work."; return FP_REJECT; + } else if (platform && tor_version_as_new_as(platform,"0.2.1.3-alpha") + && !tor_version_as_new_as(platform, "0.2.1.19")) { + /* These versions mishandled RELAY_EARLY cells on rend circuits. */ + if (msg) + *msg = "Tor version is too buggy to work."; + return FP_REJECT; } result = dirserv_get_name_status(id_digest, nickname); @@ -520,7 +546,7 @@ authdir_wants_to_reject_router(routerinfo_t *ri, const char **msg, /* Okay. Now check whether the fingerprint is recognized. */ uint32_t status = dirserv_router_get_status(ri, msg); time_t now; - int severity = complain ? LOG_NOTICE : LOG_INFO; + int severity = (complain && ri->contact_info) ? LOG_NOTICE : LOG_INFO; tor_assert(msg); if (status & FP_REJECT) return -1; /* msg is already set. */ @@ -712,6 +738,10 @@ dirserv_add_descriptor(routerinfo_t *ri, const char **msg, const char *source) desc = tor_strndup(ri->cache_info.signed_descriptor_body, desclen); nickname = tor_strdup(ri->nickname); + /* Tell if we're about to need to launch a test if we add this. */ + ri->needs_retest_if_added = + dirserv_should_launch_reachability_test(ri, ri_old); + r = router_add_to_routerlist(ri, msg, 0, 0); if (!WRA_WAS_ADDED(r)) { /* unless the routerinfo was fine, just out-of-date */ @@ -726,7 +756,7 @@ dirserv_add_descriptor(routerinfo_t *ri, const char **msg, const char *source) changed = smartlist_create(); smartlist_add(changed, ri); - control_event_descriptors_changed(changed); + routerlist_descriptors_added(changed, 0); smartlist_free(changed); if (!*msg) { *msg = ri->is_valid ? "Descriptor for valid server accepted" : @@ -835,46 +865,6 @@ directory_remove_invalid(void) routerlist_assert_ok(rl); } -/** Write a list of unregistered descriptors into a newly allocated - * string and return it. Used by dirserv operators to keep track of - * fast nodes that haven't registered. - */ -int -getinfo_helper_dirserv_unregistered(control_connection_t *control_conn, - const char *question, char **answer_out) -{ - smartlist_t *answerlist; - char buf[1024]; - char *answer; - int min_bw = atoi(question); - routerlist_t *rl = router_get_routerlist(); - - (void) control_conn; - - if (strcmpstart(question, "unregistered-servers-")) - return 0; - question += strlen("unregistered-servers-"); - - answerlist = smartlist_create(); - SMARTLIST_FOREACH(rl->routers, routerinfo_t *, ent, { - uint32_t r = dirserv_router_get_status(ent, NULL); - if (router_get_advertised_bandwidth(ent) >= (size_t)min_bw && - !(r & FP_NAMED)) { - /* then log this one */ - tor_snprintf(buf, sizeof(buf), - "%s: BW %d on '%s'.", - ent->nickname, router_get_advertised_bandwidth(ent), - ent->platform ? ent->platform : ""); - smartlist_add(answerlist, tor_strdup(buf)); - } - }); - answer = smartlist_join_strings(answerlist, "\r\n", 0, NULL); - SMARTLIST_FOREACH(answerlist, char *, cp, tor_free(cp)); - smartlist_free(answerlist); - *answer_out = answer; - return 0; -} - /** Mark the directory as <b>dirty</b> -- when we're next asked for a * directory, we will rebuild it instead of reusing the most recently * generated one. @@ -933,28 +923,66 @@ list_single_server_status(routerinfo_t *desc, int is_live) return tor_strdup(buf); } +static INLINE int +running_long_enough_to_decide_unreachable(void) +{ + return time_of_process_start + + get_options()->TestingAuthDirTimeToLearnReachability < approx_time(); +} + /** Each server needs to have passed a reachability test no more * than this number of seconds ago, or he is listed as down in * the directory. */ #define REACHABLE_TIMEOUT (45*60) +/** If we tested a router and found it reachable _at least this long_ after it + * declared itself hibernating, it is probably done hibernating and we just + * missed a descriptor from it. */ +#define HIBERNATION_PUBLICATION_SKEW (60*60) + /** Treat a router as alive if * - It's me, and I'm not hibernating. * or - We've found it reachable recently. */ void dirserv_set_router_is_running(routerinfo_t *router, time_t now) { + /*XXXX023 This function is a mess. Separate out the part that calculates + whether it's reachable and the part that tells rephist that the router was + unreachable. + */ int answer; - if (router_is_me(router) && !we_are_hibernating()) + if (router_is_me(router)) { + /* We always know if we are down ourselves. */ + answer = ! we_are_hibernating(); + } else if (router->is_hibernating && + (router->cache_info.published_on + + HIBERNATION_PUBLICATION_SKEW) > router->last_reachable) { + /* A hibernating router is down unless we (somehow) had contact with it + * since it declared itself to be hibernating. */ + answer = 0; + } else if (get_options()->AssumeReachable) { + /* If AssumeReachable, everybody is up unless they say they are down! */ answer = 1; - else - answer = get_options()->AssumeReachable || - now < router->last_reachable + REACHABLE_TIMEOUT; + } else { + /* Otherwise, a router counts as up if we found it reachable in the last + REACHABLE_TIMEOUT seconds. */ + answer = (now < router->last_reachable + REACHABLE_TIMEOUT); + } + + if (!answer && running_long_enough_to_decide_unreachable()) { + /* Not considered reachable. tell rephist about that. - if (!answer) { - /* not considered reachable. tell rephist. */ - rep_hist_note_router_unreachable(router->cache_info.identity_digest, now); + Because we launch a reachability test for each router every + REACHABILITY_TEST_CYCLE_PERIOD seconds, then the router has probably + been down since at least that time after we last successfully reached + it. + */ + time_t when = now; + if (router->last_reachable && + router->last_reachable + REACHABILITY_TEST_CYCLE_PERIOD < now) + when = router->last_reachable + REACHABILITY_TEST_CYCLE_PERIOD; + rep_hist_note_router_unreachable(router->cache_info.identity_digest, when); } router->is_running = answer; @@ -965,7 +993,6 @@ dirserv_set_router_is_running(routerinfo_t *router, time_t now) * *<b>router_status_out</b>. Return 0 on success, -1 on failure. * * If for_controller is true, include the routers with very old descriptors. - * If for_controller is >1, use the verbose nickname format. */ int list_server_status_v1(smartlist_t *routers, char **router_status_out, @@ -985,23 +1012,22 @@ list_server_status_v1(smartlist_t *routers, char **router_status_out, rs_entries = smartlist_create(); - SMARTLIST_FOREACH(routers, routerinfo_t *, ri, - { + SMARTLIST_FOREACH_BEGIN(routers, routerinfo_t *, ri) { if (authdir) { /* Update router status in routerinfo_t. */ dirserv_set_router_is_running(ri, now); } - if (for_controller == 1 || ri->cache_info.published_on >= cutoff) - smartlist_add(rs_entries, list_single_server_status(ri, ri->is_running)); - else if (for_controller > 2) { + if (for_controller) { char name_buf[MAX_VERBOSE_NICKNAME_LEN+2]; char *cp = name_buf; if (!ri->is_running) *cp++ = '!'; router_get_verbose_nickname(cp, ri); smartlist_add(rs_entries, tor_strdup(name_buf)); + } else if (ri->cache_info.published_on >= cutoff) { + smartlist_add(rs_entries, list_single_server_status(ri, ri->is_running)); } - }); + } SMARTLIST_FOREACH_END(ri); *router_status_out = smartlist_join_strings(rs_entries, " ", 0, NULL); @@ -1119,7 +1145,8 @@ dirserv_dump_directory_to_string(char **dir_out, return -1; } note_crypto_pk_op(SIGN_DIR); - if (router_append_dirobj_signature(buf,buf_len,digest,private_key)<0) { + if (router_append_dirobj_signature(buf,buf_len,digest,DIGEST_LEN, + private_key)<0) { tor_free(buf); return -1; } @@ -1144,18 +1171,21 @@ directory_fetches_from_authorities(or_options_t *options) { routerinfo_t *me; uint32_t addr; + int refuseunknown; if (options->FetchDirInfoEarly) return 1; if (options->BridgeRelay == 1) return 0; if (server_mode(options) && router_pick_published_address(options, &addr)<0) return 1; /* we don't know our IP address; ask an authority. */ - if (options->DirPort == 0) + refuseunknown = ! router_my_exit_policy_is_reject_star() && + should_refuse_unknown_exits(options); + if (options->DirPort == 0 && !refuseunknown) return 0; if (!server_mode(options) || !advertised_server_mode()) return 0; me = router_get_my_routerinfo(); - if (!me || !me->dir_port) + if (!me || (!me->dir_port && !refuseunknown)) return 0; /* if dirport not advertised, return 0 too */ return 1; } @@ -1195,7 +1225,14 @@ directory_caches_v2_dir_info(or_options_t *options) int directory_caches_dir_info(or_options_t *options) { - return options->BridgeRelay != 0 || options->DirPort != 0; + if (options->BridgeRelay || options->DirPort) + return 1; + if (!server_mode(options) || !advertised_server_mode()) + return 0; + /* We need an up-to-date view of network info if we're going to try to + * block exit attempts from unknown relays. */ + return ! router_my_exit_policy_is_reject_star() && + should_refuse_unknown_exits(options); } /** Return 1 if we want to allow remote people to ask us directory @@ -1238,14 +1275,14 @@ directory_too_idle_to_fetch_descriptors(or_options_t *options, time_t now) static cached_dir_t *cached_directory = NULL; /** The v1 runningrouters document we'll serve (as a cache or as an authority) * if requested. */ -static cached_dir_t cached_runningrouters = { NULL, NULL, 0, 0, 0, -1 }; +static cached_dir_t cached_runningrouters; /** Used for other dirservers' v2 network statuses. Map from hexdigest to * cached_dir_t. */ static digestmap_t *cached_v2_networkstatus = NULL; -/** The v3 consensus network status that we're currently serving. */ -static cached_dir_t *cached_v3_networkstatus = NULL; +/** Map from flavor name to the v3 consensuses that we're currently serving. */ +static strmap_t *cached_consensuses = NULL; /** Possibly replace the contents of <b>d</b> with the value of * <b>directory</b> published on <b>when</b>, unless <b>when</b> is older than @@ -1319,7 +1356,11 @@ clear_cached_dir(cached_dir_t *d) static void _free_cached_dir(void *_d) { - cached_dir_t *d = (cached_dir_t *)_d; + cached_dir_t *d; + if (!_d) + return; + + d = (cached_dir_t *)_d; cached_dir_decref(d); } @@ -1413,17 +1454,26 @@ dirserv_set_cached_networkstatus_v2(const char *networkstatus, } } -/** Replace the v3 consensus networkstatus that we're serving with - * <b>networkstatus</b>, published at <b>published</b>. No validation is - * performed. */ +/** Replace the v3 consensus networkstatus of type <b>flavor_name</b> that + * we're serving with <b>networkstatus</b>, published at <b>published</b>. No + * validation is performed. */ void -dirserv_set_cached_networkstatus_v3(const char *networkstatus, - time_t published) +dirserv_set_cached_consensus_networkstatus(const char *networkstatus, + const char *flavor_name, + const digests_t *digests, + time_t published) { - if (cached_v3_networkstatus) - cached_dir_decref(cached_v3_networkstatus); - cached_v3_networkstatus = new_cached_dir( - tor_strdup(networkstatus), published); + cached_dir_t *new_networkstatus; + cached_dir_t *old_networkstatus; + if (!cached_consensuses) + cached_consensuses = strmap_new(); + + new_networkstatus = new_cached_dir(tor_strdup(networkstatus), published); + memcpy(&new_networkstatus->digests, digests, sizeof(digests_t)); + old_networkstatus = strmap_set(cached_consensuses, flavor_name, + new_networkstatus); + if (old_networkstatus) + cached_dir_decref(old_networkstatus); } /** Remove any v2 networkstatus from the directory cache that was published @@ -1520,7 +1570,8 @@ dirserv_regenerate_directory(void) { char *new_directory=NULL; - if (dirserv_dump_directory_to_string(&new_directory, get_identity_key())) { + if (dirserv_dump_directory_to_string(&new_directory, + get_server_identity_key())) { log_warn(LD_BUG, "Error creating directory."); tor_free(new_directory); return NULL; @@ -1550,7 +1601,7 @@ generate_runningrouters(void) char digest[DIGEST_LEN]; char published[ISO_TIME_LEN+1]; size_t len; - crypto_pk_env_t *private_key = get_identity_key(); + crypto_pk_env_t *private_key = get_server_identity_key(); char *identity_pkey; /* Identity key, DER64-encoded. */ size_t identity_pkey_len; @@ -1577,7 +1628,8 @@ generate_runningrouters(void) goto err; } note_crypto_pk_op(SIGN_DIR); - if (router_append_dirobj_signature(s, len, digest, private_key)<0) + if (router_append_dirobj_signature(s, len, digest, DIGEST_LEN, + private_key)<0) goto err; set_cached_dir(&the_runningrouters, s, time(NULL)); @@ -1605,9 +1657,11 @@ dirserv_get_runningrouters(void) /** Return the latest downloaded consensus networkstatus in encoded, signed, * optionally compressed format, suitable for sending to clients. */ cached_dir_t * -dirserv_get_consensus(void) +dirserv_get_consensus(const char *flavor_name) { - return cached_v3_networkstatus; + if (!cached_consensuses) + return NULL; + return strmap_get(cached_consensuses, flavor_name); } /** For authoritative directories: the current (v2) network status. */ @@ -1645,7 +1699,7 @@ should_generate_v2_networkstatus(void) #define TIME_KNOWN_TO_GUARANTEE_FAMILIAR (8*24*60*60) /** Similarly, every node with sufficient WFU is around enough to be a guard. */ -#define WFU_TO_GUARANTEE_GUARD (0.995) +#define WFU_TO_GUARANTEE_GUARD (0.98) /* Thresholds for server performance: set by * dirserv_compute_performance_thresholds, and used by @@ -1700,9 +1754,12 @@ dirserv_thinks_router_is_unreliable(time_t now, { if (need_uptime) { if (!enough_mtbf_info) { - /* XXX022 Once most authorities are on v3, we should change the rule from + /* XXX023 Once most authorities are on v3, we should change the rule from * "use uptime if we don't have mtbf data" to "don't advertise Stable on - * v3 if we don't have enough mtbf data." */ + * v3 if we don't have enough mtbf data." Or maybe not, since if we ever + * hit a point where we need to reset a lot of authorities at once, + * none of them would be in a position to declare Stable. + */ long uptime = real_uptime(router, now); if ((unsigned)uptime < stable_uptime && (unsigned)uptime < UPTIME_TO_GUARANTEE_STABLE) @@ -1725,7 +1782,7 @@ dirserv_thinks_router_is_unreliable(time_t now, /** Return true iff <b>router</b> should be assigned the "HSDir" flag. * Right now this means it advertises support for it, it has a high - * uptime, and it's currently considered Running. + * uptime, it has a DirPort open, and it's currently considered Running. * * This function needs to be called after router-\>is_running has * been set. @@ -1733,9 +1790,30 @@ dirserv_thinks_router_is_unreliable(time_t now, static int dirserv_thinks_router_is_hs_dir(routerinfo_t *router, time_t now) { - long uptime = real_uptime(router, now); - return (router->wants_to_be_hs_dir && + long uptime; + + /* If we haven't been running for at least + * get_options()->MinUptimeHidServDirectoryV2 seconds, we can't + * have accurate data telling us a relay has been up for at least + * that long. We also want to allow a bit of slack: Reachability + * tests aren't instant. If we haven't been running long enough, + * trust the relay. */ + + if (stats_n_seconds_working > + get_options()->MinUptimeHidServDirectoryV2 * 1.1) + uptime = MIN(rep_hist_get_uptime(router->cache_info.identity_digest, now), + real_uptime(router, now)); + else + uptime = real_uptime(router, now); + + /* XXX We shouldn't need to check dir_port, but we do because of + * bug 1693. In the future, once relays set wants_to_be_hs_dir + * correctly, we can revert to only checking dir_port if router's + * version is too old. */ + /* XXX Unfortunately, we need to keep checking dir_port until all + * *clients* suffering from bug 2722 are obsolete. */ + return (router->wants_to_be_hs_dir && router->dir_port && uptime > get_options()->MinUptimeHidServDirectoryV2 && router->is_running); } @@ -1790,7 +1868,8 @@ dirserv_compute_performance_thresholds(routerlist_t *rl) if (router_is_active(ri, now)) { const char *id = ri->cache_info.identity_digest; uint32_t bw; - ri->is_exit = exit_policy_is_general_exit(ri->exit_policy); + ri->is_exit = (!router_exit_policy_rejects_all(ri) && + exit_policy_is_general_exit(ri->exit_policy)); uptimes[n_active] = (uint32_t)real_uptime(ri, now); mtbfs[n_active] = rep_hist_get_stability(id, now); tks [n_active] = rep_hist_get_weighted_time_known(id, now); @@ -1897,16 +1976,20 @@ version_from_platform(const char *platform) * which has at least <b>buf_len</b> free characters. Do NUL-termination. * Use the same format as in network-status documents. If <b>version</b> is * non-NULL, add a "v" line for the platform. Return 0 on success, -1 on - * failure. If <b>first_line_only</b> is true, don't include any flags - * or version line. + * failure. + * + * The format argument has three possible values: + * NS_V2 - Output an entry suitable for a V2 NS opinion document + * NS_V3_CONSENSUS - Output the first portion of a V3 NS consensus entry + * NS_V3_CONSENSUS_MICRODESC - Output the first portion of a V3 microdesc + * consensus entry. + * NS_V3_VOTE - Output a complete V3 NS vote + * NS_CONTROL_PORT - Output a NS document for the control port */ int routerstatus_format_entry(char *buf, size_t buf_len, routerstatus_t *rs, const char *version, - int first_line_only, int v2_format) -/* XXX: first_line_only and v2_format should probably be be both - * replaced by a single purpose parameter. - */ + routerstatus_format_type_t format) { int r; struct in_addr in; @@ -1925,10 +2008,11 @@ routerstatus_format_entry(char *buf, size_t buf_len, tor_inet_ntoa(&in, ipaddr, sizeof(ipaddr)); r = tor_snprintf(buf, buf_len, - "r %s %s %s %s %s %d %d\n", + "r %s %s %s%s%s %s %d %d\n", rs->nickname, identity64, - digest64, + (format==NS_V3_CONSENSUS_MICRODESC)?"":digest64, + (format==NS_V3_CONSENSUS_MICRODESC)?"":" ", published, ipaddr, (int)rs->or_port, @@ -1937,7 +2021,12 @@ routerstatus_format_entry(char *buf, size_t buf_len, log_warn(LD_BUG, "Not enough space in buffer."); return -1; } - if (first_line_only) + + /* TODO: Maybe we want to pass in what we need to build the rest of + * this here, instead of in the caller. Then we could use the + * networkstatus_type_t values, with an additional control port value + * added -MP */ + if (format == NS_V3_CONSENSUS || format == NS_V3_CONSENSUS_MICRODESC) return 0; cp = buf + strlen(buf); @@ -1974,62 +2063,87 @@ routerstatus_format_entry(char *buf, size_t buf_len, cp += strlen(cp); } - if (!v2_format) { + if (format != NS_V2) { routerinfo_t* desc = router_get_by_digest(rs->identity_digest); + uint32_t bw; + + if (format != NS_CONTROL_PORT) { + /* Blow up more or less nicely if we didn't get anything or not the + * thing we expected. + */ + if (!desc) { + char id[HEX_DIGEST_LEN+1]; + char dd[HEX_DIGEST_LEN+1]; + + base16_encode(id, sizeof(id), rs->identity_digest, DIGEST_LEN); + base16_encode(dd, sizeof(dd), rs->descriptor_digest, DIGEST_LEN); + log_warn(LD_BUG, "Cannot get any descriptor for %s " + "(wanted descriptor %s).", + id, dd); + return -1; + }; + + /* This assert can fire for the control port, because + * it can request NS documents before all descriptors + * have been fetched. */ + if (memcmp(desc->cache_info.signed_descriptor_digest, + rs->descriptor_digest, + DIGEST_LEN)) { + char rl_d[HEX_DIGEST_LEN+1]; + char rs_d[HEX_DIGEST_LEN+1]; + char id[HEX_DIGEST_LEN+1]; + + base16_encode(rl_d, sizeof(rl_d), + desc->cache_info.signed_descriptor_digest, DIGEST_LEN); + base16_encode(rs_d, sizeof(rs_d), rs->descriptor_digest, DIGEST_LEN); + base16_encode(id, sizeof(id), rs->identity_digest, DIGEST_LEN); + log_err(LD_BUG, "descriptor digest in routerlist does not match " + "the one in routerstatus: %s vs %s " + "(router %s)\n", + rl_d, rs_d, id); + + tor_assert(!memcmp(desc->cache_info.signed_descriptor_digest, + rs->descriptor_digest, + DIGEST_LEN)); + }; + } - /* Blow up more or less nicely if we didn't get anything or not the - * thing we expected. - */ - if (!desc) { - char id[HEX_DIGEST_LEN+1]; - char dd[HEX_DIGEST_LEN+1]; - - base16_encode(id, sizeof(id), rs->identity_digest, DIGEST_LEN); - base16_encode(dd, sizeof(dd), rs->descriptor_digest, DIGEST_LEN); - log_warn(LD_BUG, "Cannot get any descriptor for %s " - "(wanted descriptor %s).", - id, dd); - return -1; - }; - if (memcmp(desc->cache_info.signed_descriptor_digest, - rs->descriptor_digest, - DIGEST_LEN)) { - char rl_d[HEX_DIGEST_LEN+1]; - char rs_d[HEX_DIGEST_LEN+1]; - char id[HEX_DIGEST_LEN+1]; - - base16_encode(rl_d, sizeof(rl_d), - desc->cache_info.signed_descriptor_digest, DIGEST_LEN); - base16_encode(rs_d, sizeof(rs_d), rs->descriptor_digest, DIGEST_LEN); - base16_encode(id, sizeof(id), rs->identity_digest, DIGEST_LEN); - log_err(LD_BUG, "descriptor digest in routerlist does not match " - "the one in routerstatus: %s vs %s " - "(router %s)\n", - rl_d, rs_d, id); - - tor_assert(!memcmp(desc->cache_info.signed_descriptor_digest, - rs->descriptor_digest, - DIGEST_LEN)); - }; - + if (format == NS_CONTROL_PORT && rs->has_bandwidth) { + bw = rs->bandwidth; + } else { + tor_assert(desc); + bw = router_get_advertised_bandwidth_capped(desc) / 1000; + } r = tor_snprintf(cp, buf_len - (cp-buf), - "w Bandwidth=%d\n", - router_get_advertised_bandwidth_capped(desc) / 1024); + "w Bandwidth=%d\n", bw); + if (r<0) { log_warn(LD_BUG, "Not enough space in buffer."); return -1; } cp += strlen(cp); + if (format == NS_V3_VOTE && rs->has_measured_bw) { + *--cp = '\0'; /* Kill "\n" */ + r = tor_snprintf(cp, buf_len - (cp-buf), + " Measured=%d\n", rs->measured_bw); + if (r<0) { + log_warn(LD_BUG, "Not enough space in buffer for weight line."); + return -1; + } + cp += strlen(cp); + } - summary = policy_summarize(desc->exit_policy); - r = tor_snprintf(cp, buf_len - (cp-buf), "p %s\n", summary); - if (r<0) { - log_warn(LD_BUG, "Not enough space in buffer."); + if (desc) { + summary = policy_summarize(desc->exit_policy); + r = tor_snprintf(cp, buf_len - (cp-buf), "p %s\n", summary); + if (r<0) { + log_warn(LD_BUG, "Not enough space in buffer."); + tor_free(summary); + return -1; + } + cp += strlen(cp); tor_free(summary); - return -1; } - cp += strlen(cp); - tor_free(summary); } return 0; @@ -2132,9 +2246,7 @@ get_possible_sybil_list(const smartlist_t *routers) /** Extract status information from <b>ri</b> and from other authority * functions and store it in <b>rs</b>>. If <b>naming</b>, consider setting - * the named flag in <b>rs</b>. If not <b>exits_can_be_guards</b>, never mark - * an exit as a guard. If <b>listbadexits</b>, consider setting the badexit - * flag. + * the named flag in <b>rs</b>. * * We assume that ri-\>is_running has already been set, e.g. by * dirserv_set_router_is_running(ri, now); @@ -2142,8 +2254,8 @@ get_possible_sybil_list(const smartlist_t *routers) void set_routerstatus_from_routerinfo(routerstatus_t *rs, routerinfo_t *ri, time_t now, - int naming, int exits_can_be_guards, - int listbadexits, int listbaddirs) + int naming, int listbadexits, + int listbaddirs) { int unstable_version = !tor_version_as_new_as(ri->platform,"0.1.1.16-rc-cvs"); @@ -2172,11 +2284,10 @@ set_routerstatus_from_routerinfo(routerstatus_t *rs, rs->is_valid = ri->is_valid; if (rs->is_fast && - (!rs->is_exit || exits_can_be_guards) && (router_get_advertised_bandwidth(ri) >= BANDWIDTH_TO_GUARANTEE_GUARD || router_get_advertised_bandwidth(ri) >= - (exits_can_be_guards ? guard_bandwidth_including_exits : - guard_bandwidth_excluding_exits))) { + MIN(guard_bandwidth_including_exits, + guard_bandwidth_excluding_exits))) { long tk = rep_hist_get_weighted_time_known( ri->cache_info.identity_digest, now); double wfu = rep_hist_get_weighted_fractional_uptime( @@ -2232,6 +2343,177 @@ router_clear_status_flags(routerinfo_t *router) router->is_bad_exit = router->is_bad_directory = 0; } +/** + * Helper function to parse out a line in the measured bandwidth file + * into a measured_bw_line_t output structure. Returns -1 on failure + * or 0 on success. + */ +int +measured_bw_line_parse(measured_bw_line_t *out, const char *orig_line) +{ + char *line = tor_strdup(orig_line); + char *cp = line; + int got_bw = 0; + int got_node_id = 0; + char *strtok_state; /* lame sauce d'jour */ + cp = tor_strtok_r(cp, " \t", &strtok_state); + + if (!cp) { + log_warn(LD_DIRSERV, "Invalid line in bandwidth file: %s", + escaped(orig_line)); + tor_free(line); + return -1; + } + + if (orig_line[strlen(orig_line)-1] != '\n') { + log_warn(LD_DIRSERV, "Incomplete line in bandwidth file: %s", + escaped(orig_line)); + tor_free(line); + return -1; + } + + do { + if (strcmpstart(cp, "bw=") == 0) { + int parse_ok = 0; + char *endptr; + if (got_bw) { + log_warn(LD_DIRSERV, "Double bw= in bandwidth file line: %s", + escaped(orig_line)); + tor_free(line); + return -1; + } + cp+=strlen("bw="); + + out->bw = tor_parse_long(cp, 0, 0, LONG_MAX, &parse_ok, &endptr); + if (!parse_ok || (*endptr && !TOR_ISSPACE(*endptr))) { + log_warn(LD_DIRSERV, "Invalid bandwidth in bandwidth file line: %s", + escaped(orig_line)); + tor_free(line); + return -1; + } + got_bw=1; + } else if (strcmpstart(cp, "node_id=$") == 0) { + if (got_node_id) { + log_warn(LD_DIRSERV, "Double node_id= in bandwidth file line: %s", + escaped(orig_line)); + tor_free(line); + return -1; + } + cp+=strlen("node_id=$"); + + if (strlen(cp) != HEX_DIGEST_LEN || + base16_decode(out->node_id, DIGEST_LEN, cp, HEX_DIGEST_LEN)) { + log_warn(LD_DIRSERV, "Invalid node_id in bandwidth file line: %s", + escaped(orig_line)); + tor_free(line); + return -1; + } + strncpy(out->node_hex, cp, sizeof(out->node_hex)); + got_node_id=1; + } + } while ((cp = tor_strtok_r(NULL, " \t", &strtok_state))); + + if (got_bw && got_node_id) { + tor_free(line); + return 0; + } else { + log_warn(LD_DIRSERV, "Incomplete line in bandwidth file: %s", + escaped(orig_line)); + tor_free(line); + return -1; + } +} + +/** + * Helper function to apply a parsed measurement line to a list + * of bandwidth statuses. Returns true if a line is found, + * false otherwise. + */ +int +measured_bw_line_apply(measured_bw_line_t *parsed_line, + smartlist_t *routerstatuses) +{ + routerstatus_t *rs = NULL; + if (!routerstatuses) + return 0; + + rs = smartlist_bsearch(routerstatuses, parsed_line->node_id, + compare_digest_to_routerstatus_entry); + + if (rs) { + rs->has_measured_bw = 1; + rs->measured_bw = (uint32_t)parsed_line->bw; + } else { + log_info(LD_DIRSERV, "Node ID %s not found in routerstatus list", + parsed_line->node_hex); + } + + return rs != NULL; +} + +/** + * Read the measured bandwidth file and apply it to the list of + * routerstatuses. Returns -1 on error, 0 otherwise. + */ +int +dirserv_read_measured_bandwidths(const char *from_file, + smartlist_t *routerstatuses) +{ + char line[256]; + FILE *fp = fopen(from_file, "r"); + int applied_lines = 0; + time_t file_time; + int ok; + if (fp == NULL) { + log_warn(LD_CONFIG, "Can't open bandwidth file at configured location: %s", + from_file); + return -1; + } + + if (!fgets(line, sizeof(line), fp) + || !strlen(line) || line[strlen(line)-1] != '\n') { + log_warn(LD_DIRSERV, "Long or truncated time in bandwidth file: %s", + escaped(line)); + fclose(fp); + return -1; + } + + line[strlen(line)-1] = '\0'; + file_time = tor_parse_ulong(line, 10, 0, ULONG_MAX, &ok, NULL); + if (!ok) { + log_warn(LD_DIRSERV, "Non-integer time in bandwidth file: %s", + escaped(line)); + fclose(fp); + return -1; + } + + if ((time(NULL) - file_time) > MAX_MEASUREMENT_AGE) { + log_warn(LD_DIRSERV, "Bandwidth measurement file stale. Age: %u", + (unsigned)(time(NULL) - file_time)); + fclose(fp); + return -1; + } + + if (routerstatuses) + smartlist_sort(routerstatuses, compare_routerstatus_entries); + + while (!feof(fp)) { + measured_bw_line_t parsed_line; + if (fgets(line, sizeof(line), fp) && strlen(line)) { + if (measured_bw_line_parse(&parsed_line, line) != -1) { + if (measured_bw_line_apply(&parsed_line, routerstatuses) > 0) + applied_lines++; + } + } + } + + fclose(fp); + log_info(LD_DIRSERV, + "Bandwidth measurement file successfully read. " + "Applied %d measurements.", applied_lines); + return 0; +} + /** Return a new networkstatus_t* containing our current opinion. (For v3 * authorities) */ networkstatus_t * @@ -2249,22 +2531,18 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_env_t *private_key, int naming = options->NamingAuthoritativeDir; int listbadexits = options->AuthDirListBadExits; int listbaddirs = options->AuthDirListBadDirs; - int exits_can_be_guards; routerlist_t *rl = router_get_routerlist(); time_t now = time(NULL); time_t cutoff = now - ROUTER_MAX_AGE_TO_PUBLISH; networkstatus_voter_info_t *voter = NULL; vote_timing_t timing; digestmap_t *omit_as_sybil = NULL; - int vote_on_reachability = 1; + const int vote_on_reachability = running_long_enough_to_decide_unreachable(); + smartlist_t *microdescriptors = NULL; tor_assert(private_key); tor_assert(cert); - if (now - time_of_process_start < - options->TestingAuthDirTimeToLearnReachability) - vote_on_reachability = 0; - if (resolve_my_address(LOG_WARN, options, &addr, &hostname)<0) { log_warn(LD_NET, "Couldn't resolve my hostname"); return NULL; @@ -2299,27 +2577,24 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_env_t *private_key, dirserv_compute_performance_thresholds(rl); - /* XXXX We should take steps to keep this from oscillating if - * total_exit_bandwidth is close to total_bandwidth/3. */ - exits_can_be_guards = total_exit_bandwidth >= (total_bandwidth / 3); - routers = smartlist_create(); smartlist_add_all(routers, rl->routers); routers_sort_by_identity(routers); omit_as_sybil = get_possible_sybil_list(routers); routerstatuses = smartlist_create(); + microdescriptors = smartlist_create(); - SMARTLIST_FOREACH(routers, routerinfo_t *, ri, { + SMARTLIST_FOREACH_BEGIN(routers, routerinfo_t *, ri) { if (ri->cache_info.published_on >= cutoff) { routerstatus_t *rs; vote_routerstatus_t *vrs; + microdesc_t *md; vrs = tor_malloc_zero(sizeof(vote_routerstatus_t)); rs = &vrs->status; set_routerstatus_from_routerinfo(rs, ri, now, - naming, exits_can_be_guards, - listbadexits, listbaddirs); + naming, listbadexits, listbaddirs); if (digestmap_get(omit_as_sybil, ri->cache_info.identity_digest)) clear_status_flags_on_sybil(rs); @@ -2328,12 +2603,39 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_env_t *private_key, rs->is_running = 0; vrs->version = version_from_platform(ri->platform); + md = dirvote_create_microdescriptor(ri); + if (md) { + char buf[128]; + vote_microdesc_hash_t *h; + dirvote_format_microdesc_vote_line(buf, sizeof(buf), md); + h = tor_malloc(sizeof(vote_microdesc_hash_t)); + h->microdesc_hash_line = tor_strdup(buf); + h->next = NULL; + vrs->microdesc = h; + md->last_listed = now; + smartlist_add(microdescriptors, md); + } + smartlist_add(routerstatuses, vrs); } - }); + } SMARTLIST_FOREACH_END(ri); + + { + smartlist_t *added = + microdescs_add_list_to_cache(get_microdesc_cache(), + microdescriptors, SAVED_NOWHERE, 0); + smartlist_free(added); + smartlist_free(microdescriptors); + } + smartlist_free(routers); digestmap_free(omit_as_sybil, NULL); + if (options->V3BandwidthsFile) { + dirserv_read_measured_bandwidths(options->V3BandwidthsFile, + routerstatuses); + } + v3_out = tor_malloc_zero(sizeof(networkstatus_t)); v3_out->type = NS_TYPE_VOTE; @@ -2383,15 +2685,22 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_env_t *private_key, } smartlist_sort_strings(v3_out->known_flags); + if (options->ConsensusParams) { + v3_out->net_params = smartlist_create(); + smartlist_split_string(v3_out->net_params, + options->ConsensusParams, NULL, 0, 0); + smartlist_sort_strings(v3_out->net_params); + } + voter = tor_malloc_zero(sizeof(networkstatus_voter_info_t)); voter->nickname = tor_strdup(options->Nickname); memcpy(voter->identity_digest, identity_digest, DIGEST_LEN); + voter->sigs = smartlist_create(); voter->address = hostname; voter->addr = addr; voter->dir_port = options->DirPort; voter->or_port = options->ORPort; voter->contact = tor_strdup(contact); - memcpy(voter->signing_key_digest, signing_key_digest, DIGEST_LEN); if (options->V3AuthUseLegacyKey) { authority_cert_t *c = get_my_v3_legacy_cert(); if (c) { @@ -2435,13 +2744,12 @@ generate_v2_networkstatus_opinion(void) int versioning = options->VersioningAuthoritativeDir; int listbaddirs = options->AuthDirListBadDirs; int listbadexits = options->AuthDirListBadExits; - int exits_can_be_guards; const char *contact; char *version_lines = NULL; smartlist_t *routers = NULL; digestmap_t *omit_as_sybil = NULL; - private_key = get_identity_key(); + private_key = get_server_identity_key(); if (resolve_my_address(LOG_WARN, options, &addr, &hostname)<0) { log_warn(LD_NET, "Couldn't resolve my hostname"); @@ -2515,10 +2823,6 @@ generate_v2_networkstatus_opinion(void) dirserv_compute_performance_thresholds(rl); - /* XXXX We should take steps to keep this from oscillating if - * total_exit_bandwidth is close to total_bandwidth/3. */ - exits_can_be_guards = total_exit_bandwidth >= (total_bandwidth / 3); - routers = smartlist_create(); smartlist_add_all(routers, rl->routers); routers_sort_by_identity(routers); @@ -2531,13 +2835,12 @@ generate_v2_networkstatus_opinion(void) char *version = version_from_platform(ri->platform); set_routerstatus_from_routerinfo(&rs, ri, now, - naming, exits_can_be_guards, - listbadexits, listbaddirs); + naming, listbadexits, listbaddirs); if (digestmap_get(omit_as_sybil, ri->cache_info.identity_digest)) clear_status_flags_on_sybil(&rs); - if (routerstatus_format_entry(outp, endp-outp, &rs, version, 0, 1)) { + if (routerstatus_format_entry(outp, endp-outp, &rs, version, NS_V2)) { log_warn(LD_BUG, "Unable to print router status."); tor_free(version); goto done; @@ -2559,7 +2862,8 @@ generate_v2_networkstatus_opinion(void) outp += strlen(outp); note_crypto_pk_op(SIGN_DIR); - if (router_append_dirobj_signature(outp,endp-outp,digest,private_key)<0) { + if (router_append_dirobj_signature(outp,endp-outp,digest,DIGEST_LEN, + private_key)<0) { log_warn(LD_BUG, "Unable to sign router status."); goto done; } @@ -2591,10 +2895,8 @@ generate_v2_networkstatus_opinion(void) tor_free(status); tor_free(hostname); tor_free(identity_pkey); - if (routers) - smartlist_free(routers); - if (omit_as_sybil) - digestmap_free(omit_as_sybil, NULL); + smartlist_free(routers); + digestmap_free(omit_as_sybil, NULL); return r; } @@ -2642,7 +2944,8 @@ dirserv_get_networkstatus_v2_fingerprints(smartlist_t *result, log_info(LD_DIRSERV, "Client requested 'all' network status objects; we have none."); } else if (!strcmpstart(key, "fp/")) { - dir_split_resource_into_fingerprints(key+3, result, NULL, 1, 1); + dir_split_resource_into_fingerprints(key+3, result, NULL, + DSR_HEX|DSR_SORT_UNIQ); } } @@ -2709,10 +3012,12 @@ dirserv_get_routerdesc_fingerprints(smartlist_t *fps_out, const char *key, } else if (!strcmpstart(key, "d/")) { by_id = 0; key += strlen("d/"); - dir_split_resource_into_fingerprints(key, fps_out, NULL, 1, 1); + dir_split_resource_into_fingerprints(key, fps_out, NULL, + DSR_HEX|DSR_SORT_UNIQ); } else if (!strcmpstart(key, "fp/")) { key += strlen("fp/"); - dir_split_resource_into_fingerprints(key, fps_out, NULL, 1, 1); + dir_split_resource_into_fingerprints(key, fps_out, NULL, + DSR_HEX|DSR_SORT_UNIQ); } else { *msg = "Key not recognized"; return -1; @@ -2777,7 +3082,8 @@ dirserv_get_routerdescs(smartlist_t *descs_out, const char *key, } else if (!strcmpstart(key, "/tor/server/d/")) { smartlist_t *digests = smartlist_create(); key += strlen("/tor/server/d/"); - dir_split_resource_into_fingerprints(key, digests, NULL, 1, 1); + dir_split_resource_into_fingerprints(key, digests, NULL, + DSR_HEX|DSR_SORT_UNIQ); SMARTLIST_FOREACH(digests, const char *, d, { signed_descriptor_t *sd = router_get_by_descriptor_digest(d); @@ -2790,7 +3096,8 @@ dirserv_get_routerdescs(smartlist_t *descs_out, const char *key, smartlist_t *digests = smartlist_create(); time_t cutoff = time(NULL) - ROUTER_MAX_AGE_TO_PUBLISH; key += strlen("/tor/server/fp/"); - dir_split_resource_into_fingerprints(key, digests, NULL, 1, 1); + dir_split_resource_into_fingerprints(key, digests, NULL, + DSR_HEX|DSR_SORT_UNIQ); SMARTLIST_FOREACH(digests, const char *, d, { if (router_digest_is_me(d)) { @@ -2841,34 +3148,82 @@ dirserv_orconn_tls_done(const char *address, tor_assert(address); tor_assert(digest_rcvd); - SMARTLIST_FOREACH(rl->routers, routerinfo_t *, ri, { + /* XXX023 Doing a loop like this is stupid. We should just look up the + * router by digest_rcvd, and see if address, orport, and as_advertised + * match up. -NM */ + SMARTLIST_FOREACH_BEGIN(rl->routers, routerinfo_t *, ri) { if (!strcasecmp(address, ri->address) && or_port == ri->or_port && as_advertised && !memcmp(ri->cache_info.identity_digest, digest_rcvd, DIGEST_LEN)) { /* correct digest. mark this router reachable! */ if (!bridge_auth || ri->purpose == ROUTER_PURPOSE_BRIDGE) { - log_info(LD_DIRSERV, "Found router %s to be reachable. Yay.", - ri->nickname); - rep_hist_note_router_reachable(digest_rcvd, now); + tor_addr_t addr, *addrp=NULL; + log_info(LD_DIRSERV, "Found router %s to be reachable at %s:%d. Yay.", + ri->nickname, address, ri->or_port ); + if (tor_addr_from_str(&addr, ri->address) != -1) + addrp = &addr; + else + log_warn(LD_BUG, "Couldn't parse IP address \"%s\"", ri->address); + rep_hist_note_router_reachable(digest_rcvd, addrp, or_port, now); ri->last_reachable = now; } } - }); + } SMARTLIST_FOREACH_END(ri); /* FFFF Maybe we should reinstate the code that dumps routers with the same * addr/port but with nonmatching keys, but instead of dumping, we should * skip testing. */ } -/** Auth dir server only: if <b>try_all</b> is 1, launch connections to - * all known routers; else we want to load balance such that we only +/** Called when we, as an authority, receive a new router descriptor either as + * an upload or a download. Used to decide whether to relaunch reachability + * testing for the server. */ +int +dirserv_should_launch_reachability_test(routerinfo_t *ri, routerinfo_t *ri_old) +{ + if (!authdir_mode_handles_descs(get_options(), ri->purpose)) + return 0; + if (!ri_old) { + /* New router: Launch an immediate reachability test, so we will have an + * opinion soon in case we're generating a consensus soon */ + return 1; + } + if (ri_old->is_hibernating && !ri->is_hibernating) { + /* It just came out of hibernation; launch a reachability test */ + return 1; + } + if (! routers_have_same_or_addr(ri, ri_old)) { + /* Address or port changed; launch a reachability test */ + return 1; + } + return 0; +} + +/** Helper function for dirserv_test_reachability(). Start a TLS + * connection to <b>router</b>, and annotate it with when we started + * the test. */ +void +dirserv_single_reachability_test(time_t now, routerinfo_t *router) +{ + tor_addr_t router_addr; + log_debug(LD_OR,"Testing reachability of %s at %s:%u.", + router->nickname, router->address, router->or_port); + /* Remember when we started trying to determine reachability */ + if (!router->testing_since) + router->testing_since = now; + tor_addr_from_ipv4h(&router_addr, router->addr); + connection_or_connect(&router_addr, router->or_port, + router->cache_info.identity_digest); +} + +/** Auth dir server only: load balance such that we only * try a few connections per call. * * The load balancing is such that if we get called once every ten - * seconds, we will cycle through all the tests in 1280 seconds (a - * bit over 20 minutes). + * seconds, we will cycle through all the tests in + * REACHABILITY_TEST_CYCLE_PERIOD seconds (a bit over 20 minutes). */ void -dirserv_test_reachability(time_t now, int try_all) +dirserv_test_reachability(time_t now) { /* XXX decide what to do here; see or-talk thread "purging old router * information, revocation." -NM @@ -2885,38 +3240,33 @@ dirserv_test_reachability(time_t now, int try_all) SMARTLIST_FOREACH_BEGIN(rl->routers, routerinfo_t *, router) { const char *id_digest = router->cache_info.identity_digest; - tor_addr_t router_addr; if (router_is_me(router)) continue; if (bridge_auth && router->purpose != ROUTER_PURPOSE_BRIDGE) continue; /* bridge authorities only test reachability on bridges */ // if (router->cache_info.published_on > cutoff) // continue; - if (try_all || (((uint8_t)id_digest[0]) % 128) == ctr) { - log_debug(LD_OR,"Testing reachability of %s at %s:%u.", - router->nickname, router->address, router->or_port); - /* Remember when we started trying to determine reachability */ - if (!router->testing_since) - router->testing_since = now; - tor_addr_from_ipv4h(&router_addr, router->addr); - connection_or_connect(&router_addr, router->or_port, id_digest); + if ((((uint8_t)id_digest[0]) % REACHABILITY_MODULO_PER_TEST) == ctr) { + dirserv_single_reachability_test(now, router); } } SMARTLIST_FOREACH_END(router); - if (!try_all) /* increment ctr */ - ctr = (ctr + 1) % 128; + ctr = (ctr + 1) % REACHABILITY_MODULO_PER_TEST; /* increment ctr */ } -/** Given a fingerprint <b>fp</b> which is either set if we're looking - * for a v2 status, or zeroes if we're looking for a v3 status, return - * a pointer to the appropriate cached dir object, or NULL if there isn't - * one available. */ +/** Given a fingerprint <b>fp</b> which is either set if we're looking for a + * v2 status, or zeroes if we're looking for a v3 status, or a NUL-padded + * flavor name if we want a flavored v3 status, return a pointer to the + * appropriate cached dir object, or NULL if there isn't one available. */ static cached_dir_t * lookup_cached_dir_by_fp(const char *fp) { cached_dir_t *d = NULL; - if (tor_digest_is_zero(fp) && cached_v3_networkstatus) - d = cached_v3_networkstatus; - else if (router_digest_is_me(fp) && the_v2_networkstatus) + if (tor_digest_is_zero(fp) && cached_consensuses) + d = strmap_get(cached_consensuses, "ns"); + else if (memchr(fp, '\0', DIGEST_LEN) && cached_consensuses && + (d = strmap_get(cached_consensuses, fp))) { + /* this here interface is a nasty hack XXXX023 */; + } else if (router_digest_is_me(fp) && the_v2_networkstatus) d = the_v2_networkstatus; else if (cached_v2_networkstatus) d = digestmap_get(cached_v2_networkstatus, fp); @@ -3002,6 +3352,18 @@ dirserv_have_any_serverdesc(smartlist_t *fps, int spool_src) return 0; } +/** Return true iff any of the 256-bit elements in <b>fps</b> is the digest of + * a microdescriptor we have. */ +int +dirserv_have_any_microdesc(const smartlist_t *fps) +{ + microdesc_cache_t *cache = get_microdesc_cache(); + SMARTLIST_FOREACH(fps, const char *, fp, + if (microdesc_cache_lookup_by_digest256(cache, fp)) + return 1); + return 0; +} + /** Return an approximate estimate of the number of bytes that will * be needed to transmit the server descriptors (if is_serverdescs -- * they can be either d/ or fp/ queries) or networkstatus objects (if @@ -3033,6 +3395,17 @@ dirserv_estimate_data_size(smartlist_t *fps, int is_serverdescs, return result; } +/** Given a list of microdescriptor hashes, guess how many bytes will be + * needed to transmit them, and return the guess. */ +size_t +dirserv_estimate_microdesc_size(const smartlist_t *fps, int compressed) +{ + size_t result = smartlist_len(fps) * microdesc_average_size(NULL); + if (compressed) + result /= 2; + return result; +} + /** When we're spooling data onto our outbuf, add more whenever we dip * below this threshold. */ #define DIRSERV_BUFFER_MIN 16384 @@ -3096,6 +3469,8 @@ connection_dirserv_add_servers_to_outbuf(dir_connection_t *conn) #endif body = signed_descriptor_get_body(sd); if (conn->zlib_state) { + /* XXXX022 This 'last' business should actually happen on the last + * routerinfo, not on the last fingerprint. */ int last = ! smartlist_len(conn->fingerprint_stack); connection_write_to_buf_zlib(body, sd->signed_descriptor_len, conn, last); @@ -3119,6 +3494,44 @@ connection_dirserv_add_servers_to_outbuf(dir_connection_t *conn) return 0; } +/** Spooling helper: called when we're sending a bunch of microdescriptors, + * and the outbuf has become too empty. Pulls some entries from + * fingerprint_stack, and writes the corresponding microdescs onto outbuf. If + * we run out of entries, flushes the zlib state and sets the spool source to + * NONE. Returns 0 on success, negative on failure. + */ +static int +connection_dirserv_add_microdescs_to_outbuf(dir_connection_t *conn) +{ + microdesc_cache_t *cache = get_microdesc_cache(); + while (smartlist_len(conn->fingerprint_stack) && + buf_datalen(conn->_base.outbuf) < DIRSERV_BUFFER_MIN) { + char *fp256 = smartlist_pop_last(conn->fingerprint_stack); + microdesc_t *md = microdesc_cache_lookup_by_digest256(cache, fp256); + tor_free(fp256); + if (!md) + continue; + if (conn->zlib_state) { + /* XXXX022 This 'last' business should actually happen on the last + * routerinfo, not on the last fingerprint. */ + int last = !smartlist_len(conn->fingerprint_stack); + connection_write_to_buf_zlib(md->body, md->bodylen, conn, last); + if (last) { + tor_zlib_free(conn->zlib_state); + conn->zlib_state = NULL; + } + } else { + connection_write_to_buf(md->body, md->bodylen, TO_CONN(conn)); + } + } + if (!smartlist_len(conn->fingerprint_stack)) { + conn->dir_spool_src = DIR_SPOOL_NONE; + smartlist_free(conn->fingerprint_stack); + conn->fingerprint_stack = NULL; + } + return 0; +} + /** Spooling helper: Called when we're sending a directory or networkstatus, * and the outbuf has become too empty. Pulls some bytes from * <b>conn</b>-\>cached_dir-\>dir_z, uncompresses them if appropriate, and @@ -3201,8 +3614,7 @@ connection_dirserv_add_networkstatus_bytes_to_outbuf(dir_connection_t *conn) } } else { connection_dirserv_finish_spooling(conn); - if (conn->fingerprint_stack) - smartlist_free(conn->fingerprint_stack); + smartlist_free(conn->fingerprint_stack); conn->fingerprint_stack = NULL; return 0; } @@ -3226,6 +3638,8 @@ connection_dirserv_flushed_some(dir_connection_t *conn) case DIR_SPOOL_SERVER_BY_DIGEST: case DIR_SPOOL_SERVER_BY_FP: return connection_dirserv_add_servers_to_outbuf(conn); + case DIR_SPOOL_MICRODESC: + return connection_dirserv_add_microdescs_to_outbuf(conn); case DIR_SPOOL_CACHED_DIR: return connection_dirserv_add_dir_bytes_to_outbuf(conn); case DIR_SPOOL_NETWORKSTATUS: @@ -3247,10 +3661,10 @@ dirserv_free_all(void) cached_dir_decref(the_v2_networkstatus); cached_dir_decref(cached_directory); clear_cached_dir(&cached_runningrouters); - if (cached_v2_networkstatus) { - digestmap_free(cached_v2_networkstatus, _free_cached_dir); - cached_v2_networkstatus = NULL; - } - cached_dir_decref(cached_v3_networkstatus); + + digestmap_free(cached_v2_networkstatus, _free_cached_dir); + cached_v2_networkstatus = NULL; + strmap_free(cached_consensuses, _free_cached_dir); + cached_consensuses = NULL; } diff --git a/src/or/dirserv.h b/src/or/dirserv.h new file mode 100644 index 0000000000..569abfca2e --- /dev/null +++ b/src/or/dirserv.h @@ -0,0 +1,146 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file dirserv.h + * \brief Header file for dirserv.c. + **/ + +#ifndef _TOR_DIRSERV_H +#define _TOR_DIRSERV_H + +/** What fraction (1 over this number) of the relay ID space do we + * (as a directory authority) launch connections to at each reachability + * test? */ +#define REACHABILITY_MODULO_PER_TEST 128 + +/** How often (in seconds) do we launch reachability tests? */ +#define REACHABILITY_TEST_INTERVAL 10 + +/** How many seconds apart are the reachability tests for a given relay? */ +#define REACHABILITY_TEST_CYCLE_PERIOD \ + (REACHABILITY_TEST_INTERVAL*REACHABILITY_MODULO_PER_TEST) + +/** Maximum length of an exit policy summary. */ +#define MAX_EXITPOLICY_SUMMARY_LEN 1000 + +/** Maximum allowable length of a version line in a networkstatus. */ +#define MAX_V_LINE_LEN 128 +/** Length of "r Authority BadDirectory BadExit Exit Fast Guard HSDir Named + * Running Stable Unnamed V2Dir Valid\n". */ +#define MAX_FLAG_LINE_LEN 96 +/** Length of "w" line for weighting. Currently at most + * "w Bandwidth=<uint32t> Measured=<uint32t>\n" */ +#define MAX_WEIGHT_LINE_LEN (12+10+10+10+1) +/** Maximum length of an exit policy summary line. */ +#define MAX_POLICY_LINE_LEN (3+MAX_EXITPOLICY_SUMMARY_LEN) +/** Amount of space to allocate for each entry: r, s, and v lines. */ +#define RS_ENTRY_LEN \ + ( /* first line */ \ + MAX_NICKNAME_LEN+BASE64_DIGEST_LEN*2+ISO_TIME_LEN+INET_NTOA_BUF_LEN+ \ + 5*2 /* ports */ + 10 /* punctuation */ + \ + /* second line */ \ + MAX_FLAG_LINE_LEN + \ + /* weight line */ \ + MAX_WEIGHT_LINE_LEN + \ + /* p line. */ \ + MAX_POLICY_LINE_LEN + \ + /* v line. */ \ + MAX_V_LINE_LEN \ + ) + +#define UNNAMED_ROUTER_NICKNAME "Unnamed" + +int connection_dirserv_flushed_some(dir_connection_t *conn); + +int dirserv_add_own_fingerprint(const char *nickname, crypto_pk_env_t *pk); +int dirserv_load_fingerprint_file(void); +void dirserv_free_fingerprint_list(void); +const char *dirserv_get_nickname_by_digest(const char *digest); +enum was_router_added_t dirserv_add_multiple_descriptors( + const char *desc, uint8_t purpose, + const char *source, + const char **msg); +enum was_router_added_t dirserv_add_descriptor(routerinfo_t *ri, + const char **msg, + const char *source); +void dirserv_set_router_is_running(routerinfo_t *router, time_t now); +int list_server_status_v1(smartlist_t *routers, char **router_status_out, + int for_controller); +int dirserv_dump_directory_to_string(char **dir_out, + crypto_pk_env_t *private_key); + +int directory_fetches_from_authorities(or_options_t *options); +int directory_fetches_dir_info_early(or_options_t *options); +int directory_fetches_dir_info_later(or_options_t *options); +int directory_caches_v2_dir_info(or_options_t *options); +#define directory_caches_v1_dir_info(o) directory_caches_v2_dir_info(o) +int directory_caches_dir_info(or_options_t *options); +int directory_permits_begindir_requests(or_options_t *options); +int directory_permits_controller_requests(or_options_t *options); +int directory_too_idle_to_fetch_descriptors(or_options_t *options, time_t now); + +void directory_set_dirty(void); +cached_dir_t *dirserv_get_directory(void); +cached_dir_t *dirserv_get_runningrouters(void); +cached_dir_t *dirserv_get_consensus(const char *flavor_name); +void dirserv_set_cached_directory(const char *directory, time_t when, + int is_running_routers); +void dirserv_set_cached_networkstatus_v2(const char *directory, + const char *identity, + time_t published); +void dirserv_set_cached_consensus_networkstatus(const char *consensus, + const char *flavor_name, + const digests_t *digests, + time_t published); +void dirserv_clear_old_networkstatuses(time_t cutoff); +void dirserv_clear_old_v1_info(time_t now); +void dirserv_get_networkstatus_v2(smartlist_t *result, const char *key); +void dirserv_get_networkstatus_v2_fingerprints(smartlist_t *result, + const char *key); +int dirserv_get_routerdesc_fingerprints(smartlist_t *fps_out, const char *key, + const char **msg, + int for_unencrypted_conn, + int is_extrainfo); +int dirserv_get_routerdescs(smartlist_t *descs_out, const char *key, + const char **msg); +void dirserv_orconn_tls_done(const char *address, + uint16_t or_port, + const char *digest_rcvd, + int as_advertised); +int dirserv_should_launch_reachability_test(routerinfo_t *ri, + routerinfo_t *ri_old); +void dirserv_single_reachability_test(time_t now, routerinfo_t *router); +void dirserv_test_reachability(time_t now); +int authdir_wants_to_reject_router(routerinfo_t *ri, const char **msg, + int complain); +int dirserv_would_reject_router(routerstatus_t *rs); +int dirserv_remove_old_statuses(smartlist_t *fps, time_t cutoff); +int dirserv_have_any_serverdesc(smartlist_t *fps, int spool_src); +int dirserv_have_any_microdesc(const smartlist_t *fps); +size_t dirserv_estimate_data_size(smartlist_t *fps, int is_serverdescs, + int compressed); +size_t dirserv_estimate_microdesc_size(const smartlist_t *fps, int compressed); + +int routerstatus_format_entry(char *buf, size_t buf_len, + routerstatus_t *rs, const char *platform, + routerstatus_format_type_t format); +void dirserv_free_all(void); +void cached_dir_decref(cached_dir_t *d); +cached_dir_t *new_cached_dir(char *s, time_t published); + +#ifdef DIRSERV_PRIVATE +int measured_bw_line_parse(measured_bw_line_t *out, const char *line); + +int measured_bw_line_apply(measured_bw_line_t *parsed_line, + smartlist_t *routerstatuses); +#endif + +int dirserv_read_measured_bandwidths(const char *from_file, + smartlist_t *routerstatuses); + +#endif + diff --git a/src/or/dirvote.c b/src/or/dirvote.c index 656cb8a56e..9273dbc90d 100644 --- a/src/or/dirvote.c +++ b/src/or/dirvote.c @@ -5,27 +5,72 @@ #define DIRVOTE_PRIVATE #include "or.h" +#include "config.h" +#include "directory.h" +#include "dirserv.h" +#include "dirvote.h" +#include "microdesc.h" +#include "networkstatus.h" +#include "policies.h" +#include "rephist.h" +#include "router.h" +#include "routerlist.h" +#include "routerparse.h" /** * \file dirvote.c * \brief Functions to compute directory consensus, and schedule voting. **/ -static int dirvote_add_signatures_to_pending_consensus( +/** A consensus that we have built and are appending signatures to. Once it's + * time to publish it, it will become an active consensus if it accumulates + * enough signatures. */ +typedef struct pending_consensus_t { + /** The body of the consensus that we're currently building. Once we + * have it built, it goes into dirserv.c */ + char *body; + /** The parsed in-progress consensus document. */ + networkstatus_t *consensus; +} pending_consensus_t; + +static int dirvote_add_signatures_to_all_pending_consensuses( const char *detached_signatures_body, const char **msg_out); +static int dirvote_add_signatures_to_pending_consensus( + pending_consensus_t *pc, + ns_detached_signatures_t *sigs, + const char **msg_out); static char *list_v3_auth_ids(void); static void dirvote_fetch_missing_votes(void); static void dirvote_fetch_missing_signatures(void); static int dirvote_perform_vote(void); static void dirvote_clear_votes(int all_votes); -static int dirvote_compute_consensus(void); +static int dirvote_compute_consensuses(void); static int dirvote_publish_consensus(void); +static char *make_consensus_method_list(int low, int high, const char *sep); + +/** The highest consensus method that we currently support. */ +#define MAX_SUPPORTED_CONSENSUS_METHOD 11 + +/** Lowest consensus method that contains a 'directory-footer' marker */ +#define MIN_METHOD_FOR_FOOTER 9 + +/** Lowest consensus method that contains bandwidth weights */ +#define MIN_METHOD_FOR_BW_WEIGHTS 9 + +/** Lowest consensus method that contains consensus params */ +#define MIN_METHOD_FOR_PARAMS 7 + +/** Lowest consensus method that generates microdescriptors */ +#define MIN_METHOD_FOR_MICRODESC 8 /* ===== * Voting * =====*/ +/* Overestimated. */ +#define MICRODESC_LINE_LEN 80 + /** Return a new string containing the string representation of the vote in * <b>v3_ns</b>, signed with our v3 signing key <b>private_signing_key</b>. * For v3 authorities. */ @@ -44,6 +89,7 @@ format_networkstatus_vote(crypto_pk_env_t *private_signing_key, uint32_t addr; routerlist_t *rl = router_get_routerlist(); char *version_lines = NULL; + int r; networkstatus_voter_info_t *voter; tor_assert(private_signing_key); @@ -70,20 +116,30 @@ format_networkstatus_vote(crypto_pk_env_t *private_signing_key, version_lines = tor_malloc(v_len); cp = version_lines; if (client_versions) { - tor_snprintf(cp, v_len-(cp-version_lines), + r = tor_snprintf(cp, v_len-(cp-version_lines), "client-versions %s\n", client_versions); + if (r < 0) { + log_err(LD_BUG, "Insufficient memory for client-versions line"); + tor_assert(0); + } cp += strlen(cp); } - if (server_versions) - tor_snprintf(cp, v_len-(cp-version_lines), + if (server_versions) { + r = tor_snprintf(cp, v_len-(cp-version_lines), "server-versions %s\n", server_versions); + if (r < 0) { + log_err(LD_BUG, "Insufficient memory for server-versions line"); + tor_assert(0); + } + } } else { version_lines = tor_strdup(""); } len = 8192; len += strlen(version_lines); - len += (RS_ENTRY_LEN)*smartlist_len(rl->routers); + len += (RS_ENTRY_LEN+MICRODESC_LINE_LEN)*smartlist_len(rl->routers); + len += strlen("\ndirectory-footer\n"); len += v3_ns->cert->cache_info.signed_descriptor_len; status = tor_malloc(len); @@ -93,17 +149,25 @@ format_networkstatus_vote(crypto_pk_env_t *private_signing_key, char fu[ISO_TIME_LEN+1]; char vu[ISO_TIME_LEN+1]; char *flags = smartlist_join_strings(v3_ns->known_flags, " ", 0, NULL); + char *params; authority_cert_t *cert = v3_ns->cert; + char *methods = + make_consensus_method_list(1, MAX_SUPPORTED_CONSENSUS_METHOD, " "); format_iso_time(published, v3_ns->published); format_iso_time(va, v3_ns->valid_after); format_iso_time(fu, v3_ns->fresh_until); format_iso_time(vu, v3_ns->valid_until); + if (v3_ns->net_params) + params = smartlist_join_strings(v3_ns->net_params, " ", 0, NULL); + else + params = tor_strdup(""); + tor_assert(cert); - tor_snprintf(status, len, + r = tor_snprintf(status, len, "network-status-version 3\n" "vote-status %s\n" - "consensus-methods 1 2 3 4 5\n" + "consensus-methods %s\n" "published %s\n" "valid-after %s\n" "fresh-until %s\n" @@ -111,24 +175,38 @@ format_networkstatus_vote(crypto_pk_env_t *private_signing_key, "voting-delay %d %d\n" "%s" /* versions */ "known-flags %s\n" + "params %s\n" "dir-source %s %s %s %s %d %d\n" "contact %s\n", v3_ns->type == NS_TYPE_VOTE ? "vote" : "opinion", + methods, published, va, fu, vu, v3_ns->vote_seconds, v3_ns->dist_seconds, version_lines, flags, + params, voter->nickname, fingerprint, voter->address, - ipaddr, voter->dir_port, voter->or_port, voter->contact); + ipaddr, voter->dir_port, voter->or_port, voter->contact); + if (r < 0) { + log_err(LD_BUG, "Insufficient memory for network status line"); + tor_assert(0); + } + + tor_free(params); tor_free(flags); + tor_free(methods); outp = status + strlen(status); endp = status + len; if (!tor_digest_is_zero(voter->legacy_id_digest)) { char fpbuf[HEX_DIGEST_LEN+1]; base16_encode(fpbuf, sizeof(fpbuf), voter->legacy_id_digest, DIGEST_LEN); - tor_snprintf(outp, endp-outp, "legacy-dir-key %s\n", fpbuf); + r = tor_snprintf(outp, endp-outp, "legacy-dir-key %s\n", fpbuf); + if (r < 0) { + log_err(LD_BUG, "Insufficient memory for legacy-dir-key line"); + tor_assert(0); + } outp += strlen(outp); } @@ -139,15 +217,32 @@ format_networkstatus_vote(crypto_pk_env_t *private_signing_key, outp += cert->cache_info.signed_descriptor_len; } - SMARTLIST_FOREACH(v3_ns->routerstatus_list, vote_routerstatus_t *, vrs, - { + SMARTLIST_FOREACH_BEGIN(v3_ns->routerstatus_list, vote_routerstatus_t *, + vrs) { + vote_microdesc_hash_t *h; if (routerstatus_format_entry(outp, endp-outp, &vrs->status, - vrs->version, 0, 0) < 0) { + vrs->version, NS_V3_VOTE) < 0) { log_warn(LD_BUG, "Unable to print router status."); goto err; } outp += strlen(outp); - }); + + for (h = vrs->microdesc; h; h = h->next) { + size_t mlen = strlen(h->microdesc_hash_line); + if (outp+mlen >= endp) { + log_warn(LD_BUG, "Can't fit microdesc line in vote."); + } + memcpy(outp, h->microdesc_hash_line, mlen+1); + outp += strlen(outp); + } + } SMARTLIST_FOREACH_END(vrs); + + r = tor_snprintf(outp, endp-outp, "directory-footer\n"); + if (r < 0) { + log_err(LD_BUG, "Insufficient memory for directory-footer line"); + tor_assert(0); + } + outp += strlen(outp); { char signing_key_fingerprint[FINGERPRINT_LEN+1]; @@ -170,10 +265,10 @@ format_networkstatus_vote(crypto_pk_env_t *private_signing_key, outp += strlen(outp); } - if (router_get_networkstatus_v3_hash(status, digest)<0) + if (router_get_networkstatus_v3_hash(status, digest, DIGEST_SHA1)<0) goto err; note_crypto_pk_op(SIGN_DIR); - if (router_append_dirobj_signature(outp,endp-outp,digest, + if (router_append_dirobj_signature(outp,endp-outp,digest, DIGEST_LEN, private_signing_key)<0) { log_warn(LD_BUG, "Unable to sign networkstatus vote."); goto err; @@ -216,6 +311,20 @@ get_voter(const networkstatus_t *vote) return smartlist_get(vote->voters, 0); } +/** Return the signature made by <b>voter</b> using the algorithm + * <b>alg</b>, or NULL if none is found. */ +document_signature_t * +voter_get_sig_by_algorithm(const networkstatus_voter_info_t *voter, + digest_algorithm_t alg) +{ + if (!voter->sigs) + return NULL; + SMARTLIST_FOREACH(voter->sigs, document_signature_t *, sig, + if (sig->alg == alg) + return sig); + return NULL; +} + /** Temporary structure used in constructing a list of dir-source entries * for a consensus. One of these is generated for every vote, and one more * for every legacy key in each vote. */ @@ -275,34 +384,8 @@ get_frequent_members(smartlist_t *out, smartlist_t *in, int min) /** Given a sorted list of strings <b>lst</b>, return the member that appears * most. Break ties in favor of later-occurring members. */ -static const char * -get_most_frequent_member(smartlist_t *lst) -{ - const char *most_frequent = NULL; - int most_frequent_count = 0; - - const char *cur = NULL; - int count = 0; - - SMARTLIST_FOREACH(lst, const char *, s, - { - if (cur && !strcmp(s, cur)) { - ++count; - } else { - if (count >= most_frequent_count) { - most_frequent = cur; - most_frequent_count = count; - } - cur = s; - count = 1; - } - }); - if (count >= most_frequent_count) { - most_frequent = cur; - most_frequent_count = count; - } - return most_frequent; -} +#define get_most_frequent_member(lst) \ + smartlist_get_most_frequent_string(lst) /** Return 0 if and only if <b>a</b> and <b>b</b> are routerstatuses * that come from the same routerinfo, with the same derived elements. @@ -344,7 +427,8 @@ _compare_vote_rs(const void **_a, const void **_b) * in favor of smaller descriptor digest. */ static vote_routerstatus_t * -compute_routerstatus_consensus(smartlist_t *votes) +compute_routerstatus_consensus(smartlist_t *votes, int consensus_method, + char *microdesc_digest256_out) { vote_routerstatus_t *most = NULL, *cur = NULL; int most_n = 0, cur_n = 0; @@ -380,18 +464,45 @@ compute_routerstatus_consensus(smartlist_t *votes) } tor_assert(most); + + if (consensus_method >= MIN_METHOD_FOR_MICRODESC && + microdesc_digest256_out) { + smartlist_t *digests = smartlist_create(); + const char *best_microdesc_digest; + SMARTLIST_FOREACH_BEGIN(votes, vote_routerstatus_t *, rs) { + char d[DIGEST256_LEN]; + if (compare_vote_rs(rs, most)) + continue; + if (!vote_routerstatus_find_microdesc_hash(d, rs, consensus_method, + DIGEST_SHA256)) + smartlist_add(digests, tor_memdup(d, sizeof(d))); + } SMARTLIST_FOREACH_END(rs); + smartlist_sort_digests256(digests); + best_microdesc_digest = smartlist_get_most_frequent_digest256(digests); + if (best_microdesc_digest) + memcpy(microdesc_digest256_out, best_microdesc_digest, DIGEST256_LEN); + SMARTLIST_FOREACH(digests, char *, cp, tor_free(cp)); + smartlist_free(digests); + } + return most; } -/** Given a list of strings in <b>lst</b>, set the DIGEST_LEN-byte digest at - * <b>digest_out</b> to the hash of the concatenation of those strings. */ +/** Given a list of strings in <b>lst</b>, set the <b>len_out</b>-byte digest + * at <b>digest_out</b> to the hash of the concatenation of those strings, + * computed with the algorithm <b>alg</b>. */ static void -hash_list_members(char *digest_out, smartlist_t *lst) +hash_list_members(char *digest_out, size_t len_out, + smartlist_t *lst, digest_algorithm_t alg) { - crypto_digest_env_t *d = crypto_new_digest_env(); + crypto_digest_env_t *d; + if (alg == DIGEST_SHA1) + d = crypto_new_digest_env(); + else + d = crypto_new_digest256_env(alg); SMARTLIST_FOREACH(lst, const char *, cp, crypto_digest_add_bytes(d, cp, strlen(cp))); - crypto_digest_get_digest(d, digest_out, DIGEST_LEN); + crypto_digest_get_digest(d, digest_out, len_out); crypto_free_digest_env(d); } @@ -455,7 +566,31 @@ compute_consensus_method(smartlist_t *votes) static int consensus_method_is_supported(int method) { - return (method >= 1) && (method <= 5); + return (method >= 1) && (method <= MAX_SUPPORTED_CONSENSUS_METHOD); +} + +/** Return a newly allocated string holding the numbers between low and high + * (inclusive) that are supported consensus methods. */ +static char * +make_consensus_method_list(int low, int high, const char *separator) +{ + char *list; + + char b[32]; + int i; + smartlist_t *lst; + lst = smartlist_create(); + for (i = low; i <= high; ++i) { + if (!consensus_method_is_supported(i)) + continue; + tor_snprintf(b, sizeof(b), "%d", i); + smartlist_add(lst, tor_strdup(b)); + } + list = smartlist_join_strings(lst, separator, 0, NULL); + tor_assert(list); + SMARTLIST_FOREACH(lst, char *, cp, tor_free(cp)); + smartlist_free(lst); + return list; } /** Helper: given <b>lst</b>, a list of version strings such that every @@ -475,6 +610,728 @@ compute_consensus_versions_list(smartlist_t *lst, int n_versioning) return result; } +/** Helper: given a list of valid networkstatus_t, return a new string + * containing the contents of the consensus network parameter set. + */ +/* private */ char * +dirvote_compute_params(smartlist_t *votes) +{ + int i; + int32_t *vals; + + int cur_param_len; + const char *cur_param; + const char *eq; + char *result; + + const int n_votes = smartlist_len(votes); + smartlist_t *output; + smartlist_t *param_list = smartlist_create(); + + /* We require that the parameter lists in the votes are well-formed: that + is, that their keywords are unique and sorted, and that their values are + between INT32_MIN and INT32_MAX inclusive. This should be guaranteed by + the parsing code. */ + + vals = tor_malloc(sizeof(int)*n_votes); + + SMARTLIST_FOREACH_BEGIN(votes, networkstatus_t *, v) { + if (!v->net_params) + continue; + smartlist_add_all(param_list, v->net_params); + } SMARTLIST_FOREACH_END(v); + + if (smartlist_len(param_list) == 0) { + tor_free(vals); + smartlist_free(param_list); + return NULL; + } + + smartlist_sort_strings(param_list); + i = 0; + cur_param = smartlist_get(param_list, 0); + eq = strchr(cur_param, '='); + tor_assert(eq); + cur_param_len = (int)(eq+1 - cur_param); + + output = smartlist_create(); + + SMARTLIST_FOREACH_BEGIN(param_list, const char *, param) { + const char *next_param; + int ok=0; + eq = strchr(param, '='); + tor_assert(i<n_votes); + vals[i++] = (int32_t) + tor_parse_long(eq+1, 10, INT32_MIN, INT32_MAX, &ok, NULL); + tor_assert(ok); + + if (param_sl_idx+1 == smartlist_len(param_list)) + next_param = NULL; + else + next_param = smartlist_get(param_list, param_sl_idx+1); + if (!next_param || strncmp(next_param, param, cur_param_len)) { + /* We've reached the end of a series. */ + int32_t median = median_int32(vals, i); + char *out_string = tor_malloc(64+cur_param_len); + memcpy(out_string, param, cur_param_len); + tor_snprintf(out_string+cur_param_len,64, "%ld", (long)median); + smartlist_add(output, out_string); + + i = 0; + if (next_param) { + eq = strchr(next_param, '='); + cur_param_len = (int)(eq+1 - next_param); + } + } + } SMARTLIST_FOREACH_END(param); + + result = smartlist_join_strings(output, " ", 0, NULL); + SMARTLIST_FOREACH(output, char *, cp, tor_free(cp)); + smartlist_free(output); + smartlist_free(param_list); + tor_free(vals); + return result; +} + +#define RANGE_CHECK(a,b,c,d,e,f,g,mx) \ + ((a) >= 0 && (a) <= (mx) && (b) >= 0 && (b) <= (mx) && \ + (c) >= 0 && (c) <= (mx) && (d) >= 0 && (d) <= (mx) && \ + (e) >= 0 && (e) <= (mx) && (f) >= 0 && (f) <= (mx) && \ + (g) >= 0 && (g) <= (mx)) + +#define CHECK_EQ(a, b, margin) \ + ((a)-(b) >= 0 ? (a)-(b) <= (margin) : (b)-(a) <= (margin)) + +typedef enum { + BW_WEIGHTS_NO_ERROR = 0, + BW_WEIGHTS_RANGE_ERROR = 1, + BW_WEIGHTS_SUMG_ERROR = 2, + BW_WEIGHTS_SUME_ERROR = 3, + BW_WEIGHTS_SUMD_ERROR = 4, + BW_WEIGHTS_BALANCE_MID_ERROR = 5, + BW_WEIGHTS_BALANCE_EG_ERROR = 6 +} bw_weights_error_t; + +/** + * Verify that any weightings satisfy the balanced formulas. + */ +static bw_weights_error_t +networkstatus_check_weights(int64_t Wgg, int64_t Wgd, int64_t Wmg, + int64_t Wme, int64_t Wmd, int64_t Wee, + int64_t Wed, int64_t scale, int64_t G, + int64_t M, int64_t E, int64_t D, int64_t T, + int64_t margin, int do_balance) { + bw_weights_error_t berr = BW_WEIGHTS_NO_ERROR; + + // Wed + Wmd + Wgd == 1 + if (!CHECK_EQ(Wed + Wmd + Wgd, scale, margin)) { + berr = BW_WEIGHTS_SUMD_ERROR; + goto out; + } + + // Wmg + Wgg == 1 + if (!CHECK_EQ(Wmg + Wgg, scale, margin)) { + berr = BW_WEIGHTS_SUMG_ERROR; + goto out; + } + + // Wme + Wee == 1 + if (!CHECK_EQ(Wme + Wee, scale, margin)) { + berr = BW_WEIGHTS_SUME_ERROR; + goto out; + } + + // Verify weights within range 0->1 + if (!RANGE_CHECK(Wgg, Wgd, Wmg, Wme, Wmd, Wed, Wee, scale)) { + berr = BW_WEIGHTS_RANGE_ERROR; + goto out; + } + + if (do_balance) { + // Wgg*G + Wgd*D == Wee*E + Wed*D, already scaled + if (!CHECK_EQ(Wgg*G + Wgd*D, Wee*E + Wed*D, (margin*T)/3)) { + berr = BW_WEIGHTS_BALANCE_EG_ERROR; + goto out; + } + + // Wgg*G + Wgd*D == M*scale + Wmd*D + Wme*E + Wmg*G, already scaled + if (!CHECK_EQ(Wgg*G + Wgd*D, M*scale + Wmd*D + Wme*E + Wmg*G, + (margin*T)/3)) { + berr = BW_WEIGHTS_BALANCE_MID_ERROR; + goto out; + } + } + + out: + if (berr) { + log_info(LD_DIR, + "Bw weight mismatch %d. G="I64_FORMAT" M="I64_FORMAT + " E="I64_FORMAT" D="I64_FORMAT" T="I64_FORMAT + " Wmd=%d Wme=%d Wmg=%d Wed=%d Wee=%d" + " Wgd=%d Wgg=%d Wme=%d Wmg=%d", + berr, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + (int)Wmd, (int)Wme, (int)Wmg, (int)Wed, (int)Wee, + (int)Wgd, (int)Wgg, (int)Wme, (int)Wmg); + } + + return berr; +} + +/** + * This function computes the bandwidth weights for consensus method 10. + * + * It returns true if weights could be computed, false otherwise. + */ +static int +networkstatus_compute_bw_weights_v10(smartlist_t *chunks, int64_t G, + int64_t M, int64_t E, int64_t D, + int64_t T, int64_t weight_scale) +{ + bw_weights_error_t berr = 0; + int64_t Wgg = -1, Wgd = -1; + int64_t Wmg = -1, Wme = -1, Wmd = -1; + int64_t Wed = -1, Wee = -1; + const char *casename; + char buf[512]; + int r; + + if (G <= 0 || M <= 0 || E <= 0 || D <= 0) { + log_warn(LD_DIR, "Consensus with empty bandwidth: " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT + " D="I64_FORMAT" T="I64_FORMAT, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); + return 0; + } + + /* + * Computed from cases in 3.4.3 of dir-spec.txt + * + * 1. Neither are scarce + * 2. Both Guard and Exit are scarce + * a. R+D <= S + * b. R+D > S + * 3. One of Guard or Exit is scarce + * a. S+D < T/3 + * b. S+D >= T/3 + */ + if (3*E >= T && 3*G >= T) { // E >= T/3 && G >= T/3 + /* Case 1: Neither are scarce. */ + casename = "Case 1 (Wgd=Wmd=Wed)"; + Wgd = weight_scale/3; + Wed = weight_scale/3; + Wmd = weight_scale/3; + Wee = (weight_scale*(E+G+M))/(3*E); + Wme = weight_scale - Wee; + Wmg = (weight_scale*(2*G-E-M))/(3*G); + Wgg = weight_scale - Wmg; + + berr = networkstatus_check_weights(Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed, + weight_scale, G, M, E, D, T, 10, 1); + + if (berr) { + log_warn(LD_DIR, + "Bw Weights error %d for %s v10. G="I64_FORMAT" M="I64_FORMAT + " E="I64_FORMAT" D="I64_FORMAT" T="I64_FORMAT + " Wmd=%d Wme=%d Wmg=%d Wed=%d Wee=%d" + " Wgd=%d Wgg=%d Wme=%d Wmg=%d weight_scale=%d", + berr, casename, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + (int)Wmd, (int)Wme, (int)Wmg, (int)Wed, (int)Wee, + (int)Wgd, (int)Wgg, (int)Wme, (int)Wmg, (int)weight_scale); + return 0; + } + } else if (3*E < T && 3*G < T) { // E < T/3 && G < T/3 + int64_t R = MIN(E, G); + int64_t S = MAX(E, G); + /* + * Case 2: Both Guards and Exits are scarce + * Balance D between E and G, depending upon + * D capacity and scarcity. + */ + if (R+D < S) { // Subcase a + Wgg = weight_scale; + Wee = weight_scale; + Wmg = 0; + Wme = 0; + Wmd = 0; + if (E < G) { + casename = "Case 2a (E scarce)"; + Wed = weight_scale; + Wgd = 0; + } else { /* E >= G */ + casename = "Case 2a (G scarce)"; + Wed = 0; + Wgd = weight_scale; + } + } else { // Subcase b: R+D >= S + casename = "Case 2b1 (Wgg=1, Wmd=Wgd)"; + Wee = (weight_scale*(E - G + M))/E; + Wed = (weight_scale*(D - 2*E + 4*G - 2*M))/(3*D); + Wme = (weight_scale*(G-M))/E; + Wmg = 0; + Wgg = weight_scale; + Wmd = (weight_scale - Wed)/2; + Wgd = (weight_scale - Wed)/2; + + berr = networkstatus_check_weights(Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed, + weight_scale, G, M, E, D, T, 10, 1); + + if (berr) { + casename = "Case 2b2 (Wgg=1, Wee=1)"; + Wgg = weight_scale; + Wee = weight_scale; + Wed = (weight_scale*(D - 2*E + G + M))/(3*D); + Wmd = (weight_scale*(D - 2*M + G + E))/(3*D); + Wme = 0; + Wmg = 0; + + if (Wmd < 0) { // Can happen if M > T/3 + casename = "Case 2b3 (Wmd=0)"; + Wmd = 0; + log_warn(LD_DIR, + "Too much Middle bandwidth on the network to calculate " + "balanced bandwidth-weights. Consider increasing the " + "number of Guard nodes by lowering the requirements."); + } + Wgd = weight_scale - Wed - Wmd; + berr = networkstatus_check_weights(Wgg, Wgd, Wmg, Wme, Wmd, Wee, + Wed, weight_scale, G, M, E, D, T, 10, 1); + } + if (berr != BW_WEIGHTS_NO_ERROR && + berr != BW_WEIGHTS_BALANCE_MID_ERROR) { + log_warn(LD_DIR, + "Bw Weights error %d for %s v10. G="I64_FORMAT" M="I64_FORMAT + " E="I64_FORMAT" D="I64_FORMAT" T="I64_FORMAT + " Wmd=%d Wme=%d Wmg=%d Wed=%d Wee=%d" + " Wgd=%d Wgg=%d Wme=%d Wmg=%d weight_scale=%d", + berr, casename, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + (int)Wmd, (int)Wme, (int)Wmg, (int)Wed, (int)Wee, + (int)Wgd, (int)Wgg, (int)Wme, (int)Wmg, (int)weight_scale); + return 0; + } + } + } else { // if (E < T/3 || G < T/3) { + int64_t S = MIN(E, G); + // Case 3: Exactly one of Guard or Exit is scarce + if (!(3*E < T || 3*G < T) || !(3*G >= T || 3*E >= T)) { + log_warn(LD_BUG, + "Bw-Weights Case 3 v10 but with G="I64_FORMAT" M=" + I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT" T="I64_FORMAT, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); + } + + if (3*(S+D) < T) { // Subcase a: S+D < T/3 + if (G < E) { + casename = "Case 3a (G scarce)"; + Wgg = Wgd = weight_scale; + Wmd = Wed = Wmg = 0; + // Minor subcase, if E is more scarce than M, + // keep its bandwidth in place. + if (E < M) Wme = 0; + else Wme = (weight_scale*(E-M))/(2*E); + Wee = weight_scale-Wme; + } else { // G >= E + casename = "Case 3a (E scarce)"; + Wee = Wed = weight_scale; + Wmd = Wgd = Wme = 0; + // Minor subcase, if G is more scarce than M, + // keep its bandwidth in place. + if (G < M) Wmg = 0; + else Wmg = (weight_scale*(G-M))/(2*G); + Wgg = weight_scale-Wmg; + } + } else { // Subcase b: S+D >= T/3 + // D != 0 because S+D >= T/3 + if (G < E) { + casename = "Case 3bg (G scarce, Wgg=1, Wmd == Wed)"; + Wgg = weight_scale; + Wgd = (weight_scale*(D - 2*G + E + M))/(3*D); + Wmg = 0; + Wee = (weight_scale*(E+M))/(2*E); + Wme = weight_scale - Wee; + Wmd = (weight_scale - Wgd)/2; + Wed = (weight_scale - Wgd)/2; + + berr = networkstatus_check_weights(Wgg, Wgd, Wmg, Wme, Wmd, Wee, + Wed, weight_scale, G, M, E, D, T, 10, 1); + } else { // G >= E + casename = "Case 3be (E scarce, Wee=1, Wmd == Wgd)"; + Wee = weight_scale; + Wed = (weight_scale*(D - 2*E + G + M))/(3*D); + Wme = 0; + Wgg = (weight_scale*(G+M))/(2*G); + Wmg = weight_scale - Wgg; + Wmd = (weight_scale - Wed)/2; + Wgd = (weight_scale - Wed)/2; + + berr = networkstatus_check_weights(Wgg, Wgd, Wmg, Wme, Wmd, Wee, + Wed, weight_scale, G, M, E, D, T, 10, 1); + } + if (berr) { + log_warn(LD_DIR, + "Bw Weights error %d for %s v10. G="I64_FORMAT" M="I64_FORMAT + " E="I64_FORMAT" D="I64_FORMAT" T="I64_FORMAT + " Wmd=%d Wme=%d Wmg=%d Wed=%d Wee=%d" + " Wgd=%d Wgg=%d Wme=%d Wmg=%d weight_scale=%d", + berr, casename, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + (int)Wmd, (int)Wme, (int)Wmg, (int)Wed, (int)Wee, + (int)Wgd, (int)Wgg, (int)Wme, (int)Wmg, (int)weight_scale); + return 0; + } + } + } + + /* We cast down the weights to 32 bit ints on the assumption that + * weight_scale is ~= 10000. We need to ensure a rogue authority + * doesn't break this assumption to rig our weights */ + tor_assert(0 < weight_scale && weight_scale < INT32_MAX); + + /* + * Provide Wgm=Wgg, Wmm=1, Wem=Wee, Weg=Wed. May later determine + * that middle nodes need different bandwidth weights for dirport traffic, + * or that weird exit policies need special weight, or that bridges + * need special weight. + * + * NOTE: This list is sorted. + */ + r = tor_snprintf(buf, sizeof(buf), + "bandwidth-weights Wbd=%d Wbe=%d Wbg=%d Wbm=%d " + "Wdb=%d " + "Web=%d Wed=%d Wee=%d Weg=%d Wem=%d " + "Wgb=%d Wgd=%d Wgg=%d Wgm=%d " + "Wmb=%d Wmd=%d Wme=%d Wmg=%d Wmm=%d\n", + (int)Wmd, (int)Wme, (int)Wmg, (int)weight_scale, + (int)weight_scale, + (int)weight_scale, (int)Wed, (int)Wee, (int)Wed, (int)Wee, + (int)weight_scale, (int)Wgd, (int)Wgg, (int)Wgg, + (int)weight_scale, (int)Wmd, (int)Wme, (int)Wmg, (int)weight_scale); + if (r<0) { + log_warn(LD_BUG, + "Not enough space in buffer for bandwidth-weights line."); + *buf = '\0'; + return 0; + } + smartlist_add(chunks, tor_strdup(buf)); + + log_notice(LD_CIRC, "Computed bandwidth weights for %s with v10: " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT, + casename, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); + return 1; +} +/** + * This function computes the bandwidth weights for consensus method 9. + * + * It has been obsoleted in favor of consensus method 10. + */ +static void +networkstatus_compute_bw_weights_v9(smartlist_t *chunks, int64_t G, int64_t M, + int64_t E, int64_t D, int64_t T, + int64_t weight_scale) +{ + int64_t Wgg = -1, Wgd = -1; + int64_t Wmg = -1, Wme = -1, Wmd = -1; + int64_t Wed = -1, Wee = -1; + const char *casename; + char buf[512]; + int r; + + if (G <= 0 || M <= 0 || E <= 0 || D <= 0) { + log_warn(LD_DIR, "Consensus with empty bandwidth: " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT + " D="I64_FORMAT" T="I64_FORMAT, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); + return; + } + + /* + * Computed from cases in 3.4.3 of dir-spec.txt + * + * 1. Neither are scarce + * 2. Both Guard and Exit are scarce + * a. R+D <= S + * b. R+D > S + * 3. One of Guard or Exit is scarce + * a. S+D < T/3 + * b. S+D >= T/3 + */ + if (3*E >= T && 3*G >= T) { // E >= T/3 && G >= T/3 + bw_weights_error_t berr = 0; + /* Case 1: Neither are scarce. + * + * Attempt to ensure that we have a large amount of exit bandwidth + * in the middle position. + */ + casename = "Case 1 (Wme*E = Wmd*D)"; + Wgg = (weight_scale*(D+E+G+M))/(3*G); + if (D==0) Wmd = 0; + else Wmd = (weight_scale*(2*D + 2*E - G - M))/(6*D); + Wme = (weight_scale*(2*D + 2*E - G - M))/(6*E); + Wee = (weight_scale*(-2*D + 4*E + G + M))/(6*E); + Wgd = 0; + Wmg = weight_scale - Wgg; + Wed = weight_scale - Wmd; + + berr = networkstatus_check_weights(Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed, + weight_scale, G, M, E, D, T, 10, 1); + + if (berr) { + log_warn(LD_DIR, "Bw Weights error %d for case %s. " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT + " D="I64_FORMAT" T="I64_FORMAT, + berr, casename, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); + } + } else if (3*E < T && 3*G < T) { // E < T/3 && G < T/3 + int64_t R = MIN(E, G); + int64_t S = MAX(E, G); + /* + * Case 2: Both Guards and Exits are scarce + * Balance D between E and G, depending upon + * D capacity and scarcity. + */ + if (R+D < S) { // Subcase a + Wgg = weight_scale; + Wee = weight_scale; + Wmg = 0; + Wme = 0; + Wmd = 0; + if (E < G) { + casename = "Case 2a (E scarce)"; + Wed = weight_scale; + Wgd = 0; + } else { /* E >= G */ + casename = "Case 2a (G scarce)"; + Wed = 0; + Wgd = weight_scale; + } + } else { // Subcase b: R+D > S + bw_weights_error_t berr = 0; + casename = "Case 2b (Wme*E == Wmd*D)"; + if (D != 0) { + Wgg = weight_scale; + Wgd = (weight_scale*(D + E - 2*G + M))/(3*D); // T/3 >= G (Ok) + Wmd = (weight_scale*(D + E + G - 2*M))/(6*D); // T/3 >= M + Wme = (weight_scale*(D + E + G - 2*M))/(6*E); + Wee = (weight_scale*(-D + 5*E - G + 2*M))/(6*E); // 2E+M >= T/3 + Wmg = 0; + Wed = weight_scale - Wgd - Wmd; + + berr = networkstatus_check_weights(Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed, + weight_scale, G, M, E, D, T, 10, 1); + } + + if (D == 0 || berr) { // Can happen if M > T/3 + casename = "Case 2b (E=G)"; + Wgg = weight_scale; + Wee = weight_scale; + Wmg = 0; + Wme = 0; + Wmd = 0; + if (D == 0) Wgd = 0; + else Wgd = (weight_scale*(D+E-G))/(2*D); + Wed = weight_scale - Wgd; + berr = networkstatus_check_weights(Wgg, Wgd, Wmg, Wme, Wmd, Wee, + Wed, weight_scale, G, M, E, D, T, 10, 1); + } + if (berr != BW_WEIGHTS_NO_ERROR && + berr != BW_WEIGHTS_BALANCE_MID_ERROR) { + log_warn(LD_DIR, "Bw Weights error %d for case %s. " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT + " D="I64_FORMAT" T="I64_FORMAT, + berr, casename, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); + } + } + } else { // if (E < T/3 || G < T/3) { + int64_t S = MIN(E, G); + // Case 3: Exactly one of Guard or Exit is scarce + if (!(3*E < T || 3*G < T) || !(3*G >= T || 3*E >= T)) { + log_warn(LD_BUG, + "Bw-Weights Case 3 but with G="I64_FORMAT" M=" + I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT" T="I64_FORMAT, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); + } + + if (3*(S+D) < T) { // Subcase a: S+D < T/3 + if (G < E) { + casename = "Case 3a (G scarce)"; + Wgg = Wgd = weight_scale; + Wmd = Wed = Wmg = 0; + // Minor subcase, if E is more scarce than M, + // keep its bandwidth in place. + if (E < M) Wme = 0; + else Wme = (weight_scale*(E-M))/(2*E); + Wee = weight_scale-Wme; + } else { // G >= E + casename = "Case 3a (E scarce)"; + Wee = Wed = weight_scale; + Wmd = Wgd = Wme = 0; + // Minor subcase, if G is more scarce than M, + // keep its bandwidth in place. + if (G < M) Wmg = 0; + else Wmg = (weight_scale*(G-M))/(2*G); + Wgg = weight_scale-Wmg; + } + } else { // Subcase b: S+D >= T/3 + bw_weights_error_t berr = 0; + // D != 0 because S+D >= T/3 + if (G < E) { + casename = "Case 3b (G scarce, Wme*E == Wmd*D)"; + Wgd = (weight_scale*(D + E - 2*G + M))/(3*D); + Wmd = (weight_scale*(D + E + G - 2*M))/(6*D); + Wme = (weight_scale*(D + E + G - 2*M))/(6*E); + Wee = (weight_scale*(-D + 5*E - G + 2*M))/(6*E); + Wgg = weight_scale; + Wmg = 0; + Wed = weight_scale - Wgd - Wmd; + + berr = networkstatus_check_weights(Wgg, Wgd, Wmg, Wme, Wmd, Wee, + Wed, weight_scale, G, M, E, D, T, 10, 1); + } else { // G >= E + casename = "Case 3b (E scarce, Wme*E == Wmd*D)"; + Wgg = (weight_scale*(D + E + G + M))/(3*G); + Wmd = (weight_scale*(2*D + 2*E - G - M))/(6*D); + Wme = (weight_scale*(2*D + 2*E - G - M))/(6*E); + Wee = (weight_scale*(-2*D + 4*E + G + M))/(6*E); + Wgd = 0; + Wmg = weight_scale - Wgg; + Wed = weight_scale - Wmd; + + berr = networkstatus_check_weights(Wgg, Wgd, Wmg, Wme, Wmd, Wee, + Wed, weight_scale, G, M, E, D, T, 10, 1); + } + if (berr) { + log_warn(LD_DIR, "Bw Weights error %d for case %s. " + "G="I64_FORMAT" M="I64_FORMAT + " E="I64_FORMAT" D="I64_FORMAT" T="I64_FORMAT, + berr, casename, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); + } + } + } + + /* We cast down the weights to 32 bit ints on the assumption that + * weight_scale is ~= 10000. We need to ensure a rogue authority + * doesn't break this assumption to rig our weights */ + tor_assert(0 < weight_scale && weight_scale < INT32_MAX); + + if (Wgg < 0 || Wgg > weight_scale) { + log_warn(LD_DIR, "Bw %s: Wgg="I64_FORMAT"! G="I64_FORMAT + " M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT, + casename, I64_PRINTF_ARG(Wgg), + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); + + Wgg = MAX(MIN(Wgg, weight_scale), 0); + } + if (Wgd < 0 || Wgd > weight_scale) { + log_warn(LD_DIR, "Bw %s: Wgd="I64_FORMAT"! G="I64_FORMAT + " M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT, + casename, I64_PRINTF_ARG(Wgd), + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); + Wgd = MAX(MIN(Wgd, weight_scale), 0); + } + if (Wmg < 0 || Wmg > weight_scale) { + log_warn(LD_DIR, "Bw %s: Wmg="I64_FORMAT"! G="I64_FORMAT + " M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT, + casename, I64_PRINTF_ARG(Wmg), + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); + Wmg = MAX(MIN(Wmg, weight_scale), 0); + } + if (Wme < 0 || Wme > weight_scale) { + log_warn(LD_DIR, "Bw %s: Wme="I64_FORMAT"! G="I64_FORMAT + " M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT, + casename, I64_PRINTF_ARG(Wme), + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); + Wme = MAX(MIN(Wme, weight_scale), 0); + } + if (Wmd < 0 || Wmd > weight_scale) { + log_warn(LD_DIR, "Bw %s: Wmd="I64_FORMAT"! G="I64_FORMAT + " M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT, + casename, I64_PRINTF_ARG(Wmd), + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); + Wmd = MAX(MIN(Wmd, weight_scale), 0); + } + if (Wee < 0 || Wee > weight_scale) { + log_warn(LD_DIR, "Bw %s: Wee="I64_FORMAT"! G="I64_FORMAT + " M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT, + casename, I64_PRINTF_ARG(Wee), + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); + Wee = MAX(MIN(Wee, weight_scale), 0); + } + if (Wed < 0 || Wed > weight_scale) { + log_warn(LD_DIR, "Bw %s: Wed="I64_FORMAT"! G="I64_FORMAT + " M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT, + casename, I64_PRINTF_ARG(Wed), + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); + Wed = MAX(MIN(Wed, weight_scale), 0); + } + + // Add consensus weight keywords + smartlist_add(chunks, tor_strdup("bandwidth-weights ")); + /* + * Provide Wgm=Wgg, Wmm=1, Wem=Wee, Weg=Wed. May later determine + * that middle nodes need different bandwidth weights for dirport traffic, + * or that weird exit policies need special weight, or that bridges + * need special weight. + * + * NOTE: This list is sorted. + */ + r = tor_snprintf(buf, sizeof(buf), + "Wbd=%d Wbe=%d Wbg=%d Wbm=%d " + "Wdb=%d " + "Web=%d Wed=%d Wee=%d Weg=%d Wem=%d " + "Wgb=%d Wgd=%d Wgg=%d Wgm=%d " + "Wmb=%d Wmd=%d Wme=%d Wmg=%d Wmm=%d\n", + (int)Wmd, (int)Wme, (int)Wmg, (int)weight_scale, + (int)weight_scale, + (int)weight_scale, (int)Wed, (int)Wee, (int)Wed, (int)Wee, + (int)weight_scale, (int)Wgd, (int)Wgg, (int)Wgg, + (int)weight_scale, (int)Wmd, (int)Wme, (int)Wmg, (int)weight_scale); + if (r<0) { + log_warn(LD_BUG, + "Not enough space in buffer for bandwidth-weights line."); + *buf = '\0'; + } + smartlist_add(chunks, tor_strdup(buf)); + log_notice(LD_CIRC, "Computed bandwidth weights for %s with v9: " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT, + casename, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); +} + /** Given a list of vote networkstatus_t in <b>votes</b>, our public * authority <b>identity_key</b>, our private authority <b>signing_key</b>, * and the number of <b>total_authorities</b> that we believe exist in our @@ -489,18 +1346,27 @@ networkstatus_compute_consensus(smartlist_t *votes, crypto_pk_env_t *identity_key, crypto_pk_env_t *signing_key, const char *legacy_id_key_digest, - crypto_pk_env_t *legacy_signing_key) + crypto_pk_env_t *legacy_signing_key, + consensus_flavor_t flavor) { smartlist_t *chunks; char *result = NULL; int consensus_method; - time_t valid_after, fresh_until, valid_until; int vote_seconds, dist_seconds; char *client_versions = NULL, *server_versions = NULL; smartlist_t *flags; + const char *flavor_name; + int64_t G=0, M=0, E=0, D=0, T=0; /* For bandwidth weights */ + const routerstatus_format_type_t rs_format = + flavor == FLAV_NS ? NS_V3_CONSENSUS : NS_V3_CONSENSUS_MICRODESC; + char *params = NULL; + int added_weights = 0; + tor_assert(flavor == FLAV_NS || flavor == FLAV_MICRODESC); tor_assert(total_authorities >= smartlist_len(votes)); + flavor_name = networkstatus_get_flavor_name(flavor); + if (!smartlist_len(votes)) { log_warn(LD_DIR, "Can't compute a consensus from no votes."); return NULL; @@ -593,7 +1459,7 @@ networkstatus_compute_consensus(smartlist_t *votes, chunks = smartlist_create(); { - char buf[1024]; + char *buf=NULL; char va_buf[ISO_TIME_LEN+1], fu_buf[ISO_TIME_LEN+1], vu_buf[ISO_TIME_LEN+1]; char *flaglist; @@ -602,16 +1468,20 @@ networkstatus_compute_consensus(smartlist_t *votes, format_iso_time(vu_buf, valid_until); flaglist = smartlist_join_strings(flags, " ", 0, NULL); - smartlist_add(chunks, tor_strdup("network-status-version 3\n" - "vote-status consensus\n")); + tor_asprintf(&buf, "network-status-version 3%s%s\n" + "vote-status consensus\n", + flavor == FLAV_NS ? "" : " ", + flavor == FLAV_NS ? "" : flavor_name); + + smartlist_add(chunks, buf); if (consensus_method >= 2) { - tor_snprintf(buf, sizeof(buf), "consensus-method %d\n", + tor_asprintf(&buf, "consensus-method %d\n", consensus_method); - smartlist_add(chunks, tor_strdup(buf)); + smartlist_add(chunks, buf); } - tor_snprintf(buf, sizeof(buf), + tor_asprintf(&buf, "valid-after %s\n" "fresh-until %s\n" "valid-until %s\n" @@ -622,18 +1492,26 @@ networkstatus_compute_consensus(smartlist_t *votes, va_buf, fu_buf, vu_buf, vote_seconds, dist_seconds, client_versions, server_versions, flaglist); - smartlist_add(chunks, tor_strdup(buf)); + smartlist_add(chunks, buf); tor_free(flaglist); } + if (consensus_method >= MIN_METHOD_FOR_PARAMS) { + params = dirvote_compute_params(votes); + if (params) { + smartlist_add(chunks, tor_strdup("params ")); + smartlist_add(chunks, params); + smartlist_add(chunks, tor_strdup("\n")); + } + } + /* Sort the votes. */ smartlist_sort(votes, _compare_votes_by_authority_id); /* Add the authority sections. */ { smartlist_t *dir_sources = smartlist_create(); - SMARTLIST_FOREACH(votes, networkstatus_t *, v, - { + SMARTLIST_FOREACH_BEGIN(votes, networkstatus_t *, v) { dir_src_ent_t *e = tor_malloc_zero(sizeof(dir_src_ent_t)); e->v = v; e->digest = get_voter(v)->identity_digest; @@ -647,18 +1525,17 @@ networkstatus_compute_consensus(smartlist_t *votes, e_legacy->is_legacy = 1; smartlist_add(dir_sources, e_legacy); } - }); + } SMARTLIST_FOREACH_END(v); smartlist_sort(dir_sources, _compare_dir_src_ents_by_authority_id); - SMARTLIST_FOREACH(dir_sources, const dir_src_ent_t *, e, - { - char buf[1024]; + SMARTLIST_FOREACH_BEGIN(dir_sources, const dir_src_ent_t *, e) { struct in_addr in; char ip[INET_NTOA_BUF_LEN]; char fingerprint[HEX_DIGEST_LEN+1]; char votedigest[HEX_DIGEST_LEN+1]; networkstatus_t *v = e->v; networkstatus_voter_info_t *voter = get_voter(v); + char *buf = NULL; if (e->is_legacy) tor_assert(consensus_method >= 2); @@ -669,22 +1546,22 @@ networkstatus_compute_consensus(smartlist_t *votes, base16_encode(votedigest, sizeof(votedigest), voter->vote_digest, DIGEST_LEN); - tor_snprintf(buf, sizeof(buf), + tor_asprintf(&buf, "dir-source %s%s %s %s %s %d %d\n", voter->nickname, e->is_legacy ? "-legacy" : "", fingerprint, voter->address, ip, voter->dir_port, voter->or_port); - smartlist_add(chunks, tor_strdup(buf)); + smartlist_add(chunks, buf); if (! e->is_legacy) { - tor_snprintf(buf, sizeof(buf), + tor_asprintf(&buf, "contact %s\n" "vote-digest %s\n", voter->contact, votedigest); - smartlist_add(chunks, tor_strdup(buf)); + smartlist_add(chunks, buf); } - }); + } SMARTLIST_FOREACH_END(e); SMARTLIST_FOREACH(dir_sources, dir_src_ent_t *, e, tor_free(e)); smartlist_free(dir_sources); } @@ -701,7 +1578,10 @@ networkstatus_compute_consensus(smartlist_t *votes, smartlist_t *versions = smartlist_create(); smartlist_t *exitsummaries = smartlist_create(); uint32_t *bandwidths = tor_malloc(sizeof(uint32_t) * smartlist_len(votes)); + uint32_t *measured_bws = tor_malloc(sizeof(uint32_t) * + smartlist_len(votes)); int num_bandwidths; + int num_mbws; int *n_voter_flags; /* n_voter_flags[j] is the number of flags that * votes[j] knows about. */ @@ -813,10 +1693,12 @@ networkstatus_compute_consensus(smartlist_t *votes, const char *chosen_name = NULL; int exitsummary_disagreement = 0; int is_named = 0, is_unnamed = 0, is_running = 0; + int is_guard = 0, is_exit = 0, is_bad_exit = 0; int naming_conflict = 0; int n_listing = 0; int i; - char buf[256]; + char *buf=NULL; + char microdesc_digest[DIGEST256_LEN]; /* Of the next-to-be-considered digest in each voter, which is first? */ SMARTLIST_FOREACH(votes, networkstatus_t *, v, { @@ -835,6 +1717,7 @@ networkstatus_compute_consensus(smartlist_t *votes, smartlist_clear(chosen_flags); smartlist_clear(versions); num_bandwidths = 0; + num_mbws = 0; /* Okay, go through all the entries for this digest. */ SMARTLIST_FOREACH_BEGIN(votes, networkstatus_t *, v) { @@ -868,6 +1751,9 @@ networkstatus_compute_consensus(smartlist_t *votes, } /* count bandwidths */ + if (rs->status.has_measured_bw) + measured_bws[num_mbws++] = rs->status.measured_bw; + if (rs->status.has_bandwidth) bandwidths[num_bandwidths++] = rs->status.bandwidth; } SMARTLIST_FOREACH_END(v); @@ -879,7 +1765,9 @@ networkstatus_compute_consensus(smartlist_t *votes, /* Figure out the most popular opinion of what the most recent * routerinfo and its contents are. */ - rs = compute_routerstatus_consensus(matching_descs); + memset(microdesc_digest, 0, sizeof(microdesc_digest)); + rs = compute_routerstatus_consensus(matching_descs, consensus_method, + microdesc_digest); /* Copy bits of that into rs_out. */ tor_assert(!memcmp(lowest_id, rs->status.identity_digest, DIGEST_LEN)); memcpy(rs_out.identity_digest, lowest_id, DIGEST_LEN); @@ -925,8 +1813,14 @@ networkstatus_compute_consensus(smartlist_t *votes, } else { if (flag_counts[fl_sl_idx] > n_flag_voters[fl_sl_idx]/2) { smartlist_add(chosen_flags, (char*)fl); - if (!strcmp(fl, "Running")) + if (!strcmp(fl, "Exit")) + is_exit = 1; + else if (!strcmp(fl, "Guard")) + is_guard = 1; + else if (!strcmp(fl, "Running")) is_running = 1; + else if (!strcmp(fl, "BadExit")) + is_bad_exit = 1; } } }); @@ -945,11 +1839,36 @@ networkstatus_compute_consensus(smartlist_t *votes, } /* Pick a bandwidth */ - if (consensus_method >= 5 && num_bandwidths > 0) { + if (consensus_method >= 6 && num_mbws > 2) { + rs_out.has_bandwidth = 1; + rs_out.bandwidth = median_uint32(measured_bws, num_mbws); + } else if (consensus_method >= 5 && num_bandwidths > 0) { rs_out.has_bandwidth = 1; rs_out.bandwidth = median_uint32(bandwidths, num_bandwidths); } + /* Fix bug 2203: Do not count BadExit nodes as Exits for bw weights */ + if (consensus_method >= 11) { + is_exit = is_exit && !is_bad_exit; + } + + if (consensus_method >= MIN_METHOD_FOR_BW_WEIGHTS) { + if (rs_out.has_bandwidth) { + T += rs_out.bandwidth; + if (is_exit && is_guard) + D += rs_out.bandwidth; + else if (is_exit) + E += rs_out.bandwidth; + else if (is_guard) + G += rs_out.bandwidth; + else + M += rs_out.bandwidth; + } else { + log_warn(LD_BUG, "Missing consensus bandwidth for router %s", + rs_out.nickname); + } + } + /* Ok, we already picked a descriptor digest we want to list * previously. Now we want to use the exit policy summary from * that descriptor. If everybody plays nice all the voters who @@ -1034,11 +1953,23 @@ networkstatus_compute_consensus(smartlist_t *votes, } } - /* Okay!! Now we can write the descriptor... */ - /* First line goes into "buf". */ - routerstatus_format_entry(buf, sizeof(buf), &rs_out, NULL, 1, 0); - smartlist_add(chunks, tor_strdup(buf)); - /* Second line is all flags. The "\n" is missing. */ + { + char buf[4096]; + /* Okay!! Now we can write the descriptor... */ + /* First line goes into "buf". */ + routerstatus_format_entry(buf, sizeof(buf), &rs_out, NULL, + rs_format); + smartlist_add(chunks, tor_strdup(buf)); + } + /* Now an m line, if applicable. */ + if (flavor == FLAV_MICRODESC && + !tor_digest256_is_zero(microdesc_digest)) { + char m[BASE64_DIGEST256_LEN+1], *cp; + digest256_to_base64(m, microdesc_digest); + tor_asprintf(&cp, "m %s\n", m); + smartlist_add(chunks, cp); + } + /* Next line is all flags. The "\n" is missing. */ smartlist_add(chunks, smartlist_join_strings(chosen_flags, " ", 0, NULL)); /* Now the version line. */ @@ -1049,24 +1980,16 @@ networkstatus_compute_consensus(smartlist_t *votes, smartlist_add(chunks, tor_strdup("\n")); /* Now the weight line. */ if (rs_out.has_bandwidth) { - int r = tor_snprintf(buf, sizeof(buf), - "w Bandwidth=%d\n", rs_out.bandwidth); - if (r<0) { - log_warn(LD_BUG, "Not enough space in buffer for weight line."); - *buf = '\0'; - } - smartlist_add(chunks, tor_strdup(buf)); - }; + char *cp=NULL; + tor_asprintf(&cp, "w Bandwidth=%d\n", rs_out.bandwidth); + smartlist_add(chunks, cp); + } + /* Now the exitpolicy summary line. */ - if (rs_out.has_exitsummary) { - char buf[MAX_POLICY_LINE_LEN+1]; - int r = tor_snprintf(buf, sizeof(buf), "p %s\n", rs_out.exitsummary); - if (r<0) { - log_warn(LD_BUG, "Not enough space in buffer for exitpolicy line."); - *buf = '\0'; - } - smartlist_add(chunks, tor_strdup(buf)); - }; + if (rs_out.has_exitsummary && flavor == FLAV_NS) { + tor_asprintf(&buf, "p %s\n", rs_out.exitsummary); + smartlist_add(chunks, buf); + } /* And the loop is over and we move on to the next router */ } @@ -1087,34 +2010,97 @@ networkstatus_compute_consensus(smartlist_t *votes, smartlist_free(versions); smartlist_free(exitsummaries); tor_free(bandwidths); + tor_free(measured_bws); + } + + if (consensus_method >= MIN_METHOD_FOR_FOOTER) { + /* Starting with consensus method 9, we clearly mark the directory + * footer region */ + smartlist_add(chunks, tor_strdup("directory-footer\n")); + } + + if (consensus_method >= MIN_METHOD_FOR_BW_WEIGHTS) { + int64_t weight_scale = BW_WEIGHT_SCALE; + char *bw_weight_param = NULL; + + // Parse params, extract BW_WEIGHT_SCALE if present + // DO NOT use consensus_param_bw_weight_scale() in this code! + // The consensus is not formed yet! + if (params) { + if (strcmpstart(params, "bwweightscale=") == 0) + bw_weight_param = params; + else + bw_weight_param = strstr(params, " bwweightscale="); + } + + if (bw_weight_param) { + int ok=0; + char *eq = strchr(bw_weight_param, '='); + if (eq) { + weight_scale = tor_parse_long(eq+1, 10, INT32_MIN, INT32_MAX, &ok, + NULL); + if (!ok) { + log_warn(LD_DIR, "Bad element '%s' in bw weight param", + escaped(bw_weight_param)); + weight_scale = BW_WEIGHT_SCALE; + } + } else { + log_warn(LD_DIR, "Bad element '%s' in bw weight param", + escaped(bw_weight_param)); + weight_scale = BW_WEIGHT_SCALE; + } + } + + if (consensus_method < 10) { + networkstatus_compute_bw_weights_v9(chunks, G, M, E, D, T, weight_scale); + added_weights = 1; + } else { + added_weights = networkstatus_compute_bw_weights_v10(chunks, G, M, E, D, + T, weight_scale); + } } /* Add a signature. */ { - char digest[DIGEST_LEN]; + char digest[DIGEST256_LEN]; char fingerprint[HEX_DIGEST_LEN+1]; char signing_key_fingerprint[HEX_DIGEST_LEN+1]; + digest_algorithm_t digest_alg = + flavor == FLAV_NS ? DIGEST_SHA1 : DIGEST_SHA256; + size_t digest_len = + flavor == FLAV_NS ? DIGEST_LEN : DIGEST256_LEN; + const char *algname = crypto_digest_algorithm_get_name(digest_alg); + char *buf = NULL; + char sigbuf[4096]; - char buf[4096]; smartlist_add(chunks, tor_strdup("directory-signature ")); /* Compute the hash of the chunks. */ - hash_list_members(digest, chunks); + hash_list_members(digest, digest_len, chunks, digest_alg); /* Get the fingerprints */ crypto_pk_get_fingerprint(identity_key, fingerprint, 0); crypto_pk_get_fingerprint(signing_key, signing_key_fingerprint, 0); /* add the junk that will go at the end of the line. */ - tor_snprintf(buf, sizeof(buf), "%s %s\n", fingerprint, - signing_key_fingerprint); + if (flavor == FLAV_NS) { + tor_asprintf(&buf, "%s %s\n", fingerprint, + signing_key_fingerprint); + } else { + tor_asprintf(&buf, "%s %s %s\n", + algname, fingerprint, + signing_key_fingerprint); + } + smartlist_add(chunks, buf); /* And the signature. */ - if (router_append_dirobj_signature(buf, sizeof(buf), digest, + sigbuf[0] = '\0'; + if (router_append_dirobj_signature(sigbuf, sizeof(sigbuf), + digest, digest_len, signing_key)) { log_warn(LD_BUG, "Couldn't sign consensus networkstatus."); return NULL; /* This leaks, but it should never happen. */ } - smartlist_add(chunks, tor_strdup(buf)); + smartlist_add(chunks, tor_strdup(sigbuf)); if (legacy_id_key_digest && legacy_signing_key && consensus_method >= 3) { smartlist_add(chunks, tor_strdup("directory-signature ")); @@ -1122,14 +2108,23 @@ networkstatus_compute_consensus(smartlist_t *votes, legacy_id_key_digest, DIGEST_LEN); crypto_pk_get_fingerprint(legacy_signing_key, signing_key_fingerprint, 0); - tor_snprintf(buf, sizeof(buf), "%s %s\n", fingerprint, - signing_key_fingerprint); - if (router_append_dirobj_signature(buf, sizeof(buf), digest, + if (flavor == FLAV_NS) { + tor_asprintf(&buf, "%s %s\n", fingerprint, + signing_key_fingerprint); + } else { + tor_asprintf(&buf, "%s %s %s\n", + algname, fingerprint, + signing_key_fingerprint); + } + smartlist_add(chunks, buf); + sigbuf[0] = '\0'; + if (router_append_dirobj_signature(sigbuf, sizeof(sigbuf), + digest, digest_len, legacy_signing_key)) { log_warn(LD_BUG, "Couldn't sign consensus networkstatus."); return NULL; /* This leaks, but it should never happen. */ } - smartlist_add(chunks, tor_strdup(buf)); + smartlist_add(chunks, tor_strdup(sigbuf)); } } @@ -1146,11 +2141,15 @@ networkstatus_compute_consensus(smartlist_t *votes, networkstatus_t *c; if (!(c = networkstatus_parse_vote_from_string(result, NULL, NS_TYPE_CONSENSUS))) { - log_err(LD_BUG,"Generated a networkstatus consensus we couldn't " + log_err(LD_BUG, "Generated a networkstatus consensus we couldn't " "parse."); tor_free(result); return NULL; } + // Verify balancing parameters + if (consensus_method >= MIN_METHOD_FOR_BW_WEIGHTS && added_weights) { + networkstatus_verify_bw_weights(c); + } networkstatus_vote_free(c); } @@ -1171,10 +2170,14 @@ networkstatus_add_detached_signatures(networkstatus_t *target, const char **msg_out) { int r = 0; + const char *flavor; + smartlist_t *siglist; tor_assert(sigs); tor_assert(target); tor_assert(target->type == NS_TYPE_CONSENSUS); + flavor = networkstatus_get_flavor_name(target->flavor); + /* Do the times seem right? */ if (target->valid_after != sigs->valid_after) { *msg_out = "Valid-After times do not match " @@ -1191,79 +2194,179 @@ networkstatus_add_detached_signatures(networkstatus_t *target, "when adding detached signatures to consensus"; return -1; } - /* Are they the same consensus? */ - if (memcmp(target->networkstatus_digest, sigs->networkstatus_digest, - DIGEST_LEN)) { - *msg_out = "Digest mismatch when adding detached signatures to consensus"; + siglist = strmap_get(sigs->signatures, flavor); + if (!siglist) { + *msg_out = "No signatures for given consensus flavor"; return -1; } - /* For each voter in src... */ - SMARTLIST_FOREACH_BEGIN(sigs->signatures, networkstatus_voter_info_t *, - src_voter) { - char voter_identity[HEX_DIGEST_LEN+1]; - networkstatus_voter_info_t *target_voter = - networkstatus_get_voter_by_id(target, src_voter->identity_digest); - authority_cert_t *cert = NULL; - - base16_encode(voter_identity, sizeof(voter_identity), - src_voter->identity_digest, DIGEST_LEN); - log_info(LD_DIR, "Looking at signature from %s", voter_identity); - /* If the target doesn't know about this voter, then forget it. */ - if (!target_voter) { - log_info(LD_DIR, "We do not know about %s", voter_identity); - continue; + /** Make sure all the digests we know match, and at least one matches. */ + { + digests_t *digests = strmap_get(sigs->digests, flavor); + int n_matches = 0; + digest_algorithm_t alg; + if (!digests) { + *msg_out = "No digests for given consensus flavor"; + return -1; + } + for (alg = DIGEST_SHA1; alg < N_DIGEST_ALGORITHMS; ++alg) { + if (!tor_mem_is_zero(digests->d[alg], DIGEST256_LEN)) { + if (!memcmp(target->digests.d[alg], digests->d[alg], DIGEST256_LEN)) { + ++n_matches; + } else { + *msg_out = "Mismatched digest."; + return -1; + } } + } + if (!n_matches) { + *msg_out = "No regognized digests for given consensus flavor"; + } + } - /* If the target already has a good signature from this voter, then skip - * this one. */ - if (target_voter->good_signature) { - log_info(LD_DIR, "We already have a good signature from %s", - voter_identity); - continue; - } + /* For each voter in src... */ + SMARTLIST_FOREACH_BEGIN(siglist, document_signature_t *, sig) { + char voter_identity[HEX_DIGEST_LEN+1]; + networkstatus_voter_info_t *target_voter = + networkstatus_get_voter_by_id(target, sig->identity_digest); + authority_cert_t *cert = NULL; + const char *algorithm; + document_signature_t *old_sig = NULL; + + algorithm = crypto_digest_algorithm_get_name(sig->alg); + + base16_encode(voter_identity, sizeof(voter_identity), + sig->identity_digest, DIGEST_LEN); + log_info(LD_DIR, "Looking at signature from %s using %s", voter_identity, + algorithm); + /* If the target doesn't know about this voter, then forget it. */ + if (!target_voter) { + log_info(LD_DIR, "We do not know any voter with ID %s", voter_identity); + continue; + } - /* Try checking the signature if we haven't already. */ - if (!src_voter->good_signature && !src_voter->bad_signature) { - cert = authority_cert_get_by_digests(src_voter->identity_digest, - src_voter->signing_key_digest); - if (cert) { - networkstatus_check_voter_signature(target, src_voter, cert); - } + old_sig = voter_get_sig_by_algorithm(target_voter, sig->alg); + + /* If the target already has a good signature from this voter, then skip + * this one. */ + if (old_sig && old_sig->good_signature) { + log_info(LD_DIR, "We already have a good signature from %s using %s", + voter_identity, algorithm); + continue; + } + + /* Try checking the signature if we haven't already. */ + if (!sig->good_signature && !sig->bad_signature) { + cert = authority_cert_get_by_digests(sig->identity_digest, + sig->signing_key_digest); + if (cert) + networkstatus_check_document_signature(target, sig, cert); + } + + /* If this signature is good, or we don't have any signature yet, + * then maybe add it. */ + if (sig->good_signature || !old_sig || old_sig->bad_signature) { + log_info(LD_DIR, "Adding signature from %s with %s", voter_identity, + algorithm); + ++r; + if (old_sig) { + smartlist_remove(target_voter->sigs, old_sig); + document_signature_free(old_sig); } + smartlist_add(target_voter->sigs, document_signature_dup(sig)); + } else { + log_info(LD_DIR, "Not adding signature from %s", voter_identity); + } + } SMARTLIST_FOREACH_END(sig); + + return r; +} + +/** Return a newly allocated string containing all the signatures on + * <b>consensus</b> by all voters. If <b>for_detached_signatures</b> is true, + * then the signatures will be put in a detached signatures document, so + * prefix any non-NS-flavored signatures with "additional-signature" rather + * than "directory-signature". */ +static char * +networkstatus_format_signatures(networkstatus_t *consensus, + int for_detached_signatures) +{ + smartlist_t *elements; + char buf[4096]; + char *result = NULL; + int n_sigs = 0; + const consensus_flavor_t flavor = consensus->flavor; + const char *flavor_name = networkstatus_get_flavor_name(flavor); + const char *keyword; - /* If this signature is good, or we don't have any signature yet, - * then add it. */ - if (src_voter->good_signature || !target_voter->signature) { - log_info(LD_DIR, "Adding signature from %s", voter_identity); - ++r; - tor_free(target_voter->signature); - target_voter->signature = - tor_memdup(src_voter->signature, src_voter->signature_len); - memcpy(target_voter->signing_key_digest, src_voter->signing_key_digest, - DIGEST_LEN); - target_voter->signature_len = src_voter->signature_len; - target_voter->good_signature = src_voter->good_signature; - target_voter->bad_signature = src_voter->bad_signature; + if (for_detached_signatures && flavor != FLAV_NS) + keyword = "additional-signature"; + else + keyword = "directory-signature"; + + elements = smartlist_create(); + + SMARTLIST_FOREACH_BEGIN(consensus->voters, networkstatus_voter_info_t *, v) { + SMARTLIST_FOREACH_BEGIN(v->sigs, document_signature_t *, sig) { + char sk[HEX_DIGEST_LEN+1]; + char id[HEX_DIGEST_LEN+1]; + if (!sig->signature || sig->bad_signature) + continue; + ++n_sigs; + base16_encode(sk, sizeof(sk), sig->signing_key_digest, DIGEST_LEN); + base16_encode(id, sizeof(id), sig->identity_digest, DIGEST_LEN); + if (flavor == FLAV_NS) { + tor_snprintf(buf, sizeof(buf), + "%s %s %s\n-----BEGIN SIGNATURE-----\n", + keyword, id, sk); } else { - log_info(LD_DIR, "Not adding signature from %s", voter_identity); + const char *digest_name = + crypto_digest_algorithm_get_name(sig->alg); + tor_snprintf(buf, sizeof(buf), + "%s%s%s %s %s %s\n-----BEGIN SIGNATURE-----\n", + keyword, + for_detached_signatures ? " " : "", + for_detached_signatures ? flavor_name : "", + digest_name, id, sk); } - } SMARTLIST_FOREACH_END(src_voter); + smartlist_add(elements, tor_strdup(buf)); + base64_encode(buf, sizeof(buf), sig->signature, sig->signature_len); + strlcat(buf, "-----END SIGNATURE-----\n", sizeof(buf)); + smartlist_add(elements, tor_strdup(buf)); + } SMARTLIST_FOREACH_END(sig); + } SMARTLIST_FOREACH_END(v); - return r; + result = smartlist_join_strings(elements, "", 0, NULL); + SMARTLIST_FOREACH(elements, char *, cp, tor_free(cp)); + smartlist_free(elements); + if (!n_sigs) + tor_free(result); + return result; } /** Return a newly allocated string holding the detached-signatures document - * corresponding to the signatures on <b>consensus</b>. */ + * corresponding to the signatures on <b>consensuses</b>, which must contain + * exactly one FLAV_NS consensus, and no more than one consensus for each + * other flavor. */ char * -networkstatus_get_detached_signatures(networkstatus_t *consensus) +networkstatus_get_detached_signatures(smartlist_t *consensuses) { smartlist_t *elements; char buf[4096]; - char *result = NULL; - int n_sigs = 0; - tor_assert(consensus); - tor_assert(consensus->type == NS_TYPE_CONSENSUS); + char *result = NULL, *sigs = NULL; + networkstatus_t *consensus_ns = NULL; + tor_assert(consensuses); + + SMARTLIST_FOREACH(consensuses, networkstatus_t *, ns, { + tor_assert(ns); + tor_assert(ns->type == NS_TYPE_CONSENSUS); + if (ns && ns->flavor == FLAV_NS) + consensus_ns = ns; + }); + if (!consensus_ns) { + log_warn(LD_BUG, "No NS consensus given."); + return NULL; + } elements = smartlist_create(); @@ -1272,10 +2375,11 @@ networkstatus_get_detached_signatures(networkstatus_t *consensus) vu_buf[ISO_TIME_LEN+1]; char d[HEX_DIGEST_LEN+1]; - base16_encode(d, sizeof(d), consensus->networkstatus_digest, DIGEST_LEN); - format_iso_time(va_buf, consensus->valid_after); - format_iso_time(fu_buf, consensus->fresh_until); - format_iso_time(vu_buf, consensus->valid_until); + base16_encode(d, sizeof(d), + consensus_ns->digests.d[DIGEST_SHA1], DIGEST_LEN); + format_iso_time(va_buf, consensus_ns->valid_after); + format_iso_time(fu_buf, consensus_ns->fresh_until); + format_iso_time(vu_buf, consensus_ns->valid_until); tor_snprintf(buf, sizeof(buf), "consensus-digest %s\n" @@ -1285,45 +2389,89 @@ networkstatus_get_detached_signatures(networkstatus_t *consensus) smartlist_add(elements, tor_strdup(buf)); } - SMARTLIST_FOREACH(consensus->voters, networkstatus_voter_info_t *, v, - { - char sk[HEX_DIGEST_LEN+1]; - char id[HEX_DIGEST_LEN+1]; - if (!v->signature || v->bad_signature) + /* Get all the digests for the non-FLAV_NS consensuses */ + SMARTLIST_FOREACH_BEGIN(consensuses, networkstatus_t *, ns) { + const char *flavor_name = networkstatus_get_flavor_name(ns->flavor); + int alg; + if (ns->flavor == FLAV_NS) + continue; + + /* start with SHA256; we don't include SHA1 for anything but the basic + * consensus. */ + for (alg = DIGEST_SHA256; alg < N_DIGEST_ALGORITHMS; ++alg) { + char d[HEX_DIGEST256_LEN+1]; + const char *alg_name = + crypto_digest_algorithm_get_name(alg); + if (tor_mem_is_zero(ns->digests.d[alg], DIGEST256_LEN)) continue; - ++n_sigs; - base16_encode(sk, sizeof(sk), v->signing_key_digest, DIGEST_LEN); - base16_encode(id, sizeof(id), v->identity_digest, DIGEST_LEN); - tor_snprintf(buf, sizeof(buf), - "directory-signature %s %s\n-----BEGIN SIGNATURE-----\n", - id, sk); - smartlist_add(elements, tor_strdup(buf)); - base64_encode(buf, sizeof(buf), v->signature, v->signature_len); - strlcat(buf, "-----END SIGNATURE-----\n", sizeof(buf)); + base16_encode(d, sizeof(d), ns->digests.d[alg], DIGEST256_LEN); + tor_snprintf(buf, sizeof(buf), "additional-digest %s %s %s\n", + flavor_name, alg_name, d); smartlist_add(elements, tor_strdup(buf)); - }); + } + } SMARTLIST_FOREACH_END(ns); + + /* Now get all the sigs for non-FLAV_NS consensuses */ + SMARTLIST_FOREACH_BEGIN(consensuses, networkstatus_t *, ns) { + char *sigs; + if (ns->flavor == FLAV_NS) + continue; + sigs = networkstatus_format_signatures(ns, 1); + if (!sigs) { + log_warn(LD_DIR, "Couldn't format signatures"); + goto err; + } + smartlist_add(elements, sigs); + } SMARTLIST_FOREACH_END(ns); - result = smartlist_join_strings(elements, "", 0, NULL); + /* Now add the FLAV_NS consensus signatrures. */ + sigs = networkstatus_format_signatures(consensus_ns, 1); + if (!sigs) + goto err; + smartlist_add(elements, sigs); + result = smartlist_join_strings(elements, "", 0, NULL); + err: SMARTLIST_FOREACH(elements, char *, cp, tor_free(cp)); smartlist_free(elements); - if (!n_sigs) - tor_free(result); return result; } +/** Return a newly allocated string holding a detached-signatures document for + * all of the in-progress consensuses in the <b>n_flavors</b>-element array at + * <b>pending</b>. */ +static char * +get_detached_signatures_from_pending_consensuses(pending_consensus_t *pending, + int n_flavors) +{ + int flav; + char *signatures; + smartlist_t *c = smartlist_create(); + for (flav = 0; flav < n_flavors; ++flav) { + if (pending[flav].consensus) + smartlist_add(c, pending[flav].consensus); + } + signatures = networkstatus_get_detached_signatures(c); + smartlist_free(c); + return signatures; +} + /** Release all storage held in <b>s</b>. */ void ns_detached_signatures_free(ns_detached_signatures_t *s) { + if (!s) + return; if (s->signatures) { - SMARTLIST_FOREACH(s->signatures, networkstatus_voter_info_t *, v, - { - tor_free(v->signature); - tor_free(v); - }); - smartlist_free(s->signatures); + STRMAP_FOREACH(s->signatures, flavor, smartlist_t *, sigs) { + SMARTLIST_FOREACH(sigs, document_signature_t *, sig, + document_signature_free(sig)); + smartlist_free(sigs); + } STRMAP_FOREACH_END; + strmap_free(s->signatures, NULL); + strmap_free(s->digests, _tor_free); } + tor_free(s); } @@ -1513,7 +2661,7 @@ dirvote_act(or_options_t *options, time_t now) if (voting_schedule.voting_ends < now && !voting_schedule.have_built_consensus) { log_notice(LD_DIR, "Time to compute a consensus."); - dirvote_compute_consensus(); + dirvote_compute_consensuses(); /* XXXX We will want to try again later if we haven't got enough * votes yet. Implement this if it turns out to ever happen. */ voting_schedule.have_built_consensus = 1; @@ -1550,14 +2698,13 @@ static smartlist_t *pending_vote_list = NULL; /** List of pending_vote_t for the previous vote. After we've used them to * build a consensus, the votes go here for the next period. */ static smartlist_t *previous_vote_list = NULL; -/** The body of the consensus that we're currently building. Once we - * have it built, it goes into dirserv.c */ -static char *pending_consensus_body = NULL; + +static pending_consensus_t pending_consensuses[N_CONSENSUS_FLAVORS]; + /** The detached signatures for the consensus that we're currently * building. */ static char *pending_consensus_signatures = NULL; -/** The parsed in-progress consensus document. */ -static networkstatus_t *pending_consensus = NULL; + /** List of ns_detached_signatures_t: hold signatures that get posted to us * before we have generated the consensus on our own. */ static smartlist_t *pending_consensus_signature_list = NULL; @@ -1651,15 +2798,39 @@ dirvote_fetch_missing_votes(void) static void dirvote_fetch_missing_signatures(void) { - if (!pending_consensus) + int need_any = 0; + int i; + for (i=0; i < N_CONSENSUS_FLAVORS; ++i) { + networkstatus_t *consensus = pending_consensuses[i].consensus; + if (!consensus || + networkstatus_check_consensus_signature(consensus, -1) == 1) { + /* We have no consensus, or we have one that's signed by everybody. */ + continue; + } + need_any = 1; + } + if (!need_any) return; - if (networkstatus_check_consensus_signature(pending_consensus, -1) == 1) - return; /* we have a signature from everybody. */ directory_get_from_all_authorities(DIR_PURPOSE_FETCH_DETACHED_SIGNATURES, 0, NULL); } +/** Release all storage held by pending consensuses (those waiting for + * signatures). */ +static void +dirvote_clear_pending_consensuses(void) +{ + int i; + for (i = 0; i < N_CONSENSUS_FLAVORS; ++i) { + pending_consensus_t *pc = &pending_consensuses[i]; + tor_free(pc->body); + + networkstatus_vote_free(pc->consensus); + pc->consensus = NULL; + } +} + /** Drop all currently pending votes, consensus, and detached signatures. */ static void dirvote_clear_votes(int all_votes) @@ -1697,12 +2868,8 @@ dirvote_clear_votes(int all_votes) tor_free(cp)); smartlist_clear(pending_consensus_signature_list); } - tor_free(pending_consensus_body); tor_free(pending_consensus_signatures); - if (pending_consensus) { - networkstatus_vote_free(pending_consensus); - pending_consensus = NULL; - } + dirvote_clear_pending_consensuses(); } /** Return a newly allocated string containing the hex-encoded v3 authority @@ -1760,7 +2927,13 @@ dirvote_add_vote(const char *vote_body, const char **msg_out, int *status_out) } tor_assert(smartlist_len(vote->voters) == 1); vi = get_voter(vote); - tor_assert(vi->good_signature == 1); + { + int any_sig_good = 0; + SMARTLIST_FOREACH(vi->sigs, document_signature_t *, sig, + if (sig->good_signature) + any_sig_good = 1); + tor_assert(any_sig_good); + } ds = trusteddirserver_get_by_v3_auth_digest(vi->identity_digest); if (!ds) { char *keys = list_v3_auth_ids(); @@ -1797,6 +2970,9 @@ dirvote_add_vote(const char *vote_body, const char **msg_out, int *status_out) goto err; } + /* Fetch any new router descriptors we just learned about */ + update_consensus_router_descriptor_downloads(time(NULL), 1, vote); + /* Now see whether we already have a vote from this authority. */ SMARTLIST_FOREACH(pending_vote_list, pending_vote_t *, v, { if (! memcmp(v->vote->cert->cache_info.identity_digest, @@ -1805,7 +2981,8 @@ dirvote_add_vote(const char *vote_body, const char **msg_out, int *status_out) networkstatus_voter_info_t *vi_old = get_voter(v->vote); if (!memcmp(vi_old->vote_digest, vi->vote_digest, DIGEST_LEN)) { /* Ah, it's the same vote. Not a problem. */ - log_info(LD_DIR, "Discarding a vote we already have."); + log_info(LD_DIR, "Discarding a vote we already have (from %s).", + vi->address); if (*status_out < 200) *status_out = 200; goto discard; @@ -1832,7 +3009,7 @@ dirvote_add_vote(const char *vote_body, const char **msg_out, int *status_out) goto err; } } - }); + }); pending_vote = tor_malloc_zero(sizeof(pending_vote_t)); pending_vote->vote_body = new_cached_dir(tor_strndup(vote_body, @@ -1856,8 +3033,7 @@ dirvote_add_vote(const char *vote_body, const char **msg_out, int *status_out) *status_out = 400; discard: - if (vote) - networkstatus_vote_free(vote); + networkstatus_vote_free(vote); if (end_of_vote && !strcmpstart(end_of_vote, "network-status-version ")) { vote_body = end_of_vote; @@ -1884,14 +3060,18 @@ dirvote_add_vote(const char *vote_body, const char **msg_out, int *status_out) * pending_consensus: it won't be ready to be published until we have * everybody else's signatures collected too. (V3 Authority only) */ static int -dirvote_compute_consensus(void) +dirvote_compute_consensuses(void) { /* Have we got enough votes to try? */ - int n_votes, n_voters; + int n_votes, n_voters, n_vote_running = 0; smartlist_t *votes = NULL, *votestrings = NULL; char *consensus_body = NULL, *signatures = NULL, *votefile; networkstatus_t *consensus = NULL; authority_cert_t *my_cert; + pending_consensus_t pending[N_CONSENSUS_FLAVORS]; + int flav; + + memset(pending, 0, sizeof(pending)); if (!pending_vote_list) pending_vote_list = smartlist_create(); @@ -1900,7 +3080,20 @@ dirvote_compute_consensus(void) n_votes = smartlist_len(pending_vote_list); if (n_votes <= n_voters/2) { log_warn(LD_DIR, "We don't have enough votes to generate a consensus: " - "%d of %d", n_votes, n_voters/2); + "%d of %d", n_votes, n_voters/2+1); + goto err; + } + tor_assert(pending_vote_list); + SMARTLIST_FOREACH(pending_vote_list, pending_vote_t *, v, { + if (smartlist_string_isin(v->vote->known_flags, "Running")) + n_vote_running++; + }); + if (!n_vote_running) { + /* See task 1066. */ + log_warn(LD_DIR, "Nobody has voted on the Running flag. Generating " + "and publishing a consensus without Running nodes " + "would make many clients stop working. Not " + "generating a consensus!"); goto err; } @@ -1931,6 +3124,7 @@ dirvote_compute_consensus(void) char legacy_dbuf[DIGEST_LEN]; crypto_pk_env_t *legacy_sign=NULL; char *legacy_id_digest = NULL; + int n_generated = 0; if (get_options()->V3AuthUseLegacyKey) { authority_cert_t *cert = get_my_v3_legacy_cert(); legacy_sign = get_my_v3_legacy_signing_key(); @@ -1939,39 +3133,58 @@ dirvote_compute_consensus(void) legacy_id_digest = legacy_dbuf; } } - consensus_body = networkstatus_compute_consensus( + + for (flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) { + const char *flavor_name = networkstatus_get_flavor_name(flav); + consensus_body = networkstatus_compute_consensus( votes, n_voters, my_cert->identity_key, - get_my_v3_authority_signing_key(), legacy_id_digest, legacy_sign); - } - if (!consensus_body) { - log_warn(LD_DIR, "Couldn't generate a consensus at all!"); - goto err; - } - consensus = networkstatus_parse_vote_from_string(consensus_body, NULL, - NS_TYPE_CONSENSUS); - if (!consensus) { - log_warn(LD_DIR, "Couldn't parse consensus we generated!"); - goto err; + get_my_v3_authority_signing_key(), legacy_id_digest, legacy_sign, + flav); + + if (!consensus_body) { + log_warn(LD_DIR, "Couldn't generate a %s consensus at all!", + flavor_name); + continue; + } + consensus = networkstatus_parse_vote_from_string(consensus_body, NULL, + NS_TYPE_CONSENSUS); + if (!consensus) { + log_warn(LD_DIR, "Couldn't parse %s consensus we generated!", + flavor_name); + tor_free(consensus_body); + continue; + } + + /* 'Check' our own signature, to mark it valid. */ + networkstatus_check_consensus_signature(consensus, -1); + + pending[flav].body = consensus_body; + pending[flav].consensus = consensus; + n_generated++; + consensus_body = NULL; + consensus = NULL; + } + if (!n_generated) { + log_warn(LD_DIR, "Couldn't generate any consensus flavors at all."); + goto err; + } } - /* 'Check' our own signature, to mark it valid. */ - networkstatus_check_consensus_signature(consensus, -1); - signatures = networkstatus_get_detached_signatures(consensus); + signatures = get_detached_signatures_from_pending_consensuses( + pending, N_CONSENSUS_FLAVORS); + if (!signatures) { log_warn(LD_DIR, "Couldn't extract signatures."); goto err; } - tor_free(pending_consensus_body); - pending_consensus_body = consensus_body; + dirvote_clear_pending_consensuses(); + memcpy(pending_consensuses, pending, sizeof(pending)); + tor_free(pending_consensus_signatures); pending_consensus_signatures = signatures; - if (pending_consensus) - networkstatus_vote_free(pending_consensus); - pending_consensus = consensus; - if (pending_consensus_signature_list) { int n_sigs = 0; /* we may have gotten signatures for this consensus before we built @@ -1979,7 +3192,7 @@ dirvote_compute_consensus(void) SMARTLIST_FOREACH(pending_consensus_signature_list, char *, sig, { const char *msg = NULL; - int r = dirvote_add_signatures_to_pending_consensus(sig, &msg); + int r = dirvote_add_signatures_to_all_pending_consensuses(sig, &msg); if (r >= 0) n_sigs += r; else @@ -2003,10 +3216,10 @@ dirvote_compute_consensus(void) strlen(pending_consensus_signatures), 0); log_notice(LD_DIR, "Signature(s) posted."); + smartlist_free(votes); return 0; err: - if (votes) - smartlist_free(votes); + smartlist_free(votes); tor_free(consensus_body); tor_free(signatures); networkstatus_vote_free(consensus); @@ -2015,76 +3228,62 @@ dirvote_compute_consensus(void) } /** Helper: we just got the <b>detached_signatures_body</b> sent to us as - * signatures on the currently pending consensus. Add them to the consensus + * signatures on the currently pending consensus. Add them to <b>pc</b> * as appropriate. Return the number of signatures added. (?) */ static int dirvote_add_signatures_to_pending_consensus( - const char *detached_signatures_body, + pending_consensus_t *pc, + ns_detached_signatures_t *sigs, const char **msg_out) { - ns_detached_signatures_t *sigs = NULL; + const char *flavor_name; int r = -1; - tor_assert(detached_signatures_body); - tor_assert(msg_out); - /* Only call if we have a pending consensus right now. */ - tor_assert(pending_consensus); - tor_assert(pending_consensus_body); + tor_assert(pc->consensus); + tor_assert(pc->body); tor_assert(pending_consensus_signatures); + flavor_name = networkstatus_get_flavor_name(pc->consensus->flavor); *msg_out = NULL; - if (!(sigs = networkstatus_parse_detached_signatures( - detached_signatures_body, NULL))) { - *msg_out = "Couldn't parse detached signatures."; - goto err; + { + smartlist_t *sig_list = strmap_get(sigs->signatures, flavor_name); + log_info(LD_DIR, "Have %d signatures for adding to %s consensus.", + sig_list ? smartlist_len(sig_list) : 0, flavor_name); } - - log_info(LD_DIR, "Have %d signatures for adding to consensus.", - smartlist_len(sigs->signatures)); - r = networkstatus_add_detached_signatures(pending_consensus, - sigs, msg_out); + r = networkstatus_add_detached_signatures(pc->consensus, sigs, msg_out); log_info(LD_DIR,"Added %d signatures to consensus.", r); if (r >= 1) { - char *new_detached = - networkstatus_get_detached_signatures(pending_consensus); - const char *src; + char *new_signatures = + networkstatus_format_signatures(pc->consensus, 0); char *dst, *dst_end; size_t new_consensus_len; - if (!new_detached) { + if (!new_signatures) { *msg_out = "No signatures to add"; goto err; } new_consensus_len = - strlen(pending_consensus_body) + strlen(new_detached) + 1; - pending_consensus_body = tor_realloc(pending_consensus_body, - new_consensus_len); - dst_end = pending_consensus_body + new_consensus_len; - dst = strstr(pending_consensus_body, "directory-signature "); + strlen(pc->body) + strlen(new_signatures) + 1; + pc->body = tor_realloc(pc->body, new_consensus_len); + dst_end = pc->body + new_consensus_len; + dst = strstr(pc->body, "directory-signature "); tor_assert(dst); - src = strstr(new_detached, "directory-signature "); - tor_assert(src); - strlcpy(dst, src, dst_end-dst); + strlcpy(dst, new_signatures, dst_end-dst); /* We remove this block once it has failed to crash for a while. But * unless it shows up in profiles, we're probably better leaving it in, * just in case we break detached signature processing at some point. */ { - ns_detached_signatures_t *sigs = - networkstatus_parse_detached_signatures(new_detached, NULL); networkstatus_t *v = networkstatus_parse_vote_from_string( - pending_consensus_body, NULL, + pc->body, NULL, NS_TYPE_CONSENSUS); - tor_assert(sigs); - ns_detached_signatures_free(sigs); tor_assert(v); networkstatus_vote_free(v); } - tor_free(pending_consensus_signatures); - pending_consensus_signatures = new_detached; *msg_out = "Signatures added"; + tor_free(new_signatures); } else if (r == 0) { *msg_out = "Signatures ignored"; } else { @@ -2096,8 +3295,62 @@ dirvote_add_signatures_to_pending_consensus( if (!*msg_out) *msg_out = "Unrecognized error while adding detached signatures."; done: - if (sigs) - ns_detached_signatures_free(sigs); + return r; +} + +static int +dirvote_add_signatures_to_all_pending_consensuses( + const char *detached_signatures_body, + const char **msg_out) +{ + int r=0, i, n_added = 0, errors = 0; + ns_detached_signatures_t *sigs; + tor_assert(detached_signatures_body); + tor_assert(msg_out); + tor_assert(pending_consensus_signatures); + + if (!(sigs = networkstatus_parse_detached_signatures( + detached_signatures_body, NULL))) { + *msg_out = "Couldn't parse detached signatures."; + goto err; + } + + for (i = 0; i < N_CONSENSUS_FLAVORS; ++i) { + int res; + pending_consensus_t *pc = &pending_consensuses[i]; + if (!pc->consensus) + continue; + res = dirvote_add_signatures_to_pending_consensus(pc, sigs, msg_out); + if (res < 0) + errors++; + else + n_added += res; + } + + if (errors && !n_added) { + r = -1; + goto err; + } + + if (n_added && pending_consensuses[FLAV_NS].consensus) { + char *new_detached = + get_detached_signatures_from_pending_consensuses( + pending_consensuses, N_CONSENSUS_FLAVORS); + if (new_detached) { + tor_free(pending_consensus_signatures); + pending_consensus_signatures = new_detached; + } + } + + r = n_added; + goto done; + err: + if (!*msg_out) + *msg_out = "Unrecognized error while adding detached signatures."; + done: + ns_detached_signatures_free(sigs); + /* XXXX NM Check how return is used. We can now have an error *and* + signatures added. */ return r; } @@ -2110,10 +3363,10 @@ dirvote_add_signatures(const char *detached_signatures_body, const char *source, const char **msg) { - if (pending_consensus) { + if (pending_consensuses[FLAV_NS].consensus) { log_notice(LD_DIR, "Got a signature from %s. " "Adding it to the pending consensus.", source); - return dirvote_add_signatures_to_pending_consensus( + return dirvote_add_signatures_to_all_pending_consensuses( detached_signatures_body, msg); } else { log_notice(LD_DIR, "Got a signature from %s. " @@ -2132,17 +3385,25 @@ dirvote_add_signatures(const char *detached_signatures_body, static int dirvote_publish_consensus(void) { - /* Can we actually publish it yet? */ - if (!pending_consensus || - networkstatus_check_consensus_signature(pending_consensus, 1)<0) { - log_warn(LD_DIR, "Not enough info to publish pending consensus"); - return -1; - } + int i; + + /* Now remember all the other consensuses as if we were a directory cache. */ + for (i = 0; i < N_CONSENSUS_FLAVORS; ++i) { + pending_consensus_t *pending = &pending_consensuses[i]; + const char *name; + name = networkstatus_get_flavor_name(i); + tor_assert(name); + if (!pending->consensus || + networkstatus_check_consensus_signature(pending->consensus, 1)<0) { + log_warn(LD_DIR, "Not enough info to publish pending %s consensus",name); + continue; + } - if (networkstatus_set_current_consensus(pending_consensus_body, 0)) - log_warn(LD_DIR, "Error publishing consensus"); - else - log_notice(LD_DIR, "Consensus published."); + if (networkstatus_set_current_consensus(pending->body, name, 0)) + log_warn(LD_DIR, "Error publishing %s consensus", name); + else + log_notice(LD_DIR, "Published %s consensus", name); + } return 0; } @@ -2152,20 +3413,16 @@ void dirvote_free_all(void) { dirvote_clear_votes(1); - /* now empty as a result of clear_pending_votes. */ + /* now empty as a result of dirvote_clear_votes(). */ smartlist_free(pending_vote_list); pending_vote_list = NULL; smartlist_free(previous_vote_list); previous_vote_list = NULL; - tor_free(pending_consensus_body); + dirvote_clear_pending_consensuses(); tor_free(pending_consensus_signatures); - if (pending_consensus) { - networkstatus_vote_free(pending_consensus); - pending_consensus = NULL; - } if (pending_consensus_signature_list) { - /* now empty as a result of clear_pending_votes. */ + /* now empty as a result of dirvote_clear_votes(). */ smartlist_free(pending_consensus_signature_list); pending_consensus_signature_list = NULL; } @@ -2177,13 +3434,14 @@ dirvote_free_all(void) /** Return the body of the consensus that we're currently trying to build. */ const char * -dirvote_get_pending_consensus(void) +dirvote_get_pending_consensus(consensus_flavor_t flav) { - return pending_consensus_body; + tor_assert(((int)flav) >= 0 && flav < N_CONSENSUS_FLAVORS); + return pending_consensuses[flav].body; } /** Return the signatures that we know for the consensus that we're currently - * trying to build */ + * trying to build. */ const char * dirvote_get_pending_detached_signatures(void) { @@ -2229,15 +3487,147 @@ dirvote_get_vote(const char *fp, int flags) } else { if (pending_vote_list && include_pending) { SMARTLIST_FOREACH(pending_vote_list, pending_vote_t *, pv, - if (!memcmp(pv->vote->networkstatus_digest, fp, DIGEST_LEN)) + if (!memcmp(pv->vote->digests.d[DIGEST_SHA1], fp, DIGEST_LEN)) return pv->vote_body); } if (previous_vote_list && include_previous) { SMARTLIST_FOREACH(previous_vote_list, pending_vote_t *, pv, - if (!memcmp(pv->vote->networkstatus_digest, fp, DIGEST_LEN)) + if (!memcmp(pv->vote->digests.d[DIGEST_SHA1], fp, DIGEST_LEN)) return pv->vote_body); } } return NULL; } +/** Construct and return a new microdescriptor from a routerinfo <b>ri</b>. + * + * XXX Right now, there is only one way to generate microdescriptors from + * router descriptors. This may change in future consensus methods. If so, + * we'll need an internal way to remember which method we used, and ask for a + * particular method. + **/ +microdesc_t * +dirvote_create_microdescriptor(const routerinfo_t *ri) +{ + microdesc_t *result = NULL; + char *key = NULL, *summary = NULL, *family = NULL; + char buf[1024]; + size_t keylen; + char *out = buf, *end = buf+sizeof(buf); + + if (crypto_pk_write_public_key_to_string(ri->onion_pkey, &key, &keylen)<0) + goto done; + summary = policy_summarize(ri->exit_policy); + if (ri->declared_family) + family = smartlist_join_strings(ri->declared_family, " ", 0, NULL); + + if (tor_snprintf(out, end-out, "onion-key\n%s", key)<0) + goto done; + out += strlen(out); + if (family) { + if (tor_snprintf(out, end-out, "family %s\n", family)<0) + goto done; + out += strlen(out); + } + if (summary && strcmp(summary, "reject 1-65535")) { + if (tor_snprintf(out, end-out, "p %s\n", summary)<0) + goto done; + out += strlen(out); + } + *out = '\0'; /* Make sure it's nul-terminated. This should be a no-op */ + + { + smartlist_t *lst = microdescs_parse_from_string(buf, out, 0, 1); + if (smartlist_len(lst) != 1) { + log_warn(LD_DIR, "We generated a microdescriptor we couldn't parse."); + SMARTLIST_FOREACH(lst, microdesc_t *, md, microdesc_free(md)); + smartlist_free(lst); + goto done; + } + result = smartlist_get(lst, 0); + smartlist_free(lst); + } + + done: + tor_free(key); + tor_free(summary); + tor_free(family); + return result; +} + +/** Cached space-separated string to hold */ +static char *microdesc_consensus_methods = NULL; + +/** Format the appropriate vote line to describe the microdescriptor <b>md</b> + * in a consensus vote document. Write it into the <b>out_len</b>-byte buffer + * in <b>out</b>. Return -1 on failure and the number of characters written + * on success. */ +ssize_t +dirvote_format_microdesc_vote_line(char *out, size_t out_len, + const microdesc_t *md) +{ + char d64[BASE64_DIGEST256_LEN+1]; + if (!microdesc_consensus_methods) { + microdesc_consensus_methods = + make_consensus_method_list(MIN_METHOD_FOR_MICRODESC, + MAX_SUPPORTED_CONSENSUS_METHOD, + ","); + tor_assert(microdesc_consensus_methods); + } + if (digest256_to_base64(d64, md->digest)<0) + return -1; + + if (tor_snprintf(out, out_len, "m %s sha256=%s\n", + microdesc_consensus_methods, d64)<0) + return -1; + + return strlen(out); +} + +/** If <b>vrs</b> has a hash made for the consensus method <b>method</b> with + * the digest algorithm <b>alg</b>, decode it and copy it into + * <b>digest256_out</b> and return 0. Otherwise return -1. */ +int +vote_routerstatus_find_microdesc_hash(char *digest256_out, + const vote_routerstatus_t *vrs, + int method, + digest_algorithm_t alg) +{ + /* XXXX only returns the sha256 method. */ + const vote_microdesc_hash_t *h; + char mstr[64]; + size_t mlen; + char dstr[64]; + + tor_snprintf(mstr, sizeof(mstr), "%d", method); + mlen = strlen(mstr); + tor_snprintf(dstr, sizeof(dstr), " %s=", + crypto_digest_algorithm_get_name(alg)); + + for (h = vrs->microdesc; h; h = h->next) { + const char *cp = h->microdesc_hash_line; + size_t num_len; + /* cp looks like \d+(,\d+)* (digesttype=val )+ . Let's hunt for mstr in + * the first part. */ + while (1) { + num_len = strspn(cp, "1234567890"); + if (num_len == mlen && !memcmp(mstr, cp, mlen)) { + /* This is the line. */ + char buf[BASE64_DIGEST256_LEN+1]; + /* XXXX ignores extraneous stuff if the digest is too long. This + * seems harmless enough, right? */ + cp = strstr(cp, dstr); + if (!cp) + return -1; + cp += strlen(dstr); + strlcpy(buf, cp, sizeof(buf)); + return digest256_from_base64(digest256_out, buf); + } + if (num_len == 0 || cp[num_len] != ',') + break; + cp += num_len + 1; + } + } + return -1; +} + diff --git a/src/or/dirvote.h b/src/or/dirvote.h new file mode 100644 index 0000000000..67540a37fb --- /dev/null +++ b/src/or/dirvote.h @@ -0,0 +1,90 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file dirvote.h + * \brief Header file for dirvote.c. + **/ + +#ifndef _TOR_DIRVOTE_H +#define _TOR_DIRVOTE_H + +/** Lowest allowable value for VoteSeconds. */ +#define MIN_VOTE_SECONDS 20 +/** Lowest allowable value for DistSeconds. */ +#define MIN_DIST_SECONDS 20 +/** Smallest allowable voting interval. */ +#define MIN_VOTE_INTERVAL 300 + +void dirvote_free_all(void); + +/* vote manipulation */ +char *networkstatus_compute_consensus(smartlist_t *votes, + int total_authorities, + crypto_pk_env_t *identity_key, + crypto_pk_env_t *signing_key, + const char *legacy_identity_key_digest, + crypto_pk_env_t *legacy_signing_key, + consensus_flavor_t flavor); +int networkstatus_add_detached_signatures(networkstatus_t *target, + ns_detached_signatures_t *sigs, + const char **msg_out); +char *networkstatus_get_detached_signatures(smartlist_t *consensuses); +void ns_detached_signatures_free(ns_detached_signatures_t *s); + +/* cert manipulation */ +authority_cert_t *authority_cert_dup(authority_cert_t *cert); + +/* vote scheduling */ +void dirvote_get_preferred_voting_intervals(vote_timing_t *timing_out); +time_t dirvote_get_start_of_next_interval(time_t now, int interval); +void dirvote_recalculate_timing(or_options_t *options, time_t now); +void dirvote_act(or_options_t *options, time_t now); + +/* invoked on timers and by outside triggers. */ +struct pending_vote_t * dirvote_add_vote(const char *vote_body, + const char **msg_out, + int *status_out); +int dirvote_add_signatures(const char *detached_signatures_body, + const char *source, + const char **msg_out); + +/* Item access */ +const char *dirvote_get_pending_consensus(consensus_flavor_t flav); +const char *dirvote_get_pending_detached_signatures(void); +#define DGV_BY_ID 1 +#define DGV_INCLUDE_PENDING 2 +#define DGV_INCLUDE_PREVIOUS 4 +const cached_dir_t *dirvote_get_vote(const char *fp, int flags); +void set_routerstatus_from_routerinfo(routerstatus_t *rs, + routerinfo_t *ri, time_t now, + int naming, int listbadexits, + int listbaddirs); +void router_clear_status_flags(routerinfo_t *ri); +networkstatus_t * +dirserv_generate_networkstatus_vote_obj(crypto_pk_env_t *private_key, + authority_cert_t *cert); + +microdesc_t *dirvote_create_microdescriptor(const routerinfo_t *ri); +ssize_t dirvote_format_microdesc_vote_line(char *out, size_t out_len, + const microdesc_t *md); + +int vote_routerstatus_find_microdesc_hash(char *digest256_out, + const vote_routerstatus_t *vrs, + int method, + digest_algorithm_t alg); +document_signature_t *voter_get_sig_by_algorithm( + const networkstatus_voter_info_t *voter, + digest_algorithm_t alg); + +#ifdef DIRVOTE_PRIVATE +char *format_networkstatus_vote(crypto_pk_env_t *private_key, + networkstatus_t *v3_ns); +char *dirvote_compute_params(smartlist_t *votes); +#endif + +#endif + diff --git a/src/or/dns.c b/src/or/dns.c index 3bb9c34892..61c8f32c98 100644 --- a/src/or/dns.c +++ b/src/or/dns.c @@ -12,8 +12,70 @@ **/ #include "or.h" +#include "circuitlist.h" +#include "circuituse.h" +#include "config.h" +#include "connection.h" +#include "connection_edge.h" +#include "control.h" +#include "dns.h" +#include "main.h" +#include "policies.h" +#include "relay.h" +#include "router.h" #include "ht.h" +#ifdef HAVE_EVENT2_DNS_H +#include <event2/event.h> +#include <event2/dns.h> +#else +#include <event.h> #include "eventdns.h" +#ifndef HAVE_EVDNS_SET_DEFAULT_OUTGOING_BIND_ADDRESS +#define HAVE_EVDNS_SET_DEFAULT_OUTGOING_BIND_ADDRESS +#endif +#endif + +#ifndef HAVE_EVENT2_DNS_H +struct evdns_base; +struct evdns_request; +#define evdns_base_new(x,y) tor_malloc(1) +#define evdns_base_clear_nameservers_and_suspend(base) \ + evdns_clear_nameservers_and_suspend() +#define evdns_base_search_clear(base) evdns_search_clear() +#define evdns_base_set_default_outgoing_bind_address(base, a, len) \ + evdns_set_default_outgoing_bind_address((a),(len)) +#define evdns_base_resolv_conf_parse(base, options, fname) \ + evdns_resolv_conf_parse((options), (fname)) +#define evdns_base_count_nameservers(base) \ + evdns_count_nameservers() +#define evdns_base_resume(base) \ + evdns_resume() +#define evdns_base_config_windows_nameservers(base) \ + evdns_config_windows_nameservers() +#define evdns_base_set_option_(base, opt, val) \ + evdns_set_option((opt),(val),DNS_OPTIONS_ALL) +/* Note: our internal eventdns.c, plus Libevent 1.4, used a 1 return to + * signify failure to launch a resolve. Libevent 2.0 uses a -1 return to + * signify a failure on a resolve, though if we're on Libevent 2.0, we should + * have event2/dns.h and never hit these macros. Regardless, 0 is success. */ +#define evdns_base_resolve_ipv4(base, addr, options, cb, ptr) \ + ((evdns_resolve_ipv4((addr), (options), (cb), (ptr))!=0) \ + ? NULL : ((void*)1)) +#define evdns_base_resolve_reverse(base, addr, options, cb, ptr) \ + ((evdns_resolve_reverse((addr), (options), (cb), (ptr))!=0) \ + ? NULL : ((void*)1)) +#define evdns_base_resolve_reverse_ipv6(base, addr, options, cb, ptr) \ + ((evdns_resolve_reverse_ipv6((addr), (options), (cb), (ptr))!=0) \ + ? NULL : ((void*)1)) + +#elif defined(LIBEVENT_VERSION_NUMBER) && LIBEVENT_VERSION_NUMBER < 0x02000303 +#define evdns_base_set_option_(base, opt, val) \ + evdns_base_set_option((base), (opt),(val),DNS_OPTIONS_ALL) + +#else +#define evdns_base_set_option_ evdns_base_set_option + +#endif /** Longest hostname we're willing to resolve. */ #define MAX_ADDRESSLEN 256 @@ -28,6 +90,9 @@ #define DNS_RESOLVE_FAILED_PERMANENT 2 #define DNS_RESOLVE_SUCCEEDED 3 +/** Our evdns_base; this structure handles all our name lookups. */ +static struct evdns_base *the_evdns_base = NULL; + /** Have we currently configured nameservers with eventdns? */ static int nameservers_configured = 0; /** Did our most recent attempt to configure nameservers with eventdns fail? */ @@ -89,6 +154,8 @@ typedef struct cached_resolve_t { uint32_t ttl; /**< What TTL did the nameserver tell us? */ /** Connections that want to know when we get an answer for this resolve. */ pending_connection_t *pending_connections; + /** Position of this element in the heap*/ + int minheap_idx; } cached_resolve_t; static void purge_expired_resolves(time_t now); @@ -211,8 +278,16 @@ dns_reset(void) { or_options_t *options = get_options(); if (! server_mode(options)) { - evdns_clear_nameservers_and_suspend(); - evdns_search_clear(); + + if (!the_evdns_base) { + if (!(the_evdns_base = evdns_base_new(tor_libevent_get_base(), 0))) { + log_err(LD_BUG, "Couldn't create an evdns_base"); + return -1; + } + } + + evdns_base_clear_nameservers_and_suspend(the_evdns_base); + evdns_base_search_clear(the_evdns_base); nameservers_configured = 0; tor_free(resolv_conf_fname); resolv_conf_mtime = 0; @@ -262,6 +337,8 @@ dns_get_expiry_ttl(uint32_t ttl) static void _free_cached_resolve(cached_resolve_t *r) { + if (!r) + return; while (r->pending_connections) { pending_connection_t *victim = r->pending_connections; r->pending_connections = victim->next; @@ -303,6 +380,7 @@ set_expiry(cached_resolve_t *resolve, time_t expires) resolve->expire = expires; smartlist_pqueue_add(cached_resolve_pqueue, _compare_cached_resolves_by_expiry, + STRUCT_OFFSET(cached_resolve_t, minheap_idx), resolve); } @@ -325,8 +403,7 @@ dns_free_all(void) _free_cached_resolve(item); } HT_CLEAR(cache_map, &cache_root); - if (cached_resolve_pqueue) - smartlist_free(cached_resolve_pqueue); + smartlist_free(cached_resolve_pqueue); cached_resolve_pqueue = NULL; tor_free(resolv_conf_fname); } @@ -349,7 +426,8 @@ purge_expired_resolves(time_t now) if (resolve->expire > now) break; smartlist_pqueue_pop(cached_resolve_pqueue, - _compare_cached_resolves_by_expiry); + _compare_cached_resolves_by_expiry, + STRUCT_OFFSET(cached_resolve_t, minheap_idx)); if (resolve->state == CACHE_STATE_PENDING) { log_debug(LD_EXIT, @@ -393,7 +471,7 @@ purge_expired_resolves(time_t now) log_err(LD_BUG, "The expired resolve we purged didn't match any in" " the cache. Tried to purge %s (%p); instead got %s (%p).", resolve->address, (void*)resolve, - removed ? removed->address : "NULL", (void*)remove); + removed ? removed->address : "NULL", (void*)removed); } tor_assert(removed == resolve); } else { @@ -611,7 +689,7 @@ dns_resolve_impl(edge_connection_t *exitconn, int is_resolve, * know the answer. */ if (tor_addr_from_str(&addr, exitconn->_base.address) >= 0) { if (tor_addr_family(&addr) == AF_INET) { - tor_addr_assign(&exitconn->_base.addr, &addr); + tor_addr_copy(&exitconn->_base.addr, &addr); exitconn->address_ttl = DEFAULT_DNS_TTL; return 1; } else { @@ -711,6 +789,7 @@ dns_resolve_impl(edge_connection_t *exitconn, int is_resolve, resolve = tor_malloc_zero(sizeof(cached_resolve_t)); resolve->magic = CACHED_RESOLVE_MAGIC; resolve->state = CACHE_STATE_PENDING; + resolve->minheap_idx = -1; resolve->is_reverse = is_reverse; strlcpy(resolve->address, exitconn->_base.address, sizeof(resolve->address)); @@ -807,7 +886,8 @@ connection_dns_remove(edge_connection_t *conn) tor_free(pend); log_debug(LD_EXIT, "First connection (fd %d) no longer waiting " "for resolve of %s", - conn->_base.s, escaped_safe_str(conn->_base.address)); + conn->_base.s, + escaped_safe_str(conn->_base.address)); return; } else { for ( ; pend->next; pend = pend->next) { @@ -1108,6 +1188,14 @@ configure_nameservers(int force) conf_fname = "/etc/resolv.conf"; #endif + if (!the_evdns_base) { + if (!(the_evdns_base = evdns_base_new(tor_libevent_get_base(), 0))) { + log_err(LD_BUG, "Couldn't create an evdns_base"); + return -1; + } + } + +#ifdef HAVE_EVDNS_SET_DEFAULT_OUTGOING_BIND_ADDRESS if (options->OutboundBindAddress) { tor_addr_t addr; if (tor_addr_from_str(&addr, options->OutboundBindAddress) < 0) { @@ -1118,20 +1206,17 @@ configure_nameservers(int force) struct sockaddr_storage ss; socklen = tor_addr_to_sockaddr(&addr, 0, (struct sockaddr *)&ss, sizeof(ss)); - if (socklen < 0) { + if (socklen <= 0) { log_warn(LD_BUG, "Couldn't convert outbound bind address to sockaddr." " Ignoring."); } else { - evdns_set_default_outgoing_bind_address((struct sockaddr *)&ss, - socklen); + evdns_base_set_default_outgoing_bind_address(the_evdns_base, + (struct sockaddr *)&ss, + socklen); } } } - - if (options->ServerDNSRandomizeCase) - evdns_set_option("randomize-case:", "1", DNS_OPTIONS_ALL); - else - evdns_set_option("randomize-case:", "0", DNS_OPTIONS_ALL); +#endif evdns_set_log_fn(evdns_log_cb); if (conf_fname) { @@ -1146,16 +1231,17 @@ configure_nameservers(int force) return 0; } if (nameservers_configured) { - evdns_search_clear(); - evdns_clear_nameservers_and_suspend(); + evdns_base_search_clear(the_evdns_base); + evdns_base_clear_nameservers_and_suspend(the_evdns_base); } log_info(LD_EXIT, "Parsing resolver configuration in '%s'", conf_fname); - if ((r = evdns_resolv_conf_parse(DNS_OPTIONS_ALL, conf_fname))) { + if ((r = evdns_base_resolv_conf_parse(the_evdns_base, + DNS_OPTIONS_ALL, conf_fname))) { log_warn(LD_EXIT, "Unable to parse '%s', or no nameservers in '%s' (%d)", conf_fname, conf_fname, r); goto err; } - if (evdns_count_nameservers() == 0) { + if (evdns_base_count_nameservers(the_evdns_base) == 0) { log_warn(LD_EXIT, "Unable to find any nameservers in '%s'.", conf_fname); goto err; } @@ -1163,38 +1249,47 @@ configure_nameservers(int force) resolv_conf_fname = tor_strdup(conf_fname); resolv_conf_mtime = st.st_mtime; if (nameservers_configured) - evdns_resume(); + evdns_base_resume(the_evdns_base); } #ifdef MS_WINDOWS else { if (nameservers_configured) { - evdns_search_clear(); - evdns_clear_nameservers_and_suspend(); + evdns_base_search_clear(the_evdns_base); + evdns_base_clear_nameservers_and_suspend(the_evdns_base); } - if (evdns_config_windows_nameservers()) { + if (evdns_base_config_windows_nameservers(the_evdns_base)) { log_warn(LD_EXIT,"Could not config nameservers."); goto err; } - if (evdns_count_nameservers() == 0) { + if (evdns_base_count_nameservers(the_evdns_base) == 0) { log_warn(LD_EXIT, "Unable to find any platform nameservers in " "your Windows configuration."); goto err; } if (nameservers_configured) - evdns_resume(); + evdns_base_resume(the_evdns_base); tor_free(resolv_conf_fname); resolv_conf_mtime = 0; } #endif - if (evdns_count_nameservers() == 1) { - evdns_set_option("max-timeouts:", "16", DNS_OPTIONS_ALL); - evdns_set_option("timeout:", "10", DNS_OPTIONS_ALL); +#define SET(k,v) evdns_base_set_option_(the_evdns_base, (k), (v)) + + if (evdns_base_count_nameservers(the_evdns_base) == 1) { + SET("max-timeouts:", "16"); + SET("timeout:", "10"); } else { - evdns_set_option("max-timeouts:", "3", DNS_OPTIONS_ALL); - evdns_set_option("timeout:", "5", DNS_OPTIONS_ALL); + SET("max-timeouts:", "3"); + SET("timeout:", "5"); } + if (options->ServerDNSRandomizeCase) + SET("randomize-case:", "1"); + else + SET("randomize-case:", "0"); + +#undef SET + dns_servers_relaunch_checks(); nameservers_configured = 1; @@ -1292,6 +1387,7 @@ static int launch_resolve(edge_connection_t *exitconn) { char *addr = tor_strdup(exitconn->_base.address); + struct evdns_request *req = NULL; tor_addr_t a; int r; int options = get_options()->ServerDNSSearchDomains ? 0 @@ -1307,28 +1403,34 @@ launch_resolve(edge_connection_t *exitconn) r = tor_addr_parse_reverse_lookup_name( &a, exitconn->_base.address, AF_UNSPEC, 0); + + tor_assert(the_evdns_base); if (r == 0) { log_info(LD_EXIT, "Launching eventdns request for %s", escaped_safe_str(exitconn->_base.address)); - r = evdns_resolve_ipv4(exitconn->_base.address, options, - evdns_callback, addr); + req = evdns_base_resolve_ipv4(the_evdns_base, + exitconn->_base.address, options, + evdns_callback, addr); } else if (r == 1) { log_info(LD_EXIT, "Launching eventdns reverse request for %s", escaped_safe_str(exitconn->_base.address)); if (tor_addr_family(&a) == AF_INET) - r = evdns_resolve_reverse(tor_addr_to_in(&a), DNS_QUERY_NO_SEARCH, + req = evdns_base_resolve_reverse(the_evdns_base, + tor_addr_to_in(&a), DNS_QUERY_NO_SEARCH, evdns_callback, addr); else - r = evdns_resolve_reverse_ipv6(tor_addr_to_in6(&a), DNS_QUERY_NO_SEARCH, + req = evdns_base_resolve_reverse_ipv6(the_evdns_base, + tor_addr_to_in6(&a), DNS_QUERY_NO_SEARCH, evdns_callback, addr); } else if (r == -1) { log_warn(LD_BUG, "Somehow a malformed in-addr.arpa address reached here."); } - if (r) { - log_warn(LD_EXIT, "eventdns rejected address %s: error %d.", - escaped_safe_str(addr), r); - r = evdns_err_is_transient(r) ? -2 : -1; + r = 0; + if (!req) { + log_warn(LD_EXIT, "eventdns rejected address %s.", + escaped_safe_str(addr)); + r = -1; tor_free(addr); /* There is no evdns request in progress; stop * addr from getting leaked. */ } @@ -1449,8 +1551,8 @@ evdns_wildcard_check_callback(int result, char type, int count, int ttl, } log(dns_wildcard_one_notice_given ? LOG_INFO : LOG_NOTICE, LD_EXIT, "Your DNS provider gave an answer for \"%s\", which " - "is not supposed to exist. Apparently they are hijacking " - "DNS failures. Trying to correct for this. We've noticed %d " + "is not supposed to exist. Apparently they are hijacking " + "DNS failures. Trying to correct for this. We've noticed %d " "possibly bad address%s so far.", string_address, strmap_size(dns_wildcard_response_count), (strmap_size(dns_wildcard_response_count) == 1) ? "" : "es"); @@ -1466,17 +1568,20 @@ static void launch_wildcard_check(int min_len, int max_len, const char *suffix) { char *addr; - int r; + struct evdns_request *req; addr = crypto_random_hostname(min_len, max_len, "", suffix); log_info(LD_EXIT, "Testing whether our DNS server is hijacking nonexistent " "domains with request for bogus hostname \"%s\"", addr); - r = evdns_resolve_ipv4(/* This "addr" tells us which address to resolve */ + tor_assert(the_evdns_base); + req = evdns_base_resolve_ipv4( + the_evdns_base, + /* This "addr" tells us which address to resolve */ addr, DNS_QUERY_NO_SEARCH, evdns_wildcard_check_callback, /* This "addr" is an argument to the callback*/ addr); - if (r) { + if (!req) { /* There is no evdns request in progress; stop addr from getting leaked */ tor_free(addr); } @@ -1488,6 +1593,7 @@ static void launch_test_addresses(int fd, short event, void *args) { or_options_t *options = get_options(); + struct evdns_request *req; (void)fd; (void)event; (void)args; @@ -1499,14 +1605,19 @@ launch_test_addresses(int fd, short event, void *args) * be an exit server.*/ if (!options->ServerDNSTestAddresses) return; - SMARTLIST_FOREACH(options->ServerDNSTestAddresses, const char *, address, - { - int r = evdns_resolve_ipv4(address, DNS_QUERY_NO_SEARCH, evdns_callback, - tor_strdup(address)); - if (r) - log_info(LD_EXIT, "eventdns rejected test address %s: error %d", - escaped_safe_str(address), r); - }); + tor_assert(the_evdns_base); + SMARTLIST_FOREACH_BEGIN(options->ServerDNSTestAddresses, + const char *, address) { + char *a = tor_strdup(address); + req = evdns_base_resolve_ipv4(the_evdns_base, + address, DNS_QUERY_NO_SEARCH, evdns_callback, a); + + if (!req) { + log_info(LD_EXIT, "eventdns rejected test address %s", + escaped_safe_str(address)); + tor_free(a); + } + } SMARTLIST_FOREACH_END(address); } #define N_WILDCARD_CHECKS 2 @@ -1547,7 +1658,7 @@ dns_launch_wildcard_checks(void) void dns_launch_correctness_checks(void) { - static struct event launch_event; + static struct event *launch_event = NULL; struct timeval timeout; if (!get_options()->ServerDNSDetectHijacking) return; @@ -1555,10 +1666,12 @@ dns_launch_correctness_checks(void) /* Wait a while before launching requests for test addresses, so we can * get the results from checking for wildcarding. */ - evtimer_set(&launch_event, launch_test_addresses, NULL); + if (! launch_event) + launch_event = tor_evtimer_new(tor_libevent_get_base(), + launch_test_addresses, NULL); timeout.tv_sec = 30; timeout.tv_usec = 0; - if (evtimer_add(&launch_event, &timeout)<0) { + if (evtimer_add(launch_event, &timeout)<0) { log_warn(LD_BUG, "Couldn't add timer for checking for dns hijacking"); } } @@ -1574,10 +1687,9 @@ dns_seems_to_be_broken(void) void dns_reset_correctness_checks(void) { - if (dns_wildcard_response_count) { - strmap_free(dns_wildcard_response_count, _tor_free); - dns_wildcard_response_count = NULL; - } + strmap_free(dns_wildcard_response_count, _tor_free); + dns_wildcard_response_count = NULL; + n_wildcard_requests = 0; if (dns_wildcard_list) { @@ -1622,6 +1734,30 @@ assert_resolve_ok(cached_resolve_t *resolve) } } +/** Return the number of DNS cache entries as an int */ +static int +dns_cache_entry_count(void) +{ + return HT_SIZE(&cache_root); +} + +/** Log memory information about our internal DNS cache at level 'severity'. */ +void +dump_dns_mem_usage(int severity) +{ + /* This should never be larger than INT_MAX. */ + int hash_count = dns_cache_entry_count(); + size_t hash_mem = sizeof(struct cached_resolve_t) * hash_count; + hash_mem += HT_MEM_USAGE(&cache_root); + + /* Print out the count and estimated size of our &cache_root. It undercounts + hostnames in cached reverse resolves. + */ + log(severity, LD_MM, "Our DNS cache has %d entries.", hash_count); + log(severity, LD_MM, "Our DNS cache size is approximately %u bytes.", + (unsigned)hash_mem); +} + #ifdef DEBUG_DNS_CACHE /** Exit with an assertion if the DNS cache is corrupt. */ static void @@ -1642,7 +1778,8 @@ _assert_cache_ok(void) return; smartlist_pqueue_assert_ok(cached_resolve_pqueue, - _compare_cached_resolves_by_expiry); + _compare_cached_resolves_by_expiry, + STRUCT_OFFSET(cached_resolve_t, minheap_idx)); SMARTLIST_FOREACH(cached_resolve_pqueue, cached_resolve_t *, res, { diff --git a/src/or/dns.h b/src/or/dns.h new file mode 100644 index 0000000000..25ff86e2c6 --- /dev/null +++ b/src/or/dns.h @@ -0,0 +1,31 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file dns.h + * \brief Header file for dns.c. + **/ + +#ifndef _TOR_DNS_H +#define _TOR_DNS_H + +int dns_init(void); +int has_dns_init_failed(void); +void dns_free_all(void); +uint32_t dns_clip_ttl(uint32_t ttl); +int dns_reset(void); +void connection_dns_remove(edge_connection_t *conn); +void assert_connection_edge_not_dns_pending(edge_connection_t *conn); +void assert_all_pending_dns_resolves_ok(void); +void dns_cancel_pending_resolve(const char *question); +int dns_resolve(edge_connection_t *exitconn); +void dns_launch_correctness_checks(void); +int dns_seems_to_be_broken(void); +void dns_reset_correctness_checks(void); +void dump_dns_mem_usage(int severity); + +#endif + diff --git a/src/or/dnsserv.c b/src/or/dnsserv.c index f8b5c3cdd2..243b730cbf 100644 --- a/src/or/dnsserv.c +++ b/src/or/dnsserv.c @@ -9,7 +9,21 @@ **/ #include "or.h" +#include "dnsserv.h" +#include "config.h" +#include "connection.h" +#include "connection_edge.h" +#include "control.h" +#include "main.h" +#include "policies.h" +#ifdef HAVE_EVENT2_DNS_H +#include <event2/dns.h> +#include <event2/dns_compat.h> +/* XXXX023 this implies we want an improved evdns */ +#include <event2/dns_struct.h> +#else #include "eventdns.h" +#endif /** Helper function: called by evdns whenever the client sends a request to our * DNSPort. We need to eventually answer the request <b>req</b>. @@ -85,12 +99,7 @@ evdns_server_callback(struct evdns_server_request *req, void *_data) evdns_server_request_respond(req, DNS_ERR_NONE); return; } - if (q->type == EVDNS_TYPE_A) { - /* Refuse any attempt to resolve a noconnect address, right now. */ - if (hostname_is_noconnect_address(q->name)) { - err = DNS_ERR_REFUSED; - } - } else { + if (q->type != EVDNS_TYPE_A) { tor_assert(q->type == EVDNS_TYPE_PTR); } @@ -132,17 +141,18 @@ evdns_server_callback(struct evdns_server_request *req, void *_data) control_event_stream_status(conn, STREAM_EVENT_NEW, 0); - /* Now, throw the connection over to get rewritten (which will answer it - * immediately if it's in the cache, or completely bogus, or automapped), - * and then attached to a circuit. */ + /* Now, unless a controller asked us to leave streams unattached, + * throw the connection over to get rewritten (which will + * answer it immediately if it's in the cache, or completely bogus, or + * automapped), and then attached to a circuit. */ log_info(LD_APP, "Passing request for %s to rewrite_and_attach.", - escaped_safe_str(q->name)); + escaped_safe_str_client(q->name)); q_name = tor_strdup(q->name); /* q could be freed in rewrite_and_attach */ - connection_ap_handshake_rewrite_and_attach(conn, NULL, NULL); + connection_ap_rewrite_and_attach_if_allowed(conn, NULL, NULL); /* Now, the connection is marked if it was bad. */ - log_info(LD_APP, "Passed request for %s to rewrite_and_attach.", - escaped_safe_str(q_name)); + log_info(LD_APP, "Passed request for %s to rewrite_and_attach_if_allowed.", + escaped_safe_str_client(q_name)); tor_free(q_name); } @@ -177,17 +187,18 @@ dnsserv_launch_request(const char *name, int reverse) return -1; } - /* Now, throw the connection over to get rewritten (which will answer it - * immediately if it's in the cache, or completely bogus, or automapped), - * and then attached to a circuit. */ + /* Now, unless a controller asked us to leave streams unattached, + * throw the connection over to get rewritten (which will + * answer it immediately if it's in the cache, or completely bogus, or + * automapped), and then attached to a circuit. */ log_info(LD_APP, "Passing request for %s to rewrite_and_attach.", - escaped_safe_str(name)); + escaped_safe_str_client(name)); q_name = tor_strdup(name); /* q could be freed in rewrite_and_attach */ - connection_ap_handshake_rewrite_and_attach(conn, NULL, NULL); + connection_ap_rewrite_and_attach_if_allowed(conn, NULL, NULL); /* Now, the connection is marked if it was bad. */ - log_info(LD_APP, "Passed request for %s to rewrite_and_attach.", - escaped_safe_str(q_name)); + log_info(LD_APP, "Passed request for %s to rewrite_and_attach_if_allowed.", + escaped_safe_str_client(q_name)); tor_free(q_name); return 0; } @@ -269,7 +280,7 @@ dnsserv_resolved(edge_connection_t *conn, conn->socks_request->command == SOCKS_COMMAND_RESOLVE) { evdns_server_request_add_a_reply(req, name, - 1, (char*)answer, ttl); + 1, answer, ttl); } else if (answer_type == RESOLVED_TYPE_HOSTNAME && answer_len < 256 && conn->socks_request->command == SOCKS_COMMAND_RESOLVE_PTR) { @@ -298,8 +309,8 @@ dnsserv_configure_listener(connection_t *conn) tor_assert(conn->s >= 0); tor_assert(conn->type == CONN_TYPE_AP_DNS_LISTENER); - conn->dns_server_port = evdns_add_server_port(conn->s, 0, - evdns_server_callback, NULL); + conn->dns_server_port = + tor_evdns_add_server_port(conn->s, 0, evdns_server_callback, NULL); } /** Free the evdns server port for <b>conn</b>, which must be an diff --git a/src/or/dnsserv.h b/src/or/dnsserv.h new file mode 100644 index 0000000000..fcca868885 --- /dev/null +++ b/src/or/dnsserv.h @@ -0,0 +1,26 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file dnsserv.h + * \brief Header file for dnsserv.c. + **/ + +#ifndef _TOR_DNSSERV_H +#define _TOR_DNSSERV_H + +void dnsserv_configure_listener(connection_t *conn); +void dnsserv_close_listener(connection_t *conn); +void dnsserv_resolved(edge_connection_t *conn, + int answer_type, + size_t answer_len, + const char *answer, + int ttl); +void dnsserv_reject_request(edge_connection_t *conn); +int dnsserv_launch_request(const char *name, int is_reverse); + +#endif + diff --git a/src/or/eventdns.c b/src/or/eventdns.c index a889e803ed..cf583d0682 100644 --- a/src/or/eventdns.c +++ b/src/or/eventdns.c @@ -31,6 +31,7 @@ */ #include "eventdns_tor.h" +#include "../common/util.h" #include <sys/types.h> /* #define NDEBUG */ @@ -89,6 +90,7 @@ #include <stdarg.h> #include "eventdns.h" + #ifdef WIN32 #include <windows.h> #include <winsock2.h> @@ -173,8 +175,6 @@ struct evdns_request { /* these objects are kept in a circular list */ struct evdns_request *next, *prev; - u16 timeout_event_deleted; /**< Debugging: where was timeout_event - * deleted? 0 for "it's added." */ struct event timeout_event; u16 trans_id; /* the transaction id */ @@ -214,8 +214,6 @@ struct nameserver { struct event event; /* these objects are kept in a circular list */ struct nameserver *next, *prev; - u16 timeout_event_deleted; /**< Debugging: where was timeout_event - * deleted? 0 for "it's added." */ struct event timeout_event; /* used to keep the timeout for */ /* when we next probe this server. */ /* Valid if state == 0 */ @@ -474,51 +472,10 @@ sockaddr_eq(const struct sockaddr *sa1, const struct sockaddr *sa2, return 1; } -/* for debugging bug 929. XXXX022 */ -static int -_add_timeout_event(u16 *lineno, struct event *ev, struct timeval *to) -{ - *lineno = 0; - return evtimer_add(ev, to); -} -#define add_timeout_event(s, to) \ - (_add_timeout_event(&(s)->timeout_event_deleted, &(s)->timeout_event, (to))) - -/* for debugging bug 929. XXXX022 */ -static int -_del_timeout_event(u16 *lineno, struct event *ev, int line) -{ - if (*lineno) { - log(EVDNS_LOG_DEBUG, - "Duplicate timeout event_del from line %d: first call " - "was at %d.", line, (int)*lineno); - return 0; - } else { - *lineno = (u16)line; - return event_del(ev); - } -} -#define del_timeout_event(s) \ - (_del_timeout_event(&(s)->timeout_event_deleted, &(s)->timeout_event, \ - __LINE__)) -/* For debugging bug 929/957. XXXX022 */ -static int -_del_timeout_event_if_set(u16 *lineno, struct event *ev, int line) -{ - if (*lineno == 0) { - log(EVDNS_LOG_DEBUG, - "Event that I thought was non-added as of line %d " - "was actually added on line %d", - line, (int)*lineno); - *lineno = line; - return event_del(ev); - } - return 0; -} -#define del_timeout_event_if_set(s) \ - _del_timeout_event_if_set(&(s)->timeout_event_deleted, \ - &(s)->timeout_event, \ - __LINE__) +#define add_timeout_event(s, to) \ + (event_add(&(s)->timeout_event, (to))) +#define del_timeout_event(s) \ + (event_del(&(s)->timeout_event)) /* This walks the list of inflight requests to find the */ /* one with a matching transaction id. Returns NULL on */ @@ -555,7 +512,7 @@ static void nameserver_probe_failed(struct nameserver *const ns) { const struct timeval * timeout; del_timeout_event(ns); - CLEAR(&ns->timeout_event); + if (ns->state == 1) { /* This can happen if the nameserver acts in a way which makes us mark */ /* it as bad and then starts sending good replies. */ @@ -567,8 +524,6 @@ nameserver_probe_failed(struct nameserver *const ns) { global_nameserver_timeouts_length - 1)]; ns->failed_times++; - del_timeout_event_if_set(ns); - evtimer_set(&ns->timeout_event, nameserver_prod_callback, ns); if (add_timeout_event(ns, (struct timeval *) timeout) < 0) { log(EVDNS_LOG_WARN, "Error from libevent when adding timer event for %s", @@ -597,8 +552,6 @@ nameserver_failed(struct nameserver *const ns, const char *msg) { ns->state = 0; ns->failed_times = 1; - del_timeout_event_if_set(ns); - evtimer_set(&ns->timeout_event, nameserver_prod_callback, ns); if (add_timeout_event(ns, (struct timeval *) &global_nameserver_timeouts[0]) < 0) { log(EVDNS_LOG_WARN, "Error from libevent when adding timer event for %s", @@ -634,7 +587,6 @@ nameserver_up(struct nameserver *const ns) { log(EVDNS_LOG_WARN, "Nameserver %s is back up", debug_ntop((struct sockaddr *)&ns->address)); del_timeout_event(ns); - CLEAR(&ns->timeout_event); ns->state = 1; ns->failed_times = 0; ns->timedout = 0; @@ -666,7 +618,6 @@ request_finished(struct evdns_request *const req, struct evdns_request **head) { log(EVDNS_LOG_DEBUG, "Removing timeout for request %lx", (unsigned long) req); del_timeout_event(req); - CLEAR(&req->timeout_event); search_request_finished(req); global_requests_inflight--; @@ -1294,7 +1245,8 @@ nameserver_read(struct nameserver *ns) { for (;;) { const int r = - (int)recvfrom(ns->socket, packet, (socklen_t)sizeof(packet), 0, + (int)recvfrom(ns->socket, (void*)packet, + (socklen_t)sizeof(packet), 0, sa, &addrlen); if (r < 0) { int err = last_error(ns->socket); @@ -1325,7 +1277,7 @@ server_port_read(struct evdns_server_port *s) { for (;;) { addrlen = (socklen_t)sizeof(struct sockaddr_storage); - r = recvfrom(s->socket, packet, sizeof(packet), 0, + r = recvfrom(s->socket, (void*)packet, sizeof(packet), 0, (struct sockaddr*) &addr, &addrlen); if (r < 0) { int err = last_error(s->socket); @@ -1342,8 +1294,8 @@ server_port_read(struct evdns_server_port *s) { static void server_port_flush(struct evdns_server_port *port) { - while (port->pending_replies) { - struct server_request *req = port->pending_replies; + struct server_request *req = port->pending_replies; + while (req) { ssize_t r = sendto(port->socket, req->response, req->response_len, 0, (struct sockaddr*) &req->addr, (socklen_t)req->addrlen); if (r < 0) { @@ -1355,6 +1307,9 @@ server_port_flush(struct evdns_server_port *port) if (server_request_free(req)) { /* we released the last reference to req->port. */ return; + } else { + assert(port->pending_replies != req); + req = port->pending_replies; } } @@ -1713,7 +1668,7 @@ evdns_server_request_add_reply(struct evdns_server_request *_req, int section, c /* exported function */ int -evdns_server_request_add_a_reply(struct evdns_server_request *req, const char *name, int n, void *addrs, int ttl) +evdns_server_request_add_a_reply(struct evdns_server_request *req, const char *name, int n, const void *addrs, int ttl) { return evdns_server_request_add_reply( req, EVDNS_ANSWER_SECTION, name, TYPE_A, CLASS_INET, @@ -1722,7 +1677,7 @@ evdns_server_request_add_a_reply(struct evdns_server_request *req, const char *n /* exported function */ int -evdns_server_request_add_aaaa_reply(struct evdns_server_request *req, const char *name, int n, void *addrs, int ttl) +evdns_server_request_add_aaaa_reply(struct evdns_server_request *req, const char *name, int n, const void *addrs, int ttl) { return evdns_server_request_add_reply( req, EVDNS_ANSWER_SECTION, name, TYPE_AAAA, CLASS_INET, @@ -1949,7 +1904,7 @@ server_request_free(struct server_request *req) if (req->port) { if (req->port->pending_replies == req) { - if (req->next_pending) + if (req->next_pending && req->next_pending != req) req->port->pending_replies = req->next_pending; else req->port->pending_replies = NULL; @@ -2044,9 +1999,8 @@ evdns_request_timeout_callback(int fd, short events, void *arg) { /* retransmit it */ /* Stop waiting for the timeout. No need to do this in * request_finished; that one already deletes the timeout event. - * XXXX021 port this change to libevent. */ + * XXXX023 port this change to libevent. */ del_timeout_event(req); - CLEAR(&req->timeout_event); evdns_request_transmit(req); } } @@ -2059,7 +2013,8 @@ evdns_request_timeout_callback(int fd, short events, void *arg) { /* 2 other failure */ static int evdns_request_transmit_to(struct evdns_request *req, struct nameserver *server) { - const ssize_t r = send(server->socket, req->request, req->request_len, 0); + const ssize_t r = send(server->socket, (void*)req->request, + req->request_len, 0); if (r < 0) { int err = last_error(server->socket); if (error_is_eagain(err)) return 1; @@ -2109,8 +2064,7 @@ evdns_request_transmit(struct evdns_request *req) { /* transmitted; we need to check for timeout. */ log(EVDNS_LOG_DEBUG, "Setting timeout for request %lx", (unsigned long) req); - del_timeout_event_if_set(req); - evtimer_set(&req->timeout_event, evdns_request_timeout_callback, req); + if (add_timeout_event(req, &global_timeout) < 0) { log(EVDNS_LOG_WARN, "Error from libevent when adding timer for request %lx", @@ -2225,7 +2179,6 @@ evdns_clear_nameservers_and_suspend(void) (void) event_del(&server->event); CLEAR(&server->event); del_timeout_event(server); - CLEAR(&server->timeout_event); if (server->socket >= 0) CLOSE_SOCKET(server->socket); CLEAR(server); @@ -2243,7 +2196,6 @@ evdns_clear_nameservers_and_suspend(void) req->ns = NULL; /* ???? What to do about searches? */ del_timeout_event(req); - CLEAR(&req->timeout_event); req->trans_id = 0; req->transmit_me = 0; @@ -2292,6 +2244,21 @@ evdns_resume(void) } static int +sockaddr_is_loopback(const struct sockaddr *addr) +{ + static const char LOOPBACK_S6[16] = + "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\1"; + if (addr->sa_family == AF_INET) { + struct sockaddr_in *sin = (struct sockaddr_in *)addr; + return (ntohl(sin->sin_addr.s_addr) & 0xff000000) == 0x7f000000; + } else if (addr->sa_family == AF_INET6) { + struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)addr; + return !memcmp(sin6->sin6_addr.s6_addr, LOOPBACK_S6, 16); + } + return 0; +} + +static int _evdns_nameserver_add_impl(const struct sockaddr *address, socklen_t addrlen) { /* first check to see if we already have this nameserver */ @@ -2318,7 +2285,8 @@ _evdns_nameserver_add_impl(const struct sockaddr *address, if (!ns) return -1; memset(ns, 0, sizeof(struct nameserver)); - ns->timeout_event_deleted = __LINE__; + + evtimer_set(&ns->timeout_event, nameserver_prod_callback, ns); ns->socket = socket(PF_INET, SOCK_DGRAM, 0); if (ns->socket < 0) { err = 1; goto out1; } @@ -2331,7 +2299,8 @@ _evdns_nameserver_add_impl(const struct sockaddr *address, fcntl(ns->socket, F_SETFL, O_NONBLOCK); #endif - if (global_bind_addr_is_set) { + if (global_bind_addr_is_set && + !sockaddr_is_loopback((struct sockaddr*)&global_bind_address)) { if (bind(ns->socket, (struct sockaddr *)&global_bind_address, global_bind_addrlen) < 0) { log(EVDNS_LOG_DEBUG, "Couldn't bind to outgoing address."); @@ -2553,7 +2522,8 @@ request_new(int type, const char *name, int flags, } memset(req, 0, sizeof(struct evdns_request)); - req->timeout_event_deleted = __LINE__; + + evtimer_set(&req->timeout_event, evdns_request_timeout_callback, req); if (global_randomize_case) { unsigned i; @@ -2941,14 +2911,6 @@ evdns_resolv_set_defaults(int flags) { if (flags & DNS_OPTION_NAMESERVERS) evdns_nameserver_ip_add("127.0.0.1"); } -#ifndef HAVE_STRTOK_R -static char * -strtok_r(char *s, const char *delim, char **state) { - (void)state; - return strtok(s, delim); -} -#endif - /* helper version of atoi which returns -1 on error */ static int strtoint(const char *const str) { @@ -3025,9 +2987,9 @@ static void resolv_conf_parse_line(char *const start, int flags) { char *strtok_state; static const char *const delims = " \t"; -#define NEXT_TOKEN strtok_r(NULL, delims, &strtok_state) +#define NEXT_TOKEN tor_strtok_r(NULL, delims, &strtok_state) - char *const first_token = strtok_r(start, delims, &strtok_state); + char *const first_token = tor_strtok_r(start, delims, &strtok_state); if (!first_token) return; if (!strcmp(first_token, "nameserver") && (flags & DNS_OPTION_NAMESERVERS)) { @@ -3171,14 +3133,13 @@ load_nameservers_with_getnetworkparams(void) IP_ADDR_STRING *ns; GetNetworkParams_fn_t fn; - /* XXXX Possibly, we should hardcode the location of this DLL. */ - if (!(handle = LoadLibrary("iphlpapi.dll"))) { + if (!(handle = load_windows_system_library(TEXT("iphlpapi.dll")))) { log(EVDNS_LOG_WARN, "Could not open iphlpapi.dll"); /* right now status = 0, doesn't that mean "good" - mikec */ status = -1; goto done; } - if (!(fn = (GetNetworkParams_fn_t) GetProcAddress(handle, "GetNetworkParams"))) { + if (!(fn = (GetNetworkParams_fn_t) GetProcAddress(handle, TEXT("GetNetworkParams")))) { log(EVDNS_LOG_WARN, "Could not get address of function."); /* same as above */ status = -1; @@ -3241,9 +3202,10 @@ load_nameservers_with_getnetworkparams(void) } static int -config_nameserver_from_reg_key(HKEY key, const char *subkey) +config_nameserver_from_reg_key(HKEY key, const TCHAR *subkey) { char *buf; + char ansibuf[MAX_PATH] = {0}; DWORD bufsz = 0, type = 0; int status = 0; @@ -3255,24 +3217,30 @@ config_nameserver_from_reg_key(HKEY key, const char *subkey) if (RegQueryValueEx(key, subkey, 0, &type, (LPBYTE)buf, &bufsz) == ERROR_SUCCESS && bufsz > 1) { - status = evdns_nameserver_ip_add_line(buf); + wcstombs(ansibuf,(wchar_t*)buf,MAX_PATH);/*XXXX UNICODE */ + status = evdns_nameserver_ip_add_line(ansibuf); } mm_free(buf); return status; } -#define SERVICES_KEY "System\\CurrentControlSet\\Services\\" -#define WIN_NS_9X_KEY SERVICES_KEY "VxD\\MSTCP" -#define WIN_NS_NT_KEY SERVICES_KEY "Tcpip\\Parameters" +#define SERVICES_KEY TEXT("System\\CurrentControlSet\\Services\\") +#define WIN_NS_9X_KEY SERVICES_KEY TEXT("VxD\\MSTCP") +#define WIN_NS_NT_KEY SERVICES_KEY TEXT("Tcpip\\Parameters") static int load_nameservers_from_registry(void) { int found = 0; int r; + OSVERSIONINFO info; + memset(&info, 0, sizeof(info)); + info.dwOSVersionInfoSize = sizeof (info); + GetVersionEx(&info); + #define TRY(k, name) \ - if (!found && config_nameserver_from_reg_key(k,name) == 0) { \ + if (!found && config_nameserver_from_reg_key(k,TEXT(name)) == 0) { \ log(EVDNS_LOG_DEBUG,"Found nameservers in %s/%s",#k,name); \ found = 1; \ } else if (!found) { \ @@ -3280,7 +3248,7 @@ load_nameservers_from_registry(void) #k,#name); \ } - if (((int)GetVersion()) > 0) { /* NT */ + if (info.dwMajorVersion >= 5) { /* NT */ HKEY nt_key = 0, interfaces_key = 0; if (RegOpenKeyEx(HKEY_LOCAL_MACHINE, WIN_NS_NT_KEY, 0, @@ -3288,7 +3256,7 @@ load_nameservers_from_registry(void) log(EVDNS_LOG_DEBUG,"Couldn't open nt key, %d",(int)GetLastError()); return -1; } - r = RegOpenKeyEx(nt_key, "Interfaces", 0, + r = RegOpenKeyEx(nt_key, TEXT("Interfaces"), 0, KEY_QUERY_VALUE|KEY_ENUMERATE_SUB_KEYS, &interfaces_key); if (r != ERROR_SUCCESS) { @@ -3384,8 +3352,7 @@ evdns_shutdown(int fail_requests) if (server->socket >= 0) CLOSE_SOCKET(server->socket); (void) event_del(&server->event); - if (server->state == 0) - del_timeout_event(server); + del_timeout_event(server); CLEAR(server); mm_free(server); if (server_next == server_head) diff --git a/src/or/eventdns.h b/src/or/eventdns.h index bf3b64d08a..2fe4ac9371 100644 --- a/src/or/eventdns.h +++ b/src/or/eventdns.h @@ -323,8 +323,8 @@ struct evdns_server_port *evdns_add_server_port(int socket, int is_tcp, evdns_re void evdns_close_server_port(struct evdns_server_port *port); int evdns_server_request_add_reply(struct evdns_server_request *req, int section, const char *name, int type, int class, int ttl, int datalen, int is_name, const char *data); -int evdns_server_request_add_a_reply(struct evdns_server_request *req, const char *name, int n, void *addrs, int ttl); -int evdns_server_request_add_aaaa_reply(struct evdns_server_request *req, const char *name, int n, void *addrs, int ttl); +int evdns_server_request_add_a_reply(struct evdns_server_request *req, const char *name, int n, const void *addrs, int ttl); +int evdns_server_request_add_aaaa_reply(struct evdns_server_request *req, const char *name, int n, const void *addrs, int ttl); int evdns_server_request_add_ptr_reply(struct evdns_server_request *req, struct in_addr *in, const char *inaddr_name, const char *hostname, int ttl); int evdns_server_request_add_cname_reply(struct evdns_server_request *req, const char *name, const char *cname, int ttl); diff --git a/src/or/geoip.c b/src/or/geoip.c index 360b65f41c..5bb2410a75 100644 --- a/src/or/geoip.c +++ b/src/or/geoip.c @@ -3,15 +3,23 @@ /** * \file geoip.c - * \brief Functions related to maintaining an IP-to-country database and to - * summarizing client connections by country. + * \brief Functions related to maintaining an IP-to-country database; + * to summarizing client connections by country to entry guards, bridges, + * and directory servers; and for statistics on answering network status + * requests. */ #define GEOIP_PRIVATE #include "or.h" #include "ht.h" +#include "config.h" +#include "control.h" +#include "dnsserv.h" +#include "geoip.h" +#include "routerlist.h" static void clear_geoip_db(void); +static void init_geoip_countries(void); /** An entry from the GeoIP file: maps an IP range to a country. */ typedef struct geoip_entry_t { @@ -20,16 +28,11 @@ typedef struct geoip_entry_t { intptr_t country; /**< An index into geoip_countries */ } geoip_entry_t; -/** For how many periods should we remember per-country request history? */ -#define REQUEST_HIST_LEN 3 -/** How long are the periods for which we should remember request history? */ -#define REQUEST_HIST_PERIOD (8*60*60) - /** A per-country record for GeoIP request history. */ typedef struct geoip_country_t { char countrycode[3]; - uint32_t n_v2_ns_requests[REQUEST_HIST_LEN]; - uint32_t n_v3_ns_requests[REQUEST_HIST_LEN]; + uint32_t n_v2_ns_requests; + uint32_t n_v3_ns_requests; } geoip_country_t; /** A list of geoip_country_t */ @@ -42,7 +45,7 @@ static strmap_t *country_idxplus1_by_lc_code = NULL; static smartlist_t *geoip_entries = NULL; /** Return the index of the <b>country</b>'s entry in the GeoIP DB - * if it is a valid 2-letter country code, otherwise return zero. + * if it is a valid 2-letter country code, otherwise return -1. */ country_t geoip_get_country(const char *country) @@ -101,11 +104,11 @@ geoip_parse_entry(const char *line) { unsigned int low, high; char b[3]; - if (!geoip_countries) { - geoip_countries = smartlist_create(); + if (!geoip_countries) + init_geoip_countries(); + if (!geoip_entries) geoip_entries = smartlist_create(); - country_idxplus1_by_lc_code = strmap_new(); - } + while (TOR_ISSPACE(*line)) ++line; if (*line == '#') @@ -142,6 +145,7 @@ _geoip_compare_entries(const void **_a, const void **_b) static int _geoip_compare_key_to_entry(const void *_key, const void **_member) { + /* No alignment issue here, since _key really is a pointer to uint32_t */ const uint32_t addr = *(uint32_t *)_key; const geoip_entry_t *entry = *_member; if (addr < entry->ip_low) @@ -160,6 +164,24 @@ should_record_bridge_info(or_options_t *options) return options->BridgeRelay && options->BridgeRecordUsageByCountry; } +/** Set up a new list of geoip countries with no countries (yet) set in it, + * except for the unknown country. + */ +static void +init_geoip_countries(void) +{ + geoip_country_t *geoip_unresolved; + geoip_countries = smartlist_create(); + /* Add a geoip_country_t for requests that could not be resolved to a + * country as first element (index 0) to geoip_countries. */ + geoip_unresolved = tor_malloc_zero(sizeof(geoip_country_t)); + strlcpy(geoip_unresolved->countrycode, "??", + sizeof(geoip_unresolved->countrycode)); + smartlist_add(geoip_countries, geoip_unresolved); + country_idxplus1_by_lc_code = strmap_new(); + strmap_set_lc(country_idxplus1_by_lc_code, "??", (void*)(1)); +} + /** Clear the GeoIP database and reload it from the file * <b>filename</b>. Return 0 on success, -1 on failure. * @@ -185,16 +207,14 @@ geoip_load_file(const char *filename, or_options_t *options) filename, msg); return -1; } - if (!geoip_countries) { - geoip_countries = smartlist_create(); - country_idxplus1_by_lc_code = strmap_new(); - } + if (!geoip_countries) + init_geoip_countries(); if (geoip_entries) { SMARTLIST_FOREACH(geoip_entries, geoip_entry_t *, e, tor_free(e)); smartlist_free(geoip_entries); } geoip_entries = smartlist_create(); - log_notice(LD_GENERAL, "Parsing GEOIP file."); + log_notice(LD_GENERAL, "Parsing GEOIP file %s.", filename); while (!feof(f)) { char buf[512]; if (fgets(buf, (int)sizeof(buf), f) == NULL) @@ -215,9 +235,10 @@ geoip_load_file(const char *filename, or_options_t *options) } /** Given an IP address in host order, return a number representing the - * country to which that address belongs, or -1 for unknown. The return value - * will always be less than geoip_get_n_countries(). To decode it, - * call geoip_get_country_name(). + * country to which that address belongs, -1 for "No geoip information + * available", or 0 for the 'unknown country'. The return value will always + * be less than geoip_get_n_countries(). To decode it, call + * geoip_get_country_name(). */ int geoip_get_country_by_ip(uint32_t ipaddr) @@ -226,13 +247,15 @@ geoip_get_country_by_ip(uint32_t ipaddr) if (!geoip_entries) return -1; ent = smartlist_bsearch(geoip_entries, &ipaddr, _geoip_compare_key_to_entry); - return ent ? (int)ent->country : -1; + return ent ? (int)ent->country : 0; } /** Return the number of countries recognized by the GeoIP database. */ int geoip_get_n_countries(void) { + if (!geoip_countries) + init_geoip_countries(); return (int) smartlist_len(geoip_countries); } @@ -261,23 +284,22 @@ geoip_is_loaded(void) typedef struct clientmap_entry_t { HT_ENTRY(clientmap_entry_t) node; uint32_t ipaddr; - time_t last_seen; /* The last 2 bits of this value hold the client - * operation. */ + /** Time when we last saw this IP address, in MINUTES since the epoch. + * + * (This will run out of space around 4011 CE. If Tor is still in use around + * 4000 CE, please remember to add more bits to last_seen_in_minutes.) */ + unsigned int last_seen_in_minutes:30; + unsigned int action:2; } clientmap_entry_t; -#define ACTION_MASK 3 +/** Largest allowable value for last_seen_in_minutes. (It's a 30-bit field, + * so it can hold up to (1u<<30)-1, or 0x3fffffffu. + */ +#define MAX_LAST_SEEN_IN_MINUTES 0X3FFFFFFFu /** Map from client IP address to last time seen. */ static HT_HEAD(clientmap, clientmap_entry_t) client_history = HT_INITIALIZER(); -/** Time at which we started tracking client IP history. */ -static time_t client_history_starts = 0; - -/** When did the current period of checking per-country request history - * start? */ -static time_t current_request_period_starts = 0; -/** How many older request periods are we remembering? */ -static int n_old_request_periods = 0; /** Hashtable helper: compute a hash of a clientmap_entry_t. */ static INLINE unsigned @@ -289,7 +311,7 @@ clientmap_entry_hash(const clientmap_entry_t *a) static INLINE int clientmap_entries_eq(const clientmap_entry_t *a, const clientmap_entry_t *b) { - return a->ipaddr == b->ipaddr; + return a->ipaddr == b->ipaddr && a->action == b->action; } HT_PROTOTYPE(clientmap, clientmap_entry_t, node, clientmap_entry_hash, @@ -297,8 +319,87 @@ HT_PROTOTYPE(clientmap, clientmap_entry_t, node, clientmap_entry_hash, HT_GENERATE(clientmap, clientmap_entry_t, node, clientmap_entry_hash, clientmap_entries_eq, 0.6, malloc, realloc, free); +/** Clear history of connecting clients used by entry and bridge stats. */ +static void +client_history_clear(void) +{ + clientmap_entry_t **ent, **next, *this; + for (ent = HT_START(clientmap, &client_history); ent != NULL; + ent = next) { + if ((*ent)->action == GEOIP_CLIENT_CONNECT) { + this = *ent; + next = HT_NEXT_RMV(clientmap, &client_history, ent); + tor_free(this); + } else { + next = HT_NEXT(clientmap, &client_history, ent); + } + } +} + +/** How often do we update our estimate which share of v2 and v3 directory + * requests is sent to us? We could as well trigger updates of shares from + * network status updates, but that means adding a lot of calls into code + * that is independent from geoip stats (and keeping them up-to-date). We + * are perfectly fine with an approximation of 15-minute granularity. */ +#define REQUEST_SHARE_INTERVAL (15 * 60) + +/** When did we last determine which share of v2 and v3 directory requests + * is sent to us? */ +static time_t last_time_determined_shares = 0; + +/** Sum of products of v2 shares times the number of seconds for which we + * consider these shares as valid. */ +static double v2_share_times_seconds; + +/** Sum of products of v3 shares times the number of seconds for which we + * consider these shares as valid. */ +static double v3_share_times_seconds; + +/** Number of seconds we are determining v2 and v3 shares. */ +static int share_seconds; + +/** Try to determine which fraction of v2 and v3 directory requests aimed at + * caches will be sent to us at time <b>now</b> and store that value in + * order to take a mean value later on. */ +static void +geoip_determine_shares(time_t now) +{ + double v2_share = 0.0, v3_share = 0.0; + if (router_get_my_share_of_directory_requests(&v2_share, &v3_share) < 0) + return; + if (last_time_determined_shares) { + v2_share_times_seconds += v2_share * + ((double) (now - last_time_determined_shares)); + v3_share_times_seconds += v3_share * + ((double) (now - last_time_determined_shares)); + share_seconds += (int)(now - last_time_determined_shares); + } + last_time_determined_shares = now; +} + +/** Calculate which fraction of v2 and v3 directory requests aimed at caches + * have been sent to us since the last call of this function up to time + * <b>now</b>. Set *<b>v2_share_out</b> and *<b>v3_share_out</b> to the + * fractions of v2 and v3 protocol shares we expect to have seen. Reset + * counters afterwards. Return 0 on success, -1 on failure (e.g. when zero + * seconds have passed since the last call).*/ +static int +geoip_get_mean_shares(time_t now, double *v2_share_out, + double *v3_share_out) +{ + geoip_determine_shares(now); + if (!share_seconds) + return -1; + *v2_share_out = v2_share_times_seconds / ((double) share_seconds); + *v3_share_out = v3_share_times_seconds / ((double) share_seconds); + v2_share_times_seconds = v3_share_times_seconds = 0.0; + share_seconds = 0; + return 0; +} + /** Note that we've seen a client connect from the IP <b>addr</b> (host order) - * at time <b>now</b>. Ignored by all but bridges. */ + * at time <b>now</b>. Ignored by all but bridges and directories if + * configured accordingly. */ void geoip_note_client_seen(geoip_client_action_t action, uint32_t addr, time_t now) @@ -306,72 +407,46 @@ geoip_note_client_seen(geoip_client_action_t action, or_options_t *options = get_options(); clientmap_entry_t lookup, *ent; if (action == GEOIP_CLIENT_CONNECT) { - if (!(options->BridgeRelay && options->BridgeRecordUsageByCountry)) - return; - /* Did we recently switch from bridge to relay or back? */ - if (client_history_starts > now) + /* Only remember statistics as entry guard or as bridge. */ + if (!options->EntryStatistics && + (!(options->BridgeRelay && options->BridgeRecordUsageByCountry))) return; } else { -#ifndef ENABLE_GEOIP_STATS - return; -#else if (options->BridgeRelay || options->BridgeAuthoritativeDir || - !options->DirRecordUsageByCountry) + !options->DirReqStatistics) return; -#endif } - /* Rotate the current request period. */ - while (current_request_period_starts + REQUEST_HIST_PERIOD < now) { - if (!geoip_countries) - geoip_countries = smartlist_create(); - if (!current_request_period_starts) { - current_request_period_starts = now; - break; - } - SMARTLIST_FOREACH(geoip_countries, geoip_country_t *, c, { - memmove(&c->n_v2_ns_requests[0], &c->n_v2_ns_requests[1], - sizeof(uint32_t)*(REQUEST_HIST_LEN-1)); - memmove(&c->n_v3_ns_requests[0], &c->n_v3_ns_requests[1], - sizeof(uint32_t)*(REQUEST_HIST_LEN-1)); - c->n_v2_ns_requests[REQUEST_HIST_LEN-1] = 0; - c->n_v3_ns_requests[REQUEST_HIST_LEN-1] = 0; - }); - current_request_period_starts += REQUEST_HIST_PERIOD; - if (n_old_request_periods < REQUEST_HIST_LEN-1) - ++n_old_request_periods; - } - - /* We use the low 3 bits of the time to encode the action. Since we're - * potentially remembering tons of clients, we don't want to make - * clientmap_entry_t larger than it has to be. */ - now = (now & ~ACTION_MASK) | (((int)action) & ACTION_MASK); lookup.ipaddr = addr; + lookup.action = (int)action; ent = HT_FIND(clientmap, &client_history, &lookup); - if (ent) { - ent->last_seen = now; - } else { + if (! ent) { ent = tor_malloc_zero(sizeof(clientmap_entry_t)); ent->ipaddr = addr; - ent->last_seen = now; + ent->action = (int)action; HT_INSERT(clientmap, &client_history, ent); } + if (now / 60 <= (int)MAX_LAST_SEEN_IN_MINUTES && now >= 0) + ent->last_seen_in_minutes = (unsigned)(now/60); + else + ent->last_seen_in_minutes = 0; if (action == GEOIP_CLIENT_NETWORKSTATUS || action == GEOIP_CLIENT_NETWORKSTATUS_V2) { int country_idx = geoip_get_country_by_ip(addr); + if (country_idx < 0) + country_idx = 0; /** unresolved requests are stored at index 0. */ if (country_idx >= 0 && country_idx < smartlist_len(geoip_countries)) { geoip_country_t *country = smartlist_get(geoip_countries, country_idx); if (action == GEOIP_CLIENT_NETWORKSTATUS) - ++country->n_v3_ns_requests[REQUEST_HIST_LEN-1]; + ++country->n_v3_ns_requests; else - ++country->n_v2_ns_requests[REQUEST_HIST_LEN-1]; + ++country->n_v2_ns_requests; } - } - if (!client_history_starts) { - client_history_starts = now; - current_request_period_starts = now; + /* Periodically determine share of requests that we should see */ + if (last_time_determined_shares + REQUEST_SHARE_INTERVAL < now) + geoip_determine_shares(now); } } @@ -380,8 +455,8 @@ geoip_note_client_seen(geoip_client_action_t action, static int _remove_old_client_helper(struct clientmap_entry_t *ent, void *_cutoff) { - time_t cutoff = *(time_t*)_cutoff; - if (ent->last_seen < cutoff) { + time_t cutoff = *(time_t*)_cutoff / 60; + if (ent->last_seen_in_minutes < cutoff) { tor_free(ent); return 1; } else { @@ -389,18 +464,45 @@ _remove_old_client_helper(struct clientmap_entry_t *ent, void *_cutoff) } } -/** Forget about all clients that haven't connected since <b>cutoff</b>. - * If <b>cutoff</b> is in the future, clients won't be added to the history - * until this time is reached. This is useful to prevent relays that switch - * to bridges from reporting unbelievable numbers of clients. */ +/** Forget about all clients that haven't connected since <b>cutoff</b>. */ void geoip_remove_old_clients(time_t cutoff) { clientmap_HT_FOREACH_FN(&client_history, _remove_old_client_helper, &cutoff); - if (client_history_starts < cutoff) - client_history_starts = cutoff; +} + +/** How many responses are we giving to clients requesting v2 network + * statuses? */ +static uint32_t ns_v2_responses[GEOIP_NS_RESPONSE_NUM]; + +/** How many responses are we giving to clients requesting v3 network + * statuses? */ +static uint32_t ns_v3_responses[GEOIP_NS_RESPONSE_NUM]; + +/** Note that we've rejected a client's request for a v2 or v3 network + * status, encoded in <b>action</b> for reason <b>reason</b> at time + * <b>now</b>. */ +void +geoip_note_ns_response(geoip_client_action_t action, + geoip_ns_response_t response) +{ + static int arrays_initialized = 0; + if (!get_options()->DirReqStatistics) + return; + if (!arrays_initialized) { + memset(ns_v2_responses, 0, sizeof(ns_v2_responses)); + memset(ns_v3_responses, 0, sizeof(ns_v3_responses)); + arrays_initialized = 1; + } + tor_assert(action == GEOIP_CLIENT_NETWORKSTATUS || + action == GEOIP_CLIENT_NETWORKSTATUS_V2); + tor_assert(response < GEOIP_NS_RESPONSE_NUM); + if (action == GEOIP_CLIENT_NETWORKSTATUS) + ns_v3_responses[response]++; + else + ns_v2_responses[response]++; } /** Do not mention any country from which fewer than this number of IPs have @@ -414,13 +516,6 @@ geoip_remove_old_clients(time_t cutoff) * multiple of this value. */ #define IP_GRANULARITY 8 -/** Return the time at which we started recording geoip data. */ -time_t -geoip_get_history_start(void) -{ - return client_history_starts; -} - /** Helper type: used to sort per-country totals by value. */ typedef struct c_hist_t { char country[3]; /**< Two-letter country code. */ @@ -442,97 +537,320 @@ _c_hist_compare(const void **_a, const void **_b) return strcmp(a->country, b->country); } -/** How long do we have to have observed per-country request history before we - * are willing to talk about it? */ -#define GEOIP_MIN_OBSERVATION_TIME (12*60*60) +/** When there are incomplete directory requests at the end of a 24-hour + * period, consider those requests running for longer than this timeout as + * failed, the others as still running. */ +#define DIRREQ_TIMEOUT (10*60) -/** Return the lowest x such that x is at least <b>number</b>, and x modulo - * <b>divisor</b> == 0. */ -static INLINE unsigned -round_to_next_multiple_of(unsigned number, unsigned divisor) +/** Entry in a map from either conn->global_identifier for direct requests + * or a unique circuit identifier for tunneled requests to request time, + * response size, and completion time of a network status request. Used to + * measure download times of requests to derive average client + * bandwidths. */ +typedef struct dirreq_map_entry_t { + HT_ENTRY(dirreq_map_entry_t) node; + /** Unique identifier for this network status request; this is either the + * conn->global_identifier of the dir conn (direct request) or a new + * locally unique identifier of a circuit (tunneled request). This ID is + * only unique among other direct or tunneled requests, respectively. */ + uint64_t dirreq_id; + unsigned int state:3; /**< State of this directory request. */ + unsigned int type:1; /**< Is this a direct or a tunneled request? */ + unsigned int completed:1; /**< Is this request complete? */ + unsigned int action:2; /**< Is this a v2 or v3 request? */ + /** When did we receive the request and started sending the response? */ + struct timeval request_time; + size_t response_size; /**< What is the size of the response in bytes? */ + struct timeval completion_time; /**< When did the request succeed? */ +} dirreq_map_entry_t; + +/** Map of all directory requests asking for v2 or v3 network statuses in + * the current geoip-stats interval. Values are + * of type *<b>dirreq_map_entry_t</b>. */ +static HT_HEAD(dirreqmap, dirreq_map_entry_t) dirreq_map = + HT_INITIALIZER(); + +static int +dirreq_map_ent_eq(const dirreq_map_entry_t *a, + const dirreq_map_entry_t *b) { - number += divisor - 1; - number -= number % divisor; - return number; + return a->dirreq_id == b->dirreq_id && a->type == b->type; } -/** Return a newly allocated comma-separated string containing entries for all - * the countries from which we've seen enough clients connect. The entry - * format is cc=num where num is the number of IPs we've seen connecting from - * that country, and cc is a lowercased country code. Returns NULL if we don't - * want to export geoip data yet. */ -char * -geoip_get_client_history(time_t now, geoip_client_action_t action) +static unsigned +dirreq_map_ent_hash(const dirreq_map_entry_t *entry) +{ + unsigned u = (unsigned) entry->dirreq_id; + u += entry->type << 20; + return u; +} + +HT_PROTOTYPE(dirreqmap, dirreq_map_entry_t, node, dirreq_map_ent_hash, + dirreq_map_ent_eq); +HT_GENERATE(dirreqmap, dirreq_map_entry_t, node, dirreq_map_ent_hash, + dirreq_map_ent_eq, 0.6, malloc, realloc, free); + +/** Helper: Put <b>entry</b> into map of directory requests using + * <b>type</b> and <b>dirreq_id</b> as key parts. If there is + * already an entry for that key, print out a BUG warning and return. */ +static void +_dirreq_map_put(dirreq_map_entry_t *entry, dirreq_type_t type, + uint64_t dirreq_id) +{ + dirreq_map_entry_t *old_ent; + tor_assert(entry->type == type); + tor_assert(entry->dirreq_id == dirreq_id); + + /* XXXX we could switch this to HT_INSERT some time, since it seems that + * this bug doesn't happen. But since this function doesn't seem to be + * critical-path, it's sane to leave it alone. */ + old_ent = HT_REPLACE(dirreqmap, &dirreq_map, entry); + if (old_ent && old_ent != entry) { + log_warn(LD_BUG, "Error when putting directory request into local " + "map. There was already an entry for the same identifier."); + return; + } +} + +/** Helper: Look up and return an entry in the map of directory requests + * using <b>type</b> and <b>dirreq_id</b> as key parts. If there + * is no such entry, return NULL. */ +static dirreq_map_entry_t * +_dirreq_map_get(dirreq_type_t type, uint64_t dirreq_id) +{ + dirreq_map_entry_t lookup; + lookup.type = type; + lookup.dirreq_id = dirreq_id; + return HT_FIND(dirreqmap, &dirreq_map, &lookup); +} + +/** Note that an either direct or tunneled (see <b>type</b>) directory + * request for a network status with unique ID <b>dirreq_id</b> of size + * <b>response_size</b> and action <b>action</b> (either v2 or v3) has + * started. */ +void +geoip_start_dirreq(uint64_t dirreq_id, size_t response_size, + geoip_client_action_t action, dirreq_type_t type) +{ + dirreq_map_entry_t *ent; + if (!get_options()->DirReqStatistics) + return; + ent = tor_malloc_zero(sizeof(dirreq_map_entry_t)); + ent->dirreq_id = dirreq_id; + tor_gettimeofday(&ent->request_time); + ent->response_size = response_size; + ent->action = action; + ent->type = type; + _dirreq_map_put(ent, type, dirreq_id); +} + +/** Change the state of the either direct or tunneled (see <b>type</b>) + * directory request with <b>dirreq_id</b> to <b>new_state</b> and + * possibly mark it as completed. If no entry can be found for the given + * key parts (e.g., if this is a directory request that we are not + * measuring, or one that was started in the previous measurement period), + * or if the state cannot be advanced to <b>new_state</b>, do nothing. */ +void +geoip_change_dirreq_state(uint64_t dirreq_id, dirreq_type_t type, + dirreq_state_t new_state) +{ + dirreq_map_entry_t *ent; + if (!get_options()->DirReqStatistics) + return; + ent = _dirreq_map_get(type, dirreq_id); + if (!ent) + return; + if (new_state == DIRREQ_IS_FOR_NETWORK_STATUS) + return; + if (new_state - 1 != ent->state) + return; + ent->state = new_state; + if ((type == DIRREQ_DIRECT && + new_state == DIRREQ_FLUSHING_DIR_CONN_FINISHED) || + (type == DIRREQ_TUNNELED && + new_state == DIRREQ_OR_CONN_BUFFER_FLUSHED)) { + tor_gettimeofday(&ent->completion_time); + ent->completed = 1; + } +} + +/** Return a newly allocated comma-separated string containing statistics + * on network status downloads. The string contains the number of completed + * requests, timeouts, and still running requests as well as the download + * times by deciles and quartiles. Return NULL if we have not observed + * requests for long enough. */ +static char * +geoip_get_dirreq_history(geoip_client_action_t action, + dirreq_type_t type) { char *result = NULL; - if (!geoip_is_loaded()) + smartlist_t *dirreq_completed = NULL; + uint32_t complete = 0, timeouts = 0, running = 0; + int bufsize = 1024, written; + dirreq_map_entry_t **ptr, **next, *ent; + struct timeval now; + + tor_gettimeofday(&now); + if (action != GEOIP_CLIENT_NETWORKSTATUS && + action != GEOIP_CLIENT_NETWORKSTATUS_V2) return NULL; - if (client_history_starts < (now - GEOIP_MIN_OBSERVATION_TIME)) { - char buf[32]; - smartlist_t *chunks = NULL; - smartlist_t *entries = NULL; - int n_countries = geoip_get_n_countries(); - int i; - clientmap_entry_t **ent; - unsigned *counts = tor_malloc_zero(sizeof(unsigned)*n_countries); - unsigned total = 0; - unsigned granularity = IP_GRANULARITY; -#ifdef ENABLE_GEOIP_STATS - if (get_options()->DirRecordUsageByCountry) - granularity = get_options()->DirRecordUsageGranularity; -#endif - HT_FOREACH(ent, clientmap, &client_history) { - int country; - if (((*ent)->last_seen & ACTION_MASK) != (int)action) - continue; - country = geoip_get_country_by_ip((*ent)->ipaddr); - if (country < 0) - continue; - tor_assert(0 <= country && country < n_countries); - ++counts[country]; - ++total; - } - /* Don't record anything if we haven't seen enough IPs. */ - if (total < MIN_IPS_TO_NOTE_ANYTHING) - goto done; - /* Make a list of c_hist_t */ - entries = smartlist_create(); - for (i = 0; i < n_countries; ++i) { - unsigned c = counts[i]; - const char *countrycode; - c_hist_t *ent; - /* Only report a country if it has a minimum number of IPs. */ - if (c >= MIN_IPS_TO_NOTE_COUNTRY) { - c = round_to_next_multiple_of(c, granularity); - countrycode = geoip_get_country_name(i); - ent = tor_malloc(sizeof(c_hist_t)); - strlcpy(ent->country, countrycode, sizeof(ent->country)); - ent->total = c; - smartlist_add(entries, ent); + dirreq_completed = smartlist_create(); + for (ptr = HT_START(dirreqmap, &dirreq_map); ptr; ptr = next) { + ent = *ptr; + if (ent->action != action || ent->type != type) { + next = HT_NEXT(dirreqmap, &dirreq_map, ptr); + continue; + } else { + if (ent->completed) { + smartlist_add(dirreq_completed, ent); + complete++; + next = HT_NEXT_RMV(dirreqmap, &dirreq_map, ptr); + } else { + if (tv_mdiff(&ent->request_time, &now) / 1000 > DIRREQ_TIMEOUT) + timeouts++; + else + running++; + next = HT_NEXT_RMV(dirreqmap, &dirreq_map, ptr); + tor_free(ent); } } - /* Sort entries. Note that we must do this _AFTER_ rounding, or else - * the sort order could leak info. */ - smartlist_sort(entries, _c_hist_compare); - - /* Build the result. */ - chunks = smartlist_create(); - SMARTLIST_FOREACH(entries, c_hist_t *, ch, { - tor_snprintf(buf, sizeof(buf), "%s=%u", ch->country, ch->total); - smartlist_add(chunks, tor_strdup(buf)); - }); - result = smartlist_join_strings(chunks, ",", 0, NULL); - done: - tor_free(counts); - if (chunks) { - SMARTLIST_FOREACH(chunks, char *, c, tor_free(c)); - smartlist_free(chunks); - } - if (entries) { - SMARTLIST_FOREACH(entries, c_hist_t *, c, tor_free(c)); - smartlist_free(entries); + } +#define DIR_REQ_GRANULARITY 4 + complete = round_uint32_to_next_multiple_of(complete, + DIR_REQ_GRANULARITY); + timeouts = round_uint32_to_next_multiple_of(timeouts, + DIR_REQ_GRANULARITY); + running = round_uint32_to_next_multiple_of(running, + DIR_REQ_GRANULARITY); + result = tor_malloc_zero(bufsize); + written = tor_snprintf(result, bufsize, "complete=%u,timeout=%u," + "running=%u", complete, timeouts, running); + if (written < 0) { + tor_free(result); + goto done; + } + +#define MIN_DIR_REQ_RESPONSES 16 + if (complete >= MIN_DIR_REQ_RESPONSES) { + uint32_t *dltimes; + /* We may have rounded 'completed' up. Here we want to use the + * real value. */ + complete = smartlist_len(dirreq_completed); + dltimes = tor_malloc_zero(sizeof(uint32_t) * complete); + SMARTLIST_FOREACH_BEGIN(dirreq_completed, dirreq_map_entry_t *, ent) { + uint32_t bytes_per_second; + uint32_t time_diff = (uint32_t) tv_mdiff(&ent->request_time, + &ent->completion_time); + if (time_diff == 0) + time_diff = 1; /* Avoid DIV/0; "instant" answers are impossible + * by law of nature or something, but a milisecond + * is a bit greater than "instantly" */ + bytes_per_second = (uint32_t)(1000 * ent->response_size / time_diff); + dltimes[ent_sl_idx] = bytes_per_second; + } SMARTLIST_FOREACH_END(ent); + median_uint32(dltimes, complete); /* sorts as a side effect. */ + written = tor_snprintf(result + written, bufsize - written, + ",min=%u,d1=%u,d2=%u,q1=%u,d3=%u,d4=%u,md=%u," + "d6=%u,d7=%u,q3=%u,d8=%u,d9=%u,max=%u", + dltimes[0], + dltimes[1*complete/10-1], + dltimes[2*complete/10-1], + dltimes[1*complete/4-1], + dltimes[3*complete/10-1], + dltimes[4*complete/10-1], + dltimes[5*complete/10-1], + dltimes[6*complete/10-1], + dltimes[7*complete/10-1], + dltimes[3*complete/4-1], + dltimes[8*complete/10-1], + dltimes[9*complete/10-1], + dltimes[complete-1]); + if (written<0) + tor_free(result); + tor_free(dltimes); + } + done: + SMARTLIST_FOREACH(dirreq_completed, dirreq_map_entry_t *, ent, + tor_free(ent)); + smartlist_free(dirreq_completed); + return result; +} + +/** Return a newly allocated comma-separated string containing entries for + * all the countries from which we've seen enough clients connect as a + * bridge, directory server, or entry guard. The entry format is cc=num + * where num is the number of IPs we've seen connecting from that country, + * and cc is a lowercased country code. Returns NULL if we don't want + * to export geoip data yet. */ +char * +geoip_get_client_history(geoip_client_action_t action) +{ + char *result = NULL; + unsigned granularity = IP_GRANULARITY; + smartlist_t *chunks = NULL; + smartlist_t *entries = NULL; + int n_countries = geoip_get_n_countries(); + int i; + clientmap_entry_t **ent; + unsigned *counts = NULL; + unsigned total = 0; + + if (!geoip_is_loaded()) + return NULL; + + counts = tor_malloc_zero(sizeof(unsigned)*n_countries); + HT_FOREACH(ent, clientmap, &client_history) { + int country; + if ((*ent)->action != (int)action) + continue; + country = geoip_get_country_by_ip((*ent)->ipaddr); + if (country < 0) + country = 0; /** unresolved requests are stored at index 0. */ + tor_assert(0 <= country && country < n_countries); + ++counts[country]; + ++total; + } + /* Don't record anything if we haven't seen enough IPs. */ + if (total < MIN_IPS_TO_NOTE_ANYTHING) + goto done; + /* Make a list of c_hist_t */ + entries = smartlist_create(); + for (i = 0; i < n_countries; ++i) { + unsigned c = counts[i]; + const char *countrycode; + c_hist_t *ent; + /* Only report a country if it has a minimum number of IPs. */ + if (c >= MIN_IPS_TO_NOTE_COUNTRY) { + c = round_to_next_multiple_of(c, granularity); + countrycode = geoip_get_country_name(i); + ent = tor_malloc(sizeof(c_hist_t)); + strlcpy(ent->country, countrycode, sizeof(ent->country)); + ent->total = c; + smartlist_add(entries, ent); } } + /* Sort entries. Note that we must do this _AFTER_ rounding, or else + * the sort order could leak info. */ + smartlist_sort(entries, _c_hist_compare); + + /* Build the result. */ + chunks = smartlist_create(); + SMARTLIST_FOREACH(entries, c_hist_t *, ch, { + char *buf=NULL; + tor_asprintf(&buf, "%s=%u", ch->country, ch->total); + smartlist_add(chunks, buf); + }); + result = smartlist_join_strings(chunks, ",", 0, NULL); + done: + tor_free(counts); + if (chunks) { + SMARTLIST_FOREACH(chunks, char *, c, tor_free(c)); + smartlist_free(chunks); + } + if (entries) { + SMARTLIST_FOREACH(entries, c_hist_t *, c, tor_free(c)); + smartlist_free(entries); + } return result; } @@ -540,18 +858,12 @@ geoip_get_client_history(time_t now, geoip_client_action_t action) * for <b>action</b> in a format suitable for an extra-info document, or NULL * on failure. */ char * -geoip_get_request_history(time_t now, geoip_client_action_t action) +geoip_get_request_history(geoip_client_action_t action) { smartlist_t *entries, *strings; char *result; unsigned granularity = IP_GRANULARITY; -#ifdef ENABLE_GEOIP_STATS - if (get_options()->DirRecordUsageByCountry) - granularity = get_options()->DirRecordUsageGranularity; -#endif - if (client_history_starts >= (now - GEOIP_MIN_OBSERVATION_TIME)) - return NULL; if (action != GEOIP_CLIENT_NETWORKSTATUS && action != GEOIP_CLIENT_NETWORKSTATUS_V2) return NULL; @@ -560,13 +872,10 @@ geoip_get_request_history(time_t now, geoip_client_action_t action) entries = smartlist_create(); SMARTLIST_FOREACH(geoip_countries, geoip_country_t *, c, { - uint32_t *n = (action == GEOIP_CLIENT_NETWORKSTATUS) - ? c->n_v3_ns_requests : c->n_v2_ns_requests; uint32_t tot = 0; - int i; c_hist_t *ent; - for (i=0; i < REQUEST_HIST_LEN; ++i) - tot += n[i]; + tot = (action == GEOIP_CLIENT_NETWORKSTATUS) ? + c->n_v3_ns_requests : c->n_v2_ns_requests; if (!tot) continue; ent = tor_malloc_zero(sizeof(c_hist_t)); @@ -578,9 +887,9 @@ geoip_get_request_history(time_t now, geoip_client_action_t action) strings = smartlist_create(); SMARTLIST_FOREACH(entries, c_hist_t *, ent, { - char buf[32]; - tor_snprintf(buf, sizeof(buf), "%s=%u", ent->country, ent->total); - smartlist_add(strings, tor_strdup(buf)); + char *buf = NULL; + tor_asprintf(&buf, "%s=%u", ent->country, ent->total); + smartlist_add(strings, buf); }); result = smartlist_join_strings(strings, ",", 0, NULL); SMARTLIST_FOREACH(strings, char *, cp, tor_free(cp)); @@ -590,69 +899,450 @@ geoip_get_request_history(time_t now, geoip_client_action_t action) return result; } -/** Store all our geoip statistics into $DATADIR/geoip-stats. */ +/** Start time of directory request stats or 0 if we're not collecting + * directory request statistics. */ +static time_t start_of_dirreq_stats_interval; + +/** Initialize directory request stats. */ void -dump_geoip_stats(void) +geoip_dirreq_stats_init(time_t now) +{ + start_of_dirreq_stats_interval = now; +} + +/** Stop collecting directory request stats in a way that we can re-start + * doing so in geoip_dirreq_stats_init(). */ +void +geoip_dirreq_stats_term(void) +{ + SMARTLIST_FOREACH(geoip_countries, geoip_country_t *, c, { + c->n_v2_ns_requests = c->n_v3_ns_requests = 0; + }); + { + clientmap_entry_t **ent, **next, *this; + for (ent = HT_START(clientmap, &client_history); ent != NULL; + ent = next) { + if ((*ent)->action == GEOIP_CLIENT_NETWORKSTATUS || + (*ent)->action == GEOIP_CLIENT_NETWORKSTATUS_V2) { + this = *ent; + next = HT_NEXT_RMV(clientmap, &client_history, ent); + tor_free(this); + } else { + next = HT_NEXT(clientmap, &client_history, ent); + } + } + } + v2_share_times_seconds = v3_share_times_seconds = 0.0; + last_time_determined_shares = 0; + share_seconds = 0; + memset(ns_v2_responses, 0, sizeof(ns_v2_responses)); + memset(ns_v3_responses, 0, sizeof(ns_v3_responses)); + { + dirreq_map_entry_t **ent, **next, *this; + for (ent = HT_START(dirreqmap, &dirreq_map); ent != NULL; ent = next) { + this = *ent; + next = HT_NEXT_RMV(dirreqmap, &dirreq_map, ent); + tor_free(this); + } + } + start_of_dirreq_stats_interval = 0; +} + +/** Write dirreq statistics to $DATADIR/stats/dirreq-stats and return when + * we would next want to write. */ +time_t +geoip_dirreq_stats_write(time_t now) { -#ifdef ENABLE_GEOIP_STATS - time_t now = time(NULL); - time_t request_start; - char *filename = get_datadir_fname("geoip-stats"); + char *statsdir = NULL, *filename = NULL; char *data_v2 = NULL, *data_v3 = NULL; - char since[ISO_TIME_LEN+1], written[ISO_TIME_LEN+1]; + char written[ISO_TIME_LEN+1]; open_file_t *open_file = NULL; double v2_share = 0.0, v3_share = 0.0; FILE *out; + int i; - data_v2 = geoip_get_client_history(now, GEOIP_CLIENT_NETWORKSTATUS_V2); - data_v3 = geoip_get_client_history(now, GEOIP_CLIENT_NETWORKSTATUS); - format_iso_time(since, geoip_get_history_start()); + if (!start_of_dirreq_stats_interval) + return 0; /* Not initialized. */ + if (start_of_dirreq_stats_interval + WRITE_STATS_INTERVAL > now) + goto done; /* Not ready to write. */ + + /* Discard all items in the client history that are too old. */ + geoip_remove_old_clients(start_of_dirreq_stats_interval); + + statsdir = get_datadir_fname("stats"); + if (check_private_dir(statsdir, CPD_CREATE) < 0) + goto done; + filename = get_datadir_fname2("stats", "dirreq-stats"); + data_v2 = geoip_get_client_history(GEOIP_CLIENT_NETWORKSTATUS_V2); + data_v3 = geoip_get_client_history(GEOIP_CLIENT_NETWORKSTATUS); format_iso_time(written, now); - out = start_writing_to_stdio_file(filename, OPEN_FLAGS_REPLACE, + out = start_writing_to_stdio_file(filename, OPEN_FLAGS_APPEND, 0600, &open_file); if (!out) goto done; - if (fprintf(out, "written %s\nstarted-at %s\nns-ips %s\nns-v2-ips %s\n", - written, since, + if (fprintf(out, "dirreq-stats-end %s (%d s)\ndirreq-v3-ips %s\n" + "dirreq-v2-ips %s\n", written, + (unsigned) (now - start_of_dirreq_stats_interval), data_v3 ? data_v3 : "", data_v2 ? data_v2 : "") < 0) goto done; tor_free(data_v2); tor_free(data_v3); - request_start = current_request_period_starts - - (n_old_request_periods * REQUEST_HIST_PERIOD); - format_iso_time(since, request_start); - data_v2 = geoip_get_request_history(now, GEOIP_CLIENT_NETWORKSTATUS_V2); - data_v3 = geoip_get_request_history(now, GEOIP_CLIENT_NETWORKSTATUS); - if (fprintf(out, "requests-start %s\nn-ns-reqs %s\nn-v2-ns-reqs %s\n", - since, + data_v2 = geoip_get_request_history(GEOIP_CLIENT_NETWORKSTATUS_V2); + data_v3 = geoip_get_request_history(GEOIP_CLIENT_NETWORKSTATUS); + if (fprintf(out, "dirreq-v3-reqs %s\ndirreq-v2-reqs %s\n", data_v3 ? data_v3 : "", data_v2 ? data_v2 : "") < 0) goto done; - if (!router_get_my_share_of_directory_requests(&v2_share, &v3_share)) { - if (fprintf(out, "v2-ns-share %0.2lf%%\n", v2_share*100) < 0) + tor_free(data_v2); + tor_free(data_v3); + SMARTLIST_FOREACH(geoip_countries, geoip_country_t *, c, { + c->n_v2_ns_requests = c->n_v3_ns_requests = 0; + }); +#define RESPONSE_GRANULARITY 8 + for (i = 0; i < GEOIP_NS_RESPONSE_NUM; i++) { + ns_v2_responses[i] = round_uint32_to_next_multiple_of( + ns_v2_responses[i], RESPONSE_GRANULARITY); + ns_v3_responses[i] = round_uint32_to_next_multiple_of( + ns_v3_responses[i], RESPONSE_GRANULARITY); + } +#undef RESPONSE_GRANULARITY + if (fprintf(out, "dirreq-v3-resp ok=%u,not-enough-sigs=%u,unavailable=%u," + "not-found=%u,not-modified=%u,busy=%u\n", + ns_v3_responses[GEOIP_SUCCESS], + ns_v3_responses[GEOIP_REJECT_NOT_ENOUGH_SIGS], + ns_v3_responses[GEOIP_REJECT_UNAVAILABLE], + ns_v3_responses[GEOIP_REJECT_NOT_FOUND], + ns_v3_responses[GEOIP_REJECT_NOT_MODIFIED], + ns_v3_responses[GEOIP_REJECT_BUSY]) < 0) + goto done; + if (fprintf(out, "dirreq-v2-resp ok=%u,unavailable=%u," + "not-found=%u,not-modified=%u,busy=%u\n", + ns_v2_responses[GEOIP_SUCCESS], + ns_v2_responses[GEOIP_REJECT_UNAVAILABLE], + ns_v2_responses[GEOIP_REJECT_NOT_FOUND], + ns_v2_responses[GEOIP_REJECT_NOT_MODIFIED], + ns_v2_responses[GEOIP_REJECT_BUSY]) < 0) + goto done; + memset(ns_v2_responses, 0, sizeof(ns_v2_responses)); + memset(ns_v3_responses, 0, sizeof(ns_v3_responses)); + if (!geoip_get_mean_shares(now, &v2_share, &v3_share)) { + if (fprintf(out, "dirreq-v2-share %0.2lf%%\n", v2_share*100) < 0) goto done; - if (fprintf(out, "v3-ns-share %0.2lf%%\n", v3_share*100) < 0) + if (fprintf(out, "dirreq-v3-share %0.2lf%%\n", v3_share*100) < 0) goto done; } + data_v2 = geoip_get_dirreq_history(GEOIP_CLIENT_NETWORKSTATUS_V2, + DIRREQ_DIRECT); + data_v3 = geoip_get_dirreq_history(GEOIP_CLIENT_NETWORKSTATUS, + DIRREQ_DIRECT); + if (fprintf(out, "dirreq-v3-direct-dl %s\ndirreq-v2-direct-dl %s\n", + data_v3 ? data_v3 : "", data_v2 ? data_v2 : "") < 0) + goto done; + tor_free(data_v2); + tor_free(data_v3); + data_v2 = geoip_get_dirreq_history(GEOIP_CLIENT_NETWORKSTATUS_V2, + DIRREQ_TUNNELED); + data_v3 = geoip_get_dirreq_history(GEOIP_CLIENT_NETWORKSTATUS, + DIRREQ_TUNNELED); + if (fprintf(out, "dirreq-v3-tunneled-dl %s\ndirreq-v2-tunneled-dl %s\n", + data_v3 ? data_v3 : "", data_v2 ? data_v2 : "") < 0) + goto done; + finish_writing_to_file(open_file); open_file = NULL; + + start_of_dirreq_stats_interval = now; + done: if (open_file) abort_writing_to_file(open_file); tor_free(filename); + tor_free(statsdir); tor_free(data_v2); tor_free(data_v3); -#endif + return start_of_dirreq_stats_interval + WRITE_STATS_INTERVAL; +} + +/** Start time of bridge stats or 0 if we're not collecting bridge + * statistics. */ +static time_t start_of_bridge_stats_interval; + +/** Initialize bridge stats. */ +void +geoip_bridge_stats_init(time_t now) +{ + start_of_bridge_stats_interval = now; +} + +/** Stop collecting bridge stats in a way that we can re-start doing so in + * geoip_bridge_stats_init(). */ +void +geoip_bridge_stats_term(void) +{ + client_history_clear(); + start_of_bridge_stats_interval = 0; +} + +/** Validate a bridge statistics string as it would be written to a + * current extra-info descriptor. Return 1 if the string is valid and + * recent enough, or 0 otherwise. */ +static int +validate_bridge_stats(const char *stats_str, time_t now) +{ + char stats_end_str[ISO_TIME_LEN+1], stats_start_str[ISO_TIME_LEN+1], + *eos; + + const char *BRIDGE_STATS_END = "bridge-stats-end "; + const char *BRIDGE_IPS = "bridge-ips "; + const char *BRIDGE_IPS_EMPTY_LINE = "bridge-ips\n"; + const char *tmp; + time_t stats_end_time; + int seconds; + tor_assert(stats_str); + + /* Parse timestamp and number of seconds from + "bridge-stats-end YYYY-MM-DD HH:MM:SS (N s)" */ + tmp = find_str_at_start_of_line(stats_str, BRIDGE_STATS_END); + if (!tmp) + return 0; + tmp += strlen(BRIDGE_STATS_END); + + if (strlen(tmp) < ISO_TIME_LEN + 6) + return 0; + strlcpy(stats_end_str, tmp, sizeof(stats_end_str)); + if (parse_iso_time(stats_end_str, &stats_end_time) < 0) + return 0; + if (stats_end_time < now - (25*60*60) || + stats_end_time > now + (1*60*60)) + return 0; + seconds = (int)strtol(tmp + ISO_TIME_LEN + 2, &eos, 10); + if (!eos || seconds < 23*60*60) + return 0; + format_iso_time(stats_start_str, stats_end_time - seconds); + + /* Parse: "bridge-ips CC=N,CC=N,..." */ + tmp = find_str_at_start_of_line(stats_str, BRIDGE_IPS); + if (!tmp) { + /* Look if there is an empty "bridge-ips" line */ + tmp = find_str_at_start_of_line(stats_str, BRIDGE_IPS_EMPTY_LINE); + if (!tmp) + return 0; + } + + return 1; +} + +/** Most recent bridge statistics formatted to be written to extra-info + * descriptors. */ +static char *bridge_stats_extrainfo = NULL; + +/** Return a newly allocated string holding our bridge usage stats by country + * in a format suitable for inclusion in an extrainfo document. Return NULL on + * failure. */ +static char * +format_bridge_stats_extrainfo(time_t now) +{ + char *out = NULL, *data = NULL; + long duration = now - start_of_bridge_stats_interval; + char written[ISO_TIME_LEN+1]; + + if (duration < 0) + return NULL; + + format_iso_time(written, now); + data = geoip_get_client_history(GEOIP_CLIENT_CONNECT); + + tor_asprintf(&out, + "bridge-stats-end %s (%ld s)\n" + "bridge-ips %s\n", + written, duration, + data ? data : ""); + tor_free(data); + + return out; +} + +/** Return a newly allocated string holding our bridge usage stats by country + * in a format suitable for the answer to a controller request. Return NULL on + * failure. */ +static char * +format_bridge_stats_controller(time_t now) +{ + char *out = NULL, *data = NULL; + char started[ISO_TIME_LEN+1]; + (void) now; + + format_iso_time(started, start_of_bridge_stats_interval); + data = geoip_get_client_history(GEOIP_CLIENT_CONNECT); + + tor_asprintf(&out, + "TimeStarted=\"%s\" CountrySummary=%s", + started, data ? data : ""); + tor_free(data); + return out; +} + +/** Write bridge statistics to $DATADIR/stats/bridge-stats and return + * when we should next try to write statistics. */ +time_t +geoip_bridge_stats_write(time_t now) +{ + char *filename = NULL, *val = NULL, *statsdir = NULL; + + /* Check if 24 hours have passed since starting measurements. */ + if (now < start_of_bridge_stats_interval + WRITE_STATS_INTERVAL) + return start_of_bridge_stats_interval + WRITE_STATS_INTERVAL; + + /* Discard all items in the client history that are too old. */ + geoip_remove_old_clients(start_of_bridge_stats_interval); + + /* Generate formatted string */ + val = format_bridge_stats_extrainfo(now); + if (val == NULL) + goto done; + + /* Update the stored value. */ + tor_free(bridge_stats_extrainfo); + bridge_stats_extrainfo = val; + start_of_bridge_stats_interval = now; + + /* Write it to disk. */ + statsdir = get_datadir_fname("stats"); + if (check_private_dir(statsdir, CPD_CREATE) < 0) + goto done; + filename = get_datadir_fname2("stats", "bridge-stats"); + + write_str_to_file(filename, bridge_stats_extrainfo, 0); + + /* Tell the controller, "hey, there are clients!" */ + { + char *controller_str = format_bridge_stats_controller(now); + if (controller_str) + control_event_clients_seen(controller_str); + tor_free(controller_str); + } + done: + tor_free(filename); + tor_free(statsdir); + + return start_of_bridge_stats_interval + WRITE_STATS_INTERVAL; +} + +/** Try to load the most recent bridge statistics from disk, unless we + * have finished a measurement interval lately, and check whether they + * are still recent enough. */ +static void +load_bridge_stats(time_t now) +{ + char *fname, *contents; + if (bridge_stats_extrainfo) + return; + + fname = get_datadir_fname2("stats", "bridge-stats"); + contents = read_file_to_str(fname, RFTS_IGNORE_MISSING, NULL); + if (contents && validate_bridge_stats(contents, now)) + bridge_stats_extrainfo = contents; + + tor_free(fname); +} + +/** Return most recent bridge statistics for inclusion in extra-info + * descriptors, or NULL if we don't have recent bridge statistics. */ +const char * +geoip_get_bridge_stats_extrainfo(time_t now) +{ + load_bridge_stats(now); + return bridge_stats_extrainfo; +} + +/** Return a new string containing the recent bridge statistics to be returned + * to controller clients, or NULL if we don't have any bridge statistics. */ +char * +geoip_get_bridge_stats_controller(time_t now) +{ + return format_bridge_stats_controller(now); +} + +/** Start time of entry stats or 0 if we're not collecting entry + * statistics. */ +static time_t start_of_entry_stats_interval; + +/** Initialize entry stats. */ +void +geoip_entry_stats_init(time_t now) +{ + start_of_entry_stats_interval = now; +} + +/** Stop collecting entry stats in a way that we can re-start doing so in + * geoip_entry_stats_init(). */ +void +geoip_entry_stats_term(void) +{ + client_history_clear(); + start_of_entry_stats_interval = 0; +} + +/** Write entry statistics to $DATADIR/stats/entry-stats and return time + * when we would next want to write. */ +time_t +geoip_entry_stats_write(time_t now) +{ + char *statsdir = NULL, *filename = NULL; + char *data = NULL; + char written[ISO_TIME_LEN+1]; + open_file_t *open_file = NULL; + FILE *out; + + if (!start_of_entry_stats_interval) + return 0; /* Not initialized. */ + if (start_of_entry_stats_interval + WRITE_STATS_INTERVAL > now) + goto done; /* Not ready to write. */ + + /* Discard all items in the client history that are too old. */ + geoip_remove_old_clients(start_of_entry_stats_interval); + + statsdir = get_datadir_fname("stats"); + if (check_private_dir(statsdir, CPD_CREATE) < 0) + goto done; + filename = get_datadir_fname2("stats", "entry-stats"); + data = geoip_get_client_history(GEOIP_CLIENT_CONNECT); + format_iso_time(written, now); + out = start_writing_to_stdio_file(filename, OPEN_FLAGS_APPEND, + 0600, &open_file); + if (!out) + goto done; + if (fprintf(out, "entry-stats-end %s (%u s)\nentry-ips %s\n", + written, (unsigned) (now - start_of_entry_stats_interval), + data ? data : "") < 0) + goto done; + + start_of_entry_stats_interval = now; + + finish_writing_to_file(open_file); + open_file = NULL; + done: + if (open_file) + abort_writing_to_file(open_file); + tor_free(filename); + tor_free(statsdir); + tor_free(data); + return start_of_entry_stats_interval + WRITE_STATS_INTERVAL; } /** Helper used to implement GETINFO ip-to-country/... controller command. */ int getinfo_helper_geoip(control_connection_t *control_conn, - const char *question, char **answer) + const char *question, char **answer, + const char **errmsg) { (void)control_conn; - if (geoip_is_loaded() && !strcmpstart(question, "ip-to-country/")) { + if (!geoip_is_loaded()) { + *errmsg = "GeoIP data not loaded"; + return -1; + } + if (!strcmpstart(question, "ip-to-country/")) { int c; uint32_t ip; struct in_addr in; @@ -674,8 +1364,8 @@ clear_geoip_db(void) SMARTLIST_FOREACH(geoip_countries, geoip_country_t *, c, tor_free(c)); smartlist_free(geoip_countries); } - if (country_idxplus1_by_lc_code) - strmap_free(country_idxplus1_by_lc_code, NULL); + + strmap_free(country_idxplus1_by_lc_code, NULL); if (geoip_entries) { SMARTLIST_FOREACH(geoip_entries, geoip_entry_t *, ent, tor_free(ent)); smartlist_free(geoip_entries); @@ -689,13 +1379,24 @@ clear_geoip_db(void) void geoip_free_all(void) { - clientmap_entry_t **ent, **next, *this; - for (ent = HT_START(clientmap, &client_history); ent != NULL; ent = next) { - this = *ent; - next = HT_NEXT_RMV(clientmap, &client_history, ent); - tor_free(this); + { + clientmap_entry_t **ent, **next, *this; + for (ent = HT_START(clientmap, &client_history); ent != NULL; ent = next) { + this = *ent; + next = HT_NEXT_RMV(clientmap, &client_history, ent); + tor_free(this); + } + HT_CLEAR(clientmap, &client_history); + } + { + dirreq_map_entry_t **ent, **next, *this; + for (ent = HT_START(dirreqmap, &dirreq_map); ent != NULL; ent = next) { + this = *ent; + next = HT_NEXT_RMV(dirreqmap, &dirreq_map, ent); + tor_free(this); + } + HT_CLEAR(dirreqmap, &dirreq_map); } - HT_CLEAR(clientmap, &client_history); clear_geoip_db(); } diff --git a/src/or/geoip.h b/src/or/geoip.h new file mode 100644 index 0000000000..24f7c5b931 --- /dev/null +++ b/src/or/geoip.h @@ -0,0 +1,57 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file geoip.h + * \brief Header file for geoip.c. + **/ + +#ifndef _TOR_GEOIP_H +#define _TOR_GEOIP_H + +#ifdef GEOIP_PRIVATE +int geoip_parse_entry(const char *line); +#endif +int should_record_bridge_info(or_options_t *options); +int geoip_load_file(const char *filename, or_options_t *options); +int geoip_get_country_by_ip(uint32_t ipaddr); +int geoip_get_n_countries(void); +const char *geoip_get_country_name(country_t num); +int geoip_is_loaded(void); +country_t geoip_get_country(const char *countrycode); + +void geoip_note_client_seen(geoip_client_action_t action, + uint32_t addr, time_t now); +void geoip_remove_old_clients(time_t cutoff); + +void geoip_note_ns_response(geoip_client_action_t action, + geoip_ns_response_t response); +char *geoip_get_client_history(geoip_client_action_t action); +char *geoip_get_request_history(geoip_client_action_t action); +int getinfo_helper_geoip(control_connection_t *control_conn, + const char *question, char **answer, + const char **errmsg); +void geoip_free_all(void); + +void geoip_start_dirreq(uint64_t dirreq_id, size_t response_size, + geoip_client_action_t action, dirreq_type_t type); +void geoip_change_dirreq_state(uint64_t dirreq_id, dirreq_type_t type, + dirreq_state_t new_state); + +void geoip_dirreq_stats_init(time_t now); +time_t geoip_dirreq_stats_write(time_t now); +void geoip_dirreq_stats_term(void); +void geoip_entry_stats_init(time_t now); +time_t geoip_entry_stats_write(time_t now); +void geoip_entry_stats_term(void); +void geoip_bridge_stats_init(time_t now); +time_t geoip_bridge_stats_write(time_t now); +void geoip_bridge_stats_term(void); +const char *geoip_get_bridge_stats_extrainfo(time_t); +char *geoip_get_bridge_stats_controller(time_t); + +#endif + diff --git a/src/or/hibernate.c b/src/or/hibernate.c index 9ec63756fa..aebce4cc88 100644 --- a/src/or/hibernate.c +++ b/src/or/hibernate.c @@ -22,6 +22,12 @@ hibernating, phase 2: */ #include "or.h" +#include "config.h" +#include "connection.h" +#include "connection_edge.h" +#include "hibernate.h" +#include "main.h" +#include "router.h" /** Possible values of hibernate_state */ typedef enum { @@ -89,6 +95,13 @@ static uint64_t n_bytes_read_in_interval = 0; static uint64_t n_bytes_written_in_interval = 0; /** How many seconds have we been running this interval? */ static uint32_t n_seconds_active_in_interval = 0; +/** How many seconds were we active in this interval before we hit our soft + * limit? */ +static int n_seconds_to_hit_soft_limit = 0; +/** When in this interval was the soft limit hit. */ +static time_t soft_limit_hit_at = 0; +/** How many bytes had we read/written when we hit the soft limit? */ +static uint64_t n_bytes_at_soft_limit = 0; /** When did this accounting interval start? */ static time_t interval_start_time = 0; /** When will this accounting interval end? */ @@ -182,6 +195,9 @@ accounting_parse_options(or_options_t *options, int validate_only) case UNIT_DAY: d = 0; break; + /* Coverity dislikes unreachable default cases; some compilers warn on + * switch statements missing a case. Tell Coverity not to worry. */ + /* coverity[dead_error_begin] */ default: tor_assert(0); } @@ -332,29 +348,58 @@ start_of_accounting_period_after(time_t now) return edge_of_accounting_period_containing(now, 1); } +/** Return the length of the accounting period containing the time + * <b>now</b>. */ +static long +length_of_accounting_period_containing(time_t now) +{ + return edge_of_accounting_period_containing(now, 1) - + edge_of_accounting_period_containing(now, 0); +} + /** Initialize the accounting subsystem. */ void configure_accounting(time_t now) { + time_t s_now; /* Try to remember our recorded usage. */ if (!interval_start_time) read_bandwidth_usage(); /* If we fail, we'll leave values at zero, and * reset below.*/ - if (!interval_start_time || - start_of_accounting_period_after(interval_start_time) <= now) { - /* We didn't have recorded usage, or we don't have recorded usage - * for this interval. Start a new interval. */ + + s_now = start_of_accounting_period_containing(now); + + if (!interval_start_time) { + /* We didn't have recorded usage; Start a new interval. */ log_info(LD_ACCT, "Starting new accounting interval."); reset_accounting(now); - } else if (interval_start_time == - start_of_accounting_period_containing(interval_start_time)) { + } else if (s_now == interval_start_time) { log_info(LD_ACCT, "Continuing accounting interval."); /* We are in the interval we thought we were in. Do nothing.*/ interval_end_time = start_of_accounting_period_after(interval_start_time); } else { - log_warn(LD_ACCT, - "Mismatched accounting interval; starting a fresh one."); - reset_accounting(now); + long duration = + length_of_accounting_period_containing(interval_start_time); + double delta = ((double)(s_now - interval_start_time)) / duration; + if (-0.50 <= delta && delta <= 0.50) { + /* The start of the period is now a little later or earlier than we + * remembered. That's fine; we might lose some bytes we could otherwise + * have written, but better to err on the side of obeying people's + * accounting settings. */ + log_info(LD_ACCT, "Accounting interval moved by %.02f%%; " + "that's fine.", delta*100); + interval_end_time = start_of_accounting_period_after(now); + } else if (delta >= 0.99) { + /* This is the regular time-moved-forward case; don't be too noisy + * about it or people will complain */ + log_info(LD_ACCT, "Accounting interval elapsed; starting a new one"); + reset_accounting(now); + } else { + log_warn(LD_ACCT, + "Mismatched accounting interval: moved by %.02f%%. " + "Starting a fresh one.", delta*100); + reset_accounting(now); + } } accounting_set_wakeup_time(); } @@ -365,23 +410,42 @@ configure_accounting(time_t now) static void update_expected_bandwidth(void) { - uint64_t used, expected; - uint64_t max_configured = (get_options()->BandwidthRate * 60); - - if (n_seconds_active_in_interval < 1800) { + uint64_t expected; + or_options_t *options= get_options(); + uint64_t max_configured = (options->RelayBandwidthRate > 0 ? + options->RelayBandwidthRate : + options->BandwidthRate) * 60; + +#define MIN_TIME_FOR_MEASUREMENT (1800) + + if (soft_limit_hit_at > interval_start_time && n_bytes_at_soft_limit && + (soft_limit_hit_at - interval_start_time) > MIN_TIME_FOR_MEASUREMENT) { + /* If we hit our soft limit last time, only count the bytes up to that + * time. This is a better predictor of our actual bandwidth than + * considering the entirety of the last interval, since we likely started + * using bytes very slowly once we hit our soft limit. */ + expected = n_bytes_at_soft_limit / + (soft_limit_hit_at - interval_start_time); + expected /= 60; + } else if (n_seconds_active_in_interval >= MIN_TIME_FOR_MEASUREMENT) { + /* Otherwise, we either measured enough time in the last interval but + * never hit our soft limit, or we're using a state file from a Tor that + * doesn't know to store soft-limit info. Just take rate at which + * we were reading/writing in the last interval as our expected rate. + */ + uint64_t used = MAX(n_bytes_written_in_interval, + n_bytes_read_in_interval); + expected = used / (n_seconds_active_in_interval / 60); + } else { /* If we haven't gotten enough data last interval, set 'expected' * to 0. This will set our wakeup to the start of the interval. * Next interval, we'll choose our starting time based on how much * we sent this interval. */ expected = 0; - } else { - used = n_bytes_written_in_interval < n_bytes_read_in_interval ? - n_bytes_read_in_interval : n_bytes_written_in_interval; - expected = used / (n_seconds_active_in_interval / 60); - if (expected > max_configured) - expected = max_configured; } + if (expected > max_configured) + expected = max_configured; expected_bandwidth_usage = expected; } @@ -399,6 +463,9 @@ reset_accounting(time_t now) n_bytes_read_in_interval = 0; n_bytes_written_in_interval = 0; n_seconds_active_in_interval = 0; + n_bytes_at_soft_limit = 0; + soft_limit_hit_at = 0; + n_seconds_to_hit_soft_limit = 0; } /** Return true iff we should save our bandwidth usage to disk. */ @@ -449,35 +516,39 @@ accounting_run_housekeeping(time_t now) static void accounting_set_wakeup_time(void) { - char buf[ISO_TIME_LEN+1]; char digest[DIGEST_LEN]; crypto_digest_env_t *d_env; int time_in_interval; uint64_t time_to_exhaust_bw; int time_to_consider; - if (! identity_key_is_set()) { + if (! server_identity_key_is_set()) { if (init_keys() < 0) { log_err(LD_BUG, "Error initializing keys"); tor_assert(0); } } - format_iso_time(buf, interval_start_time); - crypto_pk_get_digest(get_identity_key(), digest); + if (server_identity_key_is_set()) { + char buf[ISO_TIME_LEN+1]; + format_iso_time(buf, interval_start_time); - d_env = crypto_new_digest_env(); - crypto_digest_add_bytes(d_env, buf, ISO_TIME_LEN); - crypto_digest_add_bytes(d_env, digest, DIGEST_LEN); - crypto_digest_get_digest(d_env, digest, DIGEST_LEN); - crypto_free_digest_env(d_env); + crypto_pk_get_digest(get_server_identity_key(), digest); + + d_env = crypto_new_digest_env(); + crypto_digest_add_bytes(d_env, buf, ISO_TIME_LEN); + crypto_digest_add_bytes(d_env, digest, DIGEST_LEN); + crypto_digest_get_digest(d_env, digest, DIGEST_LEN); + crypto_free_digest_env(d_env); + } else { + crypto_rand(digest, DIGEST_LEN); + } if (!expected_bandwidth_usage) { char buf1[ISO_TIME_LEN+1]; char buf2[ISO_TIME_LEN+1]; format_local_iso_time(buf1, interval_start_time); format_local_iso_time(buf2, interval_end_time); - time_to_exhaust_bw = GUESS_TIME_TO_USE_BANDWIDTH; interval_wakeup_time = interval_start_time; log_notice(LD_ACCT, @@ -492,8 +563,8 @@ accounting_set_wakeup_time(void) time_to_exhaust_bw = (get_options()->AccountingMax/expected_bandwidth_usage)*60; - if (time_to_exhaust_bw > TIME_MAX) { - time_to_exhaust_bw = TIME_MAX; + if (time_to_exhaust_bw > INT_MAX) { + time_to_exhaust_bw = INT_MAX; time_to_consider = 0; } else { time_to_consider = time_in_interval - (int)time_to_exhaust_bw; @@ -511,8 +582,6 @@ accounting_set_wakeup_time(void) * to be chosen than the last half. */ interval_wakeup_time = interval_start_time + (get_uint32(digest) % time_to_consider); - - format_iso_time(buf, interval_wakeup_time); } { @@ -559,6 +628,10 @@ accounting_record_bandwidth_usage(time_t now, or_state_t *state) state->AccountingSecondsActive = n_seconds_active_in_interval; state->AccountingExpectedUsage = expected_bandwidth_usage; + state->AccountingSecondsToReachSoftLimit = n_seconds_to_hit_soft_limit; + state->AccountingSoftLimitHitAt = soft_limit_hit_at; + state->AccountingBytesAtSoftLimit = n_bytes_at_soft_limit; + or_state_mark_dirty(state, now+(get_options()->AvoidDiskWrites ? 7200 : 60)); @@ -582,10 +655,6 @@ read_bandwidth_usage(void) if (!state) return -1; - /* Okay; it looks like the state file is more up-to-date than the - * bw_accounting file, or the bw_accounting file is nonexistent, - * or the bw_accounting file is corrupt. - */ log_info(LD_ACCT, "Reading bandwidth accounting data from state file"); n_bytes_read_in_interval = state->AccountingBytesReadInInterval; n_bytes_written_in_interval = state->AccountingBytesWrittenInInterval; @@ -593,6 +662,21 @@ read_bandwidth_usage(void) interval_start_time = state->AccountingIntervalStart; expected_bandwidth_usage = state->AccountingExpectedUsage; + /* Older versions of Tor (before 0.2.2.17-alpha or so) didn't generate these + * fields. If you switch back and forth, you might get an + * AccountingSoftLimitHitAt value from long before the most recent + * interval_start_time. If that's so, then ignore the softlimit-related + * values. */ + if (state->AccountingSoftLimitHitAt > interval_start_time) { + soft_limit_hit_at = state->AccountingSoftLimitHitAt; + n_bytes_at_soft_limit = state->AccountingBytesAtSoftLimit; + n_seconds_to_hit_soft_limit = state->AccountingSecondsToReachSoftLimit; + } else { + soft_limit_hit_at = 0; + n_bytes_at_soft_limit = 0; + n_seconds_to_hit_soft_limit = 0; + } + { char tbuf1[ISO_TIME_LEN+1]; char tbuf2[ISO_TIME_LEN+1]; @@ -632,8 +716,27 @@ hibernate_hard_limit_reached(void) static int hibernate_soft_limit_reached(void) { - uint64_t soft_limit = DBL_TO_U64(U64_TO_DBL(get_options()->AccountingMax) - * .95); + const uint64_t acct_max = get_options()->AccountingMax; +#define SOFT_LIM_PCT (.95) +#define SOFT_LIM_BYTES (500*1024*1024) +#define SOFT_LIM_MINUTES (3*60) + /* The 'soft limit' is a fair bit more complicated now than once it was. + * We want to stop accepting connections when ALL of the following are true: + * - We expect to use up the remaining bytes in under 3 hours + * - We have used up 95% of our bytes. + * - We have less than 500MB of bytes left. + */ + uint64_t soft_limit = DBL_TO_U64(U64_TO_DBL(acct_max) * SOFT_LIM_PCT); + if (acct_max > SOFT_LIM_BYTES && acct_max - SOFT_LIM_BYTES > soft_limit) { + soft_limit = acct_max - SOFT_LIM_BYTES; + } + if (expected_bandwidth_usage) { + const uint64_t expected_usage = + expected_bandwidth_usage * SOFT_LIM_MINUTES; + if (acct_max > expected_usage && acct_max - expected_usage > soft_limit) + soft_limit = acct_max - expected_usage; + } + if (!soft_limit) return 0; return n_bytes_read_in_interval >= soft_limit @@ -658,6 +761,14 @@ hibernate_begin(hibernate_state_t new_state, time_t now) exit(0); } + if (new_state == HIBERNATE_STATE_LOWBANDWIDTH && + hibernate_state == HIBERNATE_STATE_LIVE) { + soft_limit_hit_at = now; + n_seconds_to_hit_soft_limit = n_seconds_active_in_interval; + n_bytes_at_soft_limit = MAX(n_bytes_read_in_interval, + n_bytes_written_in_interval); + } + /* close listeners. leave control listener(s). */ while ((conn = connection_get_by_type(CONN_TYPE_OR_LISTENER)) || (conn = connection_get_by_type(CONN_TYPE_AP_LISTENER)) || @@ -673,7 +784,8 @@ hibernate_begin(hibernate_state_t new_state, time_t now) /* XXX upload rendezvous service descriptors with no intro points */ if (new_state == HIBERNATE_STATE_EXITING) { - log_notice(LD_GENERAL,"Interrupt: will shut down in %d seconds. Interrupt " + log_notice(LD_GENERAL,"Interrupt: we have stopped accepting new " + "connections, and will shut down in %d seconds. Interrupt " "again to exit now.", options->ShutdownWaitLength); shutdown_time = time(NULL) + options->ShutdownWaitLength; } else { /* soft limit reached */ @@ -830,7 +942,8 @@ consider_hibernation(time_t now) if (hibernate_state == HIBERNATE_STATE_LIVE) { if (hibernate_soft_limit_reached()) { log_notice(LD_ACCT, - "Bandwidth soft limit reached; commencing hibernation."); + "Bandwidth soft limit reached; commencing hibernation. " + "No new conncetions will be accepted"); hibernate_begin(HIBERNATE_STATE_LOWBANDWIDTH, now); } else if (accounting_enabled && now < interval_wakeup_time) { format_local_iso_time(buf,interval_wakeup_time); @@ -860,9 +973,11 @@ consider_hibernation(time_t now) * NULL. */ int getinfo_helper_accounting(control_connection_t *conn, - const char *question, char **answer) + const char *question, char **answer, + const char **errmsg) { (void) conn; + (void) errmsg; if (!strcmp(question, "accounting/enabled")) { *answer = tor_strdup(accounting_is_enabled(get_options()) ? "1" : "0"); } else if (!strcmp(question, "accounting/hibernating")) { diff --git a/src/or/hibernate.h b/src/or/hibernate.h new file mode 100644 index 0000000000..2aea0fab0c --- /dev/null +++ b/src/or/hibernate.h @@ -0,0 +1,29 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file hibernate.h + * \brief Header file for hibernate.c. + **/ + +#ifndef _TOR_HIBERNATE_H +#define _TOR_HIBERNATE_H + +int accounting_parse_options(or_options_t *options, int validate_only); +int accounting_is_enabled(or_options_t *options); +void configure_accounting(time_t now); +void accounting_run_housekeeping(time_t now); +void accounting_add_bytes(size_t n_read, size_t n_written, int seconds); +int accounting_record_bandwidth_usage(time_t now, or_state_t *state); +void hibernate_begin_shutdown(void); +int we_are_hibernating(void); +void consider_hibernation(time_t now); +int getinfo_helper_accounting(control_connection_t *conn, + const char *question, char **answer, + const char **errmsg); + +#endif + diff --git a/src/or/main.c b/src/or/main.c index 96b7b456ea..a26be39fdf 100644 --- a/src/or/main.c +++ b/src/or/main.c @@ -12,12 +12,50 @@ #define MAIN_PRIVATE #include "or.h" +#include "buffers.h" +#include "circuitbuild.h" +#include "circuitlist.h" +#include "circuituse.h" +#include "command.h" +#include "config.h" +#include "connection.h" +#include "connection_edge.h" +#include "connection_or.h" +#include "control.h" +#include "cpuworker.h" +#include "directory.h" +#include "dirserv.h" +#include "dirvote.h" +#include "dns.h" +#include "dnsserv.h" +#include "geoip.h" +#include "hibernate.h" +#include "main.h" +#include "microdesc.h" +#include "networkstatus.h" +#include "ntmain.h" +#include "onion.h" +#include "policies.h" +#include "relay.h" +#include "rendclient.h" +#include "rendcommon.h" +#include "rendservice.h" +#include "rephist.h" +#include "router.h" +#include "routerlist.h" +#include "routerparse.h" #ifdef USE_DMALLOC #include <dmalloc.h> #include <openssl/crypto.h> #endif #include "memarea.h" +#ifdef HAVE_EVENT2_EVENT_H +#include <event2/event.h> +#else +#include <event.h> +#endif + void evdns_shutdown(int); /********* PROTOTYPES **********/ @@ -26,8 +64,7 @@ static void dumpmemusage(int severity); static void dumpstats(int severity); /* log stats */ static void conn_read_callback(int fd, short event, void *_conn); static void conn_write_callback(int fd, short event, void *_conn); -static void signal_callback(int fd, short events, void *arg); -static void second_elapsed_callback(int fd, short event, void *args); +static void second_elapsed_callback(periodic_timer_t *timer, void *args); static int conn_close_if_marked(int i); static void connection_start_reading_from_linked_conn(connection_t *conn); static int connection_should_read_from_linked_conn(connection_t *conn); @@ -81,8 +118,12 @@ static smartlist_t *active_linked_connection_lst = NULL; static int called_loop_once = 0; /** We set this to 1 when we've opened a circuit, so we can print a log - * entry to inform the user that Tor is working. */ -int has_completed_circuit=0; + * entry to inform the user that Tor is working. We set it to 0 when + * we think the fact that we once opened a circuit doesn't mean we can do so + * any longer (a big time jump happened, when we notice our directory is + * heinously out-of-date, etc. + */ +int can_complete_circuit=0; /** How often do we check for router descriptors that we should download * when we have too little directory info? */ @@ -127,12 +168,10 @@ connection_add(connection_t *conn) smartlist_add(connection_array, conn); if (conn->s >= 0 || conn->linked) { - conn->read_event = tor_malloc_zero(sizeof(struct event)); - conn->write_event = tor_malloc_zero(sizeof(struct event)); - event_set(conn->read_event, conn->s, EV_READ|EV_PERSIST, - conn_read_callback, conn); - event_set(conn->write_event, conn->s, EV_WRITE|EV_PERSIST, - conn_write_callback, conn); + conn->read_event = tor_event_new(tor_libevent_get_base(), + conn->s, EV_READ|EV_PERSIST, conn_read_callback, conn); + conn->write_event = tor_event_new(tor_libevent_get_base(), + conn->s, EV_WRITE|EV_PERSIST, conn_write_callback, conn); } log_debug(LD_NET,"new conn type %s, socket %d, address %s, n_conns %d.", @@ -142,6 +181,25 @@ connection_add(connection_t *conn) return 0; } +/** Tell libevent that we don't care about <b>conn</b> any more. */ +void +connection_unregister_events(connection_t *conn) +{ + if (conn->read_event) { + if (event_del(conn->read_event)) + log_warn(LD_BUG, "Error removing read event for %d", conn->s); + tor_free(conn->read_event); + } + if (conn->write_event) { + if (event_del(conn->write_event)) + log_warn(LD_BUG, "Error removing write event for %d", conn->s); + tor_free(conn->write_event); + } + if (conn->dns_server_port) { + dnsserv_close_listener(conn); + } +} + /** Remove the connection from the global list, and remove the * corresponding poll entry. Calling this function will shift the last * connection (if any) into the position occupied by conn. @@ -246,17 +304,17 @@ get_connection_array(void) } /** Set the event mask on <b>conn</b> to <b>events</b>. (The event - * mask is a bitmask whose bits are EV_READ and EV_WRITE.) + * mask is a bitmask whose bits are READ_EVENT and WRITE_EVENT) */ void -connection_watch_events(connection_t *conn, short events) +connection_watch_events(connection_t *conn, watchable_events_t events) { - if (events & EV_READ) + if (events & READ_EVENT) connection_start_reading(conn); else connection_stop_reading(conn); - if (events & EV_WRITE) + if (events & WRITE_EVENT) connection_start_writing(conn); else connection_stop_writing(conn); @@ -393,11 +451,11 @@ connection_start_reading_from_linked_conn(connection_t *conn) smartlist_add(active_linked_connection_lst, conn); if (!called_loop_once) { /* This is the first event on the list; we won't be in LOOP_ONCE mode, - * so we need to make sure that the event_loop() actually exits at the - * end of its run through the current connections and - * lets us activate read events for linked connections. */ + * so we need to make sure that the event_base_loop() actually exits at + * the end of its run through the current connections and lets us + * activate read events for linked connections. */ struct timeval tv = { 0, 0 }; - event_loopexit(&tv); + tor_event_base_loopexit(tor_libevent_get_base(), &tv); } } else { tor_assert(smartlist_isin(active_linked_connection_lst, conn)); @@ -540,7 +598,7 @@ conn_close_if_marked(int i) log_info(LD_NET, "Conn (addr %s, fd %d, type %s, state %d) marked, but wants " "to flush %d bytes. (Marked at %s:%d)", - escaped_safe_str(conn->address), + escaped_safe_str_client(conn->address), conn->s, conn_type_to_string(conn->type), conn->state, (int)conn->outbuf_flushlen, conn->marked_for_close_file, conn->marked_for_close); @@ -593,8 +651,8 @@ conn_close_if_marked(int i) "something is wrong with theirs. " "(fd %d, type %s, state %d, marked at %s:%d).", (int)buf_datalen(conn->outbuf), - escaped_safe_str(conn->address), conn->s, - conn_type_to_string(conn->type), conn->state, + escaped_safe_str_client(conn->address), + conn->s, conn_type_to_string(conn->type), conn->state, conn->marked_for_close_file, conn->marked_for_close); } @@ -623,7 +681,7 @@ directory_all_unreachable(time_t now) log_notice(LD_NET, "Is your network connection down? " "Failing connection to '%s:%d'.", - safe_str(edge_conn->socks_request->address), + safe_str_client(edge_conn->socks_request->address), edge_conn->socks_request->port); connection_mark_unattached_ap(edge_conn, END_STREAM_REASON_NET_UNREACHABLE); @@ -651,7 +709,7 @@ directory_info_has_arrived(time_t now, int from_cache) /* if we have enough dir info, then update our guard status with * whatever we just learned. */ - entry_guards_compute_status(); + entry_guards_compute_status(options, now); /* Don't even bother trying to get extrainfo until the rest of our * directory info is up-to-date */ if (options->DownloadExtraInfo) @@ -659,7 +717,7 @@ directory_info_has_arrived(time_t now, int from_cache) } if (server_mode(options) && !we_are_hibernating() && !from_cache && - (has_completed_circuit || !any_predicted_circuits(now))) + (can_complete_circuit || !any_predicted_circuits(now))) consider_testing_reachability(1, 1); } @@ -722,6 +780,7 @@ run_connection_housekeeping(int i, time_t now) the connection or send a keepalive, depending. */ or_conn = TO_OR_CONN(conn); + tor_assert(conn->outbuf); if (or_conn->is_bad_for_new_circs && !or_conn->n_circuits) { /* It's bad for new circuits, and has no unmarked circuits on it: @@ -778,13 +837,22 @@ run_connection_housekeeping(int i, time_t now) } } -/** Honor a NEWNYM request: make future requests unlinkability to past +/** Honor a NEWNYM request: make future requests unlinkable to past * requests. */ static void signewnym_impl(time_t now) { + or_options_t *options = get_options(); + if (!proxy_mode(options)) { + log_info(LD_CONTROL, "Ignoring SIGNAL NEWNYM because client functionality " + "is disabled."); + return; + } + circuit_expire_all_dirty_circs(); addressmap_clear_transient(); + rend_cache_purge(); + rend_client_cancel_descriptor_fetches(); time_of_last_signewnym = now; signewnym_is_pending = 0; } @@ -804,16 +872,19 @@ run_scheduled_events(time_t now) static time_t time_to_try_getting_descriptors = 0; static time_t time_to_reset_descriptor_failures = 0; static time_t time_to_add_entropy = 0; - static time_t time_to_write_hs_statistics = 0; static time_t time_to_write_bridge_status_file = 0; static time_t time_to_downrate_stability = 0; static time_t time_to_save_stability = 0; static time_t time_to_clean_caches = 0; static time_t time_to_recheck_bandwidth = 0; static time_t time_to_check_for_expired_networkstatus = 0; - static time_t time_to_dump_geoip_stats = 0; + static time_t time_to_write_stats_files = 0; + static time_t time_to_write_bridge_stats = 0; + static time_t time_to_launch_reachability_tests = 0; + static int should_init_bridge_stats = 1; static time_t time_to_retry_dns_init = 0; or_options_t *options = get_options(); + int is_server = server_mode(options); int i; int have_dir_info; @@ -831,11 +902,14 @@ run_scheduled_events(time_t now) signewnym_impl(now); } + /* 0c. If we've deferred log messages for the controller, handle them now */ + flush_pending_log_callbacks(); + /** 1a. Every MIN_ONION_KEY_LIFETIME seconds, rotate the onion keys, * shut down and restart all cpuworkers, and update the directory if * necessary. */ - if (server_mode(options) && + if (is_server && get_onion_key_set_at()+MIN_ONION_KEY_LIFETIME < now) { log_info(LD_GENERAL,"Rotating onion key."); rotate_onion_key(); @@ -851,7 +925,7 @@ run_scheduled_events(time_t now) update_router_descriptor_downloads(now); update_extrainfo_downloads(now); if (options->UseBridges) - fetch_bridge_descriptors(now); + fetch_bridge_descriptors(options, now); if (router_have_minimum_dir_info()) time_to_try_getting_descriptors = now + LAZY_DESCRIPTOR_RETRY_INTERVAL; else @@ -869,7 +943,10 @@ run_scheduled_events(time_t now) last_rotated_x509_certificate = now; if (last_rotated_x509_certificate+MAX_SSL_KEY_LIFETIME < now) { log_info(LD_GENERAL,"Rotating tls context."); - if (tor_tls_context_new(get_identity_key(), MAX_SSL_KEY_LIFETIME) < 0) { + if (tor_tls_context_init(public_server_mode(options), + get_tlsclient_identity_key(), + is_server ? get_server_identity_key() : NULL, + MAX_SSL_KEY_LIFETIME) < 0) { log_warn(LD_BUG, "Error reinitializing TLS context"); /* XXX is it a bug here, that we just keep going? -RD */ } @@ -894,10 +971,12 @@ run_scheduled_events(time_t now) if (accounting_is_enabled(options)) accounting_run_housekeeping(now); - if (now % 10 == 0 && (authdir_mode_tests_reachability(options)) && - !we_are_hibernating()) { + if (time_to_launch_reachability_tests < now && + (authdir_mode_tests_reachability(options)) && + !we_are_hibernating()) { + time_to_launch_reachability_tests = now + REACHABILITY_TEST_INTERVAL; /* try to determine reachability of the other Tor relays */ - dirserv_test_reachability(now, 0); + dirserv_test_reachability(now); } /** 1d. Periodically, we discount older stability information so that new @@ -939,11 +1018,56 @@ run_scheduled_events(time_t now) time_to_check_for_expired_networkstatus = now + CHECK_EXPIRED_NS_INTERVAL; } - if (time_to_dump_geoip_stats < now) { -#define DUMP_GEOIP_STATS_INTERVAL (60*60); - if (time_to_dump_geoip_stats) - dump_geoip_stats(); - time_to_dump_geoip_stats = now + DUMP_GEOIP_STATS_INTERVAL; + /* 1g. Check whether we should write statistics to disk. + */ + if (time_to_write_stats_files < now) { +#define CHECK_WRITE_STATS_INTERVAL (60*60) + time_t next_time_to_write_stats_files = (time_to_write_stats_files > 0 ? + time_to_write_stats_files : now) + CHECK_WRITE_STATS_INTERVAL; + if (options->CellStatistics) { + time_t next_write = + rep_hist_buffer_stats_write(time_to_write_stats_files); + if (next_write && next_write < next_time_to_write_stats_files) + next_time_to_write_stats_files = next_write; + } + if (options->DirReqStatistics) { + time_t next_write = geoip_dirreq_stats_write(time_to_write_stats_files); + if (next_write && next_write < next_time_to_write_stats_files) + next_time_to_write_stats_files = next_write; + } + if (options->EntryStatistics) { + time_t next_write = geoip_entry_stats_write(time_to_write_stats_files); + if (next_write && next_write < next_time_to_write_stats_files) + next_time_to_write_stats_files = next_write; + } + if (options->ExitPortStatistics) { + time_t next_write = rep_hist_exit_stats_write(time_to_write_stats_files); + if (next_write && next_write < next_time_to_write_stats_files) + next_time_to_write_stats_files = next_write; + } + time_to_write_stats_files = next_time_to_write_stats_files; + } + + /* 1h. Check whether we should write bridge statistics to disk. + */ + if (should_record_bridge_info(options)) { + if (time_to_write_bridge_stats < now) { + if (should_init_bridge_stats) { + /* (Re-)initialize bridge statistics. */ + geoip_bridge_stats_init(now); + time_to_write_bridge_stats = now + WRITE_STATS_INTERVAL; + should_init_bridge_stats = 0; + } else { + /* Possibly write bridge statistics to disk and ask when to write + * them next time. */ + time_to_write_bridge_stats = geoip_bridge_stats_write( + time_to_write_bridge_stats); + } + } + } else if (!should_init_bridge_stats) { + /* Bridge mode was turned off. Ensure that stats are re-initialized + * next time bridge mode is turned on. */ + should_init_bridge_stats = 1; } /* Remove old information from rephist and the rend cache. */ @@ -991,7 +1115,7 @@ run_scheduled_events(time_t now) /* also, check religiously for reachability, if it's within the first * 20 minutes of our uptime. */ if (server_mode(options) && - (has_completed_circuit || !any_predicted_circuits(now)) && + (can_complete_circuit || !any_predicted_circuits(now)) && !we_are_hibernating()) { if (stats_n_seconds_working < TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT) { consider_testing_reachability(1, dirport_reachability_count==0); @@ -1034,7 +1158,9 @@ run_scheduled_events(time_t now) * We do this before step 4, so it can try building more if * it's not comfortable with the number of available circuits. */ - circuit_expire_building(now); + /* XXXX022 If our circuit build timeout is much lower than a second, maybe + we should do this more often? */ + circuit_expire_building(); /** 3b. Also look at pending streams and prune the ones that 'began' * a long time ago but haven't gotten a 'connected' yet. @@ -1067,7 +1193,7 @@ run_scheduled_events(time_t now) circuit_expire_old_circuits_serverside(now); /** 5. We do housekeeping for each connection... */ - connection_or_set_bad_connections(); + connection_or_set_bad_connections(NULL, 0); for (i=0;i<smartlist_len(connection_array);i++) { run_connection_housekeeping(i, now); } @@ -1090,7 +1216,7 @@ run_scheduled_events(time_t now) circuit_close_all_marked(); /** 7. And upload service descriptors if necessary. */ - if (has_completed_circuit && !we_are_hibernating()) { + if (can_complete_circuit && !we_are_hibernating()) { rend_consider_services_upload(now); rend_consider_descriptor_republication(); } @@ -1107,7 +1233,7 @@ run_scheduled_events(time_t now) /** 9. and if we're a server, check whether our DNS is telling stories to * us. */ - if (server_mode(options) && time_to_check_for_correct_dns < now) { + if (is_server && time_to_check_for_correct_dns < now) { if (!time_to_check_for_correct_dns) { time_to_check_for_correct_dns = now + 60 + crypto_rand_int(120); } else { @@ -1117,12 +1243,6 @@ run_scheduled_events(time_t now) } } - /** 10. write hidden service usage statistic to disk */ - if (options->HSAuthorityRecordStats && time_to_write_hs_statistics < now) { - hs_usage_write_statistics_to_file(now); -#define WRITE_HSUSAGE_INTERVAL (30*60) - time_to_write_hs_statistics = now+WRITE_HSUSAGE_INTERVAL; - } /** 10b. write bridge networkstatus file to disk */ if (options->BridgeAuthoritativeDir && time_to_write_bridge_status_file < now) { @@ -1132,39 +1252,30 @@ run_scheduled_events(time_t now) } } -/** Libevent timer: used to invoke second_elapsed_callback() once per - * second. */ -static struct event *timeout_event = NULL; +/** Timer: used to invoke second_elapsed_callback() once per second. */ +static periodic_timer_t *second_timer = NULL; /** Number of libevent errors in the last second: we die if we get too many. */ static int n_libevent_errors = 0; /** Libevent callback: invoked once every second. */ static void -second_elapsed_callback(int fd, short event, void *args) +second_elapsed_callback(periodic_timer_t *timer, void *arg) { /* XXXX This could be sensibly refactored into multiple callbacks, and we * could use Libevent's timers for this rather than checking the current * time against a bunch of timeouts every second. */ - static struct timeval one_second; static time_t current_second = 0; time_t now; size_t bytes_written; size_t bytes_read; int seconds_elapsed; or_options_t *options = get_options(); - (void)fd; - (void)event; - (void)args; - if (!timeout_event) { - timeout_event = tor_malloc_zero(sizeof(struct event)); - evtimer_set(timeout_event, second_elapsed_callback, NULL); - one_second.tv_sec = 1; - one_second.tv_usec = 0; - } + (void)timer; + (void)arg; n_libevent_errors = 0; - /* log_fn(LOG_NOTICE, "Tick."); */ + /* log_notice(LD_GENERAL, "Tick."); */ now = time(NULL); update_approx_time(now); @@ -1187,7 +1298,7 @@ second_elapsed_callback(int fd, short event, void *args) if (server_mode(options) && !we_are_hibernating() && seconds_elapsed > 0 && - has_completed_circuit && + can_complete_circuit && stats_n_seconds_working / TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT != (stats_n_seconds_working+seconds_elapsed) / TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT) { @@ -1229,18 +1340,6 @@ second_elapsed_callback(int fd, short event, void *args) run_scheduled_events(now); current_second = now; /* remember which second it is, for next time */ - -#if 0 - if (current_second % 300 == 0) { - rep_history_clean(current_second - options->RephistTrackTime); - dumpmemusage(get_min_log_level()<LOG_INFO ? - get_min_log_level() : LOG_INFO); - } -#endif - - if (evtimer_add(timeout_event, &one_second)) - log_err(LD_NET, - "Error from libevent when setting one-second timeout event"); } #ifndef MS_WINDOWS @@ -1383,7 +1482,7 @@ do_main_loop(void) /* load the private keys, if we're supposed to have them, and set up the * TLS context. */ - if (! identity_key_is_set()) { + if (! client_identity_key_is_set()) { if (init_keys() < 0) { log_err(LD_BUG,"Error initializing keys; exiting"); return -1; @@ -1401,8 +1500,10 @@ do_main_loop(void) /* initialize the bootstrap status events to know we're starting up */ control_event_bootstrap(BOOTSTRAP_STATUS_STARTING, 0); - if (trusted_dirs_reload_certs()) - return -1; + if (trusted_dirs_reload_certs()) { + log_warn(LD_DIR, + "Couldn't load all cached v3 certificates. Starting anyway."); + } if (router_reload_v2_networkstatus()) { return -1; } @@ -1419,18 +1520,23 @@ do_main_loop(void) now = time(NULL); directory_info_has_arrived(now, 1); - if (authdir_mode_tests_reachability(get_options())) { - /* the directory is already here, run startup things */ - dirserv_test_reachability(now, 1); - } - if (server_mode(get_options())) { /* launch cpuworkers. Need to do this *after* we've read the onion key. */ cpu_init(); } /* set up once-a-second callback. */ - second_elapsed_callback(0,0,NULL); + if (! second_timer) { + struct timeval one_second; + one_second.tv_sec = 1; + one_second.tv_usec = 0; + + second_timer = periodic_timer_new(tor_libevent_get_base(), + &one_second, + second_elapsed_callback, + NULL); + tor_assert(second_timer); + } for (;;) { if (nt_service_is_stopping()) @@ -1449,20 +1555,16 @@ do_main_loop(void) /* poll until we have an event, or the second ends, or until we have * some active linked connections to trigger events for. */ - loop_result = event_loop(called_loop_once ? EVLOOP_ONCE : 0); + loop_result = event_base_loop(tor_libevent_get_base(), + called_loop_once ? EVLOOP_ONCE : 0); /* let catch() handle things like ^c, and otherwise don't worry about it */ if (loop_result < 0) { int e = tor_socket_errno(-1); /* let the program survive things like ^z */ if (e != EINTR && !ERRNO_IS_EINPROGRESS(e)) { -#ifdef HAVE_EVENT_GET_METHOD log_err(LD_NET,"libevent call with %s failed: %s [%d]", - event_get_method(), tor_socket_strerror(e), e); -#else - log_err(LD_NET,"libevent call failed: %s [%d]", - tor_socket_strerror(e), e); -#endif + tor_libevent_get_method(), tor_socket_strerror(e), e); return -1; #ifndef MS_WINDOWS } else if (e == EINVAL) { @@ -1483,46 +1585,6 @@ do_main_loop(void) } } -/** Used to implement the SIGNAL control command: if we accept - * <b>the_signal</b> as a remote pseudo-signal, act on it. */ -/* We don't re-use catch() here because: - * 1. We handle a different set of signals than those allowed in catch. - * 2. Platforms without signal() are unlikely to define SIGfoo. - * 3. The control spec is defined to use fixed numeric signal values - * which just happen to match the Unix values. - */ -void -control_signal_act(int the_signal) -{ - switch (the_signal) - { - case 1: - signal_callback(0,0,(void*)(uintptr_t)SIGHUP); - break; - case 2: - signal_callback(0,0,(void*)(uintptr_t)SIGINT); - break; - case 10: - signal_callback(0,0,(void*)(uintptr_t)SIGUSR1); - break; - case 12: - signal_callback(0,0,(void*)(uintptr_t)SIGUSR2); - break; - case 15: - signal_callback(0,0,(void*)(uintptr_t)SIGTERM); - break; - case SIGNEWNYM: - signal_callback(0,0,(void*)(uintptr_t)SIGNEWNYM); - break; - case SIGCLEARDNSCACHE: - signal_callback(0,0,(void*)(uintptr_t)SIGCLEARDNSCACHE); - break; - default: - log_warn(LD_BUG, "Unrecognized signal number %d.", the_signal); - break; - } -} - /** Libevent callback: invoked when we get a signal. */ static void @@ -1531,6 +1593,14 @@ signal_callback(int fd, short events, void *arg) uintptr_t sig = (uintptr_t)arg; (void)fd; (void)events; + + process_signal(sig); +} + +/** Do the work of acting on a signal received in <b>sig</b> */ +void +process_signal(uintptr_t sig) +{ switch (sig) { case SIGTERM: @@ -1605,6 +1675,7 @@ dumpmemusage(int severity) U64_PRINTF_ARG(rephist_total_alloc), rephist_total_num); dump_routerlist_mem_usage(severity); dump_cell_pool_usage(severity); + dump_dns_mem_usage(severity); buf_dump_freelist_sizes(severity); tor_log_mallinfo(severity); } @@ -1631,7 +1702,8 @@ dumpstats(int severity) if (!connection_is_listener(conn)) { log(severity,LD_GENERAL, "Conn %d is to %s:%d.", i, - safe_str(conn->address), conn->port); + safe_str_client(conn->address), + conn->port); log(severity,LD_GENERAL, "Conn %d: %d bytes waiting on inbuf (len %d, last read %d secs ago)", i, @@ -1728,7 +1800,7 @@ handle_signals(int is_parent) { #ifndef MS_WINDOWS /* do signal stuff only on Unix */ int i; - static int signals[] = { + static const int signals[] = { SIGINT, /* do a controlled slow shutdown */ SIGTERM, /* to terminate now */ SIGPIPE, /* otherwise SIGPIPE kills us */ @@ -1740,12 +1812,13 @@ handle_signals(int is_parent) #endif SIGCHLD, /* handle dns/cpu workers that exit */ -1 }; - static struct event signal_events[16]; /* bigger than it has to be. */ + static struct event *signal_events[16]; /* bigger than it has to be. */ if (is_parent) { for (i = 0; signals[i] >= 0; ++i) { - signal_set(&signal_events[i], signals[i], signal_callback, - (void*)(uintptr_t)signals[i]); - if (signal_add(&signal_events[i], NULL)) + signal_events[i] = tor_evsignal_new( + tor_libevent_get_base(), signals[i], signal_callback, + (void*)(uintptr_t)signals[i]); + if (event_add(signal_events[i], NULL)) log_warn(LD_BUG, "Error from libevent when adding event for signal %d", signals[i]); } @@ -1834,7 +1907,9 @@ tor_init(int argc, char *argv[]) "and you probably shouldn't."); #endif - if (crypto_global_init(get_options()->HardwareAccel)) { + if (crypto_global_init(get_options()->HardwareAccel, + get_options()->AccelName, + get_options()->AccelDir)) { log_err(LD_BUG, "Unable to initialize OpenSSL. Exiting."); return -1; } @@ -1928,7 +2003,6 @@ tor_free_all(int postfork) rend_cache_free_all(); rend_service_authorization_free_all(); rep_hist_free_all(); - hs_usage_free_all(); dns_free_all(); clear_pending_onions(); circuit_free_all(); @@ -1936,6 +2010,7 @@ tor_free_all(int postfork) connection_free_all(); buf_shrink_freelists(1); memarea_clear_freelist(); + microdesc_free_all(); if (!postfork) { config_free_all(); router_free_all(); @@ -1946,13 +2021,11 @@ tor_free_all(int postfork) tor_tls_free_all(); } /* stuff in main.c */ - if (connection_array) - smartlist_free(connection_array); - if (closeable_connection_lst) - smartlist_free(closeable_connection_lst); - if (active_linked_connection_lst) - smartlist_free(active_linked_connection_lst); - tor_free(timeout_event); + + smartlist_free(connection_array); + smartlist_free(closeable_connection_lst); + smartlist_free(active_linked_connection_lst); + periodic_timer_free(second_timer); if (!postfork) { release_lockfile(); } @@ -2013,7 +2086,7 @@ do_list_fingerprint(void) log_err(LD_BUG,"Error initializing keys; can't display fingerprint"); return -1; } - if (!(k = get_identity_key())) { + if (!(k = get_server_identity_key())) { log_err(LD_GENERAL,"Error: missing identity key."); return -1; } @@ -2043,6 +2116,31 @@ do_hash_password(void) printf("16:%s\n",output); } +#if defined (WINCE) +int +find_flashcard_path(PWCHAR path, size_t size) +{ + WIN32_FIND_DATA d = {0}; + HANDLE h = NULL; + + if (!path) + return -1; + + h = FindFirstFlashCard(&d); + if (h == INVALID_HANDLE_VALUE) + return -1; + + if (wcslen(d.cFileName) == 0) { + FindClose(h); + return -1; + } + + wcsncpy(path,d.cFileName,size); + FindClose(h); + return 0; +} +#endif + /** Main entry point for the Tor process. Called from main(). */ /* This function is distinct from main() only so we can link main.c into * the unittest binary without conflicting with the unittests' main. */ @@ -2050,6 +2148,46 @@ int tor_main(int argc, char *argv[]) { int result = 0; +#if defined (WINCE) + WCHAR path [MAX_PATH] = {0}; + WCHAR fullpath [MAX_PATH] = {0}; + PWCHAR p = NULL; + FILE* redir = NULL; + FILE* redirdbg = NULL; + + // this is to facilitate debugging by opening + // a file on a folder shared by the wm emulator. + // if no flashcard (real or emulated) is present, + // log files will be written in the root folder + if (find_flashcard_path(path,MAX_PATH) == -1) + { + redir = _wfreopen( L"\\stdout.log", L"w", stdout ); + redirdbg = _wfreopen( L"\\stderr.log", L"w", stderr ); + } else { + swprintf(fullpath,L"\\%s\\tor",path); + CreateDirectory(fullpath,NULL); + + swprintf(fullpath,L"\\%s\\tor\\stdout.log",path); + redir = _wfreopen( fullpath, L"w", stdout ); + + swprintf(fullpath,L"\\%s\\tor\\stderr.log",path); + redirdbg = _wfreopen( fullpath, L"w", stderr ); + } +#endif + +#ifdef MS_WINDOWS + /* Call SetProcessDEPPolicy to permanently enable DEP. + The function will not resolve on earlier versions of Windows, + and failure is not dangerous. */ + HMODULE hMod = GetModuleHandleA("Kernel32.dll"); + if (hMod) { + typedef BOOL (WINAPI *PSETDEP)(DWORD); + PSETDEP setdeppolicy = (PSETDEP)GetProcAddress(hMod, + "SetProcessDEPPolicy"); + if (setdeppolicy) setdeppolicy(1); /* PROCESS_DEP_ENABLE */ + } +#endif + update_approx_time(time(NULL)); tor_threads_init(); init_logging(); diff --git a/src/or/main.h b/src/or/main.h new file mode 100644 index 0000000000..0551f7aaf9 --- /dev/null +++ b/src/or/main.h @@ -0,0 +1,69 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file main.h + * \brief Header file for main.c. + **/ + +#ifndef _TOR_MAIN_H +#define _TOR_MAIN_H + +extern int can_complete_circuit; + +int connection_add(connection_t *conn); +int connection_remove(connection_t *conn); +void connection_unregister_events(connection_t *conn); +int connection_in_array(connection_t *conn); +void add_connection_to_closeable_list(connection_t *conn); +int connection_is_on_closeable_list(connection_t *conn); + +smartlist_t *get_connection_array(void); + +/** Bitmask for events that we can turn on and off with + * connection_watch_events. */ +typedef enum watchable_events { + READ_EVENT=0x02, /**< We want to know when a connection is readable */ + WRITE_EVENT=0x04 /**< We want to know when a connection is writable */ +} watchable_events_t; +void connection_watch_events(connection_t *conn, watchable_events_t events); +int connection_is_reading(connection_t *conn); +void connection_stop_reading(connection_t *conn); +void connection_start_reading(connection_t *conn); + +int connection_is_writing(connection_t *conn); +void connection_stop_writing(connection_t *conn); +void connection_start_writing(connection_t *conn); + +void connection_stop_reading_from_linked_conn(connection_t *conn); + +void directory_all_unreachable(time_t now); +void directory_info_has_arrived(time_t now, int from_cache); + +void ip_address_changed(int at_interface); +void dns_servers_relaunch_checks(void); + +void handle_signals(int is_parent); +void process_signal(uintptr_t sig); + +int try_locking(or_options_t *options, int err_if_locked); +int have_lockfile(void); +void release_lockfile(void); + +void tor_cleanup(void); +void tor_free_all(int postfork); + +int tor_main(int argc, char *argv[]); + +#ifdef MAIN_PRIVATE +int do_main_loop(void); +int do_list_fingerprint(void); +void do_hash_password(void); +int tor_init(int argc, char **argv); +#endif + +#endif + diff --git a/src/or/microdesc.c b/src/or/microdesc.c new file mode 100644 index 0000000000..2c4b3435f7 --- /dev/null +++ b/src/or/microdesc.c @@ -0,0 +1,414 @@ +/* Copyright (c) 2009-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "or.h" +#include "config.h" +#include "microdesc.h" +#include "routerparse.h" + +/** A data structure to hold a bunch of cached microdescriptors. There are + * two active files in the cache: a "cache file" that we mmap, and a "journal + * file" that we append to. Periodically, we rebuild the cache file to hold + * only the microdescriptors that we want to keep */ +struct microdesc_cache_t { + /** Map from sha256-digest to microdesc_t for every microdesc_t in the + * cache. */ + HT_HEAD(microdesc_map, microdesc_t) map; + + /** Name of the cache file. */ + char *cache_fname; + /** Name of the journal file. */ + char *journal_fname; + /** Mmap'd contents of the cache file, or NULL if there is none. */ + tor_mmap_t *cache_content; + /** Number of bytes used in the journal file. */ + size_t journal_len; + + /** Total bytes of microdescriptor bodies we have added to this cache */ + uint64_t total_len_seen; + /** Total number of microdescriptors we have added to this cache */ + unsigned n_seen; +}; + +/** Helper: computes a hash of <b>md</b> to place it in a hash table. */ +static INLINE unsigned int +_microdesc_hash(microdesc_t *md) +{ + unsigned *d = (unsigned*)md->digest; +#if SIZEOF_INT == 4 + return d[0] ^ d[1] ^ d[2] ^ d[3] ^ d[4] ^ d[5] ^ d[6] ^ d[7]; +#else + return d[0] ^ d[1] ^ d[2] ^ d[3]; +#endif +} + +/** Helper: compares <b>a</b> and </b> for equality for hash-table purposes. */ +static INLINE int +_microdesc_eq(microdesc_t *a, microdesc_t *b) +{ + return !memcmp(a->digest, b->digest, DIGEST256_LEN); +} + +HT_PROTOTYPE(microdesc_map, microdesc_t, node, + _microdesc_hash, _microdesc_eq); +HT_GENERATE(microdesc_map, microdesc_t, node, + _microdesc_hash, _microdesc_eq, 0.6, + malloc, realloc, free); + +/** Write the body of <b>md</b> into <b>f</b>, with appropriate annotations. + * On success, return the total number of bytes written, and set + * *<b>annotation_len_out</b> to the number of bytes written as + * annotations. */ +static ssize_t +dump_microdescriptor(FILE *f, microdesc_t *md, size_t *annotation_len_out) +{ + ssize_t r = 0; + size_t written; + /* XXXX drops unkown annotations. */ + if (md->last_listed) { + char buf[ISO_TIME_LEN+1]; + char annotation[ISO_TIME_LEN+32]; + format_iso_time(buf, md->last_listed); + tor_snprintf(annotation, sizeof(annotation), "@last-listed %s\n", buf); + fputs(annotation, f); + r += strlen(annotation); + *annotation_len_out = r; + } else { + *annotation_len_out = 0; + } + + md->off = (off_t) ftell(f); + written = fwrite(md->body, 1, md->bodylen, f); + if (written != md->bodylen) { + log_warn(LD_DIR, + "Couldn't dump microdescriptor (wrote %lu out of %lu): %s", + (unsigned long)written, (unsigned long)md->bodylen, + strerror(ferror(f))); + return -1; + } + r += md->bodylen; + return r; +} + +/** Holds a pointer to the current microdesc_cache_t object, or NULL if no + * such object has been allocated. */ +static microdesc_cache_t *the_microdesc_cache = NULL; + +/** Return a pointer to the microdescriptor cache, loading it if necessary. */ +microdesc_cache_t * +get_microdesc_cache(void) +{ + if (PREDICT_UNLIKELY(the_microdesc_cache==NULL)) { + microdesc_cache_t *cache = tor_malloc_zero(sizeof(microdesc_cache_t)); + HT_INIT(microdesc_map, &cache->map); + cache->cache_fname = get_datadir_fname("cached-microdescs"); + cache->journal_fname = get_datadir_fname("cached-microdescs.new"); + microdesc_cache_reload(cache); + the_microdesc_cache = cache; + } + return the_microdesc_cache; +} + +/* There are three sources of microdescriptors: + 1) Generated by us while acting as a directory authority. + 2) Loaded from the cache on disk. + 3) Downloaded. +*/ + +/** Decode the microdescriptors from the string starting at <b>s</b> and + * ending at <b>eos</b>, and store them in <b>cache</b>. If <b>no-save</b>, + * mark them as non-writable to disk. If <b>where</b> is SAVED_IN_CACHE, + * leave their bodies as pointers to the mmap'd cache. If where is + * <b>SAVED_NOWHERE</b>, do not allow annotations. Return a list of the added + * microdescriptors. */ +smartlist_t * +microdescs_add_to_cache(microdesc_cache_t *cache, + const char *s, const char *eos, saved_location_t where, + int no_save) +{ + /*XXXX need an argument that sets last_listed as appropriate. */ + + smartlist_t *descriptors, *added; + const int allow_annotations = (where != SAVED_NOWHERE); + const int copy_body = (where != SAVED_IN_CACHE); + + descriptors = microdescs_parse_from_string(s, eos, + allow_annotations, + copy_body); + + added = microdescs_add_list_to_cache(cache, descriptors, where, no_save); + smartlist_free(descriptors); + return added; +} + +/* As microdescs_add_to_cache, but takes a list of micrdescriptors instead of + * a string to encode. Frees any members of <b>descriptors</b> that it does + * not add. */ +smartlist_t * +microdescs_add_list_to_cache(microdesc_cache_t *cache, + smartlist_t *descriptors, saved_location_t where, + int no_save) +{ + smartlist_t *added; + open_file_t *open_file = NULL; + FILE *f = NULL; + // int n_added = 0; + ssize_t size = 0; + + if (where == SAVED_NOWHERE && !no_save) { + f = start_writing_to_stdio_file(cache->journal_fname, + OPEN_FLAGS_APPEND|O_BINARY, + 0600, &open_file); + if (!f) { + log_warn(LD_DIR, "Couldn't append to journal in %s: %s", + cache->journal_fname, strerror(errno)); + return NULL; + } + } + + added = smartlist_create(); + SMARTLIST_FOREACH_BEGIN(descriptors, microdesc_t *, md) { + microdesc_t *md2; + md2 = HT_FIND(microdesc_map, &cache->map, md); + if (md2) { + /* We already had this one. */ + if (md2->last_listed < md->last_listed) + md2->last_listed = md->last_listed; + microdesc_free(md); + continue; + } + + /* Okay, it's a new one. */ + if (f) { + size_t annotation_len; + size = dump_microdescriptor(f, md, &annotation_len); + if (size < 0) { + /* XXX handle errors from dump_microdescriptor() */ + /* log? return -1? die? coredump the universe? */ + continue; + } + md->saved_location = SAVED_IN_JOURNAL; + cache->journal_len += size; + } else { + md->saved_location = where; + } + + md->no_save = no_save; + + HT_INSERT(microdesc_map, &cache->map, md); + smartlist_add(added, md); + ++cache->n_seen; + cache->total_len_seen += md->bodylen; + } SMARTLIST_FOREACH_END(md); + + if (f) + finish_writing_to_file(open_file); /*XXX Check me.*/ + + { + size_t old_content_len = + cache->cache_content ? cache->cache_content->size : 0; + if (cache->journal_len > 16384 + old_content_len && + cache->journal_len > old_content_len * 2) { + microdesc_cache_rebuild(cache); + } + } + + return added; +} + +/** Remove every microdescriptor in <b>cache</b>. */ +void +microdesc_cache_clear(microdesc_cache_t *cache) +{ + microdesc_t **entry, **next; + for (entry = HT_START(microdesc_map, &cache->map); entry; entry = next) { + microdesc_t *md = *entry; + next = HT_NEXT_RMV(microdesc_map, &cache->map, entry); + microdesc_free(md); + } + HT_CLEAR(microdesc_map, &cache->map); + if (cache->cache_content) { + tor_munmap_file(cache->cache_content); + cache->cache_content = NULL; + } + cache->total_len_seen = 0; + cache->n_seen = 0; +} + +/** Reload the contents of <b>cache</b> from disk. If it is empty, load it + * for the first time. Return 0 on success, -1 on failure. */ +int +microdesc_cache_reload(microdesc_cache_t *cache) +{ + struct stat st; + char *journal_content; + smartlist_t *added; + tor_mmap_t *mm; + int total = 0; + + microdesc_cache_clear(cache); + + mm = cache->cache_content = tor_mmap_file(cache->cache_fname); + if (mm) { + added = microdescs_add_to_cache(cache, mm->data, mm->data+mm->size, + SAVED_IN_CACHE, 0); + if (added) { + total += smartlist_len(added); + smartlist_free(added); + } + } + + journal_content = read_file_to_str(cache->journal_fname, + RFTS_IGNORE_MISSING, &st); + if (journal_content) { + added = microdescs_add_to_cache(cache, journal_content, + journal_content+st.st_size, + SAVED_IN_JOURNAL, 0); + if (added) { + total += smartlist_len(added); + smartlist_free(added); + } + tor_free(journal_content); + } + log_notice(LD_DIR, "Reloaded microdescriptor cache. Found %d descriptors.", + total); + return 0; +} + +/** Regenerate the main cache file for <b>cache</b>, clear the journal file, + * and update every microdesc_t in the cache with pointers to its new + * location. */ +int +microdesc_cache_rebuild(microdesc_cache_t *cache) +{ + open_file_t *open_file; + FILE *f; + microdesc_t **mdp; + smartlist_t *wrote; + ssize_t size; + off_t off = 0; + int orig_size, new_size; + + log_info(LD_DIR, "Rebuilding the microdescriptor cache..."); + orig_size = (int)(cache->cache_content ? cache->cache_content->size : 0); + orig_size += (int)cache->journal_len; + + f = start_writing_to_stdio_file(cache->cache_fname, + OPEN_FLAGS_REPLACE|O_BINARY, + 0600, &open_file); + if (!f) + return -1; + + wrote = smartlist_create(); + + HT_FOREACH(mdp, microdesc_map, &cache->map) { + microdesc_t *md = *mdp; + size_t annotation_len; + if (md->no_save) + continue; + + size = dump_microdescriptor(f, md, &annotation_len); + if (size < 0) { + /* XXX handle errors from dump_microdescriptor() */ + /* log? return -1? die? coredump the universe? */ + continue; + } + md->off = off + annotation_len; + off += size; + if (md->saved_location != SAVED_IN_CACHE) { + tor_free(md->body); + md->saved_location = SAVED_IN_CACHE; + } + smartlist_add(wrote, md); + } + + finish_writing_to_file(open_file); /*XXX Check me.*/ + + if (cache->cache_content) + tor_munmap_file(cache->cache_content); + cache->cache_content = tor_mmap_file(cache->cache_fname); + + if (!cache->cache_content && smartlist_len(wrote)) { + log_err(LD_DIR, "Couldn't map file that we just wrote to %s!", + cache->cache_fname); + smartlist_free(wrote); + return -1; + } + SMARTLIST_FOREACH_BEGIN(wrote, microdesc_t *, md) { + tor_assert(md->saved_location == SAVED_IN_CACHE); + md->body = (char*)cache->cache_content->data + md->off; + tor_assert(!memcmp(md->body, "onion-key", 9)); + } SMARTLIST_FOREACH_END(md); + + smartlist_free(wrote); + + write_str_to_file(cache->journal_fname, "", 1); + cache->journal_len = 0; + + new_size = (int)cache->cache_content->size; + log_info(LD_DIR, "Done rebuilding microdesc cache. " + "Saved %d bytes; %d still used.", + orig_size-new_size, new_size); + + return 0; +} + +/** Deallocate a single microdescriptor. Note: the microdescriptor MUST have + * previously been removed from the cache if it had ever been inserted. */ +void +microdesc_free(microdesc_t *md) +{ + if (!md) + return; + /* Must be removed from hash table! */ + if (md->onion_pkey) + crypto_free_pk_env(md->onion_pkey); + if (md->body && md->saved_location != SAVED_IN_CACHE) + tor_free(md->body); + + if (md->family) { + SMARTLIST_FOREACH(md->family, char *, cp, tor_free(cp)); + smartlist_free(md->family); + } + tor_free(md->exitsummary); + + tor_free(md); +} + +/** Free all storage held in the microdesc.c module. */ +void +microdesc_free_all(void) +{ + if (the_microdesc_cache) { + microdesc_cache_clear(the_microdesc_cache); + tor_free(the_microdesc_cache->cache_fname); + tor_free(the_microdesc_cache->journal_fname); + tor_free(the_microdesc_cache); + } +} + +/** If there is a microdescriptor in <b>cache</b> whose sha256 digest is + * <b>d</b>, return it. Otherwise return NULL. */ +microdesc_t * +microdesc_cache_lookup_by_digest256(microdesc_cache_t *cache, const char *d) +{ + microdesc_t *md, search; + if (!cache) + cache = get_microdesc_cache(); + memcpy(search.digest, d, DIGEST256_LEN); + md = HT_FIND(microdesc_map, &cache->map, &search); + return md; +} + +/** Return the mean size of decriptors added to <b>cache</b> since it was last + * cleared. Used to estimate the size of large downloads. */ +size_t +microdesc_average_size(microdesc_cache_t *cache) +{ + if (!cache) + cache = get_microdesc_cache(); + if (!cache->n_seen) + return 512; + return (size_t)(cache->total_len_seen / cache->n_seen); +} + diff --git a/src/or/microdesc.h b/src/or/microdesc.h new file mode 100644 index 0000000000..b3e12f8ef0 --- /dev/null +++ b/src/or/microdesc.h @@ -0,0 +1,37 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file microdesc.h + * \brief Header file for microdesc.c. + **/ + +#ifndef _TOR_MICRODESC_H +#define _TOR_MICRODESC_H + +microdesc_cache_t *get_microdesc_cache(void); + +smartlist_t *microdescs_add_to_cache(microdesc_cache_t *cache, + const char *s, const char *eos, saved_location_t where, + int no_save); +smartlist_t *microdescs_add_list_to_cache(microdesc_cache_t *cache, + smartlist_t *descriptors, saved_location_t where, + int no_save); + +int microdesc_cache_rebuild(microdesc_cache_t *cache); +int microdesc_cache_reload(microdesc_cache_t *cache); +void microdesc_cache_clear(microdesc_cache_t *cache); + +microdesc_t *microdesc_cache_lookup_by_digest256(microdesc_cache_t *cache, + const char *d); + +size_t microdesc_average_size(microdesc_cache_t *cache); + +void microdesc_free(microdesc_t *md); +void microdesc_free_all(void); + +#endif + diff --git a/src/or/networkstatus.c b/src/or/networkstatus.c index 7106294d54..4f6fe15409 100644 --- a/src/or/networkstatus.c +++ b/src/or/networkstatus.c @@ -11,6 +11,20 @@ */ #include "or.h" +#include "circuitbuild.h" +#include "config.h" +#include "connection.h" +#include "connection_or.h" +#include "control.h" +#include "directory.h" +#include "dirserv.h" +#include "dirvote.h" +#include "main.h" +#include "networkstatus.h" +#include "relay.h" +#include "router.h" +#include "routerlist.h" +#include "routerparse.h" /* For tracking v2 networkstatus documents. Only caches do this now. */ @@ -35,16 +49,22 @@ static networkstatus_t *current_consensus = NULL; /** A v3 consensus networkstatus that we've received, but which we don't * have enough certificates to be happy about. */ -static networkstatus_t *consensus_waiting_for_certs = NULL; -/** The encoded version of consensus_waiting_for_certs. */ -static char *consensus_waiting_for_certs_body = NULL; -/** When did we set the current value of consensus_waiting_for_certs? If this - * is too recent, we shouldn't try to fetch a new consensus for a little while, - * to give ourselves time to get certificates for this one. */ -static time_t consensus_waiting_for_certs_set_at = 0; -/** Set to 1 if we've been holding on to consensus_waiting_for_certs so long - * that we should treat it as maybe being bad. */ -static int consensus_waiting_for_certs_dl_failed = 0; +typedef struct consensus_waiting_for_certs_t { + /** The consensus itself. */ + networkstatus_t *consensus; + /** The encoded version of the consensus, nul-terminated. */ + char *body; + /** When did we set the current value of consensus_waiting_for_certs? If + * this is too recent, we shouldn't try to fetch a new consensus for a + * little while, to give ourselves time to get certificates for this one. */ + time_t set_at; + /** Set to 1 if we've been holding on to it for so long we should maybe + * treat it as being bad. */ + int dl_failed; +} consensus_waiting_for_certs_t; + +static consensus_waiting_for_certs_t + consensus_waiting_for_certs[N_CONSENSUS_FLAVORS]; /** The last time we tried to download a networkstatus, or 0 for "never". We * use this to rate-limit download attempts for directory caches (including @@ -56,7 +76,7 @@ static time_t last_networkstatus_download_attempted = 0; * before the current consensus becomes invalid. */ static time_t time_to_download_next_consensus = 0; /** Download status for the current consensus networkstatus. */ -static download_status_t consensus_dl_status = { 0, 0, DL_SCHED_CONSENSUS }; +static download_status_t consensus_dl_status[N_CONSENSUS_FLAVORS]; /** True iff we have logged a warning about this OR's version being older than * listed by the authorities. */ @@ -89,6 +109,7 @@ networkstatus_reset_warnings(void) void networkstatus_reset_download_failures(void) { + int i; const smartlist_t *networkstatus_v2_list = networkstatus_get_v2_list(); SMARTLIST_FOREACH(networkstatus_v2_list, networkstatus_v2_t *, ns, SMARTLIST_FOREACH(ns->entries, routerstatus_t *, rs, @@ -97,7 +118,8 @@ networkstatus_reset_download_failures(void) rs->need_to_mirror = 1; }));; - download_status_reset(&consensus_dl_status); + for (i=0; i < N_CONSENSUS_FLAVORS; ++i) + download_status_reset(&consensus_dl_status[i]); if (v2_download_status_map) { digestmap_iter_t *iter; digestmap_t *map = v2_download_status_map; @@ -170,7 +192,7 @@ router_reload_v2_networkstatus(void) return 0; } -/** Read the cached v3 consensus networkstatus from the disk. */ +/** Read every cached v3 consensus networkstatus from the disk. */ int router_reload_consensus_networkstatus(void) { @@ -179,31 +201,46 @@ router_reload_consensus_networkstatus(void) struct stat st; or_options_t *options = get_options(); const unsigned int flags = NSSET_FROM_CACHE | NSSET_DONT_DOWNLOAD_CERTS; + int flav; /* FFFF Suppress warnings if cached consensus is bad? */ + for (flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) { + char buf[128]; + const char *flavor = networkstatus_get_flavor_name(flav); + if (flav == FLAV_NS) { + filename = get_datadir_fname("cached-consensus"); + } else { + tor_snprintf(buf, sizeof(buf), "cached-%s-consensus", flavor); + filename = get_datadir_fname(buf); + } + s = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL); + if (s) { + if (networkstatus_set_current_consensus(s, flavor, flags) < -1) { + log_warn(LD_FS, "Couldn't load consensus %s networkstatus from \"%s\"", + flavor, filename); + } + tor_free(s); + } + tor_free(filename); - filename = get_datadir_fname("cached-consensus"); - s = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL); - if (s) { - if (networkstatus_set_current_consensus(s, flags) < -1) { - log_warn(LD_FS, "Couldn't load consensus networkstatus from \"%s\"", - filename); + if (flav == FLAV_NS) { + filename = get_datadir_fname("unverified-consensus"); + } else { + tor_snprintf(buf, sizeof(buf), "unverified-%s-consensus", flavor); + filename = get_datadir_fname(buf); } - tor_free(s); - } - tor_free(filename); - filename = get_datadir_fname("unverified-consensus"); - s = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL); - if (s) { - if (networkstatus_set_current_consensus(s, + s = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL); + if (s) { + if (networkstatus_set_current_consensus(s, flavor, flags|NSSET_WAS_WAITING_FOR_CERTS)) { - log_info(LD_FS, "Couldn't load consensus networkstatus from \"%s\"", - filename); + log_info(LD_FS, "Couldn't load consensus %s networkstatus from \"%s\"", + flavor, filename); } - tor_free(s); + tor_free(s); + } + tor_free(filename); } - tor_free(filename); if (!current_consensus || (stat(options->FallbackNetworkstatusFile, &st)==0 && @@ -211,7 +248,7 @@ router_reload_consensus_networkstatus(void) s = read_file_to_str(options->FallbackNetworkstatusFile, RFTS_IGNORE_MISSING, NULL); if (s) { - if (networkstatus_set_current_consensus(s, + if (networkstatus_set_current_consensus(s, "ns", flags|NSSET_ACCEPT_OBSOLETE)) { log_info(LD_FS, "Couldn't load consensus networkstatus from \"%s\"", options->FallbackNetworkstatusFile); @@ -242,8 +279,16 @@ router_reload_consensus_networkstatus(void) static void vote_routerstatus_free(vote_routerstatus_t *rs) { + vote_microdesc_hash_t *h, *next; + if (!rs) + return; tor_free(rs->version); tor_free(rs->status.exitsummary); + for (h = rs->microdesc; h; h = next) { + tor_free(h->microdesc_hash_line); + next = h->next; + tor_free(h); + } tor_free(rs); } @@ -251,6 +296,8 @@ vote_routerstatus_free(vote_routerstatus_t *rs) void routerstatus_free(routerstatus_t *rs) { + if (!rs) + return; tor_free(rs->exitsummary); tor_free(rs); } @@ -259,6 +306,8 @@ routerstatus_free(routerstatus_t *rs) void networkstatus_v2_free(networkstatus_v2_t *ns) { + if (!ns) + return; tor_free(ns->source_address); tor_free(ns->contact); if (ns->signing_key) @@ -273,7 +322,25 @@ networkstatus_v2_free(networkstatus_v2_t *ns) tor_free(ns); } -/** Clear all storage held in <b>ns</b>. */ +/** Free all storage held in <b>sig</b> */ +void +document_signature_free(document_signature_t *sig) +{ + tor_free(sig->signature); + tor_free(sig); +} + +/** Return a newly allocated copy of <b>sig</b> */ +document_signature_t * +document_signature_dup(const document_signature_t *sig) +{ + document_signature_t *r = tor_memdup(sig, sizeof(document_signature_t)); + if (r->signature) + r->signature = tor_memdup(sig->signature, sig->signature_len); + return r; +} + +/** Free all storage held in <b>ns</b>. */ void networkstatus_vote_free(networkstatus_t *ns) { @@ -286,6 +353,10 @@ networkstatus_vote_free(networkstatus_t *ns) SMARTLIST_FOREACH(ns->known_flags, char *, c, tor_free(c)); smartlist_free(ns->known_flags); } + if (ns->weight_params) { + SMARTLIST_FOREACH(ns->weight_params, char *, c, tor_free(c)); + smartlist_free(ns->weight_params); + } if (ns->net_params) { SMARTLIST_FOREACH(ns->net_params, char *, c, tor_free(c)); smartlist_free(ns->net_params); @@ -295,18 +366,20 @@ networkstatus_vote_free(networkstatus_t *ns) smartlist_free(ns->supported_methods); } if (ns->voters) { - SMARTLIST_FOREACH(ns->voters, networkstatus_voter_info_t *, voter, - { + SMARTLIST_FOREACH_BEGIN(ns->voters, networkstatus_voter_info_t *, voter) { tor_free(voter->nickname); tor_free(voter->address); tor_free(voter->contact); - tor_free(voter->signature); + if (voter->sigs) { + SMARTLIST_FOREACH(voter->sigs, document_signature_t *, sig, + document_signature_free(sig)); + smartlist_free(voter->sigs); + } tor_free(voter); - }); + } SMARTLIST_FOREACH_END(voter); smartlist_free(ns->voters); } - if (ns->cert) - authority_cert_free(ns->cert); + authority_cert_free(ns->cert); if (ns->routerstatus_list) { if (ns->type == NS_TYPE_VOTE || ns->type == NS_TYPE_OPINION) { @@ -319,8 +392,8 @@ networkstatus_vote_free(networkstatus_t *ns) smartlist_free(ns->routerstatus_list); } - if (ns->desc_digest_map) - digestmap_free(ns->desc_digest_map, NULL); + + digestmap_free(ns->desc_digest_map, NULL); memset(ns, 11, sizeof(*ns)); tor_free(ns); @@ -341,35 +414,39 @@ networkstatus_get_voter_by_id(networkstatus_t *vote, return NULL; } -/** Check whether the signature on <b>voter</b> is correctly signed by - * the signing key of <b>cert</b>. Return -1 if <b>cert</b> doesn't match the +/** Check whether the signature <b>sig</b> is correctly signed with the + * signing key in <b>cert</b>. Return -1 if <b>cert</b> doesn't match the * signing key; otherwise set the good_signature or bad_signature flag on * <b>voter</b>, and return 0. */ -/* (private; exposed for testing.) */ int -networkstatus_check_voter_signature(networkstatus_t *consensus, - networkstatus_voter_info_t *voter, - authority_cert_t *cert) +networkstatus_check_document_signature(const networkstatus_t *consensus, + document_signature_t *sig, + const authority_cert_t *cert) { - char d[DIGEST_LEN]; + char key_digest[DIGEST_LEN]; + const int dlen = sig->alg == DIGEST_SHA1 ? DIGEST_LEN : DIGEST256_LEN; char *signed_digest; size_t signed_digest_len; - if (crypto_pk_get_digest(cert->signing_key, d)<0) + + if (crypto_pk_get_digest(cert->signing_key, key_digest)<0) return -1; - if (memcmp(voter->signing_key_digest, d, DIGEST_LEN)) + if (memcmp(sig->signing_key_digest, key_digest, DIGEST_LEN) || + memcmp(sig->identity_digest, cert->cache_info.identity_digest, + DIGEST_LEN)) return -1; + signed_digest_len = crypto_pk_keysize(cert->signing_key); signed_digest = tor_malloc(signed_digest_len); if (crypto_pk_public_checksig(cert->signing_key, signed_digest, signed_digest_len, - voter->signature, - voter->signature_len) != DIGEST_LEN || - memcmp(signed_digest, consensus->networkstatus_digest, DIGEST_LEN)) { + sig->signature, + sig->signature_len) < dlen || + memcmp(signed_digest, consensus->digests.d[sig->alg], dlen)) { log_warn(LD_DIR, "Got a bad signature on a networkstatus vote"); - voter->bad_signature = 1; + sig->bad_signature = 1; } else { - voter->good_signature = 1; + sig->good_signature = 1; } tor_free(signed_digest); return 0; @@ -388,7 +465,7 @@ networkstatus_check_consensus_signature(networkstatus_t *consensus, int warn) { int n_good = 0; - int n_missing_key = 0; + int n_missing_key = 0, n_dl_failed_key = 0; int n_bad = 0; int n_unknown = 0; int n_no_signature = 0; @@ -402,37 +479,62 @@ networkstatus_check_consensus_signature(networkstatus_t *consensus, tor_assert(consensus->type == NS_TYPE_CONSENSUS); - SMARTLIST_FOREACH(consensus->voters, networkstatus_voter_info_t *, voter, - { - if (!voter->good_signature && !voter->bad_signature && voter->signature) { - /* we can try to check the signature. */ - int is_v3_auth = trusteddirserver_get_by_v3_auth_digest( - voter->identity_digest) != NULL; - authority_cert_t *cert = - authority_cert_get_by_digests(voter->identity_digest, - voter->signing_key_digest); - if (!is_v3_auth) { - smartlist_add(unrecognized, voter); - ++n_unknown; - continue; - } else if (!cert || cert->expires < now) { - smartlist_add(need_certs_from, voter); - ++n_missing_key; - continue; - } - if (networkstatus_check_voter_signature(consensus, voter, cert) < 0) { - smartlist_add(need_certs_from, voter); - ++n_missing_key; - continue; + SMARTLIST_FOREACH_BEGIN(consensus->voters, networkstatus_voter_info_t *, + voter) { + int good_here = 0; + int bad_here = 0; + int unknown_here = 0; + int missing_key_here = 0, dl_failed_key_here = 0; + SMARTLIST_FOREACH_BEGIN(voter->sigs, document_signature_t *, sig) { + if (!sig->good_signature && !sig->bad_signature && + sig->signature) { + /* we can try to check the signature. */ + int is_v3_auth = trusteddirserver_get_by_v3_auth_digest( + sig->identity_digest) != NULL; + authority_cert_t *cert = + authority_cert_get_by_digests(sig->identity_digest, + sig->signing_key_digest); + tor_assert(!memcmp(sig->identity_digest, voter->identity_digest, + DIGEST_LEN)); + + if (!is_v3_auth) { + smartlist_add(unrecognized, voter); + ++unknown_here; + continue; + } else if (!cert || cert->expires < now) { + smartlist_add(need_certs_from, voter); + ++missing_key_here; + if (authority_cert_dl_looks_uncertain(sig->identity_digest)) + ++dl_failed_key_here; + continue; + } + if (networkstatus_check_document_signature(consensus, sig, cert) < 0) { + smartlist_add(need_certs_from, voter); + ++missing_key_here; + if (authority_cert_dl_looks_uncertain(sig->identity_digest)) + ++dl_failed_key_here; + continue; + } } - } - if (voter->good_signature) + if (sig->good_signature) + ++good_here; + else if (sig->bad_signature) + ++bad_here; + } SMARTLIST_FOREACH_END(sig); + if (good_here) ++n_good; - else if (voter->bad_signature) + else if (bad_here) ++n_bad; - else + else if (missing_key_here) { + ++n_missing_key; + if (dl_failed_key_here) + ++n_dl_failed_key; + } else if (unknown_here) { + ++n_unknown; + } else { ++n_no_signature; - }); + } + } SMARTLIST_FOREACH_END(voter); /* Now see whether we're missing any voters entirely. */ SMARTLIST_FOREACH(router_get_trusted_dir_servers(), @@ -443,39 +545,71 @@ networkstatus_check_consensus_signature(networkstatus_t *consensus, smartlist_add(missing_authorities, ds); }); - if (warn > 1 || (warn >= 0 && n_good < n_required)) + if (warn > 1 || (warn >= 0 && + (n_good + n_missing_key - n_dl_failed_key < n_required))) { severity = LOG_WARN; - else + } else { severity = LOG_INFO; + } if (warn >= 0) { SMARTLIST_FOREACH(unrecognized, networkstatus_voter_info_t *, voter, { - log_info(LD_DIR, "Consensus includes unrecognized authority '%s' " - "at %s:%d (contact %s; identity %s)", + log(severity, LD_DIR, "Consensus includes unrecognized authority " + "'%s' at %s:%d (contact %s; identity %s)", voter->nickname, voter->address, (int)voter->dir_port, voter->contact?voter->contact:"n/a", hex_str(voter->identity_digest, DIGEST_LEN)); }); SMARTLIST_FOREACH(need_certs_from, networkstatus_voter_info_t *, voter, { - log_info(LD_DIR, "Looks like we need to download a new certificate " - "from authority '%s' at %s:%d (contact %s; identity %s)", + log(severity, LD_DIR, "Looks like we need to download a new " + "certificate from authority '%s' at %s:%d (contact %s; " + "identity %s)", voter->nickname, voter->address, (int)voter->dir_port, voter->contact?voter->contact:"n/a", hex_str(voter->identity_digest, DIGEST_LEN)); }); SMARTLIST_FOREACH(missing_authorities, trusted_dir_server_t *, ds, { - log_info(LD_DIR, "Consensus does not include configured " + log(severity, LD_DIR, "Consensus does not include configured " "authority '%s' at %s:%d (identity %s)", ds->nickname, ds->address, (int)ds->dir_port, hex_str(ds->v3_identity_digest, DIGEST_LEN)); }); - log(severity, LD_DIR, - "%d unknown, %d missing key, %d good, %d bad, %d no signature, " - "%d required", n_unknown, n_missing_key, n_good, n_bad, - n_no_signature, n_required); + { + smartlist_t *sl = smartlist_create(); + char *cp; + tor_asprintf(&cp, "A consensus needs %d good signatures from recognized " + "authorities for us to accept it. This one has %d.", + n_required, n_good); + smartlist_add(sl,cp); + if (n_no_signature) { + tor_asprintf(&cp, "%d of the authorities we know didn't sign it.", + n_no_signature); + smartlist_add(sl,cp); + } + if (n_unknown) { + tor_asprintf(&cp, "It has %d signatures from authorities we don't " + "recognize.", n_unknown); + smartlist_add(sl,cp); + } + if (n_bad) { + tor_asprintf(&cp, "%d of the signatures on it didn't verify " + "correctly.", n_bad); + smartlist_add(sl,cp); + } + if (n_missing_key) { + tor_asprintf(&cp, "We were unable to check %d of the signatures, " + "because we were missing the keys.", n_missing_key); + smartlist_add(sl,cp); + } + cp = smartlist_join_strings(sl, " ", 0, NULL); + log(severity, LD_DIR, "%s", cp); + tor_free(cp); + SMARTLIST_FOREACH(sl, char *, c, tor_free(c)); + smartlist_free(sl); + } } smartlist_free(unrecognized); @@ -785,8 +919,8 @@ networkstatus_v2_list_clean(time_t now) /** Helper for bsearching a list of routerstatus_t pointers: compare a * digest in the key to the identity digest of a routerstatus_t. */ -static int -_compare_digest_to_routerstatus_entry(const void *_key, const void **_member) +int +compare_digest_to_routerstatus_entry(const void *_key, const void **_member) { const char *key = _key; const routerstatus_t *rs = *_member; @@ -799,7 +933,7 @@ routerstatus_t * networkstatus_v2_find_entry(networkstatus_v2_t *ns, const char *digest) { return smartlist_bsearch(ns->entries, digest, - _compare_digest_to_routerstatus_entry); + compare_digest_to_routerstatus_entry); } /** Return the entry in <b>ns</b> for the identity digest <b>digest</b>, or @@ -808,7 +942,7 @@ routerstatus_t * networkstatus_vote_find_entry(networkstatus_t *ns, const char *digest) { return smartlist_bsearch(ns->routerstatus_list, digest, - _compare_digest_to_routerstatus_entry); + compare_digest_to_routerstatus_entry); } /*XXXX make this static once functions are moved into this file. */ @@ -820,7 +954,7 @@ networkstatus_vote_find_entry_idx(networkstatus_t *ns, const char *digest, int *found_out) { return smartlist_bsearch_idx(ns->routerstatus_list, digest, - _compare_digest_to_routerstatus_entry, + compare_digest_to_routerstatus_entry, found_out); } @@ -873,7 +1007,7 @@ router_get_consensus_status_by_id(const char *digest) if (!current_consensus) return NULL; return smartlist_bsearch(current_consensus->routerstatus_list, digest, - _compare_digest_to_routerstatus_entry); + compare_digest_to_routerstatus_entry); } /** Given a nickname (possibly verbose, possibly a hexadecimal digest), return @@ -1077,28 +1211,30 @@ update_v2_networkstatus_cache_downloads(time_t now) static void update_consensus_networkstatus_downloads(time_t now) { - or_options_t *options = get_options(); + int i; if (!networkstatus_get_live_consensus(now)) time_to_download_next_consensus = now; /* No live consensus? Get one now!*/ if (time_to_download_next_consensus > now) return; /* Wait until the current consensus is older. */ - if (authdir_mode_v3(options)) - return; /* Authorities never fetch a consensus */ - if (!download_status_is_ready(&consensus_dl_status, now, + /* XXXXNM Microdescs: may need to download more types. */ + if (!download_status_is_ready(&consensus_dl_status[FLAV_NS], now, CONSENSUS_NETWORKSTATUS_MAX_DL_TRIES)) return; /* We failed downloading a consensus too recently. */ if (connection_get_by_type_purpose(CONN_TYPE_DIR, DIR_PURPOSE_FETCH_CONSENSUS)) return; /* There's an in-progress download.*/ - if (consensus_waiting_for_certs) { - /* XXXX make sure this doesn't delay sane downloads. */ - if (consensus_waiting_for_certs_set_at + DELAY_WHILE_FETCHING_CERTS > now) - return; /* We're still getting certs for this one. */ - else { - if (!consensus_waiting_for_certs_dl_failed) { - download_status_failed(&consensus_dl_status, 0); - consensus_waiting_for_certs_dl_failed=1; + for (i=0; i < N_CONSENSUS_FLAVORS; ++i) { + consensus_waiting_for_certs_t *waiting = &consensus_waiting_for_certs[i]; + if (waiting->consensus) { + /* XXXX make sure this doesn't delay sane downloads. */ + if (waiting->set_at + DELAY_WHILE_FETCHING_CERTS > now) + return; /* We're still getting certs for this one. */ + else { + if (!waiting->dl_failed) { + download_status_failed(&consensus_dl_status[FLAV_NS], 0); + waiting->dl_failed=1; + } } } } @@ -1114,7 +1250,8 @@ update_consensus_networkstatus_downloads(time_t now) void networkstatus_consensus_download_failed(int status_code) { - download_status_failed(&consensus_dl_status, status_code); + /* XXXXNM Microdescs: may need to handle more types. */ + download_status_failed(&consensus_dl_status[FLAV_NS], status_code); /* Retry immediately, if appropriate. */ update_consensus_networkstatus_downloads(time(NULL)); } @@ -1148,8 +1285,15 @@ update_consensus_networkstatus_fetch_time(time_t now) /* We want to cache the next one at some point after this one * is no longer fresh... */ start = c->fresh_until + min_sec_before_caching; - /* But only in the first half-interval after that. */ - dl_interval = interval/2; + /* Some clients may need the consensus sooner than others. */ + if (options->FetchDirInfoExtraEarly || authdir_mode_v3(options)) { + dl_interval = 60; + if (min_sec_before_caching + dl_interval > interval) + dl_interval = interval/2; + } else { + /* But only in the first half-interval after that. */ + dl_interval = interval/2; + } } else { /* We're an ordinary client or a bridge. Give all the caches enough * time to download the consensus. */ @@ -1168,7 +1312,7 @@ update_consensus_networkstatus_fetch_time(time_t now) } if (dl_interval < 1) dl_interval = 1; - /* We must not try to replace c while it's still the most valid: */ + /* We must not try to replace c while it's still fresh: */ tor_assert(c->fresh_until < start); /* We must download the next one before c is invalid: */ tor_assert(start+dl_interval < c->valid_until); @@ -1189,7 +1333,6 @@ update_consensus_networkstatus_fetch_time(time_t now) time_to_download_next_consensus = now; log_info(LD_DIR, "No live consensus; we should fetch one immediately."); } - } /** Return 1 if there's a reason we shouldn't try any directory @@ -1224,10 +1367,14 @@ update_networkstatus_downloads(time_t now) void update_certificate_downloads(time_t now) { - if (consensus_waiting_for_certs) - authority_certs_fetch_missing(consensus_waiting_for_certs, now); - else - authority_certs_fetch_missing(current_consensus, now); + int i; + for (i = 0; i < N_CONSENSUS_FLAVORS; ++i) { + if (consensus_waiting_for_certs[i].consensus) + authority_certs_fetch_missing(consensus_waiting_for_certs[i].consensus, + now); + } + + authority_certs_fetch_missing(current_consensus, now); } /** Return 1 if we have a consensus but we don't have enough certificates @@ -1235,7 +1382,8 @@ update_certificate_downloads(time_t now) int consensus_is_waiting_for_certs(void) { - return consensus_waiting_for_certs ? 1 : 0; + return consensus_waiting_for_certs[USABLE_CONSENSUS_FLAVOR].consensus + ? 1 : 0; } /** Return the network status with a given identity digest. */ @@ -1404,16 +1552,31 @@ networkstatus_copy_old_consensus_info(networkstatus_t *new_c, * user, and -2 for more serious problems. */ int -networkstatus_set_current_consensus(const char *consensus, unsigned flags) +networkstatus_set_current_consensus(const char *consensus, + const char *flavor, + unsigned flags) { - networkstatus_t *c; + networkstatus_t *c=NULL; int r, result = -1; time_t now = time(NULL); + or_options_t *options = get_options(); char *unverified_fname = NULL, *consensus_fname = NULL; + int flav = networkstatus_parse_flavor_name(flavor); const unsigned from_cache = flags & NSSET_FROM_CACHE; const unsigned was_waiting_for_certs = flags & NSSET_WAS_WAITING_FOR_CERTS; const unsigned dl_certs = !(flags & NSSET_DONT_DOWNLOAD_CERTS); const unsigned accept_obsolete = flags & NSSET_ACCEPT_OBSOLETE; + const unsigned require_flavor = flags & NSSET_REQUIRE_FLAVOR; + const digests_t *current_digests = NULL; + consensus_waiting_for_certs_t *waiting = NULL; + time_t current_valid_after = 0; + int free_consensus = 1; /* Free 'c' at the end of the function */ + + if (flav < 0) { + /* XXXX we don't handle unrecognized flavors yet. */ + log_warn(LD_BUG, "Unrecognized consensus flavor %s", flavor); + return -2; + } /* Make sure it's parseable. */ c = networkstatus_parse_vote_from_string(consensus, NULL, NS_TYPE_CONSENSUS); @@ -1423,33 +1586,69 @@ networkstatus_set_current_consensus(const char *consensus, unsigned flags) goto done; } + if ((int)c->flavor != flav) { + /* This wasn't the flavor we thought we were getting. */ + if (require_flavor) { + log_warn(LD_DIR, "Got consensus with unexpected flavor %s (wanted %s)", + networkstatus_get_flavor_name(c->flavor), flavor); + goto done; + } + flav = c->flavor; + flavor = networkstatus_get_flavor_name(flav); + } + + if (flav != USABLE_CONSENSUS_FLAVOR && + !directory_caches_dir_info(options)) { + /* This consensus is totally boring to us: we won't use it, and we won't + * serve it. Drop it. */ + goto done; + } + if (from_cache && !accept_obsolete && c->valid_until < now-OLD_ROUTER_DESC_MAX_AGE) { - /* XXX022 when we try to make fallbackconsensus work again, we should + /* XXXX If we try to make fallbackconsensus work again, we should * consider taking this out. Until then, believing obsolete consensuses * is causing more harm than good. See also bug 887. */ - log_info(LD_DIR, "Loaded an obsolete consensus. Discarding."); + log_info(LD_DIR, "Loaded an expired consensus. Discarding."); goto done; } - if (current_consensus && - !memcmp(c->networkstatus_digest, current_consensus->networkstatus_digest, - DIGEST_LEN)) { + if (!strcmp(flavor, "ns")) { + consensus_fname = get_datadir_fname("cached-consensus"); + unverified_fname = get_datadir_fname("unverified-consensus"); + if (current_consensus) { + current_digests = ¤t_consensus->digests; + current_valid_after = current_consensus->valid_after; + } + } else { + cached_dir_t *cur; + char buf[128]; + tor_snprintf(buf, sizeof(buf), "cached-%s-consensus", flavor); + consensus_fname = get_datadir_fname(buf); + tor_snprintf(buf, sizeof(buf), "unverified-%s-consensus", flavor); + unverified_fname = get_datadir_fname(buf); + cur = dirserv_get_consensus(flavor); + if (cur) { + current_digests = &cur->digests; + current_valid_after = cur->published; + } + } + + if (current_digests && + !memcmp(&c->digests, current_digests, sizeof(c->digests))) { /* We already have this one. That's a failure. */ - log_info(LD_DIR, "Got a consensus we already have"); + log_info(LD_DIR, "Got a %s consensus we already have", flavor); goto done; } - if (current_consensus && c->valid_after <= current_consensus->valid_after) { + if (current_valid_after && c->valid_after <= current_valid_after) { /* We have a newer one. There's no point in accepting this one, * even if it's great. */ - log_info(LD_DIR, "Got a consensus at least as old as the one we have"); + log_info(LD_DIR, "Got a %s consensus at least as old as the one we have", + flavor); goto done; } - consensus_fname = get_datadir_fname("cached-consensus"); - unverified_fname = get_datadir_fname("unverified-consensus"); - /* Make sure it's signed enough. */ if ((r=networkstatus_check_consensus_signature(c, 1))<0) { if (r == -1) { @@ -1458,16 +1657,16 @@ networkstatus_set_current_consensus(const char *consensus, unsigned flags) log_info(LD_DIR, "Not enough certificates to check networkstatus consensus"); } - if (!current_consensus || - c->valid_after > current_consensus->valid_after) { - if (consensus_waiting_for_certs) - networkstatus_vote_free(consensus_waiting_for_certs); - tor_free(consensus_waiting_for_certs_body); - consensus_waiting_for_certs = c; - c = NULL; /* Prevent free. */ - consensus_waiting_for_certs_body = tor_strdup(consensus); - consensus_waiting_for_certs_set_at = now; - consensus_waiting_for_certs_dl_failed = 0; + if (!current_valid_after || + c->valid_after > current_valid_after) { + waiting = &consensus_waiting_for_certs[flav]; + networkstatus_vote_free(waiting->consensus); + tor_free(waiting->body); + waiting->consensus = c; + free_consensus = 0; + waiting->body = tor_strdup(consensus); + waiting->set_at = now; + waiting->dl_failed = 0; if (!from_cache) { write_str_to_file(unverified_fname, consensus, 0); } @@ -1496,57 +1695,81 @@ networkstatus_set_current_consensus(const char *consensus, unsigned flags) } } - if (!from_cache) + if (!from_cache && flav == USABLE_CONSENSUS_FLAVOR) control_event_client_status(LOG_NOTICE, "CONSENSUS_ARRIVED"); /* Are we missing any certificates at all? */ if (r != 1 && dl_certs) authority_certs_fetch_missing(c, now); - notify_control_networkstatus_changed(current_consensus, c); + if (flav == USABLE_CONSENSUS_FLAVOR) { + notify_control_networkstatus_changed(current_consensus, c); - if (current_consensus) { - networkstatus_copy_old_consensus_info(c, current_consensus); - networkstatus_vote_free(current_consensus); + if (current_consensus) { + networkstatus_copy_old_consensus_info(c, current_consensus); + networkstatus_vote_free(current_consensus); + /* Defensive programming : we should set current_consensus very soon, + * but we're about to call some stuff in the meantime, and leaving this + * dangling pointer around has proven to be trouble. */ + current_consensus = NULL; + } } - if (consensus_waiting_for_certs && - consensus_waiting_for_certs->valid_after <= c->valid_after) { - networkstatus_vote_free(consensus_waiting_for_certs); - consensus_waiting_for_certs = NULL; - if (consensus != consensus_waiting_for_certs_body) - tor_free(consensus_waiting_for_certs_body); + waiting = &consensus_waiting_for_certs[flav]; + if (waiting->consensus && + waiting->consensus->valid_after <= c->valid_after) { + networkstatus_vote_free(waiting->consensus); + waiting->consensus = NULL; + if (consensus != waiting->body) + tor_free(waiting->body); else - consensus_waiting_for_certs_body = NULL; - consensus_waiting_for_certs_set_at = 0; - consensus_waiting_for_certs_dl_failed = 0; + waiting->body = NULL; + waiting->set_at = 0; + waiting->dl_failed = 0; unlink(unverified_fname); } /* Reset the failure count only if this consensus is actually valid. */ if (c->valid_after <= now && now <= c->valid_until) { - download_status_reset(&consensus_dl_status); + download_status_reset(&consensus_dl_status[flav]); } else { if (!from_cache) - download_status_failed(&consensus_dl_status, 0); + download_status_failed(&consensus_dl_status[flav], 0); } - current_consensus = c; - c = NULL; /* Prevent free. */ + if (flav == USABLE_CONSENSUS_FLAVOR) { + current_consensus = c; + free_consensus = 0; /* Prevent free. */ + + /* XXXXNM Microdescs: needs a non-ns variant. */ + update_consensus_networkstatus_fetch_time(now); + dirvote_recalculate_timing(options, now); + routerstatus_list_update_named_server_map(); + cell_ewma_set_scale_factor(options, current_consensus); - update_consensus_networkstatus_fetch_time(now); - dirvote_recalculate_timing(get_options(), now); - routerstatus_list_update_named_server_map(); + /* XXXX023 this call might be unnecessary here: can changing the + * current consensus really alter our view of any OR's rate limits? */ + connection_or_update_token_buckets(get_connection_array(), options); + + circuit_build_times_new_consensus_params(&circ_times, current_consensus); + } + + if (directory_caches_dir_info(options)) { + dirserv_set_cached_consensus_networkstatus(consensus, + flavor, + &c->digests, + c->valid_after); + } if (!from_cache) { write_str_to_file(consensus_fname, consensus, 0); } - if (directory_caches_dir_info(get_options())) - dirserv_set_cached_networkstatus_v3(consensus, - current_consensus->valid_after); +/** If a consensus appears more than this many seconds before its declared + * valid-after time, declare that our clock is skewed. */ +#define EARLY_CONSENSUS_NOTICE_SKEW 60 - if (ftime_definitely_before(now, current_consensus->valid_after)) { + if (now < current_consensus->valid_after - EARLY_CONSENSUS_NOTICE_SKEW) { char tbuf[ISO_TIME_LEN+1]; char dbuf[64]; long delta = now - current_consensus->valid_after; @@ -1564,7 +1787,7 @@ networkstatus_set_current_consensus(const char *consensus, unsigned flags) result = 0; done: - if (c) + if (free_consensus) networkstatus_vote_free(c); tor_free(consensus_fname); tor_free(unverified_fname); @@ -1576,13 +1799,17 @@ networkstatus_set_current_consensus(const char *consensus, unsigned flags) void networkstatus_note_certs_arrived(void) { - if (consensus_waiting_for_certs) { - if (networkstatus_check_consensus_signature( - consensus_waiting_for_certs, 0)>=0) { + int i; + for (i=0; i<N_CONSENSUS_FLAVORS; ++i) { + consensus_waiting_for_certs_t *waiting = &consensus_waiting_for_certs[i]; + if (!waiting->consensus) + continue; + if (networkstatus_check_consensus_signature(waiting->consensus, 0)>=0) { if (!networkstatus_set_current_consensus( - consensus_waiting_for_certs_body, + waiting->body, + networkstatus_get_flavor_name(i), NSSET_WAS_WAITING_FOR_CERTS)) { - tor_free(consensus_waiting_for_certs_body); + tor_free(waiting->body); } } } @@ -1668,10 +1895,8 @@ download_status_map_update_from_v2_networkstatus(void) v2_download_status_map = digestmap_new(); dl_status = digestmap_new(); - SMARTLIST_FOREACH(networkstatus_v2_list, networkstatus_v2_t *, ns, - { - SMARTLIST_FOREACH(ns->entries, routerstatus_t *, rs, - { + SMARTLIST_FOREACH_BEGIN(networkstatus_v2_list, networkstatus_v2_t *, ns) { + SMARTLIST_FOREACH_BEGIN(ns->entries, routerstatus_t *, rs) { const char *d = rs->descriptor_digest; download_status_t *s; if (digestmap_get(dl_status, d)) @@ -1680,8 +1905,8 @@ download_status_map_update_from_v2_networkstatus(void) s = tor_malloc_zero(sizeof(download_status_t)); } digestmap_set(dl_status, d, s); - }); - }); + } SMARTLIST_FOREACH_END(rs); + } SMARTLIST_FOREACH_END(ns); digestmap_free(v2_download_status_map, _tor_free); v2_download_status_map = dl_status; networkstatus_v2_list_has_changed = 0; @@ -1695,11 +1920,9 @@ routerstatus_list_update_named_server_map(void) if (!current_consensus) return; - if (named_server_map) - strmap_free(named_server_map, _tor_free); + strmap_free(named_server_map, _tor_free); named_server_map = strmap_new(); - if (unnamed_server_map) - strmap_free(unnamed_server_map, NULL); + strmap_free(unnamed_server_map, NULL); unnamed_server_map = strmap_new(); SMARTLIST_FOREACH(current_consensus->routerstatus_list, routerstatus_t *, rs, { @@ -1774,6 +1997,15 @@ routers_update_status_from_consensus_networkstatus(smartlist_t *routers, router->is_bad_directory = rs->is_bad_directory; router->is_bad_exit = rs->is_bad_exit; router->is_hs_dir = rs->is_hs_dir; + } else { + /* If we _are_ an authority, we should check whether this router + * is one that will cause us to need a reachability test. */ + routerinfo_t *old_router = + router_get_by_digest(router->cache_info.identity_digest); + if (old_router != router) { + router->needs_retest_if_added = + dirserv_should_launch_reachability_test(router, old_router); + } } if (router->is_running && ds) { download_status_reset(&ds->v2_ns_dl_status); @@ -1836,7 +2068,7 @@ char * networkstatus_getinfo_helper_single(routerstatus_t *rs) { char buf[RS_ENTRY_LEN+1]; - routerstatus_format_entry(buf, sizeof(buf), rs, NULL, 0, 1); + routerstatus_format_entry(buf, sizeof(buf), rs, NULL, NS_CONTROL_PORT); return tor_strdup(buf); } @@ -1873,7 +2105,7 @@ networkstatus_getinfo_by_purpose(const char *purpose_string, time_t now) if (bridge_auth && ri->purpose == ROUTER_PURPOSE_BRIDGE) dirserv_set_router_is_running(ri, now); /* then generate and write out status lines for each of them */ - set_routerstatus_from_routerinfo(&rs, ri, now, 0, 0, 0, 0); + set_routerstatus_from_routerinfo(&rs, ri, now, 0, 0, 0); smartlist_add(statuses, networkstatus_getinfo_helper_single(&rs)); }); @@ -1898,35 +2130,118 @@ networkstatus_dump_bridge_status_to_file(time_t now) tor_free(status); } +static int32_t +get_net_param_from_list(smartlist_t *net_params, const char *param_name, + int32_t default_val, int32_t min_val, int32_t max_val) +{ + int32_t res = default_val; + size_t name_len = strlen(param_name); + + tor_assert(max_val > min_val); + tor_assert(min_val <= default_val); + tor_assert(max_val >= default_val); + + SMARTLIST_FOREACH_BEGIN(net_params, const char *, p) { + if (!strcmpstart(p, param_name) && p[name_len] == '=') { + int ok=0; + long v = tor_parse_long(p+name_len+1, 10, INT32_MIN, + INT32_MAX, &ok, NULL); + if (ok) { + res = (int32_t) v; + break; + } + } + } SMARTLIST_FOREACH_END(p); + + if (res < min_val) { + log_warn(LD_DIR, "Consensus parameter %s is too small. Got %d, raising to " + "%d.", param_name, res, min_val); + res = min_val; + } else if (res > max_val) { + log_warn(LD_DIR, "Consensus parameter %s is too large. Got %d, capping to " + "%d.", param_name, res, max_val); + res = max_val; + } + + return res; +} + /** Return the value of a integer parameter from the networkstatus <b>ns</b> * whose name is <b>param_name</b>. If <b>ns</b> is NULL, try loading the * latest consensus ourselves. Return <b>default_val</b> if no latest - * consensus, or if it has no parameter called <b>param_name</b>. */ + * consensus, or if it has no parameter called <b>param_name</b>. + * Make sure the value parsed from the consensus is at least + * <b>min_val</b> and at most <b>max_val</b> and raise/cap the parsed value + * if necessary. */ int32_t networkstatus_get_param(networkstatus_t *ns, const char *param_name, - int32_t default_val) + int32_t default_val, int32_t min_val, int32_t max_val) { - size_t name_len; - if (!ns) /* if they pass in null, go find it ourselves */ ns = networkstatus_get_latest_consensus(); if (!ns || !ns->net_params) return default_val; - name_len = strlen(param_name); + return get_net_param_from_list(ns->net_params, param_name, + default_val, min_val, max_val); +} - SMARTLIST_FOREACH_BEGIN(ns->net_params, const char *, p) { - if (!strcmpstart(p, param_name) && p[name_len] == '=') { - int ok=0; - long v = tor_parse_long(p+name_len+1, 10, INT32_MIN, INT32_MAX, &ok, - NULL); - if (ok) - return (int32_t) v; - } - } SMARTLIST_FOREACH_END(p); +/** Return the value of a integer bw weight parameter from the networkstatus + * <b>ns</b> whose name is <b>weight_name</b>. If <b>ns</b> is NULL, try + * loading the latest consensus ourselves. Return <b>default_val</b> if no + * latest consensus, or if it has no parameter called <b>weight_name</b>. */ +int32_t +networkstatus_get_bw_weight(networkstatus_t *ns, const char *weight_name, + int32_t default_val) +{ + int32_t param; + int max; + if (!ns) /* if they pass in null, go find it ourselves */ + ns = networkstatus_get_latest_consensus(); + + if (!ns || !ns->weight_params) + return default_val; + + max = circuit_build_times_get_bw_scale(ns); + param = get_net_param_from_list(ns->weight_params, weight_name, + default_val, -1, + BW_MAX_WEIGHT_SCALE); + if (param > max) { + log_warn(LD_DIR, "Value of consensus weight %s was too large, capping " + "to %d", weight_name, max); + param = max; + } + return param; +} + +/** Return the name of the consensus flavor <b>flav</b> as used to identify + * the flavor in directory documents. */ +const char * +networkstatus_get_flavor_name(consensus_flavor_t flav) +{ + switch (flav) { + case FLAV_NS: + return "ns"; + case FLAV_MICRODESC: + return "microdesc"; + default: + tor_fragile_assert(); + return "??"; + } +} - return default_val; +/** Return the consensus_flavor_t value for the flavor called <b>flavname</b>, + * or -1 if the flavor is not recognized. */ +int +networkstatus_parse_flavor_name(const char *flavname) +{ + if (!strcmp(flavname, "ns")) + return FLAV_NS; + else if (!strcmp(flavname, "microdesc")) + return FLAV_MICRODESC; + else + return -1; } /** If <b>question</b> is a string beginning with "ns/" in a format the @@ -1935,7 +2250,8 @@ networkstatus_get_param(networkstatus_t *ns, const char *param_name, * ORs. Return 0 on success, -1 on unrecognized question format. */ int getinfo_helper_networkstatus(control_connection_t *conn, - const char *question, char **answer) + const char *question, char **answer, + const char **errmsg) { routerstatus_t *status; (void) conn; @@ -1959,8 +2275,10 @@ getinfo_helper_networkstatus(control_connection_t *conn, } else if (!strcmpstart(question, "ns/id/")) { char d[DIGEST_LEN]; - if (base16_decode(d, DIGEST_LEN, question+6, strlen(question+6))) + if (base16_decode(d, DIGEST_LEN, question+6, strlen(question+6))) { + *errmsg = "Data not decodeable as hex"; return -1; + } status = router_get_consensus_status_by_id(d); } else if (!strcmpstart(question, "ns/name/")) { status = router_get_consensus_status_by_nickname(question+8, 0); @@ -1968,7 +2286,7 @@ getinfo_helper_networkstatus(control_connection_t *conn, *answer = networkstatus_getinfo_by_purpose(question+11, time(NULL)); return *answer ? 0 : -1; } else { - return -1; + return 0; } if (status) @@ -1980,30 +2298,29 @@ getinfo_helper_networkstatus(control_connection_t *conn, void networkstatus_free_all(void) { + int i; if (networkstatus_v2_list) { SMARTLIST_FOREACH(networkstatus_v2_list, networkstatus_v2_t *, ns, networkstatus_v2_free(ns)); smartlist_free(networkstatus_v2_list); networkstatus_v2_list = NULL; } - if (v2_download_status_map) { - digestmap_free(v2_download_status_map, _tor_free); - v2_download_status_map = NULL; - } - if (current_consensus) { - networkstatus_vote_free(current_consensus); - current_consensus = NULL; - } - if (consensus_waiting_for_certs) { - networkstatus_vote_free(consensus_waiting_for_certs); - consensus_waiting_for_certs = NULL; - } - tor_free(consensus_waiting_for_certs_body); - if (named_server_map) { - strmap_free(named_server_map, _tor_free); - } - if (unnamed_server_map) { - strmap_free(unnamed_server_map, NULL); + + digestmap_free(v2_download_status_map, _tor_free); + v2_download_status_map = NULL; + networkstatus_vote_free(current_consensus); + current_consensus = NULL; + + for (i=0; i < N_CONSENSUS_FLAVORS; ++i) { + consensus_waiting_for_certs_t *waiting = &consensus_waiting_for_certs[i]; + if (waiting->consensus) { + networkstatus_vote_free(waiting->consensus); + waiting->consensus = NULL; + } + tor_free(waiting->body); } + + strmap_free(named_server_map, _tor_free); + strmap_free(unnamed_server_map, NULL); } diff --git a/src/or/networkstatus.h b/src/or/networkstatus.h new file mode 100644 index 0000000000..ec2e8f884d --- /dev/null +++ b/src/or/networkstatus.h @@ -0,0 +1,99 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file networkstatus.h + * \brief Header file for networkstatus.c. + **/ + +#ifndef _TOR_NETWORKSTATUS_H +#define _TOR_NETWORKSTATUS_H + +/** How old do we allow a v2 network-status to get before removing it + * completely? */ +#define MAX_NETWORKSTATUS_AGE (10*24*60*60) + +void networkstatus_reset_warnings(void); +void networkstatus_reset_download_failures(void); +int router_reload_v2_networkstatus(void); +int router_reload_consensus_networkstatus(void); +void routerstatus_free(routerstatus_t *rs); +void networkstatus_v2_free(networkstatus_v2_t *ns); +void networkstatus_vote_free(networkstatus_t *ns); +networkstatus_voter_info_t *networkstatus_get_voter_by_id( + networkstatus_t *vote, + const char *identity); +int networkstatus_check_consensus_signature(networkstatus_t *consensus, + int warn); +int networkstatus_check_document_signature(const networkstatus_t *consensus, + document_signature_t *sig, + const authority_cert_t *cert); +char *networkstatus_get_cache_filename(const char *identity_digest); +int router_set_networkstatus_v2(const char *s, time_t arrived_at, + v2_networkstatus_source_t source, + smartlist_t *requested_fingerprints); +void networkstatus_v2_list_clean(time_t now); +int compare_digest_to_routerstatus_entry(const void *_key, + const void **_member); +routerstatus_t *networkstatus_v2_find_entry(networkstatus_v2_t *ns, + const char *digest); +routerstatus_t *networkstatus_vote_find_entry(networkstatus_t *ns, + const char *digest); +int networkstatus_vote_find_entry_idx(networkstatus_t *ns, + const char *digest, int *found_out); +const smartlist_t *networkstatus_get_v2_list(void); +download_status_t *router_get_dl_status_by_descriptor_digest(const char *d); +routerstatus_t *router_get_consensus_status_by_id(const char *digest); +routerstatus_t *router_get_consensus_status_by_descriptor_digest( + const char *digest); +routerstatus_t *router_get_consensus_status_by_nickname(const char *nickname, + int warn_if_unnamed); +const char *networkstatus_get_router_digest_by_nickname(const char *nickname); +int networkstatus_nickname_is_unnamed(const char *nickname); +void networkstatus_consensus_download_failed(int status_code); +void update_consensus_networkstatus_fetch_time(time_t now); +int should_delay_dir_fetches(or_options_t *options); +void update_networkstatus_downloads(time_t now); +void update_certificate_downloads(time_t now); +int consensus_is_waiting_for_certs(void); +networkstatus_v2_t *networkstatus_v2_get_by_digest(const char *digest); +networkstatus_t *networkstatus_get_latest_consensus(void); +networkstatus_t *networkstatus_get_live_consensus(time_t now); +networkstatus_t *networkstatus_get_reasonably_live_consensus(time_t now); +#define NSSET_FROM_CACHE 1 +#define NSSET_WAS_WAITING_FOR_CERTS 2 +#define NSSET_DONT_DOWNLOAD_CERTS 4 +#define NSSET_ACCEPT_OBSOLETE 8 +#define NSSET_REQUIRE_FLAVOR 16 +int networkstatus_set_current_consensus(const char *consensus, + const char *flavor, + unsigned flags); +void networkstatus_note_certs_arrived(void); +void routers_update_all_from_networkstatus(time_t now, int dir_version); +void routers_update_status_from_consensus_networkstatus(smartlist_t *routers, + int reset_failures); +void signed_descs_update_status_from_consensus_networkstatus( + smartlist_t *descs); + +char *networkstatus_getinfo_helper_single(routerstatus_t *rs); +char *networkstatus_getinfo_by_purpose(const char *purpose_string, time_t now); +void networkstatus_dump_bridge_status_to_file(time_t now); +int32_t networkstatus_get_param(networkstatus_t *ns, const char *param_name, + int32_t default_val, int32_t min_val, + int32_t max_val); +int getinfo_helper_networkstatus(control_connection_t *conn, + const char *question, char **answer, + const char **errmsg); +int32_t networkstatus_get_bw_weight(networkstatus_t *ns, const char *weight, + int32_t default_val); +const char *networkstatus_get_flavor_name(consensus_flavor_t flav); +int networkstatus_parse_flavor_name(const char *flavname); +void document_signature_free(document_signature_t *sig); +document_signature_t *document_signature_dup(const document_signature_t *sig); +void networkstatus_free_all(void); + +#endif + diff --git a/src/or/ntmain.c b/src/or/ntmain.c index 4dcf9b3763..b2fee648cc 100644 --- a/src/or/ntmain.c +++ b/src/or/ntmain.c @@ -5,13 +5,22 @@ #define MAIN_PRIVATE #include "or.h" - -#include <tchar.h> -#define GENSRV_SERVICENAME TEXT("tor") -#define GENSRV_DISPLAYNAME TEXT("Tor Win32 Service") +#include "config.h" +#include "main.h" +#include "ntmain.h" + +#ifdef HAVE_EVENT2_EVENT_H +#include <event2/event.h> +#else +#include <event.h> +#endif + +#include <windows.h> +#define GENSRV_SERVICENAME "tor" +#define GENSRV_DISPLAYNAME "Tor Win32 Service" #define GENSRV_DESCRIPTION \ - TEXT("Provides an anonymous Internet communication system") -#define GENSRV_USERACCT TEXT("NT AUTHORITY\\LocalService") + "Provides an anonymous Internet communication system" +#define GENSRV_USERACCT "NT AUTHORITY\\LocalService" // Cheating: using the pre-defined error codes, tricks Windows into displaying // a semi-related human-readable error message if startup fails as @@ -27,7 +36,6 @@ static SERVICE_STATUS_HANDLE hStatus; * to the NT service functions. */ static char **backup_argv; static int backup_argc; -static char* nt_strerror(uint32_t errnum); static void nt_service_control(DWORD request); static void nt_service_body(int argc, char **argv); @@ -46,6 +54,11 @@ static int nt_service_cmd_stop(void); struct service_fns { int loaded; + /** @{ */ + /** Function pointers for Windows API functions related to service + * management. These are NULL, or they point to the . They're set by + * calling the LOAD macro below. */ + BOOL (WINAPI *ChangeServiceConfig2A_fn)( SC_HANDLE hService, DWORD dwInfoLevel, @@ -61,30 +74,30 @@ struct service_fns { SC_HANDLE (WINAPI *CreateServiceA_fn)( SC_HANDLE hSCManager, - LPCTSTR lpServiceName, - LPCTSTR lpDisplayName, + LPCSTR lpServiceName, + LPCSTR lpDisplayName, DWORD dwDesiredAccess, DWORD dwServiceType, DWORD dwStartType, DWORD dwErrorControl, - LPCTSTR lpBinaryPathName, - LPCTSTR lpLoadOrderGroup, + LPCSTR lpBinaryPathName, + LPCSTR lpLoadOrderGroup, LPDWORD lpdwTagId, - LPCTSTR lpDependencies, - LPCTSTR lpServiceStartName, - LPCTSTR lpPassword); + LPCSTR lpDependencies, + LPCSTR lpServiceStartName, + LPCSTR lpPassword); BOOL (WINAPI *DeleteService_fn)( SC_HANDLE hService); SC_HANDLE (WINAPI *OpenSCManagerA_fn)( - LPCTSTR lpMachineName, - LPCTSTR lpDatabaseName, + LPCSTR lpMachineName, + LPCSTR lpDatabaseName, DWORD dwDesiredAccess); SC_HANDLE (WINAPI *OpenServiceA_fn)( SC_HANDLE hSCManager, - LPCTSTR lpServiceName, + LPCSTR lpServiceName, DWORD dwDesiredAccess); BOOL (WINAPI *QueryServiceStatus_fn)( @@ -92,28 +105,29 @@ struct service_fns { LPSERVICE_STATUS lpServiceStatus); SERVICE_STATUS_HANDLE (WINAPI *RegisterServiceCtrlHandlerA_fn)( - LPCTSTR lpServiceName, + LPCSTR lpServiceName, LPHANDLER_FUNCTION lpHandlerProc); BOOL (WINAPI *SetServiceStatus_fn)(SERVICE_STATUS_HANDLE, LPSERVICE_STATUS); BOOL (WINAPI *StartServiceCtrlDispatcherA_fn)( - const SERVICE_TABLE_ENTRY* lpServiceTable); + const SERVICE_TABLE_ENTRYA* lpServiceTable); BOOL (WINAPI *StartServiceA_fn)( SC_HANDLE hService, DWORD dwNumServiceArgs, - LPCTSTR* lpServiceArgVectors); + LPCSTR* lpServiceArgVectors); BOOL (WINAPI *LookupAccountNameA_fn)( - LPCTSTR lpSystemName, - LPCTSTR lpAccountName, + LPCSTR lpSystemName, + LPCSTR lpAccountName, PSID Sid, LPDWORD cbSid, LPTSTR ReferencedDomainName, LPDWORD cchReferencedDomainName, PSID_NAME_USE peUse); + /** @} */ } service_fns = { 0, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, @@ -130,13 +144,16 @@ nt_service_loadlibrary(void) if (service_fns.loaded) return; - /* XXXX Possibly, we should hardcode the location of this DLL. */ - if (!(library = LoadLibrary("advapi32.dll"))) { + if (!(library = load_windows_system_library(TEXT("advapi32.dll")))) { log_err(LD_GENERAL, "Couldn't open advapi32.dll. Are you trying to use " "NT services on Windows 98? That doesn't work."); goto err; } +/* Helper macro: try to load a function named <b>f</b> from "library" into + * service_functions.<b>f</b>_fn. On failure, log an error message, and goto + * err. + */ #define LOAD(f) STMT_BEGIN \ if (!(fn = GetProcAddress(library, #f))) { \ log_err(LD_BUG, \ @@ -218,7 +235,7 @@ nt_service_control(DWORD request) log_notice(LD_GENERAL, "Got stop/shutdown request; shutting down cleanly."); service_status.dwCurrentState = SERVICE_STOP_PENDING; - event_loopexit(&exit_now); + event_base_loopexit(tor_libevent_get_base(), &exit_now); return; } service_fns.SetServiceStatus_fn(hStatus, &service_status); @@ -275,20 +292,20 @@ nt_service_body(int argc, char **argv) static void nt_service_main(void) { - SERVICE_TABLE_ENTRY table[2]; + SERVICE_TABLE_ENTRYA table[2]; DWORD result = 0; char *errmsg; nt_service_loadlibrary(); table[0].lpServiceName = (char*)GENSRV_SERVICENAME; - table[0].lpServiceProc = (LPSERVICE_MAIN_FUNCTION)nt_service_body; + table[0].lpServiceProc = (LPSERVICE_MAIN_FUNCTIONA)nt_service_body; table[1].lpServiceName = NULL; table[1].lpServiceProc = NULL; if (!service_fns.StartServiceCtrlDispatcherA_fn(table)) { result = GetLastError(); - errmsg = nt_strerror(result); + errmsg = format_win32_error(result); printf("Service error %d : %s\n", (int) result, errmsg); - LocalFree(errmsg); + tor_free(errmsg); if (result == ERROR_FAILED_SERVICE_CONTROLLER_CONNECT) { if (tor_init(backup_argc, backup_argv) < 0) return; @@ -323,9 +340,9 @@ nt_service_open_scm(void) nt_service_loadlibrary(); if ((hSCManager = service_fns.OpenSCManagerA_fn( NULL, NULL, SC_MANAGER_CREATE_SERVICE)) == NULL) { - errmsg = nt_strerror(GetLastError()); + errmsg = format_win32_error(GetLastError()); printf("OpenSCManager() failed : %s\n", errmsg); - LocalFree(errmsg); + tor_free(errmsg); } return hSCManager; } @@ -340,9 +357,9 @@ nt_service_open(SC_HANDLE hSCManager) nt_service_loadlibrary(); if ((hService = service_fns.OpenServiceA_fn(hSCManager, GENSRV_SERVICENAME, SERVICE_ALL_ACCESS)) == NULL) { - errmsg = nt_strerror(GetLastError()); + errmsg = format_win32_error(GetLastError()); printf("OpenService() failed : %s\n", errmsg); - LocalFree(errmsg); + tor_free(errmsg); } return hService; } @@ -374,14 +391,14 @@ nt_service_start(SC_HANDLE hService) printf("Service started successfully\n"); return 0; } else { - errmsg = nt_strerror(service_status.dwWin32ExitCode); + errmsg = format_win32_error(service_status.dwWin32ExitCode); printf("Service failed to start : %s\n", errmsg); - LocalFree(errmsg); + tor_free(errmsg); } } else { - errmsg = nt_strerror(GetLastError()); + errmsg = format_win32_error(GetLastError()); printf("StartService() failed : %s\n", errmsg); - LocalFree(errmsg); + tor_free(errmsg); } return -1; } @@ -418,14 +435,14 @@ nt_service_stop(SC_HANDLE hService) } else if (wait_time == MAX_SERVICE_WAIT_TIME) { printf("Service did not stop within %d seconds.\n", wait_time); } else { - errmsg = nt_strerror(GetLastError()); + errmsg = format_win32_error(GetLastError()); printf("QueryServiceStatus() failed : %s\n",errmsg); - LocalFree(errmsg); + tor_free(errmsg); } } else { - errmsg = nt_strerror(GetLastError()); + errmsg = format_win32_error(GetLastError()); printf("ControlService() failed : %s\n", errmsg); - LocalFree(errmsg); + tor_free(errmsg); } return -1; } @@ -439,6 +456,7 @@ static char * nt_service_command_line(int *using_default_torrc) { TCHAR tor_exe[MAX_PATH+1]; + char tor_exe_ascii[MAX_PATH+1]; char *command, *options=NULL; smartlist_t *sl; int i, cmdlen; @@ -464,18 +482,25 @@ nt_service_command_line(int *using_default_torrc) options = smartlist_join_strings(sl,"\" \"",0,NULL); smartlist_free(sl); +#ifdef UNICODE + wcstombs(tor_exe_ascii, tor_exe, sizeof(tor_exe_ascii)); +#else + strlcpy(tor_exe_ascii, tor_exe, sizeof(tor_exe_ascii)); +#endif + /* Allocate a string for the NT service command line */ - cmdlen = strlen(tor_exe) + (options?strlen(options):0) + 32; + cmdlen = strlen(tor_exe_ascii) + (options?strlen(options):0) + 32; command = tor_malloc(cmdlen); /* Format the service command */ if (options) { if (tor_snprintf(command, cmdlen, "\"%s\" --nt-service \"%s\"", - tor_exe, options)<0) { + tor_exe_ascii, options)<0) { tor_free(command); /* sets command to NULL. */ } } else { /* ! options */ - if (tor_snprintf(command, cmdlen, "\"%s\" --nt-service", tor_exe)<0) { + if (tor_snprintf(command, cmdlen, "\"%s\" --nt-service", + tor_exe_ascii)<0) { tor_free(command); /* sets command to NULL. */ } } @@ -500,10 +525,10 @@ nt_service_install(int argc, char **argv) SC_HANDLE hSCManager = NULL; SC_HANDLE hService = NULL; - SERVICE_DESCRIPTION sdBuff; + SERVICE_DESCRIPTIONA sdBuff; char *command; char *errmsg; - const char *user_acct = GENSRV_USERACCT; + const char *user_acct = NULL; const char *password = ""; int i; OSVERSIONINFOEX info; @@ -547,13 +572,12 @@ nt_service_install(int argc, char **argv) is_win2k_or_worse = 1; } - if (user_acct == GENSRV_USERACCT) { + if (!user_acct) { if (is_win2k_or_worse) { /* On Win2k, there is no LocalService account, so we actually need to * fall back on NULL (the system account). */ printf("Running on Win2K or earlier, so the LocalService account " "doesn't exist. Falling back to SYSTEM account.\n"); - user_acct = NULL; } else { /* Genericity is apparently _so_ last year in Redmond, where some * accounts are accounts that you can look up, and some accounts @@ -562,6 +586,7 @@ nt_service_install(int argc, char **argv) */ printf("Running on a Post-Win2K OS, so we'll assume that the " "LocalService account exists.\n"); + user_acct = GENSRV_USERACCT; } } else if (0 && service_fns.LookupAccountNameA_fn(NULL, // On this system user_acct, @@ -590,10 +615,10 @@ nt_service_install(int argc, char **argv) SERVICE_AUTO_START, SERVICE_ERROR_IGNORE, command, NULL, NULL, NULL, user_acct, password)) == NULL) { - errmsg = nt_strerror(GetLastError()); + errmsg = format_win32_error(GetLastError()); printf("CreateService() failed : %s\n", errmsg); service_fns.CloseServiceHandle_fn(hSCManager); - LocalFree(errmsg); + tor_free(errmsg); tor_free(command); return -1; } @@ -634,9 +659,9 @@ nt_service_remove(void) nt_service_stop(hService); if (service_fns.DeleteService_fn(hService) == FALSE) { - errmsg = nt_strerror(GetLastError()); + errmsg = format_win32_error(GetLastError()); printf("DeleteService() failed : %s\n", errmsg); - LocalFree(errmsg); + tor_free(errmsg); service_fns.CloseServiceHandle_fn(hService); service_fns.CloseServiceHandle_fn(hSCManager); return -1; @@ -693,20 +718,6 @@ nt_service_cmd_stop(void) return stop; } -/** Given a Win32 error code, this attempts to make Windows - * return a human-readable error message. The char* returned - * is allocated by Windows, but should be freed with LocalFree() - * when finished with it. */ -static char* -nt_strerror(uint32_t errnum) -{ - char *msgbuf; - FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM, - NULL, errnum, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), - (LPSTR)&msgbuf, 0, NULL); - return msgbuf; -} - int nt_service_parse_options(int argc, char **argv, int *should_exit) { diff --git a/src/or/ntmain.h b/src/or/ntmain.h new file mode 100644 index 0000000000..acd0e1d7eb --- /dev/null +++ b/src/or/ntmain.h @@ -0,0 +1,30 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file ntmain.h + * \brief Header file for ntmain.c. + **/ + +#ifndef _TOR_NTMAIN_H +#define _TOR_NTMAIN_H + +#ifdef MS_WINDOWS +#if !defined (WINCE) +#define NT_SERVICE +#endif +#endif + +#ifdef NT_SERVICE +int nt_service_parse_options(int argc, char **argv, int *should_exit); +int nt_service_is_stopping(void); +void nt_service_set_state(DWORD state); +#else +#define nt_service_is_stopping() 0 +#endif + +#endif + diff --git a/src/or/onion.c b/src/or/onion.c index e455a52637..e1d10a60bb 100644 --- a/src/or/onion.c +++ b/src/or/onion.c @@ -11,6 +11,10 @@ **/ #include "or.h" +#include "circuitlist.h" +#include "config.h" +#include "onion.h" +#include "rephist.h" /** Type for a linked list of circuits that are waiting for a free CPU worker * to process a waiting onion handshake. */ @@ -58,11 +62,18 @@ onion_pending_add(or_circuit_t *circ, char *onionskin) tor_assert(!ol_tail->next); if (ol_length >= get_options()->MaxOnionsPending) { - log_warn(LD_GENERAL, - "Your computer is too slow to handle this many circuit " - "creation requests! Please consider using the " - "MaxAdvertisedBandwidth config option or choosing a more " - "restricted exit policy."); +#define WARN_TOO_MANY_CIRC_CREATIONS_INTERVAL (60) + static ratelim_t last_warned = + RATELIM_INIT(WARN_TOO_MANY_CIRC_CREATIONS_INTERVAL); + char *m; + if ((m = rate_limit_log(&last_warned, approx_time()))) { + log_warn(LD_GENERAL, + "Your computer is too slow to handle this many circuit " + "creation requests! Please consider using the " + "MaxAdvertisedBandwidth config option or choosing a more " + "restricted exit policy.%s",m); + tor_free(m); + } tor_free(tmp); return -1; } @@ -248,6 +259,10 @@ onion_skin_server_handshake(const char *onion_skin, /*ONIONSKIN_CHALLENGE_LEN*/ } dh = crypto_dh_new(DH_TYPE_CIRCUIT); + if (!dh) { + log_warn(LD_BUG, "Couldn't allocate DH key"); + goto err; + } if (crypto_dh_get_public(dh, handshake_reply_out, DH_KEY_LEN)) { log_info(LD_GENERAL, "crypto_dh_get_public failed."); goto err; @@ -255,8 +270,9 @@ onion_skin_server_handshake(const char *onion_skin, /*ONIONSKIN_CHALLENGE_LEN*/ key_material_len = DIGEST_LEN+key_out_len; key_material = tor_malloc(key_material_len); - len = crypto_dh_compute_secret(dh, challenge, DH_KEY_LEN, - key_material, key_material_len); + len = crypto_dh_compute_secret(LOG_PROTOCOL_WARN, dh, challenge, + DH_KEY_LEN, key_material, + key_material_len); if (len < 0) { log_info(LD_GENERAL, "crypto_dh_compute_secret failed."); goto err; @@ -306,8 +322,9 @@ onion_skin_client_handshake(crypto_dh_env_t *handshake_state, key_material_len = DIGEST_LEN + key_out_len; key_material = tor_malloc(key_material_len); - len = crypto_dh_compute_secret(handshake_state, handshake_reply, DH_KEY_LEN, - key_material, key_material_len); + len = crypto_dh_compute_secret(LOG_PROTOCOL_WARN, handshake_state, + handshake_reply, DH_KEY_LEN, key_material, + key_material_len); if (len < 0) goto err; diff --git a/src/or/onion.h b/src/or/onion.h new file mode 100644 index 0000000000..7f603b8147 --- /dev/null +++ b/src/or/onion.h @@ -0,0 +1,48 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file onion.h + * \brief Header file for onion.c. + **/ + +#ifndef _TOR_ONION_H +#define _TOR_ONION_H + +int onion_pending_add(or_circuit_t *circ, char *onionskin); +or_circuit_t *onion_next_task(char **onionskin_out); +void onion_pending_remove(or_circuit_t *circ); + +int onion_skin_create(crypto_pk_env_t *router_key, + crypto_dh_env_t **handshake_state_out, + char *onion_skin_out); + +int onion_skin_server_handshake(const char *onion_skin, + crypto_pk_env_t *private_key, + crypto_pk_env_t *prev_private_key, + char *handshake_reply_out, + char *key_out, + size_t key_out_len); + +int onion_skin_client_handshake(crypto_dh_env_t *handshake_state, + const char *handshake_reply, + char *key_out, + size_t key_out_len); + +int fast_server_handshake(const uint8_t *key_in, + uint8_t *handshake_reply_out, + uint8_t *key_out, + size_t key_out_len); + +int fast_client_handshake(const uint8_t *handshake_state, + const uint8_t *handshake_reply_out, + uint8_t *key_out, + size_t key_out_len); + +void clear_pending_onions(void); + +#endif + diff --git a/src/or/or.h b/src/or/or.h index 57e091e9eb..7d354c8fe1 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -20,9 +20,6 @@ #ifndef INSTRUMENT_DOWNLOADS #define INSTRUMENT_DOWNLOADS 1 #endif -#ifndef ENABLE_GEOIP_STATS -#define ENABLE_GEOIP_STATS 1 -#endif #endif #ifdef MS_WINDOWS @@ -62,6 +59,9 @@ #ifdef HAVE_SYS_STAT_H #include <sys/stat.h> #endif +#ifdef HAVE_NETINET_IN_H +#include <netinet/in.h> +#endif #ifdef HAVE_ARPA_INET_H #include <arpa/inet.h> #endif @@ -83,18 +83,15 @@ #define snprintf _snprintf #endif -#include "crypto.h" #include "tortls.h" -#include "log.h" -#include "compat.h" +#include "../common/torlog.h" #include "container.h" -#include "util.h" #include "torgzip.h" #include "address.h" +#include "compat_libevent.h" +#include "ht.h" -#include <event.h> - -/* These signals are defined to help control_signal_act work. +/* These signals are defined to help handle_control_signal work. */ #ifndef SIGHUP #define SIGHUP 1 @@ -161,7 +158,7 @@ #define MAX_DNS_TTL (3*60*60) /** How small can a TTL be before we stop believing it? Provides rudimentary * pinning. */ -#define MIN_DNS_TTL (60) +#define MIN_DNS_TTL 60 /** How often do we rotate onion keys? */ #define MIN_ONION_KEY_LIFETIME (7*24*60*60) @@ -221,6 +218,21 @@ typedef enum { /* !!!! If _CONN_TYPE_MAX is ever over 15, we must grow the type field in * connection_t. */ +/* Proxy client types */ +#define PROXY_NONE 0 +#define PROXY_CONNECT 1 +#define PROXY_SOCKS4 2 +#define PROXY_SOCKS5 3 + +/* Proxy client handshake states */ +#define PROXY_HTTPS_WANT_CONNECT_OK 1 +#define PROXY_SOCKS4_WANT_CONNECT_OK 2 +#define PROXY_SOCKS5_WANT_AUTH_METHOD_NONE 3 +#define PROXY_SOCKS5_WANT_AUTH_METHOD_RFC1929 4 +#define PROXY_SOCKS5_WANT_AUTH_RFC1929_OK 5 +#define PROXY_SOCKS5_WANT_CONNECT_OK 6 +#define PROXY_CONNECTED 7 + /** True iff <b>x</b> is an edge connection. */ #define CONN_IS_EDGE(x) \ ((x)->type == CONN_TYPE_EXIT || (x)->type == CONN_TYPE_AP) @@ -241,26 +253,24 @@ typedef enum { #define _OR_CONN_STATE_MIN 1 /** State for a connection to an OR: waiting for connect() to finish. */ #define OR_CONN_STATE_CONNECTING 1 -/** State for a connection to an OR: waiting for proxy command to flush. */ -#define OR_CONN_STATE_PROXY_FLUSHING 2 -/** State for a connection to an OR: waiting for proxy response. */ -#define OR_CONN_STATE_PROXY_READING 3 +/** State for a connection to an OR: waiting for proxy handshake to complete */ +#define OR_CONN_STATE_PROXY_HANDSHAKING 2 /** State for a connection to an OR or client: SSL is handshaking, not done * yet. */ -#define OR_CONN_STATE_TLS_HANDSHAKING 4 +#define OR_CONN_STATE_TLS_HANDSHAKING 3 /** State for a connection to an OR: We're doing a second SSL handshake for * renegotiation purposes. */ -#define OR_CONN_STATE_TLS_CLIENT_RENEGOTIATING 5 +#define OR_CONN_STATE_TLS_CLIENT_RENEGOTIATING 4 /** State for a connection at an OR: We're waiting for the client to * renegotiate. */ -#define OR_CONN_STATE_TLS_SERVER_RENEGOTIATING 6 +#define OR_CONN_STATE_TLS_SERVER_RENEGOTIATING 5 /** State for a connection to an OR: We're done with our SSL handshake, but we * haven't yet negotiated link protocol versions and sent a netinfo cell. */ -#define OR_CONN_STATE_OR_HANDSHAKING 7 +#define OR_CONN_STATE_OR_HANDSHAKING 6 /** State for a connection to an OR: Ready to send/receive cells. */ -#define OR_CONN_STATE_OPEN 8 -#define _OR_CONN_STATE_MAX 8 +#define OR_CONN_STATE_OPEN 7 +#define _OR_CONN_STATE_MAX 7 #define _EXIT_CONN_STATE_MIN 1 /** State for an exit connection: waiting for response from DNS farm. */ @@ -457,23 +467,23 @@ typedef enum { #define CIRCUIT_PURPOSE_C_REND_READY_INTRO_ACKED 11 /** Client-side circuit purpose: at Alice, rendezvous established. */ #define CIRCUIT_PURPOSE_C_REND_JOINED 12 - -#define _CIRCUIT_PURPOSE_C_MAX 12 - +/** This circuit is used for build time measurement only */ +#define CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT 13 +#define _CIRCUIT_PURPOSE_C_MAX 13 /** Hidden-service-side circuit purpose: at Bob, waiting for introductions. */ -#define CIRCUIT_PURPOSE_S_ESTABLISH_INTRO 13 +#define CIRCUIT_PURPOSE_S_ESTABLISH_INTRO 14 /** Hidden-service-side circuit purpose: at Bob, successfully established * intro. */ -#define CIRCUIT_PURPOSE_S_INTRO 14 +#define CIRCUIT_PURPOSE_S_INTRO 15 /** Hidden-service-side circuit purpose: at Bob, connecting to rend point. */ -#define CIRCUIT_PURPOSE_S_CONNECT_REND 15 +#define CIRCUIT_PURPOSE_S_CONNECT_REND 16 /** Hidden-service-side circuit purpose: at Bob, rendezvous established. */ -#define CIRCUIT_PURPOSE_S_REND_JOINED 16 +#define CIRCUIT_PURPOSE_S_REND_JOINED 17 /** A testing circuit; not meant to be used for actual traffic. */ -#define CIRCUIT_PURPOSE_TESTING 17 +#define CIRCUIT_PURPOSE_TESTING 18 /** A controller made this circuit and Tor should not use it. */ -#define CIRCUIT_PURPOSE_CONTROLLER 18 -#define _CIRCUIT_PURPOSE_MAX 18 +#define CIRCUIT_PURPOSE_CONTROLLER 19 +#define _CIRCUIT_PURPOSE_MAX 19 /** A catch-all for unrecognized purposes. Currently we don't expect * to make or see any circuits with this purpose. */ #define CIRCUIT_PURPOSE_UNKNOWN 255 @@ -483,7 +493,7 @@ typedef enum { #define CIRCUIT_PURPOSE_IS_ORIGIN(p) ((p)>_CIRCUIT_PURPOSE_OR_MAX) /** True iff the circuit purpose <b>p</b> is for a circuit that originated * here to serve as a client. (Hidden services don't count here.) */ -#define CIRCUIT_PURPOSE_IS_CLIENT(p) \ +#define CIRCUIT_PURPOSE_IS_CLIENT(p) \ ((p)> _CIRCUIT_PURPOSE_OR_MAX && \ (p)<=_CIRCUIT_PURPOSE_C_MAX) /** True iff the circuit_t <b>c</b> is actually an origin_circuit_t. */ @@ -573,6 +583,9 @@ typedef enum { /** This is a connection on the NATD port, and the destination IP:Port was * either ill-formed or out-of-range. */ #define END_STREAM_REASON_INVALID_NATD_DEST 261 +/** The target address is in a private network (like 127.0.0.1 or 10.0.0.1); + * you don't want to do that over a randomly chosen exit */ +#define END_STREAM_REASON_PRIVATE_ADDR 262 /** Bitwise-and this value with endreason to mask out all flags. */ #define END_STREAM_REASON_MASK 511 @@ -604,6 +617,10 @@ typedef enum { /* Negative reasons are internal: we never send them in a DESTROY or TRUNCATE * call; they only go to the controller for tracking */ +/** Our post-timeout circuit time measurement period expired. + * We must give up now */ +#define END_CIRC_REASON_MEASUREMENT_EXPIRED -3 + /** We couldn't build a path for this circuit. */ #define END_CIRC_REASON_NOPATH -2 /** Catch-all "other" reason for closing origin circuits. */ @@ -641,10 +658,6 @@ typedef enum { /** Length of a binary-encoded rendezvous service ID. */ #define REND_SERVICE_ID_LEN 10 -/** How long after we receive a hidden service descriptor do we consider - * it fresh? */ -#define NUM_SECONDS_BEFORE_HS_REFETCH (60*15) - /** Time period for which a v2 descriptor will be valid. */ #define REND_TIME_PERIOD_V2_DESC_VALIDITY (24*60*60) @@ -739,12 +752,6 @@ typedef struct rend_data_t { /** Rendezvous cookie used by both, client and service. */ char rend_cookie[REND_COOKIE_LEN]; - - /** Rendezvous descriptor version that is used by a service. Used to - * distinguish introduction and rendezvous points belonging to the same - * rendezvous service ID, but different descriptor versions. - */ - uint8_t rend_desc_version; } rend_data_t; /** Time interval for tracking possible replays of INTRODUCE2 cells. @@ -761,6 +768,8 @@ typedef enum { /** Initial value for both sides of a circuit transmission window when the * circuit is initialized. Measured in cells. */ #define CIRCWINDOW_START 1000 +#define CIRCWINDOW_START_MIN 100 +#define CIRCWINDOW_START_MAX 1000 /** Amount to increment a circuit window when we get a circuit SENDME. */ #define CIRCWINDOW_INCREMENT 100 /** Initial value on both sides of a stream transmission window when the @@ -839,9 +848,13 @@ typedef struct cell_t { /** Parsed variable-length onion routing cell. */ typedef struct var_cell_t { + /** Type of the cell: CELL_VERSIONS, etc. */ uint8_t command; + /** Circuit thich received the cell */ circid_t circ_id; + /** Number of bytes actually stored in <b>payload</b> */ uint16_t payload_len; + /** Payload of this cell */ uint8_t payload[1]; } var_cell_t; @@ -851,12 +864,28 @@ typedef struct packed_cell_t { char body[CELL_NETWORK_SIZE]; /**< Cell as packed for network. */ } packed_cell_t; +/** Number of cells added to a circuit queue including their insertion + * time on 10 millisecond detail; used for buffer statistics. */ +typedef struct insertion_time_elem_t { + struct insertion_time_elem_t *next; /**< Next element in queue. */ + uint32_t insertion_time; /**< When were cells inserted (in 10 ms steps + * starting at 0:00 of the current day)? */ + unsigned counter; /**< How many cells were inserted? */ +} insertion_time_elem_t; + +/** Queue of insertion times. */ +typedef struct insertion_time_queue_t { + struct insertion_time_elem_t *first; /**< First element in queue. */ + struct insertion_time_elem_t *last; /**< Last element in queue. */ +} insertion_time_queue_t; + /** A queue of cells on a circuit, waiting to be added to the * or_connection_t's outbuf. */ typedef struct cell_queue_t { packed_cell_t *head; /**< The first cell, or NULL if the queue is empty. */ packed_cell_t *tail; /**< The last cell, or NULL if the queue is empty. */ int n; /**< The number of cells in the queue. */ + insertion_time_queue_t *insertion_times; /**< Insertion times of cells. */ } cell_queue_t; /** Beginning of a RELAY cell payload. */ @@ -912,7 +941,7 @@ typedef struct connection_t { * again once the bandwidth throttler allows it? */ unsigned int write_blocked_on_bw:1; /**< Boolean: should we start writing * again once the bandwidth throttler allows - * reads? */ + * writes? */ unsigned int hold_open_until_flushed:1; /**< Despite this connection's being * marked for close, do we flush it * before closing it? */ @@ -937,8 +966,11 @@ typedef struct connection_t { * connection. */ unsigned int linked_conn_is_closed:1; - int s; /**< Our socket; -1 if this connection is closed, or has no - * socket. */ + /** CONNECT/SOCKS proxy client handshake state (for outgoing connections). */ + unsigned int proxy_state:4; + + /** Our socket; -1 if this connection is closed, or has no socket. */ + evutil_socket_t s; int conn_array_index; /**< Index into the global connection array. */ struct event *read_event; /**< Libevent event structure. */ struct event *write_event; /**< Libevent event structure. */ @@ -974,12 +1006,14 @@ typedef struct connection_t { /** Unique identifier for this connection on this Tor instance. */ uint64_t global_identifier; - /* XXXX022 move this field, and all the listener-only fields (just + /* XXXX023 move this field, and all the listener-only fields (just socket_family, I think), into a new listener_connection_t subtype. */ /** If the connection is a CONN_TYPE_AP_DNS_LISTENER, this field points * to the evdns_server_port is uses to listen to and answer connections. */ struct evdns_server_port *dns_server_port; + /** Unique ID for measuring tunneled network status requests. */ + uint64_t dirreq_id; } connection_t; /** Stores flags and information related to the portion of a v2 Tor OR @@ -1026,7 +1060,10 @@ typedef struct or_connection_t { * NETINFO cell listed the address we're connected to as recognized. */ unsigned int is_canonical:1; /** True iff this connection shouldn't get any new circs attached to it, - * because the connection is too old, or because there's a better one, etc. + * because the connection is too old, or because there's a better one. + * More generally, this flag is used to note an unhealthy connection; + * for example, if a bad connection fails we shouldn't assume that the + * router itself has a problem. */ unsigned int is_bad_for_new_circs:1; uint8_t link_proto; /**< What protocol version are we using? 0 for @@ -1041,12 +1078,13 @@ typedef struct or_connection_t { time_t timestamp_last_added_nonpadding; /** When did we last add a * non-padding cell to the outbuf? */ - /* bandwidth* and read_bucket only used by ORs in OPEN state: */ + /* bandwidth* and *_bucket only used by ORs in OPEN state: */ int bandwidthrate; /**< Bytes/s added to the bucket. (OPEN ORs only.) */ int bandwidthburst; /**< Max bucket size for this conn. (OPEN ORs only.) */ int read_bucket; /**< When this hits 0, stop receiving. Every second we * add 'bandwidthrate' to this, capping it at * bandwidthburst. (OPEN ORs only) */ + int write_bucket; /**< When this hits 0, stop writing. Like read_bucket. */ int n_circuits; /**< How many circuits use this connection as p_conn or * n_conn ? */ @@ -1054,6 +1092,17 @@ typedef struct or_connection_t { * free up on this connection's outbuf. Every time we pull cells from a * circuit, we advance this pointer to the next circuit in the ring. */ struct circuit_t *active_circuits; + /** Priority queue of cell_ewma_t for circuits with queued cells waiting for + * room to free up on this connection's outbuf. Kept in heap order + * according to EWMA. + * + * This is redundant with active_circuits; if we ever decide only to use the + * cell_ewma algorithm for choosing circuits, we can remove active_circuits. + */ + smartlist_t *active_circuit_pqueue; + /** The tick on which the cell_ewma_ts in active_circuit_pqueue last had + * their ewma values rescaled. */ + unsigned active_circuit_pqueue_last_recalibrated; struct or_connection_t *next_with_same_id; /**< Next connection with same * identity digest as this one. */ } or_connection_t; @@ -1102,6 +1151,13 @@ typedef struct edge_connection_t { * already retried several times. */ uint8_t num_socks_retries; +#define NUM_CIRCUITS_LAUNCHED_THRESHOLD 10 + /** Number of times we've launched a circuit to handle this stream. If + * it gets too high, that could indicate an inconsistency between our + * "launch a circuit to handle this stream" logic and our "attach our + * stream to one of the available circuits" logic. */ + unsigned int num_circuits_launched:4; + /** True iff this connection is for a DNS request only. */ unsigned int is_dns_request:1; @@ -1128,6 +1184,10 @@ typedef struct edge_connection_t { * zero, abandon the associated mapaddress. */ unsigned int chosen_exit_retries:3; + /** True iff this is an AP connection that came from a transparent or + * NATd connection */ + unsigned int is_transparent_ap:1; + /** If this is a DNSPort connection, this field holds the pending DNS * request that we're going to try to answer. */ struct evdns_server_request *dns_server_request; @@ -1150,7 +1210,8 @@ typedef struct dir_connection_t { enum { DIR_SPOOL_NONE=0, DIR_SPOOL_SERVER_BY_DIGEST, DIR_SPOOL_SERVER_BY_FP, DIR_SPOOL_EXTRA_BY_DIGEST, DIR_SPOOL_EXTRA_BY_FP, - DIR_SPOOL_CACHED_DIR, DIR_SPOOL_NETWORKSTATUS + DIR_SPOOL_CACHED_DIR, DIR_SPOOL_NETWORKSTATUS, + DIR_SPOOL_MICRODESC, /* NOTE: if we add another entry, add another bit. */ } dir_spool_src : 3; /** If we're fetching descriptors, what router purpose shall we assign * to them? */ @@ -1178,12 +1239,6 @@ typedef struct control_connection_t { uint32_t event_mask; /**< Bitfield: which events does this controller * care about? */ - unsigned int use_long_names:1; /**< True if we should use long nicknames - * on this (v1) connection. Only settable - * via v1 controllers. */ - /** For control connections only. If set, we send extended info with control - * events as appropriate. */ - unsigned int use_extended_events:1; /** True if we have sent a protocolinfo reply on this connection. */ unsigned int have_sent_protocolinfo:1; @@ -1267,6 +1322,7 @@ typedef struct cached_dir_t { size_t dir_len; /**< Length of <b>dir</b> (not counting its NUL). */ size_t dir_z_len; /**< Length of <b>dir_z</b>. */ time_t published; /**< When was this object published. */ + digests_t digests; /**< Digests of this object (networkstatus only) */ int refcnt; /**< Reference count for this cached_dir_t. */ } cached_dir_t; @@ -1427,6 +1483,9 @@ typedef struct { * directory according to the authorities. */ unsigned int policy_is_reject_star:1; /**< True iff the exit policy for this * router rejects everything. */ + /** True if, after we have added this router, we should re-launch + * tests for it. */ + unsigned int needs_retest_if_added:1; /** Tor can use this router for general positions in circuits. */ #define ROUTER_PURPOSE_GENERAL 0 @@ -1516,6 +1575,9 @@ typedef struct routerstatus_t { unsigned int has_bandwidth:1; /**< The vote/consensus had bw info */ unsigned int has_exitsummary:1; /**< The vote/consensus had exit summaries */ + unsigned int has_measured_bw:1; /**< The vote/consensus had a measured bw */ + + uint32_t measured_bw; /**< Measured bandwidth (capacity) of the router */ uint32_t bandwidth; /**< Bandwidth (capacity) of the router as reported in * the vote/consensus, in kilobytes/sec. */ @@ -1540,6 +1602,52 @@ typedef struct routerstatus_t { } routerstatus_t; +/** A microdescriptor is the smallest amount of information needed to build a + * circuit through a router. They are generated by the directory authorities, + * using information from the uploaded routerinfo documents. They are not + * self-signed, but are rather authenticated by having their hash in a signed + * networkstatus document. */ +typedef struct microdesc_t { + /** Hashtable node, used to look up the microdesc by its digest. */ + HT_ENTRY(microdesc_t) node; + + /* Cache information */ + + /** When was this microdescriptor last listed in a consensus document? + * Once a microdesc has been unlisted long enough, we can drop it. + */ + time_t last_listed; + /** Where is this microdescriptor currently stored? */ + saved_location_t saved_location : 3; + /** If true, do not attempt to cache this microdescriptor on disk. */ + unsigned int no_save : 1; + /** If saved_location == SAVED_IN_CACHE, this field holds the offset of the + * microdescriptor in the cache. */ + off_t off; + + /* The string containing the microdesc. */ + + /** A pointer to the encoded body of the microdescriptor. If the + * saved_location is SAVED_IN_CACHE, then the body is a pointer into an + * mmap'd region. Otherwise, it is a malloc'd string. The string might not + * be NUL-terminated; take the length from <b>bodylen</b>. */ + char *body; + /** The length of the microdescriptor in <b>body</b>. */ + size_t bodylen; + /** A SHA256-digest of the microdescriptor. */ + char digest[DIGEST256_LEN]; + + /* Fields in the microdescriptor. */ + + /** As routerinfo_t.onion_pkey */ + crypto_pk_env_t *onion_pkey; + /** As routerinfo_t.family */ + smartlist_t *family; + /** Encoded exit policy summary */ + char *exitsummary; /**< exit policy summary - + * XXX this probably should not stay a string. */ +} microdesc_t; + /** How many times will we try to download a router's descriptor before giving * up? */ #define MAX_ROUTERDESC_DOWNLOAD_FAILURES 8 @@ -1582,6 +1690,16 @@ typedef struct networkstatus_v2_t { * sorted by identity_digest. */ } networkstatus_v2_t; +/** Linked list of microdesc hash lines for a single router in a directory + * vote. + */ +typedef struct vote_microdesc_hash_t { + /** Next element in the list, or NULL. */ + struct vote_microdesc_hash_t *next; + /** The raw contents of the microdesc hash line, excluding the "m". */ + char *microdesc_hash_line; +} vote_microdesc_hash_t; + /** The claim about a single router, made in a vote. */ typedef struct vote_routerstatus_t { routerstatus_t status; /**< Underlying 'status' object for this router. @@ -1590,31 +1708,46 @@ typedef struct vote_routerstatus_t { * networkstatus_t.known_flags. */ char *version; /**< The version that the authority says this router is * running. */ + /** The hash or hashes that the authority claims this microdesc has. */ + vote_microdesc_hash_t *microdesc; } vote_routerstatus_t; +/** A signature of some document by an authority. */ +typedef struct document_signature_t { + /** Declared SHA-1 digest of this voter's identity key */ + char identity_digest[DIGEST_LEN]; + /** Declared SHA-1 digest of signing key used by this voter. */ + char signing_key_digest[DIGEST_LEN]; + /** Algorithm used to compute the digest of the document. */ + digest_algorithm_t alg; + /** Signature of the signed thing. */ + char *signature; + /** Length of <b>signature</b> */ + int signature_len; + unsigned int bad_signature : 1; /**< Set to true if we've tried to verify + * the sig, and we know it's bad. */ + unsigned int good_signature : 1; /**< Set to true if we've verified the sig + * as good. */ +} document_signature_t; + /** Information about a single voter in a vote or a consensus. */ typedef struct networkstatus_voter_info_t { + /** Declared SHA-1 digest of this voter's identity key */ + char identity_digest[DIGEST_LEN]; char *nickname; /**< Nickname of this voter */ - char identity_digest[DIGEST_LEN]; /**< Digest of this voter's identity key */ + /** Digest of this voter's "legacy" identity key, if any. In vote only; for + * consensuses, we treat legacy keys as additional signers. */ + char legacy_id_digest[DIGEST_LEN]; char *address; /**< Address of this voter, in string format. */ uint32_t addr; /**< Address of this voter, in IPv4, in host order. */ uint16_t dir_port; /**< Directory port of this voter */ uint16_t or_port; /**< OR port of this voter */ char *contact; /**< Contact information for this voter. */ char vote_digest[DIGEST_LEN]; /**< Digest of this voter's vote, as signed. */ - /** Digest of this voter's "legacy" identity key, if any. In vote only; for - * consensuses, we treat legacy keys as additional signers. */ - char legacy_id_digest[DIGEST_LEN]; /* Nothing from here on is signed. */ - char signing_key_digest[DIGEST_LEN]; /**< Declared digest of signing key - * used by this voter. */ - char *signature; /**< Signature from this voter. */ - int signature_len; /**< Length of <b>signature</b> */ - unsigned int bad_signature : 1; /**< Set to true if we've tried to verify - * the sig, and we know it's bad. */ - unsigned int good_signature : 1; /**< Set to true if we've verified the sig - * as good. */ + /** The signature of the document and the signature's status. */ + smartlist_t *sigs; } networkstatus_voter_info_t; /** Enumerates the possible seriousness values of a networkstatus document. */ @@ -1624,10 +1757,25 @@ typedef enum { NS_TYPE_OPINION, } networkstatus_type_t; +/** Enumerates recognized flavors of a consensus networkstatus document. All + * flavors of a consensus are generated from the same set of votes, but they + * present different types information to different versions of Tor. */ +typedef enum { + FLAV_NS = 0, + FLAV_MICRODESC = 1, +} consensus_flavor_t; + +/** Which consensus flavor do we actually want to use to build circuits? */ +#define USABLE_CONSENSUS_FLAVOR FLAV_NS + +/** How many different consensus flavors are there? */ +#define N_CONSENSUS_FLAVORS ((int)(FLAV_MICRODESC)+1) + /** A common structure to hold a v3 network status vote, or a v3 network * status consensus. */ typedef struct networkstatus_t { - networkstatus_type_t type; /**< Vote, consensus, or opinion? */ + networkstatus_type_t type : 8; /**< Vote, consensus, or opinion? */ + consensus_flavor_t flavor : 8; /**< If a consensus, what kind? */ time_t published; /**< Vote only: Time when vote was written. */ time_t valid_after; /**< Time after which this vote or consensus applies. */ time_t fresh_until; /**< Time before which this is the most recent vote or @@ -1659,6 +1807,10 @@ typedef struct networkstatus_t { * consensus, sorted by key. */ smartlist_t *net_params; + /** List of key=value strings for the bw weight parameters in the + * consensus. */ + smartlist_t *weight_params; + /** List of networkstatus_voter_info_t. For a vote, only one element * is included. For a consensus, one element is included for every voter * whose vote contributed to the consensus. */ @@ -1666,8 +1818,8 @@ typedef struct networkstatus_t { struct authority_cert_t *cert; /**< Vote only: the voter's certificate. */ - /** Digest of this document, as signed. */ - char networkstatus_digest[DIGEST_LEN]; + /** Digests of this document, as signed. */ + digests_t digests; /** List of router statuses, sorted by identity digest. For a vote, * the elements are vote_routerstatus_t; for a consensus, the elements @@ -1679,14 +1831,15 @@ typedef struct networkstatus_t { digestmap_t *desc_digest_map; } networkstatus_t; -/** A set of signatures for a networkstatus consensus. All fields are as for - * networkstatus_t. */ +/** A set of signatures for a networkstatus consensus. Unless otherwise + * noted, all fields are as for networkstatus_t. */ typedef struct ns_detached_signatures_t { time_t valid_after; time_t fresh_until; time_t valid_until; - char networkstatus_digest[DIGEST_LEN]; - smartlist_t *signatures; /* list of networkstatus_voter_info_t */ + strmap_t *digests; /**< Map from flavor name to digestset_t */ + strmap_t *signatures; /**< Map from flavor name to list of + * document_signature_t */ } ns_detached_signatures_t; /** Allowable types of desc_store_t. */ @@ -1891,6 +2044,29 @@ typedef struct { time_t expiry_time; } cpath_build_state_t; +/** + * The cell_ewma_t structure keeps track of how many cells a circuit has + * transferred recently. It keeps an EWMA (exponentially weighted moving + * average) of the number of cells flushed from the circuit queue onto a + * connection in connection_or_flush_from_first_active_circuit(). + */ +typedef struct { + /** The last 'tick' at which we recalibrated cell_count. + * + * A cell sent at exactly the start of this tick has weight 1.0. Cells sent + * since the start of this tick have weight greater than 1.0; ones sent + * earlier have less weight. */ + unsigned last_adjusted_tick; + /** The EWMA of the cell count. */ + double cell_count; + /** True iff this is the cell count for a circuit's previous + * connection. */ + unsigned int is_for_p_conn : 1; + /** The position of the circuit within the OR connection's priority + * queue. */ + int heap_index; +} cell_ewma_t; + #define ORIGIN_CIRCUIT_MAGIC 0x35315243u #define OR_CIRCUIT_MAGIC 0x98ABC04Fu @@ -1957,9 +2133,15 @@ typedef struct circuit_t { * length ONIONSKIN_CHALLENGE_LEN. */ char *n_conn_onionskin; - time_t timestamp_created; /**< When was this circuit created? */ - time_t timestamp_dirty; /**< When the circuit was first used, or 0 if the - * circuit is clean. */ + struct timeval timestamp_created; /**< When was the circuit created? */ + /** When the circuit was first used, or 0 if the circuit is clean. + * + * XXXX023 Note that some code will artifically adjust this value backward + * in time in order to indicate that a circuit shouldn't be used for new + * streams, but that it can stay alive as long as it has streams on it. + * That's a kludge we should fix. + */ + time_t timestamp_dirty; uint16_t marked_for_close; /**< Should we close this circuit at the end of * the main loop? (If true, holds the line number @@ -1976,6 +2158,14 @@ typedef struct circuit_t { * linked to an OR connection. */ struct circuit_t *prev_active_on_n_conn; struct circuit_t *next; /**< Next circuit in linked list of all circuits. */ + + /** Unique ID for measuring tunneled network status requests. */ + uint64_t dirreq_id; + + /** The EWMA count for the number of cells flushed from the + * n_conn_cells queue. Used to determine which circuit to flush from next. + */ + cell_ewma_t n_cell_ewma; } circuit_t; /** Largest number of relay_early cells that we can send on a given @@ -2008,6 +2198,13 @@ typedef struct origin_circuit_t { * to the specification? */ unsigned int remaining_relay_early_cells : 4; + /** Set if this circuit is insanely old and we already informed the user */ + unsigned int is_ancient : 1; + + /** Set if this circuit has already been opened. Used to detect + * cannibalized circuits. */ + unsigned int has_opened : 1; + /** What commands were sent over this circuit that decremented the * RELAY_EARLY counter? This is for debugging task 878. */ uint8_t relay_early_commands[MAX_RELAY_EARLY_CELLS_PER_CIRCUIT]; @@ -2098,6 +2295,19 @@ typedef struct or_circuit_t { /** True iff this circuit was made with a CREATE_FAST cell. */ unsigned int is_first_hop : 1; + + /** Number of cells that were removed from circuit queue; reset every + * time when writing buffer stats to disk. */ + uint32_t processed_cells; + + /** Total time in milliseconds that cells spent in both app-ward and + * exit-ward queues of this circuit; reset every time when writing + * buffer stats to disk. */ + uint64_t total_cell_waiting_time; + + /** The EWMA count for the number of cells flushed from the + * p_conn_cells queue. */ + cell_ewma_t p_cell_ewma; } or_circuit_t; /** Convert a circuit subtype to a circuit_t.*/ @@ -2157,6 +2367,9 @@ typedef struct { config_line_t *Logs; /**< New-style list of configuration lines * for logs */ + int LogMessageDomains; /**< Boolean: Should we log the domain(s) in which + * each log message occurs? */ + char *DebugLogFile; /**< Where to send verbose log messages. */ char *DataDirectory; /**< OR only: where to store long-term data. */ char *Nickname; /**< OR only: nickname of this onion router. */ @@ -2169,19 +2382,22 @@ typedef struct { routerset_t *EntryNodes;/**< Structure containing nicknames, digests, * country codes and IP address patterns of ORs to * consider as entry points. */ - int StrictExitNodes; /**< Boolean: When none of our ExitNodes are up, do we - * stop building circuits? */ - int StrictEntryNodes; /**< Boolean: When none of our EntryNodes are up, do we - * stop building circuits? */ + int StrictNodes; /**< Boolean: When none of our EntryNodes or ExitNodes + * are up, or we need to access a node in ExcludeNodes, + * do we just fail instead? */ routerset_t *ExcludeNodes;/**< Structure containing nicknames, digests, * country codes and IP address patterns of ORs - * not to use in circuits. */ + * not to use in circuits. But see StrictNodes + * above. */ routerset_t *ExcludeExitNodes;/**< Structure containing nicknames, digests, * country codes and IP address patterns of * ORs not to consider as exits. */ /** Union of ExcludeNodes and ExcludeExitNodes */ - struct routerset_t *_ExcludeExitNodesUnion; + routerset_t *_ExcludeExitNodesUnion; + + int DisableAllSwap; /**< Boolean: Attempt to call mlockall() on our + * process for all current and future memory. */ /** List of "entry", "middle", "exit", "introduction", "rendezvous". */ smartlist_t *AllowInvalidNodes; @@ -2197,7 +2413,7 @@ typedef struct { * connections. */ config_line_t *TransListenAddress; /** Addresses to bind for listening for transparent natd connections */ - config_line_t *NatdListenAddress; + config_line_t *NATDListenAddress; /** Addresses to bind for listening for SOCKS connections. */ config_line_t *DNSListenAddress; /** Addresses to bind for listening for OR connections. */ @@ -2221,7 +2437,7 @@ typedef struct { int SocksPort; /**< Port to listen on for SOCKS connections. */ /** Port to listen on for transparent pf/netfilter connections. */ int TransPort; - int NatdPort; /**< Port to listen on for transparent natd connections. */ + int NATDPort; /**< Port to listen on for transparent natd connections. */ int ControlPort; /**< Port to listen on for control connections. */ config_line_t *ControlSocket; /**< List of Unix Domain Sockets to listen on * for control connections. */ @@ -2237,8 +2453,6 @@ typedef struct { * for version 3 directories? */ int HSAuthoritativeDir; /**< Boolean: does this an authoritative directory * handle hidden service requests? */ - int HSAuthorityRecordStats; /**< Boolean: does this HS authoritative - * directory record statistics? */ int NamingAuthoritativeDir; /**< Boolean: is this an authoritative directory * that's willing to bind names? */ int VersioningAuthoritativeDir; /**< Boolean: is this an authoritative @@ -2267,8 +2481,6 @@ typedef struct { int AvoidDiskWrites; /**< Boolean: should we never cache things to disk? * Not used yet. */ int ClientOnly; /**< Boolean: should we never evolve into a server role? */ - /** Boolean: should we never publish a descriptor? Deprecated. */ - int NoPublish; /** To what authority types do we publish our descriptor? Choices are * "v1", "v2", "v3", "bridge", or "". */ smartlist_t *PublishServerDescriptor; @@ -2299,6 +2511,14 @@ typedef struct { int ConstrainedSockets; /**< Shrink xmit and recv socket buffers. */ uint64_t ConstrainedSockSize; /**< Size of constrained buffers. */ + /** Whether we should drop exit streams from Tors that we don't know are + * relays. One of "0" (never refuse), "1" (always refuse), or "auto" (do + * what the consensus says, defaulting to 'refuse' if the consensus says + * nothing). */ + char *RefuseUnknownExits; + /** Parsed version of RefuseUnknownExits. -1 for auto. */ + int RefuseUnknownExits_; + /** Application ports that require all nodes in circ to have sufficient * uptime. */ smartlist_t *LongLivedPorts; @@ -2328,10 +2548,18 @@ typedef struct { * connections alive? */ int SocksTimeout; /**< How long do we let a socks connection wait * unattached before we fail it? */ - int CircuitBuildTimeout; /**< Cull non-open circuits that were born - * at least this many seconds ago. */ + int LearnCircuitBuildTimeout; /**< If non-zero, we attempt to learn a value + * for CircuitBuildTimeout based on timeout + * history */ + int CircuitBuildTimeout; /**< Cull non-open circuits that were born at + * least this many seconds ago. Used until + * adaptive algorithm learns a new value. */ int CircuitIdleTimeout; /**< Cull open clean circuits that were born * at least this many seconds ago. */ + int CircuitStreamTimeout; /**< If non-zero, detach streams from circuits + * and try a new circuit if the stream has been + * waiting for this many seconds. If zero, use + * our default internal timeout schedule. */ int MaxOnionsPending; /**< How many circuit CREATE requests do we allow * to wait simultaneously before we start dropping * them? */ @@ -2349,24 +2577,36 @@ typedef struct { * willing to use for all relayed conns? */ uint64_t RelayBandwidthBurst; /**< How much bandwidth, at maximum, will we * use in a second for all relayed conns? */ - int NumCpus; /**< How many CPUs should we try to use? */ - int RunTesting; /**< If true, create testing circuits to measure how well the - * other ORs are running. */ + uint64_t PerConnBWRate; /**< Long-term bw on a single TLS conn, if set. */ + uint64_t PerConnBWBurst; /**< Allowed burst on a single TLS conn, if set. */ + int NumCPUs; /**< How many CPUs should we try to use? */ +//int RunTesting; /**< If true, create testing circuits to measure how well the +// * other ORs are running. */ config_line_t *RendConfigLines; /**< List of configuration lines * for rendezvous services. */ config_line_t *HidServAuth; /**< List of configuration lines for client-side * authorizations for hidden services */ char *ContactInfo; /**< Contact info to be published in the directory. */ - char *HttpProxy; /**< hostname[:port] to use as http proxy, if any. */ - uint32_t HttpProxyAddr; /**< Parsed IPv4 addr for http proxy, if any. */ - uint16_t HttpProxyPort; /**< Parsed port for http proxy, if any. */ - char *HttpProxyAuthenticator; /**< username:password string, if any. */ + char *HTTPProxy; /**< hostname[:port] to use as http proxy, if any. */ + tor_addr_t HTTPProxyAddr; /**< Parsed IPv4 addr for http proxy, if any. */ + uint16_t HTTPProxyPort; /**< Parsed port for http proxy, if any. */ + char *HTTPProxyAuthenticator; /**< username:password string, if any. */ + + char *HTTPSProxy; /**< hostname[:port] to use as https proxy, if any. */ + tor_addr_t HTTPSProxyAddr; /**< Parsed addr for https proxy, if any. */ + uint16_t HTTPSProxyPort; /**< Parsed port for https proxy, if any. */ + char *HTTPSProxyAuthenticator; /**< username:password string, if any. */ - char *HttpsProxy; /**< hostname[:port] to use as https proxy, if any. */ - uint32_t HttpsProxyAddr; /**< Parsed IPv4 addr for https proxy, if any. */ - uint16_t HttpsProxyPort; /**< Parsed port for https proxy, if any. */ - char *HttpsProxyAuthenticator; /**< username:password string, if any. */ + char *Socks4Proxy; /**< hostname:port to use as a SOCKS4 proxy, if any. */ + tor_addr_t Socks4ProxyAddr; /**< Derived from Socks4Proxy. */ + uint16_t Socks4ProxyPort; /**< Derived from Socks4Proxy. */ + + char *Socks5Proxy; /**< hostname:port to use as a SOCKS5 proxy, if any. */ + tor_addr_t Socks5ProxyAddr; /**< Derived from Sock5Proxy. */ + uint16_t Socks5ProxyPort; /**< Derived from Socks5Proxy. */ + char *Socks5ProxyUsername; /**< Username for SOCKS5 authentication, if any */ + char *Socks5ProxyPassword; /**< Password for SOCKS5 authentication, if any */ /** List of configuration lines for replacement directory authorities. * If you just want to replace one class of authority at a time, @@ -2429,8 +2669,13 @@ typedef struct { * or not (1)? */ int ShutdownWaitLength; /**< When we get a SIGINT and we're a server, how * long do we wait before exiting? */ - int SafeLogging; /**< Boolean: are we allowed to log sensitive strings - * such as addresses (0), or do we scrub them first (1)? */ + char *SafeLogging; /**< Contains "relay", "1", "0" (meaning no scrubbing). */ + + /* Derived from SafeLogging */ + enum { + SAFELOG_SCRUB_ALL, SAFELOG_SCRUB_RELAY, SAFELOG_SCRUB_NONE + } _SafeLogging; + int SafeSocks; /**< Boolean: should we outright refuse application * connections that use socks4 or socks5-with-local-dns? */ #define LOG_PROTOCOL_WARN (get_options()->ProtocolWarnings ? \ @@ -2441,6 +2686,8 @@ typedef struct { * log whether it was DNS-leaking or not? */ int HardwareAccel; /**< Boolean: Should we enable OpenSSL hardware * acceleration where available? */ + char *AccelName; /**< Optional hardware acceleration engine name. */ + char *AccelDir; /**< Optional hardware acceleration engine search dir. */ int UseEntryGuards; /**< Boolean: Do we try to enter from a smallish number * of fixed nodes? */ int NumEntryGuards; /**< How many entry guards do we try to establish? */ @@ -2451,6 +2698,9 @@ typedef struct { * means directly from the authorities) no matter our other config? */ int FetchDirInfoEarly; + /** Should we fetch our dir info at the start of the consensus period? */ + int FetchDirInfoExtraEarly; + char *VirtualAddrNetwork; /**< Address and mask to hand out for virtual * MAPADDRESS requests. */ int ServerDNSSearchDomains; /**< Boolean: If set, we don't force exit @@ -2499,11 +2749,42 @@ typedef struct { * exit allows it, we use it. */ int AllowSingleHopCircuits; + /** If true, we convert "www.google.com.foo.exit" addresses on the + * socks/trans/natd ports into "www.google.com" addresses that + * exit from the node "foo". Disabled by default since attacking + * websites and exit relays can use it to manipulate your path + * selection. */ + int AllowDotExit; + + /** If true, we will warn if a user gives us only an IP address + * instead of a hostname. */ + int WarnUnsafeSocks; + + /** If true, the user wants us to collect statistics on clients + * requesting network statuses from us as directory. */ + int DirReqStatistics; + + /** If true, the user wants us to collect statistics on port usage. */ + int ExitPortStatistics; + + /** If true, the user wants us to collect cell statistics. */ + int CellStatistics; + + /** If true, the user wants us to collect statistics as entry node. */ + int EntryStatistics; + + /** If true, include statistics file contents in extra-info documents. */ + int ExtraInfoStatistics; + /** If true, do not believe anybody who tells us that a domain resolves * to an internal address, or that an internal address has a PTR mapping. * Helps avoid some cross-site attacks. */ int ClientDNSRejectInternalAddresses; + /** If true, do not accept any requests to connect to internal addresses + * over randomly chosen exits. */ + int ClientRejectInternalAddresses; + /** The length of time that we think a consensus should be fresh. */ int V3AuthVotingInterval; /** The length of time we think it will take to distribute votes. */ @@ -2517,6 +2798,13 @@ typedef struct { * migration purposes? */ int V3AuthUseLegacyKey; + /** Location of bandwidth measurement file */ + char *V3BandwidthsFile; + + /** Authority only: key=value pairs that we add to our networkstatus + * consensus vote on the 'params' line. */ + char *ConsensusParams; + /** The length of time that we think an initial consensus should be fresh. * Only altered on testing networks. */ int TestingV3AuthInitialVotingInterval; @@ -2553,19 +2841,6 @@ typedef struct { * the bridge authority guess which countries have blocked access to us. */ int BridgeRecordUsageByCountry; -#ifdef ENABLE_GEOIP_STATS - /** If true, and Tor is built with GEOIP_STATS support, and we're a - * directory, record how many directory requests we get from each country. */ - int DirRecordUsageByCountry; - /** Round all GeoIP results to the next multiple of this value, to avoid - * leaking information. */ - int DirRecordUsageGranularity; - /** Time interval: purge geoip stats after this long. */ - int DirRecordUsageRetainIPs; - /** Time interval: Flush geoip data to disk this often. */ - int DirRecordUsageSaveInterval; -#endif - /** Optionally, a file with GeoIP data. */ char *GeoIPFile; @@ -2573,6 +2848,26 @@ typedef struct { * to make this false. */ int ReloadTorrcOnSIGHUP; + /* The main parameter for picking circuits within a connection. + * + * If this value is positive, when picking a cell to relay on a connection, + * we always relay from the circuit whose weighted cell count is lowest. + * Cells are weighted exponentially such that if one cell is sent + * 'CircuitPriorityHalflife' seconds before another, it counts for half as + * much. + * + * If this value is zero, we're disabling the cell-EWMA algorithm. + * + * If this value is negative, we're using the default approach + * according to either Tor or a parameter set in the consensus. + */ + double CircuitPriorityHalflife; + + /** Set to true if the TestingTorNetwork configuration option is set. + * This is used so that options_validate() has a chance to realize that + * the defaults have changed. */ + int _UsingTestNetworkDefaults; + } or_options_t; /** Persistent state for an onion router, as saved to disk. */ @@ -2591,6 +2886,9 @@ typedef struct { uint64_t AccountingBytesReadInInterval; uint64_t AccountingBytesWrittenInInterval; int AccountingSecondsActive; + int AccountingSecondsToReachSoftLimit; + time_t AccountingSoftLimitHitAt; + uint64_t AccountingBytesAtSoftLimit; uint64_t AccountingExpectedUsage; /** A list of Entry Guard-related configuration lines. */ @@ -2601,13 +2899,30 @@ typedef struct { * bandwidth usage. The "Interval" fields hold the granularity, in seconds, * of the entries of Values. The "Values" lists hold decimal string * representations of the number of bytes read or written in each - * interval. */ + * interval. The "Maxima" list holds decimal strings describing the highest + * rate achieved during the interval. + */ time_t BWHistoryReadEnds; int BWHistoryReadInterval; smartlist_t *BWHistoryReadValues; + smartlist_t *BWHistoryReadMaxima; time_t BWHistoryWriteEnds; int BWHistoryWriteInterval; smartlist_t *BWHistoryWriteValues; + smartlist_t *BWHistoryWriteMaxima; + time_t BWHistoryDirReadEnds; + int BWHistoryDirReadInterval; + smartlist_t *BWHistoryDirReadValues; + smartlist_t *BWHistoryDirReadMaxima; + time_t BWHistoryDirWriteEnds; + int BWHistoryDirWriteInterval; + smartlist_t *BWHistoryDirWriteValues; + smartlist_t *BWHistoryDirWriteMaxima; + + /** Build time histogram */ + config_line_t * BuildtimeHistogram; + unsigned int TotalBuildTimes; + unsigned int CircuitBuildAbandonedCount; /** What version of Tor wrote this state file? */ char *TorVersion; @@ -2669,209 +2984,143 @@ struct socks_request_t { /* all the function prototypes go here */ -/********************************* buffers.c ***************************/ - -buf_t *buf_new(void); -buf_t *buf_new_with_capacity(size_t size); -void buf_free(buf_t *buf); -void buf_clear(buf_t *buf); -void buf_shrink(buf_t *buf); -void buf_shrink_freelists(int free_all); -void buf_dump_freelist_sizes(int severity); - -size_t buf_datalen(const buf_t *buf); -size_t buf_allocation(const buf_t *buf); -size_t buf_slack(const buf_t *buf); -const char *_buf_peek_raw_buffer(const buf_t *buf); - -int read_to_buf(int s, size_t at_most, buf_t *buf, int *reached_eof, - int *socket_error); -int read_to_buf_tls(tor_tls_t *tls, size_t at_most, buf_t *buf); - -int flush_buf(int s, buf_t *buf, size_t sz, size_t *buf_flushlen); -int flush_buf_tls(tor_tls_t *tls, buf_t *buf, size_t sz, size_t *buf_flushlen); - -int write_to_buf(const char *string, size_t string_len, buf_t *buf); -int write_to_buf_zlib(buf_t *buf, tor_zlib_state_t *state, - const char *data, size_t data_len, int done); -int move_buf_to_buf(buf_t *buf_out, buf_t *buf_in, size_t *buf_flushlen); -int fetch_from_buf(char *string, size_t string_len, buf_t *buf); -int fetch_var_cell_from_buf(buf_t *buf, var_cell_t **out, int linkproto); -int fetch_from_buf_http(buf_t *buf, - char **headers_out, size_t max_headerlen, - char **body_out, size_t *body_used, size_t max_bodylen, - int force_complete); -int fetch_from_buf_socks(buf_t *buf, socks_request_t *req, - int log_sockstype, int safe_socks); -int fetch_from_buf_line(buf_t *buf, char *data_out, size_t *data_len); - -int peek_buf_has_control0_command(buf_t *buf); - -void assert_buf_ok(buf_t *buf); - -#ifdef BUFFERS_PRIVATE -int buf_find_string_offset(const buf_t *buf, const char *s, size_t n); -#endif - /********************************* circuitbuild.c **********************/ -char *circuit_list_path(origin_circuit_t *circ, int verbose); -char *circuit_list_path_for_controller(origin_circuit_t *circ); -void circuit_log_path(int severity, unsigned int domain, - origin_circuit_t *circ); -void circuit_rep_hist_note_result(origin_circuit_t *circ); -origin_circuit_t *origin_circuit_init(uint8_t purpose, int flags); -origin_circuit_t *circuit_establish_circuit(uint8_t purpose, - extend_info_t *exit, - int flags); -int circuit_handle_first_hop(origin_circuit_t *circ); -void circuit_n_conn_done(or_connection_t *or_conn, int status); -int inform_testing_reachability(void); -int circuit_send_next_onion_skin(origin_circuit_t *circ); -void circuit_note_clock_jumped(int seconds_elapsed); -int circuit_extend(cell_t *cell, circuit_t *circ); -int circuit_init_cpath_crypto(crypt_path_t *cpath, const char *key_data, - int reverse); -int circuit_finish_handshake(origin_circuit_t *circ, uint8_t cell_type, - const uint8_t *reply); -int circuit_truncated(origin_circuit_t *circ, crypt_path_t *layer); -int onionskin_answer(or_circuit_t *circ, uint8_t cell_type, - const char *payload, const char *keys); -int circuit_all_predicted_ports_handled(time_t now, int *need_uptime, - int *need_capacity); - -int circuit_append_new_exit(origin_circuit_t *circ, extend_info_t *info); -int circuit_extend_to_new_exit(origin_circuit_t *circ, extend_info_t *info); -void onion_append_to_cpath(crypt_path_t **head_ptr, crypt_path_t *new_hop); -extend_info_t *extend_info_alloc(const char *nickname, const char *digest, - crypto_pk_env_t *onion_key, - const tor_addr_t *addr, uint16_t port); -extend_info_t *extend_info_from_router(routerinfo_t *r); -extend_info_t *extend_info_dup(extend_info_t *info); -void extend_info_free(extend_info_t *info); -routerinfo_t *build_state_get_exit_router(cpath_build_state_t *state); -const char *build_state_get_exit_nickname(cpath_build_state_t *state); - -void entry_guards_compute_status(void); -int entry_guard_register_connect_status(const char *digest, int succeeded, - int mark_relay_status, time_t now); -void entry_nodes_should_be_added(void); -int entry_list_can_grow(or_options_t *options); -routerinfo_t *choose_random_entry(cpath_build_state_t *state); -int entry_guards_parse_state(or_state_t *state, int set, char **msg); -void entry_guards_update_state(or_state_t *state); -int getinfo_helper_entry_guards(control_connection_t *conn, - const char *question, char **answer); - -void clear_bridge_list(void); -int routerinfo_is_a_configured_bridge(routerinfo_t *ri); -void bridge_add_from_config(const tor_addr_t *addr, uint16_t port, - char *digest); -void retry_bridge_descriptor_fetch_directly(const char *digest); -void fetch_bridge_descriptors(time_t now); -void learned_bridge_descriptor(routerinfo_t *ri, int from_cache); -int any_bridge_descriptors_known(void); -int any_pending_bridge_descriptor_fetches(void); -int bridges_known_but_down(void); -void bridges_retry_all(void); - -void entry_guards_free_all(void); - -/********************************* circuitlist.c ***********************/ - -circuit_t * _circuit_get_global_list(void); -const char *circuit_state_to_string(int state); -const char *circuit_purpose_to_controller_string(uint8_t purpose); -void circuit_dump_by_conn(connection_t *conn, int severity); -void circuit_set_p_circid_orconn(or_circuit_t *circ, circid_t id, - or_connection_t *conn); -void circuit_set_n_circid_orconn(circuit_t *circ, circid_t id, - or_connection_t *conn); -void circuit_set_state(circuit_t *circ, uint8_t state); -void circuit_close_all_marked(void); -int32_t circuit_initial_package_window(void); -origin_circuit_t *origin_circuit_new(void); -or_circuit_t *or_circuit_new(circid_t p_circ_id, or_connection_t *p_conn); -circuit_t *circuit_get_by_circid_orconn(circid_t circ_id, - or_connection_t *conn); -int circuit_id_in_use_on_orconn(circid_t circ_id, or_connection_t *conn); -circuit_t *circuit_get_by_edge_conn(edge_connection_t *conn); -void circuit_unlink_all_from_or_conn(or_connection_t *conn, int reason); -origin_circuit_t *circuit_get_by_global_id(uint32_t id); -origin_circuit_t *circuit_get_by_rend_query_and_purpose(const char *rend_query, - uint8_t purpose); -origin_circuit_t *circuit_get_next_by_pk_and_purpose(origin_circuit_t *start, - const char *digest, uint8_t purpose); -or_circuit_t *circuit_get_rendezvous(const char *cookie); -or_circuit_t *circuit_get_intro_point(const char *digest); -origin_circuit_t *circuit_find_to_cannibalize(uint8_t purpose, - extend_info_t *info, int flags); -void circuit_mark_all_unused_circs(void); -void circuit_expire_all_dirty_circs(void); -void _circuit_mark_for_close(circuit_t *circ, int reason, - int line, const char *file); -int circuit_get_cpath_len(origin_circuit_t *circ); -crypt_path_t *circuit_get_cpath_hop(origin_circuit_t *circ, int hopnum); -void circuit_get_all_pending_on_or_conn(smartlist_t *out, - or_connection_t *or_conn); -int circuit_count_pending_on_or_conn(or_connection_t *or_conn); - -#define circuit_mark_for_close(c, reason) \ - _circuit_mark_for_close((c), (reason), __LINE__, _SHORT_FILE_) - -void assert_cpath_layer_ok(const crypt_path_t *cp); -void assert_circuit_ok(const circuit_t *c); -void circuit_free_all(void); - -/********************************* circuituse.c ************************/ - -void circuit_expire_building(time_t now); -void circuit_remove_handled_ports(smartlist_t *needed_ports); -int circuit_stream_is_being_handled(edge_connection_t *conn, uint16_t port, - int min); -int circuit_conforms_to_options(const origin_circuit_t *circ, - const or_options_t *options); -void circuit_build_needed_circs(time_t now); -void circuit_detach_stream(circuit_t *circ, edge_connection_t *conn); - -void circuit_expire_old_circuits_serverside(time_t now); - -void reset_bandwidth_test(void); -int circuit_enough_testing_circs(void); - -void circuit_has_opened(origin_circuit_t *circ); -void circuit_build_failed(origin_circuit_t *circ); - -/** Flag to set when a circuit should have only a single hop. */ -#define CIRCLAUNCH_ONEHOP_TUNNEL (1<<0) -/** Flag to set when a circuit needs to be built of high-uptime nodes */ -#define CIRCLAUNCH_NEED_UPTIME (1<<1) -/** Flag to set when a circuit needs to be built of high-capacity nodes */ -#define CIRCLAUNCH_NEED_CAPACITY (1<<2) -/** Flag to set when the last hop of a circuit doesn't need to be an - * exit node. */ -#define CIRCLAUNCH_IS_INTERNAL (1<<3) -origin_circuit_t *circuit_launch_by_extend_info(uint8_t purpose, - extend_info_t *info, - int flags); -origin_circuit_t *circuit_launch_by_router(uint8_t purpose, - routerinfo_t *exit, int flags); -void circuit_reset_failure_count(int timeout); -int connection_ap_handshake_attach_chosen_circuit(edge_connection_t *conn, - origin_circuit_t *circ, - crypt_path_t *cpath); -int connection_ap_handshake_attach_circuit(edge_connection_t *conn); - -/********************************* command.c ***************************/ - -void command_process_cell(cell_t *cell, or_connection_t *conn); -void command_process_var_cell(var_cell_t *cell, or_connection_t *conn); - -extern uint64_t stats_n_padding_cells_processed; -extern uint64_t stats_n_create_cells_processed; -extern uint64_t stats_n_created_cells_processed; -extern uint64_t stats_n_relay_cells_processed; -extern uint64_t stats_n_destroy_cells_processed; +/** How many hops does a general-purpose circuit have by default? */ +#define DEFAULT_ROUTE_LEN 3 + +/* Circuit Build Timeout "public" structures. */ + +/** Precision multiplier for the Bw weights */ +#define BW_WEIGHT_SCALE 10000 +#define BW_MIN_WEIGHT_SCALE 1 +#define BW_MAX_WEIGHT_SCALE INT32_MAX + +/** Total size of the circuit timeout history to accumulate. + * 1000 is approx 2.5 days worth of continual-use circuits. */ +#define CBT_NCIRCUITS_TO_OBSERVE 1000 + +/** Width of the histogram bins in milliseconds */ +#define CBT_BIN_WIDTH ((build_time_t)50) + +/** Number of modes to use in the weighted-avg computation of Xm */ +#define CBT_DEFAULT_NUM_XM_MODES 3 +#define CBT_MIN_NUM_XM_MODES 1 +#define CBT_MAX_NUM_XM_MODES 20 + +/** A build_time_t is milliseconds */ +typedef uint32_t build_time_t; + +/** + * CBT_BUILD_ABANDONED is our flag value to represent a force-closed + * circuit (Aka a 'right-censored' pareto value). + */ +#define CBT_BUILD_ABANDONED ((build_time_t)(INT32_MAX-1)) +#define CBT_BUILD_TIME_MAX ((build_time_t)(INT32_MAX)) + +/** Save state every 10 circuits */ +#define CBT_SAVE_STATE_EVERY 10 + +/* Circuit build times consensus parameters */ + +/** + * How long to wait before actually closing circuits that take too long to + * build in terms of CDF quantile. + */ +#define CBT_DEFAULT_CLOSE_QUANTILE 95 +#define CBT_MIN_CLOSE_QUANTILE CBT_MIN_QUANTILE_CUTOFF +#define CBT_MAX_CLOSE_QUANTILE CBT_MAX_QUANTILE_CUTOFF + +/** + * How many circuits count as recent when considering if the + * connection has gone gimpy or changed. + */ +#define CBT_DEFAULT_RECENT_CIRCUITS 20 +#define CBT_MIN_RECENT_CIRCUITS 3 +#define CBT_MAX_RECENT_CIRCUITS 1000 + +/** + * Maximum count of timeouts that finish the first hop in the past + * RECENT_CIRCUITS before calculating a new timeout. + * + * This tells us whether to abandon timeout history and set + * the timeout back to whatever circuit_build_times_get_initial_timeout() + * gives us. + */ +#define CBT_DEFAULT_MAX_RECENT_TIMEOUT_COUNT (CBT_DEFAULT_RECENT_CIRCUITS*9/10) +#define CBT_MIN_MAX_RECENT_TIMEOUT_COUNT 3 +#define CBT_MAX_MAX_RECENT_TIMEOUT_COUNT 10000 + +/** Minimum circuits before estimating a timeout */ +#define CBT_DEFAULT_MIN_CIRCUITS_TO_OBSERVE 100 +#define CBT_MIN_MIN_CIRCUITS_TO_OBSERVE 1 +#define CBT_MAX_MIN_CIRCUITS_TO_OBSERVE 10000 + +/** Cutoff percentile on the CDF for our timeout estimation. */ +#define CBT_DEFAULT_QUANTILE_CUTOFF 80 +#define CBT_MIN_QUANTILE_CUTOFF 10 +#define CBT_MAX_QUANTILE_CUTOFF 99 +double circuit_build_times_quantile_cutoff(void); + +/** How often in seconds should we build a test circuit */ +#define CBT_DEFAULT_TEST_FREQUENCY 60 +#define CBT_MIN_TEST_FREQUENCY 1 +#define CBT_MAX_TEST_FREQUENCY INT32_MAX + +/** Lowest allowable value for CircuitBuildTimeout in milliseconds */ +#define CBT_DEFAULT_TIMEOUT_MIN_VALUE (1500) +#define CBT_MIN_TIMEOUT_MIN_VALUE 500 +#define CBT_MAX_TIMEOUT_MIN_VALUE INT32_MAX + +/** Initial circuit build timeout in milliseconds */ +#define CBT_DEFAULT_TIMEOUT_INITIAL_VALUE (60*1000) +#define CBT_MIN_TIMEOUT_INITIAL_VALUE CBT_MIN_TIMEOUT_MIN_VALUE +#define CBT_MAX_TIMEOUT_INITIAL_VALUE INT32_MAX +int32_t circuit_build_times_initial_timeout(void); + +#if CBT_DEFAULT_MAX_RECENT_TIMEOUT_COUNT < CBT_MIN_MAX_RECENT_TIMEOUT_COUNT +#error "RECENT_CIRCUITS is set too low." +#endif + +/** Information about the state of our local network connection */ +typedef struct { + /** The timestamp we last completed a TLS handshake or received a cell */ + time_t network_last_live; + /** If the network is not live, how many timeouts has this caused? */ + int nonlive_timeouts; + /** Circular array of circuits that have made it to the first hop. Slot is + * 1 if circuit timed out, 0 if circuit succeeded */ + int8_t *timeouts_after_firsthop; + /** Number of elements allocated for the above array */ + int num_recent_circs; + /** Index into circular array. */ + int after_firsthop_idx; +} network_liveness_t; + +/** Structure for circuit build times history */ +typedef struct { + /** The circular array of recorded build times in milliseconds */ + build_time_t circuit_build_times[CBT_NCIRCUITS_TO_OBSERVE]; + /** Current index in the circuit_build_times circular array */ + int build_times_idx; + /** Total number of build times accumulated. Max CBT_NCIRCUITS_TO_OBSERVE */ + int total_build_times; + /** Information about the state of our local network connection */ + network_liveness_t liveness; + /** Last time we built a circuit. Used to decide to build new test circs */ + time_t last_circ_at; + /** "Minimum" value of our pareto distribution (actually mode) */ + build_time_t Xm; + /** alpha exponent for pareto dist. */ + double alpha; + /** Have we computed a timeout? */ + int have_computed_timeout; + /** The exact value for that timeout in milliseconds. Stored as a double + * to maintain precision from calculations to and from quantile value. */ + double timeout_ms; + /** How long we wait before actually closing the circuit. */ + double close_ms; +} circuit_build_times_t; /********************************* config.c ***************************/ @@ -2884,203 +3133,8 @@ typedef enum setopt_err_t { SETOPT_ERR_SETTING = -4, } setopt_err_t; -const char *get_dirportfrontpage(void); -or_options_t *get_options(void); -int set_options(or_options_t *new_val, char **msg); -void config_free_all(void); -const char *safe_str(const char *address); -const char *escaped_safe_str(const char *address); -const char *get_version(void); - -int config_get_lines(const char *string, config_line_t **result); -void config_free_lines(config_line_t *front); -setopt_err_t options_trial_assign(config_line_t *list, int use_defaults, - int clear_first, char **msg); -int resolve_my_address(int warn_severity, or_options_t *options, - uint32_t *addr, char **hostname_out); -int is_local_addr(const tor_addr_t *addr) ATTR_PURE; -void options_init(or_options_t *options); -int options_init_from_torrc(int argc, char **argv); -setopt_err_t options_init_from_string(const char *cf, - int command, const char *command_arg, char **msg); -int option_is_recognized(const char *key); -const char *option_get_canonical_name(const char *key); -config_line_t *option_get_assignment(or_options_t *options, - const char *key); -int options_save_current(void); -const char *get_torrc_fname(void); -char *options_get_datadir_fname2_suffix(or_options_t *options, - const char *sub1, const char *sub2, - const char *suffix); -#define get_datadir_fname2_suffix(sub1, sub2, suffix) \ - options_get_datadir_fname2_suffix(get_options(), (sub1), (sub2), (suffix)) -/** Return a newly allocated string containing datadir/sub1. See - * get_datadir_fname2_suffix. */ -#define get_datadir_fname(sub1) get_datadir_fname2_suffix((sub1), NULL, NULL) -/** Return a newly allocated string containing datadir/sub1/sub2. See - * get_datadir_fname2_suffix. */ -#define get_datadir_fname2(sub1,sub2) \ - get_datadir_fname2_suffix((sub1), (sub2), NULL) -/** Return a newly allocated string containing datadir/sub1suffix. See - * get_datadir_fname2_suffix. */ -#define get_datadir_fname_suffix(sub1, suffix) \ - get_datadir_fname2_suffix((sub1), NULL, (suffix)) - -or_state_t *get_or_state(void); -int or_state_save(time_t now); - -int options_need_geoip_info(or_options_t *options, const char **reason_out); -int getinfo_helper_config(control_connection_t *conn, - const char *question, char **answer); - -uint32_t get_effective_bwrate(or_options_t *options); -uint32_t get_effective_bwburst(or_options_t *options); - -#ifdef CONFIG_PRIVATE -/* Used only by config.c and test.c */ -or_options_t *options_new(void); -#endif - -/********************************* connection.c ***************************/ - -const char *conn_type_to_string(int type); -const char *conn_state_to_string(int type, int state); - -dir_connection_t *dir_connection_new(int socket_family); -or_connection_t *or_connection_new(int socket_family); -edge_connection_t *edge_connection_new(int type, int socket_family); -control_connection_t *control_connection_new(int socket_family); -connection_t *connection_new(int type, int socket_family); - -void connection_link_connections(connection_t *conn_a, connection_t *conn_b); -void connection_unregister_events(connection_t *conn); -void connection_free(connection_t *conn); -void connection_free_all(void); -void connection_about_to_close_connection(connection_t *conn); -void connection_close_immediate(connection_t *conn); -void _connection_mark_for_close(connection_t *conn,int line, const char *file); - -#define connection_mark_for_close(c) \ - _connection_mark_for_close((c), __LINE__, _SHORT_FILE_) - -void connection_expire_held_open(void); - -int connection_connect(connection_t *conn, const char *address, - const tor_addr_t *addr, - uint16_t port, int *socket_error); -int retry_all_listeners(smartlist_t *replaced_conns, - smartlist_t *new_conns); - -ssize_t connection_bucket_write_limit(connection_t *conn, time_t now); -int global_write_bucket_low(connection_t *conn, size_t attempt, int priority); -void connection_bucket_init(void); -void connection_bucket_refill(int seconds_elapsed, time_t now); - -int connection_handle_read(connection_t *conn); - -int connection_fetch_from_buf(char *string, size_t len, connection_t *conn); - -int connection_wants_to_flush(connection_t *conn); -int connection_outbuf_too_full(connection_t *conn); -int connection_handle_write(connection_t *conn, int force); -void _connection_write_to_buf_impl(const char *string, size_t len, - connection_t *conn, int zlib); -static void connection_write_to_buf(const char *string, size_t len, - connection_t *conn); -static void connection_write_to_buf_zlib(const char *string, size_t len, - dir_connection_t *conn, int done); -static INLINE void -connection_write_to_buf(const char *string, size_t len, connection_t *conn) -{ - _connection_write_to_buf_impl(string, len, conn, 0); -} -static INLINE void -connection_write_to_buf_zlib(const char *string, size_t len, - dir_connection_t *conn, int done) -{ - _connection_write_to_buf_impl(string, len, TO_CONN(conn), done ? -1 : 1); -} - -connection_t *connection_get_by_global_id(uint64_t id); - -connection_t *connection_get_by_type(int type); -connection_t *connection_get_by_type_purpose(int type, int purpose); -connection_t *connection_get_by_type_addr_port_purpose(int type, - const tor_addr_t *addr, - uint16_t port, int purpose); -connection_t *connection_get_by_type_state(int type, int state); -connection_t *connection_get_by_type_state_rendquery(int type, int state, - const char *rendquery, - int rendversion); - -#define connection_speaks_cells(conn) ((conn)->type == CONN_TYPE_OR) -int connection_is_listener(connection_t *conn); -int connection_state_is_open(connection_t *conn); -int connection_state_is_connecting(connection_t *conn); - -char *alloc_http_authenticator(const char *authenticator); - -void assert_connection_ok(connection_t *conn, time_t now); -int connection_or_nonopen_was_started_here(or_connection_t *conn); -void connection_dump_buffer_mem_stats(int severity); -void remove_file_if_very_old(const char *fname, time_t now); - /********************************* connection_edge.c *************************/ -#define connection_mark_unattached_ap(conn, endreason) \ - _connection_mark_unattached_ap((conn), (endreason), __LINE__, _SHORT_FILE_) - -void _connection_mark_unattached_ap(edge_connection_t *conn, int endreason, - int line, const char *file); -int connection_edge_reached_eof(edge_connection_t *conn); -int connection_edge_process_inbuf(edge_connection_t *conn, - int package_partial); -int connection_edge_destroy(circid_t circ_id, edge_connection_t *conn); -int connection_edge_end(edge_connection_t *conn, uint8_t reason); -int connection_edge_end_errno(edge_connection_t *conn); -int connection_edge_finished_flushing(edge_connection_t *conn); -int connection_edge_finished_connecting(edge_connection_t *conn); - -int connection_ap_handshake_send_begin(edge_connection_t *ap_conn); -int connection_ap_handshake_send_resolve(edge_connection_t *ap_conn); - -edge_connection_t *connection_ap_make_link(char *address, uint16_t port, - const char *digest, - int use_begindir, int want_onehop); -void connection_ap_handshake_socks_reply(edge_connection_t *conn, char *reply, - size_t replylen, - int endreason); -void connection_ap_handshake_socks_resolved(edge_connection_t *conn, - int answer_type, - size_t answer_len, - const uint8_t *answer, - int ttl, - time_t expires); - -int connection_exit_begin_conn(cell_t *cell, circuit_t *circ); -int connection_exit_begin_resolve(cell_t *cell, or_circuit_t *circ); -void connection_exit_connect(edge_connection_t *conn); -int connection_edge_is_rendezvous_stream(edge_connection_t *conn); -int connection_ap_can_use_exit(edge_connection_t *conn, routerinfo_t *exit); -void connection_ap_expire_beginning(void); -void connection_ap_attach_pending(void); -void connection_ap_fail_onehop(const char *failed_digest, - cpath_build_state_t *build_state); -void circuit_discard_optional_exit_enclaves(extend_info_t *info); -int connection_ap_detach_retriable(edge_connection_t *conn, - origin_circuit_t *circ, - int reason); -int connection_ap_process_transparent(edge_connection_t *conn); - -int address_is_invalid_destination(const char *address, int client); - -void addressmap_init(void); -void addressmap_clean(time_t now); -void addressmap_clear_configured(void); -void addressmap_clear_transient(void); -void addressmap_free_all(void); -int addressmap_rewrite(char *address, size_t maxlen, time_t *expires_out); -int addressmap_have_mapping(const char *address, int update_timeout); /** Enumerates possible origins of a client-side address mapping. */ typedef enum { /** We're remapping this address because the controller told us to. */ @@ -3095,75 +3149,6 @@ typedef enum { * Tor server that told us what its value was. */ ADDRMAPSRC_DNS, } addressmap_entry_source_t; -void addressmap_register(const char *address, char *new_address, - time_t expires, addressmap_entry_source_t source); -int parse_virtual_addr_network(const char *val, int validate_only, - char **msg); -int client_dns_incr_failures(const char *address); -void client_dns_clear_failures(const char *address); -void client_dns_set_addressmap(const char *address, uint32_t val, - const char *exitname, int ttl); -const char *addressmap_register_virtual_address(int type, char *new_address); -void addressmap_get_mappings(smartlist_t *sl, time_t min_expires, - time_t max_expires, int want_expiry); -int connection_ap_handshake_rewrite_and_attach(edge_connection_t *conn, - origin_circuit_t *circ, - crypt_path_t *cpath); -int hostname_is_noconnect_address(const char *address); - -/** Possible return values for parse_extended_hostname. */ -typedef enum hostname_type_t { - NORMAL_HOSTNAME, ONION_HOSTNAME, EXIT_HOSTNAME, BAD_HOSTNAME -} hostname_type_t; -hostname_type_t parse_extended_hostname(char *address); - -#if defined(HAVE_NET_IF_H) && defined(HAVE_NET_PFVAR_H) -int get_pf_socket(void); -#endif - -/********************************* connection_or.c ***************************/ - -void connection_or_remove_from_identity_map(or_connection_t *conn); -void connection_or_clear_identity_map(void); -or_connection_t *connection_or_get_for_extend(const char *digest, - const tor_addr_t *target_addr, - const char **msg_out, - int *launch_out); -void connection_or_set_bad_connections(void); - -int connection_or_reached_eof(or_connection_t *conn); -int connection_or_process_inbuf(or_connection_t *conn); -int connection_or_flushed_some(or_connection_t *conn); -int connection_or_finished_flushing(or_connection_t *conn); -int connection_or_finished_connecting(or_connection_t *conn); - -void connection_or_connect_failed(or_connection_t *conn, - int reason, const char *msg); -or_connection_t *connection_or_connect(const tor_addr_t *addr, uint16_t port, - const char *id_digest); - -int connection_tls_start_handshake(or_connection_t *conn, int receiving); -int connection_tls_continue_handshake(or_connection_t *conn); - -void or_handshake_state_free(or_handshake_state_t *state); -int connection_or_set_state_open(or_connection_t *conn); -void connection_or_write_cell_to_buf(const cell_t *cell, - or_connection_t *conn); -void connection_or_write_var_cell_to_buf(const var_cell_t *cell, - or_connection_t *conn); -int connection_or_send_destroy(circid_t circ_id, or_connection_t *conn, - int reason); -int connection_or_send_netinfo(or_connection_t *conn); -int connection_or_send_cert(or_connection_t *conn); -int connection_or_send_link_auth(or_connection_t *conn); -int connection_or_compute_link_auth_hmac(or_connection_t *conn, - char *hmac_out); -int is_or_protocol_version_known(uint16_t version); - -void cell_pack(packed_cell_t *dest, const cell_t *src); -void var_cell_pack_header(const var_cell_t *cell, char *hdr_out); -var_cell_t *var_cell_new(uint16_t payload_len); -void var_cell_free(var_cell_t *cell); /********************************* control.c ***************************/ @@ -3201,8 +3186,14 @@ typedef enum or_conn_status_event_t { OR_CONN_EVENT_NEW = 4, } or_conn_status_event_t; -void control_update_global_event_mask(void); -void control_adjust_event_log_severity(void); +/** Used to indicate the type of a buildtime event */ +typedef enum buildtimeout_set_event_t { + BUILDTIMEOUT_SET_EVENT_COMPUTED = 0, + BUILDTIMEOUT_SET_EVENT_RESET = 1, + BUILDTIMEOUT_SET_EVENT_SUSPENDED = 2, + BUILDTIMEOUT_SET_EVENT_DISCARD = 3, + BUILDTIMEOUT_SET_EVENT_RESUME = 4 +} buildtimeout_set_event_t; /** Execute the statement <b>stmt</b>, which may log events concerning the * connection <b>conn</b>. To prevent infinite loops, disable log messages @@ -3220,58 +3211,6 @@ void control_adjust_event_log_severity(void); enable_control_logging(); \ STMT_END -/** Log information about the connection <b>conn</b>, protecting it as with - * CONN_LOG_PROTECT. Example: - * - * LOG_FN_CONN(conn, (LOG_DEBUG, "Socket %d wants to write", conn->s)); - **/ -#define LOG_FN_CONN(conn, args) \ - CONN_LOG_PROTECT(conn, log_fn args) - -int connection_control_finished_flushing(control_connection_t *conn); -int connection_control_reached_eof(control_connection_t *conn); -int connection_control_process_inbuf(control_connection_t *conn); - -#define EVENT_AUTHDIR_NEWDESCS 0x000D -#define EVENT_NS 0x000F -int control_event_is_interesting(int event); - -int control_event_circuit_status(origin_circuit_t *circ, - circuit_status_event_t e, int reason); -int control_event_stream_status(edge_connection_t *conn, - stream_status_event_t e, - int reason); -int control_event_or_conn_status(or_connection_t *conn, - or_conn_status_event_t e, int reason); -int control_event_bandwidth_used(uint32_t n_read, uint32_t n_written); -int control_event_stream_bandwidth(edge_connection_t *edge_conn); -int control_event_stream_bandwidth_used(void); -void control_event_logmsg(int severity, unsigned int domain, const char *msg); -int control_event_descriptors_changed(smartlist_t *routers); -int control_event_address_mapped(const char *from, const char *to, - time_t expires, const char *error); -int control_event_or_authdir_new_descriptor(const char *action, - const char *desc, - size_t desclen, - const char *msg); -int control_event_my_descriptor_changed(void); -int control_event_networkstatus_changed(smartlist_t *statuses); -int control_event_newconsensus(const networkstatus_t *consensus); -int control_event_networkstatus_changed_single(routerstatus_t *rs); -int control_event_general_status(int severity, const char *format, ...) - CHECK_PRINTF(2,3); -int control_event_client_status(int severity, const char *format, ...) - CHECK_PRINTF(2,3); -int control_event_server_status(int severity, const char *format, ...) - CHECK_PRINTF(2,3); -int control_event_guard(const char *nickname, const char *digest, - const char *status); - -int init_cookie_authentication(int enabled); -smartlist_t *decode_hashed_passwords(config_line_t *passwords); -void disable_control_logging(void); -void enable_control_logging(void); - /** Enum describing various stages of bootstrapping, for use with controller * bootstrap status events. The values range from 0 to 100. */ typedef enum { @@ -3292,427 +3231,133 @@ typedef enum { BOOTSTRAP_STATUS_DONE=100 } bootstrap_status_t; -void control_event_bootstrap(bootstrap_status_t status, int progress); -void control_event_bootstrap_problem(const char *warn, int reason); - -void control_event_clients_seen(const char *timestarted, - const char *countries); - -#ifdef CONTROL_PRIVATE -/* Used only by control.c and test.c */ -size_t write_escaped_data(const char *data, size_t len, char **out); -size_t read_escaped_data(const char *data, size_t len, char **out); -#endif - -/********************************* cpuworker.c *****************************/ - -void cpu_init(void); -void cpuworkers_rotate(void); -int connection_cpu_finished_flushing(connection_t *conn); -int connection_cpu_reached_eof(connection_t *conn); -int connection_cpu_process_inbuf(connection_t *conn); -int assign_onionskin_to_cpuworker(connection_t *cpuworker, - or_circuit_t *circ, - char *onionskin); - /********************************* directory.c ***************************/ -int directories_have_accepted_server_descriptor(void); -char *authority_type_to_string(authority_type_t auth); -void directory_post_to_dirservers(uint8_t dir_purpose, uint8_t router_purpose, - authority_type_t type, const char *payload, - size_t payload_len, size_t extrainfo_len); -void directory_get_from_dirserver(uint8_t dir_purpose, uint8_t router_purpose, - const char *resource, - int pds_flags); -void directory_get_from_all_authorities(uint8_t dir_purpose, - uint8_t router_purpose, - const char *resource); -void directory_initiate_command_routerstatus(routerstatus_t *status, - uint8_t dir_purpose, - uint8_t router_purpose, - int anonymized_connection, - const char *resource, - const char *payload, - size_t payload_len, - time_t if_modified_since); -void directory_initiate_command_routerstatus_rend(routerstatus_t *status, - uint8_t dir_purpose, - uint8_t router_purpose, - int anonymized_connection, - const char *resource, - const char *payload, - size_t payload_len, - time_t if_modified_since, - const rend_data_t *rend_query); - -int parse_http_response(const char *headers, int *code, time_t *date, - compress_method_t *compression, char **response); - -int connection_dir_is_encrypted(dir_connection_t *conn); -int connection_dir_reached_eof(dir_connection_t *conn); -int connection_dir_process_inbuf(dir_connection_t *conn); -int connection_dir_finished_flushing(dir_connection_t *conn); -int connection_dir_finished_connecting(dir_connection_t *conn); -void connection_dir_request_failed(dir_connection_t *conn); -void directory_initiate_command(const char *address, const tor_addr_t *addr, - uint16_t or_port, uint16_t dir_port, - int supports_conditional_consensus, - int supports_begindir, const char *digest, - uint8_t dir_purpose, uint8_t router_purpose, - int anonymized_connection, - const char *resource, - const char *payload, size_t payload_len, - time_t if_modified_since); - -int dir_split_resource_into_fingerprints(const char *resource, - smartlist_t *fp_out, int *compresseed_out, - int decode_hex, int sort_uniq); /** A pair of digests created by dir_split_resource_info_fingerprint_pairs() */ typedef struct { char first[DIGEST_LEN]; char second[DIGEST_LEN]; } fp_pair_t; -int dir_split_resource_into_fingerprint_pairs(const char *res, - smartlist_t *pairs_out); -char *directory_dump_request_log(void); -void note_request(const char *key, size_t bytes); -int router_supports_extrainfo(const char *identity_digest, int is_authority); - -time_t download_status_increment_failure(download_status_t *dls, - int status_code, const char *item, - int server, time_t now); -/** Increment the failure count of the download_status_t <b>dls</b>, with - * the optional status code <b>sc</b>. */ -#define download_status_failed(dls, sc) \ - download_status_increment_failure((dls), (sc), NULL, \ - get_options()->DirPort, time(NULL)) - -void download_status_reset(download_status_t *dls); -static int download_status_is_ready(download_status_t *dls, time_t now, - int max_failures); -/** Return true iff, as of <b>now</b>, the resource tracked by <b>dls</b> is - * ready to get its download reattempted. */ -static INLINE int -download_status_is_ready(download_status_t *dls, time_t now, - int max_failures) -{ - return (dls->n_download_failures <= max_failures - && dls->next_attempt_at <= now); -} - -static void download_status_mark_impossible(download_status_t *dl); -/** Mark <b>dl</b> as never downloadable. */ -static INLINE void -download_status_mark_impossible(download_status_t *dl) -{ - dl->n_download_failures = IMPOSSIBLE_TO_DOWNLOAD; -} /********************************* dirserv.c ***************************/ -/** Maximum length of an exit policy summary. */ -#define MAX_EXITPOLICY_SUMMARY_LEN (1000) - -/** Maximum allowable length of a version line in a networkstatus. */ -#define MAX_V_LINE_LEN 128 -/** Length of "r Authority BadDirectory BadExit Exit Fast Guard HSDir Named - * Running Stable Unnamed V2Dir Valid\n". */ -#define MAX_FLAG_LINE_LEN 96 -/** Length of "w" line for weighting. Currently at most - * "w Bandwidth=<uint32t>\n" */ -#define MAX_WEIGHT_LINE_LEN (13+10) -/** Maximum length of an exit policy summary line. */ -#define MAX_POLICY_LINE_LEN (3+MAX_EXITPOLICY_SUMMARY_LEN) -/** Amount of space to allocate for each entry: r, s, and v lines. */ -#define RS_ENTRY_LEN \ - ( /* first line */ \ - MAX_NICKNAME_LEN+BASE64_DIGEST_LEN*2+ISO_TIME_LEN+INET_NTOA_BUF_LEN+ \ - 5*2 /* ports */ + 10 /* punctuation */ + \ - /* second line */ \ - MAX_FLAG_LINE_LEN + \ - /* weight line */ \ - MAX_WEIGHT_LINE_LEN + \ - /* p line. */ \ - MAX_POLICY_LINE_LEN + \ - /* v line. */ \ - MAX_V_LINE_LEN \ - ) -#define UNNAMED_ROUTER_NICKNAME "Unnamed" - -int connection_dirserv_flushed_some(dir_connection_t *conn); - -int dirserv_add_own_fingerprint(const char *nickname, crypto_pk_env_t *pk); -int dirserv_load_fingerprint_file(void); -void dirserv_free_fingerprint_list(void); -const char *dirserv_get_nickname_by_digest(const char *digest); -enum was_router_added_t dirserv_add_multiple_descriptors( - const char *desc, uint8_t purpose, - const char *source, - const char **msg); -enum was_router_added_t dirserv_add_descriptor(routerinfo_t *ri, - const char **msg, - const char *source); -int getinfo_helper_dirserv_unregistered(control_connection_t *conn, - const char *question, char **answer); -void dirserv_free_descriptors(void); -void dirserv_set_router_is_running(routerinfo_t *router, time_t now); -int list_server_status_v1(smartlist_t *routers, char **router_status_out, - int for_controller); -int dirserv_dump_directory_to_string(char **dir_out, - crypto_pk_env_t *private_key); - -int directory_fetches_from_authorities(or_options_t *options); -int directory_fetches_dir_info_early(or_options_t *options); -int directory_fetches_dir_info_later(or_options_t *options); -int directory_caches_v2_dir_info(or_options_t *options); -#define directory_caches_v1_dir_info(o) directory_caches_v2_dir_info(o) -int directory_caches_dir_info(or_options_t *options); -int directory_permits_begindir_requests(or_options_t *options); -int directory_permits_controller_requests(or_options_t *options); -int directory_too_idle_to_fetch_descriptors(or_options_t *options, time_t now); - -void directory_set_dirty(void); -cached_dir_t *dirserv_get_directory(void); -cached_dir_t *dirserv_get_runningrouters(void); -cached_dir_t *dirserv_get_consensus(void); -void dirserv_set_cached_directory(const char *directory, time_t when, - int is_running_routers); -void dirserv_set_cached_networkstatus_v2(const char *directory, - const char *identity, - time_t published); -void dirserv_set_cached_networkstatus_v3(const char *consensus, - time_t published); -void dirserv_clear_old_networkstatuses(time_t cutoff); -void dirserv_clear_old_v1_info(time_t now); -void dirserv_get_networkstatus_v2(smartlist_t *result, const char *key); -void dirserv_get_networkstatus_v2_fingerprints(smartlist_t *result, - const char *key); -int dirserv_get_routerdesc_fingerprints(smartlist_t *fps_out, const char *key, - const char **msg, - int for_unencrypted_conn, - int is_extrainfo); -int dirserv_get_routerdescs(smartlist_t *descs_out, const char *key, - const char **msg); -void dirserv_orconn_tls_done(const char *address, - uint16_t or_port, - const char *digest_rcvd, - int as_advertised); -void dirserv_test_reachability(time_t now, int try_all); -int authdir_wants_to_reject_router(routerinfo_t *ri, const char **msg, - int complain); -int dirserv_would_reject_router(routerstatus_t *rs); -int dirserv_remove_old_statuses(smartlist_t *fps, time_t cutoff); -int dirserv_have_any_serverdesc(smartlist_t *fps, int spool_src); -size_t dirserv_estimate_data_size(smartlist_t *fps, int is_serverdescs, - int compressed); -int routerstatus_format_entry(char *buf, size_t buf_len, - routerstatus_t *rs, const char *platform, - int first_line_only, int v2_format); -void dirserv_free_all(void); -void cached_dir_decref(cached_dir_t *d); -cached_dir_t *new_cached_dir(char *s, time_t published); -/********************************* dirvote.c ************************/ +/** An enum to describe what format we're generating a routerstatus line in. + */ +typedef enum { + /** For use in a v2 opinion */ + NS_V2, + /** For use in a consensus networkstatus document (ns flavor) */ + NS_V3_CONSENSUS, + /** For use in a vote networkstatus document */ + NS_V3_VOTE, + /** For passing to the controlport in response to a GETINFO request */ + NS_CONTROL_PORT, + /** For use in a consensus networkstatus document (microdesc flavor) */ + NS_V3_CONSENSUS_MICRODESC +} routerstatus_format_type_t; + +#ifdef DIRSERV_PRIVATE +typedef struct measured_bw_line_t { + char node_id[DIGEST_LEN]; + char node_hex[MAX_HEX_NICKNAME_LEN+1]; + long int bw; +} measured_bw_line_t; -/** Lowest allowable value for VoteSeconds. */ -#define MIN_VOTE_SECONDS 20 -/** Lowest allowable value for DistSeconds. */ -#define MIN_DIST_SECONDS 20 -/** Smallest allowable voting interval. */ -#define MIN_VOTE_INTERVAL 300 - -void dirvote_free_all(void); - -/* vote manipulation */ -char *networkstatus_compute_consensus(smartlist_t *votes, - int total_authorities, - crypto_pk_env_t *identity_key, - crypto_pk_env_t *signing_key, - const char *legacy_identity_key_digest, - crypto_pk_env_t *legacy_signing_key); -int networkstatus_add_detached_signatures(networkstatus_t *target, - ns_detached_signatures_t *sigs, - const char **msg_out); -char *networkstatus_get_detached_signatures(networkstatus_t *consensus); -void ns_detached_signatures_free(ns_detached_signatures_t *s); - -/* cert manipulation */ -authority_cert_t *authority_cert_dup(authority_cert_t *cert); +#endif + +/********************************* dirvote.c ************************/ /** Describes the schedule by which votes should be generated. */ typedef struct vote_timing_t { + /** Length in seconds between one consensus becoming valid and the next + * becoming valid. */ int vote_interval; + /** For how many intervals is a consensus valid? */ int n_intervals_valid; + /** Time in seconds allowed to propagate votes */ int vote_delay; + /** Time in seconds allowed to propagate signatures */ int dist_delay; } vote_timing_t; -/* vote scheduling */ -void dirvote_get_preferred_voting_intervals(vote_timing_t *timing_out); -time_t dirvote_get_start_of_next_interval(time_t now, int interval); -void dirvote_recalculate_timing(or_options_t *options, time_t now); -void dirvote_act(or_options_t *options, time_t now); - -/* invoked on timers and by outside triggers. */ -struct pending_vote_t * dirvote_add_vote(const char *vote_body, - const char **msg_out, - int *status_out); -int dirvote_add_signatures(const char *detached_signatures_body, - const char *source, - const char **msg_out); - -/* Item access */ -const char *dirvote_get_pending_consensus(void); -const char *dirvote_get_pending_detached_signatures(void); -#define DGV_BY_ID 1 -#define DGV_INCLUDE_PENDING 2 -#define DGV_INCLUDE_PREVIOUS 4 -const cached_dir_t *dirvote_get_vote(const char *fp, int flags); -void set_routerstatus_from_routerinfo(routerstatus_t *rs, - routerinfo_t *ri, time_t now, - int naming, int exits_can_be_guards, - int listbadexits, int listbaddirs); -void router_clear_status_flags(routerinfo_t *ri); -networkstatus_t * -dirserv_generate_networkstatus_vote_obj(crypto_pk_env_t *private_key, - authority_cert_t *cert); - -#ifdef DIRVOTE_PRIVATE -char *format_networkstatus_vote(crypto_pk_env_t *private_key, - networkstatus_t *v3_ns); -char *dirvote_compute_params(smartlist_t *votes); -#endif - -/********************************* dns.c ***************************/ - -int dns_init(void); -int has_dns_init_failed(void); -void dns_free_all(void); -uint32_t dns_clip_ttl(uint32_t ttl); -int dns_reset(void); -void connection_dns_remove(edge_connection_t *conn); -void assert_connection_edge_not_dns_pending(edge_connection_t *conn); -void assert_all_pending_dns_resolves_ok(void); -void dns_cancel_pending_resolve(const char *question); -int dns_resolve(edge_connection_t *exitconn); -void dns_launch_correctness_checks(void); -int dns_seems_to_be_broken(void); -void dns_reset_correctness_checks(void); - -/********************************* dnsserv.c ************************/ - -void dnsserv_configure_listener(connection_t *conn); -void dnsserv_close_listener(connection_t *conn); -void dnsserv_resolved(edge_connection_t *conn, - int answer_type, - size_t answer_len, - const char *answer, - int ttl); -void dnsserv_reject_request(edge_connection_t *conn); -int dnsserv_launch_request(const char *name, int is_reverse); /********************************* geoip.c **************************/ -#ifdef GEOIP_PRIVATE -int geoip_parse_entry(const char *line); -#endif -int should_record_bridge_info(or_options_t *options); -int geoip_load_file(const char *filename, or_options_t *options); -int geoip_get_country_by_ip(uint32_t ipaddr); -int geoip_get_n_countries(void); -const char *geoip_get_country_name(country_t num); -int geoip_is_loaded(void); -country_t geoip_get_country(const char *countrycode); +/** Round all GeoIP results to the next multiple of this value, to avoid + * leaking information. */ +#define DIR_RECORD_USAGE_GRANULARITY 8 +/** Time interval: Flush geoip data to disk this often. */ +#define DIR_ENTRY_RECORD_USAGE_RETAIN_IPS (24*60*60) +/** How long do we have to have observed per-country request history before + * we are willing to talk about it? */ +#define DIR_RECORD_USAGE_MIN_OBSERVATION_TIME (12*60*60) + /** Indicates an action that we might be noting geoip statistics on. * Note that if we're noticing CONNECT, we're a bridge, and if we're noticing * the others, we're not. */ typedef enum { - /** We've noticed a connection as a bridge relay. */ + /** We've noticed a connection as a bridge relay or entry guard. */ GEOIP_CLIENT_CONNECT = 0, /** We've served a networkstatus consensus as a directory server. */ GEOIP_CLIENT_NETWORKSTATUS = 1, /** We've served a v2 networkstatus consensus as a directory server. */ GEOIP_CLIENT_NETWORKSTATUS_V2 = 2, } geoip_client_action_t; -void geoip_note_client_seen(geoip_client_action_t action, - uint32_t addr, time_t now); -void geoip_remove_old_clients(time_t cutoff); -time_t geoip_get_history_start(void); -char *geoip_get_client_history(time_t now, geoip_client_action_t action); -char *geoip_get_request_history(time_t now, geoip_client_action_t action); -int getinfo_helper_geoip(control_connection_t *control_conn, - const char *question, char **answer); -void geoip_free_all(void); -void dump_geoip_stats(void); - -/********************************* hibernate.c **********************/ - -int accounting_parse_options(or_options_t *options, int validate_only); -int accounting_is_enabled(or_options_t *options); -void configure_accounting(time_t now); -void accounting_run_housekeeping(time_t now); -void accounting_add_bytes(size_t n_read, size_t n_written, int seconds); -int accounting_record_bandwidth_usage(time_t now, or_state_t *state); -void hibernate_begin_shutdown(void); -int we_are_hibernating(void); -void consider_hibernation(time_t now); -int getinfo_helper_accounting(control_connection_t *conn, - const char *question, char **answer); -void accounting_set_bandwidth_usage_from_state(or_state_t *state); - -/********************************* main.c ***************************/ - -extern int has_completed_circuit; - -int connection_add(connection_t *conn); -int connection_remove(connection_t *conn); -int connection_in_array(connection_t *conn); -void add_connection_to_closeable_list(connection_t *conn); -int connection_is_on_closeable_list(connection_t *conn); - -smartlist_t *get_connection_array(void); - -void connection_watch_events(connection_t *conn, short events); -int connection_is_reading(connection_t *conn); -void connection_stop_reading(connection_t *conn); -void connection_start_reading(connection_t *conn); - -int connection_is_writing(connection_t *conn); -void connection_stop_writing(connection_t *conn); -void connection_start_writing(connection_t *conn); - -void connection_stop_reading_from_linked_conn(connection_t *conn); - -void directory_all_unreachable(time_t now); -void directory_info_has_arrived(time_t now, int from_cache); - -void ip_address_changed(int at_interface); -void dns_servers_relaunch_checks(void); - -void control_signal_act(int the_signal); -void handle_signals(int is_parent); - -int try_locking(or_options_t *options, int err_if_locked); -int have_lockfile(void); -void release_lockfile(void); - -void tor_cleanup(void); -void tor_free_all(int postfork); - -int tor_main(int argc, char *argv[]); - -#ifdef MAIN_PRIVATE -int do_main_loop(void); -int do_list_fingerprint(void); -void do_hash_password(void); -int tor_init(int argc, char **argv); -#endif +/** Indicates either a positive reply or a reason for rejectng a network + * status request that will be included in geoip statistics. */ +typedef enum { + /** Request is answered successfully. */ + GEOIP_SUCCESS = 0, + /** V3 network status is not signed by a sufficient number of requested + * authorities. */ + GEOIP_REJECT_NOT_ENOUGH_SIGS = 1, + /** Requested network status object is unavailable. */ + GEOIP_REJECT_UNAVAILABLE = 2, + /** Requested network status not found. */ + GEOIP_REJECT_NOT_FOUND = 3, + /** Network status has not been modified since If-Modified-Since time. */ + GEOIP_REJECT_NOT_MODIFIED = 4, + /** Directory is busy. */ + GEOIP_REJECT_BUSY = 5, +} geoip_ns_response_t; +#define GEOIP_NS_RESPONSE_NUM 6 + +/** Directory requests that we are measuring can be either direct or + * tunneled. */ +typedef enum { + DIRREQ_DIRECT = 0, + DIRREQ_TUNNELED = 1, +} dirreq_type_t; -/********************************* networkstatus.c *********************/ +/** Possible states for either direct or tunneled directory requests that + * are relevant for determining network status download times. */ +typedef enum { + /** Found that the client requests a network status; applies to both + * direct and tunneled requests; initial state of a request that we are + * measuring. */ + DIRREQ_IS_FOR_NETWORK_STATUS = 0, + /** Finished writing a network status to the directory connection; + * applies to both direct and tunneled requests; completes a direct + * request. */ + DIRREQ_FLUSHING_DIR_CONN_FINISHED = 1, + /** END cell sent to circuit that initiated a tunneled request. */ + DIRREQ_END_CELL_SENT = 2, + /** Flushed last cell from queue of the circuit that initiated a + * tunneled request to the outbuf of the OR connection. */ + DIRREQ_CIRC_QUEUE_FLUSHED = 3, + /** Flushed last byte from buffer of the OR connection belonging to the + * circuit that initiated a tunneled request; completes a tunneled + * request. */ + DIRREQ_OR_CONN_BUFFER_FLUSHED = 4 +} dirreq_state_t; + +#define WRITE_STATS_INTERVAL (24*60*60) + +/********************************* microdesc.c *************************/ + +typedef struct microdesc_cache_t microdesc_cache_t; -/** How old do we allow a v2 network-status to get before removing it - * completely? */ -#define MAX_NETWORKSTATUS_AGE (10*24*60*60) +/********************************* networkstatus.c *********************/ /** Location where we found a v2 networkstatus. */ typedef enum { @@ -3733,127 +3378,8 @@ typedef enum version_status_t { VS_UNKNOWN, /**< We have no idea. */ } version_status_t; -void networkstatus_reset_warnings(void); -void networkstatus_reset_download_failures(void); -int router_reload_v2_networkstatus(void); -int router_reload_consensus_networkstatus(void); -void routerstatus_free(routerstatus_t *rs); -void networkstatus_v2_free(networkstatus_v2_t *ns); -void networkstatus_vote_free(networkstatus_t *ns); -networkstatus_voter_info_t *networkstatus_get_voter_by_id( - networkstatus_t *vote, - const char *identity); -int networkstatus_check_consensus_signature(networkstatus_t *consensus, - int warn); -int networkstatus_check_voter_signature(networkstatus_t *consensus, - networkstatus_voter_info_t *voter, - authority_cert_t *cert); -char *networkstatus_get_cache_filename(const char *identity_digest); -int router_set_networkstatus_v2(const char *s, time_t arrived_at, - v2_networkstatus_source_t source, - smartlist_t *requested_fingerprints); -void networkstatus_v2_list_clean(time_t now); -routerstatus_t *networkstatus_v2_find_entry(networkstatus_v2_t *ns, - const char *digest); -routerstatus_t *networkstatus_vote_find_entry(networkstatus_t *ns, - const char *digest); -int networkstatus_vote_find_entry_idx(networkstatus_t *ns, - const char *digest, int *found_out); -const smartlist_t *networkstatus_get_v2_list(void); -download_status_t *router_get_dl_status_by_descriptor_digest(const char *d); -routerstatus_t *router_get_consensus_status_by_id(const char *digest); -routerstatus_t *router_get_consensus_status_by_descriptor_digest( - const char *digest); -routerstatus_t *router_get_consensus_status_by_nickname(const char *nickname, - int warn_if_unnamed); -const char *networkstatus_get_router_digest_by_nickname(const char *nickname); -int networkstatus_nickname_is_unnamed(const char *nickname); -void networkstatus_consensus_download_failed(int status_code); -void update_consensus_networkstatus_fetch_time(time_t now); -int should_delay_dir_fetches(or_options_t *options); -void update_networkstatus_downloads(time_t now); -void update_certificate_downloads(time_t now); -int consensus_is_waiting_for_certs(void); -networkstatus_v2_t *networkstatus_v2_get_by_digest(const char *digest); -networkstatus_t *networkstatus_get_latest_consensus(void); -networkstatus_t *networkstatus_get_live_consensus(time_t now); -networkstatus_t *networkstatus_get_reasonably_live_consensus(time_t now); -#define NSSET_FROM_CACHE 1 -#define NSSET_WAS_WAITING_FOR_CERTS 2 -#define NSSET_DONT_DOWNLOAD_CERTS 4 -#define NSSET_ACCEPT_OBSOLETE 8 -int networkstatus_set_current_consensus(const char *consensus, unsigned flags); -void networkstatus_note_certs_arrived(void); -void routers_update_all_from_networkstatus(time_t now, int dir_version); -void routerstatus_list_update_from_consensus_networkstatus(time_t now); -void routers_update_status_from_consensus_networkstatus(smartlist_t *routers, - int reset_failures); -void signed_descs_update_status_from_consensus_networkstatus( - smartlist_t *descs); - -char *networkstatus_getinfo_helper_single(routerstatus_t *rs); -char *networkstatus_getinfo_by_purpose(const char *purpose_string, time_t now); -void networkstatus_dump_bridge_status_to_file(time_t now); -int32_t networkstatus_get_param(networkstatus_t *ns, const char *param_name, - int32_t default_val); -int getinfo_helper_networkstatus(control_connection_t *conn, - const char *question, char **answer); -void networkstatus_free_all(void); - -/********************************* ntmain.c ***************************/ -#ifdef MS_WINDOWS -#define NT_SERVICE -#endif - -#ifdef NT_SERVICE -int nt_service_parse_options(int argc, char **argv, int *should_exit); -int nt_service_is_stopping(void); -void nt_service_set_state(DWORD state); -#else -#define nt_service_is_stopping() (0) -#endif - -/********************************* onion.c ***************************/ - -int onion_pending_add(or_circuit_t *circ, char *onionskin); -or_circuit_t *onion_next_task(char **onionskin_out); -void onion_pending_remove(or_circuit_t *circ); - -int onion_skin_create(crypto_pk_env_t *router_key, - crypto_dh_env_t **handshake_state_out, - char *onion_skin_out); - -int onion_skin_server_handshake(const char *onion_skin, - crypto_pk_env_t *private_key, - crypto_pk_env_t *prev_private_key, - char *handshake_reply_out, - char *key_out, - size_t key_out_len); - -int onion_skin_client_handshake(crypto_dh_env_t *handshake_state, - const char *handshake_reply, - char *key_out, - size_t key_out_len); - -int fast_server_handshake(const uint8_t *key_in, - uint8_t *handshake_reply_out, - uint8_t *key_out, - size_t key_out_len); - -int fast_client_handshake(const uint8_t *handshake_state, - const uint8_t *handshake_reply_out, - uint8_t *key_out, - size_t key_out_len); - -void clear_pending_onions(void); - /********************************* policies.c ************************/ -/* (length of "accept 255.255.255.255/255.255.255.255:65535-65535\n" plus a - * NUL.) - */ -#define POLICY_BUF_LEN 52 - /** Outcome of applying an address policy to an address. */ typedef enum { /** The address was accepted */ @@ -3868,146 +3394,8 @@ typedef enum { ADDR_POLICY_PROBABLY_REJECTED=2 } addr_policy_result_t; -int firewall_is_fascist_or(void); -int fascist_firewall_allows_address_or(const tor_addr_t *addr, uint16_t port); -int fascist_firewall_allows_or(routerinfo_t *ri); -int fascist_firewall_allows_address_dir(const tor_addr_t *addr, uint16_t port); -int dir_policy_permits_address(const tor_addr_t *addr); -int socks_policy_permits_address(const tor_addr_t *addr); -int authdir_policy_permits_address(uint32_t addr, uint16_t port); -int authdir_policy_valid_address(uint32_t addr, uint16_t port); -int authdir_policy_baddir_address(uint32_t addr, uint16_t port); -int authdir_policy_badexit_address(uint32_t addr, uint16_t port); - -int validate_addr_policies(or_options_t *options, char **msg); -void policy_expand_private(smartlist_t **policy); -int policies_parse_from_options(or_options_t *options); - -addr_policy_t *addr_policy_get_canonical_entry(addr_policy_t *ent); -int cmp_addr_policies(smartlist_t *a, smartlist_t *b); -addr_policy_result_t compare_tor_addr_to_addr_policy(const tor_addr_t *addr, - uint16_t port, const smartlist_t *policy); -addr_policy_result_t compare_addr_to_addr_policy(uint32_t addr, - uint16_t port, const smartlist_t *policy); -int policies_parse_exit_policy(config_line_t *cfg, smartlist_t **dest, - int rejectprivate, const char *local_address); -void policies_set_router_exitpolicy_to_reject_all(routerinfo_t *exitrouter); -int exit_policy_is_general_exit(smartlist_t *policy); -int policy_is_reject_star(const smartlist_t *policy); -int getinfo_helper_policies(control_connection_t *conn, - const char *question, char **answer); -int policy_write_item(char *buf, size_t buflen, addr_policy_t *item, - int format_for_desc); - -void addr_policy_list_free(smartlist_t *p); -void addr_policy_free(addr_policy_t *p); -void policies_free_all(void); - -char *policy_summarize(smartlist_t *policy); - -/********************************* reasons.c ***************************/ - -const char *stream_end_reason_to_control_string(int reason); -const char *stream_end_reason_to_string(int reason); -socks5_reply_status_t stream_end_reason_to_socks5_response(int reason); -uint8_t errno_to_stream_end_reason(int e); - -const char *orconn_end_reason_to_control_string(int r); -int tls_error_to_orconn_end_reason(int e); -int errno_to_orconn_end_reason(int e); - -const char *circuit_end_reason_to_control_string(int reason); - -/********************************* relay.c ***************************/ - -extern uint64_t stats_n_relay_cells_relayed; -extern uint64_t stats_n_relay_cells_delivered; - -int circuit_receive_relay_cell(cell_t *cell, circuit_t *circ, - cell_direction_t cell_direction); - -void relay_header_pack(uint8_t *dest, const relay_header_t *src); -void relay_header_unpack(relay_header_t *dest, const uint8_t *src); -int relay_send_command_from_edge(streamid_t stream_id, circuit_t *circ, - uint8_t relay_command, const char *payload, - size_t payload_len, crypt_path_t *cpath_layer); -int connection_edge_send_command(edge_connection_t *fromconn, - uint8_t relay_command, const char *payload, - size_t payload_len); -int connection_edge_package_raw_inbuf(edge_connection_t *conn, - int package_partial); -void connection_edge_consider_sending_sendme(edge_connection_t *conn); - -extern uint64_t stats_n_data_cells_packaged; -extern uint64_t stats_n_data_bytes_packaged; -extern uint64_t stats_n_data_cells_received; -extern uint64_t stats_n_data_bytes_received; - -void init_cell_pool(void); -void free_cell_pool(void); -void clean_cell_pool(void); -void dump_cell_pool_usage(int severity); - -void cell_queue_clear(cell_queue_t *queue); -void cell_queue_append(cell_queue_t *queue, packed_cell_t *cell); -void cell_queue_append_packed_copy(cell_queue_t *queue, const cell_t *cell); - -void append_cell_to_circuit_queue(circuit_t *circ, or_connection_t *orconn, - cell_t *cell, cell_direction_t direction); -void connection_or_unlink_all_active_circs(or_connection_t *conn); -int connection_or_flush_from_first_active_circuit(or_connection_t *conn, - int max, time_t now); -void assert_active_circuits_ok(or_connection_t *orconn); -void make_circuit_inactive_on_conn(circuit_t *circ, or_connection_t *conn); -void make_circuit_active_on_conn(circuit_t *circ, or_connection_t *conn); - -int append_address_to_payload(uint8_t *payload_out, const tor_addr_t *addr); -const uint8_t *decode_address_from_payload(tor_addr_t *addr_out, - const uint8_t *payload, - int payload_len); - /********************************* rephist.c ***************************/ -void rep_hist_init(void); -void rep_hist_note_connect_failed(const char* nickname, time_t when); -void rep_hist_note_connect_succeeded(const char* nickname, time_t when); -void rep_hist_note_disconnect(const char* nickname, time_t when); -void rep_hist_note_connection_died(const char* nickname, time_t when); -void rep_hist_note_extend_succeeded(const char *from_name, - const char *to_name); -void rep_hist_note_extend_failed(const char *from_name, const char *to_name); -void rep_hist_dump_stats(time_t now, int severity); -void rep_hist_note_bytes_read(size_t num_bytes, time_t when); -void rep_hist_note_bytes_written(size_t num_bytes, time_t when); -int rep_hist_bandwidth_assess(void); -char *rep_hist_get_bandwidth_lines(int for_extrainfo); -void rep_hist_update_state(or_state_t *state); -int rep_hist_load_state(or_state_t *state, char **err); -void rep_history_clean(time_t before); - -void rep_hist_note_router_reachable(const char *id, time_t when); -void rep_hist_note_router_unreachable(const char *id, time_t when); -int rep_hist_record_mtbf_data(time_t now, int missing_means_down); -int rep_hist_load_mtbf_data(time_t now); - -time_t rep_hist_downrate_old_runs(time_t now); -double rep_hist_get_stability(const char *id, time_t when); -double rep_hist_get_weighted_fractional_uptime(const char *id, time_t when); -long rep_hist_get_weighted_time_known(const char *id, time_t when); -int rep_hist_have_measured_enough_stability(void); -const char *rep_hist_get_router_stability_doc(time_t now); - -void rep_hist_note_used_port(time_t now, uint16_t port); -smartlist_t *rep_hist_get_predicted_ports(time_t now); -void rep_hist_note_used_resolve(time_t now); -void rep_hist_note_used_internal(time_t now, int need_uptime, - int need_capacity); -int rep_hist_get_predicted_internal(time_t now, int *need_uptime, - int *need_capacity); - -int any_predicted_circuits(time_t now); -int rep_hist_circbuilding_dormant(time_t now); - /** Possible public/private key operations in Tor: used to keep track of where * we're spending our time. */ typedef enum { @@ -4017,48 +3405,6 @@ typedef enum { TLS_HANDSHAKE_C, TLS_HANDSHAKE_S, REND_CLIENT, REND_MID, REND_SERVER, } pk_op_t; -void note_crypto_pk_op(pk_op_t operation); -void dump_pk_ops(int severity); - -void rep_hist_free_all(void); - -/* for hidden service usage statistics */ -void hs_usage_note_publish_total(const char *service_id, time_t now); -void hs_usage_note_publish_novel(const char *service_id, time_t now); -void hs_usage_note_fetch_total(const char *service_id, time_t now); -void hs_usage_note_fetch_successful(const char *service_id, time_t now); -void hs_usage_write_statistics_to_file(time_t now); -void hs_usage_free_all(void); - -/********************************* rendclient.c ***************************/ - -void rend_client_introcirc_has_opened(origin_circuit_t *circ); -void rend_client_rendcirc_has_opened(origin_circuit_t *circ); -int rend_client_introduction_acked(origin_circuit_t *circ, - const uint8_t *request, - size_t request_len); -void rend_client_refetch_renddesc(const char *query); -void rend_client_refetch_v2_renddesc(const rend_data_t *rend_query); -int rend_client_remove_intro_point(extend_info_t *failed_intro, - const rend_data_t *rend_query); -int rend_client_rendezvous_acked(origin_circuit_t *circ, - const uint8_t *request, - size_t request_len); -int rend_client_receive_rendezvous(origin_circuit_t *circ, - const uint8_t *request, - size_t request_len); -void rend_client_desc_trynow(const char *query, int rend_version); - -extend_info_t *rend_client_get_random_intro(const rend_data_t *rend_query); - -int rend_client_send_introduction(origin_circuit_t *introcirc, - origin_circuit_t *rendcirc); -int rend_parse_service_authorization(or_options_t *options, - int validate_only); -rend_service_authorization_t *rend_client_lookup_service_authorization( - const char *onion_address); -void rend_service_authorization_free_all(void); -rend_data_t *rend_data_dup(const rend_data_t *request); /********************************* rendcommon.c ***************************/ @@ -4101,31 +3447,6 @@ typedef struct rend_service_descriptor_t { smartlist_t *successful_uploads; } rend_service_descriptor_t; -/** Free all storage associated with <b>data</b> */ -static INLINE void -rend_data_free(rend_data_t *data) -{ - tor_free(data); -} - -int rend_cmp_service_ids(const char *one, const char *two); - -void rend_process_relay_cell(circuit_t *circ, const crypt_path_t *layer_hint, - int command, size_t length, - const uint8_t *payload); - -void rend_service_descriptor_free(rend_service_descriptor_t *desc); -int rend_encode_service_descriptor(rend_service_descriptor_t *desc, - crypto_pk_env_t *key, - char **str_out, - size_t *len_out); -rend_service_descriptor_t *rend_parse_service_descriptor(const char *str, - size_t len); -int rend_get_service_id(crypto_pk_env_t *pk, char *out); -void rend_encoded_v2_service_descriptor_free( - rend_encoded_v2_service_descriptor_t *desc); -void rend_intro_point_free(rend_intro_point_t *intro); - /** A cached rendezvous descriptor. */ typedef struct rend_cache_entry_t { size_t len; /**< Length of <b>desc</b> */ @@ -4134,150 +3455,6 @@ typedef struct rend_cache_entry_t { rend_service_descriptor_t *parsed; /**< Parsed value of 'desc' */ } rend_cache_entry_t; -void rend_cache_init(void); -void rend_cache_clean(void); -void rend_cache_clean_v2_descs_as_dir(void); -void rend_cache_free_all(void); -int rend_valid_service_id(const char *query); -int rend_cache_lookup_desc(const char *query, int version, const char **desc, - size_t *desc_len); -int rend_cache_lookup_entry(const char *query, int version, - rend_cache_entry_t **entry_out); -int rend_cache_lookup_v2_desc_as_dir(const char *query, const char **desc); -int rend_cache_store(const char *desc, size_t desc_len, int published); -int rend_cache_store_v2_desc_as_client(const char *desc, - const rend_data_t *rend_query); -int rend_cache_store_v2_desc_as_dir(const char *desc); -int rend_cache_size(void); -int rend_encode_v2_descriptors(smartlist_t *descs_out, - rend_service_descriptor_t *desc, time_t now, - uint8_t period, rend_auth_type_t auth_type, - crypto_pk_env_t *client_key, - smartlist_t *client_cookies); -int rend_compute_v2_desc_id(char *desc_id_out, const char *service_id, - const char *descriptor_cookie, - time_t now, uint8_t replica); -int rend_id_is_in_interval(const char *a, const char *b, const char *c); -void rend_get_descriptor_id_bytes(char *descriptor_id_out, - const char *service_id, - const char *secret_id_part); - -/********************************* rendservice.c ***************************/ - -int num_rend_services(void); -int rend_config_services(or_options_t *options, int validate_only); -int rend_service_load_keys(void); -void rend_services_init(void); -void rend_services_introduce(void); -void rend_consider_services_upload(time_t now); -void rend_hsdir_routers_changed(void); -void rend_consider_descriptor_republication(void); - -void rend_service_intro_has_opened(origin_circuit_t *circuit); -int rend_service_intro_established(origin_circuit_t *circuit, - const uint8_t *request, - size_t request_len); -void rend_service_rendezvous_has_opened(origin_circuit_t *circuit); -int rend_service_introduce(origin_circuit_t *circuit, const uint8_t *request, - size_t request_len); -void rend_service_relaunch_rendezvous(origin_circuit_t *oldcirc); -int rend_service_set_connection_addr_port(edge_connection_t *conn, - origin_circuit_t *circ); -void rend_service_dump_stats(int severity); -void rend_service_free_all(void); - -/********************************* rendmid.c *******************************/ -int rend_mid_establish_intro(or_circuit_t *circ, const uint8_t *request, - size_t request_len); -int rend_mid_introduce(or_circuit_t *circ, const uint8_t *request, - size_t request_len); -int rend_mid_establish_rendezvous(or_circuit_t *circ, const uint8_t *request, - size_t request_len); -int rend_mid_rendezvous(or_circuit_t *circ, const uint8_t *request, - size_t request_len); - -/********************************* router.c ***************************/ - -crypto_pk_env_t *get_onion_key(void); -time_t get_onion_key_set_at(void); -void set_identity_key(crypto_pk_env_t *k); -crypto_pk_env_t *get_identity_key(void); -int identity_key_is_set(void); -authority_cert_t *get_my_v3_authority_cert(void); -crypto_pk_env_t *get_my_v3_authority_signing_key(void); -authority_cert_t *get_my_v3_legacy_cert(void); -crypto_pk_env_t *get_my_v3_legacy_signing_key(void); -void dup_onion_keys(crypto_pk_env_t **key, crypto_pk_env_t **last); -void rotate_onion_key(void); -crypto_pk_env_t *init_key_from_file(const char *fname, int generate, - int severity); -void v3_authority_check_key_expiry(void); - -int init_keys(void); - -int check_whether_orport_reachable(void); -int check_whether_dirport_reachable(void); -void consider_testing_reachability(int test_or, int test_dir); -void router_orport_found_reachable(void); -void router_dirport_found_reachable(void); -void router_perform_bandwidth_test(int num_circs, time_t now); - -int authdir_mode(or_options_t *options); -int authdir_mode_v1(or_options_t *options); -int authdir_mode_v2(or_options_t *options); -int authdir_mode_v3(or_options_t *options); -int authdir_mode_any_main(or_options_t *options); -int authdir_mode_any_nonhidserv(or_options_t *options); -int authdir_mode_handles_descs(or_options_t *options, int purpose); -int authdir_mode_publishes_statuses(or_options_t *options); -int authdir_mode_tests_reachability(or_options_t *options); -int authdir_mode_bridge(or_options_t *options); - -int server_mode(or_options_t *options); -int advertised_server_mode(void); -int proxy_mode(or_options_t *options); -void consider_publishable_server(int force); - -void router_upload_dir_desc_to_dirservers(int force); -void mark_my_descriptor_dirty_if_older_than(time_t when); -void mark_my_descriptor_dirty(void); -void check_descriptor_bandwidth_changed(time_t now); -void check_descriptor_ipaddress_changed(time_t now); -void router_new_address_suggestion(const char *suggestion, - const dir_connection_t *d_conn); -int router_compare_to_my_exit_policy(edge_connection_t *conn); -routerinfo_t *router_get_my_routerinfo(void); -extrainfo_t *router_get_my_extrainfo(void); -const char *router_get_my_descriptor(void); -int router_digest_is_me(const char *digest); -int router_extrainfo_digest_is_me(const char *digest); -int router_is_me(routerinfo_t *router); -int router_fingerprint_is_me(const char *fp); -int router_pick_published_address(or_options_t *options, uint32_t *addr); -int router_rebuild_descriptor(int force); -int router_dump_router_to_string(char *s, size_t maxlen, routerinfo_t *router, - crypto_pk_env_t *ident_key); -int extrainfo_dump_to_string(char *s, size_t maxlen, extrainfo_t *extrainfo, - crypto_pk_env_t *ident_key); -char *extrainfo_get_client_geoip_summary(time_t); -int is_legal_nickname(const char *s); -int is_legal_nickname_or_hexdigest(const char *s); -int is_legal_hexdigest(const char *s); -void router_get_verbose_nickname(char *buf, const routerinfo_t *router); -void routerstatus_get_verbose_nickname(char *buf, - const routerstatus_t *router); -void router_reset_warnings(void); -void router_reset_reachability(void); -void router_free_all(void); - -const char *router_purpose_to_string(uint8_t p); -uint8_t router_purpose_from_string(const char *s); - -#ifdef ROUTER_PRIVATE -/* Used only by router.c and test.c */ -void get_platform_str(char *platform, size_t len); -#endif - /********************************* routerlist.c ***************************/ /** Represents information about a single trusted directory server. */ @@ -4317,22 +3494,7 @@ typedef struct trusted_dir_server_t { #define ROUTER_MAX_DECLARED_BANDWIDTH INT32_MAX -int get_n_authorities(authority_type_t type); -int trusted_dirs_reload_certs(void); -int trusted_dirs_load_certs_from_string(const char *contents, int from_store, - int flush); -void trusted_dirs_flush_certs_to_disk(void); -authority_cert_t *authority_cert_get_newest_by_id(const char *id_digest); -authority_cert_t *authority_cert_get_by_sk_digest(const char *sk_digest); -authority_cert_t *authority_cert_get_by_digests(const char *id_digest, - const char *sk_digest); -void authority_cert_get_all(smartlist_t *certs_out); -void authority_cert_dl_failed(const char *id_digest, int status); -void authority_certs_fetch_missing(networkstatus_t *status, time_t now); -int router_reload_router_list(void); -smartlist_t *router_get_trusted_dir_servers(void); - -/* Flags for pick_directory_server and pick_trusteddirserver. */ +/* Flags for pick_directory_server() and pick_trusteddirserver(). */ /** Flag to indicate that we should not automatically be willing to use * ourself to answer a directory request. * Passed to router_pick_directory_server (et al).*/ @@ -4361,34 +3523,13 @@ smartlist_t *router_get_trusted_dir_servers(void); */ #define PDS_NO_EXISTING_SERVERDESC_FETCH (1<<3) #define _PDS_PREFER_TUNNELED_DIR_CONNS (1<<16) -routerstatus_t *router_pick_directory_server(authority_type_t type, int flags); -trusted_dir_server_t *router_get_trusteddirserver_by_digest(const char *d); -trusted_dir_server_t *trusteddirserver_get_by_v3_auth_digest(const char *d); -routerstatus_t *router_pick_trusteddirserver(authority_type_t type, int flags); -int router_get_my_share_of_directory_requests(double *v2_share_out, - double *v3_share_out); -void router_reset_status_download_failures(void); -void routerlist_add_family(smartlist_t *sl, routerinfo_t *router); -int routers_in_same_family(routerinfo_t *r1, routerinfo_t *r2); -void add_nickname_list_to_smartlist(smartlist_t *sl, const char *list, - int must_be_running); -int router_nickname_is_in_list(routerinfo_t *router, const char *list); -routerinfo_t *routerlist_find_my_routerinfo(void); -routerinfo_t *router_find_exact_exit_enclave(const char *address, - uint16_t port); -int router_is_unreliable(routerinfo_t *router, int need_uptime, - int need_capacity, int need_guard); -uint32_t router_get_advertised_bandwidth(routerinfo_t *router); -uint32_t router_get_advertised_bandwidth_capped(routerinfo_t *router); /** Possible ways to weight routers when choosing one randomly. See * routerlist_sl_choose_by_bandwidth() for more information.*/ -typedef enum { - NO_WEIGHTING, WEIGHT_FOR_EXIT, WEIGHT_FOR_GUARD +typedef enum bandwidth_weight_rule_t { + NO_WEIGHTING, WEIGHT_FOR_EXIT, WEIGHT_FOR_MID, WEIGHT_FOR_GUARD, + WEIGHT_FOR_DIR } bandwidth_weight_rule_t; -routerinfo_t *routerlist_sl_choose_by_bandwidth(smartlist_t *sl, - bandwidth_weight_rule_t rule); -routerstatus_t *routerstatus_sl_choose_by_bandwidth(smartlist_t *sl); /** Flags to be passed to control router_choose_random_node() to indicate what * kind of nodes to pick according to what algorithm. */ @@ -4398,44 +3539,9 @@ typedef enum { CRN_NEED_GUARD = 1<<2, CRN_ALLOW_INVALID = 1<<3, /* XXXX not used, apparently. */ - CRN_STRICT_PREFERRED = 1<<4, - /* XXXX not used, apparently. */ CRN_WEIGHT_AS_EXIT = 1<<5 } router_crn_flags_t; -routerinfo_t *router_choose_random_node(const char *preferred, - smartlist_t *excludedsmartlist, - struct routerset_t *excludedset, - router_crn_flags_t flags); - -routerinfo_t *router_get_by_nickname(const char *nickname, - int warn_if_unnamed); -int router_digest_version_as_new_as(const char *digest, const char *cutoff); -int router_digest_is_trusted_dir_type(const char *digest, - authority_type_t type); -#define router_digest_is_trusted_dir(d) \ - router_digest_is_trusted_dir_type((d), NO_AUTHORITY) - -int router_addr_is_trusted_dir(uint32_t addr); -int hexdigest_to_digest(const char *hexdigest, char *digest); -routerinfo_t *router_get_by_hexdigest(const char *hexdigest); -routerinfo_t *router_get_by_digest(const char *digest); -signed_descriptor_t *router_get_by_descriptor_digest(const char *digest); -signed_descriptor_t *router_get_by_extrainfo_digest(const char *digest); -signed_descriptor_t *extrainfo_get_by_descriptor_digest(const char *digest); -const char *signed_descriptor_get_body(signed_descriptor_t *desc); -const char *signed_descriptor_get_annotations(signed_descriptor_t *desc); -routerlist_t *router_get_routerlist(void); -void routerinfo_free(routerinfo_t *router); -void extrainfo_free(extrainfo_t *extrainfo); -void routerlist_free(routerlist_t *rl); -void dump_routerlist_mem_usage(int severity); -void routerlist_remove(routerlist_t *rl, routerinfo_t *ri, int make_old, - time_t now); -void routerlist_free_all(void); -void routerlist_reset_warnings(void); -void router_set_status(const char *digest, int up); - /** Return value for router_add_to_routerlist() and dirserv_add_descriptor() */ typedef enum was_router_added_t { ROUTER_ADDED_SUCCESSFULLY = 1, @@ -4445,111 +3551,9 @@ typedef enum was_router_added_t { ROUTER_NOT_IN_CONSENSUS = -3, ROUTER_NOT_IN_CONSENSUS_OR_NETWORKSTATUS = -4, ROUTER_AUTHDIR_REJECTS = -5, + ROUTER_WAS_NOT_WANTED = -6 } was_router_added_t; -static int WRA_WAS_ADDED(was_router_added_t s); -static int WRA_WAS_OUTDATED(was_router_added_t s); -static int WRA_WAS_REJECTED(was_router_added_t s); -/** Return true iff the descriptor was added. It might still be necessary to - * check whether the descriptor generator should be notified. - */ -static INLINE int -WRA_WAS_ADDED(was_router_added_t s) { - return s == ROUTER_ADDED_SUCCESSFULLY || s == ROUTER_ADDED_NOTIFY_GENERATOR; -} -/** Return true iff the descriptor was not added because it was either: - * - not in the consensus - * - neither in the consensus nor in any networkstatus document - * - it was outdated. - */ -static INLINE int WRA_WAS_OUTDATED(was_router_added_t s) -{ - return (s == ROUTER_WAS_NOT_NEW || - s == ROUTER_NOT_IN_CONSENSUS || - s == ROUTER_NOT_IN_CONSENSUS_OR_NETWORKSTATUS); -} -/** Return true iff the descriptor rejected because it was malformed. */ -static INLINE int WRA_WAS_REJECTED(was_router_added_t s) -{ - return (s == ROUTER_AUTHDIR_REJECTS); -} -was_router_added_t router_add_to_routerlist(routerinfo_t *router, - const char **msg, - int from_cache, - int from_fetch); -was_router_added_t router_add_extrainfo_to_routerlist( - extrainfo_t *ei, const char **msg, - int from_cache, int from_fetch); -void routerlist_remove_old_routers(void); -int router_load_single_router(const char *s, uint8_t purpose, int cache, - const char **msg); -int router_load_routers_from_string(const char *s, const char *eos, - saved_location_t saved_location, - smartlist_t *requested_fingerprints, - int descriptor_digests, - const char *prepend_annotations); -void router_load_extrainfo_from_string(const char *s, const char *eos, - saved_location_t saved_location, - smartlist_t *requested_fingerprints, - int descriptor_digests); -void routerlist_retry_directory_downloads(time_t now); -int router_exit_policy_all_routers_reject(uint32_t addr, uint16_t port, - int need_uptime); -int router_exit_policy_rejects_all(routerinfo_t *router); -trusted_dir_server_t *add_trusted_dir_server(const char *nickname, - const char *address, - uint16_t dir_port, uint16_t or_port, - const char *digest, const char *v3_auth_digest, - authority_type_t type); -void authority_cert_free(authority_cert_t *cert); -void clear_trusted_dir_servers(void); -int any_trusted_dir_is_v1_authority(void); -void update_router_descriptor_downloads(time_t now); -void update_extrainfo_downloads(time_t now); -int router_have_minimum_dir_info(void); -void router_dir_info_changed(void); -const char *get_dir_info_status_string(void); -int count_loading_descriptors_progress(void); -void router_reset_descriptor_download_failures(void); -int router_differences_are_cosmetic(routerinfo_t *r1, routerinfo_t *r2); -int routerinfo_incompatible_with_extrainfo(routerinfo_t *ri, extrainfo_t *ei, - signed_descriptor_t *sd, - const char **msg); -void routerlist_assert_ok(routerlist_t *rl); -const char *esc_router_info(routerinfo_t *router); -void routers_sort_by_identity(smartlist_t *routers); - -routerset_t *routerset_new(void); -int routerset_parse(routerset_t *target, const char *s, - const char *description); -void routerset_union(routerset_t *target, const routerset_t *source); -int routerset_is_list(const routerset_t *set); -int routerset_needs_geoip(const routerset_t *set); -int routerset_contains_router(const routerset_t *set, routerinfo_t *ri); -int routerset_contains_routerstatus(const routerset_t *set, - routerstatus_t *rs); -int routerset_contains_extendinfo(const routerset_t *set, - const extend_info_t *ei); -void routerset_get_all_routers(smartlist_t *out, const routerset_t *routerset, - int running_only); -void routersets_get_disjunction(smartlist_t *target, const smartlist_t *source, - const routerset_t *include, - const routerset_t *exclude, int running_only); -void routerset_subtract_routers(smartlist_t *out, - const routerset_t *routerset); -char *routerset_to_string(const routerset_t *routerset); -void routerset_refresh_countries(routerset_t *target); -int routerset_equal(const routerset_t *old, const routerset_t *new); -void routerset_free(routerset_t *routerset); -void routerinfo_set_country(routerinfo_t *ri); -void routerlist_refresh_countries(void); -void refresh_all_country_info(void); - -int hid_serv_get_responsible_directories(smartlist_t *responsible_dirs, - const char *id); -int hid_serv_acting_as_directory(void); -int hid_serv_responsible_for_desc_id(const char *id); - /********************************* routerparse.c ************************/ #define MAX_STATUS_TAG_LEN 32 @@ -4568,71 +3572,10 @@ typedef struct tor_version_t { int patchlevel; char status_tag[MAX_STATUS_TAG_LEN]; int svn_revision; -} tor_version_t; -int router_get_router_hash(const char *s, size_t s_len, char *digest); -int router_get_dir_hash(const char *s, char *digest); -int router_get_runningrouters_hash(const char *s, char *digest); -int router_get_networkstatus_v2_hash(const char *s, char *digest); -int router_get_networkstatus_v3_hash(const char *s, char *digest); -int router_get_extrainfo_hash(const char *s, char *digest); -int router_append_dirobj_signature(char *buf, size_t buf_len, - const char *digest, - crypto_pk_env_t *private_key); -int router_parse_list_from_string(const char **s, const char *eos, - smartlist_t *dest, - saved_location_t saved_location, - int is_extrainfo, - int allow_annotations, - const char *prepend_annotations); -int router_parse_routerlist_from_directory(const char *s, - routerlist_t **dest, - crypto_pk_env_t *pkey, - int check_version, - int write_to_cache); -int router_parse_runningrouters(const char *str); -int router_parse_directory(const char *str); -routerinfo_t *router_parse_entry_from_string(const char *s, const char *end, - int cache_copy, - int allow_annotations, - const char *prepend_annotations); -extrainfo_t *extrainfo_parse_entry_from_string(const char *s, const char *end, - int cache_copy, struct digest_ri_map_t *routermap); -addr_policy_t *router_parse_addr_policy_item_from_string(const char *s, - int assume_action); -version_status_t tor_version_is_obsolete(const char *myversion, - const char *versionlist); -int tor_version_parse(const char *s, tor_version_t *out); -int tor_version_as_new_as(const char *platform, const char *cutoff); -int tor_version_compare(tor_version_t *a, tor_version_t *b); -void sort_version_list(smartlist_t *lst, int remove_duplicates); -void assert_addr_policy_ok(smartlist_t *t); -void dump_distinct_digest_count(int severity); - -networkstatus_v2_t *networkstatus_v2_parse_from_string(const char *s); -networkstatus_t *networkstatus_parse_vote_from_string(const char *s, - const char **eos_out, - networkstatus_type_t ns_type); -ns_detached_signatures_t *networkstatus_parse_detached_signatures( - const char *s, const char *eos); - -authority_cert_t *authority_cert_parse_from_string(const char *s, - const char **end_of_string); -int rend_parse_v2_service_descriptor(rend_service_descriptor_t **parsed_out, - char *desc_id_out, - char **intro_points_encrypted_out, - size_t *intro_points_encrypted_size_out, - size_t *encoded_size_out, - const char **next_out, const char *desc); -int rend_decrypt_introduction_points(char **ipos_decrypted, - size_t *ipos_decrypted_size, - const char *descriptor_cookie, - const char *ipos_encrypted, - size_t ipos_encrypted_size); -int rend_parse_introduction_points(rend_service_descriptor_t *parsed, - const char *intro_points_encoded, - size_t intro_points_encoded_size); -int rend_parse_client_keys(strmap_t *parsed_clients, const char *str); + int git_tag_len; + char git_tag[DIGEST_LEN]; +} tor_version_t; #endif diff --git a/src/or/policies.c b/src/or/policies.c index f8c36c784b..e48f42058f 100644 --- a/src/or/policies.c +++ b/src/or/policies.c @@ -9,6 +9,10 @@ **/ #include "or.h" +#include "config.h" +#include "dirserv.h" +#include "policies.h" +#include "routerparse.h" #include "ht.h" /** Policy that addresses for incoming SOCKS connections must match. */ @@ -344,7 +348,8 @@ validate_addr_policies(or_options_t *options, char **msg) *msg = NULL; if (policies_parse_exit_policy(options->ExitPolicy, &addr_policy, - options->ExitPolicyRejectPrivate, NULL)) + options->ExitPolicyRejectPrivate, NULL, + !options->BridgeRelay)) REJECT("Error in ExitPolicy entry."); /* The rest of these calls *append* to addr_policy. So don't actually @@ -375,14 +380,8 @@ validate_addr_policies(or_options_t *options, char **msg) if (parse_addr_policy(options->ReachableDirAddresses, &addr_policy, ADDR_POLICY_ACCEPT)) REJECT("Error in ReachableDirAddresses entry."); - if (parse_addr_policy(options->AuthDirReject, &addr_policy, - ADDR_POLICY_REJECT)) - REJECT("Error in AuthDirReject entry."); - if (parse_addr_policy(options->AuthDirInvalid, &addr_policy, - ADDR_POLICY_REJECT)) - REJECT("Error in AuthDirInvalid entry."); -err: + err: addr_policy_list_free(addr_policy); return *msg ? -1 : 0; #undef REJECT @@ -829,14 +828,16 @@ exit_policy_remove_redundancies(smartlist_t *dest) "reject *:6346-6429,reject *:6699,reject *:6881-6999,accept *:*" /** Parse the exit policy <b>cfg</b> into the linked list *<b>dest</b>. If - * cfg doesn't end in an absolute accept or reject, add the default exit + * cfg doesn't end in an absolute accept or reject and if + * <b>add_default_policy</b> is true, add the default exit * policy afterwards. If <b>rejectprivate</b> is true, prepend * "reject private:*" to the policy. Return -1 if we can't parse cfg, * else return 0. */ int policies_parse_exit_policy(config_line_t *cfg, smartlist_t **dest, - int rejectprivate, const char *local_address) + int rejectprivate, const char *local_address, + int add_default_policy) { if (rejectprivate) { append_exit_policy_string(dest, "reject private:*"); @@ -848,13 +849,23 @@ policies_parse_exit_policy(config_line_t *cfg, smartlist_t **dest, } if (parse_addr_policy(cfg, dest, -1)) return -1; - append_exit_policy_string(dest, DEFAULT_EXIT_POLICY); - + if (add_default_policy) + append_exit_policy_string(dest, DEFAULT_EXIT_POLICY); + else + append_exit_policy_string(dest, "reject *:*"); exit_policy_remove_redundancies(*dest); return 0; } +/** Add "reject *:*" to the end of the policy in *<b>dest</b>, allocating + * *<b>dest</b> as needed. */ +void +policies_exit_policy_append_reject_star(smartlist_t **dest) +{ + append_exit_policy_string(dest, "reject *:*"); +} + /** Replace the exit policy of <b>r</b> with reject *:*. */ void policies_set_router_exitpolicy_to_reject_all(routerinfo_t *r) @@ -866,6 +877,51 @@ policies_set_router_exitpolicy_to_reject_all(routerinfo_t *r) smartlist_add(r->exit_policy, item); } +/** Return 1 if there is at least one /8 subnet in <b>policy</b> that + * allows exiting to <b>port</b>. Otherwise, return 0. */ +static int +exit_policy_is_general_exit_helper(smartlist_t *policy, int port) +{ + uint32_t mask, ip, i; + /* Is this /8 rejected (1), or undecided (0)? */ + char subnet_status[256]; + + memset(subnet_status, 0, sizeof(subnet_status)); + SMARTLIST_FOREACH(policy, addr_policy_t *, p, { + if (tor_addr_family(&p->addr) != AF_INET) + continue; /* IPv4 only for now */ + if (p->prt_min > port || p->prt_max < port) + continue; /* Doesn't cover our port. */ + mask = 0; + tor_assert(p->maskbits <= 32); + + if (p->maskbits) + mask = UINT32_MAX<<(32-p->maskbits); + ip = tor_addr_to_ipv4h(&p->addr); + + /* Calculate the first and last subnet that this exit policy touches + * and set it as loop boundaries. */ + for (i = ((mask & ip)>>24); i <= (~((mask & ip) ^ mask)>>24); ++i) { + tor_addr_t addr; + if (subnet_status[i] != 0) + continue; /* We already reject some part of this /8 */ + tor_addr_from_ipv4h(&addr, i<<24); + if (tor_addr_is_internal(&addr, 0)) + continue; /* Local or non-routable addresses */ + if (p->policy_type == ADDR_POLICY_ACCEPT) { + if (p->maskbits > 8) + continue; /* Narrower than a /8. */ + /* We found an allowed subnet of at least size /8. Done + * for this port! */ + return 1; + } else if (p->policy_type == ADDR_POLICY_REJECT) { + subnet_status[i] = 1; + } + } + }); + return 0; +} + /** Return true iff <b>ri</b> is "useful as an exit node", meaning * it allows exit to at least one /8 address space for at least * two of ports 80, 443, and 6667. */ @@ -879,21 +935,7 @@ exit_policy_is_general_exit(smartlist_t *policy) return 0; for (i = 0; i < 3; ++i) { - SMARTLIST_FOREACH(policy, addr_policy_t *, p, { - if (tor_addr_family(&p->addr) != AF_INET) - continue; /* IPv4 only for now */ - if (p->prt_min > ports[i] || p->prt_max < ports[i]) - continue; /* Doesn't cover our port. */ - if (p->maskbits > 8) - continue; /* Narrower than a /8. */ - if (tor_addr_is_loopback(&p->addr)) - continue; /* 127.x or ::1. */ - /* We have a match that is at least a /8. */ - if (p->policy_type == ADDR_POLICY_ACCEPT) { - ++n_allowed; - break; /* stop considering this port */ - } - }); + n_allowed += exit_policy_is_general_exit_helper(policy, ports[i]); } return n_allowed >= 2; } @@ -1240,7 +1282,7 @@ policy_summarize(smartlist_t *policy) result = tor_malloc(final_size); tor_snprintf(result, final_size, "%s %s", prefix, shorter_str); -cleanup: + cleanup: /* cleanup */ SMARTLIST_FOREACH(summary, policy_summary_item_t *, s, tor_free(s)); smartlist_free(summary); @@ -1260,9 +1302,11 @@ cleanup: * about "exit-policy/..." */ int getinfo_helper_policies(control_connection_t *conn, - const char *question, char **answer) + const char *question, char **answer, + const char **errmsg) { (void) conn; + (void) errmsg; if (!strcmp(question, "exit-policy/default")) { *answer = tor_strdup(DEFAULT_EXIT_POLICY); } @@ -1273,7 +1317,8 @@ getinfo_helper_policies(control_connection_t *conn, void addr_policy_list_free(smartlist_t *lst) { - if (!lst) return; + if (!lst) + return; SMARTLIST_FOREACH(lst, addr_policy_t *, policy, addr_policy_free(policy)); smartlist_free(lst); } @@ -1282,19 +1327,20 @@ addr_policy_list_free(smartlist_t *lst) void addr_policy_free(addr_policy_t *p) { - if (p) { - if (--p->refcnt <= 0) { - if (p->is_canonical) { - policy_map_ent_t search, *found; - search.policy = p; - found = HT_REMOVE(policy_map, &policy_root, &search); - if (found) { - tor_assert(p == found->policy); - tor_free(found); - } + if (!p) + return; + + if (--p->refcnt <= 0) { + if (p->is_canonical) { + policy_map_ent_t search, *found; + search.policy = p; + found = HT_REMOVE(policy_map, &policy_root, &search); + if (found) { + tor_assert(p == found->policy); + tor_free(found); } - tor_free(p); } + tor_free(p); } } diff --git a/src/or/policies.h b/src/or/policies.h new file mode 100644 index 0000000000..b2947c67e7 --- /dev/null +++ b/src/or/policies.h @@ -0,0 +1,61 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file policies.h + * \brief Header file for policies.c. + **/ + +#ifndef _TOR_POLICIES_H +#define _TOR_POLICIES_H + +/* (length of "accept 255.255.255.255/255.255.255.255:65535-65535\n" plus a + * NUL.) + */ +#define POLICY_BUF_LEN 52 + +int firewall_is_fascist_or(void); +int fascist_firewall_allows_address_or(const tor_addr_t *addr, uint16_t port); +int fascist_firewall_allows_or(routerinfo_t *ri); +int fascist_firewall_allows_address_dir(const tor_addr_t *addr, uint16_t port); +int dir_policy_permits_address(const tor_addr_t *addr); +int socks_policy_permits_address(const tor_addr_t *addr); +int authdir_policy_permits_address(uint32_t addr, uint16_t port); +int authdir_policy_valid_address(uint32_t addr, uint16_t port); +int authdir_policy_baddir_address(uint32_t addr, uint16_t port); +int authdir_policy_badexit_address(uint32_t addr, uint16_t port); + +int validate_addr_policies(or_options_t *options, char **msg); +void policy_expand_private(smartlist_t **policy); +int policies_parse_from_options(or_options_t *options); + +addr_policy_t *addr_policy_get_canonical_entry(addr_policy_t *ent); +int cmp_addr_policies(smartlist_t *a, smartlist_t *b); +addr_policy_result_t compare_tor_addr_to_addr_policy(const tor_addr_t *addr, + uint16_t port, const smartlist_t *policy); +addr_policy_result_t compare_addr_to_addr_policy(uint32_t addr, + uint16_t port, const smartlist_t *policy); +int policies_parse_exit_policy(config_line_t *cfg, smartlist_t **dest, + int rejectprivate, const char *local_address, + int add_default_policy); +void policies_exit_policy_append_reject_star(smartlist_t **dest); +void policies_set_router_exitpolicy_to_reject_all(routerinfo_t *exitrouter); +int exit_policy_is_general_exit(smartlist_t *policy); +int policy_is_reject_star(const smartlist_t *policy); +int getinfo_helper_policies(control_connection_t *conn, + const char *question, char **answer, + const char **errmsg); +int policy_write_item(char *buf, size_t buflen, addr_policy_t *item, + int format_for_desc); + +void addr_policy_list_free(smartlist_t *p); +void addr_policy_free(addr_policy_t *p); +void policies_free_all(void); + +char *policy_summarize(smartlist_t *policy); + +#endif + diff --git a/src/or/reasons.c b/src/or/reasons.c index 38eb4078ca..319e6c055a 100644 --- a/src/or/reasons.c +++ b/src/or/reasons.c @@ -9,6 +9,8 @@ **/ #include "or.h" +#include "config.h" +#include "reasons.h" /***************************** Edge (stream) reasons **********************/ @@ -38,6 +40,8 @@ stream_end_reason_to_control_string(int reason) case END_STREAM_REASON_NET_UNREACHABLE: return "NET_UNREACHABLE"; case END_STREAM_REASON_SOCKSPROTOCOL: return "SOCKS_PROTOCOL"; + case END_STREAM_REASON_PRIVATE_ADDR: return "PRIVATE_ADDR"; + default: return NULL; } } @@ -123,6 +127,9 @@ stream_end_reason_to_socks5_response(int reason) return SOCKS5_NET_UNREACHABLE; case END_STREAM_REASON_SOCKSPROTOCOL: return SOCKS5_GENERAL_ERROR; + case END_STREAM_REASON_PRIVATE_ADDR: + return SOCKS5_GENERAL_ERROR; + default: log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, "Reason for ending (%d) not recognized; " @@ -167,13 +174,7 @@ errno_to_stream_end_reason(int e) S_CASE(ENETUNREACH): return END_STREAM_REASON_INTERNAL; S_CASE(EHOSTUNREACH): - /* XXXX022 - * The correct behavior is END_STREAM_REASON_NOROUTE, but older - * clients don't recognize it. So we're going to continue sending - * "MISC" until 0.2.1.27 or later is "well established". - */ - /* return END_STREAM_REASON_NOROUTE; */ - return END_STREAM_REASON_MISC; + return END_STREAM_REASON_NOROUTE; S_CASE(ECONNREFUSED): return END_STREAM_REASON_CONNECTREFUSED; S_CASE(ECONNRESET): @@ -332,9 +333,78 @@ circuit_end_reason_to_control_string(int reason) return "NOPATH"; case END_CIRC_REASON_NOSUCHSERVICE: return "NOSUCHSERVICE"; + case END_CIRC_REASON_MEASUREMENT_EXPIRED: + return "MEASUREMENT_EXPIRED"; default: log_warn(LD_BUG, "Unrecognized reason code %d", (int)reason); return NULL; } } +/** Return a string corresponding to a SOCKS4 reponse code. */ +const char * +socks4_response_code_to_string(uint8_t code) +{ + switch (code) { + case 0x5a: + return "connection accepted"; + case 0x5b: + return "server rejected connection"; + case 0x5c: + return "server cannot connect to identd on this client"; + case 0x5d: + return "user id does not match identd"; + default: + return "invalid SOCKS 4 response code"; + } +} + +/** Return a string corresponding to a SOCKS5 reponse code. */ +const char * +socks5_response_code_to_string(uint8_t code) +{ + switch (code) { + case 0x00: + return "connection accepted"; + case 0x01: + return "general SOCKS server failure"; + case 0x02: + return "connection not allowed by ruleset"; + case 0x03: + return "Network unreachable"; + case 0x04: + return "Host unreachable"; + case 0x05: + return "Connection refused"; + case 0x06: + return "TTL expired"; + case 0x07: + return "Command not supported"; + case 0x08: + return "Address type not supported"; + default: + return "unknown reason"; + } +} + +/** Return a string corresponding to a bandwidht_weight_rule_t */ +const char * +bandwidth_weight_rule_to_string(bandwidth_weight_rule_t rule) +{ + switch (rule) + { + case NO_WEIGHTING: + return "no weighting"; + case WEIGHT_FOR_EXIT: + return "weight as exit"; + case WEIGHT_FOR_MID: + return "weight as middle node"; + case WEIGHT_FOR_GUARD: + return "weight as guard"; + case WEIGHT_FOR_DIR: + return "weight as directory"; + default: + return "unknown rule"; + } +} + diff --git a/src/or/reasons.h b/src/or/reasons.h new file mode 100644 index 0000000000..01f9717948 --- /dev/null +++ b/src/or/reasons.h @@ -0,0 +1,31 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file reasons.h + * \brief Header file for reasons.c. + **/ + +#ifndef _TOR_REASONS_H +#define _TOR_REASONS_H + +const char *stream_end_reason_to_control_string(int reason); +const char *stream_end_reason_to_string(int reason); +socks5_reply_status_t stream_end_reason_to_socks5_response(int reason); +uint8_t errno_to_stream_end_reason(int e); + +const char *orconn_end_reason_to_control_string(int r); +int tls_error_to_orconn_end_reason(int e); +int errno_to_orconn_end_reason(int e); + +const char *circuit_end_reason_to_control_string(int reason); +const char *socks4_response_code_to_string(uint8_t code); +const char *socks5_response_code_to_string(uint8_t code); + +const char *bandwidth_weight_rule_to_string(enum bandwidth_weight_rule_t rule); + +#endif + diff --git a/src/or/relay.c b/src/or/relay.c index b40d7ad244..807cb3d435 100644 --- a/src/or/relay.c +++ b/src/or/relay.c @@ -10,8 +10,26 @@ * receiving from circuits, plus queuing on circuits. **/ +#include <math.h> #include "or.h" +#include "buffers.h" +#include "circuitbuild.h" +#include "circuitlist.h" +#include "config.h" +#include "connection.h" +#include "connection_edge.h" +#include "connection_or.h" +#include "control.h" +#include "geoip.h" +#include "main.h" #include "mempool.h" +#include "networkstatus.h" +#include "policies.h" +#include "reasons.h" +#include "relay.h" +#include "rendcommon.h" +#include "routerlist.h" +#include "routerparse.h" static int relay_crypt(circuit_t *circ, cell_t *cell, cell_direction_t cell_direction, @@ -20,20 +38,46 @@ static edge_connection_t *relay_lookup_conn(circuit_t *circ, cell_t *cell, cell_direction_t cell_direction, crypt_path_t *layer_hint); -static int -connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, - edge_connection_t *conn, - crypt_path_t *layer_hint); -static void -circuit_consider_sending_sendme(circuit_t *circ, crypt_path_t *layer_hint); +static int connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, + edge_connection_t *conn, + crypt_path_t *layer_hint); +static void circuit_consider_sending_sendme(circuit_t *circ, + crypt_path_t *layer_hint); +static void circuit_resume_edge_reading(circuit_t *circ, + crypt_path_t *layer_hint); +static int circuit_resume_edge_reading_helper(edge_connection_t *conn, + circuit_t *circ, + crypt_path_t *layer_hint); +static int circuit_consider_stop_edge_reading(circuit_t *circ, + crypt_path_t *layer_hint); +static int circuit_queue_streams_are_blocked(circuit_t *circ); + +/* XXXX023 move this all to compat_libevent */ +/** Cache the current hi-res time; the cache gets reset when libevent + * calls us. */ +static struct timeval cached_time_hires = {0, 0}; + +/** Stop reading on edge connections when we have this many cells + * waiting on the appropriate queue. */ +#define CELL_QUEUE_HIGHWATER_SIZE 256 +/** Start reading from edge connections again when we get down to this many + * cells. */ +#define CELL_QUEUE_LOWWATER_SIZE 64 + static void -circuit_resume_edge_reading(circuit_t *circ, crypt_path_t *layer_hint); -static int -circuit_resume_edge_reading_helper(edge_connection_t *conn, - circuit_t *circ, - crypt_path_t *layer_hint); -static int -circuit_consider_stop_edge_reading(circuit_t *circ, crypt_path_t *layer_hint); +tor_gettimeofday_cached(struct timeval *tv) +{ + if (cached_time_hires.tv_sec == 0) { + tor_gettimeofday(&cached_time_hires); + } + *tv = cached_time_hires; +} + +void +tor_gettimeofday_cache_clear(void) +{ + cached_time_hires.tv_sec = 0; +} /** Stats: how many relay cells have originated at this hop, or have * been relayed onward (not recognized at this hop)? @@ -230,7 +274,7 @@ circuit_receive_relay_cell(cell_t *cell, circuit_t *circ, * we might kill the circ before we relay * the cells. */ - append_cell_to_circuit_queue(circ, or_conn, cell, cell_direction); + append_cell_to_circuit_queue(circ, or_conn, cell, cell_direction, 0); return 0; } @@ -327,7 +371,7 @@ relay_crypt(circuit_t *circ, cell_t *cell, cell_direction_t cell_direction, static int circuit_package_relay_cell(cell_t *cell, circuit_t *circ, cell_direction_t cell_direction, - crypt_path_t *layer_hint) + crypt_path_t *layer_hint, streamid_t on_stream) { or_connection_t *conn; /* where to send the cell */ @@ -371,7 +415,7 @@ circuit_package_relay_cell(cell_t *cell, circuit_t *circ, } ++stats_n_relay_cells_relayed; - append_cell_to_circuit_queue(circ, conn, cell, cell_direction); + append_cell_to_circuit_queue(circ, conn, cell, cell_direction, on_stream); return 0; } @@ -496,7 +540,7 @@ relay_command_to_string(uint8_t command) * return 0. */ int -relay_send_command_from_edge(uint16_t stream_id, circuit_t *circ, +relay_send_command_from_edge(streamid_t stream_id, circuit_t *circ, uint8_t relay_command, const char *payload, size_t payload_len, crypt_path_t *cpath_layer) { @@ -531,6 +575,12 @@ relay_send_command_from_edge(uint16_t stream_id, circuit_t *circ, log_debug(LD_OR,"delivering %d cell %s.", relay_command, cell_direction == CELL_DIRECTION_OUT ? "forward" : "backward"); + /* If we are sending an END cell and this circuit is used for a tunneled + * directory request, advance its state. */ + if (relay_command == RELAY_COMMAND_END && circ->dirreq_id) + geoip_change_dirreq_state(circ->dirreq_id, DIRREQ_TUNNELED, + DIRREQ_END_CELL_SENT); + if (cell_direction == CELL_DIRECTION_OUT && circ->n_conn) { /* if we're using relaybandwidthrate, this conn wants priority */ circ->n_conn->client_used = approx_time(); @@ -540,17 +590,11 @@ relay_send_command_from_edge(uint16_t stream_id, circuit_t *circ, origin_circuit_t *origin_circ = TO_ORIGIN_CIRCUIT(circ); if (origin_circ->remaining_relay_early_cells > 0 && (relay_command == RELAY_COMMAND_EXTEND || - (cpath_layer != origin_circ->cpath && - !CIRCUIT_PURPOSE_IS_ESTABLISHED_REND(circ->purpose)))) { - /* If we've got any relay_early cells left, and we're sending - * an extend cell or (we're not talking to the first hop and we're - * not talking to a rendezvous circuit), use one of them. - * Don't worry about the conn protocol version: + cpath_layer != origin_circ->cpath)) { + /* If we've got any relay_early cells left and (we're sending + * an extend cell or we're not talking to the first hop), use + * one of them. Don't worry about the conn protocol version: * append_cell_to_circuit_queue will fix it up. */ - /* XXX For now, clients don't use RELAY_EARLY cells when sending - * relay cells on rendezvous circuits. See bug 1038. Eventually, - * we can take this behavior away in favor of having clients avoid - * rendezvous points running 0.2.1.3-alpha through 0.2.1.18. -RD */ cell.command = CELL_RELAY_EARLY; --origin_circ->remaining_relay_early_cells; log_debug(LD_OR, "Sending a RELAY_EARLY cell; %d remaining.", @@ -578,8 +622,8 @@ relay_send_command_from_edge(uint16_t stream_id, circuit_t *circ, } } - if (circuit_package_relay_cell(&cell, circ, cell_direction, cpath_layer) - < 0) { + if (circuit_package_relay_cell(&cell, circ, cell_direction, cpath_layer, + stream_id) < 0) { log_warn(LD_BUG,"circuit_package_relay_cell failed. Closing."); circuit_mark_for_close(circ, END_CIRC_REASON_INTERNAL); return -1; @@ -747,6 +791,8 @@ connection_ap_process_end_not_open( < MAX_RESOLVE_FAILURES) { /* We haven't retried too many times; reattach the connection. */ circuit_log_path(LOG_INFO,LD_APP,circ); + /* Mark this circuit "unusable for new streams". */ + /* XXXX023 this is a kludgy way to do this. */ tor_assert(circ->_base.timestamp_dirty); circ->_base.timestamp_dirty -= get_options()->MaxCircuitDirtiness; @@ -901,7 +947,7 @@ connection_edge_process_relay_cell_not_open( } /* handle anything that might have queued */ - if (connection_edge_package_raw_inbuf(conn, 1) < 0) { + if (connection_edge_package_raw_inbuf(conn, 1, NULL) < 0) { /* (We already sent an end cell if possible) */ connection_mark_for_close(TO_CONN(conn)); return 0; @@ -999,7 +1045,8 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, relay_header_unpack(&rh, cell->payload); // log_fn(LOG_DEBUG,"command %d stream %d", rh.command, rh.stream_id); num_seen++; - log_debug(domain, "Now seen %d relay cells here.", num_seen); + log_debug(domain, "Now seen %d relay cells here (command %d, stream %d).", + num_seen, rh.command, rh.stream_id); if (rh.length > RELAY_PAYLOAD_SIZE) { log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, @@ -1038,6 +1085,16 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, "Begin cell for known stream. Dropping."); return 0; } + if (rh.command == RELAY_COMMAND_BEGIN_DIR) { + /* Assign this circuit and its app-ward OR connection a unique ID, + * so that we can measure download times. The local edge and dir + * connection will be assigned the same ID when they are created + * and linked. */ + static uint64_t next_id = 0; + circ->dirreq_id = ++next_id; + TO_CONN(TO_OR_CIRCUIT(circ)->p_conn)->dirreq_id = circ->dirreq_id; + } + return connection_exit_begin_conn(cell, circ); case RELAY_COMMAND_DATA: ++stats_n_data_cells_received; @@ -1131,6 +1188,7 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, } if (circ->n_conn) { uint8_t trunc_reason = *(uint8_t*)(cell->payload + RELAY_HEADER_SIZE); + circuit_clear_cell_queue(circ, circ->n_conn); connection_or_send_destroy(circ->n_circ_id, circ->n_conn, trunc_reason); circuit_set_n_circid_orconn(circ, 0, NULL); @@ -1179,9 +1237,13 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, conn->package_window += STREAMWINDOW_INCREMENT; log_debug(domain,"stream-level sendme, packagewindow now %d.", conn->package_window); + if (circuit_queue_streams_are_blocked(circ)) { + /* Still waiting for queue to flush; don't touch conn */ + return 0; + } connection_start_reading(TO_CONN(conn)); /* handle whatever might still be on the inbuf */ - if (connection_edge_package_raw_inbuf(conn, 1) < 0) { + if (connection_edge_package_raw_inbuf(conn, 1, NULL) < 0) { /* (We already sent an end cell if possible) */ connection_mark_for_close(TO_CONN(conn)); return 0; @@ -1247,15 +1309,19 @@ uint64_t stats_n_data_cells_received = 0; * ever received were completely full of data. */ uint64_t stats_n_data_bytes_received = 0; -/** While conn->inbuf has an entire relay payload of bytes on it, - * and the appropriate package windows aren't empty, grab a cell - * and send it down the circuit. +/** If <b>conn</b> has an entire relay payload of bytes on its inbuf (or + * <b>package_partial</b> is true), and the appropriate package windows aren't + * empty, grab a cell and send it down the circuit. + * + * If *<b>max_cells</b> is given, package no more than max_cells. Decrement + * *<b>max_cells</b> by the number of cells packaged. * * Return -1 (and send a RELAY_COMMAND_END cell if necessary) if conn should * be marked for close, else return 0. */ int -connection_edge_package_raw_inbuf(edge_connection_t *conn, int package_partial) +connection_edge_package_raw_inbuf(edge_connection_t *conn, int package_partial, + int *max_cells) { size_t amount_to_process, length; char payload[CELL_PAYLOAD_SIZE]; @@ -1271,7 +1337,10 @@ connection_edge_package_raw_inbuf(edge_connection_t *conn, int package_partial) return 0; } -repeat_connection_edge_package_raw_inbuf: + if (max_cells && *max_cells <= 0) + return 0; + + repeat_connection_edge_package_raw_inbuf: circ = circuit_get_by_edge_conn(conn); if (!circ) { @@ -1332,12 +1401,19 @@ repeat_connection_edge_package_raw_inbuf: } log_debug(domain,"conn->package_window is now %d",conn->package_window); + if (max_cells) { + *max_cells -= 1; + if (*max_cells <= 0) + return 0; + } + /* handle more if there's more, or return 0 if there isn't */ goto repeat_connection_edge_package_raw_inbuf; } -/** Called when we've just received a relay data cell, or when - * we've just finished flushing all bytes to stream <b>conn</b>. +/** Called when we've just received a relay data cell, when + * we've just finished flushing all bytes to stream <b>conn</b>, + * or when we've flushed *some* bytes to the stream <b>conn</b>. * * If conn->outbuf is not too full, and our deliver window is * low, send back a suitable number of stream-level sendme cells. @@ -1379,7 +1455,10 @@ connection_edge_consider_sending_sendme(edge_connection_t *conn) static void circuit_resume_edge_reading(circuit_t *circ, crypt_path_t *layer_hint) { - + if (circuit_queue_streams_are_blocked(circ)) { + log_debug(layer_hint?LD_APP:LD_EXIT,"Too big queue, no resuming"); + return; + } log_debug(layer_hint?LD_APP:LD_EXIT,"resuming"); if (CIRCUIT_IS_ORIGIN(circ)) @@ -1395,31 +1474,136 @@ circuit_resume_edge_reading(circuit_t *circ, crypt_path_t *layer_hint) * of a linked list of edge streams that should each be considered. */ static int -circuit_resume_edge_reading_helper(edge_connection_t *conn, +circuit_resume_edge_reading_helper(edge_connection_t *first_conn, circuit_t *circ, crypt_path_t *layer_hint) { - for ( ; conn; conn=conn->next_stream) { - if (conn->_base.marked_for_close) + edge_connection_t *conn; + int n_packaging_streams, n_streams_left; + int packaged_this_round; + int cells_on_queue; + int cells_per_conn; + edge_connection_t *chosen_stream = NULL; + + /* How many cells do we have space for? It will be the minimum of + * the number needed to exhaust the package window, and the minimum + * needed to fill the cell queue. */ + int max_to_package = circ->package_window; + if (CIRCUIT_IS_ORIGIN(circ)) { + cells_on_queue = circ->n_conn_cells.n; + } else { + or_circuit_t *or_circ = TO_OR_CIRCUIT(circ); + cells_on_queue = or_circ->p_conn_cells.n; + } + if (CELL_QUEUE_HIGHWATER_SIZE - cells_on_queue < max_to_package) + max_to_package = CELL_QUEUE_HIGHWATER_SIZE - cells_on_queue; + + /* Once we used to start listening on the streams in the order they + * appeared in the linked list. That leads to starvation on the + * streams that appeared later on the list, since the first streams + * would always get to read first. Instead, we just pick a random + * stream on the list, and enable reading for streams starting at that + * point (and wrapping around as if the list were circular). It would + * probably be better to actually remember which streams we've + * serviced in the past, but this is simple and effective. */ + + /* Select a stream uniformly at random from the linked list. We + * don't need cryptographic randomness here. */ + { + int num_streams = 0; + for (conn = first_conn; conn; conn = conn->next_stream) { + num_streams++; + if ((tor_weak_random() % num_streams)==0) + chosen_stream = conn; + /* Invariant: chosen_stream has been chosen uniformly at random from + * among the first num_streams streams on first_conn. */ + } + } + + /* Count how many non-marked streams there are that have anything on + * their inbuf, and enable reading on all of the connections. */ + n_packaging_streams = 0; + /* Activate reading starting from the chosen stream */ + for (conn=chosen_stream; conn; conn = conn->next_stream) { + /* Start reading for the streams starting from here */ + if (conn->_base.marked_for_close || conn->package_window <= 0) continue; - if ((!layer_hint && conn->package_window > 0) || - (layer_hint && conn->package_window > 0 && - conn->cpath_layer == layer_hint)) { + if (!layer_hint || conn->cpath_layer == layer_hint) { connection_start_reading(TO_CONN(conn)); + + if (buf_datalen(conn->_base.inbuf) > 0) + ++n_packaging_streams; + } + } + /* Go back and do the ones we skipped, circular-style */ + for (conn = first_conn; conn != chosen_stream; conn = conn->next_stream) { + if (conn->_base.marked_for_close || conn->package_window <= 0) + continue; + if (!layer_hint || conn->cpath_layer == layer_hint) { + connection_start_reading(TO_CONN(conn)); + + if (buf_datalen(conn->_base.inbuf) > 0) + ++n_packaging_streams; + } + } + + if (n_packaging_streams == 0) /* avoid divide-by-zero */ + return 0; + + again: + + cells_per_conn = CEIL_DIV(max_to_package, n_packaging_streams); + + packaged_this_round = 0; + n_streams_left = 0; + + /* Iterate over all connections. Package up to cells_per_conn cells on + * each. Update packaged_this_round with the total number of cells + * packaged, and n_streams_left with the number that still have data to + * package. + */ + for (conn=first_conn; conn; conn=conn->next_stream) { + if (conn->_base.marked_for_close || conn->package_window <= 0) + continue; + if (!layer_hint || conn->cpath_layer == layer_hint) { + int n = cells_per_conn, r; /* handle whatever might still be on the inbuf */ - if (connection_edge_package_raw_inbuf(conn, 1)<0) { - /* (We already sent an end cell if possible) */ + r = connection_edge_package_raw_inbuf(conn, 1, &n); + + /* Note how many we packaged */ + packaged_this_round += (cells_per_conn-n); + + if (r<0) { + /* Problem while packaging. (We already sent an end cell if + * possible) */ connection_mark_for_close(TO_CONN(conn)); continue; } + /* If there's still data to read, we'll be coming back to this stream. */ + if (buf_datalen(conn->_base.inbuf)) + ++n_streams_left; + /* If the circuit won't accept any more data, return without looking * at any more of the streams. Any connections that should be stopped * have already been stopped by connection_edge_package_raw_inbuf. */ if (circuit_consider_stop_edge_reading(circ, layer_hint)) return -1; + /* XXXX should we also stop immediately if we fill up the cell queue? + * Probably. */ } } + + /* If we made progress, and we are willing to package more, and there are + * any streams left that want to package stuff... try again! + */ + if (packaged_this_round && packaged_this_round < max_to_package && + n_streams_left) { + max_to_package -= packaged_this_round; + n_packaging_streams = n_streams_left; + goto again; + } + return 0; } @@ -1488,13 +1672,6 @@ circuit_consider_sending_sendme(circuit_t *circ, crypt_path_t *layer_hint) } } -/** Stop reading on edge connections when we have this many cells - * waiting on the appropriate queue. */ -#define CELL_QUEUE_HIGHWATER_SIZE 256 -/** Start reading from edge connections again when we get down to this many - * cells. */ -#define CELL_QUEUE_LOWWATER_SIZE 64 - #ifdef ACTIVE_CIRCUITS_PARANOIA #define assert_active_circuits_ok_paranoid(conn) \ assert_active_circuits_ok(conn) @@ -1508,6 +1685,10 @@ static int total_cells_allocated = 0; /** A memory pool to allocate packed_cell_t objects. */ static mp_pool_t *cell_pool = NULL; +/** Memory pool to allocate insertion_time_elem_t objects used for cell + * statistics. */ +static mp_pool_t *it_pool = NULL; + /** Allocate structures to hold cells. */ void init_cell_pool(void) @@ -1516,7 +1697,8 @@ init_cell_pool(void) cell_pool = mp_pool_new(sizeof(packed_cell_t), 128*1024); } -/** Free all storage used to hold cells. */ +/** Free all storage used to hold cells (and insertion times if we measure + * cell statistics). */ void free_cell_pool(void) { @@ -1525,6 +1707,10 @@ free_cell_pool(void) mp_pool_destroy(cell_pool); cell_pool = NULL; } + if (it_pool) { + mp_pool_destroy(it_pool); + it_pool = NULL; + } } /** Free excess storage in cell pool. */ @@ -1537,7 +1723,7 @@ clean_cell_pool(void) /** Release storage held by <b>cell</b>. */ static INLINE void -packed_cell_free(packed_cell_t *cell) +packed_cell_free_unchecked(packed_cell_t *cell) { --total_cells_allocated; mp_pool_release(cell); @@ -1599,7 +1785,38 @@ cell_queue_append(cell_queue_t *queue, packed_cell_t *cell) void cell_queue_append_packed_copy(cell_queue_t *queue, const cell_t *cell) { - cell_queue_append(queue, packed_cell_copy(cell)); + packed_cell_t *copy = packed_cell_copy(cell); + /* Remember the time when this cell was put in the queue. */ + if (get_options()->CellStatistics) { + struct timeval now; + uint32_t added; + insertion_time_queue_t *it_queue = queue->insertion_times; + if (!it_pool) + it_pool = mp_pool_new(sizeof(insertion_time_elem_t), 1024); + tor_gettimeofday_cached(&now); +#define SECONDS_IN_A_DAY 86400L + added = (uint32_t)(((now.tv_sec % SECONDS_IN_A_DAY) * 100L) + + ((uint32_t)now.tv_usec / (uint32_t)10000L)); + if (!it_queue) { + it_queue = tor_malloc_zero(sizeof(insertion_time_queue_t)); + queue->insertion_times = it_queue; + } + if (it_queue->last && it_queue->last->insertion_time == added) { + it_queue->last->counter++; + } else { + insertion_time_elem_t *elem = mp_pool_get(it_pool); + elem->next = NULL; + elem->insertion_time = added; + elem->counter = 1; + if (it_queue->last) { + it_queue->last->next = elem; + it_queue->last = elem; + } else { + it_queue->first = it_queue->last = elem; + } + } + } + cell_queue_append(queue, copy); } /** Remove and free every cell in <b>queue</b>. */ @@ -1610,11 +1827,19 @@ cell_queue_clear(cell_queue_t *queue) cell = queue->head; while (cell) { next = cell->next; - packed_cell_free(cell); + packed_cell_free_unchecked(cell); cell = next; } queue->head = queue->tail = NULL; queue->n = 0; + if (queue->insertion_times) { + while (queue->insertion_times->first) { + insertion_time_elem_t *elem = queue->insertion_times->first; + queue->insertion_times->first = elem->next; + mp_pool_release(elem); + } + tor_free(queue->insertion_times); + } } /** Extract and return the cell at the head of <b>queue</b>; return NULL if @@ -1666,8 +1891,226 @@ prev_circ_on_conn_p(circuit_t *circ, or_connection_t *conn) } } +/** Helper for sorting cell_ewma_t values in their priority queue. */ +static int +compare_cell_ewma_counts(const void *p1, const void *p2) +{ + const cell_ewma_t *e1=p1, *e2=p2; + if (e1->cell_count < e2->cell_count) + return -1; + else if (e1->cell_count > e2->cell_count) + return 1; + else + return 0; +} + +/** Given a cell_ewma_t, return a pointer to the circuit containing it. */ +static circuit_t * +cell_ewma_to_circuit(cell_ewma_t *ewma) +{ + if (ewma->is_for_p_conn) { + /* This is an or_circuit_t's p_cell_ewma. */ + or_circuit_t *orcirc = SUBTYPE_P(ewma, or_circuit_t, p_cell_ewma); + return TO_CIRCUIT(orcirc); + } else { + /* This is some circuit's n_cell_ewma. */ + return SUBTYPE_P(ewma, circuit_t, n_cell_ewma); + } +} + +/* ==== Functions for scaling cell_ewma_t ==== + + When choosing which cells to relay first, we favor circuits that have been + quiet recently. This gives better latency on connections that aren't + pushing lots of data, and makes the network feel more interactive. + + Conceptually, we take an exponentially weighted mean average of the number + of cells a circuit has sent, and allow active circuits (those with cells to + relay) to send cells in reverse order of their exponentially-weighted mean + average (EWMA) cell count. [That is, a cell sent N seconds ago 'counts' + F^N times as much as a cell sent now, for 0<F<1.0, and we favor the + circuit that has sent the fewest cells] + + If 'double' had infinite precision, we could do this simply by counting a + cell sent at startup as having weight 1.0, and a cell sent N seconds later + as having weight F^-N. This way, we would never need to re-scale + any already-sent cells. + + To prevent double from overflowing, we could count a cell sent now as + having weight 1.0 and a cell sent N seconds ago as having weight F^N. + This, however, would mean we'd need to re-scale *ALL* old circuits every + time we wanted to send a cell. + + So as a compromise, we divide time into 'ticks' (currently, 10-second + increments) and say that a cell sent at the start of a current tick is + worth 1.0, a cell sent N seconds before the start of the current tick is + worth F^N, and a cell sent N seconds after the start of the current tick is + worth F^-N. This way we don't overflow, and we don't need to constantly + rescale. + */ + +/** How long does a tick last (seconds)? */ +#define EWMA_TICK_LEN 10 + +/** The default per-tick scale factor, if it hasn't been overridden by a + * consensus or a configuration setting. zero means "disabled". */ +#define EWMA_DEFAULT_HALFLIFE 0.0 + +/** Given a timeval <b>now</b>, compute the cell_ewma tick in which it occurs + * and the fraction of the tick that has elapsed between the start of the tick + * and <b>now</b>. Return the former and store the latter in + * *<b>remainder_out</b>. + * + * These tick values are not meant to be shared between Tor instances, or used + * for other purposes. */ +static unsigned +cell_ewma_tick_from_timeval(const struct timeval *now, + double *remainder_out) +{ + unsigned res = (unsigned) (now->tv_sec / EWMA_TICK_LEN); + /* rem */ + double rem = (now->tv_sec % EWMA_TICK_LEN) + + ((double)(now->tv_usec)) / 1.0e6; + *remainder_out = rem / EWMA_TICK_LEN; + return res; +} + +/** Compute and return the current cell_ewma tick. */ +unsigned +cell_ewma_get_tick(void) +{ + return ((unsigned)approx_time() / EWMA_TICK_LEN); +} + +/** The per-tick scale factor to be used when computing cell-count EWMA + * values. (A cell sent N ticks before the start of the current tick + * has value ewma_scale_factor ** N.) + */ +static double ewma_scale_factor = 0.1; +static int ewma_enabled = 0; + +#define EPSILON 0.00001 +#define LOG_ONEHALF -0.69314718055994529 + +/** Adjust the global cell scale factor based on <b>options</b> */ +void +cell_ewma_set_scale_factor(or_options_t *options, networkstatus_t *consensus) +{ + int32_t halflife_ms; + double halflife; + const char *source; + if (options && options->CircuitPriorityHalflife >= -EPSILON) { + halflife = options->CircuitPriorityHalflife; + source = "CircuitPriorityHalflife in configuration"; + } else if (consensus && (halflife_ms = networkstatus_get_param( + consensus, "CircuitPriorityHalflifeMsec", + -1, -1, INT32_MAX)) >= 0) { + halflife = ((double)halflife_ms)/1000.0; + source = "CircuitPriorityHalflifeMsec in consensus"; + } else { + halflife = EWMA_DEFAULT_HALFLIFE; + source = "Default value"; + } + + if (halflife <= EPSILON) { + /* The cell EWMA algorithm is disabled. */ + ewma_scale_factor = 0.1; + ewma_enabled = 0; + log_info(LD_OR, + "Disabled cell_ewma algorithm because of value in %s", + source); + } else { + /* convert halflife into halflife-per-tick. */ + halflife /= EWMA_TICK_LEN; + /* compute per-tick scale factor. */ + ewma_scale_factor = exp( LOG_ONEHALF / halflife ); + ewma_enabled = 1; + log_info(LD_OR, + "Enabled cell_ewma algorithm because of value in %s; " + "scale factor is %lf per %d seconds", + source, ewma_scale_factor, EWMA_TICK_LEN); + } +} + +/** Return the multiplier necessary to convert the value of a cell sent in + * 'from_tick' to one sent in 'to_tick'. */ +static INLINE double +get_scale_factor(unsigned from_tick, unsigned to_tick) +{ + /* This math can wrap around, but that's okay: unsigned overflow is + well-defined */ + int diff = (int)(to_tick - from_tick); + return pow(ewma_scale_factor, diff); +} + +/** Adjust the cell count of <b>ewma</b> so that it is scaled with respect to + * <b>cur_tick</b> */ +static void +scale_single_cell_ewma(cell_ewma_t *ewma, unsigned cur_tick) +{ + double factor = get_scale_factor(ewma->last_adjusted_tick, cur_tick); + ewma->cell_count *= factor; + ewma->last_adjusted_tick = cur_tick; +} + +/** Adjust the cell count of every active circuit on <b>conn</b> so + * that they are scaled with respect to <b>cur_tick</b> */ +static void +scale_active_circuits(or_connection_t *conn, unsigned cur_tick) +{ + + double factor = get_scale_factor( + conn->active_circuit_pqueue_last_recalibrated, + cur_tick); + /** Ordinarily it isn't okay to change the value of an element in a heap, + * but it's okay here, since we are preserving the order. */ + SMARTLIST_FOREACH(conn->active_circuit_pqueue, cell_ewma_t *, e, { + tor_assert(e->last_adjusted_tick == + conn->active_circuit_pqueue_last_recalibrated); + e->cell_count *= factor; + e->last_adjusted_tick = cur_tick; + }); + conn->active_circuit_pqueue_last_recalibrated = cur_tick; +} + +/** Rescale <b>ewma</b> to the same scale as <b>conn</b>, and add it to + * <b>conn</b>'s priority queue of active circuits */ +static void +add_cell_ewma_to_conn(or_connection_t *conn, cell_ewma_t *ewma) +{ + tor_assert(ewma->heap_index == -1); + scale_single_cell_ewma(ewma, + conn->active_circuit_pqueue_last_recalibrated); + + smartlist_pqueue_add(conn->active_circuit_pqueue, + compare_cell_ewma_counts, + STRUCT_OFFSET(cell_ewma_t, heap_index), + ewma); +} + +/** Remove <b>ewma</b> from <b>conn</b>'s priority queue of active circuits */ +static void +remove_cell_ewma_from_conn(or_connection_t *conn, cell_ewma_t *ewma) +{ + tor_assert(ewma->heap_index != -1); + smartlist_pqueue_remove(conn->active_circuit_pqueue, + compare_cell_ewma_counts, + STRUCT_OFFSET(cell_ewma_t, heap_index), + ewma); +} + +/** Remove and return the first cell_ewma_t from conn's priority queue of + * active circuits. Requires that the priority queue is nonempty. */ +static cell_ewma_t * +pop_first_cell_ewma_from_conn(or_connection_t *conn) +{ + return smartlist_pqueue_pop(conn->active_circuit_pqueue, + compare_cell_ewma_counts, + STRUCT_OFFSET(cell_ewma_t, heap_index)); +} + /** Add <b>circ</b> to the list of circuits with pending cells on - * <b>conn</b>. No effect if <b>circ</b> is already unlinked. */ + * <b>conn</b>. No effect if <b>circ</b> is already linked. */ void make_circuit_active_on_conn(circuit_t *circ, or_connection_t *conn) { @@ -1679,6 +2122,8 @@ make_circuit_active_on_conn(circuit_t *circ, or_connection_t *conn) return; } + assert_active_circuits_ok_paranoid(conn); + if (! conn->active_circuits) { conn->active_circuits = circ; *prevp = *nextp = circ; @@ -1690,10 +2135,19 @@ make_circuit_active_on_conn(circuit_t *circ, or_connection_t *conn) *prev_circ_on_conn_p(head, conn) = circ; *prevp = old_tail; } + + if (circ->n_conn == conn) { + add_cell_ewma_to_conn(conn, &circ->n_cell_ewma); + } else { + or_circuit_t *orcirc = TO_OR_CIRCUIT(circ); + tor_assert(conn == orcirc->p_conn); + add_cell_ewma_to_conn(conn, &orcirc->p_cell_ewma); + } + assert_active_circuits_ok_paranoid(conn); } -/** Remove <b>circ</b> to the list of circuits with pending cells on +/** Remove <b>circ</b> from the list of circuits with pending cells on * <b>conn</b>. No effect if <b>circ</b> is already unlinked. */ void make_circuit_inactive_on_conn(circuit_t *circ, or_connection_t *conn) @@ -1707,6 +2161,8 @@ make_circuit_inactive_on_conn(circuit_t *circ, or_connection_t *conn) return; } + assert_active_circuits_ok_paranoid(conn); + tor_assert(next && prev); tor_assert(*prev_circ_on_conn_p(next, conn) == circ); tor_assert(*next_circ_on_conn_p(prev, conn) == circ); @@ -1720,6 +2176,15 @@ make_circuit_inactive_on_conn(circuit_t *circ, or_connection_t *conn) conn->active_circuits = next; } *prevp = *nextp = NULL; + + if (circ->n_conn == conn) { + remove_cell_ewma_from_conn(conn, &circ->n_cell_ewma); + } else { + or_circuit_t *orcirc = TO_OR_CIRCUIT(circ); + tor_assert(conn == orcirc->p_conn); + remove_cell_ewma_from_conn(conn, &orcirc->p_cell_ewma); + } + assert_active_circuits_ok_paranoid(conn); } @@ -1739,16 +2204,27 @@ connection_or_unlink_all_active_circs(or_connection_t *orconn) cur = next; } while (cur != head); orconn->active_circuits = NULL; + + SMARTLIST_FOREACH(orconn->active_circuit_pqueue, cell_ewma_t *, e, + e->heap_index = -1); + smartlist_clear(orconn->active_circuit_pqueue); } /** Block (if <b>block</b> is true) or unblock (if <b>block</b> is false) * every edge connection that is using <b>circ</b> to write to <b>orconn</b>, - * and start or stop reading as appropriate. */ -static void + * and start or stop reading as appropriate. + * + * If <b>stream_id</b> is nonzero, block only the edge connection whose + * stream_id matches it. + * + * Returns the number of streams whose status we changed. + */ +static int set_streams_blocked_on_circ(circuit_t *circ, or_connection_t *orconn, - int block) + int block, streamid_t stream_id) { edge_connection_t *edge = NULL; + int n = 0; if (circ->n_conn == orconn) { circ->streams_blocked_on_n_conn = block; if (CIRCUIT_IS_ORIGIN(circ)) @@ -1761,7 +2237,13 @@ set_streams_blocked_on_circ(circuit_t *circ, or_connection_t *orconn, for (; edge; edge = edge->next_stream) { connection_t *conn = TO_CONN(edge); - edge->edge_blocked_on_circ = block; + if (stream_id && edge->stream_id != stream_id) + continue; + + if (edge->edge_blocked_on_circ != block) { + ++n; + edge->edge_blocked_on_circ = block; + } if (!conn->read_event) { /* This connection is a placeholder for something; probably a DNS @@ -1778,10 +2260,12 @@ set_streams_blocked_on_circ(circuit_t *circ, or_connection_t *orconn, connection_start_reading(conn); } } + + return n; } /** Pull as many cells as possible (but no more than <b>max</b>) from the - * queue of the first active circuit on <b>conn</b>, and write then to + * queue of the first active circuit on <b>conn</b>, and write them to * <b>conn</b>->outbuf. Return the number of cells written. Advance * the active circuit pointer to the next active circuit in the ring. */ int @@ -1792,9 +2276,35 @@ connection_or_flush_from_first_active_circuit(or_connection_t *conn, int max, cell_queue_t *queue; circuit_t *circ; int streams_blocked; + + /* The current (hi-res) time */ + struct timeval now_hires; + + /* The EWMA cell counter for the circuit we're flushing. */ + cell_ewma_t *cell_ewma = NULL; + double ewma_increment = -1; + circ = conn->active_circuits; if (!circ) return 0; assert_active_circuits_ok_paranoid(conn); + + /* See if we're doing the ewma circuit selection algorithm. */ + if (ewma_enabled) { + unsigned tick; + double fractional_tick; + tor_gettimeofday_cached(&now_hires); + tick = cell_ewma_tick_from_timeval(&now_hires, &fractional_tick); + + if (tick != conn->active_circuit_pqueue_last_recalibrated) { + scale_active_circuits(conn, tick); + } + + ewma_increment = pow(ewma_scale_factor, -fractional_tick); + + cell_ewma = smartlist_get(conn->active_circuit_pqueue, 0); + circ = cell_ewma_to_circuit(cell_ewma); + } + if (circ->n_conn == conn) { queue = &circ->n_conn_cells; streams_blocked = circ->streams_blocked_on_n_conn; @@ -1808,10 +2318,60 @@ connection_or_flush_from_first_active_circuit(or_connection_t *conn, int max, packed_cell_t *cell = cell_queue_pop(queue); tor_assert(*next_circ_on_conn_p(circ,conn)); + /* Calculate the exact time that this cell has spent in the queue. */ + if (get_options()->CellStatistics && !CIRCUIT_IS_ORIGIN(circ)) { + struct timeval now; + uint32_t flushed; + uint32_t cell_waiting_time; + insertion_time_queue_t *it_queue = queue->insertion_times; + tor_gettimeofday_cached(&now); + flushed = (uint32_t)((now.tv_sec % SECONDS_IN_A_DAY) * 100L + + (uint32_t)now.tv_usec / (uint32_t)10000L); + if (!it_queue || !it_queue->first) { + log_info(LD_GENERAL, "Cannot determine insertion time of cell. " + "Looks like the CellStatistics option was " + "recently enabled."); + } else { + or_circuit_t *orcirc = TO_OR_CIRCUIT(circ); + insertion_time_elem_t *elem = it_queue->first; + cell_waiting_time = + (uint32_t)((flushed * 10L + SECONDS_IN_A_DAY * 1000L - + elem->insertion_time * 10L) % + (SECONDS_IN_A_DAY * 1000L)); +#undef SECONDS_IN_A_DAY + elem->counter--; + if (elem->counter < 1) { + it_queue->first = elem->next; + if (elem == it_queue->last) + it_queue->last = NULL; + mp_pool_release(elem); + } + orcirc->total_cell_waiting_time += cell_waiting_time; + orcirc->processed_cells++; + } + } + + /* If we just flushed our queue and this circuit is used for a + * tunneled directory request, possibly advance its state. */ + if (queue->n == 0 && TO_CONN(conn)->dirreq_id) + geoip_change_dirreq_state(TO_CONN(conn)->dirreq_id, + DIRREQ_TUNNELED, + DIRREQ_CIRC_QUEUE_FLUSHED); + connection_write_to_buf(cell->body, CELL_NETWORK_SIZE, TO_CONN(conn)); - packed_cell_free(cell); + packed_cell_free_unchecked(cell); ++n_flushed; + if (cell_ewma) { + cell_ewma_t *tmp; + cell_ewma->cell_count += ewma_increment; + /* We pop and re-add the cell_ewma_t here, not above, since we need to + * re-add it immediately to keep the priority queue consistent with + * the linked-list implementation */ + tmp = pop_first_cell_ewma_from_conn(conn); + tor_assert(tmp == cell_ewma); + add_cell_ewma_to_conn(conn, cell_ewma); + } if (circ != conn->active_circuits) { /* If this happens, the current circuit just got made inactive by * a call in connection_write_to_buf(). That's nothing to worry about: @@ -1829,9 +2389,9 @@ connection_or_flush_from_first_active_circuit(or_connection_t *conn, int max, /* Is the cell queue low enough to unblock all the streams that are waiting * to write to this circuit? */ if (streams_blocked && queue->n <= CELL_QUEUE_LOWWATER_SIZE) - set_streams_blocked_on_circ(circ, conn, 0); /* unblock streams */ + set_streams_blocked_on_circ(circ, conn, 0, 0); /* unblock streams */ - /* Did we just ran out of cells on this queue? */ + /* Did we just run out of cells on this circuit's queue? */ if (queue->n == 0) { log_debug(LD_GENERAL, "Made a circuit inactive."); make_circuit_inactive_on_conn(circ, conn); @@ -1846,10 +2406,14 @@ connection_or_flush_from_first_active_circuit(or_connection_t *conn, int max, * transmitting in <b>direction</b>. */ void append_cell_to_circuit_queue(circuit_t *circ, or_connection_t *orconn, - cell_t *cell, cell_direction_t direction) + cell_t *cell, cell_direction_t direction, + streamid_t fromstream) { cell_queue_t *queue; int streams_blocked; + if (circ->marked_for_close) + return; + if (direction == CELL_DIRECTION_OUT) { queue = &circ->n_conn_cells; streams_blocked = circ->streams_blocked_on_n_conn; @@ -1868,7 +2432,12 @@ append_cell_to_circuit_queue(circuit_t *circ, or_connection_t *orconn, /* If we have too many cells on the circuit, we should stop reading from * the edge streams for a while. */ if (!streams_blocked && queue->n >= CELL_QUEUE_HIGHWATER_SIZE) - set_streams_blocked_on_circ(circ, orconn, 1); /* block streams */ + set_streams_blocked_on_circ(circ, orconn, 1, 0); /* block streams */ + + if (streams_blocked && fromstream) { + /* This edge connection is apparently not blocked; block it. */ + set_streams_blocked_on_circ(circ, orconn, 1, fromstream); + } if (queue->n == 1) { /* This was the first cell added to the queue. We need to make this @@ -1947,6 +2516,25 @@ decode_address_from_payload(tor_addr_t *addr_out, const uint8_t *payload, return payload + 2 + payload[1]; } +/** Remove all the cells queued on <b>circ</b> for <b>orconn</b>. */ +void +circuit_clear_cell_queue(circuit_t *circ, or_connection_t *orconn) +{ + cell_queue_t *queue; + if (circ->n_conn == orconn) { + queue = &circ->n_conn_cells; + } else { + or_circuit_t *orcirc = TO_OR_CIRCUIT(circ); + tor_assert(orcirc->p_conn == orconn); + queue = &orcirc->p_conn_cells; + } + + if (queue->n) + make_circuit_inactive_on_conn(circ,orconn); + + cell_queue_clear(queue); +} + /** Fail with an assert if the active circuits ring on <b>orconn</b> is * corrupt. */ void @@ -1954,16 +2542,44 @@ assert_active_circuits_ok(or_connection_t *orconn) { circuit_t *head = orconn->active_circuits; circuit_t *cur = head; + int n = 0; if (! head) return; do { circuit_t *next = *next_circ_on_conn_p(cur, orconn); circuit_t *prev = *prev_circ_on_conn_p(cur, orconn); + cell_ewma_t *ewma; tor_assert(next); tor_assert(prev); tor_assert(*next_circ_on_conn_p(prev, orconn) == cur); tor_assert(*prev_circ_on_conn_p(next, orconn) == cur); + if (orconn == cur->n_conn) { + ewma = &cur->n_cell_ewma; + tor_assert(!ewma->is_for_p_conn); + } else { + ewma = &TO_OR_CIRCUIT(cur)->p_cell_ewma; + tor_assert(ewma->is_for_p_conn); + } + tor_assert(ewma->heap_index != -1); + tor_assert(ewma == smartlist_get(orconn->active_circuit_pqueue, + ewma->heap_index)); + n++; cur = next; } while (cur != head); + + tor_assert(n == smartlist_len(orconn->active_circuit_pqueue)); +} + +/** Return 1 if we shouldn't restart reading on this circuit, even if + * we get a SENDME. Else return 0. +*/ +static int +circuit_queue_streams_are_blocked(circuit_t *circ) +{ + if (CIRCUIT_IS_ORIGIN(circ)) { + return circ->streams_blocked_on_n_conn; + } else { + return circ->streams_blocked_on_p_conn; + } } diff --git a/src/or/relay.h b/src/or/relay.h new file mode 100644 index 0000000000..f64752da5d --- /dev/null +++ b/src/or/relay.h @@ -0,0 +1,70 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file relay.h + * \brief Header file for relay.c. + **/ + +#ifndef _TOR_RELAY_H +#define _TOR_RELAY_H + +extern uint64_t stats_n_relay_cells_relayed; +extern uint64_t stats_n_relay_cells_delivered; + +int circuit_receive_relay_cell(cell_t *cell, circuit_t *circ, + cell_direction_t cell_direction); + +void relay_header_pack(uint8_t *dest, const relay_header_t *src); +void relay_header_unpack(relay_header_t *dest, const uint8_t *src); +int relay_send_command_from_edge(streamid_t stream_id, circuit_t *circ, + uint8_t relay_command, const char *payload, + size_t payload_len, crypt_path_t *cpath_layer); +int connection_edge_send_command(edge_connection_t *fromconn, + uint8_t relay_command, const char *payload, + size_t payload_len); +int connection_edge_package_raw_inbuf(edge_connection_t *conn, + int package_partial, + int *max_cells); +void connection_edge_consider_sending_sendme(edge_connection_t *conn); + +extern uint64_t stats_n_data_cells_packaged; +extern uint64_t stats_n_data_bytes_packaged; +extern uint64_t stats_n_data_cells_received; +extern uint64_t stats_n_data_bytes_received; + +void init_cell_pool(void); +void free_cell_pool(void); +void clean_cell_pool(void); +void dump_cell_pool_usage(int severity); + +void cell_queue_clear(cell_queue_t *queue); +void cell_queue_append(cell_queue_t *queue, packed_cell_t *cell); +void cell_queue_append_packed_copy(cell_queue_t *queue, const cell_t *cell); + +void append_cell_to_circuit_queue(circuit_t *circ, or_connection_t *orconn, + cell_t *cell, cell_direction_t direction, + streamid_t fromstream); +void connection_or_unlink_all_active_circs(or_connection_t *conn); +int connection_or_flush_from_first_active_circuit(or_connection_t *conn, + int max, time_t now); +void assert_active_circuits_ok(or_connection_t *orconn); +void make_circuit_inactive_on_conn(circuit_t *circ, or_connection_t *conn); +void make_circuit_active_on_conn(circuit_t *circ, or_connection_t *conn); + +int append_address_to_payload(uint8_t *payload_out, const tor_addr_t *addr); +const uint8_t *decode_address_from_payload(tor_addr_t *addr_out, + const uint8_t *payload, + int payload_len); +unsigned cell_ewma_get_tick(void); +void cell_ewma_set_scale_factor(or_options_t *options, + networkstatus_t *consensus); +void circuit_clear_cell_queue(circuit_t *circ, or_connection_t *orconn); + +void tor_gettimeofday_cache_clear(void); + +#endif + diff --git a/src/or/rendclient.c b/src/or/rendclient.c index af91099fcb..8d024d8ebb 100644 --- a/src/or/rendclient.c +++ b/src/or/rendclient.c @@ -8,6 +8,23 @@ **/ #include "or.h" +#include "circuitbuild.h" +#include "circuitlist.h" +#include "circuituse.h" +#include "config.h" +#include "connection.h" +#include "connection_edge.h" +#include "directory.h" +#include "main.h" +#include "relay.h" +#include "rendclient.h" +#include "rendcommon.h" +#include "rephist.h" +#include "routerlist.h" + +static extend_info_t *rend_client_get_random_intro_impl( + const rend_cache_entry_t *rend_query, + const int strict, const int warnings); /** Called when we've established a circuit to an introduction point: * send the introduction request. */ @@ -49,6 +66,50 @@ rend_client_send_establish_rendezvous(origin_circuit_t *circ) return 0; } +/** Extend the introduction circuit <b>circ</b> to another valid + * introduction point for the hidden service it is trying to connect + * to, or mark it and launch a new circuit if we can't extend it. + * Return 0 on success. Return -1 and mark the introduction + * circuit on failure. + * + * On failure, the caller is responsible for marking the associated + * rendezvous circuit for close. */ +static int +rend_client_reextend_intro_circuit(origin_circuit_t *circ) +{ + extend_info_t *extend_info; + int result; + extend_info = rend_client_get_random_intro(circ->rend_data); + if (!extend_info) { + log_warn(LD_REND, + "No usable introduction points left for %s. Closing.", + safe_str_client(circ->rend_data->onion_address)); + circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_INTERNAL); + return -1; + } + if (circ->remaining_relay_early_cells) { + log_info(LD_REND, + "Re-extending circ %d, this time to %s.", + circ->_base.n_circ_id, extend_info->nickname); + result = circuit_extend_to_new_exit(circ, extend_info); + } else { + log_info(LD_REND, + "Building a new introduction circuit, this time to %s.", + extend_info->nickname); + circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_FINISHED); + if (!circuit_launch_by_extend_info(CIRCUIT_PURPOSE_C_INTRODUCING, + extend_info, + CIRCLAUNCH_IS_INTERNAL)) { + log_warn(LD_REND, "Building introduction circuit failed."); + result = -1; + } else { + result = 0; + } + } + extend_info_free(extend_info); + return result; +} + /** Called when we're trying to connect an ap conn; sends an INTRODUCE1 cell * down introcirc if possible. */ @@ -63,7 +124,7 @@ rend_client_send_introduction(origin_circuit_t *introcirc, rend_cache_entry_t *entry; crypt_path_t *cpath; off_t dh_offset; - crypto_pk_env_t *intro_key; /* either Bob's public key or an intro key. */ + crypto_pk_env_t *intro_key = NULL; tor_assert(introcirc->_base.purpose == CIRCUIT_PURPOSE_C_INTRODUCING); tor_assert(rendcirc->_base.purpose == CIRCUIT_PURPOSE_C_REND_READY); @@ -74,54 +135,51 @@ rend_client_send_introduction(origin_circuit_t *introcirc, if (rend_cache_lookup_entry(introcirc->rend_data->onion_address, -1, &entry) < 1) { - log_warn(LD_REND, - "query %s didn't have valid rend desc in cache. Failing.", - escaped_safe_str(introcirc->rend_data->onion_address)); - goto err; + log_info(LD_REND, + "query %s didn't have valid rend desc in cache. " + "Refetching descriptor.", + safe_str_client(introcirc->rend_data->onion_address)); + rend_client_refetch_v2_renddesc(introcirc->rend_data); + { + connection_t *conn; + + while ((conn = connection_get_by_type_state_rendquery(CONN_TYPE_AP, + AP_CONN_STATE_CIRCUIT_WAIT, + introcirc->rend_data->onion_address))) { + conn->state = AP_CONN_STATE_RENDDESC_WAIT; + } + } + + return -1; } /* first 20 bytes of payload are the hash of Bob's pk */ - if (entry->parsed->version == 0) { /* un-versioned descriptor */ - intro_key = entry->parsed->pk; - } else { /* versioned descriptor */ - intro_key = NULL; - SMARTLIST_FOREACH(entry->parsed->intro_nodes, rend_intro_point_t *, - intro, { - if (!memcmp(introcirc->build_state->chosen_exit->identity_digest, - intro->extend_info->identity_digest, DIGEST_LEN)) { - intro_key = intro->intro_key; - break; - } - }); - if (!intro_key) { - /** XXX This case probably means that the intro point vanished while - * we were building a circuit to it. In the future, we should find - * out how that happened and whether we should kill the circuits to - * removed intro points immediately. See task 1073. */ - int num_intro_points = smartlist_len(entry->parsed->intro_nodes); - if (rend_cache_lookup_entry(introcirc->rend_data->onion_address, - 0, &entry) > 0) { - log_info(LD_REND, "We have both a v0 and a v2 rend desc for this " - "service. The v2 desc doesn't contain the introduction " - "point (and key) to send an INTRODUCE1/2 cell to this " - "introduction point. Assuming the introduction point " - "is for v0 rend clients and using the service key " - "from the v0 desc instead. (This is probably a bug, " - "because we shouldn't even have both a v0 and a v2 " - "descriptor for the same service.)"); - /* See flyspray task 1024. */ - intro_key = entry->parsed->pk; - } else { - log_info(LD_REND, "Internal error: could not find intro key; we " - "only have a v2 rend desc with %d intro points.", - num_intro_points); - goto err; - } + intro_key = NULL; + SMARTLIST_FOREACH(entry->parsed->intro_nodes, rend_intro_point_t *, + intro, { + if (!memcmp(introcirc->build_state->chosen_exit->identity_digest, + intro->extend_info->identity_digest, DIGEST_LEN)) { + intro_key = intro->intro_key; + break; + } + }); + if (!intro_key) { + log_info(LD_REND, "Could not find intro key for %s at %s; we " + "have a v2 rend desc with %d intro points. " + "Trying a different intro point...", + safe_str_client(introcirc->rend_data->onion_address), + introcirc->build_state->chosen_exit->nickname, + smartlist_len(entry->parsed->intro_nodes)); + + if (rend_client_reextend_intro_circuit(introcirc)) { + goto perm_err; + } else { + return -1; } } if (crypto_pk_get_digest(intro_key, payload)<0) { log_warn(LD_BUG, "Internal error: couldn't hash public key."); - goto err; + goto perm_err; } /* Initialize the pending_final_cpath and start the DH handshake. */ @@ -132,11 +190,11 @@ rend_client_send_introduction(origin_circuit_t *introcirc, cpath->magic = CRYPT_PATH_MAGIC; if (!(cpath->dh_handshake_state = crypto_dh_new(DH_TYPE_REND))) { log_warn(LD_BUG, "Internal error: couldn't allocate DH."); - goto err; + goto perm_err; } if (crypto_dh_generate_public(cpath->dh_handshake_state)<0) { log_warn(LD_BUG, "Internal error: couldn't generate g^x."); - goto err; + goto perm_err; } } @@ -186,7 +244,7 @@ rend_client_send_introduction(origin_circuit_t *introcirc, if (crypto_dh_get_public(cpath->dh_handshake_state, tmp+dh_offset, DH_KEY_LEN)<0) { log_warn(LD_BUG, "Internal error: couldn't extract g^x."); - goto err; + goto perm_err; } note_crypto_pk_op(REND_CLIENT); @@ -199,7 +257,7 @@ rend_client_send_introduction(origin_circuit_t *introcirc, PK_PKCS1_OAEP_PADDING, 0); if (r<0) { log_warn(LD_BUG,"Internal error: hybrid pk encrypt failed."); - goto err; + goto perm_err; } payload_len = DIGEST_LEN + r; @@ -212,17 +270,18 @@ rend_client_send_introduction(origin_circuit_t *introcirc, introcirc->cpath->prev)<0) { /* introcirc is already marked for close. leave rendcirc alone. */ log_warn(LD_BUG, "Couldn't send INTRODUCE1 cell"); - return -1; + return -2; } /* Now, we wait for an ACK or NAK on this circuit. */ introcirc->_base.purpose = CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT; return 0; -err: - circuit_mark_for_close(TO_CIRCUIT(introcirc), END_CIRC_REASON_INTERNAL); +perm_err: + if (!introcirc->_base.marked_for_close) + circuit_mark_for_close(TO_CIRCUIT(introcirc), END_CIRC_REASON_INTERNAL); circuit_mark_for_close(TO_CIRCUIT(rendcirc), END_CIRC_REASON_INTERNAL); - return -1; + return -2; } /** Called when a rendezvous circuit is open; sends a establish @@ -283,45 +342,16 @@ rend_client_introduction_acked(origin_circuit_t *circ, * points. If any remain, extend to a new one and try again. * If none remain, refetch the service descriptor. */ + log_info(LD_REND, "Got nack for %s from %s...", + safe_str_client(circ->rend_data->onion_address), + circ->build_state->chosen_exit->nickname); if (rend_client_remove_intro_point(circ->build_state->chosen_exit, circ->rend_data) > 0) { /* There are introduction points left. Re-extend the circuit to * another intro point and try again. */ - extend_info_t *extend_info; - int result; - extend_info = rend_client_get_random_intro(circ->rend_data); - if (!extend_info) { - log_warn(LD_REND, "No introduction points left for %s. Closing.", - escaped_safe_str(circ->rend_data->onion_address)); - circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_INTERNAL); - return -1; - } - if (circ->remaining_relay_early_cells) { - log_info(LD_REND, - "Got nack for %s from %s. Re-extending circ %d, " - "this time to %s.", - escaped_safe_str(circ->rend_data->onion_address), - circ->build_state->chosen_exit->nickname, - circ->_base.n_circ_id, extend_info->nickname); - result = circuit_extend_to_new_exit(circ, extend_info); - } else { - log_info(LD_REND, - "Got nack for %s from %s. Building a new introduction " - "circuit, this time to %s.", - escaped_safe_str(circ->rend_data->onion_address), - circ->build_state->chosen_exit->nickname, - extend_info->nickname); - circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_FINISHED); - if (!circuit_launch_by_extend_info(CIRCUIT_PURPOSE_C_INTRODUCING, - extend_info, - CIRCLAUNCH_IS_INTERNAL)) { - log_warn(LD_REND, "Building introduction circuit failed."); - result = -1; - } else { - result = 0; - } - } - extend_info_free(extend_info); + int result = rend_client_reextend_intro_circuit(circ); + /* XXXX If that call failed, should we close the rend circuit, + * too? */ return result; } } @@ -412,7 +442,7 @@ directory_get_from_hs_dir(const char *desc_id, const rend_data_t *rend_query) tor_assert(rend_query); /* Determine responsible dirs. Even if we can't get all we want, * work with the ones we have. If it's empty, we'll notice below. */ - (int) hid_serv_get_responsible_directories(responsible_dirs, desc_id); + hid_serv_get_responsible_directories(responsible_dirs, desc_id); base32_encode(desc_id_base32, sizeof(desc_id_base32), desc_id, DIGEST_LEN); @@ -471,45 +501,21 @@ directory_get_from_hs_dir(const char *desc_id, const rend_data_t *rend_query) rend_query->onion_address, desc_id_base32, rend_query->auth_type, (rend_query->auth_type == REND_NO_AUTH ? "[none]" : - escaped_safe_str(descriptor_cookie_base64)), + escaped_safe_str_client(descriptor_cookie_base64)), hs_dir->nickname, hs_dir->dir_port); return 1; } -/** If we are not currently fetching a rendezvous service descriptor - * for the service ID <b>query</b>, start a directory connection to fetch a - * new one. - */ -void -rend_client_refetch_renddesc(const char *query) -{ - if (!get_options()->FetchHidServDescriptors) - return; - log_info(LD_REND, "Fetching rendezvous descriptor for service %s", - escaped_safe_str(query)); - if (connection_get_by_type_state_rendquery(CONN_TYPE_DIR, 0, query, 0)) { - log_info(LD_REND,"Would fetch a new renddesc here (for %s), but one is " - "already in progress.", escaped_safe_str(query)); - } else { - /* not one already; initiate a dir rend desc lookup */ - directory_get_from_dirserver(DIR_PURPOSE_FETCH_RENDDESC, - ROUTER_PURPOSE_GENERAL, query, - PDS_RETRY_IF_NO_SERVERS); - } -} - -/** Start a connection to a hidden service directory to fetch a v2 - * rendezvous service descriptor for the base32-encoded service ID - * <b>query</b>. - */ +/** Unless we already have a descriptor for <b>rend_query</b> with at least + * one (possibly) working introduction point in it, start a connection to a + * hidden service directory to fetch a v2 rendezvous service descriptor. */ void rend_client_refetch_v2_renddesc(const rend_data_t *rend_query) { char descriptor_id[DIGEST_LEN]; int replicas_left_to_try[REND_NUMBER_OF_NON_CONSECUTIVE_REPLICAS]; - int i, tries_left, r; + int i, tries_left; rend_cache_entry_t *e = NULL; - time_t now = time(NULL); tor_assert(rend_query); /* Are we configured to fetch descriptors? */ if (!get_options()->FetchHidServDescriptors) { @@ -518,15 +524,13 @@ rend_client_refetch_v2_renddesc(const rend_data_t *rend_query) return; } /* Before fetching, check if we already have the descriptor here. */ - r = rend_cache_lookup_entry(rend_query->onion_address, -1, &e); - if (r > 0 && now - e->received < NUM_SECONDS_BEFORE_HS_REFETCH) { + if (rend_cache_lookup_entry(rend_query->onion_address, -1, &e) > 0) { log_info(LD_REND, "We would fetch a v2 rendezvous descriptor, but we " - "already have a fresh copy of that descriptor here. " - "Not fetching."); + "already have that descriptor here. Not fetching."); return; } log_debug(LD_REND, "Fetching v2 rendezvous descriptor for service %s", - safe_str(rend_query->onion_address)); + safe_str_client(rend_query->onion_address)); /* Randomly iterate over the replicas until a descriptor can be fetched * from one of the consecutive nodes, or no options are left. */ tries_left = REND_NUMBER_OF_NON_CONSECUTIVE_REPLICAS; @@ -552,13 +556,49 @@ rend_client_refetch_v2_renddesc(const rend_data_t *rend_query) log_info(LD_REND, "Could not pick one of the responsible hidden " "service directories to fetch descriptors, because " "we already tried them all unsuccessfully."); - /* Close pending connections (unless a v0 request is still going on). */ - rend_client_desc_trynow(rend_query->onion_address, 2); + /* Close pending connections. */ + rend_client_desc_trynow(rend_query->onion_address); return; } +/** Cancel all rendezvous descriptor fetches currently in progress. + */ +void +rend_client_cancel_descriptor_fetches(void) +{ + smartlist_t *connection_array = get_connection_array(); + + SMARTLIST_FOREACH_BEGIN(connection_array, connection_t *, conn) { + if (conn->type == CONN_TYPE_DIR && + (conn->purpose == DIR_PURPOSE_FETCH_RENDDESC || + conn->purpose == DIR_PURPOSE_FETCH_RENDDESC_V2)) { + /* It's a rendezvous descriptor fetch in progress -- cancel it + * by marking the connection for close. + * + * Even if this connection has already reached EOF, this is + * enough to make sure that if the descriptor hasn't been + * processed yet, it won't be. See the end of + * connection_handle_read; connection_reached_eof (indirectly) + * processes whatever response the connection received. */ + + const rend_data_t *rd = (TO_DIR_CONN(conn))->rend_data; + if (!rd) { + log_warn(LD_BUG | LD_REND, + "Marking for close dir conn fetching rendezvous " + "descriptor for unknown service!"); + } else { + log_debug(LD_REND, "Marking for close dir conn fetching " + "rendezvous descriptor for service %s", + safe_str(rd->onion_address)); + } + connection_mark_for_close(conn); + } + } SMARTLIST_FOREACH_END(conn); +} + /** Remove failed_intro from ent. If ent now has no intro points, or * service is unrecognized, then launch a new renddesc fetch. + * * Return -1 if error, 0 if no intro points remain or service * unrecognized, 1 if recognized and some intro points remain. @@ -574,18 +614,13 @@ rend_client_remove_intro_point(extend_info_t *failed_intro, r = rend_cache_lookup_entry(rend_query->onion_address, -1, &ent); if (r<0) { log_warn(LD_BUG, "Malformed service ID %s.", - escaped_safe_str(rend_query->onion_address)); + escaped_safe_str_client(rend_query->onion_address)); return -1; } if (r==0) { log_info(LD_REND, "Unknown service %s. Re-fetching descriptor.", - escaped_safe_str(rend_query->onion_address)); - /* Fetch both, v0 and v2 rend descriptors in parallel. Use whichever - * arrives first. Exception: When using client authorization, only - * fetch v2 descriptors.*/ + escaped_safe_str_client(rend_query->onion_address)); rend_client_refetch_v2_renddesc(rend_query); - if (rend_query->auth_type == REND_NO_AUTH) - rend_client_refetch_renddesc(rend_query->onion_address); return 0; } @@ -599,21 +634,16 @@ rend_client_remove_intro_point(extend_info_t *failed_intro, } } - if (smartlist_len(ent->parsed->intro_nodes) == 0) { + if (! rend_client_any_intro_points_usable(ent)) { log_info(LD_REND, "No more intro points remain for %s. Re-fetching descriptor.", - escaped_safe_str(rend_query->onion_address)); - /* Fetch both, v0 and v2 rend descriptors in parallel. Use whichever - * arrives first. Exception: When using client authorization, only - * fetch v2 descriptors.*/ + escaped_safe_str_client(rend_query->onion_address)); rend_client_refetch_v2_renddesc(rend_query); - if (rend_query->auth_type == REND_NO_AUTH) - rend_client_refetch_renddesc(rend_query->onion_address); /* move all pending streams back to renddesc_wait */ while ((conn = connection_get_by_type_state_rendquery(CONN_TYPE_AP, AP_CONN_STATE_CIRCUIT_WAIT, - rend_query->onion_address, -1))) { + rend_query->onion_address))) { conn->state = AP_CONN_STATE_RENDDESC_WAIT; } @@ -621,7 +651,7 @@ rend_client_remove_intro_point(extend_info_t *failed_intro, } log_info(LD_REND,"%d options left for %s.", smartlist_len(ent->parsed->intro_nodes), - escaped_safe_str(rend_query->onion_address)); + escaped_safe_str_client(rend_query->onion_address)); return 1; } @@ -644,7 +674,7 @@ rend_client_rendezvous_acked(origin_circuit_t *circ, const uint8_t *request, log_info(LD_REND,"Got rendezvous ack. This circuit is now ready for " "rendezvous."); circ->_base.purpose = CIRCUIT_PURPOSE_C_REND_READY; - /* XXXX022 This is a pretty brute-force approach. It'd be better to + /* XXXX023 This is a pretty brute-force approach. It'd be better to * attach only the connections that are waiting on this circuit, rather * than trying to attach them all. See comments bug 743. */ /* If we already have the introduction circuit built, make sure we send @@ -683,7 +713,8 @@ rend_client_receive_rendezvous(origin_circuit_t *circ, const uint8_t *request, tor_assert(circ->build_state->pending_final_cpath); hop = circ->build_state->pending_final_cpath; tor_assert(hop->dh_handshake_state); - if (crypto_dh_compute_secret(hop->dh_handshake_state, (char*)request, + if (crypto_dh_compute_secret(LOG_PROTOCOL_WARN, + hop->dh_handshake_state, (char*)request, DH_KEY_LEN, keys, DIGEST_LEN+CPATH_KEY_MATERIAL_LEN)<0) { log_warn(LD_GENERAL, "Couldn't complete DH handshake."); @@ -713,7 +744,7 @@ rend_client_receive_rendezvous(origin_circuit_t *circ, const uint8_t *request, onion_append_to_cpath(&circ->cpath, hop); circ->build_state->pending_final_cpath = NULL; /* prevent double-free */ - /* XXXX022 This is a pretty brute-force approach. It'd be better to + /* XXXX023 This is a pretty brute-force approach. It'd be better to * attach only the connections that are waiting on this circuit, rather * than trying to attach them all. See comments bug 743. */ connection_ap_attach_pending(); @@ -725,24 +756,18 @@ rend_client_receive_rendezvous(origin_circuit_t *circ, const uint8_t *request, return -1; } -/** Find all the apconns in state AP_CONN_STATE_RENDDESC_WAIT that - * are waiting on query. If there's a working cache entry here - * with at least one intro point, move them to the next state. If - * <b>rend_version</b> is non-negative, fail connections that have - * requested <b>query</b> unless there are still descriptor fetch - * requests in progress for other descriptor versions than - * <b>rend_version</b>. - */ +/** Find all the apconns in state AP_CONN_STATE_RENDDESC_WAIT that are + * waiting on <b>query</b>. If there's a working cache entry here with at + * least one intro point, move them to the next state. */ void -rend_client_desc_trynow(const char *query, int rend_version) +rend_client_desc_trynow(const char *query) { edge_connection_t *conn; rend_cache_entry_t *entry; time_t now = time(NULL); smartlist_t *conns = get_connection_array(); - SMARTLIST_FOREACH(conns, connection_t *, _conn, - { + SMARTLIST_FOREACH_BEGIN(conns, connection_t *, _conn) { if (_conn->type != CONN_TYPE_AP || _conn->state != AP_CONN_STATE_RENDDESC_WAIT || _conn->marked_for_close) @@ -755,7 +780,7 @@ rend_client_desc_trynow(const char *query, int rend_version) assert_connection_ok(TO_CONN(conn), now); if (rend_cache_lookup_entry(conn->rend_data->onion_address, -1, &entry) == 1 && - smartlist_len(entry->parsed->intro_nodes) > 0) { + rend_client_any_intro_points_usable(entry)) { /* either this fetch worked, or it failed but there was a * valid entry from before which we should reuse */ log_info(LD_REND,"Rend desc is usable. Launching circuits."); @@ -774,17 +799,12 @@ rend_client_desc_trynow(const char *query, int rend_version) connection_mark_unattached_ap(conn, END_STREAM_REASON_CANT_ATTACH); } } else { /* 404, or fetch didn't get that far */ - /* Unless there are requests for another descriptor version pending, - * close the connection. */ - if (rend_version >= 0 && - !connection_get_by_type_state_rendquery(CONN_TYPE_DIR, 0, query, - rend_version == 0 ? 2 : 0)) { - log_notice(LD_REND,"Closing stream for '%s.onion': hidden service is " - "unavailable (try again later).", safe_str(query)); - connection_mark_unattached_ap(conn, END_STREAM_REASON_RESOLVEFAILED); - } + log_notice(LD_REND,"Closing stream for '%s.onion': hidden service is " + "unavailable (try again later).", + safe_str_client(query)); + connection_mark_unattached_ap(conn, END_STREAM_REASON_RESOLVEFAILED); } - }); + } SMARTLIST_FOREACH_END(_conn); } /** Return a newly allocated extend_info_t* for a randomly chosen introduction @@ -794,24 +814,63 @@ rend_client_desc_trynow(const char *query, int rend_version) extend_info_t * rend_client_get_random_intro(const rend_data_t *rend_query) { - int i; + extend_info_t *result; rend_cache_entry_t *entry; - rend_intro_point_t *intro; - routerinfo_t *router; if (rend_cache_lookup_entry(rend_query->onion_address, -1, &entry) < 1) { - log_warn(LD_REND, - "Query '%s' didn't have valid rend desc in cache. Failing.", - safe_str(rend_query->onion_address)); + log_warn(LD_REND, + "Query '%s' didn't have valid rend desc in cache. Failing.", + safe_str_client(rend_query->onion_address)); return NULL; } + /* See if we can get a node that complies with ExcludeNodes */ + if ((result = rend_client_get_random_intro_impl(entry, 1, 1))) + return result; + /* If not, and StrictNodes is not set, see if we can return any old node + */ + if (!get_options()->StrictNodes) + return rend_client_get_random_intro_impl(entry, 0, 1); + return NULL; +} + +/** As rend_client_get_random_intro, except assume that StrictNodes is set + * iff <b>strict</b> is true. If <b>warnings</b> is false, don't complain + * to the user when we're out of nodes, even if StrictNodes is true. + */ +static extend_info_t * +rend_client_get_random_intro_impl(const rend_cache_entry_t *entry, + const int strict, + const int warnings) +{ + int i; + + rend_intro_point_t *intro; + routerinfo_t *router; + or_options_t *options = get_options(); + smartlist_t *usable_nodes; + int n_excluded = 0; + + /* We'll keep a separate list of the usable nodes. If this becomes empty, + * no nodes are usable. */ + usable_nodes = smartlist_create(); + smartlist_add_all(usable_nodes, entry->parsed->intro_nodes); + again: - if (smartlist_len(entry->parsed->intro_nodes) == 0) + if (smartlist_len(usable_nodes) == 0) { + if (n_excluded && get_options()->StrictNodes && warnings) { + /* We only want to warn if StrictNodes is really set. Otherwise + * we're just about to retry anyways. + */ + log_warn(LD_REND, "All introduction points for hidden service are " + "at excluded relays, and StrictNodes is set. Skipping."); + } + smartlist_free(usable_nodes); return NULL; + } - i = crypto_rand_int(smartlist_len(entry->parsed->intro_nodes)); - intro = smartlist_get(entry->parsed->intro_nodes, i); + i = crypto_rand_int(smartlist_len(usable_nodes)); + intro = smartlist_get(usable_nodes, i); /* Do we need to look up the router or is the extend info complete? */ if (!intro->extend_info->onion_key) { if (tor_digest_is_zero(intro->extend_info->identity_digest)) @@ -821,16 +880,34 @@ rend_client_get_random_intro(const rend_data_t *rend_query) if (!router) { log_info(LD_REND, "Unknown router with nickname '%s'; trying another.", intro->extend_info->nickname); - rend_intro_point_free(intro); - smartlist_del(entry->parsed->intro_nodes, i); + smartlist_del(usable_nodes, i); goto again; } extend_info_free(intro->extend_info); intro->extend_info = extend_info_from_router(router); } + /* Check if we should refuse to talk to this router. */ + if (options->ExcludeNodes && strict && + routerset_contains_extendinfo(options->ExcludeNodes, + intro->extend_info)) { + n_excluded++; + smartlist_del(usable_nodes, i); + goto again; + } + + smartlist_free(usable_nodes); return extend_info_dup(intro->extend_info); } +/** Return true iff any introduction points still listed in <b>entry</b> are + * usable. */ +int +rend_client_any_intro_points_usable(const rend_cache_entry_t *entry) +{ + return rend_client_get_random_intro_impl( + entry, get_options()->StrictNodes, 0) != NULL; +} + /** Client-side authorizations for hidden services; map of onion address to * rend_service_authorization_t*. */ static strmap_t *auth_hid_servs = NULL; @@ -955,8 +1032,7 @@ rend_parse_service_authorization(or_options_t *options, int validate_only) err: res = -1; done: - if (auth) - rend_service_authorization_free(auth); + rend_service_authorization_free(auth); SMARTLIST_FOREACH(sl, char *, c, tor_free(c);); smartlist_free(sl); if (!validate_only && res == 0) { diff --git a/src/or/rendclient.h b/src/or/rendclient.h new file mode 100644 index 0000000000..6910c1a97b --- /dev/null +++ b/src/or/rendclient.h @@ -0,0 +1,45 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file rendclient.h + * \brief Header file for rendclient.c. + **/ + +#ifndef _TOR_RENDCLIENT_H +#define _TOR_RENDCLIENT_H + +void rend_client_introcirc_has_opened(origin_circuit_t *circ); +void rend_client_rendcirc_has_opened(origin_circuit_t *circ); +int rend_client_introduction_acked(origin_circuit_t *circ, + const uint8_t *request, + size_t request_len); +void rend_client_refetch_v2_renddesc(const rend_data_t *rend_query); +void rend_client_cancel_descriptor_fetches(void); +int rend_client_remove_intro_point(extend_info_t *failed_intro, + const rend_data_t *rend_query); +int rend_client_rendezvous_acked(origin_circuit_t *circ, + const uint8_t *request, + size_t request_len); +int rend_client_receive_rendezvous(origin_circuit_t *circ, + const uint8_t *request, + size_t request_len); +void rend_client_desc_trynow(const char *query); + +extend_info_t *rend_client_get_random_intro(const rend_data_t *rend_query); +int rend_client_any_intro_points_usable(const rend_cache_entry_t *entry); + +int rend_client_send_introduction(origin_circuit_t *introcirc, + origin_circuit_t *rendcirc); +int rend_parse_service_authorization(or_options_t *options, + int validate_only); +rend_service_authorization_t *rend_client_lookup_service_authorization( + const char *onion_address); +void rend_service_authorization_free_all(void); +rend_data_t *rend_data_dup(const rend_data_t *request); + +#endif + diff --git a/src/or/rendcommon.c b/src/or/rendcommon.c index d6f5443815..f8e7f9bbb0 100644 --- a/src/or/rendcommon.c +++ b/src/or/rendcommon.c @@ -9,6 +9,15 @@ **/ #include "or.h" +#include "circuitbuild.h" +#include "config.h" +#include "rendclient.h" +#include "rendcommon.h" +#include "rendmid.h" +#include "rendservice.h" +#include "rephist.h" +#include "routerlist.h" +#include "routerparse.h" /** Return 0 if one and two are the same service ids, else -1 or 1 */ int @@ -22,6 +31,8 @@ rend_cmp_service_ids(const char *one, const char *two) void rend_service_descriptor_free(rend_service_descriptor_t *desc) { + if (!desc) + return; if (desc->pk) crypto_free_pk_env(desc->pk); if (desc->intro_nodes) { @@ -125,7 +136,8 @@ rend_compute_v2_desc_id(char *desc_id_out, const char *service_id, if (!service_id || strlen(service_id) != REND_SERVICE_ID_LEN_BASE32) { log_warn(LD_REND, "Could not compute v2 descriptor ID: " - "Illegal service ID: %s", safe_str(service_id)); + "Illegal service ID: %s", + safe_str(service_id)); return -1; } if (replica >= REND_NUMBER_OF_NON_CONSECUTIVE_REPLICAS) { @@ -138,7 +150,7 @@ rend_compute_v2_desc_id(char *desc_id_out, const char *service_id, service_id, REND_SERVICE_ID_LEN_BASE32) < 0) { log_warn(LD_REND, "Could not compute v2 descriptor ID: " "Illegal characters in service ID: %s", - safe_str(service_id)); + safe_str_client(service_id)); return -1; } /* Calculate current time-period. */ @@ -403,8 +415,7 @@ rend_desc_v2_is_parsable(rend_encoded_v2_service_descriptor_t *desc) &test_intro_size, &test_encoded_size, &test_next, desc->desc_str); - if (test_parsed) - rend_service_descriptor_free(test_parsed); + rend_service_descriptor_free(test_parsed); tor_free(test_intro_content); return (res >= 0); } @@ -414,6 +425,8 @@ void rend_encoded_v2_service_descriptor_free( rend_encoded_v2_service_descriptor_t *desc) { + if (!desc) + return; tor_free(desc->desc_str); tor_free(desc); } @@ -422,10 +435,11 @@ rend_encoded_v2_service_descriptor_free( void rend_intro_point_free(rend_intro_point_t *intro) { - if (intro->extend_info) - extend_info_free(intro->extend_info); - if (intro->intro_key) - crypto_free_pk_env(intro->intro_key); + if (!intro) + return; + + extend_info_free(intro->extend_info); + crypto_free_pk_env(intro->intro_key); tor_free(intro); } @@ -618,7 +632,8 @@ rend_encode_v2_descriptors(smartlist_t *descs_out, } if (router_append_dirobj_signature(desc_str + written, desc_len - written, - desc_digest, service_key) < 0) { + desc_digest, DIGEST_LEN, + service_key) < 0) { log_warn(LD_BUG, "Couldn't sign desc."); rend_encoded_v2_service_descriptor_free(enc); goto err; @@ -655,63 +670,6 @@ rend_encode_v2_descriptors(smartlist_t *descs_out, return seconds_valid; } -/** Encode a service descriptor for <b>desc</b>, and sign it with - * <b>key</b>. Store the descriptor in *<b>str_out</b>, and set - * *<b>len_out</b> to its length. - */ -int -rend_encode_service_descriptor(rend_service_descriptor_t *desc, - crypto_pk_env_t *key, - char **str_out, size_t *len_out) -{ - char *cp; - char *end; - int i, r; - size_t asn1len; - size_t buflen = - PK_BYTES*2*(smartlist_len(desc->intro_nodes)+2);/*Too long, but ok*/ - cp = *str_out = tor_malloc(buflen); - end = cp + PK_BYTES*2*(smartlist_len(desc->intro_nodes)+1); - r = crypto_pk_asn1_encode(desc->pk, cp+2, end-(cp+2)); - if (r < 0) { - tor_free(*str_out); - return -1; - } - asn1len = r; - set_uint16(cp, htons((uint16_t)asn1len)); - cp += 2+asn1len; - set_uint32(cp, htonl((uint32_t)desc->timestamp)); - cp += 4; - set_uint16(cp, htons((uint16_t)smartlist_len(desc->intro_nodes))); - cp += 2; - for (i=0; i < smartlist_len(desc->intro_nodes); ++i) { - rend_intro_point_t *intro = smartlist_get(desc->intro_nodes, i); - char ipoint[HEX_DIGEST_LEN+2]; - const size_t ipoint_len = HEX_DIGEST_LEN+1; - ipoint[0] = '$'; - base16_encode(ipoint+1, HEX_DIGEST_LEN+1, - intro->extend_info->identity_digest, - DIGEST_LEN); - tor_assert(strlen(ipoint) == ipoint_len); - /* Assert that appending ipoint and its NUL won't over overrun the - * buffer. */ - tor_assert(cp + ipoint_len+1 < *str_out + buflen); - memcpy(cp, ipoint, ipoint_len+1); - cp += ipoint_len+1; - } - note_crypto_pk_op(REND_SERVER); - r = crypto_pk_private_sign_digest(key, - cp, buflen - (cp - *str_out), - *str_out, cp-*str_out); - if (r<0) { - tor_free(*str_out); - return -1; - } - cp += r; - *len_out = (size_t)(cp-*str_out); - return 0; -} - /** Parse a service descriptor at <b>str</b> (<b>len</b> bytes). On * success, return a newly alloced service_descriptor_t. On failure, * return NULL. @@ -828,22 +786,27 @@ rend_cache_init(void) /** Helper: free storage held by a single service descriptor cache entry. */ static void -_rend_cache_entry_free(void *p) +rend_cache_entry_free(rend_cache_entry_t *e) { - rend_cache_entry_t *e = p; + if (!e) + return; rend_service_descriptor_free(e->parsed); tor_free(e->desc); tor_free(e); } +static void +_rend_cache_entry_free(void *p) +{ + rend_cache_entry_free(p); +} + /** Free all storage held by the service descriptor cache. */ void rend_cache_free_all(void) { - if (rend_cache) - strmap_free(rend_cache, _rend_cache_entry_free); - if (rend_cache_v2_dir) - digestmap_free(rend_cache_v2_dir, _rend_cache_entry_free); + strmap_free(rend_cache, _rend_cache_entry_free); + digestmap_free(rend_cache_v2_dir, _rend_cache_entry_free); rend_cache = NULL; rend_cache_v2_dir = NULL; } @@ -864,13 +827,23 @@ rend_cache_clean(void) ent = (rend_cache_entry_t*)val; if (ent->parsed->timestamp < cutoff) { iter = strmap_iter_next_rmv(rend_cache, iter); - _rend_cache_entry_free(ent); + rend_cache_entry_free(ent); } else { iter = strmap_iter_next(rend_cache, iter); } } } +/** Remove ALL entries from the rendezvous service descriptor cache. + */ +void +rend_cache_purge(void) +{ + if (rend_cache) + strmap_free(rend_cache, _rend_cache_entry_free); + rend_cache = strmap_new(); +} + /** Remove all old v2 descriptors and those for which this hidden service * directory is not responsible for any more. */ void @@ -890,9 +863,9 @@ rend_cache_clean_v2_descs_as_dir(void) char key_base32[REND_DESC_ID_V2_LEN_BASE32 + 1]; base32_encode(key_base32, sizeof(key_base32), key, DIGEST_LEN); log_info(LD_REND, "Removing descriptor with ID '%s' from cache", - safe_str(key_base32)); + safe_str_client(key_base32)); iter = digestmap_iter_next_rmv(rend_cache_v2_dir, iter); - _rend_cache_entry_free(ent); + rend_cache_entry_free(ent); } else { iter = digestmap_iter_next(rend_cache_v2_dir, iter); } @@ -968,6 +941,11 @@ rend_cache_lookup_entry(const char *query, int version, rend_cache_entry_t **e) } if (!*e) return 0; + tor_assert((*e)->parsed && (*e)->parsed->intro_nodes); + /* XXX023 hack for now, to return "not found" if there are no intro + * points remaining. See bug 997. */ + if (! rend_client_any_intro_points_usable(*e)) + return 0; return 1; } @@ -1046,7 +1024,6 @@ rend_cache_store(const char *desc, size_t desc_len, int published) char query[REND_SERVICE_ID_LEN_BASE32+1]; char key[REND_SERVICE_ID_LEN_BASE32+2]; /* 0<query>\0 */ time_t now; - or_options_t *options = get_options(); tor_assert(rend_cache); parsed = rend_parse_service_descriptor(desc,desc_len); if (!parsed) { @@ -1061,13 +1038,15 @@ rend_cache_store(const char *desc, size_t desc_len, int published) now = time(NULL); if (parsed->timestamp < now-REND_CACHE_MAX_AGE-REND_CACHE_MAX_SKEW) { log_fn(LOG_PROTOCOL_WARN, LD_REND, - "Service descriptor %s is too old.", safe_str(query)); + "Service descriptor %s is too old.", + safe_str_client(query)); rend_service_descriptor_free(parsed); return -2; } if (parsed->timestamp > now+REND_CACHE_MAX_SKEW) { log_fn(LOG_PROTOCOL_WARN, LD_REND, - "Service descriptor %s is too far in the future.", safe_str(query)); + "Service descriptor %s is too far in the future.", + safe_str_client(query)); rend_service_descriptor_free(parsed); return -2; } @@ -1075,25 +1054,22 @@ rend_cache_store(const char *desc, size_t desc_len, int published) tor_snprintf(key, sizeof(key), "2%s", query); if (!published && strmap_get_lc(rend_cache, key)) { log_info(LD_REND, "We already have a v2 descriptor for service %s.", - safe_str(query)); + safe_str_client(query)); rend_service_descriptor_free(parsed); return -1; } - /* report novel publication to statistics */ - if (published && options->HSAuthorityRecordStats) { - hs_usage_note_publish_total(query, now); - } tor_snprintf(key, sizeof(key), "0%s", query); e = (rend_cache_entry_t*) strmap_get_lc(rend_cache, key); if (e && e->parsed->timestamp > parsed->timestamp) { log_info(LD_REND,"We already have a newer service descriptor %s with the " - "same ID and version.", safe_str(query)); + "same ID and version.", + safe_str_client(query)); rend_service_descriptor_free(parsed); return 0; } if (e && e->len == desc_len && !memcmp(desc,e->desc,desc_len)) { log_info(LD_REND,"We already have this service descriptor %s.", - safe_str(query)); + safe_str_client(query)); e->received = time(NULL); rend_service_descriptor_free(parsed); return 0; @@ -1101,10 +1077,6 @@ rend_cache_store(const char *desc, size_t desc_len, int published) if (!e) { e = tor_malloc_zero(sizeof(rend_cache_entry_t)); strmap_set_lc(rend_cache, key, e); - /* report novel publication to statistics */ - if (published && options->HSAuthorityRecordStats) { - hs_usage_note_publish_novel(query, now); - } } else { rend_service_descriptor_free(e->parsed); tor_free(e->desc); @@ -1116,7 +1088,7 @@ rend_cache_store(const char *desc, size_t desc_len, int published) memcpy(e->desc, desc, desc_len); log_debug(LD_REND,"Successfully stored rend desc '%s', len %d.", - safe_str(query), (int)desc_len); + safe_str_client(query), (int)desc_len); return 1; } @@ -1167,7 +1139,7 @@ rend_cache_store_v2_desc_as_dir(const char *desc) if (!hid_serv_responsible_for_desc_id(desc_id)) { log_info(LD_REND, "Service descriptor with desc ID %s is not in " "interval that we are responsible for.", - safe_str(desc_id_base32)); + safe_str_client(desc_id_base32)); goto skip; } /* Is descriptor too old? */ @@ -1296,7 +1268,8 @@ rend_cache_store_v2_desc_as_client(const char *desc, /* Decode/decrypt introduction points. */ if (intro_content) { if (rend_query->auth_type != REND_NO_AUTH && - rend_query->descriptor_cookie) { + !tor_mem_is_zero(rend_query->descriptor_cookie, + sizeof(rend_query->descriptor_cookie))) { char *ipos_decrypted = NULL; size_t ipos_decrypted_size; if (rend_decrypt_introduction_points(&ipos_decrypted, @@ -1331,14 +1304,14 @@ rend_cache_store_v2_desc_as_client(const char *desc, /* Is descriptor too old? */ if (parsed->timestamp < now - REND_CACHE_MAX_AGE-REND_CACHE_MAX_SKEW) { log_warn(LD_REND, "Service descriptor with service ID %s is too old.", - safe_str(service_id)); + safe_str_client(service_id)); retval = -2; goto err; } /* Is descriptor too far in the future? */ if (parsed->timestamp > now + REND_CACHE_MAX_SKEW) { log_warn(LD_REND, "Service descriptor with service ID %s is too far in " - "the future.", safe_str(service_id)); + "the future.", safe_str_client(service_id)); retval = -2; goto err; } @@ -1346,7 +1319,7 @@ rend_cache_store_v2_desc_as_client(const char *desc, tor_snprintf(key, sizeof(key), "0%s", service_id); if (strmap_get_lc(rend_cache, key)) { log_info(LD_REND, "We already have a v0 descriptor for service ID %s.", - safe_str(service_id)); + safe_str_client(service_id)); retval = -1; goto err; } @@ -1356,14 +1329,14 @@ rend_cache_store_v2_desc_as_client(const char *desc, if (e && e->parsed->timestamp > parsed->timestamp) { log_info(LD_REND, "We already have a newer service descriptor for " "service ID %s with the same desc ID and version.", - safe_str(service_id)); + safe_str_client(service_id)); retval = 0; goto err; } /* Do we already have this descriptor? */ if (e && !strcmp(desc, e->desc)) { log_info(LD_REND,"We already have this service descriptor %s.", - safe_str(service_id)); + safe_str_client(service_id)); e->received = time(NULL); retval = 0; goto err; @@ -1381,12 +1354,11 @@ rend_cache_store_v2_desc_as_client(const char *desc, strlcpy(e->desc, desc, encoded_size + 1); e->len = encoded_size; log_debug(LD_REND,"Successfully stored rend desc '%s', len %d.", - safe_str(service_id), (int)encoded_size); + safe_str_client(service_id), (int)encoded_size); return 1; err: - if (parsed) - rend_service_descriptor_free(parsed); + rend_service_descriptor_free(parsed); tor_free(intro_content); return retval; } diff --git a/src/or/rendcommon.h b/src/or/rendcommon.h new file mode 100644 index 0000000000..44b5227cf5 --- /dev/null +++ b/src/or/rendcommon.h @@ -0,0 +1,66 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file rendcommon.h + * \brief Header file for rendcommon.c. + **/ + +#ifndef _TOR_RENDCOMMON_H +#define _TOR_RENDCOMMON_H + +/** Free all storage associated with <b>data</b> */ +static INLINE void +rend_data_free(rend_data_t *data) +{ + tor_free(data); +} + +int rend_cmp_service_ids(const char *one, const char *two); + +void rend_process_relay_cell(circuit_t *circ, const crypt_path_t *layer_hint, + int command, size_t length, + const uint8_t *payload); + +void rend_service_descriptor_free(rend_service_descriptor_t *desc); +rend_service_descriptor_t *rend_parse_service_descriptor(const char *str, + size_t len); +int rend_get_service_id(crypto_pk_env_t *pk, char *out); +void rend_encoded_v2_service_descriptor_free( + rend_encoded_v2_service_descriptor_t *desc); +void rend_intro_point_free(rend_intro_point_t *intro); + +void rend_cache_init(void); +void rend_cache_clean(void); +void rend_cache_clean_v2_descs_as_dir(void); +void rend_cache_purge(void); +void rend_cache_free_all(void); +int rend_valid_service_id(const char *query); +int rend_cache_lookup_desc(const char *query, int version, const char **desc, + size_t *desc_len); +int rend_cache_lookup_entry(const char *query, int version, + rend_cache_entry_t **entry_out); +int rend_cache_lookup_v2_desc_as_dir(const char *query, const char **desc); +int rend_cache_store(const char *desc, size_t desc_len, int published); +int rend_cache_store_v2_desc_as_client(const char *desc, + const rend_data_t *rend_query); +int rend_cache_store_v2_desc_as_dir(const char *desc); +int rend_cache_size(void); +int rend_encode_v2_descriptors(smartlist_t *descs_out, + rend_service_descriptor_t *desc, time_t now, + uint8_t period, rend_auth_type_t auth_type, + crypto_pk_env_t *client_key, + smartlist_t *client_cookies); +int rend_compute_v2_desc_id(char *desc_id_out, const char *service_id, + const char *descriptor_cookie, + time_t now, uint8_t replica); +int rend_id_is_in_interval(const char *a, const char *b, const char *c); +void rend_get_descriptor_id_bytes(char *descriptor_id_out, + const char *service_id, + const char *secret_id_part); + +#endif + diff --git a/src/or/rendmid.c b/src/or/rendmid.c index 064f1be2e8..5e2dd986ba 100644 --- a/src/or/rendmid.c +++ b/src/or/rendmid.c @@ -8,6 +8,11 @@ **/ #include "or.h" +#include "circuitlist.h" +#include "config.h" +#include "relay.h" +#include "rendmid.h" +#include "rephist.h" /** Respond to an ESTABLISH_INTRO cell by checking the signed data and * setting the circuit's purpose and service pk digest. diff --git a/src/or/rendmid.h b/src/or/rendmid.h new file mode 100644 index 0000000000..5ed87fd2b1 --- /dev/null +++ b/src/or/rendmid.h @@ -0,0 +1,25 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file rendmid.h + * \brief Header file for rendmid.c. + **/ + +#ifndef _TOR_RENDMID_H +#define _TOR_RENDMID_H + +int rend_mid_establish_intro(or_circuit_t *circ, const uint8_t *request, + size_t request_len); +int rend_mid_introduce(or_circuit_t *circ, const uint8_t *request, + size_t request_len); +int rend_mid_establish_rendezvous(or_circuit_t *circ, const uint8_t *request, + size_t request_len); +int rend_mid_rendezvous(or_circuit_t *circ, const uint8_t *request, + size_t request_len); + +#endif + diff --git a/src/or/rendservice.c b/src/or/rendservice.c index 33e8d3e7e9..cd8f9eabeb 100644 --- a/src/or/rendservice.c +++ b/src/or/rendservice.c @@ -8,10 +8,23 @@ **/ #include "or.h" +#include "circuitbuild.h" +#include "circuitlist.h" +#include "circuituse.h" +#include "config.h" +#include "directory.h" +#include "networkstatus.h" +#include "rendclient.h" +#include "rendcommon.h" +#include "rendservice.h" +#include "router.h" +#include "relay.h" +#include "rephist.h" +#include "routerlist.h" +#include "routerparse.h" static origin_circuit_t *find_intro_circuit(rend_intro_point_t *intro, - const char *pk_digest, - int desc_version); + const char *pk_digest); /** Represents the mapping from a virtual port of a rendezvous service to * a real port on some IP. @@ -42,8 +55,6 @@ typedef struct rend_service_t { /* Fields specified in config file */ char *directory; /**< where in the filesystem it stores it */ smartlist_t *ports; /**< List of rend_service_port_config_t */ - int descriptor_version; /**< Rendezvous descriptor version that will be - * published. */ rend_auth_type_t auth_type; /**< Client authorization type or 0 if no client * authorization is performed. */ smartlist_t *clients; /**< List of rend_authorized_client_t's of @@ -58,7 +69,7 @@ typedef struct rend_service_t { * or are trying to establish. */ time_t intro_period_started; /**< Start of the current period to build * introduction points. */ - int n_intro_circuits_launched; /**< count of intro circuits we have + int n_intro_circuits_launched; /**< Count of intro circuits we have * established in this period. */ rend_service_descriptor_t *desc; /**< Current hidden service descriptor. */ time_t desc_is_dirty; /**< Time at which changes to the hidden service @@ -90,7 +101,8 @@ num_rend_services(void) static void rend_authorized_client_free(rend_authorized_client_t *client) { - if (!client) return; + if (!client) + return; if (client->client_key) crypto_free_pk_env(client->client_key); tor_free(client->client_name); @@ -109,7 +121,9 @@ rend_authorized_client_strmap_item_free(void *authorized_client) static void rend_service_free(rend_service_t *service) { - if (!service) return; + if (!service) + return; + tor_free(service->directory); SMARTLIST_FOREACH(service->ports, void*, p, tor_free(p)); smartlist_free(service->ports); @@ -120,15 +134,14 @@ rend_service_free(rend_service_t *service) rend_intro_point_free(intro);); smartlist_free(service->intro_nodes); } - if (service->desc) - rend_service_descriptor_free(service->desc); + + rend_service_descriptor_free(service->desc); if (service->clients) { SMARTLIST_FOREACH(service->clients, rend_authorized_client_t *, c, rend_authorized_client_free(c);); smartlist_free(service->clients); } - if (service->accepted_intros) - digestmap_free(service->accepted_intros, _tor_free); + digestmap_free(service->accepted_intros, _tor_free); tor_free(service); } @@ -137,9 +150,9 @@ rend_service_free(rend_service_t *service) void rend_service_free_all(void) { - if (!rend_service_list) { + if (!rend_service_list) return; - } + SMARTLIST_FOREACH(rend_service_list, rend_service_t*, ptr, rend_service_free(ptr)); smartlist_free(rend_service_list); @@ -156,36 +169,6 @@ rend_add_service(rend_service_t *service) service->intro_nodes = smartlist_create(); - /* If the service is configured to publish unversioned (v0) and versioned - * descriptors (v2 or higher), split it up into two separate services - * (unless it is configured to perform client authorization). */ - if (service->descriptor_version == -1) { - if (service->auth_type == REND_NO_AUTH) { - rend_service_t *v0_service = tor_malloc_zero(sizeof(rend_service_t)); - v0_service->directory = tor_strdup(service->directory); - v0_service->ports = smartlist_create(); - SMARTLIST_FOREACH(service->ports, rend_service_port_config_t *, p, { - rend_service_port_config_t *copy = - tor_malloc_zero(sizeof(rend_service_port_config_t)); - memcpy(copy, p, sizeof(rend_service_port_config_t)); - smartlist_add(v0_service->ports, copy); - }); - v0_service->intro_period_started = service->intro_period_started; - v0_service->descriptor_version = 0; /* Unversioned descriptor. */ - v0_service->auth_type = REND_NO_AUTH; - rend_add_service(v0_service); - } - - service->descriptor_version = 2; /* Versioned descriptor. */ - } - - if (service->auth_type != REND_NO_AUTH && !service->descriptor_version) { - log_warn(LD_CONFIG, "Hidden service with client authorization and " - "version 0 descriptors configured; ignoring."); - rend_service_free(service); - return; - } - if (service->auth_type != REND_NO_AUTH && smartlist_len(service->clients) == 0) { log_warn(LD_CONFIG, "Hidden service with client authorization but no " @@ -297,7 +280,7 @@ rend_config_services(or_options_t *options, int validate_only) for (line = options->RendConfigLines; line; line = line->next) { if (!strcasecmp(line->key, "HiddenServiceDir")) { - if (service) { + if (service) { /* register the one we just finished parsing */ if (validate_only) rend_service_free(service); else @@ -307,7 +290,6 @@ rend_config_services(or_options_t *options, int validate_only) service->directory = tor_strdup(line->value); service->ports = smartlist_create(); service->intro_period_started = time(NULL); - service->descriptor_version = -1; /**< All descriptor versions. */ continue; } if (!service) { @@ -433,35 +415,13 @@ rend_config_services(or_options_t *options, int validate_only) return -1; } } else { - smartlist_t *versions; - char *version_str; - int i, version, ver_ok=1, versions_bitmask = 0; tor_assert(!strcasecmp(line->key, "HiddenServiceVersion")); - versions = smartlist_create(); - smartlist_split_string(versions, line->value, ",", - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); - for (i = 0; i < smartlist_len(versions); i++) { - version_str = smartlist_get(versions, i); - if (strlen(version_str) != 1 || strspn(version_str, "02") != 1) { - log_warn(LD_CONFIG, - "HiddenServiceVersion can only be 0 and/or 2."); - SMARTLIST_FOREACH(versions, char *, cp, tor_free(cp)); - smartlist_free(versions); - rend_service_free(service); - return -1; - } - version = (int)tor_parse_long(version_str, 10, 0, INT_MAX, &ver_ok, - NULL); - if (!ver_ok) - continue; - versions_bitmask |= 1 << version; + if (strcmp(line->value, "2")) { + log_warn(LD_CONFIG, + "The only supported HiddenServiceVersion is 2."); + rend_service_free(service); + return -1; } - /* If exactly one version is set, change descriptor_version to that - * value; otherwise leave it at -1. */ - if (versions_bitmask == 1 << 0) service->descriptor_version = 0; - if (versions_bitmask == 1 << 2) service->descriptor_version = 2; - SMARTLIST_FOREACH(versions, char *, cp, tor_free(cp)); - smartlist_free(versions); } } if (service) { @@ -483,8 +443,7 @@ rend_config_services(or_options_t *options, int validate_only) * probably ok? */ SMARTLIST_FOREACH(rend_service_list, rend_service_t *, new, { SMARTLIST_FOREACH(old_service_list, rend_service_t *, old, { - if (!strcmp(old->directory, new->directory) && - old->descriptor_version == new->descriptor_version) { + if (!strcmp(old->directory, new->directory)) { smartlist_add_all(new->intro_nodes, old->intro_nodes); smartlist_clear(old->intro_nodes); smartlist_add(surviving_services, old); @@ -507,18 +466,16 @@ rend_config_services(or_options_t *options, int validate_only) tor_assert(oc->rend_data); SMARTLIST_FOREACH(surviving_services, rend_service_t *, ptr, { if (!memcmp(ptr->pk_digest, oc->rend_data->rend_pk_digest, - DIGEST_LEN) && - ptr->descriptor_version == oc->rend_data->rend_desc_version) { + DIGEST_LEN)) { keep_it = 1; break; } }); if (keep_it) continue; - log_info(LD_REND, "Closing intro point %s for service %s version %d.", - safe_str(oc->build_state->chosen_exit->nickname), - oc->rend_data->onion_address, - oc->rend_data->rend_desc_version); + log_info(LD_REND, "Closing intro point %s for service %s.", + safe_str_client(oc->build_state->chosen_exit->nickname), + oc->rend_data->onion_address); circuit_mark_for_close(circ, END_CIRC_REASON_FINISHED); /* XXXX Is there another reason we should use here? */ } @@ -541,14 +498,13 @@ rend_service_update_descriptor(rend_service_t *service) rend_service_descriptor_t *d; origin_circuit_t *circ; int i; - if (service->desc) { - rend_service_descriptor_free(service->desc); - service->desc = NULL; - } + + rend_service_descriptor_free(service->desc); + service->desc = NULL; + d = service->desc = tor_malloc_zero(sizeof(rend_service_descriptor_t)); d->pk = crypto_pk_dup_key(service->private_key); d->timestamp = time(NULL); - d->version = service->descriptor_version; d->intro_nodes = smartlist_create(); /* Support intro protocols 2 and 3. */ d->protocols = (1 << 2) + (1 << 3); @@ -556,7 +512,7 @@ rend_service_update_descriptor(rend_service_t *service) for (i = 0; i < smartlist_len(service->intro_nodes); ++i) { rend_intro_point_t *intro_svc = smartlist_get(service->intro_nodes, i); rend_intro_point_t *intro_desc; - circ = find_intro_circuit(intro_svc, service->pk_digest, d->version); + circ = find_intro_circuit(intro_svc, service->pk_digest); if (!circ || circ->_base.purpose != CIRCUIT_PURPOSE_S_INTRO) continue; @@ -797,17 +753,15 @@ rend_service_load_keys(void) return r; } -/** Return the service whose public key has a digest of <b>digest</b> and - * which publishes the given descriptor <b>version</b>. Return NULL if no - * such service exists. +/** Return the service whose public key has a digest of <b>digest</b>, or + * NULL if no such service exists. */ static rend_service_t * -rend_service_get_by_pk_digest_and_version(const char* digest, - uint8_t version) +rend_service_get_by_pk_digest(const char* digest) { SMARTLIST_FOREACH(rend_service_list, rend_service_t*, s, - if (!memcmp(s->pk_digest,digest,DIGEST_LEN) && - s->descriptor_version == version) return s); + if (!memcmp(s->pk_digest,digest,DIGEST_LEN)) + return s); return NULL; } @@ -894,6 +848,7 @@ clean_accepted_intros(rend_service_t *service, time_t now) /** Respond to an INTRODUCE2 cell by launching a circuit to the chosen * rendezvous point. */ + /* XXX022 this function sure could use some organizing. -RD */ int rend_service_introduce(origin_circuit_t *circuit, const uint8_t *request, size_t request_len) @@ -921,6 +876,8 @@ rend_service_introduce(origin_circuit_t *circuit, const uint8_t *request, time_t now = time(NULL); char diffie_hellman_hash[DIGEST_LEN]; time_t *access_time; + or_options_t *options = get_options(); + tor_assert(circuit->rend_data); base32_encode(serviceid, REND_SERVICE_ID_LEN_BASE32+1, @@ -944,21 +901,16 @@ rend_service_introduce(origin_circuit_t *circuit, const uint8_t *request, } /* look up service depending on circuit. */ - service = rend_service_get_by_pk_digest_and_version( - circuit->rend_data->rend_pk_digest, - circuit->rend_data->rend_desc_version); + service = rend_service_get_by_pk_digest( + circuit->rend_data->rend_pk_digest); if (!service) { log_warn(LD_REND, "Got an INTRODUCE2 cell for an unrecognized service %s.", escaped(serviceid)); return -1; } - /* if descriptor version is 2, use intro key instead of service key. */ - if (circuit->rend_data->rend_desc_version == 0) { - intro_key = service->private_key; - } else { - intro_key = circuit->intro_key; - } + /* use intro key instead of service key. */ + intro_key = circuit->intro_key; /* first DIGEST_LEN bytes of request is intro or service pk digest */ crypto_pk_get_digest(intro_key, intro_key_digest); @@ -989,7 +941,7 @@ rend_service_introduce(origin_circuit_t *circuit, const uint8_t *request, len = r; if (*buf == 3) { /* Version 3 INTRODUCE2 cell. */ - time_t ts = 0, now = time(NULL); + time_t ts = 0; v3_shift = 1; auth_type = buf[1]; switch (auth_type) { @@ -1083,7 +1035,7 @@ rend_service_introduce(origin_circuit_t *circuit, const uint8_t *request, router = router_get_by_nickname(rp_nickname, 0); if (!router) { log_info(LD_REND, "Couldn't find router %s named in introduce2 cell.", - escaped_safe_str(rp_nickname)); + escaped_safe_str_client(rp_nickname)); /* XXXX Add a no-such-router reason? */ reason = END_CIRC_REASON_TORPROTOCOL; goto err; @@ -1098,6 +1050,15 @@ rend_service_introduce(origin_circuit_t *circuit, const uint8_t *request, goto err; } + /* Check if we'd refuse to talk to this router */ + if (options->ExcludeNodes && options->StrictNodes && + routerset_contains_extendinfo(options->ExcludeNodes, extend_info)) { + log_warn(LD_REND, "Client asked to rendezvous at a relay that we " + "exclude, and StrictNodes is set. Refusing service."); + reason = END_CIRC_REASON_INTERNAL; /* XXX might leak why we refused */ + goto err; + } + r_cookie = ptr; base16_encode(hexcookie,9,r_cookie,4); @@ -1158,7 +1119,8 @@ rend_service_introduce(origin_circuit_t *circuit, const uint8_t *request, reason = END_CIRC_REASON_INTERNAL; goto err; } - if (crypto_dh_compute_secret(dh, ptr+REND_COOKIE_LEN, DH_KEY_LEN, keys, + if (crypto_dh_compute_secret(LOG_PROTOCOL_WARN, dh, ptr+REND_COOKIE_LEN, + DH_KEY_LEN, keys, DIGEST_LEN+CPATH_KEY_MATERIAL_LEN)<0) { log_warn(LD_BUG, "Internal error: couldn't complete DH handshake"); reason = END_CIRC_REASON_INTERNAL; @@ -1168,7 +1130,7 @@ rend_service_introduce(origin_circuit_t *circuit, const uint8_t *request, circ_needs_uptime = rend_service_requires_uptime(service); /* help predict this next time */ - rep_hist_note_used_internal(time(NULL), circ_needs_uptime, 1); + rep_hist_note_used_internal(now, circ_needs_uptime, 1); /* Launch a circuit to alice's chosen rendezvous point. */ @@ -1184,14 +1146,16 @@ rend_service_introduce(origin_circuit_t *circuit, const uint8_t *request, if (!launched) { /* give up */ log_warn(LD_REND, "Giving up launching first hop of circuit to rendezvous " "point %s for service %s.", - escaped_safe_str(extend_info->nickname), serviceid); + escaped_safe_str_client(extend_info->nickname), + serviceid); reason = END_CIRC_REASON_CONNECTFAILED; goto err; } log_info(LD_REND, "Accepted intro; launching circuit to %s " "(cookie %s) for service %s.", - escaped_safe_str(extend_info->nickname), hexcookie, serviceid); + escaped_safe_str_client(extend_info->nickname), + hexcookie, serviceid); tor_assert(launched->build_state); /* Fill in the circuit's state. */ launched->rend_data = tor_malloc_zero(sizeof(rend_data_t)); @@ -1201,11 +1165,10 @@ rend_service_introduce(origin_circuit_t *circuit, const uint8_t *request, memcpy(launched->rend_data->rend_cookie, r_cookie, REND_COOKIE_LEN); strlcpy(launched->rend_data->onion_address, service->service_id, sizeof(launched->rend_data->onion_address)); - launched->rend_data->rend_desc_version = service->descriptor_version; launched->build_state->pending_final_cpath = cpath = tor_malloc_zero(sizeof(crypt_path_t)); cpath->magic = CRYPT_PATH_MAGIC; - launched->build_state->expiry_time = time(NULL) + MAX_REND_TIMEOUT; + launched->build_state->expiry_time = now + MAX_REND_TIMEOUT; cpath->dh_handshake_state = dh; dh = NULL; @@ -1289,7 +1252,7 @@ rend_service_launch_establish_intro(rend_service_t *service, log_info(LD_REND, "Launching circuit to introduction point %s for service %s", - escaped_safe_str(intro->extend_info->nickname), + escaped_safe_str_client(intro->extend_info->nickname), service->service_id); rep_hist_note_used_internal(time(NULL), 1, 0); @@ -1302,7 +1265,7 @@ rend_service_launch_establish_intro(rend_service_t *service, if (!launched) { log_info(LD_REND, "Can't launch circuit to establish introduction at %s.", - escaped_safe_str(intro->extend_info->nickname)); + escaped_safe_str_client(intro->extend_info->nickname)); return -1; } @@ -1325,18 +1288,16 @@ rend_service_launch_establish_intro(rend_service_t *service, strlcpy(launched->rend_data->onion_address, service->service_id, sizeof(launched->rend_data->onion_address)); memcpy(launched->rend_data->rend_pk_digest, service->pk_digest, DIGEST_LEN); - launched->rend_data->rend_desc_version = service->descriptor_version; - if (service->descriptor_version == 2) - launched->intro_key = crypto_pk_dup_key(intro->intro_key); + launched->intro_key = crypto_pk_dup_key(intro->intro_key); if (launched->_base.state == CIRCUIT_STATE_OPEN) rend_service_intro_has_opened(launched); return 0; } /** Return the number of introduction points that are or have been - * established for the given service address and rendezvous version. */ + * established for the given service address in <b>query</b>. */ static int -count_established_intro_points(const char *query, int rend_version) +count_established_intro_points(const char *query) { int num_ipos = 0; circuit_t *circ; @@ -1347,7 +1308,6 @@ count_established_intro_points(const char *query, int rend_version) circ->purpose == CIRCUIT_PURPOSE_S_INTRO)) { origin_circuit_t *oc = TO_ORIGIN_CIRCUIT(circ); if (oc->rend_data && - oc->rend_data->rend_desc_version == rend_version && !rend_cmp_service_ids(query, oc->rend_data->onion_address)) num_ipos++; } @@ -1377,9 +1337,8 @@ rend_service_intro_has_opened(origin_circuit_t *circuit) base32_encode(serviceid, REND_SERVICE_ID_LEN_BASE32+1, circuit->rend_data->rend_pk_digest, REND_SERVICE_ID_LEN); - service = rend_service_get_by_pk_digest_and_version( - circuit->rend_data->rend_pk_digest, - circuit->rend_data->rend_desc_version); + service = rend_service_get_by_pk_digest( + circuit->rend_data->rend_pk_digest); if (!service) { log_warn(LD_REND, "Unrecognized service ID %s on introduction circuit %d.", serviceid, circuit->_base.n_circ_id); @@ -1388,28 +1347,34 @@ rend_service_intro_has_opened(origin_circuit_t *circuit) } /* If we already have enough introduction circuits for this service, - * redefine this one as a general circuit. */ - if (count_established_intro_points(serviceid, - circuit->rend_data->rend_desc_version) > NUM_INTRO_POINTS) { - log_info(LD_CIRC|LD_REND, "We have just finished an introduction " - "circuit, but we already have enough. Redefining purpose to " - "general."); - TO_CIRCUIT(circuit)->purpose = CIRCUIT_PURPOSE_C_GENERAL; - circuit_has_opened(circuit); - return; + * redefine this one as a general circuit or close it, depending. */ + if (count_established_intro_points(serviceid) > NUM_INTRO_POINTS) { + or_options_t *options = get_options(); + if (options->ExcludeNodes) { + /* XXXX in some future version, we can test whether the transition is + allowed or not given the actual nodes in the circuit. But for now, + this case, we might as well close the thing. */ + log_info(LD_CIRC|LD_REND, "We have just finished an introduction " + "circuit, but we already have enough. Closing it."); + circuit_mark_for_close(TO_CIRCUIT(circuit), END_CIRC_REASON_NONE); + return; + } else { + tor_assert(circuit->build_state->is_internal); + log_info(LD_CIRC|LD_REND, "We have just finished an introduction " + "circuit, but we already have enough. Redefining purpose to " + "general; leaving as internal."); + TO_CIRCUIT(circuit)->purpose = CIRCUIT_PURPOSE_C_GENERAL; + circuit_has_opened(circuit); + return; + } } log_info(LD_REND, "Established circuit %d as introduction point for service %s", circuit->_base.n_circ_id, serviceid); - /* If the introduction point will not be used in an unversioned - * descriptor, use the intro key instead of the service key in - * ESTABLISH_INTRO. */ - if (service->descriptor_version == 0) - intro_key = service->private_key; - else - intro_key = circuit->intro_key; + /* Use the intro key instead of the service key in ESTABLISH_INTRO. */ + intro_key = circuit->intro_key; /* Build the payload for a RELAY_ESTABLISH_INTRO cell. */ r = crypto_pk_asn1_encode(intro_key, buf+2, RELAY_PAYLOAD_SIZE-2); @@ -1453,7 +1418,7 @@ rend_service_intro_has_opened(origin_circuit_t *circuit) /** Called when we get an INTRO_ESTABLISHED cell; mark the circuit as a * live introduction point, and note that the service descriptor is - * now out-of-date.*/ + * now out-of-date. */ int rend_service_intro_established(origin_circuit_t *circuit, const uint8_t *request, @@ -1470,9 +1435,8 @@ rend_service_intro_established(origin_circuit_t *circuit, goto err; } tor_assert(circuit->rend_data); - service = rend_service_get_by_pk_digest_and_version( - circuit->rend_data->rend_pk_digest, - circuit->rend_data->rend_desc_version); + service = rend_service_get_by_pk_digest( + circuit->rend_data->rend_pk_digest); if (!service) { log_warn(LD_REND, "Unknown service on introduction circuit %d.", circuit->_base.n_circ_id); @@ -1522,9 +1486,8 @@ rend_service_rendezvous_has_opened(origin_circuit_t *circuit) "cookie %s for service %s", circuit->_base.n_circ_id, hexcookie, serviceid); - service = rend_service_get_by_pk_digest_and_version( - circuit->rend_data->rend_pk_digest, - circuit->rend_data->rend_desc_version); + service = rend_service_get_by_pk_digest( + circuit->rend_data->rend_pk_digest); if (!service) { log_warn(LD_GENERAL, "Internal error: unrecognized service ID on " "introduction circuit."); @@ -1580,13 +1543,12 @@ rend_service_rendezvous_has_opened(origin_circuit_t *circuit) */ /** Return the (possibly non-open) introduction circuit ending at - * <b>intro</b> for the service whose public key is <b>pk_digest</b> and - * which publishes descriptor of version <b>desc_version</b>. Return - * NULL if no such service is found. + * <b>intro</b> for the service whose public key is <b>pk_digest</b>. + * (<b>desc_version</b> is ignored). Return NULL if no such service is + * found. */ static origin_circuit_t * -find_intro_circuit(rend_intro_point_t *intro, const char *pk_digest, - int desc_version) +find_intro_circuit(rend_intro_point_t *intro, const char *pk_digest) { origin_circuit_t *circ = NULL; @@ -1595,8 +1557,7 @@ find_intro_circuit(rend_intro_point_t *intro, const char *pk_digest, CIRCUIT_PURPOSE_S_INTRO))) { if (!memcmp(circ->build_state->chosen_exit->identity_digest, intro->extend_info->identity_digest, DIGEST_LEN) && - circ->rend_data && - circ->rend_data->rend_desc_version == desc_version) { + circ->rend_data) { return circ; } } @@ -1606,8 +1567,7 @@ find_intro_circuit(rend_intro_point_t *intro, const char *pk_digest, CIRCUIT_PURPOSE_S_ESTABLISH_INTRO))) { if (!memcmp(circ->build_state->chosen_exit->identity_digest, intro->extend_info->identity_digest, DIGEST_LEN) && - circ->rend_data && - circ->rend_data->rend_desc_version == desc_version) { + circ->rend_data) { return circ; } } @@ -1640,6 +1600,7 @@ directory_post_to_hs_dir(rend_service_descriptor_t *renddesc, } for (j = 0; j < smartlist_len(responsible_dirs); j++) { char desc_id_base32[REND_DESC_ID_V2_LEN_BASE32 + 1]; + char *hs_dir_ip; hs_dir = smartlist_get(responsible_dirs, j); if (smartlist_digest_isin(renddesc->successful_uploads, hs_dir->identity_digest)) @@ -1661,15 +1622,18 @@ directory_post_to_hs_dir(rend_service_descriptor_t *renddesc, strlen(desc->desc_str), 0); base32_encode(desc_id_base32, sizeof(desc_id_base32), desc->desc_id, DIGEST_LEN); + hs_dir_ip = tor_dup_ip(hs_dir->addr); log_info(LD_REND, "Sending publish request for v2 descriptor for " "service '%s' with descriptor ID '%s' with validity " "of %d seconds to hidden service directory '%s' on " - "port %d.", - safe_str(service_id), - safe_str(desc_id_base32), + "%s:%d.", + safe_str_client(service_id), + safe_str_client(desc_id_base32), seconds_valid, hs_dir->nickname, - hs_dir->dir_port); + hs_dir_ip, + hs_dir->or_port); + tor_free(hs_dir_ip); /* Remember successful upload to this router for next time. */ if (!smartlist_digest_isin(successful_uploads, hs_dir->identity_digest)) smartlist_add(successful_uploads, hs_dir->identity_digest); @@ -1699,9 +1663,8 @@ directory_post_to_hs_dir(rend_service_descriptor_t *renddesc, smartlist_free(successful_uploads); } -/** Encode and sign up-to-date v0 and/or v2 service descriptors for - * <b>service</b>, and upload it/them to all the dirservers/to the - * responsible hidden service directories. +/** Encode and sign an up-to-date service descriptor for <b>service</b>, + * and upload it/them to the responsible hidden service directories. */ static void upload_service_descriptor(rend_service_t *service) @@ -1713,35 +1676,8 @@ upload_service_descriptor(rend_service_t *service) rendpostperiod = get_options()->RendPostPeriod; - /* Upload unversioned (v0) descriptor? */ - if (service->descriptor_version == 0 && - get_options()->PublishHidServDescriptors) { - char *desc; - size_t desc_len; - /* Encode the descriptor. */ - if (rend_encode_service_descriptor(service->desc, - service->private_key, - &desc, &desc_len)<0) { - log_warn(LD_BUG, "Internal error: couldn't encode service descriptor; " - "not uploading."); - return; - } - - /* Post it to the dirservers */ - rend_get_service_id(service->desc->pk, serviceid); - log_info(LD_REND, "Sending publish request for hidden service %s", - serviceid); - directory_post_to_dirservers(DIR_PURPOSE_UPLOAD_RENDDESC, - ROUTER_PURPOSE_GENERAL, - HIDSERV_AUTHORITY, desc, desc_len, 0); - tor_free(desc); - service->next_upload_time = now + rendpostperiod; - uploaded = 1; - } - - /* Upload v2 descriptor? */ - if (service->descriptor_version == 2 && - get_options()->PublishHidServDescriptors) { + /* Upload descriptor? */ + if (get_options()->PublishHidServDescriptors) { networkstatus_t *c = networkstatus_get_latest_consensus(); if (c && smartlist_len(c->routerstatus_list) > 0) { int seconds_valid, i, j, num_descs; @@ -1880,8 +1816,7 @@ rend_services_introduce(void) for (j=0; j < smartlist_len(service->intro_nodes); ++j) { intro = smartlist_get(service->intro_nodes, j); router = router_get_by_digest(intro->extend_info->identity_digest); - if (!router || !find_intro_circuit(intro, service->pk_digest, - service->descriptor_version)) { + if (!router || !find_intro_circuit(intro, service->pk_digest)) { log_info(LD_REND,"Giving up on %s as intro point for %s.", intro->extend_info->nickname, service->service_id); if (service->desc) { @@ -1933,7 +1868,7 @@ rend_services_introduce(void) router_crn_flags_t flags = CRN_NEED_UPTIME; if (get_options()->_AllowInvalid & ALLOW_INVALID_INTRODUCTION) flags |= CRN_ALLOW_INVALID; - router = router_choose_random_node(NULL, intro_routers, + router = router_choose_random_node(intro_routers, options->ExcludeNodes, flags); if (!router) { log_warn(LD_REND, @@ -1945,10 +1880,8 @@ rend_services_introduce(void) smartlist_add(intro_routers, router); intro = tor_malloc_zero(sizeof(rend_intro_point_t)); intro->extend_info = extend_info_from_router(router); - if (service->descriptor_version == 2) { - intro->intro_key = crypto_new_pk_env(); - tor_assert(!crypto_pk_generate_key(intro->intro_key)); - } + intro->intro_key = crypto_new_pk_env(); + tor_assert(!crypto_pk_generate_key(intro->intro_key)); smartlist_add(service->intro_nodes, intro); log_info(LD_REND, "Picked router %s as an intro point for %s.", router->nickname, service->service_id); @@ -2041,8 +1974,7 @@ rend_consider_descriptor_republication(void) for (i=0; i < smartlist_len(rend_service_list); ++i) { service = smartlist_get(rend_service_list, i); - if (service->descriptor_version && service->desc && - !service->desc->all_uploads_performed) { + if (service->desc && !service->desc->all_uploads_performed) { /* If we failed in uploading a descriptor last time, try again *without* * updating the descriptor's contents. */ upload_service_descriptor(service); @@ -2068,10 +2000,9 @@ rend_service_dump_stats(int severity) service->directory); for (j=0; j < smartlist_len(service->intro_nodes); ++j) { intro = smartlist_get(service->intro_nodes, j); - safe_name = safe_str(intro->extend_info->nickname); + safe_name = safe_str_client(intro->extend_info->nickname); - circ = find_intro_circuit(intro, service->pk_digest, - service->descriptor_version); + circ = find_intro_circuit(intro, service->pk_digest); if (!circ) { log(severity, LD_GENERAL, " Intro point %d at %s: no circuit", j, safe_name); @@ -2102,9 +2033,8 @@ rend_service_set_connection_addr_port(edge_connection_t *conn, log_debug(LD_REND,"beginning to hunt for addr/port"); base32_encode(serviceid, REND_SERVICE_ID_LEN_BASE32+1, circ->rend_data->rend_pk_digest, REND_SERVICE_ID_LEN); - service = rend_service_get_by_pk_digest_and_version( - circ->rend_data->rend_pk_digest, - circ->rend_data->rend_desc_version); + service = rend_service_get_by_pk_digest( + circ->rend_data->rend_pk_digest); if (!service) { log_warn(LD_REND, "Couldn't find any service associated with pk %s on " "rendezvous circuit %d; closing.", diff --git a/src/or/rendservice.h b/src/or/rendservice.h new file mode 100644 index 0000000000..70389afe9a --- /dev/null +++ b/src/or/rendservice.h @@ -0,0 +1,37 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file rendservice.h + * \brief Header file for rendservice.c. + **/ + +#ifndef _TOR_RENDSERVICE_H +#define _TOR_RENDSERVICE_H + +int num_rend_services(void); +int rend_config_services(or_options_t *options, int validate_only); +int rend_service_load_keys(void); +void rend_services_introduce(void); +void rend_consider_services_upload(time_t now); +void rend_hsdir_routers_changed(void); +void rend_consider_descriptor_republication(void); + +void rend_service_intro_has_opened(origin_circuit_t *circuit); +int rend_service_intro_established(origin_circuit_t *circuit, + const uint8_t *request, + size_t request_len); +void rend_service_rendezvous_has_opened(origin_circuit_t *circuit); +int rend_service_introduce(origin_circuit_t *circuit, const uint8_t *request, + size_t request_len); +void rend_service_relaunch_rendezvous(origin_circuit_t *oldcirc); +int rend_service_set_connection_addr_port(edge_connection_t *conn, + origin_circuit_t *circ); +void rend_service_dump_stats(int severity); +void rend_service_free_all(void); + +#endif + diff --git a/src/or/rephist.c b/src/or/rephist.c index dee74ed5c5..02db247389 100644 --- a/src/or/rephist.c +++ b/src/or/rephist.c @@ -6,15 +6,22 @@ * \file rephist.c * \brief Basic history and "reputation" functionality to remember * which servers have worked in the past, how much bandwidth we've - * been using, which ports we tend to want, and so on. + * been using, which ports we tend to want, and so on; further, + * exit port statistics and cell statistics. **/ #include "or.h" +#include "circuitlist.h" +#include "circuituse.h" +#include "config.h" +#include "networkstatus.h" +#include "rephist.h" +#include "router.h" +#include "routerlist.h" #include "ht.h" static void bw_arrays_init(void); static void predicted_ports_init(void); -static void hs_usage_init(void); /** Total number of bytes currently allocated in fields used by rephist.c. */ uint64_t rephist_total_alloc=0; @@ -67,6 +74,13 @@ typedef struct or_history_t { /** If nonzero, we have been unable to connect since this time. */ time_t down_since; + /** The address at which we most recently connected to this OR + * successfully. */ + tor_addr_t last_reached_addr; + + /** The port at which we most recently connected to this OR successfully */ + uint16_t last_reached_port; + /* === For MTBF tracking: */ /** Weighted sum total of all times that this router has been online. */ @@ -113,6 +127,7 @@ get_or_history(const char* id) rephist_total_num++; hist->link_history_map = digestmap_new(); hist->since = hist->changed = time(NULL); + tor_addr_make_unspec(&hist->last_reached_addr); digestmap_set(history_map, id, hist); } return hist; @@ -185,7 +200,6 @@ rep_hist_init(void) history_map = digestmap_new(); bw_arrays_init(); predicted_ports_init(); - hs_usage_init(); } /** Helper: note that we are no longer connected to the router with history @@ -284,13 +298,20 @@ rep_hist_note_connection_died(const char* id, time_t when) /** We have just decided that this router with identity digest <b>id</b> is * reachable, meaning we will give it a "Running" flag for the next while. */ void -rep_hist_note_router_reachable(const char *id, time_t when) +rep_hist_note_router_reachable(const char *id, const tor_addr_t *at_addr, + const uint16_t at_port, time_t when) { or_history_t *hist = get_or_history(id); int was_in_run = 1; char tbuf[ISO_TIME_LEN+1]; + int addr_changed, port_changed; tor_assert(hist); + tor_assert((!at_addr && !at_port) || (at_addr && at_port)); + + addr_changed = at_addr && + tor_addr_compare(at_addr, &hist->last_reached_addr, CMP_EXACT) != 0; + port_changed = at_port && at_port != hist->last_reached_port; if (!started_tracking_stability) started_tracking_stability = time(NULL); @@ -310,6 +331,27 @@ rep_hist_note_router_reachable(const char *id, time_t when) down_length = when - hist->start_of_downtime; hist->total_weighted_time += down_length; hist->start_of_downtime = 0; + } else if (addr_changed || port_changed) { + /* If we're reachable, but the address changed, treat this as some + * downtime. */ + int penalty = get_options()->TestingTorNetwork ? 240 : 3600; + networkstatus_t *ns; + + if ((ns = networkstatus_get_latest_consensus())) { + int fresh_interval = (int)(ns->fresh_until - ns->valid_after); + int live_interval = (int)(ns->valid_until - ns->valid_after); + /* on average, a descriptor addr change takes .5 intervals to make it + * into a consensus, and half a liveness period to make it to + * clients. */ + penalty = (int)(fresh_interval + live_interval) / 2; + } + format_local_iso_time(tbuf, hist->start_of_run); + log_info(LD_HIST,"Router %s still seems Running, but its address appears " + "to have changed since the last time it was reachable. I'm " + "going to treat it as having been down for %d seconds", + hex_str(id, DIGEST_LEN), penalty); + rep_hist_note_router_unreachable(id, when-penalty); + rep_hist_note_router_reachable(id, NULL, 0, when); } else { format_local_iso_time(tbuf, hist->start_of_run); if (was_in_run) @@ -319,6 +361,10 @@ rep_hist_note_router_reachable(const char *id, time_t when) log_info(LD_HIST,"Router %s is now Running; it was previously untracked", hex_str(id, DIGEST_LEN)); } + if (at_addr) + tor_addr_copy(&hist->last_reached_addr, at_addr); + if (at_port) + hist->last_reached_port = at_port; } /** We have just decided that this router is unreachable, meaning @@ -339,12 +385,20 @@ rep_hist_note_router_unreachable(const char *id, time_t when) long run_length = when - hist->start_of_run; format_local_iso_time(tbuf, hist->start_of_run); - hist->weighted_run_length += run_length; hist->total_run_weights += 1.0; hist->start_of_run = 0; - hist->weighted_uptime += run_length; - hist->total_weighted_time += run_length; + if (run_length < 0) { + unsigned long penalty = -run_length; +#define SUBTRACT_CLAMPED(var, penalty) \ + do { (var) = (var) < (penalty) ? 0 : (var) - (penalty); } while (0) + SUBTRACT_CLAMPED(hist->weighted_run_length, penalty); + SUBTRACT_CLAMPED(hist->weighted_uptime, penalty); + } else { + hist->weighted_run_length += run_length; + hist->weighted_uptime += run_length; + hist->total_weighted_time += run_length; + } was_running = 1; log_info(LD_HIST, "Router %s is now non-Running: it had previously been " "Running since %s. Its total weighted uptime is %lu/%lu.", @@ -417,7 +471,7 @@ rep_hist_downrate_old_runs(time_t now) static double get_stability(or_history_t *hist, time_t when) { - unsigned long total = hist->weighted_run_length; + long total = hist->weighted_run_length; double total_weights = hist->total_run_weights; if (hist->start_of_run) { @@ -453,8 +507,8 @@ get_total_weighted_time(or_history_t *hist, time_t when) static double get_weighted_fractional_uptime(or_history_t *hist, time_t when) { - unsigned long total = hist->total_weighted_time; - unsigned long up = hist->weighted_uptime; + long total = hist->total_weighted_time; + long up = hist->weighted_uptime; if (hist->start_of_run) { long run_length = (when - hist->start_of_run); @@ -474,6 +528,20 @@ get_weighted_fractional_uptime(or_history_t *hist, time_t when) return ((double) up) / total; } +/** Return how long the router whose identity digest is <b>id</b> has + * been reachable. Return 0 if the router is unknown or currently deemed + * unreachable. */ +long +rep_hist_get_uptime(const char *id, time_t when) +{ + or_history_t *hist = get_or_history(id); + if (!hist) + return 0; + if (!hist->start_of_run || when < hist->start_of_run) + return 0; + return when - hist->start_of_run; +} + /** Return an estimated MTBF for the router whose identity digest is * <b>id</b>. Return 0 if the router is unknown. */ double @@ -519,7 +587,7 @@ rep_hist_get_weighted_time_known(const char *id, time_t when) int rep_hist_have_measured_enough_stability(void) { - /* XXXX021 This doesn't do so well when we change our opinion + /* XXXX022 This doesn't do so well when we change our opinion * as to whether we're tracking router stability. */ return started_tracking_stability < time(NULL) - 4*60*60; } @@ -795,12 +863,12 @@ rep_hist_record_mtbf_data(time_t now, int missing_means_down) static char * rep_hist_format_router_status(or_history_t *hist, time_t now) { - char buf[1024]; char sor_buf[ISO_TIME_LEN+1]; char sod_buf[ISO_TIME_LEN+1]; double wfu; double mtbf; int up = 0, down = 0; + char *cp = NULL; if (hist->start_of_run) { format_iso_time(sor_buf, hist->start_of_run); @@ -813,7 +881,7 @@ rep_hist_format_router_status(or_history_t *hist, time_t now) wfu = get_weighted_fractional_uptime(hist, now); mtbf = get_stability(hist, now); - tor_snprintf(buf, sizeof(buf), + tor_asprintf(&cp, "%s%s%s" "%s%s%s" "wfu %0.3lf\n" @@ -831,8 +899,7 @@ rep_hist_format_router_status(or_history_t *hist, time_t now) hist->weighted_run_length, hist->total_run_weights ); - - return tor_strdup(buf); + return cp; } /** The last stability analysis document that we created, or NULL if we never @@ -925,10 +992,10 @@ find_next_with(smartlist_t *sl, int i, const char *prefix) return -1; } -/** How many bad times has parse_possibly_bad_iso_time parsed? */ +/** How many bad times has parse_possibly_bad_iso_time() parsed? */ static int n_bogus_times = 0; /** Parse the ISO-formatted time in <b>s</b> into *<b>time_out</b>, but - * rounds any pre-1970 date to Jan 1, 1970. */ + * round any pre-1970 date to Jan 1, 1970. */ static int parse_possibly_bad_iso_time(const char *s, time_t *time_out) { @@ -1161,6 +1228,8 @@ rep_hist_load_mtbf_data(time_t now) * totals? */ #define NUM_SECS_ROLLING_MEASURE 10 /** How large are the intervals for which we track and report bandwidth use? */ +/* XXXX Watch out! Before Tor 0.2.2.21-alpha, using any other value here would + * generate an unparseable state file. */ #define NUM_SECS_BW_SUM_INTERVAL (15*60) /** How far in the past do we remember and publish bandwidth use? */ #define NUM_SECS_BW_SUM_IS_VALID (24*60*60) @@ -1219,7 +1288,7 @@ commit_max(bw_array_t *b) b->total_in_period = 0; } -/** Shift the current observation time of 'b' forward by one second. */ +/** Shift the current observation time of <b>b</b> forward by one second. */ static INLINE void advance_obs(bw_array_t *b) { @@ -1249,14 +1318,18 @@ advance_obs(bw_array_t *b) static INLINE void add_obs(bw_array_t *b, time_t when, uint64_t n) { - /* Don't record data in the past. */ - if (when<b->cur_obs_time) - return; + if (when < b->cur_obs_time) + return; /* Don't record data in the past. */ + /* If we're currently adding observations for an earlier second than * 'when', advance b->cur_obs_time and b->cur_obs_idx by an - * appropriate number of seconds, and do all the other housekeeping */ - while (when>b->cur_obs_time) + * appropriate number of seconds, and do all the other housekeeping. */ + while (when > b->cur_obs_time) { + /* Doing this one second at a time is potentially inefficient, if we start + with a state file that is very old. Fortunately, it doesn't seem to + show up in profiles, so we can just ignore it for now. */ advance_obs(b); + } b->obs[b->cur_obs_idx] += n; b->total_in_period += n; @@ -1280,16 +1353,29 @@ bw_array_new(void) static bw_array_t *read_array = NULL; /** Recent history of bandwidth observations for write operations. */ static bw_array_t *write_array = NULL; - -/** Set up read_array and write_array. */ +/** Recent history of bandwidth observations for read operations for the + directory protocol. */ +static bw_array_t *dir_read_array = NULL; +/** Recent history of bandwidth observations for write operations for the + directory protocol. */ +static bw_array_t *dir_write_array = NULL; + +/** Set up [dir-]read_array and [dir-]write_array, freeing them if they + * already exist. */ static void bw_arrays_init(void) { + tor_free(read_array); + tor_free(write_array); + tor_free(dir_read_array); + tor_free(dir_write_array); read_array = bw_array_new(); write_array = bw_array_new(); + dir_read_array = bw_array_new(); + dir_write_array = bw_array_new(); } -/** We read <b>num_bytes</b> more bytes in second <b>when</b>. +/** Remember that we read <b>num_bytes</b> bytes in second <b>when</b>. * * Add num_bytes to the current running total for <b>when</b>. * @@ -1310,7 +1396,7 @@ rep_hist_note_bytes_written(size_t num_bytes, time_t when) add_obs(write_array, when, num_bytes); } -/** We wrote <b>num_bytes</b> more bytes in second <b>when</b>. +/** Remember that we wrote <b>num_bytes</b> bytes in second <b>when</b>. * (like rep_hist_note_bytes_written() above) */ void @@ -1320,6 +1406,24 @@ rep_hist_note_bytes_read(size_t num_bytes, time_t when) add_obs(read_array, when, num_bytes); } +/** Remember that we wrote <b>num_bytes</b> directory bytes in second + * <b>when</b>. (like rep_hist_note_bytes_written() above) + */ +void +rep_hist_note_dir_bytes_written(size_t num_bytes, time_t when) +{ + add_obs(dir_write_array, when, num_bytes); +} + +/** Remember that we read <b>num_bytes</b> directory bytes in second + * <b>when</b>. (like rep_hist_note_bytes_written() above) + */ +void +rep_hist_note_dir_bytes_read(size_t num_bytes, time_t when) +{ + add_obs(dir_read_array, when, num_bytes); +} + /** Helper: Return the largest value in b->maxima. (This is equal to the * most bandwidth used in any NUM_SECS_ROLLING_MEASURE period for the last * NUM_SECS_BW_SUM_IS_VALID seconds.) @@ -1355,14 +1459,14 @@ rep_hist_bandwidth_assess(void) return (int)(U64_TO_DBL(r)/NUM_SECS_ROLLING_MEASURE); } -/** Print the bandwidth history of b (either read_array or write_array) - * into the buffer pointed to by buf. The format is simply comma - * separated numbers, from oldest to newest. +/** Print the bandwidth history of b (either [dir-]read_array or + * [dir-]write_array) into the buffer pointed to by buf. The format is + * simply comma separated numbers, from oldest to newest. * * It returns the number of bytes written. */ static size_t -rep_hist_fill_bandwidth_history(char *buf, size_t len, bw_array_t *b) +rep_hist_fill_bandwidth_history(char *buf, size_t len, const bw_array_t *b) { char *cp = buf; int i, n; @@ -1407,151 +1511,254 @@ rep_hist_fill_bandwidth_history(char *buf, size_t len, bw_array_t *b) } /** Allocate and return lines for representing this server's bandwidth - * history in its descriptor. + * history in its descriptor. We publish these lines in our extra-info + * descriptor. */ char * -rep_hist_get_bandwidth_lines(int for_extrainfo) +rep_hist_get_bandwidth_lines(void) { char *buf, *cp; char t[ISO_TIME_LEN+1]; int r; - bw_array_t *b; + bw_array_t *b = NULL; + const char *desc = NULL; size_t len; - /* opt (read|write)-history yyyy-mm-dd HH:MM:SS (n s) n,n,n,n,n... */ - len = (60+20*NUM_TOTALS)*2; + /* opt [dirreq-](read|write)-history yyyy-mm-dd HH:MM:SS (n s) n,n,n... */ +/* The n,n,n part above. Largest representation of a uint64_t is 20 chars + * long, plus the comma. */ +#define MAX_HIST_VALUE_LEN 21*NUM_TOTALS + len = (67+MAX_HIST_VALUE_LEN)*4; buf = tor_malloc_zero(len); cp = buf; - for (r=0;r<2;++r) { - b = r?read_array:write_array; + for (r=0;r<4;++r) { + char tmp[MAX_HIST_VALUE_LEN]; + size_t slen; + switch (r) { + case 0: + b = write_array; + desc = "write-history"; + break; + case 1: + b = read_array; + desc = "read-history"; + break; + case 2: + b = dir_write_array; + desc = "dirreq-write-history"; + break; + case 3: + b = dir_read_array; + desc = "dirreq-read-history"; + break; + } tor_assert(b); + slen = rep_hist_fill_bandwidth_history(tmp, MAX_HIST_VALUE_LEN, b); + /* If we don't have anything to write, skip to the next entry. */ + if (slen == 0) + continue; format_iso_time(t, b->next_period-NUM_SECS_BW_SUM_INTERVAL); - tor_snprintf(cp, len-(cp-buf), "%s%s %s (%d s) ", - for_extrainfo ? "" : "opt ", - r ? "read-history" : "write-history", t, - NUM_SECS_BW_SUM_INTERVAL); + tor_snprintf(cp, len-(cp-buf), "%s %s (%d s) ", + desc, t, NUM_SECS_BW_SUM_INTERVAL); cp += strlen(cp); - cp += rep_hist_fill_bandwidth_history(cp, len-(cp-buf), b); + strlcat(cp, tmp, len-(cp-buf)); + cp += slen; strlcat(cp, "\n", len-(cp-buf)); ++cp; } return buf; } -/** Update <b>state</b> with the newest bandwidth history. */ +/** Write a single bw_array_t into the Values, Ends, Interval, and Maximum + * entries of an or_state_t. Done before writing out a new state file. */ +static void +rep_hist_update_bwhist_state_section(or_state_t *state, + const bw_array_t *b, + smartlist_t **s_values, + smartlist_t **s_maxima, + time_t *s_begins, + int *s_interval) +{ + char *cp; + int i,j; + uint64_t maxval; + + if (*s_values) { + SMARTLIST_FOREACH(*s_values, char *, val, tor_free(val)); + smartlist_free(*s_values); + } + if (*s_maxima) { + SMARTLIST_FOREACH(*s_maxima, char *, val, tor_free(val)); + smartlist_free(*s_maxima); + } + if (! server_mode(get_options())) { + /* Clients don't need to store bandwidth history persistently; + * force these values to the defaults. */ + /* FFFF we should pull the default out of config.c's state table, + * so we don't have two defaults. */ + if (*s_begins != 0 || *s_interval != 900) { + time_t now = time(NULL); + time_t save_at = get_options()->AvoidDiskWrites ? now+3600 : now+600; + or_state_mark_dirty(state, save_at); + } + *s_begins = 0; + *s_interval = 900; + *s_values = smartlist_create(); + *s_maxima = smartlist_create(); + return; + } + *s_begins = b->next_period; + *s_interval = NUM_SECS_BW_SUM_INTERVAL; + + *s_values = smartlist_create(); + *s_maxima = smartlist_create(); + /* Set i to first position in circular array */ + i = (b->num_maxes_set <= b->next_max_idx) ? 0 : b->next_max_idx; + for (j=0; j < b->num_maxes_set; ++j,++i) { + if (i >= NUM_TOTALS) + i = 0; + tor_asprintf(&cp, U64_FORMAT, U64_PRINTF_ARG(b->totals[i] & ~0x3ff)); + smartlist_add(*s_values, cp); + maxval = b->maxima[i] / NUM_SECS_ROLLING_MEASURE; + tor_asprintf(&cp, U64_FORMAT, U64_PRINTF_ARG(maxval & ~0x3ff)); + smartlist_add(*s_maxima, cp); + } + tor_asprintf(&cp, U64_FORMAT, U64_PRINTF_ARG(b->total_in_period & ~0x3ff)); + smartlist_add(*s_values, cp); + maxval = b->max_total / NUM_SECS_ROLLING_MEASURE; + tor_asprintf(&cp, U64_FORMAT, U64_PRINTF_ARG(maxval & ~0x3ff)); + smartlist_add(*s_maxima, cp); +} + +/** Update <b>state</b> with the newest bandwidth history. Done before + * writing out a new state file. */ void rep_hist_update_state(or_state_t *state) { - int len, r; - char *buf, *cp; - smartlist_t **s_values; - time_t *s_begins; - int *s_interval; - bw_array_t *b; - - len = 20*NUM_TOTALS+1; - buf = tor_malloc_zero(len); +#define UPDATE(arrname,st) \ + rep_hist_update_bwhist_state_section(state,\ + (arrname),\ + &state->BWHistory ## st ## Values, \ + &state->BWHistory ## st ## Maxima, \ + &state->BWHistory ## st ## Ends, \ + &state->BWHistory ## st ## Interval) - for (r=0;r<2;++r) { - b = r?read_array:write_array; - s_begins = r?&state->BWHistoryReadEnds :&state->BWHistoryWriteEnds; - s_interval= r?&state->BWHistoryReadInterval:&state->BWHistoryWriteInterval; - s_values = r?&state->BWHistoryReadValues :&state->BWHistoryWriteValues; + UPDATE(write_array, Write); + UPDATE(read_array, Read); + UPDATE(dir_write_array, DirWrite); + UPDATE(dir_read_array, DirRead); - if (*s_values) { - SMARTLIST_FOREACH(*s_values, char *, val, tor_free(val)); - smartlist_free(*s_values); - } - if (! server_mode(get_options())) { - /* Clients don't need to store bandwidth history persistently; - * force these values to the defaults. */ - /* FFFF we should pull the default out of config.c's state table, - * so we don't have two defaults. */ - if (*s_begins != 0 || *s_interval != 900) { - time_t now = time(NULL); - time_t save_at = get_options()->AvoidDiskWrites ? now+3600 : now+600; - or_state_mark_dirty(state, save_at); - } - *s_begins = 0; - *s_interval = 900; - *s_values = smartlist_create(); - continue; - } - *s_begins = b->next_period; - *s_interval = NUM_SECS_BW_SUM_INTERVAL; - cp = buf; - cp += rep_hist_fill_bandwidth_history(cp, len, b); - tor_snprintf(cp, len-(cp-buf), cp == buf ? U64_FORMAT : ","U64_FORMAT, - U64_PRINTF_ARG(b->total_in_period)); - *s_values = smartlist_create(); - if (server_mode(get_options())) - smartlist_split_string(*s_values, buf, ",", SPLIT_SKIP_SPACE, 0); - } - tor_free(buf); if (server_mode(get_options())) { - or_state_mark_dirty(get_or_state(), time(NULL)+(2*3600)); + or_state_mark_dirty(state, time(NULL)+(2*3600)); } +#undef UPDATE } -/** Set bandwidth history from our saved state. */ -int -rep_hist_load_state(or_state_t *state, char **err) +/** Load a single bw_array_t from its Values, Ends, Maxima, and Interval + * entries in an or_state_t. Done while reading the state file. */ +static int +rep_hist_load_bwhist_state_section(bw_array_t *b, + const smartlist_t *s_values, + const smartlist_t *s_maxima, + const time_t s_begins, + const int s_interval) { - time_t s_begins, start; time_t now = time(NULL); - uint64_t v; - int r,i,ok; - int all_ok = 1; - int s_interval; - smartlist_t *s_values; - bw_array_t *b; - - /* Assert they already have been malloced */ - tor_assert(read_array && write_array); + int retval = 0; + time_t start; - for (r=0;r<2;++r) { - b = r?read_array:write_array; - s_begins = r?state->BWHistoryReadEnds:state->BWHistoryWriteEnds; - s_interval = r?state->BWHistoryReadInterval:state->BWHistoryWriteInterval; - s_values = r?state->BWHistoryReadValues:state->BWHistoryWriteValues; - if (s_values && s_begins >= now - NUM_SECS_BW_SUM_INTERVAL*NUM_TOTALS) { - start = s_begins - s_interval*(smartlist_len(s_values)); - if (start > now) - continue; - b->cur_obs_time = start; - b->next_period = start + NUM_SECS_BW_SUM_INTERVAL; - SMARTLIST_FOREACH(s_values, char *, cp, { + uint64_t v, mv; + int i,ok,ok_m; + int have_maxima = (smartlist_len(s_values) == smartlist_len(s_maxima)); + + if (s_values && s_begins >= now - NUM_SECS_BW_SUM_INTERVAL*NUM_TOTALS) { + start = s_begins - s_interval*(smartlist_len(s_values)); + if (start > now) + return 0; + b->cur_obs_time = start; + b->next_period = start + NUM_SECS_BW_SUM_INTERVAL; + SMARTLIST_FOREACH_BEGIN(s_values, const char *, cp) { + const char *maxstr = NULL; v = tor_parse_uint64(cp, 10, 0, UINT64_MAX, &ok, NULL); + if (have_maxima) { + maxstr = smartlist_get(s_maxima, cp_sl_idx); + mv = tor_parse_uint64(maxstr, 10, 0, UINT64_MAX, &ok_m, NULL); + mv *= NUM_SECS_ROLLING_MEASURE; + } else { + /* No maxima known; guess average rate to be conservative. */ + mv = (v / s_interval) * NUM_SECS_ROLLING_MEASURE; + } if (!ok) { - all_ok=0; - log_notice(LD_HIST, "Could not parse '%s' into a number.'", cp); + retval = -1; + log_notice(LD_HIST, "Could not parse value '%s' into a number.'",cp); } + if (maxstr && !ok_m) { + retval = -1; + log_notice(LD_HIST, "Could not parse maximum '%s' into a number.'", + maxstr); + } + if (start < now) { - add_obs(b, start, v); - start += NUM_SECS_BW_SUM_INTERVAL; + time_t cur_start = start; + time_t actual_interval_len = s_interval; + uint64_t cur_val = 0; + /* Calculate the average per second. This is the best we can do + * because our state file doesn't have per-second resolution. */ + if (start + s_interval > now) + actual_interval_len = now - start; + cur_val = v / actual_interval_len; + /* This is potentially inefficient, but since we don't do it very + * often it should be ok. */ + while (cur_start < start + actual_interval_len) { + add_obs(b, cur_start, cur_val); + ++cur_start; + } + b->max_total = mv; + /* This will result in some fairly choppy history if s_interval + * is not the same as NUM_SECS_BW_SUM_INTERVAL. XXXX */ + start += actual_interval_len; } - }); - } + } SMARTLIST_FOREACH_END(cp); + } - /* Clean up maxima and observed */ - /* Do we really want to zero this for the purpose of max capacity? */ - for (i=0; i<NUM_SECS_ROLLING_MEASURE; ++i) { - b->obs[i] = 0; - } - b->total_obs = 0; - for (i=0; i<NUM_TOTALS; ++i) { - b->maxima[i] = 0; - } - b->max_total = 0; + /* Clean up maxima and observed */ + for (i=0; i<NUM_SECS_ROLLING_MEASURE; ++i) { + b->obs[i] = 0; } + b->total_obs = 0; + + return retval; +} + +/** Set bandwidth history from the state file we just loaded. */ +int +rep_hist_load_state(or_state_t *state, char **err) +{ + int all_ok = 1; + /* Assert they already have been malloced */ + tor_assert(read_array && write_array); + tor_assert(dir_read_array && dir_write_array); + +#define LOAD(arrname,st) \ + if (rep_hist_load_bwhist_state_section( \ + (arrname), \ + state->BWHistory ## st ## Values, \ + state->BWHistory ## st ## Maxima, \ + state->BWHistory ## st ## Ends, \ + state->BWHistory ## st ## Interval)<0) \ + all_ok = 0 + + LOAD(write_array, Write); + LOAD(read_array, Read); + LOAD(dir_write_array, DirWrite); + LOAD(dir_read_array, DirRead); + +#undef LOAD if (!all_ok) { *err = tor_strdup("Parsing of bandwidth history values failed"); /* and create fresh arrays */ - tor_free(read_array); - tor_free(write_array); - read_array = bw_array_new(); - write_array = bw_array_new(); + bw_arrays_init(); return -1; } return 0; @@ -1571,7 +1778,7 @@ static smartlist_t *predicted_ports_times=NULL; static void add_predicted_port(time_t now, uint16_t port) { - /* XXXX we could just use uintptr_t here, I think. */ + /* XXXX we could just use uintptr_t here, I think. -NM */ uint16_t *tmp_port = tor_malloc(sizeof(uint16_t)); time_t *tmp_time = tor_malloc(sizeof(time_t)); *tmp_port = port; @@ -1715,8 +1922,8 @@ rep_hist_get_predicted_internal(time_t now, int *need_uptime, return 0; /* too long ago */ if (predicted_internal_uptime_time + PREDICTED_CIRCS_RELEVANCE_TIME >= now) *need_uptime = 1; - if (predicted_internal_capacity_time + PREDICTED_CIRCS_RELEVANCE_TIME >= now) - *need_capacity = 1; + // Always predict that we need capacity. + *need_capacity = 1; return 1; } @@ -1845,564 +2052,530 @@ dump_pk_ops(int severity) pk_op_counts.n_rend_server_ops); } -/** Free all storage held by the OR/link history caches, by the - * bandwidth history arrays, or by the port history. */ +/*** Exit port statistics ***/ + +/* Some constants */ +/** To what multiple should byte numbers be rounded up? */ +#define EXIT_STATS_ROUND_UP_BYTES 1024 +/** To what multiple should stream counts be rounded up? */ +#define EXIT_STATS_ROUND_UP_STREAMS 4 +/** Number of TCP ports */ +#define EXIT_STATS_NUM_PORTS 65536 +/** Top n ports that will be included in exit stats. */ +#define EXIT_STATS_TOP_N_PORTS 10 + +/* The following data structures are arrays and no fancy smartlists or maps, + * so that all write operations can be done in constant time. This comes at + * the price of some memory (1.25 MB) and linear complexity when writing + * stats for measuring relays. */ +/** Number of bytes read in current period by exit port */ +static uint64_t *exit_bytes_read = NULL; +/** Number of bytes written in current period by exit port */ +static uint64_t *exit_bytes_written = NULL; +/** Number of streams opened in current period by exit port */ +static uint32_t *exit_streams = NULL; + +/** Start time of exit stats or 0 if we're not collecting exit stats. */ +static time_t start_of_exit_stats_interval; + +/** Initialize exit port stats. */ void -rep_hist_free_all(void) +rep_hist_exit_stats_init(time_t now) { - digestmap_free(history_map, free_or_history); - tor_free(read_array); - tor_free(write_array); - tor_free(last_stability_doc); - built_last_stability_doc_at = 0; - predicted_ports_free(); + start_of_exit_stats_interval = now; + exit_bytes_read = tor_malloc_zero(EXIT_STATS_NUM_PORTS * + sizeof(uint64_t)); + exit_bytes_written = tor_malloc_zero(EXIT_STATS_NUM_PORTS * + sizeof(uint64_t)); + exit_streams = tor_malloc_zero(EXIT_STATS_NUM_PORTS * + sizeof(uint32_t)); } -/****************** hidden service usage statistics ******************/ - -/** How large are the intervals for which we track and report hidden service - * use? */ -#define NUM_SECS_HS_USAGE_SUM_INTERVAL (15*60) -/** How far in the past do we remember and publish hidden service use? */ -#define NUM_SECS_HS_USAGE_SUM_IS_VALID (24*60*60) -/** How many hidden service usage intervals do we remember? (derived) */ -#define NUM_TOTALS_HS_USAGE (NUM_SECS_HS_USAGE_SUM_IS_VALID/ \ - NUM_SECS_HS_USAGE_SUM_INTERVAL) - -/** List element containing a service id and the count. */ -typedef struct hs_usage_list_elem_t { - /** Service id of this elem. */ - char service_id[REND_SERVICE_ID_LEN_BASE32+1]; - /** Number of occurrences for the given service id. */ - uint32_t count; - /* Pointer to next list elem */ - struct hs_usage_list_elem_t *next; -} hs_usage_list_elem_t; - -/** Ordered list that stores service ids and the number of observations. It is - * ordered by the number of occurrences in descending order. Its purpose is to - * calculate the frequency distribution when the period is over. */ -typedef struct hs_usage_list_t { - /* Pointer to the first element in the list. */ - hs_usage_list_elem_t *start; - /* Number of total occurrences for all list elements. */ - uint32_t total_count; - /* Number of service ids, i.e. number of list elements. */ - uint32_t total_service_ids; -} hs_usage_list_t; - -/** Tracks service-related observations in the current period and their - * history. */ -typedef struct hs_usage_service_related_observation_t { - /** Ordered list that stores service ids and the number of observations in - * the current period. It is ordered by the number of occurrences in - * descending order. Its purpose is to calculate the frequency distribution - * when the period is over. */ - hs_usage_list_t *list; - /** Circular arrays that store the history of observations. totals stores all - * observations, twenty (ten, five) the number of observations related to a - * service id being accounted for the top 20 (10, 5) percent of all - * observations. */ - uint32_t totals[NUM_TOTALS_HS_USAGE]; - uint32_t five[NUM_TOTALS_HS_USAGE]; - uint32_t ten[NUM_TOTALS_HS_USAGE]; - uint32_t twenty[NUM_TOTALS_HS_USAGE]; -} hs_usage_service_related_observation_t; - -/** Tracks the history of general period-related observations, i.e. those that - * cannot be related to a specific service id. */ -typedef struct hs_usage_general_period_related_observations_t { - /** Circular array that stores the history of observations. */ - uint32_t totals[NUM_TOTALS_HS_USAGE]; -} hs_usage_general_period_related_observations_t; - -/** Keeps information about the current observation period and its relation to - * the histories of observations. */ -typedef struct hs_usage_current_observation_period_t { - /** Where do we write the next history entry? */ - int next_idx; - /** How many values in history have been set ever? (upper bound!) */ - int num_set; - /** When did this period begin? */ - time_t start_of_current_period; - /** When does the next period begin? */ - time_t start_of_next_period; -} hs_usage_current_observation_period_t; - -/** Usage statistics for the current observation period. */ -static hs_usage_current_observation_period_t *current_period = NULL; - -/** Total number of descriptor publish requests in the current observation - * period. */ -static hs_usage_service_related_observation_t *publish_total = NULL; - -/** Number of descriptor publish requests for services that have not been - * seen before in the current observation period. */ -static hs_usage_service_related_observation_t *publish_novel = NULL; - -/** Total number of descriptor fetch requests in the current observation - * period. */ -static hs_usage_service_related_observation_t *fetch_total = NULL; - -/** Number of successful descriptor fetch requests in the current - * observation period. */ -static hs_usage_service_related_observation_t *fetch_successful = NULL; - -/** Number of descriptors stored in the current observation period. */ -static hs_usage_general_period_related_observations_t *descs = NULL; - -/** Creates an empty ordered list element. */ -static hs_usage_list_elem_t * -hs_usage_list_elem_new(void) +/** Reset counters for exit port statistics. */ +void +rep_hist_reset_exit_stats(time_t now) { - hs_usage_list_elem_t *e; - e = tor_malloc_zero(sizeof(hs_usage_list_elem_t)); - rephist_total_alloc += sizeof(hs_usage_list_elem_t); - e->count = 1; - e->next = NULL; - return e; + start_of_exit_stats_interval = now; + memset(exit_bytes_read, 0, EXIT_STATS_NUM_PORTS * sizeof(uint64_t)); + memset(exit_bytes_written, 0, EXIT_STATS_NUM_PORTS * sizeof(uint64_t)); + memset(exit_streams, 0, EXIT_STATS_NUM_PORTS * sizeof(uint32_t)); } -/** Creates an empty ordered list. */ -static hs_usage_list_t * -hs_usage_list_new(void) +/** Stop collecting exit port stats in a way that we can re-start doing + * so in rep_hist_exit_stats_init(). */ +void +rep_hist_exit_stats_term(void) { - hs_usage_list_t *l; - l = tor_malloc_zero(sizeof(hs_usage_list_t)); - rephist_total_alloc += sizeof(hs_usage_list_t); - l->start = NULL; - l->total_count = 0; - l->total_service_ids = 0; - return l; + start_of_exit_stats_interval = 0; + tor_free(exit_bytes_read); + tor_free(exit_bytes_written); + tor_free(exit_streams); } -/** Creates an empty structure for storing service-related observations. */ -static hs_usage_service_related_observation_t * -hs_usage_service_related_observation_new(void) +/** Helper for qsort: compare two ints. */ +static int +_compare_int(const void *x, const void *y) { - hs_usage_service_related_observation_t *h; - h = tor_malloc_zero(sizeof(hs_usage_service_related_observation_t)); - rephist_total_alloc += sizeof(hs_usage_service_related_observation_t); - h->list = hs_usage_list_new(); - return h; + return (*(int*)x - *(int*)y); } -/** Creates an empty structure for storing general period-related - * observations. */ -static hs_usage_general_period_related_observations_t * -hs_usage_general_period_related_observations_new(void) -{ - hs_usage_general_period_related_observations_t *p; - p = tor_malloc_zero(sizeof(hs_usage_general_period_related_observations_t)); - rephist_total_alloc+= sizeof(hs_usage_general_period_related_observations_t); - return p; -} +/** Return a newly allocated string containing the exit port statistics + * until <b>now</b>, or NULL if we're not collecting exit stats. */ +char * +rep_hist_format_exit_stats(time_t now) +{ + int i, j, top_elements = 0, cur_min_idx = 0, cur_port; + uint64_t top_bytes[EXIT_STATS_TOP_N_PORTS]; + int top_ports[EXIT_STATS_TOP_N_PORTS]; + uint64_t cur_bytes = 0, other_read = 0, other_written = 0, + total_read = 0, total_written = 0; + uint32_t total_streams = 0, other_streams = 0; + char *buf; + smartlist_t *written_strings, *read_strings, *streams_strings; + char *written_string, *read_string, *streams_string; + char t[ISO_TIME_LEN+1]; + char *result; -/** Creates an empty structure for storing period-specific information. */ -static hs_usage_current_observation_period_t * -hs_usage_current_observation_period_new(void) -{ - hs_usage_current_observation_period_t *c; - time_t now; - c = tor_malloc_zero(sizeof(hs_usage_current_observation_period_t)); - rephist_total_alloc += sizeof(hs_usage_current_observation_period_t); - now = time(NULL); - c->start_of_current_period = now; - c->start_of_next_period = now + NUM_SECS_HS_USAGE_SUM_INTERVAL; - return c; -} + if (!start_of_exit_stats_interval) + return NULL; /* Not initialized. */ -/** Initializes the structures for collecting hidden service usage data. */ -static void -hs_usage_init(void) -{ - current_period = hs_usage_current_observation_period_new(); - publish_total = hs_usage_service_related_observation_new(); - publish_novel = hs_usage_service_related_observation_new(); - fetch_total = hs_usage_service_related_observation_new(); - fetch_successful = hs_usage_service_related_observation_new(); - descs = hs_usage_general_period_related_observations_new(); -} + /* Go through all ports to find the n ports that saw most written and + * read bytes. + * + * Invariant: at the end of the loop for iteration i, + * total_read is the sum of all exit_bytes_read[0..i] + * total_written is the sum of all exit_bytes_written[0..i] + * total_stream is the sum of all exit_streams[0..i] + * + * top_elements = MAX(EXIT_STATS_TOP_N_PORTS, + * #{j | 0 <= j <= i && volume(i) > 0}) + * + * For all 0 <= j < top_elements, + * top_bytes[j] > 0 + * 0 <= top_ports[j] <= 65535 + * top_bytes[j] = volume(top_ports[j]) + * + * There is no j in 0..i and k in 0..top_elements such that: + * volume(j) > top_bytes[k] AND j is not in top_ports[0..top_elements] + * + * There is no j!=cur_min_idx in 0..top_elements such that: + * top_bytes[j] < top_bytes[cur_min_idx] + * + * where volume(x) == exit_bytes_read[x]+exit_bytes_written[x] + * + * Worst case: O(EXIT_STATS_NUM_PORTS * EXIT_STATS_TOP_N_PORTS) + */ + for (i = 1; i < EXIT_STATS_NUM_PORTS; i++) { + total_read += exit_bytes_read[i]; + total_written += exit_bytes_written[i]; + total_streams += exit_streams[i]; + cur_bytes = exit_bytes_read[i] + exit_bytes_written[i]; + if (cur_bytes == 0) { + continue; + } + if (top_elements < EXIT_STATS_TOP_N_PORTS) { + top_bytes[top_elements] = cur_bytes; + top_ports[top_elements++] = i; + } else if (cur_bytes > top_bytes[cur_min_idx]) { + top_bytes[cur_min_idx] = cur_bytes; + top_ports[cur_min_idx] = i; + } else { + continue; + } + cur_min_idx = 0; + for (j = 1; j < top_elements; j++) { + if (top_bytes[j] < top_bytes[cur_min_idx]) { + cur_min_idx = j; + } + } + } -/** Clears the given ordered list by resetting its attributes and releasing - * the memory allocated by its elements. */ -static void -hs_usage_list_clear(hs_usage_list_t *lst) -{ - /* walk through elements and free memory */ - hs_usage_list_elem_t *current = lst->start; - hs_usage_list_elem_t *tmp; - while (current != NULL) { - tmp = current->next; - rephist_total_alloc -= sizeof(hs_usage_list_elem_t); - tor_free(current); - current = tmp; + /* Add observations of top ports to smartlists. */ + written_strings = smartlist_create(); + read_strings = smartlist_create(); + streams_strings = smartlist_create(); + other_read = total_read; + other_written = total_written; + other_streams = total_streams; + /* Sort the ports; this puts them out of sync with top_bytes, but we + * won't be using top_bytes again anyway */ + qsort(top_ports, top_elements, sizeof(int), _compare_int); + for (j = 0; j < top_elements; j++) { + cur_port = top_ports[j]; + if (exit_bytes_written[cur_port] > 0) { + uint64_t num = round_uint64_to_next_multiple_of( + exit_bytes_written[cur_port], + EXIT_STATS_ROUND_UP_BYTES); + num /= 1024; + buf = NULL; + tor_asprintf(&buf, "%d="U64_FORMAT, cur_port, U64_PRINTF_ARG(num)); + smartlist_add(written_strings, buf); + other_written -= exit_bytes_written[cur_port]; + } + if (exit_bytes_read[cur_port] > 0) { + uint64_t num = round_uint64_to_next_multiple_of( + exit_bytes_read[cur_port], + EXIT_STATS_ROUND_UP_BYTES); + num /= 1024; + buf = NULL; + tor_asprintf(&buf, "%d="U64_FORMAT, cur_port, U64_PRINTF_ARG(num)); + smartlist_add(read_strings, buf); + other_read -= exit_bytes_read[cur_port]; + } + if (exit_streams[cur_port] > 0) { + uint32_t num = round_uint32_to_next_multiple_of( + exit_streams[cur_port], + EXIT_STATS_ROUND_UP_STREAMS); + buf = NULL; + tor_asprintf(&buf, "%d=%u", cur_port, num); + smartlist_add(streams_strings, buf); + other_streams -= exit_streams[cur_port]; + } } - /* reset attributes */ - lst->start = NULL; - lst->total_count = 0; - lst->total_service_ids = 0; - return; -} -/** Frees the memory used by the given list. */ -static void -hs_usage_list_free(hs_usage_list_t *lst) -{ - if (!lst) - return; - hs_usage_list_clear(lst); - rephist_total_alloc -= sizeof(hs_usage_list_t); - tor_free(lst); + /* Add observations of other ports in a single element. */ + other_written = round_uint64_to_next_multiple_of(other_written, + EXIT_STATS_ROUND_UP_BYTES); + other_written /= 1024; + buf = NULL; + tor_asprintf(&buf, "other="U64_FORMAT, U64_PRINTF_ARG(other_written)); + smartlist_add(written_strings, buf); + other_read = round_uint64_to_next_multiple_of(other_read, + EXIT_STATS_ROUND_UP_BYTES); + other_read /= 1024; + buf = NULL; + tor_asprintf(&buf, "other="U64_FORMAT, U64_PRINTF_ARG(other_read)); + smartlist_add(read_strings, buf); + other_streams = round_uint32_to_next_multiple_of(other_streams, + EXIT_STATS_ROUND_UP_STREAMS); + buf = NULL; + tor_asprintf(&buf, "other=%u", other_streams); + smartlist_add(streams_strings, buf); + + /* Join all observations in single strings. */ + written_string = smartlist_join_strings(written_strings, ",", 0, NULL); + read_string = smartlist_join_strings(read_strings, ",", 0, NULL); + streams_string = smartlist_join_strings(streams_strings, ",", 0, NULL); + SMARTLIST_FOREACH(written_strings, char *, cp, tor_free(cp)); + SMARTLIST_FOREACH(read_strings, char *, cp, tor_free(cp)); + SMARTLIST_FOREACH(streams_strings, char *, cp, tor_free(cp)); + smartlist_free(written_strings); + smartlist_free(read_strings); + smartlist_free(streams_strings); + + /* Put everything together. */ + format_iso_time(t, now); + tor_asprintf(&result, "exit-stats-end %s (%d s)\n" + "exit-kibibytes-written %s\n" + "exit-kibibytes-read %s\n" + "exit-streams-opened %s\n", + t, (unsigned) (now - start_of_exit_stats_interval), + written_string, + read_string, + streams_string); + tor_free(written_string); + tor_free(read_string); + tor_free(streams_string); + return result; } -/** Frees the memory used by the given service-related observations. */ -static void -hs_usage_service_related_observation_free( - hs_usage_service_related_observation_t *s) +/** If 24 hours have passed since the beginning of the current exit port + * stats period, write exit stats to $DATADIR/stats/exit-stats (possibly + * overwriting an existing file) and reset counters. Return when we would + * next want to write exit stats or 0 if we never want to write. */ +time_t +rep_hist_exit_stats_write(time_t now) { - if (!s) - return; - hs_usage_list_free(s->list); - rephist_total_alloc -= sizeof(hs_usage_service_related_observation_t); - tor_free(s); -} + char *statsdir = NULL, *filename = NULL, *str = NULL; -/** Frees the memory used by the given period-specific observations. */ -static void -hs_usage_general_period_related_observations_free( - hs_usage_general_period_related_observations_t *s) -{ - rephist_total_alloc-=sizeof(hs_usage_general_period_related_observations_t); - tor_free(s); -} + if (!start_of_exit_stats_interval) + return 0; /* Not initialized. */ + if (start_of_exit_stats_interval + WRITE_STATS_INTERVAL > now) + goto done; /* Not ready to write. */ -/** Frees the memory used by period-specific information. */ -static void -hs_usage_current_observation_period_free( - hs_usage_current_observation_period_t *s) -{ - rephist_total_alloc -= sizeof(hs_usage_current_observation_period_t); - tor_free(s); -} + log_info(LD_HIST, "Writing exit port statistics to disk."); -/** Frees all memory that was used for collecting hidden service usage data. */ -void -hs_usage_free_all(void) -{ - hs_usage_general_period_related_observations_free(descs); - descs = NULL; - hs_usage_service_related_observation_free(fetch_successful); - hs_usage_service_related_observation_free(fetch_total); - hs_usage_service_related_observation_free(publish_novel); - hs_usage_service_related_observation_free(publish_total); - fetch_successful = fetch_total = publish_novel = publish_total = NULL; - hs_usage_current_observation_period_free(current_period); - current_period = NULL; -} + /* Generate history string. */ + str = rep_hist_format_exit_stats(now); -/** Inserts a new occurrence for the given service id to the given ordered - * list. */ -static void -hs_usage_insert_value(hs_usage_list_t *lst, const char *service_id) -{ - /* search if there is already an elem with same service_id in list */ - hs_usage_list_elem_t *current = lst->start; - hs_usage_list_elem_t *previous = NULL; - while (current != NULL && strcasecmp(current->service_id,service_id)) { - previous = current; - current = current->next; - } - /* found an element with same service_id? */ - if (current == NULL) { - /* not found! append to end (which could also be the end of a zero-length - * list), don't need to sort (1 is smallest value). */ - /* create elem */ - hs_usage_list_elem_t *e = hs_usage_list_elem_new(); - /* update list attributes (one new elem, one new occurrence) */ - lst->total_count++; - lst->total_service_ids++; - /* copy service id to elem */ - strlcpy(e->service_id,service_id,sizeof(e->service_id)); - /* let either l->start or previously last elem point to new elem */ - if (lst->start == NULL) { - /* this is the first elem */ - lst->start = e; - } else { - /* there were elems in the list before */ - previous->next = e; - } - } else { - /* found! add occurrence to elem and consider resorting */ - /* update list attributes (no new elem, but one new occurrence) */ - lst->total_count++; - /* add occurrence to elem */ - current->count++; - /* is it another than the first list elem? and has previous elem fewer - * count than current? then we need to resort */ - if (previous != NULL && previous->count < current->count) { - /* yes! we need to resort */ - /* remove current elem first */ - previous->next = current->next; - /* can we prepend elem to all other elements? */ - if (lst->start->count <= current->count) { - /* yes! prepend elem */ - current->next = lst->start; - lst->start = current; - } else { - /* no! walk through list a second time and insert at correct place */ - hs_usage_list_elem_t *insert_current = lst->start->next; - hs_usage_list_elem_t *insert_previous = lst->start; - while (insert_current != NULL && - insert_current->count > current->count) { - insert_previous = insert_current; - insert_current = insert_current->next; - } - /* insert here */ - current->next = insert_current; - insert_previous->next = current; - } - } - } -} + /* Reset counters. */ + rep_hist_reset_exit_stats(now); -/** Writes the current service-related observations to the history array and - * clears the observations of the current period. */ -static void -hs_usage_write_service_related_observations_to_history( - hs_usage_current_observation_period_t *p, - hs_usage_service_related_observation_t *h) -{ - /* walk through the first 20 % of list elements and calculate frequency - * distributions */ - /* maximum indices for the three frequencies */ - int five_percent_idx = h->list->total_service_ids/20; - int ten_percent_idx = h->list->total_service_ids/10; - int twenty_percent_idx = h->list->total_service_ids/5; - /* temp values */ - uint32_t five_percent = 0; - uint32_t ten_percent = 0; - uint32_t twenty_percent = 0; - /* walk through list */ - hs_usage_list_elem_t *current = h->list->start; - int i=0; - while (current != NULL && i <= twenty_percent_idx) { - twenty_percent += current->count; - if (i <= ten_percent_idx) - ten_percent += current->count; - if (i <= five_percent_idx) - five_percent += current->count; - current = current->next; - i++; - } - /* copy frequencies */ - h->twenty[p->next_idx] = twenty_percent; - h->ten[p->next_idx] = ten_percent; - h->five[p->next_idx] = five_percent; - /* copy total number of observations */ - h->totals[p->next_idx] = h->list->total_count; - /* free memory of old list */ - hs_usage_list_clear(h->list); -} - -/** Advances to next observation period. */ -static void -hs_usage_advance_current_observation_period(void) -{ - /* aggregate observations to history, including frequency distribution - * arrays */ - hs_usage_write_service_related_observations_to_history( - current_period, publish_total); - hs_usage_write_service_related_observations_to_history( - current_period, publish_novel); - hs_usage_write_service_related_observations_to_history( - current_period, fetch_total); - hs_usage_write_service_related_observations_to_history( - current_period, fetch_successful); - /* write current number of descriptors to descs history */ - descs->totals[current_period->next_idx] = rend_cache_size(); - /* advance to next period */ - current_period->next_idx++; - if (current_period->next_idx == NUM_TOTALS_HS_USAGE) - current_period->next_idx = 0; - if (current_period->num_set < NUM_TOTALS_HS_USAGE) - ++current_period->num_set; - current_period->start_of_current_period=current_period->start_of_next_period; - current_period->start_of_next_period += NUM_SECS_HS_USAGE_SUM_INTERVAL; -} - -/** Checks if the current period is up to date, and if not, advances it. */ -static void -hs_usage_check_if_current_period_is_up_to_date(time_t now) -{ - while (now > current_period->start_of_next_period) { - hs_usage_advance_current_observation_period(); + /* Try to write to disk. */ + statsdir = get_datadir_fname("stats"); + if (check_private_dir(statsdir, CPD_CREATE) < 0) { + log_warn(LD_HIST, "Unable to create stats/ directory!"); + goto done; } -} + filename = get_datadir_fname2("stats", "exit-stats"); + if (write_str_to_file(filename, str, 0) < 0) + log_warn(LD_HIST, "Unable to write exit port statistics to disk!"); -/** Adds a service-related observation, maybe after advancing to next - * observation period. */ -static void -hs_usage_add_service_related_observation( - hs_usage_service_related_observation_t *h, - time_t now, - const char *service_id) -{ - if (now < current_period->start_of_current_period) { - /* don't record old data */ - return; - } - /* check if we are up-to-date */ - hs_usage_check_if_current_period_is_up_to_date(now); - /* add observation */ - hs_usage_insert_value(h->list, service_id); + done: + tor_free(str); + tor_free(statsdir); + tor_free(filename); + return start_of_exit_stats_interval + WRITE_STATS_INTERVAL; } -/** Adds the observation of storing a rendezvous service descriptor to our - * cache in our role as HS authoritative directory. */ +/** Note that we wrote <b>num_written</b> bytes and read <b>num_read</b> + * bytes to/from an exit connection to <b>port</b>. */ void -hs_usage_note_publish_total(const char *service_id, time_t now) +rep_hist_note_exit_bytes(uint16_t port, size_t num_written, + size_t num_read) { - hs_usage_add_service_related_observation(publish_total, now, service_id); + if (!start_of_exit_stats_interval) + return; /* Not initialized. */ + exit_bytes_written[port] += num_written; + exit_bytes_read[port] += num_read; + log_debug(LD_HIST, "Written %lu bytes and read %lu bytes to/from an " + "exit connection to port %d.", + (unsigned long)num_written, (unsigned long)num_read, port); } -/** Adds the observation of storing a novel rendezvous service descriptor to - * our cache in our role as HS authoritative directory. */ +/** Note that we opened an exit stream to <b>port</b>. */ void -hs_usage_note_publish_novel(const char *service_id, time_t now) +rep_hist_note_exit_stream_opened(uint16_t port) { - hs_usage_add_service_related_observation(publish_novel, now, service_id); + if (!start_of_exit_stats_interval) + return; /* Not initialized. */ + exit_streams[port]++; + log_debug(LD_HIST, "Opened exit stream to port %d", port); } -/** Adds the observation of being requested for a rendezvous service descriptor - * in our role as HS authoritative directory. */ +/*** cell statistics ***/ + +/** Start of the current buffer stats interval or 0 if we're not + * collecting buffer statistics. */ +static time_t start_of_buffer_stats_interval; + +/** Initialize buffer stats. */ void -hs_usage_note_fetch_total(const char *service_id, time_t now) +rep_hist_buffer_stats_init(time_t now) { - hs_usage_add_service_related_observation(fetch_total, now, service_id); + start_of_buffer_stats_interval = now; } -/** Adds the observation of being requested for a rendezvous service descriptor - * in our role as HS authoritative directory and being able to answer that - * request successfully. */ +/** Statistics from a single circuit. Collected when the circuit closes, or + * when we flush statistics to disk. */ +typedef struct circ_buffer_stats_t { + /** Average number of cells in the circuit's queue */ + double mean_num_cells_in_queue; + /** Average time a cell waits in the queue. */ + double mean_time_cells_in_queue; + /** Total number of cells sent over this circuit */ + uint32_t processed_cells; +} circ_buffer_stats_t; + +/** List of circ_buffer_stats_t. */ +static smartlist_t *circuits_for_buffer_stats = NULL; + +/** Remember cell statistics for circuit <b>circ</b> at time + * <b>end_of_interval</b> and reset cell counters in case the circuit + * remains open in the next measurement interval. */ void -hs_usage_note_fetch_successful(const char *service_id, time_t now) +rep_hist_buffer_stats_add_circ(circuit_t *circ, time_t end_of_interval) +{ + circ_buffer_stats_t *stat; + time_t start_of_interval; + int interval_length; + or_circuit_t *orcirc; + if (CIRCUIT_IS_ORIGIN(circ)) + return; + orcirc = TO_OR_CIRCUIT(circ); + if (!orcirc->processed_cells) + return; + if (!circuits_for_buffer_stats) + circuits_for_buffer_stats = smartlist_create(); + start_of_interval = circ->timestamp_created.tv_sec > + start_of_buffer_stats_interval ? + circ->timestamp_created.tv_sec : + start_of_buffer_stats_interval; + interval_length = (int) (end_of_interval - start_of_interval); + if (interval_length <= 0) + return; + stat = tor_malloc_zero(sizeof(circ_buffer_stats_t)); + stat->processed_cells = orcirc->processed_cells; + /* 1000.0 for s -> ms; 2.0 because of app-ward and exit-ward queues */ + stat->mean_num_cells_in_queue = interval_length == 0 ? 0.0 : + (double) orcirc->total_cell_waiting_time / + (double) interval_length / 1000.0 / 2.0; + stat->mean_time_cells_in_queue = + (double) orcirc->total_cell_waiting_time / + (double) orcirc->processed_cells; + smartlist_add(circuits_for_buffer_stats, stat); + orcirc->total_cell_waiting_time = 0; + orcirc->processed_cells = 0; +} + +/** Sorting helper: return -1, 1, or 0 based on comparison of two + * circ_buffer_stats_t */ +static int +_buffer_stats_compare_entries(const void **_a, const void **_b) { - hs_usage_add_service_related_observation(fetch_successful, now, service_id); + const circ_buffer_stats_t *a = *_a, *b = *_b; + if (a->processed_cells < b->processed_cells) + return 1; + else if (a->processed_cells > b->processed_cells) + return -1; + else + return 0; } -/** Writes the given circular array to a string. */ -static size_t -hs_usage_format_history(char *buf, size_t len, uint32_t *data) +/** Stop collecting cell stats in a way that we can re-start doing so in + * rep_hist_buffer_stats_init(). */ +void +rep_hist_buffer_stats_term(void) { - char *cp = buf; /* pointer where we are in the buffer */ - int i, n; - if (current_period->num_set <= current_period->next_idx) { - i = 0; /* not been through circular array */ - } else { - i = current_period->next_idx; - } - for (n = 0; n < current_period->num_set; ++n,++i) { - if (i >= NUM_TOTALS_HS_USAGE) - i -= NUM_TOTALS_HS_USAGE; - tor_assert(i < NUM_TOTALS_HS_USAGE); - if (n == (current_period->num_set-1)) - tor_snprintf(cp, len-(cp-buf), "%d", data[i]); - else - tor_snprintf(cp, len-(cp-buf), "%d,", data[i]); - cp += strlen(cp); - } - return cp-buf; + start_of_buffer_stats_interval = 0; + if (!circuits_for_buffer_stats) + circuits_for_buffer_stats = smartlist_create(); + SMARTLIST_FOREACH(circuits_for_buffer_stats, circ_buffer_stats_t *, + stat, tor_free(stat)); + smartlist_clear(circuits_for_buffer_stats); } -/** Writes the complete usage history as hidden service authoritative directory - * to a string. */ -static char * -hs_usage_format_statistics(void) +/** Write buffer statistics to $DATADIR/stats/buffer-stats and return when + * we would next want to write exit stats. */ +time_t +rep_hist_buffer_stats_write(time_t now) { - char *buf, *cp, *s = NULL; - char t[ISO_TIME_LEN+1]; - int r; - uint32_t *data = NULL; - size_t len; - len = (70+20*NUM_TOTALS_HS_USAGE)*11; - buf = tor_malloc_zero(len); - cp = buf; - for (r = 0; r < 11; ++r) { - switch (r) { - case 0: - s = (char*) "publish-total-history"; - data = publish_total->totals; - break; - case 1: - s = (char*) "publish-novel-history"; - data = publish_novel->totals; - break; - case 2: - s = (char*) "publish-top-5-percent-history"; - data = publish_total->five; - break; - case 3: - s = (char*) "publish-top-10-percent-history"; - data = publish_total->ten; - break; - case 4: - s = (char*) "publish-top-20-percent-history"; - data = publish_total->twenty; - break; - case 5: - s = (char*) "fetch-total-history"; - data = fetch_total->totals; - break; - case 6: - s = (char*) "fetch-successful-history"; - data = fetch_successful->totals; - break; - case 7: - s = (char*) "fetch-top-5-percent-history"; - data = fetch_total->five; - break; - case 8: - s = (char*) "fetch-top-10-percent-history"; - data = fetch_total->ten; - break; - case 9: - s = (char*) "fetch-top-20-percent-history"; - data = fetch_total->twenty; - break; - case 10: - s = (char*) "desc-total-history"; - data = descs->totals; - break; - } - format_iso_time(t, current_period->start_of_current_period); - tor_snprintf(cp, len-(cp-buf), "%s %s (%d s) ", s, t, - NUM_SECS_HS_USAGE_SUM_INTERVAL); - cp += strlen(cp); - cp += hs_usage_format_history(cp, len-(cp-buf), data); - strlcat(cp, "\n", len-(cp-buf)); - ++cp; + char *statsdir = NULL, *filename = NULL; + char written[ISO_TIME_LEN+1]; + open_file_t *open_file = NULL; + FILE *out; +#define SHARES 10 + int processed_cells[SHARES], circs_in_share[SHARES], + number_of_circuits, i; + double queued_cells[SHARES], time_in_queue[SHARES]; + smartlist_t *str_build = smartlist_create(); + char *str = NULL, *buf=NULL; + circuit_t *circ; + + if (!start_of_buffer_stats_interval) + return 0; /* Not initialized. */ + if (start_of_buffer_stats_interval + WRITE_STATS_INTERVAL > now) + goto done; /* Not ready to write */ + + /* add current circuits to stats */ + for (circ = _circuit_get_global_list(); circ; circ = circ->next) + rep_hist_buffer_stats_add_circ(circ, now); + /* calculate deciles */ + memset(processed_cells, 0, SHARES * sizeof(int)); + memset(circs_in_share, 0, SHARES * sizeof(int)); + memset(queued_cells, 0, SHARES * sizeof(double)); + memset(time_in_queue, 0, SHARES * sizeof(double)); + if (!circuits_for_buffer_stats) + circuits_for_buffer_stats = smartlist_create(); + smartlist_sort(circuits_for_buffer_stats, + _buffer_stats_compare_entries); + number_of_circuits = smartlist_len(circuits_for_buffer_stats); + if (number_of_circuits < 1) { + log_info(LD_HIST, "Attempt to write cell statistics to disk failed. " + "We haven't seen a single circuit to report about."); + goto done; } - return buf; + i = 0; + SMARTLIST_FOREACH_BEGIN(circuits_for_buffer_stats, + circ_buffer_stats_t *, stat) + { + int share = i++ * SHARES / number_of_circuits; + processed_cells[share] += stat->processed_cells; + queued_cells[share] += stat->mean_num_cells_in_queue; + time_in_queue[share] += stat->mean_time_cells_in_queue; + circs_in_share[share]++; + } + SMARTLIST_FOREACH_END(stat); + /* clear buffer stats history */ + SMARTLIST_FOREACH(circuits_for_buffer_stats, circ_buffer_stats_t *, + stat, tor_free(stat)); + smartlist_clear(circuits_for_buffer_stats); + /* write to file */ + statsdir = get_datadir_fname("stats"); + if (check_private_dir(statsdir, CPD_CREATE) < 0) + goto done; + filename = get_datadir_fname2("stats", "buffer-stats"); + out = start_writing_to_stdio_file(filename, OPEN_FLAGS_APPEND, + 0600, &open_file); + if (!out) + goto done; + format_iso_time(written, now); + if (fprintf(out, "cell-stats-end %s (%d s)\n", written, + (unsigned) (now - start_of_buffer_stats_interval)) < 0) + goto done; + for (i = 0; i < SHARES; i++) { + tor_asprintf(&buf,"%d", !circs_in_share[i] ? 0 : + processed_cells[i] / circs_in_share[i]); + smartlist_add(str_build, buf); + } + str = smartlist_join_strings(str_build, ",", 0, NULL); + if (fprintf(out, "cell-processed-cells %s\n", str) < 0) + goto done; + tor_free(str); + SMARTLIST_FOREACH(str_build, char *, c, tor_free(c)); + smartlist_clear(str_build); + for (i = 0; i < SHARES; i++) { + tor_asprintf(&buf, "%.2f", circs_in_share[i] == 0 ? 0.0 : + queued_cells[i] / (double) circs_in_share[i]); + smartlist_add(str_build, buf); + } + str = smartlist_join_strings(str_build, ",", 0, NULL); + if (fprintf(out, "cell-queued-cells %s\n", str) < 0) + goto done; + tor_free(str); + SMARTLIST_FOREACH(str_build, char *, c, tor_free(c)); + smartlist_clear(str_build); + for (i = 0; i < SHARES; i++) { + tor_asprintf(&buf, "%.0f", circs_in_share[i] == 0 ? 0.0 : + time_in_queue[i] / (double) circs_in_share[i]); + smartlist_add(str_build, buf); + } + str = smartlist_join_strings(str_build, ",", 0, NULL); + if (fprintf(out, "cell-time-in-queue %s\n", str) < 0) + goto done; + tor_free(str); + SMARTLIST_FOREACH(str_build, char *, c, tor_free(c)); + smartlist_free(str_build); + str_build = NULL; + if (fprintf(out, "cell-circuits-per-decile %d\n", + (number_of_circuits + SHARES - 1) / SHARES) < 0) + goto done; + finish_writing_to_file(open_file); + open_file = NULL; + start_of_buffer_stats_interval = now; + done: + if (open_file) + abort_writing_to_file(open_file); + tor_free(filename); + tor_free(statsdir); + if (str_build) { + SMARTLIST_FOREACH(str_build, char *, c, tor_free(c)); + smartlist_free(str_build); + } + tor_free(str); +#undef SHARES + return start_of_buffer_stats_interval + WRITE_STATS_INTERVAL; } -/** Write current statistics about hidden service usage to file. */ +/** Free all storage held by the OR/link history caches, by the + * bandwidth history arrays, by the port history, or by statistics . */ void -hs_usage_write_statistics_to_file(time_t now) +rep_hist_free_all(void) { - char *buf; - size_t len; - char *fname; - or_options_t *options = get_options(); - /* check if we are up-to-date */ - hs_usage_check_if_current_period_is_up_to_date(now); - buf = hs_usage_format_statistics(); - len = strlen(options->DataDirectory) + 16; - fname = tor_malloc(len); - tor_snprintf(fname, len, "%s"PATH_SEPARATOR"hsusage", - options->DataDirectory); - write_str_to_file(fname,buf,0); - tor_free(buf); - tor_free(fname); + digestmap_free(history_map, free_or_history); + tor_free(read_array); + tor_free(write_array); + tor_free(last_stability_doc); + tor_free(exit_bytes_read); + tor_free(exit_bytes_written); + tor_free(exit_streams); + built_last_stability_doc_at = 0; + predicted_ports_free(); + if (circuits_for_buffer_stats) { + SMARTLIST_FOREACH(circuits_for_buffer_stats, circ_buffer_stats_t *, s, + tor_free(s)); + smartlist_free(circuits_for_buffer_stats); + circuits_for_buffer_stats = NULL; + } } diff --git a/src/or/rephist.h b/src/or/rephist.h new file mode 100644 index 0000000000..b06a39ed59 --- /dev/null +++ b/src/or/rephist.h @@ -0,0 +1,82 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file rephist.h + * \brief Header file for rephist.c. + **/ + +#ifndef _TOR_REPHIST_H +#define _TOR_REPHIST_H + +void rep_hist_init(void); +void rep_hist_note_connect_failed(const char* nickname, time_t when); +void rep_hist_note_connect_succeeded(const char* nickname, time_t when); +void rep_hist_note_disconnect(const char* nickname, time_t when); +void rep_hist_note_connection_died(const char* nickname, time_t when); +void rep_hist_note_extend_succeeded(const char *from_name, + const char *to_name); +void rep_hist_note_extend_failed(const char *from_name, const char *to_name); +void rep_hist_dump_stats(time_t now, int severity); +void rep_hist_note_bytes_read(size_t num_bytes, time_t when); +void rep_hist_note_bytes_written(size_t num_bytes, time_t when); + +void rep_hist_note_dir_bytes_read(size_t num_bytes, time_t when); +void rep_hist_note_dir_bytes_written(size_t num_bytes, time_t when); + +int rep_hist_bandwidth_assess(void); +char *rep_hist_get_bandwidth_lines(void); +void rep_hist_update_state(or_state_t *state); +int rep_hist_load_state(or_state_t *state, char **err); +void rep_history_clean(time_t before); + +void rep_hist_note_router_reachable(const char *id, const tor_addr_t *at_addr, + uint16_t at_port, time_t when); +void rep_hist_note_router_unreachable(const char *id, time_t when); +int rep_hist_record_mtbf_data(time_t now, int missing_means_down); +int rep_hist_load_mtbf_data(time_t now); + +time_t rep_hist_downrate_old_runs(time_t now); +long rep_hist_get_uptime(const char *id, time_t when); +double rep_hist_get_stability(const char *id, time_t when); +double rep_hist_get_weighted_fractional_uptime(const char *id, time_t when); +long rep_hist_get_weighted_time_known(const char *id, time_t when); +int rep_hist_have_measured_enough_stability(void); +const char *rep_hist_get_router_stability_doc(time_t now); + +void rep_hist_note_used_port(time_t now, uint16_t port); +smartlist_t *rep_hist_get_predicted_ports(time_t now); +void rep_hist_note_used_resolve(time_t now); +void rep_hist_note_used_internal(time_t now, int need_uptime, + int need_capacity); +int rep_hist_get_predicted_internal(time_t now, int *need_uptime, + int *need_capacity); + +int any_predicted_circuits(time_t now); +int rep_hist_circbuilding_dormant(time_t now); + +void note_crypto_pk_op(pk_op_t operation); +void dump_pk_ops(int severity); + +void rep_hist_free_all(void); + +void rep_hist_exit_stats_init(time_t now); +void rep_hist_reset_exit_stats(time_t now); +void rep_hist_exit_stats_term(void); +char *rep_hist_format_exit_stats(time_t now); +time_t rep_hist_exit_stats_write(time_t now); +void rep_hist_note_exit_bytes(uint16_t port, size_t num_written, + size_t num_read); +void rep_hist_note_exit_stream_opened(uint16_t port); + +void rep_hist_buffer_stats_init(time_t now); +void rep_hist_buffer_stats_add_circ(circuit_t *circ, + time_t end_of_interval); +time_t rep_hist_buffer_stats_write(time_t now); +void rep_hist_buffer_stats_term(void); + +#endif + diff --git a/src/or/router.c b/src/or/router.c index 75f592d9e6..65afd49f7f 100644 --- a/src/or/router.c +++ b/src/or/router.c @@ -7,6 +7,24 @@ #define ROUTER_PRIVATE #include "or.h" +#include "circuitlist.h" +#include "circuituse.h" +#include "config.h" +#include "connection.h" +#include "control.h" +#include "directory.h" +#include "dirserv.h" +#include "dns.h" +#include "geoip.h" +#include "hibernate.h" +#include "main.h" +#include "networkstatus.h" +#include "policies.h" +#include "relay.h" +#include "rephist.h" +#include "router.h" +#include "routerlist.h" +#include "routerparse.h" /** * \file router.c @@ -31,11 +49,15 @@ static crypto_pk_env_t *onionkey=NULL; /** Previous private onionskin decryption key: used to decode CREATE cells * generated by clients that have an older version of our descriptor. */ static crypto_pk_env_t *lastonionkey=NULL; -/** Private "identity key": used to sign directory info and TLS +/** Private server "identity key": used to sign directory info and TLS * certificates. Never changes. */ -static crypto_pk_env_t *identitykey=NULL; -/** Digest of identitykey. */ -static char identitykey_digest[DIGEST_LEN]; +static crypto_pk_env_t *server_identitykey=NULL; +/** Digest of server_identitykey. */ +static char server_identitykey_digest[DIGEST_LEN]; +/** Private client "identity key": used to sign bridges' and clients' + * outbound TLS certificates. Regenerated on startup and on IP address + * change. */ +static crypto_pk_env_t *client_identitykey=NULL; /** Signing key used for v3 directory material; only set for authorities. */ static crypto_pk_env_t *authority_signing_key = NULL; /** Key certificate to authenticate v3 directory material; only set for @@ -61,8 +83,7 @@ static void set_onion_key(crypto_pk_env_t *k) { tor_mutex_acquire(key_lock); - if (onionkey) - crypto_free_pk_env(onionkey); + crypto_free_pk_env(onionkey); onionkey = k; onionkey_set_at = time(NULL); tor_mutex_release(key_lock); @@ -106,32 +127,78 @@ get_onion_key_set_at(void) return onionkey_set_at; } -/** Set the current identity key to k. +/** Set the current server identity key to <b>k</b>. */ void -set_identity_key(crypto_pk_env_t *k) +set_server_identity_key(crypto_pk_env_t *k) { - if (identitykey) - crypto_free_pk_env(identitykey); - identitykey = k; - crypto_pk_get_digest(identitykey, identitykey_digest); + crypto_free_pk_env(server_identitykey); + server_identitykey = k; + crypto_pk_get_digest(server_identitykey, server_identitykey_digest); } -/** Returns the current identity key; requires that the identity key has been - * set. +/** Make sure that we have set up our identity keys to match or not match as + * appropriate, and die with an assertion if we have not. */ +static void +assert_identity_keys_ok(void) +{ + tor_assert(client_identitykey); + if (public_server_mode(get_options())) { + /* assert that we have set the client and server keys to be equal */ + tor_assert(server_identitykey); + tor_assert(0==crypto_pk_cmp_keys(client_identitykey, server_identitykey)); + } else { + /* assert that we have set the client and server keys to be unequal */ + if (server_identitykey) + tor_assert(0!=crypto_pk_cmp_keys(client_identitykey, + server_identitykey)); + } +} + +/** Returns the current server identity key; requires that the key has + * been set, and that we are running as a Tor server. + */ +crypto_pk_env_t * +get_server_identity_key(void) +{ + tor_assert(server_identitykey); + tor_assert(server_mode(get_options())); + assert_identity_keys_ok(); + return server_identitykey; +} + +/** Return true iff the server identity key has been set. */ +int +server_identity_key_is_set(void) +{ + return server_identitykey != NULL; +} + +/** Set the current client identity key to <b>k</b>. + */ +void +set_client_identity_key(crypto_pk_env_t *k) +{ + crypto_free_pk_env(client_identitykey); + client_identitykey = k; +} + +/** Returns the current client identity key for use on outgoing TLS + * connections; requires that the key has been set. */ crypto_pk_env_t * -get_identity_key(void) +get_tlsclient_identity_key(void) { - tor_assert(identitykey); - return identitykey; + tor_assert(client_identitykey); + assert_identity_keys_ok(); + return client_identitykey; } -/** Return true iff the identity key has been set. */ +/** Return true iff the client identity key has been set. */ int -identity_key_is_set(void) +client_identity_key_is_set(void) { - return identitykey != NULL; + return client_identitykey != NULL; } /** Return the key certificate for this v3 (voting) authority, or NULL @@ -201,8 +268,7 @@ rotate_onion_key(void) } log_info(LD_GENERAL, "Rotating onion key"); tor_mutex_acquire(key_lock); - if (lastonionkey) - crypto_free_pk_env(lastonionkey); + crypto_free_pk_env(lastonionkey); lastonionkey = onionkey; onionkey = prkey; now = time(NULL); @@ -331,10 +397,9 @@ load_authority_keyset(int legacy, crypto_pk_env_t **key_out, goto done; } - if (*key_out) - crypto_free_pk_env(*key_out); - if (*cert_out) - authority_cert_free(*cert_out); + crypto_free_pk_env(*key_out); + authority_cert_free(*cert_out); + *key_out = signing_key; *cert_out = parsed; r = 0; @@ -344,10 +409,8 @@ load_authority_keyset(int legacy, crypto_pk_env_t **key_out, done: tor_free(fname); tor_free(cert); - if (signing_key) - crypto_free_pk_env(signing_key); - if (parsed) - authority_cert_free(parsed); + crypto_free_pk_env(signing_key); + authority_cert_free(parsed); return r; } @@ -442,7 +505,9 @@ init_keys(void) key_lock = tor_mutex_new(); /* There are a couple of paths that put us here before */ - if (crypto_global_init(get_options()->HardwareAccel)) { + if (crypto_global_init(get_options()->HardwareAccel, + get_options()->AccelName, + get_options()->AccelDir)) { log_err(LD_BUG, "Unable to initialize OpenSSL. Exiting."); return -1; } @@ -456,9 +521,12 @@ init_keys(void) crypto_free_pk_env(prkey); return -1; } - set_identity_key(prkey); - /* Create a TLS context; default the client nickname to "client". */ - if (tor_tls_context_new(get_identity_key(), MAX_SSL_KEY_LIFETIME) < 0) { + set_client_identity_key(prkey); + /* Create a TLS context. */ + if (tor_tls_context_init(0, + get_tlsclient_identity_key(), + NULL, + MAX_SSL_KEY_LIFETIME) < 0) { log_err(LD_GENERAL,"Error creating TLS context for Tor client."); return -1; } @@ -493,13 +561,28 @@ init_keys(void) } } - /* 1. Read identity key. Make it if none is found. */ + /* 1b. Read identity key. Make it if none is found. */ keydir = get_datadir_fname2("keys", "secret_id_key"); log_info(LD_GENERAL,"Reading/making identity key \"%s\"...",keydir); prkey = init_key_from_file(keydir, 1, LOG_ERR); tor_free(keydir); if (!prkey) return -1; - set_identity_key(prkey); + set_server_identity_key(prkey); + + /* 1c. If we are configured as a bridge, generate a client key; + * otherwise, set the server identity key as our client identity + * key. */ + if (public_server_mode(options)) { + set_client_identity_key(crypto_pk_dup_key(prkey)); /* set above */ + } else { + if (!(prkey = crypto_new_pk_env())) + return -1; + if (crypto_pk_generate_key(prkey)) { + crypto_free_pk_env(prkey); + return -1; + } + set_client_identity_key(prkey); + } /* 2. Read onion key. Make it if none is found. */ keydir = get_datadir_fname2("keys", "secret_onion_key"); @@ -536,18 +619,22 @@ init_keys(void) tor_free(keydir); /* 3. Initialize link key and TLS context. */ - if (tor_tls_context_new(get_identity_key(), MAX_SSL_KEY_LIFETIME) < 0) { + if (tor_tls_context_init(public_server_mode(options), + get_tlsclient_identity_key(), + get_server_identity_key(), + MAX_SSL_KEY_LIFETIME) < 0) { log_err(LD_GENERAL,"Error initializing TLS context"); return -1; } /* 4. Build our router descriptor. */ /* Must be called after keys are initialized. */ mydesc = router_get_my_descriptor(); - if (authdir_mode(options)) { + if (authdir_mode_handles_descs(options, ROUTER_PURPOSE_GENERAL)) { const char *m = NULL; routerinfo_t *ri; /* We need to add our own fingerprint so it gets recognized. */ - if (dirserv_add_own_fingerprint(options->Nickname, get_identity_key())) { + if (dirserv_add_own_fingerprint(options->Nickname, + get_server_identity_key())) { log_err(LD_GENERAL,"Error adding own fingerprint to approved set"); return -1; } @@ -568,7 +655,8 @@ init_keys(void) /* 5. Dump fingerprint to 'fingerprint' */ keydir = get_datadir_fname("fingerprint"); log_info(LD_GENERAL,"Dumping fingerprint to \"%s\"...",keydir); - if (crypto_pk_get_fingerprint(get_identity_key(), fingerprint, 0)<0) { + if (crypto_pk_get_fingerprint(get_server_identity_key(), + fingerprint, 0) < 0) { log_err(LD_GENERAL,"Error computing fingerprint"); tor_free(keydir); return -1; @@ -606,7 +694,7 @@ init_keys(void) return -1; } /* 6b. [authdirserver only] add own key to approved directories. */ - crypto_pk_get_digest(get_identity_key(), digest); + crypto_pk_get_digest(get_server_identity_key(), digest); type = ((options->V1AuthoritativeDir ? V1_AUTHORITY : NO_AUTHORITY) | (options->V2AuthoritativeDir ? V2_AUTHORITY : NO_AUTHORITY) | (options->V3AuthoritativeDir ? V3_AUTHORITY : NO_AUTHORITY) | @@ -762,9 +850,29 @@ consider_testing_reachability(int test_or, int test_dir) routerinfo_t *me = router_get_my_routerinfo(); int orport_reachable = check_whether_orport_reachable(); tor_addr_t addr; + or_options_t *options = get_options(); if (!me) return; + if (routerset_contains_router(options->ExcludeNodes, me) && + options->StrictNodes) { + /* If we've excluded ourself, and StrictNodes is set, we can't test + * ourself. */ + if (test_or || test_dir) { +#define SELF_EXCLUDED_WARN_INTERVAL 3600 + static ratelim_t warning_limit=RATELIM_INIT(SELF_EXCLUDED_WARN_INTERVAL); + char *msg; + if ((msg = rate_limit_log(&warning_limit, approx_time()))) { + log_warn(LD_CIRC, "Can't peform self-tests for this relay: we have " + "listed ourself in ExcludeNodes, and StrictNodes is set. " + "We cannot learn whether we are usable, and will not " + "be able to advertise ourself.%s", msg); + tor_free(msg); + } + } + return; + } + if (test_or && (!orport_reachable || !circuit_enough_testing_circs())) { log_info(LD_CIRC, "Testing %s of my ORPort: %s:%d.", !orport_reachable ? "reachability" : "bandwidth", @@ -793,19 +901,14 @@ consider_testing_reachability(int test_or, int test_dir) void router_orport_found_reachable(void) { - if (!can_reach_or_port) { - routerinfo_t *me = router_get_my_routerinfo(); + routerinfo_t *me = router_get_my_routerinfo(); + if (!can_reach_or_port && me) { log_notice(LD_OR,"Self-testing indicates your ORPort is reachable from " "the outside. Excellent.%s", get_options()->_PublishServerDescriptor != NO_AUTHORITY ? " Publishing server descriptor." : ""); can_reach_or_port = 1; mark_my_descriptor_dirty(); - if (!me) { /* should never happen */ - log_warn(LD_BUG, "ORPort found reachable, but I have no routerinfo " - "yet. Failing to inform controller of success."); - return; - } control_event_server_status(LOG_NOTICE, "REACHABILITY_SUCCEEDED ORADDRESS=%s:%d", me->address, me->or_port); @@ -816,18 +919,13 @@ router_orport_found_reachable(void) void router_dirport_found_reachable(void) { - if (!can_reach_dir_port) { - routerinfo_t *me = router_get_my_routerinfo(); + routerinfo_t *me = router_get_my_routerinfo(); + if (!can_reach_dir_port && me) { log_notice(LD_DIRSERV,"Self-testing indicates your DirPort is reachable " "from the outside. Excellent."); can_reach_dir_port = 1; - if (!me || decide_to_advertise_dirport(get_options(), me->dir_port)) + if (decide_to_advertise_dirport(get_options(), me->dir_port)) mark_my_descriptor_dirty(); - if (!me) { /* should never happen */ - log_warn(LD_BUG, "DirPort found reachable, but I have no routerinfo " - "yet. Failing to inform controller of success."); - return; - } control_event_server_status(LOG_NOTICE, "REACHABILITY_SUCCEEDED DIRADDRESS=%s:%d", me->address, me->dir_port); @@ -962,6 +1060,28 @@ server_mode(or_options_t *options) return (options->ORPort != 0 || options->ORListenAddress); } +/** Return true iff we are trying to be a non-bridge server. + */ +int +public_server_mode(or_options_t *options) +{ + if (!server_mode(options)) return 0; + return (!options->BridgeRelay); +} + +/** Return true iff the combination of options in <b>options</b> and parameters + * in the consensus mean that we don't want to allow exits from circuits + * we got from addresses not known to be servers. */ +int +should_refuse_unknown_exits(or_options_t *options) +{ + if (options->RefuseUnknownExits_ != -1) { + return options->RefuseUnknownExits_; + } else { + return networkstatus_get_param(NULL, "refuseunknownexits", 1, 0, 1); + } +} + /** Remember if we've advertised ourselves to the dirservers. */ static int server_is_advertised=0; @@ -987,10 +1107,10 @@ set_server_advertised(int s) int proxy_mode(or_options_t *options) { - return (options->SocksPort != 0 || options->SocksListenAddress || - options->TransPort != 0 || options->TransListenAddress || - options->NatdPort != 0 || options->NatdListenAddress || - options->DNSPort != 0 || options->DNSListenAddress); + return (options->SocksPort != 0 || + options->TransPort != 0 || + options->NATDPort != 0 || + options->DNSPort != 0); } /** Decide if we're a publishable server. We are a publishable server if: @@ -1124,12 +1244,24 @@ router_compare_to_my_exit_policy(edge_connection_t *conn) desc_routerinfo->exit_policy) != ADDR_POLICY_ACCEPTED; } +/** Return true iff my exit policy is reject *:*. Return -1 if we don't + * have a descriptor */ +int +router_my_exit_policy_is_reject_star(void) +{ + if (!router_get_my_routerinfo()) /* make sure desc_routerinfo exists */ + return -1; + + return desc_routerinfo->policy_is_reject_star; +} + /** Return true iff I'm a server and <b>digest</b> is equal to - * my identity digest. */ + * my server identity key digest. */ int router_digest_is_me(const char *digest) { - return identitykey && !memcmp(identitykey_digest, digest, DIGEST_LEN); + return (server_identitykey && + !memcmp(server_identitykey_digest, digest, DIGEST_LEN)); } /** Return true iff I'm a server and <b>digest</b> is equal to @@ -1219,6 +1351,8 @@ static int router_guess_address_from_dir_headers(uint32_t *guess); int router_pick_published_address(or_options_t *options, uint32_t *addr) { + char buf[INET_NTOA_BUF_LEN]; + struct in_addr a; if (resolve_my_address(LOG_INFO, options, addr, NULL) < 0) { log_info(LD_CONFIG, "Could not determine our address locally. " "Checking if directory headers provide any hints."); @@ -1228,6 +1362,9 @@ router_pick_published_address(or_options_t *options, uint32_t *addr) return -1; } } + a.s_addr = htonl(*addr); + tor_inet_ntoa(&a, buf, sizeof(buf)); + log_info(LD_CONFIG,"Success: chose address '%s'.", buf); return 0; } @@ -1250,7 +1387,7 @@ router_rebuild_descriptor(int force) if (router_pick_published_address(options, &addr) < 0) { /* Stop trying to rebuild our descriptor every second. We'll - * learn that it's time to try again when server_has_changed_ip() + * learn that it's time to try again when ip_address_changed() * marks it dirty. */ desc_clean_since = time(NULL); return -1; @@ -1266,7 +1403,7 @@ router_rebuild_descriptor(int force) ri->cache_info.published_on = time(NULL); ri->onion_pkey = crypto_pk_dup_key(get_onion_key()); /* must invoke from * main thread */ - ri->identity_pkey = crypto_pk_dup_key(get_identity_key()); + ri->identity_pkey = crypto_pk_dup_key(get_server_identity_key()); if (crypto_pk_get_digest(ri->identity_pkey, ri->cache_info.identity_digest)<0) { routerinfo_free(ri); @@ -1283,9 +1420,16 @@ router_rebuild_descriptor(int force) ri->bandwidthcapacity = hibernating ? 0 : rep_hist_bandwidth_assess(); - policies_parse_exit_policy(options->ExitPolicy, &ri->exit_policy, - options->ExitPolicyRejectPrivate, - ri->address); + if (dns_seems_to_be_broken() || has_dns_init_failed()) { + /* DNS is screwed up; don't claim to be an exit. */ + policies_exit_policy_append_reject_star(&ri->exit_policy); + } else { + policies_parse_exit_policy(options->ExitPolicy, &ri->exit_policy, + options->ExitPolicyRejectPrivate, + ri->address, !options->BridgeRelay); + } + ri->policy_is_reject_star = + policy_is_reject_star(ri->exit_policy); if (desc_routerinfo) { /* inherit values */ ri->is_valid = desc_routerinfo->is_valid; @@ -1356,26 +1500,30 @@ router_rebuild_descriptor(int force) ei->cache_info.published_on = ri->cache_info.published_on; memcpy(ei->cache_info.identity_digest, ri->cache_info.identity_digest, DIGEST_LEN); - ei->cache_info.signed_descriptor_body = tor_malloc(8192); - if (extrainfo_dump_to_string(ei->cache_info.signed_descriptor_body, 8192, - ei, get_identity_key()) < 0) { + if (extrainfo_dump_to_string(&ei->cache_info.signed_descriptor_body, + ei, get_server_identity_key()) < 0) { log_warn(LD_BUG, "Couldn't generate extra-info descriptor."); - routerinfo_free(ri); extrainfo_free(ei); - return -1; + ei = NULL; + } else { + ei->cache_info.signed_descriptor_len = + strlen(ei->cache_info.signed_descriptor_body); + router_get_extrainfo_hash(ei->cache_info.signed_descriptor_body, + ei->cache_info.signed_descriptor_digest); } - ei->cache_info.signed_descriptor_len = - strlen(ei->cache_info.signed_descriptor_body); - router_get_extrainfo_hash(ei->cache_info.signed_descriptor_body, - ei->cache_info.signed_descriptor_digest); /* Now finish the router descriptor. */ - memcpy(ri->cache_info.extra_info_digest, - ei->cache_info.signed_descriptor_digest, - DIGEST_LEN); + if (ei) { + memcpy(ri->cache_info.extra_info_digest, + ei->cache_info.signed_descriptor_digest, + DIGEST_LEN); + } else { + /* ri was allocated with tor_malloc_zero, so there is no need to + * zero ri->cache_info.extra_info_digest here. */ + } ri->cache_info.signed_descriptor_body = tor_malloc(8192); if (router_dump_router_to_string(ri->cache_info.signed_descriptor_body, 8192, - ri, get_identity_key())<0) { + ri, get_server_identity_key()) < 0) { log_warn(LD_BUG, "Couldn't generate router descriptor."); routerinfo_free(ri); extrainfo_free(ei); @@ -1390,7 +1538,7 @@ router_rebuild_descriptor(int force) /* Let bridges serve their own descriptors unencrypted, so they can * pass reachability testing. (If they want to be harder to notice, * they can always leave the DirPort off). */ - if (!options->BridgeRelay) + if (ei && !options->BridgeRelay) ei->cache_info.send_unencrypted = 1; router_get_router_hash(ri->cache_info.signed_descriptor_body, @@ -1399,13 +1547,13 @@ router_rebuild_descriptor(int force) routerinfo_set_country(ri); - tor_assert(! routerinfo_incompatible_with_extrainfo(ri, ei, NULL, NULL)); + if (ei) { + tor_assert(! routerinfo_incompatible_with_extrainfo(ri, ei, NULL, NULL)); + } - if (desc_routerinfo) - routerinfo_free(desc_routerinfo); + routerinfo_free(desc_routerinfo); desc_routerinfo = ri; - if (desc_extrainfo) - extrainfo_free(desc_extrainfo); + extrainfo_free(desc_extrainfo); desc_extrainfo = ei; desc_clean_since = time(NULL); @@ -1586,8 +1734,6 @@ router_guess_address_from_dir_headers(uint32_t *guess) return -1; } -extern const char tor_svn_revision[]; /* from tor_main.c */ - /** Set <b>platform</b> (max length <b>len</b>) to a NUL-terminated short * string describing the version of Tor and the operating system we're * currently running on. @@ -1618,6 +1764,7 @@ router_dump_router_to_string(char *s, size_t maxlen, routerinfo_t *router, char digest[DIGEST_LEN]; char published[ISO_TIME_LEN+1]; char fingerprint[FINGERPRINT_LEN+1]; + int has_extra_info_digest; char extra_info_digest[HEX_DIGEST_LEN+1]; size_t onion_pkeylen, identity_pkeylen; size_t written; @@ -1646,7 +1793,7 @@ router_dump_router_to_string(char *s, size_t maxlen, routerinfo_t *router, return -1; } - /* PEM-encode the identity key key */ + /* PEM-encode the identity key */ if (crypto_pk_write_public_key_to_string(router->identity_pkey, &identity_pkey,&identity_pkeylen)<0) { log_warn(LD_BUG,"write identity_pkey to string failed!"); @@ -1668,8 +1815,13 @@ router_dump_router_to_string(char *s, size_t maxlen, routerinfo_t *router, family_line = tor_strdup(""); } - base16_encode(extra_info_digest, sizeof(extra_info_digest), - router->cache_info.extra_info_digest, DIGEST_LEN); + has_extra_info_digest = + ! tor_digest_is_zero(router->cache_info.extra_info_digest); + + if (has_extra_info_digest) { + base16_encode(extra_info_digest, sizeof(extra_info_digest), + router->cache_info.extra_info_digest, DIGEST_LEN); + } /* Generate the easy portion of the router descriptor. */ result = tor_snprintf(s, maxlen, @@ -1680,7 +1832,7 @@ router_dump_router_to_string(char *s, size_t maxlen, routerinfo_t *router, "opt fingerprint %s\n" "uptime %ld\n" "bandwidth %d %d %d\n" - "opt extra-info-digest %s\n%s" + "%s%s%s%s" "onion-key\n%s" "signing-key\n%s" "%s%s%s%s", @@ -1695,7 +1847,9 @@ router_dump_router_to_string(char *s, size_t maxlen, routerinfo_t *router, (int) router->bandwidthrate, (int) router->bandwidthburst, (int) router->bandwidthcapacity, - extra_info_digest, + has_extra_info_digest ? "opt extra-info-digest " : "", + has_extra_info_digest ? extra_info_digest : "", + has_extra_info_digest ? "\n" : "", options->DownloadExtraInfo ? "opt caches-extra-info\n" : "", onion_pkey, identity_pkey, family_line, @@ -1727,9 +1881,7 @@ router_dump_router_to_string(char *s, size_t maxlen, routerinfo_t *router, } /* Write the exit policy to the end of 's'. */ - if (dns_seems_to_be_broken() || has_dns_init_failed() || - !router->exit_policy || !smartlist_len(router->exit_policy)) { - /* DNS is screwed up; don't claim to be an exit. */ + if (!router->exit_policy || !smartlist_len(router->exit_policy)) { strlcat(s+written, "reject *:*\n", maxlen-written); written += strlen("reject *:*\n"); tmpe = NULL; @@ -1752,7 +1904,8 @@ router_dump_router_to_string(char *s, size_t maxlen, routerinfo_t *router, } } - if (written+256 > maxlen) { /* Not enough room for signature. */ + if (written + DIROBJ_MAX_SIG_LEN > maxlen) { + /* Not enough room for signature. */ log_warn(LD_BUG,"not enough room left in descriptor for signature!"); return -1; } @@ -1767,7 +1920,7 @@ router_dump_router_to_string(char *s, size_t maxlen, routerinfo_t *router, note_crypto_pk_op(SIGN_RTR); if (router_append_dirobj_signature(s+written,maxlen-written, - digest,ident_key)<0) { + digest,DIGEST_LEN,ident_key)<0) { log_warn(LD_BUG, "Couldn't sign router descriptor"); return -1; } @@ -1802,11 +1955,62 @@ router_dump_router_to_string(char *s, size_t maxlen, routerinfo_t *router, return (int)written+1; } -/** Write the contents of <b>extrainfo</b> to the <b>maxlen</b>-byte string - * <b>s</b>, signing them with <b>ident_key</b>. Return 0 on success, - * negative on failure. */ +/** Load the contents of <b>filename</b>, find the last line starting with + * <b>end_line</b>, ensure that its timestamp is not more than 25 hours in + * the past or more than 1 hour in the future with respect to <b>now</b>, + * and write the file contents starting with that line to *<b>out</b>. + * Return 1 for success, 0 if the file does not exist, or -1 if the file + * does not contain a line matching these criteria or other failure. */ +static int +load_stats_file(const char *filename, const char *end_line, time_t now, + char **out) +{ + int r = -1; + char *fname = get_datadir_fname(filename); + char *contents, *start = NULL, *tmp, timestr[ISO_TIME_LEN+1]; + time_t written; + switch (file_status(fname)) { + case FN_FILE: + /* X022 Find an alternative to reading the whole file to memory. */ + if ((contents = read_file_to_str(fname, 0, NULL))) { + tmp = strstr(contents, end_line); + /* Find last block starting with end_line */ + while (tmp) { + start = tmp; + tmp = strstr(tmp + 1, end_line); + } + if (!start) + goto notfound; + if (strlen(start) < strlen(end_line) + 1 + sizeof(timestr)) + goto notfound; + strlcpy(timestr, start + 1 + strlen(end_line), sizeof(timestr)); + if (parse_iso_time(timestr, &written) < 0) + goto notfound; + if (written < now - (25*60*60) || written > now + (1*60*60)) + goto notfound; + *out = tor_strdup(start); + r = 1; + } + notfound: + tor_free(contents); + break; + case FN_NOENT: + r = 0; + break; + case FN_ERROR: + case FN_DIR: + default: + break; + } + tor_free(fname); + return r; +} + +/** Write the contents of <b>extrainfo</b> and aggregated statistics to + * *<b>s_out</b>, signing them with <b>ident_key</b>. Return 0 on + * success, negative on failure. */ int -extrainfo_dump_to_string(char *s, size_t maxlen, extrainfo_t *extrainfo, +extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo, crypto_pk_env_t *ident_key) { or_options_t *options = get_options(); @@ -1815,87 +2019,128 @@ extrainfo_dump_to_string(char *s, size_t maxlen, extrainfo_t *extrainfo, char digest[DIGEST_LEN]; char *bandwidth_usage; int result; - size_t len; + static int write_stats_to_extrainfo = 1; + char sig[DIROBJ_MAX_SIG_LEN+1]; + char *s, *pre, *contents, *cp, *s_dup = NULL; + time_t now = time(NULL); + smartlist_t *chunks = smartlist_create(); + extrainfo_t *ei_tmp = NULL; base16_encode(identity, sizeof(identity), extrainfo->cache_info.identity_digest, DIGEST_LEN); format_iso_time(published, extrainfo->cache_info.published_on); - bandwidth_usage = rep_hist_get_bandwidth_lines(1); + bandwidth_usage = rep_hist_get_bandwidth_lines(); - result = tor_snprintf(s, maxlen, - "extra-info %s %s\n" - "published %s\n%s", - extrainfo->nickname, identity, - published, bandwidth_usage); + tor_asprintf(&pre, "extra-info %s %s\npublished %s\n%s", + extrainfo->nickname, identity, + published, bandwidth_usage); tor_free(bandwidth_usage); - if (result<0) - return -1; + smartlist_add(chunks, pre); + + if (options->ExtraInfoStatistics && write_stats_to_extrainfo) { + log_info(LD_GENERAL, "Adding stats to extra-info descriptor."); + if (options->DirReqStatistics && + load_stats_file("stats"PATH_SEPARATOR"dirreq-stats", + "dirreq-stats-end", now, &contents) > 0) { + smartlist_add(chunks, contents); + } + if (options->EntryStatistics && + load_stats_file("stats"PATH_SEPARATOR"entry-stats", + "entry-stats-end", now, &contents) > 0) { + smartlist_add(chunks, contents); + } + if (options->CellStatistics && + load_stats_file("stats"PATH_SEPARATOR"buffer-stats", + "cell-stats-end", now, &contents) > 0) { + smartlist_add(chunks, contents); + } + if (options->ExitPortStatistics && + load_stats_file("stats"PATH_SEPARATOR"exit-stats", + "exit-stats-end", now, &contents) > 0) { + smartlist_add(chunks, contents); + } + } - if (should_record_bridge_info(options)) { - char *geoip_summary = extrainfo_get_client_geoip_summary(time(NULL)); - if (geoip_summary) { - char geoip_start[ISO_TIME_LEN+1]; - format_iso_time(geoip_start, geoip_get_history_start()); - result = tor_snprintf(s+strlen(s), maxlen-strlen(s), - "geoip-start-time %s\n" - "geoip-client-origins %s\n", - geoip_start, geoip_summary); - control_event_clients_seen(geoip_start, geoip_summary); - tor_free(geoip_summary); - if (result<0) - return -1; + if (should_record_bridge_info(options) && write_stats_to_extrainfo) { + const char *bridge_stats = geoip_get_bridge_stats_extrainfo(now); + if (bridge_stats) { + smartlist_add(chunks, tor_strdup(bridge_stats)); } } - len = strlen(s); - strlcat(s+len, "router-signature\n", maxlen-len); - len += strlen(s+len); - if (router_get_extrainfo_hash(s, digest)<0) - return -1; - if (router_append_dirobj_signature(s+len, maxlen-len, digest, ident_key)<0) - return -1; + smartlist_add(chunks, tor_strdup("router-signature\n")); + s = smartlist_join_strings(chunks, "", 0, NULL); + + while (strlen(s) > MAX_EXTRAINFO_UPLOAD_SIZE - DIROBJ_MAX_SIG_LEN) { + /* So long as there are at least two chunks (one for the initial + * extra-info line and one for the router-signature), we can keep removing + * things. */ + if (smartlist_len(chunks) > 2) { + /* We remove the next-to-last element (remember, len-1 is the last + element), since we need to keep the router-signature element. */ + int idx = smartlist_len(chunks) - 2; + char *e = smartlist_get(chunks, idx); + smartlist_del_keeporder(chunks, idx); + log_warn(LD_GENERAL, "We just generated an extra-info descriptor " + "with statistics that exceeds the 50 KB " + "upload limit. Removing last added " + "statistics."); + tor_free(e); + tor_free(s); + s = smartlist_join_strings(chunks, "", 0, NULL); + } else { + log_warn(LD_BUG, "We just generated an extra-info descriptors that " + "exceeds the 50 KB upload limit."); + goto err; + } + } -#ifdef DEBUG_ROUTER_DUMP_ROUTER_TO_STRING - { - char *cp, *s_dup; - extrainfo_t *ei_tmp; - cp = s_dup = tor_strdup(s); - ei_tmp = extrainfo_parse_entry_from_string(cp, NULL, 1, NULL); - if (!ei_tmp) { - log_err(LD_BUG, - "We just generated an extrainfo descriptor we can't parse."); - log_err(LD_BUG, "Descriptor was: <<%s>>", s); - tor_free(s_dup); - return -1; + memset(sig, 0, sizeof(sig)); + if (router_get_extrainfo_hash(s, digest) < 0 || + router_append_dirobj_signature(sig, sizeof(sig), digest, DIGEST_LEN, + ident_key) < 0) { + log_warn(LD_BUG, "Could not append signature to extra-info " + "descriptor."); + goto err; + } + smartlist_add(chunks, tor_strdup(sig)); + tor_free(s); + s = smartlist_join_strings(chunks, "", 0, NULL); + + cp = s_dup = tor_strdup(s); + ei_tmp = extrainfo_parse_entry_from_string(cp, NULL, 1, NULL); + if (!ei_tmp) { + if (write_stats_to_extrainfo) { + log_warn(LD_GENERAL, "We just generated an extra-info descriptor " + "with statistics that we can't parse. Not " + "adding statistics to this or any future " + "extra-info descriptors."); + write_stats_to_extrainfo = 0; + result = extrainfo_dump_to_string(s_out, extrainfo, ident_key); + goto done; + } else { + log_warn(LD_BUG, "We just generated an extrainfo descriptor we " + "can't parse."); + goto err; } - tor_free(s_dup); - extrainfo_free(ei_tmp); } -#endif - return (int)strlen(s)+1; -} + *s_out = s; + s = NULL; /* prevent free */ + result = 0; + goto done; -/** Wrapper function for geoip_get_client_history(). It first discards - * any items in the client history that are too old -- it dumps anything - * more than 48 hours old, but it only considers whether to dump at most - * once per 48 hours, so we aren't too precise to an observer (see also - * r14780). - */ -char * -extrainfo_get_client_geoip_summary(time_t now) -{ - static time_t last_purged_at = 0; - int geoip_purge_interval = 48*60*60; -#ifdef ENABLE_GEOIP_STATS - if (get_options()->DirRecordUsageByCountry) - geoip_purge_interval = get_options()->DirRecordUsageRetainIPs; -#endif - if (now > last_purged_at+geoip_purge_interval) { - geoip_remove_old_clients(now-geoip_purge_interval); - last_purged_at = now; - } - return geoip_get_client_history(now, GEOIP_CLIENT_CONNECT); + err: + result = -1; + + done: + tor_free(s); + SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp)); + smartlist_free(chunks); + tor_free(s_dup); + extrainfo_free(ei_tmp); + + return result; } /** Return true iff <b>s</b> is a legally valid server nickname. */ @@ -2022,26 +2267,17 @@ router_purpose_from_string(const char *s) void router_free_all(void) { - if (onionkey) - crypto_free_pk_env(onionkey); - if (lastonionkey) - crypto_free_pk_env(lastonionkey); - if (identitykey) - crypto_free_pk_env(identitykey); - if (key_lock) - tor_mutex_free(key_lock); - if (desc_routerinfo) - routerinfo_free(desc_routerinfo); - if (desc_extrainfo) - extrainfo_free(desc_extrainfo); - if (authority_signing_key) - crypto_free_pk_env(authority_signing_key); - if (authority_key_certificate) - authority_cert_free(authority_key_certificate); - if (legacy_signing_key) - crypto_free_pk_env(legacy_signing_key); - if (legacy_key_certificate) - authority_cert_free(legacy_key_certificate); + crypto_free_pk_env(onionkey); + crypto_free_pk_env(lastonionkey); + crypto_free_pk_env(server_identitykey); + crypto_free_pk_env(client_identitykey); + tor_mutex_free(key_lock); + routerinfo_free(desc_routerinfo); + extrainfo_free(desc_extrainfo); + crypto_free_pk_env(authority_signing_key); + authority_cert_free(authority_key_certificate); + crypto_free_pk_env(legacy_signing_key); + authority_cert_free(legacy_key_certificate); if (warned_nonexistent_family) { SMARTLIST_FOREACH(warned_nonexistent_family, char *, cp, tor_free(cp)); diff --git a/src/or/router.h b/src/or/router.h new file mode 100644 index 0000000000..5e021f6fed --- /dev/null +++ b/src/or/router.h @@ -0,0 +1,101 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file router.h + * \brief Header file for router.c. + **/ + +#ifndef _TOR_ROUTER_H +#define _TOR_ROUTER_H + +crypto_pk_env_t *get_onion_key(void); +time_t get_onion_key_set_at(void); +void set_server_identity_key(crypto_pk_env_t *k); +crypto_pk_env_t *get_server_identity_key(void); +int server_identity_key_is_set(void); +void set_client_identity_key(crypto_pk_env_t *k); +crypto_pk_env_t *get_tlsclient_identity_key(void); +int client_identity_key_is_set(void); +authority_cert_t *get_my_v3_authority_cert(void); +crypto_pk_env_t *get_my_v3_authority_signing_key(void); +authority_cert_t *get_my_v3_legacy_cert(void); +crypto_pk_env_t *get_my_v3_legacy_signing_key(void); +void dup_onion_keys(crypto_pk_env_t **key, crypto_pk_env_t **last); +void rotate_onion_key(void); +crypto_pk_env_t *init_key_from_file(const char *fname, int generate, + int severity); +void v3_authority_check_key_expiry(void); + +int init_keys(void); + +int check_whether_orport_reachable(void); +int check_whether_dirport_reachable(void); +void consider_testing_reachability(int test_or, int test_dir); +void router_orport_found_reachable(void); +void router_dirport_found_reachable(void); +void router_perform_bandwidth_test(int num_circs, time_t now); + +int authdir_mode(or_options_t *options); +int authdir_mode_v1(or_options_t *options); +int authdir_mode_v2(or_options_t *options); +int authdir_mode_v3(or_options_t *options); +int authdir_mode_any_main(or_options_t *options); +int authdir_mode_any_nonhidserv(or_options_t *options); +int authdir_mode_handles_descs(or_options_t *options, int purpose); +int authdir_mode_publishes_statuses(or_options_t *options); +int authdir_mode_tests_reachability(or_options_t *options); +int authdir_mode_bridge(or_options_t *options); + +int server_mode(or_options_t *options); +int public_server_mode(or_options_t *options); +int advertised_server_mode(void); +int proxy_mode(or_options_t *options); +void consider_publishable_server(int force); +int should_refuse_unknown_exits(or_options_t *options); + +void router_upload_dir_desc_to_dirservers(int force); +void mark_my_descriptor_dirty_if_older_than(time_t when); +void mark_my_descriptor_dirty(void); +void check_descriptor_bandwidth_changed(time_t now); +void check_descriptor_ipaddress_changed(time_t now); +void router_new_address_suggestion(const char *suggestion, + const dir_connection_t *d_conn); +int router_compare_to_my_exit_policy(edge_connection_t *conn); +int router_my_exit_policy_is_reject_star(void); +routerinfo_t *router_get_my_routerinfo(void); +extrainfo_t *router_get_my_extrainfo(void); +const char *router_get_my_descriptor(void); +int router_digest_is_me(const char *digest); +int router_extrainfo_digest_is_me(const char *digest); +int router_is_me(routerinfo_t *router); +int router_fingerprint_is_me(const char *fp); +int router_pick_published_address(or_options_t *options, uint32_t *addr); +int router_rebuild_descriptor(int force); +int router_dump_router_to_string(char *s, size_t maxlen, routerinfo_t *router, + crypto_pk_env_t *ident_key); +int extrainfo_dump_to_string(char **s, extrainfo_t *extrainfo, + crypto_pk_env_t *ident_key); +int is_legal_nickname(const char *s); +int is_legal_nickname_or_hexdigest(const char *s); +int is_legal_hexdigest(const char *s); +void router_get_verbose_nickname(char *buf, const routerinfo_t *router); +void routerstatus_get_verbose_nickname(char *buf, + const routerstatus_t *router); +void router_reset_warnings(void); +void router_reset_reachability(void); +void router_free_all(void); + +const char *router_purpose_to_string(uint8_t p); +uint8_t router_purpose_from_string(const char *s); + +#ifdef ROUTER_PRIVATE +/* Used only by router.c and test.c */ +void get_platform_str(char *platform, size_t len); +#endif + +#endif + diff --git a/src/or/routerlist.c b/src/or/routerlist.c index 5e9c82a1df..f567ccdf3c 100644 --- a/src/or/routerlist.c +++ b/src/or/routerlist.c @@ -12,6 +12,25 @@ **/ #include "or.h" +#include "circuitbuild.h" +#include "config.h" +#include "connection.h" +#include "control.h" +#include "directory.h" +#include "dirserv.h" +#include "dirvote.h" +#include "geoip.h" +#include "hibernate.h" +#include "main.h" +#include "networkstatus.h" +#include "policies.h" +#include "reasons.h" +#include "rendcommon.h" +#include "rendservice.h" +#include "rephist.h" +#include "router.h" +#include "routerlist.h" +#include "routerparse.h" // #define DEBUG_ROUTERLIST @@ -26,8 +45,8 @@ static void mark_all_trusteddirservers_up(void); static int router_nickname_matches(routerinfo_t *router, const char *nickname); static void trusted_dir_server_free(trusted_dir_server_t *ds); static void launch_router_descriptor_downloads(smartlist_t *downloadable, + routerstatus_t *source, time_t now); -static void update_consensus_router_descriptor_downloads(time_t now); static int signed_desc_digest_is_recognized(signed_descriptor_t *desc); static void update_router_have_minimum_dir_info(void); static const char *signed_descriptor_get_body_impl(signed_descriptor_t *desc, @@ -156,21 +175,24 @@ already_have_cert(authority_cert_t *cert) /** Load a bunch of new key certificates from the string <b>contents</b>. If * <b>from_store</b> is true, the certificates are from the cache, and we - * don't need to flush them to disk. If <b>from_store</b> is false, we need - * to flush any changed certificates to disk. Return 0 on success, -1 on - * failure. */ + * don't need to flush them to disk. If <b>flush</b> is true, we need + * to flush any changed certificates to disk now. Return 0 on success, -1 + * if any certs fail to parse. */ int trusted_dirs_load_certs_from_string(const char *contents, int from_store, int flush) { trusted_dir_server_t *ds; const char *s, *eos; + int failure_code = 0; for (s = contents; *s; s = eos) { authority_cert_t *cert = authority_cert_parse_from_string(s, &eos); cert_list_t *cl; - if (!cert) + if (!cert) { + failure_code = -1; break; + } ds = trusteddirserver_get_by_v3_auth_digest( cert->cache_info.identity_digest); log_debug(LD_DIR, "Parsed certificate for %s", @@ -181,15 +203,15 @@ trusted_dirs_load_certs_from_string(const char *contents, int from_store, log_info(LD_DIR, "Skipping %s certificate for %s that we " "already have.", from_store ? "cached" : "downloaded", - ds ? ds->nickname : "??"); + ds ? ds->nickname : "an old or new authority"); /* a duplicate on a download should be treated as a failure, since it * probably means we wanted a different secret key or we are trying to * replace an expired cert that has not in fact been updated. */ if (!from_store) { - log_warn(LD_DIR, "Got a certificate for %s that we already have. " - "Maybe they haven't updated it. Waiting for a while.", - ds ? ds->nickname : "??"); + log_warn(LD_DIR, "Got a certificate for %s, but we already have it. " + "Maybe they haven't updated it. Waiting for a while.", + ds ? ds->nickname : "an old or new authority"); authority_cert_dl_failed(cert->cache_info.identity_digest, 404); } @@ -224,7 +246,7 @@ trusted_dirs_load_certs_from_string(const char *contents, int from_store, ds->dir_port != cert->dir_port)) { char *a = tor_dup_ip(cert->addr); log_notice(LD_DIR, "Updating address for directory authority %s " - "from %s:%d to %s:%d based on in certificate.", + "from %s:%d to %s:%d based on certificate.", ds->nickname, ds->address, (int)ds->dir_port, a, cert->dir_port); tor_free(a); @@ -241,8 +263,11 @@ trusted_dirs_load_certs_from_string(const char *contents, int from_store, if (flush) trusted_dirs_flush_certs_to_disk(); + /* call this even if failure_code is <0, since some certs might have + * succeeded. */ networkstatus_note_certs_arrived(); - return 0; + + return failure_code; } /** Save all v3 key certificates to the cached-certs file. */ @@ -303,7 +328,7 @@ trusted_dirs_remove_old_certs(void) time_t cert_published; if (newest == cert) continue; - expired = ftime_definitely_after(now, cert->expires); + expired = now > cert->expires; cert_published = cert->cache_info.published_on; /* Store expired certs for 48 hours after a newer arrives; */ @@ -415,6 +440,23 @@ authority_cert_dl_failed(const char *id_digest, int status) download_status_failed(&cl->dl_status, status); } +/** Return true iff when we've been getting enough failures when trying to + * download the certificate with ID digest <b>id_digest</b> that we're willing + * to start bugging the user about it. */ +int +authority_cert_dl_looks_uncertain(const char *id_digest) +{ +#define N_AUTH_CERT_DL_FAILURES_TO_BUG_USER 2 + cert_list_t *cl; + int n_failures; + if (!trusted_dir_certs || + !(cl = digestmap_get(trusted_dir_certs, id_digest))) + return 0; + + n_failures = download_status_get_n_failures(&cl->dl_status); + return n_failures >= N_AUTH_CERT_DL_FAILURES_TO_BUG_USER; +} + /** How many times will we try to fetch a certificate before giving up? */ #define MAX_CERT_DL_FAILURES 8 @@ -442,17 +484,18 @@ authority_certs_fetch_missing(networkstatus_t *status, time_t now) list_pending_downloads(pending, DIR_PURPOSE_FETCH_CERTIFICATE, "fp/"); if (status) { - SMARTLIST_FOREACH(status->voters, networkstatus_voter_info_t *, voter, - { - if (tor_digest_is_zero(voter->signing_key_digest)) - continue; /* This authority never signed this consensus, so don't - * go looking for a cert with key digest 0000000000. */ - if (!cache && - !trusteddirserver_get_by_v3_auth_digest(voter->identity_digest)) - continue; /* We are not a cache, and we don't know this authority.*/ - cl = get_cert_list(voter->identity_digest); + SMARTLIST_FOREACH_BEGIN(status->voters, networkstatus_voter_info_t *, + voter) { + if (!smartlist_len(voter->sigs)) + continue; /* This authority never signed this consensus, so don't + * go looking for a cert with key digest 0000000000. */ + if (!cache && + !trusteddirserver_get_by_v3_auth_digest(voter->identity_digest)) + continue; /* We are not a cache, and we don't know this authority.*/ + cl = get_cert_list(voter->identity_digest); + SMARTLIST_FOREACH_BEGIN(voter->sigs, document_signature_t *, sig) { cert = authority_cert_get_by_digests(voter->identity_digest, - voter->signing_key_digest); + sig->signing_key_digest); if (cert) { if (now < cert->expires) download_status_reset(&cl->dl_status); @@ -463,37 +506,36 @@ authority_certs_fetch_missing(networkstatus_t *status, time_t now) !digestmap_get(pending, voter->identity_digest)) { log_notice(LD_DIR, "We're missing a certificate from authority " "with signing key %s: launching request.", - hex_str(voter->signing_key_digest, DIGEST_LEN)); - smartlist_add(missing_digests, voter->identity_digest); + hex_str(sig->signing_key_digest, DIGEST_LEN)); + smartlist_add(missing_digests, sig->identity_digest); } - }); + } SMARTLIST_FOREACH_END(sig); + } SMARTLIST_FOREACH_END(voter); } - SMARTLIST_FOREACH(trusted_dir_servers, trusted_dir_server_t *, ds, - { - int found = 0; - if (!(ds->type & V3_AUTHORITY)) - continue; - if (smartlist_digest_isin(missing_digests, ds->v3_identity_digest)) - continue; - cl = get_cert_list(ds->v3_identity_digest); - SMARTLIST_FOREACH(cl->certs, authority_cert_t *, cert, - { - if (!ftime_definitely_after(now, cert->expires)) { - /* It's not expired, and we weren't looking for something to - * verify a consensus with. Call it done. */ - download_status_reset(&cl->dl_status); - found = 1; - break; - } - }); - if (!found && - download_status_is_ready(&cl->dl_status, now,MAX_CERT_DL_FAILURES) && - !digestmap_get(pending, ds->v3_identity_digest)) { - log_notice(LD_DIR, "No current certificate known for authority %s; " - "launching request.", ds->nickname); - smartlist_add(missing_digests, ds->v3_identity_digest); + SMARTLIST_FOREACH_BEGIN(trusted_dir_servers, trusted_dir_server_t *, ds) { + int found = 0; + if (!(ds->type & V3_AUTHORITY)) + continue; + if (smartlist_digest_isin(missing_digests, ds->v3_identity_digest)) + continue; + cl = get_cert_list(ds->v3_identity_digest); + SMARTLIST_FOREACH(cl->certs, authority_cert_t *, cert, { + if (now < cert->expires) { + /* It's not expired, and we weren't looking for something to + * verify a consensus with. Call it done. */ + download_status_reset(&cl->dl_status); + found = 1; + break; } }); + if (!found && + download_status_is_ready(&cl->dl_status, now,MAX_CERT_DL_FAILURES) && + !digestmap_get(pending, ds->v3_identity_digest)) { + log_info(LD_DIR, "No current certificate known for authority %s; " + "launching request.", ds->nickname); + smartlist_add(missing_digests, ds->v3_identity_digest); + } + } SMARTLIST_FOREACH_END(ds); if (!smartlist_len(missing_digests)) { goto done; @@ -749,8 +791,7 @@ router_rebuild_store(int flags, desc_store_t *store) store->journal_len = 0; store->bytes_dropped = 0; done: - if (signed_descriptors) - smartlist_free(signed_descriptors); + smartlist_free(signed_descriptors); tor_free(fname); tor_free(fname_tmp); if (chunk_list) { @@ -968,8 +1009,9 @@ router_get_trusteddirserver_by_digest(const char *digest) return NULL; } -/** Return the trusted_dir_server_t for the directory authority whose identity - * key hashes to <b>digest</b>, or NULL if no such authority is known. +/** Return the trusted_dir_server_t for the directory authority whose + * v3 identity key hashes to <b>digest</b>, or NULL if no such authority + * is known. */ trusted_dir_server_t * trusteddirserver_get_by_v3_auth_digest(const char *digest) @@ -1028,6 +1070,7 @@ router_pick_trusteddirserver(authority_type_t type, int flags) static routerstatus_t * router_pick_directory_server_impl(authority_type_t type, int flags) { + or_options_t *options = get_options(); routerstatus_t *result; smartlist_t *direct, *tunnel; smartlist_t *trusted_direct, *trusted_tunnel; @@ -1037,10 +1080,13 @@ router_pick_directory_server_impl(authority_type_t type, int flags) int requireother = ! (flags & PDS_ALLOW_SELF); int fascistfirewall = ! (flags & PDS_IGNORE_FASCISTFIREWALL); int prefer_tunnel = (flags & _PDS_PREFER_TUNNELED_DIR_CONNS); + int try_excluding = 1, n_excluded = 0; if (!consensus) return NULL; + retry_without_exclude: + direct = smartlist_create(); tunnel = smartlist_create(); trusted_direct = smartlist_create(); @@ -1072,6 +1118,11 @@ router_pick_directory_server_impl(authority_type_t type, int flags) if ((type & EXTRAINFO_CACHE) && !router_supports_extrainfo(status->identity_digest, 0)) continue; + if (try_excluding && options->ExcludeNodes && + routerset_contains_routerstatus(options->ExcludeNodes, status)) { + ++n_excluded; + continue; + } /* XXXX IP6 proposal 118 */ tor_addr_from_ipv4h(&addr, status->addr); @@ -1089,9 +1140,10 @@ router_pick_directory_server_impl(authority_type_t type, int flags) } SMARTLIST_FOREACH_END(status); if (smartlist_len(tunnel)) { - result = routerstatus_sl_choose_by_bandwidth(tunnel); + result = routerstatus_sl_choose_by_bandwidth(tunnel, WEIGHT_FOR_DIR); } else if (smartlist_len(overloaded_tunnel)) { - result = routerstatus_sl_choose_by_bandwidth(overloaded_tunnel); + result = routerstatus_sl_choose_by_bandwidth(overloaded_tunnel, + WEIGHT_FOR_DIR); } else if (smartlist_len(trusted_tunnel)) { /* FFFF We don't distinguish between trusteds and overloaded trusteds * yet. Maybe one day we should. */ @@ -1099,9 +1151,10 @@ router_pick_directory_server_impl(authority_type_t type, int flags) * is a feature, but it could easily be a bug. -RD */ result = smartlist_choose(trusted_tunnel); } else if (smartlist_len(direct)) { - result = routerstatus_sl_choose_by_bandwidth(direct); + result = routerstatus_sl_choose_by_bandwidth(direct, WEIGHT_FOR_DIR); } else if (smartlist_len(overloaded_direct)) { - result = routerstatus_sl_choose_by_bandwidth(overloaded_direct); + result = routerstatus_sl_choose_by_bandwidth(overloaded_direct, + WEIGHT_FOR_DIR); } else { result = smartlist_choose(trusted_direct); } @@ -1111,6 +1164,15 @@ router_pick_directory_server_impl(authority_type_t type, int flags) smartlist_free(trusted_tunnel); smartlist_free(overloaded_direct); smartlist_free(overloaded_tunnel); + + if (result == NULL && try_excluding && !options->StrictNodes && n_excluded) { + /* If we got no result, and we are excluding nodes, and StrictNodes is + * not set, try again without excluding nodes. */ + try_excluding = 0; + n_excluded = 0; + goto retry_without_exclude; + } + return result; } @@ -1121,6 +1183,7 @@ static routerstatus_t * router_pick_trusteddirserver_impl(authority_type_t type, int flags, int *n_busy_out) { + or_options_t *options = get_options(); smartlist_t *direct, *tunnel; smartlist_t *overloaded_direct, *overloaded_tunnel; routerinfo_t *me = router_get_my_routerinfo(); @@ -1131,10 +1194,13 @@ router_pick_trusteddirserver_impl(authority_type_t type, int flags, const int prefer_tunnel = (flags & _PDS_PREFER_TUNNELED_DIR_CONNS); const int no_serverdesc_fetching =(flags & PDS_NO_EXISTING_SERVERDESC_FETCH); int n_busy = 0; + int try_excluding = 1, n_excluded = 0; if (!trusted_dir_servers) return NULL; + retry_without_exclude: + direct = smartlist_create(); tunnel = smartlist_create(); overloaded_direct = smartlist_create(); @@ -1153,6 +1219,12 @@ router_pick_trusteddirserver_impl(authority_type_t type, int flags, continue; if (requireother && me && router_digest_is_me(d->digest)) continue; + if (try_excluding && options->ExcludeNodes && + routerset_contains_routerstatus(options->ExcludeNodes, + &d->fake_status)) { + ++n_excluded; + continue; + } /* XXXX IP6 proposal 118 */ tor_addr_from_ipv4h(&addr, d->addr); @@ -1199,6 +1271,15 @@ router_pick_trusteddirserver_impl(authority_type_t type, int flags, smartlist_free(tunnel); smartlist_free(overloaded_direct); smartlist_free(overloaded_tunnel); + + if (result == NULL && try_excluding && !options->StrictNodes && n_excluded) { + /* If we got no result, and we are excluding nodes, and StrictNodes is + * not set, try again without excluding nodes. */ + try_excluding = 0; + n_excluded = 0; + goto retry_without_exclude; + } + return result; } @@ -1230,6 +1311,13 @@ mark_all_trusteddirservers_up(void) router_dir_info_changed(); } +/** Return true iff r1 and r2 have the same address and OR port. */ +int +routers_have_same_or_addr(const routerinfo_t *r1, const routerinfo_t *r2) +{ + return r1->addr == r2->addr && r1->or_port == r2->or_port; +} + /** Reset all internal variables used to count failed downloads of network * status objects. */ void @@ -1450,6 +1538,8 @@ routerlist_find_my_routerinfo(void) /** Find a router that's up, that has this IP address, and * that allows exit to this address:port, or return NULL if there * isn't a good one. + * Don't exit enclave to excluded relays -- it wouldn't actually + * hurt anything, but this way there are fewer confused users. */ routerinfo_t * router_find_exact_exit_enclave(const char *address, uint16_t port) @@ -1457,6 +1547,7 @@ router_find_exact_exit_enclave(const char *address, uint16_t port) uint32_t addr; struct in_addr in; tor_addr_t a; + or_options_t *options = get_options(); if (!tor_inet_aton(address, &in)) return NULL; /* it's not an IP already */ @@ -1469,7 +1560,8 @@ router_find_exact_exit_enclave(const char *address, uint16_t port) if (router->addr == addr && router->is_running && compare_tor_addr_to_addr_policy(&a, port, router->exit_policy) == - ADDR_POLICY_ACCEPTED) + ADDR_POLICY_ACCEPTED && + !routerset_contains_router(options->_ExcludeExitNodesUnion, router)) return router; }); return NULL; @@ -1521,6 +1613,29 @@ router_get_advertised_bandwidth_capped(routerinfo_t *router) return result; } +/** When weighting bridges, enforce these values as lower and upper + * bound for believable bandwidth, because there is no way for us + * to verify a bridge's bandwidth currently. */ +#define BRIDGE_MIN_BELIEVABLE_BANDWIDTH 20000 /* 20 kB/sec */ +#define BRIDGE_MAX_BELIEVABLE_BANDWIDTH 100000 /* 100 kB/sec */ + +/** Return the smaller of the router's configured BandwidthRate + * and its advertised capacity, making sure to stay within the + * interval between bridge-min-believe-bw and + * bridge-max-believe-bw. */ +static uint32_t +bridge_get_advertised_bandwidth_bounded(routerinfo_t *router) +{ + uint32_t result = router->bandwidthcapacity; + if (result > router->bandwidthrate) + result = router->bandwidthrate; + if (result > BRIDGE_MAX_BELIEVABLE_BANDWIDTH) + result = BRIDGE_MAX_BELIEVABLE_BANDWIDTH; + else if (result < BRIDGE_MIN_BELIEVABLE_BANDWIDTH) + result = BRIDGE_MIN_BELIEVABLE_BANDWIDTH; + return result; +} + /** Return bw*1000, unless bw*1000 would overflow, in which case return * INT32_MAX. */ static INLINE int32_t @@ -1531,6 +1646,218 @@ kb_to_bytes(uint32_t bw) /** Helper function: * choose a random element of smartlist <b>sl</b>, weighted by + * the advertised bandwidth of each element using the consensus + * bandwidth weights. + * + * If <b>statuses</b> is zero, then <b>sl</b> is a list of + * routerinfo_t's. Otherwise it's a list of routerstatus_t's. + * + * If <b>rule</b>==WEIGHT_FOR_EXIT. we're picking an exit node: consider all + * nodes' bandwidth equally regardless of their Exit status, since there may + * be some in the list because they exit to obscure ports. If + * <b>rule</b>==NO_WEIGHTING, we're picking a non-exit node: weight + * exit-node's bandwidth less depending on the smallness of the fraction of + * Exit-to-total bandwidth. If <b>rule</b>==WEIGHT_FOR_GUARD, we're picking a + * guard node: consider all guard's bandwidth equally. Otherwise, weight + * guards proportionally less. + */ +static void * +smartlist_choose_by_bandwidth_weights(smartlist_t *sl, + bandwidth_weight_rule_t rule, + int statuses) +{ + int64_t weight_scale; + int64_t rand_bw; + double Wg = -1, Wm = -1, We = -1, Wd = -1; + double Wgb = -1, Wmb = -1, Web = -1, Wdb = -1; + double weighted_bw = 0; + double *bandwidths; + double tmp = 0; + unsigned int i; + int have_unknown = 0; /* true iff sl contains element not in consensus. */ + + /* Can't choose exit and guard at same time */ + tor_assert(rule == NO_WEIGHTING || + rule == WEIGHT_FOR_EXIT || + rule == WEIGHT_FOR_GUARD || + rule == WEIGHT_FOR_MID || + rule == WEIGHT_FOR_DIR); + + if (smartlist_len(sl) == 0) { + log_info(LD_CIRC, + "Empty routerlist passed in to consensus weight node " + "selection for rule %s", + bandwidth_weight_rule_to_string(rule)); + return NULL; + } + + weight_scale = circuit_build_times_get_bw_scale(NULL); + + if (rule == WEIGHT_FOR_GUARD) { + Wg = networkstatus_get_bw_weight(NULL, "Wgg", -1); + Wm = networkstatus_get_bw_weight(NULL, "Wgm", -1); /* Bridges */ + We = 0; + Wd = networkstatus_get_bw_weight(NULL, "Wgd", -1); + + Wgb = networkstatus_get_bw_weight(NULL, "Wgb", -1); + Wmb = networkstatus_get_bw_weight(NULL, "Wmb", -1); + Web = networkstatus_get_bw_weight(NULL, "Web", -1); + Wdb = networkstatus_get_bw_weight(NULL, "Wdb", -1); + } else if (rule == WEIGHT_FOR_MID) { + Wg = networkstatus_get_bw_weight(NULL, "Wmg", -1); + Wm = networkstatus_get_bw_weight(NULL, "Wmm", -1); + We = networkstatus_get_bw_weight(NULL, "Wme", -1); + Wd = networkstatus_get_bw_weight(NULL, "Wmd", -1); + + Wgb = networkstatus_get_bw_weight(NULL, "Wgb", -1); + Wmb = networkstatus_get_bw_weight(NULL, "Wmb", -1); + Web = networkstatus_get_bw_weight(NULL, "Web", -1); + Wdb = networkstatus_get_bw_weight(NULL, "Wdb", -1); + } else if (rule == WEIGHT_FOR_EXIT) { + // Guards CAN be exits if they have weird exit policies + // They are d then I guess... + We = networkstatus_get_bw_weight(NULL, "Wee", -1); + Wm = networkstatus_get_bw_weight(NULL, "Wem", -1); /* Odd exit policies */ + Wd = networkstatus_get_bw_weight(NULL, "Wed", -1); + Wg = networkstatus_get_bw_weight(NULL, "Weg", -1); /* Odd exit policies */ + + Wgb = networkstatus_get_bw_weight(NULL, "Wgb", -1); + Wmb = networkstatus_get_bw_weight(NULL, "Wmb", -1); + Web = networkstatus_get_bw_weight(NULL, "Web", -1); + Wdb = networkstatus_get_bw_weight(NULL, "Wdb", -1); + } else if (rule == WEIGHT_FOR_DIR) { + We = networkstatus_get_bw_weight(NULL, "Wbe", -1); + Wm = networkstatus_get_bw_weight(NULL, "Wbm", -1); + Wd = networkstatus_get_bw_weight(NULL, "Wbd", -1); + Wg = networkstatus_get_bw_weight(NULL, "Wbg", -1); + + Wgb = Wmb = Web = Wdb = weight_scale; + } else if (rule == NO_WEIGHTING) { + Wg = Wm = We = Wd = weight_scale; + Wgb = Wmb = Web = Wdb = weight_scale; + } + + if (Wg < 0 || Wm < 0 || We < 0 || Wd < 0 || Wgb < 0 || Wmb < 0 || Wdb < 0 + || Web < 0) { + log_debug(LD_CIRC, + "Got negative bandwidth weights. Defaulting to old selection" + " algorithm."); + return NULL; // Use old algorithm. + } + + Wg /= weight_scale; + Wm /= weight_scale; + We /= weight_scale; + Wd /= weight_scale; + + Wgb /= weight_scale; + Wmb /= weight_scale; + Web /= weight_scale; + Wdb /= weight_scale; + + bandwidths = tor_malloc_zero(sizeof(double)*smartlist_len(sl)); + + // Cycle through smartlist and total the bandwidth. + for (i = 0; i < (unsigned)smartlist_len(sl); ++i) { + int is_exit = 0, is_guard = 0, is_dir = 0, this_bw = 0, is_me = 0; + double weight = 1; + if (statuses) { + routerstatus_t *status = smartlist_get(sl, i); + is_exit = status->is_exit && !status->is_bad_exit; + is_guard = status->is_possible_guard; + is_dir = (status->dir_port != 0); + if (!status->has_bandwidth) { + tor_free(bandwidths); + /* This should never happen, unless all the authorites downgrade + * to 0.2.0 or rogue routerstatuses get inserted into our consensus. */ + log_warn(LD_BUG, + "Consensus is not listing bandwidths. Defaulting back to " + "old router selection algorithm."); + return NULL; + } + this_bw = kb_to_bytes(status->bandwidth); + if (router_digest_is_me(status->identity_digest)) + is_me = 1; + } else { + routerstatus_t *rs; + routerinfo_t *router = smartlist_get(sl, i); + rs = router_get_consensus_status_by_id( + router->cache_info.identity_digest); + is_exit = router->is_exit && !router->is_bad_exit; + is_guard = router->is_possible_guard; + is_dir = (router->dir_port != 0); + if (rs && rs->has_bandwidth) { + this_bw = kb_to_bytes(rs->bandwidth); + } else { /* bridge or other descriptor not in our consensus */ + this_bw = bridge_get_advertised_bandwidth_bounded(router); + have_unknown = 1; + } + if (router_digest_is_me(router->cache_info.identity_digest)) + is_me = 1; + } + if (is_guard && is_exit) { + weight = (is_dir ? Wdb*Wd : Wd); + } else if (is_guard) { + weight = (is_dir ? Wgb*Wg : Wg); + } else if (is_exit) { + weight = (is_dir ? Web*We : We); + } else { // middle + weight = (is_dir ? Wmb*Wm : Wm); + } + + bandwidths[i] = weight*this_bw; + weighted_bw += weight*this_bw; + if (is_me) + sl_last_weighted_bw_of_me = weight*this_bw; + } + + /* XXXX023 this is a kludge to expose these values. */ + sl_last_total_weighted_bw = weighted_bw; + + log_debug(LD_CIRC, "Choosing node for rule %s based on weights " + "Wg=%lf Wm=%lf We=%lf Wd=%lf with total bw %lf", + bandwidth_weight_rule_to_string(rule), + Wg, Wm, We, Wd, weighted_bw); + + /* If there is no bandwidth, choose at random */ + if (DBL_TO_U64(weighted_bw) == 0) { + /* Don't warn when using bridges/relays not in the consensus */ + if (!have_unknown) + log_warn(LD_CIRC, + "Weighted bandwidth is %lf in node selection for rule %s", + weighted_bw, bandwidth_weight_rule_to_string(rule)); + tor_free(bandwidths); + return smartlist_choose(sl); + } + + rand_bw = crypto_rand_uint64(DBL_TO_U64(weighted_bw)); + rand_bw++; /* crypto_rand_uint64() counts from 0, and we need to count + * from 1 below. See bug 1203 for details. */ + + /* Last, count through sl until we get to the element we picked */ + tmp = 0.0; + for (i=0; i < (unsigned)smartlist_len(sl); i++) { + tmp += bandwidths[i]; + if (tmp >= rand_bw) + break; + } + + if (i == (unsigned)smartlist_len(sl)) { + /* This was once possible due to round-off error, but shouldn't be able + * to occur any longer. */ + tor_fragile_assert(); + --i; + log_warn(LD_BUG, "Round-off error in computing bandwidth had an effect on " + " which router we chose. Please tell the developers. " + "%lf " U64_FORMAT " %lf", tmp, U64_PRINTF_ARG(rand_bw), + weighted_bw); + } + tor_free(bandwidths); + return smartlist_get(sl, i); +} + +/** Helper function: + * choose a random element of smartlist <b>sl</b>, weighted by * the advertised bandwidth of each element. * * If <b>statuses</b> is zero, then <b>sl</b> is a list of @@ -1565,11 +1892,24 @@ smartlist_choose_by_bandwidth(smartlist_t *sl, bandwidth_weight_rule_t rule, bitarray_t *guard_bits; int me_idx = -1; + // This function does not support WEIGHT_FOR_DIR + // or WEIGHT_FOR_MID + if (rule == WEIGHT_FOR_DIR || rule == WEIGHT_FOR_MID) { + rule = NO_WEIGHTING; + } + /* Can't choose exit and guard at same time */ tor_assert(rule == NO_WEIGHTING || rule == WEIGHT_FOR_EXIT || rule == WEIGHT_FOR_GUARD); + if (smartlist_len(sl) == 0) { + log_info(LD_CIRC, + "Empty routerlist passed in to old node selection for rule %s", + bandwidth_weight_rule_to_string(rule)); + return NULL; + } + /* First count the total bandwidth weight, and make a list * of each value. <0 means "unknown; no routerinfo." We use the * bits of negative values to remember whether the router was fast (-x)&1 @@ -1594,7 +1934,7 @@ smartlist_choose_by_bandwidth(smartlist_t *sl, bandwidth_weight_rule_t rule, if (status->has_bandwidth) { this_bw = kb_to_bytes(status->bandwidth); } else { /* guess */ - /* XXX022 once consensuses always list bandwidths, we can take + /* XXX023 once consensuses always list bandwidths, we can take * this guessing business out. -RD */ is_known = 0; flags = status->is_fast ? 1 : 0; @@ -1613,14 +1953,14 @@ smartlist_choose_by_bandwidth(smartlist_t *sl, bandwidth_weight_rule_t rule, if (rs && rs->has_bandwidth) { this_bw = kb_to_bytes(rs->bandwidth); } else if (rs) { /* guess; don't trust the descriptor */ - /* XXX022 once consensuses always list bandwidths, we can take + /* XXX023 once consensuses always list bandwidths, we can take * this guessing business out. -RD */ is_known = 0; flags = router->is_fast ? 1 : 0; flags |= is_exit ? 2 : 0; flags |= is_guard ? 4 : 0; } else /* bridge or other descriptor not in our consensus */ - this_bw = router_get_advertised_bandwidth_capped(router); + this_bw = bridge_get_advertised_bandwidth_bounded(router); } if (is_exit) bitarray_set(exit_bits, i); @@ -1628,6 +1968,8 @@ smartlist_choose_by_bandwidth(smartlist_t *sl, bandwidth_weight_rule_t rule, bitarray_set(guard_bits, i); if (is_known) { bandwidths[i] = (int32_t) this_bw; // safe since MAX_BELIEVABLE<INT32_MAX + // XXX this is no longer true! We don't always cap the bw anymore. Can + // a consensus make us overflow?-sh tor_assert(bandwidths[i] >= 0); if (is_guard) total_guard_bw += this_bw; @@ -1691,12 +2033,12 @@ smartlist_choose_by_bandwidth(smartlist_t *sl, bandwidth_weight_rule_t rule, * For detailed derivation of this formula, see * http://archives.seul.org/or/dev/Jul-2007/msg00056.html */ - if (rule == WEIGHT_FOR_EXIT) + if (rule == WEIGHT_FOR_EXIT || !total_exit_bw) exit_weight = 1.0; else exit_weight = 1.0 - all_bw/(3.0*exit_bw); - if (rule == WEIGHT_FOR_GUARD) + if (rule == WEIGHT_FOR_GUARD || !total_guard_bw) guard_weight = 1.0; else guard_weight = 1.0 - all_bw/(3.0*guard_bw); @@ -1727,7 +2069,7 @@ smartlist_choose_by_bandwidth(smartlist_t *sl, bandwidth_weight_rule_t rule, } } - /* XXXX022 this is a kludge to expose these values. */ + /* XXXX023 this is a kludge to expose these values. */ sl_last_total_weighted_bw = total_bw; log_debug(LD_CIRC, "Total weighted bw = "U64_FORMAT @@ -1745,6 +2087,8 @@ smartlist_choose_by_bandwidth(smartlist_t *sl, bandwidth_weight_rule_t rule, /* Almost done: choose a random value from the bandwidth weights. */ rand_bw = crypto_rand_uint64(total_bw); + rand_bw++; /* crypto_rand_uint64() counts from 0, and we need to count + * from 1 below. See bug 1203 for details. */ /* Last, count through sl until we get to the element we picked */ tmp = 0; @@ -1788,26 +2132,34 @@ routerinfo_t * routerlist_sl_choose_by_bandwidth(smartlist_t *sl, bandwidth_weight_rule_t rule) { - return smartlist_choose_by_bandwidth(sl, rule, 0); + routerinfo_t *ret; + if ((ret = smartlist_choose_by_bandwidth_weights(sl, rule, 0))) { + return ret; + } else { + return smartlist_choose_by_bandwidth(sl, rule, 0); + } } /** Choose a random element of status list <b>sl</b>, weighted by * the advertised bandwidth of each status. */ routerstatus_t * -routerstatus_sl_choose_by_bandwidth(smartlist_t *sl) +routerstatus_sl_choose_by_bandwidth(smartlist_t *sl, + bandwidth_weight_rule_t rule) { /* We are choosing neither exit nor guard here. Weight accordingly. */ - return smartlist_choose_by_bandwidth(sl, NO_WEIGHTING, 1); + routerstatus_t *ret; + if ((ret = smartlist_choose_by_bandwidth_weights(sl, rule, 1))) { + return ret; + } else { + return smartlist_choose_by_bandwidth(sl, rule, 1); + } } -/** Return a random running router from the routerlist. If any node - * named in <b>preferred</b> is available, pick one of those. Never +/** Return a random running router from the routerlist. Never * pick a node whose routerinfo is in * <b>excludedsmartlist</b>, or whose routerinfo matches <b>excludedset</b>, - * even if they are the only nodes - * available. If <b>CRN_STRICT_PREFERRED</b> is set in flags, never pick - * any node besides those in <b>preferred</b>. + * even if they are the only nodes available. * If <b>CRN_NEED_UPTIME</b> is set in flags and any router has more than * a minimum uptime, return one of those. * If <b>CRN_NEED_CAPACITY</b> is set in flags, weight your choice by the @@ -1820,8 +2172,7 @@ routerstatus_sl_choose_by_bandwidth(smartlist_t *sl) * node (that is, possibly discounting exit nodes). */ routerinfo_t * -router_choose_random_node(const char *preferred, - smartlist_t *excludedsmartlist, +router_choose_random_node(smartlist_t *excludedsmartlist, routerset_t *excludedset, router_crn_flags_t flags) { @@ -1829,18 +2180,16 @@ router_choose_random_node(const char *preferred, const int need_capacity = (flags & CRN_NEED_CAPACITY) != 0; const int need_guard = (flags & CRN_NEED_GUARD) != 0; const int allow_invalid = (flags & CRN_ALLOW_INVALID) != 0; - const int strict = (flags & CRN_STRICT_PREFERRED) != 0; const int weight_for_exit = (flags & CRN_WEIGHT_AS_EXIT) != 0; - smartlist_t *sl, *excludednodes; + smartlist_t *sl=smartlist_create(), + *excludednodes=smartlist_create(); routerinfo_t *choice = NULL, *r; bandwidth_weight_rule_t rule; tor_assert(!(weight_for_exit && need_guard)); rule = weight_for_exit ? WEIGHT_FOR_EXIT : - (need_guard ? WEIGHT_FOR_GUARD : NO_WEIGHTING); - - excludednodes = smartlist_create(); + (need_guard ? WEIGHT_FOR_GUARD : WEIGHT_FOR_MID); /* Exclude relays that allow single hop exit circuits, if the user * wants to (such relays might be risky) */ @@ -1857,60 +2206,35 @@ router_choose_random_node(const char *preferred, routerlist_add_family(excludednodes, r); } - /* Try the preferred nodes first. Ignore need_uptime and need_capacity - * and need_guard, since the user explicitly asked for these nodes. */ - if (preferred) { - sl = smartlist_create(); - add_nickname_list_to_smartlist(sl,preferred,1); - smartlist_subtract(sl,excludednodes); - if (excludedsmartlist) - smartlist_subtract(sl,excludedsmartlist); - if (excludedset) - routerset_subtract_routers(sl,excludedset); - choice = smartlist_choose(sl); - smartlist_free(sl); - } - if (!choice && !strict) { - /* Then give up on our preferred choices: any node - * will do that has the required attributes. */ - sl = smartlist_create(); - router_add_running_routers_to_smartlist(sl, allow_invalid, - need_uptime, need_capacity, - need_guard); - smartlist_subtract(sl,excludednodes); - if (excludedsmartlist) - smartlist_subtract(sl,excludedsmartlist); - if (excludedset) - routerset_subtract_routers(sl,excludedset); - - if (need_capacity || need_guard) - choice = routerlist_sl_choose_by_bandwidth(sl, rule); - else - choice = smartlist_choose(sl); - - smartlist_free(sl); - if (!choice && (need_uptime || need_capacity || need_guard)) { - /* try once more -- recurse but with fewer restrictions. */ - log_info(LD_CIRC, - "We couldn't find any live%s%s%s routers; falling back " - "to list of all routers.", - need_capacity?", fast":"", - need_uptime?", stable":"", - need_guard?", guard":""); - flags &= ~ (CRN_NEED_UPTIME|CRN_NEED_CAPACITY|CRN_NEED_GUARD); - choice = router_choose_random_node( - NULL, excludedsmartlist, excludedset, flags); - } + router_add_running_routers_to_smartlist(sl, allow_invalid, + need_uptime, need_capacity, + need_guard); + smartlist_subtract(sl,excludednodes); + if (excludedsmartlist) + smartlist_subtract(sl,excludedsmartlist); + if (excludedset) + routerset_subtract_routers(sl,excludedset); + + // Always weight by bandwidth + choice = routerlist_sl_choose_by_bandwidth(sl, rule); + + smartlist_free(sl); + if (!choice && (need_uptime || need_capacity || need_guard)) { + /* try once more -- recurse but with fewer restrictions. */ + log_info(LD_CIRC, + "We couldn't find any live%s%s%s routers; falling back " + "to list of all routers.", + need_capacity?", fast":"", + need_uptime?", stable":"", + need_guard?", guard":""); + flags &= ~ (CRN_NEED_UPTIME|CRN_NEED_CAPACITY|CRN_NEED_GUARD); + choice = router_choose_random_node( + excludedsmartlist, excludedset, flags); } smartlist_free(excludednodes); if (!choice) { - if (strict) { - log_warn(LD_CIRC, "All preferred nodes were down when trying to choose " - "node, and the Strict[...]Nodes option is set. Failing."); - } else { - log_warn(LD_CIRC, - "No available nodes when trying to choose node. Failing."); - } + log_warn(LD_CIRC, + "No available nodes when trying to choose node. Failing."); } return choice; } @@ -2367,6 +2691,9 @@ extrainfo_free(extrainfo_t *extrainfo) static void signed_descriptor_free(signed_descriptor_t *sd) { + if (!sd) + return; + tor_free(sd->signed_descriptor_body); /* XXXX remove this once more bugs go away. */ @@ -2374,12 +2701,15 @@ signed_descriptor_free(signed_descriptor_t *sd) tor_free(sd); } -/** Extract a signed_descriptor_t from a routerinfo, and free the routerinfo. +/** Extract a signed_descriptor_t from a general routerinfo, and free the + * routerinfo. */ static signed_descriptor_t * signed_descriptor_from_routerinfo(routerinfo_t *ri) { - signed_descriptor_t *sd = tor_malloc_zero(sizeof(signed_descriptor_t)); + signed_descriptor_t *sd; + tor_assert(ri->purpose == ROUTER_PURPOSE_GENERAL); + sd = tor_malloc_zero(sizeof(signed_descriptor_t)); memcpy(sd, &(ri->cache_info), sizeof(signed_descriptor_t)); sd->routerlist_index = -1; ri->cache_info.signed_descriptor_body = NULL; @@ -2398,7 +2728,8 @@ _extrainfo_free(void *e) void routerlist_free(routerlist_t *rl) { - tor_assert(rl); + if (!rl) + return; rimap_free(rl->identity_map, NULL); sdmap_free(rl->desc_digest_map, NULL); sdmap_free(rl->desc_by_eid_map, NULL); @@ -2437,46 +2768,6 @@ dump_routerlist_mem_usage(int severity) "In %d old descriptors: "U64_FORMAT" bytes.", smartlist_len(routerlist->routers), U64_PRINTF_ARG(livedescs), smartlist_len(routerlist->old_routers), U64_PRINTF_ARG(olddescs)); - -#if 0 - { - const smartlist_t *networkstatus_v2_list = networkstatus_get_v2_list(); - networkstatus_t *consensus = networkstatus_get_latest_consensus(); - log(severity, LD_DIR, "Now let's look through old_descriptors!"); - SMARTLIST_FOREACH(routerlist->old_routers, signed_descriptor_t *, sd, { - int in_v2 = 0; - int in_v3 = 0; - char published[ISO_TIME_LEN+1]; - char last_valid_until[ISO_TIME_LEN+1]; - char last_served_at[ISO_TIME_LEN+1]; - char id[HEX_DIGEST_LEN+1]; - routerstatus_t *rs; - format_iso_time(published, sd->published_on); - format_iso_time(last_valid_until, sd->last_listed_as_valid_until); - format_iso_time(last_served_at, sd->last_served_at); - base16_encode(id, sizeof(id), sd->identity_digest, DIGEST_LEN); - SMARTLIST_FOREACH(networkstatus_v2_list, networkstatus_v2_t *, ns, - { - rs = networkstatus_v2_find_entry(ns, sd->identity_digest); - if (rs && !memcmp(rs->descriptor_digest, - sd->signed_descriptor_digest, DIGEST_LEN)) { - in_v2 = 1; break; - } - }); - if (consensus) { - rs = networkstatus_vote_find_entry(consensus, sd->identity_digest); - if (rs && !memcmp(rs->descriptor_digest, - sd->signed_descriptor_digest, DIGEST_LEN)) - in_v3 = 1; - } - log(severity, LD_DIR, - "Old descriptor for %s (published %s) %sin v2 ns, %sin v3 " - "consensus. Last valid until %s; last served at %s.", - id, published, in_v2 ? "" : "not ", in_v3 ? "" : "not ", - last_valid_until, last_served_at); - }); - } -#endif } /** Debugging helper: If <b>idx</b> is nonnegative, assert that <b>ri</b> is @@ -2509,6 +2800,7 @@ static void routerlist_insert(routerlist_t *rl, routerinfo_t *ri) { routerinfo_t *ri_old; + signed_descriptor_t *sd_old; { /* XXXX Remove if this slows us down. */ routerinfo_t *ri_generated = router_get_my_routerinfo(); @@ -2518,8 +2810,16 @@ routerlist_insert(routerlist_t *rl, routerinfo_t *ri) ri_old = rimap_set(rl->identity_map, ri->cache_info.identity_digest, ri); tor_assert(!ri_old); - sdmap_set(rl->desc_digest_map, ri->cache_info.signed_descriptor_digest, - &(ri->cache_info)); + + sd_old = sdmap_set(rl->desc_digest_map, + ri->cache_info.signed_descriptor_digest, + &(ri->cache_info)); + if (sd_old) { + rl->desc_store.bytes_dropped += sd_old->signed_descriptor_len; + sdmap_remove(rl->desc_by_eid_map, sd_old->extra_info_digest); + signed_descriptor_free(sd_old); + } + if (!tor_digest_is_zero(ri->cache_info.extra_info_digest)) sdmap_set(rl->desc_by_eid_map, ri->cache_info.extra_info_digest, &ri->cache_info); @@ -2738,6 +3038,7 @@ routerlist_replace(routerlist_t *rl, routerinfo_t *ri_old, routerinfo_t *ri_new) { int idx; + int same_descriptors; routerinfo_t *ri_tmp; extrainfo_t *ei_tmp; @@ -2782,8 +3083,15 @@ routerlist_replace(routerlist_t *rl, routerinfo_t *ri_old, &ri_new->cache_info); } + same_descriptors = ! memcmp(ri_old->cache_info.signed_descriptor_digest, + ri_new->cache_info.signed_descriptor_digest, + DIGEST_LEN); + if (should_cache_old_descriptors() && - ri_old->purpose == ROUTER_PURPOSE_GENERAL) { + ri_old->purpose == ROUTER_PURPOSE_GENERAL && + !same_descriptors) { + /* ri_old is going to become a signed_descriptor_t and go into + * old_routers */ signed_descriptor_t *sd = signed_descriptor_from_routerinfo(ri_old); smartlist_add(rl->old_routers, sd); sd->routerlist_index = smartlist_len(rl->old_routers)-1; @@ -2791,24 +3099,27 @@ routerlist_replace(routerlist_t *rl, routerinfo_t *ri_old, if (!tor_digest_is_zero(sd->extra_info_digest)) sdmap_set(rl->desc_by_eid_map, sd->extra_info_digest, sd); } else { - if (memcmp(ri_old->cache_info.signed_descriptor_digest, - ri_new->cache_info.signed_descriptor_digest, - DIGEST_LEN)) { - /* digests don't match; digestmap_set didn't replace */ + /* We're dropping ri_old. */ + if (!same_descriptors) { + /* digests don't match; The sdmap_set above didn't replace */ sdmap_remove(rl->desc_digest_map, ri_old->cache_info.signed_descriptor_digest); - } - ei_tmp = eimap_remove(rl->extra_info_map, - ri_old->cache_info.extra_info_digest); - if (ei_tmp) { - rl->extrainfo_store.bytes_dropped += - ei_tmp->cache_info.signed_descriptor_len; - extrainfo_free(ei_tmp); - } - if (!tor_digest_is_zero(ri_old->cache_info.extra_info_digest)) { - sdmap_remove(rl->desc_by_eid_map, - ri_old->cache_info.extra_info_digest); + if (memcmp(ri_old->cache_info.extra_info_digest, + ri_new->cache_info.extra_info_digest, DIGEST_LEN)) { + ei_tmp = eimap_remove(rl->extra_info_map, + ri_old->cache_info.extra_info_digest); + if (ei_tmp) { + rl->extrainfo_store.bytes_dropped += + ei_tmp->cache_info.signed_descriptor_len; + extrainfo_free(ei_tmp); + } + } + + if (!tor_digest_is_zero(ri_old->cache_info.extra_info_digest)) { + sdmap_remove(rl->desc_by_eid_map, + ri_old->cache_info.extra_info_digest); + } } rl->desc_store.bytes_dropped += ri_old->cache_info.signed_descriptor_len; routerinfo_free(ri_old); @@ -2846,8 +3157,7 @@ routerlist_reparse_old(routerlist_t *rl, signed_descriptor_t *sd) void routerlist_free_all(void) { - if (routerlist) - routerlist_free(routerlist); + routerlist_free(routerlist); routerlist = NULL; if (warned_nicknames) { SMARTLIST_FOREACH(warned_nicknames, char *, cp, tor_free(cp)); @@ -2940,7 +3250,8 @@ router_add_to_routerlist(routerinfo_t *router, const char **msg, int from_cache, int from_fetch) { const char *id_digest; - int authdir = authdir_mode_handles_descs(get_options(), router->purpose); + or_options_t *options = get_options(); + int authdir = authdir_mode_handles_descs(options, router->purpose); int authdir_believes_valid = 0; routerinfo_t *old_router; networkstatus_t *consensus = networkstatus_get_latest_consensus(); @@ -2954,15 +3265,33 @@ router_add_to_routerlist(routerinfo_t *router, const char **msg, id_digest = router->cache_info.identity_digest; + old_router = router_get_by_digest(id_digest); + /* Make sure that we haven't already got this exact descriptor. */ if (sdmap_get(routerlist->desc_digest_map, router->cache_info.signed_descriptor_digest)) { - log_info(LD_DIR, - "Dropping descriptor that we already have for router '%s'", - router->nickname); - *msg = "Router descriptor was not new."; - routerinfo_free(router); - return ROUTER_WAS_NOT_NEW; + /* If we have this descriptor already and the new descriptor is a bridge + * descriptor, replace it. If we had a bridge descriptor before and the + * new one is not a bridge descriptor, don't replace it. */ + + /* Only members of routerlist->identity_map can be bridges; we don't + * put bridges in old_routers. */ + const int was_bridge = old_router && + old_router->purpose == ROUTER_PURPOSE_BRIDGE; + + if (routerinfo_is_a_configured_bridge(router) && + router->purpose == ROUTER_PURPOSE_BRIDGE && + !was_bridge) { + log_info(LD_DIR, "Replacing non-bridge descriptor with bridge " + "descriptor for router '%s'", router->nickname); + } else { + log_info(LD_DIR, + "Dropping descriptor that we already have for router '%s'", + router->nickname); + *msg = "Router descriptor was not new."; + routerinfo_free(router); + return ROUTER_WAS_NOT_NEW; + } } if (authdir) { @@ -2999,15 +3328,14 @@ router_add_to_routerlist(routerinfo_t *router, const char **msg, SMARTLIST_FOREACH(networkstatus_v2_list, networkstatus_v2_t *, ns, { routerstatus_t *rs = - networkstatus_v2_find_entry(ns, router->cache_info.identity_digest); + networkstatus_v2_find_entry(ns, id_digest); if (rs && !memcmp(rs->descriptor_digest, router->cache_info.signed_descriptor_digest, DIGEST_LEN)) rs->need_to_mirror = 0; }); if (consensus) { - routerstatus_t *rs = networkstatus_vote_find_entry(consensus, - router->cache_info.identity_digest); + routerstatus_t *rs = networkstatus_vote_find_entry(consensus, id_digest); if (rs && !memcmp(rs->descriptor_digest, router->cache_info.signed_descriptor_digest, DIGEST_LEN)) { @@ -3028,14 +3356,26 @@ router_add_to_routerlist(routerinfo_t *router, const char **msg, return ROUTER_NOT_IN_CONSENSUS; } + /* If we're reading a bridge descriptor from our cache, and we don't + * recognize it as one of our currently configured bridges, drop the + * descriptor. Otherwise we could end up using it as one of our entry + * guards even if it isn't in our Bridge config lines. */ + if (router->purpose == ROUTER_PURPOSE_BRIDGE && from_cache && + !authdir_mode_bridge(options) && + !routerinfo_is_a_configured_bridge(router)) { + log_info(LD_DIR, "Dropping bridge descriptor for '%s' because we have " + "no bridge configured at that address.", router->nickname); + *msg = "Router descriptor was not a configured bridge."; + routerinfo_free(router); + return ROUTER_WAS_NOT_WANTED; + } + /* If we have a router with the same identity key, choose the newer one. */ - old_router = rimap_get(routerlist->identity_map, - router->cache_info.identity_digest); if (old_router) { if (!in_consensus && (router->cache_info.published_on <= old_router->cache_info.published_on)) { /* Same key, but old. This one is not listed in the consensus. */ - log_debug(LD_DIR, "Skipping not-new descriptor for router '%s'", + log_debug(LD_DIR, "Not-new descriptor for router '%s'", router->nickname); /* Only journal this desc if we'll be serving it. */ if (!from_cache && should_cache_old_descriptors()) @@ -3049,8 +3389,7 @@ router_add_to_routerlist(routerinfo_t *router, const char **msg, log_debug(LD_DIR, "Replacing entry for router '%s/%s' [%s]", router->nickname, old_router->nickname, hex_str(id_digest,DIGEST_LEN)); - if (router->addr == old_router->addr && - router->or_port == old_router->or_port) { + if (routers_have_same_or_addr(router, old_router)) { /* these carry over when the address and orport are unchanged. */ router->last_reachable = old_router->last_reachable; router->testing_since = old_router->testing_since; @@ -3078,9 +3417,10 @@ router_add_to_routerlist(routerinfo_t *router, const char **msg, /* We haven't seen a router with this identity before. Add it to the end of * the list. */ routerlist_insert(routerlist, router); - if (!from_cache) + if (!from_cache) { signed_desc_append_to_journal(&router->cache_info, &routerlist->desc_store); + } directory_set_dirty(); return ROUTER_ADDED_SUCCESSFULLY; } @@ -3096,7 +3436,7 @@ router_add_extrainfo_to_routerlist(extrainfo_t *ei, const char **msg, int inserted; (void)from_fetch; if (msg) *msg = NULL; - /*XXXX022 Do something with msg */ + /*XXXX023 Do something with msg */ inserted = extrainfo_insert(router_get_routerlist(), ei); @@ -3397,15 +3737,19 @@ routerlist_remove_old_routers(void) /** We just added a new set of descriptors. Take whatever extra steps * we need. */ -static void +void routerlist_descriptors_added(smartlist_t *sl, int from_cache) { tor_assert(sl); control_event_descriptors_changed(sl); - SMARTLIST_FOREACH(sl, routerinfo_t *, ri, + SMARTLIST_FOREACH_BEGIN(sl, routerinfo_t *, ri) { if (ri->purpose == ROUTER_PURPOSE_BRIDGE) learned_bridge_descriptor(ri, from_cache); - ); + if (ri->needs_retest_if_added) { + ri->needs_retest_if_added = 0; + dirserv_single_reachability_test(approx_time(), ri); + } + } SMARTLIST_FOREACH_END(ri); } /** @@ -3736,12 +4080,8 @@ add_trusted_dir_server(const char *nickname, const char *address, if (ent->or_port) ent->fake_status.version_supports_begindir = 1; -/* XX021 - wait until authorities are upgraded */ -#if 0 + ent->fake_status.version_supports_conditional_consensus = 1; -#else - ent->fake_status.version_supports_conditional_consensus = 0; -#endif smartlist_add(trusted_dir_servers, ent); router_dir_info_changed(); @@ -3756,10 +4096,8 @@ authority_cert_free(authority_cert_t *cert) return; tor_free(cert->cache_info.signed_descriptor_body); - if (cert->signing_key) - crypto_free_pk_env(cert->signing_key); - if (cert->identity_key) - crypto_free_pk_env(cert->identity_key); + crypto_free_pk_env(cert->signing_key); + crypto_free_pk_env(cert->identity_key); tor_free(cert); } @@ -3768,6 +4106,9 @@ authority_cert_free(authority_cert_t *cert) static void trusted_dir_server_free(trusted_dir_server_t *ds) { + if (!ds) + return; + tor_free(ds->nickname); tor_free(ds->description); tor_free(ds->address); @@ -3821,7 +4162,7 @@ list_pending_downloads(digestmap_t *result, const char *resource = TO_DIR_CONN(conn)->requested_resource; if (!strcmpstart(resource, prefix)) dir_split_resource_into_fingerprints(resource + p_len, - tmp, NULL, 1, 0); + tmp, NULL, DSR_HEX); } }); SMARTLIST_FOREACH(tmp, char *, d, @@ -3923,7 +4264,7 @@ client_would_use_router(routerstatus_t *rs, time_t now, or_options_t *options) * this number per server. */ #define MIN_DL_PER_REQUEST 4 /** To prevent a single screwy cache from confusing us by selective reply, - * try to split our requests into at least this this many requests. */ + * try to split our requests into at least this many requests. */ #define MIN_REQUESTS 3 /** If we want fewer than this many descriptors, wait until we * want more, or until MAX_CLIENT_INTERVAL_WITHOUT_REQUEST has @@ -3937,7 +4278,8 @@ client_would_use_router(routerstatus_t *rs, time_t now, or_options_t *options) * whether to delay fetching until we have more. If we don't want to delay, * launch one or more requests to the appropriate directory authorities. */ static void -launch_router_descriptor_downloads(smartlist_t *downloadable, time_t now) +launch_router_descriptor_downloads(smartlist_t *downloadable, + routerstatus_t *source, time_t now) { int should_delay = 0, n_downloadable; or_options_t *options = get_options(); @@ -3990,7 +4332,7 @@ launch_router_descriptor_downloads(smartlist_t *downloadable, time_t now) pds_flags |= PDS_NO_EXISTING_SERVERDESC_FETCH; } - n_per_request = (n_downloadable+MIN_REQUESTS-1) / MIN_REQUESTS; + n_per_request = CEIL_DIV(n_downloadable, MIN_REQUESTS); if (n_per_request > MAX_DL_PER_REQUEST) n_per_request = MAX_DL_PER_REQUEST; if (n_per_request < MIN_DL_PER_REQUEST) @@ -4003,11 +4345,11 @@ launch_router_descriptor_downloads(smartlist_t *downloadable, time_t now) log_info(LD_DIR, "Launching %d request%s for %d router%s, %d at a time", - (n_downloadable+n_per_request-1)/n_per_request, + CEIL_DIV(n_downloadable, n_per_request), req_plural, n_downloadable, rtr_plural, n_per_request); smartlist_sort_digests(downloadable); for (i=0; i < n_downloadable; i += n_per_request) { - initiate_descriptor_downloads(NULL, DIR_PURPOSE_FETCH_SERVERDESC, + initiate_descriptor_downloads(source, DIR_PURPOSE_FETCH_SERVERDESC, downloadable, i, i+n_per_request, pds_flags); } @@ -4163,18 +4505,18 @@ update_router_descriptor_cache_downloads_v2(time_t now) digestmap_free(map,NULL); } -/** For any descriptor that we want that's currently listed in the live - * consensus, download it as appropriate. */ -static void -update_consensus_router_descriptor_downloads(time_t now) +/** For any descriptor that we want that's currently listed in + * <b>consensus</b>, download it as appropriate. */ +void +update_consensus_router_descriptor_downloads(time_t now, int is_vote, + networkstatus_t *consensus) { or_options_t *options = get_options(); digestmap_t *map = NULL; smartlist_t *no_longer_old = smartlist_create(); smartlist_t *downloadable = smartlist_create(); + routerstatus_t *source = NULL; int authdir = authdir_mode(options); - networkstatus_t *consensus = - networkstatus_get_reasonably_live_consensus(now); int n_delayed=0, n_have=0, n_would_reject=0, n_wouldnt_use=0, n_inprogress=0, n_in_oldrouters=0; @@ -4183,10 +4525,24 @@ update_consensus_router_descriptor_downloads(time_t now) if (!consensus) goto done; + if (is_vote) { + /* where's it from, so we know whom to ask for descriptors */ + trusted_dir_server_t *ds; + networkstatus_voter_info_t *voter = smartlist_get(consensus->voters, 0); + tor_assert(voter); + ds = trusteddirserver_get_by_v3_auth_digest(voter->identity_digest); + if (ds) + source = &(ds->fake_status); + else + log_warn(LD_DIR, "couldn't lookup source from vote?"); + } + map = digestmap_new(); list_pending_descriptor_downloads(map, 0); - SMARTLIST_FOREACH(consensus->routerstatus_list, routerstatus_t *, rs, + SMARTLIST_FOREACH(consensus->routerstatus_list, void *, rsp, { + routerstatus_t *rs = + is_vote ? &(((vote_routerstatus_t *)rsp)->status) : rsp; signed_descriptor_t *sd; if ((sd = router_get_by_descriptor_digest(rs->descriptor_digest))) { routerinfo_t *ri; @@ -4221,6 +4577,18 @@ update_consensus_router_descriptor_downloads(time_t now) ++n_wouldnt_use; continue; /* We would never use it ourself. */ } + if (is_vote && source) { + char time_bufnew[ISO_TIME_LEN+1]; + char time_bufold[ISO_TIME_LEN+1]; + routerinfo_t *oldrouter = router_get_by_digest(rs->identity_digest); + format_iso_time(time_bufnew, rs->published_on); + if (oldrouter) + format_iso_time(time_bufold, oldrouter->cache_info.published_on); + log_info(LD_DIR, "Learned about %s (%s vs %s) from %s's vote (%s)", + rs->nickname, time_bufnew, + oldrouter ? time_bufold : "none", + source->nickname, oldrouter ? "known" : "unknown"); + } smartlist_add(downloadable, rs->descriptor_digest); }); @@ -4254,7 +4622,7 @@ update_consensus_router_descriptor_downloads(time_t now) smartlist_len(downloadable), n_delayed, n_have, n_in_oldrouters, n_would_reject, n_wouldnt_use, n_inprogress); - launch_router_descriptor_downloads(downloadable, now); + launch_router_descriptor_downloads(downloadable, source, now); digestmap_free(map, NULL); done: @@ -4264,7 +4632,7 @@ update_consensus_router_descriptor_downloads(time_t now) /** How often should we launch a server/authority request to be sure of getting * a guess for our IP? */ -/*XXXX021 this info should come from netinfo cells or something, or we should +/*XXXX023 this info should come from netinfo cells or something, or we should * do this only when we aren't seeing incoming data. see bug 652. */ #define DUMMY_DOWNLOAD_INTERVAL (20*60) @@ -4279,9 +4647,10 @@ update_router_descriptor_downloads(time_t now) if (directory_fetches_dir_info_early(options)) { update_router_descriptor_cache_downloads_v2(now); } - update_consensus_router_descriptor_downloads(now); + update_consensus_router_descriptor_downloads(now, 0, + networkstatus_get_reasonably_live_consensus(now)); - /* XXXX021 we could be smarter here; see notes on bug 652. */ + /* XXXX023 we could be smarter here; see notes on bug 652. */ /* If we're a server that doesn't have a configured address, we rely on * directory fetches to learn when our address changes. So if we haven't * tried to get any routerdescs in a long time, try a dummy fetch now. */ @@ -4416,16 +4785,21 @@ get_dir_info_status_string(void) /** Iterate over the servers listed in <b>consensus</b>, and count how many of * them seem like ones we'd use, and how many of <em>those</em> we have * descriptors for. Store the former in *<b>num_usable</b> and the latter in - * *<b>num_present</b>. */ + * *<b>num_present</b>. If <b>in_set</b> is non-NULL, only consider those + * routers in <b>in_set</b>. + */ static void count_usable_descriptors(int *num_present, int *num_usable, const networkstatus_t *consensus, - or_options_t *options, time_t now) + or_options_t *options, time_t now, + routerset_t *in_set) { *num_present = 0, *num_usable=0; SMARTLIST_FOREACH(consensus->routerstatus_list, routerstatus_t *, rs, { + if (in_set && ! routerset_contains_routerstatus(in_set, rs)) + continue; if (client_would_use_router(rs, now, options)) { ++*num_usable; /* the consensus says we want it. */ if (router_get_by_descriptor_digest(rs->descriptor_digest)) { @@ -4454,7 +4828,7 @@ count_loading_descriptors_progress(void) return 0; /* can't count descriptors if we have no list of them */ count_usable_descriptors(&num_present, &num_usable, - consensus, get_options(), now); + consensus, get_options(), now, NULL); if (num_usable == 0) return 0; /* don't div by 0 */ @@ -4498,22 +4872,39 @@ update_router_have_minimum_dir_info(void) goto done; } - count_usable_descriptors(&num_present, &num_usable, consensus, options, now); + count_usable_descriptors(&num_present, &num_usable, consensus, options, now, + NULL); if (num_present < num_usable/4) { tor_snprintf(dir_info_status, sizeof(dir_info_status), "We have only %d/%d usable descriptors.", num_present, num_usable); res = 0; control_event_bootstrap(BOOTSTRAP_STATUS_REQUESTING_DESCRIPTORS, 0); + goto done; } else if (num_present < 2) { tor_snprintf(dir_info_status, sizeof(dir_info_status), "Only %d descriptor%s here and believed reachable!", num_present, num_present ? "" : "s"); res = 0; - } else { - res = 1; + goto done; + } + + /* Check for entry nodes. */ + if (options->EntryNodes) { + count_usable_descriptors(&num_present, &num_usable, consensus, options, + now, options->EntryNodes); + + if (!num_usable || !num_present) { + tor_snprintf(dir_info_status, sizeof(dir_info_status), + "We have only %d/%d usable entry node descriptors.", + num_present, num_usable); + res = 0; + goto done; + } } + res = 1; + done: if (res && !have_min_dir_info) { log(LOG_NOTICE, LD_DIR, @@ -4526,6 +4917,13 @@ update_router_have_minimum_dir_info(void) log(quiet ? LOG_INFO : LOG_NOTICE, LD_DIR, "Our directory information is no longer up-to-date " "enough to build circuits: %s", dir_info_status); + + /* a) make us log when we next complete a circuit, so we know when Tor + * is back up and usable, and b) disable some activities that Tor + * should only do while circuits are working, like reachability tests + * and fetching bridge descriptors only over circuits. */ + can_complete_circuit = 0; + control_event_client_status(LOG_NOTICE, "NOT_ENOUGH_DIR_INFO"); } have_min_dir_info = res; @@ -4812,8 +5210,8 @@ esc_router_info(routerinfo_t *router) static char *info=NULL; char *esc_contact, *esc_platform; size_t len; - if (info) - tor_free(info); + tor_free(info); + if (!router) return NULL; /* we're exiting; just free the memory we use */ @@ -4948,9 +5346,8 @@ void routerset_refresh_countries(routerset_t *target) { int cc; - if (target->countries) { - bitarray_free(target->countries); - } + bitarray_free(target->countries); + if (!geoip_is_loaded()) { target->countries = NULL; target->n_countries = 0; @@ -5024,7 +5421,9 @@ routerset_parse(routerset_t *target, const char *s, const char *description) return r; } -/** DOCDOC */ +/** Called when we change a node set, or when we reload the geoip list: + * recompute all country info in all configuration node sets and in the + * routerlist. */ void refresh_all_country_info(void) { @@ -5075,7 +5474,7 @@ routerset_needs_geoip(const routerset_t *set) } /** Return true iff there are no entries in <b>set</b>. */ -static int +int routerset_is_empty(const routerset_t *set) { return !set || smartlist_len(set->list) == 0; @@ -5158,10 +5557,11 @@ routerset_contains_routerstatus(const routerset_t *set, routerstatus_t *rs) } /** Add every known routerinfo_t that is a member of <b>routerset</b> to - * <b>out</b>. If <b>running_only</b>, only add the running ones. */ + * <b>out</b>, but never add any that are part of <b>excludeset</b>. + * If <b>running_only</b>, only add the running ones. */ void routerset_get_all_routers(smartlist_t *out, const routerset_t *routerset, - int running_only) + const routerset_t *excludeset, int running_only) { tor_assert(out); if (!routerset || !routerset->list) @@ -5171,12 +5571,13 @@ routerset_get_all_routers(smartlist_t *out, const routerset_t *routerset, if (routerset_is_list(routerset)) { /* No routers are specified by type; all are given by name or digest. - * we can do a lookup in O(len(list)). */ + * we can do a lookup in O(len(routerset)). */ SMARTLIST_FOREACH(routerset->list, const char *, name, { routerinfo_t *router = router_get_by_nickname(name, 1); if (router) { if (!running_only || router->is_running) - smartlist_add(out, router); + if (!routerset_contains_router(excludeset, router)) + smartlist_add(out, router); } }); } else { @@ -5186,15 +5587,21 @@ routerset_get_all_routers(smartlist_t *out, const routerset_t *routerset, SMARTLIST_FOREACH(rl->routers, routerinfo_t *, router, { if (running_only && !router->is_running) continue; - if (routerset_contains_router(routerset, router)) + if (routerset_contains_router(routerset, router) && + !routerset_contains_router(excludeset, router)) smartlist_add(out, router); }); } } -/** Add to <b>target</b> every routerinfo_t from <b>source</b> that is in - * <b>include</b>, but not excluded in a more specific fashion by - * <b>exclude</b>. If <b>running_only</b>, only include running routers. +#if 0 +/** Add to <b>target</b> every routerinfo_t from <b>source</b> except: + * + * 1) Don't add it if <b>include</b> is non-empty and the relay isn't in + * <b>include</b>; and + * 2) Don't add it if <b>exclude</b> is non-empty and the relay is + * excluded in a more specific fashion by <b>exclude</b>. + * 3) If <b>running_only</b>, don't add non-running routers. */ void routersets_get_disjunction(smartlist_t *target, @@ -5218,6 +5625,7 @@ routersets_get_disjunction(smartlist_t *target, } }); } +#endif /** Remove every routerinfo_t from <b>lst</b> that is in <b>routerset</b>. */ void @@ -5249,10 +5657,15 @@ routerset_to_string(const routerset_t *set) int routerset_equal(const routerset_t *old, const routerset_t *new) { - if (old == NULL && new == NULL) + if (routerset_is_empty(old) && routerset_is_empty(new)) { + /* Two empty sets are equal */ return 1; - else if (old == NULL || new == NULL) + } else if (routerset_is_empty(old) || routerset_is_empty(new)) { + /* An empty set is equal to nothing else. */ return 0; + } + tor_assert(old != NULL); + tor_assert(new != NULL); if (smartlist_len(old->list) != smartlist_len(new->list)) return 0; @@ -5264,35 +5677,15 @@ routerset_equal(const routerset_t *old, const routerset_t *new) }); return 1; - -#if 0 - /* XXXX: This won't work if the names/digests are identical but in a - different order. Checking for exact equality would be heavy going, - is it worth it? -RH*/ - /* This code is totally bogus; sizeof doesn't work even remotely like this - * code seems to think. Let's revert to a string-based comparison for - * now. -NM*/ - if (sizeof(old->names) != sizeof(new->names)) - return 0; - - if (memcmp(old->names,new->names,sizeof(new->names))) - return 0; - if (sizeof(old->digests) != sizeof(new->digests)) - return 0; - if (memcmp(old->digests,new->digests,sizeof(new->digests))) - return 0; - if (sizeof(old->countries) != sizeof(new->countries)) - return 0; - if (memcmp(old->countries,new->countries,sizeof(new->countries))) - return 0; - return 1; -#endif } /** Free all storage held in <b>routerset</b>. */ void routerset_free(routerset_t *routerset) { + if (!routerset) + return; + SMARTLIST_FOREACH(routerset->list, char *, cp, tor_free(cp)); smartlist_free(routerset->list); SMARTLIST_FOREACH(routerset->policies, addr_policy_t *, p, @@ -5303,8 +5696,7 @@ routerset_free(routerset_t *routerset) strmap_free(routerset->names, NULL); digestmap_free(routerset->digests, NULL); - if (routerset->countries) - bitarray_free(routerset->countries); + bitarray_free(routerset->countries); tor_free(routerset); } diff --git a/src/or/routerlist.h b/src/or/routerlist.h new file mode 100644 index 0000000000..fec18705b3 --- /dev/null +++ b/src/or/routerlist.h @@ -0,0 +1,200 @@ +/* Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file routerlist.h + * \brief Header file for routerlist.c. + **/ + +#ifndef _TOR_ROUTERLIST_H +#define _TOR_ROUTERLIST_H + +int get_n_authorities(authority_type_t type); +int trusted_dirs_reload_certs(void); +int trusted_dirs_load_certs_from_string(const char *contents, int from_store, + int flush); +void trusted_dirs_flush_certs_to_disk(void); +authority_cert_t *authority_cert_get_newest_by_id(const char *id_digest); +authority_cert_t *authority_cert_get_by_sk_digest(const char *sk_digest); +authority_cert_t *authority_cert_get_by_digests(const char *id_digest, + const char *sk_digest); +void authority_cert_get_all(smartlist_t *certs_out); +void authority_cert_dl_failed(const char *id_digest, int status); +void authority_certs_fetch_missing(networkstatus_t *status, time_t now); +int router_reload_router_list(void); +int authority_cert_dl_looks_uncertain(const char *id_digest); +smartlist_t *router_get_trusted_dir_servers(void); + +routerstatus_t *router_pick_directory_server(authority_type_t type, int flags); +trusted_dir_server_t *router_get_trusteddirserver_by_digest(const char *d); +trusted_dir_server_t *trusteddirserver_get_by_v3_auth_digest(const char *d); +routerstatus_t *router_pick_trusteddirserver(authority_type_t type, int flags); +int router_get_my_share_of_directory_requests(double *v2_share_out, + double *v3_share_out); +void router_reset_status_download_failures(void); +void routerlist_add_family(smartlist_t *sl, routerinfo_t *router); +int routers_in_same_family(routerinfo_t *r1, routerinfo_t *r2); +int routers_have_same_or_addr(const routerinfo_t *r1, const routerinfo_t *r2); +void add_nickname_list_to_smartlist(smartlist_t *sl, const char *list, + int must_be_running); +int router_nickname_is_in_list(routerinfo_t *router, const char *list); +routerinfo_t *routerlist_find_my_routerinfo(void); +routerinfo_t *router_find_exact_exit_enclave(const char *address, + uint16_t port); +int router_is_unreliable(routerinfo_t *router, int need_uptime, + int need_capacity, int need_guard); +uint32_t router_get_advertised_bandwidth(routerinfo_t *router); +uint32_t router_get_advertised_bandwidth_capped(routerinfo_t *router); + +routerinfo_t *routerlist_sl_choose_by_bandwidth(smartlist_t *sl, + bandwidth_weight_rule_t rule); +routerstatus_t *routerstatus_sl_choose_by_bandwidth(smartlist_t *sl, + bandwidth_weight_rule_t rule); + +routerinfo_t *router_choose_random_node(smartlist_t *excludedsmartlist, + struct routerset_t *excludedset, + router_crn_flags_t flags); + +routerinfo_t *router_get_by_nickname(const char *nickname, + int warn_if_unnamed); +int router_digest_version_as_new_as(const char *digest, const char *cutoff); +int router_digest_is_trusted_dir_type(const char *digest, + authority_type_t type); +#define router_digest_is_trusted_dir(d) \ + router_digest_is_trusted_dir_type((d), NO_AUTHORITY) + +int router_addr_is_trusted_dir(uint32_t addr); +int hexdigest_to_digest(const char *hexdigest, char *digest); +routerinfo_t *router_get_by_hexdigest(const char *hexdigest); +routerinfo_t *router_get_by_digest(const char *digest); +signed_descriptor_t *router_get_by_descriptor_digest(const char *digest); +signed_descriptor_t *router_get_by_extrainfo_digest(const char *digest); +signed_descriptor_t *extrainfo_get_by_descriptor_digest(const char *digest); +const char *signed_descriptor_get_body(signed_descriptor_t *desc); +const char *signed_descriptor_get_annotations(signed_descriptor_t *desc); +routerlist_t *router_get_routerlist(void); +void routerinfo_free(routerinfo_t *router); +void extrainfo_free(extrainfo_t *extrainfo); +void routerlist_free(routerlist_t *rl); +void dump_routerlist_mem_usage(int severity); +void routerlist_remove(routerlist_t *rl, routerinfo_t *ri, int make_old, + time_t now); +void routerlist_free_all(void); +void routerlist_reset_warnings(void); +void router_set_status(const char *digest, int up); + +static int WRA_WAS_ADDED(was_router_added_t s); +static int WRA_WAS_OUTDATED(was_router_added_t s); +static int WRA_WAS_REJECTED(was_router_added_t s); +/** Return true iff the descriptor was added. It might still be necessary to + * check whether the descriptor generator should be notified. + */ +static INLINE int +WRA_WAS_ADDED(was_router_added_t s) { + return s == ROUTER_ADDED_SUCCESSFULLY || s == ROUTER_ADDED_NOTIFY_GENERATOR; +} +/** Return true iff the descriptor was not added because it was either: + * - not in the consensus + * - neither in the consensus nor in any networkstatus document + * - it was outdated. + */ +static INLINE int WRA_WAS_OUTDATED(was_router_added_t s) +{ + return (s == ROUTER_WAS_NOT_NEW || + s == ROUTER_NOT_IN_CONSENSUS || + s == ROUTER_NOT_IN_CONSENSUS_OR_NETWORKSTATUS); +} +static INLINE int WRA_WAS_REJECTED(was_router_added_t s) +{ + return (s == ROUTER_AUTHDIR_REJECTS); +} +was_router_added_t router_add_to_routerlist(routerinfo_t *router, + const char **msg, + int from_cache, + int from_fetch); +was_router_added_t router_add_extrainfo_to_routerlist( + extrainfo_t *ei, const char **msg, + int from_cache, int from_fetch); +void routerlist_descriptors_added(smartlist_t *sl, int from_cache); +void routerlist_remove_old_routers(void); +int router_load_single_router(const char *s, uint8_t purpose, int cache, + const char **msg); +int router_load_routers_from_string(const char *s, const char *eos, + saved_location_t saved_location, + smartlist_t *requested_fingerprints, + int descriptor_digests, + const char *prepend_annotations); +void router_load_extrainfo_from_string(const char *s, const char *eos, + saved_location_t saved_location, + smartlist_t *requested_fingerprints, + int descriptor_digests); + +void routerlist_retry_directory_downloads(time_t now); +int router_exit_policy_all_routers_reject(uint32_t addr, uint16_t port, + int need_uptime); +int router_exit_policy_rejects_all(routerinfo_t *router); +trusted_dir_server_t *add_trusted_dir_server(const char *nickname, + const char *address, + uint16_t dir_port, uint16_t or_port, + const char *digest, const char *v3_auth_digest, + authority_type_t type); +void authority_cert_free(authority_cert_t *cert); +void clear_trusted_dir_servers(void); +int any_trusted_dir_is_v1_authority(void); +void update_consensus_router_descriptor_downloads(time_t now, int is_vote, + networkstatus_t *consensus); +void update_router_descriptor_downloads(time_t now); +void update_extrainfo_downloads(time_t now); +int router_have_minimum_dir_info(void); +void router_dir_info_changed(void); +const char *get_dir_info_status_string(void); +int count_loading_descriptors_progress(void); +void router_reset_descriptor_download_failures(void); +int router_differences_are_cosmetic(routerinfo_t *r1, routerinfo_t *r2); +int routerinfo_incompatible_with_extrainfo(routerinfo_t *ri, extrainfo_t *ei, + signed_descriptor_t *sd, + const char **msg); + +void routerlist_assert_ok(routerlist_t *rl); +const char *esc_router_info(routerinfo_t *router); +void routers_sort_by_identity(smartlist_t *routers); + +routerset_t *routerset_new(void); +int routerset_parse(routerset_t *target, const char *s, + const char *description); +void routerset_union(routerset_t *target, const routerset_t *source); +int routerset_is_list(const routerset_t *set); +int routerset_needs_geoip(const routerset_t *set); +int routerset_is_empty(const routerset_t *set); +int routerset_contains_router(const routerset_t *set, routerinfo_t *ri); +int routerset_contains_routerstatus(const routerset_t *set, + routerstatus_t *rs); +int routerset_contains_extendinfo(const routerset_t *set, + const extend_info_t *ei); +void routerset_get_all_routers(smartlist_t *out, const routerset_t *routerset, + const routerset_t *excludeset, + int running_only); +#if 0 +void routersets_get_disjunction(smartlist_t *target, const smartlist_t *source, + const routerset_t *include, + const routerset_t *exclude, int running_only); +#endif +void routerset_subtract_routers(smartlist_t *out, + const routerset_t *routerset); +char *routerset_to_string(const routerset_t *routerset); +void routerset_refresh_countries(routerset_t *target); +int routerset_equal(const routerset_t *old, const routerset_t *new); +void routerset_free(routerset_t *routerset); +void routerinfo_set_country(routerinfo_t *ri); +void routerlist_refresh_countries(void); +void refresh_all_country_info(void); + +int hid_serv_get_responsible_directories(smartlist_t *responsible_dirs, + const char *id); +int hid_serv_acting_as_directory(void); +int hid_serv_responsible_for_desc_id(const char *id); + +#endif + diff --git a/src/or/routerparse.c b/src/or/routerparse.c index 1faa177edf..d0138e638b 100644 --- a/src/or/routerparse.c +++ b/src/or/routerparse.c @@ -10,7 +10,21 @@ **/ #include "or.h" +#include "config.h" +#include "circuitbuild.h" +#include "dirserv.h" +#include "dirvote.h" +#include "policies.h" +#include "rendcommon.h" +#include "router.h" +#include "routerlist.h" #include "memarea.h" +#include "microdesc.h" +#include "networkstatus.h" +#include "rephist.h" +#include "routerparse.h" +#undef log +#include <math.h> /****************************************************************************/ @@ -55,6 +69,7 @@ typedef enum { K_S, K_V, K_W, + K_M, K_EVENTDNS, K_EXTRA_INFO, K_EXTRA_INFO_DIGEST, @@ -62,6 +77,31 @@ typedef enum { K_HIDDEN_SERVICE_DIR, K_ALLOW_SINGLE_HOP_EXITS, + K_DIRREQ_END, + K_DIRREQ_V2_IPS, + K_DIRREQ_V3_IPS, + K_DIRREQ_V2_REQS, + K_DIRREQ_V3_REQS, + K_DIRREQ_V2_SHARE, + K_DIRREQ_V3_SHARE, + K_DIRREQ_V2_RESP, + K_DIRREQ_V3_RESP, + K_DIRREQ_V2_DIR, + K_DIRREQ_V3_DIR, + K_DIRREQ_V2_TUN, + K_DIRREQ_V3_TUN, + K_ENTRY_END, + K_ENTRY_IPS, + K_CELL_END, + K_CELL_PROCESSED, + K_CELL_QUEUED, + K_CELL_TIME, + K_CELL_CIRCS, + K_EXIT_END, + K_EXIT_WRITTEN, + K_EXIT_READ, + K_EXIT_OPENED, + K_DIR_KEY_CERTIFICATE_VERSION, K_DIR_IDENTITY_KEY, K_DIR_KEY_PUBLISHED, @@ -78,13 +118,18 @@ typedef enum { K_KNOWN_FLAGS, K_PARAMS, + K_BW_WEIGHTS, K_VOTE_DIGEST, K_CONSENSUS_DIGEST, + K_ADDITIONAL_DIGEST, + K_ADDITIONAL_SIGNATURE, K_CONSENSUS_METHODS, K_CONSENSUS_METHOD, K_LEGACY_DIR_KEY, + K_DIRECTORY_FOOTER, A_PURPOSE, + A_LAST_LISTED, _A_UNKNOWN, R_RENDEZVOUS_SERVICE_DESCRIPTOR, @@ -122,7 +167,7 @@ typedef enum { * type. * * This structure is only allocated in memareas; do not allocate it on - * the heap, or token_free() won't work. + * the heap, or token_clear() won't work. */ typedef struct directory_token_t { directory_keyword tp; /**< Type of the token. */ @@ -258,6 +303,31 @@ static token_rule_t extrainfo_token_table[] = { T0N("opt", K_OPT, CONCAT_ARGS, OBJ_OK ), T01("read-history", K_READ_HISTORY, ARGS, NO_OBJ ), T01("write-history", K_WRITE_HISTORY, ARGS, NO_OBJ ), + T01("dirreq-stats-end", K_DIRREQ_END, ARGS, NO_OBJ ), + T01("dirreq-v2-ips", K_DIRREQ_V2_IPS, ARGS, NO_OBJ ), + T01("dirreq-v3-ips", K_DIRREQ_V3_IPS, ARGS, NO_OBJ ), + T01("dirreq-v2-reqs", K_DIRREQ_V2_REQS, ARGS, NO_OBJ ), + T01("dirreq-v3-reqs", K_DIRREQ_V3_REQS, ARGS, NO_OBJ ), + T01("dirreq-v2-share", K_DIRREQ_V2_SHARE, ARGS, NO_OBJ ), + T01("dirreq-v3-share", K_DIRREQ_V3_SHARE, ARGS, NO_OBJ ), + T01("dirreq-v2-resp", K_DIRREQ_V2_RESP, ARGS, NO_OBJ ), + T01("dirreq-v3-resp", K_DIRREQ_V3_RESP, ARGS, NO_OBJ ), + T01("dirreq-v2-direct-dl", K_DIRREQ_V2_DIR, ARGS, NO_OBJ ), + T01("dirreq-v3-direct-dl", K_DIRREQ_V3_DIR, ARGS, NO_OBJ ), + T01("dirreq-v2-tunneled-dl", K_DIRREQ_V2_TUN, ARGS, NO_OBJ ), + T01("dirreq-v3-tunneled-dl", K_DIRREQ_V3_TUN, ARGS, NO_OBJ ), + T01("entry-stats-end", K_ENTRY_END, ARGS, NO_OBJ ), + T01("entry-ips", K_ENTRY_IPS, ARGS, NO_OBJ ), + T01("cell-stats-end", K_CELL_END, ARGS, NO_OBJ ), + T01("cell-processed-cells", K_CELL_PROCESSED, ARGS, NO_OBJ ), + T01("cell-queued-cells", K_CELL_QUEUED, ARGS, NO_OBJ ), + T01("cell-time-in-queue", K_CELL_TIME, ARGS, NO_OBJ ), + T01("cell-circuits-per-decile", K_CELL_CIRCS, ARGS, NO_OBJ ), + T01("exit-stats-end", K_EXIT_END, ARGS, NO_OBJ ), + T01("exit-kibibytes-written", K_EXIT_WRITTEN, ARGS, NO_OBJ ), + T01("exit-kibibytes-read", K_EXIT_READ, ARGS, NO_OBJ ), + T01("exit-streams-opened", K_EXIT_OPENED, ARGS, NO_OBJ ), + T1_START( "extra-info", K_EXTRA_INFO, GE(2), NO_OBJ ), END_OF_TABLE @@ -267,10 +337,11 @@ static token_rule_t extrainfo_token_table[] = { * documents. */ static token_rule_t rtrstatus_token_table[] = { T01("p", K_P, CONCAT_ARGS, NO_OBJ ), - T1( "r", K_R, GE(8), NO_OBJ ), + T1( "r", K_R, GE(7), NO_OBJ ), T1( "s", K_S, ARGS, NO_OBJ ), T01("v", K_V, CONCAT_ARGS, NO_OBJ ), T01("w", K_W, ARGS, NO_OBJ ), + T0N("m", K_M, CONCAT_ARGS, NO_OBJ ), T0N("opt", K_OPT, CONCAT_ARGS, OBJ_OK ), END_OF_TABLE }; @@ -375,7 +446,7 @@ static token_rule_t client_keys_token_table[] = { /** List of tokens allowed in V3 networkstatus votes. */ static token_rule_t networkstatus_token_table[] = { - T1("network-status-version", K_NETWORK_STATUS_VERSION, + T1_START("network-status-version", K_NETWORK_STATUS_VERSION, GE(1), NO_OBJ ), T1("vote-status", K_VOTE_STATUS, GE(1), NO_OBJ ), T1("published", K_PUBLISHED, CONCAT_ARGS, NO_OBJ ), @@ -403,7 +474,7 @@ static token_rule_t networkstatus_token_table[] = { /** List of tokens allowed in V3 networkstatus consensuses. */ static token_rule_t networkstatus_consensus_token_table[] = { - T1("network-status-version", K_NETWORK_STATUS_VERSION, + T1_START("network-status-version", K_NETWORK_STATUS_VERSION, GE(1), NO_OBJ ), T1("vote-status", K_VOTE_STATUS, GE(1), NO_OBJ ), T1("valid-after", K_VALID_AFTER, CONCAT_ARGS, NO_OBJ ), @@ -430,17 +501,29 @@ static token_rule_t networkstatus_consensus_token_table[] = { /** List of tokens allowable in the footer of v1/v2 directory/networkstatus * footers. */ static token_rule_t networkstatus_vote_footer_token_table[] = { - T( "directory-signature", K_DIRECTORY_SIGNATURE, GE(2), NEED_OBJ ), + T01("directory-footer", K_DIRECTORY_FOOTER, NO_ARGS, NO_OBJ ), + T01("bandwidth-weights", K_BW_WEIGHTS, ARGS, NO_OBJ ), + T( "directory-signature", K_DIRECTORY_SIGNATURE, GE(2), NEED_OBJ ), END_OF_TABLE }; /** List of tokens allowable in detached networkstatus signature documents. */ static token_rule_t networkstatus_detached_signature_token_table[] = { T1_START("consensus-digest", K_CONSENSUS_DIGEST, GE(1), NO_OBJ ), + T("additional-digest", K_ADDITIONAL_DIGEST,GE(3), NO_OBJ ), T1("valid-after", K_VALID_AFTER, CONCAT_ARGS, NO_OBJ ), T1("fresh-until", K_FRESH_UNTIL, CONCAT_ARGS, NO_OBJ ), T1("valid-until", K_VALID_UNTIL, CONCAT_ARGS, NO_OBJ ), - T1N("directory-signature", K_DIRECTORY_SIGNATURE, GE(2), NEED_OBJ ), + T("additional-signature", K_ADDITIONAL_SIGNATURE, GE(4), NEED_OBJ ), + T1N("directory-signature", K_DIRECTORY_SIGNATURE, GE(2), NEED_OBJ ), + END_OF_TABLE +}; + +static token_rule_t microdesc_token_table[] = { + T1_START("onion-key", K_ONION_KEY, NO_ARGS, NEED_KEY_1024), + T01("family", K_FAMILY, ARGS, NO_OBJ ), + T01("p", K_P, CONCAT_ARGS, NO_OBJ ), + A01("@last-listed", A_LAST_LISTED, CONCAT_ARGS, NO_OBJ ), END_OF_TABLE }; @@ -453,9 +536,13 @@ static addr_policy_t *router_parse_addr_policy_private(directory_token_t *tok); static int router_get_hash_impl(const char *s, size_t s_len, char *digest, const char *start_str, const char *end_str, - char end_char); - -static void token_free(directory_token_t *tok); + char end_char, + digest_algorithm_t alg); +static int router_get_hashes_impl(const char *s, size_t s_len, + digests_t *digests, + const char *start_str, const char *end_str, + char end_char); +static void token_clear(directory_token_t *tok); static smartlist_t *find_all_exitpolicy(smartlist_t *s); static directory_token_t *_find_by_keyword(smartlist_t *s, directory_keyword keyword, @@ -479,6 +566,7 @@ static directory_token_t *get_next_token(memarea_t *area, #define CST_CHECK_AUTHORITY (1<<0) #define CST_NO_CHECK_OBJTYPE (1<<1) static int check_signature_token(const char *digest, + ssize_t digest_len, directory_token_t *tok, crypto_pk_env_t *pkey, int flags, @@ -499,6 +587,34 @@ static int tor_version_same_series(tor_version_t *a, tor_version_t *b); #define DUMP_AREA(a,name) STMT_NIL #endif +/** Last time we dumped a descriptor to disk. */ +static time_t last_desc_dumped = 0; + +/** For debugging purposes, dump unparseable descriptor *<b>desc</b> of + * type *<b>type</b> to file $DATADIR/unparseable-desc. Do not write more + * than one descriptor to disk per minute. If there is already such a + * file in the data directory, overwrite it. */ +static void +dump_desc(const char *desc, const char *type) +{ + time_t now = time(NULL); + tor_assert(desc); + tor_assert(type); + if (!last_desc_dumped || last_desc_dumped + 60 < now) { + char *debugfile = get_datadir_fname("unparseable-desc"); + size_t filelen = 50 + strlen(type) + strlen(desc); + char *content = tor_malloc_zero(filelen); + tor_snprintf(content, filelen, "Unable to parse descriptor of type " + "%s:\n%s", type, desc); + write_str_to_file(debugfile, content, 0); + log_info(LD_DIR, "Unable to parse descriptor of type %s. See file " + "unparseable-desc in data directory for details.", type); + tor_free(content); + tor_free(debugfile); + last_desc_dumped = now; + } +} + /** Set <b>digest</b> to the SHA-1 digest of the hash of the directory in * <b>s</b>. Return 0 on success, -1 on failure. */ @@ -506,7 +622,8 @@ int router_get_dir_hash(const char *s, char *digest) { return router_get_hash_impl(s, strlen(s), digest, - "signed-directory","\ndirectory-signature",'\n'); + "signed-directory","\ndirectory-signature",'\n', + DIGEST_SHA1); } /** Set <b>digest</b> to the SHA-1 digest of the hash of the first router in @@ -516,7 +633,8 @@ int router_get_router_hash(const char *s, size_t s_len, char *digest) { return router_get_hash_impl(s, s_len, digest, - "router ","\nrouter-signature", '\n'); + "router ","\nrouter-signature", '\n', + DIGEST_SHA1); } /** Set <b>digest</b> to the SHA-1 digest of the hash of the running-routers @@ -526,7 +644,8 @@ int router_get_runningrouters_hash(const char *s, char *digest) { return router_get_hash_impl(s, strlen(s), digest, - "network-status","\ndirectory-signature", '\n'); + "network-status","\ndirectory-signature", '\n', + DIGEST_SHA1); } /** Set <b>digest</b> to the SHA-1 digest of the hash of the network-status @@ -536,18 +655,31 @@ router_get_networkstatus_v2_hash(const char *s, char *digest) { return router_get_hash_impl(s, strlen(s), digest, "network-status-version","\ndirectory-signature", - '\n'); + '\n', + DIGEST_SHA1); +} + +/** Set <b>digests</b> to all the digests of the consensus document in + * <b>s</b> */ +int +router_get_networkstatus_v3_hashes(const char *s, digests_t *digests) +{ + return router_get_hashes_impl(s,strlen(s),digests, + "network-status-version", + "\ndirectory-signature", + ' '); } /** Set <b>digest</b> to the SHA-1 digest of the hash of the network-status * string in <b>s</b>. Return 0 on success, -1 on failure. */ int -router_get_networkstatus_v3_hash(const char *s, char *digest) +router_get_networkstatus_v3_hash(const char *s, char *digest, + digest_algorithm_t alg) { return router_get_hash_impl(s, strlen(s), digest, "network-status-version", "\ndirectory-signature", - ' '); + ' ', alg); } /** Set <b>digest</b> to the SHA-1 digest of the hash of the extrainfo @@ -556,7 +688,7 @@ int router_get_extrainfo_hash(const char *s, char *digest) { return router_get_hash_impl(s, strlen(s), digest, "extra-info", - "\nrouter-signature",'\n'); + "\nrouter-signature",'\n', DIGEST_SHA1); } /** Helper: used to generate signatures for routers, directories and @@ -568,16 +700,17 @@ router_get_extrainfo_hash(const char *s, char *digest) */ int router_append_dirobj_signature(char *buf, size_t buf_len, const char *digest, - crypto_pk_env_t *private_key) + size_t digest_len, crypto_pk_env_t *private_key) { char *signature; size_t i, keysize; + int siglen; keysize = crypto_pk_keysize(private_key); signature = tor_malloc(keysize); - if (crypto_pk_private_sign(private_key, signature, keysize, - digest, DIGEST_LEN) < 0) { - + siglen = crypto_pk_private_sign(private_key, signature, keysize, + digest, digest_len); + if (siglen < 0) { log_warn(LD_BUG,"Couldn't sign digest."); goto err; } @@ -585,7 +718,7 @@ router_append_dirobj_signature(char *buf, size_t buf_len, const char *digest, goto truncated; i = strlen(buf); - if (base64_encode(buf+i, buf_len-i, signature, 128) < 0) { + if (base64_encode(buf+i, buf_len-i, signature, siglen) < 0) { log_warn(LD_BUG,"couldn't base64-encode signature"); goto err; } @@ -692,7 +825,7 @@ router_parse_directory(const char *str) char digest[DIGEST_LEN]; time_t published_on; int r; - const char *end, *cp; + const char *end, *cp, *str_dup = str; smartlist_t *tokens = NULL; crypto_pk_env_t *declared_key = NULL; memarea_t *area = memarea_new(); @@ -728,11 +861,11 @@ router_parse_directory(const char *str) } declared_key = find_dir_signing_key(str, str+strlen(str)); note_crypto_pk_op(VERIFY_DIR); - if (check_signature_token(digest, tok, declared_key, + if (check_signature_token(digest, DIGEST_LEN, tok, declared_key, CST_CHECK_AUTHORITY, "directory")<0) goto err; - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_clear(tokens); memarea_clear(area); @@ -765,11 +898,12 @@ router_parse_directory(const char *str) r = 0; goto done; err: + dump_desc(str_dup, "v1 directory"); r = -1; done: if (declared_key) crypto_free_pk_env(declared_key); if (tokens) { - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_free(tokens); } if (area) { @@ -791,7 +925,7 @@ router_parse_runningrouters(const char *str) int r = -1; crypto_pk_env_t *declared_key = NULL; smartlist_t *tokens = NULL; - const char *eos = str + strlen(str); + const char *eos = str + strlen(str), *str_dup = str; memarea_t *area = NULL; if (router_get_runningrouters_hash(str, digest)) { @@ -820,7 +954,7 @@ router_parse_runningrouters(const char *str) } declared_key = find_dir_signing_key(str, eos); note_crypto_pk_op(VERIFY_DIR); - if (check_signature_token(digest, tok, declared_key, + if (check_signature_token(digest, DIGEST_LEN, tok, declared_key, CST_CHECK_AUTHORITY, "running-routers") < 0) goto err; @@ -832,9 +966,10 @@ router_parse_runningrouters(const char *str) r = 0; err: + dump_desc(str_dup, "v1 running-routers"); if (declared_key) crypto_free_pk_env(declared_key); if (tokens) { - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_free(tokens); } if (area) { @@ -884,7 +1019,7 @@ find_dir_signing_key(const char *str, const char *eos) } done: - if (tok) token_free(tok); + if (tok) token_clear(tok); if (area) { DUMP_AREA(area, "dir-signing-key token"); memarea_drop_all(area); @@ -920,6 +1055,7 @@ dir_signing_key_is_trusted(crypto_pk_env_t *key) */ static int check_signature_token(const char *digest, + ssize_t digest_len, directory_token_t *tok, crypto_pk_env_t *pkey, int flags, @@ -952,14 +1088,14 @@ check_signature_token(const char *digest, signed_digest = tor_malloc(keysize); if (crypto_pk_public_checksig(pkey, signed_digest, keysize, tok->object_body, tok->object_size) - != DIGEST_LEN) { + < digest_len) { log_warn(LD_DIR, "Error reading %s: invalid signature.", doctype); tor_free(signed_digest); return -1; } // log_debug(LD_DIR,"Signed %s hash starts %s", doctype, // hex_str(signed_digest,4)); - if (memcmp(digest, signed_digest, DIGEST_LEN)) { + if (memcmp(digest, signed_digest, digest_len)) { log_warn(LD_DIR, "Error reading %s: signature does not match.", doctype); tor_free(signed_digest); return -1; @@ -1145,7 +1281,7 @@ router_parse_entry_from_string(const char *s, const char *end, smartlist_t *tokens = NULL, *exit_policy_tokens = NULL; directory_token_t *tok; struct in_addr in; - const char *start_of_annotations, *cp; + const char *start_of_annotations, *cp, *s_dup = s; size_t prepend_len = prepend_annotations ? strlen(prepend_annotations) : 0; int ok = 1; memarea_t *area = NULL; @@ -1429,7 +1565,7 @@ router_parse_entry_from_string(const char *s, const char *end, verified_digests = digestmap_new(); digestmap_set(verified_digests, signed_digest, (void*)(uintptr_t)1); #endif - if (check_signature_token(digest, tok, router->identity_pkey, 0, + if (check_signature_token(digest, DIGEST_LEN, tok, router->identity_pkey, 0, "router descriptor") < 0) goto err; @@ -1447,16 +1583,15 @@ router_parse_entry_from_string(const char *s, const char *end, goto done; err: + dump_desc(s_dup, "router descriptor"); routerinfo_free(router); router = NULL; done: if (tokens) { - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_free(tokens); } - if (exit_policy_tokens) { - smartlist_free(exit_policy_tokens); - } + smartlist_free(exit_policy_tokens); if (area) { DUMP_AREA(area, "routerinfo"); memarea_drop_all(area); @@ -1481,6 +1616,7 @@ extrainfo_parse_entry_from_string(const char *s, const char *end, crypto_pk_env_t *key = NULL; routerinfo_t *router = NULL; memarea_t *area = NULL; + const char *s_dup = s; if (!end) { end = s + strlen(s); @@ -1555,7 +1691,8 @@ extrainfo_parse_entry_from_string(const char *s, const char *end, if (key) { note_crypto_pk_op(VERIFY_RTR); - if (check_signature_token(digest, tok, key, 0, "extra-info") < 0) + if (check_signature_token(digest, DIGEST_LEN, tok, key, 0, + "extra-info") < 0) goto err; if (router) @@ -1569,12 +1706,12 @@ extrainfo_parse_entry_from_string(const char *s, const char *end, goto done; err: - if (extrainfo) - extrainfo_free(extrainfo); + dump_desc(s_dup, "extra-info descriptor"); + extrainfo_free(extrainfo); extrainfo = NULL; done: if (tokens) { - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_free(tokens); } if (area) { @@ -1602,6 +1739,7 @@ authority_cert_parse_from_string(const char *s, const char **end_of_string) size_t len; int found; memarea_t *area = NULL; + const char *s_dup = s; s = eat_whitespace(s); eos = strstr(s, "\ndir-key-certification"); @@ -1632,7 +1770,7 @@ authority_cert_parse_from_string(const char *s, const char **end_of_string) goto err; } if (router_get_hash_impl(s, strlen(s), digest, "dir-key-certificate-version", - "\ndir-key-certification", '\n') < 0) + "\ndir-key-certification", '\n', DIGEST_SHA1) < 0) goto err; tok = smartlist_get(tokens, 0); if (tok->tp != K_DIR_KEY_CERTIFICATE_VERSION || strcmp(tok->args[0], "3")) { @@ -1680,7 +1818,7 @@ authority_cert_parse_from_string(const char *s, const char **end_of_string) struct in_addr in; char *address = NULL; tor_assert(tok->n_args); - /* XXX021 use tor_addr_port_parse() below instead. -RD */ + /* XXX023 use tor_addr_port_parse() below instead. -RD */ if (parse_addr_port(LOG_WARN, tok->args[0], &address, NULL, &cert->dir_port)<0 || tor_inet_aton(address, &in) == 0) { @@ -1725,7 +1863,7 @@ authority_cert_parse_from_string(const char *s, const char **end_of_string) } } if (!found) { - if (check_signature_token(digest, tok, cert->identity_key, 0, + if (check_signature_token(digest, DIGEST_LEN, tok, cert->identity_key, 0, "key certificate")) { goto err; } @@ -1734,6 +1872,7 @@ authority_cert_parse_from_string(const char *s, const char **end_of_string) /* XXXX Once all authorities generate cross-certified certificates, * make this field mandatory. */ if (check_signature_token(cert->cache_info.identity_digest, + DIGEST_LEN, tok, cert->signing_key, CST_NO_CHECK_OBJTYPE, @@ -1753,7 +1892,7 @@ authority_cert_parse_from_string(const char *s, const char **end_of_string) if (end_of_string) { *end_of_string = eat_whitespace(eos); } - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_free(tokens); if (area) { DUMP_AREA(area, "authority cert"); @@ -1761,8 +1900,9 @@ authority_cert_parse_from_string(const char *s, const char **end_of_string) } return cert; err: + dump_desc(s_dup, "authority cert"); authority_cert_free(cert); - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_free(tokens); if (area) { DUMP_AREA(area, "authority cert"); @@ -1773,23 +1913,28 @@ authority_cert_parse_from_string(const char *s, const char **end_of_string) /** Helper: given a string <b>s</b>, return the start of the next router-status * object (starting with "r " at the start of a line). If none is found, - * return the start of the next directory signature. If none is found, return - * the end of the string. */ + * return the start of the directory footer, or the next directory signature. + * If none is found, return the end of the string. */ static INLINE const char * find_start_of_next_routerstatus(const char *s) { - const char *eos = strstr(s, "\nr "); - if (eos) { - const char *eos2 = tor_memstr(s, eos-s, "\ndirectory-signature"); - if (eos2 && eos2 < eos) - return eos2; - else - return eos+1; - } else { - if ((eos = strstr(s, "\ndirectory-signature"))) - return eos+1; - return s + strlen(s); - } + const char *eos, *footer, *sig; + if ((eos = strstr(s, "\nr "))) + ++eos; + else + eos = s + strlen(s); + + footer = tor_memstr(s, eos-s, "\ndirectory-footer"); + sig = tor_memstr(s, eos-s, "\ndirectory-signature"); + + if (footer && sig) + return MIN(footer, sig) + 1; + else if (footer) + return footer+1; + else if (sig) + return sig+1; + else + return eos; } /** Given a string at *<b>s</b>, containing a routerstatus object, and an @@ -1803,22 +1948,29 @@ find_start_of_next_routerstatus(const char *s) * If <b>consensus_method</b> is nonzero, this routerstatus is part of a * consensus, and we should parse it according to the method used to * make that consensus. + * + * Parse according to the syntax used by the consensus flavor <b>flav</b>. **/ static routerstatus_t * routerstatus_parse_entry_from_string(memarea_t *area, const char **s, smartlist_t *tokens, networkstatus_t *vote, vote_routerstatus_t *vote_rs, - int consensus_method) + int consensus_method, + consensus_flavor_t flav) { - const char *eos; + const char *eos, *s_dup = *s; routerstatus_t *rs = NULL; directory_token_t *tok; char timebuf[ISO_TIME_LEN+1]; struct in_addr in; + int offset = 0; tor_assert(tokens); tor_assert(bool_eq(vote, vote_rs)); + if (!consensus_method) + flav = FLAV_NS; + eos = find_start_of_next_routerstatus(*s); if (tokenize_string(area,*s, eos, tokens, rtrstatus_token_table,0)) { @@ -1830,7 +1982,15 @@ routerstatus_parse_entry_from_string(memarea_t *area, goto err; } tok = find_by_keyword(tokens, K_R); - tor_assert(tok->n_args >= 8); + tor_assert(tok->n_args >= 7); + if (flav == FLAV_NS) { + if (tok->n_args < 8) { + log_warn(LD_DIR, "Too few arguments to r"); + goto err; + } + } else { + offset = -1; + } if (vote_rs) { rs = &vote_rs->status; } else { @@ -1851,29 +2011,34 @@ routerstatus_parse_entry_from_string(memarea_t *area, goto err; } - if (digest_from_base64(rs->descriptor_digest, tok->args[2])) { - log_warn(LD_DIR, "Error decoding descriptor digest %s", - escaped(tok->args[2])); - goto err; + if (flav == FLAV_NS) { + if (digest_from_base64(rs->descriptor_digest, tok->args[2])) { + log_warn(LD_DIR, "Error decoding descriptor digest %s", + escaped(tok->args[2])); + goto err; + } } if (tor_snprintf(timebuf, sizeof(timebuf), "%s %s", - tok->args[3], tok->args[4]) < 0 || + tok->args[3+offset], tok->args[4+offset]) < 0 || parse_iso_time(timebuf, &rs->published_on)<0) { - log_warn(LD_DIR, "Error parsing time '%s %s'", - tok->args[3], tok->args[4]); + log_warn(LD_DIR, "Error parsing time '%s %s' [%d %d]", + tok->args[3+offset], tok->args[4+offset], + offset, (int)flav); goto err; } - if (tor_inet_aton(tok->args[5], &in) == 0) { + if (tor_inet_aton(tok->args[5+offset], &in) == 0) { log_warn(LD_DIR, "Error parsing router address in network-status %s", - escaped(tok->args[5])); + escaped(tok->args[5+offset])); goto err; } rs->addr = ntohl(in.s_addr); - rs->or_port =(uint16_t) tor_parse_long(tok->args[6],10,0,65535,NULL,NULL); - rs->dir_port = (uint16_t) tor_parse_long(tok->args[7],10,0,65535,NULL,NULL); + rs->or_port = (uint16_t) tor_parse_long(tok->args[6+offset], + 10,0,65535,NULL,NULL); + rs->dir_port = (uint16_t) tor_parse_long(tok->args[7+offset], + 10,0,65535,NULL,NULL); tok = find_opt_by_keyword(tokens, K_S); if (tok && vote) { @@ -1959,6 +2124,17 @@ routerstatus_parse_entry_from_string(memarea_t *area, goto err; } rs->has_bandwidth = 1; + } else if (!strcmpstart(tok->args[i], "Measured=")) { + int ok; + rs->measured_bw = + (uint32_t)tor_parse_ulong(strchr(tok->args[i], '=')+1, + 10, 0, UINT32_MAX, &ok, NULL); + if (!ok) { + log_warn(LD_DIR, "Invalid Measured Bandwidth %s", + escaped(tok->args[i])); + goto err; + } + rs->has_measured_bw = 1; } } } @@ -1980,16 +2156,29 @@ routerstatus_parse_entry_from_string(memarea_t *area, rs->has_exitsummary = 1; } + if (vote_rs) { + SMARTLIST_FOREACH_BEGIN(tokens, directory_token_t *, t) { + if (t->tp == K_M && t->n_args) { + vote_microdesc_hash_t *line = + tor_malloc(sizeof(vote_microdesc_hash_t)); + line->next = vote_rs->microdesc; + line->microdesc_hash_line = tor_strdup(t->args[0]); + vote_rs->microdesc = line; + } + } SMARTLIST_FOREACH_END(t); + } + if (!strcasecmp(rs->nickname, UNNAMED_ROUTER_NICKNAME)) rs->is_named = 0; goto done; err: + dump_desc(s_dup, "routerstatus entry"); if (rs && !vote_rs) routerstatus_free(rs); rs = NULL; done: - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_clear(tokens); if (area) { DUMP_AREA(area, "routerstatus entry"); @@ -2001,8 +2190,8 @@ routerstatus_parse_entry_from_string(memarea_t *area, } /** Helper to sort a smartlist of pointers to routerstatus_t */ -static int -_compare_routerstatus_entries(const void **_a, const void **_b) +int +compare_routerstatus_entries(const void **_a, const void **_b) { const routerstatus_t *a = *_a, *b = *_b; return memcmp(a->identity_digest, b->identity_digest, DIGEST_LEN); @@ -2026,7 +2215,7 @@ _free_duplicate_routerstatus_entry(void *e) networkstatus_v2_t * networkstatus_v2_parse_from_string(const char *s) { - const char *eos; + const char *eos, *s_dup = s; smartlist_t *tokens = smartlist_create(); smartlist_t *footer_tokens = smartlist_create(); networkstatus_v2_t *ns = NULL; @@ -2140,17 +2329,17 @@ networkstatus_v2_parse_from_string(const char *s) ns->entries = smartlist_create(); s = eos; - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_clear(tokens); memarea_clear(area); while (!strcmpstart(s, "r ")) { routerstatus_t *rs; if ((rs = routerstatus_parse_entry_from_string(area, &s, tokens, - NULL, NULL, 0))) + NULL, NULL, 0, 0))) smartlist_add(ns->entries, rs); } - smartlist_sort(ns->entries, _compare_routerstatus_entries); - smartlist_uniq(ns->entries, _compare_routerstatus_entries, + smartlist_sort(ns->entries, compare_routerstatus_entries); + smartlist_uniq(ns->entries, compare_routerstatus_entries, _free_duplicate_routerstatus_entry); if (tokenize_string(area,s, NULL, footer_tokens, dir_footer_token_table,0)) { @@ -2169,19 +2358,19 @@ networkstatus_v2_parse_from_string(const char *s) } note_crypto_pk_op(VERIFY_DIR); - if (check_signature_token(ns_digest, tok, ns->signing_key, 0, + if (check_signature_token(ns_digest, DIGEST_LEN, tok, ns->signing_key, 0, "network-status") < 0) goto err; goto done; err: - if (ns) - networkstatus_v2_free(ns); + dump_desc(s_dup, "v2 networkstatus"); + networkstatus_v2_free(ns); ns = NULL; done: - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_free(tokens); - SMARTLIST_FOREACH(footer_tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(footer_tokens, directory_token_t *, t, token_clear(t)); smartlist_free(footer_tokens); if (area) { DUMP_AREA(area, "v2 networkstatus"); @@ -2190,6 +2379,395 @@ networkstatus_v2_parse_from_string(const char *s) return ns; } +/** Verify the bandwidth weights of a network status document */ +int +networkstatus_verify_bw_weights(networkstatus_t *ns) +{ + int64_t weight_scale; + int64_t G=0, M=0, E=0, D=0, T=0; + double Wgg, Wgm, Wgd, Wmg, Wmm, Wme, Wmd, Weg, Wem, Wee, Wed; + double Gtotal=0, Mtotal=0, Etotal=0; + const char *casename = NULL; + int valid = 1; + + weight_scale = circuit_build_times_get_bw_scale(ns); + Wgg = networkstatus_get_bw_weight(ns, "Wgg", -1); + Wgm = networkstatus_get_bw_weight(ns, "Wgm", -1); + Wgd = networkstatus_get_bw_weight(ns, "Wgd", -1); + Wmg = networkstatus_get_bw_weight(ns, "Wmg", -1); + Wmm = networkstatus_get_bw_weight(ns, "Wmm", -1); + Wme = networkstatus_get_bw_weight(ns, "Wme", -1); + Wmd = networkstatus_get_bw_weight(ns, "Wmd", -1); + Weg = networkstatus_get_bw_weight(ns, "Weg", -1); + Wem = networkstatus_get_bw_weight(ns, "Wem", -1); + Wee = networkstatus_get_bw_weight(ns, "Wee", -1); + Wed = networkstatus_get_bw_weight(ns, "Wed", -1); + + if (Wgg<0 || Wgm<0 || Wgd<0 || Wmg<0 || Wmm<0 || Wme<0 || Wmd<0 || Weg<0 + || Wem<0 || Wee<0 || Wed<0) { + log_warn(LD_BUG, "No bandwidth weights produced in consensus!"); + return 0; + } + + // First, sanity check basic summing properties that hold for all cases + // We use > 1 as the check for these because they are computed as integers. + // Sometimes there are rounding errors. + if (fabs(Wmm - weight_scale) > 1) { + log_warn(LD_BUG, "Wmm=%lf != "I64_FORMAT, + Wmm, I64_PRINTF_ARG(weight_scale)); + valid = 0; + } + + if (fabs(Wem - Wee) > 1) { + log_warn(LD_BUG, "Wem=%lf != Wee=%lf", Wem, Wee); + valid = 0; + } + + if (fabs(Wgm - Wgg) > 1) { + log_warn(LD_BUG, "Wgm=%lf != Wgg=%lf", Wgm, Wgg); + valid = 0; + } + + if (fabs(Weg - Wed) > 1) { + log_warn(LD_BUG, "Wed=%lf != Weg=%lf", Wed, Weg); + valid = 0; + } + + if (fabs(Wgg + Wmg - weight_scale) > 0.001*weight_scale) { + log_warn(LD_BUG, "Wgg=%lf != "I64_FORMAT" - Wmg=%lf", Wgg, + I64_PRINTF_ARG(weight_scale), Wmg); + valid = 0; + } + + if (fabs(Wee + Wme - weight_scale) > 0.001*weight_scale) { + log_warn(LD_BUG, "Wee=%lf != "I64_FORMAT" - Wme=%lf", Wee, + I64_PRINTF_ARG(weight_scale), Wme); + valid = 0; + } + + if (fabs(Wgd + Wmd + Wed - weight_scale) > 0.001*weight_scale) { + log_warn(LD_BUG, "Wgd=%lf + Wmd=%lf + Wed=%lf != "I64_FORMAT, + Wgd, Wmd, Wed, I64_PRINTF_ARG(weight_scale)); + valid = 0; + } + + Wgg /= weight_scale; + Wgm /= weight_scale; + Wgd /= weight_scale; + + Wmg /= weight_scale; + Wmm /= weight_scale; + Wme /= weight_scale; + Wmd /= weight_scale; + + Weg /= weight_scale; + Wem /= weight_scale; + Wee /= weight_scale; + Wed /= weight_scale; + + // Then, gather G, M, E, D, T to determine case + SMARTLIST_FOREACH_BEGIN(ns->routerstatus_list, routerstatus_t *, rs) { + if (rs->has_bandwidth) { + T += rs->bandwidth; + if (rs->is_exit && rs->is_possible_guard) { + D += rs->bandwidth; + Gtotal += Wgd*rs->bandwidth; + Mtotal += Wmd*rs->bandwidth; + Etotal += Wed*rs->bandwidth; + } else if (rs->is_exit) { + E += rs->bandwidth; + Mtotal += Wme*rs->bandwidth; + Etotal += Wee*rs->bandwidth; + } else if (rs->is_possible_guard) { + G += rs->bandwidth; + Gtotal += Wgg*rs->bandwidth; + Mtotal += Wmg*rs->bandwidth; + } else { + M += rs->bandwidth; + Mtotal += Wmm*rs->bandwidth; + } + } else { + log_warn(LD_BUG, "Missing consensus bandwidth for router %s", + rs->nickname); + } + } SMARTLIST_FOREACH_END(rs); + + // Finally, check equality conditions depending upon case 1, 2 or 3 + // Full equality cases: 1, 3b + // Partial equality cases: 2b (E=G), 3a (M=E) + // Fully unknown: 2a + if (3*E >= T && 3*G >= T) { + // Case 1: Neither are scarce + casename = "Case 1"; + if (fabs(Etotal-Mtotal) > 0.01*MAX(Etotal,Mtotal)) { + log_warn(LD_DIR, + "Bw Weight Failure for %s: Etotal %lf != Mtotal %lf. " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT". " + "Wgg=%lf Wgd=%lf Wmg=%lf Wme=%lf Wmd=%lf Wee=%lf Wed=%lf", + casename, Etotal, Mtotal, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed); + valid = 0; + } + if (fabs(Etotal-Gtotal) > 0.01*MAX(Etotal,Gtotal)) { + log_warn(LD_DIR, + "Bw Weight Failure for %s: Etotal %lf != Gtotal %lf. " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT". " + "Wgg=%lf Wgd=%lf Wmg=%lf Wme=%lf Wmd=%lf Wee=%lf Wed=%lf", + casename, Etotal, Gtotal, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed); + valid = 0; + } + if (fabs(Gtotal-Mtotal) > 0.01*MAX(Gtotal,Mtotal)) { + log_warn(LD_DIR, + "Bw Weight Failure for %s: Mtotal %lf != Gtotal %lf. " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT". " + "Wgg=%lf Wgd=%lf Wmg=%lf Wme=%lf Wmd=%lf Wee=%lf Wed=%lf", + casename, Mtotal, Gtotal, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed); + valid = 0; + } + } else if (3*E < T && 3*G < T) { + int64_t R = MIN(E, G); + int64_t S = MAX(E, G); + /* + * Case 2: Both Guards and Exits are scarce + * Balance D between E and G, depending upon + * D capacity and scarcity. Devote no extra + * bandwidth to middle nodes. + */ + if (R+D < S) { // Subcase a + double Rtotal, Stotal; + if (E < G) { + Rtotal = Etotal; + Stotal = Gtotal; + } else { + Rtotal = Gtotal; + Stotal = Etotal; + } + casename = "Case 2a"; + // Rtotal < Stotal + if (Rtotal > Stotal) { + log_warn(LD_DIR, + "Bw Weight Failure for %s: Rtotal %lf > Stotal %lf. " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT". " + "Wgg=%lf Wgd=%lf Wmg=%lf Wme=%lf Wmd=%lf Wee=%lf Wed=%lf", + casename, Rtotal, Stotal, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed); + valid = 0; + } + // Rtotal < T/3 + if (3*Rtotal > T) { + log_warn(LD_DIR, + "Bw Weight Failure for %s: 3*Rtotal %lf > T " + I64_FORMAT". G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT + " D="I64_FORMAT" T="I64_FORMAT". " + "Wgg=%lf Wgd=%lf Wmg=%lf Wme=%lf Wmd=%lf Wee=%lf Wed=%lf", + casename, Rtotal*3, I64_PRINTF_ARG(T), + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed); + valid = 0; + } + // Stotal < T/3 + if (3*Stotal > T) { + log_warn(LD_DIR, + "Bw Weight Failure for %s: 3*Stotal %lf > T " + I64_FORMAT". G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT + " D="I64_FORMAT" T="I64_FORMAT". " + "Wgg=%lf Wgd=%lf Wmg=%lf Wme=%lf Wmd=%lf Wee=%lf Wed=%lf", + casename, Stotal*3, I64_PRINTF_ARG(T), + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed); + valid = 0; + } + // Mtotal > T/3 + if (3*Mtotal < T) { + log_warn(LD_DIR, + "Bw Weight Failure for %s: 3*Mtotal %lf < T " + I64_FORMAT". " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT". " + "Wgg=%lf Wgd=%lf Wmg=%lf Wme=%lf Wmd=%lf Wee=%lf Wed=%lf", + casename, Mtotal*3, I64_PRINTF_ARG(T), + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed); + valid = 0; + } + } else { // Subcase b: R+D > S + casename = "Case 2b"; + + /* Check the rare-M redirect case. */ + if (D != 0 && 3*M < T) { + casename = "Case 2b (balanced)"; + if (fabs(Etotal-Mtotal) > 0.01*MAX(Etotal,Mtotal)) { + log_warn(LD_DIR, + "Bw Weight Failure for %s: Etotal %lf != Mtotal %lf. " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT". " + "Wgg=%lf Wgd=%lf Wmg=%lf Wme=%lf Wmd=%lf Wee=%lf Wed=%lf", + casename, Etotal, Mtotal, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed); + valid = 0; + } + if (fabs(Etotal-Gtotal) > 0.01*MAX(Etotal,Gtotal)) { + log_warn(LD_DIR, + "Bw Weight Failure for %s: Etotal %lf != Gtotal %lf. " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT". " + "Wgg=%lf Wgd=%lf Wmg=%lf Wme=%lf Wmd=%lf Wee=%lf Wed=%lf", + casename, Etotal, Gtotal, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed); + valid = 0; + } + if (fabs(Gtotal-Mtotal) > 0.01*MAX(Gtotal,Mtotal)) { + log_warn(LD_DIR, + "Bw Weight Failure for %s: Mtotal %lf != Gtotal %lf. " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT". " + "Wgg=%lf Wgd=%lf Wmg=%lf Wme=%lf Wmd=%lf Wee=%lf Wed=%lf", + casename, Mtotal, Gtotal, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed); + valid = 0; + } + } else { + if (fabs(Etotal-Gtotal) > 0.01*MAX(Etotal,Gtotal)) { + log_warn(LD_DIR, + "Bw Weight Failure for %s: Etotal %lf != Gtotal %lf. " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT". " + "Wgg=%lf Wgd=%lf Wmg=%lf Wme=%lf Wmd=%lf Wee=%lf Wed=%lf", + casename, Etotal, Gtotal, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed); + valid = 0; + } + } + } + } else { // if (E < T/3 || G < T/3) { + int64_t S = MIN(E, G); + int64_t NS = MAX(E, G); + if (3*(S+D) < T) { // Subcase a: + double Stotal; + double NStotal; + if (G < E) { + casename = "Case 3a (G scarce)"; + Stotal = Gtotal; + NStotal = Etotal; + } else { // if (G >= E) { + casename = "Case 3a (E scarce)"; + NStotal = Gtotal; + Stotal = Etotal; + } + // Stotal < T/3 + if (3*Stotal > T) { + log_warn(LD_DIR, + "Bw Weight Failure for %s: 3*Stotal %lf > T " + I64_FORMAT". G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT + " D="I64_FORMAT" T="I64_FORMAT". " + "Wgg=%lf Wgd=%lf Wmg=%lf Wme=%lf Wmd=%lf Wee=%lf Wed=%lf", + casename, Stotal*3, I64_PRINTF_ARG(T), + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed); + valid = 0; + } + if (NS >= M) { + if (fabs(NStotal-Mtotal) > 0.01*MAX(NStotal,Mtotal)) { + log_warn(LD_DIR, + "Bw Weight Failure for %s: NStotal %lf != Mtotal %lf. " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT". " + "Wgg=%lf Wgd=%lf Wmg=%lf Wme=%lf Wmd=%lf Wee=%lf Wed=%lf", + casename, NStotal, Mtotal, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed); + valid = 0; + } + } else { + // if NS < M, NStotal > T/3 because only one of G or E is scarce + if (3*NStotal < T) { + log_warn(LD_DIR, + "Bw Weight Failure for %s: 3*NStotal %lf < T " + I64_FORMAT". G="I64_FORMAT" M="I64_FORMAT + " E="I64_FORMAT" D="I64_FORMAT" T="I64_FORMAT". " + "Wgg=%lf Wgd=%lf Wmg=%lf Wme=%lf Wmd=%lf Wee=%lf Wed=%lf", + casename, NStotal*3, I64_PRINTF_ARG(T), + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed); + valid = 0; + } + } + } else { // Subcase b: S+D >= T/3 + casename = "Case 3b"; + if (fabs(Etotal-Mtotal) > 0.01*MAX(Etotal,Mtotal)) { + log_warn(LD_DIR, + "Bw Weight Failure for %s: Etotal %lf != Mtotal %lf. " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT". " + "Wgg=%lf Wgd=%lf Wmg=%lf Wme=%lf Wmd=%lf Wee=%lf Wed=%lf", + casename, Etotal, Mtotal, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed); + valid = 0; + } + if (fabs(Etotal-Gtotal) > 0.01*MAX(Etotal,Gtotal)) { + log_warn(LD_DIR, + "Bw Weight Failure for %s: Etotal %lf != Gtotal %lf. " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT". " + "Wgg=%lf Wgd=%lf Wmg=%lf Wme=%lf Wmd=%lf Wee=%lf Wed=%lf", + casename, Etotal, Gtotal, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed); + valid = 0; + } + if (fabs(Gtotal-Mtotal) > 0.01*MAX(Gtotal,Mtotal)) { + log_warn(LD_DIR, + "Bw Weight Failure for %s: Mtotal %lf != Gtotal %lf. " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT". " + "Wgg=%lf Wgd=%lf Wmg=%lf Wme=%lf Wmd=%lf Wee=%lf Wed=%lf", + casename, Mtotal, Gtotal, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed); + valid = 0; + } + } + } + + if (valid) + log_notice(LD_DIR, "Bandwidth-weight %s is verified and valid.", + casename); + + return valid; +} + /** Parse a v3 networkstatus vote, opinion, or consensus (depending on * ns_type), from <b>s</b>, and return the result. Return NULL on failure. */ networkstatus_t * @@ -2200,19 +2778,21 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, smartlist_t *rs_tokens = NULL, *footer_tokens = NULL; networkstatus_voter_info_t *voter = NULL; networkstatus_t *ns = NULL; - char ns_digest[DIGEST_LEN]; - const char *cert, *end_of_header, *end_of_footer; + digests_t ns_digests; + const char *cert, *end_of_header, *end_of_footer, *s_dup = s; directory_token_t *tok; int ok; struct in_addr in; int i, inorder, n_signatures = 0; memarea_t *area = NULL, *rs_area = NULL; + consensus_flavor_t flav = FLAV_NS; + tor_assert(s); if (eos_out) *eos_out = NULL; - if (router_get_networkstatus_v3_hash(s, ns_digest)) { + if (router_get_networkstatus_v3_hashes(s, &ns_digests)) { log_warn(LD_DIR, "Unable to compute digest of network-status"); goto err; } @@ -2228,7 +2808,23 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, } ns = tor_malloc_zero(sizeof(networkstatus_t)); - memcpy(ns->networkstatus_digest, ns_digest, DIGEST_LEN); + memcpy(&ns->digests, &ns_digests, sizeof(ns_digests)); + + tok = find_by_keyword(tokens, K_NETWORK_STATUS_VERSION); + tor_assert(tok); + if (tok->n_args > 1) { + int flavor = networkstatus_parse_flavor_name(tok->args[1]); + if (flavor < 0) { + log_warn(LD_DIR, "Can't parse document with unknown flavor %s", + escaped(tok->args[2])); + goto err; + } + ns->flavor = flav = flavor; + } + if (flav != FLAV_NS && ns_type != NS_TYPE_CONSENSUS) { + log_warn(LD_DIR, "Flavor found on non-consensus networkstatus."); + goto err; + } if (ns_type != NS_TYPE_CONSENSUS) { const char *end_of_cert = NULL; @@ -2382,8 +2978,9 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, if (voter) smartlist_add(ns->voters, voter); voter = tor_malloc_zero(sizeof(networkstatus_voter_info_t)); + voter->sigs = smartlist_create(); if (ns->type != NS_TYPE_CONSENSUS) - memcpy(voter->vote_digest, ns_digest, DIGEST_LEN); + memcpy(voter->vote_digest, ns_digests.d[DIGEST_SHA1], DIGEST_LEN); voter->nickname = tor_strdup(tok->args[0]); if (strlen(tok->args[1]) != HEX_DIGEST_LEN || @@ -2475,7 +3072,7 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, if (ns->type != NS_TYPE_CONSENSUS) { vote_routerstatus_t *rs = tor_malloc_zero(sizeof(vote_routerstatus_t)); if (routerstatus_parse_entry_from_string(rs_area, &s, rs_tokens, ns, - rs, 0)) + rs, 0, 0)) smartlist_add(ns->routerstatus_list, rs); else { tor_free(rs->version); @@ -2485,7 +3082,8 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, routerstatus_t *rs; if ((rs = routerstatus_parse_entry_from_string(rs_area, &s, rs_tokens, NULL, NULL, - ns->consensus_method))) + ns->consensus_method, + flav))) smartlist_add(ns->routerstatus_list, rs); } } @@ -2518,14 +3116,73 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, goto err; } - SMARTLIST_FOREACH(footer_tokens, directory_token_t *, _tok, { + int found_sig = 0; + SMARTLIST_FOREACH_BEGIN(footer_tokens, directory_token_t *, _tok) { + tok = _tok; + if (tok->tp == K_DIRECTORY_SIGNATURE) + found_sig = 1; + else if (found_sig) { + log_warn(LD_DIR, "Extraneous token after first directory-signature"); + goto err; + } + } SMARTLIST_FOREACH_END(_tok); + } + + if ((tok = find_opt_by_keyword(footer_tokens, K_DIRECTORY_FOOTER))) { + if (tok != smartlist_get(footer_tokens, 0)) { + log_warn(LD_DIR, "Misplaced directory-footer token"); + goto err; + } + } + + tok = find_opt_by_keyword(footer_tokens, K_BW_WEIGHTS); + if (tok) { + ns->weight_params = smartlist_create(); + for (i = 0; i < tok->n_args; ++i) { + int ok=0; + char *eq = strchr(tok->args[i], '='); + if (!eq) { + log_warn(LD_DIR, "Bad element '%s' in weight params", + escaped(tok->args[i])); + goto err; + } + tor_parse_long(eq+1, 10, INT32_MIN, INT32_MAX, &ok, NULL); + if (!ok) { + log_warn(LD_DIR, "Bad element '%s' in params", escaped(tok->args[i])); + goto err; + } + smartlist_add(ns->weight_params, tor_strdup(tok->args[i])); + } + } + + SMARTLIST_FOREACH_BEGIN(footer_tokens, directory_token_t *, _tok) { char declared_identity[DIGEST_LEN]; networkstatus_voter_info_t *v; + document_signature_t *sig; + const char *id_hexdigest = NULL; + const char *sk_hexdigest = NULL; + digest_algorithm_t alg = DIGEST_SHA1; tok = _tok; if (tok->tp != K_DIRECTORY_SIGNATURE) continue; tor_assert(tok->n_args >= 2); + if (tok->n_args == 2) { + id_hexdigest = tok->args[0]; + sk_hexdigest = tok->args[1]; + } else { + const char *algname = tok->args[0]; + int a; + id_hexdigest = tok->args[1]; + sk_hexdigest = tok->args[2]; + a = crypto_digest_algorithm_parse_name(algname); + if (a<0) { + log_warn(LD_DIR, "Unknown digest algorithm %s; skipping", + escaped(algname)); + continue; + } + alg = a; + } if (!tok->object_type || strcmp(tok->object_type, "SIGNATURE") || @@ -2534,11 +3191,11 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, goto err; } - if (strlen(tok->args[0]) != HEX_DIGEST_LEN || + if (strlen(id_hexdigest) != HEX_DIGEST_LEN || base16_decode(declared_identity, sizeof(declared_identity), - tok->args[0], HEX_DIGEST_LEN) < 0) { + id_hexdigest, HEX_DIGEST_LEN) < 0) { log_warn(LD_DIR, "Error decoding declared identity %s in " - "network-status vote.", escaped(tok->args[0])); + "network-status vote.", escaped(id_hexdigest)); goto err; } if (!(v = networkstatus_get_voter_by_id(ns, declared_identity))) { @@ -2546,11 +3203,15 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, "any declared directory source."); goto err; } - if (strlen(tok->args[1]) != HEX_DIGEST_LEN || - base16_decode(v->signing_key_digest, sizeof(v->signing_key_digest), - tok->args[1], HEX_DIGEST_LEN) < 0) { - log_warn(LD_DIR, "Error decoding declared digest %s in " - "network-status vote.", escaped(tok->args[1])); + sig = tor_malloc_zero(sizeof(document_signature_t)); + memcpy(sig->identity_digest, v->identity_digest, DIGEST_LEN); + sig->alg = alg; + if (strlen(sk_hexdigest) != HEX_DIGEST_LEN || + base16_decode(sig->signing_key_digest, sizeof(sig->signing_key_digest), + sk_hexdigest, HEX_DIGEST_LEN) < 0) { + log_warn(LD_DIR, "Error decoding declared signing key digest %s in " + "network-status vote.", escaped(sk_hexdigest)); + tor_free(sig); goto err; } @@ -2559,35 +3220,49 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, DIGEST_LEN)) { log_warn(LD_DIR, "Digest mismatch between declared and actual on " "network-status vote."); + tor_free(sig); goto err; } } + if (voter_get_sig_by_algorithm(v, sig->alg)) { + /* We already parsed a vote with this algorithm from this voter. Use the + first one. */ + log_fn(LOG_PROTOCOL_WARN, LD_DIR, "We received a networkstatus " + "that contains two votes from the same voter with the same " + "algorithm. Ignoring the second vote."); + tor_free(sig); + continue; + } + if (ns->type != NS_TYPE_CONSENSUS) { - if (check_signature_token(ns_digest, tok, ns->cert->signing_key, 0, - "network-status vote")) + if (check_signature_token(ns_digests.d[DIGEST_SHA1], DIGEST_LEN, + tok, ns->cert->signing_key, 0, + "network-status vote")) { + tor_free(sig); goto err; - v->good_signature = 1; + } + sig->good_signature = 1; } else { - if (tok->object_size >= INT_MAX || tok->object_size >= SIZE_T_CEILING) + if (tok->object_size >= INT_MAX || tok->object_size >= SIZE_T_CEILING) { + tor_free(sig); goto err; - /* We already parsed a vote from this voter. Use the first one. */ - if (v->signature) { - log_fn(LOG_PROTOCOL_WARN, LD_DIR, "We received a networkstatus " - "that contains two votes from the same voter. Ignoring " - "the second vote."); - continue; } - - v->signature = tor_memdup(tok->object_body, tok->object_size); - v->signature_len = (int) tok->object_size; + sig->signature = tor_memdup(tok->object_body, tok->object_size); + sig->signature_len = (int) tok->object_size; } + smartlist_add(v->sigs, sig); + ++n_signatures; - }); + } SMARTLIST_FOREACH_END(_tok); if (! n_signatures) { log_warn(LD_DIR, "No signatures on networkstatus vote."); goto err; + } else if (ns->type == NS_TYPE_VOTE && n_signatures != 1) { + log_warn(LD_DIR, "Received more than one signature on a " + "network-status vote."); + goto err; } if (eos_out) @@ -2595,27 +3270,31 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, goto done; err: - if (ns) - networkstatus_vote_free(ns); + dump_desc(s_dup, "v3 networkstatus"); + networkstatus_vote_free(ns); ns = NULL; done: if (tokens) { - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_free(tokens); } if (voter) { + if (voter->sigs) { + SMARTLIST_FOREACH(voter->sigs, document_signature_t *, sig, + document_signature_free(sig)); + smartlist_free(voter->sigs); + } tor_free(voter->nickname); tor_free(voter->address); tor_free(voter->contact); - tor_free(voter->signature); tor_free(voter); } if (rs_tokens) { - SMARTLIST_FOREACH(rs_tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(rs_tokens, directory_token_t *, t, token_clear(t)); smartlist_free(rs_tokens); } if (footer_tokens) { - SMARTLIST_FOREACH(footer_tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(footer_tokens, directory_token_t *, t, token_clear(t)); smartlist_free(footer_tokens); } if (area) { @@ -2628,6 +3307,35 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, return ns; } +/** Return the digests_t that holds the digests of the + * <b>flavor_name</b>-flavored networkstatus according to the detached + * signatures document <b>sigs</b>, allocating a new digests_t as neeeded. */ +static digests_t * +detached_get_digests(ns_detached_signatures_t *sigs, const char *flavor_name) +{ + digests_t *d = strmap_get(sigs->digests, flavor_name); + if (!d) { + d = tor_malloc_zero(sizeof(digests_t)); + strmap_set(sigs->digests, flavor_name, d); + } + return d; +} + +/** Return the list of signatures of the <b>flavor_name</b>-flavored + * networkstatus according to the detached signatures document <b>sigs</b>, + * allocating a new digests_t as neeeded. */ +static smartlist_t * +detached_get_signatures(ns_detached_signatures_t *sigs, + const char *flavor_name) +{ + smartlist_t *sl = strmap_get(sigs->signatures, flavor_name); + if (!sl) { + sl = smartlist_create(); + strmap_set(sigs->signatures, flavor_name, sl); + } + return sl; +} + /** Parse a detached v3 networkstatus signature document between <b>s</b> and * <b>eos</b> and return the result. Return -1 on failure. */ ns_detached_signatures_t * @@ -2637,10 +3345,13 @@ networkstatus_parse_detached_signatures(const char *s, const char *eos) * networkstatus_parse_vote_from_string(). */ directory_token_t *tok; memarea_t *area = NULL; + digests_t *digests; smartlist_t *tokens = smartlist_create(); ns_detached_signatures_t *sigs = tor_malloc_zero(sizeof(ns_detached_signatures_t)); + sigs->digests = strmap_new(); + sigs->signatures = strmap_new(); if (!eos) eos = s + strlen(s); @@ -2652,18 +3363,57 @@ networkstatus_parse_detached_signatures(const char *s, const char *eos) goto err; } - tok = find_by_keyword(tokens, K_CONSENSUS_DIGEST); - if (strlen(tok->args[0]) != HEX_DIGEST_LEN) { - log_warn(LD_DIR, "Wrong length on consensus-digest in detached " - "networkstatus signatures"); - goto err; - } - if (base16_decode(sigs->networkstatus_digest, DIGEST_LEN, - tok->args[0], strlen(tok->args[0])) < 0) { - log_warn(LD_DIR, "Bad encoding on on consensus-digest in detached " - "networkstatus signatures"); - goto err; - } + /* Grab all the digest-like tokens. */ + SMARTLIST_FOREACH_BEGIN(tokens, directory_token_t *, _tok) { + const char *algname; + digest_algorithm_t alg; + const char *flavor; + const char *hexdigest; + size_t expected_length; + + tok = _tok; + + if (tok->tp == K_CONSENSUS_DIGEST) { + algname = "sha1"; + alg = DIGEST_SHA1; + flavor = "ns"; + hexdigest = tok->args[0]; + } else if (tok->tp == K_ADDITIONAL_DIGEST) { + int a = crypto_digest_algorithm_parse_name(tok->args[1]); + if (a<0) { + log_warn(LD_DIR, "Unrecognized algorithm name %s", tok->args[0]); + continue; + } + alg = (digest_algorithm_t) a; + flavor = tok->args[0]; + algname = tok->args[1]; + hexdigest = tok->args[2]; + } else { + continue; + } + + expected_length = + (alg == DIGEST_SHA1) ? HEX_DIGEST_LEN : HEX_DIGEST256_LEN; + + if (strlen(hexdigest) != expected_length) { + log_warn(LD_DIR, "Wrong length on consensus-digest in detached " + "networkstatus signatures"); + goto err; + } + digests = detached_get_digests(sigs, flavor); + tor_assert(digests); + if (!tor_mem_is_zero(digests->d[alg], DIGEST256_LEN)) { + log_warn(LD_DIR, "Multiple digests for %s with %s on detached " + "signatures document", flavor, algname); + continue; + } + if (base16_decode(digests->d[alg], DIGEST256_LEN, + hexdigest, strlen(hexdigest)) < 0) { + log_warn(LD_DIR, "Bad encoding on consensus-digest in detached " + "networkstatus signatures"); + goto err; + } + } SMARTLIST_FOREACH_END(_tok); tok = find_by_keyword(tokens, K_VALID_AFTER); if (parse_iso_time(tok->args[0], &sigs->valid_after)) { @@ -2683,57 +3433,102 @@ networkstatus_parse_detached_signatures(const char *s, const char *eos) goto err; } - sigs->signatures = smartlist_create(); - SMARTLIST_FOREACH(tokens, directory_token_t *, _tok, - { - char id_digest[DIGEST_LEN]; - char sk_digest[DIGEST_LEN]; - networkstatus_voter_info_t *voter; + SMARTLIST_FOREACH_BEGIN(tokens, directory_token_t *, _tok) { + const char *id_hexdigest; + const char *sk_hexdigest; + const char *algname; + const char *flavor; + digest_algorithm_t alg; + + char id_digest[DIGEST_LEN]; + char sk_digest[DIGEST_LEN]; + smartlist_t *siglist; + document_signature_t *sig; + int is_duplicate; - tok = _tok; - if (tok->tp != K_DIRECTORY_SIGNATURE) - continue; + tok = _tok; + if (tok->tp == K_DIRECTORY_SIGNATURE) { tor_assert(tok->n_args >= 2); + flavor = "ns"; + algname = "sha1"; + id_hexdigest = tok->args[0]; + sk_hexdigest = tok->args[1]; + } else if (tok->tp == K_ADDITIONAL_SIGNATURE) { + tor_assert(tok->n_args >= 4); + flavor = tok->args[0]; + algname = tok->args[1]; + id_hexdigest = tok->args[2]; + sk_hexdigest = tok->args[3]; + } else { + continue; + } - if (!tok->object_type || - strcmp(tok->object_type, "SIGNATURE") || - tok->object_size < 128 || tok->object_size > 512) { - log_warn(LD_DIR, "Bad object type or length on directory-signature"); - goto err; + { + int a = crypto_digest_algorithm_parse_name(algname); + if (a<0) { + log_warn(LD_DIR, "Unrecognized algorithm name %s", algname); + continue; } + alg = (digest_algorithm_t) a; + } - if (strlen(tok->args[0]) != HEX_DIGEST_LEN || - base16_decode(id_digest, sizeof(id_digest), - tok->args[0], HEX_DIGEST_LEN) < 0) { - log_warn(LD_DIR, "Error decoding declared identity %s in " - "network-status vote.", escaped(tok->args[0])); - goto err; - } - if (strlen(tok->args[1]) != HEX_DIGEST_LEN || - base16_decode(sk_digest, sizeof(sk_digest), - tok->args[1], HEX_DIGEST_LEN) < 0) { - log_warn(LD_DIR, "Error decoding declared digest %s in " - "network-status vote.", escaped(tok->args[1])); - goto err; - } + if (!tok->object_type || + strcmp(tok->object_type, "SIGNATURE") || + tok->object_size < 128 || tok->object_size > 512) { + log_warn(LD_DIR, "Bad object type or length on directory-signature"); + goto err; + } - voter = tor_malloc_zero(sizeof(networkstatus_voter_info_t)); - memcpy(voter->identity_digest, id_digest, DIGEST_LEN); - memcpy(voter->signing_key_digest, sk_digest, DIGEST_LEN); - if (tok->object_size >= INT_MAX || tok->object_size >= SIZE_T_CEILING) - goto err; - voter->signature = tor_memdup(tok->object_body, tok->object_size); - voter->signature_len = (int) tok->object_size; + if (strlen(id_hexdigest) != HEX_DIGEST_LEN || + base16_decode(id_digest, sizeof(id_digest), + id_hexdigest, HEX_DIGEST_LEN) < 0) { + log_warn(LD_DIR, "Error decoding declared identity %s in " + "network-status vote.", escaped(id_hexdigest)); + goto err; + } + if (strlen(sk_hexdigest) != HEX_DIGEST_LEN || + base16_decode(sk_digest, sizeof(sk_digest), + sk_hexdigest, HEX_DIGEST_LEN) < 0) { + log_warn(LD_DIR, "Error decoding declared signing key digest %s in " + "network-status vote.", escaped(sk_hexdigest)); + goto err; + } - smartlist_add(sigs->signatures, voter); + siglist = detached_get_signatures(sigs, flavor); + is_duplicate = 0; + SMARTLIST_FOREACH(siglist, document_signature_t *, s, { + if (s->alg == alg && + !memcmp(id_digest, s->identity_digest, DIGEST_LEN) && + !memcmp(sk_digest, s->signing_key_digest, DIGEST_LEN)) { + is_duplicate = 1; + } }); + if (is_duplicate) { + log_warn(LD_DIR, "Two signatures with identical keys and algorithm " + "found."); + continue; + } + + sig = tor_malloc_zero(sizeof(document_signature_t)); + sig->alg = alg; + memcpy(sig->identity_digest, id_digest, DIGEST_LEN); + memcpy(sig->signing_key_digest, sk_digest, DIGEST_LEN); + if (tok->object_size >= INT_MAX || tok->object_size >= SIZE_T_CEILING) { + tor_free(sig); + goto err; + } + sig->signature = tor_memdup(tok->object_body, tok->object_size); + sig->signature_len = (int) tok->object_size; + + smartlist_add(siglist, sig); + } SMARTLIST_FOREACH_END(_tok); goto done; err: ns_detached_signatures_free(sigs); sigs = NULL; done: - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_free(tokens); if (area) { DUMP_AREA(area, "detached signatures"); @@ -2789,7 +3584,7 @@ router_parse_addr_policy_item_from_string(const char *s, int assume_action) err: r = NULL; done: - token_free(tok); + token_clear(tok); if (area) { DUMP_AREA(area, "policy item"); memarea_drop_all(area); @@ -2912,9 +3707,8 @@ assert_addr_policy_ok(smartlist_t *lst) /** Free all resources allocated for <b>tok</b> */ static void -token_free(directory_token_t *tok) +token_clear(directory_token_t *tok) { - tor_assert(tok); if (tok->key) crypto_free_pk_env(tok->key); } @@ -2926,7 +3720,7 @@ token_free(directory_token_t *tok) #define RET_ERR(msg) \ STMT_BEGIN \ - if (tok) token_free(tok); \ + if (tok) token_clear(tok); \ tok = ALLOC_ZERO(sizeof(directory_token_t)); \ tok->tp = _ERR; \ tok->error = STRDUP(msg); \ @@ -3222,7 +4016,7 @@ tokenize_string(memarea_t *area, tok = get_next_token(area, s, end, table); if (tok->tp == _ERR) { log_warn(LD_DIR, "parse error: %s", tok->error); - token_free(tok); + token_clear(tok); return -1; } ++counts[tok->tp]; @@ -3336,17 +4130,11 @@ find_all_exitpolicy(smartlist_t *s) return out; } -/** Compute the SHA-1 digest of the substring of <b>s</b> taken from the first - * occurrence of <b>start_str</b> through the first instance of c after the - * first subsequent occurrence of <b>end_str</b>; store the 20-byte result in - * <b>digest</b>; return 0 on success. - * - * If no such substring exists, return -1. - */ static int -router_get_hash_impl(const char *s, size_t s_len, char *digest, +router_get_hash_impl_helper(const char *s, size_t s_len, const char *start_str, - const char *end_str, char end_c) + const char *end_str, char end_c, + const char **start_out, const char **end_out) { const char *start, *end; start = tor_memstr(s, s_len, start_str); @@ -3373,14 +4161,215 @@ router_get_hash_impl(const char *s, size_t s_len, char *digest, } ++end; - if (crypto_digest(digest, start, end-start)) { - log_warn(LD_BUG,"couldn't compute digest"); + *start_out = start; + *end_out = end; + return 0; +} + +/** Compute the digest of the substring of <b>s</b> taken from the first + * occurrence of <b>start_str</b> through the first instance of c after the + * first subsequent occurrence of <b>end_str</b>; store the 20-byte result in + * <b>digest</b>; return 0 on success. + * + * If no such substring exists, return -1. + */ +static int +router_get_hash_impl(const char *s, size_t s_len, char *digest, + const char *start_str, + const char *end_str, char end_c, + digest_algorithm_t alg) +{ + const char *start=NULL, *end=NULL; + if (router_get_hash_impl_helper(s,s_len,start_str,end_str,end_c, + &start,&end)<0) return -1; + + if (alg == DIGEST_SHA1) { + if (crypto_digest(digest, start, end-start)) { + log_warn(LD_BUG,"couldn't compute digest"); + return -1; + } + } else { + if (crypto_digest256(digest, start, end-start, alg)) { + log_warn(LD_BUG,"couldn't compute digest"); + return -1; + } } return 0; } +/** As router_get_hash_impl, but compute all hashes. */ +static int +router_get_hashes_impl(const char *s, size_t s_len, digests_t *digests, + const char *start_str, + const char *end_str, char end_c) +{ + const char *start=NULL, *end=NULL; + if (router_get_hash_impl_helper(s,s_len,start_str,end_str,end_c, + &start,&end)<0) + return -1; + + if (crypto_digest_all(digests, start, end-start)) { + log_warn(LD_BUG,"couldn't compute digests"); + return -1; + } + + return 0; +} + +/** Assuming that s starts with a microdesc, return the start of the + * *NEXT* one. Return NULL on "not found." */ +static const char * +find_start_of_next_microdesc(const char *s, const char *eos) +{ + int started_with_annotations; + s = eat_whitespace_eos(s, eos); + if (!s) + return NULL; + +#define CHECK_LENGTH() STMT_BEGIN \ + if (s+32 > eos) \ + return NULL; \ + STMT_END + +#define NEXT_LINE() STMT_BEGIN \ + s = memchr(s, '\n', eos-s); \ + if (!s || s+1 >= eos) \ + return NULL; \ + s++; \ + STMT_END + + CHECK_LENGTH(); + + started_with_annotations = (*s == '@'); + + if (started_with_annotations) { + /* Start by advancing to the first non-annotation line. */ + while (*s == '@') + NEXT_LINE(); + } + CHECK_LENGTH(); + + /* Now we should be pointed at an onion-key line. If we are, then skip + * it. */ + if (!strcmpstart(s, "onion-key")) + NEXT_LINE(); + + /* Okay, now we're pointed at the first line of the microdescriptor which is + not an annotation or onion-key. The next line that _is_ an annotation or + onion-key is the start of the next microdescriptor. */ + while (s+32 < eos) { + if (*s == '@' || !strcmpstart(s, "onion-key")) + return s; + NEXT_LINE(); + } + return NULL; + +#undef CHECK_LENGTH +#undef NEXT_LINE +} + +/** Parse as many microdescriptors as are found from the string starting at + * <b>s</b> and ending at <b>eos</b>. If allow_annotations is set, read any + * annotations we recognize and ignore ones we don't. If <b>copy_body</b> is + * true, then strdup the bodies of the microdescriptors. Return all newly + * parsed microdescriptors in a newly allocated smartlist_t. */ +smartlist_t * +microdescs_parse_from_string(const char *s, const char *eos, + int allow_annotations, int copy_body) +{ + smartlist_t *tokens; + smartlist_t *result; + microdesc_t *md = NULL; + memarea_t *area; + const char *start = s; + const char *start_of_next_microdesc; + int flags = allow_annotations ? TS_ANNOTATIONS_OK : 0; + + directory_token_t *tok; + + if (!eos) + eos = s + strlen(s); + + s = eat_whitespace_eos(s, eos); + area = memarea_new(); + result = smartlist_create(); + tokens = smartlist_create(); + + while (s < eos) { + start_of_next_microdesc = find_start_of_next_microdesc(s, eos); + if (!start_of_next_microdesc) + start_of_next_microdesc = eos; + + if (tokenize_string(area, s, start_of_next_microdesc, tokens, + microdesc_token_table, flags)) { + log_warn(LD_DIR, "Unparseable microdescriptor"); + goto next; + } + + md = tor_malloc_zero(sizeof(microdesc_t)); + { + const char *cp = tor_memstr(s, start_of_next_microdesc-s, + "onion-key"); + tor_assert(cp); + + md->bodylen = start_of_next_microdesc - cp; + if (copy_body) + md->body = tor_strndup(cp, md->bodylen); + else + md->body = (char*)cp; + md->off = cp - start; + } + + if ((tok = find_opt_by_keyword(tokens, A_LAST_LISTED))) { + if (parse_iso_time(tok->args[0], &md->last_listed)) { + log_warn(LD_DIR, "Bad last-listed time in microdescriptor"); + goto next; + } + } + + tok = find_by_keyword(tokens, K_ONION_KEY); + md->onion_pkey = tok->key; + tok->key = NULL; + + if ((tok = find_opt_by_keyword(tokens, K_FAMILY))) { + int i; + md->family = smartlist_create(); + for (i=0;i<tok->n_args;++i) { + if (!is_legal_nickname_or_hexdigest(tok->args[i])) { + log_warn(LD_DIR, "Illegal nickname %s in family line", + escaped(tok->args[i])); + goto next; + } + smartlist_add(md->family, tor_strdup(tok->args[i])); + } + } + + if ((tok = find_opt_by_keyword(tokens, K_P))) { + md->exitsummary = tor_strdup(tok->args[0]); + } + + crypto_digest256(md->digest, md->body, md->bodylen, DIGEST_SHA256); + + smartlist_add(result, md); + + md = NULL; + next: + microdesc_free(md); + md = NULL; + + memarea_clear(area); + smartlist_clear(tokens); + s = start_of_next_microdesc; + } + + memarea_drop_all(area); + smartlist_free(tokens); + + return result; +} + /** Parse the Tor version of the platform string <b>platform</b>, * and compare it to the version in <b>cutoff</b>. Return 1 if * the router is at least as new as the cutoff, else return 0. @@ -3405,7 +4394,7 @@ tor_version_as_new_as(const char *platform, const char *cutoff) if (!*start) return 0; s = (char *)find_whitespace(start); /* also finds '\0', which is fine */ s2 = (char*)eat_whitespace(s); - if (!strcmpstart(s2, "(r")) + if (!strcmpstart(s2, "(r") || !strcmpstart(s2, "(git-")) s = (char*)find_whitespace(s2); if ((size_t)(s-start+1) >= sizeof(tmp)) /* too big, no */ @@ -3501,6 +4490,23 @@ tor_version_parse(const char *s, tor_version_t *out) if (!strcmpstart(cp, "(r")) { cp += 2; out->svn_revision = (int) strtol(cp,&eos,10); + } else if (!strcmpstart(cp, "(git-")) { + char *close_paren = strchr(cp, ')'); + int hexlen; + char digest[DIGEST_LEN]; + if (! close_paren) + return -1; + cp += 5; + if (close_paren-cp > HEX_DIGEST_LEN) + return -1; + hexlen = (int)(close_paren-cp); + memset(digest, 0, sizeof(digest)); + if ( hexlen == 0 || (hexlen % 2) == 1) + return -1; + if (base16_decode(digest, hexlen/2, cp, hexlen)) + return -1; + memcpy(out->git_tag, digest, hexlen/2); + out->git_tag_len = hexlen/2; } return 0; @@ -3526,8 +4532,14 @@ tor_version_compare(tor_version_t *a, tor_version_t *b) return i; else if ((i = strcmp(a->status_tag, b->status_tag))) return i; + else if ((i = a->svn_revision - b->svn_revision)) + return i; + else if ((i = a->git_tag_len - b->git_tag_len)) + return i; + else if (a->git_tag_len) + return memcmp(a->git_tag, b->git_tag, a->git_tag_len); else - return a->svn_revision - b->svn_revision; + return 0; } /** Return true iff versions <b>a</b> and <b>b</b> belong to the same series. @@ -3616,7 +4628,7 @@ rend_parse_v2_service_descriptor(rend_service_descriptor_t **parsed_out, /* Compute descriptor hash for later validation. */ if (router_get_hash_impl(desc, strlen(desc), desc_hash, "rendezvous-service-descriptor ", - "\nsignature", '\n') < 0) { + "\nsignature", '\n', DIGEST_SHA1) < 0) { log_warn(LD_REND, "Couldn't compute descriptor hash."); goto err; } @@ -3627,10 +4639,12 @@ rend_parse_v2_service_descriptor(rend_service_descriptor_t **parsed_out, else eos = eos + 1; /* Check length. */ - if (strlen(desc) > REND_DESC_MAX_SIZE) { - log_warn(LD_REND, "Descriptor length is %i which exceeds " - "maximum rendezvous descriptor size of %i kilobytes.", - (int)strlen(desc), REND_DESC_MAX_SIZE); + if (eos-desc > REND_DESC_MAX_SIZE) { + /* XXX023 If we are parsing this descriptor as a server, this + * should be a protocol warning. */ + log_warn(LD_REND, "Descriptor length is %d which exceeds " + "maximum rendezvous descriptor size of %d bytes.", + (int)(eos-desc), REND_DESC_MAX_SIZE); goto err; } /* Tokenize descriptor. */ @@ -3735,7 +4749,7 @@ rend_parse_v2_service_descriptor(rend_service_descriptor_t **parsed_out, /* Parse and verify signature. */ tok = find_by_keyword(tokens, R_SIGNATURE); note_crypto_pk_op(VERIFY_RTR); - if (check_signature_token(desc_hash, tok, result->pk, 0, + if (check_signature_token(desc_hash, DIGEST_LEN, tok, result->pk, 0, "v2 rendezvous service descriptor") < 0) goto err; /* Verify that descriptor ID belongs to public key and secret ID part. */ @@ -3749,12 +4763,11 @@ rend_parse_v2_service_descriptor(rend_service_descriptor_t **parsed_out, } goto done; err: - if (result) - rend_service_descriptor_free(result); + rend_service_descriptor_free(result); result = NULL; done: if (tokens) { - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_free(tokens); } if (area) @@ -3912,7 +4925,7 @@ rend_parse_introduction_points(rend_service_descriptor_t *parsed, eos = eos+1; tor_assert(eos <= intro_points_encoded+intro_points_encoded_size); /* Free tokens and clear token list. */ - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_clear(tokens); memarea_clear(area); /* Tokenize string. */ @@ -3985,7 +4998,7 @@ rend_parse_introduction_points(rend_service_descriptor_t *parsed, done: /* Free tokens and clear token list. */ - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_free(tokens); if (area) memarea_drop_all(area); @@ -4024,7 +5037,7 @@ rend_parse_client_keys(strmap_t *parsed_clients, const char *ckstr) else eos = eos + 1; /* Free tokens and clear token list. */ - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_clear(tokens); memarea_clear(area); /* Tokenize string. */ @@ -4096,7 +5109,7 @@ rend_parse_client_keys(strmap_t *parsed_clients, const char *ckstr) result = -1; done: /* Free tokens and clear token list. */ - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_free(tokens); if (area) memarea_drop_all(area); diff --git a/src/or/routerparse.h b/src/or/routerparse.h new file mode 100644 index 0000000000..8b8cde25f6 --- /dev/null +++ b/src/or/routerparse.h @@ -0,0 +1,86 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file routerparse.h + * \brief Header file for routerparse.c. + **/ + +#ifndef _TOR_ROUTERPARSE_H +#define _TOR_ROUTERPARSE_H + +int router_get_router_hash(const char *s, size_t s_len, char *digest); +int router_get_dir_hash(const char *s, char *digest); +int router_get_runningrouters_hash(const char *s, char *digest); +int router_get_networkstatus_v2_hash(const char *s, char *digest); +int router_get_networkstatus_v3_hash(const char *s, char *digest, + digest_algorithm_t algorithm); +int router_get_networkstatus_v3_hashes(const char *s, digests_t *digests); +int router_get_extrainfo_hash(const char *s, char *digest); +#define DIROBJ_MAX_SIG_LEN 256 +int router_append_dirobj_signature(char *buf, size_t buf_len, + const char *digest, + size_t digest_len, + crypto_pk_env_t *private_key); +int router_parse_list_from_string(const char **s, const char *eos, + smartlist_t *dest, + saved_location_t saved_location, + int is_extrainfo, + int allow_annotations, + const char *prepend_annotations); +int router_parse_runningrouters(const char *str); +int router_parse_directory(const char *str); + +routerinfo_t *router_parse_entry_from_string(const char *s, const char *end, + int cache_copy, + int allow_annotations, + const char *prepend_annotations); +extrainfo_t *extrainfo_parse_entry_from_string(const char *s, const char *end, + int cache_copy, struct digest_ri_map_t *routermap); +addr_policy_t *router_parse_addr_policy_item_from_string(const char *s, + int assume_action); +version_status_t tor_version_is_obsolete(const char *myversion, + const char *versionlist); +int tor_version_parse(const char *s, tor_version_t *out); +int tor_version_as_new_as(const char *platform, const char *cutoff); +int tor_version_compare(tor_version_t *a, tor_version_t *b); +void sort_version_list(smartlist_t *lst, int remove_duplicates); +void assert_addr_policy_ok(smartlist_t *t); +void dump_distinct_digest_count(int severity); + +int compare_routerstatus_entries(const void **_a, const void **_b); +networkstatus_v2_t *networkstatus_v2_parse_from_string(const char *s); +int networkstatus_verify_bw_weights(networkstatus_t *ns); +networkstatus_t *networkstatus_parse_vote_from_string(const char *s, + const char **eos_out, + networkstatus_type_t ns_type); +ns_detached_signatures_t *networkstatus_parse_detached_signatures( + const char *s, const char *eos); + +smartlist_t *microdescs_parse_from_string(const char *s, const char *eos, + int allow_annotations, + int copy_body); + +authority_cert_t *authority_cert_parse_from_string(const char *s, + const char **end_of_string); +int rend_parse_v2_service_descriptor(rend_service_descriptor_t **parsed_out, + char *desc_id_out, + char **intro_points_encrypted_out, + size_t *intro_points_encrypted_size_out, + size_t *encoded_size_out, + const char **next_out, const char *desc); +int rend_decrypt_introduction_points(char **ipos_decrypted, + size_t *ipos_decrypted_size, + const char *descriptor_cookie, + const char *ipos_encrypted, + size_t ipos_encrypted_size); +int rend_parse_introduction_points(rend_service_descriptor_t *parsed, + const char *intro_points_encoded, + size_t intro_points_encoded_size); +int rend_parse_client_keys(strmap_t *parsed_clients, const char *str); + +#endif + diff --git a/src/or/test.c b/src/or/test.c deleted file mode 100644 index b08f202c20..0000000000 --- a/src/or/test.c +++ /dev/null @@ -1,4918 +0,0 @@ -/* Copyright (c) 2001-2004, Roger Dingledine. - * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2011, The Tor Project, Inc. */ -/* See LICENSE for licensing information */ - -/* Ordinarily defined in tor_main.c; this bit is just here to provide one - * since we're not linking to tor_main.c */ -const char tor_svn_revision[] = ""; - -/** - * \file test.c - * \brief Unit tests for many pieces of the lower level Tor modules. - **/ - -#include "orconfig.h" - -#include <stdio.h> -#ifdef HAVE_FCNTL_H -#include <fcntl.h> -#endif - -#ifdef MS_WINDOWS -/* For mkdir() */ -#include <direct.h> -#else -#include <dirent.h> -#endif - -/* These macros pull in declarations for some functions and structures that - * are typically file-private. */ -#define BUFFERS_PRIVATE -#define CONFIG_PRIVATE -#define CONTROL_PRIVATE -#define CRYPTO_PRIVATE -#define DIRSERV_PRIVATE -#define DIRVOTE_PRIVATE -#define GEOIP_PRIVATE -#define MEMPOOL_PRIVATE -#define ROUTER_PRIVATE - -#include "or.h" -#include "test.h" -#include "torgzip.h" -#include "mempool.h" -#include "memarea.h" - -#ifdef USE_DMALLOC -#include <dmalloc.h> -#include <openssl/crypto.h> -#endif - -/** Set to true if any unit test has failed. Mostly, this is set by the macros - * in test.h */ -int have_failed = 0; - -/** Temporary directory (set up by setup_directory) under which we store all - * our files during testing. */ -static char temp_dir[256]; - -/** Select and create the temporary directory we'll use to run our unit tests. - * Store it in <b>temp_dir</b>. Exit immediately if we can't create it. - * idempotent. */ -static void -setup_directory(void) -{ - static int is_setup = 0; - int r; - if (is_setup) return; - -#ifdef MS_WINDOWS - // XXXX - tor_snprintf(temp_dir, sizeof(temp_dir), - "c:\\windows\\temp\\tor_test_%d", (int)getpid()); - r = mkdir(temp_dir); -#else - tor_snprintf(temp_dir, sizeof(temp_dir), "/tmp/tor_test_%d", (int) getpid()); - r = mkdir(temp_dir, 0700); -#endif - if (r) { - fprintf(stderr, "Can't create directory %s:", temp_dir); - perror(""); - exit(1); - } - is_setup = 1; -} - -/** Return a filename relative to our testing temporary directory */ -static const char * -get_fname(const char *name) -{ - static char buf[1024]; - setup_directory(); - tor_snprintf(buf,sizeof(buf),"%s/%s",temp_dir,name); - return buf; -} - -/** Remove all files stored under the temporary directory, and the directory - * itself. */ -static void -remove_directory(void) -{ - smartlist_t *elements = tor_listdir(temp_dir); - if (elements) { - SMARTLIST_FOREACH(elements, const char *, cp, - { - size_t len = strlen(cp)+strlen(temp_dir)+16; - char *tmp = tor_malloc(len); - tor_snprintf(tmp, len, "%s"PATH_SEPARATOR"%s", temp_dir, cp); - unlink(tmp); - tor_free(tmp); - }); - SMARTLIST_FOREACH(elements, char *, cp, tor_free(cp)); - smartlist_free(elements); - } - rmdir(temp_dir); -} - -/** Define this if unit tests spend too much time generating public keys*/ -#undef CACHE_GENERATED_KEYS - -static crypto_pk_env_t *pregen_keys[5] = {NULL, NULL, NULL, NULL, NULL}; -#define N_PREGEN_KEYS ((int)(sizeof(pregen_keys)/sizeof(pregen_keys[0]))) - -/** Generate and return a new keypair for use in unit tests. If we're using - * the key cache optimization, we might reuse keys: we only guarantee that - * keys made with distinct values for <b>idx</b> are different. The value of - * <b>idx</b> must be at least 0, and less than N_PREGEN_KEYS. */ -static crypto_pk_env_t * -pk_generate(int idx) -{ -#ifdef CACHE_GENERATED_KEYS - tor_assert(idx < N_PREGEN_KEYS); - if (! pregen_keys[idx]) { - pregen_keys[idx] = crypto_new_pk_env(); - tor_assert(!crypto_pk_generate_key(pregen_keys[idx])); - } - return crypto_pk_dup_key(pregen_keys[idx]); -#else - crypto_pk_env_t *result; - (void) idx; - result = crypto_new_pk_env(); - tor_assert(!crypto_pk_generate_key(result)); - return result; -#endif -} - -/** Free all storage used for the cached key optimization. */ -static void -free_pregenerated_keys(void) -{ - unsigned idx; - for (idx = 0; idx < N_PREGEN_KEYS; ++idx) { - if (pregen_keys[idx]) { - crypto_free_pk_env(pregen_keys[idx]); - pregen_keys[idx] = NULL; - } - } -} - -/** Run unit tests for buffers.c */ -static void -test_buffers(void) -{ - char str[256]; - char str2[256]; - - buf_t *buf = NULL, *buf2 = NULL; - const char *cp; - - int j; - size_t r; - - /**** - * buf_new - ****/ - if (!(buf = buf_new())) - test_fail(); - - //test_eq(buf_capacity(buf), 4096); - test_eq(buf_datalen(buf), 0); - - /**** - * General pointer frobbing - */ - for (j=0;j<256;++j) { - str[j] = (char)j; - } - write_to_buf(str, 256, buf); - write_to_buf(str, 256, buf); - test_eq(buf_datalen(buf), 512); - fetch_from_buf(str2, 200, buf); - test_memeq(str, str2, 200); - test_eq(buf_datalen(buf), 312); - memset(str2, 0, sizeof(str2)); - - fetch_from_buf(str2, 256, buf); - test_memeq(str+200, str2, 56); - test_memeq(str, str2+56, 200); - test_eq(buf_datalen(buf), 56); - memset(str2, 0, sizeof(str2)); - /* Okay, now we should be 512 bytes into the 4096-byte buffer. If we add - * another 3584 bytes, we hit the end. */ - for (j=0;j<15;++j) { - write_to_buf(str, 256, buf); - } - assert_buf_ok(buf); - test_eq(buf_datalen(buf), 3896); - fetch_from_buf(str2, 56, buf); - test_eq(buf_datalen(buf), 3840); - test_memeq(str+200, str2, 56); - for (j=0;j<15;++j) { - memset(str2, 0, sizeof(str2)); - fetch_from_buf(str2, 256, buf); - test_memeq(str, str2, 256); - } - test_eq(buf_datalen(buf), 0); - buf_free(buf); - buf = NULL; - - /* Okay, now make sure growing can work. */ - buf = buf_new_with_capacity(16); - //test_eq(buf_capacity(buf), 16); - write_to_buf(str+1, 255, buf); - //test_eq(buf_capacity(buf), 256); - fetch_from_buf(str2, 254, buf); - test_memeq(str+1, str2, 254); - //test_eq(buf_capacity(buf), 256); - assert_buf_ok(buf); - write_to_buf(str, 32, buf); - //test_eq(buf_capacity(buf), 256); - assert_buf_ok(buf); - write_to_buf(str, 256, buf); - assert_buf_ok(buf); - //test_eq(buf_capacity(buf), 512); - test_eq(buf_datalen(buf), 33+256); - fetch_from_buf(str2, 33, buf); - test_eq(*str2, str[255]); - - test_memeq(str2+1, str, 32); - //test_eq(buf_capacity(buf), 512); - test_eq(buf_datalen(buf), 256); - fetch_from_buf(str2, 256, buf); - test_memeq(str, str2, 256); - - /* now try shrinking: case 1. */ - buf_free(buf); - buf = buf_new_with_capacity(33668); - for (j=0;j<67;++j) { - write_to_buf(str,255, buf); - } - //test_eq(buf_capacity(buf), 33668); - test_eq(buf_datalen(buf), 17085); - for (j=0; j < 40; ++j) { - fetch_from_buf(str2, 255,buf); - test_memeq(str2, str, 255); - } - - /* now try shrinking: case 2. */ - buf_free(buf); - buf = buf_new_with_capacity(33668); - for (j=0;j<67;++j) { - write_to_buf(str,255, buf); - } - for (j=0; j < 20; ++j) { - fetch_from_buf(str2, 255,buf); - test_memeq(str2, str, 255); - } - for (j=0;j<80;++j) { - write_to_buf(str,255, buf); - } - //test_eq(buf_capacity(buf),33668); - for (j=0; j < 120; ++j) { - fetch_from_buf(str2, 255,buf); - test_memeq(str2, str, 255); - } - - /* Move from buf to buf. */ - buf_free(buf); - buf = buf_new_with_capacity(4096); - buf2 = buf_new_with_capacity(4096); - for (j=0;j<100;++j) - write_to_buf(str, 255, buf); - test_eq(buf_datalen(buf), 25500); - for (j=0;j<100;++j) { - r = 10; - move_buf_to_buf(buf2, buf, &r); - test_eq(r, 0); - } - test_eq(buf_datalen(buf), 24500); - test_eq(buf_datalen(buf2), 1000); - for (j=0;j<3;++j) { - fetch_from_buf(str2, 255, buf2); - test_memeq(str2, str, 255); - } - r = 8192; /*big move*/ - move_buf_to_buf(buf2, buf, &r); - test_eq(r, 0); - r = 30000; /* incomplete move */ - move_buf_to_buf(buf2, buf, &r); - test_eq(r, 13692); - for (j=0;j<97;++j) { - fetch_from_buf(str2, 255, buf2); - test_memeq(str2, str, 255); - } - buf_free(buf); - buf_free(buf2); - buf = buf2 = NULL; - - buf = buf_new_with_capacity(5); - cp = "Testing. This is a moderately long Testing string."; - for (j = 0; cp[j]; j++) - write_to_buf(cp+j, 1, buf); - test_eq(0, buf_find_string_offset(buf, "Testing", 7)); - test_eq(1, buf_find_string_offset(buf, "esting", 6)); - test_eq(1, buf_find_string_offset(buf, "est", 3)); - test_eq(39, buf_find_string_offset(buf, "ing str", 7)); - test_eq(35, buf_find_string_offset(buf, "Testing str", 11)); - test_eq(32, buf_find_string_offset(buf, "ng ", 3)); - test_eq(43, buf_find_string_offset(buf, "string.", 7)); - test_eq(-1, buf_find_string_offset(buf, "shrdlu", 6)); - test_eq(-1, buf_find_string_offset(buf, "Testing thing", 13)); - test_eq(-1, buf_find_string_offset(buf, "ngx", 3)); - buf_free(buf); - buf = NULL; - -#if 0 - { - int s; - int eof; - int i; - buf_t *buf2; - /**** - * read_to_buf - ****/ - s = open(get_fname("data"), O_WRONLY|O_CREAT|O_TRUNC, 0600); - write(s, str, 256); - close(s); - - s = open(get_fname("data"), O_RDONLY, 0); - eof = 0; - errno = 0; /* XXXX */ - i = read_to_buf(s, 10, buf, &eof); - printf("%s\n", strerror(errno)); - test_eq(i, 10); - test_eq(eof, 0); - //test_eq(buf_capacity(buf), 4096); - test_eq(buf_datalen(buf), 10); - - test_memeq(str, (char*)_buf_peek_raw_buffer(buf), 10); - - /* Test reading 0 bytes. */ - i = read_to_buf(s, 0, buf, &eof); - //test_eq(buf_capacity(buf), 512*1024); - test_eq(buf_datalen(buf), 10); - test_eq(eof, 0); - test_eq(i, 0); - - /* Now test when buffer is filled exactly. */ - buf2 = buf_new_with_capacity(6); - i = read_to_buf(s, 6, buf2, &eof); - //test_eq(buf_capacity(buf2), 6); - test_eq(buf_datalen(buf2), 6); - test_eq(eof, 0); - test_eq(i, 6); - test_memeq(str+10, (char*)_buf_peek_raw_buffer(buf2), 6); - buf_free(buf2); - buf2 = NULL; - - /* Now test when buffer is filled with more data to read. */ - buf2 = buf_new_with_capacity(32); - i = read_to_buf(s, 128, buf2, &eof); - //test_eq(buf_capacity(buf2), 128); - test_eq(buf_datalen(buf2), 32); - test_eq(eof, 0); - test_eq(i, 32); - buf_free(buf2); - buf2 = NULL; - - /* Now read to eof. */ - test_assert(buf_capacity(buf) > 256); - i = read_to_buf(s, 1024, buf, &eof); - test_eq(i, (256-32-10-6)); - test_eq(buf_capacity(buf), MAX_BUF_SIZE); - test_eq(buf_datalen(buf), 256-6-32); - test_memeq(str, (char*)_buf_peek_raw_buffer(buf), 10); /* XXX Check rest. */ - test_eq(eof, 0); - - i = read_to_buf(s, 1024, buf, &eof); - test_eq(i, 0); - test_eq(buf_capacity(buf), MAX_BUF_SIZE); - test_eq(buf_datalen(buf), 256-6-32); - test_eq(eof, 1); - } -#endif - - done: - if (buf) - buf_free(buf); - if (buf2) - buf_free(buf2); -} - -/** Run unit tests for Diffie-Hellman functionality. */ -static void -test_crypto_dh(void) -{ - crypto_dh_env_t *dh1 = crypto_dh_new(DH_TYPE_CIRCUIT); - crypto_dh_env_t *dh2 = crypto_dh_new(DH_TYPE_CIRCUIT); - char p1[DH_BYTES]; - char p2[DH_BYTES]; - char s1[DH_BYTES]; - char s2[DH_BYTES]; - ssize_t s1len, s2len; - - test_eq(crypto_dh_get_bytes(dh1), DH_BYTES); - test_eq(crypto_dh_get_bytes(dh2), DH_BYTES); - - memset(p1, 0, DH_BYTES); - memset(p2, 0, DH_BYTES); - test_memeq(p1, p2, DH_BYTES); - test_assert(! crypto_dh_get_public(dh1, p1, DH_BYTES)); - test_memneq(p1, p2, DH_BYTES); - test_assert(! crypto_dh_get_public(dh2, p2, DH_BYTES)); - test_memneq(p1, p2, DH_BYTES); - - memset(s1, 0, DH_BYTES); - memset(s2, 0xFF, DH_BYTES); - s1len = crypto_dh_compute_secret(dh1, p2, DH_BYTES, s1, 50); - s2len = crypto_dh_compute_secret(dh2, p1, DH_BYTES, s2, 50); - test_assert(s1len > 0); - test_eq(s1len, s2len); - test_memeq(s1, s2, s1len); - - { - /* XXXX Now fabricate some bad values and make sure they get caught, - * Check 0, 1, N-1, >= N, etc. - */ - } - - done: - crypto_dh_free(dh1); - crypto_dh_free(dh2); -} - -/** Run unit tests for our random number generation function and its wrappers. - */ -static void -test_crypto_rng(void) -{ - int i, j, allok; - char data1[100], data2[100]; - - /* Try out RNG. */ - test_assert(! crypto_seed_rng(0)); - crypto_rand(data1, 100); - crypto_rand(data2, 100); - test_memneq(data1,data2,100); - allok = 1; - for (i = 0; i < 100; ++i) { - uint64_t big; - char *host; - j = crypto_rand_int(100); - if (i < 0 || i >= 100) - allok = 0; - big = crypto_rand_uint64(U64_LITERAL(1)<<40); - if (big >= (U64_LITERAL(1)<<40)) - allok = 0; - big = crypto_rand_uint64(U64_LITERAL(5)); - if (big >= 5) - allok = 0; - host = crypto_random_hostname(3,8,"www.",".onion"); - if (strcmpstart(host,"www.") || - strcmpend(host,".onion") || - strlen(host) < 13 || - strlen(host) > 18) - allok = 0; - tor_free(host); - } - test_assert(allok); - done: - ; -} - -/** Run unit tests for our AES functionality */ -static void -test_crypto_aes(void) -{ - char *data1 = NULL, *data2 = NULL, *data3 = NULL; - crypto_cipher_env_t *env1 = NULL, *env2 = NULL; - int i, j; - - data1 = tor_malloc(1024); - data2 = tor_malloc(1024); - data3 = tor_malloc(1024); - - /* Now, test encryption and decryption with stream cipher. */ - data1[0]='\0'; - for (i = 1023; i>0; i -= 35) - strncat(data1, "Now is the time for all good onions", i); - - memset(data2, 0, 1024); - memset(data3, 0, 1024); - env1 = crypto_new_cipher_env(); - test_neq(env1, 0); - env2 = crypto_new_cipher_env(); - test_neq(env2, 0); - j = crypto_cipher_generate_key(env1); - crypto_cipher_set_key(env2, crypto_cipher_get_key(env1)); - crypto_cipher_encrypt_init_cipher(env1); - crypto_cipher_decrypt_init_cipher(env2); - - /* Try encrypting 512 chars. */ - crypto_cipher_encrypt(env1, data2, data1, 512); - crypto_cipher_decrypt(env2, data3, data2, 512); - test_memeq(data1, data3, 512); - test_memneq(data1, data2, 512); - - /* Now encrypt 1 at a time, and get 1 at a time. */ - for (j = 512; j < 560; ++j) { - crypto_cipher_encrypt(env1, data2+j, data1+j, 1); - } - for (j = 512; j < 560; ++j) { - crypto_cipher_decrypt(env2, data3+j, data2+j, 1); - } - test_memeq(data1, data3, 560); - /* Now encrypt 3 at a time, and get 5 at a time. */ - for (j = 560; j < 1024-5; j += 3) { - crypto_cipher_encrypt(env1, data2+j, data1+j, 3); - } - for (j = 560; j < 1024-5; j += 5) { - crypto_cipher_decrypt(env2, data3+j, data2+j, 5); - } - test_memeq(data1, data3, 1024-5); - /* Now make sure that when we encrypt with different chunk sizes, we get - the same results. */ - crypto_free_cipher_env(env2); - env2 = NULL; - - memset(data3, 0, 1024); - env2 = crypto_new_cipher_env(); - test_neq(env2, 0); - crypto_cipher_set_key(env2, crypto_cipher_get_key(env1)); - crypto_cipher_encrypt_init_cipher(env2); - for (j = 0; j < 1024-16; j += 17) { - crypto_cipher_encrypt(env2, data3+j, data1+j, 17); - } - for (j= 0; j < 1024-16; ++j) { - if (data2[j] != data3[j]) { - printf("%d: %d\t%d\n", j, (int) data2[j], (int) data3[j]); - } - } - test_memeq(data2, data3, 1024-16); - crypto_free_cipher_env(env1); - env1 = NULL; - crypto_free_cipher_env(env2); - env2 = NULL; - - /* NIST test vector for aes. */ - env1 = crypto_new_cipher_env(); /* IV starts at 0 */ - crypto_cipher_set_key(env1, "\x80\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00"); - crypto_cipher_encrypt_init_cipher(env1); - crypto_cipher_encrypt(env1, data1, - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00", 16); - test_memeq_hex(data1, "0EDD33D3C621E546455BD8BA1418BEC8"); - - /* Now test rollover. All these values are originally from a python - * script. */ - crypto_cipher_set_iv(env1, "\x00\x00\x00\x00\x00\x00\x00\x00" - "\xff\xff\xff\xff\xff\xff\xff\xff"); - memset(data2, 0, 1024); - crypto_cipher_encrypt(env1, data1, data2, 32); - test_memeq_hex(data1, "335fe6da56f843199066c14a00a40231" - "cdd0b917dbc7186908a6bfb5ffd574d3"); - - crypto_cipher_set_iv(env1, "\x00\x00\x00\x00\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff"); - memset(data2, 0, 1024); - crypto_cipher_encrypt(env1, data1, data2, 32); - test_memeq_hex(data1, "e627c6423fa2d77832a02b2794094b73" - "3e63c721df790d2c6469cc1953a3ffac"); - - crypto_cipher_set_iv(env1, "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff"); - memset(data2, 0, 1024); - crypto_cipher_encrypt(env1, data1, data2, 32); - test_memeq_hex(data1, "2aed2bff0de54f9328efd070bf48f70a" - "0EDD33D3C621E546455BD8BA1418BEC8"); - - /* Now check rollover on inplace cipher. */ - crypto_cipher_set_iv(env1, "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff"); - crypto_cipher_crypt_inplace(env1, data2, 64); - test_memeq_hex(data2, "2aed2bff0de54f9328efd070bf48f70a" - "0EDD33D3C621E546455BD8BA1418BEC8" - "93e2c5243d6839eac58503919192f7ae" - "1908e67cafa08d508816659c2e693191"); - crypto_cipher_set_iv(env1, "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff"); - crypto_cipher_crypt_inplace(env1, data2, 64); - test_assert(tor_mem_is_zero(data2, 64)); - - done: - if (env1) - crypto_free_cipher_env(env1); - if (env2) - crypto_free_cipher_env(env2); - tor_free(data1); - tor_free(data2); - tor_free(data3); -} - -/** Run unit tests for our SHA-1 functionality */ -static void -test_crypto_sha(void) -{ - crypto_digest_env_t *d1 = NULL, *d2 = NULL; - int i; - char key[80]; - char digest[20]; - char data[50]; - char d_out1[DIGEST_LEN], d_out2[DIGEST_LEN]; - - /* Test SHA-1 with a test vector from the specification. */ - i = crypto_digest(data, "abc", 3); - test_memeq_hex(data, "A9993E364706816ABA3E25717850C26C9CD0D89D"); - - /* Test HMAC-SHA-1 with test cases from RFC2202. */ - - /* Case 1. */ - memset(key, 0x0b, 20); - crypto_hmac_sha1(digest, key, 20, "Hi There", 8); - test_streq(hex_str(digest, 20), - "B617318655057264E28BC0B6FB378C8EF146BE00"); - /* Case 2. */ - crypto_hmac_sha1(digest, "Jefe", 4, "what do ya want for nothing?", 28); - test_streq(hex_str(digest, 20), - "EFFCDF6AE5EB2FA2D27416D5F184DF9C259A7C79"); - - /* Case 4. */ - base16_decode(key, 25, - "0102030405060708090a0b0c0d0e0f10111213141516171819", 50); - memset(data, 0xcd, 50); - crypto_hmac_sha1(digest, key, 25, data, 50); - test_streq(hex_str(digest, 20), - "4C9007F4026250C6BC8414F9BF50C86C2D7235DA"); - - /* Case . */ - memset(key, 0xaa, 80); - crypto_hmac_sha1(digest, key, 80, - "Test Using Larger Than Block-Size Key - Hash Key First", - 54); - test_streq(hex_str(digest, 20), - "AA4AE5E15272D00E95705637CE8A3B55ED402112"); - - /* Incremental digest code. */ - d1 = crypto_new_digest_env(); - test_assert(d1); - crypto_digest_add_bytes(d1, "abcdef", 6); - d2 = crypto_digest_dup(d1); - test_assert(d2); - crypto_digest_add_bytes(d2, "ghijkl", 6); - crypto_digest_get_digest(d2, d_out1, sizeof(d_out1)); - crypto_digest(d_out2, "abcdefghijkl", 12); - test_memeq(d_out1, d_out2, DIGEST_LEN); - crypto_digest_assign(d2, d1); - crypto_digest_add_bytes(d2, "mno", 3); - crypto_digest_get_digest(d2, d_out1, sizeof(d_out1)); - crypto_digest(d_out2, "abcdefmno", 9); - test_memeq(d_out1, d_out2, DIGEST_LEN); - crypto_digest_get_digest(d1, d_out1, sizeof(d_out1)); - crypto_digest(d_out2, "abcdef", 6); - test_memeq(d_out1, d_out2, DIGEST_LEN); - - done: - if (d1) - crypto_free_digest_env(d1); - if (d2) - crypto_free_digest_env(d2); -} - -/** Run unit tests for our public key crypto functions */ -static void -test_crypto_pk(void) -{ - crypto_pk_env_t *pk1 = NULL, *pk2 = NULL; - char *encoded = NULL; - char data1[1024], data2[1024], data3[1024]; - size_t size; - int i, j, p, len; - - /* Public-key ciphers */ - pk1 = pk_generate(0); - pk2 = crypto_new_pk_env(); - test_assert(pk1 && pk2); - test_assert(! crypto_pk_write_public_key_to_string(pk1, &encoded, &size)); - test_assert(! crypto_pk_read_public_key_from_string(pk2, encoded, size)); - test_eq(0, crypto_pk_cmp_keys(pk1, pk2)); - - test_eq(128, crypto_pk_keysize(pk1)); - test_eq(128, crypto_pk_keysize(pk2)); - - test_eq(128, crypto_pk_public_encrypt(pk2, data1, sizeof(data1), - "Hello whirled.", 15, - PK_PKCS1_OAEP_PADDING)); - test_eq(128, crypto_pk_public_encrypt(pk1, data2, sizeof(data2), - "Hello whirled.", 15, - PK_PKCS1_OAEP_PADDING)); - /* oaep padding should make encryption not match */ - test_memneq(data1, data2, 128); - test_eq(15, crypto_pk_private_decrypt(pk1, data3, sizeof(data3), data1, 128, - PK_PKCS1_OAEP_PADDING,1)); - test_streq(data3, "Hello whirled."); - memset(data3, 0, 1024); - test_eq(15, crypto_pk_private_decrypt(pk1, data3, sizeof(data3), data2, 128, - PK_PKCS1_OAEP_PADDING,1)); - test_streq(data3, "Hello whirled."); - /* Can't decrypt with public key. */ - test_eq(-1, crypto_pk_private_decrypt(pk2, data3, sizeof(data3), data2, 128, - PK_PKCS1_OAEP_PADDING,1)); - /* Try again with bad padding */ - memcpy(data2+1, "XYZZY", 5); /* This has fails ~ once-in-2^40 */ - test_eq(-1, crypto_pk_private_decrypt(pk1, data3, sizeof(data3), data2, 128, - PK_PKCS1_OAEP_PADDING,1)); - - /* File operations: save and load private key */ - test_assert(! crypto_pk_write_private_key_to_filename(pk1, - get_fname("pkey1"))); - /* failing case for read: can't read. */ - test_assert(crypto_pk_read_private_key_from_filename(pk2, - get_fname("xyzzy")) < 0); - write_str_to_file(get_fname("xyzzy"), "foobar", 6); - /* Failing case for read: no key. */ - test_assert(crypto_pk_read_private_key_from_filename(pk2, - get_fname("xyzzy")) < 0); - test_assert(! crypto_pk_read_private_key_from_filename(pk2, - get_fname("pkey1"))); - test_eq(15, crypto_pk_private_decrypt(pk2, data3, sizeof(data3), data1, 128, - PK_PKCS1_OAEP_PADDING,1)); - - /* Now try signing. */ - strlcpy(data1, "Ossifrage", 1024); - test_eq(128, crypto_pk_private_sign(pk1, data2, sizeof(data2), data1, 10)); - test_eq(10, crypto_pk_public_checksig(pk1, data3, sizeof(data3), data2, 128)); - test_streq(data3, "Ossifrage"); - /* Try signing digests. */ - test_eq(128, crypto_pk_private_sign_digest(pk1, data2, sizeof(data2), - data1, 10)); - test_eq(20, crypto_pk_public_checksig(pk1, data3, sizeof(data1), data2, 128)); - test_eq(0, crypto_pk_public_checksig_digest(pk1, data1, - 10, data2, 128)); - test_eq(-1, crypto_pk_public_checksig_digest(pk1, data1, - 11, data2, 128)); - /*XXXX test failed signing*/ - - /* Try encoding */ - crypto_free_pk_env(pk2); - pk2 = NULL; - i = crypto_pk_asn1_encode(pk1, data1, 1024); - test_assert(i>0); - pk2 = crypto_pk_asn1_decode(data1, i); - test_assert(crypto_pk_cmp_keys(pk1,pk2) == 0); - - /* Try with hybrid encryption wrappers. */ - crypto_rand(data1, 1024); - for (i = 0; i < 3; ++i) { - for (j = 85; j < 140; ++j) { - memset(data2,0,1024); - memset(data3,0,1024); - if (i == 0 && j < 129) - continue; - p = (i==0)?PK_NO_PADDING: - (i==1)?PK_PKCS1_PADDING:PK_PKCS1_OAEP_PADDING; - len = crypto_pk_public_hybrid_encrypt(pk1,data2,sizeof(data2), - data1,j,p,0); - test_assert(len>=0); - len = crypto_pk_private_hybrid_decrypt(pk1,data3,sizeof(data3), - data2,len,p,1); - test_eq(len,j); - test_memeq(data1,data3,j); - } - } - - /* Try copy_full */ - crypto_free_pk_env(pk2); - pk2 = crypto_pk_copy_full(pk1); - test_assert(pk2 != NULL); - test_neq_ptr(pk1, pk2); - test_assert(crypto_pk_cmp_keys(pk1,pk2) == 0); - - done: - if (pk1) - crypto_free_pk_env(pk1); - if (pk2) - crypto_free_pk_env(pk2); - tor_free(encoded); -} - -/** Run unit tests for misc crypto functionality. */ -static void -test_crypto(void) -{ - char *data1 = NULL, *data2 = NULL, *data3 = NULL; - int i, j, idx; - - data1 = tor_malloc(1024); - data2 = tor_malloc(1024); - data3 = tor_malloc(1024); - test_assert(data1 && data2 && data3); - - /* Base64 tests */ - memset(data1, 6, 1024); - for (idx = 0; idx < 10; ++idx) { - i = base64_encode(data2, 1024, data1, idx); - test_assert(i >= 0); - j = base64_decode(data3, 1024, data2, i); - test_eq(j,idx); - test_memeq(data3, data1, idx); - } - - strlcpy(data1, "Test string that contains 35 chars.", 1024); - strlcat(data1, " 2nd string that contains 35 chars.", 1024); - - i = base64_encode(data2, 1024, data1, 71); - j = base64_decode(data3, 1024, data2, i); - test_eq(j, 71); - test_streq(data3, data1); - test_assert(data2[i] == '\0'); - - crypto_rand(data1, DIGEST_LEN); - memset(data2, 100, 1024); - digest_to_base64(data2, data1); - test_eq(BASE64_DIGEST_LEN, strlen(data2)); - test_eq(100, data2[BASE64_DIGEST_LEN+2]); - memset(data3, 99, 1024); - test_eq(digest_from_base64(data3, data2), 0); - test_memeq(data1, data3, DIGEST_LEN); - test_eq(99, data3[DIGEST_LEN+1]); - - test_assert(digest_from_base64(data3, "###") < 0); - - /* Base32 tests */ - strlcpy(data1, "5chrs", 1024); - /* bit pattern is: [35 63 68 72 73] -> - * [00110101 01100011 01101000 01110010 01110011] - * By 5s: [00110 10101 10001 10110 10000 11100 10011 10011] - */ - base32_encode(data2, 9, data1, 5); - test_streq(data2, "gvrwq4tt"); - - strlcpy(data1, "\xFF\xF5\x6D\x44\xAE\x0D\x5C\xC9\x62\xC4", 1024); - base32_encode(data2, 30, data1, 10); - test_streq(data2, "772w2rfobvomsywe"); - - /* Base16 tests */ - strlcpy(data1, "6chrs\xff", 1024); - base16_encode(data2, 13, data1, 6); - test_streq(data2, "3663687273FF"); - - strlcpy(data1, "f0d678affc000100", 1024); - i = base16_decode(data2, 8, data1, 16); - test_eq(i,0); - test_memeq(data2, "\xf0\xd6\x78\xaf\xfc\x00\x01\x00",8); - - /* now try some failing base16 decodes */ - test_eq(-1, base16_decode(data2, 8, data1, 15)); /* odd input len */ - test_eq(-1, base16_decode(data2, 7, data1, 16)); /* dest too short */ - strlcpy(data1, "f0dz!8affc000100", 1024); - test_eq(-1, base16_decode(data2, 8, data1, 16)); - - tor_free(data1); - tor_free(data2); - tor_free(data3); - - /* Add spaces to fingerprint */ - { - data1 = tor_strdup("ABCD1234ABCD56780000ABCD1234ABCD56780000"); - test_eq(strlen(data1), 40); - data2 = tor_malloc(FINGERPRINT_LEN+1); - add_spaces_to_fp(data2, FINGERPRINT_LEN+1, data1); - test_streq(data2, "ABCD 1234 ABCD 5678 0000 ABCD 1234 ABCD 5678 0000"); - tor_free(data1); - tor_free(data2); - } - - /* Check fingerprint */ - { - test_assert(crypto_pk_check_fingerprint_syntax( - "ABCD 1234 ABCD 5678 0000 ABCD 1234 ABCD 5678 0000")); - test_assert(!crypto_pk_check_fingerprint_syntax( - "ABCD 1234 ABCD 5678 0000 ABCD 1234 ABCD 5678 000")); - test_assert(!crypto_pk_check_fingerprint_syntax( - "ABCD 1234 ABCD 5678 0000 ABCD 1234 ABCD 5678 00000")); - test_assert(!crypto_pk_check_fingerprint_syntax( - "ABCD 1234 ABCD 5678 0000 ABCD1234 ABCD 5678 0000")); - test_assert(!crypto_pk_check_fingerprint_syntax( - "ABCD 1234 ABCD 5678 0000 ABCD1234 ABCD 5678 00000")); - test_assert(!crypto_pk_check_fingerprint_syntax( - "ACD 1234 ABCD 5678 0000 ABCD 1234 ABCD 5678 00000")); - } - - done: - tor_free(data1); - tor_free(data2); - tor_free(data3); -} - -/** Run unit tests for our secret-to-key passphrase hashing functionality. */ -static void -test_crypto_s2k(void) -{ - char buf[29]; - char buf2[29]; - char *buf3 = NULL; - int i; - - memset(buf, 0, sizeof(buf)); - memset(buf2, 0, sizeof(buf2)); - buf3 = tor_malloc(65536); - memset(buf3, 0, 65536); - - secret_to_key(buf+9, 20, "", 0, buf); - crypto_digest(buf2+9, buf3, 1024); - test_memeq(buf, buf2, 29); - - memcpy(buf,"vrbacrda",8); - memcpy(buf2,"vrbacrda",8); - buf[8] = 96; - buf2[8] = 96; - secret_to_key(buf+9, 20, "12345678", 8, buf); - for (i = 0; i < 65536; i += 16) { - memcpy(buf3+i, "vrbacrda12345678", 16); - } - crypto_digest(buf2+9, buf3, 65536); - test_memeq(buf, buf2, 29); - - done: - tor_free(buf3); -} - -/** Helper: return a tristate based on comparing the strings in *<b>a</b> and - * *<b>b</b>. */ -static int -_compare_strs(const void **a, const void **b) -{ - const char *s1 = *a, *s2 = *b; - return strcmp(s1, s2); -} - -/** Helper: return a tristate based on comparing the strings in *<b>a</b> and - * *<b>b</b>, excluding a's first character, and ignoring case. */ -static int -_compare_without_first_ch(const void *a, const void **b) -{ - const char *s1 = a, *s2 = *b; - return strcasecmp(s1+1, s2); -} - -/** Test basic utility functionality. */ -static void -test_util(void) -{ - struct timeval start, end; - struct tm a_time; - char timestr[RFC1123_TIME_LEN+1]; - char buf[1024]; - time_t t_res; - int i; - uint32_t u32; - uint16_t u16; - char *cp, *k, *v; - const char *str; - - start.tv_sec = 5; - start.tv_usec = 5000; - - end.tv_sec = 5; - end.tv_usec = 5000; - - test_eq(0L, tv_udiff(&start, &end)); - - end.tv_usec = 7000; - - test_eq(2000L, tv_udiff(&start, &end)); - - end.tv_sec = 6; - - test_eq(1002000L, tv_udiff(&start, &end)); - - end.tv_usec = 0; - - test_eq(995000L, tv_udiff(&start, &end)); - - end.tv_sec = 4; - - test_eq(-1005000L, tv_udiff(&start, &end)); - - end.tv_usec = 999990; - start.tv_sec = 1; - start.tv_usec = 500; - - /* The test values here are confirmed to be correct on a platform - * with a working timegm. */ - a_time.tm_year = 2003-1900; - a_time.tm_mon = 7; - a_time.tm_mday = 30; - a_time.tm_hour = 6; - a_time.tm_min = 14; - a_time.tm_sec = 55; - test_eq((time_t) 1062224095UL, tor_timegm(&a_time)); - a_time.tm_year = 2004-1900; /* Try a leap year, after feb. */ - test_eq((time_t) 1093846495UL, tor_timegm(&a_time)); - a_time.tm_mon = 1; /* Try a leap year, in feb. */ - a_time.tm_mday = 10; - test_eq((time_t) 1076393695UL, tor_timegm(&a_time)); - - format_rfc1123_time(timestr, 0); - test_streq("Thu, 01 Jan 1970 00:00:00 GMT", timestr); - format_rfc1123_time(timestr, (time_t)1091580502UL); - test_streq("Wed, 04 Aug 2004 00:48:22 GMT", timestr); - - t_res = 0; - i = parse_rfc1123_time(timestr, &t_res); - test_eq(i,0); - test_eq(t_res, (time_t)1091580502UL); - test_eq(-1, parse_rfc1123_time("Wed, zz Aug 2004 99-99x99 GMT", &t_res)); - tor_gettimeofday(&start); - - /* Tests for corner cases of strl operations */ - test_eq(5, strlcpy(buf, "Hello", 0)); - strlcpy(buf, "Hello", sizeof(buf)); - test_eq(10, strlcat(buf, "Hello", 5)); - - /* Test tor_strstrip() */ - strlcpy(buf, "Testing 1 2 3", sizeof(buf)); - tor_strstrip(buf, ",!"); - test_streq(buf, "Testing 1 2 3"); - strlcpy(buf, "!Testing 1 2 3?", sizeof(buf)); - tor_strstrip(buf, "!? "); - test_streq(buf, "Testing123"); - - /* Test parse_addr_port */ - cp = NULL; u32 = 3; u16 = 3; - test_assert(!parse_addr_port(LOG_WARN, "1.2.3.4", &cp, &u32, &u16)); - test_streq(cp, "1.2.3.4"); - test_eq(u32, 0x01020304u); - test_eq(u16, 0); - tor_free(cp); - test_assert(!parse_addr_port(LOG_WARN, "4.3.2.1:99", &cp, &u32, &u16)); - test_streq(cp, "4.3.2.1"); - test_eq(u32, 0x04030201u); - test_eq(u16, 99); - tor_free(cp); - test_assert(!parse_addr_port(LOG_WARN, "nonexistent.address:4040", - &cp, NULL, &u16)); - test_streq(cp, "nonexistent.address"); - test_eq(u16, 4040); - tor_free(cp); - test_assert(!parse_addr_port(LOG_WARN, "localhost:9999", &cp, &u32, &u16)); - test_streq(cp, "localhost"); - test_eq(u32, 0x7f000001u); - test_eq(u16, 9999); - tor_free(cp); - u32 = 3; - test_assert(!parse_addr_port(LOG_WARN, "localhost", NULL, &u32, &u16)); - test_eq(cp, NULL); - test_eq(u32, 0x7f000001u); - test_eq(u16, 0); - tor_free(cp); - test_eq(0, addr_mask_get_bits(0x0u)); - test_eq(32, addr_mask_get_bits(0xFFFFFFFFu)); - test_eq(16, addr_mask_get_bits(0xFFFF0000u)); - test_eq(31, addr_mask_get_bits(0xFFFFFFFEu)); - test_eq(1, addr_mask_get_bits(0x80000000u)); - - /* Test tor_parse_long. */ - test_eq(10L, tor_parse_long("10",10,0,100,NULL,NULL)); - test_eq(0L, tor_parse_long("10",10,50,100,NULL,NULL)); - test_eq(-50L, tor_parse_long("-50",10,-100,100,NULL,NULL)); - - /* Test tor_parse_ulong */ - test_eq(10UL, tor_parse_ulong("10",10,0,100,NULL,NULL)); - test_eq(0UL, tor_parse_ulong("10",10,50,100,NULL,NULL)); - - /* Test tor_parse_uint64. */ - test_assert(U64_LITERAL(10) == tor_parse_uint64("10 x",10,0,100, &i, &cp)); - test_assert(i == 1); - test_streq(cp, " x"); - test_assert(U64_LITERAL(12345678901) == - tor_parse_uint64("12345678901",10,0,UINT64_MAX, &i, &cp)); - test_assert(i == 1); - test_streq(cp, ""); - test_assert(U64_LITERAL(0) == - tor_parse_uint64("12345678901",10,500,INT32_MAX, &i, &cp)); - test_assert(i == 0); - - /* Test failing snprintf cases */ - test_eq(-1, tor_snprintf(buf, 0, "Foo")); - test_eq(-1, tor_snprintf(buf, 2, "Foo")); - - /* Test printf with uint64 */ - tor_snprintf(buf, sizeof(buf), "x!"U64_FORMAT"!x", - U64_PRINTF_ARG(U64_LITERAL(12345678901))); - test_streq(buf, "x!12345678901!x"); - - /* Test parse_config_line_from_str */ - strlcpy(buf, "k v\n" " key value with spaces \n" "keykey val\n" - "k2\n" - "k3 \n" "\n" " \n" "#comment\n" - "k4#a\n" "k5#abc\n" "k6 val #with comment\n" - "kseven \"a quoted 'string\"\n" - "k8 \"a \\x71uoted\\n\\\"str\\\\ing\\t\\001\\01\\1\\\"\"\n" - , sizeof(buf)); - str = buf; - - str = parse_config_line_from_str(str, &k, &v); - test_streq(k, "k"); - test_streq(v, "v"); - tor_free(k); tor_free(v); - test_assert(!strcmpstart(str, "key value with")); - - str = parse_config_line_from_str(str, &k, &v); - test_streq(k, "key"); - test_streq(v, "value with spaces"); - tor_free(k); tor_free(v); - test_assert(!strcmpstart(str, "keykey")); - - str = parse_config_line_from_str(str, &k, &v); - test_streq(k, "keykey"); - test_streq(v, "val"); - tor_free(k); tor_free(v); - test_assert(!strcmpstart(str, "k2\n")); - - str = parse_config_line_from_str(str, &k, &v); - test_streq(k, "k2"); - test_streq(v, ""); - tor_free(k); tor_free(v); - test_assert(!strcmpstart(str, "k3 \n")); - - str = parse_config_line_from_str(str, &k, &v); - test_streq(k, "k3"); - test_streq(v, ""); - tor_free(k); tor_free(v); - test_assert(!strcmpstart(str, "#comment")); - - str = parse_config_line_from_str(str, &k, &v); - test_streq(k, "k4"); - test_streq(v, ""); - tor_free(k); tor_free(v); - test_assert(!strcmpstart(str, "k5#abc")); - - str = parse_config_line_from_str(str, &k, &v); - test_streq(k, "k5"); - test_streq(v, ""); - tor_free(k); tor_free(v); - test_assert(!strcmpstart(str, "k6")); - - str = parse_config_line_from_str(str, &k, &v); - test_streq(k, "k6"); - test_streq(v, "val"); - tor_free(k); tor_free(v); - test_assert(!strcmpstart(str, "kseven")); - - str = parse_config_line_from_str(str, &k, &v); - test_streq(k, "kseven"); - test_streq(v, "a quoted \'string"); - tor_free(k); tor_free(v); - test_assert(!strcmpstart(str, "k8 ")); - - str = parse_config_line_from_str(str, &k, &v); - test_streq(k, "k8"); - test_streq(v, "a quoted\n\"str\\ing\t\x01\x01\x01\""); - tor_free(k); tor_free(v); - test_streq(str, ""); - - /* Test for strcmpstart and strcmpend. */ - test_assert(strcmpstart("abcdef", "abcdef")==0); - test_assert(strcmpstart("abcdef", "abc")==0); - test_assert(strcmpstart("abcdef", "abd")<0); - test_assert(strcmpstart("abcdef", "abb")>0); - test_assert(strcmpstart("ab", "abb")<0); - - test_assert(strcmpend("abcdef", "abcdef")==0); - test_assert(strcmpend("abcdef", "def")==0); - test_assert(strcmpend("abcdef", "deg")<0); - test_assert(strcmpend("abcdef", "dee")>0); - test_assert(strcmpend("ab", "abb")<0); - - test_assert(strcasecmpend("AbcDEF", "abcdef")==0); - test_assert(strcasecmpend("abcdef", "dEF")==0); - test_assert(strcasecmpend("abcDEf", "deg")<0); - test_assert(strcasecmpend("abcdef", "DEE")>0); - test_assert(strcasecmpend("ab", "abB")<0); - - /* Test mem_is_zero */ - memset(buf,0,128); - buf[128] = 'x'; - test_assert(tor_digest_is_zero(buf)); - test_assert(tor_mem_is_zero(buf, 10)); - test_assert(tor_mem_is_zero(buf, 20)); - test_assert(tor_mem_is_zero(buf, 128)); - test_assert(!tor_mem_is_zero(buf, 129)); - buf[60] = (char)255; - test_assert(!tor_mem_is_zero(buf, 128)); - buf[0] = (char)1; - test_assert(!tor_mem_is_zero(buf, 10)); - - /* Test inet_ntop */ - { - char tmpbuf[TOR_ADDR_BUF_LEN]; - const char *ip = "176.192.208.224"; - struct in_addr in; - tor_inet_pton(AF_INET, ip, &in); - tor_inet_ntop(AF_INET, &in, tmpbuf, sizeof(tmpbuf)); - test_streq(tmpbuf, ip); - } - - /* Test 'escaped' */ - test_streq("\"\"", escaped("")); - test_streq("\"abcd\"", escaped("abcd")); - test_streq("\"\\\\\\n\\r\\t\\\"\\'\"", escaped("\\\n\r\t\"\'")); - test_streq("\"z\\001abc\\277d\"", escaped("z\001abc\277d")); - test_assert(NULL == escaped(NULL)); - - /* Test strndup and memdup */ - { - const char *s = "abcdefghijklmnopqrstuvwxyz"; - cp = tor_strndup(s, 30); - test_streq(cp, s); /* same string, */ - test_neq(cp, s); /* but different pointers. */ - tor_free(cp); - - cp = tor_strndup(s, 5); - test_streq(cp, "abcde"); - tor_free(cp); - - s = "a\0b\0c\0d\0e\0"; - cp = tor_memdup(s,10); - test_memeq(cp, s, 10); /* same ram, */ - test_neq(cp, s); /* but different pointers. */ - tor_free(cp); - } - - /* Test str-foo functions */ - cp = tor_strdup("abcdef"); - test_assert(tor_strisnonupper(cp)); - cp[3] = 'D'; - test_assert(!tor_strisnonupper(cp)); - tor_strupper(cp); - test_streq(cp, "ABCDEF"); - test_assert(tor_strisprint(cp)); - cp[3] = 3; - test_assert(!tor_strisprint(cp)); - tor_free(cp); - - /* Test eat_whitespace. */ - { - const char *s = " \n a"; - test_eq_ptr(eat_whitespace(s), s+4); - s = "abcd"; - test_eq_ptr(eat_whitespace(s), s); - s = "#xyz\nab"; - test_eq_ptr(eat_whitespace(s), s+5); - } - - /* Test memmem and memstr */ - { - const char *haystack = "abcde"; - tor_assert(!tor_memmem(haystack, 5, "ef", 2)); - test_eq_ptr(tor_memmem(haystack, 5, "cd", 2), haystack + 2); - test_eq_ptr(tor_memmem(haystack, 5, "cde", 3), haystack + 2); - haystack = "ababcad"; - test_eq_ptr(tor_memmem(haystack, 7, "abc", 3), haystack + 2); - test_eq_ptr(tor_memstr(haystack, 7, "abc"), haystack + 2); - test_assert(!tor_memstr(haystack, 7, "fe")); - test_assert(!tor_memstr(haystack, 7, "longerthantheoriginal")); - } - - /* Test wrap_string */ - { - smartlist_t *sl = smartlist_create(); - wrap_string(sl, "This is a test of string wrapping functionality: woot.", - 10, "", ""); - cp = smartlist_join_strings(sl, "", 0, NULL); - test_streq(cp, - "This is a\ntest of\nstring\nwrapping\nfunctional\nity: woot.\n"); - tor_free(cp); - SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); - smartlist_clear(sl); - - wrap_string(sl, "This is a test of string wrapping functionality: woot.", - 16, "### ", "# "); - cp = smartlist_join_strings(sl, "", 0, NULL); - test_streq(cp, - "### This is a\n# test of string\n# wrapping\n# functionality:\n" - "# woot.\n"); - - tor_free(cp); - SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); - smartlist_free(sl); - } - - tor_gettimeofday(&start); - /* now make sure time works. */ - tor_gettimeofday(&end); - /* We might've timewarped a little. */ - test_assert(tv_udiff(&start, &end) >= -5000); - - /* Test tor_log2(). */ - test_eq(tor_log2(64), 6); - test_eq(tor_log2(65), 6); - test_eq(tor_log2(63), 5); - test_eq(tor_log2(1), 0); - test_eq(tor_log2(2), 1); - test_eq(tor_log2(3), 1); - test_eq(tor_log2(4), 2); - test_eq(tor_log2(5), 2); - test_eq(tor_log2(U64_LITERAL(40000000000000000)), 55); - test_eq(tor_log2(UINT64_MAX), 63); - - /* Test round_to_power_of_2 */ - test_eq(round_to_power_of_2(120), 128); - test_eq(round_to_power_of_2(128), 128); - test_eq(round_to_power_of_2(130), 128); - test_eq(round_to_power_of_2(U64_LITERAL(40000000000000000)), - U64_LITERAL(1)<<55); - test_eq(round_to_power_of_2(0), 2); - - done: - ; -} - -/** Helper: assert that IPv6 addresses <b>a</b> and <b>b</b> are the same. On - * failure, reports an error, describing the addresses as <b>e1</b> and - * <b>e2</b>, and reporting the line number as <b>line</b>. */ -static void -_test_eq_ip6(struct in6_addr *a, struct in6_addr *b, const char *e1, - const char *e2, int line) -{ - int i; - int ok = 1; - for (i = 0; i < 16; ++i) { - if (a->s6_addr[i] != b->s6_addr[i]) { - ok = 0; - break; - } - } - if (ok) { - printf("."); fflush(stdout); - } else { - char buf1[128], *cp1; - char buf2[128], *cp2; - have_failed = 1; - cp1 = buf1; cp2 = buf2; - for (i=0; i<16; ++i) { - tor_snprintf(cp1, sizeof(buf1)-(cp1-buf1), "%02x", a->s6_addr[i]); - tor_snprintf(cp2, sizeof(buf2)-(cp2-buf2), "%02x", b->s6_addr[i]); - cp1 += 2; cp2 += 2; - if ((i%2)==1 && i != 15) { - *cp1++ = ':'; - *cp2++ = ':'; - } - } - *cp1 = *cp2 = '\0'; - printf("Line %d: assertion failed: (%s == %s)\n" - " %s != %s\n", line, e1, e2, buf1, buf2); - fflush(stdout); - } -} - -/** Helper: Assert that two strings both decode as IPv6 addresses with - * tor_inet_pton(), and both decode to the same address. */ -#define test_pton6_same(a,b) STMT_BEGIN \ - test_eq(tor_inet_pton(AF_INET6, a, &a1), 1); \ - test_eq(tor_inet_pton(AF_INET6, b, &a2), 1); \ - _test_eq_ip6(&a1,&a2,#a,#b,__LINE__); \ - STMT_END - -/** Helper: Assert that <b>a</b> is recognized as a bad IPv6 address by - * tor_inet_pton(). */ -#define test_pton6_bad(a) \ - test_eq(0, tor_inet_pton(AF_INET6, a, &a1)) - -/** Helper: assert that <b>a</b>, when parsed by tor_inet_pton() and displayed - * with tor_inet_ntop(), yields <b>b</b>. Also assert that <b>b</b> parses to - * the same value as <b>a</b>. */ -#define test_ntop6_reduces(a,b) STMT_BEGIN \ - test_eq(tor_inet_pton(AF_INET6, a, &a1), 1); \ - test_streq(tor_inet_ntop(AF_INET6, &a1, buf, sizeof(buf)), b); \ - test_eq(tor_inet_pton(AF_INET6, b, &a2), 1); \ - _test_eq_ip6(&a1, &a2, a, b, __LINE__); \ - STMT_END - -/** Helper: assert that <b>a</b> parses by tor_inet_pton() into a address that - * passes tor_addr_is_internal() with <b>for_listening</b>. */ -#define test_internal_ip(a,for_listening) STMT_BEGIN \ - test_eq(tor_inet_pton(AF_INET6, a, &t1.addr.in6_addr), 1); \ - t1.family = AF_INET6; \ - if (!tor_addr_is_internal(&t1, for_listening)) \ - test_fail_msg( a "was not internal."); \ - STMT_END - -/** Helper: assert that <b>a</b> parses by tor_inet_pton() into a address that - * does not pass tor_addr_is_internal() with <b>for_listening</b>. */ -#define test_external_ip(a,for_listening) STMT_BEGIN \ - test_eq(tor_inet_pton(AF_INET6, a, &t1.addr.in6_addr), 1); \ - t1.family = AF_INET6; \ - if (tor_addr_is_internal(&t1, for_listening)) \ - test_fail_msg(a "was not external."); \ - STMT_END - -/** Helper: Assert that <b>a</b> and <b>b</b>, when parsed by - * tor_inet_pton(), give addresses that compare in the order defined by - * <b>op</b> with tor_addr_compare(). */ -#define test_addr_compare(a, op, b) STMT_BEGIN \ - test_eq(tor_inet_pton(AF_INET6, a, &t1.addr.in6_addr), 1); \ - test_eq(tor_inet_pton(AF_INET6, b, &t2.addr.in6_addr), 1); \ - t1.family = t2.family = AF_INET6; \ - r = tor_addr_compare(&t1,&t2,CMP_SEMANTIC); \ - if (!(r op 0)) \ - test_fail_msg("failed: tor_addr_compare("a","b") "#op" 0"); \ - STMT_END - -/** Helper: Assert that <b>a</b> and <b>b</b>, when parsed by - * tor_inet_pton(), give addresses that compare in the order defined by - * <b>op</b> with tor_addr_compare_masked() with <b>m</b> masked. */ -#define test_addr_compare_masked(a, op, b, m) STMT_BEGIN \ - test_eq(tor_inet_pton(AF_INET6, a, &t1.addr.in6_addr), 1); \ - test_eq(tor_inet_pton(AF_INET6, b, &t2.addr.in6_addr), 1); \ - t1.family = t2.family = AF_INET6; \ - r = tor_addr_compare_masked(&t1,&t2,m,CMP_SEMANTIC); \ - if (!(r op 0)) \ - test_fail_msg("failed: tor_addr_compare_masked("a","b","#m") "#op" 0"); \ - STMT_END - -/** Helper: assert that <b>xx</b> is parseable as a masked IPv6 address with - * ports by tor_parse_mask_addr_ports(), with family <b>f</b>, IP address - * as 4 32-bit words <b>ip1...ip4</b>, mask bits as <b>mm</b>, and port range - * as <b>pt1..pt2</b>. */ -#define test_addr_mask_ports_parse(xx, f, ip1, ip2, ip3, ip4, mm, pt1, pt2) \ - STMT_BEGIN \ - test_eq(tor_addr_parse_mask_ports(xx, &t1, &mask, &port1, &port2), f); \ - p1=tor_inet_ntop(AF_INET6, &t1.addr.in6_addr, bug, sizeof(bug)); \ - test_eq(htonl(ip1), tor_addr_to_in6_addr32(&t1)[0]); \ - test_eq(htonl(ip2), tor_addr_to_in6_addr32(&t1)[1]); \ - test_eq(htonl(ip3), tor_addr_to_in6_addr32(&t1)[2]); \ - test_eq(htonl(ip4), tor_addr_to_in6_addr32(&t1)[3]); \ - test_eq(mask, mm); \ - test_eq(port1, pt1); \ - test_eq(port2, pt2); \ - STMT_END - -/** Run unit tests for IPv6 encoding/decoding/manipulation functions. */ -static void -test_util_ip6_helpers(void) -{ - char buf[TOR_ADDR_BUF_LEN], bug[TOR_ADDR_BUF_LEN]; - struct in6_addr a1, a2; - tor_addr_t t1, t2; - int r, i; - uint16_t port1, port2; - maskbits_t mask; - const char *p1; - struct sockaddr_storage sa_storage; - struct sockaddr_in *sin; - struct sockaddr_in6 *sin6; - - // struct in_addr b1, b2; - /* Test tor_inet_ntop and tor_inet_pton: IPv6 */ - - /* ==== Converting to and from sockaddr_t. */ - sin = (struct sockaddr_in *)&sa_storage; - sin->sin_family = AF_INET; - sin->sin_port = 9090; - sin->sin_addr.s_addr = htonl(0x7f7f0102); /*127.127.1.2*/ - tor_addr_from_sockaddr(&t1, (struct sockaddr *)sin, NULL); - test_eq(tor_addr_family(&t1), AF_INET); - test_eq(tor_addr_to_ipv4h(&t1), 0x7f7f0102); - - memset(&sa_storage, 0, sizeof(sa_storage)); - test_eq(sizeof(struct sockaddr_in), - tor_addr_to_sockaddr(&t1, 1234, (struct sockaddr *)&sa_storage, - sizeof(sa_storage))); - test_eq(1234, ntohs(sin->sin_port)); - test_eq(0x7f7f0102, ntohl(sin->sin_addr.s_addr)); - - memset(&sa_storage, 0, sizeof(sa_storage)); - sin6 = (struct sockaddr_in6 *)&sa_storage; - sin6->sin6_family = AF_INET6; - sin6->sin6_port = htons(7070); - sin6->sin6_addr.s6_addr[0] = 128; - tor_addr_from_sockaddr(&t1, (struct sockaddr *)sin6, NULL); - test_eq(tor_addr_family(&t1), AF_INET6); - p1 = tor_addr_to_str(buf, &t1, sizeof(buf), 0); - test_streq(p1, "8000::"); - - memset(&sa_storage, 0, sizeof(sa_storage)); - test_eq(sizeof(struct sockaddr_in6), - tor_addr_to_sockaddr(&t1, 9999, (struct sockaddr *)&sa_storage, - sizeof(sa_storage))); - test_eq(AF_INET6, sin6->sin6_family); - test_eq(9999, ntohs(sin6->sin6_port)); - test_eq(0x80000000, ntohl(S6_ADDR32(sin6->sin6_addr)[0])); - - /* ==== tor_addr_lookup: static cases. (Can't test dns without knowing we - * have a good resolver. */ - test_eq(0, tor_addr_lookup("127.128.129.130", AF_UNSPEC, &t1)); - test_eq(AF_INET, tor_addr_family(&t1)); - test_eq(tor_addr_to_ipv4h(&t1), 0x7f808182); - - test_eq(0, tor_addr_lookup("9000::5", AF_UNSPEC, &t1)); - test_eq(AF_INET6, tor_addr_family(&t1)); - test_eq(0x90, tor_addr_to_in6_addr8(&t1)[0]); - test_assert(tor_mem_is_zero((char*)tor_addr_to_in6_addr8(&t1)+1, 14)); - test_eq(0x05, tor_addr_to_in6_addr8(&t1)[15]); - - /* === Test pton: valid af_inet6 */ - /* Simple, valid parsing. */ - r = tor_inet_pton(AF_INET6, - "0102:0304:0506:0708:090A:0B0C:0D0E:0F10", &a1); - test_assert(r==1); - for (i=0;i<16;++i) { test_eq(i+1, (int)a1.s6_addr[i]); } - /* ipv4 ending. */ - test_pton6_same("0102:0304:0506:0708:090A:0B0C:0D0E:0F10", - "0102:0304:0506:0708:090A:0B0C:13.14.15.16"); - /* shortened words. */ - test_pton6_same("0001:0099:BEEF:0000:0123:FFFF:0001:0001", - "1:99:BEEF:0:0123:FFFF:1:1"); - /* zeros at the beginning */ - test_pton6_same("0000:0000:0000:0000:0009:C0A8:0001:0001", - "::9:c0a8:1:1"); - test_pton6_same("0000:0000:0000:0000:0009:C0A8:0001:0001", - "::9:c0a8:0.1.0.1"); - /* zeros in the middle. */ - test_pton6_same("fe80:0000:0000:0000:0202:1111:0001:0001", - "fe80::202:1111:1:1"); - /* zeros at the end. */ - test_pton6_same("1000:0001:0000:0007:0000:0000:0000:0000", - "1000:1:0:7::"); - - /* === Test ntop: af_inet6 */ - test_ntop6_reduces("0:0:0:0:0:0:0:0", "::"); - - test_ntop6_reduces("0001:0099:BEEF:0006:0123:FFFF:0001:0001", - "1:99:beef:6:123:ffff:1:1"); - - //test_ntop6_reduces("0:0:0:0:0:0:c0a8:0101", "::192.168.1.1"); - test_ntop6_reduces("0:0:0:0:0:ffff:c0a8:0101", "::ffff:192.168.1.1"); - test_ntop6_reduces("002:0:0000:0:3::4", "2::3:0:0:4"); - test_ntop6_reduces("0:0::1:0:3", "::1:0:3"); - test_ntop6_reduces("008:0::0", "8::"); - test_ntop6_reduces("0:0:0:0:0:ffff::1", "::ffff:0.0.0.1"); - test_ntop6_reduces("abcd:0:0:0:0:0:7f00::", "abcd::7f00:0"); - test_ntop6_reduces("0000:0000:0000:0000:0009:C0A8:0001:0001", - "::9:c0a8:1:1"); - test_ntop6_reduces("fe80:0000:0000:0000:0202:1111:0001:0001", - "fe80::202:1111:1:1"); - test_ntop6_reduces("1000:0001:0000:0007:0000:0000:0000:0000", - "1000:1:0:7::"); - - /* === Test pton: invalid in6. */ - test_pton6_bad("foobar."); - test_pton6_bad("55555::"); - test_pton6_bad("9:-60::"); - test_pton6_bad("1:2:33333:4:0002:3::"); - //test_pton6_bad("1:2:3333:4:00002:3::");// BAD, but glibc doesn't say so. - test_pton6_bad("1:2:3333:4:fish:3::"); - test_pton6_bad("1:2:3:4:5:6:7:8:9"); - test_pton6_bad("1:2:3:4:5:6:7"); - test_pton6_bad("1:2:3:4:5:6:1.2.3.4.5"); - test_pton6_bad("1:2:3:4:5:6:1.2.3"); - test_pton6_bad("::1.2.3"); - test_pton6_bad("::1.2.3.4.5"); - test_pton6_bad("99"); - test_pton6_bad(""); - test_pton6_bad("1::2::3:4"); - test_pton6_bad("a:::b:c"); - test_pton6_bad(":::a:b:c"); - test_pton6_bad("a:b:c:::"); - - /* test internal checking */ - test_external_ip("fbff:ffff::2:7", 0); - test_internal_ip("fc01::2:7", 0); - test_internal_ip("fdff:ffff::f:f", 0); - test_external_ip("fe00::3:f", 0); - - test_external_ip("fe7f:ffff::2:7", 0); - test_internal_ip("fe80::2:7", 0); - test_internal_ip("febf:ffff::f:f", 0); - - test_internal_ip("fec0::2:7:7", 0); - test_internal_ip("feff:ffff::e:7:7", 0); - test_external_ip("ff00::e:7:7", 0); - - test_internal_ip("::", 0); - test_internal_ip("::1", 0); - test_internal_ip("::1", 1); - test_internal_ip("::", 0); - test_external_ip("::", 1); - test_external_ip("::2", 0); - test_external_ip("2001::", 0); - test_external_ip("ffff::", 0); - - test_external_ip("::ffff:0.0.0.0", 1); - test_internal_ip("::ffff:0.0.0.0", 0); - test_internal_ip("::ffff:0.255.255.255", 0); - test_external_ip("::ffff:1.0.0.0", 0); - - test_external_ip("::ffff:9.255.255.255", 0); - test_internal_ip("::ffff:10.0.0.0", 0); - test_internal_ip("::ffff:10.255.255.255", 0); - test_external_ip("::ffff:11.0.0.0", 0); - - test_external_ip("::ffff:126.255.255.255", 0); - test_internal_ip("::ffff:127.0.0.0", 0); - test_internal_ip("::ffff:127.255.255.255", 0); - test_external_ip("::ffff:128.0.0.0", 0); - - test_external_ip("::ffff:172.15.255.255", 0); - test_internal_ip("::ffff:172.16.0.0", 0); - test_internal_ip("::ffff:172.31.255.255", 0); - test_external_ip("::ffff:172.32.0.0", 0); - - test_external_ip("::ffff:192.167.255.255", 0); - test_internal_ip("::ffff:192.168.0.0", 0); - test_internal_ip("::ffff:192.168.255.255", 0); - test_external_ip("::ffff:192.169.0.0", 0); - - test_external_ip("::ffff:169.253.255.255", 0); - test_internal_ip("::ffff:169.254.0.0", 0); - test_internal_ip("::ffff:169.254.255.255", 0); - test_external_ip("::ffff:169.255.0.0", 0); - test_assert(is_internal_IP(0x7f000001, 0)); - - /* tor_addr_compare(tor_addr_t x2) */ - test_addr_compare("ffff::", ==, "ffff::0"); - test_addr_compare("0::3:2:1", <, "0::ffff:0.3.2.1"); - test_addr_compare("0::2:2:1", <, "0::ffff:0.3.2.1"); - test_addr_compare("0::ffff:0.3.2.1", >, "0::0:0:0"); - test_addr_compare("0::ffff:5.2.2.1", <, "::ffff:6.0.0.0"); /* XXXX wrong. */ - tor_addr_parse_mask_ports("[::ffff:2.3.4.5]", &t1, NULL, NULL, NULL); - tor_addr_parse_mask_ports("2.3.4.5", &t2, NULL, NULL, NULL); - test_assert(tor_addr_compare(&t1, &t2, CMP_SEMANTIC) == 0); - tor_addr_parse_mask_ports("[::ffff:2.3.4.4]", &t1, NULL, NULL, NULL); - tor_addr_parse_mask_ports("2.3.4.5", &t2, NULL, NULL, NULL); - test_assert(tor_addr_compare(&t1, &t2, CMP_SEMANTIC) < 0); - - /* test compare_masked */ - test_addr_compare_masked("ffff::", ==, "ffff::0", 128); - test_addr_compare_masked("ffff::", ==, "ffff::0", 64); - test_addr_compare_masked("0::2:2:1", <, "0::8000:2:1", 81); - test_addr_compare_masked("0::2:2:1", ==, "0::8000:2:1", 80); - - /* Test decorated addr_to_string. */ - test_eq(AF_INET6, tor_addr_from_str(&t1, "[123:45:6789::5005:11]")); - p1 = tor_addr_to_str(buf, &t1, sizeof(buf), 1); - test_streq(p1, "[123:45:6789::5005:11]"); - test_eq(AF_INET, tor_addr_from_str(&t1, "18.0.0.1")); - p1 = tor_addr_to_str(buf, &t1, sizeof(buf), 1); - test_streq(p1, "18.0.0.1"); - - /* Test tor_addr_parse_reverse_lookup_name */ - i = tor_addr_parse_reverse_lookup_name(&t1, "Foobar.baz", AF_UNSPEC, 0); - test_eq(0, i); - i = tor_addr_parse_reverse_lookup_name(&t1, "Foobar.baz", AF_UNSPEC, 1); - test_eq(0, i); - i = tor_addr_parse_reverse_lookup_name(&t1, "1.0.168.192.in-addr.arpa", - AF_UNSPEC, 1); - test_eq(1, i); - test_eq(tor_addr_family(&t1), AF_INET); - p1 = tor_addr_to_str(buf, &t1, sizeof(buf), 1); - test_streq(p1, "192.168.0.1"); - i = tor_addr_parse_reverse_lookup_name(&t1, "192.168.0.99", AF_UNSPEC, 0); - test_eq(0, i); - i = tor_addr_parse_reverse_lookup_name(&t1, "192.168.0.99", AF_UNSPEC, 1); - test_eq(1, i); - p1 = tor_addr_to_str(buf, &t1, sizeof(buf), 1); - test_streq(p1, "192.168.0.99"); - memset(&t1, 0, sizeof(t1)); - i = tor_addr_parse_reverse_lookup_name(&t1, - "0.1.2.3.4.5.6.7.8.9.a.b.c.d.e.f." - "f.e.e.b.1.e.b.e.e.f.f.e.e.e.d.9." - "ip6.ARPA", - AF_UNSPEC, 0); - test_eq(1, i); - p1 = tor_addr_to_str(buf, &t1, sizeof(buf), 1); - test_streq(p1, "[9dee:effe:ebe1:beef:fedc:ba98:7654:3210]"); - /* Failing cases. */ - i = tor_addr_parse_reverse_lookup_name(&t1, - "6.7.8.9.a.b.c.d.e.f." - "f.e.e.b.1.e.b.e.e.f.f.e.e.e.d.9." - "ip6.ARPA", - AF_UNSPEC, 0); - test_eq(i, -1); - i = tor_addr_parse_reverse_lookup_name(&t1, - "6.7.8.9.a.b.c.d.e.f.a.b.c.d.e.f.0." - "f.e.e.b.1.e.b.e.e.f.f.e.e.e.d.9." - "ip6.ARPA", - AF_UNSPEC, 0); - test_eq(i, -1); - i = tor_addr_parse_reverse_lookup_name(&t1, - "6.7.8.9.a.b.c.d.e.f.X.0.0.0.0.9." - "f.e.e.b.1.e.b.e.e.f.f.e.e.e.d.9." - "ip6.ARPA", - AF_UNSPEC, 0); - test_eq(i, -1); - i = tor_addr_parse_reverse_lookup_name(&t1, "32.1.1.in-addr.arpa", - AF_UNSPEC, 0); - test_eq(i, -1); - i = tor_addr_parse_reverse_lookup_name(&t1, ".in-addr.arpa", - AF_UNSPEC, 0); - test_eq(i, -1); - i = tor_addr_parse_reverse_lookup_name(&t1, "1.2.3.4.5.in-addr.arpa", - AF_UNSPEC, 0); - test_eq(i, -1); - i = tor_addr_parse_reverse_lookup_name(&t1, "1.2.3.4.5.in-addr.arpa", - AF_INET6, 0); - test_eq(i, -1); - i = tor_addr_parse_reverse_lookup_name(&t1, - "6.7.8.9.a.b.c.d.e.f.a.b.c.d.e.0." - "f.e.e.b.1.e.b.e.e.f.f.e.e.e.d.9." - "ip6.ARPA", - AF_INET, 0); - test_eq(i, -1); - - /* test tor_addr_parse_mask_ports */ - test_addr_mask_ports_parse("[::f]/17:47-95", AF_INET6, - 0, 0, 0, 0x0000000f, 17, 47, 95); - //test_addr_parse("[::fefe:4.1.1.7/120]:999-1000"); - //test_addr_parse_check("::fefe:401:107", 120, 999, 1000); - test_addr_mask_ports_parse("[::ffff:4.1.1.7]/120:443", AF_INET6, - 0, 0, 0x0000ffff, 0x04010107, 120, 443, 443); - test_addr_mask_ports_parse("[abcd:2::44a:0]:2-65000", AF_INET6, - 0xabcd0002, 0, 0, 0x044a0000, 128, 2, 65000); - - r=tor_addr_parse_mask_ports("[fefef::]/112", &t1, NULL, NULL, NULL); - test_assert(r == -1); - r=tor_addr_parse_mask_ports("efef::/112", &t1, NULL, NULL, NULL); - test_assert(r == -1); - r=tor_addr_parse_mask_ports("[f:f:f:f:f:f:f:f::]", &t1, NULL, NULL, NULL); - test_assert(r == -1); - r=tor_addr_parse_mask_ports("[::f:f:f:f:f:f:f:f]", &t1, NULL, NULL, NULL); - test_assert(r == -1); - r=tor_addr_parse_mask_ports("[f:f:f:f:f:f:f:f:f]", &t1, NULL, NULL, NULL); - test_assert(r == -1); - /* Test for V4-mapped address with mask < 96. (arguably not valid) */ - r=tor_addr_parse_mask_ports("[::ffff:1.1.2.2/33]", &t1, &mask, NULL, NULL); - test_assert(r == -1); - r=tor_addr_parse_mask_ports("1.1.2.2/33", &t1, &mask, NULL, NULL); - test_assert(r == -1); - r=tor_addr_parse_mask_ports("1.1.2.2/31", &t1, &mask, NULL, NULL); - test_assert(r == AF_INET); - r=tor_addr_parse_mask_ports("[efef::]/112", &t1, &mask, &port1, &port2); - test_assert(r == AF_INET6); - test_assert(port1 == 1); - test_assert(port2 == 65535); - - /* make sure inet address lengths >= max */ - test_assert(INET_NTOA_BUF_LEN >= sizeof("255.255.255.255")); - test_assert(TOR_ADDR_BUF_LEN >= - sizeof("ffff:ffff:ffff:ffff:ffff:ffff:255.255.255.255")); - - test_assert(sizeof(tor_addr_t) >= sizeof(struct in6_addr)); - - /* get interface addresses */ - r = get_interface_address6(LOG_DEBUG, AF_INET, &t1); - i = get_interface_address6(LOG_DEBUG, AF_INET6, &t2); -#if 0 - tor_inet_ntop(AF_INET, &t1.sa.sin_addr, buf, sizeof(buf)); - printf("\nv4 address: %s (family=%i)", buf, IN_FAMILY(&t1)); - tor_inet_ntop(AF_INET6, &t2.sa6.sin6_addr, buf, sizeof(buf)); - printf("\nv6 address: %s (family=%i)", buf, IN_FAMILY(&t2)); -#endif - - done: - ; -} - -/** Run unit tests for basic dynamic-sized array functionality. */ -static void -test_util_smartlist_basic(void) -{ - smartlist_t *sl; - - /* XXXX test sort_digests, uniq_strings, uniq_digests */ - - /* Test smartlist add, del_keeporder, insert, get. */ - sl = smartlist_create(); - smartlist_add(sl, (void*)1); - smartlist_add(sl, (void*)2); - smartlist_add(sl, (void*)3); - smartlist_add(sl, (void*)4); - smartlist_del_keeporder(sl, 1); - smartlist_insert(sl, 1, (void*)22); - smartlist_insert(sl, 0, (void*)0); - smartlist_insert(sl, 5, (void*)555); - test_eq_ptr((void*)0, smartlist_get(sl,0)); - test_eq_ptr((void*)1, smartlist_get(sl,1)); - test_eq_ptr((void*)22, smartlist_get(sl,2)); - test_eq_ptr((void*)3, smartlist_get(sl,3)); - test_eq_ptr((void*)4, smartlist_get(sl,4)); - test_eq_ptr((void*)555, smartlist_get(sl,5)); - /* Try deleting in the middle. */ - smartlist_del(sl, 1); - test_eq_ptr((void*)555, smartlist_get(sl, 1)); - /* Try deleting at the end. */ - smartlist_del(sl, 4); - test_eq(4, smartlist_len(sl)); - - /* test isin. */ - test_assert(smartlist_isin(sl, (void*)3)); - test_assert(!smartlist_isin(sl, (void*)99)); - - done: - smartlist_free(sl); -} - -/** Run unit tests for smartlist-of-strings functionality. */ -static void -test_util_smartlist_strings(void) -{ - smartlist_t *sl = smartlist_create(); - char *cp=NULL, *cp_alloc=NULL; - size_t sz; - - /* Test split and join */ - test_eq(0, smartlist_len(sl)); - smartlist_split_string(sl, "abc", ":", 0, 0); - test_eq(1, smartlist_len(sl)); - test_streq("abc", smartlist_get(sl, 0)); - smartlist_split_string(sl, "a::bc::", "::", 0, 0); - test_eq(4, smartlist_len(sl)); - test_streq("a", smartlist_get(sl, 1)); - test_streq("bc", smartlist_get(sl, 2)); - test_streq("", smartlist_get(sl, 3)); - cp_alloc = smartlist_join_strings(sl, "", 0, NULL); - test_streq(cp_alloc, "abcabc"); - tor_free(cp_alloc); - cp_alloc = smartlist_join_strings(sl, "!", 0, NULL); - test_streq(cp_alloc, "abc!a!bc!"); - tor_free(cp_alloc); - cp_alloc = smartlist_join_strings(sl, "XY", 0, NULL); - test_streq(cp_alloc, "abcXYaXYbcXY"); - tor_free(cp_alloc); - cp_alloc = smartlist_join_strings(sl, "XY", 1, NULL); - test_streq(cp_alloc, "abcXYaXYbcXYXY"); - tor_free(cp_alloc); - cp_alloc = smartlist_join_strings(sl, "", 1, NULL); - test_streq(cp_alloc, "abcabc"); - tor_free(cp_alloc); - - smartlist_split_string(sl, "/def/ /ghijk", "/", 0, 0); - test_eq(8, smartlist_len(sl)); - test_streq("", smartlist_get(sl, 4)); - test_streq("def", smartlist_get(sl, 5)); - test_streq(" ", smartlist_get(sl, 6)); - test_streq("ghijk", smartlist_get(sl, 7)); - SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); - smartlist_clear(sl); - - smartlist_split_string(sl, "a,bbd,cdef", ",", SPLIT_SKIP_SPACE, 0); - test_eq(3, smartlist_len(sl)); - test_streq("a", smartlist_get(sl,0)); - test_streq("bbd", smartlist_get(sl,1)); - test_streq("cdef", smartlist_get(sl,2)); - smartlist_split_string(sl, " z <> zhasd <> <> bnud<> ", "<>", - SPLIT_SKIP_SPACE, 0); - test_eq(8, smartlist_len(sl)); - test_streq("z", smartlist_get(sl,3)); - test_streq("zhasd", smartlist_get(sl,4)); - test_streq("", smartlist_get(sl,5)); - test_streq("bnud", smartlist_get(sl,6)); - test_streq("", smartlist_get(sl,7)); - - SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); - smartlist_clear(sl); - - smartlist_split_string(sl, " ab\tc \td ef ", NULL, - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); - test_eq(4, smartlist_len(sl)); - test_streq("ab", smartlist_get(sl,0)); - test_streq("c", smartlist_get(sl,1)); - test_streq("d", smartlist_get(sl,2)); - test_streq("ef", smartlist_get(sl,3)); - smartlist_split_string(sl, "ghi\tj", NULL, - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); - test_eq(6, smartlist_len(sl)); - test_streq("ghi", smartlist_get(sl,4)); - test_streq("j", smartlist_get(sl,5)); - - SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); - smartlist_clear(sl); - - cp_alloc = smartlist_join_strings(sl, "XY", 0, NULL); - test_streq(cp_alloc, ""); - tor_free(cp_alloc); - cp_alloc = smartlist_join_strings(sl, "XY", 1, NULL); - test_streq(cp_alloc, "XY"); - tor_free(cp_alloc); - - smartlist_split_string(sl, " z <> zhasd <> <> bnud<> ", "<>", - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); - test_eq(3, smartlist_len(sl)); - test_streq("z", smartlist_get(sl, 0)); - test_streq("zhasd", smartlist_get(sl, 1)); - test_streq("bnud", smartlist_get(sl, 2)); - smartlist_split_string(sl, " z <> zhasd <> <> bnud<> ", "<>", - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 2); - test_eq(5, smartlist_len(sl)); - test_streq("z", smartlist_get(sl, 3)); - test_streq("zhasd <> <> bnud<>", smartlist_get(sl, 4)); - SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); - smartlist_clear(sl); - - smartlist_split_string(sl, "abcd\n", "\n", - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); - test_eq(1, smartlist_len(sl)); - test_streq("abcd", smartlist_get(sl, 0)); - smartlist_split_string(sl, "efgh", "\n", - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); - test_eq(2, smartlist_len(sl)); - test_streq("efgh", smartlist_get(sl, 1)); - - SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); - smartlist_clear(sl); - - /* Test swapping, shuffling, and sorting. */ - smartlist_split_string(sl, "the,onion,router,by,arma,and,nickm", ",", 0, 0); - test_eq(7, smartlist_len(sl)); - smartlist_sort(sl, _compare_strs); - cp_alloc = smartlist_join_strings(sl, ",", 0, NULL); - test_streq(cp_alloc,"and,arma,by,nickm,onion,router,the"); - tor_free(cp_alloc); - smartlist_swap(sl, 1, 5); - cp_alloc = smartlist_join_strings(sl, ",", 0, NULL); - test_streq(cp_alloc,"and,router,by,nickm,onion,arma,the"); - tor_free(cp_alloc); - smartlist_shuffle(sl); - test_eq(7, smartlist_len(sl)); - test_assert(smartlist_string_isin(sl, "and")); - test_assert(smartlist_string_isin(sl, "router")); - test_assert(smartlist_string_isin(sl, "by")); - test_assert(smartlist_string_isin(sl, "nickm")); - test_assert(smartlist_string_isin(sl, "onion")); - test_assert(smartlist_string_isin(sl, "arma")); - test_assert(smartlist_string_isin(sl, "the")); - - /* Test bsearch. */ - smartlist_sort(sl, _compare_strs); - test_streq("nickm", smartlist_bsearch(sl, "zNicKM", - _compare_without_first_ch)); - test_streq("and", smartlist_bsearch(sl, " AND", _compare_without_first_ch)); - test_eq_ptr(NULL, smartlist_bsearch(sl, " ANz", _compare_without_first_ch)); - - /* Test bsearch_idx */ - { - int f; - test_eq(0, smartlist_bsearch_idx(sl," aaa",_compare_without_first_ch,&f)); - test_eq(f, 0); - test_eq(0, smartlist_bsearch_idx(sl," and",_compare_without_first_ch,&f)); - test_eq(f, 1); - test_eq(1, smartlist_bsearch_idx(sl," arm",_compare_without_first_ch,&f)); - test_eq(f, 0); - test_eq(1, smartlist_bsearch_idx(sl," arma",_compare_without_first_ch,&f)); - test_eq(f, 1); - test_eq(2, smartlist_bsearch_idx(sl," armb",_compare_without_first_ch,&f)); - test_eq(f, 0); - test_eq(7, smartlist_bsearch_idx(sl," zzzz",_compare_without_first_ch,&f)); - test_eq(f, 0); - } - - /* Test reverse() and pop_last() */ - smartlist_reverse(sl); - cp_alloc = smartlist_join_strings(sl, ",", 0, NULL); - test_streq(cp_alloc,"the,router,onion,nickm,by,arma,and"); - tor_free(cp_alloc); - cp_alloc = smartlist_pop_last(sl); - test_streq(cp_alloc, "and"); - tor_free(cp_alloc); - test_eq(smartlist_len(sl), 6); - SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); - smartlist_clear(sl); - cp_alloc = smartlist_pop_last(sl); - test_eq(cp_alloc, NULL); - - /* Test uniq() */ - smartlist_split_string(sl, - "50,noon,radar,a,man,a,plan,a,canal,panama,radar,noon,50", - ",", 0, 0); - smartlist_sort(sl, _compare_strs); - smartlist_uniq(sl, _compare_strs, _tor_free); - cp_alloc = smartlist_join_strings(sl, ",", 0, NULL); - test_streq(cp_alloc, "50,a,canal,man,noon,panama,plan,radar"); - tor_free(cp_alloc); - - /* Test string_isin and isin_case and num_isin */ - test_assert(smartlist_string_isin(sl, "noon")); - test_assert(!smartlist_string_isin(sl, "noonoon")); - test_assert(smartlist_string_isin_case(sl, "nOOn")); - test_assert(!smartlist_string_isin_case(sl, "nooNooN")); - test_assert(smartlist_string_num_isin(sl, 50)); - test_assert(!smartlist_string_num_isin(sl, 60)); - - /* Test smartlist_choose */ - { - int i; - int allsame = 1; - int allin = 1; - void *first = smartlist_choose(sl); - test_assert(smartlist_isin(sl, first)); - for (i = 0; i < 100; ++i) { - void *second = smartlist_choose(sl); - if (second != first) - allsame = 0; - if (!smartlist_isin(sl, second)) - allin = 0; - } - test_assert(!allsame); - test_assert(allin); - } - SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); - smartlist_clear(sl); - - /* Test string_remove and remove and join_strings2 */ - smartlist_split_string(sl, - "Some say the Earth will end in ice and some in fire", - " ", 0, 0); - cp = smartlist_get(sl, 4); - test_streq(cp, "will"); - smartlist_add(sl, cp); - smartlist_remove(sl, cp); - tor_free(cp); - cp_alloc = smartlist_join_strings(sl, ",", 0, NULL); - test_streq(cp_alloc, "Some,say,the,Earth,fire,end,in,ice,and,some,in"); - tor_free(cp_alloc); - smartlist_string_remove(sl, "in"); - cp_alloc = smartlist_join_strings2(sl, "+XX", 1, 0, &sz); - test_streq(cp_alloc, "Some+say+the+Earth+fire+end+some+ice+and"); - test_eq((int)sz, 40); - - done: - - SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); - smartlist_free(sl); - tor_free(cp_alloc); -} - -/** Run unit tests for smartlist set manipulation functions. */ -static void -test_util_smartlist_overlap(void) -{ - smartlist_t *sl = smartlist_create(); - smartlist_t *ints = smartlist_create(); - smartlist_t *odds = smartlist_create(); - smartlist_t *evens = smartlist_create(); - smartlist_t *primes = smartlist_create(); - int i; - for (i=1; i < 10; i += 2) - smartlist_add(odds, (void*)(uintptr_t)i); - for (i=0; i < 10; i += 2) - smartlist_add(evens, (void*)(uintptr_t)i); - - /* add_all */ - smartlist_add_all(ints, odds); - smartlist_add_all(ints, evens); - test_eq(smartlist_len(ints), 10); - - smartlist_add(primes, (void*)2); - smartlist_add(primes, (void*)3); - smartlist_add(primes, (void*)5); - smartlist_add(primes, (void*)7); - - /* overlap */ - test_assert(smartlist_overlap(ints, odds)); - test_assert(smartlist_overlap(odds, primes)); - test_assert(smartlist_overlap(evens, primes)); - test_assert(!smartlist_overlap(odds, evens)); - - /* intersect */ - smartlist_add_all(sl, odds); - smartlist_intersect(sl, primes); - test_eq(smartlist_len(sl), 3); - test_assert(smartlist_isin(sl, (void*)3)); - test_assert(smartlist_isin(sl, (void*)5)); - test_assert(smartlist_isin(sl, (void*)7)); - - /* subtract */ - smartlist_add_all(sl, primes); - smartlist_subtract(sl, odds); - test_eq(smartlist_len(sl), 1); - test_assert(smartlist_isin(sl, (void*)2)); - - done: - smartlist_free(odds); - smartlist_free(evens); - smartlist_free(ints); - smartlist_free(primes); - smartlist_free(sl); -} - -/** Run unit tests for smartlist-of-digests functions. */ -static void -test_util_smartlist_digests(void) -{ - smartlist_t *sl = smartlist_create(); - - /* digest_isin. */ - smartlist_add(sl, tor_memdup("AAAAAAAAAAAAAAAAAAAA", DIGEST_LEN)); - smartlist_add(sl, tor_memdup("\00090AAB2AAAAaasdAAAAA", DIGEST_LEN)); - smartlist_add(sl, tor_memdup("\00090AAB2AAAAaasdAAAAA", DIGEST_LEN)); - test_eq(0, smartlist_digest_isin(NULL, "AAAAAAAAAAAAAAAAAAAA")); - test_assert(smartlist_digest_isin(sl, "AAAAAAAAAAAAAAAAAAAA")); - test_assert(smartlist_digest_isin(sl, "\00090AAB2AAAAaasdAAAAA")); - test_eq(0, smartlist_digest_isin(sl, "\00090AAB2AAABaasdAAAAA")); - - /* sort digests */ - smartlist_sort_digests(sl); - test_memeq(smartlist_get(sl, 0), "\00090AAB2AAAAaasdAAAAA", DIGEST_LEN); - test_memeq(smartlist_get(sl, 1), "\00090AAB2AAAAaasdAAAAA", DIGEST_LEN); - test_memeq(smartlist_get(sl, 2), "AAAAAAAAAAAAAAAAAAAA", DIGEST_LEN); - test_eq(3, smartlist_len(sl)); - - /* uniq_digests */ - smartlist_uniq_digests(sl); - test_eq(2, smartlist_len(sl)); - test_memeq(smartlist_get(sl, 0), "\00090AAB2AAAAaasdAAAAA", DIGEST_LEN); - test_memeq(smartlist_get(sl, 1), "AAAAAAAAAAAAAAAAAAAA", DIGEST_LEN); - - done: - SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); - smartlist_free(sl); -} - -/** Run unit tests for concatenate-a-smartlist-of-strings functions. */ -static void -test_util_smartlist_join(void) -{ - smartlist_t *sl = smartlist_create(); - smartlist_t *sl2 = smartlist_create(), *sl3 = smartlist_create(), - *sl4 = smartlist_create(); - char *joined=NULL; - /* unique, sorted. */ - smartlist_split_string(sl, - "Abashments Ambush Anchorman Bacon Banks Borscht " - "Bunks Inhumane Insurance Knish Know Manners " - "Maraschinos Stamina Sunbonnets Unicorns Wombats", - " ", 0, 0); - /* non-unique, sorted. */ - smartlist_split_string(sl2, - "Ambush Anchorman Anchorman Anemias Anemias Bacon " - "Crossbowmen Inhumane Insurance Knish Know Manners " - "Manners Maraschinos Wombats Wombats Work", - " ", 0, 0); - SMARTLIST_FOREACH_JOIN(sl, char *, cp1, - sl2, char *, cp2, - strcmp(cp1,cp2), - smartlist_add(sl3, cp2)) { - test_streq(cp1, cp2); - smartlist_add(sl4, cp1); - } SMARTLIST_FOREACH_JOIN_END(cp1, cp2); - - SMARTLIST_FOREACH(sl3, const char *, cp, - test_assert(smartlist_isin(sl2, cp) && - !smartlist_string_isin(sl, cp))); - SMARTLIST_FOREACH(sl4, const char *, cp, - test_assert(smartlist_isin(sl, cp) && - smartlist_string_isin(sl2, cp))); - joined = smartlist_join_strings(sl3, ",", 0, NULL); - test_streq(joined, "Anemias,Anemias,Crossbowmen,Work"); - tor_free(joined); - joined = smartlist_join_strings(sl4, ",", 0, NULL); - test_streq(joined, "Ambush,Anchorman,Anchorman,Bacon,Inhumane,Insurance," - "Knish,Know,Manners,Manners,Maraschinos,Wombats,Wombats"); - tor_free(joined); - - done: - smartlist_free(sl4); - smartlist_free(sl3); - SMARTLIST_FOREACH(sl2, char *, cp, tor_free(cp)); - smartlist_free(sl2); - SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); - smartlist_free(sl); - tor_free(joined); -} - -/** Run unit tests for bitarray code */ -static void -test_util_bitarray(void) -{ - bitarray_t *ba = NULL; - int i, j, ok=1; - - ba = bitarray_init_zero(1); - test_assert(ba); - test_assert(! bitarray_is_set(ba, 0)); - bitarray_set(ba, 0); - test_assert(bitarray_is_set(ba, 0)); - bitarray_clear(ba, 0); - test_assert(! bitarray_is_set(ba, 0)); - bitarray_free(ba); - - ba = bitarray_init_zero(1023); - for (i = 1; i < 64; ) { - for (j = 0; j < 1023; ++j) { - if (j % i) - bitarray_set(ba, j); - else - bitarray_clear(ba, j); - } - for (j = 0; j < 1023; ++j) { - if (!bool_eq(bitarray_is_set(ba, j), j%i)) - ok = 0; - } - test_assert(ok); - if (i < 7) - ++i; - else if (i == 28) - i = 32; - else - i += 7; - } - - done: - if (ba) - bitarray_free(ba); -} - -/** Run unit tests for digest set code (implemented as a hashtable or as a - * bloom filter) */ -static void -test_util_digestset(void) -{ - smartlist_t *included = smartlist_create(); - char d[DIGEST_LEN]; - int i; - int ok = 1; - int false_positives = 0; - digestset_t *set = NULL; - - for (i = 0; i < 1000; ++i) { - crypto_rand(d, DIGEST_LEN); - smartlist_add(included, tor_memdup(d, DIGEST_LEN)); - } - set = digestset_new(1000); - SMARTLIST_FOREACH(included, const char *, cp, - if (digestset_isin(set, cp)) - ok = 0); - test_assert(ok); - SMARTLIST_FOREACH(included, const char *, cp, - digestset_add(set, cp)); - SMARTLIST_FOREACH(included, const char *, cp, - if (!digestset_isin(set, cp)) - ok = 0); - test_assert(ok); - for (i = 0; i < 1000; ++i) { - crypto_rand(d, DIGEST_LEN); - if (digestset_isin(set, d)) - ++false_positives; - } - test_assert(false_positives < 50); /* Should be far lower. */ - - done: - if (set) - digestset_free(set); - SMARTLIST_FOREACH(included, char *, cp, tor_free(cp)); - smartlist_free(included); -} - -/** mutex for thread test to stop the threads hitting data at the same time. */ -static tor_mutex_t *_thread_test_mutex = NULL; -/** mutexes for the thread test to make sure that the threads have to - * interleave somewhat. */ -static tor_mutex_t *_thread_test_start1 = NULL, - *_thread_test_start2 = NULL; -/** Shared strmap for the thread test. */ -static strmap_t *_thread_test_strmap = NULL; -/** The name of thread1 for the thread test */ -static char *_thread1_name = NULL; -/** The name of thread2 for the thread test */ -static char *_thread2_name = NULL; - -static void _thread_test_func(void* _s) ATTR_NORETURN; - -/** How many iterations have the threads in the unit test run? */ -static int t1_count = 0, t2_count = 0; - -/** Helper function for threading unit tests: This function runs in a - * subthread. It grabs its own mutex (start1 or start2) to make sure that it - * should start, then it repeatedly alters _test_thread_strmap protected by - * _thread_test_mutex. */ -static void -_thread_test_func(void* _s) -{ - char *s = _s; - int i, *count; - tor_mutex_t *m; - char buf[64]; - char **cp; - if (!strcmp(s, "thread 1")) { - m = _thread_test_start1; - cp = &_thread1_name; - count = &t1_count; - } else { - m = _thread_test_start2; - cp = &_thread2_name; - count = &t2_count; - } - tor_mutex_acquire(m); - - tor_snprintf(buf, sizeof(buf), "%lu", tor_get_thread_id()); - *cp = tor_strdup(buf); - - for (i=0; i<10000; ++i) { - tor_mutex_acquire(_thread_test_mutex); - strmap_set(_thread_test_strmap, "last to run", *cp); - ++*count; - tor_mutex_release(_thread_test_mutex); - } - tor_mutex_acquire(_thread_test_mutex); - strmap_set(_thread_test_strmap, s, *cp); - tor_mutex_release(_thread_test_mutex); - - tor_mutex_release(m); - - spawn_exit(); -} - -/** Run unit tests for threading logic. */ -static void -test_util_threads(void) -{ - char *s1 = NULL, *s2 = NULL; - int done = 0, timedout = 0; - time_t started; -#ifndef MS_WINDOWS - struct timeval tv; - tv.tv_sec=0; - tv.tv_usec=10; -#endif -#ifndef TOR_IS_MULTITHREADED - /* Skip this test if we aren't threading. We should be threading most - * everywhere by now. */ - if (1) - return; -#endif - _thread_test_mutex = tor_mutex_new(); - _thread_test_start1 = tor_mutex_new(); - _thread_test_start2 = tor_mutex_new(); - _thread_test_strmap = strmap_new(); - s1 = tor_strdup("thread 1"); - s2 = tor_strdup("thread 2"); - tor_mutex_acquire(_thread_test_start1); - tor_mutex_acquire(_thread_test_start2); - spawn_func(_thread_test_func, s1); - spawn_func(_thread_test_func, s2); - tor_mutex_release(_thread_test_start2); - tor_mutex_release(_thread_test_start1); - started = time(NULL); - while (!done) { - tor_mutex_acquire(_thread_test_mutex); - strmap_assert_ok(_thread_test_strmap); - if (strmap_get(_thread_test_strmap, "thread 1") && - strmap_get(_thread_test_strmap, "thread 2")) { - done = 1; - } else if (time(NULL) > started + 25) { - timedout = done = 1; - } - tor_mutex_release(_thread_test_mutex); -#ifndef MS_WINDOWS - /* Prevent the main thread from starving the worker threads. */ - select(0, NULL, NULL, NULL, &tv); -#endif - } - - tor_mutex_acquire(_thread_test_start1); - tor_mutex_release(_thread_test_start1); - tor_mutex_acquire(_thread_test_start2); - tor_mutex_release(_thread_test_start2); - - tor_mutex_free(_thread_test_mutex); - - if (timedout) { - printf("\nTimed out: %d %d", t1_count, t2_count); - test_assert(strmap_get(_thread_test_strmap, "thread 1")); - test_assert(strmap_get(_thread_test_strmap, "thread 2")); - test_assert(!timedout); - } - - /* different thread IDs. */ - test_assert(strcmp(strmap_get(_thread_test_strmap, "thread 1"), - strmap_get(_thread_test_strmap, "thread 2"))); - test_assert(!strcmp(strmap_get(_thread_test_strmap, "thread 1"), - strmap_get(_thread_test_strmap, "last to run")) || - !strcmp(strmap_get(_thread_test_strmap, "thread 2"), - strmap_get(_thread_test_strmap, "last to run"))); - - done: - tor_free(s1); - tor_free(s2); - tor_free(_thread1_name); - tor_free(_thread2_name); - if (_thread_test_strmap) - strmap_free(_thread_test_strmap, NULL); - if (_thread_test_start1) - tor_mutex_free(_thread_test_start1); - if (_thread_test_start2) - tor_mutex_free(_thread_test_start2); -} - -/** Helper: return a tristate based on comparing two strings. */ -static int -_compare_strings_for_pqueue(const void *s1, const void *s2) -{ - return strcmp((const char*)s1, (const char*)s2); -} - -/** Run unit tests for heap-based priority queue functions. */ -static void -test_util_pqueue(void) -{ - smartlist_t *sl = smartlist_create(); - int (*cmp)(const void *, const void*); -#define OK() smartlist_pqueue_assert_ok(sl, cmp) - - cmp = _compare_strings_for_pqueue; - - smartlist_pqueue_add(sl, cmp, (char*)"cows"); - smartlist_pqueue_add(sl, cmp, (char*)"zebras"); - smartlist_pqueue_add(sl, cmp, (char*)"fish"); - smartlist_pqueue_add(sl, cmp, (char*)"frogs"); - smartlist_pqueue_add(sl, cmp, (char*)"apples"); - smartlist_pqueue_add(sl, cmp, (char*)"squid"); - smartlist_pqueue_add(sl, cmp, (char*)"daschunds"); - smartlist_pqueue_add(sl, cmp, (char*)"eggplants"); - smartlist_pqueue_add(sl, cmp, (char*)"weissbier"); - smartlist_pqueue_add(sl, cmp, (char*)"lobsters"); - smartlist_pqueue_add(sl, cmp, (char*)"roquefort"); - - OK(); - - test_eq(smartlist_len(sl), 11); - test_streq(smartlist_get(sl, 0), "apples"); - test_streq(smartlist_pqueue_pop(sl, cmp), "apples"); - test_eq(smartlist_len(sl), 10); - OK(); - test_streq(smartlist_pqueue_pop(sl, cmp), "cows"); - test_streq(smartlist_pqueue_pop(sl, cmp), "daschunds"); - smartlist_pqueue_add(sl, cmp, (char*)"chinchillas"); - OK(); - smartlist_pqueue_add(sl, cmp, (char*)"fireflies"); - OK(); - test_streq(smartlist_pqueue_pop(sl, cmp), "chinchillas"); - test_streq(smartlist_pqueue_pop(sl, cmp), "eggplants"); - test_streq(smartlist_pqueue_pop(sl, cmp), "fireflies"); - OK(); - test_streq(smartlist_pqueue_pop(sl, cmp), "fish"); - test_streq(smartlist_pqueue_pop(sl, cmp), "frogs"); - test_streq(smartlist_pqueue_pop(sl, cmp), "lobsters"); - test_streq(smartlist_pqueue_pop(sl, cmp), "roquefort"); - OK(); - test_eq(smartlist_len(sl), 3); - test_streq(smartlist_pqueue_pop(sl, cmp), "squid"); - test_streq(smartlist_pqueue_pop(sl, cmp), "weissbier"); - test_streq(smartlist_pqueue_pop(sl, cmp), "zebras"); - test_eq(smartlist_len(sl), 0); - OK(); -#undef OK - - done: - - smartlist_free(sl); -} - -/** Run unit tests for compression functions */ -static void -test_util_gzip(void) -{ - char *buf1=NULL, *buf2=NULL, *buf3=NULL, *cp1, *cp2; - const char *ccp2; - size_t len1, len2; - tor_zlib_state_t *state = NULL; - - buf1 = tor_strdup("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZAAAAAAAAAAAAAAAAAAAZ"); - test_assert(detect_compression_method(buf1, strlen(buf1)) == UNKNOWN_METHOD); - if (is_gzip_supported()) { - test_assert(!tor_gzip_compress(&buf2, &len1, buf1, strlen(buf1)+1, - GZIP_METHOD)); - test_assert(buf2); - test_assert(!memcmp(buf2, "\037\213", 2)); /* Gzip magic. */ - test_assert(detect_compression_method(buf2, len1) == GZIP_METHOD); - - test_assert(!tor_gzip_uncompress(&buf3, &len2, buf2, len1, - GZIP_METHOD, 1, LOG_INFO)); - test_assert(buf3); - test_streq(buf1,buf3); - - tor_free(buf2); - tor_free(buf3); - } - - test_assert(!tor_gzip_compress(&buf2, &len1, buf1, strlen(buf1)+1, - ZLIB_METHOD)); - test_assert(buf2); - test_assert(!memcmp(buf2, "\x78\xDA", 2)); /* deflate magic. */ - test_assert(detect_compression_method(buf2, len1) == ZLIB_METHOD); - - test_assert(!tor_gzip_uncompress(&buf3, &len2, buf2, len1, - ZLIB_METHOD, 1, LOG_INFO)); - test_assert(buf3); - test_streq(buf1,buf3); - - /* Check whether we can uncompress concatenated, compressed strings. */ - tor_free(buf3); - buf2 = tor_realloc(buf2, len1*2); - memcpy(buf2+len1, buf2, len1); - test_assert(!tor_gzip_uncompress(&buf3, &len2, buf2, len1*2, - ZLIB_METHOD, 1, LOG_INFO)); - test_eq(len2, (strlen(buf1)+1)*2); - test_memeq(buf3, - "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZAAAAAAAAAAAAAAAAAAAZ\0" - "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZAAAAAAAAAAAAAAAAAAAZ\0", - (strlen(buf1)+1)*2); - - tor_free(buf1); - tor_free(buf2); - tor_free(buf3); - - /* Check whether we can uncompress partial strings. */ - buf1 = - tor_strdup("String with low redundancy that won't be compressed much."); - test_assert(!tor_gzip_compress(&buf2, &len1, buf1, strlen(buf1)+1, - ZLIB_METHOD)); - tor_assert(len1>16); - /* when we allow an incomplete string, we should succeed.*/ - tor_assert(!tor_gzip_uncompress(&buf3, &len2, buf2, len1-16, - ZLIB_METHOD, 0, LOG_INFO)); - buf3[len2]='\0'; - tor_assert(len2 > 5); - tor_assert(!strcmpstart(buf1, buf3)); - - /* when we demand a complete string, this must fail. */ - tor_free(buf3); - tor_assert(tor_gzip_uncompress(&buf3, &len2, buf2, len1-16, - ZLIB_METHOD, 1, LOG_INFO)); - tor_assert(!buf3); - - /* Now, try streaming compression. */ - tor_free(buf1); - tor_free(buf2); - tor_free(buf3); - state = tor_zlib_new(1, ZLIB_METHOD); - tor_assert(state); - cp1 = buf1 = tor_malloc(1024); - len1 = 1024; - ccp2 = "ABCDEFGHIJABCDEFGHIJ"; - len2 = 21; - test_assert(tor_zlib_process(state, &cp1, &len1, &ccp2, &len2, 0) - == TOR_ZLIB_OK); - test_eq(len2, 0); /* Make sure we compressed it all. */ - test_assert(cp1 > buf1); - - len2 = 0; - cp2 = cp1; - test_assert(tor_zlib_process(state, &cp1, &len1, &ccp2, &len2, 1) - == TOR_ZLIB_DONE); - test_eq(len2, 0); - test_assert(cp1 > cp2); /* Make sure we really added something. */ - - tor_assert(!tor_gzip_uncompress(&buf3, &len2, buf1, 1024-len1, - ZLIB_METHOD, 1, LOG_WARN)); - test_streq(buf3, "ABCDEFGHIJABCDEFGHIJ"); /*Make sure it compressed right.*/ - - done: - if (state) - tor_zlib_free(state); - tor_free(buf2); - tor_free(buf3); - tor_free(buf1); -} - -/** Run unit tests for string-to-void* map functions */ -static void -test_util_strmap(void) -{ - strmap_t *map; - strmap_iter_t *iter; - const char *k; - void *v; - char *visited = NULL; - smartlist_t *found_keys = NULL; - - map = strmap_new(); - test_assert(map); - test_eq(strmap_size(map), 0); - test_assert(strmap_isempty(map)); - v = strmap_set(map, "K1", (void*)99); - test_eq(v, NULL); - test_assert(!strmap_isempty(map)); - v = strmap_set(map, "K2", (void*)101); - test_eq(v, NULL); - v = strmap_set(map, "K1", (void*)100); - test_eq(v, (void*)99); - test_eq_ptr(strmap_get(map,"K1"), (void*)100); - test_eq_ptr(strmap_get(map,"K2"), (void*)101); - test_eq_ptr(strmap_get(map,"K-not-there"), NULL); - strmap_assert_ok(map); - - v = strmap_remove(map,"K2"); - strmap_assert_ok(map); - test_eq_ptr(v, (void*)101); - test_eq_ptr(strmap_get(map,"K2"), NULL); - test_eq_ptr(strmap_remove(map,"K2"), NULL); - - strmap_set(map, "K2", (void*)101); - strmap_set(map, "K3", (void*)102); - strmap_set(map, "K4", (void*)103); - test_eq(strmap_size(map), 4); - strmap_assert_ok(map); - strmap_set(map, "K5", (void*)104); - strmap_set(map, "K6", (void*)105); - strmap_assert_ok(map); - - /* Test iterator. */ - iter = strmap_iter_init(map); - found_keys = smartlist_create(); - while (!strmap_iter_done(iter)) { - strmap_iter_get(iter,&k,&v); - smartlist_add(found_keys, tor_strdup(k)); - test_eq_ptr(v, strmap_get(map, k)); - - if (!strcmp(k, "K2")) { - iter = strmap_iter_next_rmv(map,iter); - } else { - iter = strmap_iter_next(map,iter); - } - } - - /* Make sure we removed K2, but not the others. */ - test_eq_ptr(strmap_get(map, "K2"), NULL); - test_eq_ptr(strmap_get(map, "K5"), (void*)104); - /* Make sure we visited everyone once */ - smartlist_sort_strings(found_keys); - visited = smartlist_join_strings(found_keys, ":", 0, NULL); - test_streq(visited, "K1:K2:K3:K4:K5:K6"); - - strmap_assert_ok(map); - /* Clean up after ourselves. */ - strmap_free(map, NULL); - map = NULL; - - /* Now try some lc functions. */ - map = strmap_new(); - strmap_set_lc(map,"Ab.C", (void*)1); - test_eq_ptr(strmap_get(map,"ab.c"), (void*)1); - strmap_assert_ok(map); - test_eq_ptr(strmap_get_lc(map,"AB.C"), (void*)1); - test_eq_ptr(strmap_get(map,"AB.C"), NULL); - test_eq_ptr(strmap_remove_lc(map,"aB.C"), (void*)1); - strmap_assert_ok(map); - test_eq_ptr(strmap_get_lc(map,"AB.C"), NULL); - - done: - if (map) - strmap_free(map,NULL); - if (found_keys) { - SMARTLIST_FOREACH(found_keys, char *, cp, tor_free(cp)); - smartlist_free(found_keys); - } - tor_free(visited); -} - -/** Run unit tests for mmap() wrapper functionality. */ -static void -test_util_mmap(void) -{ - char *fname1 = tor_strdup(get_fname("mapped_1")); - char *fname2 = tor_strdup(get_fname("mapped_2")); - char *fname3 = tor_strdup(get_fname("mapped_3")); - const size_t buflen = 17000; - char *buf = tor_malloc(17000); - tor_mmap_t *mapping = NULL; - - crypto_rand(buf, buflen); - - mapping = tor_mmap_file(fname1); - test_assert(! mapping); - - write_str_to_file(fname1, "Short file.", 1); - write_bytes_to_file(fname2, buf, buflen, 1); - write_bytes_to_file(fname3, buf, 16384, 1); - - mapping = tor_mmap_file(fname1); - test_assert(mapping); - test_eq(mapping->size, strlen("Short file.")); - test_streq(mapping->data, "Short file."); -#ifdef MS_WINDOWS - tor_munmap_file(mapping); - mapping = NULL; - test_assert(unlink(fname1) == 0); -#else - /* make sure we can unlink. */ - test_assert(unlink(fname1) == 0); - test_streq(mapping->data, "Short file."); - tor_munmap_file(mapping); - mapping = NULL; -#endif - - /* Now a zero-length file. */ - write_str_to_file(fname1, "", 1); - mapping = tor_mmap_file(fname1); - test_eq(mapping, NULL); - test_eq(ERANGE, errno); - unlink(fname1); - - /* Make sure that we fail to map a no-longer-existent file. */ - mapping = tor_mmap_file(fname1); - test_assert(mapping == NULL); - - /* Now try a big file that stretches across a few pages and isn't aligned */ - mapping = tor_mmap_file(fname2); - test_assert(mapping); - test_eq(mapping->size, buflen); - test_memeq(mapping->data, buf, buflen); - tor_munmap_file(mapping); - mapping = NULL; - - /* Now try a big aligned file. */ - mapping = tor_mmap_file(fname3); - test_assert(mapping); - test_eq(mapping->size, 16384); - test_memeq(mapping->data, buf, 16384); - tor_munmap_file(mapping); - mapping = NULL; - - done: - unlink(fname1); - unlink(fname2); - unlink(fname3); - - tor_free(fname1); - tor_free(fname2); - tor_free(fname3); - tor_free(buf); - - if (mapping) - tor_munmap_file(mapping); -} - -/** Run unit tests for escaping/unescaping data for use by controllers. */ -static void -test_util_control_formats(void) -{ - char *out = NULL; - const char *inp = - "..This is a test\r\nof the emergency \nbroadcast\r\n..system.\r\nZ.\r\n"; - size_t sz; - - sz = read_escaped_data(inp, strlen(inp), &out); - test_streq(out, - ".This is a test\nof the emergency \nbroadcast\n.system.\nZ.\n"); - test_eq(sz, strlen(out)); - - done: - tor_free(out); -} - -static void -test_util_sscanf(void) -{ - unsigned u1, u2, u3; - char s1[10], s2[10], s3[10], ch; - int r; - - r = tor_sscanf("hello world", "hello world"); /* String match: success */ - test_eq(r, 0); - r = tor_sscanf("hello world 3", "hello worlb %u", &u1); /* String fail */ - test_eq(r, 0); - r = tor_sscanf("12345", "%u", &u1); /* Simple number */ - test_eq(r, 1); - test_eq(u1, 12345u); - r = tor_sscanf("", "%u", &u1); /* absent number */ - test_eq(r, 0); - r = tor_sscanf("A", "%u", &u1); /* bogus number */ - test_eq(r, 0); - r = tor_sscanf("4294967295", "%u", &u1); /* UINT32_MAX should work. */ - test_eq(r, 1); - test_eq(u1, 4294967295u); - r = tor_sscanf("4294967296", "%u", &u1); /* Always say -1 at 32 bits. */ - test_eq(r, 0); - r = tor_sscanf("123456", "%2u%u", &u1, &u2); /* Width */ - test_eq(r, 2); - test_eq(u1, 12u); - test_eq(u2, 3456u); - r = tor_sscanf("!12:3:456", "!%2u:%2u:%3u", &u1, &u2, &u3); /* separators */ - test_eq(r, 3); - test_eq(u1, 12u); - test_eq(u2, 3u); - test_eq(u3, 456u); - r = tor_sscanf("12:3:045", "%2u:%2u:%3u", &u1, &u2, &u3); /* 0s */ - test_eq(r, 3); - test_eq(u1, 12u); - test_eq(u2, 3u); - test_eq(u3, 45u); - /* %u does not match space.*/ - r = tor_sscanf("12:3: 45", "%2u:%2u:%3u", &u1, &u2, &u3); - test_eq(r, 2); - /* %u does not match negative numbers. */ - r = tor_sscanf("12:3:-4", "%2u:%2u:%3u", &u1, &u2, &u3); - test_eq(r, 2); - /* Arbitrary amounts of 0-padding are okay */ - r = tor_sscanf("12:03:000000000000000099", "%2u:%2u:%u", &u1, &u2, &u3); - test_eq(r, 3); - test_eq(u1, 12u); - test_eq(u2, 3u); - test_eq(u3, 99u); - - r = tor_sscanf("99% fresh", "%3u%% fresh", &u1); /* percents are scannable.*/ - test_eq(r, 1); - test_eq(u1, 99); - - r = tor_sscanf("hello", "%s", s1); /* %s needs a number. */ - test_eq(r, -1); - - r = tor_sscanf("hello", "%3s%7s", s1, s2); /* %s matches characters. */ - test_eq(r, 2); - test_streq(s1, "hel"); - test_streq(s2, "lo"); - r = tor_sscanf("WD40", "%2s%u", s3, &u1); /* %s%u */ - test_eq(r, 2); - test_streq(s3, "WD"); - test_eq(u1, 40); - r = tor_sscanf("76trombones", "%6u%9s", &u1, s1); /* %u%s */ - test_eq(r, 2); - test_eq(u1, 76); - test_streq(s1, "trombones"); - r = tor_sscanf("hello world", "%9s %9s", s1, s2); /* %s doesn't eat space. */ - test_eq(r, 2); - test_streq(s1, "hello"); - test_streq(s2, "world"); - r = tor_sscanf("hi", "%9s%9s%3s", s1, s2, s3); /* %s can be empty. */ - test_eq(r, 3); - test_streq(s1, "hi"); - test_streq(s2, ""); - test_streq(s3, ""); - - r = tor_sscanf("1.2.3", "%u.%u.%u%c", &u1, &u2, &u3, &ch); - test_eq(r, 3); - r = tor_sscanf("1.2.3 foobar", "%u.%u.%u%c", &u1, &u2, &u3, &ch); - test_eq(r, 4); - - done: - ; -} - -/** Run unit tests for the onion handshake code. */ -static void -test_onion_handshake(void) -{ - /* client-side */ - crypto_dh_env_t *c_dh = NULL; - char c_buf[ONIONSKIN_CHALLENGE_LEN]; - char c_keys[40]; - - /* server-side */ - char s_buf[ONIONSKIN_REPLY_LEN]; - char s_keys[40]; - - /* shared */ - crypto_pk_env_t *pk = NULL; - - pk = pk_generate(0); - - /* client handshake 1. */ - memset(c_buf, 0, ONIONSKIN_CHALLENGE_LEN); - test_assert(! onion_skin_create(pk, &c_dh, c_buf)); - - /* server handshake */ - memset(s_buf, 0, ONIONSKIN_REPLY_LEN); - memset(s_keys, 0, 40); - test_assert(! onion_skin_server_handshake(c_buf, pk, NULL, - s_buf, s_keys, 40)); - - /* client handshake 2 */ - memset(c_keys, 0, 40); - test_assert(! onion_skin_client_handshake(c_dh, s_buf, c_keys, 40)); - - if (memcmp(c_keys, s_keys, 40)) { - puts("Aiiiie"); - exit(1); - } - test_memeq(c_keys, s_keys, 40); - memset(s_buf, 0, 40); - test_memneq(c_keys, s_buf, 40); - - done: - if (c_dh) - crypto_dh_free(c_dh); - if (pk) - crypto_free_pk_env(pk); -} - -/** Run unit tests for router descriptor generation logic. */ -static void -test_dir_format(void) -{ - char buf[8192], buf2[8192]; - char platform[256]; - char fingerprint[FINGERPRINT_LEN+1]; - char *pk1_str = NULL, *pk2_str = NULL, *pk3_str = NULL, *cp; - size_t pk1_str_len, pk2_str_len, pk3_str_len; - routerinfo_t *r1=NULL, *r2=NULL; - crypto_pk_env_t *pk1 = NULL, *pk2 = NULL, *pk3 = NULL; - routerinfo_t *rp1 = NULL; - addr_policy_t *ex1, *ex2; - routerlist_t *dir1 = NULL, *dir2 = NULL; - tor_version_t ver1; - - pk1 = pk_generate(0); - pk2 = pk_generate(1); - pk3 = pk_generate(2); - - test_assert( is_legal_nickname("a")); - test_assert(!is_legal_nickname("")); - test_assert(!is_legal_nickname("abcdefghijklmnopqrst")); /* 20 chars */ - test_assert(!is_legal_nickname("hyphen-")); /* bad char */ - test_assert( is_legal_nickname("abcdefghijklmnopqrs")); /* 19 chars */ - test_assert(!is_legal_nickname("$AAAAAAAA01234AAAAAAAAAAAAAAAAAAAAAAAAAAA")); - /* valid */ - test_assert( is_legal_nickname_or_hexdigest( - "$AAAAAAAA01234AAAAAAAAAAAAAAAAAAAAAAAAAAA")); - test_assert( is_legal_nickname_or_hexdigest( - "$AAAAAAAA01234AAAAAAAAAAAAAAAAAAAAAAAAAAA=fred")); - test_assert( is_legal_nickname_or_hexdigest( - "$AAAAAAAA01234AAAAAAAAAAAAAAAAAAAAAAAAAAA~fred")); - /* too short */ - test_assert(!is_legal_nickname_or_hexdigest( - "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")); - /* illegal char */ - test_assert(!is_legal_nickname_or_hexdigest( - "$AAAAAAzAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")); - /* hex part too long */ - test_assert(!is_legal_nickname_or_hexdigest( - "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")); - test_assert(!is_legal_nickname_or_hexdigest( - "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=fred")); - /* Bad nickname */ - test_assert(!is_legal_nickname_or_hexdigest( - "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=")); - test_assert(!is_legal_nickname_or_hexdigest( - "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~")); - test_assert(!is_legal_nickname_or_hexdigest( - "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~hyphen-")); - test_assert(!is_legal_nickname_or_hexdigest( - "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~" - "abcdefghijklmnoppqrst")); - /* Bad extra char. */ - test_assert(!is_legal_nickname_or_hexdigest( - "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA!")); - test_assert(is_legal_nickname_or_hexdigest("xyzzy")); - test_assert(is_legal_nickname_or_hexdigest("abcdefghijklmnopqrs")); - test_assert(!is_legal_nickname_or_hexdigest("abcdefghijklmnopqrst")); - - get_platform_str(platform, sizeof(platform)); - r1 = tor_malloc_zero(sizeof(routerinfo_t)); - r1->address = tor_strdup("18.244.0.1"); - r1->addr = 0xc0a80001u; /* 192.168.0.1 */ - r1->cache_info.published_on = 0; - r1->or_port = 9000; - r1->dir_port = 9003; - r1->onion_pkey = crypto_pk_dup_key(pk1); - r1->identity_pkey = crypto_pk_dup_key(pk2); - r1->bandwidthrate = 1000; - r1->bandwidthburst = 5000; - r1->bandwidthcapacity = 10000; - r1->exit_policy = NULL; - r1->nickname = tor_strdup("Magri"); - r1->platform = tor_strdup(platform); - - ex1 = tor_malloc_zero(sizeof(addr_policy_t)); - ex2 = tor_malloc_zero(sizeof(addr_policy_t)); - ex1->policy_type = ADDR_POLICY_ACCEPT; - tor_addr_from_ipv4h(&ex1->addr, 0); - ex1->maskbits = 0; - ex1->prt_min = ex1->prt_max = 80; - ex2->policy_type = ADDR_POLICY_REJECT; - tor_addr_from_ipv4h(&ex2->addr, 18<<24); - ex2->maskbits = 8; - ex2->prt_min = ex2->prt_max = 24; - r2 = tor_malloc_zero(sizeof(routerinfo_t)); - r2->address = tor_strdup("1.1.1.1"); - r2->addr = 0x0a030201u; /* 10.3.2.1 */ - r2->platform = tor_strdup(platform); - r2->cache_info.published_on = 5; - r2->or_port = 9005; - r2->dir_port = 0; - r2->onion_pkey = crypto_pk_dup_key(pk2); - r2->identity_pkey = crypto_pk_dup_key(pk1); - r2->bandwidthrate = r2->bandwidthburst = r2->bandwidthcapacity = 3000; - r2->exit_policy = smartlist_create(); - smartlist_add(r2->exit_policy, ex2); - smartlist_add(r2->exit_policy, ex1); - r2->nickname = tor_strdup("Fred"); - - test_assert(!crypto_pk_write_public_key_to_string(pk1, &pk1_str, - &pk1_str_len)); - test_assert(!crypto_pk_write_public_key_to_string(pk2 , &pk2_str, - &pk2_str_len)); - test_assert(!crypto_pk_write_public_key_to_string(pk3 , &pk3_str, - &pk3_str_len)); - - memset(buf, 0, 2048); - test_assert(router_dump_router_to_string(buf, 2048, r1, pk2)>0); - - strlcpy(buf2, "router Magri 18.244.0.1 9000 0 9003\n" - "platform Tor "VERSION" on ", sizeof(buf2)); - strlcat(buf2, get_uname(), sizeof(buf2)); - strlcat(buf2, "\n" - "opt protocols Link 1 2 Circuit 1\n" - "published 1970-01-01 00:00:00\n" - "opt fingerprint ", sizeof(buf2)); - test_assert(!crypto_pk_get_fingerprint(pk2, fingerprint, 1)); - strlcat(buf2, fingerprint, sizeof(buf2)); - strlcat(buf2, "\nuptime 0\n" - /* XXX the "0" above is hard-coded, but even if we made it reflect - * uptime, that still wouldn't make it right, because the two - * descriptors might be made on different seconds... hm. */ - "bandwidth 1000 5000 10000\n" - "opt extra-info-digest 0000000000000000000000000000000000000000\n" - "onion-key\n", sizeof(buf2)); - strlcat(buf2, pk1_str, sizeof(buf2)); - strlcat(buf2, "signing-key\n", sizeof(buf2)); - strlcat(buf2, pk2_str, sizeof(buf2)); - strlcat(buf2, "opt hidden-service-dir\n", sizeof(buf2)); - strlcat(buf2, "reject *:*\nrouter-signature\n", sizeof(buf2)); - buf[strlen(buf2)] = '\0'; /* Don't compare the sig; it's never the same - * twice */ - - test_streq(buf, buf2); - - test_assert(router_dump_router_to_string(buf, 2048, r1, pk2)>0); - cp = buf; - rp1 = router_parse_entry_from_string((const char*)cp,NULL,1,0,NULL); - test_assert(rp1); - test_streq(rp1->address, r1->address); - test_eq(rp1->or_port, r1->or_port); - //test_eq(rp1->dir_port, r1->dir_port); - test_eq(rp1->bandwidthrate, r1->bandwidthrate); - test_eq(rp1->bandwidthburst, r1->bandwidthburst); - test_eq(rp1->bandwidthcapacity, r1->bandwidthcapacity); - test_assert(crypto_pk_cmp_keys(rp1->onion_pkey, pk1) == 0); - test_assert(crypto_pk_cmp_keys(rp1->identity_pkey, pk2) == 0); - //test_assert(rp1->exit_policy == NULL); - -#if 0 - /* XXX Once we have exit policies, test this again. XXX */ - strlcpy(buf2, "router tor.tor.tor 9005 0 0 3000\n", sizeof(buf2)); - strlcat(buf2, pk2_str, sizeof(buf2)); - strlcat(buf2, "signing-key\n", sizeof(buf2)); - strlcat(buf2, pk1_str, sizeof(buf2)); - strlcat(buf2, "accept *:80\nreject 18.*:24\n\n", sizeof(buf2)); - test_assert(router_dump_router_to_string(buf, 2048, &r2, pk2)>0); - test_streq(buf, buf2); - - cp = buf; - rp2 = router_parse_entry_from_string(&cp,1); - test_assert(rp2); - test_streq(rp2->address, r2.address); - test_eq(rp2->or_port, r2.or_port); - test_eq(rp2->dir_port, r2.dir_port); - test_eq(rp2->bandwidth, r2.bandwidth); - test_assert(crypto_pk_cmp_keys(rp2->onion_pkey, pk2) == 0); - test_assert(crypto_pk_cmp_keys(rp2->identity_pkey, pk1) == 0); - test_eq(rp2->exit_policy->policy_type, EXIT_POLICY_ACCEPT); - test_streq(rp2->exit_policy->string, "accept *:80"); - test_streq(rp2->exit_policy->address, "*"); - test_streq(rp2->exit_policy->port, "80"); - test_eq(rp2->exit_policy->next->policy_type, EXIT_POLICY_REJECT); - test_streq(rp2->exit_policy->next->string, "reject 18.*:24"); - test_streq(rp2->exit_policy->next->address, "18.*"); - test_streq(rp2->exit_policy->next->port, "24"); - test_assert(rp2->exit_policy->next->next == NULL); - - /* Okay, now for the directories. */ - { - fingerprint_list = smartlist_create(); - crypto_pk_get_fingerprint(pk2, buf, 1); - add_fingerprint_to_dir("Magri", buf, fingerprint_list); - crypto_pk_get_fingerprint(pk1, buf, 1); - add_fingerprint_to_dir("Fred", buf, fingerprint_list); - } - - { - char d[DIGEST_LEN]; - const char *m; - /* XXXX NM re-enable. */ - /* Make sure routers aren't too far in the past any more. */ - r1->cache_info.published_on = time(NULL); - r2->cache_info.published_on = time(NULL)-3*60*60; - test_assert(router_dump_router_to_string(buf, 2048, r1, pk2)>0); - test_eq(dirserv_add_descriptor(buf,&m,""), ROUTER_ADDED_NOTIFY_GENERATOR); - test_assert(router_dump_router_to_string(buf, 2048, r2, pk1)>0); - test_eq(dirserv_add_descriptor(buf,&m,""), ROUTER_ADDED_NOTIFY_GENERATOR); - get_options()->Nickname = tor_strdup("DirServer"); - test_assert(!dirserv_dump_directory_to_string(&cp,pk3, 0)); - crypto_pk_get_digest(pk3, d); - test_assert(!router_parse_directory(cp)); - test_eq(2, smartlist_len(dir1->routers)); - tor_free(cp); - } -#endif - dirserv_free_fingerprint_list(); - - /* Try out version parsing functionality */ - test_eq(0, tor_version_parse("0.3.4pre2-cvs", &ver1)); - test_eq(0, ver1.major); - test_eq(3, ver1.minor); - test_eq(4, ver1.micro); - test_eq(VER_PRE, ver1.status); - test_eq(2, ver1.patchlevel); - test_eq(0, tor_version_parse("0.3.4rc1", &ver1)); - test_eq(0, ver1.major); - test_eq(3, ver1.minor); - test_eq(4, ver1.micro); - test_eq(VER_RC, ver1.status); - test_eq(1, ver1.patchlevel); - test_eq(0, tor_version_parse("1.3.4", &ver1)); - test_eq(1, ver1.major); - test_eq(3, ver1.minor); - test_eq(4, ver1.micro); - test_eq(VER_RELEASE, ver1.status); - test_eq(0, ver1.patchlevel); - test_eq(0, tor_version_parse("1.3.4.999", &ver1)); - test_eq(1, ver1.major); - test_eq(3, ver1.minor); - test_eq(4, ver1.micro); - test_eq(VER_RELEASE, ver1.status); - test_eq(999, ver1.patchlevel); - test_eq(0, tor_version_parse("0.1.2.4-alpha", &ver1)); - test_eq(0, ver1.major); - test_eq(1, ver1.minor); - test_eq(2, ver1.micro); - test_eq(4, ver1.patchlevel); - test_eq(VER_RELEASE, ver1.status); - test_streq("alpha", ver1.status_tag); - test_eq(0, tor_version_parse("0.1.2.4", &ver1)); - test_eq(0, ver1.major); - test_eq(1, ver1.minor); - test_eq(2, ver1.micro); - test_eq(4, ver1.patchlevel); - test_eq(VER_RELEASE, ver1.status); - test_streq("", ver1.status_tag); - -#define test_eq_vs(vs1, vs2) test_eq_type(version_status_t, "%d", (vs1), (vs2)) -#define test_v_i_o(val, ver, lst) \ - test_eq_vs(val, tor_version_is_obsolete(ver, lst)) - - /* make sure tor_version_is_obsolete() works */ - test_v_i_o(VS_OLD, "0.0.1", "Tor 0.0.2"); - test_v_i_o(VS_OLD, "0.0.1", "0.0.2, Tor 0.0.3"); - test_v_i_o(VS_OLD, "0.0.1", "0.0.2,Tor 0.0.3"); - test_v_i_o(VS_OLD, "0.0.1","0.0.3,BetterTor 0.0.1"); - test_v_i_o(VS_RECOMMENDED, "0.0.2", "Tor 0.0.2,Tor 0.0.3"); - test_v_i_o(VS_NEW_IN_SERIES, "0.0.2", "Tor 0.0.2pre1,Tor 0.0.3"); - test_v_i_o(VS_OLD, "0.0.2", "Tor 0.0.2.1,Tor 0.0.3"); - test_v_i_o(VS_NEW, "0.1.0", "Tor 0.0.2,Tor 0.0.3"); - test_v_i_o(VS_RECOMMENDED, "0.0.7rc2", "0.0.7,Tor 0.0.7rc2,Tor 0.0.8"); - test_v_i_o(VS_OLD, "0.0.5.0", "0.0.5.1-cvs"); - test_v_i_o(VS_NEW_IN_SERIES, "0.0.5.1-cvs", "0.0.5, 0.0.6"); - /* Not on list, but newer than any in same series. */ - test_v_i_o(VS_NEW_IN_SERIES, "0.1.0.3", - "Tor 0.1.0.2,Tor 0.0.9.5,Tor 0.1.1.0"); - /* Series newer than any on list. */ - test_v_i_o(VS_NEW, "0.1.2.3", "Tor 0.1.0.2,Tor 0.0.9.5,Tor 0.1.1.0"); - /* Series older than any on list. */ - test_v_i_o(VS_OLD, "0.0.1.3", "Tor 0.1.0.2,Tor 0.0.9.5,Tor 0.1.1.0"); - /* Not on list, not newer than any on same series. */ - test_v_i_o(VS_UNRECOMMENDED, "0.1.0.1", - "Tor 0.1.0.2,Tor 0.0.9.5,Tor 0.1.1.0"); - /* On list, not newer than any on same series. */ - test_v_i_o(VS_UNRECOMMENDED, - "0.1.0.1", "Tor 0.1.0.2,Tor 0.0.9.5,Tor 0.1.1.0"); - test_eq(0, tor_version_as_new_as("Tor 0.0.5", "0.0.9pre1-cvs")); - test_eq(1, tor_version_as_new_as( - "Tor 0.0.8 on Darwin 64-121-192-100.c3-0." - "sfpo-ubr1.sfrn-sfpo.ca.cable.rcn.com Power Macintosh", - "0.0.8rc2")); - test_eq(0, tor_version_as_new_as( - "Tor 0.0.8 on Darwin 64-121-192-100.c3-0." - "sfpo-ubr1.sfrn-sfpo.ca.cable.rcn.com Power Macintosh", "0.0.8.2")); - - /* Now try svn revisions. */ - test_eq(1, tor_version_as_new_as("Tor 0.2.1.0-dev (r100)", - "Tor 0.2.1.0-dev (r99)")); - test_eq(1, tor_version_as_new_as("Tor 0.2.1.0-dev (r100) on Banana Jr", - "Tor 0.2.1.0-dev (r99) on Hal 9000")); - test_eq(1, tor_version_as_new_as("Tor 0.2.1.0-dev (r100)", - "Tor 0.2.1.0-dev on Colossus")); - test_eq(0, tor_version_as_new_as("Tor 0.2.1.0-dev (r99)", - "Tor 0.2.1.0-dev (r100)")); - test_eq(0, tor_version_as_new_as("Tor 0.2.1.0-dev (r99) on MCP", - "Tor 0.2.1.0-dev (r100) on AM")); - test_eq(0, tor_version_as_new_as("Tor 0.2.1.0-dev", - "Tor 0.2.1.0-dev (r99)")); - test_eq(1, tor_version_as_new_as("Tor 0.2.1.1", - "Tor 0.2.1.0-dev (r99)")); - done: - if (r1) - routerinfo_free(r1); - if (r2) - routerinfo_free(r2); - - tor_free(pk1_str); - tor_free(pk2_str); - tor_free(pk3_str); - if (pk1) crypto_free_pk_env(pk1); - if (pk2) crypto_free_pk_env(pk2); - if (pk3) crypto_free_pk_env(pk3); - if (rp1) routerinfo_free(rp1); - tor_free(dir1); /* XXXX And more !*/ - tor_free(dir2); /* And more !*/ -} - -/** Run unit tests for misc directory functions. */ -static void -test_dirutil(void) -{ - smartlist_t *sl = smartlist_create(); - fp_pair_t *pair; - - dir_split_resource_into_fingerprint_pairs( - /* Two pairs, out of order, with one duplicate. */ - "73656372657420646174612E0000000000FFFFFF-" - "557365204145532d32353620696e73746561642e+" - "73656372657420646174612E0000000000FFFFFF-" - "557365204145532d32353620696e73746561642e+" - "48657861646563696d616c2069736e277420736f-" - "676f6f6420666f7220686964696e6720796f7572.z", sl); - - test_eq(smartlist_len(sl), 2); - pair = smartlist_get(sl, 0); - test_memeq(pair->first, "Hexadecimal isn't so", DIGEST_LEN); - test_memeq(pair->second, "good for hiding your", DIGEST_LEN); - pair = smartlist_get(sl, 1); - test_memeq(pair->first, "secret data.\0\0\0\0\0\xff\xff\xff", DIGEST_LEN); - test_memeq(pair->second, "Use AES-256 instead.", DIGEST_LEN); - - done: - SMARTLIST_FOREACH(sl, fp_pair_t *, pair, tor_free(pair)); - smartlist_free(sl); -} - -extern const char AUTHORITY_CERT_1[]; -extern const char AUTHORITY_SIGNKEY_1[]; -extern const char AUTHORITY_CERT_2[]; -extern const char AUTHORITY_SIGNKEY_2[]; -extern const char AUTHORITY_CERT_3[]; -extern const char AUTHORITY_SIGNKEY_3[]; - -/** Helper: Test that two networkstatus_voter_info_t do in fact represent the - * same voting authority, and that they do in fact have all the same - * information. */ -static void -test_same_voter(networkstatus_voter_info_t *v1, - networkstatus_voter_info_t *v2) -{ - test_streq(v1->nickname, v2->nickname); - test_memeq(v1->identity_digest, v2->identity_digest, DIGEST_LEN); - test_streq(v1->address, v2->address); - test_eq(v1->addr, v2->addr); - test_eq(v1->dir_port, v2->dir_port); - test_eq(v1->or_port, v2->or_port); - test_streq(v1->contact, v2->contact); - test_memeq(v1->vote_digest, v2->vote_digest, DIGEST_LEN); - done: - ; -} - -/** Run unit tests for getting the median of a list. */ -static void -test_util_order_functions(void) -{ - int lst[25], n = 0; - // int a=12,b=24,c=25,d=60,e=77; - -#define median() median_int(lst, n) - - lst[n++] = 12; - test_eq(12, median()); /* 12 */ - lst[n++] = 77; - //smartlist_shuffle(sl); - test_eq(12, median()); /* 12, 77 */ - lst[n++] = 77; - //smartlist_shuffle(sl); - test_eq(77, median()); /* 12, 77, 77 */ - lst[n++] = 24; - test_eq(24, median()); /* 12,24,77,77 */ - lst[n++] = 60; - lst[n++] = 12; - lst[n++] = 25; - //smartlist_shuffle(sl); - test_eq(25, median()); /* 12,12,24,25,60,77,77 */ -#undef median - - done: - ; -} - -/** Helper: Make a new routerinfo containing the right information for a - * given vote_routerstatus_t. */ -static routerinfo_t * -generate_ri_from_rs(const vote_routerstatus_t *vrs) -{ - routerinfo_t *r; - const routerstatus_t *rs = &vrs->status; - static time_t published = 0; - - r = tor_malloc_zero(sizeof(routerinfo_t)); - memcpy(r->cache_info.identity_digest, rs->identity_digest, DIGEST_LEN); - memcpy(r->cache_info.signed_descriptor_digest, rs->descriptor_digest, - DIGEST_LEN); - r->cache_info.do_not_cache = 1; - r->cache_info.routerlist_index = -1; - r->cache_info.signed_descriptor_body = - tor_strdup("123456789012345678901234567890123"); - r->cache_info.signed_descriptor_len = - strlen(r->cache_info.signed_descriptor_body); - r->exit_policy = smartlist_create(); - r->cache_info.published_on = ++published + time(NULL); - return r; -} - -/** Run unit tests for generating and parsing V3 consensus networkstatus - * documents. */ -static void -test_v3_networkstatus(void) -{ - authority_cert_t *cert1=NULL, *cert2=NULL, *cert3=NULL; - crypto_pk_env_t *sign_skey_1=NULL, *sign_skey_2=NULL, *sign_skey_3=NULL; - crypto_pk_env_t *sign_skey_leg1=NULL; - const char *msg=NULL; - - time_t now = time(NULL); - networkstatus_voter_info_t *voter; - networkstatus_t *vote=NULL, *v1=NULL, *v2=NULL, *v3=NULL, *con=NULL; - vote_routerstatus_t *vrs; - routerstatus_t *rs; - char *v1_text=NULL, *v2_text=NULL, *v3_text=NULL, *consensus_text=NULL, *cp; - smartlist_t *votes = smartlist_create(); - - /* For generating the two other consensuses. */ - char *detached_text1=NULL, *detached_text2=NULL; - char *consensus_text2=NULL, *consensus_text3=NULL; - networkstatus_t *con2=NULL, *con3=NULL; - ns_detached_signatures_t *dsig1=NULL, *dsig2=NULL; - - /* Parse certificates and keys. */ - cert1 = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL); - test_assert(cert1); - test_assert(cert1->is_cross_certified); - cert2 = authority_cert_parse_from_string(AUTHORITY_CERT_2, NULL); - test_assert(cert2); - cert3 = authority_cert_parse_from_string(AUTHORITY_CERT_3, NULL); - test_assert(cert3); - sign_skey_1 = crypto_new_pk_env(); - sign_skey_2 = crypto_new_pk_env(); - sign_skey_3 = crypto_new_pk_env(); - sign_skey_leg1 = pk_generate(4); - - test_assert(!crypto_pk_read_private_key_from_string(sign_skey_1, - AUTHORITY_SIGNKEY_1,-1)); - test_assert(!crypto_pk_read_private_key_from_string(sign_skey_2, - AUTHORITY_SIGNKEY_2,-1)); - test_assert(!crypto_pk_read_private_key_from_string(sign_skey_3, - AUTHORITY_SIGNKEY_3,-1)); - - test_assert(!crypto_pk_cmp_keys(sign_skey_1, cert1->signing_key)); - test_assert(!crypto_pk_cmp_keys(sign_skey_2, cert2->signing_key)); - - /* - * Set up a vote; generate it; try to parse it. - */ - vote = tor_malloc_zero(sizeof(networkstatus_t)); - vote->type = NS_TYPE_VOTE; - vote->published = now; - vote->valid_after = now+1000; - vote->fresh_until = now+2000; - vote->valid_until = now+3000; - vote->vote_seconds = 100; - vote->dist_seconds = 200; - vote->supported_methods = smartlist_create(); - smartlist_split_string(vote->supported_methods, "1 2 3", NULL, 0, -1); - vote->client_versions = tor_strdup("0.1.2.14,0.1.2.15"); - vote->server_versions = tor_strdup("0.1.2.14,0.1.2.15,0.1.2.16"); - vote->known_flags = smartlist_create(); - smartlist_split_string(vote->known_flags, - "Authority Exit Fast Guard Running Stable V2Dir Valid", - 0, SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); - vote->voters = smartlist_create(); - voter = tor_malloc_zero(sizeof(networkstatus_voter_info_t)); - voter->nickname = tor_strdup("Voter1"); - voter->address = tor_strdup("1.2.3.4"); - voter->addr = 0x01020304; - voter->dir_port = 80; - voter->or_port = 9000; - voter->contact = tor_strdup("voter@example.com"); - crypto_pk_get_digest(cert1->identity_key, voter->identity_digest); - smartlist_add(vote->voters, voter); - vote->cert = authority_cert_dup(cert1); - vote->routerstatus_list = smartlist_create(); - /* add the first routerstatus. */ - vrs = tor_malloc_zero(sizeof(vote_routerstatus_t)); - rs = &vrs->status; - vrs->version = tor_strdup("0.1.2.14"); - rs->published_on = now-1500; - strlcpy(rs->nickname, "router2", sizeof(rs->nickname)); - memset(rs->identity_digest, 3, DIGEST_LEN); - memset(rs->descriptor_digest, 78, DIGEST_LEN); - rs->addr = 0x99008801; - rs->or_port = 443; - rs->dir_port = 8000; - /* all flags but running cleared */ - rs->is_running = 1; - smartlist_add(vote->routerstatus_list, vrs); - test_assert(router_add_to_routerlist(generate_ri_from_rs(vrs), &msg,0,0)>=0); - - /* add the second routerstatus. */ - vrs = tor_malloc_zero(sizeof(vote_routerstatus_t)); - rs = &vrs->status; - vrs->version = tor_strdup("0.2.0.5"); - rs->published_on = now-1000; - strlcpy(rs->nickname, "router1", sizeof(rs->nickname)); - memset(rs->identity_digest, 5, DIGEST_LEN); - memset(rs->descriptor_digest, 77, DIGEST_LEN); - rs->addr = 0x99009901; - rs->or_port = 443; - rs->dir_port = 0; - rs->is_exit = rs->is_stable = rs->is_fast = rs->is_running = - rs->is_valid = rs->is_v2_dir = rs->is_possible_guard = 1; - smartlist_add(vote->routerstatus_list, vrs); - test_assert(router_add_to_routerlist(generate_ri_from_rs(vrs), &msg,0,0)>=0); - - /* add the third routerstatus. */ - vrs = tor_malloc_zero(sizeof(vote_routerstatus_t)); - rs = &vrs->status; - vrs->version = tor_strdup("0.1.0.3"); - rs->published_on = now-1000; - strlcpy(rs->nickname, "router3", sizeof(rs->nickname)); - memset(rs->identity_digest, 33, DIGEST_LEN); - memset(rs->descriptor_digest, 79, DIGEST_LEN); - rs->addr = 0xAA009901; - rs->or_port = 400; - rs->dir_port = 9999; - rs->is_authority = rs->is_exit = rs->is_stable = rs->is_fast = - rs->is_running = rs->is_valid = rs->is_v2_dir = rs->is_possible_guard = 1; - smartlist_add(vote->routerstatus_list, vrs); - test_assert(router_add_to_routerlist(generate_ri_from_rs(vrs), &msg,0,0)>=0); - - /* add a fourth routerstatus that is not running. */ - vrs = tor_malloc_zero(sizeof(vote_routerstatus_t)); - rs = &vrs->status; - vrs->version = tor_strdup("0.1.6.3"); - rs->published_on = now-1000; - strlcpy(rs->nickname, "router4", sizeof(rs->nickname)); - memset(rs->identity_digest, 34, DIGEST_LEN); - memset(rs->descriptor_digest, 48, DIGEST_LEN); - rs->addr = 0xC0000203; - rs->or_port = 500; - rs->dir_port = 1999; - /* Running flag (and others) cleared */ - smartlist_add(vote->routerstatus_list, vrs); - test_assert(router_add_to_routerlist(generate_ri_from_rs(vrs), &msg,0,0)>=0); - - /* dump the vote and try to parse it. */ - v1_text = format_networkstatus_vote(sign_skey_1, vote); - test_assert(v1_text); - v1 = networkstatus_parse_vote_from_string(v1_text, NULL, NS_TYPE_VOTE); - test_assert(v1); - - /* Make sure the parsed thing was right. */ - test_eq(v1->type, NS_TYPE_VOTE); - test_eq(v1->published, vote->published); - test_eq(v1->valid_after, vote->valid_after); - test_eq(v1->fresh_until, vote->fresh_until); - test_eq(v1->valid_until, vote->valid_until); - test_eq(v1->vote_seconds, vote->vote_seconds); - test_eq(v1->dist_seconds, vote->dist_seconds); - test_streq(v1->client_versions, vote->client_versions); - test_streq(v1->server_versions, vote->server_versions); - test_assert(v1->voters && smartlist_len(v1->voters)); - voter = smartlist_get(v1->voters, 0); - test_streq(voter->nickname, "Voter1"); - test_streq(voter->address, "1.2.3.4"); - test_eq(voter->addr, 0x01020304); - test_eq(voter->dir_port, 80); - test_eq(voter->or_port, 9000); - test_streq(voter->contact, "voter@example.com"); - test_assert(v1->cert); - test_assert(!crypto_pk_cmp_keys(sign_skey_1, v1->cert->signing_key)); - cp = smartlist_join_strings(v1->known_flags, ":", 0, NULL); - test_streq(cp, "Authority:Exit:Fast:Guard:Running:Stable:V2Dir:Valid"); - tor_free(cp); - test_eq(smartlist_len(v1->routerstatus_list), 4); - /* Check the first routerstatus. */ - vrs = smartlist_get(v1->routerstatus_list, 0); - rs = &vrs->status; - test_streq(vrs->version, "0.1.2.14"); - test_eq(rs->published_on, now-1500); - test_streq(rs->nickname, "router2"); - test_memeq(rs->identity_digest, - "\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3", - DIGEST_LEN); - test_memeq(rs->descriptor_digest, "NNNNNNNNNNNNNNNNNNNN", DIGEST_LEN); - test_eq(rs->addr, 0x99008801); - test_eq(rs->or_port, 443); - test_eq(rs->dir_port, 8000); - test_eq(vrs->flags, U64_LITERAL(16)); // no flags except "running" - /* Check the second routerstatus. */ - vrs = smartlist_get(v1->routerstatus_list, 1); - rs = &vrs->status; - test_streq(vrs->version, "0.2.0.5"); - test_eq(rs->published_on, now-1000); - test_streq(rs->nickname, "router1"); - test_memeq(rs->identity_digest, - "\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5", - DIGEST_LEN); - test_memeq(rs->descriptor_digest, "MMMMMMMMMMMMMMMMMMMM", DIGEST_LEN); - test_eq(rs->addr, 0x99009901); - test_eq(rs->or_port, 443); - test_eq(rs->dir_port, 0); - test_eq(vrs->flags, U64_LITERAL(254)); // all flags except "authority." - - /* Generate second vote. It disagrees on some of the times, - * and doesn't list versions, and knows some crazy flags */ - vote->published = now+1; - vote->fresh_until = now+3005; - vote->dist_seconds = 300; - authority_cert_free(vote->cert); - vote->cert = authority_cert_dup(cert2); - tor_free(vote->client_versions); - tor_free(vote->server_versions); - voter = smartlist_get(vote->voters, 0); - tor_free(voter->nickname); - tor_free(voter->address); - voter->nickname = tor_strdup("Voter2"); - voter->address = tor_strdup("2.3.4.5"); - voter->addr = 0x02030405; - crypto_pk_get_digest(cert2->identity_key, voter->identity_digest); - smartlist_add(vote->known_flags, tor_strdup("MadeOfCheese")); - smartlist_add(vote->known_flags, tor_strdup("MadeOfTin")); - smartlist_sort_strings(vote->known_flags); - vrs = smartlist_get(vote->routerstatus_list, 2); - smartlist_del_keeporder(vote->routerstatus_list, 2); - tor_free(vrs->version); - tor_free(vrs); - vrs = smartlist_get(vote->routerstatus_list, 0); - vrs->status.is_fast = 1; - /* generate and parse. */ - v2_text = format_networkstatus_vote(sign_skey_2, vote); - test_assert(v2_text); - v2 = networkstatus_parse_vote_from_string(v2_text, NULL, NS_TYPE_VOTE); - test_assert(v2); - /* Check that flags come out right.*/ - cp = smartlist_join_strings(v2->known_flags, ":", 0, NULL); - test_streq(cp, "Authority:Exit:Fast:Guard:MadeOfCheese:MadeOfTin:" - "Running:Stable:V2Dir:Valid"); - tor_free(cp); - vrs = smartlist_get(v2->routerstatus_list, 1); - /* 1023 - authority(1) - madeofcheese(16) - madeoftin(32) */ - test_eq(vrs->flags, U64_LITERAL(974)); - - /* Generate the third vote. */ - vote->published = now; - vote->fresh_until = now+2003; - vote->dist_seconds = 250; - authority_cert_free(vote->cert); - vote->cert = authority_cert_dup(cert3); - smartlist_add(vote->supported_methods, tor_strdup("4")); - vote->client_versions = tor_strdup("0.1.2.14,0.1.2.17"); - vote->server_versions = tor_strdup("0.1.2.10,0.1.2.15,0.1.2.16"); - voter = smartlist_get(vote->voters, 0); - tor_free(voter->nickname); - tor_free(voter->address); - voter->nickname = tor_strdup("Voter3"); - voter->address = tor_strdup("3.4.5.6"); - voter->addr = 0x03040506; - crypto_pk_get_digest(cert3->identity_key, voter->identity_digest); - /* This one has a legacy id. */ - memset(voter->legacy_id_digest, (int)'A', DIGEST_LEN); - vrs = smartlist_get(vote->routerstatus_list, 0); - smartlist_del_keeporder(vote->routerstatus_list, 0); - tor_free(vrs->version); - tor_free(vrs); - vrs = smartlist_get(vote->routerstatus_list, 0); - memset(vrs->status.descriptor_digest, (int)'Z', DIGEST_LEN); - test_assert(router_add_to_routerlist(generate_ri_from_rs(vrs), &msg,0,0)>=0); - - v3_text = format_networkstatus_vote(sign_skey_3, vote); - test_assert(v3_text); - - v3 = networkstatus_parse_vote_from_string(v3_text, NULL, NS_TYPE_VOTE); - test_assert(v3); - - /* Compute a consensus as voter 3. */ - smartlist_add(votes, v3); - smartlist_add(votes, v1); - smartlist_add(votes, v2); - consensus_text = networkstatus_compute_consensus(votes, 3, - cert3->identity_key, - sign_skey_3, - "AAAAAAAAAAAAAAAAAAAA", - sign_skey_leg1); - test_assert(consensus_text); - con = networkstatus_parse_vote_from_string(consensus_text, NULL, - NS_TYPE_CONSENSUS); - test_assert(con); - //log_notice(LD_GENERAL, "<<%s>>\n<<%s>>\n<<%s>>\n", - // v1_text, v2_text, v3_text); - - /* Check consensus contents. */ - test_assert(con->type == NS_TYPE_CONSENSUS); - test_eq(con->published, 0); /* this field only appears in votes. */ - test_eq(con->valid_after, now+1000); - test_eq(con->fresh_until, now+2003); /* median */ - test_eq(con->valid_until, now+3000); - test_eq(con->vote_seconds, 100); - test_eq(con->dist_seconds, 250); /* median */ - test_streq(con->client_versions, "0.1.2.14"); - test_streq(con->server_versions, "0.1.2.15,0.1.2.16"); - cp = smartlist_join_strings(v2->known_flags, ":", 0, NULL); - test_streq(cp, "Authority:Exit:Fast:Guard:MadeOfCheese:MadeOfTin:" - "Running:Stable:V2Dir:Valid"); - tor_free(cp); - test_eq(4, smartlist_len(con->voters)); /*3 voters, 1 legacy key.*/ - /* The voter id digests should be in this order. */ - test_assert(memcmp(cert2->cache_info.identity_digest, - cert1->cache_info.identity_digest,DIGEST_LEN)<0); - test_assert(memcmp(cert1->cache_info.identity_digest, - cert3->cache_info.identity_digest,DIGEST_LEN)<0); - test_same_voter(smartlist_get(con->voters, 1), - smartlist_get(v2->voters, 0)); - test_same_voter(smartlist_get(con->voters, 2), - smartlist_get(v1->voters, 0)); - test_same_voter(smartlist_get(con->voters, 3), - smartlist_get(v3->voters, 0)); - - test_assert(!con->cert); - test_eq(2, smartlist_len(con->routerstatus_list)); - /* There should be two listed routers: one with identity 3, one with - * identity 5. */ - /* This one showed up in 2 digests. */ - rs = smartlist_get(con->routerstatus_list, 0); - test_memeq(rs->identity_digest, - "\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3", - DIGEST_LEN); - test_memeq(rs->descriptor_digest, "NNNNNNNNNNNNNNNNNNNN", DIGEST_LEN); - test_assert(!rs->is_authority); - test_assert(!rs->is_exit); - test_assert(!rs->is_fast); - test_assert(!rs->is_possible_guard); - test_assert(!rs->is_stable); - test_assert(rs->is_running); /* If it wasn't running it wouldn't be here */ - test_assert(!rs->is_v2_dir); - test_assert(!rs->is_valid); - test_assert(!rs->is_named); - /* XXXX check version */ - - rs = smartlist_get(con->routerstatus_list, 1); - /* This one showed up in 3 digests. Twice with ID 'M', once with 'Z'. */ - test_memeq(rs->identity_digest, - "\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5", - DIGEST_LEN); - test_streq(rs->nickname, "router1"); - test_memeq(rs->descriptor_digest, "MMMMMMMMMMMMMMMMMMMM", DIGEST_LEN); - test_eq(rs->published_on, now-1000); - test_eq(rs->addr, 0x99009901); - test_eq(rs->or_port, 443); - test_eq(rs->dir_port, 0); - test_assert(!rs->is_authority); - test_assert(rs->is_exit); - test_assert(rs->is_fast); - test_assert(rs->is_possible_guard); - test_assert(rs->is_stable); - test_assert(rs->is_running); - test_assert(rs->is_v2_dir); - test_assert(rs->is_valid); - test_assert(!rs->is_named); - /* XXXX check version */ - // x231 - // x213 - - /* Check signatures. the first voter is a pseudo-entry with a legacy key. - * The second one hasn't signed. The fourth one has signed: validate it. */ - voter = smartlist_get(con->voters, 1); - test_assert(!voter->signature); - test_assert(!voter->good_signature); - test_assert(!voter->bad_signature); - - voter = smartlist_get(con->voters, 3); - test_assert(voter->signature); - test_assert(!voter->good_signature); - test_assert(!voter->bad_signature); - test_assert(!networkstatus_check_voter_signature(con, - smartlist_get(con->voters, 3), - cert3)); - test_assert(voter->signature); - test_assert(voter->good_signature); - test_assert(!voter->bad_signature); - - { - const char *msg=NULL; - /* Compute the other two signed consensuses. */ - smartlist_shuffle(votes); - consensus_text2 = networkstatus_compute_consensus(votes, 3, - cert2->identity_key, - sign_skey_2, NULL,NULL); - smartlist_shuffle(votes); - consensus_text3 = networkstatus_compute_consensus(votes, 3, - cert1->identity_key, - sign_skey_1, NULL,NULL); - test_assert(consensus_text2); - test_assert(consensus_text3); - con2 = networkstatus_parse_vote_from_string(consensus_text2, NULL, - NS_TYPE_CONSENSUS); - con3 = networkstatus_parse_vote_from_string(consensus_text3, NULL, - NS_TYPE_CONSENSUS); - test_assert(con2); - test_assert(con3); - - /* All three should have the same digest. */ - test_memeq(con->networkstatus_digest, con2->networkstatus_digest, - DIGEST_LEN); - test_memeq(con->networkstatus_digest, con3->networkstatus_digest, - DIGEST_LEN); - - /* Extract a detached signature from con3. */ - detached_text1 = networkstatus_get_detached_signatures(con3); - tor_assert(detached_text1); - /* Try to parse it. */ - dsig1 = networkstatus_parse_detached_signatures(detached_text1, NULL); - tor_assert(dsig1); - - /* Are parsed values as expected? */ - test_eq(dsig1->valid_after, con3->valid_after); - test_eq(dsig1->fresh_until, con3->fresh_until); - test_eq(dsig1->valid_until, con3->valid_until); - test_memeq(dsig1->networkstatus_digest, con3->networkstatus_digest, - DIGEST_LEN); - test_eq(1, smartlist_len(dsig1->signatures)); - voter = smartlist_get(dsig1->signatures, 0); - test_memeq(voter->identity_digest, cert1->cache_info.identity_digest, - DIGEST_LEN); - - /* Try adding it to con2. */ - detached_text2 = networkstatus_get_detached_signatures(con2); - test_eq(1, networkstatus_add_detached_signatures(con2, dsig1, &msg)); - tor_free(detached_text2); - detached_text2 = networkstatus_get_detached_signatures(con2); - //printf("\n<%s>\n", detached_text2); - dsig2 = networkstatus_parse_detached_signatures(detached_text2, NULL); - test_assert(dsig2); - /* - printf("\n"); - SMARTLIST_FOREACH(dsig2->signatures, networkstatus_voter_info_t *, vi, { - char hd[64]; - base16_encode(hd, sizeof(hd), vi->identity_digest, DIGEST_LEN); - printf("%s\n", hd); - }); - */ - test_eq(2, smartlist_len(dsig2->signatures)); - - /* Try adding to con2 twice; verify that nothing changes. */ - test_eq(0, networkstatus_add_detached_signatures(con2, dsig1, &msg)); - - /* Add to con. */ - test_eq(2, networkstatus_add_detached_signatures(con, dsig2, &msg)); - /* Check signatures */ - test_assert(!networkstatus_check_voter_signature(con, - smartlist_get(con->voters, 1), - cert2)); - test_assert(!networkstatus_check_voter_signature(con, - smartlist_get(con->voters, 2), - cert1)); - - } - - done: - smartlist_free(votes); - tor_free(v1_text); - tor_free(v2_text); - tor_free(v3_text); - tor_free(consensus_text); - - if (vote) - networkstatus_vote_free(vote); - if (v1) - networkstatus_vote_free(v1); - if (v2) - networkstatus_vote_free(v2); - if (v3) - networkstatus_vote_free(v3); - if (con) - networkstatus_vote_free(con); - if (sign_skey_1) - crypto_free_pk_env(sign_skey_1); - if (sign_skey_2) - crypto_free_pk_env(sign_skey_2); - if (sign_skey_3) - crypto_free_pk_env(sign_skey_3); - if (sign_skey_leg1) - crypto_free_pk_env(sign_skey_leg1); - if (cert1) - authority_cert_free(cert1); - if (cert2) - authority_cert_free(cert2); - if (cert3) - authority_cert_free(cert3); - - tor_free(consensus_text2); - tor_free(consensus_text3); - tor_free(detached_text1); - tor_free(detached_text2); - if (con2) - networkstatus_vote_free(con2); - if (con3) - networkstatus_vote_free(con3); - if (dsig1) - ns_detached_signatures_free(dsig1); - if (dsig2) - ns_detached_signatures_free(dsig2); -} - -/** Helper: Parse the exit policy string in <b>policy_str</b>, and make sure - * that policies_summarize() produces the string <b>expected_summary</b> from - * it. */ -static void -test_policy_summary_helper(const char *policy_str, - const char *expected_summary) -{ - config_line_t line; - smartlist_t *policy = smartlist_create(); - char *summary = NULL; - int r; - - line.key = (char*)"foo"; - line.value = (char *)policy_str; - line.next = NULL; - - r = policies_parse_exit_policy(&line, &policy, 0, NULL); - test_eq(r, 0); - summary = policy_summarize(policy); - - test_assert(summary != NULL); - test_streq(summary, expected_summary); - - done: - tor_free(summary); - if (policy) - addr_policy_list_free(policy); -} - -/** Run unit tests for generating summary lines of exit policies */ -static void -test_policies(void) -{ - int i; - smartlist_t *policy = NULL, *policy2 = NULL; - addr_policy_t *p; - tor_addr_t tar; - config_line_t line; - smartlist_t *sm = NULL; - char *policy_str = NULL; - - policy = smartlist_create(); - - p = router_parse_addr_policy_item_from_string("reject 192.168.0.0/16:*",-1); - test_assert(p != NULL); - test_eq(ADDR_POLICY_REJECT, p->policy_type); - tor_addr_from_ipv4h(&tar, 0xc0a80000u); - test_eq(0, tor_addr_compare(&p->addr, &tar, CMP_EXACT)); - test_eq(16, p->maskbits); - test_eq(1, p->prt_min); - test_eq(65535, p->prt_max); - - smartlist_add(policy, p); - - test_assert(ADDR_POLICY_ACCEPTED == - compare_addr_to_addr_policy(0x01020304u, 2, policy)); - test_assert(ADDR_POLICY_PROBABLY_ACCEPTED == - compare_addr_to_addr_policy(0, 2, policy)); - test_assert(ADDR_POLICY_REJECTED == - compare_addr_to_addr_policy(0xc0a80102, 2, policy)); - - policy2 = NULL; - test_assert(0 == policies_parse_exit_policy(NULL, &policy2, 1, NULL)); - test_assert(policy2); - - test_assert(!exit_policy_is_general_exit(policy)); - test_assert(exit_policy_is_general_exit(policy2)); - test_assert(!exit_policy_is_general_exit(NULL)); - - test_assert(cmp_addr_policies(policy, policy2)); - test_assert(cmp_addr_policies(policy, NULL)); - test_assert(!cmp_addr_policies(policy2, policy2)); - test_assert(!cmp_addr_policies(NULL, NULL)); - - test_assert(!policy_is_reject_star(policy2)); - test_assert(policy_is_reject_star(policy)); - test_assert(policy_is_reject_star(NULL)); - - addr_policy_list_free(policy); - policy = NULL; - - /* make sure compacting logic works. */ - policy = NULL; - line.key = (char*)"foo"; - line.value = (char*)"accept *:80,reject private:*,reject *:*"; - line.next = NULL; - test_assert(0 == policies_parse_exit_policy(&line, &policy, 0, NULL)); - test_assert(policy); - //test_streq(policy->string, "accept *:80"); - //test_streq(policy->next->string, "reject *:*"); - test_eq(smartlist_len(policy), 2); - - /* test policy summaries */ - /* check if we properly ignore private IP addresses */ - test_policy_summary_helper("reject 192.168.0.0/16:*," - "reject 0.0.0.0/8:*," - "reject 10.0.0.0/8:*," - "accept *:10-30," - "accept *:90," - "reject *:*", - "accept 10-30,90"); - /* check all accept policies, and proper counting of rejects */ - test_policy_summary_helper("reject 11.0.0.0/9:80," - "reject 12.0.0.0/9:80," - "reject 13.0.0.0/9:80," - "reject 14.0.0.0/9:80," - "accept *:*", "accept 1-65535"); - test_policy_summary_helper("reject 11.0.0.0/9:80," - "reject 12.0.0.0/9:80," - "reject 13.0.0.0/9:80," - "reject 14.0.0.0/9:80," - "reject 15.0.0.0:81," - "accept *:*", "accept 1-65535"); - test_policy_summary_helper("reject 11.0.0.0/9:80," - "reject 12.0.0.0/9:80," - "reject 13.0.0.0/9:80," - "reject 14.0.0.0/9:80," - "reject 15.0.0.0:80," - "accept *:*", - "reject 80"); - /* no exits */ - test_policy_summary_helper("accept 11.0.0.0/9:80," - "reject *:*", - "reject 1-65535"); - /* port merging */ - test_policy_summary_helper("accept *:80," - "accept *:81," - "accept *:100-110," - "accept *:111," - "reject *:*", - "accept 80-81,100-111"); - /* border ports */ - test_policy_summary_helper("accept *:1," - "accept *:3," - "accept *:65535," - "reject *:*", - "accept 1,3,65535"); - /* holes */ - test_policy_summary_helper("accept *:1," - "accept *:3," - "accept *:5," - "accept *:7," - "reject *:*", - "accept 1,3,5,7"); - test_policy_summary_helper("reject *:1," - "reject *:3," - "reject *:5," - "reject *:7," - "accept *:*", - "reject 1,3,5,7"); - - /* truncation ports */ - sm = smartlist_create(); - for (i=1; i<2000; i+=2) { - char buf[POLICY_BUF_LEN]; - tor_snprintf(buf, sizeof(buf), "reject *:%d", i); - smartlist_add(sm, tor_strdup(buf)); - } - smartlist_add(sm, tor_strdup("accept *:*")); - policy_str = smartlist_join_strings(sm, ",", 0, NULL); - test_policy_summary_helper( policy_str, - "accept 2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40,42,44," - "46,48,50,52,54,56,58,60,62,64,66,68,70,72,74,76,78,80,82,84,86,88,90," - "92,94,96,98,100,102,104,106,108,110,112,114,116,118,120,122,124,126,128," - "130,132,134,136,138,140,142,144,146,148,150,152,154,156,158,160,162,164," - "166,168,170,172,174,176,178,180,182,184,186,188,190,192,194,196,198,200," - "202,204,206,208,210,212,214,216,218,220,222,224,226,228,230,232,234,236," - "238,240,242,244,246,248,250,252,254,256,258,260,262,264,266,268,270,272," - "274,276,278,280,282,284,286,288,290,292,294,296,298,300,302,304,306,308," - "310,312,314,316,318,320,322,324,326,328,330,332,334,336,338,340,342,344," - "346,348,350,352,354,356,358,360,362,364,366,368,370,372,374,376,378,380," - "382,384,386,388,390,392,394,396,398,400,402,404,406,408,410,412,414,416," - "418,420,422,424,426,428,430,432,434,436,438,440,442,444,446,448,450,452," - "454,456,458,460,462,464,466,468,470,472,474,476,478,480,482,484,486,488," - "490,492,494,496,498,500,502,504,506,508,510,512,514,516,518,520,522"); - - done: - if (policy) - addr_policy_list_free(policy); - if (policy2) - addr_policy_list_free(policy2); - tor_free(policy_str); - if (sm) { - SMARTLIST_FOREACH(sm, char *, s, tor_free(s)); - smartlist_free(sm); - } -} - -/** Run unit tests for basic rendezvous functions. */ -static void -test_rend_fns(void) -{ - char address1[] = "fooaddress.onion"; - char address2[] = "aaaaaaaaaaaaaaaa.onion"; - char address3[] = "fooaddress.exit"; - char address4[] = "www.torproject.org"; - rend_service_descriptor_t *d1 = - tor_malloc_zero(sizeof(rend_service_descriptor_t)); - rend_service_descriptor_t *d2 = NULL; - char *encoded = NULL; - size_t len; - time_t now; - int i; - crypto_pk_env_t *pk1 = pk_generate(0), *pk2 = pk_generate(1); - - /* Test unversioned (v0) descriptor */ - d1->pk = crypto_pk_dup_key(pk1); - now = time(NULL); - d1->timestamp = now; - d1->version = 0; - d1->intro_nodes = smartlist_create(); - for (i = 0; i < 3; i++) { - rend_intro_point_t *intro = tor_malloc_zero(sizeof(rend_intro_point_t)); - intro->extend_info = tor_malloc_zero(sizeof(extend_info_t)); - crypto_rand(intro->extend_info->identity_digest, DIGEST_LEN); - intro->extend_info->nickname[0] = '$'; - base16_encode(intro->extend_info->nickname+1, HEX_DIGEST_LEN+1, - intro->extend_info->identity_digest, DIGEST_LEN); - smartlist_add(d1->intro_nodes, intro); - } - test_assert(! rend_encode_service_descriptor(d1, pk1, &encoded, &len)); - d2 = rend_parse_service_descriptor(encoded, len); - test_assert(d2); - - test_assert(!crypto_pk_cmp_keys(d1->pk, d2->pk)); - test_eq(d2->timestamp, now); - test_eq(d2->version, 0); - test_eq(d2->protocols, 1<<2); - test_eq(smartlist_len(d2->intro_nodes), 3); - for (i = 0; i < 3; i++) { - rend_intro_point_t *intro1 = smartlist_get(d1->intro_nodes, i); - rend_intro_point_t *intro2 = smartlist_get(d2->intro_nodes, i); - test_streq(intro1->extend_info->nickname, - intro2->extend_info->nickname); - } - - test_assert(BAD_HOSTNAME == parse_extended_hostname(address1)); - test_assert(ONION_HOSTNAME == parse_extended_hostname(address2)); - test_assert(EXIT_HOSTNAME == parse_extended_hostname(address3)); - test_assert(NORMAL_HOSTNAME == parse_extended_hostname(address4)); - - crypto_free_pk_env(pk1); - crypto_free_pk_env(pk2); - pk1 = pk2 = NULL; - rend_service_descriptor_free(d1); - rend_service_descriptor_free(d2); - d1 = d2 = NULL; - - done: - if (pk1) - crypto_free_pk_env(pk1); - if (pk2) - crypto_free_pk_env(pk2); - if (d1) - rend_service_descriptor_free(d1); - if (d2) - rend_service_descriptor_free(d2); - tor_free(encoded); -} - -/** Run AES performance benchmarks. */ -static void -bench_aes(void) -{ - int len, i; - char *b1, *b2; - crypto_cipher_env_t *c; - struct timeval start, end; - const int iters = 100000; - uint64_t nsec; - c = crypto_new_cipher_env(); - crypto_cipher_generate_key(c); - crypto_cipher_encrypt_init_cipher(c); - for (len = 1; len <= 8192; len *= 2) { - b1 = tor_malloc_zero(len); - b2 = tor_malloc_zero(len); - tor_gettimeofday(&start); - for (i = 0; i < iters; ++i) { - crypto_cipher_encrypt(c, b1, b2, len); - } - tor_gettimeofday(&end); - tor_free(b1); - tor_free(b2); - nsec = (uint64_t) tv_udiff(&start,&end); - nsec *= 1000; - nsec /= (iters*len); - printf("%d bytes: "U64_FORMAT" nsec per byte\n", len, - U64_PRINTF_ARG(nsec)); - } - crypto_free_cipher_env(c); -} - -/** Run digestmap_t performance benchmarks. */ -static void -bench_dmap(void) -{ - smartlist_t *sl = smartlist_create(); - smartlist_t *sl2 = smartlist_create(); - struct timeval start, end, pt2, pt3, pt4; - const int iters = 10000; - const int elts = 4000; - const int fpostests = 1000000; - char d[20]; - int i,n=0, fp = 0; - digestmap_t *dm = digestmap_new(); - digestset_t *ds = digestset_new(elts); - - for (i = 0; i < elts; ++i) { - crypto_rand(d, 20); - smartlist_add(sl, tor_memdup(d, 20)); - } - for (i = 0; i < elts; ++i) { - crypto_rand(d, 20); - smartlist_add(sl2, tor_memdup(d, 20)); - } - printf("nbits=%d\n", ds->mask+1); - - tor_gettimeofday(&start); - for (i = 0; i < iters; ++i) { - SMARTLIST_FOREACH(sl, const char *, cp, digestmap_set(dm, cp, (void*)1)); - } - tor_gettimeofday(&pt2); - for (i = 0; i < iters; ++i) { - SMARTLIST_FOREACH(sl, const char *, cp, digestmap_get(dm, cp)); - SMARTLIST_FOREACH(sl2, const char *, cp, digestmap_get(dm, cp)); - } - tor_gettimeofday(&pt3); - for (i = 0; i < iters; ++i) { - SMARTLIST_FOREACH(sl, const char *, cp, digestset_add(ds, cp)); - } - tor_gettimeofday(&pt4); - for (i = 0; i < iters; ++i) { - SMARTLIST_FOREACH(sl, const char *, cp, n += digestset_isin(ds, cp)); - SMARTLIST_FOREACH(sl2, const char *, cp, n += digestset_isin(ds, cp)); - } - tor_gettimeofday(&end); - - for (i = 0; i < fpostests; ++i) { - crypto_rand(d, 20); - if (digestset_isin(ds, d)) ++fp; - } - - printf("%ld\n",(unsigned long)tv_udiff(&start, &pt2)); - printf("%ld\n",(unsigned long)tv_udiff(&pt2, &pt3)); - printf("%ld\n",(unsigned long)tv_udiff(&pt3, &pt4)); - printf("%ld\n",(unsigned long)tv_udiff(&pt4, &end)); - printf("-- %d\n", n); - printf("++ %f\n", fp/(double)fpostests); - digestmap_free(dm, NULL); - digestset_free(ds); - SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); - SMARTLIST_FOREACH(sl2, char *, cp, tor_free(cp)); - smartlist_free(sl); - smartlist_free(sl2); -} - -/** Run unittests for memory pool allocator */ -static void -test_util_mempool(void) -{ - mp_pool_t *pool = NULL; - smartlist_t *allocated = NULL; - int i; - - pool = mp_pool_new(1, 100); - test_assert(pool); - test_assert(pool->new_chunk_capacity >= 100); - test_assert(pool->item_alloc_size >= sizeof(void*)+1); - mp_pool_destroy(pool); - pool = NULL; - - pool = mp_pool_new(241, 2500); - test_assert(pool); - test_assert(pool->new_chunk_capacity >= 10); - test_assert(pool->item_alloc_size >= sizeof(void*)+241); - test_eq(pool->item_alloc_size & 0x03, 0); - test_assert(pool->new_chunk_capacity < 60); - - allocated = smartlist_create(); - for (i = 0; i < 20000; ++i) { - if (smartlist_len(allocated) < 20 || crypto_rand_int(2)) { - void *m = mp_pool_get(pool); - memset(m, 0x09, 241); - smartlist_add(allocated, m); - //printf("%d: %p\n", i, m); - //mp_pool_assert_ok(pool); - } else { - int idx = crypto_rand_int(smartlist_len(allocated)); - void *m = smartlist_get(allocated, idx); - //printf("%d: free %p\n", i, m); - smartlist_del(allocated, idx); - mp_pool_release(m); - //mp_pool_assert_ok(pool); - } - if (crypto_rand_int(777)==0) - mp_pool_clean(pool, 1, 1); - - if (i % 777) - mp_pool_assert_ok(pool); - } - - done: - if (allocated) { - SMARTLIST_FOREACH(allocated, void *, m, mp_pool_release(m)); - mp_pool_assert_ok(pool); - mp_pool_clean(pool, 0, 0); - mp_pool_assert_ok(pool); - smartlist_free(allocated); - } - - if (pool) - mp_pool_destroy(pool); -} - -/** Run unittests for memory area allocator */ -static void -test_util_memarea(void) -{ - memarea_t *area = memarea_new(); - char *p1, *p2, *p3, *p1_orig; - void *malloced_ptr = NULL; - int i; - - test_assert(area); - - p1_orig = p1 = memarea_alloc(area,64); - p2 = memarea_alloc_zero(area,52); - p3 = memarea_alloc(area,11); - - test_assert(memarea_owns_ptr(area, p1)); - test_assert(memarea_owns_ptr(area, p2)); - test_assert(memarea_owns_ptr(area, p3)); - /* Make sure we left enough space. */ - test_assert(p1+64 <= p2); - test_assert(p2+52 <= p3); - /* Make sure we aligned. */ - test_eq(((uintptr_t)p1) % sizeof(void*), 0); - test_eq(((uintptr_t)p2) % sizeof(void*), 0); - test_eq(((uintptr_t)p3) % sizeof(void*), 0); - test_assert(!memarea_owns_ptr(area, p3+8192)); - test_assert(!memarea_owns_ptr(area, p3+30)); - test_assert(tor_mem_is_zero(p2, 52)); - /* Make sure we don't overalign. */ - p1 = memarea_alloc(area, 1); - p2 = memarea_alloc(area, 1); - test_eq(p1+sizeof(void*), p2); - { - malloced_ptr = tor_malloc(64); - test_assert(!memarea_owns_ptr(area, malloced_ptr)); - tor_free(malloced_ptr); - } - - /* memarea_memdup */ - { - malloced_ptr = tor_malloc(64); - crypto_rand((char*)malloced_ptr, 64); - p1 = memarea_memdup(area, malloced_ptr, 64); - test_assert(p1 != malloced_ptr); - test_memeq(p1, malloced_ptr, 64); - tor_free(malloced_ptr); - } - - /* memarea_strdup. */ - p1 = memarea_strdup(area,""); - p2 = memarea_strdup(area, "abcd"); - test_assert(p1); - test_assert(p2); - test_streq(p1, ""); - test_streq(p2, "abcd"); - - /* memarea_strndup. */ - { - const char *s = "Ad ogni porta batte la morte e grida: il nome!"; - /* (From Turandot, act 3.) */ - size_t len = strlen(s); - p1 = memarea_strndup(area, s, 1000); - p2 = memarea_strndup(area, s, 10); - test_streq(p1, s); - test_assert(p2 >= p1 + len + 1); - test_memeq(s, p2, 10); - test_eq(p2[10], '\0'); - p3 = memarea_strndup(area, s, len); - test_streq(p3, s); - p3 = memarea_strndup(area, s, len-1); - test_memeq(s, p3, len-1); - test_eq(p3[len-1], '\0'); - } - - memarea_clear(area); - p1 = memarea_alloc(area, 1); - test_eq(p1, p1_orig); - memarea_clear(area); - - /* Check for running over an area's size. */ - for (i = 0; i < 512; ++i) { - p1 = memarea_alloc(area, crypto_rand_int(5)+1); - test_assert(memarea_owns_ptr(area, p1)); - } - memarea_assert_ok(area); - /* Make sure we can allocate a too-big object. */ - p1 = memarea_alloc_zero(area, 9000); - p2 = memarea_alloc_zero(area, 16); - test_assert(memarea_owns_ptr(area, p1)); - test_assert(memarea_owns_ptr(area, p2)); - - done: - memarea_drop_all(area); - tor_free(malloced_ptr); -} - -/** Run unit tests for utility functions to get file names relative to - * the data directory. */ -static void -test_util_datadir(void) -{ - char buf[1024]; - char *f = NULL; - - f = get_datadir_fname(NULL); - test_streq(f, temp_dir); - tor_free(f); - f = get_datadir_fname("state"); - tor_snprintf(buf, sizeof(buf), "%s"PATH_SEPARATOR"state", temp_dir); - test_streq(f, buf); - tor_free(f); - f = get_datadir_fname2("cache", "thingy"); - tor_snprintf(buf, sizeof(buf), - "%s"PATH_SEPARATOR"cache"PATH_SEPARATOR"thingy", temp_dir); - test_streq(f, buf); - tor_free(f); - f = get_datadir_fname2_suffix("cache", "thingy", ".foo"); - tor_snprintf(buf, sizeof(buf), - "%s"PATH_SEPARATOR"cache"PATH_SEPARATOR"thingy.foo", temp_dir); - test_streq(f, buf); - tor_free(f); - f = get_datadir_fname_suffix("cache", ".foo"); - tor_snprintf(buf, sizeof(buf), "%s"PATH_SEPARATOR"cache.foo", - temp_dir); - test_streq(f, buf); - - done: - tor_free(f); -} - -/** Test AES-CTR encryption and decryption with IV. */ -static void -test_crypto_aes_iv(void) -{ - crypto_cipher_env_t *cipher; - char *plain, *encrypted1, *encrypted2, *decrypted1, *decrypted2; - char plain_1[1], plain_15[15], plain_16[16], plain_17[17]; - char key1[16], key2[16]; - ssize_t encrypted_size, decrypted_size; - - plain = tor_malloc(4095); - encrypted1 = tor_malloc(4095 + 1 + 16); - encrypted2 = tor_malloc(4095 + 1 + 16); - decrypted1 = tor_malloc(4095 + 1); - decrypted2 = tor_malloc(4095 + 1); - - crypto_rand(plain, 4095); - crypto_rand(key1, 16); - crypto_rand(key2, 16); - crypto_rand(plain_1, 1); - crypto_rand(plain_15, 15); - crypto_rand(plain_16, 16); - crypto_rand(plain_17, 17); - key1[0] = key2[0] + 128; /* Make sure that contents are different. */ - /* Encrypt and decrypt with the same key. */ - cipher = crypto_create_init_cipher(key1, 1); - encrypted_size = crypto_cipher_encrypt_with_iv(cipher, encrypted1, 16 + 4095, - plain, 4095); - crypto_free_cipher_env(cipher); - cipher = NULL; - test_eq(encrypted_size, 16 + 4095); - tor_assert(encrypted_size > 0); /* This is obviously true, since 4111 is - * greater than 0, but its truth is not - * obvious to all analysis tools. */ - cipher = crypto_create_init_cipher(key1, 0); - decrypted_size = crypto_cipher_decrypt_with_iv(cipher, decrypted1, 4095, - encrypted1, encrypted_size); - crypto_free_cipher_env(cipher); - cipher = NULL; - test_eq(decrypted_size, 4095); - tor_assert(decrypted_size > 0); - test_memeq(plain, decrypted1, 4095); - /* Encrypt a second time (with a new random initialization vector). */ - cipher = crypto_create_init_cipher(key1, 1); - encrypted_size = crypto_cipher_encrypt_with_iv(cipher, encrypted2, 16 + 4095, - plain, 4095); - crypto_free_cipher_env(cipher); - cipher = NULL; - test_eq(encrypted_size, 16 + 4095); - tor_assert(encrypted_size > 0); - cipher = crypto_create_init_cipher(key1, 0); - decrypted_size = crypto_cipher_decrypt_with_iv(cipher, decrypted2, 4095, - encrypted2, encrypted_size); - crypto_free_cipher_env(cipher); - cipher = NULL; - test_eq(decrypted_size, 4095); - tor_assert(decrypted_size > 0); - test_memeq(plain, decrypted2, 4095); - test_memneq(encrypted1, encrypted2, encrypted_size); - /* Decrypt with the wrong key. */ - cipher = crypto_create_init_cipher(key2, 0); - decrypted_size = crypto_cipher_decrypt_with_iv(cipher, decrypted2, 4095, - encrypted1, encrypted_size); - crypto_free_cipher_env(cipher); - cipher = NULL; - test_memneq(plain, decrypted2, encrypted_size); - /* Alter the initialization vector. */ - encrypted1[0] += 42; - cipher = crypto_create_init_cipher(key1, 0); - decrypted_size = crypto_cipher_decrypt_with_iv(cipher, decrypted1, 4095, - encrypted1, encrypted_size); - crypto_free_cipher_env(cipher); - cipher = NULL; - test_memneq(plain, decrypted2, 4095); - /* Special length case: 1. */ - cipher = crypto_create_init_cipher(key1, 1); - encrypted_size = crypto_cipher_encrypt_with_iv(cipher, encrypted1, 16 + 1, - plain_1, 1); - crypto_free_cipher_env(cipher); - cipher = NULL; - test_eq(encrypted_size, 16 + 1); - tor_assert(encrypted_size > 0); - cipher = crypto_create_init_cipher(key1, 0); - decrypted_size = crypto_cipher_decrypt_with_iv(cipher, decrypted1, 1, - encrypted1, encrypted_size); - crypto_free_cipher_env(cipher); - cipher = NULL; - test_eq(decrypted_size, 1); - tor_assert(decrypted_size > 0); - test_memeq(plain_1, decrypted1, 1); - /* Special length case: 15. */ - cipher = crypto_create_init_cipher(key1, 1); - encrypted_size = crypto_cipher_encrypt_with_iv(cipher, encrypted1, 16 + 15, - plain_15, 15); - crypto_free_cipher_env(cipher); - cipher = NULL; - test_eq(encrypted_size, 16 + 15); - tor_assert(encrypted_size > 0); - cipher = crypto_create_init_cipher(key1, 0); - decrypted_size = crypto_cipher_decrypt_with_iv(cipher, decrypted1, 15, - encrypted1, encrypted_size); - crypto_free_cipher_env(cipher); - cipher = NULL; - test_eq(decrypted_size, 15); - tor_assert(decrypted_size > 0); - test_memeq(plain_15, decrypted1, 15); - /* Special length case: 16. */ - cipher = crypto_create_init_cipher(key1, 1); - encrypted_size = crypto_cipher_encrypt_with_iv(cipher, encrypted1, 16 + 16, - plain_16, 16); - crypto_free_cipher_env(cipher); - cipher = NULL; - test_eq(encrypted_size, 16 + 16); - tor_assert(encrypted_size > 0); - cipher = crypto_create_init_cipher(key1, 0); - decrypted_size = crypto_cipher_decrypt_with_iv(cipher, decrypted1, 16, - encrypted1, encrypted_size); - crypto_free_cipher_env(cipher); - cipher = NULL; - test_eq(decrypted_size, 16); - tor_assert(decrypted_size > 0); - test_memeq(plain_16, decrypted1, 16); - /* Special length case: 17. */ - cipher = crypto_create_init_cipher(key1, 1); - encrypted_size = crypto_cipher_encrypt_with_iv(cipher, encrypted1, 16 + 17, - plain_17, 17); - crypto_free_cipher_env(cipher); - cipher = NULL; - test_eq(encrypted_size, 16 + 17); - tor_assert(encrypted_size > 0); - cipher = crypto_create_init_cipher(key1, 0); - decrypted_size = crypto_cipher_decrypt_with_iv(cipher, decrypted1, 17, - encrypted1, encrypted_size); - test_eq(decrypted_size, 17); - tor_assert(decrypted_size > 0); - test_memeq(plain_17, decrypted1, 17); - - done: - /* Free memory. */ - tor_free(plain); - tor_free(encrypted1); - tor_free(encrypted2); - tor_free(decrypted1); - tor_free(decrypted2); - if (cipher) - crypto_free_cipher_env(cipher); -} - -/** Test base32 decoding. */ -static void -test_crypto_base32_decode(void) -{ - char plain[60], encoded[96 + 1], decoded[60]; - int res; - crypto_rand(plain, 60); - /* Encode and decode a random string. */ - base32_encode(encoded, 96 + 1, plain, 60); - res = base32_decode(decoded, 60, encoded, 96); - test_eq(res, 0); - test_memeq(plain, decoded, 60); - /* Encode, uppercase, and decode a random string. */ - base32_encode(encoded, 96 + 1, plain, 60); - tor_strupper(encoded); - res = base32_decode(decoded, 60, encoded, 96); - test_eq(res, 0); - test_memeq(plain, decoded, 60); - /* Change encoded string and decode. */ - if (encoded[0] == 'A' || encoded[0] == 'a') - encoded[0] = 'B'; - else - encoded[0] = 'A'; - res = base32_decode(decoded, 60, encoded, 96); - test_eq(res, 0); - test_memneq(plain, decoded, 60); - /* Bad encodings. */ - encoded[0] = '!'; - res = base32_decode(decoded, 60, encoded, 96); - test_assert(res < 0); - - done: - ; -} - -/** Test encoding and parsing of v2 rendezvous service descriptors. */ -static void -test_rend_fns_v2(void) -{ - rend_service_descriptor_t *generated = NULL, *parsed = NULL; - char service_id[DIGEST_LEN]; - char service_id_base32[REND_SERVICE_ID_LEN_BASE32+1]; - const char *next_desc; - smartlist_t *descs = smartlist_create(); - char computed_desc_id[DIGEST_LEN]; - char parsed_desc_id[DIGEST_LEN]; - crypto_pk_env_t *pk1 = NULL, *pk2 = NULL; - time_t now; - char *intro_points_encrypted = NULL; - size_t intro_points_size; - size_t encoded_size; - int i; - pk1 = pk_generate(0); - pk2 = pk_generate(1); - generated = tor_malloc_zero(sizeof(rend_service_descriptor_t)); - generated->pk = crypto_pk_dup_key(pk1); - crypto_pk_get_digest(generated->pk, service_id); - base32_encode(service_id_base32, REND_SERVICE_ID_LEN_BASE32+1, - service_id, REND_SERVICE_ID_LEN); - now = time(NULL); - generated->timestamp = now; - generated->version = 2; - generated->protocols = 42; - generated->intro_nodes = smartlist_create(); - - for (i = 0; i < 3; i++) { - rend_intro_point_t *intro = tor_malloc_zero(sizeof(rend_intro_point_t)); - crypto_pk_env_t *okey = pk_generate(2 + i); - intro->extend_info = tor_malloc_zero(sizeof(extend_info_t)); - intro->extend_info->onion_key = okey; - crypto_pk_get_digest(intro->extend_info->onion_key, - intro->extend_info->identity_digest); - //crypto_rand(info->identity_digest, DIGEST_LEN); /* Would this work? */ - intro->extend_info->nickname[0] = '$'; - base16_encode(intro->extend_info->nickname + 1, - sizeof(intro->extend_info->nickname) - 1, - intro->extend_info->identity_digest, DIGEST_LEN); - /* Does not cover all IP addresses. */ - tor_addr_from_ipv4h(&intro->extend_info->addr, crypto_rand_int(65536)); - intro->extend_info->port = crypto_rand_int(65536); - intro->intro_key = crypto_pk_dup_key(pk2); - smartlist_add(generated->intro_nodes, intro); - } - test_assert(rend_encode_v2_descriptors(descs, generated, now, 0, - REND_NO_AUTH, NULL, NULL) > 0); - test_assert(rend_compute_v2_desc_id(computed_desc_id, service_id_base32, - NULL, now, 0) == 0); - test_memeq(((rend_encoded_v2_service_descriptor_t *) - smartlist_get(descs, 0))->desc_id, computed_desc_id, DIGEST_LEN); - test_assert(rend_parse_v2_service_descriptor(&parsed, parsed_desc_id, - &intro_points_encrypted, - &intro_points_size, - &encoded_size, - &next_desc, - ((rend_encoded_v2_service_descriptor_t *) - smartlist_get(descs, 0))->desc_str) == 0); - test_assert(parsed); - test_memeq(((rend_encoded_v2_service_descriptor_t *) - smartlist_get(descs, 0))->desc_id, parsed_desc_id, DIGEST_LEN); - test_eq(rend_parse_introduction_points(parsed, intro_points_encrypted, - intro_points_size), 3); - test_assert(!crypto_pk_cmp_keys(generated->pk, parsed->pk)); - test_eq(parsed->timestamp, now); - test_eq(parsed->version, 2); - test_eq(parsed->protocols, 42); - test_eq(smartlist_len(parsed->intro_nodes), 3); - for (i = 0; i < smartlist_len(parsed->intro_nodes); i++) { - rend_intro_point_t *par_intro = smartlist_get(parsed->intro_nodes, i), - *gen_intro = smartlist_get(generated->intro_nodes, i); - extend_info_t *par_info = par_intro->extend_info; - extend_info_t *gen_info = gen_intro->extend_info; - test_assert(!crypto_pk_cmp_keys(gen_info->onion_key, par_info->onion_key)); - test_memeq(gen_info->identity_digest, par_info->identity_digest, - DIGEST_LEN); - test_streq(gen_info->nickname, par_info->nickname); - test_assert(tor_addr_eq(&gen_info->addr, &par_info->addr)); - test_eq(gen_info->port, par_info->port); - } - - rend_service_descriptor_free(parsed); - rend_service_descriptor_free(generated); - parsed = generated = NULL; - - done: - if (descs) { - for (i = 0; i < smartlist_len(descs); i++) - rend_encoded_v2_service_descriptor_free(smartlist_get(descs, i)); - smartlist_free(descs); - } - if (parsed) - rend_service_descriptor_free(parsed); - if (generated) - rend_service_descriptor_free(generated); - if (pk1) - crypto_free_pk_env(pk1); - if (pk2) - crypto_free_pk_env(pk2); - tor_free(intro_points_encrypted); -} - -/** Run unit tests for GeoIP code. */ -static void -test_geoip(void) -{ - int i, j; - time_t now = time(NULL); - char *s = NULL; - - /* Populate the DB a bit. Add these in order, since we can't do the final - * 'sort' step. These aren't very good IP addresses, but they're perfectly - * fine uint32_t values. */ - test_eq(0, geoip_parse_entry("10,50,AB")); - test_eq(0, geoip_parse_entry("52,90,XY")); - test_eq(0, geoip_parse_entry("95,100,AB")); - test_eq(0, geoip_parse_entry("\"105\",\"140\",\"ZZ\"")); - test_eq(0, geoip_parse_entry("\"150\",\"190\",\"XY\"")); - test_eq(0, geoip_parse_entry("\"200\",\"250\",\"AB\"")); - - /* We should have 3 countries: ab, xy, zz. */ - test_eq(3, geoip_get_n_countries()); - /* Make sure that country ID actually works. */ -#define NAMEFOR(x) geoip_get_country_name(geoip_get_country_by_ip(x)) - test_streq("ab", NAMEFOR(32)); - test_streq("??", NAMEFOR(5)); - test_streq("??", NAMEFOR(51)); - test_streq("xy", NAMEFOR(150)); - test_streq("xy", NAMEFOR(190)); - test_streq("??", NAMEFOR(2000)); -#undef NAMEFOR - - get_options()->BridgeRelay = 1; - get_options()->BridgeRecordUsageByCountry = 1; - /* Put 9 observations in AB... */ - for (i=32; i < 40; ++i) - geoip_note_client_seen(GEOIP_CLIENT_CONNECT, i, now-7200); - geoip_note_client_seen(GEOIP_CLIENT_CONNECT, 225, now-7200); - /* and 3 observations in XY, several times. */ - for (j=0; j < 10; ++j) - for (i=52; i < 55; ++i) - geoip_note_client_seen(GEOIP_CLIENT_CONNECT, i, now-3600); - /* and 17 observations in ZZ... */ - for (i=110; i < 127; ++i) - geoip_note_client_seen(GEOIP_CLIENT_CONNECT, i, now); - s = geoip_get_client_history(now+5*24*60*60, GEOIP_CLIENT_CONNECT); - test_assert(s); - test_streq("zz=24,ab=16,xy=8", s); - tor_free(s); - - /* Now clear out all the AB observations. */ - geoip_remove_old_clients(now-6000); - s = geoip_get_client_history(now+5*24*60*60, GEOIP_CLIENT_CONNECT); - test_assert(s); - test_streq("zz=24,xy=8", s); - - done: - tor_free(s); -} - -/** For test_array. Declare an CLI-invocable off-by-default function in the - * unit tests, with function name and user-visible name <b>x</b>*/ -#define DISABLED(x) { #x, x, 0, 0, 0 } -/** For test_array. Declare an CLI-invocable unit test function, with function - * name test_<b>x</b>(), and user-visible name <b>x</b> */ -#define ENT(x) { #x, test_ ## x, 0, 0, 1 } -/** For test_array. Declare an CLI-invocable unit test function, with function - * name test_<b>x</b>_<b>y</b>(), and user-visible name - * <b>x</b>/<b>y</b>. This function will be treated as a subentry of <b>x</b>, - * so that invoking <b>x</b> from the CLI invokes this test too. */ -#define SUBENT(x,y) { #x "/" #y, test_ ## x ## _ ## y, 1, 0, 1 } - -/** An array of functions and information for all the unit tests we can run. */ -static struct { - const char *test_name; /**< How does the user refer to this test from the - * command line? */ - void (*test_fn)(void); /**< What function is called to run this test? */ - int is_subent; /**< Is this a subentry of a bigger set of related tests? */ - int selected; /**< Are we planning to run this one? */ - int is_default; /**< If the user doesn't say what tests they want, do they - * get this function by default? */ -} test_array[] = { - ENT(buffers), - ENT(crypto), - SUBENT(crypto, rng), - SUBENT(crypto, aes), - SUBENT(crypto, sha), - SUBENT(crypto, pk), - SUBENT(crypto, dh), - SUBENT(crypto, s2k), - SUBENT(crypto, aes_iv), - SUBENT(crypto, base32_decode), - ENT(util), - SUBENT(util, ip6_helpers), - SUBENT(util, gzip), - SUBENT(util, datadir), - SUBENT(util, smartlist_basic), - SUBENT(util, smartlist_strings), - SUBENT(util, smartlist_overlap), - SUBENT(util, smartlist_digests), - SUBENT(util, smartlist_join), - SUBENT(util, bitarray), - SUBENT(util, digestset), - SUBENT(util, mempool), - SUBENT(util, memarea), - SUBENT(util, strmap), - SUBENT(util, control_formats), - SUBENT(util, pqueue), - SUBENT(util, mmap), - SUBENT(util, threads), - SUBENT(util, order_functions), - SUBENT(util, sscanf), - ENT(onion_handshake), - ENT(dir_format), - ENT(dirutil), - ENT(v3_networkstatus), - ENT(policies), - ENT(rend_fns), - SUBENT(rend_fns, v2), - ENT(geoip), - - DISABLED(bench_aes), - DISABLED(bench_dmap), - { NULL, NULL, 0, 0, 0 }, -}; - -static void syntax(void) ATTR_NORETURN; - -/** Print a syntax usage message, and exit.*/ -static void -syntax(void) -{ - int i; - printf("Syntax:\n" - " test [-v|--verbose] [--warn|--notice|--info|--debug]\n" - " [testname...]\n" - "Recognized tests are:\n"); - for (i = 0; test_array[i].test_name; ++i) { - printf(" %s\n", test_array[i].test_name); - } - - exit(0); -} - -/** Main entry point for unit test code: parse the command line, and run - * some unit tests. */ -int -main(int c, char**v) -{ - or_options_t *options; - char *errmsg = NULL; - int i; - int verbose = 0, any_selected = 0; - int loglevel = LOG_ERR; - -#ifdef USE_DMALLOC - { - int r = CRYPTO_set_mem_ex_functions(_tor_malloc, _tor_realloc, _tor_free); - tor_assert(r); - } -#endif - - update_approx_time(time(NULL)); - options = options_new(); - tor_threads_init(); - init_logging(); - - for (i = 1; i < c; ++i) { - if (!strcmp(v[i], "-v") || !strcmp(v[i], "--verbose")) - verbose++; - else if (!strcmp(v[i], "--warn")) - loglevel = LOG_WARN; - else if (!strcmp(v[i], "--notice")) - loglevel = LOG_NOTICE; - else if (!strcmp(v[i], "--info")) - loglevel = LOG_INFO; - else if (!strcmp(v[i], "--debug")) - loglevel = LOG_DEBUG; - else if (!strcmp(v[i], "--help") || !strcmp(v[i], "-h") || v[i][0] == '-') - syntax(); - else { - int j, found=0; - for (j = 0; test_array[j].test_name; ++j) { - if (!strcmp(v[i], test_array[j].test_name) || - (test_array[j].is_subent && - !strcmpstart(test_array[j].test_name, v[i]) && - test_array[j].test_name[strlen(v[i])] == '/') || - (v[i][0] == '=' && !strcmp(v[i]+1, test_array[j].test_name))) { - test_array[j].selected = 1; - any_selected = 1; - found = 1; - } - } - if (!found) { - printf("Unknown test: %s\n", v[i]); - syntax(); - } - } - } - - if (!any_selected) { - for (i = 0; test_array[i].test_name; ++i) { - test_array[i].selected = test_array[i].is_default; - } - } - - { - log_severity_list_t s; - memset(&s, 0, sizeof(s)); - set_log_severity_config(loglevel, LOG_ERR, &s); - add_stream_log(&s, "", fileno(stdout)); - } - - options->command = CMD_RUN_UNITTESTS; - crypto_global_init(0); - rep_hist_init(); - network_init(); - setup_directory(); - options_init(options); - options->DataDirectory = tor_strdup(temp_dir); - if (set_options(options, &errmsg) < 0) { - printf("Failed to set initial options: %s\n", errmsg); - tor_free(errmsg); - return 1; - } - - crypto_seed_rng(1); - - atexit(remove_directory); - - printf("Running Tor unit tests on %s\n", get_uname()); - - for (i = 0; test_array[i].test_name; ++i) { - if (!test_array[i].selected) - continue; - if (!test_array[i].is_subent) { - printf("\n============================== %s\n",test_array[i].test_name); - } else if (test_array[i].is_subent && verbose) { - printf("\n%s", test_array[i].test_name); - } - test_array[i].test_fn(); - } - puts(""); - - free_pregenerated_keys(); -#ifdef USE_DMALLOC - tor_free_all(0); - dmalloc_log_unfreed(); -#endif - - if (have_failed) - return 1; - else - return 0; -} - diff --git a/src/or/tor_main.c b/src/or/tor_main.c index 8fd3804b29..1ce14ab768 100644 --- a/src/or/tor_main.c +++ b/src/or/tor_main.c @@ -7,7 +7,7 @@ * built from. This string is generated by a bit of shell kludging int * src/or/Makefile.am, and is usually right. */ -const char tor_svn_revision[] = +const char tor_git_revision[] = #ifndef _MSC_VER #include "micro-revision.i" #endif diff --git a/src/test/Makefile.am b/src/test/Makefile.am new file mode 100644 index 0000000000..546fa2f4b7 --- /dev/null +++ b/src/test/Makefile.am @@ -0,0 +1,30 @@ +TESTS = test + +noinst_PROGRAMS = test + +AM_CPPFLAGS = -DSHARE_DATADIR="\"$(datadir)\"" \ + -DLOCALSTATEDIR="\"$(localstatedir)\"" \ + -DBINDIR="\"$(bindir)\"" \ + -I"$(top_srcdir)/src/or" + +# -L flags need to go in LDFLAGS. -l flags need to go in LDADD. +# This seems to matter nowhere but on windows, but I assure you that it +# matters a lot there, and is quite hard to debug if you forget to do it. + +test_SOURCES = \ + test_data.c \ + test.c \ + test_addr.c \ + test_crypto.c \ + test_dir.c \ + test_containers.c \ + test_util.c \ + tinytest.c + +test_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ \ + @TOR_LDFLAGS_libevent@ +test_LDADD = ../or/libtor.a ../common/libor.a ../common/libor-crypto.a \ + ../common/libor-event.a \ + @TOR_ZLIB_LIBS@ -lm @TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ + +noinst_HEADERS = tinytest.h tinytest_macros.h test.h diff --git a/src/test/test.c b/src/test/test.c new file mode 100644 index 0000000000..9b24a99b58 --- /dev/null +++ b/src/test/test.c @@ -0,0 +1,1292 @@ +/* Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/* Ordinarily defined in tor_main.c; this bit is just here to provide one + * since we're not linking to tor_main.c */ +const char tor_git_revision[] = ""; + +/** + * \file test.c + * \brief Unit tests for many pieces of the lower level Tor modules. + **/ + +#include "orconfig.h" + +#include <stdio.h> +#ifdef HAVE_FCNTL_H +#include <fcntl.h> +#endif + +#ifdef MS_WINDOWS +/* For mkdir() */ +#include <direct.h> +#else +#include <dirent.h> +#endif + +/* These macros pull in declarations for some functions and structures that + * are typically file-private. */ +#define BUFFERS_PRIVATE +#define CONFIG_PRIVATE +#define GEOIP_PRIVATE +#define ROUTER_PRIVATE +#define CIRCUIT_PRIVATE + +/* + * Linux doesn't provide lround in math.h by default, but mac os does... + * It's best just to leave math.h out of the picture entirely. + */ +//#include <math.h> +long int lround(double x); +double fabs(double x); + +#include "or.h" +#include "buffers.h" +#include "circuitbuild.h" +#include "config.h" +#include "connection_edge.h" +#include "geoip.h" +#include "rendcommon.h" +#include "test.h" +#include "torgzip.h" +#include "mempool.h" +#include "memarea.h" +#include "onion.h" +#include "policies.h" +#include "rephist.h" +#include "routerparse.h" + +#ifdef USE_DMALLOC +#include <dmalloc.h> +#include <openssl/crypto.h> +#include "main.h" +#endif + +/** Set to true if any unit test has failed. Mostly, this is set by the macros + * in test.h */ +int have_failed = 0; + +/** Temporary directory (set up by setup_directory) under which we store all + * our files during testing. */ +static char temp_dir[256]; +static pid_t temp_dir_setup_in_pid = 0; + +/** Select and create the temporary directory we'll use to run our unit tests. + * Store it in <b>temp_dir</b>. Exit immediately if we can't create it. + * idempotent. */ +static void +setup_directory(void) +{ + static int is_setup = 0; + int r; + if (is_setup) return; + +#ifdef MS_WINDOWS + { + char buf[MAX_PATH]; + const char *tmp = buf; + /* If this fails, we're probably screwed anyway */ + if (!GetTempPath(sizeof(buf),buf)) + tmp = "c:\\windows\\temp"; + tor_snprintf(temp_dir, sizeof(temp_dir), + "%s\\tor_test_%d", tmp, (int)getpid()); + r = mkdir(temp_dir); + } +#else + tor_snprintf(temp_dir, sizeof(temp_dir), "/tmp/tor_test_%d", (int) getpid()); + r = mkdir(temp_dir, 0700); +#endif + if (r) { + fprintf(stderr, "Can't create directory %s:", temp_dir); + perror(""); + exit(1); + } + is_setup = 1; + temp_dir_setup_in_pid = getpid(); +} + +/** Return a filename relative to our testing temporary directory */ +const char * +get_fname(const char *name) +{ + static char buf[1024]; + setup_directory(); + if (!name) + return temp_dir; + tor_snprintf(buf,sizeof(buf),"%s/%s",temp_dir,name); + return buf; +} + +/** Remove all files stored under the temporary directory, and the directory + * itself. Called by atexit(). */ +static void +remove_directory(void) +{ + smartlist_t *elements; + if (getpid() != temp_dir_setup_in_pid) { + /* Only clean out the tempdir when the main process is exiting. */ + return; + } + elements = tor_listdir(temp_dir); + if (elements) { + SMARTLIST_FOREACH(elements, const char *, cp, + { + size_t len = strlen(cp)+strlen(temp_dir)+16; + char *tmp = tor_malloc(len); + tor_snprintf(tmp, len, "%s"PATH_SEPARATOR"%s", temp_dir, cp); + unlink(tmp); + tor_free(tmp); + }); + SMARTLIST_FOREACH(elements, char *, cp, tor_free(cp)); + smartlist_free(elements); + } + rmdir(temp_dir); +} + +/** Define this if unit tests spend too much time generating public keys*/ +#undef CACHE_GENERATED_KEYS + +static crypto_pk_env_t *pregen_keys[5] = {NULL, NULL, NULL, NULL, NULL}; +#define N_PREGEN_KEYS ((int)(sizeof(pregen_keys)/sizeof(pregen_keys[0]))) + +/** Generate and return a new keypair for use in unit tests. If we're using + * the key cache optimization, we might reuse keys: we only guarantee that + * keys made with distinct values for <b>idx</b> are different. The value of + * <b>idx</b> must be at least 0, and less than N_PREGEN_KEYS. */ +crypto_pk_env_t * +pk_generate(int idx) +{ +#ifdef CACHE_GENERATED_KEYS + tor_assert(idx < N_PREGEN_KEYS); + if (! pregen_keys[idx]) { + pregen_keys[idx] = crypto_new_pk_env(); + tor_assert(!crypto_pk_generate_key(pregen_keys[idx])); + } + return crypto_pk_dup_key(pregen_keys[idx]); +#else + crypto_pk_env_t *result; + (void) idx; + result = crypto_new_pk_env(); + tor_assert(!crypto_pk_generate_key(result)); + return result; +#endif +} + +/** Free all storage used for the cached key optimization. */ +static void +free_pregenerated_keys(void) +{ + unsigned idx; + for (idx = 0; idx < N_PREGEN_KEYS; ++idx) { + if (pregen_keys[idx]) { + crypto_free_pk_env(pregen_keys[idx]); + pregen_keys[idx] = NULL; + } + } +} + +/** Run unit tests for buffers.c */ +static void +test_buffers(void) +{ + char str[256]; + char str2[256]; + + buf_t *buf = NULL, *buf2 = NULL; + const char *cp; + + int j; + size_t r; + + /**** + * buf_new + ****/ + if (!(buf = buf_new())) + test_fail(); + + //test_eq(buf_capacity(buf), 4096); + test_eq(buf_datalen(buf), 0); + + /**** + * General pointer frobbing + */ + for (j=0;j<256;++j) { + str[j] = (char)j; + } + write_to_buf(str, 256, buf); + write_to_buf(str, 256, buf); + test_eq(buf_datalen(buf), 512); + fetch_from_buf(str2, 200, buf); + test_memeq(str, str2, 200); + test_eq(buf_datalen(buf), 312); + memset(str2, 0, sizeof(str2)); + + fetch_from_buf(str2, 256, buf); + test_memeq(str+200, str2, 56); + test_memeq(str, str2+56, 200); + test_eq(buf_datalen(buf), 56); + memset(str2, 0, sizeof(str2)); + /* Okay, now we should be 512 bytes into the 4096-byte buffer. If we add + * another 3584 bytes, we hit the end. */ + for (j=0;j<15;++j) { + write_to_buf(str, 256, buf); + } + assert_buf_ok(buf); + test_eq(buf_datalen(buf), 3896); + fetch_from_buf(str2, 56, buf); + test_eq(buf_datalen(buf), 3840); + test_memeq(str+200, str2, 56); + for (j=0;j<15;++j) { + memset(str2, 0, sizeof(str2)); + fetch_from_buf(str2, 256, buf); + test_memeq(str, str2, 256); + } + test_eq(buf_datalen(buf), 0); + buf_free(buf); + buf = NULL; + + /* Okay, now make sure growing can work. */ + buf = buf_new_with_capacity(16); + //test_eq(buf_capacity(buf), 16); + write_to_buf(str+1, 255, buf); + //test_eq(buf_capacity(buf), 256); + fetch_from_buf(str2, 254, buf); + test_memeq(str+1, str2, 254); + //test_eq(buf_capacity(buf), 256); + assert_buf_ok(buf); + write_to_buf(str, 32, buf); + //test_eq(buf_capacity(buf), 256); + assert_buf_ok(buf); + write_to_buf(str, 256, buf); + assert_buf_ok(buf); + //test_eq(buf_capacity(buf), 512); + test_eq(buf_datalen(buf), 33+256); + fetch_from_buf(str2, 33, buf); + test_eq(*str2, str[255]); + + test_memeq(str2+1, str, 32); + //test_eq(buf_capacity(buf), 512); + test_eq(buf_datalen(buf), 256); + fetch_from_buf(str2, 256, buf); + test_memeq(str, str2, 256); + + /* now try shrinking: case 1. */ + buf_free(buf); + buf = buf_new_with_capacity(33668); + for (j=0;j<67;++j) { + write_to_buf(str,255, buf); + } + //test_eq(buf_capacity(buf), 33668); + test_eq(buf_datalen(buf), 17085); + for (j=0; j < 40; ++j) { + fetch_from_buf(str2, 255,buf); + test_memeq(str2, str, 255); + } + + /* now try shrinking: case 2. */ + buf_free(buf); + buf = buf_new_with_capacity(33668); + for (j=0;j<67;++j) { + write_to_buf(str,255, buf); + } + for (j=0; j < 20; ++j) { + fetch_from_buf(str2, 255,buf); + test_memeq(str2, str, 255); + } + for (j=0;j<80;++j) { + write_to_buf(str,255, buf); + } + //test_eq(buf_capacity(buf),33668); + for (j=0; j < 120; ++j) { + fetch_from_buf(str2, 255,buf); + test_memeq(str2, str, 255); + } + + /* Move from buf to buf. */ + buf_free(buf); + buf = buf_new_with_capacity(4096); + buf2 = buf_new_with_capacity(4096); + for (j=0;j<100;++j) + write_to_buf(str, 255, buf); + test_eq(buf_datalen(buf), 25500); + for (j=0;j<100;++j) { + r = 10; + move_buf_to_buf(buf2, buf, &r); + test_eq(r, 0); + } + test_eq(buf_datalen(buf), 24500); + test_eq(buf_datalen(buf2), 1000); + for (j=0;j<3;++j) { + fetch_from_buf(str2, 255, buf2); + test_memeq(str2, str, 255); + } + r = 8192; /*big move*/ + move_buf_to_buf(buf2, buf, &r); + test_eq(r, 0); + r = 30000; /* incomplete move */ + move_buf_to_buf(buf2, buf, &r); + test_eq(r, 13692); + for (j=0;j<97;++j) { + fetch_from_buf(str2, 255, buf2); + test_memeq(str2, str, 255); + } + buf_free(buf); + buf_free(buf2); + buf = buf2 = NULL; + + buf = buf_new_with_capacity(5); + cp = "Testing. This is a moderately long Testing string."; + for (j = 0; cp[j]; j++) + write_to_buf(cp+j, 1, buf); + test_eq(0, buf_find_string_offset(buf, "Testing", 7)); + test_eq(1, buf_find_string_offset(buf, "esting", 6)); + test_eq(1, buf_find_string_offset(buf, "est", 3)); + test_eq(39, buf_find_string_offset(buf, "ing str", 7)); + test_eq(35, buf_find_string_offset(buf, "Testing str", 11)); + test_eq(32, buf_find_string_offset(buf, "ng ", 3)); + test_eq(43, buf_find_string_offset(buf, "string.", 7)); + test_eq(-1, buf_find_string_offset(buf, "shrdlu", 6)); + test_eq(-1, buf_find_string_offset(buf, "Testing thing", 13)); + test_eq(-1, buf_find_string_offset(buf, "ngx", 3)); + buf_free(buf); + buf = NULL; + + done: + if (buf) + buf_free(buf); + if (buf2) + buf_free(buf2); +} + +/** Run unit tests for the onion handshake code. */ +static void +test_onion_handshake(void) +{ + /* client-side */ + crypto_dh_env_t *c_dh = NULL; + char c_buf[ONIONSKIN_CHALLENGE_LEN]; + char c_keys[40]; + + /* server-side */ + char s_buf[ONIONSKIN_REPLY_LEN]; + char s_keys[40]; + + /* shared */ + crypto_pk_env_t *pk = NULL; + + pk = pk_generate(0); + + /* client handshake 1. */ + memset(c_buf, 0, ONIONSKIN_CHALLENGE_LEN); + test_assert(! onion_skin_create(pk, &c_dh, c_buf)); + + /* server handshake */ + memset(s_buf, 0, ONIONSKIN_REPLY_LEN); + memset(s_keys, 0, 40); + test_assert(! onion_skin_server_handshake(c_buf, pk, NULL, + s_buf, s_keys, 40)); + + /* client handshake 2 */ + memset(c_keys, 0, 40); + test_assert(! onion_skin_client_handshake(c_dh, s_buf, c_keys, 40)); + + if (memcmp(c_keys, s_keys, 40)) { + puts("Aiiiie"); + exit(1); + } + test_memeq(c_keys, s_keys, 40); + memset(s_buf, 0, 40); + test_memneq(c_keys, s_buf, 40); + + done: + if (c_dh) + crypto_dh_free(c_dh); + if (pk) + crypto_free_pk_env(pk); +} + +static void +test_circuit_timeout(void) +{ + /* Plan: + * 1. Generate 1000 samples + * 2. Estimate parameters + * 3. If difference, repeat + * 4. Save state + * 5. load state + * 6. Estimate parameters + * 7. compare differences + */ + circuit_build_times_t initial; + circuit_build_times_t estimate; + circuit_build_times_t final; + double timeout1, timeout2; + or_state_t state; + int i, runs; + double close_ms; + circuit_build_times_init(&initial); + circuit_build_times_init(&estimate); + circuit_build_times_init(&final); + + memset(&state, 0, sizeof(or_state_t)); + + circuitbuild_running_unit_tests(); +#define timeout0 (build_time_t)(30*1000.0) + initial.Xm = 3000; + circuit_build_times_initial_alpha(&initial, + CBT_DEFAULT_QUANTILE_CUTOFF/100.0, + timeout0); + close_ms = MAX(circuit_build_times_calculate_timeout(&initial, + CBT_DEFAULT_CLOSE_QUANTILE/100.0), + CBT_DEFAULT_TIMEOUT_INITIAL_VALUE); + do { + for (i=0; i < CBT_DEFAULT_MIN_CIRCUITS_TO_OBSERVE; i++) { + build_time_t sample = circuit_build_times_generate_sample(&initial,0,1); + + if (sample > close_ms) { + circuit_build_times_add_time(&estimate, CBT_BUILD_ABANDONED); + } else { + circuit_build_times_add_time(&estimate, sample); + } + } + circuit_build_times_update_alpha(&estimate); + timeout1 = circuit_build_times_calculate_timeout(&estimate, + CBT_DEFAULT_QUANTILE_CUTOFF/100.0); + circuit_build_times_set_timeout(&estimate); + log_notice(LD_CIRC, "Timeout1 is %lf, Xm is %d", timeout1, estimate.Xm); + /* 2% error */ + } while (fabs(circuit_build_times_cdf(&initial, timeout0) - + circuit_build_times_cdf(&initial, timeout1)) > 0.02); + + test_assert(estimate.total_build_times <= CBT_NCIRCUITS_TO_OBSERVE); + + circuit_build_times_update_state(&estimate, &state); + test_assert(circuit_build_times_parse_state(&final, &state) == 0); + + circuit_build_times_update_alpha(&final); + timeout2 = circuit_build_times_calculate_timeout(&final, + CBT_DEFAULT_QUANTILE_CUTOFF/100.0); + + circuit_build_times_set_timeout(&final); + log_notice(LD_CIRC, "Timeout2 is %lf, Xm is %d", timeout2, final.Xm); + + /* 5% here because some accuracy is lost due to histogram conversion */ + test_assert(fabs(circuit_build_times_cdf(&initial, timeout0) - + circuit_build_times_cdf(&initial, timeout2)) < 0.05); + + for (runs = 0; runs < 50; runs++) { + int build_times_idx = 0; + int total_build_times = 0; + + final.close_ms = final.timeout_ms = CBT_DEFAULT_TIMEOUT_INITIAL_VALUE; + estimate.close_ms = estimate.timeout_ms + = CBT_DEFAULT_TIMEOUT_INITIAL_VALUE; + + for (i = 0; i < CBT_DEFAULT_RECENT_CIRCUITS*2; i++) { + circuit_build_times_network_circ_success(&estimate); + circuit_build_times_add_time(&estimate, + circuit_build_times_generate_sample(&estimate, 0, + CBT_DEFAULT_QUANTILE_CUTOFF/100.0)); + + circuit_build_times_network_circ_success(&estimate); + circuit_build_times_add_time(&final, + circuit_build_times_generate_sample(&final, 0, + CBT_DEFAULT_QUANTILE_CUTOFF/100.0)); + } + + test_assert(!circuit_build_times_network_check_changed(&estimate)); + test_assert(!circuit_build_times_network_check_changed(&final)); + + /* Reset liveness to be non-live */ + final.liveness.network_last_live = 0; + estimate.liveness.network_last_live = 0; + + build_times_idx = estimate.build_times_idx; + total_build_times = estimate.total_build_times; + + test_assert(circuit_build_times_network_check_live(&estimate)); + test_assert(circuit_build_times_network_check_live(&final)); + + circuit_build_times_count_close(&estimate, 0, + (time_t)(approx_time()-estimate.close_ms/1000.0-1)); + circuit_build_times_count_close(&final, 0, + (time_t)(approx_time()-final.close_ms/1000.0-1)); + + test_assert(!circuit_build_times_network_check_live(&estimate)); + test_assert(!circuit_build_times_network_check_live(&final)); + + log_info(LD_CIRC, "idx: %d %d, tot: %d %d", + build_times_idx, estimate.build_times_idx, + total_build_times, estimate.total_build_times); + + /* Check rollback index. Should match top of loop. */ + test_assert(build_times_idx == estimate.build_times_idx); + // This can fail if estimate.total_build_times == 1000, because + // in that case, rewind actually causes us to lose timeouts + if (total_build_times != CBT_NCIRCUITS_TO_OBSERVE) + test_assert(total_build_times == estimate.total_build_times); + + /* Now simulate that the network has become live and we need + * a change */ + circuit_build_times_network_is_live(&estimate); + circuit_build_times_network_is_live(&final); + + for (i = 0; i < CBT_DEFAULT_MAX_RECENT_TIMEOUT_COUNT; i++) { + circuit_build_times_count_timeout(&estimate, 1); + + if (i < CBT_DEFAULT_MAX_RECENT_TIMEOUT_COUNT-1) { + circuit_build_times_count_timeout(&final, 1); + } + } + + test_assert(estimate.liveness.after_firsthop_idx == 0); + test_assert(final.liveness.after_firsthop_idx == + CBT_DEFAULT_MAX_RECENT_TIMEOUT_COUNT-1); + + test_assert(circuit_build_times_network_check_live(&estimate)); + test_assert(circuit_build_times_network_check_live(&final)); + + circuit_build_times_count_timeout(&final, 1); + } + + done: + return; +} + +/** Helper: Parse the exit policy string in <b>policy_str</b>, and make sure + * that policies_summarize() produces the string <b>expected_summary</b> from + * it. */ +static void +test_policy_summary_helper(const char *policy_str, + const char *expected_summary) +{ + config_line_t line; + smartlist_t *policy = smartlist_create(); + char *summary = NULL; + int r; + + line.key = (char*)"foo"; + line.value = (char *)policy_str; + line.next = NULL; + + r = policies_parse_exit_policy(&line, &policy, 0, NULL, 1); + test_eq(r, 0); + summary = policy_summarize(policy); + + test_assert(summary != NULL); + test_streq(summary, expected_summary); + + done: + tor_free(summary); + if (policy) + addr_policy_list_free(policy); +} + +/** Run unit tests for generating summary lines of exit policies */ +static void +test_policies(void) +{ + int i; + smartlist_t *policy = NULL, *policy2 = NULL, *policy3 = NULL, + *policy4 = NULL, *policy5 = NULL, *policy6 = NULL, + *policy7 = NULL; + addr_policy_t *p; + tor_addr_t tar; + config_line_t line; + smartlist_t *sm = NULL; + char *policy_str = NULL; + + policy = smartlist_create(); + + p = router_parse_addr_policy_item_from_string("reject 192.168.0.0/16:*",-1); + test_assert(p != NULL); + test_eq(ADDR_POLICY_REJECT, p->policy_type); + tor_addr_from_ipv4h(&tar, 0xc0a80000u); + test_eq(0, tor_addr_compare(&p->addr, &tar, CMP_EXACT)); + test_eq(16, p->maskbits); + test_eq(1, p->prt_min); + test_eq(65535, p->prt_max); + + smartlist_add(policy, p); + + test_assert(ADDR_POLICY_ACCEPTED == + compare_addr_to_addr_policy(0x01020304u, 2, policy)); + test_assert(ADDR_POLICY_PROBABLY_ACCEPTED == + compare_addr_to_addr_policy(0, 2, policy)); + test_assert(ADDR_POLICY_REJECTED == + compare_addr_to_addr_policy(0xc0a80102, 2, policy)); + + test_assert(0 == policies_parse_exit_policy(NULL, &policy2, 1, NULL, 1)); + test_assert(policy2); + + policy3 = smartlist_create(); + p = router_parse_addr_policy_item_from_string("reject *:*",-1); + test_assert(p != NULL); + smartlist_add(policy3, p); + p = router_parse_addr_policy_item_from_string("accept *:*",-1); + test_assert(p != NULL); + smartlist_add(policy3, p); + + policy4 = smartlist_create(); + p = router_parse_addr_policy_item_from_string("accept *:443",-1); + test_assert(p != NULL); + smartlist_add(policy4, p); + p = router_parse_addr_policy_item_from_string("accept *:443",-1); + test_assert(p != NULL); + smartlist_add(policy4, p); + + policy5 = smartlist_create(); + p = router_parse_addr_policy_item_from_string("reject 0.0.0.0/8:*",-1); + test_assert(p != NULL); + smartlist_add(policy5, p); + p = router_parse_addr_policy_item_from_string("reject 169.254.0.0/16:*",-1); + test_assert(p != NULL); + smartlist_add(policy5, p); + p = router_parse_addr_policy_item_from_string("reject 127.0.0.0/8:*",-1); + test_assert(p != NULL); + smartlist_add(policy5, p); + p = router_parse_addr_policy_item_from_string("reject 192.168.0.0/16:*",-1); + test_assert(p != NULL); + smartlist_add(policy5, p); + p = router_parse_addr_policy_item_from_string("reject 10.0.0.0/8:*",-1); + test_assert(p != NULL); + smartlist_add(policy5, p); + p = router_parse_addr_policy_item_from_string("reject 172.16.0.0/12:*",-1); + test_assert(p != NULL); + smartlist_add(policy5, p); + p = router_parse_addr_policy_item_from_string("reject 80.190.250.90:*",-1); + test_assert(p != NULL); + smartlist_add(policy5, p); + p = router_parse_addr_policy_item_from_string("reject *:1-65534",-1); + test_assert(p != NULL); + smartlist_add(policy5, p); + p = router_parse_addr_policy_item_from_string("reject *:65535",-1); + test_assert(p != NULL); + smartlist_add(policy5, p); + p = router_parse_addr_policy_item_from_string("accept *:1-65535",-1); + test_assert(p != NULL); + smartlist_add(policy5, p); + + policy6 = smartlist_create(); + p = router_parse_addr_policy_item_from_string("accept 43.3.0.0/9:*",-1); + test_assert(p != NULL); + smartlist_add(policy6, p); + + policy7 = smartlist_create(); + p = router_parse_addr_policy_item_from_string("accept 0.0.0.0/8:*",-1); + test_assert(p != NULL); + smartlist_add(policy7, p); + + test_assert(!exit_policy_is_general_exit(policy)); + test_assert(exit_policy_is_general_exit(policy2)); + test_assert(!exit_policy_is_general_exit(NULL)); + test_assert(!exit_policy_is_general_exit(policy3)); + test_assert(!exit_policy_is_general_exit(policy4)); + test_assert(!exit_policy_is_general_exit(policy5)); + test_assert(!exit_policy_is_general_exit(policy6)); + test_assert(!exit_policy_is_general_exit(policy7)); + + test_assert(cmp_addr_policies(policy, policy2)); + test_assert(cmp_addr_policies(policy, NULL)); + test_assert(!cmp_addr_policies(policy2, policy2)); + test_assert(!cmp_addr_policies(NULL, NULL)); + + test_assert(!policy_is_reject_star(policy2)); + test_assert(policy_is_reject_star(policy)); + test_assert(policy_is_reject_star(NULL)); + + addr_policy_list_free(policy); + policy = NULL; + + /* make sure compacting logic works. */ + policy = NULL; + line.key = (char*)"foo"; + line.value = (char*)"accept *:80,reject private:*,reject *:*"; + line.next = NULL; + test_assert(0 == policies_parse_exit_policy(&line, &policy, 0, NULL, 1)); + test_assert(policy); + //test_streq(policy->string, "accept *:80"); + //test_streq(policy->next->string, "reject *:*"); + test_eq(smartlist_len(policy), 2); + + /* test policy summaries */ + /* check if we properly ignore private IP addresses */ + test_policy_summary_helper("reject 192.168.0.0/16:*," + "reject 0.0.0.0/8:*," + "reject 10.0.0.0/8:*," + "accept *:10-30," + "accept *:90," + "reject *:*", + "accept 10-30,90"); + /* check all accept policies, and proper counting of rejects */ + test_policy_summary_helper("reject 11.0.0.0/9:80," + "reject 12.0.0.0/9:80," + "reject 13.0.0.0/9:80," + "reject 14.0.0.0/9:80," + "accept *:*", "accept 1-65535"); + test_policy_summary_helper("reject 11.0.0.0/9:80," + "reject 12.0.0.0/9:80," + "reject 13.0.0.0/9:80," + "reject 14.0.0.0/9:80," + "reject 15.0.0.0:81," + "accept *:*", "accept 1-65535"); + test_policy_summary_helper("reject 11.0.0.0/9:80," + "reject 12.0.0.0/9:80," + "reject 13.0.0.0/9:80," + "reject 14.0.0.0/9:80," + "reject 15.0.0.0:80," + "accept *:*", + "reject 80"); + /* no exits */ + test_policy_summary_helper("accept 11.0.0.0/9:80," + "reject *:*", + "reject 1-65535"); + /* port merging */ + test_policy_summary_helper("accept *:80," + "accept *:81," + "accept *:100-110," + "accept *:111," + "reject *:*", + "accept 80-81,100-111"); + /* border ports */ + test_policy_summary_helper("accept *:1," + "accept *:3," + "accept *:65535," + "reject *:*", + "accept 1,3,65535"); + /* holes */ + test_policy_summary_helper("accept *:1," + "accept *:3," + "accept *:5," + "accept *:7," + "reject *:*", + "accept 1,3,5,7"); + test_policy_summary_helper("reject *:1," + "reject *:3," + "reject *:5," + "reject *:7," + "accept *:*", + "reject 1,3,5,7"); + + /* truncation ports */ + sm = smartlist_create(); + for (i=1; i<2000; i+=2) { + char buf[POLICY_BUF_LEN]; + tor_snprintf(buf, sizeof(buf), "reject *:%d", i); + smartlist_add(sm, tor_strdup(buf)); + } + smartlist_add(sm, tor_strdup("accept *:*")); + policy_str = smartlist_join_strings(sm, ",", 0, NULL); + test_policy_summary_helper( policy_str, + "accept 2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40,42,44," + "46,48,50,52,54,56,58,60,62,64,66,68,70,72,74,76,78,80,82,84,86,88,90," + "92,94,96,98,100,102,104,106,108,110,112,114,116,118,120,122,124,126,128," + "130,132,134,136,138,140,142,144,146,148,150,152,154,156,158,160,162,164," + "166,168,170,172,174,176,178,180,182,184,186,188,190,192,194,196,198,200," + "202,204,206,208,210,212,214,216,218,220,222,224,226,228,230,232,234,236," + "238,240,242,244,246,248,250,252,254,256,258,260,262,264,266,268,270,272," + "274,276,278,280,282,284,286,288,290,292,294,296,298,300,302,304,306,308," + "310,312,314,316,318,320,322,324,326,328,330,332,334,336,338,340,342,344," + "346,348,350,352,354,356,358,360,362,364,366,368,370,372,374,376,378,380," + "382,384,386,388,390,392,394,396,398,400,402,404,406,408,410,412,414,416," + "418,420,422,424,426,428,430,432,434,436,438,440,442,444,446,448,450,452," + "454,456,458,460,462,464,466,468,470,472,474,476,478,480,482,484,486,488," + "490,492,494,496,498,500,502,504,506,508,510,512,514,516,518,520,522"); + + done: + addr_policy_list_free(policy); + addr_policy_list_free(policy2); + addr_policy_list_free(policy3); + addr_policy_list_free(policy4); + addr_policy_list_free(policy5); + addr_policy_list_free(policy6); + addr_policy_list_free(policy7); + tor_free(policy_str); + if (sm) { + SMARTLIST_FOREACH(sm, char *, s, tor_free(s)); + smartlist_free(sm); + } +} + +/** Run AES performance benchmarks. */ +static void +bench_aes(void) +{ + int len, i; + char *b1, *b2; + crypto_cipher_env_t *c; + struct timeval start, end; + const int iters = 100000; + uint64_t nsec; + c = crypto_new_cipher_env(); + crypto_cipher_generate_key(c); + crypto_cipher_encrypt_init_cipher(c); + for (len = 1; len <= 8192; len *= 2) { + b1 = tor_malloc_zero(len); + b2 = tor_malloc_zero(len); + tor_gettimeofday(&start); + for (i = 0; i < iters; ++i) { + crypto_cipher_encrypt(c, b1, b2, len); + } + tor_gettimeofday(&end); + tor_free(b1); + tor_free(b2); + nsec = (uint64_t) tv_udiff(&start,&end); + nsec *= 1000; + nsec /= (iters*len); + printf("%d bytes: "U64_FORMAT" nsec per byte\n", len, + U64_PRINTF_ARG(nsec)); + } + crypto_free_cipher_env(c); +} + +/** Run digestmap_t performance benchmarks. */ +static void +bench_dmap(void) +{ + smartlist_t *sl = smartlist_create(); + smartlist_t *sl2 = smartlist_create(); + struct timeval start, end, pt2, pt3, pt4; + const int iters = 10000; + const int elts = 4000; + const int fpostests = 1000000; + char d[20]; + int i,n=0, fp = 0; + digestmap_t *dm = digestmap_new(); + digestset_t *ds = digestset_new(elts); + + for (i = 0; i < elts; ++i) { + crypto_rand(d, 20); + smartlist_add(sl, tor_memdup(d, 20)); + } + for (i = 0; i < elts; ++i) { + crypto_rand(d, 20); + smartlist_add(sl2, tor_memdup(d, 20)); + } + printf("nbits=%d\n", ds->mask+1); + + tor_gettimeofday(&start); + for (i = 0; i < iters; ++i) { + SMARTLIST_FOREACH(sl, const char *, cp, digestmap_set(dm, cp, (void*)1)); + } + tor_gettimeofday(&pt2); + for (i = 0; i < iters; ++i) { + SMARTLIST_FOREACH(sl, const char *, cp, digestmap_get(dm, cp)); + SMARTLIST_FOREACH(sl2, const char *, cp, digestmap_get(dm, cp)); + } + tor_gettimeofday(&pt3); + for (i = 0; i < iters; ++i) { + SMARTLIST_FOREACH(sl, const char *, cp, digestset_add(ds, cp)); + } + tor_gettimeofday(&pt4); + for (i = 0; i < iters; ++i) { + SMARTLIST_FOREACH(sl, const char *, cp, n += digestset_isin(ds, cp)); + SMARTLIST_FOREACH(sl2, const char *, cp, n += digestset_isin(ds, cp)); + } + tor_gettimeofday(&end); + + for (i = 0; i < fpostests; ++i) { + crypto_rand(d, 20); + if (digestset_isin(ds, d)) ++fp; + } + + printf("%ld\n",(unsigned long)tv_udiff(&start, &pt2)); + printf("%ld\n",(unsigned long)tv_udiff(&pt2, &pt3)); + printf("%ld\n",(unsigned long)tv_udiff(&pt3, &pt4)); + printf("%ld\n",(unsigned long)tv_udiff(&pt4, &end)); + printf("-- %d\n", n); + printf("++ %f\n", fp/(double)fpostests); + digestmap_free(dm, NULL); + digestset_free(ds); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + SMARTLIST_FOREACH(sl2, char *, cp, tor_free(cp)); + smartlist_free(sl); + smartlist_free(sl2); +} + +/** Test encoding and parsing of rendezvous service descriptors. */ +static void +test_rend_fns(void) +{ + rend_service_descriptor_t *generated = NULL, *parsed = NULL; + char service_id[DIGEST_LEN]; + char service_id_base32[REND_SERVICE_ID_LEN_BASE32+1]; + const char *next_desc; + smartlist_t *descs = smartlist_create(); + char computed_desc_id[DIGEST_LEN]; + char parsed_desc_id[DIGEST_LEN]; + crypto_pk_env_t *pk1 = NULL, *pk2 = NULL; + time_t now; + char *intro_points_encrypted = NULL; + size_t intro_points_size; + size_t encoded_size; + int i; + char address1[] = "fooaddress.onion"; + char address2[] = "aaaaaaaaaaaaaaaa.onion"; + char address3[] = "fooaddress.exit"; + char address4[] = "www.torproject.org"; + + test_assert(BAD_HOSTNAME == parse_extended_hostname(address1, 1)); + test_assert(ONION_HOSTNAME == parse_extended_hostname(address2, 1)); + test_assert(EXIT_HOSTNAME == parse_extended_hostname(address3, 1)); + test_assert(NORMAL_HOSTNAME == parse_extended_hostname(address4, 1)); + + pk1 = pk_generate(0); + pk2 = pk_generate(1); + generated = tor_malloc_zero(sizeof(rend_service_descriptor_t)); + generated->pk = crypto_pk_dup_key(pk1); + crypto_pk_get_digest(generated->pk, service_id); + base32_encode(service_id_base32, REND_SERVICE_ID_LEN_BASE32+1, + service_id, REND_SERVICE_ID_LEN); + now = time(NULL); + generated->timestamp = now; + generated->version = 2; + generated->protocols = 42; + generated->intro_nodes = smartlist_create(); + + for (i = 0; i < 3; i++) { + rend_intro_point_t *intro = tor_malloc_zero(sizeof(rend_intro_point_t)); + crypto_pk_env_t *okey = pk_generate(2 + i); + intro->extend_info = tor_malloc_zero(sizeof(extend_info_t)); + intro->extend_info->onion_key = okey; + crypto_pk_get_digest(intro->extend_info->onion_key, + intro->extend_info->identity_digest); + //crypto_rand(info->identity_digest, DIGEST_LEN); /* Would this work? */ + intro->extend_info->nickname[0] = '$'; + base16_encode(intro->extend_info->nickname + 1, + sizeof(intro->extend_info->nickname) - 1, + intro->extend_info->identity_digest, DIGEST_LEN); + /* Does not cover all IP addresses. */ + tor_addr_from_ipv4h(&intro->extend_info->addr, crypto_rand_int(65536)); + intro->extend_info->port = 1 + crypto_rand_int(65535); + intro->intro_key = crypto_pk_dup_key(pk2); + smartlist_add(generated->intro_nodes, intro); + } + test_assert(rend_encode_v2_descriptors(descs, generated, now, 0, + REND_NO_AUTH, NULL, NULL) > 0); + test_assert(rend_compute_v2_desc_id(computed_desc_id, service_id_base32, + NULL, now, 0) == 0); + test_memeq(((rend_encoded_v2_service_descriptor_t *) + smartlist_get(descs, 0))->desc_id, computed_desc_id, DIGEST_LEN); + test_assert(rend_parse_v2_service_descriptor(&parsed, parsed_desc_id, + &intro_points_encrypted, + &intro_points_size, + &encoded_size, + &next_desc, + ((rend_encoded_v2_service_descriptor_t *) + smartlist_get(descs, 0))->desc_str) == 0); + test_assert(parsed); + test_memeq(((rend_encoded_v2_service_descriptor_t *) + smartlist_get(descs, 0))->desc_id, parsed_desc_id, DIGEST_LEN); + test_eq(rend_parse_introduction_points(parsed, intro_points_encrypted, + intro_points_size), 3); + test_assert(!crypto_pk_cmp_keys(generated->pk, parsed->pk)); + test_eq(parsed->timestamp, now); + test_eq(parsed->version, 2); + test_eq(parsed->protocols, 42); + test_eq(smartlist_len(parsed->intro_nodes), 3); + for (i = 0; i < smartlist_len(parsed->intro_nodes); i++) { + rend_intro_point_t *par_intro = smartlist_get(parsed->intro_nodes, i), + *gen_intro = smartlist_get(generated->intro_nodes, i); + extend_info_t *par_info = par_intro->extend_info; + extend_info_t *gen_info = gen_intro->extend_info; + test_assert(!crypto_pk_cmp_keys(gen_info->onion_key, par_info->onion_key)); + test_memeq(gen_info->identity_digest, par_info->identity_digest, + DIGEST_LEN); + test_streq(gen_info->nickname, par_info->nickname); + test_assert(tor_addr_eq(&gen_info->addr, &par_info->addr)); + test_eq(gen_info->port, par_info->port); + } + + rend_service_descriptor_free(parsed); + rend_service_descriptor_free(generated); + parsed = generated = NULL; + + done: + if (descs) { + for (i = 0; i < smartlist_len(descs); i++) + rend_encoded_v2_service_descriptor_free(smartlist_get(descs, i)); + smartlist_free(descs); + } + if (parsed) + rend_service_descriptor_free(parsed); + if (generated) + rend_service_descriptor_free(generated); + if (pk1) + crypto_free_pk_env(pk1); + if (pk2) + crypto_free_pk_env(pk2); + tor_free(intro_points_encrypted); +} + +/** Run unit tests for GeoIP code. */ +static void +test_geoip(void) +{ + int i, j; + time_t now = time(NULL); + char *s = NULL; + + /* Populate the DB a bit. Add these in order, since we can't do the final + * 'sort' step. These aren't very good IP addresses, but they're perfectly + * fine uint32_t values. */ + test_eq(0, geoip_parse_entry("10,50,AB")); + test_eq(0, geoip_parse_entry("52,90,XY")); + test_eq(0, geoip_parse_entry("95,100,AB")); + test_eq(0, geoip_parse_entry("\"105\",\"140\",\"ZZ\"")); + test_eq(0, geoip_parse_entry("\"150\",\"190\",\"XY\"")); + test_eq(0, geoip_parse_entry("\"200\",\"250\",\"AB\"")); + + /* We should have 4 countries: ??, ab, xy, zz. */ + test_eq(4, geoip_get_n_countries()); + /* Make sure that country ID actually works. */ +#define NAMEFOR(x) geoip_get_country_name(geoip_get_country_by_ip(x)) + test_streq("??", NAMEFOR(3)); + test_eq(0, geoip_get_country_by_ip(3)); + test_streq("ab", NAMEFOR(32)); + test_streq("??", NAMEFOR(5)); + test_streq("??", NAMEFOR(51)); + test_streq("xy", NAMEFOR(150)); + test_streq("xy", NAMEFOR(190)); + test_streq("??", NAMEFOR(2000)); +#undef NAMEFOR + + get_options()->BridgeRelay = 1; + get_options()->BridgeRecordUsageByCountry = 1; + /* Put 9 observations in AB... */ + for (i=32; i < 40; ++i) + geoip_note_client_seen(GEOIP_CLIENT_CONNECT, i, now-7200); + geoip_note_client_seen(GEOIP_CLIENT_CONNECT, 225, now-7200); + /* and 3 observations in XY, several times. */ + for (j=0; j < 10; ++j) + for (i=52; i < 55; ++i) + geoip_note_client_seen(GEOIP_CLIENT_CONNECT, i, now-3600); + /* and 17 observations in ZZ... */ + for (i=110; i < 127; ++i) + geoip_note_client_seen(GEOIP_CLIENT_CONNECT, i, now); + s = geoip_get_client_history(GEOIP_CLIENT_CONNECT); + test_assert(s); + test_streq("zz=24,ab=16,xy=8", s); + tor_free(s); + + /* Now clear out all the AB observations. */ + geoip_remove_old_clients(now-6000); + s = geoip_get_client_history(GEOIP_CLIENT_CONNECT); + test_assert(s); + test_streq("zz=24,xy=8", s); + + done: + tor_free(s); +} + +/** Run unit tests for stats code. */ +static void +test_stats(void) +{ + time_t now = 1281533250; /* 2010-08-11 13:27:30 UTC */ + char *s = NULL; + int i; + + /* We shouldn't collect exit stats without initializing them. */ + rep_hist_note_exit_stream_opened(80); + rep_hist_note_exit_bytes(80, 100, 10000); + s = rep_hist_format_exit_stats(now + 86400); + test_assert(!s); + + /* Initialize stats, note some streams and bytes, and generate history + * string. */ + rep_hist_exit_stats_init(now); + rep_hist_note_exit_stream_opened(80); + rep_hist_note_exit_bytes(80, 100, 10000); + rep_hist_note_exit_stream_opened(443); + rep_hist_note_exit_bytes(443, 100, 10000); + rep_hist_note_exit_bytes(443, 100, 10000); + s = rep_hist_format_exit_stats(now + 86400); + test_streq("exit-stats-end 2010-08-12 13:27:30 (86400 s)\n" + "exit-kibibytes-written 80=1,443=1,other=0\n" + "exit-kibibytes-read 80=10,443=20,other=0\n" + "exit-streams-opened 80=4,443=4,other=0\n", s); + tor_free(s); + + /* Add a few bytes on 10 more ports and ensure that only the top 10 + * ports are contained in the history string. */ + for (i = 50; i < 60; i++) { + rep_hist_note_exit_bytes(i, i, i); + rep_hist_note_exit_stream_opened(i); + } + s = rep_hist_format_exit_stats(now + 86400); + test_streq("exit-stats-end 2010-08-12 13:27:30 (86400 s)\n" + "exit-kibibytes-written 52=1,53=1,54=1,55=1,56=1,57=1,58=1," + "59=1,80=1,443=1,other=1\n" + "exit-kibibytes-read 52=1,53=1,54=1,55=1,56=1,57=1,58=1," + "59=1,80=10,443=20,other=1\n" + "exit-streams-opened 52=4,53=4,54=4,55=4,56=4,57=4,58=4," + "59=4,80=4,443=4,other=4\n", s); + tor_free(s); + + /* Stop collecting stats, add some bytes, and ensure we don't generate + * a history string. */ + rep_hist_exit_stats_term(); + rep_hist_note_exit_bytes(80, 100, 10000); + s = rep_hist_format_exit_stats(now + 86400); + test_assert(!s); + + /* Re-start stats, add some bytes, reset stats, and see what history we + * get when observing no streams or bytes at all. */ + rep_hist_exit_stats_init(now); + rep_hist_note_exit_stream_opened(80); + rep_hist_note_exit_bytes(80, 100, 10000); + rep_hist_reset_exit_stats(now); + s = rep_hist_format_exit_stats(now + 86400); + test_streq("exit-stats-end 2010-08-12 13:27:30 (86400 s)\n" + "exit-kibibytes-written other=0\n" + "exit-kibibytes-read other=0\n" + "exit-streams-opened other=0\n", s); + + done: + tor_free(s); +} + +static void * +legacy_test_setup(const struct testcase_t *testcase) +{ + return testcase->setup_data; +} + +void +legacy_test_helper(void *data) +{ + void (*fn)(void) = data; + fn(); +} + +static int +legacy_test_cleanup(const struct testcase_t *testcase, void *ptr) +{ + (void)ptr; + (void)testcase; + return 1; +} + +const struct testcase_setup_t legacy_setup = { + legacy_test_setup, legacy_test_cleanup +}; + +#define ENT(name) \ + { #name, legacy_test_helper, 0, &legacy_setup, test_ ## name } +#define SUBENT(group, name) \ + { #group "_" #name, legacy_test_helper, 0, &legacy_setup, \ + test_ ## group ## _ ## name } +#define DISABLED(name) \ + { #name, legacy_test_helper, TT_SKIP, &legacy_setup, name } +#define FORK(name) \ + { #name, legacy_test_helper, TT_FORK, &legacy_setup, test_ ## name } + +static struct testcase_t test_array[] = { + ENT(buffers), + ENT(onion_handshake), + ENT(circuit_timeout), + ENT(policies), + ENT(rend_fns), + ENT(geoip), + FORK(stats), + + DISABLED(bench_aes), + DISABLED(bench_dmap), + END_OF_TESTCASES +}; + +extern struct testcase_t addr_tests[]; +extern struct testcase_t crypto_tests[]; +extern struct testcase_t container_tests[]; +extern struct testcase_t util_tests[]; +extern struct testcase_t dir_tests[]; + +static struct testgroup_t testgroups[] = { + { "", test_array }, + { "addr/", addr_tests }, + { "crypto/", crypto_tests }, + { "container/", container_tests }, + { "util/", util_tests }, + { "dir/", dir_tests }, + END_OF_GROUPS +}; + +/** Main entry point for unit test code: parse the command line, and run + * some unit tests. */ +int +main(int c, const char **v) +{ + or_options_t *options; + char *errmsg = NULL; + int i, i_out; + int loglevel = LOG_ERR; + +#ifdef USE_DMALLOC + { + int r = CRYPTO_set_mem_ex_functions(_tor_malloc, _tor_realloc, _tor_free); + tor_assert(r); + } +#endif + + update_approx_time(time(NULL)); + options = options_new(); + tor_threads_init(); + init_logging(); + + for (i_out = i = 1; i < c; ++i) { + if (!strcmp(v[i], "--warn")) { + loglevel = LOG_WARN; + } else if (!strcmp(v[i], "--notice")) { + loglevel = LOG_NOTICE; + } else if (!strcmp(v[i], "--info")) { + loglevel = LOG_INFO; + } else if (!strcmp(v[i], "--debug")) { + loglevel = LOG_DEBUG; + } else { + v[i_out++] = v[i]; + } + } + c = i_out; + + { + log_severity_list_t s; + memset(&s, 0, sizeof(s)); + set_log_severity_config(loglevel, LOG_ERR, &s); + add_stream_log(&s, "", fileno(stdout)); + } + + options->command = CMD_RUN_UNITTESTS; + crypto_global_init(0, NULL, NULL); + rep_hist_init(); + network_init(); + setup_directory(); + options_init(options); + options->DataDirectory = tor_strdup(temp_dir); + options->EntryStatistics = 1; + if (set_options(options, &errmsg) < 0) { + printf("Failed to set initial options: %s\n", errmsg); + tor_free(errmsg); + return 1; + } + + crypto_seed_rng(1); + + atexit(remove_directory); + + have_failed = (tinytest_main(c, v, testgroups) != 0); + + free_pregenerated_keys(); +#ifdef USE_DMALLOC + tor_free_all(0); + dmalloc_log_unfreed(); +#endif + + if (have_failed) + return 1; + else + return 0; +} + diff --git a/src/test/test.h b/src/test/test.h new file mode 100644 index 0000000000..f7ae46ce6d --- /dev/null +++ b/src/test/test.h @@ -0,0 +1,75 @@ +/* Copyright (c) 2001-2003, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef _TOR_TEST_H +#define _TOR_TEST_H + +/** + * \file test.h + * \brief Macros and functions used by unit tests. + */ + +#include "compat.h" +#include "tinytest.h" +#define TT_EXIT_TEST_FUNCTION STMT_BEGIN goto done; STMT_END +#include "tinytest_macros.h" + +#ifdef __GNUC__ +#define PRETTY_FUNCTION __PRETTY_FUNCTION__ +#else +#define PRETTY_FUNCTION "" +#endif + +#define test_fail_msg(msg) TT_DIE((msg)) + +#define test_fail() test_fail_msg("Assertion failed.") + +#define test_assert(expr) tt_assert(expr) + +#define test_eq(expr1, expr2) tt_int_op((expr1), ==, (expr2)) +#define test_eq_ptr(expr1, expr2) tt_ptr_op((expr1), ==, (expr2)) +#define test_neq(expr1, expr2) tt_int_op((expr1), !=, (expr2)) +#define test_neq_ptr(expr1, expr2) tt_ptr_op((expr1), !=, (expr2)) +#define test_streq(expr1, expr2) tt_str_op((expr1), ==, (expr2)) +#define test_strneq(expr1, expr2) tt_str_op((expr1), !=, (expr2)) +#define test_streq(expr1, expr2) tt_str_op((expr1), ==, (expr2)) + +#define test_mem_op(expr1, op, expr2, len) \ + tt_assert_test_fmt_type(expr1,expr2,#expr1" "#op" "#expr2, \ + const char *, \ + (memcmp(_val1, _val2, len) op 0), \ + char *, "%s", \ + { size_t printlen = (len)*2+1; \ + _print = tor_malloc(printlen); \ + base16_encode(_print, printlen, _value, \ + (len)); }, \ + { tor_free(_print); } \ + ); + +#define test_memeq(expr1, expr2, len) test_mem_op((expr1), ==, (expr2), len) +#define test_memneq(expr1, expr2, len) test_mem_op((expr1), !=, (expr2), len) + +/* As test_mem_op, but decodes 'hex' before comparing. There must be a + * local char* variable called mem_op_hex_tmp for this to work. */ +#define test_mem_op_hex(expr1, op, hex) \ + STMT_BEGIN \ + size_t length = strlen(hex); \ + tor_free(mem_op_hex_tmp); \ + mem_op_hex_tmp = tor_malloc(length/2); \ + tor_assert((length&1)==0); \ + base16_decode(mem_op_hex_tmp, length/2, hex, length); \ + test_mem_op(expr1, op, mem_op_hex_tmp, length/2); \ + STMT_END + +#define test_memeq_hex(expr1, hex) test_mem_op_hex(expr1, ==, hex) + +const char *get_fname(const char *name); +crypto_pk_env_t *pk_generate(int idx); + +void legacy_test_helper(void *data); +extern const struct testcase_setup_t legacy_setup; + +#endif + diff --git a/src/test/test_addr.c b/src/test/test_addr.c new file mode 100644 index 0000000000..6db4ee2483 --- /dev/null +++ b/src/test/test_addr.c @@ -0,0 +1,497 @@ +/* Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "orconfig.h" +#include "or.h" +#include "test.h" + +static void +test_addr_basic(void) +{ + uint32_t u32; + uint16_t u16; + char *cp; + + /* Test parse_addr_port */ + cp = NULL; u32 = 3; u16 = 3; + test_assert(!parse_addr_port(LOG_WARN, "1.2.3.4", &cp, &u32, &u16)); + test_streq(cp, "1.2.3.4"); + test_eq(u32, 0x01020304u); + test_eq(u16, 0); + tor_free(cp); + test_assert(!parse_addr_port(LOG_WARN, "4.3.2.1:99", &cp, &u32, &u16)); + test_streq(cp, "4.3.2.1"); + test_eq(u32, 0x04030201u); + test_eq(u16, 99); + tor_free(cp); + test_assert(!parse_addr_port(LOG_WARN, "nonexistent.address:4040", + &cp, NULL, &u16)); + test_streq(cp, "nonexistent.address"); + test_eq(u16, 4040); + tor_free(cp); + test_assert(!parse_addr_port(LOG_WARN, "localhost:9999", &cp, &u32, &u16)); + test_streq(cp, "localhost"); + test_eq(u32, 0x7f000001u); + test_eq(u16, 9999); + tor_free(cp); + u32 = 3; + test_assert(!parse_addr_port(LOG_WARN, "localhost", NULL, &u32, &u16)); + test_eq(cp, NULL); + test_eq(u32, 0x7f000001u); + test_eq(u16, 0); + tor_free(cp); + test_eq(0, addr_mask_get_bits(0x0u)); + test_eq(32, addr_mask_get_bits(0xFFFFFFFFu)); + test_eq(16, addr_mask_get_bits(0xFFFF0000u)); + test_eq(31, addr_mask_get_bits(0xFFFFFFFEu)); + test_eq(1, addr_mask_get_bits(0x80000000u)); + + /* Test inet_ntop */ + { + char tmpbuf[TOR_ADDR_BUF_LEN]; + const char *ip = "176.192.208.224"; + struct in_addr in; + tor_inet_pton(AF_INET, ip, &in); + tor_inet_ntop(AF_INET, &in, tmpbuf, sizeof(tmpbuf)); + test_streq(tmpbuf, ip); + } + + done: + ; +} + +#define _test_op_ip6(a,op,b,e1,e2) \ + STMT_BEGIN \ + tt_assert_test_fmt_type(a,b,e1" "#op" "e2,struct in6_addr*, \ + (memcmp(_val1->s6_addr, _val2->s6_addr, 16) op 0), \ + char *, "%s", \ + { int i; char *cp; \ + cp = _print = tor_malloc(64); \ + for (i=0;i<16;++i) { \ + tor_snprintf(cp, 3,"%02x", (unsigned)_value->s6_addr[i]);\ + cp += 2; \ + if (i != 15) *cp++ = ':'; \ + } \ + }, { tor_free(_print); } \ + ); \ + STMT_END + +/** Helper: Assert that two strings both decode as IPv6 addresses with + * tor_inet_pton(), and both decode to the same address. */ +#define test_pton6_same(a,b) STMT_BEGIN \ + test_eq(tor_inet_pton(AF_INET6, a, &a1), 1); \ + test_eq(tor_inet_pton(AF_INET6, b, &a2), 1); \ + _test_op_ip6(&a1,==,&a2,#a,#b); \ + STMT_END + +/** Helper: Assert that <b>a</b> is recognized as a bad IPv6 address by + * tor_inet_pton(). */ +#define test_pton6_bad(a) \ + test_eq(0, tor_inet_pton(AF_INET6, a, &a1)) + +/** Helper: assert that <b>a</b>, when parsed by tor_inet_pton() and displayed + * with tor_inet_ntop(), yields <b>b</b>. Also assert that <b>b</b> parses to + * the same value as <b>a</b>. */ +#define test_ntop6_reduces(a,b) STMT_BEGIN \ + test_eq(tor_inet_pton(AF_INET6, a, &a1), 1); \ + test_streq(tor_inet_ntop(AF_INET6, &a1, buf, sizeof(buf)), b); \ + test_eq(tor_inet_pton(AF_INET6, b, &a2), 1); \ + _test_op_ip6(&a1, ==, &a2, a, b); \ + STMT_END + +/** Helper: assert that <b>a</b> parses by tor_inet_pton() into a address that + * passes tor_addr_is_internal() with <b>for_listening</b>. */ +#define test_internal_ip(a,for_listening) STMT_BEGIN \ + test_eq(tor_inet_pton(AF_INET6, a, &t1.addr.in6_addr), 1); \ + t1.family = AF_INET6; \ + if (!tor_addr_is_internal(&t1, for_listening)) \ + test_fail_msg( a "was not internal."); \ + STMT_END + +/** Helper: assert that <b>a</b> parses by tor_inet_pton() into a address that + * does not pass tor_addr_is_internal() with <b>for_listening</b>. */ +#define test_external_ip(a,for_listening) STMT_BEGIN \ + test_eq(tor_inet_pton(AF_INET6, a, &t1.addr.in6_addr), 1); \ + t1.family = AF_INET6; \ + if (tor_addr_is_internal(&t1, for_listening)) \ + test_fail_msg(a "was not external."); \ + STMT_END + +/** Helper: Assert that <b>a</b> and <b>b</b>, when parsed by + * tor_inet_pton(), give addresses that compare in the order defined by + * <b>op</b> with tor_addr_compare(). */ +#define test_addr_compare(a, op, b) STMT_BEGIN \ + test_eq(tor_inet_pton(AF_INET6, a, &t1.addr.in6_addr), 1); \ + test_eq(tor_inet_pton(AF_INET6, b, &t2.addr.in6_addr), 1); \ + t1.family = t2.family = AF_INET6; \ + r = tor_addr_compare(&t1,&t2,CMP_SEMANTIC); \ + if (!(r op 0)) \ + test_fail_msg("failed: tor_addr_compare("a","b") "#op" 0"); \ + STMT_END + +/** Helper: Assert that <b>a</b> and <b>b</b>, when parsed by + * tor_inet_pton(), give addresses that compare in the order defined by + * <b>op</b> with tor_addr_compare_masked() with <b>m</b> masked. */ +#define test_addr_compare_masked(a, op, b, m) STMT_BEGIN \ + test_eq(tor_inet_pton(AF_INET6, a, &t1.addr.in6_addr), 1); \ + test_eq(tor_inet_pton(AF_INET6, b, &t2.addr.in6_addr), 1); \ + t1.family = t2.family = AF_INET6; \ + r = tor_addr_compare_masked(&t1,&t2,m,CMP_SEMANTIC); \ + if (!(r op 0)) \ + test_fail_msg("failed: tor_addr_compare_masked("a","b","#m") "#op" 0"); \ + STMT_END + +/** Helper: assert that <b>xx</b> is parseable as a masked IPv6 address with + * ports by tor_parse_mask_addr_ports(), with family <b>f</b>, IP address + * as 4 32-bit words <b>ip1...ip4</b>, mask bits as <b>mm</b>, and port range + * as <b>pt1..pt2</b>. */ +#define test_addr_mask_ports_parse(xx, f, ip1, ip2, ip3, ip4, mm, pt1, pt2) \ + STMT_BEGIN \ + test_eq(tor_addr_parse_mask_ports(xx, &t1, &mask, &port1, &port2), f); \ + p1=tor_inet_ntop(AF_INET6, &t1.addr.in6_addr, bug, sizeof(bug)); \ + test_eq(htonl(ip1), tor_addr_to_in6_addr32(&t1)[0]); \ + test_eq(htonl(ip2), tor_addr_to_in6_addr32(&t1)[1]); \ + test_eq(htonl(ip3), tor_addr_to_in6_addr32(&t1)[2]); \ + test_eq(htonl(ip4), tor_addr_to_in6_addr32(&t1)[3]); \ + test_eq(mask, mm); \ + test_eq(port1, pt1); \ + test_eq(port2, pt2); \ + STMT_END + +/** Run unit tests for IPv6 encoding/decoding/manipulation functions. */ +static void +test_addr_ip6_helpers(void) +{ + char buf[TOR_ADDR_BUF_LEN], bug[TOR_ADDR_BUF_LEN]; + struct in6_addr a1, a2; + tor_addr_t t1, t2; + int r, i; + uint16_t port1, port2; + maskbits_t mask; + const char *p1; + struct sockaddr_storage sa_storage; + struct sockaddr_in *sin; + struct sockaddr_in6 *sin6; + + // struct in_addr b1, b2; + /* Test tor_inet_ntop and tor_inet_pton: IPv6 */ + + /* ==== Converting to and from sockaddr_t. */ + sin = (struct sockaddr_in *)&sa_storage; + sin->sin_family = AF_INET; + sin->sin_port = 9090; + sin->sin_addr.s_addr = htonl(0x7f7f0102); /*127.127.1.2*/ + tor_addr_from_sockaddr(&t1, (struct sockaddr *)sin, NULL); + test_eq(tor_addr_family(&t1), AF_INET); + test_eq(tor_addr_to_ipv4h(&t1), 0x7f7f0102); + + memset(&sa_storage, 0, sizeof(sa_storage)); + test_eq(sizeof(struct sockaddr_in), + tor_addr_to_sockaddr(&t1, 1234, (struct sockaddr *)&sa_storage, + sizeof(sa_storage))); + test_eq(1234, ntohs(sin->sin_port)); + test_eq(0x7f7f0102, ntohl(sin->sin_addr.s_addr)); + + memset(&sa_storage, 0, sizeof(sa_storage)); + sin6 = (struct sockaddr_in6 *)&sa_storage; + sin6->sin6_family = AF_INET6; + sin6->sin6_port = htons(7070); + sin6->sin6_addr.s6_addr[0] = 128; + tor_addr_from_sockaddr(&t1, (struct sockaddr *)sin6, NULL); + test_eq(tor_addr_family(&t1), AF_INET6); + p1 = tor_addr_to_str(buf, &t1, sizeof(buf), 0); + test_streq(p1, "8000::"); + + memset(&sa_storage, 0, sizeof(sa_storage)); + test_eq(sizeof(struct sockaddr_in6), + tor_addr_to_sockaddr(&t1, 9999, (struct sockaddr *)&sa_storage, + sizeof(sa_storage))); + test_eq(AF_INET6, sin6->sin6_family); + test_eq(9999, ntohs(sin6->sin6_port)); + test_eq(0x80000000, ntohl(S6_ADDR32(sin6->sin6_addr)[0])); + + /* ==== tor_addr_lookup: static cases. (Can't test dns without knowing we + * have a good resolver. */ + test_eq(0, tor_addr_lookup("127.128.129.130", AF_UNSPEC, &t1)); + test_eq(AF_INET, tor_addr_family(&t1)); + test_eq(tor_addr_to_ipv4h(&t1), 0x7f808182); + + test_eq(0, tor_addr_lookup("9000::5", AF_UNSPEC, &t1)); + test_eq(AF_INET6, tor_addr_family(&t1)); + test_eq(0x90, tor_addr_to_in6_addr8(&t1)[0]); + test_assert(tor_mem_is_zero((char*)tor_addr_to_in6_addr8(&t1)+1, 14)); + test_eq(0x05, tor_addr_to_in6_addr8(&t1)[15]); + + /* === Test pton: valid af_inet6 */ + /* Simple, valid parsing. */ + r = tor_inet_pton(AF_INET6, + "0102:0304:0506:0708:090A:0B0C:0D0E:0F10", &a1); + test_assert(r==1); + for (i=0;i<16;++i) { test_eq(i+1, (int)a1.s6_addr[i]); } + /* ipv4 ending. */ + test_pton6_same("0102:0304:0506:0708:090A:0B0C:0D0E:0F10", + "0102:0304:0506:0708:090A:0B0C:13.14.15.16"); + /* shortened words. */ + test_pton6_same("0001:0099:BEEF:0000:0123:FFFF:0001:0001", + "1:99:BEEF:0:0123:FFFF:1:1"); + /* zeros at the beginning */ + test_pton6_same("0000:0000:0000:0000:0009:C0A8:0001:0001", + "::9:c0a8:1:1"); + test_pton6_same("0000:0000:0000:0000:0009:C0A8:0001:0001", + "::9:c0a8:0.1.0.1"); + /* zeros in the middle. */ + test_pton6_same("fe80:0000:0000:0000:0202:1111:0001:0001", + "fe80::202:1111:1:1"); + /* zeros at the end. */ + test_pton6_same("1000:0001:0000:0007:0000:0000:0000:0000", + "1000:1:0:7::"); + + /* === Test ntop: af_inet6 */ + test_ntop6_reduces("0:0:0:0:0:0:0:0", "::"); + + test_ntop6_reduces("0001:0099:BEEF:0006:0123:FFFF:0001:0001", + "1:99:beef:6:123:ffff:1:1"); + + //test_ntop6_reduces("0:0:0:0:0:0:c0a8:0101", "::192.168.1.1"); + test_ntop6_reduces("0:0:0:0:0:ffff:c0a8:0101", "::ffff:192.168.1.1"); + test_ntop6_reduces("002:0:0000:0:3::4", "2::3:0:0:4"); + test_ntop6_reduces("0:0::1:0:3", "::1:0:3"); + test_ntop6_reduces("008:0::0", "8::"); + test_ntop6_reduces("0:0:0:0:0:ffff::1", "::ffff:0.0.0.1"); + test_ntop6_reduces("abcd:0:0:0:0:0:7f00::", "abcd::7f00:0"); + test_ntop6_reduces("0000:0000:0000:0000:0009:C0A8:0001:0001", + "::9:c0a8:1:1"); + test_ntop6_reduces("fe80:0000:0000:0000:0202:1111:0001:0001", + "fe80::202:1111:1:1"); + test_ntop6_reduces("1000:0001:0000:0007:0000:0000:0000:0000", + "1000:1:0:7::"); + + /* === Test pton: invalid in6. */ + test_pton6_bad("foobar."); + test_pton6_bad("55555::"); + test_pton6_bad("9:-60::"); + test_pton6_bad("1:2:33333:4:0002:3::"); + //test_pton6_bad("1:2:3333:4:00002:3::");// BAD, but glibc doesn't say so. + test_pton6_bad("1:2:3333:4:fish:3::"); + test_pton6_bad("1:2:3:4:5:6:7:8:9"); + test_pton6_bad("1:2:3:4:5:6:7"); + test_pton6_bad("1:2:3:4:5:6:1.2.3.4.5"); + test_pton6_bad("1:2:3:4:5:6:1.2.3"); + test_pton6_bad("::1.2.3"); + test_pton6_bad("::1.2.3.4.5"); + test_pton6_bad("99"); + test_pton6_bad(""); + test_pton6_bad("1::2::3:4"); + test_pton6_bad("a:::b:c"); + test_pton6_bad(":::a:b:c"); + test_pton6_bad("a:b:c:::"); + + /* test internal checking */ + test_external_ip("fbff:ffff::2:7", 0); + test_internal_ip("fc01::2:7", 0); + test_internal_ip("fdff:ffff::f:f", 0); + test_external_ip("fe00::3:f", 0); + + test_external_ip("fe7f:ffff::2:7", 0); + test_internal_ip("fe80::2:7", 0); + test_internal_ip("febf:ffff::f:f", 0); + + test_internal_ip("fec0::2:7:7", 0); + test_internal_ip("feff:ffff::e:7:7", 0); + test_external_ip("ff00::e:7:7", 0); + + test_internal_ip("::", 0); + test_internal_ip("::1", 0); + test_internal_ip("::1", 1); + test_internal_ip("::", 0); + test_external_ip("::", 1); + test_external_ip("::2", 0); + test_external_ip("2001::", 0); + test_external_ip("ffff::", 0); + + test_external_ip("::ffff:0.0.0.0", 1); + test_internal_ip("::ffff:0.0.0.0", 0); + test_internal_ip("::ffff:0.255.255.255", 0); + test_external_ip("::ffff:1.0.0.0", 0); + + test_external_ip("::ffff:9.255.255.255", 0); + test_internal_ip("::ffff:10.0.0.0", 0); + test_internal_ip("::ffff:10.255.255.255", 0); + test_external_ip("::ffff:11.0.0.0", 0); + + test_external_ip("::ffff:126.255.255.255", 0); + test_internal_ip("::ffff:127.0.0.0", 0); + test_internal_ip("::ffff:127.255.255.255", 0); + test_external_ip("::ffff:128.0.0.0", 0); + + test_external_ip("::ffff:172.15.255.255", 0); + test_internal_ip("::ffff:172.16.0.0", 0); + test_internal_ip("::ffff:172.31.255.255", 0); + test_external_ip("::ffff:172.32.0.0", 0); + + test_external_ip("::ffff:192.167.255.255", 0); + test_internal_ip("::ffff:192.168.0.0", 0); + test_internal_ip("::ffff:192.168.255.255", 0); + test_external_ip("::ffff:192.169.0.0", 0); + + test_external_ip("::ffff:169.253.255.255", 0); + test_internal_ip("::ffff:169.254.0.0", 0); + test_internal_ip("::ffff:169.254.255.255", 0); + test_external_ip("::ffff:169.255.0.0", 0); + test_assert(is_internal_IP(0x7f000001, 0)); + + /* tor_addr_compare(tor_addr_t x2) */ + test_addr_compare("ffff::", ==, "ffff::0"); + test_addr_compare("0::3:2:1", <, "0::ffff:0.3.2.1"); + test_addr_compare("0::2:2:1", <, "0::ffff:0.3.2.1"); + test_addr_compare("0::ffff:0.3.2.1", >, "0::0:0:0"); + test_addr_compare("0::ffff:5.2.2.1", <, "::ffff:6.0.0.0"); /* XXXX wrong. */ + tor_addr_parse_mask_ports("[::ffff:2.3.4.5]", &t1, NULL, NULL, NULL); + tor_addr_parse_mask_ports("2.3.4.5", &t2, NULL, NULL, NULL); + test_assert(tor_addr_compare(&t1, &t2, CMP_SEMANTIC) == 0); + tor_addr_parse_mask_ports("[::ffff:2.3.4.4]", &t1, NULL, NULL, NULL); + tor_addr_parse_mask_ports("2.3.4.5", &t2, NULL, NULL, NULL); + test_assert(tor_addr_compare(&t1, &t2, CMP_SEMANTIC) < 0); + + /* test compare_masked */ + test_addr_compare_masked("ffff::", ==, "ffff::0", 128); + test_addr_compare_masked("ffff::", ==, "ffff::0", 64); + test_addr_compare_masked("0::2:2:1", <, "0::8000:2:1", 81); + test_addr_compare_masked("0::2:2:1", ==, "0::8000:2:1", 80); + + /* Test decorated addr_to_string. */ + test_eq(AF_INET6, tor_addr_from_str(&t1, "[123:45:6789::5005:11]")); + p1 = tor_addr_to_str(buf, &t1, sizeof(buf), 1); + test_streq(p1, "[123:45:6789::5005:11]"); + test_eq(AF_INET, tor_addr_from_str(&t1, "18.0.0.1")); + p1 = tor_addr_to_str(buf, &t1, sizeof(buf), 1); + test_streq(p1, "18.0.0.1"); + + /* Test tor_addr_parse_reverse_lookup_name */ + i = tor_addr_parse_reverse_lookup_name(&t1, "Foobar.baz", AF_UNSPEC, 0); + test_eq(0, i); + i = tor_addr_parse_reverse_lookup_name(&t1, "Foobar.baz", AF_UNSPEC, 1); + test_eq(0, i); + i = tor_addr_parse_reverse_lookup_name(&t1, "1.0.168.192.in-addr.arpa", + AF_UNSPEC, 1); + test_eq(1, i); + test_eq(tor_addr_family(&t1), AF_INET); + p1 = tor_addr_to_str(buf, &t1, sizeof(buf), 1); + test_streq(p1, "192.168.0.1"); + i = tor_addr_parse_reverse_lookup_name(&t1, "192.168.0.99", AF_UNSPEC, 0); + test_eq(0, i); + i = tor_addr_parse_reverse_lookup_name(&t1, "192.168.0.99", AF_UNSPEC, 1); + test_eq(1, i); + p1 = tor_addr_to_str(buf, &t1, sizeof(buf), 1); + test_streq(p1, "192.168.0.99"); + memset(&t1, 0, sizeof(t1)); + i = tor_addr_parse_reverse_lookup_name(&t1, + "0.1.2.3.4.5.6.7.8.9.a.b.c.d.e.f." + "f.e.e.b.1.e.b.e.e.f.f.e.e.e.d.9." + "ip6.ARPA", + AF_UNSPEC, 0); + test_eq(1, i); + p1 = tor_addr_to_str(buf, &t1, sizeof(buf), 1); + test_streq(p1, "[9dee:effe:ebe1:beef:fedc:ba98:7654:3210]"); + /* Failing cases. */ + i = tor_addr_parse_reverse_lookup_name(&t1, + "6.7.8.9.a.b.c.d.e.f." + "f.e.e.b.1.e.b.e.e.f.f.e.e.e.d.9." + "ip6.ARPA", + AF_UNSPEC, 0); + test_eq(i, -1); + i = tor_addr_parse_reverse_lookup_name(&t1, + "6.7.8.9.a.b.c.d.e.f.a.b.c.d.e.f.0." + "f.e.e.b.1.e.b.e.e.f.f.e.e.e.d.9." + "ip6.ARPA", + AF_UNSPEC, 0); + test_eq(i, -1); + i = tor_addr_parse_reverse_lookup_name(&t1, + "6.7.8.9.a.b.c.d.e.f.X.0.0.0.0.9." + "f.e.e.b.1.e.b.e.e.f.f.e.e.e.d.9." + "ip6.ARPA", + AF_UNSPEC, 0); + test_eq(i, -1); + i = tor_addr_parse_reverse_lookup_name(&t1, "32.1.1.in-addr.arpa", + AF_UNSPEC, 0); + test_eq(i, -1); + i = tor_addr_parse_reverse_lookup_name(&t1, ".in-addr.arpa", + AF_UNSPEC, 0); + test_eq(i, -1); + i = tor_addr_parse_reverse_lookup_name(&t1, "1.2.3.4.5.in-addr.arpa", + AF_UNSPEC, 0); + test_eq(i, -1); + i = tor_addr_parse_reverse_lookup_name(&t1, "1.2.3.4.5.in-addr.arpa", + AF_INET6, 0); + test_eq(i, -1); + i = tor_addr_parse_reverse_lookup_name(&t1, + "6.7.8.9.a.b.c.d.e.f.a.b.c.d.e.0." + "f.e.e.b.1.e.b.e.e.f.f.e.e.e.d.9." + "ip6.ARPA", + AF_INET, 0); + test_eq(i, -1); + + /* test tor_addr_parse_mask_ports */ + test_addr_mask_ports_parse("[::f]/17:47-95", AF_INET6, + 0, 0, 0, 0x0000000f, 17, 47, 95); + //test_addr_parse("[::fefe:4.1.1.7/120]:999-1000"); + //test_addr_parse_check("::fefe:401:107", 120, 999, 1000); + test_addr_mask_ports_parse("[::ffff:4.1.1.7]/120:443", AF_INET6, + 0, 0, 0x0000ffff, 0x04010107, 120, 443, 443); + test_addr_mask_ports_parse("[abcd:2::44a:0]:2-65000", AF_INET6, + 0xabcd0002, 0, 0, 0x044a0000, 128, 2, 65000); + + r=tor_addr_parse_mask_ports("[fefef::]/112", &t1, NULL, NULL, NULL); + test_assert(r == -1); + r=tor_addr_parse_mask_ports("efef::/112", &t1, NULL, NULL, NULL); + test_assert(r == -1); + r=tor_addr_parse_mask_ports("[f:f:f:f:f:f:f:f::]", &t1, NULL, NULL, NULL); + test_assert(r == -1); + r=tor_addr_parse_mask_ports("[::f:f:f:f:f:f:f:f]", &t1, NULL, NULL, NULL); + test_assert(r == -1); + r=tor_addr_parse_mask_ports("[f:f:f:f:f:f:f:f:f]", &t1, NULL, NULL, NULL); + test_assert(r == -1); + /* Test for V4-mapped address with mask < 96. (arguably not valid) */ + r=tor_addr_parse_mask_ports("[::ffff:1.1.2.2/33]", &t1, &mask, NULL, NULL); + test_assert(r == -1); + r=tor_addr_parse_mask_ports("1.1.2.2/33", &t1, &mask, NULL, NULL); + test_assert(r == -1); + r=tor_addr_parse_mask_ports("1.1.2.2/31", &t1, &mask, NULL, NULL); + test_assert(r == AF_INET); + r=tor_addr_parse_mask_ports("[efef::]/112", &t1, &mask, &port1, &port2); + test_assert(r == AF_INET6); + test_assert(port1 == 1); + test_assert(port2 == 65535); + + /* make sure inet address lengths >= max */ + test_assert(INET_NTOA_BUF_LEN >= sizeof("255.255.255.255")); + test_assert(TOR_ADDR_BUF_LEN >= + sizeof("ffff:ffff:ffff:ffff:ffff:ffff:255.255.255.255")); + + test_assert(sizeof(tor_addr_t) >= sizeof(struct in6_addr)); + + /* get interface addresses */ + r = get_interface_address6(LOG_DEBUG, AF_INET, &t1); + i = get_interface_address6(LOG_DEBUG, AF_INET6, &t2); +#if 0 + tor_inet_ntop(AF_INET, &t1.sa.sin_addr, buf, sizeof(buf)); + printf("\nv4 address: %s (family=%d)", buf, IN_FAMILY(&t1)); + tor_inet_ntop(AF_INET6, &t2.sa6.sin6_addr, buf, sizeof(buf)); + printf("\nv6 address: %s (family=%d)", buf, IN_FAMILY(&t2)); +#endif + + done: + ; +} + +#define ADDR_LEGACY(name) \ + { #name, legacy_test_helper, 0, &legacy_setup, test_addr_ ## name } + +struct testcase_t addr_tests[] = { + ADDR_LEGACY(basic), + ADDR_LEGACY(ip6_helpers), + END_OF_TESTCASES +}; + diff --git a/src/test/test_containers.c b/src/test/test_containers.c new file mode 100644 index 0000000000..af9fb1c5c9 --- /dev/null +++ b/src/test/test_containers.c @@ -0,0 +1,765 @@ +/* Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "orconfig.h" +#include "or.h" +#include "test.h" + +/** Helper: return a tristate based on comparing the strings in *<b>a</b> and + * *<b>b</b>. */ +static int +_compare_strs(const void **a, const void **b) +{ + const char *s1 = *a, *s2 = *b; + return strcmp(s1, s2); +} + +/** Helper: return a tristate based on comparing the strings in *<b>a</b> and + * *<b>b</b>, excluding a's first character, and ignoring case. */ +static int +_compare_without_first_ch(const void *a, const void **b) +{ + const char *s1 = a, *s2 = *b; + return strcasecmp(s1+1, s2); +} + +/** Run unit tests for basic dynamic-sized array functionality. */ +static void +test_container_smartlist_basic(void) +{ + smartlist_t *sl; + + /* XXXX test sort_digests, uniq_strings, uniq_digests */ + + /* Test smartlist add, del_keeporder, insert, get. */ + sl = smartlist_create(); + smartlist_add(sl, (void*)1); + smartlist_add(sl, (void*)2); + smartlist_add(sl, (void*)3); + smartlist_add(sl, (void*)4); + smartlist_del_keeporder(sl, 1); + smartlist_insert(sl, 1, (void*)22); + smartlist_insert(sl, 0, (void*)0); + smartlist_insert(sl, 5, (void*)555); + test_eq_ptr((void*)0, smartlist_get(sl,0)); + test_eq_ptr((void*)1, smartlist_get(sl,1)); + test_eq_ptr((void*)22, smartlist_get(sl,2)); + test_eq_ptr((void*)3, smartlist_get(sl,3)); + test_eq_ptr((void*)4, smartlist_get(sl,4)); + test_eq_ptr((void*)555, smartlist_get(sl,5)); + /* Try deleting in the middle. */ + smartlist_del(sl, 1); + test_eq_ptr((void*)555, smartlist_get(sl, 1)); + /* Try deleting at the end. */ + smartlist_del(sl, 4); + test_eq(4, smartlist_len(sl)); + + /* test isin. */ + test_assert(smartlist_isin(sl, (void*)3)); + test_assert(!smartlist_isin(sl, (void*)99)); + + done: + smartlist_free(sl); +} + +/** Run unit tests for smartlist-of-strings functionality. */ +static void +test_container_smartlist_strings(void) +{ + smartlist_t *sl = smartlist_create(); + char *cp=NULL, *cp_alloc=NULL; + size_t sz; + + /* Test split and join */ + test_eq(0, smartlist_len(sl)); + smartlist_split_string(sl, "abc", ":", 0, 0); + test_eq(1, smartlist_len(sl)); + test_streq("abc", smartlist_get(sl, 0)); + smartlist_split_string(sl, "a::bc::", "::", 0, 0); + test_eq(4, smartlist_len(sl)); + test_streq("a", smartlist_get(sl, 1)); + test_streq("bc", smartlist_get(sl, 2)); + test_streq("", smartlist_get(sl, 3)); + cp_alloc = smartlist_join_strings(sl, "", 0, NULL); + test_streq(cp_alloc, "abcabc"); + tor_free(cp_alloc); + cp_alloc = smartlist_join_strings(sl, "!", 0, NULL); + test_streq(cp_alloc, "abc!a!bc!"); + tor_free(cp_alloc); + cp_alloc = smartlist_join_strings(sl, "XY", 0, NULL); + test_streq(cp_alloc, "abcXYaXYbcXY"); + tor_free(cp_alloc); + cp_alloc = smartlist_join_strings(sl, "XY", 1, NULL); + test_streq(cp_alloc, "abcXYaXYbcXYXY"); + tor_free(cp_alloc); + cp_alloc = smartlist_join_strings(sl, "", 1, NULL); + test_streq(cp_alloc, "abcabc"); + tor_free(cp_alloc); + + smartlist_split_string(sl, "/def/ /ghijk", "/", 0, 0); + test_eq(8, smartlist_len(sl)); + test_streq("", smartlist_get(sl, 4)); + test_streq("def", smartlist_get(sl, 5)); + test_streq(" ", smartlist_get(sl, 6)); + test_streq("ghijk", smartlist_get(sl, 7)); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + smartlist_split_string(sl, "a,bbd,cdef", ",", SPLIT_SKIP_SPACE, 0); + test_eq(3, smartlist_len(sl)); + test_streq("a", smartlist_get(sl,0)); + test_streq("bbd", smartlist_get(sl,1)); + test_streq("cdef", smartlist_get(sl,2)); + smartlist_split_string(sl, " z <> zhasd <> <> bnud<> ", "<>", + SPLIT_SKIP_SPACE, 0); + test_eq(8, smartlist_len(sl)); + test_streq("z", smartlist_get(sl,3)); + test_streq("zhasd", smartlist_get(sl,4)); + test_streq("", smartlist_get(sl,5)); + test_streq("bnud", smartlist_get(sl,6)); + test_streq("", smartlist_get(sl,7)); + + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + smartlist_split_string(sl, " ab\tc \td ef ", NULL, + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); + test_eq(4, smartlist_len(sl)); + test_streq("ab", smartlist_get(sl,0)); + test_streq("c", smartlist_get(sl,1)); + test_streq("d", smartlist_get(sl,2)); + test_streq("ef", smartlist_get(sl,3)); + smartlist_split_string(sl, "ghi\tj", NULL, + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); + test_eq(6, smartlist_len(sl)); + test_streq("ghi", smartlist_get(sl,4)); + test_streq("j", smartlist_get(sl,5)); + + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + cp_alloc = smartlist_join_strings(sl, "XY", 0, NULL); + test_streq(cp_alloc, ""); + tor_free(cp_alloc); + cp_alloc = smartlist_join_strings(sl, "XY", 1, NULL); + test_streq(cp_alloc, "XY"); + tor_free(cp_alloc); + + smartlist_split_string(sl, " z <> zhasd <> <> bnud<> ", "<>", + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); + test_eq(3, smartlist_len(sl)); + test_streq("z", smartlist_get(sl, 0)); + test_streq("zhasd", smartlist_get(sl, 1)); + test_streq("bnud", smartlist_get(sl, 2)); + smartlist_split_string(sl, " z <> zhasd <> <> bnud<> ", "<>", + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 2); + test_eq(5, smartlist_len(sl)); + test_streq("z", smartlist_get(sl, 3)); + test_streq("zhasd <> <> bnud<>", smartlist_get(sl, 4)); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + smartlist_split_string(sl, "abcd\n", "\n", + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); + test_eq(1, smartlist_len(sl)); + test_streq("abcd", smartlist_get(sl, 0)); + smartlist_split_string(sl, "efgh", "\n", + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); + test_eq(2, smartlist_len(sl)); + test_streq("efgh", smartlist_get(sl, 1)); + + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + /* Test swapping, shuffling, and sorting. */ + smartlist_split_string(sl, "the,onion,router,by,arma,and,nickm", ",", 0, 0); + test_eq(7, smartlist_len(sl)); + smartlist_sort(sl, _compare_strs); + cp_alloc = smartlist_join_strings(sl, ",", 0, NULL); + test_streq(cp_alloc,"and,arma,by,nickm,onion,router,the"); + tor_free(cp_alloc); + smartlist_swap(sl, 1, 5); + cp_alloc = smartlist_join_strings(sl, ",", 0, NULL); + test_streq(cp_alloc,"and,router,by,nickm,onion,arma,the"); + tor_free(cp_alloc); + smartlist_shuffle(sl); + test_eq(7, smartlist_len(sl)); + test_assert(smartlist_string_isin(sl, "and")); + test_assert(smartlist_string_isin(sl, "router")); + test_assert(smartlist_string_isin(sl, "by")); + test_assert(smartlist_string_isin(sl, "nickm")); + test_assert(smartlist_string_isin(sl, "onion")); + test_assert(smartlist_string_isin(sl, "arma")); + test_assert(smartlist_string_isin(sl, "the")); + + /* Test bsearch. */ + smartlist_sort(sl, _compare_strs); + test_streq("nickm", smartlist_bsearch(sl, "zNicKM", + _compare_without_first_ch)); + test_streq("and", smartlist_bsearch(sl, " AND", _compare_without_first_ch)); + test_eq_ptr(NULL, smartlist_bsearch(sl, " ANz", _compare_without_first_ch)); + + /* Test bsearch_idx */ + { + int f; + test_eq(0, smartlist_bsearch_idx(sl," aaa",_compare_without_first_ch,&f)); + test_eq(f, 0); + test_eq(0, smartlist_bsearch_idx(sl," and",_compare_without_first_ch,&f)); + test_eq(f, 1); + test_eq(1, smartlist_bsearch_idx(sl," arm",_compare_without_first_ch,&f)); + test_eq(f, 0); + test_eq(1, smartlist_bsearch_idx(sl," arma",_compare_without_first_ch,&f)); + test_eq(f, 1); + test_eq(2, smartlist_bsearch_idx(sl," armb",_compare_without_first_ch,&f)); + test_eq(f, 0); + test_eq(7, smartlist_bsearch_idx(sl," zzzz",_compare_without_first_ch,&f)); + test_eq(f, 0); + } + + /* Test reverse() and pop_last() */ + smartlist_reverse(sl); + cp_alloc = smartlist_join_strings(sl, ",", 0, NULL); + test_streq(cp_alloc,"the,router,onion,nickm,by,arma,and"); + tor_free(cp_alloc); + cp_alloc = smartlist_pop_last(sl); + test_streq(cp_alloc, "and"); + tor_free(cp_alloc); + test_eq(smartlist_len(sl), 6); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + cp_alloc = smartlist_pop_last(sl); + test_eq(cp_alloc, NULL); + + /* Test uniq() */ + smartlist_split_string(sl, + "50,noon,radar,a,man,a,plan,a,canal,panama,radar,noon,50", + ",", 0, 0); + smartlist_sort(sl, _compare_strs); + smartlist_uniq(sl, _compare_strs, _tor_free); + cp_alloc = smartlist_join_strings(sl, ",", 0, NULL); + test_streq(cp_alloc, "50,a,canal,man,noon,panama,plan,radar"); + tor_free(cp_alloc); + + /* Test string_isin and isin_case and num_isin */ + test_assert(smartlist_string_isin(sl, "noon")); + test_assert(!smartlist_string_isin(sl, "noonoon")); + test_assert(smartlist_string_isin_case(sl, "nOOn")); + test_assert(!smartlist_string_isin_case(sl, "nooNooN")); + test_assert(smartlist_string_num_isin(sl, 50)); + test_assert(!smartlist_string_num_isin(sl, 60)); + + /* Test smartlist_choose */ + { + int i; + int allsame = 1; + int allin = 1; + void *first = smartlist_choose(sl); + test_assert(smartlist_isin(sl, first)); + for (i = 0; i < 100; ++i) { + void *second = smartlist_choose(sl); + if (second != first) + allsame = 0; + if (!smartlist_isin(sl, second)) + allin = 0; + } + test_assert(!allsame); + test_assert(allin); + } + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + /* Test string_remove and remove and join_strings2 */ + smartlist_split_string(sl, + "Some say the Earth will end in ice and some in fire", + " ", 0, 0); + cp = smartlist_get(sl, 4); + test_streq(cp, "will"); + smartlist_add(sl, cp); + smartlist_remove(sl, cp); + tor_free(cp); + cp_alloc = smartlist_join_strings(sl, ",", 0, NULL); + test_streq(cp_alloc, "Some,say,the,Earth,fire,end,in,ice,and,some,in"); + tor_free(cp_alloc); + smartlist_string_remove(sl, "in"); + cp_alloc = smartlist_join_strings2(sl, "+XX", 1, 0, &sz); + test_streq(cp_alloc, "Some+say+the+Earth+fire+end+some+ice+and"); + test_eq((int)sz, 40); + + done: + + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_free(sl); + tor_free(cp_alloc); +} + +/** Run unit tests for smartlist set manipulation functions. */ +static void +test_container_smartlist_overlap(void) +{ + smartlist_t *sl = smartlist_create(); + smartlist_t *ints = smartlist_create(); + smartlist_t *odds = smartlist_create(); + smartlist_t *evens = smartlist_create(); + smartlist_t *primes = smartlist_create(); + int i; + for (i=1; i < 10; i += 2) + smartlist_add(odds, (void*)(uintptr_t)i); + for (i=0; i < 10; i += 2) + smartlist_add(evens, (void*)(uintptr_t)i); + + /* add_all */ + smartlist_add_all(ints, odds); + smartlist_add_all(ints, evens); + test_eq(smartlist_len(ints), 10); + + smartlist_add(primes, (void*)2); + smartlist_add(primes, (void*)3); + smartlist_add(primes, (void*)5); + smartlist_add(primes, (void*)7); + + /* overlap */ + test_assert(smartlist_overlap(ints, odds)); + test_assert(smartlist_overlap(odds, primes)); + test_assert(smartlist_overlap(evens, primes)); + test_assert(!smartlist_overlap(odds, evens)); + + /* intersect */ + smartlist_add_all(sl, odds); + smartlist_intersect(sl, primes); + test_eq(smartlist_len(sl), 3); + test_assert(smartlist_isin(sl, (void*)3)); + test_assert(smartlist_isin(sl, (void*)5)); + test_assert(smartlist_isin(sl, (void*)7)); + + /* subtract */ + smartlist_add_all(sl, primes); + smartlist_subtract(sl, odds); + test_eq(smartlist_len(sl), 1); + test_assert(smartlist_isin(sl, (void*)2)); + + done: + smartlist_free(odds); + smartlist_free(evens); + smartlist_free(ints); + smartlist_free(primes); + smartlist_free(sl); +} + +/** Run unit tests for smartlist-of-digests functions. */ +static void +test_container_smartlist_digests(void) +{ + smartlist_t *sl = smartlist_create(); + + /* digest_isin. */ + smartlist_add(sl, tor_memdup("AAAAAAAAAAAAAAAAAAAA", DIGEST_LEN)); + smartlist_add(sl, tor_memdup("\00090AAB2AAAAaasdAAAAA", DIGEST_LEN)); + smartlist_add(sl, tor_memdup("\00090AAB2AAAAaasdAAAAA", DIGEST_LEN)); + test_eq(0, smartlist_digest_isin(NULL, "AAAAAAAAAAAAAAAAAAAA")); + test_assert(smartlist_digest_isin(sl, "AAAAAAAAAAAAAAAAAAAA")); + test_assert(smartlist_digest_isin(sl, "\00090AAB2AAAAaasdAAAAA")); + test_eq(0, smartlist_digest_isin(sl, "\00090AAB2AAABaasdAAAAA")); + + /* sort digests */ + smartlist_sort_digests(sl); + test_memeq(smartlist_get(sl, 0), "\00090AAB2AAAAaasdAAAAA", DIGEST_LEN); + test_memeq(smartlist_get(sl, 1), "\00090AAB2AAAAaasdAAAAA", DIGEST_LEN); + test_memeq(smartlist_get(sl, 2), "AAAAAAAAAAAAAAAAAAAA", DIGEST_LEN); + test_eq(3, smartlist_len(sl)); + + /* uniq_digests */ + smartlist_uniq_digests(sl); + test_eq(2, smartlist_len(sl)); + test_memeq(smartlist_get(sl, 0), "\00090AAB2AAAAaasdAAAAA", DIGEST_LEN); + test_memeq(smartlist_get(sl, 1), "AAAAAAAAAAAAAAAAAAAA", DIGEST_LEN); + + done: + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_free(sl); +} + +/** Run unit tests for concatenate-a-smartlist-of-strings functions. */ +static void +test_container_smartlist_join(void) +{ + smartlist_t *sl = smartlist_create(); + smartlist_t *sl2 = smartlist_create(), *sl3 = smartlist_create(), + *sl4 = smartlist_create(); + char *joined=NULL; + /* unique, sorted. */ + smartlist_split_string(sl, + "Abashments Ambush Anchorman Bacon Banks Borscht " + "Bunks Inhumane Insurance Knish Know Manners " + "Maraschinos Stamina Sunbonnets Unicorns Wombats", + " ", 0, 0); + /* non-unique, sorted. */ + smartlist_split_string(sl2, + "Ambush Anchorman Anchorman Anemias Anemias Bacon " + "Crossbowmen Inhumane Insurance Knish Know Manners " + "Manners Maraschinos Wombats Wombats Work", + " ", 0, 0); + SMARTLIST_FOREACH_JOIN(sl, char *, cp1, + sl2, char *, cp2, + strcmp(cp1,cp2), + smartlist_add(sl3, cp2)) { + test_streq(cp1, cp2); + smartlist_add(sl4, cp1); + } SMARTLIST_FOREACH_JOIN_END(cp1, cp2); + + SMARTLIST_FOREACH(sl3, const char *, cp, + test_assert(smartlist_isin(sl2, cp) && + !smartlist_string_isin(sl, cp))); + SMARTLIST_FOREACH(sl4, const char *, cp, + test_assert(smartlist_isin(sl, cp) && + smartlist_string_isin(sl2, cp))); + joined = smartlist_join_strings(sl3, ",", 0, NULL); + test_streq(joined, "Anemias,Anemias,Crossbowmen,Work"); + tor_free(joined); + joined = smartlist_join_strings(sl4, ",", 0, NULL); + test_streq(joined, "Ambush,Anchorman,Anchorman,Bacon,Inhumane,Insurance," + "Knish,Know,Manners,Manners,Maraschinos,Wombats,Wombats"); + tor_free(joined); + + done: + smartlist_free(sl4); + smartlist_free(sl3); + SMARTLIST_FOREACH(sl2, char *, cp, tor_free(cp)); + smartlist_free(sl2); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_free(sl); + tor_free(joined); +} + +/** Run unit tests for bitarray code */ +static void +test_container_bitarray(void) +{ + bitarray_t *ba = NULL; + int i, j, ok=1; + + ba = bitarray_init_zero(1); + test_assert(ba); + test_assert(! bitarray_is_set(ba, 0)); + bitarray_set(ba, 0); + test_assert(bitarray_is_set(ba, 0)); + bitarray_clear(ba, 0); + test_assert(! bitarray_is_set(ba, 0)); + bitarray_free(ba); + + ba = bitarray_init_zero(1023); + for (i = 1; i < 64; ) { + for (j = 0; j < 1023; ++j) { + if (j % i) + bitarray_set(ba, j); + else + bitarray_clear(ba, j); + } + for (j = 0; j < 1023; ++j) { + if (!bool_eq(bitarray_is_set(ba, j), j%i)) + ok = 0; + } + test_assert(ok); + if (i < 7) + ++i; + else if (i == 28) + i = 32; + else + i += 7; + } + + done: + if (ba) + bitarray_free(ba); +} + +/** Run unit tests for digest set code (implemented as a hashtable or as a + * bloom filter) */ +static void +test_container_digestset(void) +{ + smartlist_t *included = smartlist_create(); + char d[DIGEST_LEN]; + int i; + int ok = 1; + int false_positives = 0; + digestset_t *set = NULL; + + for (i = 0; i < 1000; ++i) { + crypto_rand(d, DIGEST_LEN); + smartlist_add(included, tor_memdup(d, DIGEST_LEN)); + } + set = digestset_new(1000); + SMARTLIST_FOREACH(included, const char *, cp, + if (digestset_isin(set, cp)) + ok = 0); + test_assert(ok); + SMARTLIST_FOREACH(included, const char *, cp, + digestset_add(set, cp)); + SMARTLIST_FOREACH(included, const char *, cp, + if (!digestset_isin(set, cp)) + ok = 0); + test_assert(ok); + for (i = 0; i < 1000; ++i) { + crypto_rand(d, DIGEST_LEN); + if (digestset_isin(set, d)) + ++false_positives; + } + test_assert(false_positives < 50); /* Should be far lower. */ + + done: + if (set) + digestset_free(set); + SMARTLIST_FOREACH(included, char *, cp, tor_free(cp)); + smartlist_free(included); +} + +typedef struct pq_entry_t { + const char *val; + int idx; +} pq_entry_t; + +/** Helper: return a tristate based on comparing two pq_entry_t values. */ +static int +_compare_strings_for_pqueue(const void *p1, const void *p2) +{ + const pq_entry_t *e1=p1, *e2=p2; + return strcmp(e1->val, e2->val); +} + +/** Run unit tests for heap-based priority queue functions. */ +static void +test_container_pqueue(void) +{ + smartlist_t *sl = smartlist_create(); + int (*cmp)(const void *, const void*); + const int offset = STRUCT_OFFSET(pq_entry_t, idx); +#define ENTRY(s) pq_entry_t s = { #s, -1 } + ENTRY(cows); + ENTRY(zebras); + ENTRY(fish); + ENTRY(frogs); + ENTRY(apples); + ENTRY(squid); + ENTRY(daschunds); + ENTRY(eggplants); + ENTRY(weissbier); + ENTRY(lobsters); + ENTRY(roquefort); + ENTRY(chinchillas); + ENTRY(fireflies); + +#define OK() smartlist_pqueue_assert_ok(sl, cmp, offset) + + cmp = _compare_strings_for_pqueue; + smartlist_pqueue_add(sl, cmp, offset, &cows); + smartlist_pqueue_add(sl, cmp, offset, &zebras); + smartlist_pqueue_add(sl, cmp, offset, &fish); + smartlist_pqueue_add(sl, cmp, offset, &frogs); + smartlist_pqueue_add(sl, cmp, offset, &apples); + smartlist_pqueue_add(sl, cmp, offset, &squid); + smartlist_pqueue_add(sl, cmp, offset, &daschunds); + smartlist_pqueue_add(sl, cmp, offset, &eggplants); + smartlist_pqueue_add(sl, cmp, offset, &weissbier); + smartlist_pqueue_add(sl, cmp, offset, &lobsters); + smartlist_pqueue_add(sl, cmp, offset, &roquefort); + + OK(); + + test_eq(smartlist_len(sl), 11); + test_eq_ptr(smartlist_get(sl, 0), &apples); + test_eq_ptr(smartlist_pqueue_pop(sl, cmp, offset), &apples); + test_eq(smartlist_len(sl), 10); + OK(); + test_eq_ptr(smartlist_pqueue_pop(sl, cmp, offset), &cows); + test_eq_ptr(smartlist_pqueue_pop(sl, cmp, offset), &daschunds); + smartlist_pqueue_add(sl, cmp, offset, &chinchillas); + OK(); + smartlist_pqueue_add(sl, cmp, offset, &fireflies); + OK(); + test_eq_ptr(smartlist_pqueue_pop(sl, cmp, offset), &chinchillas); + test_eq_ptr(smartlist_pqueue_pop(sl, cmp, offset), &eggplants); + test_eq_ptr(smartlist_pqueue_pop(sl, cmp, offset), &fireflies); + OK(); + test_eq_ptr(smartlist_pqueue_pop(sl, cmp, offset), &fish); + test_eq_ptr(smartlist_pqueue_pop(sl, cmp, offset), &frogs); + test_eq_ptr(smartlist_pqueue_pop(sl, cmp, offset), &lobsters); + test_eq_ptr(smartlist_pqueue_pop(sl, cmp, offset), &roquefort); + OK(); + test_eq(smartlist_len(sl), 3); + test_eq_ptr(smartlist_pqueue_pop(sl, cmp, offset), &squid); + test_eq_ptr(smartlist_pqueue_pop(sl, cmp, offset), &weissbier); + test_eq_ptr(smartlist_pqueue_pop(sl, cmp, offset), &zebras); + test_eq(smartlist_len(sl), 0); + OK(); + + /* Now test remove. */ + smartlist_pqueue_add(sl, cmp, offset, &cows); + smartlist_pqueue_add(sl, cmp, offset, &fish); + smartlist_pqueue_add(sl, cmp, offset, &frogs); + smartlist_pqueue_add(sl, cmp, offset, &apples); + smartlist_pqueue_add(sl, cmp, offset, &squid); + smartlist_pqueue_add(sl, cmp, offset, &zebras); + test_eq(smartlist_len(sl), 6); + OK(); + smartlist_pqueue_remove(sl, cmp, offset, &zebras); + test_eq(smartlist_len(sl), 5); + OK(); + smartlist_pqueue_remove(sl, cmp, offset, &cows); + test_eq(smartlist_len(sl), 4); + OK(); + smartlist_pqueue_remove(sl, cmp, offset, &apples); + test_eq(smartlist_len(sl), 3); + OK(); + test_eq_ptr(smartlist_pqueue_pop(sl, cmp, offset), &fish); + test_eq_ptr(smartlist_pqueue_pop(sl, cmp, offset), &frogs); + test_eq_ptr(smartlist_pqueue_pop(sl, cmp, offset), &squid); + test_eq(smartlist_len(sl), 0); + OK(); + +#undef OK + + done: + + smartlist_free(sl); +} + +/** Run unit tests for string-to-void* map functions */ +static void +test_container_strmap(void) +{ + strmap_t *map; + strmap_iter_t *iter; + const char *k; + void *v; + char *visited = NULL; + smartlist_t *found_keys = NULL; + + map = strmap_new(); + test_assert(map); + test_eq(strmap_size(map), 0); + test_assert(strmap_isempty(map)); + v = strmap_set(map, "K1", (void*)99); + test_eq(v, NULL); + test_assert(!strmap_isempty(map)); + v = strmap_set(map, "K2", (void*)101); + test_eq(v, NULL); + v = strmap_set(map, "K1", (void*)100); + test_eq(v, (void*)99); + test_eq_ptr(strmap_get(map,"K1"), (void*)100); + test_eq_ptr(strmap_get(map,"K2"), (void*)101); + test_eq_ptr(strmap_get(map,"K-not-there"), NULL); + strmap_assert_ok(map); + + v = strmap_remove(map,"K2"); + strmap_assert_ok(map); + test_eq_ptr(v, (void*)101); + test_eq_ptr(strmap_get(map,"K2"), NULL); + test_eq_ptr(strmap_remove(map,"K2"), NULL); + + strmap_set(map, "K2", (void*)101); + strmap_set(map, "K3", (void*)102); + strmap_set(map, "K4", (void*)103); + test_eq(strmap_size(map), 4); + strmap_assert_ok(map); + strmap_set(map, "K5", (void*)104); + strmap_set(map, "K6", (void*)105); + strmap_assert_ok(map); + + /* Test iterator. */ + iter = strmap_iter_init(map); + found_keys = smartlist_create(); + while (!strmap_iter_done(iter)) { + strmap_iter_get(iter,&k,&v); + smartlist_add(found_keys, tor_strdup(k)); + test_eq_ptr(v, strmap_get(map, k)); + + if (!strcmp(k, "K2")) { + iter = strmap_iter_next_rmv(map,iter); + } else { + iter = strmap_iter_next(map,iter); + } + } + + /* Make sure we removed K2, but not the others. */ + test_eq_ptr(strmap_get(map, "K2"), NULL); + test_eq_ptr(strmap_get(map, "K5"), (void*)104); + /* Make sure we visited everyone once */ + smartlist_sort_strings(found_keys); + visited = smartlist_join_strings(found_keys, ":", 0, NULL); + test_streq(visited, "K1:K2:K3:K4:K5:K6"); + + strmap_assert_ok(map); + /* Clean up after ourselves. */ + strmap_free(map, NULL); + map = NULL; + + /* Now try some lc functions. */ + map = strmap_new(); + strmap_set_lc(map,"Ab.C", (void*)1); + test_eq_ptr(strmap_get(map,"ab.c"), (void*)1); + strmap_assert_ok(map); + test_eq_ptr(strmap_get_lc(map,"AB.C"), (void*)1); + test_eq_ptr(strmap_get(map,"AB.C"), NULL); + test_eq_ptr(strmap_remove_lc(map,"aB.C"), (void*)1); + strmap_assert_ok(map); + test_eq_ptr(strmap_get_lc(map,"AB.C"), NULL); + + done: + if (map) + strmap_free(map,NULL); + if (found_keys) { + SMARTLIST_FOREACH(found_keys, char *, cp, tor_free(cp)); + smartlist_free(found_keys); + } + tor_free(visited); +} + +/** Run unit tests for getting the median of a list. */ +static void +test_container_order_functions(void) +{ + int lst[25], n = 0; + // int a=12,b=24,c=25,d=60,e=77; + +#define median() median_int(lst, n) + + lst[n++] = 12; + test_eq(12, median()); /* 12 */ + lst[n++] = 77; + //smartlist_shuffle(sl); + test_eq(12, median()); /* 12, 77 */ + lst[n++] = 77; + //smartlist_shuffle(sl); + test_eq(77, median()); /* 12, 77, 77 */ + lst[n++] = 24; + test_eq(24, median()); /* 12,24,77,77 */ + lst[n++] = 60; + lst[n++] = 12; + lst[n++] = 25; + //smartlist_shuffle(sl); + test_eq(25, median()); /* 12,12,24,25,60,77,77 */ +#undef median + + done: + ; +} + +#define CONTAINER_LEGACY(name) \ + { #name, legacy_test_helper, 0, &legacy_setup, test_container_ ## name } + +struct testcase_t container_tests[] = { + CONTAINER_LEGACY(smartlist_basic), + CONTAINER_LEGACY(smartlist_strings), + CONTAINER_LEGACY(smartlist_overlap), + CONTAINER_LEGACY(smartlist_digests), + CONTAINER_LEGACY(smartlist_join), + CONTAINER_LEGACY(bitarray), + CONTAINER_LEGACY(digestset), + CONTAINER_LEGACY(strmap), + CONTAINER_LEGACY(pqueue), + CONTAINER_LEGACY(order_functions), + END_OF_TESTCASES +}; + diff --git a/src/test/test_crypto.c b/src/test/test_crypto.c new file mode 100644 index 0000000000..781081a4ad --- /dev/null +++ b/src/test/test_crypto.c @@ -0,0 +1,796 @@ +/* Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "orconfig.h" +#define CRYPTO_PRIVATE +#include "or.h" +#include "test.h" + +/** Run unit tests for Diffie-Hellman functionality. */ +static void +test_crypto_dh(void) +{ + crypto_dh_env_t *dh1 = crypto_dh_new(DH_TYPE_CIRCUIT); + crypto_dh_env_t *dh2 = crypto_dh_new(DH_TYPE_CIRCUIT); + char p1[DH_BYTES]; + char p2[DH_BYTES]; + char s1[DH_BYTES]; + char s2[DH_BYTES]; + ssize_t s1len, s2len; + + test_eq(crypto_dh_get_bytes(dh1), DH_BYTES); + test_eq(crypto_dh_get_bytes(dh2), DH_BYTES); + + memset(p1, 0, DH_BYTES); + memset(p2, 0, DH_BYTES); + test_memeq(p1, p2, DH_BYTES); + test_assert(! crypto_dh_get_public(dh1, p1, DH_BYTES)); + test_memneq(p1, p2, DH_BYTES); + test_assert(! crypto_dh_get_public(dh2, p2, DH_BYTES)); + test_memneq(p1, p2, DH_BYTES); + + memset(s1, 0, DH_BYTES); + memset(s2, 0xFF, DH_BYTES); + s1len = crypto_dh_compute_secret(LOG_WARN, dh1, p2, DH_BYTES, s1, 50); + s2len = crypto_dh_compute_secret(LOG_WARN, dh2, p1, DH_BYTES, s2, 50); + test_assert(s1len > 0); + test_eq(s1len, s2len); + test_memeq(s1, s2, s1len); + + { + /* XXXX Now fabricate some bad values and make sure they get caught, + * Check 0, 1, N-1, >= N, etc. + */ + } + + done: + crypto_dh_free(dh1); + crypto_dh_free(dh2); +} + +/** Run unit tests for our random number generation function and its wrappers. + */ +static void +test_crypto_rng(void) +{ + int i, j, allok; + char data1[100], data2[100]; + double d; + + /* Try out RNG. */ + test_assert(! crypto_seed_rng(0)); + crypto_rand(data1, 100); + crypto_rand(data2, 100); + test_memneq(data1,data2,100); + allok = 1; + for (i = 0; i < 100; ++i) { + uint64_t big; + char *host; + j = crypto_rand_int(100); + if (i < 0 || i >= 100) + allok = 0; + big = crypto_rand_uint64(U64_LITERAL(1)<<40); + if (big >= (U64_LITERAL(1)<<40)) + allok = 0; + big = crypto_rand_uint64(U64_LITERAL(5)); + if (big >= 5) + allok = 0; + d = crypto_rand_double(); + test_assert(d >= 0); + test_assert(d < 1.0); + host = crypto_random_hostname(3,8,"www.",".onion"); + if (strcmpstart(host,"www.") || + strcmpend(host,".onion") || + strlen(host) < 13 || + strlen(host) > 18) + allok = 0; + tor_free(host); + } + test_assert(allok); + done: + ; +} + +/** Run unit tests for our AES functionality */ +static void +test_crypto_aes(void) +{ + char *data1 = NULL, *data2 = NULL, *data3 = NULL; + crypto_cipher_env_t *env1 = NULL, *env2 = NULL; + int i, j; + char *mem_op_hex_tmp=NULL; + + data1 = tor_malloc(1024); + data2 = tor_malloc(1024); + data3 = tor_malloc(1024); + + /* Now, test encryption and decryption with stream cipher. */ + data1[0]='\0'; + for (i = 1023; i>0; i -= 35) + strncat(data1, "Now is the time for all good onions", i); + + memset(data2, 0, 1024); + memset(data3, 0, 1024); + env1 = crypto_new_cipher_env(); + test_neq(env1, 0); + env2 = crypto_new_cipher_env(); + test_neq(env2, 0); + j = crypto_cipher_generate_key(env1); + crypto_cipher_set_key(env2, crypto_cipher_get_key(env1)); + crypto_cipher_encrypt_init_cipher(env1); + crypto_cipher_decrypt_init_cipher(env2); + + /* Try encrypting 512 chars. */ + crypto_cipher_encrypt(env1, data2, data1, 512); + crypto_cipher_decrypt(env2, data3, data2, 512); + test_memeq(data1, data3, 512); + test_memneq(data1, data2, 512); + + /* Now encrypt 1 at a time, and get 1 at a time. */ + for (j = 512; j < 560; ++j) { + crypto_cipher_encrypt(env1, data2+j, data1+j, 1); + } + for (j = 512; j < 560; ++j) { + crypto_cipher_decrypt(env2, data3+j, data2+j, 1); + } + test_memeq(data1, data3, 560); + /* Now encrypt 3 at a time, and get 5 at a time. */ + for (j = 560; j < 1024-5; j += 3) { + crypto_cipher_encrypt(env1, data2+j, data1+j, 3); + } + for (j = 560; j < 1024-5; j += 5) { + crypto_cipher_decrypt(env2, data3+j, data2+j, 5); + } + test_memeq(data1, data3, 1024-5); + /* Now make sure that when we encrypt with different chunk sizes, we get + the same results. */ + crypto_free_cipher_env(env2); + env2 = NULL; + + memset(data3, 0, 1024); + env2 = crypto_new_cipher_env(); + test_neq(env2, 0); + crypto_cipher_set_key(env2, crypto_cipher_get_key(env1)); + crypto_cipher_encrypt_init_cipher(env2); + for (j = 0; j < 1024-16; j += 17) { + crypto_cipher_encrypt(env2, data3+j, data1+j, 17); + } + for (j= 0; j < 1024-16; ++j) { + if (data2[j] != data3[j]) { + printf("%d: %d\t%d\n", j, (int) data2[j], (int) data3[j]); + } + } + test_memeq(data2, data3, 1024-16); + crypto_free_cipher_env(env1); + env1 = NULL; + crypto_free_cipher_env(env2); + env2 = NULL; + + /* NIST test vector for aes. */ + env1 = crypto_new_cipher_env(); /* IV starts at 0 */ + crypto_cipher_set_key(env1, "\x80\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00"); + crypto_cipher_encrypt_init_cipher(env1); + crypto_cipher_encrypt(env1, data1, + "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00", 16); + test_memeq_hex(data1, "0EDD33D3C621E546455BD8BA1418BEC8"); + + /* Now test rollover. All these values are originally from a python + * script. */ + crypto_cipher_set_iv(env1, "\x00\x00\x00\x00\x00\x00\x00\x00" + "\xff\xff\xff\xff\xff\xff\xff\xff"); + memset(data2, 0, 1024); + crypto_cipher_encrypt(env1, data1, data2, 32); + test_memeq_hex(data1, "335fe6da56f843199066c14a00a40231" + "cdd0b917dbc7186908a6bfb5ffd574d3"); + + crypto_cipher_set_iv(env1, "\x00\x00\x00\x00\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff"); + memset(data2, 0, 1024); + crypto_cipher_encrypt(env1, data1, data2, 32); + test_memeq_hex(data1, "e627c6423fa2d77832a02b2794094b73" + "3e63c721df790d2c6469cc1953a3ffac"); + + crypto_cipher_set_iv(env1, "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff"); + memset(data2, 0, 1024); + crypto_cipher_encrypt(env1, data1, data2, 32); + test_memeq_hex(data1, "2aed2bff0de54f9328efd070bf48f70a" + "0EDD33D3C621E546455BD8BA1418BEC8"); + + /* Now check rollover on inplace cipher. */ + crypto_cipher_set_iv(env1, "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff"); + crypto_cipher_crypt_inplace(env1, data2, 64); + test_memeq_hex(data2, "2aed2bff0de54f9328efd070bf48f70a" + "0EDD33D3C621E546455BD8BA1418BEC8" + "93e2c5243d6839eac58503919192f7ae" + "1908e67cafa08d508816659c2e693191"); + crypto_cipher_set_iv(env1, "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff"); + crypto_cipher_crypt_inplace(env1, data2, 64); + test_assert(tor_mem_is_zero(data2, 64)); + + done: + tor_free(mem_op_hex_tmp); + if (env1) + crypto_free_cipher_env(env1); + if (env2) + crypto_free_cipher_env(env2); + tor_free(data1); + tor_free(data2); + tor_free(data3); +} + +/** Run unit tests for our SHA-1 functionality */ +static void +test_crypto_sha(void) +{ + crypto_digest_env_t *d1 = NULL, *d2 = NULL; + int i; + char key[80]; + char digest[32]; + char data[50]; + char d_out1[DIGEST_LEN], d_out2[DIGEST256_LEN]; + char *mem_op_hex_tmp=NULL; + + /* Test SHA-1 with a test vector from the specification. */ + i = crypto_digest(data, "abc", 3); + test_memeq_hex(data, "A9993E364706816ABA3E25717850C26C9CD0D89D"); + + /* Test SHA-256 with a test vector from the specification. */ + i = crypto_digest256(data, "abc", 3, DIGEST_SHA256); + test_memeq_hex(data, "BA7816BF8F01CFEA414140DE5DAE2223B00361A3" + "96177A9CB410FF61F20015AD"); + + /* Test HMAC-SHA-1 with test cases from RFC2202. */ + + /* Case 1. */ + memset(key, 0x0b, 20); + crypto_hmac_sha1(digest, key, 20, "Hi There", 8); + test_streq(hex_str(digest, 20), + "B617318655057264E28BC0B6FB378C8EF146BE00"); + /* Case 2. */ + crypto_hmac_sha1(digest, "Jefe", 4, "what do ya want for nothing?", 28); + test_streq(hex_str(digest, 20), + "EFFCDF6AE5EB2FA2D27416D5F184DF9C259A7C79"); + + /* Case 4. */ + base16_decode(key, 25, + "0102030405060708090a0b0c0d0e0f10111213141516171819", 50); + memset(data, 0xcd, 50); + crypto_hmac_sha1(digest, key, 25, data, 50); + test_streq(hex_str(digest, 20), + "4C9007F4026250C6BC8414F9BF50C86C2D7235DA"); + + /* Case 5. */ + memset(key, 0xaa, 80); + crypto_hmac_sha1(digest, key, 80, + "Test Using Larger Than Block-Size Key - Hash Key First", + 54); + test_streq(hex_str(digest, 20), + "AA4AE5E15272D00E95705637CE8A3B55ED402112"); + + /* Incremental digest code. */ + d1 = crypto_new_digest_env(); + test_assert(d1); + crypto_digest_add_bytes(d1, "abcdef", 6); + d2 = crypto_digest_dup(d1); + test_assert(d2); + crypto_digest_add_bytes(d2, "ghijkl", 6); + crypto_digest_get_digest(d2, d_out1, sizeof(d_out1)); + crypto_digest(d_out2, "abcdefghijkl", 12); + test_memeq(d_out1, d_out2, DIGEST_LEN); + crypto_digest_assign(d2, d1); + crypto_digest_add_bytes(d2, "mno", 3); + crypto_digest_get_digest(d2, d_out1, sizeof(d_out1)); + crypto_digest(d_out2, "abcdefmno", 9); + test_memeq(d_out1, d_out2, DIGEST_LEN); + crypto_digest_get_digest(d1, d_out1, sizeof(d_out1)); + crypto_digest(d_out2, "abcdef", 6); + test_memeq(d_out1, d_out2, DIGEST_LEN); + crypto_free_digest_env(d1); + crypto_free_digest_env(d2); + + /* Incremental digest code with sha256 */ + d1 = crypto_new_digest256_env(DIGEST_SHA256); + test_assert(d1); + crypto_digest_add_bytes(d1, "abcdef", 6); + d2 = crypto_digest_dup(d1); + test_assert(d2); + crypto_digest_add_bytes(d2, "ghijkl", 6); + crypto_digest_get_digest(d2, d_out1, sizeof(d_out1)); + crypto_digest256(d_out2, "abcdefghijkl", 12, DIGEST_SHA256); + test_memeq(d_out1, d_out2, DIGEST_LEN); + crypto_digest_assign(d2, d1); + crypto_digest_add_bytes(d2, "mno", 3); + crypto_digest_get_digest(d2, d_out1, sizeof(d_out1)); + crypto_digest256(d_out2, "abcdefmno", 9, DIGEST_SHA256); + test_memeq(d_out1, d_out2, DIGEST_LEN); + crypto_digest_get_digest(d1, d_out1, sizeof(d_out1)); + crypto_digest256(d_out2, "abcdef", 6, DIGEST_SHA256); + test_memeq(d_out1, d_out2, DIGEST_LEN); + + done: + if (d1) + crypto_free_digest_env(d1); + if (d2) + crypto_free_digest_env(d2); + tor_free(mem_op_hex_tmp); +} + +/** Run unit tests for our public key crypto functions */ +static void +test_crypto_pk(void) +{ + crypto_pk_env_t *pk1 = NULL, *pk2 = NULL; + char *encoded = NULL; + char data1[1024], data2[1024], data3[1024]; + size_t size; + int i, j, p, len; + + /* Public-key ciphers */ + pk1 = pk_generate(0); + pk2 = crypto_new_pk_env(); + test_assert(pk1 && pk2); + test_assert(! crypto_pk_write_public_key_to_string(pk1, &encoded, &size)); + test_assert(! crypto_pk_read_public_key_from_string(pk2, encoded, size)); + test_eq(0, crypto_pk_cmp_keys(pk1, pk2)); + + test_eq(128, crypto_pk_keysize(pk1)); + test_eq(128, crypto_pk_keysize(pk2)); + + test_eq(128, crypto_pk_public_encrypt(pk2, data1, sizeof(data1), + "Hello whirled.", 15, + PK_PKCS1_OAEP_PADDING)); + test_eq(128, crypto_pk_public_encrypt(pk1, data2, sizeof(data1), + "Hello whirled.", 15, + PK_PKCS1_OAEP_PADDING)); + /* oaep padding should make encryption not match */ + test_memneq(data1, data2, 128); + test_eq(15, crypto_pk_private_decrypt(pk1, data3, sizeof(data3), data1, 128, + PK_PKCS1_OAEP_PADDING,1)); + test_streq(data3, "Hello whirled."); + memset(data3, 0, 1024); + test_eq(15, crypto_pk_private_decrypt(pk1, data3, sizeof(data3), data2, 128, + PK_PKCS1_OAEP_PADDING,1)); + test_streq(data3, "Hello whirled."); + /* Can't decrypt with public key. */ + test_eq(-1, crypto_pk_private_decrypt(pk2, data3, sizeof(data3), data2, 128, + PK_PKCS1_OAEP_PADDING,1)); + /* Try again with bad padding */ + memcpy(data2+1, "XYZZY", 5); /* This has fails ~ once-in-2^40 */ + test_eq(-1, crypto_pk_private_decrypt(pk1, data3, sizeof(data3), data2, 128, + PK_PKCS1_OAEP_PADDING,1)); + + /* File operations: save and load private key */ + test_assert(! crypto_pk_write_private_key_to_filename(pk1, + get_fname("pkey1"))); + /* failing case for read: can't read. */ + test_assert(crypto_pk_read_private_key_from_filename(pk2, + get_fname("xyzzy")) < 0); + write_str_to_file(get_fname("xyzzy"), "foobar", 6); + /* Failing case for read: no key. */ + test_assert(crypto_pk_read_private_key_from_filename(pk2, + get_fname("xyzzy")) < 0); + test_assert(! crypto_pk_read_private_key_from_filename(pk2, + get_fname("pkey1"))); + test_eq(15, crypto_pk_private_decrypt(pk2, data3, sizeof(data3), data1, 128, + PK_PKCS1_OAEP_PADDING,1)); + + /* Now try signing. */ + strlcpy(data1, "Ossifrage", 1024); + test_eq(128, crypto_pk_private_sign(pk1, data2, sizeof(data2), data1, 10)); + test_eq(10, + crypto_pk_public_checksig(pk1, data3, sizeof(data3), data2, 128)); + test_streq(data3, "Ossifrage"); + /* Try signing digests. */ + test_eq(128, crypto_pk_private_sign_digest(pk1, data2, sizeof(data2), + data1, 10)); + test_eq(20, + crypto_pk_public_checksig(pk1, data3, sizeof(data3), data2, 128)); + test_eq(0, crypto_pk_public_checksig_digest(pk1, data1, 10, data2, 128)); + test_eq(-1, crypto_pk_public_checksig_digest(pk1, data1, 11, data2, 128)); + + /*XXXX test failed signing*/ + + /* Try encoding */ + crypto_free_pk_env(pk2); + pk2 = NULL; + i = crypto_pk_asn1_encode(pk1, data1, 1024); + test_assert(i>0); + pk2 = crypto_pk_asn1_decode(data1, i); + test_assert(crypto_pk_cmp_keys(pk1,pk2) == 0); + + /* Try with hybrid encryption wrappers. */ + crypto_rand(data1, 1024); + for (i = 0; i < 3; ++i) { + for (j = 85; j < 140; ++j) { + memset(data2,0,1024); + memset(data3,0,1024); + if (i == 0 && j < 129) + continue; + p = (i==0)?PK_NO_PADDING: + (i==1)?PK_PKCS1_PADDING:PK_PKCS1_OAEP_PADDING; + len = crypto_pk_public_hybrid_encrypt(pk1,data2,sizeof(data2), + data1,j,p,0); + test_assert(len>=0); + len = crypto_pk_private_hybrid_decrypt(pk1,data3,sizeof(data3), + data2,len,p,1); + test_eq(len,j); + test_memeq(data1,data3,j); + } + } + + /* Try copy_full */ + crypto_free_pk_env(pk2); + pk2 = crypto_pk_copy_full(pk1); + test_assert(pk2 != NULL); + test_neq_ptr(pk1, pk2); + test_assert(crypto_pk_cmp_keys(pk1,pk2) == 0); + + done: + if (pk1) + crypto_free_pk_env(pk1); + if (pk2) + crypto_free_pk_env(pk2); + tor_free(encoded); +} + +/** Run unit tests for misc crypto formatting functionality (base64, base32, + * fingerprints, etc) */ +static void +test_crypto_formats(void) +{ + char *data1 = NULL, *data2 = NULL, *data3 = NULL; + int i, j, idx; + + data1 = tor_malloc(1024); + data2 = tor_malloc(1024); + data3 = tor_malloc(1024); + test_assert(data1 && data2 && data3); + + /* Base64 tests */ + memset(data1, 6, 1024); + for (idx = 0; idx < 10; ++idx) { + i = base64_encode(data2, 1024, data1, idx); + test_assert(i >= 0); + j = base64_decode(data3, 1024, data2, i); + test_eq(j,idx); + test_memeq(data3, data1, idx); + } + + strlcpy(data1, "Test string that contains 35 chars.", 1024); + strlcat(data1, " 2nd string that contains 35 chars.", 1024); + + i = base64_encode(data2, 1024, data1, 71); + test_assert(i >= 0); + j = base64_decode(data3, 1024, data2, i); + test_eq(j, 71); + test_streq(data3, data1); + test_assert(data2[i] == '\0'); + + crypto_rand(data1, DIGEST_LEN); + memset(data2, 100, 1024); + digest_to_base64(data2, data1); + test_eq(BASE64_DIGEST_LEN, strlen(data2)); + test_eq(100, data2[BASE64_DIGEST_LEN+2]); + memset(data3, 99, 1024); + test_eq(digest_from_base64(data3, data2), 0); + test_memeq(data1, data3, DIGEST_LEN); + test_eq(99, data3[DIGEST_LEN+1]); + + test_assert(digest_from_base64(data3, "###") < 0); + + /* Encoding SHA256 */ + crypto_rand(data2, DIGEST256_LEN); + memset(data2, 100, 1024); + digest256_to_base64(data2, data1); + test_eq(BASE64_DIGEST256_LEN, strlen(data2)); + test_eq(100, data2[BASE64_DIGEST256_LEN+2]); + memset(data3, 99, 1024); + test_eq(digest256_from_base64(data3, data2), 0); + test_memeq(data1, data3, DIGEST256_LEN); + test_eq(99, data3[DIGEST256_LEN+1]); + + /* Base32 tests */ + strlcpy(data1, "5chrs", 1024); + /* bit pattern is: [35 63 68 72 73] -> + * [00110101 01100011 01101000 01110010 01110011] + * By 5s: [00110 10101 10001 10110 10000 11100 10011 10011] + */ + base32_encode(data2, 9, data1, 5); + test_streq(data2, "gvrwq4tt"); + + strlcpy(data1, "\xFF\xF5\x6D\x44\xAE\x0D\x5C\xC9\x62\xC4", 1024); + base32_encode(data2, 30, data1, 10); + test_streq(data2, "772w2rfobvomsywe"); + + /* Base16 tests */ + strlcpy(data1, "6chrs\xff", 1024); + base16_encode(data2, 13, data1, 6); + test_streq(data2, "3663687273FF"); + + strlcpy(data1, "f0d678affc000100", 1024); + i = base16_decode(data2, 8, data1, 16); + test_eq(i,0); + test_memeq(data2, "\xf0\xd6\x78\xaf\xfc\x00\x01\x00",8); + + /* now try some failing base16 decodes */ + test_eq(-1, base16_decode(data2, 8, data1, 15)); /* odd input len */ + test_eq(-1, base16_decode(data2, 7, data1, 16)); /* dest too short */ + strlcpy(data1, "f0dz!8affc000100", 1024); + test_eq(-1, base16_decode(data2, 8, data1, 16)); + + tor_free(data1); + tor_free(data2); + tor_free(data3); + + /* Add spaces to fingerprint */ + { + data1 = tor_strdup("ABCD1234ABCD56780000ABCD1234ABCD56780000"); + test_eq(strlen(data1), 40); + data2 = tor_malloc(FINGERPRINT_LEN+1); + add_spaces_to_fp(data2, FINGERPRINT_LEN+1, data1); + test_streq(data2, "ABCD 1234 ABCD 5678 0000 ABCD 1234 ABCD 5678 0000"); + tor_free(data1); + tor_free(data2); + } + + /* Check fingerprint */ + { + test_assert(crypto_pk_check_fingerprint_syntax( + "ABCD 1234 ABCD 5678 0000 ABCD 1234 ABCD 5678 0000")); + test_assert(!crypto_pk_check_fingerprint_syntax( + "ABCD 1234 ABCD 5678 0000 ABCD 1234 ABCD 5678 000")); + test_assert(!crypto_pk_check_fingerprint_syntax( + "ABCD 1234 ABCD 5678 0000 ABCD 1234 ABCD 5678 00000")); + test_assert(!crypto_pk_check_fingerprint_syntax( + "ABCD 1234 ABCD 5678 0000 ABCD1234 ABCD 5678 0000")); + test_assert(!crypto_pk_check_fingerprint_syntax( + "ABCD 1234 ABCD 5678 0000 ABCD1234 ABCD 5678 00000")); + test_assert(!crypto_pk_check_fingerprint_syntax( + "ACD 1234 ABCD 5678 0000 ABCD 1234 ABCD 5678 00000")); + } + + done: + tor_free(data1); + tor_free(data2); + tor_free(data3); +} + +/** Run unit tests for our secret-to-key passphrase hashing functionality. */ +static void +test_crypto_s2k(void) +{ + char buf[29]; + char buf2[29]; + char *buf3 = NULL; + int i; + + memset(buf, 0, sizeof(buf)); + memset(buf2, 0, sizeof(buf2)); + buf3 = tor_malloc(65536); + memset(buf3, 0, 65536); + + secret_to_key(buf+9, 20, "", 0, buf); + crypto_digest(buf2+9, buf3, 1024); + test_memeq(buf, buf2, 29); + + memcpy(buf,"vrbacrda",8); + memcpy(buf2,"vrbacrda",8); + buf[8] = 96; + buf2[8] = 96; + secret_to_key(buf+9, 20, "12345678", 8, buf); + for (i = 0; i < 65536; i += 16) { + memcpy(buf3+i, "vrbacrda12345678", 16); + } + crypto_digest(buf2+9, buf3, 65536); + test_memeq(buf, buf2, 29); + + done: + tor_free(buf3); +} + +/** Test AES-CTR encryption and decryption with IV. */ +static void +test_crypto_aes_iv(void) +{ + crypto_cipher_env_t *cipher; + char *plain, *encrypted1, *encrypted2, *decrypted1, *decrypted2; + char plain_1[1], plain_15[15], plain_16[16], plain_17[17]; + char key1[16], key2[16]; + ssize_t encrypted_size, decrypted_size; + + plain = tor_malloc(4095); + encrypted1 = tor_malloc(4095 + 1 + 16); + encrypted2 = tor_malloc(4095 + 1 + 16); + decrypted1 = tor_malloc(4095 + 1); + decrypted2 = tor_malloc(4095 + 1); + + crypto_rand(plain, 4095); + crypto_rand(key1, 16); + crypto_rand(key2, 16); + crypto_rand(plain_1, 1); + crypto_rand(plain_15, 15); + crypto_rand(plain_16, 16); + crypto_rand(plain_17, 17); + key1[0] = key2[0] + 128; /* Make sure that contents are different. */ + /* Encrypt and decrypt with the same key. */ + cipher = crypto_create_init_cipher(key1, 1); + encrypted_size = crypto_cipher_encrypt_with_iv(cipher, encrypted1, 16 + 4095, + plain, 4095); + crypto_free_cipher_env(cipher); + cipher = NULL; + test_eq(encrypted_size, 16 + 4095); + tor_assert(encrypted_size > 0); /* This is obviously true, since 4111 is + * greater than 0, but its truth is not + * obvious to all analysis tools. */ + cipher = crypto_create_init_cipher(key1, 0); + decrypted_size = crypto_cipher_decrypt_with_iv(cipher, decrypted1, 4095, + encrypted1, encrypted_size); + crypto_free_cipher_env(cipher); + cipher = NULL; + test_eq(decrypted_size, 4095); + tor_assert(decrypted_size > 0); + test_memeq(plain, decrypted1, 4095); + /* Encrypt a second time (with a new random initialization vector). */ + cipher = crypto_create_init_cipher(key1, 1); + encrypted_size = crypto_cipher_encrypt_with_iv(cipher, encrypted2, 16 + 4095, + plain, 4095); + crypto_free_cipher_env(cipher); + cipher = NULL; + test_eq(encrypted_size, 16 + 4095); + tor_assert(encrypted_size > 0); + cipher = crypto_create_init_cipher(key1, 0); + decrypted_size = crypto_cipher_decrypt_with_iv(cipher, decrypted2, 4095, + encrypted2, encrypted_size); + crypto_free_cipher_env(cipher); + cipher = NULL; + test_eq(decrypted_size, 4095); + tor_assert(decrypted_size > 0); + test_memeq(plain, decrypted2, 4095); + test_memneq(encrypted1, encrypted2, encrypted_size); + /* Decrypt with the wrong key. */ + cipher = crypto_create_init_cipher(key2, 0); + decrypted_size = crypto_cipher_decrypt_with_iv(cipher, decrypted2, 4095, + encrypted1, encrypted_size); + crypto_free_cipher_env(cipher); + cipher = NULL; + test_memneq(plain, decrypted2, encrypted_size); + /* Alter the initialization vector. */ + encrypted1[0] += 42; + cipher = crypto_create_init_cipher(key1, 0); + decrypted_size = crypto_cipher_decrypt_with_iv(cipher, decrypted1, 4095, + encrypted1, encrypted_size); + crypto_free_cipher_env(cipher); + cipher = NULL; + test_memneq(plain, decrypted2, 4095); + /* Special length case: 1. */ + cipher = crypto_create_init_cipher(key1, 1); + encrypted_size = crypto_cipher_encrypt_with_iv(cipher, encrypted1, 16 + 1, + plain_1, 1); + crypto_free_cipher_env(cipher); + cipher = NULL; + test_eq(encrypted_size, 16 + 1); + tor_assert(encrypted_size > 0); + cipher = crypto_create_init_cipher(key1, 0); + decrypted_size = crypto_cipher_decrypt_with_iv(cipher, decrypted1, 1, + encrypted1, encrypted_size); + crypto_free_cipher_env(cipher); + cipher = NULL; + test_eq(decrypted_size, 1); + tor_assert(decrypted_size > 0); + test_memeq(plain_1, decrypted1, 1); + /* Special length case: 15. */ + cipher = crypto_create_init_cipher(key1, 1); + encrypted_size = crypto_cipher_encrypt_with_iv(cipher, encrypted1, 16 + 15, + plain_15, 15); + crypto_free_cipher_env(cipher); + cipher = NULL; + test_eq(encrypted_size, 16 + 15); + tor_assert(encrypted_size > 0); + cipher = crypto_create_init_cipher(key1, 0); + decrypted_size = crypto_cipher_decrypt_with_iv(cipher, decrypted1, 15, + encrypted1, encrypted_size); + crypto_free_cipher_env(cipher); + cipher = NULL; + test_eq(decrypted_size, 15); + tor_assert(decrypted_size > 0); + test_memeq(plain_15, decrypted1, 15); + /* Special length case: 16. */ + cipher = crypto_create_init_cipher(key1, 1); + encrypted_size = crypto_cipher_encrypt_with_iv(cipher, encrypted1, 16 + 16, + plain_16, 16); + crypto_free_cipher_env(cipher); + cipher = NULL; + test_eq(encrypted_size, 16 + 16); + tor_assert(encrypted_size > 0); + cipher = crypto_create_init_cipher(key1, 0); + decrypted_size = crypto_cipher_decrypt_with_iv(cipher, decrypted1, 16, + encrypted1, encrypted_size); + crypto_free_cipher_env(cipher); + cipher = NULL; + test_eq(decrypted_size, 16); + tor_assert(decrypted_size > 0); + test_memeq(plain_16, decrypted1, 16); + /* Special length case: 17. */ + cipher = crypto_create_init_cipher(key1, 1); + encrypted_size = crypto_cipher_encrypt_with_iv(cipher, encrypted1, 16 + 17, + plain_17, 17); + crypto_free_cipher_env(cipher); + cipher = NULL; + test_eq(encrypted_size, 16 + 17); + tor_assert(encrypted_size > 0); + cipher = crypto_create_init_cipher(key1, 0); + decrypted_size = crypto_cipher_decrypt_with_iv(cipher, decrypted1, 17, + encrypted1, encrypted_size); + test_eq(decrypted_size, 17); + tor_assert(decrypted_size > 0); + test_memeq(plain_17, decrypted1, 17); + + done: + /* Free memory. */ + tor_free(plain); + tor_free(encrypted1); + tor_free(encrypted2); + tor_free(decrypted1); + tor_free(decrypted2); + if (cipher) + crypto_free_cipher_env(cipher); +} + +/** Test base32 decoding. */ +static void +test_crypto_base32_decode(void) +{ + char plain[60], encoded[96 + 1], decoded[60]; + int res; + crypto_rand(plain, 60); + /* Encode and decode a random string. */ + base32_encode(encoded, 96 + 1, plain, 60); + res = base32_decode(decoded, 60, encoded, 96); + test_eq(res, 0); + test_memeq(plain, decoded, 60); + /* Encode, uppercase, and decode a random string. */ + base32_encode(encoded, 96 + 1, plain, 60); + tor_strupper(encoded); + res = base32_decode(decoded, 60, encoded, 96); + test_eq(res, 0); + test_memeq(plain, decoded, 60); + /* Change encoded string and decode. */ + if (encoded[0] == 'A' || encoded[0] == 'a') + encoded[0] = 'B'; + else + encoded[0] = 'A'; + res = base32_decode(decoded, 60, encoded, 96); + test_eq(res, 0); + test_memneq(plain, decoded, 60); + /* Bad encodings. */ + encoded[0] = '!'; + res = base32_decode(decoded, 60, encoded, 96); + test_assert(res < 0); + + done: + ; +} + +#define CRYPTO_LEGACY(name) \ + { #name, legacy_test_helper, 0, &legacy_setup, test_crypto_ ## name } + +struct testcase_t crypto_tests[] = { + CRYPTO_LEGACY(formats), + CRYPTO_LEGACY(rng), + CRYPTO_LEGACY(aes), + CRYPTO_LEGACY(sha), + CRYPTO_LEGACY(pk), + CRYPTO_LEGACY(dh), + CRYPTO_LEGACY(s2k), + CRYPTO_LEGACY(aes_iv), + CRYPTO_LEGACY(base32_decode), + END_OF_TESTCASES +}; + diff --git a/src/or/test_data.c b/src/test/test_data.c index fc85857615..fc85857615 100644 --- a/src/or/test_data.c +++ b/src/test/test_data.c diff --git a/src/test/test_dir.c b/src/test/test_dir.c new file mode 100644 index 0000000000..8fd94289a9 --- /dev/null +++ b/src/test/test_dir.c @@ -0,0 +1,1320 @@ +/* Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "orconfig.h" +#define DIRSERV_PRIVATE +#define DIRVOTE_PRIVATE +#define ROUTER_PRIVATE +#include "or.h" +#include "directory.h" +#include "dirserv.h" +#include "dirvote.h" +#include "networkstatus.h" +#include "router.h" +#include "routerlist.h" +#include "routerparse.h" +#include "test.h" + +static void +test_dir_nicknames(void) +{ + test_assert( is_legal_nickname("a")); + test_assert(!is_legal_nickname("")); + test_assert(!is_legal_nickname("abcdefghijklmnopqrst")); /* 20 chars */ + test_assert(!is_legal_nickname("hyphen-")); /* bad char */ + test_assert( is_legal_nickname("abcdefghijklmnopqrs")); /* 19 chars */ + test_assert(!is_legal_nickname("$AAAAAAAA01234AAAAAAAAAAAAAAAAAAAAAAAAAAA")); + /* valid */ + test_assert( is_legal_nickname_or_hexdigest( + "$AAAAAAAA01234AAAAAAAAAAAAAAAAAAAAAAAAAAA")); + test_assert( is_legal_nickname_or_hexdigest( + "$AAAAAAAA01234AAAAAAAAAAAAAAAAAAAAAAAAAAA=fred")); + test_assert( is_legal_nickname_or_hexdigest( + "$AAAAAAAA01234AAAAAAAAAAAAAAAAAAAAAAAAAAA~fred")); + /* too short */ + test_assert(!is_legal_nickname_or_hexdigest( + "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")); + /* illegal char */ + test_assert(!is_legal_nickname_or_hexdigest( + "$AAAAAAzAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")); + /* hex part too long */ + test_assert(!is_legal_nickname_or_hexdigest( + "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")); + test_assert(!is_legal_nickname_or_hexdigest( + "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=fred")); + /* Bad nickname */ + test_assert(!is_legal_nickname_or_hexdigest( + "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=")); + test_assert(!is_legal_nickname_or_hexdigest( + "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~")); + test_assert(!is_legal_nickname_or_hexdigest( + "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~hyphen-")); + test_assert(!is_legal_nickname_or_hexdigest( + "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~" + "abcdefghijklmnoppqrst")); + /* Bad extra char. */ + test_assert(!is_legal_nickname_or_hexdigest( + "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA!")); + test_assert(is_legal_nickname_or_hexdigest("xyzzy")); + test_assert(is_legal_nickname_or_hexdigest("abcdefghijklmnopqrs")); + test_assert(!is_legal_nickname_or_hexdigest("abcdefghijklmnopqrst")); + done: + ; +} + +/** Run unit tests for router descriptor generation logic. */ +static void +test_dir_formats(void) +{ + char buf[8192], buf2[8192]; + char platform[256]; + char fingerprint[FINGERPRINT_LEN+1]; + char *pk1_str = NULL, *pk2_str = NULL, *pk3_str = NULL, *cp; + size_t pk1_str_len, pk2_str_len, pk3_str_len; + routerinfo_t *r1=NULL, *r2=NULL; + crypto_pk_env_t *pk1 = NULL, *pk2 = NULL, *pk3 = NULL; + routerinfo_t *rp1 = NULL; + addr_policy_t *ex1, *ex2; + routerlist_t *dir1 = NULL, *dir2 = NULL; + + pk1 = pk_generate(0); + pk2 = pk_generate(1); + pk3 = pk_generate(2); + + test_assert(pk1 && pk2 && pk3); + + get_platform_str(platform, sizeof(platform)); + r1 = tor_malloc_zero(sizeof(routerinfo_t)); + r1->address = tor_strdup("18.244.0.1"); + r1->addr = 0xc0a80001u; /* 192.168.0.1 */ + r1->cache_info.published_on = 0; + r1->or_port = 9000; + r1->dir_port = 9003; + r1->onion_pkey = crypto_pk_dup_key(pk1); + r1->identity_pkey = crypto_pk_dup_key(pk2); + r1->bandwidthrate = 1000; + r1->bandwidthburst = 5000; + r1->bandwidthcapacity = 10000; + r1->exit_policy = NULL; + r1->nickname = tor_strdup("Magri"); + r1->platform = tor_strdup(platform); + + ex1 = tor_malloc_zero(sizeof(addr_policy_t)); + ex2 = tor_malloc_zero(sizeof(addr_policy_t)); + ex1->policy_type = ADDR_POLICY_ACCEPT; + tor_addr_from_ipv4h(&ex1->addr, 0); + ex1->maskbits = 0; + ex1->prt_min = ex1->prt_max = 80; + ex2->policy_type = ADDR_POLICY_REJECT; + tor_addr_from_ipv4h(&ex2->addr, 18<<24); + ex2->maskbits = 8; + ex2->prt_min = ex2->prt_max = 24; + r2 = tor_malloc_zero(sizeof(routerinfo_t)); + r2->address = tor_strdup("1.1.1.1"); + r2->addr = 0x0a030201u; /* 10.3.2.1 */ + r2->platform = tor_strdup(platform); + r2->cache_info.published_on = 5; + r2->or_port = 9005; + r2->dir_port = 0; + r2->onion_pkey = crypto_pk_dup_key(pk2); + r2->identity_pkey = crypto_pk_dup_key(pk1); + r2->bandwidthrate = r2->bandwidthburst = r2->bandwidthcapacity = 3000; + r2->exit_policy = smartlist_create(); + smartlist_add(r2->exit_policy, ex2); + smartlist_add(r2->exit_policy, ex1); + r2->nickname = tor_strdup("Fred"); + + test_assert(!crypto_pk_write_public_key_to_string(pk1, &pk1_str, + &pk1_str_len)); + test_assert(!crypto_pk_write_public_key_to_string(pk2 , &pk2_str, + &pk2_str_len)); + test_assert(!crypto_pk_write_public_key_to_string(pk3 , &pk3_str, + &pk3_str_len)); + + memset(buf, 0, 2048); + test_assert(router_dump_router_to_string(buf, 2048, r1, pk2)>0); + + strlcpy(buf2, "router Magri 18.244.0.1 9000 0 9003\n" + "platform Tor "VERSION" on ", sizeof(buf2)); + strlcat(buf2, get_uname(), sizeof(buf2)); + strlcat(buf2, "\n" + "opt protocols Link 1 2 Circuit 1\n" + "published 1970-01-01 00:00:00\n" + "opt fingerprint ", sizeof(buf2)); + test_assert(!crypto_pk_get_fingerprint(pk2, fingerprint, 1)); + strlcat(buf2, fingerprint, sizeof(buf2)); + strlcat(buf2, "\nuptime 0\n" + /* XXX the "0" above is hard-coded, but even if we made it reflect + * uptime, that still wouldn't make it right, because the two + * descriptors might be made on different seconds... hm. */ + "bandwidth 1000 5000 10000\n" + "onion-key\n", sizeof(buf2)); + strlcat(buf2, pk1_str, sizeof(buf2)); + strlcat(buf2, "signing-key\n", sizeof(buf2)); + strlcat(buf2, pk2_str, sizeof(buf2)); + strlcat(buf2, "opt hidden-service-dir\n", sizeof(buf2)); + strlcat(buf2, "reject *:*\nrouter-signature\n", sizeof(buf2)); + buf[strlen(buf2)] = '\0'; /* Don't compare the sig; it's never the same + * twice */ + + test_streq(buf, buf2); + + test_assert(router_dump_router_to_string(buf, 2048, r1, pk2)>0); + cp = buf; + rp1 = router_parse_entry_from_string((const char*)cp,NULL,1,0,NULL); + test_assert(rp1); + test_streq(rp1->address, r1->address); + test_eq(rp1->or_port, r1->or_port); + //test_eq(rp1->dir_port, r1->dir_port); + test_eq(rp1->bandwidthrate, r1->bandwidthrate); + test_eq(rp1->bandwidthburst, r1->bandwidthburst); + test_eq(rp1->bandwidthcapacity, r1->bandwidthcapacity); + test_assert(crypto_pk_cmp_keys(rp1->onion_pkey, pk1) == 0); + test_assert(crypto_pk_cmp_keys(rp1->identity_pkey, pk2) == 0); + //test_assert(rp1->exit_policy == NULL); + +#if 0 + /* XXX Once we have exit policies, test this again. XXX */ + strlcpy(buf2, "router tor.tor.tor 9005 0 0 3000\n", sizeof(buf2)); + strlcat(buf2, pk2_str, sizeof(buf2)); + strlcat(buf2, "signing-key\n", sizeof(buf2)); + strlcat(buf2, pk1_str, sizeof(buf2)); + strlcat(buf2, "accept *:80\nreject 18.*:24\n\n", sizeof(buf2)); + test_assert(router_dump_router_to_string(buf, 2048, &r2, pk2)>0); + test_streq(buf, buf2); + + cp = buf; + rp2 = router_parse_entry_from_string(&cp,1); + test_assert(rp2); + test_streq(rp2->address, r2.address); + test_eq(rp2->or_port, r2.or_port); + test_eq(rp2->dir_port, r2.dir_port); + test_eq(rp2->bandwidth, r2.bandwidth); + test_assert(crypto_pk_cmp_keys(rp2->onion_pkey, pk2) == 0); + test_assert(crypto_pk_cmp_keys(rp2->identity_pkey, pk1) == 0); + test_eq(rp2->exit_policy->policy_type, EXIT_POLICY_ACCEPT); + test_streq(rp2->exit_policy->string, "accept *:80"); + test_streq(rp2->exit_policy->address, "*"); + test_streq(rp2->exit_policy->port, "80"); + test_eq(rp2->exit_policy->next->policy_type, EXIT_POLICY_REJECT); + test_streq(rp2->exit_policy->next->string, "reject 18.*:24"); + test_streq(rp2->exit_policy->next->address, "18.*"); + test_streq(rp2->exit_policy->next->port, "24"); + test_assert(rp2->exit_policy->next->next == NULL); + + /* Okay, now for the directories. */ + { + fingerprint_list = smartlist_create(); + crypto_pk_get_fingerprint(pk2, buf, 1); + add_fingerprint_to_dir("Magri", buf, fingerprint_list); + crypto_pk_get_fingerprint(pk1, buf, 1); + add_fingerprint_to_dir("Fred", buf, fingerprint_list); + } + + { + char d[DIGEST_LEN]; + const char *m; + /* XXXX NM re-enable. */ + /* Make sure routers aren't too far in the past any more. */ + r1->cache_info.published_on = time(NULL); + r2->cache_info.published_on = time(NULL)-3*60*60; + test_assert(router_dump_router_to_string(buf, 2048, r1, pk2)>0); + test_eq(dirserv_add_descriptor(buf,&m,""), ROUTER_ADDED_NOTIFY_GENERATOR); + test_assert(router_dump_router_to_string(buf, 2048, r2, pk1)>0); + test_eq(dirserv_add_descriptor(buf,&m,""), ROUTER_ADDED_NOTIFY_GENERATOR); + get_options()->Nickname = tor_strdup("DirServer"); + test_assert(!dirserv_dump_directory_to_string(&cp,pk3, 0)); + crypto_pk_get_digest(pk3, d); + test_assert(!router_parse_directory(cp)); + test_eq(2, smartlist_len(dir1->routers)); + tor_free(cp); + } +#endif + dirserv_free_fingerprint_list(); + + done: + if (r1) + routerinfo_free(r1); + if (r2) + routerinfo_free(r2); + + tor_free(pk1_str); + tor_free(pk2_str); + tor_free(pk3_str); + if (pk1) crypto_free_pk_env(pk1); + if (pk2) crypto_free_pk_env(pk2); + if (pk3) crypto_free_pk_env(pk3); + if (rp1) routerinfo_free(rp1); + tor_free(dir1); /* XXXX And more !*/ + tor_free(dir2); /* And more !*/ +} + +static void +test_dir_versions(void) +{ + tor_version_t ver1; + + /* Try out version parsing functionality */ + test_eq(0, tor_version_parse("0.3.4pre2-cvs", &ver1)); + test_eq(0, ver1.major); + test_eq(3, ver1.minor); + test_eq(4, ver1.micro); + test_eq(VER_PRE, ver1.status); + test_eq(2, ver1.patchlevel); + test_eq(0, tor_version_parse("0.3.4rc1", &ver1)); + test_eq(0, ver1.major); + test_eq(3, ver1.minor); + test_eq(4, ver1.micro); + test_eq(VER_RC, ver1.status); + test_eq(1, ver1.patchlevel); + test_eq(0, tor_version_parse("1.3.4", &ver1)); + test_eq(1, ver1.major); + test_eq(3, ver1.minor); + test_eq(4, ver1.micro); + test_eq(VER_RELEASE, ver1.status); + test_eq(0, ver1.patchlevel); + test_eq(0, tor_version_parse("1.3.4.999", &ver1)); + test_eq(1, ver1.major); + test_eq(3, ver1.minor); + test_eq(4, ver1.micro); + test_eq(VER_RELEASE, ver1.status); + test_eq(999, ver1.patchlevel); + test_eq(0, tor_version_parse("0.1.2.4-alpha", &ver1)); + test_eq(0, ver1.major); + test_eq(1, ver1.minor); + test_eq(2, ver1.micro); + test_eq(4, ver1.patchlevel); + test_eq(VER_RELEASE, ver1.status); + test_streq("alpha", ver1.status_tag); + test_eq(0, tor_version_parse("0.1.2.4", &ver1)); + test_eq(0, ver1.major); + test_eq(1, ver1.minor); + test_eq(2, ver1.micro); + test_eq(4, ver1.patchlevel); + test_eq(VER_RELEASE, ver1.status); + test_streq("", ver1.status_tag); + +#define tt_versionstatus_op(vs1, op, vs2) \ + tt_assert_test_type(vs1,vs2,#vs1" "#op" "#vs2,version_status_t, \ + (_val1 op _val2),"%d") +#define test_v_i_o(val, ver, lst) \ + tt_versionstatus_op(val, ==, tor_version_is_obsolete(ver, lst)) + + /* make sure tor_version_is_obsolete() works */ + test_v_i_o(VS_OLD, "0.0.1", "Tor 0.0.2"); + test_v_i_o(VS_OLD, "0.0.1", "0.0.2, Tor 0.0.3"); + test_v_i_o(VS_OLD, "0.0.1", "0.0.2,Tor 0.0.3"); + test_v_i_o(VS_OLD, "0.0.1","0.0.3,BetterTor 0.0.1"); + test_v_i_o(VS_RECOMMENDED, "0.0.2", "Tor 0.0.2,Tor 0.0.3"); + test_v_i_o(VS_NEW_IN_SERIES, "0.0.2", "Tor 0.0.2pre1,Tor 0.0.3"); + test_v_i_o(VS_OLD, "0.0.2", "Tor 0.0.2.1,Tor 0.0.3"); + test_v_i_o(VS_NEW, "0.1.0", "Tor 0.0.2,Tor 0.0.3"); + test_v_i_o(VS_RECOMMENDED, "0.0.7rc2", "0.0.7,Tor 0.0.7rc2,Tor 0.0.8"); + test_v_i_o(VS_OLD, "0.0.5.0", "0.0.5.1-cvs"); + test_v_i_o(VS_NEW_IN_SERIES, "0.0.5.1-cvs", "0.0.5, 0.0.6"); + /* Not on list, but newer than any in same series. */ + test_v_i_o(VS_NEW_IN_SERIES, "0.1.0.3", + "Tor 0.1.0.2,Tor 0.0.9.5,Tor 0.1.1.0"); + /* Series newer than any on list. */ + test_v_i_o(VS_NEW, "0.1.2.3", "Tor 0.1.0.2,Tor 0.0.9.5,Tor 0.1.1.0"); + /* Series older than any on list. */ + test_v_i_o(VS_OLD, "0.0.1.3", "Tor 0.1.0.2,Tor 0.0.9.5,Tor 0.1.1.0"); + /* Not on list, not newer than any on same series. */ + test_v_i_o(VS_UNRECOMMENDED, "0.1.0.1", + "Tor 0.1.0.2,Tor 0.0.9.5,Tor 0.1.1.0"); + /* On list, not newer than any on same series. */ + test_v_i_o(VS_UNRECOMMENDED, + "0.1.0.1", "Tor 0.1.0.2,Tor 0.0.9.5,Tor 0.1.1.0"); + test_eq(0, tor_version_as_new_as("Tor 0.0.5", "0.0.9pre1-cvs")); + test_eq(1, tor_version_as_new_as( + "Tor 0.0.8 on Darwin 64-121-192-100.c3-0." + "sfpo-ubr1.sfrn-sfpo.ca.cable.rcn.com Power Macintosh", + "0.0.8rc2")); + test_eq(0, tor_version_as_new_as( + "Tor 0.0.8 on Darwin 64-121-192-100.c3-0." + "sfpo-ubr1.sfrn-sfpo.ca.cable.rcn.com Power Macintosh", "0.0.8.2")); + + /* Now try svn revisions. */ + test_eq(1, tor_version_as_new_as("Tor 0.2.1.0-dev (r100)", + "Tor 0.2.1.0-dev (r99)")); + test_eq(1, tor_version_as_new_as("Tor 0.2.1.0-dev (r100) on Banana Jr", + "Tor 0.2.1.0-dev (r99) on Hal 9000")); + test_eq(1, tor_version_as_new_as("Tor 0.2.1.0-dev (r100)", + "Tor 0.2.1.0-dev on Colossus")); + test_eq(0, tor_version_as_new_as("Tor 0.2.1.0-dev (r99)", + "Tor 0.2.1.0-dev (r100)")); + test_eq(0, tor_version_as_new_as("Tor 0.2.1.0-dev (r99) on MCP", + "Tor 0.2.1.0-dev (r100) on AM")); + test_eq(0, tor_version_as_new_as("Tor 0.2.1.0-dev", + "Tor 0.2.1.0-dev (r99)")); + test_eq(1, tor_version_as_new_as("Tor 0.2.1.1", + "Tor 0.2.1.0-dev (r99)")); + + /* Now try git revisions */ + test_eq(0, tor_version_parse("0.5.6.7 (git-ff00ff)", &ver1)); + test_eq(0, ver1.major); + test_eq(5, ver1.minor); + test_eq(6, ver1.micro); + test_eq(7, ver1.patchlevel); + test_eq(3, ver1.git_tag_len); + test_memeq(ver1.git_tag, "\xff\x00\xff", 3); + test_eq(-1, tor_version_parse("0.5.6.7 (git-ff00xx)", &ver1)); + test_eq(-1, tor_version_parse("0.5.6.7 (git-ff00fff)", &ver1)); + test_eq(0, tor_version_parse("0.5.6.7 (git ff00fff)", &ver1)); + done: + ; +} + +/** Run unit tests for directory fp_pair functions. */ +static void +test_dir_fp_pairs(void) +{ + smartlist_t *sl = smartlist_create(); + fp_pair_t *pair; + + dir_split_resource_into_fingerprint_pairs( + /* Two pairs, out of order, with one duplicate. */ + "73656372657420646174612E0000000000FFFFFF-" + "557365204145532d32353620696e73746561642e+" + "73656372657420646174612E0000000000FFFFFF-" + "557365204145532d32353620696e73746561642e+" + "48657861646563696d616c2069736e277420736f-" + "676f6f6420666f7220686964696e6720796f7572.z", sl); + + test_eq(smartlist_len(sl), 2); + pair = smartlist_get(sl, 0); + test_memeq(pair->first, "Hexadecimal isn't so", DIGEST_LEN); + test_memeq(pair->second, "good for hiding your", DIGEST_LEN); + pair = smartlist_get(sl, 1); + test_memeq(pair->first, "secret data.\0\0\0\0\0\xff\xff\xff", DIGEST_LEN); + test_memeq(pair->second, "Use AES-256 instead.", DIGEST_LEN); + + done: + SMARTLIST_FOREACH(sl, fp_pair_t *, pair, tor_free(pair)); + smartlist_free(sl); +} + +static void +test_dir_split_fps(void *testdata) +{ + smartlist_t *sl = smartlist_create(); + char *mem_op_hex_tmp = NULL; + (void)testdata; + + /* Some example hex fingerprints and their base64 equivalents */ +#define HEX1 "Fe0daff89127389bc67558691231234551193EEE" +#define HEX2 "Deadbeef99999991111119999911111111f00ba4" +#define HEX3 "b33ff00db33ff00db33ff00db33ff00db33ff00d" +#define HEX256_1 \ + "f3f3f3f3fbbbbf3f3f3f3fbbbf3f3f3f3fbbbbf3f3f3f3fbbbf3f3f3f3fbbbbf" +#define HEX256_2 \ + "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccCCc" +#define HEX256_3 \ + "0123456789ABCdef0123456789ABCdef0123456789ABCdef0123456789ABCdef" +#define B64_1 "/g2v+JEnOJvGdVhpEjEjRVEZPu4" +#define B64_2 "3q2+75mZmZERERmZmRERERHwC6Q" +#define B64_3 "sz/wDbM/8A2zP/ANsz/wDbM/8A0" +#define B64_256_1 "8/Pz8/u7vz8/Pz+7vz8/Pz+7u/Pz8/P7u/Pz8/P7u78" +#define B64_256_2 "zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMw" +#define B64_256_3 "ASNFZ4mrze8BI0VniavN7wEjRWeJq83vASNFZ4mrze8" + + /* no flags set */ + dir_split_resource_into_fingerprints("A+C+B", sl, NULL, 0); + tt_int_op(smartlist_len(sl), ==, 3); + tt_str_op(smartlist_get(sl, 0), ==, "A"); + tt_str_op(smartlist_get(sl, 1), ==, "C"); + tt_str_op(smartlist_get(sl, 2), ==, "B"); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + /* uniq strings. */ + dir_split_resource_into_fingerprints("A+C+B+A+B+B", sl, NULL, DSR_SORT_UNIQ); + tt_int_op(smartlist_len(sl), ==, 3); + tt_str_op(smartlist_get(sl, 0), ==, "A"); + tt_str_op(smartlist_get(sl, 1), ==, "B"); + tt_str_op(smartlist_get(sl, 2), ==, "C"); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + /* Decode hex. */ + dir_split_resource_into_fingerprints(HEX1"+"HEX2, sl, NULL, DSR_HEX); + tt_int_op(smartlist_len(sl), ==, 2); + test_mem_op_hex(smartlist_get(sl, 0), ==, HEX1); + test_mem_op_hex(smartlist_get(sl, 1), ==, HEX2); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + /* decode hex and drop weirdness. */ + dir_split_resource_into_fingerprints(HEX1"+bogus+"HEX2"+"HEX256_1, + sl, NULL, DSR_HEX); + tt_int_op(smartlist_len(sl), ==, 2); + test_mem_op_hex(smartlist_get(sl, 0), ==, HEX1); + test_mem_op_hex(smartlist_get(sl, 1), ==, HEX2); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + /* Decode long hex */ + dir_split_resource_into_fingerprints(HEX256_1"+"HEX256_2"+"HEX2"+"HEX256_3, + sl, NULL, DSR_HEX|DSR_DIGEST256); + tt_int_op(smartlist_len(sl), ==, 3); + test_mem_op_hex(smartlist_get(sl, 0), ==, HEX256_1); + test_mem_op_hex(smartlist_get(sl, 1), ==, HEX256_2); + test_mem_op_hex(smartlist_get(sl, 2), ==, HEX256_3); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + /* Decode hex and sort. */ + dir_split_resource_into_fingerprints(HEX1"+"HEX2"+"HEX3"+"HEX2, + sl, NULL, DSR_HEX|DSR_SORT_UNIQ); + tt_int_op(smartlist_len(sl), ==, 3); + test_mem_op_hex(smartlist_get(sl, 0), ==, HEX3); + test_mem_op_hex(smartlist_get(sl, 1), ==, HEX2); + test_mem_op_hex(smartlist_get(sl, 2), ==, HEX1); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + /* Decode long hex and sort */ + dir_split_resource_into_fingerprints(HEX256_1"+"HEX256_2"+"HEX256_3 + "+"HEX256_1, + sl, NULL, + DSR_HEX|DSR_DIGEST256|DSR_SORT_UNIQ); + tt_int_op(smartlist_len(sl), ==, 3); + test_mem_op_hex(smartlist_get(sl, 0), ==, HEX256_3); + test_mem_op_hex(smartlist_get(sl, 1), ==, HEX256_2); + test_mem_op_hex(smartlist_get(sl, 2), ==, HEX256_1); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + /* Decode base64 */ + dir_split_resource_into_fingerprints(B64_1"-"B64_2, sl, NULL, DSR_BASE64); + tt_int_op(smartlist_len(sl), ==, 2); + test_mem_op_hex(smartlist_get(sl, 0), ==, HEX1); + test_mem_op_hex(smartlist_get(sl, 1), ==, HEX2); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + /* Decode long base64 */ + dir_split_resource_into_fingerprints(B64_256_1"-"B64_256_2, + sl, NULL, DSR_BASE64|DSR_DIGEST256); + tt_int_op(smartlist_len(sl), ==, 2); + test_mem_op_hex(smartlist_get(sl, 0), ==, HEX256_1); + test_mem_op_hex(smartlist_get(sl, 1), ==, HEX256_2); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + dir_split_resource_into_fingerprints(B64_256_1, + sl, NULL, DSR_BASE64|DSR_DIGEST256); + tt_int_op(smartlist_len(sl), ==, 1); + test_mem_op_hex(smartlist_get(sl, 0), ==, HEX256_1); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + done: + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_free(sl); + tor_free(mem_op_hex_tmp); +} + +static void +test_dir_measured_bw(void) +{ + measured_bw_line_t mbwl; + int i; + const char *lines_pass[] = { + "node_id=$557365204145532d32353620696e73746561642e bw=1024\n", + "node_id=$557365204145532d32353620696e73746561642e\t bw=1024 \n", + " node_id=$557365204145532d32353620696e73746561642e bw=1024\n", + "\tnoise\tnode_id=$557365204145532d32353620696e73746561642e " + "bw=1024 junk=007\n", + "misc=junk node_id=$557365204145532d32353620696e73746561642e " + "bw=1024 junk=007\n", + "end" + }; + const char *lines_fail[] = { + /* Test possible python stupidity on input */ + "node_id=None bw=1024\n", + "node_id=$None bw=1024\n", + "node_id=$557365204145532d32353620696e73746561642e bw=None\n", + "node_id=$557365204145532d32353620696e73746561642e bw=1024.0\n", + "node_id=$557365204145532d32353620696e73746561642e bw=.1024\n", + "node_id=$557365204145532d32353620696e73746561642e bw=1.024\n", + "node_id=$557365204145532d32353620696e73746561642e bw=1024 bw=0\n", + "node_id=$557365204145532d32353620696e73746561642e bw=1024 bw=None\n", + "node_id=$557365204145532d32353620696e73746561642e bw=-1024\n", + /* Test incomplete writes due to race conditions, partial copies, etc */ + "node_i", + "node_i\n", + "node_id=", + "node_id=\n", + "node_id=$557365204145532d32353620696e73746561642e bw=", + "node_id=$557365204145532d32353620696e73746561642e bw=1024", + "node_id=$557365204145532d32353620696e73746561642e bw=\n", + "node_id=$557365204145532d32353620696e7374", + "node_id=$557365204145532d32353620696e7374\n", + "", + "\n", + " \n ", + " \n\n", + /* Test assorted noise */ + " node_id= ", + "node_id==$557365204145532d32353620696e73746561642e bw==1024\n", + "node_id=$55736520414552d32353620696e73746561642e bw=1024\n", + "node_id=557365204145532d32353620696e73746561642e bw=1024\n", + "node_id= $557365204145532d32353620696e73746561642e bw=0.23\n", + "end" + }; + + for (i = 0; strcmp(lines_fail[i], "end"); i++) { + //fprintf(stderr, "Testing: %s\n", lines_fail[i]); + test_assert(measured_bw_line_parse(&mbwl, lines_fail[i]) == -1); + } + + for (i = 0; strcmp(lines_pass[i], "end"); i++) { + //fprintf(stderr, "Testing: %s %d\n", lines_pass[i], TOR_ISSPACE('\n')); + test_assert(measured_bw_line_parse(&mbwl, lines_pass[i]) == 0); + test_assert(mbwl.bw == 1024); + test_assert(strcmp(mbwl.node_hex, + "557365204145532d32353620696e73746561642e") == 0); + } + + done: + return; +} + +static void +test_dir_param_voting(void) +{ + networkstatus_t vote1, vote2, vote3, vote4; + smartlist_t *votes = smartlist_create(); + char *res = NULL; + + /* dirvote_compute_params only looks at the net_params field of the votes, + so that's all we need to set. + */ + memset(&vote1, 0, sizeof(vote1)); + memset(&vote2, 0, sizeof(vote2)); + memset(&vote3, 0, sizeof(vote3)); + memset(&vote4, 0, sizeof(vote4)); + vote1.net_params = smartlist_create(); + vote2.net_params = smartlist_create(); + vote3.net_params = smartlist_create(); + vote4.net_params = smartlist_create(); + smartlist_split_string(vote1.net_params, + "ab=90 abcd=20 cw=50 x-yz=-99", NULL, 0, 0); + smartlist_split_string(vote2.net_params, + "ab=27 cw=5 x-yz=88", NULL, 0, 0); + smartlist_split_string(vote3.net_params, + "abcd=20 c=60 cw=500 x-yz=-9 zzzzz=101", NULL, 0, 0); + smartlist_split_string(vote4.net_params, + "ab=900 abcd=200 c=1 cw=51 x-yz=100", NULL, 0, 0); + test_eq(100, networkstatus_get_param(&vote4, "x-yz", 50, 0, 300)); + test_eq(222, networkstatus_get_param(&vote4, "foobar", 222, 0, 300)); + test_eq(80, networkstatus_get_param(&vote4, "ab", 12, 0, 80)); + test_eq(-8, networkstatus_get_param(&vote4, "ab", -12, -100, -8)); + test_eq(0, networkstatus_get_param(&vote4, "foobar", 0, -100, 8)); + + smartlist_add(votes, &vote1); + smartlist_add(votes, &vote2); + smartlist_add(votes, &vote3); + smartlist_add(votes, &vote4); + + res = dirvote_compute_params(votes); + test_streq(res, + "ab=90 abcd=20 c=1 cw=50 x-yz=-9 zzzzz=101"); + + done: + tor_free(res); + SMARTLIST_FOREACH(vote1.net_params, char *, cp, tor_free(cp)); + SMARTLIST_FOREACH(vote2.net_params, char *, cp, tor_free(cp)); + SMARTLIST_FOREACH(vote3.net_params, char *, cp, tor_free(cp)); + SMARTLIST_FOREACH(vote4.net_params, char *, cp, tor_free(cp)); + smartlist_free(vote1.net_params); + smartlist_free(vote2.net_params); + smartlist_free(vote3.net_params); + smartlist_free(vote4.net_params); + smartlist_free(votes); + + return; +} + +extern const char AUTHORITY_CERT_1[]; +extern const char AUTHORITY_SIGNKEY_1[]; +extern const char AUTHORITY_CERT_2[]; +extern const char AUTHORITY_SIGNKEY_2[]; +extern const char AUTHORITY_CERT_3[]; +extern const char AUTHORITY_SIGNKEY_3[]; + +/** Helper: Test that two networkstatus_voter_info_t do in fact represent the + * same voting authority, and that they do in fact have all the same + * information. */ +static void +test_same_voter(networkstatus_voter_info_t *v1, + networkstatus_voter_info_t *v2) +{ + test_streq(v1->nickname, v2->nickname); + test_memeq(v1->identity_digest, v2->identity_digest, DIGEST_LEN); + test_streq(v1->address, v2->address); + test_eq(v1->addr, v2->addr); + test_eq(v1->dir_port, v2->dir_port); + test_eq(v1->or_port, v2->or_port); + test_streq(v1->contact, v2->contact); + test_memeq(v1->vote_digest, v2->vote_digest, DIGEST_LEN); + done: + ; +} + +/** Helper: Make a new routerinfo containing the right information for a + * given vote_routerstatus_t. */ +static routerinfo_t * +generate_ri_from_rs(const vote_routerstatus_t *vrs) +{ + routerinfo_t *r; + const routerstatus_t *rs = &vrs->status; + static time_t published = 0; + + r = tor_malloc_zero(sizeof(routerinfo_t)); + memcpy(r->cache_info.identity_digest, rs->identity_digest, DIGEST_LEN); + memcpy(r->cache_info.signed_descriptor_digest, rs->descriptor_digest, + DIGEST_LEN); + r->cache_info.do_not_cache = 1; + r->cache_info.routerlist_index = -1; + r->cache_info.signed_descriptor_body = + tor_strdup("123456789012345678901234567890123"); + r->cache_info.signed_descriptor_len = + strlen(r->cache_info.signed_descriptor_body); + r->exit_policy = smartlist_create(); + r->cache_info.published_on = ++published + time(NULL); + return r; +} + +/** Helper: get a detached signatures document for one or two + * consensuses. */ +static char * +get_detached_sigs(networkstatus_t *ns, networkstatus_t *ns2) +{ + char *r; + smartlist_t *sl; + tor_assert(ns && ns->flavor == FLAV_NS); + sl = smartlist_create(); + smartlist_add(sl,ns); + if (ns2) + smartlist_add(sl,ns2); + r = networkstatus_get_detached_signatures(sl); + smartlist_free(sl); + return r; +} + +/** Run unit tests for generating and parsing V3 consensus networkstatus + * documents. */ +static void +test_dir_v3_networkstatus(void) +{ + authority_cert_t *cert1=NULL, *cert2=NULL, *cert3=NULL; + crypto_pk_env_t *sign_skey_1=NULL, *sign_skey_2=NULL, *sign_skey_3=NULL; + crypto_pk_env_t *sign_skey_leg1=NULL; + const char *msg=NULL; + + time_t now = time(NULL); + networkstatus_voter_info_t *voter; + document_signature_t *sig; + networkstatus_t *vote=NULL, *v1=NULL, *v2=NULL, *v3=NULL, *con=NULL, + *con_md=NULL; + vote_routerstatus_t *vrs; + routerstatus_t *rs; + char *v1_text=NULL, *v2_text=NULL, *v3_text=NULL, *consensus_text=NULL, *cp; + smartlist_t *votes = smartlist_create(); + + /* For generating the two other consensuses. */ + char *detached_text1=NULL, *detached_text2=NULL; + char *consensus_text2=NULL, *consensus_text3=NULL; + char *consensus_text_md2=NULL, *consensus_text_md3=NULL; + char *consensus_text_md=NULL; + networkstatus_t *con2=NULL, *con_md2=NULL, *con3=NULL, *con_md3=NULL; + ns_detached_signatures_t *dsig1=NULL, *dsig2=NULL; + + /* Parse certificates and keys. */ + cert1 = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL); + test_assert(cert1); + test_assert(cert1->is_cross_certified); + cert2 = authority_cert_parse_from_string(AUTHORITY_CERT_2, NULL); + test_assert(cert2); + cert3 = authority_cert_parse_from_string(AUTHORITY_CERT_3, NULL); + test_assert(cert3); + sign_skey_1 = crypto_new_pk_env(); + sign_skey_2 = crypto_new_pk_env(); + sign_skey_3 = crypto_new_pk_env(); + sign_skey_leg1 = pk_generate(4); + + test_assert(!crypto_pk_read_private_key_from_string(sign_skey_1, + AUTHORITY_SIGNKEY_1, -1)); + test_assert(!crypto_pk_read_private_key_from_string(sign_skey_2, + AUTHORITY_SIGNKEY_2, -1)); + test_assert(!crypto_pk_read_private_key_from_string(sign_skey_3, + AUTHORITY_SIGNKEY_3, -1)); + + test_assert(!crypto_pk_cmp_keys(sign_skey_1, cert1->signing_key)); + test_assert(!crypto_pk_cmp_keys(sign_skey_2, cert2->signing_key)); + + /* + * Set up a vote; generate it; try to parse it. + */ + vote = tor_malloc_zero(sizeof(networkstatus_t)); + vote->type = NS_TYPE_VOTE; + vote->published = now; + vote->valid_after = now+1000; + vote->fresh_until = now+2000; + vote->valid_until = now+3000; + vote->vote_seconds = 100; + vote->dist_seconds = 200; + vote->supported_methods = smartlist_create(); + smartlist_split_string(vote->supported_methods, "1 2 3", NULL, 0, -1); + vote->client_versions = tor_strdup("0.1.2.14,0.1.2.15"); + vote->server_versions = tor_strdup("0.1.2.14,0.1.2.15,0.1.2.16"); + vote->known_flags = smartlist_create(); + smartlist_split_string(vote->known_flags, + "Authority Exit Fast Guard Running Stable V2Dir Valid", + 0, SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); + vote->voters = smartlist_create(); + voter = tor_malloc_zero(sizeof(networkstatus_voter_info_t)); + voter->nickname = tor_strdup("Voter1"); + voter->address = tor_strdup("1.2.3.4"); + voter->addr = 0x01020304; + voter->dir_port = 80; + voter->or_port = 9000; + voter->contact = tor_strdup("voter@example.com"); + crypto_pk_get_digest(cert1->identity_key, voter->identity_digest); + smartlist_add(vote->voters, voter); + vote->cert = authority_cert_dup(cert1); + vote->net_params = smartlist_create(); + smartlist_split_string(vote->net_params, "circuitwindow=101 foo=990", + NULL, 0, 0); + vote->routerstatus_list = smartlist_create(); + /* add the first routerstatus. */ + vrs = tor_malloc_zero(sizeof(vote_routerstatus_t)); + rs = &vrs->status; + vrs->version = tor_strdup("0.1.2.14"); + rs->published_on = now-1500; + strlcpy(rs->nickname, "router2", sizeof(rs->nickname)); + memset(rs->identity_digest, 3, DIGEST_LEN); + memset(rs->descriptor_digest, 78, DIGEST_LEN); + rs->addr = 0x99008801; + rs->or_port = 443; + rs->dir_port = 8000; + /* all flags but running cleared */ + rs->is_running = 1; + smartlist_add(vote->routerstatus_list, vrs); + test_assert(router_add_to_routerlist(generate_ri_from_rs(vrs), &msg,0,0)>=0); + + /* add the second routerstatus. */ + vrs = tor_malloc_zero(sizeof(vote_routerstatus_t)); + rs = &vrs->status; + vrs->version = tor_strdup("0.2.0.5"); + rs->published_on = now-1000; + strlcpy(rs->nickname, "router1", sizeof(rs->nickname)); + memset(rs->identity_digest, 5, DIGEST_LEN); + memset(rs->descriptor_digest, 77, DIGEST_LEN); + rs->addr = 0x99009901; + rs->or_port = 443; + rs->dir_port = 0; + rs->is_exit = rs->is_stable = rs->is_fast = rs->is_running = + rs->is_valid = rs->is_v2_dir = rs->is_possible_guard = 1; + smartlist_add(vote->routerstatus_list, vrs); + test_assert(router_add_to_routerlist(generate_ri_from_rs(vrs), &msg,0,0)>=0); + + /* add the third routerstatus. */ + vrs = tor_malloc_zero(sizeof(vote_routerstatus_t)); + rs = &vrs->status; + vrs->version = tor_strdup("0.1.0.3"); + rs->published_on = now-1000; + strlcpy(rs->nickname, "router3", sizeof(rs->nickname)); + memset(rs->identity_digest, 33, DIGEST_LEN); + memset(rs->descriptor_digest, 79, DIGEST_LEN); + rs->addr = 0xAA009901; + rs->or_port = 400; + rs->dir_port = 9999; + rs->is_authority = rs->is_exit = rs->is_stable = rs->is_fast = + rs->is_running = rs->is_valid = rs->is_v2_dir = rs->is_possible_guard = 1; + smartlist_add(vote->routerstatus_list, vrs); + test_assert(router_add_to_routerlist(generate_ri_from_rs(vrs), &msg,0,0)>=0); + + /* add a fourth routerstatus that is not running. */ + vrs = tor_malloc_zero(sizeof(vote_routerstatus_t)); + rs = &vrs->status; + vrs->version = tor_strdup("0.1.6.3"); + rs->published_on = now-1000; + strlcpy(rs->nickname, "router4", sizeof(rs->nickname)); + memset(rs->identity_digest, 34, DIGEST_LEN); + memset(rs->descriptor_digest, 47, DIGEST_LEN); + rs->addr = 0xC0000203; + rs->or_port = 500; + rs->dir_port = 1999; + /* Running flag (and others) cleared */ + smartlist_add(vote->routerstatus_list, vrs); + test_assert(router_add_to_routerlist(generate_ri_from_rs(vrs), &msg,0,0)>=0); + + /* dump the vote and try to parse it. */ + v1_text = format_networkstatus_vote(sign_skey_1, vote); + test_assert(v1_text); + v1 = networkstatus_parse_vote_from_string(v1_text, NULL, NS_TYPE_VOTE); + test_assert(v1); + + /* Make sure the parsed thing was right. */ + test_eq(v1->type, NS_TYPE_VOTE); + test_eq(v1->published, vote->published); + test_eq(v1->valid_after, vote->valid_after); + test_eq(v1->fresh_until, vote->fresh_until); + test_eq(v1->valid_until, vote->valid_until); + test_eq(v1->vote_seconds, vote->vote_seconds); + test_eq(v1->dist_seconds, vote->dist_seconds); + test_streq(v1->client_versions, vote->client_versions); + test_streq(v1->server_versions, vote->server_versions); + test_assert(v1->voters && smartlist_len(v1->voters)); + voter = smartlist_get(v1->voters, 0); + test_streq(voter->nickname, "Voter1"); + test_streq(voter->address, "1.2.3.4"); + test_eq(voter->addr, 0x01020304); + test_eq(voter->dir_port, 80); + test_eq(voter->or_port, 9000); + test_streq(voter->contact, "voter@example.com"); + test_assert(v1->cert); + test_assert(!crypto_pk_cmp_keys(sign_skey_1, v1->cert->signing_key)); + cp = smartlist_join_strings(v1->known_flags, ":", 0, NULL); + test_streq(cp, "Authority:Exit:Fast:Guard:Running:Stable:V2Dir:Valid"); + tor_free(cp); + test_eq(smartlist_len(v1->routerstatus_list), 4); + /* Check the first routerstatus. */ + vrs = smartlist_get(v1->routerstatus_list, 0); + rs = &vrs->status; + test_streq(vrs->version, "0.1.2.14"); + test_eq(rs->published_on, now-1500); + test_streq(rs->nickname, "router2"); + test_memeq(rs->identity_digest, + "\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3", + DIGEST_LEN); + test_memeq(rs->descriptor_digest, "NNNNNNNNNNNNNNNNNNNN", DIGEST_LEN); + test_eq(rs->addr, 0x99008801); + test_eq(rs->or_port, 443); + test_eq(rs->dir_port, 8000); + test_eq(vrs->flags, U64_LITERAL(16)); // no flags except "running" + /* Check the second routerstatus. */ + vrs = smartlist_get(v1->routerstatus_list, 1); + rs = &vrs->status; + test_streq(vrs->version, "0.2.0.5"); + test_eq(rs->published_on, now-1000); + test_streq(rs->nickname, "router1"); + test_memeq(rs->identity_digest, + "\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5", + DIGEST_LEN); + test_memeq(rs->descriptor_digest, "MMMMMMMMMMMMMMMMMMMM", DIGEST_LEN); + test_eq(rs->addr, 0x99009901); + test_eq(rs->or_port, 443); + test_eq(rs->dir_port, 0); + test_eq(vrs->flags, U64_LITERAL(254)); // all flags except "authority." + + { + measured_bw_line_t mbw; + memset(mbw.node_id, 33, sizeof(mbw.node_id)); + mbw.bw = 1024; + test_assert(measured_bw_line_apply(&mbw, + v1->routerstatus_list) == 1); + vrs = smartlist_get(v1->routerstatus_list, 2); + test_assert(vrs->status.has_measured_bw && + vrs->status.measured_bw == 1024); + } + + /* Generate second vote. It disagrees on some of the times, + * and doesn't list versions, and knows some crazy flags */ + vote->published = now+1; + vote->fresh_until = now+3005; + vote->dist_seconds = 300; + authority_cert_free(vote->cert); + vote->cert = authority_cert_dup(cert2); + vote->net_params = smartlist_create(); + smartlist_split_string(vote->net_params, "bar=2000000000 circuitwindow=20", + NULL, 0, 0); + tor_free(vote->client_versions); + tor_free(vote->server_versions); + voter = smartlist_get(vote->voters, 0); + tor_free(voter->nickname); + tor_free(voter->address); + voter->nickname = tor_strdup("Voter2"); + voter->address = tor_strdup("2.3.4.5"); + voter->addr = 0x02030405; + crypto_pk_get_digest(cert2->identity_key, voter->identity_digest); + smartlist_add(vote->known_flags, tor_strdup("MadeOfCheese")); + smartlist_add(vote->known_flags, tor_strdup("MadeOfTin")); + smartlist_sort_strings(vote->known_flags); + vrs = smartlist_get(vote->routerstatus_list, 2); + smartlist_del_keeporder(vote->routerstatus_list, 2); + tor_free(vrs->version); + tor_free(vrs); + vrs = smartlist_get(vote->routerstatus_list, 0); + vrs->status.is_fast = 1; + /* generate and parse. */ + v2_text = format_networkstatus_vote(sign_skey_2, vote); + test_assert(v2_text); + v2 = networkstatus_parse_vote_from_string(v2_text, NULL, NS_TYPE_VOTE); + test_assert(v2); + /* Check that flags come out right.*/ + cp = smartlist_join_strings(v2->known_flags, ":", 0, NULL); + test_streq(cp, "Authority:Exit:Fast:Guard:MadeOfCheese:MadeOfTin:" + "Running:Stable:V2Dir:Valid"); + tor_free(cp); + vrs = smartlist_get(v2->routerstatus_list, 1); + /* 1023 - authority(1) - madeofcheese(16) - madeoftin(32) */ + test_eq(vrs->flags, U64_LITERAL(974)); + + /* Generate the third vote. */ + vote->published = now; + vote->fresh_until = now+2003; + vote->dist_seconds = 250; + authority_cert_free(vote->cert); + vote->cert = authority_cert_dup(cert3); + vote->net_params = smartlist_create(); + smartlist_split_string(vote->net_params, "circuitwindow=80 foo=660", + NULL, 0, 0); + smartlist_add(vote->supported_methods, tor_strdup("4")); + vote->client_versions = tor_strdup("0.1.2.14,0.1.2.17"); + vote->server_versions = tor_strdup("0.1.2.10,0.1.2.15,0.1.2.16"); + voter = smartlist_get(vote->voters, 0); + tor_free(voter->nickname); + tor_free(voter->address); + voter->nickname = tor_strdup("Voter3"); + voter->address = tor_strdup("3.4.5.6"); + voter->addr = 0x03040506; + crypto_pk_get_digest(cert3->identity_key, voter->identity_digest); + /* This one has a legacy id. */ + memset(voter->legacy_id_digest, (int)'A', DIGEST_LEN); + vrs = smartlist_get(vote->routerstatus_list, 0); + smartlist_del_keeporder(vote->routerstatus_list, 0); + tor_free(vrs->version); + tor_free(vrs); + vrs = smartlist_get(vote->routerstatus_list, 0); + memset(vrs->status.descriptor_digest, (int)'Z', DIGEST_LEN); + test_assert(router_add_to_routerlist(generate_ri_from_rs(vrs), &msg,0,0)>=0); + + v3_text = format_networkstatus_vote(sign_skey_3, vote); + test_assert(v3_text); + + v3 = networkstatus_parse_vote_from_string(v3_text, NULL, NS_TYPE_VOTE); + test_assert(v3); + + /* Compute a consensus as voter 3. */ + smartlist_add(votes, v3); + smartlist_add(votes, v1); + smartlist_add(votes, v2); + consensus_text = networkstatus_compute_consensus(votes, 3, + cert3->identity_key, + sign_skey_3, + "AAAAAAAAAAAAAAAAAAAA", + sign_skey_leg1, + FLAV_NS); + test_assert(consensus_text); + con = networkstatus_parse_vote_from_string(consensus_text, NULL, + NS_TYPE_CONSENSUS); + test_assert(con); + //log_notice(LD_GENERAL, "<<%s>>\n<<%s>>\n<<%s>>\n", + // v1_text, v2_text, v3_text); + consensus_text_md = networkstatus_compute_consensus(votes, 3, + cert3->identity_key, + sign_skey_3, + "AAAAAAAAAAAAAAAAAAAA", + sign_skey_leg1, + FLAV_MICRODESC); + test_assert(consensus_text_md); + con_md = networkstatus_parse_vote_from_string(consensus_text_md, NULL, + NS_TYPE_CONSENSUS); + test_assert(con_md); + test_eq(con_md->flavor, FLAV_MICRODESC); + + /* Check consensus contents. */ + test_assert(con->type == NS_TYPE_CONSENSUS); + test_eq(con->published, 0); /* this field only appears in votes. */ + test_eq(con->valid_after, now+1000); + test_eq(con->fresh_until, now+2003); /* median */ + test_eq(con->valid_until, now+3000); + test_eq(con->vote_seconds, 100); + test_eq(con->dist_seconds, 250); /* median */ + test_streq(con->client_versions, "0.1.2.14"); + test_streq(con->server_versions, "0.1.2.15,0.1.2.16"); + cp = smartlist_join_strings(v2->known_flags, ":", 0, NULL); + test_streq(cp, "Authority:Exit:Fast:Guard:MadeOfCheese:MadeOfTin:" + "Running:Stable:V2Dir:Valid"); + tor_free(cp); + cp = smartlist_join_strings(con->net_params, ":", 0, NULL); + test_streq(cp, "bar=2000000000:circuitwindow=80:foo=660"); + tor_free(cp); + + test_eq(4, smartlist_len(con->voters)); /*3 voters, 1 legacy key.*/ + /* The voter id digests should be in this order. */ + test_assert(memcmp(cert2->cache_info.identity_digest, + cert1->cache_info.identity_digest,DIGEST_LEN)<0); + test_assert(memcmp(cert1->cache_info.identity_digest, + cert3->cache_info.identity_digest,DIGEST_LEN)<0); + test_same_voter(smartlist_get(con->voters, 1), + smartlist_get(v2->voters, 0)); + test_same_voter(smartlist_get(con->voters, 2), + smartlist_get(v1->voters, 0)); + test_same_voter(smartlist_get(con->voters, 3), + smartlist_get(v3->voters, 0)); + + test_assert(!con->cert); + test_eq(2, smartlist_len(con->routerstatus_list)); + /* There should be two listed routers: one with identity 3, one with + * identity 5. */ + /* This one showed up in 2 digests. */ + rs = smartlist_get(con->routerstatus_list, 0); + test_memeq(rs->identity_digest, + "\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3", + DIGEST_LEN); + test_memeq(rs->descriptor_digest, "NNNNNNNNNNNNNNNNNNNN", DIGEST_LEN); + test_assert(!rs->is_authority); + test_assert(!rs->is_exit); + test_assert(!rs->is_fast); + test_assert(!rs->is_possible_guard); + test_assert(!rs->is_stable); + test_assert(rs->is_running); /* If it wasn't running it wouldn't be here */ + test_assert(!rs->is_v2_dir); + test_assert(!rs->is_valid); + test_assert(!rs->is_named); + /* XXXX check version */ + + rs = smartlist_get(con->routerstatus_list, 1); + /* This one showed up in 3 digests. Twice with ID 'M', once with 'Z'. */ + test_memeq(rs->identity_digest, + "\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5", + DIGEST_LEN); + test_streq(rs->nickname, "router1"); + test_memeq(rs->descriptor_digest, "MMMMMMMMMMMMMMMMMMMM", DIGEST_LEN); + test_eq(rs->published_on, now-1000); + test_eq(rs->addr, 0x99009901); + test_eq(rs->or_port, 443); + test_eq(rs->dir_port, 0); + test_assert(!rs->is_authority); + test_assert(rs->is_exit); + test_assert(rs->is_fast); + test_assert(rs->is_possible_guard); + test_assert(rs->is_stable); + test_assert(rs->is_running); + test_assert(rs->is_v2_dir); + test_assert(rs->is_valid); + test_assert(!rs->is_named); + /* XXXX check version */ + + /* Check signatures. the first voter is a pseudo-entry with a legacy key. + * The second one hasn't signed. The fourth one has signed: validate it. */ + voter = smartlist_get(con->voters, 1); + test_eq(smartlist_len(voter->sigs), 0); + + voter = smartlist_get(con->voters, 3); + test_eq(smartlist_len(voter->sigs), 1); + sig = smartlist_get(voter->sigs, 0); + test_assert(sig->signature); + test_assert(!sig->good_signature); + test_assert(!sig->bad_signature); + + test_assert(!networkstatus_check_document_signature(con, sig, cert3)); + test_assert(sig->signature); + test_assert(sig->good_signature); + test_assert(!sig->bad_signature); + + { + const char *msg=NULL; + /* Compute the other two signed consensuses. */ + smartlist_shuffle(votes); + consensus_text2 = networkstatus_compute_consensus(votes, 3, + cert2->identity_key, + sign_skey_2, NULL,NULL, + FLAV_NS); + consensus_text_md2 = networkstatus_compute_consensus(votes, 3, + cert2->identity_key, + sign_skey_2, NULL,NULL, + FLAV_MICRODESC); + smartlist_shuffle(votes); + consensus_text3 = networkstatus_compute_consensus(votes, 3, + cert1->identity_key, + sign_skey_1, NULL,NULL, + FLAV_NS); + consensus_text_md3 = networkstatus_compute_consensus(votes, 3, + cert1->identity_key, + sign_skey_1, NULL,NULL, + FLAV_MICRODESC); + test_assert(consensus_text2); + test_assert(consensus_text3); + test_assert(consensus_text_md2); + test_assert(consensus_text_md3); + con2 = networkstatus_parse_vote_from_string(consensus_text2, NULL, + NS_TYPE_CONSENSUS); + con3 = networkstatus_parse_vote_from_string(consensus_text3, NULL, + NS_TYPE_CONSENSUS); + con_md2 = networkstatus_parse_vote_from_string(consensus_text_md2, NULL, + NS_TYPE_CONSENSUS); + con_md3 = networkstatus_parse_vote_from_string(consensus_text_md3, NULL, + NS_TYPE_CONSENSUS); + test_assert(con2); + test_assert(con3); + test_assert(con_md2); + test_assert(con_md3); + + /* All three should have the same digest. */ + test_memeq(&con->digests, &con2->digests, sizeof(digests_t)); + test_memeq(&con->digests, &con3->digests, sizeof(digests_t)); + + test_memeq(&con_md->digests, &con_md2->digests, sizeof(digests_t)); + test_memeq(&con_md->digests, &con_md3->digests, sizeof(digests_t)); + + /* Extract a detached signature from con3. */ + detached_text1 = get_detached_sigs(con3, con_md3); + tor_assert(detached_text1); + /* Try to parse it. */ + dsig1 = networkstatus_parse_detached_signatures(detached_text1, NULL); + tor_assert(dsig1); + + /* Are parsed values as expected? */ + test_eq(dsig1->valid_after, con3->valid_after); + test_eq(dsig1->fresh_until, con3->fresh_until); + test_eq(dsig1->valid_until, con3->valid_until); + { + digests_t *dsig_digests = strmap_get(dsig1->digests, "ns"); + test_assert(dsig_digests); + test_memeq(dsig_digests->d[DIGEST_SHA1], con3->digests.d[DIGEST_SHA1], + DIGEST_LEN); + dsig_digests = strmap_get(dsig1->digests, "microdesc"); + test_assert(dsig_digests); + test_memeq(dsig_digests->d[DIGEST_SHA256], + con_md3->digests.d[DIGEST_SHA256], + DIGEST256_LEN); + } + { + smartlist_t *dsig_signatures = strmap_get(dsig1->signatures, "ns"); + test_assert(dsig_signatures); + test_eq(1, smartlist_len(dsig_signatures)); + sig = smartlist_get(dsig_signatures, 0); + test_memeq(sig->identity_digest, cert1->cache_info.identity_digest, + DIGEST_LEN); + test_eq(sig->alg, DIGEST_SHA1); + + dsig_signatures = strmap_get(dsig1->signatures, "microdesc"); + test_assert(dsig_signatures); + test_eq(1, smartlist_len(dsig_signatures)); + sig = smartlist_get(dsig_signatures, 0); + test_memeq(sig->identity_digest, cert1->cache_info.identity_digest, + DIGEST_LEN); + test_eq(sig->alg, DIGEST_SHA256); + } + + /* Try adding it to con2. */ + detached_text2 = get_detached_sigs(con2,con_md2); + test_eq(1, networkstatus_add_detached_signatures(con2, dsig1, &msg)); + tor_free(detached_text2); + test_eq(1, networkstatus_add_detached_signatures(con_md2, dsig1, &msg)); + tor_free(detached_text2); + detached_text2 = get_detached_sigs(con2,con_md2); + //printf("\n<%s>\n", detached_text2); + dsig2 = networkstatus_parse_detached_signatures(detached_text2, NULL); + test_assert(dsig2); + /* + printf("\n"); + SMARTLIST_FOREACH(dsig2->signatures, networkstatus_voter_info_t *, vi, { + char hd[64]; + base16_encode(hd, sizeof(hd), vi->identity_digest, DIGEST_LEN); + printf("%s\n", hd); + }); + */ + test_eq(2, + smartlist_len((smartlist_t*)strmap_get(dsig2->signatures, "ns"))); + test_eq(2, + smartlist_len((smartlist_t*)strmap_get(dsig2->signatures, + "microdesc"))); + + /* Try adding to con2 twice; verify that nothing changes. */ + test_eq(0, networkstatus_add_detached_signatures(con2, dsig1, &msg)); + + /* Add to con. */ + test_eq(2, networkstatus_add_detached_signatures(con, dsig2, &msg)); + /* Check signatures */ + voter = smartlist_get(con->voters, 1); + sig = smartlist_get(voter->sigs, 0); + test_assert(sig); + test_assert(!networkstatus_check_document_signature(con, sig, cert2)); + voter = smartlist_get(con->voters, 2); + sig = smartlist_get(voter->sigs, 0); + test_assert(sig); + test_assert(!networkstatus_check_document_signature(con, sig, cert1)); + } + + done: + smartlist_free(votes); + tor_free(v1_text); + tor_free(v2_text); + tor_free(v3_text); + tor_free(consensus_text); + tor_free(consensus_text_md); + + if (vote) + networkstatus_vote_free(vote); + if (v1) + networkstatus_vote_free(v1); + if (v2) + networkstatus_vote_free(v2); + if (v3) + networkstatus_vote_free(v3); + if (con) + networkstatus_vote_free(con); + if (con_md) + networkstatus_vote_free(con_md); + if (sign_skey_1) + crypto_free_pk_env(sign_skey_1); + if (sign_skey_2) + crypto_free_pk_env(sign_skey_2); + if (sign_skey_3) + crypto_free_pk_env(sign_skey_3); + if (sign_skey_leg1) + crypto_free_pk_env(sign_skey_leg1); + if (cert1) + authority_cert_free(cert1); + if (cert2) + authority_cert_free(cert2); + if (cert3) + authority_cert_free(cert3); + + tor_free(consensus_text2); + tor_free(consensus_text3); + tor_free(consensus_text_md2); + tor_free(consensus_text_md3); + tor_free(detached_text1); + tor_free(detached_text2); + if (con2) + networkstatus_vote_free(con2); + if (con3) + networkstatus_vote_free(con3); + if (con_md2) + networkstatus_vote_free(con_md2); + if (con_md3) + networkstatus_vote_free(con_md3); + if (dsig1) + ns_detached_signatures_free(dsig1); + if (dsig2) + ns_detached_signatures_free(dsig2); +} + +#define DIR_LEGACY(name) \ + { #name, legacy_test_helper, 0, &legacy_setup, test_dir_ ## name } + +#define DIR(name) \ + { #name, test_dir_##name, 0, NULL, NULL } + +struct testcase_t dir_tests[] = { + DIR_LEGACY(nicknames), + DIR_LEGACY(formats), + DIR_LEGACY(versions), + DIR_LEGACY(fp_pairs), + DIR(split_fps), + DIR_LEGACY(measured_bw), + DIR_LEGACY(param_voting), + DIR_LEGACY(v3_networkstatus), + END_OF_TESTCASES +}; + diff --git a/src/test/test_util.c b/src/test/test_util.c new file mode 100644 index 0000000000..b1fafc84b2 --- /dev/null +++ b/src/test/test_util.c @@ -0,0 +1,1240 @@ +/* Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2011, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "orconfig.h" +#define CONTROL_PRIVATE +#define MEMPOOL_PRIVATE +#include "or.h" +#include "config.h" +#include "control.h" +#include "test.h" +#include "mempool.h" +#include "memarea.h" + +static void +test_util_time(void) +{ + struct timeval start, end; + struct tm a_time; + char timestr[RFC1123_TIME_LEN+1]; + time_t t_res; + int i; + + start.tv_sec = 5; + start.tv_usec = 5000; + + end.tv_sec = 5; + end.tv_usec = 5000; + + test_eq(0L, tv_udiff(&start, &end)); + + end.tv_usec = 7000; + + test_eq(2000L, tv_udiff(&start, &end)); + + end.tv_sec = 6; + + test_eq(1002000L, tv_udiff(&start, &end)); + + end.tv_usec = 0; + + test_eq(995000L, tv_udiff(&start, &end)); + + end.tv_sec = 4; + + test_eq(-1005000L, tv_udiff(&start, &end)); + + end.tv_usec = 999990; + start.tv_sec = 1; + start.tv_usec = 500; + + /* The test values here are confirmed to be correct on a platform + * with a working timegm. */ + a_time.tm_year = 2003-1900; + a_time.tm_mon = 7; + a_time.tm_mday = 30; + a_time.tm_hour = 6; + a_time.tm_min = 14; + a_time.tm_sec = 55; + test_eq((time_t) 1062224095UL, tor_timegm(&a_time)); + a_time.tm_year = 2004-1900; /* Try a leap year, after feb. */ + test_eq((time_t) 1093846495UL, tor_timegm(&a_time)); + a_time.tm_mon = 1; /* Try a leap year, in feb. */ + a_time.tm_mday = 10; + test_eq((time_t) 1076393695UL, tor_timegm(&a_time)); + + format_rfc1123_time(timestr, 0); + test_streq("Thu, 01 Jan 1970 00:00:00 GMT", timestr); + format_rfc1123_time(timestr, (time_t)1091580502UL); + test_streq("Wed, 04 Aug 2004 00:48:22 GMT", timestr); + + t_res = 0; + i = parse_rfc1123_time(timestr, &t_res); + test_eq(i,0); + test_eq(t_res, (time_t)1091580502UL); + test_eq(-1, parse_rfc1123_time("Wed, zz Aug 2004 99-99x99 GMT", &t_res)); + + tor_gettimeofday(&start); + /* now make sure time works. */ + tor_gettimeofday(&end); + /* We might've timewarped a little. */ + tt_int_op(tv_udiff(&start, &end), >=, -5000); + + done: + ; +} + +static void +test_util_config_line(void) +{ + char buf[1024]; + char *k=NULL, *v=NULL; + const char *str; + + /* Test parse_config_line_from_str */ + strlcpy(buf, "k v\n" " key value with spaces \n" "keykey val\n" + "k2\n" + "k3 \n" "\n" " \n" "#comment\n" + "k4#a\n" "k5#abc\n" "k6 val #with comment\n" + "kseven \"a quoted 'string\"\n" + "k8 \"a \\x71uoted\\n\\\"str\\\\ing\\t\\001\\01\\1\\\"\"\n" + "k9 a line that\\\n spans two lines.\n\n" + "k10 more than\\\n one contin\\\nuation\n" + "k11 \\\ncontinuation at the start\n" + "k12 line with a\\\n#comment\n embedded\n" + "k13\\\ncontinuation at the very start\n" + "k14 a line that has a comment and # ends with a slash \\\n" + "k15 this should be the next new line\n" + "k16 a line that has a comment and # ends without a slash \n" + "k17 this should be the next new line\n" + , sizeof(buf)); + str = buf; + + str = parse_config_line_from_str(str, &k, &v); + test_streq(k, "k"); + test_streq(v, "v"); + tor_free(k); tor_free(v); + test_assert(!strcmpstart(str, "key value with")); + + str = parse_config_line_from_str(str, &k, &v); + test_streq(k, "key"); + test_streq(v, "value with spaces"); + tor_free(k); tor_free(v); + test_assert(!strcmpstart(str, "keykey")); + + str = parse_config_line_from_str(str, &k, &v); + test_streq(k, "keykey"); + test_streq(v, "val"); + tor_free(k); tor_free(v); + test_assert(!strcmpstart(str, "k2\n")); + + str = parse_config_line_from_str(str, &k, &v); + test_streq(k, "k2"); + test_streq(v, ""); + tor_free(k); tor_free(v); + test_assert(!strcmpstart(str, "k3 \n")); + + str = parse_config_line_from_str(str, &k, &v); + test_streq(k, "k3"); + test_streq(v, ""); + tor_free(k); tor_free(v); + test_assert(!strcmpstart(str, "#comment")); + + str = parse_config_line_from_str(str, &k, &v); + test_streq(k, "k4"); + test_streq(v, ""); + tor_free(k); tor_free(v); + test_assert(!strcmpstart(str, "k5#abc")); + + str = parse_config_line_from_str(str, &k, &v); + test_streq(k, "k5"); + test_streq(v, ""); + tor_free(k); tor_free(v); + test_assert(!strcmpstart(str, "k6")); + + str = parse_config_line_from_str(str, &k, &v); + test_streq(k, "k6"); + test_streq(v, "val"); + tor_free(k); tor_free(v); + test_assert(!strcmpstart(str, "kseven")); + + str = parse_config_line_from_str(str, &k, &v); + test_streq(k, "kseven"); + test_streq(v, "a quoted \'string"); + tor_free(k); tor_free(v); + test_assert(!strcmpstart(str, "k8 ")); + + str = parse_config_line_from_str(str, &k, &v); + test_streq(k, "k8"); + test_streq(v, "a quoted\n\"str\\ing\t\x01\x01\x01\""); + tor_free(k); tor_free(v); + + str = parse_config_line_from_str(str, &k, &v); + test_streq(k, "k9"); + test_streq(v, "a line that spans two lines."); + tor_free(k); tor_free(v); + + str = parse_config_line_from_str(str, &k, &v); + test_streq(k, "k10"); + test_streq(v, "more than one continuation"); + tor_free(k); tor_free(v); + + str = parse_config_line_from_str(str, &k, &v); + test_streq(k, "k11"); + test_streq(v, "continuation at the start"); + tor_free(k); tor_free(v); + + str = parse_config_line_from_str(str, &k, &v); + test_streq(k, "k12"); + test_streq(v, "line with a embedded"); + tor_free(k); tor_free(v); + + str = parse_config_line_from_str(str, &k, &v); + test_streq(k, "k13"); + test_streq(v, "continuation at the very start"); + tor_free(k); tor_free(v); + + str = parse_config_line_from_str(str, &k, &v); + test_streq(k, "k14"); + test_streq(v, "a line that has a comment and" ); + tor_free(k); tor_free(v); + + str = parse_config_line_from_str(str, &k, &v); + test_streq(k, "k15"); + test_streq(v, "this should be the next new line"); + tor_free(k); tor_free(v); + + str = parse_config_line_from_str(str, &k, &v); + test_streq(k, "k16"); + test_streq(v, "a line that has a comment and" ); + tor_free(k); tor_free(v); + + str = parse_config_line_from_str(str, &k, &v); + test_streq(k, "k17"); + test_streq(v, "this should be the next new line"); + tor_free(k); tor_free(v); + + test_streq(str, ""); + + done: + tor_free(k); + tor_free(v); +} + +/** Test basic string functionality. */ +static void +test_util_strmisc(void) +{ + char buf[1024]; + int i; + char *cp; + + /* Tests for corner cases of strl operations */ + test_eq(5, strlcpy(buf, "Hello", 0)); + strlcpy(buf, "Hello", sizeof(buf)); + test_eq(10, strlcat(buf, "Hello", 5)); + + /* Test tor_strstrip() */ + strlcpy(buf, "Testing 1 2 3", sizeof(buf)); + tor_strstrip(buf, ",!"); + test_streq(buf, "Testing 1 2 3"); + strlcpy(buf, "!Testing 1 2 3?", sizeof(buf)); + tor_strstrip(buf, "!? "); + test_streq(buf, "Testing123"); + + /* Test tor_parse_long. */ + test_eq(10L, tor_parse_long("10",10,0,100,NULL,NULL)); + test_eq(0L, tor_parse_long("10",10,50,100,NULL,NULL)); + test_eq(-50L, tor_parse_long("-50",10,-100,100,NULL,NULL)); + + /* Test tor_parse_ulong */ + test_eq(10UL, tor_parse_ulong("10",10,0,100,NULL,NULL)); + test_eq(0UL, tor_parse_ulong("10",10,50,100,NULL,NULL)); + + /* Test tor_parse_uint64. */ + test_assert(U64_LITERAL(10) == tor_parse_uint64("10 x",10,0,100, &i, &cp)); + test_assert(i == 1); + test_streq(cp, " x"); + test_assert(U64_LITERAL(12345678901) == + tor_parse_uint64("12345678901",10,0,UINT64_MAX, &i, &cp)); + test_assert(i == 1); + test_streq(cp, ""); + test_assert(U64_LITERAL(0) == + tor_parse_uint64("12345678901",10,500,INT32_MAX, &i, &cp)); + test_assert(i == 0); + + { + /* Test tor_parse_double. */ + double d = tor_parse_double("10", 0, UINT64_MAX,&i,NULL); + test_assert(i == 1); + test_assert(DBL_TO_U64(d) == 10); + d = tor_parse_double("0", 0, UINT64_MAX,&i,NULL); + test_assert(i == 1); + test_assert(DBL_TO_U64(d) == 0); + d = tor_parse_double(" ", 0, UINT64_MAX,&i,NULL); + test_assert(i == 0); + d = tor_parse_double(".0a", 0, UINT64_MAX,&i,NULL); + test_assert(i == 0); + d = tor_parse_double(".0a", 0, UINT64_MAX,&i,&cp); + test_assert(i == 1); + d = tor_parse_double("-.0", 0, UINT64_MAX,&i,NULL); + test_assert(i == 1); + } + + /* Test failing snprintf cases */ + test_eq(-1, tor_snprintf(buf, 0, "Foo")); + test_eq(-1, tor_snprintf(buf, 2, "Foo")); + + /* Test printf with uint64 */ + tor_snprintf(buf, sizeof(buf), "x!"U64_FORMAT"!x", + U64_PRINTF_ARG(U64_LITERAL(12345678901))); + test_streq(buf, "x!12345678901!x"); + + /* Test for strcmpstart and strcmpend. */ + test_assert(strcmpstart("abcdef", "abcdef")==0); + test_assert(strcmpstart("abcdef", "abc")==0); + test_assert(strcmpstart("abcdef", "abd")<0); + test_assert(strcmpstart("abcdef", "abb")>0); + test_assert(strcmpstart("ab", "abb")<0); + + test_assert(strcmpend("abcdef", "abcdef")==0); + test_assert(strcmpend("abcdef", "def")==0); + test_assert(strcmpend("abcdef", "deg")<0); + test_assert(strcmpend("abcdef", "dee")>0); + test_assert(strcmpend("ab", "abb")<0); + + test_assert(strcasecmpend("AbcDEF", "abcdef")==0); + test_assert(strcasecmpend("abcdef", "dEF")==0); + test_assert(strcasecmpend("abcDEf", "deg")<0); + test_assert(strcasecmpend("abcdef", "DEE")>0); + test_assert(strcasecmpend("ab", "abB")<0); + + /* Test mem_is_zero */ + memset(buf,0,128); + buf[128] = 'x'; + test_assert(tor_digest_is_zero(buf)); + test_assert(tor_mem_is_zero(buf, 10)); + test_assert(tor_mem_is_zero(buf, 20)); + test_assert(tor_mem_is_zero(buf, 128)); + test_assert(!tor_mem_is_zero(buf, 129)); + buf[60] = (char)255; + test_assert(!tor_mem_is_zero(buf, 128)); + buf[0] = (char)1; + test_assert(!tor_mem_is_zero(buf, 10)); + + /* Test 'escaped' */ + test_streq("\"\"", escaped("")); + test_streq("\"abcd\"", escaped("abcd")); + test_streq("\"\\\\\\n\\r\\t\\\"\\'\"", escaped("\\\n\r\t\"\'")); + test_streq("\"z\\001abc\\277d\"", escaped("z\001abc\277d")); + test_assert(NULL == escaped(NULL)); + + /* Test strndup and memdup */ + { + const char *s = "abcdefghijklmnopqrstuvwxyz"; + cp = tor_strndup(s, 30); + test_streq(cp, s); /* same string, */ + test_neq(cp, s); /* but different pointers. */ + tor_free(cp); + + cp = tor_strndup(s, 5); + test_streq(cp, "abcde"); + tor_free(cp); + + s = "a\0b\0c\0d\0e\0"; + cp = tor_memdup(s,10); + test_memeq(cp, s, 10); /* same ram, */ + test_neq(cp, s); /* but different pointers. */ + tor_free(cp); + } + + /* Test str-foo functions */ + cp = tor_strdup("abcdef"); + test_assert(tor_strisnonupper(cp)); + cp[3] = 'D'; + test_assert(!tor_strisnonupper(cp)); + tor_strupper(cp); + test_streq(cp, "ABCDEF"); + test_assert(tor_strisprint(cp)); + cp[3] = 3; + test_assert(!tor_strisprint(cp)); + tor_free(cp); + + /* Test eat_whitespace. */ + { + const char *s = " \n a"; + test_eq_ptr(eat_whitespace(s), s+4); + s = "abcd"; + test_eq_ptr(eat_whitespace(s), s); + s = "#xyz\nab"; + test_eq_ptr(eat_whitespace(s), s+5); + } + + /* Test memmem and memstr */ + { + const char *haystack = "abcde"; + tor_assert(!tor_memmem(haystack, 5, "ef", 2)); + test_eq_ptr(tor_memmem(haystack, 5, "cd", 2), haystack + 2); + test_eq_ptr(tor_memmem(haystack, 5, "cde", 3), haystack + 2); + haystack = "ababcad"; + test_eq_ptr(tor_memmem(haystack, 7, "abc", 3), haystack + 2); + test_eq_ptr(tor_memstr(haystack, 7, "abc"), haystack + 2); + test_assert(!tor_memstr(haystack, 7, "fe")); + test_assert(!tor_memstr(haystack, 7, "longerthantheoriginal")); + } + + /* Test wrap_string */ + { + smartlist_t *sl = smartlist_create(); + wrap_string(sl, "This is a test of string wrapping functionality: woot.", + 10, "", ""); + cp = smartlist_join_strings(sl, "", 0, NULL); + test_streq(cp, + "This is a\ntest of\nstring\nwrapping\nfunctional\nity: woot.\n"); + tor_free(cp); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + wrap_string(sl, "This is a test of string wrapping functionality: woot.", + 16, "### ", "# "); + cp = smartlist_join_strings(sl, "", 0, NULL); + test_streq(cp, + "### This is a\n# test of string\n# wrapping\n# functionality:\n" + "# woot.\n"); + + tor_free(cp); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_free(sl); + } + done: + ; +} + +static void +test_util_pow2(void) +{ + /* Test tor_log2(). */ + test_eq(tor_log2(64), 6); + test_eq(tor_log2(65), 6); + test_eq(tor_log2(63), 5); + test_eq(tor_log2(1), 0); + test_eq(tor_log2(2), 1); + test_eq(tor_log2(3), 1); + test_eq(tor_log2(4), 2); + test_eq(tor_log2(5), 2); + test_eq(tor_log2(U64_LITERAL(40000000000000000)), 55); + test_eq(tor_log2(UINT64_MAX), 63); + + /* Test round_to_power_of_2 */ + test_eq(round_to_power_of_2(120), 128); + test_eq(round_to_power_of_2(128), 128); + test_eq(round_to_power_of_2(130), 128); + test_eq(round_to_power_of_2(U64_LITERAL(40000000000000000)), + U64_LITERAL(1)<<55); + test_eq(round_to_power_of_2(0), 2); + + done: + ; +} + +/** mutex for thread test to stop the threads hitting data at the same time. */ +static tor_mutex_t *_thread_test_mutex = NULL; +/** mutexes for the thread test to make sure that the threads have to + * interleave somewhat. */ +static tor_mutex_t *_thread_test_start1 = NULL, + *_thread_test_start2 = NULL; +/** Shared strmap for the thread test. */ +static strmap_t *_thread_test_strmap = NULL; +/** The name of thread1 for the thread test */ +static char *_thread1_name = NULL; +/** The name of thread2 for the thread test */ +static char *_thread2_name = NULL; + +static void _thread_test_func(void* _s) ATTR_NORETURN; + +/** How many iterations have the threads in the unit test run? */ +static int t1_count = 0, t2_count = 0; + +/** Helper function for threading unit tests: This function runs in a + * subthread. It grabs its own mutex (start1 or start2) to make sure that it + * should start, then it repeatedly alters _test_thread_strmap protected by + * _thread_test_mutex. */ +static void +_thread_test_func(void* _s) +{ + char *s = _s; + int i, *count; + tor_mutex_t *m; + char buf[64]; + char **cp; + if (!strcmp(s, "thread 1")) { + m = _thread_test_start1; + cp = &_thread1_name; + count = &t1_count; + } else { + m = _thread_test_start2; + cp = &_thread2_name; + count = &t2_count; + } + + tor_snprintf(buf, sizeof(buf), "%lu", tor_get_thread_id()); + *cp = tor_strdup(buf); + + tor_mutex_acquire(m); + + for (i=0; i<10000; ++i) { + tor_mutex_acquire(_thread_test_mutex); + strmap_set(_thread_test_strmap, "last to run", *cp); + ++*count; + tor_mutex_release(_thread_test_mutex); + } + tor_mutex_acquire(_thread_test_mutex); + strmap_set(_thread_test_strmap, s, *cp); + tor_mutex_release(_thread_test_mutex); + + tor_mutex_release(m); + + spawn_exit(); +} + +/** Run unit tests for threading logic. */ +static void +test_util_threads(void) +{ + char *s1 = NULL, *s2 = NULL; + int done = 0, timedout = 0; + time_t started; +#ifndef MS_WINDOWS + struct timeval tv; + tv.tv_sec=0; + tv.tv_usec=10; +#endif +#ifndef TOR_IS_MULTITHREADED + /* Skip this test if we aren't threading. We should be threading most + * everywhere by now. */ + if (1) + return; +#endif + _thread_test_mutex = tor_mutex_new(); + _thread_test_start1 = tor_mutex_new(); + _thread_test_start2 = tor_mutex_new(); + _thread_test_strmap = strmap_new(); + s1 = tor_strdup("thread 1"); + s2 = tor_strdup("thread 2"); + tor_mutex_acquire(_thread_test_start1); + tor_mutex_acquire(_thread_test_start2); + spawn_func(_thread_test_func, s1); + spawn_func(_thread_test_func, s2); + tor_mutex_release(_thread_test_start2); + tor_mutex_release(_thread_test_start1); + started = time(NULL); + while (!done) { + tor_mutex_acquire(_thread_test_mutex); + strmap_assert_ok(_thread_test_strmap); + if (strmap_get(_thread_test_strmap, "thread 1") && + strmap_get(_thread_test_strmap, "thread 2")) { + done = 1; + } else if (time(NULL) > started + 25) { + timedout = done = 1; + } + tor_mutex_release(_thread_test_mutex); +#ifndef MS_WINDOWS + /* Prevent the main thread from starving the worker threads. */ + select(0, NULL, NULL, NULL, &tv); +#endif + } + tor_mutex_acquire(_thread_test_start1); + tor_mutex_release(_thread_test_start1); + tor_mutex_acquire(_thread_test_start2); + tor_mutex_release(_thread_test_start2); + + tor_mutex_free(_thread_test_mutex); + + if (timedout) { + printf("\nTimed out: %d %d", t1_count, t2_count); + test_assert(strmap_get(_thread_test_strmap, "thread 1")); + test_assert(strmap_get(_thread_test_strmap, "thread 2")); + test_assert(!timedout); + } + + /* different thread IDs. */ + test_assert(strcmp(strmap_get(_thread_test_strmap, "thread 1"), + strmap_get(_thread_test_strmap, "thread 2"))); + test_assert(!strcmp(strmap_get(_thread_test_strmap, "thread 1"), + strmap_get(_thread_test_strmap, "last to run")) || + !strcmp(strmap_get(_thread_test_strmap, "thread 2"), + strmap_get(_thread_test_strmap, "last to run"))); + + done: + tor_free(s1); + tor_free(s2); + tor_free(_thread1_name); + tor_free(_thread2_name); + if (_thread_test_strmap) + strmap_free(_thread_test_strmap, NULL); + if (_thread_test_start1) + tor_mutex_free(_thread_test_start1); + if (_thread_test_start2) + tor_mutex_free(_thread_test_start2); +} + +/** Run unit tests for compression functions */ +static void +test_util_gzip(void) +{ + char *buf1=NULL, *buf2=NULL, *buf3=NULL, *cp1, *cp2; + const char *ccp2; + size_t len1, len2; + tor_zlib_state_t *state = NULL; + + buf1 = tor_strdup("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZAAAAAAAAAAAAAAAAAAAZ"); + test_assert(detect_compression_method(buf1, strlen(buf1)) == UNKNOWN_METHOD); + if (is_gzip_supported()) { + test_assert(!tor_gzip_compress(&buf2, &len1, buf1, strlen(buf1)+1, + GZIP_METHOD)); + test_assert(buf2); + test_assert(!memcmp(buf2, "\037\213", 2)); /* Gzip magic. */ + test_assert(detect_compression_method(buf2, len1) == GZIP_METHOD); + + test_assert(!tor_gzip_uncompress(&buf3, &len2, buf2, len1, + GZIP_METHOD, 1, LOG_INFO)); + test_assert(buf3); + test_streq(buf1,buf3); + + tor_free(buf2); + tor_free(buf3); + } + + test_assert(!tor_gzip_compress(&buf2, &len1, buf1, strlen(buf1)+1, + ZLIB_METHOD)); + test_assert(buf2); + test_assert(!memcmp(buf2, "\x78\xDA", 2)); /* deflate magic. */ + test_assert(detect_compression_method(buf2, len1) == ZLIB_METHOD); + + test_assert(!tor_gzip_uncompress(&buf3, &len2, buf2, len1, + ZLIB_METHOD, 1, LOG_INFO)); + test_assert(buf3); + test_streq(buf1,buf3); + + /* Check whether we can uncompress concatenated, compressed strings. */ + tor_free(buf3); + buf2 = tor_realloc(buf2, len1*2); + memcpy(buf2+len1, buf2, len1); + test_assert(!tor_gzip_uncompress(&buf3, &len2, buf2, len1*2, + ZLIB_METHOD, 1, LOG_INFO)); + test_eq(len2, (strlen(buf1)+1)*2); + test_memeq(buf3, + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZAAAAAAAAAAAAAAAAAAAZ\0" + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZAAAAAAAAAAAAAAAAAAAZ\0", + (strlen(buf1)+1)*2); + + tor_free(buf1); + tor_free(buf2); + tor_free(buf3); + + /* Check whether we can uncompress partial strings. */ + buf1 = + tor_strdup("String with low redundancy that won't be compressed much."); + test_assert(!tor_gzip_compress(&buf2, &len1, buf1, strlen(buf1)+1, + ZLIB_METHOD)); + tor_assert(len1>16); + /* when we allow an incomplete string, we should succeed.*/ + tor_assert(!tor_gzip_uncompress(&buf3, &len2, buf2, len1-16, + ZLIB_METHOD, 0, LOG_INFO)); + buf3[len2]='\0'; + tor_assert(len2 > 5); + tor_assert(!strcmpstart(buf1, buf3)); + + /* when we demand a complete string, this must fail. */ + tor_free(buf3); + tor_assert(tor_gzip_uncompress(&buf3, &len2, buf2, len1-16, + ZLIB_METHOD, 1, LOG_INFO)); + tor_assert(!buf3); + + /* Now, try streaming compression. */ + tor_free(buf1); + tor_free(buf2); + tor_free(buf3); + state = tor_zlib_new(1, ZLIB_METHOD); + tor_assert(state); + cp1 = buf1 = tor_malloc(1024); + len1 = 1024; + ccp2 = "ABCDEFGHIJABCDEFGHIJ"; + len2 = 21; + test_assert(tor_zlib_process(state, &cp1, &len1, &ccp2, &len2, 0) + == TOR_ZLIB_OK); + test_eq(len2, 0); /* Make sure we compressed it all. */ + test_assert(cp1 > buf1); + + len2 = 0; + cp2 = cp1; + test_assert(tor_zlib_process(state, &cp1, &len1, &ccp2, &len2, 1) + == TOR_ZLIB_DONE); + test_eq(len2, 0); + test_assert(cp1 > cp2); /* Make sure we really added something. */ + + tor_assert(!tor_gzip_uncompress(&buf3, &len2, buf1, 1024-len1, + ZLIB_METHOD, 1, LOG_WARN)); + test_streq(buf3, "ABCDEFGHIJABCDEFGHIJ"); /*Make sure it compressed right.*/ + + done: + if (state) + tor_zlib_free(state); + tor_free(buf2); + tor_free(buf3); + tor_free(buf1); +} + +/** Run unit tests for mmap() wrapper functionality. */ +static void +test_util_mmap(void) +{ + char *fname1 = tor_strdup(get_fname("mapped_1")); + char *fname2 = tor_strdup(get_fname("mapped_2")); + char *fname3 = tor_strdup(get_fname("mapped_3")); + const size_t buflen = 17000; + char *buf = tor_malloc(17000); + tor_mmap_t *mapping = NULL; + + crypto_rand(buf, buflen); + + mapping = tor_mmap_file(fname1); + test_assert(! mapping); + + write_str_to_file(fname1, "Short file.", 1); + write_bytes_to_file(fname2, buf, buflen, 1); + write_bytes_to_file(fname3, buf, 16384, 1); + + mapping = tor_mmap_file(fname1); + test_assert(mapping); + test_eq(mapping->size, strlen("Short file.")); + test_streq(mapping->data, "Short file."); +#ifdef MS_WINDOWS + tor_munmap_file(mapping); + mapping = NULL; + test_assert(unlink(fname1) == 0); +#else + /* make sure we can unlink. */ + test_assert(unlink(fname1) == 0); + test_streq(mapping->data, "Short file."); + tor_munmap_file(mapping); + mapping = NULL; +#endif + + /* Now a zero-length file. */ + write_str_to_file(fname1, "", 1); + mapping = tor_mmap_file(fname1); + test_eq(mapping, NULL); + test_eq(ERANGE, errno); + unlink(fname1); + + /* Make sure that we fail to map a no-longer-existent file. */ + mapping = tor_mmap_file(fname1); + test_assert(mapping == NULL); + + /* Now try a big file that stretches across a few pages and isn't aligned */ + mapping = tor_mmap_file(fname2); + test_assert(mapping); + test_eq(mapping->size, buflen); + test_memeq(mapping->data, buf, buflen); + tor_munmap_file(mapping); + mapping = NULL; + + /* Now try a big aligned file. */ + mapping = tor_mmap_file(fname3); + test_assert(mapping); + test_eq(mapping->size, 16384); + test_memeq(mapping->data, buf, 16384); + tor_munmap_file(mapping); + mapping = NULL; + + done: + unlink(fname1); + unlink(fname2); + unlink(fname3); + + tor_free(fname1); + tor_free(fname2); + tor_free(fname3); + tor_free(buf); + + if (mapping) + tor_munmap_file(mapping); +} + +/** Run unit tests for escaping/unescaping data for use by controllers. */ +static void +test_util_control_formats(void) +{ + char *out = NULL; + const char *inp = + "..This is a test\r\nof the emergency \nbroadcast\r\n..system.\r\nZ.\r\n"; + size_t sz; + + sz = read_escaped_data(inp, strlen(inp), &out); + test_streq(out, + ".This is a test\nof the emergency \nbroadcast\n.system.\nZ.\n"); + test_eq(sz, strlen(out)); + + done: + tor_free(out); +} + +static void +test_util_sscanf(void) +{ + unsigned u1, u2, u3; + char s1[10], s2[10], s3[10], ch; + int r; + + r = tor_sscanf("hello world", "hello world"); /* String match: success */ + test_eq(r, 0); + r = tor_sscanf("hello world 3", "hello worlb %u", &u1); /* String fail */ + test_eq(r, 0); + r = tor_sscanf("12345", "%u", &u1); /* Simple number */ + test_eq(r, 1); + test_eq(u1, 12345u); + r = tor_sscanf("", "%u", &u1); /* absent number */ + test_eq(r, 0); + r = tor_sscanf("A", "%u", &u1); /* bogus number */ + test_eq(r, 0); + r = tor_sscanf("4294967295", "%u", &u1); /* UINT32_MAX should work. */ + test_eq(r, 1); + test_eq(u1, 4294967295u); + r = tor_sscanf("4294967296", "%u", &u1); /* Always say -1 at 32 bits. */ + test_eq(r, 0); + r = tor_sscanf("123456", "%2u%u", &u1, &u2); /* Width */ + test_eq(r, 2); + test_eq(u1, 12u); + test_eq(u2, 3456u); + r = tor_sscanf("!12:3:456", "!%2u:%2u:%3u", &u1, &u2, &u3); /* separators */ + test_eq(r, 3); + test_eq(u1, 12u); + test_eq(u2, 3u); + test_eq(u3, 456u); + r = tor_sscanf("12:3:045", "%2u:%2u:%3u", &u1, &u2, &u3); /* 0s */ + test_eq(r, 3); + test_eq(u1, 12u); + test_eq(u2, 3u); + test_eq(u3, 45u); + /* %u does not match space.*/ + r = tor_sscanf("12:3: 45", "%2u:%2u:%3u", &u1, &u2, &u3); + test_eq(r, 2); + /* %u does not match negative numbers. */ + r = tor_sscanf("12:3:-4", "%2u:%2u:%3u", &u1, &u2, &u3); + test_eq(r, 2); + /* Arbitrary amounts of 0-padding are okay */ + r = tor_sscanf("12:03:000000000000000099", "%2u:%2u:%u", &u1, &u2, &u3); + test_eq(r, 3); + test_eq(u1, 12u); + test_eq(u2, 3u); + test_eq(u3, 99u); + + r = tor_sscanf("99% fresh", "%3u%% fresh", &u1); /* percents are scannable.*/ + test_eq(r, 1); + test_eq(u1, 99); + + r = tor_sscanf("hello", "%s", s1); /* %s needs a number. */ + test_eq(r, -1); + + r = tor_sscanf("hello", "%3s%7s", s1, s2); /* %s matches characters. */ + test_eq(r, 2); + test_streq(s1, "hel"); + test_streq(s2, "lo"); + r = tor_sscanf("WD40", "%2s%u", s3, &u1); /* %s%u */ + test_eq(r, 2); + test_streq(s3, "WD"); + test_eq(u1, 40); + r = tor_sscanf("76trombones", "%6u%9s", &u1, s1); /* %u%s */ + test_eq(r, 2); + test_eq(u1, 76); + test_streq(s1, "trombones"); + r = tor_sscanf("hello world", "%9s %9s", s1, s2); /* %s doesn't eat space. */ + test_eq(r, 2); + test_streq(s1, "hello"); + test_streq(s2, "world"); + r = tor_sscanf("hi", "%9s%9s%3s", s1, s2, s3); /* %s can be empty. */ + test_eq(r, 3); + test_streq(s1, "hi"); + test_streq(s2, ""); + test_streq(s3, ""); + + r = tor_sscanf("1.2.3", "%u.%u.%u%c", &u1, &u2, &u3, &ch); + test_eq(r, 3); + r = tor_sscanf("1.2.3 foobar", "%u.%u.%u%c", &u1, &u2, &u3, &ch); + test_eq(r, 4); + + done: + ; +} + +/** Run unittests for memory pool allocator */ +static void +test_util_mempool(void) +{ + mp_pool_t *pool = NULL; + smartlist_t *allocated = NULL; + int i; + + pool = mp_pool_new(1, 100); + test_assert(pool); + test_assert(pool->new_chunk_capacity >= 100); + test_assert(pool->item_alloc_size >= sizeof(void*)+1); + mp_pool_destroy(pool); + pool = NULL; + + pool = mp_pool_new(241, 2500); + test_assert(pool); + test_assert(pool->new_chunk_capacity >= 10); + test_assert(pool->item_alloc_size >= sizeof(void*)+241); + test_eq(pool->item_alloc_size & 0x03, 0); + test_assert(pool->new_chunk_capacity < 60); + + allocated = smartlist_create(); + for (i = 0; i < 20000; ++i) { + if (smartlist_len(allocated) < 20 || crypto_rand_int(2)) { + void *m = mp_pool_get(pool); + memset(m, 0x09, 241); + smartlist_add(allocated, m); + //printf("%d: %p\n", i, m); + //mp_pool_assert_ok(pool); + } else { + int idx = crypto_rand_int(smartlist_len(allocated)); + void *m = smartlist_get(allocated, idx); + //printf("%d: free %p\n", i, m); + smartlist_del(allocated, idx); + mp_pool_release(m); + //mp_pool_assert_ok(pool); + } + if (crypto_rand_int(777)==0) + mp_pool_clean(pool, 1, 1); + + if (i % 777) + mp_pool_assert_ok(pool); + } + + done: + if (allocated) { + SMARTLIST_FOREACH(allocated, void *, m, mp_pool_release(m)); + mp_pool_assert_ok(pool); + mp_pool_clean(pool, 0, 0); + mp_pool_assert_ok(pool); + smartlist_free(allocated); + } + + if (pool) + mp_pool_destroy(pool); +} + +/** Run unittests for memory area allocator */ +static void +test_util_memarea(void) +{ + memarea_t *area = memarea_new(); + char *p1, *p2, *p3, *p1_orig; + void *malloced_ptr = NULL; + int i; + + test_assert(area); + + p1_orig = p1 = memarea_alloc(area,64); + p2 = memarea_alloc_zero(area,52); + p3 = memarea_alloc(area,11); + + test_assert(memarea_owns_ptr(area, p1)); + test_assert(memarea_owns_ptr(area, p2)); + test_assert(memarea_owns_ptr(area, p3)); + /* Make sure we left enough space. */ + test_assert(p1+64 <= p2); + test_assert(p2+52 <= p3); + /* Make sure we aligned. */ + test_eq(((uintptr_t)p1) % sizeof(void*), 0); + test_eq(((uintptr_t)p2) % sizeof(void*), 0); + test_eq(((uintptr_t)p3) % sizeof(void*), 0); + test_assert(!memarea_owns_ptr(area, p3+8192)); + test_assert(!memarea_owns_ptr(area, p3+30)); + test_assert(tor_mem_is_zero(p2, 52)); + /* Make sure we don't overalign. */ + p1 = memarea_alloc(area, 1); + p2 = memarea_alloc(area, 1); + test_eq(p1+sizeof(void*), p2); + { + malloced_ptr = tor_malloc(64); + test_assert(!memarea_owns_ptr(area, malloced_ptr)); + tor_free(malloced_ptr); + } + + /* memarea_memdup */ + { + malloced_ptr = tor_malloc(64); + crypto_rand((char*)malloced_ptr, 64); + p1 = memarea_memdup(area, malloced_ptr, 64); + test_assert(p1 != malloced_ptr); + test_memeq(p1, malloced_ptr, 64); + tor_free(malloced_ptr); + } + + /* memarea_strdup. */ + p1 = memarea_strdup(area,""); + p2 = memarea_strdup(area, "abcd"); + test_assert(p1); + test_assert(p2); + test_streq(p1, ""); + test_streq(p2, "abcd"); + + /* memarea_strndup. */ + { + const char *s = "Ad ogni porta batte la morte e grida: il nome!"; + /* (From Turandot, act 3.) */ + size_t len = strlen(s); + p1 = memarea_strndup(area, s, 1000); + p2 = memarea_strndup(area, s, 10); + test_streq(p1, s); + test_assert(p2 >= p1 + len + 1); + test_memeq(s, p2, 10); + test_eq(p2[10], '\0'); + p3 = memarea_strndup(area, s, len); + test_streq(p3, s); + p3 = memarea_strndup(area, s, len-1); + test_memeq(s, p3, len-1); + test_eq(p3[len-1], '\0'); + } + + memarea_clear(area); + p1 = memarea_alloc(area, 1); + test_eq(p1, p1_orig); + memarea_clear(area); + + /* Check for running over an area's size. */ + for (i = 0; i < 512; ++i) { + p1 = memarea_alloc(area, crypto_rand_int(5)+1); + test_assert(memarea_owns_ptr(area, p1)); + } + memarea_assert_ok(area); + /* Make sure we can allocate a too-big object. */ + p1 = memarea_alloc_zero(area, 9000); + p2 = memarea_alloc_zero(area, 16); + test_assert(memarea_owns_ptr(area, p1)); + test_assert(memarea_owns_ptr(area, p2)); + + done: + memarea_drop_all(area); + tor_free(malloced_ptr); +} + +/** Run unit tests for utility functions to get file names relative to + * the data directory. */ +static void +test_util_datadir(void) +{ + char buf[1024]; + char *f = NULL; + char *temp_dir = NULL; + + temp_dir = get_datadir_fname(NULL); + f = get_datadir_fname("state"); + tor_snprintf(buf, sizeof(buf), "%s"PATH_SEPARATOR"state", temp_dir); + test_streq(f, buf); + tor_free(f); + f = get_datadir_fname2("cache", "thingy"); + tor_snprintf(buf, sizeof(buf), + "%s"PATH_SEPARATOR"cache"PATH_SEPARATOR"thingy", temp_dir); + test_streq(f, buf); + tor_free(f); + f = get_datadir_fname2_suffix("cache", "thingy", ".foo"); + tor_snprintf(buf, sizeof(buf), + "%s"PATH_SEPARATOR"cache"PATH_SEPARATOR"thingy.foo", temp_dir); + test_streq(f, buf); + tor_free(f); + f = get_datadir_fname_suffix("cache", ".foo"); + tor_snprintf(buf, sizeof(buf), "%s"PATH_SEPARATOR"cache.foo", + temp_dir); + test_streq(f, buf); + + done: + tor_free(f); + tor_free(temp_dir); +} + +static void +test_util_strtok(void) +{ + char buf[128]; + char buf2[128]; + char *cp1, *cp2; + strlcpy(buf, "Graved on the dark in gestures of descent", sizeof(buf)); + strlcpy(buf2, "they.seemed;their!own;most.perfect;monument", sizeof(buf2)); + /* -- "Year's End", Richard Wilbur */ + + test_streq("Graved", tor_strtok_r_impl(buf, " ", &cp1)); + test_streq("they", tor_strtok_r_impl(buf2, ".!..;!", &cp2)); +#define S1() tor_strtok_r_impl(NULL, " ", &cp1) +#define S2() tor_strtok_r_impl(NULL, ".!..;!", &cp2) + test_streq("on", S1()); + test_streq("the", S1()); + test_streq("dark", S1()); + test_streq("seemed", S2()); + test_streq("their", S2()); + test_streq("own", S2()); + test_streq("in", S1()); + test_streq("gestures", S1()); + test_streq("of", S1()); + test_streq("most", S2()); + test_streq("perfect", S2()); + test_streq("descent", S1()); + test_streq("monument", S2()); + test_assert(NULL == S1()); + test_assert(NULL == S2()); + done: + ; +} + +static void +test_util_find_str_at_start_of_line(void *ptr) +{ + const char *long_string = + "hello world. hello world. hello hello. howdy.\n" + "hello hello world\n"; + + (void)ptr; + + /* not-found case. */ + tt_assert(! find_str_at_start_of_line(long_string, "fred")); + + /* not-found case where haystack doesn't end with \n */ + tt_assert(! find_str_at_start_of_line("foobar\nbaz", "fred")); + + /* start-of-string case */ + tt_assert(long_string == + find_str_at_start_of_line(long_string, "hello world.")); + + /* start-of-line case */ + tt_assert(strchr(long_string,'\n')+1 == + find_str_at_start_of_line(long_string, "hello hello")); + done: + ; +} + +static void +test_util_asprintf(void *ptr) +{ +#define LOREMIPSUM \ + "Lorem ipsum dolor sit amet, consectetur adipisicing elit" + char *cp=NULL, *cp2=NULL; + int r; + (void)ptr; + + /* empty string. */ + r = tor_asprintf(&cp, "%s", ""); + tt_assert(cp); + tt_int_op(r, ==, strlen(cp)); + tt_str_op(cp, ==, ""); + + /* Short string with some printing in it. */ + r = tor_asprintf(&cp2, "First=%d, Second=%d", 101, 202); + tt_assert(cp2); + tt_int_op(r, ==, strlen(cp2)); + tt_str_op(cp2, ==, "First=101, Second=202"); + tt_assert(cp != cp2); + tor_free(cp); + tor_free(cp2); + + /* Glass-box test: a string exactly 128 characters long. */ + r = tor_asprintf(&cp, "Lorem1: %sLorem2: %s", LOREMIPSUM, LOREMIPSUM); + tt_assert(cp); + tt_int_op(r, ==, 128); + tt_assert(cp[128] == '\0'); + tt_str_op(cp, ==, + "Lorem1: "LOREMIPSUM"Lorem2: "LOREMIPSUM); + tor_free(cp); + + /* String longer than 128 characters */ + r = tor_asprintf(&cp, "1: %s 2: %s 3: %s", + LOREMIPSUM, LOREMIPSUM, LOREMIPSUM); + tt_assert(cp); + tt_int_op(r, ==, strlen(cp)); + tt_str_op(cp, ==, "1: "LOREMIPSUM" 2: "LOREMIPSUM" 3: "LOREMIPSUM); + + done: + tor_free(cp); + tor_free(cp2); +} + +static void +test_util_listdir(void *ptr) +{ + smartlist_t *dir_contents = NULL; + char *fname1=NULL, *fname2=NULL, *dirname=NULL; + (void)ptr; + + fname1 = tor_strdup(get_fname("hopscotch")); + fname2 = tor_strdup(get_fname("mumblety-peg")); + dirname = tor_strdup(get_fname(NULL)); + + tt_int_op(write_str_to_file(fname1, "X\n", 0), ==, 0); + tt_int_op(write_str_to_file(fname2, "Y\n", 0), ==, 0); + + dir_contents = tor_listdir(dirname); + tt_assert(dir_contents); + /* make sure that each filename is listed. */ + tt_assert(smartlist_string_isin_case(dir_contents, "hopscotch")); + tt_assert(smartlist_string_isin_case(dir_contents, "mumblety-peg")); + + tt_assert(!smartlist_string_isin(dir_contents, ".")); + tt_assert(!smartlist_string_isin(dir_contents, "..")); + + done: + tor_free(fname1); + tor_free(fname2); + tor_free(dirname); + if (dir_contents) { + SMARTLIST_FOREACH(dir_contents, char *, cp, tor_free(cp)); + smartlist_free(dir_contents); + } +} + +#ifdef MS_WINDOWS +static void +test_util_load_win_lib(void *ptr) +{ + HANDLE h = load_windows_system_library("advapi32.dll"); + (void) ptr; + + tt_assert(h); + done: + if (h) + CloseHandle(h); +} +#endif + +#define UTIL_LEGACY(name) \ + { #name, legacy_test_helper, 0, &legacy_setup, test_util_ ## name } + +#define UTIL_TEST(name, flags) \ + { #name, test_util_ ## name, flags, NULL, NULL } + +struct testcase_t util_tests[] = { + UTIL_LEGACY(time), + UTIL_LEGACY(config_line), + UTIL_LEGACY(strmisc), + UTIL_LEGACY(pow2), + UTIL_LEGACY(gzip), + UTIL_LEGACY(datadir), + UTIL_LEGACY(mempool), + UTIL_LEGACY(memarea), + UTIL_LEGACY(control_formats), + UTIL_LEGACY(mmap), + UTIL_LEGACY(threads), + UTIL_LEGACY(sscanf), + UTIL_LEGACY(strtok), + UTIL_TEST(find_str_at_start_of_line, 0), + UTIL_TEST(asprintf, 0), + UTIL_TEST(listdir, 0), +#ifdef MS_WINDOWS + UTIL_TEST(load_win_lib, 0), +#endif + END_OF_TESTCASES +}; + diff --git a/src/test/tinytest.c b/src/test/tinytest.c new file mode 100644 index 0000000000..11ffc2fe56 --- /dev/null +++ b/src/test/tinytest.c @@ -0,0 +1,380 @@ +/* tinytest.c -- Copyright 2009-2010 Nick Mathewson + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <assert.h> + +#ifdef WIN32 +#include <windows.h> +#else +#include <sys/types.h> +#include <sys/wait.h> +#include <unistd.h> +#endif + +#ifndef __GNUC__ +#define __attribute__(x) +#endif + +#ifdef TINYTEST_LOCAL +#include "tinytest_local.h" +#endif +#include "tinytest.h" +#include "tinytest_macros.h" + +#define LONGEST_TEST_NAME 16384 + +static int in_tinytest_main = 0; /**< true if we're in tinytest_main().*/ +static int n_ok = 0; /**< Number of tests that have passed */ +static int n_bad = 0; /**< Number of tests that have failed. */ +static int n_skipped = 0; /**< Number of tests that have been skipped. */ + +static int opt_forked = 0; /**< True iff we're called from inside a win32 fork*/ +static int opt_nofork = 0; /**< Suppress calls to fork() for debugging. */ +static int opt_verbosity = 1; /**< -==quiet,0==terse,1==normal,2==verbose */ +const char *verbosity_flag = ""; + +enum outcome { SKIP=2, OK=1, FAIL=0 }; +static enum outcome cur_test_outcome = 0; +const char *cur_test_prefix = NULL; /**< prefix of the current test group */ +/** Name of the current test, if we haven't logged is yet. Used for --quiet */ +const char *cur_test_name = NULL; + +#ifdef WIN32 +/** Pointer to argv[0] for win32. */ +static const char *commandname = NULL; +#endif + +static void usage(struct testgroup_t *groups, int list_groups) + __attribute__((noreturn)); + +static enum outcome +_testcase_run_bare(const struct testcase_t *testcase) +{ + void *env = NULL; + int outcome; + if (testcase->setup) { + env = testcase->setup->setup_fn(testcase); + if (!env) + return FAIL; + else if (env == (void*)TT_SKIP) + return SKIP; + } + + cur_test_outcome = OK; + testcase->fn(env); + outcome = cur_test_outcome; + + if (testcase->setup) { + if (testcase->setup->cleanup_fn(testcase, env) == 0) + outcome = FAIL; + } + + return outcome; +} + +#define MAGIC_EXITCODE 42 + +static enum outcome +_testcase_run_forked(const struct testgroup_t *group, + const struct testcase_t *testcase) +{ +#ifdef WIN32 + /* Fork? On Win32? How primitive! We'll do what the smart kids do: + we'll invoke our own exe (whose name we recall from the command + line) with a command line that tells it to run just the test we + want, and this time without forking. + + (No, threads aren't an option. The whole point of forking is to + share no state between tests.) + */ + int ok; + char buffer[LONGEST_TEST_NAME+256]; + STARTUPINFOA si; + PROCESS_INFORMATION info; + DWORD exitcode; + + if (!in_tinytest_main) { + printf("\nERROR. On Windows, _testcase_run_forked must be" + " called from within tinytest_main.\n"); + abort(); + } + if (opt_verbosity>0) + printf("[forking] "); + + snprintf(buffer, sizeof(buffer), "%s --RUNNING-FORKED %s %s%s", + commandname, verbosity_flag, group->prefix, testcase->name); + + memset(&si, 0, sizeof(si)); + memset(&info, 0, sizeof(info)); + si.cb = sizeof(si); + + ok = CreateProcessA(commandname, buffer, NULL, NULL, 0, + 0, NULL, NULL, &si, &info); + if (!ok) { + printf("CreateProcess failed!\n"); + return 0; + } + WaitForSingleObject(info.hProcess, INFINITE); + GetExitCodeProcess(info.hProcess, &exitcode); + CloseHandle(info.hProcess); + CloseHandle(info.hThread); + if (exitcode == 0) + return OK; + else if (exitcode == MAGIC_EXITCODE) + return SKIP; + else + return FAIL; +#else + int outcome_pipe[2]; + pid_t pid; + (void)group; + + if (pipe(outcome_pipe)) + perror("opening pipe"); + + if (opt_verbosity>0) + printf("[forking] "); + pid = fork(); + if (!pid) { + /* child. */ + int test_r, write_r; + char b[1]; + close(outcome_pipe[0]); + test_r = _testcase_run_bare(testcase); + assert(0<=(int)test_r && (int)test_r<=2); + b[0] = "NYS"[test_r]; + write_r = (int)write(outcome_pipe[1], b, 1); + if (write_r != 1) { + perror("write outcome to pipe"); + exit(1); + } + exit(0); + } else { + /* parent */ + int status, r; + char b[1]; + /* Close this now, so that if the other side closes it, + * our read fails. */ + close(outcome_pipe[1]); + r = (int)read(outcome_pipe[0], b, 1); + if (r == 0) { + printf("[Lost connection!] "); + return 0; + } else if (r != 1) { + perror("read outcome from pipe"); + } + waitpid(pid, &status, 0); + close(outcome_pipe[0]); + return b[0]=='Y' ? OK : (b[0]=='S' ? SKIP : FAIL); + } +#endif +} + +int +testcase_run_one(const struct testgroup_t *group, + const struct testcase_t *testcase) +{ + enum outcome outcome; + + if (testcase->flags & TT_SKIP) { + if (opt_verbosity>0) + printf("%s%s: SKIPPED\n", + group->prefix, testcase->name); + ++n_skipped; + return SKIP; + } + + if (opt_verbosity>0 && !opt_forked) { + printf("%s%s: ", group->prefix, testcase->name); + } else { + if (opt_verbosity==0) printf("."); + cur_test_prefix = group->prefix; + cur_test_name = testcase->name; + } + + if ((testcase->flags & TT_FORK) && !(opt_forked||opt_nofork)) { + outcome = _testcase_run_forked(group, testcase); + } else { + outcome = _testcase_run_bare(testcase); + } + + if (outcome == OK) { + ++n_ok; + if (opt_verbosity>0 && !opt_forked) + puts(opt_verbosity==1?"OK":""); + } else if (outcome == SKIP) { + ++n_skipped; + if (opt_verbosity>0 && !opt_forked) + puts("SKIPPED"); + } else { + ++n_bad; + if (!opt_forked) + printf("\n [%s FAILED]\n", testcase->name); + } + + if (opt_forked) { + exit(outcome==OK ? 0 : (outcome==SKIP?MAGIC_EXITCODE : 1)); + } else { + return (int)outcome; + } +} + +int +_tinytest_set_flag(struct testgroup_t *groups, const char *arg, unsigned long flag) +{ + int i, j; + size_t length = LONGEST_TEST_NAME; + char fullname[LONGEST_TEST_NAME]; + int found=0; + if (strstr(arg, "..")) + length = strstr(arg,"..")-arg; + for (i=0; groups[i].prefix; ++i) { + for (j=0; groups[i].cases[j].name; ++j) { + snprintf(fullname, sizeof(fullname), "%s%s", + groups[i].prefix, groups[i].cases[j].name); + if (!flag) /* Hack! */ + printf(" %s\n", fullname); + if (!strncmp(fullname, arg, length)) { + groups[i].cases[j].flags |= flag; + ++found; + } + } + } + return found; +} + +static void +usage(struct testgroup_t *groups, int list_groups) +{ + puts("Options are: [--verbose|--quiet|--terse] [--no-fork]"); + puts(" Specify tests by name, or using a prefix ending with '..'"); + puts(" To skip a test, list give its name prefixed with a colon."); + puts(" Use --list-tests for a list of tests."); + if (list_groups) { + puts("Known tests are:"); + _tinytest_set_flag(groups, "..", 0); + } + exit(0); +} + +int +tinytest_main(int c, const char **v, struct testgroup_t *groups) +{ + int i, j, n=0; + +#ifdef WIN32 + commandname = v[0]; +#endif + for (i=1; i<c; ++i) { + if (v[i][0] == '-') { + if (!strcmp(v[i], "--RUNNING-FORKED")) { + opt_forked = 1; + } else if (!strcmp(v[i], "--no-fork")) { + opt_nofork = 1; + } else if (!strcmp(v[i], "--quiet")) { + opt_verbosity = -1; + verbosity_flag = "--quiet"; + } else if (!strcmp(v[i], "--verbose")) { + opt_verbosity = 2; + verbosity_flag = "--verbose"; + } else if (!strcmp(v[i], "--terse")) { + opt_verbosity = 0; + verbosity_flag = "--terse"; + } else if (!strcmp(v[i], "--help")) { + usage(groups, 0); + } else if (!strcmp(v[i], "--list-tests")) { + usage(groups, 1); + } else { + printf("Unknown option %s. Try --help\n",v[i]); + return -1; + } + } else { + const char *test = v[i]; + int flag = _TT_ENABLED; + if (test[0] == ':') { + ++test; + flag = TT_SKIP; + } else { + ++n; + } + if (!_tinytest_set_flag(groups, test, flag)) { + printf("No such test as %s!\n", v[i]); + return -1; + } + } + } + if (!n) + _tinytest_set_flag(groups, "..", _TT_ENABLED); + + setvbuf(stdout, NULL, _IONBF, 0); + + ++in_tinytest_main; + for (i=0; groups[i].prefix; ++i) + for (j=0; groups[i].cases[j].name; ++j) + if (groups[i].cases[j].flags & _TT_ENABLED) + testcase_run_one(&groups[i], + &groups[i].cases[j]); + + --in_tinytest_main; + + if (opt_verbosity==0) + puts(""); + + if (n_bad) + printf("%d/%d TESTS FAILED. (%d skipped)\n", n_bad, + n_bad+n_ok,n_skipped); + else if (opt_verbosity >= 1) + printf("%d tests ok. (%d skipped)\n", n_ok, n_skipped); + + return (n_bad == 0) ? 0 : 1; +} + +int +_tinytest_get_verbosity(void) +{ + return opt_verbosity; +} + +void +_tinytest_set_test_failed(void) +{ + if (opt_verbosity <= 0 && cur_test_name) { + if (opt_verbosity==0) puts(""); + printf("%s%s: ", cur_test_prefix, cur_test_name); + cur_test_name = NULL; + } + cur_test_outcome = 0; +} + +void +_tinytest_set_test_skipped(void) +{ + if (cur_test_outcome==OK) + cur_test_outcome = SKIP; +} + diff --git a/src/test/tinytest.h b/src/test/tinytest.h new file mode 100644 index 0000000000..cbe28b7f51 --- /dev/null +++ b/src/test/tinytest.h @@ -0,0 +1,87 @@ +/* tinytest.h -- Copyright 2009-2010 Nick Mathewson + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _TINYTEST_H +#define _TINYTEST_H + +/** Flag for a test that needs to run in a subprocess. */ +#define TT_FORK (1<<0) +/** Runtime flag for a test we've decided to skip. */ +#define TT_SKIP (1<<1) +/** Internal runtime flag for a test we've decided to run. */ +#define _TT_ENABLED (1<<2) +/** If you add your own flags, make them start at this point. */ +#define TT_FIRST_USER_FLAG (1<<3) + +typedef void (*testcase_fn)(void *); + +struct testcase_t; + +/** Functions to initialize/teardown a structure for a testcase. */ +struct testcase_setup_t { + /** Return a new structure for use by a given testcase. */ + void *(*setup_fn)(const struct testcase_t *); + /** Clean/free a structure from setup_fn. Return 1 if ok, 0 on err. */ + int (*cleanup_fn)(const struct testcase_t *, void *); +}; + +/** A single test-case that you can run. */ +struct testcase_t { + const char *name; /**< An identifier for this case. */ + testcase_fn fn; /**< The function to run to implement this case. */ + unsigned long flags; /**< Bitfield of TT_* flags. */ + const struct testcase_setup_t *setup; /**< Optional setup/cleanup fns*/ + void *setup_data; /**< Extra data usable by setup function */ +}; +#define END_OF_TESTCASES { NULL, NULL, 0, NULL, NULL } + +/** A group of tests that are selectable together. */ +struct testgroup_t { + const char *prefix; /**< Prefix to prepend to testnames. */ + struct testcase_t *cases; /** Array, ending with END_OF_TESTCASES */ +}; +#define END_OF_GROUPS { NULL, NULL} + +/** Implementation: called from a test to indicate failure, before logging. */ +void _tinytest_set_test_failed(void); +/** Implementation: called from a test to indicate that we're skipping. */ +void _tinytest_set_test_skipped(void); +/** Implementation: return 0 for quiet, 1 for normal, 2 for loud. */ +int _tinytest_get_verbosity(void); +/** Implementation: Set a flag on tests matching a name; returns number + * of tests that matched. */ +int _tinytest_set_flag(struct testgroup_t *, const char *, unsigned long); + +/** Set all tests in 'groups' matching the name 'named' to be skipped. */ +#define tinytest_skip(groups, named) \ + _tinytest_set_flag(groups, named, TT_SKIP) + +/** Run a single testcase in a single group. */ +int testcase_run_one(const struct testgroup_t *,const struct testcase_t *); +/** Run a set of testcases from an END_OF_GROUPS-terminated array of groups, + as selected from the command line. */ +int tinytest_main(int argc, const char **argv, struct testgroup_t *groups); + +#endif diff --git a/src/test/tinytest_demo.c b/src/test/tinytest_demo.c new file mode 100644 index 0000000000..bd33cc37fa --- /dev/null +++ b/src/test/tinytest_demo.c @@ -0,0 +1,215 @@ +/* tinytest_demo.c -- Copyright 2009 Nick Mathewson + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + + +/* Welcome to the example file for tinytest! I'll show you how to set up + * some simple and not-so-simple testcases. */ + +/* Make sure you include these headers. */ +#include "tinytest.h" +#include "tinytest_macros.h" + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <errno.h> + +/* ============================================================ */ + +/* First, let's see if strcmp is working. (All your test cases should be + * functions declared to take a single void * as) an argument. */ +void +test_strcmp(void *data) +{ + (void)data; /* This testcase takes no data. */ + + /* Let's make sure the empty string is equal to itself */ + if (strcmp("","")) { + /* This macro tells tinytest to stop the current test + * and go straight to the "end" label. */ + tt_abort_msg("The empty string was not equal to itself"); + } + + /* Pretty often, calling tt_abort_msg to indicate failure is more + heavy-weight than you want. Instead, just say: */ + tt_assert(strcmp("testcase", "testcase") == 0); + + /* Occasionally, you don't want to stop the current testcase just + because a single assertion has failed. In that case, use + tt_want: */ + tt_want(strcmp("tinytest", "testcase") > 0); + + /* You can use the tt_*_op family of macros to compare values and to + fail unless they have the relationship you want. They produce + more useful output than tt_assert, since they display the actual + values of the failing things. + + Fail unless strcmp("abc, "abc") == 0 */ + tt_int_op(strcmp("abc", "abc"), ==, 0); + + /* Fail unless strcmp("abc, "abcd") is less than 0 */ + tt_int_op(strcmp("abc", "abcd"), < , 0); + + /* Incidentally, there's a test_str_op that uses strcmp internally. */ + tt_str_op("abc", <, "abcd"); + + + /* Every test-case function needs to finish with an "end:" + label and (optionally) code to clean up local variables. */ + end: + ; +} + +/* ============================================================ */ + +/* Now let's mess with setup and teardown functions! These are handy if + you have a bunch of tests that all need a similar environment, and you + want to reconstruct that environment freshly for each one. */ + +/* First you declare a type to hold the environment info, and functions to + set it up and tear it down. */ +struct data_buffer { + /* We're just going to have couple of character buffer. Using + setup/teardown functions is probably overkill for this case. + + You could also do file descriptors, complicated handles, temporary + files, etc. */ + char buffer1[512]; + char buffer2[512]; +}; +/* The setup function needs to take a const struct testcase_t and return + void* */ +void * +setup_data_buffer(const struct testcase_t *testcase) +{ + struct data_buffer *db = malloc(sizeof(struct data_buffer)); + + /* If you had a complicated set of setup rules, you might behave + differently here depending on testcase->flags or + testcase->setup_data or even or testcase->name. */ + + /* Returning a NULL here would mean that we couldn't set up for this + test, so we don't need to test db for null. */ + return db; +} +/* The clean function deallocates storage carefully and returns true on + success. */ +int +clean_data_buffer(const struct testcase_t *testcase, void *ptr) +{ + struct data_buffer *db = ptr; + + if (db) { + free(db); + return 1; + } + return 0; +} +/* Finally, declare a testcase_setup_t with these functions. */ +struct testcase_setup_t data_buffer_setup = { + setup_data_buffer, clean_data_buffer +}; + + +/* Now let's write our test. */ +void +test_memcpy(void *ptr) +{ + /* This time, we use the argument. */ + struct data_buffer *db = ptr; + + /* We'll also introduce a local variable that might need cleaning up. */ + char *mem = NULL; + + /* Let's make sure that memcpy does what we'd like. */ + strcpy(db->buffer1, "String 0"); + memcpy(db->buffer2, db->buffer1, sizeof(db->buffer1)); + tt_str_op(db->buffer1, ==, db->buffer2); + + /* Now we've allocated memory that's referenced by a local variable. + The end block of the function will clean it up. */ + mem = strdup("Hello world."); + tt_assert(mem); + + /* Another rather trivial test. */ + tt_str_op(db->buffer1, !=, mem); + + end: + /* This time our end block has something to do. */ + if (mem) + free(mem); +} + +/* ============================================================ */ + +/* Now we need to make sure that our tests get invoked. First, you take + a bunch of related tests and put them into an array of struct testcase_t. +*/ + +struct testcase_t demo_tests[] = { + /* Here's a really simple test: it has a name you can refer to it + with, and a function to invoke it. */ + { "strcmp", test_strcmp, }, + + /* The second test has a flag, "TT_FORK", to make it run in a + subprocess, and a pointer to the testcase_setup_t that configures + its environment. */ + { "memcpy", test_memcpy, TT_FORK, &data_buffer_setup }, + + /* The array has to end with END_OF_TESTCASES. */ + END_OF_TESTCASES +}; + +/* Next, we make an array of testgroups. This is mandatory. Unlike more + heavy-duty testing frameworks, groups can't nest. */ +struct testgroup_t groups[] = { + + /* Every group has a 'prefix', and an array of tests. That's it. */ + { "demo/", demo_tests }, + + END_OF_GROUPS +}; + + +int +main(int c, const char **v) +{ + /* Finally, just call tinytest_main(). It lets you specify verbose + or quiet output with --verbose and --quiet. You can list + specific tests: + + tinytest-demo demo/memcpy + + or use a ..-wildcard to select multiple tests with a common + prefix: + + tinytest-demo demo/.. + + If you list no tests, you get them all by default, so that + "tinytest-demo" and "tinytest-demo .." mean the same thing. + + */ + return tinytest_main(c, v, groups); +} diff --git a/src/test/tinytest_macros.h b/src/test/tinytest_macros.h new file mode 100644 index 0000000000..a7fa64a824 --- /dev/null +++ b/src/test/tinytest_macros.h @@ -0,0 +1,167 @@ +/* tinytest_macros.h -- Copyright 2009-2010 Nick Mathewson + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _TINYTEST_MACROS_H +#define _TINYTEST_MACROS_H + +/* Helpers for defining statement-like macros */ +#define TT_STMT_BEGIN do { +#define TT_STMT_END } while (0) + +/* Redefine this if your test functions want to abort with something besides + * "goto end;" */ +#ifndef TT_EXIT_TEST_FUNCTION +#define TT_EXIT_TEST_FUNCTION TT_STMT_BEGIN goto end; TT_STMT_END +#endif + +/* Redefine this if you want to note success/failure in some different way. */ +#ifndef TT_DECLARE +#define TT_DECLARE(prefix, args) \ + TT_STMT_BEGIN \ + printf("\n %s %s:%d: ",prefix,__FILE__,__LINE__); \ + printf args ; \ + TT_STMT_END +#endif + +/* Announce a failure. Args are parenthesized printf args. */ +#define TT_GRIPE(args) TT_DECLARE("FAIL", args) + +/* Announce a non-failure if we're verbose. */ +#define TT_BLATHER(args) \ + TT_STMT_BEGIN \ + if (_tinytest_get_verbosity()>1) TT_DECLARE(" OK", args); \ + TT_STMT_END + +#define TT_DIE(args) \ + TT_STMT_BEGIN \ + _tinytest_set_test_failed(); \ + TT_GRIPE(args); \ + TT_EXIT_TEST_FUNCTION; \ + TT_STMT_END + +#define TT_FAIL(args) \ + TT_STMT_BEGIN \ + _tinytest_set_test_failed(); \ + TT_GRIPE(args); \ + TT_STMT_END + +/* Fail and abort the current test for the reason in msg */ +#define tt_abort_printf(msg) TT_DIE(msg) +#define tt_abort_perror(op) TT_DIE(("%s: %s [%d]",(op),strerror(errno), errno)) +#define tt_abort_msg(msg) TT_DIE(("%s", msg)) +#define tt_abort() TT_DIE(("%s", "(Failed.)")) + +/* Fail but do not abort the current test for the reason in msg. */ +#define tt_fail_printf(msg) TT_FAIL(msg) +#define tt_fail_perror(op) TT_FAIL(("%s: %s [%d]",(op),strerror(errno), errno)) +#define tt_fail_msg(msg) TT_FAIL(("%s", msg)) +#define tt_fail() TT_FAIL(("%s", "(Failed.)")) + +/* End the current test, and indicate we are skipping it. */ +#define tt_skip() \ + TT_STMT_BEGIN \ + _tinytest_set_test_skipped(); \ + TT_EXIT_TEST_FUNCTION; \ + TT_STMT_END + +#define _tt_want(b, msg, fail) \ + TT_STMT_BEGIN \ + if (!(b)) { \ + _tinytest_set_test_failed(); \ + TT_GRIPE((msg)); \ + fail; \ + } else { \ + TT_BLATHER((msg)); \ + } \ + TT_STMT_END + +/* Assert b, but do not stop the test if b fails. Log msg on failure. */ +#define tt_want_msg(b, msg) \ + _tt_want(b, msg, ); + +/* Assert b and stop the test if b fails. Log msg on failure. */ +#define tt_assert_msg(b, msg) \ + _tt_want(b, msg, TT_EXIT_TEST_FUNCTION); + +/* Assert b, but do not stop the test if b fails. */ +#define tt_want(b) tt_want_msg( (b), "want("#b")") +/* Assert b, and stop the test if b fails. */ +#define tt_assert(b) tt_assert_msg((b), "assert("#b")") + +#define tt_assert_test_fmt_type(a,b,str_test,type,test,printf_type,printf_fmt, \ + setup_block,cleanup_block) \ + TT_STMT_BEGIN \ + type _val1 = (type)(a); \ + type _val2 = (type)(b); \ + int _tt_status = (test); \ + if (!_tt_status || _tinytest_get_verbosity()>1) { \ + printf_type _print; \ + printf_type _print1; \ + printf_type _print2; \ + type _value = _val1; \ + setup_block; \ + _print1 = _print; \ + _value = _val2; \ + setup_block; \ + _print2 = _print; \ + TT_DECLARE(_tt_status?" OK":"FAIL", \ + ("assert(%s): "printf_fmt" vs "printf_fmt, \ + str_test, _print1, _print2)); \ + _print = _print1; \ + cleanup_block; \ + _print = _print2; \ + cleanup_block; \ + if (!_tt_status) { \ + _tinytest_set_test_failed(); \ + TT_EXIT_TEST_FUNCTION; \ + } \ + } \ + TT_STMT_END + +#define tt_assert_test_type(a,b,str_test,type,test,fmt) \ + tt_assert_test_fmt_type(a,b,str_test,type,test,type,fmt, \ + {_print=_value;},{}) + +/* Helper: assert that a op b, when cast to type. Format the values with + * printf format fmt on failure. */ +#define tt_assert_op_type(a,op,b,type,fmt) \ + tt_assert_test_type(a,b,#a" "#op" "#b,type,(_val1 op _val2),fmt) + +#define tt_int_op(a,op,b) \ + tt_assert_test_type(a,b,#a" "#op" "#b,long,(_val1 op _val2),"%ld") + +#define tt_uint_op(a,op,b) \ + tt_assert_test_type(a,b,#a" "#op" "#b,unsigned long, \ + (_val1 op _val2),"%lu") + +#define tt_ptr_op(a,op,b) \ + tt_assert_test_type(a,b,#a" "#op" "#b,void*, \ + (_val1 op _val2),"%p") + +#define tt_str_op(a,op,b) \ + tt_assert_test_type(a,b,#a" "#op" "#b,const char *, \ + (strcmp(_val1,_val2) op 0),"<%s>") + +#endif diff --git a/src/tools/Makefile.am b/src/tools/Makefile.am index 41786e4378..1bb5076849 100644 --- a/src/tools/Makefile.am +++ b/src/tools/Makefile.am @@ -3,16 +3,16 @@ noinst_PROGRAMS = tor-checkkey tor_resolve_SOURCES = tor-resolve.c tor_resolve_LDFLAGS = @TOR_LDFLAGS_libevent@ -tor_resolve_LDADD = ../common/libor.a @TOR_LIBEVENT_LIBS@ @TOR_LIB_WS32@ +tor_resolve_LDADD = ../common/libor.a -lm @TOR_LIBEVENT_LIBS@ @TOR_LIB_WS32@ tor_gencert_SOURCES = tor-gencert.c tor_gencert_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ \ @TOR_LDFLAGS_libevent@ tor_gencert_LDADD = ../common/libor.a ../common/libor-crypto.a \ - -lz @TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ + -lm @TOR_ZLIB_LIBS@ @TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ tor_checkkey_SOURCES = tor-checkkey.c tor_checkkey_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ \ @TOR_LDFLAGS_libevent@ tor_checkkey_LDADD = ../common/libor.a ../common/libor-crypto.a \ - -lz @TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ + -lm @TOR_ZLIB_LIBS@ @TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ diff --git a/src/tools/tor-checkkey.c b/src/tools/tor-checkkey.c index b29b52d8db..94c8cbd44c 100644 --- a/src/tools/tor-checkkey.c +++ b/src/tools/tor-checkkey.c @@ -6,19 +6,21 @@ #include <stdio.h> #include <stdlib.h> #include "crypto.h" -#include "log.h" -#include "util.h" +#include "torlog.h" +#include "../common/util.h" #include "compat.h" #include <openssl/bn.h> #include <openssl/rsa.h> -int main(int c, char **v) +int +main(int c, char **v) { crypto_pk_env_t *env; char *str; RSA *rsa; int wantdigest=0; int fname_idx; + char *fname=NULL; init_logging(); if (c < 2) { @@ -29,7 +31,7 @@ int main(int c, char **v) return 1; } - if (crypto_global_init(0)) { + if (crypto_global_init(0, NULL, NULL)) { fprintf(stderr, "Couldn't initialize crypto library.\n"); return 1; } @@ -46,7 +48,9 @@ int main(int c, char **v) fname_idx = 1; } - str = read_file_to_str(v[fname_idx], 0, NULL); + fname = expand_filename(v[fname_idx]); + str = read_file_to_str(fname, 0, NULL); + tor_free(fname); if (!str) { fprintf(stderr, "Couldn't read %s\n", v[fname_idx]); return 1; diff --git a/src/tools/tor-gencert.c b/src/tools/tor-gencert.c index e6b09963bd..a04eddafc7 100644 --- a/src/tools/tor-gencert.c +++ b/src/tools/tor-gencert.c @@ -13,6 +13,7 @@ #include <openssl/evp.h> #include <openssl/pem.h> +#include <openssl/rsa.h> #include <openssl/objects.h> #include <openssl/obj_mac.h> #include <openssl/err.h> @@ -27,8 +28,8 @@ #define CRYPTO_PRIVATE #include "compat.h" -#include "util.h" -#include "log.h" +#include "../common/util.h" +#include "../common/torlog.h" #include "crypto.h" #include "address.h" @@ -63,7 +64,6 @@ show_help(void) "[-c certificate_file]\n" " [-m lifetime_in_months] [-a address:port] " "[--passphrase-fd <fd>]\n"); - } /* XXXX copied from crypto.c */ @@ -218,6 +218,20 @@ parse_commandline(int argc, char **argv) return 0; } +static RSA * +generate_key(int bits) +{ + RSA *rsa = NULL; + crypto_pk_env_t *env = crypto_new_pk_env(); + if (crypto_pk_generate_key_with_bits(env,bits)<0) + goto done; + rsa = _crypto_pk_env_get_rsa(env); + rsa = RSAPrivateKey_dup(rsa); + done: + crypto_free_pk_env(env); + return rsa; +} + /** Try to read the identity key from <b>identity_key_file</b>. If no such * file exists and create_identity_key is set, make a new identity key and * store it. Return 0 on success, nonzero on failure. @@ -238,7 +252,7 @@ load_identity_key(void) } log_notice(LD_GENERAL, "Generating %d-bit RSA identity key.", IDENTITY_KEY_BITS); - if (!(key = RSA_generate_key(IDENTITY_KEY_BITS, 65537, NULL, NULL))) { + if (!(key = generate_key(IDENTITY_KEY_BITS))) { log_err(LD_GENERAL, "Couldn't generate identity key."); crypto_log_errors(LOG_ERR, "Generating identity key"); return 1; @@ -323,7 +337,7 @@ generate_signing_key(void) RSA *key; log_notice(LD_GENERAL, "Generating %d-bit RSA signing key.", SIGNING_KEY_BITS); - if (!(key = RSA_generate_key(SIGNING_KEY_BITS, 65537, NULL, NULL))) { + if (!(key = generate_key(SIGNING_KEY_BITS))) { log_err(LD_GENERAL, "Couldn't generate signing key."); crypto_log_errors(LOG_ERR, "Generating signing key"); return 1; @@ -393,7 +407,6 @@ get_fingerprint(EVP_PKEY *pkey, char *out) return r; } - /** Set <b>out</b> to the hex-encoded fingerprint of <b>pkey</b>. */ static int get_digest(EVP_PKEY *pkey, char *out) @@ -487,7 +500,6 @@ generate_certificate(void) return 0; } - /** Entry point to tor-gencert */ int main(int argc, char **argv) @@ -496,7 +508,7 @@ main(int argc, char **argv) init_logging(); /* Don't bother using acceleration. */ - if (crypto_global_init(0)) { + if (crypto_global_init(0, NULL, NULL)) { fprintf(stderr, "Couldn't initialize crypto library.\n"); return 1; } diff --git a/src/tools/tor-resolve.c b/src/tools/tor-resolve.c index 4d9d57a31c..12349d9d12 100644 --- a/src/tools/tor-resolve.c +++ b/src/tools/tor-resolve.c @@ -6,9 +6,9 @@ #include "orconfig.h" #include "compat.h" -#include "util.h" +#include "../common/util.h" #include "address.h" -#include "log.h" +#include "../common/torlog.h" #include <stdio.h> #include <stdlib.h> @@ -148,7 +148,7 @@ parse_socks4a_resolve_response(const char *hostname, static const char * socks5_reason_to_string(char reason) { - switch(reason) { + switch (reason) { case SOCKS5_SUCCEEDED: return "succeeded"; case SOCKS5_GENERAL_ERROR: @@ -251,7 +251,7 @@ do_resolve(const char *hostname, uint32_t sockshost, uint16_t socksport, } if (parse_socks4a_resolve_response(hostname, reply_buf, RESPONSE_LEN_4, - result_addr)<0){ + result_addr)<0) { return -1; } } else { diff --git a/src/win32/orconfig.h b/src/win32/orconfig.h index 65804fe19d..b3cd1db50b 100644 --- a/src/win32/orconfig.h +++ b/src/win32/orconfig.h @@ -88,11 +88,18 @@ #define HAVE_STRING_H /* Define to 1 if you have the `strlcat' function. */ +#if defined (WINCE) +#define HAVE_STRLCAT +#else #undef HAVE_STRLCAT +#endif /* Define to 1 if you have the `strlcpy' function. */ +#if defined (WINCE) +#define HAVE_STRLCPY +#else #undef HAVE_STRLCPY - +#endif /* Define to 1 if you have the `strptime' function. */ #undef HAVE_STRPTIME @@ -226,6 +233,5 @@ #define USING_TWOS_COMPLEMENT /* Version number of package */ -#define VERSION "0.2.1.25" - +#define VERSION "0.2.2.19-alpha" |