diff options
Diffstat (limited to 'src')
123 files changed, 33591 insertions, 14882 deletions
diff --git a/src/Makefile.am b/src/Makefile.am index ae647b2d61..fa2dd560a6 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -1,5 +1,5 @@ # leave in dependency order, since common must be built first -SUBDIRS = common or tools win32 config -DIST_SUBDIRS = common or tools win32 config +SUBDIRS = common or test tools win32 config +DIST_SUBDIRS = common or test tools win32 config diff --git a/src/common/Makefile.am b/src/common/Makefile.am index 105c413343..b1e03cd710 100644 --- a/src/common/Makefile.am +++ b/src/common/Makefile.am @@ -1,5 +1,7 @@ -noinst_LIBRARIES = libor.a libor-crypto.a +noinst_LIBRARIES = libor.a libor-crypto.a libor-event.a + +EXTRA_DIST = common_sha1.i sha256.c #CFLAGS = -Wall -Wpointer-arith -O2 @@ -10,7 +12,21 @@ libor_extra_source= endif libor_a_SOURCES = address.c log.c util.c compat.c container.c mempool.c \ - memarea.c $(libor_extra_source) + memarea.c util_codedigest.c $(libor_extra_source) libor_crypto_a_SOURCES = crypto.c aes.c tortls.c torgzip.c +libor_event_a_SOURCES = compat_libevent.c + +noinst_HEADERS = address.h torlog.h crypto.h util.h compat.h aes.h torint.h tortls.h strlcpy.c strlcat.c torgzip.h container.h ht.h mempool.h memarea.h ciphers.inc compat_libevent.h tortls_states.h + +common_sha1.i: $(libor_SOURCES) $(libor_crypto_a_SOURCES) $(noinst_HEADERS) + if test "@SHA1SUM@" != none; then \ + @SHA1SUM@ $(libor_SOURCES) $(libor_crypto_a_SOURCES) $(noinst_HEADERS) | @SED@ -n 's/^\(.*\)$$/"\1\\n"/p' > common_sha1.i; \ + elif test "@OPENSSL@" != none; then \ + @OPENSSL@ sha1 $(libor_SOURCES) $(libor_crypto_a_SOURCES) $(noinst_HEADERS) | @SED@ -n 's/SHA1(\(.*\))= \(.*\)/"\2 \1\\n"/p' > common_sha1.i; \ + else \ + rm common_sha1.i; \ + touch common_sha1.i; \ + fi -noinst_HEADERS = address.h log.h crypto.h test.h util.h compat.h aes.h torint.h tortls.h strlcpy.c strlcat.c torgzip.h container.h ht.h mempool.h memarea.h ciphers.inc +util_codedigest.o: common_sha1.i +crypto.o: sha256.c diff --git a/src/common/address.c b/src/common/address.c index 58ead3075a..c35f04c18b 100644 --- a/src/common/address.c +++ b/src/common/address.c @@ -12,7 +12,7 @@ #include "compat.h" #include "util.h" #include "address.h" -#include "log.h" +#include "torlog.h" #ifdef MS_WINDOWS #include <process.h> @@ -726,7 +726,7 @@ tor_addr_is_loopback(const tor_addr_t *addr) } /** Set <b>dest</b> to equal the IPv4 address in <b>v4addr</b> (given in - * network order. */ + * network order). */ void tor_addr_from_ipv4n(tor_addr_t *dest, uint32_t v4addr) { @@ -760,6 +760,8 @@ tor_addr_from_in6(tor_addr_t *dest, const struct in6_addr *in6) void tor_addr_copy(tor_addr_t *dest, const tor_addr_t *src) { + if (src == dest) + return; tor_assert(src); tor_assert(dest); memcpy(dest, src, sizeof(tor_addr_t)); @@ -912,13 +914,6 @@ tor_dup_addr(const tor_addr_t *addr) return tor_strdup(buf); } -/** Copy the address in <b>src</b> to <b>dest</b> */ -void -tor_addr_assign(tor_addr_t *dest, const tor_addr_t *src) -{ - memcpy(dest, src, sizeof(tor_addr_t)); -} - /** Return a string representing the address <b>addr</b>. This string is * statically allocated, and must not be freed. Each call to * <b>fmt_addr</b> invalidates the last result of the function. This diff --git a/src/common/address.h b/src/common/address.h index 352e0d7237..6116bb4b1c 100644 --- a/src/common/address.h +++ b/src/common/address.h @@ -107,7 +107,6 @@ tor_addr_eq_ipv4h(const tor_addr_t *a, uint32_t u) int tor_addr_lookup(const char *name, uint16_t family, tor_addr_t *addr_out); char *tor_dup_addr(const tor_addr_t *addr) ATTR_MALLOC; -void tor_addr_assign(tor_addr_t *dest, const tor_addr_t *src); const char *fmt_addr(const tor_addr_t *addr); int get_interface_address6(int severity, sa_family_t family, tor_addr_t *addr); diff --git a/src/common/aes.c b/src/common/aes.c index 5a6bab242d..a17328317e 100644 --- a/src/common/aes.c +++ b/src/common/aes.c @@ -21,7 +21,7 @@ #include "compat.h" #include "aes.h" #include "util.h" -#include "log.h" +#include "torlog.h" /* We have 3 strategies for getting AES: Via OpenSSL's AES_encrypt function, * via OpenSSL's EVP_EncryptUpdate function, or via the built-in AES @@ -263,7 +263,8 @@ aes_set_key(aes_cnt_cipher_t *cipher, const char *key, int key_bits) void aes_free_cipher(aes_cnt_cipher_t *cipher) { - tor_assert(cipher); + if (!cipher) + return; #ifdef USE_OPENSSL_EVP EVP_CIPHER_CTX_cleanup(&cipher->key); #endif diff --git a/src/common/compat.c b/src/common/compat.c index b2dab5c341..e9101a8d7a 100644 --- a/src/common/compat.c +++ b/src/common/compat.c @@ -88,7 +88,7 @@ #include <sys/prctl.h> #endif -#include "log.h" +#include "torlog.h" #include "util.h" #include "container.h" #include "address.h" @@ -169,12 +169,13 @@ tor_munmap_file(tor_mmap_t *handle) tor_mmap_t * tor_mmap_file(const char *filename) { + WCHAR wfilename[MAX_PATH]= {0}; tor_mmap_t *res = tor_malloc_zero(sizeof(tor_mmap_t)); int empty = 0; res->file_handle = INVALID_HANDLE_VALUE; res->mmap_handle = NULL; - - res->file_handle = CreateFile(filename, + mbstowcs(wfilename,filename,MAX_PATH); + res->file_handle = CreateFileW(wfilename, GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, @@ -307,6 +308,100 @@ tor_vsnprintf(char *str, size_t size, const char *format, va_list args) return r; } +/** + * Portable asprintf implementation. Does a printf() into a newly malloc'd + * string. Sets *<b>strp</b> to this string, and returns its length (not + * including the terminating NUL character). + * + * You can treat this function as if its implementation were something like + <pre> + char buf[_INFINITY_]; + tor_snprintf(buf, sizeof(buf), fmt, args); + *strp = tor_strdup(buf); + return strlen(*strp): + </pre> + * Where _INFINITY_ is an imaginary constant so big that any string can fit + * into it. + */ +int +tor_asprintf(char **strp, const char *fmt, ...) +{ + int r; + va_list args; + va_start(args, fmt); + r = tor_vasprintf(strp, fmt, args); + va_end(args); + if (!*strp || r < 0) { + log_err(LD_BUG, "Internal error in asprintf"); + tor_assert(0); + } + return r; +} + +/** + * Portable vasprintf implementation. Does a printf() into a newly malloc'd + * string. Differs from regular vasprintf in the same ways that + * tor_asprintf() differs from regular asprintf. + */ +int +tor_vasprintf(char **strp, const char *fmt, va_list args) +{ + /* use a temporary variable in case *strp is in args. */ + char *strp_tmp=NULL; +#ifdef HAVE_VASPRINTF + /* If the platform gives us one, use it. */ + int r = vasprintf(&strp_tmp, fmt, args); + if (r < 0) + *strp = NULL; + else + *strp = strp_tmp; + return r; +#elif defined(_MSC_VER) + /* On Windows, _vsnprintf won't tell us the length of the string if it + * overflows, so we need to use _vcsprintf to tell how much to allocate */ + int len, r; + char *res; + len = _vscprintf(fmt, args); + if (len < 0) { + *strp = NULL; + return -1; + } + strp_tmp = tor_malloc(len + 1); + r = _vsnprintf(strp_tmp, len+1, fmt, args); + if (r != len) { + tor_free(strp_tmp); + *strp = NULL; + return -1; + } + *strp = strp_tmp; + return len; +#else + /* Everywhere else, we have a decent vsnprintf that tells us how many + * characters we need. We give it a try on a short buffer first, since + * it might be nice to avoid the second vsnprintf call. + */ + char buf[128]; + int len, r; + va_list tmp_args; + va_copy(tmp_args, args); + len = vsnprintf(buf, sizeof(buf), fmt, tmp_args); + va_end(tmp_args); + if (len < (int)sizeof(buf)) { + *strp = tor_strdup(buf); + return len; + } + strp_tmp = tor_malloc(len+1); + r = vsnprintf(strp_tmp, len+1, fmt, args); + if (r != len) { + tor_free(strp_tmp); + *strp = NULL; + return -1; + } + *strp = strp_tmp; + return len; +#endif +} + /** Given <b>hlen</b> bytes at <b>haystack</b> and <b>nlen</b> bytes at * <b>needle</b>, return a pointer to the first occurrence of the needle * within the haystack, or NULL if there is no such occurrence. @@ -398,6 +493,37 @@ const char TOR_TOLOWER_TABLE[256] = { 240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255, }; +/** Implementation of strtok_r for platforms whose coders haven't figured out + * how to write one. Hey guys! You can use this code here for free! */ +char * +tor_strtok_r_impl(char *str, const char *sep, char **lasts) +{ + char *cp, *start; + if (str) + start = cp = *lasts = str; + else if (!*lasts) + return NULL; + else + start = cp = *lasts; + + tor_assert(*sep); + if (sep[1]) { + while (*cp && !strchr(sep, *cp)) + ++cp; + } else { + tor_assert(strlen(sep) == 1); + cp = strchr(cp, *sep); + } + + if (!cp || !*cp) { + *lasts = NULL; + } else { + *cp++ = '\0'; + *lasts = cp; + } + return start; +} + #ifdef MS_WINDOWS /** Take a filename and return a pointer to its final element. This * function is called on __FILE__ to fix a MSVC nit where __FILE__ @@ -449,8 +575,8 @@ get_uint32(const char *cp) return v; } /** - * Read a 32-bit value beginning at <b>cp</b>. Equivalent to - * *(uint32_t*)(cp), but will not cause segfaults on platforms that forbid + * Read a 64-bit value beginning at <b>cp</b>. Equivalent to + * *(uint64_t*)(cp), but will not cause segfaults on platforms that forbid * unaligned memory access. */ uint64_t @@ -978,9 +1104,6 @@ set_max_file_descriptors(rlim_t limit, int *max_out) #if defined(CYGWIN) || defined(__CYGWIN__) const char *platform = "Cygwin"; const unsigned long MAX_CONNECTIONS = 3200; -#elif defined(IPHONE) - const char *platform = "iPhone"; - const unsigned long MAX_CONNECTIONS = 9999; #elif defined(MS_WINDOWS) const char *platform = "Windows"; const unsigned long MAX_CONNECTIONS = 15000; @@ -1575,15 +1698,20 @@ get_uname(void) #endif { #ifdef MS_WINDOWS - OSVERSIONINFOEX info; +#if defined (WINCE) + OSVERSIONINFO info; +#else + OSVERSIONINFOEXW info; +#endif int i; - unsigned int leftover_mask; const char *plat = NULL; const char *extra = NULL; + char acsd[MAX_PATH] = {0}; static struct { unsigned major; unsigned minor; const char *version; } win_version_table[] = { - { 6, 0, "Windows \"Longhorn\"" }, + { 6, 1, "Windows 7" }, + { 6, 0, "Windows Vista" }, { 5, 2, "Windows Server 2003" }, { 5, 1, "Windows XP" }, { 5, 0, "Windows 2000" }, @@ -1594,41 +1722,23 @@ get_uname(void) { 3, 51, "Windows NT 3.51" }, { 0, 0, NULL } }; -#ifdef VER_SUITE_BACKOFFICE - static struct { - unsigned int mask; const char *str; - } win_mask_table[] = { - { VER_SUITE_BACKOFFICE, " {backoffice}" }, - { VER_SUITE_BLADE, " {\"blade\" (2003, web edition)}" }, - { VER_SUITE_DATACENTER, " {datacenter}" }, - { VER_SUITE_ENTERPRISE, " {enterprise}" }, - { VER_SUITE_EMBEDDEDNT, " {embedded}" }, - { VER_SUITE_PERSONAL, " {personal}" }, - { VER_SUITE_SINGLEUSERTS, - " {terminal services, single user}" }, - { VER_SUITE_SMALLBUSINESS, " {small business}" }, - { VER_SUITE_SMALLBUSINESS_RESTRICTED, - " {small business, restricted}" }, - { VER_SUITE_TERMINAL, " {terminal services}" }, - { 0, NULL }, - }; -#endif memset(&info, 0, sizeof(info)); info.dwOSVersionInfoSize = sizeof(info); - if (! GetVersionEx((LPOSVERSIONINFO)&info)) { + if (! GetVersionExW((LPOSVERSIONINFOW)&info)) { strlcpy(uname_result, "Bizarre version of Windows where GetVersionEx" " doesn't work.", sizeof(uname_result)); uname_result_is_set = 1; return uname_result; } + wcstombs(acsd, info.szCSDVersion, MAX_PATH); if (info.dwMajorVersion == 4 && info.dwMinorVersion == 0) { if (info.dwPlatformId == VER_PLATFORM_WIN32_NT) plat = "Windows NT 4.0"; else plat = "Windows 95"; - if (info.szCSDVersion[1] == 'B') + if (acsd[1] == 'B') extra = "OSR2 (B)"; - else if (info.szCSDVersion[1] == 'C') + else if (acsd[1] == 'C') extra = "OSR2 (C)"; } else { for (i=0; win_version_table[i].major>0; ++i) { @@ -1640,29 +1750,30 @@ get_uname(void) } } if (plat && !strcmp(plat, "Windows 98")) { - if (info.szCSDVersion[1] == 'A') + if (acsd[1] == 'A') extra = "SE (A)"; - else if (info.szCSDVersion[1] == 'B') + else if (acsd[1] == 'B') extra = "SE (B)"; } if (plat) { if (!extra) - extra = info.szCSDVersion; + extra = acsd; tor_snprintf(uname_result, sizeof(uname_result), "%s %s", plat, extra); } else { if (info.dwMajorVersion > 6 || - (info.dwMajorVersion==6 && info.dwMinorVersion>0)) + (info.dwMajorVersion==6 && info.dwMinorVersion>1)) tor_snprintf(uname_result, sizeof(uname_result), "Very recent version of Windows [major=%d,minor=%d] %s", (int)info.dwMajorVersion,(int)info.dwMinorVersion, - info.szCSDVersion); + acsd); else tor_snprintf(uname_result, sizeof(uname_result), "Unrecognized version of Windows [major=%d,minor=%d] %s", (int)info.dwMajorVersion,(int)info.dwMinorVersion, - info.szCSDVersion); + acsd); } +#if !defined (WINCE) #ifdef VER_SUITE_BACKOFFICE if (info.wProductType == VER_NT_DOMAIN_CONTROLLER) { strlcat(uname_result, " [domain controller]", sizeof(uname_result)); @@ -1671,18 +1782,7 @@ get_uname(void) } else if (info.wProductType == VER_NT_WORKSTATION) { strlcat(uname_result, " [workstation]", sizeof(uname_result)); } - leftover_mask = info.wSuiteMask; - for (i = 0; win_mask_table[i].mask; ++i) { - if (info.wSuiteMask & win_mask_table[i].mask) { - strlcat(uname_result, win_mask_table[i].str, sizeof(uname_result)); - leftover_mask &= ~win_mask_table[i].mask; - } - } - if (leftover_mask) { - size_t len = strlen(uname_result); - tor_snprintf(uname_result+len, sizeof(uname_result)-len, - " {0x%x}", info.wSuiteMask); - } +#endif #endif #else strlcpy(uname_result, "Unknown platform", sizeof(uname_result)); @@ -1792,7 +1892,6 @@ spawn_exit(void) * call _exit, not exit, from child processes. */ _exit(0); #endif - } /** Set *timeval to the current time of day. On error, log and terminate. @@ -1812,8 +1911,15 @@ tor_gettimeofday(struct timeval *timeval) uint64_t ft_64; FILETIME ft_ft; } ft; +#if defined (WINCE) + /* wince do not have GetSystemTimeAsFileTime */ + SYSTEMTIME stime; + GetSystemTime(&stime); + SystemTimeToFileTime(&stime,&ft.ft_ft); +#else /* number of 100-nsec units since Jan 1, 1601 */ GetSystemTimeAsFileTime(&ft.ft_ft); +#endif if (ft.ft_64 < EPOCH_BIAS) { log_err(LD_GENERAL,"System time is before 1970; failing."); exit(1); @@ -2013,6 +2119,8 @@ tor_mutex_new(void) void tor_mutex_free(tor_mutex_t *m) { + if (!m) + return; tor_mutex_uninit(m); tor_free(m); } @@ -2040,7 +2148,8 @@ tor_cond_new(void) void tor_cond_free(tor_cond_t *cond) { - tor_assert(cond); + if (!cond) + return; if (pthread_cond_destroy(&cond->cond)) { log_warn(LD_GENERAL,"Error freeing condition: %s", strerror(errno)); return; @@ -2097,7 +2206,8 @@ tor_cond_new(void) void tor_cond_free(tor_cond_t *cond) { - tor_assert(cond); + if (!cond) + return; DeleteCriticalSection(&cond->mutex); /* XXXX notify? */ smartlist_free(cond->events); @@ -2173,6 +2283,89 @@ tor_threads_init(void) } #endif +#if defined(HAVE_MLOCKALL) && HAVE_DECL_MLOCKALL && defined(RLIMIT_MEMLOCK) +/** Attempt to raise the current and max rlimit to infinity for our process. + * This only needs to be done once and can probably only be done when we have + * not already dropped privileges. + */ +static int +tor_set_max_memlock(void) +{ + /* Future consideration for Windows is probably SetProcessWorkingSetSize + * This is similar to setting the memory rlimit of RLIMIT_MEMLOCK + * http://msdn.microsoft.com/en-us/library/ms686234(VS.85).aspx + */ + + struct rlimit limit; + + /* RLIM_INFINITY is -1 on some platforms. */ + limit.rlim_cur = RLIM_INFINITY; + limit.rlim_max = RLIM_INFINITY; + + if (setrlimit(RLIMIT_MEMLOCK, &limit) == -1) { + if (errno == EPERM) { + log_warn(LD_GENERAL, "You appear to lack permissions to change memory " + "limits. Are you root?"); + } + log_warn(LD_GENERAL, "Unable to raise RLIMIT_MEMLOCK: %s", + strerror(errno)); + return -1; + } + + return 0; +} +#endif + +/** Attempt to lock all current and all future memory pages. + * This should only be called once and while we're privileged. + * Like mlockall() we return 0 when we're successful and -1 when we're not. + * Unlike mlockall() we return 1 if we've already attempted to lock memory. + */ +int +tor_mlockall(void) +{ + static int memory_lock_attempted = 0; + + if (memory_lock_attempted) { + return 1; + } + + memory_lock_attempted = 1; + + /* + * Future consideration for Windows may be VirtualLock + * VirtualLock appears to implement mlock() but not mlockall() + * + * http://msdn.microsoft.com/en-us/library/aa366895(VS.85).aspx + */ + +#if defined(HAVE_MLOCKALL) && HAVE_DECL_MLOCKALL && defined(RLIMIT_MEMLOCK) + if (tor_set_max_memlock() == 0) { + log_debug(LD_GENERAL, "RLIMIT_MEMLOCK is now set to RLIM_INFINITY."); + } + + if (mlockall(MCL_CURRENT|MCL_FUTURE) == 0) { + log_info(LD_GENERAL, "Insecure OS paging is effectively disabled."); + return 0; + } else { + if (errno == ENOSYS) { + /* Apple - it's 2009! I'm looking at you. Grrr. */ + log_notice(LD_GENERAL, "It appears that mlockall() is not available on " + "your platform."); + } else if (errno == EPERM) { + log_notice(LD_GENERAL, "It appears that you lack the permissions to " + "lock memory. Are you root?"); + } + log_notice(LD_GENERAL, "Unable to lock all current and future memory " + "pages: %s", strerror(errno)); + return -1; + } +#else + log_warn(LD_GENERAL, "Unable to lock memory pages. mlockall() unsupported?"); + return -1; +#endif +} + /** Identity of the "main" thread */ static unsigned long main_thread_id = -1; @@ -2325,19 +2518,21 @@ char * format_win32_error(DWORD err) { LPVOID str = NULL; + char abuf[1024] = {0}; char *result; /* Somebody once decided that this interface was better than strerror(). */ - FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | + FormatMessageW(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, NULL, err, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), - (LPTSTR) &str, + (LPWSTR) &str, 0, NULL); if (str) { - result = tor_strdup((char*)str); + wcstombs(abuf,str,1024); + result = tor_strdup((char*)abuf); LocalFree(str); /* LocalFree != free() */ } else { result = tor_strdup("<unformattable error>"); diff --git a/src/common/compat.h b/src/common/compat.h index fbbbb3fc92..7d59501e2b 100644 --- a/src/common/compat.h +++ b/src/common/compat.h @@ -51,6 +51,22 @@ #include <netinet6/in6.h> #endif +#if defined (WINCE) +#include <fcntl.h> +#include <io.h> +#include <math.h> +#include <projects.h> +#define snprintf _snprintf +/* this is not exported as W .... */ +#define SHGetPathFromIDListW SHGetPathFromIDList +/* wcecompat has vasprintf */ +#define HAVE_VASPRINTF +/* no service here */ +#ifdef NT_SERVICE +#undef NT_SERVICE +#endif +#endif // WINCE + #ifndef NULL_REP_IS_ZERO_BYTES #error "It seems your platform does not represent NULL as zero. We can't cope." #endif @@ -177,8 +193,8 @@ extern INLINE double U64_TO_DBL(uint64_t x) { /* ===== String compatibility */ #ifdef MS_WINDOWS /* Windows names string functions differently from most other platforms. */ -#define strncasecmp strnicmp -#define strcasecmp stricmp +#define strncasecmp _strnicmp +#define strcasecmp _stricmp #endif #ifndef HAVE_STRLCAT size_t strlcat(char *dst, const char *src, size_t siz) ATTR_NONNULL((1,2)); @@ -196,18 +212,26 @@ size_t strlcpy(char *dst, const char *src, size_t siz) ATTR_NONNULL((1,2)); #define U64_SCANF_ARG(a) (a) /** Expands to a literal uint64_t-typed constant for the value <b>n</b>. */ #define U64_LITERAL(n) (n ## ui64) +#define I64_PRINTF_ARG(a) (a) +#define I64_SCANF_ARG(a) (a) +#define I64_LITERAL(n) (n ## i64) #else #define U64_PRINTF_ARG(a) ((long long unsigned int)(a)) #define U64_SCANF_ARG(a) ((long long unsigned int*)(a)) #define U64_LITERAL(n) (n ## llu) +#define I64_PRINTF_ARG(a) ((long long signed int)(a)) +#define I64_SCANF_ARG(a) ((long long signed int*)(a)) +#define I64_LITERAL(n) (n ## ll) #endif #if defined(_MSC_VER) || defined(__MINGW32__) || defined(__MINGW64__) /** The formatting string used to put a uint64_t value in a printf() or * scanf() function. See also U64_PRINTF_ARG and U64_SCANF_ARG. */ #define U64_FORMAT "%I64u" +#define I64_FORMAT "%I64d" #else #define U64_FORMAT "%llu" +#define I64_FORMAT "%lld" #endif /** Represents an mmaped file. Allocated via tor_mmap_file; freed with @@ -235,6 +259,10 @@ int tor_snprintf(char *str, size_t size, const char *format, ...) int tor_vsnprintf(char *str, size_t size, const char *format, va_list args) ATTR_NONNULL((1,3)); +int tor_asprintf(char **strp, const char *fmt, ...) + CHECK_PRINTF(2,3); +int tor_vasprintf(char **strp, const char *fmt, va_list args); + const void *tor_memmem(const void *haystack, size_t hlen, const void *needle, size_t nlen) ATTR_PURE ATTR_NONNULL((1,3)); static const void *tor_memstr(const void *haystack, size_t hlen, @@ -267,6 +295,13 @@ extern const char TOR_TOLOWER_TABLE[]; #define TOR_TOLOWER(c) (TOR_TOLOWER_TABLE[(uint8_t)c]) #define TOR_TOUPPER(c) (TOR_TOUPPER_TABLE[(uint8_t)c]) +char *tor_strtok_r_impl(char *str, const char *sep, char **lasts); +#ifdef HAVE_STRTOK_R +#define tor_strtok_r(str, sep, lasts) strtok_r(str, sep, lasts) +#else +#define tor_strtok_r(str, sep, lasts) tor_strtok_r_impl(str, sep, lasts) +#endif + #ifdef MS_WINDOWS #define _SHORT_FILE_ (tor_fix_source_file(__FILE__)) const char *tor_fix_source_file(const char *fname); @@ -502,6 +537,8 @@ typedef struct tor_mutex_t { #endif } tor_mutex_t; +int tor_mlockall(void); + #ifdef TOR_IS_MULTITHREADED tor_mutex_t *tor_mutex_new(void); void tor_mutex_init(tor_mutex_t *m); @@ -536,6 +573,19 @@ void tor_cond_signal_all(tor_cond_t *cond); #endif #endif +/** Macros for MIN/MAX. Never use these when the arguments could have + * side-effects. + * {With GCC extensions we could probably define a safer MIN/MAX. But + * depending on that safety would be dangerous, since not every platform + * has it.} + **/ +#ifndef MAX +#define MAX(a,b) ( ((a)<(b)) ? (b) : (a) ) +#endif +#ifndef MIN +#define MIN(a,b) ( ((a)>(b)) ? (b) : (a) ) +#endif + /* Platform-specific helpers. */ #ifdef MS_WINDOWS char *format_win32_error(DWORD err); diff --git a/src/common/compat_libevent.c b/src/common/compat_libevent.c new file mode 100644 index 0000000000..250fa2bdb7 --- /dev/null +++ b/src/common/compat_libevent.c @@ -0,0 +1,553 @@ +/* Copyright (c) 2009-2010, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file compat_libevent.c + * \brief Wrappers to handle porting between different versions of libevent. + * + * In an ideal world, we'd just use Libevent 2.0 from now on. But as of June + * 2009, Libevent 2.0 is still in alpha, and we will have old versions of + * Libevent for the forseeable future. + **/ + +#include "orconfig.h" +#include "compat.h" +#include "compat_libevent.h" + +#include "util.h" +#include "torlog.h" + +#ifdef HAVE_EVENT2_EVENT_H +#include <event2/event.h> +#else +#include <event.h> +#endif + +/** A number representing a version of Libevent. + + This is a 4-byte number, with the first three bytes representing the + major, minor, and patchlevel respectively of the library. The fourth + byte is unused. + + This is equivalent to the format of LIBEVENT_VERSION_NUMBER on Libevent + 2.0.1 or later. For versions of Libevent before 1.4.0, which followed the + format of "1.0, 1.0a, 1.0b", we define 1.0 to be equivalent to 1.0.0, 1.0a + to be equivalent to 1.0.1, and so on. +*/ +typedef uint32_t le_version_t; + +/* Macros: returns the number of a libevent version. */ +#define V(major, minor, patch) \ + (((major) << 24) | ((minor) << 16) | ((patch) << 8)) +#define V_OLD(major, minor, patch) \ + V((major), (minor), (patch)-'a'+1) + +#define LE_OLD V(0,0,0) +#define LE_OTHER V(0,0,99) + +static le_version_t tor_get_libevent_version(const char **v_out); + +#ifdef HAVE_EVENT_SET_LOG_CALLBACK +/** A string which, if it appears in a libevent log, should be ignored. */ +static const char *suppress_msg = NULL; +/** Callback function passed to event_set_log() so we can intercept + * log messages from libevent. */ +static void +libevent_logging_callback(int severity, const char *msg) +{ + char buf[1024]; + size_t n; + if (suppress_msg && strstr(msg, suppress_msg)) + return; + n = strlcpy(buf, msg, sizeof(buf)); + if (n && n < sizeof(buf) && buf[n-1] == '\n') { + buf[n-1] = '\0'; + } + switch (severity) { + case _EVENT_LOG_DEBUG: + log(LOG_DEBUG, LD_NET, "Message from libevent: %s", buf); + break; + case _EVENT_LOG_MSG: + log(LOG_INFO, LD_NET, "Message from libevent: %s", buf); + break; + case _EVENT_LOG_WARN: + log(LOG_WARN, LD_GENERAL, "Warning from libevent: %s", buf); + break; + case _EVENT_LOG_ERR: + log(LOG_ERR, LD_GENERAL, "Error from libevent: %s", buf); + break; + default: + log(LOG_WARN, LD_GENERAL, "Message [%d] from libevent: %s", + severity, buf); + break; + } +} +/** Set hook to intercept log messages from libevent. */ +void +configure_libevent_logging(void) +{ + event_set_log_callback(libevent_logging_callback); +} +/** Ignore any libevent log message that contains <b>msg</b>. */ +void +suppress_libevent_log_msg(const char *msg) +{ + suppress_msg = msg; +} +#else +void +configure_libevent_logging(void) +{ +} +void +suppress_libevent_log_msg(const char *msg) +{ + (void)msg; +} +#endif + +#ifndef HAVE_EVENT2_EVENT_H +/** Work-alike replacement for event_new() on pre-Libevent-2.0 systems. */ +struct event * +tor_event_new(struct event_base *base, int sock, short what, + void (*cb)(int, short, void *), void *arg) +{ + struct event *e = tor_malloc_zero(sizeof(struct event)); + event_set(e, sock, what, cb, arg); + if (! base) + base = tor_libevent_get_base(); + event_base_set(base, e); + return e; +} +/** Work-alike replacement for evtimer_new() on pre-Libevent-2.0 systems. */ +struct event * +tor_evtimer_new(struct event_base *base, + void (*cb)(int, short, void *), void *arg) +{ + return tor_event_new(base, -1, 0, cb, arg); +} +/** Work-alike replacement for evsignal_new() on pre-Libevent-2.0 systems. */ +struct event * +tor_evsignal_new(struct event_base * base, int sig, + void (*cb)(int, short, void *), void *arg) +{ + return tor_event_new(base, sig, EV_SIGNAL|EV_PERSIST, cb, arg); +} +/** Work-alike replacement for event_free() on pre-Libevent-2.0 systems. */ +void +tor_event_free(struct event *ev) +{ + event_del(ev); + tor_free(ev); +} +#endif + +/** Global event base for use by the main thread. */ +struct event_base *the_event_base = NULL; + +/* This is what passes for version detection on OSX. We set + * MACOSX_KQUEUE_IS_BROKEN to true iff we're on a version of OSX before + * 10.4.0 (aka 1040). */ +#ifdef __APPLE__ +#ifdef __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ +#define MACOSX_KQUEUE_IS_BROKEN \ + (__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 1040) +#else +#define MACOSX_KQUEUE_IS_BROKEN 0 +#endif +#endif + +/** Initialize the Libevent library and set up the event base. */ +void +tor_libevent_initialize(void) +{ + tor_assert(the_event_base == NULL); + +#ifdef __APPLE__ + if (MACOSX_KQUEUE_IS_BROKEN || + tor_get_libevent_version(NULL) < V_OLD(1,1,'b')) { + setenv("EVENT_NOKQUEUE","1",1); + } +#endif + +#ifdef HAVE_EVENT2_EVENT_H + the_event_base = event_base_new(); +#else + the_event_base = event_init(); +#endif + +#if defined(HAVE_EVENT_GET_VERSION) && defined(HAVE_EVENT_GET_METHOD) + /* Making this a NOTICE for now so we can link bugs to a libevent versions + * or methods better. */ + log(LOG_NOTICE, LD_GENERAL, + "Initialized libevent version %s using method %s. Good.", + event_get_version(), tor_libevent_get_method()); +#else + log(LOG_NOTICE, LD_GENERAL, + "Initialized old libevent (version 1.0b or earlier)."); + log(LOG_WARN, LD_GENERAL, + "You have a *VERY* old version of libevent. It is likely to be buggy; " + "please build Tor with a more recent version."); +#endif +} + +/** Return the current Libevent event base that we're set up to use. */ +struct event_base * +tor_libevent_get_base(void) +{ + return the_event_base; +} + +#ifndef HAVE_EVENT_BASE_LOOPEXIT +/* Replacement for event_base_loopexit on some very old versions of Libevent + that we are not yet brave enough to deprecate. */ +int +tor_event_base_loopexit(struct event_base *base, struct timeval *tv) +{ + tor_assert(base == the_event_base); + return event_loopexit(tv); +} +#endif + +/** Return the name of the Libevent backend we're using. */ +const char * +tor_libevent_get_method(void) +{ +#ifdef HAVE_EVENT2_EVENT_H + return event_base_get_method(the_event_base); +#elif defined(HAVE_EVENT_GET_METHOD) + return event_get_method(); +#else + return "<unknown>"; +#endif +} + +/** Return the le_version_t for the current version of libevent. If the + * version is very new, return LE_OTHER. If the version is so old that it + * doesn't support event_get_version(), return LE_OLD. DOCDOC */ +static le_version_t +tor_decode_libevent_version(const char *v) +{ + unsigned major, minor, patchlevel; + char c, e, extra; + int fields; + + /* Try the new preferred "1.4.11-stable" format. + * Also accept "1.4.14b-stable". */ + fields = sscanf(v, "%u.%u.%u%c%c", &major, &minor, &patchlevel, &c, &e); + if (fields == 3 || + ((fields == 4 || fields == 5 ) && (c == '-' || c == '_')) || + (fields == 5 && TOR_ISALPHA(c) && (e == '-' || e == '_'))) { + return V(major,minor,patchlevel); + } + + /* Try the old "1.3e" format. */ + fields = sscanf(v, "%u.%u%c%c", &major, &minor, &c, &extra); + if (fields == 3 && TOR_ISALPHA(c)) { + return V_OLD(major, minor, c); + } else if (fields == 2) { + return V(major, minor, 0); + } + + return LE_OTHER; +} + +/** Return an integer representing the binary interface of a Libevent library. + * Two different versions with different numbers are sure not to be binary + * compatible. Two different versions with the same numbers have a decent + * chance of binary compatibility.*/ +static int +le_versions_compatibility(le_version_t v) +{ + if (v == LE_OTHER) + return 0; + if (v < V_OLD(1,0,'c')) + return 1; + else if (v < V(1,4,0)) + return 2; + else if (v < V(1,4,99)) + return 3; + else if (v < V(2,0,1)) + return 4; + else /* Everything 2.0 and later should be compatible. */ + return 5; +} + +/** Return the version number of the currently running version of Libevent. + See le_version_t for info on the format. + */ +static le_version_t +tor_get_libevent_version(const char **v_out) +{ + const char *v; + le_version_t r; +#if defined(HAVE_EVENT_GET_VERSION_NUMBER) + v = event_get_version(); + r = event_get_version_number(); +#elif defined (HAVE_EVENT_GET_VERSION) + v = event_get_version(); + r = tor_decode_libevent_version(v); +#else + v = "pre-1.0c"; + r = LE_OLD; +#endif + if (v_out) + *v_out = v; + return r; +} + +/** Return a string representation of the version of the currently running + * version of Libevent. */ +const char * +tor_libevent_get_version_str(void) +{ +#ifdef HAVE_EVENT_GET_VERSION + return event_get_version(); +#else + return "pre-1.0c"; +#endif +} + +/** + * Compare the current Libevent method and version to a list of versions + * which are known not to work. Warn the user as appropriate. + */ +void +tor_check_libevent_version(const char *m, int server, + const char **badness_out) +{ + int buggy = 0, iffy = 0, slow = 0, thread_unsafe = 0; + le_version_t version; + const char *v = NULL; + const char *badness = NULL; + const char *sad_os = ""; + + version = tor_get_libevent_version(&v); + + /* XXX Would it be worthwhile disabling the methods that we know + * are buggy, rather than just warning about them and then proceeding + * to use them? If so, we should probably not wrap this whole thing + * in HAVE_EVENT_GET_VERSION and HAVE_EVENT_GET_METHOD. -RD */ + /* XXXX The problem is that it's not trivial to get libevent to change it's + * method once it's initialized, and it's not trivial to tell what method it + * will use without initializing it. I guess we could preemptively disable + * buggy libevent modes based on the version _before_ initializing it, + * though, but then there's no good way (afaict) to warn "I would have used + * kqueue, but instead I'm using select." -NM */ + /* XXXX022 revist the above; it is fixable now. */ + if (!strcmp(m, "kqueue")) { + if (version < V_OLD(1,1,'b')) + buggy = 1; + } else if (!strcmp(m, "epoll")) { + if (version < V(1,1,0)) + iffy = 1; + } else if (!strcmp(m, "poll")) { + if (version < V_OLD(1,0,'e')) + buggy = 1; + if (version < V(1,1,0)) + slow = 1; + } else if (!strcmp(m, "select")) { + if (version < V(1,1,0)) + slow = 1; + } else if (!strcmp(m, "win32")) { + if (version < V_OLD(1,1,'b')) + buggy = 1; + } + + /* Libevent versions before 1.3b do very badly on operating systems with + * user-space threading implementations. */ +#if defined(__OpenBSD__) || defined(__FreeBSD__) || defined(__NetBSD__) + if (server && version < V_OLD(1,3,'b')) { + thread_unsafe = 1; + sad_os = "BSD variants"; + } +#elif defined(__APPLE__) || defined(__darwin__) + if (server && version < V_OLD(1,3,'b')) { + thread_unsafe = 1; + sad_os = "Mac OS X"; + } +#endif + + if (thread_unsafe) { + log(LOG_WARN, LD_GENERAL, + "Libevent version %s often crashes when running a Tor server with %s. " + "Please use the latest version of libevent (1.3b or later)",v,sad_os); + badness = "BROKEN"; + } else if (buggy) { + log(LOG_WARN, LD_GENERAL, + "There are serious bugs in using %s with libevent %s. " + "Please use the latest version of libevent.", m, v); + badness = "BROKEN"; + } else if (iffy) { + log(LOG_WARN, LD_GENERAL, + "There are minor bugs in using %s with libevent %s. " + "You may want to use the latest version of libevent.", m, v); + badness = "BUGGY"; + } else if (slow && server) { + log(LOG_WARN, LD_GENERAL, + "libevent %s can be very slow with %s. " + "When running a server, please use the latest version of libevent.", + v,m); + badness = "SLOW"; + } + + *badness_out = badness; +} + +#if defined(LIBEVENT_VERSION) +#define HEADER_VERSION LIBEVENT_VERSION +#elif defined(_EVENT_VERSION) +#define HEADER_VERSION _EVENT_VERSION +#endif + +/** See whether the headers we were built against differ from the library we + * linked against so much that we're likely to crash. If so, warn the + * user. */ +void +tor_check_libevent_header_compatibility(void) +{ + (void) le_versions_compatibility; + (void) tor_decode_libevent_version; + + /* In libevent versions before 2.0, it's hard to keep binary compatibility + * between upgrades, and unpleasant to detect when the version we compiled + * against is unlike the version we have linked against. Here's how. */ +#if defined(HEADER_VERSION) && defined(HAVE_EVENT_GET_VERSION) + /* We have a header-file version and a function-call version. Easy. */ + if (strcmp(HEADER_VERSION, event_get_version())) { + le_version_t v1, v2; + int compat1 = -1, compat2 = -1; + int verybad; + v1 = tor_decode_libevent_version(HEADER_VERSION); + v2 = tor_decode_libevent_version(event_get_version()); + compat1 = le_versions_compatibility(v1); + compat2 = le_versions_compatibility(v2); + + verybad = compat1 != compat2; + + log(verybad ? LOG_WARN : LOG_NOTICE, + LD_GENERAL, "We were compiled with headers from version %s " + "of Libevent, but we're using a Libevent library that says it's " + "version %s.", HEADER_VERSION, event_get_version()); + if (verybad) + log_warn(LD_GENERAL, "This will almost certainly make Tor crash."); + else + log_info(LD_GENERAL, "I think these versions are binary-compatible."); + } +#elif defined(HAVE_EVENT_GET_VERSION) + /* event_get_version but no _EVENT_VERSION. We might be in 1.4.0-beta or + earlier, where that's normal. To see whether we were compiled with an + earlier version, let's see whether the struct event defines MIN_HEAP_IDX. + */ +#ifdef HAVE_STRUCT_EVENT_MIN_HEAP_IDX + /* The header files are 1.4.0-beta or later. If the version is not + * 1.4.0-beta, we are incompatible. */ + { + if (strcmp(event_get_version(), "1.4.0-beta")) { + log_warn(LD_GENERAL, "It's a little hard to tell, but you seem to have " + "Libevent 1.4.0-beta header files, whereas you have linked " + "against Libevent %s. This will probably make Tor crash.", + event_get_version()); + } + } +#else + /* Our headers are 1.3e or earlier. If the library version is not 1.4.x or + later, we're probably fine. */ + { + const char *v = event_get_version(); + if ((v[0] == '1' && v[2] == '.' && v[3] > '3') || v[0] > '1') { + log_warn(LD_GENERAL, "It's a little hard to tell, but you seem to have " + "Libevent header file from 1.3e or earlier, whereas you have " + "linked against Libevent %s. This will probably make Tor " + "crash.", event_get_version()); + } + } +#endif + +#elif defined(HEADER_VERSION) +#warn "_EVENT_VERSION is defined but not get_event_version(): Libevent is odd." +#else + /* Your libevent is ancient. */ +#endif +} + +/* + If possible, we're going to try to use Libevent's periodic timer support, + since it does a pretty good job of making sure that periodic events get + called exactly M seconds apart, rather than starting each one exactly M + seconds after the time that the last one was run. + */ +#ifdef HAVE_EVENT2_EVENT_H +#define HAVE_PERIODIC +#define PERIODIC_FLAGS EV_PERSIST +#else +#define PERIODIC_FLAGS 0 +#endif + +/** Represents a timer that's run every N microseconds by Libevent. */ +struct periodic_timer_t { + /** Underlying event used to implement this periodic event. */ + struct event *ev; + /** The callback we'll be invoking whenever the event triggers */ + void (*cb)(struct periodic_timer_t *, void *); + /** User-supplied data for the callback */ + void *data; +#ifndef HAVE_PERIODIC + /** If Libevent doesn't know how to invoke events every N microseconds, + * we'll need to remember the timeout interval here. */ + struct timeval tv; +#endif +}; + +/** Libevent callback to implement a periodic event. */ +static void +periodic_timer_cb(evutil_socket_t fd, short what, void *arg) +{ + periodic_timer_t *timer = arg; + (void) what; + (void) fd; +#ifndef HAVE_PERIODIC + /** reschedule the event as needed. */ + event_add(timer->ev, &timer->tv); +#endif + timer->cb(timer, timer->data); +} + +/** Create and schedule a new timer that will run every <b>tv</b> in + * the event loop of <b>base</b>. When the timer fires, it will + * run the timer in <b>cb</b> with the user-supplied data in <b>data</b>. */ +periodic_timer_t * +periodic_timer_new(struct event_base *base, + const struct timeval *tv, + void (*cb)(periodic_timer_t *timer, void *data), + void *data) +{ + periodic_timer_t *timer; + tor_assert(base); + tor_assert(tv); + tor_assert(cb); + timer = tor_malloc_zero(sizeof(periodic_timer_t)); + if (!(timer->ev = tor_event_new(base, -1, PERIODIC_FLAGS, + periodic_timer_cb, timer))) { + tor_free(timer); + return NULL; + } + timer->cb = cb; + timer->data = data; +#ifndef HAVE_PERIODIC + memcpy(&timer->tv, tv, sizeof(struct timeval)); +#endif + event_add(timer->ev, (struct timeval *)tv); /*drop const for old libevent*/ + return timer; +} + +/** Stop and free a periodic timer */ +void +periodic_timer_free(periodic_timer_t *timer) +{ + if (!timer) + return; + tor_event_free(timer->ev); + tor_free(timer); +} + diff --git a/src/common/compat_libevent.h b/src/common/compat_libevent.h new file mode 100644 index 0000000000..fdf5e0a18f --- /dev/null +++ b/src/common/compat_libevent.h @@ -0,0 +1,65 @@ +/* Copyright (c) 2009, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef _TOR_COMPAT_LIBEVENT_H +#define _TOR_COMPAT_LIBEVENT_H + +#include "orconfig.h" + +struct event; +struct event_base; + +#ifdef HAVE_EVENT2_EVENT_H +#include <event2/util.h> +#else +#define evutil_socket_t int +#endif + +void configure_libevent_logging(void); +void suppress_libevent_log_msg(const char *msg); + +#ifdef HAVE_EVENT2_EVENT_H +#define tor_event_new event_new +#define tor_evtimer_new evtimer_new +#define tor_evsignal_new evsignal_new +#define tor_event_free event_free +#define tor_evdns_add_server_port(sock, tcp, cb, data) \ + evdns_add_server_port_with_base(tor_libevent_get_base(), \ + (sock),(tcp),(cb),(data)); + +#else +struct event *tor_event_new(struct event_base * base, evutil_socket_t sock, + short what, void (*cb)(evutil_socket_t, short, void *), void *arg); +struct event *tor_evtimer_new(struct event_base * base, + void (*cb)(evutil_socket_t, short, void *), void *arg); +struct event *tor_evsignal_new(struct event_base * base, int sig, + void (*cb)(evutil_socket_t, short, void *), void *arg); +void tor_event_free(struct event *ev); +#define tor_evdns_add_server_port evdns_add_server_port +#endif + +typedef struct periodic_timer_t periodic_timer_t; + +periodic_timer_t *periodic_timer_new(struct event_base *base, + const struct timeval *tv, + void (*cb)(periodic_timer_t *timer, void *data), + void *data); +void periodic_timer_free(periodic_timer_t *); + +#ifdef HAVE_EVENT_BASE_LOOPEXIT +#define tor_event_base_loopexit event_base_loopexit +#else +struct timeval; +int tor_event_base_loopexit(struct event_base *base, struct timeval *tv); +#endif + +void tor_libevent_initialize(void); +struct event_base *tor_libevent_get_base(void); +const char *tor_libevent_get_method(void); +void tor_check_libevent_version(const char *m, int server, + const char **badness_out); +void tor_check_libevent_header_compatibility(void); +const char *tor_libevent_get_version_str(void); + +#endif + diff --git a/src/common/container.c b/src/common/container.c index 08fab5ced1..72f3470344 100644 --- a/src/common/container.c +++ b/src/common/container.c @@ -13,7 +13,7 @@ #include "compat.h" #include "util.h" -#include "log.h" +#include "torlog.h" #include "container.h" #include "crypto.h" @@ -44,7 +44,8 @@ smartlist_create(void) void smartlist_free(smartlist_t *sl) { - tor_assert(sl != NULL); + if (!sl) + return; tor_free(sl->list); tor_free(sl); } @@ -459,6 +460,42 @@ smartlist_sort(smartlist_t *sl, int (*compare)(const void **a, const void **b)) (int (*)(const void *,const void*))compare); } +/** Given a smartlist <b>sl</b> sorted with the function <b>compare</b>, + * return the most frequent member in the list. Break ties in favor of + * later elements. If the list is empty, return NULL. + */ +void * +smartlist_get_most_frequent(const smartlist_t *sl, + int (*compare)(const void **a, const void **b)) +{ + const void *most_frequent = NULL; + int most_frequent_count = 0; + + const void *cur = NULL; + int i, count=0; + + if (!sl->num_used) + return NULL; + for (i = 0; i < sl->num_used; ++i) { + const void *item = sl->list[i]; + if (cur && 0 == compare(&cur, &item)) { + ++count; + } else { + if (cur && count >= most_frequent_count) { + most_frequent = cur; + most_frequent_count = count; + } + cur = item; + count = 1; + } + } + if (cur && count >= most_frequent_count) { + most_frequent = cur; + most_frequent_count = count; + } + return (void*)most_frequent; +} + /** Given a sorted smartlist <b>sl</b> and the comparison function used to * sort it, remove all duplicate members. If free_fn is provided, calls * free_fn on each duplicate. Otherwise, just removes them. Preserves order. @@ -550,6 +587,13 @@ smartlist_sort_strings(smartlist_t *sl) smartlist_sort(sl, _compare_string_ptrs); } +/** Return the most frequent string in the sorted list <b>sl</b> */ +char * +smartlist_get_most_frequent_string(smartlist_t *sl) +{ + return smartlist_get_most_frequent(sl, _compare_string_ptrs); +} + /** Remove duplicate strings from a sorted list, and free them with tor_free(). */ void @@ -561,6 +605,38 @@ smartlist_uniq_strings(smartlist_t *sl) /* Heap-based priority queue implementation for O(lg N) insert and remove. * Recall that the heap property is that, for every index I, h[I] < * H[LEFT_CHILD[I]] and h[I] < H[RIGHT_CHILD[I]]. + * + * For us to remove items other than the topmost item, each item must store + * its own index within the heap. When calling the pqueue functions, tell + * them about the offset of the field that stores the index within the item. + * + * Example: + * + * typedef struct timer_t { + * struct timeval tv; + * int heap_index; + * } timer_t; + * + * static int compare(const void *p1, const void *p2) { + * const timer_t *t1 = p1, *t2 = p2; + * if (t1->tv.tv_sec < t2->tv.tv_sec) { + * return -1; + * } else if (t1->tv.tv_sec > t2->tv.tv_sec) { + * return 1; + * } else { + * return t1->tv.tv_usec - t2->tv_usec; + * } + * } + * + * void timer_heap_insert(smartlist_t *heap, timer_t *timer) { + * smartlist_pqueue_add(heap, compare, STRUCT_OFFSET(timer_t, heap_index), + * timer); + * } + * + * void timer_heap_pop(smartlist_t *heap) { + * return smartlist_pqueue_pop(heap, compare, + * STRUCT_OFFSET(timer_t, heap_index)); + * } */ /* For a 1-indexed array, we would use LEFT_CHILD[x] = 2*x and RIGHT_CHILD[x] @@ -572,12 +648,22 @@ smartlist_uniq_strings(smartlist_t *sl) #define RIGHT_CHILD(i) ( 2*(i) + 2 ) #define PARENT(i) ( ((i)-1) / 2 ) +#define IDXP(p) ((int*)STRUCT_VAR_P(p, idx_field_offset)) + +#define UPDATE_IDX(i) do { \ + void *updated = sl->list[i]; \ + *IDXP(updated) = i; \ + } while (0) + +#define IDX_OF_ITEM(p) (*IDXP(p)) + /** Helper. <b>sl</b> may have at most one violation of the heap property: * the item at <b>idx</b> may be greater than one or both of its children. * Restore the heap property. */ static INLINE void smartlist_heapify(smartlist_t *sl, int (*compare)(const void *a, const void *b), + int idx_field_offset, int idx) { while (1) { @@ -600,21 +686,28 @@ smartlist_heapify(smartlist_t *sl, void *tmp = sl->list[idx]; sl->list[idx] = sl->list[best_idx]; sl->list[best_idx] = tmp; + UPDATE_IDX(idx); + UPDATE_IDX(best_idx); idx = best_idx; } } } -/** Insert <b>item</b> into the heap stored in <b>sl</b>, where order - * is determined by <b>compare</b>. */ +/** Insert <b>item</b> into the heap stored in <b>sl</b>, where order is + * determined by <b>compare</b> and the offset of the item in the heap is + * stored in an int-typed field at position <b>idx_field_offset</b> within + * item. + */ void smartlist_pqueue_add(smartlist_t *sl, int (*compare)(const void *a, const void *b), + int idx_field_offset, void *item) { int idx; smartlist_add(sl,item); + UPDATE_IDX(sl->num_used-1); for (idx = sl->num_used - 1; idx; ) { int parent = PARENT(idx); @@ -622,6 +715,8 @@ smartlist_pqueue_add(smartlist_t *sl, void *tmp = sl->list[parent]; sl->list[parent] = sl->list[idx]; sl->list[idx] = tmp; + UPDATE_IDX(parent); + UPDATE_IDX(idx); idx = parent; } else { return; @@ -630,32 +725,63 @@ smartlist_pqueue_add(smartlist_t *sl, } /** Remove and return the top-priority item from the heap stored in <b>sl</b>, - * where order is determined by <b>compare</b>. <b>sl</b> must not be - * empty. */ + * where order is determined by <b>compare</b> and the item's position is + * stored at position <b>idx_field_offset</b> within the item. <b>sl</b> must + * not be empty. */ void * smartlist_pqueue_pop(smartlist_t *sl, - int (*compare)(const void *a, const void *b)) + int (*compare)(const void *a, const void *b), + int idx_field_offset) { void *top; tor_assert(sl->num_used); top = sl->list[0]; + *IDXP(top)=-1; if (--sl->num_used) { sl->list[0] = sl->list[sl->num_used]; - smartlist_heapify(sl, compare, 0); + UPDATE_IDX(0); + smartlist_heapify(sl, compare, idx_field_offset, 0); } return top; } +/** Remove the item <b>item</b> from the heap stored in <b>sl</b>, + * where order is determined by <b>compare</b> and the item's position is + * stored at position <b>idx_field_offset</b> within the item. <b>sl</b> must + * not be empty. */ +void +smartlist_pqueue_remove(smartlist_t *sl, + int (*compare)(const void *a, const void *b), + int idx_field_offset, + void *item) +{ + int idx = IDX_OF_ITEM(item); + tor_assert(idx >= 0); + tor_assert(sl->list[idx] == item); + --sl->num_used; + *IDXP(item) = -1; + if (idx == sl->num_used) { + return; + } else { + sl->list[idx] = sl->list[sl->num_used]; + UPDATE_IDX(idx); + smartlist_heapify(sl, compare, idx_field_offset, idx); + } +} + /** Assert that the heap property is correctly maintained by the heap stored * in <b>sl</b>, where order is determined by <b>compare</b>. */ void smartlist_pqueue_assert_ok(smartlist_t *sl, - int (*compare)(const void *a, const void *b)) + int (*compare)(const void *a, const void *b), + int idx_field_offset) { int i; - for (i = sl->num_used - 1; i > 0; --i) { - tor_assert(compare(sl->list[PARENT(i)], sl->list[i]) <= 0); + for (i = sl->num_used - 1; i >= 0; --i) { + if (i>0) + tor_assert(compare(sl->list[PARENT(i)], sl->list[i]) <= 0); + tor_assert(IDX_OF_ITEM(sl->list[i]) == i); } } @@ -681,6 +807,37 @@ smartlist_uniq_digests(smartlist_t *sl) smartlist_uniq(sl, _compare_digests, _tor_free); } +/** Helper: compare two DIGEST256_LEN digests. */ +static int +_compare_digests256(const void **_a, const void **_b) +{ + return memcmp((const char*)*_a, (const char*)*_b, DIGEST256_LEN); +} + +/** Sort the list of DIGEST256_LEN-byte digests into ascending order. */ +void +smartlist_sort_digests256(smartlist_t *sl) +{ + smartlist_sort(sl, _compare_digests256); +} + +/** Return the most frequent member of the sorted list of DIGEST256_LEN + * digests in <b>sl</b> */ +char * +smartlist_get_most_frequent_digest256(smartlist_t *sl) +{ + return smartlist_get_most_frequent(sl, _compare_digests256); +} + +/** Remove duplicate 256-bit digests from a sorted list, and free them with + * tor_free(). + */ +void +smartlist_uniq_digests256(smartlist_t *sl) +{ + smartlist_uniq(sl, _compare_digests256, _tor_free); +} + /** Helper: Declare an entry type and a map type to implement a mapping using * ht.h. The map type will be called <b>maptype</b>. The key part of each * entry is declared using the C declaration <b>keydecl</b>. All functions @@ -1113,6 +1270,9 @@ void strmap_free(strmap_t *map, void (*free_val)(void*)) { strmap_entry_t **ent, **next, *this; + if (!map) + return; + for (ent = HT_START(strmap_impl, &map->head); ent != NULL; ent = next) { this = *ent; next = HT_NEXT_RMV(strmap_impl, &map->head, ent); @@ -1134,6 +1294,8 @@ void digestmap_free(digestmap_t *map, void (*free_val)(void*)) { digestmap_entry_t **ent, **next, *this; + if (!map) + return; for (ent = HT_START(digestmap_impl, &map->head); ent != NULL; ent = next) { this = *ent; next = HT_NEXT_RMV(digestmap_impl, &map->head, ent); @@ -1220,6 +1382,7 @@ IMPLEMENT_ORDER_FUNC(find_nth_int, int) IMPLEMENT_ORDER_FUNC(find_nth_time, time_t) IMPLEMENT_ORDER_FUNC(find_nth_double, double) IMPLEMENT_ORDER_FUNC(find_nth_uint32, uint32_t) +IMPLEMENT_ORDER_FUNC(find_nth_int32, int32_t) IMPLEMENT_ORDER_FUNC(find_nth_long, long) /** Return a newly allocated digestset_t, optimized to hold a total of @@ -1248,6 +1411,8 @@ digestset_new(int max_elements) void digestset_free(digestset_t *set) { + if (!set) + return; bitarray_free(set->ba); tor_free(set); } diff --git a/src/common/container.h b/src/common/container.h index c42b86977d..3568de0159 100644 --- a/src/common/container.h +++ b/src/common/container.h @@ -93,13 +93,22 @@ void smartlist_del_keeporder(smartlist_t *sl, int idx); void smartlist_insert(smartlist_t *sl, int idx, void *val); void smartlist_sort(smartlist_t *sl, int (*compare)(const void **a, const void **b)); +void *smartlist_get_most_frequent(const smartlist_t *sl, + int (*compare)(const void **a, const void **b)); void smartlist_uniq(smartlist_t *sl, int (*compare)(const void **a, const void **b), void (*free_fn)(void *elt)); + void smartlist_sort_strings(smartlist_t *sl); void smartlist_sort_digests(smartlist_t *sl); +void smartlist_sort_digests256(smartlist_t *sl); + +char *smartlist_get_most_frequent_string(smartlist_t *sl); +char *smartlist_get_most_frequent_digest256(smartlist_t *sl); + void smartlist_uniq_strings(smartlist_t *sl); void smartlist_uniq_digests(smartlist_t *sl); +void smartlist_uniq_digests256(smartlist_t *sl); void *smartlist_bsearch(smartlist_t *sl, const void *key, int (*compare)(const void *key, const void **member)) ATTR_PURE; @@ -109,11 +118,18 @@ int smartlist_bsearch_idx(const smartlist_t *sl, const void *key, void smartlist_pqueue_add(smartlist_t *sl, int (*compare)(const void *a, const void *b), + int idx_field_offset, void *item); void *smartlist_pqueue_pop(smartlist_t *sl, - int (*compare)(const void *a, const void *b)); + int (*compare)(const void *a, const void *b), + int idx_field_offset); +void smartlist_pqueue_remove(smartlist_t *sl, + int (*compare)(const void *a, const void *b), + int idx_field_offset, + void *item); void smartlist_pqueue_assert_ok(smartlist_t *sl, - int (*compare)(const void *a, const void *b)); + int (*compare)(const void *a, const void *b), + int idx_field_offset); #define SPLIT_SKIP_SPACE 0x01 #define SPLIT_IGNORE_BLANK 0x02 @@ -627,6 +643,7 @@ void digestset_free(digestset_t* set); int find_nth_int(int *array, int n_elements, int nth); time_t find_nth_time(time_t *array, int n_elements, int nth); double find_nth_double(double *array, int n_elements, int nth); +int32_t find_nth_int32(int32_t *array, int n_elements, int nth); uint32_t find_nth_uint32(uint32_t *array, int n_elements, int nth); long find_nth_long(long *array, int n_elements, int nth); static INLINE int @@ -649,6 +666,11 @@ median_uint32(uint32_t *array, int n_elements) { return find_nth_uint32(array, n_elements, (n_elements-1)/2); } +static INLINE int32_t +median_int32(int32_t *array, int n_elements) +{ + return find_nth_int32(array, n_elements, (n_elements-1)/2); +} static INLINE long median_long(long *array, int n_elements) { diff --git a/src/common/crypto.c b/src/common/crypto.c index 8aef4771c3..bfb81d3ccd 100644 --- a/src/common/crypto.c +++ b/src/common/crypto.c @@ -27,6 +27,7 @@ #include <openssl/rsa.h> #include <openssl/pem.h> #include <openssl/evp.h> +#include <openssl/engine.h> #include <openssl/rand.h> #include <openssl/opensslv.h> #include <openssl/bn.h> @@ -49,9 +50,9 @@ #define CRYPTO_PRIVATE #include "crypto.h" -#include "log.h" +#include "../common/torlog.h" #include "aes.h" -#include "util.h" +#include "../common/util.h" #include "container.h" #include "compat.h" @@ -61,6 +62,33 @@ #include <openssl/engine.h> +#ifdef ANDROID +/* Android's OpenSSL seems to have removed all of its Engine support. */ +#define DISABLE_ENGINES +#endif + +#if OPENSSL_VERSION_NUMBER < 0x00908000l +/* On OpenSSL versions before 0.9.8, there is no working SHA256 + * implementation, so we use Tom St Denis's nice speedy one, slightly adapted + * to our needs */ +#define SHA256_CTX sha256_state +#define SHA256_Init sha256_init +#define SHA256_Update sha256_process +#define LTC_ARGCHK(x) tor_assert(x) +#include "sha256.c" +#define SHA256_Final(a,b) sha256_done(b,a) + +static unsigned char * +SHA256(const unsigned char *m, size_t len, unsigned char *d) +{ + SHA256_CTX ctx; + SHA256_Init(&ctx); + SHA256_Update(&ctx, m, len); + SHA256_Final(d, &ctx); + return d; +} +#endif + /** Macro: is k a valid RSA public or private key? */ #define PUBLIC_KEY_OK(k) ((k) && (k)->key && (k)->key->n) /** Macro: is k a valid RSA private key? */ @@ -94,7 +122,7 @@ struct crypto_dh_env_t { }; static int setup_openssl_threading(void); -static int tor_check_dh_key(BIGNUM *bn); +static int tor_check_dh_key(int severity, BIGNUM *bn); /** Return the number of bytes added by padding method <b>padding</b>. */ @@ -151,6 +179,7 @@ crypto_log_errors(int severity, const char *doing) } } +#ifndef DISABLE_ENGINES /** Log any OpenSSL engines we're using at NOTICE. */ static void log_engine(const char *fn, ENGINE *e) @@ -165,37 +194,82 @@ log_engine(const char *fn, ENGINE *e) log(LOG_INFO, LD_CRYPTO, "Using default implementation for %s", fn); } } +#endif + +#ifndef DISABLE_ENGINES +/** Try to load an engine in a shared library via fully qualified path. + */ +static ENGINE * +try_load_engine(const char *path, const char *engine) +{ + ENGINE *e = ENGINE_by_id("dynamic"); + if (e) { + if (!ENGINE_ctrl_cmd_string(e, "ID", engine, 0) || + !ENGINE_ctrl_cmd_string(e, "DIR_LOAD", "2", 0) || + !ENGINE_ctrl_cmd_string(e, "DIR_ADD", path, 0) || + !ENGINE_ctrl_cmd_string(e, "LOAD", NULL, 0)) { + ENGINE_free(e); + e = NULL; + } + } + return e; +} +#endif /** Initialize the crypto library. Return 0 on success, -1 on failure. */ int -crypto_global_init(int useAccel) +crypto_global_init(int useAccel, const char *accelName, const char *accelDir) { if (!_crypto_global_initialized) { ERR_load_crypto_strings(); OpenSSL_add_all_algorithms(); _crypto_global_initialized = 1; setup_openssl_threading(); - /* XXX the below is a bug, since we can't know if we're supposed - * to be using hardware acceleration or not. we should arrange - * for this function to be called before init_keys. But make it - * not complain loudly, at least until we make acceleration work. */ - if (useAccel < 0) { - log_info(LD_CRYPTO, "Initializing OpenSSL via tor_tls_init()."); - } if (useAccel > 0) { +#ifdef DISABLE_ENGINES + (void)accelName; + (void)accelDir; + log_warn(LD_CRYPTO, "No OpenSSL hardware acceleration support enabled."); +#else + ENGINE *e = NULL; + log_info(LD_CRYPTO, "Initializing OpenSSL engine support."); ENGINE_load_builtin_engines(); - if (!ENGINE_register_all_complete()) - return -1; - - /* XXXX make sure this isn't leaking. */ + ENGINE_register_all_complete(); + + if (accelName) { + if (accelDir) { + log_info(LD_CRYPTO, "Trying to load dynamic OpenSSL engine \"%s\"" + " via path \"%s\".", accelName, accelDir); + e = try_load_engine(accelName, accelDir); + } else { + log_info(LD_CRYPTO, "Initializing dynamic OpenSSL engine \"%s\"" + " acceleration support.", accelName); + e = ENGINE_by_id(accelName); + } + if (!e) { + log_warn(LD_CRYPTO, "Unable to load dynamic OpenSSL engine \"%s\".", + accelName); + } else { + log_info(LD_CRYPTO, "Loaded dynamic OpenSSL engine \"%s\".", + accelName); + } + } + if (e) { + log_info(LD_CRYPTO, "Loaded OpenSSL hardware acceleration engine," + " setting default ciphers."); + ENGINE_set_default(e, ENGINE_METHOD_ALL); + } log_engine("RSA", ENGINE_get_default_RSA()); log_engine("DH", ENGINE_get_default_DH()); log_engine("RAND", ENGINE_get_default_RAND()); log_engine("SHA1", ENGINE_get_digest_engine(NID_sha1)); log_engine("3DES", ENGINE_get_cipher_engine(NID_des_ede3_ecb)); log_engine("AES", ENGINE_get_cipher_engine(NID_aes_128_ecb)); +#endif + } else { + log_info(LD_CRYPTO, "NOT using OpenSSL engine support."); } return crypto_seed_rng(1); } @@ -217,7 +291,11 @@ crypto_global_cleanup(void) EVP_cleanup(); ERR_remove_state(0); ERR_free_strings(); + +#ifndef DISABLE_ENGINES ENGINE_cleanup(); +#endif + CONF_modules_unload(1); CRYPTO_cleanup_all_ex_data(); #ifdef TOR_IS_MULTITHREADED @@ -259,7 +337,8 @@ _crypto_new_pk_env_evp_pkey(EVP_PKEY *pkey) return _crypto_new_pk_env_rsa(rsa); } -/** Helper, used by tor-checkkey.c. Return the RSA from a crypto_pk_env_t. */ +/** Helper, used by tor-checkkey.c and tor-gencert.c. Return the RSA from a + * crypto_pk_env_t. */ RSA * _crypto_pk_env_get_rsa(crypto_pk_env_t *env) { @@ -321,7 +400,8 @@ crypto_new_pk_env(void) void crypto_free_pk_env(crypto_pk_env_t *env) { - tor_assert(env); + if (!env) + return; if (--env->refs > 0) return; @@ -347,10 +427,7 @@ crypto_create_init_cipher(const char *key, int encrypt_mode) return NULL; } - if (crypto_cipher_set_key(crypto, key)) { - crypto_log_errors(LOG_WARN, "setting symmetric key"); - goto error; - } + crypto_cipher_set_key(crypto, key); if (encrypt_mode) r = crypto_cipher_encrypt_init_cipher(crypto); @@ -384,7 +461,8 @@ crypto_new_cipher_env(void) void crypto_free_cipher_env(crypto_cipher_env_t *env) { - tor_assert(env); + if (!env) + return; tor_assert(env->cipher); aes_free_cipher(env->cipher); @@ -394,11 +472,11 @@ crypto_free_cipher_env(crypto_cipher_env_t *env) /* public key crypto */ -/** Generate a new public/private keypair in <b>env</b>. Return 0 on - * success, -1 on failure. +/** Generate a <b>bits</b>-bit new public/private keypair in <b>env</b>. + * Return 0 on success, -1 on failure. */ int -crypto_pk_generate_key(crypto_pk_env_t *env) +crypto_pk_generate_key_with_bits(crypto_pk_env_t *env, int bits) { tor_assert(env); @@ -406,7 +484,7 @@ crypto_pk_generate_key(crypto_pk_env_t *env) RSA_free(env->key); #if OPENSSL_VERSION_NUMBER < 0x00908000l /* In OpenSSL 0.9.7, RSA_generate_key is all we have. */ - env->key = RSA_generate_key(PK_BYTES*8,65537, NULL, NULL); + env->key = RSA_generate_key(bits, 65537, NULL, NULL); #else /* In OpenSSL 0.9.8, RSA_generate_key is deprecated. */ { @@ -419,7 +497,7 @@ crypto_pk_generate_key(crypto_pk_env_t *env) r = RSA_new(); if (!r) goto done; - if (RSA_generate_key_ex(r, PK_BYTES*8, e, NULL) == -1) + if (RSA_generate_key_ex(r, bits, e, NULL) == -1) goto done; env->key = r; @@ -701,14 +779,25 @@ crypto_pk_env_t * crypto_pk_copy_full(crypto_pk_env_t *env) { RSA *new_key; + int privatekey = 0; tor_assert(env); tor_assert(env->key); if (PRIVATE_KEY_OK(env)) { new_key = RSAPrivateKey_dup(env->key); + privatekey = 1; } else { new_key = RSAPublicKey_dup(env->key); } + if (!new_key) { + log_err(LD_CRYPTO, "Unable to duplicate a %s key: openssl failed.", + privatekey?"private":"public"); + crypto_log_errors(LOG_ERR, + privatekey ? "Duplicating a private key" : + "Duplicating a public key"); + tor_fragile_assert(); + return NULL; + } return _crypto_new_pk_env_rsa(new_key); } @@ -1172,19 +1261,14 @@ crypto_cipher_generate_key(crypto_cipher_env_t *env) /** Set the symmetric key for the cipher in <b>env</b> to the first * CIPHER_KEY_LEN bytes of <b>key</b>. Does not initialize the cipher. - * Return 0 on success, -1 on failure. */ -int +void crypto_cipher_set_key(crypto_cipher_env_t *env, const char *key) { tor_assert(env); tor_assert(key); - if (!env->key) - return -1; - memcpy(env->key, key, CIPHER_KEY_LEN); - return 0; } /** Generate an initialization vector for our AES-CTR cipher; store it @@ -1358,9 +1442,69 @@ crypto_digest(char *digest, const char *m, size_t len) return (SHA1((const unsigned char*)m,len,(unsigned char*)digest) == NULL); } +int +crypto_digest256(char *digest, const char *m, size_t len, + digest_algorithm_t algorithm) +{ + tor_assert(m); + tor_assert(digest); + tor_assert(algorithm == DIGEST_SHA256); + return (SHA256((const unsigned char*)m,len,(unsigned char*)digest) == NULL); +} + +/** Set the digests_t in <b>ds_out</b> to contain every digest on the + * <b>len</b> bytes in <b>m</b> that we know how to compute. Return 0 on + * success, -1 on failure. */ +int +crypto_digest_all(digests_t *ds_out, const char *m, size_t len) +{ + digest_algorithm_t i; + tor_assert(ds_out); + memset(ds_out, 0, sizeof(*ds_out)); + if (crypto_digest(ds_out->d[DIGEST_SHA1], m, len) < 0) + return -1; + for (i = DIGEST_SHA256; i < N_DIGEST_ALGORITHMS; ++i) { + if (crypto_digest256(ds_out->d[i], m, len, i) < 0) + return -1; + } + return 0; +} + +/** Return the name of an algorithm, as used in directory documents. */ +const char * +crypto_digest_algorithm_get_name(digest_algorithm_t alg) +{ + switch (alg) { + case DIGEST_SHA1: + return "sha1"; + case DIGEST_SHA256: + return "sha256"; + default: + tor_fragile_assert(); + return "??unknown_digest??"; + } +} + +/** Given the name of a digest algorithm, return its integer value, or -1 if + * the name is not recognized. */ +int +crypto_digest_algorithm_parse_name(const char *name) +{ + if (!strcmp(name, "sha1")) + return DIGEST_SHA1; + else if (!strcmp(name, "sha256")) + return DIGEST_SHA256; + else + return -1; +} + /** Intermediate information about the digest of a stream of data. */ struct crypto_digest_env_t { - SHA_CTX d; + union { + SHA_CTX sha1; + SHA256_CTX sha2; + } d; + digest_algorithm_t algorithm : 8; }; /** Allocate and return a new digest object. @@ -1370,7 +1514,19 @@ crypto_new_digest_env(void) { crypto_digest_env_t *r; r = tor_malloc(sizeof(crypto_digest_env_t)); - SHA1_Init(&r->d); + SHA1_Init(&r->d.sha1); + r->algorithm = DIGEST_SHA1; + return r; +} + +crypto_digest_env_t * +crypto_new_digest256_env(digest_algorithm_t algorithm) +{ + crypto_digest_env_t *r; + tor_assert(algorithm == DIGEST_SHA256); + r = tor_malloc(sizeof(crypto_digest_env_t)); + SHA256_Init(&r->d.sha2); + r->algorithm = algorithm; return r; } @@ -1379,6 +1535,8 @@ crypto_new_digest_env(void) void crypto_free_digest_env(crypto_digest_env_t *digest) { + if (!digest) + return; memset(digest, 0, sizeof(crypto_digest_env_t)); tor_free(digest); } @@ -1391,30 +1549,51 @@ crypto_digest_add_bytes(crypto_digest_env_t *digest, const char *data, { tor_assert(digest); tor_assert(data); - /* Using the SHA1_*() calls directly means we don't support doing - * SHA1 in hardware. But so far the delay of getting the question + /* Using the SHA*_*() calls directly means we don't support doing + * SHA in hardware. But so far the delay of getting the question * to the hardware, and hearing the answer, is likely higher than * just doing it ourselves. Hashes are fast. */ - SHA1_Update(&digest->d, (void*)data, len); + switch (digest->algorithm) { + case DIGEST_SHA1: + SHA1_Update(&digest->d.sha1, (void*)data, len); + break; + case DIGEST_SHA256: + SHA256_Update(&digest->d.sha2, (void*)data, len); + break; + default: + tor_fragile_assert(); + break; + } } /** Compute the hash of the data that has been passed to the digest * object; write the first out_len bytes of the result to <b>out</b>. - * <b>out_len</b> must be \<= DIGEST_LEN. + * <b>out_len</b> must be \<= DIGEST256_LEN. */ void crypto_digest_get_digest(crypto_digest_env_t *digest, char *out, size_t out_len) { - unsigned char r[DIGEST_LEN]; - SHA_CTX tmpctx; + unsigned char r[DIGEST256_LEN]; + crypto_digest_env_t tmpenv; tor_assert(digest); tor_assert(out); - tor_assert(out_len <= DIGEST_LEN); - /* memcpy into a temporary ctx, since SHA1_Final clears the context */ - memcpy(&tmpctx, &digest->d, sizeof(SHA_CTX)); - SHA1_Final(r, &tmpctx); + /* memcpy into a temporary ctx, since SHA*_Final clears the context */ + memcpy(&tmpenv, digest, sizeof(crypto_digest_env_t)); + switch (digest->algorithm) { + case DIGEST_SHA1: + tor_assert(out_len <= DIGEST_LEN); + SHA1_Final(r, &tmpenv.d.sha1); + break; + case DIGEST_SHA256: + tor_assert(out_len <= DIGEST256_LEN); + SHA256_Final(r, &tmpenv.d.sha2); + break; + default: + tor_fragile_assert(); + break; + } memcpy(out, r, out_len); memset(r, 0, sizeof(r)); } @@ -1550,7 +1729,7 @@ crypto_dh_generate_public(crypto_dh_env_t *dh) crypto_log_errors(LOG_WARN, "generating DH key"); return -1; } - if (tor_check_dh_key(dh->dh->pub_key)<0) { + if (tor_check_dh_key(LOG_WARN, dh->dh->pub_key)<0) { log_warn(LD_CRYPTO, "Weird! Our own DH key was invalid. I guess once-in-" "the-universe chances really do happen. Trying again."); /* Free and clear the keys, so OpenSSL will actually try again. */ @@ -1597,7 +1776,7 @@ crypto_dh_get_public(crypto_dh_env_t *dh, char *pubkey, size_t pubkey_len) * See http://www.cl.cam.ac.uk/ftp/users/rja14/psandqs.ps.gz for some tips. */ static int -tor_check_dh_key(BIGNUM *bn) +tor_check_dh_key(int severity, BIGNUM *bn) { BIGNUM *x; char *s; @@ -1608,13 +1787,13 @@ tor_check_dh_key(BIGNUM *bn) init_dh_param(); BN_set_word(x, 1); if (BN_cmp(bn,x)<=0) { - log_warn(LD_CRYPTO, "DH key must be at least 2."); + log_fn(severity, LD_CRYPTO, "DH key must be at least 2."); goto err; } BN_copy(x,dh_param_p); BN_sub_word(x, 1); if (BN_cmp(bn,x)>=0) { - log_warn(LD_CRYPTO, "DH key must be at most p-2."); + log_fn(severity, LD_CRYPTO, "DH key must be at most p-2."); goto err; } BN_free(x); @@ -1622,7 +1801,7 @@ tor_check_dh_key(BIGNUM *bn) err: BN_free(x); s = BN_bn2hex(bn); - log_warn(LD_CRYPTO, "Rejecting insecure DH key [%s]", s); + log_fn(severity, LD_CRYPTO, "Rejecting insecure DH key [%s]", s); OPENSSL_free(s); return -1; } @@ -1640,7 +1819,7 @@ tor_check_dh_key(BIGNUM *bn) * where || is concatenation.) */ ssize_t -crypto_dh_compute_secret(crypto_dh_env_t *dh, +crypto_dh_compute_secret(int severity, crypto_dh_env_t *dh, const char *pubkey, size_t pubkey_len, char *secret_out, size_t secret_bytes_out) { @@ -1655,9 +1834,9 @@ crypto_dh_compute_secret(crypto_dh_env_t *dh, if (!(pubkey_bn = BN_bin2bn((const unsigned char*)pubkey, (int)pubkey_len, NULL))) goto error; - if (tor_check_dh_key(pubkey_bn)<0) { + if (tor_check_dh_key(severity, pubkey_bn)<0) { /* Check for invalid public keys. */ - log_warn(LD_CRYPTO,"Rejected invalid g^x"); + log_fn(severity, LD_CRYPTO,"Rejected invalid g^x"); goto error; } secret_tmp = tor_malloc(crypto_dh_get_bytes(dh)); @@ -1729,7 +1908,8 @@ crypto_expand_key_material(const char *key_in, size_t key_in_len, void crypto_dh_free(crypto_dh_env_t *dh) { - tor_assert(dh); + if (!dh) + return; tor_assert(dh->dh); DH_free(dh->dh); tor_free(dh); @@ -1887,6 +2067,26 @@ crypto_rand_uint64(uint64_t max) } } +/** Return a pseudorandom double d, chosen uniformly from the range + * 0.0 <= d < 1.0. + */ +double +crypto_rand_double(void) +{ + /* We just use an unsigned int here; we don't really care about getting + * more than 32 bits of resolution */ + unsigned int uint; + crypto_rand((char*)&uint, sizeof(uint)); +#if SIZEOF_INT == 4 +#define UINT_MAX_AS_DOUBLE 4294967296.0 +#elif SIZEOF_INT == 8 +#define UINT_MAX_AS_DOUBLE 1.8446744073709552e+19 +#else +#error SIZEOF_INT is neither 4 nor 8 +#endif + return ((double)uint) / UINT_MAX_AS_DOUBLE; +} + /** Generate and return a new random hostname starting with <b>prefix</b>, * ending with <b>suffix</b>, and containing no less than * <b>min_rand_len</b> and no more than <b>max_rand_len</b> random base32 @@ -2147,6 +2347,44 @@ digest_from_base64(char *digest, const char *d64) #endif } +/** Base-64 encode DIGEST256_LINE bytes from <b>digest</b>, remove the + * trailing = and newline characters, and store the nul-terminated result in + * the first BASE64_DIGEST256_LEN+1 bytes of <b>d64</b>. */ +int +digest256_to_base64(char *d64, const char *digest) +{ + char buf[256]; + base64_encode(buf, sizeof(buf), digest, DIGEST256_LEN); + buf[BASE64_DIGEST256_LEN] = '\0'; + memcpy(d64, buf, BASE64_DIGEST256_LEN+1); + return 0; +} + +/** Given a base-64 encoded, nul-terminated digest in <b>d64</b> (without + * trailing newline or = characters), decode it and store the result in the + * first DIGEST256_LEN bytes at <b>digest</b>. */ +int +digest256_from_base64(char *digest, const char *d64) +{ +#ifdef USE_OPENSSL_BASE64 + char buf_in[BASE64_DIGEST256_LEN+3]; + char buf[256]; + if (strlen(d64) != BASE64_DIGEST256_LEN) + return -1; + memcpy(buf_in, d64, BASE64_DIGEST256_LEN); + memcpy(buf_in+BASE64_DIGEST256_LEN, "=\n\0", 3); + if (base64_decode(buf, sizeof(buf), buf_in, strlen(buf_in)) != DIGEST256_LEN) + return -1; + memcpy(digest, buf, DIGEST256_LEN); + return 0; +#else + if (base64_decode(digest, DIGEST256_LEN, d64, strlen(d64)) == DIGEST256_LEN) + return 0; + else + return -1; +#endif +} + /** Implements base32 encoding as in rfc3548. Limitation: Requires * that srclen*8 is a multiple of 5. */ diff --git a/src/common/crypto.h b/src/common/crypto.h index 20fdfa4d2f..a30e5bcbae 100644 --- a/src/common/crypto.h +++ b/src/common/crypto.h @@ -18,6 +18,9 @@ /** Length of the output of our message digest. */ #define DIGEST_LEN 20 +/** Length of the output of our second (improved) message digests. (For now + * this is just sha256, but any it can be any other 256-byte digest). */ +#define DIGEST256_LEN 32 /** Length of our symmetric cipher's keys. */ #define CIPHER_KEY_LEN 16 /** Length of our symmetric cipher's IV. */ @@ -27,9 +30,12 @@ /** Length of our DH keys. */ #define DH_BYTES (1024/8) -/** Length of a message digest when encoded in base64 with trailing = signs - * removed. */ +/** Length of a sha1 message digest when encoded in base64 with trailing = + * signs removed. */ #define BASE64_DIGEST_LEN 27 +/** Length of a sha256 message digest when encoded in base64 with trailing = + * signs removed. */ +#define BASE64_DIGEST256_LEN 43 /** Constants used to indicate no padding for public-key encryption */ #define PK_NO_PADDING 60000 @@ -48,6 +54,26 @@ #define FINGERPRINT_LEN 49 /** Length of hex encoding of SHA1 digest, not including final NUL. */ #define HEX_DIGEST_LEN 40 +/** Length of hex encoding of SHA256 digest, not including final NUL. */ +#define HEX_DIGEST256_LEN 64 + +typedef enum { + DIGEST_SHA1 = 0, + DIGEST_SHA256 = 1, +} digest_algorithm_t; +#define N_DIGEST_ALGORITHMS (DIGEST_SHA256+1) + +/** A set of all the digests we know how to compute, taken on a single + * string. Any digests that are shorter than 256 bits are right-padded + * with 0 bits. + * + * Note that this representation wastes 12 bytes for the SHA1 case, so + * don't use it for anything where we need to allocate a whole bunch at + * once. + **/ +typedef struct { + char d[N_DIGEST_ALGORITHMS][DIGEST256_LEN]; +} digests_t; typedef struct crypto_pk_env_t crypto_pk_env_t; typedef struct crypto_cipher_env_t crypto_cipher_env_t; @@ -55,7 +81,9 @@ typedef struct crypto_digest_env_t crypto_digest_env_t; typedef struct crypto_dh_env_t crypto_dh_env_t; /* global state */ -int crypto_global_init(int hardwareAccel); +int crypto_global_init(int hardwareAccel, + const char *accelName, + const char *accelPath); void crypto_thread_cleanup(void); int crypto_global_cleanup(void); @@ -71,7 +99,9 @@ crypto_cipher_env_t *crypto_new_cipher_env(void); void crypto_free_cipher_env(crypto_cipher_env_t *env); /* public key crypto */ -int crypto_pk_generate_key(crypto_pk_env_t *env); +int crypto_pk_generate_key_with_bits(crypto_pk_env_t *env, int bits); +#define crypto_pk_generate_key(env) \ + crypto_pk_generate_key_with_bits((env), (PK_BYTES*8)) int crypto_pk_read_private_key_from_filename(crypto_pk_env_t *env, const char *keyfile); @@ -121,7 +151,7 @@ int crypto_pk_check_fingerprint_syntax(const char *s); /* symmetric crypto */ int crypto_cipher_generate_key(crypto_cipher_env_t *env); -int crypto_cipher_set_key(crypto_cipher_env_t *env, const char *key); +void crypto_cipher_set_key(crypto_cipher_env_t *env, const char *key); void crypto_cipher_generate_iv(char *iv_out); int crypto_cipher_set_iv(crypto_cipher_env_t *env, const char *iv); const char *crypto_cipher_get_key(crypto_cipher_env_t *env); @@ -141,9 +171,15 @@ int crypto_cipher_decrypt_with_iv(crypto_cipher_env_t *env, char *to, size_t tolen, const char *from, size_t fromlen); -/* SHA-1 */ +/* SHA-1 and other digests. */ int crypto_digest(char *digest, const char *m, size_t len); +int crypto_digest256(char *digest, const char *m, size_t len, + digest_algorithm_t algorithm); +int crypto_digest_all(digests_t *ds_out, const char *m, size_t len); +const char *crypto_digest_algorithm_get_name(digest_algorithm_t alg); +int crypto_digest_algorithm_parse_name(const char *name); crypto_digest_env_t *crypto_new_digest_env(void); +crypto_digest_env_t *crypto_new_digest256_env(digest_algorithm_t algorithm); void crypto_free_digest_env(crypto_digest_env_t *digest); void crypto_digest_add_bytes(crypto_digest_env_t *digest, const char *data, size_t len); @@ -162,7 +198,7 @@ int crypto_dh_get_bytes(crypto_dh_env_t *dh); int crypto_dh_generate_public(crypto_dh_env_t *dh); int crypto_dh_get_public(crypto_dh_env_t *dh, char *pubkey_out, size_t pubkey_out_len); -ssize_t crypto_dh_compute_secret(crypto_dh_env_t *dh, +ssize_t crypto_dh_compute_secret(int severity, crypto_dh_env_t *dh, const char *pubkey, size_t pubkey_len, char *secret_out, size_t secret_out_len); void crypto_dh_free(crypto_dh_env_t *dh); @@ -174,6 +210,7 @@ int crypto_seed_rng(int startup); int crypto_rand(char *to, size_t n); int crypto_rand_int(unsigned int max); uint64_t crypto_rand_uint64(uint64_t max); +double crypto_rand_double(void); char *crypto_random_hostname(int min_rand_len, int max_rand_len, const char *prefix, const char *suffix); @@ -191,6 +228,8 @@ int base32_decode(char *dest, size_t destlen, const char *src, size_t srclen); int digest_to_base64(char *d64, const char *digest); int digest_from_base64(char *digest, const char *d64); +int digest256_to_base64(char *d64, const char *digest); +int digest256_from_base64(char *digest, const char *d64); /** Length of RFC2440-style S2K specifier: the first 8 bytes are a salt, the * 9th describes how much iteration to do. */ diff --git a/src/common/ht.h b/src/common/ht.h index e6dd3d2b24..f598856d8a 100644 --- a/src/common/ht.h +++ b/src/common/ht.h @@ -42,6 +42,10 @@ #define HT_SIZE(head) \ ((head)->hth_n_entries) +/* Return memory usage for a hashtable (not counting the entries themselves) */ +#define HT_MEM_USAGE(head) \ + (sizeof(*head) + (head)->hth_table_length * sizeof(void*)) + #define HT_FIND(name, head, elm) name##_HT_FIND((head), (elm)) #define HT_INSERT(name, head, elm) name##_HT_INSERT((head), (elm)) #define HT_REPLACE(name, head, elm) name##_HT_REPLACE((head), (elm)) diff --git a/src/common/log.c b/src/common/log.c index 6baef8e26c..b639e7a781 100644 --- a/src/common/log.c +++ b/src/common/log.c @@ -33,11 +33,9 @@ #include "compat.h" #include "util.h" #define LOG_PRIVATE -#include "log.h" +#include "torlog.h" #include "container.h" -#include <event.h> - #define TRUNCATED_STR "[...truncated]" #define TRUNCATED_STR_LEN 14 @@ -83,7 +81,7 @@ should_log_function_name(log_domain_mask_t domain, int severity) /* All debugging messages occur in interesting places. */ return 1; case LOG_NOTICE: - case LOG_WARN: + case LOG_WARN: case LOG_ERR: /* We care about places where bugs occur. */ return (domain == LD_BUG); @@ -330,7 +328,7 @@ logv(int severity, log_domain_mask_t domain, const char *funcname, /** Output a message to the log. */ void -_log(int severity, log_domain_mask_t domain, const char *format, ...) +tor_log(int severity, log_domain_mask_t domain, const char *format, ...) { va_list ap; if (severity > _log_global_min_severity) @@ -428,6 +426,8 @@ _log_err(log_domain_mask_t domain, const char *format, ...) static void log_free(logfile_t *victim) { + if (!victim) + return; tor_free(victim->severities); tor_free(victim->filename); tor_free(victim); @@ -678,13 +678,11 @@ add_file_log(const log_severity_list_t *severity, const char *filename) logfiles->needs_close = 1; lf = logfiles; _log_global_min_severity = get_min_log_level(); - UNLOCK_LOGS(); if (log_tor_version(lf, 0) < 0) { - LOCK_LOGS(); delete_log(lf); - UNLOCK_LOGS(); } + UNLOCK_LOGS(); return 0; } @@ -747,7 +745,7 @@ log_level_to_string(int level) static const char *domain_list[] = { "GENERAL", "CRYPTO", "NET", "CONFIG", "FS", "PROTOCOL", "MM", "HTTP", "APP", "CONTROL", "CIRC", "REND", "BUG", "DIR", "DIRSERV", - "OR", "EDGE", "ACCT", "HIST", NULL + "OR", "EDGE", "ACCT", "HIST", "HANDSHAKE", NULL }; /** Return a bitmask for the log domain for which <b>domain</b> is the name, @@ -925,65 +923,6 @@ switch_logs_debug(void) UNLOCK_LOGS(); } -#ifdef HAVE_EVENT_SET_LOG_CALLBACK -/** A string which, if it appears in a libevent log, should be ignored. */ -static const char *suppress_msg = NULL; -/** Callback function passed to event_set_log() so we can intercept - * log messages from libevent. */ -static void -libevent_logging_callback(int severity, const char *msg) -{ - char buf[1024]; - size_t n; - if (suppress_msg && strstr(msg, suppress_msg)) - return; - n = strlcpy(buf, msg, sizeof(buf)); - if (n && n < sizeof(buf) && buf[n-1] == '\n') { - buf[n-1] = '\0'; - } - switch (severity) { - case _EVENT_LOG_DEBUG: - log(LOG_DEBUG, LD_NET, "Message from libevent: %s", buf); - break; - case _EVENT_LOG_MSG: - log(LOG_INFO, LD_NET, "Message from libevent: %s", buf); - break; - case _EVENT_LOG_WARN: - log(LOG_WARN, LD_GENERAL, "Warning from libevent: %s", buf); - break; - case _EVENT_LOG_ERR: - log(LOG_ERR, LD_GENERAL, "Error from libevent: %s", buf); - break; - default: - log(LOG_WARN, LD_GENERAL, "Message [%d] from libevent: %s", - severity, buf); - break; - } -} -/** Set hook to intercept log messages from libevent. */ -void -configure_libevent_logging(void) -{ - event_set_log_callback(libevent_logging_callback); -} -/** Ignore any libevent log message that contains <b>msg</b>. */ -void -suppress_libevent_log_msg(const char *msg) -{ - suppress_msg = msg; -} -#else -void -configure_libevent_logging(void) -{ -} -void -suppress_libevent_log_msg(const char *msg) -{ - (void)msg; -} -#endif - #if 0 static void dump_log_info(logfile_t *lf) diff --git a/src/common/memarea.c b/src/common/memarea.c index 9d908bae50..77579e63be 100644 --- a/src/common/memarea.c +++ b/src/common/memarea.c @@ -11,7 +11,11 @@ #include "memarea.h" #include "util.h" #include "compat.h" -#include "log.h" +#include "torlog.h" + +/** If true, we try to detect any attempts to write beyond the length of a + * memarea. */ +#define USE_SENTINELS /** All returned pointers should be aligned to the nearest multiple of this * value. */ @@ -25,6 +29,24 @@ #error "void* is neither 4 nor 8 bytes long. I don't know how to align stuff." #endif +#ifdef USE_SENTINELS +#define SENTINEL_VAL 0x90806622u +#define SENTINEL_LEN sizeof(uint32_t) +#define SET_SENTINEL(chunk) \ + STMT_BEGIN \ + set_uint32( &(chunk)->u.mem[chunk->mem_size], SENTINEL_VAL ); \ + STMT_END +#define CHECK_SENTINEL(chunk) \ + STMT_BEGIN \ + uint32_t sent_val = get_uint32(&(chunk)->u.mem[chunk->mem_size]); \ + tor_assert(sent_val == SENTINEL_VAL); \ + STMT_END +#else +#define SENTINEL_LEN 0 +#define SET_SENTINEL(chunk) STMT_NIL +#define CHECK_SENTINEL(chunk) STMT_NIL +#endif + /** Increment <b>ptr</b> until it is aligned to MEMAREA_ALIGN. */ static INLINE void * realign_pointer(void *ptr) @@ -78,15 +100,20 @@ alloc_chunk(size_t sz, int freelist_ok) freelist = res->next_chunk; res->next_chunk = NULL; --freelist_len; + CHECK_SENTINEL(res); return res; } else { size_t chunk_size = freelist_ok ? CHUNK_SIZE : sz; - memarea_chunk_t *res = tor_malloc_roundup(&chunk_size); + memarea_chunk_t *res; + chunk_size += SENTINEL_LEN; + res = tor_malloc_roundup(&chunk_size); res->next_chunk = NULL; - res->mem_size = chunk_size - CHUNK_HEADER_SIZE; + res->mem_size = chunk_size - CHUNK_HEADER_SIZE - SENTINEL_LEN; res->next_mem = res->u.mem; - tor_assert(res->next_mem+res->mem_size == ((char*)res)+chunk_size); + tor_assert(res->next_mem+res->mem_size+SENTINEL_LEN == + ((char*)res)+chunk_size); tor_assert(realign_pointer(res->next_mem) == res->next_mem); + SET_SENTINEL(res); return res; } } @@ -94,8 +121,9 @@ alloc_chunk(size_t sz, int freelist_ok) /** Release <b>chunk</b> from a memarea, either by adding it to the freelist * or by freeing it if the freelist is already too big. */ static void -chunk_free(memarea_chunk_t *chunk) +chunk_free_unchecked(memarea_chunk_t *chunk) { + CHECK_SENTINEL(chunk); if (freelist_len < MAX_FREELIST_LEN) { ++freelist_len; chunk->next_chunk = freelist; @@ -123,7 +151,7 @@ memarea_drop_all(memarea_t *area) memarea_chunk_t *chunk, *next; for (chunk = area->first; chunk; chunk = next) { next = chunk->next_chunk; - chunk_free(chunk); + chunk_free_unchecked(chunk); } area->first = NULL; /*fail fast on */ tor_free(area); @@ -139,7 +167,7 @@ memarea_clear(memarea_t *area) if (area->first->next_chunk) { for (chunk = area->first->next_chunk; chunk; chunk = next) { next = chunk->next_chunk; - chunk_free(chunk); + chunk_free_unchecked(chunk); } area->first->next_chunk = NULL; } @@ -182,6 +210,7 @@ memarea_alloc(memarea_t *area, size_t sz) memarea_chunk_t *chunk = area->first; char *result; tor_assert(chunk); + CHECK_SENTINEL(chunk); if (sz == 0) sz = 1; if (chunk->next_mem+sz > chunk->u.mem+chunk->mem_size) { @@ -258,6 +287,7 @@ memarea_get_stats(memarea_t *area, size_t *allocated_out, size_t *used_out) size_t a = 0, u = 0; memarea_chunk_t *chunk; for (chunk = area->first; chunk; chunk = chunk->next_chunk) { + CHECK_SENTINEL(chunk); a += CHUNK_HEADER_SIZE + chunk->mem_size; tor_assert(chunk->next_mem >= chunk->u.mem); u += CHUNK_HEADER_SIZE + (chunk->next_mem - chunk->u.mem); @@ -274,6 +304,7 @@ memarea_assert_ok(memarea_t *area) tor_assert(area->first); for (chunk = area->first; chunk; chunk = chunk->next_chunk) { + CHECK_SENTINEL(chunk); tor_assert(chunk->next_mem >= chunk->u.mem); tor_assert(chunk->next_mem <= (char*) realign_pointer(chunk->u.mem+chunk->mem_size)); diff --git a/src/common/mempool.c b/src/common/mempool.c index 256388a9fd..c795d83f0c 100644 --- a/src/common/mempool.c +++ b/src/common/mempool.c @@ -65,7 +65,7 @@ #include "orconfig.h" #include "util.h" #include "compat.h" -#include "log.h" +#include "torlog.h" #define ALLOC(x) tor_malloc(x) #define FREE(x) tor_free(x) #define ASSERT(x) tor_assert(x) diff --git a/src/common/sha256.c b/src/common/sha256.c new file mode 100644 index 0000000000..4236d48f5f --- /dev/null +++ b/src/common/sha256.c @@ -0,0 +1,331 @@ +/* Copyright (c) 2009-2010, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ +/* This SHA256 implementation is adapted from the public domain one in + LibTomCrypt, version 1.6. Tor uses it on platforms where OpenSSL doesn't + have a SHA256. */ + + +typedef struct sha256_state { + uint64_t length; + uint32_t state[8], curlen; + unsigned char buf[64]; +} sha256_state; + +#define CRYPT_OK 0 +#define CRYPT_NOP -1 +#define CRYPT_INVALID_ARG -2 + +#define LOAD32H(x,y) STMT_BEGIN x = ntohl(get_uint32((const char*)y)); STMT_END +#define STORE32H(x,y) STMT_BEGIN set_uint32((char*)y, htonl(x)); STMT_END +#define STORE64H(x,y) STMT_BEGIN \ + set_uint32((char*)y, htonl((uint32_t)((x)>>32))); \ + set_uint32(((char*)y)+4, htonl((uint32_t)((x)&0xffffffff))); \ + STMT_END +#define RORc(x, y) ( ((((unsigned long)(x)&0xFFFFFFFFUL)>>(unsigned long)((y)&31)) | ((unsigned long)(x)<<(unsigned long)(32-((y)&31)))) & 0xFFFFFFFFUL) +#ifndef MIN + #define MIN(x, y) ( ((x)<(y))?(x):(y) ) +#endif + + +/* LibTomCrypt, modular cryptographic library -- Tom St Denis + * + * LibTomCrypt is a library that provides various cryptographic + * algorithms in a highly modular and flexible manner. + * + * The library is free for all purposes without any express + * guarantee it works. + * + * Tom St Denis, tomstdenis@gmail.com, http://libtomcrypt.com + */ + +/** + @file sha256.c + SHA256 by Tom St Denis +*/ + + +#ifdef LTC_SMALL_CODE +/* the K array */ +static const uint32_t K[64] = { + 0x428a2f98UL, 0x71374491UL, 0xb5c0fbcfUL, 0xe9b5dba5UL, 0x3956c25bUL, + 0x59f111f1UL, 0x923f82a4UL, 0xab1c5ed5UL, 0xd807aa98UL, 0x12835b01UL, + 0x243185beUL, 0x550c7dc3UL, 0x72be5d74UL, 0x80deb1feUL, 0x9bdc06a7UL, + 0xc19bf174UL, 0xe49b69c1UL, 0xefbe4786UL, 0x0fc19dc6UL, 0x240ca1ccUL, + 0x2de92c6fUL, 0x4a7484aaUL, 0x5cb0a9dcUL, 0x76f988daUL, 0x983e5152UL, + 0xa831c66dUL, 0xb00327c8UL, 0xbf597fc7UL, 0xc6e00bf3UL, 0xd5a79147UL, + 0x06ca6351UL, 0x14292967UL, 0x27b70a85UL, 0x2e1b2138UL, 0x4d2c6dfcUL, + 0x53380d13UL, 0x650a7354UL, 0x766a0abbUL, 0x81c2c92eUL, 0x92722c85UL, + 0xa2bfe8a1UL, 0xa81a664bUL, 0xc24b8b70UL, 0xc76c51a3UL, 0xd192e819UL, + 0xd6990624UL, 0xf40e3585UL, 0x106aa070UL, 0x19a4c116UL, 0x1e376c08UL, + 0x2748774cUL, 0x34b0bcb5UL, 0x391c0cb3UL, 0x4ed8aa4aUL, 0x5b9cca4fUL, + 0x682e6ff3UL, 0x748f82eeUL, 0x78a5636fUL, 0x84c87814UL, 0x8cc70208UL, + 0x90befffaUL, 0xa4506cebUL, 0xbef9a3f7UL, 0xc67178f2UL +}; +#endif + +/* Various logical functions */ +#define Ch(x,y,z) (z ^ (x & (y ^ z))) +#define Maj(x,y,z) (((x | y) & z) | (x & y)) +#define S(x, n) RORc((x),(n)) +#define R(x, n) (((x)&0xFFFFFFFFUL)>>(n)) +#define Sigma0(x) (S(x, 2) ^ S(x, 13) ^ S(x, 22)) +#define Sigma1(x) (S(x, 6) ^ S(x, 11) ^ S(x, 25)) +#define Gamma0(x) (S(x, 7) ^ S(x, 18) ^ R(x, 3)) +#define Gamma1(x) (S(x, 17) ^ S(x, 19) ^ R(x, 10)) + +/* compress 512-bits */ +#ifdef LTC_CLEAN_STACK +static int _sha256_compress(sha256_state * md, unsigned char *buf) +#else +static int sha256_compress(sha256_state * md, unsigned char *buf) +#endif +{ + uint32_t S[8], W[64], t0, t1; +#ifdef LTC_SMALL_CODE + uint32_t t; +#endif + int i; + + /* copy state into S */ + for (i = 0; i < 8; i++) { + S[i] = md->state[i]; + } + + /* copy the state into 512-bits into W[0..15] */ + for (i = 0; i < 16; i++) { + LOAD32H(W[i], buf + (4*i)); + } + + /* fill W[16..63] */ + for (i = 16; i < 64; i++) { + W[i] = Gamma1(W[i - 2]) + W[i - 7] + Gamma0(W[i - 15]) + W[i - 16]; + } + + /* Compress */ +#ifdef LTC_SMALL_CODE +#define RND(a,b,c,d,e,f,g,h,i) \ + t0 = h + Sigma1(e) + Ch(e, f, g) + K[i] + W[i]; \ + t1 = Sigma0(a) + Maj(a, b, c); \ + d += t0; \ + h = t0 + t1; + + for (i = 0; i < 64; ++i) { + RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],i); + t = S[7]; S[7] = S[6]; S[6] = S[5]; S[5] = S[4]; + S[4] = S[3]; S[3] = S[2]; S[2] = S[1]; S[1] = S[0]; S[0] = t; + } +#else +#define RND(a,b,c,d,e,f,g,h,i,ki) \ + t0 = h + Sigma1(e) + Ch(e, f, g) + ki + W[i]; \ + t1 = Sigma0(a) + Maj(a, b, c); \ + d += t0; \ + h = t0 + t1; + + RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],0,0x428a2f98); + RND(S[7],S[0],S[1],S[2],S[3],S[4],S[5],S[6],1,0x71374491); + RND(S[6],S[7],S[0],S[1],S[2],S[3],S[4],S[5],2,0xb5c0fbcf); + RND(S[5],S[6],S[7],S[0],S[1],S[2],S[3],S[4],3,0xe9b5dba5); + RND(S[4],S[5],S[6],S[7],S[0],S[1],S[2],S[3],4,0x3956c25b); + RND(S[3],S[4],S[5],S[6],S[7],S[0],S[1],S[2],5,0x59f111f1); + RND(S[2],S[3],S[4],S[5],S[6],S[7],S[0],S[1],6,0x923f82a4); + RND(S[1],S[2],S[3],S[4],S[5],S[6],S[7],S[0],7,0xab1c5ed5); + RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],8,0xd807aa98); + RND(S[7],S[0],S[1],S[2],S[3],S[4],S[5],S[6],9,0x12835b01); + RND(S[6],S[7],S[0],S[1],S[2],S[3],S[4],S[5],10,0x243185be); + RND(S[5],S[6],S[7],S[0],S[1],S[2],S[3],S[4],11,0x550c7dc3); + RND(S[4],S[5],S[6],S[7],S[0],S[1],S[2],S[3],12,0x72be5d74); + RND(S[3],S[4],S[5],S[6],S[7],S[0],S[1],S[2],13,0x80deb1fe); + RND(S[2],S[3],S[4],S[5],S[6],S[7],S[0],S[1],14,0x9bdc06a7); + RND(S[1],S[2],S[3],S[4],S[5],S[6],S[7],S[0],15,0xc19bf174); + RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],16,0xe49b69c1); + RND(S[7],S[0],S[1],S[2],S[3],S[4],S[5],S[6],17,0xefbe4786); + RND(S[6],S[7],S[0],S[1],S[2],S[3],S[4],S[5],18,0x0fc19dc6); + RND(S[5],S[6],S[7],S[0],S[1],S[2],S[3],S[4],19,0x240ca1cc); + RND(S[4],S[5],S[6],S[7],S[0],S[1],S[2],S[3],20,0x2de92c6f); + RND(S[3],S[4],S[5],S[6],S[7],S[0],S[1],S[2],21,0x4a7484aa); + RND(S[2],S[3],S[4],S[5],S[6],S[7],S[0],S[1],22,0x5cb0a9dc); + RND(S[1],S[2],S[3],S[4],S[5],S[6],S[7],S[0],23,0x76f988da); + RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],24,0x983e5152); + RND(S[7],S[0],S[1],S[2],S[3],S[4],S[5],S[6],25,0xa831c66d); + RND(S[6],S[7],S[0],S[1],S[2],S[3],S[4],S[5],26,0xb00327c8); + RND(S[5],S[6],S[7],S[0],S[1],S[2],S[3],S[4],27,0xbf597fc7); + RND(S[4],S[5],S[6],S[7],S[0],S[1],S[2],S[3],28,0xc6e00bf3); + RND(S[3],S[4],S[5],S[6],S[7],S[0],S[1],S[2],29,0xd5a79147); + RND(S[2],S[3],S[4],S[5],S[6],S[7],S[0],S[1],30,0x06ca6351); + RND(S[1],S[2],S[3],S[4],S[5],S[6],S[7],S[0],31,0x14292967); + RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],32,0x27b70a85); + RND(S[7],S[0],S[1],S[2],S[3],S[4],S[5],S[6],33,0x2e1b2138); + RND(S[6],S[7],S[0],S[1],S[2],S[3],S[4],S[5],34,0x4d2c6dfc); + RND(S[5],S[6],S[7],S[0],S[1],S[2],S[3],S[4],35,0x53380d13); + RND(S[4],S[5],S[6],S[7],S[0],S[1],S[2],S[3],36,0x650a7354); + RND(S[3],S[4],S[5],S[6],S[7],S[0],S[1],S[2],37,0x766a0abb); + RND(S[2],S[3],S[4],S[5],S[6],S[7],S[0],S[1],38,0x81c2c92e); + RND(S[1],S[2],S[3],S[4],S[5],S[6],S[7],S[0],39,0x92722c85); + RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],40,0xa2bfe8a1); + RND(S[7],S[0],S[1],S[2],S[3],S[4],S[5],S[6],41,0xa81a664b); + RND(S[6],S[7],S[0],S[1],S[2],S[3],S[4],S[5],42,0xc24b8b70); + RND(S[5],S[6],S[7],S[0],S[1],S[2],S[3],S[4],43,0xc76c51a3); + RND(S[4],S[5],S[6],S[7],S[0],S[1],S[2],S[3],44,0xd192e819); + RND(S[3],S[4],S[5],S[6],S[7],S[0],S[1],S[2],45,0xd6990624); + RND(S[2],S[3],S[4],S[5],S[6],S[7],S[0],S[1],46,0xf40e3585); + RND(S[1],S[2],S[3],S[4],S[5],S[6],S[7],S[0],47,0x106aa070); + RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],48,0x19a4c116); + RND(S[7],S[0],S[1],S[2],S[3],S[4],S[5],S[6],49,0x1e376c08); + RND(S[6],S[7],S[0],S[1],S[2],S[3],S[4],S[5],50,0x2748774c); + RND(S[5],S[6],S[7],S[0],S[1],S[2],S[3],S[4],51,0x34b0bcb5); + RND(S[4],S[5],S[6],S[7],S[0],S[1],S[2],S[3],52,0x391c0cb3); + RND(S[3],S[4],S[5],S[6],S[7],S[0],S[1],S[2],53,0x4ed8aa4a); + RND(S[2],S[3],S[4],S[5],S[6],S[7],S[0],S[1],54,0x5b9cca4f); + RND(S[1],S[2],S[3],S[4],S[5],S[6],S[7],S[0],55,0x682e6ff3); + RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],56,0x748f82ee); + RND(S[7],S[0],S[1],S[2],S[3],S[4],S[5],S[6],57,0x78a5636f); + RND(S[6],S[7],S[0],S[1],S[2],S[3],S[4],S[5],58,0x84c87814); + RND(S[5],S[6],S[7],S[0],S[1],S[2],S[3],S[4],59,0x8cc70208); + RND(S[4],S[5],S[6],S[7],S[0],S[1],S[2],S[3],60,0x90befffa); + RND(S[3],S[4],S[5],S[6],S[7],S[0],S[1],S[2],61,0xa4506ceb); + RND(S[2],S[3],S[4],S[5],S[6],S[7],S[0],S[1],62,0xbef9a3f7); + RND(S[1],S[2],S[3],S[4],S[5],S[6],S[7],S[0],63,0xc67178f2); + +#undef RND + +#endif + + /* feedback */ + for (i = 0; i < 8; i++) { + md->state[i] = md->state[i] + S[i]; + } + return CRYPT_OK; +} + +#ifdef LTC_CLEAN_STACK +static int sha256_compress(sha256_state * md, unsigned char *buf) +{ + int err; + err = _sha256_compress(md, buf); + burn_stack(sizeof(uint32_t) * 74); + return err; +} +#endif + +/** + Initialize the hash state + @param md The hash state you wish to initialize + @return CRYPT_OK if successful +*/ +static int sha256_init(sha256_state * md) +{ + LTC_ARGCHK(md != NULL); + + md->curlen = 0; + md->length = 0; + md->state[0] = 0x6A09E667UL; + md->state[1] = 0xBB67AE85UL; + md->state[2] = 0x3C6EF372UL; + md->state[3] = 0xA54FF53AUL; + md->state[4] = 0x510E527FUL; + md->state[5] = 0x9B05688CUL; + md->state[6] = 0x1F83D9ABUL; + md->state[7] = 0x5BE0CD19UL; + return CRYPT_OK; +} + +/** + Process a block of memory though the hash + @param md The hash state + @param in The data to hash + @param inlen The length of the data (octets) + @return CRYPT_OK if successful +*/ +static int sha256_process (sha256_state * md, const unsigned char *in, unsigned long inlen) +{ + unsigned long n; + int err; + LTC_ARGCHK(md != NULL); + LTC_ARGCHK(in != NULL); + if (md->curlen > sizeof(md->buf)) { + return CRYPT_INVALID_ARG; + } + while (inlen > 0) { + if (md->curlen == 0 && inlen >= 64) { + if ((err = sha256_compress (md, (unsigned char *)in)) != CRYPT_OK) { + return err; + } + md->length += 64 * 8; + in += 64; + inlen -= 64; + } else { + n = MIN(inlen, (64 - md->curlen)); + memcpy(md->buf + md->curlen, in, (size_t)n); + md->curlen += n; + in += n; + inlen -= n; + if (md->curlen == 64) { + if ((err = sha256_compress (md, md->buf)) != CRYPT_OK) { + return err; + } + md->length += 8*64; + md->curlen = 0; + } + } + } + return CRYPT_OK; +} + +/** + Terminate the hash to get the digest + @param md The hash state + @param out [out] The destination of the hash (32 bytes) + @return CRYPT_OK if successful +*/ +static int sha256_done(sha256_state * md, unsigned char *out) +{ + int i; + + LTC_ARGCHK(md != NULL); + LTC_ARGCHK(out != NULL); + + if (md->curlen >= sizeof(md->buf)) { + return CRYPT_INVALID_ARG; + } + + + /* increase the length of the message */ + md->length += md->curlen * 8; + + /* append the '1' bit */ + md->buf[md->curlen++] = (unsigned char)0x80; + + /* if the length is currently above 56 bytes we append zeros + * then compress. Then we can fall back to padding zeros and length + * encoding like normal. + */ + if (md->curlen > 56) { + while (md->curlen < 64) { + md->buf[md->curlen++] = (unsigned char)0; + } + sha256_compress(md, md->buf); + md->curlen = 0; + } + + /* pad upto 56 bytes of zeroes */ + while (md->curlen < 56) { + md->buf[md->curlen++] = (unsigned char)0; + } + + /* store length */ + STORE64H(md->length, md->buf+56); + sha256_compress(md, md->buf); + + /* copy output */ + for (i = 0; i < 8; i++) { + STORE32H(md->state[i], out+(4*i)); + } +#ifdef LTC_CLEAN_STACK + zeromem(md, sizeof(sha256_state)); +#endif + return CRYPT_OK; +} + +/* $Source: /cvs/libtom/libtomcrypt/src/hashes/sha2/sha256.c,v $ */ +/* $Revision: 1.9 $ */ +/* $Date: 2006/11/01 09:28:17 $ */ diff --git a/src/common/test.h b/src/common/test.h deleted file mode 100644 index 1e3db58333..0000000000 --- a/src/common/test.h +++ /dev/null @@ -1,184 +0,0 @@ -/* Copyright (c) 2001-2003, Roger Dingledine. - * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2010, The Tor Project, Inc. */ -/* See LICENSE for licensing information */ - -#ifndef _TOR_TEST_H -#define _TOR_TEST_H - -/** - * \file test.h - * \brief Macros used by unit tests. - */ - -#include "compat.h" - -#ifdef __GNUC__ -#define PRETTY_FUNCTION __PRETTY_FUNCTION__ -#else -#define PRETTY_FUNCTION "" -#endif - -#define test_fail_msg(msg) \ - STMT_BEGIN \ - have_failed = 1; \ - printf("\nFile %s: line %d (%s): %s", \ - _SHORT_FILE_, \ - __LINE__, \ - PRETTY_FUNCTION, \ - msg); \ - goto done; \ - STMT_END - -#define test_fail() test_fail_msg("Assertion failed.") - -#define test_assert(expr) \ - STMT_BEGIN \ - if (expr) { printf("."); fflush(stdout); } else { \ - have_failed = 1; \ - printf("\nFile %s: line %d (%s): assertion failed: (%s)\n", \ - _SHORT_FILE_, \ - __LINE__, \ - PRETTY_FUNCTION, \ - #expr); \ - goto done; \ - } STMT_END - -#define test_eq_type(tp, fmt, expr1, expr2) \ - STMT_BEGIN \ - tp _test_v1=(tp)(expr1); \ - tp _test_v2=(tp)(expr2); \ - if (_test_v1==_test_v2) { printf("."); fflush(stdout); } else { \ - have_failed = 1; \ - printf("\nFile %s: line %d (%s): Assertion failed: (%s==%s)\n" \ - " "fmt "!="fmt"\n", \ - _SHORT_FILE_, \ - __LINE__, \ - PRETTY_FUNCTION, \ - #expr1, #expr2, \ - _test_v1, _test_v2); \ - goto done; \ - } STMT_END - -#define test_eq(expr1, expr2) \ - test_eq_type(long, "%ld", expr1, expr2) - -#define test_eq_ptr(expr1, expr2) \ - test_eq_type(void*, "%p", expr1, expr2) - -#define test_neq_type(tp, fmt, expr1, expr2) \ - STMT_BEGIN \ - tp _test_v1=(tp)(expr1); \ - tp _test_v2=(tp)(expr2); \ - if (_test_v1!=_test_v2) { printf("."); fflush(stdout); } else { \ - have_failed = 1; \ - printf("\nFile %s: line %d (%s): Assertion failed: (%s!=%s)\n" \ - " ("fmt" == "fmt")\n", \ - _SHORT_FILE_, \ - __LINE__, \ - PRETTY_FUNCTION, \ - #expr1, #expr2, \ - _test_v1, _test_v2); \ - goto done; \ - } STMT_END - -#define test_neq(expr1, expr2) \ - test_neq_type(long, "%ld", expr1, expr2) - -#define test_neq_ptr(expr1, expr2) \ - test_neq_type(void *, "%p", expr1, expr2) - -#define test_streq(expr1, expr2) \ - STMT_BEGIN \ - const char *_test_v1=(expr1), *_test_v2=(expr2); \ - if (!strcmp(_test_v1,_test_v2)) { printf("."); fflush(stdout); } else { \ - have_failed = 1; \ - printf("\nFile %s: line %d (%s): Assertion failed: (%s==%s)\n"\ - " (\"%s\" != \"%s\")\n", \ - _SHORT_FILE_, \ - __LINE__, \ - PRETTY_FUNCTION, \ - #expr1, #expr2, \ - _test_v1, _test_v2); \ - goto done; \ - } STMT_END - -#define test_strneq(expr1, expr2) \ - STMT_BEGIN \ - const char *_test_v1=(expr1), *_test_v2=(expr2); \ - if (strcmp(_test_v1,_test_v2)) { printf("."); fflush(stdout); } else { \ - have_failed = 1; \ - printf("\nFile %s: line %d (%s): Assertion failed: (%s!=%s)\n"\ - " (\"%s\" == \"%s\")\n", \ - _SHORT_FILE_, \ - __LINE__, \ - PRETTY_FUNCTION, \ - #expr1, #expr2, \ - _test_v1, _test_v2); \ - goto done; \ - } STMT_END - -#define test_memeq(expr1, expr2, len) \ - STMT_BEGIN \ - const void *_test_v1=(expr1), *_test_v2=(expr2); \ - char *mem1, *mem2; \ - if (!memcmp(_test_v1,_test_v2,(len))) { \ - printf("."); fflush(stdout); } else { \ - have_failed = 1; \ - mem1 = tor_malloc(len*2+1); \ - mem2 = tor_malloc(len*2+1); \ - base16_encode(mem1, len*2+1, _test_v1, len); \ - base16_encode(mem2, len*2+1, _test_v2, len); \ - printf("\nFile %s: line %d (%s): Assertion failed: (%s==%s)\n" \ - " %s != %s\n", \ - _SHORT_FILE_, \ - __LINE__, \ - PRETTY_FUNCTION, \ - #expr1, #expr2, mem1, mem2); \ - tor_free(mem1); \ - tor_free(mem2); \ - goto done; \ - } STMT_END - -#define test_memeq_hex(expr1, hex) \ - STMT_BEGIN \ - const char *_test_v1 = (char*)(expr1); \ - const char *_test_v2 = (hex); \ - size_t _len_v2 = strlen(_test_v2); \ - char *_mem2 = tor_malloc(_len_v2/2); \ - tor_assert((_len_v2 & 1) == 0); \ - base16_decode(_mem2, _len_v2/2, _test_v2, _len_v2); \ - if (!memcmp(_mem2, _test_v1, _len_v2/2)) { \ - printf("."); fflush(stdout); } else { \ - char *_mem1 = tor_malloc(_len_v2+1); \ - base16_encode(_mem1, _len_v2+1, _test_v1, _len_v2/2); \ - printf("\nFile %s: line %d (%s): Assertion failed: (%s==%s)\n" \ - " %s != %s\n", \ - _SHORT_FILE_, \ - __LINE__, \ - PRETTY_FUNCTION, \ - #expr1, _test_v2, _mem1, _test_v2); \ - tor_free(_mem1); \ - tor_free(_mem2); \ - goto done; \ - } \ - tor_free(_mem2); \ - STMT_END - -#define test_memneq(expr1, expr2, len) \ - STMT_BEGIN \ - void *_test_v1=(expr1), *_test_v2=(expr2); \ - if (memcmp(_test_v1,_test_v2,(len))) { \ - printf("."); fflush(stdout); \ - } else { \ - have_failed = 1; \ - printf("\nFile %s: line %d (%s): Assertion failed: (%s!=%s)\n", \ - _SHORT_FILE_, \ - __LINE__, \ - PRETTY_FUNCTION, \ - #expr1, #expr2); \ - goto done; \ - } STMT_END - -#endif - diff --git a/src/common/torgzip.c b/src/common/torgzip.c index 10bce33283..a247d6c177 100644 --- a/src/common/torgzip.c +++ b/src/common/torgzip.c @@ -13,20 +13,42 @@ #include <stdlib.h> #include <stdio.h> #include <assert.h> -#ifdef _MSC_VER -#include "..\..\contrib\zlib\zlib.h" -#else -#include <zlib.h> -#endif #include <string.h> #ifdef HAVE_NETINET_IN_H #include <netinet/in.h> #endif +#include "torint.h" #include "util.h" -#include "log.h" +#include "torlog.h" #include "torgzip.h" +/* zlib 1.2.4 and 1.2.5 do some "clever" things with macros. Instead of + saying "(defined(FOO) ? FOO : 0)" they like to say "FOO-0", on the theory + that nobody will care if the compile outputs a no-such-identifier warning. + + Sorry, but we like -Werror over here, so I guess we need to define these. + I hope that zlib 1.2.6 doesn't break these too. +*/ +#ifndef _LARGEFILE64_SOURCE +#define _LARGEFILE64_SOURCE 0 +#endif +#ifndef _LFS64_LARGEFILE +#define _LFS64_LARGEFILE 0 +#endif +#ifndef _FILE_OFFSET_BITS +#define _FILE_OFFSET_BITS 0 +#endif +#ifndef off64_t +#define off64_t int64_t +#endif + +#ifdef _MSC_VER +#include "..\..\contrib\zlib\zlib.h" +#else +#include <zlib.h> +#endif + /** Set to 1 if zlib is a version that supports gzip; set to 0 if it doesn't; * set to -1 if we haven't checked yet. */ static int gzip_is_supported = -1; @@ -165,9 +187,7 @@ tor_gzip_compress(char **out, size_t *out_len, deflateEnd(stream); tor_free(stream); } - if (*out) { - tor_free(*out); - } + tor_free(*out); return -1; } @@ -423,7 +443,8 @@ tor_zlib_process(tor_zlib_state_t *state, void tor_zlib_free(tor_zlib_state_t *state) { - tor_assert(state); + if (!state) + return; if (state->compress) deflateEnd(&state->stream); diff --git a/src/common/torint.h b/src/common/torint.h index d9722868d2..57f18212ad 100644 --- a/src/common/torint.h +++ b/src/common/torint.h @@ -117,11 +117,10 @@ typedef unsigned int uint32_t; #ifndef INT32_MAX #define INT32_MAX 0x7fffffffu #endif -#endif - #ifndef INT32_MIN #define INT32_MIN (-2147483647-1) #endif +#endif #if (SIZEOF_LONG == 4) #ifndef HAVE_INT32_T diff --git a/src/common/log.h b/src/common/torlog.h index 6a45fc3d0b..21219569e3 100644 --- a/src/common/log.h +++ b/src/common/torlog.h @@ -90,9 +90,10 @@ #define LD_ACCT (1u<<17) /** Router history */ #define LD_HIST (1u<<18) - +/** OR handshaking */ +#define LD_HANDSHAKE (1u<<19) /** Number of logging domains in the code. */ -#define N_LOGGING_DOMAINS 19 +#define N_LOGGING_DOMAINS 20 typedef uint32_t log_domain_mask_t; @@ -134,16 +135,14 @@ void add_temp_log(int min_severity); void close_temp_logs(void); void rollback_log_changes(void); void mark_logs_temp(void); -void configure_libevent_logging(void); -void suppress_libevent_log_msg(const char *msg); void change_callback_log_severity(int loglevelMin, int loglevelMax, log_callback cb); void log_set_application_name(const char *name); /* Outputs a message to stdout */ -void _log(int severity, log_domain_mask_t domain, const char *format, ...) +void tor_log(int severity, log_domain_mask_t domain, const char *format, ...) CHECK_PRINTF(3,4); -#define log _log /* hack it so we don't conflict with log() as much */ +#define log tor_log /* hack it so we don't conflict with log() as much */ #ifdef __GNUC__ extern int _log_global_min_severity; diff --git a/src/common/tortls.c b/src/common/tortls.c index 25f21a9892..7dfdca6af9 100644 --- a/src/common/tortls.c +++ b/src/common/tortls.c @@ -16,7 +16,22 @@ #include "orconfig.h" +#if defined (WINCE) +#include <WinSock2.h> +#endif + #include <assert.h> +#ifdef MS_WINDOWS /*wrkard for dtls1.h >= 0.9.8m of "#include <winsock.h>"*/ + #define WIN32_WINNT 0x400 + #define _WIN32_WINNT 0x400 + #define WIN32_LEAN_AND_MEAN + #if defined(_MSC_VER) && (_MSC_VER < 1300) + #include <winsock.h> + #else + #include <winsock2.h> + #include <ws2tcpip.h> + #endif +#endif #include <openssl/ssl.h> #include <openssl/ssl3.h> #include <openssl/err.h> @@ -34,7 +49,7 @@ #include "crypto.h" #include "tortls.h" #include "util.h" -#include "log.h" +#include "torlog.h" #include "container.h" #include "ht.h" #include <string.h> @@ -184,30 +199,51 @@ static int tls_library_is_initialized = 0; #define _TOR_TLS_SYSCALL (_MIN_TOR_TLS_ERROR_VAL - 2) #define _TOR_TLS_ZERORETURN (_MIN_TOR_TLS_ERROR_VAL - 1) +#include "tortls_states.h" + +/** Return the symbolic name of an OpenSSL state. */ +static const char * +ssl_state_to_string(int ssl_state) +{ + static char buf[40]; + int i; + for (i = 0; state_map[i].name; ++i) { + if (state_map[i].state == ssl_state) + return state_map[i].name; + } + tor_snprintf(buf, sizeof(buf), "Unknown state %d", ssl_state); + return buf; +} + /** Log all pending tls errors at level <b>severity</b>. Use * <b>doing</b> to describe our current activities. */ static void -tls_log_errors(tor_tls_t *tls, int severity, const char *doing) +tls_log_errors(tor_tls_t *tls, int severity, int domain, const char *doing) { + const char *state = NULL; + int st; unsigned long err; const char *msg, *lib, *func, *addr; addr = tls ? tls->address : NULL; + st = (tls && tls->ssl) ? tls->ssl->state : -1; while ((err = ERR_get_error()) != 0) { msg = (const char*)ERR_reason_error_string(err); lib = (const char*)ERR_lib_error_string(err); func = (const char*)ERR_func_error_string(err); + if (!state) + state = (st>=0)?ssl_state_to_string(st):"---"; if (!msg) msg = "(null)"; if (!lib) lib = "(null)"; if (!func) func = "(null)"; if (doing) { - log(severity, LD_NET, "TLS error while %s%s%s: %s (in %s:%s)", + log(severity, domain, "TLS error while %s%s%s: %s (in %s:%s:%s)", doing, addr?" with ":"", addr?addr:"", - msg, lib, func); + msg, lib, func, state); } else { - log(severity, LD_NET, "TLS error%s%s: %s (in %s:%s)", + log(severity, domain, "TLS error%s%s: %s (in %s:%s:%s)", addr?" with ":"", addr?addr:"", - msg, lib, func); + msg, lib, func, state); } } } @@ -283,7 +319,7 @@ tor_tls_err_to_string(int err) */ static int tor_tls_get_error(tor_tls_t *tls, int r, int extra, - const char *doing, int severity) + const char *doing, int severity, int domain) { int err = SSL_get_error(tls->ssl, r); int tor_error = TOR_TLS_ERROR_MISC; @@ -298,25 +334,28 @@ tor_tls_get_error(tor_tls_t *tls, int r, int extra, if (extra&CATCH_SYSCALL) return _TOR_TLS_SYSCALL; if (r == 0) { - log(severity, LD_NET, "TLS error: unexpected close while %s", doing); + log(severity, LD_NET, "TLS error: unexpected close while %s (%s)", + doing, ssl_state_to_string(tls->ssl->state)); tor_error = TOR_TLS_ERROR_IO; } else { int e = tor_socket_errno(tls->socket); log(severity, LD_NET, - "TLS error: <syscall error while %s> (errno=%d: %s)", - doing, e, tor_socket_strerror(e)); + "TLS error: <syscall error while %s> (errno=%d: %s; state=%s)", + doing, e, tor_socket_strerror(e), + ssl_state_to_string(tls->ssl->state)); tor_error = tor_errno_to_tls_error(e); } - tls_log_errors(tls, severity, doing); + tls_log_errors(tls, severity, domain, doing); return tor_error; case SSL_ERROR_ZERO_RETURN: if (extra&CATCH_ZERO) return _TOR_TLS_ZERORETURN; - log(severity, LD_NET, "TLS connection closed while %s", doing); - tls_log_errors(tls, severity, doing); + log(severity, LD_NET, "TLS connection closed while %s in state %s", + doing, ssl_state_to_string(tls->ssl->state)); + tls_log_errors(tls, severity, domain, doing); return TOR_TLS_CLOSE; default: - tls_log_errors(tls, severity, doing); + tls_log_errors(tls, severity, domain, doing); return TOR_TLS_ERROR_MISC; } } @@ -330,11 +369,10 @@ tor_tls_init(void) long version; SSL_library_init(); SSL_load_error_strings(); - crypto_global_init(-1); version = SSLeay(); - /* OpenSSL 0.9.8l introdeced SSL3_FLAGS_ALLOW_UNSAGE_LEGACY_RENEGOTIATION + /* OpenSSL 0.9.8l introduced SSL3_FLAGS_ALLOW_UNSAGE_LEGACY_RENEGOTIATION * here, but without thinking too hard about it: it turns out that the * flag in question needed to be set at the last minute, and that it * conflicted with an existing flag number that had already been added @@ -353,8 +391,8 @@ tor_tls_init(void) * leave their headers out of sync with their libraries. * * Yes, it _is_ almost as if the OpenSSL developers decided that no - * program should be allowed to use renegotiation its first passed an - * test of intelligence and determination. + * program should be allowed to use renegotiation unless it first passed + * a test of intelligence and determination. */ if (version >= 0x009080c0L && version < 0x009080d0L) { log_notice(LD_GENERAL, "OpenSSL %s looks like version 0.9.8l; " @@ -500,7 +538,7 @@ tor_tls_create_certificate(crypto_pk_env_t *rsa, x509 = NULL; } done: - tls_log_errors(NULL, LOG_WARN, "generating certificate"); + tls_log_errors(NULL, LOG_WARN, LD_NET, "generating certificate"); if (sign_pkey) EVP_PKEY_free(sign_pkey); if (pkey) @@ -695,7 +733,7 @@ tor_tls_context_new(crypto_pk_env_t *identity, unsigned int key_lifetime) return 0; error: - tls_log_errors(NULL, LOG_WARN, "creating TLS context"); + tls_log_errors(NULL, LOG_WARN, LD_NET, "creating TLS context"); tor_free(nickname); tor_free(nn2); if (pkey) @@ -723,11 +761,11 @@ tor_tls_client_is_using_v2_ciphers(const SSL *ssl, const char *address) /* If we reached this point, we just got a client hello. See if there is * a cipher list. */ if (!(session = SSL_get_session((SSL *)ssl))) { - log_warn(LD_NET, "No session on TLS?"); + log_info(LD_NET, "No session on TLS?"); return 0; } if (!session->ciphers) { - log_warn(LD_NET, "No ciphers on session"); + log_info(LD_NET, "No ciphers on session"); return 0; } /* Now we need to see if there are any ciphers whose presence means we're @@ -739,8 +777,7 @@ tor_tls_client_is_using_v2_ciphers(const SSL *ssl, const char *address) strcmp(ciphername, TLS1_TXT_DHE_RSA_WITH_AES_256_SHA) && strcmp(ciphername, SSL3_TXT_EDH_RSA_DES_192_CBC3_SHA) && strcmp(ciphername, "(NONE)")) { - /* XXXX should be ld_debug */ - log_info(LD_NET, "Got a non-version-1 cipher called '%s'", ciphername); + log_debug(LD_NET, "Got a non-version-1 cipher called '%s'", ciphername); // return 1; goto dump_list; } @@ -756,8 +793,8 @@ tor_tls_client_is_using_v2_ciphers(const SSL *ssl, const char *address) smartlist_add(elts, (char*)ciphername); } s = smartlist_join_strings(elts, ":", 0, NULL); - log_info(LD_NET, "Got a non-version-1 cipher list from %s. It is: '%s'", - address, s); + log_debug(LD_NET, "Got a non-version-1 cipher list from %s. It is: '%s'", + address, s); tor_free(s); smartlist_free(elts); } @@ -891,7 +928,7 @@ tor_tls_new(int sock, int isServer) tor_assert(global_tls_context); /* make sure somebody made it first */ if (!(result->ssl = SSL_new(global_tls_context->ctx))) { - tls_log_errors(NULL, LOG_WARN, "generating TLS context"); + tls_log_errors(NULL, LOG_WARN, LD_NET, "generating TLS context"); tor_free(result); return NULL; } @@ -907,7 +944,7 @@ tor_tls_new(int sock, int isServer) if (!SSL_set_cipher_list(result->ssl, isServer ? SERVER_CIPHER_LIST : CLIENT_CIPHER_LIST)) { - tls_log_errors(NULL, LOG_WARN, "setting ciphers"); + tls_log_errors(NULL, LOG_WARN, LD_NET, "setting ciphers"); #ifdef SSL_set_tlsext_host_name SSL_set_tlsext_host_name(result->ssl, NULL); #endif @@ -920,7 +957,7 @@ tor_tls_new(int sock, int isServer) result->socket = sock; bio = BIO_new_socket(sock, BIO_NOCLOSE); if (! bio) { - tls_log_errors(NULL, LOG_WARN, "opening BIO"); + tls_log_errors(NULL, LOG_WARN, LD_NET, "opening BIO"); #ifdef SSL_set_tlsext_host_name SSL_set_tlsext_host_name(result->ssl, NULL); #endif @@ -948,7 +985,7 @@ tor_tls_new(int sock, int isServer) #endif /* Not expected to get called. */ - tls_log_errors(NULL, LOG_WARN, "generating TLS context"); + tls_log_errors(NULL, LOG_WARN, LD_NET, "generating TLS context"); return result; } @@ -1027,7 +1064,9 @@ void tor_tls_free(tor_tls_t *tls) { tor_tls_t *removed; - tor_assert(tls && tls->ssl); + if (!tls) + return; + tor_assert(tls->ssl); removed = HT_REMOVE(tlsmap, &tlsmap_root, tls); if (!removed) { log_warn(LD_BUG, "Freeing a TLS that was not in the ssl->tls map."); @@ -1070,7 +1109,7 @@ tor_tls_read(tor_tls_t *tls, char *cp, size_t len) #endif return r; } - err = tor_tls_get_error(tls, r, CATCH_ZERO, "reading", LOG_DEBUG); + err = tor_tls_get_error(tls, r, CATCH_ZERO, "reading", LOG_DEBUG, LD_NET); if (err == _TOR_TLS_ZERORETURN || err == TOR_TLS_CLOSE) { log_debug(LD_NET,"read returned r=%d; TLS is closed",r); tls->state = TOR_TLS_ST_CLOSED; @@ -1106,7 +1145,7 @@ tor_tls_write(tor_tls_t *tls, const char *cp, size_t n) tls->wantwrite_n = 0; } r = SSL_write(tls->ssl, cp, (int)n); - err = tor_tls_get_error(tls, r, 0, "writing", LOG_INFO); + err = tor_tls_get_error(tls, r, 0, "writing", LOG_INFO, LD_NET); if (err == TOR_TLS_DONE) { return r; } @@ -1124,21 +1163,30 @@ int tor_tls_handshake(tor_tls_t *tls) { int r; + int oldstate; tor_assert(tls); tor_assert(tls->ssl); tor_assert(tls->state == TOR_TLS_ST_HANDSHAKE); check_no_tls_errors(); + oldstate = tls->ssl->state; if (tls->isServer) { + log_debug(LD_HANDSHAKE, "About to call SSL_accept on %p (%s)", tls, + ssl_state_to_string(tls->ssl->state)); r = SSL_accept(tls->ssl); } else { + log_debug(LD_HANDSHAKE, "About to call SSL_connect on %p (%s)", tls, + ssl_state_to_string(tls->ssl->state)); r = SSL_connect(tls->ssl); } + if (oldstate != tls->ssl->state) + log_debug(LD_HANDSHAKE, "After call, %p was in state %s", + tls, ssl_state_to_string(tls->ssl->state)); /* We need to call this here and not earlier, since OpenSSL has a penchant * for clearing its flags when you say accept or connect. */ tor_tls_unblock_renegotiation(tls); - r = tor_tls_get_error(tls,r,0, "handshaking", LOG_INFO); + r = tor_tls_get_error(tls,r,0, "handshaking", LOG_INFO, LD_HANDSHAKE); if (ERR_peek_error() != 0) { - tls_log_errors(tls, tls->isServer ? LOG_INFO : LOG_WARN, + tls_log_errors(tls, tls->isServer ? LOG_INFO : LOG_WARN, LD_HANDSHAKE, "handshaking"); return TOR_TLS_ERROR_MISC; } @@ -1159,7 +1207,8 @@ tor_tls_handshake(tor_tls_t *tls) " get set. Fixing that."); } tls->wasV2Handshake = 1; - log_debug(LD_NET, "Completed V2 TLS handshake with client; waiting " + log_debug(LD_HANDSHAKE, + "Completed V2 TLS handshake with client; waiting " "for renegotiation."); } else { tls->wasV2Handshake = 0; @@ -1171,10 +1220,13 @@ tor_tls_handshake(tor_tls_t *tls) X509 *cert = SSL_get_peer_certificate(tls->ssl); STACK_OF(X509) *chain = SSL_get_peer_cert_chain(tls->ssl); int n_certs = sk_X509_num(chain); - if (n_certs > 1 || (n_certs == 1 && cert != sk_X509_value(chain, 0))) + if (n_certs > 1 || (n_certs == 1 && cert != sk_X509_value(chain, 0))) { + log_debug(LD_HANDSHAKE, "Server sent back multiple certificates; it " + "looks like a v1 handshake on %p", tls); tls->wasV2Handshake = 0; - else { - log_debug(LD_NET, "Server sent back a single certificate; looks like " + } else { + log_debug(LD_HANDSHAKE, + "Server sent back a single certificate; looks like " "a v2 handshake on %p.", tls); tls->wasV2Handshake = 1; } @@ -1182,7 +1234,7 @@ tor_tls_handshake(tor_tls_t *tls) X509_free(cert); #endif if (SSL_set_cipher_list(tls->ssl, SERVER_CIPHER_LIST) == 0) { - tls_log_errors(NULL, LOG_WARN, "re-setting ciphers"); + tls_log_errors(NULL, LOG_WARN, LD_HANDSHAKE, "re-setting ciphers"); r = TOR_TLS_ERROR_MISC; } } @@ -1205,7 +1257,8 @@ tor_tls_renegotiate(tor_tls_t *tls) if (tls->state != TOR_TLS_ST_RENEGOTIATE) { int r = SSL_renegotiate(tls->ssl); if (r <= 0) { - return tor_tls_get_error(tls, r, 0, "renegotiating", LOG_WARN); + return tor_tls_get_error(tls, r, 0, "renegotiating", LOG_WARN, + LD_HANDSHAKE); } tls->state = TOR_TLS_ST_RENEGOTIATE; } @@ -1214,7 +1267,8 @@ tor_tls_renegotiate(tor_tls_t *tls) tls->state = TOR_TLS_ST_OPEN; return TOR_TLS_DONE; } else - return tor_tls_get_error(tls, r, 0, "renegotiating handshake", LOG_INFO); + return tor_tls_get_error(tls, r, 0, "renegotiating handshake", LOG_INFO, + LD_HANDSHAKE); } /** Shut down an open tls connection <b>tls</b>. When finished, returns @@ -1238,7 +1292,7 @@ tor_tls_shutdown(tor_tls_t *tls) r = SSL_read(tls->ssl, buf, 128); } while (r>0); err = tor_tls_get_error(tls, r, CATCH_ZERO, "reading to shut down", - LOG_INFO); + LOG_INFO, LD_NET); if (err == _TOR_TLS_ZERORETURN) { tls->state = TOR_TLS_ST_GOTCLOSE; /* fall through... */ @@ -1254,7 +1308,7 @@ tor_tls_shutdown(tor_tls_t *tls) return TOR_TLS_DONE; } err = tor_tls_get_error(tls, r, CATCH_SYSCALL|CATCH_ZERO, "shutting down", - LOG_INFO); + LOG_INFO, LD_NET); if (err == _TOR_TLS_SYSCALL) { /* The underlying TCP connection closed while we were shutting down. */ tls->state = TOR_TLS_ST_CLOSED; @@ -1286,7 +1340,7 @@ tor_tls_peer_has_cert(tor_tls_t *tls) { X509 *cert; cert = SSL_get_peer_certificate(tls->ssl); - tls_log_errors(tls, LOG_WARN, "getting peer certificate"); + tls_log_errors(tls, LOG_WARN, LD_HANDSHAKE, "getting peer certificate"); if (!cert) return 0; X509_free(cert); @@ -1313,7 +1367,7 @@ log_cert_lifetime(X509 *cert, const char *problem) log_warn(LD_GENERAL, "Couldn't allocate BIO!"); goto end; } if (!(ASN1_TIME_print(bio, X509_get_notBefore(cert)))) { - tls_log_errors(NULL, LOG_WARN, "printing certificate lifetime"); + tls_log_errors(NULL, LOG_WARN, LD_NET, "printing certificate lifetime"); goto end; } BIO_get_mem_ptr(bio, &buf); @@ -1321,7 +1375,7 @@ log_cert_lifetime(X509 *cert, const char *problem) (void)BIO_reset(bio); if (!(ASN1_TIME_print(bio, X509_get_notAfter(cert)))) { - tls_log_errors(NULL, LOG_WARN, "printing certificate lifetime"); + tls_log_errors(NULL, LOG_WARN, LD_NET, "printing certificate lifetime"); goto end; } BIO_get_mem_ptr(bio, &buf); @@ -1335,13 +1389,11 @@ log_cert_lifetime(X509 *cert, const char *problem) end: /* Not expected to get invoked */ - tls_log_errors(NULL, LOG_WARN, "getting certificate lifetime"); + tls_log_errors(NULL, LOG_WARN, LD_NET, "getting certificate lifetime"); if (bio) BIO_free(bio); - if (s1) - tor_free(s1); - if (s2) - tor_free(s2); + tor_free(s1); + tor_free(s2); } /** Helper function: try to extract a link certificate and an identity @@ -1409,7 +1461,7 @@ tor_tls_verify(int severity, tor_tls_t *tls, crypto_pk_env_t **identity_key) if (!(id_pkey = X509_get_pubkey(id_cert)) || X509_verify(cert, id_pkey) <= 0) { log_fn(severity,LD_PROTOCOL,"X509_verify on cert and pkey returned <= 0"); - tls_log_errors(tls, severity,"verifying certificate"); + tls_log_errors(tls, severity, LD_HANDSHAKE, "verifying certificate"); goto done; } @@ -1428,7 +1480,7 @@ tor_tls_verify(int severity, tor_tls_t *tls, crypto_pk_env_t **identity_key) /* This should never get invoked, but let's make sure in case OpenSSL * acts unexpectedly. */ - tls_log_errors(tls, LOG_WARN, "finishing tor_tls_verify"); + tls_log_errors(tls, LOG_WARN, LD_HANDSHAKE, "finishing tor_tls_verify"); return r; } @@ -1467,7 +1519,7 @@ tor_tls_check_lifetime(tor_tls_t *tls, int tolerance) if (cert) X509_free(cert); /* Not expected to get invoked */ - tls_log_errors(tls, LOG_WARN, "checking certificate lifetime"); + tls_log_errors(tls, LOG_WARN, LD_NET, "checking certificate lifetime"); return r; } @@ -1535,7 +1587,7 @@ _check_no_tls_errors(const char *fname, int line) return; log(LOG_WARN, LD_CRYPTO, "Unhandled OpenSSL errors found at %s:%d: ", tor_fix_source_file(fname), line); - tls_log_errors(NULL, LOG_WARN, NULL); + tls_log_errors(NULL, LOG_WARN, LD_NET, NULL); } /** Return true iff the initial TLS connection at <b>tls</b> did not use a v2 diff --git a/src/common/tortls_states.h b/src/common/tortls_states.h new file mode 100644 index 0000000000..00f476dd66 --- /dev/null +++ b/src/common/tortls_states.h @@ -0,0 +1,414 @@ +/* Copyright (c) 2003, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2010, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/* Helper file: included only in tortls.c */ + +#ifndef _TORTLS_STATES_H +#define _TORTLS_STATES_H + +/* The main body of this file was mechanically generated with this + perl script: + + my %keys = (); + for $fn (@ARGV) { + open(F, $fn); + while (<F>) { + next unless /^#define ((?:SSL|DTLS)\w*_ST_\w*)/; + $keys{$1} = 1; + } + close(F); + } + for $k (sort keys %keys) { + print "#ifdef $k\n S($k),\n#endif\n" + } +*/ + +/** Mapping from allowed value of SSL.state to the name of C macro for that + * state. Used for debugging an openssl connection. */ +static const struct { int state; const char *name; } state_map[] = { +#define S(state) { state, #state } +#ifdef DTLS1_ST_CR_HELLO_VERIFY_REQUEST_A + S(DTLS1_ST_CR_HELLO_VERIFY_REQUEST_A), +#endif +#ifdef DTLS1_ST_CR_HELLO_VERIFY_REQUEST_B + S(DTLS1_ST_CR_HELLO_VERIFY_REQUEST_B), +#endif +#ifdef DTLS1_ST_SW_HELLO_VERIFY_REQUEST_A + S(DTLS1_ST_SW_HELLO_VERIFY_REQUEST_A), +#endif +#ifdef DTLS1_ST_SW_HELLO_VERIFY_REQUEST_B + S(DTLS1_ST_SW_HELLO_VERIFY_REQUEST_B), +#endif +#ifdef SSL23_ST_CR_SRVR_HELLO_A + S(SSL23_ST_CR_SRVR_HELLO_A), +#endif +#ifdef SSL23_ST_CR_SRVR_HELLO_B + S(SSL23_ST_CR_SRVR_HELLO_B), +#endif +#ifdef SSL23_ST_CW_CLNT_HELLO_A + S(SSL23_ST_CW_CLNT_HELLO_A), +#endif +#ifdef SSL23_ST_CW_CLNT_HELLO_B + S(SSL23_ST_CW_CLNT_HELLO_B), +#endif +#ifdef SSL23_ST_SR_CLNT_HELLO_A + S(SSL23_ST_SR_CLNT_HELLO_A), +#endif +#ifdef SSL23_ST_SR_CLNT_HELLO_B + S(SSL23_ST_SR_CLNT_HELLO_B), +#endif +#ifdef SSL2_ST_CLIENT_START_ENCRYPTION + S(SSL2_ST_CLIENT_START_ENCRYPTION), +#endif +#ifdef SSL2_ST_GET_CLIENT_FINISHED_A + S(SSL2_ST_GET_CLIENT_FINISHED_A), +#endif +#ifdef SSL2_ST_GET_CLIENT_FINISHED_B + S(SSL2_ST_GET_CLIENT_FINISHED_B), +#endif +#ifdef SSL2_ST_GET_CLIENT_HELLO_A + S(SSL2_ST_GET_CLIENT_HELLO_A), +#endif +#ifdef SSL2_ST_GET_CLIENT_HELLO_B + S(SSL2_ST_GET_CLIENT_HELLO_B), +#endif +#ifdef SSL2_ST_GET_CLIENT_HELLO_C + S(SSL2_ST_GET_CLIENT_HELLO_C), +#endif +#ifdef SSL2_ST_GET_CLIENT_MASTER_KEY_A + S(SSL2_ST_GET_CLIENT_MASTER_KEY_A), +#endif +#ifdef SSL2_ST_GET_CLIENT_MASTER_KEY_B + S(SSL2_ST_GET_CLIENT_MASTER_KEY_B), +#endif +#ifdef SSL2_ST_GET_SERVER_FINISHED_A + S(SSL2_ST_GET_SERVER_FINISHED_A), +#endif +#ifdef SSL2_ST_GET_SERVER_FINISHED_B + S(SSL2_ST_GET_SERVER_FINISHED_B), +#endif +#ifdef SSL2_ST_GET_SERVER_HELLO_A + S(SSL2_ST_GET_SERVER_HELLO_A), +#endif +#ifdef SSL2_ST_GET_SERVER_HELLO_B + S(SSL2_ST_GET_SERVER_HELLO_B), +#endif +#ifdef SSL2_ST_GET_SERVER_VERIFY_A + S(SSL2_ST_GET_SERVER_VERIFY_A), +#endif +#ifdef SSL2_ST_GET_SERVER_VERIFY_B + S(SSL2_ST_GET_SERVER_VERIFY_B), +#endif +#ifdef SSL2_ST_SEND_CLIENT_CERTIFICATE_A + S(SSL2_ST_SEND_CLIENT_CERTIFICATE_A), +#endif +#ifdef SSL2_ST_SEND_CLIENT_CERTIFICATE_B + S(SSL2_ST_SEND_CLIENT_CERTIFICATE_B), +#endif +#ifdef SSL2_ST_SEND_CLIENT_CERTIFICATE_C + S(SSL2_ST_SEND_CLIENT_CERTIFICATE_C), +#endif +#ifdef SSL2_ST_SEND_CLIENT_CERTIFICATE_D + S(SSL2_ST_SEND_CLIENT_CERTIFICATE_D), +#endif +#ifdef SSL2_ST_SEND_CLIENT_FINISHED_A + S(SSL2_ST_SEND_CLIENT_FINISHED_A), +#endif +#ifdef SSL2_ST_SEND_CLIENT_FINISHED_B + S(SSL2_ST_SEND_CLIENT_FINISHED_B), +#endif +#ifdef SSL2_ST_SEND_CLIENT_HELLO_A + S(SSL2_ST_SEND_CLIENT_HELLO_A), +#endif +#ifdef SSL2_ST_SEND_CLIENT_HELLO_B + S(SSL2_ST_SEND_CLIENT_HELLO_B), +#endif +#ifdef SSL2_ST_SEND_CLIENT_MASTER_KEY_A + S(SSL2_ST_SEND_CLIENT_MASTER_KEY_A), +#endif +#ifdef SSL2_ST_SEND_CLIENT_MASTER_KEY_B + S(SSL2_ST_SEND_CLIENT_MASTER_KEY_B), +#endif +#ifdef SSL2_ST_SEND_REQUEST_CERTIFICATE_A + S(SSL2_ST_SEND_REQUEST_CERTIFICATE_A), +#endif +#ifdef SSL2_ST_SEND_REQUEST_CERTIFICATE_B + S(SSL2_ST_SEND_REQUEST_CERTIFICATE_B), +#endif +#ifdef SSL2_ST_SEND_REQUEST_CERTIFICATE_C + S(SSL2_ST_SEND_REQUEST_CERTIFICATE_C), +#endif +#ifdef SSL2_ST_SEND_REQUEST_CERTIFICATE_D + S(SSL2_ST_SEND_REQUEST_CERTIFICATE_D), +#endif +#ifdef SSL2_ST_SEND_SERVER_FINISHED_A + S(SSL2_ST_SEND_SERVER_FINISHED_A), +#endif +#ifdef SSL2_ST_SEND_SERVER_FINISHED_B + S(SSL2_ST_SEND_SERVER_FINISHED_B), +#endif +#ifdef SSL2_ST_SEND_SERVER_HELLO_A + S(SSL2_ST_SEND_SERVER_HELLO_A), +#endif +#ifdef SSL2_ST_SEND_SERVER_HELLO_B + S(SSL2_ST_SEND_SERVER_HELLO_B), +#endif +#ifdef SSL2_ST_SEND_SERVER_VERIFY_A + S(SSL2_ST_SEND_SERVER_VERIFY_A), +#endif +#ifdef SSL2_ST_SEND_SERVER_VERIFY_B + S(SSL2_ST_SEND_SERVER_VERIFY_B), +#endif +#ifdef SSL2_ST_SEND_SERVER_VERIFY_C + S(SSL2_ST_SEND_SERVER_VERIFY_C), +#endif +#ifdef SSL2_ST_SERVER_START_ENCRYPTION + S(SSL2_ST_SERVER_START_ENCRYPTION), +#endif +#ifdef SSL2_ST_X509_GET_CLIENT_CERTIFICATE + S(SSL2_ST_X509_GET_CLIENT_CERTIFICATE), +#endif +#ifdef SSL2_ST_X509_GET_SERVER_CERTIFICATE + S(SSL2_ST_X509_GET_SERVER_CERTIFICATE), +#endif +#ifdef SSL3_ST_CR_CERT_A + S(SSL3_ST_CR_CERT_A), +#endif +#ifdef SSL3_ST_CR_CERT_B + S(SSL3_ST_CR_CERT_B), +#endif +#ifdef SSL3_ST_CR_CERT_REQ_A + S(SSL3_ST_CR_CERT_REQ_A), +#endif +#ifdef SSL3_ST_CR_CERT_REQ_B + S(SSL3_ST_CR_CERT_REQ_B), +#endif +#ifdef SSL3_ST_CR_CERT_STATUS_A + S(SSL3_ST_CR_CERT_STATUS_A), +#endif +#ifdef SSL3_ST_CR_CERT_STATUS_B + S(SSL3_ST_CR_CERT_STATUS_B), +#endif +#ifdef SSL3_ST_CR_CHANGE_A + S(SSL3_ST_CR_CHANGE_A), +#endif +#ifdef SSL3_ST_CR_CHANGE_B + S(SSL3_ST_CR_CHANGE_B), +#endif +#ifdef SSL3_ST_CR_FINISHED_A + S(SSL3_ST_CR_FINISHED_A), +#endif +#ifdef SSL3_ST_CR_FINISHED_B + S(SSL3_ST_CR_FINISHED_B), +#endif +#ifdef SSL3_ST_CR_KEY_EXCH_A + S(SSL3_ST_CR_KEY_EXCH_A), +#endif +#ifdef SSL3_ST_CR_KEY_EXCH_B + S(SSL3_ST_CR_KEY_EXCH_B), +#endif +#ifdef SSL3_ST_CR_SESSION_TICKET_A + S(SSL3_ST_CR_SESSION_TICKET_A), +#endif +#ifdef SSL3_ST_CR_SESSION_TICKET_B + S(SSL3_ST_CR_SESSION_TICKET_B), +#endif +#ifdef SSL3_ST_CR_SRVR_DONE_A + S(SSL3_ST_CR_SRVR_DONE_A), +#endif +#ifdef SSL3_ST_CR_SRVR_DONE_B + S(SSL3_ST_CR_SRVR_DONE_B), +#endif +#ifdef SSL3_ST_CR_SRVR_HELLO_A + S(SSL3_ST_CR_SRVR_HELLO_A), +#endif +#ifdef SSL3_ST_CR_SRVR_HELLO_B + S(SSL3_ST_CR_SRVR_HELLO_B), +#endif +#ifdef SSL3_ST_CW_CERT_A + S(SSL3_ST_CW_CERT_A), +#endif +#ifdef SSL3_ST_CW_CERT_B + S(SSL3_ST_CW_CERT_B), +#endif +#ifdef SSL3_ST_CW_CERT_C + S(SSL3_ST_CW_CERT_C), +#endif +#ifdef SSL3_ST_CW_CERT_D + S(SSL3_ST_CW_CERT_D), +#endif +#ifdef SSL3_ST_CW_CERT_VRFY_A + S(SSL3_ST_CW_CERT_VRFY_A), +#endif +#ifdef SSL3_ST_CW_CERT_VRFY_B + S(SSL3_ST_CW_CERT_VRFY_B), +#endif +#ifdef SSL3_ST_CW_CHANGE_A + S(SSL3_ST_CW_CHANGE_A), +#endif +#ifdef SSL3_ST_CW_CHANGE_B + S(SSL3_ST_CW_CHANGE_B), +#endif +#ifdef SSL3_ST_CW_CLNT_HELLO_A + S(SSL3_ST_CW_CLNT_HELLO_A), +#endif +#ifdef SSL3_ST_CW_CLNT_HELLO_B + S(SSL3_ST_CW_CLNT_HELLO_B), +#endif +#ifdef SSL3_ST_CW_FINISHED_A + S(SSL3_ST_CW_FINISHED_A), +#endif +#ifdef SSL3_ST_CW_FINISHED_B + S(SSL3_ST_CW_FINISHED_B), +#endif +#ifdef SSL3_ST_CW_FLUSH + S(SSL3_ST_CW_FLUSH), +#endif +#ifdef SSL3_ST_CW_KEY_EXCH_A + S(SSL3_ST_CW_KEY_EXCH_A), +#endif +#ifdef SSL3_ST_CW_KEY_EXCH_B + S(SSL3_ST_CW_KEY_EXCH_B), +#endif +#ifdef SSL3_ST_SR_CERT_A + S(SSL3_ST_SR_CERT_A), +#endif +#ifdef SSL3_ST_SR_CERT_B + S(SSL3_ST_SR_CERT_B), +#endif +#ifdef SSL3_ST_SR_CERT_VRFY_A + S(SSL3_ST_SR_CERT_VRFY_A), +#endif +#ifdef SSL3_ST_SR_CERT_VRFY_B + S(SSL3_ST_SR_CERT_VRFY_B), +#endif +#ifdef SSL3_ST_SR_CHANGE_A + S(SSL3_ST_SR_CHANGE_A), +#endif +#ifdef SSL3_ST_SR_CHANGE_B + S(SSL3_ST_SR_CHANGE_B), +#endif +#ifdef SSL3_ST_SR_CLNT_HELLO_A + S(SSL3_ST_SR_CLNT_HELLO_A), +#endif +#ifdef SSL3_ST_SR_CLNT_HELLO_B + S(SSL3_ST_SR_CLNT_HELLO_B), +#endif +#ifdef SSL3_ST_SR_CLNT_HELLO_C + S(SSL3_ST_SR_CLNT_HELLO_C), +#endif +#ifdef SSL3_ST_SR_FINISHED_A + S(SSL3_ST_SR_FINISHED_A), +#endif +#ifdef SSL3_ST_SR_FINISHED_B + S(SSL3_ST_SR_FINISHED_B), +#endif +#ifdef SSL3_ST_SR_KEY_EXCH_A + S(SSL3_ST_SR_KEY_EXCH_A), +#endif +#ifdef SSL3_ST_SR_KEY_EXCH_B + S(SSL3_ST_SR_KEY_EXCH_B), +#endif +#ifdef SSL3_ST_SW_CERT_A + S(SSL3_ST_SW_CERT_A), +#endif +#ifdef SSL3_ST_SW_CERT_B + S(SSL3_ST_SW_CERT_B), +#endif +#ifdef SSL3_ST_SW_CERT_REQ_A + S(SSL3_ST_SW_CERT_REQ_A), +#endif +#ifdef SSL3_ST_SW_CERT_REQ_B + S(SSL3_ST_SW_CERT_REQ_B), +#endif +#ifdef SSL3_ST_SW_CERT_STATUS_A + S(SSL3_ST_SW_CERT_STATUS_A), +#endif +#ifdef SSL3_ST_SW_CERT_STATUS_B + S(SSL3_ST_SW_CERT_STATUS_B), +#endif +#ifdef SSL3_ST_SW_CHANGE_A + S(SSL3_ST_SW_CHANGE_A), +#endif +#ifdef SSL3_ST_SW_CHANGE_B + S(SSL3_ST_SW_CHANGE_B), +#endif +#ifdef SSL3_ST_SW_FINISHED_A + S(SSL3_ST_SW_FINISHED_A), +#endif +#ifdef SSL3_ST_SW_FINISHED_B + S(SSL3_ST_SW_FINISHED_B), +#endif +#ifdef SSL3_ST_SW_FLUSH + S(SSL3_ST_SW_FLUSH), +#endif +#ifdef SSL3_ST_SW_HELLO_REQ_A + S(SSL3_ST_SW_HELLO_REQ_A), +#endif +#ifdef SSL3_ST_SW_HELLO_REQ_B + S(SSL3_ST_SW_HELLO_REQ_B), +#endif +#ifdef SSL3_ST_SW_HELLO_REQ_C + S(SSL3_ST_SW_HELLO_REQ_C), +#endif +#ifdef SSL3_ST_SW_KEY_EXCH_A + S(SSL3_ST_SW_KEY_EXCH_A), +#endif +#ifdef SSL3_ST_SW_KEY_EXCH_B + S(SSL3_ST_SW_KEY_EXCH_B), +#endif +#ifdef SSL3_ST_SW_SESSION_TICKET_A + S(SSL3_ST_SW_SESSION_TICKET_A), +#endif +#ifdef SSL3_ST_SW_SESSION_TICKET_B + S(SSL3_ST_SW_SESSION_TICKET_B), +#endif +#ifdef SSL3_ST_SW_SRVR_DONE_A + S(SSL3_ST_SW_SRVR_DONE_A), +#endif +#ifdef SSL3_ST_SW_SRVR_DONE_B + S(SSL3_ST_SW_SRVR_DONE_B), +#endif +#ifdef SSL3_ST_SW_SRVR_HELLO_A + S(SSL3_ST_SW_SRVR_HELLO_A), +#endif +#ifdef SSL3_ST_SW_SRVR_HELLO_B + S(SSL3_ST_SW_SRVR_HELLO_B), +#endif +#ifdef SSL_ST_ACCEPT + S(SSL_ST_ACCEPT), +#endif +#ifdef SSL_ST_BEFORE + S(SSL_ST_BEFORE), +#endif +#ifdef SSL_ST_CONNECT + S(SSL_ST_CONNECT), +#endif +#ifdef SSL_ST_INIT + S(SSL_ST_INIT), +#endif +#ifdef SSL_ST_MASK + S(SSL_ST_MASK), +#endif +#ifdef SSL_ST_OK + S(SSL_ST_OK), +#endif +#ifdef SSL_ST_READ_BODY + S(SSL_ST_READ_BODY), +#endif +#ifdef SSL_ST_READ_DONE + S(SSL_ST_READ_DONE), +#endif +#ifdef SSL_ST_READ_HEADER + S(SSL_ST_READ_HEADER), +#endif +#ifdef SSL_ST_RENEGOTIATE + S(SSL_ST_RENEGOTIATE), +#endif + { 0, NULL } +}; + +#endif + diff --git a/src/common/util.c b/src/common/util.c index 34bb9cc376..6830ef3aa4 100644 --- a/src/common/util.c +++ b/src/common/util.c @@ -15,7 +15,8 @@ #include "orconfig.h" #include "util.h" -#include "log.h" +#include "torlog.h" +#undef log #include "crypto.h" #include "torint.h" #include "container.h" @@ -30,6 +31,11 @@ #include <pwd.h> #endif +/* math.h needs this on Linux */ +#ifndef __USE_ISOC99 +#define __USE_ISOC99 1 +#endif +#include <math.h> #include <stdlib.h> #include <stdio.h> #include <string.h> @@ -278,7 +284,7 @@ tor_log_mallinfo(int severity) struct mallinfo mi; memset(&mi, 0, sizeof(mi)); mi = mallinfo(); - log(severity, LD_MM, + tor_log(severity, LD_MM, "mallinfo() said: arena=%d, ordblks=%d, smblks=%d, hblks=%d, " "hblkhd=%d, usmblks=%d, fsmblks=%d, uordblks=%d, fordblks=%d, " "keepcost=%d", @@ -301,6 +307,25 @@ tor_log_mallinfo(int severity) * Math * ===== */ +/** + * Returns the natural logarithm of d base 2. We define this wrapper here so + * as to make it easier not to conflict with Tor's log() macro. + */ +double +tor_mathlog(double d) +{ + return log(d); +} + +/** Return the long integer closest to d. We define this wrapper here so + * that not all users of math.h need to use the right incancations to get + * the c99 functions. */ +long +tor_lround(double d) +{ + return lround(d); +} + /** Returns floor(log2(u64)). If u64 is 0, (incorrectly) returns 0. */ int tor_log2(uint64_t u64) @@ -345,6 +370,36 @@ round_to_power_of_2(uint64_t u64) return low; } +/** Return the lowest x such that x is at least <b>number</b>, and x modulo + * <b>divisor</b> == 0. */ +unsigned +round_to_next_multiple_of(unsigned number, unsigned divisor) +{ + number += divisor - 1; + number -= number % divisor; + return number; +} + +/** Return the lowest x such that x is at least <b>number</b>, and x modulo + * <b>divisor</b> == 0. */ +uint32_t +round_uint32_to_next_multiple_of(uint32_t number, uint32_t divisor) +{ + number += divisor - 1; + number -= number % divisor; + return number; +} + +/** Return the lowest x such that x is at least <b>number</b>, and x modulo + * <b>divisor</b> == 0. */ +uint64_t +round_uint64_to_next_multiple_of(uint64_t number, uint64_t divisor) +{ + number += divisor - 1; + number -= number % divisor; + return number; +} + /* ===== * String manipulation * ===== */ @@ -627,6 +682,29 @@ find_whitespace_eos(const char *s, const char *eos) return s; } +/** Return the first occurrence of <b>needle</b> in <b>haystack</b> that + * occurs at the start of a line (that is, at the beginning of <b>haystack</b> + * or immediately after a newline). Return NULL if no such string is found. + */ +const char * +find_str_at_start_of_line(const char *haystack, const char *needle) +{ + size_t needle_len = strlen(needle); + + do { + if (!strncmp(haystack, needle, needle_len)) + return haystack; + + haystack = strchr(haystack, '\n'); + if (!haystack) + return NULL; + else + ++haystack; + } while (*haystack); + + return NULL; +} + /** Return true iff the 'len' bytes at 'mem' are all zero. */ int tor_mem_is_zero(const char *mem, size_t len) @@ -654,6 +732,13 @@ tor_digest_is_zero(const char *digest) return tor_mem_is_zero(digest, DIGEST_LEN); } +/** Return true iff the DIGEST256_LEN bytes in digest are all zero. */ +int +tor_digest256_is_zero(const char *digest) +{ + return tor_mem_is_zero(digest, DIGEST256_LEN); +} + /* Helper: common code to check whether the result of a strtol or strtoul or * strtoll is correct. */ #define CHECK_STRTOX_RESULT() \ @@ -705,7 +790,18 @@ tor_parse_ulong(const char *s, int base, unsigned long min, CHECK_STRTOX_RESULT(); } -/** As tor_parse_log, but return a unit64_t. Only base 10 is guaranteed to +/** As tor_parse_long(), but return a double. */ +double +tor_parse_double(const char *s, double min, double max, int *ok, char **next) +{ + char *endptr; + double r; + + r = strtod(s, &endptr); + CHECK_STRTOX_RESULT(); +} + +/** As tor_parse_long, but return a uint64_t. Only base 10 is guaranteed to * work for now. */ uint64_t tor_parse_uint64(const char *s, int base, uint64_t min, @@ -844,6 +940,9 @@ esc_for_log(const char *s) case '\\': case '\"': case '\'': + case '\r': + case '\n': + case '\t': len += 2; break; default: @@ -905,8 +1004,7 @@ const char * escaped(const char *s) { static char *_escaped_val = NULL; - if (_escaped_val) - tor_free(_escaped_val); + tor_free(_escaped_val); if (s) _escaped_val = esc_for_log(s); @@ -993,6 +1091,42 @@ wrap_string(smartlist_t *out, const char *string, size_t width, * Time * ===== */ +/** + * Converts struct timeval to a double value. + * Preserves microsecond precision, but just barely. + * Error is approx +/- 0.1 usec when dealing with epoch values. + */ +double +tv_to_double(const struct timeval *tv) +{ + double conv = tv->tv_sec; + conv += tv->tv_usec/1000000.0; + return conv; +} + +/** + * Converts timeval to milliseconds. + */ +int64_t +tv_to_msec(const struct timeval *tv) +{ + int64_t conv = ((int64_t)tv->tv_sec)*1000L; + /* Round ghetto-style */ + conv += ((int64_t)tv->tv_usec+500)/1000L; + return conv; +} + +/** + * Converts timeval to microseconds. + */ +int64_t +tv_to_usec(const struct timeval *tv) +{ + int64_t conv = ((int64_t)tv->tv_sec)*1000000L; + conv += tv->tv_usec; + return conv; +} + /** Return the number of microseconds elapsed between *start and *end. */ long @@ -1002,7 +1136,8 @@ tv_udiff(const struct timeval *start, const struct timeval *end) long secdiff = end->tv_sec - start->tv_sec; if (labs(secdiff+1) > LONG_MAX/1000000) { - log_warn(LD_GENERAL, "comparing times too far apart."); + log_warn(LD_GENERAL, "comparing times on microsecond detail too far " + "apart: %ld seconds", secdiff); return LONG_MAX; } @@ -1010,6 +1145,26 @@ tv_udiff(const struct timeval *start, const struct timeval *end) return udiff; } +/** Return the number of milliseconds elapsed between *start and *end. + */ +long +tv_mdiff(const struct timeval *start, const struct timeval *end) +{ + long mdiff; + long secdiff = end->tv_sec - start->tv_sec; + + if (labs(secdiff+1) > LONG_MAX/1000) { + log_warn(LD_GENERAL, "comparing times on millisecond detail too far " + "apart: %ld seconds", secdiff); + return LONG_MAX; + } + + /* Subtract and round */ + mdiff = secdiff*1000L + + ((long)end->tv_usec - (long)start->tv_usec + 500L) / 1000L; + return mdiff; +} + /** Yield true iff <b>y</b> is a leap-year. */ #define IS_LEAPYEAR(y) (!(y % 4) && ((y % 100) || !(y % 400))) /** Helper: Return the number of leap-days between Jan 1, y1 and Jan 1, y2. */ @@ -1085,7 +1240,7 @@ format_rfc1123_time(char *buf, time_t t) memcpy(buf+8, MONTH_NAMES[tm.tm_mon], 3); } -/** Parse the the RFC1123 encoding of some time (in GMT) from <b>buf</b>, +/** Parse the RFC1123 encoding of some time (in GMT) from <b>buf</b>, * and store the result in *<b>t</b>. * * Return 0 on success, -1 on failure. @@ -1546,22 +1701,22 @@ check_private_dir(const char *dirname, cpd_check_t check) tor_free(f); if (r) { if (errno != ENOENT) { - log(LOG_WARN, LD_FS, "Directory %s cannot be read: %s", dirname, - strerror(errno)); + log_warn(LD_FS, "Directory %s cannot be read: %s", dirname, + strerror(errno)); return -1; } if (check == CPD_NONE) { - log(LOG_WARN, LD_FS, "Directory %s does not exist.", dirname); + log_warn(LD_FS, "Directory %s does not exist.", dirname); return -1; } else if (check == CPD_CREATE) { log_info(LD_GENERAL, "Creating directory %s", dirname); -#ifdef MS_WINDOWS +#if defined (MS_WINDOWS) && !defined (WINCE) r = mkdir(dirname); #else r = mkdir(dirname, 0700); #endif if (r) { - log(LOG_WARN, LD_FS, "Error creating directory %s: %s", dirname, + log_warn(LD_FS, "Error creating directory %s: %s", dirname, strerror(errno)); return -1; } @@ -1571,7 +1726,7 @@ check_private_dir(const char *dirname, cpd_check_t check) return 0; } if (!(st.st_mode & S_IFDIR)) { - log(LOG_WARN, LD_FS, "%s is not a directory", dirname); + log_warn(LD_FS, "%s is not a directory", dirname); return -1; } #ifndef MS_WINDOWS @@ -1584,7 +1739,7 @@ check_private_dir(const char *dirname, cpd_check_t check) pw = getpwuid(st.st_uid); - log(LOG_WARN, LD_FS, "%s is not owned by this user (%s, %d) but by " + log_warn(LD_FS, "%s is not owned by this user (%s, %d) but by " "%s (%d). Perhaps you are running Tor as the wrong user?", dirname, process_ownername, (int)getuid(), pw ? pw->pw_name : "<unknown>", (int)st.st_uid); @@ -1593,9 +1748,9 @@ check_private_dir(const char *dirname, cpd_check_t check) return -1; } if (st.st_mode & 0077) { - log(LOG_WARN, LD_FS, "Fixing permissions on directory %s", dirname); + log_warn(LD_FS, "Fixing permissions on directory %s", dirname); if (chmod(dirname, 0700)) { - log(LOG_WARN, LD_FS, "Could not chmod directory %s: %s", dirname, + log_warn(LD_FS, "Could not chmod directory %s: %s", dirname, strerror(errno)); return -1; } else { @@ -1626,12 +1781,13 @@ write_str_to_file(const char *fname, const char *str, int bin) } /** Represents a file that we're writing to, with support for atomic commit: - * we can write into a a temporary file, and either remove the file on + * we can write into a temporary file, and either remove the file on * failure, or replace the original file on success. */ struct open_file_t { char *tempname; /**< Name of the temporary file. */ char *filename; /**< Name of the original file. */ - int rename_on_close; /**< Are we using the temporary file or not? */ + unsigned rename_on_close:1; /**< Are we using the temporary file or not? */ + unsigned binary:1; /**< Did we open in binary mode? */ int fd; /**< fd for the open file. */ FILE *stdio_file; /**< stdio wrapper for <b>fd</b>. */ }; @@ -1679,7 +1835,7 @@ start_writing_to_file(const char *fname, int open_flags, int mode, } else { open_name = new_file->tempname = tor_malloc(tempname_len); if (tor_snprintf(new_file->tempname, tempname_len, "%s.tmp", fname)<0) { - log(LOG_WARN, LD_GENERAL, "Failed to generate filename"); + log_warn(LD_GENERAL, "Failed to generate filename"); goto err; } /* We always replace an existing temporary file if there is one. */ @@ -1687,9 +1843,12 @@ start_writing_to_file(const char *fname, int open_flags, int mode, open_flags &= ~O_EXCL; new_file->rename_on_close = 1; } + if (open_flags & O_BINARY) + new_file->binary = 1; - if ((new_file->fd = open(open_name, open_flags, mode)) < 0) { - log(LOG_WARN, LD_FS, "Couldn't open \"%s\" (%s) for writing: %s", + new_file->fd = open(open_name, open_flags, mode); + if (new_file->fd < 0) { + log_warn(LD_FS, "Couldn't open \"%s\" (%s) for writing: %s", open_name, fname, strerror(errno)); goto err; } @@ -1725,7 +1884,8 @@ fdopen_file(open_file_t *file_data) if (file_data->stdio_file) return file_data->stdio_file; tor_assert(file_data->fd >= 0); - if (!(file_data->stdio_file = fdopen(file_data->fd, "a"))) { + if (!(file_data->stdio_file = fdopen(file_data->fd, + file_data->binary?"ab":"a"))) { log_warn(LD_FS, "Couldn't fdopen \"%s\" [%d]: %s", file_data->filename, file_data->fd, strerror(errno)); } @@ -1825,7 +1985,7 @@ write_chunks_to_file_impl(const char *fname, const smartlist_t *chunks, { result = write_all(fd, chunk->bytes, chunk->len, 0); if (result < 0) { - log(LOG_WARN, LD_FS, "Error writing to \"%s\": %s", fname, + log_warn(LD_FS, "Error writing to \"%s\": %s", fname, strerror(errno)); goto err; } @@ -2156,19 +2316,22 @@ char * expand_filename(const char *filename) { tor_assert(filename); +#ifdef MS_WINDOWS + return tor_strdup(filename); +#else if (*filename == '~') { - size_t len; - char *home, *result; + char *home, *result=NULL; const char *rest; if (filename[1] == '/' || filename[1] == '\0') { home = getenv("HOME"); if (!home) { log_warn(LD_CONFIG, "Couldn't find $HOME environment variable while " - "expanding \"%s\"", filename); - return NULL; + "expanding \"%s\"; defaulting to \"\".", filename); + home = tor_strdup(""); + } else { + home = tor_strdup(home); } - home = tor_strdup(home); rest = strlen(filename)>=2?(filename+2):""; } else { #ifdef HAVE_PWD_H @@ -2195,21 +2358,19 @@ expand_filename(const char *filename) if (strlen(home)>1 && !strcmpend(home,PATH_SEPARATOR)) { home[strlen(home)-1] = '\0'; } - /* Plus one for /, plus one for NUL. - * Round up to 16 in case we can't do math. */ - len = strlen(home)+strlen(rest)+16; - result = tor_malloc(len); - tor_snprintf(result,len,"%s"PATH_SEPARATOR"%s",home,rest); + tor_asprintf(&result,"%s"PATH_SEPARATOR"%s",home,rest); tor_free(home); return result; } else { return tor_strdup(filename); } +#endif } #define MAX_SCANF_WIDTH 9999 -/** DOCDOC */ +/** Helper: given an ASCII-encoded decimal digit, return its numeric value. + * NOTE: requires that its input be in-bounds. */ static int digit_to_num(char d) { @@ -2218,7 +2379,10 @@ digit_to_num(char d) return num; } -/** DOCDOC */ +/** Helper: Read an unsigned int from *<b>bufp</b> of up to <b>width</b> + * characters. (Handle arbitrary width if <b>width</b> is less than 0.) On + * success, store the result in <b>out</b>, advance bufp to the next + * character, and return 0. On failure, return -1. */ static int scan_unsigned(const char **bufp, unsigned *out, int width) { @@ -2245,7 +2409,9 @@ scan_unsigned(const char **bufp, unsigned *out, int width) return 0; } -/** DOCDOC */ +/** Helper: copy up to <b>width</b> non-space characters from <b>bufp</b> to + * <b>out</b>. Make sure <b>out</b> is nul-terminated. Advance <b>bufp</b> + * to the next non-space character or the EOS. */ static int scan_string(const char **bufp, char *out, int width) { @@ -2334,7 +2500,12 @@ tor_vsscanf(const char *buf, const char *pattern, va_list ap) * and store the results in the corresponding argument fields. Differs from * sscanf in that it: Only handles %u and %Ns. Does not handle arbitrarily * long widths. %u does not consume any space. Is locale-independent. - * Returns -1 on malformed patterns. */ + * Returns -1 on malformed patterns. + * + * (As with other locale-independent functions, we need this to parse data that + * is in ASCII without worrying that the C library's locale-handling will make + * miscellaneous characters look like numbers, spaces, and so on.) + */ int tor_sscanf(const char *buf, const char *pattern, ...) { @@ -2355,22 +2526,26 @@ tor_listdir(const char *dirname) smartlist_t *result; #ifdef MS_WINDOWS char *pattern; + WCHAR wpattern[MAX_PATH] = {0}; + char name[MAX_PATH] = {0}; HANDLE handle; - WIN32_FIND_DATA findData; + WIN32_FIND_DATAW findData; size_t pattern_len = strlen(dirname)+16; pattern = tor_malloc(pattern_len); tor_snprintf(pattern, pattern_len, "%s\\*", dirname); - if (INVALID_HANDLE_VALUE == (handle = FindFirstFile(pattern, &findData))) { + mbstowcs(wpattern,pattern,MAX_PATH); + if (INVALID_HANDLE_VALUE == (handle = FindFirstFileW(wpattern, &findData))) { tor_free(pattern); return NULL; } + wcstombs(name,findData.cFileName,MAX_PATH); result = smartlist_create(); while (1) { - if (strcmp(findData.cFileName, ".") && - strcmp(findData.cFileName, "..")) { - smartlist_add(result, tor_strdup(findData.cFileName)); + if (strcmp(name, ".") && + strcmp(name, "..")) { + smartlist_add(result, tor_strdup(name)); } - if (!FindNextFile(handle, &findData)) { + if (!FindNextFileW(handle, &findData)) { DWORD err; if ((err = GetLastError()) != ERROR_NO_MORE_FILES) { char *errstr = format_win32_error(err); diff --git a/src/common/util.h b/src/common/util.h index fa66d7cf40..ba38f4c7ed 100644 --- a/src/common/util.h +++ b/src/common/util.h @@ -43,7 +43,7 @@ * stderr. */ #define tor_assert(expr) STMT_BEGIN \ if (PREDICT_UNLIKELY(!(expr))) { \ - log(LOG_ERR, LD_BUG, "%s:%d: %s: Assertion %s failed; aborting.", \ + log_err(LD_BUG, "%s:%d: %s: Assertion %s failed; aborting.", \ _SHORT_FILE_, __LINE__, __func__, #expr); \ fprintf(stderr,"%s:%d %s: Assertion %s failed; aborting.\n", \ _SHORT_FILE_, __LINE__, __func__, #expr); \ @@ -152,8 +152,13 @@ void tor_log_mallinfo(int severity); #define bool_neq(a,b) (!(a)!=!(b)) /* Math functions */ +double tor_mathlog(double d) ATTR_CONST; +long tor_lround(double d) ATTR_CONST; int tor_log2(uint64_t u64) ATTR_CONST; uint64_t round_to_power_of_2(uint64_t u64); +unsigned round_to_next_multiple_of(unsigned number, unsigned divisor); +uint32_t round_uint32_to_next_multiple_of(uint32_t number, uint32_t divisor); +uint64_t round_uint64_to_next_multiple_of(uint64_t number, uint64_t divisor); /* String manipulation */ @@ -179,6 +184,8 @@ long tor_parse_long(const char *s, int base, long min, long max, int *ok, char **next); unsigned long tor_parse_ulong(const char *s, int base, unsigned long min, unsigned long max, int *ok, char **next); +double tor_parse_double(const char *s, double min, double max, int *ok, + char **next); uint64_t tor_parse_uint64(const char *s, int base, uint64_t min, uint64_t max, int *ok, char **next); const char *hex_str(const char *from, size_t fromlen) ATTR_NONNULL((1)); @@ -188,8 +195,11 @@ const char *eat_whitespace_no_nl(const char *s) ATTR_PURE; const char *eat_whitespace_eos_no_nl(const char *s, const char *eos) ATTR_PURE; const char *find_whitespace(const char *s) ATTR_PURE; const char *find_whitespace_eos(const char *s, const char *eos) ATTR_PURE; +const char *find_str_at_start_of_line(const char *haystack, const char *needle) + ATTR_PURE; int tor_mem_is_zero(const char *mem, size_t len) ATTR_PURE; int tor_digest_is_zero(const char *digest) ATTR_PURE; +int tor_digest256_is_zero(const char *digest) ATTR_PURE; char *esc_for_log(const char *string) ATTR_MALLOC; const char *escaped(const char *string); struct smartlist_t; @@ -207,7 +217,11 @@ void base16_encode(char *dest, size_t destlen, const char *src, size_t srclen); int base16_decode(char *dest, size_t destlen, const char *src, size_t srclen); /* Time helpers */ +double tv_to_double(const struct timeval *tv); +int64_t tv_to_msec(const struct timeval *tv); +int64_t tv_to_usec(const struct timeval *tv); long tv_udiff(const struct timeval *start, const struct timeval *end); +long tv_mdiff(const struct timeval *start, const struct timeval *end); time_t tor_timegm(struct tm *tm); #define RFC1123_TIME_LEN 29 void format_rfc1123_time(char *buf, time_t t); @@ -294,5 +308,7 @@ void start_daemon(void); void finish_daemon(const char *desired_cwd); void write_pidfile(char *filename); +const char *libor_get_digests(void); + #endif diff --git a/src/common/util_codedigest.c b/src/common/util_codedigest.c new file mode 100644 index 0000000000..88fe508b92 --- /dev/null +++ b/src/common/util_codedigest.c @@ -0,0 +1,11 @@ + +#include "util.h" + +const char * +libor_get_digests(void) +{ + return "" +#include "common_sha1.i" + ; +} + diff --git a/src/config/geoip b/src/config/geoip index 4ecdc9873f..4ebafe4c63 100644 --- a/src/config/geoip +++ b/src/config/geoip @@ -1,27 +1,50 @@ -# Last updated based on June 1 2010 Maxmind GeoLite Country +# Last updated based on August 1 2010 Maxmind GeoLite Country # wget http://geolite.maxmind.com/download/geoip/database/GeoIPCountryCSV.zip # cut -d, -f3-5 < GeoIPCountryWhois.csv|sed 's/"//g' > geoip 16777216,17301503,AU +17367040,17432575,MY 17435136,17435391,AU +17498112,17563647,KR 17563648,17825791,CN 17825792,18087935,KR +18153472,18219007,JP 18219008,18350079,IN 18350080,18874367,CN +18939904,19005439,JP 19005440,19136511,TW +19136512,19202047,HK 19267584,19398655,IN 19398656,19726335,AU +19726336,19791871,CN 19791872,19922943,TH 19922944,20054015,CN 20054016,20058111,AP +20119552,20185087,CN 20185088,20447231,VN 20447232,20971519,CN +20971520,21102591,HK +21102592,21233663,JP +21233664,21495807,CN +21495808,22020095,JP +23068672,24117247,KR +24117248,24379391,JP +24379392,24641535,CN 24641536,27262975,AU 27262976,28311551,TW -29097984,29360127,CN +28311552,28442623,KR +28442624,28573695,AU +28573696,28966911,CN +28966912,29032447,IN +29097984,29884415,CN 29884416,29885439,AP -30146560,30408703,CN +29949952,30015487,KR +30015488,30408703,CN +30408704,33554431,KR +33554432,34603007,FR 35651584,36700159,IT 36700160,36962303,AE +37748736,38273023,SE +38273024,38797311,KZ 38797312,39059455,PT 39059456,39321599,GR 39321600,39583743,SA @@ -70,8 +93,10 @@ 134730496,135192575,US 135192576,135200767,MX 135200768,135603199,US -135603200,135603455,CA -135603456,152305663,US +135603200,135604223,CA +135604224,135606783,US +135606784,135607039,CA +135607040,152305663,US 152305664,152338431,GB 152338432,167772159,US 184549376,201405503,US @@ -240,7 +265,9 @@ 209868800,209869055,IR 209869056,209965359,US 209965360,209965367,PR -209965368,210022479,US +209965368,209988527,US +209988528,209988535,VI +209988536,210022479,US 210022480,210022487,PR 210022488,210784255,US 210784256,210784383,BO @@ -306,7 +333,9 @@ 212105496,212105503,VI 212105504,212107575,US 212107576,212107583,PR -212107584,212122039,US +212107584,212119767,US +212119768,212119775,PR +212119776,212122039,US 212122040,212122047,PR 212122048,212161487,US 212161488,212161511,VI @@ -320,13 +349,14 @@ 212517200,212517215,VI 212517216,212520815,US 212520816,212520831,VI -212520832,212787199,US +212520832,212733527,US +212733528,212733535,PR +212733536,212787199,US 212787200,212788223,PR 212788224,212788479,US 212788480,212788607,VI 212788608,212788631,US -212788632,212788639,VI -212788640,212788647,US +212788632,212788647,VI 212788648,212788655,PR 212788656,212788663,VI 212788664,212788671,US @@ -358,23 +388,32 @@ 212789248,212791295,US 212791296,212791447,VI 212791448,212791455,US -212791456,212791807,PR +212791456,212791479,VI +212791480,212791807,PR 212791808,212791815,US 212791816,212791839,VI 212791840,212792191,US -212792192,212792263,PR +212792192,212792199,PR +212792200,212792239,VI +212792240,212792263,PR 212792264,212792271,US 212792272,212792279,VI 212792280,212792287,US 212792288,212792319,PR -212792320,212793087,US -212793088,212793151,VI -212793152,212793167,US +212792320,212793103,US +212793104,212793111,VI +212793112,212793119,US +212793120,212793127,VI +212793128,212793135,PR +212793136,212793143,VI +212793144,212793167,US 212793168,212793175,VI 212793176,212793199,US 212793200,212793207,VI 212793208,212793215,PR -212793216,212793343,US +212793216,212793311,US +212793312,212793327,PR +212793328,212793343,US 212793344,212794367,VI 212794368,212794559,PR 212794560,212794575,US @@ -394,7 +433,9 @@ 212794984,212794991,US 212794992,212795007,VI 212795008,212795135,PR -212795136,212838527,US +212795136,212822199,US +212822200,212822215,PR +212822216,212838527,US 212838528,212838535,PR 212838536,212890775,US 212890776,212890783,PR @@ -479,9 +520,14 @@ 214698160,214698167,PR 214698168,214698175,VI 214698176,214698239,PR -214698240,214698319,VI -214698320,214698367,PR -214698368,214698375,VI +214698240,214698271,VI +214698272,214698279,US +214698280,214698319,VI +214698320,214698335,PR +214698336,214698343,US +214698344,214698351,VI +214698352,214698359,US +214698360,214698375,VI 214698376,214698383,US 214698384,214698639,PR 214698640,214698655,VI @@ -490,16 +536,20 @@ 214699296,214699303,US 214699304,214699311,VI 214699312,214699335,PR -214699336,214699375,VI -214699376,214699383,US -214699384,214699415,VI -214699416,214699519,US +214699336,214699367,VI +214699368,214699383,US +214699384,214699455,VI +214699456,214699471,US +214699472,214699487,VI +214699488,214699519,US 214699520,214699647,PR 214699648,214858655,US 214858656,214858671,NL 214858672,216417663,US 216417664,216417727,PR -216417728,216637639,US +216417728,216627279,US +216627280,216627295,PR +216627296,216637639,US 216637640,216637647,PR 216637648,216820479,US 216820480,216820487,PR @@ -516,7 +566,7 @@ 217028008,217046775,US 217046776,217046783,PR 217046784,234881023,US -243269632,251658239,AP +243269632,243269887,AP 251658240,289011535,US 289011536,289011543,IT 289011544,323243895,US @@ -545,7 +595,10 @@ 405946368,405962751,CA 405962752,405970943,US 405979136,405995519,PR -405995520,406126591,US +406003712,406011903,US +406011904,406028287,BS +406044672,406052863,US +406061056,406126591,US 406142976,406147071,US 406147072,406151167,CA 406159360,406175743,US @@ -563,7 +616,9 @@ 408502272,408518655,CA 408518656,408535039,US 408535040,408551423,CA -408551424,409337855,US +408551424,408719359,US +408719360,408723455,VC +408723456,409337855,US 409337856,409354239,CA 409370624,409731071,US 409731072,409862143,CA @@ -599,7 +654,7 @@ 411779072,411828223,PR 411828224,411893759,US 411893760,411975679,CA -411983872,411992063,A2 +411983872,411992063,US 412024832,412221439,US 412221440,412229631,CA 412286976,412483583,US @@ -608,7 +663,9 @@ 412614656,412647423,CL 412647424,412680191,US 412680192,412688383,CA -412696576,412909567,US +412688384,412704767,US +412704768,412708863,CA +412708864,412909567,US 412909568,412917759,CA 412942336,412946431,US 413007872,413908991,US @@ -661,7 +718,7 @@ 418775040,419430399,US 419430400,436207615,GB 436207616,452984831,US -452984832,452985343,AP +452984832,452985855,JP 452985856,452986879,MY 452986880,452987903,SG 452987904,452988927,VN @@ -683,9 +740,9 @@ 455278592,455344127,KR 455344128,456130559,CN 456130560,456261631,IN -456261632,456262655,AP +456261632,456262655,PH 456262656,456263679,IN -456263680,456264703,AP +456263680,456264703,NP 456264704,456265727,JP 456265728,456269823,ID 456269824,456271871,HK @@ -712,7 +769,6 @@ 459294720,459297791,JP 459297792,459298815,IN 459298816,459299839,PK -459299840,459300863,AP 459300864,459309055,SG 459309056,459325439,KR 459325440,459333631,AU @@ -723,11 +779,91 @@ 459472896,459505663,AU 459538432,459539455,AU 459540480,459541503,JP +459541504,459542527,IN 459542528,459544575,HK +459544576,459545599,AU +459546624,459547647,JP +459547648,459548671,CN +459548672,459550719,TH +459550720,459554815,JP 459554816,459571199,TW 459571200,459735039,KR 459735040,459800575,CN 459800576,459866111,SG +459866112,459931647,TW +459931648,459964415,IN +459964416,459980799,CN +459980800,459984895,AU +459984896,459986943,JP +459986944,459988991,PH +459988992,459997183,JP +459997184,460062719,IN +460062720,460128255,PH +460128256,460136447,IN +460144640,460152831,PH +460152832,460154879,JP +460154880,460155903,SG +460155904,460156927,AU +460156928,460158975,KH +460158976,460159999,JP +460160000,460161023,ID +460161024,460193791,MO +460193792,460210175,JP +460210176,460214271,HK +460214272,460218367,MY +460218368,460224511,JP +460224512,460226559,NZ +460226560,460259327,PH +460259328,460261375,AU +460261376,460262399,KH +460262400,460263423,NZ +460263424,460267519,NP +460267520,460275711,ID +460275712,460277759,AU +460277760,460278783,PG +460278784,460279807,JP +460279808,460283903,AU +460283904,460292095,KR +460292096,460300287,JP +460300288,460312575,HK +460312576,460320767,SG +460320768,460324863,AU +460324864,460341247,CN +460341248,460343295,ID +460343296,460344319,AU +460344320,460345343,ID +460345344,460349439,CN +460349440,460351487,AU +460351488,460353535,JP +460353536,460355583,TH +460355584,460356607,IN +460356608,460357631,JP +460357632,460423167,KR +460423168,460439551,CN +460439552,460451839,JP +460451840,460453887,AU +460453888,460454911,JP +460454912,460455935,IN +460455936,460488703,JP +460488704,460505087,MV +460505088,460521471,SG +460521472,460554239,CN +460554240,460587007,KR +460587008,460591103,JP +460591104,460593151,IN +460593152,460595199,JP +460595200,460596223,IN +460596224,460597247,AU +460597248,460598271,JP +460598272,460599295,CN +460599296,460600319,IN +460601344,460602367,AF +460602368,460603391,KH +460603392,460718079,KR +460718080,460722175,JP +460722176,460726271,VN +460726272,460734463,IN +460734464,460849151,KR 461373440,461504511,CN 461504512,461570047,TH 461570048,461572095,ID @@ -759,7 +895,9 @@ 469237760,469499903,IN 469499904,469565439,NZ 469630976,469696511,TH -469696512,469762047,AP +469696512,469712895,PK +469712896,469729279,KR +469729280,469762047,IN 469762048,539624567,US 539624568,539624575,IE 539624576,539624703,US @@ -779,7 +917,13 @@ 539625392,539625407,GB 539625408,539626495,US 539626496,539626543,GB -539626544,540680895,US +539626544,539627391,US +539627392,539627399,JP +539627400,539627407,US +539627408,539627423,JP +539627424,539627503,US +539627504,539627519,NL +539627520,540680895,US 540680896,540680959,BE 540680960,540683775,US 540683776,540683783,DE @@ -848,7 +992,8 @@ 540813696,540814015,TW 540814016,540814079,US 540814080,540814271,TW -540814272,540819455,US +540814272,540814279,DE +540814280,540819455,US 540819456,540823551,CA 540823552,540826671,US 540826672,540826719,CA @@ -908,8 +1053,11 @@ 692256768,692273151,SD 692273152,692289535,EG 692289536,692305919,NG +692305920,692322303,CM 692322304,692453375,EG 692453376,692486143,MW +692486144,692518911,AO +692518912,692551679,ZA 692584448,692600831,AO 692600832,692609023,EG 692609024,692617215,ZM @@ -932,6 +1080,9 @@ 692740096,692744191,ZA 692744192,692748287,TZ 692748288,692752383,CV +692752384,692756479,ZA +692756480,692760575,RW +692760576,692768767,ZA 692830208,692834303,NG 692834304,692838399,TZ 692838400,692842495,ZA @@ -952,6 +1103,12 @@ 692881408,692883455,CG 692883456,692885503,MZ 692885504,692887551,NG +692887552,692889599,UG +692889600,692891647,KE +692891648,692893695,ZA +692893696,692895743,KE +692895744,692897791,NG +692897792,692899839,ZA 692969472,692971519,TZ 692973568,692975615,MZ 692975616,692977663,EG @@ -983,17 +1140,28 @@ 693008384,693009407,CF 693009408,693010431,GH 693010432,693011455,ZA +693011456,693012479,SZ +693012480,693013503,TG +693013504,693014527,ZA +693014528,693015551,GH +693015552,693016575,EG 693101568,693102591,KE 693102592,693103615,CD 693103616,693104639,GN 693104640,693105663,ZA 693105664,693106687,MZ 693106688,693107711,TZ -693107712,693108735,KE +693107712,693239807,KE +693239808,693370879,SN 693501952,693510143,LR 693510144,693518335,SC 693518336,693534719,ZA 693534720,693542911,NG +693542912,693551103,TZ +693551104,693559295,ZA +693559296,693567487,BJ +693567488,693575679,MW +693575680,693583871,KE 693633024,693698559,EG 693698560,693829631,KE 693829632,693895167,EG @@ -1375,19 +1543,97 @@ 704380928,704643071,LY 721420288,738197503,JP 738197504,771751935,US +773062656,773066751,TR +773324800,773586943,ES +773586944,773588991,IT +773588992,773591039,PL +773591040,773593087,RU +773593088,773595135,IT +773595136,773597183,DE +773597184,773599231,BE +773599232,773601279,GB +773601280,773603327,LV +773603328,773605375,UA +773605376,773607423,IR +773607424,773609471,DE +773609472,773611519,RU +773611520,773613567,BY +773613568,773615615,CZ +773615616,773617663,DE +773617664,773619711,IT +773619712,773621759,RU +773621760,773623807,PS +773623808,773625855,SE +773625856,773627903,IS +773627904,773629951,TR +773629952,773631999,FR +773632000,773634047,CY +773634048,773636095,DE +773636096,773638143,UA +773638144,773640191,GB +773640192,773642239,ES +773642240,773644287,HU +773644288,773646335,RU +773646336,773648383,TR +773648384,773650431,PL +773650432,773652479,GB +773652480,773654527,SK +773654528,773656575,RU +774373376,774389759,RS +774389760,774406143,BG +774406144,774422527,IT +774422528,774438911,OM +774438912,774455295,DE +774455296,774471679,RU +774471680,774488063,PL +774488064,774504447,RO +774897664,774963199,RU +774963200,774995967,GE +774995968,775028735,RO +775028736,775061503,PT +775061504,775094271,RU +775094272,775127039,RO +775127040,775159807,IE +775946240,776077311,GB +776077312,776208383,NO +776208384,776339455,GB +776339456,776470527,AM +776470528,776601599,RU +776601600,776732671,AT +776732672,776863743,PL +776994816,778043391,DE +778043392,778108927,RU +778108928,778174463,RO +778174464,778239999,UA +778240000,778305535,AL +778305536,778371071,IR +778371072,778436607,RU +778436608,778502143,RO +778502144,778567679,CY +778567680,778633215,TR +778633216,778698751,IL +778698752,778764287,TR +778764288,778829823,HU +778829824,778895359,RO +778895360,778960895,LV +778960896,779026431,DE +779026432,779091967,RU 779091968,779223039,PL 779223040,779354111,DE 779354112,779485183,IL 779485184,779616255,UA 779616256,779747327,IL 779747328,779878399,SI +779878400,780009471,AT +780009472,780140543,CH 788529152,805306367,CA 805306368,822083583,US +838860800,839909375,US 855638016,872415231,GB 872415232,889192447,US 889192448,905969663,DE 905969664,956301311,US -956301312,973078527,EU +956301312,973078527,FR 973078528,973209599,JP 973209600,973275135,IN 973275136,973471743,JP @@ -1831,6 +2077,7 @@ 1039400960,1039466495,HK 1039466496,1039499263,JP 1039499264,1039507455,AP +1039507456,1039511551,LK 1039515648,1039523839,JP 1039523840,1039532031,KR 1039532032,1039597567,IN @@ -1932,11 +2179,11 @@ 1040973824,1040982015,NL 1040982016,1040982279,A2 1040982280,1040982287,NG -1040982288,1040982631,A2 +1040982288,1040982583,A2 +1040982584,1040982607,DK +1040982608,1040982631,A2 1040982632,1040982639,DE -1040982640,1040983551,A2 -1040983552,1040983567,NG -1040983568,1040983807,A2 +1040982640,1040983807,A2 1040983808,1040983815,NG 1040983816,1040984143,A2 1040984144,1040984151,NG @@ -2002,7 +2249,9 @@ 1041697896,1041697903,GB 1041697904,1041697919,FR 1041697920,1041697983,GB -1041697984,1041698135,FR +1041697984,1041698111,FR +1041698112,1041698119,GB +1041698120,1041698135,FR 1041698136,1041698143,GB 1041698144,1041698247,FR 1041698248,1041698255,GB @@ -2025,10 +2274,14 @@ 1041699592,1041699615,GB 1041699616,1041699623,FR 1041699624,1041699631,GB -1041699632,1041699767,FR +1041699632,1041699679,FR +1041699680,1041699695,GB +1041699696,1041699767,FR 1041699768,1041699807,GB -1041699808,1041700383,FR -1041700384,1041700607,GB +1041699808,1041700391,FR +1041700392,1041700399,GB +1041700400,1041700407,FR +1041700408,1041700607,GB 1041700608,1041700863,FR 1041700864,1041700895,GB 1041700896,1041700919,FR @@ -2051,7 +2304,9 @@ 1041701728,1041701735,GB 1041701736,1041701783,FR 1041701784,1041701791,GB -1041701792,1041701839,FR +1041701792,1041701823,FR +1041701824,1041701831,GB +1041701832,1041701839,FR 1041701840,1041701847,GB 1041701848,1041701943,FR 1041701944,1041701951,GB @@ -2079,8 +2334,8 @@ 1041703952,1041703967,GB 1041703968,1041703991,FR 1041703992,1041703999,GB -1041704000,1041704015,FR -1041704016,1041704039,GB +1041704000,1041704007,FR +1041704008,1041704039,GB 1041704040,1041704111,FR 1041704112,1041704127,GB 1041704128,1041704159,FR @@ -2090,8 +2345,8 @@ 1041704272,1041704279,FR 1041704280,1041704287,GB 1041704288,1041704319,FR -1041704320,1041704335,GB -1041704336,1041704399,FR +1041704320,1041704383,GB +1041704384,1041704399,FR 1041704400,1041704431,GB 1041704432,1041704439,FR 1041704440,1041704455,GB @@ -2151,7 +2406,9 @@ 1041705936,1041705951,GB 1041705952,1041705999,FR 1041706000,1041706007,GB -1041706008,1041706111,FR +1041706008,1041706087,FR +1041706088,1041706095,GB +1041706096,1041706111,FR 1041706112,1041706143,GB 1041706144,1041706191,FR 1041706192,1041706239,GB @@ -2207,9 +2464,7 @@ 1041707680,1041707687,GB 1041707688,1041707711,FR 1041707712,1041707743,GB -1041707744,1041707807,FR -1041707808,1041707815,GB -1041707816,1041707895,FR +1041707744,1041707895,FR 1041707896,1041707903,GB 1041707904,1041707999,FR 1041708000,1041708007,GB @@ -2226,9 +2481,7 @@ 1041708384,1041708415,FR 1041708416,1041708431,GB 1041708432,1041708447,FR -1041708448,1041708463,GB -1041708464,1041708479,FR -1041708480,1041708487,GB +1041708448,1041708487,GB 1041708488,1041708495,FR 1041708496,1041708511,GB 1041708512,1041708551,FR @@ -2253,8 +2506,8 @@ 1041708992,1041708999,GB 1041709000,1041709023,FR 1041709024,1041709055,GB -1041709056,1041709319,FR -1041709320,1041709375,GB +1041709056,1041709327,FR +1041709328,1041709375,GB 1041709376,1041709463,FR 1041709464,1041709471,GB 1041709472,1041709479,FR @@ -2263,9 +2516,7 @@ 1041709768,1041709775,GB 1041709776,1041709783,FR 1041709784,1041709791,GB -1041709792,1041709823,FR -1041709824,1041709831,GB -1041709832,1041709903,FR +1041709792,1041709903,FR 1041709904,1041709911,GB 1041709912,1041709919,FR 1041709920,1041709927,GB @@ -2299,9 +2550,7 @@ 1041710560,1041710583,GB 1041710584,1041710663,FR 1041710664,1041710671,GB -1041710672,1041710687,FR -1041710688,1041710695,GB -1041710696,1041710703,FR +1041710672,1041710703,FR 1041710704,1041710711,GB 1041710712,1041710719,FR 1041710720,1041710727,GB @@ -2387,9 +2636,9 @@ 1041714856,1041714879,GB 1041714880,1041714911,FR 1041714912,1041714919,GB -1041714920,1041714943,FR -1041714944,1041714975,GB -1041714976,1041714991,FR +1041714920,1041714951,FR +1041714952,1041714959,GB +1041714960,1041714991,FR 1041714992,1041715023,GB 1041715024,1041715055,FR 1041715056,1041715071,GB @@ -2403,7 +2652,9 @@ 1041715152,1041715167,GB 1041715168,1041715183,FR 1041715184,1041715199,GB -1041715200,1041715503,FR +1041715200,1041715463,FR +1041715464,1041715471,GB +1041715472,1041715503,FR 1041715504,1041715535,GB 1041715536,1041715551,FR 1041715552,1041715615,GB @@ -2439,8 +2690,8 @@ 1041716560,1041716607,GB 1041716608,1041716671,FR 1041716672,1041716703,GB -1041716704,1041717039,FR -1041717040,1041717103,GB +1041716704,1041717047,FR +1041717048,1041717103,GB 1041717104,1041717111,FR 1041717112,1041717119,GB 1041717120,1041717167,FR @@ -2499,7 +2750,9 @@ 1041718976,1041718991,GB 1041718992,1041718999,FR 1041719000,1041719007,GB -1041719008,1041719203,FR +1041719008,1041719167,FR +1041719168,1041719175,GB +1041719176,1041719203,FR 1041719204,1041719207,GB 1041719208,1041719215,FR 1041719216,1041719223,GB @@ -2554,8 +2807,8 @@ 1041720848,1041720895,FR 1041720896,1041720927,GB 1041720928,1041720959,FR -1041720960,1041721007,GB -1041721008,1041721039,FR +1041720960,1041721023,GB +1041721024,1041721039,FR 1041721040,1041721087,GB 1041721088,1041721343,FR 1041721344,1041721375,GB @@ -2568,8 +2821,8 @@ 1041721792,1041721807,FR 1041721808,1041721855,GB 1041721856,1041722119,FR -1041722120,1041722127,GB -1041722128,1041722143,FR +1041722120,1041722135,GB +1041722136,1041722143,FR 1041722144,1041722159,GB 1041722160,1041722167,FR 1041722168,1041722191,GB @@ -2603,8 +2856,8 @@ 1041723232,1041723263,GB 1041723264,1041723295,FR 1041723296,1041723391,GB -1041723392,1041723647,FR -1041723648,1041723775,GB +1041723392,1041723655,FR +1041723656,1041723775,GB 1041723776,1041723839,FR 1041723840,1041723903,GB 1041723904,1041724455,FR @@ -2617,7 +2870,9 @@ 1041724592,1041724607,GB 1041724608,1041724671,FR 1041724672,1041724927,GB -1041724928,1041725007,FR +1041724928,1041724991,FR +1041724992,1041724999,GB +1041725000,1041725007,FR 1041725008,1041725023,GB 1041725024,1041725119,FR 1041725120,1041725167,GB @@ -2627,9 +2882,7 @@ 1041725256,1041725271,GB 1041725272,1041725343,FR 1041725344,1041725407,GB -1041725408,1041725703,FR -1041725704,1041725711,GB -1041725712,1041725799,FR +1041725408,1041725799,FR 1041725800,1041725807,GB 1041725808,1041725815,FR 1041725816,1041725823,GB @@ -2641,9 +2894,7 @@ 1041726128,1041726151,GB 1041726152,1041726159,FR 1041726160,1041726167,GB -1041726168,1041726463,FR -1041726464,1041726471,GB -1041726472,1041726487,FR +1041726168,1041726487,FR 1041726488,1041726543,GB 1041726544,1041726559,FR 1041726560,1041726591,GB @@ -2727,8 +2978,8 @@ 1041730640,1041730655,GB 1041730656,1041730815,FR 1041730816,1041731071,GB -1041731072,1041731903,FR -1041731904,1041731935,GB +1041731072,1041731911,FR +1041731912,1041731935,GB 1041731936,1041731975,FR 1041731976,1041732031,GB 1041732032,1041732055,FR @@ -2741,9 +2992,7 @@ 1041732256,1041732295,GB 1041732296,1041732303,FR 1041732304,1041732311,GB -1041732312,1041732375,FR -1041732376,1041732383,GB -1041732384,1041732447,FR +1041732312,1041732447,FR 1041732448,1041732455,GB 1041732456,1041732463,FR 1041732464,1041732479,GB @@ -2757,7 +3006,9 @@ 1041732992,1041733103,GB 1041733104,1041733503,FR 1041733504,1041733535,GB -1041733536,1041733887,FR +1041733536,1041733567,FR +1041733568,1041733631,GB +1041733632,1041733887,FR 1041733888,1041734143,GB 1041734144,1041734927,FR 1041734928,1041734943,GB @@ -2777,8 +3028,8 @@ 1041735568,1041735583,GB 1041735584,1041735615,FR 1041735616,1041735663,GB -1041735664,1041736191,FR -1041736192,1041736207,GB +1041735664,1041736199,FR +1041736200,1041736207,GB 1041736208,1041736255,FR 1041736256,1041736263,GB 1041736264,1041736367,FR @@ -2833,7 +3084,9 @@ 1041737848,1041737855,GB 1041737856,1041737871,FR 1041737872,1041737879,GB -1041737880,1041737919,FR +1041737880,1041737903,FR +1041737904,1041737911,GB +1041737912,1041737919,FR 1041737920,1041737935,GB 1041737936,1041738159,FR 1041738160,1041738167,GB @@ -2929,7 +3182,9 @@ 1041742184,1041742191,GB 1041742192,1041742223,FR 1041742224,1041742239,GB -1041742240,1041742279,FR +1041742240,1041742263,FR +1041742264,1041742271,GB +1041742272,1041742279,FR 1041742280,1041742287,GB 1041742288,1041742303,FR 1041742304,1041742335,GB @@ -2979,7 +3234,9 @@ 1041744272,1041744295,GB 1041744296,1041744311,FR 1041744312,1041744327,GB -1041744328,1041744399,FR +1041744328,1041744359,FR +1041744360,1041744367,GB +1041744368,1041744399,FR 1041744400,1041744407,GB 1041744408,1041744479,FR 1041744480,1041744543,GB @@ -3068,8 +3325,8 @@ 1041749816,1041749847,FR 1041749848,1041749855,GB 1041749856,1041749967,FR -1041749968,1041749975,GB -1041749976,1041749999,FR +1041749968,1041749983,GB +1041749984,1041749999,FR 1041750000,1041750007,GB 1041750008,1041750527,FR 1041750528,1041750543,GB @@ -3349,23 +3606,222 @@ 1043357696,1043365887,CH 1043365888,1043398655,PT 1043398656,1043464191,GB -1043464192,1043465215,NL -1043465216,1043466503,GB +1043464192,1043464799,NL +1043464800,1043464807,GB +1043464808,1043464823,NL +1043464824,1043464831,GB +1043464832,1043464927,NL +1043464928,1043465055,GB +1043465056,1043465071,NL +1043465072,1043465087,GB +1043465088,1043465775,NL +1043465776,1043465783,GB +1043465784,1043465839,NL +1043465840,1043465855,GB +1043465856,1043465871,NL +1043465872,1043465919,GB +1043465920,1043465983,NL +1043465984,1043466247,GB +1043466248,1043466319,NL +1043466320,1043466327,GB +1043466328,1043466335,NL +1043466336,1043466351,GB +1043466352,1043466431,NL +1043466432,1043466447,GB +1043466448,1043466455,NL +1043466456,1043466463,GB +1043466464,1043466495,NL +1043466496,1043466503,GB 1043466504,1043466511,NL -1043466512,1043466687,GB -1043466688,1043466711,NL -1043466712,1043474703,GB -1043474704,1043474719,DE -1043474720,1043480575,GB +1043466512,1043466519,GB +1043466520,1043466559,NL +1043466560,1043466575,GB +1043466576,1043466583,NL +1043466584,1043466607,GB +1043466608,1043466887,NL +1043466888,1043466895,GB +1043466896,1043466911,NL +1043466912,1043466927,GB +1043466928,1043466943,NL +1043466944,1043466991,GB +1043466992,1043467007,NL +1043467008,1043467039,GB +1043467040,1043467071,NL +1043467072,1043467087,GB +1043467088,1043467103,NL +1043467104,1043467199,GB +1043467200,1043467231,NL +1043467232,1043467519,GB +1043467520,1043467535,NL +1043467536,1043467583,GB +1043467584,1043467599,NL +1043467600,1043467615,GB +1043467616,1043467663,NL +1043467664,1043467679,GB +1043467680,1043467695,NL +1043467696,1043467759,GB +1043467760,1043467775,NL +1043467776,1043467815,GB +1043467816,1043467839,NL +1043467840,1043467871,GB +1043467872,1043467967,NL +1043467968,1043467983,GB +1043467984,1043467999,NL +1043468000,1043468063,GB +1043468064,1043468127,NL +1043468128,1043468159,GB +1043468160,1043468191,NL +1043468192,1043468543,GB +1043468544,1043468575,NL +1043468576,1043468655,GB +1043468656,1043468663,NL +1043468664,1043468671,GB +1043468672,1043468751,NL +1043468752,1043468767,GB +1043468768,1043468783,NL +1043468784,1043468791,GB +1043468792,1043468799,NL +1043468800,1043468831,GB +1043468832,1043468863,NL +1043468864,1043468895,GB +1043468896,1043468903,NL +1043468904,1043468911,GB +1043468912,1043468927,NL +1043468928,1043468935,GB +1043468936,1043468959,NL +1043468960,1043468967,GB +1043468968,1043469023,NL +1043469024,1043469055,GB +1043469056,1043469087,NL +1043469088,1043469143,GB +1043469144,1043469159,NL +1043469160,1043469183,GB +1043469184,1043469199,NL +1043469200,1043469207,GB +1043469208,1043469223,NL +1043469224,1043469231,GB +1043469232,1043469239,NL +1043469240,1043469247,GB +1043469248,1043469359,NL +1043469360,1043469375,GB +1043469376,1043469399,NL +1043469400,1043469415,GB +1043469416,1043469423,NL +1043469424,1043469439,GB +1043469440,1043469559,NL +1043469560,1043469663,GB +1043469664,1043469679,NL +1043469680,1043469695,GB +1043469696,1043469727,NL +1043469728,1043469823,GB +1043469824,1043469843,NL +1043469844,1043469847,GB +1043469848,1043469919,NL +1043469920,1043469939,GB +1043469940,1043469943,NL +1043469944,1043470127,GB +1043470128,1043470223,NL +1043470224,1043470271,GB +1043470272,1043470303,NL +1043470304,1043470335,GB +1043470336,1043470847,NL +1043470848,1043472415,GB +1043472416,1043472423,DE +1043472424,1043472431,GB +1043472432,1043472439,DE +1043472440,1043472487,GB +1043472488,1043472495,DE +1043472496,1043472503,GB +1043472504,1043472895,DE +1043472896,1043473151,GB +1043473152,1043473411,DE +1043473412,1043473415,GB +1043473416,1043473579,DE +1043473580,1043473583,GB +1043473584,1043474031,DE +1043474032,1043474047,GB +1043474048,1043474487,DE +1043474488,1043474495,GB +1043474496,1043475591,DE +1043475592,1043475711,GB +1043475712,1043475887,DE +1043475888,1043475919,GB +1043475920,1043475983,DE +1043475984,1043475991,GB +1043475992,1043476319,DE +1043476320,1043476351,GB +1043476352,1043476735,DE +1043476736,1043476767,GB +1043476768,1043476815,DE +1043476816,1043476991,GB +1043476992,1043477503,DE +1043477504,1043477759,GB +1043477760,1043478179,DE +1043478180,1043478271,GB +1043478272,1043478511,DE +1043478512,1043478527,GB +1043478528,1043478623,DE +1043478624,1043478639,GB +1043478640,1043478655,DE +1043478656,1043478671,GB +1043478672,1043478735,DE +1043478736,1043478751,GB +1043478752,1043478783,DE +1043478784,1043479295,GB +1043479296,1043479551,DE +1043479552,1043480063,GB +1043480064,1043480351,DE +1043480352,1043480359,GB +1043480360,1043480363,DE +1043480364,1043480575,GB 1043480576,1043488767,CH 1043488768,1043496959,DE -1043496960,1043505151,GB +1043496960,1043497055,FR +1043497056,1043497079,GB +1043497080,1043497215,FR +1043497216,1043497759,GB +1043497760,1043497767,FR +1043497768,1043497863,GB +1043497864,1043497871,FR +1043497872,1043497903,GB +1043497904,1043497911,FR +1043497912,1043497927,GB +1043497928,1043497935,FR +1043497936,1043497983,GB +1043497984,1043498015,FR +1043498016,1043498023,GB +1043498024,1043498031,FR +1043498032,1043498039,GB +1043498040,1043498071,FR +1043498072,1043498095,GB +1043498096,1043498111,FR +1043498112,1043498151,GB +1043498152,1043498183,FR +1043498184,1043498195,GB +1043498196,1043498207,FR +1043498208,1043498223,GB +1043498224,1043498239,FR +1043498240,1043498243,GB +1043498244,1043498247,FR +1043498248,1043498495,GB +1043498496,1043498535,FR +1043498536,1043498543,GB +1043498544,1043498639,FR +1043498640,1043498655,GB +1043498656,1043498671,FR +1043498672,1043498815,GB +1043498816,1043498911,FR +1043498912,1043499267,GB +1043499268,1043499519,FR +1043499520,1043505151,GB 1043505152,1043513343,NO 1043513344,1043529727,GB 1043529728,1043595263,NL 1043595264,1043600639,DE 1043600640,1043600895,FR -1043600896,1043607679,DE +1043600896,1043606255,DE +1043606256,1043606271,GB +1043606272,1043607679,DE 1043607680,1043607711,FR 1043607712,1043609279,DE 1043609280,1043609343,AT @@ -3887,7 +4343,8 @@ 1044930880,1044930911,BE 1044930912,1044930927,DE 1044930928,1044930967,BE -1044930968,1044930979,GB +1044930968,1044930975,FR +1044930976,1044930979,GB 1044930980,1044931375,BE 1044931376,1044931391,GB 1044931392,1044931407,BE @@ -3930,8 +4387,8 @@ 1044932736,1044932799,GB 1044932800,1044932831,BE 1044932832,1044932839,GB -1044932840,1044932855,BE -1044932856,1044932863,GB +1044932840,1044932847,BE +1044932848,1044932863,GB 1044932864,1044932939,BE 1044932940,1044932947,GB 1044932948,1044932951,BE @@ -3952,7 +4409,9 @@ 1044933408,1044933411,GB 1044933412,1044933415,BE 1044933416,1044933439,GB -1044933440,1044933615,BE +1044933440,1044933511,BE +1044933512,1044933519,GB +1044933520,1044933615,BE 1044933616,1044933623,GB 1044933624,1044933711,BE 1044933712,1044933719,GB @@ -3966,7 +4425,9 @@ 1044933844,1044933847,GB 1044933848,1044933863,BE 1044933864,1044933871,GB -1044933872,1044934191,BE +1044933872,1044934095,BE +1044934096,1044934111,GB +1044934112,1044934191,BE 1044934192,1044934199,GB 1044934200,1044934239,BE 1044934240,1044934247,GB @@ -3986,17 +4447,21 @@ 1044935028,1044935031,GB 1044935032,1044935035,BE 1044935036,1044935039,GB -1044935040,1044935231,BE +1044935040,1044935183,BE +1044935184,1044935199,GB +1044935200,1044935231,BE 1044935232,1044935247,GB -1044935248,1044935279,BE -1044935280,1044935295,GB +1044935248,1044935263,BE +1044935264,1044935295,GB 1044935296,1044935331,BE 1044935332,1044935343,GB 1044935344,1044935355,BE 1044935356,1044935391,GB 1044935392,1044935407,BE 1044935408,1044935423,GB -1044935424,1044935595,BE +1044935424,1044935551,BE +1044935552,1044935559,GB +1044935560,1044935595,BE 1044935596,1044935599,GB 1044935600,1044935623,BE 1044935624,1044935631,GB @@ -4090,17 +4555,23 @@ 1044971520,1044979711,FR 1044979712,1044987903,FI 1044987904,1045004287,BG -1045004288,1045018207,GB +1045004288,1045013231,GB +1045013232,1045013247,IE +1045013248,1045013455,GB +1045013456,1045013471,IE +1045013472,1045018207,GB 1045018208,1045018231,FI 1045018232,1045018367,GB 1045018368,1045018423,ES 1045018424,1045018431,GB 1045018432,1045018495,ES -1045018496,1045020159,GB +1045018496,1045018623,GB +1045018624,1045018751,ES +1045018752,1045020159,GB 1045020160,1045020255,ES 1045020256,1045020287,GB -1045020288,1045020415,ES -1045020416,1045020671,GB +1045020288,1045020519,ES +1045020520,1045020671,GB 1045020672,1045037055,NO 1045037056,1045119231,GR 1045119232,1045119743,AL @@ -4119,9 +4590,7 @@ 1045160960,1045160991,TR 1045160992,1045161023,DE 1045161024,1045161055,AT -1045161056,1045161087,DE -1045161088,1045161119,PT -1045161120,1045161151,DE +1045161056,1045161151,DE 1045161152,1045161183,PT 1045161184,1045168127,DE 1045168128,1045233663,RU @@ -4177,8 +4646,7 @@ 1045452288,1045452799,SK 1045452800,1045453055,CZ 1045453056,1045454847,SK -1045454848,1045460735,DE -1045460736,1045460991,NL +1045454848,1045460991,DE 1045460992,1045461503,FR 1045461504,1045461631,DE 1045461632,1045461695,FR @@ -4202,23 +4670,23 @@ 1045716992,1045725183,RU 1045725184,1045733375,CZ 1045733376,1045741567,GB -1045741568,1045742015,SE +1045741568,1045741887,SE +1045741888,1045741951,GB +1045741952,1045742015,SE 1045742016,1045742023,GB 1045742024,1045742031,SE 1045742032,1045742039,GB -1045742040,1045742047,SE -1045742048,1045742111,GB -1045742112,1045742127,SE -1045742128,1045742175,GB -1045742176,1045742199,SE -1045742200,1045742207,GB -1045742208,1045742239,SE +1045742040,1045742063,SE +1045742064,1045742111,GB +1045742112,1045742135,SE +1045742136,1045742143,NL +1045742144,1045742151,SE +1045742152,1045742159,DK +1045742160,1045742167,SE +1045742168,1045742175,DK +1045742176,1045742239,SE 1045742240,1045742247,GB -1045742248,1045742271,SE -1045742272,1045742335,GB -1045742336,1045742351,SE -1045742352,1045742367,GB -1045742368,1045742383,SE +1045742248,1045742383,SE 1045742384,1045742391,GB 1045742392,1045742495,SE 1045742496,1045742535,GB @@ -4232,19 +4700,20 @@ 1045742840,1045742847,GB 1045742848,1045743023,SE 1045743024,1045743031,GB -1045743032,1045743043,SE -1045743044,1045743063,GB +1045743032,1045743039,SE +1045743040,1045743047,GB +1045743048,1045743063,NL 1045743064,1045743095,SE 1045743096,1045743099,GB -1045743100,1045743247,SE +1045743100,1045743103,SE +1045743104,1045743167,GB +1045743168,1045743247,SE 1045743248,1045743251,GB 1045743252,1045743295,SE 1045743296,1045743299,GB 1045743300,1045743679,SE 1045743680,1045743743,GB -1045743744,1045743783,SE -1045743784,1045743791,GB -1045743792,1045743803,SE +1045743744,1045743803,SE 1045743804,1045743807,GB 1045743808,1045743999,SE 1045744000,1045744007,LU @@ -4252,7 +4721,9 @@ 1045744016,1045744031,SE 1045744032,1045744063,GB 1045744064,1045744639,SE -1045744640,1045745407,GB +1045744640,1045744671,BE +1045744672,1045744703,SE +1045744704,1045745407,GB 1045745408,1045745607,SE 1045745608,1045745615,FI 1045745616,1045745623,GB @@ -4304,8 +4775,8 @@ 1045748396,1045748399,GB 1045748400,1045748407,SE 1045748408,1045748463,GB -1045748464,1045748479,SE -1045748480,1045749503,GB +1045748464,1045748735,SE +1045748736,1045749503,GB 1045749504,1045749759,SE 1045749760,1045757951,GB 1045757952,1045790719,LV @@ -4330,10 +4801,14 @@ 1046225656,1046229111,NO 1046229112,1046229119,SE 1046229120,1046282239,NO -1046282240,1046288383,DE +1046282240,1046284287,DE +1046284288,1046285311,BZ +1046285312,1046286663,DE +1046286664,1046286671,BZ +1046286672,1046288383,DE 1046288384,1046288663,CZ -1046288664,1046289407,DE -1046289408,1046290431,AG +1046288664,1046288895,DE +1046288896,1046290431,AG 1046290432,1046298623,PL 1046298624,1046299903,AT 1046299904,1046300159,MK @@ -4392,8 +4867,7 @@ 1046337536,1046338047,DE 1046338048,1046339839,EU 1046339840,1046340095,FR -1046340096,1046340351,NL -1046340352,1046340607,EU +1046340096,1046340607,EU 1046340608,1046341119,NL 1046341120,1046341631,EU 1046341632,1046342143,FR @@ -4422,19 +4896,21 @@ 1046385112,1046413311,IT 1046413312,1046446079,SE 1046446080,1046479535,DE -1046479536,1046479550,GB -1046479551,1046479687,DE +1046479536,1046479551,GB +1046479552,1046479687,DE 1046479688,1046479695,GB -1046479696,1046480475,DE -1046480476,1046480479,GB +1046479696,1046480047,DE +1046480048,1046480055,GB +1046480056,1046480471,DE +1046480472,1046480479,GB 1046480480,1046480719,DE 1046480720,1046480735,GB 1046480736,1046481327,DE 1046481328,1046481375,GB 1046481376,1046481535,DE 1046481536,1046481919,GB -1046481920,1046481963,DE -1046481964,1046481967,GB +1046481920,1046481959,DE +1046481960,1046481967,GB 1046481968,1046482751,DE 1046482752,1046482943,GB 1046482944,1046483231,DE @@ -4458,14 +4934,8 @@ 1046485288,1046485367,DE 1046485368,1046485375,GB 1046485376,1046485495,DE -1046485496,1046485543,GB -1046485544,1046485647,DE -1046485648,1046485655,GB -1046485656,1046485759,DE -1046485760,1046485791,GB -1046485792,1046485823,DE -1046485824,1046485855,GB -1046485856,1046486055,DE +1046485496,1046485503,GB +1046485504,1046486055,DE 1046486056,1046486063,GB 1046486064,1046486159,DE 1046486160,1046486175,GB @@ -4524,24 +4994,8 @@ 1046494336,1046494607,DE 1046494608,1046494615,GB 1046494616,1046494623,DE -1046494624,1046494751,GB -1046494752,1046494767,DE -1046494768,1046494775,GB -1046494776,1046494815,DE -1046494816,1046494823,GB -1046494824,1046494831,DE -1046494832,1046494839,GB -1046494840,1046494975,DE -1046494976,1046495007,GB -1046495008,1046495063,DE -1046495064,1046495071,GB -1046495072,1046495079,DE -1046495080,1046495087,GB -1046495088,1046495103,DE -1046495104,1046495127,GB -1046495128,1046495223,DE -1046495224,1046495231,GB -1046495232,1046495351,DE +1046494624,1046494719,GB +1046494720,1046495351,DE 1046495352,1046495359,GB 1046495360,1046495407,DE 1046495408,1046495415,GB @@ -4622,289 +5076,19 @@ 1046499328,1046502655,DE 1046502656,1046502719,GB 1046502720,1046502863,DE -1046502864,1046502951,GB -1046502952,1046502967,DE -1046502968,1046502975,GB -1046502976,1046503135,DE -1046503136,1046503143,GB -1046503144,1046503167,DE -1046503168,1046503207,GB -1046503208,1046503343,DE -1046503344,1046503351,GB -1046503352,1046504415,DE -1046504416,1046504487,GB -1046504488,1046504551,DE -1046504552,1046504559,GB -1046504560,1046504599,DE -1046504600,1046504615,GB -1046504616,1046504671,DE -1046504672,1046504679,GB -1046504680,1046504703,DE -1046504704,1046504735,GB -1046504736,1046504799,DE -1046504800,1046504807,GB -1046504808,1046504871,DE -1046504872,1046504879,GB -1046504880,1046504887,DE -1046504888,1046504895,GB -1046504896,1046504919,DE -1046504920,1046504927,GB -1046504928,1046504943,DE -1046504944,1046504951,GB -1046504952,1046504959,DE -1046504960,1046504999,GB -1046505000,1046505031,DE -1046505032,1046505039,GB -1046505040,1046505055,DE -1046505056,1046505063,GB -1046505064,1046505071,DE -1046505072,1046505087,GB -1046505088,1046505159,DE -1046505160,1046505167,GB -1046505168,1046505183,DE -1046505184,1046505191,GB -1046505192,1046505215,DE -1046505216,1046505247,GB -1046505248,1046505343,DE -1046505344,1046505359,GB -1046505360,1046505415,DE -1046505416,1046505423,GB -1046505424,1046505431,DE -1046505432,1046505439,GB -1046505440,1046505463,DE -1046505464,1046505511,GB -1046505512,1046505551,DE -1046505552,1046505559,GB -1046505560,1046505583,DE -1046505584,1046505591,GB -1046505592,1046505615,DE -1046505616,1046505631,GB -1046505632,1046505663,DE -1046505664,1046505671,GB -1046505672,1046505679,DE -1046505680,1046505687,GB -1046505688,1046505695,DE -1046505696,1046505703,GB -1046505704,1046505727,DE -1046505728,1046505759,GB -1046505760,1046505807,DE -1046505808,1046505823,GB -1046505824,1046505847,DE -1046505848,1046505879,GB -1046505880,1046505895,DE -1046505896,1046505903,GB -1046505904,1046505983,DE -1046505984,1046506015,GB -1046506016,1046506039,DE -1046506040,1046506047,GB -1046506048,1046506119,DE -1046506120,1046506135,GB -1046506136,1046506239,DE -1046506240,1046506271,GB -1046506272,1046506295,DE -1046506296,1046506303,GB -1046506304,1046506327,DE -1046506328,1046506343,GB -1046506344,1046506375,DE -1046506376,1046506383,GB -1046506384,1046506391,DE -1046506392,1046506407,GB -1046506408,1046506415,DE -1046506416,1046506423,GB -1046506424,1046506439,DE -1046506440,1046506447,GB -1046506448,1046506479,DE -1046506480,1046506487,GB -1046506488,1046506495,DE -1046506496,1046506535,GB -1046506536,1046506575,DE -1046506576,1046506591,GB -1046506592,1046506599,DE -1046506600,1046506607,GB -1046506608,1046506623,DE -1046506624,1046506639,GB -1046506640,1046506647,DE -1046506648,1046506655,GB -1046506656,1046506671,DE -1046506672,1046506679,GB -1046506680,1046506687,DE -1046506688,1046506695,GB -1046506696,1046506711,DE -1046506712,1046506743,GB -1046506744,1046506751,DE -1046506752,1046506783,GB -1046506784,1046506791,DE -1046506792,1046506807,GB -1046506808,1046506823,DE -1046506824,1046506839,GB -1046506840,1046506895,DE -1046506896,1046506903,GB -1046506904,1046506927,DE -1046506928,1046506935,GB -1046506936,1046506951,DE -1046506952,1046506959,GB -1046506960,1046506991,DE -1046506992,1046506999,GB -1046507000,1046507007,DE -1046507008,1046507039,GB -1046507040,1046507047,DE -1046507048,1046507055,GB -1046507056,1046507071,DE -1046507072,1046507111,GB -1046507112,1046507143,DE -1046507144,1046507151,GB -1046507152,1046507159,DE -1046507160,1046507175,GB -1046507176,1046507199,DE -1046507200,1046507207,GB -1046507208,1046507223,DE -1046507224,1046507247,GB -1046507248,1046507263,DE -1046507264,1046507519,GB -1046507520,1046511615,DE -1046511616,1046511647,GB -1046511648,1046511695,DE -1046511696,1046511711,GB -1046511712,1046511743,DE -1046511744,1046511751,GB -1046511752,1046511759,DE -1046511760,1046511815,GB -1046511816,1046511847,DE -1046511848,1046511863,GB -1046511864,1046511871,DE -1046511872,1046511903,GB -1046511904,1046511919,DE -1046511920,1046511927,GB -1046511928,1046511935,DE -1046511936,1046511967,GB -1046511968,1046511975,DE -1046511976,1046511991,GB -1046511992,1046512007,DE -1046512008,1046512015,GB -1046512016,1046512031,DE -1046512032,1046512055,GB -1046512056,1046512063,DE -1046512064,1046512095,GB -1046512096,1046512103,DE -1046512104,1046512167,GB -1046512168,1046512175,DE -1046512176,1046512199,GB -1046512200,1046512215,DE -1046512216,1046512223,GB -1046512224,1046512231,DE -1046512232,1046512271,GB -1046512272,1046512303,DE -1046512304,1046512311,GB -1046512312,1046512327,DE -1046512328,1046512335,GB -1046512336,1046512383,DE -1046512384,1046512415,GB -1046512416,1046512447,DE -1046512448,1046512455,GB -1046512456,1046512535,DE -1046512536,1046512543,GB -1046512544,1046512551,DE -1046512552,1046512567,GB -1046512568,1046512607,DE -1046512608,1046512623,GB -1046512624,1046512631,DE -1046512632,1046512671,GB -1046512672,1046512687,DE -1046512688,1046512711,GB -1046512712,1046512727,DE -1046512728,1046512735,GB -1046512736,1046512751,DE -1046512752,1046512759,GB -1046512760,1046512847,DE -1046512848,1046512879,GB -1046512880,1046512895,DE -1046512896,1046512927,GB -1046512928,1046512943,DE -1046512944,1046512967,GB -1046512968,1046512983,DE -1046512984,1046512991,GB -1046512992,1046512999,DE -1046513000,1046513007,GB -1046513008,1046513023,DE -1046513024,1046513031,GB -1046513032,1046513039,DE -1046513040,1046513047,GB -1046513048,1046513055,DE -1046513056,1046513063,GB -1046513064,1046513079,DE -1046513080,1046513111,GB -1046513112,1046513143,DE -1046513144,1046513231,GB -1046513232,1046513239,DE -1046513240,1046513247,GB -1046513248,1046513255,DE -1046513256,1046513263,GB -1046513264,1046513295,DE -1046513296,1046513303,GB -1046513304,1046513327,DE -1046513328,1046513343,GB -1046513344,1046513391,DE -1046513392,1046513439,GB -1046513440,1046513519,DE -1046513520,1046513527,GB -1046513528,1046513551,DE -1046513552,1046513559,GB -1046513560,1046513591,DE -1046513592,1046513607,GB -1046513608,1046513615,DE -1046513616,1046513623,GB -1046513624,1046513655,DE -1046513656,1046513719,GB -1046513720,1046513735,DE -1046513736,1046513759,GB -1046513760,1046513791,DE -1046513792,1046513799,GB -1046513800,1046513831,DE -1046513832,1046513847,GB -1046513848,1046513863,DE -1046513864,1046513871,GB -1046513872,1046513879,DE -1046513880,1046513887,GB -1046513888,1046513919,DE -1046513920,1046514207,GB -1046514208,1046514223,DE -1046514224,1046514239,GB -1046514240,1046514247,DE -1046514248,1046514255,GB -1046514256,1046514279,DE -1046514280,1046514287,GB -1046514288,1046514311,DE -1046514312,1046514319,GB -1046514320,1046514335,DE -1046514336,1046514343,GB -1046514344,1046514359,DE -1046514360,1046514367,GB -1046514368,1046514383,DE -1046514384,1046514391,GB -1046514392,1046514399,DE -1046514400,1046514415,GB -1046514416,1046514423,DE -1046514424,1046514687,GB -1046514688,1046516287,DE +1046502864,1046502911,GB +1046502912,1046504415,DE +1046504416,1046504447,GB +1046504448,1046516287,DE 1046516288,1046516735,GB 1046516736,1046516751,DE 1046516752,1046516991,GB 1046516992,1046517031,DE 1046517032,1046517039,GB 1046517040,1046517087,DE -1046517088,1046517799,GB -1046517800,1046517807,DE -1046517808,1046517815,GB -1046517816,1046517831,DE -1046517832,1046517839,GB -1046517840,1046517847,DE -1046517848,1046517855,GB -1046517856,1046517863,DE -1046517864,1046517871,GB -1046517872,1046517903,DE -1046517904,1046517911,GB -1046517912,1046517927,DE -1046517928,1046524159,GB +1046517088,1046517759,GB +1046517760,1046518271,DE +1046518272,1046524159,GB 1046524160,1046524191,DE 1046524192,1046524207,GB 1046524208,1046524215,DE @@ -4949,8 +5133,8 @@ 1046528604,1046528607,GB 1046528608,1046528639,DE 1046528640,1046528671,GB -1046528672,1046528703,DE -1046528704,1046529279,GB +1046528672,1046528767,DE +1046528768,1046529279,GB 1046529280,1046529535,DE 1046529536,1046530047,GB 1046530048,1046530687,DE @@ -4961,9 +5145,7 @@ 1046530948,1046530967,GB 1046530968,1046530975,DE 1046530976,1046531007,GB -1046531008,1046531087,DE -1046531088,1046531095,GB -1046531096,1046531127,DE +1046531008,1046531127,DE 1046531128,1046531839,GB 1046531840,1046532095,DE 1046532096,1046534015,GB @@ -4971,11 +5153,11 @@ 1046534048,1046534063,GB 1046534064,1046534071,DE 1046534072,1046534143,GB -1046534144,1046534655,DE -1046534656,1046534911,GB -1046534912,1046535167,DE -1046535168,1046535295,GB -1046535296,1046535311,DE +1046534144,1046534743,DE +1046534744,1046534847,GB +1046534848,1046534895,DE +1046534896,1046534911,GB +1046534912,1046535311,DE 1046535312,1046535359,GB 1046535360,1046535423,DE 1046535424,1046535487,GB @@ -5015,19 +5197,17 @@ 1046537072,1046537087,GB 1046537088,1046537151,DE 1046537152,1046537215,GB -1046537216,1046537983,DE -1046537984,1046538431,GB +1046537216,1046538239,DE +1046538240,1046538431,GB 1046538432,1046538447,DE 1046538448,1046538463,GB 1046538464,1046538495,DE 1046538496,1046538751,GB -1046538752,1046539519,DE -1046539520,1046540031,GB +1046538752,1046539775,DE +1046539776,1046540031,GB 1046540032,1046540287,DE 1046540288,1046540543,GB -1046540544,1046541055,DE -1046541056,1046541311,GB -1046541312,1046541439,DE +1046540544,1046541439,DE 1046541440,1046541503,GB 1046541504,1046541567,DE 1046541568,1046541631,GB @@ -5037,9 +5217,7 @@ 1046541760,1046541767,GB 1046541768,1046541783,DE 1046541784,1046541807,GB -1046541808,1046542335,DE -1046542336,1046542847,GB -1046542848,1046543103,DE +1046541808,1046543103,DE 1046543104,1046543263,GB 1046543264,1046543295,DE 1046543296,1046543327,GB @@ -5202,8 +5380,7 @@ 1047552000,1047560191,RU 1047560192,1047561367,DE 1047561368,1047561371,CN -1047561372,1047561375,AU -1047561376,1047561747,DE +1047561372,1047561747,DE 1047561748,1047561751,ES 1047561752,1047561887,DE 1047561888,1047561891,US @@ -5216,8 +5393,10 @@ 1047563312,1047563319,DE 1047563320,1047563323,CH 1047563324,1047563347,DE -1047563348,1047563351,CH -1047563352,1047563403,DE +1047563348,1047563355,CH +1047563356,1047563363,DE +1047563364,1047563367,ES +1047563368,1047563403,DE 1047563404,1047563407,NL 1047563408,1047563411,CH 1047563412,1047563415,DE @@ -5341,7 +5520,9 @@ 1047568236,1047568239,CH 1047568240,1047568247,DE 1047568248,1047568251,CH -1047568252,1047568383,DE +1047568252,1047568363,DE +1047568364,1047568367,CH +1047568368,1047568383,DE 1047568384,1047576575,NL 1047576576,1047584767,HU 1047584768,1047592959,NL @@ -5431,9 +5612,11 @@ 1048604744,1048604759,EE 1048604760,1048604927,UA 1048604928,1048604965,LT -1048604966,1048604975,UA -1048604976,1048605055,LT -1048605056,1048607231,UA +1048604966,1048604967,UA +1048604968,1048604971,LT +1048604972,1048604975,UA +1048604976,1048605063,LT +1048605064,1048607231,UA 1048607232,1048607247,EE 1048607248,1048607487,UA 1048607488,1048607519,EE @@ -6187,14 +6370,16 @@ 1049020480,1049020543,GE 1049020544,1049026559,DE 1049026560,1049026815,EU -1049026816,1049032159,DE -1049032160,1049032167,EU +1049026816,1049032063,DE +1049032064,1049032127,EU +1049032128,1049032143,DE +1049032144,1049032167,EU 1049032168,1049032171,DE 1049032172,1049032175,EU 1049032176,1049032287,DE 1049032288,1049032295,EU -1049032296,1049032351,DE -1049032352,1049032399,EU +1049032296,1049032383,DE +1049032384,1049032399,EU 1049032400,1049032639,DE 1049032640,1049032647,EU 1049032648,1049032699,DE @@ -6281,7 +6466,8 @@ 1049709056,1049709311,NL 1049709312,1049709567,GB 1049709568,1049709823,A2 -1049709824,1049710591,NL +1049709824,1049710079,US +1049710080,1049710591,NL 1049710592,1049712639,GB 1049712640,1049713055,IR 1049713056,1049713087,MA @@ -6603,9 +6789,7 @@ 1051574624,1051574655,GM 1051574656,1051575999,GB 1051576000,1051576015,IT -1051576016,1051576903,GB -1051576904,1051576911,US -1051576912,1051577503,GB +1051576016,1051577503,GB 1051577504,1051577519,NL 1051577520,1051577535,GB 1051577536,1051577567,NL @@ -6711,12 +6895,12 @@ 1052000520,1052000527,DE 1052000528,1052000535,EU 1052000536,1052000539,DE -1052000540,1052000575,EU -1052000576,1052000623,DE +1052000540,1052000583,EU +1052000584,1052000623,DE 1052000624,1052000631,EU 1052000632,1052002303,DE -1052002304,1052002311,EU -1052002312,1052002335,DE +1052002304,1052002319,EU +1052002320,1052002335,DE 1052002336,1052002367,EU 1052002368,1052002431,DE 1052002432,1052002439,EU @@ -6792,14 +6976,14 @@ 1052010656,1052010751,EU 1052010752,1052011135,DE 1052011136,1052011263,EU -1052011264,1052012335,DE +1052011264,1052012287,DE +1052012288,1052012303,EU +1052012304,1052012335,DE 1052012336,1052012375,EU 1052012376,1052012399,DE 1052012400,1052012543,EU 1052012544,1052013055,DE -1052013056,1052013059,EU -1052013060,1052013063,DE -1052013064,1052013071,EU +1052013056,1052013071,EU 1052013072,1052013087,DE 1052013088,1052013119,EU 1052013120,1052013127,DE @@ -6954,7 +7138,7 @@ 1052119040,1052121087,RO 1052121088,1052129279,RU 1052129280,1052131327,BE -1052131328,1052135423,RU +1052133376,1052135423,RU 1052135424,1052137471,UA 1052137472,1052139519,RU 1052139520,1052141567,DE @@ -6962,11 +7146,13 @@ 1052143616,1052145663,BG 1052145664,1052147711,UA 1052147712,1052151807,RU +1052151808,1052153855,PL 1052153856,1052157951,UA 1052157952,1052159999,DK 1052160000,1052162047,LU 1052162048,1052164095,CH 1052164096,1052168191,RU +1052168192,1052170239,PL 1052170240,1052172287,SE 1052172288,1052174335,PL 1052174336,1052176383,CZ @@ -7840,16 +8026,16 @@ 1053296992,1053297023,IT 1053297024,1053297039,EU 1053297040,1053297055,IT -1053297056,1053297111,EU +1053297056,1053297071,EU +1053297072,1053297087,IT +1053297088,1053297111,EU 1053297112,1053297135,IT 1053297136,1053297143,EU 1053297144,1053297151,IT 1053297152,1053298175,EU 1053298176,1053299199,CH 1053299200,1053299711,EU -1053299712,1053300147,GB -1053300148,1053300151,EU -1053300152,1053300159,GB +1053299712,1053300159,GB 1053300160,1053300223,EU 1053300224,1053300735,GB 1053300736,1053300991,CH @@ -7890,9 +8076,7 @@ 1053306880,1053307903,EU 1053307904,1053307951,GB 1053307952,1053307959,EU -1053307960,1053307999,GB -1053308000,1053308031,EU -1053308032,1053308095,GB +1053307960,1053308095,GB 1053308096,1053308159,EU 1053308160,1053308543,GB 1053308544,1053308671,EU @@ -7928,8 +8112,7 @@ 1053314048,1053314063,EU 1053314064,1053314079,CZ 1053314080,1053315071,EU -1053315072,1053316863,GB -1053316864,1053317119,EU +1053315072,1053317119,GB 1053317120,1053317295,ZA 1053317296,1053317375,EU 1053317376,1053317631,GB @@ -8053,7 +8236,9 @@ 1053349632,1053349887,EU 1053349888,1053349951,IE 1053349952,1053350143,EU -1053350144,1053350359,IE +1053350144,1053350335,IE +1053350336,1053350351,EU +1053350352,1053350359,IE 1053350360,1053350367,EU 1053350368,1053350383,IE 1053350384,1053350399,EU @@ -8069,8 +8254,7 @@ 1053353112,1053353135,EU 1053353136,1053353215,IE 1053353216,1053353223,GB -1053353224,1053353231,EU -1053353232,1053353255,IE +1053353224,1053353255,IE 1053353256,1053353263,EU 1053353264,1053353279,IE 1053353280,1053353983,EU @@ -8294,8 +8478,12 @@ 1054359552,1054367743,RO 1054367744,1054375935,FI 1054375936,1054381055,GB -1054381056,1054381823,EG -1054381824,1054384127,GB +1054381056,1054381567,EG +1054381568,1054381583,SG +1054381584,1054381599,HK +1054381600,1054381615,GB +1054381616,1054381631,US +1054381632,1054384127,GB 1054384128,1054400511,DE 1054400512,1054408703,GB 1054408704,1054416895,FR @@ -8379,8 +8567,9 @@ 1054969040,1054978815,DE 1054978816,1054979071,FR 1054979072,1055129599,DE -1055129600,1055162367,CY -1055162368,1055195135,GR +1055129600,1055186943,CY +1055186944,1055194623,GR +1055194624,1055195135,CY 1055195136,1055196159,EU 1055196160,1055197823,CH 1055197824,1055197919,EU @@ -8560,7 +8749,11 @@ 1056210944,1056219135,PL 1056219136,1056227327,GB 1056227328,1056243711,TR -1056243712,1056244447,CH +1056243712,1056244255,CH +1056244256,1056244287,FR +1056244288,1056244335,MC +1056244336,1056244351,FR +1056244352,1056244447,CH 1056244448,1056244479,MC 1056244480,1056251903,CH 1056251904,1056260095,RU @@ -8679,9 +8872,7 @@ 1062725104,1062725111,IL 1062725112,1062871551,US 1062871552,1062872319,PR -1062872320,1063335422,US -1063335423,1063335423,CO -1063335424,1063390207,US +1062872320,1063390207,US 1063390208,1063390463,PR 1063390464,1063497863,US 1063497864,1063497871,CA @@ -9561,7 +9752,9 @@ 1075601280,1075601295,CA 1075601296,1075609599,US 1075609600,1075613695,TT -1075613696,1075855359,US +1075613696,1075744767,US +1075744768,1075748863,A2 +1075748864,1075855359,US 1075855360,1075871743,DO 1075871744,1075970047,US 1075970048,1075972095,CA @@ -10736,7 +10929,9 @@ 1077632448,1077632479,NL 1077632480,1077633823,US 1077633824,1077633855,AN -1077633856,1077641215,US +1077633856,1077640799,US +1077640800,1077640831,EC +1077640832,1077641215,US 1077641216,1077657599,CA 1077657600,1077839103,US 1077839104,1077839119,ES @@ -10800,7 +10995,9 @@ 1077869760,1077869823,GR 1077869824,1077870335,US 1077870336,1077870591,CN -1077870592,1077977087,US +1077870592,1077963775,US +1077963776,1077964031,CA +1077964032,1077977087,US 1077977088,1077985279,CA 1077985280,1077993471,US 1077993472,1078001663,CA @@ -10811,8 +11008,8 @@ 1078124544,1078128639,CA 1078128640,1078247423,US 1078247424,1078251519,CA -1078251520,1078280447,US -1078280448,1078280575,CA +1078251520,1078280191,US +1078280192,1078280575,CA 1078280576,1078280591,US 1078280592,1078280639,CA 1078280640,1078280655,CR @@ -10824,14 +11021,12 @@ 1078281080,1078281087,US 1078281088,1078281279,CA 1078281280,1078281295,US -1078281296,1078281511,CA -1078281512,1078281527,US -1078281528,1078281599,CA -1078281600,1078281679,US +1078281296,1078281663,CA +1078281664,1078281679,US 1078281680,1078281711,CA 1078281712,1078281719,US -1078281720,1078281727,CA -1078281728,1078281743,US +1078281720,1078281735,CA +1078281736,1078281743,US 1078281744,1078281759,CA 1078281760,1078281791,KN 1078281792,1078282119,CA @@ -10841,7 +11036,7 @@ 1078282240,1078283015,CA 1078283016,1078283023,VG 1078283024,1078283103,CA -1078283104,1078283111,US +1078283104,1078283111,MX 1078283112,1078283119,CA 1078283120,1078283135,US 1078283136,1078283263,CA @@ -10871,37 +11066,35 @@ 1078284352,1078284367,CA 1078284368,1078284383,BV 1078284384,1078284399,US -1078284400,1078284479,CA -1078284480,1078284543,US -1078284544,1078284687,CA -1078284688,1078284703,US -1078284704,1078284863,CA -1078284864,1078284927,US -1078284928,1078284991,CA +1078284400,1078284687,CA +1078284688,1078284695,US +1078284696,1078284703,MX +1078284704,1078284991,CA 1078284992,1078285007,US 1078285008,1078285015,CA 1078285016,1078285023,US 1078285024,1078285031,CA 1078285032,1078285039,US -1078285040,1078285151,CA -1078285152,1078285167,US -1078285168,1078285175,CA -1078285176,1078285247,US +1078285040,1078285191,CA +1078285192,1078285215,US +1078285216,1078285231,CA +1078285232,1078285247,US 1078285248,1078285255,CA 1078285256,1078285263,US 1078285264,1078285279,CA -1078285280,1078285295,US +1078285280,1078285287,MX +1078285288,1078285295,US 1078285296,1078285303,CA 1078285304,1078285567,US 1078285568,1078285639,CA -1078285640,1078285655,US -1078285656,1078285695,CA +1078285640,1078285647,US +1078285648,1078285695,CA 1078285696,1078285703,MX 1078285704,1078285711,US 1078285712,1078285919,CA -1078285920,1078285951,US -1078285952,1078286015,CA -1078286016,1078286079,US +1078285920,1078285943,US +1078285944,1078286047,CA +1078286048,1078286079,US 1078286080,1078286111,CA 1078286112,1078286115,US 1078286116,1078286143,CA @@ -10909,19 +11102,23 @@ 1078286208,1078286335,CA 1078286336,1078286351,US 1078286352,1078286367,CA -1078286368,1078286463,US +1078286368,1078286399,US +1078286400,1078286431,CA +1078286432,1078286463,US 1078286464,1078286559,CA 1078286560,1078286575,TW 1078286576,1078286591,CA -1078286592,1078286751,US +1078286592,1078286655,US +1078286656,1078286687,CA +1078286688,1078286751,US 1078286752,1078286759,CA 1078286760,1078286783,US 1078286784,1078286791,MX 1078286792,1078286847,US 1078286848,1078286975,CA 1078286976,1078286991,US -1078286992,1078287023,CA -1078287024,1078287039,US +1078286992,1078287031,CA +1078287032,1078287039,US 1078287040,1078287071,CA 1078287072,1078287103,US 1078287104,1078287183,CA @@ -11198,17 +11395,9 @@ 1078957056,1079320575,US 1079320576,1079320671,CA 1079320672,1079320703,US -1079320704,1079320719,CA -1079320720,1079320735,US -1079320736,1079320823,CA -1079320824,1079320831,US -1079320832,1079321087,CA -1079321088,1079321103,US -1079321104,1079321199,CA -1079321200,1079321207,US -1079321208,1079321215,CA -1079321216,1079321231,US -1079321232,1079322703,CA +1079320704,1079322191,CA +1079322192,1079322199,US +1079322200,1079322703,CA 1079322704,1079322719,US 1079322720,1079322815,CA 1079322816,1079322839,US @@ -11331,8 +11520,10 @@ 1079402496,1079403007,CA 1079403008,1079403519,US 1079403520,1079403775,CA -1079403776,1079403967,US -1079403968,1079403999,CA +1079403776,1079403807,US +1079403808,1079403839,CA +1079403840,1079403935,US +1079403936,1079403999,CA 1079404000,1079404799,US 1079404800,1079405023,CA 1079405024,1079405247,US @@ -11393,24 +11584,22 @@ 1079435776,1079436799,HT 1079436800,1079437311,CA 1079437312,1079439359,US -1079439360,1079443199,CA +1079439360,1079442431,CA +1079442432,1079442687,US +1079442688,1079443199,CA 1079443200,1079443231,PW 1079443232,1079443327,CA 1079443328,1079443455,AU 1079443456,1079459839,US 1079459840,1079508991,CA -1079508992,1079566335,US -1079566336,1079566463,A2 -1079566464,1079566559,US -1079566560,1079566847,A2 +1079508992,1079566847,US 1079566848,1079567103,GB -1079567104,1079567359,A2 +1079567104,1079567359,US 1079567360,1079567615,AU -1079567616,1079568383,A2 -1079568384,1079569407,US -1079569408,1079570431,A2 -1079570432,1079574527,US -1079574528,1079578623,PR +1079567616,1079574527,US +1079574528,1079576487,PR +1079576488,1079576495,US +1079576496,1079578623,PR 1079578624,1079585391,US 1079585392,1079585407,IN 1079585408,1079664639,US @@ -11565,22 +11754,28 @@ 1080750602,1080750611,GB 1080750612,1080820871,US 1080820872,1080820879,PR -1080820880,1080950783,US -1080953344,1080953855,US +1080820880,1080954879,US 1080955904,1080957183,US -1080957952,1080958207,US +1080957952,1080958207,A2 1080958208,1080958463,BH -1080958464,1080958719,US +1080958464,1080958719,A2 1080958720,1080958975,BH -1080958976,1080960255,US +1080958976,1080960255,A2 1080960256,1080960511,BH -1080960512,1080968191,US +1080960512,1080963839,A2 +1080963840,1080967167,US +1080967168,1080968191,A2 1080968192,1080968447,BH -1080968448,1080969471,US +1080968448,1080969471,A2 1080969472,1080969727,BH -1080969728,1080972575,US +1080969728,1080972575,A2 1080972576,1080972607,ID -1080972608,1080999935,US +1080972608,1080983551,A2 +1080983552,1080987647,US +1080987648,1080988671,A2 +1080988672,1080989951,US +1080989952,1080990207,A2 +1080990208,1080999935,US 1080999936,1081016319,CA 1081016320,1081037311,US 1081037312,1081037567,CA @@ -11866,7 +12061,9 @@ 1081609824,1081609831,LK 1081609832,1081609887,US 1081609888,1081609919,IN -1081609920,1081618951,US +1081609920,1081616831,US +1081616832,1081616839,GB +1081616840,1081618951,US 1081618952,1081618959,GB 1081618960,1081803287,US 1081803288,1081803295,PR @@ -11904,7 +12101,9 @@ 1082097056,1082097071,CA 1082097072,1082097143,US 1082097144,1082097151,AU -1082097152,1082314751,US +1082097152,1082138623,US +1082138624,1082140671,A2 +1082140672,1082314751,US 1082314752,1082315263,CA 1082315264,1082315471,US 1082315472,1082318847,CA @@ -11947,7 +12146,9 @@ 1082350656,1082350671,IT 1082350672,1082350911,US 1082350912,1082350943,RU -1082350944,1082434239,US +1082350944,1082419455,US +1082419456,1082419711,A2 +1082419712,1082434239,US 1082434240,1082434255,IN 1082434256,1082621987,US 1082621988,1082621991,ME @@ -12488,8 +12689,8 @@ 1090146304,1090150399,CA 1090150400,1090154495,US 1090158592,1090207743,US -1090207744,1090220031,CA -1090220032,1090355199,US +1090207744,1090215935,CA +1090215936,1090355199,US 1090355200,1090356327,CA 1090356328,1090356335,US 1090356336,1090357535,CA @@ -12519,7 +12720,9 @@ 1090447488,1090448127,CA 1090448128,1090448255,US 1090448256,1090453503,CA -1090453504,1090497903,US +1090453504,1090495487,US +1090495488,1090495743,NO +1090495744,1090497903,US 1090497904,1090497919,AU 1090497920,1090499583,US 1090499584,1090499839,NO @@ -12733,7 +12936,9 @@ 1093110912,1093110927,CA 1093110928,1093110935,US 1093110936,1093111711,CA -1093111712,1093111759,US +1093111712,1093111727,US +1093111728,1093111743,CA +1093111744,1093111759,US 1093111760,1093111871,CA 1093111872,1093111903,US 1093111904,1093111935,CA @@ -12756,11 +12961,11 @@ 1093114848,1093114863,US 1093114864,1093115839,CA 1093115840,1093115903,US -1093115904,1093116767,CA -1093116768,1093116799,US -1093116800,1093116863,CA -1093116864,1093117439,US -1093117440,1093117519,CA +1093115904,1093116927,CA +1093116928,1093117439,US +1093117440,1093117455,CA +1093117456,1093117503,US +1093117504,1093117519,CA 1093117520,1093117527,US 1093117528,1093117567,CA 1093117568,1093117599,US @@ -12775,7 +12980,9 @@ 1093118528,1093118531,CA 1093118532,1093118591,US 1093118592,1093118687,CA -1093118688,1093118993,US +1093118688,1093118983,US +1093118984,1093118991,CA +1093118992,1093118993,US 1093118994,1093119006,CA 1093119007,1093119015,US 1093119016,1093119023,CA @@ -12834,9 +13041,7 @@ 1093124288,1093124351,US 1093124352,1093124559,CA 1093124560,1093124575,US -1093124576,1093125103,CA -1093125104,1093125111,US -1093125112,1093125695,CA +1093124576,1093125695,CA 1093125696,1093125719,US 1093125720,1093125735,CA 1093125736,1093125743,US @@ -12873,8 +13078,8 @@ 1093127776,1093127791,CA 1093127792,1093128191,US 1093128192,1093128415,CA -1093128416,1093128959,US -1093128960,1093129215,CA +1093128416,1093128447,US +1093128448,1093129215,CA 1093129216,1093130367,US 1093130368,1093130415,CA 1093130416,1093130495,US @@ -12895,9 +13100,7 @@ 1093133312,1093133407,US 1093133408,1093133439,CA 1093133440,1093133567,US -1093133568,1093134143,CA -1093134144,1093134207,US -1093134208,1093134239,CA +1093133568,1093134239,CA 1093134240,1093134255,US 1093134256,1093134263,CA 1093134264,1093134271,US @@ -12906,12 +13109,12 @@ 1093135360,1093135615,CA 1093135616,1093135631,US 1093135632,1093135639,CA -1093135640,1093135679,US -1093135680,1093135871,CA -1093135872,1093136895,US -1093136896,1093137663,CA -1093137664,1093137791,US -1093137792,1093138431,CA +1093135640,1093135647,US +1093135648,1093135871,CA +1093135872,1093136127,US +1093136128,1093136383,CA +1093136384,1093136895,US +1093136896,1093138431,CA 1093138432,1093138559,US 1093138560,1093138623,CA 1093138624,1093139455,US @@ -12948,7 +13151,9 @@ 1093737424,1093737431,GB 1093737432,1093740095,US 1093740096,1093740159,JP -1093740160,1093740335,US +1093740160,1093740167,US +1093740168,1093740191,CN +1093740192,1093740335,US 1093740336,1093740351,SG 1093740352,1093740415,JP 1093740416,1093741599,US @@ -13546,7 +13751,9 @@ 1097777152,1097798495,US 1097798496,1097798527,CA 1097798528,1097799167,US -1097799168,1097799423,CA +1097799168,1097799199,CA +1097799200,1097799231,US +1097799232,1097799423,CA 1097799424,1097799679,US 1097799680,1097799711,CA 1097799712,1097799775,US @@ -13952,7 +14159,11 @@ 1106814720,1106814975,EC 1106814976,1107220223,US 1107220224,1107220239,AX -1107220240,1107243007,US +1107220240,1107241215,US +1107241216,1107241343,CN +1107241344,1107241471,US +1107241472,1107241599,CN +1107241600,1107243007,US 1107243008,1107247103,TC 1107247104,1107275775,US 1107275776,1107279871,CA @@ -13966,7 +14177,11 @@ 1107801368,1107801375,CA 1107801376,1107805695,US 1107805696,1107805951,TR -1107805952,1107820543,US +1107805952,1107808842,US +1107808843,1107808846,PE +1107808847,1107808877,US +1107808878,1107808881,PE +1107808882,1107820543,US 1107820544,1107853311,ZA 1107853312,1107895039,US 1107895040,1107895807,GB @@ -14047,9 +14262,7 @@ 1108057080,1108057087,US 1108057088,1108057135,CA 1108057136,1108057151,US -1108057152,1108057199,CA -1108057200,1108057207,MT -1108057208,1108057215,CA +1108057152,1108057215,CA 1108057216,1108057239,US 1108057240,1108057335,CA 1108057336,1108057343,GB @@ -14190,7 +14403,9 @@ 1109954352,1109954367,ES 1109954368,1110126591,US 1110126592,1110130687,CA -1110130688,1110310911,US +1110130688,1110307903,US +1110307904,1110307967,AU +1110307968,1110310911,US 1110310912,1110376447,CA 1110376448,1110441983,US 1110441984,1110442239,CA @@ -14278,7 +14493,9 @@ 1110460224,1110460319,CA 1110460320,1110460383,US 1110460384,1110460447,CA -1110460448,1110460671,US +1110460448,1110460479,US +1110460480,1110460511,CA +1110460512,1110460671,US 1110460672,1110460719,CA 1110460720,1110460743,US 1110460744,1110460775,CA @@ -14383,7 +14600,15 @@ 1113598464,1113599487,US 1113599488,1113599999,VE 1113600000,1113600255,US -1113600256,1113603071,VE +1113600256,1113602399,VE +1113602400,1113602527,US +1113602528,1113602543,VE +1113602544,1113602559,US +1113602560,1113602655,VE +1113602656,1113602687,US +1113602688,1113602783,VE +1113602784,1113602815,US +1113602816,1113603071,VE 1113603072,1113603327,CL 1113603328,1113603583,SV 1113603584,1113603839,US @@ -14450,7 +14675,7 @@ 1114515456,1114517503,GU 1114517504,1114520063,CA 1114520064,1114520319,PH -1114520320,1114520575,CA +1114520320,1114520575,US 1114520576,1114520831,PH 1114520832,1114524159,CA 1114524160,1114524671,ZA @@ -14470,13 +14695,42 @@ 1114681344,1114685439,CA 1114685440,1114730495,US 1114730496,1114734591,CA -1114734592,1114877183,US +1114734592,1114865711,US +1114865712,1114865727,CA +1114865728,1114875391,US +1114875392,1114876159,CA +1114876160,1114876447,US +1114876448,1114876463,CA +1114876464,1114876479,US +1114876480,1114876543,CA +1114876544,1114876575,GB +1114876576,1114876927,US +1114876928,1114877183,CA 1114877184,1114877439,CY -1114877440,1114877951,US +1114877440,1114877695,US +1114877696,1114877759,SK +1114877760,1114877791,US +1114877792,1114877807,CA +1114877808,1114877951,US 1114877952,1114878207,PA -1114878208,1114878849,US +1114878208,1114878463,US +1114878464,1114878719,NL +1114878720,1114878849,US 1114878850,1114878977,LT -1114878978,1114966815,US +1114878978,1114880079,US +1114880080,1114880095,CY +1114880096,1114880383,US +1114880384,1114880399,CY +1114880400,1114880407,GB +1114880408,1114881343,US +1114881344,1114881407,CY +1114881408,1114881471,US +1114881472,1114881535,CY +1114881536,1114881743,US +1114881744,1114881759,CA +1114881760,1114930175,US +1114930176,1114930303,GB +1114930304,1114966815,US 1114966816,1114966831,GB 1114966832,1115104511,US 1115104512,1115104639,CA @@ -14581,9 +14835,17 @@ 1115795456,1115799551,CA 1115799552,1115815935,US 1115815936,1115947007,CA -1115947008,1116041279,US -1116041280,1116041295,CA -1116041296,1116158463,US +1115947008,1116014079,US +1116014080,1116014335,A2 +1116014336,1116014591,US +1116014592,1116014847,A2 +1116014848,1116015871,US +1116015872,1116016127,A2 +1116016128,1116027135,US +1116027136,1116027903,A2 +1116027904,1116041791,US +1116041792,1116041807,CA +1116041808,1116158463,US 1116158464,1116158495,AU 1116158496,1116168191,US 1116168192,1116169311,CA @@ -14876,29 +15138,27 @@ 1117979504,1117979519,US 1117979520,1117982639,CA 1117982640,1117982655,US -1117982656,1117983023,CA +1117982656,1117982999,CA +1117983000,1117983007,US +1117983008,1117983023,CA 1117983024,1117983039,US 1117983040,1117983215,CA 1117983216,1117983223,US -1117983224,1117984191,CA -1117984192,1117984199,US -1117984200,1117985055,CA +1117983224,1117983831,CA +1117983832,1117983839,US +1117983840,1117984023,CA +1117984024,1117984047,US +1117984048,1117985055,CA 1117985056,1117985079,US 1117985080,1117985423,CA 1117985424,1117985447,US 1117985448,1117985463,CA 1117985464,1117985471,US -1117985472,1117985607,CA -1117985608,1117985663,US -1117985664,1117985671,CA -1117985672,1117985679,US -1117985680,1117985695,CA -1117985696,1117985711,US -1117985712,1117985719,CA -1117985720,1117985743,US -1117985744,1117985767,CA -1117985768,1117985783,US -1117985784,1117986575,CA +1117985472,1117985615,CA +1117985616,1117985647,US +1117985648,1117985767,CA +1117985768,1117985775,US +1117985776,1117986575,CA 1117986576,1117986687,US 1117986688,1117986815,CA 1117986816,1117995007,US @@ -14906,10 +15166,8 @@ 1117999104,1118027783,US 1118027784,1118029471,CA 1118029472,1118029519,US -1118029520,1118029527,CA -1118029528,1118029535,US -1118029536,1118029543,CA -1118029544,1118029567,US +1118029520,1118029551,CA +1118029552,1118029567,US 1118029568,1118030991,CA 1118030992,1118030999,US 1118031000,1118031071,CA @@ -14923,8 +15181,8 @@ 1118031384,1118031431,CA 1118031432,1118031439,US 1118031440,1118031759,CA -1118031760,1118031807,US -1118031808,1118031871,CA +1118031760,1118031799,US +1118031800,1118031871,CA 1118031872,1118126079,US 1118126080,1118126591,CA 1118126592,1118126847,US @@ -15048,7 +15306,8 @@ 1118157696,1118157711,NL 1118157712,1118157759,US 1118157760,1118157775,GY -1118157776,1118158079,US +1118157776,1118157791,AR +1118157792,1118158079,US 1118158080,1118158087,PA 1118158088,1118158319,US 1118158320,1118158323,SV @@ -15057,7 +15316,9 @@ 1118158336,1118158847,US 1118158848,1118167039,CA 1118167040,1118474239,US -1118474240,1118474591,CA +1118474240,1118474535,CA +1118474536,1118474543,US +1118474544,1118474591,CA 1118474592,1118474599,US 1118474600,1118474607,CA 1118474608,1118474615,US @@ -15071,9 +15332,9 @@ 1118475216,1118475223,US 1118475224,1118475231,CA 1118475232,1118475247,US -1118475248,1118476751,CA -1118476752,1118476759,US -1118476760,1118476887,CA +1118475248,1118476807,CA +1118476808,1118476831,US +1118476832,1118476887,CA 1118476888,1118476895,US 1118476896,1118477535,CA 1118477536,1118477543,US @@ -15655,7 +15916,9 @@ 1120152320,1120152575,US 1120152576,1120153599,CA 1120153600,1120272383,US -1120272384,1120281087,CA +1120272384,1120280855,CA +1120280856,1120280863,US +1120280864,1120281087,CA 1120281088,1120281095,PG 1120281096,1120281103,US 1120281104,1120281111,CA @@ -15668,29 +15931,35 @@ 1120281408,1120281599,CA 1120281600,1120282367,US 1120282368,1120282623,SY -1120282624,1120282879,CA +1120282624,1120282879,US 1120282880,1120283135,SY -1120283136,1120283647,CA +1120283136,1120283391,US +1120283392,1120283647,CA 1120283648,1120284671,EC 1120284672,1120286719,CA 1120286720,1120288767,US -1120288768,1120299007,CA +1120288768,1120293119,CA +1120293120,1120293631,US +1120293632,1120299007,CA 1120299008,1120299263,PK -1120299264,1120299519,CA +1120299264,1120299519,US 1120299520,1120300799,PK 1120300800,1120306175,CA 1120306176,1120306943,PH -1120306944,1120307199,CA +1120306944,1120307199,US 1120307200,1120307967,EC 1120307968,1120308223,PH 1120308224,1120309247,HT 1120309248,1120310015,CA 1120310016,1120310783,PH -1120310784,1120312063,CA +1120310784,1120311807,CA +1120311808,1120312063,US 1120312064,1120312447,PH 1120312448,1120312575,US 1120312576,1120312831,PH -1120312832,1120315135,CA +1120312832,1120313343,CA +1120313344,1120313599,US +1120313600,1120315135,CA 1120315136,1120315391,US 1120315392,1120321535,CA 1120321536,1120346111,US @@ -15709,23 +15978,24 @@ 1120372736,1120374047,CA 1120374048,1120374063,US 1120374064,1120374271,CA -1120374272,1120374335,US -1120374336,1120374399,CA +1120374272,1120374303,US +1120374304,1120374399,CA 1120374400,1120374415,US 1120374416,1120374479,CA 1120374480,1120374487,US 1120374488,1120374511,CA 1120374512,1120374527,US -1120374528,1120375183,CA -1120375184,1120375199,US -1120375200,1120375231,CA +1120374528,1120375231,CA 1120375232,1120375239,US 1120375240,1120375243,CA 1120375244,1120375263,US 1120375264,1120376063,CA -1120376064,1120376127,US +1120376064,1120376079,US +1120376080,1120376095,CA +1120376096,1120376127,US 1120376128,1120376223,CA -1120376224,1120376255,US +1120376224,1120376239,US +1120376240,1120376255,MX 1120376256,1120376287,CA 1120376288,1120376303,US 1120376304,1120376327,CA @@ -15733,9 +16003,10 @@ 1120376384,1120376447,CA 1120376448,1120376479,US 1120376480,1120376494,CA -1120376495,1120376575,US -1120376576,1120376647,CA -1120376648,1120376679,US +1120376495,1120376559,US +1120376560,1120376647,CA +1120376648,1120376671,US +1120376672,1120376679,MX 1120376680,1120376687,CA 1120376688,1120376767,US 1120376768,1120376815,CA @@ -15744,11 +16015,9 @@ 1120377168,1120377199,US 1120377200,1120377215,CA 1120377216,1120377247,US -1120377248,1120377599,CA -1120377600,1120377727,US -1120377728,1120377791,CA -1120377792,1120377855,US -1120377856,1120378367,CA +1120377248,1120377791,CA +1120377792,1120377823,US +1120377824,1120378367,CA 1120378368,1120378623,US 1120378624,1120378919,CA 1120378920,1120378927,US @@ -15757,9 +16026,9 @@ 1120379072,1120379935,CA 1120379936,1120379967,US 1120379968,1120380027,CA -1120380028,1120380047,US -1120380048,1120380063,CA -1120380064,1120380127,US +1120380028,1120380031,US +1120380032,1120380095,CA +1120380096,1120380127,US 1120380128,1120380131,CA 1120380132,1120380415,US 1120380416,1120380431,VG @@ -15767,7 +16036,9 @@ 1120380448,1120380479,US 1120380480,1120380639,CA 1120380640,1120380655,US -1120380656,1120380671,CA +1120380656,1120380659,CA +1120380660,1120380663,US +1120380664,1120380671,CA 1120380672,1120380927,US 1120380928,1120381055,CA 1120381056,1120381071,US @@ -15852,9 +16123,7 @@ 1120495360,1120495615,CA 1120495616,1120495743,US 1120495744,1120495871,SG -1120495872,1120495967,US -1120495968,1120495999,CA -1120496000,1120496223,US +1120495872,1120496223,US 1120496224,1120496255,BE 1120496256,1120496287,CA 1120496288,1120496385,US @@ -15877,7 +16146,10 @@ 1120498944,1120499199,CA 1120499200,1120499487,US 1120499488,1120499519,CA -1120499520,1120500639,US +1120499520,1120499583,US +1120499584,1120499615,FR +1120499616,1120499647,IN +1120499648,1120500639,US 1120500640,1120500671,BE 1120500672,1120501247,US 1120501248,1120501503,CA @@ -15924,7 +16196,9 @@ 1120862208,1120870399,US 1120874496,1120874519,A2 1120874520,1120874527,US -1120874528,1120874655,A2 +1120874528,1120874631,A2 +1120874632,1120874639,US +1120874640,1120874655,A2 1120874656,1120874703,US 1120874704,1120874719,CA 1120874720,1120874727,A2 @@ -16077,9 +16351,7 @@ 1121250208,1121250239,CA 1121250240,1121250303,GB 1121250304,1121250815,BZ -1121250816,1121250831,AG -1121250832,1121250847,CA -1121250848,1121250879,AG +1121250816,1121250879,AG 1121250880,1121250911,CA 1121250912,1121250927,VG 1121250928,1121250935,CA @@ -16114,7 +16386,7 @@ 1121251776,1121251791,CA 1121251792,1121251807,VG 1121251808,1121251831,CA -1121251832,1121251839,AG +1121251832,1121251839,VG 1121251840,1121251847,BZ 1121251848,1121251871,CA 1121251872,1121251887,VG @@ -16127,9 +16399,7 @@ 1121251984,1121251991,CA 1121251992,1121251999,AG 1121252000,1121252063,CA -1121252064,1121252095,AG -1121252096,1121252351,CA -1121252352,1121252359,AG +1121252064,1121252359,AG 1121252360,1121252375,CA 1121252376,1121252383,PA 1121252384,1121252391,CR @@ -16155,9 +16425,294 @@ 1121254232,1121254255,CR 1121254256,1121254271,AG 1121254272,1121255423,CA -1121255424,1121652991,US -1121652992,1121653119,EE -1121653120,1121655071,US +1121255424,1121468463,US +1121468464,1121468479,AR +1121468480,1121468495,US +1121468496,1121468511,PK +1121468512,1121468519,AU +1121468520,1121468527,IN +1121468528,1121468551,US +1121468552,1121468559,CA +1121468560,1121468567,US +1121468568,1121468575,AU +1121468576,1121468591,US +1121468592,1121468600,AU +1121468601,1121468911,US +1121468912,1121468927,GB +1121468928,1121468967,US +1121468968,1121468975,CA +1121468976,1121469175,US +1121469176,1121469183,CA +1121469184,1121469367,US +1121469368,1121469375,CA +1121469376,1121469839,US +1121469840,1121469847,AU +1121469848,1121469871,US +1121469872,1121469879,SE +1121469880,1121469895,US +1121469896,1121469903,GB +1121469904,1121469911,US +1121469912,1121469919,PH +1121469920,1121470015,US +1121470016,1121470047,AU +1121470048,1121470175,US +1121470176,1121470191,CA +1121470192,1121470215,US +1121470216,1121470223,CA +1121470224,1121470263,US +1121470264,1121470271,CA +1121470272,1121470311,US +1121470312,1121470319,IL +1121470320,1121470423,US +1121470424,1121470463,NL +1121470464,1121470551,US +1121470552,1121470559,LB +1121470560,1121470583,US +1121470584,1121470591,AU +1121470592,1121470703,US +1121470704,1121470711,VU +1121470712,1121470727,US +1121470728,1121470743,CA +1121470744,1121470759,US +1121470760,1121470767,FR +1121470768,1121470815,US +1121470816,1121470831,AU +1121470832,1121470847,US +1121470848,1121470863,AU +1121470864,1121470871,US +1121470872,1121470879,AU +1121470880,1121471063,US +1121471064,1121471071,AU +1121471072,1121471087,US +1121471088,1121471095,AU +1121471096,1121471127,US +1121471128,1121471135,AU +1121471136,1121471151,US +1121471152,1121471159,CH +1121471160,1121471167,US +1121471168,1121471175,CA +1121471176,1121471279,US +1121471280,1121471295,GB +1121471296,1121471375,US +1121471376,1121471391,DK +1121471392,1121471471,US +1121471472,1121471487,CA +1121471488,1121471503,MY +1121471504,1121471511,US +1121471512,1121471519,BR +1121471520,1121471527,US +1121471528,1121471535,EC +1121471536,1121471559,US +1121471560,1121471575,CA +1121471576,1121471591,US +1121471592,1121471599,CA +1121471600,1121471623,US +1121471624,1121471631,CA +1121471632,1121471655,US +1121471656,1121471663,CA +1121471664,1121471671,AR +1121471672,1121471703,US +1121471704,1121471711,JP +1121471712,1121471719,UA +1121471720,1121471727,IT +1121471728,1121471935,US +1121471936,1121471943,CA +1121471944,1121471951,IN +1121471952,1121471975,US +1121471976,1121471983,CA +1121471984,1121471999,US +1121472000,1121472007,BE +1121472008,1121472047,US +1121472048,1121472055,MY +1121472056,1121472263,US +1121472264,1121472271,LK +1121472272,1121472287,US +1121472288,1121472303,IN +1121472304,1121472375,US +1121472376,1121472447,AU +1121472448,1121472463,US +1121472464,1121472471,CA +1121472472,1121472479,ES +1121472480,1121472511,US +1121472512,1121472519,NZ +1121472520,1121472535,CA +1121472536,1121472559,US +1121472560,1121472567,IN +1121472568,1121472655,US +1121472656,1121472663,UA +1121472664,1121472727,US +1121472728,1121472735,CA +1121472736,1121472743,LB +1121472744,1121472751,GB +1121472752,1121472775,US +1121472776,1121472807,RU +1121472808,1121472831,US +1121472832,1121472839,LB +1121472840,1121472847,US +1121472848,1121472863,IN +1121472864,1121472871,US +1121472872,1121472879,SE +1121472880,1121472887,IT +1121472888,1121472927,US +1121472928,1121472935,GB +1121472936,1121473175,US +1121473176,1121473183,FR +1121473184,1121473199,US +1121473200,1121473207,GB +1121473208,1121473239,US +1121473240,1121473247,ES +1121473248,1121473279,US +1121473280,1121473295,AU +1121473296,1121473303,US +1121473304,1121473311,AU +1121473312,1121473327,US +1121473328,1121473335,AG +1121473336,1121473439,US +1121473440,1121473471,AU +1121473472,1121473479,US +1121473480,1121473487,CA +1121473488,1121473599,US +1121473600,1121473607,GB +1121473608,1121473615,CA +1121473616,1121473871,US +1121473872,1121473887,NL +1121473888,1121474007,US +1121474008,1121474015,DE +1121474016,1121474031,US +1121474032,1121474039,AG +1121474040,1121474063,US +1121474064,1121474071,GB +1121474072,1121474079,SI +1121474080,1121474095,US +1121474096,1121474111,CA +1121474112,1121474151,US +1121474152,1121474159,CA +1121474160,1121474303,US +1121474304,1121474319,VE +1121474320,1121474351,US +1121474352,1121474359,NZ +1121474360,1121474367,US +1121474368,1121474375,AU +1121474376,1121474383,US +1121474384,1121474415,AU +1121474416,1121474447,US +1121474448,1121474455,AU +1121474456,1121474511,US +1121474512,1121474527,UA +1121474528,1121474535,US +1121474536,1121474543,GB +1121474544,1121474591,US +1121474592,1121474607,CA +1121474608,1121474927,US +1121474928,1121474935,AU +1121474936,1121475135,US +1121475136,1121475151,FR +1121475152,1121475199,US +1121475200,1121475215,IN +1121475216,1121475295,US +1121475296,1121475327,BD +1121475328,1121475967,US +1121475968,1121476031,AU +1121476032,1121476191,US +1121476192,1121476223,BD +1121476224,1121476431,US +1121476432,1121476439,JP +1121476440,1121476927,US +1121476928,1121476935,IN +1121476936,1121477391,US +1121477392,1121477399,IN +1121477400,1121477415,US +1121477416,1121477423,NZ +1121477424,1121477431,US +1121477432,1121477439,IE +1121477440,1121477447,ID +1121477448,1121477551,US +1121477552,1121477567,CA +1121477568,1121477599,US +1121477600,1121477607,AU +1121477608,1121477935,US +1121477936,1121477943,CA +1121477944,1121478023,US +1121478024,1121478031,GB +1121478032,1121478079,US +1121478080,1121478087,CA +1121478088,1121478095,US +1121478096,1121478111,CA +1121478112,1121478167,US +1121478168,1121478175,AU +1121478176,1121478207,US +1121478208,1121478239,GB +1121478240,1121478343,US +1121478344,1121478367,AU +1121478368,1121478463,US +1121478464,1121478471,GR +1121478472,1121478495,US +1121478496,1121478503,NO +1121478504,1121478511,US +1121478512,1121478519,RU +1121478520,1121478527,KE +1121478528,1121478591,US +1121478592,1121478655,AU +1121478656,1121478687,US +1121478688,1121478703,NZ +1121478704,1121478967,US +1121478968,1121478975,AU +1121478976,1121478991,US +1121478992,1121478999,IN +1121479000,1121479023,US +1121479024,1121479031,CY +1121479032,1121479143,US +1121479144,1121479167,EG +1121479168,1121479215,US +1121479216,1121479231,CA +1121479232,1121479295,US +1121479296,1121479303,HK +1121479304,1121479343,CA +1121479344,1121479359,US +1121479360,1121479367,NZ +1121479368,1121479407,US +1121479408,1121479415,CA +1121479416,1121479679,US +1121479680,1121479687,GB +1121479688,1121479695,CA +1121479696,1121479759,US +1121479760,1121479783,AU +1121479784,1121479831,US +1121479832,1121479839,NZ +1121479840,1121479847,IN +1121479848,1121479871,US +1121479872,1121479887,PK +1121479888,1121479919,US +1121479920,1121479927,AU +1121479928,1121480783,US +1121480784,1121480791,TH +1121480792,1121480799,US +1121480800,1121480807,CA +1121480808,1121480847,US +1121480848,1121480855,CY +1121480856,1121480903,US +1121480904,1121480911,HK +1121480912,1121480959,US +1121480960,1121480967,AU +1121480968,1121480991,US +1121480992,1121481023,AU +1121481024,1121481039,US +1121481040,1121481047,AU +1121481048,1121481055,US +1121481056,1121481087,AU +1121481088,1121481239,US +1121481240,1121481247,AU +1121481248,1121481407,US +1121481408,1121481415,GB +1121481416,1121481439,US +1121481440,1121481447,AT +1121481448,1121482799,US +1121482800,1121482807,AU +1121482808,1121483567,US +1121483568,1121483575,PT +1121483576,1121483759,US +1121483760,1121483775,FR +1121483776,1121655071,US 1121655072,1121655103,RU 1121655104,1121655231,US 1121655232,1121655263,AR @@ -16165,9 +16720,7 @@ 1121655440,1121655447,VE 1121655448,1121656063,US 1121656064,1121656095,NZ -1121656096,1121656159,US -1121656160,1121656191,NZ -1121656192,1121714975,US +1121656096,1121714975,US 1121714976,1121714991,AU 1121714992,1121715967,US 1121715968,1121716095,IT @@ -16233,7 +16786,9 @@ 1122451456,1122455551,CO 1122455552,1122476031,US 1122476032,1122480127,PR -1122480128,1122480207,US +1122480128,1122480159,US +1122480160,1122480167,RU +1122480168,1122480207,US 1122480208,1122480223,GB 1122480224,1122480231,PL 1122480232,1122480343,US @@ -16242,7 +16797,9 @@ 1122480416,1122480423,CL 1122480424,1122480447,US 1122480448,1122480455,PR -1122480456,1122481231,US +1122480456,1122480479,US +1122480480,1122480487,CA +1122480488,1122481231,US 1122481232,1122481247,GB 1122481248,1122493055,US 1122493056,1122493183,ES @@ -16314,13 +16871,7 @@ 1123584704,1123584735,CA 1123584736,1123588063,US 1123588064,1123588095,CY -1123588096,1123589631,US -1123589632,1123589663,CA -1123589664,1123589759,US -1123589760,1123589775,CA -1123589776,1123589791,US -1123589792,1123589823,CA -1123589824,1123590143,US +1123588096,1123590143,US 1123590144,1123598335,VI 1123598336,1123651583,US 1123651584,1123655679,JM @@ -16533,7 +17084,9 @@ 1125479232,1125498879,US 1125498880,1125508095,CA 1125508096,1125508351,PA -1125508352,1125515263,CA +1125508352,1125514495,CA +1125514496,1125514751,US +1125514752,1125515263,CA 1125515264,1125531647,US 1125531648,1125543935,CA 1125543936,1125545983,US @@ -16698,7 +17251,9 @@ 1133785376,1133785383,NE 1133785384,1133785471,US 1133785472,1133785503,GB -1133785504,1134029055,US +1133785504,1133785567,US +1133785568,1133785599,BE +1133785600,1134029055,US 1134029056,1134029567,A2 1134029568,1134444543,US 1134444544,1134448639,CA @@ -16714,9 +17269,725 @@ 1136517008,1136517023,GB 1136517024,1136517039,US 1136517040,1136517055,GB -1136517056,1136721919,US +1136517056,1136656503,US +1136656504,1136656511,CA +1136656512,1136656583,US +1136656584,1136656591,GB +1136656592,1136656615,US +1136656616,1136656631,AU +1136656632,1136656863,US +1136656864,1136656895,CA +1136656896,1136657039,US +1136657040,1136657047,CA +1136657048,1136657615,US +1136657616,1136657623,GB +1136657624,1136657775,US +1136657776,1136657783,CA +1136657784,1136657815,US +1136657816,1136657823,CA +1136657824,1136657871,US +1136657872,1136657887,GT +1136657888,1136657911,US +1136657912,1136657919,CA +1136657920,1136658087,US +1136658088,1136658095,CN +1136658096,1136658111,MX +1136658112,1136658119,US +1136658120,1136658127,JP +1136658128,1136658135,US +1136658136,1136658143,AU +1136658144,1136658151,US +1136658152,1136658159,AU +1136658160,1136658439,US +1136658440,1136658447,MX +1136658448,1136658711,US +1136658712,1136658719,IN +1136658720,1136658727,NZ +1136658728,1136658991,US +1136658992,1136658999,AU +1136659000,1136659023,US +1136659024,1136659031,AU +1136659032,1136659039,US +1136659040,1136659047,AU +1136659048,1136659151,US +1136659152,1136659167,CA +1136659168,1136659599,US +1136659600,1136659607,MX +1136659608,1136659639,US +1136659640,1136659647,FR +1136659648,1136659767,US +1136659768,1136659775,CA +1136659776,1136659927,US +1136659928,1136659935,CA +1136659936,1136659967,US +1136659968,1136659975,CA +1136659976,1136660047,US +1136660048,1136660055,GB +1136660056,1136660223,US +1136660224,1136660231,CA +1136660232,1136660239,US +1136660240,1136660247,GB +1136660248,1136660471,US +1136660472,1136660479,GB +1136660480,1136660487,US +1136660488,1136660495,NO +1136660496,1136660511,US +1136660512,1136660519,HR +1136660520,1136660759,US +1136660760,1136660767,CA +1136660768,1136660775,US +1136660776,1136660791,CA +1136660792,1136660823,US +1136660824,1136660839,CA +1136660840,1136660871,US +1136660872,1136660879,AU +1136660880,1136660943,US +1136660944,1136660951,IN +1136660952,1136661295,US +1136661296,1136661303,TW +1136661304,1136661327,US +1136661328,1136661335,IN +1136661336,1136661343,US +1136661344,1136661351,CN +1136661352,1136661367,US +1136661368,1136661375,IN +1136661376,1136661399,US +1136661400,1136661407,MX +1136661408,1136661631,US +1136661632,1136661639,IN +1136661640,1136661687,US +1136661688,1136661695,GB +1136661696,1136661727,US +1136661728,1136661743,CA +1136661744,1136661919,US +1136661920,1136661951,CA +1136661952,1136662087,US +1136662088,1136662095,CA +1136662096,1136662151,US +1136662152,1136662159,MX +1136662160,1136662575,US +1136662576,1136662583,CA +1136662584,1136662663,US +1136662664,1136662671,CA +1136662672,1136662815,US +1136662816,1136662823,IN +1136662824,1136662911,US +1136662912,1136662919,IN +1136662920,1136663247,US +1136663248,1136663255,CL +1136663256,1136663575,US +1136663576,1136663583,CN +1136663584,1136663679,US +1136663680,1136663687,IN +1136663688,1136663695,JP +1136663696,1136664543,US +1136664544,1136664551,AU +1136664552,1136664639,US +1136664640,1136664647,CA +1136664648,1136664791,US +1136664792,1136664799,GB +1136664800,1136664815,US +1136664816,1136664823,IL +1136664824,1136664847,US +1136664848,1136664863,CA +1136664864,1136665103,US +1136665104,1136665111,AU +1136665112,1136665135,US +1136665136,1136665143,GB +1136665144,1136665655,US +1136665656,1136665663,IN +1136665664,1136665735,US +1136665736,1136665743,GB +1136665744,1136665759,US +1136665760,1136665767,CA +1136665768,1136665831,US +1136665832,1136665839,CA +1136665840,1136665919,US +1136665920,1136665927,DE +1136665928,1136666143,US +1136666144,1136666151,CA +1136666152,1136666223,US +1136666224,1136666231,PT +1136666232,1136666351,US +1136666352,1136666359,IL +1136666360,1136666447,US +1136666448,1136666463,GB +1136666464,1136666559,US +1136666560,1136666575,GB +1136666576,1136666743,US +1136666744,1136666751,CA +1136666752,1136666791,US +1136666792,1136666799,IL +1136666800,1136666863,US +1136666864,1136666871,GB +1136666872,1136666991,US +1136666992,1136667007,CA +1136667008,1136667087,US +1136667088,1136667103,IL +1136667104,1136667447,US +1136667448,1136667455,GB +1136667456,1136667551,US +1136667552,1136667559,AU +1136667560,1136667599,US +1136667600,1136667615,IN +1136667616,1136667655,US +1136667656,1136667663,CA +1136667664,1136667759,US +1136667760,1136667767,KW +1136667768,1136668023,US +1136668024,1136668031,GB +1136668032,1136668127,US +1136668128,1136668159,AU +1136668160,1136668191,US +1136668192,1136668207,MX +1136668208,1136668287,US +1136668288,1136668303,IN +1136668304,1136668319,MX +1136668320,1136668335,US +1136668336,1136668351,MX +1136668352,1136668383,US +1136668384,1136668415,IN +1136668416,1136668447,US +1136668448,1136668455,IN +1136668456,1136668559,US +1136668560,1136668567,CA +1136668568,1136668671,US +1136668672,1136668679,IN +1136668680,1136668887,US +1136668888,1136668895,CL +1136668896,1136669143,US +1136669144,1136669151,IN +1136669152,1136669263,US +1136669264,1136669279,GB +1136669280,1136669343,US +1136669344,1136669359,CO +1136669360,1136669487,US +1136669488,1136669495,CA +1136669496,1136669511,US +1136669512,1136669519,CA +1136669520,1136669567,US +1136669568,1136669575,CA +1136669576,1136669583,MX +1136669584,1136669775,US +1136669776,1136669783,NL +1136669784,1136669911,US +1136669912,1136669919,ES +1136669920,1136670063,US +1136670064,1136670071,SG +1136670072,1136670111,US +1136670112,1136670119,NZ +1136670120,1136670151,US +1136670152,1136670159,SG +1136670160,1136670719,US +1136670720,1136670975,GB +1136670976,1136672999,US +1136673000,1136673007,CA +1136673008,1136673039,US +1136673040,1136673047,IN +1136673048,1136673231,US +1136673232,1136673239,GB +1136673240,1136673279,US +1136673280,1136673287,CA +1136673288,1136673327,US +1136673328,1136673335,CA +1136673336,1136673423,US +1136673424,1136673431,CA +1136673432,1136673559,US +1136673560,1136673567,VE +1136673568,1136673703,US +1136673704,1136673711,MX +1136673712,1136673855,US +1136673856,1136673863,CA +1136673864,1136673903,US +1136673904,1136673919,CA +1136673920,1136673927,GB +1136673928,1136674079,US +1136674080,1136674095,AU +1136674096,1136674447,US +1136674448,1136674463,AU +1136674464,1136674583,US +1136674584,1136674591,CA +1136674592,1136674607,MX +1136674608,1136674807,US +1136674808,1136674815,GB +1136674816,1136674863,US +1136674864,1136674871,IN +1136674872,1136674895,US +1136674896,1136674903,IL +1136674904,1136674911,GB +1136674912,1136674967,US +1136674968,1136674975,CA +1136674976,1136675007,US +1136675008,1136675015,AU +1136675016,1136675023,CL +1136675024,1136675055,US +1136675056,1136675063,ES +1136675064,1136675311,US +1136675312,1136675319,GB +1136675320,1136675751,US +1136675752,1136675759,IN +1136675760,1136675815,US +1136675816,1136675823,AE +1136675824,1136675839,US +1136675840,1136676095,GB +1136676096,1136676127,US +1136676128,1136676143,IE +1136676144,1136676255,US +1136676256,1136676271,CA +1136676272,1136676551,US +1136676552,1136676559,GB +1136676560,1136676583,US +1136676584,1136676591,AU +1136676592,1136676671,US +1136676672,1136676679,CA +1136676680,1136676735,US +1136676736,1136676743,AU +1136676744,1136677375,US +1136677376,1136677391,AU +1136677392,1136677463,US +1136677464,1136677471,CA +1136677472,1136677615,US +1136677616,1136677631,CA +1136677632,1136678103,US +1136678104,1136678111,IN +1136678112,1136678303,US +1136678304,1136678319,MX +1136678320,1136678431,US +1136678432,1136678447,GB +1136678448,1136679071,US +1136679072,1136679087,CA +1136679088,1136679135,US +1136679136,1136679167,BR +1136679168,1136679215,US +1136679216,1136679223,GB +1136679224,1136679231,US +1136679232,1136679263,IL +1136679264,1136679279,US +1136679280,1136679295,CA +1136679296,1136680055,US +1136680056,1136680063,CA +1136680064,1136680143,US +1136680144,1136680151,NO +1136680152,1136680159,US +1136680160,1136680175,BD +1136680176,1136680311,US +1136680312,1136680319,IN +1136680320,1136680743,US +1136680744,1136680751,BD +1136680752,1136680759,IN +1136680760,1136681167,US +1136681168,1136681175,CA +1136681176,1136681191,US +1136681192,1136681199,MX +1136681200,1136681215,US +1136681216,1136681223,HN +1136681224,1136681231,US +1136681232,1136681239,IL +1136681240,1136681311,US +1136681312,1136681319,BR +1136681320,1136681439,US +1136681440,1136681447,GB +1136681448,1136681455,US +1136681456,1136681471,IL +1136681472,1136681487,US +1136681488,1136681495,GB +1136681496,1136681559,US +1136681560,1136681567,CA +1136681568,1136682271,US +1136682272,1136682303,GB +1136682304,1136682495,US +1136682496,1136682503,CA +1136682504,1136682527,US +1136682528,1136682559,CA +1136682560,1136682583,US +1136682584,1136682591,CA +1136682592,1136682695,US +1136682696,1136682703,CA +1136682704,1136682863,US +1136682864,1136682871,CA +1136682872,1136683095,US +1136683096,1136683103,IL +1136683104,1136683127,US +1136683128,1136683135,MX +1136683136,1136683367,US +1136683368,1136683375,GB +1136683376,1136683383,ZA +1136683384,1136683527,US +1136683528,1136683535,AU +1136683536,1136683543,US +1136683544,1136683551,AE +1136683552,1136684047,US +1136684048,1136684063,CA +1136684064,1136684087,US +1136684088,1136684095,MX +1136684096,1136684103,US +1136684104,1136684111,CA +1136684112,1136684231,US +1136684232,1136684239,CA +1136684240,1136684247,IN +1136684248,1136684463,US +1136684464,1136684471,CA +1136684472,1136684559,US +1136684560,1136684575,CA +1136684576,1136684607,US +1136684608,1136684615,CA +1136684616,1136684623,US +1136684624,1136684631,IN +1136684632,1136684663,US +1136684664,1136684671,AU +1136684672,1136684679,GB +1136684680,1136684703,US +1136684704,1136684719,MX +1136684720,1136684735,US +1136684736,1136684751,IN +1136684752,1136684807,US +1136684808,1136684815,IT +1136684816,1136684895,US +1136684896,1136684911,CA +1136684912,1136684919,HR +1136684920,1136684975,US +1136684976,1136684983,CA +1136684984,1136684991,US +1136684992,1136684999,JP +1136685000,1136685055,US +1136685056,1136685183,CA +1136685184,1136685191,US +1136685192,1136685199,BM +1136685200,1136685231,US +1136685232,1136685239,CN +1136685240,1136685247,US +1136685248,1136685263,CA +1136685264,1136685791,US +1136685792,1136685799,IL +1136685800,1136685871,US +1136685872,1136685879,CA +1136685880,1136685903,US +1136685904,1136685911,HR +1136685912,1136685935,US +1136685936,1136685943,IT +1136685944,1136686047,US +1136686048,1136686055,IE +1136686056,1136689159,US +1136689160,1136689167,BR +1136689168,1136689271,US +1136689272,1136689279,MX +1136689280,1136689431,US +1136689432,1136689439,BM +1136689440,1136689503,US +1136689504,1136689511,VE +1136689512,1136689551,US +1136689552,1136689559,IN +1136689560,1136689735,US +1136689736,1136689743,GB +1136689744,1136689815,US +1136689816,1136689823,GB +1136689824,1136689951,US +1136689952,1136689959,CA +1136689960,1136689975,US +1136689976,1136689983,IL +1136689984,1136690023,US +1136690024,1136690031,IN +1136690032,1136690055,US +1136690056,1136690063,CA +1136690064,1136690079,US +1136690080,1136690087,AU +1136690088,1136690407,US +1136690408,1136690415,BD +1136690416,1136690847,US +1136690848,1136690863,GB +1136690864,1136691415,US +1136691416,1136691423,CA +1136691424,1136691935,US +1136691936,1136691943,CA +1136691944,1136692039,US +1136692040,1136692047,AU +1136692048,1136692071,US +1136692072,1136692079,CA +1136692080,1136692239,US +1136692240,1136692247,AR +1136692248,1136692319,US +1136692320,1136692327,NL +1136692328,1136692535,US +1136692536,1136692543,GB +1136692544,1136692799,US +1136692800,1136692807,NZ +1136692808,1136692919,US +1136692920,1136692927,IN +1136692928,1136692991,US +1136692992,1136693055,IR +1136693056,1136693063,AU +1136693064,1136693503,US +1136693504,1136693511,GB +1136693512,1136693519,CA +1136693520,1136693887,US +1136693888,1136693903,CA +1136693904,1136693951,US +1136693952,1136693959,CA +1136693960,1136693967,AU +1136693968,1136693975,US +1136693976,1136693983,GB +1136693984,1136694079,US +1136694080,1136694087,BG +1136694088,1136694263,US +1136694264,1136694271,GB +1136694272,1136694303,US +1136694304,1136694311,KR +1136694312,1136694319,US +1136694320,1136694327,KR +1136694328,1136694399,US +1136694400,1136694415,CA +1136694416,1136694895,US +1136694896,1136694903,AU +1136694904,1136694991,US +1136694992,1136694999,GB +1136695000,1136695503,US +1136695504,1136695511,GB +1136695512,1136695703,US +1136695704,1136695711,DK +1136695712,1136695719,US +1136695720,1136695727,IL +1136695728,1136695743,DK +1136695744,1136696087,US +1136696088,1136696095,AU +1136696096,1136696111,US +1136696112,1136696127,AU +1136696128,1136696263,US +1136696264,1136696271,CA +1136696272,1136696647,US +1136696648,1136696655,GB +1136696656,1136696671,US +1136696672,1136696679,IL +1136696680,1136696711,US +1136696712,1136696719,GT +1136696720,1136696767,US +1136696768,1136696775,IL +1136696776,1136696855,US +1136696856,1136696863,GB +1136696864,1136697047,US +1136697048,1136697055,BM +1136697056,1136697455,US +1136697456,1136697463,CA +1136697464,1136697647,US +1136697648,1136697655,CA +1136697656,1136697767,US +1136697768,1136697775,GB +1136697776,1136697975,US +1136697976,1136697983,CA +1136697984,1136698023,US +1136698024,1136698031,GB +1136698032,1136698039,US +1136698040,1136698111,IE +1136698112,1136698815,US +1136698816,1136698823,GB +1136698824,1136699023,US +1136699024,1136699039,GB +1136699040,1136699087,US +1136699088,1136699095,GB +1136699096,1136699487,US +1136699488,1136699503,CA +1136699504,1136699511,US +1136699512,1136699519,SE +1136699520,1136699583,US +1136699584,1136699599,CA +1136699600,1136699623,US +1136699624,1136699631,CA +1136699632,1136699647,IN +1136699648,1136699695,US +1136699696,1136699703,AU +1136699704,1136699839,US +1136699840,1136699847,MT +1136699848,1136699935,US +1136699936,1136699943,AU +1136699944,1136699999,US +1136700000,1136700007,GB +1136700008,1136700031,US +1136700032,1136700039,CA +1136700040,1136700103,US +1136700104,1136700111,NZ +1136700112,1136700263,US +1136700264,1136700271,IL +1136700272,1136700343,US +1136700344,1136700351,MX +1136700352,1136700479,US +1136700480,1136700487,NZ +1136700488,1136700583,US +1136700584,1136700591,AU +1136700592,1136700663,US +1136700664,1136700671,DK +1136700672,1136700783,US +1136700784,1136700791,CA +1136700792,1136700847,US +1136700848,1136700855,CL +1136700856,1136700911,US +1136700912,1136700919,CA +1136700920,1136701119,US +1136701120,1136701183,AU +1136701184,1136701279,US +1136701280,1136701287,CA +1136701288,1136701335,US +1136701336,1136701343,GB +1136701344,1136701455,US +1136701456,1136701463,IN +1136701464,1136701527,US +1136701528,1136701543,CA +1136701544,1136702711,US +1136702712,1136702719,GB +1136702720,1136702727,CA +1136702728,1136702799,US +1136702800,1136702807,GB +1136702808,1136702871,US +1136702872,1136702879,AT +1136702880,1136702911,US +1136702912,1136702919,GB +1136702920,1136703095,US +1136703096,1136703103,CA +1136703104,1136703279,US +1136703280,1136703287,CA +1136703288,1136704639,US +1136704640,1136704703,GB +1136704704,1136705575,US +1136705576,1136705583,CA +1136705584,1136705671,US +1136705672,1136705679,CA +1136705680,1136705807,US +1136705808,1136705815,IN +1136705816,1136705823,RU +1136705824,1136706087,US +1136706088,1136706095,IN +1136706096,1136706119,US +1136706120,1136706127,CA +1136706128,1136706231,US +1136706232,1136706239,MX +1136706240,1136706359,US +1136706360,1136706367,BM +1136706368,1136706455,US +1136706456,1136706463,GB +1136706464,1136706471,CA +1136706472,1136706495,US +1136706496,1136706559,IE +1136706560,1136706591,US +1136706592,1136706599,AU +1136706600,1136706703,US +1136706704,1136706711,CA +1136706712,1136706935,US +1136706936,1136706943,AU +1136706944,1136707087,US +1136707088,1136707095,GB +1136707096,1136707175,US +1136707176,1136707183,ZA +1136707184,1136707263,US +1136707264,1136707271,IN +1136707272,1136707751,US +1136707752,1136707759,GB +1136707760,1136707775,US +1136707776,1136707783,BR +1136707784,1136707847,US +1136707848,1136707855,SG +1136707856,1136707887,US +1136707888,1136707895,CA +1136707896,1136708039,US +1136708040,1136708047,CA +1136708048,1136708679,US +1136708680,1136708687,GB +1136708688,1136708871,US +1136708872,1136708879,GB +1136708880,1136709023,US +1136709024,1136709031,BD +1136709032,1136709647,US +1136709648,1136709655,CA +1136709656,1136709679,US +1136709680,1136709695,AU +1136709696,1136709767,US +1136709768,1136709775,GB +1136709776,1136709815,US +1136709816,1136709823,CA +1136709824,1136709927,US +1136709928,1136709935,CA +1136709936,1136710191,US +1136710192,1136710199,GB +1136710200,1136710415,US +1136710416,1136710431,AU +1136710432,1136711479,US +1136711480,1136711487,ZA +1136711488,1136711927,US +1136711928,1136711935,GB +1136711936,1136711991,US +1136711992,1136711999,AU +1136712000,1136712607,US +1136712608,1136712615,GB +1136712616,1136712695,US +1136712696,1136712703,CA +1136712704,1136712991,US +1136712992,1136712999,CA +1136713000,1136713087,US +1136713088,1136713103,CA +1136713104,1136713151,US +1136713152,1136713167,CA +1136713168,1136713471,US +1136713472,1136713479,SG +1136713480,1136713583,US +1136713584,1136713591,CA +1136713592,1136713919,US +1136713920,1136713935,NO +1136713936,1136714031,US +1136714032,1136714039,BR +1136714040,1136714167,US +1136714168,1136714175,IE +1136714176,1136714255,US +1136714256,1136714263,GB +1136714264,1136714271,US +1136714272,1136714279,GB +1136714280,1136714639,US +1136714640,1136714647,DE +1136714648,1136714671,US +1136714672,1136714679,CN +1136714680,1136715135,US +1136715136,1136715151,MX +1136715152,1136715623,US +1136715624,1136715631,AU +1136715632,1136716287,US +1136716288,1136716295,ZA +1136716296,1136716431,US +1136716432,1136716439,IN +1136716440,1136716447,US +1136716448,1136716479,CA +1136716480,1136716631,US +1136716632,1136716639,AU +1136716640,1136716703,US +1136716704,1136716711,IN +1136716712,1136716759,US +1136716760,1136716767,CA +1136716768,1136716775,US +1136716776,1136716783,NZ +1136716784,1136716799,US +1136716800,1136716831,VI +1136716832,1136717167,US +1136717168,1136717183,IT +1136717184,1136717399,US +1136717400,1136717407,GB +1136717408,1136717551,US +1136717552,1136717559,CA +1136717560,1136717583,US +1136717584,1136717599,GB +1136717600,1136717935,US +1136717936,1136717943,CA +1136717944,1136718391,US +1136718392,1136718399,MX +1136718400,1136718439,US +1136718440,1136718447,IN +1136718448,1136718487,US +1136718488,1136718495,IN +1136718496,1136718759,US +1136718760,1136718767,AU +1136718768,1136718791,US +1136718792,1136718799,CA +1136718800,1136718879,US +1136718880,1136718895,AU +1136718896,1136721919,US 1136721920,1136787455,CA -1136787456,1137134083,US +1136787456,1137090815,US +1137090816,1137091327,NZ +1137091328,1137134083,US 1137134084,1137134099,CA 1137134100,1137134283,US 1137134284,1137134331,CA @@ -16736,8 +18007,8 @@ 1137283072,1137287167,US 1137287168,1137295359,CA 1137295360,1137337599,US -1137337600,1137338367,CA -1137338368,1137369087,US +1137337600,1137337855,CA +1137337856,1137369087,US 1137369088,1137369519,CA 1137369520,1137369535,US 1137369536,1137370111,CA @@ -16746,8 +18017,10 @@ 1137376336,1137376351,US 1137376352,1137376367,CA 1137376368,1137376383,US -1137376384,1137376447,CA -1137376448,1137377471,US +1137376384,1137376463,CA +1137376464,1137377311,US +1137377312,1137377327,DE +1137377328,1137377471,US 1137377472,1137377479,IN 1137377480,1137383455,US 1137383456,1137383471,DE @@ -16764,7 +18037,7 @@ 1137704968,1137705287,CA 1137705288,1137705295,US 1137705296,1137705383,CA -1137705384,1137705391,US +1137705384,1137705391,NL 1137705392,1137705447,CA 1137705448,1137705455,US 1137705456,1137705727,CA @@ -16789,17 +18062,15 @@ 1137706808,1137706823,US 1137706824,1137706831,CA 1137706832,1137706839,US -1137706840,1137706871,CA -1137706872,1137706887,US +1137706840,1137706879,CA +1137706880,1137706887,US 1137706888,1137706903,CA 1137706904,1137706943,US 1137706944,1137706991,CA 1137706992,1137707007,US 1137707008,1137707071,CA 1137707072,1137707095,US -1137707096,1137707103,CA -1137707104,1137707111,NL -1137707112,1137707127,CA +1137707096,1137707127,CA 1137707128,1137707143,US 1137707144,1137707151,CA 1137707152,1137707159,NL @@ -16810,8 +18081,8 @@ 1137707224,1137707255,CA 1137707256,1137707263,US 1137707264,1137707519,CA -1137707520,1137707567,US -1137707568,1137707583,NL +1137707520,1137707575,US +1137707576,1137707583,NL 1137707584,1137707623,CA 1137707624,1137707647,US 1137707648,1137707663,CA @@ -16892,8 +18163,8 @@ 1137711280,1137711287,NL 1137711288,1137711295,US 1137711296,1137711303,NL -1137711304,1137711351,US -1137711352,1137711367,CA +1137711304,1137711343,US +1137711344,1137711367,CA 1137711368,1137711375,US 1137711376,1137711399,CA 1137711400,1137711407,US @@ -16912,7 +18183,9 @@ 1137711656,1137711687,CA 1137711688,1137711703,US 1137711704,1137711743,CA -1137711744,1137711767,US +1137711744,1137711751,US +1137711752,1137711759,CA +1137711760,1137711767,US 1137711768,1137711791,CA 1137711792,1137711807,US 1137711808,1137711863,CA @@ -16952,7 +18225,15 @@ 1137713072,1137713087,CA 1137713088,1137713103,US 1137713104,1137713111,CA -1137713112,1137758207,US +1137713112,1137724495,US +1137724496,1137724511,CA +1137724512,1137724543,US +1137724544,1137724575,CA +1137724576,1137724655,US +1137724656,1137724687,CA +1137724688,1137724847,US +1137724848,1137724863,CA +1137724864,1137758207,US 1137758208,1137758463,GB 1137758464,1137758495,US 1137758496,1137758527,AU @@ -17260,7 +18541,9 @@ 1138655520,1138655521,MX 1138655522,1138655609,US 1138655610,1138655617,MX -1138655618,1138655715,US +1138655618,1138655695,US +1138655696,1138655703,MX +1138655704,1138655715,US 1138655716,1138655723,GB 1138655724,1138655841,US 1138655842,1138655849,CN @@ -17308,8 +18591,8 @@ 1138657006,1138657041,US 1138657042,1138657056,SA 1138657057,1138657123,US -1138657124,1138657133,MO -1138657134,1138657159,US +1138657124,1138657131,EG +1138657132,1138657159,US 1138657160,1138657173,MO 1138657174,1138657181,EG 1138657182,1138657193,US @@ -17479,8 +18762,7 @@ 1138662058,1138662107,EG 1138662108,1138662169,US 1138662170,1138662233,EG -1138662234,1138662241,IN -1138662242,1138662273,US +1138662234,1138662273,US 1138662274,1138662309,EG 1138662310,1138662409,US 1138662410,1138662473,EG @@ -17489,12 +18771,18 @@ 1138662538,1138662545,EG 1138662546,1138662575,US 1138662576,1138662607,MX -1138662608,1138663037,US +1138662608,1138662623,US +1138662624,1138662639,GB +1138662640,1138663037,US 1138663038,1138663166,GB 1138663167,1138712575,US 1138712576,1138713583,CA 1138713584,1138713599,IR -1138713600,1138714039,CA +1138713600,1138713927,CA +1138713928,1138713935,US +1138713936,1138713951,CA +1138713952,1138713959,US +1138713960,1138714039,CA 1138714040,1138714047,US 1138714048,1138714071,CA 1138714072,1138714079,US @@ -17504,7 +18792,9 @@ 1138720768,1138728959,CA 1138728960,1138774015,US 1138774016,1138778111,CA -1138778112,1138786303,US +1138778112,1138780159,US +1138780160,1138780191,CA +1138780192,1138786303,US 1138786304,1138819071,PR 1138819072,1138851839,CA 1138851840,1138851847,US @@ -17513,7 +18803,9 @@ 1138865563,1138865566,IN 1138865567,1138865578,US 1138865579,1138865586,IN -1138865587,1138879999,US +1138865587,1138874252,US +1138874253,1138874264,UY +1138874265,1138879999,US 1138880000,1138880511,CA 1138880512,1138882331,US 1138882332,1138882339,GB @@ -17554,7 +18846,7 @@ 1145315328,1145376767,US 1145376768,1145380863,CA 1145380864,1145405439,US -1145405440,1145409535,CA +1145405440,1145413631,CA 1145413632,1145421823,US 1145421824,1145430015,CA 1145430016,1145434111,US @@ -17624,7 +18916,7 @@ 1151945920,1151945935,JP 1151945936,1151946239,US 1151946240,1151946751,GB -1151946752,1151950847,US +1151946752,1151954943,US 1151959040,1152057343,US 1152057344,1152061439,CA 1152061440,1152073727,US @@ -17937,7 +19229,313 @@ 1158774784,1158791167,CA 1158791168,1158799359,BM 1158799360,1158807551,CA -1158807552,1158995967,US +1158807552,1158940823,US +1158940824,1158940831,ES +1158940832,1158940911,US +1158940912,1158940919,CA +1158940920,1158941247,US +1158941248,1158941255,CY +1158941256,1158941351,US +1158941352,1158941359,CR +1158941360,1158941367,CA +1158941368,1158941399,US +1158941400,1158941407,CA +1158941408,1158941775,US +1158941776,1158941791,JM +1158941792,1158941799,IE +1158941800,1158942271,US +1158942272,1158942279,IE +1158942280,1158942311,US +1158942312,1158942319,CA +1158942320,1158943271,US +1158943272,1158943279,TH +1158943280,1158943423,US +1158943424,1158943431,PK +1158943432,1158943535,US +1158943536,1158943543,CA +1158943544,1158943599,US +1158943600,1158943615,GB +1158943616,1158943863,US +1158943864,1158943871,CR +1158943872,1158943887,GB +1158943888,1158943895,US +1158943896,1158943903,GB +1158943904,1158943975,US +1158943976,1158943983,CA +1158943984,1158944279,US +1158944280,1158944287,AE +1158944288,1158944863,US +1158944864,1158944871,CA +1158944872,1158944895,US +1158944896,1158944903,ZA +1158944904,1158944951,US +1158944952,1158944959,IN +1158944960,1158945095,US +1158945096,1158945103,TH +1158945104,1158945111,RU +1158945112,1158945119,CA +1158945120,1158945271,US +1158945272,1158945287,CA +1158945288,1158945319,US +1158945320,1158945335,CA +1158945336,1158945535,US +1158945536,1158945543,CA +1158945544,1158945735,US +1158945736,1158945743,TR +1158945744,1158945791,US +1158945792,1158945815,IE +1158945816,1158945823,US +1158945824,1158945831,JP +1158945832,1158945935,US +1158945936,1158945943,CA +1158945944,1158946047,US +1158946048,1158946055,CA +1158946056,1158946063,US +1158946064,1158946071,IL +1158946072,1158946151,US +1158946152,1158946159,IE +1158946160,1158946207,US +1158946208,1158946215,CA +1158946216,1158946223,US +1158946224,1158946239,CR +1158946240,1158946335,US +1158946336,1158946343,SE +1158946344,1158946487,US +1158946488,1158946495,GB +1158946496,1158946551,US +1158946552,1158946559,CH +1158946560,1158946575,US +1158946576,1158946583,IN +1158946584,1158946631,US +1158946632,1158946639,IT +1158946640,1158946647,IL +1158946648,1158946711,US +1158946712,1158946719,HK +1158946720,1158947087,US +1158947088,1158947095,ZA +1158947096,1158947391,US +1158947392,1158947399,CA +1158947400,1158947407,GB +1158947408,1158947591,US +1158947592,1158947599,JP +1158947600,1158947647,US +1158947648,1158947655,CA +1158947656,1158947663,US +1158947664,1158947695,IE +1158947696,1158947711,US +1158947712,1158947719,CA +1158947720,1158947735,US +1158947736,1158947743,IN +1158947744,1158947791,US +1158947792,1158947799,IN +1158947800,1158947831,US +1158947832,1158947839,TR +1158947840,1158948383,US +1158948384,1158948391,IT +1158948392,1158948415,US +1158948416,1158948423,CA +1158948424,1158948895,US +1158948896,1158948903,CA +1158948904,1158948967,US +1158948968,1158948975,CA +1158948976,1158949007,US +1158949008,1158949015,IL +1158949016,1158949287,US +1158949288,1158949295,CA +1158949296,1158949407,US +1158949408,1158949415,ZA +1158949416,1158949775,US +1158949776,1158949783,GB +1158949784,1158950319,US +1158950320,1158950327,CA +1158950328,1158950335,US +1158950336,1158950351,CA +1158950352,1158950359,IN +1158950360,1158950367,CA +1158950368,1158950767,US +1158950768,1158950775,TH +1158950776,1158950783,US +1158950784,1158950791,SA +1158950792,1158951167,US +1158951168,1158951183,CA +1158951184,1158951215,US +1158951216,1158951223,CA +1158951224,1158951231,US +1158951232,1158951263,TH +1158951264,1158951423,US +1158951424,1158951439,CA +1158951440,1158951447,NL +1158951448,1158951551,US +1158951552,1158951559,NL +1158951560,1158951575,US +1158951576,1158951583,CA +1158951584,1158951743,US +1158951744,1158951751,IN +1158951752,1158951839,US +1158951840,1158951855,CA +1158951856,1158952743,US +1158952744,1158952751,FI +1158952752,1158952815,US +1158952816,1158952823,CA +1158952824,1158952895,US +1158952896,1158952903,IL +1158952904,1158954815,US +1158954816,1158954823,GB +1158954824,1158954831,CA +1158954832,1158955039,US +1158955040,1158955047,GB +1158955048,1158956063,US +1158956064,1158956079,NL +1158956080,1158956095,US +1158956096,1158956103,CA +1158956104,1158956295,US +1158956296,1158956303,CA +1158956304,1158957455,US +1158957456,1158957471,NL +1158957472,1158957647,US +1158957648,1158957655,AU +1158957656,1158957663,PE +1158957664,1158957695,US +1158957696,1158957703,TH +1158957704,1158958295,US +1158958296,1158958303,AG +1158958304,1158958311,ES +1158958312,1158958367,US +1158958368,1158958375,CA +1158958376,1158958591,US +1158958592,1158958599,ZA +1158958600,1158958671,US +1158958672,1158958679,SA +1158958680,1158958719,US +1158958720,1158958751,CA +1158958752,1158959031,US +1158959032,1158959039,CA +1158959040,1158959135,US +1158959136,1158959151,BD +1158959152,1158959679,US +1158959680,1158959687,CR +1158959688,1158960023,US +1158960024,1158960031,GB +1158960032,1158960055,US +1158960056,1158960063,GB +1158960064,1158960455,US +1158960456,1158960463,IN +1158960464,1158960535,US +1158960536,1158960543,NZ +1158960544,1158961063,US +1158961064,1158961071,TW +1158961072,1158961151,US +1158961152,1158961159,GB +1158961160,1158961175,US +1158961176,1158961183,CA +1158961184,1158961255,US +1158961256,1158961263,ES +1158961264,1158961775,US +1158961776,1158961783,CA +1158961784,1158961823,US +1158961824,1158961855,BD +1158961856,1158962175,US +1158962176,1158962183,GB +1158962184,1158962287,US +1158962288,1158962295,ZA +1158962296,1158962591,US +1158962592,1158962607,GB +1158962608,1158962615,US +1158962616,1158962623,GB +1158962624,1158963271,US +1158963272,1158963279,IE +1158963280,1158963335,US +1158963336,1158963343,SE +1158963344,1158963551,US +1158963552,1158963559,IN +1158963560,1158963647,US +1158963648,1158963663,IN +1158963664,1158963727,US +1158963728,1158963735,GB +1158963736,1158964103,US +1158964104,1158964111,IN +1158964112,1158964255,US +1158964256,1158964263,IN +1158964264,1158964287,US +1158964288,1158964319,CA +1158964320,1158964631,US +1158964632,1158964639,CA +1158964640,1158964647,MX +1158964648,1158964701,US +1158964702,1158964703,CA +1158964704,1158964799,US +1158964800,1158964807,IE +1158964808,1158964831,US +1158964832,1158964839,GB +1158964840,1158964943,US +1158964944,1158964951,MX +1158964952,1158964959,TW +1158964960,1158965087,US +1158965088,1158965111,CA +1158965112,1158965119,US +1158965120,1158965135,AR +1158965136,1158965159,US +1158965160,1158965167,IL +1158965168,1158965199,US +1158965200,1158965215,CA +1158965216,1158965247,US +1158965248,1158965255,RU +1158965256,1158965559,US +1158965560,1158965567,IN +1158965568,1158965743,US +1158965744,1158965751,IN +1158965752,1158966127,US +1158966128,1158966143,FR +1158966144,1158966335,US +1158966336,1158966343,PL +1158966344,1158966359,CA +1158966360,1158966567,US +1158966568,1158966591,CA +1158966592,1158966799,US +1158966800,1158966807,CA +1158966808,1158966887,US +1158966888,1158966895,EG +1158966896,1158966911,US +1158966912,1158966919,MX +1158966920,1158966991,US +1158966992,1158966999,CA +1158967000,1158967263,US +1158967264,1158967279,TH +1158967280,1158967327,US +1158967328,1158967335,JP +1158967336,1158967503,US +1158967504,1158967511,GB +1158967512,1158967527,US +1158967528,1158967535,CA +1158967536,1158967543,IE +1158967544,1158967559,US +1158967560,1158967567,CA +1158967568,1158967615,US +1158967616,1158967623,GT +1158967624,1158967631,US +1158967632,1158967679,CA +1158967680,1158968071,US +1158968072,1158968079,MT +1158968080,1158968103,US +1158968104,1158968111,HU +1158968112,1158968207,US +1158968208,1158968239,CA +1158968240,1158969447,US +1158969448,1158969455,CA +1158969456,1158969663,US +1158969664,1158969679,MX +1158969680,1158969703,US +1158969704,1158969711,CA +1158969712,1158970127,US +1158970128,1158970135,CA +1158970136,1158970287,US +1158970288,1158970295,CA +1158970296,1158970303,MY +1158970304,1158970919,US +1158970920,1158970927,AG +1158970928,1158970943,US +1158970944,1158970975,CA +1158970976,1158995967,US 1158995968,1158997919,CA 1158997920,1158997935,GB 1158997936,1158999271,CA @@ -18765,22 +20363,22 @@ 1161832704,1161832799,US 1161832800,1161832831,ZA 1161832832,1161832863,DE -1161832864,1161832927,US -1161832928,1161832959,TR -1161832960,1161833967,US +1161832864,1161833967,US 1161833968,1161833975,DE 1161833976,1161835341,US 1161835342,1161835353,GR 1161835354,1161836031,US 1161836032,1161836063,CA -1161836064,1161836191,US +1161836064,1161836159,US +1161836160,1161836191,BR 1161836192,1161836223,MX 1161836224,1161836255,US 1161836256,1161836287,UA 1161836288,1161836319,RS 1161836320,1161836383,US 1161836384,1161836415,CY -1161836416,1161836479,US +1161836416,1161836447,BR +1161836448,1161836479,US 1161836480,1161836511,UA 1161836512,1161837567,US 1161837568,1161837823,JP @@ -19419,6 +21017,7 @@ 1168900096,1168916479,US 1168916480,1168932863,CA 1168932864,1168936959,US +1168936960,1168949247,CA 1168949248,1168950271,US 1168950272,1168951823,CA 1168951824,1168952319,US @@ -19659,7 +21258,350 @@ 1208079168,1208082431,CA 1208082432,1208090623,US 1208090624,1208107007,CA -1208107008,1208301455,US +1208107008,1208189951,US +1208189952,1208190015,GB +1208190016,1208190543,US +1208190544,1208190559,GB +1208190560,1208190583,US +1208190584,1208190591,GB +1208190592,1208190607,US +1208190608,1208190615,GB +1208190616,1208190999,US +1208191000,1208191007,CA +1208191008,1208191047,US +1208191048,1208191055,AU +1208191056,1208191087,US +1208191088,1208191095,NO +1208191096,1208191199,US +1208191200,1208191215,CA +1208191216,1208191223,US +1208191224,1208191231,BG +1208191232,1208191311,US +1208191312,1208191319,GB +1208191320,1208191359,US +1208191360,1208191367,GB +1208191368,1208191391,US +1208191392,1208191423,NO +1208191424,1208191447,US +1208191448,1208191455,IE +1208191456,1208191567,US +1208191568,1208191575,SG +1208191576,1208191647,US +1208191648,1208191655,AG +1208191656,1208192167,US +1208192168,1208192175,BM +1208192176,1208192359,US +1208192360,1208192367,GB +1208192368,1208192879,US +1208192880,1208192895,MY +1208192896,1208192903,US +1208192904,1208192911,CA +1208192912,1208192959,US +1208192960,1208192967,IL +1208192968,1208193135,US +1208193136,1208193151,MY +1208193152,1208193303,US +1208193304,1208193311,CA +1208193312,1208193391,US +1208193392,1208193407,AU +1208193408,1208193455,US +1208193456,1208193463,PA +1208193464,1208193599,US +1208193600,1208193615,CA +1208193616,1208193871,US +1208193872,1208193879,AU +1208193880,1208193935,US +1208193936,1208193943,CA +1208193944,1208194055,US +1208194056,1208194063,AU +1208194064,1208194087,US +1208194088,1208194095,AU +1208194096,1208194191,US +1208194192,1208194223,MY +1208194224,1208195359,US +1208195360,1208195375,MY +1208195376,1208195391,US +1208195392,1208195399,IN +1208195400,1208195447,US +1208195448,1208195455,GB +1208195456,1208195535,US +1208195536,1208195551,GB +1208195552,1208195703,US +1208195704,1208195711,CA +1208195712,1208195727,TH +1208195728,1208195759,US +1208195760,1208195775,BG +1208195776,1208195847,US +1208195848,1208195855,FR +1208195856,1208195871,GB +1208195872,1208195887,US +1208195888,1208195903,ID +1208195904,1208196143,US +1208196144,1208196151,AU +1208196152,1208196327,US +1208196328,1208196335,CA +1208196336,1208196607,US +1208196608,1208196615,CA +1208196616,1208196703,US +1208196704,1208196719,CA +1208196720,1208197023,US +1208197024,1208197031,CR +1208197032,1208197135,US +1208197136,1208197167,MY +1208197168,1208197175,US +1208197176,1208197183,AU +1208197184,1208197239,US +1208197240,1208197247,CA +1208197248,1208197519,US +1208197520,1208197527,AU +1208197528,1208197559,US +1208197560,1208197567,AU +1208197568,1208197631,US +1208197632,1208197639,AU +1208197640,1208197671,US +1208197672,1208197679,MX +1208197680,1208197783,US +1208197784,1208197791,CA +1208197792,1208197967,US +1208197968,1208197983,AU +1208197984,1208198039,US +1208198040,1208198047,BS +1208198048,1208198135,US +1208198136,1208198143,AU +1208198144,1208198279,US +1208198280,1208198287,ZA +1208198288,1208198303,US +1208198304,1208198319,FR +1208198320,1208198463,US +1208198464,1208198471,CA +1208198472,1208198559,US +1208198560,1208198575,IN +1208198576,1208198743,US +1208198744,1208198751,GB +1208198752,1208198791,US +1208198792,1208198799,CN +1208198800,1208199055,US +1208199056,1208199063,AU +1208199064,1208199087,US +1208199088,1208199103,MY +1208199104,1208199215,US +1208199216,1208199223,AU +1208199224,1208199343,US +1208199344,1208199351,CR +1208199352,1208199359,MY +1208199360,1208199767,US +1208199768,1208199775,MX +1208199776,1208199783,MY +1208199784,1208199823,US +1208199824,1208199847,CA +1208199848,1208199935,US +1208199936,1208199943,AU +1208199944,1208200167,US +1208200168,1208200175,CA +1208200176,1208200247,US +1208200248,1208200255,MC +1208200256,1208200271,US +1208200272,1208200287,BD +1208200288,1208200303,US +1208200304,1208200319,AU +1208200320,1208200639,US +1208200640,1208200647,FR +1208200648,1208200655,US +1208200656,1208200663,BS +1208200664,1208200815,US +1208200816,1208200887,BS +1208200888,1208200959,US +1208200960,1208200963,ID +1208200964,1208201063,US +1208201064,1208201071,IN +1208201072,1208201255,US +1208201256,1208201263,ES +1208201264,1208201535,US +1208201536,1208201543,GB +1208201544,1208201631,US +1208201632,1208201639,CA +1208201640,1208201727,US +1208201728,1208201743,MY +1208201744,1208201951,US +1208201952,1208201959,MY +1208201960,1208202007,US +1208202008,1208202015,CA +1208202016,1208202031,US +1208202032,1208202039,GB +1208202040,1208202135,US +1208202136,1208202143,MY +1208202144,1208202239,US +1208202240,1208202247,TW +1208202248,1208202255,US +1208202256,1208202287,MY +1208202288,1208202367,US +1208202368,1208202375,TW +1208202376,1208202495,US +1208202496,1208202511,GB +1208202512,1208202519,AU +1208202520,1208202559,US +1208202560,1208202575,JP +1208202576,1208203023,US +1208203024,1208203071,MY +1208203072,1208203343,US +1208203344,1208203351,AU +1208203352,1208203647,US +1208203648,1208203655,PR +1208203656,1208204007,US +1208204008,1208204015,NL +1208204016,1208204031,US +1208204032,1208204039,GB +1208204040,1208204047,US +1208204048,1208204055,JP +1208204056,1208204767,US +1208204768,1208204775,MX +1208204776,1208204783,CA +1208204784,1208204927,US +1208204928,1208204935,AU +1208204936,1208204951,US +1208204952,1208204959,MY +1208204960,1208205063,US +1208205064,1208205079,MY +1208205080,1208205119,US +1208205120,1208205127,CA +1208205128,1208205167,US +1208205168,1208205231,MY +1208205232,1208205247,US +1208205248,1208205263,IN +1208205264,1208205303,US +1208205304,1208205311,CO +1208205312,1208205407,US +1208205408,1208205423,MY +1208205424,1208205455,US +1208205456,1208205463,CA +1208205464,1208205703,US +1208205704,1208205711,CA +1208205712,1208205959,US +1208205960,1208205967,SG +1208205968,1208205983,US +1208205984,1208205991,GB +1208205992,1208206439,US +1208206440,1208206463,CA +1208206464,1208207183,US +1208207184,1208207191,AU +1208207192,1208207303,US +1208207304,1208207311,ZA +1208207312,1208207359,US +1208207360,1208207399,CA +1208207400,1208207463,US +1208207464,1208207471,CA +1208207472,1208207583,US +1208207584,1208207591,AU +1208207592,1208207599,US +1208207600,1208207607,AU +1208207608,1208207623,US +1208207624,1208207631,GB +1208207632,1208207983,US +1208207984,1208207999,NL +1208208000,1208208215,US +1208208216,1208208223,CA +1208208224,1208208231,AU +1208208232,1208208239,US +1208208240,1208208255,GB +1208208256,1208208287,US +1208208288,1208208303,GB +1208208304,1208208311,KR +1208208312,1208208799,US +1208208800,1208208807,AU +1208208808,1208208815,US +1208208816,1208208831,CA +1208208832,1208208927,US +1208208928,1208208943,AU +1208208944,1208209023,US +1208209024,1208209031,CO +1208209032,1208209175,US +1208209176,1208209183,CA +1208209184,1208209215,US +1208209216,1208209247,MY +1208209248,1208209359,US +1208209360,1208209375,MY +1208209376,1208209391,US +1208209392,1208209407,MY +1208209408,1208209543,US +1208209544,1208209551,AU +1208209552,1208209719,US +1208209720,1208209727,TH +1208209728,1208209863,US +1208209864,1208209871,GT +1208209872,1208210055,US +1208210056,1208210063,IN +1208210064,1208210071,HK +1208210072,1208210079,US +1208210080,1208210095,GB +1208210096,1208210303,US +1208210304,1208210351,GB +1208210352,1208210431,US +1208210432,1208210447,JP +1208210448,1208210511,US +1208210512,1208210519,CA +1208210520,1208210751,US +1208210752,1208210759,TR +1208210760,1208211239,US +1208211240,1208211247,JM +1208211248,1208213087,US +1208213088,1208213103,NG +1208213104,1208213383,US +1208213384,1208213391,CA +1208213392,1208215935,US +1208215936,1208215951,GB +1208215952,1208216367,US +1208216368,1208216375,CA +1208216376,1208217447,US +1208217448,1208217455,AE +1208217456,1208218415,US +1208218416,1208218423,IN +1208218424,1208219159,US +1208219160,1208219167,AU +1208219168,1208219311,US +1208219312,1208219319,JP +1208219320,1208220655,US +1208220656,1208220663,AU +1208220664,1208220671,NZ +1208220672,1208220743,US +1208220744,1208220751,GB +1208220752,1208250367,US +1208250368,1208250375,CA +1208250376,1208250623,US +1208250624,1208250631,NL +1208250632,1208250711,US +1208250712,1208250719,GB +1208250720,1208250735,US +1208250736,1208250743,IN +1208250744,1208251199,US +1208251200,1208251207,CA +1208251208,1208251215,US +1208251216,1208251223,AU +1208251224,1208251415,US +1208251416,1208251423,GB +1208251424,1208251431,IL +1208251432,1208251447,ES +1208251448,1208251479,US +1208251480,1208251487,GB +1208251488,1208251495,BB +1208251496,1208251631,US +1208251632,1208251647,GB +1208251648,1208251679,US +1208251680,1208251695,GB +1208251696,1208251807,US +1208251808,1208251823,CA +1208251824,1208251839,GR +1208251840,1208251871,US +1208251872,1208251887,GB +1208251888,1208251903,IL +1208251904,1208252423,US +1208252424,1208252431,CA +1208252432,1208252703,US +1208252704,1208252735,AU +1208252736,1208252927,US +1208252928,1208253183,MX +1208253184,1208253599,US +1208253600,1208253615,IN +1208253616,1208301455,US 1208301456,1208301471,CA 1208301472,1208333567,US 1208333568,1208333823,GB @@ -20127,7 +22069,630 @@ 1209893504,1209893519,MX 1209893520,1209917439,US 1209917440,1209925631,CA -1209925632,1210253311,US +1209925632,1210057039,US +1210057040,1210057047,JM +1210057048,1210057055,US +1210057056,1210057071,DE +1210057072,1210057519,US +1210057520,1210057535,IN +1210057536,1210057567,US +1210057568,1210057583,NG +1210057584,1210057615,US +1210057616,1210057623,CA +1210057624,1210057879,US +1210057880,1210057887,IL +1210057888,1210057895,US +1210057896,1210057903,MX +1210057904,1210057911,GB +1210057912,1210058047,US +1210058048,1210058055,CA +1210058056,1210058159,US +1210058160,1210058167,CA +1210058168,1210058191,US +1210058192,1210058199,CA +1210058200,1210058615,US +1210058616,1210058623,AU +1210058624,1210059271,US +1210059272,1210059279,ES +1210059280,1210059775,US +1210059776,1210059791,GB +1210059792,1210060031,US +1210060032,1210060047,AE +1210060048,1210060351,US +1210060352,1210060359,IN +1210060360,1210060383,US +1210060384,1210060407,IL +1210060408,1210060623,US +1210060624,1210060639,CA +1210060640,1210060799,US +1210060800,1210060815,BR +1210060816,1210060839,US +1210060840,1210060847,AU +1210060848,1210061327,US +1210061328,1210061335,CA +1210061336,1210061367,US +1210061368,1210061375,IN +1210061376,1210061407,US +1210061408,1210061415,AU +1210061416,1210061463,US +1210061464,1210061471,HK +1210061472,1210061599,US +1210061600,1210061631,IL +1210061632,1210061647,US +1210061648,1210061655,IN +1210061656,1210061663,GB +1210061664,1210061727,US +1210061728,1210061735,GB +1210061736,1210061991,US +1210061992,1210061999,AU +1210062000,1210062007,US +1210062008,1210062015,CA +1210062016,1210062079,US +1210062080,1210062087,IN +1210062088,1210062095,US +1210062096,1210062103,NZ +1210062104,1210062463,US +1210062464,1210062479,IN +1210062480,1210062487,US +1210062488,1210062495,MX +1210062496,1210062815,US +1210062816,1210062831,ZA +1210062832,1210062847,IL +1210062848,1210062927,US +1210062928,1210062935,AU +1210062936,1210062975,US +1210062976,1210062983,CR +1210062984,1210064311,US +1210064312,1210064319,IE +1210064320,1210064527,US +1210064528,1210064535,AU +1210064536,1210064663,US +1210064664,1210064671,AU +1210064672,1210064951,US +1210064952,1210064959,KW +1210064960,1210064999,US +1210065000,1210065007,BR +1210065008,1210065023,US +1210065024,1210065031,CA +1210065032,1210065319,US +1210065320,1210065327,AU +1210065328,1210065535,US +1210065536,1210065543,IN +1210065544,1210065551,US +1210065552,1210065559,GB +1210065560,1210065567,US +1210065568,1210065575,GB +1210065576,1210065615,US +1210065616,1210065623,CA +1210065624,1210065663,US +1210065664,1210065679,IN +1210065680,1210065695,AU +1210065696,1210065703,IN +1210065704,1210065735,US +1210065736,1210065743,DO +1210065744,1210065751,US +1210065752,1210065759,GB +1210065760,1210065783,US +1210065784,1210065791,PA +1210065792,1210065823,US +1210065824,1210065855,AU +1210065856,1210066255,US +1210066256,1210066263,CA +1210066264,1210066287,US +1210066288,1210066295,IN +1210066296,1210066519,US +1210066520,1210066527,BM +1210066528,1210066807,US +1210066808,1210066831,CA +1210066832,1210067263,US +1210067264,1210067287,GB +1210067288,1210067535,US +1210067536,1210067543,AU +1210067544,1210067632,US +1210067633,1210067640,AU +1210067641,1210067967,US +1210067968,1210067983,IL +1210067984,1210068039,US +1210068040,1210068047,GB +1210068048,1210068151,US +1210068152,1210068159,CA +1210068160,1210068319,US +1210068320,1210068327,CA +1210068328,1210068343,US +1210068344,1210068351,MX +1210068352,1210068447,US +1210068448,1210068471,NO +1210068472,1210068543,US +1210068544,1210068559,CA +1210068560,1210068607,US +1210068608,1210068735,IL +1210068736,1210068759,US +1210068760,1210068767,IL +1210068768,1210068775,CA +1210068776,1210068903,US +1210068904,1210068927,AU +1210068928,1210069007,US +1210069008,1210069023,CA +1210069024,1210069159,US +1210069160,1210069167,AU +1210069168,1210070071,US +1210070072,1210070079,GB +1210070080,1210070175,US +1210070176,1210070183,IL +1210070184,1210070431,US +1210070432,1210070439,IL +1210070440,1210070503,US +1210070504,1210070511,CA +1210070512,1210070631,US +1210070632,1210070639,MX +1210070640,1210070703,US +1210070704,1210070711,GB +1210070712,1210070799,US +1210070800,1210070807,AU +1210070808,1210070863,US +1210070864,1210070871,BH +1210070872,1210070943,US +1210070944,1210070951,IN +1210070952,1210070959,US +1210070960,1210070967,JP +1210070968,1210071007,US +1210071008,1210071015,GB +1210071016,1210071039,US +1210071040,1210071167,GB +1210071168,1210072255,US +1210072256,1210072319,CA +1210072320,1210072519,US +1210072520,1210072527,NZ +1210072528,1210072559,US +1210072560,1210072567,GB +1210072568,1210072783,US +1210072784,1210072791,KW +1210072792,1210073199,US +1210073200,1210073207,DE +1210073208,1210075663,US +1210075664,1210075671,IN +1210075672,1210076039,US +1210076040,1210076047,CA +1210076048,1210079903,US +1210079904,1210079911,IT +1210079912,1210079919,BM +1210079920,1210079999,US +1210080000,1210080015,IN +1210080016,1210080079,US +1210080080,1210080111,GB +1210080112,1210080407,US +1210080408,1210080455,NL +1210080456,1210080615,US +1210080616,1210080623,CA +1210080624,1210080847,US +1210080848,1210080855,SV +1210080856,1210081087,US +1210081088,1210081103,CA +1210081104,1210081335,US +1210081336,1210081343,CR +1210081344,1210081559,US +1210081560,1210081567,PK +1210081568,1210081823,US +1210081824,1210081831,MX +1210081832,1210081839,US +1210081840,1210081847,MX +1210081848,1210081879,US +1210081880,1210081887,MX +1210081888,1210081911,US +1210081912,1210081919,AU +1210081920,1210081943,US +1210081944,1210081951,CA +1210081952,1210082047,US +1210082048,1210082055,IN +1210082056,1210082199,US +1210082200,1210082207,CA +1210082208,1210082447,US +1210082448,1210082463,AU +1210082464,1210082655,US +1210082656,1210082663,CA +1210082664,1210082831,US +1210082832,1210082839,AU +1210082840,1210082847,IN +1210082848,1210082975,US +1210082976,1210082991,GB +1210082992,1210083071,US +1210083072,1210083079,JP +1210083080,1210083719,US +1210083720,1210083735,IN +1210083736,1210083871,US +1210083872,1210083879,CA +1210083880,1210083983,US +1210083984,1210083991,IN +1210083992,1210084215,US +1210084216,1210084223,VE +1210084224,1210084263,US +1210084264,1210084271,NZ +1210084272,1210084359,US +1210084360,1210084367,BG +1210084368,1210084415,US +1210084416,1210084479,CA +1210084480,1210084711,US +1210084712,1210084719,AU +1210084720,1210084783,US +1210084784,1210084799,BD +1210084800,1210086503,US +1210086504,1210086511,GB +1210086512,1210086551,US +1210086552,1210086559,SE +1210086560,1210086583,US +1210086584,1210086591,SE +1210086592,1210086599,US +1210086600,1210086607,NZ +1210086608,1210086623,US +1210086624,1210086631,SE +1210086632,1210087295,US +1210087296,1210087303,GB +1210087304,1210087423,US +1210087424,1210087431,MX +1210087432,1210088959,US +1210088960,1210088967,GB +1210088968,1210089047,US +1210089048,1210089055,MX +1210089056,1210089823,US +1210089824,1210089855,CA +1210089856,1210090071,US +1210090072,1210090079,MX +1210090080,1210090287,US +1210090288,1210090295,AU +1210090296,1210090367,US +1210090368,1210090375,JM +1210090376,1210090383,US +1210090384,1210090399,CA +1210090400,1210090623,US +1210090624,1210090639,BD +1210090640,1210090767,US +1210090768,1210090783,BD +1210090784,1210090815,US +1210090816,1210090823,CA +1210090824,1210090895,US +1210090896,1210090903,AU +1210090904,1210091079,US +1210091080,1210091087,SG +1210091088,1210091135,US +1210091136,1210091151,CA +1210091152,1210091175,US +1210091176,1210091183,SR +1210091184,1210091351,US +1210091352,1210091359,AU +1210091360,1210091535,US +1210091536,1210091543,CA +1210091544,1210091679,US +1210091680,1210091687,AU +1210091688,1210091791,US +1210091792,1210091807,CA +1210091808,1210091839,US +1210091840,1210091847,CA +1210091848,1210091987,US +1210091988,1210091991,GB +1210091992,1210092287,US +1210092288,1210092543,ES +1210092544,1210092567,US +1210092568,1210092575,AU +1210092576,1210092583,CA +1210092584,1210092591,MX +1210092592,1210092607,US +1210092608,1210092639,GB +1210092640,1210092799,US +1210092800,1210092815,CA +1210092816,1210093143,US +1210093144,1210093151,ID +1210093152,1210093191,US +1210093192,1210093199,BR +1210093200,1210093407,US +1210093408,1210093423,MX +1210093424,1210093575,US +1210093576,1210093583,MY +1210093584,1210093639,US +1210093640,1210093647,CA +1210093648,1210093663,US +1210093664,1210093671,CA +1210093672,1210093679,AU +1210093680,1210093751,US +1210093752,1210093759,AU +1210093760,1210093847,US +1210093848,1210093855,AU +1210093856,1210094175,US +1210094176,1210094183,TT +1210094184,1210094407,US +1210094408,1210094415,GB +1210094416,1210094831,US +1210094832,1210094847,SG +1210094848,1210094943,US +1210094944,1210094951,GB +1210094952,1210094959,US +1210094960,1210094967,GB +1210094968,1210095263,US +1210095264,1210095271,GB +1210095272,1210095503,US +1210095504,1210095519,IN +1210095520,1210095527,NZ +1210095528,1210095711,US +1210095712,1210095727,GB +1210095728,1210095935,US +1210095936,1210095951,IN +1210095952,1210095959,US +1210095960,1210095967,GB +1210095968,1210096295,US +1210096296,1210096303,GB +1210096304,1210096543,US +1210096544,1210096551,MX +1210096552,1210096559,US +1210096560,1210096575,DE +1210096576,1210096871,US +1210096872,1210096879,CA +1210096880,1210097151,US +1210097152,1210097159,GB +1210097160,1210097303,US +1210097304,1210097311,MX +1210097312,1210097415,US +1210097416,1210097423,ZA +1210097424,1210097567,US +1210097568,1210097575,GB +1210097576,1210097655,US +1210097656,1210097663,GB +1210097664,1210097783,US +1210097784,1210097791,GB +1210097792,1210097999,US +1210098000,1210098007,CA +1210098008,1210098111,US +1210098112,1210098119,IL +1210098120,1210098127,US +1210098128,1210098143,CA +1210098144,1210098215,US +1210098216,1210098223,IN +1210098224,1210098319,US +1210098320,1210098327,CA +1210098328,1210098527,US +1210098528,1210098535,SG +1210098536,1210098615,US +1210098616,1210098623,NZ +1210098624,1210098743,US +1210098744,1210098751,BO +1210098752,1210098775,US +1210098776,1210098783,CN +1210098784,1210098959,US +1210098960,1210098975,CA +1210098976,1210099383,US +1210099384,1210099399,AE +1210099400,1210099495,US +1210099496,1210099503,BG +1210099504,1210099511,US +1210099512,1210099519,MX +1210099520,1210099583,US +1210099584,1210099591,CA +1210099592,1210099695,US +1210099696,1210099711,AU +1210099712,1210099783,US +1210099784,1210099791,NZ +1210099792,1210099879,US +1210099880,1210099895,GB +1210099896,1210099967,US +1210099968,1210099975,MT +1210099976,1210100103,US +1210100104,1210100111,MT +1210100112,1210100231,US +1210100232,1210100239,CA +1210100240,1210100255,PK +1210100256,1210100287,CA +1210100288,1210100359,US +1210100360,1210100367,AU +1210100368,1210100463,US +1210100464,1210100479,CA +1210100480,1210100631,US +1210100632,1210100639,NZ +1210100640,1210101023,US +1210101024,1210101039,GB +1210101040,1210101263,US +1210101264,1210101279,MY +1210101280,1210101311,US +1210101312,1210101319,IE +1210101320,1210101327,IL +1210101328,1210101559,US +1210101560,1210101567,AU +1210101568,1210101727,US +1210101728,1210101743,GB +1210101744,1210101879,US +1210101880,1210101887,CA +1210101888,1210102399,US +1210102400,1210102407,AU +1210102408,1210102815,US +1210102816,1210102823,CA +1210102824,1210105063,US +1210105064,1210105071,SE +1210105072,1210107967,US +1210107968,1210107975,MO +1210107976,1210108295,US +1210108296,1210108303,IN +1210108304,1210108335,US +1210108336,1210108343,CA +1210108344,1210108351,IL +1210108352,1210108359,US +1210108360,1210108367,CA +1210108368,1210108423,US +1210108424,1210108431,PR +1210108432,1210108631,US +1210108632,1210108639,GB +1210108640,1210108903,US +1210108904,1210108911,AU +1210108912,1210108919,GB +1210108920,1210109247,US +1210109248,1210109255,DK +1210109256,1210109263,IL +1210109264,1210109271,CA +1210109272,1210109343,US +1210109344,1210109351,FI +1210109352,1210109751,US +1210109752,1210109759,IL +1210109760,1210109767,GB +1210109768,1210109775,IN +1210109776,1210109935,US +1210109936,1210109943,BB +1210109944,1210111007,US +1210111008,1210111015,GB +1210111016,1210111095,US +1210111096,1210111103,CA +1210111104,1210111279,US +1210111280,1210111287,VG +1210111288,1210111295,US +1210111296,1210111303,AU +1210111304,1210111999,US +1210112000,1210112007,CA +1210112008,1210112039,US +1210112040,1210112047,IL +1210112048,1210112455,US +1210112456,1210112463,ZA +1210112464,1210112679,US +1210112680,1210112687,CA +1210112688,1210112727,US +1210112728,1210112735,PY +1210112736,1210112815,US +1210112816,1210112823,MX +1210112824,1210112839,US +1210112840,1210112847,LT +1210112848,1210112855,IN +1210112856,1210113055,US +1210113056,1210113063,CA +1210113064,1210113095,US +1210113096,1210113103,BR +1210113104,1210113135,US +1210113136,1210113143,CA +1210113144,1210113535,US +1210113536,1210113551,CA +1210113552,1210113615,US +1210113616,1210113631,GB +1210113632,1210113991,US +1210113992,1210113999,IE +1210114000,1210114095,US +1210114096,1210114103,CR +1210114104,1210114151,US +1210114152,1210114159,MY +1210114160,1210114167,US +1210114168,1210114175,HN +1210114176,1210114279,US +1210114280,1210114287,AU +1210114288,1210114351,US +1210114352,1210114367,AU +1210114368,1210114439,US +1210114440,1210114447,GB +1210114448,1210114471,US +1210114472,1210114479,AU +1210114480,1210114487,CL +1210114488,1210114495,AU +1210114496,1210114503,US +1210114504,1210114511,BB +1210114512,1210114551,US +1210114552,1210114559,AU +1210114560,1210114623,US +1210114624,1210114631,CA +1210114632,1210114711,US +1210114712,1210114719,ZA +1210114720,1210114759,US +1210114760,1210114775,MT +1210114776,1210114831,US +1210114832,1210114839,AU +1210114840,1210114991,US +1210114992,1210114999,AU +1210115000,1210115079,US +1210115080,1210115087,AU +1210115088,1210115135,US +1210115136,1210115143,AU +1210115144,1210115199,US +1210115200,1210115207,GB +1210115208,1210115231,US +1210115232,1210115239,AU +1210115240,1210115335,US +1210115336,1210115343,GB +1210115344,1210115471,US +1210115472,1210115479,IN +1210115480,1210115495,US +1210115496,1210115503,AU +1210115504,1210115543,US +1210115544,1210115575,BB +1210115576,1210115591,US +1210115592,1210115599,CA +1210115600,1210115719,US +1210115720,1210115727,AU +1210115728,1210115791,US +1210115792,1210115799,GB +1210115800,1210115911,US +1210115912,1210115919,GB +1210115920,1210116255,US +1210116256,1210116263,IN +1210116264,1210116375,US +1210116376,1210116383,CA +1210116384,1210116423,US +1210116424,1210116431,GB +1210116432,1210116527,US +1210116528,1210116535,PR +1210116536,1210116671,US +1210116672,1210116679,AU +1210116680,1210116687,GB +1210116688,1210116695,NG +1210116696,1210116863,US +1210116864,1210116871,GB +1210116872,1210117191,US +1210117192,1210117199,CA +1210117200,1210117567,US +1210117568,1210117599,CA +1210117600,1210117631,US +1210117632,1210117639,IN +1210117640,1210117647,GB +1210117648,1210117655,AE +1210117656,1210117871,US +1210117872,1210117879,CA +1210117880,1210117887,US +1210117888,1210117895,TH +1210117896,1210118023,US +1210118024,1210118031,GB +1210118032,1210118191,US +1210118192,1210118199,NZ +1210118200,1210118239,US +1210118240,1210118255,TH +1210118256,1210118431,US +1210118432,1210118439,AU +1210118440,1210118655,US +1210118656,1210118663,IN +1210118664,1210118687,US +1210118688,1210118703,IN +1210118704,1210118887,US +1210118888,1210118895,BR +1210118896,1210119119,US +1210119120,1210119135,GB +1210119136,1210119183,US +1210119184,1210119191,CA +1210119192,1210119255,US +1210119256,1210119263,MY +1210119264,1210119311,US +1210119312,1210119319,AN +1210119320,1210119327,MX +1210119328,1210119551,US +1210119552,1210119559,MX +1210119560,1210119823,US +1210119824,1210119831,CA +1210119832,1210120007,US +1210120008,1210120015,IN +1210120016,1210120847,US +1210120848,1210120855,MY +1210120856,1210120895,US +1210120896,1210120903,MX +1210120904,1210120991,US +1210120992,1210120999,IN +1210121000,1210121023,US +1210121024,1210121039,AU +1210121040,1210121079,US +1210121080,1210121087,BR +1210121088,1210121887,US +1210121888,1210121903,MO +1210121904,1210122031,US +1210122032,1210122039,CA +1210122040,1210122063,US +1210122064,1210122071,MX +1210122072,1210253311,US 1210253312,1210254703,CA 1210254704,1210254719,NZ 1210254720,1210261503,CA @@ -20502,7 +23067,9 @@ 1249050624,1249052671,A1 1249052672,1249082367,US 1249082368,1249083391,CA -1249083392,1249099775,US +1249083392,1249091583,US +1249091584,1249092607,A1 +1249092608,1249099775,US 1249099776,1249101823,CA 1249101824,1249102847,PR 1249102848,1249103871,CA @@ -20650,7 +23217,521 @@ 1254725120,1254725151,GB 1254725152,1254924687,US 1254924688,1254924703,RO -1254924704,1254989823,US +1254924704,1254948927,US +1254948928,1254948935,SG +1254948936,1254948959,US +1254948960,1254948975,CA +1254948976,1254949247,US +1254949248,1254949279,IN +1254949280,1254950671,US +1254950672,1254950679,RU +1254950680,1254950687,CA +1254950688,1254950719,US +1254950720,1254950727,SG +1254950728,1254950751,US +1254950752,1254950759,IL +1254950760,1254950767,US +1254950768,1254950775,GR +1254950776,1254950847,US +1254950848,1254950855,IL +1254950856,1254950919,US +1254950920,1254950927,CA +1254950928,1254950951,US +1254950952,1254950959,IN +1254950960,1254951247,US +1254951248,1254951263,CA +1254951264,1254951431,US +1254951432,1254951439,CA +1254951440,1254951471,US +1254951472,1254951479,MX +1254951480,1254951495,US +1254951496,1254951503,PR +1254951504,1254951591,US +1254951592,1254951599,IL +1254951600,1254951607,US +1254951608,1254951615,IN +1254951616,1254951631,US +1254951632,1254951639,JP +1254951640,1254951647,CA +1254951648,1254951655,US +1254951656,1254951663,CA +1254951664,1254951871,US +1254951872,1254951887,TH +1254951888,1254952015,US +1254952016,1254952023,CA +1254952024,1254952047,US +1254952048,1254952055,MX +1254952056,1254952239,US +1254952240,1254952247,AU +1254952248,1254952255,US +1254952256,1254952263,CA +1254952264,1254952279,US +1254952280,1254952287,CA +1254952288,1254952295,IL +1254952296,1254952591,US +1254952592,1254952599,CA +1254952600,1254952927,US +1254952928,1254952935,CA +1254952936,1254953031,US +1254953032,1254953039,CA +1254953040,1254953047,ZA +1254953048,1254953055,US +1254953056,1254953063,IL +1254953064,1254953071,US +1254953072,1254953079,CA +1254953080,1254953087,US +1254953088,1254953095,CA +1254953096,1254953279,US +1254953280,1254953295,BD +1254953296,1254953327,US +1254953328,1254953343,NL +1254953344,1254953359,US +1254953360,1254953367,CA +1254953368,1254953615,US +1254953616,1254953623,GB +1254953624,1254953639,US +1254953640,1254953647,TW +1254953648,1254953671,US +1254953672,1254953679,MT +1254953680,1254953687,US +1254953688,1254953703,CA +1254953704,1254953743,US +1254953744,1254953751,IN +1254953752,1254953759,US +1254953760,1254953767,IN +1254953768,1254953775,GB +1254953776,1254953815,US +1254953816,1254953823,GT +1254953824,1254953855,US +1254953856,1254953879,NL +1254953880,1254953919,US +1254953920,1254953935,NL +1254953936,1254954255,US +1254954256,1254954263,CA +1254954264,1254954287,US +1254954288,1254954295,CA +1254954296,1254954303,US +1254954304,1254954311,MX +1254954312,1254954319,US +1254954320,1254954327,CA +1254954328,1254954343,US +1254954344,1254954351,TH +1254954352,1254954639,US +1254954640,1254954647,GB +1254954648,1254954655,CA +1254954656,1254954663,US +1254954664,1254954671,GB +1254954672,1254954887,US +1254954888,1254954895,IL +1254954896,1254955599,US +1254955600,1254955607,GB +1254955608,1254955735,US +1254955736,1254955743,GB +1254955744,1254956135,US +1254956136,1254956143,IL +1254956144,1254956383,US +1254956384,1254956391,IL +1254956392,1254957055,US +1254957056,1254957063,GB +1254957064,1254957071,MX +1254957072,1254957239,US +1254957240,1254957247,GB +1254957248,1254957263,MT +1254957264,1254957287,US +1254957288,1254957295,IN +1254957296,1254957375,US +1254957376,1254957391,CA +1254957392,1254957407,US +1254957408,1254957423,IN +1254957424,1254957743,US +1254957744,1254957751,GB +1254957752,1254957983,US +1254957984,1254957991,MX +1254957992,1254958303,US +1254958304,1254958319,GB +1254958320,1254958351,US +1254958352,1254958359,BO +1254958360,1254958367,US +1254958368,1254958383,CA +1254958384,1254958423,US +1254958424,1254958431,NO +1254958432,1254958535,US +1254958536,1254958543,CA +1254958544,1254958855,US +1254958856,1254958863,NZ +1254958864,1254958895,US +1254958896,1254958903,CA +1254958904,1254959151,US +1254959152,1254959183,CA +1254959184,1254959391,US +1254959392,1254959399,GB +1254959400,1254959671,US +1254959672,1254959679,GB +1254959680,1254959775,US +1254959776,1254959783,KR +1254959784,1254959799,US +1254959800,1254959807,GB +1254959808,1254959927,US +1254959928,1254959935,MX +1254959936,1254959943,US +1254959944,1254959951,CA +1254959952,1254959991,US +1254959992,1254959999,MX +1254960000,1254960039,US +1254960040,1254960047,IN +1254960048,1254960279,US +1254960280,1254960287,JM +1254960288,1254960351,US +1254960352,1254960359,AU +1254960360,1254960367,CA +1254960368,1254960415,US +1254960416,1254960423,CA +1254960424,1254960447,US +1254960448,1254960455,AU +1254960456,1254960463,US +1254960464,1254960471,SG +1254960472,1254960487,AU +1254960488,1254960495,GB +1254960496,1254960559,US +1254960560,1254960567,MY +1254960568,1254960583,US +1254960584,1254960591,MY +1254960592,1254960655,US +1254960656,1254960663,CA +1254960664,1254960879,US +1254960880,1254960887,CA +1254960888,1254960903,US +1254960904,1254960911,NZ +1254960912,1254960935,US +1254960936,1254960943,IN +1254960944,1254960967,US +1254960968,1254960975,IN +1254960976,1254961279,US +1254961280,1254961295,GB +1254961296,1254961567,US +1254961568,1254961599,BD +1254961600,1254961743,US +1254961744,1254961751,ZA +1254961752,1254961767,GB +1254961768,1254961823,US +1254961824,1254961831,AE +1254961832,1254961911,US +1254961912,1254961919,CA +1254961920,1254962015,US +1254962016,1254962023,AE +1254962024,1254962191,US +1254962192,1254962199,GB +1254962200,1254962351,US +1254962352,1254962359,GB +1254962360,1254962511,US +1254962512,1254962527,JP +1254962528,1254962695,US +1254962696,1254962703,GB +1254962704,1254962799,US +1254962800,1254962807,CA +1254962808,1254962855,US +1254962856,1254962863,CA +1254962864,1254962951,US +1254962952,1254962959,NZ +1254962960,1254962975,US +1254962976,1254962983,CA +1254962984,1254963047,US +1254963048,1254963055,MX +1254963056,1254963575,US +1254963576,1254963591,AU +1254963592,1254963663,US +1254963664,1254963671,GB +1254963672,1254964079,US +1254964080,1254964087,DK +1254964088,1254964231,US +1254964232,1254964239,PR +1254964240,1254964255,US +1254964256,1254964263,BR +1254964264,1254964335,US +1254964336,1254964343,NZ +1254964344,1254964351,PR +1254964352,1254964391,US +1254964392,1254964399,AU +1254964400,1254964799,US +1254964800,1254964815,CA +1254964816,1254964927,US +1254964928,1254964943,CA +1254964944,1254965095,US +1254965096,1254965103,IN +1254965104,1254965207,US +1254965208,1254965215,AU +1254965216,1254965239,US +1254965240,1254965247,GB +1254965248,1254965391,US +1254965392,1254965399,TC +1254965400,1254965431,US +1254965432,1254965439,GB +1254965440,1254965511,US +1254965512,1254965519,AU +1254965520,1254965735,US +1254965736,1254965743,CA +1254965744,1254965751,AU +1254965752,1254965887,US +1254965888,1254965895,MX +1254965896,1254966031,US +1254966032,1254966039,CA +1254966040,1254966135,US +1254966136,1254966143,GB +1254966144,1254966159,US +1254966160,1254966167,CA +1254966168,1254966239,US +1254966240,1254966247,AU +1254966248,1254966383,US +1254966384,1254966391,ZA +1254966392,1254966487,US +1254966488,1254966495,IE +1254966496,1254966567,US +1254966568,1254966575,ZA +1254966576,1254966607,US +1254966608,1254966615,JO +1254966616,1254966655,US +1254966656,1254966663,GB +1254966664,1254966767,US +1254966768,1254966775,ZA +1254966776,1254966863,US +1254966864,1254966879,IL +1254966880,1254966927,US +1254966928,1254966943,FR +1254966944,1254966975,US +1254966976,1254966991,IL +1254966992,1254967007,US +1254967008,1254967023,AE +1254967024,1254967079,US +1254967080,1254967087,PH +1254967088,1254967127,US +1254967128,1254967143,BM +1254967144,1254967263,US +1254967264,1254967271,AU +1254967272,1254967519,US +1254967520,1254967535,CA +1254967536,1254967607,US +1254967608,1254967615,CA +1254967616,1254967647,US +1254967648,1254967655,EG +1254967656,1254967887,US +1254967888,1254967895,CA +1254967896,1254967975,US +1254967976,1254967991,IN +1254967992,1254968039,US +1254968040,1254968047,CA +1254968048,1254968103,US +1254968104,1254968111,UY +1254968112,1254968183,US +1254968184,1254968191,CA +1254968192,1254968295,US +1254968296,1254968303,IT +1254968304,1254968391,US +1254968392,1254968399,SG +1254968400,1254968407,US +1254968408,1254968415,CA +1254968416,1254968487,US +1254968488,1254968495,CA +1254968496,1254968503,US +1254968504,1254968511,GB +1254968512,1254968671,US +1254968672,1254968679,DO +1254968680,1254968751,US +1254968752,1254968759,GB +1254968760,1254968927,US +1254968928,1254968935,CA +1254968936,1254968951,US +1254968952,1254968959,GB +1254968960,1254968991,US +1254968992,1254968999,CA +1254969000,1254969111,US +1254969112,1254969127,AU +1254969128,1254969175,US +1254969176,1254969183,AU +1254969184,1254969247,US +1254969248,1254969255,CO +1254969256,1254969263,BD +1254969264,1254969287,US +1254969288,1254969295,GB +1254969296,1254969311,US +1254969312,1254969319,CA +1254969320,1254969399,US +1254969400,1254969407,CA +1254969408,1254969423,US +1254969424,1254969431,CA +1254969432,1254969439,GB +1254969440,1254969447,CA +1254969448,1254969455,US +1254969456,1254969463,IL +1254969464,1254969471,GB +1254969472,1254969495,US +1254969496,1254969503,DK +1254969504,1254969551,US +1254969552,1254969559,CA +1254969560,1254969567,GB +1254969568,1254969575,US +1254969576,1254969583,AE +1254969584,1254969663,US +1254969664,1254969671,AU +1254969672,1254970047,US +1254970048,1254970063,CA +1254970064,1254970095,US +1254970096,1254970111,AE +1254970112,1254970167,US +1254970168,1254970175,GB +1254970176,1254970183,US +1254970184,1254970191,GB +1254970192,1254970199,US +1254970200,1254970207,CA +1254970208,1254970215,IL +1254970216,1254970287,US +1254970288,1254970295,IN +1254970296,1254970335,US +1254970336,1254970343,CH +1254970344,1254970527,US +1254970528,1254970543,GB +1254970544,1254970607,US +1254970608,1254970623,GH +1254970624,1254970631,MX +1254970632,1254970903,US +1254970904,1254970911,CA +1254970912,1254971007,US +1254971008,1254971015,JP +1254971016,1254971023,IN +1254971024,1254971055,US +1254971056,1254971063,GB +1254971064,1254971143,US +1254971144,1254971151,CA +1254971152,1254971231,US +1254971232,1254971239,IN +1254971240,1254971255,US +1254971256,1254971263,CA +1254971264,1254971319,US +1254971320,1254971327,MX +1254971328,1254971951,US +1254971952,1254971959,GB +1254971960,1254971967,IN +1254971968,1254972239,US +1254972240,1254972247,CA +1254972248,1254972407,US +1254972408,1254972415,AU +1254972416,1254972431,US +1254972432,1254972439,AU +1254972440,1254972447,US +1254972448,1254972455,GG +1254972456,1254972671,US +1254972672,1254972679,GB +1254972680,1254972727,US +1254972728,1254972735,AU +1254972736,1254972879,US +1254972880,1254972895,MX +1254972896,1254972983,US +1254972984,1254972991,CA +1254972992,1254973015,US +1254973016,1254973023,BR +1254973024,1254973047,US +1254973048,1254973055,AU +1254973056,1254973471,US +1254973472,1254973479,IN +1254973480,1254973511,US +1254973512,1254973519,IE +1254973520,1254973575,US +1254973576,1254973583,MX +1254973584,1254973663,US +1254973664,1254973671,PH +1254973672,1254973679,US +1254973680,1254973687,IN +1254973688,1254973695,US +1254973696,1254973703,MY +1254973704,1254973807,US +1254973808,1254973815,IN +1254973816,1254973895,US +1254973896,1254973903,MX +1254973904,1254974167,US +1254974168,1254974175,MX +1254974176,1254974327,US +1254974328,1254974335,MX +1254974336,1254974367,US +1254974368,1254974375,CA +1254974376,1254974391,US +1254974392,1254974399,CA +1254974400,1254974511,US +1254974512,1254974519,IL +1254974520,1254974647,US +1254974648,1254974655,CA +1254974656,1254974767,US +1254974768,1254974775,GB +1254974776,1254974887,US +1254974888,1254974895,AU +1254974896,1254974951,US +1254974952,1254974959,NZ +1254974960,1254975231,US +1254975232,1254975239,AU +1254975240,1254975415,US +1254975416,1254975423,CA +1254975424,1254975471,US +1254975472,1254975487,GB +1254975488,1254975615,US +1254975616,1254975623,IN +1254975624,1254975631,IL +1254975632,1254975887,US +1254975888,1254975895,IN +1254975896,1254975919,US +1254975920,1254975927,IN +1254975928,1254975935,CA +1254975936,1254976023,US +1254976024,1254976031,GB +1254976032,1254976127,US +1254976128,1254976135,AU +1254976136,1254976151,US +1254976152,1254976167,CA +1254976168,1254976303,US +1254976304,1254976311,GB +1254976312,1254976351,US +1254976352,1254976359,GB +1254976360,1254976399,US +1254976400,1254976407,MX +1254976408,1254976735,US +1254976736,1254976751,CA +1254976752,1254976767,US +1254976768,1254976775,GB +1254976776,1254976791,US +1254976792,1254976799,IN +1254976800,1254976903,US +1254976904,1254976911,MX +1254976912,1254977095,US +1254977096,1254977103,CO +1254977104,1254977135,US +1254977136,1254977143,NZ +1254977144,1254977159,US +1254977160,1254977167,IN +1254977168,1254977247,US +1254977248,1254977255,CL +1254977256,1254977455,US +1254977456,1254977471,IN +1254977472,1254977487,US +1254977488,1254977503,TH +1254977504,1254977623,US +1254977624,1254977631,CA +1254977632,1254977647,US +1254977648,1254977655,AE +1254977656,1254977735,US +1254977736,1254977743,NZ +1254977744,1254977791,US +1254977792,1254977799,GB +1254977800,1254978183,US +1254978184,1254978191,GB +1254978192,1254978327,US +1254978328,1254978335,IN +1254978336,1254978375,US +1254978376,1254978383,BR +1254978384,1254978391,CA +1254978392,1254978447,US +1254978448,1254978463,IN +1254978464,1254978495,US +1254978496,1254978559,NL +1254978560,1254979407,US +1254979408,1254979423,CA +1254979424,1254989823,US 1254989824,1254998015,CA 1254998016,1255002111,US 1255002112,1255006207,CA @@ -21141,7 +24222,9 @@ 1275920080,1275920095,CA 1275920096,1279262719,US 1279262720,1279787007,CA -1279787008,1279940879,US +1279787008,1279848447,US +1279848448,1279852543,PR +1279852544,1279940879,US 1279940880,1279940887,DE 1279940888,1279940943,US 1279940944,1279940959,DE @@ -21726,7 +24809,8 @@ 1296251328,1296251359,IE 1296251360,1296251391,DE 1296251392,1296251775,NL -1296251776,1296252143,FR +1296251776,1296252127,FR +1296252128,1296252143,ES 1296252144,1296252159,US 1296252160,1296252175,GB 1296252176,1296252191,PL @@ -21740,7 +24824,9 @@ 1296252320,1296252367,FR 1296252368,1296252383,DE 1296252384,1296252415,IE -1296252416,1296252719,FR +1296252416,1296252703,FR +1296252704,1296252711,DE +1296252712,1296252719,FR 1296252720,1296252727,BE 1296252728,1296252735,IL 1296252736,1296252739,FR @@ -21762,7 +24848,7 @@ 1296255744,1296255999,FR 1296256000,1296257023,DE 1296257024,1296257535,US -1296257536,1296257791,FR +1296257536,1296257791,DE 1296257792,1296258047,US 1296258048,1296258303,TW 1296258304,1296259071,NL @@ -21822,11 +24908,9 @@ 1296466400,1296466415,AO 1296466416,1296466431,NG 1296466432,1296466559,NO -1296466560,1296466575,NG -1296466576,1296466591,NO -1296466592,1296466607,NG -1296466608,1296466623,NO -1296466624,1296466655,NG +1296466560,1296466583,NG +1296466584,1296466591,NO +1296466592,1296466655,NG 1296466656,1296466671,BJ 1296466672,1296466679,NG 1296466680,1296466687,NO @@ -21841,7 +24925,9 @@ 1296472416,1296473087,NO 1296473088,1296474623,LT 1296474624,1296476159,US -1296476160,1296498687,LT +1296476160,1296480255,LT +1296480256,1296481791,NO +1296481792,1296498687,LT 1296498688,1296531455,BG 1296531456,1296564223,MT 1296564224,1296566271,GB @@ -21859,7 +24945,6 @@ 1296590848,1296592895,ES 1296592896,1296594943,NL 1296594944,1296596991,RU -1296596992,1296599039,CH 1296599040,1296601087,RU 1296601088,1296603135,DE 1296603136,1296605183,NL @@ -21881,7 +24966,7 @@ 1296621568,1296623615,SE 1296623616,1296625663,BE 1296625664,1296629759,RU -1296631808,1296633855,GB +1296629760,1296633855,GB 1296633856,1296635903,NL 1296635904,1296637951,RU 1296637952,1296639999,CH @@ -22035,7 +25120,10 @@ 1296990208,1297006591,BG 1297006592,1297018623,BA 1297018624,1297019135,MD -1297019136,1297022975,BA +1297019136,1297020927,BA +1297020928,1297021183,SR +1297021184,1297021439,MD +1297021440,1297022975,BA 1297022976,1297039359,LT 1297039360,1297055743,TR 1297055744,1297072127,RU @@ -22066,6 +25154,7 @@ 1297569792,1297571839,RO 1297571840,1297573887,RU 1297573888,1297575935,FR +1297575936,1297577983,RU 1297577984,1297580031,PL 1297580032,1297582079,GB 1297582080,1297584127,PL @@ -22078,7 +25167,7 @@ 1297598464,1297602559,RU 1297602560,1297604607,PL 1297604608,1297606655,DE -1297606656,1297608703,CZ +1297606656,1297610751,CZ 1297610752,1297612799,DE 1297612800,1297629183,RU 1297629184,1297645567,NO @@ -22105,9 +25194,7 @@ 1297867688,1297867695,CY 1297867696,1297867871,RU 1297867872,1297867879,ES -1297867880,1297868607,RU -1297868608,1297868671,CY -1297868672,1297868799,RU +1297867880,1297868799,RU 1297868800,1297870847,DE 1297870848,1297872895,FR 1297872896,1297874943,GB @@ -22119,9 +25206,7 @@ 1297879041,1297883135,TR 1297883136,1297891327,PL 1297891328,1297899519,GB -1297899520,1297910527,TR -1297910528,1297910783,CZ -1297910784,1297915903,TR +1297899520,1297915903,TR 1297915904,1297924095,BH 1297924096,1297932287,CZ 1297932288,1297940479,GE @@ -22143,12 +25228,14 @@ 1298015488,1298015743,LB 1298015744,1298015999,IQ 1298016000,1298016511,LB -1298016512,1298017279,RU -1298017280,1298018303,KG +1298016512,1298017023,RU +1298017024,1298017791,BY +1298017792,1298018303,RU 1298018304,1298018559,TM 1298018560,1298019327,RU 1298019328,1298019839,LT -1298019840,1298022399,RU +1298019840,1298020351,RU +1298020352,1298022399,BY 1298022400,1298030591,IT 1298030592,1298038783,RU 1298038784,1298046975,SI @@ -22196,6 +25283,7 @@ 1298137088,1298661375,GB 1298661376,1298677759,FR 1298677760,1298694143,IR +1298694144,1298710527,RO 1298710528,1298726911,CZ 1298726912,1298743295,RS 1298743296,1298759679,FI @@ -22212,7 +25300,8 @@ 1298956288,1298972671,GB 1298972672,1298989055,RU 1298989056,1299005439,UA -1299005440,1299005951,BE +1299005440,1299005695,RU +1299005696,1299005951,BE 1299005952,1299008511,NL 1299008512,1299009791,BE 1299009792,1299010047,NL @@ -22226,8 +25315,8 @@ 1299012608,1299013119,GB 1299013120,1299013375,FR 1299013376,1299013631,DE -1299013632,1299016447,BE -1299016448,1299016703,NL +1299013632,1299015679,BE +1299015680,1299016703,NL 1299016704,1299016959,BE 1299016960,1299017215,NL 1299017216,1299017727,RU @@ -22297,10 +25386,8 @@ 1306525696,1307049983,ES 1307049984,1307066367,RU 1307066368,1307069695,EU -1307069696,1307069823,DE -1307069824,1307069855,EU -1307069856,1307069887,DE -1307069888,1307074559,EU +1307069696,1307069919,DE +1307069920,1307074559,EU 1307074560,1307082751,SA 1307082752,1307090943,RU 1307090944,1307099135,GB @@ -22339,7 +25426,9 @@ 1307361280,1307369471,CZ 1307369472,1307377663,RU 1307377664,1307385855,PL -1307385856,1307394047,AT +1307385856,1307393919,AT +1307393920,1307393983,CZ +1307393984,1307394047,AT 1307394048,1307402239,IR 1307402240,1307410431,HR 1307410432,1307418623,CZ @@ -22348,7 +25437,6 @@ 1307435008,1307443199,DE 1307443200,1307451391,IT 1307451392,1307459583,SE -1307459584,1307467775,UZ 1307467776,1307484159,RU 1307484160,1307488767,GB 1307488768,1307489023,IE @@ -22378,21 +25466,21 @@ 1307623424,1307627519,FR 1307627520,1307631615,SE 1307631616,1307635711,IT -1307635712,1307636991,EE -1307636992,1307637031,LV -1307637032,1307638015,EE -1307638016,1307638095,LT -1307638096,1307638375,EE -1307638376,1307638383,LT -1307638384,1307638447,EE -1307638448,1307638463,LT -1307638464,1307638575,EE -1307638576,1307638607,LT -1307638608,1307638783,EE -1307638784,1307639295,LT -1307639296,1307639495,EE -1307639496,1307639503,LT -1307639504,1307639807,EE +1307635712,1307636735,EE +1307636736,1307636807,LV +1307636808,1307636815,EE +1307636816,1307636847,LV +1307636848,1307636863,EE +1307636864,1307637359,LV +1307637360,1307637367,EE +1307637368,1307637383,LV +1307637384,1307637407,EE +1307637408,1307637471,LV +1307637472,1307637491,EE +1307637492,1307637751,LV +1307637752,1307637759,EE +1307637760,1307639791,LT +1307639792,1307639807,EE 1307639808,1307643903,IT 1307643904,1307652095,RU 1307652096,1307656191,ES @@ -22432,7 +25520,13 @@ 1307756448,1307756543,GB 1307756544,1307756847,FR 1307756848,1307756863,US -1307756864,1307757567,FR +1307756864,1307756903,FR +1307756904,1307756911,US +1307756912,1307757063,FR +1307757064,1307757071,US +1307757072,1307757183,FR +1307757184,1307757247,US +1307757248,1307757567,FR 1307757568,1307758591,GB 1307758592,1307762687,SM 1307762688,1307766783,PL @@ -22778,7 +25872,6 @@ 1310251008,1310255103,RU 1310255104,1310257151,CZ 1310257152,1310259199,FR -1310259200,1310261247,DE 1310261248,1310277631,UA 1310277632,1310310399,RU 1310310400,1310326783,GB @@ -22787,7 +25880,7 @@ 1310359552,1310392319,UA 1310392320,1310408703,NL 1310408704,1310425087,FI -1310441472,1310457855,UA +1310425088,1310457855,UA 1310457856,1310474239,PL 1310474240,1310490623,EE 1310490624,1310507007,BA @@ -22838,7 +25931,9 @@ 1311256576,1311258623,FR 1311258624,1311262719,GB 1311262720,1311262975,FR -1311262976,1311263359,CH +1311262976,1311263247,CH +1311263248,1311263263,FR +1311263264,1311263359,CH 1311263360,1311263375,FR 1311263376,1311263407,CH 1311263408,1311263423,BE @@ -22877,6 +25972,7 @@ 1311324160,1311326207,LB 1311326208,1311328255,BE 1311328256,1311330303,NL +1311330304,1311332351,DE 1311332352,1311338495,RU 1311338496,1311340543,AT 1311340544,1311342591,NO @@ -22917,7 +26013,9 @@ 1311369088,1311369215,BT 1311369216,1311371263,FR 1311371264,1311373311,SE -1311373312,1311375359,FR +1311373312,1311374335,FR +1311374336,1311374351,ES +1311374352,1311375359,FR 1311375360,1311506431,DE 1311506432,1311637503,CZ 1311637504,1311707655,DE @@ -23113,10 +26211,12 @@ 1315898000,1315898015,AE 1315898016,1315898031,IR 1315898032,1315898111,AE -1315898112,1315900799,IR +1315898112,1315900671,IR +1315900672,1315900735,AE +1315900736,1315900799,IR 1315900800,1315900927,AE -1315900928,1315901215,IR -1315901216,1315901439,AE +1315900928,1315901223,IR +1315901224,1315901439,AE 1315901440,1315905535,UA 1315905536,1315907583,BA 1315907584,1315908095,SI @@ -23146,9 +26246,7 @@ 1317118208,1317118463,IN 1317118464,1317119103,GB 1317119104,1317119167,IT -1317119168,1317119487,GB -1317119488,1317119743,KY -1317119744,1317126399,GB +1317119168,1317126399,GB 1317126400,1317126655,US 1317126656,1317129279,GB 1317129280,1317129343,CA @@ -23743,7 +26841,9 @@ 1317675952,1317675959,NG 1317675960,1317676287,A2 1317676288,1317676543,LR -1317676544,1317676807,A2 +1317676544,1317676663,A2 +1317676664,1317676671,NG +1317676672,1317676807,A2 1317676808,1317676815,NG 1317676816,1317676823,A2 1317676824,1317676831,NG @@ -23770,9 +26870,7 @@ 1317686336,1317686399,NL 1317686400,1317695743,DE 1317695744,1317695999,CH -1317696000,1317696191,DE -1317696192,1317696255,CN -1317696256,1317698559,DE +1317696000,1317698559,DE 1317698560,1317698687,NL 1317698688,1317699583,DE 1317699584,1317715967,RU @@ -23792,12 +26890,18 @@ 1317835648,1317836863,NL 1317836864,1317836927,CY 1317836928,1317839359,NL -1317839360,1317839615,BZ -1317839616,1317841407,NL +1317839360,1317839615,RU +1317839616,1317839679,HK +1317839680,1317841407,NL 1317841408,1317841439,SC 1317841440,1317841727,NL 1317841728,1317841855,CY -1317841856,1317847039,NL +1317841856,1317842943,NL +1317842944,1317843071,CY +1317843072,1317843135,HK +1317843136,1317843199,CY +1317843200,1317843327,US +1317843328,1317847039,NL 1317847040,1317863423,RU 1317863424,1317879807,GB 1317879808,1317896191,SK @@ -23808,12 +26912,14 @@ 1317965048,1317965055,DE 1317965056,1317978111,AT 1317978112,1317994495,RU -1317994496,1317996287,NL -1317996288,1317996415,DE -1317996416,1317996543,NL -1317996544,1317997567,DE +1317994496,1317996031,NL +1317996032,1317996095,DE +1317996096,1317996287,NL +1317996288,1317997567,DE 1317997568,1317998591,NL -1317998592,1318000383,DE +1317998592,1317999615,DE +1317999616,1317999647,NL +1317999648,1318000383,DE 1318000384,1318000447,NL 1318000448,1318002175,DE 1318002176,1318002431,NL @@ -23821,7 +26927,9 @@ 1318005632,1318005759,NL 1318005760,1318006271,DE 1318006272,1318006783,NL -1318006784,1318009423,DE +1318006784,1318007999,DE +1318008000,1318008031,NL +1318008032,1318009423,DE 1318009424,1318009471,NL 1318009472,1318010527,DE 1318010528,1318010879,NL @@ -23896,7 +27004,7 @@ 1318920192,1318928383,IR 1318928384,1318936575,LT 1318936576,1318944767,DK -1318952960,1318956287,CZ +1318944768,1318956287,CZ 1318956288,1318957055,PL 1318957056,1318958847,CZ 1318958848,1318961151,PL @@ -23972,6 +27080,7 @@ 1331841024,1331843071,CZ 1331843072,1331845119,GB 1331845120,1331847167,RU +1331847168,1331849215,FR 1331849216,1331851263,RU 1331851264,1331853311,AL 1331853312,1331855359,AE @@ -24011,7 +27120,7 @@ 1331929088,1331931135,FR 1331931136,1331933183,SE 1331933184,1331935231,TR -1331935232,1331937535,GB +1331937280,1331937535,GB 1331937536,1331937567,SM 1331937568,1331937583,US 1331937584,1331937599,ZA @@ -24044,9 +27153,7 @@ 1332412416,1332477951,GR 1332477952,1332609023,ES 1332609024,1332613119,PL -1332613120,1332621311,UA -1332621312,1332625407,KZ -1332625408,1332629503,UA +1332613120,1332629503,UA 1332629504,1332633599,DE 1332633600,1332637695,UA 1332637696,1332641791,BG @@ -24064,10 +27171,7 @@ 1333428224,1333460991,RU 1333460992,1333493759,GB 1333493760,1333526527,TR -1333526528,1333542911,BG -1333542912,1333551103,RO -1333551104,1333555199,UA -1333555200,1333559295,RO +1333526528,1333559295,BG 1333559296,1333592063,UA 1333592064,1333624831,IE 1333624832,1333657599,MK @@ -24094,7 +27198,6 @@ 1334165504,1334173695,PL 1334173696,1334181887,LT 1334181888,1334190079,RU -1334190080,1334198271,RO 1334198272,1334206463,RU 1334206464,1334214655,FI 1334214656,1334222847,JO @@ -24141,9 +27244,7 @@ 1334584904,1334584911,AT 1334584912,1334584999,DE 1334585000,1334585007,US -1334585008,1334585055,DE -1334585056,1334585087,US -1334585088,1334585111,DE +1334585008,1334585111,DE 1334585112,1334585119,AT 1334585120,1334591487,DE 1334591488,1334595583,RU @@ -24174,7 +27275,7 @@ 1334632448,1334636543,KZ 1334636544,1334640639,RU 1334640640,1334644735,FR -1334644736,1334644991,IQ +1334644736,1334644991,LU 1334644992,1334645247,A2 1334645248,1334648319,IQ 1334648320,1334648831,A2 @@ -24236,7 +27337,6 @@ 1334730740,1334734847,RU 1334734848,1334738943,LT 1334738944,1334743039,CH -1334743040,1334747135,UA 1334747136,1334751231,RU 1334751232,1334755327,BH 1334755328,1334759423,UA @@ -24388,8 +27488,8 @@ 1343218688,1343219711,FR 1343219712,1343219967,KR 1343219968,1343220479,FR -1343220480,1343220607,DE -1343220608,1343220735,FR +1343220480,1343220639,DE +1343220640,1343220735,FR 1343220736,1343220863,GB 1343220864,1343221023,FR 1343221024,1343221027,DE @@ -24431,12 +27531,13 @@ 1346379776,1346383871,ES 1346383872,1346387967,GB 1346387968,1346392063,HU +1346392064,1346396159,UA 1346396160,1346400255,RU 1346400256,1346404351,IT 1346404352,1346404859,AT 1346404860,1346404863,NL -1346404864,1346404919,AT -1346404920,1346404943,NL +1346404864,1346404927,AT +1346404928,1346404943,NL 1346404944,1346405087,AT 1346405088,1346405103,NL 1346405104,1346405135,AT @@ -24612,11 +27713,7 @@ 1346793472,1346797567,IT 1346797568,1346801663,IL 1346801664,1346805759,SE -1346805760,1346806079,DE -1346806080,1346806111,GB -1346806112,1346806167,DE -1346806168,1346806207,SE -1346806208,1346818047,DE +1346805760,1346818047,DE 1346818048,1346822143,FR 1346822144,1346826239,RU 1346826240,1346830335,RS @@ -24626,7 +27723,6 @@ 1346846720,1346854911,DK 1346854912,1346859007,MT 1346859008,1346863103,IR -1346863104,1346867199,TR 1346867200,1346871295,BA 1346871296,1346879487,RU 1346879488,1346883583,FI @@ -24712,7 +27808,7 @@ 1347158016,1347162111,CH 1347162112,1347166207,IT 1347166208,1347174399,GR -1347174400,1347186687,IT +1347174400,1347182591,IT 1347186688,1347190783,GB 1347190784,1347194879,RU 1347194880,1347198975,SE @@ -24750,8 +27846,7 @@ 1347239936,1347244031,DK 1347244032,1347247359,GB 1347247360,1347247871,RU -1347247872,1347247872,GB -1347247873,1347248127,US +1347247872,1347248127,US 1347248128,1347248863,SE 1347248864,1347248864,FI 1347248865,1347249343,SE @@ -24771,9 +27866,7 @@ 1347256320,1347260415,FO 1347260416,1347264511,MK 1347264512,1347268607,RU -1347268608,1347271935,LB -1347271936,1347272447,IQ -1347272448,1347272703,LB +1347268608,1347272703,LB 1347272704,1347276799,MT 1347276800,1347278831,DE 1347278832,1347278847,BR @@ -24831,7 +27924,7 @@ 1347293724,1347293727,IQ 1347293728,1347293735,A2 1347293736,1347293759,GN -1347293760,1347293767,NG +1347293760,1347293767,US 1347293768,1347293775,CM 1347293776,1347293799,NG 1347293800,1347293807,IQ @@ -24956,22 +28049,13 @@ 1347342336,1347346431,SE 1347346432,1347350527,BG 1347350528,1347354623,RU -1347354624,1347358719,PT 1347358720,1347362815,CZ 1347362816,1347366911,NL 1347366912,1347371007,IT 1347371008,1347375103,RU 1347375104,1347379199,GB 1347379200,1347383295,NL -1347383296,1347383695,EE -1347383696,1347383699,IL -1347383700,1347384231,EE -1347384232,1347384235,IL -1347384236,1347385031,EE -1347385032,1347385035,IL -1347385036,1347385171,EE -1347385172,1347385175,IL -1347385176,1347387391,EE +1347383296,1347387391,EE 1347387392,1347391487,GB 1347391488,1347395583,LB 1347395584,1347399679,SE @@ -24995,7 +28079,6 @@ 1347424352,1347427135,DK 1347427136,1347427327,NO 1347427328,1347428351,DK -1347428352,1347432447,IT 1347432448,1347436543,HR 1347436544,1347440639,SE 1347440640,1347444735,ES @@ -25027,8 +28110,10 @@ 1347504960,1347505023,GB 1347505024,1347505151,ES 1347505152,1347505407,GB -1347505408,1347506175,ES -1347506176,1347510271,FR +1347505408,1347505663,DE +1347505664,1347506175,ES +1347506176,1347506431,GB +1347506432,1347510271,FR 1347510272,1347518463,ES 1347518464,1347522559,AT 1347522560,1347526655,LB @@ -25094,9 +28179,7 @@ 1347716152,1347716159,LB 1347716160,1347716207,ES 1347716208,1347716223,CY -1347716224,1347718367,ES -1347718368,1347718399,IR -1347718400,1347719711,ES +1347716224,1347719711,ES 1347719712,1347719719,SO 1347719720,1347722495,ES 1347722496,1347722751,LB @@ -25128,9 +28211,8 @@ 1347755192,1347755223,CY 1347755224,1347755231,GR 1347755232,1347755263,CY -1347755264,1347755519,GR -1347755520,1347755775,CY -1347755776,1347756031,GR +1347755264,1347755775,GR +1347755776,1347756031,CY 1347756032,1347760127,NL 1347760128,1347764223,HU 1347764224,1347772415,GB @@ -25263,12 +28345,8 @@ 1347978408,1347978447,A2 1347978448,1347978463,NG 1347978464,1347978575,A2 -1347978576,1347978663,NG -1347978664,1347978671,A2 -1347978672,1347978679,NG -1347978680,1347978687,A2 -1347978688,1347978695,NG -1347978696,1347978719,A2 +1347978576,1347978655,NG +1347978656,1347978719,A2 1347978720,1347978735,NG 1347978736,1347978775,A2 1347978776,1347978783,NG @@ -25288,17 +28366,13 @@ 1347979232,1347979247,NG 1347979248,1347979375,A2 1347979376,1347979399,NG -1347979400,1347979527,A2 -1347979528,1347979543,NG -1347979544,1347979567,A2 +1347979400,1347979567,A2 1347979568,1347979575,NG 1347979576,1347979583,A2 1347979584,1347979591,NG 1347979592,1347979743,A2 1347979744,1347979751,NG -1347979752,1347979775,A2 -1347979776,1347979783,NG -1347979784,1347980079,A2 +1347979752,1347980079,A2 1347980080,1347980095,GH 1347980096,1347980111,NG 1347980112,1347980127,GN @@ -25311,9 +28385,7 @@ 1347980416,1347980479,NG 1347980480,1347980543,A2 1347980544,1347980559,DK -1347980560,1347980671,A2 -1347980672,1347980703,NG -1347980704,1347981007,A2 +1347980560,1347981007,A2 1347981008,1347981015,NG 1347981016,1347981023,A2 1347981024,1347981031,NG @@ -25361,9 +28433,7 @@ 1347983984,1347984127,NG 1347984128,1347984207,A2 1347984208,1347984215,NG -1347984216,1347984383,A2 -1347984384,1347984391,NG -1347984392,1347984415,A2 +1347984216,1347984415,A2 1347984416,1347984423,NG 1347984424,1347984647,A2 1347984648,1347984655,NG @@ -25371,12 +28441,12 @@ 1347984720,1347984735,NG 1347984736,1347984879,A2 1347984880,1347984895,NG -1347984896,1347984927,A2 -1347984928,1347984943,NG +1347984896,1347984935,A2 +1347984936,1347984943,NG 1347984944,1347984959,A2 1347984960,1347984991,NG -1347984992,1347985023,A2 -1347985024,1347985039,NG +1347984992,1347985031,A2 +1347985032,1347985039,NG 1347985040,1347985095,A2 1347985096,1347985119,NG 1347985120,1347985151,A2 @@ -25439,18 +28509,22 @@ 1348206592,1348218879,RU 1348218880,1348219295,DE 1348219296,1348219327,A2 -1348219328,1348220415,DE -1348220416,1348220479,SA -1348220480,1348220511,DE -1348220512,1348220543,SA -1348220544,1348220687,DE +1348219328,1348219343,DE +1348219344,1348219359,ZM +1348219360,1348219391,DE +1348219392,1348219647,US +1348219648,1348219903,DE +1348219904,1348220415,LB +1348220416,1348220687,DE 1348220688,1348220703,SA 1348220704,1348220711,DE 1348220712,1348220735,SA 1348220736,1348220799,BH 1348220800,1348220863,DE 1348220864,1348220895,BH -1348220896,1348222975,DE +1348220896,1348220927,DE +1348220928,1348221183,US +1348221184,1348222975,DE 1348222976,1348231167,LU 1348231168,1348235263,RU 1348235264,1348239359,AT @@ -25587,7 +28661,9 @@ 1351965184,1351965439,US 1351965440,1351991295,DE 1351991296,1351991551,IT -1351991552,1352010239,DE +1351991552,1352002047,DE +1352002048,1352002303,US +1352002304,1352010239,DE 1352010240,1352010495,US 1352010496,1352011447,DE 1352011448,1352011455,IT @@ -25614,7 +28690,11 @@ 1352148152,1352148159,IT 1352148160,1352148751,DE 1352148752,1352148759,US -1352148760,1352277535,DE +1352148760,1352149415,DE +1352149416,1352149423,FI +1352149424,1352149431,DE +1352149432,1352149447,FR +1352149448,1352277535,DE 1352277536,1352277567,IT 1352277568,1352402791,DE 1352402792,1352402799,BE @@ -25630,15 +28710,9 @@ 1353255072,1353255087,CA 1353255088,1353255839,GB 1353255840,1353255855,FR -1353255856,1353257015,GB -1353257016,1353257023,IE -1353257024,1353257111,GB +1353255856,1353257111,GB 1353257112,1353257119,ES -1353257120,1353257151,GB -1353257152,1353257159,IE -1353257160,1353257191,GB -1353257192,1353257199,IE -1353257200,1353257983,GB +1353257120,1353257983,GB 1353257984,1353258247,SE 1353258248,1353258255,GB 1353258256,1353258263,SE @@ -25687,9 +28761,7 @@ 1353272848,1353272879,GB 1353272880,1353272927,ES 1353272928,1353272959,GB -1353272960,1353273007,ES -1353273008,1353273031,GB -1353273032,1353273047,ES +1353272960,1353273047,ES 1353273048,1353273055,GB 1353273056,1353273343,ES 1353273344,1353273631,BE @@ -25697,17 +28769,13 @@ 1353273640,1353273711,BE 1353273712,1353273719,GB 1353273720,1353274367,BE -1353274368,1353274655,ES -1353274656,1353274663,GB -1353274664,1353274671,ES +1353274368,1353274671,ES 1353274672,1353274679,GB 1353274680,1353274687,ES 1353274688,1353274703,GB 1353274704,1353274759,ES 1353274760,1353274767,GB -1353274768,1353274771,ES -1353274772,1353274775,GB -1353274776,1353274791,ES +1353274768,1353274791,ES 1353274792,1353274799,GB 1353274800,1353274831,ES 1353274832,1353274839,GB @@ -25736,16 +28804,12 @@ 1353279760,1353279767,GB 1353279768,1353279783,IT 1353279784,1353279791,GB -1353279792,1353280591,IT -1353280592,1353280607,GB -1353280608,1353280655,IT -1353280656,1353280663,GB -1353280664,1353280671,IT -1353280672,1353280679,GB -1353280680,1353281023,IT +1353279792,1353280599,IT +1353280600,1353280607,GB +1353280608,1353281023,IT 1353281024,1353281535,BE -1353281536,1353282055,GB -1353282056,1353282559,IT +1353281536,1353282047,GB +1353282048,1353282559,IT 1353282560,1353283071,GB 1353283072,1353283327,IT 1353283328,1353287327,GB @@ -25772,8 +28836,8 @@ 1353288832,1353288839,GB 1353288840,1353288847,IE 1353288848,1353288855,GB -1353288856,1353288863,IE -1353288864,1353288895,GB +1353288856,1353288871,IE +1353288872,1353288895,GB 1353288896,1353289247,IE 1353289248,1353289255,GB 1353289256,1353289359,IE @@ -25783,7 +28847,9 @@ 1353289408,1353289623,IE 1353289624,1353289631,GB 1353289632,1353289647,IE -1353289648,1353290751,GB +1353289648,1353289663,GB +1353289664,1353289727,IE +1353289728,1353290751,GB 1353290752,1353290815,DE 1353290816,1353290927,GB 1353290928,1353290935,IE @@ -25801,9 +28867,7 @@ 1353299712,1353299839,GB 1353299840,1353299855,SE 1353299856,1353299863,GB -1353299864,1353299871,SE -1353299872,1353299887,GB -1353299888,1353300063,SE +1353299864,1353300063,SE 1353300064,1353300071,GB 1353300072,1353300079,SE 1353300080,1353300103,GB @@ -25818,17 +28882,19 @@ 1353300280,1353300735,SE 1353300736,1353301095,GB 1353301096,1353301103,US -1353301104,1353304879,GB -1353304880,1353304895,ES -1353304896,1353305599,GB +1353301104,1353305599,GB 1353305600,1353305607,IE 1353305608,1353306111,GB 1353306112,1353306623,ES 1353306624,1353306624,BE 1353306625,1353306687,GB -1353306688,1353306735,BE +1353306688,1353306695,BE +1353306696,1353306703,GB +1353306704,1353306735,BE 1353306736,1353306751,GB -1353306752,1353307135,BE +1353306752,1353306831,BE +1353306832,1353306847,GB +1353306848,1353307135,BE 1353307136,1353307143,IT 1353307144,1353308159,GB 1353308160,1353309183,FR @@ -25841,7 +28907,9 @@ 1353311184,1353311231,IT 1353311232,1353312255,GB 1353312256,1353312767,CH -1353312768,1353313191,IT +1353312768,1353313111,IT +1353313112,1353313119,GB +1353313120,1353313191,IT 1353313192,1353313199,GB 1353313200,1353313279,IT 1353313280,1353313791,IE @@ -25891,8 +28959,8 @@ 1353956864,1353968639,IL 1353968640,1353968895,A2 1353968896,1353970431,IL -1353970432,1353970943,A2 -1353970944,1353973759,IL +1353970432,1353970687,A2 +1353970688,1353973759,IL 1353973760,1354235903,IT 1354235904,1354301439,KW 1354301440,1354366975,FR @@ -26093,7 +29161,10 @@ 1357351168,1357351423,PL 1357351424,1357351935,EU 1357351936,1357352959,GB -1357352960,1357356031,EU +1357352960,1357355007,EU +1357355008,1357355263,NL +1357355264,1357355775,FR +1357355776,1357356031,GB 1357356032,1357356543,ES 1357356544,1357357055,EU 1357357056,1357357567,ES @@ -26163,7 +29234,9 @@ 1357643776,1357676543,FR 1357676544,1357709311,BE 1357709312,1357742079,RU -1357742080,1357759183,DE +1357742080,1357759079,DE +1357759080,1357759087,AM +1357759088,1357759183,DE 1357759184,1357759191,AT 1357759192,1357765823,DE 1357765824,1357765831,AT @@ -26187,10 +29260,7 @@ 1357875712,1357875967,PL 1357875968,1357876143,DE 1357876144,1357876151,SE -1357876152,1357876199,DE -1357876200,1357876207,EU -1357876208,1357876215,DE -1357876216,1357876223,EU +1357876152,1357876223,DE 1357876224,1357876239,NO 1357876240,1357876287,EU 1357876288,1357876303,PL @@ -26229,34 +29299,33 @@ 1357883520,1357883535,EU 1357883536,1357883551,FR 1357883552,1357883583,EU -1357883584,1357883719,FR -1357883720,1357883759,EU +1357883584,1357883727,FR +1357883728,1357883759,EU 1357883760,1357883935,FR 1357883936,1357883999,EU 1357884000,1357884031,FR 1357884032,1357884159,EU 1357884160,1357884423,FR -1357884424,1357884439,EU +1357884424,1357884427,BE +1357884428,1357884439,EU 1357884440,1357884447,FR 1357884448,1357884511,EU 1357884512,1357884935,FR -1357884936,1357884991,EU -1357884992,1357885055,FR -1357885056,1357885119,EU -1357885120,1357885183,FR -1357885184,1357885199,EU -1357885200,1357885247,ES +1357884936,1357884959,EU +1357884960,1357884991,RU +1357884992,1357885183,FR +1357885184,1357885215,EU +1357885216,1357885247,ES 1357885248,1357885439,DE 1357885440,1357885695,FR 1357885696,1357885951,DE 1357885952,1357886463,FR 1357886464,1357887487,US 1357887488,1357888511,FR -1357888512,1357888639,EU -1357888640,1357888703,DE +1357888512,1357888703,DE 1357888704,1357888767,EU 1357888768,1357889023,DE -1357889024,1357889535,EU +1357889024,1357889535,GB 1357889536,1357890815,PL 1357890816,1357890827,NL 1357890828,1357890847,EU @@ -26271,7 +29340,7 @@ 1357891456,1357891583,SE 1357891584,1357891647,EU 1357891648,1357891679,NL -1357891680,1357891711,EU +1357891680,1357891711,GB 1357891712,1357891839,NL 1357891840,1357892095,GB 1357892096,1357892351,EU @@ -26291,7 +29360,9 @@ 1357899048,1357899055,DE 1357899056,1357899071,EU 1357899072,1357899199,DE -1357899200,1357899327,EU +1357899200,1357899279,EU +1357899280,1357899287,RU +1357899288,1357899327,EU 1357899328,1357899391,PL 1357899392,1357899551,EU 1357899552,1357899567,AT @@ -26304,13 +29375,14 @@ 1357900544,1357900799,NL 1357900800,1357901823,DE 1357901824,1357902335,ES -1357902336,1357902847,A1 +1357902336,1357902847,PL 1357902848,1357903359,GB 1357903360,1357903615,DE 1357903616,1357903871,SE 1357903872,1357904383,EU 1357904384,1357904895,DE -1357904896,1357905919,EU +1357904896,1357905407,GB +1357905408,1357905919,EU 1357905920,1357910015,LT 1357910016,1357914111,GR 1357914112,1357922303,RU @@ -26339,13 +29411,16 @@ 1357988752,1357988775,DE 1357988776,1357988783,GB 1357988784,1357988863,DE -1357988864,1357989647,GB +1357988864,1357989375,GB +1357989376,1357989383,CH +1357989384,1357989647,GB 1357989648,1357989655,FR 1357989656,1357989663,GB 1357989664,1357989703,FR 1357989704,1357989711,GB 1357989712,1357989727,FR -1357989728,1357989759,GB +1357989728,1357989743,GB +1357989744,1357989759,CH 1357989760,1357989783,FR 1357989784,1357989791,GB 1357989792,1357989807,FR @@ -26438,8 +29513,8 @@ 1358223872,1358223911,DE 1358223912,1358223927,NL 1358223928,1358223967,DE -1358223968,1358224063,NL -1358224064,1358224511,DE +1358223968,1358224055,NL +1358224056,1358224511,DE 1358224512,1358224519,BE 1358224520,1358224611,DE 1358224612,1358224651,BE @@ -26469,8 +29544,8 @@ 1358242624,1358244351,FR 1358244352,1358244607,IT 1358244608,1358245695,FR -1358245696,1358245727,BE -1358245728,1358249215,FR +1358245696,1358245759,BE +1358245760,1358249215,FR 1358249216,1358249471,IT 1358249472,1358249983,FR 1358249984,1358251815,DE @@ -26621,7 +29696,6 @@ 1358528512,1358536703,RU 1358536704,1358540799,HU 1358540800,1358544895,RU -1358544896,1358548991,LV 1358548992,1358553087,JO 1358553088,1358557183,UA 1358557184,1358559503,IE @@ -26667,26 +29741,40 @@ 1358668984,1358668991,GB 1358668992,1358669351,PT 1358669352,1358669359,GB -1358669360,1358669423,PT -1358669424,1358669439,GB +1358669360,1358669431,PT +1358669432,1358669439,GB 1358669440,1358669455,PT 1358669456,1358669471,GB 1358669472,1358669503,PT 1358669504,1358669519,GB 1358669520,1358669543,PT 1358669544,1358669551,GB -1358669552,1358670015,PT +1358669552,1358669967,PT +1358669968,1358669999,GB +1358670000,1358670015,PT 1358670016,1358670023,GB 1358670024,1358670071,PT 1358670072,1358670079,GB 1358670080,1358670135,PT -1358670136,1358670335,GB +1358670136,1358670143,GB +1358670144,1358670175,PT +1358670176,1358670335,GB 1358670336,1358671415,PT 1358671416,1358671423,GB 1358671424,1358671431,PT 1358671432,1358671439,GB -1358671440,1358671871,PT -1358671872,1358675967,GB +1358671440,1358671839,PT +1358671840,1358671855,ES +1358671856,1358671967,PT +1358671968,1358671983,GB +1358671984,1358672479,PT +1358672480,1358672511,GB +1358672512,1358672687,PT +1358672688,1358672703,GB +1358672704,1358673535,PT +1358673536,1358673567,US +1358673568,1358673663,PT +1358673664,1358675967,GB 1358675968,1358676735,SE 1358676736,1358677759,DK 1358677760,1358679295,SE @@ -26807,9 +29895,7 @@ 1358876672,1358880767,LV 1358880768,1358884863,GB 1358884864,1358888959,SE -1358888960,1358890943,DE -1358890944,1358891007,AT -1358891008,1358893055,DE +1358888960,1358893055,DE 1358893056,1358897151,RU 1358897152,1358898175,A2 1358898176,1358898239,CA @@ -27247,7 +30333,6 @@ 1360556032,1360564223,RU 1360564224,1360568319,AT 1360568320,1360572415,ES -1360572416,1360576511,GR 1360576512,1360580607,GB 1360580608,1360584703,UA 1360584704,1360588799,IT @@ -27286,6 +30371,7 @@ 1360699392,1360703487,DE 1360703488,1360707583,RU 1360707584,1360709631,IT +1360709632,1360711679,AL 1360711680,1360715775,NL 1360715776,1360728063,SE 1360728064,1360732159,FR @@ -27440,7 +30526,7 @@ 1361035544,1361035547,ES 1361035548,1361035551,DE 1361035552,1361035555,IQ -1361035556,1361035559,GI +1361035556,1361035559,NL 1361035560,1361035563,IQ 1361035564,1361035567,DE 1361035568,1361035575,ES @@ -27472,7 +30558,7 @@ 1361035752,1361035755,GB 1361035756,1361035759,IQ 1361035760,1361035763,ES -1361035764,1361035767,IE +1361035764,1361035767,NL 1361035768,1361035775,IQ 1361035776,1361035779,GB 1361035780,1361035783,PL @@ -27916,6 +31002,7 @@ 1364905984,1364910079,RU 1364910080,1364914175,UA 1364914176,1364918271,CZ +1364918272,1364922367,SE 1364922368,1364926463,TR 1364926464,1364934655,RU 1364934656,1364938751,FR @@ -27938,12 +31025,13 @@ 1364963328,1364967423,RU 1364967424,1364969983,GB 1364969984,1364970239,FR -1364970240,1364971519,GB +1364970240,1364970495,US +1364970496,1364971519,GB 1364971520,1364975615,CZ 1364975616,1364979711,BJ 1364979712,1364983039,GB -1364983040,1364983295,CH -1364983296,1364983807,GB +1364983040,1364983551,CH +1364983552,1364983807,GB 1364983808,1364991999,DE 1364992000,1364996095,IT 1364996096,1365000191,GR @@ -28273,7 +31361,8 @@ 1372045088,1372045119,A2 1372045120,1372045151,NG 1372045152,1372045567,A2 -1372045568,1372045887,NG +1372045568,1372045823,ZM +1372045824,1372045887,NG 1372045888,1372046559,A2 1372046560,1372046591,NG 1372046592,1372047231,A2 @@ -28318,7 +31407,9 @@ 1372152824,1372152831,GB 1372152832,1372159999,DE 1372160000,1372164095,GB -1372164096,1372168191,DE +1372164096,1372166863,DE +1372166864,1372166879,US +1372166880,1372168191,DE 1372168192,1372172287,GB 1372172288,1372176383,FR 1372176384,1372180479,CH @@ -28357,8 +31448,8 @@ 1372691456,1372691711,US 1372691712,1372691967,EU 1372691968,1372694271,DE -1372694272,1372694335,EU -1372694336,1372694367,DE +1372694272,1372694303,EU +1372694304,1372694367,DE 1372694368,1372694383,EU 1372694384,1372694399,DE 1372694400,1372694559,EU @@ -28370,9 +31461,7 @@ 1372694816,1372694831,EU 1372694832,1372695039,DE 1372695040,1372695167,EU -1372695168,1372695295,DE -1372695296,1372695423,EU -1372695424,1372695551,DE +1372695168,1372695551,DE 1372695552,1372695807,CZ 1372695808,1372696007,DK 1372696008,1372696063,EU @@ -28415,9 +31504,7 @@ 1372848128,1373110271,TR 1373110272,1373175807,SE 1373175808,1373241343,AT -1373241344,1373299967,IL -1373299968,1373300223,NL -1373300224,1373306879,IL +1373241344,1373306879,IL 1373306880,1373372415,PL 1373372416,1373437951,FR 1373437952,1373503487,CH @@ -28500,7 +31587,13 @@ 1382154240,1382170623,FR 1382170624,1382171776,DE 1382171777,1382171791,CH -1382171792,1382177983,DE +1382171792,1382173663,DE +1382173664,1382173671,GB +1382173672,1382173675,DE +1382173676,1382173679,CH +1382173680,1382176099,DE +1382176100,1382176103,CH +1382176104,1382177983,DE 1382177984,1382178303,LI 1382178304,1382179247,DE 1382179248,1382179279,GB @@ -28508,7 +31601,9 @@ 1382179840,1382180863,CN 1382180864,1382182327,DE 1382182328,1382182335,GB -1382182336,1382182799,DE +1382182336,1382182547,DE +1382182548,1382182551,CH +1382182552,1382182799,DE 1382182800,1382182831,GB 1382182832,1382182895,DE 1382182896,1382182911,GB @@ -28564,7 +31659,9 @@ 1382218912,1382218959,NL 1382218960,1382218991,GB 1382218992,1382219775,NL -1382219776,1382224287,SE +1382219776,1382223551,SE +1382223552,1382223615,NO +1382223616,1382224287,SE 1382224288,1382224319,NO 1382224320,1382226271,SE 1382226272,1382226303,NO @@ -28617,12 +31714,19 @@ 1383096320,1383096383,PL 1383096384,1383096447,GB 1383096448,1383096575,FR -1383096576,1383096607,PL -1383096608,1383096831,GB +1383096576,1383096831,GB 1383096832,1383096863,FR -1383096864,1383099391,GB -1383099392,1383099615,DE -1383099616,1383103271,GB +1383096864,1383098111,GB +1383098112,1383098367,DE +1383098368,1383099391,GB +1383099392,1383099903,DE +1383099904,1383100159,GB +1383100160,1383100255,FR +1383100256,1383100831,GB +1383100832,1383100847,IE +1383100848,1383100879,GB +1383100880,1383100895,IE +1383100896,1383103271,GB 1383103272,1383103279,ES 1383103280,1383103471,GB 1383103472,1383103487,NL @@ -28764,8 +31868,8 @@ 1383596032,1384120319,FR 1384120320,1384153087,NG 1384153088,1384185855,FI -1384185856,1384187903,DE -1384187904,1384188031,CH +1384185856,1384187967,DE +1384187968,1384188031,CH 1384188032,1384189055,DE 1384189056,1384189183,CH 1384189184,1384189439,AT @@ -28914,7 +32018,7 @@ 1385259008,1385267199,IT 1385267200,1385275391,SE 1385275392,1385283583,IT -1385283584,1385291775,KW +1385283584,1385287679,KW 1385291776,1385299967,TR 1385299968,1385308159,BG 1385308160,1385310207,BE @@ -28976,19 +32080,21 @@ 1385565440,1385566207,EU 1385566208,1385566399,FR 1385566400,1385566431,EU -1385566432,1385566847,FR +1385566432,1385566447,FR +1385566448,1385566455,EU +1385566456,1385566847,FR 1385566848,1385566927,EU 1385566928,1385566935,FR -1385566936,1385566943,EU -1385566944,1385566991,FR +1385566936,1385566959,EU +1385566960,1385566991,FR 1385566992,1385566999,EU 1385567000,1385567023,FR 1385567024,1385567087,EU 1385567088,1385567103,FR 1385567104,1385567135,EU 1385567136,1385567155,FR -1385567156,1385567167,EU -1385567168,1385567215,FR +1385567156,1385567159,EU +1385567160,1385567215,FR 1385567216,1385567223,EU 1385567224,1385567227,FR 1385567228,1385567231,EU @@ -28997,9 +32103,7 @@ 1385568768,1385569279,EU 1385569280,1385569407,FR 1385569408,1385569487,CZ -1385569488,1385569527,EU -1385569528,1385569535,GB -1385569536,1385569791,EU +1385569488,1385569791,EU 1385569792,1385570303,FR 1385570304,1385578495,HU 1385578496,1385586687,TR @@ -29079,8 +32183,8 @@ 1388389440,1388389471,NG 1388389472,1388389567,IT 1388389568,1388389631,NG -1388389632,1388390399,IT -1388390400,1388394495,NG +1388389632,1388390143,IT +1388390144,1388394495,NG 1388394496,1388395519,IT 1388395520,1388396287,NG 1388396288,1388396543,IT @@ -29167,7 +32271,6 @@ 1388642304,1388650495,FI 1388650496,1388658687,PL 1388658688,1388666879,GB -1388666880,1388675071,GR 1388675072,1388675327,NL 1388675328,1388675583,EU 1388675584,1388676095,DE @@ -29226,7 +32329,8 @@ 1388731137,1388731391,AT 1388731392,1388732415,DE 1388732416,1388740607,ES -1388740608,1388740719,IE +1388740608,1388740623,GB +1388740624,1388740719,IE 1388740720,1388740735,GB 1388740736,1388741745,IE 1388741746,1388741819,GB @@ -29238,7 +32342,9 @@ 1388742020,1388742023,GB 1388742024,1388742024,IE 1388742025,1388742031,GB -1388742032,1388743043,IE +1388742032,1388742655,IE +1388742656,1388742739,GB +1388742740,1388743043,IE 1388743044,1388743055,GB 1388743056,1388743435,IE 1388743436,1388743439,GB @@ -29246,9 +32352,11 @@ 1388744088,1388744095,GB 1388744096,1388744117,IE 1388744118,1388744127,GB -1388744128,1388744259,IE -1388744260,1388744271,GB -1388744272,1388744893,IE +1388744128,1388744215,IE +1388744216,1388744219,GB +1388744220,1388744251,IE +1388744252,1388744255,GB +1388744256,1388744893,IE 1388744894,1388744895,GB 1388744896,1388744911,IE 1388744912,1388744915,GB @@ -29267,9 +32375,11 @@ 1388745728,1388745927,IE 1388745928,1388745935,GB 1388745936,1388745959,IE -1388745960,1388745971,GB -1388745972,1388746171,IE -1388746172,1388746195,GB +1388745960,1388745975,GB +1388745976,1388746143,IE +1388746144,1388746155,GB +1388746156,1388746159,IE +1388746160,1388746195,GB 1388746196,1388746207,IE 1388746208,1388746239,GB 1388746240,1388746659,IE @@ -29316,7 +32426,9 @@ 1388765184,1388773375,GB 1388773376,1388781567,NO 1388781568,1388789759,ES -1388789760,1388797951,NL +1388789760,1388789887,NL +1388789888,1388789951,GB +1388789952,1388797951,NL 1388797952,1388806143,RU 1388806144,1388814335,DE 1388814336,1388821119,AX @@ -29716,7 +32828,9 @@ 1389505024,1389510655,DE 1389510656,1389527039,NL 1389527040,1389543423,PL -1389543424,1389548991,DE +1389543424,1389544927,DE +1389544928,1389544943,SG +1389544944,1389548991,DE 1389548992,1389549055,BE 1389549056,1389554023,DE 1389554024,1389554039,FR @@ -29875,7 +32989,9 @@ 1400716288,1400717311,EU 1400717312,1400718335,DE 1400718336,1400718847,EU -1400718848,1400722431,DE +1400718848,1400721407,DE +1400721408,1400721919,EU +1400721920,1400722431,DE 1400722432,1400729599,EU 1400729600,1400730639,DE 1400730640,1400731135,EU @@ -29943,11 +33059,11 @@ 1401477120,1401479167,BE 1401479168,1401481215,PL 1401481216,1401485311,RU -1401485312,1401487359,IT 1401487360,1401489407,GB 1401489408,1401491455,SE 1401491456,1401493503,NL 1401493504,1401495551,CH +1401495552,1401497599,DE 1401497600,1401499647,AT 1401499648,1401501695,RU 1401501696,1401501791,IT @@ -30020,9 +33136,13 @@ 1401686912,1401686927,DE 1401686928,1401691519,SE 1401691520,1401691535,DE -1401691536,1401697815,SE +1401691536,1401695263,SE +1401695264,1401695267,GB +1401695268,1401697815,SE 1401697816,1401697823,DE -1401697824,1401708543,SE +1401697824,1401698223,SE +1401698224,1401698227,GB +1401698228,1401708543,SE 1401708544,1401709055,FR 1401709056,1401709311,SE 1401709312,1401709567,GB @@ -30032,7 +33152,8 @@ 1401717760,1401718015,NL 1401718016,1401719935,SE 1401719936,1401719951,DK -1401719952,1401727743,SE +1401719952,1401719955,GB +1401719956,1401727743,SE 1401727744,1401727999,GB 1401728000,1401728335,SE 1401728336,1401728351,NL @@ -30054,7 +33175,9 @@ 1401743312,1401743327,GB 1401743328,1401743343,SE 1401743344,1401743359,GB -1401743360,1401745487,SE +1401743360,1401745439,SE +1401745440,1401745443,GB +1401745444,1401745487,SE 1401745488,1401745503,ES 1401745504,1401745919,SE 1401745920,1401745935,IT @@ -30063,14 +33186,16 @@ 1401745968,1401745983,SE 1401745984,1401745999,NL 1401746000,1401746015,ES -1401746016,1401746175,SE +1401746016,1401746019,GB +1401746020,1401746175,SE 1401746176,1401746191,DK 1401746192,1401746215,SE 1401746216,1401746223,GB 1401746224,1401746239,SE 1401746240,1401746255,NL 1401746256,1401746271,ES -1401746272,1401746431,SE +1401746272,1401746275,GB +1401746276,1401746431,SE 1401746432,1401746447,NL 1401746448,1401746467,SE 1401746468,1401746471,NL @@ -30080,7 +33205,8 @@ 1401746512,1401746527,ES 1401746528,1401746623,SE 1401746624,1401746639,DE -1401746640,1401746687,SE +1401746640,1401746655,SE +1401746656,1401746687,IE 1401746688,1401746703,NO 1401746704,1401746751,SE 1401746752,1401746767,NL @@ -30102,7 +33228,9 @@ 1401747392,1401747407,DE 1401747408,1401747487,SE 1401747488,1401747495,DK -1401747496,1401747647,SE +1401747496,1401747499,SE +1401747500,1401747503,GB +1401747504,1401747647,SE 1401747648,1401747663,DE 1401747664,1401747711,SE 1401747712,1401747967,FR @@ -30356,25 +33484,26 @@ 1404026880,1404043263,ES 1404043264,1404076031,SE 1404076032,1404084223,DE -1404084224,1404088319,BE -1404088320,1404184063,SE +1404084224,1404184063,SE 1404184064,1404184575,NO 1404184576,1404186623,SE 1404186624,1404187647,NO 1404187648,1404188671,SE 1404188672,1404189183,LT 1404189184,1404189695,EE -1404189696,1404192767,SE +1404189696,1404190719,NO +1404190720,1404192767,SE 1404192768,1404194815,LV 1404194816,1404195839,LT -1404195840,1404196351,SE +1404195840,1404196351,NO 1404196352,1404197375,LT 1404197376,1404197887,SE 1404197888,1404198911,LT 1404198912,1404200959,SE 1404200960,1404203007,NL 1404203008,1404204031,HR -1404204032,1404210175,SE +1404204032,1404207103,SE +1404207104,1404210175,NO 1404210176,1404212223,LV 1404212224,1404215295,SE 1404215296,1404219391,LV @@ -30386,7 +33515,8 @@ 1404225536,1404227071,RU 1404227072,1404227583,HR 1404227584,1404231679,LV -1404231680,1404232703,SE +1404231680,1404232191,SE +1404232192,1404232703,NO 1404232704,1404233215,CH 1404233216,1404234239,SE 1404234240,1404235775,HR @@ -30398,12 +33528,9 @@ 1404387328,1404395519,DE 1404395520,1404403711,SE 1404403712,1404411903,NL -1404411904,1404436479,SE -1404436480,1404502015,BE -1404502016,1404645375,SE +1404411904,1404645375,SE 1404645376,1404645887,HR -1404645888,1404698623,SE -1404698624,1404764159,BE +1404645888,1404764159,SE 1404764160,1404768255,NL 1404768256,1404802047,SE 1404802048,1404802559,EE @@ -30427,8 +33554,7 @@ 1405063168,1405066239,NO 1405066240,1405067263,SE 1405067264,1405075455,DE -1405075456,1405083647,FR -1405083648,1405091839,SE +1405075456,1405091839,SE 1405091840,1406140415,FR 1406140416,1406205951,CZ 1406205952,1406271487,SE @@ -30458,33 +33584,38 @@ 1406716928,1406717439,AT 1406717440,1406717695,NL 1406717696,1406717951,DE -1406717952,1406719999,NL +1406717952,1406719999,GB 1406720000,1406721023,AT -1406721024,1406722047,NL +1406721024,1406722047,GB 1406722048,1406730239,DE 1406730240,1406746623,RU -1406746624,1406747135,NL -1406747136,1406747391,BE +1406746624,1406747391,BE 1406747392,1406747679,NL 1406747680,1406747743,BE -1406747744,1406748287,NL -1406748288,1406748319,BE -1406748320,1406749727,NL -1406749728,1406749775,BE +1406747744,1406747775,NL +1406747776,1406749183,BE +1406749184,1406749439,NL +1406749440,1406749567,BE +1406749568,1406749695,NL +1406749696,1406749775,BE 1406749776,1406749823,NL 1406749824,1406749839,BE 1406749840,1406749919,NL 1406749920,1406749951,BE -1406749952,1406750303,NL -1406750304,1406750319,BE -1406750320,1406751775,NL -1406751776,1406751807,BE +1406749952,1406750207,NL +1406750208,1406750319,BE +1406750320,1406750335,NL +1406750336,1406750975,BE +1406750976,1406751743,NL +1406751744,1406751807,BE 1406751808,1406751903,NL 1406751904,1406751967,BE -1406751968,1406752383,NL +1406751968,1406751999,NL +1406752000,1406752255,BE +1406752256,1406752383,NL 1406752384,1406752447,BE -1406752448,1406752703,NL -1406752704,1406752735,BE +1406752448,1406752511,NL +1406752512,1406752735,BE 1406752736,1406752751,NL 1406752752,1406752767,BE 1406752768,1406754815,NL @@ -30513,12 +33644,11 @@ 1406926848,1406935039,IT 1406935040,1406951423,RU 1406951424,1406959615,PL -1406959616,1406960831,DE -1406960832,1406960895,CN -1406960896,1406964287,DE +1406959616,1406964287,DE 1406964288,1406964319,CH 1406964320,1406964327,BG -1406964328,1406964735,DE +1406964328,1406964351,NL +1406964352,1406964735,DE 1406964736,1406964927,US 1406964928,1406967295,DE 1406967296,1406967327,GB @@ -30590,7 +33720,7 @@ 1407517440,1407517823,A2 1407517824,1407517887,NG 1407517888,1407517919,CD -1407517920,1407517951,ZW +1407517920,1407517951,A2 1407517952,1407518015,SL 1407518016,1407518031,ZA 1407518032,1407518039,A2 @@ -30685,8 +33815,7 @@ 1407520424,1407520439,A2 1407520440,1407520447,CD 1407520448,1407520463,NG -1407520464,1407520511,A2 -1407520512,1407520519,NG +1407520464,1407520519,A2 1407520520,1407520527,CD 1407520528,1407520543,NG 1407520544,1407520551,A2 @@ -30723,8 +33852,7 @@ 1407520904,1407520911,NG 1407520912,1407520919,ZW 1407520920,1407520927,IQ -1407520928,1407520935,A2 -1407520936,1407520943,CD +1407520928,1407520943,A2 1407520944,1407520951,IQ 1407520952,1407520959,ZW 1407520960,1407520967,TZ @@ -30811,7 +33939,9 @@ 1407524632,1407524639,BF 1407524640,1407524655,A2 1407524656,1407524663,NG -1407524664,1407526911,A2 +1407524664,1407524687,A2 +1407524688,1407524703,MZ +1407524704,1407526911,A2 1407526912,1407527935,MZ 1407527936,1407529023,A2 1407529024,1407529087,NG @@ -30820,7 +33950,7 @@ 1407529104,1407529111,NG 1407529112,1407529127,A2 1407529128,1407529135,ZW -1407529136,1407529143,MW +1407529136,1407529143,US 1407529144,1407529151,ZW 1407529152,1407529183,A2 1407529184,1407529191,NG @@ -30932,9 +34062,7 @@ 1407537360,1407537367,LR 1407537368,1407537391,NG 1407537392,1407537399,LR -1407537400,1407537407,A2 -1407537408,1407537415,NG -1407537416,1407537423,A2 +1407537400,1407537423,A2 1407537424,1407537455,NG 1407537456,1407537479,A2 1407537480,1407537487,CM @@ -31190,12 +34318,9 @@ 1407712224,1407712767,GB 1407712768,1407712831,DE 1407712832,1407712887,GB -1407712888,1407713239,DE -1407713240,1407713279,GB +1407712888,1407713279,DE 1407713280,1407778815,NL -1407778816,1407842303,SE -1407842304,1407842367,NO -1407842368,1407844351,SE +1407778816,1407844351,SE 1407844352,1407909887,RU 1407909888,1407975423,GR 1407975424,1408040959,DE @@ -31305,7 +34430,8 @@ 1410013664,1410013695,AE 1410013696,1410013727,KW 1410013728,1410013759,AE -1410013760,1410013823,IR +1410013760,1410013791,IR +1410013792,1410013823,DE 1410013824,1410013887,AF 1410013888,1410013919,DE 1410013920,1410013951,KW @@ -31600,27 +34726,22 @@ 1410745856,1410746111,CH 1410746112,1410746623,AO 1410746624,1410748415,A2 -1410748416,1410748671,DE -1410748672,1410752511,A2 +1410748416,1410752511,DE 1410752512,1410752639,AQ 1410752640,1410752763,DE 1410752764,1410752785,AQ 1410752786,1410752799,DE 1410752800,1410752827,AQ 1410752828,1410753023,DE -1410753024,1410753127,AQ -1410753128,1410753303,DE +1410753024,1410753151,AQ +1410753152,1410753303,DE 1410753304,1410753327,AQ 1410753328,1410753335,DE 1410753336,1410753343,AQ 1410753344,1410753351,DE 1410753352,1410753367,AQ 1410753368,1410753391,DE -1410753392,1410754559,AQ -1410754560,1410754815,DE -1410754816,1410754823,AQ -1410754824,1410754827,DE -1410754828,1410754831,AQ +1410753392,1410754831,AQ 1410754832,1410754855,DE 1410754856,1410754859,AQ 1410754860,1410754867,DE @@ -31703,6 +34824,7 @@ 1411809280,1411813375,LV 1411813376,1411817471,NO 1411817472,1411821567,PL +1411821568,1411825663,RU 1411825664,1411829759,DE 1411829760,1411833855,PL 1411833856,1411837951,NL @@ -31746,7 +34868,9 @@ 1411917568,1411917639,GB 1411917640,1411917663,EU 1411917664,1411917695,GB -1411917696,1411919871,EU +1411917696,1411917759,EU +1411917760,1411917791,GB +1411917792,1411919871,EU 1411919872,1411923967,DE 1411923968,1411940351,BG 1411940352,1411973119,PL @@ -31787,7 +34911,9 @@ 1412003040,1412003055,SI 1412003056,1412003119,BA 1412003120,1412003135,SI -1412003136,1412003215,BA +1412003136,1412003151,BA +1412003152,1412003167,SI +1412003168,1412003215,BA 1412003216,1412003223,SI 1412003224,1412003231,BA 1412003232,1412003247,SI @@ -31806,8 +34932,8 @@ 1412003536,1412003551,BA 1412003552,1412003583,SI 1412003584,1412003839,BA -1412003840,1412003847,SI -1412003848,1412003903,BG +1412003840,1412003855,SI +1412003856,1412003903,BG 1412003904,1412003935,SI 1412003936,1412004607,BG 1412004608,1412005887,SI @@ -31820,7 +34946,9 @@ 1412202496,1412235263,DE 1412235264,1412241111,GB 1412241112,1412241119,NL -1412241120,1412257535,GB +1412241120,1412254719,GB +1412254720,1412254975,NL +1412254976,1412257535,GB 1412257536,1412257599,NL 1412257600,1412258559,GB 1412258560,1412258815,DE @@ -31838,7 +34966,8 @@ 1412628480,1412641791,TR 1412641792,1412642815,DE 1412642816,1412644863,TR -1412644864,1412661247,RU +1412644864,1412653055,UA +1412653056,1412661247,RU 1412661248,1412677631,GB 1412677632,1412685823,RU 1412685824,1412686239,IE @@ -31894,12 +35023,9 @@ 1422399104,1422399231,IT 1422399232,1422399999,DE 1422400000,1422400255,IT -1422400256,1422401023,DE -1422401024,1422401087,RO +1422400256,1422401087,DE 1422401088,1422401151,GB -1422401152,1422401279,DE -1422401280,1422401283,RO -1422401284,1422462207,DE +1422401152,1422462207,DE 1422462208,1422462463,TR 1422462464,1422468671,DE 1422468672,1422468735,IT @@ -31951,13 +35077,9 @@ 1424556032,1424588799,EG 1424588800,1424588839,DE 1424588840,1424588847,GB -1424588848,1424588927,DE -1424588928,1424588935,GB -1424588936,1424588959,DE +1424588848,1424588959,DE 1424588960,1424588963,IT -1424588964,1424589007,DE -1424589008,1424589015,GB -1424589016,1424589311,DE +1424588964,1424589311,DE 1424589312,1424589567,FR 1424589568,1424590719,GB 1424590720,1424590767,SE @@ -31975,9 +35097,7 @@ 1424591616,1424591839,GB 1424591840,1424591871,ZA 1424591872,1424592639,GB -1424592640,1424592847,FR -1424592848,1424592863,GB -1424592864,1424593151,FR +1424592640,1424593151,FR 1424593152,1424593407,NL 1424593408,1424593663,DE 1424593664,1424593727,SK @@ -31987,7 +35107,9 @@ 1424593968,1424593983,GB 1424593984,1424594055,FR 1424594056,1424594063,IT -1424594064,1424594175,FR +1424594064,1424594095,FR +1424594096,1424594111,GB +1424594112,1424594175,FR 1424594176,1424594431,GB 1424594432,1424594495,IE 1424594496,1424595455,GB @@ -31995,8 +35117,12 @@ 1424595528,1424595535,GB 1424595536,1424595575,IT 1424595576,1424595583,ES -1424595584,1424596479,IT -1424596480,1424596991,FR +1424595584,1424595791,IT +1424595792,1424595839,GB +1424595840,1424596479,IT +1424596480,1424596563,FR +1424596564,1424596567,GB +1424596568,1424596991,FR 1424596992,1424597255,CZ 1424597256,1424597263,GB 1424597264,1424597311,CZ @@ -32043,17 +35169,18 @@ 1424603136,1424603391,SK 1424603392,1424603647,DE 1424603648,1424603903,GB -1424603904,1424604047,ES -1424604048,1424604159,GB -1424604160,1424604463,NL -1424604464,1424604479,GB +1424603904,1424604067,ES +1424604068,1424604095,GB +1424604096,1424604159,ES +1424604160,1424604471,NL +1424604472,1424604479,GB 1424604480,1424604543,NL 1424604544,1424604671,GB 1424604672,1424604799,NL 1424604800,1424604927,GB 1424604928,1424605119,NL -1424605120,1424605151,GB -1424605152,1424605183,NL +1424605120,1424605135,GB +1424605136,1424605183,NL 1424605184,1424605439,GB 1424605440,1424605583,BG 1424605584,1424605607,GB @@ -32076,35 +35203,38 @@ 1424607120,1424607167,GB 1424607168,1424607199,SK 1424607200,1424607215,GB -1424607216,1424607231,SK -1424607232,1424607487,GB +1424607216,1424607223,SK +1424607224,1424607487,GB 1424607488,1424607871,DE 1424607872,1424608031,GB 1424608032,1424608083,FR -1424608084,1424608095,GB -1424608096,1424608127,FR +1424608084,1424608087,GB +1424608088,1424608127,FR 1424608128,1424608279,GB 1424608280,1424608383,FR 1424608384,1424608399,ES 1424608400,1424608511,FR 1424608512,1424608567,ES -1424608568,1424608639,GB -1424608640,1424608687,ES +1424608568,1424608575,GB +1424608576,1424608639,ES +1424608640,1424608671,GB +1424608672,1424608687,ES 1424608688,1424608695,GB -1424608696,1424608703,ES -1424608704,1424608719,GB -1424608720,1424609023,ES +1424608696,1424609023,ES 1424609024,1424609259,DE 1424609260,1424609271,GB 1424609272,1424609279,DE 1424609280,1424609395,CH 1424609396,1424609399,GB -1424609400,1424609791,CH +1424609400,1424609535,CH +1424609536,1424609551,GB +1424609552,1424609791,CH 1424609792,1424610303,GB 1424610304,1424610559,TZ 1424610560,1424610815,PL 1424610816,1424611071,FR -1424611072,1424611143,BE +1424611072,1424611135,GB +1424611136,1424611143,BE 1424611144,1424611151,GB 1424611152,1424611327,BE 1424611328,1424611583,PL @@ -32123,9 +35253,7 @@ 1424614136,1424614143,IT 1424614144,1424614399,GB 1424614400,1424614415,FR -1424614416,1424614439,IT -1424614440,1424614447,GB -1424614448,1424614655,IT +1424614416,1424614655,IT 1424614656,1424614911,FR 1424614912,1424615167,RO 1424615168,1424615679,GB @@ -32139,12 +35267,14 @@ 1424616488,1424616607,ES 1424616608,1424616615,PT 1424616616,1424616623,GB -1424616624,1424616687,ES -1424616688,1424616703,GB +1424616624,1424616655,ES +1424616656,1424616659,GB +1424616660,1424616695,ES +1424616696,1424616703,GB 1424616704,1424616959,US 1424616960,1424617215,FR -1424617216,1424617343,IT -1424617344,1424617391,GB +1424617216,1424617351,IT +1424617352,1424617391,GB 1424617392,1424617395,IT 1424617396,1424617399,GB 1424617400,1424617407,IT @@ -32154,7 +35284,9 @@ 1424617440,1424617463,IT 1424617464,1424617471,GB 1424617472,1424617727,US -1424617728,1424618239,IT +1424617728,1424617983,IT +1424617984,1424618015,GB +1424618016,1424618239,IT 1424618240,1424618495,NL 1424618496,1424618751,GB 1424618752,1424618983,FR @@ -32176,7 +35308,6 @@ 1424633856,1424637951,MD 1424637952,1424642047,FI 1424642048,1424646143,DK -1424646144,1424650239,RO 1424650240,1424654335,PL 1424654336,1424687103,NO 1424687104,1424711679,SA @@ -32234,8 +35365,7 @@ 1424751616,1424752127,IE 1424752128,1424752383,DE 1424752384,1424752639,NL -1424752640,1424784383,HU -1424784384,1424785407,RS +1424752640,1424785407,HU 1424785408,1424818175,ES 1424818176,1424850943,RU 1424850944,1424883711,LV @@ -32608,9 +35738,7 @@ 1426859520,1426864511,NL 1426864512,1426864639,PA 1426864640,1426864895,FR -1426864896,1426865151,NL -1426865152,1426865407,BE -1426865408,1426865663,NL +1426864896,1426865663,NL 1426865664,1426865919,BE 1426865920,1426866175,ES 1426866176,1426882559,GB @@ -32627,9 +35755,7 @@ 1426967672,1426967679,ES 1426967680,1426968343,GB 1426968344,1426968351,ES -1426968352,1426968447,GB -1426968448,1426968455,ES -1426968456,1426968591,GB +1426968352,1426968591,GB 1426968592,1426968600,ES 1426968601,1426968791,GB 1426968792,1426968800,ES @@ -32663,10 +35789,7 @@ 1427032736,1427032743,AT 1427032744,1427032959,DE 1427032960,1427032967,NL -1427032968,1427032991,DE -1427032992,1427032999,CH -1427033000,1427033007,GR -1427033008,1427033247,DE +1427032968,1427033247,DE 1427033248,1427033263,MZ 1427033264,1427033287,DE 1427033288,1427033295,SE @@ -32711,11 +35834,14 @@ 1427728800,1427728831,RU 1427728832,1427728863,DE 1427728864,1427728895,CY -1427728896,1427728959,DE +1427728896,1427728927,DE +1427728928,1427728959,IL 1427728960,1427728991,RU 1427728992,1427729055,DE 1427729056,1427729087,CA -1427729088,1427729311,DE +1427729088,1427729119,DE +1427729120,1427729151,RO +1427729152,1427729311,DE 1427729312,1427729343,GR 1427729344,1427742719,DE 1427742720,1427742751,IO @@ -32727,19 +35853,19 @@ 1427743456,1427743487,CY 1427743488,1427743615,DE 1427743616,1427743647,DK -1427743648,1427743839,DE -1427743840,1427743871,IT -1427743872,1427744031,DE -1427744032,1427744063,BR -1427744064,1427744159,DE +1427743648,1427744159,DE 1427744160,1427744191,RU 1427744192,1427744255,DE 1427744256,1427744287,US -1427744288,1427744351,DE +1427744288,1427744319,TR +1427744320,1427744351,DE 1427744352,1427744383,UA 1427744384,1427744511,DE 1427744512,1427744543,BE -1427744544,1427744735,DE +1427744544,1427744639,DE +1427744640,1427744671,TR +1427744672,1427744703,DE +1427744704,1427744735,CA 1427744736,1427744767,DK 1427744768,1427744863,DE 1427744864,1427744927,TR @@ -32747,13 +35873,17 @@ 1427744992,1427745023,DK 1427745024,1427745151,DE 1427745152,1427745183,RO -1427745184,1427745503,DE +1427745184,1427745247,DE +1427745248,1427745279,TR +1427745280,1427745503,DE 1427745504,1427745535,RU 1427745536,1427745567,CN 1427745568,1427745599,RU -1427745600,1427745663,DE +1427745600,1427745631,DE +1427745632,1427745663,BM 1427745664,1427745695,CY -1427745696,1427745791,DE +1427745696,1427745727,US +1427745728,1427745791,DE 1427745792,1427745823,GB 1427745824,1427746079,DE 1427746080,1427746111,GB @@ -32763,13 +35893,19 @@ 1427746368,1427746399,SE 1427746400,1427747839,DE 1427747840,1427747871,FI -1427747872,1427749503,DE +1427747872,1427748543,DE +1427748544,1427748575,MX +1427748576,1427748831,DE +1427748832,1427748863,US +1427748864,1427749503,DE 1427749504,1427749535,UA 1427749536,1427749567,DE 1427749568,1427749599,CY 1427749600,1427749919,DE 1427749920,1427749983,RU -1427749984,1427750239,DE +1427749984,1427750143,DE +1427750144,1427750175,TR +1427750176,1427750239,DE 1427750240,1427750271,TR 1427750272,1427750303,DE 1427750304,1427750335,GB @@ -32779,18 +35915,24 @@ 1427759936,1427759967,HR 1427759968,1427760191,DE 1427760192,1427760223,TR -1427760224,1427760255,DE +1427760224,1427760255,US 1427760256,1427760319,BR -1427760320,1427760351,DE -1427760352,1427760383,DK +1427760320,1427760383,DE 1427760384,1427760415,TR 1427760416,1427760799,DE 1427760800,1427760831,NL 1427760832,1427760959,DE 1427760960,1427760991,CZ 1427760992,1427761023,DE -1427761024,1427761055,CR -1427761056,1427761535,DE +1427761024,1427761055,HR +1427761056,1427761087,DE +1427761088,1427761119,CH +1427761120,1427761247,DE +1427761248,1427761279,BR +1427761280,1427761375,DE +1427761376,1427761407,TW +1427761408,1427761503,DE +1427761504,1427761535,PL 1427761536,1427761567,US 1427761568,1427761599,RU 1427761600,1427767295,DE @@ -32809,7 +35951,6 @@ 1427980288,1427996671,FI 1427996672,1428013055,KZ 1428013056,1428029439,EE -1428029440,1428045823,GR 1428045824,1428062207,MK 1428062208,1428078591,SE 1428078592,1428094975,RU @@ -32911,7 +36052,9 @@ 1431953408,1431961599,DK 1431961600,1431969791,CH 1431969792,1431977983,GB -1431977984,1431980151,NL +1431977984,1431979479,NL +1431979480,1431979487,GB +1431979488,1431980151,NL 1431980152,1431980159,US 1431980160,1431986175,NL 1431986176,1431994367,RU @@ -32937,9 +36080,7 @@ 1432131584,1432133631,PH 1432133632,1432150015,GB 1432150016,1432158207,BA -1432158208,1432159042,DE -1432159043,1432159046,AE -1432159047,1432159743,DE +1432158208,1432159743,DE 1432159744,1432159871,PL 1432159872,1432160255,DE 1432160256,1432160511,CH @@ -32967,14 +36108,12 @@ 1432322048,1432338431,RU 1432338432,1432346623,FR 1432346624,1433403391,TR -1433403392,1433407711,ES -1433407712,1433408767,NL -1433408768,1433409023,ES -1433409024,1433409279,NL -1433409280,1433409535,ES -1433409536,1433411071,NL -1433411072,1433411072,ES -1433411073,1433411583,TR +1433403392,1433407487,ES +1433407488,1433410559,NL +1433410560,1433411071,ES +1433411072,1433411391,TR +1433411392,1433411407,NL +1433411408,1433411583,TR 1433411584,1433419775,RU 1433419776,1433427967,LB 1433427968,1433436159,RU @@ -33051,7 +36190,9 @@ 1433843776,1433843839,SE 1433843840,1433843967,NL 1433843968,1433843983,RS -1433843984,1433845759,NL +1433843984,1433845119,NL +1433845120,1433845247,EU +1433845248,1433845759,NL 1433845760,1433847807,BG 1433847808,1433849855,SE 1433849856,1433851903,BE @@ -33116,17 +36257,14 @@ 1434583040,1434615807,BG 1434615808,1434648575,IL 1434648576,1434681343,FI -1434681344,1434681599,DE -1434681600,1434681631,NL -1434681632,1434681743,DE -1434681744,1434681791,NL -1434681792,1434681855,DE +1434681344,1434681855,DE 1434681856,1434682111,MY -1434682112,1434682367,NL -1434682368,1434682623,DE -1434682624,1434683391,NL -1434683392,1434683903,DE -1434683904,1434684671,NL +1434682112,1434682303,DE +1434682304,1434682367,NL +1434682368,1434683007,DE +1434683008,1434683391,NL +1434683392,1434684415,DE +1434684416,1434684671,NL 1434684672,1434685439,DE 1434685440,1434687231,NL 1434687232,1434687359,DE @@ -33143,7 +36281,9 @@ 1434705696,1434705919,NL 1434705920,1434710591,DE 1434710592,1434710783,NL -1434710784,1434712959,DE +1434710784,1434712063,DE +1434712064,1434712319,NL +1434712320,1434712959,DE 1434712960,1434712975,NL 1434712976,1434713855,DE 1434713856,1434714111,NL @@ -33488,7 +36628,11 @@ 1441385104,1441385119,ES 1441385120,1441386111,FR 1441386112,1441386239,IL -1441386240,1441390591,FR +1441386240,1441388671,FR +1441388672,1441388799,CA +1441388800,1441389055,FR +1441389056,1441389119,IE +1441389120,1441390591,FR 1441390592,1441398783,DK 1441398784,1441415167,RU 1441415168,1441423359,GB @@ -33643,9 +36787,25 @@ 1445436960,1445437182,BE 1445437183,1445437311,NL 1445437312,1445437319,BE -1445437320,1445447679,NL -1445447680,1445447935,BE -1445447936,1445462015,NL +1445437320,1445437439,NL +1445437440,1445437695,BE +1445437696,1445437711,NL +1445437712,1445437743,BE +1445437744,1445437823,NL +1445437824,1445438271,BE +1445438272,1445439999,NL +1445440000,1445440255,BE +1445440256,1445442559,NL +1445442560,1445443327,BE +1445443328,1445445119,NL +1445445120,1445445855,BE +1445445856,1445447759,NL +1445447760,1445451519,BE +1445451520,1445451647,NL +1445451648,1445456126,BE +1445456127,1445456127,NL +1445456128,1445458175,BE +1445458176,1445462015,NL 1445462016,1445986303,IE 1445986304,1446051839,DK 1446051840,1446117375,CZ @@ -33675,8 +36835,7 @@ 1446862080,1446862591,HR 1446862592,1446871039,SI 1446871040,1446903807,CZ -1446903808,1446904063,AE -1446904064,1446904071,A2 +1446903808,1446904071,A2 1446904072,1446904079,CY 1446904080,1446904095,BH 1446904096,1446904111,SA @@ -33850,7 +37009,8 @@ 1467367616,1467367647,SE 1467367648,1467367679,DE 1467367680,1467367695,IT -1467367696,1467367871,DE +1467367696,1467367807,DE +1467367808,1467367871,CZ 1467367872,1467367903,SE 1467367904,1467367935,DE 1467367936,1467368191,SI @@ -33898,9 +37058,7 @@ 1467467904,1467468031,CH 1467468032,1467468111,DE 1467468112,1467468119,BR -1467468120,1467468479,DE -1467468480,1467468543,CN -1467468544,1467473919,DE +1467468120,1467473919,DE 1467473920,1467482111,RU 1467482112,1467613183,BG 1467613184,1467744255,DE @@ -33998,16 +37156,18 @@ 1475175056,1475176447,NL 1475176448,1475178495,RO 1475178496,1475180543,RU -1475180544,1475184639,DE +1475180544,1475181455,DE +1475181456,1475181471,SC +1475181472,1475181519,DE +1475181520,1475181535,GB +1475181536,1475184639,DE 1475184640,1475186687,RU 1475186688,1475188735,ES 1475188736,1475190783,SE 1475190784,1475192831,GB 1475192832,1475194879,CH 1475194880,1475196927,DE -1475196928,1475197071,FR -1475197072,1475197079,DZ -1475197080,1475197791,FR +1475196928,1475197791,FR 1475197792,1475197807,ES 1475197808,1475198975,FR 1475198976,1475201023,BH @@ -34066,7 +37226,7 @@ 1475244320,1475244543,CV 1475244544,1475245055,DE 1475245056,1475245183,IE -1475245184,1475245311,DE +1475245184,1475245311,SD 1475245312,1475245567,CV 1475245568,1475246078,SA 1475246079,1475246079,DE @@ -34127,7 +37287,7 @@ 1475319808,1475321855,BG 1475321856,1475323903,RO 1475323904,1475327999,PL -1475328000,1475330047,US +1475328000,1475330047,NL 1475330048,1475332095,PL 1475332096,1475334143,LI 1475334144,1475336191,CH @@ -34144,15 +37304,14 @@ 1475412472,1475412479,ES 1475412480,1475417975,IT 1475417976,1475417983,A2 -1475417984,1475427207,IT -1475427208,1475427215,FR -1475427216,1475428351,IT +1475417984,1475428351,IT 1475428352,1475444735,SE 1475444736,1475461119,AM 1475461120,1475477503,RU 1475477504,1475493887,HU 1475493888,1475510271,RU 1475510272,1475543039,GB +1475543040,1475559423,RO 1475559424,1475575807,GR 1475575808,1475592191,AT 1475592192,1475608575,GB @@ -34190,9 +37349,7 @@ 1475639217,1475639223,JE 1475639224,1475639263,GB 1475639264,1475639271,BB -1475639272,1475639279,JE -1475639280,1475639287,GB -1475639288,1475639295,JE +1475639272,1475639295,JE 1475639296,1475639343,GB 1475639344,1475639351,JE 1475639352,1475639367,GB @@ -34497,7 +37654,9 @@ 1486323200,1486323215,GU 1486323216,1486323231,NL 1486323232,1486323239,GB -1486323240,1486323711,EU +1486323240,1486323447,EU +1486323448,1486323455,GB +1486323456,1486323711,EU 1486323712,1486325759,GB 1486325760,1486327807,SA 1486327808,1486329855,CH @@ -34579,8 +37738,8 @@ 1489993728,1490026495,LU 1490026496,1490028543,US 1490028544,1490029055,UA -1490029056,1490040831,NL -1490040832,1490041855,UA +1490029056,1490040839,NL +1490040840,1490041855,UA 1490041856,1490049879,CZ 1490049880,1490049887,AT 1490049888,1490049919,CZ @@ -34725,7 +37884,8 @@ 1494581248,1494589439,RU 1494589440,1494589695,NL 1494589696,1494589723,IQ -1494589724,1494595071,NL +1494589724,1494594559,NL +1494594560,1494595071,US 1494595072,1494595327,A2 1494595328,1494595583,NL 1494595584,1494596607,US @@ -34778,7 +37938,9 @@ 1495150592,1495151103,GB 1495151104,1495151359,NL 1495151360,1495154687,FR -1495154688,1495162367,EU +1495154688,1495161599,EU +1495161600,1495161855,FR +1495161856,1495162367,EU 1495162368,1495162879,US 1495162880,1495163903,FR 1495163904,1495164415,EU @@ -34797,7 +37959,7 @@ 1495169024,1495169279,FR 1495169280,1495169535,EU 1495169536,1495169791,FR -1495169792,1495170047,EU +1495169792,1495170047,NL 1495170048,1495170079,FR 1495170080,1495170159,EU 1495170160,1495170175,FR @@ -34813,6 +37975,7 @@ 1495205888,1495207935,CZ 1495207936,1495209983,RU 1495209984,1495212031,KZ +1495212032,1495214079,RU 1495214080,1495216127,CZ 1495216128,1495218175,GB 1495218176,1495220223,IT @@ -34829,7 +37992,9 @@ 1495237376,1495237711,AT 1495237712,1495237727,NL 1495237728,1495237731,AT -1495237732,1495237887,EU +1495237732,1495237759,EU +1495237760,1495237775,AT +1495237776,1495237887,EU 1495237888,1495238207,AT 1495238208,1495238223,EU 1495238224,1495238231,AT @@ -34906,7 +38071,11 @@ 1500104064,1500104071,DE 1500104072,1500106527,NL 1500106528,1500106559,DE -1500106560,1500110847,NL +1500106560,1500107415,NL +1500107416,1500107423,DE +1500107424,1500107775,NL +1500107776,1500107903,DE +1500107904,1500110847,NL 1500110848,1500119039,UA 1500119040,1500127231,TR 1500127232,1500135423,FI @@ -35027,8 +38196,7 @@ 1502478336,1502605311,SI 1502605312,1502606335,HR 1502606336,1502609407,SI -1502609408,1502610175,AT -1502610176,1502625791,DE +1502609408,1502625791,DE 1502625792,1502642175,SA 1502642176,1502658559,IR 1502658560,1502674943,AT @@ -35047,7 +38215,9 @@ 1502793504,1502793511,NL 1502793512,1502794239,DE 1502794240,1502794495,NL -1502794496,1502806015,DE +1502794496,1502795767,DE +1502795768,1502795771,NL +1502795772,1502806015,DE 1502806016,1502822399,SA 1502822400,1502838783,HU 1502838784,1502855167,SE @@ -35137,25 +38307,19 @@ 1503789056,1503821823,NO 1503821824,1503854591,UA 1503854592,1503887359,RU -1503887360,1503895615,DE -1503895616,1503895623,NL +1503887360,1503895607,DE +1503895608,1503895623,NL 1503895624,1503895631,DE 1503895632,1503895639,AT 1503895640,1503895671,DE 1503895672,1503895679,PL -1503895680,1503895751,DE -1503895752,1503895759,US -1503895760,1503895767,DE +1503895680,1503895767,DE 1503895768,1503895783,IT 1503895784,1503895799,DE 1503895800,1503895807,CA 1503895808,1503896175,DE 1503896176,1503896183,NL -1503896184,1503896303,DE -1503896304,1503896311,GR -1503896312,1503896319,DE -1503896320,1503896327,GR -1503896328,1503896351,DE +1503896184,1503896351,DE 1503896352,1503896359,AT 1503896360,1503896367,CH 1503896368,1503896375,DE @@ -35166,7 +38330,8 @@ 1503896440,1503896447,NO 1503896448,1503896519,DE 1503896520,1503896527,AT -1503896528,1503896551,DE +1503896528,1503896543,DE +1503896544,1503896551,FR 1503896552,1503896559,AT 1503896560,1503897303,DE 1503897304,1503897311,BE @@ -35178,11 +38343,17 @@ 1503897400,1503897407,DE 1503897408,1503897415,IT 1503897416,1503897423,CH -1503897424,1503897463,DE +1503897424,1503897431,DE +1503897432,1503897439,IT +1503897440,1503897463,DE 1503897464,1503897471,GR -1503897472,1503897479,DE +1503897472,1503897479,AT 1503897480,1503897487,SI -1503897488,1503908351,DE +1503897488,1503897575,DE +1503897576,1503897583,GB +1503897584,1503898119,DE +1503898120,1503898135,TH +1503898136,1503908351,DE 1503908352,1503909375,IT 1503909376,1503920127,DE 1503920128,1503985663,HR @@ -35240,8 +38411,8 @@ 1505273088,1505273095,NZ 1505273096,1505279999,NL 1505280000,1505280007,IR -1505280008,1505285119,AE -1505285120,1505288191,IR +1505280008,1505284863,AE +1505284864,1505288191,IR 1505288192,1505296383,RU 1505296384,1505304575,UA 1505304576,1505305198,FR @@ -35274,7 +38445,17 @@ 1505306352,1505306367,IT 1505306368,1505312767,FR 1505312768,1505320959,RU -1505320960,1505329151,AT +1505320960,1505321151,AT +1505321152,1505321183,NL +1505321184,1505321343,AT +1505321344,1505321471,NL +1505321472,1505321631,AT +1505321632,1505321983,NL +1505321984,1505322287,AT +1505322288,1505322415,NL +1505322416,1505322879,AT +1505322880,1505325055,NL +1505325056,1505329151,AT 1505329152,1505329215,GB 1505329216,1505329375,IE 1505329376,1505329407,GB @@ -35402,15 +38583,17 @@ 1506428240,1506437631,DE 1506437632,1506437887,MU 1506437888,1506437903,CA -1506437904,1506437919,DE -1506437920,1506437951,IN +1506437904,1506437951,US 1506437952,1506437983,DE 1506437984,1506437991,US 1506437992,1506437999,DE 1506438000,1506438015,CA -1506438016,1506438143,DE +1506438016,1506438019,US +1506438020,1506438143,DE 1506438144,1506438271,HK -1506438272,1506438399,DE +1506438272,1506438367,GB +1506438368,1506438384,IL +1506438385,1506438399,DE 1506438400,1506438527,US 1506438528,1506438783,DE 1506438784,1506438799,KR @@ -35420,7 +38603,9 @@ 1506438872,1506438879,FR 1506438880,1506438911,DE 1506438912,1506439039,US -1506439040,1506439935,DE +1506439040,1506439463,DE +1506439464,1506439471,GB +1506439472,1506439935,DE 1506439936,1506440191,US 1506440192,1506440447,DE 1506440448,1506440575,US @@ -35491,11 +38676,14 @@ 1506445338,1506445343,GB 1506445344,1506445519,FR 1506445520,1506445527,GB -1506445528,1506445695,FR -1506445696,1506445703,GB +1506445528,1506445703,FR 1506445704,1506445711,NL 1506445712,1506445719,GB -1506445720,1506445807,FR +1506445720,1506445759,FR +1506445760,1506445767,GB +1506445768,1506445775,FR +1506445776,1506445791,GB +1506445792,1506445807,FR 1506445808,1506445823,GB 1506445824,1506446159,FR 1506446160,1506446175,GB @@ -35506,8 +38694,8 @@ 1506448640,1506448647,GB 1506448648,1506448663,AT 1506448664,1506448671,GB -1506448672,1506448719,AT -1506448720,1506448727,GB +1506448672,1506448703,AT +1506448704,1506448727,GB 1506448728,1506448735,AT 1506448736,1506448895,GB 1506448896,1506449167,BE @@ -35525,9 +38713,7 @@ 1506451008,1506451199,GB 1506451200,1506451895,ES 1506451896,1506451903,GB -1506451904,1506452079,ES -1506452080,1506452087,GB -1506452088,1506452223,ES +1506451904,1506452223,ES 1506452224,1506452479,GB 1506452480,1506452735,US 1506452736,1506452743,RO @@ -35538,7 +38724,9 @@ 1506453400,1506453415,SE 1506453416,1506453423,GB 1506453424,1506453439,SE -1506453440,1506453503,GB +1506453440,1506453455,GB +1506453456,1506453471,SE +1506453472,1506453503,GB 1506453504,1506453759,DE 1506453760,1506454015,FR 1506454016,1506454271,AT @@ -35557,20 +38745,18 @@ 1506457088,1506458239,GB 1506458240,1506458623,CH 1506458624,1506459135,GB -1506459136,1506459247,BE -1506459248,1506459263,GB -1506459264,1506459279,BE -1506459280,1506459327,GB -1506459328,1506459647,BE +1506459136,1506459647,BE 1506459648,1506460047,FR 1506460048,1506460055,GB -1506460056,1506460111,FR -1506460112,1506460127,GB -1506460128,1506460255,FR -1506460256,1506460263,GB -1506460264,1506460311,FR -1506460312,1506460319,GB -1506460320,1506460335,FR +1506460056,1506460063,FR +1506460064,1506460079,GB +1506460080,1506460087,FR +1506460088,1506460095,GB +1506460096,1506460126,FR +1506460127,1506460127,GB +1506460128,1506460287,FR +1506460288,1506460295,GB +1506460296,1506460335,FR 1506460336,1506460343,GB 1506460344,1506460671,FR 1506460672,1506460927,AT @@ -35594,9 +38780,7 @@ 1506464000,1506464767,GB 1506464768,1506465111,NL 1506465112,1506465119,GB -1506465120,1506465135,NL -1506465136,1506465151,GB -1506465152,1506465279,NL +1506465120,1506465279,NL 1506465280,1506465791,GB 1506465792,1506466047,DE 1506466048,1506466303,BE @@ -35626,7 +38810,8 @@ 1506474304,1506474495,IT 1506474496,1506474751,FR 1506474752,1506475519,IT -1506475520,1506476031,GB +1506475520,1506475527,AT +1506475528,1506476031,GB 1506476032,1506508799,KW 1506508800,1506541567,CZ 1506541568,1506574335,RU @@ -35706,7 +38891,7 @@ 1507666560,1507666591,FR 1507666592,1507666639,GR 1507666640,1507666655,IL -1507666656,1507666687,IT +1507666656,1507666687,SG 1507666688,1507666943,GB 1507666944,1507667455,IT 1507667456,1507667711,DE @@ -35765,13 +38950,13 @@ 1508642816,1508646911,SE 1508646912,1508646927,DK 1508646928,1508646935,SE -1508646936,1508646999,DK -1508647000,1508648447,SE +1508646936,1508647015,DK +1508647016,1508648447,SE 1508648448,1508648703,DK 1508648704,1508650751,SE 1508650752,1508650815,DK -1508650816,1508651007,SE -1508651008,1508651263,DK +1508650816,1508650879,SE +1508650880,1508651263,DK 1508651264,1508652543,SE 1508652544,1508654079,DK 1508654080,1508655103,SE @@ -35810,6 +38995,7 @@ 1509445632,1509449727,LV 1509449728,1509453823,ES 1509453824,1509457919,RU +1509457920,1509462015,NL 1509462016,1509465599,LI 1509465600,1509466111,CH 1509466112,1509466879,NL @@ -35834,7 +39020,7 @@ 1509478400,1509482495,FR 1509482496,1509486591,RU 1509486592,1509489407,CZ -1509489408,1509489535,US +1509489408,1509489535,DE 1509489536,1509490687,CZ 1509490688,1509494783,DK 1509494784,1509498879,RU @@ -35889,8 +39075,7 @@ 1509666816,1509670911,IT 1509670912,1509675007,GB 1509675008,1509677823,NO -1509677824,1509678591,GB -1509678592,1509679103,NO +1509677824,1509679103,GB 1509679104,1509683199,ES 1509683200,1509687295,CH 1509687296,1509703679,UA @@ -35920,9 +39105,7 @@ 1515520000,1518338047,FR 1518338048,1518370815,DE 1518370816,1518403583,NL -1518403584,1518436351,SE -1518436352,1518437375,BE -1518437376,1518452735,SE +1518403584,1518452735,SE 1518452736,1518460927,AT 1518460928,1518501887,SE 1518501888,1518503935,EE @@ -36265,7 +39448,8 @@ 1534714000,1534714015,FR 1534714016,1534714031,DE 1534714032,1534714047,BE -1534714048,1534714055,FR +1534714048,1534714051,FR +1534714052,1534714055,IT 1534714056,1534714059,DE 1534714060,1534714063,FR 1534714064,1534714079,ES @@ -36284,7 +39468,10 @@ 1534714308,1534714311,ES 1534714312,1534714315,FR 1534714316,1534714319,GB -1534714320,1534714351,FR +1534714320,1534714327,DE +1534714328,1534714331,FR +1534714332,1534714335,PL +1534714336,1534714351,FR 1534714352,1534714367,CH 1534714368,1534714383,GB 1534714384,1534714399,FR @@ -36294,8 +39481,10 @@ 1534714464,1534714495,GB 1534714496,1534714511,FR 1534714512,1534714527,PL -1534714528,1534714543,GB -1534714544,1534714547,FR +1534714528,1534714531,FR +1534714532,1534714535,PL +1534714536,1534714539,GB +1534714540,1534714547,FR 1534714548,1534714551,ES 1534714552,1534714559,GB 1534714560,1534714575,NL @@ -36369,8 +39558,7 @@ 1534716016,1534716047,FR 1534716048,1534716063,GB 1534716064,1534716127,FR -1534716128,1534716131,NL -1534716132,1534716143,ES +1534716128,1534716143,ES 1534716144,1534716159,PL 1534716160,1534716163,IT 1534716164,1534716167,FR @@ -36388,13 +39576,12 @@ 1534716396,1534716399,GB 1534716400,1534716431,FR 1534716432,1534716447,PL -1534716448,1534716463,FR -1534716464,1534716479,GB +1534716448,1534716479,FR 1534716480,1534716495,ES 1534716496,1534716511,FR 1534716512,1534716527,DE -1534716528,1534716591,FR -1534716592,1534716607,PL +1534716528,1534716575,FR +1534716576,1534716607,PL 1534716608,1534716639,FR 1534716640,1534716647,BE 1534716648,1534716655,FR @@ -36435,7 +39622,7 @@ 1534717376,1534717503,FR 1534717504,1534717519,DE 1534717520,1534717535,FR -1534717536,1534717551,DE +1534717536,1534717551,PL 1534717552,1534717567,CZ 1534717568,1534717583,PL 1534717584,1534717647,FR @@ -36472,7 +39659,9 @@ 1534718096,1534718111,PL 1534718112,1534718127,IT 1534718128,1534718143,ES -1534718144,1534718207,FR +1534718144,1534718159,FR +1534718160,1534718175,DE +1534718176,1534718207,PL 1534718208,1534718271,ES 1534718272,1534718399,FR 1534718400,1534718415,DE @@ -36499,7 +39688,7 @@ 1534718848,1534718911,BE 1534718912,1534718927,IT 1534718928,1534718943,FR -1534718944,1534718959,PL +1534718944,1534718959,DE 1534718960,1534718975,ES 1534718976,1534719039,FR 1534719040,1534719167,BE @@ -36534,8 +39723,8 @@ 1534719888,1534719951,FR 1534719952,1534719967,DE 1534719968,1534719983,PL -1534719984,1534719999,FR -1534720000,1534720015,PL +1534719984,1534720007,FR +1534720008,1534720015,PL 1534720016,1534720031,GB 1534720032,1534720047,DE 1534720048,1534720063,ES @@ -36555,7 +39744,8 @@ 1534720436,1534720439,FR 1534720440,1534720443,GB 1534720444,1534720447,NL -1534720448,1534720463,GB +1534720448,1534720455,FR +1534720456,1534720463,IE 1534720464,1534720467,FR 1534720468,1534720471,GB 1534720472,1534720479,DE @@ -36570,12 +39760,15 @@ 1534720608,1534720623,GB 1534720624,1534720655,FR 1534720656,1534720671,PL -1534720672,1534720743,FR +1534720672,1534720739,FR +1534720740,1534720743,PL 1534720744,1534720747,NL 1534720748,1534720751,FR 1534720752,1534720767,GB 1534720768,1534720783,ES -1534720784,1534720799,BE +1534720784,1534720791,PL +1534720792,1534720795,IT +1534720796,1534720799,ES 1534720800,1534720815,PL 1534720816,1534720831,FR 1534720832,1534720863,PL @@ -36618,10 +39811,9 @@ 1534721344,1534721359,PL 1534721360,1534721375,FR 1534721376,1534721391,ES -1534721392,1534721407,GB +1534721392,1534721407,PL 1534721408,1534721439,DE -1534721440,1534721455,BE -1534721456,1534721487,FR +1534721440,1534721487,FR 1534721488,1534721491,GB 1534721492,1534721495,FR 1534721496,1534721519,PL @@ -36666,7 +39858,7 @@ 1534722032,1534722047,BE 1534722048,1534787583,RU 1534787584,1534791679,RO -1534791680,1534795775,RU +1534791680,1534795775,UA 1534795776,1534803967,NO 1534803968,1534808063,LV 1534808064,1534812159,RU @@ -36755,7 +39947,6 @@ 1536048704,1536048719,IQ 1536048720,1536048735,NL 1536048736,1536049151,A2 -1536051200,1536053247,NL 1536053248,1536057343,RU 1536057344,1536061439,IE 1536061440,1536065535,SE @@ -36893,7 +40084,8 @@ 1538859008,1538875391,RU 1538875392,1538883583,RS 1538883584,1538891775,BE -1538891776,1538899967,DE +1538891776,1538898431,DE +1538898432,1538899967,FR 1538899968,1538904031,SI 1538904032,1538904039,BH 1538904040,1538908159,SI @@ -36965,7 +40157,11 @@ 1539050408,1539050415,AL 1539050416,1539055471,IT 1539055472,1539055487,FR -1539055488,1539055615,IT +1539055488,1539055511,IT +1539055512,1539055519,DE +1539055520,1539055567,IT +1539055568,1539055575,FR +1539055576,1539055615,IT 1539055616,1539063807,LV 1539063808,1539071999,FR 1539072000,1539080191,PL @@ -37014,7 +40210,6 @@ 1539212544,1539213311,CZ 1539213312,1539215359,SE 1539215360,1539219455,DE -1539219456,1539221503,DK 1539221504,1539222287,FR 1539222288,1539222303,HK 1539222304,1539222527,FR @@ -37022,7 +40217,6 @@ 1539222784,1539223551,FR 1539223552,1539225599,DE 1539225600,1539227647,HU -1539227648,1539229695,KW 1539229696,1539231743,DE 1539231744,1539233791,BE 1539233792,1539235839,GR @@ -37128,7 +40322,7 @@ 1539402752,1539403263,GB 1539403264,1539403775,RU 1539403776,1539404799,GR -1539404800,1539405823,LV +1539404800,1539405823,EE 1539405824,1539406847,PL 1539406848,1539408895,UA 1539408896,1539409919,CZ @@ -37477,7 +40671,7 @@ 1539680256,1539681279,UA 1539681280,1539684351,RU 1539684352,1539685375,UA -1539685376,1539688447,RU +1539686400,1539688447,RU 1539688448,1539689471,SA 1539689472,1539690495,RU 1539690496,1539691519,FI @@ -37495,6 +40689,7 @@ 1539703040,1539703295,UA 1539703296,1539703551,DE 1539703552,1539703807,GB +1539703808,1539704063,UA 1539704064,1539704319,IL 1539704320,1539704575,CH 1539704576,1539704831,HU @@ -37709,6 +40904,7 @@ 1539761920,1539762175,RU 1539762176,1539762431,PL 1539762432,1539762687,UA +1539762688,1539762943,NL 1539762944,1539763199,UA 1539763200,1539763455,MD 1539763456,1539763711,DK @@ -37936,13 +41132,13 @@ 1539824128,1539824383,CH 1539824384,1539824639,NL 1539824640,1539824895,HU +1539824896,1539825151,RU 1539825152,1539825407,UA 1539825408,1539825919,RU 1539825920,1539826175,DE 1539826176,1539826431,AM 1539826432,1539826687,BE 1539826688,1539826943,CH -1539826944,1539827199,DE 1539827200,1539827455,RU 1539827456,1539827711,MK 1539827712,1539827967,RU @@ -37990,7 +41186,6 @@ 1539860480,1539861503,UA 1539861504,1539862527,DE 1539862528,1539863551,UA -1539863552,1539864575,MD 1539864576,1539865599,RO 1539865600,1539866623,UA 1539866624,1539867647,IT @@ -38025,7 +41220,9 @@ 1539902464,1539903487,LV 1539903488,1539904511,PL 1539904512,1539905535,CZ -1539905536,1539910655,UA +1539905536,1539907583,UA +1539907584,1539908607,EU +1539908608,1539910655,UA 1539910656,1539911679,PL 1539911680,1539913727,RU 1539913728,1539914751,CH @@ -38139,12 +41336,15 @@ 1540047872,1540048895,LV 1540048896,1540049919,GB 1540049920,1540050943,UA -1540050944,1540052991,RU +1540050944,1540051967,RU 1540052992,1540054052,UA 1540054053,1540054062,MD 1540054063,1540055039,UA -1540055040,1540056063,A1 +1540055040,1540056063,NO 1540056064,1540057087,NL +1540057088,1540057343,PL +1540057600,1540057855,US +1540057856,1540058111,UA 1540058112,1540059135,DE 1540059136,1540060159,UA 1540060160,1540061183,NO @@ -38155,6 +41355,7 @@ 1540065280,1540068351,UA 1540068352,1540069375,FR 1540069376,1540070399,RU +1540070400,1540071423,PL 1540071424,1540072447,SE 1540072448,1540073471,UA 1540073472,1540074495,RU @@ -38261,6 +41462,7 @@ 1540208640,1540209663,NO 1540209664,1540211711,RU 1540211712,1540212735,DE +1540212736,1540213759,RU 1540213760,1540214783,UA 1540214784,1540215807,RU 1540215808,1540216831,NL @@ -38292,6 +41494,7 @@ 1540237312,1540238847,RO 1540238848,1540239359,DE 1540239360,1540239871,UA +1540239872,1540240383,FR 1540240384,1540240895,CH 1540240896,1540242431,RU 1540242432,1540242943,DE @@ -38424,7 +41627,7 @@ 1540318720,1540319231,RO 1540319232,1540320255,UA 1540320256,1540320767,RU -1540320768,1540321279,FR +1540320768,1540321279,DE 1540321280,1540321791,SE 1540321792,1540322303,RU 1540322304,1540322815,UA @@ -38437,7 +41640,9 @@ 1540325888,1540326399,LI 1540326400,1540326911,RU 1540326912,1540327423,AT -1540327424,1540329983,RU +1540327424,1540327935,RU +1540327936,1540328447,IR +1540328448,1540329983,RU 1540329984,1540330495,UA 1540330496,1540331007,PL 1540331008,1540331519,IT @@ -38522,6 +41727,7 @@ 1540366336,1540366591,PL 1540366592,1540366847,RU 1540366848,1540367103,AT +1540367104,1540367359,RU 1540367360,1540367615,HU 1540367616,1540367871,BE 1540367872,1540368127,DE @@ -38563,6 +41769,7 @@ 1540377344,1540377599,RO 1540377600,1540377855,AM 1540377856,1540378111,ES +1540378112,1540378367,PL 1540378368,1540378623,CH 1540378624,1540378879,AT 1540378880,1540379135,SE @@ -38792,7 +41999,6 @@ 1540442624,1540442879,IE 1540442880,1540443135,AT 1540443136,1540443647,RU -1540443648,1540443903,NL 1540443904,1540444159,GB 1540444160,1540444415,RU 1540444416,1540444671,DE @@ -38932,7 +42138,9 @@ 1540483840,1540484095,UA 1540484096,1540484351,DK 1540484352,1540484607,SI -1540484608,1540485119,UA +1540484608,1540484863,EU +1540484864,1540485119,UA +1540485120,1540485375,SE 1540485376,1540485631,RO 1540485632,1540485887,IR 1540485888,1540486143,ES @@ -39219,6 +42427,7 @@ 1540668672,1540668927,IL 1540668928,1540669695,RO 1540669696,1540669951,KZ +1540669952,1540670207,UA 1540670208,1540670463,RU 1540670464,1540670719,CH 1540670720,1540670975,RU @@ -39308,7 +42517,6 @@ 1540693504,1540693759,ES 1540693760,1540694015,UZ 1540694016,1540694271,CH -1540694272,1540694527,RU 1540694528,1540695039,RO 1540695040,1540695295,DE 1540695296,1540695551,NL @@ -39402,6 +42610,7 @@ 1540719872,1540720127,CH 1540720128,1540720383,NL 1540720384,1540720639,RO +1540720640,1540720895,CH 1540720896,1540721151,PL 1540721152,1540721663,RU 1540721664,1540721919,GB @@ -39604,6 +42813,7 @@ 1540873216,1540875263,RU 1540875264,1540876287,PL 1540876288,1540877311,RU +1540877312,1540878335,PL 1540878336,1540879359,AT 1540879360,1540880383,RU 1540880384,1540881407,UA @@ -39685,7 +42895,7 @@ 1540902912,1540903167,RO 1540903168,1540903423,RU 1540903424,1540903679,GB -1540903680,1540904447,CH +1540903680,1540904447,FR 1540904448,1540904703,EE 1540904704,1540904959,SI 1540904960,1540905471,GB @@ -39737,6 +42947,148 @@ 1540918528,1540918783,EU 1540918784,1540919039,NL 1540919040,1540919295,GB +1540919296,1540919551,RO +1540919552,1540919807,BE +1540919808,1540920063,GB +1540920064,1540920319,RU +1540920320,1540920575,TR +1540920576,1540920831,PL +1540920832,1540921087,HU +1540921088,1540921599,RO +1540921600,1540922111,EU +1540922112,1540922367,ES +1540922368,1540922879,RO +1540922880,1540923135,DE +1540923136,1540923391,GB +1540923392,1540923647,PL +1540923648,1540923903,UA +1540923904,1540924159,DE +1540924160,1540924415,LT +1540924416,1540924671,RU +1540924672,1540924927,UA +1540924928,1540925183,MD +1540925184,1540925439,RU +1540925440,1540925695,CZ +1540925696,1540925951,AE +1540925952,1540926207,TR +1540926208,1540926463,FR +1540926464,1540926719,SI +1540926720,1540926975,UA +1540926976,1540927231,BG +1540927232,1540927487,RU +1540927488,1540927743,GE +1540927744,1540927999,RO +1540928000,1540928255,KZ +1540928256,1540928511,CZ +1540928512,1540928767,GR +1540928768,1540929023,GB +1540929024,1540929279,DK +1540929280,1540929535,GB +1540929536,1540929791,NL +1540929792,1540930047,CZ +1540930048,1540930303,FR +1540930304,1540930559,RU +1540930560,1540930815,KW +1540930816,1540931071,KZ +1540931072,1540931327,EU +1540931328,1540931583,PL +1540931584,1540931839,RU +1540931840,1540932095,GB +1540932096,1540932351,PL +1540932352,1540932607,FR +1540932608,1540932863,UA +1540932864,1540933119,GB +1540933120,1540933375,RU +1540933376,1540933631,RO +1540933632,1540933887,NL +1540933888,1540934143,TR +1540934144,1540934399,PL +1540934400,1540934655,FR +1540934656,1540934911,PL +1540934912,1540935167,RU +1540935168,1540935423,GR +1540935424,1540935679,NL +1540935680,1540935935,RU +1540935936,1540936191,FR +1540936192,1540936447,CY +1540936448,1540936959,RU +1540936960,1540937471,PL +1540937472,1540937727,RU +1540937728,1540937983,DE +1540937984,1540938239,RU +1540938240,1540938751,ES +1540938752,1540939007,KZ +1540939008,1540939263,FR +1540939264,1540939519,IL +1540939520,1540939775,TR +1540939776,1540940031,PL +1540940032,1540940287,RO +1540940288,1540940543,DK +1540940544,1540940799,RO +1540940800,1540941055,FR +1540941056,1540941311,CH +1540941312,1540941567,AT +1540941568,1540941823,UA +1540941824,1540942079,RU +1540942080,1540942335,RO +1540942336,1540942591,FR +1540942592,1540942847,DE +1540942848,1540943103,IE +1540943104,1540943359,UA +1540943360,1540943615,BE +1540943616,1540943871,PL +1540943872,1540944127,UA +1540944128,1540944383,IE +1540944384,1540944639,AT +1540944640,1540944895,DE +1540944896,1540945151,UA +1540945152,1540945407,DE +1540945408,1540945663,GB +1540945664,1540945919,AT +1540945920,1540946175,DE +1540946432,1540946687,RU +1540946688,1540946943,UA +1540947200,1540947455,BG +1540947456,1540947711,PL +1540947712,1540947967,IS +1541013504,1541014527,RO +1541014528,1541015551,AM +1541015552,1541016575,RU +1541016576,1541018623,UA +1541018624,1541019647,ES +1541019648,1541020671,RU +1541020672,1541023743,UA +1541023744,1541024767,RU +1541024768,1541026815,UA +1541026816,1541027839,LV +1541027840,1541028863,RU +1541028864,1541029887,PL +1541029888,1541030911,UA +1541030912,1541031935,DE +1541031936,1541032959,UA +1541032960,1541033983,PL +1541033984,1541035007,BG +1541035008,1541036031,RU +1541036032,1541037055,UA +1541037056,1541038079,RU +1541038080,1541039103,UA +1541039104,1541040127,RU +1541040128,1541041151,UA +1541041152,1541042175,RU +1541042176,1541043199,DE +1541043200,1541044223,RU +1541044224,1541045247,US +1541045248,1541046271,NL +1541046272,1541051391,RU +1541051392,1541052415,NL +1541052416,1541053439,RO +1541053440,1541054463,PL +1541054464,1541055487,RU +1541055488,1541056511,NL +1541056512,1541057535,TJ +1541057536,1541058559,RS +1541058560,1541059583,RU +1541059584,1541060607,AM 1543503872,1545601023,GB 1545601024,1545863167,SE 1545863168,1545895935,RU @@ -39800,7 +43152,6 @@ 1546270720,1546272767,GB 1546272768,1546274815,NO 1546274816,1546276863,SE -1546276864,1546278911,GB 1546278912,1546280959,RU 1546280960,1546283007,IT 1546283008,1546285055,CH @@ -39865,11 +43216,13 @@ 1546665984,1546673823,GB 1546673824,1546673839,RS 1546673840,1546677503,GB -1546677504,1546677557,IN +1546677504,1546677551,IN +1546677552,1546677555,GB +1546677556,1546677557,IN 1546677558,1546677559,GB 1546677560,1546677561,IN -1546677562,1546677565,GB -1546677566,1546677569,IN +1546677562,1546677567,GB +1546677568,1546677569,IN 1546677570,1546677571,GB 1546677572,1546677573,IN 1546677574,1546677575,GB @@ -39901,7 +43254,6 @@ 1546895360,1546911743,IE 1546911744,1546928127,SK 1546928128,1546944511,GB -1546944512,1546960895,UA 1546960896,1546977279,HU 1546977280,1546993663,MK 1546993664,1547010047,RU @@ -40023,7 +43375,9 @@ 1547621304,1547621335,NG 1547621336,1547621351,A2 1547621352,1547621375,NG -1547621376,1547624447,A2 +1547621376,1547622463,A2 +1547622464,1547622471,NG +1547622472,1547624447,A2 1547624448,1547628543,CZ 1547628544,1547632639,BG 1547632640,1547636735,TR @@ -40115,8 +43469,7 @@ 1559339776,1559341695,RU 1559341696,1559341703,ES 1559341704,1559347199,RU -1559347200,1559351295,SE -1559351296,1559355391,DK +1559347200,1559355391,SE 1559355392,1559388159,RU 1559388160,1559396351,UA 1559396352,1559404543,GB @@ -40159,9 +43512,7 @@ 1559855104,1559887871,HU 1559887872,1559920639,PT 1559920640,1559932927,LU -1559932928,1559943167,DE -1559943168,1559950335,LU -1559950336,1559953407,DE +1559932928,1559953407,DE 1559953408,1559986175,MT 1559986176,1560018943,IE 1560018944,1560051711,DE @@ -40192,7 +43543,9 @@ 1566097408,1566101503,GB 1566101504,1566105599,RU 1566105600,1566109695,DE -1566109696,1566113791,CH +1566109696,1566111231,CH +1566111232,1566111295,FR +1566111296,1566113791,CH 1566113792,1566117887,DE 1566117888,1566121983,TR 1566121984,1566126079,NO @@ -40209,7 +43562,6 @@ 1566158848,1566162943,RU 1566162944,1566167039,PL 1566167040,1566171135,UA -1566171136,1566175231,HU 1566175232,1566179327,IE 1566179328,1566183423,DK 1566183424,1566187519,ES @@ -40281,8 +43633,7 @@ 1566382080,1566384127,TR 1566384128,1566386175,RU 1566386176,1566388223,FR -1566388224,1566388735,HU -1566388736,1566388991,PT +1566388224,1566388991,HU 1566388992,1566389247,SC 1566389248,1566389503,HU 1566389504,1566389759,SC @@ -40336,7 +43687,9 @@ 1566447616,1566451711,IT 1566451712,1566453759,IL 1566453760,1566455807,IQ -1566455808,1566457855,PT +1566455808,1566456351,PT +1566456352,1566456359,AO +1566456360,1566457855,PT 1566457856,1566459903,CH 1566459904,1566461951,GB 1566461952,1566463999,DE @@ -40447,7 +43800,9 @@ 1568295792,1568295799,GB 1568295800,1568295855,DE 1568295856,1568295863,GB -1568295864,1568309247,DE +1568295864,1568296239,DE +1568296240,1568296247,ZA +1568296248,1568309247,DE 1568309248,1568342015,RO 1568342016,1568374783,BG 1568374784,1568440319,RU @@ -40456,14 +43811,11 @@ 1568505856,1568538623,NL 1568538624,1568555007,IR 1568555008,1568571391,UA -1568571392,1568579583,RO -1568579584,1568581631,BG -1568581632,1568604159,RO 1568604160,1568636927,UA 1568636928,1568662271,DE 1568662272,1568663039,GB -1568663040,1568665599,DE -1568665600,1568669695,MD +1568663040,1568669439,DE +1568669440,1568669695,MD 1568669696,1569193983,DE 1569193984,1569718271,HR 1569718272,1570242559,IT @@ -40479,7 +43831,6 @@ 1570504704,1570570239,ES 1570570240,1570572287,NL 1570572288,1570574335,UA -1570574336,1570576383,RU 1570576384,1570578431,UA 1570578432,1570580479,CH 1570580480,1570582527,RU @@ -40499,7 +43850,9 @@ 1570621440,1570625535,RU 1570625536,1570627583,GB 1570627584,1570635775,RU -1570635776,1570652159,FR +1570635776,1570644991,FR +1570644992,1570645039,GB +1570645040,1570652159,FR 1570652160,1570652543,SE 1570652544,1570652551,FR 1570652552,1570652891,SE @@ -40547,7 +43900,9 @@ 1570665968,1570666175,SE 1570666176,1570666191,GB 1570666192,1570666223,FR -1570666224,1570666367,SE +1570666224,1570666227,SE +1570666228,1570666231,NO +1570666232,1570666367,SE 1570666368,1570666383,FR 1570666384,1570666431,SE 1570666432,1570666447,GB @@ -40601,7 +43956,9 @@ 1570667792,1570667807,ES 1570667808,1570667839,SE 1570667840,1570667903,FR -1570667904,1570667967,SE +1570667904,1570667907,SE +1570667908,1570667911,RU +1570667912,1570667967,SE 1570667968,1570667983,FR 1570667984,1570668031,SE 1570668032,1570668035,NL @@ -40615,14 +43972,14 @@ 1570717696,1570725887,HR 1570725888,1570734079,DE 1570734080,1570750463,PL -1570750464,1570752511,NO 1570752512,1570754559,GB 1570754560,1570756607,ES 1570756608,1570764799,RU 1570764800,1570766847,DE 1570766848,1571291135,DK 1571291136,1571422207,RO -1571422208,1571553279,CZ +1571422208,1571426303,RU +1571426304,1571553279,CZ 1571553280,1571684351,IL 1571684352,1571686399,ES 1571686400,1571688447,GB @@ -40634,9 +43991,7 @@ 1571700736,1571702783,IT 1571702784,1571704831,RU 1571704832,1571706879,GB -1571706880,1571707647,NL -1571707648,1571707903,BZ -1571707904,1571708927,NL +1571706880,1571708927,NL 1571708928,1571709439,GB 1571709440,1571709567,DE 1571709568,1571709695,GB @@ -40700,7 +44055,6 @@ 1572028416,1572044799,RU 1572044800,1572061183,IT 1572061184,1572077567,PL -1572077568,1572093951,RU 1572093952,1572110335,BG 1572110336,1572126719,RU 1572126720,1572143103,UA @@ -40936,7 +44290,6 @@ 1572831232,1572833279,BH 1572833280,1572835327,ES 1572835328,1572837375,IE -1572837376,1572839423,AM 1572839424,1572841471,DE 1572841472,1572842239,EU 1572842240,1572842495,LV @@ -40976,7 +44329,7 @@ 1578590276,1578590279,PT 1578590280,1578590283,PL 1578590284,1578590287,GB -1578590288,1578590303,PL +1578590288,1578590303,DE 1578590304,1578590311,IT 1578590312,1578590319,FR 1578590320,1578590335,CH @@ -41002,8 +44355,7 @@ 1578590656,1578590659,FR 1578590660,1578590663,CZ 1578590664,1578590667,FR -1578590668,1578590671,IT -1578590672,1578590687,PL +1578590668,1578590687,PL 1578590688,1578590699,FR 1578590700,1578590719,PL 1578590720,1578590799,FR @@ -41046,7 +44398,8 @@ 1578591392,1578591407,CH 1578591408,1578591431,GB 1578591432,1578591435,DE -1578591436,1578591463,FR +1578591436,1578591439,ES +1578591440,1578591463,FR 1578591464,1578591487,PL 1578591488,1578591503,FR 1578591504,1578591519,CH @@ -41080,7 +44433,7 @@ 1578592096,1578592111,FR 1578592112,1578592127,GB 1578592128,1578592143,DE -1578592144,1578592159,PL +1578592144,1578592159,NL 1578592160,1578592163,FR 1578592164,1578592171,IT 1578592172,1578592175,FR @@ -41121,17 +44474,19 @@ 1578592752,1578592783,DE 1578592784,1578592815,FR 1578592816,1578592823,PL -1578592824,1578592831,FR +1578592824,1578592827,FR +1578592828,1578592831,IE 1578592832,1578592847,PL 1578592848,1578592863,FR 1578592864,1578592879,PL -1578592880,1578592895,CH +1578592880,1578592883,FR +1578592884,1578592891,GB +1578592892,1578592895,FR 1578592896,1578592959,IT 1578592960,1578593007,FR 1578593008,1578593279,DE 1578593280,1578593295,PL -1578593296,1578593323,FR -1578593324,1578593327,ES +1578593296,1578593327,FR 1578593328,1578593343,IT 1578593344,1578593359,FR 1578593360,1578593407,GB @@ -41224,10 +44579,19 @@ 1578595132,1578595151,FR 1578595152,1578595167,ES 1578595168,1578595199,FR -1578595200,1578595327,GB +1578595200,1578595203,DE +1578595204,1578595207,FR +1578595208,1578595215,PL +1578595216,1578595231,DE +1578595232,1578595263,BE +1578595264,1578595267,ES +1578595268,1578595271,FR +1578595272,1578595275,PL +1578595276,1578595279,ES +1578595280,1578595295,PT +1578595296,1578595327,FR 1578595328,1578595343,IT -1578595344,1578595359,PL -1578595360,1578595363,FR +1578595344,1578595363,FR 1578595364,1578595367,DE 1578595368,1578595455,FR 1578595456,1578595463,DE @@ -41240,12 +44604,16 @@ 1578595616,1578595647,PL 1578595648,1578595679,FR 1578595680,1578595695,BE -1578595696,1578595759,FR -1578595760,1578595775,GB +1578595696,1578595763,FR +1578595764,1578595775,ES 1578595776,1578595807,CH 1578595808,1578595983,FR 1578595984,1578595991,NL -1578595992,1578596111,FR +1578595992,1578595995,IT +1578595996,1578596095,FR +1578596096,1578596099,DE +1578596100,1578596103,GB +1578596104,1578596111,FR 1578596112,1578596127,ES 1578596128,1578596143,PL 1578596144,1578596147,DE @@ -41253,7 +44621,8 @@ 1578596152,1578596155,DE 1578596156,1578596159,FR 1578596160,1578596175,PL -1578596176,1578596191,GB +1578596176,1578596183,GB +1578596184,1578596191,BE 1578596192,1578596207,ES 1578596208,1578596255,FR 1578596256,1578596287,GB @@ -41292,7 +44661,8 @@ 1578611296,1578611303,GB 1578611304,1578611311,FR 1578611312,1578611327,PL -1578611328,1578611359,FR +1578611328,1578611343,BE +1578611344,1578611359,FR 1578611360,1578611375,CH 1578611376,1578611391,FR 1578611392,1578611407,DE @@ -41375,7 +44745,11 @@ 1579089920,1579090575,NL 1579090576,1579090687,GB 1579090688,1579090943,NL -1579090944,1579091967,GB +1579090944,1579091759,GB +1579091760,1579091775,IL +1579091776,1579091839,GB +1579091840,1579091855,US +1579091856,1579091967,GB 1579091968,1579092223,DE 1579092224,1579104511,GB 1579104512,1579104767,NL @@ -41441,7 +44815,10 @@ 1582301184,1583349759,IT 1583349760,1583611903,SA 1583611904,1583615999,LV -1583616000,1583620095,NL +1583616000,1583617791,NL +1583617792,1583618047,US +1583618048,1583619839,NL +1583619840,1583620095,US 1583620096,1583624191,IT 1583624192,1583628287,RS 1583628288,1583632383,DE @@ -41456,8 +44833,8 @@ 1583669248,1583673343,GE 1583673344,1583673991,DE 1583673992,1583673999,GB -1583674000,1583674015,DE -1583674016,1583674039,NL +1583674000,1583674031,DE +1583674032,1583674039,NL 1583674040,1583677439,DE 1583677440,1583681535,FI 1583681536,1583685631,PL @@ -41500,7 +44877,9 @@ 1583813680,1583813683,GB 1583813684,1583813711,NL 1583813712,1583813727,US -1583813728,1583816703,NL +1583813728,1583813735,NL +1583813736,1583813743,DE +1583813744,1583816703,NL 1583816704,1583820799,TR 1583820800,1583824895,LV 1583824896,1583828991,SI @@ -41561,9 +44940,18 @@ 1585254400,1585256447,GB 1585256448,1585258495,DE 1585258496,1585260543,GB -1585262592,1585264639,RU +1585260544,1585264639,RU 1585264640,1585265663,MT -1585265664,1585266687,IM +1585265664,1585265695,FR +1585265696,1585265727,IM +1585265728,1585265767,FR +1585265768,1585265775,IM +1585265776,1585265943,FR +1585265944,1585265951,IM +1585265952,1585265967,FR +1585265968,1585265983,IM +1585265984,1585266047,FR +1585266048,1585266687,IM 1585266688,1585270783,DE 1585270784,1585272831,IT 1585272832,1585274879,RU @@ -41599,7 +44987,9 @@ 1585336320,1585338367,SE 1585338368,1585340415,RU 1585340416,1585342463,AT -1585342464,1585344511,GB +1585342464,1585343311,GB +1585343312,1585343319,DE +1585343320,1585344511,GB 1585344512,1585346559,FR 1585346560,1585348607,GB 1585348608,1585350655,CH @@ -41710,6 +45100,7 @@ 1586315264,1586323455,UZ 1586323456,1586331647,RU 1586331648,1586339839,PL +1586339840,1586348031,RU 1586348032,1586356223,SY 1586356224,1586372607,RU 1586372608,1586380799,JO @@ -41743,7 +45134,6 @@ 1586438144,1586446335,RU 1586446336,1586448383,NO 1586448384,1586450431,ES -1586450432,1586452479,RU 1586452480,1586454527,CH 1586454528,1586456575,IT 1586456576,1586458623,GB @@ -41805,15 +45195,14 @@ 1589510144,1589542911,RU 1589542912,1589575679,BG 1589575680,1589608447,RU -1589608448,1589610495,DK -1589610496,1589620735,SE -1589620736,1589623807,DK -1589623808,1589628927,SE -1589628928,1589641215,DK +1589608448,1589612543,DK +1589612544,1589620735,SE +1589620736,1589641215,DK 1589641216,1590034431,GB 1590034432,1590036479,RU 1590036480,1590038527,GB 1590038528,1590040575,RU +1590040576,1590042623,NL 1590042624,1590048767,RU 1590048768,1590050815,DE 1590050816,1590052863,SE @@ -41847,6 +45236,7 @@ 1590099968,1590102015,RU 1590102016,1590104063,IT 1590104064,1590106111,TJ +1590106112,1590108159,GB 1590108160,1590110207,TR 1590110208,1590112255,CZ 1590112256,1590114303,HU @@ -41897,7 +45287,9 @@ 1592054272,1592054527,AE 1592054528,1592054783,NL 1592054784,1592055295,AE -1592055296,1592056959,NL +1592055296,1592055935,NL +1592055936,1592055999,SG +1592056000,1592056959,NL 1592056960,1592057023,SG 1592057024,1592057855,NL 1592057856,1592061951,RS @@ -41958,7 +45350,7 @@ 1592287232,1592289279,RO 1592289280,1592291327,RU 1592291328,1592293375,NL -1592293376,1592299519,RU +1592295424,1592299519,RU 1592299520,1592303615,PL 1592303616,1592305663,NL 1592305664,1592307711,IR @@ -42028,7 +45420,7 @@ 1593209168,1593210879,SE 1593210880,1593211391,NO 1593211392,1593212415,SE -1593212416,1593212927,A1 +1593212416,1593212927,NO 1593212928,1593229311,PL 1593229312,1593245695,EE 1593245696,1593247743,NL @@ -42123,7 +45515,6 @@ 1599094784,1599111167,RU 1599111168,1599127551,IR 1599127552,1599143935,CZ -1599143936,1599160319,UA 1599160320,1599176703,IR 1599176704,1599193087,FR 1599193088,1599209471,RU @@ -42241,7 +45632,6 @@ 1602318336,1602320383,ES 1602320384,1602324479,NL 1602324480,1602326527,RU -1602326528,1602328575,GB 1602328576,1602330623,ES 1602330624,1602336767,RU 1602336768,1602338815,GB @@ -42309,7 +45699,9 @@ 1602449408,1602451455,LV 1602451456,1602453503,DE 1602453504,1602455551,SK -1602455552,1602457599,FR +1602455552,1602456015,FR +1602456016,1602456023,ES +1602456024,1602457599,FR 1602457600,1602459647,RU 1602459648,1602461695,GB 1602461696,1602465791,ES @@ -42320,6 +45712,7 @@ 1602473984,1602476031,RU 1602476032,1602478079,GB 1602478080,1602480127,RU +1602480128,1602482175,MT 1602482176,1602484223,FR 1602484224,1602486271,GB 1602486272,1602748415,UA @@ -42363,7 +45756,8 @@ 1603146080,1603146239,NL 1603146240,1603146751,US 1603146752,1603147007,NL -1603147008,1603147775,IL +1603147008,1603147263,US +1603147264,1603147775,IL 1603147776,1603148031,US 1603148032,1603149567,IL 1603149568,1603149823,US @@ -42384,7 +45778,10 @@ 1603167168,1603167231,NL 1603167232,1603167743,BE 1603167744,1603167871,NO -1603167872,1603170303,NL +1603167872,1603169919,NL +1603169920,1603169983,US +1603169984,1603170047,SE +1603170048,1603170303,NL 1603170304,1603174399,IE 1603174400,1603178495,TR 1603178496,1603182591,DK @@ -42443,9 +45840,7 @@ 1603982784,1603982847,AN 1603982848,1603984895,GB 1603984896,1603985151,PT -1603985152,1603986687,GB -1603986688,1603986751,TR -1603986752,1603990271,GB +1603985152,1603990271,GB 1603990272,1603990527,SA 1603990528,1603993599,GB 1603993600,1604009983,ME @@ -42472,9 +45867,7 @@ 1604517888,1604632320,RU 1604632321,1604632575,UA 1604632576,1604647679,RU -1604647680,1604647935,UA -1604647936,1604648191,RU -1604648192,1604648959,UA +1604647680,1604648959,UA 1604648960,1604714495,RU 1604714496,1604780031,DK 1604780032,1604845567,RU @@ -42522,9 +45915,7 @@ 1605025792,1605033983,IQ 1605033984,1605042175,SK 1605042176,1605050367,RU -1605050368,1605052039,IT -1605052040,1605052047,DE -1605052048,1605058559,IT +1605050368,1605058559,IT 1605058560,1605066751,HU 1605066752,1605074943,PL 1605074944,1605083135,FR @@ -42552,7 +45943,9 @@ 1605173248,1605181439,TR 1605181440,1605189631,LT 1605189632,1605189663,SK -1605189664,1605197823,CZ +1605189664,1605189759,CZ +1605189760,1605189887,SK +1605189888,1605197823,CZ 1605197824,1605206015,DE 1605206016,1605214207,RU 1605214208,1605222399,TR @@ -42602,7 +45995,9 @@ 1607610368,1607612415,IE 1607612416,1607614463,A2 1607614464,1607616511,GR -1607616512,1607663615,A2 +1607616512,1607630847,A2 +1607630848,1607632895,IE +1607632896,1607663615,A2 1607663616,1607729151,NL 1607729152,1607761919,EG 1607761920,1607766015,SY @@ -42624,6 +46019,7 @@ 1607942144,1607944191,RU 1607944192,1607945215,FR 1607946240,1607947263,UA +1607947264,1607948287,RU 1607948288,1607949311,GB 1607949312,1607950335,UA 1607952384,1607953407,UA @@ -42720,7 +46116,593 @@ 1652293632,1652310015,CA 1652310016,1652481279,US 1652481280,1652481791,CN -1652481792,1653534719,US +1652481792,1652621327,US +1652621328,1652621335,CA +1652621336,1652621343,AU +1652621344,1652621359,CA +1652621360,1652621495,US +1652621496,1652621503,CA +1652621504,1652621663,US +1652621664,1652621671,GB +1652621672,1652621679,US +1652621680,1652621687,CA +1652621688,1652621695,US +1652621696,1652621703,MX +1652621704,1652621911,US +1652621912,1652621919,GB +1652621920,1652622119,US +1652622120,1652622127,GB +1652622128,1652622175,US +1652622176,1652622207,GB +1652622208,1652622271,US +1652622272,1652622279,AU +1652622280,1652622575,US +1652622576,1652622583,GB +1652622584,1652622895,US +1652622896,1652622903,MX +1652622904,1652623903,US +1652623904,1652623935,IN +1652623936,1652624111,US +1652624112,1652624119,MX +1652624120,1652624511,US +1652624512,1652624519,IN +1652624520,1652624527,US +1652624528,1652624535,GB +1652624536,1652624591,US +1652624592,1652624599,IE +1652624600,1652624655,US +1652624656,1652624663,ZA +1652624664,1652625119,US +1652625120,1652625127,ZA +1652625128,1652625303,US +1652625304,1652625311,CA +1652625312,1652625495,US +1652625496,1652625503,CA +1652625504,1652625695,US +1652625696,1652625711,IN +1652625712,1652626055,US +1652626056,1652626063,GB +1652626064,1652626119,US +1652626120,1652626127,MX +1652626128,1652626319,US +1652626320,1652626327,GB +1652626328,1652626399,US +1652626400,1652626407,IE +1652626408,1652626487,US +1652626488,1652626495,GB +1652626496,1652626727,US +1652626728,1652626735,IL +1652626736,1652627519,US +1652627520,1652627567,MY +1652627568,1652627575,US +1652627576,1652627583,GB +1652627584,1652628055,US +1652628056,1652628063,IN +1652628064,1652628311,US +1652628312,1652628319,BO +1652628320,1652628351,US +1652628352,1652628359,CA +1652628360,1652629551,US +1652629552,1652629559,NZ +1652629560,1652630031,US +1652630032,1652630039,GB +1652630040,1652630247,US +1652630248,1652630255,AU +1652630256,1652630655,US +1652630656,1652630663,IL +1652630664,1652630887,US +1652630888,1652630895,AU +1652630896,1652631015,US +1652631016,1652631023,IN +1652631024,1652631343,US +1652631344,1652631351,CA +1652631352,1652631535,US +1652631536,1652631551,MY +1652631552,1652631751,US +1652631752,1652631759,GB +1652631760,1652631999,US +1652632000,1652632015,AE +1652632016,1652632255,US +1652632256,1652632263,IL +1652632264,1652632431,US +1652632432,1652632439,CA +1652632440,1652632647,US +1652632648,1652632655,MY +1652632656,1652632671,US +1652632672,1652632751,MY +1652632752,1652632863,US +1652632864,1652632879,CA +1652632880,1652632887,US +1652632888,1652632895,AU +1652632896,1652632911,US +1652632912,1652632919,CA +1652632920,1652632935,US +1652632936,1652632943,PH +1652632944,1652633159,US +1652633160,1652633167,CA +1652633168,1652633999,US +1652634000,1652634007,GB +1652634008,1652634143,US +1652634144,1652634151,AE +1652634152,1652634423,US +1652634424,1652634431,CA +1652634432,1652634647,US +1652634648,1652634655,PH +1652634656,1652634663,AU +1652634664,1652634767,US +1652634768,1652634775,AU +1652634776,1652634791,US +1652634792,1652634799,CA +1652634800,1652634855,US +1652634856,1652634863,CA +1652634864,1652635047,US +1652635048,1652635055,GB +1652635056,1652635295,US +1652635296,1652635311,CA +1652635312,1652635591,US +1652635592,1652635599,CA +1652635600,1652635903,US +1652635904,1652635911,AU +1652635912,1652636423,US +1652636424,1652636431,CA +1652636432,1652636631,US +1652636632,1652636639,IL +1652636640,1652636655,US +1652636656,1652636663,CA +1652636664,1652636839,US +1652636840,1652636847,CA +1652636848,1652637207,US +1652637208,1652637215,FR +1652637216,1652637623,US +1652637624,1652637631,GB +1652637632,1652637887,US +1652637888,1652637895,IL +1652637896,1652638143,US +1652638144,1652638151,CH +1652638152,1652638191,US +1652638192,1652638207,IN +1652638208,1652638215,US +1652638216,1652638223,AU +1652638224,1652638703,US +1652638704,1652638711,IN +1652638712,1652638855,US +1652638856,1652638863,CA +1652638864,1652638895,US +1652638896,1652638903,GB +1652638904,1652639015,US +1652639016,1652639023,MX +1652639024,1652639135,US +1652639136,1652639143,AU +1652639144,1652639455,US +1652639456,1652639471,IN +1652639472,1652639535,US +1652639536,1652639543,CA +1652639544,1652639671,US +1652639672,1652639679,GB +1652639680,1652640631,US +1652640632,1652640639,NL +1652640640,1652641183,US +1652641184,1652641191,BR +1652641192,1652641431,US +1652641432,1652641439,IN +1652641440,1652641879,US +1652641880,1652641887,GB +1652641888,1652641895,US +1652641896,1652641911,GB +1652641912,1652641919,CA +1652641920,1652641983,US +1652641984,1652642015,AE +1652642016,1652642023,CA +1652642024,1652643215,US +1652643216,1652643223,IT +1652643224,1652643351,US +1652643352,1652643359,CA +1652643360,1652643431,US +1652643432,1652643447,CA +1652643448,1652643463,US +1652643464,1652643471,CA +1652643472,1652643575,US +1652643576,1652643583,EG +1652643584,1652643631,US +1652643632,1652643639,IN +1652643640,1652643655,US +1652643656,1652643663,GB +1652643664,1652643679,US +1652643680,1652643687,GB +1652643688,1652643799,US +1652643800,1652643807,GB +1652643808,1652643839,US +1652643840,1652643847,GB +1652643848,1652643879,US +1652643880,1652643887,AU +1652643888,1652643895,CA +1652643896,1652644223,US +1652644224,1652644231,AU +1652644232,1652644247,US +1652644248,1652644255,GB +1652644256,1652644311,US +1652644312,1652644319,CA +1652644320,1652644375,US +1652644376,1652644383,CA +1652644384,1652644591,US +1652644592,1652644639,BD +1652644640,1652644647,US +1652644648,1652644655,BO +1652644656,1652644799,US +1652644800,1652644807,CA +1652644808,1652644863,US +1652644864,1652644871,AU +1652644872,1652645103,US +1652645104,1652645111,CA +1652645112,1652645751,US +1652645752,1652645759,NZ +1652645760,1652645791,US +1652645792,1652645799,IN +1652645800,1652645879,US +1652645880,1652645887,NZ +1652645888,1652645919,US +1652645920,1652645935,AE +1652645936,1652646271,US +1652646272,1652646279,AU +1652646280,1652646487,US +1652646488,1652646495,CA +1652646496,1652646503,US +1652646504,1652646511,CA +1652646512,1652646815,US +1652646816,1652646823,BD +1652646824,1652647199,US +1652647200,1652647207,IL +1652647208,1652647215,GB +1652647216,1652647303,US +1652647304,1652647311,AU +1652647312,1652647319,US +1652647320,1652647327,IL +1652647328,1652647351,US +1652647352,1652647359,GB +1652647360,1652647511,US +1652647512,1652647519,CA +1652647520,1652647647,US +1652647648,1652647655,MX +1652647656,1652647823,US +1652647824,1652647831,CA +1652647832,1652647855,US +1652647856,1652647871,CA +1652647872,1652647919,US +1652647920,1652647935,IE +1652647936,1652648103,US +1652648104,1652648111,MX +1652648112,1652648255,US +1652648256,1652648263,GB +1652648264,1652648399,US +1652648400,1652648407,IN +1652648408,1652648639,US +1652648640,1652648647,MY +1652648648,1652648879,US +1652648880,1652648887,BD +1652648888,1652648983,US +1652648984,1652648991,NZ +1652648992,1652649463,US +1652649464,1652649471,GB +1652649472,1652649519,US +1652649520,1652649527,SE +1652649528,1652649543,US +1652649544,1652649551,CA +1652649552,1652650303,US +1652650304,1652650311,SE +1652650312,1652650335,US +1652650336,1652650343,CA +1652650344,1652650679,US +1652650680,1652650687,CA +1652650688,1652650751,US +1652650752,1652650759,AU +1652650760,1652650791,US +1652650792,1652650799,GB +1652650800,1652651063,US +1652651064,1652651071,SG +1652651072,1652651079,US +1652651080,1652651087,GB +1652651088,1652651111,US +1652651112,1652651119,NZ +1652651120,1652651271,US +1652651272,1652651279,IN +1652651280,1652651351,US +1652651352,1652651359,CA +1652651360,1652651495,US +1652651496,1652651503,AR +1652651504,1652652151,US +1652652152,1652652159,IN +1652652160,1652652327,US +1652652328,1652652335,SG +1652652336,1652652591,US +1652652592,1652652599,BD +1652652600,1652652655,US +1652652656,1652652663,GB +1652652664,1652652895,US +1652652896,1652652927,CA +1652652928,1652653231,US +1652653232,1652653239,CA +1652653240,1652653247,US +1652653248,1652653279,BD +1652653280,1652653343,US +1652653344,1652653359,BD +1652653360,1652653631,US +1652653632,1652653639,CA +1652653640,1652653663,US +1652653664,1652653671,IN +1652653672,1652653679,GB +1652653680,1652653855,US +1652653856,1652653863,AR +1652653864,1652654023,US +1652654024,1652654031,CA +1652654032,1652654527,US +1652654528,1652654535,NZ +1652654536,1652654543,CA +1652654544,1652654719,US +1652654720,1652654727,GB +1652654728,1652654783,US +1652654784,1652654815,BD +1652654816,1652654895,US +1652654896,1652654911,ZA +1652654912,1652654927,US +1652654928,1652654935,GB +1652654936,1652655007,US +1652655008,1652655015,GB +1652655016,1652655063,US +1652655064,1652655071,GU +1652655072,1652655231,US +1652655232,1652655263,CA +1652655264,1652655303,US +1652655304,1652655311,IN +1652655312,1652655431,US +1652655432,1652655439,GU +1652655440,1652655511,US +1652655512,1652655519,AU +1652655520,1652655727,US +1652655728,1652655743,GB +1652655744,1652655871,US +1652655872,1652655879,CA +1652655880,1652655887,US +1652655888,1652655895,IN +1652655896,1652655903,GB +1652655904,1652655943,US +1652655944,1652655951,JM +1652655952,1652655983,US +1652655984,1652655999,GB +1652656000,1652656063,US +1652656064,1652656071,GB +1652656072,1652656383,US +1652656384,1652656391,CA +1652656392,1652656431,US +1652656432,1652656439,ZA +1652656440,1652656447,CA +1652656448,1652656671,US +1652656672,1652656679,GB +1652656680,1652656703,US +1652656704,1652656711,CA +1652656712,1652656719,US +1652656720,1652656727,NZ +1652656728,1652656759,US +1652656760,1652656767,CN +1652656768,1652656799,US +1652656800,1652656807,GB +1652656808,1652656855,US +1652656856,1652656863,PH +1652656864,1652656927,US +1652656928,1652656935,GB +1652656936,1652657295,US +1652657296,1652657311,AU +1652657312,1652657439,US +1652657440,1652657447,MX +1652657448,1652657567,US +1652657568,1652657575,JM +1652657576,1652657863,US +1652657864,1652657871,GB +1652657872,1652658223,US +1652658224,1652658231,GB +1652658232,1652658367,US +1652658368,1652658375,MX +1652658376,1652658431,US +1652658432,1652658439,GB +1652658440,1652658535,US +1652658536,1652658543,GB +1652658544,1652658559,US +1652658560,1652658567,CA +1652658568,1652658687,US +1652658688,1652658695,NZ +1652658696,1652658727,US +1652658728,1652658735,IN +1652658736,1652659151,US +1652659152,1652659159,GB +1652659160,1652659367,US +1652659368,1652659375,CA +1652659376,1652660007,US +1652660008,1652660015,CA +1652660016,1652660127,US +1652660128,1652660135,CA +1652660136,1652660183,US +1652660184,1652660191,MX +1652660192,1652660287,US +1652660288,1652660295,IN +1652660296,1652660303,MX +1652660304,1652660551,US +1652660552,1652660559,GB +1652660560,1652660639,US +1652660640,1652660663,AU +1652660664,1652660959,US +1652660960,1652660967,BR +1652660968,1652661087,US +1652661088,1652661095,MY +1652661096,1652661135,US +1652661136,1652661143,GB +1652661144,1652661151,US +1652661152,1652661159,IL +1652661160,1652661191,US +1652661192,1652661199,CA +1652661200,1652662735,US +1652662736,1652662751,GB +1652662752,1652662919,US +1652662920,1652662927,AU +1652662928,1652663039,US +1652663040,1652663047,MX +1652663048,1652663135,US +1652663136,1652663143,CA +1652663144,1652664527,US +1652664528,1652664543,CA +1652664544,1652665079,US +1652665080,1652665087,CA +1652665088,1652665103,US +1652665104,1652665111,CA +1652665112,1652665319,US +1652665320,1652665327,BR +1652665328,1652665335,PR +1652665336,1652665455,US +1652665456,1652665463,CA +1652665464,1652665479,US +1652665480,1652665487,GB +1652665488,1652665671,US +1652665672,1652665679,MX +1652665680,1652665687,IE +1652665688,1652665807,US +1652665808,1652665815,GB +1652665816,1652665863,US +1652665864,1652665871,AU +1652665872,1652665951,US +1652665952,1652665959,IT +1652665960,1652665983,US +1652665984,1652665991,CA +1652665992,1652666175,US +1652666176,1652666183,CA +1652666184,1652666207,US +1652666208,1652666215,GB +1652666216,1652666327,US +1652666328,1652666335,MX +1652666336,1652666495,US +1652666496,1652666511,CA +1652666512,1652666527,US +1652666528,1652666543,GB +1652666544,1652668951,US +1652668952,1652668959,IE +1652668960,1652668967,US +1652668968,1652668975,AU +1652668976,1652669191,US +1652669192,1652669199,IN +1652669200,1652669231,US +1652669232,1652669239,CA +1652669240,1652669567,US +1652669568,1652669575,GB +1652669576,1652670015,US +1652670016,1652670023,CA +1652670024,1652670311,US +1652670312,1652670319,BR +1652670320,1652670327,US +1652670328,1652670335,BR +1652670336,1652670463,CA +1652670464,1652670871,US +1652670872,1652670879,CA +1652670880,1652671167,US +1652671168,1652671175,NL +1652671176,1652671207,US +1652671208,1652671215,NL +1652671216,1652671271,US +1652671272,1652671279,GB +1652671280,1652671343,US +1652671344,1652671351,GB +1652671352,1652671359,MX +1652671360,1652671375,US +1652671376,1652671383,CA +1652671384,1652671407,US +1652671408,1652671415,GB +1652671416,1652671783,US +1652671784,1652671791,NL +1652671792,1652672023,US +1652672024,1652672031,MX +1652672032,1652672063,US +1652672064,1652672071,GB +1652672072,1652672159,US +1652672160,1652672167,JP +1652672168,1652672191,US +1652672192,1652672199,PK +1652672200,1652672295,US +1652672296,1652672303,MT +1652672304,1652672335,US +1652672336,1652672343,MT +1652672344,1652672439,US +1652672440,1652672447,CA +1652672448,1652672455,GB +1652672456,1652672463,US +1652672464,1652672471,IN +1652672472,1652672479,ZA +1652672480,1652672487,IN +1652672488,1652672799,US +1652672800,1652672807,CA +1652672808,1652673279,US +1652673280,1652673535,CA +1652673536,1652674063,US +1652674064,1652674071,CA +1652674072,1652674135,US +1652674136,1652674143,MX +1652674144,1652674327,US +1652674328,1652674335,GB +1652674336,1652674343,US +1652674344,1652674351,NA +1652674352,1652674367,US +1652674368,1652674375,GB +1652674376,1652674879,US +1652674880,1652674887,CA +1652674888,1652674919,US +1652674920,1652674927,CH +1652674928,1652674943,US +1652674944,1652674951,CA +1652674952,1652675047,US +1652675048,1652675055,MX +1652675056,1652675103,US +1652675104,1652675111,GB +1652675112,1652675391,US +1652675392,1652675399,IN +1652675400,1652675487,US +1652675488,1652675495,NZ +1652675496,1652675951,US +1652675952,1652675959,KE +1652675960,1652676031,US +1652676032,1652676047,IN +1652676048,1652676191,US +1652676192,1652676199,IN +1652676200,1652676615,US +1652676616,1652676623,AU +1652676624,1652676679,US +1652676680,1652676687,CA +1652676688,1652677007,US +1652677008,1652677015,IN +1652677016,1652677039,US +1652677040,1652677047,MT +1652677048,1652677335,US +1652677336,1652677343,KE +1652677344,1652677367,US +1652677368,1652677375,GB +1652677376,1652677407,US +1652677408,1652677415,IN +1652677416,1652677575,US +1652677576,1652677583,CH +1652677584,1652678999,US +1652679000,1652679007,SG +1652679008,1652679031,US +1652679032,1652679039,BR +1652679040,1652679103,US +1652679104,1652679111,MX +1652679112,1652679119,NO +1652679120,1652680743,US +1652680744,1652680751,CA +1652680752,1652681047,US +1652681048,1652681055,GB +1652681056,1652681159,US +1652681160,1652681167,CA +1652681168,1652681455,US +1652681456,1652681463,CA +1652681464,1652681471,US +1652681472,1652681479,GB +1652681480,1652681511,US +1652681512,1652681519,IL +1652681520,1653534719,US 1653534720,1653538815,CA 1653538816,1653555199,US 1653555200,1653567487,CA @@ -42760,8 +46742,7 @@ 1673562752,1673562767,CA 1673562768,1673562895,US 1673562896,1673562911,CA -1673562912,1673562943,US -1673562944,1673562975,BG +1673562912,1673562975,US 1673562976,1673563007,NL 1673563008,1673563071,DE 1673563072,1673563135,NL @@ -42840,7 +46821,8 @@ 1673986048,1674051583,CA 1674051584,1674575871,US 1674575872,1677721599,CA -1811939328,1820327935,US +1795162112,1795686399,US +1811939328,1828716543,US 1828716544,1830813695,FR 1830813696,1831337983,NL 1831337984,1831862271,DE @@ -42854,7 +46836,8 @@ 1832747008,1832779775,RU 1832779776,1832812543,FR 1832812544,1832845311,RU -1832845312,1832910847,BH +1832845312,1832878079,BH +1832878080,1832910847,RU 1832910848,1833172991,IL 1833172992,1833177087,GB 1833177088,1833179135,RU @@ -42891,12 +46874,11 @@ 1833246720,1833248767,FI 1833248768,1833250815,MK 1833250816,1833254911,GB -1833254912,1833256047,DE -1833256048,1833256063,US -1833256064,1833256159,DE +1833254912,1833256159,DE 1833256160,1833256175,GB 1833256176,1833256191,DE -1833256192,1833256223,US +1833256192,1833256207,US +1833256208,1833256223,GB 1833256224,1833256255,DE 1833256256,1833256271,GB 1833256272,1833256287,DE @@ -42931,10 +46913,14 @@ 1833299968,1833302015,LU 1833302016,1833304063,AT 1833304064,1833308159,NL -1833308160,1833310207,FR +1833308160,1833308755,FR +1833308756,1833308759,NL +1833308760,1833310207,FR 1833310208,1833312255,RU 1833312256,1833314303,ES -1833314304,1833316351,IM +1833314304,1833315903,IM +1833315904,1833315919,GB +1833315920,1833316351,IM 1833316352,1833318399,DK 1833318400,1833320447,GB 1833320448,1833322495,AE @@ -43012,11 +46998,16 @@ 1833459712,1833463807,ME 1833463808,1833467903,UA 1833467904,1833468079,RU -1833468080,1833468575,CH +1833468080,1833468159,CH +1833468160,1833468287,RU +1833468288,1833468575,CH 1833468576,1833468591,RU 1833468592,1833471999,CH 1833472000,1833476095,EU -1833476096,1833484287,NL +1833476096,1833477375,NL +1833477376,1833477503,GB +1833477504,1833477631,DE +1833477632,1833484287,NL 1833484288,1833488383,IR 1833488384,1833492479,GB 1833492480,1833504767,RU @@ -43039,7 +47030,8 @@ 1833558016,1833562111,RO 1833562112,1833566207,PS 1833566208,1833570303,SE -1833570304,1833574399,NL +1833570304,1833571327,GB +1833571328,1833574399,NL 1833574400,1833578495,YE 1833578496,1833582591,HU 1833582592,1833586687,TJ @@ -43054,7 +47046,9 @@ 1833623552,1833627647,IR 1833627648,1833631743,GB 1833631744,1833635839,CZ -1833635840,1833639935,DE +1833635840,1833636367,DE +1833636368,1833636375,AL +1833636376,1833639935,DE 1833639936,1833644031,AM 1833644032,1833644063,TJ 1833644064,1833644287,RU @@ -43065,7 +47059,10 @@ 1833659024,1833659039,BZ 1833659040,1833659903,DE 1833659904,1833659919,NA -1833659920,1833660415,DE +1833659920,1833660159,DE +1833660160,1833660223,CH +1833660224,1833660351,DE +1833660352,1833660415,US 1833660416,1833664511,IT 1833664512,1833668607,RU 1833668608,1833672703,CZ @@ -43269,7 +47266,9 @@ 1839801600,1839801855,US 1839801856,1839802111,GB 1839802112,1839802239,RO -1839802240,1839816703,GB +1839802240,1839806975,GB +1839806976,1839808511,US +1839808512,1839816703,GB 1839816704,1839824895,NO 1839824896,1839890431,RU 1839890432,1839923199,GB @@ -43314,7 +47313,6 @@ 1841635328,1841639423,PL 1841639424,1841641471,RU 1841641472,1841643519,PL -1841643520,1841645567,SI 1841645568,1841647615,RU 1841647616,1841649663,RO 1841649664,1841651711,RU @@ -43344,7 +47342,9 @@ 1841799168,1841807359,DE 1841807360,1841815551,NO 1841815552,1841823743,BG -1841823744,1841831935,GB +1841823744,1841827079,GB +1841827080,1841827087,ES +1841827088,1841831935,GB 1841831936,1841840127,MT 1841840128,1841848319,PL 1841848320,1841856511,RU @@ -43370,7 +47370,9 @@ 1842028544,1842036735,CH 1842036736,1842038783,FR 1842038784,1842044927,LU -1842044928,1842053119,GB +1842044928,1842047369,GB +1842047370,1842047371,IN +1842047372,1842053119,GB 1842053120,1842061311,ES 1842061312,1842069503,IR 1842069504,1842077695,RU @@ -43431,7 +47433,8 @@ 1842323456,1842331647,RU 1842331648,1842335743,UA 1842335744,1842339839,RO -1842339840,1842348031,RU +1842339840,1842343935,RU +1842343936,1842348031,UZ 1842348032,1843396607,FR 1843396608,1843412991,IQ 1843412992,1843429375,CZ @@ -43453,7 +47456,9 @@ 1843691520,1843707903,SE 1843707904,1843724287,TR 1843724288,1843732479,UA -1843732480,1843740671,RU +1843732480,1843765247,RU +1843773440,1843781631,LV +1843781632,1843789823,RU 1843789824,1843806207,SK 1843806208,1843822591,IR 1843822592,1843838975,RU @@ -43477,7 +47482,7 @@ 1843945472,1843947519,FR 1843947520,1843949567,RU 1843949568,1843951615,BY -1843953664,1843955711,DE +1843951616,1843955711,DE 1843955712,1843957759,IT 1843957760,1843959807,CZ 1843959808,1843961855,GB @@ -43557,8 +47562,7 @@ 1844123648,1844125695,RU 1844125696,1844127743,NL 1844127744,1844129791,DE -1844129792,1844130047,SE -1844130048,1844131839,NL +1844129792,1844131839,NL 1844131840,1844133887,DE 1844133888,1844135935,LT 1844135936,1844137983,NL @@ -43598,6 +47602,8 @@ 1844207616,1844211711,RU 1844211712,1844215807,SK 1844215808,1844219903,BE +1844219904,1844220159,KE +1844220160,1844223999,DE 1844224000,1844228095,GB 1844228096,1844232191,DK 1844232192,1844235775,ES @@ -43758,7 +47764,7 @@ 1851527168,1851528191,NZ 1851529216,1851531263,PH 1851531264,1851539455,JP -1851539456,1851542527,ID +1851541504,1851542527,ID 1851542528,1851543551,HK 1851543552,1851547647,JP 1851547648,1851555839,SG @@ -43782,7 +47788,7 @@ 1855979520,1856241663,JP 1856241664,1856307199,TH 1856307200,1856315391,KR -1856315392,1856323583,HK +1856315392,1856323583,CN 1856323584,1856339967,KR 1856339968,1856372735,JP 1856372736,1856503807,CN @@ -44672,6 +48678,8 @@ 1966424064,1966440447,KR 1966440448,1966444543,AU 1966444544,1966446591,NZ +1966446592,1966447615,MY +1966447616,1966448639,NZ 1966448640,1966452735,AU 1966452736,1966456831,CN 1966456832,1966473215,KR @@ -45344,7 +49352,7 @@ 2056290304,2056323071,CN 2056323072,2056388607,JP 2056388608,2056519679,TW -2056519680,2056650751,AU +2056519680,2056781823,AU 2056781824,2056794111,JP 2056794112,2056796159,BD 2056798208,2056806399,JP @@ -46289,7 +50297,8 @@ 2208235520,2208301055,DE 2208301056,2208366591,FI 2208432128,2208563199,CA -2208563200,2208759807,DK +2208563200,2208694271,DK +2208694272,2208759807,EU 2208759808,2208890879,US 2208890880,2208956415,DE 2208956416,2209021951,AU @@ -47705,7 +51714,7 @@ 2468175360,2468189663,DK 2468189664,2468189695,GB 2468189696,2468216831,DK -2468216832,2468282367,DE +2468216832,2468282367,EU 2468282368,2468347903,US 2468347904,2468478975,CZ 2468478976,2468937727,US @@ -47751,7 +51760,9 @@ 2473394176,2473459711,AU 2473459712,2473525247,ZA 2473525248,2473656319,NO -2473656320,2474049535,US +2473656320,2473721855,US +2473721856,2473787391,GB +2473787392,2474049535,US 2474049536,2474115071,GB 2474115072,2474246143,US 2474246144,2474377215,IT @@ -48358,7 +52369,8 @@ 2644639744,2644770815,DE 2644770816,2644836351,BE 2644836352,2644967423,US -2644967424,2645098495,AT +2644967424,2645032959,AT +2645032960,2645098495,CH 2645098496,2645164031,FR 2645164032,2645229567,US 2645229568,2645295103,SE @@ -49005,8 +53017,7 @@ 2751922176,2751987711,FR 2751987712,2752053247,SI 2752053248,2752184319,SE -2752184320,2752249855,GB -2752249856,2752315391,DE +2752184320,2752315391,GB 2752315392,2752380927,FI 2752380928,2752446463,CH 2752446464,2752511999,BE @@ -49264,7 +53275,9 @@ 2788229936,2788229943,GB 2788229944,2788230663,US 2788230664,2788230679,GB -2788230680,2788233263,US +2788230680,2788230703,US +2788230704,2788230719,GB +2788230720,2788233263,US 2788233264,2788233271,GB 2788233272,2788234279,US 2788234280,2788234287,CA @@ -49408,7 +53421,13 @@ 2816003440,2816003455,US 2816003456,2816003855,IL 2816003856,2816003871,US -2816003872,2816010495,IL +2816003872,2816004351,IL +2816004352,2816004367,US +2816004368,2816004415,IL +2816004416,2816004479,US +2816004480,2816008191,IL +2816008192,2816008207,IE +2816008208,2816010495,IL 2816010496,2816010751,US 2816010752,2816014239,IL 2816014240,2816014271,US @@ -49636,31 +53655,38 @@ 2899378176,2899443711,GB 2899443712,2899574783,FR 2899574784,2899902463,GB -2902458368,2904555519,US +2902458368,2902462463,A1 +2902462464,2902507519,US +2902507520,2902515711,CA +2902515712,2904555519,US 2904555520,2904817663,CA 2904817664,2905376767,US 2905376768,2905377535,CA 2905377536,2905377791,US 2905377792,2905378047,CA -2905378048,2905379071,US +2905378048,2905378303,US +2905378304,2905378815,CA +2905378816,2905379071,US 2905379072,2905379583,CA -2905379584,2905381887,US +2905379584,2905380863,US +2905380864,2905381119,CA +2905381120,2905381887,US 2905381888,2905382016,CA 2905382017,2905384191,US 2905384192,2905385471,CA 2905385472,2905386239,US 2905386240,2905386495,CA 2905386496,2905388031,US -2905388032,2905389311,CA -2905389312,2905390079,US -2905390080,2905390335,CA -2905390336,2905391871,US +2905388032,2905388287,CA +2905388288,2905391871,US 2905391872,2905392127,CA -2905392128,2905393151,US -2905393152,2905394943,CA +2905392128,2905392895,US +2905392896,2905394943,CA 2905394944,2905395455,US 2905395456,2905396991,CA -2905396992,2905404671,US +2905396992,2905397759,US +2905397760,2905398527,CA +2905398528,2905404671,US 2905404672,2905405439,CA 2905405440,2905407743,US 2905407744,2905407999,TW @@ -49668,8 +53694,8 @@ 2905428968,2905428975,AE 2905428976,2905432975,US 2905432976,2905432983,AE -2905432984,2905450495,US -2905450496,2905451007,CA +2905432984,2905449983,US +2905449984,2905451007,CA 2905451008,2905451519,US 2905451520,2905451647,PA 2905451648,2905451775,US @@ -49726,9 +53752,29 @@ 2915768488,2915768495,CA 2915768496,2915768703,US 2915768704,2915768767,GB -2915768768,2915794959,US +2915768768,2915768903,US +2915768904,2915768911,GR +2915768912,2915769095,US +2915769096,2915769103,BB +2915769104,2915769111,US +2915769112,2915769119,BB +2915769120,2915769239,US +2915769240,2915769247,CA +2915769248,2915769295,US +2915769296,2915769303,BR +2915769304,2915772671,US +2915772672,2915772679,IN +2915772680,2915772711,US +2915772712,2915772719,GB +2915772720,2915773023,US +2915773024,2915773039,IN +2915773040,2915773175,US +2915773176,2915773183,KW +2915773184,2915794959,US 2915794960,2915794975,MX -2915794976,2915795263,US +2915794976,2915795007,US +2915795008,2915795023,BR +2915795024,2915795263,US 2915795264,2915795279,NZ 2915795280,2915795343,US 2915795344,2915795359,NZ @@ -49922,7 +53968,21 @@ 2915807440,2915807447,CA 2915807448,2915807455,US 2915807456,2915807463,PT -2915807464,2915811135,US +2915807464,2915807519,US +2915807520,2915807527,CA +2915807528,2915807559,US +2915807560,2915807567,CA +2915807568,2915807607,US +2915807608,2915807615,IL +2915807616,2915807951,US +2915807952,2915807959,CA +2915807960,2915808095,US +2915808096,2915808103,CA +2915808104,2915808183,US +2915808184,2915808191,CA +2915808192,2915808247,US +2915808248,2915808255,DE +2915808256,2915811135,US 2915811136,2915811199,IN 2915811200,2915958783,US 2915958784,2916024319,CA @@ -49932,7 +53992,9 @@ 2916092928,2916093183,IN 2916093184,2916093695,US 2916093696,2916093951,IN -2916093952,2916163583,US +2916093952,2916110623,US +2916110624,2916110639,CA +2916110640,2916163583,US 2916163584,2916171775,CA 2916171776,2916188159,US 2916188160,2916196351,CA @@ -49964,7 +54026,7 @@ 2916447232,2916448255,CA 2916448256,2916449279,US 2916449280,2916450303,CA -2916450304,2916499455,US +2916450304,2916515839,US 2916548608,2916581375,US 2916581376,2916614143,PR 2916614144,2917195775,US @@ -49979,9 +54041,17 @@ 2917265408,2917269503,JM 2917269504,2917572607,US 2917572608,2917580799,CA -2917580800,2917665791,US +2917580800,2917597439,US +2917597440,2917597695,GB +2917597696,2917621759,US +2917629952,2917646335,US +2917662720,2917663231,US +2917663232,2917663487,CA +2917663488,2917665791,US 2917665792,2917666303,CA -2917666304,2917675263,US +2917666304,2917667710,US +2917667711,2917671935,CA +2917671936,2917675263,US 2917675264,2917675775,CA 2917675776,2917676031,US 2917676032,2917677055,CA @@ -49995,14 +54065,12 @@ 2917685248,2917686015,CA 2917686016,2917690367,US 2917690368,2917690879,CA -2917690880,2917699583,US -2917699584,2917699839,CA -2917699840,2917701119,US +2917690880,2917693183,US +2917693184,2917693951,CA +2917693952,2917701119,US 2917701120,2917701375,CA 2917701376,2917702399,US -2917702400,2917704703,CA -2917704704,2917706239,US -2917706240,2917707519,CA +2917702400,2917707519,CA 2917707520,2917711871,US 2917711872,2917714431,CA 2917714432,2917718527,US @@ -50010,8 +54078,8 @@ 2917719040,2917719295,US 2917719296,2917719551,CA 2917719552,2917722367,US -2917722368,2917723135,CA -2917723136,2918055935,US +2917722368,2917722879,CA +2917722880,2918055935,US 2918055936,2918121471,CA 2918121472,2918154239,US 2918154240,2918170623,CA @@ -50032,11 +54100,29 @@ 2918391808,2918395903,CA 2918395904,2918404095,US 2918404096,2918408191,PR -2918449152,2918580223,US +2918408192,2918432767,US +2918432768,2918436863,CA +2918436864,2918464511,US +2918464512,2918465023,CA +2918465024,2918580223,US 2918580224,2918588415,CA -2918711296,2918760447,US +2918588416,2918596607,US +2918596608,2918604799,CA +2918604800,2918608895,US +2918612992,2918617087,CA +2918621184,2918653951,US +2918653952,2918662143,CA +2918662144,2918760447,US 2918760448,2918776831,CA -2918776832,2918793215,US +2918776832,2918842367,US +2918842368,2918875135,CA +2918875136,2918973439,US +2918973440,2918989823,CA +2919170048,2919174143,US +2919174144,2919178239,CA +2919178240,2919186431,US +2919186432,2919190527,CA +2919190528,2919202815,US 2919206912,2919211007,CA 2919235584,2919759871,CA 2919759872,2921496895,US @@ -50049,7 +54135,9 @@ 2921497416,2921497423,IN 2921497424,2921497471,US 2921497472,2921497599,IN -2921497600,2921503695,US +2921497600,2921503607,US +2921503608,2921503615,GB +2921503616,2921503695,US 2921503696,2921503703,GB 2921503704,2921508095,US 2921508096,2921508103,IN @@ -50058,7 +54146,8 @@ 2921508352,2921508639,US 2921508640,2921508671,IN 2921508672,2921508679,US -2921508680,2921508727,IN +2921508680,2921508719,IN +2921508720,2921508727,SE 2921508728,2921511351,US 2921511352,2921511359,IN 2921511360,2921511743,US @@ -50188,7 +54277,9 @@ 2928270480,2928270487,NL 2928270488,2928270551,CA 2928270552,2928270559,US -2928270560,2928270607,CA +2928270560,2928270583,CA +2928270584,2928270591,US +2928270592,2928270607,CA 2928270608,2928270615,US 2928270616,2928270623,CA 2928270624,2928270631,US @@ -50290,7 +54381,9 @@ 2928275720,2928275727,US 2928275728,2928275767,CA 2928275768,2928275775,US -2928275776,2928276071,CA +2928275776,2928275815,CA +2928275816,2928275823,US +2928275824,2928276071,CA 2928276072,2928276079,US 2928276080,2928276127,CA 2928276128,2928276135,US @@ -51012,7 +55105,7 @@ 2987458560,2987462655,CZ 2987462656,2987466751,RS 2987466752,2987470847,GB -2987470848,2987474943,IQ +2987470848,2987474943,DE 2987474944,2987479039,GB 2987479040,2987487231,CZ 2987487232,2987491327,HR @@ -51028,7 +55121,8 @@ 2987515904,2987519999,A2 2987520000,2987524095,GB 2987524096,2987528191,RU -2987528192,2987532287,NL +2987528192,2987529215,US +2987529216,2987532287,NL 2987532288,2987536383,MD 2987536384,2987540479,FR 2987540480,2987544575,SK @@ -51055,7 +55149,8 @@ 2987593728,2987597823,LT 2987597824,2987601919,ES 2987601920,2987606015,IS -2987606016,2987610111,DE +2987606016,2987606527,US +2987606528,2987610111,DE 2987610112,2987614207,RU 2987614208,2987618303,PL 2987618304,2987622399,NL @@ -51165,6 +55260,7 @@ 2987859968,2987862015,FR 2987862016,2987864063,BE 2987864064,2987866111,DE +2987866112,2987868159,ES 2987868160,2987870207,RO 2987870208,2987872255,GR 2987872256,2987874303,AT @@ -51218,7 +55314,11 @@ 2988442440,2988442447,ES 2988442448,2988442463,GB 2988442464,2988442895,FR -2988442896,2988442919,NL +2988442896,2988442899,PL +2988442900,2988442903,FR +2988442904,2988442907,IT +2988442908,2988442911,FR +2988442912,2988442919,NL 2988442920,2988442923,LT 2988442924,2988442927,GB 2988442928,2988442975,FR @@ -51234,7 +55334,10 @@ 2988443112,2988443119,ES 2988443120,2988443391,FR 2988443392,2988443407,ES -2988443408,2988443455,FR +2988443408,2988443439,FR +2988443440,2988443443,CZ +2988443444,2988443447,DE +2988443448,2988443455,FR 2988443456,2988443487,BE 2988443488,2988443539,FR 2988443540,2988443547,ES @@ -51299,8 +55402,7 @@ 2988448000,2988448127,DE 2988448128,2988448255,ES 2988448256,2988448511,DE -2988448512,2988448515,ES -2988448516,2988448519,FR +2988448512,2988448519,FR 2988448520,2988448543,PL 2988448544,2988448559,FR 2988448560,2988448563,DE @@ -51342,11 +55444,13 @@ 2988449760,2988449791,CH 2988449792,2988451839,FR 2988451840,2988453887,BE -2988453888,2988457987,GB +2988453888,2988457983,GB +2988457984,2988457987,FR 2988457988,2988457991,PL 2988457992,2988458031,FR 2988458032,2988458047,IT -2988458048,2988458063,FR +2988458048,2988458055,FR +2988458056,2988458063,CH 2988458064,2988458067,ES 2988458068,2988458075,FR 2988458076,2988458111,PL @@ -51357,7 +55461,8 @@ 2988458296,2988458299,GB 2988458300,2988458319,ES 2988458320,2988458323,CZ -2988458324,2988458335,FR +2988458324,2988458331,FR +2988458332,2988458335,GB 2988458336,2988458367,DE 2988458368,2988458399,FR 2988458400,2988458431,PL @@ -51378,8 +55483,7 @@ 2988459080,2988459083,DE 2988459084,2988459087,FR 2988459088,2988459103,GB -2988459104,2988459107,FR -2988459108,2988459111,IT +2988459104,2988459111,FR 2988459112,2988459119,ES 2988459120,2988459127,IT 2988459128,2988459135,PL @@ -51400,10 +55504,10 @@ 2988459632,2988459639,FR 2988459640,2988459643,IT 2988459644,2988459647,CH -2988459648,2988459711,NL +2988459648,2988459711,FR 2988459712,2988459715,ES -2988459716,2988459719,FR -2988459720,2988459727,GB +2988459716,2988459723,FR +2988459724,2988459727,DE 2988459728,2988459731,FR 2988459732,2988459735,BE 2988459736,2988459743,PL @@ -51416,13 +55520,12 @@ 2988460104,2988460107,PL 2988460108,2988460111,DE 2988460112,2988460119,FR -2988460120,2988460123,ES +2988460120,2988460123,DE 2988460124,2988460127,FR 2988460128,2988460131,PL 2988460132,2988460135,CZ -2988460136,2988460147,FR -2988460148,2988460151,GB -2988460152,2988460159,ES +2988460136,2988460143,FR +2988460144,2988460159,ES 2988460160,2988460191,GB 2988460192,2988460195,FR 2988460196,2988460199,CZ @@ -51436,8 +55539,8 @@ 2988460272,2988460275,FR 2988460276,2988460279,NL 2988460280,2988460287,PL -2988460288,2988460319,FR -2988460320,2988460335,GB +2988460288,2988460323,FR +2988460324,2988460335,PL 2988460336,2988460351,DE 2988460352,2988460375,GB 2988460376,2988460543,FR @@ -51448,11 +55551,11 @@ 2988460592,2988460607,GB 2988460608,2988460683,FR 2988460684,2988460687,DE -2988460688,2988460703,FR -2988460704,2988460711,CH -2988460712,2988460719,FR +2988460688,2988460719,FR 2988460720,2988460735,ES -2988460736,2988460767,FR +2988460736,2988460759,FR +2988460760,2988460763,PT +2988460764,2988460767,FR 2988460768,2988460799,PL 2988460800,2988460863,CH 2988460864,2988460927,FR @@ -51487,11 +55590,12 @@ 2988461476,2988461479,IE 2988461480,2988461483,CZ 2988461484,2988461487,LT -2988461488,2988461495,FR +2988461488,2988461491,FR +2988461492,2988461495,PL 2988461496,2988461499,ES 2988461500,2988461503,BE 2988461504,2988461519,PL -2988461520,2988461523,ES +2988461520,2988461523,GB 2988461524,2988461559,FR 2988461560,2988461567,GB 2988461568,2988461599,FR @@ -51527,9 +55631,7 @@ 2988461936,2988461951,GB 2988461952,2988462079,PL 2988462080,2988462095,CZ -2988462096,2988462111,FR -2988462112,2988462119,PL -2988462120,2988462127,FR +2988462096,2988462127,FR 2988462128,2988462131,ES 2988462132,2988462143,FR 2988462144,2988462151,PL @@ -51549,16 +55651,14 @@ 2988462464,2988462495,GB 2988462496,2988462527,DE 2988462528,2988462559,NL -2988462560,2988462575,FR -2988462576,2988462583,PL -2988462584,2988462587,FR +2988462560,2988462587,FR 2988462588,2988462591,DE 2988462592,2988462599,FR 2988462600,2988462603,IT 2988462604,2988462607,ES 2988462608,2988462735,FR -2988462736,2988462751,BE -2988462752,2988462767,FR +2988462736,2988462743,IT +2988462744,2988462767,FR 2988462768,2988462775,GB 2988462776,2988462779,PL 2988462780,2988462783,FR @@ -51572,7 +55672,11 @@ 2988463120,2988463127,GB 2988463128,2988463131,FR 2988463132,2988463135,PL -2988463136,2988463167,IT +2988463136,2988463143,NL +2988463144,2988463147,FR +2988463148,2988463151,ES +2988463152,2988463159,FR +2988463160,2988463167,ES 2988463168,2988463199,NL 2988463200,2988463203,GB 2988463204,2988463207,DE @@ -51654,10 +55758,12 @@ 2988464956,2988464959,GB 2988464960,2988465215,FR 2988465216,2988465223,ES -2988465224,2988465279,FR +2988465224,2988465235,FR +2988465236,2988465239,ES +2988465240,2988465279,FR 2988465280,2988465295,DE -2988465296,2988465303,GB -2988465304,2988465343,FR +2988465296,2988465299,IT +2988465300,2988465343,FR 2988465344,2988465363,DE 2988465364,2988465371,PL 2988465372,2988465375,FR @@ -51707,7 +55813,9 @@ 2988507204,2988507207,PL 2988507208,2988507211,ES 2988507212,2988507215,PL -2988507216,2988507243,FR +2988507216,2988507231,FR +2988507232,2988507239,ES +2988507240,2988507243,FR 2988507244,2988507247,ES 2988507248,2988507263,FR 2988507264,2988507279,PL @@ -51717,10 +55825,19 @@ 2988507336,2988507423,FR 2988507424,2988507431,PL 2988507432,2988507439,DE -2988507440,2988507455,BE -2988507456,2988507471,IT -2988507472,2988507487,DE -2988507488,2988507519,FR +2988507440,2988507443,IT +2988507444,2988507447,NL +2988507448,2988507451,PL +2988507452,2988507459,FR +2988507460,2988507463,BE +2988507464,2988507467,CH +2988507468,2988507471,ES +2988507472,2988507475,FR +2988507476,2988507479,GB +2988507480,2988507483,IE +2988507484,2988507487,IT +2988507488,2988507503,FR +2988507504,2988507519,IT 2988507520,2988507527,PL 2988507528,2988507531,DE 2988507532,2988507535,PL @@ -51737,10 +55854,11 @@ 2988507608,2988507611,ES 2988507612,2988507615,DE 2988507616,2988507623,IT -2988507624,2988507627,FR +2988507624,2988507627,GB 2988507628,2988507631,PT 2988507632,2988507639,IT -2988507640,2988507647,ES +2988507640,2988507643,FR +2988507644,2988507647,IT 2988507648,2988507711,DE 2988507712,2988507775,IT 2988507776,2988507903,FR @@ -51772,7 +55890,7 @@ 2988508160,2988508219,FR 2988508220,2988508223,DE 2988508224,2988508287,FR -2988508288,2988508303,IT +2988508288,2988508303,PL 2988508304,2988508315,FR 2988508316,2988508319,ES 2988508320,2988508343,FR @@ -51821,7 +55939,7 @@ 2988509152,2988509183,ES 2988509184,2988509279,FR 2988509280,2988509283,DE -2988509284,2988509287,ES +2988509284,2988509287,IT 2988509288,2988509291,BE 2988509292,2988509295,FR 2988509296,2988509311,DE @@ -51845,18 +55963,369 @@ 2988509504,2988509507,PL 2988509508,2988509511,FR 2988509512,2988509515,GB -2988509516,2988515327,FR +2988509516,2988509535,FR +2988509536,2988509543,ES +2988509544,2988509547,PL +2988509548,2988509551,GB +2988509552,2988509575,FR +2988509576,2988509583,PL +2988509584,2988509619,FR +2988509620,2988509623,DE +2988509624,2988509627,CZ +2988509628,2988509631,IT +2988509632,2988509711,FR +2988509712,2988509715,IE +2988509716,2988509719,GB +2988509720,2988509727,IT +2988509728,2988509743,FR +2988509744,2988509751,ES +2988509752,2988509759,DE +2988509760,2988509767,PL +2988509768,2988509775,IT +2988509776,2988509779,DE +2988509780,2988509783,GB +2988509784,2988509787,FR +2988509788,2988509791,ES +2988509792,2988509823,NL +2988509824,2988509839,FR +2988509840,2988509855,ES +2988509856,2988509859,FR +2988509860,2988509871,DE +2988509872,2988509903,FR +2988509904,2988509907,PL +2988509908,2988509919,IT +2988509920,2988509931,FR +2988509932,2988509951,PL +2988509952,2988510207,GB +2988510208,2988510211,FR +2988510212,2988510215,DE +2988510216,2988510255,FR +2988510256,2988510259,GB +2988510260,2988510263,FR +2988510264,2988510271,PL +2988510272,2988510287,FR +2988510288,2988510303,PL +2988510304,2988510307,ES +2988510308,2988510311,FR +2988510312,2988510319,BE +2988510320,2988510323,FR +2988510324,2988510327,GB +2988510328,2988510399,FR +2988510400,2988510403,DE +2988510404,2988510407,FR +2988510408,2988510415,PL +2988510416,2988510431,FR +2988510432,2988510435,IT +2988510436,2988510751,FR +2988510752,2988510759,PL +2988510760,2988510767,FR +2988510768,2988510775,ES +2988510776,2988510847,FR +2988510848,2988510911,PL +2988510912,2988510943,FR +2988510944,2988510975,GB +2988510976,2988510991,PL +2988510992,2988511007,FR +2988511008,2988511015,PL +2988511016,2988511023,IT +2988511024,2988511071,FR +2988511072,2988511079,PL +2988511080,2988511083,FR +2988511084,2988511087,PL +2988511088,2988511103,FR +2988511104,2988511167,PL +2988511168,2988511183,FR +2988511184,2988511187,CZ +2988511188,2988511191,GB +2988511192,2988511487,FR +2988511488,2988511551,PL +2988511552,2988511559,FR +2988511560,2988511567,GB +2988511568,2988511575,FR +2988511576,2988511583,GB +2988511584,2988511663,FR +2988511664,2988511671,DE +2988511672,2988511675,FR +2988511676,2988511679,GB +2988511680,2988511683,NL +2988511684,2988511687,ES +2988511688,2988511691,DE +2988511692,2988511695,FR +2988511696,2988511699,BE +2988511700,2988511703,IT +2988511704,2988511711,PL +2988511712,2988511719,DE +2988511720,2988511723,FR +2988511724,2988511727,GB +2988511728,2988511999,FR +2988512000,2988512031,PL +2988512032,2988512055,FR +2988512056,2988512059,PL +2988512060,2988512063,FR +2988512064,2988512095,CH +2988512096,2988512127,FR +2988512128,2988512143,PL +2988512144,2988512151,GB +2988512152,2988512155,ES +2988512156,2988512159,FR +2988512160,2988512191,GB +2988512192,2988512199,FR +2988512200,2988512207,ES +2988512208,2988512239,FR +2988512240,2988512247,LT +2988512248,2988512251,GB +2988512252,2988512255,ES +2988512256,2988512287,FR +2988512288,2988512303,PL +2988512304,2988512307,IT +2988512308,2988512311,GB +2988512312,2988512315,FR +2988512316,2988512319,CH +2988512320,2988512335,FR +2988512336,2988512339,DE +2988512340,2988512343,GB +2988512344,2988512351,FR +2988512352,2988512383,IT +2988512384,2988512415,FR +2988512416,2988512435,PL +2988512436,2988512439,FR +2988512440,2988512447,NL +2988512448,2988512455,PL +2988512456,2988512459,ES +2988512460,2988512467,FR +2988512468,2988512471,GB +2988512472,2988512479,FR +2988512480,2988512483,ES +2988512484,2988512523,FR +2988512524,2988512527,IT +2988512528,2988512543,FR +2988512544,2988512551,PL +2988512552,2988512579,FR +2988512580,2988512583,ES +2988512584,2988512587,FR +2988512588,2988512591,PL +2988512592,2988512595,BE +2988512596,2988512599,ES +2988512600,2988512607,PT +2988512608,2988512639,FR +2988512640,2988512647,GB +2988512648,2988512651,ES +2988512652,2988512655,DE +2988512656,2988512663,PT +2988512664,2988512667,FR +2988512668,2988512671,ES +2988512672,2988512687,FR +2988512688,2988512703,PL +2988512704,2988512767,FR +2988512768,2988512831,PL +2988512832,2988512835,FR +2988512836,2988512839,ES +2988512840,2988512847,DE +2988512848,2988512879,FR +2988512880,2988512895,DE +2988512896,2988512899,PL +2988512900,2988512903,GB +2988512904,2988512911,ES +2988512912,2988512919,FR +2988512920,2988512927,CH +2988512928,2988512943,FR +2988512944,2988512951,GB +2988512952,2988512955,FR +2988512956,2988512959,LT +2988512960,2988512975,BE +2988512976,2988512995,FR +2988512996,2988512999,GB +2988513000,2988513003,FR +2988513004,2988513007,PL +2988513008,2988513015,IT +2988513016,2988513019,GB +2988513020,2988513023,NL +2988513024,2988513151,FR +2988513152,2988513183,ES +2988513184,2988513191,DE +2988513192,2988513195,FR +2988513196,2988513199,ES +2988513200,2988513207,PT +2988513208,2988513219,FR +2988513220,2988513223,DE +2988513224,2988513227,PT +2988513228,2988513271,FR +2988513272,2988513275,DE +2988513276,2988513279,CH +2988513280,2988513283,FR +2988513284,2988513287,ES +2988513288,2988513307,FR +2988513308,2988513311,PL +2988513312,2988513327,PT +2988513328,2988513331,ES +2988513332,2988513335,FR +2988513336,2988513343,GB +2988513344,2988513351,ES +2988513352,2988513359,IT +2988513360,2988513375,GB +2988513376,2988513391,IT +2988513392,2988513407,GB +2988513408,2988513471,PL +2988513472,2988513503,FR +2988513504,2988513507,CH +2988513508,2988513511,DE +2988513512,2988513515,FR +2988513516,2988513519,DE +2988513520,2988513535,GB +2988513536,2988513551,DE +2988513552,2988513555,IT +2988513556,2988513567,FR +2988513568,2988513575,GB +2988513576,2988513583,PL +2988513584,2988513599,IE +2988513600,2988513671,FR +2988513672,2988513675,ES +2988513676,2988513679,FR +2988513680,2988513683,ES +2988513684,2988513691,FR +2988513692,2988513695,DE +2988513696,2988513723,FR +2988513724,2988513727,DE +2988513728,2988513731,FR +2988513732,2988513735,BE +2988513736,2988513743,DE +2988513744,2988513759,GB +2988513760,2988513879,FR +2988513880,2988513883,ES +2988513884,2988513895,FR +2988513896,2988513899,IT +2988513900,2988513903,PL +2988513904,2988513951,FR +2988513952,2988513983,PT +2988513984,2988514015,GB +2988514016,2988514019,FR +2988514020,2988514023,NL +2988514024,2988514031,IE +2988514032,2988514111,FR +2988514112,2988514115,DE +2988514116,2988514119,FR +2988514120,2988514127,PT +2988514128,2988514131,IE +2988514132,2988514135,IT +2988514136,2988514159,FR +2988514160,2988514163,IT +2988514164,2988514167,DE +2988514168,2988514171,FR +2988514172,2988514175,DE +2988514176,2988514207,FR +2988514208,2988514211,ES +2988514212,2988514219,FR +2988514220,2988514223,GB +2988514224,2988514239,PL +2988514240,2988514243,FR +2988514244,2988514251,GB +2988514252,2988514255,FR +2988514256,2988514271,ES +2988514272,2988514279,FR +2988514280,2988514283,PL +2988514284,2988514287,GB +2988514288,2988514303,ES +2988514304,2988514335,FR +2988514336,2988514339,IE +2988514340,2988514343,FR +2988514344,2988514351,GB +2988514352,2988514399,FR +2988514400,2988514431,GB +2988514432,2988514447,FR +2988514448,2988514463,ES +2988514464,2988514527,FR +2988514528,2988514559,GB +2988514560,2988514623,FR +2988514624,2988514655,DE +2988514656,2988514671,FR +2988514672,2988514675,GB +2988514676,2988514679,IT +2988514680,2988514687,FR +2988514688,2988514719,GB +2988514720,2988514735,DE +2988514736,2988514739,FR +2988514740,2988514743,ES +2988514744,2988514747,BE +2988514748,2988514751,IT +2988514752,2988514815,FI +2988514816,2988514831,FR +2988514832,2988514839,PL +2988514840,2988514879,ES +2988514880,2988514943,PL +2988514944,2988514959,ES +2988514960,2988514975,PL +2988514976,2988514979,BE +2988514980,2988514983,PL +2988514984,2988514991,IT +2988514992,2988514999,FR +2988515000,2988515007,PL +2988515008,2988515039,FR +2988515040,2988515071,GB +2988515072,2988515327,FR 2988515328,2988517375,DE -2988517376,2988572671,FR +2988517376,2988519423,FR +2988519424,2988521471,PL +2988521472,2988539935,FR +2988539936,2988539967,GB +2988539968,2988539971,ES +2988539972,2988539975,IT +2988539976,2988539983,DE +2988539984,2988540003,PL +2988540004,2988540007,CZ +2988540008,2988540023,FR +2988540024,2988540027,PL +2988540028,2988540031,ES +2988540032,2988540207,FR +2988540208,2988540211,GB +2988540212,2988540215,DE +2988540216,2988540219,FR +2988540220,2988540223,GB +2988540224,2988540255,BE +2988540256,2988540271,PL +2988540272,2988540287,FR +2988540288,2988540303,BE +2988540304,2988540319,FR +2988540320,2988540351,PT +2988540352,2988540363,FR +2988540364,2988540367,PL +2988540368,2988540375,FR +2988540376,2988540379,GB +2988540380,2988540423,FR +2988540424,2988540427,GB +2988540428,2988540431,NL +2988540432,2988540435,FR +2988540436,2988540439,DE +2988540440,2988540447,PL +2988540448,2988540451,IT +2988540452,2988540455,FR +2988540456,2988540459,DE +2988540460,2988540463,ES +2988540464,2988540471,PL +2988540472,2988540479,DE +2988540480,2988540483,FR +2988540484,2988540487,DE +2988540488,2988540491,ES +2988540492,2988540503,FR +2988540504,2988540507,DE +2988540508,2988540511,ES +2988540512,2988540543,FR +2988540544,2988540607,PL +2988540608,2988540639,FR +2988540640,2988540647,PL +2988540648,2988540651,GB +2988540652,2988540663,PL +2988540664,2988540671,IT +2988540672,2988540679,PL +2988540680,2988540683,IT +2988540684,2988572671,FR 2988572672,2988703743,RU 2988703744,2988834815,PL 2988834816,2988965887,CH 2988965888,2989096959,SK 2989096960,2989228031,PL 2989228032,2989490175,RU -2989490176,2989555711,UA 2989555712,2989621247,RU -2989621248,2989686783,BE +2989621248,2989752319,BE 2989752320,2989817855,SY 2989817856,2989883391,KW 2989883392,2989916159,UA @@ -51868,15 +56337,18 @@ 2990276608,2990342143,ES 2990342144,2990407679,KW 2990407680,2990473215,RU -2990473216,2990534655,DE +2990473216,2990518015,DE +2990518016,2990518079,IT +2990518080,2990525247,DE +2990525248,2990525311,ES +2990525312,2990534655,DE 2990534656,2990534783,EG 2990534784,2990535935,DE 2990535936,2990535967,PL 2990535968,2990538751,DE 2990538752,2991063039,RU 2991063040,2991067135,SE -2991067136,2991068159,DK -2991068160,2991071231,SE +2991067136,2991071231,DK 2991071232,2991079423,NO 2991079424,2991095807,RU 2991095808,2991112191,AM @@ -51884,7 +56356,56 @@ 2991128576,2991144959,PL 2991144960,2991161343,SA 2991161344,2991177727,FR -2991177728,2991194111,SE +2991177728,2991185919,SE +2991185920,2991185951,GB +2991185952,2991186175,SE +2991186176,2991186207,GB +2991186208,2991186431,SE +2991186432,2991186463,GB +2991186464,2991186687,SE +2991186688,2991186719,GB +2991186720,2991186943,SE +2991186944,2991186975,GB +2991186976,2991187199,SE +2991187200,2991187231,GB +2991187232,2991187455,SE +2991187456,2991187487,GB +2991187488,2991187711,SE +2991187712,2991187743,GB +2991187744,2991187967,SE +2991187968,2991187999,GB +2991188000,2991188223,SE +2991188224,2991188255,GB +2991188256,2991188479,SE +2991188480,2991188511,GB +2991188512,2991188735,SE +2991188736,2991188767,GB +2991188768,2991188991,SE +2991188992,2991189023,GB +2991189024,2991189247,SE +2991189248,2991189279,GB +2991189280,2991189503,SE +2991189504,2991189535,GB +2991189536,2991189759,SE +2991189760,2991189791,GB +2991189792,2991190015,SE +2991190016,2991190047,GB +2991190048,2991190271,SE +2991190272,2991190303,GB +2991190304,2991190527,SE +2991190528,2991190559,GB +2991190560,2991190783,SE +2991190784,2991190815,GB +2991190816,2991191039,SE +2991191040,2991191071,GB +2991191072,2991191295,SE +2991191296,2991191327,GB +2991191328,2991191551,SE +2991191552,2991191583,GB +2991191584,2991191807,SE +2991191808,2991192063,FI +2991192064,2991192319,DK +2991192320,2991194111,SE 2991194112,2991210495,NO 2991210496,2991243263,RU 2991243264,2991259647,UA @@ -51905,14 +56426,16 @@ 2991505408,2991521791,SE 2991521792,2991538175,RS 2991538176,2991554559,SI -2991554560,2991570943,GB -2991570944,2991587327,EU +2991554560,2991571455,GB +2991571456,2991587327,IT 2991587328,2991718399,RU 2991718400,2991849471,CH 2991849472,2991980543,NL 2991980544,2991980807,UA 2991980808,2991980815,NA -2991980816,2991981375,UA +2991980816,2991981311,UA +2991981312,2991981315,RU +2991981316,2991981375,UA 2991981376,2991981383,NA 2991981384,2991981415,UA 2991981416,2991981423,NA @@ -51930,7 +56453,13 @@ 2991981648,2991981655,NA 2991981656,2991981687,UA 2991981688,2991981695,NA -2991981696,2992111615,UA +2991981696,2991981911,UA +2991981912,2991981919,RU +2991981920,2991981935,UA +2991981936,2991981943,RU +2991981944,2991982535,UA +2991982536,2991982543,GL +2991982544,2992111615,UA 2992111616,2992373759,KZ 2992373760,2992635903,UA 2992635904,2993684479,GB @@ -51955,8 +56484,16 @@ 2996174848,2996305919,UA 2996305920,2996436991,QA 2996436992,2996469759,BY -2996469760,2996535295,RU +2996469760,2996479487,RU +2996479488,2996479491,US +2996479492,2996535295,RU 2996535296,2996568063,DK +2996568064,2996600831,ES +2996600832,2996633599,RO +2996633600,2996649983,IR +2996649984,2996666367,RO +2996666368,2996682751,RU +2996682752,2996699135,DK 2996830208,2996862975,HR 2996862976,2996895743,AM 2996895744,2996928511,KW @@ -51973,7 +56510,9 @@ 2996995712,2996995775,BY 2996995776,2996995839,RU 2996995840,2996996095,UA -2996996096,2996996287,DE +2996996096,2996996127,DE +2996996128,2996996159,CA +2996996160,2996996287,DE 2996996288,2996996351,RU 2996996352,2996996383,DE 2996996384,2996996447,RU @@ -51987,19 +56526,25 @@ 2996997376,2996997631,US 2996997632,2996998143,BY 2996998144,2996998271,UA -2996998272,2996998399,US -2996998400,2996998463,DE +2996998272,2996998463,DE 2996998464,2996998527,US 2996998528,2996998655,UA -2996998656,2996998783,DE -2996998784,2996998910,RU -2996998911,2997001727,DE +2996998656,2996998719,DE +2996998720,2996998911,RU +2996998912,2996999231,DE +2996999232,2996999359,IN +2996999360,2996999679,DE +2996999680,2996999935,GB +2996999936,2997000063,DE +2997000064,2997000191,CN +2997000192,2997001727,DE 2997001728,2997001983,UA 2997001984,2997003071,DE 2997003072,2997003135,RU -2997003136,2997003199,NL +2997003136,2997003199,US 2997003200,2997003263,RU -2997003264,2997003327,DE +2997003264,2997003295,DE +2997003296,2997003327,CA 2997003328,2997003391,RU 2997003392,2997003583,DE 2997003584,2997003647,UA @@ -52010,9 +56555,24 @@ 2997004608,2997004799,RU 2997004800,2997005055,BZ 2997005056,2997005183,GB -2997005184,2997005439,DE -2997005440,2997006847,GB -2997006848,2997026815,DE +2997005184,2997005375,DE +2997005376,2997005439,GR +2997005440,2997005567,GB +2997005568,2997005823,BZ +2997005824,2997008191,DE +2997008192,2997008255,GR +2997008256,2997008383,LT +2997008384,2997008447,DE +2997008448,2997008511,RU +2997008512,2997008639,LT +2997008640,2997008895,TR +2997008896,2997008959,DE +2997008960,2997009023,RU +2997009024,2997009663,DE +2997009664,2997009919,US +2997009920,2997010431,DE +2997010432,2997018623,US +2997018624,2997026815,DE 2997026816,2997059583,RU 2997059584,2997092351,BY 2997092352,2997125119,NO @@ -52027,7 +56587,9 @@ 2997420032,2997452799,RU 2997452800,2997485567,BG 2997485568,2997518335,FR -2997518336,2997583871,RU +2997518336,2997528063,RU +2997528064,2997528319,UA +2997528320,2997583871,RU 2997583872,2997616639,SY 2997616640,2997649407,SI 2997649408,2997682175,BY @@ -52045,8 +56607,8 @@ 2999451648,2999713791,DE 2999713792,2999975935,RU 2999975936,2999984127,FR -2999984128,2999985151,BE -2999985152,2999992319,NL +2999984128,2999985663,BE +2999985664,2999992319,NL 2999992320,3000000511,RU 3000000512,3000008703,DE 3000008704,3000016895,RU @@ -52057,6 +56619,43 @@ 3000049664,3000057855,CH 3000057856,3000066047,UA 3000066048,3000074239,RU +3000074240,3000082431,CZ +3000082432,3000090623,DK +3000090624,3000131583,RU +3000131584,3000139775,DE +3000139776,3000147967,UA +3000147968,3000156159,TR +3000156160,3000164351,UA +3000164352,3000172543,HU +3000172544,3000180735,RU +3000180736,3000188927,TR +3000188928,3000197119,IT +3000197120,3000213503,UA +3000213504,3000221695,RU +3000221696,3000229887,TR +3000238080,3000240127,KZ +3000240128,3000242175,RU +3000242176,3000244223,AT +3000244224,3000246271,PL +3000246272,3000248319,RO +3000248320,3000252415,PL +3000252416,3000254463,RO +3000254464,3000256511,UA +3000256512,3000260607,RU +3000260608,3000262655,RS +3000262656,3000264703,UA +3000369152,3000377343,PL +3000377344,3000385535,RU +3000385536,3000393727,PS +3000393728,3000401919,NL +3000434688,3000451071,IR +3000451072,3000467455,RU +3000467456,3000471551,GB +3000471552,3000475647,DK +3000475648,3000483839,UA +3000483840,3000487935,PL +3000487936,3000492031,PS +3000492032,3000496127,UA 3000500224,3000506367,UA 3000506368,3000508415,PL 3000508416,3000510463,UA @@ -52093,6 +56692,21 @@ 3000588288,3000590335,RO 3000590336,3000594431,RU 3000594432,3000596479,PL +3000596480,3000598527,RU +3000598528,3000600575,FI +3000600576,3000602623,PL +3000602624,3000604671,CZ +3000604672,3000606719,RU +3000606720,3000608767,UA +3000608768,3000610815,KG +3000610816,3000612863,RU +3000612864,3000616959,PL +3000616960,3000621055,UA +3000621056,3000623103,PL +3000623104,3000625151,RU +3000625152,3000627199,RO +3000627200,3000629247,CZ +3000629248,3000631295,PL 3000631296,3000647679,RU 3000647680,3000651775,UA 3000651776,3000664063,RU @@ -52112,6 +56726,13 @@ 3000717312,3000721407,UA 3000721408,3000729599,PL 3000729600,3000733695,NL +3000733696,3000737791,RU +3000737792,3000741887,UA +3000741888,3000745983,RU +3000745984,3000750079,UA +3000750080,3000754175,RU +3000754176,3000758271,IR +3000758272,3000762367,UA 3000762368,3001024511,RS 3001024512,3001548799,NL 3001548800,3001614335,NO @@ -52121,6 +56742,52 @@ 3001810944,3001815039,IE 3001815040,3001819135,RU 3001819136,3001823231,IR +3001823232,3001827327,GE +3001827328,3001831423,SE +3001831424,3001835519,BA +3001835520,3001839615,RU +3001839616,3001843711,ES +3001843712,3001845759,RU +3001845760,3001851903,GB +3001851904,3001855999,IT +3001856000,3001858559,NL +3001858560,3001859071,RO +3001859072,3001859327,MD +3001859328,3001860095,NL +3001860096,3001864191,EE +3001864192,3001868287,RU +3001868288,3001872383,FR +3001872384,3001876479,RU +3001876480,3001880575,IT +3001880576,3001884671,RU +3001884672,3001888767,NL +3001888768,3001892863,BH +3001892864,3001896959,AZ +3001896960,3001901055,CH +3001901056,3001905151,FR +3001905152,3001909247,GB +3001909248,3001917439,ES +3001917440,3001921535,GB +3001921536,3001929727,RU +3001929728,3001933823,RS +3001933824,3001937919,ES +3001937920,3001942015,RU +3001942016,3001946111,BY +3001946112,3001950207,RU +3001950208,3001954303,CZ +3001954304,3001958399,IT +3001958400,3001962495,KZ +3001962496,3001966591,GB +3001966592,3001970687,NL +3001970688,3001974783,RU +3001974784,3001982975,GB +3001982976,3001987071,CH +3001987072,3001991167,JO +3001991168,3001995263,IR +3001995264,3001999359,HU +3001999360,3002003455,DE +3002003456,3002011647,PL +3002011648,3002015743,BA 3002073088,3002597375,TR 3002597376,3002599423,RU 3002599424,3002601471,CZ @@ -52134,7 +56801,11 @@ 3002615808,3002617855,PL 3002617856,3002619903,RU 3002619904,3002619911,GB -3002619912,3002621951,IT +3002619912,3002619951,IT +3002619952,3002619967,GB +3002619968,3002619983,IT +3002619984,3002619991,GB +3002619992,3002621951,IT 3002621952,3002623999,NO 3002624000,3002628095,GB 3002628096,3002630143,NO @@ -52183,9 +56854,109 @@ 3002724352,3002726399,RU 3002726400,3002728447,NL 3002728448,3002730495,FR +3002730496,3002732543,DE +3002732544,3002734591,NL +3002734592,3002736639,CH +3002736640,3002738687,FR +3002738688,3002740735,PL +3002740736,3002742783,GB +3002742784,3002744831,LI +3002744832,3002746879,IT +3002746880,3002748927,GB +3002748928,3002750975,DE +3002750976,3002753023,TR +3002753024,3002755071,GB +3002755072,3002757119,FR +3002757120,3002759167,IE +3002759168,3002761215,DE +3002761216,3002763263,CH +3002763264,3002765311,RS +3002765312,3002767359,NL +3002767360,3002769407,RU +3002769408,3002773503,DE +3002773504,3002775551,GB +3002775552,3002777599,RO +3002777600,3002779647,NL +3002779648,3002781695,IT +3002781696,3002783743,FR +3002783744,3002785791,NO +3002785792,3002789887,DE +3002789888,3002791935,RU +3002791936,3002793983,AE +3002793984,3002796031,DK +3002796032,3002798079,DE +3002798080,3002800127,FR +3002800128,3002802175,NL +3002802176,3002804223,GB +3002804224,3002806271,TR +3002806272,3002808319,DE +3002808320,3002810367,FI +3002810368,3002812415,AT +3002812416,3002814463,IT +3002814464,3002816511,FR +3002816512,3002818559,DE +3002818560,3002820607,RU +3002820608,3002822655,UA +3002822656,3002824703,DE +3002824704,3002826751,NL +3002826752,3002828799,SE +3002828800,3002830847,RU +3002830848,3002834943,FI +3002834944,3002836991,LU +3002836992,3002841087,GB +3002841088,3002843135,CZ +3002843136,3002845183,NL +3002845184,3002847231,FR +3002847232,3002849279,IR +3002849280,3002851327,RU +3002851328,3002853375,DE +3002853376,3002855423,GB +3002855424,3002857471,SE +3002857472,3002859519,FR 3002859520,3002875903,PL 3002875904,3002892287,RU 3002892288,3002908671,IR +3002908672,3002925055,RU +3002925056,3002941439,IR +3002941440,3002957823,SY +3002957824,3002974207,SK +3002974208,3002990591,RS +3002990592,3003006975,DE +3003006976,3003023359,LU +3003023360,3003039743,RS +3003039744,3003056127,BG +3003056128,3003058175,DE +3003058176,3003058431,MT +3003058432,3003058687,FI +3003058688,3003060223,EE +3003060224,3003062271,DE +3003062272,3003064319,NL +3003064320,3003066367,RO +3003066368,3003068415,PL +3003068416,3003070463,NL +3003070464,3003074559,GB +3003074560,3003076607,IT +3003076608,3003080703,GB +3003080704,3003082751,FR +3003082752,3003084799,ES +3003084800,3003086847,IT +3003086848,3003088895,RU +3003088896,3003090943,FR +3003090944,3003092991,DE +3003092992,3003095039,NO +3003095040,3003097087,AT +3003097088,3003099135,FR +3003099136,3003101183,CZ +3003101184,3003103231,UA +3003103232,3003105279,IT +3003105280,3003107327,NL +3003107328,3003109375,RU +3003109376,3003111423,FR +3003111424,3003113471,IT +3003113472,3003115519,ES +3003115520,3003117567,RU +3003117568,3003119615,IT +3003119616,3003121663,RU 3019898880,3024093183,JP 3024093184,3024617471,KR 3024617472,3024879615,MY @@ -52311,8 +57082,14 @@ 3032334336,3032342527,NZ 3032350720,3032743935,PH 3032743936,3033038847,JP -3033038848,3033071615,AU -3033071616,3033268223,CN +3033038848,3033063423,AU +3033063424,3033065471,JP +3033065472,3033066495,MY +3033066496,3033067519,BD +3033067520,3033068543,IN +3033068544,3033069567,AU +3033069568,3033070591,JP +3033070592,3033268223,CN 3033268224,3033530367,TW 3033530368,3033661439,CN 3033661440,3033694207,KR @@ -52610,30 +57387,45 @@ 3091726336,3093168127,US 3093168128,3093200895,CA 3093233664,3093237759,PR -3093299200,3093856255,US -3093954560,3094020095,US +3093299200,3094020095,US 3094020096,3094085631,CA 3094085632,3096444927,US 3096444928,3096969215,CA +3096969216,3097493503,US +3097493504,3097755647,CA +3097755648,3098017791,US 3098542080,3103784959,US -3120562176,3120578559,CO +3120562176,3120594943,CO 3120594944,3120599039,AR +3120599040,3120601087,EC +3120601088,3120602111,AR +3120602112,3120603135,BO +3120603136,3120608255,AR +3120609280,3120610303,AR +3120610304,3120611327,PY +3120611328,3120627711,AR 3120627712,3120644095,NI +3120660480,3120668671,PA +3120676864,3120680959,HT 3120693248,3120709631,BO +3120726016,3120734207,HN 3120758784,3120824319,EC 3120824320,3120840703,CR 3120857088,3120889855,EC +3120955392,3121020927,DO 3121086464,3121151999,UY -3121152000,3121283071,CL +3121152000,3121348607,CL 3121348608,3121479679,AR 3121479680,3121545215,VE 3121545216,3121610751,CR 3121610752,3121741823,PY 3121741824,3121872895,AR +3121872896,3121938431,CL 3122135040,3122397183,VE 3122397184,3122659327,CO 3122659328,3122724863,GT -3122921472,3122987007,CL +3122790400,3122987007,CL +3123052544,3123118079,AR 3123183616,3123216383,CL 3123314688,3123380223,EC 3123380224,3123412991,CO @@ -52649,21 +57441,19 @@ 3126853632,3126919167,AR 3126919168,3127115775,CO 3127115776,3127181311,AR +3127246848,3127377919,CO 3127377920,3127508991,CL 3127640064,3127902207,AR 3127902208,3128164351,CO 3128426496,3128492031,DO 3128492032,3128524799,CO -3128557568,3128623103,AR -3128688640,3128950783,AR +3128557568,3128950783,AR 3128950784,3129016319,UY -3129016320,3129212927,AR -3129475072,3129999359,AR +3129016320,3129999359,AR 3129999360,3130261503,CO 3130523648,3130654719,AR 3131047936,3131310079,PE -3133145088,3135242239,BR -3137339392,3145727999,BR +3133145088,3145727999,BR 3145728000,3154116607,MX 3154116608,3154182143,EU 3154182144,3154247679,DE @@ -52697,7 +57487,9 @@ 3156771221,3156771221,FR 3156771222,3156791439,DE 3156791440,3156791455,NL -3156791456,3156803583,DE +3156791456,3156802271,DE +3156802272,3156802303,US +3156802304,3156803583,DE 3156803584,3156869119,TR 3156869120,3156934655,LU 3156934656,3157065727,RU @@ -52885,18 +57677,11 @@ 3160188928,3160190975,SE 3160190976,3160193023,RU 3160193024,3160193279,IT -3160193280,3160193535,US -3160193536,3160193791,IT -3160193792,3160193807,US +3160193280,3160193807,A2 3160193808,3160193871,LY -3160193872,3160194047,IT +3160193872,3160194047,A2 3160194048,3160194175,LY -3160194176,3160194463,IT -3160194464,3160194527,A2 -3160194528,3160194623,IT -3160194624,3160194751,A2 -3160194752,3160194815,IT -3160194816,3160195071,A2 +3160194176,3160195071,A2 3160195072,3160197119,RU 3160197120,3160199167,DE 3160199168,3160201215,GE @@ -52936,11 +57721,11 @@ 3160264704,3160266751,RU 3160266752,3160268799,DE 3160268800,3160271615,PT -3160271616,3160271871,ES -3160271872,3160272895,PT +3160271616,3160271935,ES +3160271936,3160271951,PT +3160271952,3160272895,ES 3160272896,3160274943,RU 3160274944,3160276991,AT -3160276992,3160279039,RU 3160279040,3160281087,ES 3160281088,3160283135,GB 3160283136,3160285183,DE @@ -53112,11 +57897,7 @@ 3162701824,3162734591,MD 3162734592,3162767359,RU 3162767360,3162800127,SA -3162800128,3162802175,TR -3162802176,3162803199,CZ -3162803200,3162821631,TR -3162821632,3162821887,CZ -3162821888,3162832895,TR +3162800128,3162832895,TR 3162832896,3162865663,GR 3162865664,3162931199,RU 3162931200,3162963967,CZ @@ -53135,25 +57916,28 @@ 3163162112,3163162143,TR 3163162144,3163162175,DE 3163162176,3163162207,US -3163162208,3163162239,DE +3163162208,3163162239,IT 3163162240,3163162271,CH 3163162272,3163162303,NL 3163162304,3163162495,DE 3163162496,3163162527,RU 3163162528,3163163743,DE 3163163744,3163163775,GB -3163163776,3163164543,DE +3163163776,3163163871,DE +3163163872,3163163903,RU +3163163904,3163163935,DE +3163163936,3163163967,VG +3163163968,3163164543,DE 3163164544,3163164575,RU -3163164576,3163165727,DE -3163165728,3163165759,US +3163164576,3163165695,DE +3163165696,3163165727,TR +3163165728,3163165759,DE 3163165760,3163165791,PL -3163165792,3163166111,DE -3163166112,3163166143,PT -3163166144,3163166175,DE +3163165792,3163166175,DE 3163166176,3163166207,US 3163166208,3163166495,DE 3163166496,3163166527,RO -3163166528,3163166559,TR +3163166528,3163166559,DE 3163166560,3163166591,HU 3163166592,3163167807,DE 3163167808,3163167839,TR @@ -53161,11 +57945,14 @@ 3163167968,3163167999,TR 3163168000,3163168031,DE 3163168032,3163168063,RU -3163168064,3163168127,DE +3163168064,3163168095,DE +3163168096,3163168127,TR 3163168128,3163168159,IL 3163168160,3163168319,DE 3163168320,3163168351,DK -3163168352,3163168575,DE +3163168352,3163168415,DE +3163168416,3163168447,CA +3163168448,3163168575,DE 3163168576,3163168607,TR 3163168608,3163168671,DE 3163168672,3163168703,US @@ -53173,54 +57960,55 @@ 3163169824,3163169855,RU 3163169856,3163169951,DE 3163169952,3163169983,GB -3163169984,3163170015,DE -3163170016,3163170047,TR -3163170048,3163170079,DE +3163169984,3163170079,DE 3163170080,3163170111,TR 3163170112,3163170207,DE 3163170208,3163170239,RU 3163170240,3163170303,DE 3163170304,3163170335,IT -3163170336,3163170495,DE +3163170336,3163170367,DE +3163170368,3163170399,IR +3163170400,3163170495,DE 3163170496,3163170527,BR 3163170528,3163170591,DE 3163170592,3163170623,BR -3163170624,3163170655,DE -3163170656,3163170687,DK -3163170688,3163172127,DE +3163170624,3163172127,DE 3163172128,3163172159,TR 3163172160,3163172191,RU 3163172192,3163172223,DE 3163172224,3163172255,TR 3163172256,3163172287,PL 3163172288,3163172319,TR -3163172320,3163172351,DE +3163172320,3163172351,NL 3163172352,3163172383,GB 3163172384,3163172607,DE 3163172608,3163172639,DK -3163172640,3163172671,DE -3163172672,3163172703,TR -3163172704,3163172735,DE +3163172640,3163172735,DE 3163172736,3163172767,GR 3163172768,3163174047,DE 3163174048,3163174079,DK -3163174080,3163174207,DE -3163174208,3163174239,TR -3163174240,3163174335,DE +3163174080,3163174335,DE 3163174336,3163174367,RU 3163174368,3163174399,DE 3163174400,3163174431,DK 3163174432,3163174591,DE 3163174592,3163174623,SE -3163174624,3163176127,DE +3163174624,3163174655,GB +3163174656,3163174687,DE +3163174688,3163174719,US +3163174720,3163174751,NL +3163174752,3163174783,US +3163174784,3163176127,DE 3163176128,3163176159,TR -3163176160,3163176351,DE -3163176352,3163176383,PT -3163176384,3163176575,DE +3163176160,3163176479,DE +3163176480,3163176543,GB +3163176544,3163176575,DE 3163176576,3163176607,GR 3163176608,3163176767,DE 3163176768,3163176799,PL -3163176800,3163193343,DE +3163176800,3163176895,DE +3163176896,3163176927,SE +3163176928,3163193343,DE 3163226112,3163258879,SA 3163258880,3163291647,SY 3163291648,3163324415,PT @@ -53229,7 +58017,6 @@ 3163389952,3163422719,FR 3163422720,3163455487,NL 3163455488,3163488255,HU -3163488256,3163521023,UA 3163521024,3163553791,RU 3163553792,3163684863,DE 3163684864,3163815935,PL @@ -53253,8 +58040,7 @@ 3164946440,3164946447,ES 3164946448,3164946479,FR 3164946480,3164946483,ES -3164946484,3164946487,PL -3164946488,3164946495,FR +3164946484,3164946495,PL 3164946496,3164946499,IT 3164946500,3164946503,GB 3164946504,3164946507,BE @@ -53263,8 +58049,12 @@ 3164946528,3164946559,GB 3164946560,3164946783,FR 3164946784,3164946799,BE -3164946800,3164947071,FR -3164947072,3164947199,DE +3164946800,3164947039,FR +3164947040,3164947043,ES +3164947044,3164947047,GB +3164947048,3164947063,FR +3164947064,3164947067,NL +3164947068,3164947199,DE 3164947200,3164947455,FR 3164947456,3164947711,DE 3164947712,3164947743,FR @@ -53278,7 +58068,7 @@ 3164948992,3164949087,GB 3164949088,3164949095,FR 3164949096,3164949103,IT -3164949104,3164949119,FR +3164949104,3164949119,GB 3164949120,3164949131,DE 3164949132,3164949151,FR 3164949152,3164949155,NL @@ -53309,7 +58099,8 @@ 3164950656,3164950687,FR 3164950688,3164950703,ES 3164950704,3164950719,PL -3164950720,3164950735,GB +3164950720,3164950723,GB +3164950724,3164950735,FR 3164950736,3164950751,ES 3164950752,3164950759,GB 3164950760,3164950783,ES @@ -53324,15 +58115,20 @@ 3164951520,3164951535,GB 3164951536,3164951543,BE 3164951544,3164951547,NL -3164951548,3164951551,FR -3164951552,3164951583,IT -3164951584,3164951615,NL -3164951616,3164951647,GB -3164951648,3164951679,NL -3164951680,3164951711,GB -3164951712,3164951743,FR -3164951744,3164951775,NL -3164951776,3164951807,FR +3164951548,3164951559,FR +3164951560,3164951567,PL +3164951568,3164951571,BE +3164951572,3164951575,CZ +3164951576,3164951583,NL +3164951584,3164951591,GB +3164951592,3164951595,IE +3164951596,3164951615,FR +3164951616,3164951663,GB +3164951664,3164951671,PL +3164951672,3164951679,FR +3164951680,3164951695,ES +3164951696,3164951711,IE +3164951712,3164951807,FR 3164951808,3164951823,NL 3164951824,3164951831,CH 3164951832,3164951839,FR @@ -53417,13 +58213,13 @@ 3164954520,3164954543,GB 3164954544,3164954591,FR 3164954592,3164954623,PL -3164954624,3164958847,FR +3164954624,3164956479,FR +3164956480,3164956543,GB +3164956544,3164958847,FR 3164958848,3164958911,IT 3164958912,3164958927,CH 3164958928,3164958943,ES -3164958944,3164958947,FR -3164958948,3164958951,GB -3164958952,3164958955,FR +3164958944,3164958955,FR 3164958956,3164958959,GB 3164958960,3164959039,FR 3164959040,3164959135,DE @@ -53488,26 +58284,30 @@ 3164961972,3164961975,ES 3164961976,3164961979,GB 3164961980,3164961999,DE -3164962000,3164962015,FR -3164962016,3164962047,NL +3164962000,3164962031,FR +3164962032,3164962047,NL 3164962048,3164962079,FR 3164962080,3164962111,ES 3164962112,3164962143,GB 3164962144,3164962175,FR 3164962176,3164962239,NL 3164962240,3164962251,FR -3164962252,3164962255,ES +3164962252,3164962255,DE 3164962256,3164962259,NL -3164962260,3164962263,GB +3164962260,3164962263,FR 3164962264,3164962279,ES 3164962280,3164962283,DE 3164962284,3164962295,ES 3164962296,3164962303,PL 3164962304,3164962335,NL 3164962336,3164962367,DE -3164962368,3164962431,FR -3164962432,3164962495,ES -3164962496,3164962527,FR +3164962368,3164962431,PL +3164962432,3164962439,GB +3164962440,3164962447,PL +3164962448,3164962451,PT +3164962452,3164962455,FR +3164962456,3164962459,IE +3164962460,3164962527,FR 3164962528,3164962535,PL 3164962536,3164962539,CH 3164962540,3164962543,CZ @@ -53583,23 +58383,24 @@ 3164970464,3164970495,GB 3164970496,3164970559,PL 3164970560,3164970567,FR -3164970568,3164970571,GB -3164970572,3164970575,ES +3164970568,3164970575,ES 3164970576,3164970607,FR 3164970608,3164970615,ES 3164970616,3164970623,CH -3164970624,3164970627,NL -3164970628,3164970631,FR +3164970624,3164970631,FR 3164970632,3164970639,CH 3164970640,3164970647,ES -3164970648,3164970655,FR +3164970648,3164970651,GB +3164970652,3164970655,FR 3164970656,3164970687,IT 3164970688,3164970691,GB 3164970692,3164970695,FR 3164970696,3164970719,PL 3164970720,3164970783,FR 3164970784,3164970791,PL -3164970792,3164970831,FR +3164970792,3164970823,FR +3164970824,3164970827,BE +3164970828,3164970831,CH 3164970832,3164970835,IT 3164970836,3164970839,PL 3164970840,3164970847,GB @@ -53663,7 +58464,9 @@ 3164973888,3164973919,ES 3164973920,3164973935,FR 3164973936,3164973939,IT -3164973940,3164974591,FR +3164973940,3164973943,FR +3164973944,3164973951,PL +3164973952,3164974591,FR 3164974592,3164975103,GB 3164975104,3164975167,CH 3164975168,3164975231,GB @@ -53671,7 +58474,9 @@ 3164975300,3164975307,ES 3164975308,3164975311,GB 3164975312,3164975343,ES -3164975344,3164975359,NL +3164975344,3164975351,GB +3164975352,3164975355,NL +3164975356,3164975359,FR 3164975360,3164975615,ES 3164975616,3164976127,GB 3164976128,3164976639,IT @@ -53765,9 +58570,7 @@ 3167223808,3167748095,NL 3167748096,3167940095,RO 3167940096,3167940351,CY -3167940352,3167985663,RO -3167985664,3167989759,GB -3167989760,3168092159,RO +3167940352,3168092159,RO 3168092160,3168096255,GB 3168096256,3168100351,MD 3168100352,3168104447,RO @@ -53778,8 +58581,10 @@ 3168120832,3168124927,GB 3168124928,3168129023,RO 3168129024,3168141311,GB -3168141312,3168153599,RO -3168153600,3168194559,GB +3168141312,3168157695,RO +3168157696,3168178175,GB +3168178176,3168182271,RO +3168182272,3168194559,GB 3168194560,3168198655,RO 3168198656,3168202751,GB 3168202752,3168207103,RO @@ -53888,7 +58693,7 @@ 3188047872,3188051967,CO 3188056064,3188064255,PA 3188064256,3188080639,VE -3188080640,3188084735,CU +3188080640,3188088831,CU 3188088832,3188097023,AR 3188097024,3188105215,DO 3188105216,3188113407,CO @@ -53913,7 +58718,7 @@ 3188310016,3188326399,CO 3188326400,3188359167,CR 3188359168,3188400127,EC -3188400128,3188404223,AR +3188400128,3188408319,AR 3188408320,3188416511,BO 3188416512,3188445183,AR 3188449280,3188453375,HN @@ -54061,6 +58866,9 @@ 3194511360,3194515455,AW 3194519552,3194535935,PY 3194552320,3194585087,AR +3194585088,3194589183,HN +3194601472,3194602495,AN +3194602496,3194605567,AR 3194617856,3194626047,NI 3194634240,3194638335,CR 3194642432,3194646527,AR @@ -54073,7 +58881,7 @@ 3194691584,3194695679,BO 3194699776,3194703871,EC 3194707968,3194716159,AR -3194716160,3194720255,HN +3194716160,3194724351,HN 3194724352,3194728447,PA 3194732544,3194736639,AR 3194740736,3194742783,CL @@ -54084,7 +58892,11 @@ 3194765312,3194767359,EC 3194769408,3194771455,PE 3194781696,3194798079,CL +3194798080,3194799103,AR +3194799104,3194802175,GY +3194806272,3194810367,AR 3194818560,3194822655,AR +3194830848,3194839039,PA 3194847232,3194863615,CO 3194863616,3194871807,HN 3194880000,3194896383,DO @@ -54104,7 +58916,9 @@ 3195092992,3195097087,AR 3195101184,3195105279,CR 3195109376,3195117567,AR +3195125760,3195133951,PE 3195142144,3195150335,VE +3195158528,3195162623,AR 3195174912,3195183103,AR 3195191296,3195199487,AR 3195199488,3195201535,PY @@ -54141,7 +58955,7 @@ 3195809792,3195811839,PE 3195811840,3195813887,AR 3195813888,3195822079,DO -3195830272,3195834367,AR +3195830272,3195838463,AR 3195838464,3195840511,HN 3195846656,3195852799,AR 3195852800,3195853823,AN @@ -54175,7 +58989,9 @@ 3198156800,3198484479,CL 3198484480,3198550015,CO 3198550016,3198681087,DO -3198681088,3198943231,VE +3198681088,3198877695,VE +3198877696,3198894079,CR +3198910464,3198926847,CL 3198943232,3199500799,AR 3199500800,3199501311,US 3199501312,3199533055,AR @@ -54185,6 +59001,7 @@ 3199664128,3199729663,AR 3199729664,3199762431,NI 3199795200,3199827967,AN +3199827968,3199844351,AR 3199860736,3199926271,BO 3199926272,3199991807,PE 3199991808,3200516095,AR @@ -54201,11 +59018,11 @@ 3201875968,3201880063,CO 3201880064,3201888255,AR 3201888256,3201892351,VE -3201957888,3202023423,PA +3201957888,3202088959,PA 3202088960,3202220031,AR 3202220032,3202351103,PE 3202351104,3202875391,AR -3202875392,3203137535,PE +3202875392,3203399679,PE 3203399680,3203432447,CO 3203465216,3203476479,CR 3203476480,3203476735,CO @@ -54510,7 +59327,6 @@ 3223965440,3223966207,US 3223966208,3223966463,GB 3223966464,3223967743,US -3223967744,3223967999,EU 3223968000,3223968255,US 3223968256,3223968511,IT 3223968512,3223970303,US @@ -54756,6 +59572,7 @@ 3224884480,3224884991,US 3224885248,3224885503,CA 3224885760,3224886015,US +3224886016,3224886271,AU 3224886272,3224886527,JP 3224886528,3224887295,US 3224887296,3224887551,CA @@ -55092,7 +59909,11 @@ 3226376704,3226376959,AT 3226376960,3226384639,US 3226384640,3226385407,BE -3226385408,3226468351,US +3226385408,3226393599,US +3226393600,3226393855,DE +3226393856,3226397695,US +3226397696,3226400255,DE +3226400256,3226468351,US 3226470400,3226473471,US 3226473472,3226473983,PT 3226473984,3226474495,US @@ -58038,7 +62859,6 @@ 3239173632,3239173887,AT 3239173888,3239174143,PL 3239174144,3239174399,DE -3239174400,3239174655,FR 3239174656,3239174911,GB 3239174912,3239175167,SI 3239175168,3239175423,UA @@ -58276,6 +63096,7 @@ 3239670272,3239670783,DE 3239670784,3239671295,GB 3239671296,3239671807,DE +3239671808,3239672319,UA 3239672320,3239672831,DE 3239672832,3239673343,TR 3239673344,3239673855,SE @@ -58423,7 +63244,6 @@ 3239834624,3239835135,AT 3239835136,3239836159,RU 3239836160,3239836671,DK -3239836672,3239837183,DE 3239837184,3239837695,SE 3239837696,3239837951,PL 3239837952,3239839231,DE @@ -58537,6 +63357,7 @@ 3239939072,3239951103,DE 3239951104,3239951359,AT 3239951360,3239954431,DE +3239954432,3239954687,UA 3239954688,3239954943,DK 3239954944,3239955199,ES 3239955200,3239955711,UA @@ -58592,7 +63413,9 @@ 3240038400,3240098815,DE 3240098816,3240099327,CH 3240099328,3240099839,DE -3240099840,3240104703,GB +3240099840,3240102911,GB +3240102912,3240103935,UA +3240103936,3240104703,GB 3240104704,3240104959,NL 3240104960,3240105215,RU 3240105216,3240105471,UA @@ -58606,6 +63429,7 @@ 3240165888,3240166399,ES 3240166400,3240166911,PL 3240166912,3240167423,RO +3240167424,3240167935,PL 3240167936,3240168447,RU 3240168448,3240168959,FR 3240168960,3240169471,CZ @@ -58638,6 +63462,7 @@ 3240183616,3240183647,CR 3240183648,3240183807,NL 3240183808,3240184319,GB +3240184320,3240184831,UA 3240184832,3240185343,GB 3240185344,3240185855,FR 3240185856,3240187391,RU @@ -58703,6 +63528,7 @@ 3240219648,3240220159,UA 3240220160,3240220671,NL 3240220672,3240221183,AT +3240221184,3240221695,SE 3240221696,3240222207,NL 3240222208,3240222719,FR 3240222720,3240223231,KW @@ -58747,6 +63573,7 @@ 3240263680,3240264191,UA 3240264192,3240264703,FR 3240264704,3240265215,TR +3240265216,3240265727,HU 3240265728,3240266239,CH 3240266240,3240266751,PL 3240266752,3240267263,UA @@ -58783,11 +63610,12 @@ 3240281216,3240281343,FR 3240281344,3240281471,PL 3240281472,3240281599,IR +3240281600,3240281727,JO 3240281728,3240281855,PL 3240282112,3240282239,DE 3240282240,3240282367,UA 3240282368,3240282495,RO -3240282496,3240282879,SE +3240282624,3240282879,SE 3240282880,3240283007,UA 3240283008,3240283135,PL 3240284160,3240285183,RU @@ -58841,6 +63669,7 @@ 3240460288,3240461055,IL 3240461056,3240461567,DE 3240461568,3240461823,UA +3240461824,3240462079,EU 3240462080,3240462335,TR 3240462336,3240462591,RO 3240462592,3240462847,RU @@ -58930,7 +63759,6 @@ 3240706048,3240707071,BG 3240707072,3240707839,NL 3240707840,3240709119,FR -3240709120,3240710143,IT 3240710144,3240710399,UA 3240710400,3240710655,LT 3240710656,3240710911,DE @@ -59000,6 +63828,7 @@ 3240745984,3240746495,GB 3240746496,3240747007,FR 3240747008,3240747263,SE +3240747264,3240747519,TR 3240747520,3240747775,RU 3240747776,3240748031,ES 3240748032,3240748287,AT @@ -59096,7 +63925,6 @@ 3240867584,3240867839,CH 3240867840,3240873983,IT 3240873984,3240874495,RU -3240874496,3240875007,FR 3240875008,3240875519,AT 3240875520,3240876031,FR 3240876032,3240876543,DK @@ -59176,7 +64004,7 @@ 3241064704,3241065471,PL 3241065472,3241068543,AT 3241068544,3241068799,GB -3241068800,3241069055,UA +3241068800,3241069311,UA 3241069312,3241069567,FR 3241069568,3241070079,GB 3241070080,3241070335,FR @@ -59231,7 +64059,6 @@ 3241119488,3241119743,BE 3241119744,3241119999,RO 3241120000,3241120255,UA -3241120256,3241120511,CZ 3241120512,3241120767,GB 3241120768,3241121023,RU 3241121024,3241121279,CH @@ -59334,7 +64161,7 @@ 3241867008,3241867263,FR 3241867264,3241867519,UA 3241867520,3241867775,PL -3241867776,3241868031,CH +3241868032,3241868287,RU 3241868288,3241868543,HR 3241868544,3241868799,DE 3241868800,3241869055,UA @@ -59659,7 +64486,7 @@ 3244846592,3244846847,CZ 3244846848,3244847103,NL 3244847104,3244847359,SE -3244847360,3244847615,GB +3244847360,3244847615,A1 3244847616,3244847871,DK 3244847872,3244848127,IL 3244848128,3244848383,CH @@ -59696,7 +64523,7 @@ 3244856832,3244857087,RO 3244857088,3244857343,IT 3244857344,3244857599,GB -3244857600,3244857855,LU +3244857600,3244857855,MC 3244857856,3244858111,PL 3244858112,3244858367,DE 3244858368,3244858623,PL @@ -59721,6 +64548,7 @@ 3244863488,3244863743,KZ 3244863744,3244863999,RU 3244864000,3244864255,UA +3244864256,3244864511,GB 3244864512,3244864767,RO 3244864768,3244865023,RU 3244865024,3244865535,UA @@ -59831,6 +64659,7 @@ 3244894976,3244895231,FR 3244895232,3244895487,DK 3244895488,3244895743,PL +3244895744,3244895999,DE 3244896000,3244896511,GB 3244896512,3244896767,LT 3244896768,3244897023,NL @@ -59946,7 +64775,7 @@ 3244926976,3244927231,DE 3244927232,3244927487,UA 3244927488,3244927743,GB -3244927744,3244927999,UA +3244927744,3244927999,US 3244928000,3244928255,GB 3244928512,3244928767,UA 3244928768,3244929023,RU @@ -60043,6 +64872,7 @@ 3244977152,3244978175,RU 3244978176,3244979199,GB 3244979200,3244980223,FR +3244980224,3244981247,DK 3244981248,3244982271,RO 3244982272,3244983295,PL 3244983296,3244984319,RU @@ -60074,6 +64904,7 @@ 3245014016,3245015039,GR 3245015040,3245016063,RU 3245016064,3245017087,RS +3245017088,3245018111,PL 3245018112,3245019135,RU 3245019136,3245020159,SC 3245020160,3245021183,RU @@ -60141,7 +64972,7 @@ 3245088512,3245088767,FR 3245088768,3245089279,BG 3245089280,3245089791,PL -3245089792,3245090303,BG +3245089792,3245090303,GB 3245090304,3245090815,UA 3245090816,3245091327,AT 3245091328,3245091839,SE @@ -60192,8 +65023,7 @@ 3245127936,3245128191,LV 3245128192,3245128447,IT 3245128448,3245128703,CH -3245128704,3245128959,GB -3245128960,3245129471,DK +3245128704,3245129471,DK 3245129472,3245129983,GB 3245129984,3245130239,DE 3245130240,3245130495,PL @@ -60360,7 +65190,7 @@ 3245232640,3245232895,BG 3245232896,3245233151,IT 3245233152,3245233407,DK -3245233408,3245233663,EU +3245233408,3245233663,CH 3245233664,3245233919,GR 3245233920,3245234175,DK 3245234176,3245234431,GB @@ -60597,7 +65427,9 @@ 3245924688,3245924695,GB 3245924696,3245924751,IE 3245924752,3245924767,GB -3245924768,3245927959,IE +3245924768,3245927583,IE +3245927584,3245927647,GB +3245927648,3245927959,IE 3245927960,3245927967,US 3245927968,3245931263,IE 3245931264,3245931519,GB @@ -60612,15 +65444,13 @@ 3246141440,3246141695,UA 3246141696,3246142975,RU 3246142976,3246143231,UA -3246143232,3246143743,RU -3246143744,3246143999,UA -3246144000,3246144255,RU -3246144256,3246145023,UA -3246145024,3246147839,RU +3246143232,3246144511,RU +3246144512,3246144767,UA +3246144768,3246147839,RU 3246147840,3246148095,UA 3246148096,3246148607,RU -3246148608,3246149631,UA -3246149632,3246260223,RU +3246148608,3246148863,UA +3246148864,3246260223,RU 3246260224,3246325759,PT 3246325760,3246328831,ES 3246328832,3246329087,EU @@ -61113,7 +65943,6 @@ 3247901696,3247902719,UA 3247902720,3247903743,GB 3247903744,3247904767,BG -3247904768,3247905791,GB 3247905792,3247906815,RO 3247906816,3247907839,DE 3247907840,3247908863,PL @@ -61142,7 +65971,9 @@ 3248095232,3248095823,AT 3248095824,3248095831,HU 3248095832,3248160767,AT -3248160768,3248226303,DE +3248160768,3248221215,DE +3248221216,3248221223,PL +3248221224,3248226303,DE 3248226304,3248235007,NO 3248235008,3248235263,PK 3248235264,3248357375,NO @@ -61158,10 +65989,33 @@ 3248372272,3248372319,DE 3248372320,3248372327,BE 3248372328,3248488447,DE -3248488448,3248525311,NO +3248488448,3248488703,FR +3248488704,3248491519,NO +3248491520,3248492031,RU +3248492032,3248496895,NO +3248496896,3248497151,IL +3248497152,3248498431,NO +3248498432,3248498687,DE +3248498688,3248513279,NO +3248513280,3248513535,UA +3248513536,3248514047,AT +3248514048,3248521983,NO +3248521984,3248522239,RU +3248522240,3248525311,NO 3248525312,3248525567,DE 3248525568,3248525823,DK -3248525824,3248619519,NO +3248525824,3248545791,NO +3248545792,3248546815,UA +3248546816,3248547839,RU +3248547840,3248553727,NO +3248553728,3248553983,RU +3248553984,3248554239,RO +3248554240,3248557055,NO +3248557056,3248558079,UA +3248558080,3248575487,NO +3248575488,3248576511,CZ +3248576512,3248619263,NO +3248619264,3248619519,UA 3248619520,3248638463,DK 3248638464,3248638719,GB 3248638720,3248750591,DK @@ -61200,8 +66054,8 @@ 3248790784,3248791039,PL 3248791040,3248791295,BE 3248791296,3248791551,DE -3248791552,3248792383,GB -3248792384,3248796607,EU +3248791552,3248792447,GB +3248792448,3248796607,EU 3248796608,3248796863,GB 3248796864,3248798975,EU 3248798976,3248799231,GB @@ -61254,6 +66108,7 @@ 3249079296,3249079807,RU 3249079808,3249080831,RO 3249080832,3249081343,DE +3249081344,3249081855,LV 3249081856,3249082367,GB 3249082368,3249082879,HU 3249082880,3249083391,PL @@ -61338,6 +66193,7 @@ 3249132032,3249132543,RU 3249132544,3249133055,DE 3249133056,3249133567,UA +3249133568,3249134079,GB 3249134080,3249134591,UA 3249134592,3249135615,RU 3249135616,3249136127,GB @@ -61378,7 +66234,9 @@ 3249574400,3249574655,SE 3249574656,3249574911,UA 3249574912,3249668095,NL -3249668096,3249676799,IE +3249668096,3249676287,IE +3249676288,3249676543,GB +3249676544,3249676799,IE 3249676800,3249677055,UA 3249677056,3249677311,SE 3249677312,3249679103,IE @@ -61462,8 +66320,8 @@ 3249731584,3249732607,UA 3249732608,3249733631,IT 3249733632,3249799167,CZ -3249799168,3249932287,SE -3249932288,3249934335,GB +3249799168,3249932031,SE +3249932032,3249934335,GB 3249934336,3250061311,SE 3250061312,3250061635,FI 3250061636,3250061639,AX @@ -61969,10 +66827,10 @@ 3251209216,3251210239,GB 3251210240,3251211263,PL 3251211264,3251212287,UA -3251212288,3251212415,DE 3251212416,3251212671,FI 3251212672,3251212799,SE 3251212800,3251212927,FR +3251212928,3251213055,RO 3251213056,3251213119,SE 3251213120,3251213151,NL 3251213152,3251213183,FR @@ -61989,7 +66847,7 @@ 3251213888,3251214015,FI 3251214016,3251214079,UA 3251214080,3251214143,FR -3251214144,3251214207,MC +3251214144,3251214207,AF 3251214208,3251214271,RU 3251214272,3251214335,DE 3251214336,3251214463,RU @@ -62124,15 +66982,9 @@ 3251319808,3251320831,HU 3251320832,3251321855,PL 3251321856,3251322879,RU -3251322880,3251329535,GB -3251329536,3251329567,FR -3251329568,3251329727,GB +3251322880,3251329727,GB 3251329728,3251329791,FR -3251329792,3251330047,GB -3251330048,3251330079,IT -3251330080,3251330239,GB -3251330240,3251330303,IT -3251330304,3251331071,GB +3251329792,3251331071,GB 3251331072,3251331583,FR 3251331584,3251332095,PL 3251332096,3251333119,RU @@ -62313,22 +67165,21 @@ 3252408192,3252408319,NO 3252408320,3252408335,TR 3252408336,3252408343,GN -3252408344,3252408359,LT -3252408360,3252408367,GN +3252408344,3252408351,LT +3252408352,3252408367,GN 3252408368,3252408375,NG 3252408376,3252408383,LT 3252408384,3252408391,BI 3252408392,3252408415,LT 3252408416,3252408479,NO 3252408480,3252408495,FI -3252408496,3252408543,LT +3252408496,3252408511,LT +3252408512,3252408519,AF +3252408520,3252408543,LT 3252408544,3252408551,AF -3252408552,3252408559,LT -3252408560,3252408575,IQ -3252408576,3252408639,LT +3252408552,3252408639,LT 3252408640,3252408671,CF -3252408672,3252408687,LT -3252408688,3252408703,SO +3252408672,3252408703,SO 3252408704,3252408735,NG 3252408736,3252408767,LT 3252408768,3252408783,NG @@ -62360,13 +67211,15 @@ 3252409520,3252409527,NG 3252409528,3252409535,LT 3252409536,3252409543,BI -3252409544,3252409631,LT +3252409544,3252409599,LT +3252409600,3252409615,MA +3252409616,3252409631,LT 3252409632,3252409647,AO 3252409648,3252409855,LT 3252409856,3252410111,BW 3252410112,3252410367,LT -3252410368,3252410383,BW -3252410384,3252410431,LT +3252410368,3252410391,BW +3252410392,3252410431,LT 3252410432,3252410463,BI 3252410464,3252410623,LT 3252410624,3252410751,ZW @@ -62386,8 +67239,10 @@ 3252411824,3252411839,LT 3252411840,3252411855,MR 3252411856,3252411879,LT -3252411880,3252411895,CD -3252411896,3252414463,LT +3252411880,3252411903,CD +3252411904,3252412927,LT +3252412928,3252413439,BW +3252413440,3252414463,LT 3252414464,3252414479,GH 3252414480,3252414495,ML 3252414496,3252414511,LT @@ -62400,8 +67255,10 @@ 3252414648,3252414655,NE 3252414656,3252414719,ER 3252414720,3252414975,LT -3252414976,3252415095,IQ -3252415096,3252415551,LT +3252414976,3252415007,IQ +3252415008,3252415015,LT +3252415016,3252415119,IQ +3252415120,3252415551,LT 3252415552,3252415567,NG 3252415568,3252415743,LT 3252415744,3252415775,GB @@ -62413,8 +67270,8 @@ 3252416960,3252417023,GN 3252417024,3252417279,LT 3252417280,3252417287,IQ -3252417288,3252417319,AF -3252417320,3252417535,LT +3252417288,3252417359,AF +3252417360,3252417535,LT 3252417536,3252417855,NG 3252417856,3252417919,LT 3252417920,3252417935,MW @@ -62444,22 +67301,30 @@ 3252419328,3252419343,GH 3252419344,3252419359,LT 3252419360,3252419423,GH -3252419424,3252419583,LT -3252419584,3252419775,IQ -3252419776,3252419839,LT -3252419840,3252419975,IQ -3252419976,3252419983,LT -3252419984,3252420055,IQ +3252419424,3252419839,LT +3252419840,3252419911,IQ +3252419912,3252419919,LT +3252419920,3252419951,IQ +3252419952,3252419959,LT +3252419960,3252419967,IQ +3252419968,3252419991,LT +3252419992,3252420007,IQ +3252420008,3252420031,LT +3252420032,3252420055,IQ 3252420056,3252420063,LT -3252420064,3252420111,IQ +3252420064,3252420079,IQ +3252420080,3252420095,LT +3252420096,3252420111,IQ 3252420112,3252420119,GB 3252420120,3252420143,IQ -3252420144,3252420151,LT -3252420152,3252420159,IQ -3252420160,3252420191,LT -3252420192,3252420287,IQ -3252420288,3252420295,LT -3252420296,3252420351,IQ +3252420144,3252420191,LT +3252420192,3252420223,IQ +3252420224,3252420239,LT +3252420240,3252420247,IQ +3252420248,3252420263,LT +3252420264,3252420279,IQ +3252420280,3252420303,LT +3252420304,3252420351,IQ 3252420352,3252420367,LT 3252420368,3252420415,IQ 3252420416,3252420431,GB @@ -62468,8 +67333,10 @@ 3252420456,3252420463,IQ 3252420464,3252420471,AF 3252420472,3252420479,LT -3252420480,3252420607,IQ -3252420608,3252421119,LT +3252420480,3252420527,IQ +3252420528,3252420575,LT +3252420576,3252420591,IQ +3252420592,3252421119,LT 3252421120,3252421631,NO 3252421632,3252424703,LT 3252424704,3252424719,GN @@ -62498,8 +67365,8 @@ 3252428448,3252430079,LT 3252430080,3252430335,BJ 3252430336,3252430463,NE -3252430464,3252430503,LT -3252430504,3252430519,BF +3252430464,3252430511,LT +3252430512,3252430519,BF 3252430520,3252430527,LT 3252430528,3252430543,BF 3252430544,3252430559,BJ @@ -62525,8 +67392,8 @@ 3252434736,3252435199,LT 3252435200,3252435247,TZ 3252435248,3252435295,LT -3252435296,3252435303,TZ -3252435304,3252435319,LT +3252435296,3252435311,TZ +3252435312,3252435319,LT 3252435320,3252435327,MZ 3252435328,3252435343,CD 3252435344,3252435359,LT @@ -62540,10 +67407,11 @@ 3252435872,3252435887,GH 3252435888,3252435919,BW 3252435920,3252435935,BF -3252435936,3252435967,CF +3252435936,3252435951,CF +3252435952,3252435967,LT 3252435968,3252436223,TZ 3252436224,3252436239,GN -3252436240,3252436255,NA +3252436240,3252436255,LT 3252436256,3252436271,ML 3252436272,3252436287,BF 3252436288,3252436303,LT @@ -62552,10 +67420,11 @@ 3252436336,3252436351,ZM 3252436352,3252436383,GN 3252436384,3252436399,SN -3252436400,3252436415,TZ -3252436416,3252436447,LT +3252436400,3252436447,LT 3252436448,3252436479,ER -3252436480,3252438015,LT +3252436480,3252436991,LT +3252436992,3252437503,NG +3252437504,3252438015,LT 3252438016,3252438271,TR 3252438272,3252438527,MD 3252438528,3252438783,CM @@ -62630,9 +67499,7 @@ 3252453120,3252453151,NG 3252453152,3252453247,LT 3252453248,3252453271,NG -3252453272,3252454527,LT -3252454528,3252454583,IQ -3252454584,3252454655,LT +3252453272,3252454655,LT 3252454656,3252454911,IQ 3252454912,3252455167,GB 3252455168,3252455295,NG @@ -62822,7 +67689,7 @@ 3252980992,3252981247,GB 3252981248,3252981503,RU 3252981504,3252981759,SE -3252981760,3252982015,EU +3252981760,3252982015,DE 3252982016,3252982271,RO 3252982272,3252982527,DE 3252982528,3252982783,FR @@ -62938,7 +67805,7 @@ 3253683200,3253685247,UA 3253685248,3253686271,FR 3253686272,3253687295,PL -3253687296,3253688319,UA +3253687296,3253688319,RU 3253688320,3253690367,NL 3253690368,3253691391,DK 3253691392,3253692415,PL @@ -62955,6 +67822,7 @@ 3253700608,3253701119,PL 3253701120,3253702143,RO 3253702144,3253702655,UA +3253702656,3253703167,PL 3253703168,3253703679,RO 3253703680,3253704191,HU 3253704192,3253704703,RU @@ -62963,7 +67831,7 @@ 3253705728,3253706239,DE 3253706240,3253706751,RO 3253706752,3253707263,GB -3253707776,3253708287,DE +3253707264,3253708287,DE 3253708288,3253708799,RO 3253708800,3253709311,DK 3253709312,3253709823,UA @@ -63252,7 +68120,6 @@ 3254704128,3254705663,UA 3254705664,3254706175,GR 3254706176,3254706687,RU -3254706688,3254707199,UA 3254707200,3254707711,TR 3254707712,3254708223,RO 3254708224,3254708735,UA @@ -63337,8 +68204,7 @@ 3254823424,3254823679,NL 3254823680,3254823935,PL 3254823936,3254824191,CH -3254824192,3254824447,BE -3254824448,3254824959,NL +3254824192,3254824959,BE 3254824960,3254825215,FR 3254825216,3254825471,RO 3254825472,3254825727,TR @@ -63571,6 +68437,7 @@ 3255205888,3255214079,FR 3255214080,3255222271,CH 3255222272,3255223295,DE +3255223296,3255224319,RU 3255224320,3255225343,RO 3255225344,3255226367,PL 3255226368,3255227391,DE @@ -63632,9 +68499,7 @@ 3255252528,3255252543,LU 3255252544,3255252559,BE 3255252560,3255252575,LU -3255252576,3255254847,BE -3255254848,3255254879,LU -3255254880,3255255559,BE +3255252576,3255255559,BE 3255255560,3255255567,LU 3255255568,3255255599,BE 3255255600,3255255607,LU @@ -63644,9 +68509,7 @@ 3255255752,3255255759,DK 3255255760,3255256319,BE 3255256320,3255256575,LU -3255256576,3255259199,BE -3255259200,3255259215,LU -3255259216,3255259327,BE +3255256576,3255259327,BE 3255259328,3255259359,LU 3255259360,3255259919,BE 3255259920,3255259927,LU @@ -63700,7 +68563,11 @@ 3255280032,3255280039,LU 3255280040,3255280047,BE 3255280048,3255280055,LU -3255280056,3255283927,BE +3255280056,3255281407,BE +3255281408,3255281439,LU +3255281440,3255283415,BE +3255283416,3255283423,LU +3255283424,3255283927,BE 3255283928,3255283931,LU 3255283932,3255284239,BE 3255284240,3255284255,LU @@ -63960,6 +68827,7 @@ 3255565568,3255566079,DE 3255566080,3255566335,CH 3255566336,3255582719,SE +3255590912,3255599103,UA 3255599104,3255615487,CH 3255615488,3255623679,DE 3255623680,3255631871,BG @@ -64121,6 +68989,7 @@ 3256674304,3256675327,RO 3256675328,3256677375,RU 3256677376,3256678399,GB +3256678400,3256680447,DK 3256680448,3256688639,CH 3256688640,3256692735,IL 3256692736,3256693759,GR @@ -64384,6 +69253,7 @@ 3257559552,3257560063,UA 3257560064,3257560575,DE 3257560576,3257561087,PL +3257561088,3257561599,UA 3257561600,3257562111,DE 3257562112,3257563647,PL 3257564160,3257564671,GB @@ -65316,7 +70186,7 @@ 3258085888,3258086143,UA 3258086144,3258086399,AT 3258086400,3258086655,UA -3258086656,3258087167,GB +3258086912,3258087167,GB 3258087168,3258087423,UA 3258087424,3258087679,RU 3258087680,3258087935,GB @@ -65428,9 +70298,7 @@ 3258504960,3258505215,IL 3258505216,3258515455,CH 3258515456,3258580991,FR -3258580992,3258601983,RU -3258601984,3258602495,UA -3258602496,3258625791,RU +3258580992,3258625791,RU 3258625792,3258626047,UA 3258626048,3258646527,RU 3258646528,3258691583,DE @@ -65497,8 +70365,16 @@ 3258949632,3258972159,GR 3258972160,3258974207,NO 3258974208,3259023103,DE -3259023104,3259023119,IT -3259023120,3259032551,DE +3259023104,3259023107,ES +3259023108,3259031655,DE +3259031656,3259031659,ES +3259031660,3259031895,DE +3259031896,3259031899,ES +3259031900,3259032199,DE +3259032200,3259032203,ES +3259032204,3259032439,DE +3259032440,3259032443,ES +3259032444,3259032551,DE 3259032552,3259032559,IT 3259032560,3259032831,DE 3259032832,3259032879,GB @@ -65506,7 +70382,9 @@ 3259034320,3259034327,GB 3259034328,3259035447,DE 3259035448,3259035455,IT -3259035456,3259039743,DE +3259035456,3259036031,DE +3259036032,3259036035,ES +3259036036,3259039743,DE 3259039744,3259062015,PT 3259062016,3259062271,GW 3259062272,3259105279,PT @@ -65906,14 +70784,14 @@ 3261071360,3261136895,AT 3261136896,3261150143,DE 3261150144,3261150207,US -3261150208,3261173987,DE +3261150208,3261170687,DE +3261170688,3261170943,CH +3261170944,3261173987,DE 3261173988,3261173991,AT 3261173992,3261174007,DE 3261174008,3261174015,AT 3261174016,3261202431,DE -3261202432,3261236863,FR -3261236864,3261236991,GB -3261236992,3261267967,FR +3261202432,3261267967,FR 3261267968,3261297663,DE 3261297664,3261297919,RU 3261297920,3261298175,PL @@ -69252,9 +74130,7 @@ 3263689472,3263689727,UA 3263689728,3263689983,RU 3263689984,3263690239,UA -3263690240,3263691007,RU -3263691008,3263691263,UA -3263691264,3263691775,RU +3263690240,3263691775,RU 3263691776,3263692031,UA 3263692032,3263692287,RU 3263692288,3263692799,UA @@ -69361,6 +74237,7 @@ 3264312320,3264312575,PL 3264312576,3264312831,UA 3264312832,3264313087,DE +3264313088,3264313343,NL 3264313344,3264313599,RO 3264313600,3264313855,PT 3264313856,3264314623,DE @@ -69397,6 +74274,7 @@ 3264332800,3264333311,GB 3264333312,3264333823,UA 3264333824,3264334335,NL +3264334336,3264334847,UA 3264334848,3264335359,PL 3264335360,3264335871,SK 3264335872,3264336383,CY @@ -69448,6 +74326,7 @@ 3264386048,3264387071,IR 3264387072,3264388095,LB 3264388096,3264389119,UA +3264389120,3264390143,RU 3264390144,3264391167,IR 3264391168,3264392191,SE 3264392192,3264393215,FR @@ -69488,7 +74367,10 @@ 3264565248,3264567295,NL 3264567296,3264567551,PL 3264567552,3264567807,GB -3264567808,3264569343,RU +3264567808,3264568319,RU +3264568576,3264568831,RU +3264568832,3264569087,FR +3264569088,3264569343,RU 3264569344,3264577535,CZ 3264577536,3264585727,MK 3264585728,3264593919,FR @@ -69686,17 +74568,18 @@ 3264845952,3264846079,GB 3264846080,3264846207,UG 3264846208,3264846335,PL +3264846336,3264846463,GB 3264846464,3264846591,NO 3264846592,3264846719,US 3264846720,3264846847,RU 3264846848,3264846911,DK +3264846912,3264846975,SK 3264847040,3264847103,GB 3264847104,3264847135,FR 3264847136,3264847167,ES 3264847168,3264847199,IE 3264847200,3264847231,NO 3264847264,3264847295,LI -3264847328,3264847359,EU 3264847360,3264847487,FR 3264847488,3264847615,RU 3264847616,3264847679,PL @@ -69736,7 +74619,9 @@ 3264906333,3264906335,GR 3264906336,3264906339,CY 3264906340,3264906351,GR -3264906352,3264906367,CY +3264906352,3264906355,CY +3264906356,3264906359,GR +3264906360,3264906367,CY 3264906368,3264906411,GR 3264906412,3264906423,CY 3264906424,3264906435,GR @@ -69961,7 +74846,11 @@ 3265464848,3265464863,AT 3265464864,3265465343,DE 3265465344,3265465599,AT -3265465600,3265527807,DE +3265465600,3265467519,DE +3265467520,3265467523,ES +3265467524,3265527055,DE +3265527056,3265527059,ES +3265527060,3265527807,DE 3265527808,3265569279,GB 3265569280,3265569791,AE 3265569792,3265582335,GB @@ -70036,8 +74925,7 @@ 3265906944,3265907199,ES 3265907200,3265907455,DE 3265907456,3265907711,FR -3265907712,3265907967,MD -3265907968,3265908735,UA +3265907712,3265908735,UA 3265908736,3265908991,NL 3265908992,3265909247,RO 3265909248,3265909503,UA @@ -70232,6 +75120,7 @@ 3267046400,3267047423,NL 3267047424,3267048447,UA 3267048448,3267049471,PL +3267049472,3267050495,GB 3267050496,3267051519,SE 3267051520,3267052543,DE 3267052544,3267053567,LV @@ -70409,12 +75298,11 @@ 3267630880,3267630943,FR 3267630944,3267630959,EU 3267630960,3267631055,IT -3267631056,3267631087,EU +3267631056,3267631071,GB +3267631072,3267631087,EU 3267631088,3267631103,IT 3267631104,3267631615,GB -3267631616,3267631935,DE -3267631936,3267631999,EU -3267632000,3267632127,DE +3267631616,3267632127,DE 3267632128,3267632383,EU 3267632384,3267632511,BE 3267632512,3267632639,EU @@ -70425,7 +75313,9 @@ 3267633728,3267633855,NO 3267633856,3267634175,EU 3267634176,3267634431,GB -3267634432,3267634943,EU +3267634432,3267634559,EU +3267634560,3267634687,GB +3267634688,3267634943,EU 3267634944,3267635711,GB 3267635712,3267635967,EU 3267635968,3267636223,BE @@ -70440,9 +75330,7 @@ 3267637504,3267637695,IT 3267637696,3267637759,EU 3267637760,3267637823,ES -3267637824,3267637855,EU -3267637856,3267637887,ES -3267637888,3267638015,EU +3267637824,3267638015,EU 3267638016,3267638271,ES 3267638272,3267638527,BE 3267638528,3267638783,DK @@ -70463,7 +75351,9 @@ 3267642624,3267642879,NO 3267642880,3267643135,SI 3267643136,3267643391,BE -3267643392,3267643439,RU +3267643392,3267643407,RU +3267643408,3267643423,EU +3267643424,3267643439,RU 3267643440,3267643455,EU 3267643456,3267643487,RU 3267643488,3267643519,EU @@ -70584,7 +75474,9 @@ 3267666264,3267666271,EU 3267666272,3267666431,GR 3267666432,3267666447,GB -3267666448,3267666495,EU +3267666448,3267666463,EU +3267666464,3267666479,GB +3267666480,3267666495,EU 3267666496,3267666543,GB 3267666544,3267666575,EU 3267666576,3267666951,GB @@ -70594,23 +75486,29 @@ 3267667456,3267667967,GB 3267667968,3267670015,EU 3267670016,3267671551,ZA -3267671552,3267671807,DE +3267671552,3267671679,DE +3267671680,3267671711,EU +3267671712,3267671807,DE 3267671808,3267672063,NO 3267672064,3267672223,DE 3267672224,3267672255,EU 3267672256,3267672319,DE 3267672320,3267672575,EU 3267672576,3267672831,AT -3267672832,3267672927,FR -3267672928,3267672935,EU -3267672936,3267672999,FR +3267672832,3267672847,FR +3267672848,3267672855,EU +3267672856,3267672927,FR +3267672928,3267672943,EU +3267672944,3267672999,FR 3267673000,3267673007,EU 3267673008,3267673015,FR 3267673016,3267673023,EU 3267673024,3267673087,FR 3267673088,3267673439,DE 3267673440,3267673471,EU -3267673472,3267673503,DE +3267673472,3267673479,DE +3267673480,3267673487,EU +3267673488,3267673503,DE 3267673504,3267673599,EU 3267673600,3267673759,DE 3267673760,3267673807,EU @@ -70626,8 +75524,8 @@ 3267674368,3267674879,BE 3267674880,3267675135,EU 3267675136,3267675391,AT -3267675392,3267675615,CH -3267675616,3267675623,EU +3267675392,3267675619,CH +3267675620,3267675623,EU 3267675624,3267675631,CH 3267675632,3267675647,EU 3267675648,3267675775,CY @@ -70683,8 +75581,8 @@ 3267683568,3267683575,PL 3267683576,3267684383,EU 3267684384,3267684399,GB -3267684400,3267684415,EU -3267684416,3267684783,GB +3267684400,3267684407,EU +3267684408,3267684783,GB 3267684784,3267684791,EU 3267684792,3267685119,GB 3267685120,3267685375,DE @@ -70785,7 +75683,9 @@ 3268223232,3268224767,EU 3268224768,3268225023,US 3268225024,3268226367,EU -3268226368,3268226815,GB +3268226368,3268226631,GB +3268226632,3268226687,EU +3268226688,3268226815,GB 3268226816,3268227327,EU 3268227328,3268227391,GB 3268227392,3268227519,EU @@ -70839,8 +75739,7 @@ 3268238360,3268238367,DE 3268238368,3268238463,GB 3268238464,3268238471,DE -3268238472,3268238479,EU -3268238480,3268238527,GB +3268238472,3268238527,GB 3268238528,3268239103,EU 3268239104,3268239359,GB 3268239360,3268239583,EU @@ -70849,7 +75748,9 @@ 3268240160,3268240191,GB 3268240192,3268240383,EU 3268240384,3268240399,GB -3268240400,3268240687,EU +3268240400,3268240479,EU +3268240480,3268240487,GB +3268240488,3268240687,EU 3268240688,3268240695,GB 3268240696,3268240711,EU 3268240712,3268240735,GB @@ -71086,6 +75987,21 @@ 3268747008,3268747263,DE 3268747264,3268755455,CZ 3268755456,3268763647,SE +3268763648,3268764159,RU +3268764160,3268764671,DE +3268764672,3268765183,LV +3268765184,3268765695,RU +3268765696,3268766719,UA +3268766720,3268767231,IL +3268767232,3268767743,RU +3268767744,3268768255,NL +3268768256,3268768767,BE +3268768768,3268769279,RU +3268769280,3268769791,SE +3268769792,3268770303,UA +3268770304,3268770815,RU +3268770816,3268771327,FR +3268771328,3268771839,NL 3268771840,3268788223,NO 3268788224,3268804607,CZ 3268804608,3268848895,FR @@ -71120,11 +76036,7 @@ 3269200640,3269200895,UA 3269200896,3269212671,RU 3269212672,3269213183,UA -3269213184,3269231871,RU -3269231872,3269232127,UA -3269232128,3269233407,RU -3269233408,3269233663,UA -3269233664,3269239039,RU +3269213184,3269239039,RU 3269239040,3269239295,UA 3269239296,3269263359,RU 3269263360,3269264639,EU @@ -71141,8 +76053,8 @@ 3269272704,3269272831,DE 3269272832,3269272847,GB 3269272848,3269272863,NL -3269272864,3269272879,GB -3269272880,3269272895,EU +3269272864,3269272887,GB +3269272888,3269272895,EU 3269272896,3269273087,NL 3269273088,3269273343,DE 3269273344,3269273599,EU @@ -71299,7 +76211,11 @@ 3269936064,3269936079,NL 3269936080,3269984255,DE 3269984256,3270049791,AT -3270049792,3270115327,DE +3270049792,3270062191,DE +3270062192,3270062195,ES +3270062196,3270078391,DE +3270078392,3270078395,ES +3270078396,3270115327,DE 3270115328,3270117503,EU 3270117504,3270117631,DK 3270117632,3270122239,EU @@ -71461,6 +76377,7 @@ 3270639616,3270640127,PL 3270640128,3270642175,RU 3270642176,3270642687,NO +3270642688,3270643199,DE 3270643200,3270643711,RU 3270643712,3270644223,AT 3270644224,3270644735,NL @@ -71550,7 +76467,7 @@ 3270973952,3270974463,FR 3270974464,3270974975,IT 3270974976,3270975487,NL -3270975488,3270991871,GB +3270983680,3270991871,GB 3270991872,3271000063,NL 3271000064,3271008255,RU 3271008256,3271008767,BE @@ -71576,8 +76493,7 @@ 3271098368,3271163903,SI 3271163904,3271200767,FR 3271200768,3271200775,GB -3271200776,3271200783,ES -3271200784,3271229439,FR +3271200776,3271229439,FR 3271229440,3271244191,FI 3271244192,3271244223,AX 3271244224,3271247867,FI @@ -72047,13 +76963,9 @@ 3272138752,3272146943,DE 3272146944,3272161023,RU 3272161024,3272161279,KZ -3272161280,3272169215,RU -3272169216,3272169471,UA -3272169472,3272178559,RU +3272161280,3272178559,RU 3272178560,3272178623,UA -3272178624,3272210175,RU -3272210176,3272210431,UA -3272210432,3272212479,RU +3272178624,3272212479,RU 3272212480,3272212991,GB 3272212992,3272213375,EU 3272213376,3272213391,IT @@ -72256,10 +77168,12 @@ 3272265984,3272266239,GB 3272266240,3272266495,UA 3272266496,3272266751,SE +3272266752,3272267007,NL 3272267008,3272267263,GB 3272267264,3272267519,SE 3272267520,3272267775,GB 3272267776,3272268031,LV +3272268288,3272268543,UA 3272268544,3272268799,RO 3272268800,3272269055,IL 3272269056,3272269311,GB @@ -72295,22 +77209,21 @@ 3272402040,3272402047,US 3272402048,3272402079,EU 3272402080,3272402111,SE -3272402112,3272402143,FR -3272402144,3272402175,DE +3272402112,3272402175,EU 3272402176,3272402191,GB 3272402192,3272402239,EU 3272402240,3272402303,GB 3272402304,3272402431,FR 3272402432,3272402447,EU 3272402448,3272402455,FR -3272402456,3272402527,EU -3272402528,3272402559,NL +3272402456,3272402559,EU 3272402560,3272402623,GB 3272402624,3272402687,EU 3272402688,3272402815,SE 3272402816,3272402879,GB 3272402880,3272403007,EU -3272403008,3272403039,FR +3272403008,3272403023,FR +3272403024,3272403039,EU 3272403040,3272403055,DE 3272403056,3272403071,NL 3272403072,3272403199,FR @@ -72368,6 +77281,16 @@ 3272482560,3272482815,BE 3272482816,3272491007,RU 3272491008,3272499199,GB +3272499200,3272499711,RU +3272499712,3272500223,US +3272500224,3272500735,RU +3272500736,3272501247,IT +3272501248,3272502783,RU +3272502784,3272503295,KW +3272503296,3272503807,UA +3272505344,3272505855,AT +3272505856,3272506879,UA +3272506880,3272507391,CH 3272507392,3272515583,CY 3272515584,3272523775,LT 3272523776,3272540159,DE @@ -72477,6 +77400,11 @@ 3272733984,3272733991,RU 3272733992,3272736767,SE 3272736768,3272737791,IT +3272737792,3272738815,GB +3272738816,3272739839,IL +3272739840,3272741887,UA +3272741888,3272743935,RU +3272743936,3272744959,UA 3272744960,3272753151,RU 3272753152,3272769535,NL 3272769536,3272802303,NO @@ -72520,6 +77448,7 @@ 3272921088,3272921599,CH 3272921600,3272922111,DE 3272922112,3272922623,PL +3272922624,3272923135,GB 3272923136,3272923647,ES 3272923648,3272924159,PL 3272924160,3272924671,DE @@ -72851,7 +77780,9 @@ 3273327288,3273327359,EU 3273327360,3273327423,IE 3273327424,3273327495,GB -3273327496,3273328511,EU +3273327496,3273327519,EU +3273327520,3273327551,IE +3273327552,3273328511,EU 3273328512,3273328639,DE 3273328640,3273329199,GB 3273329200,3273329215,DE @@ -73054,13 +77985,14 @@ 3273390080,3273390111,DE 3273390112,3273390143,EU 3273390144,3273390207,DE -3273390208,3273390367,EU -3273390368,3273390463,DE +3273390208,3273390335,EU +3273390336,3273390463,DE 3273390464,3273391231,EU 3273391232,3273391327,DE 3273391328,3273391343,EU 3273391344,3273391359,DE -3273391360,3273392127,EU +3273391360,3273391871,EU +3273391872,3273392127,DE 3273392128,3273392639,UA 3273392640,3273393663,RO 3273393664,3273394175,BG @@ -73294,6 +78226,7 @@ 3274338304,3274338815,PL 3274338816,3274339327,DE 3274339328,3274339839,RO +3274339840,3274340351,RU 3274340352,3274340863,IE 3274340864,3274341375,RU 3274341376,3274341887,FR @@ -73308,6 +78241,7 @@ 3274345984,3274346495,DE 3274346496,3274347007,FR 3274347008,3274347519,RU +3274347520,3274348031,FR 3274348032,3274348543,DK 3274348544,3274349055,PL 3274349056,3274358783,RU @@ -73336,6 +78270,13 @@ 3274393600,3274394111,PL 3274394112,3274394623,AM 3274394624,3274395135,CZ +3274395136,3274395647,DK +3274395648,3274396671,LV +3274396672,3274397183,PL +3274397184,3274397695,EE +3274397696,3274398207,DE +3274398208,3274399231,RU +3274399232,3274399743,SC 3274399744,3274407935,LU 3274407936,3274408191,ES 3274408192,3274408447,IT @@ -73537,6 +78478,7 @@ 3274505856,3274506239,EU 3274506240,3274571775,IT 3274571776,3274579967,GR +3274579968,3274580991,UA 3274580992,3274582015,NL 3274582016,3274583039,PL 3274583040,3274584063,DE @@ -73551,7 +78493,9 @@ 3274620928,3274629119,FR 3274629120,3274629151,LU 3274629152,3274629159,A2 -3274629160,3274637311,LU +3274629160,3274637175,LU +3274637176,3274637183,A2 +3274637184,3274637311,LU 3274637312,3274670079,CZ 3274670080,3274686463,DK 3274686464,3274686719,RU @@ -73637,7 +78581,6 @@ 3274821632,3274823679,KZ 3274823680,3274825727,TR 3274825728,3274827775,DE -3274827776,3274829823,RU 3274829824,3274831871,SE 3274831872,3274833919,RO 3274833920,3274842111,BY @@ -73825,11 +78768,38 @@ 3274964928,3274964991,GB 3274964992,3275030527,DE 3275030528,3275096063,ES -3275096064,3275104255,RU +3275096064,3275105279,RU +3275105280,3275105791,GB +3275105792,3275106303,NL +3275106304,3275106815,RU +3275106816,3275107327,UA +3275107328,3275107839,BG +3275107840,3275108351,PL +3275108352,3275108863,FR +3275108864,3275109375,PL +3275109376,3275109887,UA +3275109888,3275110399,DE +3275110400,3275110911,NL +3275110912,3275111423,GB +3275111424,3275111935,IT +3275111936,3275112447,FR 3275112448,3275120639,DK 3275120640,3275136639,SE 3275136640,3275136767,DK 3275136768,3275137023,SE +3275137024,3275137535,DE +3275137536,3275138047,UA +3275138048,3275138559,DE +3275138560,3275139071,UA +3275139072,3275139583,PT +3275139584,3275140095,MD +3275140096,3275140607,AT +3275140608,3275141119,NL +3275141120,3275141631,RU +3275141632,3275142143,GE +3275142144,3275142655,KG +3275142656,3275143167,SE +3275143168,3275145215,PL 3275145216,3275153407,RU 3275153408,3275161599,GB 3275161600,3275227135,ES @@ -73847,15 +78817,15 @@ 3275423744,3275423751,GB 3275423752,3275423775,EU 3275423776,3275423807,GB -3275423808,3275423999,EU -3275424000,3275424255,GB +3275423808,3275423871,EU +3275423872,3275424255,GB 3275424256,3275425791,EU 3275425792,3275426599,GB -3275426600,3275426815,EU -3275426816,3275427071,GB -3275427072,3275427103,EU -3275427104,3275427135,GB -3275427136,3275427839,EU +3275426600,3275426607,EU +3275426608,3275426623,GB +3275426624,3275426815,EU +3275426816,3275427199,GB +3275427200,3275427839,EU 3275427840,3275428375,GB 3275428376,3275429887,EU 3275429888,3275430143,GB @@ -73876,32 +78846,32 @@ 3275442720,3275444223,EU 3275444224,3275444735,GB 3275444736,3275446271,EU -3275446272,3275446815,GB -3275446816,3275446847,EU -3275446848,3275446911,GB -3275446912,3275448319,EU +3275446272,3275446831,GB +3275446832,3275446847,EU +3275446848,3275446943,GB +3275446944,3275448319,EU 3275448320,3275449519,GB 3275449520,3275449527,FR 3275449528,3275450879,GB 3275450880,3275451231,EU 3275451232,3275451263,GB 3275451264,3275452415,EU -3275452416,3275453823,GB -3275453824,3275454463,EU -3275454464,3275455255,GB -3275455256,3275455263,EU -3275455264,3275455279,GB -3275455280,3275455295,EU -3275455296,3275455999,GB -3275456000,3275456511,EU +3275452416,3275453959,GB +3275453960,3275453967,EU +3275453968,3275454047,GB +3275454048,3275454463,EU +3275454464,3275456383,GB +3275456384,3275456511,EU 3275456512,3275457023,GB 3275457024,3275457791,FK 3275457792,3275458559,GB 3275458560,3275460095,IE 3275460096,3275460607,EU 3275460608,3275460863,HK -3275460864,3275463455,GB -3275463456,3275463679,EU +3275460864,3275463471,GB +3275463472,3275463487,EU +3275463488,3275463503,GB +3275463504,3275463679,EU 3275463680,3275464031,GB 3275464032,3275464047,IE 3275464048,3275468655,GB @@ -73931,14 +78901,13 @@ 3275509984,3275510015,PT 3275510016,3275510079,SE 3275510080,3275510143,ES -3275510208,3275510271,FR -3275510272,3275510335,LV 3275510336,3275510399,IE 3275510400,3275510463,NL 3275510464,3275510527,GB 3275510528,3275510559,PL 3275510560,3275510591,DE 3275510624,3275510655,EE +3275510656,3275510687,FR 3275510784,3275510911,RU 3275510912,3275511167,GB 3275511168,3275511295,LV @@ -73971,7 +78940,7 @@ 3275530752,3275531263,AT 3275531264,3275531775,GB 3275531776,3275532287,IE -3275532288,3275532799,FR +3275532288,3275532799,CH 3275532800,3275533823,GB 3275533824,3275534335,UA 3275534336,3275534847,IL @@ -74237,7 +79206,7 @@ 3275796480,3275796991,DE 3275796992,3275797503,RO 3275797504,3275798015,DE -3275798528,3275799039,UA +3275798016,3275799039,UA 3275799040,3275799551,ES 3275799552,3275800063,IT 3275800064,3275800575,UA @@ -74316,6 +79285,7 @@ 3275917312,3275918847,UA 3275918848,3275919359,RO 3275919360,3275919871,PL +3275919872,3275920383,UA 3275920384,3275920895,GB 3275920896,3275921407,ES 3275921408,3275921919,UA @@ -74382,7 +79352,9 @@ 3276015248,3276015263,GB 3276015264,3276015295,FR 3276015296,3276015327,GB -3276015328,3276015559,FR +3276015328,3276015415,FR +3276015416,3276015423,GB +3276015424,3276015559,FR 3276015560,3276015583,GB 3276015584,3276015879,FR 3276015880,3276015887,GB @@ -74430,7 +79402,9 @@ 3276018184,3276018207,GB 3276018208,3276018375,FR 3276018376,3276018383,GB -3276018384,3276018495,FR +3276018384,3276018423,FR +3276018424,3276018431,GB +3276018432,3276018495,FR 3276018496,3276018527,GB 3276018528,3276018543,FR 3276018544,3276018623,GB @@ -74484,7 +79458,9 @@ 3276020880,3276020887,GB 3276020888,3276020943,FR 3276020944,3276020991,GB -3276020992,3276021591,FR +3276020992,3276021103,FR +3276021104,3276021111,GB +3276021112,3276021591,FR 3276021592,3276021599,GB 3276021600,3276022039,FR 3276022040,3276022047,GB @@ -74492,9 +79468,13 @@ 3276022088,3276022095,GB 3276022096,3276022127,FR 3276022128,3276022143,GB -3276022144,3276022255,FR +3276022144,3276022215,FR +3276022216,3276022223,GB +3276022224,3276022255,FR 3276022256,3276022271,GB -3276022272,3276022455,FR +3276022272,3276022343,FR +3276022344,3276022351,GB +3276022352,3276022455,FR 3276022456,3276022463,GB 3276022464,3276022479,FR 3276022480,3276022495,GB @@ -74506,7 +79486,9 @@ 3276022656,3276022687,GB 3276022688,3276022751,FR 3276022752,3276022783,GB -3276022784,3276022983,FR +3276022784,3276022791,FR +3276022792,3276022799,GB +3276022800,3276022983,FR 3276022984,3276022991,GB 3276022992,3276023031,FR 3276023032,3276023039,GB @@ -74526,8 +79508,8 @@ 3276023944,3276023951,GB 3276023952,3276023959,FR 3276023960,3276023967,GB -3276023968,3276024703,FR -3276024704,3276024831,GB +3276023968,3276024719,FR +3276024720,3276024831,GB 3276024832,3276025111,FR 3276025112,3276025135,GB 3276025136,3276025151,FR @@ -74563,9 +79545,7 @@ 3276026608,3276026703,FR 3276026704,3276026719,GB 3276026720,3276026727,FR -3276026728,3276026743,GB -3276026744,3276026751,FR -3276026752,3276026783,GB +3276026728,3276026783,GB 3276026784,3276026815,FR 3276026816,3276026823,GB 3276026824,3276026847,FR @@ -74651,8 +79631,8 @@ 3276031384,3276031423,FR 3276031424,3276031439,GB 3276031440,3276031519,FR -3276031520,3276031535,GB -3276031536,3276031559,FR +3276031520,3276031551,GB +3276031552,3276031559,FR 3276031560,3276031575,GB 3276031576,3276031583,FR 3276031584,3276031591,GB @@ -74971,17 +79951,15 @@ 3276227608,3276227615,MT 3276227616,3276227727,DE 3276227728,3276227743,MT -3276227744,3276234303,DE +3276227744,3276233791,DE +3276233792,3276233799,RO +3276233800,3276233807,DE +3276233808,3276233823,RO +3276233824,3276234303,DE 3276234304,3276234367,GB -3276234368,3276235519,DE -3276235520,3276235583,NL -3276235584,3276241759,DE +3276234368,3276241759,DE 3276241760,3276241767,FR -3276241768,3276242871,DE -3276242872,3276242879,NL -3276242880,3276244351,DE -3276244352,3276244367,NL -3276244368,3276244447,DE +3276241768,3276244447,DE 3276244448,3276244479,RO 3276244480,3276249279,DE 3276249280,3276249287,FR @@ -77106,15 +82084,10 @@ 3276475952,3276475983,EU 3276475984,3276476039,IT 3276476040,3276476047,EU -3276476048,3276476055,IT -3276476056,3276476063,EU -3276476064,3276476071,IT -3276476072,3276476079,EU -3276476080,3276476111,IT +3276476048,3276476111,IT 3276476112,3276476119,EU 3276476120,3276476159,IT -3276476160,3276476287,GB -3276476288,3276476415,EU +3276476160,3276476415,EU 3276476416,3276476671,CH 3276476672,3276477439,EU 3276477440,3276477983,CH @@ -77130,7 +82103,7 @@ 3276479200,3276479215,FR 3276479216,3276479223,EU 3276479224,3276479343,FR -3276479344,3276479351,GB +3276479344,3276479351,EU 3276479352,3276479615,FR 3276479616,3276479647,EU 3276479648,3276479743,FR @@ -77193,9 +82166,7 @@ 3276492488,3276492495,EU 3276492496,3276493215,GB 3276493216,3276493247,EU -3276493248,3276494335,GB -3276494336,3276494367,EU -3276494368,3276494383,GB +3276493248,3276494383,GB 3276494384,3276494415,EU 3276494416,3276495503,GB 3276495504,3276495519,EU @@ -77245,8 +82216,8 @@ 3276501024,3276501055,FR 3276501056,3276501119,EU 3276501120,3276501167,DE -3276501168,3276501247,EU -3276501248,3276501503,DE +3276501168,3276501183,EU +3276501184,3276501503,DE 3276501504,3276502271,EU 3276502272,3276505087,DE 3276505088,3276505231,ZA @@ -77457,15 +82428,15 @@ 3276533792,3276533823,EU 3276533824,3276533887,IE 3276533888,3276533919,EU -3276533920,3276534015,IE +3276533920,3276533951,IE +3276533952,3276533983,EU +3276533984,3276534015,IE 3276534016,3276534399,EU 3276534400,3276534415,GB 3276534416,3276534431,NL 3276534432,3276534463,GB 3276534464,3276534495,FR -3276534496,3276534511,EU -3276534512,3276534527,NL -3276534528,3276534543,EU +3276534496,3276534543,EU 3276534544,3276534591,NL 3276534592,3276534623,EU 3276534624,3276534671,NL @@ -77482,8 +82453,8 @@ 3276535040,3276535063,RU 3276535064,3276535071,LV 3276535072,3276535103,RU -3276535104,3276535295,EU -3276535296,3276535319,FI +3276535104,3276535311,EU +3276535312,3276535319,FI 3276535320,3276535335,EU 3276535336,3276535359,FI 3276535360,3276535375,EU @@ -77554,14 +82525,14 @@ 3276697088,3276697599,GB 3276697600,3276698111,UA 3276698112,3276699647,RU -3276699648,3276700159,FR +3276699648,3276700159,DE 3276700160,3276700671,KZ 3276700672,3276701183,RO 3276701184,3276701695,RU 3276701696,3276709887,SE 3276709888,3276718079,DE 3276718080,3276726271,IT -3276726272,3276727295,PK +3276726272,3276727295,DE 3276727296,3276728319,ES 3276728320,3276729343,UA 3276729344,3276730367,PL @@ -77983,12 +82954,11 @@ 3276872480,3276872511,GB 3276872512,3276872703,DE 3276872704,3276873727,GB -3276873728,3276873735,ES -3276873736,3276873743,GB -3276873744,3276874303,ES +3276873728,3276874303,ES 3276874304,3276874367,GB 3276874368,3276874751,ES -3276874752,3276875263,GB +3276874752,3276875007,NL +3276875008,3276875263,GB 3276875264,3276875775,NL 3276875776,3276876799,GB 3276876800,3276876823,AT @@ -78003,7 +82973,10 @@ 3276877824,3276877831,BG 3276877832,3276878079,GB 3276878080,3276878335,BG -3276878336,3276879871,GB +3276878336,3276879359,GB +3276879360,3276879423,TR +3276879424,3276879615,GB +3276879616,3276879871,TR 3276879872,3276880639,DK 3276880640,3276881919,GB 3276881920,3276883391,IT @@ -78013,21 +82986,25 @@ 3276883968,3276884487,PL 3276884488,3276886015,GB 3276886016,3276886271,RO -3276886272,3276888575,GB +3276886272,3276887807,GB +3276887808,3276888063,DE +3276888064,3276888575,GB 3276888576,3276888831,AT 3276888832,3276889087,GB 3276889088,3276890111,AT 3276890112,3276890135,US 3276890136,3276890143,GB -3276890144,3276890175,US -3276890176,3276890367,GB +3276890144,3276890191,US +3276890192,3276890367,GB 3276890368,3276890623,US 3276890624,3276891135,IT -3276891136,3276891903,GB -3276891904,3276892159,US -3276892160,3276893183,IT -3276893184,3276895231,GB -3276895232,3276895999,IT +3276891136,3276891391,GB +3276891392,3276892159,US +3276892160,3276893695,IT +3276893696,3276894463,GB +3276894464,3276894719,IT +3276894720,3276894975,GB +3276894976,3276895999,IT 3276896000,3276896255,CZ 3276896256,3276896831,BE 3276896832,3276896847,SE @@ -78036,12 +83013,12 @@ 3276896928,3276897023,GB 3276897024,3276897279,BE 3276897280,3276898303,GB -3276898304,3276898423,CH -3276898424,3276898431,GB -3276898432,3276898775,CH +3276898304,3276898775,CH 3276898776,3276898783,GB -3276898784,3276900031,CH -3276900032,3276900095,GB +3276898784,3276899135,CH +3276899136,3276899167,GB +3276899168,3276900039,CH +3276900040,3276900095,GB 3276900096,3276900351,CH 3276900352,3276902143,GB 3276902144,3276902151,CH @@ -78052,9 +83029,7 @@ 3276902576,3276902655,GB 3276902656,3276903422,SE 3276903423,3276903423,GB -3276903424,3276903487,SE -3276903488,3276903551,GB -3276903552,3276903679,SE +3276903424,3276903679,SE 3276903680,3276905471,GB 3276905472,3276905727,SE 3276905728,3276906239,BE @@ -78062,8 +83037,8 @@ 3276906280,3276906495,GB 3276906496,3276906863,NL 3276906864,3276906879,GB -3276906880,3276907263,NL -3276907264,3276910079,GB +3276906880,3276907519,NL +3276907520,3276910079,GB 3276910080,3276910591,NL 3276910592,3276910967,IT 3276910968,3276910975,GB @@ -78071,9 +83046,7 @@ 3276912616,3276912623,GB 3276912624,3276913919,IT 3276913920,3276913983,US -3276913984,3276914095,IT -3276914096,3276914111,GB -3276914112,3276914687,IT +3276913984,3276914687,IT 3276914688,3276915455,ES 3276915456,3276915487,GB 3276915488,3276916175,ES @@ -78107,8 +83080,8 @@ 3276924160,3276924287,FR 3276924288,3276924415,GB 3276924416,3276925951,FR -3276925952,3276926719,GB -3276926720,3276926735,FR +3276925952,3276926463,GB +3276926464,3276926735,FR 3276926736,3276931071,GB 3276931072,3276939263,KZ 3276939264,3276955647,DE @@ -78206,7 +83179,9 @@ 3277330432,3277330943,RO 3277330944,3277331455,UA 3277331456,3277332479,PL +3277332480,3277332991,NL 3277332992,3277333503,AT +3277333504,3277334015,DK 3277334016,3277334527,RO 3277334528,3277335039,RU 3277335040,3277335551,UA @@ -78226,6 +83201,7 @@ 3277342720,3277343231,UA 3277343232,3277343743,BG 3277343744,3277344767,UA +3277344768,3277345279,RO 3277345280,3277345791,SI 3277345792,3277346303,FR 3277346304,3277346815,GB @@ -78402,7 +83378,7 @@ 3277710848,3277711359,PL 3277711360,3277711871,NO 3277711872,3277712383,IL -3277712384,3277712895,FR +3277712384,3277712895,DE 3277712896,3277713407,NL 3277713408,3277713919,RU 3277713920,3277714943,DE @@ -78411,9 +83387,7 @@ 3277716480,3277716991,SE 3277716992,3277717503,IT 3277717504,3277725695,YE -3277725696,3277728367,CH -3277728368,3277728383,NL -3277728384,3277733887,CH +3277725696,3277733887,CH 3277733888,3277742079,DE 3277742080,3277750271,FI 3277750272,3277766655,IT @@ -78561,7 +83535,11 @@ 3278168120,3278168175,SE 3278168176,3278168191,DK 3278168192,3278176255,SE -3278176256,3278232511,FR +3278176256,3278220799,FR +3278220800,3278221055,GB +3278221056,3278221087,FR +3278221088,3278221119,GB +3278221120,3278232511,FR 3278232512,3278232575,MC 3278232576,3278241791,FR 3278241792,3278307327,GB @@ -78601,7 +83579,19 @@ 3278781952,3278782463,RU 3278782464,3278782975,GB 3278782976,3278783231,DE -3278783232,3278790655,GB +3278783232,3278784703,GB +3278784704,3278784735,US +3278784736,3278786911,GB +3278786912,3278786943,LU +3278786944,3278787567,GB +3278787568,3278787583,FR +3278787584,3278788095,GB +3278788096,3278788223,US +3278788224,3278789935,GB +3278789936,3278789951,US +3278789952,3278790095,GB +3278790096,3278790111,ES +3278790112,3278790655,GB 3278790656,3278807039,IT 3278807040,3278815231,GB 3278815232,3278823423,AT @@ -81086,7 +86076,7 @@ 3279057920,3279058431,RO 3279058432,3279058943,RU 3279058944,3279059455,UA -3279059968,3279060479,RU +3279059456,3279060479,RU 3279060480,3279060991,PL 3279060992,3279069183,UA 3279069184,3279077375,PL @@ -81295,9 +86285,7 @@ 3279593488,3279593535,GB 3279593536,3279593567,DE 3279593568,3279593599,GB -3279593600,3279593727,DE -3279593728,3279593983,GB -3279593984,3279594495,DE +3279593600,3279594495,DE 3279594496,3279595903,GB 3279595904,3279595999,DE 3279596000,3279596023,GB @@ -81333,6 +86321,7 @@ 3279986688,3279987199,NL 3279987200,3279987711,RU 3279987712,3279988223,RO +3279988224,3279988735,AM 3279988736,3279989247,RO 3279989248,3279989759,AT 3279989760,3279990271,PL @@ -81687,7 +86676,7 @@ 3280785408,3280787455,DE 3280787456,3280788479,GB 3280788480,3280789503,PL -3280789504,3280790527,UA +3280789504,3280791551,UA 3280791552,3280792575,DE 3280792576,3280793599,PL 3280793600,3280794623,NL @@ -82230,11 +87219,41 @@ 3280948720,3280948735,DE 3280948736,3280949503,GB 3280949504,3280950015,DE -3280950016,3280951039,GB -3280951040,3280951807,DE -3280951808,3280952327,GB -3280952328,3280952335,DE -3280952336,3280953104,GB +3280950016,3280950655,GB +3280950656,3280950687,DE +3280950688,3280951039,GB +3280951040,3280952063,DE +3280952064,3280952067,GB +3280952068,3280952079,DE +3280952080,3280952087,GB +3280952088,3280952095,DE +3280952096,3280952111,GB +3280952112,3280952119,DE +3280952120,3280952127,GB +3280952128,3280952159,DE +3280952160,3280952183,GB +3280952184,3280952191,DE +3280952192,3280952207,GB +3280952208,3280952223,DE +3280952224,3280952319,GB +3280952320,3280952323,DE +3280952324,3280952327,GB +3280952328,3280952343,DE +3280952344,3280952351,GB +3280952352,3280952359,DE +3280952360,3280952383,GB +3280952384,3280952559,DE +3280952560,3280952575,GB +3280952576,3280952623,DE +3280952624,3280952631,GB +3280952632,3280952687,DE +3280952688,3280952695,GB +3280952696,3280952703,DE +3280952704,3280952743,GB +3280952744,3280952767,DE +3280952768,3280952783,GB +3280952784,3280952815,DE +3280952816,3280953104,GB 3280953105,3280953135,DE 3280953136,3280953343,GB 3280953344,3280953371,DE @@ -82302,13 +87321,15 @@ 3280958288,3280958335,DE 3280958336,3280958343,GB 3280958344,3280958350,DE -3280958351,3280958591,GB -3280958592,3280958599,DE -3280958600,3280958607,GB -3280958608,3280958615,DE -3280958616,3280958655,GB -3280958656,3280958663,DE -3280958664,3280958975,GB +3280958351,3280958463,GB +3280958464,3280958851,DE +3280958852,3280958863,GB +3280958864,3280958871,DE +3280958872,3280958879,GB +3280958880,3280958915,DE +3280958916,3280958919,GB +3280958920,3280958927,DE +3280958928,3280958975,GB 3280958976,3280959231,DE 3280959232,3280959487,GB 3280959488,3280959711,DE @@ -82330,7 +87351,11 @@ 3280961888,3280961919,DE 3280961920,3280962111,GB 3280962112,3280962175,DE -3280962176,3280963199,GB +3280962176,3280962815,GB +3280962816,3280962831,DE +3280962832,3280962847,GB +3280962848,3280962887,DE +3280962888,3280963199,GB 3280963200,3280963263,DE 3280963264,3280963303,GB 3280963304,3280963311,DE @@ -82379,8 +87404,7 @@ 3280969604,3280969631,GB 3280969632,3280969647,DE 3280969648,3280970239,GB -3280970240,3280970367,NL -3280970368,3280970399,DE +3280970240,3280970399,DE 3280970400,3280970431,GB 3280970432,3280970463,DE 3280970464,3280970507,GB @@ -82425,15 +87449,17 @@ 3280972352,3280972415,DE 3280972416,3280974079,GB 3280974080,3280974335,DE -3280974336,3280974591,GB +3280974336,3280974399,GB +3280974400,3280974415,DE +3280974416,3280974591,GB 3280974592,3280974847,DE 3280974848,3280976695,GB 3280976696,3280976719,DE 3280976720,3280976831,GB 3280976832,3280976847,DE 3280976848,3280976895,GB -3280976896,3280977152,DE -3280977153,3280977439,GB +3280976896,3280977407,DE +3280977408,3280977439,GB 3280977440,3280977479,DE 3280977480,3280977487,GB 3280977488,3280977495,DE @@ -83015,8 +88041,8 @@ 3280993200,3280993215,DE 3280993216,3280993247,GB 3280993248,3280993263,DE -3280993264,3280993791,GB -3280993792,3280994303,DE +3280993264,3280993535,GB +3280993536,3280994303,DE 3280994304,3280994559,GB 3280994560,3280994815,RU 3280994816,3280995071,NL @@ -83292,8 +88318,7 @@ 3282284544,3282284551,FI 3282284552,3282284567,SE 3282284568,3282284575,FR -3282284576,3282284579,SE -3282284580,3282284583,FI +3282284576,3282284583,FI 3282284584,3282284591,SE 3282284592,3282284595,FR 3282284596,3282284599,SE @@ -83488,7 +88513,9 @@ 3283279872,3283288063,SE 3283288064,3283288575,DE 3283288576,3283288579,BE -3283288580,3283291999,DE +3283288580,3283291487,DE +3283291488,3283291491,ES +3283291492,3283291999,DE 3283292000,3283292015,CH 3283292016,3283304919,DE 3283304920,3283304923,ES @@ -83496,14 +88523,22 @@ 3283305088,3283305127,FR 3283305128,3283305711,DE 3283305712,3283305715,ES -3283305716,3283318975,DE +3283305716,3283318687,DE +3283318688,3283318691,ES +3283318692,3283318975,DE 3283318976,3283318991,NL 3283318992,3283318999,ES -3283319000,3283321087,DE +3283319000,3283319167,DE +3283319168,3283319171,ES +3283319172,3283321087,DE 3283321088,3283321119,GB 3283321120,3283321151,DE 3283321152,3283321155,ES -3283321156,3283329923,DE +3283321156,3283321711,DE +3283321712,3283321715,ES +3283321716,3283321871,DE +3283321872,3283321875,ES +3283321876,3283329923,DE 3283329924,3283329927,ES 3283329928,3283419135,DE 3283419136,3283484671,DK @@ -83965,7 +89000,7 @@ 3283635712,3283636223,RU 3283636224,3283636735,UA 3283636736,3283637247,PL -3283637248,3283637759,FR +3283637248,3283637759,DE 3283637760,3283638271,UA 3283638272,3283638783,PL 3283638784,3283639295,RU @@ -84086,7 +89121,7 @@ 3283961856,3283962879,RU 3283962880,3283963903,PL 3283963904,3283964927,RU -3283965952,3283966975,PL +3283964928,3283966975,PL 3283966976,3283967999,DE 3283968000,3283969023,UA 3283969024,3283970047,RU @@ -84102,7 +89137,7 @@ 3283977728,3283978751,RU 3283978752,3283979263,RO 3283979264,3283979775,IL -3283979776,3283980287,A1 +3283979776,3283980287,NO 3283980288,3283980799,RO 3283980800,3283981823,CH 3283981824,3283983359,RO @@ -84238,7 +89273,7 @@ 3284043776,3284044031,HR 3284044032,3284044287,FR 3284044288,3284044799,DE -3284044800,3284045055,CH +3284044800,3284045055,CZ 3284045056,3284045311,LT 3284045312,3284045567,DE 3284045568,3284045823,AT @@ -84293,9 +89328,9 @@ 3284091392,3284091903,GB 3284091904,3284092415,DE 3284092416,3284092927,GB +3284092928,3284093439,UA 3284093440,3284093951,IR 3284093952,3284094463,RU -3284094464,3284094975,DK 3284094976,3284095487,UA 3284095488,3284095999,RU 3284096000,3284096511,IT @@ -84307,10 +89342,11 @@ 3284099072,3284099583,RO 3284099584,3284100095,GR 3284100096,3284100607,AM -3284100608,3284101119,HK +3284100608,3284101119,EU 3284101120,3284101631,UA 3284101632,3284102143,GB 3284102144,3284102655,DK +3284102656,3284103167,AT 3284103168,3284103679,NL 3284103680,3284104191,DE 3284104192,3284104703,NO @@ -84365,6 +89401,7 @@ 3284131840,3284132351,DE 3284132352,3284132863,UA 3284132864,3284133375,HU +3284133376,3284133887,TM 3284133888,3284134399,PL 3284134400,3284134911,UA 3284134912,3284135935,RU @@ -84409,7 +89446,7 @@ 3284697216,3284697279,FR 3284697280,3284705279,GB 3284705280,3284713471,FR -3284713472,3284713727,EU +3284713472,3284713727,DE 3284713728,3284713983,GR 3284713984,3284714239,DE 3284714240,3284714495,FR @@ -84451,6 +89488,7 @@ 3284726272,3284726783,DE 3284726784,3284727295,UA 3284727296,3284727807,MK +3284727808,3284728319,RU 3284728320,3284728831,UA 3284728832,3284729343,PT 3284729344,3284729855,FR @@ -84517,7 +89555,7 @@ 3285118976,3285119487,RU 3285119488,3285119999,RO 3285120000,3285120511,RU -3285120512,3285121023,MD +3285120512,3285121023,RO 3285121024,3285121535,SE 3285121536,3285122047,CY 3285122048,3285123071,RU @@ -84664,9 +89702,9 @@ 3285457792,3285457919,IT 3285457920,3285457951,GB 3285457952,3285457983,EU -3285457984,3285458031,GB -3285458032,3285458039,EU -3285458040,3285458111,GB +3285457984,3285458095,GB +3285458096,3285458103,EU +3285458104,3285458111,GB 3285458112,3285458175,EU 3285458176,3285458943,GB 3285458944,3285458975,DK @@ -84738,8 +89776,8 @@ 3285463520,3285463615,BE 3285463616,3285463647,FR 3285463648,3285463663,EU -3285463664,3285463759,BE -3285463760,3285463775,EU +3285463664,3285463743,BE +3285463744,3285463775,EU 3285463776,3285463839,BE 3285463840,3285463855,EU 3285463856,3285463991,BE @@ -84759,14 +89797,16 @@ 3285465088,3285465103,DK 3285465104,3285465151,EU 3285465152,3285465175,DK -3285465176,3285465183,EU -3285465184,3285465231,DK +3285465176,3285465215,EU +3285465216,3285465231,DK 3285465232,3285465247,EU 3285465248,3285465343,DK 3285465344,3285465663,EU 3285465664,3285465727,DE 3285465728,3285465855,EU -3285465856,3285466367,DE +3285465856,3285465903,DE +3285465904,3285465911,EU +3285465912,3285466367,DE 3285466368,3285466447,CH 3285466448,3285466455,EU 3285466456,3285466463,CH @@ -84896,9 +89936,7 @@ 3285479424,3285479807,CH 3285479808,3285480447,EU 3285480448,3285480575,CH -3285480576,3285480703,EU -3285480704,3285480719,CH -3285480720,3285480735,EU +3285480576,3285480735,EU 3285480736,3285480767,CH 3285480768,3285480775,EU 3285480776,3285480895,CH @@ -84938,7 +89976,8 @@ 3285487360,3285487391,NL 3285487392,3285487423,EU 3285487424,3285487487,GB -3285487488,3285487679,NL +3285487488,3285487615,EU +3285487616,3285487679,NL 3285487680,3285487743,EU 3285487744,3285488127,NL 3285488128,3285488895,EU @@ -84990,7 +90029,7 @@ 3285496496,3285496527,FR 3285496528,3285496543,EU 3285496544,3285496575,FR -3285496576,3285496607,GB +3285496576,3285496607,EU 3285496608,3285496815,DE 3285496816,3285496831,EU 3285496832,3285497855,DE @@ -85929,7 +90968,7 @@ 3285762560,3285763071,RU 3285763072,3285763583,GB 3285763584,3285764095,DE -3285764608,3285765119,RU +3285764096,3285765119,RU 3285765120,3285766143,UA 3285766144,3285766655,DE 3285766656,3285767679,UA @@ -85978,8 +91017,7 @@ 3285909504,3285910015,GB 3285910016,3285910271,EU 3285910272,3285910303,ES -3285910304,3285910319,GB -3285910320,3285910335,EU +3285910304,3285910335,GB 3285910336,3285910399,ES 3285910400,3285910407,IT 3285910408,3285910463,GB @@ -86004,7 +91042,8 @@ 3285916928,3285917183,YE 3285917184,3285917439,ES 3285917440,3285917695,YE -3285917696,3285918975,EU +3285917696,3285918719,EU +3285918720,3285918975,SE 3285918976,3285919231,YE 3285919232,3285919487,GB 3285919488,3285919743,UA @@ -86020,7 +91059,7 @@ 3285922192,3285922207,DE 3285922208,3285922239,ES 3285922240,3285922303,RU -3285922304,3285922559,EU +3285922304,3285922559,DE 3285922560,3285922815,GB 3285922816,3285923327,KW 3285923328,3285924415,ES @@ -86099,8 +91138,7 @@ 3285943560,3285943567,EU 3285943568,3285943575,ES 3285943576,3285943583,DE -3285943584,3285943615,ES -3285943616,3285943679,EU +3285943584,3285943679,EU 3285943680,3285943807,GB 3285943808,3285944319,EU 3285944320,3285944831,US @@ -86119,9 +91157,9 @@ 3285947904,3285948159,NL 3285948160,3285948671,GB 3285948672,3285948927,SA -3285948928,3285949183,EU +3285948928,3285949183,GB 3285949184,3285949439,ES -3285949440,3285949695,EU +3285949440,3285949695,DE 3285949696,3285949823,ES 3285949824,3285949855,EU 3285949856,3285949887,ES @@ -86132,7 +91170,8 @@ 3285950784,3285950975,EU 3285950976,3285951231,NL 3285951232,3285951487,IT -3285951488,3285951647,EU +3285951488,3285951615,EU +3285951616,3285951647,GB 3285951648,3285951679,ES 3285951680,3285951743,EU 3285951744,3285951999,GB @@ -86151,7 +91190,7 @@ 3285958848,3285958895,NL 3285958896,3285959039,GB 3285959040,3285959167,DE -3285959168,3285959423,EU +3285959168,3285959423,NL 3285959424,3285959935,GB 3285959936,3285960191,ES 3285960192,3285960447,FR @@ -86216,6 +91255,7 @@ 3286312960,3286313983,RO 3286313984,3286314495,CH 3286314496,3286315007,IL +3286315008,3286315519,UA 3286315520,3286316031,DE 3286316032,3286316543,UA 3286316544,3286317055,NL @@ -86272,7 +91312,37 @@ 3286384640,3286401023,DE 3286401024,3286409215,GB 3286409216,3286417407,DE +3286417408,3286417663,UA 3286417664,3286417919,IT +3286417920,3286418175,DE +3286418176,3286418431,AT +3286418432,3286418687,TR +3286418688,3286418943,UA +3286418944,3286419199,PL +3286419200,3286419455,DK +3286419456,3286419711,DE +3286419712,3286419967,GB +3286419968,3286420223,FI +3286420224,3286420735,RU +3286420736,3286420991,NL +3286420992,3286421247,RO +3286421248,3286421503,PL +3286421504,3286421759,KZ +3286421760,3286422015,RU +3286422016,3286422271,MD +3286422272,3286422527,FR +3286422528,3286422783,RO +3286422784,3286423039,UA +3286423040,3286423295,RO +3286423296,3286423551,UA +3286423552,3286423807,RU +3286423808,3286424063,CZ +3286424064,3286424319,LV +3286424320,3286424575,FR +3286424576,3286424831,UA +3286424832,3286425087,TR +3286425088,3286425343,RU +3286425344,3286425599,IT 3286425600,3286433791,KW 3286433792,3286499327,DE 3286499328,3286519039,HU @@ -86425,7 +91495,7 @@ 3286914048,3286915071,BG 3286915072,3286916095,FR 3286916096,3286917119,DE -3286917120,3286917631,RU +3286917120,3286918143,RU 3286918144,3286919167,AT 3286919168,3286920191,DE 3286920192,3286921215,RU @@ -86535,9 +91605,7 @@ 3287182336,3287185407,UA 3287185408,3287186431,SE 3287186432,3287187455,FI -3287187456,3287196159,DE -3287196160,3287196415,GB -3287196416,3287196671,DE +3287187456,3287196671,DE 3287196672,3287198719,PL 3287198720,3287199743,GB 3287199744,3287201791,UA @@ -86612,7 +91680,6 @@ 3287448064,3287448575,GB 3287448576,3287449087,PL 3287449088,3287449599,BG -3287449600,3287450111,MD 3287450112,3287450623,DK 3287450624,3287451135,SE 3287451136,3287451647,PL @@ -86657,6 +91724,7 @@ 3287465728,3287465983,FR 3287465984,3287467007,DK 3287467008,3287468031,SA +3287468032,3287469055,UA 3287469056,3287471103,DE 3287471104,3287472127,GB 3287472128,3287473151,FI @@ -86818,6 +91886,7 @@ 3287704064,3287704575,NL 3287704576,3287705087,LU 3287705088,3287705599,RU +3287705600,3287706111,SA 3287706112,3287706623,RU 3287706624,3287707135,GB 3287707136,3287707647,FR @@ -86827,6 +91896,7 @@ 3287709184,3287709695,RU 3287709696,3287710207,SE 3287710208,3287710719,RU +3287710720,3287711231,UA 3287711232,3287711743,PL 3287711744,3287711983,GB 3287711984,3287711999,FR @@ -86871,6 +91941,7 @@ 3287732224,3287734271,IT 3287734272,3287734527,PL 3287734528,3287734783,UA +3287734784,3287735039,RU 3287735040,3287735295,SE 3287735296,3287736319,IT 3287736320,3287736831,PL @@ -87264,6 +92335,7 @@ 3290486784,3290488831,SC 3290488832,3290489343,NG 3290489344,3290489855,KE +3290489856,3290490367,EG 3290497024,3290955775,ZA 3290955776,3290980351,CR 3290980352,3290984447,ZA @@ -87304,6 +92376,7 @@ 3291331584,3291332607,MW 3291332608,3291348991,NG 3291348992,3291353087,MZ +3291353088,3291353343,KE 3291353344,3291353599,EG 3291353600,3291354111,ZM 3291354112,3291354367,NG @@ -87315,6 +92388,8 @@ 3291355648,3291355903,EG 3291355904,3291356159,NG 3291356160,3291356415,NA +3291356416,3291356671,EG +3291356672,3291357183,ZA 3291357184,3291365375,MU 3291365376,3291367423,ZA 3291367424,3291369471,AO @@ -87422,6 +92497,7 @@ 3302534144,3302535167,ZA 3302535168,3302536191,UG 3302536192,3302537215,GH +3302537216,3302538239,NG 3302552576,3302552831,TZ 3302552832,3302553087,KE 3302553088,3302553343,EG @@ -87457,10 +92533,14 @@ 3302950912,3302951423,ZA 3302951424,3302951679,SL 3302951680,3302951935,KE +3302951936,3302952191,RW 3305111552,3307208703,TN +3315597312,3316121599,EG 3317694464,3318218751,EG 3318218752,3318743039,DZ -3319791616,3321888767,MU +3319791616,3320053759,MU +3320053760,3320315903,ZA +3321884672,3321888767,MU 3321954304,3321970687,US 3322019840,3322023935,US 3322023936,3322028031,CL @@ -87937,7 +93017,9 @@ 3332737024,3332738047,PM 3332738048,3332744191,CA 3332744192,3332745215,PM -3332745216,3332947967,CA +3332745216,3332906495,CA +3332906496,3332909567,US +3332909568,3332947967,CA 3332947968,3332948223,US 3332948224,3332966143,CA 3332966144,3332966399,US @@ -88167,7 +93249,9 @@ 3339082752,3339086335,US 3339086336,3339086591,JP 3339086592,3339091967,US -3339091968,3339094015,NL +3339091968,3339092991,NL +3339092992,3339093503,US +3339093504,3339094015,NL 3339094016,3339142655,US 3339142656,3339142911,NL 3339142912,3339145727,US @@ -88177,11 +93261,32 @@ 3339147008,3339147775,CA 3339147776,3339148031,MS 3339148032,3339149311,US -3339190272,3339669503,US +3339157504,3339159551,US +3339159552,3339160575,CA +3339160576,3339164671,US +3339164672,3339165695,CA +3339165696,3339167743,US +3339167744,3339168767,PR +3339168768,3339177983,US +3339177984,3339179007,PR +3339179008,3339180031,US +3339180032,3339181055,CA +3339181056,3339184127,US +3339184128,3339186175,CA +3339186176,3339669503,US 3339669504,3339671807,CA 3339672576,3339673599,US -3339714560,3339923455,US +3339681792,3339747327,US +3339747328,3339748351,CA +3339748352,3339753471,US +3339753472,3339754495,CA +3339754496,3339760639,US +3339760640,3339761663,CA +3339761664,3339923455,US 3339923456,3339927551,CA +3339943936,3339952127,US +3339952128,3339956223,CA +3339956224,3339968511,US 3339976704,3340107775,US 3340107776,3340369919,CA 3340369920,3340386559,US @@ -88198,9 +93303,7 @@ 3340925072,3340925079,CA 3340925080,3340925199,US 3340925200,3340925207,RU -3340925208,3340925215,US -3340925216,3340925223,AU -3340925224,3340925239,US +3340925208,3340925239,US 3340925240,3340925247,MX 3340925248,3340925255,AR 3340925256,3340925279,US @@ -88213,40 +93316,43 @@ 3340925472,3340925479,CH 3340925480,3340925535,US 3340925536,3340925543,RU -3340925544,3340925599,US -3340925600,3340925607,HU +3340925544,3340925607,US 3340925608,3340925615,AU 3340925616,3340925631,US 3340925632,3340925639,BR 3340925640,3340925647,CO 3340925648,3340925703,US 3340925704,3340925711,FR -3340925712,3340925727,US +3340925712,3340925719,US +3340925720,3340925727,CA 3340925728,3340925735,BR 3340925736,3340925743,UA 3340925744,3340925823,US 3340925824,3340925887,GB -3340925888,3340925919,US +3340925888,3340925895,CA +3340925896,3340925919,US 3340925920,3340925927,PT 3340925928,3340925935,US 3340925936,3340925943,DE -3340925944,3340925991,US -3340925992,3340925999,CA +3340925944,3340925999,US 3340926000,3340926007,RU 3340926008,3340926015,US 3340926016,3340926023,RU 3340926024,3340926079,US 3340926080,3340926087,CA 3340926088,3340926095,US -3340926096,3340926111,CA -3340926112,3340926151,US +3340926096,3340926103,AU +3340926104,3340926111,CA +3340926112,3340926143,US +3340926144,3340926151,BR 3340926152,3340926159,CN 3340926160,3340926175,AR -3340926176,3340926215,US +3340926176,3340926199,US +3340926200,3340926207,DE +3340926208,3340926215,US 3340926216,3340926223,BO 3340926224,3340926231,PT -3340926232,3340926239,US -3340926240,3340926247,CA +3340926232,3340926247,CA 3340926248,3340926255,AR 3340926256,3340926263,AU 3340926264,3340926287,US @@ -88257,24 +93363,34 @@ 3340926344,3340926351,GB 3340926352,3340926359,US 3340926360,3340926367,DE -3340926368,3340926439,US +3340926368,3340926407,US +3340926408,3340926415,GB +3340926416,3340926439,US 3340926440,3340926447,NO 3340926448,3340926455,DE 3340926456,3340926463,US 3340926464,3340926479,MX -3340926480,3340926503,US +3340926480,3340926495,US +3340926496,3340926503,DE 3340926504,3340926511,MX 3340926512,3340926543,US 3340926544,3340926559,GB -3340926560,3340926623,US +3340926560,3340926599,US +3340926600,3340926607,CA +3340926608,3340926615,RU +3340926616,3340926623,US 3340926624,3340926655,GB 3340926656,3340926663,BE 3340926664,3340926671,GB -3340926672,3340926791,US +3340926672,3340926679,US +3340926680,3340926687,CL +3340926688,3340926791,US 3340926792,3340926799,GB -3340926800,3340926823,US -3340926824,3340926831,CA -3340926832,3340926927,US +3340926800,3340926815,US +3340926816,3340926831,CA +3340926832,3340926903,US +3340926904,3340926911,IL +3340926912,3340926927,US 3340926928,3340926935,ES 3340926936,3340926951,US 3340926952,3340926959,BR @@ -88564,7 +93680,73 @@ 3351642112,3351698431,CA 3351698432,3351904255,US 3351904256,3351969791,CA -3351969792,3352035327,IL +3351969792,3351970047,IL +3351970048,3351970815,US +3351970816,3351971071,IL +3351971072,3351971583,US +3351971584,3351973119,IL +3351973120,3351973887,US +3351973888,3351987199,IL +3351987200,3351987455,US +3351987456,3351990783,IL +3351990784,3351991039,US +3351991040,3351996671,IL +3351996672,3351996927,US +3351996928,3352002303,IL +3352002304,3352002559,US +3352002560,3352002815,IL +3352002816,3352003071,US +3352003072,3352006911,IL +3352006912,3352007167,US +3352007168,3352007423,IL +3352007424,3352007583,US +3352007584,3352007679,IL +3352007680,3352007967,US +3352007968,3352008063,IL +3352008064,3352009311,US +3352009312,3352009343,IL +3352009344,3352010815,US +3352010816,3352010879,IL +3352010880,3352011199,US +3352011200,3352011263,IL +3352011264,3352014719,US +3352014720,3352014847,IL +3352014848,3352014927,US +3352014928,3352014943,IL +3352014944,3352015999,US +3352016000,3352016127,IL +3352016128,3352020671,US +3352020672,3352020703,IL +3352020704,3352020991,US +3352020992,3352023167,IL +3352023168,3352023343,US +3352023344,3352023359,IL +3352023360,3352023679,US +3352023680,3352023807,IL +3352023808,3352024319,US +3352024320,3352024447,IL +3352024448,3352025087,US +3352025088,3352025215,IL +3352025216,3352026623,US +3352026624,3352026751,IL +3352026752,3352027295,US +3352027296,3352027327,IL +3352027328,3352028799,US +3352028800,3352028927,IL +3352028928,3352030047,US +3352030048,3352030079,IL +3352030080,3352030143,US +3352030144,3352030175,IL +3352030176,3352030383,US +3352030384,3352030399,IL +3352030400,3352032255,US +3352032256,3352032767,IL +3352032768,3352033279,US +3352033280,3352033791,IL +3352033792,3352034303,US +3352034304,3352034815,IL +3352034816,3352035071,US +3352035072,3352035327,IL 3352035328,3352036351,CA 3352036352,3352046591,US 3352046592,3352049663,CA @@ -88614,11 +93796,12 @@ 3354955776,3354956031,AR 3354956032,3355013119,US 3355013120,3355017215,CA -3355017216,3355052031,US -3355052032,3355054079,CA +3355017216,3355052287,US +3355052288,3355053311,CA +3355053312,3355053567,US 3355054080,3355260927,US 3355260928,3355262975,CA -3355262976,3355310335,US +3355262976,3355310591,US 3355310592,3355311103,CA 3355312128,3355319295,US 3355319296,3355320319,CA @@ -88635,7 +93818,9 @@ 3355407360,3355408383,PR 3355408384,3355412479,US 3355412480,3355412991,BE -3355412992,3355422719,US +3355412992,3355430911,US +3355430912,3355432959,A1 +3355432960,3355443199,US 3355443200,3355445247,CO 3355445248,3355447295,BR 3355447296,3355447551,CU @@ -89020,6 +94205,7 @@ 3356272640,3356272895,US 3356272896,3356273151,EC 3356273152,3356273407,CO +3356273408,3356273663,CL 3356273664,3356274431,SV 3356274432,3356278783,CO 3356279040,3356279295,VE @@ -89369,7 +94555,9 @@ 3358725120,3358725375,AN 3358725376,3358728191,PR 3358728192,3358736383,CL -3358736384,3358738175,BB +3358736384,3358737111,BB +3358737112,3358737119,CA +3358737120,3358738175,BB 3358738176,3358738687,GD 3358738688,3358739711,BB 3358739712,3358739967,KN @@ -89523,7 +94711,11 @@ 3359498240,3359501311,PY 3359501312,3359502335,SR 3359506432,3359508479,US -3359508480,3359514623,VE +3359508480,3359511343,VE +3359511344,3359511348,US +3359511349,3359511447,VE +3359511448,3359511519,US +3359511520,3359514623,VE 3359514624,3359516671,GT 3359516672,3359516927,VG 3359516928,3359517183,US @@ -89543,7 +94735,6 @@ 3359589376,3359589631,UY 3359589632,3359596543,AR 3359596544,3359598591,CL -3359598592,3359600639,AN 3359600640,3359621119,AR 3359621120,3359637503,EC 3359637504,3359789583,MX @@ -90266,6 +95457,7 @@ 3389610240,3389610495,IN 3389610496,3389610751,HK 3389610752,3389611519,IN +3389611520,3389612031,AU 3389612032,3389616127,MY 3389616128,3389640703,IN 3389640704,3389644799,JP @@ -90541,7 +95733,6 @@ 3391843328,3391844351,VN 3391844352,3391845887,ID 3391845888,3391846399,PK -3391846400,3391847423,TW 3391847424,3391848447,PK 3391848448,3391852543,AU 3391852544,3391856639,CN @@ -90663,11 +95854,47 @@ 3392406528,3392407551,TH 3392407552,3392409599,ID 3392409600,3392413695,JP +3392413696,3392413951,PK +3392413952,3392414207,HK +3392414208,3392414463,PH +3392414464,3392414719,HK +3392414720,3392415231,AU +3392415232,3392415487,JP +3392415488,3392415743,VN +3392415744,3392415999,IN +3392416000,3392416255,AU +3392416256,3392416767,HK +3392416768,3392417023,IN +3392417024,3392417535,AU +3392417536,3392417791,HK +3392417792,3392418559,ID +3392418560,3392418815,SG +3392418816,3392419071,ID +3392419072,3392419327,IN +3392419328,3392419839,PH +3392419840,3392420351,IN +3392420352,3392420863,ID +3392420864,3392421119,AU +3392421120,3392421375,MY +3392421376,3392421887,IN 3392421888,3392430271,SG 3392430272,3392430303,TW 3392430304,3392432511,SG 3392432512,3392432543,TW 3392432544,3392438271,SG +3392438272,3392438527,IN +3392438528,3392438783,AU +3392438784,3392439039,IN +3392439040,3392439551,ID +3392439552,3392439807,PG +3392439808,3392440063,IN +3392440064,3392440319,AU +3392440320,3392440575,HK +3392440576,3392440831,ID +3392440832,3392441343,BD +3392441344,3392441855,ID +3392441856,3392442111,AU +3392442112,3392442367,IN 3392446464,3392450559,PH 3392450560,3392454655,JP 3392454656,3392462847,HK @@ -91149,7 +96376,6 @@ 3394838528,3394846719,NZ 3394846720,3394850815,IN 3394850816,3394854911,JP -3394854912,3394855935,PH 3394855936,3394856959,AP 3394856960,3394859007,IN 3394859008,3394860031,JP @@ -91398,7 +96624,8 @@ 3397410816,3397411327,AU 3397411328,3397411583,SG 3397411584,3397411839,HK -3397411840,3397412863,KR +3397411840,3397412351,KR +3397412352,3397412863,AU 3397412864,3397413375,HK 3397413376,3397413631,AU 3397413632,3397413887,IN @@ -93296,6 +98523,7 @@ 3413595904,3413596159,HK 3413596160,3413596671,AP 3413596672,3413597183,NP +3413597184,3413597695,AP 3413639168,3413704703,SG 3413704704,3413737471,MY 3413737472,3413753855,TH @@ -93472,7 +98700,7 @@ 3415814656,3415815167,TH 3415815168,3415816191,IN 3415816192,3415817215,JP -3415817216,3415818239,ID +3415817216,3415817727,ID 3415818240,3415822335,HK 3415822336,3415826431,MY 3415826432,3415834623,ID @@ -94632,7 +99860,9 @@ 3423184216,3423184287,CA 3423184288,3423184319,AU 3423184320,3423184895,CA -3423184896,3423207663,US +3423184896,3423204095,US +3423204096,3423204351,CA +3423204352,3423207663,US 3423207664,3423207671,AN 3423207672,3423208193,US 3423208194,3423208383,AN @@ -94660,7 +99890,8 @@ 3423264832,3423264863,NG 3423264864,3423265247,US 3423265248,3423265263,EC -3423265264,3423268863,US +3423265264,3423266815,US +3423266872,3423266879,US 3423268864,3423269135,CA 3423269136,3423269151,US 3423269152,3423269887,CA @@ -94693,7 +99924,219 @@ 3423313408,3423313919,VI 3423313920,3423329567,US 3423329568,3423329615,IL -3423329616,3423371263,US +3423329616,3423338527,US +3423338528,3423338535,AE +3423338536,3423338927,US +3423338928,3423338943,DE +3423338944,3423339375,US +3423339376,3423339383,DE +3423339384,3423340399,US +3423340400,3423340407,AU +3423340408,3423341511,US +3423341512,3423341519,PE +3423341520,3423341543,US +3423341544,3423341551,GB +3423341552,3423341567,US +3423341568,3423341599,GB +3423341600,3423342279,US +3423342280,3423342287,IL +3423342288,3423342631,US +3423342632,3423342639,IN +3423342640,3423342759,US +3423342760,3423342775,GB +3423342776,3423342967,US +3423342968,3423342975,AU +3423342976,3423343039,US +3423343040,3423343055,IN +3423343056,3423343135,US +3423343136,3423343151,PK +3423343152,3423343167,RO +3423343168,3423343327,US +3423343328,3423343343,AU +3423343344,3423344767,US +3423344768,3423344791,PK +3423344792,3423344807,CA +3423344808,3423344991,US +3423344992,3423344999,IE +3423345000,3423345087,US +3423345088,3423345095,CA +3423345096,3423345111,US +3423345112,3423345119,IE +3423345120,3423345399,US +3423345400,3423345407,IE +3423345408,3423345431,US +3423345432,3423345439,GT +3423345440,3423345503,US +3423345504,3423345535,AU +3423345536,3423345583,US +3423345584,3423345591,JO +3423345592,3423345623,US +3423345624,3423345631,GB +3423345632,3423345695,US +3423345696,3423345719,CA +3423345720,3423346127,US +3423346128,3423346143,CA +3423346144,3423346319,US +3423346320,3423346327,DK +3423346328,3423346487,US +3423346488,3423346495,AE +3423346496,3423346831,US +3423346832,3423346847,IM +3423346848,3423346943,US +3423346944,3423346951,CA +3423346952,3423346983,US +3423346984,3423346991,AE +3423346992,3423346999,AU +3423347000,3423347207,US +3423347208,3423347231,CA +3423347232,3423347447,US +3423347448,3423347455,CA +3423347456,3423347503,US +3423347504,3423347519,DE +3423347520,3423347783,US +3423347784,3423347791,SA +3423347792,3423347823,US +3423347824,3423347831,FR +3423347832,3423347919,US +3423347920,3423347927,AU +3423347928,3423348007,US +3423348008,3423348023,GB +3423348024,3423348071,US +3423348072,3423348095,AU +3423348096,3423349119,US +3423349120,3423349135,IT +3423349136,3423349247,US +3423349248,3423349263,AU +3423349264,3423349295,US +3423349296,3423349303,AU +3423349304,3423349327,US +3423349328,3423349335,DE +3423349336,3423349359,US +3423349360,3423349383,DE +3423349384,3423349679,US +3423349680,3423349687,IL +3423349688,3423350663,US +3423350664,3423350671,AU +3423350672,3423350719,US +3423350720,3423350727,CA +3423350728,3423350735,US +3423350736,3423350743,SA +3423350744,3423351831,US +3423351832,3423351839,AU +3423351840,3423352047,US +3423352048,3423352063,GB +3423352064,3423352255,US +3423352256,3423352271,CA +3423352272,3423352439,US +3423352440,3423352447,IL +3423352448,3423352503,US +3423352504,3423352511,IL +3423352512,3423352679,US +3423352680,3423352687,NL +3423352688,3423353031,US +3423353032,3423353039,AU +3423353040,3423353143,US +3423353144,3423353151,KH +3423353152,3423353295,US +3423353296,3423353303,BR +3423353304,3423353351,US +3423353352,3423353367,PA +3423353368,3423353471,US +3423353472,3423353479,KH +3423353480,3423353543,US +3423353544,3423353551,IN +3423353552,3423353791,US +3423353792,3423353807,GB +3423353808,3423353919,US +3423353920,3423353927,BB +3423353928,3423354031,US +3423354032,3423354039,IL +3423354040,3423354047,US +3423354048,3423354063,PK +3423354064,3423354095,US +3423354096,3423354103,CA +3423354104,3423354143,US +3423354144,3423354207,CA +3423354208,3423354335,US +3423354336,3423354359,PE +3423354360,3423354583,US +3423354584,3423354591,AU +3423354592,3423354655,US +3423354656,3423354663,AE +3423354664,3423354783,US +3423354784,3423354791,PA +3423354792,3423357111,US +3423357112,3423357127,AE +3423357128,3423357311,US +3423357312,3423357319,CO +3423357320,3423357511,US +3423357512,3423357519,DE +3423357520,3423357527,CO +3423357528,3423363463,US +3423363464,3423363471,AU +3423363472,3423363487,CA +3423363488,3423363511,US +3423363512,3423363519,MX +3423363520,3423363567,US +3423363568,3423363583,ZA +3423363584,3423363615,US +3423363616,3423363631,AU +3423363632,3423365775,US +3423365776,3423365783,CA +3423365784,3423365903,US +3423365904,3423365911,AE +3423365912,3423366223,US +3423366224,3423366239,AU +3423366240,3423366351,US +3423366352,3423366367,AU +3423366368,3423366479,US +3423366480,3423366495,IT +3423366496,3423366703,US +3423366704,3423366711,GB +3423366712,3423366991,US +3423366992,3423366999,CA +3423367000,3423367223,US +3423367224,3423367231,AU +3423367232,3423367279,US +3423367280,3423367303,BR +3423367304,3423367311,AU +3423367312,3423367343,US +3423367344,3423367359,NL +3423367360,3423367367,US +3423367368,3423367375,GB +3423367376,3423367807,US +3423367808,3423367823,IT +3423367824,3423367831,US +3423367832,3423367847,MX +3423367848,3423367903,US +3423367904,3423367919,IL +3423367920,3423367983,US +3423367984,3423368007,CA +3423368008,3423368103,US +3423368104,3423368111,CA +3423368112,3423368143,US +3423368144,3423368167,IL +3423368168,3423368255,US +3423368256,3423368287,CA +3423368288,3423368319,US +3423368320,3423368327,CA +3423368328,3423369327,US +3423369328,3423369343,BR +3423369344,3423369591,US +3423369592,3423369599,BR +3423369600,3423369719,US +3423369720,3423369727,AU +3423369728,3423370303,US +3423370304,3423370319,IL +3423370320,3423370551,US +3423370552,3423370559,IE +3423370560,3423370719,US +3423370720,3423370727,VG +3423370728,3423370775,US +3423370776,3423370783,LB +3423370784,3423370871,US +3423370872,3423370879,VG +3423370880,3423371263,US 3423371264,3423375359,ZA 3423375360,3423379455,CA 3423379456,3423393903,US @@ -94713,50 +100156,38 @@ 3423401192,3423402943,US 3423402944,3423402951,RU 3423402952,3423412223,US -3423412224,3423412239,CA 3423412240,3423412255,GR 3423412256,3423412287,TK 3423412288,3423412319,RU -3423412320,3423412415,CA +3423412320,3423412335,CA 3423412416,3423412479,SA 3423412480,3423412495,CA 3423412496,3423412511,US -3423412512,3423412543,CA 3423412544,3423412623,FR 3423412624,3423412639,NO -3423412640,3423412655,CA 3423412656,3423412671,RU -3423412672,3423412703,CA +3423412688,3423412703,CA 3423412704,3423412719,RU 3423412720,3423412727,US -3423412728,3423412751,CA +3423412736,3423412751,CA 3423412752,3423412767,US 3423412768,3423412783,FI -3423412784,3423412799,CA 3423412800,3423412815,US 3423412816,3423412831,GR -3423412832,3423412847,CA 3423412848,3423412863,US -3423412864,3423412927,CA 3423412928,3423412991,NL -3423412992,3423413055,CA 3423413056,3423413071,US -3423413072,3423413151,CA +3423413072,3423413135,CA 3423413152,3423413167,US -3423413168,3423413183,CA 3423413184,3423413247,RS 3423413248,3423413279,US 3423413280,3423413295,SE -3423413296,3423413311,CA 3423413312,3423413327,PT 3423413328,3423413343,US -3423413344,3423413375,CA 3423413376,3423413759,US -3423413760,3423414143,CA +3423414016,3423414143,CA 3423414144,3423414159,FR -3423414160,3423414191,CA 3423414192,3423414207,US -3423414208,3423414271,CA 3423414272,3423416319,US 3423417344,3423417470,US 3423417471,3423417480,AU @@ -94785,8 +100216,7 @@ 3423480528,3423480543,US 3423480544,3423480575,A2 3423480576,3423480831,DM -3423480832,3423480895,GY -3423480896,3423480927,A2 +3423480832,3423480927,A2 3423480928,3423480935,US 3423480936,3423480943,AO 3423480944,3423481007,A2 @@ -94797,11 +100227,13 @@ 3423481536,3423481551,NG 3423481552,3423481855,A2 3423481856,3423493631,US -3423493632,3423493887,CA +3423493632,3423493887,RU 3423493888,3423493903,ID 3423493904,3423493911,ES 3423493912,3423493927,TT -3423493928,3423498079,US +3423493928,3423493967,US +3423493968,3423493975,AT +3423493976,3423498079,US 3423498080,3423498087,CA 3423498088,3423533055,US 3423533056,3423535103,AI @@ -94860,7 +100292,7 @@ 3423585928,3423585935,NZ 3423585936,3423585999,CA 3423586000,3423586007,US -3423586008,3423586015,CA +3423586008,3423586015,PA 3423586016,3423586031,US 3423586032,3423586039,NZ 3423586040,3423586159,CA @@ -94966,7 +100398,9 @@ 3425912816,3425912831,DO 3425912832,3425914159,US 3425914160,3425914239,A2 -3425914240,3425916479,US +3425914240,3425915807,US +3425915808,3425915823,NI +3425915824,3425916479,US 3425916480,3425916543,DK 3425916544,3426013183,US 3426013184,3426013439,IL @@ -95097,8 +100531,8 @@ 3427779072,3427779327,GB 3427779328,3427780095,US 3427780096,3427780351,GB -3427780352,3427780607,DE -3427780608,3427781375,US +3427780352,3427780863,DE +3427780864,3427781375,US 3427781376,3427781631,GB 3427781632,3427781887,BE 3427781888,3427782399,US @@ -95394,8 +100828,7 @@ 3429408768,3429411583,US 3429411584,3429411839,DE 3429411840,3429500927,US -3429500928,3429501951,A2 -3429501952,3429502975,CA +3429500928,3429502975,A2 3429502976,3429775359,US 3429775360,3429777407,TC 3429777408,3429892095,US @@ -95571,9 +101004,7 @@ 3434571776,3434573823,CO 3434573824,3434575359,US 3434575360,3434575615,BR -3434575616,3434577407,US -3434577408,3434577663,CA -3434577664,3434583039,US +3434575616,3434583039,US 3434583040,3434584063,NL 3434584064,3434807295,US 3434807296,3434831359,CA @@ -95928,7 +101359,9 @@ 3438592264,3438600319,CA 3438600320,3438600351,US 3438600352,3438608383,CA -3438608384,3438813183,US +3438608384,3438689535,US +3438689536,3438689791,CA +3438689792,3438813183,US 3438813184,3438814207,GH 3438814208,3438895103,US 3438895104,3438896895,HN @@ -96007,14 +101440,19 @@ 3448380840,3448380847,SG 3448380848,3448380927,US 3448380928,3448380959,SG -3448380960,3448381183,US +3448380960,3448380975,US +3448380976,3448380991,AU +3448380992,3448381183,US 3448381184,3448381439,SG 3448381440,3448381695,GB 3448381696,3448382343,US 3448382344,3448382351,IE -3448382352,3448382415,US +3448382352,3448382367,GB +3448382368,3448382415,US 3448382416,3448382447,GB -3448382448,3448397887,US +3448382448,3448382567,US +3448382568,3448382583,GB +3448382584,3448397887,US 3448397888,3448397967,CA 3448397968,3448398335,US 3448398336,3448399103,CA @@ -96046,7 +101484,20 @@ 3448462256,3448462271,GB 3448462272,3448500479,US 3448500480,3448500735,SG -3448500736,3448556671,US +3448500736,3448545599,US +3448545600,3448545679,AU +3448545680,3448545695,US +3448545696,3448545727,AU +3448545728,3448545791,US +3448545792,3448545823,SG +3448545824,3448545855,AU +3448545856,3448545879,US +3448545880,3448545919,SG +3448545920,3448545951,US +3448545952,3448545967,SG +3448545968,3448545975,MY +3448545976,3448546007,AU +3448546008,3448556671,US 3448556672,3448556735,GB 3448556736,3448556799,US 3448556800,3448556815,GB @@ -96054,7 +101505,9 @@ 3448558912,3448558919,GB 3448558920,3448559103,US 3448559104,3448559359,GB -3448559360,3449001245,US +3448559360,3448563015,US +3448563016,3448563031,GB +3448563032,3449001245,US 3449001246,3449001246,MC 3449001247,3449159679,US 3449159680,3449160703,CA @@ -96325,9 +101778,17 @@ 3451896320,3451896575,DE 3451896576,3451912191,US 3451912192,3452174335,CA -3452174336,3452436479,US +3452174336,3452178719,US +3452178720,3452178751,CA +3452178752,3452179391,US +3452179392,3452179399,CA +3452179400,3452179940,US +3452179941,3452179941,CA +3452179942,3452436479,US 3452436480,3452502015,CA -3452502016,3452764159,US +3452502016,3452678143,US +3452678144,3452682239,BE +3452682240,3452764159,US 3452764160,3452764439,CA 3452764440,3452764447,US 3452764448,3452764487,CA @@ -96623,7 +102084,19 @@ 3453554880,3453555711,US 3453555712,3453555767,GB 3453555768,3453607935,US -3453607936,3453616127,AG +3453607936,3453608959,KN +3453608960,3453609983,LC +3453609984,3453610495,AG +3453610496,3453610751,MS +3453610752,3453611007,AG +3453611008,3453611263,MS +3453611264,3453612543,DM +3453612544,3453613055,KN +3453613056,3453614591,AG +3453614592,3453615103,DM +3453615104,3453615359,AG +3453615360,3453615615,KN +3453615616,3453616127,AG 3453616128,3454003013,US 3454003014,3454003014,ES 3454003015,3454004997,US @@ -96717,9 +102190,7 @@ 3455244848,3455244855,CA 3455244856,3455320063,US 3455320064,3455322111,FR -3455322112,3455323623,US -3455323624,3455323631,CA -3455323632,3455328255,US +3455322112,3455328255,US 3455328256,3455329279,DO 3455329280,3455330815,US 3455330816,3455331327,ES @@ -96729,9 +102200,7 @@ 3455359232,3455359487,ES 3455359488,3455360671,US 3455360672,3455360687,PR -3455360688,3455378351,US -3455378352,3455378367,CA -3455378368,3455418367,US +3455360688,3455418367,US 3455418368,3455418879,KY 3455418880,3455421439,US 3455421440,3455421951,BE @@ -96874,8 +102343,8 @@ 3457598464,3457600511,US 3457600512,3457600767,CA 3457600768,3457683967,US -3457683968,3457684735,CA -3457684736,3457723535,US +3457683968,3457684991,CA +3457684992,3457723535,US 3457723536,3457723551,GB 3457723552,3457859839,US 3457859840,3457860095,CA @@ -97029,20 +102498,43 @@ 3459358720,3459358975,VE 3459358976,3459359999,US 3459360000,3459360511,VE -3459360512,3459362815,US -3459362816,3459364863,CO -3459364864,3459366911,AR -3459366912,3459367167,VE -3459367168,3459367423,US -3459367424,3459367455,VE +3459360512,3459363583,US +3459363584,3459363871,CO +3459363872,3459363935,US +3459363936,3459363967,CO +3459363968,3459364031,US +3459364032,3459364063,CO +3459364064,3459364159,US +3459364160,3459364191,CO +3459364192,3459364351,US +3459364352,3459364383,CO +3459364384,3459364415,US +3459364416,3459364575,CO +3459364576,3459365919,US +3459365920,3459365951,AR +3459365952,3459365983,US +3459365984,3459366015,AR +3459366016,3459366047,US +3459366048,3459366079,AR +3459366080,3459366127,US +3459366128,3459366135,AR +3459366136,3459366143,US +3459366144,3459366303,AR +3459366304,3459366335,US +3459366336,3459366911,AR +3459366912,3459367455,US 3459367456,3459367535,VN -3459367536,3459367551,VE +3459367536,3459367551,US 3459367552,3459367647,VN -3459367648,3459367935,VE +3459367648,3459367679,US +3459367680,3459367935,VE 3459367936,3459367999,VN -3459368000,3459368063,VE +3459368000,3459368063,US 3459368064,3459368095,VN -3459368096,3459368703,VE +3459368096,3459368127,US +3459368128,3459368159,VE +3459368160,3459368191,US +3459368192,3459368703,VE 3459368704,3459368959,US 3459368960,3459371007,MX 3459371008,3459373055,PE @@ -97094,7 +102586,65 @@ 3460453632,3460453887,BS 3460453888,3460507647,US 3460507648,3460507903,MX -3460507904,3460891135,US +3460507904,3460854911,US +3460854912,3460854943,GB +3460854944,3460855023,US +3460855024,3460855031,BR +3460855032,3460855247,US +3460855248,3460855255,AU +3460855256,3460855271,US +3460855272,3460855279,GB +3460855280,3460855287,CA +3460855288,3460855311,US +3460855312,3460855319,CA +3460855320,3460855463,US +3460855464,3460855471,SA +3460855472,3460855535,US +3460855536,3460855543,MX +3460855544,3460855551,AU +3460855552,3460855631,US +3460855632,3460855647,IT +3460855648,3460855703,US +3460855704,3460855711,AU +3460855712,3460855743,US +3460855744,3460855775,CA +3460855776,3460855807,US +3460855808,3460855815,GB +3460855816,3460855823,NZ +3460855824,3460855855,US +3460855856,3460855863,GB +3460855864,3460855871,US +3460855872,3460855879,GB +3460855880,3460855887,GU +3460855888,3460855927,US +3460855928,3460855935,GB +3460855936,3460856007,US +3460856008,3460856015,CA +3460856016,3460856127,US +3460856128,3460856135,JP +3460856136,3460856191,US +3460856192,3460856199,ZA +3460856200,3460856215,US +3460856216,3460856223,GB +3460856224,3460856287,US +3460856288,3460856295,IN +3460856296,3460856383,US +3460856384,3460856399,GB +3460856400,3460856447,US +3460856448,3460856495,GB +3460856496,3460856559,US +3460856560,3460856575,CA +3460856576,3460856607,US +3460856608,3460856639,NZ +3460856640,3460856815,US +3460856816,3460856831,NZ +3460856832,3460857055,US +3460857056,3460857087,NZ +3460857088,3460857151,US +3460857152,3460857183,CA +3460857184,3460857599,US +3460857600,3460857855,AU +3460857856,3460891135,US 3460891136,3460891199,CL 3460891200,3460891967,US 3460891968,3460891999,BR @@ -97422,7 +102972,7 @@ 3460954432,3460954463,US 3460954464,3460954479,MY 3460954480,3460954527,US -3460954528,3460954559,KP +3460954528,3460954559,KR 3460954560,3460954623,US 3460954624,3460954879,DE 3460954880,3460955135,LU @@ -97466,8 +103016,8 @@ 3461514496,3461514751,QA 3461514752,3461516287,CA 3461516288,3461516543,IL -3461516544,3461517311,CA -3461517312,3461554175,US +3461516544,3461517055,CA +3461517056,3461554175,US 3461554176,3461556223,CA 3461558272,3461597887,US 3461597888,3461597951,CO @@ -97559,7 +103109,13 @@ 3462340608,3462340863,CA 3462340864,3462350847,US 3462350848,3462351103,CA -3462351104,3462571095,US +3462351104,3462478559,US +3462478560,3462478591,IL +3462478592,3462478623,US +3462478624,3462478655,IL +3462478656,3462478943,US +3462478944,3462478975,IL +3462478976,3462571095,US 3462571096,3462571103,CA 3462571104,3462571663,US 3462571664,3462571671,CA @@ -97628,7 +103184,11 @@ 3463583568,3463583583,UM 3463583584,3463602687,US 3463602688,3463602943,PH -3463602944,3463708671,US +3463602944,3463632383,US +3463632384,3463632391,CA +3463632392,3463632399,US +3463632400,3463632415,CA +3463632416,3463708671,US 3463708672,3463774207,CA 3463774208,3464108031,US 3464108032,3464108287,DE @@ -97645,9 +103205,7 @@ 3464184496,3464184511,CA 3464184512,3464185759,US 3464185760,3464185791,PR -3464185792,3464189439,US -3464189440,3464189951,GB -3464189952,3464190463,US +3464185792,3464190463,US 3464190464,3464190719,CA 3464190720,3464195543,US 3464195544,3464195551,IT @@ -97673,7 +103231,9 @@ 3464310592,3464310655,AU 3464310656,3464340479,US 3464340480,3464341503,CA -3464341504,3464341823,US +3464341504,3464341759,US +3464341760,3464341775,JP +3464341776,3464341823,US 3464341824,3464341831,PT 3464341832,3464342543,US 3464342544,3464342559,SE @@ -97689,7 +103249,9 @@ 3464343240,3464343247,MX 3464343248,3464343271,US 3464343272,3464343279,CA -3464343280,3464382463,US +3464343280,3464343423,US +3464343424,3464343431,SG +3464343432,3464382463,US 3464384512,3464388607,CA 3464388608,3464394751,US 3464394752,3464396799,VC @@ -98231,11 +103793,13 @@ 3469186560,3469893631,US 3469893632,3469901823,CA 3469901824,3470131199,US -3470131200,3470139391,AG +3470131200,3470137343,AG +3470137344,3470139391,VG 3470147584,3470148095,US 3470148096,3470148351,CA 3470148352,3470148607,US -3470148608,3470149119,CA +3470148608,3470148863,CR +3470148864,3470149119,CA 3470149120,3470151935,US 3470151936,3470152703,CA 3470152704,3470153215,US @@ -98244,7 +103808,479 @@ 3470154496,3470154751,CA 3470154752,3470155007,US 3470155008,3470155775,CA -3470155776,3470458879,US +3470155776,3470184454,US +3470184455,3470184458,LK +3470184459,3470184460,RU +3470184461,3470184476,US +3470184477,3470184480,CN +3470184481,3470184484,US +3470184485,3470184488,IN +3470184489,3470184492,US +3470184493,3470184496,ID +3470184497,3470184512,US +3470184513,3470184516,TH +3470184517,3470184528,US +3470184529,3470184532,CA +3470184533,3470184540,US +3470184541,3470184544,ID +3470184545,3470184545,MY +3470184546,3470184549,CA +3470184550,3470184553,SI +3470184554,3470184557,IN +3470184558,3470184561,US +3470184562,3470184565,IN +3470184566,3470184574,US +3470184575,3470184575,BE +3470184576,3470184577,CA +3470184578,3470184581,MX +3470184582,3470184585,CA +3470184586,3470184593,US +3470184594,3470184597,JP +3470184598,3470184613,US +3470184614,3470184617,IN +3470184618,3470184619,NZ +3470184620,3470184639,US +3470184640,3470184643,TW +3470184644,3470184671,US +3470184672,3470184675,MY +3470184676,3470184683,US +3470184684,3470184687,BR +3470184688,3470184691,CA +3470184692,3470184699,US +3470184700,3470184701,AU +3470184702,3470184722,US +3470184723,3470184723,MY +3470184724,3470184731,US +3470184732,3470184733,NZ +3470184734,3470184737,US +3470184738,3470184741,SG +3470184742,3470184746,US +3470184747,3470184750,MY +3470184751,3470184769,US +3470184770,3470184773,ID +3470184774,3470184785,US +3470184786,3470184789,NZ +3470184790,3470184801,US +3470184802,3470184805,TW +3470184806,3470184809,US +3470184810,3470184813,MY +3470184814,3470184817,CA +3470184818,3470184821,US +3470184822,3470184825,AU +3470184826,3470184829,US +3470184830,3470184833,JP +3470184834,3470184837,US +3470184838,3470184841,GB +3470184842,3470184849,US +3470184850,3470184853,JP +3470184854,3470184857,SG +3470184858,3470184861,RO +3470184862,3470184865,US +3470184866,3470184869,AU +3470184870,3470184873,HK +3470184874,3470184881,US +3470184882,3470184884,AU +3470184885,3470184885,US +3470184886,3470184889,CA +3470184890,3470184905,US +3470184906,3470184907,SG +3470184908,3470184919,US +3470184920,3470184923,MX +3470184924,3470184929,US +3470184930,3470184933,AU +3470184934,3470184957,US +3470184958,3470184958,LK +3470184959,3470184969,US +3470184970,3470184971,RU +3470184972,3470184975,US +3470184976,3470184976,MY +3470184977,3470184990,US +3470184991,3470184994,NZ +3470184995,3470185010,US +3470185011,3470185014,GB +3470185015,3470185016,AU +3470185017,3470185020,US +3470185021,3470185021,BE +3470185022,3470185050,US +3470185051,3470185054,IN +3470185055,3470185062,US +3470185063,3470185066,BR +3470185067,3470185070,PL +3470185071,3470185082,US +3470185083,3470185086,AU +3470185087,3470185090,IN +3470185091,3470185138,US +3470185139,3470185142,ID +3470185143,3470185146,US +3470185147,3470185150,AR +3470185151,3470185154,AU +3470185155,3470185166,US +3470185167,3470185170,CN +3470185171,3470185199,US +3470185200,3470185203,ID +3470185204,3470185207,GB +3470185208,3470185213,US +3470185214,3470185214,LK +3470185215,3470185234,US +3470185235,3470185238,AE +3470185239,3470185242,NL +3470185243,3470185250,US +3470185251,3470185254,CA +3470185255,3470185258,SG +3470185259,3470185262,AU +3470185263,3470185270,US +3470185271,3470185274,ID +3470185275,3470185278,AU +3470185279,3470185282,CA +3470185283,3470185286,US +3470185287,3470185290,CN +3470185291,3470185294,US +3470185295,3470185295,MY +3470185296,3470185312,US +3470185313,3470185316,TW +3470185317,3470185336,US +3470185337,3470185340,AU +3470185341,3470185344,US +3470185345,3470185348,EC +3470185349,3470185352,US +3470185353,3470185353,NZ +3470185354,3470185357,AR +3470185358,3470185365,US +3470185366,3470185367,AU +3470185368,3470185371,PH +3470185372,3470185379,US +3470185380,3470185383,AU +3470185384,3470185399,US +3470185400,3470185403,ID +3470185404,3470185415,US +3470185416,3470185419,BE +3470185420,3470185423,NL +3470185424,3470185427,CA +3470185428,3470185431,US +3470185432,3470185435,CN +3470185436,3470185451,US +3470185452,3470185455,VE +3470185456,3470185463,US +3470185464,3470185467,NZ +3470185468,3470185468,US +3470185469,3470185469,LK +3470185470,3470186003,US +3470186004,3470186007,MX +3470186008,3470186015,CA +3470186016,3470186019,AU +3470186020,3470186048,US +3470186049,3470186052,PY +3470186053,3470186060,US +3470186061,3470186064,CN +3470186065,3470186068,AU +3470186069,3470186072,US +3470186073,3470186076,IN +3470186077,3470186084,US +3470186085,3470186085,BE +3470186086,3470186089,US +3470186090,3470186093,SG +3470186094,3470186097,CA +3470186098,3470186113,US +3470186114,3470186117,AU +3470186118,3470186251,US +3470186252,3470186255,CN +3470186256,3470186267,US +3470186268,3470186271,MX +3470186272,3470186275,SG +3470186276,3470186283,US +3470186284,3470186287,CA +3470186288,3470186306,US +3470186307,3470186310,MY +3470186311,3470186314,TH +3470186315,3470186318,AU +3470186319,3470186324,US +3470186325,3470186328,CN +3470186329,3470186329,NZ +3470186330,3470186338,US +3470186339,3470186342,UA +3470186343,3470186513,US +3470186514,3470186517,CA +3470186518,3470186525,US +3470186526,3470186529,ID +3470186530,3470186537,GB +3470186538,3470186545,US +3470186546,3470186549,MY +3470186550,3470186573,US +3470186574,3470186577,CH +3470186578,3470186593,US +3470186594,3470186597,CA +3470186598,3470186601,TR +3470186602,3470186605,US +3470186606,3470186609,CA +3470186610,3470186613,ES +3470186614,3470186617,US +3470186618,3470186621,RO +3470186622,3470186625,BR +3470186626,3470186637,US +3470186638,3470186641,BR +3470186642,3470186645,US +3470186646,3470186649,ES +3470186650,3470186669,US +3470186670,3470186673,AU +3470186674,3470186677,US +3470186678,3470186681,CA +3470186682,3470186685,US +3470186686,3470186689,IL +3470186690,3470186693,IN +3470186694,3470186697,ES +3470186698,3470186701,CA +3470186702,3470186705,AU +3470186706,3470186721,US +3470186722,3470186725,IN +3470186726,3470186729,US +3470186730,3470186733,IN +3470186734,3470186737,US +3470186738,3470186741,CA +3470186742,3470186745,IL +3470186746,3470186757,US +3470186758,3470186761,CA +3470186762,3470186765,US +3470186766,3470186769,RO +3470186770,3470186785,US +3470186786,3470186789,GB +3470186790,3470186805,US +3470186806,3470186809,IN +3470186810,3470186813,US +3470186814,3470186817,EE +3470186818,3470186825,US +3470186826,3470186829,GB +3470186830,3470186833,IN +3470186834,3470186837,US +3470186838,3470186841,AR +3470186842,3470186845,CA +3470186846,3470186849,EG +3470186850,3470186853,US +3470186854,3470186857,KW +3470186858,3470186861,US +3470186862,3470186865,AR +3470186866,3470186873,US +3470186874,3470186877,AE +3470186878,3470186881,CA +3470186882,3470186885,US +3470186886,3470186889,FR +3470186890,3470186891,CA +3470186892,3470186895,FR +3470186896,3470186899,AU +3470186900,3470186903,US +3470186904,3470186907,IN +3470186908,3470186933,US +3470186934,3470186935,BR +3470186936,3470186943,US +3470186944,3470186947,CA +3470186948,3470186963,US +3470186964,3470186967,TR +3470186968,3470186989,US +3470186990,3470186993,PE +3470186994,3470186997,RU +3470186998,3470187001,GB +3470187002,3470187041,US +3470187042,3470187045,GB +3470187046,3470187057,US +3470187058,3470187061,MY +3470187062,3470187073,US +3470187074,3470187077,IT +3470187078,3470187081,CA +3470187082,3470187085,CZ +3470187086,3470187089,US +3470187090,3470187093,TR +3470187094,3470187097,GB +3470187098,3470187101,IN +3470187102,3470187109,US +3470187110,3470187113,BR +3470187114,3470187117,VN +3470187118,3470187125,US +3470187126,3470187129,GB +3470187130,3470187133,AR +3470187134,3470187137,BA +3470187138,3470187141,US +3470187142,3470187145,CA +3470187146,3470187153,IN +3470187154,3470187157,ID +3470187158,3470187161,US +3470187162,3470187165,AU +3470187166,3470187169,NO +3470187170,3470187181,US +3470187182,3470187185,GR +3470187186,3470187229,US +3470187230,3470187231,CA +3470187232,3470187235,GB +3470187236,3470187247,US +3470187248,3470187251,GB +3470187252,3470187255,US +3470187256,3470187259,ES +3470187260,3470187269,US +3470187270,3470187273,AU +3470187274,3470187309,US +3470187310,3470187313,ES +3470187314,3470187335,US +3470187336,3470187339,CA +3470187340,3470187343,GR +3470187344,3470187356,US +3470187357,3470187360,GB +3470187361,3470187364,BG +3470187365,3470187368,SA +3470187369,3470187372,GR +3470187373,3470187380,US +3470187381,3470187384,AR +3470187385,3470187388,US +3470187389,3470187392,GB +3470187393,3470187396,IN +3470187397,3470187400,RU +3470187401,3470187404,GB +3470187405,3470187408,DO +3470187409,3470187416,US +3470187417,3470187420,IN +3470187421,3470187429,US +3470187430,3470187431,RU +3470187432,3470187437,US +3470187438,3470187441,PH +3470187442,3470187446,US +3470187447,3470187450,CA +3470187451,3470187454,US +3470187455,3470187458,IT +3470187459,3470187470,US +3470187471,3470187474,PT +3470187475,3470187502,US +3470187503,3470187503,JP +3470187504,3470187511,US +3470187512,3470187515,UA +3470187516,3470187525,US +3470187526,3470187529,EC +3470187530,3470187557,US +3470187558,3470187561,AE +3470187562,3470187569,US +3470187570,3470187573,IN +3470187574,3470187577,US +3470187578,3470187581,GB +3470187582,3470187589,US +3470187590,3470187593,IT +3470187594,3470187606,US +3470187607,3470187610,IL +3470187611,3470187614,PL +3470187615,3470187618,US +3470187619,3470187622,IT +3470187623,3470187634,US +3470187635,3470187638,SG +3470187639,3470187642,US +3470187643,3470187646,AU +3470187647,3470187650,IT +3470187651,3470187654,AR +3470187655,3470187666,US +3470187667,3470187670,IN +3470187671,3470187674,RS +3470187675,3470187678,ES +3470187679,3470187682,PT +3470187683,3470187686,US +3470187687,3470187690,JP +3470187691,3470187702,US +3470187703,3470187706,GB +3470187707,3470187725,US +3470187726,3470187729,GB +3470187730,3470187741,US +3470187742,3470187745,VN +3470187746,3470187755,US +3470187756,3470187759,PH +3470187760,3470187763,ZA +3470187764,3470187781,US +3470187782,3470187785,SI +3470187786,3470187801,US +3470187802,3470187805,IN +3470187806,3470187813,US +3470187814,3470187817,HK +3470187818,3470187825,US +3470187826,3470187829,CA +3470187830,3470187841,US +3470187842,3470187845,TR +3470187846,3470187857,US +3470187858,3470187861,MX +3470187862,3470187867,US +3470187868,3470187871,IL +3470187872,3470187885,US +3470187886,3470187889,IN +3470187890,3470187940,US +3470187941,3470187944,SA +3470187945,3470187952,US +3470187953,3470187956,NO +3470187957,3470187960,US +3470187961,3470187964,IN +3470187965,3470187976,US +3470187977,3470187980,ES +3470187981,3470187988,US +3470187989,3470187992,CA +3470187993,3470188004,US +3470188005,3470188008,AU +3470188009,3470188016,SE +3470188017,3470188024,US +3470188025,3470188028,GB +3470188029,3470188041,US +3470188042,3470188045,TH +3470188046,3470188049,GB +3470188050,3470188053,CA +3470188054,3470188057,US +3470188058,3470188061,GB +3470188062,3470188069,US +3470188070,3470188073,NZ +3470188074,3470188095,US +3470188096,3470188099,GB +3470188100,3470188103,US +3470188104,3470188107,CA +3470188108,3470188111,US +3470188112,3470188115,CA +3470188116,3470188131,US +3470188132,3470188135,AU +3470188136,3470188151,US +3470188152,3470188155,KW +3470188156,3470188159,US +3470188160,3470188163,BE +3470188164,3470188167,ID +3470188168,3470188171,US +3470188172,3470188175,NL +3470188176,3470188187,US +3470188188,3470188191,ES +3470188192,3470188199,US +3470188200,3470188203,AR +3470188204,3470188207,US +3470188208,3470188211,ES +3470188212,3470188215,TR +3470188216,3470188227,US +3470188228,3470188231,GB +3470188232,3470188243,US +3470188244,3470188247,CA +3470188248,3470188251,BE +3470188252,3470188255,US +3470188256,3470188259,CA +3470188260,3470188275,US +3470188276,3470188279,IN +3470188280,3470188283,GR +3470188284,3470188293,US +3470188294,3470188295,RU +3470188296,3470188299,US +3470188300,3470188303,ES +3470188304,3470188347,US +3470188348,3470188351,HU +3470188352,3470188393,US +3470188394,3470188397,GB +3470188398,3470188405,US +3470188406,3470188409,GR +3470188410,3470188421,US +3470188422,3470188425,GB +3470188426,3470188433,ES +3470188434,3470188437,AR +3470188438,3470188483,US +3470188484,3470188487,PH +3470188488,3470188503,US +3470188504,3470188507,CA +3470188508,3470188509,US +3470188510,3470188510,IL +3470188511,3470188521,US +3470188522,3470188541,HK +3470188542,3470192639,US +3470196736,3470458879,US 3470458880,3470475263,KR 3470475264,3470558207,US 3470558208,3470559231,HK @@ -98252,15 +104288,98 @@ 3470573568,3470575615,CA 3470575616,3470610431,US 3470610432,3470614527,BR -3470614528,3470651391,US +3470614528,3470645511,US +3470645512,3470645519,GB +3470645520,3470645527,FR +3470645528,3470645535,BR +3470645536,3470645559,US +3470645560,3470645567,IT +3470645568,3470645575,CA +3470645576,3470645583,US +3470645584,3470645591,FR +3470645592,3470645623,US +3470645624,3470645631,BR +3470645632,3470645655,US +3470645656,3470645663,RU +3470645664,3470645687,US +3470645688,3470645703,ES +3470645704,3470645731,US +3470645732,3470645735,AU +3470645736,3470645739,CN +3470645740,3470645743,US +3470645744,3470645747,ES +3470645748,3470645751,US +3470645752,3470645759,BR +3470645760,3470646015,US +3470646016,3470646019,BR +3470646020,3470646035,US +3470646036,3470646039,VI +3470646040,3470646087,US +3470646088,3470646095,ES +3470646096,3470646103,US +3470646104,3470646119,BR +3470646120,3470646135,US +3470646136,3470646143,AU +3470646144,3470646151,CA +3470646152,3470646167,AU +3470646168,3470646175,BR +3470646176,3470646215,US +3470646216,3470646223,BR +3470646224,3470646231,IL +3470646232,3470646239,US +3470646240,3470646247,IL +3470646248,3470646255,DE +3470646256,3470646263,AU +3470646264,3470646271,AR +3470646272,3470646359,US +3470646360,3470646367,ZA +3470646368,3470646375,US +3470646376,3470646383,BR +3470646384,3470646423,US +3470646424,3470646431,FR +3470646432,3470646455,US +3470646456,3470646463,FR +3470646464,3470646487,US +3470646488,3470646495,PL +3470646496,3470646503,BR +3470646504,3470646511,US +3470646512,3470646519,BR +3470646520,3470646527,US +3470646528,3470646543,FR +3470646544,3470646567,US +3470646568,3470646575,BR +3470646576,3470646583,US +3470646584,3470646591,LK +3470646592,3470646599,CN +3470646600,3470646607,US +3470646608,3470646623,ZA +3470646624,3470646631,DE +3470646632,3470646639,BR +3470646640,3470646663,US +3470646664,3470646671,PL +3470646672,3470646679,BR +3470646680,3470646687,CA +3470646688,3470646703,US +3470646704,3470646711,CN +3470646712,3470646727,US +3470646728,3470646735,GB +3470646736,3470646743,US +3470646744,3470646751,BR +3470646752,3470646767,FR +3470646768,3470646775,US +3470646776,3470646783,IT +3470646784,3470646791,US +3470646792,3470646799,CZ +3470646800,3470646807,BR +3470646808,3470646831,US +3470646832,3470646847,IL +3470646848,3470651391,US 3470651392,3470655487,CA 3470655488,3470656063,US 3470656064,3470656095,IL 3470656096,3470656191,US 3470656192,3470656199,IL -3470656200,3470656231,US -3470656232,3470656239,BD -3470656240,3470656383,US +3470656200,3470656383,US 3470656384,3470656415,IL 3470656416,3470656471,US 3470656472,3470656479,IL @@ -98419,9 +104538,7 @@ 3471060992,3471061247,NL 3471061248,3471061503,US 3471061504,3471061759,NL -3471061760,3471095271,US -3471095272,3471095279,CA -3471095280,3471204679,US +3471061760,3471204679,US 3471204680,3471204687,PR 3471204688,3471262719,US 3471262720,3471263743,EC @@ -98578,7 +104695,9 @@ 3475884032,3475884287,US 3475884288,3475884319,CA 3475884320,3475884327,US -3475884328,3475890175,CA +3475884328,3475888127,CA +3475888128,3475888383,US +3475888384,3475890175,CA 3475890176,3475996671,US 3475996672,3476029439,CA 3476029440,3476041551,US @@ -98609,7 +104728,9 @@ 3476451376,3476455423,CA 3476455424,3476718616,US 3476718617,3476718617,IN -3476718618,3476720377,US +3476718618,3476720363,US +3476720364,3476720367,LV +3476720368,3476720377,US 3476720378,3476720381,PT 3476720382,3476721159,US 3476721160,3476721167,IT @@ -99139,18 +105260,32 @@ 3484320256,3484320815,US 3484320816,3484320823,DE 3484320832,3484321279,US -3484321792,3484450815,US -3484450816,3484451839,CA +3484321792,3484451071,US +3484451072,3484451839,CA 3484451840,3484451871,US -3484451872,3484455263,CA +3484451872,3484451935,CA +3484451936,3484451967,US +3484451968,3484452863,CA +3484452864,3484453119,US +3484453120,3484453375,CA +3484453376,3484453631,US +3484453632,3484455167,CA +3484455168,3484455199,US +3484455200,3484455263,CA 3484455264,3484455295,US -3484455296,3484455807,CA -3484455808,3484455871,US -3484455872,3484456191,CA +3484455296,3484455327,CA +3484455328,3484455391,US +3484455392,3484455679,CA +3484455680,3484455935,US +3484455936,3484456191,CA 3484456192,3484456703,US -3484456704,3484457727,CA +3484456704,3484456959,CA +3484456960,3484457215,US +3484457216,3484457727,CA 3484457728,3484457983,US -3484457984,3484458783,CA +3484457984,3484458239,CA +3484458240,3484458495,US +3484458496,3484458783,CA 3484458784,3484458791,US 3484458792,3484458807,CA 3484458808,3484458815,US @@ -99161,31 +105296,29 @@ 3484458920,3484458959,CA 3484458960,3484458967,US 3484458968,3484459791,CA -3484459792,3484459807,US -3484459808,3484459823,CA -3484459824,3484459839,US +3484459792,3484459839,US 3484459840,3484459855,CA -3484459856,3484459871,US -3484459872,3484459887,CA -3484459888,3484459919,US -3484459920,3484459967,CA -3484459968,3484459999,US +3484459856,3484459919,US +3484459920,3484459951,CA +3484459952,3484459999,US 3484460000,3484460015,CA 3484460016,3484460031,US 3484460032,3484460287,CA 3484460288,3484460543,US -3484460544,3484460575,CA -3484460576,3484460591,US -3484460592,3484460703,CA +3484460544,3484460559,CA +3484460560,3484460591,US +3484460592,3484460623,CA +3484460624,3484460671,US +3484460672,3484460703,CA 3484460704,3484460719,US -3484460720,3484460783,CA +3484460720,3484460751,CA +3484460752,3484460767,US +3484460768,3484460783,CA 3484460784,3484460799,US 3484460800,3484461079,CA 3484461080,3484461095,US 3484461096,3484461103,CA -3484461104,3484461135,US -3484461136,3484461143,CA -3484461144,3484461199,US +3484461104,3484461199,US 3484461200,3484461231,CA 3484461232,3484461263,US 3484461264,3484461271,CA @@ -99194,53 +105327,81 @@ 3484461376,3484461439,US 3484461440,3484461503,CA 3484461504,3484461631,US -3484461632,3484462103,CA +3484461632,3484461759,CA +3484461760,3484461823,US +3484461824,3484462103,CA 3484462104,3484462143,US 3484462144,3484462159,CA 3484462160,3484462175,US -3484462176,3484462207,CA -3484462208,3484462215,US -3484462216,3484462279,CA -3484462280,3484462295,US -3484462296,3484462975,CA -3484462976,3484463039,US -3484463040,3484463151,CA +3484462176,3484462199,CA +3484462200,3484462215,US +3484462216,3484462263,CA +3484462264,3484462271,US +3484462272,3484462279,CA +3484462280,3484462303,US +3484462304,3484462911,CA +3484462912,3484463039,US +3484463040,3484463119,CA +3484463120,3484463135,US +3484463136,3484463151,CA 3484463152,3484463183,US -3484463184,3484464143,CA -3484464144,3484464159,US -3484464160,3484464175,CA -3484464176,3484464191,US -3484464192,3484464223,CA -3484464224,3484464287,US -3484464288,3484464303,CA -3484464304,3484464319,US -3484464320,3484464335,CA -3484464336,3484464383,US -3484464384,3484464703,CA -3484464704,3484464735,US -3484464736,3484465151,CA -3484465152,3484465407,US -3484465408,3484466431,CA +3484463184,3484463231,CA +3484463232,3484463279,US +3484463280,3484463311,CA +3484463312,3484463359,US +3484463360,3484464143,CA +3484464144,3484464191,US +3484464192,3484464207,CA +3484464208,3484464447,US +3484464448,3484464575,CA +3484464576,3484464671,US +3484464672,3484464703,CA +3484464704,3484464767,US +3484464768,3484464799,CA +3484464800,3484464863,US +3484464864,3484465151,CA +3484465152,3484465471,US +3484465472,3484465535,CA +3484465536,3484465663,US +3484465664,3484466047,CA +3484466048,3484466111,US +3484466112,3484466431,CA 3484466432,3484466687,US -3484466688,3484468287,CA -3484468288,3484468479,US -3484468480,3484469119,CA -3484469120,3484469247,US -3484469248,3484469759,CA -3484469760,3484469791,US -3484469792,3484469887,CA -3484469888,3484469951,US -3484469952,3484470527,CA -3484470528,3484471295,US -3484471296,3484471551,CA -3484471552,3484471807,US -3484471808,3484472135,CA +3484466688,3484467711,CA +3484467712,3484467967,US +3484467968,3484468287,CA +3484468288,3484469791,US +3484469792,3484469823,CA +3484469824,3484469855,US +3484469856,3484469887,CA +3484469888,3484470015,US +3484470016,3484470271,CA +3484470272,3484471423,US +3484471424,3484471487,CA +3484471488,3484471807,US +3484471808,3484472071,CA +3484472072,3484472087,US +3484472088,3484472095,CA +3484472096,3484472103,US +3484472104,3484472119,CA +3484472120,3484472127,US +3484472128,3484472135,CA 3484472136,3484472167,US 3484472168,3484472183,CA 3484472184,3484472199,US 3484472200,3484472223,CA 3484472224,3484472263,US -3484472264,3484472839,CA +3484472264,3484472271,CA +3484472272,3484472279,US +3484472280,3484472295,CA +3484472296,3484472303,US +3484472304,3484472311,CA +3484472312,3484472319,US +3484472320,3484472639,CA +3484472640,3484472703,US +3484472704,3484472767,CA +3484472768,3484472831,US +3484472832,3484472839,CA 3484472840,3484472855,US 3484472856,3484473007,CA 3484473008,3484473015,US @@ -99249,13 +105410,15 @@ 3484473032,3484473047,CA 3484473048,3484473055,US 3484473056,3484473087,CA -3484473088,3484473151,US -3484473152,3484473599,CA -3484473600,3484473727,US -3484473728,3484473791,CA -3484473792,3484473855,US -3484473856,3484477183,CA -3484477184,3484477695,US +3484473088,3484473215,US +3484473216,3484473343,CA +3484473344,3484474111,US +3484474112,3484474623,CA +3484474624,3484474879,US +3484474880,3484474943,CA +3484474944,3484475071,US +3484475072,3484476927,CA +3484476928,3484477695,US 3484477696,3484478207,CA 3484478208,3484478719,US 3484478720,3484478975,CA @@ -99264,10 +105427,18 @@ 3484479488,3484479743,US 3484479744,3484479999,CA 3484480000,3484480255,US -3484480256,3484482047,CA +3484480256,3484480511,CA +3484480512,3484480767,US +3484480768,3484481279,CA +3484481280,3484481343,US +3484481344,3484481535,CA +3484481536,3484481791,US +3484481792,3484482047,CA 3484482048,3484482559,US -3484482560,3484483583,CA -3484483584,3484647423,US +3484482560,3484482815,CA +3484482816,3484483071,US +3484483072,3484483327,CA +3484483328,3484647423,US 3484647424,3484663807,CA 3484663808,3484746815,US 3484746816,3484746943,GB @@ -99277,9 +105448,7 @@ 3484775776,3484777663,CA 3484777664,3484777695,US 3484777696,3484778495,CA -3484778496,3484850303,US -3484850304,3484850311,IR -3484850312,3484884991,US +3484778496,3484884991,US 3484884992,3484893183,CA 3484893184,3485220863,US 3485220864,3485229055,CA @@ -99305,9 +105474,7 @@ 3485885128,3485885141,CA 3485885142,3485886917,US 3485886918,3485886931,IN -3485886932,3485887747,US -3485887748,3485887777,AU -3485887778,3486023679,US +3485886932,3486023679,US 3486023680,3486031871,CA 3486031872,3486210303,US 3486210304,3486210319,FM @@ -99393,9 +105560,7 @@ 3486601536,3486601727,US 3486601728,3486601983,BE 3486601984,3486602239,NZ -3486602240,3486602431,US -3486602432,3486602495,CO -3486602496,3486603007,US +3486602240,3486603007,US 3486603008,3486603263,SG 3486603264,3486603295,US 3486603296,3486603327,BR @@ -99411,9 +105576,7 @@ 3486604544,3486604575,US 3486604576,3486604607,SG 3486604608,3486604615,MY -3486604616,3486604671,US -3486604672,3486604735,CO -3486604736,3486605055,US +3486604616,3486605055,US 3486605056,3486605311,ES 3486605312,3486605567,NL 3486605568,3486606079,US @@ -99463,7 +105626,7 @@ 3486617344,3486617599,US 3486617600,3486617855,CH 3486617856,3486618111,AU -3486618112,3486618367,NO +3486618112,3486618367,US 3486618368,3486618623,ES 3486618624,3486618879,BE 3486618880,3486619135,FR @@ -99553,8 +105716,7 @@ 3486640640,3486642175,US 3486642176,3486642431,BR 3486642432,3486642687,PT -3486642688,3486642943,DK -3486642944,3486646015,US +3486642688,3486646015,US 3486646016,3486646143,JP 3486646144,3486646271,US 3486646272,3486662655,CA @@ -99564,13 +105726,9 @@ 3486700400,3486700407,CA 3486700408,3486700439,US 3486700440,3486700447,IT -3486700448,3486700479,US -3486700480,3486700503,IN -3486700504,3486701311,US +3486700448,3486701311,US 3486701312,3486701567,CA -3486701568,3486705919,US -3486705920,3486706175,IN -3486706176,3486711551,US +3486701568,3486711551,US 3486711552,3486711807,CA 3486711808,3487039487,US 3487039488,3487105023,CA @@ -99587,8 +105745,10 @@ 3487203328,3487236095,US 3487236096,3487301631,CA 3487301632,3487766527,US -3487766528,3487768575,CA -3487768576,3487842303,US +3487766528,3487767551,CA +3487767552,3487767807,US +3487767808,3487768063,CA +3487768064,3487842303,US 3487842304,3487858687,CA 3487858688,3487859199,KY 3487859200,3487861759,BM @@ -99609,16 +105769,32 @@ 3488072704,3488079871,US 3488079872,3488083967,CA 3488083968,3488088063,US -3488088064,3488104447,IL -3488104448,3488112639,US +3488088064,3488088319,IL +3488088320,3488088575,US +3488088576,3488091391,IL +3488091392,3488091519,US +3488091520,3488093055,IL +3488093056,3488093951,US +3488093952,3488094015,IL +3488094016,3488094719,US +3488094720,3488094735,IL +3488094736,3488094751,US +3488094752,3488094783,IL +3488094784,3488094831,US +3488094832,3488094847,IL +3488094848,3488095103,US +3488095104,3488095119,IL +3488095120,3488112639,US 3488112640,3488113839,CA -3488113840,3488113855,US -3488113856,3488116863,CA +3488113840,3488113847,US +3488113848,3488116863,CA 3488116864,3488116871,US -3488116872,3488118047,CA -3488118048,3488118055,US -3488118056,3488120831,CA -3488120832,3488350207,US +3488116872,3488118559,CA +3488118560,3488118567,US +3488118568,3488120831,CA +3488120832,3488300287,US +3488300288,3488300543,A2 +3488300544,3488350207,US 3488350208,3488361215,CA 3488361216,3488361471,US 3488361472,3488415743,CA @@ -99636,11 +105812,8 @@ 3488721408,3488722175,BA 3488722176,3488722431,RO 3488722432,3488722687,AL -3488722688,3488723199,JM -3488723200,3488727039,A2 -3488727040,3488901887,US -3488901888,3488902143,CA -3488902144,3488940031,US +3488722688,3488727039,A2 +3488727040,3488940031,US 3488940032,3488956415,CA 3488956416,3488989183,US 3488989184,3489005567,CA @@ -99727,7 +105900,9 @@ 3491381248,3491389439,BM 3491389440,3491476991,US 3491476992,3491478527,VI -3491478528,3491506631,US +3491478528,3491478591,US +3491478592,3491478599,PR +3491478600,3491506631,US 3491506632,3491506639,DK 3491506640,3491508223,US 3491508224,3491512319,CA @@ -99825,7 +106000,9 @@ 3493135168,3493135199,CY 3493135200,3493135871,US 3493135872,3493136127,CA -3493136128,3493139711,US +3493136128,3493138207,US +3493138208,3493138239,DE +3493138240,3493139711,US 3493139712,3493139775,CA 3493139776,3493139839,US 3493139840,3493139935,CA @@ -99915,7 +106092,9 @@ 3494094848,3494095871,CA 3494095872,3494102623,US 3494102624,3494102639,SB -3494102640,3494109540,US +3494102640,3494102735,US +3494102736,3494102751,VI +3494102752,3494109540,US 3494109541,3494109548,GB 3494109549,3494109697,US 3494109698,3494109722,GB @@ -100035,13 +106214,13 @@ 3494299680,3494299687,US 3494299688,3494299695,TW 3494299696,3494300359,US -3494300360,3494300375,TW -3494300376,3494300927,US +3494300360,3494300383,TW +3494300384,3494300927,US 3494300928,3494301055,TW 3494301056,3494301247,US 3494301248,3494301311,SG -3494301312,3494301407,US -3494301408,3494301695,TW +3494301312,3494301439,US +3494301440,3494301695,TW 3494301696,3494302719,US 3494302720,3494303743,CA 3494303744,3494310911,US @@ -100311,7 +106490,9 @@ 3494698888,3494698895,EG 3494698896,3494700031,US 3494700032,3494701055,CA -3494701056,3494727679,US +3494701056,3494712319,US +3494712320,3494713343,CA +3494713344,3494727679,US 3494727680,3494729727,CA 3494729728,3494730751,US 3494730752,3494731775,CA @@ -100353,7 +106534,7 @@ 3494776832,3494777855,CA 3494777856,3494785023,US 3494785024,3494787071,MF -3494787072,3494787199,CA +3494787072,3494787199,A2 3494787200,3494787231,GB 3494787232,3494789119,A2 3494789120,3494852607,US @@ -100424,18 +106605,16 @@ 3495154304,3495154431,US 3495154432,3495154751,CA 3495154752,3495154759,US -3495154760,3495154775,CA -3495154776,3495154783,US -3495154784,3495154807,CA -3495154808,3495154815,US -3495154816,3495154823,CA +3495154760,3495154823,CA 3495154824,3495154831,US 3495154832,3495154839,CA 3495154840,3495154847,US 3495154848,3495155711,CA 3495155712,3495157039,US 3495157040,3495157047,SE -3495157048,3495192575,US +3495157048,3495164239,US +3495164240,3495164247,CA +3495164248,3495192575,US 3495192576,3495193599,CA 3495193600,3495215103,US 3495215104,3495217151,VI @@ -100457,7 +106636,9 @@ 3495358464,3495359487,CA 3495359488,3495367679,US 3495367680,3495368703,CA -3495368704,3495370751,US +3495368704,3495370239,US +3495370240,3495370495,CA +3495370496,3495370751,US 3495370752,3495372799,BS 3495372800,3495375871,US 3495375872,3495376895,CA @@ -100873,9 +107054,7 @@ 3497222408,3497222415,IN 3497222416,3497222463,US 3497222464,3497222479,IN -3497222480,3497223295,US -3497223296,3497223359,CH -3497223360,3497223791,US +3497222480,3497223791,US 3497223792,3497223807,CH 3497223808,3497223839,US 3497223840,3497223871,CH @@ -100891,11 +107070,13 @@ 3497225472,3497225535,IN 3497225536,3497226295,US 3497226296,3497226303,SG -3497226304,3497226783,US +3497226304,3497226391,US +3497226392,3497226399,IR +3497226400,3497226783,US 3497226784,3497226815,CH 3497226816,3497226831,US -3497226832,3497226855,CH -3497226856,3497226943,US +3497226832,3497226847,CH +3497226848,3497226943,US 3497226944,3497226959,CH 3497226960,3497227023,US 3497227024,3497227039,GB @@ -100907,9 +107088,7 @@ 3497227392,3497227407,CN 3497227408,3497227527,US 3497227528,3497227535,CN -3497227536,3497228015,US -3497228016,3497228023,CN -3497228024,3497228079,US +3497227536,3497228079,US 3497228080,3497228087,CN 3497228088,3497233407,US 3497233408,3497233663,CN @@ -100981,8 +107160,10 @@ 3498708992,3498713639,JM 3498713640,3498713647,CA 3498713648,3498717183,JM -3498717184,3498760191,US -3498760192,3498762239,PR +3498717184,3498761215,US +3498761216,3498761727,PR +3498761728,3498761983,US +3498761984,3498762239,PR 3498762240,3498812415,US 3498812416,3498812671,PF 3498812672,3499186591,US @@ -101210,9 +107391,11 @@ 3505818624,3505819647,BS 3505819648,3505999359,US 3505999360,3506000127,PR -3506000128,3506043135,US -3506043136,3506044927,PA -3506044928,3506052543,US +3506000128,3506043136,US +3506043137,3506043390,PA +3506043391,3506043904,US +3506043905,3506044158,PA +3506044159,3506052543,US 3506052544,3506052559,DE 3506052560,3506135263,US 3506135264,3506135295,GB @@ -102004,7 +108187,7 @@ 3509775665,3509775792,US 3509775793,3509775870,CA 3509775871,3509775871,US -3509775872,3509775879,PK +3509775872,3509775879,RU 3509775880,3509775903,US 3509775904,3509775919,AU 3509775920,3509775943,US @@ -103426,16 +109609,15 @@ 3516383232,3516514303,US 3516514304,3516530687,CA 3516530688,3516899327,US -3516899328,3516899847,A2 +3516899328,3516899839,A2 +3516899840,3516899847,US 3516899848,3516899855,NG -3516899856,3516899871,US -3516899872,3516899887,A2 +3516899856,3516899887,US 3516899888,3516899903,NG 3516899904,3516899935,EC 3516899936,3516900015,NG -3516900016,3516900031,A2 -3516900032,3516900063,US -3516900064,3516900351,A2 +3516900016,3516900095,US +3516900096,3516900351,A2 3516900352,3516900607,NG 3516900608,3516900863,A2 3516900864,3516901119,NG @@ -103472,8 +109654,8 @@ 3517116416,3517120511,CA 3517120512,3517233151,US 3517233152,3517235199,GU -3517235200,3517309951,US -3517309952,3517313023,CA +3517235200,3517308927,US +3517308928,3517313023,CA 3517313024,3517382655,US 3517382656,3517383679,CA 3517383680,3517383935,US @@ -103601,7 +109783,9 @@ 3517418368,3517418399,US 3517418400,3517419103,CA 3517419104,3517419135,US -3517419136,3517420159,CA +3517419136,3517419519,CA +3517419520,3517419527,US +3517419528,3517420159,CA 3517420160,3517420287,US 3517420288,3517420303,CA 3517420304,3517420311,US @@ -103771,7 +109955,9 @@ 3517596672,3517596927,NO 3517596928,3517597183,SE 3517597184,3517597695,US -3517597696,3517598591,SE +3517597696,3517598207,SE +3517598208,3517598463,IE +3517598464,3517598591,SE 3517598592,3517599231,US 3517599232,3517600767,SE 3517600768,3517601279,US @@ -103806,8 +109992,7 @@ 3517610048,3517610191,SE 3517610192,3517610199,US 3517610200,3517610207,SE -3517610208,3517610239,US -3517610240,3517610495,SE +3517610208,3517610495,US 3517610496,3517611263,IE 3517611264,3517611295,SE 3517611296,3517611303,US @@ -103973,7 +110158,9 @@ 3519352896,3519352903,NL 3519352904,3519381503,US 3519381504,3519397887,CA -3519397888,3519406127,US +3519397888,3519406079,US +3519406080,3519406111,IT +3519406112,3519406127,US 3519406128,3519406143,DE 3519406144,3519406207,US 3519406208,3519406215,SA @@ -104049,11 +110236,15 @@ 3519413760,3519414271,CA 3519414272,3519417087,US 3519417088,3519417151,IT -3519417152,3519428095,US -3519428096,3519428351,RO -3519428352,3519475711,US +3519417152,3519417215,DE +3519417216,3519420767,US +3519420768,3519420775,HK +3519420776,3519422495,US +3519422496,3519422527,CA +3519422528,3519475711,US 3519475712,3519476223,BH -3519476224,3519566871,US +3519476224,3519477759,A2 +3519477760,3519566871,US 3519566872,3519566879,IE 3519566880,3519566943,US 3519566944,3519566951,GB @@ -104179,13 +110370,7 @@ 3520020984,3520021015,US 3520021016,3520021055,CA 3520021056,3520021071,US -3520021072,3520021079,CA -3520021080,3520021095,US -3520021096,3520021103,CA -3520021104,3520021111,US -3520021112,3520021127,CA -3520021128,3520021143,US -3520021144,3520021183,CA +3520021072,3520021183,CA 3520021184,3520021191,US 3520021192,3520021319,CA 3520021320,3520021359,US @@ -104281,7 +110466,9 @@ 3520026784,3520026791,US 3520026792,3520026895,CA 3520026896,3520026903,US -3520026904,3520027047,CA +3520026904,3520026991,CA +3520026992,3520026999,US +3520027000,3520027047,CA 3520027048,3520027055,US 3520027056,3520027111,CA 3520027112,3520027119,US @@ -104397,7 +110584,9 @@ 3520034960,3520035055,CA 3520035056,3520035071,US 3520035072,3520035135,CA -3520035136,3520035287,US +3520035136,3520035263,US +3520035264,3520035279,CA +3520035280,3520035287,US 3520035288,3520035327,CA 3520035328,3520035335,US 3520035336,3520035487,CA @@ -104406,7 +110595,9 @@ 3520035520,3520035543,US 3520035544,3520035583,CA 3520035584,3520035631,US -3520035632,3520036255,CA +3520035632,3520036191,CA +3520036192,3520036199,US +3520036200,3520036255,CA 3520036256,3520036271,US 3520036272,3520036287,CA 3520036288,3520036295,US @@ -104498,14 +110689,15 @@ 3520749872,3520749879,CA 3520749880,3520937983,US 3520937984,3520954367,CA -3520954368,3520958783,US +3520954368,3520956155,US +3520956156,3520956159,CH +3520956160,3520958783,US 3520958784,3520958847,CA 3520958848,3520966111,US 3520966112,3520966119,CA 3520966120,3520978943,US 3520978944,3520979711,BZ -3520979712,3520983039,A2 -3520983040,3520999423,US +3520979712,3520999423,US 3520999424,3521003519,CA 3521003520,3521007103,US 3521007104,3521007111,IN @@ -104595,7 +110787,9 @@ 3521966080,3522029439,US 3522029440,3522029503,FI 3522029504,3522029567,CA -3522029568,3522031871,US +3522029568,3522029823,US +3522029824,3522029855,A1 +3522029856,3522031871,US 3522031872,3522032127,AT 3522032128,3522034447,US 3522034448,3522034463,GB @@ -104617,7 +110811,7 @@ 3522132544,3522132575,US 3522132576,3522132607,UA 3522132608,3522132639,CA -3522132640,3522132671,TR +3522132640,3522132671,PE 3522132672,3522132703,BR 3522132704,3522132735,LK 3522132736,3522132767,US @@ -105122,7 +111316,9 @@ 3557028032,3557028063,GB 3557028064,3557028095,BE 3557028096,3557028351,GB -3557028352,3557028735,BE +3557028352,3557028543,BE +3557028544,3557028607,GB +3557028608,3557028735,BE 3557028736,3557028799,GB 3557028800,3557028911,BE 3557028912,3557028927,GB @@ -105203,8 +111399,10 @@ 3557335952,3557335967,LU 3557335968,3557336191,BE 3557336192,3557336255,EU -3557336256,3557336639,BE -3557336640,3557336703,EU +3557336256,3557336319,BE +3557336320,3557336575,EU +3557336576,3557336655,BE +3557336656,3557336703,EU 3557336704,3557336767,BE 3557336768,3557338111,EU 3557338112,3557338367,BE @@ -105221,9 +111419,9 @@ 3557340160,3557340191,EU 3557340192,3557340927,BE 3557340928,3557341183,EU -3557341184,3557341455,BE -3557341456,3557341471,EU -3557341472,3557341527,BE +3557341184,3557341439,BE +3557341440,3557341487,EU +3557341488,3557341527,BE 3557341528,3557341535,EU 3557341536,3557341551,BE 3557341552,3557341559,EU @@ -105260,8 +111458,8 @@ 3557360624,3557360639,JE 3557360640,3557360680,GB 3557360681,3557360687,JE -3557360688,3557360767,GB -3557360768,3557360895,JE +3557360688,3557360807,GB +3557360808,3557360895,JE 3557360896,3557360927,GB 3557360928,3557360943,JE 3557360944,3557360959,GB @@ -105283,13 +111481,15 @@ 3557361424,3557361607,GB 3557361608,3557361615,JE 3557361616,3557361919,GB -3557361920,3557362431,JE +3557361920,3557361983,JE +3557361984,3557362047,GB +3557362048,3557362431,JE 3557362432,3557362687,GB 3557362688,3557363455,JE 3557363456,3557363471,GB 3557363472,3557363479,JE -3557363480,3557363599,GB -3557363600,3557363783,JE +3557363480,3557363639,GB +3557363640,3557363783,JE 3557363784,3557363791,GB 3557363792,3557364223,JE 3557364224,3557364479,GB @@ -105334,13 +111534,13 @@ 3557365912,3557365919,JE 3557365920,3557365935,GB 3557365936,3557366015,JE -3557366016,3557366111,GB +3557366016,3557366055,GB +3557366056,3557366063,JE +3557366064,3557366111,GB 3557366112,3557366119,JE 3557366120,3557366263,GB 3557366264,3557366271,JE -3557366272,3557366295,GB -3557366296,3557366303,JE -3557366304,3557367807,GB +3557366272,3557367807,GB 3557367808,3557375999,DE 3557376000,3557384191,ES 3557384192,3557392383,GB @@ -105462,7 +111662,7 @@ 3558155008,3558155135,DE 3558155136,3558155263,SD 3558155264,3558155391,ET -3558155392,3558155519,DE +3558155392,3558155519,GB 3558155520,3558156031,SD 3558156032,3558156032,DE 3558156033,3558156287,KG @@ -105484,7 +111684,9 @@ 3558196192,3558196199,ES 3558196200,3558196271,IT 3558196272,3558196351,ES -3558196352,3558196415,IT +3558196352,3558196359,IT +3558196360,3558196367,ES +3558196368,3558196415,IT 3558196416,3558196543,ES 3558196544,3558196607,IT 3558196608,3558196719,ES @@ -105580,9 +111782,10 @@ 3558292288,3558292543,NL 3558292544,3558292607,GB 3558292608,3558292735,NL -3558292736,3558292863,GB -3558292864,3558292871,DE -3558292872,3558293055,NL +3558292736,3558292871,GB +3558292872,3558292879,NL +3558292880,3558292895,GB +3558292896,3558293055,NL 3558293056,3558293247,GB 3558293248,3558293503,NL 3558293504,3558301695,RU @@ -105708,42 +111911,96 @@ 3558737280,3558737919,GG 3558737920,3558738431,GB 3558738432,3558738687,GG -3558738688,3558738991,GB +3558738688,3558738751,GB +3558738752,3558738943,GG +3558738944,3558738991,GB 3558738992,3558739007,GG 3558739008,3558739039,GB 3558739040,3558739055,GG 3558739056,3558739071,GB 3558739072,3558739087,GG -3558739088,3558739183,GB -3558739184,3558739263,GG -3558739264,3558739295,GB -3558739296,3558739423,GG -3558739424,3558739455,GB -3558739456,3558739711,GG +3558739088,3558739199,GB +3558739200,3558739231,GG +3558739232,3558739295,GB +3558739296,3558739711,GG 3558739712,3558739975,GB 3558739976,3558739983,GG -3558739984,3558740031,GB +3558739984,3558740015,GB +3558740016,3558740023,GG +3558740024,3558740031,GB 3558740032,3558740039,GG -3558740040,3558740143,GB -3558740144,3558740151,GG -3558740152,3558740183,GB +3558740040,3558740063,GB +3558740064,3558740079,GG +3558740080,3558740087,GB +3558740088,3558740095,GG +3558740096,3558740183,GB 3558740184,3558740199,GG 3558740200,3558740207,GB 3558740208,3558740215,GG 3558740216,3558740222,GB -3558740223,3558740239,GG -3558740240,3558740255,GB -3558740256,3558740303,GG -3558740304,3558740391,GB -3558740392,3558740607,GG +3558740223,3558740223,GG +3558740224,3558740255,GB +3558740256,3558740271,GG +3558740272,3558740287,GB +3558740288,3558740295,GG +3558740296,3558740343,GB +3558740344,3558740351,GG +3558740352,3558740359,GB +3558740360,3558740367,GG +3558740368,3558740383,GB +3558740384,3558740607,GG 3558740608,3558740687,GB -3558740688,3558741247,GG -3558741248,3558741503,GB +3558740688,3558741255,GG +3558741256,3558741271,GB +3558741272,3558741279,GG +3558741280,3558741287,GB +3558741288,3558741295,GG +3558741296,3558741311,GB +3558741312,3558741335,GG +3558741336,3558741359,GB +3558741360,3558741367,GG +3558741368,3558741391,GB +3558741392,3558741399,GG +3558741400,3558741423,GB +3558741424,3558741431,GG +3558741432,3558741439,GB +3558741440,3558741447,GG +3558741448,3558741471,GB +3558741472,3558741487,GG +3558741488,3558741503,GB 3558741504,3558742015,GG -3558742016,3558742271,GB -3558742272,3558742527,GG -3558742528,3558742783,GB -3558742784,3558743039,GG +3558742016,3558742039,GB +3558742040,3558742047,GG +3558742048,3558742055,GB +3558742056,3558742071,GG +3558742072,3558742079,GB +3558742080,3558742087,GG +3558742088,3558742103,GB +3558742104,3558742135,GG +3558742136,3558742159,GB +3558742160,3558742191,GG +3558742192,3558742215,GB +3558742216,3558742223,GG +3558742224,3558742239,GB +3558742240,3558742247,GG +3558742248,3558742255,GB +3558742256,3558742527,GG +3558742528,3558742543,GB +3558742544,3558742551,GG +3558742552,3558742567,GB +3558742568,3558742583,GG +3558742584,3558742623,GB +3558742624,3558742631,GG +3558742632,3558742655,GB +3558742656,3558742663,GG +3558742664,3558742679,GB +3558742680,3558742695,GG +3558742696,3558742727,GB +3558742728,3558742735,GG +3558742736,3558742743,GB +3558742744,3558742751,GG +3558742752,3558742775,GB +3558742776,3558743039,GG 3558743040,3558743107,GB 3558743108,3558743111,GG 3558743112,3558743279,GB @@ -105849,8 +112106,8 @@ 3559089608,3559089611,GB 3559089612,3559089623,BE 3559089624,3559089627,GB -3559089628,3559089639,BE -3559089640,3559089643,GB +3559089628,3559089631,BE +3559089632,3559089643,GB 3559089644,3559090071,BE 3559090072,3559090079,NL 3559090080,3559090815,BE @@ -105871,7 +112128,9 @@ 3559091400,3559091407,GB 3559091408,3559091423,BE 3559091424,3559091427,GB -3559091428,3559091507,BE +3559091428,3559091431,BE +3559091432,3559091439,GB +3559091440,3559091507,BE 3559091508,3559091511,GB 3559091512,3559091519,BE 3559091520,3559091543,GB @@ -105945,7 +112204,9 @@ 3559095592,3559095623,GB 3559095624,3559095631,BE 3559095632,3559095647,GB -3559095648,3559096087,BE +3559095648,3559095871,BE +3559095872,3559095887,GB +3559095888,3559096087,BE 3559096088,3559096095,GB 3559096096,3559096123,BE 3559096124,3559096127,GB @@ -105963,7 +112224,46 @@ 3559129088,3559137279,BG 3559137280,3559145471,GB 3559145472,3559153663,ES -3559153664,3559177215,GB +3559153664,3559161855,GB +3559161856,3559161887,FR +3559161888,3559161919,GB +3559161920,3559161983,SE +3559161984,3559162047,GB +3559162048,3559162079,BE +3559162080,3559163903,GB +3559163904,3559163935,FR +3559163936,3559163999,GB +3559164000,3559164031,NL +3559164032,3559164087,GB +3559164088,3559164095,SE +3559164096,3559164151,GB +3559164152,3559164159,IT +3559164160,3559166335,GB +3559166336,3559166351,CH +3559166352,3559169023,GB +3559169024,3559169055,CH +3559169056,3559174703,GB +3559174704,3559174711,IT +3559174712,3559174743,GB +3559174744,3559174751,DE +3559174752,3559174767,GB +3559174768,3559174775,DE +3559174776,3559174783,GB +3559174784,3559174791,DK +3559174792,3559174799,GB +3559174800,3559174807,NO +3559174808,3559174943,GB +3559174944,3559174951,IT +3559174952,3559174975,GB +3559174976,3559174983,DE +3559174984,3559175071,GB +3559175072,3559175095,FR +3559175096,3559175127,GB +3559175128,3559175135,DE +3559175136,3559175143,FR +3559175144,3559175151,GB +3559175152,3559175167,FR +3559175168,3559177215,GB 3559177216,3559177471,DE 3559177472,3559177727,GB 3559177728,3559178239,DE @@ -106019,14 +112319,9 @@ 3559490752,3559490783,ES 3559490784,3559490791,CH 3559490792,3559490799,BE -3559490800,3559490815,ES -3559490816,3559491071,NL -3559491072,3559491103,ES -3559491104,3559491135,NL +3559490800,3559491135,NL 3559491136,3559491199,ES -3559491200,3559491247,NL -3559491248,3559491263,ES -3559491264,3559491327,NL +3559491200,3559491327,NL 3559491328,3559491359,ES 3559491360,3559491447,NL 3559491448,3559491455,ES @@ -106036,8 +112331,8 @@ 3559491840,3559491871,ES 3559491872,3559491903,GB 3559491904,3559491967,ES -3559491968,3559491999,NL -3559492000,3559492007,ES +3559491968,3559492003,NL +3559492004,3559492007,ES 3559492008,3559492013,NL 3559492014,3559492031,ES 3559492032,3559493247,NL @@ -106108,11 +112403,7 @@ 3559899392,3559899395,EE 3559899396,3559899487,UA 3559899488,3559899519,EE -3559899520,3559899791,UA -3559899792,3559899807,EE -3559899808,3559899855,UA -3559899856,3559899871,EE -3559899872,3559899903,UA +3559899520,3559899903,UA 3559899904,3559899907,EE 3559899908,3559899935,RU 3559899936,3559899951,UA @@ -106175,16 +112466,18 @@ 3559905152,3559905155,EE 3559905156,3559905159,UA 3559905160,3559905297,EE -3559905298,3559905535,UA +3559905298,3559905299,UA +3559905300,3559905303,EE +3559905304,3559905535,UA 3559905536,3559905623,EE 3559905624,3559905631,UA -3559905632,3559905719,EE -3559905720,3559905727,UA -3559905728,3559906255,EE +3559905632,3559905911,EE +3559905912,3559905919,UA +3559905920,3559905951,EE +3559905952,3559905983,UA +3559905984,3559906255,EE 3559906256,3559906263,UA -3559906264,3559906735,EE -3559906736,3559906751,UA -3559906752,3559906783,EE +3559906264,3559906783,EE 3559906784,3559906815,UA 3559906816,3559906975,EE 3559906976,3559907071,UA @@ -106245,7 +112538,6 @@ 3560316928,3560325119,NL 3560325120,3560333311,DK 3560333312,3560341503,RO -3560341504,3560349695,PT 3560349696,3560357887,GB 3560357888,3560366079,GR 3560366080,3560366687,CH @@ -106938,8 +113230,7 @@ 3560942588,3560942590,ES 3560942591,3560942599,DE 3560942600,3560942603,DK -3560942604,3560942607,LU -3560942608,3560942619,DE +3560942604,3560942619,DE 3560942620,3560942623,BE 3560942624,3560942627,IT 3560942628,3560942631,GB @@ -107623,7 +113914,7 @@ 3560943837,3560943837,CH 3560943838,3560943838,DE 3560943839,3560943839,NO -3560943840,3560943840,LU +3560943840,3560943840,DE 3560943841,3560943841,GB 3560943842,3560943842,DE 3560943843,3560943843,GB @@ -108300,11 +114591,17 @@ 3560947712,3560950863,SE 3560950864,3560950867,DK 3560950868,3560950871,ES -3560950872,3560951027,SE +3560950872,3560950956,SE +3560950957,3560950958,PL +3560950959,3560951027,SE 3560951028,3560951031,DK 3560951032,3560951039,SE 3560951040,3560951043,NO -3560951044,3560951103,SE +3560951044,3560951088,SE +3560951089,3560951090,PL +3560951091,3560951091,SE +3560951092,3560951095,IT +3560951096,3560951103,SE 3560951104,3560951107,DE 3560951108,3560951111,DK 3560951112,3560951115,FR @@ -108314,27 +114611,46 @@ 3560951208,3560951211,DK 3560951212,3560951307,SE 3560951308,3560951311,IT -3560951312,3560951383,SE +3560951312,3560951376,SE +3560951377,3560951378,PL +3560951379,3560951380,SE +3560951381,3560951382,FI +3560951383,3560951383,SE 3560951384,3560951387,NO -3560951388,3560951391,SE +3560951388,3560951388,SE +3560951389,3560951390,PT +3560951391,3560951391,SE 3560951392,3560951395,IT 3560951396,3560951399,DE 3560951400,3560951403,SE 3560951404,3560951407,NO 3560951408,3560951423,SE 3560951424,3560951427,NL -3560951428,3560951447,SE +3560951428,3560951431,DK +3560951432,3560951440,SE +3560951441,3560951442,JP +3560951443,3560951444,SE +3560951445,3560951446,HU +3560951447,3560951447,SE 3560951448,3560951451,FR 3560951452,3560951551,SE 3560951552,3560951555,ES 3560951556,3560951559,NO 3560951560,3560951583,SE 3560951584,3560951587,FI -3560951588,3560951679,SE +3560951588,3560951591,SE +3560951592,3560951595,NO +3560951596,3560951679,SE 3560951680,3560951683,DE 3560951684,3560951807,SE 3560951808,3560951811,US -3560951812,3560951915,SE +3560951812,3560951816,SE +3560951817,3560951818,ES +3560951819,3560951820,SE +3560951821,3560951822,IT +3560951823,3560951831,SE +3560951832,3560951835,DE +3560951836,3560951915,SE 3560951916,3560951919,PL 3560951920,3560951935,SE 3560951936,3560951940,NL @@ -108342,7 +114658,8 @@ 3560952224,3560952227,CZ 3560952228,3560952231,SE 3560952232,3560952235,US -3560952236,3560953103,SE +3560952236,3560952239,PT +3560952240,3560953103,SE 3560953104,3560953119,JP 3560953120,3560954239,SE 3560954240,3560954367,AX @@ -108428,7 +114745,9 @@ 3561496888,3561496891,NL 3561496892,3561496955,GB 3561496956,3561496959,NL -3561496960,3561497215,GB +3561496960,3561497087,GB +3561497088,3561497119,NL +3561497120,3561497215,GB 3561497216,3561497423,NL 3561497424,3561497471,GB 3561497472,3561497535,NL @@ -108492,8 +114811,8 @@ 3561603264,3561603295,GB 3561603296,3561603455,FR 3561603456,3561603487,GB -3561603488,3561604351,FR -3561604352,3561604895,GB +3561603488,3561604607,FR +3561604608,3561604895,GB 3561604896,3561604911,FR 3561604912,3561605119,GB 3561605120,3561607231,FR @@ -108566,7 +114885,7 @@ 3561709568,3561717759,PL 3561717760,3561725951,RU 3561725952,3561734143,ES -3561734144,3561742335,RU +3561734144,3561750527,RU 3561750528,3561758719,KZ 3561758720,3561766911,PL 3561766912,3561775103,SA @@ -108593,8 +114912,11 @@ 3561910208,3561910223,GB 3561910224,3561914367,DE 3561914368,3561922559,ES -3561922560,3561922855,NL -3561922856,3561922911,GB +3561922560,3561922815,NL +3561922816,3561922847,GB +3561922848,3561922863,NL +3561922864,3561922871,FR +3561922872,3561922911,GB 3561922912,3561922975,NL 3561922976,3561922991,GB 3561922992,3561923015,NL @@ -108604,12 +114926,15 @@ 3561923072,3561923327,NL 3561923328,3561923403,GB 3561923404,3561923407,NL -3561923408,3561923455,GB -3561923456,3561923551,NL +3561923408,3561923423,GB +3561923424,3561923551,NL 3561923552,3561923583,GB 3561923584,3561923679,NL -3561923680,3561923871,GB -3561923872,3561924351,NL +3561923680,3561923711,GB +3561923712,3561923743,FR +3561923744,3561923775,NL +3561923776,3561923839,GB +3561923840,3561924351,NL 3561924352,3561924607,GB 3561924608,3561924647,NL 3561924648,3561924655,GB @@ -108618,33 +114943,47 @@ 3561924680,3561924687,NL 3561924688,3561924703,GB 3561924704,3561924719,NL -3561924720,3561924911,GB -3561924912,3561924959,NL -3561924960,3561925007,GB -3561925008,3561925039,NL +3561924720,3561924895,GB +3561924896,3561924903,FR +3561924904,3561924911,GB +3561924912,3561925039,NL 3561925040,3561925087,GB 3561925088,3561925119,NL 3561925120,3561926943,GB 3561926944,3561926975,NL -3561926976,3561927167,GB -3561927168,3561927423,NL -3561927424,3561927551,GB +3561926976,3561926983,GB +3561926984,3561926999,NL +3561927000,3561927007,GB +3561927008,3561927039,NL +3561927040,3561927103,GB +3561927104,3561927231,NL +3561927232,3561927551,GB 3561927552,3561927647,NL 3561927648,3561927663,GB -3561927664,3561927679,NL -3561927680,3561927823,GB -3561927824,3561927839,NL -3561927840,3561928191,GB -3561928192,3561929727,NL +3561927664,3561929727,NL 3561929728,3561929743,GB -3561929744,3561929791,NL -3561929792,3561929879,GB +3561929744,3561929823,NL +3561929824,3561929855,CH +3561929856,3561929879,GB 3561929880,3561929919,NL 3561929920,3561929967,GB 3561929968,3561930335,NL -3561930336,3561930503,GB -3561930504,3561930511,NL -3561930512,3561930751,GB +3561930336,3561930495,GB +3561930496,3561930511,NL +3561930512,3561930519,BE +3561930520,3561930527,FR +3561930528,3561930535,CH +3561930536,3561930543,GB +3561930544,3561930551,FR +3561930552,3561930575,NL +3561930576,3561930583,DE +3561930584,3561930599,NL +3561930600,3561930615,GB +3561930616,3561930671,NL +3561930672,3561930679,GB +3561930680,3561930719,NL +3561930720,3561930727,GB +3561930728,3561938943,NL 3561938944,3561940991,IE 3561940992,3561942015,GB 3561942016,3561947135,IE @@ -108652,12 +114991,13 @@ 3561963144,3561963151,AU 3561963152,3561963519,DE 3561963520,3561971711,BE -3561971712,3561975807,TR 3561975808,3561979903,UA 3561979904,3561988095,ES 3561988096,3562004479,DE 3562004480,3562012671,NL -3562012672,3562020863,IR +3562012672,3562020095,IR +3562020096,3562020351,AE +3562020352,3562020863,IR 3562020864,3562029055,UA 3562029056,3562037247,CZ 3562037248,3562045439,CH @@ -108867,9 +115207,7 @@ 3562321232,3562321239,BE 3562321240,3562322743,GB 3562322744,3562322751,GG -3562322752,3562323007,GB -3562323008,3562323015,US -3562323016,3562323087,GB +3562322752,3562323087,GB 3562323088,3562323095,ES 3562323096,3562323135,GB 3562323136,3562323143,BE @@ -108882,12 +115220,14 @@ 3562356736,3562364927,SE 3562364928,3562373119,PL 3562373120,3562381311,FR +3562381312,3562389503,SI 3562389504,3562395647,GH 3562395648,3562396159,TZ 3562396160,3562397695,SZ 3562397696,3562405887,GB 3562405888,3562414079,FI 3562414080,3562422271,DE +3562422272,3562430463,IR 3562430464,3562438655,UA 3562438656,3562463231,DE 3562463232,3562471423,UA @@ -109274,15 +115614,11 @@ 3563872256,3563880447,RU 3563880448,3563888639,TR 3563888640,3563896831,DE -3563896832,3563899391,HU -3563899392,3563899647,SK -3563899648,3563899903,HU +3563896832,3563899903,HU 3563899904,3563900159,SK 3563900160,3563900415,HU 3563900416,3563900671,SK -3563900672,3563902975,HU -3563902976,3563903231,SK -3563903232,3563913215,HU +3563900672,3563913215,HU 3563913216,3563921407,RU 3563921408,3563929599,UA 3563929600,3563937791,CH @@ -109299,18 +115635,14 @@ 3564011520,3564019711,UA 3564019712,3564021247,GB 3564021248,3564021279,IT -3564021280,3564023967,GB -3564023968,3564023983,IT -3564023984,3564024095,GB +3564021280,3564024095,GB 3564024096,3564024103,IT 3564024104,3564024135,GB 3564024136,3564024143,IT -3564024144,3564024287,GB -3564024288,3564024319,IT -3564024320,3564024343,GB -3564024344,3564024447,IT -3564024448,3564024463,GB -3564024464,3564024831,IT +3564024144,3564024447,GB +3564024448,3564024463,IT +3564024464,3564024671,GB +3564024672,3564024831,IT 3564024832,3564027903,GB 3564027904,3564041215,DE 3564041216,3564041727,RU @@ -109323,7 +115655,13 @@ 3564093440,3564101631,UA 3564101632,3564109823,DE 3564109824,3564126207,SE -3564126208,3564134399,IT +3564126208,3564128287,IT +3564128288,3564128303,BG +3564128304,3564129727,IT +3564129728,3564129743,PH +3564129744,3564130655,IT +3564130656,3564130671,IN +3564130672,3564134399,IT 3564134400,3564142591,KG 3564142592,3564146175,DE 3564146176,3564150783,DK @@ -109520,8 +115858,7 @@ 3564470272,3564473439,IL 3564473440,3564473455,A2 3564473456,3564478463,IL -3564478464,3564486399,LU -3564486400,3564486655,A1 +3564478464,3564486655,LU 3564486656,3564494847,CH 3564494848,3564494907,GB 3564494908,3564494911,FR @@ -109607,8 +115944,8 @@ 3564561792,3564561919,GB 3564561920,3564562431,CH 3564562432,3564562687,GB -3564562688,3564562821,US -3564562822,3564562879,GB +3564562688,3564562831,US +3564562832,3564562879,GB 3564562880,3564562943,US 3564562944,3564563711,GB 3564563712,3564563967,US @@ -109667,8 +116004,8 @@ 3564634112,3564642303,MK 3564642304,3564650495,CH 3564650496,3564666879,NL -3564666880,3564668943,IE -3564668944,3564668991,GB +3564666880,3564668959,IE +3564668960,3564668991,GB 3564668992,3564669103,IE 3564669104,3564669119,GB 3564669120,3564669407,IE @@ -109694,26 +116031,24 @@ 3564699648,3564716031,GB 3564716032,3564724223,IT 3564724224,3564732415,NL -3564732416,3564734239,DE -3564734240,3564734271,GB -3564734272,3564734287,DE +3564732416,3564734287,DE 3564734288,3564734303,GB 3564734304,3564734335,DE 3564734336,3564734431,GB -3564734432,3564734719,DE -3564734720,3564734807,GB -3564734808,3564734815,DE +3564734432,3564734815,DE 3564734816,3564734819,GB -3564734820,3564735231,DE -3564735232,3564735487,GB -3564735488,3564736511,DE +3564734820,3564734847,DE +3564734848,3564734975,GB +3564734976,3564736511,DE 3564736512,3564736527,GB 3564736528,3564736583,DE 3564736584,3564736603,GB 3564736604,3564736607,DE 3564736608,3564736611,GB -3564736612,3564736767,DE -3564736768,3564736887,GB +3564736612,3564736623,DE +3564736624,3564736631,GB +3564736632,3564736871,DE +3564736872,3564736887,GB 3564736888,3564736895,DE 3564736896,3564736903,GB 3564736904,3564736911,DE @@ -109722,35 +116057,23 @@ 3564736968,3564736975,GB 3564736976,3564736991,DE 3564736992,3564737023,GB -3564737024,3564737535,DE -3564737536,3564737791,GB -3564737792,3564738047,DE -3564738048,3564738303,GB -3564738304,3564739327,DE -3564739328,3564739335,GB -3564739336,3564739351,DE -3564739352,3564739391,GB -3564739392,3564739455,DE -3564739456,3564739503,GB -3564739504,3564739519,DE +3564737024,3564739383,DE +3564739384,3564739391,ES +3564739392,3564739495,DE +3564739496,3564739503,GB +3564739504,3564739519,AT 3564739520,3564739807,GB 3564739808,3564739815,DE 3564739816,3564739823,GB 3564739824,3564740351,DE 3564740352,3564740383,GB -3564740384,3564740415,DE -3564740416,3564740447,GB -3564740448,3564740455,DE +3564740384,3564740455,DE 3564740456,3564740459,GB -3564740460,3564740463,DE -3564740464,3564740471,GB -3564740472,3564740479,DE +3564740460,3564740479,DE 3564740480,3564740495,GB 3564740496,3564740503,DE 3564740504,3564740527,GB -3564740528,3564740543,DE -3564740544,3564740575,GB -3564740576,3564740591,DE +3564740528,3564740591,DE 3564740592,3564740595,GB 3564740596,3564740599,DE 3564740600,3564740603,GB @@ -109781,9 +116104,7 @@ 3564880992,3564881007,NL 3564881008,3564881103,GB 3564881104,3564881151,NL -3564881152,3564881183,GB -3564881184,3564881199,NL -3564881200,3564881231,GB +3564881152,3564881231,GB 3564881232,3564881263,NL 3564881264,3564881343,GB 3564881344,3564881359,NL @@ -109811,9 +116132,7 @@ 3564882144,3564882239,NL 3564882240,3564882255,GB 3564882256,3564882271,NL -3564882272,3564882335,GB -3564882336,3564882367,NL -3564882368,3564882399,GB +3564882272,3564882399,GB 3564882400,3564882943,NL 3564882944,3564883007,GB 3564883008,3564883039,NL @@ -109909,7 +116228,8 @@ 3564888672,3564888735,BE 3564888736,3564888815,NL 3564888816,3564888823,GB -3564888824,3564888991,NL +3564888824,3564888959,NL +3564888960,3564888991,GB 3564888992,3564889023,FR 3564889024,3564889087,NL 3564889088,3564889119,GB @@ -110028,7 +116348,9 @@ 3564956720,3564956735,GB 3564956736,3564956751,AT 3564956752,3564956767,US -3564956768,3564956855,AT +3564956768,3564956831,AT +3564956832,3564956839,GB +3564956840,3564956855,AT 3564956856,3564956903,GB 3564956904,3564956907,AT 3564956908,3564956911,CH @@ -110196,13 +116518,13 @@ 3565036288,3565036543,GB 3565036544,3565037119,IE 3565037120,3565037135,GB -3565037136,3565037167,IE -3565037168,3565037567,GB +3565037136,3565037199,IE +3565037200,3565037215,GB +3565037216,3565037247,IE +3565037248,3565037567,GB 3565037568,3565037823,IE 3565037824,3565038591,GB -3565038592,3565038643,IE -3565038644,3565038647,GB -3565038648,3565038663,IE +3565038592,3565038663,IE 3565038664,3565043711,GB 3565043712,3565047807,AT 3565047808,3565048063,LI @@ -110681,7 +117003,9 @@ 3567152992,3567153023,EU 3567153024,3567153055,GB 3567153056,3567153599,EU -3567153600,3567153623,GB +3567153600,3567153607,GB +3567153608,3567153615,EU +3567153616,3567153623,GB 3567153624,3567153647,EU 3567153648,3567153663,GB 3567153664,3567153687,EU @@ -110706,7 +117030,15 @@ 3567250176,3567250943,NL 3567250944,3567255551,EU 3567255552,3567321087,PL -3567321088,3567353855,GB +3567321088,3567323135,GB +3567323136,3567323391,IE +3567323392,3567350807,GB +3567350808,3567350815,US +3567350816,3567350839,GB +3567350840,3567350847,CH +3567350848,3567350935,GB +3567350936,3567350943,CH +3567350944,3567353855,GB 3567353856,3567386623,CH 3567386624,3567386879,GB 3567386880,3567386883,DE @@ -111038,11 +117370,7 @@ 3569241888,3569241903,LU 3569241904,3569242047,BE 3569242048,3569242111,LU -3569242112,3569242551,BE -3569242552,3569242559,LU -3569242560,3569242879,BE -3569242880,3569243135,US -3569243136,3569243903,BE +3569242112,3569243903,BE 3569243904,3569244031,NL 3569244032,3569244303,BE 3569244304,3569244319,LU @@ -111057,11 +117385,9 @@ 3569252992,3569253007,LU 3569253008,3569253095,BE 3569253096,3569253103,LU -3569253104,3569254911,BE -3569254912,3569254943,LU -3569254944,3569255007,BE -3569255008,3569255055,LU -3569255056,3569255071,BE +3569253104,3569255007,BE +3569255008,3569255039,LU +3569255040,3569255071,BE 3569255072,3569255103,LU 3569255104,3569258975,BE 3569258976,3569259007,LU @@ -111075,9 +117401,7 @@ 3569271936,3569271943,IT 3569271944,3569272063,BE 3569272064,3569272079,LU -3569272080,3569272239,BE -3569272240,3569272255,LU -3569272256,3569273167,BE +3569272080,3569273167,BE 3569273168,3569273183,LU 3569273184,3569273791,BE 3569273792,3569273823,LU @@ -111386,9 +117710,7 @@ 3569942528,3570064759,DE 3570064760,3570064767,A2 3570064768,3570073599,DE -3570073600,3570076943,NL -3570076944,3570076951,PH -3570076952,3570081791,NL +3570073600,3570081791,NL 3570081792,3570102463,CH 3570102464,3570102479,ES 3570102480,3570106367,CH @@ -111401,16 +117723,16 @@ 3570215680,3570215807,DE 3570215808,3570223103,GR 3570223104,3570223231,DE -3570223232,3570236927,GR +3570223232,3570236159,GR +3570236160,3570236415,DE +3570236416,3570236927,GR 3570236928,3570237183,DE 3570237184,3570241791,GR 3570241792,3570242047,DE 3570242048,3570270207,GR 3570270208,3570335743,NL 3570335744,3570401279,GB -3570401280,3570442687,FR -3570442688,3570442695,IT -3570442696,3570466815,FR +3570401280,3570466815,FR 3570466816,3570507439,SE 3570507440,3570507455,CH 3570507456,3570532351,SE @@ -112065,13 +118387,13 @@ 3574155728,3574155731,AT 3574155732,3574155739,DE 3574155740,3574155743,AT -3574155744,3574155819,DE -3574155820,3574155823,NO -3574155824,3574155851,DE +3574155744,3574155851,DE 3574155852,3574155855,NO 3574155856,3574155859,DE 3574155860,3574155863,NO -3574155864,3574159919,DE +3574155864,3574155883,DE +3574155884,3574155887,NL +3574155888,3574159919,DE 3574159920,3574159927,NL 3574159928,3574160931,DE 3574160932,3574160935,NL @@ -112081,9 +118403,7 @@ 3574161056,3574161087,NL 3574161088,3574161107,DE 3574161108,3574161111,NL -3574161112,3574161319,DE -3574161320,3574161323,NO -3574161324,3574161335,DE +3574161112,3574161335,DE 3574161336,3574161339,AI 3574161340,3574161391,DE 3574161392,3574161395,NO @@ -112165,7 +118485,11 @@ 3574917376,3574917631,CH 3574917632,3574919359,DE 3574919360,3574919391,FR -3574919392,3574923263,DE +3574919392,3574920223,DE +3574920224,3574920239,RO +3574920240,3574920503,DE +3574920504,3574920511,RO +3574920512,3574923263,DE 3574923264,3574939647,RU 3574939648,3574956031,SE 3574956032,3574972415,IT @@ -112560,15 +118884,104 @@ 3575775488,3575775743,AT 3575775744,3575824383,NL 3575824384,3575832575,KW -3575832576,3575857151,NL +3575832576,3575840767,NL +3575840768,3575848991,GB +3575848992,3575849007,NL +3575849008,3575850255,GB +3575850256,3575850263,NL +3575850264,3575850543,GB +3575850544,3575850559,NL +3575850560,3575850783,GB +3575850784,3575850815,NL +3575850816,3575852799,GB +3575852800,3575853055,NL +3575853056,3575853311,GB +3575853312,3575853375,NL +3575853376,3575853439,GB +3575853440,3575853503,NL +3575853504,3575853567,GB +3575853568,3575853579,NL +3575853580,3575853587,GB +3575853588,3575853591,NL +3575853592,3575853619,GB +3575853620,3575853631,NL +3575853632,3575853635,GB +3575853636,3575853647,NL +3575853648,3575853655,GB +3575853656,3575853663,NL +3575853664,3575853691,GB +3575853692,3575853695,NL +3575853696,3575853719,GB +3575853720,3575853723,NL +3575853724,3575853739,GB +3575853740,3575853743,NL +3575853744,3575853763,GB +3575853764,3575853767,NL +3575853768,3575853799,GB +3575853800,3575853807,NL +3575853808,3575853815,GB +3575853816,3575853823,NL +3575853824,3575853831,GB +3575853832,3575853863,NL +3575853864,3575853871,GB +3575853872,3575853879,NL +3575853880,3575853895,GB +3575853896,3575853927,NL +3575853928,3575853943,GB +3575853944,3575853951,NL +3575853952,3575853959,GB +3575853960,3575853975,NL +3575853976,3575853999,GB +3575854000,3575854015,NL +3575854016,3575854031,GB +3575854032,3575854039,NL +3575854040,3575857151,GB 3575857152,3575857407,BE -3575857408,3575858529,NL +3575857408,3575858529,GB 3575858530,3575858543,US -3575858544,3575860299,NL +3575858544,3575860299,GB 3575860300,3575860303,BE -3575860304,3575866367,NL +3575860304,3575860735,GB +3575860736,3575860863,NL +3575860864,3575860991,GB +3575860992,3575861247,NL +3575861248,3575861503,GB +3575861504,3575861519,NL +3575861520,3575864327,GB +3575864328,3575864331,NL +3575864332,3575864343,GB +3575864344,3575864347,NL +3575864348,3575864359,GB +3575864360,3575864363,NL +3575864364,3575864380,GB +3575864381,3575864382,NL +3575864383,3575864387,GB +3575864388,3575864391,NL +3575864392,3575864395,GB +3575864396,3575864403,NL +3575864404,3575864583,GB +3575864584,3575864599,NL +3575864600,3575864607,GB +3575864608,3575864615,NL +3575864616,3575864631,GB +3575864632,3575864639,NL +3575864640,3575864663,GB +3575864664,3575864671,NL +3575864672,3575864687,GB +3575864688,3575864703,NL +3575864704,3575864775,GB +3575864776,3575864783,NL +3575864784,3575865087,GB +3575865088,3575865215,NL +3575865216,3575866367,GB 3575866368,3575867135,US -3575867136,3575873535,NL +3575867136,3575872047,GB +3575872048,3575872063,NL +3575872064,3575872079,GB +3575872080,3575872095,NL +3575872096,3575872255,GB +3575872256,3575872767,NL +3575872768,3575873535,GB 3575873536,3575906303,EE 3575906304,3575971839,FR 3575971840,3576037375,ES @@ -112625,7 +119038,9 @@ 3576042240,3576042495,EU 3576042496,3576042623,GB 3576042624,3576042687,EU -3576042688,3576042815,GB +3576042688,3576042695,GB +3576042696,3576042751,EU +3576042752,3576042815,GB 3576042816,3576045567,EU 3576045568,3576045599,GB 3576045600,3576045663,EU @@ -113011,24 +119426,14 @@ 3576237232,3576237279,GB 3576237280,3576237455,FR 3576237456,3576237503,GB -3576237504,3576237567,FR -3576237568,3576237583,GB -3576237584,3576237599,FR -3576237600,3576237647,GB -3576237648,3576237711,FR +3576237504,3576237615,FR +3576237616,3576237631,GB +3576237632,3576237711,FR 3576237712,3576237743,GB -3576237744,3576237855,FR -3576237856,3576237871,GB -3576237872,3576237887,FR -3576237888,3576237919,GB -3576237920,3576237935,FR -3576237936,3576237967,GB -3576237968,3576237983,FR -3576237984,3576238015,GB -3576238016,3576238143,FR +3576237744,3576238143,FR 3576238144,3576238159,GB -3576238160,3576238271,FR -3576238272,3576238335,GB +3576238160,3576238305,FR +3576238306,3576238335,GB 3576238336,3576238447,FR 3576238448,3576238463,GB 3576238464,3576238511,FR @@ -113038,7 +119443,9 @@ 3576238576,3576238607,FR 3576238608,3576238623,GB 3576238624,3576238639,FR -3576238640,3576238879,GB +3576238640,3576238655,GB +3576238656,3576238863,FR +3576238864,3576238879,GB 3576238880,3576238895,FR 3576238896,3576238911,GB 3576238912,3576238927,FR @@ -113049,14 +119456,14 @@ 3576239040,3576239071,GB 3576239072,3576239087,FR 3576239088,3576239103,GB -3576239104,3576239623,FR -3576239624,3576239871,GB -3576239872,3576240039,FR +3576239104,3576239663,FR +3576239664,3576239671,GB +3576239672,3576240039,FR 3576240040,3576240047,GB 3576240048,3576240111,FR 3576240112,3576240127,GB -3576240128,3576240383,FR -3576240384,3576240511,GB +3576240128,3576240475,FR +3576240476,3576240511,GB 3576240512,3576240575,FR 3576240576,3576240895,GB 3576240896,3576241183,FR @@ -113067,21 +119474,33 @@ 3576241344,3576241351,GB 3576241352,3576241383,FR 3576241384,3576241399,GB -3576241400,3576241951,FR -3576241952,3576242047,GB +3576241400,3576241935,FR +3576241936,3576241943,GB +3576241944,3576241951,FR +3576241952,3576241991,GB +3576241992,3576242007,FR +3576242008,3576242031,GB +3576242032,3576242039,FR +3576242040,3576242047,GB 3576242048,3576242111,FR -3576242112,3576242175,GB -3576242176,3576242183,FR +3576242112,3576242143,GB +3576242144,3576242183,FR 3576242184,3576242199,GB -3576242200,3576243967,FR -3576243968,3576244223,GB -3576244224,3576246399,FR -3576246400,3576246527,GB -3576246528,3576246656,FR +3576242200,3576242319,FR +3576242320,3576242327,GB +3576242328,3576243967,FR +3576243968,3576243999,GB +3576244000,3576244031,FR +3576244032,3576244047,GB +3576244048,3576244127,FR +3576244128,3576244143,GB +3576244144,3576246399,FR +3576246400,3576246463,GB +3576246464,3576246656,FR 3576246657,3576246727,GB -3576246728,3576246735,FR -3576246736,3576246783,GB -3576246784,3576249359,FR +3576246728,3576246743,FR +3576246744,3576246751,GB +3576246752,3576249359,FR 3576249360,3576249367,GB 3576249368,3576249463,FR 3576249464,3576249471,GB @@ -113103,19 +119522,15 @@ 3576251616,3576251647,GB 3576251648,3576252415,FR 3576252416,3576252671,GB -3576252672,3576253007,FR -3576253008,3576253023,GB -3576253024,3576253159,FR -3576253160,3576253183,GB -3576253184,3576254487,FR +3576252672,3576254487,FR 3576254488,3576254495,GB 3576254496,3576254551,FR 3576254552,3576254559,GB 3576254560,3576254607,FR 3576254608,3576254615,GB 3576254616,3576254623,FR -3576254624,3576254639,GB -3576254640,3576254655,FR +3576254624,3576254647,GB +3576254648,3576254655,FR 3576254656,3576254687,GB 3576254688,3576254847,FR 3576254848,3576254855,GB @@ -113123,44 +119538,34 @@ 3576254864,3576254879,GB 3576254880,3576254895,FR 3576254896,3576254903,GB -3576254904,3576255087,FR -3576255088,3576255199,GB +3576254904,3576255151,FR +3576255152,3576255199,GB 3576255200,3576255215,FR 3576255216,3576255263,GB -3576255264,3576255319,FR -3576255320,3576255327,GB -3576255328,3576255375,FR +3576255264,3576255375,FR 3576255376,3576255383,GB 3576255384,3576255407,FR 3576255408,3576255423,GB 3576255424,3576255431,FR 3576255432,3576255439,GB -3576255440,3576255503,FR -3576255504,3576255511,GB -3576255512,3576255519,FR +3576255440,3576255519,FR 3576255520,3576255527,GB 3576255528,3576255543,FR 3576255544,3576255551,GB -3576255552,3576255575,FR -3576255576,3576255583,GB -3576255584,3576255631,FR +3576255552,3576255631,FR 3576255632,3576255647,GB 3576255648,3576255671,FR 3576255672,3576255679,GB 3576255680,3576255703,FR 3576255704,3576255719,GB 3576255720,3576255743,FR -3576255744,3576255759,GB -3576255760,3576255839,FR -3576255840,3576255871,GB -3576255872,3576255887,FR +3576255744,3576255751,GB +3576255752,3576255887,FR 3576255888,3576255919,GB -3576255920,3576255943,FR -3576255944,3576255975,GB -3576255976,3576255983,FR -3576255984,3576256047,GB -3576256048,3576256111,FR -3576256112,3576256143,GB +3576255920,3576255983,FR +3576255984,3576255999,GB +3576256000,3576256127,FR +3576256128,3576256143,GB 3576256144,3576256239,FR 3576256240,3576256255,GB 3576256256,3576256351,FR @@ -113237,7 +119642,9 @@ 3576258672,3576258687,GB 3576258688,3576258703,FR 3576258704,3576258783,GB -3576258784,3576258863,FR +3576258784,3576258815,FR +3576258816,3576258847,GB +3576258848,3576258863,FR 3576258864,3576258895,GB 3576258896,3576258943,FR 3576258944,3576259007,GB @@ -113266,10 +119673,10 @@ 3576259976,3576259991,FR 3576259992,3576259999,GB 3576260000,3576260015,FR -3576260016,3576260031,GB -3576260032,3576260079,FR -3576260080,3576260095,GB -3576260096,3576260175,FR +3576260016,3576260047,GB +3576260048,3576260079,FR +3576260080,3576260111,GB +3576260112,3576260175,FR 3576260176,3576260191,GB 3576260192,3576260207,FR 3576260208,3576260223,GB @@ -113339,21 +119746,31 @@ 3576264576,3576264623,GB 3576264624,3576264639,FR 3576264640,3576264687,GB -3576264688,3576265375,FR +3576264688,3576265279,FR +3576265280,3576265287,GB +3576265288,3576265303,FR +3576265304,3576265311,GB +3576265312,3576265327,FR +3576265328,3576265335,GB +3576265336,3576265359,FR +3576265360,3576265367,GB +3576265368,3576265375,FR 3576265376,3576265383,GB 3576265384,3576265399,FR -3576265400,3576265423,GB -3576265424,3576265463,FR +3576265400,3576265431,GB +3576265432,3576265463,FR 3576265464,3576265471,GB -3576265472,3576265807,FR +3576265472,3576265759,FR +3576265760,3576265775,GB +3576265776,3576265807,FR 3576265808,3576265823,GB 3576265824,3576265831,FR 3576265832,3576265839,GB 3576265840,3576265847,FR 3576265848,3576265863,GB 3576265864,3576265879,FR -3576265880,3576265895,GB -3576265896,3576266663,FR +3576265880,3576265903,GB +3576265904,3576266663,FR 3576266664,3576266671,GB 3576266672,3576266687,FR 3576266688,3576266751,GB @@ -113393,8 +119810,8 @@ 3576889344,3576954879,NL 3576954880,3576987647,NO 3576987648,3577001983,GB -3577001984,3577002495,NL -3577002496,3577020415,GB +3577001984,3577003519,NL +3577003520,3577020415,GB 3577020416,3577085951,NL 3577085952,3577151487,DE 3577151488,3577167871,GB @@ -113429,9 +119846,7 @@ 3577562392,3577562399,GB 3577562400,3577577231,DE 3577577232,3577577247,US -3577577248,3577579039,DE -3577579040,3577579047,BE -3577579048,3577591295,DE +3577577248,3577591295,DE 3577591296,3577591551,GB 3577591552,3577592431,DE 3577592432,3577592447,FR @@ -113531,8 +119946,8 @@ 3577638968,3577639239,FR 3577639240,3577639247,EU 3577639248,3577639679,FR -3577639680,3577639759,EU -3577639760,3577639783,FR +3577639680,3577639767,EU +3577639768,3577639783,FR 3577639784,3577639799,EU 3577639800,3577639807,FR 3577639808,3577639839,EU @@ -113541,8 +119956,8 @@ 3577639864,3577639871,FR 3577639872,3577639935,EU 3577639936,3577640719,FR -3577640720,3577640727,EU -3577640728,3577641151,FR +3577640720,3577640735,EU +3577640736,3577641151,FR 3577641152,3577641159,EU 3577641160,3577641175,FR 3577641176,3577641183,EU @@ -113551,15 +119966,14 @@ 3577641216,3577641391,FR 3577641392,3577641399,EU 3577641400,3577641423,FR -3577641424,3577641471,EU -3577641472,3577641727,FR -3577641728,3577641983,EU +3577641424,3577641983,EU 3577641984,3577642023,GB 3577642024,3577642031,EU 3577642032,3577642111,GB 3577642112,3577642175,EU 3577642176,3577642239,GB -3577642240,3577642623,FR +3577642240,3577642495,EU +3577642496,3577642623,FR 3577642624,3577642751,EU 3577642752,3577643007,FR 3577643008,3577643231,NL @@ -113588,8 +120002,8 @@ 3577647936,3577647967,DK 3577647968,3577647999,EU 3577648000,3577648127,DK -3577648128,3577648383,EU -3577648384,3577649151,NL +3577648128,3577648639,EU +3577648640,3577649151,NL 3577649152,3577651199,EU 3577651200,3577651455,GB 3577651456,3577652223,EU @@ -113694,14 +120108,20 @@ 3578997936,3578997943,FR 3578997944,3578998575,DE 3578998576,3578998583,GB -3578998584,3579002879,DE +3578998584,3578999647,DE +3578999648,3578999651,CH +3578999652,3579002879,DE 3579002880,3579019263,GB 3579019264,3579052031,DK 3579052032,3579117567,NL 3579117568,3579132159,RU 3579132160,3579132415,BY 3579132416,3579183103,RU -3579183104,3579191759,GB +3579183104,3579188239,GB +3579188240,3579188255,BE +3579188256,3579188415,GB +3579188416,3579188431,IS +3579188432,3579191759,GB 3579191760,3579191775,DE 3579191776,3579193599,GB 3579193600,3579193703,NL @@ -113712,18 +120132,68 @@ 3579193728,3579193815,NL 3579193816,3579193823,ES 3579193824,3579193855,NL -3579193856,3579197055,GB +3579193856,3579194103,GB +3579194104,3579194111,US +3579194112,3579197055,GB 3579197056,3579197183,US 3579197184,3579197887,GB 3579197888,3579197903,IT -3579197904,3579228191,GB +3579197904,3579205631,GB +3579205632,3579205887,IE +3579205888,3579210079,GB +3579210080,3579210087,BE +3579210088,3579221071,GB +3579221072,3579221087,DE +3579221088,3579221103,GB +3579221104,3579221119,IT +3579221120,3579221471,GB +3579221472,3579221503,ES +3579221504,3579223071,GB +3579223072,3579223087,BE +3579223088,3579223103,FR +3579223104,3579226143,GB +3579226144,3579226151,DE +3579226152,3579226327,GB +3579226328,3579226351,FR +3579226352,3579228191,GB 3579228192,3579228207,IT 3579228208,3579228223,ES 3579228224,3579228671,GB 3579228672,3579228799,US -3579228800,3579241151,GB +3579228800,3579231815,GB +3579231816,3579231831,FR +3579231832,3579231919,GB +3579231920,3579231927,US +3579231928,3579237183,GB +3579237184,3579237215,DE +3579237216,3579237247,NL +3579237248,3579240143,GB +3579240144,3579240159,CH +3579240160,3579241151,GB 3579241152,3579241215,IE -3579241216,3579248639,GB +3579241216,3579244375,GB +3579244376,3579244383,CH +3579244384,3579247199,GB +3579247200,3579247207,DE +3579247208,3579247223,GB +3579247224,3579247231,FR +3579247232,3579247271,GB +3579247272,3579247279,US +3579247280,3579247311,GB +3579247312,3579247319,FR +3579247320,3579247391,GB +3579247392,3579247399,US +3579247400,3579247407,GB +3579247408,3579247415,US +3579247416,3579247535,GB +3579247536,3579247543,IE +3579247544,3579247567,GB +3579247568,3579247575,DE +3579247576,3579247583,GB +3579247584,3579247615,US +3579247616,3579248135,GB +3579248136,3579248143,DE +3579248144,3579248639,GB 3579248640,3579346943,RU 3579346944,3579361535,SE 3579361536,3579361791,NO @@ -113921,8 +120391,7 @@ 3580239872,3580241919,SE 3580241920,3580243967,CH 3580243968,3580248063,NL -3580248064,3580250111,ES -3580250112,3580252671,SE +3580248064,3580252671,SE 3580252672,3580254207,EE 3580254208,3580260351,DE 3580260352,3580265727,AT @@ -113975,6 +120444,7 @@ 3580633088,3580635135,RU 3580635136,3580637183,UA 3580637184,3580639231,KZ +3580639232,3580641279,PL 3580641280,3580643327,FR 3580643328,3580645375,UA 3580645376,3580647423,PL @@ -114068,7 +120538,10 @@ 3582066688,3582074879,UA 3582074880,3582078631,GB 3582078632,3582078639,DE -3582078640,3582083071,GB +3582078640,3582081023,GB +3582081024,3582081535,ES +3582081536,3582081791,DE +3582081792,3582083071,GB 3582083072,3582091263,BG 3582091264,3582099455,QA 3582099456,3582107647,GB @@ -114101,9 +120574,7 @@ 3582195136,3582195143,CH 3582195144,3582196183,DE 3582196184,3582196191,BE -3582196192,3582196991,DE -3582196992,3582197055,CN -3582197056,3582197127,DE +3582196192,3582197127,DE 3582197128,3582197135,BZ 3582197136,3582197759,DE 3582197760,3582205951,DK @@ -114321,9 +120792,7 @@ 3582571692,3582571707,IE 3582571708,3582571727,EU 3582571728,3582571839,IE -3582571840,3582571855,EU -3582571856,3582571871,IE -3582571872,3582571999,EU +3582571840,3582571999,EU 3582572000,3582572015,IE 3582572016,3582572023,EU 3582572024,3582572031,IE @@ -114376,8 +120845,8 @@ 3582574016,3582574431,EU 3582574432,3582574435,CH 3582574436,3582574455,EU -3582574456,3582574479,CH -3582574480,3582574591,EU +3582574456,3582574463,CH +3582574464,3582574591,EU 3582574592,3582582783,DE 3582582784,3582590975,RU 3582590976,3582599167,FR @@ -114414,6 +120883,7 @@ 3582746624,3582754815,RU 3582754816,3582763007,GR 3582763008,3582771199,FI +3582771200,3582779391,CY 3582779392,3582787583,RU 3582787584,3582795775,PT 3582795776,3582803967,ES @@ -114423,7 +120893,6 @@ 3582828544,3582836735,KZ 3582836736,3582853119,RU 3582853120,3582861311,SE -3582861312,3582869503,RU 3582869504,3582877695,NO 3582877696,3582885887,AT 3582885888,3582894079,TR @@ -114447,7 +120916,8 @@ 3583019376,3583023215,CH 3583023216,3583023223,DE 3583023224,3583025151,CH -3583025152,3583029119,IT +3583025152,3583028991,IT +3583028992,3583029119,DJ 3583029120,3583029247,US 3583029248,3583029375,SG 3583029376,3583029503,FR @@ -114572,6 +121042,7 @@ 3583524864,3583533055,EG 3583533056,3583541247,DE 3583541248,3583549439,RU +3583549440,3583557631,NL 3583557632,3583558399,IT 3583558400,3583558527,SM 3583558528,3583565823,IT @@ -114719,18 +121190,20 @@ 3583708608,3583708623,NA 3583708624,3583709447,UA 3583709448,3583709455,NA -3583709456,3583709479,UA +3583709456,3583709463,DE +3583709464,3583709479,UA 3583709480,3583709487,NA 3583709488,3583709503,UA -3583709504,3583709519,NA -3583709520,3583709527,UA +3583709504,3583709511,NA +3583709512,3583709527,UA 3583709528,3583709543,NA 3583709544,3583709551,UA 3583709552,3583709559,NA 3583709560,3583709575,UA 3583709576,3583709583,NA 3583709584,3583709599,UA -3583709600,3583709615,NA +3583709600,3583709607,NA +3583709608,3583709615,RU 3583709616,3583709623,UA 3583709624,3583709639,NA 3583709640,3583709663,UA @@ -114748,7 +121221,9 @@ 3583709832,3583709839,UA 3583709840,3583709863,NA 3583709864,3583709871,UA -3583709872,3583709911,NA +3583709872,3583709879,NA +3583709880,3583709887,UA +3583709888,3583709911,NA 3583709912,3583709927,UA 3583709928,3583709943,NA 3583709944,3583710055,UA @@ -114773,7 +121248,9 @@ 3583710496,3583710503,NA 3583710504,3583710519,UA 3583710520,3583710527,NA -3583710528,3583710631,UA +3583710528,3583710535,UA +3583710536,3583710543,RU +3583710544,3583710631,UA 3583710632,3583710639,NA 3583710640,3583710671,UA 3583710672,3583710679,NA @@ -114781,7 +121258,8 @@ 3583710704,3583710711,NA 3583710712,3583710823,UA 3583710824,3583710831,NA -3583710832,3583710855,UA +3583710832,3583710847,UA +3583710848,3583710855,RU 3583710856,3583710863,NA 3583710864,3583710871,UA 3583710872,3583710879,NA @@ -114799,9 +121277,7 @@ 3583711080,3583711095,NA 3583711096,3583711247,UA 3583711248,3583711255,NA -3583711256,3583711295,UA -3583711296,3583711303,NA -3583711304,3583711311,UA +3583711256,3583711311,UA 3583711312,3583711319,NA 3583711320,3583711327,UA 3583711328,3583711335,NA @@ -114847,7 +121323,8 @@ 3583712400,3583712407,NA 3583712408,3583712415,UA 3583712416,3583712423,NA -3583712424,3583712439,UA +3583712424,3583712431,SG +3583712432,3583712439,UA 3583712440,3583712479,NA 3583712480,3583712591,UA 3583712592,3583712599,NA @@ -114871,7 +121348,9 @@ 3583712912,3583712943,NA 3583712944,3583712959,UA 3583712960,3583712983,NA -3583712984,3583713031,UA +3583712984,3583713007,UA +3583713008,3583713015,RU +3583713016,3583713031,UA 3583713032,3583713039,NA 3583713040,3583713071,UA 3583713072,3583713079,NA @@ -114898,7 +121377,9 @@ 3583741696,3583741951,NL 3583741952,3583742719,EU 3583742720,3583743487,DE -3583743488,3583743975,EU +3583743488,3583743519,EU +3583743520,3583743551,FR +3583743552,3583743975,EU 3583743976,3583743983,GB 3583743984,3583744067,EU 3583744068,3583744071,GB @@ -114922,7 +121403,6 @@ 3583745808,3583745823,GB 3583745824,3583746047,EU 3583746048,3583754239,PL -3583754240,3583762431,IT 3583762432,3583770623,CZ 3583770624,3583772351,NL 3583772352,3583772367,IT @@ -114939,7 +121419,7 @@ 3583827968,3583836159,CH 3583836160,3583844351,DE 3583844352,3583852543,KZ -3583852544,3583853055,US +3583852544,3583853055,SG 3583853056,3583854591,FI 3583854592,3583854726,GB 3583854727,3583854735,FI @@ -115116,7 +121596,9 @@ 3584598016,3584606207,CZ 3584606208,3584614399,DE 3584614400,3584614463,GB -3584614464,3584614623,IE +3584614464,3584614527,IE +3584614528,3584614591,GB +3584614592,3584614623,IE 3584614624,3584614655,GB 3584614656,3584614751,IE 3584614752,3584614759,GB @@ -115183,14 +121665,12 @@ 3584983808,3584984063,UA 3584984064,3584984319,US 3584984320,3584984575,UA -3584984576,3584988311,US -3584988312,3584988319,UA -3584988320,3584988607,US +3584984576,3584988607,US 3584988608,3584988623,UA -3584988624,3584988639,US -3584988640,3584988671,UA -3584988672,3584990239,US -3584990240,3584990463,UA +3584988624,3584988655,US +3584988656,3584988671,UA +3584988672,3584990287,US +3584990288,3584990463,UA 3584990464,3584990495,NL 3584990496,3584990527,UA 3584990528,3584990591,US @@ -115240,7 +121720,9 @@ 3585302528,3585310719,ES 3585310720,3585318911,DE 3585318912,3585327103,DZ -3585327104,3585335295,NL +3585327104,3585331327,NL +3585331328,3585331343,FI +3585331344,3585335295,NL 3585335296,3585343487,UA 3585343488,3585351679,EE 3585351680,3585359871,CZ @@ -115355,7 +121837,6 @@ 3585716424,3585720319,AT 3585720320,3585728511,GB 3585728512,3585736703,SE -3585736704,3585744895,GB 3585744896,3585750335,FR 3585750336,3585750351,GB 3585750352,3585750367,DE @@ -115394,7 +121875,7 @@ 3585835984,3585836007,NL 3585836008,3585836023,IQ 3585836024,3585836287,NL -3585836288,3585836543,SA +3585836288,3585836543,CH 3585836544,3585837055,DZ 3585837056,3585837567,IQ 3585837568,3585838143,AO @@ -115535,7 +122016,9 @@ 3586272832,3586272895,NL 3586272896,3586272959,IT 3586272960,3586272991,ES -3586272992,3586273023,IT +3586272992,3586273007,IT +3586273008,3586273015,ES +3586273016,3586273023,IT 3586273024,3586276351,NL 3586276352,3586276863,ES 3586276864,3586277375,NL @@ -115561,10 +122044,13 @@ 3586476320,3586476351,HU 3586476352,3586476383,AT 3586476384,3586476407,UA -3586476408,3586478079,HU +3586476408,3586476431,HU +3586476432,3586476439,SK +3586476440,3586478079,HU 3586478080,3586478591,SK 3586478592,3586478847,TR -3586478848,3586490367,HU +3586478848,3586479103,SK +3586479104,3586490367,HU 3586490368,3586506751,LT 3586506752,3586523135,NL 3586523136,3586543519,DE @@ -115787,59 +122273,33 @@ 3587213912,3587213919,UY 3587213920,3587219455,AT 3587219456,3587227647,RU -3587227648,3587227655,NL -3587227656,3587227663,GB -3587227664,3587227679,NL -3587227680,3587227759,GB -3587227760,3587227791,NL -3587227792,3587227807,GB -3587227808,3587227823,NL -3587227824,3587227871,GB -3587227872,3587227887,NL -3587227888,3587227919,GB +3587227648,3587227903,NL +3587227904,3587227919,GB 3587227920,3587227935,NL 3587227936,3587227967,GB 3587227968,3587227983,DE -3587227984,3587228015,NL -3587228016,3587228047,GB +3587227984,3587227999,NL +3587228000,3587228047,GB 3587228048,3587228087,NL 3587228088,3587228127,GB -3587228128,3587228207,NL -3587228208,3587228223,GB -3587228224,3587228239,NL -3587228240,3587228255,GB -3587228256,3587228271,NL +3587228128,3587228271,NL 3587228272,3587228287,GB -3587228288,3587228319,NL -3587228320,3587228335,GB -3587228336,3587228367,NL -3587228368,3587228447,GB -3587228448,3587228463,NL -3587228464,3587228647,GB -3587228648,3587228663,NL -3587228664,3587228667,GB -3587228668,3587228927,NL +3587228288,3587228591,NL +3587228592,3587228607,GB +3587228608,3587228640,NL +3587228641,3587228647,GB +3587228648,3587228927,NL 3587228928,3587229455,GB -3587229456,3587229487,NL -3587229488,3587229535,GB -3587229536,3587229583,NL -3587229584,3587229647,GB -3587229648,3587229663,NL -3587229664,3587229791,GB -3587229792,3587229807,NL -3587229808,3587229855,GB -3587229856,3587229887,NL -3587229888,3587229951,GB -3587229952,3587229967,NL -3587229968,3587230047,GB -3587230048,3587230079,NL -3587230080,3587230095,GB -3587230096,3587230111,NL -3587230112,3587230239,GB -3587230240,3587230255,NL -3587230256,3587230319,GB -3587230320,3587230351,NL -3587230352,3587230447,GB +3587229456,3587229823,NL +3587229824,3587229839,GB +3587229840,3587229920,NL +3587229921,3587229935,GB +3587229936,3587230159,NL +3587230160,3587230207,GB +3587230208,3587230399,NL +3587230400,3587230415,GB +3587230416,3587230431,NL +3587230432,3587230447,GB 3587230448,3587230463,NL 3587230464,3587230511,GB 3587230512,3587230519,NL @@ -115851,9 +122311,7 @@ 3587230688,3587230719,NL 3587230720,3587230791,GB 3587230792,3587230799,NL -3587230800,3587230807,GB -3587230808,3587230815,NL -3587230816,3587230847,GB +3587230800,3587230847,GB 3587230848,3587230911,NL 3587230912,3587230943,GB 3587230944,3587230967,DE @@ -115861,37 +122319,13 @@ 3587230992,3587231295,GB 3587231296,3587231327,NL 3587231328,3587231359,GB -3587231360,3587232255,NL -3587232256,3587232415,GB -3587232416,3587232439,NL -3587232440,3587232511,GB -3587232512,3587232535,NL +3587231360,3587232535,NL 3587232536,3587232711,GB 3587232712,3587232719,NL 3587232720,3587232767,GB -3587232768,3587232831,NL -3587232832,3587232847,GB -3587232848,3587232863,NL -3587232864,3587232879,GB -3587232880,3587232895,NL -3587232896,3587232911,GB -3587232912,3587232927,NL -3587232928,3587232967,GB -3587232968,3587232975,NL -3587232976,3587232991,GB -3587232992,3587233007,NL -3587233008,3587233023,GB -3587233024,3587233095,NL -3587233096,3587233103,GB -3587233104,3587233119,NL -3587233120,3587233151,GB -3587233152,3587233175,NL -3587233176,3587233199,GB -3587233200,3587233215,NL -3587233216,3587233231,GB -3587233232,3587233295,NL -3587233296,3587233311,GB -3587233312,3587233343,NL +3587232768,3587233295,NL +3587233296,3587233319,GB +3587233320,3587233343,NL 3587233344,3587233359,GB 3587233360,3587233407,NL 3587233408,3587233471,GB @@ -115902,8 +122336,8 @@ 3587233600,3587233807,NL 3587233808,3587233823,GB 3587233824,3587233839,NL -3587233840,3587233903,GB -3587233904,3587233935,NL +3587233840,3587233919,GB +3587233920,3587233935,NL 3587233936,3587233951,GB 3587233952,3587233967,NL 3587233968,3587233983,GB @@ -115917,8 +122351,8 @@ 3587234144,3587234159,GB 3587234160,3587234207,NL 3587234208,3587234815,GB -3587234816,3587236471,NL -3587236472,3587236479,GB +3587234816,3587236447,NL +3587236448,3587236479,GB 3587236480,3587236511,NL 3587236512,3587236543,GB 3587236544,3587236607,NL @@ -115927,9 +122361,7 @@ 3587237376,3587237391,GB 3587237392,3587237399,NL 3587237400,3587237407,GB -3587237408,3587237439,NL -3587237440,3587237447,GB -3587237448,3587237471,NL +3587237408,3587237471,NL 3587237472,3587237487,GB 3587237488,3587237495,NL 3587237496,3587237519,GB @@ -115941,45 +122373,39 @@ 3587238536,3587238543,GB 3587238544,3587238607,NL 3587238608,3587238911,GB -3587238912,3587239295,NL -3587239296,3587239311,GB -3587239312,3587239767,NL +3587238912,3587239303,NL +3587239304,3587239311,GB +3587239312,3587239679,NL +3587239680,3587239687,GB +3587239688,3587239767,NL 3587239768,3587239775,GB 3587239776,3587239791,NL -3587239792,3587239935,GB -3587239936,3587240055,NL -3587240056,3587240063,GB -3587240064,3587240087,NL +3587239792,3587239807,GB +3587239808,3587239871,NL +3587239872,3587239935,GB +3587239936,3587240087,NL 3587240088,3587240095,GB 3587240096,3587240103,NL 3587240104,3587240107,GB -3587240108,3587240111,NL -3587240112,3587240119,GB -3587240120,3587240431,NL -3587240432,3587240439,GB -3587240440,3587240511,NL +3587240108,3587240511,NL 3587240512,3587240575,GB 3587240576,3587240615,NL 3587240616,3587240623,GB -3587240624,3587241015,NL -3587241016,3587241031,GB -3587241032,3587241087,NL -3587241088,3587241095,GB -3587241096,3587241119,NL -3587241120,3587241151,GB -3587241152,3587241215,NL -3587241216,3587241279,GB -3587241280,3587241295,NL -3587241296,3587241311,GB -3587241312,3587241343,NL -3587241344,3587241663,GB -3587241664,3587242063,NL +3587240624,3587241143,NL +3587241144,3587241151,GB +3587241152,3587241223,NL +3587241224,3587241247,GB +3587241248,3587241263,NL +3587241264,3587241271,GB +3587241272,3587241343,NL +3587241344,3587241471,GB +3587241472,3587242063,NL 3587242064,3587242071,BE -3587242072,3587243647,NL -3587243648,3587243775,GB -3587243776,3587243935,NL +3587242072,3587243935,NL 3587243936,3587243943,GB -3587243944,3587244031,NL +3587243944,3587243967,NL +3587243968,3587243975,GB +3587243976,3587244031,NL 3587244032,3587260415,IT 3587260416,3587284991,DE 3587284992,3587285135,A2 @@ -116109,7 +122535,10 @@ 3587375104,3587383295,TR 3587383296,3587391487,CZ 3587391488,3587407871,KZ -3587407872,3587424255,NL +3587407872,3587408127,NL +3587408128,3587415039,BE +3587415040,3587422719,NL +3587422720,3587424255,BE 3587424256,3587440639,DE 3587440640,3587457023,SE 3587457024,3587459263,GB @@ -116130,10 +122559,8 @@ 3587637248,3587653631,FR 3587653632,3587670015,SK 3587670016,3587672623,IT -3587672624,3587672635,GB -3587672636,3587673475,IT -3587673476,3587673487,GB -3587673488,3587673763,IT +3587672624,3587672627,GB +3587672628,3587673763,IT 3587673764,3587673767,GB 3587673768,3587673787,IT 3587673788,3587673792,GB @@ -116151,7 +122578,9 @@ 3587687592,3587687599,NL 3587687600,3587699255,IT 3587699256,3587699263,A2 -3587699264,3587702783,IT +3587699264,3587701295,IT +3587701296,3587701303,FR +3587701304,3587702783,IT 3587702784,3587710975,DE 3587710976,3587719167,CZ 3587719168,3587735551,PL @@ -116197,7 +122626,7 @@ 3588104192,3588112383,BG 3588112384,3588128767,FR 3588128768,3588145151,HU -3588145152,3588161535,PL +3588145152,3588153343,PL 3588161536,3588227071,FR 3588227072,3588292607,BE 3588292608,3588308991,AT @@ -116277,65 +122706,34 @@ 3588997120,3589013503,AT 3589013504,3589021695,ES 3589021696,3589029887,SA -3589029888,3589029919,NL -3589029920,3589029951,ES -3589029952,3589030015,NL -3589030016,3589030143,ES -3589030144,3589030183,NL +3589029888,3589030183,NL 3589030184,3589030191,FR -3589030192,3589030207,NL -3589030208,3589030239,ES -3589030240,3589030271,NL -3589030272,3589030303,ES +3589030192,3589030303,NL 3589030304,3589030335,FR -3589030336,3589030399,NL -3589030400,3589030655,ES -3589030656,3589031423,NL -3589031424,3589031935,ES -3589031936,3589034015,NL -3589034016,3589034111,ES -3589034112,3589034143,NL +3589030336,3589034143,NL 3589034144,3589034159,ES 3589034160,3589034167,GB -3589034168,3589034175,NL -3589034176,3589034183,ES +3589034168,3589034183,NL 3589034184,3589034191,GB 3589034192,3589034207,DE -3589034208,3589034223,ES -3589034224,3589034271,NL -3589034272,3589034303,ES +3589034208,3589034303,NL 3589034304,3589034335,DE 3589034336,3589034367,FR 3589034368,3589034431,NL 3589034432,3589034495,FR -3589034496,3589037311,ES -3589037312,3589037343,NL +3589034496,3589037055,ES +3589037056,3589037343,NL 3589037344,3589037375,SE -3589037376,3589037391,NL -3589037392,3589037407,ES -3589037408,3589037423,NL +3589037376,3589037423,NL 3589037424,3589037439,ES -3589037440,3589037487,NL -3589037488,3589037495,ES +3589037440,3589037495,NL 3589037496,3589037503,CH -3589037504,3589037535,NL -3589037536,3589037551,ES +3589037504,3589037551,NL 3589037552,3589037559,BE 3589037560,3589037567,NL 3589037568,3589038079,ES -3589038080,3589040127,NL -3589040128,3589042207,ES -3589042208,3589042367,NL -3589042368,3589043231,ES -3589043232,3589043327,NL -3589043328,3589043455,ES -3589043456,3589043711,NL -3589043712,3589044223,ES -3589044224,3589044319,NL -3589044320,3589046271,ES -3589046272,3589066751,NO -3589066752,3589067007,A1 -3589067008,3589079039,NO +3589038080,3589046271,NL +3589046272,3589079039,NO 3589079040,3589092223,DK 3589092224,3589092227,SE 3589092228,3589128191,DK @@ -116474,27 +122872,140 @@ 3589685248,3589718015,GB 3589718016,3589719343,BE 3589719344,3589719347,GB -3589719348,3589720603,BE +3589719348,3589719847,BE +3589719848,3589719855,AT +3589719856,3589719967,BE +3589719968,3589719971,GB +3589719972,3589719972,BE +3589719973,3589719975,GB +3589719976,3589720071,BE +3589720072,3589720079,FR +3589720080,3589720135,BE +3589720136,3589720143,GB +3589720144,3589720263,BE +3589720264,3589720271,GB +3589720272,3589720279,FR +3589720280,3589720303,BE +3589720304,3589720311,FR +3589720312,3589720575,BE +3589720576,3589720583,ES +3589720584,3589720599,FR +3589720600,3589720603,BE 3589720604,3589720607,GB -3589720608,3589721143,BE -3589721144,3589721151,GB -3589721152,3589722015,BE -3589722016,3589722016,GB -3589722017,3589722059,BE +3589720608,3589720623,BE +3589720624,3589720631,GB +3589720632,3589720639,FR +3589720640,3589720655,BE +3589720656,3589720663,ES +3589720664,3589720687,BE +3589720688,3589720695,GB +3589720696,3589720703,FR +3589720704,3589720727,BE +3589720728,3589720735,GB +3589720736,3589720743,BE +3589720744,3589720767,GB +3589720768,3589720783,BE +3589720784,3589720799,GB +3589720800,3589720815,BE +3589720816,3589720823,GB +3589720824,3589720839,BE +3589720840,3589720847,GB +3589720848,3589720879,BE +3589720880,3589720887,FR +3589720888,3589720895,BE +3589720896,3589720903,IT +3589720904,3589720935,BE +3589720936,3589720943,GB +3589720944,3589720951,FR +3589720952,3589720983,GB +3589720984,3589720995,BE +3589720996,3589721003,GB +3589721004,3589721015,BE +3589721016,3589721031,GB +3589721032,3589721039,BE +3589721040,3589721047,GB +3589721048,3589721055,ES +3589721056,3589721063,BE +3589721064,3589721071,GB +3589721072,3589721147,BE +3589721148,3589721151,GB +3589721152,3589721159,BE +3589721160,3589721167,GB +3589721168,3589721203,BE +3589721204,3589721211,FR +3589721212,3589721231,BE +3589721232,3589721247,GB +3589721248,3589721255,FR +3589721256,3589721279,GB +3589721280,3589721295,BE +3589721296,3589721311,FR +3589721312,3589721399,BE +3589721400,3589721407,GB +3589721408,3589721855,BE +3589721856,3589721863,GB +3589721864,3589721867,BE +3589721868,3589721883,GB +3589721884,3589721903,BE +3589721904,3589721911,GB +3589721912,3589721951,BE +3589721952,3589721967,GB +3589721968,3589722023,BE +3589722024,3589722031,FR +3589722032,3589722059,BE 3589722060,3589722063,GB -3589722064,3589722295,BE +3589722064,3589722111,BE +3589722112,3589722143,GB +3589722144,3589722159,BE +3589722160,3589722167,FR +3589722168,3589722175,BE +3589722176,3589722191,GB +3589722192,3589722199,BE +3589722200,3589722207,FR +3589722208,3589722263,BE +3589722264,3589722271,GB +3589722272,3589722279,BE +3589722280,3589722287,FR +3589722288,3589722295,BE 3589722296,3589722299,GB -3589722300,3589722387,BE -3589722388,3589722391,GB +3589722300,3589722303,BE +3589722304,3589722311,FR +3589722312,3589722319,GB +3589722320,3589722335,FR +3589722336,3589722343,BE +3589722344,3589722359,FR +3589722360,3589722383,BE +3589722384,3589722391,FR 3589722392,3589722395,BE 3589722396,3589722399,GB -3589722400,3589722859,BE +3589722400,3589722439,BE +3589722440,3589722447,GB +3589722448,3589722451,BE +3589722452,3589722455,FR +3589722456,3589722459,BE +3589722460,3589722463,FR +3589722464,3589722623,BE +3589722624,3589722631,GB +3589722632,3589722639,BE +3589722640,3589722647,GB +3589722648,3589722703,BE +3589722704,3589722711,FR +3589722712,3589722791,BE +3589722792,3589722799,GB +3589722800,3589722835,BE +3589722836,3589722843,GB +3589722844,3589722859,BE 3589722860,3589722863,GB -3589722864,3589722979,BE -3589722980,3589722983,GB -3589722984,3589722995,BE +3589722864,3589722943,BE +3589722944,3589722951,GB +3589722952,3589722959,FR +3589722960,3589722979,BE +3589722980,3589722991,GB +3589722992,3589722995,BE 3589722996,3589722999,GB -3589723000,3589723983,BE +3589723000,3589723023,BE +3589723024,3589723031,FR +3589723032,3589723975,BE +3589723976,3589723983,FR 3589723984,3589723987,GB 3589723988,3589724031,BE 3589724032,3589724039,ES @@ -116510,11 +123021,23 @@ 3589724612,3589724615,GB 3589724616,3589724619,BE 3589724620,3589724631,GB -3589724632,3589724775,BE +3589724632,3589724671,BE +3589724672,3589724679,FR +3589724680,3589724695,GB +3589724696,3589724703,BE +3589724704,3589724719,FR +3589724720,3589724727,GB +3589724728,3589724775,BE 3589724776,3589724779,GB -3589724780,3589724875,BE +3589724780,3589724791,BE +3589724792,3589724807,GB +3589724808,3589724875,BE 3589724876,3589724879,GB -3589724880,3589725271,BE +3589724880,3589725039,BE +3589725040,3589725055,GB +3589725056,3589725247,BE +3589725248,3589725263,GB +3589725264,3589725271,BE 3589725272,3589725275,GB 3589725276,3589725383,BE 3589725384,3589725391,GB @@ -116522,49 +123045,66 @@ 3589725444,3589725447,GB 3589725448,3589725451,BE 3589725452,3589725463,GB -3589725464,3589725575,BE +3589725464,3589725495,BE +3589725496,3589725503,PT +3589725504,3589725575,BE 3589725576,3589725583,GB 3589725584,3589725619,BE 3589725620,3589725623,GB -3589725624,3589725779,BE +3589725624,3589725663,BE +3589725664,3589725695,GB +3589725696,3589725779,BE 3589725780,3589725783,GB 3589725784,3589725791,BE 3589725792,3589725799,GB -3589725800,3589725831,BE +3589725800,3589725807,IE +3589725808,3589725831,BE 3589725832,3589725835,GB 3589725836,3589725839,BE 3589725840,3589725843,GB -3589725844,3589725887,BE +3589725844,3589725847,BE +3589725848,3589725855,IE +3589725856,3589725887,BE 3589725888,3589725895,GB 3589725896,3589725907,BE -3589725908,3589725911,GB -3589725912,3589725919,BE -3589725920,3589725927,GB +3589725908,3589725927,GB 3589725928,3589725935,BE 3589725936,3589725943,GB -3589725944,3589725967,BE +3589725944,3589725959,BE +3589725960,3589725967,LU 3589725968,3589725983,GB -3589725984,3589726027,BE +3589725984,3589725999,BE +3589726000,3589726007,FR +3589726008,3589726027,BE 3589726028,3589726031,GB -3589726032,3589726079,BE -3589726080,3589726083,GB -3589726084,3589726091,BE +3589726032,3589726047,BE +3589726048,3589726055,FR +3589726056,3589726091,BE 3589726092,3589726095,GB -3589726096,3589726191,BE +3589726096,3589726111,BE +3589726112,3589726119,GB +3589726120,3589726191,BE 3589726192,3589726195,GB -3589726196,3589726395,BE +3589726196,3589726367,BE +3589726368,3589726375,FR +3589726376,3589726395,BE 3589726396,3589726403,GB -3589726404,3589726415,BE -3589726416,3589726423,GB -3589726424,3589726435,BE +3589726404,3589726419,BE +3589726420,3589726423,GB +3589726424,3589726431,FR +3589726432,3589726435,BE 3589726436,3589726439,GB -3589726440,3589726483,BE -3589726484,3589726487,GB -3589726488,3589726547,BE +3589726440,3589726455,BE +3589726456,3589726463,FR +3589726464,3589726471,BE +3589726472,3589726479,GB +3589726480,3589726483,BE +3589726484,3589726495,GB +3589726496,3589726535,BE +3589726536,3589726543,GB +3589726544,3589726547,BE 3589726548,3589726551,GB -3589726552,3589726627,BE -3589726628,3589726631,GB -3589726632,3589726655,BE +3589726552,3589726655,BE 3589726656,3589726663,GB 3589726664,3589726695,BE 3589726696,3589726699,GB @@ -116594,8 +123134,8 @@ 3589729328,3589729331,GB 3589729332,3589729339,BE 3589729340,3589729343,GB -3589729344,3589729551,BE -3589729552,3589729559,GB +3589729344,3589729555,BE +3589729556,3589729559,GB 3589729560,3589729571,BE 3589729572,3589729579,GB 3589729580,3589729607,BE @@ -116608,7 +123148,9 @@ 3589729656,3589729659,GB 3589729660,3589729671,BE 3589729672,3589729675,GB -3589729676,3589729779,BE +3589729676,3589729743,BE +3589729744,3589729751,GB +3589729752,3589729779,BE 3589729780,3589729783,GB 3589729784,3589729815,BE 3589729816,3589729819,GB @@ -116624,7 +123166,11 @@ 3589730056,3589730063,GB 3589730064,3589730071,BE 3589730072,3589730075,GB -3589730076,3589730139,BE +3589730076,3589730087,BE +3589730088,3589730095,GB +3589730096,3589730111,BE +3589730112,3589730119,GB +3589730120,3589730139,BE 3589730140,3589730143,GB 3589730144,3589730271,BE 3589730272,3589730279,GB @@ -116654,15 +123200,9 @@ 3589732292,3589732295,GB 3589732296,3589732303,BE 3589732304,3589732311,GB -3589732312,3589733123,BE -3589733124,3589733139,GB -3589733140,3589733375,BE +3589732312,3589733375,BE 3589733376,3589733439,LU -3589733440,3589733639,BE -3589733640,3589733643,GB -3589733644,3589733651,BE -3589733652,3589733667,GB -3589733668,3589734399,BE +3589733440,3589734399,BE 3589734400,3589742591,EG 3589742592,3589742687,NL 3589742688,3589742695,AT @@ -116769,11 +123309,11 @@ 3590156432,3590156447,SI 3590156448,3590156679,HR 3590156680,3590156687,SI -3590156688,3590156959,HR +3590156688,3590156719,HR +3590156720,3590156727,SI +3590156728,3590156959,HR 3590156960,3590156991,AE -3590156992,3590157023,SI -3590157024,3590157031,HR -3590157032,3590157039,SI +3590156992,3590157039,SI 3590157040,3590157311,HR 3590157312,3590157343,RS 3590157344,3590157351,SI @@ -116817,11 +123357,10 @@ 3590225920,3590234111,TR 3590234112,3590242303,GB 3590242304,3590244351,US -3590244352,3590244607,FR -3590244608,3590244863,DE +3590244352,3590244863,DE 3590244864,3590245119,GR -3590245120,3590245151,BE -3590245152,3590245279,FR +3590245120,3590245167,BE +3590245168,3590245279,FR 3590245280,3590245311,BE 3590245312,3590245439,US 3590245440,3590245503,FR @@ -116849,7 +123388,7 @@ 3590247552,3590247679,IT 3590247680,3590247687,GB 3590247688,3590247711,FR -3590247712,3590247743,HK +3590247712,3590247743,A1 3590247744,3590248447,FR 3590248448,3590248959,US 3590248960,3590249471,ES @@ -116881,18 +123420,19 @@ 3590254144,3590254271,US 3590254272,3590254303,FR 3590254304,3590254335,US -3590254336,3590254591,FR -3590254592,3590254879,ES +3590254336,3590254463,FR +3590254464,3590254879,ES 3590254880,3590254911,FR 3590254912,3590254975,ES -3590254976,3590255047,FR +3590254976,3590255039,DE +3590255040,3590255047,FR 3590255048,3590255071,ES 3590255072,3590255103,FR 3590255104,3590255359,ES 3590255360,3590255615,FR 3590255616,3590255871,BE -3590255872,3590255999,US -3590256000,3590256383,FR +3590255872,3590256127,US +3590256128,3590256383,ES 3590256384,3590256639,IT 3590256640,3590256703,US 3590256704,3590256719,BE @@ -117174,7 +123714,9 @@ 3590323904,3590323911,CD 3590323912,3590324031,A2 3590324032,3590324095,NG -3590324096,3590324223,A2 +3590324096,3590324103,A2 +3590324104,3590324111,ZW +3590324112,3590324223,A2 3590324224,3623890943,US 3623890944,3623891199,ZA 3623891200,3623891455,US @@ -117204,7 +123746,9 @@ 3624288256,3624290303,IN 3624290304,3624292351,CA 3624292352,3624292607,GP -3624292608,3624295935,CA +3624292608,3624294143,CA +3624294144,3624294399,US +3624294400,3624295935,CA 3624295936,3624296191,LY 3624296192,3624297471,CA 3624297472,3624298495,CO @@ -117288,7 +123832,8 @@ 3624381472,3624381487,AU 3624381488,3624381535,US 3624381536,3624381551,CA -3624381552,3624381567,US +3624381552,3624381559,US +3624381560,3624381567,CA 3624381568,3624381583,MY 3624381584,3624381631,US 3624381632,3624381647,GB @@ -117353,8 +123898,8 @@ 3624547584,3624547591,PN 3624547592,3624547647,A2 3624547648,3624547711,US -3624547712,3624547751,A2 -3624547752,3624547839,US +3624547712,3624547743,A2 +3624547744,3624547839,US 3624547840,3624547847,A2 3624547848,3624547863,US 3624547864,3624547871,A2 @@ -117433,7 +123978,9 @@ 3624896288,3624896319,GB 3624896320,3624896415,US 3624896416,3624896447,AT -3624896448,3624898815,US +3624896448,3624897791,US +3624897792,3624898047,IN +3624898048,3624898815,US 3624898816,3624899071,AU 3624899072,3624899327,CA 3624899328,3624899583,US @@ -117539,14 +124086,18 @@ 3624911200,3624911231,BE 3624911232,3624911359,US 3624911360,3624911615,CA -3624911616,3624911999,US +3624911616,3624911647,US +3624911648,3624911651,RU +3624911652,3624911999,US 3624912000,3624912127,CA 3624912128,3624913023,US 3624913024,3624913087,IN 3624913088,3624913311,US 3624913312,3624913343,CA 3624913344,3624913375,PK -3624913376,3624914047,US +3624913376,3624913535,US +3624913536,3624913663,IN +3624913664,3624914047,US 3624914048,3624914079,IN 3624914080,3624914143,US 3624914144,3624914175,GB @@ -117690,7 +124241,15 @@ 3625288888,3625288895,US 3625288896,3625289959,CA 3625289960,3625289967,US -3625289968,3625295871,CA +3625289968,3625292575,CA +3625292576,3625292583,US +3625292584,3625292671,CA +3625292672,3625292679,US +3625292680,3625292927,CA +3625292928,3625292935,US +3625292936,3625293607,CA +3625293608,3625293615,US +3625293616,3625295871,CA 3625295872,3625320447,US 3625320448,3625320467,CA 3625320468,3625321727,US @@ -117836,7 +124395,9 @@ 3626228464,3626228479,AE 3626228480,3626270719,US 3626270720,3626287103,CA -3626287104,3626381317,US +3626287104,3626332159,US +3626332160,3626336255,CA +3626336256,3626381317,US 3626381318,3626381321,AU 3626381322,3626381325,CR 3626381326,3626381329,IT @@ -118040,9 +124601,7 @@ 3626382535,3626382538,BR 3626382539,3626382546,US 3626382547,3626382550,IN -3626382551,3626382558,US -3626382559,3626382562,CA -3626382563,3626382570,US +3626382551,3626382570,US 3626382571,3626382574,IT 3626382575,3626382578,US 3626382579,3626382582,IN @@ -118244,9 +124803,7 @@ 3626385394,3626508287,US 3626508288,3626512383,CA 3626512384,3626524671,US -3626524672,3626524823,CA -3626524824,3626524831,CR -3626524832,3626524927,CA +3626524672,3626524927,CA 3626524928,3626524943,US 3626524944,3626524967,CA 3626524968,3626524975,US @@ -118271,9 +124828,9 @@ 3626526624,3626526631,US 3626526632,3626526711,CA 3626526712,3626526719,US -3626526720,3626526975,CA -3626526976,3626527031,US -3626527032,3626527135,CA +3626526720,3626526983,CA +3626526984,3626527023,US +3626527024,3626527135,CA 3626527136,3626527159,US 3626527160,3626527167,CA 3626527168,3626527183,US @@ -118283,7 +124840,9 @@ 3626527472,3626527479,US 3626527480,3626527487,CA 3626527488,3626527495,US -3626527496,3626527903,CA +3626527496,3626527551,CA +3626527552,3626527559,US +3626527560,3626527903,CA 3626527904,3626527911,US 3626527912,3626528239,CA 3626528240,3626528247,US @@ -118305,18 +124864,20 @@ 3626529920,3626529927,US 3626529928,3626530039,CA 3626530040,3626530055,US -3626530056,3626530207,CA +3626530056,3626530151,CA +3626530152,3626530159,US +3626530160,3626530207,CA 3626530208,3626530215,US 3626530216,3626530287,CA 3626530288,3626530295,US 3626530296,3626530327,CA 3626530328,3626530343,US 3626530344,3626530391,CA -3626530392,3626530415,US -3626530416,3626530439,CA +3626530392,3626530407,US +3626530408,3626530439,CA 3626530440,3626530455,US -3626530456,3626530463,CA -3626530464,3626530479,US +3626530456,3626530471,CA +3626530472,3626530479,US 3626530480,3626530487,CA 3626530488,3626530495,US 3626530496,3626530543,CA @@ -118329,7 +124890,9 @@ 3626530944,3626530951,US 3626530952,3626530991,CA 3626530992,3626531007,US -3626531008,3626531031,CA +3626531008,3626531015,CA +3626531016,3626531023,US +3626531024,3626531031,CA 3626531032,3626531047,US 3626531048,3626531143,CA 3626531144,3626531151,US @@ -118465,8 +125028,7 @@ 3627746400,3627746431,CA 3627746432,3627746559,US 3627746560,3627746815,IN -3627746816,3627747071,NZ -3627747072,3627753471,US +3627746816,3627753471,US 3627753472,3627753727,AR 3627753728,3627761663,US 3627778048,3627802623,US @@ -118576,8 +125138,16 @@ 3628332384,3628332391,IN 3628332392,3628332423,US 3628332424,3628332431,IN -3628332432,3628598271,US -3628598272,3628599295,CO +3628332432,3628598335,US +3628598336,3628598367,CO +3628598368,3628598399,US +3628598400,3628598431,CO +3628598432,3628598463,US +3628598464,3628598495,CO +3628598496,3628598527,US +3628598528,3628598687,CO +3628598688,3628598719,US +3628598720,3628599295,CO 3628599296,3628603391,CL 3628603392,3628604415,US 3628604416,3628605439,GT @@ -118612,7 +125182,9 @@ 3628631808,3628633855,US 3628633856,3628634111,CO 3628634112,3628636159,AR -3628636160,3628636671,VE +3628636160,3628636399,VE +3628636400,3628636415,US +3628636416,3628636671,VE 3628636672,3628636927,US 3628636928,3628637183,VE 3628637184,3628637695,CO @@ -118623,16 +125195,20 @@ 3628647424,3628647679,SZ 3628647680,3628648959,CO 3628648960,3628649215,US -3628649216,3628649727,CO -3628649728,3628654079,US +3628649216,3628649471,CO +3628649472,3628654079,US 3628654080,3628654591,PA 3628654592,3628654847,US 3628654848,3628655103,CR 3628655104,3628655359,VE -3628655360,3628656383,CO +3628655360,3628655743,CO +3628655744,3628655871,US +3628655872,3628656383,CO 3628656384,3628656639,EC -3628656640,3628657407,CO -3628657408,3628657663,US +3628656640,3628657087,CO +3628657088,3628657151,US +3628657152,3628657183,CO +3628657184,3628657663,US 3628657664,3628657919,VE 3628657920,3628658175,BS 3628658176,3628679167,US @@ -118819,7 +125395,9 @@ 3630054240,3630054335,US 3630054336,3630054495,CA 3630054496,3630054511,US -3630054512,3630055519,CA +3630054512,3630054911,CA +3630054912,3630054919,US +3630054920,3630055519,CA 3630055520,3630055551,US 3630055552,3630055679,CA 3630055680,3630055935,US @@ -119208,11 +125786,11 @@ 3631015140,3631015147,CN 3631015148,3631015253,US 3631015254,3631015269,CA -3631015270,3631016243,US +3631015270,3631015295,US +3631015296,3631015311,CN +3631015312,3631016243,US 3631016244,3631016257,BD -3631016258,3631016451,US -3631016452,3631016483,CN -3631016484,3631016555,US +3631016258,3631016555,US 3631016556,3631016571,CN 3631016572,3631016581,US 3631016582,3631016613,BD @@ -119264,7 +125842,9 @@ 3631333376,3631333679,CA 3631333680,3631333695,US 3631333696,3631341567,CA -3631341568,3631480831,US +3631341568,3631435007,US +3631435008,3631435263,GB +3631435264,3631480831,US 3631480832,3631484927,CA 3631484928,3631644671,US 3631644672,3631652863,CA @@ -119347,7 +125927,9 @@ 3632381952,3632390143,CA 3632390144,3632414719,US 3632414720,3632422911,CA -3632422912,3632480319,US +3632422912,3632451583,US +3632451584,3632455679,CA +3632455680,3632480319,US 3632480320,3632480335,GB 3632480336,3632480367,US 3632480368,3632480383,GB @@ -119694,7 +126276,9 @@ 3632493072,3632493079,ID 3632493080,3632493087,US 3632493088,3632493119,IO -3632493120,3632493191,US +3632493120,3632493151,US +3632493152,3632493159,GR +3632493160,3632493191,US 3632493192,3632493199,CA 3632493200,3632493207,GB 3632493208,3632493215,US @@ -119807,83 +126391,104 @@ 3632988160,3633029119,US 3633029120,3633029631,PY 3633029632,3633030143,NI -3633030144,3633030215,A2 +3633030144,3633030159,US +3633030160,3633030167,A2 +3633030168,3633030175,US +3633030176,3633030183,A2 +3633030184,3633030191,US +3633030192,3633030199,A2 +3633030200,3633030207,US +3633030208,3633030215,A2 3633030216,3633030223,NG -3633030224,3633030655,A2 +3633030224,3633030231,A2 +3633030232,3633030287,US +3633030288,3633030303,A2 +3633030304,3633030335,US +3633030336,3633030359,A2 +3633030360,3633030655,US 3633030656,3633030911,SN -3633030912,3633031487,A2 -3633031488,3633031495,US -3633031496,3633031519,A2 -3633031520,3633031535,US -3633031536,3633031647,A2 +3633030912,3633031167,A2 +3633031168,3633031615,US +3633031616,3633031623,A2 +3633031624,3633031647,US 3633031648,3633031655,NG -3633031656,3633031687,A2 +3633031656,3633031687,US 3633031688,3633031695,NG -3633031696,3633031703,A2 -3633031704,3633031711,US -3633031712,3633031719,A2 -3633031720,3633031743,US -3633031744,3633031871,A2 -3633031872,3633031887,US -3633031888,3633031959,A2 +3633031696,3633031959,US 3633031960,3633031967,NL -3633031968,3633031983,A2 +3633031968,3633031983,US 3633031984,3633031991,PG -3633031992,3633032087,A2 +3633031992,3633032087,US 3633032088,3633032095,NL -3633032096,3633032167,A2 +3633032096,3633032167,US 3633032168,3633032175,NL -3633032176,3633032223,A2 +3633032176,3633032223,US 3633032224,3633032231,NG -3633032232,3633032279,A2 +3633032232,3633032239,US +3633032240,3633032247,A2 +3633032248,3633032271,US +3633032272,3633032279,A2 3633032280,3633032287,US 3633032288,3633032319,GQ -3633032320,3633032415,A2 +3633032320,3633032415,US 3633032416,3633032431,NG -3633032432,3633032543,A2 +3633032432,3633032439,US +3633032440,3633032447,A2 +3633032448,3633032543,US 3633032544,3633032567,NG -3633032568,3633033287,A2 +3633032568,3633032591,US +3633032592,3633032607,A2 +3633032608,3633032959,US +3633032960,3633033215,A2 +3633033216,3633033247,US +3633033248,3633033255,A2 +3633033256,3633033263,US +3633033264,3633033271,A2 +3633033272,3633033287,US 3633033288,3633033295,GH 3633033296,3633033303,NG 3633033304,3633033319,A2 3633033320,3633033327,NG -3633033328,3633033359,A2 +3633033328,3633033335,US +3633033336,3633033359,A2 3633033360,3633033367,NG 3633033368,3633033375,A2 3633033376,3633033391,NG -3633033392,3633033727,A2 -3633033728,3633033919,US -3633033920,3633034111,A2 -3633034112,3633034239,US -3633034240,3633034519,A2 +3633033392,3633033399,A2 +3633033400,3633034047,US +3633034048,3633034079,A2 +3633034080,3633034519,US 3633034520,3633034527,NG -3633034528,3633034559,A2 +3633034528,3633034559,US 3633034560,3633034751,NG -3633034752,3633035015,A2 +3633034752,3633035015,US 3633035016,3633035023,NL -3633035024,3633035071,A2 +3633035024,3633035071,US 3633035072,3633035087,NR 3633035088,3633035095,CN -3633035096,3633035135,A2 +3633035096,3633035135,US 3633035136,3633035151,NR -3633035152,3633035199,A2 +3633035152,3633035199,US 3633035200,3633035207,ID -3633035208,3633035311,A2 +3633035208,3633035287,US +3633035288,3633035295,A2 +3633035296,3633035311,US 3633035312,3633035319,ID -3633035320,3633035439,A2 +3633035320,3633035439,US 3633035440,3633035447,ID -3633035448,3633035455,A2 +3633035448,3633035455,US 3633035456,3633035487,ID -3633035488,3633035527,A2 +3633035488,3633035527,US 3633035528,3633035535,NG -3633035536,3633036031,A2 +3633035536,3633035551,US +3633035552,3633035599,A2 +3633035600,3633036031,US 3633036032,3633036287,HT -3633036288,3633036479,A2 +3633036288,3633036479,US 3633036480,3633036511,TT -3633036512,3633036591,A2 -3633036592,3633036599,US -3633036600,3633037055,A2 -3633037056,3633039503,US +3633036512,3633036607,US +3633036608,3633036671,A2 +3633036672,3633039503,US 3633039504,3633039519,CA 3633039520,3633166079,US 3633166080,3633166207,MX @@ -120131,9 +126736,11 @@ 3636068352,3636150495,US 3636150496,3636150527,CA 3636150528,3636150783,US -3636150784,3636150831,CA -3636150832,3636150911,US -3636150912,3636151007,CA +3636150784,3636150839,CA +3636150840,3636150911,US +3636150912,3636150935,CA +3636150936,3636150943,US +3636150944,3636151007,CA 3636151008,3636151023,BS 3636151024,3636151031,CA 3636151032,3636151039,US @@ -120143,31 +126750,30 @@ 3636151456,3636151479,US 3636151480,3636151488,CA 3636151489,3636151535,US -3636151536,3636151583,CA -3636151584,3636151591,US -3636151592,3636151759,CA +3636151536,3636151759,CA 3636151760,3636151775,US 3636151776,3636152575,CA 3636152576,3636152591,US 3636152592,3636152639,CA -3636152640,3636152703,US +3636152640,3636152647,US +3636152648,3636152655,CA +3636152656,3636152703,US 3636152704,3636152767,CA -3636152768,3636152783,US +3636152768,3636152775,MX +3636152776,3636152783,US 3636152784,3636152791,CA -3636152792,3636152831,US -3636152832,3636152847,CA +3636152792,3636152799,US +3636152800,3636152847,CA 3636152848,3636152895,US 3636152896,3636153023,CA 3636153024,3636153055,KN -3636153056,3636153087,CA -3636153088,3636153215,US -3636153216,3636153343,CA +3636153056,3636153343,CA 3636153344,3636153375,US 3636153376,3636153823,CA 3636153824,3636153839,US 3636153840,3636153847,CA -3636153848,3636154111,US -3636154112,3636154895,CA +3636153848,3636153855,US +3636153856,3636154895,CA 3636154896,3636154911,US 3636154912,3636154975,CA 3636154976,3636154979,US @@ -120178,7 +126784,9 @@ 3636155072,3636155135,CA 3636155136,3636155407,US 3636155408,3636155423,CA -3636155424,3636155519,US +3636155424,3636155439,US +3636155440,3636155455,CA +3636155456,3636155519,US 3636155520,3636155775,CA 3636155776,3636155839,US 3636155840,3636156159,CA @@ -120197,15 +126805,13 @@ 3636158016,3636158031,US 3636158032,3636158063,CA 3636158064,3636158079,US -3636158080,3636158207,CA -3636158208,3636158223,US +3636158080,3636158215,CA +3636158216,3636158223,US 3636158224,3636158335,CA 3636158336,3636158351,US 3636158352,3636158367,CA 3636158368,3636158399,US -3636158400,3636158431,CA -3636158432,3636158463,US -3636158464,3636158719,CA +3636158400,3636158719,CA 3636158720,3636158751,US 3636158752,3636158815,CA 3636158816,3636158831,US @@ -120225,8 +126831,8 @@ 3636160896,3636160927,CA 3636160928,3636160975,US 3636160976,3636161279,CA -3636161280,3636161343,US -3636161344,3636161535,CA +3636161280,3636161327,US +3636161328,3636161535,CA 3636161536,3636161791,US 3636161792,3636161871,CA 3636161872,3636161885,US @@ -120249,7 +126855,11 @@ 3636166144,3636166655,CA 3636166656,3636206079,US 3636206080,3636206335,AU -3636206336,3636396031,US +3636206336,3636265535,US +3636265536,3636265599,DE +3636265600,3636266879,US +3636266880,3636266911,HK +3636266912,3636396031,US 3636396032,3636461567,CA 3636461568,3636577647,US 3636577648,3636577663,CA @@ -120415,7 +127025,7 @@ 3637071888,3637071903,AD 3637071904,3637071935,US 3637071936,3637071943,SK -3637071944,3637071951,US +3637071944,3637071951,IL 3637071952,3637071967,CY 3637071968,3637071999,US 3637072000,3637072015,CA @@ -120594,9 +127204,13 @@ 3638526912,3638526919,US 3638526920,3638528751,CA 3638528752,3638528759,US -3638528760,3638530239,CA +3638528760,3638530047,CA +3638530048,3638530063,US +3638530064,3638530239,CA 3638530240,3638530255,US -3638530256,3638534143,CA +3638530256,3638533695,CA +3638533696,3638533703,US +3638533704,3638534143,CA 3638534144,3638697983,US 3638697984,3638706175,CA 3638706176,3638707199,A2 @@ -120713,7 +127327,9 @@ 3639439632,3639439639,RO 3639439640,3639440767,US 3639440768,3639440895,IN -3639440896,3639558143,US +3639440896,3639476223,US +3639507456,3639507967,US +3639541760,3639558143,US 3639558144,3639566335,CA 3639566336,3639607295,US 3639607296,3639611391,CA @@ -120852,39 +127468,33 @@ 3640209408,3640213503,CA 3640213504,3640287231,US 3640287232,3640291327,CA -3640291328,3640312063,US -3640312064,3640312079,A2 +3640291328,3640312079,US 3640312080,3640312095,ID 3640312096,3640312103,GB -3640312104,3640312159,A2 +3640312104,3640312159,US 3640312160,3640312191,ID -3640312192,3640312295,A2 -3640312296,3640312303,US -3640312304,3640312319,A2 +3640312192,3640312319,US 3640312320,3640312575,NI 3640312576,3640312607,NG -3640312608,3640312639,A2 -3640312640,3640312703,US -3640312704,3640315263,A2 -3640315264,3640315327,US -3640315328,3640315391,A2 +3640312608,3640314879,US +3640314880,3640315135,A2 +3640315136,3640315391,US 3640315392,3640315647,TR -3640315648,3640316415,A2 -3640316416,3640316671,US -3640316672,3640316927,A2 -3640316928,3640317183,US -3640317184,3640317503,A2 +3640315648,3640317327,US +3640317328,3640317343,A2 +3640317344,3640317407,US +3640317408,3640317423,A2 +3640317424,3640317503,US 3640317504,3640317535,ID -3640317536,3640317559,A2 +3640317536,3640317559,US 3640317560,3640317567,NL -3640317568,3640317695,A2 -3640317696,3640317823,US -3640317824,3640318207,A2 +3640317568,3640318207,US 3640318208,3640318975,NC 3640318976,3640319103,GQ -3640319104,3640319231,A2 +3640319104,3640319231,US 3640319232,3640319487,NG -3640319488,3640319999,A2 +3640319488,3640319743,US +3640319744,3640319999,A2 3640320000,3640344959,US 3640344960,3640345007,MX 3640345008,3640345023,US @@ -120925,7 +127535,17 @@ 3640360960,3640369151,CA 3640369152,3640410111,US 3640410112,3640418303,CA -3640418304,3640557567,US +3640418304,3640430591,US +3640430592,3640432639,A2 +3640432640,3640433407,US +3640433408,3640433663,A2 +3640433664,3640434943,US +3640438784,3640442879,A2 +3640442880,3640446975,US +3640449024,3640450047,A2 +3640450048,3640451071,US +3640451072,3640459263,A2 +3640459264,3640557567,US 3640557568,3640560511,CA 3640560512,3640560527,US 3640560528,3640564455,CA @@ -121376,6 +127996,7 @@ 3641999360,3642003455,HU 3642007552,3642015743,UA 3642015744,3642019839,CH +3642019840,3642023935,SY 3642023936,3642028031,EG 3642028032,3642028287,A2 3642028288,3642028543,NG @@ -121396,7 +128017,6 @@ 3642031744,3642031871,A2 3642031872,3642032127,GB 3642032128,3642036223,PS -3642036224,3642040319,RU 3642040320,3642048511,BY 3642048512,3642053439,A2 3642053440,3642053631,TZ @@ -121404,7 +128024,6 @@ 3642056704,3642060799,RU 3642060800,3642064895,KE 3642064896,3642068991,SE -3642068992,3642073087,NL 3642073088,3642077183,LV 3642077184,3642081279,BE 3642081280,3642085375,NL @@ -121416,8 +128035,8 @@ 3642109952,3642110463,BE 3642110464,3642110991,NL 3642110992,3642111023,BE -3642111024,3642111359,NL -3642111360,3642111486,BE +3642111024,3642111103,NL +3642111104,3642111486,BE 3642111487,3642111791,NL 3642111792,3642111839,BE 3642111840,3642111887,NL @@ -121543,7 +128162,7 @@ 3642368000,3642376191,CH 3642376192,3642380287,RS 3642380288,3642384383,IT -3642388480,3642392575,NL +3642384384,3642392575,NL 3642392576,3642396671,AM 3642396672,3642404863,MD 3642404864,3642408959,RU @@ -121586,7 +128205,7 @@ 3642499072,3642503167,SE 3642503168,3642507263,LV 3642507264,3642509311,FR -3642511360,3642515455,RU +3642509312,3642515455,RU 3642515456,3642519551,DE 3642519552,3642523647,IT 3642523648,3642527743,GB @@ -121613,12 +128232,12 @@ 3642552848,3642553095,UA 3642553096,3642553097,LV 3642553098,3642553103,UA -3642553104,3642553119,LV -3642553120,3642553151,UA -3642553152,3642553175,LV +3642553104,3642553139,LV +3642553140,3642553143,UA +3642553144,3642553175,LV 3642553176,3642553183,UA -3642553184,3642553327,LV -3642553328,3642553343,UA +3642553184,3642553335,LV +3642553336,3642553343,UA 3642553344,3642553371,RU 3642553372,3642553375,UA 3642553376,3642553377,RU @@ -121641,8 +128260,8 @@ 3642553600,3642553855,UA 3642553856,3642553925,RU 3642553926,3642553935,UA -3642553936,3642553951,RU -3642553952,3642554111,UA +3642553936,3642553959,RU +3642553960,3642554111,UA 3642554112,3642554119,RU 3642554120,3642554123,UA 3642554124,3642554151,RU @@ -121653,17 +128272,14 @@ 3642554224,3642554239,UA 3642554240,3642554271,RU 3642554272,3642554367,UA -3642554368,3642554395,LT -3642554396,3642554399,UA -3642554400,3642554623,LT +3642554368,3642554623,LT 3642554624,3642554631,UA 3642554632,3642554671,LV 3642554672,3642554675,UA -3642554676,3642554679,EE -3642554680,3642554687,LV +3642554676,3642554687,LV 3642554688,3642554751,UA -3642554752,3642554787,LV -3642554788,3642554807,UA +3642554752,3642554791,LV +3642554792,3642554807,UA 3642554808,3642554815,LV 3642554816,3642554847,UA 3642554848,3642554879,LV @@ -121702,8 +128318,8 @@ 3642555200,3642555223,LT 3642555224,3642555227,LV 3642555228,3642555391,LT -3642555392,3642555489,PL -3642555490,3642555647,UA +3642555392,3642555499,PL +3642555500,3642555647,UA 3642555648,3642555683,SE 3642555684,3642555687,UA 3642555688,3642555695,SE @@ -121715,8 +128331,7 @@ 3642555736,3642555743,GB 3642555744,3642555759,SE 3642555760,3642555771,GB -3642555772,3642555775,UA -3642555776,3642555783,SE +3642555772,3642555783,SE 3642555784,3642555787,GB 3642555788,3642555789,UA 3642555790,3642555791,SE @@ -121982,7 +128597,9 @@ 3645091840,3645095935,RU 3645095936,3645104127,PL 3645104128,3645112319,NL -3645112320,3645116415,DE +3645112320,3645113735,DE +3645113736,3645113743,MK +3645113744,3645116415,DE 3645116416,3645120511,GB 3645120512,3645124607,SE 3645124608,3645128703,NL @@ -122019,8 +128636,8 @@ 3645183872,3645183903,FR 3645183904,3645183935,IE 3645183936,3645184256,FR -3645184257,3645184383,GB -3645184384,3645186047,FR +3645184257,3645184447,GB +3645184448,3645186047,FR 3645186048,3645190143,GB 3645190144,3645194239,FI 3645194240,3645202431,DE @@ -122150,7 +128767,9 @@ 3645563136,3645564671,LS 3645564672,3645564927,ZW 3645564928,3645565055,SO -3645565056,3645565695,GB +3645565056,3645565183,GB +3645565184,3645565439,ZW +3645565440,3645565695,GB 3645565696,3645566975,ZW 3645566976,3645571071,RU 3645571072,3645575167,IE @@ -123098,14 +129717,12 @@ 3645779968,3645784063,PS 3645784064,3645788159,NG 3645788160,3645792255,GB -3645792256,3645796351,NO 3645796352,3645800447,CH 3645800448,3645804543,DE 3645804544,3645808639,GB 3645808640,3645812735,DE 3645812736,3645816831,RU 3645816832,3645825023,FI -3645825024,3645829119,GR 3645829120,3645833215,NL 3645833216,3645841407,RU 3645841408,3645845503,BG @@ -123180,9 +129797,7 @@ 3647926396,3647926399,GB 3647926400,3647926783,DE 3647926784,3647926791,GB -3647926792,3647933823,DE -3647933824,3647933855,GB -3647933856,3647936255,DE +3647926792,3647936255,DE 3647936256,3647936383,GB 3647936384,3647936823,DE 3647936824,3647936827,GB @@ -123196,7 +129811,9 @@ 3647954232,3647954239,GB 3647954240,3647954783,DE 3647954784,3647954815,CH -3647954816,3647959567,DE +3647954816,3647957511,DE +3647957512,3647957519,GB +3647957520,3647959567,DE 3647959568,3647959575,ES 3647959576,3647961215,DE 3647961216,3647961247,IT @@ -123242,26 +129859,17 @@ 3647967232,3647968255,BE 3647968256,3647968703,FR 3647968704,3647968767,GB -3647968768,3647968871,FR -3647968872,3647968879,DE -3647968880,3647968895,FR -3647968896,3647968911,DE -3647968912,3647968943,FR -3647968944,3647969279,DE +3647968768,3647969279,FR 3647969280,3647969791,IT 3647969792,3647970303,BE 3647970304,3647971327,FR 3647971328,3647971671,GB 3647971672,3647971679,DE -3647971680,3647972191,GB -3647972192,3647972207,DE -3647972208,3647972215,GB +3647971680,3647972215,GB 3647972216,3647972223,DE 3647972224,3647972259,GB 3647972260,3647972263,DE -3647972264,3647972279,GB -3647972280,3647972287,DE -3647972288,3647972335,GB +3647972264,3647972335,GB 3647972336,3647972351,DE 3647972352,3647973375,GB 3647973376,3647973783,IT @@ -123274,20 +129882,35 @@ 3647976560,3647976575,FR 3647976576,3647976607,BE 3647976608,3647976639,BR -3647976640,3647977471,BE +3647976640,3647976647,BE +3647976648,3647976663,DE +3647976664,3647976719,BE +3647976720,3647976727,DE +3647976728,3647976743,BE +3647976744,3647976751,DE +3647976752,3647976759,BE +3647976760,3647976783,DE +3647976784,3647976791,BE +3647976792,3647976799,DE +3647976800,3647976927,BE +3647976928,3647976935,DE +3647976936,3647977039,BE +3647977040,3647977215,DE +3647977216,3647977471,BE 3647977472,3647977791,GB 3647977792,3647977855,IE 3647977856,3647978495,GB 3647978496,3647979007,NL 3647979008,3647979519,DE 3647979520,3647980543,FR -3647980544,3647981567,GB -3647981568,3647995903,DE +3647980544,3647981055,GB +3647981056,3647981567,IE +3647981568,3647982591,BE +3647982592,3647983615,IT +3647983616,3647995903,DE 3647995904,3648004095,RU 3648004096,3648004607,GB -3648004608,3648004863,RU -3648004864,3648005119,GB -3648005120,3648006271,RU +3648004608,3648006271,RU 3648006272,3648006399,GB 3648006400,3648007167,RU 3648007168,3648008191,GB @@ -123394,7 +130017,9 @@ 3648180768,3648180799,AT 3648180800,3648181023,DE 3648181024,3648181055,AT -3648181056,3648181311,DE +3648181056,3648181215,DE +3648181216,3648181231,RU +3648181232,3648181311,DE 3648181312,3648181343,AT 3648181344,3648181375,DE 3648181376,3648181407,RU @@ -123434,6 +130059,7 @@ 3648200704,3648208895,SE 3648208896,3648212991,DE 3648212992,3648217087,RU +3648217088,3648221183,UA 3648221184,3648225279,IE 3648225280,3648231263,DE 3648231264,3648231295,NL @@ -123461,8 +130087,7 @@ 3648316080,3648316087,SE 3648316088,3648316415,DK 3648316416,3648316671,NL -3648316672,3648316927,AT -3648316928,3648317247,DK +3648316672,3648317247,DK 3648317248,3648317263,CA 3648317264,3648319527,DK 3648319528,3648319535,CH @@ -123570,9 +130195,7 @@ 3649512192,3649512447,FR 3649512448,3649568767,GB 3649568768,3649634303,BE -3649634304,3649699839,GB -3649699840,3649700095,US -3649700096,3649700159,GB +3649634304,3649700159,GB 3649700160,3649700247,US 3649700248,3649700251,BB 3649700252,3649700287,US @@ -123805,7 +130428,9 @@ 3650879488,3650912255,RO 3650912256,3650912639,GB 3650912640,3650912671,FR -3650912672,3650945023,GB +3650912672,3650923103,GB +3650923104,3650923135,FR +3650923136,3650945023,GB 3650945024,3651010559,DK 3651010560,3651076095,GB 3651076096,3651077375,DE @@ -123876,7 +130501,8 @@ 3651108864,3651141631,EE 3651141632,3651173375,DE 3651173376,3651173631,ES -3651173632,3651199743,DE +3651173632,3651199487,DE +3651199488,3651199743,GB 3651199744,3651200255,ES 3651200256,3651201023,PL 3651201024,3651201279,ES @@ -124005,8 +130631,8 @@ 3651874816,3651878911,PL 3651878912,3651883007,RU 3651883008,3651885823,BE -3651885824,3651885843,CD -3651885844,3651885851,BE +3651885824,3651885847,CD +3651885848,3651885851,BE 3651885852,3651885855,CD 3651885856,3651885867,BE 3651885868,3651885875,CD @@ -124014,14 +130640,12 @@ 3651885904,3651885919,CD 3651885920,3651886335,BE 3651886336,3651886347,CD -3651886348,3651886367,BE -3651886368,3651886383,CD -3651886384,3651886387,BE -3651886388,3651886395,CD -3651886396,3651886399,BE -3651886400,3651886403,CD -3651886404,3651886431,BE -3651886432,3651886451,CD +3651886348,3651886379,BE +3651886380,3651886383,CD +3651886384,3651886391,BE +3651886392,3651886395,CD +3651886396,3651886435,BE +3651886436,3651886451,CD 3651886452,3651886455,BE 3651886456,3651886459,CD 3651886460,3651886463,BE @@ -124029,8 +130653,8 @@ 3651886472,3651886479,BE 3651886480,3651886495,CD 3651886496,3651886511,BE -3651886512,3651886519,CD -3651886520,3651887103,BE +3651886512,3651886527,CD +3651886528,3651887103,BE 3651887104,3651891199,GB 3651891200,3651895295,DE 3651895296,3651899391,GB @@ -124126,7 +130750,7 @@ 3651969024,3651977215,DE 3651977216,3651985407,IT 3651985408,3651993599,PL -3651993600,3652001791,RU +3651997696,3652001791,RU 3652001792,3652005887,HU 3652005888,3652009983,GB 3652009984,3652011519,RU @@ -124160,8 +130784,7 @@ 3652020480,3652020735,IE 3652020736,3652020767,GB 3652020768,3652020799,IE -3652020800,3652020831,DE -3652020832,3652020863,GB +3652020800,3652020863,DE 3652020864,3652020959,IE 3652020960,3652021791,GB 3652021792,3652021887,IE @@ -124196,7 +130819,9 @@ 3652034384,3652034399,VG 3652034400,3652034559,GD 3652034560,3652046847,PL -3652046848,3652050499,IE +3652046848,3652050271,IE +3652050272,3652050299,GB +3652050300,3652050499,IE 3652050500,3652050503,GB 3652050504,3652050527,IE 3652050528,3652050535,GB @@ -124232,7 +130857,8 @@ 3652143360,3652143871,A2 3652143872,3652144127,GB 3652144128,3652144383,KE -3652144384,3652145151,A2 +3652144384,3652144639,CM +3652144640,3652145151,A2 3652145152,3652149247,UA 3652149248,3652153343,DE 3652153344,3652157439,SE @@ -124248,11 +130874,11 @@ 3652186880,3652187007,AT 3652187008,3652190207,DE 3652190208,3652714495,GB -3652714496,3652984927,DE -3652984928,3652984935,NL -3652984936,3652985039,DE +3652714496,3652985039,DE 3652985040,3652985047,FR -3652985048,3653039103,DE +3652985048,3653029775,DE +3653029776,3653029791,RO +3653029792,3653039103,DE 3653039104,3653039359,IT 3653039360,3653238783,DE 3653238784,3653334141,CH @@ -124425,7 +131051,7 @@ 3653410048,3653410063,YT 3653410064,3653410079,UG 3653410080,3653410087,A2 -3653410088,3653410095,NG +3653410088,3653410095,IQ 3653410096,3653410111,ZW 3653410112,3653410119,A2 3653410120,3653410127,NG @@ -124517,16 +131143,8 @@ 3653525216,3653525503,AT 3653525504,3653529599,RU 3653529600,3653533695,CZ -3653533696,3653537791,HU 3653537792,3653541887,AT -3653541888,3653545087,UA -3653545088,3653545215,AT -3653545216,3653545343,UA -3653545344,3653545471,SE -3653545472,3653545727,UA -3653545728,3653545791,SE -3653545792,3653545855,UA -3653545856,3653545983,SE +3653541888,3653545983,UA 3653545984,3653550079,CH 3653550080,3653554175,GB 3653554176,3653558271,CZ @@ -124709,6 +131327,8 @@ 3664004096,3664004351,NZ 3664004352,3664004607,MY 3664004608,3664004863,SG +3664004864,3664005119,KH +3664005120,3664005631,ID 3664052224,3664084991,NZ 3664084992,3664117759,KR 3664117760,3664248831,HK @@ -124931,4 +131551,31 @@ 3740925952,3741024255,TW 3741024256,3741057023,KR 3741057024,3741319167,VN -3749707776,3758096383,AP +3741319168,3749838847,CN +3749838848,3749839103,AP +3749839872,3749840895,IN +3749847040,3749855231,HK +3749871616,3749969919,KR +3749969920,3750232063,JP +3750232064,3750756351,TW +3751804928,3752067071,CN +3752067072,3752132607,ID +3752133632,3752134655,ID +3752140800,3752148991,JP +3752198144,3752329215,CN +3752329216,3752853503,KR +3753902080,3754033151,CN +3754033152,3754164223,KR +3754164224,3754229759,IN +3754426368,3754688511,CN +3754688512,3754950655,TH +3755474944,3755737087,JP +3755737088,3755868159,CN +3755868160,3755933695,KR +3755988992,3755990015,HK +3755991040,3755999231,JP +3757572096,3757834239,CN +3757834240,3757867007,AU +3757899776,3757965311,KR +3757965312,3758030847,CN +3758094336,3758095359,AU diff --git a/src/config/torrc.bridge.in b/src/config/torrc.bridge.in new file mode 100644 index 0000000000..557b7adf46 --- /dev/null +++ b/src/config/torrc.bridge.in @@ -0,0 +1,171 @@ +## Configuration file for a typical Tor user +## Last updated 16 July 2009 for Tor 0.2.2.1-alpha. +## (May or may not work for much older or much newer versions of Tor.) +## +## Lines that begin with "## " try to explain what's going on. Lines +## that begin with just "#" are disabled commands: you can enable them +## by removing the "#" symbol. +## +## See 'man tor', or https://www.torproject.org/tor-manual.html, +## for more options you can use in this file. +## +## Tor will look for this file in various places based on your platform: +## https://wiki.torproject.org/noreply/TheOnionRouter/TorFAQ#torrc + + +## Replace this with "SocksPort 0" if you plan to run Tor only as a +## relay, and not make any local application connections yourself. +SocksPort 9050 # what port to open for local application connections +SocksListenAddress 127.0.0.1 # accept connections only from localhost +#SocksListenAddress 192.168.0.1:9100 # listen on this IP:port also + +## Entry policies to allow/deny SOCKS requests based on IP address. +## First entry that matches wins. If no SocksPolicy is set, we accept +## all (and only) requests from SocksListenAddress. +#SocksPolicy accept 192.168.0.0/16 +#SocksPolicy reject * + +## Logs go to stdout at level "notice" unless redirected by something +## else, like one of the below lines. You can have as many Log lines as +## you want. +## +## We advise using "notice" in most cases, since anything more verbose +## may provide sensitive information to an attacker who obtains the logs. +## +## Send all messages of level 'notice' or higher to @LOCALSTATEDIR@/log/tor/notices.log +#Log notice file @LOCALSTATEDIR@/log/tor/notices.log +## Send every possible message to @LOCALSTATEDIR@/log/tor/debug.log +#Log debug file @LOCALSTATEDIR@/log/tor/debug.log +## Use the system log instead of Tor's logfiles +#Log notice syslog +## To send all messages to stderr: +#Log debug stderr + +## Uncomment this to start the process in the background... or use +## --runasdaemon 1 on the command line. This is ignored on Windows; +## see the FAQ entry if you want Tor to run as an NT service. +#RunAsDaemon 1 + +## The directory for keeping all the keys/etc. By default, we store +## things in $HOME/.tor on Unix, and in Application Data\tor on Windows. +#DataDirectory @LOCALSTATEDIR@/lib/tor + +## The port on which Tor will listen for local connections from Tor +## controller applications, as documented in control-spec.txt. +#ControlPort 9051 +## If you enable the controlport, be sure to enable one of these +## authentication methods, to prevent attackers from accessing it. +#HashedControlPassword 16:872860B76453A77D60CA2BB8C1A7042072093276A3D701AD684053EC4C +#CookieAuthentication 1 + +############### This section is just for location-hidden services ### + +## Once you have configured a hidden service, you can look at the +## contents of the file ".../hidden_service/hostname" for the address +## to tell people. +## +## HiddenServicePort x y:z says to redirect requests on port x to the +## address y:z. + +#HiddenServiceDir @LOCALSTATEDIR@/lib/tor/hidden_service/ +#HiddenServicePort 80 127.0.0.1:80 + +#HiddenServiceDir @LOCALSTATEDIR@/lib/tor/other_hidden_service/ +#HiddenServicePort 80 127.0.0.1:80 +#HiddenServicePort 22 127.0.0.1:22 + +################ This section is just for relays ##################### +# +## See https://www.torproject.org/docs/tor-doc-relay for details. + +## Required: what port to advertise for incoming Tor connections. +ORPort 9001 +## If you want to listen on a port other than the one advertised +## in ORPort (e.g. to advertise 443 but bind to 9090), uncomment the +## line below too. You'll need to do ipchains or other port forwarding +## yourself to make this work. +#ORListenAddress 0.0.0.0:9090 + +## A handle for your relay, so people don't have to refer to it by key. +Nickname Unnamed + +## The IP address or full DNS name for your relay. Leave commented out +## and Tor will guess. +#Address noname.example.com + +## Define these to limit how much relayed traffic you will allow. Your +## own traffic is still unthrottled. Note that RelayBandwidthRate must +## be at least 20 KB. +#RelayBandwidthRate 100 KB # Throttle traffic to 100KB/s (800Kbps) +#RelayBandwidthBurst 200 KB # But allow bursts up to 200KB/s (1600Kbps) +RelayBandwidthBurst 10485760 +RelayBandwidthRate 5242880 + +## Use these to restrict the maximum traffic per day, week, or month. +## Note that this threshold applies to sent _and_ to received bytes, +## not to their sum: Setting "4 GB" may allow up to 8 GB +## total before hibernating. +## +## Set a maximum of 4 gigabytes each way per period. +#AccountingMax 4 GB +## Each period starts daily at midnight (AccountingMax is per day) +#AccountingStart day 00:00 +## Each period starts on the 3rd of the month at 15:00 (AccountingMax +## is per month) +#AccountingStart month 3 15:00 + +## Contact info to be published in the directory, so we can contact you +## if your relay is misconfigured or something else goes wrong. Google +## indexes this, so spammers might also collect it. +#ContactInfo Random Person <nobody AT example dot com> +## You might also include your PGP or GPG fingerprint if you have one: +#ContactInfo 1234D/FFFFFFFF Random Person <nobody AT example dot com> + +## Uncomment this to mirror directory information for others. Please do +## if you have enough bandwidth. +DirPort 9030 # what port to advertise for directory connections +## If you want to listen on a port other than the one advertised +## in DirPort (e.g. to advertise 80 but bind to 9091), uncomment the line +## below too. You'll need to do ipchains or other port forwarding yourself +## to make this work. +#DirListenAddress 0.0.0.0:9091 +## Uncomment to return an arbitrary blob of html on your DirPort. Now you +## can explain what Tor is if anybody wonders why your IP address is +## contacting them. See contrib/tor-exit-notice.html in Tor's source +## distribution for a sample. +#DirPortFrontPage @CONFDIR@/tor-exit-notice.html + +## Uncomment this if you run more than one Tor relay, and add the identity +## key fingerprint of each Tor relay you control, even if they're on +## different networks. You declare it here so Tor clients can avoid +## using more than one of your relays in a single circuit. See +## https://wiki.torproject.org/noreply/TheOnionRouter/TorFAQ#MultipleServers +#MyFamily $keyid,$keyid,... + +## A comma-separated list of exit policies. They're considered first +## to last, and the first match wins. If you want to _replace_ +## the default exit policy, end this with either a reject *:* or an +## accept *:*. Otherwise, you're _augmenting_ (prepending to) the +## default exit policy. Leave commented to just use the default, which is +## described in the man page or at +## https://www.torproject.org/documentation.html +## +## Look at https://www.torproject.org/faq-abuse.html#TypicalAbuses +## for issues you might encounter if you use the default exit policy. +## +## If certain IPs and ports are blocked externally, e.g. by your firewall, +## you should update your exit policy to reflect this -- otherwise Tor +## users will be told that those destinations are down. +## +#ExitPolicy accept *:6660-6667,reject *:* # allow irc ports but no more +#ExitPolicy accept *:119 # accept nntp as well as default exit policy +#ExitPolicy reject *:* # no exits allowed +# +## Bridge relays (or "bridges") are Tor relays that aren't listed in the +## main directory. Since there is no complete public list of them, even if an +## ISP is filtering connections to all the known Tor relays, they probably +## won't be able to block all the bridges. Also, websites won't treat you +## differently because they won't know you're running Tor. If you can +## be a real relay, please do; but if not, be a bridge! +BridgeRelay 1 +ExitPolicy reject *:* diff --git a/src/config/torrc.complete.in b/src/config/torrc.complete.in index 310458a5c0..6dbec2fbf9 100644 --- a/src/config/torrc.complete.in +++ b/src/config/torrc.complete.in @@ -1,5 +1,3 @@ -# $Id$ -# Last updated on $Date$ #################################################################### ## This config file is divided into four sections. They are: ## 1. Global Options (clients and servers) @@ -81,6 +79,9 @@ #DirServer moria2 v1 18.244.0.114:80 719B E45D E224 B607 C537 07D0 E214 3E2D 423E 74CF #DirServer tor26 v1 86.59.21.38:80 847B 1F85 0344 D787 6491 A548 92F9 0493 4E4E B85D +## Attempt to lock current and future memory pages and effectively disable swap +# DisableAllSwap 0|1 + ## On startup, setgid to this user. #Group GID diff --git a/src/config/torrc.sample.in b/src/config/torrc.sample.in index d0b1ee1591..f0c78ce5a9 100644 --- a/src/config/torrc.sample.in +++ b/src/config/torrc.sample.in @@ -1,5 +1,5 @@ ## Configuration file for a typical Tor user -## Last updated 12 April 2009 for Tor 0.2.1.14-rc. +## Last updated 16 July 2009 for Tor 0.2.2.1-alpha. ## (May or may not work for much older or much newer versions of Tor.) ## ## Lines that begin with "## " try to explain what's going on. Lines @@ -95,9 +95,22 @@ SocksListenAddress 127.0.0.1 # accept connections only from localhost ## Define these to limit how much relayed traffic you will allow. Your ## own traffic is still unthrottled. Note that RelayBandwidthRate must -## be at least 20 KBytes. -#RelayBandwidthRate 100 KBytes # Throttle traffic to 100KB/s (800Kbps) -#RelayBandwidthBurst 200 KBytes # But allow bursts up to 200KB/s (1600Kbps) +## be at least 20 KB. +#RelayBandwidthRate 100 KB # Throttle traffic to 100KB/s (800Kbps) +#RelayBandwidthBurst 200 KB # But allow bursts up to 200KB/s (1600Kbps) + +## Use these to restrict the maximum traffic per day, week, or month. +## Note that this threshold applies to sent _and_ to received bytes, +## not to their sum: Setting "4 GB" may allow up to 8 GB +## total before hibernating. +## +## Set a maximum of 4 gigabytes each way per period. +#AccountingMax 4 GB +## Each period starts daily at midnight (AccountingMax is per day) +#AccountingStart day 00:00 +## Each period starts on the 3rd of the month at 15:00 (AccountingMax +## is per month) +#AccountingStart month 3 15:00 ## Contact info to be published in the directory, so we can contact you ## if your relay is misconfigured or something else goes wrong. Google @@ -116,8 +129,9 @@ SocksListenAddress 127.0.0.1 # accept connections only from localhost #DirListenAddress 0.0.0.0:9091 ## Uncomment to return an arbitrary blob of html on your DirPort. Now you ## can explain what Tor is if anybody wonders why your IP address is -## contacting them. See contrib/tor-exit-notice.html for a sample. -#DirPortFrontPage /etc/tor/exit-notice.html +## contacting them. See contrib/tor-exit-notice.html in Tor's source +## distribution for a sample. +#DirPortFrontPage @CONFDIR@/tor-exit-notice.html ## Uncomment this if you run more than one Tor relay, and add the identity ## key fingerprint of each Tor relay you control, even if they're on diff --git a/src/or/Makefile.am b/src/or/Makefile.am index ad2476ff15..a9ac3cdee1 100644 --- a/src/or/Makefile.am +++ b/src/or/Makefile.am @@ -1,8 +1,5 @@ -TESTS = test - -noinst_PROGRAMS = test - bin_PROGRAMS = tor +noinst_LIBRARIES = libtor.a if BUILD_NT_SERVICES tor_platform_source=ntmain.c @@ -10,18 +7,30 @@ else tor_platform_source= endif -EXTRA_DIST=ntmain.c +EXTRA_DIST=ntmain.c or_sha1.i + +if USE_EXTERNAL_EVDNS +evdns_source= +else +evdns_source=eventdns.c +endif -tor_SOURCES = buffers.c circuitbuild.c circuitlist.c \ +libtor_a_SOURCES = buffers.c circuitbuild.c circuitlist.c \ circuituse.c command.c config.c \ connection.c connection_edge.c connection_or.c control.c \ cpuworker.c directory.c dirserv.c dirvote.c \ dns.c dnsserv.c geoip.c hibernate.c main.c $(tor_platform_source) \ + microdesc.c \ networkstatus.c onion.c policies.c \ reasons.c relay.c rendcommon.c rendclient.c rendmid.c \ rendservice.c rephist.c router.c routerlist.c routerparse.c \ - eventdns.c \ - tor_main.c + $(evdns_source) config_codedigest.c + +#libtor_a_LIBADD = ../common/libor.a ../common/libor-crypto.a \ +# ../common/libor-event.a + + +tor_SOURCES = tor_main.c AM_CPPFLAGS = -DSHARE_DATADIR="\"$(datadir)\"" \ -DLOCALSTATEDIR="\"$(localstatedir)\"" \ @@ -32,76 +41,48 @@ AM_CPPFLAGS = -DSHARE_DATADIR="\"$(datadir)\"" \ # matters a lot there, and is quite hard to debug if you forget to do it. tor_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ @TOR_LDFLAGS_libevent@ -tor_LDADD = ../common/libor.a ../common/libor-crypto.a \ - -lz @TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ -test_SOURCES = buffers.c circuitbuild.c circuitlist.c \ - circuituse.c command.c config.c \ - connection.c connection_edge.c connection_or.c control.c \ - cpuworker.c directory.c dirserv.c dirvote.c \ - dns.c dnsserv.c geoip.c hibernate.c main.c $(tor_platform_source) \ - networkstatus.c onion.c policies.c \ - reasons.c relay.c rendcommon.c rendclient.c rendmid.c \ - rendservice.c rephist.c router.c routerlist.c routerparse.c \ - eventdns.c \ - test_data.c test.c +tor_LDADD = ./libtor.a ../common/libor.a ../common/libor-crypto.a \ + ../common/libor-event.a \ + @TOR_ZLIB_LIBS@ -lm @TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ -test_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ \ - @TOR_LDFLAGS_libevent@ -test_LDADD = ../common/libor.a ../common/libor-crypto.a \ - -lz @TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ +noinst_HEADERS = buffers.h circuitbuild.h circuitlist.h circuituse.h \ + command.h config.h connection_edge.h connection.h connection_or.h \ + control.h cpuworker.h directory.h dirserv.h dirvote.h dns.h \ + dnsserv.h geoip.h hibernate.h main.h microdesc.h networkstatus.h \ + ntmain.h onion.h policies.h reasons.h relay.h rendclient.h \ + rendcommon.h rendmid.h rendservice.h rephist.h router.h routerlist.h \ + routerparse.h or.h eventdns.h eventdns_tor.h micro-revision.i -noinst_HEADERS = or.h eventdns.h eventdns_tor.h micro-revision.i +config_codedigest.o: or_sha1.i tor_main.o: micro-revision.i micro-revision.i: FORCE - @svkdir=$$SVKROOT; \ - if test "x$$svkdir" = x ; then \ - svkdir=$$HOME/.svk; \ - fi; \ - if test -d ../../.git && test -x "`which git 2>&1;true`" ; then \ - if test -d ../../.git/svn && test -x "`which git-svn 2>&1;true`" ; then \ - git-svn info ../../README | \ - sed -n 's/^Revision: \([0-9][0-9]*\).*/"\1"/p' \ - > micro-revision.tmp \ - || true; \ - fi; \ - elif test -d ../../.svn && test -x "`which svn 2>&1;true`" ; then \ - svn info ../.. | \ - sed -n 's/^Revision: \([0-9][0-9]*\).*/"\1"/p' > micro-revision.tmp \ - || true; \ - elif test -x "`which svk 2>&1;true`" && test -d $$svkdir/local; then \ - location=../..; \ - rev=x; \ - while test x$$rev = xx; do \ - x=`svk info $$location | \ - sed -n 's/^Mirrored From:.*, Rev\. \([0-9][0-9]*\)/\1/p'`; \ - if test x$$x != x; then \ - rev=$$x; \ - break; \ - else \ - loc=`svk info $$location | \ - sed -n 's/^Copied From: \(.*\), Rev\. [0-9][0-9]*/\1/p' | \ - head -1`; \ - if test x$$loc = x; then \ - break; \ - else \ - location=/$$loc; \ - fi; \ - fi; \ - done; \ - if test x$$rev != xx; then \ - echo \"$$rev\" > micro-revision.tmp; \ - fi; \ - fi; \ - if test ! -f micro-revision.tmp ; then \ - if test ! -f micro-revision.i ; then \ - echo '""' > micro-revision.i; \ - fi; \ - elif test ! -f micro-revision.i || \ + @rm -f micro-revision.tmp; \ + if test -d ../../.git && test -x "`which git 2>&1;true`"; then \ + HASH="`git rev-parse --short=16 HEAD`"; \ + echo \"$$HASH\" > micro-revision.tmp; \ + fi; \ + if test ! -f micro-revision.tmp ; then \ + if test ! -f micro-revision.i ; then \ + echo '""' > micro-revision.i; \ + fi; \ + elif test ! -f micro-revision.i || \ test x"`cat micro-revision.tmp`" != x"`cat micro-revision.i`"; then \ - mv micro-revision.tmp micro-revision.i; \ + mv micro-revision.tmp micro-revision.i; \ fi; true +or_sha1.i: $(tor_SOURCES) + if test "@SHA1SUM@" != none; then \ + @SHA1SUM@ $(tor_SOURCES) | @SED@ -n 's/^\(.*\)$$/"\1\\n"/p' > or_sha1.i; \ + elif test "@OPENSSL@" != none; then \ + @OPENSSL@ sha1 $(tor_SOURCES) | @SED@ -n 's/SHA1(\(.*\))= \(.*\)/"\2 \1\\n"/p' > or_sha1.i; \ + else \ + rm or_sha1.i; \ + touch or_sha1.i; \ + fi + +CLEANFILES = micro-revision.i + #Dummy target to ensure that micro-revision.i _always_ gets built. FORCE: diff --git a/src/or/buffers.c b/src/or/buffers.c index 571f86de1f..3ba2760bf3 100644 --- a/src/or/buffers.c +++ b/src/or/buffers.c @@ -12,6 +12,14 @@ **/ #define BUFFERS_PRIVATE #include "or.h" +#include "buffers.h" +#include "config.h" +#include "connection_edge.h" +#include "connection_or.h" +#include "control.h" +#include "reasons.h" +#include "../common/util.h" +#include "../common/torlog.h" #ifdef HAVE_UNISTD_H #include <unistd.h> #endif @@ -145,10 +153,13 @@ get_freelist(size_t alloc) /** Deallocate a chunk or put it on a freelist */ static void -chunk_free(chunk_t *chunk) +chunk_free_unchecked(chunk_t *chunk) { - size_t alloc = CHUNK_ALLOC_SIZE(chunk->memlen); - chunk_freelist_t *freelist = get_freelist(alloc); + size_t alloc; + chunk_freelist_t *freelist; + + alloc = CHUNK_ALLOC_SIZE(chunk->memlen); + freelist = get_freelist(alloc); if (freelist && freelist->cur_length < freelist->max_length) { chunk->next = freelist->head; freelist->head = chunk; @@ -193,7 +204,7 @@ chunk_new_with_alloc_size(size_t alloc) } #else static void -chunk_free(chunk_t *chunk) +chunk_free_unchecked(chunk_t *chunk) { tor_free(chunk); } @@ -401,7 +412,7 @@ buf_pullup(buf_t *buf, size_t bytes, int nulterminate) dest->next = src->next; if (buf->tail == src) buf->tail = dest; - chunk_free(src); + chunk_free_unchecked(src); } else { memcpy(CHUNK_WRITE_PTR(dest), src->data, n); dest->datalen += n; @@ -447,7 +458,7 @@ buf_remove_from_front(buf_t *buf, size_t n) buf->head = victim->next; if (buf->tail == victim) buf->tail = NULL; - chunk_free(victim); + chunk_free_unchecked(victim); } } check(); @@ -481,7 +492,7 @@ buf_clear(buf_t *buf) buf->datalen = 0; for (chunk = buf->head; chunk; chunk = next) { next = chunk->next; - chunk_free(chunk); + chunk_free_unchecked(chunk); } buf->head = buf->tail = NULL; } @@ -520,6 +531,8 @@ buf_slack(const buf_t *buf) void buf_free(buf_t *buf) { + if (!buf) + return; buf_clear(buf); buf->magic = 0xdeadbeef; tor_free(buf); @@ -1395,19 +1408,21 @@ fetch_from_buf_socks(buf_t *buf, socks_request_t *req, if (req->command != SOCKS_COMMAND_RESOLVE_PTR && !addressmap_have_mapping(req->address,0) && !have_warned_about_unsafe_socks) { - log_warn(LD_APP, - "Your application (using socks5 to port %d) is giving " - "Tor only an IP address. Applications that do DNS resolves " - "themselves may leak information. Consider using Socks4A " - "(e.g. via privoxy or socat) instead. For more information, " - "please see http://wiki.noreply.org/noreply/TheOnionRouter/" - "TorFAQ#SOCKSAndDNS.%s", req->port, - safe_socks ? " Rejecting." : ""); - /*have_warned_about_unsafe_socks = 1;*/ + if (get_options()->WarnUnsafeSocks) { + log_warn(LD_APP, + "Your application (using socks5 to port %d) is giving " + "Tor only an IP address. Applications that do DNS resolves " + "themselves may leak information. Consider using Socks4A " + "(e.g. via privoxy or socat) instead. For more information, " + "please see https://wiki.torproject.org/TheOnionRouter/" + "TorFAQ#SOCKSAndDNS.%s", req->port, + safe_socks ? " Rejecting." : ""); + /*have_warned_about_unsafe_socks = 1;*/ /*(for now, warn every time)*/ control_event_client_status(LOG_WARN, "DANGEROUS_SOCKS PROTOCOL=SOCKS5 ADDRESS=%s:%d", req->address, req->port); + } if (safe_socks) return -1; } @@ -1488,7 +1503,8 @@ fetch_from_buf_socks(buf_t *buf, socks_request_t *req, return -1; } log_debug(LD_APP, - "socks4: successfully read destip (%s)", safe_str(tmpbuf)); + "socks4: successfully read destip (%s)", + safe_str_client(tmpbuf)); socks4_prot = socks4; } @@ -1508,18 +1524,21 @@ fetch_from_buf_socks(buf_t *buf, socks_request_t *req, if (socks4_prot != socks4a && !addressmap_have_mapping(tmpbuf,0) && !have_warned_about_unsafe_socks) { - log_warn(LD_APP, + if (get_options()->WarnUnsafeSocks) { + log_warn(LD_APP, "Your application (using socks4 to port %d) is giving Tor " "only an IP address. Applications that do DNS resolves " "themselves may leak information. Consider using Socks4A " "(e.g. via privoxy or socat) instead. For more information, " - "please see http://wiki.noreply.org/noreply/TheOnionRouter/" + "please see https://wiki.torproject.org/TheOnionRouter/" "TorFAQ#SOCKSAndDNS.%s", req->port, safe_socks ? " Rejecting." : ""); - /*have_warned_about_unsafe_socks = 1;*/ /*(for now, warn every time)*/ - control_event_client_status(LOG_WARN, + /*have_warned_about_unsafe_socks = 1;*/ + /*(for now, warn every time)*/ + control_event_client_status(LOG_WARN, "DANGEROUS_SOCKS PROTOCOL=SOCKS4 ADDRESS=%s:%d", tmpbuf, req->port); + } if (safe_socks) return -1; } @@ -1611,6 +1630,132 @@ fetch_from_buf_socks(buf_t *buf, socks_request_t *req, } } +/** Inspect a reply from SOCKS server stored in <b>buf</b> according + * to <b>state</b>, removing the protocol data upon success. Return 0 on + * incomplete response, 1 on success and -1 on error, in which case + * <b>reason</b> is set to a descriptive message (free() when finished + * with it). + * + * As a special case, 2 is returned when user/pass is required + * during SOCKS5 handshake and user/pass is configured. + */ +int +fetch_from_buf_socks_client(buf_t *buf, int state, char **reason) +{ + unsigned char *data; + size_t addrlen; + + if (buf->datalen < 2) + return 0; + + buf_pullup(buf, 128, 0); + tor_assert(buf->head && buf->head->datalen >= 2); + + data = (unsigned char *) buf->head->data; + + switch (state) { + case PROXY_SOCKS4_WANT_CONNECT_OK: + /* Wait for the complete response */ + if (buf->head->datalen < 8) + return 0; + + if (data[1] != 0x5a) { + *reason = tor_strdup(socks4_response_code_to_string(data[1])); + return -1; + } + + /* Success */ + buf_remove_from_front(buf, 8); + return 1; + + case PROXY_SOCKS5_WANT_AUTH_METHOD_NONE: + /* we don't have any credentials */ + if (data[1] != 0x00) { + *reason = tor_strdup("server doesn't support any of our " + "available authentication methods"); + return -1; + } + + log_info(LD_NET, "SOCKS 5 client: continuing without authentication"); + buf_clear(buf); + return 1; + + case PROXY_SOCKS5_WANT_AUTH_METHOD_RFC1929: + /* we have a username and password. return 1 if we can proceed without + * providing authentication, or 2 otherwise. */ + switch (data[1]) { + case 0x00: + log_info(LD_NET, "SOCKS 5 client: we have auth details but server " + "doesn't require authentication."); + buf_clear(buf); + return 1; + case 0x02: + log_info(LD_NET, "SOCKS 5 client: need authentication."); + buf_clear(buf); + return 2; + /* fall through */ + } + + *reason = tor_strdup("server doesn't support any of our available " + "authentication methods"); + return -1; + + case PROXY_SOCKS5_WANT_AUTH_RFC1929_OK: + /* handle server reply to rfc1929 authentication */ + if (data[1] != 0x00) { + *reason = tor_strdup("authentication failed"); + return -1; + } + + log_info(LD_NET, "SOCKS 5 client: authentication successful."); + buf_clear(buf); + return 1; + + case PROXY_SOCKS5_WANT_CONNECT_OK: + /* response is variable length. BND.ADDR, etc, isn't needed + * (don't bother with buf_pullup()), but make sure to eat all + * the data used */ + + /* wait for address type field to arrive */ + if (buf->datalen < 4) + return 0; + + switch (data[3]) { + case 0x01: /* ip4 */ + addrlen = 4; + break; + case 0x04: /* ip6 */ + addrlen = 16; + break; + case 0x03: /* fqdn (can this happen here?) */ + if (buf->datalen < 5) + return 0; + addrlen = 1 + data[4]; + break; + default: + *reason = tor_strdup("invalid response to connect request"); + return -1; + } + + /* wait for address and port */ + if (buf->datalen < 6 + addrlen) + return 0; + + if (data[1] != 0x00) { + *reason = tor_strdup(socks5_response_code_to_string(data[1])); + return -1; + } + + buf_remove_from_front(buf, 6 + addrlen); + return 1; + } + + /* shouldn't get here... */ + tor_assert(0); + + return -1; +} + /** Return 1 iff buf looks more like it has an (obsolete) v0 controller * command on it than any valid v1 controller command. */ int diff --git a/src/or/buffers.h b/src/or/buffers.h new file mode 100644 index 0000000000..42d92dd89b --- /dev/null +++ b/src/or/buffers.h @@ -0,0 +1,59 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2010, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file buffers.h + * \brief Header file for buffers.c. + **/ + +#ifndef _TOR_BUFFERS_H +#define _TOR_BUFFERS_H + +buf_t *buf_new(void); +buf_t *buf_new_with_capacity(size_t size); +void buf_free(buf_t *buf); +void buf_clear(buf_t *buf); +void buf_shrink(buf_t *buf); +void buf_shrink_freelists(int free_all); +void buf_dump_freelist_sizes(int severity); + +size_t buf_datalen(const buf_t *buf); +size_t buf_allocation(const buf_t *buf); +size_t buf_slack(const buf_t *buf); +const char *_buf_peek_raw_buffer(const buf_t *buf); + +int read_to_buf(int s, size_t at_most, buf_t *buf, int *reached_eof, + int *socket_error); +int read_to_buf_tls(tor_tls_t *tls, size_t at_most, buf_t *buf); + +int flush_buf(int s, buf_t *buf, size_t sz, size_t *buf_flushlen); +int flush_buf_tls(tor_tls_t *tls, buf_t *buf, size_t sz, size_t *buf_flushlen); + +int write_to_buf(const char *string, size_t string_len, buf_t *buf); +int write_to_buf_zlib(buf_t *buf, tor_zlib_state_t *state, + const char *data, size_t data_len, int done); +int move_buf_to_buf(buf_t *buf_out, buf_t *buf_in, size_t *buf_flushlen); +int fetch_from_buf(char *string, size_t string_len, buf_t *buf); +int fetch_var_cell_from_buf(buf_t *buf, var_cell_t **out, int linkproto); +int fetch_from_buf_http(buf_t *buf, + char **headers_out, size_t max_headerlen, + char **body_out, size_t *body_used, size_t max_bodylen, + int force_complete); +int fetch_from_buf_socks(buf_t *buf, socks_request_t *req, + int log_sockstype, int safe_socks); +int fetch_from_buf_socks_client(buf_t *buf, int state, char **reason); +int fetch_from_buf_line(buf_t *buf, char *data_out, size_t *data_len); + +int peek_buf_has_control0_command(buf_t *buf); + +void assert_buf_ok(buf_t *buf); + +#ifdef BUFFERS_PRIVATE +int buf_find_string_offset(const buf_t *buf, const char *s, size_t n); +#endif + +#endif + diff --git a/src/or/circuitbuild.c b/src/or/circuitbuild.c index 3d0f728d5c..5bb9d70d5d 100644 --- a/src/or/circuitbuild.c +++ b/src/or/circuitbuild.c @@ -9,9 +9,45 @@ * \brief The actual details of building circuits. **/ +#define CIRCUIT_PRIVATE + #include "or.h" +#include "circuitbuild.h" +#include "circuitlist.h" +#include "circuituse.h" +#include "config.h" +#include "connection.h" +#include "connection_edge.h" +#include "connection_or.h" +#include "control.h" +#include "directory.h" +#include "main.h" +#include "networkstatus.h" +#include "onion.h" +#include "policies.h" +#include "relay.h" +#include "rephist.h" +#include "router.h" +#include "routerlist.h" +#include "routerparse.h" +#include "crypto.h" +#undef log +#include <math.h> + +#ifndef MIN +#define MIN(a,b) ((a)<(b)?(a):(b)) +#endif + +#define CBT_BIN_TO_MS(bin) ((bin)*CBT_BIN_WIDTH + (CBT_BIN_WIDTH/2)) /********* START VARIABLES **********/ +/** Global list of circuit build times */ +// FIXME: Add this as a member for entry_guard_t instead of global? +// Then we could do per-guard statistics, as guards are likely to +// vary in their own latency. The downside of this is that guards +// can change frequently, so we'd be building a lot more circuits +// most likely. +circuit_build_times_t circ_times; /** A global list of all circuits at this hop. */ extern circuit_t *global_circuitlist; @@ -47,6 +83,10 @@ static smartlist_t *entry_guards = NULL; * and those changes need to be flushed to disk. */ static int entry_guards_dirty = 0; +/** If set, we're running the unit tests: we should avoid clobbering + * our state file or accessing get_options() or get_or_state() */ +static int unit_tests = 0; + /********* END VARIABLES ************/ static int circuit_deliver_create_cell(circuit_t *circ, @@ -59,6 +99,1210 @@ static int onion_append_hop(crypt_path_t **head_ptr, extend_info_t *choice); static void entry_guards_changed(void); +static int +circuit_build_times_disabled(void) +{ + if (unit_tests) { + return 0; + } else { + int consensus_disabled = networkstatus_get_param(NULL, "cbtdisabled", + 0); + int config_disabled = !get_options()->LearnCircuitBuildTimeout; + int dirauth_disabled = get_options()->AuthoritativeDir; + int state_disabled = (get_or_state()->LastWritten == -1); + + if (consensus_disabled || config_disabled || dirauth_disabled || + state_disabled) { + log_info(LD_CIRC, + "CircuitBuildTime learning is disabled. " + "Consensus=%d, Config=%d, AuthDir=%d, StateFile=%d", + consensus_disabled, config_disabled, dirauth_disabled, + state_disabled); + return 1; + } else { + return 0; + } + } +} + +static int32_t +circuit_build_times_max_timeouts(void) +{ + int32_t num = networkstatus_get_param(NULL, "cbtmaxtimeouts", + CBT_DEFAULT_MAX_RECENT_TIMEOUT_COUNT); + return num; +} + +static int32_t +circuit_build_times_default_num_xm_modes(void) +{ + int32_t num = networkstatus_get_param(NULL, "cbtnummodes", + CBT_DEFAULT_NUM_XM_MODES); + return num; +} + +static int32_t +circuit_build_times_min_circs_to_observe(void) +{ + int32_t num = networkstatus_get_param(NULL, "cbtmincircs", + CBT_DEFAULT_MIN_CIRCUITS_TO_OBSERVE); + return num; +} + +double +circuit_build_times_quantile_cutoff(void) +{ + int32_t num = networkstatus_get_param(NULL, "cbtquantile", + CBT_DEFAULT_QUANTILE_CUTOFF); + return num/100.0; +} + +static double +circuit_build_times_close_quantile(void) +{ + int32_t num = networkstatus_get_param(NULL, "cbtclosequantile", + CBT_DEFAULT_CLOSE_QUANTILE); + + return num/100.0; +} + +static int32_t +circuit_build_times_test_frequency(void) +{ + int32_t num = networkstatus_get_param(NULL, "cbttestfreq", + CBT_DEFAULT_TEST_FREQUENCY); + return num; +} + +static int32_t +circuit_build_times_min_timeout(void) +{ + int32_t num = networkstatus_get_param(NULL, "cbtmintimeout", + CBT_DEFAULT_TIMEOUT_MIN_VALUE); + return num; +} + +int32_t +circuit_build_times_initial_timeout(void) +{ + int32_t num = networkstatus_get_param(NULL, "cbtinitialtimeout", + CBT_DEFAULT_TIMEOUT_INITIAL_VALUE); + return num; +} + +static int32_t +circuit_build_times_recent_circuit_count(void) +{ + int32_t num = networkstatus_get_param(NULL, "cbtrecentcount", + CBT_DEFAULT_RECENT_CIRCUITS); + return num; +} + +/** + * This function is called when we get a consensus update. + * + * It checks to see if we have changed any consensus parameters + * that require reallocation or discard of previous stats. + */ +void +circuit_build_times_new_consensus_params(circuit_build_times_t *cbt, + networkstatus_t *ns) +{ + int32_t num = networkstatus_get_param(ns, "cbtrecentcount", + CBT_DEFAULT_RECENT_CIRCUITS); + + if (num > 0 && num != cbt->liveness.num_recent_circs) { + int8_t *recent_circs; + log_notice(LD_CIRC, "Changing recent timeout size from %d to %d", + cbt->liveness.num_recent_circs, num); + + tor_assert(cbt->liveness.timeouts_after_firsthop); + + /* + * Technically this is a circular array that we are reallocating + * and memcopying. However, since it only consists of either 1s + * or 0s, and is only used in a statistical test to determine when + * we should discard our history after a sufficient number of 1's + * have been reached, it is fine if order is not preserved or + * elements are lost. + * + * cbtrecentcount should only be changing in cases of severe network + * distress anyway, so memory correctness here is paramount over + * doing acrobatics to preserve the array. + */ + recent_circs = tor_malloc_zero(sizeof(int8_t)*num); + memcpy(recent_circs, cbt->liveness.timeouts_after_firsthop, + sizeof(int8_t)*MIN(num, cbt->liveness.num_recent_circs)); + + // Adjust the index if it needs it. + if (num < cbt->liveness.num_recent_circs) { + cbt->liveness.after_firsthop_idx = MIN(num-1, + cbt->liveness.after_firsthop_idx); + } + + tor_free(cbt->liveness.timeouts_after_firsthop); + cbt->liveness.timeouts_after_firsthop = recent_circs; + cbt->liveness.num_recent_circs = num; + } +} + +/** Make a note that we're running unit tests (rather than running Tor + * itself), so we avoid clobbering our state file. */ +void +circuitbuild_running_unit_tests(void) +{ + unit_tests = 1; +} + +/** + * Return the initial default or configured timeout in milliseconds + */ +static double +circuit_build_times_get_initial_timeout(void) +{ + double timeout; + if (!unit_tests && get_options()->CircuitBuildTimeout) { + timeout = get_options()->CircuitBuildTimeout*1000; + if (timeout < circuit_build_times_min_timeout()) { + log_warn(LD_CIRC, "Config CircuitBuildTimeout too low. Setting to %ds", + circuit_build_times_min_timeout()/1000); + timeout = circuit_build_times_min_timeout(); + } + } else { + timeout = circuit_build_times_initial_timeout(); + } + return timeout; +} + +/** + * Reset the build time state. + * + * Leave estimated parameters, timeout and network liveness intact + * for future use. + */ +void +circuit_build_times_reset(circuit_build_times_t *cbt) +{ + memset(cbt->circuit_build_times, 0, sizeof(cbt->circuit_build_times)); + cbt->total_build_times = 0; + cbt->build_times_idx = 0; + cbt->have_computed_timeout = 0; +} + +/** + * Initialize the buildtimes structure for first use. + * + * Sets the initial timeout value based to either the + * config setting or BUILD_TIMEOUT_INITIAL_VALUE. + */ +void +circuit_build_times_init(circuit_build_times_t *cbt) +{ + memset(cbt, 0, sizeof(*cbt)); + cbt->liveness.num_recent_circs = circuit_build_times_recent_circuit_count(); + cbt->liveness.timeouts_after_firsthop = tor_malloc_zero(sizeof(int8_t)* + cbt->liveness.num_recent_circs); + cbt->close_ms = cbt->timeout_ms = circuit_build_times_get_initial_timeout(); + control_event_buildtimeout_set(cbt, BUILDTIMEOUT_SET_EVENT_RESET); +} + +/** + * Rewind our build time history by n positions. + */ +static void +circuit_build_times_rewind_history(circuit_build_times_t *cbt, int n) +{ + int i = 0; + + cbt->build_times_idx -= n; + cbt->build_times_idx %= CBT_NCIRCUITS_TO_OBSERVE; + + for (i = 0; i < n; i++) { + cbt->circuit_build_times[(i+cbt->build_times_idx) + %CBT_NCIRCUITS_TO_OBSERVE]=0; + } + + if (cbt->total_build_times > n) { + cbt->total_build_times -= n; + } else { + cbt->total_build_times = 0; + } + + log_info(LD_CIRC, + "Rewound history by %d places. Current index: %d. " + "Total: %d", n, cbt->build_times_idx, cbt->total_build_times); +} + +/** + * Add a new build time value <b>time</b> to the set of build times. Time + * units are milliseconds. + * + * circuit_build_times <b>cbt</a> is a circular array, so loop around when + * array is full. + */ +int +circuit_build_times_add_time(circuit_build_times_t *cbt, build_time_t time) +{ + if (time <= 0 || time > CBT_BUILD_TIME_MAX) { + log_warn(LD_BUG, "Circuit build time is too large (%u)." + "This is probably a bug.", time); + tor_fragile_assert(); + return -1; + } + + log_debug(LD_CIRC, "Adding circuit build time %u", time); + + cbt->circuit_build_times[cbt->build_times_idx] = time; + cbt->build_times_idx = (cbt->build_times_idx + 1) % CBT_NCIRCUITS_TO_OBSERVE; + if (cbt->total_build_times < CBT_NCIRCUITS_TO_OBSERVE) + cbt->total_build_times++; + + if ((cbt->total_build_times % CBT_SAVE_STATE_EVERY) == 0) { + /* Save state every n circuit builds */ + if (!unit_tests && !get_options()->AvoidDiskWrites) + or_state_mark_dirty(get_or_state(), 0); + } + + return 0; +} + +/** + * Return maximum circuit build time + */ +static build_time_t +circuit_build_times_max(circuit_build_times_t *cbt) +{ + int i = 0; + build_time_t max_build_time = 0; + for (i = 0; i < CBT_NCIRCUITS_TO_OBSERVE; i++) { + if (cbt->circuit_build_times[i] > max_build_time + && cbt->circuit_build_times[i] != CBT_BUILD_ABANDONED) + max_build_time = cbt->circuit_build_times[i]; + } + return max_build_time; +} + +#if 0 +/** Return minimum circuit build time */ +build_time_t +circuit_build_times_min(circuit_build_times_t *cbt) +{ + int i = 0; + build_time_t min_build_time = CBT_BUILD_TIME_MAX; + for (i = 0; i < CBT_NCIRCUITS_TO_OBSERVE; i++) { + if (cbt->circuit_build_times[i] && /* 0 <-> uninitialized */ + cbt->circuit_build_times[i] < min_build_time) + min_build_time = cbt->circuit_build_times[i]; + } + if (min_build_time == CBT_BUILD_TIME_MAX) { + log_warn(LD_CIRC, "No build times less than CBT_BUILD_TIME_MAX!"); + } + return min_build_time; +} +#endif + +/** + * Calculate and return a histogram for the set of build times. + * + * Returns an allocated array of histrogram bins representing + * the frequency of index*CBT_BIN_WIDTH millisecond + * build times. Also outputs the number of bins in nbins. + * + * The return value must be freed by the caller. + */ +static uint32_t * +circuit_build_times_create_histogram(circuit_build_times_t *cbt, + build_time_t *nbins) +{ + uint32_t *histogram; + build_time_t max_build_time = circuit_build_times_max(cbt); + int i, c; + + *nbins = 1 + (max_build_time / CBT_BIN_WIDTH); + histogram = tor_malloc_zero(*nbins * sizeof(build_time_t)); + + // calculate histogram + for (i = 0; i < CBT_NCIRCUITS_TO_OBSERVE; i++) { + if (cbt->circuit_build_times[i] == 0 + || cbt->circuit_build_times[i] == CBT_BUILD_ABANDONED) + continue; /* 0 <-> uninitialized */ + + c = (cbt->circuit_build_times[i] / CBT_BIN_WIDTH); + histogram[c]++; + } + + return histogram; +} + +/** + * Return the Pareto start-of-curve parameter Xm. + * + * Because we are not a true Pareto curve, we compute this as the + * weighted average of the N=3 most frequent build time bins. + */ +static build_time_t +circuit_build_times_get_xm(circuit_build_times_t *cbt) +{ + build_time_t i, nbins; + build_time_t *nth_max_bin; + int32_t bin_counts=0; + build_time_t ret = 0; + uint32_t *histogram = circuit_build_times_create_histogram(cbt, &nbins); + int n=0; + int num_modes = circuit_build_times_default_num_xm_modes(); + + // Only use one mode if < 1000 buildtimes. Not enough data + // for multiple. + if (cbt->total_build_times < CBT_NCIRCUITS_TO_OBSERVE) + num_modes = 1; + + nth_max_bin = (build_time_t*)tor_malloc_zero(num_modes*sizeof(build_time_t)); + + for (i = 0; i < nbins; i++) { + if (histogram[i] >= histogram[nth_max_bin[0]]) { + nth_max_bin[0] = i; + } + + for (n = 1; n < num_modes; n++) { + if (histogram[i] >= histogram[nth_max_bin[n]] && + (!histogram[nth_max_bin[n-1]] + || histogram[i] < histogram[nth_max_bin[n-1]])) { + nth_max_bin[n] = i; + } + } + } + + for (n = 0; n < num_modes; n++) { + bin_counts += histogram[nth_max_bin[n]]; + ret += CBT_BIN_TO_MS(nth_max_bin[n])*histogram[nth_max_bin[n]]; + log_info(LD_CIRC, "Xm mode #%d: %u %u", n, CBT_BIN_TO_MS(nth_max_bin[n]), + histogram[nth_max_bin[n]]); + } + + ret /= bin_counts; + tor_free(histogram); + tor_free(nth_max_bin); + + return ret; +} + +/** + * Output a histogram of current circuit build times to + * the or_state_t state structure. + */ +void +circuit_build_times_update_state(circuit_build_times_t *cbt, + or_state_t *state) +{ + uint32_t *histogram; + build_time_t i = 0; + build_time_t nbins = 0; + config_line_t **next, *line; + + histogram = circuit_build_times_create_histogram(cbt, &nbins); + // write to state + config_free_lines(state->BuildtimeHistogram); + next = &state->BuildtimeHistogram; + *next = NULL; + + state->TotalBuildTimes = cbt->total_build_times; + state->CircuitBuildAbandonedCount = 0; + + for (i = 0; i < CBT_NCIRCUITS_TO_OBSERVE; i++) { + if (cbt->circuit_build_times[i] == CBT_BUILD_ABANDONED) + state->CircuitBuildAbandonedCount++; + } + + for (i = 0; i < nbins; i++) { + // compress the histogram by skipping the blanks + if (histogram[i] == 0) continue; + *next = line = tor_malloc_zero(sizeof(config_line_t)); + line->key = tor_strdup("CircuitBuildTimeBin"); + line->value = tor_malloc(25); + tor_snprintf(line->value, 25, "%d %d", + CBT_BIN_TO_MS(i), histogram[i]); + next = &(line->next); + } + + if (!unit_tests) { + if (!get_options()->AvoidDiskWrites) + or_state_mark_dirty(get_or_state(), 0); + } + + tor_free(histogram); +} + +/** + * Shuffle the build times array. + * + * Stolen from http://en.wikipedia.org/wiki/Fisher\u2013Yates_shuffle + */ +static void +circuit_build_times_shuffle_and_store_array(circuit_build_times_t *cbt, + build_time_t *raw_times, + int num_times) +{ + int n = num_times; + if (num_times > CBT_NCIRCUITS_TO_OBSERVE) { + log_notice(LD_CIRC, "Decreasing circuit_build_times size from %d to %d", + num_times, CBT_NCIRCUITS_TO_OBSERVE); + } + + /* This code can only be run on a compact array */ + while (n-- > 1) { + int k = crypto_rand_int(n + 1); /* 0 <= k <= n. */ + build_time_t tmp = raw_times[k]; + raw_times[k] = raw_times[n]; + raw_times[n] = tmp; + } + + /* Since the times are now shuffled, take a random CBT_NCIRCUITS_TO_OBSERVE + * subset (ie the first CBT_NCIRCUITS_TO_OBSERVE values) */ + for (n = 0; n < MIN(num_times, CBT_NCIRCUITS_TO_OBSERVE); n++) { + circuit_build_times_add_time(cbt, raw_times[n]); + } +} + +/** + * Filter old synthetic timeouts that were created before the + * new right-censored Pareto calculation was deployed. + * + * Once all clients before 0.2.1.13-alpha are gone, this code + * will be unused. + */ +static int +circuit_build_times_filter_timeouts(circuit_build_times_t *cbt) +{ + int num_filtered=0, i=0; + double timeout_rate = 0; + build_time_t max_timeout = 0; + + timeout_rate = circuit_build_times_timeout_rate(cbt); + max_timeout = (build_time_t)cbt->close_ms; + + for (i = 0; i < CBT_NCIRCUITS_TO_OBSERVE; i++) { + if (cbt->circuit_build_times[i] > max_timeout) { + build_time_t replaced = cbt->circuit_build_times[i]; + num_filtered++; + cbt->circuit_build_times[i] = CBT_BUILD_ABANDONED; + + log_debug(LD_CIRC, "Replaced timeout %d with %d", replaced, + cbt->circuit_build_times[i]); + } + } + + log_info(LD_CIRC, + "We had %d timeouts out of %d build times, " + "and filtered %d above the max of %u", + (int)(cbt->total_build_times*timeout_rate), + cbt->total_build_times, num_filtered, max_timeout); + + return num_filtered; +} + +/** + * Load histogram from <b>state</b>, shuffling the resulting array + * after we do so. Use this result to estimate parameters and + * calculate the timeout. + * + * Returns -1 and sets msg on error. Msg must be freed by the caller. + */ +int +circuit_build_times_parse_state(circuit_build_times_t *cbt, + or_state_t *state, char **msg) +{ + int tot_values = 0; + uint32_t loaded_cnt = 0, N = 0; + config_line_t *line; + unsigned int i; + build_time_t *loaded_times; + circuit_build_times_init(cbt); + *msg = NULL; + + if (circuit_build_times_disabled()) { + return 0; + } + + /* build_time_t 0 means uninitialized */ + loaded_times = tor_malloc_zero(sizeof(build_time_t)*state->TotalBuildTimes); + + for (line = state->BuildtimeHistogram; line; line = line->next) { + smartlist_t *args = smartlist_create(); + smartlist_split_string(args, line->value, " ", + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); + if (smartlist_len(args) < 2) { + *msg = tor_strdup("Unable to parse circuit build times: " + "Too few arguments to CircuitBuildTime"); + SMARTLIST_FOREACH(args, char*, cp, tor_free(cp)); + smartlist_free(args); + break; + } else { + const char *ms_str = smartlist_get(args,0); + const char *count_str = smartlist_get(args,1); + uint32_t count, k; + build_time_t ms; + int ok; + ms = (build_time_t)tor_parse_ulong(ms_str, 0, 0, + CBT_BUILD_TIME_MAX, &ok, NULL); + if (!ok) { + *msg = tor_strdup("Unable to parse circuit build times: " + "Unparsable bin number"); + SMARTLIST_FOREACH(args, char*, cp, tor_free(cp)); + smartlist_free(args); + break; + } + count = (uint32_t)tor_parse_ulong(count_str, 0, 0, + UINT32_MAX, &ok, NULL); + if (!ok) { + *msg = tor_strdup("Unable to parse circuit build times: " + "Unparsable bin count"); + SMARTLIST_FOREACH(args, char*, cp, tor_free(cp)); + smartlist_free(args); + break; + } + + if (loaded_cnt+count+state->CircuitBuildAbandonedCount + > state->TotalBuildTimes) { + log_warn(LD_CIRC, + "Too many build times in state file. " + "Stopping short before %d", + loaded_cnt+count); + SMARTLIST_FOREACH(args, char*, cp, tor_free(cp)); + smartlist_free(args); + break; + } + + for (k = 0; k < count; k++) { + loaded_times[loaded_cnt++] = ms; + } + N++; + SMARTLIST_FOREACH(args, char*, cp, tor_free(cp)); + smartlist_free(args); + } + } + + log_info(LD_CIRC, + "Adding %d timeouts.", state->CircuitBuildAbandonedCount); + for (i=0; i < state->CircuitBuildAbandonedCount; i++) { + loaded_times[loaded_cnt++] = CBT_BUILD_ABANDONED; + } + + if (loaded_cnt != state->TotalBuildTimes) { + log_warn(LD_CIRC, + "Corrupt state file? Build times count mismatch. " + "Read %d times, but file says %d", loaded_cnt, + state->TotalBuildTimes); + *msg = tor_strdup("Build times count mismatch."); + circuit_build_times_reset(cbt); + tor_free(loaded_times); + return -1; + } + + circuit_build_times_shuffle_and_store_array(cbt, loaded_times, loaded_cnt); + + /* Verify that we didn't overwrite any indexes */ + for (i=0; i < CBT_NCIRCUITS_TO_OBSERVE; i++) { + if (!cbt->circuit_build_times[i]) + break; + tot_values++; + } + log_info(LD_CIRC, + "Loaded %d/%d values from %d lines in circuit time histogram", + tot_values, cbt->total_build_times, N); + + if (cbt->total_build_times != tot_values + || cbt->total_build_times > CBT_NCIRCUITS_TO_OBSERVE) { + log_warn(LD_CIRC, + "Corrupt state file? Shuffled build times mismatch. " + "Read %d times, but file says %d", tot_values, + state->TotalBuildTimes); + *msg = tor_strdup("Build times count mismatch."); + circuit_build_times_reset(cbt); + tor_free(loaded_times); + return -1; + } + + circuit_build_times_set_timeout(cbt); + + if (!state->CircuitBuildAbandonedCount && cbt->total_build_times) { + circuit_build_times_filter_timeouts(cbt); + } + + tor_free(loaded_times); + return *msg ? -1 : 0; +} + +/** + * Estimates the Xm and Alpha parameters using + * http://en.wikipedia.org/wiki/Pareto_distribution#Parameter_estimation + * + * The notable difference is that we use mode instead of min to estimate Xm. + * This is because our distribution is frechet-like. We claim this is + * an acceptable approximation because we are only concerned with the + * accuracy of the CDF of the tail. + */ +int +circuit_build_times_update_alpha(circuit_build_times_t *cbt) +{ + build_time_t *x=cbt->circuit_build_times; + double a = 0; + int n=0,i=0,abandoned_count=0; + build_time_t max_time=0; + + /* http://en.wikipedia.org/wiki/Pareto_distribution#Parameter_estimation */ + /* We sort of cheat here and make our samples slightly more pareto-like + * and less frechet-like. */ + cbt->Xm = circuit_build_times_get_xm(cbt); + + tor_assert(cbt->Xm > 0); + + for (i=0; i< CBT_NCIRCUITS_TO_OBSERVE; i++) { + if (!x[i]) { + continue; + } + + if (x[i] < cbt->Xm) { + a += tor_mathlog(cbt->Xm); + } else if (x[i] == CBT_BUILD_ABANDONED) { + abandoned_count++; + } else { + a += tor_mathlog(x[i]); + if (x[i] > max_time) + max_time = x[i]; + } + n++; + } + + /* + * We are erring and asserting here because this can only happen + * in codepaths other than startup. The startup state parsing code + * performs this same check, and resets state if it hits it. If we + * hit it at runtime, something serious has gone wrong. + */ + if (n!=cbt->total_build_times) { + log_err(LD_CIRC, "Discrepancy in build times count: %d vs %d", n, + cbt->total_build_times); + } + tor_assert(n==cbt->total_build_times); + + if (max_time <= 0) { + /* This can happen if Xm is actually the *maximum* value in the set. + * It can also happen if we've abandoned every single circuit somehow. + * In either case, tell the caller not to compute a new build timeout. */ + log_warn(LD_BUG, + "Could not determine largest build time (%d). " + "Xm is %dms and we've abandoned %d out of %d circuits.", max_time, + cbt->Xm, abandoned_count, n); + return 0; + } + + a += abandoned_count*tor_mathlog(max_time); + + a -= n*tor_mathlog(cbt->Xm); + // Estimator comes from Eq #4 in: + // "Bayesian estimation based on trimmed samples from Pareto populations" + // by Arturo J. Fernández. We are right-censored only. + a = (n-abandoned_count)/a; + + cbt->alpha = a; + + return 1; +} + +/** + * This is the Pareto Quantile Function. It calculates the point x + * in the distribution such that F(x) = quantile (ie quantile*100% + * of the mass of the density function is below x on the curve). + * + * We use it to calculate the timeout and also to generate synthetic + * values of time for circuits that timeout before completion. + * + * See http://en.wikipedia.org/wiki/Quantile_function, + * http://en.wikipedia.org/wiki/Inverse_transform_sampling and + * http://en.wikipedia.org/wiki/Pareto_distribution#Generating_a_ + * random_sample_from_Pareto_distribution + * That's right. I'll cite wikipedia all day long. + * + * Return value is in milliseconds. + */ +double +circuit_build_times_calculate_timeout(circuit_build_times_t *cbt, + double quantile) +{ + double ret; + tor_assert(quantile >= 0); + tor_assert(1.0-quantile > 0); + tor_assert(cbt->Xm > 0); + + ret = cbt->Xm/pow(1.0-quantile,1.0/cbt->alpha); + if (ret > INT32_MAX) { + ret = INT32_MAX; + } + tor_assert(ret > 0); + return ret; +} + +/** Pareto CDF */ +double +circuit_build_times_cdf(circuit_build_times_t *cbt, double x) +{ + double ret; + tor_assert(cbt->Xm > 0); + ret = 1.0-pow(cbt->Xm/x,cbt->alpha); + tor_assert(0 <= ret && ret <= 1.0); + return ret; +} + +/** + * Generate a synthetic time using our distribution parameters. + * + * The return value will be within the [q_lo, q_hi) quantile points + * on the CDF. + */ +build_time_t +circuit_build_times_generate_sample(circuit_build_times_t *cbt, + double q_lo, double q_hi) +{ + double randval = crypto_rand_double(); + build_time_t ret; + double u; + + /* Generate between [q_lo, q_hi) */ + /*XXXX This is what nextafter is supposed to be for; we should use it on the + * platforms that support it. */ + q_hi -= 1.0/(INT32_MAX); + + tor_assert(q_lo >= 0); + tor_assert(q_hi < 1); + tor_assert(q_lo < q_hi); + + u = q_lo + (q_hi-q_lo)*randval; + + tor_assert(0 <= u && u < 1.0); + /* circuit_build_times_calculate_timeout returns <= INT32_MAX */ + ret = (build_time_t) + tor_lround(circuit_build_times_calculate_timeout(cbt, u)); + tor_assert(ret > 0); + return ret; +} + +/** + * Estimate an initial alpha parameter by solving the quantile + * function with a quantile point and a specific timeout value. + */ +void +circuit_build_times_initial_alpha(circuit_build_times_t *cbt, + double quantile, double timeout_ms) +{ + // Q(u) = Xm/((1-u)^(1/a)) + // Q(0.8) = Xm/((1-0.8))^(1/a)) = CircBuildTimeout + // CircBuildTimeout = Xm/((1-0.8))^(1/a)) + // CircBuildTimeout = Xm*((1-0.8))^(-1/a)) + // ln(CircBuildTimeout) = ln(Xm)+ln(((1-0.8)))*(-1/a) + // -ln(1-0.8)/(ln(CircBuildTimeout)-ln(Xm))=a + tor_assert(quantile >= 0); + tor_assert(cbt->Xm > 0); + cbt->alpha = tor_mathlog(1.0-quantile)/ + (tor_mathlog(cbt->Xm)-tor_mathlog(timeout_ms)); + tor_assert(cbt->alpha > 0); +} + +/** + * Returns true if we need circuits to be built + */ +int +circuit_build_times_needs_circuits(circuit_build_times_t *cbt) +{ + /* Return true if < MIN_CIRCUITS_TO_OBSERVE */ + if (cbt->total_build_times < circuit_build_times_min_circs_to_observe()) + return 1; + return 0; +} + +/** + * Returns true if we should build a timeout test circuit + * right now. + */ +int +circuit_build_times_needs_circuits_now(circuit_build_times_t *cbt) +{ + return circuit_build_times_needs_circuits(cbt) && + approx_time()-cbt->last_circ_at > circuit_build_times_test_frequency(); +} + +/** + * Called to indicate that the network showed some signs of liveness. + * + * This function is called every time we receive a cell. Avoid + * syscalls, events, and other high-intensity work. + */ +void +circuit_build_times_network_is_live(circuit_build_times_t *cbt) +{ + cbt->liveness.network_last_live = approx_time(); + cbt->liveness.nonlive_discarded = 0; + cbt->liveness.nonlive_timeouts = 0; +} + +/** + * Called to indicate that we completed a circuit. Because this circuit + * succeeded, it doesn't count as a timeout-after-the-first-hop. + */ +void +circuit_build_times_network_circ_success(circuit_build_times_t *cbt) +{ + cbt->liveness.timeouts_after_firsthop[cbt->liveness.after_firsthop_idx] = 0; + cbt->liveness.after_firsthop_idx++; + cbt->liveness.after_firsthop_idx %= cbt->liveness.num_recent_circs; +} + +/** + * A circuit just timed out. If it failed after the first hop, record it + * in our history for later deciding if the network speed has changed. + */ +static void +circuit_build_times_network_timeout(circuit_build_times_t *cbt, + int did_onehop) +{ + if (did_onehop) { + cbt->liveness.timeouts_after_firsthop[cbt->liveness.after_firsthop_idx]=1; + cbt->liveness.after_firsthop_idx++; + cbt->liveness.after_firsthop_idx %= cbt->liveness.num_recent_circs; + } +} + +/** + * A circuit was just forcibly closed. If there has been no recent network + * activity at all, but this circuit was launched back when we thought the + * network was live, increment the number of "nonlive" circuit timeouts. + */ +static void +circuit_build_times_network_close(circuit_build_times_t *cbt, + int did_onehop, time_t start_time) +{ + time_t now = time(NULL); + /* + * Check if this is a timeout that was for a circuit that spent its + * entire existence during a time where we have had no network activity. + * + * Also double check that it is a valid timeout after we have possibly + * just recently reset cbt->close_ms. + * + * We use close_ms here because timeouts aren't actually counted as timeouts + * until close_ms elapses. + */ + if (cbt->liveness.network_last_live <= start_time && + start_time <= (now - cbt->close_ms/1000.0)) { + if (did_onehop) { + char last_live_buf[ISO_TIME_LEN+1]; + char start_time_buf[ISO_TIME_LEN+1]; + char now_buf[ISO_TIME_LEN+1]; + format_local_iso_time(last_live_buf, cbt->liveness.network_last_live); + format_local_iso_time(start_time_buf, start_time); + format_local_iso_time(now_buf, now); + log_warn(LD_BUG, + "Circuit somehow completed a hop while the network was " + "not live. Network was last live at %s, but circuit launched " + "at %s. It's now %s.", last_live_buf, start_time_buf, + now_buf); + } + cbt->liveness.nonlive_timeouts++; + } +} + +/** + * Returns false if the network has not received a cell or tls handshake + * in the past NETWORK_NOTLIVE_TIMEOUT_COUNT circuits. + * + * Also has the side effect of rewinding the circuit time history + * in the case of recent liveness changes. + */ +int +circuit_build_times_network_check_live(circuit_build_times_t *cbt) +{ + time_t now = approx_time(); + if (cbt->liveness.nonlive_timeouts >= CBT_NETWORK_NONLIVE_DISCARD_COUNT) { + if (!cbt->liveness.nonlive_discarded) { + cbt->liveness.nonlive_discarded = 1; + log_notice(LD_CIRC, "Network is no longer live (too many recent " + "circuit timeouts). Dead for %ld seconds.", + (long int)(now - cbt->liveness.network_last_live)); + /* Only discard NETWORK_NONLIVE_TIMEOUT_COUNT-1 because we stopped + * counting after that */ + circuit_build_times_rewind_history(cbt, + CBT_NETWORK_NONLIVE_TIMEOUT_COUNT-1); + control_event_buildtimeout_set(cbt, BUILDTIMEOUT_SET_EVENT_DISCARD); + } + return 0; + } else if (cbt->liveness.nonlive_timeouts >= + CBT_NETWORK_NONLIVE_TIMEOUT_COUNT) { + if (cbt->timeout_ms < circuit_build_times_get_initial_timeout()) { + log_notice(LD_CIRC, + "Network is flaky. No activity for %ld seconds. " + "Temporarily raising timeout to %lds.", + (long int)(now - cbt->liveness.network_last_live), + tor_lround(circuit_build_times_get_initial_timeout()/1000)); + cbt->liveness.suspended_timeout = cbt->timeout_ms; + cbt->liveness.suspended_close_timeout = cbt->close_ms; + cbt->close_ms = cbt->timeout_ms + = circuit_build_times_get_initial_timeout(); + control_event_buildtimeout_set(cbt, BUILDTIMEOUT_SET_EVENT_SUSPENDED); + } + + return 0; + } else if (cbt->liveness.suspended_timeout > 0) { + log_notice(LD_CIRC, + "Network activity has resumed. " + "Resuming circuit timeout calculations."); + cbt->timeout_ms = cbt->liveness.suspended_timeout; + cbt->close_ms = cbt->liveness.suspended_close_timeout; + cbt->liveness.suspended_timeout = 0; + cbt->liveness.suspended_close_timeout = 0; + control_event_buildtimeout_set(cbt, BUILDTIMEOUT_SET_EVENT_RESUME); + } + + return 1; +} + +/** + * Returns true if we have seen more than MAX_RECENT_TIMEOUT_COUNT of + * the past RECENT_CIRCUITS time out after the first hop. Used to detect + * if the network connection has changed significantly. + * + * Also resets the entire timeout history in this case and causes us + * to restart the process of building test circuits and estimating a + * new timeout. + */ +int +circuit_build_times_network_check_changed(circuit_build_times_t *cbt) +{ + int total_build_times = cbt->total_build_times; + int timeout_count=0; + int i; + + /* how many of our recent circuits made it to the first hop but then + * timed out? */ + for (i = 0; i < cbt->liveness.num_recent_circs; i++) { + timeout_count += cbt->liveness.timeouts_after_firsthop[i]; + } + + /* If 80% of our recent circuits are timing out after the first hop, + * we need to re-estimate a new initial alpha and timeout. */ + if (timeout_count < circuit_build_times_max_timeouts()) { + return 0; + } + + circuit_build_times_reset(cbt); + memset(cbt->liveness.timeouts_after_firsthop, 0, + sizeof(*cbt->liveness.timeouts_after_firsthop)* + cbt->liveness.num_recent_circs); + cbt->liveness.after_firsthop_idx = 0; + + /* Check to see if this has happened before. If so, double the timeout + * to give people on abysmally bad network connections a shot at access */ + if (cbt->timeout_ms >= circuit_build_times_get_initial_timeout()) { + if (cbt->timeout_ms > INT32_MAX/2 || cbt->close_ms > INT32_MAX/2) { + log_warn(LD_CIRC, "Insanely large circuit build timeout value. " + "(timeout = %lfmsec, close = %lfmsec)", + cbt->timeout_ms, cbt->close_ms); + } else { + cbt->timeout_ms *= 2; + cbt->close_ms *= 2; + } + } else { + cbt->close_ms = cbt->timeout_ms + = circuit_build_times_get_initial_timeout(); + } + + control_event_buildtimeout_set(cbt, BUILDTIMEOUT_SET_EVENT_RESET); + + log_notice(LD_CIRC, + "Network connection speed appears to have changed. Resetting " + "timeout to %lds after %d timeouts and %d buildtimes.", + tor_lround(cbt->timeout_ms/1000), timeout_count, + total_build_times); + + return 1; +} + +/** + * Count the number of timeouts in a set of cbt data. + */ +double +circuit_build_times_timeout_rate(const circuit_build_times_t *cbt) +{ + int i=0,timeouts=0; + for (i = 0; i < CBT_NCIRCUITS_TO_OBSERVE; i++) { + if (cbt->circuit_build_times[i] >= cbt->timeout_ms) { + timeouts++; + } + } + + if (!cbt->total_build_times) + return 0; + + return ((double)timeouts)/cbt->total_build_times; +} + +/** + * Count the number of closed circuits in a set of cbt data. + */ +double +circuit_build_times_close_rate(const circuit_build_times_t *cbt) +{ + int i=0,closed=0; + for (i = 0; i < CBT_NCIRCUITS_TO_OBSERVE; i++) { + if (cbt->circuit_build_times[i] == CBT_BUILD_ABANDONED) { + closed++; + } + } + + if (!cbt->total_build_times) + return 0; + + return ((double)closed)/cbt->total_build_times; +} + +/** + * Store a timeout as a synthetic value. + * + * Returns true if the store was successful and we should possibly + * update our timeout estimate. + */ +int +circuit_build_times_count_close(circuit_build_times_t *cbt, + int did_onehop, + time_t start_time) +{ + if (circuit_build_times_disabled()) { + cbt->close_ms = cbt->timeout_ms + = circuit_build_times_get_initial_timeout(); + return 0; + } + + /* Record this force-close to help determine if the network is dead */ + circuit_build_times_network_close(cbt, did_onehop, start_time); + + /* Only count timeouts if network is live.. */ + if (!circuit_build_times_network_check_live(cbt)) { + return 0; + } + + circuit_build_times_add_time(cbt, CBT_BUILD_ABANDONED); + return 1; +} + +/** + * Update timeout counts to determine if we need to expire + * our build time history due to excessive timeouts. + */ +void +circuit_build_times_count_timeout(circuit_build_times_t *cbt, + int did_onehop) +{ + if (circuit_build_times_disabled()) { + cbt->close_ms = cbt->timeout_ms + = circuit_build_times_get_initial_timeout(); + return; + } + + circuit_build_times_network_timeout(cbt, did_onehop); + + /* If there are a ton of timeouts, we should reset + * the circuit build timeout. + */ + circuit_build_times_network_check_changed(cbt); +} + +/** + * Estimate a new timeout based on history and set our timeout + * variable accordingly. + */ +static int +circuit_build_times_set_timeout_worker(circuit_build_times_t *cbt) +{ + if (cbt->total_build_times < circuit_build_times_min_circs_to_observe()) { + return 0; + } + + if (!circuit_build_times_update_alpha(cbt)) + return 0; + + cbt->timeout_ms = circuit_build_times_calculate_timeout(cbt, + circuit_build_times_quantile_cutoff()); + + cbt->close_ms = circuit_build_times_calculate_timeout(cbt, + circuit_build_times_close_quantile()); + + /* Sometimes really fast guard nodes give us such a steep curve + * that this ends up being not that much greater than timeout_ms. + * Make it be at least 1 min to handle this case. */ + cbt->close_ms = MAX(cbt->close_ms, circuit_build_times_initial_timeout()); + + cbt->have_computed_timeout = 1; + return 1; +} + +/** + * Exposed function to compute a new timeout. Dispatches events and + * also filters out extremely high timeout values. + */ +void +circuit_build_times_set_timeout(circuit_build_times_t *cbt) +{ + long prev_timeout = tor_lround(cbt->timeout_ms/1000); + double timeout_rate; + + if (!circuit_build_times_set_timeout_worker(cbt)) + return; + + if (cbt->timeout_ms < circuit_build_times_min_timeout()) { + log_warn(LD_CIRC, "Set buildtimeout to low value %lfms. Setting to %dms", + cbt->timeout_ms, circuit_build_times_min_timeout()); + cbt->timeout_ms = circuit_build_times_min_timeout(); + if (cbt->close_ms < cbt->timeout_ms) { + /* This shouldn't happen because of MAX() in timeout_worker above, + * but doing it just in case */ + cbt->close_ms = circuit_build_times_initial_timeout(); + } + } + + control_event_buildtimeout_set(cbt, BUILDTIMEOUT_SET_EVENT_COMPUTED); + + timeout_rate = circuit_build_times_timeout_rate(cbt); + + if (prev_timeout > tor_lround(cbt->timeout_ms/1000)) { + log_notice(LD_CIRC, + "Based on %d circuit times, it looks like we don't need to " + "wait so long for circuits to finish. We will now assume a " + "circuit is too slow to use after waiting %ld seconds.", + cbt->total_build_times, + tor_lround(cbt->timeout_ms/1000)); + log_info(LD_CIRC, + "Circuit timeout data: %lfms, %lfms, Xm: %d, a: %lf, r: %lf", + cbt->timeout_ms, cbt->close_ms, cbt->Xm, cbt->alpha, + timeout_rate); + } else if (prev_timeout < tor_lround(cbt->timeout_ms/1000)) { + log_notice(LD_CIRC, + "Based on %d circuit times, it looks like we need to wait " + "longer for circuits to finish. We will now assume a " + "circuit is too slow to use after waiting %ld seconds.", + cbt->total_build_times, + tor_lround(cbt->timeout_ms/1000)); + log_info(LD_CIRC, + "Circuit timeout data: %lfms, %lfms, Xm: %d, a: %lf, r: %lf", + cbt->timeout_ms, cbt->close_ms, cbt->Xm, cbt->alpha, + timeout_rate); + } else { + log_info(LD_CIRC, + "Set circuit build timeout to %lds (%lfms, %lfms, Xm: %d, a: %lf," + " r: %lf) based on %d circuit times", + tor_lround(cbt->timeout_ms/1000), + cbt->timeout_ms, cbt->close_ms, cbt->Xm, cbt->alpha, timeout_rate, + cbt->total_build_times); + } +} + /** Iterate over values of circ_id, starting from conn-\>next_circ_id, * and with the high bit specified by conn-\>circ_id_type, until we get * a circ_id that is not in use by any other circuit on that conn. @@ -112,21 +1356,21 @@ circuit_list_path_impl(origin_circuit_t *circ, int verbose, int verbose_names) crypt_path_t *hop; smartlist_t *elements; const char *states[] = {"closed", "waiting for keys", "open"}; - char buf[128]; char *s; elements = smartlist_create(); if (verbose) { const char *nickname = build_state_get_exit_nickname(circ->build_state); - tor_snprintf(buf, sizeof(buf), "%s%s circ (length %d%s%s):", + char *cp; + tor_asprintf(&cp, "%s%s circ (length %d%s%s):", circ->build_state->is_internal ? "internal" : "exit", circ->build_state->need_uptime ? " (high-uptime)" : "", circ->build_state->desired_path_len, circ->_base.state == CIRCUIT_STATE_OPEN ? "" : ", exit ", circ->_base.state == CIRCUIT_STATE_OPEN ? "" : (nickname?nickname:"*unnamed*")); - smartlist_add(elements, tor_strdup(buf)); + smartlist_add(elements, cp); } hop = circ->cpath; @@ -148,8 +1392,7 @@ circuit_list_path_impl(origin_circuit_t *circ, int verbose, int verbose_names) router_get_verbose_nickname(elt, ri); } else if ((rs = router_get_consensus_status_by_id(id))) { routerstatus_get_verbose_nickname(elt, rs); - } else if (hop->extend_info->nickname && - is_legal_nickname(hop->extend_info->nickname)) { + } else if (is_legal_nickname(hop->extend_info->nickname)) { elt[0] = '$'; base16_encode(elt+1, HEX_DIGEST_LEN+1, id, DIGEST_LEN); elt[HEX_DIGEST_LEN+1]= '~'; @@ -217,7 +1460,7 @@ void circuit_log_path(int severity, unsigned int domain, origin_circuit_t *circ) { char *s = circuit_list_path(circ,1); - log(severity,domain,"%s",s); + tor_log(severity,domain,"%s",s); tor_free(s); } @@ -361,9 +1604,10 @@ circuit_handle_first_hop(origin_circuit_t *circ) if (!n_conn) { /* not currently connected in a useful way. */ - const char *name = firsthop->extend_info->nickname ? + const char *name = strlen(firsthop->extend_info->nickname) ? firsthop->extend_info->nickname : fmt_addr(&firsthop->extend_info->addr); - log_info(LD_CIRC, "Next router is %s: %s ", safe_str(name), msg?msg:"???"); + log_info(LD_CIRC, "Next router is %s: %s ", + safe_str_client(name), msg?msg:"???"); circ->_base.n_hop = extend_info_dup(firsthop->extend_info); if (should_launch) { @@ -536,7 +1780,7 @@ inform_testing_reachability(void) "CHECKING_REACHABILITY DIRADDRESS=%s:%d", me->address, me->dir_port); } - log(LOG_NOTICE, LD_OR, "Now checking whether ORPort %s:%d%s %s reachable... " + log_notice(LD_OR, "Now checking whether ORPort %s:%d%s %s reachable... " "(this may take up to %d minutes -- look for log " "messages indicating success)", me->address, me->or_port, @@ -642,6 +1886,31 @@ circuit_send_next_onion_skin(origin_circuit_t *circ) if (!hop) { /* done building the circuit. whew. */ circuit_set_state(TO_CIRCUIT(circ), CIRCUIT_STATE_OPEN); + if (!circ->build_state->onehop_tunnel) { + struct timeval end; + long timediff; + tor_gettimeofday(&end); + timediff = tv_mdiff(&circ->_base.highres_created, &end); + /* + * If the circuit build time is much greater than we would have cut + * it off at, we probably had a suspend event along this codepath, + * and we should discard the value. + */ + if (timediff < 0 || timediff > 2*circ_times.close_ms+1000) { + log_notice(LD_CIRC, "Strange value for circuit build time: %ldmsec. " + "Assuming clock jump.", timediff); + } else if (!circuit_build_times_disabled()) { + /* Don't count circuit times if the network was not live */ + if (circuit_build_times_network_check_live(&circ_times)) { + circuit_build_times_add_time(&circ_times, (build_time_t)timediff); + circuit_build_times_set_timeout(&circ_times); + } + + if (circ->_base.purpose != CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT) { + circuit_build_times_network_circ_success(&circ_times); + } + } + } log_info(LD_CIRC,"circuit built!"); circuit_reset_failure_count(0); if (circ->build_state->onehop_tunnel) @@ -650,7 +1919,7 @@ circuit_send_next_onion_skin(origin_circuit_t *circ) or_options_t *options = get_options(); has_completed_circuit=1; /* FFFF Log a count of known routers here */ - log(LOG_NOTICE, LD_GENERAL, + log_notice(LD_GENERAL, "Tor has successfully opened a circuit. " "Looks like client functionality is working."); control_event_bootstrap(BOOTSTRAP_STATUS_DONE, 0); @@ -662,6 +1931,10 @@ circuit_send_next_onion_skin(origin_circuit_t *circ) } circuit_rep_hist_note_result(circ); circuit_has_opened(circ); /* do other actions as necessary */ + + /* We're done with measurement circuits here. Just close them */ + if (circ->_base.purpose == CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT) + circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_FINISHED); return 0; } @@ -705,7 +1978,7 @@ void circuit_note_clock_jumped(int seconds_elapsed) { int severity = server_mode(get_options()) ? LOG_WARN : LOG_NOTICE; - log(severity, LD_GENERAL, "Your system clock just jumped %d seconds %s; " + tor_log(severity, LD_GENERAL, "Your system clock just jumped %d seconds %s; " "assuming established circuits no longer work.", seconds_elapsed >=0 ? seconds_elapsed : -seconds_elapsed, seconds_elapsed >=0 ? "forward" : "backward"); @@ -942,10 +2215,9 @@ circuit_finish_handshake(origin_circuit_t *circ, uint8_t reply_type, return -END_CIRC_REASON_TORPROTOCOL; } - if (hop->dh_handshake_state) { - crypto_dh_free(hop->dh_handshake_state); /* don't need it anymore */ - hop->dh_handshake_state = NULL; - } + crypto_dh_free(hop->dh_handshake_state); /* don't need it anymore */ + hop->dh_handshake_state = NULL; + memset(hop->fast_handshake_state, 0, sizeof(hop->fast_handshake_state)); if (circuit_init_cpath_crypto(hop, keys, 0)<0) { @@ -1084,7 +2356,7 @@ new_route_len(uint8_t purpose, extend_info_t *exit, tor_assert(routers); - routelen = 3; + routelen = DEFAULT_ROUTE_LEN; if (exit && purpose != CIRCUIT_PURPOSE_TESTING && purpose != CIRCUIT_PURPOSE_S_ESTABLISH_INTRO) @@ -1149,6 +2421,8 @@ circuit_all_predicted_ports_handled(time_t now, int *need_uptime, smartlist_t *LongLivedServices = get_options()->LongLivedPorts; tor_assert(need_uptime); tor_assert(need_capacity); + // Always predict need_capacity + *need_capacity = 1; enough = (smartlist_len(sl) == 0); for (i = 0; i < smartlist_len(sl); ++i) { port = smartlist_get(sl, i); @@ -1253,9 +2527,16 @@ choose_good_exit_server_general(routerlist_t *dir, int need_uptime, n_supported[i] = -1; continue; /* skip routers that are known to be down or bad exits */ } - if (router_is_unreliable(router, need_uptime, need_capacity, 0)) { + if (router_is_unreliable(router, need_uptime, need_capacity, 0) && + (!options->ExitNodes || + !routerset_contains_router(options->ExitNodes, router))) { + /* FFFF Someday, differentiate between a routerset that names + * routers, and a routerset that names countries, and only do this + * check if they've asked for specific exit relays. Or if the country + * they ask for is rare. Or something. */ n_supported[i] = -1; - continue; /* skip routers that are not suitable */ + continue; /* skip routers that are not suitable, unless we have + * ExitNodes set, in which case we asked for it */ } if (!(router->is_valid || options->_AllowInvalid & ALLOW_INVALID_EXIT)) { /* if it's invalid and we don't want it */ @@ -1280,7 +2561,7 @@ choose_good_exit_server_general(routerlist_t *dir, int need_uptime, { if (!ap_stream_wants_exit_attention(conn)) continue; /* Skip everything but APs in CIRCUIT_WAIT */ - if (connection_ap_can_use_exit(TO_EDGE_CONN(conn), router)) { + if (connection_ap_can_use_exit(TO_EDGE_CONN(conn), router, 1)) { ++n_supported[i]; // log_fn(LOG_DEBUG,"%s is supported. n_supported[%d] now %d.", // router->nickname, i, n_supported[i]); @@ -1322,7 +2603,8 @@ choose_good_exit_server_general(routerlist_t *dir, int need_uptime, routersets_get_disjunction(use, supporting, options->ExitNodes, options->_ExcludeExitNodesUnion, 1); - if (smartlist_len(use) == 0 && !options->StrictExitNodes) { + if (smartlist_len(use) == 0 && options->ExitNodes && + !options->StrictNodes) { /* give up on exitnodes and try again */ routersets_get_disjunction(use, supporting, NULL, options->_ExcludeExitNodesUnion, 1); } @@ -1334,7 +2616,7 @@ choose_good_exit_server_general(routerlist_t *dir, int need_uptime, * possibly support any of them. Choose a router at random that satisfies * at least one predicted exit port. */ - int try; + int attempt; smartlist_t *needed_ports, *supporting, *use; if (best_support == -1) { @@ -1347,19 +2629,20 @@ choose_good_exit_server_general(routerlist_t *dir, int need_uptime, tor_free(n_supported); return choose_good_exit_server_general(dir, 0, 0); } - log_notice(LD_CIRC, "All routers are down or won't exit -- choosing a " - "doomed exit at random."); + log_notice(LD_CIRC, "All routers are down or won't exit%s -- " + "choosing a doomed exit at random.", + options->_ExcludeExitNodesUnion ? " or are Excluded" : ""); } supporting = smartlist_create(); use = smartlist_create(); needed_ports = circuit_get_unhandled_ports(time(NULL)); - for (try = 0; try < 2; try++) { + for (attempt = 0; attempt < 2; attempt++) { /* try once to pick only from routers that satisfy a needed port, * then if there are none, pick from any that support exiting. */ for (i = 0; i < smartlist_len(dir->routers); i++) { router = smartlist_get(dir->routers, i); if (n_supported[i] != -1 && - (try || router_handles_some_port(router, needed_ports))) { + (attempt || router_handles_some_port(router, needed_ports))) { // log_fn(LOG_DEBUG,"Try %d: '%s' is a possibility.", // try, router->nickname); smartlist_add(supporting, router); @@ -1368,12 +2651,14 @@ choose_good_exit_server_general(routerlist_t *dir, int need_uptime, routersets_get_disjunction(use, supporting, options->ExitNodes, options->_ExcludeExitNodesUnion, 1); - if (smartlist_len(use) == 0 && !options->StrictExitNodes) { + if (smartlist_len(use) == 0 && options->ExitNodes && + !options->StrictNodes) { /* give up on exitnodes and try again */ routersets_get_disjunction(use, supporting, NULL, options->_ExcludeExitNodesUnion, 1); } - /* XXX sometimes the above results in null, when the requested - * exit node is down. we should pick it anyway. */ + /* FFF sometimes the above results in null, when the requested + * exit node is considered down by the consensus. we should pick + * it anyway, since the user asked for it. */ router = routerlist_sl_choose_by_bandwidth(use, WEIGHT_FOR_EXIT); if (router) break; @@ -1391,10 +2676,10 @@ choose_good_exit_server_general(routerlist_t *dir, int need_uptime, log_info(LD_CIRC, "Chose exit server '%s'", router->nickname); return router; } - if (options->StrictExitNodes) { + if (options->ExitNodes && options->StrictNodes) { log_warn(LD_CIRC, "No specified exit routers seem to be running, and " - "StrictExitNodes is set: can't choose an exit."); + "StrictNodes is set: can't choose an exit."); } return NULL; } @@ -1425,15 +2710,13 @@ choose_good_exit_server(uint8_t purpose, routerlist_t *dir, if (options->_AllowInvalid & ALLOW_INVALID_MIDDLE) flags |= CRN_ALLOW_INVALID; if (is_internal) /* pick it like a middle hop */ - return router_choose_random_node(NULL, NULL, - options->ExcludeNodes, flags); + return router_choose_random_node(NULL, options->ExcludeNodes, flags); else return choose_good_exit_server_general(dir,need_uptime,need_capacity); case CIRCUIT_PURPOSE_C_ESTABLISH_REND: if (options->_AllowInvalid & ALLOW_INVALID_RENDEZVOUS) flags |= CRN_ALLOW_INVALID; - return router_choose_random_node(NULL, NULL, - options->ExcludeNodes, flags); + return router_choose_random_node(NULL, options->ExcludeNodes, flags); } log_warn(LD_BUG,"Unhandled purpose %d", purpose); tor_fragile_assert(); @@ -1553,8 +2836,7 @@ circuit_append_new_exit(origin_circuit_t *circ, extend_info_t *exit) state = circ->build_state; tor_assert(state); - if (state->chosen_exit) - extend_info_free(state->chosen_exit); + extend_info_free(state->chosen_exit); state->chosen_exit = extend_info_dup(exit); ++circ->build_state->desired_path_len; @@ -1676,8 +2958,7 @@ choose_good_middle_server(uint8_t purpose, flags |= CRN_NEED_CAPACITY; if (options->_AllowInvalid & ALLOW_INVALID_MIDDLE) flags |= CRN_ALLOW_INVALID; - choice = router_choose_random_node(NULL, - excluded, options->ExcludeNodes, flags); + choice = router_choose_random_node(excluded, options->ExcludeNodes, flags); smartlist_free(excluded); return choice; } @@ -1741,11 +3022,7 @@ choose_good_entry_server(uint8_t purpose, cpath_build_state_t *state) if (options->_AllowInvalid & ALLOW_INVALID_ENTRY) flags |= CRN_ALLOW_INVALID; - choice = router_choose_random_node( - NULL, - excluded, - options->ExcludeNodes, - flags); + choice = router_choose_random_node(excluded, options->ExcludeNodes, flags); smartlist_free(excluded); return choice; } @@ -1866,9 +3143,9 @@ extend_info_from_router(routerinfo_t *r) void extend_info_free(extend_info_t *info) { - tor_assert(info); - if (info->onion_key) - crypto_free_pk_env(info->onion_key); + if (!info) + return; + crypto_free_pk_env(info->onion_key); tor_free(info); } @@ -1991,35 +3268,58 @@ entry_is_time_to_retry(entry_guard_t *e, time_t now) * - Listed as either up or never yet contacted; * - Present in the routerlist; * - Listed as 'stable' or 'fast' by the current dirserver consensus, - * if demanded by <b>need_uptime</b> or <b>need_capacity</b>; - * (This check is currently redundant with the Guard flag, but in - * the future that might change. Best to leave it in for now.) + * if demanded by <b>need_uptime</b> or <b>need_capacity</b> + * (unless it's a configured EntryNode); * - Allowed by our current ReachableORAddresses config option; and - * - Currently thought to be reachable by us (unless assume_reachable + * - Currently thought to be reachable by us (unless <b>assume_reachable</b> * is true). + * + * If the answer is no, set *<b>msg</b> to an explanation of why. */ static INLINE routerinfo_t * entry_is_live(entry_guard_t *e, int need_uptime, int need_capacity, - int assume_reachable) + int assume_reachable, const char **msg) { routerinfo_t *r; - if (e->bad_since) + or_options_t *options = get_options(); + tor_assert(msg); + + if (e->bad_since) { + *msg = "bad"; return NULL; + } /* no good if it's unreachable, unless assume_unreachable or can_retry. */ if (!assume_reachable && !e->can_retry && - e->unreachable_since && !entry_is_time_to_retry(e, time(NULL))) + e->unreachable_since && !entry_is_time_to_retry(e, time(NULL))) { + *msg = "unreachable"; return NULL; + } r = router_get_by_digest(e->identity); - if (!r) + if (!r) { + *msg = "no descriptor"; return NULL; - if (get_options()->UseBridges && r->purpose != ROUTER_PURPOSE_BRIDGE) + } + if (get_options()->UseBridges && r->purpose != ROUTER_PURPOSE_BRIDGE) { + *msg = "not a bridge"; return NULL; - if (!get_options()->UseBridges && r->purpose != ROUTER_PURPOSE_GENERAL) + } + if (!get_options()->UseBridges && r->purpose != ROUTER_PURPOSE_GENERAL) { + *msg = "not general-purpose"; return NULL; - if (router_is_unreliable(r, need_uptime, need_capacity, 0)) + } + if (options->EntryNodes && + routerset_contains_router(options->EntryNodes, r)) { + /* they asked for it, they get it */ + need_uptime = need_capacity = 0; + } + if (router_is_unreliable(r, need_uptime, need_capacity, 0)) { + *msg = "not fast/stable"; return NULL; - if (!fascist_firewall_allows_or(r)) + } + if (!fascist_firewall_allows_or(r)) { + *msg = "unreachable by config"; return NULL; + } return r; } @@ -2028,11 +3328,12 @@ static int num_live_entry_guards(void) { int n = 0; + const char *msg; if (! entry_guards) return 0; SMARTLIST_FOREACH(entry_guards, entry_guard_t *, entry, { - if (entry_is_live(entry, 0, 1, 0)) + if (entry_is_live(entry, 0, 1, 0, &msg)) ++n; }); return n; @@ -2056,16 +3357,21 @@ static void log_entry_guards(int severity) { smartlist_t *elements = smartlist_create(); - char buf[1024]; char *s; SMARTLIST_FOREACH(entry_guards, entry_guard_t *, e, { - tor_snprintf(buf, sizeof(buf), "%s (%s%s)", - e->nickname, - entry_is_live(e, 0, 1, 0) ? "up " : "down ", - e->made_contact ? "made-contact" : "never-contacted"); - smartlist_add(elements, tor_strdup(buf)); + const char *msg = NULL; + char *cp; + if (entry_is_live(e, 0, 1, 0, &msg)) + tor_asprintf(&cp, "%s (up %s)", + e->nickname, + e->made_contact ? "made-contact" : "never-contacted"); + else + tor_asprintf(&cp, "%s (%s, %s)", + e->nickname, msg, + e->made_contact ? "made-contact" : "never-contacted"); + smartlist_add(elements, cp); }); s = smartlist_join_strings(elements, ",", 0, NULL); @@ -2089,12 +3395,13 @@ control_event_guard_deferred(void) **/ #if 0 int n = 0; + const char *msg; or_options_t *options = get_options(); if (!entry_guards) return; SMARTLIST_FOREACH(entry_guards, entry_guard_t *, entry, { - if (entry_is_live(entry, 0, 1, 0)) { + if (entry_is_live(entry, 0, 1, 0, &msg)) { if (n++ == options->NumEntryGuards) { control_event_guard(entry->nickname, entry->identity, "DEFERRED"); return; @@ -2180,7 +3487,8 @@ pick_entry_guards(void) static void entry_guard_free(entry_guard_t *e) { - tor_assert(e); + if (!e) + return; tor_free(e->chosen_by_version); tor_free(e); } @@ -2298,10 +3606,13 @@ entry_guards_compute_status(void) int severity = LOG_DEBUG; or_options_t *options; digestmap_t *reasons; + if (! entry_guards) return; options = get_options(); + if (options->EntryNodes) /* reshuffle the entry guard list if needed */ + entry_nodes_should_be_added(); now = time(NULL); @@ -2328,13 +3639,16 @@ entry_guards_compute_status(void) if (changed) { SMARTLIST_FOREACH_BEGIN(entry_guards, entry_guard_t *, entry) { const char *reason = digestmap_get(reasons, entry->identity); - log_info(LD_CIRC, "Summary: Entry '%s' is %s, %s%s%s, and %s.", + const char *live_msg = ""; + routerinfo_t *r = entry_is_live(entry, 0, 1, 0, &live_msg); + log_info(LD_CIRC, "Summary: Entry '%s' is %s, %s%s%s, and %s%s.", entry->nickname, entry->unreachable_since ? "unreachable" : "reachable", entry->bad_since ? "unusable" : "usable", reason ? ", ": "", reason ? reason : "", - entry_is_live(entry, 0, 1, 0) ? "live" : "not live"); + r ? "live" : "not live / ", + r ? "" : live_msg); } SMARTLIST_FOREACH_END(entry); log_info(LD_CIRC, " (%d/%d entry guards are usable/new)", num_live_entry_guards(), smartlist_len(entry_guards)); @@ -2405,6 +3719,7 @@ entry_guard_register_connect_status(const char *digest, int succeeded, "Removing from the list. %d/%d entry guards usable/new.", entry->nickname, buf, num_live_entry_guards()-1, smartlist_len(entry_guards)-1); + control_event_guard(entry->nickname, entry->identity, "DROPPED"); entry_guard_free(entry); smartlist_del_keeporder(entry_guards, idx); log_entry_guards(LOG_INFO); @@ -2441,7 +3756,8 @@ entry_guard_register_connect_status(const char *digest, int succeeded, if (e == entry) break; if (e->made_contact) { - routerinfo_t *r = entry_is_live(e, 0, 1, 1); + const char *msg; + routerinfo_t *r = entry_is_live(e, 0, 1, 1, &msg); if (r && e->unreachable_since) { refuse_conn = 1; e->can_retry = 1; @@ -2472,7 +3788,8 @@ static int should_add_entry_nodes = 0; void entry_nodes_should_be_added(void) { - log_info(LD_CIRC, "New EntryNodes config option detected. Will use."); + log_info(LD_CIRC, "EntryNodes config option set. Putting configured " + "relays at the front of the entry guard list."); should_add_entry_nodes = 1; } @@ -2496,7 +3813,7 @@ entry_guards_prepend_from_config(void) return; } - if (options->EntryNodes) { + { char *string = routerset_to_string(options->EntryNodes); log_info(LD_CIRC,"Adding configured EntryNodes '%s'.", string); tor_free(string); @@ -2540,8 +3857,9 @@ entry_guards_prepend_from_config(void) SMARTLIST_FOREACH(entry_routers, routerinfo_t *, ri, { add_an_entry_guard(ri, 0); }); - /* Finally, the remaining EntryNodes, unless we're strict */ - if (options->StrictEntryNodes) { + /* Finally, the remaining previously configured guards that are not in + * EntryNodes, unless we're strict in which case we drop them */ + if (options->StrictNodes) { SMARTLIST_FOREACH(old_entry_guards_not_on_list, entry_guard_t *, e, entry_guard_free(e)); } else { @@ -2555,16 +3873,30 @@ entry_guards_prepend_from_config(void) entry_guards_changed(); } -/** Return 1 if we're fine adding arbitrary routers out of the - * directory to our entry guard list. Else return 0. */ +/** Return 0 if we're fine adding arbitrary routers out of the + * directory to our entry guard list, or return 1 if we have a + * list already and we'd prefer to stick to it. + */ int -entry_list_can_grow(or_options_t *options) +entry_list_is_constrained(or_options_t *options) { - if (options->StrictEntryNodes) - return 0; + if (options->EntryNodes) + return 1; if (options->UseBridges) - return 0; - return 1; + return 1; + return 0; +} + +/* Are we dead set against changing our entry guard list, or would we + * change it if it means keeping Tor usable? */ +static int +entry_list_is_totally_static(or_options_t *options) +{ + if (options->EntryNodes && options->StrictNodes) + return 1; + if (options->UseBridges) + return 1; + return 0; } /** Pick a live (up and listed) entry guard from entry_guards. If @@ -2582,7 +3914,7 @@ choose_random_entry(cpath_build_state_t *state) routerinfo_t *r = NULL; int need_uptime = state ? state->need_uptime : 0; int need_capacity = state ? state->need_capacity : 0; - int consider_exit_family = 0; + int preferred_min, consider_exit_family = 0; if (chosen_exit) { smartlist_add(exit_family, chosen_exit); @@ -2596,36 +3928,60 @@ choose_random_entry(cpath_build_state_t *state) if (should_add_entry_nodes) entry_guards_prepend_from_config(); - if (entry_list_can_grow(options) && - (! entry_guards || - smartlist_len(entry_guards) < options->NumEntryGuards)) + if (!entry_list_is_constrained(options) && + smartlist_len(entry_guards) < options->NumEntryGuards) pick_entry_guards(); retry: smartlist_clear(live_entry_guards); SMARTLIST_FOREACH(entry_guards, entry_guard_t *, entry, { - r = entry_is_live(entry, need_uptime, need_capacity, 0); - if (r && (!consider_exit_family || !smartlist_isin(exit_family, r))) { - smartlist_add(live_entry_guards, r); - if (!entry->made_contact) { - /* Always start with the first not-yet-contacted entry - * guard. Otherwise we might add several new ones, pick - * the second new one, and now we've expanded our entry - * guard list without needing to. */ - goto choose_and_finish; + const char *msg; + r = entry_is_live(entry, need_uptime, need_capacity, 0, &msg); + if (!r) + continue; /* down, no point */ + if (consider_exit_family && smartlist_isin(exit_family, r)) + continue; /* avoid relays that are family members of our exit */ + if (options->EntryNodes && + !routerset_contains_router(options->EntryNodes, r)) { + /* We've come to the end of our preferred entry nodes. */ + if (smartlist_len(live_entry_guards)) + goto choose_and_finish; /* only choose from the ones we like */ + if (options->StrictNodes) { + /* in theory this case should never happen, since + * entry_guards_prepend_from_config() drops unwanted relays */ + tor_fragile_assert(); + } else { + log_info(LD_CIRC, + "No relays from EntryNodes available. Using others."); } - if (smartlist_len(live_entry_guards) >= options->NumEntryGuards) - break; /* we have enough */ } + smartlist_add(live_entry_guards, r); + if (!entry->made_contact) { + /* Always start with the first not-yet-contacted entry + * guard. Otherwise we might add several new ones, pick + * the second new one, and now we've expanded our entry + * guard list without needing to. */ + goto choose_and_finish; + } + if (smartlist_len(live_entry_guards) >= options->NumEntryGuards) + break; /* we have enough */ }); - /* Try to have at least 2 choices available. This way we don't - * get stuck with a single live-but-crummy entry and just keep - * using him. - * (We might get 2 live-but-crummy entry guards, but so be it.) */ - if (smartlist_len(live_entry_guards) < 2) { - if (entry_list_can_grow(options)) { + if (entry_list_is_constrained(options)) { + /* If we prefer the entry nodes we've got, and we have at least + * one choice, that's great. Use it. */ + preferred_min = 1; + } else { + /* Try to have at least 2 choices available. This way we don't + * get stuck with a single live-but-crummy entry and just keep + * using him. + * (We might get 2 live-but-crummy entry guards, but so be it.) */ + preferred_min = 2; + } + + if (smartlist_len(live_entry_guards) < preferred_min) { + if (!entry_list_is_totally_static(options)) { /* still no? try adding a new entry then */ /* XXX if guard doesn't imply fast and stable, then we need * to tell add_an_entry_guard below what we want, or it might @@ -2650,7 +4006,7 @@ choose_random_entry(cpath_build_state_t *state) need_capacity = 0; goto retry; } - if (!r && !entry_list_can_grow(options) && consider_exit_family) { + if (!r && entry_list_is_constrained(options) && consider_exit_family) { /* still no? if we're using bridges or have strictentrynodes * set, and our chosen exit is in the same family as all our * bridges/entry guards, then be flexible about families. */ @@ -2661,15 +4017,15 @@ choose_random_entry(cpath_build_state_t *state) } choose_and_finish: - if (entry_list_can_grow(options)) { + if (entry_list_is_constrained(options)) { + /* We need to weight by bandwidth, because our bridges or entryguards + * were not already selected proportional to their bandwidth. */ + r = routerlist_sl_choose_by_bandwidth(live_entry_guards, WEIGHT_FOR_GUARD); + } else { /* We choose uniformly at random here, because choose_good_entry_server() * already weights its choices by bandwidth, so we don't want to * *double*-weight our guard selection. */ r = smartlist_choose(live_entry_guards); - } else { - /* We need to weight by bandwidth, because our bridges or entryguards - * were not already selected proportional to their bandwidth. */ - r = routerlist_sl_choose_by_bandwidth(live_entry_guards, WEIGHT_FOR_GUARD); } smartlist_free(live_entry_guards); smartlist_free(exit_family); @@ -2902,9 +4258,11 @@ entry_guards_update_state(or_state_t *state) * */ int getinfo_helper_entry_guards(control_connection_t *conn, - const char *question, char **answer) + const char *question, char **answer, + const char **errmsg) { - int use_long_names = conn->use_long_names; + (void) conn; + (void) errmsg; if (!strcmp(question,"entry-guards") || !strcmp(question,"helper-nodes")) { @@ -2913,12 +4271,13 @@ getinfo_helper_entry_guards(control_connection_t *conn, char nbuf[MAX_VERBOSE_NICKNAME_LEN+1]; if (!entry_guards) entry_guards = smartlist_create(); - SMARTLIST_FOREACH(entry_guards, entry_guard_t *, e, - { + SMARTLIST_FOREACH_BEGIN(entry_guards, entry_guard_t *, e) { size_t len = MAX_VERBOSE_NICKNAME_LEN+ISO_TIME_LEN+32; char *c = tor_malloc(len); const char *status = NULL; time_t when = 0; + routerinfo_t *ri; + if (!e->made_contact) { status = "never-connected"; } else if (e->bad_since) { @@ -2927,19 +4286,17 @@ getinfo_helper_entry_guards(control_connection_t *conn, } else { status = "up"; } - if (use_long_names) { - routerinfo_t *ri = router_get_by_digest(e->identity); - if (ri) { - router_get_verbose_nickname(nbuf, ri); - } else { - nbuf[0] = '$'; - base16_encode(nbuf+1, sizeof(nbuf)-1, e->identity, DIGEST_LEN); - /* e->nickname field is not very reliable if we don't know about - * this router any longer; don't include it. */ - } + + ri = router_get_by_digest(e->identity); + if (ri) { + router_get_verbose_nickname(nbuf, ri); } else { - base16_encode(nbuf, sizeof(nbuf), e->identity, DIGEST_LEN); + nbuf[0] = '$'; + base16_encode(nbuf+1, sizeof(nbuf)-1, e->identity, DIGEST_LEN); + /* e->nickname field is not very reliable if we don't know about + * this router any longer; don't include it. */ } + if (when) { format_iso_time(tbuf, when); tor_snprintf(c, len, "%s %s %s\n", nbuf, status, tbuf); @@ -2947,7 +4304,7 @@ getinfo_helper_entry_guards(control_connection_t *conn, tor_snprintf(c, len, "%s %s\n", nbuf, status); } smartlist_add(sl, c); - }); + } SMARTLIST_FOREACH_END(e); *answer = smartlist_join_strings(sl, "", 0, NULL); SMARTLIST_FOREACH(sl, char *, c, tor_free(c)); smartlist_free(sl); @@ -2988,29 +4345,56 @@ clear_bridge_list(void) * (either by comparing keys if possible, else by comparing addr/port). * Else return NULL. */ static bridge_info_t * -routerinfo_get_configured_bridge(routerinfo_t *ri) +get_configured_bridge_by_addr_port_digest(tor_addr_t *addr, uint16_t port, + const char *digest) { if (!bridge_list) return NULL; SMARTLIST_FOREACH_BEGIN(bridge_list, bridge_info_t *, bridge) { if (tor_digest_is_zero(bridge->identity) && - tor_addr_eq_ipv4h(&bridge->addr, ri->addr) && - bridge->port == ri->or_port) + !tor_addr_compare(&bridge->addr, addr, CMP_EXACT) && + bridge->port == port) return bridge; - if (!memcmp(bridge->identity, ri->cache_info.identity_digest, - DIGEST_LEN)) + if (!memcmp(bridge->identity, digest, DIGEST_LEN)) return bridge; } SMARTLIST_FOREACH_END(bridge); return NULL; } +/** Wrapper around get_configured_bridge_by_addr_port_digest() to look + * it up via router descriptor <b>ri</b>. */ +static bridge_info_t * +get_configured_bridge_by_routerinfo(routerinfo_t *ri) +{ + tor_addr_t addr; + tor_addr_from_ipv4h(&addr, ri->addr); + return get_configured_bridge_by_addr_port_digest(&addr, + ri->or_port, ri->cache_info.identity_digest); +} + /** Return 1 if <b>ri</b> is one of our known bridges, else 0. */ int routerinfo_is_a_configured_bridge(routerinfo_t *ri) { - return routerinfo_get_configured_bridge(ri) ? 1 : 0; + return get_configured_bridge_by_routerinfo(ri) ? 1 : 0; +} + +/** We made a connection to a router at <b>addr</b>:<b>port</b> + * without knowing its digest. Its digest turned out to be <b>digest</b>. + * If it was a bridge, and we still don't know its digest, record it. + */ +void +learned_router_identity(tor_addr_t *addr, uint16_t port, const char *digest) +{ + bridge_info_t *bridge = + get_configured_bridge_by_addr_port_digest(addr, port, digest); + if (bridge && tor_digest_is_zero(bridge->identity)) { + memcpy(bridge->identity, digest, DIGEST_LEN); + log_notice(LD_DIR, "Learned fingerprint %s for bridge %s:%d", + hex_str(digest, DIGEST_LEN), fmt_addr(addr), port); + } } /** Remember a new bridge at <b>addr</b>:<b>port</b>. If <b>digest</b> @@ -3150,7 +4534,7 @@ learned_bridge_descriptor(routerinfo_t *ri, int from_cache) tor_assert(ri->purpose == ROUTER_PURPOSE_BRIDGE); if (get_options()->UseBridges) { int first = !any_bridge_descriptors_known(); - bridge_info_t *bridge = routerinfo_get_configured_bridge(ri); + bridge_info_t *bridge = get_configured_bridge_by_routerinfo(ri); time_t now = time(NULL); ri->is_running = 1; diff --git a/src/or/circuitbuild.h b/src/or/circuitbuild.h new file mode 100644 index 0000000000..3a02f04202 --- /dev/null +++ b/src/or/circuitbuild.h @@ -0,0 +1,125 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2010, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file circuitbuild.h + * \brief Header file for circuitbuild.c. + **/ + +#ifndef _TOR_CIRCUITBUILD_H +#define _TOR_CIRCUITBUILD_H + +char *circuit_list_path(origin_circuit_t *circ, int verbose); +char *circuit_list_path_for_controller(origin_circuit_t *circ); +void circuit_log_path(int severity, unsigned int domain, + origin_circuit_t *circ); +void circuit_rep_hist_note_result(origin_circuit_t *circ); +origin_circuit_t *origin_circuit_init(uint8_t purpose, int flags); +origin_circuit_t *circuit_establish_circuit(uint8_t purpose, + extend_info_t *exit, + int flags); +int circuit_handle_first_hop(origin_circuit_t *circ); +void circuit_n_conn_done(or_connection_t *or_conn, int status); +int inform_testing_reachability(void); +int circuit_send_next_onion_skin(origin_circuit_t *circ); +void circuit_note_clock_jumped(int seconds_elapsed); +int circuit_extend(cell_t *cell, circuit_t *circ); +int circuit_init_cpath_crypto(crypt_path_t *cpath, const char *key_data, + int reverse); +int circuit_finish_handshake(origin_circuit_t *circ, uint8_t cell_type, + const char *reply); +int circuit_truncated(origin_circuit_t *circ, crypt_path_t *layer); +int onionskin_answer(or_circuit_t *circ, uint8_t cell_type, + const char *payload, const char *keys); +int circuit_all_predicted_ports_handled(time_t now, int *need_uptime, + int *need_capacity); + +int circuit_append_new_exit(origin_circuit_t *circ, extend_info_t *info); +int circuit_extend_to_new_exit(origin_circuit_t *circ, extend_info_t *info); +void onion_append_to_cpath(crypt_path_t **head_ptr, crypt_path_t *new_hop); +extend_info_t *extend_info_alloc(const char *nickname, const char *digest, + crypto_pk_env_t *onion_key, + const tor_addr_t *addr, uint16_t port); +extend_info_t *extend_info_from_router(routerinfo_t *r); +extend_info_t *extend_info_dup(extend_info_t *info); +void extend_info_free(extend_info_t *info); +routerinfo_t *build_state_get_exit_router(cpath_build_state_t *state); +const char *build_state_get_exit_nickname(cpath_build_state_t *state); + +void entry_guards_compute_status(void); +int entry_guard_register_connect_status(const char *digest, int succeeded, + int mark_relay_status, time_t now); +void entry_nodes_should_be_added(void); +int entry_list_is_constrained(or_options_t *options); +routerinfo_t *choose_random_entry(cpath_build_state_t *state); +int entry_guards_parse_state(or_state_t *state, int set, char **msg); +void entry_guards_update_state(or_state_t *state); +int getinfo_helper_entry_guards(control_connection_t *conn, + const char *question, char **answer, + const char **errmsg); + +void clear_bridge_list(void); +int routerinfo_is_a_configured_bridge(routerinfo_t *ri); +void +learned_router_identity(tor_addr_t *addr, uint16_t port, const char *digest); +void bridge_add_from_config(const tor_addr_t *addr, uint16_t port, + char *digest); +void retry_bridge_descriptor_fetch_directly(const char *digest); +void fetch_bridge_descriptors(time_t now); +void learned_bridge_descriptor(routerinfo_t *ri, int from_cache); +int any_bridge_descriptors_known(void); +int any_pending_bridge_descriptor_fetches(void); +int bridges_known_but_down(void); +void bridges_retry_all(void); + +void entry_guards_free_all(void); + +extern circuit_build_times_t circ_times; +void circuit_build_times_update_state(circuit_build_times_t *cbt, + or_state_t *state); +int circuit_build_times_parse_state(circuit_build_times_t *cbt, + or_state_t *state, char **msg); +void circuit_build_times_count_timeout(circuit_build_times_t *cbt, + int did_onehop); +int circuit_build_times_count_close(circuit_build_times_t *cbt, + int did_onehop, time_t start_time); +void circuit_build_times_set_timeout(circuit_build_times_t *cbt); +int circuit_build_times_add_time(circuit_build_times_t *cbt, + build_time_t time); +int circuit_build_times_needs_circuits(circuit_build_times_t *cbt); + +int circuit_build_times_needs_circuits_now(circuit_build_times_t *cbt); +void circuit_build_times_init(circuit_build_times_t *cbt); +void circuit_build_times_new_consensus_params(circuit_build_times_t *cbt, + networkstatus_t *ns); +double circuit_build_times_timeout_rate(const circuit_build_times_t *cbt); +double circuit_build_times_close_rate(const circuit_build_times_t *cbt); + +#ifdef CIRCUIT_PRIVATE +double circuit_build_times_calculate_timeout(circuit_build_times_t *cbt, + double quantile); +build_time_t circuit_build_times_generate_sample(circuit_build_times_t *cbt, + double q_lo, double q_hi); +void circuit_build_times_initial_alpha(circuit_build_times_t *cbt, + double quantile, double time_ms); +int circuit_build_times_update_alpha(circuit_build_times_t *cbt); +double circuit_build_times_cdf(circuit_build_times_t *cbt, double x); +void circuit_build_times_add_timeout_worker(circuit_build_times_t *cbt, + double quantile_cutoff); +void circuitbuild_running_unit_tests(void); +void circuit_build_times_reset(circuit_build_times_t *cbt); + +/* Network liveness functions */ +int circuit_build_times_network_check_changed(circuit_build_times_t *cbt); +#endif + +/* Network liveness functions */ +void circuit_build_times_network_is_live(circuit_build_times_t *cbt); +int circuit_build_times_network_check_live(circuit_build_times_t *cbt); +void circuit_build_times_network_circ_success(circuit_build_times_t *cbt); + +#endif + diff --git a/src/or/circuitlist.c b/src/or/circuitlist.c index b84d7f7623..c581365f8b 100644 --- a/src/or/circuitlist.c +++ b/src/or/circuitlist.c @@ -10,6 +10,21 @@ **/ #include "or.h" +#include "circuitbuild.h" +#include "circuitlist.h" +#include "circuituse.h" +#include "connection.h" +#include "config.h" +#include "connection_edge.h" +#include "connection_or.h" +#include "control.h" +#include "networkstatus.h" +#include "onion.h" +#include "relay.h" +#include "rendclient.h" +#include "rendcommon.h" +#include "rephist.h" +#include "routerlist.h" #include "ht.h" /********* START VARIABLES **********/ @@ -352,6 +367,8 @@ circuit_purpose_to_controller_string(uint8_t purpose) case CIRCUIT_PURPOSE_TESTING: return "TESTING"; + case CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT: + return "EXPIRED"; case CIRCUIT_PURPOSE_CONTROLLER: return "CONTROLLER"; @@ -380,10 +397,17 @@ static void init_circuit_base(circuit_t *circ) { circ->timestamp_created = time(NULL); + tor_gettimeofday(&circ->highres_created); circ->package_window = circuit_initial_package_window(); circ->deliver_window = CIRCWINDOW_START; + /* Initialize the cell_ewma_t structure */ + circ->n_cell_ewma.last_adjusted_tick = cell_ewma_get_tick(); + circ->n_cell_ewma.cell_count = 0.0; + circ->n_cell_ewma.heap_index = -1; + circ->n_cell_ewma.is_for_p_conn = 0; + circuit_add(circ); } @@ -408,6 +432,8 @@ origin_circuit_new(void) init_circuit_base(TO_CIRCUIT(circ)); + circ_times.last_circ_at = approx_time(); + return circ; } @@ -429,6 +455,16 @@ or_circuit_new(circid_t p_circ_id, or_connection_t *p_conn) init_circuit_base(TO_CIRCUIT(circ)); + /* Initialize the cell_ewma_t structure */ + + /* Initialize the cell counts to 0 */ + circ->p_cell_ewma.cell_count = 0.0; + circ->p_cell_ewma.last_adjusted_tick = cell_ewma_get_tick(); + circ->p_cell_ewma.is_for_p_conn = 1; + + /* It's not in any heap yet. */ + circ->p_cell_ewma.heap_index = -1; + return circ; } @@ -439,39 +475,37 @@ circuit_free(circuit_t *circ) { void *mem; size_t memlen; - tor_assert(circ); + if (!circ) + return; + if (CIRCUIT_IS_ORIGIN(circ)) { origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ); mem = ocirc; memlen = sizeof(origin_circuit_t); tor_assert(circ->magic == ORIGIN_CIRCUIT_MAGIC); if (ocirc->build_state) { - if (ocirc->build_state->chosen_exit) extend_info_free(ocirc->build_state->chosen_exit); - if (ocirc->build_state->pending_final_cpath) circuit_free_cpath_node(ocirc->build_state->pending_final_cpath); } tor_free(ocirc->build_state); circuit_free_cpath(ocirc->cpath); - if (ocirc->intro_key) - crypto_free_pk_env(ocirc->intro_key); - if (ocirc->rend_data) - rend_data_free(ocirc->rend_data); + + crypto_free_pk_env(ocirc->intro_key); + rend_data_free(ocirc->rend_data); } else { or_circuit_t *ocirc = TO_OR_CIRCUIT(circ); + /* Remember cell statistics for this circuit before deallocating. */ + if (get_options()->CellStatistics) + rep_hist_buffer_stats_add_circ(circ, time(NULL)); mem = ocirc; memlen = sizeof(or_circuit_t); tor_assert(circ->magic == OR_CIRCUIT_MAGIC); - if (ocirc->p_crypto) - crypto_free_cipher_env(ocirc->p_crypto); - if (ocirc->p_digest) - crypto_free_digest_env(ocirc->p_digest); - if (ocirc->n_crypto) - crypto_free_cipher_env(ocirc->n_crypto); - if (ocirc->n_digest) - crypto_free_digest_env(ocirc->n_digest); + crypto_free_cipher_env(ocirc->p_crypto); + crypto_free_digest_env(ocirc->p_digest); + crypto_free_cipher_env(ocirc->n_crypto); + crypto_free_digest_env(ocirc->n_digest); if (ocirc->rend_splice) { or_circuit_t *other = ocirc->rend_splice; @@ -487,8 +521,7 @@ circuit_free(circuit_t *circ) cell_queue_clear(ô->p_conn_cells); } - if (circ->n_hop) - extend_info_free(circ->n_hop); + extend_info_free(circ->n_hop); tor_free(circ->n_conn_onionskin); /* Remove from map. */ @@ -498,7 +531,7 @@ circuit_free(circuit_t *circ) * "active" checks will be violated. */ cell_queue_clear(&circ->n_conn_cells); - memset(circ, 0xAA, memlen); /* poison memory */ + memset(mem, 0xAA, memlen); /* poison memory */ tor_free(mem); } @@ -541,10 +574,10 @@ circuit_free_all(void) circuit_free(global_circuitlist); global_circuitlist = next; } - if (circuits_pending_or_conns) { - smartlist_free(circuits_pending_or_conns); - circuits_pending_or_conns = NULL; - } + + smartlist_free(circuits_pending_or_conns); + circuits_pending_or_conns = NULL; + HT_CLEAR(orconn_circid_map, &orconn_circid_circuit_map); } @@ -552,18 +585,15 @@ circuit_free_all(void) static void circuit_free_cpath_node(crypt_path_t *victim) { - if (victim->f_crypto) - crypto_free_cipher_env(victim->f_crypto); - if (victim->b_crypto) - crypto_free_cipher_env(victim->b_crypto); - if (victim->f_digest) - crypto_free_digest_env(victim->f_digest); - if (victim->b_digest) - crypto_free_digest_env(victim->b_digest); - if (victim->dh_handshake_state) - crypto_dh_free(victim->dh_handshake_state); - if (victim->extend_info) - extend_info_free(victim->extend_info); + if (!victim) + return; + + crypto_free_cipher_env(victim->f_crypto); + crypto_free_cipher_env(victim->b_crypto); + crypto_free_digest_env(victim->f_digest); + crypto_free_digest_env(victim->b_digest); + crypto_dh_free(victim->dh_handshake_state); + extend_info_free(victim->extend_info); memset(victim, 0xBB, sizeof(crypt_path_t)); /* poison memory */ tor_free(victim); @@ -892,6 +922,10 @@ circuit_find_to_cannibalize(uint8_t purpose, extend_info_t *info, int need_capacity = (flags & CIRCLAUNCH_NEED_CAPACITY) != 0; int internal = (flags & CIRCLAUNCH_IS_INTERNAL) != 0; + /* Make sure we're not trying to create a onehop circ by + * cannibalization. */ + tor_assert(!(flags & CIRCLAUNCH_ONEHOP_TUNNEL)); + log_debug(LD_CIRC, "Hunting for a circ to cannibalize: purpose %d, uptime %d, " "capacity %d, internal %d", @@ -907,7 +941,8 @@ circuit_find_to_cannibalize(uint8_t purpose, extend_info_t *info, if ((!need_uptime || circ->build_state->need_uptime) && (!need_capacity || circ->build_state->need_capacity) && (internal == circ->build_state->is_internal) && - circ->remaining_relay_early_cells) { + circ->remaining_relay_early_cells && + !circ->build_state->onehop_tunnel) { if (info) { /* need to make sure we don't duplicate hops */ crypt_path_t *hop = circ->cpath; @@ -1083,9 +1118,9 @@ _circuit_mark_for_close(circuit_t *circ, int reason, int line, tor_assert(ocirc->rend_data); /* treat this like getting a nack from it */ log_info(LD_REND, "Failed intro circ %s to %s (awaiting ack). " - "Removing from descriptor.", - safe_str(ocirc->rend_data->onion_address), - safe_str(build_state_get_exit_nickname(ocirc->build_state))); + "Removing from descriptor.", + safe_str_client(ocirc->rend_data->onion_address), + safe_str_client(build_state_get_exit_nickname(ocirc->build_state))); rend_client_remove_intro_point(ocirc->build_state->chosen_exit, ocirc->rend_data); } @@ -1236,11 +1271,6 @@ assert_circuit_ok(const circuit_t *c) tor_assert(c == c2); } } -#if 0 /* false now that rendezvous exits are attached to p_streams */ - if (origin_circ) - for (conn = origin_circ->p_streams; conn; conn = conn->next_stream) - tor_assert(conn->_base.type == CONN_TYPE_AP); -#endif if (or_circ) for (conn = or_circ->n_streams; conn; conn = conn->next_stream) tor_assert(conn->_base.type == CONN_TYPE_EXIT); diff --git a/src/or/circuitlist.h b/src/or/circuitlist.h new file mode 100644 index 0000000000..caca614c8c --- /dev/null +++ b/src/or/circuitlist.h @@ -0,0 +1,60 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2010, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file circuitlist.h + * \brief Header file for circuitlist.c. + **/ + +#ifndef _TOR_CIRCUITLIST_H +#define _TOR_CIRCUITLIST_H + +circuit_t * _circuit_get_global_list(void); +const char *circuit_state_to_string(int state); +const char *circuit_purpose_to_controller_string(uint8_t purpose); +void circuit_dump_by_conn(connection_t *conn, int severity); +void circuit_set_p_circid_orconn(or_circuit_t *circ, circid_t id, + or_connection_t *conn); +void circuit_set_n_circid_orconn(circuit_t *circ, circid_t id, + or_connection_t *conn); +void circuit_set_state(circuit_t *circ, uint8_t state); +void circuit_close_all_marked(void); +int32_t circuit_initial_package_window(void); +origin_circuit_t *origin_circuit_new(void); +or_circuit_t *or_circuit_new(circid_t p_circ_id, or_connection_t *p_conn); +circuit_t *circuit_get_by_circid_orconn(circid_t circ_id, + or_connection_t *conn); +int circuit_id_in_use_on_orconn(circid_t circ_id, or_connection_t *conn); +circuit_t *circuit_get_by_edge_conn(edge_connection_t *conn); +void circuit_unlink_all_from_or_conn(or_connection_t *conn, int reason); +origin_circuit_t *circuit_get_by_global_id(uint32_t id); +origin_circuit_t *circuit_get_by_rend_query_and_purpose(const char *rend_query, + uint8_t purpose); +origin_circuit_t *circuit_get_next_by_pk_and_purpose(origin_circuit_t *start, + const char *digest, uint8_t purpose); +or_circuit_t *circuit_get_rendezvous(const char *cookie); +or_circuit_t *circuit_get_intro_point(const char *digest); +origin_circuit_t *circuit_find_to_cannibalize(uint8_t purpose, + extend_info_t *info, int flags); +void circuit_mark_all_unused_circs(void); +void circuit_expire_all_dirty_circs(void); +void _circuit_mark_for_close(circuit_t *circ, int reason, + int line, const char *file); +int circuit_get_cpath_len(origin_circuit_t *circ); +crypt_path_t *circuit_get_cpath_hop(origin_circuit_t *circ, int hopnum); +void circuit_get_all_pending_on_or_conn(smartlist_t *out, + or_connection_t *or_conn); +int circuit_count_pending_on_or_conn(or_connection_t *or_conn); + +#define circuit_mark_for_close(c, reason) \ + _circuit_mark_for_close((c), (reason), __LINE__, _SHORT_FILE_) + +void assert_cpath_layer_ok(const crypt_path_t *cp); +void assert_circuit_ok(const circuit_t *c); +void circuit_free_all(void); + +#endif + diff --git a/src/or/circuituse.c b/src/or/circuituse.c index 827c4bbf2b..e9335b18d6 100644 --- a/src/or/circuituse.c +++ b/src/or/circuituse.c @@ -10,6 +10,20 @@ **/ #include "or.h" +#include "circuitbuild.h" +#include "circuitlist.h" +#include "circuituse.h" +#include "config.h" +#include "connection.h" +#include "connection_edge.h" +#include "control.h" +#include "policies.h" +#include "rendclient.h" +#include "rendcommon.h" +#include "rendservice.h" +#include "rephist.h" +#include "router.h" +#include "routerlist.h" /********* START VARIABLES **********/ @@ -20,6 +34,8 @@ extern circuit_t *global_circuitlist; /* from circuitlist.c */ static void circuit_expire_old_circuits_clientside(time_t now); static void circuit_increment_failure_count(void); +long int lround(double x); + /** Return 1 if <b>circ</b> could be returned by circuit_get_best(). * Else return 0. */ @@ -113,7 +129,7 @@ circuit_is_acceptable(circuit_t *circ, edge_connection_t *conn, return 0; } } - if (exitrouter && !connection_ap_can_use_exit(conn, exitrouter)) { + if (exitrouter && !connection_ap_can_use_exit(conn, exitrouter, 0)) { /* can't exit from this router */ return 0; } @@ -263,16 +279,19 @@ circuit_conforms_to_options(const origin_circuit_t *circ, void circuit_expire_building(time_t now) { - circuit_t *victim, *circ = global_circuitlist; - time_t general_cutoff = now - get_options()->CircuitBuildTimeout; - time_t begindir_cutoff = now - get_options()->CircuitBuildTimeout/2; + circuit_t *victim, *next_circ = global_circuitlist; + /* circ_times.timeout is BUILD_TIMEOUT_INITIAL_VALUE if we haven't + * decided on a customized one yet */ + time_t general_cutoff = now - lround(circ_times.timeout_ms/1000); + time_t begindir_cutoff = now - lround(circ_times.timeout_ms/2000); + time_t close_cutoff = now - lround(circ_times.close_ms/1000); time_t introcirc_cutoff = begindir_cutoff; cpath_build_state_t *build_state; - while (circ) { + while (next_circ) { time_t cutoff; - victim = circ; - circ = circ->next; + victim = next_circ; + next_circ = next_circ->next; if (!CIRCUIT_IS_ORIGIN(victim) || /* didn't originate here */ victim->marked_for_close) /* don't mess with marked circs */ continue; @@ -282,8 +301,11 @@ circuit_expire_building(time_t now) cutoff = begindir_cutoff; else if (victim->purpose == CIRCUIT_PURPOSE_C_INTRODUCING) cutoff = introcirc_cutoff; + else if (victim->purpose == CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT) + cutoff = close_cutoff; else cutoff = general_cutoff; + if (victim->timestamp_created > cutoff) continue; /* it's still young, leave it alone */ @@ -343,6 +365,43 @@ circuit_expire_building(time_t now) continue; break; } + } else { /* circuit not open, consider recording failure as timeout */ + int first_hop_succeeded = TO_ORIGIN_CIRCUIT(victim)->cpath && + TO_ORIGIN_CIRCUIT(victim)->cpath->state == CPATH_STATE_OPEN; + + if (TO_ORIGIN_CIRCUIT(victim)->p_streams != NULL) { + log_warn(LD_BUG, "Circuit %d (purpose %d) has timed out, " + "yet has attached streams!", + TO_ORIGIN_CIRCUIT(victim)->global_identifier, + victim->purpose); + tor_fragile_assert(); + continue; + } + + /* circuits are allowed to last longer for measurement. + * Switch their purpose and wait. */ + if (victim->purpose != CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT) { + victim->purpose = CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT; + circuit_build_times_count_timeout(&circ_times, + first_hop_succeeded); + continue; + } + + /* + * If the circuit build time is much greater than we would have cut + * it off at, we probably had a suspend event along this codepath, + * and we should discard the value. + */ + if (now - victim->timestamp_created > 2*circ_times.close_ms/1000+1) { + log_notice(LD_CIRC, + "Extremely large value for circuit build timeout: %lds. " + "Assuming clock jump.", + (long)(now - victim->timestamp_created)); + } else if (circuit_build_times_count_close(&circ_times, + first_hop_succeeded, + victim->timestamp_created)) { + circuit_build_times_set_timeout(&circ_times); + } } if (victim->n_conn) @@ -414,7 +473,7 @@ circuit_stream_is_being_handled(edge_connection_t *conn, if (exitrouter && (!need_uptime || build_state->need_uptime)) { int ok; if (conn) { - ok = connection_ap_can_use_exit(conn, exitrouter); + ok = connection_ap_can_use_exit(conn, exitrouter, 0); } else { addr_policy_result_t r = compare_addr_to_addr_policy( 0, port, exitrouter->exit_policy); @@ -431,11 +490,11 @@ circuit_stream_is_being_handled(edge_connection_t *conn, } /** Don't keep more than this many unused open circuits around. */ -#define MAX_UNUSED_OPEN_CIRCUITS 12 +#define MAX_UNUSED_OPEN_CIRCUITS 14 /** Figure out how many circuits we have open that are clean. Make * sure it's enough for all the upcoming behaviors we predict we'll have. - * But if we have too many, close the not-so-useful ones. + * But put an upper bound on the total number of circuits. */ static void circuit_predict_and_launch_new(void) @@ -517,6 +576,19 @@ circuit_predict_and_launch_new(void) circuit_launch_by_router(CIRCUIT_PURPOSE_C_GENERAL, NULL, flags); return; } + + /* Finally, check to see if we still need more circuits to learn + * a good build timeout. But if we're close to our max number we + * want, don't do another -- we want to leave a few slots open so + * we can still build circuits preemptively as needed. */ + if (num < MAX_UNUSED_OPEN_CIRCUITS-2 && + circuit_build_times_needs_circuits_now(&circ_times)) { + flags = CIRCLAUNCH_NEED_CAPACITY; + log_info(LD_CIRC, + "Have %d clean circs need another buildtime test circ.", num); + circuit_launch_by_router(CIRCUIT_PURPOSE_C_GENERAL, NULL, flags); + return; + } } /** Build a new test circuit every 5 minutes */ @@ -624,6 +696,11 @@ circuit_detach_stream(circuit_t *circ, edge_connection_t *conn) tor_fragile_assert(); } +/** If we haven't yet decided on a good timeout value for circuit + * building, we close idles circuits aggressively so we can get more + * data points. */ +#define IDLE_TIMEOUT_WHILE_LEARNING (10*60) + /** Find each circuit that has been unused for too long, or dirty * for too long and has no streams on it: mark it for close. */ @@ -631,10 +708,18 @@ static void circuit_expire_old_circuits_clientside(time_t now) { circuit_t *circ; - time_t cutoff = now - get_options()->CircuitIdleTimeout; + time_t cutoff; + + if (circuit_build_times_needs_circuits(&circ_times)) { + /* Circuits should be shorter lived if we need more of them + * for learning a good build timeout */ + cutoff = now - IDLE_TIMEOUT_WHILE_LEARNING; + } else { + cutoff = now - get_options()->CircuitIdleTimeout; + } for (circ = global_circuitlist; circ; circ = circ->next) { - if (circ->marked_for_close || ! CIRCUIT_IS_ORIGIN(circ)) + if (circ->marked_for_close || !CIRCUIT_IS_ORIGIN(circ)) continue; /* If the circuit has been dirty for too long, and there are no streams * on it, mark it for close. @@ -647,14 +732,38 @@ circuit_expire_old_circuits_clientside(time_t now) circ->n_circ_id, (int)(now - circ->timestamp_dirty), circ->purpose); circuit_mark_for_close(circ, END_CIRC_REASON_FINISHED); - } else if (!circ->timestamp_dirty && - circ->state == CIRCUIT_STATE_OPEN && - circ->purpose == CIRCUIT_PURPOSE_C_GENERAL) { + } else if (!circ->timestamp_dirty && circ->state == CIRCUIT_STATE_OPEN) { if (circ->timestamp_created < cutoff) { - log_debug(LD_CIRC, - "Closing circuit that has been unused for %d seconds.", - (int)(now - circ->timestamp_created)); - circuit_mark_for_close(circ, END_CIRC_REASON_FINISHED); + if (circ->purpose == CIRCUIT_PURPOSE_C_GENERAL || + circ->purpose == CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT || + circ->purpose == CIRCUIT_PURPOSE_S_ESTABLISH_INTRO || + circ->purpose == CIRCUIT_PURPOSE_TESTING || + (circ->purpose >= CIRCUIT_PURPOSE_C_INTRODUCING && + circ->purpose <= CIRCUIT_PURPOSE_C_REND_READY_INTRO_ACKED) || + circ->purpose == CIRCUIT_PURPOSE_S_CONNECT_REND) { + log_debug(LD_CIRC, + "Closing circuit that has been unused for %ld seconds.", + (long)(now - circ->timestamp_created)); + circuit_mark_for_close(circ, END_CIRC_REASON_FINISHED); + } else if (!TO_ORIGIN_CIRCUIT(circ)->is_ancient) { + /* Server-side rend joined circuits can end up really old, because + * they are reused by clients for longer than normal. The client + * controls their lifespan. (They never become dirty, because + * connection_exit_begin_conn() never marks anything as dirty.) + * Similarly, server-side intro circuits last a long time. */ + if (circ->purpose != CIRCUIT_PURPOSE_S_REND_JOINED && + circ->purpose != CIRCUIT_PURPOSE_S_INTRO) { + log_notice(LD_CIRC, + "Ancient non-dirty circuit %d is still around after " + "%ld seconds. Purpose: %d", + TO_ORIGIN_CIRCUIT(circ)->global_identifier, + (long)(now - circ->timestamp_created), + circ->purpose); + /* FFFF implement a new circuit_purpose_to_string() so we don't + * just print out a number for circ->purpose */ + TO_ORIGIN_CIRCUIT(circ)->is_ancient = 1; + } + } } } } @@ -853,6 +962,9 @@ circuit_build_failed(origin_circuit_t *circ) "(%s:%d). I'm going to try to rotate to a better connection.", n_conn->_base.address, n_conn->_base.port); n_conn->is_bad_for_new_circs = 1; + } else { + log_info(LD_OR, + "Our circuit died before the first hop with no connection"); } if (n_conn_id) { entry_guard_register_connect_status(n_conn_id, 0, 1, time(NULL)); @@ -936,8 +1048,8 @@ circuit_launch_by_router(uint8_t purpose, if (exit) info = extend_info_from_router(exit); circ = circuit_launch_by_extend_info(purpose, info, flags); - if (info) - extend_info_free(info); + + extend_info_free(info); return circ; } @@ -1082,13 +1194,13 @@ circuit_get_open_circ_or_launch(edge_connection_t *conn, * as loudly. the user doesn't even know it's happening. */ if (options->UseBridges && bridges_known_but_down()) { log_fn(severity, LD_APP|LD_DIR, - "Application request when we're believed to be " - "offline. Optimistically trying known bridges again."); + "Application request when we haven't used client functionality " + "lately. Optimistically trying known bridges again."); bridges_retry_all(); } else if (!options->UseBridges || any_bridge_descriptors_known()) { log_fn(severity, LD_APP|LD_DIR, - "Application request when we're believed to be " - "offline. Optimistically trying directory fetches again."); + "Application request when we haven't used client functionality " + "lately. Optimistically trying directory fetches again."); routerlist_retry_directory_downloads(time(NULL)); } } @@ -1111,7 +1223,7 @@ circuit_get_open_circ_or_launch(edge_connection_t *conn, need_uptime)) { log_notice(LD_APP, "No Tor server allows exit to %s:%d. Rejecting.", - safe_str(conn->socks_request->address), + safe_str_client(conn->socks_request->address), conn->socks_request->port); return -1; } @@ -1119,7 +1231,7 @@ circuit_get_open_circ_or_launch(edge_connection_t *conn, /* XXXX022 Duplicates checks in connection_ap_handshake_attach_circuit */ routerinfo_t *router = router_get_by_nickname(conn->chosen_exit_name, 1); int opt = conn->chosen_exit_optional; - if (router && !connection_ap_can_use_exit(conn, router)) { + if (router && !connection_ap_can_use_exit(conn, router, 0)) { log_fn(opt ? LOG_INFO : LOG_WARN, LD_APP, "Requested exit point '%s' would refuse request. %s.", conn->chosen_exit_name, opt ? "Trying others" : "Closing"); @@ -1152,19 +1264,14 @@ circuit_get_open_circ_or_launch(edge_connection_t *conn, if (!extend_info) { log_info(LD_REND, "No intro points for '%s': re-fetching service descriptor.", - safe_str(conn->rend_data->onion_address)); - /* Fetch both, v0 and v2 rend descriptors in parallel. Use whichever - * arrives first. Exception: When using client authorization, only - * fetch v2 descriptors.*/ + safe_str_client(conn->rend_data->onion_address)); rend_client_refetch_v2_renddesc(conn->rend_data); - if (conn->rend_data->auth_type == REND_NO_AUTH) - rend_client_refetch_renddesc(conn->rend_data->onion_address); conn->_base.state = AP_CONN_STATE_RENDDESC_WAIT; return 0; } log_info(LD_REND,"Chose '%s' as intro point for '%s'.", extend_info->nickname, - safe_str(conn->rend_data->onion_address)); + safe_str_client(conn->rend_data->onion_address)); } /* If we have specified a particular exit node for our @@ -1193,7 +1300,7 @@ circuit_get_open_circ_or_launch(edge_connection_t *conn, } if (tor_addr_from_str(&addr, conn->socks_request->address) < 0) { log_info(LD_DIR, "Broken address %s on tunnel conn. Closing.", - escaped_safe_str(conn->socks_request->address)); + escaped_safe_str_client(conn->socks_request->address)); return -1; } extend_info = extend_info_alloc(conn->chosen_exit_name+1, @@ -1235,8 +1342,7 @@ circuit_get_open_circ_or_launch(edge_connection_t *conn, flags); } - if (extend_info) - extend_info_free(extend_info); + extend_info_free(extend_info); if (desired_circuit_purpose != CIRCUIT_PURPOSE_C_GENERAL) { /* help predict this next time */ @@ -1418,7 +1524,7 @@ connection_ap_handshake_attach_circuit(edge_connection_t *conn) LOG_INFO : LOG_NOTICE; log_fn(severity, LD_APP, "Tried for %d seconds to get a connection to %s:%d. Giving up.", - conn_age, safe_str(conn->socks_request->address), + conn_age, safe_str_client(conn->socks_request->address), conn->socks_request->port); return -1; } @@ -1445,7 +1551,7 @@ connection_ap_handshake_attach_circuit(edge_connection_t *conn) } return -1; } - if (router && !connection_ap_can_use_exit(conn, router)) { + if (router && !connection_ap_can_use_exit(conn, router, 0)) { log_fn(opt ? LOG_INFO : LOG_WARN, LD_APP, "Requested exit point '%s' would refuse request. %s.", conn->chosen_exit_name, opt ? "Trying others" : "Closing"); diff --git a/src/or/circuituse.h b/src/or/circuituse.h new file mode 100644 index 0000000000..269b6c5f8f --- /dev/null +++ b/src/or/circuituse.h @@ -0,0 +1,53 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2010, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file circuituse.h + * \brief Header file for circuituse.c. + **/ + +#ifndef _TOR_CIRCUITUSE_H +#define _TOR_CIRCUITUSE_H + +void circuit_expire_building(time_t now); +void circuit_remove_handled_ports(smartlist_t *needed_ports); +int circuit_stream_is_being_handled(edge_connection_t *conn, uint16_t port, + int min); +int circuit_conforms_to_options(const origin_circuit_t *circ, + const or_options_t *options); +void circuit_build_needed_circs(time_t now); +void circuit_detach_stream(circuit_t *circ, edge_connection_t *conn); + +void circuit_expire_old_circuits_serverside(time_t now); + +void reset_bandwidth_test(void); +int circuit_enough_testing_circs(void); + +void circuit_has_opened(origin_circuit_t *circ); +void circuit_build_failed(origin_circuit_t *circ); + +/** Flag to set when a circuit should have only a single hop. */ +#define CIRCLAUNCH_ONEHOP_TUNNEL (1<<0) +/** Flag to set when a circuit needs to be built of high-uptime nodes */ +#define CIRCLAUNCH_NEED_UPTIME (1<<1) +/** Flag to set when a circuit needs to be built of high-capacity nodes */ +#define CIRCLAUNCH_NEED_CAPACITY (1<<2) +/** Flag to set when the last hop of a circuit doesn't need to be an + * exit node. */ +#define CIRCLAUNCH_IS_INTERNAL (1<<3) +origin_circuit_t *circuit_launch_by_extend_info(uint8_t purpose, + extend_info_t *info, + int flags); +origin_circuit_t *circuit_launch_by_router(uint8_t purpose, + routerinfo_t *exit, int flags); +void circuit_reset_failure_count(int timeout); +int connection_ap_handshake_attach_chosen_circuit(edge_connection_t *conn, + origin_circuit_t *circ, + crypt_path_t *cpath); +int connection_ap_handshake_attach_circuit(edge_connection_t *conn); + +#endif + diff --git a/src/or/command.c b/src/or/command.c index 181bad8bcd..0460e25c25 100644 --- a/src/or/command.c +++ b/src/or/command.c @@ -16,6 +16,19 @@ */ #include "or.h" +#include "circuitbuild.h" +#include "circuitlist.h" +#include "command.h" +#include "connection.h" +#include "connection_or.h" +#include "config.h" +#include "control.h" +#include "cpuworker.h" +#include "hibernate.h" +#include "onion.h" +#include "relay.h" +#include "router.h" +#include "routerlist.h" /** How many CELL_PADDING cells have we received, ever? */ uint64_t stats_n_padding_cells_processed = 0; @@ -395,15 +408,18 @@ command_process_relay_cell(cell_t *cell, or_connection_t *conn) * gotten no more than MAX_RELAY_EARLY_CELLS_PER_CIRCUIT of them. */ if (cell->command == CELL_RELAY_EARLY) { if (direction == CELL_DIRECTION_IN) { - /* XXX Allow an unlimited number of inbound relay_early cells for - * now, for hidden service compatibility. See bug 1038. -RD */ + /* Allow an unlimited number of inbound relay_early cells, + * for hidden service compatibility. There isn't any way to make + * a long circuit through inbound relay_early cells anyway. See + * bug 1038. -RD */ } else { or_circuit_t *or_circ = TO_OR_CIRCUIT(circ); if (or_circ->remaining_relay_early_cells == 0) { log_fn(LOG_PROTOCOL_WARN, LD_OR, "Received too many RELAY_EARLY cells on circ %d from %s:%d." " Closing circuit.", - cell->circ_id, safe_str(conn->_base.address), conn->_base.port); + cell->circ_id, safe_str(conn->_base.address), + conn->_base.port); circuit_mark_for_close(circ, END_CIRC_REASON_TORPROTOCOL); return; } @@ -511,7 +527,8 @@ command_process_versions_cell(var_cell_t *cell, or_connection_t *conn) conn->handshake_state->received_versions = 1; log_info(LD_OR, "Negotiated version %d with %s:%d; sending NETINFO.", - highest_supported_version, safe_str(conn->_base.address), + highest_supported_version, + safe_str_client(conn->_base.address), conn->_base.port); tor_assert(conn->link_proto >= 2); @@ -625,8 +642,8 @@ command_process_netinfo_cell(cell_t *cell, or_connection_t *conn) else log_info(LD_OR, "Got good NETINFO cell from %s:%d; OR connection is now " "open, using protocol version %d", - safe_str(conn->_base.address), conn->_base.port, - (int)conn->link_proto); + safe_str_client(conn->_base.address), + conn->_base.port, (int)conn->link_proto); assert_connection_ok(TO_CONN(conn),time(NULL)); } diff --git a/src/or/command.h b/src/or/command.h new file mode 100644 index 0000000000..1aa56207f6 --- /dev/null +++ b/src/or/command.h @@ -0,0 +1,25 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2010, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file command.h + * \brief Header file for command.c. + **/ + +#ifndef _TOR_COMMAND_H +#define _TOR_COMMAND_H + +void command_process_cell(cell_t *cell, or_connection_t *conn); +void command_process_var_cell(var_cell_t *cell, or_connection_t *conn); + +extern uint64_t stats_n_padding_cells_processed; +extern uint64_t stats_n_create_cells_processed; +extern uint64_t stats_n_created_cells_processed; +extern uint64_t stats_n_relay_cells_processed; +extern uint64_t stats_n_destroy_cells_processed; + +#endif + diff --git a/src/or/config.c b/src/or/config.c index 7966557063..ef2b2ddeab 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -12,6 +12,28 @@ #define CONFIG_PRIVATE #include "or.h" +#include "circuitbuild.h" +#include "circuitlist.h" +#include "config.h" +#include "connection.h" +#include "connection_edge.h" +#include "connection_or.h" +#include "control.h" +#include "cpuworker.h" +#include "dirserv.h" +#include "dirvote.h" +#include "dns.h" +#include "geoip.h" +#include "hibernate.h" +#include "main.h" +#include "networkstatus.h" +#include "policies.h" +#include "relay.h" +#include "rendclient.h" +#include "rendservice.h" +#include "rephist.h" +#include "router.h" +#include "routerlist.h" #ifdef MS_WINDOWS #include <shlobj.h> #endif @@ -66,6 +88,7 @@ static config_abbrev_t _option_abbrevs[] = { PLURAL(RendExcludeNode), PLURAL(StrictEntryNode), PLURAL(StrictExitNode), + PLURAL(StrictNode), { "l", "Log", 1, 0}, { "AllowUnverifiedNodes", "AllowInvalidNodes", 0, 0}, { "AutomapHostSuffixes", "AutomapHostsSuffixes", 0, 0}, @@ -83,10 +106,12 @@ static config_abbrev_t _option_abbrevs[] = { { "NumEntryNodes", "NumEntryGuards", 0, 0}, { "ResolvConf", "ServerDNSResolvConfFile", 0, 1}, { "SearchDomains", "ServerDNSSearchDomains", 0, 1}, - { "ServerDNSAllowBrokenResolvConf", "ServerDNSAllowBrokenConfig", 0, 0 }, + { "ServerDNSAllowBrokenResolvConf", "ServerDNSAllowBrokenConfig", 0, 0}, { "PreferTunnelledDirConns", "PreferTunneledDirConns", 0, 0}, { "BridgeAuthoritativeDirectory", "BridgeAuthoritativeDir", 0, 0}, { "HashedControlPassword", "__HashedControlSessionPassword", 1, 0}, + { "StrictEntryNodes", "StrictNodes", 0, 1}, + { "StrictExitNodes", "StrictNodes", 0, 1}, { NULL, NULL, 0, 0}, }; @@ -134,6 +159,7 @@ static config_var_t _option_vars[] = { V(AccountingMax, MEMUNIT, "0 bytes"), V(AccountingStart, STRING, NULL), V(Address, STRING, NULL), + V(AllowDotExit, BOOL, "0"), V(AllowInvalidNodes, CSV, "middle,rendezvous"), V(AllowNonRFC953Hostnames, BOOL, "0"), V(AllowSingleHopCircuits, BOOL, "0"), @@ -162,10 +188,15 @@ static config_var_t _option_vars[] = { V(BridgePassword, STRING, NULL), V(BridgeRecordUsageByCountry, BOOL, "1"), V(BridgeRelay, BOOL, "0"), - V(CircuitBuildTimeout, INTERVAL, "1 minute"), + V(CellStatistics, BOOL, "0"), + V(LearnCircuitBuildTimeout, BOOL, "1"), + V(CircuitBuildTimeout, INTERVAL, "0"), V(CircuitIdleTimeout, INTERVAL, "1 hour"), + V(CircuitStreamTimeout, INTERVAL, "0"), + V(CircuitPriorityHalflife, DOUBLE, "-100.0"), /*negative:'Use default'*/ V(ClientDNSRejectInternalAddresses, BOOL,"1"), V(ClientOnly, BOOL, "0"), + V(ConsensusParams, STRING, NULL), V(ConnLimit, UINT, "1000"), V(ConstrainedSockets, BOOL, "0"), V(ConstrainedSockSize, MEMUNIT, "8192"), @@ -186,18 +217,19 @@ static config_var_t _option_vars[] = { V(DirPort, UINT, "0"), V(DirPortFrontPage, FILENAME, NULL), OBSOLETE("DirPostPeriod"), -#ifdef ENABLE_GEOIP_STATS - V(DirRecordUsageByCountry, BOOL, "0"), - V(DirRecordUsageGranularity, UINT, "4"), - V(DirRecordUsageRetainIPs, INTERVAL, "14 days"), - V(DirRecordUsageSaveInterval, INTERVAL, "6 hours"), -#endif + OBSOLETE("DirRecordUsageByCountry"), + OBSOLETE("DirRecordUsageGranularity"), + OBSOLETE("DirRecordUsageRetainIPs"), + OBSOLETE("DirRecordUsageSaveInterval"), + V(DirReqStatistics, BOOL, "0"), VAR("DirServer", LINELIST, DirServers, NULL), + V(DisableAllSwap, BOOL, "0"), V(DNSPort, UINT, "0"), V(DNSListenAddress, LINELIST, NULL), V(DownloadExtraInfo, BOOL, "0"), V(EnforceDistinctSubnets, BOOL, "1"), V(EntryNodes, ROUTERSET, NULL), + V(EntryStatistics, BOOL, "0"), V(TestingEstimatedDescriptorPropagationTime, INTERVAL, "10 minutes"), V(ExcludeNodes, ROUTERSET, NULL), V(ExcludeExitNodes, ROUTERSET, NULL), @@ -205,12 +237,20 @@ static config_var_t _option_vars[] = { V(ExitNodes, ROUTERSET, NULL), V(ExitPolicy, LINELIST, NULL), V(ExitPolicyRejectPrivate, BOOL, "1"), + V(ExitPortStatistics, BOOL, "0"), + V(ExtraInfoStatistics, BOOL, "0"), + +#if defined (WINCE) + V(FallbackNetworkstatusFile, FILENAME, "fallback-consensus"), +#else V(FallbackNetworkstatusFile, FILENAME, SHARE_DATADIR PATH_SEPARATOR "tor" PATH_SEPARATOR "fallback-consensus"), +#endif V(FascistFirewall, BOOL, "0"), V(FirewallPorts, CSV, ""), V(FastFirstHopPK, BOOL, "1"), V(FetchDirInfoEarly, BOOL, "0"), + V(FetchDirInfoExtraEarly, BOOL, "0"), V(FetchServerDescriptors, BOOL, "1"), V(FetchHidServDescriptors, BOOL, "1"), V(FetchUselessDescriptors, BOOL, "0"), @@ -222,6 +262,8 @@ static config_var_t _option_vars[] = { #endif OBSOLETE("Group"), V(HardwareAccel, BOOL, "0"), + V(AccelName, STRING, NULL), + V(AccelDir, FILENAME, NULL), V(HashedControlPassword, LINELIST, NULL), V(HidServDirectoryV2, BOOL, "1"), VAR("HiddenServiceDir", LINELIST_S, RendConfigLines, NULL), @@ -233,11 +275,15 @@ static config_var_t _option_vars[] = { VAR("HiddenServiceAuthorizeClient",LINELIST_S,RendConfigLines, NULL), V(HidServAuth, LINELIST, NULL), V(HSAuthoritativeDir, BOOL, "0"), - V(HSAuthorityRecordStats, BOOL, "0"), + OBSOLETE("HSAuthorityRecordStats"), V(HttpProxy, STRING, NULL), V(HttpProxyAuthenticator, STRING, NULL), V(HttpsProxy, STRING, NULL), V(HttpsProxyAuthenticator, STRING, NULL), + V(Socks4Proxy, STRING, NULL), + V(Socks5Proxy, STRING, NULL), + V(Socks5ProxyUsername, STRING, NULL), + V(Socks5ProxyPassword, STRING, NULL), OBSOLETE("IgnoreVersion"), V(KeepalivePeriod, INTERVAL, "5 minutes"), VAR("Log", LINELIST, Logs, NULL), @@ -257,6 +303,7 @@ static config_var_t _option_vars[] = { V(NatdListenAddress, LINELIST, NULL), V(NatdPort, UINT, "0"), V(Nickname, STRING, NULL), + V(WarnUnsafeSocks, BOOL, "1"), V(NoPublish, BOOL, "0"), VAR("NodeFamily", LINELIST, NodeFamilies, NULL), V(NumCpus, UINT, "1"), @@ -265,6 +312,8 @@ static config_var_t _option_vars[] = { V(ORPort, UINT, "0"), V(OutboundBindAddress, STRING, NULL), OBSOLETE("PathlenCoinWeight"), + V(PerConnBWBurst, MEMUNIT, "0"), + V(PerConnBWRate, MEMUNIT, "0"), V(PidFile, STRING, NULL), V(TestingTorNetwork, BOOL, "0"), V(PreferTunneledDirConns, BOOL, "1"), @@ -278,6 +327,7 @@ static config_var_t _option_vars[] = { V(RecommendedClientVersions, LINELIST, NULL), V(RecommendedServerVersions, LINELIST, NULL), OBSOLETE("RedirectExit"), + V(RefuseUnknownExits, BOOL, "0"), V(RejectPlaintextPorts, CSV, ""), V(RelayBandwidthBurst, MEMUNIT, "0"), V(RelayBandwidthRate, MEMUNIT, "0"), @@ -288,7 +338,7 @@ static config_var_t _option_vars[] = { OBSOLETE("RouterFile"), V(RunAsDaemon, BOOL, "0"), V(RunTesting, BOOL, "0"), - V(SafeLogging, BOOL, "1"), + V(SafeLogging, STRING, "1"), V(SafeSocks, BOOL, "0"), V(ServerDNSAllowBrokenConfig, BOOL, "1"), V(ServerDNSAllowNonRFC953Hostnames, BOOL,"0"), @@ -304,8 +354,7 @@ static config_var_t _option_vars[] = { V(SocksPort, UINT, "9050"), V(SocksTimeout, INTERVAL, "2 minutes"), OBSOLETE("StatusFetchPeriod"), - V(StrictEntryNodes, BOOL, "0"), - V(StrictExitNodes, BOOL, "0"), + V(StrictNodes, BOOL, "0"), OBSOLETE("SysLog"), V(TestSocks, BOOL, "0"), OBSOLETE("TestVia"), @@ -330,6 +379,7 @@ static config_var_t _option_vars[] = { V(V3AuthDistDelay, INTERVAL, "5 minutes"), V(V3AuthNIntervalsValid, UINT, "3"), V(V3AuthUseLegacyKey, BOOL, "0"), + V(V3BandwidthsFile, FILENAME, NULL), VAR("VersioningAuthoritativeDirectory",BOOL,VersioningAuthoritativeDir, "0"), V(VirtualAddrNetwork, STRING, "127.192.0.0/10"), V(WarnPlaintextPorts, CSV, "23,109,110,143"), @@ -340,6 +390,7 @@ static config_var_t _option_vars[] = { VAR("__HashedControlSessionPassword", LINELIST, HashedControlSessionPassword, NULL), V(MinUptimeHidServDirectoryV2, INTERVAL, "24 hours"), + { NULL, CONFIG_TYPE_OBSOLETE, 0, NULL } }; @@ -396,6 +447,11 @@ static config_var_t _state_vars[] = { V(LastRotatedOnionKey, ISOTIME, NULL), V(LastWritten, ISOTIME, NULL), + V(TotalBuildTimes, UINT, NULL), + V(CircuitBuildAbandonedCount, UINT, "0"), + VAR("CircuitBuildTimeBin", LINELIST_S, BuildtimeHistogram, NULL), + VAR("BuildtimeHistogram", LINELIST_V, BuildtimeHistogram, NULL), + { NULL, CONFIG_TYPE_OBSOLETE, 0, NULL } }; @@ -410,213 +466,6 @@ typedef struct config_var_description_t { const char *description; } config_var_description_t; -/** Descriptions of the configuration options, to be displayed by online - * option browsers */ -/* XXXX022 did anybody want this? at all? If not, kill it.*/ -static config_var_description_t options_description[] = { - /* ==== general options */ - { "AvoidDiskWrites", "If non-zero, try to write to disk less frequently than" - " we would otherwise." }, - { "BandwidthRate", "A token bucket limits the average incoming bandwidth on " - "this node to the specified number of bytes per second." }, - { "BandwidthBurst", "Limit the maximum token buffer size (also known as " - "burst) to the given number of bytes." }, - { "ConnLimit", "Minimum number of simultaneous sockets we must have." }, - { "ConstrainedSockets", "Shrink tx and rx buffers for sockets to avoid " - "system limits on vservers and related environments. See man page for " - "more information regarding this option." }, - { "ConstrainedSockSize", "Limit socket buffers to this size when " - "ConstrainedSockets is enabled." }, - /* ControlListenAddress */ - { "ControlPort", "If set, Tor will accept connections from the same machine " - "(localhost only) on this port, and allow those connections to control " - "the Tor process using the Tor Control Protocol (described in " - "control-spec.txt).", }, - { "CookieAuthentication", "If this option is set to 1, don't allow any " - "connections to the control port except when the connecting process " - "can read a file that Tor creates in its data directory." }, - { "DataDirectory", "Store working data, state, keys, and caches here." }, - { "DirServer", "Tor only trusts directories signed with one of these " - "servers' keys. Used to override the standard list of directory " - "authorities." }, - /* { "FastFirstHopPK", "" }, */ - /* FetchServerDescriptors, FetchHidServDescriptors, - * FetchUselessDescriptors */ - { "HardwareAccel", "If set, Tor tries to use hardware crypto accelerators " - "when it can." }, - /* HashedControlPassword */ - { "HTTPProxy", "Force Tor to make all HTTP directory requests through this " - "host:port (or host:80 if port is not set)." }, - { "HTTPProxyAuthenticator", "A username:password pair to be used with " - "HTTPProxy." }, - { "HTTPSProxy", "Force Tor to make all TLS (SSL) connections through this " - "host:port (or host:80 if port is not set)." }, - { "HTTPSProxyAuthenticator", "A username:password pair to be used with " - "HTTPSProxy." }, - { "KeepalivePeriod", "Send a padding cell every N seconds to keep firewalls " - "from closing our connections while Tor is not in use." }, - { "Log", "Where to send logging messages. Format is " - "minSeverity[-maxSeverity] (stderr|stdout|syslog|file FILENAME)." }, - { "OutboundBindAddress", "Make all outbound connections originate from the " - "provided IP address (only useful for multiple network interfaces)." }, - { "PIDFile", "On startup, write our PID to this file. On clean shutdown, " - "remove the file." }, - { "PreferTunneledDirConns", "If non-zero, avoid directory servers that " - "don't support tunneled connections." }, - /* PreferTunneledDirConns */ - /* ProtocolWarnings */ - /* RephistTrackTime */ - { "RunAsDaemon", "If set, Tor forks and daemonizes to the background when " - "started. Unix only." }, - { "SafeLogging", "If set to 0, Tor logs potentially sensitive strings " - "rather than replacing them with the string [scrubbed]." }, - { "TunnelDirConns", "If non-zero, when a directory server we contact " - "supports it, we will build a one-hop circuit and make an encrypted " - "connection via its ORPort." }, - { "User", "On startup, setuid to this user." }, - - /* ==== client options */ - { "AllowInvalidNodes", "Where on our circuits should Tor allow servers " - "that the directory authorities haven't called \"valid\"?" }, - { "AllowNonRFC953Hostnames", "If set to 1, we don't automatically reject " - "hostnames for having invalid characters." }, - /* CircuitBuildTimeout, CircuitIdleTimeout */ - { "ClientOnly", "If set to 1, Tor will under no circumstances run as a " - "server, even if ORPort is enabled." }, - { "EntryNodes", "A list of preferred entry nodes to use for the first hop " - "in circuits, when possible." }, - /* { "EnforceDistinctSubnets" , "" }, */ - { "ExitNodes", "A list of preferred nodes to use for the last hop in " - "circuits, when possible." }, - { "ExcludeNodes", "A list of nodes never to use when building a circuit." }, - { "FascistFirewall", "If set, Tor will only create outgoing connections to " - "servers running on the ports listed in FirewallPorts." }, - { "FirewallPorts", "A list of ports that we can connect to. Only used " - "when FascistFirewall is set." }, - { "LongLivedPorts", "A list of ports for services that tend to require " - "high-uptime connections." }, - { "MapAddress", "Force Tor to treat all requests for one address as if " - "they were for another." }, - { "NewCircuitPeriod", "Force Tor to consider whether to build a new circuit " - "every NUM seconds." }, - { "MaxCircuitDirtiness", "Do not attach new streams to a circuit that has " - "been used more than this many seconds ago." }, - /* NatdPort, NatdListenAddress */ - { "NodeFamily", "A list of servers that constitute a 'family' and should " - "never be used in the same circuit." }, - { "NumEntryGuards", "How many entry guards should we keep at a time?" }, - /* PathlenCoinWeight */ - { "ReachableAddresses", "Addresses we can connect to, as IP/bits:port-port. " - "By default, we assume all addresses are reachable." }, - /* reachablediraddresses, reachableoraddresses. */ - /* SafeSOCKS */ - { "SOCKSPort", "The port where we listen for SOCKS connections from " - "applications." }, - { "SOCKSListenAddress", "Bind to this address to listen to connections from " - "SOCKS-speaking applications." }, - { "SOCKSPolicy", "Set an entry policy to limit which addresses can connect " - "to the SOCKSPort." }, - /* SocksTimeout */ - { "StrictExitNodes", "If set, Tor will fail to operate when none of the " - "configured ExitNodes can be used." }, - { "StrictEntryNodes", "If set, Tor will fail to operate when none of the " - "configured EntryNodes can be used." }, - /* TestSocks */ - { "TrackHostsExit", "Hosts and domains which should, if possible, be " - "accessed from the same exit node each time we connect to them." }, - { "TrackHostsExitExpire", "Time after which we forget which exit we were " - "using to connect to hosts in TrackHostsExit." }, - /* "TransPort", "TransListenAddress */ - { "UseEntryGuards", "Set to 0 if we want to pick from the whole set of " - "servers for the first position in each circuit, rather than picking a " - "set of 'Guards' to prevent profiling attacks." }, - - /* === server options */ - { "Address", "The advertised (external) address we should use." }, - /* Accounting* options. */ - /* AssumeReachable */ - { "ContactInfo", "Administrative contact information to advertise for this " - "server." }, - { "ExitPolicy", "Address/port ranges for which to accept or reject outgoing " - "connections on behalf of Tor users." }, - /* { "ExitPolicyRejectPrivate, "" }, */ - { "MaxAdvertisedBandwidth", "If set, we will not advertise more than this " - "amount of bandwidth for our bandwidth rate, regardless of how much " - "bandwidth we actually detect." }, - { "MaxOnionsPending", "Reject new attempts to extend circuits when we " - "already have this many pending." }, - { "MyFamily", "Declare a list of other servers as belonging to the same " - "family as this one, so that clients will not use two from the same " - "family in the same circuit." }, - { "Nickname", "Set the server nickname." }, - { "NoPublish", "{DEPRECATED}" }, - { "NumCPUs", "How many processes to use at once for public-key crypto." }, - { "ORPort", "Advertise this port to listen for connections from Tor clients " - "and servers." }, - { "ORListenAddress", "Bind to this address to listen for connections from " - "clients and servers, instead of the default 0.0.0.0:ORPort." }, - { "PublishServerDescriptor", "Set to 0 to keep the server from " - "uploading info to the directory authorities." }, - /* ServerDNS: DetectHijacking, ResolvConfFile, SearchDomains */ - { "ShutdownWaitLength", "Wait this long for clients to finish when " - "shutting down because of a SIGINT." }, - - /* === directory cache options */ - { "DirPort", "Serve directory information from this port, and act as a " - "directory cache." }, - { "DirPortFrontPage", "Serve a static html disclaimer on DirPort." }, - { "DirListenAddress", "Bind to this address to listen for connections from " - "clients and servers, instead of the default 0.0.0.0:DirPort." }, - { "DirPolicy", "Set a policy to limit who can connect to the directory " - "port." }, - - /* Authority options: AuthDirBadExit, AuthDirInvalid, AuthDirReject, - * AuthDirRejectUnlisted, AuthDirListBadExits, AuthoritativeDirectory, - * DirAllowPrivateAddresses, HSAuthoritativeDir, - * NamingAuthoritativeDirectory, RecommendedVersions, - * RecommendedClientVersions, RecommendedServerVersions, RendPostPeriod, - * RunTesting, V1AuthoritativeDirectory, VersioningAuthoritativeDirectory, */ - - /* Hidden service options: HiddenService: dir,excludenodes, nodes, - * options, port. PublishHidServDescriptor */ - - /* Nonpersistent options: __LeaveStreamsUnattached, __AllDirActionsPrivate */ - { NULL, NULL }, -}; - -/** Online description of state variables. */ -static config_var_description_t state_description[] = { - { "AccountingBytesReadInInterval", - "How many bytes have we read in this accounting period?" }, - { "AccountingBytesWrittenInInterval", - "How many bytes have we written in this accounting period?" }, - { "AccountingExpectedUsage", - "How many bytes did we expect to use per minute? (0 for no estimate.)" }, - { "AccountingIntervalStart", "When did this accounting period begin?" }, - { "AccountingSecondsActive", "How long have we been awake in this period?" }, - - { "BWHistoryReadEnds", "When does the last-recorded read-interval end?" }, - { "BWHistoryReadInterval", "How long is each read-interval (in seconds)?" }, - { "BWHistoryReadValues", "Number of bytes read in each interval." }, - { "BWHistoryWriteEnds", "When does the last-recorded write-interval end?" }, - { "BWHistoryWriteInterval", "How long is each write-interval (in seconds)?"}, - { "BWHistoryWriteValues", "Number of bytes written in each interval." }, - - { "EntryGuard", "One of the nodes we have chosen as a fixed entry" }, - { "EntryGuardDownSince", - "The last entry guard has been unreachable since this time." }, - { "EntryGuardUnlistedSince", - "The last entry guard has been unusable since this time." }, - - { "LastRotatedOnionKey", - "The last time at which we changed the medium-term private key used for " - "building circuits." }, - { "LastWritten", "When was this state file last regenerated?" }, - - { "TorVersion", "Which version of Tor generated this state file?" }, - { NULL, NULL }, -}; - /** Type of a callback to validate whether a given configuration is * well-formed and consistent. See options_trial_assign() for documentation * of arguments. */ @@ -635,8 +484,6 @@ typedef struct { config_var_t *vars; /**< List of variables we recognize, their default * values, and where we stick them in the structure. */ validate_fn_t validate_fn; /**< Function to validate config. */ - /** Documentation for configuration variables. */ - config_var_description_t *descriptions; /** If present, extra is a LINELIST variable for unrecognized * lines. Otherwise, unrecognized lines are an error. */ config_var_t *extra; @@ -700,20 +547,6 @@ static uint64_t config_parse_memunit(const char *s, int *ok); static int config_parse_interval(const char *s, int *ok); static void init_libevent(void); static int opt_streq(const char *s1, const char *s2); -/** Versions of libevent. */ -typedef enum { - /* Note: we compare these, so it's important that "old" precede everything, - * and that "other" come last. */ - LE_OLD=0, LE_10C, LE_10D, LE_10E, LE_11, LE_11A, LE_11B, LE_12, LE_12A, - LE_13, LE_13A, LE_13B, LE_13C, LE_13D, LE_13E, - LE_140, LE_141, LE_142, LE_143, LE_144, LE_145, LE_146, LE_147, LE_148, - LE_1499, - LE_OTHER -} le_version_t; -static le_version_t decode_libevent_version(const char *v, int *bincompat_out); -#if defined(HAVE_EVENT_GET_VERSION) && defined(HAVE_EVENT_GET_METHOD) -static void check_libevent_version(const char *m, int server); -#endif /** Magic value for or_options_t. */ #define OR_OPTIONS_MAGIC 9090909 @@ -726,7 +559,6 @@ static config_format_t options_format = { _option_abbrevs, _option_vars, (validate_fn_t)options_validate, - options_description, NULL }; @@ -747,7 +579,6 @@ static config_format_t state_format = { _state_abbrevs, _state_vars, (validate_fn_t)or_state_validate, - state_description, &state_extra_var, }; @@ -812,13 +643,13 @@ set_options(or_options_t *new_val, char **msg) "Acting on config options left us in a broken state. Dying."); exit(1); } - if (old_options) - config_free(&options_format, old_options); + + config_free(&options_format, old_options); return 0; } -extern const char tor_svn_revision[]; /* from tor_main.c */ +extern const char tor_git_revision[]; /* from tor_main.c */ /** The version of this Tor process, as parsed. */ static char *_version = NULL; @@ -828,10 +659,10 @@ const char * get_version(void) { if (_version == NULL) { - if (strlen(tor_svn_revision)) { - size_t len = strlen(VERSION)+strlen(tor_svn_revision)+8; + if (strlen(tor_git_revision)) { + size_t len = strlen(VERSION)+strlen(tor_git_revision)+16; _version = tor_malloc(len); - tor_snprintf(_version, len, "%s (r%s)", VERSION, tor_svn_revision); + tor_snprintf(_version, len, "%s (git-%s)", VERSION, tor_git_revision); } else { _version = tor_strdup(VERSION); } @@ -844,8 +675,10 @@ get_version(void) static void or_options_free(or_options_t *options) { - if (options->_ExcludeExitNodesUnion) - routerset_free(options->_ExcludeExitNodesUnion); + if (!options) + return; + + routerset_free(options->_ExcludeExitNodesUnion); config_free(&options_format, options); } @@ -854,43 +687,72 @@ or_options_free(or_options_t *options) void config_free_all(void) { - if (global_options) { - or_options_free(global_options); - global_options = NULL; - } - if (global_state) { - config_free(&state_format, global_state); - global_state = NULL; - } - if (global_cmdline_options) { - config_free_lines(global_cmdline_options); - global_cmdline_options = NULL; - } + or_options_free(global_options); + global_options = NULL; + + config_free(&state_format, global_state); + global_state = NULL; + + config_free_lines(global_cmdline_options); + global_cmdline_options = NULL; + tor_free(torrc_fname); tor_free(_version); tor_free(global_dirfrontpagecontents); } -/** If options->SafeLogging is on, return a not very useful string, - * else return address. +/** Make <b>address</b> -- a piece of information related to our operation as + * a client -- safe to log according to the settings in options->SafeLogging, + * and return it. + * + * (We return "[scrubbed]" if SafeLogging is "1", and address otherwise.) + */ +const char * +safe_str_client(const char *address) +{ + tor_assert(address); + if (get_options()->_SafeLogging == SAFELOG_SCRUB_ALL) + return "[scrubbed]"; + else + return address; +} + +/** Make <b>address</b> -- a piece of information of unspecified sensitivity + * -- safe to log according to the settings in options->SafeLogging, and + * return it. + * + * (We return "[scrubbed]" if SafeLogging is anything besides "0", and address + * otherwise.) */ const char * safe_str(const char *address) { tor_assert(address); - if (get_options()->SafeLogging) + if (get_options()->_SafeLogging != SAFELOG_SCRUB_NONE) return "[scrubbed]"; else return address; } +/** Equivalent to escaped(safe_str_client(address)). See reentrancy note on + * escaped(): don't use this outside the main thread, or twice in the same + * log statement. */ +const char * +escaped_safe_str_client(const char *address) +{ + if (get_options()->_SafeLogging == SAFELOG_SCRUB_ALL) + return "[scrubbed]"; + else + return escaped(address); +} + /** Equivalent to escaped(safe_str(address)). See reentrancy note on * escaped(): don't use this outside the main thread, or twice in the same * log statement. */ const char * escaped_safe_str(const char *address) { - if (get_options()->SafeLogging) + if (get_options()->_SafeLogging != SAFELOG_SCRUB_NONE) return "[scrubbed]"; else return escaped(address); @@ -1089,10 +951,12 @@ options_act_reversible(or_options_t *old_options, char **msg) } /* Launch the listeners. (We do this before we setuid, so we can bind to - * ports under 1024.) */ - if (retry_all_listeners(replaced_listeners, new_listeners) < 0) { - *msg = tor_strdup("Failed to bind one of the listener ports."); - goto rollback; + * ports under 1024.) We don't want to rebind if we're hibernating. */ + if (!we_are_hibernating()) { + if (retry_all_listeners(replaced_listeners, new_listeners) < 0) { + *msg = tor_strdup("Failed to bind one of the listener ports."); + goto rollback; + } } } @@ -1106,6 +970,15 @@ options_act_reversible(or_options_t *old_options, char **msg) } #endif + /* Attempt to lock all current and future memory with mlockall() only once */ + if (options->DisableAllSwap) { + if (tor_mlockall() == -1) { + *msg = tor_strdup("DisableAllSwap failure. Do you have proper " + "permissions?"); + goto done; + } + } + /* Setuid/setgid as appropriate */ if (options->User) { if (switch_id(options->User) != 0) { @@ -1118,11 +991,9 @@ options_act_reversible(or_options_t *old_options, char **msg) /* Ensure data directory is private; create if possible. */ if (check_private_dir(options->DataDirectory, running_tor ? CPD_CREATE : CPD_CHECK)<0) { - char buf[1024]; - int tmp = tor_snprintf(buf, sizeof(buf), + tor_asprintf(msg, "Couldn't access/create private data directory \"%s\"", options->DataDirectory); - *msg = tor_strdup(tmp >= 0 ? buf : "internal error"); goto done; /* No need to roll back, since you can't change the value. */ } @@ -1133,10 +1004,8 @@ options_act_reversible(or_options_t *old_options, char **msg) tor_snprintf(fn, len, "%s"PATH_SEPARATOR"cached-status", options->DataDirectory); if (check_private_dir(fn, running_tor ? CPD_CREATE : CPD_CHECK) < 0) { - char buf[1024]; - int tmp = tor_snprintf(buf, sizeof(buf), + tor_asprintf(msg, "Couldn't access/create private data directory \"%s\"", fn); - *msg = tor_strdup(tmp >= 0 ? buf : "internal error"); tor_free(fn); goto done; } @@ -1236,7 +1105,6 @@ get_effective_bwrate(or_options_t *options) bw = options->MaxAdvertisedBandwidth; if (options->RelayBandwidthRate > 0 && bw > options->RelayBandwidthRate) bw = options->RelayBandwidthRate; - /* ensure_bandwidth_cap() makes sure that this cast can't overflow. */ return (uint32_t)bw; } @@ -1314,14 +1182,14 @@ options_act(or_options_t *old_options) return 0; /* Finish backgrounding the process */ - if (running_tor && options->RunAsDaemon) { + if (options->RunAsDaemon) { /* We may be calling this for the n'th time (on SIGHUP), but it's safe. */ finish_daemon(options->DataDirectory); } /* Write our PID to the PID file. If we do not have write permissions we * will log a warning */ - if (running_tor && options->PidFile) + if (options->PidFile) write_pidfile(options->PidFile); /* Register addressmap directives */ @@ -1354,18 +1222,53 @@ options_act(or_options_t *old_options) if (accounting_is_enabled(options)) configure_accounting(time(NULL)); + /* Change the cell EWMA settings */ + cell_ewma_set_scale_factor(options, networkstatus_get_latest_consensus()); + /* Check for transitions that need action. */ if (old_options) { - if (options->UseEntryGuards && !old_options->UseEntryGuards) { + + if ((options->UseEntryGuards && !old_options->UseEntryGuards) || + (options->ExcludeNodes && + !routerset_equal(old_options->ExcludeNodes,options->ExcludeNodes)) || + (options->ExcludeExitNodes && + !routerset_equal(old_options->ExcludeExitNodes, + options->ExcludeExitNodes)) || + (options->EntryNodes && + !routerset_equal(old_options->EntryNodes, options->EntryNodes)) || + (options->ExitNodes && + !routerset_equal(old_options->ExitNodes, options->ExitNodes)) || + options->StrictNodes != old_options->StrictNodes) { log_info(LD_CIRC, - "Switching to entry guards; abandoning previous circuits"); + "Changed to using entry guards, or changed preferred or " + "excluded node lists. Abandoning previous circuits."); circuit_mark_all_unused_circs(); circuit_expire_all_dirty_circs(); } +/* How long should we delay counting bridge stats after becoming a bridge? + * We use this so we don't count people who used our bridge thinking it is + * a relay. If you change this, don't forget to change the log message + * below. */ +#define RELAY_BRIDGE_STATS_DELAY (2 * 60 * 60) + if (! bool_eq(options->BridgeRelay, old_options->BridgeRelay)) { - log_info(LD_GENERAL, "Bridge status changed. Forgetting GeoIP stats."); - geoip_remove_old_clients(time(NULL)+(2*60*60)); + int was_relay = 0; + if (options->BridgeRelay) { + time_t int_start = time(NULL); + if (old_options->ORPort == options->ORPort) { + int_start += RELAY_BRIDGE_STATS_DELAY; + was_relay = 1; + } + geoip_bridge_stats_init(int_start); + log_info(LD_CONFIG, "We are acting as a bridge now. Starting new " + "GeoIP stats interval%s.", was_relay ? " in 2 " + "hours from now" : ""); + } else { + geoip_bridge_stats_term(); + log_info(LD_GENERAL, "We are no longer acting as a bridge. " + "Forgetting GeoIP stats."); + } } if (options_transition_affects_workers(old_options, options)) { @@ -1390,6 +1293,10 @@ options_act(or_options_t *old_options) if (options->V3AuthoritativeDir && !old_options->V3AuthoritativeDir) init_keys(); + + if (options->PerConnBWRate != old_options->PerConnBWRate || + options->PerConnBWBurst != old_options->PerConnBWBurst) + connection_or_update_token_buckets(get_connection_array(), options); } /* Maybe load geoip file */ @@ -1412,13 +1319,63 @@ options_act(or_options_t *old_options) geoip_load_file(actual_fname, options); tor_free(actual_fname); } -#ifdef ENABLE_GEOIP_STATS - log_warn(LD_CONFIG, "We are configured to measure GeoIP statistics, but " - "the way these statistics are measured has changed " - "significantly in later versions of Tor. The results may not be " - "as expected if you are used to later versions. Be sure you " - "know what you are doing."); -#endif + + if (options->DirReqStatistics && !geoip_is_loaded()) { + /* Check if GeoIP database could be loaded. */ + log_warn(LD_CONFIG, "Configured to measure directory request " + "statistics, but no GeoIP database found!"); + return -1; + } + + if (options->EntryStatistics) { + if (should_record_bridge_info(options)) { + /* Don't allow measuring statistics on entry guards when configured + * as bridge. */ + log_warn(LD_CONFIG, "Bridges cannot be configured to measure " + "additional GeoIP statistics as entry guards."); + return -1; + } else if (!geoip_is_loaded()) { + /* Check if GeoIP database could be loaded. */ + log_warn(LD_CONFIG, "Configured to measure entry node statistics, " + "but no GeoIP database found!"); + return -1; + } + } + + if (options->CellStatistics || options->DirReqStatistics || + options->EntryStatistics || options->ExitPortStatistics) { + time_t now = time(NULL); + if ((!old_options || !old_options->CellStatistics) && + options->CellStatistics) + rep_hist_buffer_stats_init(now); + if ((!old_options || !old_options->DirReqStatistics) && + options->DirReqStatistics) + geoip_dirreq_stats_init(now); + if ((!old_options || !old_options->EntryStatistics) && + options->EntryStatistics) + geoip_entry_stats_init(now); + if ((!old_options || !old_options->ExitPortStatistics) && + options->ExitPortStatistics) + rep_hist_exit_stats_init(now); + if (!old_options) + log_notice(LD_CONFIG, "Configured to measure statistics. Look for " + "the *-stats files that will first be written to the " + "data directory in 24 hours from now."); + } + + if (old_options && old_options->CellStatistics && + !options->CellStatistics) + rep_hist_buffer_stats_term(); + if (old_options && old_options->DirReqStatistics && + !options->DirReqStatistics) + geoip_dirreq_stats_term(); + if (old_options && old_options->EntryStatistics && + !options->EntryStatistics) + geoip_entry_stats_term(); + if (old_options && old_options->ExitPortStatistics && + !options->ExitPortStatistics) + rep_hist_exit_stats_term(); + /* Check if we need to parse and add the EntryNodes config option. */ if (options->EntryNodes && (!old_options || @@ -1491,7 +1448,10 @@ expand_abbrev(config_format_t *fmt, const char *option, int command_line, fmt->abbrevs[i].abbreviated, fmt->abbrevs[i].full); } - return fmt->abbrevs[i].full; + /* Keep going through the list in case we want to rewrite it more. + * (We could imagine recursing here, but I don't want to get the + * user into an infinite loop if we craft our list wrong.) */ + option = fmt->abbrevs[i].full; } } return option; @@ -1536,7 +1496,10 @@ config_get_commandlines(int argc, char **argv, config_line_t **result) *new = tor_malloc_zero(sizeof(config_line_t)); s = argv[i]; - while (*s == '-') + /* Each keyword may be prefixed with one or two dashes. */ + if (*s == '-') + s++; + if (*s == '-') s++; (*new)->key = tor_strdup(expand_abbrev(&options_format, s, 1, 1)); @@ -1628,19 +1591,6 @@ config_free_lines(config_line_t *front) } } -/** Return the description for a given configuration variable, or NULL if no - * description exists. */ -static const char * -config_find_description(config_format_t *fmt, const char *name) -{ - int i; - for (i=0; fmt->descriptions[i].name; ++i) { - if (!strcasecmp(name, fmt->descriptions[i].name)) - return fmt->descriptions[i].description; - } - return NULL; -} - /** If <b>key</b> is a configuration option, return the corresponding * config_var_t. Otherwise, if <b>key</b> is a non-standard abbreviation, * warn, and return the corresponding config_var_t. Otherwise return NULL. @@ -1671,6 +1621,16 @@ config_find_option(config_format_t *fmt, const char *key) return NULL; } +/** Return the number of option entries in <b>fmt</b>. */ +static int +config_count_options(config_format_t *fmt) +{ + int i; + for (i=0; fmt->vars[i].name; ++i) + ; + return i; +} + /* * Functions to assign config options. */ @@ -1684,8 +1644,7 @@ static int config_assign_value(config_format_t *fmt, or_options_t *options, config_line_t *c, char **msg) { - int i, r, ok; - char buf[1024]; + int i, ok; config_var_t *var; void *lvalue; @@ -1701,10 +1660,9 @@ config_assign_value(config_format_t *fmt, or_options_t *options, case CONFIG_TYPE_UINT: i = (int)tor_parse_long(c->value, 10, 0, INT_MAX, &ok, NULL); if (!ok) { - r = tor_snprintf(buf, sizeof(buf), + tor_asprintf(msg, "Int keyword '%s %s' is malformed or out of bounds.", c->key, c->value); - *msg = tor_strdup(r >= 0 ? buf : "internal error"); return -1; } *(int *)lvalue = i; @@ -1713,10 +1671,9 @@ config_assign_value(config_format_t *fmt, or_options_t *options, case CONFIG_TYPE_INTERVAL: { i = config_parse_interval(c->value, &ok); if (!ok) { - r = tor_snprintf(buf, sizeof(buf), + tor_asprintf(msg, "Interval '%s %s' is malformed or out of bounds.", c->key, c->value); - *msg = tor_strdup(r >= 0 ? buf : "internal error"); return -1; } *(int *)lvalue = i; @@ -1726,10 +1683,9 @@ config_assign_value(config_format_t *fmt, or_options_t *options, case CONFIG_TYPE_MEMUNIT: { uint64_t u64 = config_parse_memunit(c->value, &ok); if (!ok) { - r = tor_snprintf(buf, sizeof(buf), + tor_asprintf(msg, "Value '%s %s' is malformed or out of bounds.", c->key, c->value); - *msg = tor_strdup(r >= 0 ? buf : "internal error"); return -1; } *(uint64_t *)lvalue = u64; @@ -1739,10 +1695,9 @@ config_assign_value(config_format_t *fmt, or_options_t *options, case CONFIG_TYPE_BOOL: i = (int)tor_parse_long(c->value, 10, 0, 1, &ok, NULL); if (!ok) { - r = tor_snprintf(buf, sizeof(buf), + tor_asprintf(msg, "Boolean '%s %s' expects 0 or 1.", c->key, c->value); - *msg = tor_strdup(r >= 0 ? buf : "internal error"); return -1; } *(int *)lvalue = i; @@ -1760,9 +1715,8 @@ config_assign_value(config_format_t *fmt, or_options_t *options, case CONFIG_TYPE_ISOTIME: if (parse_iso_time(c->value, (time_t *)lvalue)) { - r = tor_snprintf(buf, sizeof(buf), + tor_asprintf(msg, "Invalid time '%s' for keyword '%s'", c->value, c->key); - *msg = tor_strdup(r >= 0 ? buf : "internal error"); return -1; } break; @@ -1773,9 +1727,8 @@ config_assign_value(config_format_t *fmt, or_options_t *options, } *(routerset_t**)lvalue = routerset_new(); if (routerset_parse(*(routerset_t**)lvalue, c->value, c->key)<0) { - tor_snprintf(buf, sizeof(buf), "Invalid exit list '%s' for option '%s'", + tor_asprintf(msg, "Invalid exit list '%s' for option '%s'", c->value, c->key); - *msg = tor_strdup(buf); return -1; } break; @@ -1800,9 +1753,8 @@ config_assign_value(config_format_t *fmt, or_options_t *options, log_warn(LD_CONFIG, "Skipping obsolete configuration option '%s'", c->key); break; case CONFIG_TYPE_LINELIST_V: - r = tor_snprintf(buf, sizeof(buf), + tor_asprintf(msg, "You may not provide a value for virtual option '%s'", c->key); - *msg = tor_strdup(r >= 0 ? buf : "internal error"); return -1; default: tor_assert(0); @@ -1823,7 +1775,7 @@ config_assign_value(config_format_t *fmt, or_options_t *options, static int config_assign_line(config_format_t *fmt, or_options_t *options, config_line_t *c, int use_defaults, - int clear_first, char **msg) + int clear_first, bitarray_t *options_seen, char **msg) { config_var_t *var; @@ -1838,13 +1790,12 @@ config_assign_line(config_format_t *fmt, or_options_t *options, config_line_append((config_line_t**)lvalue, c->key, c->value); return 0; } else { - char buf[1024]; - int tmp = tor_snprintf(buf, sizeof(buf), + tor_asprintf(msg, "Unknown option '%s'. Failing.", c->key); - *msg = tor_strdup(tmp >= 0 ? buf : "internal error"); return -1; } } + /* Put keyword into canonical case. */ if (strcmp(var->name, c->key)) { tor_free(c->key); @@ -1867,6 +1818,18 @@ config_assign_line(config_format_t *fmt, or_options_t *options, return 0; } + if (options_seen && (var->type != CONFIG_TYPE_LINELIST && + var->type != CONFIG_TYPE_LINELIST_S)) { + /* We're tracking which options we've seen, and this option is not + * supposed to occur more than once. */ + int var_index = (int)(var - fmt->vars); + if (bitarray_is_set(options_seen, var_index)) { + log_warn(LD_CONFIG, "Option '%s' used more than once; all but the last " + "value will be ignored.", var->name); + } + bitarray_set(options_seen, var_index); + } + if (config_assign_value(fmt, options, c, msg) < 0) return -2; return 0; @@ -1967,7 +1930,6 @@ get_assigned_option(config_format_t *fmt, void *options, { config_var_t *var; const void *value; - char buf[32]; config_line_t *result; tor_assert(options && key); @@ -2008,19 +1970,16 @@ get_assigned_option(config_format_t *fmt, void *options, case CONFIG_TYPE_UINT: /* This means every or_options_t uint or bool element * needs to be an int. Not, say, a uint16_t or char. */ - tor_snprintf(buf, sizeof(buf), "%d", *(int*)value); - result->value = tor_strdup(buf); + tor_asprintf(&result->value, "%d", *(int*)value); escape_val = 0; /* Can't need escape. */ break; case CONFIG_TYPE_MEMUNIT: - tor_snprintf(buf, sizeof(buf), U64_FORMAT, + tor_asprintf(&result->value, U64_FORMAT, U64_PRINTF_ARG(*(uint64_t*)value)); - result->value = tor_strdup(buf); escape_val = 0; /* Can't need escape. */ break; case CONFIG_TYPE_DOUBLE: - tor_snprintf(buf, sizeof(buf), "%f", *(double*)value); - result->value = tor_strdup(buf); + tor_asprintf(&result->value, "%f", *(double*)value); escape_val = 0; /* Can't need escape. */ break; case CONFIG_TYPE_BOOL: @@ -2139,6 +2098,8 @@ config_assign(config_format_t *fmt, void *options, config_line_t *list, int use_defaults, int clear_first, char **msg) { config_line_t *p; + bitarray_t *options_seen; + const int n_options = config_count_options(fmt); CHECK(fmt, options); @@ -2158,14 +2119,18 @@ config_assign(config_format_t *fmt, void *options, config_line_t *list, config_reset_line(fmt, options, p->key, use_defaults); } + options_seen = bitarray_init_zero(n_options); /* pass 3: assign. */ while (list) { int r; if ((r=config_assign_line(fmt, options, list, use_defaults, - clear_first, msg))) + clear_first, options_seen, msg))) { + bitarray_free(options_seen); return r; + } list = list->next; } + bitarray_free(options_seen); return 0; } @@ -2308,20 +2273,10 @@ list_torrc_options(void) smartlist_t *lines = smartlist_create(); for (i = 0; _option_vars[i].name; ++i) { config_var_t *var = &_option_vars[i]; - const char *desc; if (var->type == CONFIG_TYPE_OBSOLETE || var->type == CONFIG_TYPE_LINELIST_V) continue; - desc = config_find_description(&options_format, var->name); printf("%s\n", var->name); - if (desc) { - wrap_string(lines, desc, 76, " ", " "); - SMARTLIST_FOREACH(lines, char *, cp, { - printf("%s", cp); - tor_free(cp); - }); - smartlist_clear(lines); - } } smartlist_free(lines); } @@ -2340,7 +2295,7 @@ resolve_my_address(int warn_severity, or_options_t *options, uint32_t *addr_out, char **hostname_out) { struct in_addr in; - uint32_t addr; + uint32_t addr; /* host order */ char hostname[256]; int explicit_ip=1; int explicit_hostname=1; @@ -2370,8 +2325,8 @@ resolve_my_address(int warn_severity, or_options_t *options, if (tor_inet_aton(hostname, &in) == 0) { /* then we have to resolve it */ explicit_ip = 0; - if (tor_lookup_hostname(hostname, &addr)) { - uint32_t interface_ip; + if (tor_lookup_hostname(hostname, &addr)) { /* failed to resolve */ + uint32_t interface_ip; /* host order */ if (explicit_hostname) { log_fn(warn_severity, LD_CONFIG, @@ -2392,7 +2347,7 @@ resolve_my_address(int warn_severity, or_options_t *options, log_fn(notice_severity, LD_CONFIG, "Learned IP address '%s' for " "local interface. Using that.", tmpbuf); strlcpy(hostname, "<guessed from interfaces>", sizeof(hostname)); - } else { + } else { /* resolved hostname into addr */ in.s_addr = htonl(addr); if (!explicit_hostname && @@ -2567,7 +2522,10 @@ config_free(config_format_t *fmt, void *options) { int i; - tor_assert(options); + if (!options) + return; + + tor_assert(fmt); for (i=0; fmt->vars[i].name; ++i) option_clear(fmt, options, &(fmt->vars[i])); @@ -2669,6 +2627,8 @@ is_listening_on_low_port(uint16_t port_option, const config_line_t *listen_options) { #ifdef MS_WINDOWS + (void) port_option; + (void) listen_options; return 0; /* No port is too low for windows. */ #else const config_line_t *l; @@ -2718,7 +2678,6 @@ config_dump(config_format_t *fmt, void *options, int minimal, config_line_t *line, *assigned; char *result; int i; - const char *desc; char *msg = NULL; defaults = config_alloc(fmt); @@ -2746,24 +2705,13 @@ config_dump(config_format_t *fmt, void *options, int minimal, option_is_same(fmt, options, defaults, fmt->vars[i].name)) comment_option = 1; - desc = config_find_description(fmt, fmt->vars[i].name); line = assigned = get_assigned_option(fmt, options, fmt->vars[i].name, 1); - if (line && desc) { - /* Only dump the description if there's something to describe. */ - wrap_string(elements, desc, 78, "# ", "# "); - } - for (; line; line = line->next) { - size_t len = strlen(line->key) + strlen(line->value) + 5; char *tmp; - tmp = tor_malloc(len); - if (tor_snprintf(tmp, len, "%s%s %s\n", - comment_option ? "# " : "", - line->key, line->value)<0) { - log_err(LD_BUG,"Internal error writing option value"); - tor_assert(0); - } + tor_asprintf(&tmp, "%s%s %s\n", + comment_option ? "# " : "", + line->key, line->value); smartlist_add(elements, tmp); } config_free_lines(assigned); @@ -2772,13 +2720,8 @@ config_dump(config_format_t *fmt, void *options, int minimal, if (fmt->extra) { line = *(config_line_t**)STRUCT_VAR_P(options, fmt->extra->var_offset); for (; line; line = line->next) { - size_t len = strlen(line->key) + strlen(line->value) + 3; char *tmp; - tmp = tor_malloc(len); - if (tor_snprintf(tmp, len, "%s %s\n", line->key, line->value)<0) { - log_err(LD_BUG,"Internal error writing option value"); - tor_assert(0); - } + tor_asprintf(&tmp, "%s %s\n", line->key, line->value); smartlist_add(elements, tmp); } } @@ -2794,7 +2737,7 @@ config_dump(config_format_t *fmt, void *options, int minimal, * the configuration in <b>options</b>. If <b>minimal</b> is true, do not * include options that are the same as Tor's defaults. */ -static char * +char * options_dump(or_options_t *options, int minimal) { return config_dump(&options_format, options, minimal, 0); @@ -2807,7 +2750,6 @@ static int validate_ports_csv(smartlist_t *sl, const char *name, char **msg) { int i; - char buf[1024]; tor_assert(name); if (!sl) @@ -2817,9 +2759,7 @@ validate_ports_csv(smartlist_t *sl, const char *name, char **msg) { i = atoi(cp); if (i < 1 || i > 65535) { - int r = tor_snprintf(buf, sizeof(buf), - "Port '%s' out of range in %s", cp, name); - *msg = tor_strdup(r >= 0 ? buf : "internal error"); + tor_asprintf(msg, "Port '%s' out of range in %s", cp, name); return -1; } }); @@ -2833,18 +2773,15 @@ validate_ports_csv(smartlist_t *sl, const char *name, char **msg) static int ensure_bandwidth_cap(uint64_t *value, const char *desc, char **msg) { - int r; - char buf[1024]; if (*value > ROUTER_MAX_DECLARED_BANDWIDTH) { /* This handles an understandable special case where somebody says "2gb" * whereas our actual maximum is 2gb-1 (INT_MAX) */ --*value; } if (*value > ROUTER_MAX_DECLARED_BANDWIDTH) { - r = tor_snprintf(buf, sizeof(buf), "%s ("U64_FORMAT") must be at most %d", - desc, U64_PRINTF_ARG(*value), - ROUTER_MAX_DECLARED_BANDWIDTH); - *msg = tor_strdup(r >= 0 ? buf : "internal error"); + tor_asprintf(msg, "%s ("U64_FORMAT") must be at most %d", + desc, U64_PRINTF_ARG(*value), + ROUTER_MAX_DECLARED_BANDWIDTH); return -1; } return 0; @@ -2895,15 +2832,14 @@ compute_publishserverdescriptor(or_options_t *options) /** Highest allowable value for RendPostPeriod. */ #define MAX_DIR_PERIOD (MIN_ONION_KEY_LIFETIME/2) -/** Lowest allowable value for CircuitBuildTimeout; values too low will - * increase network load because of failing connections being retried, and - * might prevent users from connecting to the network at all. */ -#define MIN_CIRCUIT_BUILD_TIMEOUT 30 - /** Lowest allowable value for MaxCircuitDirtiness; if this is too low, Tor * will generate too many circuits and potentially overload the network. */ #define MIN_MAX_CIRCUIT_DIRTINESS 10 +/** Lowest allowable value for CircuitStreamTimeout; if this is too low, Tor + * will generate too many circuits and potentially overload the network. */ +#define MIN_CIRCUIT_STREAM_TIMEOUT 10 + /** Return 0 if every setting in <b>options</b> is reasonable, and a * permissible transition from <b>old_options</b>. Else return -1. * Should have no side effects, except for normalizing the contents of @@ -2920,10 +2856,9 @@ static int options_validate(or_options_t *old_options, or_options_t *options, int from_setconf, char **msg) { - int i, r; + int i; config_line_t *cl; const char *uname = get_uname(); - char buf[1024]; #define REJECT(arg) \ STMT_BEGIN *msg = tor_strdup(arg); return -1; STMT_END #define COMPLAIN(arg) STMT_BEGIN log(LOG_WARN, LD_CONFIG, arg); STMT_END @@ -2940,7 +2875,7 @@ options_validate(or_options_t *old_options, or_options_t *options, !strcmpstart(uname, "Windows Me"))) { log(LOG_WARN, LD_CONFIG, "Tor is running as a server, but you are " "running %s; this probably won't work. See " - "http://wiki.noreply.org/noreply/TheOnionRouter/TorFAQ#ServerOS " + "https://wiki.torproject.org/TheOnionRouter/TorFAQ#ServerOS " "for details.", uname); } @@ -3018,10 +2953,9 @@ options_validate(or_options_t *old_options, or_options_t *options, } } else { if (!is_legal_nickname(options->Nickname)) { - r = tor_snprintf(buf, sizeof(buf), + tor_asprintf(msg, "Nickname '%s' is wrong length or contains illegal characters.", options->Nickname); - *msg = tor_strdup(r >= 0 ? buf : "internal error"); return -1; } } @@ -3108,19 +3042,11 @@ options_validate(or_options_t *old_options, or_options_t *options, routerset_union(options->_ExcludeExitNodesUnion,options->ExcludeNodes); } - if (options->StrictExitNodes && - (!options->ExitNodes) && - (!old_options || - (old_options->StrictExitNodes != options->StrictExitNodes) || - (!routerset_equal(old_options->ExitNodes,options->ExitNodes)))) - COMPLAIN("StrictExitNodes set, but no ExitNodes listed."); - - if (options->StrictEntryNodes && - (!options->EntryNodes) && - (!old_options || - (old_options->StrictEntryNodes != options->StrictEntryNodes) || - (!routerset_equal(old_options->EntryNodes,options->EntryNodes)))) - COMPLAIN("StrictEntryNodes set, but no EntryNodes listed."); + if (options->ExcludeNodes && options->StrictNodes) { + COMPLAIN("You have asked to exclude certain relays from all positions " + "in your circuits. Expect hidden services and other Tor " + "features to be broken in unpredictable ways."); + } if (options->EntryNodes && !routerset_is_list(options->EntryNodes)) { /* XXXX fix this; see entry_guards_prepend_from_config(). */ @@ -3158,6 +3084,10 @@ options_validate(or_options_t *old_options, or_options_t *options, options->V3AuthoritativeDir)) REJECT("AuthoritativeDir is set, but none of " "(Bridge/HS/V1/V2/V3)AuthoritativeDir is set."); + /* If we have a v3bandwidthsfile and it's broken, complain on startup */ + if (options->V3BandwidthsFile && !old_options) { + dirserv_read_measured_bandwidths(options->V3BandwidthsFile, NULL); + } } if (options->AuthoritativeDir && !options->DirPort) @@ -3169,15 +3099,14 @@ options_validate(or_options_t *old_options, or_options_t *options, if (options->AuthoritativeDir && options->ClientOnly) REJECT("Running as authoritative directory, but ClientOnly also set."); - if (options->HSAuthorityRecordStats && !options->HSAuthoritativeDir) - REJECT("HSAuthorityRecordStats is set but we're not running as " - "a hidden service authority."); + if (options->FetchDirInfoExtraEarly && !options->FetchDirInfoEarly) + REJECT("FetchDirInfoExtraEarly requires that you also set " + "FetchDirInfoEarly"); if (options->ConnLimit <= 0) { - r = tor_snprintf(buf, sizeof(buf), + tor_asprintf(msg, "ConnLimit must be greater than 0, but was set to %d", options->ConnLimit); - *msg = tor_strdup(r >= 0 ? buf : "internal error"); return -1; } @@ -3296,18 +3225,29 @@ options_validate(or_options_t *old_options, or_options_t *options, else if (!strcasecmp(cp, "rendezvous")) options->_AllowInvalid |= ALLOW_INVALID_RENDEZVOUS; else { - r = tor_snprintf(buf, sizeof(buf), + tor_asprintf(msg, "Unrecognized value '%s' in AllowInvalidNodes", cp); - *msg = tor_strdup(r >= 0 ? buf : "internal error"); return -1; } }); } + if (!options->SafeLogging || + !strcasecmp(options->SafeLogging, "0")) { + options->_SafeLogging = SAFELOG_SCRUB_NONE; + } else if (!strcasecmp(options->SafeLogging, "relay")) { + options->_SafeLogging = SAFELOG_SCRUB_RELAY; + } else if (!strcasecmp(options->SafeLogging, "1")) { + options->_SafeLogging = SAFELOG_SCRUB_ALL; + } else { + tor_asprintf(msg, + "Unrecognized value '%s' in SafeLogging", + escaped(options->SafeLogging)); + return -1; + } + if (compute_publishserverdescriptor(options) < 0) { - r = tor_snprintf(buf, sizeof(buf), - "Unrecognized value in PublishServerDescriptor"); - *msg = tor_strdup(r >= 0 ? buf : "internal error"); + tor_asprintf(msg, "Unrecognized value in PublishServerDescriptor"); return -1; } @@ -3327,29 +3267,30 @@ options_validate(or_options_t *old_options, or_options_t *options, } if (options->RendPostPeriod < MIN_REND_POST_PERIOD) { - log(LOG_WARN,LD_CONFIG,"RendPostPeriod option is too short; " - "raising to %d seconds.", MIN_REND_POST_PERIOD); + log_warn(LD_CONFIG, "RendPostPeriod option is too short; " + "raising to %d seconds.", MIN_REND_POST_PERIOD); options->RendPostPeriod = MIN_REND_POST_PERIOD; } if (options->RendPostPeriod > MAX_DIR_PERIOD) { - log(LOG_WARN, LD_CONFIG, "RendPostPeriod is too large; clipping to %ds.", - MAX_DIR_PERIOD); + log_warn(LD_CONFIG, "RendPostPeriod is too large; clipping to %ds.", + MAX_DIR_PERIOD); options->RendPostPeriod = MAX_DIR_PERIOD; } - if (options->CircuitBuildTimeout < MIN_CIRCUIT_BUILD_TIMEOUT) { - log(LOG_WARN, LD_CONFIG, "CircuitBuildTimeout option is too short; " - "raising to %d seconds.", MIN_CIRCUIT_BUILD_TIMEOUT); - options->CircuitBuildTimeout = MIN_CIRCUIT_BUILD_TIMEOUT; - } - if (options->MaxCircuitDirtiness < MIN_MAX_CIRCUIT_DIRTINESS) { - log(LOG_WARN, LD_CONFIG, "MaxCircuitDirtiness option is too short; " - "raising to %d seconds.", MIN_MAX_CIRCUIT_DIRTINESS); + log_warn(LD_CONFIG, "MaxCircuitDirtiness option is too short; " + "raising to %d seconds.", MIN_MAX_CIRCUIT_DIRTINESS); options->MaxCircuitDirtiness = MIN_MAX_CIRCUIT_DIRTINESS; } + if (options->CircuitStreamTimeout && + options->CircuitStreamTimeout < MIN_CIRCUIT_STREAM_TIMEOUT) { + log_warn(LD_CONFIG, "CircuitStreamTimeout option is too short; " + "raising to %d seconds.", MIN_CIRCUIT_STREAM_TIMEOUT); + options->CircuitStreamTimeout = MIN_CIRCUIT_STREAM_TIMEOUT; + } + if (options->KeepalivePeriod < 1) REJECT("KeepalivePeriod option must be positive."); @@ -3368,34 +3309,37 @@ options_validate(or_options_t *old_options, or_options_t *options, if (ensure_bandwidth_cap(&options->RelayBandwidthBurst, "RelayBandwidthBurst", msg) < 0) return -1; + if (ensure_bandwidth_cap(&options->PerConnBWRate, + "PerConnBWRate", msg) < 0) + return -1; + if (ensure_bandwidth_cap(&options->PerConnBWBurst, + "PerConnBWBurst", msg) < 0) + return -1; if (server_mode(options)) { if (options->BandwidthRate < ROUTER_REQUIRED_MIN_BANDWIDTH) { - r = tor_snprintf(buf, sizeof(buf), + tor_asprintf(msg, "BandwidthRate is set to %d bytes/second. " "For servers, it must be at least %d.", (int)options->BandwidthRate, ROUTER_REQUIRED_MIN_BANDWIDTH); - *msg = tor_strdup(r >= 0 ? buf : "internal error"); return -1; } else if (options->MaxAdvertisedBandwidth < ROUTER_REQUIRED_MIN_BANDWIDTH/2) { - r = tor_snprintf(buf, sizeof(buf), + tor_asprintf(msg, "MaxAdvertisedBandwidth is set to %d bytes/second. " "For servers, it must be at least %d.", (int)options->MaxAdvertisedBandwidth, ROUTER_REQUIRED_MIN_BANDWIDTH/2); - *msg = tor_strdup(r >= 0 ? buf : "internal error"); return -1; } if (options->RelayBandwidthRate && options->RelayBandwidthRate < ROUTER_REQUIRED_MIN_BANDWIDTH) { - r = tor_snprintf(buf, sizeof(buf), + tor_asprintf(msg, "RelayBandwidthRate is set to %d bytes/second. " "For servers, it must be at least %d.", (int)options->RelayBandwidthRate, ROUTER_REQUIRED_MIN_BANDWIDTH); - *msg = tor_strdup(r >= 0 ? buf : "internal error"); return -1; } } @@ -3421,7 +3365,7 @@ options_validate(or_options_t *old_options, or_options_t *options, REJECT("Failed to parse accounting options. See logs for details."); if (options->HttpProxy) { /* parse it now */ - if (parse_addr_port(LOG_WARN, options->HttpProxy, NULL, + if (tor_addr_port_parse(options->HttpProxy, &options->HttpProxyAddr, &options->HttpProxyPort) < 0) REJECT("HttpProxy failed to parse or resolve. Please fix."); if (options->HttpProxyPort == 0) { /* give it a default */ @@ -3435,7 +3379,7 @@ options_validate(or_options_t *old_options, or_options_t *options, } if (options->HttpsProxy) { /* parse it now */ - if (parse_addr_port(LOG_WARN, options->HttpsProxy, NULL, + if (tor_addr_port_parse(options->HttpsProxy, &options->HttpsProxyAddr, &options->HttpsProxyPort) <0) REJECT("HttpsProxy failed to parse or resolve. Please fix."); if (options->HttpsProxyPort == 0) { /* give it a default */ @@ -3448,6 +3392,45 @@ options_validate(or_options_t *old_options, or_options_t *options, REJECT("HttpsProxyAuthenticator is too long (>= 48 chars)."); } + if (options->Socks4Proxy) { /* parse it now */ + if (tor_addr_port_parse(options->Socks4Proxy, + &options->Socks4ProxyAddr, + &options->Socks4ProxyPort) <0) + REJECT("Socks4Proxy failed to parse or resolve. Please fix."); + if (options->Socks4ProxyPort == 0) { /* give it a default */ + options->Socks4ProxyPort = 1080; + } + } + + if (options->Socks5Proxy) { /* parse it now */ + if (tor_addr_port_parse(options->Socks5Proxy, + &options->Socks5ProxyAddr, + &options->Socks5ProxyPort) <0) + REJECT("Socks5Proxy failed to parse or resolve. Please fix."); + if (options->Socks5ProxyPort == 0) { /* give it a default */ + options->Socks5ProxyPort = 1080; + } + } + + if (options->Socks4Proxy && options->Socks5Proxy) + REJECT("You cannot specify both Socks4Proxy and SOCKS5Proxy"); + + if (options->Socks5ProxyUsername) { + size_t len; + + len = strlen(options->Socks5ProxyUsername); + if (len < 1 || len > 255) + REJECT("Socks5ProxyUsername must be between 1 and 255 characters."); + + if (!options->Socks5ProxyPassword) + REJECT("Socks5ProxyPassword must be included with Socks5ProxyUsername."); + + len = strlen(options->Socks5ProxyPassword); + if (len < 1 || len > 255) + REJECT("Socks5ProxyPassword must be between 1 and 255 characters."); + } else if (options->Socks5ProxyPassword) + REJECT("Socks5ProxyPassword must be included with Socks5ProxyUsername."); + if (options->HashedControlPassword) { smartlist_t *sl = decode_hashed_passwords(options->HashedControlPassword); if (!sl) { @@ -3544,11 +3527,10 @@ options_validate(or_options_t *old_options, or_options_t *options, if (options->ConstrainedSockSize < MIN_CONSTRAINED_TCP_BUFFER || options->ConstrainedSockSize > MAX_CONSTRAINED_TCP_BUFFER || options->ConstrainedSockSize % 1024) { - r = tor_snprintf(buf, sizeof(buf), + tor_asprintf(msg, "ConstrainedSockSize is invalid. Must be a value between %d and %d " "in 1024 byte increments.", MIN_CONSTRAINED_TCP_BUFFER, MAX_CONSTRAINED_TCP_BUFFER); - *msg = tor_strdup(r >= 0 ? buf : "internal error"); return -1; } if (options->DirPort) { @@ -3596,6 +3578,12 @@ options_validate(or_options_t *old_options, or_options_t *options, if (options->PreferTunneledDirConns && !options->TunnelDirConns) REJECT("Must set TunnelDirConns if PreferTunneledDirConns is set."); + if ((options->Socks4Proxy || options->Socks5Proxy) && + !options->HttpProxy && !options->PreferTunneledDirConns) + REJECT("When Socks4Proxy or Socks5Proxy is configured, " + "PreferTunneledDirConns and TunnelDirConns must both be " + "set to 1, or HttpProxy must be configured."); + if (options->AutomapHostsSuffixes) { SMARTLIST_FOREACH(options->AutomapHostsSuffixes, char *, suf, { @@ -3675,6 +3663,26 @@ options_validate(or_options_t *old_options, or_options_t *options, "testing Tor network!"); } + if (options->AccelName && !options->HardwareAccel) + options->HardwareAccel = 1; + if (options->AccelDir && !options->AccelName) + REJECT("Can't use hardware crypto accelerator dir without engine name."); + + if (options->PublishServerDescriptor) + SMARTLIST_FOREACH(options->PublishServerDescriptor, const char *, pubdes, { + if (!strcmp(pubdes, "1") || !strcmp(pubdes, "0")) + if (smartlist_len(options->PublishServerDescriptor) > 1) { + COMPLAIN("You have passed a list of multiple arguments to the " + "PublishServerDescriptor option that includes 0 or 1. " + "0 or 1 should only be used as the sole argument. " + "This configuration will be rejected in a future release."); + break; + } + }); + + if (options->BridgeRelay == 1 && options->ORPort == 0) + REJECT("BridgeRelay is 1, ORPort is 0. This is an invalid combination."); + return 0; #undef REJECT #undef COMPLAIN @@ -3713,12 +3721,10 @@ options_transition_allowed(or_options_t *old, or_options_t *new_val, } if (strcmp(old->DataDirectory,new_val->DataDirectory)!=0) { - char buf[1024]; - int r = tor_snprintf(buf, sizeof(buf), + tor_asprintf(msg, "While Tor is running, changing DataDirectory " "(\"%s\"->\"%s\") is not allowed.", old->DataDirectory, new_val->DataDirectory); - *msg = tor_strdup(r >= 0 ? buf : "internal error"); return -1; } @@ -3727,19 +3733,22 @@ options_transition_allowed(or_options_t *old, or_options_t *new_val, return -1; } - if (!opt_streq(old->Group, new_val->Group)) { - *msg = tor_strdup("While Tor is running, changing Group is not allowed."); + if ((old->HardwareAccel != new_val->HardwareAccel) + || !opt_streq(old->AccelName, new_val->AccelName) + || !opt_streq(old->AccelDir, new_val->AccelDir)) { + *msg = tor_strdup("While Tor is running, changing OpenSSL hardware " + "acceleration engine is not allowed."); return -1; } - if (old->HardwareAccel != new_val->HardwareAccel) { - *msg = tor_strdup("While Tor is running, changing HardwareAccel is " - "not allowed."); + if (old->TestingTorNetwork != new_val->TestingTorNetwork) { + *msg = tor_strdup("While Tor is running, changing TestingTorNetwork " + "is not allowed."); return -1; } - if (old->TestingTorNetwork != new_val->TestingTorNetwork) { - *msg = tor_strdup("While Tor is running, changing TestingTorNetwork " + if (old->DisableAllSwap != new_val->DisableAllSwap) { + *msg = tor_strdup("While Tor is running, changing DisableAllSwap " "is not allowed."); return -1; } @@ -3809,6 +3818,7 @@ get_windows_conf_root(void) { static int is_set = 0; static char path[MAX_PATH+1]; + WCHAR wpath[MAX_PATH] = {0}; LPITEMIDLIST idl; IMalloc *m; @@ -3826,7 +3836,7 @@ get_windows_conf_root(void) #define APPDATA_PATH CSIDL_APPDATA #endif if (!SUCCEEDED(SHGetSpecialFolderLocation(NULL, APPDATA_PATH, &idl))) { - GetCurrentDirectory(MAX_PATH, path); + getcwd(path,MAX_PATH); is_set = 1; log_warn(LD_CONFIG, "I couldn't find your application data folder: are you " @@ -3835,8 +3845,11 @@ get_windows_conf_root(void) return path; } /* Convert the path from an "ID List" (whatever that is!) to a path. */ - result = SHGetPathFromIDList(idl, path); - /* Now we need to free the */ + result = SHGetPathFromIDListW(idl, wpath); + wcstombs(path,wpath,MAX_PATH); + + /* Now we need to free the memory that the path-idl was stored in. In + * typical Windows fashion, we can't just call 'free()' on it. */ SHGetMalloc(&m); if (m) { m->lpVtbl->Free(m, idl); @@ -3884,10 +3897,7 @@ check_nickname_list(const char *lst, const char *name, char **msg) SMARTLIST_FOREACH(sl, const char *, s, { if (!is_legal_nickname_or_hexdigest(s)) { - char buf[1024]; - int tmp = tor_snprintf(buf, sizeof(buf), - "Invalid nickname '%s' in %s line", s, name); - *msg = tor_strdup(tmp >= 0 ? buf : "internal error"); + tor_asprintf(msg, "Invalid nickname '%s' in %s line", s, name); r = -1; break; } @@ -3911,13 +3921,7 @@ find_torrc_filename(int argc, char **argv, log(LOG_WARN, LD_CONFIG, "Duplicate -f options on command line."); tor_free(fname); } -#ifdef MS_WINDOWS - /* XXX one day we might want to extend expand_filename to work - * under Windows as well. */ - fname = tor_strdup(argv[i+1]); -#else fname = expand_filename(argv[i+1]); -#endif *using_default_torrc = 0; ++i; } else if (!strcmp(argv[i],"--ignore-missing-torrc")) { @@ -4023,6 +4027,12 @@ options_init_from_torrc(int argc, char **argv) printf("Tor version %s.\n",get_version()); exit(0); } + if (argc > 1 && (!strcmp(argv[1],"--digests"))) { + printf("Tor version %s.\n",get_version()); + printf("%s", libor_get_digests()); + printf("%s", tor_get_digests()); + exit(0); + } /* Go through command-line variables */ if (!global_cmdline_options) { @@ -4190,12 +4200,9 @@ options_init_from_string(const char *cf, err: config_free(&options_format, newoptions); if (*msg) { - int len = (int)strlen(*msg)+256; - char *newmsg = tor_malloc(len); - - tor_snprintf(newmsg, len, "Failed to parse/validate config: %s", *msg); - tor_free(*msg); - *msg = newmsg; + char *old_msg = *msg; + tor_asprintf(msg, "Failed to parse/validate config: %s", old_msg); + tor_free(old_msg); } return err; } @@ -4583,7 +4590,7 @@ normalize_data_directory(or_options_t *options) } /** Check and normalize the value of options->DataDirectory; return 0 if it - * sane, -1 otherwise. */ + * is sane, -1 otherwise. */ static int validate_data_directory(or_options_t *options) { @@ -4615,7 +4622,6 @@ write_configuration_file(const char *fname, or_options_t *options) { char *old_val=NULL, *new_val=NULL, *new_conf=NULL; int rename_old = 0, r; - size_t len; tor_assert(fname); @@ -4642,9 +4648,7 @@ write_configuration_file(const char *fname, or_options_t *options) goto err; } - len = strlen(new_conf)+256; - new_val = tor_malloc(len); - tor_snprintf(new_val, len, "%s\n%s\n\n%s", + tor_asprintf(&new_val, "%s\n%s\n\n%s", GENERATED_FILE_PREFIX, GENERATED_FILE_COMMENT, new_conf); if (rename_old) { @@ -4694,15 +4698,12 @@ write_configuration_file(const char *fname, or_options_t *options) int options_save_current(void) { - if (torrc_fname) { - /* This fails if we can't write to our configuration file. - * - * If we try falling back to datadirectory or something, we have a better - * chance of saving the configuration, but a better chance of doing - * something the user never expected. Let's just warn instead. */ - return write_configuration_file(torrc_fname, get_options()); - } - return write_configuration_file(get_default_conf_file(), get_options()); + /* This fails if we can't write to our configuration file. + * + * If we try falling back to datadirectory or something, we have a better + * chance of saving the configuration, but a better chance of doing + * something the user never expected. */ + return write_configuration_file(get_torrc_fname(), get_options()); } /** Mapping from a unit name to a multiplier for converting that unit into a @@ -4767,30 +4768,47 @@ static struct unit_table_t time_units[] = { static uint64_t config_parse_units(const char *val, struct unit_table_t *u, int *ok) { - uint64_t v; + uint64_t v = 0; + double d = 0; + int use_float = 0; char *cp; tor_assert(ok); v = tor_parse_uint64(val, 10, 0, UINT64_MAX, ok, &cp); - if (!*ok) - return 0; + if (!*ok || (cp && *cp == '.')) { + d = tor_parse_double(val, 0, UINT64_MAX, ok, &cp); + if (!*ok) + goto done; + use_float = 1; + } + if (!cp) { *ok = 1; - return v; + v = use_float ? DBL_TO_U64(d) : v; + goto done; } - while (TOR_ISSPACE(*cp)) - ++cp; + + cp = (char*) eat_whitespace(cp); + for ( ;u->unit;++u) { if (!strcasecmp(u->unit, cp)) { - v *= u->multiplier; + if (use_float) + v = u->multiplier * d; + else + v *= u->multiplier; *ok = 1; - return v; + goto done; } } log_warn(LD_CONFIG, "Unknown unit '%s'.", cp); *ok = 0; - return 0; + done: + + if (*ok) + return v; + else + return 0; } /** Parse a string in the format "number unit", where unit is a unit of @@ -4800,7 +4818,8 @@ config_parse_units(const char *val, struct unit_table_t *u, int *ok) static uint64_t config_parse_memunit(const char *s, int *ok) { - return config_parse_units(s, memory_units, ok); + uint64_t u = config_parse_units(s, memory_units, ok); + return u; } /** Parse a string in the format "number unit", where unit is a unit of time. @@ -4822,256 +4841,37 @@ config_parse_interval(const char *s, int *ok) return (int)r; } -/* This is what passes for version detection on OSX. We set - * MACOSX_KQUEUE_IS_BROKEN to true iff we're on a version of OSX before - * 10.4.0 (aka 1040). */ -#ifdef __APPLE__ -#ifdef __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ -#define MACOSX_KQUEUE_IS_BROKEN \ - (__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 1040) -#else -#define MACOSX_KQUEUE_IS_BROKEN 0 -#endif -#endif - /** * Initialize the libevent library. */ static void init_libevent(void) { + const char *badness=NULL; + configure_libevent_logging(); /* If the kernel complains that some method (say, epoll) doesn't * exist, we don't care about it, since libevent will cope. */ suppress_libevent_log_msg("Function not implemented"); -#ifdef __APPLE__ - if (MACOSX_KQUEUE_IS_BROKEN || - decode_libevent_version(event_get_version(), NULL) < LE_11B) { - setenv("EVENT_NOKQUEUE","1",1); - } -#endif - /* In libevent versions before 2.0, it's hard to keep binary compatibility - * between upgrades, and unpleasant to detect when the version we compiled - * against is unlike the version we have linked against. Here's how. */ -#if defined(_EVENT_VERSION) && defined(HAVE_EVENT_GET_VERSION) - /* We have a header-file version and a function-call version. Easy. */ - if (strcmp(_EVENT_VERSION, event_get_version())) { - int compat1 = -1, compat2 = -1; - int verybad, prettybad ; - decode_libevent_version(_EVENT_VERSION, &compat1); - decode_libevent_version(event_get_version(), &compat2); - verybad = compat1 != compat2; - prettybad = (compat1 == -1 || compat2 == -1) && compat1 != compat2; - - log(verybad ? LOG_WARN : (prettybad ? LOG_NOTICE : LOG_INFO), - LD_GENERAL, "We were compiled with headers from version %s " - "of Libevent, but we're using a Libevent library that says it's " - "version %s.", _EVENT_VERSION, event_get_version()); - if (verybad) - log_warn(LD_GENERAL, "This will almost certainly make Tor crash."); - else if (prettybad) - log_notice(LD_GENERAL, "If Tor crashes, this might be why."); - else - log_info(LD_GENERAL, "I think these versions are binary-compatible."); - } -#elif defined(HAVE_EVENT_GET_VERSION) - /* event_get_version but no _EVENT_VERSION. We might be in 1.4.0-beta or - earlier, where that's normal. To see whether we were compiled with an - earlier version, let's see whether the struct event defines MIN_HEAP_IDX. - */ -#ifdef HAVE_STRUCT_EVENT_MIN_HEAP_IDX - /* The header files are 1.4.0-beta or later. If the version is not - * 1.4.0-beta, we are incompatible. */ - { - if (strcmp(event_get_version(), "1.4.0-beta")) { - log_warn(LD_GENERAL, "It's a little hard to tell, but you seem to have " - "Libevent 1.4.0-beta header files, whereas you have linked " - "against Libevent %s. This will probably make Tor crash.", - event_get_version()); - } - } -#else - /* Our headers are 1.3e or earlier. If the library version is not 1.4.x or - later, we're probably fine. */ - { - const char *v = event_get_version(); - if ((v[0] == '1' && v[2] == '.' && v[3] > '3') || v[0] > '1') { - log_warn(LD_GENERAL, "It's a little hard to tell, but you seem to have " - "Libevent header file from 1.3e or earlier, whereas you have " - "linked against Libevent %s. This will probably make Tor " - "crash.", event_get_version()); - } - } -#endif + tor_check_libevent_header_compatibility(); -#elif defined(_EVENT_VERSION) -#warn "_EVENT_VERSION is defined but not get_event_version(): Libevent is odd." -#else - /* Your libevent is ancient. */ -#endif + tor_libevent_initialize(); - event_init(); suppress_libevent_log_msg(NULL); -#if defined(HAVE_EVENT_GET_VERSION) && defined(HAVE_EVENT_GET_METHOD) - /* Making this a NOTICE for now so we can link bugs to a libevent versions - * or methods better. */ - log(LOG_NOTICE, LD_GENERAL, - "Initialized libevent version %s using method %s. Good.", - event_get_version(), event_get_method()); - check_libevent_version(event_get_method(), get_options()->ORPort != 0); -#else - log(LOG_NOTICE, LD_GENERAL, - "Initialized old libevent (version 1.0b or earlier)."); - log(LOG_WARN, LD_GENERAL, - "You have a *VERY* old version of libevent. It is likely to be buggy; " - "please build Tor with a more recent version."); -#endif -} - -/** Table mapping return value of event_get_version() to le_version_t. */ -static const struct { - const char *name; le_version_t version; int bincompat; -} le_version_table[] = { - /* earlier versions don't have get_version. */ - { "1.0c", LE_10C, 1}, - { "1.0d", LE_10D, 1}, - { "1.0e", LE_10E, 1}, - { "1.1", LE_11, 1 }, - { "1.1a", LE_11A, 1 }, - { "1.1b", LE_11B, 1 }, - { "1.2", LE_12, 1 }, - { "1.2a", LE_12A, 1 }, - { "1.3", LE_13, 1 }, - { "1.3a", LE_13A, 1 }, - { "1.3b", LE_13B, 1 }, - { "1.3c", LE_13C, 1 }, - { "1.3d", LE_13D, 1 }, - { "1.3e", LE_13E, 1 }, - { "1.4.0-beta", LE_140, 2 }, - { "1.4.1-beta", LE_141, 2 }, - { "1.4.2-rc", LE_142, 2 }, - { "1.4.3-stable", LE_143, 2 }, - { "1.4.4-stable", LE_144, 2 }, - { "1.4.5-stable", LE_145, 2 }, - { "1.4.6-stable", LE_146, 2 }, - { "1.4.7-stable", LE_147, 2 }, - { "1.4.8-stable", LE_148, 2 }, - { "1.4.99-trunk", LE_1499, 3 }, - { NULL, LE_OTHER, 0 } -}; -/** Return the le_version_t for the current version of libevent. If the - * version is very new, return LE_OTHER. If the version is so old that it - * doesn't support event_get_version(), return LE_OLD. */ -static le_version_t -decode_libevent_version(const char *v, int *bincompat_out) -{ - int i; - for (i=0; le_version_table[i].name; ++i) { - if (!strcmp(le_version_table[i].name, v)) { - if (bincompat_out) - *bincompat_out = le_version_table[i].bincompat; - return le_version_table[i].version; - } - } - if (v[0] != '1' && bincompat_out) - *bincompat_out = 100; - else if (!strcmpstart(v, "1.4") && bincompat_out) - *bincompat_out = 2; - return LE_OTHER; -} - -#if defined(HAVE_EVENT_GET_VERSION) && defined(HAVE_EVENT_GET_METHOD) -/** - * Compare the given libevent method and version to a list of versions - * which are known not to work. Warn the user as appropriate. - */ -static void -check_libevent_version(const char *m, int server) -{ - int buggy = 0, iffy = 0, slow = 0, thread_unsafe = 0; - le_version_t version; - const char *v = event_get_version(); - const char *badness = NULL; - const char *sad_os = ""; - - version = decode_libevent_version(v, NULL); - - /* XXX Would it be worthwhile disabling the methods that we know - * are buggy, rather than just warning about them and then proceeding - * to use them? If so, we should probably not wrap this whole thing - * in HAVE_EVENT_GET_VERSION and HAVE_EVENT_GET_METHOD. -RD */ - /* XXXX The problem is that it's not trivial to get libevent to change it's - * method once it's initialized, and it's not trivial to tell what method it - * will use without initializing it. I guess we could preemptively disable - * buggy libevent modes based on the version _before_ initializing it, - * though, but then there's no good way (afaict) to warn "I would have used - * kqueue, but instead I'm using select." -NM */ - if (!strcmp(m, "kqueue")) { - if (version < LE_11B) - buggy = 1; - } else if (!strcmp(m, "epoll")) { - if (version < LE_11) - iffy = 1; - } else if (!strcmp(m, "poll")) { - if (version < LE_10E) - buggy = 1; - else if (version < LE_11) - slow = 1; - } else if (!strcmp(m, "select")) { - if (version < LE_11) - slow = 1; - } else if (!strcmp(m, "win32")) { - if (version < LE_11B) - buggy = 1; - } - - /* Libevent versions before 1.3b do very badly on operating systems with - * user-space threading implementations. */ -#if defined(__OpenBSD__) || defined(__FreeBSD__) || defined(__NetBSD__) - if (server && version < LE_13B) { - thread_unsafe = 1; - sad_os = "BSD variants"; - } -#elif defined(__APPLE__) || defined(__darwin__) - if (server && version < LE_13B) { - thread_unsafe = 1; - sad_os = "Mac OS X"; - } -#endif - - if (thread_unsafe) { - log(LOG_WARN, LD_GENERAL, - "Libevent version %s often crashes when running a Tor server with %s. " - "Please use the latest version of libevent (1.3b or later)",v,sad_os); - badness = "BROKEN"; - } else if (buggy) { - log(LOG_WARN, LD_GENERAL, - "There are serious bugs in using %s with libevent %s. " - "Please use the latest version of libevent.", m, v); - badness = "BROKEN"; - } else if (iffy) { - log(LOG_WARN, LD_GENERAL, - "There are minor bugs in using %s with libevent %s. " - "You may want to use the latest version of libevent.", m, v); - badness = "BUGGY"; - } else if (slow && server) { - log(LOG_WARN, LD_GENERAL, - "libevent %s can be very slow with %s. " - "When running a server, please use the latest version of libevent.", - v,m); - badness = "SLOW"; - } + tor_check_libevent_version(tor_libevent_get_method(), + get_options()->ORPort != 0, + &badness); if (badness) { + const char *v = tor_libevent_get_version_str(); + const char *m = tor_libevent_get_method(); control_event_general_status(LOG_WARN, "BAD_LIBEVENT VERSION=%s METHOD=%s BADNESS=%s RECOVERED=NO", v, m, badness); } - } -#endif /** Return the persistent state struct for this Tor. */ or_state_t * @@ -5153,22 +4953,63 @@ or_state_validate(or_state_t *old_state, or_state_t *state, } /** Replace the current persistent state with <b>new_state</b> */ -static void +static int or_state_set(or_state_t *new_state) { char *err = NULL; + int ret = 0; tor_assert(new_state); - if (global_state) - config_free(&state_format, global_state); + config_free(&state_format, global_state); global_state = new_state; if (entry_guards_parse_state(global_state, 1, &err)<0) { log_warn(LD_GENERAL,"%s",err); tor_free(err); + ret = -1; } if (rep_hist_load_state(global_state, &err)<0) { log_warn(LD_GENERAL,"Unparseable bandwidth history state: %s",err); tor_free(err); + ret = -1; } + if (circuit_build_times_parse_state(&circ_times, global_state, &err) < 0) { + log_warn(LD_GENERAL,"%s",err); + tor_free(err); + ret = -1; + } + return ret; +} + +/** + * Save a broken state file to a backup location. + */ +static void +or_state_save_broken(char *fname) +{ + int i; + file_status_t status; + size_t len = strlen(fname)+16; + char *fname2 = tor_malloc(len); + for (i = 0; i < 100; ++i) { + tor_snprintf(fname2, len, "%s.%d", fname, i); + status = file_status(fname2); + if (status == FN_NOENT) + break; + } + if (i == 100) { + log_warn(LD_BUG, "Unable to parse state in \"%s\"; too many saved bad " + "state files to move aside. Discarding the old state file.", + fname); + unlink(fname); + } else { + log_warn(LD_BUG, "Unable to parse state in \"%s\". Moving it aside " + "to \"%s\". This could be a bug in Tor; please tell " + "the developers.", fname, fname2); + if (rename(fname, fname2) < 0) { + log_warn(LD_BUG, "Weirdly, I couldn't even move the state aside. The " + "OS gave an error of %s", strerror(errno)); + } + } + tor_free(fname2); } /** Reload the persistent state from disk, generating a new state as needed. @@ -5230,31 +5071,8 @@ or_state_load(void) " This is a bug in Tor."); goto done; } else if (badstate && contents) { - int i; - file_status_t status; - size_t len = strlen(fname)+16; - char *fname2 = tor_malloc(len); - for (i = 0; i < 100; ++i) { - tor_snprintf(fname2, len, "%s.%d", fname, i); - status = file_status(fname2); - if (status == FN_NOENT) - break; - } - if (i == 100) { - log_warn(LD_BUG, "Unable to parse state in \"%s\"; too many saved bad " - "state files to move aside. Discarding the old state file.", - fname); - unlink(fname); - } else { - log_warn(LD_BUG, "Unable to parse state in \"%s\". Moving it aside " - "to \"%s\". This could be a bug in Tor; please tell " - "the developers.", fname, fname2); - if (rename(fname, fname2) < 0) { - log_warn(LD_BUG, "Weirdly, I couldn't even move the state aside. The " - "OS gave an error of %s", strerror(errno)); - } - } - tor_free(fname2); + or_state_save_broken(fname); + tor_free(contents); config_free(&state_format, new_state); @@ -5266,7 +5084,9 @@ or_state_load(void) } else { log_info(LD_GENERAL, "Initialized state"); } - or_state_set(new_state); + if (or_state_set(new_state) == -1) { + or_state_save_broken(fname); + } new_state = NULL; if (!contents) { global_state->next_write = 0; @@ -5289,7 +5109,6 @@ or_state_save(time_t now) { char *state, *contents; char tbuf[ISO_TIME_LEN+1]; - size_t len; char *fname; tor_assert(global_state); @@ -5301,20 +5120,16 @@ or_state_save(time_t now) * to avoid redundant writes. */ entry_guards_update_state(global_state); rep_hist_update_state(global_state); + circuit_build_times_update_state(&circ_times, global_state); if (accounting_is_enabled(get_options())) accounting_run_housekeeping(now); - global_state->LastWritten = time(NULL); tor_free(global_state->TorVersion); - len = strlen(get_version())+8; - global_state->TorVersion = tor_malloc(len); - tor_snprintf(global_state->TorVersion, len, "Tor %s", get_version()); + tor_asprintf(&global_state->TorVersion, "Tor %s", get_version()); state = config_dump(&state_format, global_state, 1, 0); - len = strlen(state)+256; - contents = tor_malloc(len); format_local_iso_time(tbuf, time(NULL)); - tor_snprintf(contents, len, + tor_asprintf(&contents, "# Tor state file last generated on %s local time\n" "# Other times below are in GMT\n" "# You *do not* need to edit this file.\n\n%s", @@ -5323,10 +5138,13 @@ or_state_save(time_t now) fname = get_datadir_fname("state"); if (write_str_to_file(fname, contents, 0)<0) { log_warn(LD_FS, "Unable to write state to file \"%s\"", fname); + global_state->LastWritten = -1; tor_free(fname); tor_free(contents); return -1; } + + global_state->LastWritten = time(NULL); log_info(LD_GENERAL, "Saved state to \"%s\"", fname); tor_free(fname); tor_free(contents); @@ -5358,18 +5176,18 @@ remove_file_if_very_old(const char *fname, time_t now) * types. */ int getinfo_helper_config(control_connection_t *conn, - const char *question, char **answer) + const char *question, char **answer, + const char **errmsg) { (void) conn; + (void) errmsg; if (!strcmp(question, "config/names")) { smartlist_t *sl = smartlist_create(); int i; for (i = 0; _option_vars[i].name; ++i) { config_var_t *var = &_option_vars[i]; - const char *type, *desc; + const char *type; char *line; - size_t len; - desc = config_find_description(&options_format, var->name); switch (var->type) { case CONFIG_TYPE_STRING: type = "String"; break; case CONFIG_TYPE_FILENAME: type = "Filename"; break; @@ -5390,14 +5208,7 @@ getinfo_helper_config(control_connection_t *conn, } if (!type) continue; - len = strlen(var->name)+strlen(type)+16; - if (desc) - len += strlen(desc); - line = tor_malloc(len); - if (desc) - tor_snprintf(line, len, "%s %s %s\n",var->name,type,desc); - else - tor_snprintf(line, len, "%s %s\n",var->name,type); + tor_asprintf(&line, "%s %s\n",var->name,type); smartlist_add(sl, line); } *answer = smartlist_join_strings(sl, "", 0, NULL); diff --git a/src/or/config.h b/src/or/config.h new file mode 100644 index 0000000000..7a4ba5c60f --- /dev/null +++ b/src/or/config.h @@ -0,0 +1,78 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2010, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file config.h + * \brief Header file for config.c. + **/ + +#ifndef _TOR_CONFIG_H +#define _TOR_CONFIG_H + +const char *get_dirportfrontpage(void); +or_options_t *get_options(void); +int set_options(or_options_t *new_val, char **msg); +void config_free_all(void); +const char *safe_str_client(const char *address); +const char *safe_str(const char *address); +const char *escaped_safe_str_client(const char *address); +const char *escaped_safe_str(const char *address); +const char *get_version(void); + +int config_get_lines(const char *string, config_line_t **result); +void config_free_lines(config_line_t *front); +setopt_err_t options_trial_assign(config_line_t *list, int use_defaults, + int clear_first, char **msg); +int resolve_my_address(int warn_severity, or_options_t *options, + uint32_t *addr, char **hostname_out); +int is_local_addr(const tor_addr_t *addr) ATTR_PURE; +void options_init(or_options_t *options); +char *options_dump(or_options_t *options, int minimal); +int options_init_from_torrc(int argc, char **argv); +setopt_err_t options_init_from_string(const char *cf, + int command, const char *command_arg, char **msg); +int option_is_recognized(const char *key); +const char *option_get_canonical_name(const char *key); +config_line_t *option_get_assignment(or_options_t *options, + const char *key); +int options_save_current(void); +const char *get_torrc_fname(void); +char *options_get_datadir_fname2_suffix(or_options_t *options, + const char *sub1, const char *sub2, + const char *suffix); +#define get_datadir_fname2_suffix(sub1, sub2, suffix) \ + options_get_datadir_fname2_suffix(get_options(), (sub1), (sub2), (suffix)) +/** Return a newly allocated string containing datadir/sub1. See + * get_datadir_fname2_suffix. */ +#define get_datadir_fname(sub1) get_datadir_fname2_suffix((sub1), NULL, NULL) +/** Return a newly allocated string containing datadir/sub1/sub2. See + * get_datadir_fname2_suffix. */ +#define get_datadir_fname2(sub1,sub2) \ + get_datadir_fname2_suffix((sub1), (sub2), NULL) +/** Return a newly allocated string containing datadir/sub1suffix. See + * get_datadir_fname2_suffix. */ +#define get_datadir_fname_suffix(sub1, suffix) \ + get_datadir_fname2_suffix((sub1), NULL, (suffix)) + +or_state_t *get_or_state(void); +int or_state_save(time_t now); + +int options_need_geoip_info(or_options_t *options, const char **reason_out); +int getinfo_helper_config(control_connection_t *conn, + const char *question, char **answer, + const char **errmsg); + +const char *tor_get_digests(void); +uint32_t get_effective_bwrate(or_options_t *options); +uint32_t get_effective_bwburst(or_options_t *options); + +#ifdef CONFIG_PRIVATE +/* Used only by config.c and test.c */ +or_options_t *options_new(void); +#endif + +#endif + diff --git a/src/or/config_codedigest.c b/src/or/config_codedigest.c new file mode 100644 index 0000000000..be9eaa331d --- /dev/null +++ b/src/or/config_codedigest.c @@ -0,0 +1,11 @@ + +const char *tor_get_digests(void); + +const char * +tor_get_digests(void) +{ + return "" +#include "or_sha1.i" + ; +} + diff --git a/src/or/connection.c b/src/or/connection.c index 5036be4507..0eb05b74ab 100644 --- a/src/or/connection.c +++ b/src/or/connection.c @@ -11,6 +11,30 @@ **/ #include "or.h" +#include "buffers.h" +#include "circuitbuild.h" +#include "circuitlist.h" +#include "circuituse.h" +#include "config.h" +#include "connection.h" +#include "connection_edge.h" +#include "connection_or.h" +#include "control.h" +#include "cpuworker.h" +#include "directory.h" +#include "dirserv.h" +#include "dns.h" +#include "dnsserv.h" +#include "geoip.h" +#include "main.h" +#include "policies.h" +#include "reasons.h" +#include "relay.h" +#include "rendclient.h" +#include "rendcommon.h" +#include "rephist.h" +#include "router.h" +#include "routerparse.h" static connection_t *connection_create_listener( struct sockaddr *listensockaddr, @@ -21,7 +45,8 @@ static void connection_init(time_t now, connection_t *conn, int type, static int connection_init_accepted_conn(connection_t *conn, uint8_t listener_type); static int connection_handle_listener_read(connection_t *conn, int new_type); -static int connection_read_bucket_should_increase(or_connection_t *conn); +static int connection_bucket_should_increase(int bucket, + or_connection_t *conn); static int connection_finished_flushing(connection_t *conn); static int connection_flushed_some(connection_t *conn); static int connection_finished_connecting(connection_t *conn); @@ -32,6 +57,10 @@ static int connection_process_inbuf(connection_t *conn, int package_partial); static void client_check_address_changed(int sock); static void set_constrained_socket_buffers(int sock, int size); +static const char *connection_proxy_state_to_string(int state); +static int connection_read_https_proxy_response(connection_t *conn); +static void connection_send_socks5_connect(connection_t *conn); + /** The last IPv4 address that our network interface seemed to have been * binding to, in host order. We use this to detect when our IP changes. */ static uint32_t last_interface_ip = 0; @@ -92,8 +121,7 @@ conn_state_to_string(int type, int state) case CONN_TYPE_OR: switch (state) { case OR_CONN_STATE_CONNECTING: return "connect()ing"; - case OR_CONN_STATE_PROXY_FLUSHING: return "proxy flushing"; - case OR_CONN_STATE_PROXY_READING: return "proxy reading"; + case OR_CONN_STATE_PROXY_HANDSHAKING: return "handshaking (proxy)"; case OR_CONN_STATE_TLS_HANDSHAKING: return "handshaking (TLS)"; case OR_CONN_STATE_TLS_CLIENT_RENEGOTIATING: return "renegotiating (TLS)"; @@ -177,6 +205,9 @@ or_connection_new(int socket_family) or_conn->timestamp_last_added_nonpadding = time(NULL); or_conn->next_circ_id = crypto_rand_int(1<<15); + or_conn->active_circuit_pqueue = smartlist_create(); + or_conn->active_circuit_pqueue_last_recalibrated = cell_ewma_get_tick(); + return or_conn; } @@ -202,6 +233,7 @@ control_connection_new(int socket_family) tor_malloc_zero(sizeof(control_connection_t)); connection_init(time(NULL), TO_CONN(control_conn), CONN_TYPE_CONTROL, socket_family); + log_notice(LD_CONTROL, "New control connection opened."); return control_conn; } @@ -299,25 +331,6 @@ connection_link_connections(connection_t *conn_a, connection_t *conn_b) conn_b->linked_conn = conn_a; } -/** Tell libevent that we don't care about <b>conn</b> any more. */ -void -connection_unregister_events(connection_t *conn) -{ - if (conn->read_event) { - if (event_del(conn->read_event)) - log_warn(LD_BUG, "Error removing read event for %d", conn->s); - tor_free(conn->read_event); - } - if (conn->write_event) { - if (event_del(conn->write_event)) - log_warn(LD_BUG, "Error removing write event for %d", conn->s); - tor_free(conn->write_event); - } - if (conn->dns_server_port) { - dnsserv_close_listener(conn); - } -} - /** Deallocate memory used by <b>conn</b>. Deallocate its buffers if * necessary, close its socket if necessary, and mark the directory as dirty * if <b>conn</b> is an OR or OP connection. @@ -327,6 +340,9 @@ _connection_free(connection_t *conn) { void *mem; size_t memlen; + if (!conn) + return; + switch (conn->type) { case CONN_TYPE_OR: tor_assert(conn->magic == OR_CONNECTION_MAGIC); @@ -384,14 +400,11 @@ _connection_free(connection_t *conn) if (connection_speaks_cells(conn)) { or_connection_t *or_conn = TO_OR_CONN(conn); - if (or_conn->tls) { - tor_tls_free(or_conn->tls); - or_conn->tls = NULL; - } - if (or_conn->handshake_state) { - or_handshake_state_free(or_conn->handshake_state); - or_conn->handshake_state = NULL; - } + tor_tls_free(or_conn->tls); + or_conn->tls = NULL; + or_handshake_state_free(or_conn->handshake_state); + or_conn->handshake_state = NULL; + smartlist_free(or_conn->active_circuit_pqueue); tor_free(or_conn->nickname); } if (CONN_IS_EDGE(conn)) { @@ -401,8 +414,8 @@ _connection_free(connection_t *conn) memset(edge_conn->socks_request, 0xcc, sizeof(socks_request_t)); tor_free(edge_conn->socks_request); } - if (edge_conn->rend_data) - rend_data_free(edge_conn->rend_data); + + rend_data_free(edge_conn->rend_data); } if (conn->type == CONN_TYPE_CONTROL) { control_connection_t *control_conn = TO_CONTROL_CONN(conn); @@ -415,16 +428,15 @@ _connection_free(connection_t *conn) if (conn->type == CONN_TYPE_DIR) { dir_connection_t *dir_conn = TO_DIR_CONN(conn); tor_free(dir_conn->requested_resource); - if (dir_conn->zlib_state) - tor_zlib_free(dir_conn->zlib_state); + + tor_zlib_free(dir_conn->zlib_state); if (dir_conn->fingerprint_stack) { SMARTLIST_FOREACH(dir_conn->fingerprint_stack, char *, cp, tor_free(cp)); smartlist_free(dir_conn->fingerprint_stack); } - if (dir_conn->cached_dir) - cached_dir_decref(dir_conn->cached_dir); - if (dir_conn->rend_data) - rend_data_free(dir_conn->rend_data); + + cached_dir_decref(dir_conn->cached_dir); + rend_data_free(dir_conn->rend_data); } if (conn->s >= 0) { @@ -439,7 +451,7 @@ _connection_free(connection_t *conn) connection_or_remove_from_identity_map(TO_OR_CONN(conn)); } - memset(conn, 0xAA, memlen); /* poison memory */ + memset(mem, 0xCC, memlen); /* poison memory */ tor_free(mem); } @@ -448,7 +460,8 @@ _connection_free(connection_t *conn) void connection_free(connection_t *conn) { - tor_assert(conn); + if (!conn) + return; tor_assert(!connection_is_on_closeable_list(conn)); tor_assert(!connection_in_array(conn)); if (conn->linked_conn) { @@ -544,13 +557,6 @@ connection_about_to_close_connection(connection_t *conn) * failed: forget about this router, and maybe try again. */ connection_dir_request_failed(dir_conn); } - if (conn->purpose == DIR_PURPOSE_FETCH_RENDDESC && dir_conn->rend_data) { - /* Give it a try. However, there is no re-fetching for v0 rend - * descriptors; if the response is empty or the descriptor is - * unusable, close pending connections (unless a v2 request is - * still in progress). */ - rend_client_desc_trynow(dir_conn->rend_data->onion_address, 0); - } /* If we were trying to fetch a v2 rend desc and did not succeed, * retry as needed. (If a fetch is successful, the connection state * is changed to DIR_PURPOSE_HAS_FETCHED_RENDDESC to mark that @@ -589,7 +595,7 @@ connection_about_to_close_connection(connection_t *conn) rep_hist_note_disconnect(or_conn->identity_digest, now); control_event_or_conn_status(or_conn, OR_CONN_EVENT_CLOSED, tls_error_to_orconn_end_reason(or_conn->tls_error)); - } else if (or_conn->identity_digest) { + } else if (!tor_digest_is_zero(or_conn->identity_digest)) { rep_hist_note_connection_died(or_conn->identity_digest, now); control_event_or_conn_status(or_conn, OR_CONN_EVENT_CLOSED, tls_error_to_orconn_end_reason(or_conn->tls_error)); @@ -840,9 +846,9 @@ warn_too_many_conns(void) log_warn(LD_NET,"Failing because we have %d connections already. Please " "raise your ulimit -n.", n_conns); last_warned = now; + control_event_general_status(LOG_WARN, "TOO_MANY_CONNECTIONS CURRENT=%d", + n_conns); } - control_event_general_status(LOG_WARN, "TOO_MANY_CONNECTIONS CURRENT=%d", - n_conns); } /** Bind a new non-blocking socket listening to the socket described @@ -1254,7 +1260,7 @@ connection_connect(connection_t *conn, const char *address, return -1; } - if (options->OutboundBindAddress) { + if (options->OutboundBindAddress && !tor_addr_is_loopback(addr)) { struct sockaddr_in ext_addr; memset(&ext_addr, 0, sizeof(ext_addr)); @@ -1285,7 +1291,8 @@ connection_connect(connection_t *conn, const char *address, dest_addr_len = tor_addr_to_sockaddr(addr, port, dest_addr, sizeof(addrbuf)); tor_assert(dest_addr_len > 0); - log_debug(LD_NET,"Connecting to %s:%u.",escaped_safe_str(address),port); + log_debug(LD_NET, "Connecting to %s:%u.", + escaped_safe_str_client(address), port); if (connect(s, dest_addr, dest_addr_len) < 0) { int e = tor_socket_errno(s); @@ -1293,7 +1300,8 @@ connection_connect(connection_t *conn, const char *address, /* yuck. kill it. */ *socket_error = e; log_info(LD_NET, - "connect() to %s:%u failed: %s",escaped_safe_str(address), + "connect() to %s:%u failed: %s", + escaped_safe_str_client(address), port, tor_socket_strerror(e)); tor_close_socket(s); return -1; @@ -1307,7 +1315,8 @@ connection_connect(connection_t *conn, const char *address, /* it succeeded. we're connected. */ log_fn(inprogress?LOG_DEBUG:LOG_INFO, LD_NET, - "Connection to %s:%u %s (sock %d).",escaped_safe_str(address), + "Connection to %s:%u %s (sock %d).", + escaped_safe_str_client(address), port, inprogress?"in progress":"established", s); conn->s = s; if (connection_add(conn) < 0) /* no space, forget it */ @@ -1315,6 +1324,353 @@ connection_connect(connection_t *conn, const char *address, return inprogress ? 0 : 1; } +/** Convert state number to string representation for logging purposes. + */ +static const char * +connection_proxy_state_to_string(int state) +{ + static const char *unknown = "???"; + static const char *states[] = { + "PROXY_NONE", + "PROXY_HTTPS_WANT_CONNECT_OK", + "PROXY_SOCKS4_WANT_CONNECT_OK", + "PROXY_SOCKS5_WANT_AUTH_METHOD_NONE", + "PROXY_SOCKS5_WANT_AUTH_METHOD_RFC1929", + "PROXY_SOCKS5_WANT_AUTH_RFC1929_OK", + "PROXY_SOCKS5_WANT_CONNECT_OK", + "PROXY_CONNECTED", + }; + + if (state < PROXY_NONE || state > PROXY_CONNECTED) + return unknown; + + return states[state]; +} + +/** Write a proxy request of <b>type</b> (socks4, socks5, https) to conn + * for conn->addr:conn->port, authenticating with the auth details given + * in the configuration (if available). SOCKS 5 and HTTP CONNECT proxies + * support authentication. + * + * Returns -1 if conn->addr is incompatible with the proxy protocol, and + * 0 otherwise. + * + * Use connection_read_proxy_handshake() to complete the handshake. + */ +int +connection_proxy_connect(connection_t *conn, int type) +{ + or_options_t *options; + + tor_assert(conn); + + options = get_options(); + + switch (type) { + case PROXY_CONNECT: { + char buf[1024]; + char *base64_authenticator=NULL; + const char *authenticator = options->HttpsProxyAuthenticator; + + /* Send HTTP CONNECT and authentication (if available) in + * one request */ + + if (authenticator) { + base64_authenticator = alloc_http_authenticator(authenticator); + if (!base64_authenticator) + log_warn(LD_OR, "Encoding https authenticator failed"); + } + + if (base64_authenticator) { + tor_snprintf(buf, sizeof(buf), "CONNECT %s:%d HTTP/1.1\r\n" + "Proxy-Authorization: Basic %s\r\n\r\n", + fmt_addr(&conn->addr), + conn->port, base64_authenticator); + tor_free(base64_authenticator); + } else { + tor_snprintf(buf, sizeof(buf), "CONNECT %s:%d HTTP/1.0\r\n\r\n", + fmt_addr(&conn->addr), conn->port); + } + + connection_write_to_buf(buf, strlen(buf), conn); + conn->proxy_state = PROXY_HTTPS_WANT_CONNECT_OK; + break; + } + + case PROXY_SOCKS4: { + unsigned char buf[9]; + uint16_t portn; + uint32_t ip4addr; + + /* Send a SOCKS4 connect request with empty user id */ + + if (tor_addr_family(&conn->addr) != AF_INET) { + log_warn(LD_NET, "SOCKS4 client is incompatible with IPv6"); + return -1; + } + + ip4addr = tor_addr_to_ipv4n(&conn->addr); + portn = htons(conn->port); + + buf[0] = 4; /* version */ + buf[1] = SOCKS_COMMAND_CONNECT; /* command */ + memcpy(buf + 2, &portn, 2); /* port */ + memcpy(buf + 4, &ip4addr, 4); /* addr */ + buf[8] = 0; /* userid (empty) */ + + connection_write_to_buf((char *)buf, sizeof(buf), conn); + conn->proxy_state = PROXY_SOCKS4_WANT_CONNECT_OK; + break; + } + + case PROXY_SOCKS5: { + unsigned char buf[4]; /* fields: vers, num methods, method list */ + + /* Send a SOCKS5 greeting (connect request must wait) */ + + buf[0] = 5; /* version */ + + /* number of auth methods */ + if (options->Socks5ProxyUsername) { + buf[1] = 2; + buf[2] = 0x00; /* no authentication */ + buf[3] = 0x02; /* rfc1929 Username/Passwd auth */ + conn->proxy_state = PROXY_SOCKS5_WANT_AUTH_METHOD_RFC1929; + } else { + buf[1] = 1; + buf[2] = 0x00; /* no authentication */ + conn->proxy_state = PROXY_SOCKS5_WANT_AUTH_METHOD_NONE; + } + + connection_write_to_buf((char *)buf, 2 + buf[1], conn); + break; + } + + default: + log_err(LD_BUG, "Invalid proxy protocol, %d", type); + tor_fragile_assert(); + return -1; + } + + log_debug(LD_NET, "set state %s", + connection_proxy_state_to_string(conn->proxy_state)); + + return 0; +} + +/** Read conn's inbuf. If the http response from the proxy is all + * here, make sure it's good news, then return 1. If it's bad news, + * return -1. Else return 0 and hope for better luck next time. + */ +static int +connection_read_https_proxy_response(connection_t *conn) +{ + char *headers; + char *reason=NULL; + int status_code; + time_t date_header; + + switch (fetch_from_buf_http(conn->inbuf, + &headers, MAX_HEADERS_SIZE, + NULL, NULL, 10000, 0)) { + case -1: /* overflow */ + log_warn(LD_PROTOCOL, + "Your https proxy sent back an oversized response. Closing."); + return -1; + case 0: + log_info(LD_NET,"https proxy response not all here yet. Waiting."); + return 0; + /* case 1, fall through */ + } + + if (parse_http_response(headers, &status_code, &date_header, + NULL, &reason) < 0) { + log_warn(LD_NET, + "Unparseable headers from proxy (connecting to '%s'). Closing.", + conn->address); + tor_free(headers); + return -1; + } + if (!reason) reason = tor_strdup("[no reason given]"); + + if (status_code == 200) { + log_info(LD_NET, + "HTTPS connect to '%s' successful! (200 %s) Starting TLS.", + conn->address, escaped(reason)); + tor_free(reason); + return 1; + } + /* else, bad news on the status code */ + log_warn(LD_NET, + "The https proxy sent back an unexpected status code %d (%s). " + "Closing.", + status_code, escaped(reason)); + tor_free(reason); + return -1; +} + +/** Send SOCKS5 CONNECT command to <b>conn</b>, copying <b>conn->addr</b> + * and <b>conn->port</b> into the request. + */ +static void +connection_send_socks5_connect(connection_t *conn) +{ + unsigned char buf[1024]; + size_t reqsize = 6; + uint16_t port = htons(conn->port); + + buf[0] = 5; /* version */ + buf[1] = SOCKS_COMMAND_CONNECT; /* command */ + buf[2] = 0; /* reserved */ + + if (tor_addr_family(&conn->addr) == AF_INET) { + uint32_t addr = tor_addr_to_ipv4n(&conn->addr); + + buf[3] = 1; + reqsize += 4; + memcpy(buf + 4, &addr, 4); + memcpy(buf + 8, &port, 2); + } else { /* AF_INET6 */ + buf[3] = 4; + reqsize += 16; + memcpy(buf + 4, tor_addr_to_in6(&conn->addr), 16); + memcpy(buf + 20, &port, 2); + } + + connection_write_to_buf((char *)buf, reqsize, conn); + + conn->proxy_state = PROXY_SOCKS5_WANT_CONNECT_OK; +} + +/** Call this from connection_*_process_inbuf() to advance the proxy + * handshake. + * + * No matter what proxy protocol is used, if this function returns 1, the + * handshake is complete, and the data remaining on inbuf may contain the + * start of the communication with the requested server. + * + * Returns 0 if the current buffer contains an incomplete response, and -1 + * on error. + */ +int +connection_read_proxy_handshake(connection_t *conn) +{ + int ret = 0; + char *reason = NULL; + + log_debug(LD_NET, "enter state %s", + connection_proxy_state_to_string(conn->proxy_state)); + + switch (conn->proxy_state) { + case PROXY_HTTPS_WANT_CONNECT_OK: + ret = connection_read_https_proxy_response(conn); + if (ret == 1) + conn->proxy_state = PROXY_CONNECTED; + break; + + case PROXY_SOCKS4_WANT_CONNECT_OK: + ret = fetch_from_buf_socks_client(conn->inbuf, + conn->proxy_state, + &reason); + if (ret == 1) + conn->proxy_state = PROXY_CONNECTED; + break; + + case PROXY_SOCKS5_WANT_AUTH_METHOD_NONE: + ret = fetch_from_buf_socks_client(conn->inbuf, + conn->proxy_state, + &reason); + /* no auth needed, do connect */ + if (ret == 1) { + connection_send_socks5_connect(conn); + ret = 0; + } + break; + + case PROXY_SOCKS5_WANT_AUTH_METHOD_RFC1929: + ret = fetch_from_buf_socks_client(conn->inbuf, + conn->proxy_state, + &reason); + + /* send auth if needed, otherwise do connect */ + if (ret == 1) { + connection_send_socks5_connect(conn); + ret = 0; + } else if (ret == 2) { + unsigned char buf[1024]; + size_t reqsize, usize, psize; + const char *user, *pass; + + user = get_options()->Socks5ProxyUsername; + pass = get_options()->Socks5ProxyPassword; + tor_assert(user && pass); + + /* XXX len of user and pass must be <= 255 !!! */ + usize = strlen(user); + psize = strlen(pass); + tor_assert(usize <= 255 && psize <= 255); + reqsize = 3 + usize + psize; + + buf[0] = 1; /* negotiation version */ + buf[1] = usize; + memcpy(buf + 2, user, usize); + buf[2 + usize] = psize; + memcpy(buf + 3 + usize, pass, psize); + + connection_write_to_buf((char *)buf, reqsize, conn); + + conn->proxy_state = PROXY_SOCKS5_WANT_AUTH_RFC1929_OK; + ret = 0; + } + break; + + case PROXY_SOCKS5_WANT_AUTH_RFC1929_OK: + ret = fetch_from_buf_socks_client(conn->inbuf, + conn->proxy_state, + &reason); + /* send the connect request */ + if (ret == 1) { + connection_send_socks5_connect(conn); + ret = 0; + } + break; + + case PROXY_SOCKS5_WANT_CONNECT_OK: + ret = fetch_from_buf_socks_client(conn->inbuf, + conn->proxy_state, + &reason); + if (ret == 1) + conn->proxy_state = PROXY_CONNECTED; + break; + + default: + log_err(LD_BUG, "Invalid proxy_state for reading, %d", + conn->proxy_state); + tor_fragile_assert(); + ret = -1; + break; + } + + log_debug(LD_NET, "leaving state %s", + connection_proxy_state_to_string(conn->proxy_state)); + + if (ret < 0) { + if (reason) { + log_warn(LD_NET, "Proxy Client: unable to connect to %s:%d (%s)", + conn->address, conn->port, escaped(reason)); + tor_free(reason); + } else { + log_warn(LD_NET, "Proxy Client: unable to connect to %s:%d", + conn->address, conn->port); + } + } else if (ret == 1) { + log_info(LD_NET, "Proxy Client: connection to %s:%d successful", + conn->address, conn->port); + } + + return ret; +} + /** * Launch any configured listener connections of type <b>type</b>. (A * listener is configured if <b>port_option</b> is non-zero. If any @@ -1643,6 +1999,7 @@ connection_bucket_write_limit(connection_t *conn, time_t now) int base = connection_speaks_cells(conn) ? CELL_NETWORK_SIZE : RELAY_PAYLOAD_SIZE; int priority = conn->type != CONN_TYPE_DIR; + int conn_bucket = (int)conn->outbuf_flushlen; int global_bucket = global_write_bucket; if (!connection_is_rate_limited(conn)) { @@ -1650,12 +2007,22 @@ connection_bucket_write_limit(connection_t *conn, time_t now) return conn->outbuf_flushlen; } + if (connection_speaks_cells(conn)) { + /* use the per-conn write limit if it's lower, but if it's less + * than zero just use zero */ + or_connection_t *or_conn = TO_OR_CONN(conn); + if (conn->state == OR_CONN_STATE_OPEN) + if (or_conn->write_bucket < conn_bucket) + conn_bucket = or_conn->write_bucket >= 0 ? + or_conn->write_bucket : 0; + } + if (connection_counts_as_relayed_traffic(conn, now) && global_relayed_write_bucket <= global_write_bucket) global_bucket = global_relayed_write_bucket; - return connection_bucket_round_robin(base, priority, global_bucket, - conn->outbuf_flushlen); + return connection_bucket_round_robin(base, priority, + global_bucket, conn_bucket); } /** Return 1 if the global write buckets are low enough that we @@ -1709,8 +2076,8 @@ global_write_bucket_low(connection_t *conn, size_t attempt, int priority) return 0; } -/** We just read num_read and wrote num_written onto conn. - * Decrement buckets appropriately. */ +/** We just read <b>num_read</b> and wrote <b>num_written</b> bytes + * onto <b>conn</b>. Decrement buckets appropriately. */ static void connection_buckets_decrement(connection_t *conn, time_t now, size_t num_read, size_t num_written) @@ -1728,10 +2095,14 @@ connection_buckets_decrement(connection_t *conn, time_t now, tor_fragile_assert(); } - if (num_read > 0) + if (num_read > 0) { rep_hist_note_bytes_read(num_read, now); - if (num_written > 0) + } + if (num_written > 0) { rep_hist_note_bytes_written(num_written, now); + } + if (conn->type == CONN_TYPE_EXIT) + rep_hist_note_exit_bytes(conn->port, num_written, num_read); if (connection_counts_as_relayed_traffic(conn, now)) { global_relayed_read_bucket -= (int)num_read; @@ -1739,8 +2110,10 @@ connection_buckets_decrement(connection_t *conn, time_t now, } global_read_bucket -= (int)num_read; global_write_bucket -= (int)num_written; - if (connection_speaks_cells(conn) && conn->state == OR_CONN_STATE_OPEN) + if (connection_speaks_cells(conn) && conn->state == OR_CONN_STATE_OPEN) { TO_OR_CONN(conn)->read_bucket -= (int)num_read; + TO_OR_CONN(conn)->write_bucket -= (int)num_written; + } } /** If we have exhausted our global buckets, or the buckets for conn, @@ -1779,12 +2152,10 @@ connection_consider_empty_write_buckets(connection_t *conn) } else if (connection_counts_as_relayed_traffic(conn, approx_time()) && global_relayed_write_bucket <= 0) { reason = "global relayed write bucket exhausted. Pausing."; -#if 0 } else if (connection_speaks_cells(conn) && conn->state == OR_CONN_STATE_OPEN && TO_OR_CONN(conn)->write_bucket <= 0) { reason = "connection write bucket exhausted. Pausing."; -#endif } else return; /* all good, no need to stop it */ @@ -1880,14 +2251,19 @@ connection_bucket_refill(int seconds_elapsed, time_t now) { if (connection_speaks_cells(conn)) { or_connection_t *or_conn = TO_OR_CONN(conn); - if (connection_read_bucket_should_increase(or_conn)) { + if (connection_bucket_should_increase(or_conn->read_bucket, or_conn)) { connection_bucket_refill_helper(&or_conn->read_bucket, or_conn->bandwidthrate, or_conn->bandwidthburst, seconds_elapsed, "or_conn->read_bucket"); - //log_fn(LOG_DEBUG,"Receiver bucket %d now %d.", i, - // conn->read_bucket); + } + if (connection_bucket_should_increase(or_conn->write_bucket, or_conn)) { + connection_bucket_refill_helper(&or_conn->write_bucket, + or_conn->bandwidthrate, + or_conn->bandwidthburst, + seconds_elapsed, + "or_conn->write_bucket"); } } @@ -1908,8 +2284,10 @@ connection_bucket_refill(int seconds_elapsed, time_t now) if (conn->write_blocked_on_bw == 1 && global_write_bucket > 0 /* and we're allowed to write */ && (!connection_counts_as_relayed_traffic(conn, now) || - global_relayed_write_bucket > 0)) { - /* even if we're relayed traffic */ + global_relayed_write_bucket > 0) /* even if it's relayed traffic */ + && (!connection_speaks_cells(conn) || + conn->state != OR_CONN_STATE_OPEN || + TO_OR_CONN(conn)->write_bucket > 0)) { LOG_FN_CONN(conn, (LOG_DEBUG,LD_NET, "waking up conn (fd %d) for write", conn->s)); conn->write_blocked_on_bw = 0; @@ -1918,17 +2296,17 @@ connection_bucket_refill(int seconds_elapsed, time_t now) }); } -/** Is the receiver bucket for connection <b>conn</b> low enough that we +/** Is the <b>bucket</b> for connection <b>conn</b> low enough that we * should add another pile of tokens to it? */ static int -connection_read_bucket_should_increase(or_connection_t *conn) +connection_bucket_should_increase(int bucket, or_connection_t *conn) { tor_assert(conn); if (conn->_base.state != OR_CONN_STATE_OPEN) return 0; /* only open connections play the rate limiting game */ - if (conn->read_bucket >= conn->bandwidthburst) + if (bucket >= conn->bandwidthburst) return 0; return 1; @@ -1946,8 +2324,8 @@ connection_read_bucket_should_increase(or_connection_t *conn) * Mark the connection and return -1 if you want to close it, else * return 0. */ -int -connection_handle_read(connection_t *conn) +static int +connection_handle_read_impl(connection_t *conn) { int max_to_read=-1, try_to_read; size_t before, n_read = 0; @@ -2016,13 +2394,13 @@ loop_again: return -1; } if (conn->linked_conn) { - /* The other side's handle_write will never actually get called, so + /* The other side's handle_write() will never actually get called, so * we need to invoke the appropriate callbacks ourself. */ connection_t *linked = conn->linked_conn; if (n_read) { /* Probably a no-op, but hey. */ - connection_buckets_decrement(linked, approx_time(), 0, n_read); + connection_buckets_decrement(linked, approx_time(), n_read, 0); if (connection_flushed_some(linked) < 0) connection_mark_for_close(linked); @@ -2033,7 +2411,7 @@ loop_again: if (!buf_datalen(linked->outbuf) && conn->active_on_link) connection_stop_reading_from_linked_conn(conn); } - /* If we hit the EOF, call connection_reached_eof. */ + /* If we hit the EOF, call connection_reached_eof(). */ if (!conn->marked_for_close && conn->inbuf_reached_eof && connection_reached_eof(conn) < 0) { @@ -2042,6 +2420,16 @@ loop_again: return 0; } +int +connection_handle_read(connection_t *conn) +{ + int res; + + tor_gettimeofday_cache_clear(); + res = connection_handle_read_impl(conn); + return res; +} + /** Pull in new bytes from conn-\>s or conn-\>linked_conn onto conn-\>inbuf, * either directly or via TLS. Reduce the token buckets by the number of bytes * read. @@ -2075,7 +2463,7 @@ connection_read_to_buf(connection_t *conn, int *max_to_read, int *socket_error) } if (connection_speaks_cells(conn) && - conn->state > OR_CONN_STATE_PROXY_READING) { + conn->state > OR_CONN_STATE_PROXY_HANDSHAKING) { int pending; or_connection_t *or_conn = TO_OR_CONN(conn); size_t initial_size; @@ -2243,8 +2631,8 @@ connection_outbuf_too_full(connection_t *conn) * Mark the connection and return -1 if you want to close it, else * return 0. */ -int -connection_handle_write(connection_t *conn, int force) +static int +connection_handle_write_impl(connection_t *conn, int force) { int e; socklen_t len=(socklen_t)sizeof(e); @@ -2259,7 +2647,7 @@ connection_handle_write(connection_t *conn, int force) return 0; /* do nothing */ if (conn->in_flushed_some) { - log_warn(LD_BUG, "called recursively from inside conn->in_flushed_some()"); + log_warn(LD_BUG, "called recursively from inside conn->in_flushed_some"); return 0; } @@ -2303,7 +2691,7 @@ connection_handle_write(connection_t *conn, int force) : connection_bucket_write_limit(conn, now); if (connection_speaks_cells(conn) && - conn->state > OR_CONN_STATE_PROXY_READING) { + conn->state > OR_CONN_STATE_PROXY_HANDSHAKING) { or_connection_t *or_conn = TO_OR_CONN(conn); if (conn->state == OR_CONN_STATE_TLS_HANDSHAKING || conn->state == OR_CONN_STATE_TLS_CLIENT_RENEGOTIATING) { @@ -2322,6 +2710,13 @@ connection_handle_write(connection_t *conn, int force) /* else open, or closing */ result = flush_buf_tls(or_conn->tls, conn->outbuf, max_to_write, &conn->outbuf_flushlen); + + /* If we just flushed the last bytes, check if this tunneled dir + * request is done. */ + if (buf_datalen(conn->outbuf) == 0 && conn->dirreq_id) + geoip_change_dirreq_state(conn->dirreq_id, DIRREQ_TUNNELED, + DIRREQ_OR_CONN_BUFFER_FLUSHED); + switch (result) { CASE_TOR_TLS_ERROR_ANY: case TOR_TLS_CLOSE: @@ -2341,8 +2736,8 @@ connection_handle_write(connection_t *conn, int force) if (!connection_is_reading(conn)) { connection_stop_writing(conn); conn->write_blocked_on_bw = 1; - /* we'll start reading again when the next second arrives, - * and then also start writing again. + /* we'll start reading again when we get more tokens in our + * read bucket; then we'll start writing again too. */ } /* else no problem, we're already reading */ @@ -2404,6 +2799,15 @@ connection_handle_write(connection_t *conn, int force) return 0; } +int +connection_handle_write(connection_t *conn, int force) +{ + int res; + tor_gettimeofday_cache_clear(); + res = connection_handle_write_impl(conn, force); + return res; +} + /** OpenSSL TLS record size is 16383; this is close. The goal here is to * push data out as soon as we know there's enough for a TLS record, so * during periods of high load we won't read entire megabytes from @@ -2577,13 +2981,11 @@ connection_get_by_type_state(int type, int state) /** Return a connection of type <b>type</b> that has rendquery equal * to <b>rendquery</b>, and that is not marked for close. If state - * is non-zero, conn must be of that state too. If rendversion is - * nonnegative, conn must be fetching that rendversion, too. + * is non-zero, conn must be of that state too. */ connection_t * connection_get_by_type_state_rendquery(int type, int state, - const char *rendquery, - int rendversion) + const char *rendquery) { smartlist_t *conns = get_connection_array(); @@ -2598,8 +3000,6 @@ connection_get_by_type_state_rendquery(int type, int state, (!state || state == conn->state)) { if (type == CONN_TYPE_DIR && TO_DIR_CONN(conn)->rend_data && - (rendversion < 0 || - rendversion == TO_DIR_CONN(conn)->rend_data->rend_desc_version) && !rend_cmp_service_ids(rendquery, TO_DIR_CONN(conn)->rend_data->onion_address)) return conn; @@ -2717,10 +3117,10 @@ alloc_http_authenticator(const char *authenticator) static void client_check_address_changed(int sock) { - uint32_t iface_ip, ip_out; + uint32_t iface_ip, ip_out; /* host order */ struct sockaddr_in out_addr; socklen_t out_addr_len = (socklen_t) sizeof(out_addr); - uint32_t *ip; + uint32_t *ip; /* host order */ if (!last_interface_ip) get_interface_address(LOG_INFO, &last_interface_ip); @@ -2734,7 +3134,7 @@ client_check_address_changed(int sock) return; } - /* Okay. If we've used this address previously, we're okay. */ + /* If we've used this address previously, we're okay. */ ip_out = ntohl(out_addr.sin_addr.s_addr); SMARTLIST_FOREACH(outgoing_addrs, uint32_t*, ip_ptr, if (*ip_ptr == ip_out) return; @@ -3033,7 +3433,7 @@ assert_connection_ok(connection_t *conn, time_t now) } // tor_assert(conn->addr && conn->port); tor_assert(conn->address); - if (conn->state > OR_CONN_STATE_PROXY_READING) + if (conn->state > OR_CONN_STATE_PROXY_HANDSHAKING) tor_assert(or_conn->tls); } diff --git a/src/or/connection.h b/src/or/connection.h new file mode 100644 index 0000000000..f38927e788 --- /dev/null +++ b/src/or/connection.h @@ -0,0 +1,100 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2010, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file connection.h + * \brief Header file for connection.c. + **/ + +#ifndef _TOR_CONNECTION_H +#define _TOR_CONNECTION_H + +const char *conn_type_to_string(int type); +const char *conn_state_to_string(int type, int state); + +dir_connection_t *dir_connection_new(int socket_family); +or_connection_t *or_connection_new(int socket_family); +edge_connection_t *edge_connection_new(int type, int socket_family); +control_connection_t *control_connection_new(int socket_family); +connection_t *connection_new(int type, int socket_family); + +void connection_link_connections(connection_t *conn_a, connection_t *conn_b); +void connection_free(connection_t *conn); +void connection_free_all(void); +void connection_about_to_close_connection(connection_t *conn); +void connection_close_immediate(connection_t *conn); +void _connection_mark_for_close(connection_t *conn,int line, const char *file); + +#define connection_mark_for_close(c) \ + _connection_mark_for_close((c), __LINE__, _SHORT_FILE_) + +void connection_expire_held_open(void); + +int connection_connect(connection_t *conn, const char *address, + const tor_addr_t *addr, + uint16_t port, int *socket_error); + +int connection_proxy_connect(connection_t *conn, int type); +int connection_read_proxy_handshake(connection_t *conn); + +int retry_all_listeners(smartlist_t *replaced_conns, + smartlist_t *new_conns); + +ssize_t connection_bucket_write_limit(connection_t *conn, time_t now); +int global_write_bucket_low(connection_t *conn, size_t attempt, int priority); +void connection_bucket_init(void); +void connection_bucket_refill(int seconds_elapsed, time_t now); + +int connection_handle_read(connection_t *conn); + +int connection_fetch_from_buf(char *string, size_t len, connection_t *conn); + +int connection_wants_to_flush(connection_t *conn); +int connection_outbuf_too_full(connection_t *conn); +int connection_handle_write(connection_t *conn, int force); +void _connection_write_to_buf_impl(const char *string, size_t len, + connection_t *conn, int zlib); +static void connection_write_to_buf(const char *string, size_t len, + connection_t *conn); +static void connection_write_to_buf_zlib(const char *string, size_t len, + dir_connection_t *conn, int done); +static INLINE void +connection_write_to_buf(const char *string, size_t len, connection_t *conn) +{ + _connection_write_to_buf_impl(string, len, conn, 0); +} +static INLINE void +connection_write_to_buf_zlib(const char *string, size_t len, + dir_connection_t *conn, int done) +{ + _connection_write_to_buf_impl(string, len, TO_CONN(conn), done ? -1 : 1); +} + +connection_t *connection_get_by_global_id(uint64_t id); + +connection_t *connection_get_by_type(int type); +connection_t *connection_get_by_type_purpose(int type, int purpose); +connection_t *connection_get_by_type_addr_port_purpose(int type, + const tor_addr_t *addr, + uint16_t port, int purpose); +connection_t *connection_get_by_type_state(int type, int state); +connection_t *connection_get_by_type_state_rendquery(int type, int state, + const char *rendquery); + +#define connection_speaks_cells(conn) ((conn)->type == CONN_TYPE_OR) +int connection_is_listener(connection_t *conn); +int connection_state_is_open(connection_t *conn); +int connection_state_is_connecting(connection_t *conn); + +char *alloc_http_authenticator(const char *authenticator); + +void assert_connection_ok(connection_t *conn, time_t now); +int connection_or_nonopen_was_started_here(or_connection_t *conn); +void connection_dump_buffer_mem_stats(int severity); +void remove_file_if_very_old(const char *fname, time_t now); + +#endif + diff --git a/src/or/connection_edge.c b/src/or/connection_edge.c index ded02e936d..e83028faef 100644 --- a/src/or/connection_edge.c +++ b/src/or/connection_edge.c @@ -10,6 +10,28 @@ **/ #include "or.h" +#include "buffers.h" +#include "circuitlist.h" +#include "circuituse.h" +#include "config.h" +#include "connection.h" +#include "connection_edge.h" +#include "connection_or.h" +#include "control.h" +#include "dns.h" +#include "dnsserv.h" +#include "dirserv.h" +#include "hibernate.h" +#include "main.h" +#include "policies.h" +#include "reasons.h" +#include "relay.h" +#include "rendclient.h" +#include "rendcommon.h" +#include "rendservice.h" +#include "rephist.h" +#include "router.h" +#include "routerlist.h" #ifdef HAVE_LINUX_TYPES_H #include <linux/types.h> @@ -330,11 +352,13 @@ connection_edge_finished_connecting(edge_connection_t *edge_conn) tor_assert(conn->state == EXIT_CONN_STATE_CONNECTING); log_info(LD_EXIT,"Exit connection to %s:%u (%s) established.", - escaped_safe_str(conn->address),conn->port, + escaped_safe_str(conn->address), conn->port, safe_str(fmt_addr(&conn->addr))); + rep_hist_note_exit_stream_opened(conn->port); + conn->state = EXIT_CONN_STATE_OPEN; - connection_watch_events(conn, EV_READ); /* stop writing, continue reading */ + connection_watch_events(conn, READ_EVENT); /* stop writing, keep reading */ if (connection_wants_to_flush(conn)) /* in case there are any queued relay * cells */ connection_start_writing(conn); @@ -375,13 +399,16 @@ connection_edge_finished_connecting(edge_connection_t *edge_conn) static int compute_retry_timeout(edge_connection_t *conn) { + int timeout = get_options()->CircuitStreamTimeout; + if (timeout) /* if our config options override the default, use them */ + return timeout; if (conn->num_socks_retries < 2) /* try 0 and try 1 */ return 10; return 15; } /** Find all general-purpose AP streams waiting for a response that sent their - * begin/resolve cell >=15 seconds ago. Detach from their current circuit, and + * begin/resolve cell too long ago. Detach from their current circuit, and * mark their current circuit as unsuitable for new streams. Then call * connection_ap_handshake_attach_circuit() to attach to a new circuit (if * available) or launch a new one. @@ -423,7 +450,8 @@ connection_ap_expire_beginning(void) log_fn(severity, LD_APP, "Tried for %d seconds to get a connection to %s:%d. " "Giving up. (%s)", - seconds_since_born, safe_str(conn->socks_request->address), + seconds_since_born, + safe_str_client(conn->socks_request->address), conn->socks_request->port, conn_state_to_string(CONN_TYPE_AP, conn->_base.state)); connection_mark_unattached_ap(conn, END_STREAM_REASON_TIMEOUT); @@ -440,7 +468,7 @@ connection_ap_expire_beginning(void) circ = circuit_get_by_edge_conn(conn); if (!circ) { /* it's vanished? */ log_info(LD_APP,"Conn is waiting (address %s), but lost its circ.", - safe_str(conn->socks_request->address)); + safe_str_client(conn->socks_request->address)); connection_mark_unattached_ap(conn, END_STREAM_REASON_TIMEOUT); continue; } @@ -450,7 +478,7 @@ connection_ap_expire_beginning(void) "Rend stream is %d seconds late. Giving up on address" " '%s.onion'.", seconds_idle, - safe_str(conn->socks_request->address)); + safe_str_client(conn->socks_request->address)); connection_edge_end(conn, END_STREAM_REASON_TIMEOUT); connection_mark_unattached_ap(conn, END_STREAM_REASON_TIMEOUT); } @@ -460,7 +488,8 @@ connection_ap_expire_beginning(void) log_fn(cutoff < 15 ? LOG_INFO : severity, LD_APP, "We tried for %d seconds to connect to '%s' using exit '%s'." " Retrying on a new circuit.", - seconds_idle, safe_str(conn->socks_request->address), + seconds_idle, + safe_str_client(conn->socks_request->address), conn->cpath_layer ? conn->cpath_layer->extend_info->nickname : "*unnamed*"); /* send an end down the circuit */ @@ -577,8 +606,8 @@ circuit_discard_optional_exit_enclaves(extend_info_t *info) tor_assert(edge_conn->socks_request); if (edge_conn->chosen_exit_optional) { log_info(LD_APP, "Giving up on enclave exit '%s' for destination %s.", - safe_str(edge_conn->chosen_exit_name), - escaped_safe_str(edge_conn->socks_request->address)); + safe_str_client(edge_conn->chosen_exit_name), + escaped_safe_str_client(edge_conn->socks_request->address)); edge_conn->chosen_exit_optional = 0; tor_free(edge_conn->chosen_exit_name); /* clears it */ /* if this port is dangerous, warn or reject it now that we don't @@ -683,7 +712,11 @@ addressmap_init(void) static void addressmap_ent_free(void *_ent) { - addressmap_entry_t *ent = _ent; + addressmap_entry_t *ent; + if (!_ent) + return; + + ent = _ent; tor_free(ent->new_address); tor_free(ent); } @@ -692,7 +725,11 @@ addressmap_ent_free(void *_ent) static void addressmap_virtaddress_ent_free(void *_ent) { - virtaddress_entry_t *ent = _ent; + virtaddress_entry_t *ent; + if (!_ent) + return; + + ent = _ent; tor_free(ent->ipv4_address); tor_free(ent->hostname_address); tor_free(ent); @@ -782,14 +819,11 @@ addressmap_clean(time_t now) void addressmap_free_all(void) { - if (addressmap) { - strmap_free(addressmap, addressmap_ent_free); - addressmap = NULL; - } - if (virtaddress_reversemap) { - strmap_free(virtaddress_reversemap, addressmap_virtaddress_ent_free); - virtaddress_reversemap = NULL; - } + strmap_free(addressmap, addressmap_ent_free); + addressmap = NULL; + + strmap_free(virtaddress_reversemap, addressmap_virtaddress_ent_free); + virtaddress_reversemap = NULL; } /** Look at address, and rewrite it until it doesn't want any @@ -816,9 +850,9 @@ addressmap_rewrite(char *address, size_t maxlen, time_t *expires_out) return (rewrites > 0); /* done, no rewrite needed */ } - cp = tor_strdup(escaped_safe_str(ent->new_address)); + cp = tor_strdup(escaped_safe_str_client(ent->new_address)); log_info(LD_APP, "Addressmap: rewriting %s to %s", - escaped_safe_str(address), cp); + escaped_safe_str_client(address), cp); if (ent->expires > 1 && ent->expires < expires) expires = ent->expires; tor_free(cp); @@ -826,7 +860,7 @@ addressmap_rewrite(char *address, size_t maxlen, time_t *expires_out) } log_warn(LD_CONFIG, "Loop detected: we've rewritten %s 16 times! Using it as-is.", - escaped_safe_str(address)); + escaped_safe_str_client(address)); /* it's fine to rewrite a rewrite, but don't loop forever */ if (expires_out) *expires_out = TIME_MAX; @@ -848,9 +882,9 @@ addressmap_rewrite_reverse(char *address, size_t maxlen, time_t *expires_out) tor_snprintf(s, len, "REVERSE[%s]", address); ent = strmap_get(addressmap, s); if (ent) { - cp = tor_strdup(escaped_safe_str(ent->new_address)); + cp = tor_strdup(escaped_safe_str_client(ent->new_address)); log_info(LD_APP, "Rewrote reverse lookup %s -> %s", - escaped_safe_str(s), cp); + escaped_safe_str_client(s), cp); tor_free(cp); strlcpy(address, ent->new_address, maxlen); r = 1; @@ -912,7 +946,9 @@ addressmap_register(const char *address, char *new_address, time_t expires, if (expires > 1) { log_info(LD_APP,"Temporary addressmap ('%s' to '%s') not performed, " "since it's already mapped to '%s'", - safe_str(address), safe_str(new_address), safe_str(ent->new_address)); + safe_str_client(address), + safe_str_client(new_address), + safe_str_client(ent->new_address)); tor_free(new_address); return; } @@ -931,7 +967,8 @@ addressmap_register(const char *address, char *new_address, time_t expires, ent->source = source; log_info(LD_CONFIG, "Addressmap: (re)mapped '%s' to '%s'", - safe_str(address), safe_str(ent->new_address)); + safe_str_client(address), + safe_str_client(ent->new_address)); control_event_address_mapped(address, ent->new_address, expires, NULL); } @@ -951,7 +988,8 @@ client_dns_incr_failures(const char *address) if (ent->num_resolve_failures < SHORT_MAX) ++ent->num_resolve_failures; /* don't overflow */ log_info(LD_APP, "Address %s now has %d resolve failures.", - safe_str(address), ent->num_resolve_failures); + safe_str_client(address), + ent->num_resolve_failures); return ent->num_resolve_failures; } @@ -1230,8 +1268,10 @@ addressmap_register_virtual_address(int type, char *new_address) log_warn(LD_BUG, "Internal confusion: I thought that '%s' was mapped to by " "'%s', but '%s' really maps to '%s'. This is a harmless bug.", - safe_str(new_address), safe_str(*addrp), safe_str(*addrp), - ent?safe_str(ent->new_address):"(nothing)"); + safe_str_client(new_address), + safe_str_client(*addrp), + safe_str_client(*addrp), + ent?safe_str_client(ent->new_address):"(nothing)"); } tor_free(*addrp); @@ -1252,7 +1292,8 @@ addressmap_register_virtual_address(int type, char *new_address) (type == RESOLVED_TYPE_IPV4) ? vent->ipv4_address : vent->hostname_address)); log_info(LD_APP, "Map from %s to %s okay.", - safe_str(*addrp),safe_str(new_address)); + safe_str_client(*addrp), + safe_str_client(new_address)); } #endif @@ -1400,7 +1441,7 @@ connection_ap_handshake_rewrite_and_attach(edge_connection_t *conn, tor_strlower(socks->address); /* normalize it */ strlcpy(orig_address, socks->address, sizeof(orig_address)); log_debug(LD_APP,"Client asked for %s:%d", - safe_str(socks->address), + safe_str_client(socks->address), socks->port); if (socks->command == SOCKS_COMMAND_RESOLVE && @@ -1417,7 +1458,8 @@ connection_ap_handshake_rewrite_and_attach(edge_connection_t *conn, RESOLVED_TYPE_IPV4, tor_strdup(socks->address)); tor_assert(new_addr); log_info(LD_APP, "Automapping %s to %s", - escaped_safe_str(socks->address), safe_str(new_addr)); + escaped_safe_str_client(socks->address), + safe_str_client(new_addr)); strlcpy(socks->address, new_addr, sizeof(socks->address)); } } @@ -1473,7 +1515,7 @@ connection_ap_handshake_rewrite_and_attach(edge_connection_t *conn, * information. */ log_warn(LD_APP,"Missing mapping for virtual address '%s'. Refusing.", - socks->address); /* don't safe_str() this yet. */ + safe_str_client(socks->address)); connection_mark_unattached_ap(conn, END_STREAM_REASON_INTERNAL); return -1; } @@ -1481,11 +1523,12 @@ connection_ap_handshake_rewrite_and_attach(edge_connection_t *conn, /* Parse the address provided by SOCKS. Modify it in-place if it * specifies a hidden-service (.onion) or particular exit node (.exit). */ - addresstype = parse_extended_hostname(socks->address); + addresstype = parse_extended_hostname(socks->address, + remapped_to_exit || options->AllowDotExit); if (addresstype == BAD_HOSTNAME) { log_warn(LD_APP, "Invalid onion hostname %s; rejecting", - safe_str(socks->address)); + safe_str_client(socks->address)); control_event_client_status(LOG_WARN, "SOCKS_BAD_HOSTNAME HOSTNAME=%s", escaped(socks->address)); connection_mark_unattached_ap(conn, END_STREAM_REASON_TORPROTOCOL); @@ -1494,7 +1537,7 @@ connection_ap_handshake_rewrite_and_attach(edge_connection_t *conn, if (addresstype == EXIT_HOSTNAME) { /* foo.exit -- modify conn->chosen_exit_node to specify the exit - * node, and conn->address to hold only the address portion.*/ + * node, and conn->address to hold only the address portion. */ char *s = strrchr(socks->address,'.'); tor_assert(!automap); if (s) { @@ -1505,7 +1548,7 @@ connection_ap_handshake_rewrite_and_attach(edge_connection_t *conn, *s = 0; } else { log_warn(LD_APP,"Malformed exit address '%s.exit'. Refusing.", - safe_str(socks->address)); + safe_str_client(socks->address)); control_event_client_status(LOG_WARN, "SOCKS_BAD_HOSTNAME HOSTNAME=%s", escaped(socks->address)); connection_mark_unattached_ap(conn, END_STREAM_REASON_TORPROTOCOL); @@ -1521,7 +1564,7 @@ connection_ap_handshake_rewrite_and_attach(edge_connection_t *conn, } else { log_warn(LD_APP, "Unrecognized server in exit address '%s.exit'. Refusing.", - safe_str(socks->address)); + safe_str_client(socks->address)); connection_mark_unattached_ap(conn, END_STREAM_REASON_TORPROTOCOL); return -1; } @@ -1535,7 +1578,7 @@ connection_ap_handshake_rewrite_and_attach(edge_connection_t *conn, escaped(socks->address)); log_warn(LD_APP, "Destination '%s' seems to be an invalid hostname. Failing.", - safe_str(socks->address)); + safe_str_client(socks->address)); connection_mark_unattached_ap(conn, END_STREAM_REASON_TORPROTOCOL); return -1; } @@ -1544,18 +1587,6 @@ connection_ap_handshake_rewrite_and_attach(edge_connection_t *conn, uint32_t answer; struct in_addr in; /* Reply to resolves immediately if we can. */ - if (strlen(socks->address) > RELAY_PAYLOAD_SIZE) { - log_warn(LD_APP,"Address to be resolved is too large. Failing."); - control_event_client_status(LOG_WARN, "SOCKS_BAD_HOSTNAME HOSTNAME=%s", - escaped(socks->address)); - connection_ap_handshake_socks_resolved(conn, - RESOLVED_TYPE_ERROR_TRANSIENT, - 0,NULL,-1,TIME_MAX); - connection_mark_unattached_ap(conn, - END_STREAM_REASON_SOCKSPROTOCOL | - END_STREAM_REASON_FLAG_ALREADY_SOCKS_REPLIED); - return -1; - } if (tor_inet_aton(socks->address, &in)) { /* see if it's an IP already */ /* leave it in network order */ answer = in.s_addr; @@ -1585,7 +1616,7 @@ connection_ap_handshake_rewrite_and_attach(edge_connection_t *conn, if (r) { log_info(LD_APP, "Redirecting address %s to exit at enclave router %s", - safe_str(socks->address), r->nickname); + safe_str_client(socks->address), r->nickname); /* use the hex digest, not nickname, in case there are two routers with this nickname */ conn->chosen_exit_name = @@ -1649,12 +1680,12 @@ connection_ap_handshake_rewrite_and_attach(edge_connection_t *conn, strlcpy(conn->rend_data->onion_address, socks->address, sizeof(conn->rend_data->onion_address)); log_info(LD_REND,"Got a hidden service request for ID '%s'", - safe_str(conn->rend_data->onion_address)); + safe_str_client(conn->rend_data->onion_address)); /* see if we already have it cached */ r = rend_cache_lookup_entry(conn->rend_data->onion_address, -1, &entry); if (r<0) { log_warn(LD_BUG,"Invalid service name '%s'", - safe_str(conn->rend_data->onion_address)); + safe_str_client(conn->rend_data->onion_address)); connection_mark_unattached_ap(conn, END_STREAM_REASON_TORPROTOCOL); return -1; } @@ -1676,32 +1707,15 @@ connection_ap_handshake_rewrite_and_attach(edge_connection_t *conn, if (r==0) { conn->_base.state = AP_CONN_STATE_RENDDESC_WAIT; log_info(LD_REND, "Unknown descriptor %s. Fetching.", - safe_str(conn->rend_data->onion_address)); - /* Fetch both, v0 and v2 rend descriptors in parallel. Use whichever - * arrives first. Exception: When using client authorization, only - * fetch v2 descriptors.*/ + safe_str_client(conn->rend_data->onion_address)); rend_client_refetch_v2_renddesc(conn->rend_data); - if (conn->rend_data->auth_type == REND_NO_AUTH) - rend_client_refetch_renddesc(conn->rend_data->onion_address); } else { /* r > 0 */ - if (now - entry->received < NUM_SECONDS_BEFORE_HS_REFETCH) { - conn->_base.state = AP_CONN_STATE_CIRCUIT_WAIT; - log_info(LD_REND, "Descriptor is here and fresh enough. Great."); - if (connection_ap_handshake_attach_circuit(conn) < 0) { - if (!conn->_base.marked_for_close) - connection_mark_unattached_ap(conn, END_STREAM_REASON_CANT_ATTACH); - return -1; - } - } else { - conn->_base.state = AP_CONN_STATE_RENDDESC_WAIT; - log_info(LD_REND, "Stale descriptor %s. Re-fetching.", - safe_str(conn->rend_data->onion_address)); - /* Fetch both, v0 and v2 rend descriptors in parallel. Use whichever - * arrives first. Exception: When using client authorization, only - * fetch v2 descriptors.*/ - rend_client_refetch_v2_renddesc(conn->rend_data); - if (conn->rend_data->auth_type == REND_NO_AUTH) - rend_client_refetch_renddesc(conn->rend_data->onion_address); + conn->_base.state = AP_CONN_STATE_CIRCUIT_WAIT; + log_info(LD_REND, "Descriptor is here. Great."); + if (connection_ap_handshake_attach_circuit(conn) < 0) { + if (!conn->_base.marked_for_close) + connection_mark_unattached_ap(conn, END_STREAM_REASON_CANT_ATTACH); + return -1; } } return 0; @@ -1889,14 +1903,6 @@ connection_ap_handshake_process_socks(edge_connection_t *conn) return -1; } /* else socks handshake is done, continue processing */ - if (hostname_is_noconnect_address(socks->address)) - { - control_event_stream_status(conn, STREAM_EVENT_NEW, 0); - control_event_stream_status(conn, STREAM_EVENT_CLOSED, 0); - connection_mark_unattached_ap(conn, END_STREAM_REASON_DONE); - return -1; - } - if (SOCKS_COMMAND_IS_CONNECT(socks->command)) control_event_stream_status(conn, STREAM_EVENT_NEW, 0); else @@ -2160,7 +2166,7 @@ connection_ap_handshake_send_resolve(edge_connection_t *ap_conn) r = tor_addr_parse_reverse_lookup_name(&addr, a, AF_INET, 1); if (r <= 0) { log_warn(LD_APP, "Rejecting ill-formed reverse lookup of %s", - safe_str(a)); + safe_str_client(a)); connection_mark_unattached_ap(ap_conn, END_STREAM_REASON_INTERNAL); return -1; } @@ -2168,7 +2174,7 @@ connection_ap_handshake_send_resolve(edge_connection_t *ap_conn) r = tor_addr_to_reverse_lookup_name(inaddr_buf, sizeof(inaddr_buf), &addr); if (r < 0) { log_warn(LD_BUG, "Couldn't generate reverse lookup hostname of %s", - safe_str(a)); + safe_str_client(a)); connection_mark_unattached_ap(ap_conn, END_STREAM_REASON_INTERNAL); return -1; } @@ -2178,12 +2184,6 @@ connection_ap_handshake_send_resolve(edge_connection_t *ap_conn) tor_assert(payload_len <= (int)sizeof(inaddr_buf)); } - if (payload_len > RELAY_PAYLOAD_SIZE) { - /* This should be impossible: we don't accept addresses this big. */ - connection_mark_unattached_ap(ap_conn, END_STREAM_REASON_INTERNAL); - return -1; - } - log_debug(LD_APP, "Sending relay cell to begin stream %d.", ap_conn->stream_id); @@ -2215,7 +2215,8 @@ connection_ap_make_link(char *address, uint16_t port, edge_connection_t *conn; log_info(LD_APP,"Making internal %s tunnel to %s:%d ...", - want_onehop ? "direct" : "anonymized" , safe_str(address),port); + want_onehop ? "direct" : "anonymized", + safe_str_client(address), port); conn = edge_connection_new(CONN_TYPE_AP, AF_INET); conn->_base.linked = 1; /* so that we can add it safely below. */ @@ -2526,16 +2527,28 @@ connection_exit_begin_conn(cell_t *cell, circuit_t *circ) tor_free(address); return 0; } - if (or_circ && or_circ->is_first_hop && - !get_options()->AllowSingleHopExits) { + if (or_circ && or_circ->p_conn && !get_options()->AllowSingleHopExits && + (or_circ->is_first_hop || + (!connection_or_digest_is_known_relay( + or_circ->p_conn->identity_digest) && +// XXX022 commented out so we can test it first in 0.2.2.11 -RD +// networkstatus_get_param(NULL, "refuseunknownexits", 1)))) { + get_options()->RefuseUnknownExits))) { /* Don't let clients use us as a single-hop proxy, unless the user - * has explicitly allowed that in the config. It attracts attackers + * has explicitly allowed that in the config. It attracts attackers * and users who'd be better off with, well, single-hop proxies. */ - log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, - "Attempt to open a stream on first hop of circuit. Closing."); +// log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, + log_notice(LD_PROTOCOL, + "Attempt by %s to open a stream %s. Closing.", + safe_str(or_circ->p_conn->_base.address), + or_circ->is_first_hop ? "on first hop of circuit" : + "from unknown relay"); relay_send_end_cell_from_edge(rh.stream_id, circ, - END_STREAM_REASON_TORPROTOCOL, NULL); + or_circ->is_first_hop ? + END_STREAM_REASON_TORPROTOCOL : + END_STREAM_REASON_MISC, + NULL); tor_free(address); return 0; } @@ -2566,6 +2579,11 @@ connection_exit_begin_conn(cell_t *cell, circuit_t *circ) log_debug(LD_EXIT,"Creating new exit connection."); n_stream = edge_connection_new(CONN_TYPE_EXIT, AF_INET); + + /* Remember the tunneled request ID in the new edge connection, so that + * we can measure download times. */ + TO_CONN(n_stream)->dirreq_id = circ->dirreq_id; + n_stream->_base.purpose = EXIT_PURPOSE_CONNECT; n_stream->stream_id = rh.stream_id; @@ -2596,7 +2614,7 @@ connection_exit_begin_conn(cell_t *cell, circuit_t *circ) log_debug(LD_REND,"Finished assigning addr/port"); n_stream->cpath_layer = origin_circ->cpath->prev; /* link it */ - /* add it into the linked list of n_streams on this circuit */ + /* add it into the linked list of p_streams on this circuit */ n_stream->next_stream = origin_circ->p_streams; n_stream->on_circuit = circ; origin_circ->p_streams = n_stream; @@ -2623,7 +2641,7 @@ connection_exit_begin_conn(cell_t *cell, circuit_t *circ) if (rh.command == RELAY_COMMAND_BEGIN_DIR) { tor_assert(or_circ); if (or_circ->p_conn && !tor_addr_is_null(&or_circ->p_conn->real_addr)) - tor_addr_assign(&n_stream->_base.addr, &or_circ->p_conn->real_addr); + tor_addr_copy(&n_stream->_base.addr, &or_circ->p_conn->real_addr); return connection_exit_connect_dir(n_stream); } @@ -2713,7 +2731,7 @@ connection_exit_connect(edge_connection_t *edge_conn) if (!connection_edge_is_rendezvous_stream(edge_conn) && router_compare_to_my_exit_policy(edge_conn)) { log_info(LD_EXIT,"%s:%d failed exit policy. Closing.", - escaped_safe_str(conn->address), conn->port); + escaped_safe_str_client(conn->address), conn->port); connection_edge_end(edge_conn, END_STREAM_REASON_EXITPOLICY); circuit_detach_stream(circuit_get_by_edge_conn(edge_conn), edge_conn); connection_free(conn); @@ -2735,7 +2753,7 @@ connection_exit_connect(edge_connection_t *edge_conn) case 0: conn->state = EXIT_CONN_STATE_CONNECTING; - connection_watch_events(conn, EV_WRITE | EV_READ); + connection_watch_events(conn, READ_EVENT | WRITE_EVENT); /* writable indicates finish; * readable/error indicates broken link in windows-land. */ return; @@ -2748,7 +2766,7 @@ connection_exit_connect(edge_connection_t *edge_conn) log_warn(LD_BUG,"newly connected conn had data waiting!"); // connection_start_writing(conn); } - connection_watch_events(conn, EV_READ); + connection_watch_events(conn, READ_EVENT); /* also, deliver a 'connected' cell back through the circuit. */ if (connection_edge_is_rendezvous_stream(edge_conn)) { @@ -2795,13 +2813,17 @@ connection_exit_connect_dir(edge_connection_t *exitconn) dirconn = dir_connection_new(AF_INET); - tor_addr_assign(&dirconn->_base.addr, &exitconn->_base.addr); + tor_addr_copy(&dirconn->_base.addr, &exitconn->_base.addr); dirconn->_base.port = 0; dirconn->_base.address = tor_strdup(exitconn->_base.address); dirconn->_base.type = CONN_TYPE_DIR; dirconn->_base.purpose = DIR_PURPOSE_SERVER; dirconn->_base.state = DIR_CONN_STATE_SERVER_COMMAND_WAIT; + /* Note that the new dir conn belongs to the same tunneled request as + * the edge conn, so that we can measure download times. */ + TO_CONN(dirconn)->dirreq_id = TO_CONN(exitconn)->dirreq_id; + connection_link_connections(TO_CONN(dirconn), TO_CONN(exitconn)); if (connection_add(TO_CONN(exitconn))<0) { @@ -2852,10 +2874,16 @@ connection_edge_is_rendezvous_stream(edge_connection_t *conn) * to exit from it, or 0 if it probably will not allow it. * (We might be uncertain if conn's destination address has not yet been * resolved.) + * + * If <b>excluded_means_no</b> is 1 and Exclude*Nodes is set and excludes + * this relay, return 0. */ int -connection_ap_can_use_exit(edge_connection_t *conn, routerinfo_t *exit) +connection_ap_can_use_exit(edge_connection_t *conn, routerinfo_t *exit, + int excluded_means_no) { + or_options_t *options = get_options(); + tor_assert(conn); tor_assert(conn->_base.type == CONN_TYPE_AP); tor_assert(conn->socks_request); @@ -2901,20 +2929,35 @@ connection_ap_can_use_exit(edge_connection_t *conn, routerinfo_t *exit) if (!conn->chosen_exit_name && policy_is_reject_star(exit->exit_policy)) return 0; } + if (options->_ExcludeExitNodesUnion && + (options->StrictNodes || excluded_means_no) && + routerset_contains_router(options->_ExcludeExitNodesUnion, exit)) { + /* If we are trying to avoid this node as exit, and we have StrictNodes + * set, then this is not a suitable exit. Refuse it. + * + * If we don't have StrictNodes set, then this function gets called in + * two contexts. First, we've got a circuit open and we want to know + * whether we can use it. In that case, we somehow built this circuit + * despite having the last hop in ExcludeExitNodes, so we should be + * willing to use it. Second, we are evaluating whether this is an + * acceptable exit for a new circuit. In that case, skip it. */ + return 0; + } + return 1; } /** If address is of the form "y.onion" with a well-formed handle y: * Put a NUL after y, lower-case it, and return ONION_HOSTNAME. * - * If address is of the form "y.exit": + * If address is of the form "y.exit" and <b>allowdotexit</b> is true: * Put a NUL after y and return EXIT_HOSTNAME. * * Otherwise: * Return NORMAL_HOSTNAME and change nothing. */ hostname_type_t -parse_extended_hostname(char *address) +parse_extended_hostname(char *address, int allowdotexit) { char *s; char query[REND_SERVICE_ID_LEN_BASE32+1]; @@ -2923,8 +2966,16 @@ parse_extended_hostname(char *address) if (!s) return NORMAL_HOSTNAME; /* no dot, thus normal */ if (!strcmp(s+1,"exit")) { - *s = 0; /* NUL-terminate it */ - return EXIT_HOSTNAME; /* .exit */ + if (allowdotexit) { + *s = 0; /* NUL-terminate it */ + return EXIT_HOSTNAME; /* .exit */ + } else { + log_warn(LD_APP, "The \".exit\" notation is disabled in Tor due to " + "security risks. Set AllowDotExit in your torrc to enable " + "it."); + /* FFFF send a controller event too to notify Vidalia users */ + return BAD_HOSTNAME; + } } if (strcmp(s+1,"onion")) return NORMAL_HOSTNAME; /* neither .exit nor .onion, thus normal */ @@ -2943,11 +2994,3 @@ failed: return BAD_HOSTNAME; } -/** Check if the address is of the form "y.noconnect" - */ -int -hostname_is_noconnect_address(const char *address) -{ - return ! strcasecmpend(address, ".noconnect"); -} - diff --git a/src/or/connection_edge.h b/src/or/connection_edge.h new file mode 100644 index 0000000000..c3d6098c5a --- /dev/null +++ b/src/or/connection_edge.h @@ -0,0 +1,98 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2010, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file connection_edge.h + * \brief Header file for connection_edge.c. + **/ + +#ifndef _TOR_CONNECTION_EDGE_H +#define _TOR_CONNECTION_EDGE_H + +#define connection_mark_unattached_ap(conn, endreason) \ + _connection_mark_unattached_ap((conn), (endreason), __LINE__, _SHORT_FILE_) + +void _connection_mark_unattached_ap(edge_connection_t *conn, int endreason, + int line, const char *file); +int connection_edge_reached_eof(edge_connection_t *conn); +int connection_edge_process_inbuf(edge_connection_t *conn, + int package_partial); +int connection_edge_destroy(circid_t circ_id, edge_connection_t *conn); +int connection_edge_end(edge_connection_t *conn, uint8_t reason); +int connection_edge_end_errno(edge_connection_t *conn); +int connection_edge_finished_flushing(edge_connection_t *conn); +int connection_edge_finished_connecting(edge_connection_t *conn); + +int connection_ap_handshake_send_begin(edge_connection_t *ap_conn); +int connection_ap_handshake_send_resolve(edge_connection_t *ap_conn); + +edge_connection_t *connection_ap_make_link(char *address, uint16_t port, + const char *digest, + int use_begindir, int want_onehop); +void connection_ap_handshake_socks_reply(edge_connection_t *conn, char *reply, + size_t replylen, + int endreason); +void connection_ap_handshake_socks_resolved(edge_connection_t *conn, + int answer_type, + size_t answer_len, + const char *answer, + int ttl, + time_t expires); + +int connection_exit_begin_conn(cell_t *cell, circuit_t *circ); +int connection_exit_begin_resolve(cell_t *cell, or_circuit_t *circ); +void connection_exit_connect(edge_connection_t *conn); +int connection_edge_is_rendezvous_stream(edge_connection_t *conn); +int connection_ap_can_use_exit(edge_connection_t *conn, routerinfo_t *exit, + int excluded_means_no); +void connection_ap_expire_beginning(void); +void connection_ap_attach_pending(void); +void connection_ap_fail_onehop(const char *failed_digest, + cpath_build_state_t *build_state); +void circuit_discard_optional_exit_enclaves(extend_info_t *info); +int connection_ap_detach_retriable(edge_connection_t *conn, + origin_circuit_t *circ, + int reason); +int connection_ap_process_transparent(edge_connection_t *conn); + +int address_is_invalid_destination(const char *address, int client); + +void addressmap_init(void); +void addressmap_clean(time_t now); +void addressmap_clear_configured(void); +void addressmap_clear_transient(void); +void addressmap_free_all(void); +int addressmap_rewrite(char *address, size_t maxlen, time_t *expires_out); +int addressmap_have_mapping(const char *address, int update_timeout); + +void addressmap_register(const char *address, char *new_address, + time_t expires, addressmap_entry_source_t source); +int parse_virtual_addr_network(const char *val, int validate_only, + char **msg); +int client_dns_incr_failures(const char *address); +void client_dns_clear_failures(const char *address); +void client_dns_set_addressmap(const char *address, uint32_t val, + const char *exitname, int ttl); +const char *addressmap_register_virtual_address(int type, char *new_address); +void addressmap_get_mappings(smartlist_t *sl, time_t min_expires, + time_t max_expires, int want_expiry); +int connection_ap_handshake_rewrite_and_attach(edge_connection_t *conn, + origin_circuit_t *circ, + crypt_path_t *cpath); +int hostname_is_noconnect_address(const char *address); + +/** Possible return values for parse_extended_hostname. */ +typedef enum hostname_type_t { + NORMAL_HOSTNAME, ONION_HOSTNAME, EXIT_HOSTNAME, BAD_HOSTNAME +} hostname_type_t; +hostname_type_t parse_extended_hostname(char *address, int allowdotexit); + +#if defined(HAVE_NET_IF_H) && defined(HAVE_NET_PFVAR_H) +int get_pf_socket(void); +#endif + +#endif + diff --git a/src/or/connection_or.c b/src/or/connection_or.c index 3ac0bf5452..0ba1bca31f 100644 --- a/src/or/connection_or.c +++ b/src/or/connection_or.c @@ -11,6 +11,22 @@ **/ #include "or.h" +#include "buffers.h" +#include "circuitbuild.h" +#include "command.h" +#include "config.h" +#include "connection.h" +#include "connection_or.h" +#include "control.h" +#include "dirserv.h" +#include "geoip.h" +#include "main.h" +#include "networkstatus.h" +#include "reasons.h" +#include "relay.h" +#include "rephist.h" +#include "router.h" +#include "routerlist.h" static int connection_tls_finish_handshake(or_connection_t *conn); static int connection_or_process_cells_from_inbuf(or_connection_t *conn); @@ -80,10 +96,8 @@ connection_or_clear_identity_map(void) } }); - if (orconn_identity_map) { - digestmap_free(orconn_identity_map, NULL); - orconn_identity_map = NULL; - } + digestmap_free(orconn_identity_map, NULL); + orconn_identity_map = NULL; } /** Change conn->identity_digest to digest, and add conn into @@ -187,66 +201,6 @@ connection_or_reached_eof(or_connection_t *conn) return 0; } -/** Read conn's inbuf. If the http response from the proxy is all - * here, make sure it's good news, and begin the tls handshake. If - * it's bad news, close the connection and return -1. Else return 0 - * and hope for better luck next time. - */ -static int -connection_or_read_proxy_response(or_connection_t *or_conn) -{ - char *headers; - char *reason=NULL; - int status_code; - time_t date_header; - connection_t *conn = TO_CONN(or_conn); - - switch (fetch_from_buf_http(conn->inbuf, - &headers, MAX_HEADERS_SIZE, - NULL, NULL, 10000, 0)) { - case -1: /* overflow */ - log_warn(LD_PROTOCOL, - "Your https proxy sent back an oversized response. Closing."); - return -1; - case 0: - log_info(LD_OR,"https proxy response not all here yet. Waiting."); - return 0; - /* case 1, fall through */ - } - - if (parse_http_response(headers, &status_code, &date_header, - NULL, &reason) < 0) { - log_warn(LD_OR, - "Unparseable headers from proxy (connecting to '%s'). Closing.", - conn->address); - tor_free(headers); - return -1; - } - if (!reason) reason = tor_strdup("[no reason given]"); - - if (status_code == 200) { - log_info(LD_OR, - "HTTPS connect to '%s' successful! (200 %s) Starting TLS.", - conn->address, escaped(reason)); - tor_free(reason); - if (connection_tls_start_handshake(or_conn, 0) < 0) { - /* TLS handshaking error of some kind. */ - connection_mark_for_close(conn); - - return -1; - } - return 0; - } - /* else, bad news on the status code */ - log_warn(LD_OR, - "The https proxy sent back an unexpected status code %d (%s). " - "Closing.", - status_code, escaped(reason)); - tor_free(reason); - connection_mark_for_close(conn); - return -1; -} - /** Handle any new bytes that have come in on connection <b>conn</b>. * If conn is in 'open' state, hand it to * connection_or_process_cells_from_inbuf() @@ -255,11 +209,24 @@ connection_or_read_proxy_response(or_connection_t *or_conn) int connection_or_process_inbuf(or_connection_t *conn) { + int ret; tor_assert(conn); switch (conn->_base.state) { - case OR_CONN_STATE_PROXY_READING: - return connection_or_read_proxy_response(conn); + case OR_CONN_STATE_PROXY_HANDSHAKING: + ret = connection_read_proxy_handshake(TO_CONN(conn)); + + /* start TLS after handshake completion, or deal with error */ + if (ret == 1) { + tor_assert(TO_CONN(conn)->proxy_state == PROXY_CONNECTED); + if (connection_tls_start_handshake(conn, 0) < 0) + ret = -1; + } + if (ret < 0) { + connection_mark_for_close(TO_CONN(conn)); + } + + return ret; case OR_CONN_STATE_OPEN: case OR_CONN_STATE_OR_HANDSHAKING: return connection_or_process_cells_from_inbuf(conn); @@ -312,11 +279,7 @@ connection_or_finished_flushing(or_connection_t *conn) assert_connection_ok(TO_CONN(conn),0); switch (conn->_base.state) { - case OR_CONN_STATE_PROXY_FLUSHING: - log_debug(LD_OR,"finished sending CONNECT to proxy."); - conn->_base.state = OR_CONN_STATE_PROXY_READING; - connection_stop_writing(TO_CONN(conn)); - break; + case OR_CONN_STATE_PROXY_HANDSHAKING: case OR_CONN_STATE_OPEN: case OR_CONN_STATE_OR_HANDSHAKING: connection_stop_writing(TO_CONN(conn)); @@ -334,37 +297,34 @@ connection_or_finished_flushing(or_connection_t *conn) int connection_or_finished_connecting(or_connection_t *or_conn) { + int proxy_type; connection_t *conn; tor_assert(or_conn); conn = TO_CONN(or_conn); tor_assert(conn->state == OR_CONN_STATE_CONNECTING); - log_debug(LD_OR,"OR connect() to router at %s:%u finished.", + log_debug(LD_HANDSHAKE,"OR connect() to router at %s:%u finished.", conn->address,conn->port); control_event_bootstrap(BOOTSTRAP_STATUS_HANDSHAKE, 0); - if (get_options()->HttpsProxy) { - char buf[1024]; - char *base64_authenticator=NULL; - const char *authenticator = get_options()->HttpsProxyAuthenticator; + proxy_type = PROXY_NONE; - if (authenticator) { - base64_authenticator = alloc_http_authenticator(authenticator); - if (!base64_authenticator) - log_warn(LD_OR, "Encoding https authenticator failed"); - } - if (base64_authenticator) { - tor_snprintf(buf, sizeof(buf), "CONNECT %s:%d HTTP/1.1\r\n" - "Proxy-Authorization: Basic %s\r\n\r\n", - fmt_addr(&conn->addr), - conn->port, base64_authenticator); - tor_free(base64_authenticator); - } else { - tor_snprintf(buf, sizeof(buf), "CONNECT %s:%d HTTP/1.0\r\n\r\n", - fmt_addr(&conn->addr), conn->port); + if (get_options()->HttpsProxy) + proxy_type = PROXY_CONNECT; + else if (get_options()->Socks4Proxy) + proxy_type = PROXY_SOCKS4; + else if (get_options()->Socks5Proxy) + proxy_type = PROXY_SOCKS5; + + if (proxy_type != PROXY_NONE) { + /* start proxy handshake */ + if (connection_proxy_connect(conn, proxy_type) < 0) { + connection_mark_for_close(conn); + return -1; } - connection_write_to_buf(buf, strlen(buf), conn); - conn->state = OR_CONN_STATE_PROXY_FLUSHING; + + connection_start_reading(conn); + conn->state = OR_CONN_STATE_PROXY_HANDSHAKING; return 0; } @@ -376,6 +336,74 @@ connection_or_finished_connecting(or_connection_t *or_conn) return 0; } +/** Return 1 if identity digest <b>id_digest</b> is known to be a + * currently or recently running relay. Otherwise return 0. */ +int +connection_or_digest_is_known_relay(const char *id_digest) +{ + if (router_get_consensus_status_by_id(id_digest)) + return 1; /* It's in the consensus: "yes" */ + if (router_get_by_digest(id_digest)) + return 1; /* Not in the consensus, but we have a descriptor for + * it. Probably it was in a recent consensus. "Yes". */ + return 0; +} + +/** Set the per-conn read and write limits for <b>conn</b>. If it's a known + * relay, we will rely on the global read and write buckets, so give it + * per-conn limits that are big enough they'll never matter. But if it's + * not a known relay, first check if we set PerConnBwRate/Burst, then + * check if the consensus sets them, else default to 'big enough'. + */ +static void +connection_or_update_token_buckets_helper(or_connection_t *conn, int reset, + or_options_t *options) +{ + int rate, burst; /* per-connection rate limiting params */ + if (connection_or_digest_is_known_relay(conn->identity_digest)) { + /* It's in the consensus, or we have a descriptor for it meaning it + * was probably in a recent consensus. It's a recognized relay: + * give it full bandwidth. */ + rate = (int)options->BandwidthRate; + burst = (int)options->BandwidthBurst; + } else { + /* Not a recognized relay. Squeeze it down based on the suggested + * bandwidth parameters in the consensus, but allow local config + * options to override. */ + rate = options->PerConnBWRate ? (int)options->PerConnBWRate : + (int)networkstatus_get_param(NULL, "bwconnrate", + (int)options->BandwidthRate); + burst = options->PerConnBWBurst ? (int)options->PerConnBWBurst : + (int)networkstatus_get_param(NULL, "bwconnburst", + (int)options->BandwidthBurst); + } + + conn->bandwidthrate = rate; + conn->bandwidthburst = burst; + if (reset) { /* set up the token buckets to be full */ + conn->read_bucket = conn->write_bucket = burst; + return; + } + /* If the new token bucket is smaller, take out the extra tokens. + * (If it's larger, don't -- the buckets can grow to reach the cap.) */ + if (conn->read_bucket > burst) + conn->read_bucket = burst; + if (conn->write_bucket > burst) + conn->write_bucket = burst; +} + +/** Either our set of relays or our per-conn rate limits have changed. + * Go through all the OR connections and update their token buckets. */ +void +connection_or_update_token_buckets(smartlist_t *conns, or_options_t *options) +{ + SMARTLIST_FOREACH(conns, connection_t *, conn, + { + if (connection_speaks_cells(conn)) + connection_or_update_token_buckets_helper(TO_OR_CONN(conn), 0, options); + }); +} + /** If we don't necessarily know the router we're connecting to, but we * have an addr/port/id_digest, then fill in as much as we can. Start * by checking to see if this describes a router we know. */ @@ -385,11 +413,9 @@ connection_or_init_conn_from_address(or_connection_t *conn, const char *id_digest, int started_here) { - or_options_t *options = get_options(); routerinfo_t *r = router_get_by_digest(id_digest); - conn->bandwidthrate = (int)options->BandwidthRate; - conn->read_bucket = conn->bandwidthburst = (int)options->BandwidthBurst; connection_or_set_identity_digest(conn, id_digest); + connection_or_update_token_buckets_helper(conn, 1, get_options()); conn->_base.port = port; tor_addr_copy(&conn->_base.addr, addr); @@ -631,7 +657,7 @@ connection_or_group_set_badness(or_connection_t *head) /* We have at least one open canonical connection to this router, * and this one is open but not canonical. Mark it bad. */ log_info(LD_OR, - "Marking OR conn to %s:%d as too old for new circuits: " + "Marking OR conn to %s:%d as unsuitable for new circuits: " "(fd %d, %d secs old). It is not canonical, and we have " "another connection to that OR that is.", or_conn->_base.address, or_conn->_base.port, or_conn->_base.s, @@ -671,7 +697,7 @@ connection_or_group_set_badness(or_connection_t *head) even when we're being forgiving. */ if (best->is_canonical) { log_info(LD_OR, - "Marking OR conn to %s:%d as too old for new circuits: " + "Marking OR conn to %s:%d as unsuitable for new circuits: " "(fd %d, %d secs old). We have a better canonical one " "(fd %d; %d secs old).", or_conn->_base.address, or_conn->_base.port, or_conn->_base.s, @@ -681,9 +707,9 @@ connection_or_group_set_badness(or_connection_t *head) } else if (!tor_addr_compare(&or_conn->real_addr, &best->real_addr, CMP_EXACT)) { log_info(LD_OR, - "Marking OR conn to %s:%d as too old for new circuits: " - "(fd %d, %d secs old). We have a better one " - "(fd %d; %d secs old).", + "Marking OR conn to %s:%d as unsuitable for new circuits: " + "(fd %d, %d secs old). We have a better one with the " + "same address (fd %d; %d secs old).", or_conn->_base.address, or_conn->_base.port, or_conn->_base.s, (int)(now - or_conn->_base.timestamp_created), best->_base.s, (int)(now - best->_base.timestamp_created)); @@ -753,6 +779,7 @@ connection_or_connect(const tor_addr_t *_addr, uint16_t port, or_connection_t *conn; or_options_t *options = get_options(); int socket_error = 0; + int using_proxy = 0; tor_addr_t addr; tor_assert(_addr); @@ -771,19 +798,27 @@ connection_or_connect(const tor_addr_t *_addr, uint16_t port, conn->_base.state = OR_CONN_STATE_CONNECTING; control_event_or_conn_status(conn, OR_CONN_EVENT_LAUNCHED, 0); + /* use a proxy server if available */ if (options->HttpsProxy) { - /* we shouldn't connect directly. use the https proxy instead. */ - tor_addr_from_ipv4h(&addr, options->HttpsProxyAddr); + using_proxy = 1; + tor_addr_copy(&addr, &options->HttpsProxyAddr); port = options->HttpsProxyPort; + } else if (options->Socks4Proxy) { + using_proxy = 1; + tor_addr_copy(&addr, &options->Socks4ProxyAddr); + port = options->Socks4ProxyPort; + } else if (options->Socks5Proxy) { + using_proxy = 1; + tor_addr_copy(&addr, &options->Socks5ProxyAddr); + port = options->Socks5ProxyPort; } switch (connection_connect(TO_CONN(conn), conn->_base.address, &addr, port, &socket_error)) { case -1: /* If the connection failed immediately, and we're using - * an https proxy, our https proxy is down. Don't blame the - * Tor server. */ - if (!options->HttpsProxy) + * a proxy, our proxy is down. Don't blame the Tor server. */ + if (!using_proxy) entry_guard_register_connect_status(conn->identity_digest, 0, 1, time(NULL)); connection_or_connect_failed(conn, @@ -792,7 +827,7 @@ connection_or_connect(const tor_addr_t *_addr, uint16_t port, connection_free(TO_CONN(conn)); return NULL; case 0: - connection_watch_events(TO_CONN(conn), EV_READ | EV_WRITE); + connection_watch_events(TO_CONN(conn), READ_EVENT | WRITE_EVENT); /* writable indicates finish, readable indicates broken link, error indicates broken link on windows */ return conn; @@ -819,13 +854,14 @@ connection_tls_start_handshake(or_connection_t *conn, int receiving) { conn->_base.state = OR_CONN_STATE_TLS_HANDSHAKING; conn->tls = tor_tls_new(conn->_base.s, receiving); - tor_tls_set_logged_address(conn->tls, escaped_safe_str(conn->_base.address)); + tor_tls_set_logged_address(conn->tls, // XXX client and relay? + escaped_safe_str(conn->_base.address)); if (!conn->tls) { log_warn(LD_BUG,"tor_tls_new failed. Closing."); return -1; } connection_start_reading(TO_CONN(conn)); - log_debug(LD_OR,"starting TLS handshake on fd %d", conn->_base.s); + log_debug(LD_HANDSHAKE,"starting TLS handshake on fd %d", conn->_base.s); note_crypto_pk_op(receiving ? TLS_HANDSHAKE_S : TLS_HANDSHAKE_C); if (connection_tls_continue_handshake(conn) < 0) { @@ -932,23 +968,26 @@ connection_or_nonopen_was_started_here(or_connection_t *conn) * return -1 if he is lying, broken, or otherwise something is wrong. * * If we initiated this connection (<b>started_here</b> is true), make sure - * the other side sent sent a correctly formed certificate. If I initiated the + * the other side sent a correctly formed certificate. If I initiated the * connection, make sure it's the right guy. * * Otherwise (if we _didn't_ initiate this connection), it's okay for * the certificate to be weird or absent. * * If we return 0, and the certificate is as expected, write a hash of the - * identity key into digest_rcvd, which must have DIGEST_LEN space in it. (If - * we return -1 this buffer is undefined.) If the certificate is invalid - * or missing on an incoming connection, we return 0 and set digest_rcvd to - * DIGEST_LEN 0 bytes. + * identity key into <b>digest_rcvd_out</b>, which must have DIGEST_LEN + * space in it. + * If the certificate is invalid or missing on an incoming connection, + * we return 0 and set <b>digest_rcvd_out</b> to DIGEST_LEN NUL bytes. + * (If we return -1, the contents of this buffer are undefined.) * * As side effects, * 1) Set conn->circ_id_type according to tor-spec.txt. * 2) If we're an authdirserver and we initiated the connection: drop all * descriptors that claim to be on that IP/port but that aren't * this guy; and note that this guy is reachable. + * 3) If this is a bridge and we didn't configure its identity + * fingerprint, remember the keyid we just learned. */ static int connection_or_check_valid_tls_handshake(or_connection_t *conn, @@ -959,19 +998,20 @@ connection_or_check_valid_tls_handshake(or_connection_t *conn, or_options_t *options = get_options(); int severity = server_mode(options) ? LOG_PROTOCOL_WARN : LOG_WARN; const char *safe_address = - started_here ? conn->_base.address : safe_str(conn->_base.address); + started_here ? conn->_base.address : + safe_str_client(conn->_base.address); const char *conn_type = started_here ? "outgoing" : "incoming"; int has_cert = 0, has_identity=0; check_no_tls_errors(); has_cert = tor_tls_peer_has_cert(conn->tls); if (started_here && !has_cert) { - log_info(LD_PROTOCOL,"Tried connecting to router at %s:%d, but it didn't " + log_info(LD_HANDSHAKE,"Tried connecting to router at %s:%d, but it didn't " "send a cert! Closing.", safe_address, conn->_base.port); return -1; } else if (!has_cert) { - log_debug(LD_PROTOCOL,"Got incoming connection with no certificate. " + log_debug(LD_HANDSHAKE,"Got incoming connection with no certificate. " "That's ok."); } check_no_tls_errors(); @@ -980,15 +1020,16 @@ connection_or_check_valid_tls_handshake(or_connection_t *conn, int v = tor_tls_verify(started_here?severity:LOG_INFO, conn->tls, &identity_rcvd); if (started_here && v<0) { - log_fn(severity,LD_OR,"Tried connecting to router at %s:%d: It" + log_fn(severity,LD_HANDSHAKE,"Tried connecting to router at %s:%d: It" " has a cert but it's invalid. Closing.", safe_address, conn->_base.port); return -1; } else if (v<0) { - log_info(LD_PROTOCOL,"Incoming connection gave us an invalid cert " + log_info(LD_HANDSHAKE,"Incoming connection gave us an invalid cert " "chain; ignoring."); } else { - log_debug(LD_OR,"The certificate seems to be valid on %s connection " + log_debug(LD_HANDSHAKE, + "The certificate seems to be valid on %s connection " "with %s:%d", conn_type, safe_address, conn->_base.port); } check_no_tls_errors(); @@ -1015,9 +1056,13 @@ connection_or_check_valid_tls_handshake(or_connection_t *conn, conn->nickname[0] = '$'; base16_encode(conn->nickname+1, HEX_DIGEST_LEN+1, conn->identity_digest, DIGEST_LEN); - log_info(LD_OR, "Connected to router %s at %s:%d without knowing " + log_info(LD_HANDSHAKE, "Connected to router %s at %s:%d without knowing " "its key. Hoping for the best.", conn->nickname, conn->_base.address, conn->_base.port); + /* if it's a bridge and we didn't know its identity fingerprint, now + * we do -- remember it for future attempts. */ + learned_router_identity(&conn->_base.addr, conn->_base.port, + digest_rcvd_out); } if (started_here) { @@ -1031,7 +1076,7 @@ connection_or_check_valid_tls_handshake(or_connection_t *conn, base16_encode(seen, sizeof(seen), digest_rcvd_out, DIGEST_LEN); base16_encode(expected, sizeof(expected), conn->identity_digest, DIGEST_LEN); - log_fn(severity, LD_OR, + log_fn(severity, LD_HANDSHAKE, "Tried connecting to router at %s:%d, but identity key was not " "as expected: wanted %s but got %s.", conn->_base.address, conn->_base.port, expected, seen); @@ -1073,8 +1118,8 @@ connection_tls_finish_handshake(or_connection_t *conn) char digest_rcvd[DIGEST_LEN]; int started_here = connection_or_nonopen_was_started_here(conn); - log_debug(LD_OR,"tls handshake with %s done. verifying.", - safe_str(conn->_base.address)); + log_debug(LD_HANDSHAKE,"tls handshake with %s done. verifying.", + safe_str_client(conn->_base.address)); directory_set_dirty(); @@ -1082,6 +1127,8 @@ connection_tls_finish_handshake(or_connection_t *conn) digest_rcvd) < 0) return -1; + circuit_build_times_network_is_live(&circ_times); + if (tor_tls_used_v1_handshake(conn->tls)) { conn->link_proto = 1; if (!started_here) { @@ -1117,7 +1164,8 @@ connection_init_or_handshake_state(or_connection_t *conn, int started_here) void or_handshake_state_free(or_handshake_state_t *state) { - tor_assert(state); + if (!state) + return; memset(state, 0xBE, sizeof(or_handshake_state_t)); tor_free(state); } @@ -1134,6 +1182,7 @@ connection_or_set_state_open(or_connection_t *conn) control_event_or_conn_status(conn, OR_CONN_EVENT_CONNECTED, 0); if (started_here) { + circuit_build_times_network_is_live(&circ_times); rep_hist_note_connect_succeeded(conn->identity_digest, now); if (entry_guard_register_connect_status(conn->identity_digest, 1, 0, now) < 0) { @@ -1158,10 +1207,10 @@ connection_or_set_state_open(or_connection_t *conn) } } } - if (conn->handshake_state) { - or_handshake_state_free(conn->handshake_state); - conn->handshake_state = NULL; - } + + or_handshake_state_free(conn->handshake_state); + conn->handshake_state = NULL; + connection_start_reading(TO_CONN(conn)); circuit_n_conn_done(conn, 1); /* send the pending creates, if any. */ @@ -1234,6 +1283,7 @@ connection_or_process_cells_from_inbuf(or_connection_t *conn) if (connection_fetch_var_cell_from_buf(conn, &var_cell)) { if (!var_cell) return 0; /* not yet. */ + circuit_build_times_network_is_live(&circ_times); command_process_var_cell(var_cell, conn); var_cell_free(var_cell); } else { @@ -1243,6 +1293,7 @@ connection_or_process_cells_from_inbuf(or_connection_t *conn) available? */ return 0; /* not yet */ + circuit_build_times_network_is_live(&circ_times); connection_fetch_from_buf(buf, CELL_NETWORK_SIZE, TO_CONN(conn)); /* retrieve cell info from buf (create the host-order struct from the diff --git a/src/or/connection_or.h b/src/or/connection_or.h new file mode 100644 index 0000000000..e4e99a6ea6 --- /dev/null +++ b/src/or/connection_or.h @@ -0,0 +1,61 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2010, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file connection_or.h + * \brief Header file for connection_or.c. + **/ + +#ifndef _TOR_CONNECTION_OR_H +#define _TOR_CONNECTION_OR_H + +void connection_or_remove_from_identity_map(or_connection_t *conn); +void connection_or_clear_identity_map(void); +or_connection_t *connection_or_get_for_extend(const char *digest, + const tor_addr_t *target_addr, + const char **msg_out, + int *launch_out); +void connection_or_set_bad_connections(void); + +int connection_or_reached_eof(or_connection_t *conn); +int connection_or_process_inbuf(or_connection_t *conn); +int connection_or_flushed_some(or_connection_t *conn); +int connection_or_finished_flushing(or_connection_t *conn); +int connection_or_finished_connecting(or_connection_t *conn); +int connection_or_digest_is_known_relay(const char *id_digest); +void connection_or_update_token_buckets(smartlist_t *conns, + or_options_t *options); + +void connection_or_connect_failed(or_connection_t *conn, + int reason, const char *msg); +or_connection_t *connection_or_connect(const tor_addr_t *addr, uint16_t port, + const char *id_digest); + +int connection_tls_start_handshake(or_connection_t *conn, int receiving); +int connection_tls_continue_handshake(or_connection_t *conn); + +void or_handshake_state_free(or_handshake_state_t *state); +int connection_or_set_state_open(or_connection_t *conn); +void connection_or_write_cell_to_buf(const cell_t *cell, + or_connection_t *conn); +void connection_or_write_var_cell_to_buf(const var_cell_t *cell, + or_connection_t *conn); +int connection_or_send_destroy(circid_t circ_id, or_connection_t *conn, + int reason); +int connection_or_send_netinfo(or_connection_t *conn); +int connection_or_send_cert(or_connection_t *conn); +int connection_or_send_link_auth(or_connection_t *conn); +int connection_or_compute_link_auth_hmac(or_connection_t *conn, + char *hmac_out); +int is_or_protocol_version_known(uint16_t version); + +void cell_pack(packed_cell_t *dest, const cell_t *src); +void var_cell_pack_header(const var_cell_t *cell, char *hdr_out); +var_cell_t *var_cell_new(uint16_t payload_len); +void var_cell_free(var_cell_t *cell); + +#endif + diff --git a/src/or/control.c b/src/or/control.c index 7eb2e042f2..7cbb1bd1f6 100644 --- a/src/or/control.c +++ b/src/or/control.c @@ -11,6 +11,26 @@ #define CONTROL_PRIVATE #include "or.h" +#include "buffers.h" +#include "circuitbuild.h" +#include "circuitlist.h" +#include "circuituse.h" +#include "config.h" +#include "connection.h" +#include "connection_edge.h" +#include "control.h" +#include "directory.h" +#include "dirserv.h" +#include "dnsserv.h" +#include "geoip.h" +#include "hibernate.h" +#include "main.h" +#include "networkstatus.h" +#include "policies.h" +#include "reasons.h" +#include "router.h" +#include "routerlist.h" +#include "routerparse.h" /** Yield true iff <b>s</b> is the state of a control_connection_t that has * finished authentication and is accepting commands. */ @@ -43,7 +63,8 @@ #define EVENT_STREAM_BANDWIDTH_USED 0x0014 #define EVENT_CLIENTS_SEEN 0x0015 #define EVENT_NEWCONSENSUS 0x0016 -#define _EVENT_MAX 0x0016 +#define EVENT_BUILDTIMEOUT_SET 0x0017 +#define _EVENT_MAX 0x0017 /* If _EVENT_MAX ever hits 0x0020, we need to make the mask wider. */ /** Bitfield: The bit 1<<e is set if <b>any</b> open control @@ -54,13 +75,9 @@ **/ typedef uint32_t event_mask_t; -/** An event mask of all the events that controller with the LONG_NAMES option - * set is interested in receiving. */ -static event_mask_t global_event_mask1long = 0; - -/** An event mask of all the events that controller with the SHORT_NAMES option - * set is interested in receiving. */ -static event_mask_t global_event_mask1short = 0; +/** An event mask of all the events that any controller is interested in + * receiving. */ +static event_mask_t global_event_mask = 0; /** True iff we have disabled log messages from being sent to the controller */ static int disable_log_messages = 0; @@ -68,13 +85,7 @@ static int disable_log_messages = 0; /** Macro: true if any control connection is interested in events of type * <b>e</b>. */ #define EVENT_IS_INTERESTING(e) \ - ((global_event_mask1long|global_event_mask1short) & (1<<(e))) -/** Macro: true if any control connection with the LONG_NAMES option is - * interested in events of type <b>e</b>. */ -#define EVENT_IS_INTERESTING1L(e) (global_event_mask1long & (1<<(e))) -/** Macro: true if any control connection with the SHORT_NAMES option is - * interested in events of type <b>e</b>. */ -#define EVENT_IS_INTERESTING1S(e) (global_event_mask1short & (1<<(e))) + (global_event_mask & (1<<(e))) /** If we're using cookie-type authentication, how long should our cookies be? */ @@ -95,25 +106,13 @@ static char authentication_cookie[AUTHENTICATION_COOKIE_LEN]; * of this so we can respond to getinfo status/bootstrap-phase queries. */ static char last_sent_bootstrap_message[BOOTSTRAP_MSG_LEN]; -/** Flag for event_format_t. Indicates that we should use the old - * name format of nickname|hexdigest - */ -#define SHORT_NAMES 1 -/** Flag for event_format_t. Indicates that we should use the new - * name format of $hexdigest[=~]nickname +/** Flag for event_format_t. Indicates that we should use the one standard + format. */ -#define LONG_NAMES 2 -#define ALL_NAMES (SHORT_NAMES|LONG_NAMES) -/** Flag for event_format_t. Indicates that we should use the new event - * format where extra event fields are allowed using a NAME=VAL format. */ -#define EXTENDED_FORMAT 4 -/** Flag for event_format_t. Indicates that we are using the old event format - * where extra fields aren't allowed. */ -#define NONEXTENDED_FORMAT 8 -#define ALL_FORMATS (EXTENDED_FORMAT|NONEXTENDED_FORMAT) +#define ALL_FORMATS 1 /** Bit field of flags to select how to format a controller event. Recognized - * flags are SHORT_NAMES, LONG_NAMES, EXTENDED_FORMAT, NONEXTENDED_FORMAT. */ + * flag is ALL_FORMATS. */ typedef int event_format_t; static void connection_printf_to_buf(control_connection_t *conn, @@ -123,9 +122,6 @@ static void send_control_done(control_connection_t *conn); static void send_control_event(uint16_t event, event_format_t which, const char *format, ...) CHECK_PRINTF(3,4); -static void send_control_event_extended(uint16_t event, event_format_t which, - const char *format, ...) - CHECK_PRINTF(3,4); static int handle_control_setconf(control_connection_t *conn, uint32_t len, char *body); static int handle_control_resetconf(control_connection_t *conn, uint32_t len, @@ -139,8 +135,6 @@ static int handle_control_setevents(control_connection_t *conn, uint32_t len, static int handle_control_authenticate(control_connection_t *conn, uint32_t len, const char *body); -static int handle_control_saveconf(control_connection_t *conn, uint32_t len, - const char *body); static int handle_control_signal(control_connection_t *conn, uint32_t len, const char *body); static int handle_control_mapaddress(control_connection_t *conn, uint32_t len, @@ -174,7 +168,7 @@ static int handle_control_usefeature(control_connection_t *conn, const char *body); static int write_stream_target_to_buf(edge_connection_t *conn, char *buf, size_t len); -static void orconn_target_get_name(int long_names, char *buf, size_t len, +static void orconn_target_get_name(char *buf, size_t len, or_connection_t *conn); static char *get_cookie_file(void); @@ -214,25 +208,19 @@ control_update_global_event_mask(void) { smartlist_t *conns = get_connection_array(); event_mask_t old_mask, new_mask; - old_mask = global_event_mask1short; - old_mask |= global_event_mask1long; + old_mask = global_event_mask; - global_event_mask1short = 0; - global_event_mask1long = 0; + global_event_mask = 0; SMARTLIST_FOREACH(conns, connection_t *, _conn, { if (_conn->type == CONN_TYPE_CONTROL && STATE_IS_OPEN(_conn->state)) { control_connection_t *conn = TO_CONTROL_CONN(_conn); - if (conn->use_long_names) - global_event_mask1long |= conn->event_mask; - else - global_event_mask1short |= conn->event_mask; + global_event_mask |= conn->event_mask; } }); - new_mask = global_event_mask1short; - new_mask |= global_event_mask1long; + new_mask = global_event_mask; /* Handle the aftermath. Set up the log callback to tell us only what * we want to hear...*/ @@ -312,7 +300,7 @@ connection_write_str_to_buf(const char *s, control_connection_t *conn) /** Given a <b>len</b>-character string in <b>data</b>, made of lines * terminated by CRLF, allocate a new string in *<b>out</b>, and copy the * contents of <b>data</b> into *<b>out</b>, adding a period before any period - * that that appears at the start of a line, and adding a period-CRLF line at + * that appears at the start of a line, and adding a period-CRLF line at * the end. Replace all LF characters sequences with CRLF. Return the number * of bytes in *<b>out</b>. */ @@ -542,28 +530,15 @@ send_control_event_string(uint16_t event, event_format_t which, const char *msg) { smartlist_t *conns = get_connection_array(); + (void)which; tor_assert(event >= _EVENT_MIN && event <= _EVENT_MAX); - SMARTLIST_FOREACH(conns, connection_t *, conn, - { + SMARTLIST_FOREACH_BEGIN(conns, connection_t *, conn) { if (conn->type == CONN_TYPE_CONTROL && !conn->marked_for_close && conn->state == CONTROL_CONN_STATE_OPEN) { control_connection_t *control_conn = TO_CONTROL_CONN(conn); - if (control_conn->use_long_names) { - if (!(which & LONG_NAMES)) - continue; - } else { - if (!(which & SHORT_NAMES)) - continue; - } - if (control_conn->use_extended_events) { - if (!(which & EXTENDED_FORMAT)) - continue; - } else { - if (!(which & NONEXTENDED_FORMAT)) - continue; - } + if (control_conn->event_mask & (1<<event)) { int is_err = 0; connection_write_to_buf(msg, strlen(msg), TO_CONN(control_conn)); @@ -579,7 +554,7 @@ send_control_event_string(uint16_t event, event_format_t which, connection_handle_write(TO_CONN(control_conn), 1); } } - }); + } SMARTLIST_FOREACH_END(conn); } /** Helper for send_control1_event and send_control1_event_extended: @@ -587,22 +562,17 @@ send_control_event_string(uint16_t event, event_format_t which, * <b>event</b>. The event's body is created by the printf-style format in * <b>format</b>, and other arguments as provided. * - * If <b>extended</b> is true, and the format contains a single '@' character, - * it will be replaced with a space and all text after that character will be - * sent only to controllers that have enabled extended events. - * * Currently the length of the message is limited to 1024 (including the * ending \\r\\n\\0). */ static void -send_control_event_impl(uint16_t event, event_format_t which, int extended, - const char *format, va_list ap) +send_control_event_impl(uint16_t event, event_format_t which, + const char *format, va_list ap) { /* This is just a little longer than the longest allowed log message */ #define SEND_CONTROL1_EVENT_BUFFERSIZE 10064 int r; char buf[SEND_CONTROL1_EVENT_BUFFERSIZE]; size_t len; - char *cp; r = tor_vsnprintf(buf, sizeof(buf), format, ap); if (r<0) { @@ -618,15 +588,7 @@ send_control_event_impl(uint16_t event, event_format_t which, int extended, buf[SEND_CONTROL1_EVENT_BUFFERSIZE-3] = '\r'; } - if (extended && (cp = strchr(buf, '@'))) { - which &= ~ALL_FORMATS; - *cp = ' '; - send_control_event_string(event, which|EXTENDED_FORMAT, buf); - memcpy(cp, "\r\n\0", 3); - send_control_event_string(event, which|NONEXTENDED_FORMAT, buf); - } else { - send_control_event_string(event, which|ALL_FORMATS, buf); - } + send_control_event_string(event, which|ALL_FORMATS, buf); } /** Send an event to all v1 controllers that are listening for code @@ -641,27 +603,7 @@ send_control_event(uint16_t event, event_format_t which, { va_list ap; va_start(ap, format); - send_control_event_impl(event, which, 0, format, ap); - va_end(ap); -} - -/** Send an event to all v1 controllers that are listening for code - * <b>event</b>. The event's body is created by the printf-style format in - * <b>format</b>, and other arguments as provided. - * - * If the format contains a single '@' character, it will be replaced with a - * space and all text after that character will be sent only to controllers - * that have enabled extended events. - * - * Currently the length of the message is limited to 1024 (including the - * ending \\n\\r\\0. */ -static void -send_control_event_extended(uint16_t event, event_format_t which, - const char *format, ...) -{ - va_list ap; - va_start(ap, format); - send_control_event_impl(event, which, 1, format, ap); + send_control_event_impl(event, which, format, ap); va_end(ap); } @@ -907,36 +849,37 @@ handle_control_loadconf(control_connection_t *conn, uint32_t len, retval = options_init_from_string(body, CMD_RUN_TOR, NULL, &errstring); - if (retval != SETOPT_OK) { + if (retval != SETOPT_OK) log_warn(LD_CONTROL, "Controller gave us config file that didn't validate: %s", errstring); - switch (retval) { - case SETOPT_ERR_PARSE: - msg = "552 Invalid config file"; - break; - case SETOPT_ERR_TRANSITION: - msg = "553 Transition not allowed"; - break; - case SETOPT_ERR_SETTING: - msg = "553 Unable to set option"; - break; - case SETOPT_ERR_MISC: - default: - msg = "550 Unable to load config"; - break; - case SETOPT_OK: - tor_fragile_assert(); - break; - } + + switch (retval) { + case SETOPT_ERR_PARSE: + msg = "552 Invalid config file"; + break; + case SETOPT_ERR_TRANSITION: + msg = "553 Transition not allowed"; + break; + case SETOPT_ERR_SETTING: + msg = "553 Unable to set option"; + break; + case SETOPT_ERR_MISC: + default: + msg = "550 Unable to load config"; + break; + case SETOPT_OK: + break; + } + if (msg) { if (errstring) connection_printf_to_buf(conn, "%s: %s\r\n", msg, errstring); else connection_printf_to_buf(conn, "%s\r\n", msg); - tor_free(errstring); - return 0; + } else { + send_control_done(conn); } - send_control_done(conn); + tor_free(errstring); return 0; } @@ -948,7 +891,6 @@ handle_control_setevents(control_connection_t *conn, uint32_t len, { uint16_t event_code; uint32_t event_mask = 0; - unsigned int extended = 0; smartlist_t *events = smartlist_create(); (void) len; @@ -958,7 +900,6 @@ handle_control_setevents(control_connection_t *conn, uint32_t len, SMARTLIST_FOREACH_BEGIN(events, const char *, ev) { if (!strcasecmp(ev, "EXTENDED")) { - extended = 1; continue; } else if (!strcasecmp(ev, "CIRC")) event_code = EVENT_CIRCUIT_STATUS; @@ -1002,6 +943,8 @@ handle_control_setevents(control_connection_t *conn, uint32_t len, event_code = EVENT_CLIENTS_SEEN; else if (!strcasecmp(ev, "NEWCONSENSUS")) event_code = EVENT_NEWCONSENSUS; + else if (!strcasecmp(ev, "BUILDTIMEOUT_SET")) + event_code = EVENT_BUILDTIMEOUT_SET; else { connection_printf_to_buf(conn, "552 Unrecognized event \"%s\"\r\n", ev); @@ -1016,8 +959,6 @@ handle_control_setevents(control_connection_t *conn, uint32_t len, smartlist_free(events); conn->event_mask = event_mask; - if (extended) - conn->use_extended_events = 1; control_update_global_event_mask(); send_control_done(conn); @@ -1328,7 +1269,7 @@ handle_control_mapaddress(control_connection_t *conn, uint32_t len, smartlist_add(reply, ans); log_warn(LD_CONTROL, "Unable to allocate address for '%s' in MapAddress msg", - safe_str(line)); + safe_str_client(line)); } else { tor_snprintf(ans, anslen, "250-%s=%s", address, to); smartlist_add(reply, ans); @@ -1345,7 +1286,8 @@ handle_control_mapaddress(control_connection_t *conn, uint32_t len, "not of expected form 'foo=bar'.", line); smartlist_add(reply, ans); log_info(LD_CONTROL, "Skipping MapAddress '%s': wrong " - "number of items.", safe_str(line)); + "number of items.", + safe_str_client(line)); } SMARTLIST_FOREACH(elts, char *, cp, tor_free(cp)); smartlist_clear(elts); @@ -1374,13 +1316,15 @@ handle_control_mapaddress(control_connection_t *conn, uint32_t len, * trivial-to-implement questions. */ static int getinfo_helper_misc(control_connection_t *conn, const char *question, - char **answer) + char **answer, const char **errmsg) { (void) conn; if (!strcmp(question, "version")) { *answer = tor_strdup(get_version()); } else if (!strcmp(question, "config-file")) { *answer = tor_strdup(get_torrc_fname()); + } else if (!strcmp(question, "config-text")) { + *answer = options_dump(get_options(), 1); } else if (!strcmp(question, "info/names")) { *answer = list_getinfo_options(); } else if (!strcmp(question, "events/names")) { @@ -1392,15 +1336,19 @@ getinfo_helper_misc(control_connection_t *conn, const char *question, *answer = tor_strdup("VERBOSE_NAMES EXTENDED_EVENTS"); } else if (!strcmp(question, "address")) { uint32_t addr; - if (router_pick_published_address(get_options(), &addr) < 0) + if (router_pick_published_address(get_options(), &addr) < 0) { + *errmsg = "Address unknown"; return -1; + } *answer = tor_dup_ip(addr); } else if (!strcmp(question, "dir-usage")) { *answer = directory_dump_request_log(); } else if (!strcmp(question, "fingerprint")) { routerinfo_t *me = router_get_my_routerinfo(); - if (!me) + if (!me) { + *errmsg = "No routerdesc known; am I really a server?"; return -1; + } *answer = tor_malloc(HEX_DIGEST_LEN+1); base16_encode(*answer, HEX_DIGEST_LEN+1, me->cache_info.identity_digest, DIGEST_LEN); @@ -1461,8 +1409,10 @@ munge_extrainfo_into_routerinfo(const char *ri_body, signed_descriptor_t *ri, * directory information. */ static int getinfo_helper_dir(control_connection_t *control_conn, - const char *question, char **answer) + const char *question, char **answer, + const char **errmsg) { + (void) control_conn; if (!strcmpstart(question, "desc/id/")) { routerinfo_t *ri = router_get_by_hexdigest(question+strlen("desc/id/")); if (ri) { @@ -1537,6 +1487,7 @@ getinfo_helper_dir(control_connection_t *control_conn, log_warn(LD_CONTROL, "getinfo '%s': %s", question, msg); smartlist_free(descs); tor_free(url); + *errmsg = msg; return -1; } SMARTLIST_FOREACH(descs, signed_descriptor_t *, sd, @@ -1587,7 +1538,7 @@ getinfo_helper_dir(control_connection_t *control_conn, } } else if (!strcmp(question, "dir/status-vote/current/consensus")) { /* v3 */ if (directory_caches_dir_info(get_options())) { - const cached_dir_t *consensus = dirserv_get_consensus(); + const cached_dir_t *consensus = dirserv_get_consensus("ns"); if (consensus) *answer = tor_strdup(consensus->dir); } @@ -1598,10 +1549,8 @@ getinfo_helper_dir(control_connection_t *control_conn, } } else if (!strcmp(question, "network-status")) { /* v1 */ routerlist_t *routerlist = router_get_routerlist(); - int verbose = control_conn->use_long_names; if (!routerlist || !routerlist->routers || - list_server_status_v1(routerlist->routers, answer, - verbose ? 2 : 1) < 0) { + list_server_status_v1(routerlist->routers, answer, 1) < 0) { return -1; } } else if (!strcmpstart(question, "extra-info/digest/")) { @@ -1635,8 +1584,10 @@ getinfo_helper_dir(control_connection_t *control_conn, * current states of things we send events about. */ static int getinfo_helper_events(control_connection_t *control_conn, - const char *question, char **answer) + const char *question, char **answer, + const char **errmsg) { + (void) control_conn; if (!strcmp(question, "circuit-status")) { circuit_t *circ; smartlist_t *status = smartlist_create(); @@ -1647,10 +1598,9 @@ getinfo_helper_events(control_connection_t *control_conn, const char *purpose; if (! CIRCUIT_IS_ORIGIN(circ) || circ->marked_for_close) continue; - if (control_conn->use_long_names) - path = circuit_list_path_for_controller(TO_ORIGIN_CIRCUIT(circ)); - else - path = circuit_list_path(TO_ORIGIN_CIRCUIT(circ),0); + + path = circuit_list_path_for_controller(TO_ORIGIN_CIRCUIT(circ)); + if (circ->state == CIRCUIT_STATE_OPEN) state = "BUILT"; else if (strlen(path)) @@ -1728,8 +1678,7 @@ getinfo_helper_events(control_connection_t *control_conn, } else if (!strcmp(question, "orconn-status")) { smartlist_t *conns = get_connection_array(); smartlist_t *status = smartlist_create(); - SMARTLIST_FOREACH(conns, connection_t *, base_conn, - { + SMARTLIST_FOREACH_BEGIN(conns, connection_t *, base_conn) { const char *state; char *s; char name[128]; @@ -1744,29 +1693,19 @@ getinfo_helper_events(control_connection_t *control_conn, state = "LAUNCHED"; else state = "NEW"; - orconn_target_get_name(control_conn->use_long_names, name, sizeof(name), - conn); + orconn_target_get_name(name, sizeof(name), conn); slen = strlen(name)+strlen(state)+2; s = tor_malloc(slen+1); tor_snprintf(s, slen, "%s %s", name, state); smartlist_add(status, s); - }); + } SMARTLIST_FOREACH_END(base_conn); *answer = smartlist_join_strings(status, "\r\n", 0, NULL); SMARTLIST_FOREACH(status, char *, cp, tor_free(cp)); smartlist_free(status); - } else if (!strcmpstart(question, "addr-mappings/") || - !strcmpstart(question, "address-mappings/")) { + } else if (!strcmpstart(question, "address-mappings/")) { time_t min_e, max_e; smartlist_t *mappings; - int want_expiry = !strcmpstart(question, "address-mappings/"); - if (!strcmpstart(question, "addr-mappings/")) { - /* XXXX022 This has been deprecated since 0.2.0.3-alpha, and has - generated a warning since 0.2.1.10-alpha; remove late in 0.2.2.x. */ - log_warn(LD_CONTROL, "Controller used obsolete addr-mappings/ GETINFO " - "key; use address-mappings/ instead."); - } - question += strlen(want_expiry ? "address-mappings/" - : "addr-mappings/"); + question += strlen("address-mappings/"); if (!strcmp(question, "all")) { min_e = 0; max_e = TIME_MAX; } else if (!strcmp(question, "cache")) { @@ -1779,7 +1718,7 @@ getinfo_helper_events(control_connection_t *control_conn, return 0; } mappings = smartlist_create(); - addressmap_get_mappings(mappings, min_e, max_e, want_expiry); + addressmap_get_mappings(mappings, min_e, max_e, 1); *answer = smartlist_join_strings(mappings, "\r\n", 0, NULL); SMARTLIST_FOREACH(mappings, char *, cp, tor_free(cp)); smartlist_free(mappings); @@ -1846,21 +1785,12 @@ getinfo_helper_events(control_connection_t *control_conn, "information", question); } } else if (!strcmp(question, "status/clients-seen")) { - char geoip_start[ISO_TIME_LEN+1]; - size_t answer_len; - char *geoip_summary = extrainfo_get_client_geoip_summary(time(NULL)); - - if (!geoip_summary) + const char *bridge_stats = geoip_get_bridge_stats_controller(time(NULL)); + if (!bridge_stats) { + *errmsg = "No bridge-client stats available"; return -1; - - answer_len = strlen("TimeStarted=\"\" CountrySummary=") + - ISO_TIME_LEN + strlen(geoip_summary) + 1; - *answer = tor_malloc(answer_len); - format_iso_time(geoip_start, geoip_get_history_start()); - tor_snprintf(*answer, answer_len, - "TimeStarted=\"%s\" CountrySummary=%s", - geoip_start, geoip_summary); - tor_free(geoip_summary); + } + *answer = tor_strdup(bridge_stats); } else { return 0; } @@ -1870,11 +1800,14 @@ getinfo_helper_events(control_connection_t *control_conn, /** Callback function for GETINFO: on a given control connection, try to * answer the question <b>q</b> and store the newly-allocated answer in - * *<b>a</b>. If there's no answer, or an error occurs, just don't set - * <b>a</b>. Return 0. + * *<b>a</b>. If an internal error occurs, return -1 and optionally set + * *<b>error_out</b> to point to an error message to be delivered to the + * controller. On success, _or if the key is not recognized_, return 0. Do not + * set <b>a</b> if the key is not recognized. */ typedef int (*getinfo_helper_t)(control_connection_t *, - const char *q, char **a); + const char *q, char **a, + const char **error_out); /** A single item for the GETINFO question-to-answer-function table. */ typedef struct getinfo_item_t { @@ -1894,6 +1827,8 @@ typedef struct getinfo_item_t { static const getinfo_item_t getinfo_items[] = { ITEM("version", misc, "The current version of Tor."), ITEM("config-file", misc, "Current location of the \"torrc\" file."), + ITEM("config-text", misc, + "Return the string that would be written by a saveconf command."), ITEM("accounting/bytes", accounting, "Number of bytes read/written so far in the accounting interval."), ITEM("accounting/bytes-left", accounting, @@ -1933,7 +1868,6 @@ static const getinfo_item_t getinfo_items[] = { PREFIX("ns/purpose/", networkstatus, "Brief summary of router status by purpose (v2 directory format)."), - PREFIX("unregistered-servers-", dirserv_unregistered, NULL), ITEM("network-status", dir, "Brief summary of router status (v1 directory format)"), ITEM("circuit-status", events, "List of current circuits originating here."), @@ -1945,14 +1879,6 @@ static const getinfo_item_t getinfo_items[] = { DOC("address-mappings/config", "Current address mappings from configuration."), DOC("address-mappings/control", "Current address mappings from controller."), - PREFIX("addr-mappings/", events, NULL), - DOC("addr-mappings/all", "Current address mappings without expiry times."), - DOC("addr-mappings/cache", - "Current cached DNS replies without expiry times."), - DOC("addr-mappings/config", - "Current address mappings from configuration without expiry times."), - DOC("addr-mappings/control", - "Current address mappings from controller without expiry times."), PREFIX("status/", events, NULL), DOC("status/circuit-established", "Whether we think client functionality is working."), @@ -1988,18 +1914,18 @@ static char * list_getinfo_options(void) { int i; - char buf[300]; + char *buf=NULL; smartlist_t *lines = smartlist_create(); char *ans; for (i = 0; getinfo_items[i].varname; ++i) { if (!getinfo_items[i].desc) continue; - tor_snprintf(buf, sizeof(buf), "%s%s -- %s\n", + tor_asprintf(&buf, "%s%s -- %s\n", getinfo_items[i].varname, getinfo_items[i].is_prefix ? "*" : "", getinfo_items[i].desc); - smartlist_add(lines, tor_strdup(buf)); + smartlist_add(lines, buf); } smartlist_sort_strings(lines); @@ -2016,7 +1942,8 @@ list_getinfo_options(void) * internal error. */ static int handle_getinfo_helper(control_connection_t *control_conn, - const char *question, char **answer) + const char *question, char **answer, + const char **err_out) { int i; *answer = NULL; /* unrecognized key by default */ @@ -2029,7 +1956,7 @@ handle_getinfo_helper(control_connection_t *control_conn, match = !strcmp(question, getinfo_items[i].varname); if (match) { tor_assert(getinfo_items[i].fn); - return getinfo_items[i].fn(control_conn, question, answer); + return getinfo_items[i].fn(control_conn, question, answer, err_out); } } @@ -2051,10 +1978,12 @@ handle_control_getinfo(control_connection_t *conn, uint32_t len, smartlist_split_string(questions, body, " ", SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); - SMARTLIST_FOREACH(questions, const char *, q, - { - if (handle_getinfo_helper(conn, q, &ans) < 0) { - connection_write_str_to_buf("551 Internal error\r\n", conn); + SMARTLIST_FOREACH_BEGIN(questions, const char *, q) { + const char *errmsg = NULL; + if (handle_getinfo_helper(conn, q, &ans, &errmsg) < 0) { + if (!errmsg) + errmsg = "Internal error"; + connection_printf_to_buf(conn, "551 %s\r\n", errmsg); goto done; } if (!ans) { @@ -2063,7 +1992,7 @@ handle_control_getinfo(control_connection_t *conn, uint32_t len, smartlist_add(answers, tor_strdup(q)); smartlist_add(answers, ans); } - }); + } SMARTLIST_FOREACH_END(q); if (smartlist_len(unrecognized)) { for (i=0; i < smartlist_len(unrecognized)-1; ++i) connection_printf_to_buf(conn, @@ -2108,12 +2037,12 @@ handle_control_getinfo(control_connection_t *conn, uint32_t len, static uint8_t circuit_purpose_from_string(const char *string) { - if (!strcmpstart(string, "purpose=")) + if (!strcasecmpstart(string, "purpose=")) string += strlen("purpose="); - if (!strcmp(string, "general")) + if (!strcasecmp(string, "general")) return CIRCUIT_PURPOSE_C_GENERAL; - else if (!strcmp(string, "controller")) + else if (!strcasecmp(string, "controller")) return CIRCUIT_PURPOSE_CONTROLLER; else return CIRCUIT_PURPOSE_UNKNOWN; @@ -2145,6 +2074,31 @@ getargs_helper(const char *command, control_connection_t *conn, return NULL; } +/** Helper. Return the first element of <b>sl</b> at index <b>start_at</b> or + * higher that starts with <b>prefix</b>, case-insensitive. Return NULL if no + * such element exists. */ +static const char * +find_element_starting_with(smartlist_t *sl, int start_at, const char *prefix) +{ + int i; + for (i = start_at; i < smartlist_len(sl); ++i) { + const char *elt = smartlist_get(sl, i); + if (!strcasecmpstart(elt, prefix)) + return elt; + } + return NULL; +} + +/** Helper. Return true iff s is an argument that we should treat as a + * key-value pair. */ +static int +is_keyval_pair(const char *s) +{ + /* An argument is a key-value pair if it has an =, and it isn't of the form + * $fingeprint=name */ + return strchr(s, '=') && s[0] != '$'; +} + /** Called when we get an EXTENDCIRCUIT message. Try to extend the listed * circuit, and report success or failure. */ static int @@ -2160,33 +2114,57 @@ handle_control_extendcircuit(control_connection_t *conn, uint32_t len, router_nicknames = smartlist_create(); - args = getargs_helper("EXTENDCIRCUIT", conn, body, 2, -1); + args = getargs_helper("EXTENDCIRCUIT", conn, body, 1, -1); if (!args) goto done; zero_circ = !strcmp("0", (char*)smartlist_get(args,0)); - if (!zero_circ && !(circ = get_circ(smartlist_get(args,0)))) { - connection_printf_to_buf(conn, "552 Unknown circuit \"%s\"\r\n", - (char*)smartlist_get(args, 0)); - } - smartlist_split_string(router_nicknames, smartlist_get(args,1), ",", 0, 0); - if (zero_circ && smartlist_len(args)>2) { - char *purp = smartlist_get(args,2); - intended_purpose = circuit_purpose_from_string(purp); - if (intended_purpose == CIRCUIT_PURPOSE_UNKNOWN) { - connection_printf_to_buf(conn, "552 Unknown purpose \"%s\"\r\n", purp); + if (zero_circ) { + const char *purp = find_element_starting_with(args, 1, "PURPOSE="); + + if (purp) { + intended_purpose = circuit_purpose_from_string(purp); + if (intended_purpose == CIRCUIT_PURPOSE_UNKNOWN) { + connection_printf_to_buf(conn, "552 Unknown purpose \"%s\"\r\n", purp); + SMARTLIST_FOREACH(args, char *, cp, tor_free(cp)); + smartlist_free(args); + goto done; + } + } + + if ((smartlist_len(args) == 1) || + (smartlist_len(args) >= 2 && is_keyval_pair(smartlist_get(args, 1)))) { + // "EXTENDCIRCUIT 0" || EXTENDCIRCUIT 0 foo=bar" + circ = circuit_launch_by_router(intended_purpose, NULL, + CIRCLAUNCH_NEED_CAPACITY); + if (!circ) { + connection_write_str_to_buf("551 Couldn't start circuit\r\n", conn); + } else { + connection_printf_to_buf(conn, "250 EXTENDED %lu\r\n", + (unsigned long)circ->global_identifier); + } SMARTLIST_FOREACH(args, char *, cp, tor_free(cp)); smartlist_free(args); goto done; } + // "EXTENDCIRCUIT 0 router1,router2" || + // "EXTENDCIRCUIT 0 router1,router2 PURPOSE=foo" } - SMARTLIST_FOREACH(args, char *, cp, tor_free(cp)); - smartlist_free(args); - if (!zero_circ && !circ) { + + if (!zero_circ && !(circ = get_circ(smartlist_get(args,0)))) { + connection_printf_to_buf(conn, "552 Unknown circuit \"%s\"\r\n", + (char*)smartlist_get(args, 0)); + SMARTLIST_FOREACH(args, char *, cp, tor_free(cp)); + smartlist_free(args); goto done; } + smartlist_split_string(router_nicknames, smartlist_get(args,1), ",", 0, 0); + + SMARTLIST_FOREACH(args, char *, cp, tor_free(cp)); + smartlist_free(args); + routers = smartlist_create(); SMARTLIST_FOREACH(router_nicknames, const char *, n, { @@ -2244,8 +2222,7 @@ handle_control_extendcircuit(control_connection_t *conn, uint32_t len, done: SMARTLIST_FOREACH(router_nicknames, char *, n, tor_free(n)); smartlist_free(router_nicknames); - if (routers) - smartlist_free(routers); + smartlist_free(routers); return 0; } @@ -2271,7 +2248,7 @@ handle_control_setcircuitpurpose(control_connection_t *conn, } { - char *purp = smartlist_get(args,1); + const char *purp = find_element_starting_with(args,1,"PURPOSE="); new_purpose = circuit_purpose_from_string(purp); if (new_purpose == CIRCUIT_PURPOSE_UNKNOWN) { connection_printf_to_buf(conn, "552 Unknown purpose \"%s\"\r\n", purp); @@ -2316,9 +2293,9 @@ handle_control_attachstream(control_connection_t *conn, uint32_t len, } else if (!zero_circ && !(circ = get_circ(smartlist_get(args, 1)))) { connection_printf_to_buf(conn, "552 Unknown circuit \"%s\"\r\n", (char*)smartlist_get(args, 1)); - } else if (circ && smartlist_len(args) > 2) { - char *hopstring = smartlist_get(args, 2); - if (!strcasecmpstart(hopstring, "HOP=")) { + } else if (circ) { + const char *hopstring = find_element_starting_with(args,2,"HOP="); + if (hopstring) { hopstring += strlen("HOP="); hop = (int) tor_parse_ulong(hopstring, 10, 0, INT_MAX, &hop_line_ok, NULL); @@ -2365,7 +2342,7 @@ handle_control_attachstream(control_connection_t *conn, uint32_t len, char* exit_digest; if (circ->build_state && circ->build_state->chosen_exit && - circ->build_state->chosen_exit->identity_digest) { + !tor_digest_is_zero(circ->build_state->chosen_exit->identity_digest)) { exit_digest = circ->build_state->chosen_exit->identity_digest; r = router_get_by_digest(exit_digest); } @@ -2426,9 +2403,9 @@ handle_control_postdescriptor(control_connection_t *conn, uint32_t len, } } else if (!strcasecmpstart(option, "cache=")) { option += strlen("cache="); - if (!strcmp(option, "no")) + if (!strcasecmp(option, "no")) cache = 0; - else if (!strcmp(option, "yes")) + else if (!strcasecmp(option, "yes")) cache = 1; else { connection_printf_to_buf(conn, "552 Unknown cache request \"%s\"\r\n", @@ -2610,17 +2587,17 @@ handle_control_resolve(control_connection_t *conn, uint32_t len, args = smartlist_create(); smartlist_split_string(args, body, " ", SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); - if (smartlist_len(args) && - !strcasecmp(smartlist_get(args, 0), "mode=reverse")) { - char *cp = smartlist_get(args, 0); - smartlist_del_keeporder(args, 0); - tor_free(cp); - is_reverse = 1; + { + const char *modearg = find_element_starting_with(args, 0, "mode="); + if (modearg && !strcasecmp(modearg, "mode=reverse")) + is_reverse = 1; } failed = smartlist_create(); SMARTLIST_FOREACH(args, const char *, arg, { - if (dnsserv_launch_request(arg, is_reverse)<0) - smartlist_add(failed, (char*)arg); + if (!is_keyval_pair(arg)) { + if (dnsserv_launch_request(arg, is_reverse)<0) + smartlist_add(failed, (char*)arg); + } }); send_control_done(conn); @@ -2710,7 +2687,6 @@ handle_control_usefeature(control_connection_t *conn, const char *body) { smartlist_t *args; - int verbose_names = 0, extended_events = 0; int bad = 0; (void) len; /* body is nul-terminated; it's safe to ignore the length */ args = smartlist_create(); @@ -2718,9 +2694,9 @@ handle_control_usefeature(control_connection_t *conn, SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); SMARTLIST_FOREACH(args, const char *, arg, { if (!strcasecmp(arg, "VERBOSE_NAMES")) - verbose_names = 1; + ; else if (!strcasecmp(arg, "EXTENDED_EVENTS")) - extended_events = 1; + ; else { connection_printf_to_buf(conn, "552 Unrecognized feature \"%s\"\r\n", arg); @@ -2730,12 +2706,6 @@ handle_control_usefeature(control_connection_t *conn, }); if (!bad) { - if (verbose_names) { - conn->use_long_names = 1; - control_update_global_event_mask(); - } - if (extended_events) - conn->use_extended_events = 1; send_control_done(conn); } @@ -3039,20 +3009,11 @@ control_event_circuit_status(origin_circuit_t *circ, circuit_status_event_t tp, tor_free(reason); } - if (EVENT_IS_INTERESTING1S(EVENT_CIRCUIT_STATUS)) { - char *path = circuit_list_path(circ,0); - const char *sp = strlen(path) ? " " : ""; - send_control_event_extended(EVENT_CIRCUIT_STATUS, SHORT_NAMES, - "650 CIRC %lu %s%s%s@%s\r\n", - (unsigned long)circ->global_identifier, - status, sp, path, extended_buf); - tor_free(path); - } - if (EVENT_IS_INTERESTING1L(EVENT_CIRCUIT_STATUS)) { + { char *vpath = circuit_list_path_for_controller(circ); const char *sp = strlen(vpath) ? " " : ""; - send_control_event_extended(EVENT_CIRCUIT_STATUS, LONG_NAMES, - "650 CIRC %lu %s%s%s@%s\r\n", + send_control_event(EVENT_CIRCUIT_STATUS, ALL_FORMATS, + "650 CIRC %lu %s%s%s %s\r\n", (unsigned long)circ->global_identifier, status, sp, vpath, extended_buf); tor_free(vpath); @@ -3131,26 +3092,26 @@ control_event_stream_status(edge_connection_t *conn, stream_status_event_t tp, char *r = NULL; if (!reason_str) { r = tor_malloc(16); - tor_snprintf(r, 16, "UNKNOWN_%d", reason_code); + tor_snprintf(r, 16, " UNKNOWN_%d", reason_code); reason_str = r; } if (reason_code & END_STREAM_REASON_FLAG_REMOTE) tor_snprintf(reason_buf, sizeof(reason_buf), - "REASON=END REMOTE_REASON=%s", reason_str); + " REASON=END REMOTE_REASON=%s", reason_str); else tor_snprintf(reason_buf, sizeof(reason_buf), - "REASON=%s", reason_str); + " REASON=%s", reason_str); tor_free(r); } else if (reason_code && tp == STREAM_EVENT_REMAP) { switch (reason_code) { case REMAP_STREAM_SOURCE_CACHE: - strlcpy(reason_buf, "SOURCE=CACHE", sizeof(reason_buf)); + strlcpy(reason_buf, " SOURCE=CACHE", sizeof(reason_buf)); break; case REMAP_STREAM_SOURCE_EXIT: - strlcpy(reason_buf, "SOURCE=EXIT", sizeof(reason_buf)); + strlcpy(reason_buf, " SOURCE=EXIT", sizeof(reason_buf)); break; default: - tor_snprintf(reason_buf, sizeof(reason_buf), "REASON=UNKNOWN_%d", + tor_snprintf(reason_buf, sizeof(reason_buf), " REASON=UNKNOWN_%d", reason_code); /* XXX do we want SOURCE=UNKNOWN_%d above instead? -RD */ break; @@ -3158,8 +3119,7 @@ control_event_stream_status(edge_connection_t *conn, stream_status_event_t tp, } if (tp == STREAM_EVENT_NEW) { - tor_snprintf(addrport_buf,sizeof(addrport_buf), "%sSOURCE_ADDR=%s:%d", - strlen(reason_buf) ? " " : "", + tor_snprintf(addrport_buf,sizeof(addrport_buf), " SOURCE_ADDR=%s:%d", TO_CONN(conn)->address, TO_CONN(conn)->port ); } else { addrport_buf[0] = '\0'; @@ -3188,8 +3148,8 @@ control_event_stream_status(edge_connection_t *conn, stream_status_event_t tp, circ = circuit_get_by_edge_conn(conn); if (circ && CIRCUIT_IS_ORIGIN(circ)) origin_circ = TO_ORIGIN_CIRCUIT(circ); - send_control_event_extended(EVENT_STREAM_STATUS, ALL_NAMES, - "650 STREAM "U64_FORMAT" %s %lu %s@%s%s%s\r\n", + send_control_event(EVENT_STREAM_STATUS, ALL_FORMATS, + "650 STREAM "U64_FORMAT" %s %lu %s%s%s%s\r\n", U64_PRINTF_ARG(conn->_base.global_identifier), status, origin_circ? (unsigned long)origin_circ->global_identifier : 0ul, @@ -3202,30 +3162,21 @@ control_event_stream_status(edge_connection_t *conn, stream_status_event_t tp, /** Figure out the best name for the target router of an OR connection * <b>conn</b>, and write it into the <b>len</b>-character buffer - * <b>name</b>. Use verbose names if <b>long_names</b> is set. */ + * <b>name</b>. */ static void -orconn_target_get_name(int long_names, - char *name, size_t len, or_connection_t *conn) -{ - if (! long_names) { - if (conn->nickname) - strlcpy(name, conn->nickname, len); - else - tor_snprintf(name, len, "%s:%d", - conn->_base.address, conn->_base.port); +orconn_target_get_name(char *name, size_t len, or_connection_t *conn) +{ + routerinfo_t *ri = router_get_by_digest(conn->identity_digest); + if (ri) { + tor_assert(len > MAX_VERBOSE_NICKNAME_LEN); + router_get_verbose_nickname(name, ri); + } else if (! tor_digest_is_zero(conn->identity_digest)) { + name[0] = '$'; + base16_encode(name+1, len-1, conn->identity_digest, + DIGEST_LEN); } else { - routerinfo_t *ri = router_get_by_digest(conn->identity_digest); - if (ri) { - tor_assert(len > MAX_VERBOSE_NICKNAME_LEN); - router_get_verbose_nickname(name, ri); - } else if (! tor_digest_is_zero(conn->identity_digest)) { - name[0] = '$'; - base16_encode(name+1, len-1, conn->identity_digest, - DIGEST_LEN); - } else { - tor_snprintf(name, len, "%s:%d", - conn->_base.address, conn->_base.port); - } + tor_snprintf(name, len, "%s:%d", + conn->_base.address, conn->_base.port); } } @@ -3264,24 +3215,13 @@ control_event_or_conn_status(or_connection_t *conn, or_conn_status_event_t tp, reason ? " " : "", ncircs); } - if (EVENT_IS_INTERESTING1S(EVENT_OR_CONN_STATUS)) { - orconn_target_get_name(0, name, sizeof(name), conn); - send_control_event_extended(EVENT_OR_CONN_STATUS, SHORT_NAMES, - "650 ORCONN %s %s@%s%s%s\r\n", - name, status, - reason ? "REASON=" : "", - orconn_end_reason_to_control_string(reason), - ncircs_buf); - } - if (EVENT_IS_INTERESTING1L(EVENT_OR_CONN_STATUS)) { - orconn_target_get_name(1, name, sizeof(name), conn); - send_control_event_extended(EVENT_OR_CONN_STATUS, LONG_NAMES, - "650 ORCONN %s %s@%s%s%s\r\n", - name, status, - reason ? "REASON=" : "", - orconn_end_reason_to_control_string(reason), - ncircs_buf); - } + orconn_target_get_name(name, sizeof(name), conn); + send_control_event(EVENT_OR_CONN_STATUS, ALL_FORMATS, + "650 ORCONN %s %s %s%s%s\r\n", + name, status, + reason ? "REASON=" : "", + orconn_end_reason_to_control_string(reason), + ncircs_buf); return 0; } @@ -3296,7 +3236,7 @@ control_event_stream_bandwidth(edge_connection_t *edge_conn) if (!edge_conn->n_read && !edge_conn->n_written) return 0; - send_control_event(EVENT_STREAM_BANDWIDTH_USED, ALL_NAMES, + send_control_event(EVENT_STREAM_BANDWIDTH_USED, ALL_FORMATS, "650 STREAM_BW "U64_FORMAT" %lu %lu\r\n", U64_PRINTF_ARG(edge_conn->_base.global_identifier), (unsigned long)edge_conn->n_read, @@ -3325,7 +3265,7 @@ control_event_stream_bandwidth_used(void) if (!edge_conn->n_read && !edge_conn->n_written) continue; - send_control_event(EVENT_STREAM_BANDWIDTH_USED, ALL_NAMES, + send_control_event(EVENT_STREAM_BANDWIDTH_USED, ALL_FORMATS, "650 STREAM_BW "U64_FORMAT" %lu %lu\r\n", U64_PRINTF_ARG(edge_conn->_base.global_identifier), (unsigned long)edge_conn->n_read, @@ -3345,7 +3285,7 @@ int control_event_bandwidth_used(uint32_t n_read, uint32_t n_written) { if (EVENT_IS_INTERESTING(EVENT_BANDWIDTH_USED)) { - send_control_event(EVENT_BANDWIDTH_USED, ALL_NAMES, + send_control_event(EVENT_BANDWIDTH_USED, ALL_FORMATS, "650 BW %lu %lu\r\n", (unsigned long)n_read, (unsigned long)n_written); @@ -3415,7 +3355,7 @@ control_event_logmsg(int severity, uint32_t domain, const char *msg) default: s = "UnknownLogSeverity"; break; } ++disable_log_messages; - send_control_event(event, ALL_NAMES, "650 %s %s\r\n", s, b?b:msg); + send_control_event(event, ALL_FORMATS, "650 %s %s\r\n", s, b?b:msg); --disable_log_messages; tor_free(b); } @@ -3428,31 +3368,12 @@ control_event_logmsg(int severity, uint32_t domain, const char *msg) int control_event_descriptors_changed(smartlist_t *routers) { - size_t len; char *msg; - smartlist_t *identities = NULL; - char buf[HEX_DIGEST_LEN+1]; if (!EVENT_IS_INTERESTING(EVENT_NEW_DESC)) return 0; - if (EVENT_IS_INTERESTING1S(EVENT_NEW_DESC)) { - identities = smartlist_create(); - SMARTLIST_FOREACH(routers, routerinfo_t *, r, - { - base16_encode(buf,sizeof(buf),r->cache_info.identity_digest,DIGEST_LEN); - smartlist_add(identities, tor_strdup(buf)); - }); - } - if (EVENT_IS_INTERESTING1S(EVENT_NEW_DESC)) { - char *ids = smartlist_join_strings(identities, " ", 0, &len); - size_t ids_len = strlen(ids)+32; - msg = tor_malloc(ids_len); - tor_snprintf(msg, ids_len, "650 NEWDESC %s\r\n", ids); - send_control_event_string(EVENT_NEW_DESC, SHORT_NAMES|ALL_FORMATS, msg); - tor_free(ids); - tor_free(msg); - } - if (EVENT_IS_INTERESTING1L(EVENT_NEW_DESC)) { + + { smartlist_t *names = smartlist_create(); char *ids; size_t names_len; @@ -3465,16 +3386,12 @@ control_event_descriptors_changed(smartlist_t *routers) names_len = strlen(ids)+32; msg = tor_malloc(names_len); tor_snprintf(msg, names_len, "650 NEWDESC %s\r\n", ids); - send_control_event_string(EVENT_NEW_DESC, LONG_NAMES|ALL_FORMATS, msg); + send_control_event_string(EVENT_NEW_DESC, ALL_FORMATS, msg); tor_free(ids); tor_free(msg); SMARTLIST_FOREACH(names, char *, cp, tor_free(cp)); smartlist_free(names); } - if (identities) { - SMARTLIST_FOREACH(identities, char *, cp, tor_free(cp)); - smartlist_free(identities); - } return 0; } @@ -3491,17 +3408,17 @@ control_event_address_mapped(const char *from, const char *to, time_t expires, return 0; if (expires < 3 || expires == TIME_MAX) - send_control_event_extended(EVENT_ADDRMAP, ALL_NAMES, - "650 ADDRMAP %s %s NEVER@%s\r\n", from, to, + send_control_event(EVENT_ADDRMAP, ALL_FORMATS, + "650 ADDRMAP %s %s NEVER %s\r\n", from, to, error?error:""); else { char buf[ISO_TIME_LEN+1]; char buf2[ISO_TIME_LEN+1]; format_local_iso_time(buf,expires); format_iso_time(buf2,expires); - send_control_event_extended(EVENT_ADDRMAP, ALL_NAMES, + send_control_event(EVENT_ADDRMAP, ALL_FORMATS, "650 ADDRMAP %s %s \"%s\"" - "@%s%sEXPIRES=\"%s\"\r\n", + " %s%sEXPIRES=\"%s\"\r\n", from, to, buf, error?error:"", error?" ":"", buf2); @@ -3541,9 +3458,9 @@ control_event_or_authdir_new_descriptor(const char *action, buf = tor_malloc(totallen); strlcpy(buf, firstline, totallen); strlcpy(buf+strlen(firstline), esc, totallen); - send_control_event_string(EVENT_AUTHDIR_NEWDESCS, ALL_NAMES|ALL_FORMATS, + send_control_event_string(EVENT_AUTHDIR_NEWDESCS, ALL_FORMATS, buf); - send_control_event_string(EVENT_AUTHDIR_NEWDESCS, ALL_NAMES|ALL_FORMATS, + send_control_event_string(EVENT_AUTHDIR_NEWDESCS, ALL_FORMATS, "650 OK\r\n"); tor_free(esc); tor_free(buf); @@ -3581,8 +3498,8 @@ control_event_networkstatus_changed_helper(smartlist_t *statuses, SMARTLIST_FOREACH(strs, char *, cp, tor_free(cp)); smartlist_free(strs); tor_free(s); - send_control_event_string(event, ALL_NAMES|ALL_FORMATS, esc); - send_control_event_string(event, ALL_NAMES|ALL_FORMATS, + send_control_event_string(event, ALL_FORMATS, esc); + send_control_event_string(event, ALL_FORMATS, "650 OK\r\n"); tor_free(esc); @@ -3608,6 +3525,55 @@ control_event_newconsensus(const networkstatus_t *consensus) consensus->routerstatus_list, EVENT_NEWCONSENSUS, "NEWCONSENSUS"); } +/** Called when we compute a new circuitbuildtimeout */ +int +control_event_buildtimeout_set(const circuit_build_times_t *cbt, + buildtimeout_set_event_t type) +{ + const char *type_string = NULL; + double qnt = circuit_build_times_quantile_cutoff(); + + if (!control_event_is_interesting(EVENT_BUILDTIMEOUT_SET)) + return 0; + + switch (type) { + case BUILDTIMEOUT_SET_EVENT_COMPUTED: + type_string = "COMPUTED"; + break; + case BUILDTIMEOUT_SET_EVENT_RESET: + type_string = "RESET"; + qnt = 1.0; + break; + case BUILDTIMEOUT_SET_EVENT_SUSPENDED: + type_string = "SUSPENDED"; + qnt = 1.0; + break; + case BUILDTIMEOUT_SET_EVENT_DISCARD: + type_string = "DISCARD"; + qnt = 1.0; + break; + case BUILDTIMEOUT_SET_EVENT_RESUME: + type_string = "RESUME"; + break; + default: + type_string = "UNKNOWN"; + break; + } + + send_control_event(EVENT_BUILDTIMEOUT_SET, ALL_FORMATS, + "650 BUILDTIMEOUT_SET %s TOTAL_TIMES=%lu " + "TIMEOUT_MS=%lu XM=%lu ALPHA=%lf CUTOFF_QUANTILE=%lf " + "TIMEOUT_RATE=%lf CLOSE_MS=%lu CLOSE_RATE=%lf\r\n", + type_string, (unsigned long)cbt->total_build_times, + (unsigned long)cbt->timeout_ms, + (unsigned long)cbt->Xm, cbt->alpha, qnt, + circuit_build_times_timeout_rate(cbt), + (unsigned long)cbt->close_ms, + circuit_build_times_close_rate(cbt)); + + return 0; +} + /** Called when a single local_routerstatus_t has changed: Sends an NS event * to any controller that cares. */ int @@ -3631,7 +3597,7 @@ control_event_networkstatus_changed_single(routerstatus_t *rs) int control_event_my_descriptor_changed(void) { - send_control_event(EVENT_DESCCHANGED, ALL_NAMES, "650 DESCCHANGED\r\n"); + send_control_event(EVENT_DESCCHANGED, ALL_FORMATS, "650 DESCCHANGED\r\n"); return 0; } @@ -3679,7 +3645,7 @@ control_event_status(int type, int severity, const char *format, va_list args) return -1; } - send_control_event_impl(type, ALL_NAMES|ALL_FORMATS, 0, format_buf, args); + send_control_event_impl(type, ALL_FORMATS, format_buf, args); return 0; } @@ -3743,7 +3709,7 @@ control_event_guard(const char *nickname, const char *digest, if (!EVENT_IS_INTERESTING(EVENT_GUARD)) return 0; - if (EVENT_IS_INTERESTING1L(EVENT_GUARD)) { + { char buf[MAX_VERBOSE_NICKNAME_LEN+1]; routerinfo_t *ri = router_get_by_digest(digest); if (ri) { @@ -3751,13 +3717,9 @@ control_event_guard(const char *nickname, const char *digest, } else { tor_snprintf(buf, sizeof(buf), "$%s~%s", hbuf, nickname); } - send_control_event(EVENT_GUARD, LONG_NAMES, + send_control_event(EVENT_GUARD, ALL_FORMATS, "650 GUARD ENTRY %s %s\r\n", buf, status); } - if (EVENT_IS_INTERESTING1S(EVENT_GUARD)) { - send_control_event(EVENT_GUARD, SHORT_NAMES, - "650 GUARD ENTRY $%s %s\r\n", hbuf, status); - } return 0; } @@ -4008,10 +3970,9 @@ control_event_bootstrap_problem(const char *warn, int reason) * from recently. Send a copy to the controller in case it wants to * display it for the user. */ void -control_event_clients_seen(const char *timestarted, const char *countries) +control_event_clients_seen(const char *controller_str) { send_control_event(EVENT_CLIENTS_SEEN, 0, - "650 CLIENTS_SEEN TimeStarted=\"%s\" CountrySummary=%s\r\n", - timestarted, countries); + "650 CLIENTS_SEEN %s\r\n", controller_str); } diff --git a/src/or/control.h b/src/or/control.h new file mode 100644 index 0000000000..27ef5c37f7 --- /dev/null +++ b/src/or/control.h @@ -0,0 +1,85 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2010, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file control.h + * \brief Header file for control.c. + **/ + +#ifndef _TOR_CONTROL_H +#define _TOR_CONTROL_H + +void control_update_global_event_mask(void); +void control_adjust_event_log_severity(void); + +/** Log information about the connection <b>conn</b>, protecting it as with + * CONN_LOG_PROTECT. Example: + * + * LOG_FN_CONN(conn, (LOG_DEBUG, "Socket %d wants to write", conn->s)); + **/ +#define LOG_FN_CONN(conn, args) \ + CONN_LOG_PROTECT(conn, log_fn args) + +int connection_control_finished_flushing(control_connection_t *conn); +int connection_control_reached_eof(control_connection_t *conn); +int connection_control_process_inbuf(control_connection_t *conn); + +#define EVENT_AUTHDIR_NEWDESCS 0x000D +#define EVENT_NS 0x000F +int control_event_is_interesting(int event); + +int control_event_circuit_status(origin_circuit_t *circ, + circuit_status_event_t e, int reason); +int control_event_stream_status(edge_connection_t *conn, + stream_status_event_t e, + int reason); +int control_event_or_conn_status(or_connection_t *conn, + or_conn_status_event_t e, int reason); +int control_event_bandwidth_used(uint32_t n_read, uint32_t n_written); +int control_event_stream_bandwidth(edge_connection_t *edge_conn); +int control_event_stream_bandwidth_used(void); +void control_event_logmsg(int severity, unsigned int domain, const char *msg); +int control_event_descriptors_changed(smartlist_t *routers); +int control_event_address_mapped(const char *from, const char *to, + time_t expires, const char *error); +int control_event_or_authdir_new_descriptor(const char *action, + const char *desc, + size_t desclen, + const char *msg); +int control_event_my_descriptor_changed(void); +int control_event_networkstatus_changed(smartlist_t *statuses); + +int control_event_newconsensus(const networkstatus_t *consensus); +int control_event_networkstatus_changed_single(routerstatus_t *rs); +int control_event_general_status(int severity, const char *format, ...) + CHECK_PRINTF(2,3); +int control_event_client_status(int severity, const char *format, ...) + CHECK_PRINTF(2,3); +int control_event_server_status(int severity, const char *format, ...) + CHECK_PRINTF(2,3); +int control_event_guard(const char *nickname, const char *digest, + const char *status); +int control_event_buildtimeout_set(const circuit_build_times_t *cbt, + buildtimeout_set_event_t type); + +int init_cookie_authentication(int enabled); +smartlist_t *decode_hashed_passwords(config_line_t *passwords); +void disable_control_logging(void); +void enable_control_logging(void); + +void control_event_bootstrap(bootstrap_status_t status, int progress); +void control_event_bootstrap_problem(const char *warn, int reason); + +void control_event_clients_seen(const char *controller_str); + +#ifdef CONTROL_PRIVATE +/* Used only by control.c and test.c */ +size_t write_escaped_data(const char *data, size_t len, char **out); +size_t read_escaped_data(const char *data, size_t len, char **out); +#endif + +#endif + diff --git a/src/or/cpuworker.c b/src/or/cpuworker.c index fde149978c..2760d96665 100644 --- a/src/or/cpuworker.c +++ b/src/or/cpuworker.c @@ -13,6 +13,15 @@ **/ #include "or.h" +#include "buffers.h" +#include "circuitbuild.h" +#include "circuitlist.h" +#include "config.h" +#include "connection.h" +#include "cpuworker.h" +#include "main.h" +#include "onion.h" +#include "router.h" /** The maximum number of cpuworker processes we will keep around. */ #define MAX_CPUWORKERS 16 diff --git a/src/or/cpuworker.h b/src/or/cpuworker.h new file mode 100644 index 0000000000..e09703f217 --- /dev/null +++ b/src/or/cpuworker.h @@ -0,0 +1,25 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2010, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file cpuworker.h + * \brief Header file for cpuworker.c. + **/ + +#ifndef _TOR_CPUWORKER_H +#define _TOR_CPUWORKER_H + +void cpu_init(void); +void cpuworkers_rotate(void); +int connection_cpu_finished_flushing(connection_t *conn); +int connection_cpu_reached_eof(connection_t *conn); +int connection_cpu_process_inbuf(connection_t *conn); +int assign_onionskin_to_cpuworker(connection_t *cpuworker, + or_circuit_t *circ, + char *onionskin); + +#endif + diff --git a/src/or/directory.c b/src/or/directory.c index 7ec97d48e0..a3e575ac97 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -4,6 +4,26 @@ /* See LICENSE for licensing information */ #include "or.h" +#include "buffers.h" +#include "circuitbuild.h" +#include "config.h" +#include "connection.h" +#include "connection_edge.h" +#include "control.h" +#include "directory.h" +#include "dirserv.h" +#include "dirvote.h" +#include "geoip.h" +#include "main.h" +#include "networkstatus.h" +#include "policies.h" +#include "rendclient.h" +#include "rendcommon.h" +#include "rephist.h" +#include "router.h" +#include "routerlist.h" +#include "routerparse.h" + #if defined(EXPORTMALLINFO) && defined(HAVE_MALLOC_H) && defined(HAVE_MALLINFO) #ifndef OPENBSD #include <malloc.h> @@ -92,11 +112,13 @@ static void directory_initiate_command_rend(const char *address, #define ROUTERDESC_CACHE_LIFETIME (30*60) #define ROUTERDESC_BY_DIGEST_CACHE_LIFETIME (48*60*60) #define ROBOTS_CACHE_LIFETIME (24*60*60) +#define MICRODESC_CACHE_LIFETIME (48*60*60) /********* END VARIABLES ************/ -/** Return true iff the directory purpose 'purpose' must use an - * anonymous connection to a directory. */ +/** Return true iff the directory purpose <b>dir_purpose</b> (and if it's + * fetching descriptors, it's fetching them for <b>router_purpose</b>) + * must use an anonymous connection to a directory. */ static int purpose_needs_anonymity(uint8_t dir_purpose, uint8_t router_purpose) { @@ -230,7 +252,7 @@ directories_have_accepted_server_descriptor(void) /** Start a connection to every suitable directory authority, using * connection purpose 'purpose' and uploading the payload 'payload' - * (length 'payload_len'). The purpose should be one of + * (length 'payload_len'). dir_purpose should be one of * 'DIR_PURPOSE_UPLOAD_DIR' or 'DIR_PURPOSE_UPLOAD_RENDDESC'. * * <b>type</b> specifies what sort of dir authorities (V1, V2, @@ -560,7 +582,7 @@ connection_dir_request_failed(dir_connection_t *conn) if (directory_conn_is_self_reachability_test(conn)) { return; /* this was a test fetch. don't retry. */ } - if (entry_list_can_grow(get_options())) + if (entry_list_is_constrained(get_options())) router_set_status(conn->identity_digest, 0); /* don't try him again */ if (conn->_base.purpose == DIR_PURPOSE_FETCH_V2_NETWORKSTATUS) { log_info(LD_DIR, "Giving up on directory server at '%s'; retrying", @@ -614,7 +636,7 @@ connection_dir_download_networkstatus_failed(dir_connection_t *conn, * failed, and possibly retry them later.*/ smartlist_t *failed = smartlist_create(); dir_split_resource_into_fingerprints(conn->requested_resource+3, - failed, NULL, 0, 0); + failed, NULL, 0); if (smartlist_len(failed)) { dir_networkstatus_download_failed(failed, status_code); SMARTLIST_FOREACH(failed, char *, cp, tor_free(cp)); @@ -651,7 +673,7 @@ connection_dir_download_cert_failed(dir_connection_t *conn, int status) return; failed = smartlist_create(); dir_split_resource_into_fingerprints(conn->requested_resource+3, - failed, NULL, 1, 0); + failed, NULL, DSR_HEX); SMARTLIST_FOREACH(failed, char *, cp, { authority_cert_dl_failed(cp, status); @@ -667,7 +689,7 @@ connection_dir_download_cert_failed(dir_connection_t *conn, int status) * 1) If or_port is 0, or it's a direct conn and or_port is firewalled * or we're a dir mirror, no. * 2) If we prefer to avoid begindir conns, and we're not fetching or - * publishing a bridge relay descriptor, no. + * publishing a bridge relay descriptor, no. * 3) Else yes. */ static int @@ -746,6 +768,15 @@ directory_initiate_command_rend(const char *address, const tor_addr_t *_addr, log_debug(LD_DIR, "Initiating %s", dir_conn_purpose_to_string(dir_purpose)); + /* ensure that we don't make direct connections when a SOCKS server is + * configured. */ + if (!anonymized_connection && !use_begindir && !options->HttpProxy && + (options->Socks4Proxy || options->Socks5Proxy)) { + log_warn(LD_DIR, "Cannot connect to a directory server through a " + "SOCKS proxy!"); + return; + } + conn = dir_connection_new(AF_INET); /* set up conn so it's got all the data we need to remember */ @@ -771,7 +802,7 @@ directory_initiate_command_rend(const char *address, const tor_addr_t *_addr, /* then we want to connect to dirport directly */ if (options->HttpProxy) { - tor_addr_from_ipv4h(&addr, options->HttpProxyAddr); + tor_addr_copy(&addr, &options->HttpProxyAddr); dir_port = options->HttpProxyPort; } @@ -793,7 +824,7 @@ directory_initiate_command_rend(const char *address, const tor_addr_t *_addr, payload, payload_len, supports_conditional_consensus, if_modified_since); - connection_watch_events(TO_CONN(conn), EV_READ | EV_WRITE); + connection_watch_events(TO_CONN(conn), READ_EVENT | WRITE_EVENT); /* writable indicates finish, readable indicates broken link, error indicates broken link in windowsland. */ } @@ -832,7 +863,7 @@ directory_initiate_command_rend(const char *address, const tor_addr_t *_addr, payload, payload_len, supports_conditional_consensus, if_modified_since); - connection_watch_events(TO_CONN(conn), EV_READ | EV_WRITE); + connection_watch_events(TO_CONN(conn), READ_EVENT|WRITE_EVENT); connection_start_reading(TO_CONN(linked_conn)); } } @@ -880,7 +911,7 @@ directory_get_consensus_url(int supports_conditional_consensus) if (supports_conditional_consensus) { char *authority_id_list; - smartlist_t *authority_digets = smartlist_create(); + smartlist_t *authority_digests = smartlist_create(); SMARTLIST_FOREACH(router_get_trusted_dir_servers(), trusted_dir_server_t *, ds, @@ -892,10 +923,10 @@ directory_get_consensus_url(int supports_conditional_consensus) hex = tor_malloc(2*CONDITIONAL_CONSENSUS_FPR_LEN+1); base16_encode(hex, 2*CONDITIONAL_CONSENSUS_FPR_LEN+1, ds->v3_identity_digest, CONDITIONAL_CONSENSUS_FPR_LEN); - smartlist_add(authority_digets, hex); + smartlist_add(authority_digests, hex); }); - smartlist_sort(authority_digets, _compare_strs); - authority_id_list = smartlist_join_strings(authority_digets, + smartlist_sort(authority_digests, _compare_strs); + authority_id_list = smartlist_join_strings(authority_digests, "+", 0, NULL); len = strlen(authority_id_list)+64; @@ -903,8 +934,8 @@ directory_get_consensus_url(int supports_conditional_consensus) tor_snprintf(url, len, "/tor/status-vote/current/consensus/%s.z", authority_id_list); - SMARTLIST_FOREACH(authority_digets, char *, cp, tor_free(cp)); - smartlist_free(authority_digets); + SMARTLIST_FOREACH(authority_digests, char *, cp, tor_free(cp)); + smartlist_free(authority_digests); tor_free(authority_id_list); } else { url = tor_strdup("/tor/status-vote/current/consensus.z"); @@ -913,7 +944,7 @@ directory_get_consensus_url(int supports_conditional_consensus) } /** Queue an appropriate HTTP command on conn-\>outbuf. The other args - * are as in directory_initiate_command. + * are as in directory_initiate_command(). */ static void directory_send_command(dir_connection_t *conn, @@ -1049,31 +1080,10 @@ directory_send_command(dir_connection_t *conn, httpcommand = "POST"; url = tor_strdup("/tor/post/consensus-signature"); break; - case DIR_PURPOSE_FETCH_RENDDESC: - tor_assert(resource); - tor_assert(!payload); - - /* this must be true or we wouldn't be doing the lookup */ - tor_assert(strlen(resource) <= REND_SERVICE_ID_LEN_BASE32); - /* This breaks the function abstraction. */ - conn->rend_data = tor_malloc_zero(sizeof(rend_data_t)); - strlcpy(conn->rend_data->onion_address, resource, - sizeof(conn->rend_data->onion_address)); - conn->rend_data->rend_desc_version = 0; - - httpcommand = "GET"; - /* Request the most recent versioned descriptor. */ - // (XXXX We were going to switch this to fetch rendezvous1 descriptors, - // but that never got testing, and it wasn't a good design.) - len = strlen(resource)+32; - url = tor_malloc(len); - tor_snprintf(url, len, "/tor/rendezvous/%s", resource); - break; case DIR_PURPOSE_FETCH_RENDDESC_V2: tor_assert(resource); tor_assert(strlen(resource) <= REND_DESC_ID_V2_LEN_BASE32); tor_assert(!payload); - conn->rend_data->rend_desc_version = 2; httpcommand = "GET"; len = strlen(resource) + 32; url = tor_malloc(len); @@ -1162,7 +1172,7 @@ parse_http_url(const char *headers, char **url) if (s-tmp >= 3 && !strcmpstart(tmp,"://")) { tmp = strchr(tmp+3, '/'); if (tmp && tmp < s) { - log_debug(LD_DIR,"Skipping over 'http[s]://hostname' string"); + log_debug(LD_DIR,"Skipping over 'http[s]://hostname/' string"); start = tmp; } } @@ -1478,21 +1488,22 @@ connection_dir_client_reached_eof(dir_connection_t *conn) } (void) skewed; /* skewed isn't used yet. */ - if (status_code == 503 && body_len < 16) { - routerstatus_t *rs; - trusted_dir_server_t *ds; - log_info(LD_DIR,"Received http status code %d (%s) from server " - "'%s:%d'. I'll try again soon.", - status_code, escaped(reason), conn->_base.address, - conn->_base.port); - if ((rs = router_get_consensus_status_by_id(conn->identity_digest))) - rs->last_dir_503_at = now; - if ((ds = router_get_trusteddirserver_by_digest(conn->identity_digest))) - ds->fake_status.last_dir_503_at = now; + if (status_code == 503) { + if (body_len < 16) { + routerstatus_t *rs; + trusted_dir_server_t *ds; + log_info(LD_DIR,"Received http status code %d (%s) from server " + "'%s:%d'. I'll try again soon.", + status_code, escaped(reason), conn->_base.address, + conn->_base.port); + if ((rs = router_get_consensus_status_by_id(conn->identity_digest))) + rs->last_dir_503_at = now; + if ((ds = router_get_trusteddirserver_by_digest(conn->identity_digest))) + ds->fake_status.last_dir_503_at = now; - tor_free(body); tor_free(headers); tor_free(reason); - return -1; - } else if (status_code == 503) { + tor_free(body); tor_free(headers); tor_free(reason); + return -1; + } /* XXXX022 Remove this once every server with bug 539 is obsolete. */ log_info(LD_DIR, "Server at '%s:%d' sent us a 503 response, but included " "a body anyway. We'll pretend it gave us a 200.", @@ -1564,7 +1575,7 @@ connection_dir_client_reached_eof(dir_connection_t *conn) v2_networkstatus_source_t source; char *cp; log_info(LD_DIR,"Received networkstatus objects (size %d) from server " - "'%s:%d'",(int) body_len, conn->_base.address, conn->_base.port); + "'%s:%d'", (int)body_len, conn->_base.address, conn->_base.port); if (status_code != 200) { log_warn(LD_DIR, "Received http status code %d (%s) from server " @@ -1580,7 +1591,7 @@ connection_dir_client_reached_eof(dir_connection_t *conn) source = NS_FROM_DIR_BY_FP; which = smartlist_create(); dir_split_resource_into_fingerprints(conn->requested_resource+3, - which, NULL, 0, 0); + which, NULL, 0); } else if (conn->requested_resource && !strcmpstart(conn->requested_resource, "all")) { source = NS_FROM_DIR_ALL; @@ -1638,8 +1649,8 @@ connection_dir_client_reached_eof(dir_connection_t *conn) return -1; } log_info(LD_DIR,"Received consensus directory (size %d) from server " - "'%s:%d'",(int) body_len, conn->_base.address, conn->_base.port); - if ((r=networkstatus_set_current_consensus(body, 0))<0) { + "'%s:%d'", (int)body_len, conn->_base.address, conn->_base.port); + if ((r=networkstatus_set_current_consensus(body, "ns", 0))<0) { log_fn(r<-1?LOG_WARN:LOG_INFO, LD_DIR, "Unable to load consensus directory downloaded from " "server '%s:%d'. I'll try again soon.", @@ -1666,9 +1677,11 @@ connection_dir_client_reached_eof(dir_connection_t *conn) return -1; } log_info(LD_DIR,"Received authority certificates (size %d) from server " - "'%s:%d'",(int) body_len, conn->_base.address, conn->_base.port); + "'%s:%d'", (int)body_len, conn->_base.address, conn->_base.port); if (trusted_dirs_load_certs_from_string(body, 0, 1)<0) { log_warn(LD_DIR, "Unable to parse fetched certificates"); + /* if we fetched more than one and only some failed, the successful + * ones got flushed to disk so it's safe to call this on them */ connection_dir_download_cert_failed(conn, status_code); } else { directory_info_has_arrived(now, 0); @@ -1679,7 +1692,7 @@ connection_dir_client_reached_eof(dir_connection_t *conn) const char *msg; int st; log_info(LD_DIR,"Got votes (size %d) from server %s:%d", - (int) body_len, conn->_base.address, conn->_base.port); + (int)body_len, conn->_base.address, conn->_base.port); if (status_code != 200) { log_warn(LD_DIR, "Received http status code %d (%s) from server " @@ -1699,11 +1712,11 @@ connection_dir_client_reached_eof(dir_connection_t *conn) if (conn->_base.purpose == DIR_PURPOSE_FETCH_DETACHED_SIGNATURES) { const char *msg = NULL; log_info(LD_DIR,"Got detached signatures (size %d) from server %s:%d", - (int) body_len, conn->_base.address, conn->_base.port); + (int)body_len, conn->_base.address, conn->_base.port); if (status_code != 200) { log_warn(LD_DIR, - "Received http status code %d (%s) from server " - "'%s:%d' while fetching \"/tor/status-vote/consensus-signatures.z\".", + "Received http status code %d (%s) from server '%s:%d' while fetching " + "\"/tor/status-vote/next/consensus-signatures.z\".", status_code, escaped(reason), conn->_base.address, conn->_base.port); tor_free(body); tor_free(headers); tor_free(reason); @@ -1731,7 +1744,7 @@ connection_dir_client_reached_eof(dir_connection_t *conn) which = smartlist_create(); dir_split_resource_into_fingerprints(conn->requested_resource + (descriptor_digests ? 2 : 3), - which, NULL, 0, 0); + which, NULL, 0); n_asked_for = smartlist_len(which); } if (status_code != 200) { @@ -1919,7 +1932,7 @@ connection_dir_client_reached_eof(dir_connection_t *conn) /* Success, or at least there's a v2 descriptor already * present. Notify pending connections about this. */ conn->_base.purpose = DIR_PURPOSE_HAS_FETCHED_RENDDESC; - rend_client_desc_trynow(conn->rend_data->onion_address, -1); + rend_client_desc_trynow(conn->rend_data->onion_address); } break; case 404: @@ -1966,7 +1979,7 @@ connection_dir_client_reached_eof(dir_connection_t *conn) log_info(LD_REND, "Successfully fetched v2 rendezvous " "descriptor."); conn->_base.purpose = DIR_PURPOSE_HAS_FETCHED_RENDDESC; - rend_client_desc_trynow(conn->rend_data->onion_address, -1); + rend_client_desc_trynow(conn->rend_data->onion_address); break; } break; @@ -2009,12 +2022,6 @@ connection_dir_client_reached_eof(dir_connection_t *conn) "'%s:%d'. Malformed rendezvous descriptor?", escaped(reason), conn->_base.address, conn->_base.port); break; - case 503: - log_info(LD_REND,"http status 503 (%s) response from dirserver " - "'%s:%d'. Node is (currently) not acting as v2 hidden " - "service directory.", - escaped(reason), conn->_base.address, conn->_base.port); - break; default: log_warn(LD_REND,"http status %d (%s) response unexpected (server " "'%s:%d').", @@ -2321,7 +2328,7 @@ directory_dump_request_log(void) } #endif -/** Decide whether a client would accept the consensus we have +/** Decide whether a client would accept the consensus we have. * * Clients can say they only want a consensus if it's signed by more * than half the authorities in a list. They pass this list in @@ -2342,31 +2349,32 @@ client_likes_consensus(networkstatus_t *v, const char *want_url) int need_at_least; int have = 0; - dir_split_resource_into_fingerprints(want_url, want_authorities, NULL, 0, 0); + dir_split_resource_into_fingerprints(want_url, want_authorities, NULL, 0); need_at_least = smartlist_len(want_authorities)/2+1; - SMARTLIST_FOREACH(want_authorities, const char *, d, { + SMARTLIST_FOREACH_BEGIN(want_authorities, const char *, d) { char want_digest[DIGEST_LEN]; size_t want_len = strlen(d)/2; if (want_len > DIGEST_LEN) want_len = DIGEST_LEN; if (base16_decode(want_digest, DIGEST_LEN, d, want_len*2) < 0) { - log_warn(LD_DIR,"Failed to decode requested authority digest %s.", d); + log_fn(LOG_PROTOCOL_WARN, LD_DIR, + "Failed to decode requested authority digest %s.", d); continue; }; - SMARTLIST_FOREACH(v->voters, networkstatus_voter_info_t *, vi, { - if (vi->signature && + SMARTLIST_FOREACH_BEGIN(v->voters, networkstatus_voter_info_t *, vi) { + if (smartlist_len(vi->sigs) && !memcmp(vi->identity_digest, want_digest, want_len)) { have++; break; }; - }); + } SMARTLIST_FOREACH_END(vi); /* early exit, if we already have enough */ if (have >= need_at_least) break; - }); + } SMARTLIST_FOREACH_END(d); SMARTLIST_FOREACH(want_authorities, char *, d, tor_free(d)); smartlist_free(want_authorities); @@ -2513,9 +2521,12 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, /* v2 or v3 network status fetch. */ smartlist_t *dir_fps = smartlist_create(); int is_v3 = !strcmpstart(url, "/tor/status-vote"); + geoip_client_action_t act = + is_v3 ? GEOIP_CLIENT_NETWORKSTATUS : GEOIP_CLIENT_NETWORKSTATUS_V2; const char *request_type = NULL; const char *key = url + strlen("/tor/status/"); long lifetime = NETWORKSTATUS_CACHE_LIFETIME; + if (!is_v3) { dirserv_get_networkstatus_v2_fingerprints(dir_fps, key); if (!strcmpstart(key, "fp/")) @@ -2530,18 +2541,44 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, } else { networkstatus_t *v = networkstatus_get_latest_consensus(); time_t now = time(NULL); + const char *want_fps = NULL; + char *flavor = NULL; #define CONSENSUS_URL_PREFIX "/tor/status-vote/current/consensus/" - if (v && - !strcmpstart(url, CONSENSUS_URL_PREFIX) && - !client_likes_consensus(v, url + strlen(CONSENSUS_URL_PREFIX))) { + #define CONSENSUS_FLAVORED_PREFIX "/tor/status-vote/current/consensus-" + /* figure out the flavor if any, and who we wanted to sign the thing */ + if (!strcmpstart(url, CONSENSUS_FLAVORED_PREFIX)) { + const char *f, *cp; + f = url + strlen(CONSENSUS_FLAVORED_PREFIX); + cp = strchr(f, '/'); + if (cp) { + want_fps = cp+1; + flavor = tor_strndup(f, cp-f); + } else { + flavor = tor_strdup(f); + } + } else { + if (!strcmpstart(url, CONSENSUS_URL_PREFIX)) + want_fps = url+strlen(CONSENSUS_URL_PREFIX); + } + + /* XXXX MICRODESC NM NM should check document of correct flavor */ + if (v && want_fps && + !client_likes_consensus(v, want_fps)) { write_http_status_line(conn, 404, "Consensus not signed by sufficient " "number of requested authorities"); smartlist_free(dir_fps); + geoip_note_ns_response(act, GEOIP_REJECT_NOT_ENOUGH_SIGS); + tor_free(flavor); goto done; } - smartlist_add(dir_fps, tor_memdup("\0\0\0\0\0\0\0\0\0\0" - "\0\0\0\0\0\0\0\0\0\0", 20)); + { + char *fp = tor_malloc_zero(DIGEST_LEN); + if (flavor) + strlcpy(fp, flavor, DIGEST_LEN); + tor_free(flavor); + smartlist_add(dir_fps, fp); + } request_type = compressed?"v3.z":"v3"; lifetime = (v && v->fresh_until > now) ? v->fresh_until - now : 0; } @@ -2549,6 +2586,7 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, if (!smartlist_len(dir_fps)) { /* we failed to create/cache cp */ write_http_status_line(conn, 503, "Network status object unavailable"); smartlist_free(dir_fps); + geoip_note_ns_response(act, GEOIP_REJECT_UNAVAILABLE); goto done; } @@ -2556,11 +2594,13 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, write_http_status_line(conn, 404, "Not found"); SMARTLIST_FOREACH(dir_fps, char *, cp, tor_free(cp)); smartlist_free(dir_fps); + geoip_note_ns_response(act, GEOIP_REJECT_NOT_FOUND); goto done; } else if (!smartlist_len(dir_fps)) { write_http_status_line(conn, 304, "Not modified"); SMARTLIST_FOREACH(dir_fps, char *, cp, tor_free(cp)); smartlist_free(dir_fps); + geoip_note_ns_response(act, GEOIP_REJECT_NOT_MODIFIED); goto done; } @@ -2572,18 +2612,25 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, write_http_status_line(conn, 503, "Directory busy, try again later"); SMARTLIST_FOREACH(dir_fps, char *, fp, tor_free(fp)); smartlist_free(dir_fps); + geoip_note_ns_response(act, GEOIP_REJECT_BUSY); goto done; } -#ifdef ENABLE_GEOIP_STATS { - geoip_client_action_t act = - is_v3 ? GEOIP_CLIENT_NETWORKSTATUS : GEOIP_CLIENT_NETWORKSTATUS_V2; struct in_addr in; - if (tor_inet_aton((TO_CONN(conn))->address, &in)) + if (tor_inet_aton((TO_CONN(conn))->address, &in)) { geoip_note_client_seen(act, ntohl(in.s_addr), time(NULL)); + geoip_note_ns_response(act, GEOIP_SUCCESS); + /* Note that a request for a network status has started, so that we + * can measure the download time later on. */ + if (TO_CONN(conn)->dirreq_id) + geoip_start_dirreq(TO_CONN(conn)->dirreq_id, dlen, act, + DIRREQ_TUNNELED); + else + geoip_start_dirreq(TO_CONN(conn)->global_identifier, dlen, act, + DIRREQ_DIRECT); + } } -#endif // note_request(request_type,dlen); (void) request_type; @@ -2619,7 +2666,7 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, const char *item; tor_assert(!current); /* we handle current consensus specially above, * since it wants to be spooled. */ - if ((item = dirvote_get_pending_consensus())) + if ((item = dirvote_get_pending_consensus(FLAV_NS))) smartlist_add(items, (char*)item); } else if (!current && !strcmp(url, "consensus-signatures")) { /* XXXX the spec says that we should implement @@ -2645,7 +2692,8 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, flags = DGV_BY_ID | (current ? DGV_INCLUDE_PREVIOUS : DGV_INCLUDE_PENDING); } - dir_split_resource_into_fingerprints(url, fps, NULL, 1, 1); + dir_split_resource_into_fingerprints(url, fps, NULL, + DSR_HEX|DSR_SORT_UNIQ); SMARTLIST_FOREACH(fps, char *, fp, { if ((d = dirvote_get_vote(fp, flags))) smartlist_add(dir_items, (cached_dir_t*)d); @@ -2698,6 +2746,41 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, goto done; } + if (!strcmpstart(url, "/tor/micro/d/")) { + smartlist_t *fps = smartlist_create(); + + dir_split_resource_into_fingerprints(url+strlen("/tor/micro/d/"), + fps, NULL, + DSR_DIGEST256|DSR_BASE64|DSR_SORT_UNIQ); + + if (!dirserv_have_any_microdesc(fps)) { + write_http_status_line(conn, 404, "Not found"); + SMARTLIST_FOREACH(fps, char *, fp, tor_free(fp)); + smartlist_free(fps); + goto done; + } + dlen = dirserv_estimate_microdesc_size(fps, compressed); + if (global_write_bucket_low(TO_CONN(conn), dlen, 2)) { + log_info(LD_DIRSERV, + "Client asked for server descriptors, but we've been " + "writing too many bytes lately. Sending 503 Dir busy."); + write_http_status_line(conn, 503, "Directory busy, try again later"); + SMARTLIST_FOREACH(fps, char *, fp, tor_free(fp)); + smartlist_free(fps); + goto done; + } + + write_http_response_header(conn, -1, compressed, MICRODESC_CACHE_LIFETIME); + conn->dir_spool_src = DIR_SPOOL_MICRODESC; + conn->fingerprint_stack = fps; + + if (compressed) + conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD); + + connection_dirserv_flushed_some(conn); + goto done; + } + if (!strcmpstart(url,"/tor/server/") || (!options->BridgeAuthoritativeDir && !options->BridgeRelay && !strcmpstart(url,"/tor/extra/"))) { @@ -2779,7 +2862,8 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, } else if (!strcmpstart(url, "/tor/keys/fp/")) { smartlist_t *fps = smartlist_create(); dir_split_resource_into_fingerprints(url+strlen("/tor/keys/fp/"), - fps, NULL, 1, 1); + fps, NULL, + DSR_HEX|DSR_SORT_UNIQ); SMARTLIST_FOREACH(fps, char *, d, { authority_cert_t *c = authority_cert_get_newest_by_id(d); if (c) smartlist_add(certs, c); @@ -2789,7 +2873,8 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, } else if (!strcmpstart(url, "/tor/keys/sk/")) { smartlist_t *fps = smartlist_create(); dir_split_resource_into_fingerprints(url+strlen("/tor/keys/sk/"), - fps, NULL, 1, 1); + fps, NULL, + DSR_HEX|DSR_SORT_UNIQ); SMARTLIST_FOREACH(fps, char *, d, { authority_cert_t *c = authority_cert_get_by_sk_digest(d); if (c) smartlist_add(certs, c); @@ -2891,18 +2976,9 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, note_request("/tor/rendezvous?/", desc_len); /* need to send descp separately, because it may include NULs */ connection_write_to_buf(descp, desc_len, TO_CONN(conn)); - /* report successful fetch to statistic */ - if (options->HSAuthorityRecordStats) { - hs_usage_note_fetch_total(query, time(NULL)); - hs_usage_note_fetch_successful(query, time(NULL)); - } break; case 0: /* well-formed but not present */ write_http_status_line(conn, 404, "Not found"); - /* report (unsuccessful) fetch to statistic */ - if (options->HSAuthorityRecordStats) { - hs_usage_note_fetch_total(query, time(NULL)); - } break; case -1: /* not well-formed */ write_http_status_line(conn, 400, "Bad request"); @@ -3179,8 +3255,8 @@ directory_handle_command(dir_connection_t *conn) &body, &body_len, MAX_DIR_UL_SIZE, 0)) { case -1: /* overflow */ log_warn(LD_DIRSERV, - "Invalid input from address '%s'. Closing.", - conn->_base.address); + "Request too large from address '%s' to DirPort. Closing.", + safe_str(conn->_base.address)); return -1; case 0: log_debug(LD_DIRSERV,"command not all here yet."); @@ -3216,6 +3292,16 @@ connection_dir_finished_flushing(dir_connection_t *conn) tor_assert(conn); tor_assert(conn->_base.type == CONN_TYPE_DIR); + /* Note that we have finished writing the directory response. For direct + * connections this means we're done, for tunneled connections its only + * an intermediate step. */ + if (TO_CONN(conn)->dirreq_id) + geoip_change_dirreq_state(TO_CONN(conn)->dirreq_id, DIRREQ_TUNNELED, + DIRREQ_FLUSHING_DIR_CONN_FINISHED); + else + geoip_change_dirreq_state(TO_CONN(conn)->global_identifier, + DIRREQ_DIRECT, + DIRREQ_FLUSHING_DIR_CONN_FINISHED); switch (conn->_base.state) { case DIR_CONN_STATE_CLIENT_SENDING: log_debug(LD_DIR,"client finished sending command."); @@ -3515,19 +3601,37 @@ dir_split_resource_into_fingerprint_pairs(const char *res, /** Given a directory <b>resource</b> request, containing zero * or more strings separated by plus signs, followed optionally by ".z", store * the strings, in order, into <b>fp_out</b>. If <b>compressed_out</b> is - * non-NULL, set it to 1 if the resource ends in ".z", else set it to 0. If - * decode_hex is true, then delete all elements that aren't hex digests, and - * decode the rest. If sort_uniq is true, then sort the list and remove - * all duplicates. + * non-NULL, set it to 1 if the resource ends in ".z", else set it to 0. + * + * If (flags & DSR_HEX), then delete all elements that aren't hex digests, and + * decode the rest. If (flags & DSR_BASE64), then use "-" rather than "+" as + * a separator, delete all the elements that aren't base64-encoded digests, + * and decode the rest. If (flags & DSR_DIGEST256), these digests should be + * 256 bits long; else they should be 160. + * + * If (flags & DSR_SORT_UNIQ), then sort the list and remove all duplicates. */ int dir_split_resource_into_fingerprints(const char *resource, smartlist_t *fp_out, int *compressed_out, - int decode_hex, int sort_uniq) + int flags) { + const int decode_hex = flags & DSR_HEX; + const int decode_base64 = flags & DSR_BASE64; + const int digests_are_256 = flags & DSR_DIGEST256; + const int sort_uniq = flags & DSR_SORT_UNIQ; + + const int digest_len = digests_are_256 ? DIGEST256_LEN : DIGEST_LEN; + const int hex_digest_len = digests_are_256 ? + HEX_DIGEST256_LEN : HEX_DIGEST_LEN; + const int base64_digest_len = digests_are_256 ? + BASE64_DIGEST256_LEN : BASE64_DIGEST_LEN; smartlist_t *fp_tmp = smartlist_create(); + + tor_assert(!(decode_hex && decode_base64)); tor_assert(fp_out); - smartlist_split_string(fp_tmp, resource, "+", 0, 0); + + smartlist_split_string(fp_tmp, resource, decode_base64?"-":"+", 0, 0); if (compressed_out) *compressed_out = 0; if (smartlist_len(fp_tmp)) { @@ -3539,22 +3643,25 @@ dir_split_resource_into_fingerprints(const char *resource, *compressed_out = 1; } } - if (decode_hex) { + if (decode_hex || decode_base64) { + const size_t encoded_len = decode_hex ? hex_digest_len : base64_digest_len; int i; char *cp, *d = NULL; for (i = 0; i < smartlist_len(fp_tmp); ++i) { cp = smartlist_get(fp_tmp, i); - if (strlen(cp) != HEX_DIGEST_LEN) { + if (strlen(cp) != encoded_len) { log_info(LD_DIR, "Skipping digest %s with non-standard length.", escaped(cp)); smartlist_del_keeporder(fp_tmp, i--); goto again; } - d = tor_malloc_zero(DIGEST_LEN); - if (base16_decode(d, DIGEST_LEN, cp, HEX_DIGEST_LEN)<0) { - log_info(LD_DIR, "Skipping non-decodable digest %s", escaped(cp)); - smartlist_del_keeporder(fp_tmp, i--); - goto again; + d = tor_malloc_zero(digest_len); + if (decode_hex ? + (base16_decode(d, digest_len, cp, hex_digest_len)<0) : + (base64_decode(d, digest_len, cp, base64_digest_len)<0)) { + log_info(LD_DIR, "Skipping non-decodable digest %s", escaped(cp)); + smartlist_del_keeporder(fp_tmp, i--); + goto again; } smartlist_set(fp_tmp, i, d); d = NULL; @@ -3564,26 +3671,18 @@ dir_split_resource_into_fingerprints(const char *resource, } } if (sort_uniq) { - smartlist_t *fp_tmp2 = smartlist_create(); - int i; - if (decode_hex) - smartlist_sort_digests(fp_tmp); - else + if (decode_hex || decode_base64) { + if (digests_are_256) { + smartlist_sort_digests256(fp_tmp); + smartlist_uniq_digests256(fp_tmp); + } else { + smartlist_sort_digests(fp_tmp); + smartlist_uniq_digests(fp_tmp); + } + } else { smartlist_sort_strings(fp_tmp); - if (smartlist_len(fp_tmp)) - smartlist_add(fp_tmp2, smartlist_get(fp_tmp, 0)); - for (i = 1; i < smartlist_len(fp_tmp); ++i) { - char *cp = smartlist_get(fp_tmp, i); - char *last = smartlist_get(fp_tmp2, smartlist_len(fp_tmp2)-1); - - if ((decode_hex && memcmp(cp, last, DIGEST_LEN)) - || (!decode_hex && strcasecmp(cp, last))) - smartlist_add(fp_tmp2, cp); - else - tor_free(cp); + smartlist_uniq_strings(fp_tmp); } - smartlist_free(fp_tmp); - fp_tmp = fp_tmp2; } smartlist_add_all(fp_out, fp_tmp); smartlist_free(fp_tmp); diff --git a/src/or/directory.h b/src/or/directory.h new file mode 100644 index 0000000000..36b4cf2b18 --- /dev/null +++ b/src/or/directory.h @@ -0,0 +1,108 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2010, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file directory.h + * \brief Header file for directory.c. + **/ + +#ifndef _TOR_DIRECTORY_H +#define _TOR_DIRECTORY_H + +int directories_have_accepted_server_descriptor(void); +char *authority_type_to_string(authority_type_t auth); +void directory_post_to_dirservers(uint8_t dir_purpose, uint8_t router_purpose, + authority_type_t type, const char *payload, + size_t payload_len, size_t extrainfo_len); +void directory_get_from_dirserver(uint8_t dir_purpose, uint8_t router_purpose, + const char *resource, + int pds_flags); +void directory_get_from_all_authorities(uint8_t dir_purpose, + uint8_t router_purpose, + const char *resource); +void directory_initiate_command_routerstatus(routerstatus_t *status, + uint8_t dir_purpose, + uint8_t router_purpose, + int anonymized_connection, + const char *resource, + const char *payload, + size_t payload_len, + time_t if_modified_since); +void directory_initiate_command_routerstatus_rend(routerstatus_t *status, + uint8_t dir_purpose, + uint8_t router_purpose, + int anonymized_connection, + const char *resource, + const char *payload, + size_t payload_len, + time_t if_modified_since, + const rend_data_t *rend_query); + +int parse_http_response(const char *headers, int *code, time_t *date, + compress_method_t *compression, char **response); + +int connection_dir_is_encrypted(dir_connection_t *conn); +int connection_dir_reached_eof(dir_connection_t *conn); +int connection_dir_process_inbuf(dir_connection_t *conn); +int connection_dir_finished_flushing(dir_connection_t *conn); +int connection_dir_finished_connecting(dir_connection_t *conn); +void connection_dir_request_failed(dir_connection_t *conn); +void directory_initiate_command(const char *address, const tor_addr_t *addr, + uint16_t or_port, uint16_t dir_port, + int supports_conditional_consensus, + int supports_begindir, const char *digest, + uint8_t dir_purpose, uint8_t router_purpose, + int anonymized_connection, + const char *resource, + const char *payload, size_t payload_len, + time_t if_modified_since); + +#define DSR_HEX (1<<0) +#define DSR_BASE64 (1<<1) +#define DSR_DIGEST256 (1<<2) +#define DSR_SORT_UNIQ (1<<3) +int dir_split_resource_into_fingerprints(const char *resource, + smartlist_t *fp_out, int *compressed_out, + int flags); + +int dir_split_resource_into_fingerprint_pairs(const char *res, + smartlist_t *pairs_out); +char *directory_dump_request_log(void); +void note_request(const char *key, size_t bytes); +int router_supports_extrainfo(const char *identity_digest, int is_authority); + +time_t download_status_increment_failure(download_status_t *dls, + int status_code, const char *item, + int server, time_t now); +/** Increment the failure count of the download_status_t <b>dls</b>, with + * the optional status code <b>sc</b>. */ +#define download_status_failed(dls, sc) \ + download_status_increment_failure((dls), (sc), NULL, \ + get_options()->DirPort, time(NULL)) + +void download_status_reset(download_status_t *dls); +static int download_status_is_ready(download_status_t *dls, time_t now, + int max_failures); +/** Return true iff, as of <b>now</b>, the resource tracked by <b>dls</b> is + * ready to get its download reattempted. */ +static INLINE int +download_status_is_ready(download_status_t *dls, time_t now, + int max_failures) +{ + return (dls->n_download_failures <= max_failures + && dls->next_attempt_at <= now); +} + +static void download_status_mark_impossible(download_status_t *dl); +/** Mark <b>dl</b> as never downloadable. */ +static INLINE void +download_status_mark_impossible(download_status_t *dl) +{ + dl->n_download_failures = IMPOSSIBLE_TO_DOWNLOAD; +} + +#endif + diff --git a/src/or/dirserv.c b/src/or/dirserv.c index 851fc5cbdd..86cd186111 100644 --- a/src/or/dirserv.c +++ b/src/or/dirserv.c @@ -5,6 +5,22 @@ #define DIRSERV_PRIVATE #include "or.h" +#include "buffers.h" +#include "config.h" +#include "connection.h" +#include "connection_or.h" +#include "control.h" +#include "directory.h" +#include "dirserv.h" +#include "dirvote.h" +#include "hibernate.h" +#include "microdesc.h" +#include "networkstatus.h" +#include "policies.h" +#include "rephist.h" +#include "router.h" +#include "routerlist.h" +#include "routerparse.h" /** * \file dirserv.c @@ -41,7 +57,7 @@ static time_t the_v2_networkstatus_is_dirty = 1; static cached_dir_t *the_directory = NULL; /** For authoritative directories: the current (v1) network status. */ -static cached_dir_t the_runningrouters = { NULL, NULL, 0, 0, 0, -1 }; +static cached_dir_t the_runningrouters; static void directory_remove_invalid(void); static cached_dir_t *dirserv_regenerate_directory(void); @@ -63,13 +79,16 @@ static signed_descriptor_t *get_signed_descriptor_by_fp(const char *fp, time_t publish_cutoff); static int dirserv_add_extrainfo(extrainfo_t *ei, const char **msg); +/************** Measured Bandwidth parsing code ******/ +#define MAX_MEASUREMENT_AGE (3*24*60*60) /* 3 days */ + /************** Fingerprint handling code ************/ #define FP_NAMED 1 /**< Listed in fingerprint file. */ #define FP_INVALID 2 /**< Believed invalid. */ #define FP_REJECT 4 /**< We will not publish this router. */ #define FP_BADDIR 8 /**< We'll tell clients to avoid using this as a dir. */ -#define FP_BADEXIT 16 /**< We'll tell clients not to use this as an exit. */ +#define FP_BADEXIT 16 /**< We'll tell clients not to use this as an exit. */ #define FP_UNNAMED 32 /**< Another router has this name in fingerprint file. */ /** Encapsulate a nickname and an FP_* status; target of status_by_digest @@ -99,7 +118,7 @@ authdir_config_new(void) return list; } -/** Add the fingerprint <b>fp</b> for the nickname <b>nickname</b> to +/** Add the fingerprint <b>fp</b> for <b>nickname</b> to * the smartlist of fingerprint_entry_t's <b>list</b>. Return 0 if it's * new, or 1 if we replaced the old value. */ @@ -181,8 +200,7 @@ dirserv_add_own_fingerprint(const char *nickname, crypto_pk_env_t *pk) * file. The file format is line-based, with each non-blank holding one * nickname, some space, and a fingerprint for that nickname. On success, * replace the current fingerprint list with the new list and return 0. On - * failure, leave the current fingerprint list untouched, and - * return -1. */ + * failure, leave the current fingerprint list untouched, and return -1. */ int dirserv_load_fingerprint_file(void) { @@ -368,10 +386,10 @@ dirserv_get_status_impl(const char *id_digest, const char *nickname, strmap_size(fingerprint_list->fp_by_name), digestmap_size(fingerprint_list->status_by_digest)); - /* 0.1.1.17-rc was the first version that claimed to be stable, doesn't - * crash and drop circuits all the time, and is even vaguely compatible with - * the current network */ - if (platform && !tor_version_as_new_as(platform,"0.1.1.17-rc")) { + /* Tor 0.1.2.x is pretty old, but there are a lot of them running still, + * and there aren't any critical relay-side vulnerabilities. Once more + * of them die off, we should raise this minimum to 0.2.0.x. */ + if (platform && !tor_version_as_new_as(platform,"0.1.2.14")) { if (msg) *msg = "Tor version is far too old to work."; return FP_REJECT; @@ -520,7 +538,7 @@ authdir_wants_to_reject_router(routerinfo_t *ri, const char **msg, /* Okay. Now check whether the fingerprint is recognized. */ uint32_t status = dirserv_router_get_status(ri, msg); time_t now; - int severity = complain ? LOG_NOTICE : LOG_INFO; + int severity = (complain && ri->contact_info) ? LOG_NOTICE : LOG_INFO; tor_assert(msg); if (status & FP_REJECT) return -1; /* msg is already set. */ @@ -835,46 +853,6 @@ directory_remove_invalid(void) routerlist_assert_ok(rl); } -/** Write a list of unregistered descriptors into a newly allocated - * string and return it. Used by dirserv operators to keep track of - * fast nodes that haven't registered. - */ -int -getinfo_helper_dirserv_unregistered(control_connection_t *control_conn, - const char *question, char **answer_out) -{ - smartlist_t *answerlist; - char buf[1024]; - char *answer; - int min_bw = atoi(question); - routerlist_t *rl = router_get_routerlist(); - - (void) control_conn; - - if (strcmpstart(question, "unregistered-servers-")) - return 0; - question += strlen("unregistered-servers-"); - - answerlist = smartlist_create(); - SMARTLIST_FOREACH(rl->routers, routerinfo_t *, ent, { - uint32_t r = dirserv_router_get_status(ent, NULL); - if (router_get_advertised_bandwidth(ent) >= (size_t)min_bw && - !(r & FP_NAMED)) { - /* then log this one */ - tor_snprintf(buf, sizeof(buf), - "%s: BW %d on '%s'.", - ent->nickname, router_get_advertised_bandwidth(ent), - ent->platform ? ent->platform : ""); - smartlist_add(answerlist, tor_strdup(buf)); - } - }); - answer = smartlist_join_strings(answerlist, "\r\n", 0, NULL); - SMARTLIST_FOREACH(answerlist, char *, cp, tor_free(cp)); - smartlist_free(answerlist); - *answer_out = answer; - return 0; -} - /** Mark the directory as <b>dirty</b> -- when we're next asked for a * directory, we will rebuild it instead of reusing the most recently * generated one. @@ -933,6 +911,13 @@ list_single_server_status(routerinfo_t *desc, int is_live) return tor_strdup(buf); } +static INLINE int +running_long_enough_to_decide_unreachable(void) +{ + return time_of_process_start + + get_options()->TestingAuthDirTimeToLearnReachability < approx_time(); +} + /** Each server needs to have passed a reachability test no more * than this number of seconds ago, or he is listed as down in * the directory. */ @@ -944,6 +929,10 @@ list_single_server_status(routerinfo_t *desc, int is_live) void dirserv_set_router_is_running(routerinfo_t *router, time_t now) { + /*XXXX022 This function is a mess. Separate out the part that calculates + whether it's reachable and the part that tells rephist that the router was + unreachable. + */ int answer; if (router_is_me(router) && !we_are_hibernating()) @@ -952,7 +941,7 @@ dirserv_set_router_is_running(routerinfo_t *router, time_t now) answer = get_options()->AssumeReachable || now < router->last_reachable + REACHABLE_TIMEOUT; - if (!answer) { + if (!answer && running_long_enough_to_decide_unreachable()) { /* not considered reachable. tell rephist. */ rep_hist_note_router_unreachable(router->cache_info.identity_digest, now); } @@ -965,7 +954,6 @@ dirserv_set_router_is_running(routerinfo_t *router, time_t now) * *<b>router_status_out</b>. Return 0 on success, -1 on failure. * * If for_controller is true, include the routers with very old descriptors. - * If for_controller is >1, use the verbose nickname format. */ int list_server_status_v1(smartlist_t *routers, char **router_status_out, @@ -985,23 +973,22 @@ list_server_status_v1(smartlist_t *routers, char **router_status_out, rs_entries = smartlist_create(); - SMARTLIST_FOREACH(routers, routerinfo_t *, ri, - { + SMARTLIST_FOREACH_BEGIN(routers, routerinfo_t *, ri) { if (authdir) { /* Update router status in routerinfo_t. */ dirserv_set_router_is_running(ri, now); } - if (for_controller == 1 || ri->cache_info.published_on >= cutoff) - smartlist_add(rs_entries, list_single_server_status(ri, ri->is_running)); - else if (for_controller > 2) { + if (for_controller) { char name_buf[MAX_VERBOSE_NICKNAME_LEN+2]; char *cp = name_buf; if (!ri->is_running) *cp++ = '!'; router_get_verbose_nickname(cp, ri); smartlist_add(rs_entries, tor_strdup(name_buf)); + } else if (ri->cache_info.published_on >= cutoff) { + smartlist_add(rs_entries, list_single_server_status(ri, ri->is_running)); } - }); + } SMARTLIST_FOREACH_END(ri); *router_status_out = smartlist_join_strings(rs_entries, " ", 0, NULL); @@ -1119,7 +1106,8 @@ dirserv_dump_directory_to_string(char **dir_out, return -1; } note_crypto_pk_op(SIGN_DIR); - if (router_append_dirobj_signature(buf,buf_len,digest,private_key)<0) { + if (router_append_dirobj_signature(buf,buf_len,digest,DIGEST_LEN, + private_key)<0) { tor_free(buf); return -1; } @@ -1150,12 +1138,12 @@ directory_fetches_from_authorities(or_options_t *options) return 0; if (server_mode(options) && router_pick_published_address(options, &addr)<0) return 1; /* we don't know our IP address; ask an authority. */ - if (options->DirPort == 0) + if (options->DirPort == 0 && !options->RefuseUnknownExits) return 0; if (!server_mode(options) || !advertised_server_mode()) return 0; me = router_get_my_routerinfo(); - if (!me || !me->dir_port) + if (!me || (!me->dir_port && !options->RefuseUnknownExits)) return 0; /* if dirport not advertised, return 0 too */ return 1; } @@ -1195,7 +1183,11 @@ directory_caches_v2_dir_info(or_options_t *options) int directory_caches_dir_info(or_options_t *options) { - return options->BridgeRelay != 0 || options->DirPort != 0; + if (options->BridgeRelay || options->DirPort) + return 1; + if (!server_mode(options) || !advertised_server_mode()) + return 0; + return options->RefuseUnknownExits; } /** Return 1 if we want to allow remote people to ask us directory @@ -1238,14 +1230,14 @@ directory_too_idle_to_fetch_descriptors(or_options_t *options, time_t now) static cached_dir_t *cached_directory = NULL; /** The v1 runningrouters document we'll serve (as a cache or as an authority) * if requested. */ -static cached_dir_t cached_runningrouters = { NULL, NULL, 0, 0, 0, -1 }; +static cached_dir_t cached_runningrouters; /** Used for other dirservers' v2 network statuses. Map from hexdigest to * cached_dir_t. */ static digestmap_t *cached_v2_networkstatus = NULL; -/** The v3 consensus network status that we're currently serving. */ -static cached_dir_t *cached_v3_networkstatus = NULL; +/** Map from flavor name to the v3 consensuses that we're currently serving. */ +static strmap_t *cached_consensuses = NULL; /** Possibly replace the contents of <b>d</b> with the value of * <b>directory</b> published on <b>when</b>, unless <b>when</b> is older than @@ -1319,7 +1311,11 @@ clear_cached_dir(cached_dir_t *d) static void _free_cached_dir(void *_d) { - cached_dir_t *d = (cached_dir_t *)_d; + cached_dir_t *d; + if (!_d) + return; + + d = (cached_dir_t *)_d; cached_dir_decref(d); } @@ -1413,17 +1409,26 @@ dirserv_set_cached_networkstatus_v2(const char *networkstatus, } } -/** Replace the v3 consensus networkstatus that we're serving with - * <b>networkstatus</b>, published at <b>published</b>. No validation is - * performed. */ +/** Replace the v3 consensus networkstatus of type <b>flavor_name</b> that + * we're serving with <b>networkstatus</b>, published at <b>published</b>. No + * validation is performed. */ void -dirserv_set_cached_networkstatus_v3(const char *networkstatus, - time_t published) +dirserv_set_cached_consensus_networkstatus(const char *networkstatus, + const char *flavor_name, + const digests_t *digests, + time_t published) { - if (cached_v3_networkstatus) - cached_dir_decref(cached_v3_networkstatus); - cached_v3_networkstatus = new_cached_dir( - tor_strdup(networkstatus), published); + cached_dir_t *new_networkstatus; + cached_dir_t *old_networkstatus; + if (!cached_consensuses) + cached_consensuses = strmap_new(); + + new_networkstatus = new_cached_dir(tor_strdup(networkstatus), published); + memcpy(&new_networkstatus->digests, digests, sizeof(digests_t)); + old_networkstatus = strmap_set(cached_consensuses, flavor_name, + new_networkstatus); + if (old_networkstatus) + cached_dir_decref(old_networkstatus); } /** Remove any v2 networkstatus from the directory cache that was published @@ -1577,7 +1582,8 @@ generate_runningrouters(void) goto err; } note_crypto_pk_op(SIGN_DIR); - if (router_append_dirobj_signature(s, len, digest, private_key)<0) + if (router_append_dirobj_signature(s, len, digest, DIGEST_LEN, + private_key)<0) goto err; set_cached_dir(&the_runningrouters, s, time(NULL)); @@ -1605,9 +1611,11 @@ dirserv_get_runningrouters(void) /** Return the latest downloaded consensus networkstatus in encoded, signed, * optionally compressed format, suitable for sending to clients. */ cached_dir_t * -dirserv_get_consensus(void) +dirserv_get_consensus(const char *flavor_name) { - return cached_v3_networkstatus; + if (!cached_consensuses) + return NULL; + return strmap_get(cached_consensuses, flavor_name); } /** For authoritative directories: the current (v2) network status. */ @@ -1645,7 +1653,7 @@ should_generate_v2_networkstatus(void) #define TIME_KNOWN_TO_GUARANTEE_FAMILIAR (8*24*60*60) /** Similarly, every node with sufficient WFU is around enough to be a guard. */ -#define WFU_TO_GUARANTEE_GUARD (0.995) +#define WFU_TO_GUARANTEE_GUARD (0.98) /* Thresholds for server performance: set by * dirserv_compute_performance_thresholds, and used by @@ -1725,7 +1733,7 @@ dirserv_thinks_router_is_unreliable(time_t now, /** Return true iff <b>router</b> should be assigned the "HSDir" flag. * Right now this means it advertises support for it, it has a high - * uptime, and it's currently considered Running. + * uptime, it has a DirPort open, and it's currently considered Running. * * This function needs to be called after router-\>is_running has * been set. @@ -1735,7 +1743,11 @@ dirserv_thinks_router_is_hs_dir(routerinfo_t *router, time_t now) { long uptime = real_uptime(router, now); - return (router->wants_to_be_hs_dir && + /* XXX We shouldn't need to check dir_port, but we do because of + * bug 1693. In the future, once relays set wants_to_be_hs_dir + * correctly, we can revert to only checking dir_port if router's + * version is too old. */ + return (router->wants_to_be_hs_dir && router->dir_port && uptime > get_options()->MinUptimeHidServDirectoryV2 && router->is_running); } @@ -1790,7 +1802,8 @@ dirserv_compute_performance_thresholds(routerlist_t *rl) if (router_is_active(ri, now)) { const char *id = ri->cache_info.identity_digest; uint32_t bw; - ri->is_exit = exit_policy_is_general_exit(ri->exit_policy); + ri->is_exit = (!router_exit_policy_rejects_all(ri) && + exit_policy_is_general_exit(ri->exit_policy)); uptimes[n_active] = (uint32_t)real_uptime(ri, now); mtbfs[n_active] = rep_hist_get_stability(id, now); tks [n_active] = rep_hist_get_weighted_time_known(id, now); @@ -1897,16 +1910,20 @@ version_from_platform(const char *platform) * which has at least <b>buf_len</b> free characters. Do NUL-termination. * Use the same format as in network-status documents. If <b>version</b> is * non-NULL, add a "v" line for the platform. Return 0 on success, -1 on - * failure. If <b>first_line_only</b> is true, don't include any flags - * or version line. + * failure. + * + * The format argument has three possible values: + * NS_V2 - Output an entry suitable for a V2 NS opinion document + * NS_V3_CONSENSUS - Output the first portion of a V3 NS consensus entry + * NS_V3_CONSENSUS_MICRODESC - Output the first portion of a V3 microdesc + * consensus entry. + * NS_V3_VOTE - Output a complete V3 NS vote + * NS_CONTROL_PORT - Output a NS document for the control port */ int routerstatus_format_entry(char *buf, size_t buf_len, routerstatus_t *rs, const char *version, - int first_line_only, int v2_format) -/* XXX: first_line_only and v2_format should probably be be both - * replaced by a single purpose parameter. - */ + routerstatus_format_type_t format) { int r; struct in_addr in; @@ -1925,10 +1942,11 @@ routerstatus_format_entry(char *buf, size_t buf_len, tor_inet_ntoa(&in, ipaddr, sizeof(ipaddr)); r = tor_snprintf(buf, buf_len, - "r %s %s %s %s %s %d %d\n", + "r %s %s %s%s%s %s %d %d\n", rs->nickname, identity64, - digest64, + (format==NS_V3_CONSENSUS_MICRODESC)?"":digest64, + (format==NS_V3_CONSENSUS_MICRODESC)?"":" ", published, ipaddr, (int)rs->or_port, @@ -1937,7 +1955,12 @@ routerstatus_format_entry(char *buf, size_t buf_len, log_warn(LD_BUG, "Not enough space in buffer."); return -1; } - if (first_line_only) + + /* TODO: Maybe we want to pass in what we need to build the rest of + * this here, instead of in the caller. Then we could use the + * networkstatus_type_t values, with an additional control port value + * added -MP */ + if (format == NS_V3_CONSENSUS || format == NS_V3_CONSENSUS_MICRODESC) return 0; cp = buf + strlen(buf); @@ -1974,62 +1997,87 @@ routerstatus_format_entry(char *buf, size_t buf_len, cp += strlen(cp); } - if (!v2_format) { + if (format != NS_V2) { routerinfo_t* desc = router_get_by_digest(rs->identity_digest); + uint32_t bw; + + if (format != NS_CONTROL_PORT) { + /* Blow up more or less nicely if we didn't get anything or not the + * thing we expected. + */ + if (!desc) { + char id[HEX_DIGEST_LEN+1]; + char dd[HEX_DIGEST_LEN+1]; + + base16_encode(id, sizeof(id), rs->identity_digest, DIGEST_LEN); + base16_encode(dd, sizeof(dd), rs->descriptor_digest, DIGEST_LEN); + log_warn(LD_BUG, "Cannot get any descriptor for %s " + "(wanted descriptor %s).", + id, dd); + return -1; + }; + + /* This assert can fire for the control port, because + * it can request NS documents before all descriptors + * have been fetched. */ + if (memcmp(desc->cache_info.signed_descriptor_digest, + rs->descriptor_digest, + DIGEST_LEN)) { + char rl_d[HEX_DIGEST_LEN+1]; + char rs_d[HEX_DIGEST_LEN+1]; + char id[HEX_DIGEST_LEN+1]; + + base16_encode(rl_d, sizeof(rl_d), + desc->cache_info.signed_descriptor_digest, DIGEST_LEN); + base16_encode(rs_d, sizeof(rs_d), rs->descriptor_digest, DIGEST_LEN); + base16_encode(id, sizeof(id), rs->identity_digest, DIGEST_LEN); + log_err(LD_BUG, "descriptor digest in routerlist does not match " + "the one in routerstatus: %s vs %s " + "(router %s)\n", + rl_d, rs_d, id); + + tor_assert(!memcmp(desc->cache_info.signed_descriptor_digest, + rs->descriptor_digest, + DIGEST_LEN)); + }; + } - /* Blow up more or less nicely if we didn't get anything or not the - * thing we expected. - */ - if (!desc) { - char id[HEX_DIGEST_LEN+1]; - char dd[HEX_DIGEST_LEN+1]; - - base16_encode(id, sizeof(id), rs->identity_digest, DIGEST_LEN); - base16_encode(dd, sizeof(dd), rs->descriptor_digest, DIGEST_LEN); - log_warn(LD_BUG, "Cannot get any descriptor for %s " - "(wanted descriptor %s).", - id, dd); - return -1; - }; - if (memcmp(desc->cache_info.signed_descriptor_digest, - rs->descriptor_digest, - DIGEST_LEN)) { - char rl_d[HEX_DIGEST_LEN+1]; - char rs_d[HEX_DIGEST_LEN+1]; - char id[HEX_DIGEST_LEN+1]; - - base16_encode(rl_d, sizeof(rl_d), - desc->cache_info.signed_descriptor_digest, DIGEST_LEN); - base16_encode(rs_d, sizeof(rs_d), rs->descriptor_digest, DIGEST_LEN); - base16_encode(id, sizeof(id), rs->identity_digest, DIGEST_LEN); - log_err(LD_BUG, "descriptor digest in routerlist does not match " - "the one in routerstatus: %s vs %s " - "(router %s)\n", - rl_d, rs_d, id); - - tor_assert(!memcmp(desc->cache_info.signed_descriptor_digest, - rs->descriptor_digest, - DIGEST_LEN)); - }; - + if (format == NS_CONTROL_PORT && rs->has_bandwidth) { + bw = rs->bandwidth; + } else { + tor_assert(desc); + bw = router_get_advertised_bandwidth_capped(desc) / 1000; + } r = tor_snprintf(cp, buf_len - (cp-buf), - "w Bandwidth=%d\n", - router_get_advertised_bandwidth_capped(desc) / 1024); + "w Bandwidth=%d\n", bw); + if (r<0) { log_warn(LD_BUG, "Not enough space in buffer."); return -1; } cp += strlen(cp); + if (format == NS_V3_VOTE && rs->has_measured_bw) { + *--cp = '\0'; /* Kill "\n" */ + r = tor_snprintf(cp, buf_len - (cp-buf), + " Measured=%d\n", rs->measured_bw); + if (r<0) { + log_warn(LD_BUG, "Not enough space in buffer for weight line."); + return -1; + } + cp += strlen(cp); + } - summary = policy_summarize(desc->exit_policy); - r = tor_snprintf(cp, buf_len - (cp-buf), "p %s\n", summary); - if (r<0) { - log_warn(LD_BUG, "Not enough space in buffer."); + if (desc) { + summary = policy_summarize(desc->exit_policy); + r = tor_snprintf(cp, buf_len - (cp-buf), "p %s\n", summary); + if (r<0) { + log_warn(LD_BUG, "Not enough space in buffer."); + tor_free(summary); + return -1; + } + cp += strlen(cp); tor_free(summary); - return -1; } - cp += strlen(cp); - tor_free(summary); } return 0; @@ -2132,9 +2180,7 @@ get_possible_sybil_list(const smartlist_t *routers) /** Extract status information from <b>ri</b> and from other authority * functions and store it in <b>rs</b>>. If <b>naming</b>, consider setting - * the named flag in <b>rs</b>. If not <b>exits_can_be_guards</b>, never mark - * an exit as a guard. If <b>listbadexits</b>, consider setting the badexit - * flag. + * the named flag in <b>rs</b>. * * We assume that ri-\>is_running has already been set, e.g. by * dirserv_set_router_is_running(ri, now); @@ -2142,8 +2188,8 @@ get_possible_sybil_list(const smartlist_t *routers) void set_routerstatus_from_routerinfo(routerstatus_t *rs, routerinfo_t *ri, time_t now, - int naming, int exits_can_be_guards, - int listbadexits, int listbaddirs) + int naming, int listbadexits, + int listbaddirs) { int unstable_version = !tor_version_as_new_as(ri->platform,"0.1.1.16-rc-cvs"); @@ -2172,11 +2218,10 @@ set_routerstatus_from_routerinfo(routerstatus_t *rs, rs->is_valid = ri->is_valid; if (rs->is_fast && - (!rs->is_exit || exits_can_be_guards) && (router_get_advertised_bandwidth(ri) >= BANDWIDTH_TO_GUARANTEE_GUARD || router_get_advertised_bandwidth(ri) >= - (exits_can_be_guards ? guard_bandwidth_including_exits : - guard_bandwidth_excluding_exits))) { + MIN(guard_bandwidth_including_exits, + guard_bandwidth_excluding_exits))) { long tk = rep_hist_get_weighted_time_known( ri->cache_info.identity_digest, now); double wfu = rep_hist_get_weighted_fractional_uptime( @@ -2232,6 +2277,177 @@ router_clear_status_flags(routerinfo_t *router) router->is_bad_exit = router->is_bad_directory = 0; } +/** + * Helper function to parse out a line in the measured bandwidth file + * into a measured_bw_line_t output structure. Returns -1 on failure + * or 0 on success. + */ +int +measured_bw_line_parse(measured_bw_line_t *out, const char *orig_line) +{ + char *line = tor_strdup(orig_line); + char *cp = line; + int got_bw = 0; + int got_node_id = 0; + char *strtok_state; /* lame sauce d'jour */ + cp = tor_strtok_r(cp, " \t", &strtok_state); + + if (!cp) { + log_warn(LD_DIRSERV, "Invalid line in bandwidth file: %s", + escaped(orig_line)); + tor_free(line); + return -1; + } + + if (orig_line[strlen(orig_line)-1] != '\n') { + log_warn(LD_DIRSERV, "Incomplete line in bandwidth file: %s", + escaped(orig_line)); + tor_free(line); + return -1; + } + + do { + if (strcmpstart(cp, "bw=") == 0) { + int parse_ok = 0; + char *endptr; + if (got_bw) { + log_warn(LD_DIRSERV, "Double bw= in bandwidth file line: %s", + escaped(orig_line)); + tor_free(line); + return -1; + } + cp+=strlen("bw="); + + out->bw = tor_parse_long(cp, 0, 0, LONG_MAX, &parse_ok, &endptr); + if (!parse_ok || (*endptr && !TOR_ISSPACE(*endptr))) { + log_warn(LD_DIRSERV, "Invalid bandwidth in bandwidth file line: %s", + escaped(orig_line)); + tor_free(line); + return -1; + } + got_bw=1; + } else if (strcmpstart(cp, "node_id=$") == 0) { + if (got_node_id) { + log_warn(LD_DIRSERV, "Double node_id= in bandwidth file line: %s", + escaped(orig_line)); + tor_free(line); + return -1; + } + cp+=strlen("node_id=$"); + + if (strlen(cp) != HEX_DIGEST_LEN || + base16_decode(out->node_id, DIGEST_LEN, cp, HEX_DIGEST_LEN)) { + log_warn(LD_DIRSERV, "Invalid node_id in bandwidth file line: %s", + escaped(orig_line)); + tor_free(line); + return -1; + } + strncpy(out->node_hex, cp, sizeof(out->node_hex)); + got_node_id=1; + } + } while ((cp = tor_strtok_r(NULL, " \t", &strtok_state))); + + if (got_bw && got_node_id) { + tor_free(line); + return 0; + } else { + log_warn(LD_DIRSERV, "Incomplete line in bandwidth file: %s", + escaped(orig_line)); + tor_free(line); + return -1; + } +} + +/** + * Helper function to apply a parsed measurement line to a list + * of bandwidth statuses. Returns true if a line is found, + * false otherwise. + */ +int +measured_bw_line_apply(measured_bw_line_t *parsed_line, + smartlist_t *routerstatuses) +{ + routerstatus_t *rs = NULL; + if (!routerstatuses) + return 0; + + rs = smartlist_bsearch(routerstatuses, parsed_line->node_id, + compare_digest_to_routerstatus_entry); + + if (rs) { + rs->has_measured_bw = 1; + rs->measured_bw = (uint32_t)parsed_line->bw; + } else { + log_info(LD_DIRSERV, "Node ID %s not found in routerstatus list", + parsed_line->node_hex); + } + + return rs != NULL; +} + +/** + * Read the measured bandwidth file and apply it to the list of + * routerstatuses. Returns -1 on error, 0 otherwise. + */ +int +dirserv_read_measured_bandwidths(const char *from_file, + smartlist_t *routerstatuses) +{ + char line[256]; + FILE *fp = fopen(from_file, "r"); + int applied_lines = 0; + time_t file_time; + int ok; + if (fp == NULL) { + log_warn(LD_CONFIG, "Can't open bandwidth file at configured location: %s", + from_file); + return -1; + } + + if (!fgets(line, sizeof(line), fp) + || !strlen(line) || line[strlen(line)-1] != '\n') { + log_warn(LD_DIRSERV, "Long or truncated time in bandwidth file: %s", + escaped(line)); + fclose(fp); + return -1; + } + + line[strlen(line)-1] = '\0'; + file_time = tor_parse_ulong(line, 10, 0, ULONG_MAX, &ok, NULL); + if (!ok) { + log_warn(LD_DIRSERV, "Non-integer time in bandwidth file: %s", + escaped(line)); + fclose(fp); + return -1; + } + + if ((time(NULL) - file_time) > MAX_MEASUREMENT_AGE) { + log_warn(LD_DIRSERV, "Bandwidth measurement file stale. Age: %u", + (unsigned)(time(NULL) - file_time)); + fclose(fp); + return -1; + } + + if (routerstatuses) + smartlist_sort(routerstatuses, compare_routerstatus_entries); + + while (!feof(fp)) { + measured_bw_line_t parsed_line; + if (fgets(line, sizeof(line), fp) && strlen(line)) { + if (measured_bw_line_parse(&parsed_line, line) != -1) { + if (measured_bw_line_apply(&parsed_line, routerstatuses) > 0) + applied_lines++; + } + } + } + + fclose(fp); + log_info(LD_DIRSERV, + "Bandwidth measurement file successfully read. " + "Applied %d measurements.", applied_lines); + return 0; +} + /** Return a new networkstatus_t* containing our current opinion. (For v3 * authorities) */ networkstatus_t * @@ -2249,22 +2465,18 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_env_t *private_key, int naming = options->NamingAuthoritativeDir; int listbadexits = options->AuthDirListBadExits; int listbaddirs = options->AuthDirListBadDirs; - int exits_can_be_guards; routerlist_t *rl = router_get_routerlist(); time_t now = time(NULL); time_t cutoff = now - ROUTER_MAX_AGE_TO_PUBLISH; networkstatus_voter_info_t *voter = NULL; vote_timing_t timing; digestmap_t *omit_as_sybil = NULL; - int vote_on_reachability = 1; + const int vote_on_reachability = running_long_enough_to_decide_unreachable(); + smartlist_t *microdescriptors = NULL; tor_assert(private_key); tor_assert(cert); - if (now - time_of_process_start < - options->TestingAuthDirTimeToLearnReachability) - vote_on_reachability = 0; - if (resolve_my_address(LOG_WARN, options, &addr, &hostname)<0) { log_warn(LD_NET, "Couldn't resolve my hostname"); return NULL; @@ -2299,27 +2511,24 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_env_t *private_key, dirserv_compute_performance_thresholds(rl); - /* XXXX We should take steps to keep this from oscillating if - * total_exit_bandwidth is close to total_bandwidth/3. */ - exits_can_be_guards = total_exit_bandwidth >= (total_bandwidth / 3); - routers = smartlist_create(); smartlist_add_all(routers, rl->routers); routers_sort_by_identity(routers); omit_as_sybil = get_possible_sybil_list(routers); routerstatuses = smartlist_create(); + microdescriptors = smartlist_create(); - SMARTLIST_FOREACH(routers, routerinfo_t *, ri, { + SMARTLIST_FOREACH_BEGIN(routers, routerinfo_t *, ri) { if (ri->cache_info.published_on >= cutoff) { routerstatus_t *rs; vote_routerstatus_t *vrs; + microdesc_t *md; vrs = tor_malloc_zero(sizeof(vote_routerstatus_t)); rs = &vrs->status; set_routerstatus_from_routerinfo(rs, ri, now, - naming, exits_can_be_guards, - listbadexits, listbaddirs); + naming, listbadexits, listbaddirs); if (digestmap_get(omit_as_sybil, ri->cache_info.identity_digest)) clear_status_flags_on_sybil(rs); @@ -2328,12 +2537,39 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_env_t *private_key, rs->is_running = 0; vrs->version = version_from_platform(ri->platform); + md = dirvote_create_microdescriptor(ri); + if (md) { + char buf[128]; + vote_microdesc_hash_t *h; + dirvote_format_microdesc_vote_line(buf, sizeof(buf), md); + h = tor_malloc(sizeof(vote_microdesc_hash_t)); + h->microdesc_hash_line = tor_strdup(buf); + h->next = NULL; + vrs->microdesc = h; + md->last_listed = now; + smartlist_add(microdescriptors, md); + } + smartlist_add(routerstatuses, vrs); } - }); + } SMARTLIST_FOREACH_END(ri); + + { + smartlist_t *added = + microdescs_add_list_to_cache(get_microdesc_cache(), + microdescriptors, SAVED_NOWHERE, 0); + smartlist_free(added); + smartlist_free(microdescriptors); + } + smartlist_free(routers); digestmap_free(omit_as_sybil, NULL); + if (options->V3BandwidthsFile) { + dirserv_read_measured_bandwidths(options->V3BandwidthsFile, + routerstatuses); + } + v3_out = tor_malloc_zero(sizeof(networkstatus_t)); v3_out->type = NS_TYPE_VOTE; @@ -2383,15 +2619,22 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_env_t *private_key, } smartlist_sort_strings(v3_out->known_flags); + if (options->ConsensusParams) { + v3_out->net_params = smartlist_create(); + smartlist_split_string(v3_out->net_params, + options->ConsensusParams, NULL, 0, 0); + smartlist_sort_strings(v3_out->net_params); + } + voter = tor_malloc_zero(sizeof(networkstatus_voter_info_t)); voter->nickname = tor_strdup(options->Nickname); memcpy(voter->identity_digest, identity_digest, DIGEST_LEN); + voter->sigs = smartlist_create(); voter->address = hostname; voter->addr = addr; voter->dir_port = options->DirPort; voter->or_port = options->ORPort; voter->contact = tor_strdup(contact); - memcpy(voter->signing_key_digest, signing_key_digest, DIGEST_LEN); if (options->V3AuthUseLegacyKey) { authority_cert_t *c = get_my_v3_legacy_cert(); if (c) { @@ -2435,7 +2678,6 @@ generate_v2_networkstatus_opinion(void) int versioning = options->VersioningAuthoritativeDir; int listbaddirs = options->AuthDirListBadDirs; int listbadexits = options->AuthDirListBadExits; - int exits_can_be_guards; const char *contact; char *version_lines = NULL; smartlist_t *routers = NULL; @@ -2515,10 +2757,6 @@ generate_v2_networkstatus_opinion(void) dirserv_compute_performance_thresholds(rl); - /* XXXX We should take steps to keep this from oscillating if - * total_exit_bandwidth is close to total_bandwidth/3. */ - exits_can_be_guards = total_exit_bandwidth >= (total_bandwidth / 3); - routers = smartlist_create(); smartlist_add_all(routers, rl->routers); routers_sort_by_identity(routers); @@ -2531,13 +2769,12 @@ generate_v2_networkstatus_opinion(void) char *version = version_from_platform(ri->platform); set_routerstatus_from_routerinfo(&rs, ri, now, - naming, exits_can_be_guards, - listbadexits, listbaddirs); + naming, listbadexits, listbaddirs); if (digestmap_get(omit_as_sybil, ri->cache_info.identity_digest)) clear_status_flags_on_sybil(&rs); - if (routerstatus_format_entry(outp, endp-outp, &rs, version, 0, 1)) { + if (routerstatus_format_entry(outp, endp-outp, &rs, version, NS_V2)) { log_warn(LD_BUG, "Unable to print router status."); tor_free(version); goto done; @@ -2559,7 +2796,8 @@ generate_v2_networkstatus_opinion(void) outp += strlen(outp); note_crypto_pk_op(SIGN_DIR); - if (router_append_dirobj_signature(outp,endp-outp,digest,private_key)<0) { + if (router_append_dirobj_signature(outp,endp-outp,digest,DIGEST_LEN, + private_key)<0) { log_warn(LD_BUG, "Unable to sign router status."); goto done; } @@ -2591,10 +2829,8 @@ generate_v2_networkstatus_opinion(void) tor_free(status); tor_free(hostname); tor_free(identity_pkey); - if (routers) - smartlist_free(routers); - if (omit_as_sybil) - digestmap_free(omit_as_sybil, NULL); + smartlist_free(routers); + digestmap_free(omit_as_sybil, NULL); return r; } @@ -2642,7 +2878,8 @@ dirserv_get_networkstatus_v2_fingerprints(smartlist_t *result, log_info(LD_DIRSERV, "Client requested 'all' network status objects; we have none."); } else if (!strcmpstart(key, "fp/")) { - dir_split_resource_into_fingerprints(key+3, result, NULL, 1, 1); + dir_split_resource_into_fingerprints(key+3, result, NULL, + DSR_HEX|DSR_SORT_UNIQ); } } @@ -2707,10 +2944,12 @@ dirserv_get_routerdesc_fingerprints(smartlist_t *fps_out, const char *key, } else if (!strcmpstart(key, "d/")) { by_id = 0; key += strlen("d/"); - dir_split_resource_into_fingerprints(key, fps_out, NULL, 1, 1); + dir_split_resource_into_fingerprints(key, fps_out, NULL, + DSR_HEX|DSR_SORT_UNIQ); } else if (!strcmpstart(key, "fp/")) { key += strlen("fp/"); - dir_split_resource_into_fingerprints(key, fps_out, NULL, 1, 1); + dir_split_resource_into_fingerprints(key, fps_out, NULL, + DSR_HEX|DSR_SORT_UNIQ); } else { *msg = "Key not recognized"; return -1; @@ -2775,7 +3014,8 @@ dirserv_get_routerdescs(smartlist_t *descs_out, const char *key, } else if (!strcmpstart(key, "/tor/server/d/")) { smartlist_t *digests = smartlist_create(); key += strlen("/tor/server/d/"); - dir_split_resource_into_fingerprints(key, digests, NULL, 1, 1); + dir_split_resource_into_fingerprints(key, digests, NULL, + DSR_HEX|DSR_SORT_UNIQ); SMARTLIST_FOREACH(digests, const char *, d, { signed_descriptor_t *sd = router_get_by_descriptor_digest(d); @@ -2788,7 +3028,8 @@ dirserv_get_routerdescs(smartlist_t *descs_out, const char *key, smartlist_t *digests = smartlist_create(); time_t cutoff = time(NULL) - ROUTER_MAX_AGE_TO_PUBLISH; key += strlen("/tor/server/fp/"); - dir_split_resource_into_fingerprints(key, digests, NULL, 1, 1); + dir_split_resource_into_fingerprints(key, digests, NULL, + DSR_HEX|DSR_SORT_UNIQ); SMARTLIST_FOREACH(digests, const char *, d, { if (router_digest_is_me(d)) { @@ -2857,8 +3098,24 @@ dirserv_orconn_tls_done(const char *address, * skip testing. */ } -/** Auth dir server only: if <b>try_all</b> is 1, launch connections to - * all known routers; else we want to load balance such that we only +/** Helper function for dirserv_test_reachability(). Start a TLS + * connection to <b>router</b>, and annotate it with when we started + * the test. */ +void +dirserv_single_reachability_test(time_t now, routerinfo_t *router) +{ + tor_addr_t router_addr; + log_debug(LD_OR,"Testing reachability of %s at %s:%u.", + router->nickname, router->address, router->or_port); + /* Remember when we started trying to determine reachability */ + if (!router->testing_since) + router->testing_since = now; + tor_addr_from_ipv4h(&router_addr, router->addr); + connection_or_connect(&router_addr, router->or_port, + router->cache_info.identity_digest); +} + +/** Auth dir server only: load balance such that we only * try a few connections per call. * * The load balancing is such that if we get called once every ten @@ -2866,7 +3123,7 @@ dirserv_orconn_tls_done(const char *address, * bit over 20 minutes). */ void -dirserv_test_reachability(time_t now, int try_all) +dirserv_test_reachability(time_t now) { /* XXX decide what to do here; see or-talk thread "purging old router * information, revocation." -NM @@ -2883,38 +3140,33 @@ dirserv_test_reachability(time_t now, int try_all) SMARTLIST_FOREACH_BEGIN(rl->routers, routerinfo_t *, router) { const char *id_digest = router->cache_info.identity_digest; - tor_addr_t router_addr; if (router_is_me(router)) continue; if (bridge_auth && router->purpose != ROUTER_PURPOSE_BRIDGE) continue; /* bridge authorities only test reachability on bridges */ // if (router->cache_info.published_on > cutoff) // continue; - if (try_all || (((uint8_t)id_digest[0]) % 128) == ctr) { - log_debug(LD_OR,"Testing reachability of %s at %s:%u.", - router->nickname, router->address, router->or_port); - /* Remember when we started trying to determine reachability */ - if (!router->testing_since) - router->testing_since = now; - tor_addr_from_ipv4h(&router_addr, router->addr); - connection_or_connect(&router_addr, router->or_port, id_digest); + if ((((uint8_t)id_digest[0]) % 128) == ctr) { + dirserv_single_reachability_test(now, router); } } SMARTLIST_FOREACH_END(router); - if (!try_all) /* increment ctr */ - ctr = (ctr + 1) % 128; + ctr = (ctr + 1) % 128; /* increment ctr */ } -/** Given a fingerprint <b>fp</b> which is either set if we're looking - * for a v2 status, or zeroes if we're looking for a v3 status, return - * a pointer to the appropriate cached dir object, or NULL if there isn't - * one available. */ +/** Given a fingerprint <b>fp</b> which is either set if we're looking for a + * v2 status, or zeroes if we're looking for a v3 status, or a NUL-padded + * flavor name if we want a flavored v3 status, return a pointer to the + * appropriate cached dir object, or NULL if there isn't one available. */ static cached_dir_t * lookup_cached_dir_by_fp(const char *fp) { cached_dir_t *d = NULL; - if (tor_digest_is_zero(fp) && cached_v3_networkstatus) - d = cached_v3_networkstatus; - else if (router_digest_is_me(fp) && the_v2_networkstatus) + if (tor_digest_is_zero(fp) && cached_consensuses) + d = strmap_get(cached_consensuses, "ns"); + else if (memchr(fp, '\0', DIGEST_LEN) && cached_consensuses && + (d = strmap_get(cached_consensuses, fp))) { + /* this here interface is a nasty hack XXXX022 */; + } else if (router_digest_is_me(fp) && the_v2_networkstatus) d = the_v2_networkstatus; else if (cached_v2_networkstatus) d = digestmap_get(cached_v2_networkstatus, fp); @@ -3000,6 +3252,18 @@ dirserv_have_any_serverdesc(smartlist_t *fps, int spool_src) return 0; } +/** Return true iff any of the 256-bit elements in <b>fps</b> is the digest of + * a microdescriptor we have. */ +int +dirserv_have_any_microdesc(const smartlist_t *fps) +{ + microdesc_cache_t *cache = get_microdesc_cache(); + SMARTLIST_FOREACH(fps, const char *, fp, + if (microdesc_cache_lookup_by_digest256(cache, fp)) + return 1); + return 0; +} + /** Return an approximate estimate of the number of bytes that will * be needed to transmit the server descriptors (if is_serverdescs -- * they can be either d/ or fp/ queries) or networkstatus objects (if @@ -3031,6 +3295,17 @@ dirserv_estimate_data_size(smartlist_t *fps, int is_serverdescs, return result; } +/** Given a list of microdescriptor hashes, guess how many bytes will be + * needed to transmit them, and return the guess. */ +size_t +dirserv_estimate_microdesc_size(const smartlist_t *fps, int compressed) +{ + size_t result = smartlist_len(fps) * microdesc_average_size(NULL); + if (compressed) + result /= 2; + return result; +} + /** When we're spooling data onto our outbuf, add more whenever we dip * below this threshold. */ #define DIRSERV_BUFFER_MIN 16384 @@ -3094,6 +3369,8 @@ connection_dirserv_add_servers_to_outbuf(dir_connection_t *conn) #endif body = signed_descriptor_get_body(sd); if (conn->zlib_state) { + /* XXXX022 This 'last' business should actually happen on the last + * routerinfo, not on the last fingerprint. */ int last = ! smartlist_len(conn->fingerprint_stack); connection_write_to_buf_zlib(body, sd->signed_descriptor_len, conn, last); @@ -3117,6 +3394,44 @@ connection_dirserv_add_servers_to_outbuf(dir_connection_t *conn) return 0; } +/** Spooling helper: called when we're sending a bunch of microdescriptors, + * and the outbuf has become too empty. Pulls some entries from + * fingerprint_stack, and writes the corresponding microdescs onto outbuf. If + * we run out of entries, flushes the zlib state and sets the spool source to + * NONE. Returns 0 on success, negative on failure. + */ +static int +connection_dirserv_add_microdescs_to_outbuf(dir_connection_t *conn) +{ + microdesc_cache_t *cache = get_microdesc_cache(); + while (smartlist_len(conn->fingerprint_stack) && + buf_datalen(conn->_base.outbuf) < DIRSERV_BUFFER_MIN) { + char *fp256 = smartlist_pop_last(conn->fingerprint_stack); + microdesc_t *md = microdesc_cache_lookup_by_digest256(cache, fp256); + tor_free(fp256); + if (!md) + continue; + if (conn->zlib_state) { + /* XXXX022 This 'last' business should actually happen on the last + * routerinfo, not on the last fingerprint. */ + int last = !smartlist_len(conn->fingerprint_stack); + connection_write_to_buf_zlib(md->body, md->bodylen, conn, last); + if (last) { + tor_zlib_free(conn->zlib_state); + conn->zlib_state = NULL; + } + } else { + connection_write_to_buf(md->body, md->bodylen, TO_CONN(conn)); + } + } + if (!smartlist_len(conn->fingerprint_stack)) { + conn->dir_spool_src = DIR_SPOOL_NONE; + smartlist_free(conn->fingerprint_stack); + conn->fingerprint_stack = NULL; + } + return 0; +} + /** Spooling helper: Called when we're sending a directory or networkstatus, * and the outbuf has become too empty. Pulls some bytes from * <b>conn</b>-\>cached_dir-\>dir_z, uncompresses them if appropriate, and @@ -3199,8 +3514,7 @@ connection_dirserv_add_networkstatus_bytes_to_outbuf(dir_connection_t *conn) } } else { connection_dirserv_finish_spooling(conn); - if (conn->fingerprint_stack) - smartlist_free(conn->fingerprint_stack); + smartlist_free(conn->fingerprint_stack); conn->fingerprint_stack = NULL; return 0; } @@ -3224,6 +3538,8 @@ connection_dirserv_flushed_some(dir_connection_t *conn) case DIR_SPOOL_SERVER_BY_DIGEST: case DIR_SPOOL_SERVER_BY_FP: return connection_dirserv_add_servers_to_outbuf(conn); + case DIR_SPOOL_MICRODESC: + return connection_dirserv_add_microdescs_to_outbuf(conn); case DIR_SPOOL_CACHED_DIR: return connection_dirserv_add_dir_bytes_to_outbuf(conn); case DIR_SPOOL_NETWORKSTATUS: @@ -3245,10 +3561,10 @@ dirserv_free_all(void) cached_dir_decref(the_v2_networkstatus); cached_dir_decref(cached_directory); clear_cached_dir(&cached_runningrouters); - if (cached_v2_networkstatus) { - digestmap_free(cached_v2_networkstatus, _free_cached_dir); - cached_v2_networkstatus = NULL; - } - cached_dir_decref(cached_v3_networkstatus); + + digestmap_free(cached_v2_networkstatus, _free_cached_dir); + cached_v2_networkstatus = NULL; + strmap_free(cached_consensuses, _free_cached_dir); + cached_consensuses = NULL; } diff --git a/src/or/dirserv.h b/src/or/dirserv.h new file mode 100644 index 0000000000..fc5a5549c5 --- /dev/null +++ b/src/or/dirserv.h @@ -0,0 +1,133 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2010, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file dirserv.h + * \brief Header file for dirserv.c. + **/ + +#ifndef _TOR_DIRSERV_H +#define _TOR_DIRSERV_H + +/** Maximum length of an exit policy summary. */ +#define MAX_EXITPOLICY_SUMMARY_LEN 1000 + +/** Maximum allowable length of a version line in a networkstatus. */ +#define MAX_V_LINE_LEN 128 +/** Length of "r Authority BadDirectory BadExit Exit Fast Guard HSDir Named + * Running Stable Unnamed V2Dir Valid\n". */ +#define MAX_FLAG_LINE_LEN 96 +/** Length of "w" line for weighting. Currently at most + * "w Bandwidth=<uint32t> Measured=<uint32t>\n" */ +#define MAX_WEIGHT_LINE_LEN (12+10+10+10+1) +/** Maximum length of an exit policy summary line. */ +#define MAX_POLICY_LINE_LEN (3+MAX_EXITPOLICY_SUMMARY_LEN) +/** Amount of space to allocate for each entry: r, s, and v lines. */ +#define RS_ENTRY_LEN \ + ( /* first line */ \ + MAX_NICKNAME_LEN+BASE64_DIGEST_LEN*2+ISO_TIME_LEN+INET_NTOA_BUF_LEN+ \ + 5*2 /* ports */ + 10 /* punctuation */ + \ + /* second line */ \ + MAX_FLAG_LINE_LEN + \ + /* weight line */ \ + MAX_WEIGHT_LINE_LEN + \ + /* p line. */ \ + MAX_POLICY_LINE_LEN + \ + /* v line. */ \ + MAX_V_LINE_LEN \ + ) + +#define UNNAMED_ROUTER_NICKNAME "Unnamed" + +int connection_dirserv_flushed_some(dir_connection_t *conn); + +int dirserv_add_own_fingerprint(const char *nickname, crypto_pk_env_t *pk); +int dirserv_load_fingerprint_file(void); +void dirserv_free_fingerprint_list(void); +const char *dirserv_get_nickname_by_digest(const char *digest); +enum was_router_added_t dirserv_add_multiple_descriptors( + const char *desc, uint8_t purpose, + const char *source, + const char **msg); +enum was_router_added_t dirserv_add_descriptor(routerinfo_t *ri, + const char **msg, + const char *source); +void dirserv_free_descriptors(void); +void dirserv_set_router_is_running(routerinfo_t *router, time_t now); +int list_server_status_v1(smartlist_t *routers, char **router_status_out, + int for_controller); +int dirserv_dump_directory_to_string(char **dir_out, + crypto_pk_env_t *private_key); + +int directory_fetches_from_authorities(or_options_t *options); +int directory_fetches_dir_info_early(or_options_t *options); +int directory_fetches_dir_info_later(or_options_t *options); +int directory_caches_v2_dir_info(or_options_t *options); +#define directory_caches_v1_dir_info(o) directory_caches_v2_dir_info(o) +int directory_caches_dir_info(or_options_t *options); +int directory_permits_begindir_requests(or_options_t *options); +int directory_permits_controller_requests(or_options_t *options); +int directory_too_idle_to_fetch_descriptors(or_options_t *options, time_t now); + +void directory_set_dirty(void); +cached_dir_t *dirserv_get_directory(void); +cached_dir_t *dirserv_get_runningrouters(void); +cached_dir_t *dirserv_get_consensus(const char *flavor_name); +void dirserv_set_cached_directory(const char *directory, time_t when, + int is_running_routers); +void dirserv_set_cached_networkstatus_v2(const char *directory, + const char *identity, + time_t published); +void dirserv_set_cached_consensus_networkstatus(const char *consensus, + const char *flavor_name, + const digests_t *digests, + time_t published); +void dirserv_clear_old_networkstatuses(time_t cutoff); +void dirserv_clear_old_v1_info(time_t now); +void dirserv_get_networkstatus_v2(smartlist_t *result, const char *key); +void dirserv_get_networkstatus_v2_fingerprints(smartlist_t *result, + const char *key); +int dirserv_get_routerdesc_fingerprints(smartlist_t *fps_out, const char *key, + const char **msg, + int for_unencrypted_conn, + int is_extrainfo); +int dirserv_get_routerdescs(smartlist_t *descs_out, const char *key, + const char **msg); +void dirserv_orconn_tls_done(const char *address, + uint16_t or_port, + const char *digest_rcvd, + int as_advertised); +void dirserv_single_reachability_test(time_t now, routerinfo_t *router); +void dirserv_test_reachability(time_t now); +int authdir_wants_to_reject_router(routerinfo_t *ri, const char **msg, + int complain); +int dirserv_would_reject_router(routerstatus_t *rs); +int dirserv_remove_old_statuses(smartlist_t *fps, time_t cutoff); +int dirserv_have_any_serverdesc(smartlist_t *fps, int spool_src); +int dirserv_have_any_microdesc(const smartlist_t *fps); +size_t dirserv_estimate_data_size(smartlist_t *fps, int is_serverdescs, + int compressed); +size_t dirserv_estimate_microdesc_size(const smartlist_t *fps, int compressed); + +int routerstatus_format_entry(char *buf, size_t buf_len, + routerstatus_t *rs, const char *platform, + routerstatus_format_type_t format); +void dirserv_free_all(void); +void cached_dir_decref(cached_dir_t *d); +cached_dir_t *new_cached_dir(char *s, time_t published); + +#ifdef DIRSERV_PRIVATE +int measured_bw_line_parse(measured_bw_line_t *out, const char *line); + +int measured_bw_line_apply(measured_bw_line_t *parsed_line, + smartlist_t *routerstatuses); +#endif + +int dirserv_read_measured_bandwidths(const char *from_file, + smartlist_t *routerstatuses); + +#endif + diff --git a/src/or/dirvote.c b/src/or/dirvote.c index 64e07cc0a2..fd4d742ccb 100644 --- a/src/or/dirvote.c +++ b/src/or/dirvote.c @@ -5,27 +5,72 @@ #define DIRVOTE_PRIVATE #include "or.h" +#include "config.h" +#include "directory.h" +#include "dirserv.h" +#include "dirvote.h" +#include "microdesc.h" +#include "networkstatus.h" +#include "policies.h" +#include "rephist.h" +#include "router.h" +#include "routerlist.h" +#include "routerparse.h" /** * \file dirvote.c * \brief Functions to compute directory consensus, and schedule voting. **/ -static int dirvote_add_signatures_to_pending_consensus( +/** A consensus that we have built and are appending signatures to. Once it's + * time to publish it, it will become an active consensus if it accumulates + * enough signatures. */ +typedef struct pending_consensus_t { + /** The body of the consensus that we're currently building. Once we + * have it built, it goes into dirserv.c */ + char *body; + /** The parsed in-progress consensus document. */ + networkstatus_t *consensus; +} pending_consensus_t; + +static int dirvote_add_signatures_to_all_pending_consensuses( const char *detached_signatures_body, const char **msg_out); +static int dirvote_add_signatures_to_pending_consensus( + pending_consensus_t *pc, + ns_detached_signatures_t *sigs, + const char **msg_out); static char *list_v3_auth_ids(void); static void dirvote_fetch_missing_votes(void); static void dirvote_fetch_missing_signatures(void); static int dirvote_perform_vote(void); static void dirvote_clear_votes(int all_votes); -static int dirvote_compute_consensus(void); +static int dirvote_compute_consensuses(void); static int dirvote_publish_consensus(void); +static char *make_consensus_method_list(int low, int high, const char *sep); + +/** The highest consensus method that we currently support. */ +#define MAX_SUPPORTED_CONSENSUS_METHOD 9 + +/** Lowest consensus method that contains a 'directory-footer' marker */ +#define MIN_METHOD_FOR_FOOTER 9 + +/** Lowest consensus method that contains bandwidth weights */ +#define MIN_METHOD_FOR_BW_WEIGHTS 9 + +/** Lowest consensus method that contains consensus params */ +#define MIN_METHOD_FOR_PARAMS 7 + +/** Lowest consensus method that generates microdescriptors */ +#define MIN_METHOD_FOR_MICRODESC 8 /* ===== * Voting * =====*/ +/* Overestimated. */ +#define MICRODESC_LINE_LEN 80 + /** Return a new string containing the string representation of the vote in * <b>v3_ns</b>, signed with our v3 signing key <b>private_signing_key</b>. * For v3 authorities. */ @@ -44,6 +89,7 @@ format_networkstatus_vote(crypto_pk_env_t *private_signing_key, uint32_t addr; routerlist_t *rl = router_get_routerlist(); char *version_lines = NULL; + int r; networkstatus_voter_info_t *voter; tor_assert(private_signing_key); @@ -70,20 +116,30 @@ format_networkstatus_vote(crypto_pk_env_t *private_signing_key, version_lines = tor_malloc(v_len); cp = version_lines; if (client_versions) { - tor_snprintf(cp, v_len-(cp-version_lines), + r = tor_snprintf(cp, v_len-(cp-version_lines), "client-versions %s\n", client_versions); + if (r < 0) { + log_err(LD_BUG, "Insufficient memory for client-versions line"); + tor_assert(0); + } cp += strlen(cp); } - if (server_versions) - tor_snprintf(cp, v_len-(cp-version_lines), + if (server_versions) { + r = tor_snprintf(cp, v_len-(cp-version_lines), "server-versions %s\n", server_versions); + if (r < 0) { + log_err(LD_BUG, "Insufficient memory for server-versions line"); + tor_assert(0); + } + } } else { version_lines = tor_strdup(""); } len = 8192; len += strlen(version_lines); - len += (RS_ENTRY_LEN)*smartlist_len(rl->routers); + len += (RS_ENTRY_LEN+MICRODESC_LINE_LEN)*smartlist_len(rl->routers); + len += strlen("\ndirectory-footer\n"); len += v3_ns->cert->cache_info.signed_descriptor_len; status = tor_malloc(len); @@ -93,17 +149,25 @@ format_networkstatus_vote(crypto_pk_env_t *private_signing_key, char fu[ISO_TIME_LEN+1]; char vu[ISO_TIME_LEN+1]; char *flags = smartlist_join_strings(v3_ns->known_flags, " ", 0, NULL); + char *params; authority_cert_t *cert = v3_ns->cert; + char *methods = + make_consensus_method_list(1, MAX_SUPPORTED_CONSENSUS_METHOD, " "); format_iso_time(published, v3_ns->published); format_iso_time(va, v3_ns->valid_after); format_iso_time(fu, v3_ns->fresh_until); format_iso_time(vu, v3_ns->valid_until); + if (v3_ns->net_params) + params = smartlist_join_strings(v3_ns->net_params, " ", 0, NULL); + else + params = tor_strdup(""); + tor_assert(cert); - tor_snprintf(status, len, + r = tor_snprintf(status, len, "network-status-version 3\n" "vote-status %s\n" - "consensus-methods 1 2 3 4 5\n" + "consensus-methods %s\n" "published %s\n" "valid-after %s\n" "fresh-until %s\n" @@ -111,24 +175,38 @@ format_networkstatus_vote(crypto_pk_env_t *private_signing_key, "voting-delay %d %d\n" "%s" /* versions */ "known-flags %s\n" + "params %s\n" "dir-source %s %s %s %s %d %d\n" "contact %s\n", v3_ns->type == NS_TYPE_VOTE ? "vote" : "opinion", + methods, published, va, fu, vu, v3_ns->vote_seconds, v3_ns->dist_seconds, version_lines, flags, + params, voter->nickname, fingerprint, voter->address, - ipaddr, voter->dir_port, voter->or_port, voter->contact); + ipaddr, voter->dir_port, voter->or_port, voter->contact); + + if (r < 0) { + log_err(LD_BUG, "Insufficient memory for network status line"); + tor_assert(0); + } + tor_free(params); tor_free(flags); + tor_free(methods); outp = status + strlen(status); endp = status + len; if (!tor_digest_is_zero(voter->legacy_id_digest)) { char fpbuf[HEX_DIGEST_LEN+1]; base16_encode(fpbuf, sizeof(fpbuf), voter->legacy_id_digest, DIGEST_LEN); - tor_snprintf(outp, endp-outp, "legacy-dir-key %s\n", fpbuf); + r = tor_snprintf(outp, endp-outp, "legacy-dir-key %s\n", fpbuf); + if (r < 0) { + log_err(LD_BUG, "Insufficient memory for legacy-dir-key line"); + tor_assert(0); + } outp += strlen(outp); } @@ -139,15 +217,32 @@ format_networkstatus_vote(crypto_pk_env_t *private_signing_key, outp += cert->cache_info.signed_descriptor_len; } - SMARTLIST_FOREACH(v3_ns->routerstatus_list, vote_routerstatus_t *, vrs, - { + SMARTLIST_FOREACH_BEGIN(v3_ns->routerstatus_list, vote_routerstatus_t *, + vrs) { + vote_microdesc_hash_t *h; if (routerstatus_format_entry(outp, endp-outp, &vrs->status, - vrs->version, 0, 0) < 0) { + vrs->version, NS_V3_VOTE) < 0) { log_warn(LD_BUG, "Unable to print router status."); goto err; } outp += strlen(outp); - }); + + for (h = vrs->microdesc; h; h = h->next) { + size_t mlen = strlen(h->microdesc_hash_line); + if (outp+mlen >= endp) { + log_warn(LD_BUG, "Can't fit microdesc line in vote."); + } + memcpy(outp, h->microdesc_hash_line, mlen+1); + outp += strlen(outp); + } + } SMARTLIST_FOREACH_END(vrs); + + r = tor_snprintf(outp, endp-outp, "directory-footer\n"); + if (r < 0) { + log_err(LD_BUG, "Insufficient memory for directory-footer line"); + tor_assert(0); + } + outp += strlen(outp); { char signing_key_fingerprint[FINGERPRINT_LEN+1]; @@ -170,10 +265,10 @@ format_networkstatus_vote(crypto_pk_env_t *private_signing_key, outp += strlen(outp); } - if (router_get_networkstatus_v3_hash(status, digest)<0) + if (router_get_networkstatus_v3_hash(status, digest, DIGEST_SHA1)<0) goto err; note_crypto_pk_op(SIGN_DIR); - if (router_append_dirobj_signature(outp,endp-outp,digest, + if (router_append_dirobj_signature(outp,endp-outp,digest, DIGEST_LEN, private_signing_key)<0) { log_warn(LD_BUG, "Unable to sign networkstatus vote."); goto err; @@ -216,6 +311,20 @@ get_voter(const networkstatus_t *vote) return smartlist_get(vote->voters, 0); } +/** Return the signature made by <b>voter</b> using the algorithm + * <b>alg</b>, or NULL if none is found. */ +document_signature_t * +voter_get_sig_by_algorithm(const networkstatus_voter_info_t *voter, + digest_algorithm_t alg) +{ + if (!voter->sigs) + return NULL; + SMARTLIST_FOREACH(voter->sigs, document_signature_t *, sig, + if (sig->alg == alg) + return sig); + return NULL; +} + /** Temporary structure used in constructing a list of dir-source entries * for a consensus. One of these is generated for every vote, and one more * for every legacy key in each vote. */ @@ -275,34 +384,8 @@ get_frequent_members(smartlist_t *out, smartlist_t *in, int min) /** Given a sorted list of strings <b>lst</b>, return the member that appears * most. Break ties in favor of later-occurring members. */ -static const char * -get_most_frequent_member(smartlist_t *lst) -{ - const char *most_frequent = NULL; - int most_frequent_count = 0; - - const char *cur = NULL; - int count = 0; - - SMARTLIST_FOREACH(lst, const char *, s, - { - if (cur && !strcmp(s, cur)) { - ++count; - } else { - if (count >= most_frequent_count) { - most_frequent = cur; - most_frequent_count = count; - } - cur = s; - count = 1; - } - }); - if (count >= most_frequent_count) { - most_frequent = cur; - most_frequent_count = count; - } - return most_frequent; -} +#define get_most_frequent_member(lst) \ + smartlist_get_most_frequent_string(lst) /** Return 0 if and only if <b>a</b> and <b>b</b> are routerstatuses * that come from the same routerinfo, with the same derived elements. @@ -344,7 +427,8 @@ _compare_vote_rs(const void **_a, const void **_b) * in favor of smaller descriptor digest. */ static vote_routerstatus_t * -compute_routerstatus_consensus(smartlist_t *votes) +compute_routerstatus_consensus(smartlist_t *votes, int consensus_method, + char *microdesc_digest256_out) { vote_routerstatus_t *most = NULL, *cur = NULL; int most_n = 0, cur_n = 0; @@ -380,18 +464,45 @@ compute_routerstatus_consensus(smartlist_t *votes) } tor_assert(most); + + if (consensus_method >= MIN_METHOD_FOR_MICRODESC && + microdesc_digest256_out) { + smartlist_t *digests = smartlist_create(); + const char *best_microdesc_digest; + SMARTLIST_FOREACH_BEGIN(votes, vote_routerstatus_t *, rs) { + char d[DIGEST256_LEN]; + if (compare_vote_rs(rs, most)) + continue; + if (!vote_routerstatus_find_microdesc_hash(d, rs, consensus_method, + DIGEST_SHA256)) + smartlist_add(digests, tor_memdup(d, sizeof(d))); + } SMARTLIST_FOREACH_END(rs); + smartlist_sort_digests256(digests); + best_microdesc_digest = smartlist_get_most_frequent_digest256(digests); + if (best_microdesc_digest) + memcpy(microdesc_digest256_out, best_microdesc_digest, DIGEST256_LEN); + SMARTLIST_FOREACH(digests, char *, cp, tor_free(cp)); + smartlist_free(digests); + } + return most; } -/** Given a list of strings in <b>lst</b>, set the DIGEST_LEN-byte digest at - * <b>digest_out</b> to the hash of the concatenation of those strings. */ +/** Given a list of strings in <b>lst</b>, set the <b>len_out</b>-byte digest + * at <b>digest_out</b> to the hash of the concatenation of those strings, + * computed with the algorithm <b>alg</b>. */ static void -hash_list_members(char *digest_out, smartlist_t *lst) +hash_list_members(char *digest_out, size_t len_out, + smartlist_t *lst, digest_algorithm_t alg) { - crypto_digest_env_t *d = crypto_new_digest_env(); + crypto_digest_env_t *d; + if (alg == DIGEST_SHA1) + d = crypto_new_digest_env(); + else + d = crypto_new_digest256_env(alg); SMARTLIST_FOREACH(lst, const char *, cp, crypto_digest_add_bytes(d, cp, strlen(cp))); - crypto_digest_get_digest(d, digest_out, DIGEST_LEN); + crypto_digest_get_digest(d, digest_out, len_out); crypto_free_digest_env(d); } @@ -455,7 +566,31 @@ compute_consensus_method(smartlist_t *votes) static int consensus_method_is_supported(int method) { - return (method >= 1) && (method <= 5); + return (method >= 1) && (method <= MAX_SUPPORTED_CONSENSUS_METHOD); +} + +/** Return a newly allocated string holding the numbers between low and high + * (inclusive) that are supported consensus methods. */ +static char * +make_consensus_method_list(int low, int high, const char *separator) +{ + char *list; + + char b[32]; + int i; + smartlist_t *lst; + lst = smartlist_create(); + for (i = low; i <= high; ++i) { + if (!consensus_method_is_supported(i)) + continue; + tor_snprintf(b, sizeof(b), "%d", i); + smartlist_add(lst, tor_strdup(b)); + } + list = smartlist_join_strings(lst, separator, 0, NULL); + tor_assert(list); + SMARTLIST_FOREACH(lst, char *, cp, tor_free(cp)); + smartlist_free(lst); + return list; } /** Helper: given <b>lst</b>, a list of version strings such that every @@ -475,6 +610,468 @@ compute_consensus_versions_list(smartlist_t *lst, int n_versioning) return result; } +/** Helper: given a list of valid networkstatus_t, return a new string + * containing the contents of the consensus network parameter set. + */ +/* private */ char * +dirvote_compute_params(smartlist_t *votes) +{ + int i; + int32_t *vals; + + int cur_param_len; + const char *cur_param; + const char *eq; + char *result; + + const int n_votes = smartlist_len(votes); + smartlist_t *output; + smartlist_t *param_list = smartlist_create(); + + /* We require that the parameter lists in the votes are well-formed: that + is, that their keywords are unique and sorted, and that their values are + between INT32_MIN and INT32_MAX inclusive. This should be guaranteed by + the parsing code. */ + + vals = tor_malloc(sizeof(int)*n_votes); + + SMARTLIST_FOREACH_BEGIN(votes, networkstatus_t *, v) { + if (!v->net_params) + continue; + smartlist_add_all(param_list, v->net_params); + } SMARTLIST_FOREACH_END(v); + + if (smartlist_len(param_list) == 0) { + tor_free(vals); + smartlist_free(param_list); + return NULL; + } + + smartlist_sort_strings(param_list); + i = 0; + cur_param = smartlist_get(param_list, 0); + eq = strchr(cur_param, '='); + tor_assert(eq); + cur_param_len = (int)(eq+1 - cur_param); + + output = smartlist_create(); + + SMARTLIST_FOREACH_BEGIN(param_list, const char *, param) { + const char *next_param; + int ok=0; + eq = strchr(param, '='); + tor_assert(i<n_votes); + vals[i++] = (int32_t) + tor_parse_long(eq+1, 10, INT32_MIN, INT32_MAX, &ok, NULL); + tor_assert(ok); + + if (param_sl_idx+1 == smartlist_len(param_list)) + next_param = NULL; + else + next_param = smartlist_get(param_list, param_sl_idx+1); + if (!next_param || strncmp(next_param, param, cur_param_len)) { + /* We've reached the end of a series. */ + int32_t median = median_int32(vals, i); + char *out_string = tor_malloc(64+cur_param_len); + memcpy(out_string, param, cur_param_len); + tor_snprintf(out_string+cur_param_len,64, "%ld", (long)median); + smartlist_add(output, out_string); + + i = 0; + if (next_param) { + eq = strchr(next_param, '='); + cur_param_len = (int)(eq+1 - next_param); + } + } + } SMARTLIST_FOREACH_END(param); + + result = smartlist_join_strings(output, " ", 0, NULL); + SMARTLIST_FOREACH(output, char *, cp, tor_free(cp)); + smartlist_free(output); + smartlist_free(param_list); + tor_free(vals); + return result; +} + +#define RANGE_CHECK(a,b,c,d,e,f,g,mx) \ + ((a) >= 0 && (a) <= (mx) && (b) >= 0 && (b) <= (mx) && \ + (c) >= 0 && (c) <= (mx) && (d) >= 0 && (d) <= (mx) && \ + (e) >= 0 && (e) <= (mx) && (f) >= 0 && (f) <= (mx) && \ + (g) >= 0 && (g) <= (mx)) + +#define CHECK_EQ(a, b, margin) \ + ((a)-(b) >= 0 ? (a)-(b) <= (margin) : (b)-(a) <= (margin)) + +typedef enum { + BW_WEIGHTS_NO_ERROR = 0, + BW_WEIGHTS_RANGE_ERROR = 1, + BW_WEIGHTS_SUMG_ERROR = 2, + BW_WEIGHTS_SUME_ERROR = 3, + BW_WEIGHTS_SUMD_ERROR = 4, + BW_WEIGHTS_BALANCE_MID_ERROR = 5, + BW_WEIGHTS_BALANCE_EG_ERROR = 6 +} bw_weights_error_t; + +/** + * Verify that any weightings satisfy the balanced formulas. + */ +static bw_weights_error_t +networkstatus_check_weights(int64_t Wgg, int64_t Wgd, int64_t Wmg, + int64_t Wme, int64_t Wmd, int64_t Wee, + int64_t Wed, int64_t scale, int64_t G, + int64_t M, int64_t E, int64_t D, int64_t T, + int64_t margin, int do_balance) { + bw_weights_error_t berr = BW_WEIGHTS_NO_ERROR; + + // Wed + Wmd + Wgd == 1 + if (!CHECK_EQ(Wed + Wmd + Wgd, scale, margin)) { + berr = BW_WEIGHTS_SUMD_ERROR; + goto out; + } + + // Wmg + Wgg == 1 + if (!CHECK_EQ(Wmg + Wgg, scale, margin)) { + berr = BW_WEIGHTS_SUMG_ERROR; + goto out; + } + + // Wme + Wee == 1 + if (!CHECK_EQ(Wme + Wee, scale, margin)) { + berr = BW_WEIGHTS_SUME_ERROR; + goto out; + } + + // Verify weights within range 0->1 + if (!RANGE_CHECK(Wgg, Wgd, Wmg, Wme, Wmd, Wed, Wee, scale)) { + berr = BW_WEIGHTS_RANGE_ERROR; + goto out; + } + + if (do_balance) { + // Wgg*G + Wgd*D == Wee*E + Wed*D, already scaled + if (!CHECK_EQ(Wgg*G + Wgd*D, Wee*E + Wed*D, (margin*T)/3)) { + berr = BW_WEIGHTS_BALANCE_EG_ERROR; + goto out; + } + + // Wgg*G + Wgd*D == M*scale + Wmd*D + Wme*E + Wmg*G, already scaled + if (!CHECK_EQ(Wgg*G + Wgd*D, M*scale + Wmd*D + Wme*E + Wmg*G, + (margin*T)/3)) { + berr = BW_WEIGHTS_BALANCE_MID_ERROR; + goto out; + } + } + +out: + if (berr) { + log_info(LD_DIR, + "Bw weight mismatch %d. G="I64_FORMAT" M="I64_FORMAT + " E="I64_FORMAT" D="I64_FORMAT" T="I64_FORMAT, + berr, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); + } + + return berr; +} + +static void +networkstatus_compute_bw_weights_v9(smartlist_t *chunks, int64_t G, int64_t M, + int64_t E, int64_t D, int64_t T, + int64_t weight_scale) +{ + int64_t Wgg = -1, Wgd = -1; + int64_t Wmg = -1, Wme = -1, Wmd = -1; + int64_t Wed = -1, Wee = -1; + const char *casename; + char buf[512]; + int r; + + if (G <= 0 || M <= 0 || E <= 0 || D <= 0) { + log_warn(LD_DIR, "Consensus with empty bandwidth: " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT + " D="I64_FORMAT" T="I64_FORMAT, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); + return; + } + + /* + * Computed from cases in 3.4.3 of dir-spec.txt + * + * 1. Neither are scarce + * 2. Both Guard and Exit are scarce + * a. R+D <= S + * b. R+D > S + * 3. One of Guard or Exit is scarce + * a. S+D < T/3 + * b. S+D >= T/3 + */ + if (3*E >= T && 3*G >= T) { // E >= T/3 && G >= T/3 + bw_weights_error_t berr = 0; + /* Case 1: Neither are scarce. + * + * Attempt to ensure that we have a large amount of exit bandwidth + * in the middle position. + */ + casename = "Case 1 (Wme*E = Wmd*D)"; + Wgg = (weight_scale*(D+E+G+M))/(3*G); + if (D==0) Wmd = 0; + else Wmd = (weight_scale*(2*D + 2*E - G - M))/(6*D); + Wme = (weight_scale*(2*D + 2*E - G - M))/(6*E); + Wee = (weight_scale*(-2*D + 4*E + G + M))/(6*E); + Wgd = 0; + Wmg = weight_scale - Wgg; + Wed = weight_scale - Wmd; + + berr = networkstatus_check_weights(Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed, + weight_scale, G, M, E, D, T, 10, 1); + + if (berr) { + log_warn(LD_DIR, "Bw Weights error %d for case %s. " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT + " D="I64_FORMAT" T="I64_FORMAT, + berr, casename, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); + } + } else if (3*E < T && 3*G < T) { // E < T/3 && G < T/3 + int64_t R = MIN(E, G); + int64_t S = MAX(E, G); + /* + * Case 2: Both Guards and Exits are scarce + * Balance D between E and G, depending upon + * D capacity and scarcity. + */ + if (R+D < S) { // Subcase a + Wgg = weight_scale; + Wee = weight_scale; + Wmg = 0; + Wme = 0; + Wmd = 0; + if (E < G) { + casename = "Case 2a (E scarce)"; + Wed = weight_scale; + Wgd = 0; + } else { /* E >= G */ + casename = "Case 2a (G scarce)"; + Wed = 0; + Wgd = weight_scale; + } + } else { // Subcase b: R+D > S + bw_weights_error_t berr = 0; + casename = "Case 2b (Wme*E == Wmd*D)"; + if (D != 0) { + Wgg = weight_scale; + Wgd = (weight_scale*(D + E - 2*G + M))/(3*D); // T/3 >= G (Ok) + Wmd = (weight_scale*(D + E + G - 2*M))/(6*D); // T/3 >= M + Wme = (weight_scale*(D + E + G - 2*M))/(6*E); + Wee = (weight_scale*(-D + 5*E - G + 2*M))/(6*E); // 2E+M >= T/3 + Wmg = 0; + Wed = weight_scale - Wgd - Wmd; + + berr = networkstatus_check_weights(Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed, + weight_scale, G, M, E, D, T, 10, 1); + } + + if (D == 0 || berr) { // Can happen if M > T/3 + casename = "Case 2b (E=G)"; + Wgg = weight_scale; + Wee = weight_scale; + Wmg = 0; + Wme = 0; + Wmd = 0; + if (D == 0) Wgd = 0; + else Wgd = (weight_scale*(D+E-G))/(2*D); + Wed = weight_scale - Wgd; + berr = networkstatus_check_weights(Wgg, Wgd, Wmg, Wme, Wmd, Wee, + Wed, weight_scale, G, M, E, D, T, 10, 1); + } + if (berr != BW_WEIGHTS_NO_ERROR && + berr != BW_WEIGHTS_BALANCE_MID_ERROR) { + log_warn(LD_DIR, "Bw Weights error %d for case %s. " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT + " D="I64_FORMAT" T="I64_FORMAT, + berr, casename, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); + } + } + } else { // if (E < T/3 || G < T/3) { + int64_t S = MIN(E, G); + // Case 3: Exactly one of Guard or Exit is scarce + if (!(3*E < T || 3*G < T) || !(3*G >= T || 3*E >= T)) { + log_warn(LD_BUG, + "Bw-Weights Case 3 but with G="I64_FORMAT" M=" + I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT" T="I64_FORMAT, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); + } + + if (3*(S+D) < T) { // Subcase a: S+D < T/3 + if (G < E) { + casename = "Case 3a (G scarce)"; + Wgg = Wgd = weight_scale; + Wmd = Wed = Wmg = 0; + // Minor subcase, if E is more scarce than M, + // keep its bandwidth in place. + if (E < M) Wme = 0; + else Wme = (weight_scale*(E-M))/(2*E); + Wee = weight_scale-Wme; + } else { // G >= E + casename = "Case 3a (E scarce)"; + Wee = Wed = weight_scale; + Wmd = Wgd = Wme = 0; + // Minor subcase, if G is more scarce than M, + // keep its bandwidth in place. + if (G < M) Wmg = 0; + else Wmg = (weight_scale*(G-M))/(2*G); + Wgg = weight_scale-Wmg; + } + } else { // Subcase b: S+D >= T/3 + bw_weights_error_t berr = 0; + // D != 0 because S+D >= T/3 + if (G < E) { + casename = "Case 3b (G scarce, Wme*E == Wmd*D)"; + Wgd = (weight_scale*(D + E - 2*G + M))/(3*D); + Wmd = (weight_scale*(D + E + G - 2*M))/(6*D); + Wme = (weight_scale*(D + E + G - 2*M))/(6*E); + Wee = (weight_scale*(-D + 5*E - G + 2*M))/(6*E); + Wgg = weight_scale; + Wmg = 0; + Wed = weight_scale - Wgd - Wmd; + + berr = networkstatus_check_weights(Wgg, Wgd, Wmg, Wme, Wmd, Wee, + Wed, weight_scale, G, M, E, D, T, 10, 1); + } else { // G >= E + casename = "Case 3b (E scarce, Wme*E == Wmd*D)"; + Wgg = (weight_scale*(D + E + G + M))/(3*G); + Wmd = (weight_scale*(2*D + 2*E - G - M))/(6*D); + Wme = (weight_scale*(2*D + 2*E - G - M))/(6*E); + Wee = (weight_scale*(-2*D + 4*E + G + M))/(6*E); + Wgd = 0; + Wmg = weight_scale - Wgg; + Wed = weight_scale - Wmd; + + berr = networkstatus_check_weights(Wgg, Wgd, Wmg, Wme, Wmd, Wee, + Wed, weight_scale, G, M, E, D, T, 10, 1); + } + if (berr) { + log_warn(LD_DIR, "Bw Weights error %d for case %s. " + "G="I64_FORMAT" M="I64_FORMAT + " E="I64_FORMAT" D="I64_FORMAT" T="I64_FORMAT, + berr, casename, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); + } + } + } + + /* We cast down the weights to 32 bit ints on the assumption that + * weight_scale is ~= 10000. We need to ensure a rogue authority + * doesn't break this assumption to rig our weights */ + tor_assert(0 < weight_scale && weight_scale < INT32_MAX); + + if (Wgg < 0 || Wgg > weight_scale) { + log_warn(LD_DIR, "Bw %s: Wgg="I64_FORMAT"! G="I64_FORMAT + " M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT, + casename, I64_PRINTF_ARG(Wgg), + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); + + Wgg = MAX(MIN(Wgg, weight_scale), 0); + } + if (Wgd < 0 || Wgd > weight_scale) { + log_warn(LD_DIR, "Bw %s: Wgd="I64_FORMAT"! G="I64_FORMAT + " M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT, + casename, I64_PRINTF_ARG(Wgd), + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); + Wgd = MAX(MIN(Wgd, weight_scale), 0); + } + if (Wmg < 0 || Wmg > weight_scale) { + log_warn(LD_DIR, "Bw %s: Wmg="I64_FORMAT"! G="I64_FORMAT + " M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT, + casename, I64_PRINTF_ARG(Wmg), + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); + Wmg = MAX(MIN(Wmg, weight_scale), 0); + } + if (Wme < 0 || Wme > weight_scale) { + log_warn(LD_DIR, "Bw %s: Wme="I64_FORMAT"! G="I64_FORMAT + " M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT, + casename, I64_PRINTF_ARG(Wme), + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); + Wme = MAX(MIN(Wme, weight_scale), 0); + } + if (Wmd < 0 || Wmd > weight_scale) { + log_warn(LD_DIR, "Bw %s: Wmd="I64_FORMAT"! G="I64_FORMAT + " M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT, + casename, I64_PRINTF_ARG(Wmd), + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); + Wmd = MAX(MIN(Wmd, weight_scale), 0); + } + if (Wee < 0 || Wee > weight_scale) { + log_warn(LD_DIR, "Bw %s: Wee="I64_FORMAT"! G="I64_FORMAT + " M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT, + casename, I64_PRINTF_ARG(Wee), + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); + Wee = MAX(MIN(Wee, weight_scale), 0); + } + if (Wed < 0 || Wed > weight_scale) { + log_warn(LD_DIR, "Bw %s: Wed="I64_FORMAT"! G="I64_FORMAT + " M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT, + casename, I64_PRINTF_ARG(Wed), + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); + Wed = MAX(MIN(Wed, weight_scale), 0); + } + + // Add consensus weight keywords + smartlist_add(chunks, tor_strdup("bandwidth-weights ")); + /* + * Provide Wgm=Wgg, Wmm=1, Wem=Wee, Weg=Wed. May later determine + * that middle nodes need different bandwidth weights for dirport traffic, + * or that weird exit policies need special weight, or that bridges + * need special weight. + * + * NOTE: This list is sorted. + */ + r = tor_snprintf(buf, sizeof(buf), + "Wbd=%d Wbe=%d Wbg=%d Wbm=%d " + "Wdb=%d " + "Web=%d Wed=%d Wee=%d Weg=%d Wem=%d " + "Wgb=%d Wgd=%d Wgg=%d Wgm=%d " + "Wmb=%d Wmd=%d Wme=%d Wmg=%d Wmm=%d\n", + (int)Wmd, (int)Wme, (int)Wmg, (int)weight_scale, + (int)weight_scale, + (int)weight_scale, (int)Wed, (int)Wee, (int)Wed, (int)Wee, + (int)weight_scale, (int)Wgd, (int)Wgg, (int)Wgg, + (int)weight_scale, (int)Wmd, (int)Wme, (int)Wmg, (int)weight_scale); + if (r<0) { + log_warn(LD_BUG, + "Not enough space in buffer for bandwidth-weights line."); + *buf = '\0'; + } + smartlist_add(chunks, tor_strdup(buf)); + log_notice(LD_CIRC, "Computed bandwidth weights for %s: " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT, + casename, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); +} + /** Given a list of vote networkstatus_t in <b>votes</b>, our public * authority <b>identity_key</b>, our private authority <b>signing_key</b>, * and the number of <b>total_authorities</b> that we believe exist in our @@ -489,18 +1086,26 @@ networkstatus_compute_consensus(smartlist_t *votes, crypto_pk_env_t *identity_key, crypto_pk_env_t *signing_key, const char *legacy_id_key_digest, - crypto_pk_env_t *legacy_signing_key) + crypto_pk_env_t *legacy_signing_key, + consensus_flavor_t flavor) { smartlist_t *chunks; char *result = NULL; int consensus_method; - time_t valid_after, fresh_until, valid_until; int vote_seconds, dist_seconds; char *client_versions = NULL, *server_versions = NULL; smartlist_t *flags; + const char *flavor_name; + int64_t G=0, M=0, E=0, D=0, T=0; /* For bandwidth weights */ + const routerstatus_format_type_t rs_format = + flavor == FLAV_NS ? NS_V3_CONSENSUS : NS_V3_CONSENSUS_MICRODESC; + char *params = NULL; + tor_assert(flavor == FLAV_NS || flavor == FLAV_MICRODESC); tor_assert(total_authorities >= smartlist_len(votes)); + flavor_name = networkstatus_get_flavor_name(flavor); + if (!smartlist_len(votes)) { log_warn(LD_DIR, "Can't compute a consensus from no votes."); return NULL; @@ -593,7 +1198,7 @@ networkstatus_compute_consensus(smartlist_t *votes, chunks = smartlist_create(); { - char buf[1024]; + char *buf=NULL; char va_buf[ISO_TIME_LEN+1], fu_buf[ISO_TIME_LEN+1], vu_buf[ISO_TIME_LEN+1]; char *flaglist; @@ -602,16 +1207,20 @@ networkstatus_compute_consensus(smartlist_t *votes, format_iso_time(vu_buf, valid_until); flaglist = smartlist_join_strings(flags, " ", 0, NULL); - smartlist_add(chunks, tor_strdup("network-status-version 3\n" - "vote-status consensus\n")); + tor_asprintf(&buf, "network-status-version 3%s%s\n" + "vote-status consensus\n", + flavor == FLAV_NS ? "" : " ", + flavor == FLAV_NS ? "" : flavor_name); + + smartlist_add(chunks, buf); if (consensus_method >= 2) { - tor_snprintf(buf, sizeof(buf), "consensus-method %d\n", + tor_asprintf(&buf, "consensus-method %d\n", consensus_method); - smartlist_add(chunks, tor_strdup(buf)); + smartlist_add(chunks, buf); } - tor_snprintf(buf, sizeof(buf), + tor_asprintf(&buf, "valid-after %s\n" "fresh-until %s\n" "valid-until %s\n" @@ -622,18 +1231,26 @@ networkstatus_compute_consensus(smartlist_t *votes, va_buf, fu_buf, vu_buf, vote_seconds, dist_seconds, client_versions, server_versions, flaglist); - smartlist_add(chunks, tor_strdup(buf)); + smartlist_add(chunks, buf); tor_free(flaglist); } + if (consensus_method >= MIN_METHOD_FOR_PARAMS) { + params = dirvote_compute_params(votes); + if (params) { + smartlist_add(chunks, tor_strdup("params ")); + smartlist_add(chunks, params); + smartlist_add(chunks, tor_strdup("\n")); + } + } + /* Sort the votes. */ smartlist_sort(votes, _compare_votes_by_authority_id); /* Add the authority sections. */ { smartlist_t *dir_sources = smartlist_create(); - SMARTLIST_FOREACH(votes, networkstatus_t *, v, - { + SMARTLIST_FOREACH_BEGIN(votes, networkstatus_t *, v) { dir_src_ent_t *e = tor_malloc_zero(sizeof(dir_src_ent_t)); e->v = v; e->digest = get_voter(v)->identity_digest; @@ -647,18 +1264,17 @@ networkstatus_compute_consensus(smartlist_t *votes, e_legacy->is_legacy = 1; smartlist_add(dir_sources, e_legacy); } - }); + } SMARTLIST_FOREACH_END(v); smartlist_sort(dir_sources, _compare_dir_src_ents_by_authority_id); - SMARTLIST_FOREACH(dir_sources, const dir_src_ent_t *, e, - { - char buf[1024]; + SMARTLIST_FOREACH_BEGIN(dir_sources, const dir_src_ent_t *, e) { struct in_addr in; char ip[INET_NTOA_BUF_LEN]; char fingerprint[HEX_DIGEST_LEN+1]; char votedigest[HEX_DIGEST_LEN+1]; networkstatus_t *v = e->v; networkstatus_voter_info_t *voter = get_voter(v); + char *buf = NULL; if (e->is_legacy) tor_assert(consensus_method >= 2); @@ -669,22 +1285,22 @@ networkstatus_compute_consensus(smartlist_t *votes, base16_encode(votedigest, sizeof(votedigest), voter->vote_digest, DIGEST_LEN); - tor_snprintf(buf, sizeof(buf), + tor_asprintf(&buf, "dir-source %s%s %s %s %s %d %d\n", voter->nickname, e->is_legacy ? "-legacy" : "", fingerprint, voter->address, ip, voter->dir_port, voter->or_port); - smartlist_add(chunks, tor_strdup(buf)); + smartlist_add(chunks, buf); if (! e->is_legacy) { - tor_snprintf(buf, sizeof(buf), + tor_asprintf(&buf, "contact %s\n" "vote-digest %s\n", voter->contact, votedigest); - smartlist_add(chunks, tor_strdup(buf)); + smartlist_add(chunks, buf); } - }); + } SMARTLIST_FOREACH_END(e); SMARTLIST_FOREACH(dir_sources, dir_src_ent_t *, e, tor_free(e)); smartlist_free(dir_sources); } @@ -701,7 +1317,10 @@ networkstatus_compute_consensus(smartlist_t *votes, smartlist_t *versions = smartlist_create(); smartlist_t *exitsummaries = smartlist_create(); uint32_t *bandwidths = tor_malloc(sizeof(uint32_t) * smartlist_len(votes)); + uint32_t *measured_bws = tor_malloc(sizeof(uint32_t) * + smartlist_len(votes)); int num_bandwidths; + int num_mbws; int *n_voter_flags; /* n_voter_flags[j] is the number of flags that * votes[j] knows about. */ @@ -813,10 +1432,12 @@ networkstatus_compute_consensus(smartlist_t *votes, const char *chosen_name = NULL; int exitsummary_disagreement = 0; int is_named = 0, is_unnamed = 0, is_running = 0; + int is_guard = 0, is_exit = 0; int naming_conflict = 0; int n_listing = 0; int i; - char buf[256]; + char *buf=NULL; + char microdesc_digest[DIGEST256_LEN]; /* Of the next-to-be-considered digest in each voter, which is first? */ SMARTLIST_FOREACH(votes, networkstatus_t *, v, { @@ -835,6 +1456,7 @@ networkstatus_compute_consensus(smartlist_t *votes, smartlist_clear(chosen_flags); smartlist_clear(versions); num_bandwidths = 0; + num_mbws = 0; /* Okay, go through all the entries for this digest. */ SMARTLIST_FOREACH_BEGIN(votes, networkstatus_t *, v) { @@ -868,6 +1490,9 @@ networkstatus_compute_consensus(smartlist_t *votes, } /* count bandwidths */ + if (rs->status.has_measured_bw) + measured_bws[num_mbws++] = rs->status.measured_bw; + if (rs->status.has_bandwidth) bandwidths[num_bandwidths++] = rs->status.bandwidth; } SMARTLIST_FOREACH_END(v); @@ -879,7 +1504,9 @@ networkstatus_compute_consensus(smartlist_t *votes, /* Figure out the most popular opinion of what the most recent * routerinfo and its contents are. */ - rs = compute_routerstatus_consensus(matching_descs); + memset(microdesc_digest, 0, sizeof(microdesc_digest)); + rs = compute_routerstatus_consensus(matching_descs, consensus_method, + microdesc_digest); /* Copy bits of that into rs_out. */ tor_assert(!memcmp(lowest_id, rs->status.identity_digest, DIGEST_LEN)); memcpy(rs_out.identity_digest, lowest_id, DIGEST_LEN); @@ -925,7 +1552,11 @@ networkstatus_compute_consensus(smartlist_t *votes, } else { if (flag_counts[fl_sl_idx] > n_flag_voters[fl_sl_idx]/2) { smartlist_add(chosen_flags, (char*)fl); - if (!strcmp(fl, "Running")) + if (!strcmp(fl, "Exit")) + is_exit = 1; + else if (!strcmp(fl, "Guard")) + is_guard = 1; + else if (!strcmp(fl, "Running")) is_running = 1; } } @@ -945,11 +1576,31 @@ networkstatus_compute_consensus(smartlist_t *votes, } /* Pick a bandwidth */ - if (consensus_method >= 5 && num_bandwidths > 0) { + if (consensus_method >= 6 && num_mbws > 2) { + rs_out.has_bandwidth = 1; + rs_out.bandwidth = median_uint32(measured_bws, num_mbws); + } else if (consensus_method >= 5 && num_bandwidths > 0) { rs_out.has_bandwidth = 1; rs_out.bandwidth = median_uint32(bandwidths, num_bandwidths); } + if (consensus_method >= MIN_METHOD_FOR_BW_WEIGHTS) { + if (rs_out.has_bandwidth) { + T += rs_out.bandwidth; + if (is_exit && is_guard) + D += rs_out.bandwidth; + else if (is_exit) + E += rs_out.bandwidth; + else if (is_guard) + G += rs_out.bandwidth; + else + M += rs_out.bandwidth; + } else { + log_warn(LD_BUG, "Missing consensus bandwidth for router %s", + rs_out.nickname); + } + } + /* Ok, we already picked a descriptor digest we want to list * previously. Now we want to use the exit policy summary from * that descriptor. If everybody plays nice all the voters who @@ -1034,11 +1685,23 @@ networkstatus_compute_consensus(smartlist_t *votes, } } - /* Okay!! Now we can write the descriptor... */ - /* First line goes into "buf". */ - routerstatus_format_entry(buf, sizeof(buf), &rs_out, NULL, 1, 0); - smartlist_add(chunks, tor_strdup(buf)); - /* Second line is all flags. The "\n" is missing. */ + { + char buf[4096]; + /* Okay!! Now we can write the descriptor... */ + /* First line goes into "buf". */ + routerstatus_format_entry(buf, sizeof(buf), &rs_out, NULL, + rs_format); + smartlist_add(chunks, tor_strdup(buf)); + } + /* Now an m line, if applicable. */ + if (flavor == FLAV_MICRODESC && + !tor_digest256_is_zero(microdesc_digest)) { + char m[BASE64_DIGEST256_LEN+1], *cp; + digest256_to_base64(m, microdesc_digest); + tor_asprintf(&cp, "m %s\n", m); + smartlist_add(chunks, cp); + } + /* Next line is all flags. The "\n" is missing. */ smartlist_add(chunks, smartlist_join_strings(chosen_flags, " ", 0, NULL)); /* Now the version line. */ @@ -1049,24 +1712,16 @@ networkstatus_compute_consensus(smartlist_t *votes, smartlist_add(chunks, tor_strdup("\n")); /* Now the weight line. */ if (rs_out.has_bandwidth) { - int r = tor_snprintf(buf, sizeof(buf), - "w Bandwidth=%d\n", rs_out.bandwidth); - if (r<0) { - log_warn(LD_BUG, "Not enough space in buffer for weight line."); - *buf = '\0'; - } - smartlist_add(chunks, tor_strdup(buf)); - }; + char *cp=NULL; + tor_asprintf(&cp, "w Bandwidth=%d\n", rs_out.bandwidth); + smartlist_add(chunks, cp); + } + /* Now the exitpolicy summary line. */ - if (rs_out.has_exitsummary) { - char buf[MAX_POLICY_LINE_LEN+1]; - int r = tor_snprintf(buf, sizeof(buf), "p %s\n", rs_out.exitsummary); - if (r<0) { - log_warn(LD_BUG, "Not enough space in buffer for exitpolicy line."); - *buf = '\0'; - } - smartlist_add(chunks, tor_strdup(buf)); - }; + if (rs_out.has_exitsummary && flavor == FLAV_NS) { + tor_asprintf(&buf, "p %s\n", rs_out.exitsummary); + smartlist_add(chunks, buf); + } /* And the loop is over and we move on to the next router */ } @@ -1087,34 +1742,91 @@ networkstatus_compute_consensus(smartlist_t *votes, smartlist_free(versions); smartlist_free(exitsummaries); tor_free(bandwidths); + tor_free(measured_bws); + } + + if (consensus_method >= MIN_METHOD_FOR_FOOTER) { + /* Starting with consensus method 9, we clearly mark the directory + * footer region */ + smartlist_add(chunks, tor_strdup("directory-footer\n")); + } + + if (consensus_method >= MIN_METHOD_FOR_BW_WEIGHTS) { + int64_t weight_scale = BW_WEIGHT_SCALE; + char *bw_weight_param = NULL; + + // Parse params, extract BW_WEIGHT_SCALE if present + // DO NOT use consensus_param_bw_weight_scale() in this code! + // The consensus is not formed yet! + if (params) { + if (strcmpstart(params, "bwweightscale=") == 0) + bw_weight_param = params; + else + bw_weight_param = strstr(params, " bwweightscale="); + } + + if (bw_weight_param) { + int ok=0; + char *eq = strchr(bw_weight_param, '='); + if (eq) { + weight_scale = tor_parse_long(eq+1, 10, INT32_MIN, INT32_MAX, &ok, + NULL); + if (!ok) { + log_warn(LD_DIR, "Bad element '%s' in bw weight param", + escaped(bw_weight_param)); + weight_scale = BW_WEIGHT_SCALE; + } + } else { + log_warn(LD_DIR, "Bad element '%s' in bw weight param", + escaped(bw_weight_param)); + weight_scale = BW_WEIGHT_SCALE; + } + } + + networkstatus_compute_bw_weights_v9(chunks, G, M, E, D, T, weight_scale); } /* Add a signature. */ { - char digest[DIGEST_LEN]; + char digest[DIGEST256_LEN]; char fingerprint[HEX_DIGEST_LEN+1]; char signing_key_fingerprint[HEX_DIGEST_LEN+1]; + digest_algorithm_t digest_alg = + flavor == FLAV_NS ? DIGEST_SHA1 : DIGEST_SHA256; + size_t digest_len = + flavor == FLAV_NS ? DIGEST_LEN : DIGEST256_LEN; + const char *algname = crypto_digest_algorithm_get_name(digest_alg); + char *buf = NULL; + char sigbuf[4096]; - char buf[4096]; smartlist_add(chunks, tor_strdup("directory-signature ")); /* Compute the hash of the chunks. */ - hash_list_members(digest, chunks); + hash_list_members(digest, digest_len, chunks, digest_alg); /* Get the fingerprints */ crypto_pk_get_fingerprint(identity_key, fingerprint, 0); crypto_pk_get_fingerprint(signing_key, signing_key_fingerprint, 0); /* add the junk that will go at the end of the line. */ - tor_snprintf(buf, sizeof(buf), "%s %s\n", fingerprint, - signing_key_fingerprint); + if (flavor == FLAV_NS) { + tor_asprintf(&buf, "%s %s\n", fingerprint, + signing_key_fingerprint); + } else { + tor_asprintf(&buf, "%s %s %s\n", + algname, fingerprint, + signing_key_fingerprint); + } + smartlist_add(chunks, buf); /* And the signature. */ - if (router_append_dirobj_signature(buf, sizeof(buf), digest, + sigbuf[0] = '\0'; + if (router_append_dirobj_signature(sigbuf, sizeof(sigbuf), + digest, digest_len, signing_key)) { log_warn(LD_BUG, "Couldn't sign consensus networkstatus."); return NULL; /* This leaks, but it should never happen. */ } - smartlist_add(chunks, tor_strdup(buf)); + smartlist_add(chunks, tor_strdup(sigbuf)); if (legacy_id_key_digest && legacy_signing_key && consensus_method >= 3) { smartlist_add(chunks, tor_strdup("directory-signature ")); @@ -1122,14 +1834,23 @@ networkstatus_compute_consensus(smartlist_t *votes, legacy_id_key_digest, DIGEST_LEN); crypto_pk_get_fingerprint(legacy_signing_key, signing_key_fingerprint, 0); - tor_snprintf(buf, sizeof(buf), "%s %s\n", fingerprint, - signing_key_fingerprint); - if (router_append_dirobj_signature(buf, sizeof(buf), digest, + if (flavor == FLAV_NS) { + tor_asprintf(&buf, "%s %s\n", fingerprint, + signing_key_fingerprint); + } else { + tor_asprintf(&buf, "%s %s %s\n", + algname, fingerprint, + signing_key_fingerprint); + } + smartlist_add(chunks, buf); + sigbuf[0] = '\0'; + if (router_append_dirobj_signature(sigbuf, sizeof(sigbuf), + digest, digest_len, legacy_signing_key)) { log_warn(LD_BUG, "Couldn't sign consensus networkstatus."); return NULL; /* This leaks, but it should never happen. */ } - smartlist_add(chunks, tor_strdup(buf)); + smartlist_add(chunks, tor_strdup(sigbuf)); } } @@ -1146,11 +1867,15 @@ networkstatus_compute_consensus(smartlist_t *votes, networkstatus_t *c; if (!(c = networkstatus_parse_vote_from_string(result, NULL, NS_TYPE_CONSENSUS))) { - log_err(LD_BUG,"Generated a networkstatus consensus we couldn't " + log_err(LD_BUG, "Generated a networkstatus consensus we couldn't " "parse."); tor_free(result); return NULL; } + // Verify balancing parameters + if (consensus_method >= MIN_METHOD_FOR_BW_WEIGHTS) { + networkstatus_verify_bw_weights(c); + } networkstatus_vote_free(c); } @@ -1171,10 +1896,14 @@ networkstatus_add_detached_signatures(networkstatus_t *target, const char **msg_out) { int r = 0; + const char *flavor; + smartlist_t *siglist; tor_assert(sigs); tor_assert(target); tor_assert(target->type == NS_TYPE_CONSENSUS); + flavor = networkstatus_get_flavor_name(target->flavor); + /* Do the times seem right? */ if (target->valid_after != sigs->valid_after) { *msg_out = "Valid-After times do not match " @@ -1191,79 +1920,179 @@ networkstatus_add_detached_signatures(networkstatus_t *target, "when adding detached signatures to consensus"; return -1; } - /* Are they the same consensus? */ - if (memcmp(target->networkstatus_digest, sigs->networkstatus_digest, - DIGEST_LEN)) { - *msg_out = "Digest mismatch when adding detached signatures to consensus"; + siglist = strmap_get(sigs->signatures, flavor); + if (!siglist) { + *msg_out = "No signatures for given consensus flavor"; return -1; } - /* For each voter in src... */ - SMARTLIST_FOREACH_BEGIN(sigs->signatures, networkstatus_voter_info_t *, - src_voter) { - char voter_identity[HEX_DIGEST_LEN+1]; - networkstatus_voter_info_t *target_voter = - networkstatus_get_voter_by_id(target, src_voter->identity_digest); - authority_cert_t *cert = NULL; - - base16_encode(voter_identity, sizeof(voter_identity), - src_voter->identity_digest, DIGEST_LEN); - log_info(LD_DIR, "Looking at signature from %s", voter_identity); - /* If the target doesn't know about this voter, then forget it. */ - if (!target_voter) { - log_info(LD_DIR, "We do not know about %s", voter_identity); - continue; + /** Make sure all the digests we know match, and at least one matches. */ + { + digests_t *digests = strmap_get(sigs->digests, flavor); + int n_matches = 0; + digest_algorithm_t alg; + if (!digests) { + *msg_out = "No digests for given consensus flavor"; + return -1; + } + for (alg = DIGEST_SHA1; alg < N_DIGEST_ALGORITHMS; ++alg) { + if (!tor_mem_is_zero(digests->d[alg], DIGEST256_LEN)) { + if (!memcmp(target->digests.d[alg], digests->d[alg], DIGEST256_LEN)) { + ++n_matches; + } else { + *msg_out = "Mismatched digest."; + return -1; + } } + } + if (!n_matches) { + *msg_out = "No regognized digests for given consensus flavor"; + } + } - /* If the target already has a good signature from this voter, then skip - * this one. */ - if (target_voter->good_signature) { - log_info(LD_DIR, "We already have a good signature from %s", - voter_identity); - continue; - } + /* For each voter in src... */ + SMARTLIST_FOREACH_BEGIN(siglist, document_signature_t *, sig) { + char voter_identity[HEX_DIGEST_LEN+1]; + networkstatus_voter_info_t *target_voter = + networkstatus_get_voter_by_id(target, sig->identity_digest); + authority_cert_t *cert = NULL; + const char *algorithm; + document_signature_t *old_sig = NULL; + + algorithm = crypto_digest_algorithm_get_name(sig->alg); + + base16_encode(voter_identity, sizeof(voter_identity), + sig->identity_digest, DIGEST_LEN); + log_info(LD_DIR, "Looking at signature from %s using %s", voter_identity, + algorithm); + /* If the target doesn't know about this voter, then forget it. */ + if (!target_voter) { + log_info(LD_DIR, "We do not know any voter with ID %s", voter_identity); + continue; + } - /* Try checking the signature if we haven't already. */ - if (!src_voter->good_signature && !src_voter->bad_signature) { - cert = authority_cert_get_by_digests(src_voter->identity_digest, - src_voter->signing_key_digest); - if (cert) { - networkstatus_check_voter_signature(target, src_voter, cert); - } + old_sig = voter_get_sig_by_algorithm(target_voter, sig->alg); + + /* If the target already has a good signature from this voter, then skip + * this one. */ + if (old_sig && old_sig->good_signature) { + log_info(LD_DIR, "We already have a good signature from %s using %s", + voter_identity, algorithm); + continue; + } + + /* Try checking the signature if we haven't already. */ + if (!sig->good_signature && !sig->bad_signature) { + cert = authority_cert_get_by_digests(sig->identity_digest, + sig->signing_key_digest); + if (cert) + networkstatus_check_document_signature(target, sig, cert); + } + + /* If this signature is good, or we don't have any signature yet, + * then maybe add it. */ + if (sig->good_signature || !old_sig || old_sig->bad_signature) { + log_info(LD_DIR, "Adding signature from %s with %s", voter_identity, + algorithm); + ++r; + if (old_sig) { + smartlist_remove(target_voter->sigs, old_sig); + document_signature_free(old_sig); } + smartlist_add(target_voter->sigs, document_signature_dup(sig)); + } else { + log_info(LD_DIR, "Not adding signature from %s", voter_identity); + } + } SMARTLIST_FOREACH_END(sig); - /* If this signature is good, or we don't have any signature yet, - * then add it. */ - if (src_voter->good_signature || !target_voter->signature) { - log_info(LD_DIR, "Adding signature from %s", voter_identity); - ++r; - tor_free(target_voter->signature); - target_voter->signature = - tor_memdup(src_voter->signature, src_voter->signature_len); - memcpy(target_voter->signing_key_digest, src_voter->signing_key_digest, - DIGEST_LEN); - target_voter->signature_len = src_voter->signature_len; - target_voter->good_signature = src_voter->good_signature; - target_voter->bad_signature = src_voter->bad_signature; + return r; +} + +/** Return a newly allocated string containing all the signatures on + * <b>consensus</b> by all voters. If <b>for_detached_signatures</b> is true, + * then the signatures will be put in a detached signatures document, so + * prefix any non-NS-flavored signatures with "additional-signature" rather + * than "directory-signature". */ +static char * +networkstatus_format_signatures(networkstatus_t *consensus, + int for_detached_signatures) +{ + smartlist_t *elements; + char buf[4096]; + char *result = NULL; + int n_sigs = 0; + const consensus_flavor_t flavor = consensus->flavor; + const char *flavor_name = networkstatus_get_flavor_name(flavor); + const char *keyword; + + if (for_detached_signatures && flavor != FLAV_NS) + keyword = "additional-signature"; + else + keyword = "directory-signature"; + + elements = smartlist_create(); + + SMARTLIST_FOREACH_BEGIN(consensus->voters, networkstatus_voter_info_t *, v) { + SMARTLIST_FOREACH_BEGIN(v->sigs, document_signature_t *, sig) { + char sk[HEX_DIGEST_LEN+1]; + char id[HEX_DIGEST_LEN+1]; + if (!sig->signature || sig->bad_signature) + continue; + ++n_sigs; + base16_encode(sk, sizeof(sk), sig->signing_key_digest, DIGEST_LEN); + base16_encode(id, sizeof(id), sig->identity_digest, DIGEST_LEN); + if (flavor == FLAV_NS) { + tor_snprintf(buf, sizeof(buf), + "%s %s %s\n-----BEGIN SIGNATURE-----\n", + keyword, id, sk); } else { - log_info(LD_DIR, "Not adding signature from %s", voter_identity); + const char *digest_name = + crypto_digest_algorithm_get_name(sig->alg); + tor_snprintf(buf, sizeof(buf), + "%s%s%s %s %s %s\n-----BEGIN SIGNATURE-----\n", + keyword, + for_detached_signatures ? " " : "", + for_detached_signatures ? flavor_name : "", + digest_name, id, sk); } - } SMARTLIST_FOREACH_END(src_voter); + smartlist_add(elements, tor_strdup(buf)); + base64_encode(buf, sizeof(buf), sig->signature, sig->signature_len); + strlcat(buf, "-----END SIGNATURE-----\n", sizeof(buf)); + smartlist_add(elements, tor_strdup(buf)); + } SMARTLIST_FOREACH_END(sig); + } SMARTLIST_FOREACH_END(v); - return r; + result = smartlist_join_strings(elements, "", 0, NULL); + SMARTLIST_FOREACH(elements, char *, cp, tor_free(cp)); + smartlist_free(elements); + if (!n_sigs) + tor_free(result); + return result; } /** Return a newly allocated string holding the detached-signatures document - * corresponding to the signatures on <b>consensus</b>. */ + * corresponding to the signatures on <b>consensuses</b>, which must contain + * exactly one FLAV_NS consensus, and no more than one consensus for each + * other flavor. */ char * -networkstatus_get_detached_signatures(networkstatus_t *consensus) +networkstatus_get_detached_signatures(smartlist_t *consensuses) { smartlist_t *elements; char buf[4096]; - char *result = NULL; - int n_sigs = 0; - tor_assert(consensus); - tor_assert(consensus->type == NS_TYPE_CONSENSUS); + char *result = NULL, *sigs = NULL; + networkstatus_t *consensus_ns = NULL; + tor_assert(consensuses); + + SMARTLIST_FOREACH(consensuses, networkstatus_t *, ns, { + tor_assert(ns); + tor_assert(ns->type == NS_TYPE_CONSENSUS); + if (ns && ns->flavor == FLAV_NS) + consensus_ns = ns; + }); + if (!consensus_ns) { + log_warn(LD_BUG, "No NS consensus given."); + return NULL; + } elements = smartlist_create(); @@ -1272,10 +2101,11 @@ networkstatus_get_detached_signatures(networkstatus_t *consensus) vu_buf[ISO_TIME_LEN+1]; char d[HEX_DIGEST_LEN+1]; - base16_encode(d, sizeof(d), consensus->networkstatus_digest, DIGEST_LEN); - format_iso_time(va_buf, consensus->valid_after); - format_iso_time(fu_buf, consensus->fresh_until); - format_iso_time(vu_buf, consensus->valid_until); + base16_encode(d, sizeof(d), + consensus_ns->digests.d[DIGEST_SHA1], DIGEST_LEN); + format_iso_time(va_buf, consensus_ns->valid_after); + format_iso_time(fu_buf, consensus_ns->fresh_until); + format_iso_time(vu_buf, consensus_ns->valid_until); tor_snprintf(buf, sizeof(buf), "consensus-digest %s\n" @@ -1285,45 +2115,89 @@ networkstatus_get_detached_signatures(networkstatus_t *consensus) smartlist_add(elements, tor_strdup(buf)); } - SMARTLIST_FOREACH(consensus->voters, networkstatus_voter_info_t *, v, - { - char sk[HEX_DIGEST_LEN+1]; - char id[HEX_DIGEST_LEN+1]; - if (!v->signature || v->bad_signature) + /* Get all the digests for the non-FLAV_NS consensuses */ + SMARTLIST_FOREACH_BEGIN(consensuses, networkstatus_t *, ns) { + const char *flavor_name = networkstatus_get_flavor_name(ns->flavor); + int alg; + if (ns->flavor == FLAV_NS) + continue; + + /* start with SHA256; we don't include SHA1 for anything but the basic + * consensus. */ + for (alg = DIGEST_SHA256; alg < N_DIGEST_ALGORITHMS; ++alg) { + char d[HEX_DIGEST256_LEN+1]; + const char *alg_name = + crypto_digest_algorithm_get_name(alg); + if (tor_mem_is_zero(ns->digests.d[alg], DIGEST256_LEN)) continue; - ++n_sigs; - base16_encode(sk, sizeof(sk), v->signing_key_digest, DIGEST_LEN); - base16_encode(id, sizeof(id), v->identity_digest, DIGEST_LEN); - tor_snprintf(buf, sizeof(buf), - "directory-signature %s %s\n-----BEGIN SIGNATURE-----\n", - id, sk); - smartlist_add(elements, tor_strdup(buf)); - base64_encode(buf, sizeof(buf), v->signature, v->signature_len); - strlcat(buf, "-----END SIGNATURE-----\n", sizeof(buf)); + base16_encode(d, sizeof(d), ns->digests.d[alg], DIGEST256_LEN); + tor_snprintf(buf, sizeof(buf), "additional-digest %s %s %s\n", + flavor_name, alg_name, d); smartlist_add(elements, tor_strdup(buf)); - }); + } + } SMARTLIST_FOREACH_END(ns); + + /* Now get all the sigs for non-FLAV_NS consensuses */ + SMARTLIST_FOREACH_BEGIN(consensuses, networkstatus_t *, ns) { + char *sigs; + if (ns->flavor == FLAV_NS) + continue; + sigs = networkstatus_format_signatures(ns, 1); + if (!sigs) { + log_warn(LD_DIR, "Couldn't format signatures"); + goto err; + } + smartlist_add(elements, sigs); + } SMARTLIST_FOREACH_END(ns); - result = smartlist_join_strings(elements, "", 0, NULL); + /* Now add the FLAV_NS consensus signatrures. */ + sigs = networkstatus_format_signatures(consensus_ns, 1); + if (!sigs) + goto err; + smartlist_add(elements, sigs); + result = smartlist_join_strings(elements, "", 0, NULL); + err: SMARTLIST_FOREACH(elements, char *, cp, tor_free(cp)); smartlist_free(elements); - if (!n_sigs) - tor_free(result); return result; } +/** Return a newly allocated string holding a detached-signatures document for + * all of the in-progress consensuses in the <b>n_flavors</b>-element array at + * <b>pending</b>. */ +static char * +get_detached_signatures_from_pending_consensuses(pending_consensus_t *pending, + int n_flavors) +{ + int flav; + char *signatures; + smartlist_t *c = smartlist_create(); + for (flav = 0; flav < n_flavors; ++flav) { + if (pending[flav].consensus) + smartlist_add(c, pending[flav].consensus); + } + signatures = networkstatus_get_detached_signatures(c); + smartlist_free(c); + return signatures; +} + /** Release all storage held in <b>s</b>. */ void ns_detached_signatures_free(ns_detached_signatures_t *s) { + if (!s) + return; if (s->signatures) { - SMARTLIST_FOREACH(s->signatures, networkstatus_voter_info_t *, v, - { - tor_free(v->signature); - tor_free(v); - }); - smartlist_free(s->signatures); + STRMAP_FOREACH(s->signatures, flavor, smartlist_t *, sigs) { + SMARTLIST_FOREACH(sigs, document_signature_t *, sig, + document_signature_free(sig)); + smartlist_free(sigs); + } STRMAP_FOREACH_END; + strmap_free(s->signatures, NULL); + strmap_free(s->digests, _tor_free); } + tor_free(s); } @@ -1513,7 +2387,7 @@ dirvote_act(or_options_t *options, time_t now) if (voting_schedule.voting_ends < now && !voting_schedule.have_built_consensus) { log_notice(LD_DIR, "Time to compute a consensus."); - dirvote_compute_consensus(); + dirvote_compute_consensuses(); /* XXXX We will want to try again later if we haven't got enough * votes yet. Implement this if it turns out to ever happen. */ voting_schedule.have_built_consensus = 1; @@ -1550,14 +2424,13 @@ static smartlist_t *pending_vote_list = NULL; /** List of pending_vote_t for the previous vote. After we've used them to * build a consensus, the votes go here for the next period. */ static smartlist_t *previous_vote_list = NULL; -/** The body of the consensus that we're currently building. Once we - * have it built, it goes into dirserv.c */ -static char *pending_consensus_body = NULL; + +static pending_consensus_t pending_consensuses[N_CONSENSUS_FLAVORS]; + /** The detached signatures for the consensus that we're currently * building. */ static char *pending_consensus_signatures = NULL; -/** The parsed in-progress consensus document. */ -static networkstatus_t *pending_consensus = NULL; + /** List of ns_detached_signatures_t: hold signatures that get posted to us * before we have generated the consensus on our own. */ static smartlist_t *pending_consensus_signature_list = NULL; @@ -1651,15 +2524,39 @@ dirvote_fetch_missing_votes(void) static void dirvote_fetch_missing_signatures(void) { - if (!pending_consensus) + int need_any = 0; + int i; + for (i=0; i < N_CONSENSUS_FLAVORS; ++i) { + networkstatus_t *consensus = pending_consensuses[i].consensus; + if (!consensus || + networkstatus_check_consensus_signature(consensus, -1) == 1) { + /* We have no consensus, or we have one that's signed by everybody. */ + continue; + } + need_any = 1; + } + if (!need_any) return; - if (networkstatus_check_consensus_signature(pending_consensus, -1) == 1) - return; /* we have a signature from everybody. */ directory_get_from_all_authorities(DIR_PURPOSE_FETCH_DETACHED_SIGNATURES, 0, NULL); } +/** Release all storage held by pending consensuses (those waiting for + * signatures). */ +static void +dirvote_clear_pending_consensuses(void) +{ + int i; + for (i = 0; i < N_CONSENSUS_FLAVORS; ++i) { + pending_consensus_t *pc = &pending_consensuses[i]; + tor_free(pc->body); + + networkstatus_vote_free(pc->consensus); + pc->consensus = NULL; + } +} + /** Drop all currently pending votes, consensus, and detached signatures. */ static void dirvote_clear_votes(int all_votes) @@ -1697,12 +2594,8 @@ dirvote_clear_votes(int all_votes) tor_free(cp)); smartlist_clear(pending_consensus_signature_list); } - tor_free(pending_consensus_body); tor_free(pending_consensus_signatures); - if (pending_consensus) { - networkstatus_vote_free(pending_consensus); - pending_consensus = NULL; - } + dirvote_clear_pending_consensuses(); } /** Return a newly allocated string containing the hex-encoded v3 authority @@ -1760,7 +2653,13 @@ dirvote_add_vote(const char *vote_body, const char **msg_out, int *status_out) } tor_assert(smartlist_len(vote->voters) == 1); vi = get_voter(vote); - tor_assert(vi->good_signature == 1); + { + int any_sig_good = 0; + SMARTLIST_FOREACH(vi->sigs, document_signature_t *, sig, + if (sig->good_signature) + any_sig_good = 1); + tor_assert(any_sig_good); + } ds = trusteddirserver_get_by_v3_auth_digest(vi->identity_digest); if (!ds) { char *keys = list_v3_auth_ids(); @@ -1797,6 +2696,9 @@ dirvote_add_vote(const char *vote_body, const char **msg_out, int *status_out) goto err; } + /* Fetch any new router descriptors we just learned about */ + update_consensus_router_descriptor_downloads(time(NULL), 1, vote); + /* Now see whether we already have a vote from this authority. */ SMARTLIST_FOREACH(pending_vote_list, pending_vote_t *, v, { if (! memcmp(v->vote->cert->cache_info.identity_digest, @@ -1805,7 +2707,8 @@ dirvote_add_vote(const char *vote_body, const char **msg_out, int *status_out) networkstatus_voter_info_t *vi_old = get_voter(v->vote); if (!memcmp(vi_old->vote_digest, vi->vote_digest, DIGEST_LEN)) { /* Ah, it's the same vote. Not a problem. */ - log_info(LD_DIR, "Discarding a vote we already have."); + log_info(LD_DIR, "Discarding a vote we already have (from %s).", + vi->address); if (*status_out < 200) *status_out = 200; goto discard; @@ -1832,7 +2735,7 @@ dirvote_add_vote(const char *vote_body, const char **msg_out, int *status_out) goto err; } } - }); + }); pending_vote = tor_malloc_zero(sizeof(pending_vote_t)); pending_vote->vote_body = new_cached_dir(tor_strndup(vote_body, @@ -1856,8 +2759,7 @@ dirvote_add_vote(const char *vote_body, const char **msg_out, int *status_out) *status_out = 400; discard: - if (vote) - networkstatus_vote_free(vote); + networkstatus_vote_free(vote); if (end_of_vote && !strcmpstart(end_of_vote, "network-status-version ")) { vote_body = end_of_vote; @@ -1884,14 +2786,18 @@ dirvote_add_vote(const char *vote_body, const char **msg_out, int *status_out) * pending_consensus: it won't be ready to be published until we have * everybody else's signatures collected too. (V3 Authority only) */ static int -dirvote_compute_consensus(void) +dirvote_compute_consensuses(void) { /* Have we got enough votes to try? */ - int n_votes, n_voters; + int n_votes, n_voters, n_vote_running = 0; smartlist_t *votes = NULL, *votestrings = NULL; char *consensus_body = NULL, *signatures = NULL, *votefile; networkstatus_t *consensus = NULL; authority_cert_t *my_cert; + pending_consensus_t pending[N_CONSENSUS_FLAVORS]; + int flav; + + memset(pending, 0, sizeof(pending)); if (!pending_vote_list) pending_vote_list = smartlist_create(); @@ -1903,6 +2809,19 @@ dirvote_compute_consensus(void) "%d of %d", n_votes, n_voters/2); goto err; } + tor_assert(pending_vote_list); + SMARTLIST_FOREACH(pending_vote_list, pending_vote_t *, v, { + if (smartlist_string_isin(v->vote->known_flags, "Running")) + n_vote_running++; + }); + if (!n_vote_running) { + /* See task 1066. */ + log_warn(LD_DIR, "Nobody has voted on the Running flag. Generating " + "and publishing a consensus without Running nodes " + "would make many clients stop working. Not " + "generating a consensus!"); + goto err; + } if (!(my_cert = get_my_v3_authority_cert())) { log_warn(LD_DIR, "Can't generate consensus without a certificate."); @@ -1931,6 +2850,7 @@ dirvote_compute_consensus(void) char legacy_dbuf[DIGEST_LEN]; crypto_pk_env_t *legacy_sign=NULL; char *legacy_id_digest = NULL; + int n_generated = 0; if (get_options()->V3AuthUseLegacyKey) { authority_cert_t *cert = get_my_v3_legacy_cert(); legacy_sign = get_my_v3_legacy_signing_key(); @@ -1939,39 +2859,58 @@ dirvote_compute_consensus(void) legacy_id_digest = legacy_dbuf; } } - consensus_body = networkstatus_compute_consensus( + + for (flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) { + const char *flavor_name = networkstatus_get_flavor_name(flav); + consensus_body = networkstatus_compute_consensus( votes, n_voters, my_cert->identity_key, - get_my_v3_authority_signing_key(), legacy_id_digest, legacy_sign); - } - if (!consensus_body) { - log_warn(LD_DIR, "Couldn't generate a consensus at all!"); - goto err; - } - consensus = networkstatus_parse_vote_from_string(consensus_body, NULL, - NS_TYPE_CONSENSUS); - if (!consensus) { - log_warn(LD_DIR, "Couldn't parse consensus we generated!"); - goto err; + get_my_v3_authority_signing_key(), legacy_id_digest, legacy_sign, + flav); + + if (!consensus_body) { + log_warn(LD_DIR, "Couldn't generate a %s consensus at all!", + flavor_name); + continue; + } + consensus = networkstatus_parse_vote_from_string(consensus_body, NULL, + NS_TYPE_CONSENSUS); + if (!consensus) { + log_warn(LD_DIR, "Couldn't parse %s consensus we generated!", + flavor_name); + tor_free(consensus_body); + continue; + } + + /* 'Check' our own signature, to mark it valid. */ + networkstatus_check_consensus_signature(consensus, -1); + + pending[flav].body = consensus_body; + pending[flav].consensus = consensus; + n_generated++; + consensus_body = NULL; + consensus = NULL; + } + if (!n_generated) { + log_warn(LD_DIR, "Couldn't generate any consensus flavors at all."); + goto err; + } } - /* 'Check' our own signature, to mark it valid. */ - networkstatus_check_consensus_signature(consensus, -1); - signatures = networkstatus_get_detached_signatures(consensus); + signatures = get_detached_signatures_from_pending_consensuses( + pending, N_CONSENSUS_FLAVORS); + if (!signatures) { log_warn(LD_DIR, "Couldn't extract signatures."); goto err; } - tor_free(pending_consensus_body); - pending_consensus_body = consensus_body; + dirvote_clear_pending_consensuses(); + memcpy(pending_consensuses, pending, sizeof(pending)); + tor_free(pending_consensus_signatures); pending_consensus_signatures = signatures; - if (pending_consensus) - networkstatus_vote_free(pending_consensus); - pending_consensus = consensus; - if (pending_consensus_signature_list) { int n_sigs = 0; /* we may have gotten signatures for this consensus before we built @@ -1979,7 +2918,7 @@ dirvote_compute_consensus(void) SMARTLIST_FOREACH(pending_consensus_signature_list, char *, sig, { const char *msg = NULL; - int r = dirvote_add_signatures_to_pending_consensus(sig, &msg); + int r = dirvote_add_signatures_to_all_pending_consensuses(sig, &msg); if (r >= 0) n_sigs += r; else @@ -2005,8 +2944,7 @@ dirvote_compute_consensus(void) return 0; err: - if (votes) - smartlist_free(votes); + smartlist_free(votes); tor_free(consensus_body); tor_free(signatures); networkstatus_vote_free(consensus); @@ -2015,75 +2953,60 @@ dirvote_compute_consensus(void) } /** Helper: we just got the <b>detached_signatures_body</b> sent to us as - * signatures on the currently pending consensus. Add them to the consensus + * signatures on the currently pending consensus. Add them to <b>pc</b> * as appropriate. Return the number of signatures added. (?) */ static int dirvote_add_signatures_to_pending_consensus( - const char *detached_signatures_body, + pending_consensus_t *pc, + ns_detached_signatures_t *sigs, const char **msg_out) { - ns_detached_signatures_t *sigs = NULL; + const char *flavor_name; int r = -1; - tor_assert(detached_signatures_body); - tor_assert(msg_out); - /* Only call if we have a pending consensus right now. */ - tor_assert(pending_consensus); - tor_assert(pending_consensus_body); + tor_assert(pc->consensus); + tor_assert(pc->body); tor_assert(pending_consensus_signatures); + flavor_name = networkstatus_get_flavor_name(pc->consensus->flavor); *msg_out = NULL; - if (!(sigs = networkstatus_parse_detached_signatures( - detached_signatures_body, NULL))) { - *msg_out = "Couldn't parse detached signatures."; - goto err; + { + smartlist_t *sig_list = strmap_get(sigs->signatures, flavor_name); + log_info(LD_DIR, "Have %d signatures for adding to %s consensus.", + sig_list ? smartlist_len(sig_list) : 0, flavor_name); } - - log_info(LD_DIR, "Have %d signatures for adding to consensus.", - smartlist_len(sigs->signatures)); - r = networkstatus_add_detached_signatures(pending_consensus, - sigs, msg_out); + r = networkstatus_add_detached_signatures(pc->consensus, sigs, msg_out); log_info(LD_DIR,"Added %d signatures to consensus.", r); if (r >= 1) { - char *new_detached = - networkstatus_get_detached_signatures(pending_consensus); - const char *src; + char *new_signatures = + networkstatus_format_signatures(pc->consensus, 0); char *dst, *dst_end; size_t new_consensus_len; - if (!new_detached) { + if (!new_signatures) { *msg_out = "No signatures to add"; goto err; } new_consensus_len = - strlen(pending_consensus_body) + strlen(new_detached) + 1; - pending_consensus_body = tor_realloc(pending_consensus_body, - new_consensus_len); - dst_end = pending_consensus_body + new_consensus_len; - dst = strstr(pending_consensus_body, "directory-signature "); + strlen(pc->body) + strlen(new_signatures) + 1; + pc->body = tor_realloc(pc->body, new_consensus_len); + dst_end = pc->body + new_consensus_len; + dst = strstr(pc->body, "directory-signature "); tor_assert(dst); - src = strstr(new_detached, "directory-signature "); - tor_assert(src); - strlcpy(dst, src, dst_end-dst); + strlcpy(dst, new_signatures, dst_end-dst); /* We remove this block once it has failed to crash for a while. But * unless it shows up in profiles, we're probably better leaving it in, * just in case we break detached signature processing at some point. */ { - ns_detached_signatures_t *sigs = - networkstatus_parse_detached_signatures(new_detached, NULL); networkstatus_t *v = networkstatus_parse_vote_from_string( - pending_consensus_body, NULL, + pc->body, NULL, NS_TYPE_CONSENSUS); - tor_assert(sigs); - ns_detached_signatures_free(sigs); tor_assert(v); networkstatus_vote_free(v); } - tor_free(pending_consensus_signatures); - pending_consensus_signatures = new_detached; *msg_out = "Signatures added"; } else if (r == 0) { *msg_out = "Signatures ignored"; @@ -2096,8 +3019,62 @@ dirvote_add_signatures_to_pending_consensus( if (!*msg_out) *msg_out = "Unrecognized error while adding detached signatures."; done: - if (sigs) - ns_detached_signatures_free(sigs); + return r; +} + +static int +dirvote_add_signatures_to_all_pending_consensuses( + const char *detached_signatures_body, + const char **msg_out) +{ + int r=0, i, n_added = 0, errors = 0; + ns_detached_signatures_t *sigs; + tor_assert(detached_signatures_body); + tor_assert(msg_out); + tor_assert(pending_consensus_signatures); + + if (!(sigs = networkstatus_parse_detached_signatures( + detached_signatures_body, NULL))) { + *msg_out = "Couldn't parse detached signatures."; + goto err; + } + + for (i = 0; i < N_CONSENSUS_FLAVORS; ++i) { + int res; + pending_consensus_t *pc = &pending_consensuses[i]; + if (!pc->consensus) + continue; + res = dirvote_add_signatures_to_pending_consensus(pc, sigs, msg_out); + if (res < 0) + errors++; + else + n_added += res; + } + + if (errors && !n_added) { + r = -1; + goto err; + } + + if (n_added && pending_consensuses[FLAV_NS].consensus) { + char *new_detached = + get_detached_signatures_from_pending_consensuses( + pending_consensuses, N_CONSENSUS_FLAVORS); + if (new_detached) { + tor_free(pending_consensus_signatures); + pending_consensus_signatures = new_detached; + } + } + + r = n_added; + goto done; + err: + if (!*msg_out) + *msg_out = "Unrecognized error while adding detached signatures."; + done: + ns_detached_signatures_free(sigs); + /* XXXX NM Check how return is used. We can now have an error *and* + signatures added. */ return r; } @@ -2110,10 +3087,10 @@ dirvote_add_signatures(const char *detached_signatures_body, const char *source, const char **msg) { - if (pending_consensus) { + if (pending_consensuses[FLAV_NS].consensus) { log_notice(LD_DIR, "Got a signature from %s. " "Adding it to the pending consensus.", source); - return dirvote_add_signatures_to_pending_consensus( + return dirvote_add_signatures_to_all_pending_consensuses( detached_signatures_body, msg); } else { log_notice(LD_DIR, "Got a signature from %s. " @@ -2132,17 +3109,25 @@ dirvote_add_signatures(const char *detached_signatures_body, static int dirvote_publish_consensus(void) { - /* Can we actually publish it yet? */ - if (!pending_consensus || - networkstatus_check_consensus_signature(pending_consensus, 1)<0) { - log_warn(LD_DIR, "Not enough info to publish pending consensus"); - return -1; - } + int i; + + /* Now remember all the other consensuses as if we were a directory cache. */ + for (i = 0; i < N_CONSENSUS_FLAVORS; ++i) { + pending_consensus_t *pending = &pending_consensuses[i]; + const char *name; + name = networkstatus_get_flavor_name(i); + tor_assert(name); + if (!pending->consensus || + networkstatus_check_consensus_signature(pending->consensus, 1)<0) { + log_warn(LD_DIR, "Not enough info to publish pending %s consensus",name); + continue; + } - if (networkstatus_set_current_consensus(pending_consensus_body, 0)) - log_warn(LD_DIR, "Error publishing consensus"); - else - log_notice(LD_DIR, "Consensus published."); + if (networkstatus_set_current_consensus(pending->body, name, 0)) + log_warn(LD_DIR, "Error publishing %s consensus", name); + else + log_notice(LD_DIR, "Published %s consensus", name); + } return 0; } @@ -2158,12 +3143,8 @@ dirvote_free_all(void) smartlist_free(previous_vote_list); previous_vote_list = NULL; - tor_free(pending_consensus_body); + dirvote_clear_pending_consensuses(); tor_free(pending_consensus_signatures); - if (pending_consensus) { - networkstatus_vote_free(pending_consensus); - pending_consensus = NULL; - } if (pending_consensus_signature_list) { /* now empty as a result of clear_pending_votes. */ smartlist_free(pending_consensus_signature_list); @@ -2177,13 +3158,14 @@ dirvote_free_all(void) /** Return the body of the consensus that we're currently trying to build. */ const char * -dirvote_get_pending_consensus(void) +dirvote_get_pending_consensus(consensus_flavor_t flav) { - return pending_consensus_body; + tor_assert(((int)flav) >= 0 && flav < N_CONSENSUS_FLAVORS); + return pending_consensuses[flav].body; } /** Return the signatures that we know for the consensus that we're currently - * trying to build */ + * trying to build. */ const char * dirvote_get_pending_detached_signatures(void) { @@ -2229,15 +3211,147 @@ dirvote_get_vote(const char *fp, int flags) } else { if (pending_vote_list && include_pending) { SMARTLIST_FOREACH(pending_vote_list, pending_vote_t *, pv, - if (!memcmp(pv->vote->networkstatus_digest, fp, DIGEST_LEN)) + if (!memcmp(pv->vote->digests.d[DIGEST_SHA1], fp, DIGEST_LEN)) return pv->vote_body); } if (previous_vote_list && include_previous) { SMARTLIST_FOREACH(previous_vote_list, pending_vote_t *, pv, - if (!memcmp(pv->vote->networkstatus_digest, fp, DIGEST_LEN)) + if (!memcmp(pv->vote->digests.d[DIGEST_SHA1], fp, DIGEST_LEN)) return pv->vote_body); } } return NULL; } +/** Construct and return a new microdescriptor from a routerinfo <b>ri</b>. + * + * XXX Right now, there is only one way to generate microdescriptors from + * router descriptors. This may change in future consensus methods. If so, + * we'll need an internal way to remember which method we used, and ask for a + * particular method. + **/ +microdesc_t * +dirvote_create_microdescriptor(const routerinfo_t *ri) +{ + microdesc_t *result = NULL; + char *key = NULL, *summary = NULL, *family = NULL; + char buf[1024]; + size_t keylen; + char *out = buf, *end = buf+sizeof(buf); + + if (crypto_pk_write_public_key_to_string(ri->onion_pkey, &key, &keylen)<0) + goto done; + summary = policy_summarize(ri->exit_policy); + if (ri->declared_family) + family = smartlist_join_strings(ri->declared_family, " ", 0, NULL); + + if (tor_snprintf(out, end-out, "onion-key\n%s", key)<0) + goto done; + out += strlen(out); + if (family) { + if (tor_snprintf(out, end-out, "family %s\n", family)<0) + goto done; + out += strlen(out); + } + if (summary && strcmp(summary, "reject 1-65535")) { + if (tor_snprintf(out, end-out, "p %s\n", summary)<0) + goto done; + out += strlen(out); + } + *out = '\0'; /* Make sure it's nul-terminated. This should be a no-op */ + + { + smartlist_t *lst = microdescs_parse_from_string(buf, out, 0, 1); + if (smartlist_len(lst) != 1) { + log_warn(LD_DIR, "We generated a microdescriptor we couldn't parse."); + SMARTLIST_FOREACH(lst, microdesc_t *, md, microdesc_free(md)); + smartlist_free(lst); + goto done; + } + result = smartlist_get(lst, 0); + smartlist_free(lst); + } + + done: + tor_free(key); + tor_free(summary); + tor_free(family); + return result; +} + +/** Cached space-separated string to hold */ +static char *microdesc_consensus_methods = NULL; + +/** Format the appropriate vote line to describe the microdescriptor <b>md</b> + * in a consensus vote document. Write it into the <b>out_len</b>-byte buffer + * in <b>out</b>. Return -1 on failure and the number of characters written + * on success. */ +ssize_t +dirvote_format_microdesc_vote_line(char *out, size_t out_len, + const microdesc_t *md) +{ + char d64[BASE64_DIGEST256_LEN+1]; + if (!microdesc_consensus_methods) { + microdesc_consensus_methods = + make_consensus_method_list(MIN_METHOD_FOR_MICRODESC, + MAX_SUPPORTED_CONSENSUS_METHOD, + ","); + tor_assert(microdesc_consensus_methods); + } + if (digest256_to_base64(d64, md->digest)<0) + return -1; + + if (tor_snprintf(out, out_len, "m %s sha256=%s\n", + microdesc_consensus_methods, d64)<0) + return -1; + + return strlen(out); +} + +/** If <b>vrs</b> has a hash made for the consensus method <b>method</b> with + * the digest algorithm <b>alg</b>, decode it and copy it into + * <b>digest256_out</b> and return 0. Otherwise return -1. */ +int +vote_routerstatus_find_microdesc_hash(char *digest256_out, + const vote_routerstatus_t *vrs, + int method, + digest_algorithm_t alg) +{ + /* XXXX only returns the sha256 method. */ + const vote_microdesc_hash_t *h; + char mstr[64]; + size_t mlen; + char dstr[64]; + + tor_snprintf(mstr, sizeof(mstr), "%d", method); + mlen = strlen(mstr); + tor_snprintf(dstr, sizeof(dstr), " %s=", + crypto_digest_algorithm_get_name(alg)); + + for (h = vrs->microdesc; h; h = h->next) { + const char *cp = h->microdesc_hash_line; + size_t num_len; + /* cp looks like \d+(,\d+)* (digesttype=val )+ . Let's hunt for mstr in + * the first part. */ + while (1) { + num_len = strspn(cp, "1234567890"); + if (num_len == mlen && !memcmp(mstr, cp, mlen)) { + /* This is the line. */ + char buf[BASE64_DIGEST256_LEN+1]; + /* XXXX ignores extraneous stuff if the digest is too long. This + * seems harmless enough, right? */ + cp = strstr(cp, dstr); + if (!cp) + return -1; + cp += strlen(dstr); + strlcpy(buf, cp, sizeof(buf)); + return digest256_from_base64(digest256_out, buf); + } + if (num_len == 0 || cp[num_len] != ',') + break; + cp += num_len + 1; + } + } + return -1; +} + diff --git a/src/or/dirvote.h b/src/or/dirvote.h new file mode 100644 index 0000000000..e384dc53b3 --- /dev/null +++ b/src/or/dirvote.h @@ -0,0 +1,93 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2010, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file dirvote.h + * \brief Header file for dirvote.c. + **/ + +#ifndef _TOR_DIRVOTE_H +#define _TOR_DIRVOTE_H + +/** Lowest allowable value for VoteSeconds. */ +#define MIN_VOTE_SECONDS 20 +/** Lowest allowable value for DistSeconds. */ +#define MIN_DIST_SECONDS 20 +/** Smallest allowable voting interval. */ +#define MIN_VOTE_INTERVAL 300 + +/** Precision multiplier for the Bw weights */ +#define BW_WEIGHT_SCALE 10000 + +void dirvote_free_all(void); + +/* vote manipulation */ +char *networkstatus_compute_consensus(smartlist_t *votes, + int total_authorities, + crypto_pk_env_t *identity_key, + crypto_pk_env_t *signing_key, + const char *legacy_identity_key_digest, + crypto_pk_env_t *legacy_signing_key, + consensus_flavor_t flavor); +int networkstatus_add_detached_signatures(networkstatus_t *target, + ns_detached_signatures_t *sigs, + const char **msg_out); +char *networkstatus_get_detached_signatures(smartlist_t *consensuses); +void ns_detached_signatures_free(ns_detached_signatures_t *s); + +/* cert manipulation */ +authority_cert_t *authority_cert_dup(authority_cert_t *cert); + +/* vote scheduling */ +void dirvote_get_preferred_voting_intervals(vote_timing_t *timing_out); +time_t dirvote_get_start_of_next_interval(time_t now, int interval); +void dirvote_recalculate_timing(or_options_t *options, time_t now); +void dirvote_act(or_options_t *options, time_t now); + +/* invoked on timers and by outside triggers. */ +struct pending_vote_t * dirvote_add_vote(const char *vote_body, + const char **msg_out, + int *status_out); +int dirvote_add_signatures(const char *detached_signatures_body, + const char *source, + const char **msg_out); + +/* Item access */ +const char *dirvote_get_pending_consensus(consensus_flavor_t flav); +const char *dirvote_get_pending_detached_signatures(void); +#define DGV_BY_ID 1 +#define DGV_INCLUDE_PENDING 2 +#define DGV_INCLUDE_PREVIOUS 4 +const cached_dir_t *dirvote_get_vote(const char *fp, int flags); +void set_routerstatus_from_routerinfo(routerstatus_t *rs, + routerinfo_t *ri, time_t now, + int naming, int listbadexits, + int listbaddirs); +void router_clear_status_flags(routerinfo_t *ri); +networkstatus_t * +dirserv_generate_networkstatus_vote_obj(crypto_pk_env_t *private_key, + authority_cert_t *cert); + +microdesc_t *dirvote_create_microdescriptor(const routerinfo_t *ri); +ssize_t dirvote_format_microdesc_vote_line(char *out, size_t out_len, + const microdesc_t *md); + +int vote_routerstatus_find_microdesc_hash(char *digest256_out, + const vote_routerstatus_t *vrs, + int method, + digest_algorithm_t alg); +document_signature_t *voter_get_sig_by_algorithm( + const networkstatus_voter_info_t *voter, + digest_algorithm_t alg); + +#ifdef DIRVOTE_PRIVATE +char *format_networkstatus_vote(crypto_pk_env_t *private_key, + networkstatus_t *v3_ns); +char *dirvote_compute_params(smartlist_t *votes); +#endif + +#endif + diff --git a/src/or/dns.c b/src/or/dns.c index 98eb27f8b3..4e319b7d83 100644 --- a/src/or/dns.c +++ b/src/or/dns.c @@ -12,8 +12,63 @@ **/ #include "or.h" +#include "circuitlist.h" +#include "circuituse.h" +#include "config.h" +#include "connection.h" +#include "connection_edge.h" +#include "control.h" +#include "dns.h" +#include "main.h" +#include "policies.h" +#include "relay.h" +#include "router.h" #include "ht.h" +#ifdef HAVE_EVENT2_DNS_H +#include <event2/event.h> +#include <event2/dns.h> +#else +#include <event.h> #include "eventdns.h" +#ifndef HAVE_EVDNS_SET_DEFAULT_OUTGOING_BIND_ADDRESS +#define HAVE_EVDNS_SET_DEFAULT_OUTGOING_BIND_ADDRESS +#endif +#endif + +#ifndef HAVE_EVENT2_DNS_H +struct evdns_base; +struct evdns_request; +#define evdns_base_new(x,y) tor_malloc(1) +#define evdns_base_clear_nameservers_and_suspend(base) \ + evdns_clear_nameservers_and_suspend() +#define evdns_base_search_clear(base) evdns_search_clear() +#define evdns_base_set_default_outgoing_bind_address(base, a, len) \ + evdns_set_default_outgoing_bind_address((a),(len)) +#define evdns_base_resolv_conf_parse(base, options, fname) \ + evdns_resolv_conf_parse((options), (fname)) +#define evdns_base_count_nameservers(base) \ + evdns_count_nameservers() +#define evdns_base_resume(base) \ + evdns_resume() +#define evdns_base_config_windows_nameservers(base) \ + evdns_config_windows_nameservers() +#define evdns_base_set_option_(base, opt, val) \ + evdns_set_option((opt),(val),DNS_OPTIONS_ALL) +#define evdns_base_resolve_ipv4(base, addr, options, cb, ptr) \ + ((evdns_resolve_ipv4(addr, options, cb, ptr)<0) ? NULL : ((void*)1)) +#define evdns_base_resolve_reverse(base, addr, options, cb, ptr) \ + ((evdns_resolve_reverse(addr, options, cb, ptr)<0) ? NULL : ((void*)1)) +#define evdns_base_resolve_reverse_ipv6(base, addr, options, cb, ptr) \ + ((evdns_resolve_reverse_ipv6(addr, options, cb, ptr)<0) ? NULL : ((void*)1)) + +#elif defined(LIBEVENT_VERSION_NUMBER) && LIBEVENT_VERSION_NUMBER < 0x02000303 +#define evdns_base_set_option_(base, opt, val) \ + evdns_base_set_option((base), (opt),(val),DNS_OPTIONS_ALL) + +#else +#define evdns_base_set_option_ evdns_base_set_option + +#endif /** Longest hostname we're willing to resolve. */ #define MAX_ADDRESSLEN 256 @@ -28,6 +83,9 @@ #define DNS_RESOLVE_FAILED_PERMANENT 2 #define DNS_RESOLVE_SUCCEEDED 3 +/** Our evdns_base; this structure handles all our name lookups. */ +static struct evdns_base *the_evdns_base = NULL; + /** Have we currently configured nameservers with eventdns? */ static int nameservers_configured = 0; /** Did our most recent attempt to configure nameservers with eventdns fail? */ @@ -89,6 +147,8 @@ typedef struct cached_resolve_t { uint32_t ttl; /**< What TTL did the nameserver tell us? */ /** Connections that want to know when we get an answer for this resolve. */ pending_connection_t *pending_connections; + /** Position of this element in the heap*/ + int minheap_idx; } cached_resolve_t; static void purge_expired_resolves(time_t now); @@ -211,8 +271,16 @@ dns_reset(void) { or_options_t *options = get_options(); if (! server_mode(options)) { - evdns_clear_nameservers_and_suspend(); - evdns_search_clear(); + + if (!the_evdns_base) { + if (!(the_evdns_base = evdns_base_new(tor_libevent_get_base(), 0))) { + log_err(LD_BUG, "Couldn't create an evdns_base"); + return -1; + } + } + + evdns_base_clear_nameservers_and_suspend(the_evdns_base); + evdns_base_search_clear(the_evdns_base); nameservers_configured = 0; tor_free(resolv_conf_fname); resolv_conf_mtime = 0; @@ -262,6 +330,8 @@ dns_get_expiry_ttl(uint32_t ttl) static void _free_cached_resolve(cached_resolve_t *r) { + if (!r) + return; while (r->pending_connections) { pending_connection_t *victim = r->pending_connections; r->pending_connections = victim->next; @@ -303,6 +373,7 @@ set_expiry(cached_resolve_t *resolve, time_t expires) resolve->expire = expires; smartlist_pqueue_add(cached_resolve_pqueue, _compare_cached_resolves_by_expiry, + STRUCT_OFFSET(cached_resolve_t, minheap_idx), resolve); } @@ -325,8 +396,7 @@ dns_free_all(void) _free_cached_resolve(item); } HT_CLEAR(cache_map, &cache_root); - if (cached_resolve_pqueue) - smartlist_free(cached_resolve_pqueue); + smartlist_free(cached_resolve_pqueue); cached_resolve_pqueue = NULL; tor_free(resolv_conf_fname); } @@ -349,7 +419,8 @@ purge_expired_resolves(time_t now) if (resolve->expire > now) break; smartlist_pqueue_pop(cached_resolve_pqueue, - _compare_cached_resolves_by_expiry); + _compare_cached_resolves_by_expiry, + STRUCT_OFFSET(cached_resolve_t, minheap_idx)); if (resolve->state == CACHE_STATE_PENDING) { log_debug(LD_EXIT, @@ -393,7 +464,7 @@ purge_expired_resolves(time_t now) log_err(LD_BUG, "The expired resolve we purged didn't match any in" " the cache. Tried to purge %s (%p); instead got %s (%p).", resolve->address, (void*)resolve, - removed ? removed->address : "NULL", (void*)remove); + removed ? removed->address : "NULL", (void*)removed); } tor_assert(removed == resolve); } else { @@ -611,7 +682,7 @@ dns_resolve_impl(edge_connection_t *exitconn, int is_resolve, * know the answer. */ if (tor_addr_from_str(&addr, exitconn->_base.address) >= 0) { if (tor_addr_family(&addr) == AF_INET) { - tor_addr_assign(&exitconn->_base.addr, &addr); + tor_addr_copy(&exitconn->_base.addr, &addr); exitconn->address_ttl = DEFAULT_DNS_TTL; return 1; } else { @@ -711,6 +782,7 @@ dns_resolve_impl(edge_connection_t *exitconn, int is_resolve, resolve = tor_malloc_zero(sizeof(cached_resolve_t)); resolve->magic = CACHED_RESOLVE_MAGIC; resolve->state = CACHE_STATE_PENDING; + resolve->minheap_idx = -1; resolve->is_reverse = is_reverse; strlcpy(resolve->address, exitconn->_base.address, sizeof(resolve->address)); @@ -807,7 +879,8 @@ connection_dns_remove(edge_connection_t *conn) tor_free(pend); log_debug(LD_EXIT, "First connection (fd %d) no longer waiting " "for resolve of %s", - conn->_base.s, escaped_safe_str(conn->_base.address)); + conn->_base.s, + escaped_safe_str(conn->_base.address)); return; } else { for ( ; pend->next; pend = pend->next) { @@ -1108,6 +1181,14 @@ configure_nameservers(int force) conf_fname = "/etc/resolv.conf"; #endif + if (!the_evdns_base) { + if (!(the_evdns_base = evdns_base_new(tor_libevent_get_base(), 0))) { + log_err(LD_BUG, "Couldn't create an evdns_base"); + return -1; + } + } + +#ifdef HAVE_EVDNS_SET_DEFAULT_OUTGOING_BIND_ADDRESS if (options->OutboundBindAddress) { tor_addr_t addr; if (tor_addr_from_str(&addr, options->OutboundBindAddress) < 0) { @@ -1122,16 +1203,13 @@ configure_nameservers(int force) log_warn(LD_BUG, "Couldn't convert outbound bind address to sockaddr." " Ignoring."); } else { - evdns_set_default_outgoing_bind_address((struct sockaddr *)&ss, - socklen); + evdns_base_set_default_outgoing_bind_address(the_evdns_base, + (struct sockaddr *)&ss, + socklen); } } } - - if (options->ServerDNSRandomizeCase) - evdns_set_option("randomize-case:", "1", DNS_OPTIONS_ALL); - else - evdns_set_option("randomize-case:", "0", DNS_OPTIONS_ALL); +#endif evdns_set_log_fn(evdns_log_cb); if (conf_fname) { @@ -1146,16 +1224,17 @@ configure_nameservers(int force) return 0; } if (nameservers_configured) { - evdns_search_clear(); - evdns_clear_nameservers_and_suspend(); + evdns_base_search_clear(the_evdns_base); + evdns_base_clear_nameservers_and_suspend(the_evdns_base); } log_info(LD_EXIT, "Parsing resolver configuration in '%s'", conf_fname); - if ((r = evdns_resolv_conf_parse(DNS_OPTIONS_ALL, conf_fname))) { + if ((r = evdns_base_resolv_conf_parse(the_evdns_base, + DNS_OPTIONS_ALL, conf_fname))) { log_warn(LD_EXIT, "Unable to parse '%s', or no nameservers in '%s' (%d)", conf_fname, conf_fname, r); goto err; } - if (evdns_count_nameservers() == 0) { + if (evdns_base_count_nameservers(the_evdns_base) == 0) { log_warn(LD_EXIT, "Unable to find any nameservers in '%s'.", conf_fname); goto err; } @@ -1163,38 +1242,47 @@ configure_nameservers(int force) resolv_conf_fname = tor_strdup(conf_fname); resolv_conf_mtime = st.st_mtime; if (nameservers_configured) - evdns_resume(); + evdns_base_resume(the_evdns_base); } #ifdef MS_WINDOWS else { if (nameservers_configured) { - evdns_search_clear(); - evdns_clear_nameservers_and_suspend(); + evdns_base_search_clear(the_evdns_base); + evdns_base_clear_nameservers_and_suspend(the_evdns_base); } - if (evdns_config_windows_nameservers()) { + if (evdns_base_config_windows_nameservers(the_evdns_base)) { log_warn(LD_EXIT,"Could not config nameservers."); goto err; } - if (evdns_count_nameservers() == 0) { + if (evdns_base_count_nameservers(the_evdns_base) == 0) { log_warn(LD_EXIT, "Unable to find any platform nameservers in " "your Windows configuration."); goto err; } if (nameservers_configured) - evdns_resume(); + evdns_base_resume(the_evdns_base); tor_free(resolv_conf_fname); resolv_conf_mtime = 0; } #endif - if (evdns_count_nameservers() == 1) { - evdns_set_option("max-timeouts:", "16", DNS_OPTIONS_ALL); - evdns_set_option("timeout:", "10", DNS_OPTIONS_ALL); +#define SET(k,v) evdns_base_set_option_(the_evdns_base, (k), (v)) + + if (evdns_base_count_nameservers(the_evdns_base) == 1) { + SET("max-timeouts:", "16"); + SET("timeout:", "10"); } else { - evdns_set_option("max-timeouts:", "3", DNS_OPTIONS_ALL); - evdns_set_option("timeout:", "5", DNS_OPTIONS_ALL); + SET("max-timeouts:", "3"); + SET("timeout:", "5"); } + if (options->ServerDNSRandomizeCase) + SET("randomize-case:", "1"); + else + SET("randomize-case:", "0"); + +#undef SET + dns_servers_relaunch_checks(); nameservers_configured = 1; @@ -1292,6 +1380,7 @@ static int launch_resolve(edge_connection_t *exitconn) { char *addr = tor_strdup(exitconn->_base.address); + struct evdns_request *req = NULL; tor_addr_t a; int r; int options = get_options()->ServerDNSSearchDomains ? 0 @@ -1307,28 +1396,34 @@ launch_resolve(edge_connection_t *exitconn) r = tor_addr_parse_reverse_lookup_name( &a, exitconn->_base.address, AF_UNSPEC, 0); + + tor_assert(the_evdns_base); if (r == 0) { log_info(LD_EXIT, "Launching eventdns request for %s", escaped_safe_str(exitconn->_base.address)); - r = evdns_resolve_ipv4(exitconn->_base.address, options, - evdns_callback, addr); + req = evdns_base_resolve_ipv4(the_evdns_base, + exitconn->_base.address, options, + evdns_callback, addr); } else if (r == 1) { log_info(LD_EXIT, "Launching eventdns reverse request for %s", escaped_safe_str(exitconn->_base.address)); if (tor_addr_family(&a) == AF_INET) - r = evdns_resolve_reverse(tor_addr_to_in(&a), DNS_QUERY_NO_SEARCH, + req = evdns_base_resolve_reverse(the_evdns_base, + tor_addr_to_in(&a), DNS_QUERY_NO_SEARCH, evdns_callback, addr); else - r = evdns_resolve_reverse_ipv6(tor_addr_to_in6(&a), DNS_QUERY_NO_SEARCH, + req = evdns_base_resolve_reverse_ipv6(the_evdns_base, + tor_addr_to_in6(&a), DNS_QUERY_NO_SEARCH, evdns_callback, addr); } else if (r == -1) { log_warn(LD_BUG, "Somehow a malformed in-addr.arpa address reached here."); } - if (r) { - log_warn(LD_EXIT, "eventdns rejected address %s: error %d.", - escaped_safe_str(addr), r); - r = evdns_err_is_transient(r) ? -2 : -1; + r = 0; + if (!req) { + log_warn(LD_EXIT, "eventdns rejected address %s.", + escaped_safe_str(addr)); + r = -1; tor_free(addr); /* There is no evdns request in progress; stop * addr from getting leaked. */ } @@ -1449,8 +1544,8 @@ evdns_wildcard_check_callback(int result, char type, int count, int ttl, } log(dns_wildcard_one_notice_given ? LOG_INFO : LOG_NOTICE, LD_EXIT, "Your DNS provider gave an answer for \"%s\", which " - "is not supposed to exist. Apparently they are hijacking " - "DNS failures. Trying to correct for this. We've noticed %d " + "is not supposed to exist. Apparently they are hijacking " + "DNS failures. Trying to correct for this. We've noticed %d " "possibly bad address%s so far.", string_address, strmap_size(dns_wildcard_response_count), (strmap_size(dns_wildcard_response_count) == 1) ? "" : "es"); @@ -1466,17 +1561,20 @@ static void launch_wildcard_check(int min_len, int max_len, const char *suffix) { char *addr; - int r; + struct evdns_request *req; addr = crypto_random_hostname(min_len, max_len, "", suffix); log_info(LD_EXIT, "Testing whether our DNS server is hijacking nonexistent " "domains with request for bogus hostname \"%s\"", addr); - r = evdns_resolve_ipv4(/* This "addr" tells us which address to resolve */ + tor_assert(the_evdns_base); + req = evdns_base_resolve_ipv4( + the_evdns_base, + /* This "addr" tells us which address to resolve */ addr, DNS_QUERY_NO_SEARCH, evdns_wildcard_check_callback, /* This "addr" is an argument to the callback*/ addr); - if (r) { + if (!req) { /* There is no evdns request in progress; stop addr from getting leaked */ tor_free(addr); } @@ -1488,6 +1586,7 @@ static void launch_test_addresses(int fd, short event, void *args) { or_options_t *options = get_options(); + struct evdns_request *req; (void)fd; (void)event; (void)args; @@ -1499,14 +1598,19 @@ launch_test_addresses(int fd, short event, void *args) * be an exit server.*/ if (!options->ServerDNSTestAddresses) return; - SMARTLIST_FOREACH(options->ServerDNSTestAddresses, const char *, address, - { - int r = evdns_resolve_ipv4(address, DNS_QUERY_NO_SEARCH, evdns_callback, - tor_strdup(address)); - if (r) - log_info(LD_EXIT, "eventdns rejected test address %s: error %d", - escaped_safe_str(address), r); - }); + tor_assert(the_evdns_base); + SMARTLIST_FOREACH_BEGIN(options->ServerDNSTestAddresses, + const char *, address) { + char *a = tor_strdup(address); + req = evdns_base_resolve_ipv4(the_evdns_base, + address, DNS_QUERY_NO_SEARCH, evdns_callback, a); + + if (!req) { + log_info(LD_EXIT, "eventdns rejected test address %s", + escaped_safe_str(address)); + tor_free(a); + } + } SMARTLIST_FOREACH_END(address); } #define N_WILDCARD_CHECKS 2 @@ -1547,7 +1651,7 @@ dns_launch_wildcard_checks(void) void dns_launch_correctness_checks(void) { - static struct event launch_event; + static struct event *launch_event = NULL; struct timeval timeout; if (!get_options()->ServerDNSDetectHijacking) return; @@ -1555,10 +1659,12 @@ dns_launch_correctness_checks(void) /* Wait a while before launching requests for test addresses, so we can * get the results from checking for wildcarding. */ - evtimer_set(&launch_event, launch_test_addresses, NULL); + if (! launch_event) + launch_event = tor_evtimer_new(tor_libevent_get_base(), + launch_test_addresses, NULL); timeout.tv_sec = 30; timeout.tv_usec = 0; - if (evtimer_add(&launch_event, &timeout)<0) { + if (evtimer_add(launch_event, &timeout)<0) { log_warn(LD_BUG, "Couldn't add timer for checking for dns hijacking"); } } @@ -1574,10 +1680,9 @@ dns_seems_to_be_broken(void) void dns_reset_correctness_checks(void) { - if (dns_wildcard_response_count) { - strmap_free(dns_wildcard_response_count, _tor_free); - dns_wildcard_response_count = NULL; - } + strmap_free(dns_wildcard_response_count, _tor_free); + dns_wildcard_response_count = NULL; + n_wildcard_requests = 0; if (dns_wildcard_list) { @@ -1622,6 +1727,30 @@ assert_resolve_ok(cached_resolve_t *resolve) } } +/** Return the number of DNS cache entries as an int */ +static int +dns_cache_entry_count(void) +{ + return HT_SIZE(&cache_root); +} + +/** Log memory information about our internal DNS cache at level 'severity'. */ +void +dump_dns_mem_usage(int severity) +{ + /* This should never be larger than INT_MAX. */ + int hash_count = dns_cache_entry_count(); + size_t hash_mem = sizeof(struct cached_resolve_t) * hash_count; + hash_mem += HT_MEM_USAGE(&cache_root); + + /* Print out the count and estimated size of our &cache_root. It undercounts + hostnames in cached reverse resolves. + */ + log(severity, LD_MM, "Our DNS cache has %d entries.", hash_count); + log(severity, LD_MM, "Our DNS cache size is approximately %u bytes.", + (unsigned)hash_mem); +} + #ifdef DEBUG_DNS_CACHE /** Exit with an assertion if the DNS cache is corrupt. */ static void @@ -1642,7 +1771,8 @@ _assert_cache_ok(void) return; smartlist_pqueue_assert_ok(cached_resolve_pqueue, - _compare_cached_resolves_by_expiry); + _compare_cached_resolves_by_expiry, + STRUCT_OFFSET(cached_resolve_t, minheap_idx)); SMARTLIST_FOREACH(cached_resolve_pqueue, cached_resolve_t *, res, { diff --git a/src/or/dns.h b/src/or/dns.h new file mode 100644 index 0000000000..c4fd4d1fb7 --- /dev/null +++ b/src/or/dns.h @@ -0,0 +1,31 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2010, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file dns.h + * \brief Header file for dns.c. + **/ + +#ifndef _TOR_DNS_H +#define _TOR_DNS_H + +int dns_init(void); +int has_dns_init_failed(void); +void dns_free_all(void); +uint32_t dns_clip_ttl(uint32_t ttl); +int dns_reset(void); +void connection_dns_remove(edge_connection_t *conn); +void assert_connection_edge_not_dns_pending(edge_connection_t *conn); +void assert_all_pending_dns_resolves_ok(void); +void dns_cancel_pending_resolve(const char *question); +int dns_resolve(edge_connection_t *exitconn); +void dns_launch_correctness_checks(void); +int dns_seems_to_be_broken(void); +void dns_reset_correctness_checks(void); +void dump_dns_mem_usage(int severity); + +#endif + diff --git a/src/or/dnsserv.c b/src/or/dnsserv.c index bceaa462b3..e231b655f4 100644 --- a/src/or/dnsserv.c +++ b/src/or/dnsserv.c @@ -9,7 +9,21 @@ **/ #include "or.h" +#include "dnsserv.h" +#include "config.h" +#include "connection.h" +#include "connection_edge.h" +#include "control.h" +#include "main.h" +#include "policies.h" +#ifdef HAVE_EVENT2_DNS_H +#include <event2/dns.h> +#include <event2/dns_compat.h> +/* XXXX022 this implies we want an improved evdns */ +#include <event2/dns_struct.h> +#else #include "eventdns.h" +#endif /** Helper function: called by evdns whenever the client sends a request to our * DNSPort. We need to eventually answer the request <b>req</b>. @@ -85,12 +99,7 @@ evdns_server_callback(struct evdns_server_request *req, void *_data) evdns_server_request_respond(req, DNS_ERR_NONE); return; } - if (q->type == EVDNS_TYPE_A) { - /* Refuse any attempt to resolve a noconnect address, right now. */ - if (hostname_is_noconnect_address(q->name)) { - err = DNS_ERR_REFUSED; - } - } else { + if (q->type != EVDNS_TYPE_A) { tor_assert(q->type == EVDNS_TYPE_PTR); } @@ -136,13 +145,13 @@ evdns_server_callback(struct evdns_server_request *req, void *_data) * immediately if it's in the cache, or completely bogus, or automapped), * and then attached to a circuit. */ log_info(LD_APP, "Passing request for %s to rewrite_and_attach.", - escaped_safe_str(q->name)); + escaped_safe_str_client(q->name)); q_name = tor_strdup(q->name); /* q could be freed in rewrite_and_attach */ connection_ap_handshake_rewrite_and_attach(conn, NULL, NULL); /* Now, the connection is marked if it was bad. */ log_info(LD_APP, "Passed request for %s to rewrite_and_attach.", - escaped_safe_str(q_name)); + escaped_safe_str_client(q_name)); tor_free(q_name); } @@ -181,13 +190,13 @@ dnsserv_launch_request(const char *name, int reverse) * immediately if it's in the cache, or completely bogus, or automapped), * and then attached to a circuit. */ log_info(LD_APP, "Passing request for %s to rewrite_and_attach.", - escaped_safe_str(name)); + escaped_safe_str_client(name)); q_name = tor_strdup(name); /* q could be freed in rewrite_and_attach */ connection_ap_handshake_rewrite_and_attach(conn, NULL, NULL); /* Now, the connection is marked if it was bad. */ log_info(LD_APP, "Passed request for %s to rewrite_and_attach.", - escaped_safe_str(q_name)); + escaped_safe_str_client(q_name)); tor_free(q_name); return 0; } @@ -297,8 +306,8 @@ dnsserv_configure_listener(connection_t *conn) tor_assert(conn->s >= 0); tor_assert(conn->type == CONN_TYPE_AP_DNS_LISTENER); - conn->dns_server_port = evdns_add_server_port(conn->s, 0, - evdns_server_callback, NULL); + conn->dns_server_port = + tor_evdns_add_server_port(conn->s, 0, evdns_server_callback, NULL); } /** Free the evdns server port for <b>conn</b>, which must be an diff --git a/src/or/dnsserv.h b/src/or/dnsserv.h new file mode 100644 index 0000000000..5bf154ebb2 --- /dev/null +++ b/src/or/dnsserv.h @@ -0,0 +1,26 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2010, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file dnsserv.h + * \brief Header file for dnsserv.c. + **/ + +#ifndef _TOR_DNSSERV_H +#define _TOR_DNSSERV_H + +void dnsserv_configure_listener(connection_t *conn); +void dnsserv_close_listener(connection_t *conn); +void dnsserv_resolved(edge_connection_t *conn, + int answer_type, + size_t answer_len, + const char *answer, + int ttl); +void dnsserv_reject_request(edge_connection_t *conn); +int dnsserv_launch_request(const char *name, int is_reverse); + +#endif + diff --git a/src/or/eventdns.c b/src/or/eventdns.c index a889e803ed..4e44d15163 100644 --- a/src/or/eventdns.c +++ b/src/or/eventdns.c @@ -31,6 +31,7 @@ */ #include "eventdns_tor.h" +#include "../common/util.h" #include <sys/types.h> /* #define NDEBUG */ @@ -89,6 +90,7 @@ #include <stdarg.h> #include "eventdns.h" + #ifdef WIN32 #include <windows.h> #include <winsock2.h> @@ -173,8 +175,6 @@ struct evdns_request { /* these objects are kept in a circular list */ struct evdns_request *next, *prev; - u16 timeout_event_deleted; /**< Debugging: where was timeout_event - * deleted? 0 for "it's added." */ struct event timeout_event; u16 trans_id; /* the transaction id */ @@ -214,8 +214,6 @@ struct nameserver { struct event event; /* these objects are kept in a circular list */ struct nameserver *next, *prev; - u16 timeout_event_deleted; /**< Debugging: where was timeout_event - * deleted? 0 for "it's added." */ struct event timeout_event; /* used to keep the timeout for */ /* when we next probe this server. */ /* Valid if state == 0 */ @@ -474,51 +472,10 @@ sockaddr_eq(const struct sockaddr *sa1, const struct sockaddr *sa2, return 1; } -/* for debugging bug 929. XXXX022 */ -static int -_add_timeout_event(u16 *lineno, struct event *ev, struct timeval *to) -{ - *lineno = 0; - return evtimer_add(ev, to); -} -#define add_timeout_event(s, to) \ - (_add_timeout_event(&(s)->timeout_event_deleted, &(s)->timeout_event, (to))) - -/* for debugging bug 929. XXXX022 */ -static int -_del_timeout_event(u16 *lineno, struct event *ev, int line) -{ - if (*lineno) { - log(EVDNS_LOG_DEBUG, - "Duplicate timeout event_del from line %d: first call " - "was at %d.", line, (int)*lineno); - return 0; - } else { - *lineno = (u16)line; - return event_del(ev); - } -} -#define del_timeout_event(s) \ - (_del_timeout_event(&(s)->timeout_event_deleted, &(s)->timeout_event, \ - __LINE__)) -/* For debugging bug 929/957. XXXX022 */ -static int -_del_timeout_event_if_set(u16 *lineno, struct event *ev, int line) -{ - if (*lineno == 0) { - log(EVDNS_LOG_DEBUG, - "Event that I thought was non-added as of line %d " - "was actually added on line %d", - line, (int)*lineno); - *lineno = line; - return event_del(ev); - } - return 0; -} -#define del_timeout_event_if_set(s) \ - _del_timeout_event_if_set(&(s)->timeout_event_deleted, \ - &(s)->timeout_event, \ - __LINE__) +#define add_timeout_event(s, to) \ + (event_add(&(s)->timeout_event, (to))) +#define del_timeout_event(s) \ + (event_del(&(s)->timeout_event)) /* This walks the list of inflight requests to find the */ /* one with a matching transaction id. Returns NULL on */ @@ -555,7 +512,7 @@ static void nameserver_probe_failed(struct nameserver *const ns) { const struct timeval * timeout; del_timeout_event(ns); - CLEAR(&ns->timeout_event); + if (ns->state == 1) { /* This can happen if the nameserver acts in a way which makes us mark */ /* it as bad and then starts sending good replies. */ @@ -567,8 +524,6 @@ nameserver_probe_failed(struct nameserver *const ns) { global_nameserver_timeouts_length - 1)]; ns->failed_times++; - del_timeout_event_if_set(ns); - evtimer_set(&ns->timeout_event, nameserver_prod_callback, ns); if (add_timeout_event(ns, (struct timeval *) timeout) < 0) { log(EVDNS_LOG_WARN, "Error from libevent when adding timer event for %s", @@ -597,8 +552,6 @@ nameserver_failed(struct nameserver *const ns, const char *msg) { ns->state = 0; ns->failed_times = 1; - del_timeout_event_if_set(ns); - evtimer_set(&ns->timeout_event, nameserver_prod_callback, ns); if (add_timeout_event(ns, (struct timeval *) &global_nameserver_timeouts[0]) < 0) { log(EVDNS_LOG_WARN, "Error from libevent when adding timer event for %s", @@ -634,7 +587,6 @@ nameserver_up(struct nameserver *const ns) { log(EVDNS_LOG_WARN, "Nameserver %s is back up", debug_ntop((struct sockaddr *)&ns->address)); del_timeout_event(ns); - CLEAR(&ns->timeout_event); ns->state = 1; ns->failed_times = 0; ns->timedout = 0; @@ -666,7 +618,6 @@ request_finished(struct evdns_request *const req, struct evdns_request **head) { log(EVDNS_LOG_DEBUG, "Removing timeout for request %lx", (unsigned long) req); del_timeout_event(req); - CLEAR(&req->timeout_event); search_request_finished(req); global_requests_inflight--; @@ -1342,8 +1293,8 @@ server_port_read(struct evdns_server_port *s) { static void server_port_flush(struct evdns_server_port *port) { - while (port->pending_replies) { - struct server_request *req = port->pending_replies; + struct server_request *req = port->pending_replies; + while (req) { ssize_t r = sendto(port->socket, req->response, req->response_len, 0, (struct sockaddr*) &req->addr, (socklen_t)req->addrlen); if (r < 0) { @@ -1355,6 +1306,9 @@ server_port_flush(struct evdns_server_port *port) if (server_request_free(req)) { /* we released the last reference to req->port. */ return; + } else { + assert(port->pending_replies != req); + req = port->pending_replies; } } @@ -2046,7 +2000,6 @@ evdns_request_timeout_callback(int fd, short events, void *arg) { * request_finished; that one already deletes the timeout event. * XXXX021 port this change to libevent. */ del_timeout_event(req); - CLEAR(&req->timeout_event); evdns_request_transmit(req); } } @@ -2109,8 +2062,7 @@ evdns_request_transmit(struct evdns_request *req) { /* transmitted; we need to check for timeout. */ log(EVDNS_LOG_DEBUG, "Setting timeout for request %lx", (unsigned long) req); - del_timeout_event_if_set(req); - evtimer_set(&req->timeout_event, evdns_request_timeout_callback, req); + if (add_timeout_event(req, &global_timeout) < 0) { log(EVDNS_LOG_WARN, "Error from libevent when adding timer for request %lx", @@ -2225,7 +2177,6 @@ evdns_clear_nameservers_and_suspend(void) (void) event_del(&server->event); CLEAR(&server->event); del_timeout_event(server); - CLEAR(&server->timeout_event); if (server->socket >= 0) CLOSE_SOCKET(server->socket); CLEAR(server); @@ -2243,7 +2194,6 @@ evdns_clear_nameservers_and_suspend(void) req->ns = NULL; /* ???? What to do about searches? */ del_timeout_event(req); - CLEAR(&req->timeout_event); req->trans_id = 0; req->transmit_me = 0; @@ -2292,6 +2242,21 @@ evdns_resume(void) } static int +sockaddr_is_loopback(const struct sockaddr *addr) +{ + static const char LOOPBACK_S6[16] = + "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\1"; + if (addr->sa_family == AF_INET) { + struct sockaddr_in *sin = (struct sockaddr_in *)addr; + return (ntohl(sin->sin_addr.s_addr) & 0xff000000) == 0x7f000000; + } else if (addr->sa_family == AF_INET6) { + struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)addr; + return !memcmp(sin6->sin6_addr.s6_addr, LOOPBACK_S6, 16); + } + return 0; +} + +static int _evdns_nameserver_add_impl(const struct sockaddr *address, socklen_t addrlen) { /* first check to see if we already have this nameserver */ @@ -2318,7 +2283,8 @@ _evdns_nameserver_add_impl(const struct sockaddr *address, if (!ns) return -1; memset(ns, 0, sizeof(struct nameserver)); - ns->timeout_event_deleted = __LINE__; + + evtimer_set(&ns->timeout_event, nameserver_prod_callback, ns); ns->socket = socket(PF_INET, SOCK_DGRAM, 0); if (ns->socket < 0) { err = 1; goto out1; } @@ -2331,7 +2297,8 @@ _evdns_nameserver_add_impl(const struct sockaddr *address, fcntl(ns->socket, F_SETFL, O_NONBLOCK); #endif - if (global_bind_addr_is_set) { + if (global_bind_addr_is_set && + !sockaddr_is_loopback((struct sockaddr*)&global_bind_address)) { if (bind(ns->socket, (struct sockaddr *)&global_bind_address, global_bind_addrlen) < 0) { log(EVDNS_LOG_DEBUG, "Couldn't bind to outgoing address."); @@ -2553,7 +2520,8 @@ request_new(int type, const char *name, int flags, } memset(req, 0, sizeof(struct evdns_request)); - req->timeout_event_deleted = __LINE__; + + evtimer_set(&req->timeout_event, evdns_request_timeout_callback, req); if (global_randomize_case) { unsigned i; @@ -2941,14 +2909,6 @@ evdns_resolv_set_defaults(int flags) { if (flags & DNS_OPTION_NAMESERVERS) evdns_nameserver_ip_add("127.0.0.1"); } -#ifndef HAVE_STRTOK_R -static char * -strtok_r(char *s, const char *delim, char **state) { - (void)state; - return strtok(s, delim); -} -#endif - /* helper version of atoi which returns -1 on error */ static int strtoint(const char *const str) { @@ -3025,9 +2985,9 @@ static void resolv_conf_parse_line(char *const start, int flags) { char *strtok_state; static const char *const delims = " \t"; -#define NEXT_TOKEN strtok_r(NULL, delims, &strtok_state) +#define NEXT_TOKEN tor_strtok_r(NULL, delims, &strtok_state) - char *const first_token = strtok_r(start, delims, &strtok_state); + char *const first_token = tor_strtok_r(start, delims, &strtok_state); if (!first_token) return; if (!strcmp(first_token, "nameserver") && (flags & DNS_OPTION_NAMESERVERS)) { @@ -3172,13 +3132,13 @@ load_nameservers_with_getnetworkparams(void) GetNetworkParams_fn_t fn; /* XXXX Possibly, we should hardcode the location of this DLL. */ - if (!(handle = LoadLibrary("iphlpapi.dll"))) { + if (!(handle = LoadLibraryW(L"iphlpapi.dll"))) { log(EVDNS_LOG_WARN, "Could not open iphlpapi.dll"); /* right now status = 0, doesn't that mean "good" - mikec */ status = -1; goto done; } - if (!(fn = (GetNetworkParams_fn_t) GetProcAddress(handle, "GetNetworkParams"))) { + if (!(fn = (GetNetworkParams_fn_t) GetProcAddress(handle, TEXT("GetNetworkParams")))) { log(EVDNS_LOG_WARN, "Could not get address of function."); /* same as above */ status = -1; @@ -3245,32 +3205,40 @@ config_nameserver_from_reg_key(HKEY key, const char *subkey) { char *buf; DWORD bufsz = 0, type = 0; + WCHAR wsubkey[MAX_PATH] = {0}; + char ansibuf[MAX_PATH] = {0}; int status = 0; - if (RegQueryValueEx(key, subkey, 0, &type, NULL, &bufsz) + mbstowcs(wsubkey,subkey,MAX_PATH); + if (RegQueryValueExW(key, wsubkey, 0, &type, NULL, &bufsz) != ERROR_MORE_DATA) return -1; if (!(buf = mm_malloc(bufsz))) return -1; - if (RegQueryValueEx(key, subkey, 0, &type, (LPBYTE)buf, &bufsz) + if (RegQueryValueExW(key, wsubkey, 0, &type, (LPBYTE)buf, &bufsz) == ERROR_SUCCESS && bufsz > 1) { - status = evdns_nameserver_ip_add_line(buf); + wcstombs(ansibuf,(wchar_t*)buf,MAX_PATH); + status = evdns_nameserver_ip_add_line(ansibuf); } mm_free(buf); return status; } -#define SERVICES_KEY "System\\CurrentControlSet\\Services\\" -#define WIN_NS_9X_KEY SERVICES_KEY "VxD\\MSTCP" -#define WIN_NS_NT_KEY SERVICES_KEY "Tcpip\\Parameters" +#define SERVICES_KEY L"System\\CurrentControlSet\\Services\\" +#define WIN_NS_9X_KEY SERVICES_KEY L"VxD\\MSTCP" +#define WIN_NS_NT_KEY SERVICES_KEY L"Tcpip\\Parameters" static int load_nameservers_from_registry(void) { int found = 0; int r; + OSVERSIONINFO info = {0}; + info.dwOSVersionInfoSize = sizeof (info); + GetVersionExW((LPOSVERSIONINFO)&info); + #define TRY(k, name) \ if (!found && config_nameserver_from_reg_key(k,name) == 0) { \ log(EVDNS_LOG_DEBUG,"Found nameservers in %s/%s",#k,name); \ @@ -3280,15 +3248,15 @@ load_nameservers_from_registry(void) #k,#name); \ } - if (((int)GetVersion()) > 0) { /* NT */ + if (info.dwMajorVersion >= 5) { /* NT */ HKEY nt_key = 0, interfaces_key = 0; - if (RegOpenKeyEx(HKEY_LOCAL_MACHINE, WIN_NS_NT_KEY, 0, + if (RegOpenKeyExW(HKEY_LOCAL_MACHINE, WIN_NS_NT_KEY, 0, KEY_READ, &nt_key) != ERROR_SUCCESS) { log(EVDNS_LOG_DEBUG,"Couldn't open nt key, %d",(int)GetLastError()); return -1; } - r = RegOpenKeyEx(nt_key, "Interfaces", 0, + r = RegOpenKeyExW(nt_key, L"Interfaces", 0, KEY_QUERY_VALUE|KEY_ENUMERATE_SUB_KEYS, &interfaces_key); if (r != ERROR_SUCCESS) { @@ -3303,7 +3271,7 @@ load_nameservers_from_registry(void) RegCloseKey(nt_key); } else { HKEY win_key = 0; - if (RegOpenKeyEx(HKEY_LOCAL_MACHINE, WIN_NS_9X_KEY, 0, + if (RegOpenKeyExW(HKEY_LOCAL_MACHINE, WIN_NS_9X_KEY, 0, KEY_READ, &win_key) != ERROR_SUCCESS) { log(EVDNS_LOG_DEBUG, "Couldn't open registry key, %d", (int)GetLastError()); return -1; @@ -3384,8 +3352,7 @@ evdns_shutdown(int fail_requests) if (server->socket >= 0) CLOSE_SOCKET(server->socket); (void) event_del(&server->event); - if (server->state == 0) - del_timeout_event(server); + del_timeout_event(server); CLEAR(server); mm_free(server); if (server_next == server_head) diff --git a/src/or/geoip.c b/src/or/geoip.c index eb8460e24e..cde9cabdb3 100644 --- a/src/or/geoip.c +++ b/src/or/geoip.c @@ -3,15 +3,23 @@ /** * \file geoip.c - * \brief Functions related to maintaining an IP-to-country database and to - * summarizing client connections by country. + * \brief Functions related to maintaining an IP-to-country database; + * to summarizing client connections by country to entry guards, bridges, + * and directory servers; and for statistics on answering network status + * requests. */ #define GEOIP_PRIVATE #include "or.h" #include "ht.h" +#include "config.h" +#include "control.h" +#include "dnsserv.h" +#include "geoip.h" +#include "routerlist.h" static void clear_geoip_db(void); +static void init_geoip_countries(void); /** An entry from the GeoIP file: maps an IP range to a country. */ typedef struct geoip_entry_t { @@ -20,16 +28,11 @@ typedef struct geoip_entry_t { intptr_t country; /**< An index into geoip_countries */ } geoip_entry_t; -/** For how many periods should we remember per-country request history? */ -#define REQUEST_HIST_LEN 3 -/** How long are the periods for which we should remember request history? */ -#define REQUEST_HIST_PERIOD (8*60*60) - /** A per-country record for GeoIP request history. */ typedef struct geoip_country_t { char countrycode[3]; - uint32_t n_v2_ns_requests[REQUEST_HIST_LEN]; - uint32_t n_v3_ns_requests[REQUEST_HIST_LEN]; + uint32_t n_v2_ns_requests; + uint32_t n_v3_ns_requests; } geoip_country_t; /** A list of geoip_country_t */ @@ -42,7 +45,7 @@ static strmap_t *country_idxplus1_by_lc_code = NULL; static smartlist_t *geoip_entries = NULL; /** Return the index of the <b>country</b>'s entry in the GeoIP DB - * if it is a valid 2-letter country code, otherwise return zero. + * if it is a valid 2-letter country code, otherwise return -1. */ country_t geoip_get_country(const char *country) @@ -101,11 +104,11 @@ geoip_parse_entry(const char *line) { unsigned int low, high; char b[3]; - if (!geoip_countries) { - geoip_countries = smartlist_create(); + if (!geoip_countries) + init_geoip_countries(); + if (!geoip_entries) geoip_entries = smartlist_create(); - country_idxplus1_by_lc_code = strmap_new(); - } + while (TOR_ISSPACE(*line)) ++line; if (*line == '#') @@ -160,6 +163,24 @@ should_record_bridge_info(or_options_t *options) return options->BridgeRelay && options->BridgeRecordUsageByCountry; } +/** Set up a new list of geoip countries with no countries (yet) set in it, + * except for the unknown country. + */ +static void +init_geoip_countries(void) +{ + geoip_country_t *geoip_unresolved; + geoip_countries = smartlist_create(); + /* Add a geoip_country_t for requests that could not be resolved to a + * country as first element (index 0) to geoip_countries. */ + geoip_unresolved = tor_malloc_zero(sizeof(geoip_country_t)); + strlcpy(geoip_unresolved->countrycode, "??", + sizeof(geoip_unresolved->countrycode)); + smartlist_add(geoip_countries, geoip_unresolved); + country_idxplus1_by_lc_code = strmap_new(); + strmap_set_lc(country_idxplus1_by_lc_code, "??", (void*)(1)); +} + /** Clear the GeoIP database and reload it from the file * <b>filename</b>. Return 0 on success, -1 on failure. * @@ -185,10 +206,8 @@ geoip_load_file(const char *filename, or_options_t *options) filename, msg); return -1; } - if (!geoip_countries) { - geoip_countries = smartlist_create(); - country_idxplus1_by_lc_code = strmap_new(); - } + if (!geoip_countries) + init_geoip_countries(); if (geoip_entries) { SMARTLIST_FOREACH(geoip_entries, geoip_entry_t *, e, tor_free(e)); smartlist_free(geoip_entries); @@ -215,9 +234,10 @@ geoip_load_file(const char *filename, or_options_t *options) } /** Given an IP address in host order, return a number representing the - * country to which that address belongs, or -1 for unknown. The return value - * will always be less than geoip_get_n_countries(). To decode it, - * call geoip_get_country_name(). + * country to which that address belongs, -1 for "No geoip information + * available", or 0 for the 'unknown country'. The return value will always + * be less than geoip_get_n_countries(). To decode it, call + * geoip_get_country_name(). */ int geoip_get_country_by_ip(uint32_t ipaddr) @@ -226,7 +246,7 @@ geoip_get_country_by_ip(uint32_t ipaddr) if (!geoip_entries) return -1; ent = smartlist_bsearch(geoip_entries, &ipaddr, _geoip_compare_key_to_entry); - return ent ? (int)ent->country : -1; + return ent ? (int)ent->country : 0; } /** Return the number of countries recognized by the GeoIP database. */ @@ -261,8 +281,8 @@ geoip_is_loaded(void) typedef struct clientmap_entry_t { HT_ENTRY(clientmap_entry_t) node; uint32_t ipaddr; - time_t last_seen; /* The last 2 bits of this value hold the client - * operation. */ + unsigned int last_seen_in_minutes:30; + unsigned int action:2; } clientmap_entry_t; #define ACTION_MASK 3 @@ -270,14 +290,6 @@ typedef struct clientmap_entry_t { /** Map from client IP address to last time seen. */ static HT_HEAD(clientmap, clientmap_entry_t) client_history = HT_INITIALIZER(); -/** Time at which we started tracking client IP history. */ -static time_t client_history_starts = 0; - -/** When did the current period of checking per-country request history - * start? */ -static time_t current_request_period_starts = 0; -/** How many older request periods are we remembering? */ -static int n_old_request_periods = 0; /** Hashtable helper: compute a hash of a clientmap_entry_t. */ static INLINE unsigned @@ -289,7 +301,7 @@ clientmap_entry_hash(const clientmap_entry_t *a) static INLINE int clientmap_entries_eq(const clientmap_entry_t *a, const clientmap_entry_t *b) { - return a->ipaddr == b->ipaddr; + return a->ipaddr == b->ipaddr && a->action == b->action; } HT_PROTOTYPE(clientmap, clientmap_entry_t, node, clientmap_entry_hash, @@ -297,8 +309,87 @@ HT_PROTOTYPE(clientmap, clientmap_entry_t, node, clientmap_entry_hash, HT_GENERATE(clientmap, clientmap_entry_t, node, clientmap_entry_hash, clientmap_entries_eq, 0.6, malloc, realloc, free); +/** Clear history of connecting clients used by entry and bridge stats. */ +static void +client_history_clear(void) +{ + clientmap_entry_t **ent, **next, *this; + for (ent = HT_START(clientmap, &client_history); ent != NULL; + ent = next) { + if ((*ent)->action == GEOIP_CLIENT_CONNECT) { + this = *ent; + next = HT_NEXT_RMV(clientmap, &client_history, ent); + tor_free(this); + } else { + next = HT_NEXT(clientmap, &client_history, ent); + } + } +} + +/** How often do we update our estimate which share of v2 and v3 directory + * requests is sent to us? We could as well trigger updates of shares from + * network status updates, but that means adding a lot of calls into code + * that is independent from geoip stats (and keeping them up-to-date). We + * are perfectly fine with an approximation of 15-minute granularity. */ +#define REQUEST_SHARE_INTERVAL (15 * 60) + +/** When did we last determine which share of v2 and v3 directory requests + * is sent to us? */ +static time_t last_time_determined_shares = 0; + +/** Sum of products of v2 shares times the number of seconds for which we + * consider these shares as valid. */ +static double v2_share_times_seconds; + +/** Sum of products of v3 shares times the number of seconds for which we + * consider these shares as valid. */ +static double v3_share_times_seconds; + +/** Number of seconds we are determining v2 and v3 shares. */ +static int share_seconds; + +/** Try to determine which fraction of v2 and v3 directory requests aimed at + * caches will be sent to us at time <b>now</b> and store that value in + * order to take a mean value later on. */ +static void +geoip_determine_shares(time_t now) +{ + double v2_share = 0.0, v3_share = 0.0; + if (router_get_my_share_of_directory_requests(&v2_share, &v3_share) < 0) + return; + if (last_time_determined_shares) { + v2_share_times_seconds += v2_share * + ((double) (now - last_time_determined_shares)); + v3_share_times_seconds += v3_share * + ((double) (now - last_time_determined_shares)); + share_seconds += (int)(now - last_time_determined_shares); + } + last_time_determined_shares = now; +} + +/** Calculate which fraction of v2 and v3 directory requests aimed at caches + * have been sent to us since the last call of this function up to time + * <b>now</b>. Set *<b>v2_share_out</b> and *<b>v3_share_out</b> to the + * fractions of v2 and v3 protocol shares we expect to have seen. Reset + * counters afterwards. Return 0 on success, -1 on failure (e.g. when zero + * seconds have passed since the last call).*/ +static int +geoip_get_mean_shares(time_t now, double *v2_share_out, + double *v3_share_out) +{ + geoip_determine_shares(now); + if (!share_seconds) + return -1; + *v2_share_out = v2_share_times_seconds / ((double) share_seconds); + *v3_share_out = v3_share_times_seconds / ((double) share_seconds); + v2_share_times_seconds = v3_share_times_seconds = 0.0; + share_seconds = 0; + return 0; +} + /** Note that we've seen a client connect from the IP <b>addr</b> (host order) - * at time <b>now</b>. Ignored by all but bridges. */ + * at time <b>now</b>. Ignored by all but bridges and directories if + * configured accordingly. */ void geoip_note_client_seen(geoip_client_action_t action, uint32_t addr, time_t now) @@ -306,72 +397,45 @@ geoip_note_client_seen(geoip_client_action_t action, or_options_t *options = get_options(); clientmap_entry_t lookup, *ent; if (action == GEOIP_CLIENT_CONNECT) { - if (!(options->BridgeRelay && options->BridgeRecordUsageByCountry)) - return; - /* Did we recently switch from bridge to relay or back? */ - if (client_history_starts > now) + /* Only remember statistics as entry guard or as bridge. */ + if (!options->EntryStatistics && + (!(options->BridgeRelay && options->BridgeRecordUsageByCountry))) return; } else { -#ifndef ENABLE_GEOIP_STATS - return; -#else if (options->BridgeRelay || options->BridgeAuthoritativeDir || - !options->DirRecordUsageByCountry) + !options->DirReqStatistics) return; -#endif } - /* Rotate the current request period. */ - while (current_request_period_starts + REQUEST_HIST_PERIOD < now) { - if (!geoip_countries) - geoip_countries = smartlist_create(); - if (!current_request_period_starts) { - current_request_period_starts = now; - break; - } - SMARTLIST_FOREACH(geoip_countries, geoip_country_t *, c, { - memmove(&c->n_v2_ns_requests[0], &c->n_v2_ns_requests[1], - sizeof(uint32_t)*(REQUEST_HIST_LEN-1)); - memmove(&c->n_v3_ns_requests[0], &c->n_v3_ns_requests[1], - sizeof(uint32_t)*(REQUEST_HIST_LEN-1)); - c->n_v2_ns_requests[REQUEST_HIST_LEN-1] = 0; - c->n_v3_ns_requests[REQUEST_HIST_LEN-1] = 0; - }); - current_request_period_starts += REQUEST_HIST_PERIOD; - if (n_old_request_periods < REQUEST_HIST_LEN-1) - ++n_old_request_periods; - } - - /* We use the low 3 bits of the time to encode the action. Since we're - * potentially remembering tons of clients, we don't want to make - * clientmap_entry_t larger than it has to be. */ - now = (now & ~ACTION_MASK) | (((int)action) & ACTION_MASK); lookup.ipaddr = addr; + lookup.action = (int)action; ent = HT_FIND(clientmap, &client_history, &lookup); if (ent) { - ent->last_seen = now; + ent->last_seen_in_minutes = now / 60; } else { ent = tor_malloc_zero(sizeof(clientmap_entry_t)); ent->ipaddr = addr; - ent->last_seen = now; + ent->last_seen_in_minutes = now / 60; + ent->action = (int)action; HT_INSERT(clientmap, &client_history, ent); } if (action == GEOIP_CLIENT_NETWORKSTATUS || action == GEOIP_CLIENT_NETWORKSTATUS_V2) { int country_idx = geoip_get_country_by_ip(addr); + if (country_idx < 0) + country_idx = 0; /** unresolved requests are stored at index 0. */ if (country_idx >= 0 && country_idx < smartlist_len(geoip_countries)) { geoip_country_t *country = smartlist_get(geoip_countries, country_idx); if (action == GEOIP_CLIENT_NETWORKSTATUS) - ++country->n_v3_ns_requests[REQUEST_HIST_LEN-1]; + ++country->n_v3_ns_requests; else - ++country->n_v2_ns_requests[REQUEST_HIST_LEN-1]; + ++country->n_v2_ns_requests; } - } - if (!client_history_starts) { - client_history_starts = now; - current_request_period_starts = now; + /* Periodically determine share of requests that we should see */ + if (last_time_determined_shares + REQUEST_SHARE_INTERVAL < now) + geoip_determine_shares(now); } } @@ -380,8 +444,8 @@ geoip_note_client_seen(geoip_client_action_t action, static int _remove_old_client_helper(struct clientmap_entry_t *ent, void *_cutoff) { - time_t cutoff = *(time_t*)_cutoff; - if (ent->last_seen < cutoff) { + time_t cutoff = *(time_t*)_cutoff / 60; + if (ent->last_seen_in_minutes < cutoff) { tor_free(ent); return 1; } else { @@ -389,18 +453,45 @@ _remove_old_client_helper(struct clientmap_entry_t *ent, void *_cutoff) } } -/** Forget about all clients that haven't connected since <b>cutoff</b>. - * If <b>cutoff</b> is in the future, clients won't be added to the history - * until this time is reached. This is useful to prevent relays that switch - * to bridges from reporting unbelievable numbers of clients. */ +/** Forget about all clients that haven't connected since <b>cutoff</b>. */ void geoip_remove_old_clients(time_t cutoff) { clientmap_HT_FOREACH_FN(&client_history, _remove_old_client_helper, &cutoff); - if (client_history_starts < cutoff) - client_history_starts = cutoff; +} + +/** How many responses are we giving to clients requesting v2 network + * statuses? */ +static uint32_t ns_v2_responses[GEOIP_NS_RESPONSE_NUM]; + +/** How many responses are we giving to clients requesting v3 network + * statuses? */ +static uint32_t ns_v3_responses[GEOIP_NS_RESPONSE_NUM]; + +/** Note that we've rejected a client's request for a v2 or v3 network + * status, encoded in <b>action</b> for reason <b>reason</b> at time + * <b>now</b>. */ +void +geoip_note_ns_response(geoip_client_action_t action, + geoip_ns_response_t response) +{ + static int arrays_initialized = 0; + if (!get_options()->DirReqStatistics) + return; + if (!arrays_initialized) { + memset(ns_v2_responses, 0, sizeof(ns_v2_responses)); + memset(ns_v3_responses, 0, sizeof(ns_v3_responses)); + arrays_initialized = 1; + } + tor_assert(action == GEOIP_CLIENT_NETWORKSTATUS || + action == GEOIP_CLIENT_NETWORKSTATUS_V2); + tor_assert(response < GEOIP_NS_RESPONSE_NUM); + if (action == GEOIP_CLIENT_NETWORKSTATUS) + ns_v3_responses[response]++; + else + ns_v2_responses[response]++; } /** Do not mention any country from which fewer than this number of IPs have @@ -414,13 +505,6 @@ geoip_remove_old_clients(time_t cutoff) * multiple of this value. */ #define IP_GRANULARITY 8 -/** Return the time at which we started recording geoip data. */ -time_t -geoip_get_history_start(void) -{ - return client_history_starts; -} - /** Helper type: used to sort per-country totals by value. */ typedef struct c_hist_t { char country[3]; /**< Two-letter country code. */ @@ -442,97 +526,319 @@ _c_hist_compare(const void **_a, const void **_b) return strcmp(a->country, b->country); } -/** How long do we have to have observed per-country request history before we - * are willing to talk about it? */ -#define GEOIP_MIN_OBSERVATION_TIME (12*60*60) +/** When there are incomplete directory requests at the end of a 24-hour + * period, consider those requests running for longer than this timeout as + * failed, the others as still running. */ +#define DIRREQ_TIMEOUT (10*60) -/** Return the lowest x such that x is at least <b>number</b>, and x modulo - * <b>divisor</b> == 0. */ -static INLINE unsigned -round_to_next_multiple_of(unsigned number, unsigned divisor) +/** Entry in a map from either conn->global_identifier for direct requests + * or a unique circuit identifier for tunneled requests to request time, + * response size, and completion time of a network status request. Used to + * measure download times of requests to derive average client + * bandwidths. */ +typedef struct dirreq_map_entry_t { + HT_ENTRY(dirreq_map_entry_t) node; + /** Unique identifier for this network status request; this is either the + * conn->global_identifier of the dir conn (direct request) or a new + * locally unique identifier of a circuit (tunneled request). This ID is + * only unique among other direct or tunneled requests, respectively. */ + uint64_t dirreq_id; + unsigned int state:3; /**< State of this directory request. */ + unsigned int type:1; /**< Is this a direct or a tunneled request? */ + unsigned int completed:1; /**< Is this request complete? */ + unsigned int action:2; /**< Is this a v2 or v3 request? */ + /** When did we receive the request and started sending the response? */ + struct timeval request_time; + size_t response_size; /**< What is the size of the response in bytes? */ + struct timeval completion_time; /**< When did the request succeed? */ +} dirreq_map_entry_t; + +/** Map of all directory requests asking for v2 or v3 network statuses in + * the current geoip-stats interval. Values are + * of type *<b>dirreq_map_entry_t</b>. */ +static HT_HEAD(dirreqmap, dirreq_map_entry_t) dirreq_map = + HT_INITIALIZER(); + +static int +dirreq_map_ent_eq(const dirreq_map_entry_t *a, + const dirreq_map_entry_t *b) { - number += divisor - 1; - number -= number % divisor; - return number; + return a->dirreq_id == b->dirreq_id && a->type == b->type; } -/** Return a newly allocated comma-separated string containing entries for all - * the countries from which we've seen enough clients connect. The entry - * format is cc=num where num is the number of IPs we've seen connecting from - * that country, and cc is a lowercased country code. Returns NULL if we don't - * want to export geoip data yet. */ -char * -geoip_get_client_history(time_t now, geoip_client_action_t action) +static unsigned +dirreq_map_ent_hash(const dirreq_map_entry_t *entry) +{ + unsigned u = (unsigned) entry->dirreq_id; + u += entry->type << 20; + return u; +} + +HT_PROTOTYPE(dirreqmap, dirreq_map_entry_t, node, dirreq_map_ent_hash, + dirreq_map_ent_eq); +HT_GENERATE(dirreqmap, dirreq_map_entry_t, node, dirreq_map_ent_hash, + dirreq_map_ent_eq, 0.6, malloc, realloc, free); + +/** Helper: Put <b>entry</b> into map of directory requests using + * <b>type</b> and <b>dirreq_id</b> as key parts. If there is + * already an entry for that key, print out a BUG warning and return. */ +static void +_dirreq_map_put(dirreq_map_entry_t *entry, dirreq_type_t type, + uint64_t dirreq_id) +{ + dirreq_map_entry_t *old_ent; + tor_assert(entry->type == type); + tor_assert(entry->dirreq_id == dirreq_id); + + /* XXXX022 once we're sure the bug case never happens, we can switch + * to HT_INSERT */ + old_ent = HT_REPLACE(dirreqmap, &dirreq_map, entry); + if (old_ent && old_ent != entry) { + log_warn(LD_BUG, "Error when putting directory request into local " + "map. There was already an entry for the same identifier."); + return; + } +} + +/** Helper: Look up and return an entry in the map of directory requests + * using <b>type</b> and <b>dirreq_id</b> as key parts. If there + * is no such entry, return NULL. */ +static dirreq_map_entry_t * +_dirreq_map_get(dirreq_type_t type, uint64_t dirreq_id) +{ + dirreq_map_entry_t lookup; + lookup.type = type; + lookup.dirreq_id = dirreq_id; + return HT_FIND(dirreqmap, &dirreq_map, &lookup); +} + +/** Note that an either direct or tunneled (see <b>type</b>) directory + * request for a network status with unique ID <b>dirreq_id</b> of size + * <b>response_size</b> and action <b>action</b> (either v2 or v3) has + * started. */ +void +geoip_start_dirreq(uint64_t dirreq_id, size_t response_size, + geoip_client_action_t action, dirreq_type_t type) +{ + dirreq_map_entry_t *ent; + if (!get_options()->DirReqStatistics) + return; + ent = tor_malloc_zero(sizeof(dirreq_map_entry_t)); + ent->dirreq_id = dirreq_id; + tor_gettimeofday(&ent->request_time); + ent->response_size = response_size; + ent->action = action; + ent->type = type; + _dirreq_map_put(ent, type, dirreq_id); +} + +/** Change the state of the either direct or tunneled (see <b>type</b>) + * directory request with <b>dirreq_id</b> to <b>new_state</b> and + * possibly mark it as completed. If no entry can be found for the given + * key parts (e.g., if this is a directory request that we are not + * measuring, or one that was started in the previous measurement period), + * or if the state cannot be advanced to <b>new_state</b>, do nothing. */ +void +geoip_change_dirreq_state(uint64_t dirreq_id, dirreq_type_t type, + dirreq_state_t new_state) +{ + dirreq_map_entry_t *ent; + if (!get_options()->DirReqStatistics) + return; + ent = _dirreq_map_get(type, dirreq_id); + if (!ent) + return; + if (new_state == DIRREQ_IS_FOR_NETWORK_STATUS) + return; + if (new_state - 1 != ent->state) + return; + ent->state = new_state; + if ((type == DIRREQ_DIRECT && + new_state == DIRREQ_FLUSHING_DIR_CONN_FINISHED) || + (type == DIRREQ_TUNNELED && + new_state == DIRREQ_OR_CONN_BUFFER_FLUSHED)) { + tor_gettimeofday(&ent->completion_time); + ent->completed = 1; + } +} + +/** Return a newly allocated comma-separated string containing statistics + * on network status downloads. The string contains the number of completed + * requests, timeouts, and still running requests as well as the download + * times by deciles and quartiles. Return NULL if we have not observed + * requests for long enough. */ +static char * +geoip_get_dirreq_history(geoip_client_action_t action, + dirreq_type_t type) { char *result = NULL; - if (!geoip_is_loaded()) + smartlist_t *dirreq_completed = NULL; + uint32_t complete = 0, timeouts = 0, running = 0; + int bufsize = 1024, written; + dirreq_map_entry_t **ptr, **next, *ent; + struct timeval now; + + tor_gettimeofday(&now); + if (action != GEOIP_CLIENT_NETWORKSTATUS && + action != GEOIP_CLIENT_NETWORKSTATUS_V2) return NULL; - if (client_history_starts < (now - GEOIP_MIN_OBSERVATION_TIME)) { - char buf[32]; - smartlist_t *chunks = NULL; - smartlist_t *entries = NULL; - int n_countries = geoip_get_n_countries(); - int i; - clientmap_entry_t **ent; - unsigned *counts = tor_malloc_zero(sizeof(unsigned)*n_countries); - unsigned total = 0; - unsigned granularity = IP_GRANULARITY; -#ifdef ENABLE_GEOIP_STATS - if (get_options()->DirRecordUsageByCountry) - granularity = get_options()->DirRecordUsageGranularity; -#endif - HT_FOREACH(ent, clientmap, &client_history) { - int country; - if (((*ent)->last_seen & ACTION_MASK) != (int)action) - continue; - country = geoip_get_country_by_ip((*ent)->ipaddr); - if (country < 0) - continue; - tor_assert(0 <= country && country < n_countries); - ++counts[country]; - ++total; - } - /* Don't record anything if we haven't seen enough IPs. */ - if (total < MIN_IPS_TO_NOTE_ANYTHING) - goto done; - /* Make a list of c_hist_t */ - entries = smartlist_create(); - for (i = 0; i < n_countries; ++i) { - unsigned c = counts[i]; - const char *countrycode; - c_hist_t *ent; - /* Only report a country if it has a minimum number of IPs. */ - if (c >= MIN_IPS_TO_NOTE_COUNTRY) { - c = round_to_next_multiple_of(c, granularity); - countrycode = geoip_get_country_name(i); - ent = tor_malloc(sizeof(c_hist_t)); - strlcpy(ent->country, countrycode, sizeof(ent->country)); - ent->total = c; - smartlist_add(entries, ent); + dirreq_completed = smartlist_create(); + for (ptr = HT_START(dirreqmap, &dirreq_map); ptr; ptr = next) { + ent = *ptr; + if (ent->action != action || ent->type != type) { + next = HT_NEXT(dirreqmap, &dirreq_map, ptr); + continue; + } else { + if (ent->completed) { + smartlist_add(dirreq_completed, ent); + complete++; + next = HT_NEXT_RMV(dirreqmap, &dirreq_map, ptr); + } else { + if (tv_mdiff(&ent->request_time, &now) / 1000 > DIRREQ_TIMEOUT) + timeouts++; + else + running++; + next = HT_NEXT_RMV(dirreqmap, &dirreq_map, ptr); + tor_free(ent); } } - /* Sort entries. Note that we must do this _AFTER_ rounding, or else - * the sort order could leak info. */ - smartlist_sort(entries, _c_hist_compare); - - /* Build the result. */ - chunks = smartlist_create(); - SMARTLIST_FOREACH(entries, c_hist_t *, ch, { - tor_snprintf(buf, sizeof(buf), "%s=%u", ch->country, ch->total); - smartlist_add(chunks, tor_strdup(buf)); - }); - result = smartlist_join_strings(chunks, ",", 0, NULL); - done: - tor_free(counts); - if (chunks) { - SMARTLIST_FOREACH(chunks, char *, c, tor_free(c)); - smartlist_free(chunks); - } - if (entries) { - SMARTLIST_FOREACH(entries, c_hist_t *, c, tor_free(c)); - smartlist_free(entries); + } +#define DIR_REQ_GRANULARITY 4 + complete = round_uint32_to_next_multiple_of(complete, + DIR_REQ_GRANULARITY); + timeouts = round_uint32_to_next_multiple_of(timeouts, + DIR_REQ_GRANULARITY); + running = round_uint32_to_next_multiple_of(running, + DIR_REQ_GRANULARITY); + result = tor_malloc_zero(bufsize); + written = tor_snprintf(result, bufsize, "complete=%u,timeout=%u," + "running=%u", complete, timeouts, running); + if (written < 0) { + tor_free(result); + goto done; + } + +#define MIN_DIR_REQ_RESPONSES 16 + if (complete >= MIN_DIR_REQ_RESPONSES) { + uint32_t *dltimes; + /* We may have rounded 'completed' up. Here we want to use the + * real value. */ + complete = smartlist_len(dirreq_completed); + dltimes = tor_malloc_zero(sizeof(uint32_t) * complete); + SMARTLIST_FOREACH_BEGIN(dirreq_completed, dirreq_map_entry_t *, ent) { + uint32_t bytes_per_second; + uint32_t time_diff = (uint32_t) tv_mdiff(&ent->request_time, + &ent->completion_time); + if (time_diff == 0) + time_diff = 1; /* Avoid DIV/0; "instant" answers are impossible + * by law of nature or something, but a milisecond + * is a bit greater than "instantly" */ + bytes_per_second = (uint32_t)(1000 * ent->response_size / time_diff); + dltimes[ent_sl_idx] = bytes_per_second; + } SMARTLIST_FOREACH_END(ent); + median_uint32(dltimes, complete); /* sorts as a side effect. */ + written = tor_snprintf(result + written, bufsize - written, + ",min=%u,d1=%u,d2=%u,q1=%u,d3=%u,d4=%u,md=%u," + "d6=%u,d7=%u,q3=%u,d8=%u,d9=%u,max=%u", + dltimes[0], + dltimes[1*complete/10-1], + dltimes[2*complete/10-1], + dltimes[1*complete/4-1], + dltimes[3*complete/10-1], + dltimes[4*complete/10-1], + dltimes[5*complete/10-1], + dltimes[6*complete/10-1], + dltimes[7*complete/10-1], + dltimes[3*complete/4-1], + dltimes[8*complete/10-1], + dltimes[9*complete/10-1], + dltimes[complete-1]); + if (written<0) + tor_free(result); + tor_free(dltimes); + } + done: + SMARTLIST_FOREACH(dirreq_completed, dirreq_map_entry_t *, ent, + tor_free(ent)); + smartlist_free(dirreq_completed); + return result; +} + +/** Return a newly allocated comma-separated string containing entries for + * all the countries from which we've seen enough clients connect as a + * bridge, directory server, or entry guard. The entry format is cc=num + * where num is the number of IPs we've seen connecting from that country, + * and cc is a lowercased country code. Returns NULL if we don't want + * to export geoip data yet. */ +char * +geoip_get_client_history(geoip_client_action_t action) +{ + char *result = NULL; + unsigned granularity = IP_GRANULARITY; + smartlist_t *chunks = NULL; + smartlist_t *entries = NULL; + int n_countries = geoip_get_n_countries(); + int i; + clientmap_entry_t **ent; + unsigned *counts = NULL; + unsigned total = 0; + + if (!geoip_is_loaded()) + return NULL; + + counts = tor_malloc_zero(sizeof(unsigned)*n_countries); + HT_FOREACH(ent, clientmap, &client_history) { + int country; + if ((*ent)->action != (int)action) + continue; + country = geoip_get_country_by_ip((*ent)->ipaddr); + if (country < 0) + country = 0; /** unresolved requests are stored at index 0. */ + tor_assert(0 <= country && country < n_countries); + ++counts[country]; + ++total; + } + /* Don't record anything if we haven't seen enough IPs. */ + if (total < MIN_IPS_TO_NOTE_ANYTHING) + goto done; + /* Make a list of c_hist_t */ + entries = smartlist_create(); + for (i = 0; i < n_countries; ++i) { + unsigned c = counts[i]; + const char *countrycode; + c_hist_t *ent; + /* Only report a country if it has a minimum number of IPs. */ + if (c >= MIN_IPS_TO_NOTE_COUNTRY) { + c = round_to_next_multiple_of(c, granularity); + countrycode = geoip_get_country_name(i); + ent = tor_malloc(sizeof(c_hist_t)); + strlcpy(ent->country, countrycode, sizeof(ent->country)); + ent->total = c; + smartlist_add(entries, ent); } } + /* Sort entries. Note that we must do this _AFTER_ rounding, or else + * the sort order could leak info. */ + smartlist_sort(entries, _c_hist_compare); + + /* Build the result. */ + chunks = smartlist_create(); + SMARTLIST_FOREACH(entries, c_hist_t *, ch, { + char *buf=NULL; + tor_asprintf(&buf, "%s=%u", ch->country, ch->total); + smartlist_add(chunks, buf); + }); + result = smartlist_join_strings(chunks, ",", 0, NULL); +done: + tor_free(counts); + if (chunks) { + SMARTLIST_FOREACH(chunks, char *, c, tor_free(c)); + smartlist_free(chunks); + } + if (entries) { + SMARTLIST_FOREACH(entries, c_hist_t *, c, tor_free(c)); + smartlist_free(entries); + } return result; } @@ -540,18 +846,12 @@ geoip_get_client_history(time_t now, geoip_client_action_t action) * for <b>action</b> in a format suitable for an extra-info document, or NULL * on failure. */ char * -geoip_get_request_history(time_t now, geoip_client_action_t action) +geoip_get_request_history(geoip_client_action_t action) { smartlist_t *entries, *strings; char *result; unsigned granularity = IP_GRANULARITY; -#ifdef ENABLE_GEOIP_STATS - if (get_options()->DirRecordUsageByCountry) - granularity = get_options()->DirRecordUsageGranularity; -#endif - if (client_history_starts >= (now - GEOIP_MIN_OBSERVATION_TIME)) - return NULL; if (action != GEOIP_CLIENT_NETWORKSTATUS && action != GEOIP_CLIENT_NETWORKSTATUS_V2) return NULL; @@ -560,13 +860,10 @@ geoip_get_request_history(time_t now, geoip_client_action_t action) entries = smartlist_create(); SMARTLIST_FOREACH(geoip_countries, geoip_country_t *, c, { - uint32_t *n = (action == GEOIP_CLIENT_NETWORKSTATUS) - ? c->n_v3_ns_requests : c->n_v2_ns_requests; uint32_t tot = 0; - int i; c_hist_t *ent; - for (i=0; i < REQUEST_HIST_LEN; ++i) - tot += n[i]; + tot = (action == GEOIP_CLIENT_NETWORKSTATUS) ? + c->n_v3_ns_requests : c->n_v2_ns_requests; if (!tot) continue; ent = tor_malloc_zero(sizeof(c_hist_t)); @@ -578,9 +875,9 @@ geoip_get_request_history(time_t now, geoip_client_action_t action) strings = smartlist_create(); SMARTLIST_FOREACH(entries, c_hist_t *, ent, { - char buf[32]; - tor_snprintf(buf, sizeof(buf), "%s=%u", ent->country, ent->total); - smartlist_add(strings, tor_strdup(buf)); + char *buf = NULL; + tor_asprintf(&buf, "%s=%u", ent->country, ent->total); + smartlist_add(strings, buf); }); result = smartlist_join_strings(strings, ",", 0, NULL); SMARTLIST_FOREACH(strings, char *, cp, tor_free(cp)); @@ -590,69 +887,434 @@ geoip_get_request_history(time_t now, geoip_client_action_t action) return result; } -/** Store all our geoip statistics into $DATADIR/geoip-stats. */ +/** Start time of directory request stats or 0 if we're not collecting + * directory request statistics. */ +static time_t start_of_dirreq_stats_interval; + +/** Initialize directory request stats. */ void -dump_geoip_stats(void) +geoip_dirreq_stats_init(time_t now) { -#ifdef ENABLE_GEOIP_STATS - time_t now = time(NULL); - time_t request_start; - char *filename = get_datadir_fname("geoip-stats"); + start_of_dirreq_stats_interval = now; +} + +/** Stop collecting directory request stats in a way that we can re-start + * doing so in geoip_dirreq_stats_init(). */ +void +geoip_dirreq_stats_term(void) +{ + SMARTLIST_FOREACH(geoip_countries, geoip_country_t *, c, { + c->n_v2_ns_requests = c->n_v3_ns_requests = 0; + }); + { + clientmap_entry_t **ent, **next, *this; + for (ent = HT_START(clientmap, &client_history); ent != NULL; + ent = next) { + if ((*ent)->action == GEOIP_CLIENT_NETWORKSTATUS || + (*ent)->action == GEOIP_CLIENT_NETWORKSTATUS_V2) { + this = *ent; + next = HT_NEXT_RMV(clientmap, &client_history, ent); + tor_free(this); + } else { + next = HT_NEXT(clientmap, &client_history, ent); + } + } + } + v2_share_times_seconds = v3_share_times_seconds = 0.0; + last_time_determined_shares = 0; + share_seconds = 0; + memset(ns_v2_responses, 0, sizeof(ns_v2_responses)); + memset(ns_v3_responses, 0, sizeof(ns_v3_responses)); + { + dirreq_map_entry_t **ent, **next, *this; + for (ent = HT_START(dirreqmap, &dirreq_map); ent != NULL; ent = next) { + this = *ent; + next = HT_NEXT_RMV(dirreqmap, &dirreq_map, ent); + tor_free(this); + } + } + start_of_dirreq_stats_interval = 0; +} + +/** Write dirreq statistics to $DATADIR/stats/dirreq-stats and return when + * we would next want to write. */ +time_t +geoip_dirreq_stats_write(time_t now) +{ + char *statsdir = NULL, *filename = NULL; char *data_v2 = NULL, *data_v3 = NULL; - char since[ISO_TIME_LEN+1], written[ISO_TIME_LEN+1]; + char written[ISO_TIME_LEN+1]; open_file_t *open_file = NULL; double v2_share = 0.0, v3_share = 0.0; FILE *out; + int i; + + if (!start_of_dirreq_stats_interval) + return 0; /* Not initialized. */ + if (start_of_dirreq_stats_interval + WRITE_STATS_INTERVAL > now) + goto done; /* Not ready to write. */ - data_v2 = geoip_get_client_history(now, GEOIP_CLIENT_NETWORKSTATUS_V2); - data_v3 = geoip_get_client_history(now, GEOIP_CLIENT_NETWORKSTATUS); - format_iso_time(since, geoip_get_history_start()); + /* Discard all items in the client history that are too old. */ + geoip_remove_old_clients(start_of_dirreq_stats_interval); + + statsdir = get_datadir_fname("stats"); + if (check_private_dir(statsdir, CPD_CREATE) < 0) + goto done; + filename = get_datadir_fname2("stats", "dirreq-stats"); + data_v2 = geoip_get_client_history(GEOIP_CLIENT_NETWORKSTATUS_V2); + data_v3 = geoip_get_client_history(GEOIP_CLIENT_NETWORKSTATUS); format_iso_time(written, now); - out = start_writing_to_stdio_file(filename, OPEN_FLAGS_REPLACE, + out = start_writing_to_stdio_file(filename, OPEN_FLAGS_APPEND, 0600, &open_file); if (!out) goto done; - if (fprintf(out, "written %s\nstarted-at %s\nns-ips %s\nns-v2-ips %s\n", - written, since, + if (fprintf(out, "dirreq-stats-end %s (%d s)\ndirreq-v3-ips %s\n" + "dirreq-v2-ips %s\n", written, + (unsigned) (now - start_of_dirreq_stats_interval), data_v3 ? data_v3 : "", data_v2 ? data_v2 : "") < 0) goto done; tor_free(data_v2); tor_free(data_v3); - request_start = current_request_period_starts - - (n_old_request_periods * REQUEST_HIST_PERIOD); - format_iso_time(since, request_start); - data_v2 = geoip_get_request_history(now, GEOIP_CLIENT_NETWORKSTATUS_V2); - data_v3 = geoip_get_request_history(now, GEOIP_CLIENT_NETWORKSTATUS); - if (fprintf(out, "requests-start %s\nn-ns-reqs %s\nn-v2-ns-reqs %s\n", - since, + data_v2 = geoip_get_request_history(GEOIP_CLIENT_NETWORKSTATUS_V2); + data_v3 = geoip_get_request_history(GEOIP_CLIENT_NETWORKSTATUS); + if (fprintf(out, "dirreq-v3-reqs %s\ndirreq-v2-reqs %s\n", data_v3 ? data_v3 : "", data_v2 ? data_v2 : "") < 0) goto done; - if (!router_get_my_share_of_directory_requests(&v2_share, &v3_share)) { - if (fprintf(out, "v2-ns-share %0.2lf%%\n", v2_share*100) < 0) + tor_free(data_v2); + tor_free(data_v3); + SMARTLIST_FOREACH(geoip_countries, geoip_country_t *, c, { + c->n_v2_ns_requests = c->n_v3_ns_requests = 0; + }); +#define RESPONSE_GRANULARITY 8 + for (i = 0; i < GEOIP_NS_RESPONSE_NUM; i++) { + ns_v2_responses[i] = round_uint32_to_next_multiple_of( + ns_v2_responses[i], RESPONSE_GRANULARITY); + ns_v3_responses[i] = round_uint32_to_next_multiple_of( + ns_v3_responses[i], RESPONSE_GRANULARITY); + } +#undef RESPONSE_GRANULARITY + if (fprintf(out, "dirreq-v3-resp ok=%u,not-enough-sigs=%u,unavailable=%u," + "not-found=%u,not-modified=%u,busy=%u\n", + ns_v3_responses[GEOIP_SUCCESS], + ns_v3_responses[GEOIP_REJECT_NOT_ENOUGH_SIGS], + ns_v3_responses[GEOIP_REJECT_UNAVAILABLE], + ns_v3_responses[GEOIP_REJECT_NOT_FOUND], + ns_v3_responses[GEOIP_REJECT_NOT_MODIFIED], + ns_v3_responses[GEOIP_REJECT_BUSY]) < 0) + goto done; + if (fprintf(out, "dirreq-v2-resp ok=%u,unavailable=%u," + "not-found=%u,not-modified=%u,busy=%u\n", + ns_v2_responses[GEOIP_SUCCESS], + ns_v2_responses[GEOIP_REJECT_UNAVAILABLE], + ns_v2_responses[GEOIP_REJECT_NOT_FOUND], + ns_v2_responses[GEOIP_REJECT_NOT_MODIFIED], + ns_v2_responses[GEOIP_REJECT_BUSY]) < 0) + goto done; + memset(ns_v2_responses, 0, sizeof(ns_v2_responses)); + memset(ns_v3_responses, 0, sizeof(ns_v3_responses)); + if (!geoip_get_mean_shares(now, &v2_share, &v3_share)) { + if (fprintf(out, "dirreq-v2-share %0.2lf%%\n", v2_share*100) < 0) goto done; - if (fprintf(out, "v3-ns-share %0.2lf%%\n", v3_share*100) < 0) + if (fprintf(out, "dirreq-v3-share %0.2lf%%\n", v3_share*100) < 0) goto done; } + data_v2 = geoip_get_dirreq_history(GEOIP_CLIENT_NETWORKSTATUS_V2, + DIRREQ_DIRECT); + data_v3 = geoip_get_dirreq_history(GEOIP_CLIENT_NETWORKSTATUS, + DIRREQ_DIRECT); + if (fprintf(out, "dirreq-v3-direct-dl %s\ndirreq-v2-direct-dl %s\n", + data_v3 ? data_v3 : "", data_v2 ? data_v2 : "") < 0) + goto done; + tor_free(data_v2); + tor_free(data_v3); + data_v2 = geoip_get_dirreq_history(GEOIP_CLIENT_NETWORKSTATUS_V2, + DIRREQ_TUNNELED); + data_v3 = geoip_get_dirreq_history(GEOIP_CLIENT_NETWORKSTATUS, + DIRREQ_TUNNELED); + if (fprintf(out, "dirreq-v3-tunneled-dl %s\ndirreq-v2-tunneled-dl %s\n", + data_v3 ? data_v3 : "", data_v2 ? data_v2 : "") < 0) + goto done; + finish_writing_to_file(open_file); open_file = NULL; + + start_of_dirreq_stats_interval = now; + done: if (open_file) abort_writing_to_file(open_file); tor_free(filename); + tor_free(statsdir); tor_free(data_v2); tor_free(data_v3); -#endif + return start_of_dirreq_stats_interval + WRITE_STATS_INTERVAL; +} + +/** Start time of bridge stats or 0 if we're not collecting bridge + * statistics. */ +static time_t start_of_bridge_stats_interval; + +/** Initialize bridge stats. */ +void +geoip_bridge_stats_init(time_t now) +{ + start_of_bridge_stats_interval = now; +} + +/** Stop collecting bridge stats in a way that we can re-start doing so in + * geoip_bridge_stats_init(). */ +void +geoip_bridge_stats_term(void) +{ + client_history_clear(); + start_of_bridge_stats_interval = 0; +} + +/** Parse the bridge statistics as they are written to extra-info + * descriptors for being returned to controller clients. Return the + * controller string if successful, or NULL otherwise. */ +static char * +parse_bridge_stats_controller(const char *stats_str, time_t now) +{ + char stats_end_str[ISO_TIME_LEN+1], stats_start_str[ISO_TIME_LEN+1], + *controller_str, *eos, *eol, *summary; + + const char *BRIDGE_STATS_END = "bridge-stats-end "; + const char *BRIDGE_IPS = "bridge-ips "; + const char *BRIDGE_IPS_EMPTY_LINE = "bridge-ips\n"; + const char *tmp; + time_t stats_end_time; + int seconds; + tor_assert(stats_str); + + /* Parse timestamp and number of seconds from + "bridge-stats-end YYYY-MM-DD HH:MM:SS (N s)" */ + tmp = find_str_at_start_of_line(stats_str, BRIDGE_STATS_END); + if (!tmp) + return NULL; + tmp += strlen(BRIDGE_STATS_END); + + if (strlen(tmp) < ISO_TIME_LEN + 6) + return NULL; + strlcpy(stats_end_str, tmp, sizeof(stats_end_str)); + if (parse_iso_time(stats_end_str, &stats_end_time) < 0) + return NULL; + if (stats_end_time < now - (25*60*60) || + stats_end_time > now + (1*60*60)) + return NULL; + seconds = (int)strtol(tmp + ISO_TIME_LEN + 2, &eos, 10); + if (!eos || seconds < 23*60*60) + return NULL; + format_iso_time(stats_start_str, stats_end_time - seconds); + + /* Parse: "bridge-ips CC=N,CC=N,..." */ + tmp = find_str_at_start_of_line(stats_str, BRIDGE_IPS); + if (tmp) { + tmp += strlen(BRIDGE_IPS); + tmp = eat_whitespace_no_nl(tmp); + eol = strchr(tmp, '\n'); + if (eol) + summary = tor_strndup(tmp, eol-tmp); + else + summary = tor_strdup(tmp); + } else { + /* Look if there is an empty "bridge-ips" line */ + tmp = find_str_at_start_of_line(stats_str, BRIDGE_IPS_EMPTY_LINE); + if (!tmp) + return NULL; + summary = tor_strdup(""); + } + + tor_asprintf(&controller_str, + "TimeStarted=\"%s\" CountrySummary=%s", + stats_start_str, summary); + tor_free(summary); + return controller_str; +} + +/** Most recent bridge statistics formatted to be written to extra-info + * descriptors. */ +static char *bridge_stats_extrainfo = NULL; + +/** Most recent bridge statistics formatted to be returned to controller + * clients. */ +static char *bridge_stats_controller = NULL; + +/** Write bridge statistics to $DATADIR/stats/bridge-stats and return + * when we should next try to write statistics. */ +time_t +geoip_bridge_stats_write(time_t now) +{ + char *statsdir = NULL, *filename = NULL, *data = NULL, + written[ISO_TIME_LEN+1], *out = NULL, *controller_str; + size_t len; + + /* Check if 24 hours have passed since starting measurements. */ + if (now < start_of_bridge_stats_interval + WRITE_STATS_INTERVAL) + return start_of_bridge_stats_interval + WRITE_STATS_INTERVAL; + + /* Discard all items in the client history that are too old. */ + geoip_remove_old_clients(start_of_bridge_stats_interval); + + statsdir = get_datadir_fname("stats"); + if (check_private_dir(statsdir, CPD_CREATE) < 0) + goto done; + filename = get_datadir_fname2("stats", "bridge-stats"); + data = geoip_get_client_history(GEOIP_CLIENT_CONNECT); + format_iso_time(written, now); + len = strlen("bridge-stats-end (999999 s)\nbridge-ips \n") + + ISO_TIME_LEN + (data ? strlen(data) : 0) + 42; + out = tor_malloc(len); + if (tor_snprintf(out, len, "bridge-stats-end %s (%u s)\nbridge-ips %s\n", + written, (unsigned) (now - start_of_bridge_stats_interval), + data ? data : "") < 0) + goto done; + write_str_to_file(filename, out, 0); + controller_str = parse_bridge_stats_controller(out, now); + if (!controller_str) + goto done; + start_of_bridge_stats_interval = now; + tor_free(bridge_stats_extrainfo); + tor_free(bridge_stats_controller); + bridge_stats_extrainfo = out; + out = NULL; + bridge_stats_controller = controller_str; + control_event_clients_seen(controller_str); + done: + tor_free(filename); + tor_free(statsdir); + tor_free(data); + tor_free(out); + return start_of_bridge_stats_interval + + WRITE_STATS_INTERVAL; +} + +/** Try to load the most recent bridge statistics from disk, unless we + * have finished a measurement interval lately. */ +static void +load_bridge_stats(time_t now) +{ + char *statsdir, *fname=NULL, *contents, *controller_str; + if (bridge_stats_extrainfo) + return; + statsdir = get_datadir_fname("stats"); + if (check_private_dir(statsdir, CPD_CREATE) < 0) + goto done; + fname = get_datadir_fname2("stats", "bridge-stats"); + contents = read_file_to_str(fname, RFTS_IGNORE_MISSING, NULL); + if (contents) { + controller_str = parse_bridge_stats_controller(contents, now); + if (controller_str) { + bridge_stats_extrainfo = contents; + bridge_stats_controller = controller_str; + } else { + tor_free(contents); + } + } + done: + tor_free(fname); + tor_free(statsdir); +} + +/** Return most recent bridge statistics for inclusion in extra-info + * descriptors, or NULL if we don't have recent bridge statistics. */ +const char * +geoip_get_bridge_stats_extrainfo(time_t now) +{ + load_bridge_stats(now); + return bridge_stats_extrainfo; +} + +/** Return most recent bridge statistics to be returned to controller + * clients, or NULL if we don't have recent bridge statistics. */ +const char * +geoip_get_bridge_stats_controller(time_t now) +{ + load_bridge_stats(now); + return bridge_stats_controller; +} + +/** Start time of entry stats or 0 if we're not collecting entry + * statistics. */ +static time_t start_of_entry_stats_interval; + +/** Initialize entry stats. */ +void +geoip_entry_stats_init(time_t now) +{ + start_of_entry_stats_interval = now; +} + +/** Stop collecting entry stats in a way that we can re-start doing so in + * geoip_entry_stats_init(). */ +void +geoip_entry_stats_term(void) +{ + client_history_clear(); + start_of_entry_stats_interval = 0; +} + +/** Write entry statistics to $DATADIR/stats/entry-stats and return time + * when we would next want to write. */ +time_t +geoip_entry_stats_write(time_t now) +{ + char *statsdir = NULL, *filename = NULL; + char *data = NULL; + char written[ISO_TIME_LEN+1]; + open_file_t *open_file = NULL; + FILE *out; + + if (!start_of_entry_stats_interval) + return 0; /* Not initialized. */ + if (start_of_entry_stats_interval + WRITE_STATS_INTERVAL > now) + goto done; /* Not ready to write. */ + + /* Discard all items in the client history that are too old. */ + geoip_remove_old_clients(start_of_entry_stats_interval); + + statsdir = get_datadir_fname("stats"); + if (check_private_dir(statsdir, CPD_CREATE) < 0) + goto done; + filename = get_datadir_fname2("stats", "entry-stats"); + data = geoip_get_client_history(GEOIP_CLIENT_CONNECT); + format_iso_time(written, now); + out = start_writing_to_stdio_file(filename, OPEN_FLAGS_APPEND, + 0600, &open_file); + if (!out) + goto done; + if (fprintf(out, "entry-stats-end %s (%u s)\nentry-ips %s\n", + written, (unsigned) (now - start_of_entry_stats_interval), + data ? data : "") < 0) + goto done; + + start_of_entry_stats_interval = now; + + finish_writing_to_file(open_file); + open_file = NULL; + done: + if (open_file) + abort_writing_to_file(open_file); + tor_free(filename); + tor_free(statsdir); + tor_free(data); + return start_of_entry_stats_interval + WRITE_STATS_INTERVAL; } /** Helper used to implement GETINFO ip-to-country/... controller command. */ int getinfo_helper_geoip(control_connection_t *control_conn, - const char *question, char **answer) + const char *question, char **answer, + const char **errmsg) { (void)control_conn; - if (geoip_is_loaded() && !strcmpstart(question, "ip-to-country/")) { + if (!geoip_is_loaded()) { + *errmsg = "GeoIP data not loaded"; + return -1; + } + if (!strcmpstart(question, "ip-to-country/")) { int c; uint32_t ip; struct in_addr in; @@ -674,8 +1336,8 @@ clear_geoip_db(void) SMARTLIST_FOREACH(geoip_countries, geoip_country_t *, c, tor_free(c)); smartlist_free(geoip_countries); } - if (country_idxplus1_by_lc_code) - strmap_free(country_idxplus1_by_lc_code, NULL); + + strmap_free(country_idxplus1_by_lc_code, NULL); if (geoip_entries) { SMARTLIST_FOREACH(geoip_entries, geoip_entry_t *, ent, tor_free(ent)); smartlist_free(geoip_entries); @@ -689,13 +1351,24 @@ clear_geoip_db(void) void geoip_free_all(void) { - clientmap_entry_t **ent, **next, *this; - for (ent = HT_START(clientmap, &client_history); ent != NULL; ent = next) { - this = *ent; - next = HT_NEXT_RMV(clientmap, &client_history, ent); - tor_free(this); + { + clientmap_entry_t **ent, **next, *this; + for (ent = HT_START(clientmap, &client_history); ent != NULL; ent = next) { + this = *ent; + next = HT_NEXT_RMV(clientmap, &client_history, ent); + tor_free(this); + } + HT_CLEAR(clientmap, &client_history); + } + { + dirreq_map_entry_t **ent, **next, *this; + for (ent = HT_START(dirreqmap, &dirreq_map); ent != NULL; ent = next) { + this = *ent; + next = HT_NEXT_RMV(dirreqmap, &dirreq_map, ent); + tor_free(this); + } + HT_CLEAR(dirreqmap, &dirreq_map); } - HT_CLEAR(clientmap, &client_history); clear_geoip_db(); } diff --git a/src/or/geoip.h b/src/or/geoip.h new file mode 100644 index 0000000000..68e01deecc --- /dev/null +++ b/src/or/geoip.h @@ -0,0 +1,57 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2010, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file geoip.h + * \brief Header file for geoip.c. + **/ + +#ifndef _TOR_GEOIP_H +#define _TOR_GEOIP_H + +#ifdef GEOIP_PRIVATE +int geoip_parse_entry(const char *line); +#endif +int should_record_bridge_info(or_options_t *options); +int geoip_load_file(const char *filename, or_options_t *options); +int geoip_get_country_by_ip(uint32_t ipaddr); +int geoip_get_n_countries(void); +const char *geoip_get_country_name(country_t num); +int geoip_is_loaded(void); +country_t geoip_get_country(const char *countrycode); + +void geoip_note_client_seen(geoip_client_action_t action, + uint32_t addr, time_t now); +void geoip_remove_old_clients(time_t cutoff); + +void geoip_note_ns_response(geoip_client_action_t action, + geoip_ns_response_t response); +char *geoip_get_client_history(geoip_client_action_t action); +char *geoip_get_request_history(geoip_client_action_t action); +int getinfo_helper_geoip(control_connection_t *control_conn, + const char *question, char **answer, + const char **errmsg); +void geoip_free_all(void); + +void geoip_start_dirreq(uint64_t dirreq_id, size_t response_size, + geoip_client_action_t action, dirreq_type_t type); +void geoip_change_dirreq_state(uint64_t dirreq_id, dirreq_type_t type, + dirreq_state_t new_state); + +void geoip_dirreq_stats_init(time_t now); +time_t geoip_dirreq_stats_write(time_t now); +void geoip_dirreq_stats_term(void); +void geoip_entry_stats_init(time_t now); +time_t geoip_entry_stats_write(time_t now); +void geoip_entry_stats_term(void); +void geoip_bridge_stats_init(time_t now); +time_t geoip_bridge_stats_write(time_t now); +void geoip_bridge_stats_term(void); +const char *geoip_get_bridge_stats_extrainfo(time_t); +const char *geoip_get_bridge_stats_controller(time_t); + +#endif + diff --git a/src/or/hibernate.c b/src/or/hibernate.c index 5ebe5b1c5a..d50d05ed5e 100644 --- a/src/or/hibernate.c +++ b/src/or/hibernate.c @@ -22,6 +22,12 @@ hibernating, phase 2: */ #include "or.h" +#include "config.h" +#include "connection.h" +#include "connection_edge.h" +#include "hibernate.h" +#include "main.h" +#include "router.h" /** Possible values of hibernate_state */ typedef enum { @@ -182,6 +188,9 @@ accounting_parse_options(or_options_t *options, int validate_only) case UNIT_DAY: d = 0; break; + /* Coverity dislikes unreachable default cases; some compilers warn on + * switch statements missing a case. Tell Coverity not to worry. */ + /* coverity[dead_error_begin] */ default: tor_assert(0); } @@ -860,9 +869,11 @@ consider_hibernation(time_t now) * NULL. */ int getinfo_helper_accounting(control_connection_t *conn, - const char *question, char **answer) + const char *question, char **answer, + const char **errmsg) { (void) conn; + (void) errmsg; if (!strcmp(question, "accounting/enabled")) { *answer = tor_strdup(accounting_is_enabled(get_options()) ? "1" : "0"); } else if (!strcmp(question, "accounting/hibernating")) { diff --git a/src/or/hibernate.h b/src/or/hibernate.h new file mode 100644 index 0000000000..8192ac5d8e --- /dev/null +++ b/src/or/hibernate.h @@ -0,0 +1,30 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2010, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file hibernate.h + * \brief Header file for hibernate.c. + **/ + +#ifndef _TOR_HIBERNATE_H +#define _TOR_HIBERNATE_H + +int accounting_parse_options(or_options_t *options, int validate_only); +int accounting_is_enabled(or_options_t *options); +void configure_accounting(time_t now); +void accounting_run_housekeeping(time_t now); +void accounting_add_bytes(size_t n_read, size_t n_written, int seconds); +int accounting_record_bandwidth_usage(time_t now, or_state_t *state); +void hibernate_begin_shutdown(void); +int we_are_hibernating(void); +void consider_hibernation(time_t now); +int getinfo_helper_accounting(control_connection_t *conn, + const char *question, char **answer, + const char **errmsg); +void accounting_set_bandwidth_usage_from_state(or_state_t *state); + +#endif + diff --git a/src/or/main.c b/src/or/main.c index 9052f7c570..2d75a58088 100644 --- a/src/or/main.c +++ b/src/or/main.c @@ -12,12 +12,50 @@ #define MAIN_PRIVATE #include "or.h" +#include "buffers.h" +#include "circuitbuild.h" +#include "circuitlist.h" +#include "circuituse.h" +#include "command.h" +#include "config.h" +#include "connection.h" +#include "connection_edge.h" +#include "connection_or.h" +#include "control.h" +#include "cpuworker.h" +#include "directory.h" +#include "dirserv.h" +#include "dirvote.h" +#include "dns.h" +#include "dnsserv.h" +#include "geoip.h" +#include "hibernate.h" +#include "main.h" +#include "microdesc.h" +#include "networkstatus.h" +#include "ntmain.h" +#include "onion.h" +#include "policies.h" +#include "relay.h" +#include "rendclient.h" +#include "rendcommon.h" +#include "rendservice.h" +#include "rephist.h" +#include "router.h" +#include "routerlist.h" +#include "routerparse.h" #ifdef USE_DMALLOC #include <dmalloc.h> #include <openssl/crypto.h> #endif #include "memarea.h" +#ifdef HAVE_EVENT2_EVENT_H +#include <event2/event.h> +#else +#include <event.h> +#endif + void evdns_shutdown(int); /********* PROTOTYPES **********/ @@ -27,7 +65,7 @@ static void dumpstats(int severity); /* log stats */ static void conn_read_callback(int fd, short event, void *_conn); static void conn_write_callback(int fd, short event, void *_conn); static void signal_callback(int fd, short events, void *arg); -static void second_elapsed_callback(int fd, short event, void *args); +static void second_elapsed_callback(periodic_timer_t *timer, void *args); static int conn_close_if_marked(int i); static void connection_start_reading_from_linked_conn(connection_t *conn); static int connection_should_read_from_linked_conn(connection_t *conn); @@ -127,12 +165,10 @@ connection_add(connection_t *conn) smartlist_add(connection_array, conn); if (conn->s >= 0 || conn->linked) { - conn->read_event = tor_malloc_zero(sizeof(struct event)); - conn->write_event = tor_malloc_zero(sizeof(struct event)); - event_set(conn->read_event, conn->s, EV_READ|EV_PERSIST, - conn_read_callback, conn); - event_set(conn->write_event, conn->s, EV_WRITE|EV_PERSIST, - conn_write_callback, conn); + conn->read_event = tor_event_new(tor_libevent_get_base(), + conn->s, EV_READ|EV_PERSIST, conn_read_callback, conn); + conn->write_event = tor_event_new(tor_libevent_get_base(), + conn->s, EV_WRITE|EV_PERSIST, conn_write_callback, conn); } log_debug(LD_NET,"new conn type %s, socket %d, address %s, n_conns %d.", @@ -142,6 +178,25 @@ connection_add(connection_t *conn) return 0; } +/** Tell libevent that we don't care about <b>conn</b> any more. */ +void +connection_unregister_events(connection_t *conn) +{ + if (conn->read_event) { + if (event_del(conn->read_event)) + log_warn(LD_BUG, "Error removing read event for %d", conn->s); + tor_free(conn->read_event); + } + if (conn->write_event) { + if (event_del(conn->write_event)) + log_warn(LD_BUG, "Error removing write event for %d", conn->s); + tor_free(conn->write_event); + } + if (conn->dns_server_port) { + dnsserv_close_listener(conn); + } +} + /** Remove the connection from the global list, and remove the * corresponding poll entry. Calling this function will shift the last * connection (if any) into the position occupied by conn. @@ -246,17 +301,17 @@ get_connection_array(void) } /** Set the event mask on <b>conn</b> to <b>events</b>. (The event - * mask is a bitmask whose bits are EV_READ and EV_WRITE.) + * mask is a bitmask whose bits are READ_EVENT and WRITE_EVENT) */ void -connection_watch_events(connection_t *conn, short events) +connection_watch_events(connection_t *conn, watchable_events_t events) { - if (events & EV_READ) + if (events & READ_EVENT) connection_start_reading(conn); else connection_stop_reading(conn); - if (events & EV_WRITE) + if (events & WRITE_EVENT) connection_start_writing(conn); else connection_stop_writing(conn); @@ -393,11 +448,11 @@ connection_start_reading_from_linked_conn(connection_t *conn) smartlist_add(active_linked_connection_lst, conn); if (!called_loop_once) { /* This is the first event on the list; we won't be in LOOP_ONCE mode, - * so we need to make sure that the event_loop() actually exits at the - * end of its run through the current connections and - * lets us activate read events for linked connections. */ + * so we need to make sure that the event_base_loop() actually exits at + * the end of its run through the current connections and lets us + * activate read events for linked connections. */ struct timeval tv = { 0, 0 }; - event_loopexit(&tv); + tor_event_base_loopexit(tor_libevent_get_base(), &tv); } } else { tor_assert(smartlist_isin(active_linked_connection_lst, conn)); @@ -540,7 +595,7 @@ conn_close_if_marked(int i) log_info(LD_NET, "Conn (addr %s, fd %d, type %s, state %d) marked, but wants " "to flush %d bytes. (Marked at %s:%d)", - escaped_safe_str(conn->address), + escaped_safe_str_client(conn->address), conn->s, conn_type_to_string(conn->type), conn->state, (int)conn->outbuf_flushlen, conn->marked_for_close_file, conn->marked_for_close); @@ -593,8 +648,8 @@ conn_close_if_marked(int i) "something is wrong with theirs. " "(fd %d, type %s, state %d, marked at %s:%d).", (int)buf_datalen(conn->outbuf), - escaped_safe_str(conn->address), conn->s, - conn_type_to_string(conn->type), conn->state, + escaped_safe_str_client(conn->address), + conn->s, conn_type_to_string(conn->type), conn->state, conn->marked_for_close_file, conn->marked_for_close); } @@ -623,7 +678,7 @@ directory_all_unreachable(time_t now) log_notice(LD_NET, "Is your network connection down? " "Failing connection to '%s:%d'.", - safe_str(edge_conn->socks_request->address), + safe_str_client(edge_conn->socks_request->address), edge_conn->socks_request->port); connection_mark_unattached_ap(edge_conn, END_STREAM_REASON_NET_UNREACHABLE); @@ -722,6 +777,7 @@ run_connection_housekeeping(int i, time_t now) the connection or send a keepalive, depending. */ or_conn = TO_OR_CONN(conn); + tor_assert(conn->outbuf); if (or_conn->is_bad_for_new_circs && !or_conn->n_circuits) { /* It's bad for new circuits, and has no unmarked circuits on it: @@ -777,7 +833,7 @@ run_connection_housekeeping(int i, time_t now) } } -/** Honor a NEWNYM request: make future requests unlinkability to past +/** Honor a NEWNYM request: make future requests unlinkable to past * requests. */ static void signewnym_impl(time_t now) @@ -803,14 +859,15 @@ run_scheduled_events(time_t now) static time_t time_to_try_getting_descriptors = 0; static time_t time_to_reset_descriptor_failures = 0; static time_t time_to_add_entropy = 0; - static time_t time_to_write_hs_statistics = 0; static time_t time_to_write_bridge_status_file = 0; static time_t time_to_downrate_stability = 0; static time_t time_to_save_stability = 0; static time_t time_to_clean_caches = 0; static time_t time_to_recheck_bandwidth = 0; static time_t time_to_check_for_expired_networkstatus = 0; - static time_t time_to_dump_geoip_stats = 0; + static time_t time_to_write_stats_files = 0; + static time_t time_to_write_bridge_stats = 0; + static int should_init_bridge_stats = 1; static time_t time_to_retry_dns_init = 0; or_options_t *options = get_options(); int i; @@ -896,7 +953,7 @@ run_scheduled_events(time_t now) if (now % 10 == 0 && (authdir_mode_tests_reachability(options)) && !we_are_hibernating()) { /* try to determine reachability of the other Tor relays */ - dirserv_test_reachability(now, 0); + dirserv_test_reachability(now); } /** 1d. Periodically, we discount older stability information so that new @@ -938,11 +995,56 @@ run_scheduled_events(time_t now) time_to_check_for_expired_networkstatus = now + CHECK_EXPIRED_NS_INTERVAL; } - if (time_to_dump_geoip_stats < now) { -#define DUMP_GEOIP_STATS_INTERVAL (60*60); - if (time_to_dump_geoip_stats) - dump_geoip_stats(); - time_to_dump_geoip_stats = now + DUMP_GEOIP_STATS_INTERVAL; + /* 1g. Check whether we should write statistics to disk. + */ + if (time_to_write_stats_files < now) { +#define CHECK_WRITE_STATS_INTERVAL (60*60) + time_t next_time_to_write_stats_files = (time_to_write_stats_files > 0 ? + time_to_write_stats_files : now) + CHECK_WRITE_STATS_INTERVAL; + if (options->CellStatistics) { + time_t next_write = + rep_hist_buffer_stats_write(time_to_write_stats_files); + if (next_write && next_write < next_time_to_write_stats_files) + next_time_to_write_stats_files = next_write; + } + if (options->DirReqStatistics) { + time_t next_write = geoip_dirreq_stats_write(time_to_write_stats_files); + if (next_write && next_write < next_time_to_write_stats_files) + next_time_to_write_stats_files = next_write; + } + if (options->EntryStatistics) { + time_t next_write = geoip_entry_stats_write(time_to_write_stats_files); + if (next_write && next_write < next_time_to_write_stats_files) + next_time_to_write_stats_files = next_write; + } + if (options->ExitPortStatistics) { + time_t next_write = rep_hist_exit_stats_write(time_to_write_stats_files); + if (next_write && next_write < next_time_to_write_stats_files) + next_time_to_write_stats_files = next_write; + } + time_to_write_stats_files = next_time_to_write_stats_files; + } + + /* 1h. Check whether we should write bridge statistics to disk. + */ + if (should_record_bridge_info(options)) { + if (time_to_write_bridge_stats < now) { + if (should_init_bridge_stats) { + /* (Re-)initialize bridge statistics. */ + geoip_bridge_stats_init(now); + time_to_write_bridge_stats = now + WRITE_STATS_INTERVAL; + should_init_bridge_stats = 0; + } else { + /* Possibly write bridge statistics to disk and ask when to write + * them next time. */ + time_to_write_bridge_stats = geoip_bridge_stats_write( + time_to_write_bridge_stats); + } + } + } else if (!should_init_bridge_stats) { + /* Bridge mode was turned off. Ensure that stats are re-initialized + * next time bridge mode is turned on. */ + should_init_bridge_stats = 1; } /* Remove old information from rephist and the rend cache. */ @@ -1116,12 +1218,6 @@ run_scheduled_events(time_t now) } } - /** 10. write hidden service usage statistic to disk */ - if (options->HSAuthorityRecordStats && time_to_write_hs_statistics < now) { - hs_usage_write_statistics_to_file(now); -#define WRITE_HSUSAGE_INTERVAL (30*60) - time_to_write_hs_statistics = now+WRITE_HSUSAGE_INTERVAL; - } /** 10b. write bridge networkstatus file to disk */ if (options->BridgeAuthoritativeDir && time_to_write_bridge_status_file < now) { @@ -1131,39 +1227,30 @@ run_scheduled_events(time_t now) } } -/** Libevent timer: used to invoke second_elapsed_callback() once per - * second. */ -static struct event *timeout_event = NULL; +/** Timer: used to invoke second_elapsed_callback() once per second. */ +static periodic_timer_t *second_timer = NULL; /** Number of libevent errors in the last second: we die if we get too many. */ static int n_libevent_errors = 0; /** Libevent callback: invoked once every second. */ static void -second_elapsed_callback(int fd, short event, void *args) +second_elapsed_callback(periodic_timer_t *timer, void *arg) { /* XXXX This could be sensibly refactored into multiple callbacks, and we * could use Libevent's timers for this rather than checking the current * time against a bunch of timeouts every second. */ - static struct timeval one_second; static time_t current_second = 0; time_t now; size_t bytes_written; size_t bytes_read; int seconds_elapsed; or_options_t *options = get_options(); - (void)fd; - (void)event; - (void)args; - if (!timeout_event) { - timeout_event = tor_malloc_zero(sizeof(struct event)); - evtimer_set(timeout_event, second_elapsed_callback, NULL); - one_second.tv_sec = 1; - one_second.tv_usec = 0; - } + (void)timer; + (void)arg; n_libevent_errors = 0; - /* log_fn(LOG_NOTICE, "Tick."); */ + /* log_notice(LD_GENERAL, "Tick."); */ now = time(NULL); update_approx_time(now); @@ -1228,18 +1315,6 @@ second_elapsed_callback(int fd, short event, void *args) run_scheduled_events(now); current_second = now; /* remember which second it is, for next time */ - -#if 0 - if (current_second % 300 == 0) { - rep_history_clean(current_second - options->RephistTrackTime); - dumpmemusage(get_min_log_level()<LOG_INFO ? - get_min_log_level() : LOG_INFO); - } -#endif - - if (evtimer_add(timeout_event, &one_second)) - log_err(LD_NET, - "Error from libevent when setting one-second timeout event"); } #ifndef MS_WINDOWS @@ -1400,8 +1475,10 @@ do_main_loop(void) /* initialize the bootstrap status events to know we're starting up */ control_event_bootstrap(BOOTSTRAP_STATUS_STARTING, 0); - if (trusted_dirs_reload_certs()) - return -1; + if (trusted_dirs_reload_certs()) { + log_warn(LD_DIR, + "Couldn't load all cached v3 certificates. Starting anyway."); + } if (router_reload_v2_networkstatus()) { return -1; } @@ -1418,18 +1495,23 @@ do_main_loop(void) now = time(NULL); directory_info_has_arrived(now, 1); - if (authdir_mode_tests_reachability(get_options())) { - /* the directory is already here, run startup things */ - dirserv_test_reachability(now, 1); - } - if (server_mode(get_options())) { /* launch cpuworkers. Need to do this *after* we've read the onion key. */ cpu_init(); } /* set up once-a-second callback. */ - second_elapsed_callback(0,0,NULL); + if (! second_timer) { + struct timeval one_second; + one_second.tv_sec = 1; + one_second.tv_usec = 0; + + second_timer = periodic_timer_new(tor_libevent_get_base(), + &one_second, + second_elapsed_callback, + NULL); + tor_assert(second_timer); + } for (;;) { if (nt_service_is_stopping()) @@ -1448,20 +1530,16 @@ do_main_loop(void) /* poll until we have an event, or the second ends, or until we have * some active linked connections to trigger events for. */ - loop_result = event_loop(called_loop_once ? EVLOOP_ONCE : 0); + loop_result = event_base_loop(tor_libevent_get_base(), + called_loop_once ? EVLOOP_ONCE : 0); /* let catch() handle things like ^c, and otherwise don't worry about it */ if (loop_result < 0) { int e = tor_socket_errno(-1); /* let the program survive things like ^z */ if (e != EINTR && !ERRNO_IS_EINPROGRESS(e)) { -#ifdef HAVE_EVENT_GET_METHOD log_err(LD_NET,"libevent call with %s failed: %s [%d]", - event_get_method(), tor_socket_strerror(e), e); -#else - log_err(LD_NET,"libevent call failed: %s [%d]", - tor_socket_strerror(e), e); -#endif + tor_libevent_get_method(), tor_socket_strerror(e), e); return -1; #ifndef MS_WINDOWS } else if (e == EINVAL) { @@ -1604,6 +1682,7 @@ dumpmemusage(int severity) U64_PRINTF_ARG(rephist_total_alloc), rephist_total_num); dump_routerlist_mem_usage(severity); dump_cell_pool_usage(severity); + dump_dns_mem_usage(severity); buf_dump_freelist_sizes(severity); tor_log_mallinfo(severity); } @@ -1630,7 +1709,8 @@ dumpstats(int severity) if (!connection_is_listener(conn)) { log(severity,LD_GENERAL, "Conn %d is to %s:%d.", i, - safe_str(conn->address), conn->port); + safe_str_client(conn->address), + conn->port); log(severity,LD_GENERAL, "Conn %d: %d bytes waiting on inbuf (len %d, last read %d secs ago)", i, @@ -1727,7 +1807,7 @@ handle_signals(int is_parent) { #ifndef MS_WINDOWS /* do signal stuff only on Unix */ int i; - static int signals[] = { + static const int signals[] = { SIGINT, /* do a controlled slow shutdown */ SIGTERM, /* to terminate now */ SIGPIPE, /* otherwise SIGPIPE kills us */ @@ -1739,12 +1819,13 @@ handle_signals(int is_parent) #endif SIGCHLD, /* handle dns/cpu workers that exit */ -1 }; - static struct event signal_events[16]; /* bigger than it has to be. */ + static struct event *signal_events[16]; /* bigger than it has to be. */ if (is_parent) { for (i = 0; signals[i] >= 0; ++i) { - signal_set(&signal_events[i], signals[i], signal_callback, - (void*)(uintptr_t)signals[i]); - if (signal_add(&signal_events[i], NULL)) + signal_events[i] = tor_evsignal_new( + tor_libevent_get_base(), signals[i], signal_callback, + (void*)(uintptr_t)signals[i]); + if (event_add(signal_events[i], NULL)) log_warn(LD_BUG, "Error from libevent when adding event for signal %d", signals[i]); } @@ -1833,7 +1914,9 @@ tor_init(int argc, char *argv[]) "and you probably shouldn't."); #endif - if (crypto_global_init(get_options()->HardwareAccel)) { + if (crypto_global_init(get_options()->HardwareAccel, + get_options()->AccelName, + get_options()->AccelDir)) { log_err(LD_BUG, "Unable to initialize OpenSSL. Exiting."); return -1; } @@ -1927,7 +2010,6 @@ tor_free_all(int postfork) rend_cache_free_all(); rend_service_authorization_free_all(); rep_hist_free_all(); - hs_usage_free_all(); dns_free_all(); clear_pending_onions(); circuit_free_all(); @@ -1935,6 +2017,7 @@ tor_free_all(int postfork) connection_free_all(); buf_shrink_freelists(1); memarea_clear_freelist(); + microdesc_free_all(); if (!postfork) { config_free_all(); router_free_all(); @@ -1945,13 +2028,11 @@ tor_free_all(int postfork) tor_tls_free_all(); } /* stuff in main.c */ - if (connection_array) - smartlist_free(connection_array); - if (closeable_connection_lst) - smartlist_free(closeable_connection_lst); - if (active_linked_connection_lst) - smartlist_free(active_linked_connection_lst); - tor_free(timeout_event); + + smartlist_free(connection_array); + smartlist_free(closeable_connection_lst); + smartlist_free(active_linked_connection_lst); + periodic_timer_free(second_timer); if (!postfork) { release_lockfile(); } @@ -2042,6 +2123,31 @@ do_hash_password(void) printf("16:%s\n",output); } +#if defined (WINCE) +int +find_flashcard_path(PWCHAR path, size_t size) +{ + WIN32_FIND_DATA d = {0}; + HANDLE h = NULL; + + if (!path) + return -1; + + h = FindFirstFlashCard(&d); + if (h == INVALID_HANDLE_VALUE) + return -1; + + if (wcslen(d.cFileName) == 0) { + FindClose(h); + return -1; + } + + wcsncpy(path,d.cFileName,size); + FindClose(h); + return 0; +} +#endif + /** Main entry point for the Tor process. Called from main(). */ /* This function is distinct from main() only so we can link main.c into * the unittest binary without conflicting with the unittests' main. */ @@ -2049,6 +2155,33 @@ int tor_main(int argc, char *argv[]) { int result = 0; +#if defined (WINCE) + WCHAR path [MAX_PATH] = {0}; + WCHAR fullpath [MAX_PATH] = {0}; + PWCHAR p = NULL; + FILE* redir = NULL; + FILE* redirdbg = NULL; + + // this is to facilitate debugging by opening + // a file on a folder shared by the wm emulator. + // if no flashcard (real or emulated) is present, + // log files will be written in the root folder + if (find_flashcard_path(path,MAX_PATH) == -1) + { + redir = _wfreopen( L"\\stdout.log", L"w", stdout ); + redirdbg = _wfreopen( L"\\stderr.log", L"w", stderr ); + } else { + swprintf(fullpath,L"\\%s\\tor",path); + CreateDirectory(fullpath,NULL); + + swprintf(fullpath,L"\\%s\\tor\\stdout.log",path); + redir = _wfreopen( fullpath, L"w", stdout ); + + swprintf(fullpath,L"\\%s\\tor\\stderr.log",path); + redirdbg = _wfreopen( fullpath, L"w", stderr ); + } +#endif + update_approx_time(time(NULL)); tor_threads_init(); init_logging(); diff --git a/src/or/main.h b/src/or/main.h new file mode 100644 index 0000000000..6eeb95449a --- /dev/null +++ b/src/or/main.h @@ -0,0 +1,67 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2010, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file main.h + * \brief Header file for main.c. + **/ + +#ifndef _TOR_MAIN_H +#define _TOR_MAIN_H + +extern int has_completed_circuit; + +int connection_add(connection_t *conn); +int connection_remove(connection_t *conn); +void connection_unregister_events(connection_t *conn); +int connection_in_array(connection_t *conn); +void add_connection_to_closeable_list(connection_t *conn); +int connection_is_on_closeable_list(connection_t *conn); + +smartlist_t *get_connection_array(void); + +typedef enum watchable_events { + READ_EVENT=0x02, + WRITE_EVENT=0x04 +} watchable_events_t; +void connection_watch_events(connection_t *conn, watchable_events_t events); +int connection_is_reading(connection_t *conn); +void connection_stop_reading(connection_t *conn); +void connection_start_reading(connection_t *conn); + +int connection_is_writing(connection_t *conn); +void connection_stop_writing(connection_t *conn); +void connection_start_writing(connection_t *conn); + +void connection_stop_reading_from_linked_conn(connection_t *conn); + +void directory_all_unreachable(time_t now); +void directory_info_has_arrived(time_t now, int from_cache); + +void ip_address_changed(int at_interface); +void dns_servers_relaunch_checks(void); + +void control_signal_act(int the_signal); +void handle_signals(int is_parent); + +int try_locking(or_options_t *options, int err_if_locked); +int have_lockfile(void); +void release_lockfile(void); + +void tor_cleanup(void); +void tor_free_all(int postfork); + +int tor_main(int argc, char *argv[]); + +#ifdef MAIN_PRIVATE +int do_main_loop(void); +int do_list_fingerprint(void); +void do_hash_password(void); +int tor_init(int argc, char **argv); +#endif + +#endif + diff --git a/src/or/microdesc.c b/src/or/microdesc.c new file mode 100644 index 0000000000..f56ccd9ee2 --- /dev/null +++ b/src/or/microdesc.c @@ -0,0 +1,414 @@ +/* Copyright (c) 2009-2010, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "or.h" +#include "config.h" +#include "microdesc.h" +#include "routerparse.h" + +/** A data structure to hold a bunch of cached microdescriptors. There are + * two active files in the cache: a "cache file" that we mmap, and a "journal + * file" that we append to. Periodically, we rebuild the cache file to hold + * only the microdescriptors that we want to keep */ +struct microdesc_cache_t { + /** Map from sha256-digest to microdesc_t for every microdesc_t in the + * cache. */ + HT_HEAD(microdesc_map, microdesc_t) map; + + /** Name of the cache file. */ + char *cache_fname; + /** Name of the journal file. */ + char *journal_fname; + /** Mmap'd contents of the cache file, or NULL if there is none. */ + tor_mmap_t *cache_content; + /** Number of bytes used in the journal file. */ + size_t journal_len; + + /** Total bytes of microdescriptor bodies we have added to this cache */ + uint64_t total_len_seen; + /** Total number of microdescriptors we have added to this cache */ + unsigned n_seen; +}; + +/** Helper: computes a hash of <b>md</b> to place it in a hash table. */ +static INLINE unsigned int +_microdesc_hash(microdesc_t *md) +{ + unsigned *d = (unsigned*)md->digest; +#if SIZEOF_INT == 4 + return d[0] ^ d[1] ^ d[2] ^ d[3] ^ d[4] ^ d[5] ^ d[6] ^ d[7]; +#else + return d[0] ^ d[1] ^ d[2] ^ d[3]; +#endif +} + +/** Helper: compares <b>a</b> and </b> for equality for hash-table purposes. */ +static INLINE int +_microdesc_eq(microdesc_t *a, microdesc_t *b) +{ + return !memcmp(a->digest, b->digest, DIGEST256_LEN); +} + +HT_PROTOTYPE(microdesc_map, microdesc_t, node, + _microdesc_hash, _microdesc_eq); +HT_GENERATE(microdesc_map, microdesc_t, node, + _microdesc_hash, _microdesc_eq, 0.6, + _tor_malloc, _tor_realloc, _tor_free); + +/** Write the body of <b>md</b> into <b>f</b>, with appropriate annotations. + * On success, return the total number of bytes written, and set + * *<b>annotation_len_out</b> to the number of bytes written as + * annotations. */ +static ssize_t +dump_microdescriptor(FILE *f, microdesc_t *md, size_t *annotation_len_out) +{ + ssize_t r = 0; + size_t written; + /* XXXX drops unkown annotations. */ + if (md->last_listed) { + char buf[ISO_TIME_LEN+1]; + char annotation[ISO_TIME_LEN+32]; + format_iso_time(buf, md->last_listed); + tor_snprintf(annotation, sizeof(annotation), "@last-listed %s\n", buf); + fputs(annotation, f); + r += strlen(annotation); + *annotation_len_out = r; + } else { + *annotation_len_out = 0; + } + + md->off = (off_t) ftell(f); + written = fwrite(md->body, 1, md->bodylen, f); + if (written != md->bodylen) { + log_warn(LD_DIR, + "Couldn't dump microdescriptor (wrote %lu out of %lu): %s", + (unsigned long)written, (unsigned long)md->bodylen, + strerror(ferror(f))); + return -1; + } + r += md->bodylen; + return r; +} + +/** Holds a pointer to the current microdesc_cache_t object, or NULL if no + * such object has been allocated. */ +static microdesc_cache_t *the_microdesc_cache = NULL; + +/** Return a pointer to the microdescriptor cache, loading it if necessary. */ +microdesc_cache_t * +get_microdesc_cache(void) +{ + if (PREDICT_UNLIKELY(the_microdesc_cache==NULL)) { + microdesc_cache_t *cache = tor_malloc_zero(sizeof(microdesc_cache_t)); + HT_INIT(microdesc_map, &cache->map); + cache->cache_fname = get_datadir_fname("cached-microdescs"); + cache->journal_fname = get_datadir_fname("cached-microdescs.new"); + microdesc_cache_reload(cache); + the_microdesc_cache = cache; + } + return the_microdesc_cache; +} + +/* There are three sources of microdescriptors: + 1) Generated by us while acting as a directory authority. + 2) Loaded from the cache on disk. + 3) Downloaded. +*/ + +/** Decode the microdescriptors from the string starting at <b>s</b> and + * ending at <b>eos</b>, and store them in <b>cache</b>. If <b>no-save</b>, + * mark them as non-writable to disk. If <b>where</b> is SAVED_IN_CACHE, + * leave their bodies as pointers to the mmap'd cache. If where is + * <b>SAVED_NOWHERE</b>, do not allow annotations. Return a list of the added + * microdescriptors. */ +smartlist_t * +microdescs_add_to_cache(microdesc_cache_t *cache, + const char *s, const char *eos, saved_location_t where, + int no_save) +{ + /*XXXX need an argument that sets last_listed as appropriate. */ + + smartlist_t *descriptors, *added; + const int allow_annotations = (where != SAVED_NOWHERE); + const int copy_body = (where != SAVED_IN_CACHE); + + descriptors = microdescs_parse_from_string(s, eos, + allow_annotations, + copy_body); + + added = microdescs_add_list_to_cache(cache, descriptors, where, no_save); + smartlist_free(descriptors); + return added; +} + +/* As microdescs_add_to_cache, but takes a list of micrdescriptors instead of + * a string to encode. Frees any members of <b>descriptors</b> that it does + * not add. */ +smartlist_t * +microdescs_add_list_to_cache(microdesc_cache_t *cache, + smartlist_t *descriptors, saved_location_t where, + int no_save) +{ + smartlist_t *added; + open_file_t *open_file = NULL; + FILE *f = NULL; + // int n_added = 0; + ssize_t size = 0; + + if (where == SAVED_NOWHERE && !no_save) { + f = start_writing_to_stdio_file(cache->journal_fname, + OPEN_FLAGS_APPEND|O_BINARY, + 0600, &open_file); + if (!f) { + log_warn(LD_DIR, "Couldn't append to journal in %s: %s", + cache->journal_fname, strerror(errno)); + return NULL; + } + } + + added = smartlist_create(); + SMARTLIST_FOREACH_BEGIN(descriptors, microdesc_t *, md) { + microdesc_t *md2; + md2 = HT_FIND(microdesc_map, &cache->map, md); + if (md2) { + /* We already had this one. */ + if (md2->last_listed < md->last_listed) + md2->last_listed = md->last_listed; + microdesc_free(md); + continue; + } + + /* Okay, it's a new one. */ + if (f) { + size_t annotation_len; + size = dump_microdescriptor(f, md, &annotation_len); + if (size < 0) { + /* XXX handle errors from dump_microdescriptor() */ + /* log? return -1? die? coredump the universe? */ + continue; + } + md->saved_location = SAVED_IN_JOURNAL; + cache->journal_len += size; + } else { + md->saved_location = where; + } + + md->no_save = no_save; + + HT_INSERT(microdesc_map, &cache->map, md); + smartlist_add(added, md); + ++cache->n_seen; + cache->total_len_seen += md->bodylen; + } SMARTLIST_FOREACH_END(md); + + if (f) + finish_writing_to_file(open_file); /*XXX Check me.*/ + + { + size_t old_content_len = + cache->cache_content ? cache->cache_content->size : 0; + if (cache->journal_len > 16384 + old_content_len && + cache->journal_len > old_content_len * 2) { + microdesc_cache_rebuild(cache); + } + } + + return added; +} + +/** Remove every microdescriptor in <b>cache</b>. */ +void +microdesc_cache_clear(microdesc_cache_t *cache) +{ + microdesc_t **entry, **next; + for (entry = HT_START(microdesc_map, &cache->map); entry; entry = next) { + microdesc_t *md = *entry; + next = HT_NEXT_RMV(microdesc_map, &cache->map, entry); + microdesc_free(md); + } + HT_CLEAR(microdesc_map, &cache->map); + if (cache->cache_content) { + tor_munmap_file(cache->cache_content); + cache->cache_content = NULL; + } + cache->total_len_seen = 0; + cache->n_seen = 0; +} + +/** Reload the contents of <b>cache</b> from disk. If it is empty, load it + * for the first time. Return 0 on success, -1 on failure. */ +int +microdesc_cache_reload(microdesc_cache_t *cache) +{ + struct stat st; + char *journal_content; + smartlist_t *added; + tor_mmap_t *mm; + int total = 0; + + microdesc_cache_clear(cache); + + mm = cache->cache_content = tor_mmap_file(cache->cache_fname); + if (mm) { + added = microdescs_add_to_cache(cache, mm->data, mm->data+mm->size, + SAVED_IN_CACHE, 0); + if (added) { + total += smartlist_len(added); + smartlist_free(added); + } + } + + journal_content = read_file_to_str(cache->journal_fname, + RFTS_IGNORE_MISSING, &st); + if (journal_content) { + added = microdescs_add_to_cache(cache, journal_content, + journal_content+st.st_size, + SAVED_IN_JOURNAL, 0); + if (added) { + total += smartlist_len(added); + smartlist_free(added); + } + tor_free(journal_content); + } + log_notice(LD_DIR, "Reloaded microdescriptor cache. Found %d descriptors.", + total); + return 0; +} + +/** Regenerate the main cache file for <b>cache</b>, clear the journal file, + * and update every microdesc_t in the cache with pointers to its new + * location. */ +int +microdesc_cache_rebuild(microdesc_cache_t *cache) +{ + open_file_t *open_file; + FILE *f; + microdesc_t **mdp; + smartlist_t *wrote; + ssize_t size; + off_t off = 0; + int orig_size, new_size; + + log_info(LD_DIR, "Rebuilding the microdescriptor cache..."); + orig_size = (int)(cache->cache_content ? cache->cache_content->size : 0); + orig_size += (int)cache->journal_len; + + f = start_writing_to_stdio_file(cache->cache_fname, + OPEN_FLAGS_REPLACE|O_BINARY, + 0600, &open_file); + if (!f) + return -1; + + wrote = smartlist_create(); + + HT_FOREACH(mdp, microdesc_map, &cache->map) { + microdesc_t *md = *mdp; + size_t annotation_len; + if (md->no_save) + continue; + + size = dump_microdescriptor(f, md, &annotation_len); + if (size < 0) { + /* XXX handle errors from dump_microdescriptor() */ + /* log? return -1? die? coredump the universe? */ + continue; + } + md->off = off + annotation_len; + off += size; + if (md->saved_location != SAVED_IN_CACHE) { + tor_free(md->body); + md->saved_location = SAVED_IN_CACHE; + } + smartlist_add(wrote, md); + } + + finish_writing_to_file(open_file); /*XXX Check me.*/ + + if (cache->cache_content) + tor_munmap_file(cache->cache_content); + cache->cache_content = tor_mmap_file(cache->cache_fname); + + if (!cache->cache_content && smartlist_len(wrote)) { + log_err(LD_DIR, "Couldn't map file that we just wrote to %s!", + cache->cache_fname); + smartlist_free(wrote); + return -1; + } + SMARTLIST_FOREACH_BEGIN(wrote, microdesc_t *, md) { + tor_assert(md->saved_location == SAVED_IN_CACHE); + md->body = (char*)cache->cache_content->data + md->off; + tor_assert(!memcmp(md->body, "onion-key", 9)); + } SMARTLIST_FOREACH_END(md); + + smartlist_free(wrote); + + write_str_to_file(cache->journal_fname, "", 1); + cache->journal_len = 0; + + new_size = (int)cache->cache_content->size; + log_info(LD_DIR, "Done rebuilding microdesc cache. " + "Saved %d bytes; %d still used.", + orig_size-new_size, new_size); + + return 0; +} + +/** Deallocate a single microdescriptor. Note: the microdescriptor MUST have + * previously been removed from the cache if it had ever been inserted. */ +void +microdesc_free(microdesc_t *md) +{ + if (!md) + return; + /* Must be removed from hash table! */ + if (md->onion_pkey) + crypto_free_pk_env(md->onion_pkey); + if (md->body && md->saved_location != SAVED_IN_CACHE) + tor_free(md->body); + + if (md->family) { + SMARTLIST_FOREACH(md->family, char *, cp, tor_free(cp)); + smartlist_free(md->family); + } + tor_free(md->exitsummary); + + tor_free(md); +} + +/** Free all storage held in the microdesc.c module. */ +void +microdesc_free_all(void) +{ + if (the_microdesc_cache) { + microdesc_cache_clear(the_microdesc_cache); + tor_free(the_microdesc_cache->cache_fname); + tor_free(the_microdesc_cache->journal_fname); + tor_free(the_microdesc_cache); + } +} + +/** If there is a microdescriptor in <b>cache</b> whose sha256 digest is + * <b>d</b>, return it. Otherwise return NULL. */ +microdesc_t * +microdesc_cache_lookup_by_digest256(microdesc_cache_t *cache, const char *d) +{ + microdesc_t *md, search; + if (!cache) + cache = get_microdesc_cache(); + memcpy(search.digest, d, DIGEST256_LEN); + md = HT_FIND(microdesc_map, &cache->map, &search); + return md; +} + +/** Return the mean size of decriptors added to <b>cache</b> since it was last + * cleared. Used to estimate the size of large downloads. */ +size_t +microdesc_average_size(microdesc_cache_t *cache) +{ + if (!cache) + cache = get_microdesc_cache(); + if (!cache->n_seen) + return 512; + return (size_t)(cache->total_len_seen / cache->n_seen); +} + diff --git a/src/or/microdesc.h b/src/or/microdesc.h new file mode 100644 index 0000000000..2d1a60ad0a --- /dev/null +++ b/src/or/microdesc.h @@ -0,0 +1,37 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2010, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file microdesc.h + * \brief Header file for microdesc.c. + **/ + +#ifndef _TOR_MICRODESC_H +#define _TOR_MICRODESC_H + +microdesc_cache_t *get_microdesc_cache(void); + +smartlist_t *microdescs_add_to_cache(microdesc_cache_t *cache, + const char *s, const char *eos, saved_location_t where, + int no_save); +smartlist_t *microdescs_add_list_to_cache(microdesc_cache_t *cache, + smartlist_t *descriptors, saved_location_t where, + int no_save); + +int microdesc_cache_rebuild(microdesc_cache_t *cache); +int microdesc_cache_reload(microdesc_cache_t *cache); +void microdesc_cache_clear(microdesc_cache_t *cache); + +microdesc_t *microdesc_cache_lookup_by_digest256(microdesc_cache_t *cache, + const char *d); + +size_t microdesc_average_size(microdesc_cache_t *cache); + +void microdesc_free(microdesc_t *md); +void microdesc_free_all(void); + +#endif + diff --git a/src/or/networkstatus.c b/src/or/networkstatus.c index a43ed52547..0ec69cb29b 100644 --- a/src/or/networkstatus.c +++ b/src/or/networkstatus.c @@ -11,6 +11,20 @@ */ #include "or.h" +#include "circuitbuild.h" +#include "config.h" +#include "connection.h" +#include "connection_or.h" +#include "control.h" +#include "directory.h" +#include "dirserv.h" +#include "dirvote.h" +#include "main.h" +#include "networkstatus.h" +#include "relay.h" +#include "router.h" +#include "routerlist.h" +#include "routerparse.h" /* For tracking v2 networkstatus documents. Only caches do this now. */ @@ -35,16 +49,22 @@ static networkstatus_t *current_consensus = NULL; /** A v3 consensus networkstatus that we've received, but which we don't * have enough certificates to be happy about. */ -static networkstatus_t *consensus_waiting_for_certs = NULL; -/** The encoded version of consensus_waiting_for_certs. */ -static char *consensus_waiting_for_certs_body = NULL; -/** When did we set the current value of consensus_waiting_for_certs? If this - * is too recent, we shouldn't try to fetch a new consensus for a little while, - * to give ourselves time to get certificates for this one. */ -static time_t consensus_waiting_for_certs_set_at = 0; -/** Set to 1 if we've been holding on to consensus_waiting_for_certs so long - * that we should treat it as maybe being bad. */ -static int consensus_waiting_for_certs_dl_failed = 0; +typedef struct consensus_waiting_for_certs_t { + /** The consensus itself. */ + networkstatus_t *consensus; + /** The encoded version of the consensus, nul-terminated. */ + char *body; + /** When did we set the current value of consensus_waiting_for_certs? If + * this is too recent, we shouldn't try to fetch a new consensus for a + * little while, to give ourselves time to get certificates for this one. */ + time_t set_at; + /** Set to 1 if we've been holding on to it for so long we should maybe + * treat it as being bad. */ + int dl_failed; +} consensus_waiting_for_certs_t; + +static consensus_waiting_for_certs_t + consensus_waiting_for_certs[N_CONSENSUS_FLAVORS]; /** The last time we tried to download a networkstatus, or 0 for "never". We * use this to rate-limit download attempts for directory caches (including @@ -56,7 +76,7 @@ static time_t last_networkstatus_download_attempted = 0; * before the current consensus becomes invalid. */ static time_t time_to_download_next_consensus = 0; /** Download status for the current consensus networkstatus. */ -static download_status_t consensus_dl_status = { 0, 0, DL_SCHED_CONSENSUS }; +static download_status_t consensus_dl_status[N_CONSENSUS_FLAVORS]; /** True iff we have logged a warning about this OR's version being older than * listed by the authorities. */ @@ -89,6 +109,7 @@ networkstatus_reset_warnings(void) void networkstatus_reset_download_failures(void) { + int i; const smartlist_t *networkstatus_v2_list = networkstatus_get_v2_list(); SMARTLIST_FOREACH(networkstatus_v2_list, networkstatus_v2_t *, ns, SMARTLIST_FOREACH(ns->entries, routerstatus_t *, rs, @@ -97,7 +118,8 @@ networkstatus_reset_download_failures(void) rs->need_to_mirror = 1; }));; - download_status_reset(&consensus_dl_status); + for (i=0; i < N_CONSENSUS_FLAVORS; ++i) + download_status_reset(&consensus_dl_status[i]); if (v2_download_status_map) { digestmap_iter_t *iter; digestmap_t *map = v2_download_status_map; @@ -170,7 +192,7 @@ router_reload_v2_networkstatus(void) return 0; } -/** Read the cached v3 consensus networkstatus from the disk. */ +/** Read every cached v3 consensus networkstatus from the disk. */ int router_reload_consensus_networkstatus(void) { @@ -179,31 +201,46 @@ router_reload_consensus_networkstatus(void) struct stat st; or_options_t *options = get_options(); const unsigned int flags = NSSET_FROM_CACHE | NSSET_DONT_DOWNLOAD_CERTS; + int flav; /* FFFF Suppress warnings if cached consensus is bad? */ + for (flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) { + char buf[128]; + const char *flavor = networkstatus_get_flavor_name(flav); + if (flav == FLAV_NS) { + filename = get_datadir_fname("cached-consensus"); + } else { + tor_snprintf(buf, sizeof(buf), "cached-%s-consensus", flavor); + filename = get_datadir_fname(buf); + } + s = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL); + if (s) { + if (networkstatus_set_current_consensus(s, flavor, flags) < -1) { + log_warn(LD_FS, "Couldn't load consensus %s networkstatus from \"%s\"", + flavor, filename); + } + tor_free(s); + } + tor_free(filename); - filename = get_datadir_fname("cached-consensus"); - s = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL); - if (s) { - if (networkstatus_set_current_consensus(s, flags) < -1) { - log_warn(LD_FS, "Couldn't load consensus networkstatus from \"%s\"", - filename); + if (flav == FLAV_NS) { + filename = get_datadir_fname("unverified-consensus"); + } else { + tor_snprintf(buf, sizeof(buf), "unverified-%s-consensus", flavor); + filename = get_datadir_fname(buf); } - tor_free(s); - } - tor_free(filename); - filename = get_datadir_fname("unverified-consensus"); - s = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL); - if (s) { - if (networkstatus_set_current_consensus(s, + s = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL); + if (s) { + if (networkstatus_set_current_consensus(s, flavor, flags|NSSET_WAS_WAITING_FOR_CERTS)) { - log_info(LD_FS, "Couldn't load consensus networkstatus from \"%s\"", - filename); + log_info(LD_FS, "Couldn't load consensus %s networkstatus from \"%s\"", + flavor, filename); + } + tor_free(s); } - tor_free(s); + tor_free(filename); } - tor_free(filename); if (!current_consensus || (stat(options->FallbackNetworkstatusFile, &st)==0 && @@ -211,7 +248,7 @@ router_reload_consensus_networkstatus(void) s = read_file_to_str(options->FallbackNetworkstatusFile, RFTS_IGNORE_MISSING, NULL); if (s) { - if (networkstatus_set_current_consensus(s, + if (networkstatus_set_current_consensus(s, "ns", flags|NSSET_ACCEPT_OBSOLETE)) { log_info(LD_FS, "Couldn't load consensus networkstatus from \"%s\"", options->FallbackNetworkstatusFile); @@ -242,8 +279,16 @@ router_reload_consensus_networkstatus(void) static void vote_routerstatus_free(vote_routerstatus_t *rs) { + vote_microdesc_hash_t *h, *next; + if (!rs) + return; tor_free(rs->version); tor_free(rs->status.exitsummary); + for (h = rs->microdesc; h; h = next) { + tor_free(h->microdesc_hash_line); + next = h->next; + tor_free(h); + } tor_free(rs); } @@ -251,6 +296,8 @@ vote_routerstatus_free(vote_routerstatus_t *rs) void routerstatus_free(routerstatus_t *rs) { + if (!rs) + return; tor_free(rs->exitsummary); tor_free(rs); } @@ -259,6 +306,8 @@ routerstatus_free(routerstatus_t *rs) void networkstatus_v2_free(networkstatus_v2_t *ns) { + if (!ns) + return; tor_free(ns->source_address); tor_free(ns->contact); if (ns->signing_key) @@ -273,7 +322,25 @@ networkstatus_v2_free(networkstatus_v2_t *ns) tor_free(ns); } -/** Clear all storage held in <b>ns</b>. */ +/** Free all storage held in <b>sig</b> */ +void +document_signature_free(document_signature_t *sig) +{ + tor_free(sig->signature); + tor_free(sig); +} + +/** Return a newly allocated copy of <b>sig</b> */ +document_signature_t * +document_signature_dup(const document_signature_t *sig) +{ + document_signature_t *r = tor_memdup(sig, sizeof(document_signature_t)); + if (r->signature) + r->signature = tor_memdup(sig->signature, sig->signature_len); + return r; +} + +/** Free all storage held in <b>ns</b>. */ void networkstatus_vote_free(networkstatus_t *ns) { @@ -286,6 +353,10 @@ networkstatus_vote_free(networkstatus_t *ns) SMARTLIST_FOREACH(ns->known_flags, char *, c, tor_free(c)); smartlist_free(ns->known_flags); } + if (ns->weight_params) { + SMARTLIST_FOREACH(ns->weight_params, char *, c, tor_free(c)); + smartlist_free(ns->weight_params); + } if (ns->net_params) { SMARTLIST_FOREACH(ns->net_params, char *, c, tor_free(c)); smartlist_free(ns->net_params); @@ -295,18 +366,20 @@ networkstatus_vote_free(networkstatus_t *ns) smartlist_free(ns->supported_methods); } if (ns->voters) { - SMARTLIST_FOREACH(ns->voters, networkstatus_voter_info_t *, voter, - { + SMARTLIST_FOREACH_BEGIN(ns->voters, networkstatus_voter_info_t *, voter) { tor_free(voter->nickname); tor_free(voter->address); tor_free(voter->contact); - tor_free(voter->signature); + if (voter->sigs) { + SMARTLIST_FOREACH(voter->sigs, document_signature_t *, sig, + document_signature_free(sig)); + smartlist_free(voter->sigs); + } tor_free(voter); - }); + } SMARTLIST_FOREACH_END(voter); smartlist_free(ns->voters); } - if (ns->cert) - authority_cert_free(ns->cert); + authority_cert_free(ns->cert); if (ns->routerstatus_list) { if (ns->type == NS_TYPE_VOTE || ns->type == NS_TYPE_OPINION) { @@ -319,8 +392,8 @@ networkstatus_vote_free(networkstatus_t *ns) smartlist_free(ns->routerstatus_list); } - if (ns->desc_digest_map) - digestmap_free(ns->desc_digest_map, NULL); + + digestmap_free(ns->desc_digest_map, NULL); memset(ns, 11, sizeof(*ns)); tor_free(ns); @@ -341,34 +414,38 @@ networkstatus_get_voter_by_id(networkstatus_t *vote, return NULL; } -/** Check whether the signature on <b>voter</b> is correctly signed by - * the signing key of <b>cert</b>. Return -1 if <b>cert</b> doesn't match the +/** Check whether the signature <b>sig</b> is correctly signed with the + * signing key in <b>cert</b>. Return -1 if <b>cert</b> doesn't match the * signing key; otherwise set the good_signature or bad_signature flag on * <b>voter</b>, and return 0. */ -/* (private; exposed for testing.) */ int -networkstatus_check_voter_signature(networkstatus_t *consensus, - networkstatus_voter_info_t *voter, - authority_cert_t *cert) +networkstatus_check_document_signature(const networkstatus_t *consensus, + document_signature_t *sig, + const authority_cert_t *cert) { - char d[DIGEST_LEN]; + char key_digest[DIGEST_LEN]; + const int dlen = sig->alg == DIGEST_SHA1 ? DIGEST_LEN : DIGEST256_LEN; char *signed_digest; size_t signed_digest_len; - if (crypto_pk_get_digest(cert->signing_key, d)<0) + + if (crypto_pk_get_digest(cert->signing_key, key_digest)<0) return -1; - if (memcmp(voter->signing_key_digest, d, DIGEST_LEN)) + if (memcmp(sig->signing_key_digest, key_digest, DIGEST_LEN) || + memcmp(sig->identity_digest, cert->cache_info.identity_digest, + DIGEST_LEN)) return -1; + signed_digest_len = crypto_pk_keysize(cert->signing_key); signed_digest = tor_malloc(signed_digest_len); if (crypto_pk_public_checksig(cert->signing_key, signed_digest, - voter->signature, - voter->signature_len) != DIGEST_LEN || - memcmp(signed_digest, consensus->networkstatus_digest, DIGEST_LEN)) { + sig->signature, + sig->signature_len) < dlen || + memcmp(signed_digest, consensus->digests.d[sig->alg], dlen)) { log_warn(LD_DIR, "Got a bad signature on a networkstatus vote"); - voter->bad_signature = 1; + sig->bad_signature = 1; } else { - voter->good_signature = 1; + sig->good_signature = 1; } tor_free(signed_digest); return 0; @@ -401,37 +478,52 @@ networkstatus_check_consensus_signature(networkstatus_t *consensus, tor_assert(consensus->type == NS_TYPE_CONSENSUS); - SMARTLIST_FOREACH(consensus->voters, networkstatus_voter_info_t *, voter, - { - if (!voter->good_signature && !voter->bad_signature && voter->signature) { - /* we can try to check the signature. */ - int is_v3_auth = trusteddirserver_get_by_v3_auth_digest( - voter->identity_digest) != NULL; - authority_cert_t *cert = - authority_cert_get_by_digests(voter->identity_digest, - voter->signing_key_digest); - if (!is_v3_auth) { - smartlist_add(unrecognized, voter); - ++n_unknown; - continue; - } else if (!cert || cert->expires < now) { - smartlist_add(need_certs_from, voter); - ++n_missing_key; - continue; - } - if (networkstatus_check_voter_signature(consensus, voter, cert) < 0) { - smartlist_add(need_certs_from, voter); - ++n_missing_key; - continue; + SMARTLIST_FOREACH_BEGIN(consensus->voters, networkstatus_voter_info_t *, + voter) { + int good_here = 0; + int bad_here = 0; + int missing_key_here = 0; + SMARTLIST_FOREACH_BEGIN(voter->sigs, document_signature_t *, sig) { + if (!sig->good_signature && !sig->bad_signature && + sig->signature) { + /* we can try to check the signature. */ + int is_v3_auth = trusteddirserver_get_by_v3_auth_digest( + sig->identity_digest) != NULL; + authority_cert_t *cert = + authority_cert_get_by_digests(sig->identity_digest, + sig->signing_key_digest); + tor_assert(!memcmp(sig->identity_digest, voter->identity_digest, + DIGEST_LEN)); + + if (!is_v3_auth) { + smartlist_add(unrecognized, voter); + ++n_unknown; + continue; + } else if (!cert || cert->expires < now) { + smartlist_add(need_certs_from, voter); + ++missing_key_here; + continue; + } + if (networkstatus_check_document_signature(consensus, sig, cert) < 0) { + smartlist_add(need_certs_from, voter); + ++missing_key_here; + continue; + } } - } - if (voter->good_signature) + if (sig->good_signature) + ++good_here; + else if (sig->bad_signature) + ++bad_here; + } SMARTLIST_FOREACH_END(sig); + if (good_here) ++n_good; - else if (voter->bad_signature) + else if (bad_here) ++n_bad; + else if (missing_key_here) + ++n_missing_key; else ++n_no_signature; - }); + } SMARTLIST_FOREACH_END(voter); /* Now see whether we're missing any voters entirely. */ SMARTLIST_FOREACH(router_get_trusted_dir_servers(), @@ -784,8 +876,8 @@ networkstatus_v2_list_clean(time_t now) /** Helper for bsearching a list of routerstatus_t pointers: compare a * digest in the key to the identity digest of a routerstatus_t. */ -static int -_compare_digest_to_routerstatus_entry(const void *_key, const void **_member) +int +compare_digest_to_routerstatus_entry(const void *_key, const void **_member) { const char *key = _key; const routerstatus_t *rs = *_member; @@ -798,7 +890,7 @@ routerstatus_t * networkstatus_v2_find_entry(networkstatus_v2_t *ns, const char *digest) { return smartlist_bsearch(ns->entries, digest, - _compare_digest_to_routerstatus_entry); + compare_digest_to_routerstatus_entry); } /** Return the entry in <b>ns</b> for the identity digest <b>digest</b>, or @@ -807,7 +899,7 @@ routerstatus_t * networkstatus_vote_find_entry(networkstatus_t *ns, const char *digest) { return smartlist_bsearch(ns->routerstatus_list, digest, - _compare_digest_to_routerstatus_entry); + compare_digest_to_routerstatus_entry); } /*XXXX make this static once functions are moved into this file. */ @@ -819,7 +911,7 @@ networkstatus_vote_find_entry_idx(networkstatus_t *ns, const char *digest, int *found_out) { return smartlist_bsearch_idx(ns->routerstatus_list, digest, - _compare_digest_to_routerstatus_entry, + compare_digest_to_routerstatus_entry, found_out); } @@ -872,7 +964,7 @@ router_get_consensus_status_by_id(const char *digest) if (!current_consensus) return NULL; return smartlist_bsearch(current_consensus->routerstatus_list, digest, - _compare_digest_to_routerstatus_entry); + compare_digest_to_routerstatus_entry); } /** Given a nickname (possibly verbose, possibly a hexadecimal digest), return @@ -1077,27 +1169,32 @@ static void update_consensus_networkstatus_downloads(time_t now) { or_options_t *options = get_options(); + int i; if (!networkstatus_get_live_consensus(now)) time_to_download_next_consensus = now; /* No live consensus? Get one now!*/ if (time_to_download_next_consensus > now) return; /* Wait until the current consensus is older. */ if (authdir_mode_v3(options)) return; /* Authorities never fetch a consensus */ - if (!download_status_is_ready(&consensus_dl_status, now, + /* XXXXNM Microdescs: may need to download more types. */ + if (!download_status_is_ready(&consensus_dl_status[FLAV_NS], now, CONSENSUS_NETWORKSTATUS_MAX_DL_TRIES)) return; /* We failed downloading a consensus too recently. */ if (connection_get_by_type_purpose(CONN_TYPE_DIR, DIR_PURPOSE_FETCH_CONSENSUS)) return; /* There's an in-progress download.*/ - if (consensus_waiting_for_certs) { - /* XXXX make sure this doesn't delay sane downloads. */ - if (consensus_waiting_for_certs_set_at + DELAY_WHILE_FETCHING_CERTS > now) - return; /* We're still getting certs for this one. */ - else { - if (!consensus_waiting_for_certs_dl_failed) { - download_status_failed(&consensus_dl_status, 0); - consensus_waiting_for_certs_dl_failed=1; + for (i=0; i < N_CONSENSUS_FLAVORS; ++i) { + consensus_waiting_for_certs_t *waiting = &consensus_waiting_for_certs[i]; + if (waiting->consensus) { + /* XXXX make sure this doesn't delay sane downloads. */ + if (waiting->set_at + DELAY_WHILE_FETCHING_CERTS > now) + return; /* We're still getting certs for this one. */ + else { + if (!waiting->dl_failed) { + download_status_failed(&consensus_dl_status[FLAV_NS], 0); + waiting->dl_failed=1; + } } } } @@ -1113,7 +1210,8 @@ update_consensus_networkstatus_downloads(time_t now) void networkstatus_consensus_download_failed(int status_code) { - download_status_failed(&consensus_dl_status, status_code); + /* XXXXNM Microdescs: may need to handle more types. */ + download_status_failed(&consensus_dl_status[FLAV_NS], status_code); /* Retry immediately, if appropriate. */ update_consensus_networkstatus_downloads(time(NULL)); } @@ -1137,8 +1235,13 @@ update_consensus_networkstatus_fetch_time(time_t now) /* We want to cache the next one at some point after this one * is no longer fresh... */ start = c->fresh_until + CONSENSUS_MIN_SECONDS_BEFORE_CACHING; - /* But only in the first half-interval after that. */ - dl_interval = interval/2; + /* Some clients may need the consensus sooner than others. */ + if (options->FetchDirInfoExtraEarly) { + dl_interval = 60; + } else { + /* But only in the first half-interval after that. */ + dl_interval = interval/2; + } } else { /* We're an ordinary client or a bridge. Give all the caches enough * time to download the consensus. */ @@ -1158,7 +1261,7 @@ update_consensus_networkstatus_fetch_time(time_t now) } if (dl_interval < 1) dl_interval = 1; - /* We must not try to replace c while it's still the most valid: */ + /* We must not try to replace c while it's still fresh: */ tor_assert(c->fresh_until < start); /* We must download the next one before c is invalid: */ tor_assert(start+dl_interval < c->valid_until); @@ -1179,7 +1282,6 @@ update_consensus_networkstatus_fetch_time(time_t now) time_to_download_next_consensus = now; log_info(LD_DIR, "No live consensus; we should fetch one immediately."); } - } /** Return 1 if there's a reason we shouldn't try any directory @@ -1214,10 +1316,14 @@ update_networkstatus_downloads(time_t now) void update_certificate_downloads(time_t now) { - if (consensus_waiting_for_certs) - authority_certs_fetch_missing(consensus_waiting_for_certs, now); - else - authority_certs_fetch_missing(current_consensus, now); + int i; + for (i = 0; i < N_CONSENSUS_FLAVORS; ++i) { + if (consensus_waiting_for_certs[i].consensus) + authority_certs_fetch_missing(consensus_waiting_for_certs[i].consensus, + now); + } + + authority_certs_fetch_missing(current_consensus, now); } /** Return 1 if we have a consensus but we don't have enough certificates @@ -1225,7 +1331,8 @@ update_certificate_downloads(time_t now) int consensus_is_waiting_for_certs(void) { - return consensus_waiting_for_certs ? 1 : 0; + return consensus_waiting_for_certs[USABLE_CONSENSUS_FLAVOR].consensus + ? 1 : 0; } /** Return the network status with a given identity digest. */ @@ -1394,16 +1501,30 @@ networkstatus_copy_old_consensus_info(networkstatus_t *new_c, * user, and -2 for more serious problems. */ int -networkstatus_set_current_consensus(const char *consensus, unsigned flags) +networkstatus_set_current_consensus(const char *consensus, + const char *flavor, + unsigned flags) { - networkstatus_t *c; + networkstatus_t *c=NULL; int r, result = -1; time_t now = time(NULL); + or_options_t *options = get_options(); char *unverified_fname = NULL, *consensus_fname = NULL; + int flav = networkstatus_parse_flavor_name(flavor); const unsigned from_cache = flags & NSSET_FROM_CACHE; const unsigned was_waiting_for_certs = flags & NSSET_WAS_WAITING_FOR_CERTS; const unsigned dl_certs = !(flags & NSSET_DONT_DOWNLOAD_CERTS); const unsigned accept_obsolete = flags & NSSET_ACCEPT_OBSOLETE; + const unsigned require_flavor = flags & NSSET_REQUIRE_FLAVOR; + const digests_t *current_digests = NULL; + consensus_waiting_for_certs_t *waiting = NULL; + time_t current_valid_after = 0; + + if (flav < 0) { + /* XXXX we don't handle unrecognized flavors yet. */ + log_warn(LD_BUG, "Unrecognized consensus flavor %s", flavor); + return -2; + } /* Make sure it's parseable. */ c = networkstatus_parse_vote_from_string(consensus, NULL, NS_TYPE_CONSENSUS); @@ -1413,33 +1534,69 @@ networkstatus_set_current_consensus(const char *consensus, unsigned flags) goto done; } + if ((int)c->flavor != flav) { + /* This wasn't the flavor we thought we were getting. */ + if (require_flavor) { + log_warn(LD_DIR, "Got consensus with unexpected flavor %s (wanted %s)", + networkstatus_get_flavor_name(c->flavor), flavor); + goto done; + } + flav = c->flavor; + flavor = networkstatus_get_flavor_name(flav); + } + + if (flav != USABLE_CONSENSUS_FLAVOR && + !directory_caches_dir_info(options)) { + /* This consensus is totally boring to us: we won't use it, and we won't + * serve it. Drop it. */ + goto done; + } + if (from_cache && !accept_obsolete && c->valid_until < now-OLD_ROUTER_DESC_MAX_AGE) { /* XXX022 when we try to make fallbackconsensus work again, we should * consider taking this out. Until then, believing obsolete consensuses * is causing more harm than good. See also bug 887. */ - log_info(LD_DIR, "Loaded an obsolete consensus. Discarding."); + log_info(LD_DIR, "Loaded an expired consensus. Discarding."); goto done; } - if (current_consensus && - !memcmp(c->networkstatus_digest, current_consensus->networkstatus_digest, - DIGEST_LEN)) { + if (!strcmp(flavor, "ns")) { + consensus_fname = get_datadir_fname("cached-consensus"); + unverified_fname = get_datadir_fname("unverified-consensus"); + if (current_consensus) { + current_digests = ¤t_consensus->digests; + current_valid_after = current_consensus->valid_after; + } + } else { + cached_dir_t *cur; + char buf[128]; + tor_snprintf(buf, sizeof(buf), "cached-%s-consensus", flavor); + consensus_fname = get_datadir_fname(buf); + tor_snprintf(buf, sizeof(buf), "unverified-%s-consensus", flavor); + unverified_fname = get_datadir_fname(buf); + cur = dirserv_get_consensus(flavor); + if (cur) { + current_digests = &cur->digests; + current_valid_after = cur->published; + } + } + + if (current_digests && + !memcmp(&c->digests, current_digests, sizeof(c->digests))) { /* We already have this one. That's a failure. */ - log_info(LD_DIR, "Got a consensus we already have"); + log_info(LD_DIR, "Got a %s consensus we already have", flavor); goto done; } - if (current_consensus && c->valid_after <= current_consensus->valid_after) { + if (current_valid_after && c->valid_after <= current_valid_after) { /* We have a newer one. There's no point in accepting this one, * even if it's great. */ - log_info(LD_DIR, "Got a consensus at least as old as the one we have"); + log_info(LD_DIR, "Got a %s consensus at least as old as the one we have", + flavor); goto done; } - consensus_fname = get_datadir_fname("cached-consensus"); - unverified_fname = get_datadir_fname("unverified-consensus"); - /* Make sure it's signed enough. */ if ((r=networkstatus_check_consensus_signature(c, 1))<0) { if (r == -1) { @@ -1448,16 +1605,16 @@ networkstatus_set_current_consensus(const char *consensus, unsigned flags) log_info(LD_DIR, "Not enough certificates to check networkstatus consensus"); } - if (!current_consensus || - c->valid_after > current_consensus->valid_after) { - if (consensus_waiting_for_certs) - networkstatus_vote_free(consensus_waiting_for_certs); - tor_free(consensus_waiting_for_certs_body); - consensus_waiting_for_certs = c; + if (!current_valid_after || + c->valid_after > current_valid_after) { + waiting = &consensus_waiting_for_certs[flav]; + networkstatus_vote_free(waiting->consensus); + tor_free(waiting->body); + waiting->consensus = c; c = NULL; /* Prevent free. */ - consensus_waiting_for_certs_body = tor_strdup(consensus); - consensus_waiting_for_certs_set_at = now; - consensus_waiting_for_certs_dl_failed = 0; + waiting->body = tor_strdup(consensus); + waiting->set_at = now; + waiting->dl_failed = 0; if (!from_cache) { write_str_to_file(unverified_fname, consensus, 0); } @@ -1486,56 +1643,71 @@ networkstatus_set_current_consensus(const char *consensus, unsigned flags) } } - if (!from_cache) + if (!from_cache && flav == USABLE_CONSENSUS_FLAVOR) control_event_client_status(LOG_NOTICE, "CONSENSUS_ARRIVED"); /* Are we missing any certificates at all? */ if (r != 1 && dl_certs) authority_certs_fetch_missing(c, now); - notify_control_networkstatus_changed(current_consensus, c); + if (flav == USABLE_CONSENSUS_FLAVOR) { + notify_control_networkstatus_changed(current_consensus, c); - if (current_consensus) { - networkstatus_copy_old_consensus_info(c, current_consensus); - networkstatus_vote_free(current_consensus); + if (current_consensus) { + networkstatus_copy_old_consensus_info(c, current_consensus); + networkstatus_vote_free(current_consensus); + } } - if (consensus_waiting_for_certs && - consensus_waiting_for_certs->valid_after <= c->valid_after) { - networkstatus_vote_free(consensus_waiting_for_certs); - consensus_waiting_for_certs = NULL; - if (consensus != consensus_waiting_for_certs_body) - tor_free(consensus_waiting_for_certs_body); + waiting = &consensus_waiting_for_certs[flav]; + if (waiting->consensus && + waiting->consensus->valid_after <= c->valid_after) { + networkstatus_vote_free(waiting->consensus); + waiting->consensus = NULL; + if (consensus != waiting->body) + tor_free(waiting->body); else - consensus_waiting_for_certs_body = NULL; - consensus_waiting_for_certs_set_at = 0; - consensus_waiting_for_certs_dl_failed = 0; + waiting->body = NULL; + waiting->set_at = 0; + waiting->dl_failed = 0; unlink(unverified_fname); } /* Reset the failure count only if this consensus is actually valid. */ if (c->valid_after <= now && now <= c->valid_until) { - download_status_reset(&consensus_dl_status); + download_status_reset(&consensus_dl_status[flav]); } else { if (!from_cache) - download_status_failed(&consensus_dl_status, 0); + download_status_failed(&consensus_dl_status[flav], 0); } - current_consensus = c; - c = NULL; /* Prevent free. */ + if (directory_caches_dir_info(options)) { + dirserv_set_cached_consensus_networkstatus(consensus, + flavor, + &c->digests, + c->valid_after); + } + + if (flav == USABLE_CONSENSUS_FLAVOR) { + current_consensus = c; + c = NULL; /* Prevent free. */ + + /* XXXXNM Microdescs: needs a non-ns variant. */ + update_consensus_networkstatus_fetch_time(now); + dirvote_recalculate_timing(options, now); + routerstatus_list_update_named_server_map(); + cell_ewma_set_scale_factor(options, current_consensus); - update_consensus_networkstatus_fetch_time(now); - dirvote_recalculate_timing(get_options(), now); - routerstatus_list_update_named_server_map(); + /* XXX022 where is the right place to put this call? */ + connection_or_update_token_buckets(get_connection_array(), options); + + circuit_build_times_new_consensus_params(&circ_times, current_consensus); + } if (!from_cache) { write_str_to_file(consensus_fname, consensus, 0); } - if (directory_caches_dir_info(get_options())) - dirserv_set_cached_networkstatus_v3(consensus, - current_consensus->valid_after); - if (ftime_definitely_before(now, current_consensus->valid_after)) { char tbuf[ISO_TIME_LEN+1]; char dbuf[64]; @@ -1554,8 +1726,7 @@ networkstatus_set_current_consensus(const char *consensus, unsigned flags) result = 0; done: - if (c) - networkstatus_vote_free(c); + networkstatus_vote_free(c); tor_free(consensus_fname); tor_free(unverified_fname); return result; @@ -1566,13 +1737,17 @@ networkstatus_set_current_consensus(const char *consensus, unsigned flags) void networkstatus_note_certs_arrived(void) { - if (consensus_waiting_for_certs) { - if (networkstatus_check_consensus_signature( - consensus_waiting_for_certs, 0)>=0) { + int i; + for (i=0; i<N_CONSENSUS_FLAVORS; ++i) { + consensus_waiting_for_certs_t *waiting = &consensus_waiting_for_certs[i]; + if (!waiting->consensus) + continue; + if (networkstatus_check_consensus_signature(waiting->consensus, 0)>=0) { if (!networkstatus_set_current_consensus( - consensus_waiting_for_certs_body, + waiting->body, + networkstatus_get_flavor_name(i), NSSET_WAS_WAITING_FOR_CERTS)) { - tor_free(consensus_waiting_for_certs_body); + tor_free(waiting->body); } } } @@ -1658,10 +1833,8 @@ download_status_map_update_from_v2_networkstatus(void) v2_download_status_map = digestmap_new(); dl_status = digestmap_new(); - SMARTLIST_FOREACH(networkstatus_v2_list, networkstatus_v2_t *, ns, - { - SMARTLIST_FOREACH(ns->entries, routerstatus_t *, rs, - { + SMARTLIST_FOREACH_BEGIN(networkstatus_v2_list, networkstatus_v2_t *, ns) { + SMARTLIST_FOREACH_BEGIN(ns->entries, routerstatus_t *, rs) { const char *d = rs->descriptor_digest; download_status_t *s; if (digestmap_get(dl_status, d)) @@ -1670,8 +1843,8 @@ download_status_map_update_from_v2_networkstatus(void) s = tor_malloc_zero(sizeof(download_status_t)); } digestmap_set(dl_status, d, s); - }); - }); + } SMARTLIST_FOREACH_END(rs); + } SMARTLIST_FOREACH_END(ns); digestmap_free(v2_download_status_map, _tor_free); v2_download_status_map = dl_status; networkstatus_v2_list_has_changed = 0; @@ -1685,11 +1858,9 @@ routerstatus_list_update_named_server_map(void) if (!current_consensus) return; - if (named_server_map) - strmap_free(named_server_map, _tor_free); + strmap_free(named_server_map, _tor_free); named_server_map = strmap_new(); - if (unnamed_server_map) - strmap_free(unnamed_server_map, NULL); + strmap_free(unnamed_server_map, NULL); unnamed_server_map = strmap_new(); SMARTLIST_FOREACH(current_consensus->routerstatus_list, routerstatus_t *, rs, { @@ -1826,7 +1997,7 @@ char * networkstatus_getinfo_helper_single(routerstatus_t *rs) { char buf[RS_ENTRY_LEN+1]; - routerstatus_format_entry(buf, sizeof(buf), rs, NULL, 0, 1); + routerstatus_format_entry(buf, sizeof(buf), rs, NULL, NS_CONTROL_PORT); return tor_strdup(buf); } @@ -1863,7 +2034,7 @@ networkstatus_getinfo_by_purpose(const char *purpose_string, time_t now) if (bridge_auth && ri->purpose == ROUTER_PURPOSE_BRIDGE) dirserv_set_router_is_running(ri, now); /* then generate and write out status lines for each of them */ - set_routerstatus_from_routerinfo(&rs, ri, now, 0, 0, 0, 0); + set_routerstatus_from_routerinfo(&rs, ri, now, 0, 0, 0); smartlist_add(statuses, networkstatus_getinfo_helper_single(&rs)); }); @@ -1888,6 +2059,25 @@ networkstatus_dump_bridge_status_to_file(time_t now) tor_free(status); } +int32_t +get_net_param_from_list(smartlist_t *net_params, const char *param_name, + int default_val) +{ + size_t name_len = strlen(param_name); + + SMARTLIST_FOREACH_BEGIN(net_params, const char *, p) { + if (!strcmpstart(p, param_name) && p[name_len] == '=') { + int ok=0; + long v = tor_parse_long(p+name_len+1, 10, INT32_MIN, + INT32_MAX, &ok, NULL); + if (ok) + return (int32_t) v; + } + } SMARTLIST_FOREACH_END(p); + + return default_val; +} + /** Return the value of a integer parameter from the networkstatus <b>ns</b> * whose name is <b>param_name</b>. If <b>ns</b> is NULL, try loading the * latest consensus ourselves. Return <b>default_val</b> if no latest @@ -1896,27 +2086,59 @@ int32_t networkstatus_get_param(networkstatus_t *ns, const char *param_name, int32_t default_val) { - size_t name_len; - if (!ns) /* if they pass in null, go find it ourselves */ ns = networkstatus_get_latest_consensus(); if (!ns || !ns->net_params) return default_val; - name_len = strlen(param_name); + return get_net_param_from_list(ns->net_params, param_name, default_val); +} - SMARTLIST_FOREACH_BEGIN(ns->net_params, const char *, p) { - if (!strcmpstart(p, param_name) && p[name_len] == '=') { - int ok=0; - long v = tor_parse_long(p+name_len+1, 10, INT32_MIN, INT32_MAX, &ok, - NULL); - if (ok) - return (int32_t) v; - } - } SMARTLIST_FOREACH_END(p); +/** Return the value of a integer bw weight parameter from the networkstatus + * <b>ns</b> whose name is <b>weight_name</b>. If <b>ns</b> is NULL, try + * loading the latest consensus ourselves. Return <b>default_val</b> if no + * latest consensus, or if it has no parameter called <b>param_name</b>. */ +int32_t +networkstatus_get_bw_weight(networkstatus_t *ns, const char *weight_name, + int32_t default_val) +{ + if (!ns) /* if they pass in null, go find it ourselves */ + ns = networkstatus_get_latest_consensus(); - return default_val; + if (!ns || !ns->weight_params) + return default_val; + + return get_net_param_from_list(ns->weight_params, weight_name, default_val); +} + +/** Return the name of the consensus flavor <b>flav</b> as used to identify + * the flavor in directory documents. */ +const char * +networkstatus_get_flavor_name(consensus_flavor_t flav) +{ + switch (flav) { + case FLAV_NS: + return "ns"; + case FLAV_MICRODESC: + return "microdesc"; + default: + tor_fragile_assert(); + return "??"; + } +} + +/** Return the consensus_flavor_t value for the flavor called <b>flavname</b>, + * or -1 if the flavor is not recognized. */ +int +networkstatus_parse_flavor_name(const char *flavname) +{ + if (!strcmp(flavname, "ns")) + return FLAV_NS; + else if (!strcmp(flavname, "microdesc")) + return FLAV_MICRODESC; + else + return -1; } /** If <b>question</b> is a string beginning with "ns/" in a format the @@ -1925,7 +2147,8 @@ networkstatus_get_param(networkstatus_t *ns, const char *param_name, * ORs. Return 0 on success, -1 on unrecognized question format. */ int getinfo_helper_networkstatus(control_connection_t *conn, - const char *question, char **answer) + const char *question, char **answer, + const char **errmsg) { routerstatus_t *status; (void) conn; @@ -1949,8 +2172,10 @@ getinfo_helper_networkstatus(control_connection_t *conn, } else if (!strcmpstart(question, "ns/id/")) { char d[DIGEST_LEN]; - if (base16_decode(d, DIGEST_LEN, question+6, strlen(question+6))) + if (base16_decode(d, DIGEST_LEN, question+6, strlen(question+6))) { + *errmsg = "Data not decodeable as hex"; return -1; + } status = router_get_consensus_status_by_id(d); } else if (!strcmpstart(question, "ns/name/")) { status = router_get_consensus_status_by_nickname(question+8, 0); @@ -1958,7 +2183,7 @@ getinfo_helper_networkstatus(control_connection_t *conn, *answer = networkstatus_getinfo_by_purpose(question+11, time(NULL)); return *answer ? 0 : -1; } else { - return -1; + return 0; } if (status) @@ -1970,30 +2195,29 @@ getinfo_helper_networkstatus(control_connection_t *conn, void networkstatus_free_all(void) { + int i; if (networkstatus_v2_list) { SMARTLIST_FOREACH(networkstatus_v2_list, networkstatus_v2_t *, ns, networkstatus_v2_free(ns)); smartlist_free(networkstatus_v2_list); networkstatus_v2_list = NULL; } - if (v2_download_status_map) { - digestmap_free(v2_download_status_map, _tor_free); - v2_download_status_map = NULL; - } - if (current_consensus) { - networkstatus_vote_free(current_consensus); - current_consensus = NULL; - } - if (consensus_waiting_for_certs) { - networkstatus_vote_free(consensus_waiting_for_certs); - consensus_waiting_for_certs = NULL; - } - tor_free(consensus_waiting_for_certs_body); - if (named_server_map) { - strmap_free(named_server_map, _tor_free); - } - if (unnamed_server_map) { - strmap_free(unnamed_server_map, NULL); + + digestmap_free(v2_download_status_map, _tor_free); + v2_download_status_map = NULL; + networkstatus_vote_free(current_consensus); + current_consensus = NULL; + + for (i=0; i < N_CONSENSUS_FLAVORS; ++i) { + consensus_waiting_for_certs_t *waiting = &consensus_waiting_for_certs[i]; + if (waiting->consensus) { + networkstatus_vote_free(waiting->consensus); + waiting->consensus = NULL; + } + tor_free(waiting->body); } + + strmap_free(named_server_map, _tor_free); + strmap_free(unnamed_server_map, NULL); } diff --git a/src/or/networkstatus.h b/src/or/networkstatus.h new file mode 100644 index 0000000000..4059dead51 --- /dev/null +++ b/src/or/networkstatus.h @@ -0,0 +1,101 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2010, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file networkstatus.h + * \brief Header file for networkstatus.c. + **/ + +#ifndef _TOR_NETWORKSTATUS_H +#define _TOR_NETWORKSTATUS_H + +/** How old do we allow a v2 network-status to get before removing it + * completely? */ +#define MAX_NETWORKSTATUS_AGE (10*24*60*60) + +void networkstatus_reset_warnings(void); +void networkstatus_reset_download_failures(void); +int router_reload_v2_networkstatus(void); +int router_reload_consensus_networkstatus(void); +void routerstatus_free(routerstatus_t *rs); +void networkstatus_v2_free(networkstatus_v2_t *ns); +void networkstatus_vote_free(networkstatus_t *ns); +networkstatus_voter_info_t *networkstatus_get_voter_by_id( + networkstatus_t *vote, + const char *identity); +int networkstatus_check_consensus_signature(networkstatus_t *consensus, + int warn); +int networkstatus_check_document_signature(const networkstatus_t *consensus, + document_signature_t *sig, + const authority_cert_t *cert); +char *networkstatus_get_cache_filename(const char *identity_digest); +int router_set_networkstatus_v2(const char *s, time_t arrived_at, + v2_networkstatus_source_t source, + smartlist_t *requested_fingerprints); +void networkstatus_v2_list_clean(time_t now); +int compare_digest_to_routerstatus_entry(const void *_key, + const void **_member); +routerstatus_t *networkstatus_v2_find_entry(networkstatus_v2_t *ns, + const char *digest); +routerstatus_t *networkstatus_vote_find_entry(networkstatus_t *ns, + const char *digest); +int networkstatus_vote_find_entry_idx(networkstatus_t *ns, + const char *digest, int *found_out); +const smartlist_t *networkstatus_get_v2_list(void); +download_status_t *router_get_dl_status_by_descriptor_digest(const char *d); +routerstatus_t *router_get_consensus_status_by_id(const char *digest); +routerstatus_t *router_get_consensus_status_by_descriptor_digest( + const char *digest); +routerstatus_t *router_get_consensus_status_by_nickname(const char *nickname, + int warn_if_unnamed); +const char *networkstatus_get_router_digest_by_nickname(const char *nickname); +int networkstatus_nickname_is_unnamed(const char *nickname); +void networkstatus_consensus_download_failed(int status_code); +void update_consensus_networkstatus_fetch_time(time_t now); +int should_delay_dir_fetches(or_options_t *options); +void update_networkstatus_downloads(time_t now); +void update_certificate_downloads(time_t now); +int consensus_is_waiting_for_certs(void); +networkstatus_v2_t *networkstatus_v2_get_by_digest(const char *digest); +networkstatus_t *networkstatus_get_latest_consensus(void); +networkstatus_t *networkstatus_get_live_consensus(time_t now); +networkstatus_t *networkstatus_get_reasonably_live_consensus(time_t now); +#define NSSET_FROM_CACHE 1 +#define NSSET_WAS_WAITING_FOR_CERTS 2 +#define NSSET_DONT_DOWNLOAD_CERTS 4 +#define NSSET_ACCEPT_OBSOLETE 8 +#define NSSET_REQUIRE_FLAVOR 16 +int networkstatus_set_current_consensus(const char *consensus, + const char *flavor, + unsigned flags); +void networkstatus_note_certs_arrived(void); +void routers_update_all_from_networkstatus(time_t now, int dir_version); +void routerstatus_list_update_from_consensus_networkstatus(time_t now); +void routers_update_status_from_consensus_networkstatus(smartlist_t *routers, + int reset_failures); +void signed_descs_update_status_from_consensus_networkstatus( + smartlist_t *descs); + +char *networkstatus_getinfo_helper_single(routerstatus_t *rs); +char *networkstatus_getinfo_by_purpose(const char *purpose_string, time_t now); +void networkstatus_dump_bridge_status_to_file(time_t now); +int32_t get_net_param_from_list(smartlist_t *net_params, const char *name, + int default_val); +int32_t networkstatus_get_param(networkstatus_t *ns, const char *param_name, + int32_t default_val); +int getinfo_helper_networkstatus(control_connection_t *conn, + const char *question, char **answer, + const char **errmsg); +int32_t networkstatus_get_bw_weight(networkstatus_t *ns, const char *weight, + int32_t default_val); +const char *networkstatus_get_flavor_name(consensus_flavor_t flav); +int networkstatus_parse_flavor_name(const char *flavname); +void document_signature_free(document_signature_t *sig); +document_signature_t *document_signature_dup(const document_signature_t *sig); +void networkstatus_free_all(void); + +#endif + diff --git a/src/or/ntmain.c b/src/or/ntmain.c index ad6698e722..9bcb7047eb 100644 --- a/src/or/ntmain.c +++ b/src/or/ntmain.c @@ -5,6 +5,14 @@ #define MAIN_PRIVATE #include "or.h" +#include "config.h" +#include "main.h" + +#ifdef HAVE_EVENT2_EVENT_H +#include <event2/event.h> +#else +#include <event.h> +#endif #include <tchar.h> #define GENSRV_SERVICENAME TEXT("tor") @@ -218,7 +226,7 @@ nt_service_control(DWORD request) log_notice(LD_GENERAL, "Got stop/shutdown request; shutting down cleanly."); service_status.dwCurrentState = SERVICE_STOP_PENDING; - event_loopexit(&exit_now); + event_base_loopexit(tor_libevent_get_base(), &exit_now); return; } service_fns.SetServiceStatus_fn(hStatus, &service_status); diff --git a/src/or/ntmain.h b/src/or/ntmain.h new file mode 100644 index 0000000000..2cfa653c3d --- /dev/null +++ b/src/or/ntmain.h @@ -0,0 +1,30 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2010, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file ntmain.h + * \brief Header file for ntmain.c. + **/ + +#ifndef _TOR_NTMAIN_H +#define _TOR_NTMAIN_H + +#ifdef MS_WINDOWS +#if !defined (WINCE) +#define NT_SERVICE +#endif +#endif + +#ifdef NT_SERVICE +int nt_service_parse_options(int argc, char **argv, int *should_exit); +int nt_service_is_stopping(void); +void nt_service_set_state(DWORD state); +#else +#define nt_service_is_stopping() 0 +#endif + +#endif + diff --git a/src/or/onion.c b/src/or/onion.c index 8ee59f2847..ebc3583648 100644 --- a/src/or/onion.c +++ b/src/or/onion.c @@ -11,6 +11,10 @@ **/ #include "or.h" +#include "circuitlist.h" +#include "config.h" +#include "onion.h" +#include "rephist.h" /** Type for a linked list of circuits that are waiting for a free CPU worker * to process a waiting onion handshake. */ @@ -58,11 +62,17 @@ onion_pending_add(or_circuit_t *circ, char *onionskin) tor_assert(!ol_tail->next); if (ol_length >= get_options()->MaxOnionsPending) { - log_warn(LD_GENERAL, - "Your computer is too slow to handle this many circuit " - "creation requests! Please consider using the " - "MaxAdvertisedBandwidth config option or choosing a more " - "restricted exit policy."); +#define WARN_TOO_MANY_CIRC_CREATIONS_INTERVAL (60) + static time_t last_warned = 0; + time_t now = time(NULL); + if (last_warned + WARN_TOO_MANY_CIRC_CREATIONS_INTERVAL < now) { + log_warn(LD_GENERAL, + "Your computer is too slow to handle this many circuit " + "creation requests! Please consider using the " + "MaxAdvertisedBandwidth config option or choosing a more " + "restricted exit policy."); + last_warned = now; + } tor_free(tmp); return -1; } @@ -253,8 +263,9 @@ onion_skin_server_handshake(const char *onion_skin, /*ONIONSKIN_CHALLENGE_LEN*/ key_material_len = DIGEST_LEN+key_out_len; key_material = tor_malloc(key_material_len); - len = crypto_dh_compute_secret(dh, challenge, DH_KEY_LEN, - key_material, key_material_len); + len = crypto_dh_compute_secret(LOG_PROTOCOL_WARN, dh, challenge, + DH_KEY_LEN, key_material, + key_material_len); if (len < 0) { log_info(LD_GENERAL, "crypto_dh_compute_secret failed."); goto err; @@ -304,8 +315,9 @@ onion_skin_client_handshake(crypto_dh_env_t *handshake_state, key_material_len = DIGEST_LEN + key_out_len; key_material = tor_malloc(key_material_len); - len = crypto_dh_compute_secret(handshake_state, handshake_reply, DH_KEY_LEN, - key_material, key_material_len); + len = crypto_dh_compute_secret(LOG_PROTOCOL_WARN, handshake_state, + handshake_reply, DH_KEY_LEN, key_material, + key_material_len); if (len < 0) goto err; diff --git a/src/or/onion.h b/src/or/onion.h new file mode 100644 index 0000000000..e84dbb47be --- /dev/null +++ b/src/or/onion.h @@ -0,0 +1,48 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2010, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file onion.h + * \brief Header file for onion.c. + **/ + +#ifndef _TOR_ONION_H +#define _TOR_ONION_H + +int onion_pending_add(or_circuit_t *circ, char *onionskin); +or_circuit_t *onion_next_task(char **onionskin_out); +void onion_pending_remove(or_circuit_t *circ); + +int onion_skin_create(crypto_pk_env_t *router_key, + crypto_dh_env_t **handshake_state_out, + char *onion_skin_out); + +int onion_skin_server_handshake(const char *onion_skin, + crypto_pk_env_t *private_key, + crypto_pk_env_t *prev_private_key, + char *handshake_reply_out, + char *key_out, + size_t key_out_len); + +int onion_skin_client_handshake(crypto_dh_env_t *handshake_state, + const char *handshake_reply, + char *key_out, + size_t key_out_len); + +int fast_server_handshake(const char *key_in, + char *handshake_reply_out, + char *key_out, + size_t key_out_len); + +int fast_client_handshake(const char *handshake_state, + const char *handshake_reply_out, + char *key_out, + size_t key_out_len); + +void clear_pending_onions(void); + +#endif + diff --git a/src/or/or.h b/src/or/or.h index 910e612a0d..4098cd284e 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -20,9 +20,6 @@ #ifndef INSTRUMENT_DOWNLOADS #define INSTRUMENT_DOWNLOADS 1 #endif -#ifndef ENABLE_GEOIP_STATS -#define ENABLE_GEOIP_STATS 1 -#endif #endif #ifdef MS_WINDOWS @@ -83,16 +80,13 @@ #define snprintf _snprintf #endif -#include "crypto.h" #include "tortls.h" -#include "log.h" -#include "compat.h" +#include "../common/torlog.h" #include "container.h" -#include "util.h" #include "torgzip.h" #include "address.h" - -#include <event.h> +#include "compat_libevent.h" +#include "ht.h" /* These signals are defined to help control_signal_act work. */ @@ -161,7 +155,7 @@ #define MAX_DNS_TTL (3*60*60) /** How small can a TTL be before we stop believing it? Provides rudimentary * pinning. */ -#define MIN_DNS_TTL (60) +#define MIN_DNS_TTL 60 /** How often do we rotate onion keys? */ #define MIN_ONION_KEY_LIFETIME (7*24*60*60) @@ -221,6 +215,21 @@ typedef enum { /* !!!! If _CONN_TYPE_MAX is ever over 15, we must grow the type field in * connection_t. */ +/* Proxy client types */ +#define PROXY_NONE 0 +#define PROXY_CONNECT 1 +#define PROXY_SOCKS4 2 +#define PROXY_SOCKS5 3 + +/* Proxy client handshake states */ +#define PROXY_HTTPS_WANT_CONNECT_OK 1 +#define PROXY_SOCKS4_WANT_CONNECT_OK 2 +#define PROXY_SOCKS5_WANT_AUTH_METHOD_NONE 3 +#define PROXY_SOCKS5_WANT_AUTH_METHOD_RFC1929 4 +#define PROXY_SOCKS5_WANT_AUTH_RFC1929_OK 5 +#define PROXY_SOCKS5_WANT_CONNECT_OK 6 +#define PROXY_CONNECTED 7 + /** True iff <b>x</b> is an edge connection. */ #define CONN_IS_EDGE(x) \ ((x)->type == CONN_TYPE_EXIT || (x)->type == CONN_TYPE_AP) @@ -241,26 +250,24 @@ typedef enum { #define _OR_CONN_STATE_MIN 1 /** State for a connection to an OR: waiting for connect() to finish. */ #define OR_CONN_STATE_CONNECTING 1 -/** State for a connection to an OR: waiting for proxy command to flush. */ -#define OR_CONN_STATE_PROXY_FLUSHING 2 -/** State for a connection to an OR: waiting for proxy response. */ -#define OR_CONN_STATE_PROXY_READING 3 +/** State for a connection to an OR: waiting for proxy handshake to complete */ +#define OR_CONN_STATE_PROXY_HANDSHAKING 2 /** State for a connection to an OR or client: SSL is handshaking, not done * yet. */ -#define OR_CONN_STATE_TLS_HANDSHAKING 4 +#define OR_CONN_STATE_TLS_HANDSHAKING 3 /** State for a connection to an OR: We're doing a second SSL handshake for * renegotiation purposes. */ -#define OR_CONN_STATE_TLS_CLIENT_RENEGOTIATING 5 +#define OR_CONN_STATE_TLS_CLIENT_RENEGOTIATING 4 /** State for a connection at an OR: We're waiting for the client to * renegotiate. */ -#define OR_CONN_STATE_TLS_SERVER_RENEGOTIATING 6 +#define OR_CONN_STATE_TLS_SERVER_RENEGOTIATING 5 /** State for a connection to an OR: We're done with our SSL handshake, but we * haven't yet negotiated link protocol versions and sent a netinfo cell. */ -#define OR_CONN_STATE_OR_HANDSHAKING 7 +#define OR_CONN_STATE_OR_HANDSHAKING 6 /** State for a connection to an OR: Ready to send/receive cells. */ -#define OR_CONN_STATE_OPEN 8 -#define _OR_CONN_STATE_MAX 8 +#define OR_CONN_STATE_OPEN 7 +#define _OR_CONN_STATE_MAX 7 #define _EXIT_CONN_STATE_MIN 1 /** State for an exit connection: waiting for response from DNS farm. */ @@ -457,23 +464,23 @@ typedef enum { #define CIRCUIT_PURPOSE_C_REND_READY_INTRO_ACKED 11 /** Client-side circuit purpose: at Alice, rendezvous established. */ #define CIRCUIT_PURPOSE_C_REND_JOINED 12 - -#define _CIRCUIT_PURPOSE_C_MAX 12 - +/** This circuit is used for build time measurement only */ +#define CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT 13 +#define _CIRCUIT_PURPOSE_C_MAX 13 /** Hidden-service-side circuit purpose: at Bob, waiting for introductions. */ -#define CIRCUIT_PURPOSE_S_ESTABLISH_INTRO 13 +#define CIRCUIT_PURPOSE_S_ESTABLISH_INTRO 14 /** Hidden-service-side circuit purpose: at Bob, successfully established * intro. */ -#define CIRCUIT_PURPOSE_S_INTRO 14 +#define CIRCUIT_PURPOSE_S_INTRO 15 /** Hidden-service-side circuit purpose: at Bob, connecting to rend point. */ -#define CIRCUIT_PURPOSE_S_CONNECT_REND 15 +#define CIRCUIT_PURPOSE_S_CONNECT_REND 16 /** Hidden-service-side circuit purpose: at Bob, rendezvous established. */ -#define CIRCUIT_PURPOSE_S_REND_JOINED 16 +#define CIRCUIT_PURPOSE_S_REND_JOINED 17 /** A testing circuit; not meant to be used for actual traffic. */ -#define CIRCUIT_PURPOSE_TESTING 17 +#define CIRCUIT_PURPOSE_TESTING 18 /** A controller made this circuit and Tor should not use it. */ -#define CIRCUIT_PURPOSE_CONTROLLER 18 -#define _CIRCUIT_PURPOSE_MAX 18 +#define CIRCUIT_PURPOSE_CONTROLLER 19 +#define _CIRCUIT_PURPOSE_MAX 19 /** A catch-all for unrecognized purposes. Currently we don't expect * to make or see any circuits with this purpose. */ #define CIRCUIT_PURPOSE_UNKNOWN 255 @@ -483,7 +490,7 @@ typedef enum { #define CIRCUIT_PURPOSE_IS_ORIGIN(p) ((p)>_CIRCUIT_PURPOSE_OR_MAX) /** True iff the circuit purpose <b>p</b> is for a circuit that originated * here to serve as a client. (Hidden services don't count here.) */ -#define CIRCUIT_PURPOSE_IS_CLIENT(p) \ +#define CIRCUIT_PURPOSE_IS_CLIENT(p) \ ((p)> _CIRCUIT_PURPOSE_OR_MAX && \ (p)<=_CIRCUIT_PURPOSE_C_MAX) /** True iff the circuit_t <b>c</b> is actually an origin_circuit_t. */ @@ -641,10 +648,6 @@ typedef enum { /** Length of a binary-encoded rendezvous service ID. */ #define REND_SERVICE_ID_LEN 10 -/** How long after we receive a hidden service descriptor do we consider - * it fresh? */ -#define NUM_SECONDS_BEFORE_HS_REFETCH (60*15) - /** Time period for which a v2 descriptor will be valid. */ #define REND_TIME_PERIOD_V2_DESC_VALIDITY (24*60*60) @@ -739,12 +742,6 @@ typedef struct rend_data_t { /** Rendezvous cookie used by both, client and service. */ char rend_cookie[REND_COOKIE_LEN]; - - /** Rendezvous descriptor version that is used by a service. Used to - * distinguish introduction and rendezvous points belonging to the same - * rendezvous service ID, but different descriptor versions. - */ - uint8_t rend_desc_version; } rend_data_t; /** Time interval for tracking possible replays of INTRODUCE2 cells. @@ -851,12 +848,28 @@ typedef struct packed_cell_t { char body[CELL_NETWORK_SIZE]; /**< Cell as packed for network. */ } packed_cell_t; +/** Number of cells added to a circuit queue including their insertion + * time on 10 millisecond detail; used for buffer statistics. */ +typedef struct insertion_time_elem_t { + struct insertion_time_elem_t *next; /**< Next element in queue. */ + uint32_t insertion_time; /**< When were cells inserted (in 10 ms steps + * starting at 0:00 of the current day)? */ + unsigned counter; /**< How many cells were inserted? */ +} insertion_time_elem_t; + +/** Queue of insertion times. */ +typedef struct insertion_time_queue_t { + struct insertion_time_elem_t *first; /**< First element in queue. */ + struct insertion_time_elem_t *last; /**< Last element in queue. */ +} insertion_time_queue_t; + /** A queue of cells on a circuit, waiting to be added to the * or_connection_t's outbuf. */ typedef struct cell_queue_t { packed_cell_t *head; /**< The first cell, or NULL if the queue is empty. */ packed_cell_t *tail; /**< The last cell, or NULL if the queue is empty. */ int n; /**< The number of cells in the queue. */ + insertion_time_queue_t *insertion_times; /**< Insertion times of cells. */ } cell_queue_t; /** Beginning of a RELAY cell payload. */ @@ -912,7 +925,7 @@ typedef struct connection_t { * again once the bandwidth throttler allows it? */ unsigned int write_blocked_on_bw:1; /**< Boolean: should we start writing * again once the bandwidth throttler allows - * reads? */ + * writes? */ unsigned int hold_open_until_flushed:1; /**< Despite this connection's being * marked for close, do we flush it * before closing it? */ @@ -937,8 +950,11 @@ typedef struct connection_t { * connection. */ unsigned int linked_conn_is_closed:1; - int s; /**< Our socket; -1 if this connection is closed, or has no - * socket. */ + /** CONNECT/SOCKS proxy client handshake state (for outgoing connections). */ + unsigned int proxy_state:4; + + /** Our socket; -1 if this connection is closed, or has no socket. */ + evutil_socket_t s; int conn_array_index; /**< Index into the global connection array. */ struct event *read_event; /**< Libevent event structure. */ struct event *write_event; /**< Libevent event structure. */ @@ -980,6 +996,8 @@ typedef struct connection_t { * to the evdns_server_port is uses to listen to and answer connections. */ struct evdns_server_port *dns_server_port; + /** Unique ID for measuring tunneled network status requests. */ + uint64_t dirreq_id; } connection_t; /** Stores flags and information related to the portion of a v2 Tor OR @@ -1041,12 +1059,13 @@ typedef struct or_connection_t { time_t timestamp_last_added_nonpadding; /** When did we last add a * non-padding cell to the outbuf? */ - /* bandwidth* and read_bucket only used by ORs in OPEN state: */ + /* bandwidth* and *_bucket only used by ORs in OPEN state: */ int bandwidthrate; /**< Bytes/s added to the bucket. (OPEN ORs only.) */ int bandwidthburst; /**< Max bucket size for this conn. (OPEN ORs only.) */ int read_bucket; /**< When this hits 0, stop receiving. Every second we * add 'bandwidthrate' to this, capping it at * bandwidthburst. (OPEN ORs only) */ + int write_bucket; /**< When this hits 0, stop writing. Like read_bucket. */ int n_circuits; /**< How many circuits use this connection as p_conn or * n_conn ? */ @@ -1054,6 +1073,17 @@ typedef struct or_connection_t { * free up on this connection's outbuf. Every time we pull cells from a * circuit, we advance this pointer to the next circuit in the ring. */ struct circuit_t *active_circuits; + /** Priority queue of cell_ewma_t for circuits with queued cells waiting for + * room to free up on this connection's outbuf. Kept in heap order + * according to EWMA. + * + * This is redundant with active_circuits; if we ever decide only to use the + * cell_ewma algorithm for choosing circuits, we can remove active_circuits. + */ + smartlist_t *active_circuit_pqueue; + /** The tick on which the cell_ewma_ts in active_circuit_pqueue last had + * their ewma values rescaled. */ + unsigned active_circuit_pqueue_last_recalibrated; struct or_connection_t *next_with_same_id; /**< Next connection with same * identity digest as this one. */ } or_connection_t; @@ -1150,7 +1180,8 @@ typedef struct dir_connection_t { enum { DIR_SPOOL_NONE=0, DIR_SPOOL_SERVER_BY_DIGEST, DIR_SPOOL_SERVER_BY_FP, DIR_SPOOL_EXTRA_BY_DIGEST, DIR_SPOOL_EXTRA_BY_FP, - DIR_SPOOL_CACHED_DIR, DIR_SPOOL_NETWORKSTATUS + DIR_SPOOL_CACHED_DIR, DIR_SPOOL_NETWORKSTATUS, + DIR_SPOOL_MICRODESC, /* NOTE: if we add another entry, add another bit. */ } dir_spool_src : 3; /** If we're fetching descriptors, what router purpose shall we assign * to them? */ @@ -1178,12 +1209,6 @@ typedef struct control_connection_t { uint32_t event_mask; /**< Bitfield: which events does this controller * care about? */ - unsigned int use_long_names:1; /**< True if we should use long nicknames - * on this (v1) connection. Only settable - * via v1 controllers. */ - /** For control connections only. If set, we send extended info with control - * events as appropriate. */ - unsigned int use_extended_events:1; /** True if we have sent a protocolinfo reply on this connection. */ unsigned int have_sent_protocolinfo:1; @@ -1267,6 +1292,7 @@ typedef struct cached_dir_t { size_t dir_len; /**< Length of <b>dir</b> (not counting its NUL). */ size_t dir_z_len; /**< Length of <b>dir_z</b>. */ time_t published; /**< When was this object published. */ + digests_t digests; /**< Digests of this object (networkstatus only) */ int refcnt; /**< Reference count for this cached_dir_t. */ } cached_dir_t; @@ -1516,6 +1542,9 @@ typedef struct routerstatus_t { unsigned int has_bandwidth:1; /**< The vote/consensus had bw info */ unsigned int has_exitsummary:1; /**< The vote/consensus had exit summaries */ + unsigned int has_measured_bw:1; /**< The vote/consensus had a measured bw */ + + uint32_t measured_bw; /**< Measured bandwidth (capacity) of the router */ uint32_t bandwidth; /**< Bandwidth (capacity) of the router as reported in * the vote/consensus, in kilobytes/sec. */ @@ -1540,6 +1569,52 @@ typedef struct routerstatus_t { } routerstatus_t; +/** A microdescriptor is the smallest amount of information needed to build a + * circuit through a router. They are generated by the directory authorities, + * using information from the uploaded routerinfo documents. They are not + * self-signed, but are rather authenticated by having their hash in a signed + * networkstatus document. */ +typedef struct microdesc_t { + /** Hashtable node, used to look up the microdesc by its digest. */ + HT_ENTRY(microdesc_t) node; + + /* Cache information */ + + /** When was this microdescriptor last listed in a consensus document? + * Once a microdesc has been unlisted long enough, we can drop it. + */ + time_t last_listed; + /** Where is this microdescriptor currently stored? */ + saved_location_t saved_location : 3; + /** If true, do not attempt to cache this microdescriptor on disk. */ + unsigned int no_save : 1; + /** If saved_location == SAVED_IN_CACHE, this field holds the offset of the + * microdescriptor in the cache. */ + off_t off; + + /* The string containing the microdesc. */ + + /** A pointer to the encoded body of the microdescriptor. If the + * saved_location is SAVED_IN_CACHE, then the body is a pointer into an + * mmap'd region. Otherwise, it is a malloc'd string. The string might not + * be NUL-terminated; take the length from <b>bodylen</b>. */ + char *body; + /** The length of the microdescriptor in <b>body</b>. */ + size_t bodylen; + /** A SHA256-digest of the microdescriptor. */ + char digest[DIGEST256_LEN]; + + /* Fields in the microdescriptor. */ + + /** As routerinfo_t.onion_pkey */ + crypto_pk_env_t *onion_pkey; + /** As routerinfo_t.family */ + smartlist_t *family; + /** Encoded exit policy summary */ + char *exitsummary; /**< exit policy summary - + * XXX this probably should not stay a string. */ +} microdesc_t; + /** How many times will we try to download a router's descriptor before giving * up? */ #define MAX_ROUTERDESC_DOWNLOAD_FAILURES 8 @@ -1582,6 +1657,11 @@ typedef struct networkstatus_v2_t { * sorted by identity_digest. */ } networkstatus_v2_t; +typedef struct vote_microdesc_hash_t { + struct vote_microdesc_hash_t *next; + char *microdesc_hash_line; +} vote_microdesc_hash_t; + /** The claim about a single router, made in a vote. */ typedef struct vote_routerstatus_t { routerstatus_t status; /**< Underlying 'status' object for this router. @@ -1590,31 +1670,45 @@ typedef struct vote_routerstatus_t { * networkstatus_t.known_flags. */ char *version; /**< The version that the authority says this router is * running. */ + vote_microdesc_hash_t *microdesc; } vote_routerstatus_t; +/** A signature of some document by an authority. */ +typedef struct document_signature_t { + /** Declared SHA-1 digest of this voter's identity key */ + char identity_digest[DIGEST_LEN]; + /** Declared SHA-1 digest of signing key used by this voter. */ + char signing_key_digest[DIGEST_LEN]; + /** Algorithm used to compute the digest of the document. */ + digest_algorithm_t alg; + /** Signature of the signed thing. */ + char *signature; + /** Length of <b>signature</b> */ + int signature_len; + unsigned int bad_signature : 1; /**< Set to true if we've tried to verify + * the sig, and we know it's bad. */ + unsigned int good_signature : 1; /**< Set to true if we've verified the sig + * as good. */ +} document_signature_t; + /** Information about a single voter in a vote or a consensus. */ typedef struct networkstatus_voter_info_t { + /** Declared SHA-1 digest of this voter's identity key */ + char identity_digest[DIGEST_LEN]; char *nickname; /**< Nickname of this voter */ - char identity_digest[DIGEST_LEN]; /**< Digest of this voter's identity key */ + /** Digest of this voter's "legacy" identity key, if any. In vote only; for + * consensuses, we treat legacy keys as additional signers. */ + char legacy_id_digest[DIGEST_LEN]; char *address; /**< Address of this voter, in string format. */ uint32_t addr; /**< Address of this voter, in IPv4, in host order. */ uint16_t dir_port; /**< Directory port of this voter */ uint16_t or_port; /**< OR port of this voter */ char *contact; /**< Contact information for this voter. */ char vote_digest[DIGEST_LEN]; /**< Digest of this voter's vote, as signed. */ - /** Digest of this voter's "legacy" identity key, if any. In vote only; for - * consensuses, we treat legacy keys as additional signers. */ - char legacy_id_digest[DIGEST_LEN]; /* Nothing from here on is signed. */ - char signing_key_digest[DIGEST_LEN]; /**< Declared digest of signing key - * used by this voter. */ - char *signature; /**< Signature from this voter. */ - int signature_len; /**< Length of <b>signature</b> */ - unsigned int bad_signature : 1; /**< Set to true if we've tried to verify - * the sig, and we know it's bad. */ - unsigned int good_signature : 1; /**< Set to true if we've verified the sig - * as good. */ + /** The signature of the document and the signature's status. */ + smartlist_t *sigs; } networkstatus_voter_info_t; /** Enumerates the possible seriousness values of a networkstatus document. */ @@ -1624,10 +1718,25 @@ typedef enum { NS_TYPE_OPINION, } networkstatus_type_t; +/** Enumerates recognized flavors of a consensus networkstatus document. All + * flavors of a consensus are generated from the same set of votes, but they + * present different types information to different versions of Tor. */ +typedef enum { + FLAV_NS = 0, + FLAV_MICRODESC = 1, +} consensus_flavor_t; + +/** Which consensus flavor do we actually want to use to build circuits? */ +#define USABLE_CONSENSUS_FLAVOR FLAV_NS + +/** How many different consensus flavors are there? */ +#define N_CONSENSUS_FLAVORS ((int)(FLAV_MICRODESC)+1) + /** A common structure to hold a v3 network status vote, or a v3 network * status consensus. */ typedef struct networkstatus_t { - networkstatus_type_t type; /**< Vote, consensus, or opinion? */ + networkstatus_type_t type : 8; /**< Vote, consensus, or opinion? */ + consensus_flavor_t flavor : 8; /**< If a consensus, what kind? */ time_t published; /**< Vote only: Time when vote was written. */ time_t valid_after; /**< Time after which this vote or consensus applies. */ time_t fresh_until; /**< Time before which this is the most recent vote or @@ -1659,6 +1768,10 @@ typedef struct networkstatus_t { * consensus, sorted by key. */ smartlist_t *net_params; + /** List of key=value strings for the bw weight parameters in the + * consensus. */ + smartlist_t *weight_params; + /** List of networkstatus_voter_info_t. For a vote, only one element * is included. For a consensus, one element is included for every voter * whose vote contributed to the consensus. */ @@ -1666,8 +1779,8 @@ typedef struct networkstatus_t { struct authority_cert_t *cert; /**< Vote only: the voter's certificate. */ - /** Digest of this document, as signed. */ - char networkstatus_digest[DIGEST_LEN]; + /** Digests of this document, as signed. */ + digests_t digests; /** List of router statuses, sorted by identity digest. For a vote, * the elements are vote_routerstatus_t; for a consensus, the elements @@ -1679,14 +1792,15 @@ typedef struct networkstatus_t { digestmap_t *desc_digest_map; } networkstatus_t; -/** A set of signatures for a networkstatus consensus. All fields are as for - * networkstatus_t. */ +/** A set of signatures for a networkstatus consensus. Unless otherwise + * noted, all fields are as for networkstatus_t. */ typedef struct ns_detached_signatures_t { time_t valid_after; time_t fresh_until; time_t valid_until; - char networkstatus_digest[DIGEST_LEN]; - smartlist_t *signatures; /* list of networkstatus_voter_info_t */ + strmap_t *digests; /**< Map from flavor name to digestset_t */ + strmap_t *signatures; /**< Map from flavor name to list of + * document_signature_t */ } ns_detached_signatures_t; /** Allowable types of desc_store_t. */ @@ -1891,6 +2005,29 @@ typedef struct { time_t expiry_time; } cpath_build_state_t; +/** + * The cell_ewma_t structure keeps track of how many cells a circuit has + * transferred recently. It keeps an EWMA (exponentially weighted moving + * average) of the number of cells flushed from the circuit queue onto a + * connection in connection_or_flush_from_first_active_circuit(). + */ +typedef struct { + /** The last 'tick' at which we recalibrated cell_count. + * + * A cell sent at exactly the start of this tick has weight 1.0. Cells sent + * since the start of this tick have weight greater than 1.0; ones sent + * earlier have less weight. */ + unsigned last_adjusted_tick; + /** The EWMA of the cell count. */ + double cell_count; + /** True iff this is the cell count for a circuit's previous + * connection. */ + unsigned int is_for_p_conn : 1; + /** The position of the circuit within the OR connection's priority + * queue. */ + int heap_index; +} cell_ewma_t; + #define ORIGIN_CIRCUIT_MAGIC 0x35315243u #define OR_CIRCUIT_MAGIC 0x98ABC04Fu @@ -1960,6 +2097,7 @@ typedef struct circuit_t { time_t timestamp_created; /**< When was this circuit created? */ time_t timestamp_dirty; /**< When the circuit was first used, or 0 if the * circuit is clean. */ + struct timeval highres_created; /**< When exactly was the circuit created? */ uint16_t marked_for_close; /**< Should we close this circuit at the end of * the main loop? (If true, holds the line number @@ -1976,6 +2114,14 @@ typedef struct circuit_t { * linked to an OR connection. */ struct circuit_t *prev_active_on_n_conn; struct circuit_t *next; /**< Next circuit in linked list of all circuits. */ + + /** Unique ID for measuring tunneled network status requests. */ + uint64_t dirreq_id; + + /** The EWMA count for the number of cells flushed from the + * n_conn_cells queue. Used to determine which circuit to flush from next. + */ + cell_ewma_t n_cell_ewma; } circuit_t; /** Largest number of relay_early cells that we can send on a given @@ -2008,6 +2154,9 @@ typedef struct origin_circuit_t { * to the specification? */ unsigned int remaining_relay_early_cells : 4; + /** Set if this circuit insanely old and if we already informed the user */ + unsigned int is_ancient : 1; + /** What commands were sent over this circuit that decremented the * RELAY_EARLY counter? This is for debugging task 878. */ uint8_t relay_early_commands[MAX_RELAY_EARLY_CELLS_PER_CIRCUIT]; @@ -2098,6 +2247,19 @@ typedef struct or_circuit_t { /** True iff this circuit was made with a CREATE_FAST cell. */ unsigned int is_first_hop : 1; + + /** Number of cells that were removed from circuit queue; reset every + * time when writing buffer stats to disk. */ + uint32_t processed_cells; + + /** Total time in milliseconds that cells spent in both app-ward and + * exit-ward queues of this circuit; reset every time when writing + * buffer stats to disk. */ + uint64_t total_cell_waiting_time; + + /** The EWMA count for the number of cells flushed from the + * p_conn_cells queue. */ + cell_ewma_t p_cell_ewma; } or_circuit_t; /** Convert a circuit subtype to a circuit_t.*/ @@ -2169,13 +2331,13 @@ typedef struct { routerset_t *EntryNodes;/**< Structure containing nicknames, digests, * country codes and IP address patterns of ORs to * consider as entry points. */ - int StrictExitNodes; /**< Boolean: When none of our ExitNodes are up, do we - * stop building circuits? */ - int StrictEntryNodes; /**< Boolean: When none of our EntryNodes are up, do we - * stop building circuits? */ + int StrictNodes; /**< Boolean: When none of our EntryNodes or ExitNodes + * are up, or we need to access a node in ExcludeNodes, + * do we just fail instead? */ routerset_t *ExcludeNodes;/**< Structure containing nicknames, digests, * country codes and IP address patterns of ORs - * not to use in circuits. */ + * not to use in circuits. But see StrictNodes + * above. */ routerset_t *ExcludeExitNodes;/**< Structure containing nicknames, digests, * country codes and IP address patterns of * ORs not to consider as exits. */ @@ -2183,6 +2345,9 @@ typedef struct { /** Union of ExcludeNodes and ExcludeExitNodes */ struct routerset_t *_ExcludeExitNodesUnion; + int DisableAllSwap; /**< Boolean: Attempt to call mlockall() on our + * process for all current and future memory. */ + /** List of "entry", "middle", "exit", "introduction", "rendezvous". */ smartlist_t *AllowInvalidNodes; /** Bitmask; derived from AllowInvalidNodes. */ @@ -2237,8 +2402,6 @@ typedef struct { * for version 3 directories? */ int HSAuthoritativeDir; /**< Boolean: does this an authoritative directory * handle hidden service requests? */ - int HSAuthorityRecordStats; /**< Boolean: does this HS authoritative - * directory record statistics? */ int NamingAuthoritativeDir; /**< Boolean: is this an authoritative directory * that's willing to bind names? */ int VersioningAuthoritativeDir; /**< Boolean: is this an authoritative @@ -2299,6 +2462,11 @@ typedef struct { int ConstrainedSockets; /**< Shrink xmit and recv socket buffers. */ uint64_t ConstrainedSockSize; /**< Size of constrained buffers. */ + /** Whether we should drop exit streams from Tors that we don't know + * are relays. XXX022 In here for 0.2.2.11 as a temporary test before + * we switch over to putting it in consensusparams. -RD */ + int RefuseUnknownExits; + /** Application ports that require all nodes in circ to have sufficient * uptime. */ smartlist_t *LongLivedPorts; @@ -2328,10 +2496,18 @@ typedef struct { * connections alive? */ int SocksTimeout; /**< How long do we let a socks connection wait * unattached before we fail it? */ - int CircuitBuildTimeout; /**< Cull non-open circuits that were born - * at least this many seconds ago. */ + int LearnCircuitBuildTimeout; /**< If non-zero, we attempt to learn a value + * for CircuitBuildTimeout based on timeout + * history */ + int CircuitBuildTimeout; /**< Cull non-open circuits that were born at + * least this many seconds ago. Used until + * adaptive algorithm learns a new value. */ int CircuitIdleTimeout; /**< Cull open clean circuits that were born * at least this many seconds ago. */ + int CircuitStreamTimeout; /**< If non-zero, detach streams from circuits + * and try a new circuit if the stream has been + * waiting for this many seconds. If zero, use + * our default internal timeout schedule. */ int MaxOnionsPending; /**< How many circuit CREATE requests do we allow * to wait simultaneously before we start dropping * them? */ @@ -2349,6 +2525,8 @@ typedef struct { * willing to use for all relayed conns? */ uint64_t RelayBandwidthBurst; /**< How much bandwidth, at maximum, will we * use in a second for all relayed conns? */ + uint64_t PerConnBWRate; /**< Long-term bw on a single TLS conn, if set. */ + uint64_t PerConnBWBurst; /**< Allowed burst on a single TLS conn, if set. */ int NumCpus; /**< How many CPUs should we try to use? */ int RunTesting; /**< If true, create testing circuits to measure how well the * other ORs are running. */ @@ -2359,15 +2537,25 @@ typedef struct { char *ContactInfo; /**< Contact info to be published in the directory. */ char *HttpProxy; /**< hostname[:port] to use as http proxy, if any. */ - uint32_t HttpProxyAddr; /**< Parsed IPv4 addr for http proxy, if any. */ + tor_addr_t HttpProxyAddr; /**< Parsed IPv4 addr for http proxy, if any. */ uint16_t HttpProxyPort; /**< Parsed port for http proxy, if any. */ char *HttpProxyAuthenticator; /**< username:password string, if any. */ char *HttpsProxy; /**< hostname[:port] to use as https proxy, if any. */ - uint32_t HttpsProxyAddr; /**< Parsed IPv4 addr for https proxy, if any. */ + tor_addr_t HttpsProxyAddr; /**< Parsed addr for https proxy, if any. */ uint16_t HttpsProxyPort; /**< Parsed port for https proxy, if any. */ char *HttpsProxyAuthenticator; /**< username:password string, if any. */ + char *Socks4Proxy; /**< hostname:port to use as a SOCKS4 proxy, if any. */ + tor_addr_t Socks4ProxyAddr; /**< Derived from Socks4Proxy. */ + uint16_t Socks4ProxyPort; /**< Derived from Socks4Proxy. */ + + char *Socks5Proxy; /**< hostname:port to use as a SOCKS5 proxy, if any. */ + tor_addr_t Socks5ProxyAddr; /**< Derived from Sock5Proxy. */ + uint16_t Socks5ProxyPort; /**< Derived from Socks5Proxy. */ + char *Socks5ProxyUsername; /**< Username for SOCKS5 authentication, if any */ + char *Socks5ProxyPassword; /**< Password for SOCKS5 authentication, if any */ + /** List of configuration lines for replacement directory authorities. * If you just want to replace one class of authority at a time, * use the "Alternate*Authority" options below instead. */ @@ -2429,8 +2617,13 @@ typedef struct { * or not (1)? */ int ShutdownWaitLength; /**< When we get a SIGINT and we're a server, how * long do we wait before exiting? */ - int SafeLogging; /**< Boolean: are we allowed to log sensitive strings - * such as addresses (0), or do we scrub them first (1)? */ + char *SafeLogging; /**< Contains "relay", "1", "0" (meaning no scrubbing). */ + + /* Derived from SafeLogging */ + enum { + SAFELOG_SCRUB_ALL, SAFELOG_SCRUB_RELAY, SAFELOG_SCRUB_NONE + } _SafeLogging; + int SafeSocks; /**< Boolean: should we outright refuse application * connections that use socks4 or socks5-with-local-dns? */ #define LOG_PROTOCOL_WARN (get_options()->ProtocolWarnings ? \ @@ -2441,6 +2634,8 @@ typedef struct { * log whether it was DNS-leaking or not? */ int HardwareAccel; /**< Boolean: Should we enable OpenSSL hardware * acceleration where available? */ + char *AccelName; /**< Optional hardware acceleration engine name. */ + char *AccelDir; /**< Optional hardware acceleration engine search dir. */ int UseEntryGuards; /**< Boolean: Do we try to enter from a smallish number * of fixed nodes? */ int NumEntryGuards; /**< How many entry guards do we try to establish? */ @@ -2451,6 +2646,9 @@ typedef struct { * means directly from the authorities) no matter our other config? */ int FetchDirInfoEarly; + /** Should we fetch our dir info at the start of the consensus period? */ + int FetchDirInfoExtraEarly; + char *VirtualAddrNetwork; /**< Address and mask to hand out for virtual * MAPADDRESS requests. */ int ServerDNSSearchDomains; /**< Boolean: If set, we don't force exit @@ -2499,6 +2697,33 @@ typedef struct { * exit allows it, we use it. */ int AllowSingleHopCircuits; + /** If true, we convert "www.google.com.foo.exit" addresses on the + * socks/trans/natd ports into "www.google.com" addresses that + * exit from the node "foo". Disabled by default since attacking + * websites and exit relays can use it to manipulate your path + * selection. */ + int AllowDotExit; + + /** If true, we will warn if a user gives us only an IP address + * instead of a hostname. */ + int WarnUnsafeSocks; + + /** If true, the user wants us to collect statistics on clients + * requesting network statuses from us as directory. */ + int DirReqStatistics; + + /** If true, the user wants us to collect statistics on port usage. */ + int ExitPortStatistics; + + /** If true, the user wants us to collect cell statistics. */ + int CellStatistics; + + /** If true, the user wants us to collect statistics as entry node. */ + int EntryStatistics; + + /** If true, include statistics file contents in extra-info documents. */ + int ExtraInfoStatistics; + /** If true, do not believe anybody who tells us that a domain resolves * to an internal address, or that an internal address has a PTR mapping. * Helps avoid some cross-site attacks. */ @@ -2517,6 +2742,13 @@ typedef struct { * migration purposes? */ int V3AuthUseLegacyKey; + /** Location of bandwidth measurement file */ + char *V3BandwidthsFile; + + /** Authority only: key=value pairs that we add to our networkstatus + * consensus vote on the 'params' line. */ + char *ConsensusParams; + /** The length of time that we think an initial consensus should be fresh. * Only altered on testing networks. */ int TestingV3AuthInitialVotingInterval; @@ -2553,19 +2785,6 @@ typedef struct { * the bridge authority guess which countries have blocked access to us. */ int BridgeRecordUsageByCountry; -#ifdef ENABLE_GEOIP_STATS - /** If true, and Tor is built with GEOIP_STATS support, and we're a - * directory, record how many directory requests we get from each country. */ - int DirRecordUsageByCountry; - /** Round all GeoIP results to the next multiple of this value, to avoid - * leaking information. */ - int DirRecordUsageGranularity; - /** Time interval: purge geoip stats after this long. */ - int DirRecordUsageRetainIPs; - /** Time interval: Flush geoip data to disk this often. */ - int DirRecordUsageSaveInterval; -#endif - /** Optionally, a file with GeoIP data. */ char *GeoIPFile; @@ -2573,6 +2792,21 @@ typedef struct { * to make this false. */ int ReloadTorrcOnSIGHUP; + /* The main parameter for picking circuits within a connection. + * + * If this value is positive, when picking a cell to relay on a connection, + * we always relay from the circuit whose weighted cell count is lowest. + * Cells are weighted exponentially such that if one cell is sent + * 'CircuitPriorityHalflife' seconds before another, it counts for half as + * much. + * + * If this value is zero, we're disabling the cell-EWMA algorithm. + * + * If this value is negative, we're using the default approach + * according to either Tor or a parameter set in the consensus. + */ + double CircuitPriorityHalflife; + } or_options_t; /** Persistent state for an onion router, as saved to disk. */ @@ -2609,6 +2843,11 @@ typedef struct { int BWHistoryWriteInterval; smartlist_t *BWHistoryWriteValues; + /** Build time histogram */ + config_line_t * BuildtimeHistogram; + unsigned int TotalBuildTimes; + unsigned int CircuitBuildAbandonedCount; + /** What version of Tor wrote this state file? */ char *TorVersion; @@ -2669,209 +2908,149 @@ struct socks_request_t { /* all the function prototypes go here */ -/********************************* buffers.c ***************************/ - -buf_t *buf_new(void); -buf_t *buf_new_with_capacity(size_t size); -void buf_free(buf_t *buf); -void buf_clear(buf_t *buf); -void buf_shrink(buf_t *buf); -void buf_shrink_freelists(int free_all); -void buf_dump_freelist_sizes(int severity); - -size_t buf_datalen(const buf_t *buf); -size_t buf_allocation(const buf_t *buf); -size_t buf_slack(const buf_t *buf); -const char *_buf_peek_raw_buffer(const buf_t *buf); - -int read_to_buf(int s, size_t at_most, buf_t *buf, int *reached_eof, - int *socket_error); -int read_to_buf_tls(tor_tls_t *tls, size_t at_most, buf_t *buf); - -int flush_buf(int s, buf_t *buf, size_t sz, size_t *buf_flushlen); -int flush_buf_tls(tor_tls_t *tls, buf_t *buf, size_t sz, size_t *buf_flushlen); - -int write_to_buf(const char *string, size_t string_len, buf_t *buf); -int write_to_buf_zlib(buf_t *buf, tor_zlib_state_t *state, - const char *data, size_t data_len, int done); -int move_buf_to_buf(buf_t *buf_out, buf_t *buf_in, size_t *buf_flushlen); -int fetch_from_buf(char *string, size_t string_len, buf_t *buf); -int fetch_var_cell_from_buf(buf_t *buf, var_cell_t **out, int linkproto); -int fetch_from_buf_http(buf_t *buf, - char **headers_out, size_t max_headerlen, - char **body_out, size_t *body_used, size_t max_bodylen, - int force_complete); -int fetch_from_buf_socks(buf_t *buf, socks_request_t *req, - int log_sockstype, int safe_socks); -int fetch_from_buf_line(buf_t *buf, char *data_out, size_t *data_len); - -int peek_buf_has_control0_command(buf_t *buf); - -void assert_buf_ok(buf_t *buf); - -#ifdef BUFFERS_PRIVATE -int buf_find_string_offset(const buf_t *buf, const char *s, size_t n); -#endif - /********************************* circuitbuild.c **********************/ -char *circuit_list_path(origin_circuit_t *circ, int verbose); -char *circuit_list_path_for_controller(origin_circuit_t *circ); -void circuit_log_path(int severity, unsigned int domain, - origin_circuit_t *circ); -void circuit_rep_hist_note_result(origin_circuit_t *circ); -origin_circuit_t *origin_circuit_init(uint8_t purpose, int flags); -origin_circuit_t *circuit_establish_circuit(uint8_t purpose, - extend_info_t *exit, - int flags); -int circuit_handle_first_hop(origin_circuit_t *circ); -void circuit_n_conn_done(or_connection_t *or_conn, int status); -int inform_testing_reachability(void); -int circuit_send_next_onion_skin(origin_circuit_t *circ); -void circuit_note_clock_jumped(int seconds_elapsed); -int circuit_extend(cell_t *cell, circuit_t *circ); -int circuit_init_cpath_crypto(crypt_path_t *cpath, const char *key_data, - int reverse); -int circuit_finish_handshake(origin_circuit_t *circ, uint8_t cell_type, - const char *reply); -int circuit_truncated(origin_circuit_t *circ, crypt_path_t *layer); -int onionskin_answer(or_circuit_t *circ, uint8_t cell_type, - const char *payload, const char *keys); -int circuit_all_predicted_ports_handled(time_t now, int *need_uptime, - int *need_capacity); - -int circuit_append_new_exit(origin_circuit_t *circ, extend_info_t *info); -int circuit_extend_to_new_exit(origin_circuit_t *circ, extend_info_t *info); -void onion_append_to_cpath(crypt_path_t **head_ptr, crypt_path_t *new_hop); -extend_info_t *extend_info_alloc(const char *nickname, const char *digest, - crypto_pk_env_t *onion_key, - const tor_addr_t *addr, uint16_t port); -extend_info_t *extend_info_from_router(routerinfo_t *r); -extend_info_t *extend_info_dup(extend_info_t *info); -void extend_info_free(extend_info_t *info); -routerinfo_t *build_state_get_exit_router(cpath_build_state_t *state); -const char *build_state_get_exit_nickname(cpath_build_state_t *state); - -void entry_guards_compute_status(void); -int entry_guard_register_connect_status(const char *digest, int succeeded, - int mark_relay_status, time_t now); -void entry_nodes_should_be_added(void); -int entry_list_can_grow(or_options_t *options); -routerinfo_t *choose_random_entry(cpath_build_state_t *state); -int entry_guards_parse_state(or_state_t *state, int set, char **msg); -void entry_guards_update_state(or_state_t *state); -int getinfo_helper_entry_guards(control_connection_t *conn, - const char *question, char **answer); - -void clear_bridge_list(void); -int routerinfo_is_a_configured_bridge(routerinfo_t *ri); -void bridge_add_from_config(const tor_addr_t *addr, uint16_t port, - char *digest); -void retry_bridge_descriptor_fetch_directly(const char *digest); -void fetch_bridge_descriptors(time_t now); -void learned_bridge_descriptor(routerinfo_t *ri, int from_cache); -int any_bridge_descriptors_known(void); -int any_pending_bridge_descriptor_fetches(void); -int bridges_known_but_down(void); -void bridges_retry_all(void); - -void entry_guards_free_all(void); - -/********************************* circuitlist.c ***********************/ - -circuit_t * _circuit_get_global_list(void); -const char *circuit_state_to_string(int state); -const char *circuit_purpose_to_controller_string(uint8_t purpose); -void circuit_dump_by_conn(connection_t *conn, int severity); -void circuit_set_p_circid_orconn(or_circuit_t *circ, circid_t id, - or_connection_t *conn); -void circuit_set_n_circid_orconn(circuit_t *circ, circid_t id, - or_connection_t *conn); -void circuit_set_state(circuit_t *circ, uint8_t state); -void circuit_close_all_marked(void); -int32_t circuit_initial_package_window(void); -origin_circuit_t *origin_circuit_new(void); -or_circuit_t *or_circuit_new(circid_t p_circ_id, or_connection_t *p_conn); -circuit_t *circuit_get_by_circid_orconn(circid_t circ_id, - or_connection_t *conn); -int circuit_id_in_use_on_orconn(circid_t circ_id, or_connection_t *conn); -circuit_t *circuit_get_by_edge_conn(edge_connection_t *conn); -void circuit_unlink_all_from_or_conn(or_connection_t *conn, int reason); -origin_circuit_t *circuit_get_by_global_id(uint32_t id); -origin_circuit_t *circuit_get_by_rend_query_and_purpose(const char *rend_query, - uint8_t purpose); -origin_circuit_t *circuit_get_next_by_pk_and_purpose(origin_circuit_t *start, - const char *digest, uint8_t purpose); -or_circuit_t *circuit_get_rendezvous(const char *cookie); -or_circuit_t *circuit_get_intro_point(const char *digest); -origin_circuit_t *circuit_find_to_cannibalize(uint8_t purpose, - extend_info_t *info, int flags); -void circuit_mark_all_unused_circs(void); -void circuit_expire_all_dirty_circs(void); -void _circuit_mark_for_close(circuit_t *circ, int reason, - int line, const char *file); -int circuit_get_cpath_len(origin_circuit_t *circ); -crypt_path_t *circuit_get_cpath_hop(origin_circuit_t *circ, int hopnum); -void circuit_get_all_pending_on_or_conn(smartlist_t *out, - or_connection_t *or_conn); -int circuit_count_pending_on_or_conn(or_connection_t *or_conn); - -#define circuit_mark_for_close(c, reason) \ - _circuit_mark_for_close((c), (reason), __LINE__, _SHORT_FILE_) - -void assert_cpath_layer_ok(const crypt_path_t *cp); -void assert_circuit_ok(const circuit_t *c); -void circuit_free_all(void); - -/********************************* circuituse.c ************************/ - -void circuit_expire_building(time_t now); -void circuit_remove_handled_ports(smartlist_t *needed_ports); -int circuit_stream_is_being_handled(edge_connection_t *conn, uint16_t port, - int min); -int circuit_conforms_to_options(const origin_circuit_t *circ, - const or_options_t *options); -void circuit_build_needed_circs(time_t now); -void circuit_detach_stream(circuit_t *circ, edge_connection_t *conn); - -void circuit_expire_old_circuits_serverside(time_t now); - -void reset_bandwidth_test(void); -int circuit_enough_testing_circs(void); - -void circuit_has_opened(origin_circuit_t *circ); -void circuit_build_failed(origin_circuit_t *circ); - -/** Flag to set when a circuit should have only a single hop. */ -#define CIRCLAUNCH_ONEHOP_TUNNEL (1<<0) -/** Flag to set when a circuit needs to be built of high-uptime nodes */ -#define CIRCLAUNCH_NEED_UPTIME (1<<1) -/** Flag to set when a circuit needs to be built of high-capacity nodes */ -#define CIRCLAUNCH_NEED_CAPACITY (1<<2) -/** Flag to set when the last hop of a circuit doesn't need to be an - * exit node. */ -#define CIRCLAUNCH_IS_INTERNAL (1<<3) -origin_circuit_t *circuit_launch_by_extend_info(uint8_t purpose, - extend_info_t *info, - int flags); -origin_circuit_t *circuit_launch_by_router(uint8_t purpose, - routerinfo_t *exit, int flags); -void circuit_reset_failure_count(int timeout); -int connection_ap_handshake_attach_chosen_circuit(edge_connection_t *conn, - origin_circuit_t *circ, - crypt_path_t *cpath); -int connection_ap_handshake_attach_circuit(edge_connection_t *conn); - -/********************************* command.c ***************************/ - -void command_process_cell(cell_t *cell, or_connection_t *conn); -void command_process_var_cell(var_cell_t *cell, or_connection_t *conn); - -extern uint64_t stats_n_padding_cells_processed; -extern uint64_t stats_n_create_cells_processed; -extern uint64_t stats_n_created_cells_processed; -extern uint64_t stats_n_relay_cells_processed; -extern uint64_t stats_n_destroy_cells_processed; +/** How many hops does a general-purpose circuit have by default? */ +#define DEFAULT_ROUTE_LEN 3 + +/* Circuit Build Timeout "public" structures. */ + +/** Total size of the circuit timeout history to accumulate. + * 1000 is approx 2.5 days worth of continual-use circuits. */ +#define CBT_NCIRCUITS_TO_OBSERVE 1000 + +/** Width of the histogram bins in milliseconds */ +#define CBT_BIN_WIDTH ((build_time_t)50) + +/** Number of modes to use in the weighted-avg computation of Xm */ +#define CBT_DEFAULT_NUM_XM_MODES 3 + +/** A build_time_t is milliseconds */ +typedef uint32_t build_time_t; + +/** + * CBT_BUILD_ABANDONED is our flag value to represent a force-closed + * circuit (Aka a 'right-censored' pareto value). + */ +#define CBT_BUILD_ABANDONED ((build_time_t)(INT32_MAX-1)) +#define CBT_BUILD_TIME_MAX ((build_time_t)(INT32_MAX)) + +/** Save state every 10 circuits */ +#define CBT_SAVE_STATE_EVERY 10 + +/* Circuit Build Timeout network liveness constants */ + +/** + * Have we received a cell in the last N circ attempts? + * + * This tells us when to temporarily switch back to + * BUILD_TIMEOUT_INITIAL_VALUE until we start getting cells, + * at which point we switch back to computing the timeout from + * our saved history. + */ +#define CBT_NETWORK_NONLIVE_TIMEOUT_COUNT 3 + +/** + * This tells us when to toss out the last streak of N timeouts. + * + * If instead we start getting cells, we switch back to computing the timeout + * from our saved history. + */ +#define CBT_NETWORK_NONLIVE_DISCARD_COUNT (CBT_NETWORK_NONLIVE_TIMEOUT_COUNT*2) + +/* Circuit build times consensus parameters */ + +/** + * How long to wait before actually closing circuits that take too long to + * build in terms of CDF quantile. + */ +#define CBT_DEFAULT_CLOSE_QUANTILE 95 + +/** + * How many circuits count as recent when considering if the + * connection has gone gimpy or changed. + */ +#define CBT_DEFAULT_RECENT_CIRCUITS 20 + +/** + * Maximum count of timeouts that finish the first hop in the past + * RECENT_CIRCUITS before calculating a new timeout. + * + * This tells us to abandon timeout history and set + * the timeout back to BUILD_TIMEOUT_INITIAL_VALUE. + */ +#define CBT_DEFAULT_MAX_RECENT_TIMEOUT_COUNT (CBT_DEFAULT_RECENT_CIRCUITS*9/10) + +/** Minimum circuits before estimating a timeout */ +#define CBT_DEFAULT_MIN_CIRCUITS_TO_OBSERVE 100 + +/** Cutoff percentile on the CDF for our timeout estimation. */ +#define CBT_DEFAULT_QUANTILE_CUTOFF 80 +double circuit_build_times_quantile_cutoff(void); + +/** How often in seconds should we build a test circuit */ +#define CBT_DEFAULT_TEST_FREQUENCY 60 + +/** Lowest allowable value for CircuitBuildTimeout in milliseconds */ +#define CBT_DEFAULT_TIMEOUT_MIN_VALUE (2*1000) + +/** Initial circuit build timeout in milliseconds */ +#define CBT_DEFAULT_TIMEOUT_INITIAL_VALUE (60*1000) +int32_t circuit_build_times_initial_timeout(void); + +#if CBT_DEFAULT_MAX_RECENT_TIMEOUT_COUNT < 1 || \ + CBT_NETWORK_NONLIVE_DISCARD_COUNT < 1 || \ + CBT_NETWORK_NONLIVE_TIMEOUT_COUNT < 1 +#error "RECENT_CIRCUITS is set too low." +#endif + +/** Information about the state of our local network connection */ +typedef struct { + /** The timestamp we last completed a TLS handshake or received a cell */ + time_t network_last_live; + /** If the network is not live, how many timeouts has this caused? */ + int nonlive_timeouts; + /** If the network is not live, have we yet discarded our history? */ + int nonlive_discarded; + /** Circular array of circuits that have made it to the first hop. Slot is + * 1 if circuit timed out, 0 if circuit succeeded */ + int8_t *timeouts_after_firsthop; + /** Number of elements allocated for the above array */ + int num_recent_circs; + /** Index into circular array. */ + int after_firsthop_idx; + /** Timeout gathering is suspended if non-zero. The old timeout value + * is stored here in that case. */ + double suspended_timeout; + /** Timeout gathering is suspended if non-zero. The old close value + * is stored here in that case. */ + double suspended_close_timeout; +} network_liveness_t; + +/** Structure for circuit build times history */ +typedef struct { + /** The circular array of recorded build times in milliseconds */ + build_time_t circuit_build_times[CBT_NCIRCUITS_TO_OBSERVE]; + /** Current index in the circuit_build_times circular array */ + int build_times_idx; + /** Total number of build times accumulated. Max CBT_NCIRCUITS_TO_OBSERVE */ + int total_build_times; + /** Information about the state of our local network connection */ + network_liveness_t liveness; + /** Last time we built a circuit. Used to decide to build new test circs */ + time_t last_circ_at; + /** "Minimum" value of our pareto distribution (actually mode) */ + build_time_t Xm; + /** alpha exponent for pareto dist. */ + double alpha; + /** Have we computed a timeout? */ + int have_computed_timeout; + /** The exact value for that timeout in milliseconds. Stored as a double + * to maintain precision from calculations to and from quantile value. */ + double timeout_ms; + /** How long we wait before actually closing the circuit. */ + double close_ms; +} circuit_build_times_t; /********************************* config.c ***************************/ @@ -2884,203 +3063,8 @@ typedef enum setopt_err_t { SETOPT_ERR_SETTING = -4, } setopt_err_t; -const char *get_dirportfrontpage(void); -or_options_t *get_options(void); -int set_options(or_options_t *new_val, char **msg); -void config_free_all(void); -const char *safe_str(const char *address); -const char *escaped_safe_str(const char *address); -const char *get_version(void); - -int config_get_lines(const char *string, config_line_t **result); -void config_free_lines(config_line_t *front); -setopt_err_t options_trial_assign(config_line_t *list, int use_defaults, - int clear_first, char **msg); -int resolve_my_address(int warn_severity, or_options_t *options, - uint32_t *addr, char **hostname_out); -int is_local_addr(const tor_addr_t *addr) ATTR_PURE; -void options_init(or_options_t *options); -int options_init_from_torrc(int argc, char **argv); -setopt_err_t options_init_from_string(const char *cf, - int command, const char *command_arg, char **msg); -int option_is_recognized(const char *key); -const char *option_get_canonical_name(const char *key); -config_line_t *option_get_assignment(or_options_t *options, - const char *key); -int options_save_current(void); -const char *get_torrc_fname(void); -char *options_get_datadir_fname2_suffix(or_options_t *options, - const char *sub1, const char *sub2, - const char *suffix); -#define get_datadir_fname2_suffix(sub1, sub2, suffix) \ - options_get_datadir_fname2_suffix(get_options(), (sub1), (sub2), (suffix)) -/** Return a newly allocated string containing datadir/sub1. See - * get_datadir_fname2_suffix. */ -#define get_datadir_fname(sub1) get_datadir_fname2_suffix((sub1), NULL, NULL) -/** Return a newly allocated string containing datadir/sub1/sub2. See - * get_datadir_fname2_suffix. */ -#define get_datadir_fname2(sub1,sub2) \ - get_datadir_fname2_suffix((sub1), (sub2), NULL) -/** Return a newly allocated string containing datadir/sub1suffix. See - * get_datadir_fname2_suffix. */ -#define get_datadir_fname_suffix(sub1, suffix) \ - get_datadir_fname2_suffix((sub1), NULL, (suffix)) - -or_state_t *get_or_state(void); -int or_state_save(time_t now); - -int options_need_geoip_info(or_options_t *options, const char **reason_out); -int getinfo_helper_config(control_connection_t *conn, - const char *question, char **answer); - -uint32_t get_effective_bwrate(or_options_t *options); -uint32_t get_effective_bwburst(or_options_t *options); - -#ifdef CONFIG_PRIVATE -/* Used only by config.c and test.c */ -or_options_t *options_new(void); -#endif - -/********************************* connection.c ***************************/ - -const char *conn_type_to_string(int type); -const char *conn_state_to_string(int type, int state); - -dir_connection_t *dir_connection_new(int socket_family); -or_connection_t *or_connection_new(int socket_family); -edge_connection_t *edge_connection_new(int type, int socket_family); -control_connection_t *control_connection_new(int socket_family); -connection_t *connection_new(int type, int socket_family); - -void connection_link_connections(connection_t *conn_a, connection_t *conn_b); -void connection_unregister_events(connection_t *conn); -void connection_free(connection_t *conn); -void connection_free_all(void); -void connection_about_to_close_connection(connection_t *conn); -void connection_close_immediate(connection_t *conn); -void _connection_mark_for_close(connection_t *conn,int line, const char *file); - -#define connection_mark_for_close(c) \ - _connection_mark_for_close((c), __LINE__, _SHORT_FILE_) - -void connection_expire_held_open(void); - -int connection_connect(connection_t *conn, const char *address, - const tor_addr_t *addr, - uint16_t port, int *socket_error); -int retry_all_listeners(smartlist_t *replaced_conns, - smartlist_t *new_conns); - -ssize_t connection_bucket_write_limit(connection_t *conn, time_t now); -int global_write_bucket_low(connection_t *conn, size_t attempt, int priority); -void connection_bucket_init(void); -void connection_bucket_refill(int seconds_elapsed, time_t now); - -int connection_handle_read(connection_t *conn); - -int connection_fetch_from_buf(char *string, size_t len, connection_t *conn); - -int connection_wants_to_flush(connection_t *conn); -int connection_outbuf_too_full(connection_t *conn); -int connection_handle_write(connection_t *conn, int force); -void _connection_write_to_buf_impl(const char *string, size_t len, - connection_t *conn, int zlib); -static void connection_write_to_buf(const char *string, size_t len, - connection_t *conn); -static void connection_write_to_buf_zlib(const char *string, size_t len, - dir_connection_t *conn, int done); -static INLINE void -connection_write_to_buf(const char *string, size_t len, connection_t *conn) -{ - _connection_write_to_buf_impl(string, len, conn, 0); -} -static INLINE void -connection_write_to_buf_zlib(const char *string, size_t len, - dir_connection_t *conn, int done) -{ - _connection_write_to_buf_impl(string, len, TO_CONN(conn), done ? -1 : 1); -} - -connection_t *connection_get_by_global_id(uint64_t id); - -connection_t *connection_get_by_type(int type); -connection_t *connection_get_by_type_purpose(int type, int purpose); -connection_t *connection_get_by_type_addr_port_purpose(int type, - const tor_addr_t *addr, - uint16_t port, int purpose); -connection_t *connection_get_by_type_state(int type, int state); -connection_t *connection_get_by_type_state_rendquery(int type, int state, - const char *rendquery, - int rendversion); - -#define connection_speaks_cells(conn) ((conn)->type == CONN_TYPE_OR) -int connection_is_listener(connection_t *conn); -int connection_state_is_open(connection_t *conn); -int connection_state_is_connecting(connection_t *conn); - -char *alloc_http_authenticator(const char *authenticator); - -void assert_connection_ok(connection_t *conn, time_t now); -int connection_or_nonopen_was_started_here(or_connection_t *conn); -void connection_dump_buffer_mem_stats(int severity); -void remove_file_if_very_old(const char *fname, time_t now); - /********************************* connection_edge.c *************************/ -#define connection_mark_unattached_ap(conn, endreason) \ - _connection_mark_unattached_ap((conn), (endreason), __LINE__, _SHORT_FILE_) - -void _connection_mark_unattached_ap(edge_connection_t *conn, int endreason, - int line, const char *file); -int connection_edge_reached_eof(edge_connection_t *conn); -int connection_edge_process_inbuf(edge_connection_t *conn, - int package_partial); -int connection_edge_destroy(circid_t circ_id, edge_connection_t *conn); -int connection_edge_end(edge_connection_t *conn, uint8_t reason); -int connection_edge_end_errno(edge_connection_t *conn); -int connection_edge_finished_flushing(edge_connection_t *conn); -int connection_edge_finished_connecting(edge_connection_t *conn); - -int connection_ap_handshake_send_begin(edge_connection_t *ap_conn); -int connection_ap_handshake_send_resolve(edge_connection_t *ap_conn); - -edge_connection_t *connection_ap_make_link(char *address, uint16_t port, - const char *digest, - int use_begindir, int want_onehop); -void connection_ap_handshake_socks_reply(edge_connection_t *conn, char *reply, - size_t replylen, - int endreason); -void connection_ap_handshake_socks_resolved(edge_connection_t *conn, - int answer_type, - size_t answer_len, - const char *answer, - int ttl, - time_t expires); - -int connection_exit_begin_conn(cell_t *cell, circuit_t *circ); -int connection_exit_begin_resolve(cell_t *cell, or_circuit_t *circ); -void connection_exit_connect(edge_connection_t *conn); -int connection_edge_is_rendezvous_stream(edge_connection_t *conn); -int connection_ap_can_use_exit(edge_connection_t *conn, routerinfo_t *exit); -void connection_ap_expire_beginning(void); -void connection_ap_attach_pending(void); -void connection_ap_fail_onehop(const char *failed_digest, - cpath_build_state_t *build_state); -void circuit_discard_optional_exit_enclaves(extend_info_t *info); -int connection_ap_detach_retriable(edge_connection_t *conn, - origin_circuit_t *circ, - int reason); -int connection_ap_process_transparent(edge_connection_t *conn); - -int address_is_invalid_destination(const char *address, int client); - -void addressmap_init(void); -void addressmap_clean(time_t now); -void addressmap_clear_configured(void); -void addressmap_clear_transient(void); -void addressmap_free_all(void); -int addressmap_rewrite(char *address, size_t maxlen, time_t *expires_out); -int addressmap_have_mapping(const char *address, int update_timeout); /** Enumerates possible origins of a client-side address mapping. */ typedef enum { /** We're remapping this address because the controller told us to. */ @@ -3095,75 +3079,6 @@ typedef enum { * Tor server that told us what its value was. */ ADDRMAPSRC_DNS, } addressmap_entry_source_t; -void addressmap_register(const char *address, char *new_address, - time_t expires, addressmap_entry_source_t source); -int parse_virtual_addr_network(const char *val, int validate_only, - char **msg); -int client_dns_incr_failures(const char *address); -void client_dns_clear_failures(const char *address); -void client_dns_set_addressmap(const char *address, uint32_t val, - const char *exitname, int ttl); -const char *addressmap_register_virtual_address(int type, char *new_address); -void addressmap_get_mappings(smartlist_t *sl, time_t min_expires, - time_t max_expires, int want_expiry); -int connection_ap_handshake_rewrite_and_attach(edge_connection_t *conn, - origin_circuit_t *circ, - crypt_path_t *cpath); -int hostname_is_noconnect_address(const char *address); - -/** Possible return values for parse_extended_hostname. */ -typedef enum hostname_type_t { - NORMAL_HOSTNAME, ONION_HOSTNAME, EXIT_HOSTNAME, BAD_HOSTNAME -} hostname_type_t; -hostname_type_t parse_extended_hostname(char *address); - -#if defined(HAVE_NET_IF_H) && defined(HAVE_NET_PFVAR_H) -int get_pf_socket(void); -#endif - -/********************************* connection_or.c ***************************/ - -void connection_or_remove_from_identity_map(or_connection_t *conn); -void connection_or_clear_identity_map(void); -or_connection_t *connection_or_get_for_extend(const char *digest, - const tor_addr_t *target_addr, - const char **msg_out, - int *launch_out); -void connection_or_set_bad_connections(void); - -int connection_or_reached_eof(or_connection_t *conn); -int connection_or_process_inbuf(or_connection_t *conn); -int connection_or_flushed_some(or_connection_t *conn); -int connection_or_finished_flushing(or_connection_t *conn); -int connection_or_finished_connecting(or_connection_t *conn); - -void connection_or_connect_failed(or_connection_t *conn, - int reason, const char *msg); -or_connection_t *connection_or_connect(const tor_addr_t *addr, uint16_t port, - const char *id_digest); - -int connection_tls_start_handshake(or_connection_t *conn, int receiving); -int connection_tls_continue_handshake(or_connection_t *conn); - -void or_handshake_state_free(or_handshake_state_t *state); -int connection_or_set_state_open(or_connection_t *conn); -void connection_or_write_cell_to_buf(const cell_t *cell, - or_connection_t *conn); -void connection_or_write_var_cell_to_buf(const var_cell_t *cell, - or_connection_t *conn); -int connection_or_send_destroy(circid_t circ_id, or_connection_t *conn, - int reason); -int connection_or_send_netinfo(or_connection_t *conn); -int connection_or_send_cert(or_connection_t *conn); -int connection_or_send_link_auth(or_connection_t *conn); -int connection_or_compute_link_auth_hmac(or_connection_t *conn, - char *hmac_out); -int is_or_protocol_version_known(uint16_t version); - -void cell_pack(packed_cell_t *dest, const cell_t *src); -void var_cell_pack_header(const var_cell_t *cell, char *hdr_out); -var_cell_t *var_cell_new(uint16_t payload_len); -void var_cell_free(var_cell_t *cell); /********************************* control.c ***************************/ @@ -3201,8 +3116,14 @@ typedef enum or_conn_status_event_t { OR_CONN_EVENT_NEW = 4, } or_conn_status_event_t; -void control_update_global_event_mask(void); -void control_adjust_event_log_severity(void); +/** Used to indicate the type of a buildtime event */ +typedef enum buildtimeout_set_event_t { + BUILDTIMEOUT_SET_EVENT_COMPUTED = 0, + BUILDTIMEOUT_SET_EVENT_RESET = 1, + BUILDTIMEOUT_SET_EVENT_SUSPENDED = 2, + BUILDTIMEOUT_SET_EVENT_DISCARD = 3, + BUILDTIMEOUT_SET_EVENT_RESUME = 4 +} buildtimeout_set_event_t; /** Execute the statement <b>stmt</b>, which may log events concerning the * connection <b>conn</b>. To prevent infinite loops, disable log messages @@ -3220,58 +3141,6 @@ void control_adjust_event_log_severity(void); enable_control_logging(); \ STMT_END -/** Log information about the connection <b>conn</b>, protecting it as with - * CONN_LOG_PROTECT. Example: - * - * LOG_FN_CONN(conn, (LOG_DEBUG, "Socket %d wants to write", conn->s)); - **/ -#define LOG_FN_CONN(conn, args) \ - CONN_LOG_PROTECT(conn, log_fn args) - -int connection_control_finished_flushing(control_connection_t *conn); -int connection_control_reached_eof(control_connection_t *conn); -int connection_control_process_inbuf(control_connection_t *conn); - -#define EVENT_AUTHDIR_NEWDESCS 0x000D -#define EVENT_NS 0x000F -int control_event_is_interesting(int event); - -int control_event_circuit_status(origin_circuit_t *circ, - circuit_status_event_t e, int reason); -int control_event_stream_status(edge_connection_t *conn, - stream_status_event_t e, - int reason); -int control_event_or_conn_status(or_connection_t *conn, - or_conn_status_event_t e, int reason); -int control_event_bandwidth_used(uint32_t n_read, uint32_t n_written); -int control_event_stream_bandwidth(edge_connection_t *edge_conn); -int control_event_stream_bandwidth_used(void); -void control_event_logmsg(int severity, unsigned int domain, const char *msg); -int control_event_descriptors_changed(smartlist_t *routers); -int control_event_address_mapped(const char *from, const char *to, - time_t expires, const char *error); -int control_event_or_authdir_new_descriptor(const char *action, - const char *desc, - size_t desclen, - const char *msg); -int control_event_my_descriptor_changed(void); -int control_event_networkstatus_changed(smartlist_t *statuses); -int control_event_newconsensus(const networkstatus_t *consensus); -int control_event_networkstatus_changed_single(routerstatus_t *rs); -int control_event_general_status(int severity, const char *format, ...) - CHECK_PRINTF(2,3); -int control_event_client_status(int severity, const char *format, ...) - CHECK_PRINTF(2,3); -int control_event_server_status(int severity, const char *format, ...) - CHECK_PRINTF(2,3); -int control_event_guard(const char *nickname, const char *digest, - const char *status); - -int init_cookie_authentication(int enabled); -smartlist_t *decode_hashed_passwords(config_line_t *passwords); -void disable_control_logging(void); -void enable_control_logging(void); - /** Enum describing various stages of bootstrapping, for use with controller * bootstrap status events. The values range from 0 to 100. */ typedef enum { @@ -3292,252 +3161,30 @@ typedef enum { BOOTSTRAP_STATUS_DONE=100 } bootstrap_status_t; -void control_event_bootstrap(bootstrap_status_t status, int progress); -void control_event_bootstrap_problem(const char *warn, int reason); - -void control_event_clients_seen(const char *timestarted, - const char *countries); - -#ifdef CONTROL_PRIVATE -/* Used only by control.c and test.c */ -size_t write_escaped_data(const char *data, size_t len, char **out); -size_t read_escaped_data(const char *data, size_t len, char **out); -#endif - -/********************************* cpuworker.c *****************************/ - -void cpu_init(void); -void cpuworkers_rotate(void); -int connection_cpu_finished_flushing(connection_t *conn); -int connection_cpu_reached_eof(connection_t *conn); -int connection_cpu_process_inbuf(connection_t *conn); -int assign_onionskin_to_cpuworker(connection_t *cpuworker, - or_circuit_t *circ, - char *onionskin); - /********************************* directory.c ***************************/ -int directories_have_accepted_server_descriptor(void); -char *authority_type_to_string(authority_type_t auth); -void directory_post_to_dirservers(uint8_t dir_purpose, uint8_t router_purpose, - authority_type_t type, const char *payload, - size_t payload_len, size_t extrainfo_len); -void directory_get_from_dirserver(uint8_t dir_purpose, uint8_t router_purpose, - const char *resource, - int pds_flags); -void directory_get_from_all_authorities(uint8_t dir_purpose, - uint8_t router_purpose, - const char *resource); -void directory_initiate_command_routerstatus(routerstatus_t *status, - uint8_t dir_purpose, - uint8_t router_purpose, - int anonymized_connection, - const char *resource, - const char *payload, - size_t payload_len, - time_t if_modified_since); -void directory_initiate_command_routerstatus_rend(routerstatus_t *status, - uint8_t dir_purpose, - uint8_t router_purpose, - int anonymized_connection, - const char *resource, - const char *payload, - size_t payload_len, - time_t if_modified_since, - const rend_data_t *rend_query); - -int parse_http_response(const char *headers, int *code, time_t *date, - compress_method_t *compression, char **response); - -int connection_dir_is_encrypted(dir_connection_t *conn); -int connection_dir_reached_eof(dir_connection_t *conn); -int connection_dir_process_inbuf(dir_connection_t *conn); -int connection_dir_finished_flushing(dir_connection_t *conn); -int connection_dir_finished_connecting(dir_connection_t *conn); -void connection_dir_request_failed(dir_connection_t *conn); -void directory_initiate_command(const char *address, const tor_addr_t *addr, - uint16_t or_port, uint16_t dir_port, - int supports_conditional_consensus, - int supports_begindir, const char *digest, - uint8_t dir_purpose, uint8_t router_purpose, - int anonymized_connection, - const char *resource, - const char *payload, size_t payload_len, - time_t if_modified_since); - -int dir_split_resource_into_fingerprints(const char *resource, - smartlist_t *fp_out, int *compresseed_out, - int decode_hex, int sort_uniq); /** A pair of digests created by dir_split_resource_info_fingerprint_pairs() */ typedef struct { char first[DIGEST_LEN]; char second[DIGEST_LEN]; } fp_pair_t; -int dir_split_resource_into_fingerprint_pairs(const char *res, - smartlist_t *pairs_out); -char *directory_dump_request_log(void); -void note_request(const char *key, size_t bytes); -int router_supports_extrainfo(const char *identity_digest, int is_authority); - -time_t download_status_increment_failure(download_status_t *dls, - int status_code, const char *item, - int server, time_t now); -/** Increment the failure count of the download_status_t <b>dls</b>, with - * the optional status code <b>sc</b>. */ -#define download_status_failed(dls, sc) \ - download_status_increment_failure((dls), (sc), NULL, \ - get_options()->DirPort, time(NULL)) - -void download_status_reset(download_status_t *dls); -static int download_status_is_ready(download_status_t *dls, time_t now, - int max_failures); -/** Return true iff, as of <b>now</b>, the resource tracked by <b>dls</b> is - * ready to get its download reattempted. */ -static INLINE int -download_status_is_ready(download_status_t *dls, time_t now, - int max_failures) -{ - return (dls->n_download_failures <= max_failures - && dls->next_attempt_at <= now); -} - -static void download_status_mark_impossible(download_status_t *dl); -/** Mark <b>dl</b> as never downloadable. */ -static INLINE void -download_status_mark_impossible(download_status_t *dl) -{ - dl->n_download_failures = IMPOSSIBLE_TO_DOWNLOAD; -} /********************************* dirserv.c ***************************/ -/** Maximum length of an exit policy summary. */ -#define MAX_EXITPOLICY_SUMMARY_LEN (1000) - -/** Maximum allowable length of a version line in a networkstatus. */ -#define MAX_V_LINE_LEN 128 -/** Length of "r Authority BadDirectory BadExit Exit Fast Guard HSDir Named - * Running Stable Unnamed V2Dir Valid\n". */ -#define MAX_FLAG_LINE_LEN 96 -/** Length of "w" line for weighting. Currently at most - * "w Bandwidth=<uint32t>\n" */ -#define MAX_WEIGHT_LINE_LEN (13+10) -/** Maximum length of an exit policy summary line. */ -#define MAX_POLICY_LINE_LEN (3+MAX_EXITPOLICY_SUMMARY_LEN) -/** Amount of space to allocate for each entry: r, s, and v lines. */ -#define RS_ENTRY_LEN \ - ( /* first line */ \ - MAX_NICKNAME_LEN+BASE64_DIGEST_LEN*2+ISO_TIME_LEN+INET_NTOA_BUF_LEN+ \ - 5*2 /* ports */ + 10 /* punctuation */ + \ - /* second line */ \ - MAX_FLAG_LINE_LEN + \ - /* weight line */ \ - MAX_WEIGHT_LINE_LEN + \ - /* p line. */ \ - MAX_POLICY_LINE_LEN + \ - /* v line. */ \ - MAX_V_LINE_LEN \ - ) -#define UNNAMED_ROUTER_NICKNAME "Unnamed" - -int connection_dirserv_flushed_some(dir_connection_t *conn); - -int dirserv_add_own_fingerprint(const char *nickname, crypto_pk_env_t *pk); -int dirserv_load_fingerprint_file(void); -void dirserv_free_fingerprint_list(void); -const char *dirserv_get_nickname_by_digest(const char *digest); -enum was_router_added_t dirserv_add_multiple_descriptors( - const char *desc, uint8_t purpose, - const char *source, - const char **msg); -enum was_router_added_t dirserv_add_descriptor(routerinfo_t *ri, - const char **msg, - const char *source); -int getinfo_helper_dirserv_unregistered(control_connection_t *conn, - const char *question, char **answer); -void dirserv_free_descriptors(void); -void dirserv_set_router_is_running(routerinfo_t *router, time_t now); -int list_server_status_v1(smartlist_t *routers, char **router_status_out, - int for_controller); -int dirserv_dump_directory_to_string(char **dir_out, - crypto_pk_env_t *private_key); - -int directory_fetches_from_authorities(or_options_t *options); -int directory_fetches_dir_info_early(or_options_t *options); -int directory_fetches_dir_info_later(or_options_t *options); -int directory_caches_v2_dir_info(or_options_t *options); -#define directory_caches_v1_dir_info(o) directory_caches_v2_dir_info(o) -int directory_caches_dir_info(or_options_t *options); -int directory_permits_begindir_requests(or_options_t *options); -int directory_permits_controller_requests(or_options_t *options); -int directory_too_idle_to_fetch_descriptors(or_options_t *options, time_t now); - -void directory_set_dirty(void); -cached_dir_t *dirserv_get_directory(void); -cached_dir_t *dirserv_get_runningrouters(void); -cached_dir_t *dirserv_get_consensus(void); -void dirserv_set_cached_directory(const char *directory, time_t when, - int is_running_routers); -void dirserv_set_cached_networkstatus_v2(const char *directory, - const char *identity, - time_t published); -void dirserv_set_cached_networkstatus_v3(const char *consensus, - time_t published); -void dirserv_clear_old_networkstatuses(time_t cutoff); -void dirserv_clear_old_v1_info(time_t now); -void dirserv_get_networkstatus_v2(smartlist_t *result, const char *key); -void dirserv_get_networkstatus_v2_fingerprints(smartlist_t *result, - const char *key); -int dirserv_get_routerdesc_fingerprints(smartlist_t *fps_out, const char *key, - const char **msg, - int for_unencrypted_conn, - int is_extrainfo); -int dirserv_get_routerdescs(smartlist_t *descs_out, const char *key, - const char **msg); -void dirserv_orconn_tls_done(const char *address, - uint16_t or_port, - const char *digest_rcvd, - int as_advertised); -void dirserv_test_reachability(time_t now, int try_all); -int authdir_wants_to_reject_router(routerinfo_t *ri, const char **msg, - int complain); -int dirserv_would_reject_router(routerstatus_t *rs); -int dirserv_remove_old_statuses(smartlist_t *fps, time_t cutoff); -int dirserv_have_any_serverdesc(smartlist_t *fps, int spool_src); -size_t dirserv_estimate_data_size(smartlist_t *fps, int is_serverdescs, - int compressed); -int routerstatus_format_entry(char *buf, size_t buf_len, - routerstatus_t *rs, const char *platform, - int first_line_only, int v2_format); -void dirserv_free_all(void); -void cached_dir_decref(cached_dir_t *d); -cached_dir_t *new_cached_dir(char *s, time_t published); +typedef enum { + NS_V2, NS_V3_CONSENSUS, NS_V3_VOTE, NS_CONTROL_PORT, + NS_V3_CONSENSUS_MICRODESC +} routerstatus_format_type_t; -/********************************* dirvote.c ************************/ +#ifdef DIRSERV_PRIVATE +typedef struct measured_bw_line_t { + char node_id[DIGEST_LEN]; + char node_hex[MAX_HEX_NICKNAME_LEN+1]; + long int bw; +} measured_bw_line_t; -/** Lowest allowable value for VoteSeconds. */ -#define MIN_VOTE_SECONDS 20 -/** Lowest allowable value for DistSeconds. */ -#define MIN_DIST_SECONDS 20 -/** Smallest allowable voting interval. */ -#define MIN_VOTE_INTERVAL 300 - -void dirvote_free_all(void); - -/* vote manipulation */ -char *networkstatus_compute_consensus(smartlist_t *votes, - int total_authorities, - crypto_pk_env_t *identity_key, - crypto_pk_env_t *signing_key, - const char *legacy_identity_key_digest, - crypto_pk_env_t *legacy_signing_key); -int networkstatus_add_detached_signatures(networkstatus_t *target, - ns_detached_signatures_t *sigs, - const char **msg_out); -char *networkstatus_get_detached_signatures(networkstatus_t *consensus); -void ns_detached_signatures_free(ns_detached_signatures_t *s); - -/* cert manipulation */ -authority_cert_t *authority_cert_dup(authority_cert_t *cert); +#endif + +/********************************* dirvote.c ************************/ /** Describes the schedule by which votes should be generated. */ typedef struct vote_timing_t { @@ -3546,173 +3193,85 @@ typedef struct vote_timing_t { int vote_delay; int dist_delay; } vote_timing_t; -/* vote scheduling */ -void dirvote_get_preferred_voting_intervals(vote_timing_t *timing_out); -time_t dirvote_get_start_of_next_interval(time_t now, int interval); -void dirvote_recalculate_timing(or_options_t *options, time_t now); -void dirvote_act(or_options_t *options, time_t now); - -/* invoked on timers and by outside triggers. */ -struct pending_vote_t * dirvote_add_vote(const char *vote_body, - const char **msg_out, - int *status_out); -int dirvote_add_signatures(const char *detached_signatures_body, - const char *source, - const char **msg_out); - -/* Item access */ -const char *dirvote_get_pending_consensus(void); -const char *dirvote_get_pending_detached_signatures(void); -#define DGV_BY_ID 1 -#define DGV_INCLUDE_PENDING 2 -#define DGV_INCLUDE_PREVIOUS 4 -const cached_dir_t *dirvote_get_vote(const char *fp, int flags); -void set_routerstatus_from_routerinfo(routerstatus_t *rs, - routerinfo_t *ri, time_t now, - int naming, int exits_can_be_guards, - int listbadexits, int listbaddirs); -void router_clear_status_flags(routerinfo_t *ri); -networkstatus_t * -dirserv_generate_networkstatus_vote_obj(crypto_pk_env_t *private_key, - authority_cert_t *cert); - -#ifdef DIRVOTE_PRIVATE -char *format_networkstatus_vote(crypto_pk_env_t *private_key, - networkstatus_t *v3_ns); -char *dirvote_compute_params(smartlist_t *votes); -#endif - -/********************************* dns.c ***************************/ - -int dns_init(void); -int has_dns_init_failed(void); -void dns_free_all(void); -uint32_t dns_clip_ttl(uint32_t ttl); -int dns_reset(void); -void connection_dns_remove(edge_connection_t *conn); -void assert_connection_edge_not_dns_pending(edge_connection_t *conn); -void assert_all_pending_dns_resolves_ok(void); -void dns_cancel_pending_resolve(const char *question); -int dns_resolve(edge_connection_t *exitconn); -void dns_launch_correctness_checks(void); -int dns_seems_to_be_broken(void); -void dns_reset_correctness_checks(void); - -/********************************* dnsserv.c ************************/ - -void dnsserv_configure_listener(connection_t *conn); -void dnsserv_close_listener(connection_t *conn); -void dnsserv_resolved(edge_connection_t *conn, - int answer_type, - size_t answer_len, - const char *answer, - int ttl); -void dnsserv_reject_request(edge_connection_t *conn); -int dnsserv_launch_request(const char *name, int is_reverse); /********************************* geoip.c **************************/ -#ifdef GEOIP_PRIVATE -int geoip_parse_entry(const char *line); -#endif -int should_record_bridge_info(or_options_t *options); -int geoip_load_file(const char *filename, or_options_t *options); -int geoip_get_country_by_ip(uint32_t ipaddr); -int geoip_get_n_countries(void); -const char *geoip_get_country_name(country_t num); -int geoip_is_loaded(void); -country_t geoip_get_country(const char *countrycode); +/** Round all GeoIP results to the next multiple of this value, to avoid + * leaking information. */ +#define DIR_RECORD_USAGE_GRANULARITY 8 +/** Time interval: Flush geoip data to disk this often. */ +#define DIR_ENTRY_RECORD_USAGE_RETAIN_IPS (24*60*60) +/** How long do we have to have observed per-country request history before + * we are willing to talk about it? */ +#define DIR_RECORD_USAGE_MIN_OBSERVATION_TIME (12*60*60) + /** Indicates an action that we might be noting geoip statistics on. * Note that if we're noticing CONNECT, we're a bridge, and if we're noticing * the others, we're not. */ typedef enum { - /** We've noticed a connection as a bridge relay. */ + /** We've noticed a connection as a bridge relay or entry guard. */ GEOIP_CLIENT_CONNECT = 0, /** We've served a networkstatus consensus as a directory server. */ GEOIP_CLIENT_NETWORKSTATUS = 1, /** We've served a v2 networkstatus consensus as a directory server. */ GEOIP_CLIENT_NETWORKSTATUS_V2 = 2, } geoip_client_action_t; -void geoip_note_client_seen(geoip_client_action_t action, - uint32_t addr, time_t now); -void geoip_remove_old_clients(time_t cutoff); -time_t geoip_get_history_start(void); -char *geoip_get_client_history(time_t now, geoip_client_action_t action); -char *geoip_get_request_history(time_t now, geoip_client_action_t action); -int getinfo_helper_geoip(control_connection_t *control_conn, - const char *question, char **answer); -void geoip_free_all(void); -void dump_geoip_stats(void); - -/********************************* hibernate.c **********************/ - -int accounting_parse_options(or_options_t *options, int validate_only); -int accounting_is_enabled(or_options_t *options); -void configure_accounting(time_t now); -void accounting_run_housekeeping(time_t now); -void accounting_add_bytes(size_t n_read, size_t n_written, int seconds); -int accounting_record_bandwidth_usage(time_t now, or_state_t *state); -void hibernate_begin_shutdown(void); -int we_are_hibernating(void); -void consider_hibernation(time_t now); -int getinfo_helper_accounting(control_connection_t *conn, - const char *question, char **answer); -void accounting_set_bandwidth_usage_from_state(or_state_t *state); - -/********************************* main.c ***************************/ - -extern int has_completed_circuit; - -int connection_add(connection_t *conn); -int connection_remove(connection_t *conn); -int connection_in_array(connection_t *conn); -void add_connection_to_closeable_list(connection_t *conn); -int connection_is_on_closeable_list(connection_t *conn); - -smartlist_t *get_connection_array(void); - -void connection_watch_events(connection_t *conn, short events); -int connection_is_reading(connection_t *conn); -void connection_stop_reading(connection_t *conn); -void connection_start_reading(connection_t *conn); - -int connection_is_writing(connection_t *conn); -void connection_stop_writing(connection_t *conn); -void connection_start_writing(connection_t *conn); - -void connection_stop_reading_from_linked_conn(connection_t *conn); - -void directory_all_unreachable(time_t now); -void directory_info_has_arrived(time_t now, int from_cache); - -void ip_address_changed(int at_interface); -void dns_servers_relaunch_checks(void); - -void control_signal_act(int the_signal); -void handle_signals(int is_parent); - -int try_locking(or_options_t *options, int err_if_locked); -int have_lockfile(void); -void release_lockfile(void); - -void tor_cleanup(void); -void tor_free_all(int postfork); - -int tor_main(int argc, char *argv[]); - -#ifdef MAIN_PRIVATE -int do_main_loop(void); -int do_list_fingerprint(void); -void do_hash_password(void); -int tor_init(int argc, char **argv); -#endif +/** Indicates either a positive reply or a reason for rejectng a network + * status request that will be included in geoip statistics. */ +typedef enum { + /** Request is answered successfully. */ + GEOIP_SUCCESS = 0, + /** V3 network status is not signed by a sufficient number of requested + * authorities. */ + GEOIP_REJECT_NOT_ENOUGH_SIGS = 1, + /** Requested network status object is unavailable. */ + GEOIP_REJECT_UNAVAILABLE = 2, + /** Requested network status not found. */ + GEOIP_REJECT_NOT_FOUND = 3, + /** Network status has not been modified since If-Modified-Since time. */ + GEOIP_REJECT_NOT_MODIFIED = 4, + /** Directory is busy. */ + GEOIP_REJECT_BUSY = 5, +} geoip_ns_response_t; +#define GEOIP_NS_RESPONSE_NUM 6 + +/** Directory requests that we are measuring can be either direct or + * tunneled. */ +typedef enum { + DIRREQ_DIRECT = 0, + DIRREQ_TUNNELED = 1, +} dirreq_type_t; -/********************************* networkstatus.c *********************/ +/** Possible states for either direct or tunneled directory requests that + * are relevant for determining network status download times. */ +typedef enum { + /** Found that the client requests a network status; applies to both + * direct and tunneled requests; initial state of a request that we are + * measuring. */ + DIRREQ_IS_FOR_NETWORK_STATUS = 0, + /** Finished writing a network status to the directory connection; + * applies to both direct and tunneled requests; completes a direct + * request. */ + DIRREQ_FLUSHING_DIR_CONN_FINISHED = 1, + /** END cell sent to circuit that initiated a tunneled request. */ + DIRREQ_END_CELL_SENT = 2, + /** Flushed last cell from queue of the circuit that initiated a + * tunneled request to the outbuf of the OR connection. */ + DIRREQ_CIRC_QUEUE_FLUSHED = 3, + /** Flushed last byte from buffer of the OR connection belonging to the + * circuit that initiated a tunneled request; completes a tunneled + * request. */ + DIRREQ_OR_CONN_BUFFER_FLUSHED = 4 +} dirreq_state_t; + +#define WRITE_STATS_INTERVAL (24*60*60) + +/********************************* microdesc.c *************************/ + +typedef struct microdesc_cache_t microdesc_cache_t; -/** How old do we allow a v2 network-status to get before removing it - * completely? */ -#define MAX_NETWORKSTATUS_AGE (10*24*60*60) +/********************************* networkstatus.c *********************/ /** Location where we found a v2 networkstatus. */ typedef enum { @@ -3733,127 +3292,8 @@ typedef enum version_status_t { VS_UNKNOWN, /**< We have no idea. */ } version_status_t; -void networkstatus_reset_warnings(void); -void networkstatus_reset_download_failures(void); -int router_reload_v2_networkstatus(void); -int router_reload_consensus_networkstatus(void); -void routerstatus_free(routerstatus_t *rs); -void networkstatus_v2_free(networkstatus_v2_t *ns); -void networkstatus_vote_free(networkstatus_t *ns); -networkstatus_voter_info_t *networkstatus_get_voter_by_id( - networkstatus_t *vote, - const char *identity); -int networkstatus_check_consensus_signature(networkstatus_t *consensus, - int warn); -int networkstatus_check_voter_signature(networkstatus_t *consensus, - networkstatus_voter_info_t *voter, - authority_cert_t *cert); -char *networkstatus_get_cache_filename(const char *identity_digest); -int router_set_networkstatus_v2(const char *s, time_t arrived_at, - v2_networkstatus_source_t source, - smartlist_t *requested_fingerprints); -void networkstatus_v2_list_clean(time_t now); -routerstatus_t *networkstatus_v2_find_entry(networkstatus_v2_t *ns, - const char *digest); -routerstatus_t *networkstatus_vote_find_entry(networkstatus_t *ns, - const char *digest); -int networkstatus_vote_find_entry_idx(networkstatus_t *ns, - const char *digest, int *found_out); -const smartlist_t *networkstatus_get_v2_list(void); -download_status_t *router_get_dl_status_by_descriptor_digest(const char *d); -routerstatus_t *router_get_consensus_status_by_id(const char *digest); -routerstatus_t *router_get_consensus_status_by_descriptor_digest( - const char *digest); -routerstatus_t *router_get_consensus_status_by_nickname(const char *nickname, - int warn_if_unnamed); -const char *networkstatus_get_router_digest_by_nickname(const char *nickname); -int networkstatus_nickname_is_unnamed(const char *nickname); -void networkstatus_consensus_download_failed(int status_code); -void update_consensus_networkstatus_fetch_time(time_t now); -int should_delay_dir_fetches(or_options_t *options); -void update_networkstatus_downloads(time_t now); -void update_certificate_downloads(time_t now); -int consensus_is_waiting_for_certs(void); -networkstatus_v2_t *networkstatus_v2_get_by_digest(const char *digest); -networkstatus_t *networkstatus_get_latest_consensus(void); -networkstatus_t *networkstatus_get_live_consensus(time_t now); -networkstatus_t *networkstatus_get_reasonably_live_consensus(time_t now); -#define NSSET_FROM_CACHE 1 -#define NSSET_WAS_WAITING_FOR_CERTS 2 -#define NSSET_DONT_DOWNLOAD_CERTS 4 -#define NSSET_ACCEPT_OBSOLETE 8 -int networkstatus_set_current_consensus(const char *consensus, unsigned flags); -void networkstatus_note_certs_arrived(void); -void routers_update_all_from_networkstatus(time_t now, int dir_version); -void routerstatus_list_update_from_consensus_networkstatus(time_t now); -void routers_update_status_from_consensus_networkstatus(smartlist_t *routers, - int reset_failures); -void signed_descs_update_status_from_consensus_networkstatus( - smartlist_t *descs); - -char *networkstatus_getinfo_helper_single(routerstatus_t *rs); -char *networkstatus_getinfo_by_purpose(const char *purpose_string, time_t now); -void networkstatus_dump_bridge_status_to_file(time_t now); -int32_t networkstatus_get_param(networkstatus_t *ns, const char *param_name, - int32_t default_val); -int getinfo_helper_networkstatus(control_connection_t *conn, - const char *question, char **answer); -void networkstatus_free_all(void); - -/********************************* ntmain.c ***************************/ -#ifdef MS_WINDOWS -#define NT_SERVICE -#endif - -#ifdef NT_SERVICE -int nt_service_parse_options(int argc, char **argv, int *should_exit); -int nt_service_is_stopping(void); -void nt_service_set_state(DWORD state); -#else -#define nt_service_is_stopping() (0) -#endif - -/********************************* onion.c ***************************/ - -int onion_pending_add(or_circuit_t *circ, char *onionskin); -or_circuit_t *onion_next_task(char **onionskin_out); -void onion_pending_remove(or_circuit_t *circ); - -int onion_skin_create(crypto_pk_env_t *router_key, - crypto_dh_env_t **handshake_state_out, - char *onion_skin_out); - -int onion_skin_server_handshake(const char *onion_skin, - crypto_pk_env_t *private_key, - crypto_pk_env_t *prev_private_key, - char *handshake_reply_out, - char *key_out, - size_t key_out_len); - -int onion_skin_client_handshake(crypto_dh_env_t *handshake_state, - const char *handshake_reply, - char *key_out, - size_t key_out_len); - -int fast_server_handshake(const char *key_in, - char *handshake_reply_out, - char *key_out, - size_t key_out_len); - -int fast_client_handshake(const char *handshake_state, - const char *handshake_reply_out, - char *key_out, - size_t key_out_len); - -void clear_pending_onions(void); - /********************************* policies.c ************************/ -/* (length of "accept 255.255.255.255/255.255.255.255:65535-65535\n" plus a - * NUL.) - */ -#define POLICY_BUF_LEN 52 - /** Outcome of applying an address policy to an address. */ typedef enum { /** The address was accepted */ @@ -3868,146 +3308,8 @@ typedef enum { ADDR_POLICY_PROBABLY_REJECTED=2 } addr_policy_result_t; -int firewall_is_fascist_or(void); -int fascist_firewall_allows_address_or(const tor_addr_t *addr, uint16_t port); -int fascist_firewall_allows_or(routerinfo_t *ri); -int fascist_firewall_allows_address_dir(const tor_addr_t *addr, uint16_t port); -int dir_policy_permits_address(const tor_addr_t *addr); -int socks_policy_permits_address(const tor_addr_t *addr); -int authdir_policy_permits_address(uint32_t addr, uint16_t port); -int authdir_policy_valid_address(uint32_t addr, uint16_t port); -int authdir_policy_baddir_address(uint32_t addr, uint16_t port); -int authdir_policy_badexit_address(uint32_t addr, uint16_t port); - -int validate_addr_policies(or_options_t *options, char **msg); -void policy_expand_private(smartlist_t **policy); -int policies_parse_from_options(or_options_t *options); - -addr_policy_t *addr_policy_get_canonical_entry(addr_policy_t *ent); -int cmp_addr_policies(smartlist_t *a, smartlist_t *b); -addr_policy_result_t compare_tor_addr_to_addr_policy(const tor_addr_t *addr, - uint16_t port, const smartlist_t *policy); -addr_policy_result_t compare_addr_to_addr_policy(uint32_t addr, - uint16_t port, const smartlist_t *policy); -int policies_parse_exit_policy(config_line_t *cfg, smartlist_t **dest, - int rejectprivate, const char *local_address); -void policies_set_router_exitpolicy_to_reject_all(routerinfo_t *exitrouter); -int exit_policy_is_general_exit(smartlist_t *policy); -int policy_is_reject_star(const smartlist_t *policy); -int getinfo_helper_policies(control_connection_t *conn, - const char *question, char **answer); -int policy_write_item(char *buf, size_t buflen, addr_policy_t *item, - int format_for_desc); - -void addr_policy_list_free(smartlist_t *p); -void addr_policy_free(addr_policy_t *p); -void policies_free_all(void); - -char *policy_summarize(smartlist_t *policy); - -/********************************* reasons.c ***************************/ - -const char *stream_end_reason_to_control_string(int reason); -const char *stream_end_reason_to_string(int reason); -socks5_reply_status_t stream_end_reason_to_socks5_response(int reason); -uint8_t errno_to_stream_end_reason(int e); - -const char *orconn_end_reason_to_control_string(int r); -int tls_error_to_orconn_end_reason(int e); -int errno_to_orconn_end_reason(int e); - -const char *circuit_end_reason_to_control_string(int reason); - -/********************************* relay.c ***************************/ - -extern uint64_t stats_n_relay_cells_relayed; -extern uint64_t stats_n_relay_cells_delivered; - -int circuit_receive_relay_cell(cell_t *cell, circuit_t *circ, - cell_direction_t cell_direction); - -void relay_header_pack(char *dest, const relay_header_t *src); -void relay_header_unpack(relay_header_t *dest, const char *src); -int relay_send_command_from_edge(streamid_t stream_id, circuit_t *circ, - uint8_t relay_command, const char *payload, - size_t payload_len, crypt_path_t *cpath_layer); -int connection_edge_send_command(edge_connection_t *fromconn, - uint8_t relay_command, const char *payload, - size_t payload_len); -int connection_edge_package_raw_inbuf(edge_connection_t *conn, - int package_partial); -void connection_edge_consider_sending_sendme(edge_connection_t *conn); - -extern uint64_t stats_n_data_cells_packaged; -extern uint64_t stats_n_data_bytes_packaged; -extern uint64_t stats_n_data_cells_received; -extern uint64_t stats_n_data_bytes_received; - -void init_cell_pool(void); -void free_cell_pool(void); -void clean_cell_pool(void); -void dump_cell_pool_usage(int severity); - -void cell_queue_clear(cell_queue_t *queue); -void cell_queue_append(cell_queue_t *queue, packed_cell_t *cell); -void cell_queue_append_packed_copy(cell_queue_t *queue, const cell_t *cell); - -void append_cell_to_circuit_queue(circuit_t *circ, or_connection_t *orconn, - cell_t *cell, cell_direction_t direction); -void connection_or_unlink_all_active_circs(or_connection_t *conn); -int connection_or_flush_from_first_active_circuit(or_connection_t *conn, - int max, time_t now); -void assert_active_circuits_ok(or_connection_t *orconn); -void make_circuit_inactive_on_conn(circuit_t *circ, or_connection_t *conn); -void make_circuit_active_on_conn(circuit_t *circ, or_connection_t *conn); - -int append_address_to_payload(char *payload_out, const tor_addr_t *addr); -const char *decode_address_from_payload(tor_addr_t *addr_out, - const char *payload, - int payload_len); - /********************************* rephist.c ***************************/ -void rep_hist_init(void); -void rep_hist_note_connect_failed(const char* nickname, time_t when); -void rep_hist_note_connect_succeeded(const char* nickname, time_t when); -void rep_hist_note_disconnect(const char* nickname, time_t when); -void rep_hist_note_connection_died(const char* nickname, time_t when); -void rep_hist_note_extend_succeeded(const char *from_name, - const char *to_name); -void rep_hist_note_extend_failed(const char *from_name, const char *to_name); -void rep_hist_dump_stats(time_t now, int severity); -void rep_hist_note_bytes_read(size_t num_bytes, time_t when); -void rep_hist_note_bytes_written(size_t num_bytes, time_t when); -int rep_hist_bandwidth_assess(void); -char *rep_hist_get_bandwidth_lines(int for_extrainfo); -void rep_hist_update_state(or_state_t *state); -int rep_hist_load_state(or_state_t *state, char **err); -void rep_history_clean(time_t before); - -void rep_hist_note_router_reachable(const char *id, time_t when); -void rep_hist_note_router_unreachable(const char *id, time_t when); -int rep_hist_record_mtbf_data(time_t now, int missing_means_down); -int rep_hist_load_mtbf_data(time_t now); - -time_t rep_hist_downrate_old_runs(time_t now); -double rep_hist_get_stability(const char *id, time_t when); -double rep_hist_get_weighted_fractional_uptime(const char *id, time_t when); -long rep_hist_get_weighted_time_known(const char *id, time_t when); -int rep_hist_have_measured_enough_stability(void); -const char *rep_hist_get_router_stability_doc(time_t now); - -void rep_hist_note_used_port(time_t now, uint16_t port); -smartlist_t *rep_hist_get_predicted_ports(time_t now); -void rep_hist_note_used_resolve(time_t now); -void rep_hist_note_used_internal(time_t now, int need_uptime, - int need_capacity); -int rep_hist_get_predicted_internal(time_t now, int *need_uptime, - int *need_capacity); - -int any_predicted_circuits(time_t now); -int rep_hist_circbuilding_dormant(time_t now); - /** Possible public/private key operations in Tor: used to keep track of where * we're spending our time. */ typedef enum { @@ -4017,45 +3319,6 @@ typedef enum { TLS_HANDSHAKE_C, TLS_HANDSHAKE_S, REND_CLIENT, REND_MID, REND_SERVER, } pk_op_t; -void note_crypto_pk_op(pk_op_t operation); -void dump_pk_ops(int severity); - -void rep_hist_free_all(void); - -/* for hidden service usage statistics */ -void hs_usage_note_publish_total(const char *service_id, time_t now); -void hs_usage_note_publish_novel(const char *service_id, time_t now); -void hs_usage_note_fetch_total(const char *service_id, time_t now); -void hs_usage_note_fetch_successful(const char *service_id, time_t now); -void hs_usage_write_statistics_to_file(time_t now); -void hs_usage_free_all(void); - -/********************************* rendclient.c ***************************/ - -void rend_client_introcirc_has_opened(origin_circuit_t *circ); -void rend_client_rendcirc_has_opened(origin_circuit_t *circ); -int rend_client_introduction_acked(origin_circuit_t *circ, const char *request, - size_t request_len); -void rend_client_refetch_renddesc(const char *query); -void rend_client_refetch_v2_renddesc(const rend_data_t *rend_query); -int rend_client_remove_intro_point(extend_info_t *failed_intro, - const rend_data_t *rend_query); -int rend_client_rendezvous_acked(origin_circuit_t *circ, const char *request, - size_t request_len); -int rend_client_receive_rendezvous(origin_circuit_t *circ, const char *request, - size_t request_len); -void rend_client_desc_trynow(const char *query, int rend_version); - -extend_info_t *rend_client_get_random_intro(const rend_data_t *rend_query); - -int rend_client_send_introduction(origin_circuit_t *introcirc, - origin_circuit_t *rendcirc); -int rend_parse_service_authorization(or_options_t *options, - int validate_only); -rend_service_authorization_t *rend_client_lookup_service_authorization( - const char *onion_address); -void rend_service_authorization_free_all(void); -rend_data_t *rend_data_dup(const rend_data_t *request); /********************************* rendcommon.c ***************************/ @@ -4098,30 +3361,6 @@ typedef struct rend_service_descriptor_t { smartlist_t *successful_uploads; } rend_service_descriptor_t; -/** Free all storage associated with <b>data</b> */ -static INLINE void -rend_data_free(rend_data_t *data) -{ - tor_free(data); -} - -int rend_cmp_service_ids(const char *one, const char *two); - -void rend_process_relay_cell(circuit_t *circ, const crypt_path_t *layer_hint, - int command, size_t length, const char *payload); - -void rend_service_descriptor_free(rend_service_descriptor_t *desc); -int rend_encode_service_descriptor(rend_service_descriptor_t *desc, - crypto_pk_env_t *key, - char **str_out, - size_t *len_out); -rend_service_descriptor_t *rend_parse_service_descriptor(const char *str, - size_t len); -int rend_get_service_id(crypto_pk_env_t *pk, char *out); -void rend_encoded_v2_service_descriptor_free( - rend_encoded_v2_service_descriptor_t *desc); -void rend_intro_point_free(rend_intro_point_t *intro); - /** A cached rendezvous descriptor. */ typedef struct rend_cache_entry_t { size_t len; /**< Length of <b>desc</b> */ @@ -4130,150 +3369,6 @@ typedef struct rend_cache_entry_t { rend_service_descriptor_t *parsed; /**< Parsed value of 'desc' */ } rend_cache_entry_t; -void rend_cache_init(void); -void rend_cache_clean(void); -void rend_cache_clean_v2_descs_as_dir(void); -void rend_cache_free_all(void); -int rend_valid_service_id(const char *query); -int rend_cache_lookup_desc(const char *query, int version, const char **desc, - size_t *desc_len); -int rend_cache_lookup_entry(const char *query, int version, - rend_cache_entry_t **entry_out); -int rend_cache_lookup_v2_desc_as_dir(const char *query, const char **desc); -int rend_cache_store(const char *desc, size_t desc_len, int published); -int rend_cache_store_v2_desc_as_client(const char *desc, - const rend_data_t *rend_query); -int rend_cache_store_v2_desc_as_dir(const char *desc); -int rend_cache_size(void); -int rend_encode_v2_descriptors(smartlist_t *descs_out, - rend_service_descriptor_t *desc, time_t now, - uint8_t period, rend_auth_type_t auth_type, - crypto_pk_env_t *client_key, - smartlist_t *client_cookies); -int rend_compute_v2_desc_id(char *desc_id_out, const char *service_id, - const char *descriptor_cookie, - time_t now, uint8_t replica); -int rend_id_is_in_interval(const char *a, const char *b, const char *c); -void rend_get_descriptor_id_bytes(char *descriptor_id_out, - const char *service_id, - const char *secret_id_part); - -/********************************* rendservice.c ***************************/ - -int num_rend_services(void); -int rend_config_services(or_options_t *options, int validate_only); -int rend_service_load_keys(void); -void rend_services_init(void); -void rend_services_introduce(void); -void rend_consider_services_upload(time_t now); -void rend_hsdir_routers_changed(void); -void rend_consider_descriptor_republication(void); - -void rend_service_intro_has_opened(origin_circuit_t *circuit); -int rend_service_intro_established(origin_circuit_t *circuit, - const char *request, - size_t request_len); -void rend_service_rendezvous_has_opened(origin_circuit_t *circuit); -int rend_service_introduce(origin_circuit_t *circuit, const char *request, - size_t request_len); -void rend_service_relaunch_rendezvous(origin_circuit_t *oldcirc); -int rend_service_set_connection_addr_port(edge_connection_t *conn, - origin_circuit_t *circ); -void rend_service_dump_stats(int severity); -void rend_service_free_all(void); - -/********************************* rendmid.c *******************************/ -int rend_mid_establish_intro(or_circuit_t *circ, const char *request, - size_t request_len); -int rend_mid_introduce(or_circuit_t *circ, const char *request, - size_t request_len); -int rend_mid_establish_rendezvous(or_circuit_t *circ, const char *request, - size_t request_len); -int rend_mid_rendezvous(or_circuit_t *circ, const char *request, - size_t request_len); - -/********************************* router.c ***************************/ - -crypto_pk_env_t *get_onion_key(void); -time_t get_onion_key_set_at(void); -void set_identity_key(crypto_pk_env_t *k); -crypto_pk_env_t *get_identity_key(void); -int identity_key_is_set(void); -authority_cert_t *get_my_v3_authority_cert(void); -crypto_pk_env_t *get_my_v3_authority_signing_key(void); -authority_cert_t *get_my_v3_legacy_cert(void); -crypto_pk_env_t *get_my_v3_legacy_signing_key(void); -void dup_onion_keys(crypto_pk_env_t **key, crypto_pk_env_t **last); -void rotate_onion_key(void); -crypto_pk_env_t *init_key_from_file(const char *fname, int generate, - int severity); -void v3_authority_check_key_expiry(void); - -int init_keys(void); - -int check_whether_orport_reachable(void); -int check_whether_dirport_reachable(void); -void consider_testing_reachability(int test_or, int test_dir); -void router_orport_found_reachable(void); -void router_dirport_found_reachable(void); -void router_perform_bandwidth_test(int num_circs, time_t now); - -int authdir_mode(or_options_t *options); -int authdir_mode_v1(or_options_t *options); -int authdir_mode_v2(or_options_t *options); -int authdir_mode_v3(or_options_t *options); -int authdir_mode_any_main(or_options_t *options); -int authdir_mode_any_nonhidserv(or_options_t *options); -int authdir_mode_handles_descs(or_options_t *options, int purpose); -int authdir_mode_publishes_statuses(or_options_t *options); -int authdir_mode_tests_reachability(or_options_t *options); -int authdir_mode_bridge(or_options_t *options); - -int server_mode(or_options_t *options); -int advertised_server_mode(void); -int proxy_mode(or_options_t *options); -void consider_publishable_server(int force); - -void router_upload_dir_desc_to_dirservers(int force); -void mark_my_descriptor_dirty_if_older_than(time_t when); -void mark_my_descriptor_dirty(void); -void check_descriptor_bandwidth_changed(time_t now); -void check_descriptor_ipaddress_changed(time_t now); -void router_new_address_suggestion(const char *suggestion, - const dir_connection_t *d_conn); -int router_compare_to_my_exit_policy(edge_connection_t *conn); -routerinfo_t *router_get_my_routerinfo(void); -extrainfo_t *router_get_my_extrainfo(void); -const char *router_get_my_descriptor(void); -int router_digest_is_me(const char *digest); -int router_extrainfo_digest_is_me(const char *digest); -int router_is_me(routerinfo_t *router); -int router_fingerprint_is_me(const char *fp); -int router_pick_published_address(or_options_t *options, uint32_t *addr); -int router_rebuild_descriptor(int force); -int router_dump_router_to_string(char *s, size_t maxlen, routerinfo_t *router, - crypto_pk_env_t *ident_key); -int extrainfo_dump_to_string(char *s, size_t maxlen, extrainfo_t *extrainfo, - crypto_pk_env_t *ident_key); -char *extrainfo_get_client_geoip_summary(time_t); -int is_legal_nickname(const char *s); -int is_legal_nickname_or_hexdigest(const char *s); -int is_legal_hexdigest(const char *s); -void router_get_verbose_nickname(char *buf, const routerinfo_t *router); -void routerstatus_get_verbose_nickname(char *buf, - const routerstatus_t *router); -void router_reset_warnings(void); -void router_reset_reachability(void); -void router_free_all(void); - -const char *router_purpose_to_string(uint8_t p); -uint8_t router_purpose_from_string(const char *s); - -#ifdef ROUTER_PRIVATE -/* Used only by router.c and test.c */ -void get_platform_str(char *platform, size_t len); -#endif - /********************************* routerlist.c ***************************/ /** Represents information about a single trusted directory server. */ @@ -4313,21 +3408,6 @@ typedef struct trusted_dir_server_t { #define ROUTER_MAX_DECLARED_BANDWIDTH INT32_MAX -int get_n_authorities(authority_type_t type); -int trusted_dirs_reload_certs(void); -int trusted_dirs_load_certs_from_string(const char *contents, int from_store, - int flush); -void trusted_dirs_flush_certs_to_disk(void); -authority_cert_t *authority_cert_get_newest_by_id(const char *id_digest); -authority_cert_t *authority_cert_get_by_sk_digest(const char *sk_digest); -authority_cert_t *authority_cert_get_by_digests(const char *id_digest, - const char *sk_digest); -void authority_cert_get_all(smartlist_t *certs_out); -void authority_cert_dl_failed(const char *id_digest, int status); -void authority_certs_fetch_missing(networkstatus_t *status, time_t now); -int router_reload_router_list(void); -smartlist_t *router_get_trusted_dir_servers(void); - /* Flags for pick_directory_server and pick_trusteddirserver. */ /** Flag to indicate that we should not automatically be willing to use * ourself to answer a directory request. @@ -4357,34 +3437,13 @@ smartlist_t *router_get_trusted_dir_servers(void); */ #define PDS_NO_EXISTING_SERVERDESC_FETCH (1<<3) #define _PDS_PREFER_TUNNELED_DIR_CONNS (1<<16) -routerstatus_t *router_pick_directory_server(authority_type_t type, int flags); -trusted_dir_server_t *router_get_trusteddirserver_by_digest(const char *d); -trusted_dir_server_t *trusteddirserver_get_by_v3_auth_digest(const char *d); -routerstatus_t *router_pick_trusteddirserver(authority_type_t type, int flags); -int router_get_my_share_of_directory_requests(double *v2_share_out, - double *v3_share_out); -void router_reset_status_download_failures(void); -void routerlist_add_family(smartlist_t *sl, routerinfo_t *router); -int routers_in_same_family(routerinfo_t *r1, routerinfo_t *r2); -void add_nickname_list_to_smartlist(smartlist_t *sl, const char *list, - int must_be_running); -int router_nickname_is_in_list(routerinfo_t *router, const char *list); -routerinfo_t *routerlist_find_my_routerinfo(void); -routerinfo_t *router_find_exact_exit_enclave(const char *address, - uint16_t port); -int router_is_unreliable(routerinfo_t *router, int need_uptime, - int need_capacity, int need_guard); -uint32_t router_get_advertised_bandwidth(routerinfo_t *router); -uint32_t router_get_advertised_bandwidth_capped(routerinfo_t *router); /** Possible ways to weight routers when choosing one randomly. See * routerlist_sl_choose_by_bandwidth() for more information.*/ -typedef enum { - NO_WEIGHTING, WEIGHT_FOR_EXIT, WEIGHT_FOR_GUARD +typedef enum bandwidth_weight_rule_t { + NO_WEIGHTING, WEIGHT_FOR_EXIT, WEIGHT_FOR_MID, WEIGHT_FOR_GUARD, + WEIGHT_FOR_DIR } bandwidth_weight_rule_t; -routerinfo_t *routerlist_sl_choose_by_bandwidth(smartlist_t *sl, - bandwidth_weight_rule_t rule); -routerstatus_t *routerstatus_sl_choose_by_bandwidth(smartlist_t *sl); /** Flags to be passed to control router_choose_random_node() to indicate what * kind of nodes to pick according to what algorithm. */ @@ -4394,44 +3453,9 @@ typedef enum { CRN_NEED_GUARD = 1<<2, CRN_ALLOW_INVALID = 1<<3, /* XXXX not used, apparently. */ - CRN_STRICT_PREFERRED = 1<<4, - /* XXXX not used, apparently. */ CRN_WEIGHT_AS_EXIT = 1<<5 } router_crn_flags_t; -routerinfo_t *router_choose_random_node(const char *preferred, - smartlist_t *excludedsmartlist, - struct routerset_t *excludedset, - router_crn_flags_t flags); - -routerinfo_t *router_get_by_nickname(const char *nickname, - int warn_if_unnamed); -int router_digest_version_as_new_as(const char *digest, const char *cutoff); -int router_digest_is_trusted_dir_type(const char *digest, - authority_type_t type); -#define router_digest_is_trusted_dir(d) \ - router_digest_is_trusted_dir_type((d), NO_AUTHORITY) - -int router_addr_is_trusted_dir(uint32_t addr); -int hexdigest_to_digest(const char *hexdigest, char *digest); -routerinfo_t *router_get_by_hexdigest(const char *hexdigest); -routerinfo_t *router_get_by_digest(const char *digest); -signed_descriptor_t *router_get_by_descriptor_digest(const char *digest); -signed_descriptor_t *router_get_by_extrainfo_digest(const char *digest); -signed_descriptor_t *extrainfo_get_by_descriptor_digest(const char *digest); -const char *signed_descriptor_get_body(signed_descriptor_t *desc); -const char *signed_descriptor_get_annotations(signed_descriptor_t *desc); -routerlist_t *router_get_routerlist(void); -void routerinfo_free(routerinfo_t *router); -void extrainfo_free(extrainfo_t *extrainfo); -void routerlist_free(routerlist_t *rl); -void dump_routerlist_mem_usage(int severity); -void routerlist_remove(routerlist_t *rl, routerinfo_t *ri, int make_old, - time_t now); -void routerlist_free_all(void); -void routerlist_reset_warnings(void); -void router_set_status(const char *digest, int up); - /** Return value for router_add_to_routerlist() and dirserv_add_descriptor() */ typedef enum was_router_added_t { ROUTER_ADDED_SUCCESSFULLY = 1, @@ -4443,109 +3467,6 @@ typedef enum was_router_added_t { ROUTER_AUTHDIR_REJECTS = -5, } was_router_added_t; -static int WRA_WAS_ADDED(was_router_added_t s); -static int WRA_WAS_OUTDATED(was_router_added_t s); -static int WRA_WAS_REJECTED(was_router_added_t s); -/** Return true iff the descriptor was added. It might still be necessary to - * check whether the descriptor generator should be notified. - */ -static INLINE int -WRA_WAS_ADDED(was_router_added_t s) { - return s == ROUTER_ADDED_SUCCESSFULLY || s == ROUTER_ADDED_NOTIFY_GENERATOR; -} -/** Return true iff the descriptor was not added because it was either: - * - not in the consensus - * - neither in the consensus nor in any networkstatus document - * - it was outdated. - */ -static INLINE int WRA_WAS_OUTDATED(was_router_added_t s) -{ - return (s == ROUTER_WAS_NOT_NEW || - s == ROUTER_NOT_IN_CONSENSUS || - s == ROUTER_NOT_IN_CONSENSUS_OR_NETWORKSTATUS); -} -/** Return true iff the descriptor rejected because it was malformed. */ -static INLINE int WRA_WAS_REJECTED(was_router_added_t s) -{ - return (s == ROUTER_AUTHDIR_REJECTS); -} -was_router_added_t router_add_to_routerlist(routerinfo_t *router, - const char **msg, - int from_cache, - int from_fetch); -was_router_added_t router_add_extrainfo_to_routerlist( - extrainfo_t *ei, const char **msg, - int from_cache, int from_fetch); -void routerlist_remove_old_routers(void); -int router_load_single_router(const char *s, uint8_t purpose, int cache, - const char **msg); -int router_load_routers_from_string(const char *s, const char *eos, - saved_location_t saved_location, - smartlist_t *requested_fingerprints, - int descriptor_digests, - const char *prepend_annotations); -void router_load_extrainfo_from_string(const char *s, const char *eos, - saved_location_t saved_location, - smartlist_t *requested_fingerprints, - int descriptor_digests); -void routerlist_retry_directory_downloads(time_t now); -int router_exit_policy_all_routers_reject(uint32_t addr, uint16_t port, - int need_uptime); -int router_exit_policy_rejects_all(routerinfo_t *router); -trusted_dir_server_t *add_trusted_dir_server(const char *nickname, - const char *address, - uint16_t dir_port, uint16_t or_port, - const char *digest, const char *v3_auth_digest, - authority_type_t type); -void authority_cert_free(authority_cert_t *cert); -void clear_trusted_dir_servers(void); -int any_trusted_dir_is_v1_authority(void); -void update_router_descriptor_downloads(time_t now); -void update_extrainfo_downloads(time_t now); -int router_have_minimum_dir_info(void); -void router_dir_info_changed(void); -const char *get_dir_info_status_string(void); -int count_loading_descriptors_progress(void); -void router_reset_descriptor_download_failures(void); -int router_differences_are_cosmetic(routerinfo_t *r1, routerinfo_t *r2); -int routerinfo_incompatible_with_extrainfo(routerinfo_t *ri, extrainfo_t *ei, - signed_descriptor_t *sd, - const char **msg); -void routerlist_assert_ok(routerlist_t *rl); -const char *esc_router_info(routerinfo_t *router); -void routers_sort_by_identity(smartlist_t *routers); - -routerset_t *routerset_new(void); -int routerset_parse(routerset_t *target, const char *s, - const char *description); -void routerset_union(routerset_t *target, const routerset_t *source); -int routerset_is_list(const routerset_t *set); -int routerset_needs_geoip(const routerset_t *set); -int routerset_contains_router(const routerset_t *set, routerinfo_t *ri); -int routerset_contains_routerstatus(const routerset_t *set, - routerstatus_t *rs); -int routerset_contains_extendinfo(const routerset_t *set, - const extend_info_t *ei); -void routerset_get_all_routers(smartlist_t *out, const routerset_t *routerset, - int running_only); -void routersets_get_disjunction(smartlist_t *target, const smartlist_t *source, - const routerset_t *include, - const routerset_t *exclude, int running_only); -void routerset_subtract_routers(smartlist_t *out, - const routerset_t *routerset); -char *routerset_to_string(const routerset_t *routerset); -void routerset_refresh_countries(routerset_t *target); -int routerset_equal(const routerset_t *old, const routerset_t *new); -void routerset_free(routerset_t *routerset); -void routerinfo_set_country(routerinfo_t *ri); -void routerlist_refresh_countries(void); -void refresh_all_country_info(void); - -int hid_serv_get_responsible_directories(smartlist_t *responsible_dirs, - const char *id); -int hid_serv_acting_as_directory(void); -int hid_serv_responsible_for_desc_id(const char *id); - /********************************* routerparse.c ************************/ #define MAX_STATUS_TAG_LEN 32 @@ -4564,71 +3485,10 @@ typedef struct tor_version_t { int patchlevel; char status_tag[MAX_STATUS_TAG_LEN]; int svn_revision; -} tor_version_t; -int router_get_router_hash(const char *s, size_t s_len, char *digest); -int router_get_dir_hash(const char *s, char *digest); -int router_get_runningrouters_hash(const char *s, char *digest); -int router_get_networkstatus_v2_hash(const char *s, char *digest); -int router_get_networkstatus_v3_hash(const char *s, char *digest); -int router_get_extrainfo_hash(const char *s, char *digest); -int router_append_dirobj_signature(char *buf, size_t buf_len, - const char *digest, - crypto_pk_env_t *private_key); -int router_parse_list_from_string(const char **s, const char *eos, - smartlist_t *dest, - saved_location_t saved_location, - int is_extrainfo, - int allow_annotations, - const char *prepend_annotations); -int router_parse_routerlist_from_directory(const char *s, - routerlist_t **dest, - crypto_pk_env_t *pkey, - int check_version, - int write_to_cache); -int router_parse_runningrouters(const char *str); -int router_parse_directory(const char *str); -routerinfo_t *router_parse_entry_from_string(const char *s, const char *end, - int cache_copy, - int allow_annotations, - const char *prepend_annotations); -extrainfo_t *extrainfo_parse_entry_from_string(const char *s, const char *end, - int cache_copy, struct digest_ri_map_t *routermap); -addr_policy_t *router_parse_addr_policy_item_from_string(const char *s, - int assume_action); -version_status_t tor_version_is_obsolete(const char *myversion, - const char *versionlist); -int tor_version_parse(const char *s, tor_version_t *out); -int tor_version_as_new_as(const char *platform, const char *cutoff); -int tor_version_compare(tor_version_t *a, tor_version_t *b); -void sort_version_list(smartlist_t *lst, int remove_duplicates); -void assert_addr_policy_ok(smartlist_t *t); -void dump_distinct_digest_count(int severity); - -networkstatus_v2_t *networkstatus_v2_parse_from_string(const char *s); -networkstatus_t *networkstatus_parse_vote_from_string(const char *s, - const char **eos_out, - networkstatus_type_t ns_type); -ns_detached_signatures_t *networkstatus_parse_detached_signatures( - const char *s, const char *eos); - -authority_cert_t *authority_cert_parse_from_string(const char *s, - const char **end_of_string); -int rend_parse_v2_service_descriptor(rend_service_descriptor_t **parsed_out, - char *desc_id_out, - char **intro_points_encrypted_out, - size_t *intro_points_encrypted_size_out, - size_t *encoded_size_out, - const char **next_out, const char *desc); -int rend_decrypt_introduction_points(char **ipos_decrypted, - size_t *ipos_decrypted_size, - const char *descriptor_cookie, - const char *ipos_encrypted, - size_t ipos_encrypted_size); -int rend_parse_introduction_points(rend_service_descriptor_t *parsed, - const char *intro_points_encoded, - size_t intro_points_encoded_size); -int rend_parse_client_keys(strmap_t *parsed_clients, const char *str); + int git_tag_len; + char git_tag[DIGEST_LEN]; +} tor_version_t; #endif diff --git a/src/or/policies.c b/src/or/policies.c index 7299b7db3c..db3c6d886b 100644 --- a/src/or/policies.c +++ b/src/or/policies.c @@ -9,6 +9,10 @@ **/ #include "or.h" +#include "config.h" +#include "dirserv.h" +#include "policies.h" +#include "routerparse.h" #include "ht.h" /** Policy that addresses for incoming SOCKS connections must match. */ @@ -344,7 +348,8 @@ validate_addr_policies(or_options_t *options, char **msg) *msg = NULL; if (policies_parse_exit_policy(options->ExitPolicy, &addr_policy, - options->ExitPolicyRejectPrivate, NULL)) + options->ExitPolicyRejectPrivate, NULL, + !options->BridgeRelay)) REJECT("Error in ExitPolicy entry."); /* The rest of these calls *append* to addr_policy. So don't actually @@ -375,12 +380,6 @@ validate_addr_policies(or_options_t *options, char **msg) if (parse_addr_policy(options->ReachableDirAddresses, &addr_policy, ADDR_POLICY_ACCEPT)) REJECT("Error in ReachableDirAddresses entry."); - if (parse_addr_policy(options->AuthDirReject, &addr_policy, - ADDR_POLICY_REJECT)) - REJECT("Error in AuthDirReject entry."); - if (parse_addr_policy(options->AuthDirInvalid, &addr_policy, - ADDR_POLICY_REJECT)) - REJECT("Error in AuthDirInvalid entry."); err: addr_policy_list_free(addr_policy); @@ -829,14 +828,16 @@ exit_policy_remove_redundancies(smartlist_t *dest) "reject *:6346-6429,reject *:6699,reject *:6881-6999,accept *:*" /** Parse the exit policy <b>cfg</b> into the linked list *<b>dest</b>. If - * cfg doesn't end in an absolute accept or reject, add the default exit + * cfg doesn't end in an absolute accept or reject and if + * <b>add_default_policy</b> is true, add the default exit * policy afterwards. If <b>rejectprivate</b> is true, prepend * "reject private:*" to the policy. Return -1 if we can't parse cfg, * else return 0. */ int policies_parse_exit_policy(config_line_t *cfg, smartlist_t **dest, - int rejectprivate, const char *local_address) + int rejectprivate, const char *local_address, + int add_default_policy) { if (rejectprivate) { append_exit_policy_string(dest, "reject private:*"); @@ -848,8 +849,10 @@ policies_parse_exit_policy(config_line_t *cfg, smartlist_t **dest, } if (parse_addr_policy(cfg, dest, -1)) return -1; - append_exit_policy_string(dest, DEFAULT_EXIT_POLICY); - + if (add_default_policy) + append_exit_policy_string(dest, DEFAULT_EXIT_POLICY); + else + append_exit_policy_string(dest, "reject *:*"); exit_policy_remove_redundancies(*dest); return 0; @@ -866,6 +869,49 @@ policies_set_router_exitpolicy_to_reject_all(routerinfo_t *r) smartlist_add(r->exit_policy, item); } +/** Return 1 if there is at least one /8 subnet in <b>policy</b> that + * allows exiting to <b>port</b>. Otherwise, return 0. */ +static int +exit_policy_is_general_exit_helper(smartlist_t *policy, int port) +{ + uint32_t mask, ip, i; + /* Is this /8 rejected (1), or undecided (0)? */ + char subnet_status[256]; + + memset(subnet_status, 0, sizeof(subnet_status)); + SMARTLIST_FOREACH(policy, addr_policy_t *, p, { + if (p->prt_min > port || p->prt_max < port) + continue; /* Doesn't cover our port. */ + mask = 0; + tor_assert(p->maskbits <= 32); + + if (p->maskbits) + mask = UINT32_MAX<<(32-p->maskbits); + ip = tor_addr_to_ipv4h(&p->addr); + + /* Calculate the first and last subnet that this exit policy touches + * and set it as loop boundaries. */ + for (i = ((mask & ip)>>24); i <= (~((mask & ip) ^ mask)>>24); ++i) { + tor_addr_t addr; + if (subnet_status[i] != 0) + continue; /* We already reject some part of this /8 */ + tor_addr_from_ipv4h(&addr, i<<24); + if (tor_addr_is_internal(&addr, 0)) + continue; /* Local or non-routable addresses */ + if (p->policy_type == ADDR_POLICY_ACCEPT) { + if (p->maskbits > 8) + continue; /* Narrower than a /8. */ + /* We found an allowed subnet of at least size /8. Done + * for this port! */ + return 1; + } else if (p->policy_type == ADDR_POLICY_REJECT) { + subnet_status[i] = 1; + } + } + }); + return 0; +} + /** Return true iff <b>ri</b> is "useful as an exit node", meaning * it allows exit to at least one /8 address space for at least * two of ports 80, 443, and 6667. */ @@ -879,19 +925,7 @@ exit_policy_is_general_exit(smartlist_t *policy) return 0; for (i = 0; i < 3; ++i) { - SMARTLIST_FOREACH(policy, addr_policy_t *, p, { - if (p->prt_min > ports[i] || p->prt_max < ports[i]) - continue; /* Doesn't cover our port. */ - if (p->maskbits > 8) - continue; /* Narrower than a /8. */ - if (tor_addr_is_loopback(&p->addr)) - continue; /* 127.x or ::1. */ - /* We have a match that is at least a /8. */ - if (p->policy_type == ADDR_POLICY_ACCEPT) { - ++n_allowed; - break; /* stop considering this port */ - } - }); + n_allowed += exit_policy_is_general_exit_helper(policy, ports[i]); } return n_allowed >= 2; } @@ -1258,9 +1292,11 @@ cleanup: * about "exit-policy/..." */ int getinfo_helper_policies(control_connection_t *conn, - const char *question, char **answer) + const char *question, char **answer, + const char **errmsg) { (void) conn; + (void) errmsg; if (!strcmp(question, "exit-policy/default")) { *answer = tor_strdup(DEFAULT_EXIT_POLICY); } @@ -1271,7 +1307,8 @@ getinfo_helper_policies(control_connection_t *conn, void addr_policy_list_free(smartlist_t *lst) { - if (!lst) return; + if (!lst) + return; SMARTLIST_FOREACH(lst, addr_policy_t *, policy, addr_policy_free(policy)); smartlist_free(lst); } @@ -1280,19 +1317,20 @@ addr_policy_list_free(smartlist_t *lst) void addr_policy_free(addr_policy_t *p) { - if (p) { - if (--p->refcnt <= 0) { - if (p->is_canonical) { - policy_map_ent_t search, *found; - search.policy = p; - found = HT_REMOVE(policy_map, &policy_root, &search); - if (found) { - tor_assert(p == found->policy); - tor_free(found); - } + if (!p) + return; + + if (--p->refcnt <= 0) { + if (p->is_canonical) { + policy_map_ent_t search, *found; + search.policy = p; + found = HT_REMOVE(policy_map, &policy_root, &search); + if (found) { + tor_assert(p == found->policy); + tor_free(found); } - tor_free(p); } + tor_free(p); } } diff --git a/src/or/policies.h b/src/or/policies.h new file mode 100644 index 0000000000..dd46f4de9c --- /dev/null +++ b/src/or/policies.h @@ -0,0 +1,60 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2010, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file policies.h + * \brief Header file for policies.c. + **/ + +#ifndef _TOR_POLICIES_H +#define _TOR_POLICIES_H + +/* (length of "accept 255.255.255.255/255.255.255.255:65535-65535\n" plus a + * NUL.) + */ +#define POLICY_BUF_LEN 52 + +int firewall_is_fascist_or(void); +int fascist_firewall_allows_address_or(const tor_addr_t *addr, uint16_t port); +int fascist_firewall_allows_or(routerinfo_t *ri); +int fascist_firewall_allows_address_dir(const tor_addr_t *addr, uint16_t port); +int dir_policy_permits_address(const tor_addr_t *addr); +int socks_policy_permits_address(const tor_addr_t *addr); +int authdir_policy_permits_address(uint32_t addr, uint16_t port); +int authdir_policy_valid_address(uint32_t addr, uint16_t port); +int authdir_policy_baddir_address(uint32_t addr, uint16_t port); +int authdir_policy_badexit_address(uint32_t addr, uint16_t port); + +int validate_addr_policies(or_options_t *options, char **msg); +void policy_expand_private(smartlist_t **policy); +int policies_parse_from_options(or_options_t *options); + +addr_policy_t *addr_policy_get_canonical_entry(addr_policy_t *ent); +int cmp_addr_policies(smartlist_t *a, smartlist_t *b); +addr_policy_result_t compare_tor_addr_to_addr_policy(const tor_addr_t *addr, + uint16_t port, const smartlist_t *policy); +addr_policy_result_t compare_addr_to_addr_policy(uint32_t addr, + uint16_t port, const smartlist_t *policy); +int policies_parse_exit_policy(config_line_t *cfg, smartlist_t **dest, + int rejectprivate, const char *local_address, + int add_default_policy); +void policies_set_router_exitpolicy_to_reject_all(routerinfo_t *exitrouter); +int exit_policy_is_general_exit(smartlist_t *policy); +int policy_is_reject_star(const smartlist_t *policy); +int getinfo_helper_policies(control_connection_t *conn, + const char *question, char **answer, + const char **errmsg); +int policy_write_item(char *buf, size_t buflen, addr_policy_t *item, + int format_for_desc); + +void addr_policy_list_free(smartlist_t *p); +void addr_policy_free(addr_policy_t *p); +void policies_free_all(void); + +char *policy_summarize(smartlist_t *policy); + +#endif + diff --git a/src/or/reasons.c b/src/or/reasons.c index 0679119b6c..8d71f3b39e 100644 --- a/src/or/reasons.c +++ b/src/or/reasons.c @@ -9,6 +9,8 @@ **/ #include "or.h" +#include "config.h" +#include "reasons.h" /***************************** Edge (stream) reasons **********************/ @@ -332,3 +334,70 @@ circuit_end_reason_to_control_string(int reason) } } +/** Return a string corresponding to a SOCKS4 reponse code. */ +const char * +socks4_response_code_to_string(uint8_t code) +{ + switch (code) { + case 0x5a: + return "connection accepted"; + case 0x5b: + return "server rejected connection"; + case 0x5c: + return "server cannot connect to identd on this client"; + case 0x5d: + return "user id does not match identd"; + default: + return "invalid SOCKS 4 response code"; + } +} + +/** Return a string corresponding to a SOCKS5 reponse code. */ +const char * +socks5_response_code_to_string(uint8_t code) +{ + switch (code) { + case 0x00: + return "connection accepted"; + case 0x01: + return "general SOCKS server failure"; + case 0x02: + return "connection not allowed by ruleset"; + case 0x03: + return "Network unreachable"; + case 0x04: + return "Host unreachable"; + case 0x05: + return "Connection refused"; + case 0x06: + return "TTL expired"; + case 0x07: + return "Command not supported"; + case 0x08: + return "Address type not supported"; + default: + return "unknown reason"; + } +} + +/** Return a string corresponding to a bandwidht_weight_rule_t */ +const char * +bandwidth_weight_rule_to_string(bandwidth_weight_rule_t rule) +{ + switch (rule) + { + case NO_WEIGHTING: + return "no weighting"; + case WEIGHT_FOR_EXIT: + return "weight as exit"; + case WEIGHT_FOR_MID: + return "weight as middle node"; + case WEIGHT_FOR_GUARD: + return "weight as guard"; + case WEIGHT_FOR_DIR: + return "weight as directory"; + default: + return "unknown rule"; + } +} + diff --git a/src/or/reasons.h b/src/or/reasons.h new file mode 100644 index 0000000000..b87d36a77a --- /dev/null +++ b/src/or/reasons.h @@ -0,0 +1,31 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2010, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file reasons.h + * \brief Header file for reasons.c. + **/ + +#ifndef _TOR_REASONS_H +#define _TOR_REASONS_H + +const char *stream_end_reason_to_control_string(int reason); +const char *stream_end_reason_to_string(int reason); +socks5_reply_status_t stream_end_reason_to_socks5_response(int reason); +uint8_t errno_to_stream_end_reason(int e); + +const char *orconn_end_reason_to_control_string(int r); +int tls_error_to_orconn_end_reason(int e); +int errno_to_orconn_end_reason(int e); + +const char *circuit_end_reason_to_control_string(int reason); +const char *socks4_response_code_to_string(uint8_t code); +const char *socks5_response_code_to_string(uint8_t code); + +const char *bandwidth_weight_rule_to_string(enum bandwidth_weight_rule_t rule); + +#endif + diff --git a/src/or/relay.c b/src/or/relay.c index 320c27037b..ad3e9afe47 100644 --- a/src/or/relay.c +++ b/src/or/relay.c @@ -10,8 +10,26 @@ * receiving from circuits, plus queuing on circuits. **/ +#include <math.h> #include "or.h" +#include "buffers.h" +#include "circuitbuild.h" +#include "circuitlist.h" +#include "config.h" +#include "connection.h" +#include "connection_edge.h" +#include "connection_or.h" +#include "control.h" +#include "geoip.h" +#include "main.h" #include "mempool.h" +#include "networkstatus.h" +#include "policies.h" +#include "reasons.h" +#include "relay.h" +#include "rendcommon.h" +#include "routerlist.h" +#include "routerparse.h" static int relay_crypt(circuit_t *circ, cell_t *cell, cell_direction_t cell_direction, @@ -35,6 +53,26 @@ circuit_resume_edge_reading_helper(edge_connection_t *conn, static int circuit_consider_stop_edge_reading(circuit_t *circ, crypt_path_t *layer_hint); +/** Cache the current hi-res time; the cache gets reset when libevent + * calls us. */ + +static struct timeval cached_time_hires = {0, 0}; + +static void +tor_gettimeofday_cached(struct timeval *tv) +{ + if (cached_time_hires.tv_sec == 0) { + tor_gettimeofday(&cached_time_hires); + } + *tv = cached_time_hires; +} + +void +tor_gettimeofday_cache_clear(void) +{ + cached_time_hires.tv_sec = 0; +} + /** Stats: how many relay cells have originated at this hop, or have * been relayed onward (not recognized at this hop)? */ @@ -533,6 +571,12 @@ relay_send_command_from_edge(uint16_t stream_id, circuit_t *circ, log_debug(LD_OR,"delivering %d cell %s.", relay_command, cell_direction == CELL_DIRECTION_OUT ? "forward" : "backward"); + /* If we are sending an END cell and this circuit is used for a tunneled + * directory request, advance its state. */ + if (relay_command == RELAY_COMMAND_END && circ->dirreq_id) + geoip_change_dirreq_state(circ->dirreq_id, DIRREQ_TUNNELED, + DIRREQ_END_CELL_SENT); + if (cell_direction == CELL_DIRECTION_OUT && circ->n_conn) { /* if we're using relaybandwidthrate, this conn wants priority */ circ->n_conn->client_used = approx_time(); @@ -550,9 +594,9 @@ relay_send_command_from_edge(uint16_t stream_id, circuit_t *circ, * Don't worry about the conn protocol version: * append_cell_to_circuit_queue will fix it up. */ /* XXX For now, clients don't use RELAY_EARLY cells when sending - * relay cells on rendezvous circuits. See bug 1038. Eventually, - * we can take this behavior away in favor of having clients avoid - * rendezvous points running 0.2.1.3-alpha through 0.2.1.18. -RD */ + * relay cells on rendezvous circuits. See bug 1038. Once no relays + * (and thus no rendezvous points) are running 0.2.1.3-alpha through + * 0.2.1.18, we can take out that exception. -RD */ cell.command = CELL_RELAY_EARLY; --origin_circ->remaining_relay_early_cells; log_debug(LD_OR, "Sending a RELAY_EARLY cell; %d remaining.", @@ -1000,7 +1044,8 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, relay_header_unpack(&rh, cell->payload); // log_fn(LOG_DEBUG,"command %d stream %d", rh.command, rh.stream_id); num_seen++; - log_debug(domain, "Now seen %d relay cells here.", num_seen); + log_debug(domain, "Now seen %d relay cells here (command %d, stream %d).", + num_seen, rh.command, rh.stream_id); if (rh.length > RELAY_PAYLOAD_SIZE) { log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, @@ -1039,6 +1084,16 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, "Begin cell for known stream. Dropping."); return 0; } + if (rh.command == RELAY_COMMAND_BEGIN_DIR) { + /* Assign this circuit and its app-ward OR connection a unique ID, + * so that we can measure download times. The local edge and dir + * connection will be assigned the same ID when they are created + * and linked. */ + static uint64_t next_id = 0; + circ->dirreq_id = ++next_id; + TO_CONN(TO_OR_CIRCUIT(circ)->p_conn)->dirreq_id = circ->dirreq_id; + } + return connection_exit_begin_conn(cell, circ); case RELAY_COMMAND_DATA: ++stats_n_data_cells_received; @@ -1509,6 +1564,10 @@ static int total_cells_allocated = 0; /** A memory pool to allocate packed_cell_t objects. */ static mp_pool_t *cell_pool = NULL; +/** Memory pool to allocate insertion_time_elem_t objects used for cell + * statistics. */ +static mp_pool_t *it_pool = NULL; + /** Allocate structures to hold cells. */ void init_cell_pool(void) @@ -1517,7 +1576,8 @@ init_cell_pool(void) cell_pool = mp_pool_new(sizeof(packed_cell_t), 128*1024); } -/** Free all storage used to hold cells. */ +/** Free all storage used to hold cells (and insertion times if we measure + * cell statistics). */ void free_cell_pool(void) { @@ -1526,6 +1586,10 @@ free_cell_pool(void) mp_pool_destroy(cell_pool); cell_pool = NULL; } + if (it_pool) { + mp_pool_destroy(it_pool); + it_pool = NULL; + } } /** Free excess storage in cell pool. */ @@ -1538,7 +1602,7 @@ clean_cell_pool(void) /** Release storage held by <b>cell</b>. */ static INLINE void -packed_cell_free(packed_cell_t *cell) +packed_cell_free_unchecked(packed_cell_t *cell) { --total_cells_allocated; mp_pool_release(cell); @@ -1600,7 +1664,38 @@ cell_queue_append(cell_queue_t *queue, packed_cell_t *cell) void cell_queue_append_packed_copy(cell_queue_t *queue, const cell_t *cell) { - cell_queue_append(queue, packed_cell_copy(cell)); + packed_cell_t *copy = packed_cell_copy(cell); + /* Remember the time when this cell was put in the queue. */ + if (get_options()->CellStatistics) { + struct timeval now; + uint32_t added; + insertion_time_queue_t *it_queue = queue->insertion_times; + if (!it_pool) + it_pool = mp_pool_new(sizeof(insertion_time_elem_t), 1024); + tor_gettimeofday_cached(&now); +#define SECONDS_IN_A_DAY 86400L + added = (uint32_t)(((now.tv_sec % SECONDS_IN_A_DAY) * 100L) + + ((uint32_t)now.tv_usec / (uint32_t)10000L)); + if (!it_queue) { + it_queue = tor_malloc_zero(sizeof(insertion_time_queue_t)); + queue->insertion_times = it_queue; + } + if (it_queue->last && it_queue->last->insertion_time == added) { + it_queue->last->counter++; + } else { + insertion_time_elem_t *elem = mp_pool_get(it_pool); + elem->next = NULL; + elem->insertion_time = added; + elem->counter = 1; + if (it_queue->last) { + it_queue->last->next = elem; + it_queue->last = elem; + } else { + it_queue->first = it_queue->last = elem; + } + } + } + cell_queue_append(queue, copy); } /** Remove and free every cell in <b>queue</b>. */ @@ -1611,11 +1706,19 @@ cell_queue_clear(cell_queue_t *queue) cell = queue->head; while (cell) { next = cell->next; - packed_cell_free(cell); + packed_cell_free_unchecked(cell); cell = next; } queue->head = queue->tail = NULL; queue->n = 0; + if (queue->insertion_times) { + while (queue->insertion_times->first) { + insertion_time_elem_t *elem = queue->insertion_times->first; + queue->insertion_times->first = elem->next; + mp_pool_release(elem); + } + tor_free(queue->insertion_times); + } } /** Extract and return the cell at the head of <b>queue</b>; return NULL if @@ -1667,8 +1770,226 @@ prev_circ_on_conn_p(circuit_t *circ, or_connection_t *conn) } } +/** Helper for sorting cell_ewma_t values in their priority queue. */ +static int +compare_cell_ewma_counts(const void *p1, const void *p2) +{ + const cell_ewma_t *e1=p1, *e2=p2; + if (e1->cell_count < e2->cell_count) + return -1; + else if (e1->cell_count > e2->cell_count) + return 1; + else + return 0; +} + +/** Given a cell_ewma_t, return a pointer to the circuit containing it. */ +static circuit_t * +cell_ewma_to_circuit(cell_ewma_t *ewma) +{ + if (ewma->is_for_p_conn) { + /* This is an or_circuit_t's p_cell_ewma. */ + or_circuit_t *orcirc = SUBTYPE_P(ewma, or_circuit_t, p_cell_ewma); + return TO_CIRCUIT(orcirc); + } else { + /* This is some circuit's n_cell_ewma. */ + return SUBTYPE_P(ewma, circuit_t, n_cell_ewma); + } +} + +/* ==== Functions for scaling cell_ewma_t ==== + + When choosing which cells to relay first, we favor circuits that have been + quiet recently. This gives better latency on connections that aren't + pushing lots of data, and makes the network feel more interactive. + + Conceptually, we take an exponentially weighted mean average of the number + of cells a circuit has sent, and allow active circuits (those with cells to + relay) to send cells in reverse order of their exponentially-weighted mean + average (EWMA) cell count. [That is, a cell sent N seconds ago 'counts' + F^N times as much as a cell sent now, for 0<F<1.0, and we favor the + circuit that has sent the fewest cells] + + If 'double' had infinite precision, we could do this simply by counting a + cell sent at startup as having weight 1.0, and a cell sent N seconds later + as having weight F^-N. This way, we would never need to re-scale + any already-sent cells. + + To prevent double from overflowing, we could count a cell sent now as + having weight 1.0 and a cell sent N seconds ago as having weight F^N. + This, however, would mean we'd need to re-scale *ALL* old circuits every + time we wanted to send a cell. + + So as a compromise, we divide time into 'ticks' (currently, 10-second + increments) and say that a cell sent at the start of a current tick is + worth 1.0, a cell sent N seconds before the start of the current tick is + worth F^N, and a cell sent N seconds after the start of the current tick is + worth F^-N. This way we don't overflow, and we don't need to constantly + rescale. + */ + +/** How long does a tick last (seconds)? */ +#define EWMA_TICK_LEN 10 + +/** The default per-tick scale factor, if it hasn't been overridden by a + * consensus or a configuration setting. zero means "disabled". */ +#define EWMA_DEFAULT_HALFLIFE 0.0 + +/** Given a timeval <b>now</b>, compute the cell_ewma tick in which it occurs + * and the fraction of the tick that has elapsed between the start of the tick + * and <b>now</b>. Return the former and store the latter in + * *<b>remainder_out</b>. + * + * These tick values are not meant to be shared between Tor instances, or used + * for other purposes. */ +static unsigned +cell_ewma_tick_from_timeval(const struct timeval *now, + double *remainder_out) +{ + unsigned res = (unsigned) (now->tv_sec / EWMA_TICK_LEN); + /* rem */ + double rem = (now->tv_sec % EWMA_TICK_LEN) + + ((double)(now->tv_usec)) / 1.0e6; + *remainder_out = rem / EWMA_TICK_LEN; + return res; +} + +/** Compute and return the current cell_ewma tick. */ +unsigned +cell_ewma_get_tick(void) +{ + return ((unsigned)approx_time() / EWMA_TICK_LEN); +} + +/** The per-tick scale factor to be used when computing cell-count EWMA + * values. (A cell sent N ticks before the start of the current tick + * has value ewma_scale_factor ** N.) + */ +static double ewma_scale_factor = 0.1; +static int ewma_enabled = 0; + +#define EPSILON 0.00001 +#define LOG_ONEHALF -0.69314718055994529 + +/** Adjust the global cell scale factor based on <b>options</b> */ +void +cell_ewma_set_scale_factor(or_options_t *options, networkstatus_t *consensus) +{ + int32_t halflife_ms; + double halflife; + const char *source; + if (options && options->CircuitPriorityHalflife >= -EPSILON) { + halflife = options->CircuitPriorityHalflife; + source = "CircuitPriorityHalflife in configuration"; + } else if (consensus && + (halflife_ms = networkstatus_get_param( + consensus, "CircuitPriorityHalflifeMsec", -1)) >= 0) { + halflife = ((double)halflife_ms)/1000.0; + source = "CircuitPriorityHalflifeMsec in consensus"; + } else { + halflife = EWMA_DEFAULT_HALFLIFE; + source = "Default value"; + } + + if (halflife <= EPSILON) { + /* The cell EWMA algorithm is disabled. */ + ewma_scale_factor = 0.1; + ewma_enabled = 0; + log_info(LD_OR, + "Disabled cell_ewma algorithm because of value in %s", + source); + } else { + /* convert halflife into halflife-per-tick. */ + halflife /= EWMA_TICK_LEN; + /* compute per-tick scale factor. */ + ewma_scale_factor = exp( LOG_ONEHALF / halflife ); + ewma_enabled = 1; + log_info(LD_OR, + "Enabled cell_ewma algorithm because of value in %s; " + "scale factor is %lf per %d seconds", + source, ewma_scale_factor, EWMA_TICK_LEN); + } +} + +/** Return the multiplier necessary to convert the value of a cell sent in + * 'from_tick' to one sent in 'to_tick'. */ +static INLINE double +get_scale_factor(unsigned from_tick, unsigned to_tick) +{ + /* This math can wrap around, but that's okay: unsigned overflow is + well-defined */ + int diff = (int)(to_tick - from_tick); + return pow(ewma_scale_factor, diff); +} + +/** Adjust the cell count of <b>ewma</b> so that it is scaled with respect to + * <b>cur_tick</b> */ +static void +scale_single_cell_ewma(cell_ewma_t *ewma, unsigned cur_tick) +{ + double factor = get_scale_factor(ewma->last_adjusted_tick, cur_tick); + ewma->cell_count *= factor; + ewma->last_adjusted_tick = cur_tick; +} + +/** Adjust the cell count of every active circuit on <b>conn</b> so + * that they are scaled with respect to <b>cur_tick</b> */ +static void +scale_active_circuits(or_connection_t *conn, unsigned cur_tick) +{ + + double factor = get_scale_factor( + conn->active_circuit_pqueue_last_recalibrated, + cur_tick); + /** Ordinarily it isn't okay to change the value of an element in a heap, + * but it's okay here, since we are preserving the order. */ + SMARTLIST_FOREACH(conn->active_circuit_pqueue, cell_ewma_t *, e, { + tor_assert(e->last_adjusted_tick == + conn->active_circuit_pqueue_last_recalibrated); + e->cell_count *= factor; + e->last_adjusted_tick = cur_tick; + }); + conn->active_circuit_pqueue_last_recalibrated = cur_tick; +} + +/** Rescale <b>ewma</b> to the same scale as <b>conn</b>, and add it to + * <b>conn</b>'s priority queue of active circuits */ +static void +add_cell_ewma_to_conn(or_connection_t *conn, cell_ewma_t *ewma) +{ + tor_assert(ewma->heap_index == -1); + scale_single_cell_ewma(ewma, + conn->active_circuit_pqueue_last_recalibrated); + + smartlist_pqueue_add(conn->active_circuit_pqueue, + compare_cell_ewma_counts, + STRUCT_OFFSET(cell_ewma_t, heap_index), + ewma); +} + +/** Remove <b>ewma</b> from <b>conn</b>'s priority queue of active circuits */ +static void +remove_cell_ewma_from_conn(or_connection_t *conn, cell_ewma_t *ewma) +{ + tor_assert(ewma->heap_index != -1); + smartlist_pqueue_remove(conn->active_circuit_pqueue, + compare_cell_ewma_counts, + STRUCT_OFFSET(cell_ewma_t, heap_index), + ewma); +} + +/** Remove and return the first cell_ewma_t from conn's priority queue of + * active circuits. Requires that the priority queue is nonempty. */ +static cell_ewma_t * +pop_first_cell_ewma_from_conn(or_connection_t *conn) +{ + return smartlist_pqueue_pop(conn->active_circuit_pqueue, + compare_cell_ewma_counts, + STRUCT_OFFSET(cell_ewma_t, heap_index)); +} + /** Add <b>circ</b> to the list of circuits with pending cells on - * <b>conn</b>. No effect if <b>circ</b> is already unlinked. */ + * <b>conn</b>. No effect if <b>circ</b> is already linked. */ void make_circuit_active_on_conn(circuit_t *circ, or_connection_t *conn) { @@ -1680,6 +2001,8 @@ make_circuit_active_on_conn(circuit_t *circ, or_connection_t *conn) return; } + assert_active_circuits_ok_paranoid(conn); + if (! conn->active_circuits) { conn->active_circuits = circ; *prevp = *nextp = circ; @@ -1691,10 +2014,19 @@ make_circuit_active_on_conn(circuit_t *circ, or_connection_t *conn) *prev_circ_on_conn_p(head, conn) = circ; *prevp = old_tail; } + + if (circ->n_conn == conn) { + add_cell_ewma_to_conn(conn, &circ->n_cell_ewma); + } else { + or_circuit_t *orcirc = TO_OR_CIRCUIT(circ); + tor_assert(conn == orcirc->p_conn); + add_cell_ewma_to_conn(conn, &orcirc->p_cell_ewma); + } + assert_active_circuits_ok_paranoid(conn); } -/** Remove <b>circ</b> to the list of circuits with pending cells on +/** Remove <b>circ</b> from the list of circuits with pending cells on * <b>conn</b>. No effect if <b>circ</b> is already unlinked. */ void make_circuit_inactive_on_conn(circuit_t *circ, or_connection_t *conn) @@ -1708,6 +2040,8 @@ make_circuit_inactive_on_conn(circuit_t *circ, or_connection_t *conn) return; } + assert_active_circuits_ok_paranoid(conn); + tor_assert(next && prev); tor_assert(*prev_circ_on_conn_p(next, conn) == circ); tor_assert(*next_circ_on_conn_p(prev, conn) == circ); @@ -1721,6 +2055,15 @@ make_circuit_inactive_on_conn(circuit_t *circ, or_connection_t *conn) conn->active_circuits = next; } *prevp = *nextp = NULL; + + if (circ->n_conn == conn) { + remove_cell_ewma_from_conn(conn, &circ->n_cell_ewma); + } else { + or_circuit_t *orcirc = TO_OR_CIRCUIT(circ); + tor_assert(conn == orcirc->p_conn); + remove_cell_ewma_from_conn(conn, &orcirc->p_cell_ewma); + } + assert_active_circuits_ok_paranoid(conn); } @@ -1740,6 +2083,10 @@ connection_or_unlink_all_active_circs(or_connection_t *orconn) cur = next; } while (cur != head); orconn->active_circuits = NULL; + + SMARTLIST_FOREACH(orconn->active_circuit_pqueue, cell_ewma_t *, e, + e->heap_index = -1); + smartlist_clear(orconn->active_circuit_pqueue); } /** Block (if <b>block</b> is true) or unblock (if <b>block</b> is false) @@ -1782,7 +2129,7 @@ set_streams_blocked_on_circ(circuit_t *circ, or_connection_t *orconn, } /** Pull as many cells as possible (but no more than <b>max</b>) from the - * queue of the first active circuit on <b>conn</b>, and write then to + * queue of the first active circuit on <b>conn</b>, and write them to * <b>conn</b>->outbuf. Return the number of cells written. Advance * the active circuit pointer to the next active circuit in the ring. */ int @@ -1793,9 +2140,35 @@ connection_or_flush_from_first_active_circuit(or_connection_t *conn, int max, cell_queue_t *queue; circuit_t *circ; int streams_blocked; + + /* The current (hi-res) time */ + struct timeval now_hires; + + /* The EWMA cell counter for the circuit we're flushing. */ + cell_ewma_t *cell_ewma = NULL; + double ewma_increment = -1; + circ = conn->active_circuits; if (!circ) return 0; assert_active_circuits_ok_paranoid(conn); + + /* See if we're doing the ewma circuit selection algorithm. */ + if (ewma_enabled) { + unsigned tick; + double fractional_tick; + tor_gettimeofday_cached(&now_hires); + tick = cell_ewma_tick_from_timeval(&now_hires, &fractional_tick); + + if (tick != conn->active_circuit_pqueue_last_recalibrated) { + scale_active_circuits(conn, tick); + } + + ewma_increment = pow(ewma_scale_factor, -fractional_tick); + + cell_ewma = smartlist_get(conn->active_circuit_pqueue, 0); + circ = cell_ewma_to_circuit(cell_ewma); + } + if (circ->n_conn == conn) { queue = &circ->n_conn_cells; streams_blocked = circ->streams_blocked_on_n_conn; @@ -1809,10 +2182,58 @@ connection_or_flush_from_first_active_circuit(or_connection_t *conn, int max, packed_cell_t *cell = cell_queue_pop(queue); tor_assert(*next_circ_on_conn_p(circ,conn)); + /* Calculate the exact time that this cell has spent in the queue. */ + if (get_options()->CellStatistics && !CIRCUIT_IS_ORIGIN(circ)) { + struct timeval now; + uint32_t flushed; + uint32_t cell_waiting_time; + insertion_time_queue_t *it_queue = queue->insertion_times; + tor_gettimeofday_cached(&now); + flushed = (uint32_t)((now.tv_sec % SECONDS_IN_A_DAY) * 100L + + (uint32_t)now.tv_usec / (uint32_t)10000L); + if (!it_queue || !it_queue->first) { + log_warn(LD_BUG, "Cannot determine insertion time of cell."); + } else { + or_circuit_t *orcirc = TO_OR_CIRCUIT(circ); + insertion_time_elem_t *elem = it_queue->first; + cell_waiting_time = + (uint32_t)((flushed * 10L + SECONDS_IN_A_DAY * 1000L - + elem->insertion_time * 10L) % + (SECONDS_IN_A_DAY * 1000L)); +#undef SECONDS_IN_A_DAY + elem->counter--; + if (elem->counter < 1) { + it_queue->first = elem->next; + if (elem == it_queue->last) + it_queue->last = NULL; + mp_pool_release(elem); + } + orcirc->total_cell_waiting_time += cell_waiting_time; + orcirc->processed_cells++; + } + } + + /* If we just flushed our queue and this circuit is used for a + * tunneled directory request, possibly advance its state. */ + if (queue->n == 0 && TO_CONN(conn)->dirreq_id) + geoip_change_dirreq_state(TO_CONN(conn)->dirreq_id, + DIRREQ_TUNNELED, + DIRREQ_CIRC_QUEUE_FLUSHED); + connection_write_to_buf(cell->body, CELL_NETWORK_SIZE, TO_CONN(conn)); - packed_cell_free(cell); + packed_cell_free_unchecked(cell); ++n_flushed; + if (cell_ewma) { + cell_ewma_t *tmp; + cell_ewma->cell_count += ewma_increment; + /* We pop and re-add the cell_ewma_t here, not above, since we need to + * re-add it immediately to keep the priority queue consistent with + * the linked-list implementation */ + tmp = pop_first_cell_ewma_from_conn(conn); + tor_assert(tmp == cell_ewma); + add_cell_ewma_to_conn(conn, cell_ewma); + } if (circ != conn->active_circuits) { /* If this happens, the current circuit just got made inactive by * a call in connection_write_to_buf(). That's nothing to worry about: @@ -1832,7 +2253,7 @@ connection_or_flush_from_first_active_circuit(or_connection_t *conn, int max, if (streams_blocked && queue->n <= CELL_QUEUE_LOWWATER_SIZE) set_streams_blocked_on_circ(circ, conn, 0); /* unblock streams */ - /* Did we just ran out of cells on this queue? */ + /* Did we just run out of cells on this circuit's queue? */ if (queue->n == 0) { log_debug(LD_GENERAL, "Made a circuit inactive."); make_circuit_inactive_on_conn(circ, conn); @@ -1955,16 +2376,31 @@ assert_active_circuits_ok(or_connection_t *orconn) { circuit_t *head = orconn->active_circuits; circuit_t *cur = head; + int n = 0; if (! head) return; do { circuit_t *next = *next_circ_on_conn_p(cur, orconn); circuit_t *prev = *prev_circ_on_conn_p(cur, orconn); + cell_ewma_t *ewma; tor_assert(next); tor_assert(prev); tor_assert(*next_circ_on_conn_p(prev, orconn) == cur); tor_assert(*prev_circ_on_conn_p(next, orconn) == cur); + if (orconn == cur->n_conn) { + ewma = &cur->n_cell_ewma; + tor_assert(!ewma->is_for_p_conn); + } else { + ewma = &TO_OR_CIRCUIT(cur)->p_cell_ewma; + tor_assert(ewma->is_for_p_conn); + } + tor_assert(ewma->heap_index != -1); + tor_assert(ewma == smartlist_get(orconn->active_circuit_pqueue, + ewma->heap_index)); + n++; cur = next; } while (cur != head); + + tor_assert(n == smartlist_len(orconn->active_circuit_pqueue)); } diff --git a/src/or/relay.h b/src/or/relay.h new file mode 100644 index 0000000000..73855a52bf --- /dev/null +++ b/src/or/relay.h @@ -0,0 +1,65 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2010, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file relay.h + * \brief Header file for relay.c. + **/ + +#ifndef _TOR_RELAY_H +#define _TOR_RELAY_H + +extern uint64_t stats_n_relay_cells_relayed; +extern uint64_t stats_n_relay_cells_delivered; + +int circuit_receive_relay_cell(cell_t *cell, circuit_t *circ, + cell_direction_t cell_direction); + +void relay_header_pack(char *dest, const relay_header_t *src); +void relay_header_unpack(relay_header_t *dest, const char *src); +int relay_send_command_from_edge(streamid_t stream_id, circuit_t *circ, + uint8_t relay_command, const char *payload, + size_t payload_len, crypt_path_t *cpath_layer); +int connection_edge_send_command(edge_connection_t *fromconn, + uint8_t relay_command, const char *payload, + size_t payload_len); +int connection_edge_package_raw_inbuf(edge_connection_t *conn, + int package_partial); +void connection_edge_consider_sending_sendme(edge_connection_t *conn); + +extern uint64_t stats_n_data_cells_packaged; +extern uint64_t stats_n_data_bytes_packaged; +extern uint64_t stats_n_data_cells_received; +extern uint64_t stats_n_data_bytes_received; + +void init_cell_pool(void); +void free_cell_pool(void); +void clean_cell_pool(void); +void dump_cell_pool_usage(int severity); + +void cell_queue_clear(cell_queue_t *queue); +void cell_queue_append(cell_queue_t *queue, packed_cell_t *cell); +void cell_queue_append_packed_copy(cell_queue_t *queue, const cell_t *cell); + +void append_cell_to_circuit_queue(circuit_t *circ, or_connection_t *orconn, + cell_t *cell, cell_direction_t direction); +void connection_or_unlink_all_active_circs(or_connection_t *conn); +int connection_or_flush_from_first_active_circuit(or_connection_t *conn, + int max, time_t now); +void assert_active_circuits_ok(or_connection_t *orconn); +void make_circuit_inactive_on_conn(circuit_t *circ, or_connection_t *conn); +void make_circuit_active_on_conn(circuit_t *circ, or_connection_t *conn); + +int append_address_to_payload(char *payload_out, const tor_addr_t *addr); +const char *decode_address_from_payload(tor_addr_t *addr_out, + const char *payload, + int payload_len); +unsigned cell_ewma_get_tick(void); +void cell_ewma_set_scale_factor(or_options_t *options, + networkstatus_t *consensus); + +#endif + diff --git a/src/or/rendclient.c b/src/or/rendclient.c index 7023b7f271..0377f121cc 100644 --- a/src/or/rendclient.c +++ b/src/or/rendclient.c @@ -8,6 +8,19 @@ **/ #include "or.h" +#include "circuitbuild.h" +#include "circuitlist.h" +#include "circuituse.h" +#include "config.h" +#include "connection.h" +#include "connection_edge.h" +#include "directory.h" +#include "main.h" +#include "relay.h" +#include "rendclient.h" +#include "rendcommon.h" +#include "rephist.h" +#include "routerlist.h" /** Called when we've established a circuit to an introduction point: * send the introduction request. */ @@ -63,7 +76,7 @@ rend_client_send_introduction(origin_circuit_t *introcirc, rend_cache_entry_t *entry; crypt_path_t *cpath; off_t dh_offset; - crypto_pk_env_t *intro_key; /* either Bob's public key or an intro key. */ + crypto_pk_env_t *intro_key = NULL; tor_assert(introcirc->_base.purpose == CIRCUIT_PURPOSE_C_INTRODUCING); tor_assert(rendcirc->_base.purpose == CIRCUIT_PURPOSE_C_REND_READY); @@ -76,48 +89,26 @@ rend_client_send_introduction(origin_circuit_t *introcirc, &entry) < 1) { log_warn(LD_REND, "query %s didn't have valid rend desc in cache. Failing.", - escaped_safe_str(introcirc->rend_data->onion_address)); + escaped_safe_str_client(introcirc->rend_data->onion_address)); goto err; } - /* first 20 bytes of payload are the hash of Bob's pk */ - if (entry->parsed->version == 0) { /* un-versioned descriptor */ - intro_key = entry->parsed->pk; - } else { /* versioned descriptor */ - intro_key = NULL; - SMARTLIST_FOREACH(entry->parsed->intro_nodes, rend_intro_point_t *, - intro, { - if (!memcmp(introcirc->build_state->chosen_exit->identity_digest, - intro->extend_info->identity_digest, DIGEST_LEN)) { - intro_key = intro->intro_key; - break; - } - }); - if (!intro_key) { - /** XXX This case probably means that the intro point vanished while - * we were building a circuit to it. In the future, we should find - * out how that happened and whether we should kill the circuits to - * removed intro points immediately. See task 1073. */ - int num_intro_points = smartlist_len(entry->parsed->intro_nodes); - if (rend_cache_lookup_entry(introcirc->rend_data->onion_address, - 0, &entry) > 0) { - log_info(LD_REND, "We have both a v0 and a v2 rend desc for this " - "service. The v2 desc doesn't contain the introduction " - "point (and key) to send an INTRODUCE1/2 cell to this " - "introduction point. Assuming the introduction point " - "is for v0 rend clients and using the service key " - "from the v0 desc instead. (This is probably a bug, " - "because we shouldn't even have both a v0 and a v2 " - "descriptor for the same service.)"); - /* See flyspray task 1024. */ - intro_key = entry->parsed->pk; - } else { - log_info(LD_REND, "Internal error: could not find intro key; we " - "only have a v2 rend desc with %d intro points.", - num_intro_points); - goto err; - } + /* first 20 bytes of payload are the hash of the intro key */ + intro_key = NULL; + SMARTLIST_FOREACH(entry->parsed->intro_nodes, rend_intro_point_t *, + intro, { + if (!memcmp(introcirc->build_state->chosen_exit->identity_digest, + intro->extend_info->identity_digest, DIGEST_LEN)) { + intro_key = intro->intro_key; + break; } + }); + if (!intro_key) { + log_info(LD_REND, "Our introduction point knowledge changed in " + "mid-connect! Could not find intro key; we only have a " + "v2 rend desc with %d intro points. Giving up.", + smartlist_len(entry->parsed->intro_nodes)); + goto err; } if (crypto_pk_get_digest(intro_key, payload)<0) { log_warn(LD_BUG, "Internal error: couldn't hash public key."); @@ -291,7 +282,7 @@ rend_client_introduction_acked(origin_circuit_t *circ, extend_info = rend_client_get_random_intro(circ->rend_data); if (!extend_info) { log_warn(LD_REND, "No introduction points left for %s. Closing.", - escaped_safe_str(circ->rend_data->onion_address)); + escaped_safe_str_client(circ->rend_data->onion_address)); circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_INTERNAL); return -1; } @@ -299,7 +290,7 @@ rend_client_introduction_acked(origin_circuit_t *circ, log_info(LD_REND, "Got nack for %s from %s. Re-extending circ %d, " "this time to %s.", - escaped_safe_str(circ->rend_data->onion_address), + escaped_safe_str_client(circ->rend_data->onion_address), circ->build_state->chosen_exit->nickname, circ->_base.n_circ_id, extend_info->nickname); result = circuit_extend_to_new_exit(circ, extend_info); @@ -307,7 +298,7 @@ rend_client_introduction_acked(origin_circuit_t *circ, log_info(LD_REND, "Got nack for %s from %s. Building a new introduction " "circuit, this time to %s.", - escaped_safe_str(circ->rend_data->onion_address), + escaped_safe_str_client(circ->rend_data->onion_address), circ->build_state->chosen_exit->nickname, extend_info->nickname); circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_FINISHED); @@ -470,45 +461,21 @@ directory_get_from_hs_dir(const char *desc_id, const rend_data_t *rend_query) rend_query->onion_address, desc_id_base32, rend_query->auth_type, (rend_query->auth_type == REND_NO_AUTH ? "[none]" : - escaped_safe_str(descriptor_cookie_base64)), + escaped_safe_str_client(descriptor_cookie_base64)), hs_dir->nickname, hs_dir->dir_port); return 1; } -/** If we are not currently fetching a rendezvous service descriptor - * for the service ID <b>query</b>, start a directory connection to fetch a - * new one. - */ -void -rend_client_refetch_renddesc(const char *query) -{ - if (!get_options()->FetchHidServDescriptors) - return; - log_info(LD_REND, "Fetching rendezvous descriptor for service %s", - escaped_safe_str(query)); - if (connection_get_by_type_state_rendquery(CONN_TYPE_DIR, 0, query, 0)) { - log_info(LD_REND,"Would fetch a new renddesc here (for %s), but one is " - "already in progress.", escaped_safe_str(query)); - } else { - /* not one already; initiate a dir rend desc lookup */ - directory_get_from_dirserver(DIR_PURPOSE_FETCH_RENDDESC, - ROUTER_PURPOSE_GENERAL, query, - PDS_RETRY_IF_NO_SERVERS); - } -} - -/** Start a connection to a hidden service directory to fetch a v2 - * rendezvous service descriptor for the base32-encoded service ID - * <b>query</b>. - */ +/** Unless we already have a descriptor for <b>rend_query</b> with at least + * one (possibly) working introduction point in it, start a connection to a + * hidden service directory to fetch a v2 rendezvous service descriptor. */ void rend_client_refetch_v2_renddesc(const rend_data_t *rend_query) { char descriptor_id[DIGEST_LEN]; int replicas_left_to_try[REND_NUMBER_OF_NON_CONSECUTIVE_REPLICAS]; - int i, tries_left, r; + int i, tries_left; rend_cache_entry_t *e = NULL; - time_t now = time(NULL); tor_assert(rend_query); /* Are we configured to fetch descriptors? */ if (!get_options()->FetchHidServDescriptors) { @@ -517,15 +484,13 @@ rend_client_refetch_v2_renddesc(const rend_data_t *rend_query) return; } /* Before fetching, check if we already have the descriptor here. */ - r = rend_cache_lookup_entry(rend_query->onion_address, -1, &e); - if (r > 0 && now - e->received < NUM_SECONDS_BEFORE_HS_REFETCH) { + if (rend_cache_lookup_entry(rend_query->onion_address, -1, &e) > 0) { log_info(LD_REND, "We would fetch a v2 rendezvous descriptor, but we " - "already have a fresh copy of that descriptor here. " - "Not fetching."); + "already have that descriptor here. Not fetching."); return; } log_debug(LD_REND, "Fetching v2 rendezvous descriptor for service %s", - safe_str(rend_query->onion_address)); + safe_str_client(rend_query->onion_address)); /* Randomly iterate over the replicas until a descriptor can be fetched * from one of the consecutive nodes, or no options are left. */ tries_left = REND_NUMBER_OF_NON_CONSECUTIVE_REPLICAS; @@ -551,8 +516,8 @@ rend_client_refetch_v2_renddesc(const rend_data_t *rend_query) log_info(LD_REND, "Could not pick one of the responsible hidden " "service directories to fetch descriptors, because " "we already tried them all unsuccessfully."); - /* Close pending connections (unless a v0 request is still going on). */ - rend_client_desc_trynow(rend_query->onion_address, 2); + /* Close pending connections. */ + rend_client_desc_trynow(rend_query->onion_address); return; } @@ -573,18 +538,13 @@ rend_client_remove_intro_point(extend_info_t *failed_intro, r = rend_cache_lookup_entry(rend_query->onion_address, -1, &ent); if (r<0) { log_warn(LD_BUG, "Malformed service ID %s.", - escaped_safe_str(rend_query->onion_address)); + escaped_safe_str_client(rend_query->onion_address)); return -1; } if (r==0) { log_info(LD_REND, "Unknown service %s. Re-fetching descriptor.", - escaped_safe_str(rend_query->onion_address)); - /* Fetch both, v0 and v2 rend descriptors in parallel. Use whichever - * arrives first. Exception: When using client authorization, only - * fetch v2 descriptors.*/ + escaped_safe_str_client(rend_query->onion_address)); rend_client_refetch_v2_renddesc(rend_query); - if (rend_query->auth_type == REND_NO_AUTH) - rend_client_refetch_renddesc(rend_query->onion_address); return 0; } @@ -601,18 +561,13 @@ rend_client_remove_intro_point(extend_info_t *failed_intro, if (smartlist_len(ent->parsed->intro_nodes) == 0) { log_info(LD_REND, "No more intro points remain for %s. Re-fetching descriptor.", - escaped_safe_str(rend_query->onion_address)); - /* Fetch both, v0 and v2 rend descriptors in parallel. Use whichever - * arrives first. Exception: When using client authorization, only - * fetch v2 descriptors.*/ + escaped_safe_str_client(rend_query->onion_address)); rend_client_refetch_v2_renddesc(rend_query); - if (rend_query->auth_type == REND_NO_AUTH) - rend_client_refetch_renddesc(rend_query->onion_address); /* move all pending streams back to renddesc_wait */ while ((conn = connection_get_by_type_state_rendquery(CONN_TYPE_AP, AP_CONN_STATE_CIRCUIT_WAIT, - rend_query->onion_address, -1))) { + rend_query->onion_address))) { conn->state = AP_CONN_STATE_RENDDESC_WAIT; } @@ -620,7 +575,7 @@ rend_client_remove_intro_point(extend_info_t *failed_intro, } log_info(LD_REND,"%d options left for %s.", smartlist_len(ent->parsed->intro_nodes), - escaped_safe_str(rend_query->onion_address)); + escaped_safe_str_client(rend_query->onion_address)); return 1; } @@ -682,8 +637,9 @@ rend_client_receive_rendezvous(origin_circuit_t *circ, const char *request, tor_assert(circ->build_state->pending_final_cpath); hop = circ->build_state->pending_final_cpath; tor_assert(hop->dh_handshake_state); - if (crypto_dh_compute_secret(hop->dh_handshake_state, request, DH_KEY_LEN, - keys, DIGEST_LEN+CPATH_KEY_MATERIAL_LEN)<0) { + if (crypto_dh_compute_secret(LOG_PROTOCOL_WARN, hop->dh_handshake_state, + request, DH_KEY_LEN, keys, + DIGEST_LEN+CPATH_KEY_MATERIAL_LEN)<0) { log_warn(LD_GENERAL, "Couldn't complete DH handshake."); goto err; } @@ -721,24 +677,18 @@ rend_client_receive_rendezvous(origin_circuit_t *circ, const char *request, return -1; } -/** Find all the apconns in state AP_CONN_STATE_RENDDESC_WAIT that - * are waiting on query. If there's a working cache entry here - * with at least one intro point, move them to the next state. If - * <b>rend_version</b> is non-negative, fail connections that have - * requested <b>query</b> unless there are still descriptor fetch - * requests in progress for other descriptor versions than - * <b>rend_version</b>. - */ +/** Find all the apconns in state AP_CONN_STATE_RENDDESC_WAIT that are + * waiting on <b>query</b>. If there's a working cache entry here with at + * least one intro point, move them to the next state. */ void -rend_client_desc_trynow(const char *query, int rend_version) +rend_client_desc_trynow(const char *query) { edge_connection_t *conn; rend_cache_entry_t *entry; time_t now = time(NULL); smartlist_t *conns = get_connection_array(); - SMARTLIST_FOREACH(conns, connection_t *, _conn, - { + SMARTLIST_FOREACH_BEGIN(conns, connection_t *, _conn) { if (_conn->type != CONN_TYPE_AP || _conn->state != AP_CONN_STATE_RENDDESC_WAIT || _conn->marked_for_close) @@ -770,17 +720,12 @@ rend_client_desc_trynow(const char *query, int rend_version) connection_mark_unattached_ap(conn, END_STREAM_REASON_CANT_ATTACH); } } else { /* 404, or fetch didn't get that far */ - /* Unless there are requests for another descriptor version pending, - * close the connection. */ - if (rend_version >= 0 && - !connection_get_by_type_state_rendquery(CONN_TYPE_DIR, 0, query, - rend_version == 0 ? 2 : 0)) { - log_notice(LD_REND,"Closing stream for '%s.onion': hidden service is " - "unavailable (try again later).", safe_str(query)); - connection_mark_unattached_ap(conn, END_STREAM_REASON_RESOLVEFAILED); - } + log_notice(LD_REND,"Closing stream for '%s.onion': hidden service is " + "unavailable (try again later).", + safe_str_client(query)); + connection_mark_unattached_ap(conn, END_STREAM_REASON_RESOLVEFAILED); } - }); + } SMARTLIST_FOREACH_END(_conn); } /** Return a newly allocated extend_info_t* for a randomly chosen introduction @@ -798,7 +743,7 @@ rend_client_get_random_intro(const rend_data_t *rend_query) if (rend_cache_lookup_entry(rend_query->onion_address, -1, &entry) < 1) { log_warn(LD_REND, "Query '%s' didn't have valid rend desc in cache. Failing.", - safe_str(rend_query->onion_address)); + safe_str_client(rend_query->onion_address)); return NULL; } @@ -948,8 +893,7 @@ rend_parse_service_authorization(or_options_t *options, int validate_only) err: res = -1; done: - if (auth) - rend_service_authorization_free(auth); + rend_service_authorization_free(auth); SMARTLIST_FOREACH(sl, char *, c, tor_free(c);); smartlist_free(sl); if (!validate_only && res == 0) { diff --git a/src/or/rendclient.h b/src/or/rendclient.h new file mode 100644 index 0000000000..842fe0b8ce --- /dev/null +++ b/src/or/rendclient.h @@ -0,0 +1,40 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2010, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file rendclient.h + * \brief Header file for rendclient.c. + **/ + +#ifndef _TOR_RENDCLIENT_H +#define _TOR_RENDCLIENT_H + +void rend_client_introcirc_has_opened(origin_circuit_t *circ); +void rend_client_rendcirc_has_opened(origin_circuit_t *circ); +int rend_client_introduction_acked(origin_circuit_t *circ, const char *request, + size_t request_len); +void rend_client_refetch_v2_renddesc(const rend_data_t *rend_query); +int rend_client_remove_intro_point(extend_info_t *failed_intro, + const rend_data_t *rend_query); +int rend_client_rendezvous_acked(origin_circuit_t *circ, const char *request, + size_t request_len); +int rend_client_receive_rendezvous(origin_circuit_t *circ, const char *request, + size_t request_len); +void rend_client_desc_trynow(const char *query); + +extend_info_t *rend_client_get_random_intro(const rend_data_t *rend_query); + +int rend_client_send_introduction(origin_circuit_t *introcirc, + origin_circuit_t *rendcirc); +int rend_parse_service_authorization(or_options_t *options, + int validate_only); +rend_service_authorization_t *rend_client_lookup_service_authorization( + const char *onion_address); +void rend_service_authorization_free_all(void); +rend_data_t *rend_data_dup(const rend_data_t *request); + +#endif + diff --git a/src/or/rendcommon.c b/src/or/rendcommon.c index 1f9596de2e..ec6680b1e6 100644 --- a/src/or/rendcommon.c +++ b/src/or/rendcommon.c @@ -9,6 +9,15 @@ **/ #include "or.h" +#include "circuitbuild.h" +#include "config.h" +#include "rendclient.h" +#include "rendcommon.h" +#include "rendmid.h" +#include "rendservice.h" +#include "rephist.h" +#include "routerlist.h" +#include "routerparse.h" /** Return 0 if one and two are the same service ids, else -1 or 1 */ int @@ -22,6 +31,8 @@ rend_cmp_service_ids(const char *one, const char *two) void rend_service_descriptor_free(rend_service_descriptor_t *desc) { + if (!desc) + return; if (desc->pk) crypto_free_pk_env(desc->pk); if (desc->intro_nodes) { @@ -125,7 +136,8 @@ rend_compute_v2_desc_id(char *desc_id_out, const char *service_id, if (!service_id || strlen(service_id) != REND_SERVICE_ID_LEN_BASE32) { log_warn(LD_REND, "Could not compute v2 descriptor ID: " - "Illegal service ID: %s", safe_str(service_id)); + "Illegal service ID: %s", + safe_str(service_id)); return -1; } if (replica >= REND_NUMBER_OF_NON_CONSECUTIVE_REPLICAS) { @@ -138,7 +150,7 @@ rend_compute_v2_desc_id(char *desc_id_out, const char *service_id, service_id, REND_SERVICE_ID_LEN_BASE32) < 0) { log_warn(LD_REND, "Could not compute v2 descriptor ID: " "Illegal characters in service ID: %s", - safe_str(service_id)); + safe_str_client(service_id)); return -1; } /* Calculate current time-period. */ @@ -403,8 +415,7 @@ rend_desc_v2_is_parsable(rend_encoded_v2_service_descriptor_t *desc) &test_intro_size, &test_encoded_size, &test_next, desc->desc_str); - if (test_parsed) - rend_service_descriptor_free(test_parsed); + rend_service_descriptor_free(test_parsed); tor_free(test_intro_content); return (res >= 0); } @@ -414,6 +425,8 @@ void rend_encoded_v2_service_descriptor_free( rend_encoded_v2_service_descriptor_t *desc) { + if (!desc) + return; tor_free(desc->desc_str); tor_free(desc); } @@ -422,10 +435,11 @@ rend_encoded_v2_service_descriptor_free( void rend_intro_point_free(rend_intro_point_t *intro) { - if (intro->extend_info) - extend_info_free(intro->extend_info); - if (intro->intro_key) - crypto_free_pk_env(intro->intro_key); + if (!intro) + return; + + extend_info_free(intro->extend_info); + crypto_free_pk_env(intro->intro_key); tor_free(intro); } @@ -618,7 +632,8 @@ rend_encode_v2_descriptors(smartlist_t *descs_out, } if (router_append_dirobj_signature(desc_str + written, desc_len - written, - desc_digest, service_key) < 0) { + desc_digest, DIGEST_LEN, + service_key) < 0) { log_warn(LD_BUG, "Couldn't sign desc."); rend_encoded_v2_service_descriptor_free(enc); goto err; @@ -655,61 +670,6 @@ rend_encode_v2_descriptors(smartlist_t *descs_out, return seconds_valid; } -/** Encode a service descriptor for <b>desc</b>, and sign it with - * <b>key</b>. Store the descriptor in *<b>str_out</b>, and set - * *<b>len_out</b> to its length. - */ -int -rend_encode_service_descriptor(rend_service_descriptor_t *desc, - crypto_pk_env_t *key, - char **str_out, size_t *len_out) -{ - char *cp; - char *end; - int i, r; - size_t asn1len; - size_t buflen = - PK_BYTES*2*(smartlist_len(desc->intro_nodes)+2);/*Too long, but ok*/ - cp = *str_out = tor_malloc(buflen); - end = cp + PK_BYTES*2*(smartlist_len(desc->intro_nodes)+1); - r = crypto_pk_asn1_encode(desc->pk, cp+2, end-(cp+2)); - if (r < 0) { - tor_free(*str_out); - return -1; - } - asn1len = r; - set_uint16(cp, htons((uint16_t)asn1len)); - cp += 2+asn1len; - set_uint32(cp, htonl((uint32_t)desc->timestamp)); - cp += 4; - set_uint16(cp, htons((uint16_t)smartlist_len(desc->intro_nodes))); - cp += 2; - for (i=0; i < smartlist_len(desc->intro_nodes); ++i) { - rend_intro_point_t *intro = smartlist_get(desc->intro_nodes, i); - char ipoint[HEX_DIGEST_LEN+2]; - const size_t ipoint_len = HEX_DIGEST_LEN+1; - ipoint[0] = '$'; - base16_encode(ipoint+1, HEX_DIGEST_LEN+1, - intro->extend_info->identity_digest, - DIGEST_LEN); - tor_assert(strlen(ipoint) == ipoint_len); - /* Assert that appending ipoint and its NUL won't over overrun the - * buffer. */ - tor_assert(cp + ipoint_len+1 < *str_out + buflen); - memcpy(cp, ipoint, ipoint_len+1); - cp += ipoint_len+1; - } - note_crypto_pk_op(REND_SERVER); - r = crypto_pk_private_sign_digest(key, cp, *str_out, cp-*str_out); - if (r<0) { - tor_free(*str_out); - return -1; - } - cp += r; - *len_out = (size_t)(cp-*str_out); - return 0; -} - /** Parse a service descriptor at <b>str</b> (<b>len</b> bytes). On * success, return a newly alloced service_descriptor_t. On failure, * return NULL. @@ -826,22 +786,27 @@ rend_cache_init(void) /** Helper: free storage held by a single service descriptor cache entry. */ static void -_rend_cache_entry_free(void *p) +rend_cache_entry_free(rend_cache_entry_t *e) { - rend_cache_entry_t *e = p; + if (!e) + return; rend_service_descriptor_free(e->parsed); tor_free(e->desc); tor_free(e); } +static void +_rend_cache_entry_free(void *p) +{ + rend_cache_entry_free(p); +} + /** Free all storage held by the service descriptor cache. */ void rend_cache_free_all(void) { - if (rend_cache) - strmap_free(rend_cache, _rend_cache_entry_free); - if (rend_cache_v2_dir) - digestmap_free(rend_cache_v2_dir, _rend_cache_entry_free); + strmap_free(rend_cache, _rend_cache_entry_free); + digestmap_free(rend_cache_v2_dir, _rend_cache_entry_free); rend_cache = NULL; rend_cache_v2_dir = NULL; } @@ -862,7 +827,7 @@ rend_cache_clean(void) ent = (rend_cache_entry_t*)val; if (ent->parsed->timestamp < cutoff) { iter = strmap_iter_next_rmv(rend_cache, iter); - _rend_cache_entry_free(ent); + rend_cache_entry_free(ent); } else { iter = strmap_iter_next(rend_cache, iter); } @@ -888,9 +853,9 @@ rend_cache_clean_v2_descs_as_dir(void) char key_base32[REND_DESC_ID_V2_LEN_BASE32 + 1]; base32_encode(key_base32, sizeof(key_base32), key, DIGEST_LEN); log_info(LD_REND, "Removing descriptor with ID '%s' from cache", - safe_str(key_base32)); + safe_str_client(key_base32)); iter = digestmap_iter_next_rmv(rend_cache_v2_dir, iter); - _rend_cache_entry_free(ent); + rend_cache_entry_free(ent); } else { iter = digestmap_iter_next(rend_cache_v2_dir, iter); } @@ -966,6 +931,11 @@ rend_cache_lookup_entry(const char *query, int version, rend_cache_entry_t **e) } if (!*e) return 0; + tor_assert((*e)->parsed && (*e)->parsed->intro_nodes); + /* XXX022 hack for now, to return "not found" if there are no intro + * points remaining. See bug 997. */ + if (smartlist_len((*e)->parsed->intro_nodes) == 0) + return 0; return 1; } @@ -1044,7 +1014,6 @@ rend_cache_store(const char *desc, size_t desc_len, int published) char query[REND_SERVICE_ID_LEN_BASE32+1]; char key[REND_SERVICE_ID_LEN_BASE32+2]; /* 0<query>\0 */ time_t now; - or_options_t *options = get_options(); tor_assert(rend_cache); parsed = rend_parse_service_descriptor(desc,desc_len); if (!parsed) { @@ -1059,13 +1028,15 @@ rend_cache_store(const char *desc, size_t desc_len, int published) now = time(NULL); if (parsed->timestamp < now-REND_CACHE_MAX_AGE-REND_CACHE_MAX_SKEW) { log_fn(LOG_PROTOCOL_WARN, LD_REND, - "Service descriptor %s is too old.", safe_str(query)); + "Service descriptor %s is too old.", + safe_str_client(query)); rend_service_descriptor_free(parsed); return -2; } if (parsed->timestamp > now+REND_CACHE_MAX_SKEW) { log_fn(LOG_PROTOCOL_WARN, LD_REND, - "Service descriptor %s is too far in the future.", safe_str(query)); + "Service descriptor %s is too far in the future.", + safe_str_client(query)); rend_service_descriptor_free(parsed); return -2; } @@ -1073,25 +1044,22 @@ rend_cache_store(const char *desc, size_t desc_len, int published) tor_snprintf(key, sizeof(key), "2%s", query); if (!published && strmap_get_lc(rend_cache, key)) { log_info(LD_REND, "We already have a v2 descriptor for service %s.", - safe_str(query)); + safe_str_client(query)); rend_service_descriptor_free(parsed); return -1; } - /* report novel publication to statistics */ - if (published && options->HSAuthorityRecordStats) { - hs_usage_note_publish_total(query, now); - } tor_snprintf(key, sizeof(key), "0%s", query); e = (rend_cache_entry_t*) strmap_get_lc(rend_cache, key); if (e && e->parsed->timestamp > parsed->timestamp) { log_info(LD_REND,"We already have a newer service descriptor %s with the " - "same ID and version.", safe_str(query)); + "same ID and version.", + safe_str_client(query)); rend_service_descriptor_free(parsed); return 0; } if (e && e->len == desc_len && !memcmp(desc,e->desc,desc_len)) { log_info(LD_REND,"We already have this service descriptor %s.", - safe_str(query)); + safe_str_client(query)); e->received = time(NULL); rend_service_descriptor_free(parsed); return 0; @@ -1099,10 +1067,6 @@ rend_cache_store(const char *desc, size_t desc_len, int published) if (!e) { e = tor_malloc_zero(sizeof(rend_cache_entry_t)); strmap_set_lc(rend_cache, key, e); - /* report novel publication to statistics */ - if (published && options->HSAuthorityRecordStats) { - hs_usage_note_publish_novel(query, now); - } } else { rend_service_descriptor_free(e->parsed); tor_free(e->desc); @@ -1114,7 +1078,7 @@ rend_cache_store(const char *desc, size_t desc_len, int published) memcpy(e->desc, desc, desc_len); log_debug(LD_REND,"Successfully stored rend desc '%s', len %d.", - safe_str(query), (int)desc_len); + safe_str_client(query), (int)desc_len); return 1; } @@ -1165,7 +1129,7 @@ rend_cache_store_v2_desc_as_dir(const char *desc) if (!hid_serv_responsible_for_desc_id(desc_id)) { log_info(LD_REND, "Service descriptor with desc ID %s is not in " "interval that we are responsible for.", - safe_str(desc_id_base32)); + safe_str_client(desc_id_base32)); goto skip; } /* Is descriptor too old? */ @@ -1294,7 +1258,8 @@ rend_cache_store_v2_desc_as_client(const char *desc, /* Decode/decrypt introduction points. */ if (intro_content) { if (rend_query->auth_type != REND_NO_AUTH && - rend_query->descriptor_cookie) { + !tor_mem_is_zero(rend_query->descriptor_cookie, + sizeof(rend_query->descriptor_cookie))) { char *ipos_decrypted = NULL; size_t ipos_decrypted_size; if (rend_decrypt_introduction_points(&ipos_decrypted, @@ -1329,14 +1294,14 @@ rend_cache_store_v2_desc_as_client(const char *desc, /* Is descriptor too old? */ if (parsed->timestamp < now - REND_CACHE_MAX_AGE-REND_CACHE_MAX_SKEW) { log_warn(LD_REND, "Service descriptor with service ID %s is too old.", - safe_str(service_id)); + safe_str_client(service_id)); retval = -2; goto err; } /* Is descriptor too far in the future? */ if (parsed->timestamp > now + REND_CACHE_MAX_SKEW) { log_warn(LD_REND, "Service descriptor with service ID %s is too far in " - "the future.", safe_str(service_id)); + "the future.", safe_str_client(service_id)); retval = -2; goto err; } @@ -1344,7 +1309,7 @@ rend_cache_store_v2_desc_as_client(const char *desc, tor_snprintf(key, sizeof(key), "0%s", service_id); if (strmap_get_lc(rend_cache, key)) { log_info(LD_REND, "We already have a v0 descriptor for service ID %s.", - safe_str(service_id)); + safe_str_client(service_id)); retval = -1; goto err; } @@ -1354,14 +1319,14 @@ rend_cache_store_v2_desc_as_client(const char *desc, if (e && e->parsed->timestamp > parsed->timestamp) { log_info(LD_REND, "We already have a newer service descriptor for " "service ID %s with the same desc ID and version.", - safe_str(service_id)); + safe_str_client(service_id)); retval = 0; goto err; } /* Do we already have this descriptor? */ if (e && !strcmp(desc, e->desc)) { log_info(LD_REND,"We already have this service descriptor %s.", - safe_str(service_id)); + safe_str_client(service_id)); e->received = time(NULL); retval = 0; goto err; @@ -1379,12 +1344,11 @@ rend_cache_store_v2_desc_as_client(const char *desc, strlcpy(e->desc, desc, encoded_size + 1); e->len = encoded_size; log_debug(LD_REND,"Successfully stored rend desc '%s', len %d.", - safe_str(service_id), (int)encoded_size); + safe_str_client(service_id), (int)encoded_size); return 1; err: - if (parsed) - rend_service_descriptor_free(parsed); + rend_service_descriptor_free(parsed); tor_free(intro_content); return retval; } diff --git a/src/or/rendcommon.h b/src/or/rendcommon.h new file mode 100644 index 0000000000..00705022a2 --- /dev/null +++ b/src/or/rendcommon.h @@ -0,0 +1,64 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2010, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file rendcommon.h + * \brief Header file for rendcommon.c. + **/ + +#ifndef _TOR_RENDCOMMON_H +#define _TOR_RENDCOMMON_H + +/** Free all storage associated with <b>data</b> */ +static INLINE void +rend_data_free(rend_data_t *data) +{ + tor_free(data); +} + +int rend_cmp_service_ids(const char *one, const char *two); + +void rend_process_relay_cell(circuit_t *circ, const crypt_path_t *layer_hint, + int command, size_t length, const char *payload); + +void rend_service_descriptor_free(rend_service_descriptor_t *desc); +rend_service_descriptor_t *rend_parse_service_descriptor(const char *str, + size_t len); +int rend_get_service_id(crypto_pk_env_t *pk, char *out); +void rend_encoded_v2_service_descriptor_free( + rend_encoded_v2_service_descriptor_t *desc); +void rend_intro_point_free(rend_intro_point_t *intro); + +void rend_cache_init(void); +void rend_cache_clean(void); +void rend_cache_clean_v2_descs_as_dir(void); +void rend_cache_free_all(void); +int rend_valid_service_id(const char *query); +int rend_cache_lookup_desc(const char *query, int version, const char **desc, + size_t *desc_len); +int rend_cache_lookup_entry(const char *query, int version, + rend_cache_entry_t **entry_out); +int rend_cache_lookup_v2_desc_as_dir(const char *query, const char **desc); +int rend_cache_store(const char *desc, size_t desc_len, int published); +int rend_cache_store_v2_desc_as_client(const char *desc, + const rend_data_t *rend_query); +int rend_cache_store_v2_desc_as_dir(const char *desc); +int rend_cache_size(void); +int rend_encode_v2_descriptors(smartlist_t *descs_out, + rend_service_descriptor_t *desc, time_t now, + uint8_t period, rend_auth_type_t auth_type, + crypto_pk_env_t *client_key, + smartlist_t *client_cookies); +int rend_compute_v2_desc_id(char *desc_id_out, const char *service_id, + const char *descriptor_cookie, + time_t now, uint8_t replica); +int rend_id_is_in_interval(const char *a, const char *b, const char *c); +void rend_get_descriptor_id_bytes(char *descriptor_id_out, + const char *service_id, + const char *secret_id_part); + +#endif + diff --git a/src/or/rendmid.c b/src/or/rendmid.c index c9f4bf8b70..d392f8e53a 100644 --- a/src/or/rendmid.c +++ b/src/or/rendmid.c @@ -8,6 +8,11 @@ **/ #include "or.h" +#include "circuitlist.h" +#include "config.h" +#include "relay.h" +#include "rendmid.h" +#include "rephist.h" /** Respond to an ESTABLISH_INTRO cell by checking the signed data and * setting the circuit's purpose and service pk digest. diff --git a/src/or/rendmid.h b/src/or/rendmid.h new file mode 100644 index 0000000000..4d08d4c8b9 --- /dev/null +++ b/src/or/rendmid.h @@ -0,0 +1,25 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2010, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file rendmid.h + * \brief Header file for rendmid.c. + **/ + +#ifndef _TOR_RENDMID_H +#define _TOR_RENDMID_H + +int rend_mid_establish_intro(or_circuit_t *circ, const char *request, + size_t request_len); +int rend_mid_introduce(or_circuit_t *circ, const char *request, + size_t request_len); +int rend_mid_establish_rendezvous(or_circuit_t *circ, const char *request, + size_t request_len); +int rend_mid_rendezvous(or_circuit_t *circ, const char *request, + size_t request_len); + +#endif + diff --git a/src/or/rendservice.c b/src/or/rendservice.c index 289d64d399..b0d791529b 100644 --- a/src/or/rendservice.c +++ b/src/or/rendservice.c @@ -8,10 +8,23 @@ **/ #include "or.h" +#include "circuitbuild.h" +#include "circuitlist.h" +#include "circuituse.h" +#include "config.h" +#include "directory.h" +#include "networkstatus.h" +#include "rendclient.h" +#include "rendcommon.h" +#include "rendservice.h" +#include "router.h" +#include "relay.h" +#include "rephist.h" +#include "routerlist.h" +#include "routerparse.h" static origin_circuit_t *find_intro_circuit(rend_intro_point_t *intro, - const char *pk_digest, - int desc_version); + const char *pk_digest); /** Represents the mapping from a virtual port of a rendezvous service to * a real port on some IP. @@ -42,8 +55,6 @@ typedef struct rend_service_t { /* Fields specified in config file */ char *directory; /**< where in the filesystem it stores it */ smartlist_t *ports; /**< List of rend_service_port_config_t */ - int descriptor_version; /**< Rendezvous descriptor version that will be - * published. */ rend_auth_type_t auth_type; /**< Client authorization type or 0 if no client * authorization is performed. */ smartlist_t *clients; /**< List of rend_authorized_client_t's of @@ -58,7 +69,7 @@ typedef struct rend_service_t { * or are trying to establish. */ time_t intro_period_started; /**< Start of the current period to build * introduction points. */ - int n_intro_circuits_launched; /**< count of intro circuits we have + int n_intro_circuits_launched; /**< Count of intro circuits we have * established in this period. */ rend_service_descriptor_t *desc; /**< Current hidden service descriptor. */ time_t desc_is_dirty; /**< Time at which changes to the hidden service @@ -90,7 +101,8 @@ num_rend_services(void) static void rend_authorized_client_free(rend_authorized_client_t *client) { - if (!client) return; + if (!client) + return; if (client->client_key) crypto_free_pk_env(client->client_key); tor_free(client->client_name); @@ -109,7 +121,9 @@ rend_authorized_client_strmap_item_free(void *authorized_client) static void rend_service_free(rend_service_t *service) { - if (!service) return; + if (!service) + return; + tor_free(service->directory); SMARTLIST_FOREACH(service->ports, void*, p, tor_free(p)); smartlist_free(service->ports); @@ -120,15 +134,14 @@ rend_service_free(rend_service_t *service) rend_intro_point_free(intro);); smartlist_free(service->intro_nodes); } - if (service->desc) - rend_service_descriptor_free(service->desc); + + rend_service_descriptor_free(service->desc); if (service->clients) { SMARTLIST_FOREACH(service->clients, rend_authorized_client_t *, c, rend_authorized_client_free(c);); smartlist_free(service->clients); } - if (service->accepted_intros) - digestmap_free(service->accepted_intros, _tor_free); + digestmap_free(service->accepted_intros, _tor_free); tor_free(service); } @@ -137,9 +150,9 @@ rend_service_free(rend_service_t *service) void rend_service_free_all(void) { - if (!rend_service_list) { + if (!rend_service_list) return; - } + SMARTLIST_FOREACH(rend_service_list, rend_service_t*, ptr, rend_service_free(ptr)); smartlist_free(rend_service_list); @@ -156,36 +169,6 @@ rend_add_service(rend_service_t *service) service->intro_nodes = smartlist_create(); - /* If the service is configured to publish unversioned (v0) and versioned - * descriptors (v2 or higher), split it up into two separate services - * (unless it is configured to perform client authorization). */ - if (service->descriptor_version == -1) { - if (service->auth_type == REND_NO_AUTH) { - rend_service_t *v0_service = tor_malloc_zero(sizeof(rend_service_t)); - v0_service->directory = tor_strdup(service->directory); - v0_service->ports = smartlist_create(); - SMARTLIST_FOREACH(service->ports, rend_service_port_config_t *, p, { - rend_service_port_config_t *copy = - tor_malloc_zero(sizeof(rend_service_port_config_t)); - memcpy(copy, p, sizeof(rend_service_port_config_t)); - smartlist_add(v0_service->ports, copy); - }); - v0_service->intro_period_started = service->intro_period_started; - v0_service->descriptor_version = 0; /* Unversioned descriptor. */ - v0_service->auth_type = REND_NO_AUTH; - rend_add_service(v0_service); - } - - service->descriptor_version = 2; /* Versioned descriptor. */ - } - - if (service->auth_type != REND_NO_AUTH && !service->descriptor_version) { - log_warn(LD_CONFIG, "Hidden service with client authorization and " - "version 0 descriptors configured; ignoring."); - rend_service_free(service); - return; - } - if (service->auth_type != REND_NO_AUTH && smartlist_len(service->clients) == 0) { log_warn(LD_CONFIG, "Hidden service with client authorization but no " @@ -297,7 +280,7 @@ rend_config_services(or_options_t *options, int validate_only) for (line = options->RendConfigLines; line; line = line->next) { if (!strcasecmp(line->key, "HiddenServiceDir")) { - if (service) { + if (service) { /* register the one we just finished parsing */ if (validate_only) rend_service_free(service); else @@ -307,7 +290,6 @@ rend_config_services(or_options_t *options, int validate_only) service->directory = tor_strdup(line->value); service->ports = smartlist_create(); service->intro_period_started = time(NULL); - service->descriptor_version = -1; /**< All descriptor versions. */ continue; } if (!service) { @@ -433,35 +415,13 @@ rend_config_services(or_options_t *options, int validate_only) return -1; } } else { - smartlist_t *versions; - char *version_str; - int i, version, ver_ok=1, versions_bitmask = 0; tor_assert(!strcasecmp(line->key, "HiddenServiceVersion")); - versions = smartlist_create(); - smartlist_split_string(versions, line->value, ",", - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); - for (i = 0; i < smartlist_len(versions); i++) { - version_str = smartlist_get(versions, i); - if (strlen(version_str) != 1 || strspn(version_str, "02") != 1) { - log_warn(LD_CONFIG, - "HiddenServiceVersion can only be 0 and/or 2."); - SMARTLIST_FOREACH(versions, char *, cp, tor_free(cp)); - smartlist_free(versions); - rend_service_free(service); - return -1; - } - version = (int)tor_parse_long(version_str, 10, 0, INT_MAX, &ver_ok, - NULL); - if (!ver_ok) - continue; - versions_bitmask |= 1 << version; + if (strcmp(line->value, "2")) { + log_warn(LD_CONFIG, + "The only supported HiddenServiceVersion is 2."); + rend_service_free(service); + return -1; } - /* If exactly one version is set, change descriptor_version to that - * value; otherwise leave it at -1. */ - if (versions_bitmask == 1 << 0) service->descriptor_version = 0; - if (versions_bitmask == 1 << 2) service->descriptor_version = 2; - SMARTLIST_FOREACH(versions, char *, cp, tor_free(cp)); - smartlist_free(versions); } } if (service) { @@ -483,8 +443,7 @@ rend_config_services(or_options_t *options, int validate_only) * probably ok? */ SMARTLIST_FOREACH(rend_service_list, rend_service_t *, new, { SMARTLIST_FOREACH(old_service_list, rend_service_t *, old, { - if (!strcmp(old->directory, new->directory) && - old->descriptor_version == new->descriptor_version) { + if (!strcmp(old->directory, new->directory)) { smartlist_add_all(new->intro_nodes, old->intro_nodes); smartlist_clear(old->intro_nodes); smartlist_add(surviving_services, old); @@ -507,18 +466,16 @@ rend_config_services(or_options_t *options, int validate_only) tor_assert(oc->rend_data); SMARTLIST_FOREACH(surviving_services, rend_service_t *, ptr, { if (!memcmp(ptr->pk_digest, oc->rend_data->rend_pk_digest, - DIGEST_LEN) && - ptr->descriptor_version == oc->rend_data->rend_desc_version) { + DIGEST_LEN)) { keep_it = 1; break; } }); if (keep_it) continue; - log_info(LD_REND, "Closing intro point %s for service %s version %d.", - safe_str(oc->build_state->chosen_exit->nickname), - oc->rend_data->onion_address, - oc->rend_data->rend_desc_version); + log_info(LD_REND, "Closing intro point %s for service %s.", + safe_str_client(oc->build_state->chosen_exit->nickname), + oc->rend_data->onion_address); circuit_mark_for_close(circ, END_CIRC_REASON_FINISHED); /* XXXX Is there another reason we should use here? */ } @@ -541,14 +498,13 @@ rend_service_update_descriptor(rend_service_t *service) rend_service_descriptor_t *d; origin_circuit_t *circ; int i; - if (service->desc) { - rend_service_descriptor_free(service->desc); - service->desc = NULL; - } + + rend_service_descriptor_free(service->desc); + service->desc = NULL; + d = service->desc = tor_malloc_zero(sizeof(rend_service_descriptor_t)); d->pk = crypto_pk_dup_key(service->private_key); d->timestamp = time(NULL); - d->version = service->descriptor_version; d->intro_nodes = smartlist_create(); /* Support intro protocols 2 and 3. */ d->protocols = (1 << 2) + (1 << 3); @@ -556,7 +512,7 @@ rend_service_update_descriptor(rend_service_t *service) for (i = 0; i < smartlist_len(service->intro_nodes); ++i) { rend_intro_point_t *intro_svc = smartlist_get(service->intro_nodes, i); rend_intro_point_t *intro_desc; - circ = find_intro_circuit(intro_svc, service->pk_digest, d->version); + circ = find_intro_circuit(intro_svc, service->pk_digest); if (!circ || circ->_base.purpose != CIRCUIT_PURPOSE_S_INTRO) continue; @@ -797,17 +753,15 @@ rend_service_load_keys(void) return r; } -/** Return the service whose public key has a digest of <b>digest</b> and - * which publishes the given descriptor <b>version</b>. Return NULL if no - * such service exists. +/** Return the service whose public key has a digest of <b>digest</b>, or + * NULL if no such service exists. */ static rend_service_t * -rend_service_get_by_pk_digest_and_version(const char* digest, - uint8_t version) +rend_service_get_by_pk_digest(const char* digest) { SMARTLIST_FOREACH(rend_service_list, rend_service_t*, s, - if (!memcmp(s->pk_digest,digest,DIGEST_LEN) && - s->descriptor_version == version) return s); + if (!memcmp(s->pk_digest,digest,DIGEST_LEN)) + return s); return NULL; } @@ -944,21 +898,16 @@ rend_service_introduce(origin_circuit_t *circuit, const char *request, } /* look up service depending on circuit. */ - service = rend_service_get_by_pk_digest_and_version( - circuit->rend_data->rend_pk_digest, - circuit->rend_data->rend_desc_version); + service = rend_service_get_by_pk_digest( + circuit->rend_data->rend_pk_digest); if (!service) { log_warn(LD_REND, "Got an INTRODUCE2 cell for an unrecognized service %s.", escaped(serviceid)); return -1; } - /* if descriptor version is 2, use intro key instead of service key. */ - if (circuit->rend_data->rend_desc_version == 0) { - intro_key = service->private_key; - } else { - intro_key = circuit->intro_key; - } + /* use intro key instead of service key. */ + intro_key = circuit->intro_key; /* first DIGEST_LEN bytes of request is intro or service pk digest */ crypto_pk_get_digest(intro_key, intro_key_digest); @@ -988,7 +937,7 @@ rend_service_introduce(origin_circuit_t *circuit, const char *request, len = r; if (*buf == 3) { /* Version 3 INTRODUCE2 cell. */ - time_t ts = 0, now = time(NULL); + time_t ts = 0; v3_shift = 1; auth_type = buf[1]; switch (auth_type) { @@ -1082,7 +1031,7 @@ rend_service_introduce(origin_circuit_t *circuit, const char *request, router = router_get_by_nickname(rp_nickname, 0); if (!router) { log_info(LD_REND, "Couldn't find router %s named in introduce2 cell.", - escaped_safe_str(rp_nickname)); + escaped_safe_str_client(rp_nickname)); /* XXXX Add a no-such-router reason? */ reason = END_CIRC_REASON_TORPROTOCOL; goto err; @@ -1157,7 +1106,8 @@ rend_service_introduce(origin_circuit_t *circuit, const char *request, reason = END_CIRC_REASON_INTERNAL; goto err; } - if (crypto_dh_compute_secret(dh, ptr+REND_COOKIE_LEN, DH_KEY_LEN, keys, + if (crypto_dh_compute_secret(LOG_PROTOCOL_WARN, dh, ptr+REND_COOKIE_LEN, + DH_KEY_LEN, keys, DIGEST_LEN+CPATH_KEY_MATERIAL_LEN)<0) { log_warn(LD_BUG, "Internal error: couldn't complete DH handshake"); reason = END_CIRC_REASON_INTERNAL; @@ -1167,7 +1117,7 @@ rend_service_introduce(origin_circuit_t *circuit, const char *request, circ_needs_uptime = rend_service_requires_uptime(service); /* help predict this next time */ - rep_hist_note_used_internal(time(NULL), circ_needs_uptime, 1); + rep_hist_note_used_internal(now, circ_needs_uptime, 1); /* Launch a circuit to alice's chosen rendezvous point. */ @@ -1183,14 +1133,16 @@ rend_service_introduce(origin_circuit_t *circuit, const char *request, if (!launched) { /* give up */ log_warn(LD_REND, "Giving up launching first hop of circuit to rendezvous " "point %s for service %s.", - escaped_safe_str(extend_info->nickname), serviceid); + escaped_safe_str_client(extend_info->nickname), + serviceid); reason = END_CIRC_REASON_CONNECTFAILED; goto err; } log_info(LD_REND, "Accepted intro; launching circuit to %s " "(cookie %s) for service %s.", - escaped_safe_str(extend_info->nickname), hexcookie, serviceid); + escaped_safe_str_client(extend_info->nickname), + hexcookie, serviceid); tor_assert(launched->build_state); /* Fill in the circuit's state. */ launched->rend_data = tor_malloc_zero(sizeof(rend_data_t)); @@ -1200,11 +1152,10 @@ rend_service_introduce(origin_circuit_t *circuit, const char *request, memcpy(launched->rend_data->rend_cookie, r_cookie, REND_COOKIE_LEN); strlcpy(launched->rend_data->onion_address, service->service_id, sizeof(launched->rend_data->onion_address)); - launched->rend_data->rend_desc_version = service->descriptor_version; launched->build_state->pending_final_cpath = cpath = tor_malloc_zero(sizeof(crypt_path_t)); cpath->magic = CRYPT_PATH_MAGIC; - launched->build_state->expiry_time = time(NULL) + MAX_REND_TIMEOUT; + launched->build_state->expiry_time = now + MAX_REND_TIMEOUT; cpath->dh_handshake_state = dh; dh = NULL; @@ -1286,7 +1237,7 @@ rend_service_launch_establish_intro(rend_service_t *service, log_info(LD_REND, "Launching circuit to introduction point %s for service %s", - escaped_safe_str(intro->extend_info->nickname), + escaped_safe_str_client(intro->extend_info->nickname), service->service_id); rep_hist_note_used_internal(time(NULL), 1, 0); @@ -1299,7 +1250,7 @@ rend_service_launch_establish_intro(rend_service_t *service, if (!launched) { log_info(LD_REND, "Can't launch circuit to establish introduction at %s.", - escaped_safe_str(intro->extend_info->nickname)); + escaped_safe_str_client(intro->extend_info->nickname)); return -1; } @@ -1322,18 +1273,16 @@ rend_service_launch_establish_intro(rend_service_t *service, strlcpy(launched->rend_data->onion_address, service->service_id, sizeof(launched->rend_data->onion_address)); memcpy(launched->rend_data->rend_pk_digest, service->pk_digest, DIGEST_LEN); - launched->rend_data->rend_desc_version = service->descriptor_version; - if (service->descriptor_version == 2) - launched->intro_key = crypto_pk_dup_key(intro->intro_key); + launched->intro_key = crypto_pk_dup_key(intro->intro_key); if (launched->_base.state == CIRCUIT_STATE_OPEN) rend_service_intro_has_opened(launched); return 0; } /** Return the number of introduction points that are or have been - * established for the given service address and rendezvous version. */ + * established for the given service address in <b>query</b>. */ static int -count_established_intro_points(const char *query, int rend_version) +count_established_intro_points(const char *query) { int num_ipos = 0; circuit_t *circ; @@ -1344,7 +1293,6 @@ count_established_intro_points(const char *query, int rend_version) circ->purpose == CIRCUIT_PURPOSE_S_INTRO)) { origin_circuit_t *oc = TO_ORIGIN_CIRCUIT(circ); if (oc->rend_data && - oc->rend_data->rend_desc_version == rend_version && !rend_cmp_service_ids(query, oc->rend_data->onion_address)) num_ipos++; } @@ -1374,9 +1322,8 @@ rend_service_intro_has_opened(origin_circuit_t *circuit) base32_encode(serviceid, REND_SERVICE_ID_LEN_BASE32+1, circuit->rend_data->rend_pk_digest, REND_SERVICE_ID_LEN); - service = rend_service_get_by_pk_digest_and_version( - circuit->rend_data->rend_pk_digest, - circuit->rend_data->rend_desc_version); + service = rend_service_get_by_pk_digest( + circuit->rend_data->rend_pk_digest); if (!service) { log_warn(LD_REND, "Unrecognized service ID %s on introduction circuit %d.", serviceid, circuit->_base.n_circ_id); @@ -1386,8 +1333,7 @@ rend_service_intro_has_opened(origin_circuit_t *circuit) /* If we already have enough introduction circuits for this service, * redefine this one as a general circuit. */ - if (count_established_intro_points(serviceid, - circuit->rend_data->rend_desc_version) > NUM_INTRO_POINTS) { + if (count_established_intro_points(serviceid) > NUM_INTRO_POINTS) { log_info(LD_CIRC|LD_REND, "We have just finished an introduction " "circuit, but we already have enough. Redefining purpose to " "general."); @@ -1400,13 +1346,8 @@ rend_service_intro_has_opened(origin_circuit_t *circuit) "Established circuit %d as introduction point for service %s", circuit->_base.n_circ_id, serviceid); - /* If the introduction point will not be used in an unversioned - * descriptor, use the intro key instead of the service key in - * ESTABLISH_INTRO. */ - if (service->descriptor_version == 0) - intro_key = service->private_key; - else - intro_key = circuit->intro_key; + /* Use the intro key instead of the service key in ESTABLISH_INTRO. */ + intro_key = circuit->intro_key; /* Build the payload for a RELAY_ESTABLISH_INTRO cell. */ r = crypto_pk_asn1_encode(intro_key, buf+2, RELAY_PAYLOAD_SIZE-2); @@ -1465,9 +1406,8 @@ rend_service_intro_established(origin_circuit_t *circuit, const char *request, goto err; } tor_assert(circuit->rend_data); - service = rend_service_get_by_pk_digest_and_version( - circuit->rend_data->rend_pk_digest, - circuit->rend_data->rend_desc_version); + service = rend_service_get_by_pk_digest( + circuit->rend_data->rend_pk_digest); if (!service) { log_warn(LD_REND, "Unknown service on introduction circuit %d.", circuit->_base.n_circ_id); @@ -1517,9 +1457,8 @@ rend_service_rendezvous_has_opened(origin_circuit_t *circuit) "cookie %s for service %s", circuit->_base.n_circ_id, hexcookie, serviceid); - service = rend_service_get_by_pk_digest_and_version( - circuit->rend_data->rend_pk_digest, - circuit->rend_data->rend_desc_version); + service = rend_service_get_by_pk_digest( + circuit->rend_data->rend_pk_digest); if (!service) { log_warn(LD_GENERAL, "Internal error: unrecognized service ID on " "introduction circuit."); @@ -1575,13 +1514,12 @@ rend_service_rendezvous_has_opened(origin_circuit_t *circuit) */ /** Return the (possibly non-open) introduction circuit ending at - * <b>intro</b> for the service whose public key is <b>pk_digest</b> and - * which publishes descriptor of version <b>desc_version</b>. Return - * NULL if no such service is found. + * <b>intro</b> for the service whose public key is <b>pk_digest</b>. + * (<b>desc_version</b> is ignored). Return NULL if no such service is + * found. */ static origin_circuit_t * -find_intro_circuit(rend_intro_point_t *intro, const char *pk_digest, - int desc_version) +find_intro_circuit(rend_intro_point_t *intro, const char *pk_digest) { origin_circuit_t *circ = NULL; @@ -1590,8 +1528,7 @@ find_intro_circuit(rend_intro_point_t *intro, const char *pk_digest, CIRCUIT_PURPOSE_S_INTRO))) { if (!memcmp(circ->build_state->chosen_exit->identity_digest, intro->extend_info->identity_digest, DIGEST_LEN) && - circ->rend_data && - circ->rend_data->rend_desc_version == desc_version) { + circ->rend_data) { return circ; } } @@ -1601,8 +1538,7 @@ find_intro_circuit(rend_intro_point_t *intro, const char *pk_digest, CIRCUIT_PURPOSE_S_ESTABLISH_INTRO))) { if (!memcmp(circ->build_state->chosen_exit->identity_digest, intro->extend_info->identity_digest, DIGEST_LEN) && - circ->rend_data && - circ->rend_data->rend_desc_version == desc_version) { + circ->rend_data) { return circ; } } @@ -1635,6 +1571,7 @@ directory_post_to_hs_dir(rend_service_descriptor_t *renddesc, } for (j = 0; j < smartlist_len(responsible_dirs); j++) { char desc_id_base32[REND_DESC_ID_V2_LEN_BASE32 + 1]; + char *hs_dir_ip; hs_dir = smartlist_get(responsible_dirs, j); if (smartlist_digest_isin(renddesc->successful_uploads, hs_dir->identity_digest)) @@ -1656,15 +1593,18 @@ directory_post_to_hs_dir(rend_service_descriptor_t *renddesc, strlen(desc->desc_str), 0); base32_encode(desc_id_base32, sizeof(desc_id_base32), desc->desc_id, DIGEST_LEN); + hs_dir_ip = tor_dup_ip(hs_dir->addr); log_info(LD_REND, "Sending publish request for v2 descriptor for " "service '%s' with descriptor ID '%s' with validity " "of %d seconds to hidden service directory '%s' on " - "port %d.", - safe_str(service_id), - safe_str(desc_id_base32), + "%s:%d.", + safe_str_client(service_id), + safe_str_client(desc_id_base32), seconds_valid, hs_dir->nickname, - hs_dir->dir_port); + hs_dir_ip, + hs_dir->or_port); + tor_free(hs_dir_ip); /* Remember successful upload to this router for next time. */ if (!smartlist_digest_isin(successful_uploads, hs_dir->identity_digest)) smartlist_add(successful_uploads, hs_dir->identity_digest); @@ -1694,9 +1634,8 @@ directory_post_to_hs_dir(rend_service_descriptor_t *renddesc, smartlist_free(successful_uploads); } -/** Encode and sign up-to-date v0 and/or v2 service descriptors for - * <b>service</b>, and upload it/them to all the dirservers/to the - * responsible hidden service directories. +/** Encode and sign an up-to-date service descriptor for <b>service</b>, + * and upload it/them to the responsible hidden service directories. */ static void upload_service_descriptor(rend_service_t *service) @@ -1708,35 +1647,8 @@ upload_service_descriptor(rend_service_t *service) rendpostperiod = get_options()->RendPostPeriod; - /* Upload unversioned (v0) descriptor? */ - if (service->descriptor_version == 0 && - get_options()->PublishHidServDescriptors) { - char *desc; - size_t desc_len; - /* Encode the descriptor. */ - if (rend_encode_service_descriptor(service->desc, - service->private_key, - &desc, &desc_len)<0) { - log_warn(LD_BUG, "Internal error: couldn't encode service descriptor; " - "not uploading."); - return; - } - - /* Post it to the dirservers */ - rend_get_service_id(service->desc->pk, serviceid); - log_info(LD_REND, "Sending publish request for hidden service %s", - serviceid); - directory_post_to_dirservers(DIR_PURPOSE_UPLOAD_RENDDESC, - ROUTER_PURPOSE_GENERAL, - HIDSERV_AUTHORITY, desc, desc_len, 0); - tor_free(desc); - service->next_upload_time = now + rendpostperiod; - uploaded = 1; - } - - /* Upload v2 descriptor? */ - if (service->descriptor_version == 2 && - get_options()->PublishHidServDescriptors) { + /* Upload descriptor? */ + if (get_options()->PublishHidServDescriptors) { networkstatus_t *c = networkstatus_get_latest_consensus(); if (c && smartlist_len(c->routerstatus_list) > 0) { int seconds_valid, i, j, num_descs; @@ -1875,8 +1787,7 @@ rend_services_introduce(void) for (j=0; j < smartlist_len(service->intro_nodes); ++j) { intro = smartlist_get(service->intro_nodes, j); router = router_get_by_digest(intro->extend_info->identity_digest); - if (!router || !find_intro_circuit(intro, service->pk_digest, - service->descriptor_version)) { + if (!router || !find_intro_circuit(intro, service->pk_digest)) { log_info(LD_REND,"Giving up on %s as intro point for %s.", intro->extend_info->nickname, service->service_id); if (service->desc) { @@ -1928,7 +1839,7 @@ rend_services_introduce(void) router_crn_flags_t flags = CRN_NEED_UPTIME; if (get_options()->_AllowInvalid & ALLOW_INVALID_INTRODUCTION) flags |= CRN_ALLOW_INVALID; - router = router_choose_random_node(NULL, intro_routers, + router = router_choose_random_node(intro_routers, options->ExcludeNodes, flags); if (!router) { log_warn(LD_REND, @@ -1940,10 +1851,8 @@ rend_services_introduce(void) smartlist_add(intro_routers, router); intro = tor_malloc_zero(sizeof(rend_intro_point_t)); intro->extend_info = extend_info_from_router(router); - if (service->descriptor_version == 2) { - intro->intro_key = crypto_new_pk_env(); - tor_assert(!crypto_pk_generate_key(intro->intro_key)); - } + intro->intro_key = crypto_new_pk_env(); + tor_assert(!crypto_pk_generate_key(intro->intro_key)); smartlist_add(service->intro_nodes, intro); log_info(LD_REND, "Picked router %s as an intro point for %s.", router->nickname, service->service_id); @@ -2036,8 +1945,7 @@ rend_consider_descriptor_republication(void) for (i=0; i < smartlist_len(rend_service_list); ++i) { service = smartlist_get(rend_service_list, i); - if (service->descriptor_version && service->desc && - !service->desc->all_uploads_performed) { + if (service->desc && !service->desc->all_uploads_performed) { /* If we failed in uploading a descriptor last time, try again *without* * updating the descriptor's contents. */ upload_service_descriptor(service); @@ -2063,10 +1971,9 @@ rend_service_dump_stats(int severity) service->directory); for (j=0; j < smartlist_len(service->intro_nodes); ++j) { intro = smartlist_get(service->intro_nodes, j); - safe_name = safe_str(intro->extend_info->nickname); + safe_name = safe_str_client(intro->extend_info->nickname); - circ = find_intro_circuit(intro, service->pk_digest, - service->descriptor_version); + circ = find_intro_circuit(intro, service->pk_digest); if (!circ) { log(severity, LD_GENERAL, " Intro point %d at %s: no circuit", j, safe_name); @@ -2097,9 +2004,8 @@ rend_service_set_connection_addr_port(edge_connection_t *conn, log_debug(LD_REND,"beginning to hunt for addr/port"); base32_encode(serviceid, REND_SERVICE_ID_LEN_BASE32+1, circ->rend_data->rend_pk_digest, REND_SERVICE_ID_LEN); - service = rend_service_get_by_pk_digest_and_version( - circ->rend_data->rend_pk_digest, - circ->rend_data->rend_desc_version); + service = rend_service_get_by_pk_digest( + circ->rend_data->rend_pk_digest); if (!service) { log_warn(LD_REND, "Couldn't find any service associated with pk %s on " "rendezvous circuit %d; closing.", diff --git a/src/or/rendservice.h b/src/or/rendservice.h new file mode 100644 index 0000000000..f979a39417 --- /dev/null +++ b/src/or/rendservice.h @@ -0,0 +1,38 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2010, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file rendservice.h + * \brief Header file for rendservice.c. + **/ + +#ifndef _TOR_RENDSERVICE_H +#define _TOR_RENDSERVICE_H + +int num_rend_services(void); +int rend_config_services(or_options_t *options, int validate_only); +int rend_service_load_keys(void); +void rend_services_init(void); +void rend_services_introduce(void); +void rend_consider_services_upload(time_t now); +void rend_hsdir_routers_changed(void); +void rend_consider_descriptor_republication(void); + +void rend_service_intro_has_opened(origin_circuit_t *circuit); +int rend_service_intro_established(origin_circuit_t *circuit, + const char *request, + size_t request_len); +void rend_service_rendezvous_has_opened(origin_circuit_t *circuit); +int rend_service_introduce(origin_circuit_t *circuit, const char *request, + size_t request_len); +void rend_service_relaunch_rendezvous(origin_circuit_t *oldcirc); +int rend_service_set_connection_addr_port(edge_connection_t *conn, + origin_circuit_t *circ); +void rend_service_dump_stats(int severity); +void rend_service_free_all(void); + +#endif + diff --git a/src/or/rephist.c b/src/or/rephist.c index 71ec0bd53c..b6a19d4b16 100644 --- a/src/or/rephist.c +++ b/src/or/rephist.c @@ -6,15 +6,21 @@ * \file rephist.c * \brief Basic history and "reputation" functionality to remember * which servers have worked in the past, how much bandwidth we've - * been using, which ports we tend to want, and so on. + * been using, which ports we tend to want, and so on; further, + * exit port statistics and cell statistics. **/ #include "or.h" +#include "circuitlist.h" +#include "circuituse.h" +#include "config.h" +#include "rephist.h" +#include "router.h" +#include "routerlist.h" #include "ht.h" static void bw_arrays_init(void); static void predicted_ports_init(void); -static void hs_usage_init(void); /** Total number of bytes currently allocated in fields used by rephist.c. */ uint64_t rephist_total_alloc=0; @@ -185,7 +191,6 @@ rep_hist_init(void) history_map = digestmap_new(); bw_arrays_init(); predicted_ports_init(); - hs_usage_init(); } /** Helper: note that we are no longer connected to the router with history @@ -795,12 +800,12 @@ rep_hist_record_mtbf_data(time_t now, int missing_means_down) static char * rep_hist_format_router_status(or_history_t *hist, time_t now) { - char buf[1024]; char sor_buf[ISO_TIME_LEN+1]; char sod_buf[ISO_TIME_LEN+1]; double wfu; double mtbf; int up = 0, down = 0; + char *cp = NULL; if (hist->start_of_run) { format_iso_time(sor_buf, hist->start_of_run); @@ -813,7 +818,7 @@ rep_hist_format_router_status(or_history_t *hist, time_t now) wfu = get_weighted_fractional_uptime(hist, now); mtbf = get_stability(hist, now); - tor_snprintf(buf, sizeof(buf), + tor_asprintf(&cp, "%s%s%s" "%s%s%s" "wfu %0.3lf\n" @@ -831,8 +836,7 @@ rep_hist_format_router_status(or_history_t *hist, time_t now) hist->weighted_run_length, hist->total_run_weights ); - - return tor_strdup(buf); + return cp; } /** The last stability analysis document that we created, or NULL if we never @@ -1419,7 +1423,7 @@ rep_hist_get_bandwidth_lines(int for_extrainfo) size_t len; /* opt (read|write)-history yyyy-mm-dd HH:MM:SS (n s) n,n,n,n,n... */ - len = (60+20*NUM_TOTALS)*2; + len = (60+21*NUM_TOTALS)*2; buf = tor_malloc_zero(len); cp = buf; for (r=0;r<2;++r) { @@ -1715,8 +1719,8 @@ rep_hist_get_predicted_internal(time_t now, int *need_uptime, return 0; /* too long ago */ if (predicted_internal_uptime_time + PREDICTED_CIRCS_RELEVANCE_TIME >= now) *need_uptime = 1; - if (predicted_internal_capacity_time + PREDICTED_CIRCS_RELEVANCE_TIME >= now) - *need_capacity = 1; + // Always predict that we need capacity. + *need_capacity = 1; return 1; } @@ -1845,564 +1849,459 @@ dump_pk_ops(int severity) pk_op_counts.n_rend_server_ops); } -/** Free all storage held by the OR/link history caches, by the - * bandwidth history arrays, or by the port history. */ +/*** Exit port statistics ***/ + +/* Some constants */ +/** To what multiple should byte numbers be rounded up? */ +#define EXIT_STATS_ROUND_UP_BYTES 1024 +/** To what multiple should stream counts be rounded up? */ +#define EXIT_STATS_ROUND_UP_STREAMS 4 +/** Number of TCP ports */ +#define EXIT_STATS_NUM_PORTS 65536 +/** Reciprocal of threshold (= 0.01%) of total bytes that a port needs to + * see in order to be included in exit stats. */ +#define EXIT_STATS_THRESHOLD_RECIPROCAL 10000 + +/* The following data structures are arrays and no fancy smartlists or maps, + * so that all write operations can be done in constant time. This comes at + * the price of some memory (1.25 MB) and linear complexity when writing + * stats for measuring relays. */ +/** Number of bytes read in current period by exit port */ +static uint64_t *exit_bytes_read = NULL; +/** Number of bytes written in current period by exit port */ +static uint64_t *exit_bytes_written = NULL; +/** Number of streams opened in current period by exit port */ +static uint32_t *exit_streams = NULL; + +/** Start time of exit stats or 0 if we're not collecting exit stats. */ +static time_t start_of_exit_stats_interval; + +/** Initialize exit port stats. */ void -rep_hist_free_all(void) -{ - digestmap_free(history_map, free_or_history); - tor_free(read_array); - tor_free(write_array); - tor_free(last_stability_doc); - built_last_stability_doc_at = 0; - predicted_ports_free(); -} - -/****************** hidden service usage statistics ******************/ - -/** How large are the intervals for which we track and report hidden service - * use? */ -#define NUM_SECS_HS_USAGE_SUM_INTERVAL (15*60) -/** How far in the past do we remember and publish hidden service use? */ -#define NUM_SECS_HS_USAGE_SUM_IS_VALID (24*60*60) -/** How many hidden service usage intervals do we remember? (derived) */ -#define NUM_TOTALS_HS_USAGE (NUM_SECS_HS_USAGE_SUM_IS_VALID/ \ - NUM_SECS_HS_USAGE_SUM_INTERVAL) - -/** List element containing a service id and the count. */ -typedef struct hs_usage_list_elem_t { - /** Service id of this elem. */ - char service_id[REND_SERVICE_ID_LEN_BASE32+1]; - /** Number of occurrences for the given service id. */ - uint32_t count; - /* Pointer to next list elem */ - struct hs_usage_list_elem_t *next; -} hs_usage_list_elem_t; - -/** Ordered list that stores service ids and the number of observations. It is - * ordered by the number of occurrences in descending order. Its purpose is to - * calculate the frequency distribution when the period is over. */ -typedef struct hs_usage_list_t { - /* Pointer to the first element in the list. */ - hs_usage_list_elem_t *start; - /* Number of total occurrences for all list elements. */ - uint32_t total_count; - /* Number of service ids, i.e. number of list elements. */ - uint32_t total_service_ids; -} hs_usage_list_t; - -/** Tracks service-related observations in the current period and their - * history. */ -typedef struct hs_usage_service_related_observation_t { - /** Ordered list that stores service ids and the number of observations in - * the current period. It is ordered by the number of occurrences in - * descending order. Its purpose is to calculate the frequency distribution - * when the period is over. */ - hs_usage_list_t *list; - /** Circular arrays that store the history of observations. totals stores all - * observations, twenty (ten, five) the number of observations related to a - * service id being accounted for the top 20 (10, 5) percent of all - * observations. */ - uint32_t totals[NUM_TOTALS_HS_USAGE]; - uint32_t five[NUM_TOTALS_HS_USAGE]; - uint32_t ten[NUM_TOTALS_HS_USAGE]; - uint32_t twenty[NUM_TOTALS_HS_USAGE]; -} hs_usage_service_related_observation_t; - -/** Tracks the history of general period-related observations, i.e. those that - * cannot be related to a specific service id. */ -typedef struct hs_usage_general_period_related_observations_t { - /** Circular array that stores the history of observations. */ - uint32_t totals[NUM_TOTALS_HS_USAGE]; -} hs_usage_general_period_related_observations_t; - -/** Keeps information about the current observation period and its relation to - * the histories of observations. */ -typedef struct hs_usage_current_observation_period_t { - /** Where do we write the next history entry? */ - int next_idx; - /** How many values in history have been set ever? (upper bound!) */ - int num_set; - /** When did this period begin? */ - time_t start_of_current_period; - /** When does the next period begin? */ - time_t start_of_next_period; -} hs_usage_current_observation_period_t; - -/** Usage statistics for the current observation period. */ -static hs_usage_current_observation_period_t *current_period = NULL; - -/** Total number of descriptor publish requests in the current observation - * period. */ -static hs_usage_service_related_observation_t *publish_total = NULL; - -/** Number of descriptor publish requests for services that have not been - * seen before in the current observation period. */ -static hs_usage_service_related_observation_t *publish_novel = NULL; - -/** Total number of descriptor fetch requests in the current observation - * period. */ -static hs_usage_service_related_observation_t *fetch_total = NULL; - -/** Number of successful descriptor fetch requests in the current - * observation period. */ -static hs_usage_service_related_observation_t *fetch_successful = NULL; - -/** Number of descriptors stored in the current observation period. */ -static hs_usage_general_period_related_observations_t *descs = NULL; - -/** Creates an empty ordered list element. */ -static hs_usage_list_elem_t * -hs_usage_list_elem_new(void) -{ - hs_usage_list_elem_t *e; - e = tor_malloc_zero(sizeof(hs_usage_list_elem_t)); - rephist_total_alloc += sizeof(hs_usage_list_elem_t); - e->count = 1; - e->next = NULL; - return e; -} - -/** Creates an empty ordered list. */ -static hs_usage_list_t * -hs_usage_list_new(void) +rep_hist_exit_stats_init(time_t now) { - hs_usage_list_t *l; - l = tor_malloc_zero(sizeof(hs_usage_list_t)); - rephist_total_alloc += sizeof(hs_usage_list_t); - l->start = NULL; - l->total_count = 0; - l->total_service_ids = 0; - return l; + start_of_exit_stats_interval = now; + exit_bytes_read = tor_malloc_zero(EXIT_STATS_NUM_PORTS * + sizeof(uint64_t)); + exit_bytes_written = tor_malloc_zero(EXIT_STATS_NUM_PORTS * + sizeof(uint64_t)); + exit_streams = tor_malloc_zero(EXIT_STATS_NUM_PORTS * + sizeof(uint32_t)); } -/** Creates an empty structure for storing service-related observations. */ -static hs_usage_service_related_observation_t * -hs_usage_service_related_observation_new(void) +/** Reset counters for exit port statistics. */ +void +rep_hist_reset_exit_stats(time_t now) { - hs_usage_service_related_observation_t *h; - h = tor_malloc_zero(sizeof(hs_usage_service_related_observation_t)); - rephist_total_alloc += sizeof(hs_usage_service_related_observation_t); - h->list = hs_usage_list_new(); - return h; + start_of_exit_stats_interval = now; + memset(exit_bytes_read, 0, EXIT_STATS_NUM_PORTS * sizeof(uint64_t)); + memset(exit_bytes_written, 0, EXIT_STATS_NUM_PORTS * sizeof(uint64_t)); + memset(exit_streams, 0, EXIT_STATS_NUM_PORTS * sizeof(uint32_t)); } -/** Creates an empty structure for storing general period-related - * observations. */ -static hs_usage_general_period_related_observations_t * -hs_usage_general_period_related_observations_new(void) +/** Stop collecting exit port stats in a way that we can re-start doing + * so in rep_hist_exit_stats_init(). */ +void +rep_hist_exit_stats_term(void) { - hs_usage_general_period_related_observations_t *p; - p = tor_malloc_zero(sizeof(hs_usage_general_period_related_observations_t)); - rephist_total_alloc+= sizeof(hs_usage_general_period_related_observations_t); - return p; + start_of_exit_stats_interval = 0; + tor_free(exit_bytes_read); + tor_free(exit_bytes_written); + tor_free(exit_streams); } -/** Creates an empty structure for storing period-specific information. */ -static hs_usage_current_observation_period_t * -hs_usage_current_observation_period_new(void) +/** Return a newly allocated string containing the exit port statistics + * until <b>now</b>, or NULL if we're not collecting exit stats. */ +char * +rep_hist_format_exit_stats(time_t now) { - hs_usage_current_observation_period_t *c; - time_t now; - c = tor_malloc_zero(sizeof(hs_usage_current_observation_period_t)); - rephist_total_alloc += sizeof(hs_usage_current_observation_period_t); - now = time(NULL); - c->start_of_current_period = now; - c->start_of_next_period = now + NUM_SECS_HS_USAGE_SUM_INTERVAL; - return c; -} + int i; + uint64_t total_bytes = 0, threshold_bytes, other_read = 0, + other_written = 0; + uint32_t other_streams = 0; + char *buf; + smartlist_t *written_strings, *read_strings, *streams_strings; + char *written_string, *read_string, *streams_string; + char t[ISO_TIME_LEN+1]; + char *result; -/** Initializes the structures for collecting hidden service usage data. */ -static void -hs_usage_init(void) -{ - current_period = hs_usage_current_observation_period_new(); - publish_total = hs_usage_service_related_observation_new(); - publish_novel = hs_usage_service_related_observation_new(); - fetch_total = hs_usage_service_related_observation_new(); - fetch_successful = hs_usage_service_related_observation_new(); - descs = hs_usage_general_period_related_observations_new(); -} + if (!start_of_exit_stats_interval) + return NULL; /* Not initialized. */ -/** Clears the given ordered list by resetting its attributes and releasing - * the memory allocated by its elements. */ -static void -hs_usage_list_clear(hs_usage_list_t *lst) -{ - /* walk through elements and free memory */ - hs_usage_list_elem_t *current = lst->start; - hs_usage_list_elem_t *tmp; - while (current != NULL) { - tmp = current->next; - rephist_total_alloc -= sizeof(hs_usage_list_elem_t); - tor_free(current); - current = tmp; + /* Count total number of bytes, so that we can attribute observations + * below or equal to a threshold of 1 / EXIT_STATS_THRESHOLD_RECIPROCAL + * of all bytes to a special port 'other'. */ + for (i = 1; i < EXIT_STATS_NUM_PORTS; i++) { + total_bytes += exit_bytes_read[i]; + total_bytes += exit_bytes_written[i]; } - /* reset attributes */ - lst->start = NULL; - lst->total_count = 0; - lst->total_service_ids = 0; - return; -} - -/** Frees the memory used by the given list. */ -static void -hs_usage_list_free(hs_usage_list_t *lst) -{ - if (!lst) - return; - hs_usage_list_clear(lst); - rephist_total_alloc -= sizeof(hs_usage_list_t); - tor_free(lst); + threshold_bytes = total_bytes / EXIT_STATS_THRESHOLD_RECIPROCAL; + + /* Add observations of all ports above the threshold to smartlists and + * join them to single strings. Also count bytes and streams of ports + * below or equal to the threshold. */ + written_strings = smartlist_create(); + read_strings = smartlist_create(); + streams_strings = smartlist_create(); + for (i = 1; i < EXIT_STATS_NUM_PORTS; i++) { + if (exit_bytes_read[i] + exit_bytes_written[i] > threshold_bytes) { + if (exit_bytes_written[i] > 0) { + uint64_t num = round_uint64_to_next_multiple_of( + exit_bytes_written[i], EXIT_STATS_ROUND_UP_BYTES); + num /= 1024; + buf = NULL; + tor_asprintf(&buf, "%d="U64_FORMAT, i, U64_PRINTF_ARG(num)); + smartlist_add(written_strings, buf); + } + if (exit_bytes_read[i] > 0) { + uint64_t num = round_uint64_to_next_multiple_of( + exit_bytes_read[i], EXIT_STATS_ROUND_UP_BYTES); + num /= 1024; + buf = NULL; + tor_asprintf(&buf, "%d="U64_FORMAT, i, U64_PRINTF_ARG(num)); + smartlist_add(read_strings, buf); + } + if (exit_streams[i] > 0) { + uint32_t num = round_uint32_to_next_multiple_of(exit_streams[i], + EXIT_STATS_ROUND_UP_STREAMS); + buf = NULL; + tor_asprintf(&buf, "%d=%u", i, num); + smartlist_add(streams_strings, buf); + } + } else { + other_read += exit_bytes_read[i]; + other_written += exit_bytes_written[i]; + other_streams += exit_streams[i]; + } + } + other_written = round_uint64_to_next_multiple_of(other_written, + EXIT_STATS_ROUND_UP_BYTES); + other_written /= 1024; + buf = NULL; + tor_asprintf(&buf, "other="U64_FORMAT, U64_PRINTF_ARG(other_written)); + smartlist_add(written_strings, buf); + other_read = round_uint64_to_next_multiple_of(other_read, + EXIT_STATS_ROUND_UP_BYTES); + other_read /= 1024; + buf = NULL; + tor_asprintf(&buf, "other="U64_FORMAT, U64_PRINTF_ARG(other_read)); + smartlist_add(read_strings, buf); + other_streams = round_uint32_to_next_multiple_of(other_streams, + EXIT_STATS_ROUND_UP_STREAMS); + buf = NULL; + tor_asprintf(&buf, "other=%u", other_streams); + smartlist_add(streams_strings, buf); + written_string = smartlist_join_strings(written_strings, ",", 0, NULL); + read_string = smartlist_join_strings(read_strings, ",", 0, NULL); + streams_string = smartlist_join_strings(streams_strings, ",", 0, NULL); + SMARTLIST_FOREACH(written_strings, char *, cp, tor_free(cp)); + SMARTLIST_FOREACH(read_strings, char *, cp, tor_free(cp)); + SMARTLIST_FOREACH(streams_strings, char *, cp, tor_free(cp)); + smartlist_free(written_strings); + smartlist_free(read_strings); + smartlist_free(streams_strings); + + /* Put everything together. */ + format_iso_time(t, now); + tor_asprintf(&result, "exit-stats-end %s (%d s)\n" + "exit-kibibytes-written %s\n" + "exit-kibibytes-read %s\n" + "exit-streams-opened %s\n", + t, (unsigned) (now - start_of_exit_stats_interval), + written_string, + read_string, + streams_string); + tor_free(written_string); + tor_free(read_string); + tor_free(streams_string); + return result; } -/** Frees the memory used by the given service-related observations. */ -static void -hs_usage_service_related_observation_free( - hs_usage_service_related_observation_t *s) +/** If 24 hours have passed since the beginning of the current exit port + * stats period, write exit stats to $DATADIR/stats/exit-stats (possibly + * overwriting an existing file) and reset counters. Return when we would + * next want to write exit stats or 0 if we never want to write. */ +time_t +rep_hist_exit_stats_write(time_t now) { - if (!s) - return; - hs_usage_list_free(s->list); - rephist_total_alloc -= sizeof(hs_usage_service_related_observation_t); - tor_free(s); -} + char *statsdir = NULL, *filename = NULL, *str = NULL; -/** Frees the memory used by the given period-specific observations. */ -static void -hs_usage_general_period_related_observations_free( - hs_usage_general_period_related_observations_t *s) -{ - rephist_total_alloc-=sizeof(hs_usage_general_period_related_observations_t); - tor_free(s); -} + if (!start_of_exit_stats_interval) + return 0; /* Not initialized. */ + if (start_of_exit_stats_interval + WRITE_STATS_INTERVAL > now) + goto done; /* Not ready to write. */ -/** Frees the memory used by period-specific information. */ -static void -hs_usage_current_observation_period_free( - hs_usage_current_observation_period_t *s) -{ - rephist_total_alloc -= sizeof(hs_usage_current_observation_period_t); - tor_free(s); -} + log_info(LD_HIST, "Writing exit port statistics to disk."); -/** Frees all memory that was used for collecting hidden service usage data. */ -void -hs_usage_free_all(void) -{ - hs_usage_general_period_related_observations_free(descs); - descs = NULL; - hs_usage_service_related_observation_free(fetch_successful); - hs_usage_service_related_observation_free(fetch_total); - hs_usage_service_related_observation_free(publish_novel); - hs_usage_service_related_observation_free(publish_total); - fetch_successful = fetch_total = publish_novel = publish_total = NULL; - hs_usage_current_observation_period_free(current_period); - current_period = NULL; -} + /* Generate history string. */ + str = rep_hist_format_exit_stats(now); -/** Inserts a new occurrence for the given service id to the given ordered - * list. */ -static void -hs_usage_insert_value(hs_usage_list_t *lst, const char *service_id) -{ - /* search if there is already an elem with same service_id in list */ - hs_usage_list_elem_t *current = lst->start; - hs_usage_list_elem_t *previous = NULL; - while (current != NULL && strcasecmp(current->service_id,service_id)) { - previous = current; - current = current->next; - } - /* found an element with same service_id? */ - if (current == NULL) { - /* not found! append to end (which could also be the end of a zero-length - * list), don't need to sort (1 is smallest value). */ - /* create elem */ - hs_usage_list_elem_t *e = hs_usage_list_elem_new(); - /* update list attributes (one new elem, one new occurrence) */ - lst->total_count++; - lst->total_service_ids++; - /* copy service id to elem */ - strlcpy(e->service_id,service_id,sizeof(e->service_id)); - /* let either l->start or previously last elem point to new elem */ - if (lst->start == NULL) { - /* this is the first elem */ - lst->start = e; - } else { - /* there were elems in the list before */ - previous->next = e; - } - } else { - /* found! add occurrence to elem and consider resorting */ - /* update list attributes (no new elem, but one new occurrence) */ - lst->total_count++; - /* add occurrence to elem */ - current->count++; - /* is it another than the first list elem? and has previous elem fewer - * count than current? then we need to resort */ - if (previous != NULL && previous->count < current->count) { - /* yes! we need to resort */ - /* remove current elem first */ - previous->next = current->next; - /* can we prepend elem to all other elements? */ - if (lst->start->count <= current->count) { - /* yes! prepend elem */ - current->next = lst->start; - lst->start = current; - } else { - /* no! walk through list a second time and insert at correct place */ - hs_usage_list_elem_t *insert_current = lst->start->next; - hs_usage_list_elem_t *insert_previous = lst->start; - while (insert_current != NULL && - insert_current->count > current->count) { - insert_previous = insert_current; - insert_current = insert_current->next; - } - /* insert here */ - current->next = insert_current; - insert_previous->next = current; - } - } - } -} + /* Reset counters. */ + rep_hist_reset_exit_stats(now); -/** Writes the current service-related observations to the history array and - * clears the observations of the current period. */ -static void -hs_usage_write_service_related_observations_to_history( - hs_usage_current_observation_period_t *p, - hs_usage_service_related_observation_t *h) -{ - /* walk through the first 20 % of list elements and calculate frequency - * distributions */ - /* maximum indices for the three frequencies */ - int five_percent_idx = h->list->total_service_ids/20; - int ten_percent_idx = h->list->total_service_ids/10; - int twenty_percent_idx = h->list->total_service_ids/5; - /* temp values */ - uint32_t five_percent = 0; - uint32_t ten_percent = 0; - uint32_t twenty_percent = 0; - /* walk through list */ - hs_usage_list_elem_t *current = h->list->start; - int i=0; - while (current != NULL && i <= twenty_percent_idx) { - twenty_percent += current->count; - if (i <= ten_percent_idx) - ten_percent += current->count; - if (i <= five_percent_idx) - five_percent += current->count; - current = current->next; - i++; + /* Try to write to disk. */ + statsdir = get_datadir_fname("stats"); + if (check_private_dir(statsdir, CPD_CREATE) < 0) { + log_warn(LD_HIST, "Unable to create stats/ directory!"); + goto done; } - /* copy frequencies */ - h->twenty[p->next_idx] = twenty_percent; - h->ten[p->next_idx] = ten_percent; - h->five[p->next_idx] = five_percent; - /* copy total number of observations */ - h->totals[p->next_idx] = h->list->total_count; - /* free memory of old list */ - hs_usage_list_clear(h->list); -} + filename = get_datadir_fname2("stats", "exit-stats"); + if (write_str_to_file(filename, str, 0) < 0) + log_warn(LD_HIST, "Unable to write exit port statistics to disk!"); -/** Advances to next observation period. */ -static void -hs_usage_advance_current_observation_period(void) -{ - /* aggregate observations to history, including frequency distribution - * arrays */ - hs_usage_write_service_related_observations_to_history( - current_period, publish_total); - hs_usage_write_service_related_observations_to_history( - current_period, publish_novel); - hs_usage_write_service_related_observations_to_history( - current_period, fetch_total); - hs_usage_write_service_related_observations_to_history( - current_period, fetch_successful); - /* write current number of descriptors to descs history */ - descs->totals[current_period->next_idx] = rend_cache_size(); - /* advance to next period */ - current_period->next_idx++; - if (current_period->next_idx == NUM_TOTALS_HS_USAGE) - current_period->next_idx = 0; - if (current_period->num_set < NUM_TOTALS_HS_USAGE) - ++current_period->num_set; - current_period->start_of_current_period=current_period->start_of_next_period; - current_period->start_of_next_period += NUM_SECS_HS_USAGE_SUM_INTERVAL; -} - -/** Checks if the current period is up to date, and if not, advances it. */ -static void -hs_usage_check_if_current_period_is_up_to_date(time_t now) -{ - while (now > current_period->start_of_next_period) { - hs_usage_advance_current_observation_period(); - } + done: + tor_free(str); + tor_free(statsdir); + tor_free(filename); + return start_of_exit_stats_interval + WRITE_STATS_INTERVAL; } -/** Adds a service-related observation, maybe after advancing to next - * observation period. */ -static void -hs_usage_add_service_related_observation( - hs_usage_service_related_observation_t *h, - time_t now, - const char *service_id) +/** Note that we wrote <b>num_written</b> bytes and read <b>num_read</b> + * bytes to/from an exit connection to <b>port</b>. */ +void +rep_hist_note_exit_bytes(uint16_t port, size_t num_written, + size_t num_read) { - if (now < current_period->start_of_current_period) { - /* don't record old data */ - return; - } - /* check if we are up-to-date */ - hs_usage_check_if_current_period_is_up_to_date(now); - /* add observation */ - hs_usage_insert_value(h->list, service_id); + if (!start_of_exit_stats_interval) + return; /* Not initialized. */ + exit_bytes_written[port] += num_written; + exit_bytes_read[port] += num_read; + log_debug(LD_HIST, "Written %lu bytes and read %lu bytes to/from an " + "exit connection to port %d.", + (unsigned long)num_written, (unsigned long)num_read, port); } -/** Adds the observation of storing a rendezvous service descriptor to our - * cache in our role as HS authoritative directory. */ +/** Note that we opened an exit stream to <b>port</b>. */ void -hs_usage_note_publish_total(const char *service_id, time_t now) +rep_hist_note_exit_stream_opened(uint16_t port) { - hs_usage_add_service_related_observation(publish_total, now, service_id); + if (!start_of_exit_stats_interval) + return; /* Not initialized. */ + exit_streams[port]++; + log_debug(LD_HIST, "Opened exit stream to port %d", port); } -/** Adds the observation of storing a novel rendezvous service descriptor to - * our cache in our role as HS authoritative directory. */ +/*** cell statistics ***/ + +/** Start of the current buffer stats interval or 0 if we're not + * collecting buffer statistics. */ +static time_t start_of_buffer_stats_interval; + +/** Initialize buffer stats. */ void -hs_usage_note_publish_novel(const char *service_id, time_t now) +rep_hist_buffer_stats_init(time_t now) { - hs_usage_add_service_related_observation(publish_novel, now, service_id); + start_of_buffer_stats_interval = now; } -/** Adds the observation of being requested for a rendezvous service descriptor - * in our role as HS authoritative directory. */ +typedef struct circ_buffer_stats_t { + uint32_t processed_cells; + double mean_num_cells_in_queue; + double mean_time_cells_in_queue; + uint32_t local_circ_id; +} circ_buffer_stats_t; + +/** Holds stats. */ +smartlist_t *circuits_for_buffer_stats = NULL; + +/** Remember cell statistics for circuit <b>circ</b> at time + * <b>end_of_interval</b> and reset cell counters in case the circuit + * remains open in the next measurement interval. */ void -hs_usage_note_fetch_total(const char *service_id, time_t now) +rep_hist_buffer_stats_add_circ(circuit_t *circ, time_t end_of_interval) +{ + circ_buffer_stats_t *stat; + time_t start_of_interval; + int interval_length; + or_circuit_t *orcirc; + if (CIRCUIT_IS_ORIGIN(circ)) + return; + orcirc = TO_OR_CIRCUIT(circ); + if (!orcirc->processed_cells) + return; + if (!circuits_for_buffer_stats) + circuits_for_buffer_stats = smartlist_create(); + start_of_interval = circ->timestamp_created > + start_of_buffer_stats_interval ? + circ->timestamp_created : + start_of_buffer_stats_interval; + interval_length = (int) (end_of_interval - start_of_interval); + stat = tor_malloc_zero(sizeof(circ_buffer_stats_t)); + stat->processed_cells = orcirc->processed_cells; + /* 1000.0 for s -> ms; 2.0 because of app-ward and exit-ward queues */ + stat->mean_num_cells_in_queue = interval_length == 0 ? 0.0 : + (double) orcirc->total_cell_waiting_time / + (double) interval_length / 1000.0 / 2.0; + stat->mean_time_cells_in_queue = + (double) orcirc->total_cell_waiting_time / + (double) orcirc->processed_cells; + smartlist_add(circuits_for_buffer_stats, stat); + orcirc->total_cell_waiting_time = 0; + orcirc->processed_cells = 0; +} + +/** Sorting helper: return -1, 1, or 0 based on comparison of two + * circ_buffer_stats_t */ +static int +_buffer_stats_compare_entries(const void **_a, const void **_b) { - hs_usage_add_service_related_observation(fetch_total, now, service_id); + const circ_buffer_stats_t *a = *_a, *b = *_b; + if (a->processed_cells < b->processed_cells) + return 1; + else if (a->processed_cells > b->processed_cells) + return -1; + else + return 0; } -/** Adds the observation of being requested for a rendezvous service descriptor - * in our role as HS authoritative directory and being able to answer that - * request successfully. */ +/** Stop collecting cell stats in a way that we can re-start doing so in + * rep_hist_buffer_stats_init(). */ void -hs_usage_note_fetch_successful(const char *service_id, time_t now) +rep_hist_buffer_stats_term(void) { - hs_usage_add_service_related_observation(fetch_successful, now, service_id); + start_of_buffer_stats_interval = 0; + if (!circuits_for_buffer_stats) + circuits_for_buffer_stats = smartlist_create(); + SMARTLIST_FOREACH(circuits_for_buffer_stats, circ_buffer_stats_t *, + stat, tor_free(stat)); + smartlist_clear(circuits_for_buffer_stats); } -/** Writes the given circular array to a string. */ -static size_t -hs_usage_format_history(char *buf, size_t len, uint32_t *data) +/** Write buffer statistics to $DATADIR/stats/buffer-stats and return when + * we would next want to write exit stats. */ +time_t +rep_hist_buffer_stats_write(time_t now) { - char *cp = buf; /* pointer where we are in the buffer */ - int i, n; - if (current_period->num_set <= current_period->next_idx) { - i = 0; /* not been through circular array */ - } else { - i = current_period->next_idx; + char *statsdir = NULL, *filename = NULL; + char written[ISO_TIME_LEN+1]; + open_file_t *open_file = NULL; + FILE *out; +#define SHARES 10 + int processed_cells[SHARES], circs_in_share[SHARES], + number_of_circuits, i; + double queued_cells[SHARES], time_in_queue[SHARES]; + smartlist_t *str_build = smartlist_create(); + char *str = NULL, *buf=NULL; + circuit_t *circ; + + if (!start_of_buffer_stats_interval) + return 0; /* Not initialized. */ + if (start_of_buffer_stats_interval + WRITE_STATS_INTERVAL > now) + goto done; /* Not ready to write */ + + /* add current circuits to stats */ + for (circ = _circuit_get_global_list(); circ; circ = circ->next) + rep_hist_buffer_stats_add_circ(circ, now); + /* calculate deciles */ + memset(processed_cells, 0, SHARES * sizeof(int)); + memset(circs_in_share, 0, SHARES * sizeof(int)); + memset(queued_cells, 0, SHARES * sizeof(double)); + memset(time_in_queue, 0, SHARES * sizeof(double)); + if (!circuits_for_buffer_stats) + circuits_for_buffer_stats = smartlist_create(); + smartlist_sort(circuits_for_buffer_stats, + _buffer_stats_compare_entries); + number_of_circuits = smartlist_len(circuits_for_buffer_stats); + if (number_of_circuits < 1) { + log_info(LD_HIST, "Attempt to write cell statistics to disk failed. " + "We haven't seen a single circuit to report about."); + goto done; } - for (n = 0; n < current_period->num_set; ++n,++i) { - if (i >= NUM_TOTALS_HS_USAGE) - i -= NUM_TOTALS_HS_USAGE; - tor_assert(i < NUM_TOTALS_HS_USAGE); - if (n == (current_period->num_set-1)) - tor_snprintf(cp, len-(cp-buf), "%d", data[i]); - else - tor_snprintf(cp, len-(cp-buf), "%d,", data[i]); - cp += strlen(cp); + i = 0; + SMARTLIST_FOREACH_BEGIN(circuits_for_buffer_stats, + circ_buffer_stats_t *, stat) + { + int share = i++ * SHARES / number_of_circuits; + processed_cells[share] += stat->processed_cells; + queued_cells[share] += stat->mean_num_cells_in_queue; + time_in_queue[share] += stat->mean_time_cells_in_queue; + circs_in_share[share]++; } - return cp-buf; -} - -/** Writes the complete usage history as hidden service authoritative directory - * to a string. */ -static char * -hs_usage_format_statistics(void) -{ - char *buf, *cp, *s = NULL; - char t[ISO_TIME_LEN+1]; - int r; - uint32_t *data = NULL; - size_t len; - len = (70+20*NUM_TOTALS_HS_USAGE)*11; - buf = tor_malloc_zero(len); - cp = buf; - for (r = 0; r < 11; ++r) { - switch (r) { - case 0: - s = (char*) "publish-total-history"; - data = publish_total->totals; - break; - case 1: - s = (char*) "publish-novel-history"; - data = publish_novel->totals; - break; - case 2: - s = (char*) "publish-top-5-percent-history"; - data = publish_total->five; - break; - case 3: - s = (char*) "publish-top-10-percent-history"; - data = publish_total->ten; - break; - case 4: - s = (char*) "publish-top-20-percent-history"; - data = publish_total->twenty; - break; - case 5: - s = (char*) "fetch-total-history"; - data = fetch_total->totals; - break; - case 6: - s = (char*) "fetch-successful-history"; - data = fetch_successful->totals; - break; - case 7: - s = (char*) "fetch-top-5-percent-history"; - data = fetch_total->five; - break; - case 8: - s = (char*) "fetch-top-10-percent-history"; - data = fetch_total->ten; - break; - case 9: - s = (char*) "fetch-top-20-percent-history"; - data = fetch_total->twenty; - break; - case 10: - s = (char*) "desc-total-history"; - data = descs->totals; - break; - } - format_iso_time(t, current_period->start_of_current_period); - tor_snprintf(cp, len-(cp-buf), "%s %s (%d s) ", s, t, - NUM_SECS_HS_USAGE_SUM_INTERVAL); - cp += strlen(cp); - cp += hs_usage_format_history(cp, len-(cp-buf), data); - strlcat(cp, "\n", len-(cp-buf)); - ++cp; + SMARTLIST_FOREACH_END(stat); + /* clear buffer stats history */ + SMARTLIST_FOREACH(circuits_for_buffer_stats, circ_buffer_stats_t *, + stat, tor_free(stat)); + smartlist_clear(circuits_for_buffer_stats); + /* write to file */ + statsdir = get_datadir_fname("stats"); + if (check_private_dir(statsdir, CPD_CREATE) < 0) + goto done; + filename = get_datadir_fname2("stats", "buffer-stats"); + out = start_writing_to_stdio_file(filename, OPEN_FLAGS_APPEND, + 0600, &open_file); + if (!out) + goto done; + format_iso_time(written, now); + if (fprintf(out, "cell-stats-end %s (%d s)\n", written, + (unsigned) (now - start_of_buffer_stats_interval)) < 0) + goto done; + for (i = 0; i < SHARES; i++) { + tor_asprintf(&buf,"%d", !circs_in_share[i] ? 0 : + processed_cells[i] / circs_in_share[i]); + smartlist_add(str_build, buf); } - return buf; + str = smartlist_join_strings(str_build, ",", 0, NULL); + if (fprintf(out, "cell-processed-cells %s\n", str) < 0) + goto done; + tor_free(str); + SMARTLIST_FOREACH(str_build, char *, c, tor_free(c)); + smartlist_clear(str_build); + for (i = 0; i < SHARES; i++) { + tor_asprintf(&buf, "%.2f", circs_in_share[i] == 0 ? 0.0 : + queued_cells[i] / (double) circs_in_share[i]); + smartlist_add(str_build, buf); + } + str = smartlist_join_strings(str_build, ",", 0, NULL); + if (fprintf(out, "cell-queued-cells %s\n", str) < 0) + goto done; + tor_free(str); + SMARTLIST_FOREACH(str_build, char *, c, tor_free(c)); + smartlist_clear(str_build); + for (i = 0; i < SHARES; i++) { + tor_asprintf(&buf, "%.0f", circs_in_share[i] == 0 ? 0.0 : + time_in_queue[i] / (double) circs_in_share[i]); + smartlist_add(str_build, buf); + } + str = smartlist_join_strings(str_build, ",", 0, NULL); + if (fprintf(out, "cell-time-in-queue %s\n", str) < 0) + goto done; + tor_free(str); + SMARTLIST_FOREACH(str_build, char *, c, tor_free(c)); + smartlist_free(str_build); + str_build = NULL; + if (fprintf(out, "cell-circuits-per-decile %d\n", + (number_of_circuits + SHARES - 1) / SHARES) < 0) + goto done; + finish_writing_to_file(open_file); + open_file = NULL; + start_of_buffer_stats_interval = now; + done: + if (open_file) + abort_writing_to_file(open_file); + tor_free(filename); + tor_free(statsdir); + if (str_build) { + SMARTLIST_FOREACH(str_build, char *, c, tor_free(c)); + smartlist_free(str_build); + } + tor_free(str); +#undef SHARES + return start_of_buffer_stats_interval + WRITE_STATS_INTERVAL; } -/** Write current statistics about hidden service usage to file. */ +/** Free all storage held by the OR/link history caches, by the + * bandwidth history arrays, by the port history, or by statistics . */ void -hs_usage_write_statistics_to_file(time_t now) +rep_hist_free_all(void) { - char *buf; - size_t len; - char *fname; - or_options_t *options = get_options(); - /* check if we are up-to-date */ - hs_usage_check_if_current_period_is_up_to_date(now); - buf = hs_usage_format_statistics(); - len = strlen(options->DataDirectory) + 16; - fname = tor_malloc(len); - tor_snprintf(fname, len, "%s"PATH_SEPARATOR"hsusage", - options->DataDirectory); - write_str_to_file(fname,buf,0); - tor_free(buf); - tor_free(fname); + digestmap_free(history_map, free_or_history); + tor_free(read_array); + tor_free(write_array); + tor_free(last_stability_doc); + tor_free(exit_bytes_read); + tor_free(exit_bytes_written); + tor_free(exit_streams); + built_last_stability_doc_at = 0; + predicted_ports_free(); } diff --git a/src/or/rephist.h b/src/or/rephist.h new file mode 100644 index 0000000000..f655500eb8 --- /dev/null +++ b/src/or/rephist.h @@ -0,0 +1,84 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2010, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file rephist.h + * \brief Header file for rephist.c. + **/ + +#ifndef _TOR_REPHIST_H +#define _TOR_REPHIST_H + +void rep_hist_init(void); +void rep_hist_note_connect_failed(const char* nickname, time_t when); +void rep_hist_note_connect_succeeded(const char* nickname, time_t when); +void rep_hist_note_disconnect(const char* nickname, time_t when); +void rep_hist_note_connection_died(const char* nickname, time_t when); +void rep_hist_note_extend_succeeded(const char *from_name, + const char *to_name); +void rep_hist_note_extend_failed(const char *from_name, const char *to_name); +void rep_hist_dump_stats(time_t now, int severity); +void rep_hist_note_bytes_read(size_t num_bytes, time_t when); +void rep_hist_note_bytes_written(size_t num_bytes, time_t when); +int rep_hist_bandwidth_assess(void); +char *rep_hist_get_bandwidth_lines(int for_extrainfo); +void rep_hist_update_state(or_state_t *state); +int rep_hist_load_state(or_state_t *state, char **err); +void rep_history_clean(time_t before); + +void rep_hist_note_router_reachable(const char *id, time_t when); +void rep_hist_note_router_unreachable(const char *id, time_t when); +int rep_hist_record_mtbf_data(time_t now, int missing_means_down); +int rep_hist_load_mtbf_data(time_t now); + +time_t rep_hist_downrate_old_runs(time_t now); +double rep_hist_get_stability(const char *id, time_t when); +double rep_hist_get_weighted_fractional_uptime(const char *id, time_t when); +long rep_hist_get_weighted_time_known(const char *id, time_t when); +int rep_hist_have_measured_enough_stability(void); +const char *rep_hist_get_router_stability_doc(time_t now); + +void rep_hist_note_used_port(time_t now, uint16_t port); +smartlist_t *rep_hist_get_predicted_ports(time_t now); +void rep_hist_note_used_resolve(time_t now); +void rep_hist_note_used_internal(time_t now, int need_uptime, + int need_capacity); +int rep_hist_get_predicted_internal(time_t now, int *need_uptime, + int *need_capacity); + +int any_predicted_circuits(time_t now); +int rep_hist_circbuilding_dormant(time_t now); + +void note_crypto_pk_op(pk_op_t operation); +void dump_pk_ops(int severity); + +void rep_hist_free_all(void); + +/* for hidden service usage statistics */ +void hs_usage_note_publish_total(const char *service_id, time_t now); +void hs_usage_note_publish_novel(const char *service_id, time_t now); +void hs_usage_note_fetch_total(const char *service_id, time_t now); +void hs_usage_note_fetch_successful(const char *service_id, time_t now); +void hs_usage_write_statistics_to_file(time_t now); +void hs_usage_free_all(void); + +void rep_hist_exit_stats_init(time_t now); +void rep_hist_reset_exit_stats(time_t now); +void rep_hist_exit_stats_term(void); +char *rep_hist_format_exit_stats(time_t now); +time_t rep_hist_exit_stats_write(time_t now); +void rep_hist_note_exit_bytes(uint16_t port, size_t num_written, + size_t num_read); +void rep_hist_note_exit_stream_opened(uint16_t port); + +void rep_hist_buffer_stats_init(time_t now); +void rep_hist_buffer_stats_add_circ(circuit_t *circ, + time_t end_of_interval); +time_t rep_hist_buffer_stats_write(time_t now); +void rep_hist_buffer_stats_term(void); + +#endif + diff --git a/src/or/router.c b/src/or/router.c index 8661e7a224..978078bf78 100644 --- a/src/or/router.c +++ b/src/or/router.c @@ -7,6 +7,23 @@ #define ROUTER_PRIVATE #include "or.h" +#include "circuitlist.h" +#include "circuituse.h" +#include "config.h" +#include "connection.h" +#include "control.h" +#include "directory.h" +#include "dirserv.h" +#include "dns.h" +#include "geoip.h" +#include "hibernate.h" +#include "main.h" +#include "policies.h" +#include "relay.h" +#include "rephist.h" +#include "router.h" +#include "routerlist.h" +#include "routerparse.h" /** * \file router.c @@ -61,8 +78,7 @@ static void set_onion_key(crypto_pk_env_t *k) { tor_mutex_acquire(key_lock); - if (onionkey) - crypto_free_pk_env(onionkey); + crypto_free_pk_env(onionkey); onionkey = k; onionkey_set_at = time(NULL); tor_mutex_release(key_lock); @@ -111,8 +127,7 @@ get_onion_key_set_at(void) void set_identity_key(crypto_pk_env_t *k) { - if (identitykey) - crypto_free_pk_env(identitykey); + crypto_free_pk_env(identitykey); identitykey = k; crypto_pk_get_digest(identitykey, identitykey_digest); } @@ -201,8 +216,7 @@ rotate_onion_key(void) } log_info(LD_GENERAL, "Rotating onion key"); tor_mutex_acquire(key_lock); - if (lastonionkey) - crypto_free_pk_env(lastonionkey); + crypto_free_pk_env(lastonionkey); lastonionkey = onionkey; onionkey = prkey; now = time(NULL); @@ -331,10 +345,9 @@ load_authority_keyset(int legacy, crypto_pk_env_t **key_out, goto done; } - if (*key_out) - crypto_free_pk_env(*key_out); - if (*cert_out) - authority_cert_free(*cert_out); + crypto_free_pk_env(*key_out); + authority_cert_free(*cert_out); + *key_out = signing_key; *cert_out = parsed; r = 0; @@ -344,10 +357,8 @@ load_authority_keyset(int legacy, crypto_pk_env_t **key_out, done: tor_free(fname); tor_free(cert); - if (signing_key) - crypto_free_pk_env(signing_key); - if (parsed) - authority_cert_free(parsed); + crypto_free_pk_env(signing_key); + authority_cert_free(parsed); return r; } @@ -442,7 +453,9 @@ init_keys(void) key_lock = tor_mutex_new(); /* There are a couple of paths that put us here before */ - if (crypto_global_init(get_options()->HardwareAccel)) { + if (crypto_global_init(get_options()->HardwareAccel, + get_options()->AccelName, + get_options()->AccelDir)) { log_err(LD_BUG, "Unable to initialize OpenSSL. Exiting."); return -1; } @@ -1243,6 +1256,7 @@ router_rebuild_descriptor(int force) uint32_t addr; char platform[256]; int hibernating = we_are_hibernating(); + size_t ei_size; or_options_t *options = get_options(); if (desc_clean_since && !force) @@ -1285,7 +1299,7 @@ router_rebuild_descriptor(int force) policies_parse_exit_policy(options->ExitPolicy, &ri->exit_policy, options->ExitPolicyRejectPrivate, - ri->address); + ri->address, !options->BridgeRelay); if (desc_routerinfo) { /* inherit values */ ri->is_valid = desc_routerinfo->is_valid; @@ -1356,9 +1370,10 @@ router_rebuild_descriptor(int force) ei->cache_info.published_on = ri->cache_info.published_on; memcpy(ei->cache_info.identity_digest, ri->cache_info.identity_digest, DIGEST_LEN); - ei->cache_info.signed_descriptor_body = tor_malloc(8192); - if (extrainfo_dump_to_string(ei->cache_info.signed_descriptor_body, 8192, - ei, get_identity_key()) < 0) { + ei_size = options->ExtraInfoStatistics ? MAX_EXTRAINFO_UPLOAD_SIZE : 8192; + ei->cache_info.signed_descriptor_body = tor_malloc(ei_size); + if (extrainfo_dump_to_string(ei->cache_info.signed_descriptor_body, + ei_size, ei, get_identity_key()) < 0) { log_warn(LD_BUG, "Couldn't generate extra-info descriptor."); routerinfo_free(ri); extrainfo_free(ei); @@ -1401,11 +1416,9 @@ router_rebuild_descriptor(int force) tor_assert(! routerinfo_incompatible_with_extrainfo(ri, ei, NULL, NULL)); - if (desc_routerinfo) - routerinfo_free(desc_routerinfo); + routerinfo_free(desc_routerinfo); desc_routerinfo = ri; - if (desc_extrainfo) - extrainfo_free(desc_extrainfo); + extrainfo_free(desc_extrainfo); desc_extrainfo = ei; desc_clean_since = time(NULL); @@ -1586,8 +1599,6 @@ router_guess_address_from_dir_headers(uint32_t *guess) return -1; } -extern const char tor_svn_revision[]; /* from tor_main.c */ - /** Set <b>platform</b> (max length <b>len</b>) to a NUL-terminated short * string describing the version of Tor and the operating system we're * currently running on. @@ -1646,7 +1657,7 @@ router_dump_router_to_string(char *s, size_t maxlen, routerinfo_t *router, return -1; } - /* PEM-encode the identity key key */ + /* PEM-encode the identity key */ if (crypto_pk_write_public_key_to_string(router->identity_pkey, &identity_pkey,&identity_pkeylen)<0) { log_warn(LD_BUG,"write identity_pkey to string failed!"); @@ -1767,7 +1778,7 @@ router_dump_router_to_string(char *s, size_t maxlen, routerinfo_t *router, note_crypto_pk_op(SIGN_RTR); if (router_append_dirobj_signature(s+written,maxlen-written, - digest,ident_key)<0) { + digest,DIGEST_LEN,ident_key)<0) { log_warn(LD_BUG, "Couldn't sign router descriptor"); return -1; } @@ -1802,6 +1813,57 @@ router_dump_router_to_string(char *s, size_t maxlen, routerinfo_t *router, return (int)written+1; } +/** Load the contents of <b>filename</b>, find the last line starting with + * <b>end_line</b>, ensure that its timestamp is not more than 25 hours in + * the past or more than 1 hour in the future with respect to <b>now</b>, + * and write the file contents starting with that line to *<b>out</b>. + * Return 1 for success, 0 if the file does not exist, or -1 if the file + * does not contain a line matching these criteria or other failure. */ +static int +load_stats_file(const char *filename, const char *end_line, time_t now, + char **out) +{ + int r = -1; + char *fname = get_datadir_fname(filename); + char *contents, *start = NULL, *tmp, timestr[ISO_TIME_LEN+1]; + time_t written; + switch (file_status(fname)) { + case FN_FILE: + /* X022 Find an alternative to reading the whole file to memory. */ + if ((contents = read_file_to_str(fname, 0, NULL))) { + tmp = strstr(contents, end_line); + /* Find last block starting with end_line */ + while (tmp) { + start = tmp; + tmp = strstr(tmp + 1, end_line); + } + if (!start) + goto notfound; + if (strlen(start) < strlen(end_line) + 1 + sizeof(timestr)) + goto notfound; + strlcpy(timestr, start + 1 + strlen(end_line), sizeof(timestr)); + if (parse_iso_time(timestr, &written) < 0) + goto notfound; + if (written < now - (25*60*60) || written > now + (1*60*60)) + goto notfound; + *out = tor_strdup(start); + r = 1; + } + notfound: + tor_free(contents); + break; + case FN_NOENT: + r = 0; + break; + case FN_ERROR: + case FN_DIR: + default: + break; + } + tor_free(fname); + return r; +} + /** Write the contents of <b>extrainfo</b> to the <b>maxlen</b>-byte string * <b>s</b>, signing them with <b>ident_key</b>. Return 0 on success, * negative on failure. */ @@ -1816,6 +1878,8 @@ extrainfo_dump_to_string(char *s, size_t maxlen, extrainfo_t *extrainfo, char *bandwidth_usage; int result; size_t len; + static int write_stats_to_extrainfo = 1; + time_t now = time(NULL); base16_encode(identity, sizeof(identity), extrainfo->cache_info.identity_digest, DIGEST_LEN); @@ -1827,23 +1891,79 @@ extrainfo_dump_to_string(char *s, size_t maxlen, extrainfo_t *extrainfo, "published %s\n%s", extrainfo->nickname, identity, published, bandwidth_usage); + tor_free(bandwidth_usage); if (result<0) return -1; - if (should_record_bridge_info(options)) { - char *geoip_summary = extrainfo_get_client_geoip_summary(time(NULL)); - if (geoip_summary) { - char geoip_start[ISO_TIME_LEN+1]; - format_iso_time(geoip_start, geoip_get_history_start()); - result = tor_snprintf(s+strlen(s), maxlen-strlen(s), - "geoip-start-time %s\n" - "geoip-client-origins %s\n", - geoip_start, geoip_summary); - control_event_clients_seen(geoip_start, geoip_summary); - tor_free(geoip_summary); - if (result<0) - return -1; + if (options->ExtraInfoStatistics && write_stats_to_extrainfo) { + char *contents = NULL; + log_info(LD_GENERAL, "Adding stats to extra-info descriptor."); + if (options->DirReqStatistics && + load_stats_file("stats"PATH_SEPARATOR"dirreq-stats", + "dirreq-stats-end", now, &contents) > 0) { + size_t pos = strlen(s); + if (strlcpy(s + pos, contents, maxlen - strlen(s)) != + strlen(contents)) { + log_warn(LD_DIR, "Could not write dirreq-stats to extra-info " + "descriptor."); + s[pos] = '\0'; + write_stats_to_extrainfo = 0; + } + tor_free(contents); + } + if (options->EntryStatistics && + load_stats_file("stats"PATH_SEPARATOR"entry-stats", + "entry-stats-end", now, &contents) > 0) { + size_t pos = strlen(s); + if (strlcpy(s + pos, contents, maxlen - strlen(s)) != + strlen(contents)) { + log_warn(LD_DIR, "Could not write entry-stats to extra-info " + "descriptor."); + s[pos] = '\0'; + write_stats_to_extrainfo = 0; + } + tor_free(contents); + } + if (options->CellStatistics && + load_stats_file("stats"PATH_SEPARATOR"buffer-stats", + "cell-stats-end", now, &contents) > 0) { + size_t pos = strlen(s); + if (strlcpy(s + pos, contents, maxlen - strlen(s)) != + strlen(contents)) { + log_warn(LD_DIR, "Could not write buffer-stats to extra-info " + "descriptor."); + s[pos] = '\0'; + write_stats_to_extrainfo = 0; + } + tor_free(contents); + } + if (options->ExitPortStatistics && + load_stats_file("stats"PATH_SEPARATOR"exit-stats", + "exit-stats-end", now, &contents) > 0) { + size_t pos = strlen(s); + if (strlcpy(s + pos, contents, maxlen - strlen(s)) != + strlen(contents)) { + log_warn(LD_DIR, "Could not write exit-stats to extra-info " + "descriptor."); + s[pos] = '\0'; + write_stats_to_extrainfo = 0; + } + tor_free(contents); + } + } + + if (should_record_bridge_info(options) && write_stats_to_extrainfo) { + const char *bridge_stats = geoip_get_bridge_stats_extrainfo(now); + if (bridge_stats) { + size_t pos = strlen(s); + if (strlcpy(s + pos, bridge_stats, maxlen - strlen(s)) != + strlen(bridge_stats)) { + log_warn(LD_DIR, "Could not write bridge-stats to extra-info " + "descriptor."); + s[pos] = '\0'; + write_stats_to_extrainfo = 0; + } } } @@ -1852,10 +1972,10 @@ extrainfo_dump_to_string(char *s, size_t maxlen, extrainfo_t *extrainfo, len += strlen(s+len); if (router_get_extrainfo_hash(s, digest)<0) return -1; - if (router_append_dirobj_signature(s+len, maxlen-len, digest, ident_key)<0) + if (router_append_dirobj_signature(s+len, maxlen-len, digest, DIGEST_LEN, + ident_key)<0) return -1; -#ifdef DEBUG_ROUTER_DUMP_ROUTER_TO_STRING { char *cp, *s_dup; extrainfo_t *ei_tmp; @@ -1871,31 +1991,26 @@ extrainfo_dump_to_string(char *s, size_t maxlen, extrainfo_t *extrainfo, tor_free(s_dup); extrainfo_free(ei_tmp); } -#endif - - return (int)strlen(s)+1; -} -/** Wrapper function for geoip_get_client_history(). It first discards - * any items in the client history that are too old -- it dumps anything - * more than 48 hours old, but it only considers whether to dump at most - * once per 48 hours, so we aren't too precise to an observer (see also - * r14780). - */ -char * -extrainfo_get_client_geoip_summary(time_t now) -{ - static time_t last_purged_at = 0; - int geoip_purge_interval = 48*60*60; -#ifdef ENABLE_GEOIP_STATS - if (get_options()->DirRecordUsageByCountry) - geoip_purge_interval = get_options()->DirRecordUsageRetainIPs; -#endif - if (now > last_purged_at+geoip_purge_interval) { - geoip_remove_old_clients(now-geoip_purge_interval); - last_purged_at = now; + if (options->ExtraInfoStatistics && write_stats_to_extrainfo) { + char *cp, *s_dup; + extrainfo_t *ei_tmp; + cp = s_dup = tor_strdup(s); + ei_tmp = extrainfo_parse_entry_from_string(cp, NULL, 1, NULL); + if (!ei_tmp) { + log_warn(LD_GENERAL, + "We just generated an extra-info descriptor with " + "statistics that we can't parse. Not adding statistics to " + "this or any future extra-info descriptors. Descriptor " + "was:\n%s", s); + write_stats_to_extrainfo = 0; + extrainfo_dump_to_string(s, maxlen, extrainfo, ident_key); + } + tor_free(s_dup); + extrainfo_free(ei_tmp); } - return geoip_get_client_history(now, GEOIP_CLIENT_CONNECT); + + return (int)strlen(s)+1; } /** Return true iff <b>s</b> is a legally valid server nickname. */ @@ -2022,26 +2137,16 @@ router_purpose_from_string(const char *s) void router_free_all(void) { - if (onionkey) - crypto_free_pk_env(onionkey); - if (lastonionkey) - crypto_free_pk_env(lastonionkey); - if (identitykey) - crypto_free_pk_env(identitykey); - if (key_lock) - tor_mutex_free(key_lock); - if (desc_routerinfo) - routerinfo_free(desc_routerinfo); - if (desc_extrainfo) - extrainfo_free(desc_extrainfo); - if (authority_signing_key) - crypto_free_pk_env(authority_signing_key); - if (authority_key_certificate) - authority_cert_free(authority_key_certificate); - if (legacy_signing_key) - crypto_free_pk_env(legacy_signing_key); - if (legacy_key_certificate) - authority_cert_free(legacy_key_certificate); + crypto_free_pk_env(onionkey); + crypto_free_pk_env(lastonionkey); + crypto_free_pk_env(identitykey); + tor_mutex_free(key_lock); + routerinfo_free(desc_routerinfo); + extrainfo_free(desc_extrainfo); + crypto_free_pk_env(authority_signing_key); + authority_cert_free(authority_key_certificate); + crypto_free_pk_env(legacy_signing_key); + authority_cert_free(legacy_key_certificate); if (warned_nonexistent_family) { SMARTLIST_FOREACH(warned_nonexistent_family, char *, cp, tor_free(cp)); diff --git a/src/or/router.h b/src/or/router.h new file mode 100644 index 0000000000..d90a7cff90 --- /dev/null +++ b/src/or/router.h @@ -0,0 +1,95 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2010, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file router.h + * \brief Header file for router.c. + **/ + +#ifndef _TOR_ROUTER_H +#define _TOR_ROUTER_H + +crypto_pk_env_t *get_onion_key(void); +time_t get_onion_key_set_at(void); +void set_identity_key(crypto_pk_env_t *k); +crypto_pk_env_t *get_identity_key(void); +int identity_key_is_set(void); +authority_cert_t *get_my_v3_authority_cert(void); +crypto_pk_env_t *get_my_v3_authority_signing_key(void); +authority_cert_t *get_my_v3_legacy_cert(void); +crypto_pk_env_t *get_my_v3_legacy_signing_key(void); +void dup_onion_keys(crypto_pk_env_t **key, crypto_pk_env_t **last); +void rotate_onion_key(void); +crypto_pk_env_t *init_key_from_file(const char *fname, int generate, + int severity); +void v3_authority_check_key_expiry(void); + +int init_keys(void); + +int check_whether_orport_reachable(void); +int check_whether_dirport_reachable(void); +void consider_testing_reachability(int test_or, int test_dir); +void router_orport_found_reachable(void); +void router_dirport_found_reachable(void); +void router_perform_bandwidth_test(int num_circs, time_t now); + +int authdir_mode(or_options_t *options); +int authdir_mode_v1(or_options_t *options); +int authdir_mode_v2(or_options_t *options); +int authdir_mode_v3(or_options_t *options); +int authdir_mode_any_main(or_options_t *options); +int authdir_mode_any_nonhidserv(or_options_t *options); +int authdir_mode_handles_descs(or_options_t *options, int purpose); +int authdir_mode_publishes_statuses(or_options_t *options); +int authdir_mode_tests_reachability(or_options_t *options); +int authdir_mode_bridge(or_options_t *options); + +int server_mode(or_options_t *options); +int advertised_server_mode(void); +int proxy_mode(or_options_t *options); +void consider_publishable_server(int force); + +void router_upload_dir_desc_to_dirservers(int force); +void mark_my_descriptor_dirty_if_older_than(time_t when); +void mark_my_descriptor_dirty(void); +void check_descriptor_bandwidth_changed(time_t now); +void check_descriptor_ipaddress_changed(time_t now); +void router_new_address_suggestion(const char *suggestion, + const dir_connection_t *d_conn); +int router_compare_to_my_exit_policy(edge_connection_t *conn); +routerinfo_t *router_get_my_routerinfo(void); +extrainfo_t *router_get_my_extrainfo(void); +const char *router_get_my_descriptor(void); +int router_digest_is_me(const char *digest); +int router_extrainfo_digest_is_me(const char *digest); +int router_is_me(routerinfo_t *router); +int router_fingerprint_is_me(const char *fp); +int router_pick_published_address(or_options_t *options, uint32_t *addr); +int router_rebuild_descriptor(int force); +int router_dump_router_to_string(char *s, size_t maxlen, routerinfo_t *router, + crypto_pk_env_t *ident_key); +int extrainfo_dump_to_string(char *s, size_t maxlen, extrainfo_t *extrainfo, + crypto_pk_env_t *ident_key); +int is_legal_nickname(const char *s); +int is_legal_nickname_or_hexdigest(const char *s); +int is_legal_hexdigest(const char *s); +void router_get_verbose_nickname(char *buf, const routerinfo_t *router); +void routerstatus_get_verbose_nickname(char *buf, + const routerstatus_t *router); +void router_reset_warnings(void); +void router_reset_reachability(void); +void router_free_all(void); + +const char *router_purpose_to_string(uint8_t p); +uint8_t router_purpose_from_string(const char *s); + +#ifdef ROUTER_PRIVATE +/* Used only by router.c and test.c */ +void get_platform_str(char *platform, size_t len); +#endif + +#endif + diff --git a/src/or/routerlist.c b/src/or/routerlist.c index 2fc80738f8..8808f56db9 100644 --- a/src/or/routerlist.c +++ b/src/or/routerlist.c @@ -12,6 +12,25 @@ **/ #include "or.h" +#include "circuitbuild.h" +#include "config.h" +#include "connection.h" +#include "control.h" +#include "directory.h" +#include "dirserv.h" +#include "dirvote.h" +#include "geoip.h" +#include "hibernate.h" +#include "main.h" +#include "networkstatus.h" +#include "policies.h" +#include "reasons.h" +#include "rendcommon.h" +#include "rendservice.h" +#include "rephist.h" +#include "router.h" +#include "routerlist.h" +#include "routerparse.h" // #define DEBUG_ROUTERLIST @@ -26,8 +45,8 @@ static void mark_all_trusteddirservers_up(void); static int router_nickname_matches(routerinfo_t *router, const char *nickname); static void trusted_dir_server_free(trusted_dir_server_t *ds); static void launch_router_descriptor_downloads(smartlist_t *downloadable, + routerstatus_t *source, time_t now); -static void update_consensus_router_descriptor_downloads(time_t now); static int signed_desc_digest_is_recognized(signed_descriptor_t *desc); static void update_router_have_minimum_dir_info(void); static const char *signed_descriptor_get_body_impl(signed_descriptor_t *desc, @@ -156,21 +175,24 @@ already_have_cert(authority_cert_t *cert) /** Load a bunch of new key certificates from the string <b>contents</b>. If * <b>from_store</b> is true, the certificates are from the cache, and we - * don't need to flush them to disk. If <b>from_store</b> is false, we need - * to flush any changed certificates to disk. Return 0 on success, -1 on - * failure. */ + * don't need to flush them to disk. If <b>flush</b> is true, we need + * to flush any changed certificates to disk now. Return 0 on success, -1 + * if any certs fail to parse. */ int trusted_dirs_load_certs_from_string(const char *contents, int from_store, int flush) { trusted_dir_server_t *ds; const char *s, *eos; + int failure_code = 0; for (s = contents; *s; s = eos) { authority_cert_t *cert = authority_cert_parse_from_string(s, &eos); cert_list_t *cl; - if (!cert) + if (!cert) { + failure_code = -1; break; + } ds = trusteddirserver_get_by_v3_auth_digest( cert->cache_info.identity_digest); log_debug(LD_DIR, "Parsed certificate for %s", @@ -181,15 +203,15 @@ trusted_dirs_load_certs_from_string(const char *contents, int from_store, log_info(LD_DIR, "Skipping %s certificate for %s that we " "already have.", from_store ? "cached" : "downloaded", - ds ? ds->nickname : "??"); + ds ? ds->nickname : "an old or new authority"); /* a duplicate on a download should be treated as a failure, since it * probably means we wanted a different secret key or we are trying to * replace an expired cert that has not in fact been updated. */ if (!from_store) { - log_warn(LD_DIR, "Got a certificate for %s that we already have. " - "Maybe they haven't updated it. Waiting for a while.", - ds ? ds->nickname : "??"); + log_warn(LD_DIR, "Got a certificate for %s, but we already have it. " + "Maybe they haven't updated it. Waiting for a while.", + ds ? ds->nickname : "an old or new authority"); authority_cert_dl_failed(cert->cache_info.identity_digest, 404); } @@ -224,7 +246,7 @@ trusted_dirs_load_certs_from_string(const char *contents, int from_store, ds->dir_port != cert->dir_port)) { char *a = tor_dup_ip(cert->addr); log_notice(LD_DIR, "Updating address for directory authority %s " - "from %s:%d to %s:%d based on in certificate.", + "from %s:%d to %s:%d based on certificate.", ds->nickname, ds->address, (int)ds->dir_port, a, cert->dir_port); tor_free(a); @@ -241,8 +263,11 @@ trusted_dirs_load_certs_from_string(const char *contents, int from_store, if (flush) trusted_dirs_flush_certs_to_disk(); + /* call this even if failure_code is <0, since some certs might have + * succeeded. */ networkstatus_note_certs_arrived(); - return 0; + + return failure_code; } /** Save all v3 key certificates to the cached-certs file. */ @@ -442,17 +467,18 @@ authority_certs_fetch_missing(networkstatus_t *status, time_t now) list_pending_downloads(pending, DIR_PURPOSE_FETCH_CERTIFICATE, "fp/"); if (status) { - SMARTLIST_FOREACH(status->voters, networkstatus_voter_info_t *, voter, - { - if (tor_digest_is_zero(voter->signing_key_digest)) - continue; /* This authority never signed this consensus, so don't - * go looking for a cert with key digest 0000000000. */ - if (!cache && - !trusteddirserver_get_by_v3_auth_digest(voter->identity_digest)) - continue; /* We are not a cache, and we don't know this authority.*/ - cl = get_cert_list(voter->identity_digest); + SMARTLIST_FOREACH_BEGIN(status->voters, networkstatus_voter_info_t *, + voter) { + if (!smartlist_len(voter->sigs)) + continue; /* This authority never signed this consensus, so don't + * go looking for a cert with key digest 0000000000. */ + if (!cache && + !trusteddirserver_get_by_v3_auth_digest(voter->identity_digest)) + continue; /* We are not a cache, and we don't know this authority.*/ + cl = get_cert_list(voter->identity_digest); + SMARTLIST_FOREACH_BEGIN(voter->sigs, document_signature_t *, sig) { cert = authority_cert_get_by_digests(voter->identity_digest, - voter->signing_key_digest); + sig->signing_key_digest); if (cert) { if (now < cert->expires) download_status_reset(&cl->dl_status); @@ -463,37 +489,36 @@ authority_certs_fetch_missing(networkstatus_t *status, time_t now) !digestmap_get(pending, voter->identity_digest)) { log_notice(LD_DIR, "We're missing a certificate from authority " "with signing key %s: launching request.", - hex_str(voter->signing_key_digest, DIGEST_LEN)); - smartlist_add(missing_digests, voter->identity_digest); + hex_str(sig->signing_key_digest, DIGEST_LEN)); + smartlist_add(missing_digests, sig->identity_digest); } - }); + } SMARTLIST_FOREACH_END(sig); + } SMARTLIST_FOREACH_END(voter); } - SMARTLIST_FOREACH(trusted_dir_servers, trusted_dir_server_t *, ds, - { - int found = 0; - if (!(ds->type & V3_AUTHORITY)) - continue; - if (smartlist_digest_isin(missing_digests, ds->v3_identity_digest)) - continue; - cl = get_cert_list(ds->v3_identity_digest); - SMARTLIST_FOREACH(cl->certs, authority_cert_t *, cert, - { - if (!ftime_definitely_after(now, cert->expires)) { - /* It's not expired, and we weren't looking for something to - * verify a consensus with. Call it done. */ - download_status_reset(&cl->dl_status); - found = 1; - break; - } - }); - if (!found && - download_status_is_ready(&cl->dl_status, now,MAX_CERT_DL_FAILURES) && - !digestmap_get(pending, ds->v3_identity_digest)) { - log_notice(LD_DIR, "No current certificate known for authority %s; " - "launching request.", ds->nickname); - smartlist_add(missing_digests, ds->v3_identity_digest); + SMARTLIST_FOREACH_BEGIN(trusted_dir_servers, trusted_dir_server_t *, ds) { + int found = 0; + if (!(ds->type & V3_AUTHORITY)) + continue; + if (smartlist_digest_isin(missing_digests, ds->v3_identity_digest)) + continue; + cl = get_cert_list(ds->v3_identity_digest); + SMARTLIST_FOREACH(cl->certs, authority_cert_t *, cert, { + if (!ftime_definitely_after(now, cert->expires)) { + /* It's not expired, and we weren't looking for something to + * verify a consensus with. Call it done. */ + download_status_reset(&cl->dl_status); + found = 1; + break; } }); + if (!found && + download_status_is_ready(&cl->dl_status, now,MAX_CERT_DL_FAILURES) && + !digestmap_get(pending, ds->v3_identity_digest)) { + log_notice(LD_DIR, "No current certificate known for authority %s; " + "launching request.", ds->nickname); + smartlist_add(missing_digests, ds->v3_identity_digest); + } + } SMARTLIST_FOREACH_END(ds); if (!smartlist_len(missing_digests)) { goto done; @@ -749,8 +774,7 @@ router_rebuild_store(int flags, desc_store_t *store) store->journal_len = 0; store->bytes_dropped = 0; done: - if (signed_descriptors) - smartlist_free(signed_descriptors); + smartlist_free(signed_descriptors); tor_free(fname); tor_free(fname_tmp); if (chunk_list) { @@ -968,8 +992,9 @@ router_get_trusteddirserver_by_digest(const char *digest) return NULL; } -/** Return the trusted_dir_server_t for the directory authority whose identity - * key hashes to <b>digest</b>, or NULL if no such authority is known. +/** Return the trusted_dir_server_t for the directory authority whose + * v3 identity key hashes to <b>digest</b>, or NULL if no such authority + * is known. */ trusted_dir_server_t * trusteddirserver_get_by_v3_auth_digest(const char *digest) @@ -1089,9 +1114,10 @@ router_pick_directory_server_impl(authority_type_t type, int flags) } SMARTLIST_FOREACH_END(status); if (smartlist_len(tunnel)) { - result = routerstatus_sl_choose_by_bandwidth(tunnel); + result = routerstatus_sl_choose_by_bandwidth(tunnel, WEIGHT_FOR_DIR); } else if (smartlist_len(overloaded_tunnel)) { - result = routerstatus_sl_choose_by_bandwidth(overloaded_tunnel); + result = routerstatus_sl_choose_by_bandwidth(overloaded_tunnel, + WEIGHT_FOR_DIR); } else if (smartlist_len(trusted_tunnel)) { /* FFFF We don't distinguish between trusteds and overloaded trusteds * yet. Maybe one day we should. */ @@ -1099,9 +1125,10 @@ router_pick_directory_server_impl(authority_type_t type, int flags) * is a feature, but it could easily be a bug. -RD */ result = smartlist_choose(trusted_tunnel); } else if (smartlist_len(direct)) { - result = routerstatus_sl_choose_by_bandwidth(direct); + result = routerstatus_sl_choose_by_bandwidth(direct, WEIGHT_FOR_DIR); } else if (smartlist_len(overloaded_direct)) { - result = routerstatus_sl_choose_by_bandwidth(overloaded_direct); + result = routerstatus_sl_choose_by_bandwidth(overloaded_direct, + WEIGHT_FOR_DIR); } else { result = smartlist_choose(trusted_direct); } @@ -1531,6 +1558,215 @@ kb_to_bytes(uint32_t bw) /** Helper function: * choose a random element of smartlist <b>sl</b>, weighted by + * the advertised bandwidth of each element using the consensus + * bandwidth weights. + * + * If <b>statuses</b> is zero, then <b>sl</b> is a list of + * routerinfo_t's. Otherwise it's a list of routerstatus_t's. + * + * If <b>rule</b>==WEIGHT_FOR_EXIT. we're picking an exit node: consider all + * nodes' bandwidth equally regardless of their Exit status, since there may + * be some in the list because they exit to obscure ports. If + * <b>rule</b>==NO_WEIGHTING, we're picking a non-exit node: weight + * exit-node's bandwidth less depending on the smallness of the fraction of + * Exit-to-total bandwidth. If <b>rule</b>==WEIGHT_FOR_GUARD, we're picking a + * guard node: consider all guard's bandwidth equally. Otherwise, weight + * guards proportionally less. + */ +static void * +smartlist_choose_by_bandwidth_weights(smartlist_t *sl, + bandwidth_weight_rule_t rule, + int statuses) +{ + int64_t weight_scale; + int64_t rand_bw; + double Wg = -1, Wm = -1, We = -1, Wd = -1; + double Wgb = -1, Wmb = -1, Web = -1, Wdb = -1; + double weighted_bw = 0; + double *bandwidths; + double tmp = 0; + unsigned int i; + + /* Can't choose exit and guard at same time */ + tor_assert(rule == NO_WEIGHTING || + rule == WEIGHT_FOR_EXIT || + rule == WEIGHT_FOR_GUARD || + rule == WEIGHT_FOR_MID || + rule == WEIGHT_FOR_DIR); + + if (smartlist_len(sl) == 0) { + log_info(LD_CIRC, + "Empty routerlist passed in to consensus weight node " + "selection for rule %s", + bandwidth_weight_rule_to_string(rule)); + return NULL; + } + + weight_scale = networkstatus_get_param(NULL, "bwweightscale", + BW_WEIGHT_SCALE); + + if (rule == WEIGHT_FOR_GUARD) { + Wg = networkstatus_get_bw_weight(NULL, "Wgg", -1); + Wm = networkstatus_get_bw_weight(NULL, "Wgm", -1); /* Bridges */ + We = 0; + Wd = networkstatus_get_bw_weight(NULL, "Wgd", -1); + + Wgb = networkstatus_get_bw_weight(NULL, "Wgb", -1); + Wmb = networkstatus_get_bw_weight(NULL, "Wmb", -1); + Web = networkstatus_get_bw_weight(NULL, "Web", -1); + Wdb = networkstatus_get_bw_weight(NULL, "Wdb", -1); + } else if (rule == WEIGHT_FOR_MID) { + Wg = networkstatus_get_bw_weight(NULL, "Wmg", -1); + Wm = networkstatus_get_bw_weight(NULL, "Wmm", -1); + We = networkstatus_get_bw_weight(NULL, "Wme", -1); + Wd = networkstatus_get_bw_weight(NULL, "Wmd", -1); + + Wgb = networkstatus_get_bw_weight(NULL, "Wgb", -1); + Wmb = networkstatus_get_bw_weight(NULL, "Wmb", -1); + Web = networkstatus_get_bw_weight(NULL, "Web", -1); + Wdb = networkstatus_get_bw_weight(NULL, "Wdb", -1); + } else if (rule == WEIGHT_FOR_EXIT) { + // Guards CAN be exits if they have weird exit policies + // They are d then I guess... + We = networkstatus_get_bw_weight(NULL, "Wee", -1); + Wm = networkstatus_get_bw_weight(NULL, "Wem", -1); /* Odd exit policies */ + Wd = networkstatus_get_bw_weight(NULL, "Wed", -1); + Wg = networkstatus_get_bw_weight(NULL, "Weg", -1); /* Odd exit policies */ + + Wgb = networkstatus_get_bw_weight(NULL, "Wgb", -1); + Wmb = networkstatus_get_bw_weight(NULL, "Wmb", -1); + Web = networkstatus_get_bw_weight(NULL, "Web", -1); + Wdb = networkstatus_get_bw_weight(NULL, "Wdb", -1); + } else if (rule == WEIGHT_FOR_DIR) { + We = networkstatus_get_bw_weight(NULL, "Wbe", -1); + Wm = networkstatus_get_bw_weight(NULL, "Wbm", -1); + Wd = networkstatus_get_bw_weight(NULL, "Wbd", -1); + Wg = networkstatus_get_bw_weight(NULL, "Wbg", -1); + + Wgb = Wmb = Web = Wdb = weight_scale; + } else if (rule == NO_WEIGHTING) { + Wg = Wm = We = Wd = weight_scale; + Wgb = Wmb = Web = Wdb = weight_scale; + } + + if (Wg < 0 || Wm < 0 || We < 0 || Wd < 0 || Wgb < 0 || Wmb < 0 || Wdb < 0 + || Web < 0) { + log_debug(LD_CIRC, + "Got negative bandwidth weights. Defaulting to old selection" + " algorithm."); + return NULL; // Use old algorithm. + } + + Wg /= weight_scale; + Wm /= weight_scale; + We /= weight_scale; + Wd /= weight_scale; + + Wgb /= weight_scale; + Wmb /= weight_scale; + Web /= weight_scale; + Wdb /= weight_scale; + + bandwidths = tor_malloc_zero(sizeof(double)*smartlist_len(sl)); + + // Cycle through smartlist and total the bandwidth. + for (i = 0; i < (unsigned)smartlist_len(sl); ++i) { + int is_exit = 0, is_guard = 0, is_dir = 0, this_bw = 0, is_me = 0; + double weight = 1; + if (statuses) { + routerstatus_t *status = smartlist_get(sl, i); + is_exit = status->is_exit; + is_guard = status->is_possible_guard; + is_dir = (status->dir_port != 0); + if (!status->has_bandwidth) { + tor_free(bandwidths); + /* This should never happen, unless all the authorites downgrade + * to 0.2.0 or rogue routerstatuses get inserted into our consensus. */ + log_warn(LD_BUG, + "Consensus is not listing bandwidths. Defaulting back to " + "old router selection algorithm."); + return NULL; + } + this_bw = kb_to_bytes(status->bandwidth); + if (router_digest_is_me(status->identity_digest)) + is_me = 1; + } else { + routerstatus_t *rs; + routerinfo_t *router = smartlist_get(sl, i); + rs = router_get_consensus_status_by_id( + router->cache_info.identity_digest); + is_exit = router->is_exit; + is_guard = router->is_possible_guard; + is_dir = (router->dir_port != 0); + if (rs && rs->has_bandwidth) { + this_bw = kb_to_bytes(rs->bandwidth); + } else { /* bridge or other descriptor not in our consensus */ + this_bw = router_get_advertised_bandwidth_capped(router); + } + if (router_digest_is_me(router->cache_info.identity_digest)) + is_me = 1; + } + if (is_guard && is_exit) { + weight = (is_dir ? Wdb*Wd : Wd); + } else if (is_guard) { + weight = (is_dir ? Wgb*Wg : Wg); + } else if (is_exit) { + weight = (is_dir ? Web*We : We); + } else { // middle + weight = (is_dir ? Wmb*Wm : Wm); + } + + bandwidths[i] = weight*this_bw; + weighted_bw += weight*this_bw; + if (is_me) + sl_last_weighted_bw_of_me = weight*this_bw; + } + + /* XXXX022 this is a kludge to expose these values. */ + sl_last_total_weighted_bw = weighted_bw; + + log_debug(LD_CIRC, "Choosing node for rule %s based on weights " + "Wg=%lf Wm=%lf We=%lf Wd=%lf with total bw %lf", + bandwidth_weight_rule_to_string(rule), + Wg, Wm, We, Wd, weighted_bw); + + /* If there is no bandwidth, choose at random */ + if (DBL_TO_U64(weighted_bw) == 0) { + log_warn(LD_CIRC, + "Weighted bandwidth is %lf in node selection for rule %s", + weighted_bw, bandwidth_weight_rule_to_string(rule)); + tor_free(bandwidths); + return smartlist_choose(sl); + } + + rand_bw = crypto_rand_uint64(DBL_TO_U64(weighted_bw)); + rand_bw++; /* crypto_rand_uint64() counts from 0, and we need to count + * from 1 below. See bug 1203 for details. */ + + /* Last, count through sl until we get to the element we picked */ + tmp = 0.0; + for (i=0; i < (unsigned)smartlist_len(sl); i++) { + tmp += bandwidths[i]; + if (tmp >= rand_bw) + break; + } + + if (i == (unsigned)smartlist_len(sl)) { + /* This was once possible due to round-off error, but shouldn't be able + * to occur any longer. */ + tor_fragile_assert(); + --i; + log_warn(LD_BUG, "Round-off error in computing bandwidth had an effect on " + " which router we chose. Please tell the developers. " + "%lf " U64_FORMAT " %lf", tmp, U64_PRINTF_ARG(rand_bw), + weighted_bw); + } + tor_free(bandwidths); + return smartlist_get(sl, i); +} + +/** Helper function: + * choose a random element of smartlist <b>sl</b>, weighted by * the advertised bandwidth of each element. * * If <b>statuses</b> is zero, then <b>sl</b> is a list of @@ -1565,11 +1801,24 @@ smartlist_choose_by_bandwidth(smartlist_t *sl, bandwidth_weight_rule_t rule, bitarray_t *guard_bits; int me_idx = -1; + // This function does not support WEIGHT_FOR_DIR + // or WEIGHT_FOR_MID + if (rule == WEIGHT_FOR_DIR || rule == WEIGHT_FOR_MID) { + rule = NO_WEIGHTING; + } + /* Can't choose exit and guard at same time */ tor_assert(rule == NO_WEIGHTING || rule == WEIGHT_FOR_EXIT || rule == WEIGHT_FOR_GUARD); + if (smartlist_len(sl) == 0) { + log_info(LD_CIRC, + "Empty routerlist passed in to old node selection for rule %s", + bandwidth_weight_rule_to_string(rule)); + return NULL; + } + /* First count the total bandwidth weight, and make a list * of each value. <0 means "unknown; no routerinfo." We use the * bits of negative values to remember whether the router was fast (-x)&1 @@ -1691,12 +1940,12 @@ smartlist_choose_by_bandwidth(smartlist_t *sl, bandwidth_weight_rule_t rule, * For detailed derivation of this formula, see * http://archives.seul.org/or/dev/Jul-2007/msg00056.html */ - if (rule == WEIGHT_FOR_EXIT) + if (rule == WEIGHT_FOR_EXIT || !total_exit_bw) exit_weight = 1.0; else exit_weight = 1.0 - all_bw/(3.0*exit_bw); - if (rule == WEIGHT_FOR_GUARD) + if (rule == WEIGHT_FOR_GUARD || !total_guard_bw) guard_weight = 1.0; else guard_weight = 1.0 - all_bw/(3.0*guard_bw); @@ -1745,6 +1994,8 @@ smartlist_choose_by_bandwidth(smartlist_t *sl, bandwidth_weight_rule_t rule, /* Almost done: choose a random value from the bandwidth weights. */ rand_bw = crypto_rand_uint64(total_bw); + rand_bw++; /* crypto_rand_uint64() counts from 0, and we need to count + * from 1 below. See bug 1203 for details. */ /* Last, count through sl until we get to the element we picked */ tmp = 0; @@ -1788,26 +2039,34 @@ routerinfo_t * routerlist_sl_choose_by_bandwidth(smartlist_t *sl, bandwidth_weight_rule_t rule) { - return smartlist_choose_by_bandwidth(sl, rule, 0); + routerinfo_t *ret; + if ((ret = smartlist_choose_by_bandwidth_weights(sl, rule, 0))) { + return ret; + } else { + return smartlist_choose_by_bandwidth(sl, rule, 0); + } } /** Choose a random element of status list <b>sl</b>, weighted by * the advertised bandwidth of each status. */ routerstatus_t * -routerstatus_sl_choose_by_bandwidth(smartlist_t *sl) +routerstatus_sl_choose_by_bandwidth(smartlist_t *sl, + bandwidth_weight_rule_t rule) { /* We are choosing neither exit nor guard here. Weight accordingly. */ - return smartlist_choose_by_bandwidth(sl, NO_WEIGHTING, 1); + routerstatus_t *ret; + if ((ret = smartlist_choose_by_bandwidth_weights(sl, rule, 1))) { + return ret; + } else { + return smartlist_choose_by_bandwidth(sl, rule, 1); + } } -/** Return a random running router from the routerlist. If any node - * named in <b>preferred</b> is available, pick one of those. Never +/** Return a random running router from the routerlist. Never * pick a node whose routerinfo is in * <b>excludedsmartlist</b>, or whose routerinfo matches <b>excludedset</b>, - * even if they are the only nodes - * available. If <b>CRN_STRICT_PREFERRED</b> is set in flags, never pick - * any node besides those in <b>preferred</b>. + * even if they are the only nodes available. * If <b>CRN_NEED_UPTIME</b> is set in flags and any router has more than * a minimum uptime, return one of those. * If <b>CRN_NEED_CAPACITY</b> is set in flags, weight your choice by the @@ -1820,8 +2079,7 @@ routerstatus_sl_choose_by_bandwidth(smartlist_t *sl) * node (that is, possibly discounting exit nodes). */ routerinfo_t * -router_choose_random_node(const char *preferred, - smartlist_t *excludedsmartlist, +router_choose_random_node(smartlist_t *excludedsmartlist, routerset_t *excludedset, router_crn_flags_t flags) { @@ -1829,18 +2087,16 @@ router_choose_random_node(const char *preferred, const int need_capacity = (flags & CRN_NEED_CAPACITY) != 0; const int need_guard = (flags & CRN_NEED_GUARD) != 0; const int allow_invalid = (flags & CRN_ALLOW_INVALID) != 0; - const int strict = (flags & CRN_STRICT_PREFERRED) != 0; const int weight_for_exit = (flags & CRN_WEIGHT_AS_EXIT) != 0; - smartlist_t *sl, *excludednodes; + smartlist_t *sl=smartlist_create(), + *excludednodes=smartlist_create(); routerinfo_t *choice = NULL, *r; bandwidth_weight_rule_t rule; tor_assert(!(weight_for_exit && need_guard)); rule = weight_for_exit ? WEIGHT_FOR_EXIT : - (need_guard ? WEIGHT_FOR_GUARD : NO_WEIGHTING); - - excludednodes = smartlist_create(); + (need_guard ? WEIGHT_FOR_GUARD : WEIGHT_FOR_MID); /* Exclude relays that allow single hop exit circuits, if the user * wants to (such relays might be risky) */ @@ -1857,60 +2113,35 @@ router_choose_random_node(const char *preferred, routerlist_add_family(excludednodes, r); } - /* Try the preferred nodes first. Ignore need_uptime and need_capacity - * and need_guard, since the user explicitly asked for these nodes. */ - if (preferred) { - sl = smartlist_create(); - add_nickname_list_to_smartlist(sl,preferred,1); - smartlist_subtract(sl,excludednodes); - if (excludedsmartlist) - smartlist_subtract(sl,excludedsmartlist); - if (excludedset) - routerset_subtract_routers(sl,excludedset); - choice = smartlist_choose(sl); - smartlist_free(sl); - } - if (!choice && !strict) { - /* Then give up on our preferred choices: any node - * will do that has the required attributes. */ - sl = smartlist_create(); - router_add_running_routers_to_smartlist(sl, allow_invalid, - need_uptime, need_capacity, - need_guard); - smartlist_subtract(sl,excludednodes); - if (excludedsmartlist) - smartlist_subtract(sl,excludedsmartlist); - if (excludedset) - routerset_subtract_routers(sl,excludedset); - - if (need_capacity || need_guard) - choice = routerlist_sl_choose_by_bandwidth(sl, rule); - else - choice = smartlist_choose(sl); - - smartlist_free(sl); - if (!choice && (need_uptime || need_capacity || need_guard)) { - /* try once more -- recurse but with fewer restrictions. */ - log_info(LD_CIRC, - "We couldn't find any live%s%s%s routers; falling back " - "to list of all routers.", - need_capacity?", fast":"", - need_uptime?", stable":"", - need_guard?", guard":""); - flags &= ~ (CRN_NEED_UPTIME|CRN_NEED_CAPACITY|CRN_NEED_GUARD); - choice = router_choose_random_node( - NULL, excludedsmartlist, excludedset, flags); - } + router_add_running_routers_to_smartlist(sl, allow_invalid, + need_uptime, need_capacity, + need_guard); + smartlist_subtract(sl,excludednodes); + if (excludedsmartlist) + smartlist_subtract(sl,excludedsmartlist); + if (excludedset) + routerset_subtract_routers(sl,excludedset); + + // Always weight by bandwidth + choice = routerlist_sl_choose_by_bandwidth(sl, rule); + + smartlist_free(sl); + if (!choice && (need_uptime || need_capacity || need_guard)) { + /* try once more -- recurse but with fewer restrictions. */ + log_info(LD_CIRC, + "We couldn't find any live%s%s%s routers; falling back " + "to list of all routers.", + need_capacity?", fast":"", + need_uptime?", stable":"", + need_guard?", guard":""); + flags &= ~ (CRN_NEED_UPTIME|CRN_NEED_CAPACITY|CRN_NEED_GUARD); + choice = router_choose_random_node( + excludedsmartlist, excludedset, flags); } smartlist_free(excludednodes); if (!choice) { - if (strict) { - log_warn(LD_CIRC, "All preferred nodes were down when trying to choose " - "node, and the Strict[...]Nodes option is set. Failing."); - } else { - log_warn(LD_CIRC, - "No available nodes when trying to choose node. Failing."); - } + log_warn(LD_CIRC, + "No available nodes when trying to choose node. Failing."); } return choice; } @@ -2370,6 +2601,9 @@ extrainfo_free(extrainfo_t *extrainfo) static void signed_descriptor_free(signed_descriptor_t *sd) { + if (!sd) + return; + tor_free(sd->signed_descriptor_body); /* XXXX remove this once more bugs go away. */ @@ -2401,7 +2635,8 @@ _extrainfo_free(void *e) void routerlist_free(routerlist_t *rl) { - tor_assert(rl); + if (!rl) + return; rimap_free(rl->identity_map, NULL); sdmap_free(rl->desc_digest_map, NULL); sdmap_free(rl->desc_by_eid_map, NULL); @@ -2440,46 +2675,6 @@ dump_routerlist_mem_usage(int severity) "In %d old descriptors: "U64_FORMAT" bytes.", smartlist_len(routerlist->routers), U64_PRINTF_ARG(livedescs), smartlist_len(routerlist->old_routers), U64_PRINTF_ARG(olddescs)); - -#if 0 - { - const smartlist_t *networkstatus_v2_list = networkstatus_get_v2_list(); - networkstatus_t *consensus = networkstatus_get_latest_consensus(); - log(severity, LD_DIR, "Now let's look through old_descriptors!"); - SMARTLIST_FOREACH(routerlist->old_routers, signed_descriptor_t *, sd, { - int in_v2 = 0; - int in_v3 = 0; - char published[ISO_TIME_LEN+1]; - char last_valid_until[ISO_TIME_LEN+1]; - char last_served_at[ISO_TIME_LEN+1]; - char id[HEX_DIGEST_LEN+1]; - routerstatus_t *rs; - format_iso_time(published, sd->published_on); - format_iso_time(last_valid_until, sd->last_listed_as_valid_until); - format_iso_time(last_served_at, sd->last_served_at); - base16_encode(id, sizeof(id), sd->identity_digest, DIGEST_LEN); - SMARTLIST_FOREACH(networkstatus_v2_list, networkstatus_v2_t *, ns, - { - rs = networkstatus_v2_find_entry(ns, sd->identity_digest); - if (rs && !memcmp(rs->descriptor_digest, - sd->signed_descriptor_digest, DIGEST_LEN)) { - in_v2 = 1; break; - } - }); - if (consensus) { - rs = networkstatus_vote_find_entry(consensus, sd->identity_digest); - if (rs && !memcmp(rs->descriptor_digest, - sd->signed_descriptor_digest, DIGEST_LEN)) - in_v3 = 1; - } - log(severity, LD_DIR, - "Old descriptor for %s (published %s) %sin v2 ns, %sin v3 " - "consensus. Last valid until %s; last served at %s.", - id, published, in_v2 ? "" : "not ", in_v3 ? "" : "not ", - last_valid_until, last_served_at); - }); - } -#endif } /** Debugging helper: If <b>idx</b> is nonnegative, assert that <b>ri</b> is @@ -2849,8 +3044,7 @@ routerlist_reparse_old(routerlist_t *rl, signed_descriptor_t *sd) void routerlist_free_all(void) { - if (routerlist) - routerlist_free(routerlist); + routerlist_free(routerlist); routerlist = NULL; if (warned_nicknames) { SMARTLIST_FOREACH(warned_nicknames, char *, cp, tor_free(cp)); @@ -2957,15 +3151,23 @@ router_add_to_routerlist(routerinfo_t *router, const char **msg, id_digest = router->cache_info.identity_digest; + old_router = router_get_by_digest(id_digest); + /* Make sure that we haven't already got this exact descriptor. */ if (sdmap_get(routerlist->desc_digest_map, router->cache_info.signed_descriptor_digest)) { - log_info(LD_DIR, - "Dropping descriptor that we already have for router '%s'", - router->nickname); - *msg = "Router descriptor was not new."; - routerinfo_free(router); - return ROUTER_WAS_NOT_NEW; + /* If we have this descriptor already and the new descriptor is a bridge + * descriptor, replace it. If we had a bridge descriptor before and the + * new one is not a bridge descriptor, don't replace it. */ + if (old_router && (!routerinfo_is_a_configured_bridge(router) || + routerinfo_is_a_configured_bridge(old_router))) { + log_info(LD_DIR, + "Dropping descriptor that we already have for router '%s'", + router->nickname); + *msg = "Router descriptor was not new."; + routerinfo_free(router); + return ROUTER_WAS_NOT_NEW; + } } if (authdir) { @@ -3002,15 +3204,14 @@ router_add_to_routerlist(routerinfo_t *router, const char **msg, SMARTLIST_FOREACH(networkstatus_v2_list, networkstatus_v2_t *, ns, { routerstatus_t *rs = - networkstatus_v2_find_entry(ns, router->cache_info.identity_digest); + networkstatus_v2_find_entry(ns, id_digest); if (rs && !memcmp(rs->descriptor_digest, router->cache_info.signed_descriptor_digest, DIGEST_LEN)) rs->need_to_mirror = 0; }); if (consensus) { - routerstatus_t *rs = networkstatus_vote_find_entry(consensus, - router->cache_info.identity_digest); + routerstatus_t *rs = networkstatus_vote_find_entry(consensus, id_digest); if (rs && !memcmp(rs->descriptor_digest, router->cache_info.signed_descriptor_digest, DIGEST_LEN)) { @@ -3032,13 +3233,11 @@ router_add_to_routerlist(routerinfo_t *router, const char **msg, } /* If we have a router with the same identity key, choose the newer one. */ - old_router = rimap_get(routerlist->identity_map, - router->cache_info.identity_digest); if (old_router) { if (!in_consensus && (router->cache_info.published_on <= old_router->cache_info.published_on)) { /* Same key, but old. This one is not listed in the consensus. */ - log_debug(LD_DIR, "Skipping not-new descriptor for router '%s'", + log_debug(LD_DIR, "Not-new descriptor for router '%s'", router->nickname); /* Only journal this desc if we'll be serving it. */ if (!from_cache && should_cache_old_descriptors()) @@ -3081,9 +3280,15 @@ router_add_to_routerlist(routerinfo_t *router, const char **msg, /* We haven't seen a router with this identity before. Add it to the end of * the list. */ routerlist_insert(routerlist, router); - if (!from_cache) + if (!from_cache) { + if (authdir) { + /* launch an immediate reachability test, so we will have an opinion + * soon in case we're generating a consensus soon */ + dirserv_single_reachability_test(time(NULL), router); + } signed_desc_append_to_journal(&router->cache_info, &routerlist->desc_store); + } directory_set_dirty(); return ROUTER_ADDED_SUCCESSFULLY; } @@ -3739,12 +3944,8 @@ add_trusted_dir_server(const char *nickname, const char *address, if (ent->or_port) ent->fake_status.version_supports_begindir = 1; -/* XX021 - wait until authorities are upgraded */ -#if 0 + ent->fake_status.version_supports_conditional_consensus = 1; -#else - ent->fake_status.version_supports_conditional_consensus = 0; -#endif smartlist_add(trusted_dir_servers, ent); router_dir_info_changed(); @@ -3759,10 +3960,8 @@ authority_cert_free(authority_cert_t *cert) return; tor_free(cert->cache_info.signed_descriptor_body); - if (cert->signing_key) - crypto_free_pk_env(cert->signing_key); - if (cert->identity_key) - crypto_free_pk_env(cert->identity_key); + crypto_free_pk_env(cert->signing_key); + crypto_free_pk_env(cert->identity_key); tor_free(cert); } @@ -3771,6 +3970,9 @@ authority_cert_free(authority_cert_t *cert) static void trusted_dir_server_free(trusted_dir_server_t *ds) { + if (!ds) + return; + tor_free(ds->nickname); tor_free(ds->description); tor_free(ds->address); @@ -3824,7 +4026,7 @@ list_pending_downloads(digestmap_t *result, const char *resource = TO_DIR_CONN(conn)->requested_resource; if (!strcmpstart(resource, prefix)) dir_split_resource_into_fingerprints(resource + p_len, - tmp, NULL, 1, 0); + tmp, NULL, DSR_HEX); } }); SMARTLIST_FOREACH(tmp, char *, d, @@ -3926,7 +4128,7 @@ client_would_use_router(routerstatus_t *rs, time_t now, or_options_t *options) * this number per server. */ #define MIN_DL_PER_REQUEST 4 /** To prevent a single screwy cache from confusing us by selective reply, - * try to split our requests into at least this this many requests. */ + * try to split our requests into at least this many requests. */ #define MIN_REQUESTS 3 /** If we want fewer than this many descriptors, wait until we * want more, or until MAX_CLIENT_INTERVAL_WITHOUT_REQUEST has @@ -3940,7 +4142,8 @@ client_would_use_router(routerstatus_t *rs, time_t now, or_options_t *options) * whether to delay fetching until we have more. If we don't want to delay, * launch one or more requests to the appropriate directory authorities. */ static void -launch_router_descriptor_downloads(smartlist_t *downloadable, time_t now) +launch_router_descriptor_downloads(smartlist_t *downloadable, + routerstatus_t *source, time_t now) { int should_delay = 0, n_downloadable; or_options_t *options = get_options(); @@ -4010,7 +4213,7 @@ launch_router_descriptor_downloads(smartlist_t *downloadable, time_t now) req_plural, n_downloadable, rtr_plural, n_per_request); smartlist_sort_digests(downloadable); for (i=0; i < n_downloadable; i += n_per_request) { - initiate_descriptor_downloads(NULL, DIR_PURPOSE_FETCH_SERVERDESC, + initiate_descriptor_downloads(source, DIR_PURPOSE_FETCH_SERVERDESC, downloadable, i, i+n_per_request, pds_flags); } @@ -4166,18 +4369,18 @@ update_router_descriptor_cache_downloads_v2(time_t now) digestmap_free(map,NULL); } -/** For any descriptor that we want that's currently listed in the live - * consensus, download it as appropriate. */ -static void -update_consensus_router_descriptor_downloads(time_t now) +/** For any descriptor that we want that's currently listed in + * <b>consensus</b>, download it as appropriate. */ +void +update_consensus_router_descriptor_downloads(time_t now, int is_vote, + networkstatus_t *consensus) { or_options_t *options = get_options(); digestmap_t *map = NULL; smartlist_t *no_longer_old = smartlist_create(); smartlist_t *downloadable = smartlist_create(); + routerstatus_t *source = NULL; int authdir = authdir_mode(options); - networkstatus_t *consensus = - networkstatus_get_reasonably_live_consensus(now); int n_delayed=0, n_have=0, n_would_reject=0, n_wouldnt_use=0, n_inprogress=0, n_in_oldrouters=0; @@ -4186,10 +4389,24 @@ update_consensus_router_descriptor_downloads(time_t now) if (!consensus) goto done; + if (is_vote) { + /* where's it from, so we know whom to ask for descriptors */ + trusted_dir_server_t *ds; + networkstatus_voter_info_t *voter = smartlist_get(consensus->voters, 0); + tor_assert(voter); + ds = trusteddirserver_get_by_v3_auth_digest(voter->identity_digest); + if (ds) + source = &(ds->fake_status); + else + log_warn(LD_DIR, "couldn't lookup source from vote?"); + } + map = digestmap_new(); list_pending_descriptor_downloads(map, 0); - SMARTLIST_FOREACH(consensus->routerstatus_list, routerstatus_t *, rs, + SMARTLIST_FOREACH(consensus->routerstatus_list, void *, rsp, { + routerstatus_t *rs = + is_vote ? &(((vote_routerstatus_t *)rsp)->status) : rsp; signed_descriptor_t *sd; if ((sd = router_get_by_descriptor_digest(rs->descriptor_digest))) { routerinfo_t *ri; @@ -4224,6 +4441,18 @@ update_consensus_router_descriptor_downloads(time_t now) ++n_wouldnt_use; continue; /* We would never use it ourself. */ } + if (is_vote && source) { + char time_bufnew[ISO_TIME_LEN+1]; + char time_bufold[ISO_TIME_LEN+1]; + routerinfo_t *oldrouter = router_get_by_digest(rs->identity_digest); + format_iso_time(time_bufnew, rs->published_on); + if (oldrouter) + format_iso_time(time_bufold, oldrouter->cache_info.published_on); + log_info(LD_DIR, "Learned about %s (%s vs %s) from %s's vote (%s)", + rs->nickname, time_bufnew, + oldrouter ? time_bufold : "none", + source->nickname, oldrouter ? "known" : "unknown"); + } smartlist_add(downloadable, rs->descriptor_digest); }); @@ -4257,7 +4486,7 @@ update_consensus_router_descriptor_downloads(time_t now) smartlist_len(downloadable), n_delayed, n_have, n_in_oldrouters, n_would_reject, n_wouldnt_use, n_inprogress); - launch_router_descriptor_downloads(downloadable, now); + launch_router_descriptor_downloads(downloadable, source, now); digestmap_free(map, NULL); done: @@ -4282,7 +4511,8 @@ update_router_descriptor_downloads(time_t now) if (directory_fetches_dir_info_early(options)) { update_router_descriptor_cache_downloads_v2(now); } - update_consensus_router_descriptor_downloads(now); + update_consensus_router_descriptor_downloads(now, 0, + networkstatus_get_reasonably_live_consensus(now)); /* XXXX021 we could be smarter here; see notes on bug 652. */ /* If we're a server that doesn't have a configured address, we rely on @@ -4814,8 +5044,8 @@ esc_router_info(routerinfo_t *router) static char *info=NULL; char *esc_contact, *esc_platform; size_t len; - if (info) - tor_free(info); + tor_free(info); + if (!router) return NULL; /* we're exiting; just free the memory we use */ @@ -4950,9 +5180,8 @@ void routerset_refresh_countries(routerset_t *target) { int cc; - if (target->countries) { - bitarray_free(target->countries); - } + bitarray_free(target->countries); + if (!geoip_is_loaded()) { target->countries = NULL; target->n_countries = 0; @@ -5026,7 +5255,9 @@ routerset_parse(routerset_t *target, const char *s, const char *description) return r; } -/** DOCDOC */ +/** Called when we change a node set, or when we reload the geoip list: + * recompute all country info in all configuration node sets and in the + * routerlist. */ void refresh_all_country_info(void) { @@ -5194,9 +5425,13 @@ routerset_get_all_routers(smartlist_t *out, const routerset_t *routerset, } } -/** Add to <b>target</b> every routerinfo_t from <b>source</b> that is in - * <b>include</b>, but not excluded in a more specific fashion by - * <b>exclude</b>. If <b>running_only</b>, only include running routers. +/** Add to <b>target</b> every routerinfo_t from <b>source</b> except: + * + * 1) Don't add it if <b>include</b> is non-empty and the relay isn't in + * <b>include</b>; and + * 2) Don't add it if <b>exclude</b> is non-empty and the relay is + * excluded in a more specific fashion by <b>exclude</b>. + * 3) If <b>running_only</b>, don't add non-running routers. */ void routersets_get_disjunction(smartlist_t *target, @@ -5266,35 +5501,15 @@ routerset_equal(const routerset_t *old, const routerset_t *new) }); return 1; - -#if 0 - /* XXXX: This won't work if the names/digests are identical but in a - different order. Checking for exact equality would be heavy going, - is it worth it? -RH*/ - /* This code is totally bogus; sizeof doesn't work even remotely like this - * code seems to think. Let's revert to a string-based comparison for - * now. -NM*/ - if (sizeof(old->names) != sizeof(new->names)) - return 0; - - if (memcmp(old->names,new->names,sizeof(new->names))) - return 0; - if (sizeof(old->digests) != sizeof(new->digests)) - return 0; - if (memcmp(old->digests,new->digests,sizeof(new->digests))) - return 0; - if (sizeof(old->countries) != sizeof(new->countries)) - return 0; - if (memcmp(old->countries,new->countries,sizeof(new->countries))) - return 0; - return 1; -#endif } /** Free all storage held in <b>routerset</b>. */ void routerset_free(routerset_t *routerset) { + if (!routerset) + return; + SMARTLIST_FOREACH(routerset->list, char *, cp, tor_free(cp)); smartlist_free(routerset->list); SMARTLIST_FOREACH(routerset->policies, addr_policy_t *, p, @@ -5305,8 +5520,7 @@ routerset_free(routerset_t *routerset) strmap_free(routerset->names, NULL); digestmap_free(routerset->digests, NULL); - if (routerset->countries) - bitarray_free(routerset->countries); + bitarray_free(routerset->countries); tor_free(routerset); } diff --git a/src/or/routerlist.h b/src/or/routerlist.h new file mode 100644 index 0000000000..e31b07aef5 --- /dev/null +++ b/src/or/routerlist.h @@ -0,0 +1,193 @@ +/* Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2010, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file routerlist.h + * \brief Header file for routerlist.c. + **/ + +#ifndef _TOR_ROUTERLIST_H +#define _TOR_ROUTERLIST_H + +int get_n_authorities(authority_type_t type); +int trusted_dirs_reload_certs(void); +int trusted_dirs_load_certs_from_string(const char *contents, int from_store, + int flush); +void trusted_dirs_flush_certs_to_disk(void); +authority_cert_t *authority_cert_get_newest_by_id(const char *id_digest); +authority_cert_t *authority_cert_get_by_sk_digest(const char *sk_digest); +authority_cert_t *authority_cert_get_by_digests(const char *id_digest, + const char *sk_digest); +void authority_cert_get_all(smartlist_t *certs_out); +void authority_cert_dl_failed(const char *id_digest, int status); +void authority_certs_fetch_missing(networkstatus_t *status, time_t now); +int router_reload_router_list(void); +smartlist_t *router_get_trusted_dir_servers(void); + +routerstatus_t *router_pick_directory_server(authority_type_t type, int flags); +trusted_dir_server_t *router_get_trusteddirserver_by_digest(const char *d); +trusted_dir_server_t *trusteddirserver_get_by_v3_auth_digest(const char *d); +routerstatus_t *router_pick_trusteddirserver(authority_type_t type, int flags); +int router_get_my_share_of_directory_requests(double *v2_share_out, + double *v3_share_out); +void router_reset_status_download_failures(void); +void routerlist_add_family(smartlist_t *sl, routerinfo_t *router); +int routers_in_same_family(routerinfo_t *r1, routerinfo_t *r2); +void add_nickname_list_to_smartlist(smartlist_t *sl, const char *list, + int must_be_running); +int router_nickname_is_in_list(routerinfo_t *router, const char *list); +routerinfo_t *routerlist_find_my_routerinfo(void); +routerinfo_t *router_find_exact_exit_enclave(const char *address, + uint16_t port); +int router_is_unreliable(routerinfo_t *router, int need_uptime, + int need_capacity, int need_guard); +uint32_t router_get_advertised_bandwidth(routerinfo_t *router); +uint32_t router_get_advertised_bandwidth_capped(routerinfo_t *router); + +routerinfo_t *routerlist_sl_choose_by_bandwidth(smartlist_t *sl, + bandwidth_weight_rule_t rule); +routerstatus_t *routerstatus_sl_choose_by_bandwidth(smartlist_t *sl, + bandwidth_weight_rule_t rule); + +routerinfo_t *router_choose_random_node(smartlist_t *excludedsmartlist, + struct routerset_t *excludedset, + router_crn_flags_t flags); + +routerinfo_t *router_get_by_nickname(const char *nickname, + int warn_if_unnamed); +int router_digest_version_as_new_as(const char *digest, const char *cutoff); +int router_digest_is_trusted_dir_type(const char *digest, + authority_type_t type); +#define router_digest_is_trusted_dir(d) \ + router_digest_is_trusted_dir_type((d), NO_AUTHORITY) + +int router_addr_is_trusted_dir(uint32_t addr); +int hexdigest_to_digest(const char *hexdigest, char *digest); +routerinfo_t *router_get_by_hexdigest(const char *hexdigest); +routerinfo_t *router_get_by_digest(const char *digest); +signed_descriptor_t *router_get_by_descriptor_digest(const char *digest); +signed_descriptor_t *router_get_by_extrainfo_digest(const char *digest); +signed_descriptor_t *extrainfo_get_by_descriptor_digest(const char *digest); +const char *signed_descriptor_get_body(signed_descriptor_t *desc); +const char *signed_descriptor_get_annotations(signed_descriptor_t *desc); +routerlist_t *router_get_routerlist(void); +void routerinfo_free(routerinfo_t *router); +void extrainfo_free(extrainfo_t *extrainfo); +void routerlist_free(routerlist_t *rl); +void dump_routerlist_mem_usage(int severity); +void routerlist_remove(routerlist_t *rl, routerinfo_t *ri, int make_old, + time_t now); +void routerlist_free_all(void); +void routerlist_reset_warnings(void); +void router_set_status(const char *digest, int up); + +static int WRA_WAS_ADDED(was_router_added_t s); +static int WRA_WAS_OUTDATED(was_router_added_t s); +static int WRA_WAS_REJECTED(was_router_added_t s); +/** Return true iff the descriptor was added. It might still be necessary to + * check whether the descriptor generator should be notified. + */ +static INLINE int +WRA_WAS_ADDED(was_router_added_t s) { + return s == ROUTER_ADDED_SUCCESSFULLY || s == ROUTER_ADDED_NOTIFY_GENERATOR; +} +/** Return true iff the descriptor was not added because it was either: + * - not in the consensus + * - neither in the consensus nor in any networkstatus document + * - it was outdated. + */ +static INLINE int WRA_WAS_OUTDATED(was_router_added_t s) +{ + return (s == ROUTER_WAS_NOT_NEW || + s == ROUTER_NOT_IN_CONSENSUS || + s == ROUTER_NOT_IN_CONSENSUS_OR_NETWORKSTATUS); +} +static INLINE int WRA_WAS_REJECTED(was_router_added_t s) +{ + return (s == ROUTER_AUTHDIR_REJECTS); +} +was_router_added_t router_add_to_routerlist(routerinfo_t *router, + const char **msg, + int from_cache, + int from_fetch); +was_router_added_t router_add_extrainfo_to_routerlist( + extrainfo_t *ei, const char **msg, + int from_cache, int from_fetch); +void routerlist_remove_old_routers(void); +int router_load_single_router(const char *s, uint8_t purpose, int cache, + const char **msg); +int router_load_routers_from_string(const char *s, const char *eos, + saved_location_t saved_location, + smartlist_t *requested_fingerprints, + int descriptor_digests, + const char *prepend_annotations); +void router_load_extrainfo_from_string(const char *s, const char *eos, + saved_location_t saved_location, + smartlist_t *requested_fingerprints, + int descriptor_digests); + +void routerlist_retry_directory_downloads(time_t now); +int router_exit_policy_all_routers_reject(uint32_t addr, uint16_t port, + int need_uptime); +int router_exit_policy_rejects_all(routerinfo_t *router); +trusted_dir_server_t *add_trusted_dir_server(const char *nickname, + const char *address, + uint16_t dir_port, uint16_t or_port, + const char *digest, const char *v3_auth_digest, + authority_type_t type); +void authority_cert_free(authority_cert_t *cert); +void clear_trusted_dir_servers(void); +int any_trusted_dir_is_v1_authority(void); +void update_consensus_router_descriptor_downloads(time_t now, int is_vote, + networkstatus_t *consensus); +void update_router_descriptor_downloads(time_t now); +void update_extrainfo_downloads(time_t now); +int router_have_minimum_dir_info(void); +void router_dir_info_changed(void); +const char *get_dir_info_status_string(void); +int count_loading_descriptors_progress(void); +void router_reset_descriptor_download_failures(void); +int router_differences_are_cosmetic(routerinfo_t *r1, routerinfo_t *r2); +int routerinfo_incompatible_with_extrainfo(routerinfo_t *ri, extrainfo_t *ei, + signed_descriptor_t *sd, + const char **msg); + +void routerlist_assert_ok(routerlist_t *rl); +const char *esc_router_info(routerinfo_t *router); +void routers_sort_by_identity(smartlist_t *routers); + +routerset_t *routerset_new(void); +int routerset_parse(routerset_t *target, const char *s, + const char *description); +void routerset_union(routerset_t *target, const routerset_t *source); +int routerset_is_list(const routerset_t *set); +int routerset_needs_geoip(const routerset_t *set); +int routerset_contains_router(const routerset_t *set, routerinfo_t *ri); +int routerset_contains_routerstatus(const routerset_t *set, + routerstatus_t *rs); +int routerset_contains_extendinfo(const routerset_t *set, + const extend_info_t *ei); +void routerset_get_all_routers(smartlist_t *out, const routerset_t *routerset, + int running_only); +void routersets_get_disjunction(smartlist_t *target, const smartlist_t *source, + const routerset_t *include, + const routerset_t *exclude, int running_only); +void routerset_subtract_routers(smartlist_t *out, + const routerset_t *routerset); +char *routerset_to_string(const routerset_t *routerset); +void routerset_refresh_countries(routerset_t *target); +int routerset_equal(const routerset_t *old, const routerset_t *new); +void routerset_free(routerset_t *routerset); +void routerinfo_set_country(routerinfo_t *ri); +void routerlist_refresh_countries(void); +void refresh_all_country_info(void); + +int hid_serv_get_responsible_directories(smartlist_t *responsible_dirs, + const char *id); +int hid_serv_acting_as_directory(void); +int hid_serv_responsible_for_desc_id(const char *id); + +#endif + diff --git a/src/or/routerparse.c b/src/or/routerparse.c index aa1aba423c..96749e5a74 100644 --- a/src/or/routerparse.c +++ b/src/or/routerparse.c @@ -10,7 +10,20 @@ **/ #include "or.h" +#include "config.h" +#include "dirserv.h" +#include "dirvote.h" +#include "policies.h" +#include "rendcommon.h" +#include "router.h" +#include "routerlist.h" #include "memarea.h" +#include "microdesc.h" +#include "networkstatus.h" +#include "rephist.h" +#include "routerparse.h" +#undef log +#include <math.h> /****************************************************************************/ @@ -55,6 +68,7 @@ typedef enum { K_S, K_V, K_W, + K_M, K_EVENTDNS, K_EXTRA_INFO, K_EXTRA_INFO_DIGEST, @@ -62,6 +76,31 @@ typedef enum { K_HIDDEN_SERVICE_DIR, K_ALLOW_SINGLE_HOP_EXITS, + K_DIRREQ_END, + K_DIRREQ_V2_IPS, + K_DIRREQ_V3_IPS, + K_DIRREQ_V2_REQS, + K_DIRREQ_V3_REQS, + K_DIRREQ_V2_SHARE, + K_DIRREQ_V3_SHARE, + K_DIRREQ_V2_RESP, + K_DIRREQ_V3_RESP, + K_DIRREQ_V2_DIR, + K_DIRREQ_V3_DIR, + K_DIRREQ_V2_TUN, + K_DIRREQ_V3_TUN, + K_ENTRY_END, + K_ENTRY_IPS, + K_CELL_END, + K_CELL_PROCESSED, + K_CELL_QUEUED, + K_CELL_TIME, + K_CELL_CIRCS, + K_EXIT_END, + K_EXIT_WRITTEN, + K_EXIT_READ, + K_EXIT_OPENED, + K_DIR_KEY_CERTIFICATE_VERSION, K_DIR_IDENTITY_KEY, K_DIR_KEY_PUBLISHED, @@ -78,13 +117,18 @@ typedef enum { K_KNOWN_FLAGS, K_PARAMS, + K_BW_WEIGHTS, K_VOTE_DIGEST, K_CONSENSUS_DIGEST, + K_ADDITIONAL_DIGEST, + K_ADDITIONAL_SIGNATURE, K_CONSENSUS_METHODS, K_CONSENSUS_METHOD, K_LEGACY_DIR_KEY, + K_DIRECTORY_FOOTER, A_PURPOSE, + A_LAST_LISTED, _A_UNKNOWN, R_RENDEZVOUS_SERVICE_DESCRIPTOR, @@ -122,7 +166,7 @@ typedef enum { * type. * * This structure is only allocated in memareas; do not allocate it on - * the heap, or token_free() won't work. + * the heap, or token_clear() won't work. */ typedef struct directory_token_t { directory_keyword tp; /**< Type of the token. */ @@ -258,6 +302,31 @@ static token_rule_t extrainfo_token_table[] = { T0N("opt", K_OPT, CONCAT_ARGS, OBJ_OK ), T01("read-history", K_READ_HISTORY, ARGS, NO_OBJ ), T01("write-history", K_WRITE_HISTORY, ARGS, NO_OBJ ), + T01("dirreq-stats-end", K_DIRREQ_END, ARGS, NO_OBJ ), + T01("dirreq-v2-ips", K_DIRREQ_V2_IPS, ARGS, NO_OBJ ), + T01("dirreq-v3-ips", K_DIRREQ_V3_IPS, ARGS, NO_OBJ ), + T01("dirreq-v2-reqs", K_DIRREQ_V2_REQS, ARGS, NO_OBJ ), + T01("dirreq-v3-reqs", K_DIRREQ_V3_REQS, ARGS, NO_OBJ ), + T01("dirreq-v2-share", K_DIRREQ_V2_SHARE, ARGS, NO_OBJ ), + T01("dirreq-v3-share", K_DIRREQ_V3_SHARE, ARGS, NO_OBJ ), + T01("dirreq-v2-resp", K_DIRREQ_V2_RESP, ARGS, NO_OBJ ), + T01("dirreq-v3-resp", K_DIRREQ_V3_RESP, ARGS, NO_OBJ ), + T01("dirreq-v2-direct-dl", K_DIRREQ_V2_DIR, ARGS, NO_OBJ ), + T01("dirreq-v3-direct-dl", K_DIRREQ_V3_DIR, ARGS, NO_OBJ ), + T01("dirreq-v2-tunneled-dl", K_DIRREQ_V2_TUN, ARGS, NO_OBJ ), + T01("dirreq-v3-tunneled-dl", K_DIRREQ_V3_TUN, ARGS, NO_OBJ ), + T01("entry-stats-end", K_ENTRY_END, ARGS, NO_OBJ ), + T01("entry-ips", K_ENTRY_IPS, ARGS, NO_OBJ ), + T01("cell-stats-end", K_CELL_END, ARGS, NO_OBJ ), + T01("cell-processed-cells", K_CELL_PROCESSED, ARGS, NO_OBJ ), + T01("cell-queued-cells", K_CELL_QUEUED, ARGS, NO_OBJ ), + T01("cell-time-in-queue", K_CELL_TIME, ARGS, NO_OBJ ), + T01("cell-circuits-per-decile", K_CELL_CIRCS, ARGS, NO_OBJ ), + T01("exit-stats-end", K_EXIT_END, ARGS, NO_OBJ ), + T01("exit-kibibytes-written", K_EXIT_WRITTEN, ARGS, NO_OBJ ), + T01("exit-kibibytes-read", K_EXIT_READ, ARGS, NO_OBJ ), + T01("exit-streams-opened", K_EXIT_OPENED, ARGS, NO_OBJ ), + T1_START( "extra-info", K_EXTRA_INFO, GE(2), NO_OBJ ), END_OF_TABLE @@ -267,10 +336,11 @@ static token_rule_t extrainfo_token_table[] = { * documents. */ static token_rule_t rtrstatus_token_table[] = { T01("p", K_P, CONCAT_ARGS, NO_OBJ ), - T1( "r", K_R, GE(8), NO_OBJ ), + T1( "r", K_R, GE(7), NO_OBJ ), T1( "s", K_S, ARGS, NO_OBJ ), T01("v", K_V, CONCAT_ARGS, NO_OBJ ), T01("w", K_W, ARGS, NO_OBJ ), + T0N("m", K_M, CONCAT_ARGS, NO_OBJ ), T0N("opt", K_OPT, CONCAT_ARGS, OBJ_OK ), END_OF_TABLE }; @@ -375,7 +445,7 @@ static token_rule_t client_keys_token_table[] = { /** List of tokens allowed in V3 networkstatus votes. */ static token_rule_t networkstatus_token_table[] = { - T1("network-status-version", K_NETWORK_STATUS_VERSION, + T1_START("network-status-version", K_NETWORK_STATUS_VERSION, GE(1), NO_OBJ ), T1("vote-status", K_VOTE_STATUS, GE(1), NO_OBJ ), T1("published", K_PUBLISHED, CONCAT_ARGS, NO_OBJ ), @@ -403,7 +473,7 @@ static token_rule_t networkstatus_token_table[] = { /** List of tokens allowed in V3 networkstatus consensuses. */ static token_rule_t networkstatus_consensus_token_table[] = { - T1("network-status-version", K_NETWORK_STATUS_VERSION, + T1_START("network-status-version", K_NETWORK_STATUS_VERSION, GE(1), NO_OBJ ), T1("vote-status", K_VOTE_STATUS, GE(1), NO_OBJ ), T1("valid-after", K_VALID_AFTER, CONCAT_ARGS, NO_OBJ ), @@ -430,17 +500,29 @@ static token_rule_t networkstatus_consensus_token_table[] = { /** List of tokens allowable in the footer of v1/v2 directory/networkstatus * footers. */ static token_rule_t networkstatus_vote_footer_token_table[] = { - T( "directory-signature", K_DIRECTORY_SIGNATURE, GE(2), NEED_OBJ ), + T01("directory-footer", K_DIRECTORY_FOOTER, NO_ARGS, NO_OBJ ), + T01("bandwidth-weights", K_BW_WEIGHTS, ARGS, NO_OBJ ), + T( "directory-signature", K_DIRECTORY_SIGNATURE, GE(2), NEED_OBJ ), END_OF_TABLE }; /** List of tokens allowable in detached networkstatus signature documents. */ static token_rule_t networkstatus_detached_signature_token_table[] = { T1_START("consensus-digest", K_CONSENSUS_DIGEST, GE(1), NO_OBJ ), + T("additional-digest", K_ADDITIONAL_DIGEST,GE(3), NO_OBJ ), T1("valid-after", K_VALID_AFTER, CONCAT_ARGS, NO_OBJ ), T1("fresh-until", K_FRESH_UNTIL, CONCAT_ARGS, NO_OBJ ), T1("valid-until", K_VALID_UNTIL, CONCAT_ARGS, NO_OBJ ), - T1N("directory-signature", K_DIRECTORY_SIGNATURE, GE(2), NEED_OBJ ), + T("additional-signature", K_ADDITIONAL_SIGNATURE, GE(4), NEED_OBJ ), + T1N("directory-signature", K_DIRECTORY_SIGNATURE, GE(2), NEED_OBJ ), + END_OF_TABLE +}; + +static token_rule_t microdesc_token_table[] = { + T1_START("onion-key", K_ONION_KEY, NO_ARGS, NEED_KEY_1024), + T01("family", K_FAMILY, ARGS, NO_OBJ ), + T01("p", K_P, CONCAT_ARGS, NO_OBJ ), + A01("@last-listed", A_LAST_LISTED, CONCAT_ARGS, NO_OBJ ), END_OF_TABLE }; @@ -453,9 +535,13 @@ static addr_policy_t *router_parse_addr_policy_private(directory_token_t *tok); static int router_get_hash_impl(const char *s, size_t s_len, char *digest, const char *start_str, const char *end_str, - char end_char); - -static void token_free(directory_token_t *tok); + char end_char, + digest_algorithm_t alg); +static int router_get_hashes_impl(const char *s, size_t s_len, + digests_t *digests, + const char *start_str, const char *end_str, + char end_char); +static void token_clear(directory_token_t *tok); static smartlist_t *find_all_exitpolicy(smartlist_t *s); static directory_token_t *_find_by_keyword(smartlist_t *s, directory_keyword keyword, @@ -479,6 +565,7 @@ static directory_token_t *get_next_token(memarea_t *area, #define CST_CHECK_AUTHORITY (1<<0) #define CST_NO_CHECK_OBJTYPE (1<<1) static int check_signature_token(const char *digest, + ssize_t digest_len, directory_token_t *tok, crypto_pk_env_t *pkey, int flags, @@ -499,6 +586,34 @@ static int tor_version_same_series(tor_version_t *a, tor_version_t *b); #define DUMP_AREA(a,name) STMT_NIL #endif +/** Last time we dumped a descriptor to disk. */ +static time_t last_desc_dumped = 0; + +/** For debugging purposes, dump unparseable descriptor *<b>desc</b> of + * type *<b>type</b> to file $DATADIR/unparseable-desc. Do not write more + * than one descriptor to disk per minute. If there is already such a + * file in the data directory, overwrite it. */ +static void +dump_desc(const char *desc, const char *type) +{ + time_t now = time(NULL); + tor_assert(desc); + tor_assert(type); + if (!last_desc_dumped || last_desc_dumped + 60 < now) { + char *debugfile = get_datadir_fname("unparseable-desc"); + size_t filelen = 50 + strlen(type) + strlen(desc); + char *content = tor_malloc_zero(filelen); + tor_snprintf(content, filelen, "Unable to parse descriptor of type " + "%s:\n%s", type, desc); + write_str_to_file(debugfile, content, 0); + log_info(LD_DIR, "Unable to parse descriptor of type %s. See file " + "unparseable-desc in data directory for details.", type); + tor_free(content); + tor_free(debugfile); + last_desc_dumped = now; + } +} + /** Set <b>digest</b> to the SHA-1 digest of the hash of the directory in * <b>s</b>. Return 0 on success, -1 on failure. */ @@ -506,7 +621,8 @@ int router_get_dir_hash(const char *s, char *digest) { return router_get_hash_impl(s, strlen(s), digest, - "signed-directory","\ndirectory-signature",'\n'); + "signed-directory","\ndirectory-signature",'\n', + DIGEST_SHA1); } /** Set <b>digest</b> to the SHA-1 digest of the hash of the first router in @@ -516,7 +632,8 @@ int router_get_router_hash(const char *s, size_t s_len, char *digest) { return router_get_hash_impl(s, s_len, digest, - "router ","\nrouter-signature", '\n'); + "router ","\nrouter-signature", '\n', + DIGEST_SHA1); } /** Set <b>digest</b> to the SHA-1 digest of the hash of the running-routers @@ -526,7 +643,8 @@ int router_get_runningrouters_hash(const char *s, char *digest) { return router_get_hash_impl(s, strlen(s), digest, - "network-status","\ndirectory-signature", '\n'); + "network-status","\ndirectory-signature", '\n', + DIGEST_SHA1); } /** Set <b>digest</b> to the SHA-1 digest of the hash of the network-status @@ -536,18 +654,31 @@ router_get_networkstatus_v2_hash(const char *s, char *digest) { return router_get_hash_impl(s, strlen(s), digest, "network-status-version","\ndirectory-signature", - '\n'); + '\n', + DIGEST_SHA1); +} + +/** Set <b>digests</b> to all the digests of the consensus document in + * <b>s</b> */ +int +router_get_networkstatus_v3_hashes(const char *s, digests_t *digests) +{ + return router_get_hashes_impl(s,strlen(s),digests, + "network-status-version", + "\ndirectory-signature", + ' '); } /** Set <b>digest</b> to the SHA-1 digest of the hash of the network-status * string in <b>s</b>. Return 0 on success, -1 on failure. */ int -router_get_networkstatus_v3_hash(const char *s, char *digest) +router_get_networkstatus_v3_hash(const char *s, char *digest, + digest_algorithm_t alg) { return router_get_hash_impl(s, strlen(s), digest, "network-status-version", "\ndirectory-signature", - ' '); + ' ', alg); } /** Set <b>digest</b> to the SHA-1 digest of the hash of the extrainfo @@ -556,7 +687,7 @@ int router_get_extrainfo_hash(const char *s, char *digest) { return router_get_hash_impl(s, strlen(s), digest, "extra-info", - "\nrouter-signature",'\n'); + "\nrouter-signature",'\n', DIGEST_SHA1); } /** Helper: used to generate signatures for routers, directories and @@ -568,14 +699,15 @@ router_get_extrainfo_hash(const char *s, char *digest) */ int router_append_dirobj_signature(char *buf, size_t buf_len, const char *digest, - crypto_pk_env_t *private_key) + size_t digest_len, crypto_pk_env_t *private_key) { char *signature; size_t i; + int siglen; signature = tor_malloc(crypto_pk_keysize(private_key)); - if (crypto_pk_private_sign(private_key, signature, digest, DIGEST_LEN) < 0) { - + siglen = crypto_pk_private_sign(private_key, signature, digest, digest_len); + if (siglen < 0) { log_warn(LD_BUG,"Couldn't sign digest."); goto err; } @@ -583,7 +715,7 @@ router_append_dirobj_signature(char *buf, size_t buf_len, const char *digest, goto truncated; i = strlen(buf); - if (base64_encode(buf+i, buf_len-i, signature, 128) < 0) { + if (base64_encode(buf+i, buf_len-i, signature, siglen) < 0) { log_warn(LD_BUG,"couldn't base64-encode signature"); goto err; } @@ -690,7 +822,7 @@ router_parse_directory(const char *str) char digest[DIGEST_LEN]; time_t published_on; int r; - const char *end, *cp; + const char *end, *cp, *str_dup = str; smartlist_t *tokens = NULL; crypto_pk_env_t *declared_key = NULL; memarea_t *area = memarea_new(); @@ -726,11 +858,11 @@ router_parse_directory(const char *str) } declared_key = find_dir_signing_key(str, str+strlen(str)); note_crypto_pk_op(VERIFY_DIR); - if (check_signature_token(digest, tok, declared_key, + if (check_signature_token(digest, DIGEST_LEN, tok, declared_key, CST_CHECK_AUTHORITY, "directory")<0) goto err; - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_clear(tokens); memarea_clear(area); @@ -763,11 +895,12 @@ router_parse_directory(const char *str) r = 0; goto done; err: + dump_desc(str_dup, "v1 directory"); r = -1; done: if (declared_key) crypto_free_pk_env(declared_key); if (tokens) { - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_free(tokens); } if (area) { @@ -789,7 +922,7 @@ router_parse_runningrouters(const char *str) int r = -1; crypto_pk_env_t *declared_key = NULL; smartlist_t *tokens = NULL; - const char *eos = str + strlen(str); + const char *eos = str + strlen(str), *str_dup = str; memarea_t *area = NULL; if (router_get_runningrouters_hash(str, digest)) { @@ -818,7 +951,7 @@ router_parse_runningrouters(const char *str) } declared_key = find_dir_signing_key(str, eos); note_crypto_pk_op(VERIFY_DIR); - if (check_signature_token(digest, tok, declared_key, + if (check_signature_token(digest, DIGEST_LEN, tok, declared_key, CST_CHECK_AUTHORITY, "running-routers") < 0) goto err; @@ -830,9 +963,10 @@ router_parse_runningrouters(const char *str) r = 0; err: + dump_desc(str_dup, "v1 running-routers"); if (declared_key) crypto_free_pk_env(declared_key); if (tokens) { - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_free(tokens); } if (area) { @@ -882,7 +1016,7 @@ find_dir_signing_key(const char *str, const char *eos) } done: - if (tok) token_free(tok); + if (tok) token_clear(tok); if (area) { DUMP_AREA(area, "dir-signing-key token"); memarea_drop_all(area); @@ -918,6 +1052,7 @@ dir_signing_key_is_trusted(crypto_pk_env_t *key) */ static int check_signature_token(const char *digest, + ssize_t digest_len, directory_token_t *tok, crypto_pk_env_t *pkey, int flags, @@ -948,14 +1083,14 @@ check_signature_token(const char *digest, signed_digest = tor_malloc(tok->object_size); if (crypto_pk_public_checksig(pkey, signed_digest, tok->object_body, tok->object_size) - != DIGEST_LEN) { + < digest_len) { log_warn(LD_DIR, "Error reading %s: invalid signature.", doctype); tor_free(signed_digest); return -1; } // log_debug(LD_DIR,"Signed %s hash starts %s", doctype, // hex_str(signed_digest,4)); - if (memcmp(digest, signed_digest, DIGEST_LEN)) { + if (memcmp(digest, signed_digest, digest_len)) { log_warn(LD_DIR, "Error reading %s: signature does not match.", doctype); tor_free(signed_digest); return -1; @@ -1141,7 +1276,7 @@ router_parse_entry_from_string(const char *s, const char *end, smartlist_t *tokens = NULL, *exit_policy_tokens = NULL; directory_token_t *tok; struct in_addr in; - const char *start_of_annotations, *cp; + const char *start_of_annotations, *cp, *s_dup = s; size_t prepend_len = prepend_annotations ? strlen(prepend_annotations) : 0; int ok = 1; memarea_t *area = NULL; @@ -1413,7 +1548,7 @@ router_parse_entry_from_string(const char *s, const char *end, verified_digests = digestmap_new(); digestmap_set(verified_digests, signed_digest, (void*)(uintptr_t)1); #endif - if (check_signature_token(digest, tok, router->identity_pkey, 0, + if (check_signature_token(digest, DIGEST_LEN, tok, router->identity_pkey, 0, "router descriptor") < 0) goto err; @@ -1431,16 +1566,15 @@ router_parse_entry_from_string(const char *s, const char *end, goto done; err: + dump_desc(s_dup, "router descriptor"); routerinfo_free(router); router = NULL; done: if (tokens) { - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_free(tokens); } - if (exit_policy_tokens) { - smartlist_free(exit_policy_tokens); - } + smartlist_free(exit_policy_tokens); if (area) { DUMP_AREA(area, "routerinfo"); memarea_drop_all(area); @@ -1465,6 +1599,7 @@ extrainfo_parse_entry_from_string(const char *s, const char *end, crypto_pk_env_t *key = NULL; routerinfo_t *router = NULL; memarea_t *area = NULL; + const char *s_dup = s; if (!end) { end = s + strlen(s); @@ -1539,7 +1674,8 @@ extrainfo_parse_entry_from_string(const char *s, const char *end, if (key) { note_crypto_pk_op(VERIFY_RTR); - if (check_signature_token(digest, tok, key, 0, "extra-info") < 0) + if (check_signature_token(digest, DIGEST_LEN, tok, key, 0, + "extra-info") < 0) goto err; if (router) @@ -1553,12 +1689,12 @@ extrainfo_parse_entry_from_string(const char *s, const char *end, goto done; err: - if (extrainfo) - extrainfo_free(extrainfo); + dump_desc(s_dup, "extra-info descriptor"); + extrainfo_free(extrainfo); extrainfo = NULL; done: if (tokens) { - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_free(tokens); } if (area) { @@ -1582,6 +1718,7 @@ authority_cert_parse_from_string(const char *s, const char **end_of_string) size_t len; int found; memarea_t *area = NULL; + const char *s_dup = s; s = eat_whitespace(s); eos = strstr(s, "\ndir-key-certification"); @@ -1606,7 +1743,7 @@ authority_cert_parse_from_string(const char *s, const char **end_of_string) goto err; } if (router_get_hash_impl(s, strlen(s), digest, "dir-key-certificate-version", - "\ndir-key-certification", '\n') < 0) + "\ndir-key-certification", '\n', DIGEST_SHA1) < 0) goto err; tok = smartlist_get(tokens, 0); if (tok->tp != K_DIR_KEY_CERTIFICATE_VERSION || strcmp(tok->args[0], "3")) { @@ -1699,7 +1836,7 @@ authority_cert_parse_from_string(const char *s, const char **end_of_string) } } if (!found) { - if (check_signature_token(digest, tok, cert->identity_key, 0, + if (check_signature_token(digest, DIGEST_LEN, tok, cert->identity_key, 0, "key certificate")) { goto err; } @@ -1708,6 +1845,7 @@ authority_cert_parse_from_string(const char *s, const char **end_of_string) /* XXXX Once all authorities generate cross-certified certificates, * make this field mandatory. */ if (check_signature_token(cert->cache_info.identity_digest, + DIGEST_LEN, tok, cert->signing_key, CST_NO_CHECK_OBJTYPE, @@ -1727,7 +1865,7 @@ authority_cert_parse_from_string(const char *s, const char **end_of_string) if (end_of_string) { *end_of_string = eat_whitespace(eos); } - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_free(tokens); if (area) { DUMP_AREA(area, "authority cert"); @@ -1735,8 +1873,9 @@ authority_cert_parse_from_string(const char *s, const char **end_of_string) } return cert; err: + dump_desc(s_dup, "authority cert"); authority_cert_free(cert); - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_free(tokens); if (area) { DUMP_AREA(area, "authority cert"); @@ -1747,23 +1886,28 @@ authority_cert_parse_from_string(const char *s, const char **end_of_string) /** Helper: given a string <b>s</b>, return the start of the next router-status * object (starting with "r " at the start of a line). If none is found, - * return the start of the next directory signature. If none is found, return - * the end of the string. */ + * return the start of the directory footer, or the next directory signature. + * If none is found, return the end of the string. */ static INLINE const char * find_start_of_next_routerstatus(const char *s) { - const char *eos = strstr(s, "\nr "); - if (eos) { - const char *eos2 = tor_memstr(s, eos-s, "\ndirectory-signature"); - if (eos2 && eos2 < eos) - return eos2; - else - return eos+1; - } else { - if ((eos = strstr(s, "\ndirectory-signature"))) - return eos+1; - return s + strlen(s); - } + const char *eos, *footer, *sig; + if ((eos = strstr(s, "\nr "))) + ++eos; + else + eos = s + strlen(s); + + footer = tor_memstr(s, eos-s, "\ndirectory-footer"); + sig = tor_memstr(s, eos-s, "\ndirectory-signature"); + + if (footer && sig) + return MIN(footer, sig) + 1; + else if (footer) + return footer+1; + else if (sig) + return sig+1; + else + return eos; } /** Given a string at *<b>s</b>, containing a routerstatus object, and an @@ -1777,22 +1921,29 @@ find_start_of_next_routerstatus(const char *s) * If <b>consensus_method</b> is nonzero, this routerstatus is part of a * consensus, and we should parse it according to the method used to * make that consensus. + * + * Parse according to the syntax used by the consensus flavor <b>flav</b>. **/ static routerstatus_t * routerstatus_parse_entry_from_string(memarea_t *area, const char **s, smartlist_t *tokens, networkstatus_t *vote, vote_routerstatus_t *vote_rs, - int consensus_method) + int consensus_method, + consensus_flavor_t flav) { - const char *eos; + const char *eos, *s_dup = *s; routerstatus_t *rs = NULL; directory_token_t *tok; char timebuf[ISO_TIME_LEN+1]; struct in_addr in; + int offset = 0; tor_assert(tokens); tor_assert(bool_eq(vote, vote_rs)); + if (!consensus_method) + flav = FLAV_NS; + eos = find_start_of_next_routerstatus(*s); if (tokenize_string(area,*s, eos, tokens, rtrstatus_token_table,0)) { @@ -1804,7 +1955,15 @@ routerstatus_parse_entry_from_string(memarea_t *area, goto err; } tok = find_by_keyword(tokens, K_R); - tor_assert(tok->n_args >= 8); + tor_assert(tok->n_args >= 7); + if (flav == FLAV_NS) { + if (tok->n_args < 8) { + log_warn(LD_DIR, "Too few arguments to r"); + goto err; + } + } else { + offset = -1; + } if (vote_rs) { rs = &vote_rs->status; } else { @@ -1825,29 +1984,34 @@ routerstatus_parse_entry_from_string(memarea_t *area, goto err; } - if (digest_from_base64(rs->descriptor_digest, tok->args[2])) { - log_warn(LD_DIR, "Error decoding descriptor digest %s", - escaped(tok->args[2])); - goto err; + if (flav == FLAV_NS) { + if (digest_from_base64(rs->descriptor_digest, tok->args[2])) { + log_warn(LD_DIR, "Error decoding descriptor digest %s", + escaped(tok->args[2])); + goto err; + } } if (tor_snprintf(timebuf, sizeof(timebuf), "%s %s", - tok->args[3], tok->args[4]) < 0 || + tok->args[3+offset], tok->args[4+offset]) < 0 || parse_iso_time(timebuf, &rs->published_on)<0) { - log_warn(LD_DIR, "Error parsing time '%s %s'", - tok->args[3], tok->args[4]); + log_warn(LD_DIR, "Error parsing time '%s %s' [%d %d]", + tok->args[3+offset], tok->args[4+offset], + offset, (int)flav); goto err; } - if (tor_inet_aton(tok->args[5], &in) == 0) { + if (tor_inet_aton(tok->args[5+offset], &in) == 0) { log_warn(LD_DIR, "Error parsing router address in network-status %s", - escaped(tok->args[5])); + escaped(tok->args[5+offset])); goto err; } rs->addr = ntohl(in.s_addr); - rs->or_port =(uint16_t) tor_parse_long(tok->args[6],10,0,65535,NULL,NULL); - rs->dir_port = (uint16_t) tor_parse_long(tok->args[7],10,0,65535,NULL,NULL); + rs->or_port = (uint16_t) tor_parse_long(tok->args[6+offset], + 10,0,65535,NULL,NULL); + rs->dir_port = (uint16_t) tor_parse_long(tok->args[7+offset], + 10,0,65535,NULL,NULL); tok = find_opt_by_keyword(tokens, K_S); if (tok && vote) { @@ -1933,6 +2097,17 @@ routerstatus_parse_entry_from_string(memarea_t *area, goto err; } rs->has_bandwidth = 1; + } else if (!strcmpstart(tok->args[i], "Measured=")) { + int ok; + rs->measured_bw = + (uint32_t)tor_parse_ulong(strchr(tok->args[i], '=')+1, + 10, 0, UINT32_MAX, &ok, NULL); + if (!ok) { + log_warn(LD_DIR, "Invalid Measured Bandwidth %s", + escaped(tok->args[i])); + goto err; + } + rs->has_measured_bw = 1; } } } @@ -1954,16 +2129,29 @@ routerstatus_parse_entry_from_string(memarea_t *area, rs->has_exitsummary = 1; } + if (vote_rs) { + SMARTLIST_FOREACH_BEGIN(tokens, directory_token_t *, t) { + if (t->tp == K_M && t->n_args) { + vote_microdesc_hash_t *line = + tor_malloc(sizeof(vote_microdesc_hash_t)); + line->next = vote_rs->microdesc; + line->microdesc_hash_line = tor_strdup(t->args[0]); + vote_rs->microdesc = line; + } + } SMARTLIST_FOREACH_END(t); + } + if (!strcasecmp(rs->nickname, UNNAMED_ROUTER_NICKNAME)) rs->is_named = 0; goto done; err: + dump_desc(s_dup, "routerstatus entry"); if (rs && !vote_rs) routerstatus_free(rs); rs = NULL; done: - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_clear(tokens); if (area) { DUMP_AREA(area, "routerstatus entry"); @@ -1975,8 +2163,8 @@ routerstatus_parse_entry_from_string(memarea_t *area, } /** Helper to sort a smartlist of pointers to routerstatus_t */ -static int -_compare_routerstatus_entries(const void **_a, const void **_b) +int +compare_routerstatus_entries(const void **_a, const void **_b) { const routerstatus_t *a = *_a, *b = *_b; return memcmp(a->identity_digest, b->identity_digest, DIGEST_LEN); @@ -2000,7 +2188,7 @@ _free_duplicate_routerstatus_entry(void *e) networkstatus_v2_t * networkstatus_v2_parse_from_string(const char *s) { - const char *eos; + const char *eos, *s_dup = s; smartlist_t *tokens = smartlist_create(); smartlist_t *footer_tokens = smartlist_create(); networkstatus_v2_t *ns = NULL; @@ -2114,17 +2302,17 @@ networkstatus_v2_parse_from_string(const char *s) ns->entries = smartlist_create(); s = eos; - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_clear(tokens); memarea_clear(area); while (!strcmpstart(s, "r ")) { routerstatus_t *rs; if ((rs = routerstatus_parse_entry_from_string(area, &s, tokens, - NULL, NULL, 0))) + NULL, NULL, 0, 0))) smartlist_add(ns->entries, rs); } - smartlist_sort(ns->entries, _compare_routerstatus_entries); - smartlist_uniq(ns->entries, _compare_routerstatus_entries, + smartlist_sort(ns->entries, compare_routerstatus_entries); + smartlist_uniq(ns->entries, compare_routerstatus_entries, _free_duplicate_routerstatus_entry); if (tokenize_string(area,s, NULL, footer_tokens, dir_footer_token_table,0)) { @@ -2143,19 +2331,19 @@ networkstatus_v2_parse_from_string(const char *s) } note_crypto_pk_op(VERIFY_DIR); - if (check_signature_token(ns_digest, tok, ns->signing_key, 0, + if (check_signature_token(ns_digest, DIGEST_LEN, tok, ns->signing_key, 0, "network-status") < 0) goto err; goto done; err: - if (ns) - networkstatus_v2_free(ns); + dump_desc(s_dup, "v2 networkstatus"); + networkstatus_v2_free(ns); ns = NULL; done: - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_free(tokens); - SMARTLIST_FOREACH(footer_tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(footer_tokens, directory_token_t *, t, token_clear(t)); smartlist_free(footer_tokens); if (area) { DUMP_AREA(area, "v2 networkstatus"); @@ -2164,6 +2352,394 @@ networkstatus_v2_parse_from_string(const char *s) return ns; } +/** Verify the bandwidth weights of a network status document */ +int +networkstatus_verify_bw_weights(networkstatus_t *ns) +{ + int64_t weight_scale; + int64_t G=0, M=0, E=0, D=0, T=0; + double Wgg, Wgm, Wgd, Wmg, Wmm, Wme, Wmd, Weg, Wem, Wee, Wed; + double Gtotal=0, Mtotal=0, Etotal=0; + const char *casename = NULL; + int valid = 1; + + weight_scale = networkstatus_get_param(ns, "bwweightscale", BW_WEIGHT_SCALE); + Wgg = networkstatus_get_bw_weight(ns, "Wgg", -1); + Wgm = networkstatus_get_bw_weight(ns, "Wgm", -1); + Wgd = networkstatus_get_bw_weight(ns, "Wgd", -1); + Wmg = networkstatus_get_bw_weight(ns, "Wmg", -1); + Wmm = networkstatus_get_bw_weight(ns, "Wmm", -1); + Wme = networkstatus_get_bw_weight(ns, "Wme", -1); + Wmd = networkstatus_get_bw_weight(ns, "Wmd", -1); + Weg = networkstatus_get_bw_weight(ns, "Weg", -1); + Wem = networkstatus_get_bw_weight(ns, "Wem", -1); + Wee = networkstatus_get_bw_weight(ns, "Wee", -1); + Wed = networkstatus_get_bw_weight(ns, "Wed", -1); + + if (Wgg<0 || Wgm<0 || Wgd<0 || Wmg<0 || Wmm<0 || Wme<0 || Wmd<0 || Weg<0 + || Wem<0 || Wee<0 || Wed<0) { + log_warn(LD_BUG, "No bandwidth weights produced in consensus!"); + return 0; + } + + // First, sanity check basic summing properties that hold for all cases + // We use > 1 as the check for these because they are computed as integers. + // Sometimes there are rounding errors. + if (fabs(Wmm - weight_scale) > 1) { + log_warn(LD_BUG, "Wmm=%lf != "I64_FORMAT, + Wmm, I64_PRINTF_ARG(weight_scale)); + valid = 0; + } + + if (fabs(Wem - Wee) > 1) { + log_warn(LD_BUG, "Wem=%lf != Wee=%lf", Wem, Wee); + valid = 0; + } + + if (fabs(Wgm - Wgg) > 1) { + log_warn(LD_BUG, "Wgm=%lf != Wgg=%lf", Wgm, Wgg); + valid = 0; + } + + if (fabs(Weg - Wed) > 1) { + log_warn(LD_BUG, "Wed=%lf != Weg=%lf", Wed, Weg); + valid = 0; + } + + if (fabs(Wgg + Wmg - weight_scale) > 0.001*weight_scale) { + log_warn(LD_BUG, "Wgg=%lf != "I64_FORMAT" - Wmg=%lf", Wgg, + I64_PRINTF_ARG(weight_scale), Wmg); + valid = 0; + } + + if (fabs(Wee + Wme - weight_scale) > 0.001*weight_scale) { + log_warn(LD_BUG, "Wee=%lf != "I64_FORMAT" - Wme=%lf", Wee, + I64_PRINTF_ARG(weight_scale), Wme); + valid = 0; + } + + if (fabs(Wgd + Wmd + Wed - weight_scale) > 0.001*weight_scale) { + log_warn(LD_BUG, "Wgd=%lf + Wmd=%lf + Wed=%lf != "I64_FORMAT, + Wgd, Wmd, Wed, I64_PRINTF_ARG(weight_scale)); + valid = 0; + } + + Wgg /= weight_scale; + Wgm /= weight_scale; + Wgd /= weight_scale; + + Wmg /= weight_scale; + Wmm /= weight_scale; + Wme /= weight_scale; + Wmd /= weight_scale; + + Weg /= weight_scale; + Wem /= weight_scale; + Wee /= weight_scale; + Wed /= weight_scale; + + // Then, gather G, M, E, D, T to determine case + SMARTLIST_FOREACH_BEGIN(ns->routerstatus_list, routerstatus_t *, rs) { + if (rs->has_bandwidth) { + T += rs->bandwidth; + if (rs->is_exit && rs->is_possible_guard) { + D += rs->bandwidth; + Gtotal += Wgd*rs->bandwidth; + Mtotal += Wmd*rs->bandwidth; + Etotal += Wed*rs->bandwidth; + } else if (rs->is_exit) { + E += rs->bandwidth; + Mtotal += Wme*rs->bandwidth; + Etotal += Wee*rs->bandwidth; + } else if (rs->is_possible_guard) { + G += rs->bandwidth; + Gtotal += Wgg*rs->bandwidth; + Mtotal += Wmg*rs->bandwidth; + } else { + M += rs->bandwidth; + Mtotal += Wmm*rs->bandwidth; + } + } else { + log_warn(LD_BUG, "Missing consensus bandwidth for router %s", + rs->nickname); + } + } SMARTLIST_FOREACH_END(rs); + + // Finally, check equality conditions depending upon case 1, 2 or 3 + // Full equality cases: 1, 3b + // Partial equality cases: 2b (E=G), 3a (M=E) + // Fully unknown: 2a + if (3*E >= T && 3*G >= T) { + // Case 1: Neither are scarce + casename = "Case 1"; + if (fabs(Etotal-Mtotal) > 0.01*MAX(Etotal,Mtotal)) { + log_warn(LD_DIR, + "Bw Weight Failure for %s: Etotal %lf != Mtotal %lf. " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT". " + "Wgg=%lf Wgd=%lf Wmg=%lf Wme=%lf Wmd=%lf Wee=%lf Wed=%lf", + casename, Etotal, Mtotal, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed); + valid = 0; + } + if (fabs(Etotal-Gtotal) > 0.01*MAX(Etotal,Gtotal)) { + log_warn(LD_DIR, + "Bw Weight Failure for %s: Etotal %lf != Gtotal %lf. " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT". " + "Wgg=%lf Wgd=%lf Wmg=%lf Wme=%lf Wmd=%lf Wee=%lf Wed=%lf", + casename, Etotal, Gtotal, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed); + valid = 0; + } + if (fabs(Gtotal-Mtotal) > 0.01*MAX(Gtotal,Mtotal)) { + log_warn(LD_DIR, + "Bw Weight Failure for %s: Mtotal %lf != Gtotal %lf. " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT". " + "Wgg=%lf Wgd=%lf Wmg=%lf Wme=%lf Wmd=%lf Wee=%lf Wed=%lf", + casename, Mtotal, Gtotal, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed); + valid = 0; + } + } else if (3*E < T && 3*G < T) { + int64_t R = MIN(E, G); + int64_t S = MAX(E, G); + /* + * Case 2: Both Guards and Exits are scarce + * Balance D between E and G, depending upon + * D capacity and scarcity. Devote no extra + * bandwidth to middle nodes. + */ + if (R+D < S) { // Subcase a + double Rtotal, Stotal; + if (E < G) { + Rtotal = Etotal; + Stotal = Gtotal; + } else { + Rtotal = Gtotal; + Stotal = Etotal; + } + casename = "Case 2a"; + // Rtotal < Stotal + if (Rtotal > Stotal) { + log_warn(LD_DIR, + "Bw Weight Failure for %s: Rtotal %lf > Stotal %lf. " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT". " + "Wgg=%lf Wgd=%lf Wmg=%lf Wme=%lf Wmd=%lf Wee=%lf Wed=%lf", + casename, Rtotal, Stotal, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed); + valid = 0; + } + // Rtotal < T/3 + if (3*Rtotal > T) { + log_warn(LD_DIR, + "Bw Weight Failure for %s: 3*Rtotal %lf > T " + I64_FORMAT". G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT + " D="I64_FORMAT" T="I64_FORMAT". " + "Wgg=%lf Wgd=%lf Wmg=%lf Wme=%lf Wmd=%lf Wee=%lf Wed=%lf", + casename, Rtotal*3, I64_PRINTF_ARG(T), + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed); + valid = 0; + } + // Stotal < T/3 + if (3*Stotal > T) { + log_warn(LD_DIR, + "Bw Weight Failure for %s: 3*Stotal %lf > T " + I64_FORMAT". G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT + " D="I64_FORMAT" T="I64_FORMAT". " + "Wgg=%lf Wgd=%lf Wmg=%lf Wme=%lf Wmd=%lf Wee=%lf Wed=%lf", + casename, Stotal*3, I64_PRINTF_ARG(T), + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed); + valid = 0; + } + // Mtotal > T/3 + if (3*Mtotal < T) { + log_warn(LD_DIR, + "Bw Weight Failure for %s: 3*Mtotal %lf < T " + I64_FORMAT". " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT". " + "Wgg=%lf Wgd=%lf Wmg=%lf Wme=%lf Wmd=%lf Wee=%lf Wed=%lf", + casename, Mtotal*3, I64_PRINTF_ARG(T), + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed); + valid = 0; + } + } else { // Subcase b: R+D > S + casename = "Case 2b"; + + /* Check the rare-M redirect case. */ + if (D != 0 && 3*M < T) { + casename = "Case 2b (balanced)"; + if (fabs(Etotal-Mtotal) > 0.01*MAX(Etotal,Mtotal)) { + log_warn(LD_DIR, + "Bw Weight Failure for %s: Etotal %lf != Mtotal %lf. " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT". " + "Wgg=%lf Wgd=%lf Wmg=%lf Wme=%lf Wmd=%lf Wee=%lf Wed=%lf", + casename, Etotal, Mtotal, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed); + valid = 0; + } + if (fabs(Etotal-Gtotal) > 0.01*MAX(Etotal,Gtotal)) { + log_warn(LD_DIR, + "Bw Weight Failure for %s: Etotal %lf != Gtotal %lf. " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT". " + "Wgg=%lf Wgd=%lf Wmg=%lf Wme=%lf Wmd=%lf Wee=%lf Wed=%lf", + casename, Etotal, Gtotal, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed); + valid = 0; + } + if (fabs(Gtotal-Mtotal) > 0.01*MAX(Gtotal,Mtotal)) { + log_warn(LD_DIR, + "Bw Weight Failure for %s: Mtotal %lf != Gtotal %lf. " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT". " + "Wgg=%lf Wgd=%lf Wmg=%lf Wme=%lf Wmd=%lf Wee=%lf Wed=%lf", + casename, Mtotal, Gtotal, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed); + valid = 0; + } + } else { + if (fabs(Etotal-Gtotal) > 0.01*MAX(Etotal,Gtotal)) { + log_warn(LD_DIR, + "Bw Weight Failure for %s: Etotal %lf != Gtotal %lf. " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT". " + "Wgg=%lf Wgd=%lf Wmg=%lf Wme=%lf Wmd=%lf Wee=%lf Wed=%lf", + casename, Etotal, Gtotal, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed); + valid = 0; + } + } + } + } else { // if (E < T/3 || G < T/3) { + int64_t S = MIN(E, G); + int64_t NS = MAX(E, G); + if (3*(S+D) < T) { // Subcase a: + double Stotal; + double NStotal; + if (G < E) { + casename = "Case 3a (G scarce)"; + Stotal = Gtotal; + NStotal = Etotal; + } else { // if (G >= E) { + casename = "Case 3a (E scarce)"; + NStotal = Gtotal; + Stotal = Etotal; + } + // Stotal < T/3 + if (3*Stotal > T) { + log_warn(LD_DIR, + "Bw Weight Failure for %s: 3*Stotal %lf > T " + I64_FORMAT". G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT + " D="I64_FORMAT" T="I64_FORMAT". " + "Wgg=%lf Wgd=%lf Wmg=%lf Wme=%lf Wmd=%lf Wee=%lf Wed=%lf", + casename, Stotal*3, I64_PRINTF_ARG(T), + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed); + valid = 0; + } + if (NS >= M) { + if (fabs(NStotal-Mtotal) > 0.01*MAX(NStotal,Mtotal)) { + log_warn(LD_DIR, + "Bw Weight Failure for %s: NStotal %lf != Mtotal %lf. " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT". " + "Wgg=%lf Wgd=%lf Wmg=%lf Wme=%lf Wmd=%lf Wee=%lf Wed=%lf", + casename, NStotal, Mtotal, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed); + valid = 0; + } + } else { + // if NS < M, NStotal > T/3 because only one of G or E is scarce + if (3*NStotal < T) { + log_warn(LD_DIR, + "Bw Weight Failure for %s: 3*NStotal %lf < T " + I64_FORMAT". G="I64_FORMAT" M="I64_FORMAT + " E="I64_FORMAT" D="I64_FORMAT" T="I64_FORMAT". " + "Wgg=%lf Wgd=%lf Wmg=%lf Wme=%lf Wmd=%lf Wee=%lf Wed=%lf", + casename, NStotal*3, I64_PRINTF_ARG(T), + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed); + valid = 0; + } + } + } else { // Subcase b: S+D >= T/3 + casename = "Case 3b"; + if (fabs(Etotal-Mtotal) > 0.01*MAX(Etotal,Mtotal)) { + log_warn(LD_DIR, + "Bw Weight Failure for %s: Etotal %lf != Mtotal %lf. " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT". " + "Wgg=%lf Wgd=%lf Wmg=%lf Wme=%lf Wmd=%lf Wee=%lf Wed=%lf", + casename, Etotal, Mtotal, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed); + valid = 0; + } + if (fabs(Etotal-Gtotal) > 0.01*MAX(Etotal,Gtotal)) { + log_warn(LD_DIR, + "Bw Weight Failure for %s: Etotal %lf != Gtotal %lf. " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT". " + "Wgg=%lf Wgd=%lf Wmg=%lf Wme=%lf Wmd=%lf Wee=%lf Wed=%lf", + casename, Etotal, Gtotal, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed); + valid = 0; + } + if (fabs(Gtotal-Mtotal) > 0.01*MAX(Gtotal,Mtotal)) { + log_warn(LD_DIR, + "Bw Weight Failure for %s: Mtotal %lf != Gtotal %lf. " + "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT + " T="I64_FORMAT". " + "Wgg=%lf Wgd=%lf Wmg=%lf Wme=%lf Wmd=%lf Wee=%lf Wed=%lf", + casename, Mtotal, Gtotal, + I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), + I64_PRINTF_ARG(D), I64_PRINTF_ARG(T), + Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed); + valid = 0; + } + } + } + + if (valid) + log_notice(LD_DIR, "Bandwidth-weight %s is verified and valid.", + casename); + + return valid; +} /** Parse a v3 networkstatus vote, opinion, or consensus (depending on * ns_type), from <b>s</b>, and return the result. Return NULL on failure. */ @@ -2175,19 +2751,21 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, smartlist_t *rs_tokens = NULL, *footer_tokens = NULL; networkstatus_voter_info_t *voter = NULL; networkstatus_t *ns = NULL; - char ns_digest[DIGEST_LEN]; - const char *cert, *end_of_header, *end_of_footer; + digests_t ns_digests; + const char *cert, *end_of_header, *end_of_footer, *s_dup = s; directory_token_t *tok; int ok; struct in_addr in; int i, inorder, n_signatures = 0; memarea_t *area = NULL, *rs_area = NULL; + consensus_flavor_t flav = FLAV_NS; + tor_assert(s); if (eos_out) *eos_out = NULL; - if (router_get_networkstatus_v3_hash(s, ns_digest)) { + if (router_get_networkstatus_v3_hashes(s, &ns_digests)) { log_warn(LD_DIR, "Unable to compute digest of network-status"); goto err; } @@ -2203,7 +2781,23 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, } ns = tor_malloc_zero(sizeof(networkstatus_t)); - memcpy(ns->networkstatus_digest, ns_digest, DIGEST_LEN); + memcpy(&ns->digests, &ns_digests, sizeof(ns_digests)); + + tok = find_by_keyword(tokens, K_NETWORK_STATUS_VERSION); + tor_assert(tok); + if (tok->n_args > 1) { + int flavor = networkstatus_parse_flavor_name(tok->args[1]); + if (flavor < 0) { + log_warn(LD_DIR, "Can't parse document with unknown flavor %s", + escaped(tok->args[2])); + goto err; + } + ns->flavor = flav = flavor; + } + if (flav != FLAV_NS && ns_type != NS_TYPE_CONSENSUS) { + log_warn(LD_DIR, "Flavor found on non-consenus networkstatus."); + goto err; + } if (ns_type != NS_TYPE_CONSENSUS) { const char *end_of_cert = NULL; @@ -2357,8 +2951,9 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, if (voter) smartlist_add(ns->voters, voter); voter = tor_malloc_zero(sizeof(networkstatus_voter_info_t)); + voter->sigs = smartlist_create(); if (ns->type != NS_TYPE_CONSENSUS) - memcpy(voter->vote_digest, ns_digest, DIGEST_LEN); + memcpy(voter->vote_digest, ns_digests.d[DIGEST_SHA1], DIGEST_LEN); voter->nickname = tor_strdup(tok->args[0]); if (strlen(tok->args[1]) != HEX_DIGEST_LEN || @@ -2450,7 +3045,7 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, if (ns->type != NS_TYPE_CONSENSUS) { vote_routerstatus_t *rs = tor_malloc_zero(sizeof(vote_routerstatus_t)); if (routerstatus_parse_entry_from_string(rs_area, &s, rs_tokens, ns, - rs, 0)) + rs, 0, 0)) smartlist_add(ns->routerstatus_list, rs); else { tor_free(rs->version); @@ -2460,7 +3055,8 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, routerstatus_t *rs; if ((rs = routerstatus_parse_entry_from_string(rs_area, &s, rs_tokens, NULL, NULL, - ns->consensus_method))) + ns->consensus_method, + flav))) smartlist_add(ns->routerstatus_list, rs); } } @@ -2493,14 +3089,73 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, goto err; } - SMARTLIST_FOREACH(footer_tokens, directory_token_t *, _tok, { + int found_sig = 0; + SMARTLIST_FOREACH_BEGIN(footer_tokens, directory_token_t *, _tok) { + tok = _tok; + if (tok->tp == K_DIRECTORY_SIGNATURE) + found_sig = 1; + else if (found_sig) { + log_warn(LD_DIR, "Extraneous token after first directory-signature"); + goto err; + } + } SMARTLIST_FOREACH_END(_tok); + } + + if ((tok = find_opt_by_keyword(footer_tokens, K_DIRECTORY_FOOTER))) { + if (tok != smartlist_get(footer_tokens, 0)) { + log_warn(LD_DIR, "Misplaced directory-footer token"); + goto err; + } + } + + tok = find_opt_by_keyword(footer_tokens, K_BW_WEIGHTS); + if (tok) { + ns->weight_params = smartlist_create(); + for (i = 0; i < tok->n_args; ++i) { + int ok=0; + char *eq = strchr(tok->args[i], '='); + if (!eq) { + log_warn(LD_DIR, "Bad element '%s' in weight params", + escaped(tok->args[i])); + goto err; + } + tor_parse_long(eq+1, 10, INT32_MIN, INT32_MAX, &ok, NULL); + if (!ok) { + log_warn(LD_DIR, "Bad element '%s' in params", escaped(tok->args[i])); + goto err; + } + smartlist_add(ns->weight_params, tor_strdup(tok->args[i])); + } + } + + SMARTLIST_FOREACH_BEGIN(footer_tokens, directory_token_t *, _tok) { char declared_identity[DIGEST_LEN]; networkstatus_voter_info_t *v; + document_signature_t *sig; + const char *id_hexdigest = NULL; + const char *sk_hexdigest = NULL; + digest_algorithm_t alg = DIGEST_SHA1; tok = _tok; if (tok->tp != K_DIRECTORY_SIGNATURE) continue; tor_assert(tok->n_args >= 2); + if (tok->n_args == 2) { + id_hexdigest = tok->args[0]; + sk_hexdigest = tok->args[1]; + } else { + const char *algname = tok->args[0]; + int a; + id_hexdigest = tok->args[1]; + sk_hexdigest = tok->args[2]; + a = crypto_digest_algorithm_parse_name(algname); + if (a<0) { + log_warn(LD_DIR, "Unknown digest algorithm %s; skipping", + escaped(algname)); + continue; + } + alg = a; + } if (!tok->object_type || strcmp(tok->object_type, "SIGNATURE") || @@ -2509,11 +3164,11 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, goto err; } - if (strlen(tok->args[0]) != HEX_DIGEST_LEN || + if (strlen(id_hexdigest) != HEX_DIGEST_LEN || base16_decode(declared_identity, sizeof(declared_identity), - tok->args[0], HEX_DIGEST_LEN) < 0) { + id_hexdigest, HEX_DIGEST_LEN) < 0) { log_warn(LD_DIR, "Error decoding declared identity %s in " - "network-status vote.", escaped(tok->args[0])); + "network-status vote.", escaped(id_hexdigest)); goto err; } if (!(v = networkstatus_get_voter_by_id(ns, declared_identity))) { @@ -2521,11 +3176,15 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, "any declared directory source."); goto err; } - if (strlen(tok->args[1]) != HEX_DIGEST_LEN || - base16_decode(v->signing_key_digest, sizeof(v->signing_key_digest), - tok->args[1], HEX_DIGEST_LEN) < 0) { - log_warn(LD_DIR, "Error decoding declared digest %s in " - "network-status vote.", escaped(tok->args[1])); + sig = tor_malloc_zero(sizeof(document_signature_t)); + memcpy(sig->identity_digest, v->identity_digest, DIGEST_LEN); + sig->alg = alg; + if (strlen(sk_hexdigest) != HEX_DIGEST_LEN || + base16_decode(sig->signing_key_digest, sizeof(sig->signing_key_digest), + sk_hexdigest, HEX_DIGEST_LEN) < 0) { + log_warn(LD_DIR, "Error decoding declared signing key digest %s in " + "network-status vote.", escaped(sk_hexdigest)); + tor_free(sig); goto err; } @@ -2534,35 +3193,49 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, DIGEST_LEN)) { log_warn(LD_DIR, "Digest mismatch between declared and actual on " "network-status vote."); + tor_free(sig); goto err; } } + if (voter_get_sig_by_algorithm(v, sig->alg)) { + /* We already parsed a vote with this algorithm from this voter. Use the + first one. */ + log_fn(LOG_PROTOCOL_WARN, LD_DIR, "We received a networkstatus " + "that contains two votes from the same voter with the same " + "algorithm. Ignoring the second vote."); + tor_free(sig); + continue; + } + if (ns->type != NS_TYPE_CONSENSUS) { - if (check_signature_token(ns_digest, tok, ns->cert->signing_key, 0, - "network-status vote")) + if (check_signature_token(ns_digests.d[DIGEST_SHA1], DIGEST_LEN, + tok, ns->cert->signing_key, 0, + "network-status vote")) { + tor_free(sig); goto err; - v->good_signature = 1; + } + sig->good_signature = 1; } else { - if (tok->object_size >= INT_MAX) + if (tok->object_size >= INT_MAX) { + tor_free(sig); goto err; - /* We already parsed a vote from this voter. Use the first one. */ - if (v->signature) { - log_fn(LOG_PROTOCOL_WARN, LD_DIR, "We received a networkstatus " - "that contains two votes from the same voter. Ignoring " - "the second vote."); - continue; } - - v->signature = tor_memdup(tok->object_body, tok->object_size); - v->signature_len = (int) tok->object_size; + sig->signature = tor_memdup(tok->object_body, tok->object_size); + sig->signature_len = (int) tok->object_size; } + smartlist_add(v->sigs, sig); + ++n_signatures; - }); + } SMARTLIST_FOREACH_END(_tok); if (! n_signatures) { log_warn(LD_DIR, "No signatures on networkstatus vote."); goto err; + } else if (ns->type == NS_TYPE_VOTE && n_signatures != 1) { + log_warn(LD_DIR, "Received more than one signature on a " + "network-status vote."); + goto err; } if (eos_out) @@ -2570,27 +3243,31 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, goto done; err: - if (ns) - networkstatus_vote_free(ns); + dump_desc(s_dup, "v3 networkstatus"); + networkstatus_vote_free(ns); ns = NULL; done: if (tokens) { - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_free(tokens); } if (voter) { + if (voter->sigs) { + SMARTLIST_FOREACH(voter->sigs, document_signature_t *, sig, + document_signature_free(sig)); + smartlist_free(voter->sigs); + } tor_free(voter->nickname); tor_free(voter->address); tor_free(voter->contact); - tor_free(voter->signature); tor_free(voter); } if (rs_tokens) { - SMARTLIST_FOREACH(rs_tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(rs_tokens, directory_token_t *, t, token_clear(t)); smartlist_free(rs_tokens); } if (footer_tokens) { - SMARTLIST_FOREACH(footer_tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(footer_tokens, directory_token_t *, t, token_clear(t)); smartlist_free(footer_tokens); } if (area) { @@ -2603,6 +3280,35 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, return ns; } +/** Return the digests_t that holds the digests of the + * <b>flavor_name</b>-flavored networkstatus according to the detached + * signatures document <b>sigs</b>, allocating a new digests_t as neeeded. */ +static digests_t * +detached_get_digests(ns_detached_signatures_t *sigs, const char *flavor_name) +{ + digests_t *d = strmap_get(sigs->digests, flavor_name); + if (!d) { + d = tor_malloc_zero(sizeof(digests_t)); + strmap_set(sigs->digests, flavor_name, d); + } + return d; +} + +/** Return the list of signatures of the <b>flavor_name</b>-flavored + * networkstatus according to the detached signatures document <b>sigs</b>, + * allocating a new digests_t as neeeded. */ +static smartlist_t * +detached_get_signatures(ns_detached_signatures_t *sigs, + const char *flavor_name) +{ + smartlist_t *sl = strmap_get(sigs->signatures, flavor_name); + if (!sl) { + sl = smartlist_create(); + strmap_set(sigs->signatures, flavor_name, sl); + } + return sl; +} + /** Parse a detached v3 networkstatus signature document between <b>s</b> and * <b>eos</b> and return the result. Return -1 on failure. */ ns_detached_signatures_t * @@ -2612,10 +3318,13 @@ networkstatus_parse_detached_signatures(const char *s, const char *eos) * networkstatus_parse_vote_from_string(). */ directory_token_t *tok; memarea_t *area = NULL; + digests_t *digests; smartlist_t *tokens = smartlist_create(); ns_detached_signatures_t *sigs = tor_malloc_zero(sizeof(ns_detached_signatures_t)); + sigs->digests = strmap_new(); + sigs->signatures = strmap_new(); if (!eos) eos = s + strlen(s); @@ -2627,18 +3336,57 @@ networkstatus_parse_detached_signatures(const char *s, const char *eos) goto err; } - tok = find_by_keyword(tokens, K_CONSENSUS_DIGEST); - if (strlen(tok->args[0]) != HEX_DIGEST_LEN) { - log_warn(LD_DIR, "Wrong length on consensus-digest in detached " - "networkstatus signatures"); - goto err; - } - if (base16_decode(sigs->networkstatus_digest, DIGEST_LEN, - tok->args[0], strlen(tok->args[0])) < 0) { - log_warn(LD_DIR, "Bad encoding on on consensus-digest in detached " - "networkstatus signatures"); - goto err; - } + /* Grab all the digest-like tokens. */ + SMARTLIST_FOREACH_BEGIN(tokens, directory_token_t *, _tok) { + const char *algname; + digest_algorithm_t alg; + const char *flavor; + const char *hexdigest; + size_t expected_length; + + tok = _tok; + + if (tok->tp == K_CONSENSUS_DIGEST) { + algname = "sha1"; + alg = DIGEST_SHA1; + flavor = "ns"; + hexdigest = tok->args[0]; + } else if (tok->tp == K_ADDITIONAL_DIGEST) { + int a = crypto_digest_algorithm_parse_name(tok->args[1]); + if (a<0) { + log_warn(LD_DIR, "Unrecognized algorithm name %s", tok->args[0]); + continue; + } + alg = (digest_algorithm_t) a; + flavor = tok->args[0]; + algname = tok->args[1]; + hexdigest = tok->args[2]; + } else { + continue; + } + + expected_length = + (alg == DIGEST_SHA1) ? HEX_DIGEST_LEN : HEX_DIGEST256_LEN; + + if (strlen(hexdigest) != expected_length) { + log_warn(LD_DIR, "Wrong length on consensus-digest in detached " + "networkstatus signatures"); + goto err; + } + digests = detached_get_digests(sigs, flavor); + tor_assert(digests); + if (!tor_mem_is_zero(digests->d[alg], DIGEST256_LEN)) { + log_warn(LD_DIR, "Multiple digests for %s with %s on detached " + "signatures document", flavor, algname); + continue; + } + if (base16_decode(digests->d[alg], DIGEST256_LEN, + hexdigest, strlen(hexdigest)) < 0) { + log_warn(LD_DIR, "Bad encoding on consensus-digest in detached " + "networkstatus signatures"); + goto err; + } + } SMARTLIST_FOREACH_END(_tok); tok = find_by_keyword(tokens, K_VALID_AFTER); if (parse_iso_time(tok->args[0], &sigs->valid_after)) { @@ -2658,57 +3406,102 @@ networkstatus_parse_detached_signatures(const char *s, const char *eos) goto err; } - sigs->signatures = smartlist_create(); - SMARTLIST_FOREACH(tokens, directory_token_t *, _tok, - { - char id_digest[DIGEST_LEN]; - char sk_digest[DIGEST_LEN]; - networkstatus_voter_info_t *voter; + SMARTLIST_FOREACH_BEGIN(tokens, directory_token_t *, _tok) { + const char *id_hexdigest; + const char *sk_hexdigest; + const char *algname; + const char *flavor; + digest_algorithm_t alg; + + char id_digest[DIGEST_LEN]; + char sk_digest[DIGEST_LEN]; + smartlist_t *siglist; + document_signature_t *sig; + int is_duplicate; - tok = _tok; - if (tok->tp != K_DIRECTORY_SIGNATURE) - continue; + tok = _tok; + if (tok->tp == K_DIRECTORY_SIGNATURE) { tor_assert(tok->n_args >= 2); + flavor = "ns"; + algname = "sha1"; + id_hexdigest = tok->args[0]; + sk_hexdigest = tok->args[1]; + } else if (tok->tp == K_ADDITIONAL_SIGNATURE) { + tor_assert(tok->n_args >= 4); + flavor = tok->args[0]; + algname = tok->args[1]; + id_hexdigest = tok->args[2]; + sk_hexdigest = tok->args[3]; + } else { + continue; + } - if (!tok->object_type || - strcmp(tok->object_type, "SIGNATURE") || - tok->object_size < 128 || tok->object_size > 512) { - log_warn(LD_DIR, "Bad object type or length on directory-signature"); - goto err; + { + int a = crypto_digest_algorithm_parse_name(algname); + if (a<0) { + log_warn(LD_DIR, "Unrecognized algorithm name %s", algname); + continue; } + alg = (digest_algorithm_t) a; + } - if (strlen(tok->args[0]) != HEX_DIGEST_LEN || - base16_decode(id_digest, sizeof(id_digest), - tok->args[0], HEX_DIGEST_LEN) < 0) { - log_warn(LD_DIR, "Error decoding declared identity %s in " - "network-status vote.", escaped(tok->args[0])); - goto err; - } - if (strlen(tok->args[1]) != HEX_DIGEST_LEN || - base16_decode(sk_digest, sizeof(sk_digest), - tok->args[1], HEX_DIGEST_LEN) < 0) { - log_warn(LD_DIR, "Error decoding declared digest %s in " - "network-status vote.", escaped(tok->args[1])); - goto err; - } + if (!tok->object_type || + strcmp(tok->object_type, "SIGNATURE") || + tok->object_size < 128 || tok->object_size > 512) { + log_warn(LD_DIR, "Bad object type or length on directory-signature"); + goto err; + } - voter = tor_malloc_zero(sizeof(networkstatus_voter_info_t)); - memcpy(voter->identity_digest, id_digest, DIGEST_LEN); - memcpy(voter->signing_key_digest, sk_digest, DIGEST_LEN); - if (tok->object_size >= INT_MAX) - goto err; - voter->signature = tor_memdup(tok->object_body, tok->object_size); - voter->signature_len = (int) tok->object_size; + if (strlen(id_hexdigest) != HEX_DIGEST_LEN || + base16_decode(id_digest, sizeof(id_digest), + id_hexdigest, HEX_DIGEST_LEN) < 0) { + log_warn(LD_DIR, "Error decoding declared identity %s in " + "network-status vote.", escaped(id_hexdigest)); + goto err; + } + if (strlen(sk_hexdigest) != HEX_DIGEST_LEN || + base16_decode(sk_digest, sizeof(sk_digest), + sk_hexdigest, HEX_DIGEST_LEN) < 0) { + log_warn(LD_DIR, "Error decoding declared signing key digest %s in " + "network-status vote.", escaped(sk_hexdigest)); + goto err; + } - smartlist_add(sigs->signatures, voter); + siglist = detached_get_signatures(sigs, flavor); + is_duplicate = 0; + SMARTLIST_FOREACH(siglist, document_signature_t *, s, { + if (s->alg == alg && + !memcmp(id_digest, s->identity_digest, DIGEST_LEN) && + !memcmp(sk_digest, s->signing_key_digest, DIGEST_LEN)) { + is_duplicate = 1; + } }); + if (is_duplicate) { + log_warn(LD_DIR, "Two signatures with identical keys and algorithm " + "found."); + continue; + } + + sig = tor_malloc_zero(sizeof(document_signature_t)); + sig->alg = alg; + memcpy(sig->identity_digest, id_digest, DIGEST_LEN); + memcpy(sig->signing_key_digest, sk_digest, DIGEST_LEN); + if (tok->object_size >= INT_MAX) { + tor_free(sig); + goto err; + } + sig->signature = tor_memdup(tok->object_body, tok->object_size); + sig->signature_len = (int) tok->object_size; + + smartlist_add(siglist, sig); + } SMARTLIST_FOREACH_END(_tok); goto done; err: ns_detached_signatures_free(sigs); sigs = NULL; done: - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_free(tokens); if (area) { DUMP_AREA(area, "detached signatures"); @@ -2764,7 +3557,7 @@ router_parse_addr_policy_item_from_string(const char *s, int assume_action) err: r = NULL; done: - token_free(tok); + token_clear(tok); if (area) { DUMP_AREA(area, "policy item"); memarea_drop_all(area); @@ -2887,9 +3680,8 @@ assert_addr_policy_ok(smartlist_t *lst) /** Free all resources allocated for <b>tok</b> */ static void -token_free(directory_token_t *tok) +token_clear(directory_token_t *tok) { - tor_assert(tok); if (tok->key) crypto_free_pk_env(tok->key); } @@ -2901,7 +3693,7 @@ token_free(directory_token_t *tok) #define RET_ERR(msg) \ STMT_BEGIN \ - if (tok) token_free(tok); \ + if (tok) token_clear(tok); \ tok = ALLOC_ZERO(sizeof(directory_token_t)); \ tok->tp = _ERR; \ tok->error = STRDUP(msg); \ @@ -3179,7 +3971,7 @@ tokenize_string(memarea_t *area, tok = get_next_token(area, s, end, table); if (tok->tp == _ERR) { log_warn(LD_DIR, "parse error: %s", tok->error); - token_free(tok); + token_clear(tok); return -1; } ++counts[tok->tp]; @@ -3293,17 +4085,11 @@ find_all_exitpolicy(smartlist_t *s) return out; } -/** Compute the SHA-1 digest of the substring of <b>s</b> taken from the first - * occurrence of <b>start_str</b> through the first instance of c after the - * first subsequent occurrence of <b>end_str</b>; store the 20-byte result in - * <b>digest</b>; return 0 on success. - * - * If no such substring exists, return -1. - */ static int -router_get_hash_impl(const char *s, size_t s_len, char *digest, +router_get_hash_impl_helper(const char *s, size_t s_len, const char *start_str, - const char *end_str, char end_c) + const char *end_str, char end_c, + const char **start_out, const char **end_out) { const char *start, *end; start = tor_memstr(s, s_len, start_str); @@ -3330,14 +4116,214 @@ router_get_hash_impl(const char *s, size_t s_len, char *digest, } ++end; - if (crypto_digest(digest, start, end-start)) { - log_warn(LD_BUG,"couldn't compute digest"); + *start_out = start; + *end_out = end; + return 0; +} + +/** Compute the digest of the substring of <b>s</b> taken from the first + * occurrence of <b>start_str</b> through the first instance of c after the + * first subsequent occurrence of <b>end_str</b>; store the 20-byte result in + * <b>digest</b>; return 0 on success. + * + * If no such substring exists, return -1. + */ +static int +router_get_hash_impl(const char *s, size_t s_len, char *digest, + const char *start_str, + const char *end_str, char end_c, + digest_algorithm_t alg) +{ + const char *start=NULL, *end=NULL; + if (router_get_hash_impl_helper(s,s_len,start_str,end_str,end_c, + &start,&end)<0) return -1; + + if (alg == DIGEST_SHA1) { + if (crypto_digest(digest, start, end-start)) { + log_warn(LD_BUG,"couldn't compute digest"); + return -1; + } + } else { + if (crypto_digest256(digest, start, end-start, alg)) { + log_warn(LD_BUG,"couldn't compute digest"); + return -1; + } } return 0; } +/** As router_get_hash_impl, but compute all hashes. */ +static int +router_get_hashes_impl(const char *s, size_t s_len, digests_t *digests, + const char *start_str, + const char *end_str, char end_c) +{ + const char *start=NULL, *end=NULL; + if (router_get_hash_impl_helper(s,s_len,start_str,end_str,end_c, + &start,&end)<0) + return -1; + + if (crypto_digest_all(digests, start, end-start)) { + log_warn(LD_BUG,"couldn't compute digests"); + return -1; + } + + return 0; +} + +/** Assuming that s starts with a microdesc, return the start of the + * *NEXT* one. Return NULL on "not found." */ +static const char * +find_start_of_next_microdesc(const char *s, const char *eos) +{ + int started_with_annotations; + s = eat_whitespace_eos(s, eos); + if (!s) + return NULL; + +#define CHECK_LENGTH() STMT_BEGIN \ + if (s+32 > eos) \ + return NULL; \ + STMT_END + +#define NEXT_LINE() STMT_BEGIN \ + s = memchr(s, '\n', eos-s); \ + if (!s || s+1 >= eos) \ + return NULL; \ + s++; \ + STMT_END + + CHECK_LENGTH(); + + started_with_annotations = (*s == '@'); + + if (started_with_annotations) { + /* Start by advancing to the first non-annotation line. */ + while (*s == '@') + NEXT_LINE(); + } + CHECK_LENGTH(); + + /* Now we should be pointed at an onion-key line. If we are, then skip + * it. */ + if (!strcmpstart(s, "onion-key")) + NEXT_LINE(); + + /* Okay, now we're pointed at the first line of the microdescriptor which is + not an annotation or onion-key. The next line that _is_ an annotation or + onion-key is the start of the next microdescriptor. */ + while (s+32 < eos) { + if (*s == '@' || !strcmpstart(s, "onion-key")) + return s; + NEXT_LINE(); + } + return NULL; + +#undef CHECK_LENGTH +#undef NEXT_LINE +} + +/** Parse as many microdescriptors as are found from the string starting at + * <b>s</b> and ending at <b>eos</b>. If allow_annotations is set, read any + * annotations we recognize and ignore ones we don't. If <b>copy_body</b> is + * true, then strdup the bodies of the microdescriptors. Return all newly + * parsed microdescriptors in a newly allocated smartlist_t. */ +smartlist_t * +microdescs_parse_from_string(const char *s, const char *eos, + int allow_annotations, int copy_body) +{ + smartlist_t *tokens; + smartlist_t *result; + microdesc_t *md = NULL; + memarea_t *area; + const char *start = s; + const char *start_of_next_microdesc; + int flags = allow_annotations ? TS_ANNOTATIONS_OK : 0; + + directory_token_t *tok; + + if (!eos) + eos = s + strlen(s); + + s = eat_whitespace_eos(s, eos); + area = memarea_new(); + result = smartlist_create(); + tokens = smartlist_create(); + + while (s < eos) { + start_of_next_microdesc = find_start_of_next_microdesc(s, eos); + if (!start_of_next_microdesc) + start_of_next_microdesc = eos; + + if (tokenize_string(area, s, start_of_next_microdesc, tokens, + microdesc_token_table, flags)) { + log_warn(LD_DIR, "Unparseable microdescriptor"); + goto next; + } + + md = tor_malloc_zero(sizeof(microdesc_t)); + { + const char *cp = tor_memstr(s, start_of_next_microdesc-s, + "onion-key"); + tor_assert(cp); + + md->bodylen = start_of_next_microdesc - cp; + if (copy_body) + md->body = tor_strndup(cp, md->bodylen); + else + md->body = (char*)cp; + md->off = cp - start; + } + + if ((tok = find_opt_by_keyword(tokens, A_LAST_LISTED))) { + if (parse_iso_time(tok->args[0], &md->last_listed)) { + log_warn(LD_DIR, "Bad last-listed time in microdescriptor"); + goto next; + } + } + + tok = find_by_keyword(tokens, K_ONION_KEY); + md->onion_pkey = tok->key; + tok->key = NULL; + + if ((tok = find_opt_by_keyword(tokens, K_FAMILY))) { + int i; + md->family = smartlist_create(); + for (i=0;i<tok->n_args;++i) { + if (!is_legal_nickname_or_hexdigest(tok->args[i])) { + log_warn(LD_DIR, "Illegal nickname %s in family line", + escaped(tok->args[i])); + goto next; + } + smartlist_add(md->family, tor_strdup(tok->args[i])); + } + } + + if ((tok = find_opt_by_keyword(tokens, K_P))) { + md->exitsummary = tor_strdup(tok->args[0]); + } + + crypto_digest256(md->digest, md->body, md->bodylen, DIGEST_SHA256); + + smartlist_add(result, md); + + md = NULL; + next: + microdesc_free(md); + + memarea_clear(area); + smartlist_clear(tokens); + s = start_of_next_microdesc; + } + + memarea_drop_all(area); + smartlist_free(tokens); + + return result; +} + /** Parse the Tor version of the platform string <b>platform</b>, * and compare it to the version in <b>cutoff</b>. Return 1 if * the router is at least as new as the cutoff, else return 0. @@ -3362,7 +4348,7 @@ tor_version_as_new_as(const char *platform, const char *cutoff) if (!*start) return 0; s = (char *)find_whitespace(start); /* also finds '\0', which is fine */ s2 = (char*)eat_whitespace(s); - if (!strcmpstart(s2, "(r")) + if (!strcmpstart(s2, "(r") || !strcmpstart(s2, "(git-")) s = (char*)find_whitespace(s2); if ((size_t)(s-start+1) >= sizeof(tmp)) /* too big, no */ @@ -3458,6 +4444,23 @@ tor_version_parse(const char *s, tor_version_t *out) if (!strcmpstart(cp, "(r")) { cp += 2; out->svn_revision = (int) strtol(cp,&eos,10); + } else if (!strcmpstart(cp, "(git-")) { + char *close_paren = strchr(cp, ')'); + int hexlen; + char digest[DIGEST_LEN]; + if (! close_paren) + return -1; + cp += 5; + if (close_paren-cp > HEX_DIGEST_LEN) + return -1; + hexlen = (int)(close_paren-cp); + memset(digest, 0, sizeof(digest)); + if ( hexlen == 0 || (hexlen % 2) == 1) + return -1; + if (base16_decode(digest, hexlen/2, cp, hexlen)) + return -1; + memcpy(out->git_tag, digest, hexlen/2); + out->git_tag_len = hexlen/2; } return 0; @@ -3483,8 +4486,14 @@ tor_version_compare(tor_version_t *a, tor_version_t *b) return i; else if ((i = strcmp(a->status_tag, b->status_tag))) return i; + else if ((i = a->svn_revision - b->svn_revision)) + return i; + else if ((i = a->git_tag_len - b->git_tag_len)) + return i; + else if (a->git_tag_len) + return memcmp(a->git_tag, b->git_tag, a->git_tag_len); else - return a->svn_revision - b->svn_revision; + return 0; } /** Return true iff versions <b>a</b> and <b>b</b> belong to the same series. @@ -3573,7 +4582,7 @@ rend_parse_v2_service_descriptor(rend_service_descriptor_t **parsed_out, /* Compute descriptor hash for later validation. */ if (router_get_hash_impl(desc, strlen(desc), desc_hash, "rendezvous-service-descriptor ", - "\nsignature", '\n') < 0) { + "\nsignature", '\n', DIGEST_SHA1) < 0) { log_warn(LD_REND, "Couldn't compute descriptor hash."); goto err; } @@ -3692,7 +4701,7 @@ rend_parse_v2_service_descriptor(rend_service_descriptor_t **parsed_out, /* Parse and verify signature. */ tok = find_by_keyword(tokens, R_SIGNATURE); note_crypto_pk_op(VERIFY_RTR); - if (check_signature_token(desc_hash, tok, result->pk, 0, + if (check_signature_token(desc_hash, DIGEST_LEN, tok, result->pk, 0, "v2 rendezvous service descriptor") < 0) goto err; /* Verify that descriptor ID belongs to public key and secret ID part. */ @@ -3706,12 +4715,11 @@ rend_parse_v2_service_descriptor(rend_service_descriptor_t **parsed_out, } goto done; err: - if (result) - rend_service_descriptor_free(result); + rend_service_descriptor_free(result); result = NULL; done: if (tokens) { - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_free(tokens); } if (area) @@ -3869,7 +4877,7 @@ rend_parse_introduction_points(rend_service_descriptor_t *parsed, eos = eos+1; tor_assert(eos <= intro_points_encoded+intro_points_encoded_size); /* Free tokens and clear token list. */ - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_clear(tokens); memarea_clear(area); /* Tokenize string. */ @@ -3942,7 +4950,7 @@ rend_parse_introduction_points(rend_service_descriptor_t *parsed, done: /* Free tokens and clear token list. */ - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_free(tokens); if (area) memarea_drop_all(area); @@ -3981,7 +4989,7 @@ rend_parse_client_keys(strmap_t *parsed_clients, const char *ckstr) else eos = eos + 1; /* Free tokens and clear token list. */ - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_clear(tokens); memarea_clear(area); /* Tokenize string. */ @@ -4053,7 +5061,7 @@ rend_parse_client_keys(strmap_t *parsed_clients, const char *ckstr) result = -1; done: /* Free tokens and clear token list. */ - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_free(t)); + SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_free(tokens); if (area) memarea_drop_all(area); diff --git a/src/or/routerparse.h b/src/or/routerparse.h new file mode 100644 index 0000000000..2313f68e02 --- /dev/null +++ b/src/or/routerparse.h @@ -0,0 +1,92 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2010, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file routerpase.h + * \brief Header file for routerparse.c. + **/ + +#ifndef _TOR_ROUTERPARSE_H +#define _TOR_ROUTERPARSE_H + +int router_get_router_hash(const char *s, size_t s_len, char *digest); +int router_get_dir_hash(const char *s, char *digest); +int router_get_runningrouters_hash(const char *s, char *digest); +int router_get_networkstatus_v2_hash(const char *s, char *digest); +int router_get_networkstatus_v3_hash(const char *s, char *digest, + digest_algorithm_t algorithm); +int router_get_networkstatus_v3_hashes(const char *s, digests_t *digests); +int router_get_extrainfo_hash(const char *s, char *digest); +int router_append_dirobj_signature(char *buf, size_t buf_len, + const char *digest, + size_t digest_len, + crypto_pk_env_t *private_key); +int router_parse_list_from_string(const char **s, const char *eos, + smartlist_t *dest, + saved_location_t saved_location, + int is_extrainfo, + int allow_annotations, + const char *prepend_annotations); +int router_parse_routerlist_from_directory(const char *s, + routerlist_t **dest, + crypto_pk_env_t *pkey, + int check_version, + int write_to_cache); +int router_parse_runningrouters(const char *str); +int router_parse_directory(const char *str); + +routerinfo_t *router_parse_entry_from_string(const char *s, const char *end, + int cache_copy, + int allow_annotations, + const char *prepend_annotations); +extrainfo_t *extrainfo_parse_entry_from_string(const char *s, const char *end, + int cache_copy, struct digest_ri_map_t *routermap); +addr_policy_t *router_parse_addr_policy_item_from_string(const char *s, + int assume_action); +version_status_t tor_version_is_obsolete(const char *myversion, + const char *versionlist); +int tor_version_parse(const char *s, tor_version_t *out); +int tor_version_as_new_as(const char *platform, const char *cutoff); +int tor_version_compare(tor_version_t *a, tor_version_t *b); +void sort_version_list(smartlist_t *lst, int remove_duplicates); +void assert_addr_policy_ok(smartlist_t *t); +void dump_distinct_digest_count(int severity); + +int compare_routerstatus_entries(const void **_a, const void **_b); +networkstatus_v2_t *networkstatus_v2_parse_from_string(const char *s); +int networkstatus_verify_bw_weights(networkstatus_t *ns); +networkstatus_t *networkstatus_parse_vote_from_string(const char *s, + const char **eos_out, + networkstatus_type_t ns_type); +ns_detached_signatures_t *networkstatus_parse_detached_signatures( + const char *s, const char *eos); + +smartlist_t *microdescs_parse_from_string(const char *s, const char *eos, + int allow_annotations, + int copy_body); + +authority_cert_t *authority_cert_parse_from_string(const char *s, + const char **end_of_string); +int rend_parse_v2_service_descriptor(rend_service_descriptor_t **parsed_out, + char *desc_id_out, + char **intro_points_encrypted_out, + size_t *intro_points_encrypted_size_out, + size_t *encoded_size_out, + const char **next_out, const char *desc); +int rend_decrypt_introduction_points(char **ipos_decrypted, + size_t *ipos_decrypted_size, + const char *descriptor_cookie, + const char *ipos_encrypted, + size_t ipos_encrypted_size); +int rend_parse_introduction_points(rend_service_descriptor_t *parsed, + const char *intro_points_encoded, + size_t intro_points_encoded_size); +int rend_parse_client_keys(strmap_t *parsed_clients, const char *str); + +void tor_gettimeofday_cache_clear(void); + +#endif + diff --git a/src/or/test.c b/src/or/test.c deleted file mode 100644 index 14ba953544..0000000000 --- a/src/or/test.c +++ /dev/null @@ -1,4911 +0,0 @@ -/* Copyright (c) 2001-2004, Roger Dingledine. - * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2010, The Tor Project, Inc. */ -/* See LICENSE for licensing information */ - -/* Ordinarily defined in tor_main.c; this bit is just here to provide one - * since we're not linking to tor_main.c */ -const char tor_svn_revision[] = ""; - -/** - * \file test.c - * \brief Unit tests for many pieces of the lower level Tor modules. - **/ - -#include "orconfig.h" - -#include <stdio.h> -#ifdef HAVE_FCNTL_H -#include <fcntl.h> -#endif - -#ifdef MS_WINDOWS -/* For mkdir() */ -#include <direct.h> -#else -#include <dirent.h> -#endif - -/* These macros pull in declarations for some functions and structures that - * are typically file-private. */ -#define BUFFERS_PRIVATE -#define CONFIG_PRIVATE -#define CONTROL_PRIVATE -#define CRYPTO_PRIVATE -#define DIRSERV_PRIVATE -#define DIRVOTE_PRIVATE -#define GEOIP_PRIVATE -#define MEMPOOL_PRIVATE -#define ROUTER_PRIVATE - -#include "or.h" -#include "test.h" -#include "torgzip.h" -#include "mempool.h" -#include "memarea.h" - -#ifdef USE_DMALLOC -#include <dmalloc.h> -#include <openssl/crypto.h> -#endif - -/** Set to true if any unit test has failed. Mostly, this is set by the macros - * in test.h */ -int have_failed = 0; - -/** Temporary directory (set up by setup_directory) under which we store all - * our files during testing. */ -static char temp_dir[256]; - -/** Select and create the temporary directory we'll use to run our unit tests. - * Store it in <b>temp_dir</b>. Exit immediately if we can't create it. - * idempotent. */ -static void -setup_directory(void) -{ - static int is_setup = 0; - int r; - if (is_setup) return; - -#ifdef MS_WINDOWS - // XXXX - tor_snprintf(temp_dir, sizeof(temp_dir), - "c:\\windows\\temp\\tor_test_%d", (int)getpid()); - r = mkdir(temp_dir); -#else - tor_snprintf(temp_dir, sizeof(temp_dir), "/tmp/tor_test_%d", (int) getpid()); - r = mkdir(temp_dir, 0700); -#endif - if (r) { - fprintf(stderr, "Can't create directory %s:", temp_dir); - perror(""); - exit(1); - } - is_setup = 1; -} - -/** Return a filename relative to our testing temporary directory */ -static const char * -get_fname(const char *name) -{ - static char buf[1024]; - setup_directory(); - tor_snprintf(buf,sizeof(buf),"%s/%s",temp_dir,name); - return buf; -} - -/** Remove all files stored under the temporary directory, and the directory - * itself. */ -static void -remove_directory(void) -{ - smartlist_t *elements = tor_listdir(temp_dir); - if (elements) { - SMARTLIST_FOREACH(elements, const char *, cp, - { - size_t len = strlen(cp)+strlen(temp_dir)+16; - char *tmp = tor_malloc(len); - tor_snprintf(tmp, len, "%s"PATH_SEPARATOR"%s", temp_dir, cp); - unlink(tmp); - tor_free(tmp); - }); - SMARTLIST_FOREACH(elements, char *, cp, tor_free(cp)); - smartlist_free(elements); - } - rmdir(temp_dir); -} - -/** Define this if unit tests spend too much time generating public keys*/ -#undef CACHE_GENERATED_KEYS - -static crypto_pk_env_t *pregen_keys[5] = {NULL, NULL, NULL, NULL, NULL}; -#define N_PREGEN_KEYS ((int)(sizeof(pregen_keys)/sizeof(pregen_keys[0]))) - -/** Generate and return a new keypair for use in unit tests. If we're using - * the key cache optimization, we might reuse keys: we only guarantee that - * keys made with distinct values for <b>idx</b> are different. The value of - * <b>idx</b> must be at least 0, and less than N_PREGEN_KEYS. */ -static crypto_pk_env_t * -pk_generate(int idx) -{ -#ifdef CACHE_GENERATED_KEYS - tor_assert(idx < N_PREGEN_KEYS); - if (! pregen_keys[idx]) { - pregen_keys[idx] = crypto_new_pk_env(); - tor_assert(!crypto_pk_generate_key(pregen_keys[idx])); - } - return crypto_pk_dup_key(pregen_keys[idx]); -#else - crypto_pk_env_t *result; - (void) idx; - result = crypto_new_pk_env(); - tor_assert(!crypto_pk_generate_key(result)); - return result; -#endif -} - -/** Free all storage used for the cached key optimization. */ -static void -free_pregenerated_keys(void) -{ - unsigned idx; - for (idx = 0; idx < N_PREGEN_KEYS; ++idx) { - if (pregen_keys[idx]) { - crypto_free_pk_env(pregen_keys[idx]); - pregen_keys[idx] = NULL; - } - } -} - -/** Run unit tests for buffers.c */ -static void -test_buffers(void) -{ - char str[256]; - char str2[256]; - - buf_t *buf = NULL, *buf2 = NULL; - const char *cp; - - int j; - size_t r; - - /**** - * buf_new - ****/ - if (!(buf = buf_new())) - test_fail(); - - //test_eq(buf_capacity(buf), 4096); - test_eq(buf_datalen(buf), 0); - - /**** - * General pointer frobbing - */ - for (j=0;j<256;++j) { - str[j] = (char)j; - } - write_to_buf(str, 256, buf); - write_to_buf(str, 256, buf); - test_eq(buf_datalen(buf), 512); - fetch_from_buf(str2, 200, buf); - test_memeq(str, str2, 200); - test_eq(buf_datalen(buf), 312); - memset(str2, 0, sizeof(str2)); - - fetch_from_buf(str2, 256, buf); - test_memeq(str+200, str2, 56); - test_memeq(str, str2+56, 200); - test_eq(buf_datalen(buf), 56); - memset(str2, 0, sizeof(str2)); - /* Okay, now we should be 512 bytes into the 4096-byte buffer. If we add - * another 3584 bytes, we hit the end. */ - for (j=0;j<15;++j) { - write_to_buf(str, 256, buf); - } - assert_buf_ok(buf); - test_eq(buf_datalen(buf), 3896); - fetch_from_buf(str2, 56, buf); - test_eq(buf_datalen(buf), 3840); - test_memeq(str+200, str2, 56); - for (j=0;j<15;++j) { - memset(str2, 0, sizeof(str2)); - fetch_from_buf(str2, 256, buf); - test_memeq(str, str2, 256); - } - test_eq(buf_datalen(buf), 0); - buf_free(buf); - buf = NULL; - - /* Okay, now make sure growing can work. */ - buf = buf_new_with_capacity(16); - //test_eq(buf_capacity(buf), 16); - write_to_buf(str+1, 255, buf); - //test_eq(buf_capacity(buf), 256); - fetch_from_buf(str2, 254, buf); - test_memeq(str+1, str2, 254); - //test_eq(buf_capacity(buf), 256); - assert_buf_ok(buf); - write_to_buf(str, 32, buf); - //test_eq(buf_capacity(buf), 256); - assert_buf_ok(buf); - write_to_buf(str, 256, buf); - assert_buf_ok(buf); - //test_eq(buf_capacity(buf), 512); - test_eq(buf_datalen(buf), 33+256); - fetch_from_buf(str2, 33, buf); - test_eq(*str2, str[255]); - - test_memeq(str2+1, str, 32); - //test_eq(buf_capacity(buf), 512); - test_eq(buf_datalen(buf), 256); - fetch_from_buf(str2, 256, buf); - test_memeq(str, str2, 256); - - /* now try shrinking: case 1. */ - buf_free(buf); - buf = buf_new_with_capacity(33668); - for (j=0;j<67;++j) { - write_to_buf(str,255, buf); - } - //test_eq(buf_capacity(buf), 33668); - test_eq(buf_datalen(buf), 17085); - for (j=0; j < 40; ++j) { - fetch_from_buf(str2, 255,buf); - test_memeq(str2, str, 255); - } - - /* now try shrinking: case 2. */ - buf_free(buf); - buf = buf_new_with_capacity(33668); - for (j=0;j<67;++j) { - write_to_buf(str,255, buf); - } - for (j=0; j < 20; ++j) { - fetch_from_buf(str2, 255,buf); - test_memeq(str2, str, 255); - } - for (j=0;j<80;++j) { - write_to_buf(str,255, buf); - } - //test_eq(buf_capacity(buf),33668); - for (j=0; j < 120; ++j) { - fetch_from_buf(str2, 255,buf); - test_memeq(str2, str, 255); - } - - /* Move from buf to buf. */ - buf_free(buf); - buf = buf_new_with_capacity(4096); - buf2 = buf_new_with_capacity(4096); - for (j=0;j<100;++j) - write_to_buf(str, 255, buf); - test_eq(buf_datalen(buf), 25500); - for (j=0;j<100;++j) { - r = 10; - move_buf_to_buf(buf2, buf, &r); - test_eq(r, 0); - } - test_eq(buf_datalen(buf), 24500); - test_eq(buf_datalen(buf2), 1000); - for (j=0;j<3;++j) { - fetch_from_buf(str2, 255, buf2); - test_memeq(str2, str, 255); - } - r = 8192; /*big move*/ - move_buf_to_buf(buf2, buf, &r); - test_eq(r, 0); - r = 30000; /* incomplete move */ - move_buf_to_buf(buf2, buf, &r); - test_eq(r, 13692); - for (j=0;j<97;++j) { - fetch_from_buf(str2, 255, buf2); - test_memeq(str2, str, 255); - } - buf_free(buf); - buf_free(buf2); - buf = buf2 = NULL; - - buf = buf_new_with_capacity(5); - cp = "Testing. This is a moderately long Testing string."; - for (j = 0; cp[j]; j++) - write_to_buf(cp+j, 1, buf); - test_eq(0, buf_find_string_offset(buf, "Testing", 7)); - test_eq(1, buf_find_string_offset(buf, "esting", 6)); - test_eq(1, buf_find_string_offset(buf, "est", 3)); - test_eq(39, buf_find_string_offset(buf, "ing str", 7)); - test_eq(35, buf_find_string_offset(buf, "Testing str", 11)); - test_eq(32, buf_find_string_offset(buf, "ng ", 3)); - test_eq(43, buf_find_string_offset(buf, "string.", 7)); - test_eq(-1, buf_find_string_offset(buf, "shrdlu", 6)); - test_eq(-1, buf_find_string_offset(buf, "Testing thing", 13)); - test_eq(-1, buf_find_string_offset(buf, "ngx", 3)); - buf_free(buf); - buf = NULL; - -#if 0 - { - int s; - int eof; - int i; - buf_t *buf2; - /**** - * read_to_buf - ****/ - s = open(get_fname("data"), O_WRONLY|O_CREAT|O_TRUNC, 0600); - write(s, str, 256); - close(s); - - s = open(get_fname("data"), O_RDONLY, 0); - eof = 0; - errno = 0; /* XXXX */ - i = read_to_buf(s, 10, buf, &eof); - printf("%s\n", strerror(errno)); - test_eq(i, 10); - test_eq(eof, 0); - //test_eq(buf_capacity(buf), 4096); - test_eq(buf_datalen(buf), 10); - - test_memeq(str, (char*)_buf_peek_raw_buffer(buf), 10); - - /* Test reading 0 bytes. */ - i = read_to_buf(s, 0, buf, &eof); - //test_eq(buf_capacity(buf), 512*1024); - test_eq(buf_datalen(buf), 10); - test_eq(eof, 0); - test_eq(i, 0); - - /* Now test when buffer is filled exactly. */ - buf2 = buf_new_with_capacity(6); - i = read_to_buf(s, 6, buf2, &eof); - //test_eq(buf_capacity(buf2), 6); - test_eq(buf_datalen(buf2), 6); - test_eq(eof, 0); - test_eq(i, 6); - test_memeq(str+10, (char*)_buf_peek_raw_buffer(buf2), 6); - buf_free(buf2); - buf2 = NULL; - - /* Now test when buffer is filled with more data to read. */ - buf2 = buf_new_with_capacity(32); - i = read_to_buf(s, 128, buf2, &eof); - //test_eq(buf_capacity(buf2), 128); - test_eq(buf_datalen(buf2), 32); - test_eq(eof, 0); - test_eq(i, 32); - buf_free(buf2); - buf2 = NULL; - - /* Now read to eof. */ - test_assert(buf_capacity(buf) > 256); - i = read_to_buf(s, 1024, buf, &eof); - test_eq(i, (256-32-10-6)); - test_eq(buf_capacity(buf), MAX_BUF_SIZE); - test_eq(buf_datalen(buf), 256-6-32); - test_memeq(str, (char*)_buf_peek_raw_buffer(buf), 10); /* XXX Check rest. */ - test_eq(eof, 0); - - i = read_to_buf(s, 1024, buf, &eof); - test_eq(i, 0); - test_eq(buf_capacity(buf), MAX_BUF_SIZE); - test_eq(buf_datalen(buf), 256-6-32); - test_eq(eof, 1); - } -#endif - - done: - if (buf) - buf_free(buf); - if (buf2) - buf_free(buf2); -} - -/** Run unit tests for Diffie-Hellman functionality. */ -static void -test_crypto_dh(void) -{ - crypto_dh_env_t *dh1 = crypto_dh_new(); - crypto_dh_env_t *dh2 = crypto_dh_new(); - char p1[DH_BYTES]; - char p2[DH_BYTES]; - char s1[DH_BYTES]; - char s2[DH_BYTES]; - ssize_t s1len, s2len; - - test_eq(crypto_dh_get_bytes(dh1), DH_BYTES); - test_eq(crypto_dh_get_bytes(dh2), DH_BYTES); - - memset(p1, 0, DH_BYTES); - memset(p2, 0, DH_BYTES); - test_memeq(p1, p2, DH_BYTES); - test_assert(! crypto_dh_get_public(dh1, p1, DH_BYTES)); - test_memneq(p1, p2, DH_BYTES); - test_assert(! crypto_dh_get_public(dh2, p2, DH_BYTES)); - test_memneq(p1, p2, DH_BYTES); - - memset(s1, 0, DH_BYTES); - memset(s2, 0xFF, DH_BYTES); - s1len = crypto_dh_compute_secret(dh1, p2, DH_BYTES, s1, 50); - s2len = crypto_dh_compute_secret(dh2, p1, DH_BYTES, s2, 50); - test_assert(s1len > 0); - test_eq(s1len, s2len); - test_memeq(s1, s2, s1len); - - { - /* XXXX Now fabricate some bad values and make sure they get caught, - * Check 0, 1, N-1, >= N, etc. - */ - } - - done: - crypto_dh_free(dh1); - crypto_dh_free(dh2); -} - -/** Run unit tests for our random number generation function and its wrappers. - */ -static void -test_crypto_rng(void) -{ - int i, j, allok; - char data1[100], data2[100]; - - /* Try out RNG. */ - test_assert(! crypto_seed_rng(0)); - crypto_rand(data1, 100); - crypto_rand(data2, 100); - test_memneq(data1,data2,100); - allok = 1; - for (i = 0; i < 100; ++i) { - uint64_t big; - char *host; - j = crypto_rand_int(100); - if (i < 0 || i >= 100) - allok = 0; - big = crypto_rand_uint64(U64_LITERAL(1)<<40); - if (big >= (U64_LITERAL(1)<<40)) - allok = 0; - big = crypto_rand_uint64(U64_LITERAL(5)); - if (big >= 5) - allok = 0; - host = crypto_random_hostname(3,8,"www.",".onion"); - if (strcmpstart(host,"www.") || - strcmpend(host,".onion") || - strlen(host) < 13 || - strlen(host) > 18) - allok = 0; - tor_free(host); - } - test_assert(allok); - done: - ; -} - -/** Run unit tests for our AES functionality */ -static void -test_crypto_aes(void) -{ - char *data1 = NULL, *data2 = NULL, *data3 = NULL; - crypto_cipher_env_t *env1 = NULL, *env2 = NULL; - int i, j; - - data1 = tor_malloc(1024); - data2 = tor_malloc(1024); - data3 = tor_malloc(1024); - - /* Now, test encryption and decryption with stream cipher. */ - data1[0]='\0'; - for (i = 1023; i>0; i -= 35) - strncat(data1, "Now is the time for all good onions", i); - - memset(data2, 0, 1024); - memset(data3, 0, 1024); - env1 = crypto_new_cipher_env(); - test_neq(env1, 0); - env2 = crypto_new_cipher_env(); - test_neq(env2, 0); - j = crypto_cipher_generate_key(env1); - crypto_cipher_set_key(env2, crypto_cipher_get_key(env1)); - crypto_cipher_encrypt_init_cipher(env1); - crypto_cipher_decrypt_init_cipher(env2); - - /* Try encrypting 512 chars. */ - crypto_cipher_encrypt(env1, data2, data1, 512); - crypto_cipher_decrypt(env2, data3, data2, 512); - test_memeq(data1, data3, 512); - test_memneq(data1, data2, 512); - - /* Now encrypt 1 at a time, and get 1 at a time. */ - for (j = 512; j < 560; ++j) { - crypto_cipher_encrypt(env1, data2+j, data1+j, 1); - } - for (j = 512; j < 560; ++j) { - crypto_cipher_decrypt(env2, data3+j, data2+j, 1); - } - test_memeq(data1, data3, 560); - /* Now encrypt 3 at a time, and get 5 at a time. */ - for (j = 560; j < 1024-5; j += 3) { - crypto_cipher_encrypt(env1, data2+j, data1+j, 3); - } - for (j = 560; j < 1024-5; j += 5) { - crypto_cipher_decrypt(env2, data3+j, data2+j, 5); - } - test_memeq(data1, data3, 1024-5); - /* Now make sure that when we encrypt with different chunk sizes, we get - the same results. */ - crypto_free_cipher_env(env2); - env2 = NULL; - - memset(data3, 0, 1024); - env2 = crypto_new_cipher_env(); - test_neq(env2, 0); - crypto_cipher_set_key(env2, crypto_cipher_get_key(env1)); - crypto_cipher_encrypt_init_cipher(env2); - for (j = 0; j < 1024-16; j += 17) { - crypto_cipher_encrypt(env2, data3+j, data1+j, 17); - } - for (j= 0; j < 1024-16; ++j) { - if (data2[j] != data3[j]) { - printf("%d: %d\t%d\n", j, (int) data2[j], (int) data3[j]); - } - } - test_memeq(data2, data3, 1024-16); - crypto_free_cipher_env(env1); - env1 = NULL; - crypto_free_cipher_env(env2); - env2 = NULL; - - /* NIST test vector for aes. */ - env1 = crypto_new_cipher_env(); /* IV starts at 0 */ - crypto_cipher_set_key(env1, "\x80\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00"); - crypto_cipher_encrypt_init_cipher(env1); - crypto_cipher_encrypt(env1, data1, - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00", 16); - test_memeq_hex(data1, "0EDD33D3C621E546455BD8BA1418BEC8"); - - /* Now test rollover. All these values are originally from a python - * script. */ - crypto_cipher_set_iv(env1, "\x00\x00\x00\x00\x00\x00\x00\x00" - "\xff\xff\xff\xff\xff\xff\xff\xff"); - memset(data2, 0, 1024); - crypto_cipher_encrypt(env1, data1, data2, 32); - test_memeq_hex(data1, "335fe6da56f843199066c14a00a40231" - "cdd0b917dbc7186908a6bfb5ffd574d3"); - - crypto_cipher_set_iv(env1, "\x00\x00\x00\x00\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff"); - memset(data2, 0, 1024); - crypto_cipher_encrypt(env1, data1, data2, 32); - test_memeq_hex(data1, "e627c6423fa2d77832a02b2794094b73" - "3e63c721df790d2c6469cc1953a3ffac"); - - crypto_cipher_set_iv(env1, "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff"); - memset(data2, 0, 1024); - crypto_cipher_encrypt(env1, data1, data2, 32); - test_memeq_hex(data1, "2aed2bff0de54f9328efd070bf48f70a" - "0EDD33D3C621E546455BD8BA1418BEC8"); - - /* Now check rollover on inplace cipher. */ - crypto_cipher_set_iv(env1, "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff"); - crypto_cipher_crypt_inplace(env1, data2, 64); - test_memeq_hex(data2, "2aed2bff0de54f9328efd070bf48f70a" - "0EDD33D3C621E546455BD8BA1418BEC8" - "93e2c5243d6839eac58503919192f7ae" - "1908e67cafa08d508816659c2e693191"); - crypto_cipher_set_iv(env1, "\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff"); - crypto_cipher_crypt_inplace(env1, data2, 64); - test_assert(tor_mem_is_zero(data2, 64)); - - done: - if (env1) - crypto_free_cipher_env(env1); - if (env2) - crypto_free_cipher_env(env2); - tor_free(data1); - tor_free(data2); - tor_free(data3); -} - -/** Run unit tests for our SHA-1 functionality */ -static void -test_crypto_sha(void) -{ - crypto_digest_env_t *d1 = NULL, *d2 = NULL; - int i; - char key[80]; - char digest[20]; - char data[50]; - char d_out1[DIGEST_LEN], d_out2[DIGEST_LEN]; - - /* Test SHA-1 with a test vector from the specification. */ - i = crypto_digest(data, "abc", 3); - test_memeq_hex(data, "A9993E364706816ABA3E25717850C26C9CD0D89D"); - - /* Test HMAC-SHA-1 with test cases from RFC2202. */ - - /* Case 1. */ - memset(key, 0x0b, 20); - crypto_hmac_sha1(digest, key, 20, "Hi There", 8); - test_streq(hex_str(digest, 20), - "B617318655057264E28BC0B6FB378C8EF146BE00"); - /* Case 2. */ - crypto_hmac_sha1(digest, "Jefe", 4, "what do ya want for nothing?", 28); - test_streq(hex_str(digest, 20), - "EFFCDF6AE5EB2FA2D27416D5F184DF9C259A7C79"); - - /* Case 4. */ - base16_decode(key, 25, - "0102030405060708090a0b0c0d0e0f10111213141516171819", 50); - memset(data, 0xcd, 50); - crypto_hmac_sha1(digest, key, 25, data, 50); - test_streq(hex_str(digest, 20), - "4C9007F4026250C6BC8414F9BF50C86C2D7235DA"); - - /* Case . */ - memset(key, 0xaa, 80); - crypto_hmac_sha1(digest, key, 80, - "Test Using Larger Than Block-Size Key - Hash Key First", - 54); - test_streq(hex_str(digest, 20), - "AA4AE5E15272D00E95705637CE8A3B55ED402112"); - - /* Incremental digest code. */ - d1 = crypto_new_digest_env(); - test_assert(d1); - crypto_digest_add_bytes(d1, "abcdef", 6); - d2 = crypto_digest_dup(d1); - test_assert(d2); - crypto_digest_add_bytes(d2, "ghijkl", 6); - crypto_digest_get_digest(d2, d_out1, sizeof(d_out1)); - crypto_digest(d_out2, "abcdefghijkl", 12); - test_memeq(d_out1, d_out2, DIGEST_LEN); - crypto_digest_assign(d2, d1); - crypto_digest_add_bytes(d2, "mno", 3); - crypto_digest_get_digest(d2, d_out1, sizeof(d_out1)); - crypto_digest(d_out2, "abcdefmno", 9); - test_memeq(d_out1, d_out2, DIGEST_LEN); - crypto_digest_get_digest(d1, d_out1, sizeof(d_out1)); - crypto_digest(d_out2, "abcdef", 6); - test_memeq(d_out1, d_out2, DIGEST_LEN); - - done: - if (d1) - crypto_free_digest_env(d1); - if (d2) - crypto_free_digest_env(d2); -} - -/** Run unit tests for our public key crypto functions */ -static void -test_crypto_pk(void) -{ - crypto_pk_env_t *pk1 = NULL, *pk2 = NULL; - char *encoded = NULL; - char data1[1024], data2[1024], data3[1024]; - size_t size; - int i, j, p, len; - - /* Public-key ciphers */ - pk1 = pk_generate(0); - pk2 = crypto_new_pk_env(); - test_assert(pk1 && pk2); - test_assert(! crypto_pk_write_public_key_to_string(pk1, &encoded, &size)); - test_assert(! crypto_pk_read_public_key_from_string(pk2, encoded, size)); - test_eq(0, crypto_pk_cmp_keys(pk1, pk2)); - - test_eq(128, crypto_pk_keysize(pk1)); - test_eq(128, crypto_pk_keysize(pk2)); - - test_eq(128, crypto_pk_public_encrypt(pk2, data1, "Hello whirled.", 15, - PK_PKCS1_OAEP_PADDING)); - test_eq(128, crypto_pk_public_encrypt(pk1, data2, "Hello whirled.", 15, - PK_PKCS1_OAEP_PADDING)); - /* oaep padding should make encryption not match */ - test_memneq(data1, data2, 128); - test_eq(15, crypto_pk_private_decrypt(pk1, data3, data1, 128, - PK_PKCS1_OAEP_PADDING,1)); - test_streq(data3, "Hello whirled."); - memset(data3, 0, 1024); - test_eq(15, crypto_pk_private_decrypt(pk1, data3, data2, 128, - PK_PKCS1_OAEP_PADDING,1)); - test_streq(data3, "Hello whirled."); - /* Can't decrypt with public key. */ - test_eq(-1, crypto_pk_private_decrypt(pk2, data3, data2, 128, - PK_PKCS1_OAEP_PADDING,1)); - /* Try again with bad padding */ - memcpy(data2+1, "XYZZY", 5); /* This has fails ~ once-in-2^40 */ - test_eq(-1, crypto_pk_private_decrypt(pk1, data3, data2, 128, - PK_PKCS1_OAEP_PADDING,1)); - - /* File operations: save and load private key */ - test_assert(! crypto_pk_write_private_key_to_filename(pk1, - get_fname("pkey1"))); - /* failing case for read: can't read. */ - test_assert(crypto_pk_read_private_key_from_filename(pk2, - get_fname("xyzzy")) < 0); - write_str_to_file(get_fname("xyzzy"), "foobar", 6); - /* Failing case for read: no key. */ - test_assert(crypto_pk_read_private_key_from_filename(pk2, - get_fname("xyzzy")) < 0); - test_assert(! crypto_pk_read_private_key_from_filename(pk2, - get_fname("pkey1"))); - test_eq(15, crypto_pk_private_decrypt(pk2, data3, data1, 128, - PK_PKCS1_OAEP_PADDING,1)); - - /* Now try signing. */ - strlcpy(data1, "Ossifrage", 1024); - test_eq(128, crypto_pk_private_sign(pk1, data2, data1, 10)); - test_eq(10, crypto_pk_public_checksig(pk1, data3, data2, 128)); - test_streq(data3, "Ossifrage"); - /* Try signing digests. */ - test_eq(128, crypto_pk_private_sign_digest(pk1, data2, data1, 10)); - test_eq(20, crypto_pk_public_checksig(pk1, data3, data2, 128)); - test_eq(0, crypto_pk_public_checksig_digest(pk1, data1, 10, data2, 128)); - test_eq(-1, crypto_pk_public_checksig_digest(pk1, data1, 11, data2, 128)); - /*XXXX test failed signing*/ - - /* Try encoding */ - crypto_free_pk_env(pk2); - pk2 = NULL; - i = crypto_pk_asn1_encode(pk1, data1, 1024); - test_assert(i>0); - pk2 = crypto_pk_asn1_decode(data1, i); - test_assert(crypto_pk_cmp_keys(pk1,pk2) == 0); - - /* Try with hybrid encryption wrappers. */ - crypto_rand(data1, 1024); - for (i = 0; i < 3; ++i) { - for (j = 85; j < 140; ++j) { - memset(data2,0,1024); - memset(data3,0,1024); - if (i == 0 && j < 129) - continue; - p = (i==0)?PK_NO_PADDING: - (i==1)?PK_PKCS1_PADDING:PK_PKCS1_OAEP_PADDING; - len = crypto_pk_public_hybrid_encrypt(pk1,data2,data1,j,p,0); - test_assert(len>=0); - len = crypto_pk_private_hybrid_decrypt(pk1,data3,data2,len,p,1); - test_eq(len,j); - test_memeq(data1,data3,j); - } - } - - /* Try copy_full */ - crypto_free_pk_env(pk2); - pk2 = crypto_pk_copy_full(pk1); - test_assert(pk2 != NULL); - test_neq_ptr(pk1, pk2); - test_assert(crypto_pk_cmp_keys(pk1,pk2) == 0); - - done: - if (pk1) - crypto_free_pk_env(pk1); - if (pk2) - crypto_free_pk_env(pk2); - tor_free(encoded); -} - -/** Run unit tests for misc crypto functionality. */ -static void -test_crypto(void) -{ - char *data1 = NULL, *data2 = NULL, *data3 = NULL; - int i, j, idx; - - data1 = tor_malloc(1024); - data2 = tor_malloc(1024); - data3 = tor_malloc(1024); - test_assert(data1 && data2 && data3); - - /* Base64 tests */ - memset(data1, 6, 1024); - for (idx = 0; idx < 10; ++idx) { - i = base64_encode(data2, 1024, data1, idx); - test_assert(i >= 0); - j = base64_decode(data3, 1024, data2, i); - test_eq(j,idx); - test_memeq(data3, data1, idx); - } - - strlcpy(data1, "Test string that contains 35 chars.", 1024); - strlcat(data1, " 2nd string that contains 35 chars.", 1024); - - i = base64_encode(data2, 1024, data1, 71); - j = base64_decode(data3, 1024, data2, i); - test_eq(j, 71); - test_streq(data3, data1); - test_assert(data2[i] == '\0'); - - crypto_rand(data1, DIGEST_LEN); - memset(data2, 100, 1024); - digest_to_base64(data2, data1); - test_eq(BASE64_DIGEST_LEN, strlen(data2)); - test_eq(100, data2[BASE64_DIGEST_LEN+2]); - memset(data3, 99, 1024); - test_eq(digest_from_base64(data3, data2), 0); - test_memeq(data1, data3, DIGEST_LEN); - test_eq(99, data3[DIGEST_LEN+1]); - - test_assert(digest_from_base64(data3, "###") < 0); - - /* Base32 tests */ - strlcpy(data1, "5chrs", 1024); - /* bit pattern is: [35 63 68 72 73] -> - * [00110101 01100011 01101000 01110010 01110011] - * By 5s: [00110 10101 10001 10110 10000 11100 10011 10011] - */ - base32_encode(data2, 9, data1, 5); - test_streq(data2, "gvrwq4tt"); - - strlcpy(data1, "\xFF\xF5\x6D\x44\xAE\x0D\x5C\xC9\x62\xC4", 1024); - base32_encode(data2, 30, data1, 10); - test_streq(data2, "772w2rfobvomsywe"); - - /* Base16 tests */ - strlcpy(data1, "6chrs\xff", 1024); - base16_encode(data2, 13, data1, 6); - test_streq(data2, "3663687273FF"); - - strlcpy(data1, "f0d678affc000100", 1024); - i = base16_decode(data2, 8, data1, 16); - test_eq(i,0); - test_memeq(data2, "\xf0\xd6\x78\xaf\xfc\x00\x01\x00",8); - - /* now try some failing base16 decodes */ - test_eq(-1, base16_decode(data2, 8, data1, 15)); /* odd input len */ - test_eq(-1, base16_decode(data2, 7, data1, 16)); /* dest too short */ - strlcpy(data1, "f0dz!8affc000100", 1024); - test_eq(-1, base16_decode(data2, 8, data1, 16)); - - tor_free(data1); - tor_free(data2); - tor_free(data3); - - /* Add spaces to fingerprint */ - { - data1 = tor_strdup("ABCD1234ABCD56780000ABCD1234ABCD56780000"); - test_eq(strlen(data1), 40); - data2 = tor_malloc(FINGERPRINT_LEN+1); - add_spaces_to_fp(data2, FINGERPRINT_LEN+1, data1); - test_streq(data2, "ABCD 1234 ABCD 5678 0000 ABCD 1234 ABCD 5678 0000"); - tor_free(data1); - tor_free(data2); - } - - /* Check fingerprint */ - { - test_assert(crypto_pk_check_fingerprint_syntax( - "ABCD 1234 ABCD 5678 0000 ABCD 1234 ABCD 5678 0000")); - test_assert(!crypto_pk_check_fingerprint_syntax( - "ABCD 1234 ABCD 5678 0000 ABCD 1234 ABCD 5678 000")); - test_assert(!crypto_pk_check_fingerprint_syntax( - "ABCD 1234 ABCD 5678 0000 ABCD 1234 ABCD 5678 00000")); - test_assert(!crypto_pk_check_fingerprint_syntax( - "ABCD 1234 ABCD 5678 0000 ABCD1234 ABCD 5678 0000")); - test_assert(!crypto_pk_check_fingerprint_syntax( - "ABCD 1234 ABCD 5678 0000 ABCD1234 ABCD 5678 00000")); - test_assert(!crypto_pk_check_fingerprint_syntax( - "ACD 1234 ABCD 5678 0000 ABCD 1234 ABCD 5678 00000")); - } - - done: - tor_free(data1); - tor_free(data2); - tor_free(data3); -} - -/** Run unit tests for our secret-to-key passphrase hashing functionality. */ -static void -test_crypto_s2k(void) -{ - char buf[29]; - char buf2[29]; - char *buf3 = NULL; - int i; - - memset(buf, 0, sizeof(buf)); - memset(buf2, 0, sizeof(buf2)); - buf3 = tor_malloc(65536); - memset(buf3, 0, 65536); - - secret_to_key(buf+9, 20, "", 0, buf); - crypto_digest(buf2+9, buf3, 1024); - test_memeq(buf, buf2, 29); - - memcpy(buf,"vrbacrda",8); - memcpy(buf2,"vrbacrda",8); - buf[8] = 96; - buf2[8] = 96; - secret_to_key(buf+9, 20, "12345678", 8, buf); - for (i = 0; i < 65536; i += 16) { - memcpy(buf3+i, "vrbacrda12345678", 16); - } - crypto_digest(buf2+9, buf3, 65536); - test_memeq(buf, buf2, 29); - - done: - tor_free(buf3); -} - -/** Helper: return a tristate based on comparing the strings in *<b>a</b> and - * *<b>b</b>. */ -static int -_compare_strs(const void **a, const void **b) -{ - const char *s1 = *a, *s2 = *b; - return strcmp(s1, s2); -} - -/** Helper: return a tristate based on comparing the strings in *<b>a</b> and - * *<b>b</b>, excluding a's first character, and ignoring case. */ -static int -_compare_without_first_ch(const void *a, const void **b) -{ - const char *s1 = a, *s2 = *b; - return strcasecmp(s1+1, s2); -} - -/** Test basic utility functionality. */ -static void -test_util(void) -{ - struct timeval start, end; - struct tm a_time; - char timestr[RFC1123_TIME_LEN+1]; - char buf[1024]; - time_t t_res; - int i; - uint32_t u32; - uint16_t u16; - char *cp, *k, *v; - const char *str; - - start.tv_sec = 5; - start.tv_usec = 5000; - - end.tv_sec = 5; - end.tv_usec = 5000; - - test_eq(0L, tv_udiff(&start, &end)); - - end.tv_usec = 7000; - - test_eq(2000L, tv_udiff(&start, &end)); - - end.tv_sec = 6; - - test_eq(1002000L, tv_udiff(&start, &end)); - - end.tv_usec = 0; - - test_eq(995000L, tv_udiff(&start, &end)); - - end.tv_sec = 4; - - test_eq(-1005000L, tv_udiff(&start, &end)); - - end.tv_usec = 999990; - start.tv_sec = 1; - start.tv_usec = 500; - - /* The test values here are confirmed to be correct on a platform - * with a working timegm. */ - a_time.tm_year = 2003-1900; - a_time.tm_mon = 7; - a_time.tm_mday = 30; - a_time.tm_hour = 6; - a_time.tm_min = 14; - a_time.tm_sec = 55; - test_eq((time_t) 1062224095UL, tor_timegm(&a_time)); - a_time.tm_year = 2004-1900; /* Try a leap year, after feb. */ - test_eq((time_t) 1093846495UL, tor_timegm(&a_time)); - a_time.tm_mon = 1; /* Try a leap year, in feb. */ - a_time.tm_mday = 10; - test_eq((time_t) 1076393695UL, tor_timegm(&a_time)); - - format_rfc1123_time(timestr, 0); - test_streq("Thu, 01 Jan 1970 00:00:00 GMT", timestr); - format_rfc1123_time(timestr, (time_t)1091580502UL); - test_streq("Wed, 04 Aug 2004 00:48:22 GMT", timestr); - - t_res = 0; - i = parse_rfc1123_time(timestr, &t_res); - test_eq(i,0); - test_eq(t_res, (time_t)1091580502UL); - test_eq(-1, parse_rfc1123_time("Wed, zz Aug 2004 99-99x99 GMT", &t_res)); - tor_gettimeofday(&start); - - /* Tests for corner cases of strl operations */ - test_eq(5, strlcpy(buf, "Hello", 0)); - strlcpy(buf, "Hello", sizeof(buf)); - test_eq(10, strlcat(buf, "Hello", 5)); - - /* Test tor_strstrip() */ - strlcpy(buf, "Testing 1 2 3", sizeof(buf)); - tor_strstrip(buf, ",!"); - test_streq(buf, "Testing 1 2 3"); - strlcpy(buf, "!Testing 1 2 3?", sizeof(buf)); - tor_strstrip(buf, "!? "); - test_streq(buf, "Testing123"); - - /* Test parse_addr_port */ - cp = NULL; u32 = 3; u16 = 3; - test_assert(!parse_addr_port(LOG_WARN, "1.2.3.4", &cp, &u32, &u16)); - test_streq(cp, "1.2.3.4"); - test_eq(u32, 0x01020304u); - test_eq(u16, 0); - tor_free(cp); - test_assert(!parse_addr_port(LOG_WARN, "4.3.2.1:99", &cp, &u32, &u16)); - test_streq(cp, "4.3.2.1"); - test_eq(u32, 0x04030201u); - test_eq(u16, 99); - tor_free(cp); - test_assert(!parse_addr_port(LOG_WARN, "nonexistent.address:4040", - &cp, NULL, &u16)); - test_streq(cp, "nonexistent.address"); - test_eq(u16, 4040); - tor_free(cp); - test_assert(!parse_addr_port(LOG_WARN, "localhost:9999", &cp, &u32, &u16)); - test_streq(cp, "localhost"); - test_eq(u32, 0x7f000001u); - test_eq(u16, 9999); - tor_free(cp); - u32 = 3; - test_assert(!parse_addr_port(LOG_WARN, "localhost", NULL, &u32, &u16)); - test_eq(cp, NULL); - test_eq(u32, 0x7f000001u); - test_eq(u16, 0); - tor_free(cp); - test_eq(0, addr_mask_get_bits(0x0u)); - test_eq(32, addr_mask_get_bits(0xFFFFFFFFu)); - test_eq(16, addr_mask_get_bits(0xFFFF0000u)); - test_eq(31, addr_mask_get_bits(0xFFFFFFFEu)); - test_eq(1, addr_mask_get_bits(0x80000000u)); - - /* Test tor_parse_long. */ - test_eq(10L, tor_parse_long("10",10,0,100,NULL,NULL)); - test_eq(0L, tor_parse_long("10",10,50,100,NULL,NULL)); - test_eq(-50L, tor_parse_long("-50",10,-100,100,NULL,NULL)); - - /* Test tor_parse_ulong */ - test_eq(10UL, tor_parse_ulong("10",10,0,100,NULL,NULL)); - test_eq(0UL, tor_parse_ulong("10",10,50,100,NULL,NULL)); - - /* Test tor_parse_uint64. */ - test_assert(U64_LITERAL(10) == tor_parse_uint64("10 x",10,0,100, &i, &cp)); - test_assert(i == 1); - test_streq(cp, " x"); - test_assert(U64_LITERAL(12345678901) == - tor_parse_uint64("12345678901",10,0,UINT64_MAX, &i, &cp)); - test_assert(i == 1); - test_streq(cp, ""); - test_assert(U64_LITERAL(0) == - tor_parse_uint64("12345678901",10,500,INT32_MAX, &i, &cp)); - test_assert(i == 0); - - /* Test failing snprintf cases */ - test_eq(-1, tor_snprintf(buf, 0, "Foo")); - test_eq(-1, tor_snprintf(buf, 2, "Foo")); - - /* Test printf with uint64 */ - tor_snprintf(buf, sizeof(buf), "x!"U64_FORMAT"!x", - U64_PRINTF_ARG(U64_LITERAL(12345678901))); - test_streq(buf, "x!12345678901!x"); - - /* Test parse_config_line_from_str */ - strlcpy(buf, "k v\n" " key value with spaces \n" "keykey val\n" - "k2\n" - "k3 \n" "\n" " \n" "#comment\n" - "k4#a\n" "k5#abc\n" "k6 val #with comment\n" - "kseven \"a quoted 'string\"\n" - "k8 \"a \\x71uoted\\n\\\"str\\\\ing\\t\\001\\01\\1\\\"\"\n" - , sizeof(buf)); - str = buf; - - str = parse_config_line_from_str(str, &k, &v); - test_streq(k, "k"); - test_streq(v, "v"); - tor_free(k); tor_free(v); - test_assert(!strcmpstart(str, "key value with")); - - str = parse_config_line_from_str(str, &k, &v); - test_streq(k, "key"); - test_streq(v, "value with spaces"); - tor_free(k); tor_free(v); - test_assert(!strcmpstart(str, "keykey")); - - str = parse_config_line_from_str(str, &k, &v); - test_streq(k, "keykey"); - test_streq(v, "val"); - tor_free(k); tor_free(v); - test_assert(!strcmpstart(str, "k2\n")); - - str = parse_config_line_from_str(str, &k, &v); - test_streq(k, "k2"); - test_streq(v, ""); - tor_free(k); tor_free(v); - test_assert(!strcmpstart(str, "k3 \n")); - - str = parse_config_line_from_str(str, &k, &v); - test_streq(k, "k3"); - test_streq(v, ""); - tor_free(k); tor_free(v); - test_assert(!strcmpstart(str, "#comment")); - - str = parse_config_line_from_str(str, &k, &v); - test_streq(k, "k4"); - test_streq(v, ""); - tor_free(k); tor_free(v); - test_assert(!strcmpstart(str, "k5#abc")); - - str = parse_config_line_from_str(str, &k, &v); - test_streq(k, "k5"); - test_streq(v, ""); - tor_free(k); tor_free(v); - test_assert(!strcmpstart(str, "k6")); - - str = parse_config_line_from_str(str, &k, &v); - test_streq(k, "k6"); - test_streq(v, "val"); - tor_free(k); tor_free(v); - test_assert(!strcmpstart(str, "kseven")); - - str = parse_config_line_from_str(str, &k, &v); - test_streq(k, "kseven"); - test_streq(v, "a quoted \'string"); - tor_free(k); tor_free(v); - test_assert(!strcmpstart(str, "k8 ")); - - str = parse_config_line_from_str(str, &k, &v); - test_streq(k, "k8"); - test_streq(v, "a quoted\n\"str\\ing\t\x01\x01\x01\""); - tor_free(k); tor_free(v); - test_streq(str, ""); - - /* Test for strcmpstart and strcmpend. */ - test_assert(strcmpstart("abcdef", "abcdef")==0); - test_assert(strcmpstart("abcdef", "abc")==0); - test_assert(strcmpstart("abcdef", "abd")<0); - test_assert(strcmpstart("abcdef", "abb")>0); - test_assert(strcmpstart("ab", "abb")<0); - - test_assert(strcmpend("abcdef", "abcdef")==0); - test_assert(strcmpend("abcdef", "def")==0); - test_assert(strcmpend("abcdef", "deg")<0); - test_assert(strcmpend("abcdef", "dee")>0); - test_assert(strcmpend("ab", "abb")<0); - - test_assert(strcasecmpend("AbcDEF", "abcdef")==0); - test_assert(strcasecmpend("abcdef", "dEF")==0); - test_assert(strcasecmpend("abcDEf", "deg")<0); - test_assert(strcasecmpend("abcdef", "DEE")>0); - test_assert(strcasecmpend("ab", "abB")<0); - - /* Test mem_is_zero */ - memset(buf,0,128); - buf[128] = 'x'; - test_assert(tor_digest_is_zero(buf)); - test_assert(tor_mem_is_zero(buf, 10)); - test_assert(tor_mem_is_zero(buf, 20)); - test_assert(tor_mem_is_zero(buf, 128)); - test_assert(!tor_mem_is_zero(buf, 129)); - buf[60] = (char)255; - test_assert(!tor_mem_is_zero(buf, 128)); - buf[0] = (char)1; - test_assert(!tor_mem_is_zero(buf, 10)); - - /* Test inet_ntop */ - { - char tmpbuf[TOR_ADDR_BUF_LEN]; - const char *ip = "176.192.208.224"; - struct in_addr in; - tor_inet_pton(AF_INET, ip, &in); - tor_inet_ntop(AF_INET, &in, tmpbuf, sizeof(tmpbuf)); - test_streq(tmpbuf, ip); - } - - /* Test 'escaped' */ - test_streq("\"\"", escaped("")); - test_streq("\"abcd\"", escaped("abcd")); - test_streq("\"\\\\\\n\\r\\t\\\"\\'\"", escaped("\\\n\r\t\"\'")); - test_streq("\"z\\001abc\\277d\"", escaped("z\001abc\277d")); - test_assert(NULL == escaped(NULL)); - - /* Test strndup and memdup */ - { - const char *s = "abcdefghijklmnopqrstuvwxyz"; - cp = tor_strndup(s, 30); - test_streq(cp, s); /* same string, */ - test_neq(cp, s); /* but different pointers. */ - tor_free(cp); - - cp = tor_strndup(s, 5); - test_streq(cp, "abcde"); - tor_free(cp); - - s = "a\0b\0c\0d\0e\0"; - cp = tor_memdup(s,10); - test_memeq(cp, s, 10); /* same ram, */ - test_neq(cp, s); /* but different pointers. */ - tor_free(cp); - } - - /* Test str-foo functions */ - cp = tor_strdup("abcdef"); - test_assert(tor_strisnonupper(cp)); - cp[3] = 'D'; - test_assert(!tor_strisnonupper(cp)); - tor_strupper(cp); - test_streq(cp, "ABCDEF"); - test_assert(tor_strisprint(cp)); - cp[3] = 3; - test_assert(!tor_strisprint(cp)); - tor_free(cp); - - /* Test eat_whitespace. */ - { - const char *s = " \n a"; - test_eq_ptr(eat_whitespace(s), s+4); - s = "abcd"; - test_eq_ptr(eat_whitespace(s), s); - s = "#xyz\nab"; - test_eq_ptr(eat_whitespace(s), s+5); - } - - /* Test memmem and memstr */ - { - const char *haystack = "abcde"; - tor_assert(!tor_memmem(haystack, 5, "ef", 2)); - test_eq_ptr(tor_memmem(haystack, 5, "cd", 2), haystack + 2); - test_eq_ptr(tor_memmem(haystack, 5, "cde", 3), haystack + 2); - haystack = "ababcad"; - test_eq_ptr(tor_memmem(haystack, 7, "abc", 3), haystack + 2); - test_eq_ptr(tor_memstr(haystack, 7, "abc"), haystack + 2); - test_assert(!tor_memstr(haystack, 7, "fe")); - test_assert(!tor_memstr(haystack, 7, "longerthantheoriginal")); - } - - /* Test wrap_string */ - { - smartlist_t *sl = smartlist_create(); - wrap_string(sl, "This is a test of string wrapping functionality: woot.", - 10, "", ""); - cp = smartlist_join_strings(sl, "", 0, NULL); - test_streq(cp, - "This is a\ntest of\nstring\nwrapping\nfunctional\nity: woot.\n"); - tor_free(cp); - SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); - smartlist_clear(sl); - - wrap_string(sl, "This is a test of string wrapping functionality: woot.", - 16, "### ", "# "); - cp = smartlist_join_strings(sl, "", 0, NULL); - test_streq(cp, - "### This is a\n# test of string\n# wrapping\n# functionality:\n" - "# woot.\n"); - - tor_free(cp); - SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); - smartlist_free(sl); - } - - tor_gettimeofday(&start); - /* now make sure time works. */ - tor_gettimeofday(&end); - /* We might've timewarped a little. */ - test_assert(tv_udiff(&start, &end) >= -5000); - - /* Test tor_log2(). */ - test_eq(tor_log2(64), 6); - test_eq(tor_log2(65), 6); - test_eq(tor_log2(63), 5); - test_eq(tor_log2(1), 0); - test_eq(tor_log2(2), 1); - test_eq(tor_log2(3), 1); - test_eq(tor_log2(4), 2); - test_eq(tor_log2(5), 2); - test_eq(tor_log2(U64_LITERAL(40000000000000000)), 55); - test_eq(tor_log2(UINT64_MAX), 63); - - /* Test round_to_power_of_2 */ - test_eq(round_to_power_of_2(120), 128); - test_eq(round_to_power_of_2(128), 128); - test_eq(round_to_power_of_2(130), 128); - test_eq(round_to_power_of_2(U64_LITERAL(40000000000000000)), - U64_LITERAL(1)<<55); - test_eq(round_to_power_of_2(0), 2); - - done: - ; -} - -/** Helper: assert that IPv6 addresses <b>a</b> and <b>b</b> are the same. On - * failure, reports an error, describing the addresses as <b>e1</b> and - * <b>e2</b>, and reporting the line number as <b>line</b>. */ -static void -_test_eq_ip6(struct in6_addr *a, struct in6_addr *b, const char *e1, - const char *e2, int line) -{ - int i; - int ok = 1; - for (i = 0; i < 16; ++i) { - if (a->s6_addr[i] != b->s6_addr[i]) { - ok = 0; - break; - } - } - if (ok) { - printf("."); fflush(stdout); - } else { - char buf1[128], *cp1; - char buf2[128], *cp2; - have_failed = 1; - cp1 = buf1; cp2 = buf2; - for (i=0; i<16; ++i) { - tor_snprintf(cp1, sizeof(buf1)-(cp1-buf1), "%02x", a->s6_addr[i]); - tor_snprintf(cp2, sizeof(buf2)-(cp2-buf2), "%02x", b->s6_addr[i]); - cp1 += 2; cp2 += 2; - if ((i%2)==1 && i != 15) { - *cp1++ = ':'; - *cp2++ = ':'; - } - } - *cp1 = *cp2 = '\0'; - printf("Line %d: assertion failed: (%s == %s)\n" - " %s != %s\n", line, e1, e2, buf1, buf2); - fflush(stdout); - } -} - -/** Helper: Assert that two strings both decode as IPv6 addresses with - * tor_inet_pton(), and both decode to the same address. */ -#define test_pton6_same(a,b) STMT_BEGIN \ - test_eq(tor_inet_pton(AF_INET6, a, &a1), 1); \ - test_eq(tor_inet_pton(AF_INET6, b, &a2), 1); \ - _test_eq_ip6(&a1,&a2,#a,#b,__LINE__); \ - STMT_END - -/** Helper: Assert that <b>a</b> is recognized as a bad IPv6 address by - * tor_inet_pton(). */ -#define test_pton6_bad(a) \ - test_eq(0, tor_inet_pton(AF_INET6, a, &a1)) - -/** Helper: assert that <b>a</b>, when parsed by tor_inet_pton() and displayed - * with tor_inet_ntop(), yields <b>b</b>. Also assert that <b>b</b> parses to - * the same value as <b>a</b>. */ -#define test_ntop6_reduces(a,b) STMT_BEGIN \ - test_eq(tor_inet_pton(AF_INET6, a, &a1), 1); \ - test_streq(tor_inet_ntop(AF_INET6, &a1, buf, sizeof(buf)), b); \ - test_eq(tor_inet_pton(AF_INET6, b, &a2), 1); \ - _test_eq_ip6(&a1, &a2, a, b, __LINE__); \ - STMT_END - -/** Helper: assert that <b>a</b> parses by tor_inet_pton() into a address that - * passes tor_addr_is_internal() with <b>for_listening</b>. */ -#define test_internal_ip(a,for_listening) STMT_BEGIN \ - test_eq(tor_inet_pton(AF_INET6, a, &t1.addr.in6_addr), 1); \ - t1.family = AF_INET6; \ - if (!tor_addr_is_internal(&t1, for_listening)) \ - test_fail_msg( a "was not internal."); \ - STMT_END - -/** Helper: assert that <b>a</b> parses by tor_inet_pton() into a address that - * does not pass tor_addr_is_internal() with <b>for_listening</b>. */ -#define test_external_ip(a,for_listening) STMT_BEGIN \ - test_eq(tor_inet_pton(AF_INET6, a, &t1.addr.in6_addr), 1); \ - t1.family = AF_INET6; \ - if (tor_addr_is_internal(&t1, for_listening)) \ - test_fail_msg(a "was not external."); \ - STMT_END - -/** Helper: Assert that <b>a</b> and <b>b</b>, when parsed by - * tor_inet_pton(), give addresses that compare in the order defined by - * <b>op</b> with tor_addr_compare(). */ -#define test_addr_compare(a, op, b) STMT_BEGIN \ - test_eq(tor_inet_pton(AF_INET6, a, &t1.addr.in6_addr), 1); \ - test_eq(tor_inet_pton(AF_INET6, b, &t2.addr.in6_addr), 1); \ - t1.family = t2.family = AF_INET6; \ - r = tor_addr_compare(&t1,&t2,CMP_SEMANTIC); \ - if (!(r op 0)) \ - test_fail_msg("failed: tor_addr_compare("a","b") "#op" 0"); \ - STMT_END - -/** Helper: Assert that <b>a</b> and <b>b</b>, when parsed by - * tor_inet_pton(), give addresses that compare in the order defined by - * <b>op</b> with tor_addr_compare_masked() with <b>m</b> masked. */ -#define test_addr_compare_masked(a, op, b, m) STMT_BEGIN \ - test_eq(tor_inet_pton(AF_INET6, a, &t1.addr.in6_addr), 1); \ - test_eq(tor_inet_pton(AF_INET6, b, &t2.addr.in6_addr), 1); \ - t1.family = t2.family = AF_INET6; \ - r = tor_addr_compare_masked(&t1,&t2,m,CMP_SEMANTIC); \ - if (!(r op 0)) \ - test_fail_msg("failed: tor_addr_compare_masked("a","b","#m") "#op" 0"); \ - STMT_END - -/** Helper: assert that <b>xx</b> is parseable as a masked IPv6 address with - * ports by tor_parse_mask_addr_ports(), with family <b>f</b>, IP address - * as 4 32-bit words <b>ip1...ip4</b>, mask bits as <b>mm</b>, and port range - * as <b>pt1..pt2</b>. */ -#define test_addr_mask_ports_parse(xx, f, ip1, ip2, ip3, ip4, mm, pt1, pt2) \ - STMT_BEGIN \ - test_eq(tor_addr_parse_mask_ports(xx, &t1, &mask, &port1, &port2), f); \ - p1=tor_inet_ntop(AF_INET6, &t1.addr.in6_addr, bug, sizeof(bug)); \ - test_eq(htonl(ip1), tor_addr_to_in6_addr32(&t1)[0]); \ - test_eq(htonl(ip2), tor_addr_to_in6_addr32(&t1)[1]); \ - test_eq(htonl(ip3), tor_addr_to_in6_addr32(&t1)[2]); \ - test_eq(htonl(ip4), tor_addr_to_in6_addr32(&t1)[3]); \ - test_eq(mask, mm); \ - test_eq(port1, pt1); \ - test_eq(port2, pt2); \ - STMT_END - -/** Run unit tests for IPv6 encoding/decoding/manipulation functions. */ -static void -test_util_ip6_helpers(void) -{ - char buf[TOR_ADDR_BUF_LEN], bug[TOR_ADDR_BUF_LEN]; - struct in6_addr a1, a2; - tor_addr_t t1, t2; - int r, i; - uint16_t port1, port2; - maskbits_t mask; - const char *p1; - struct sockaddr_storage sa_storage; - struct sockaddr_in *sin; - struct sockaddr_in6 *sin6; - - // struct in_addr b1, b2; - /* Test tor_inet_ntop and tor_inet_pton: IPv6 */ - - /* ==== Converting to and from sockaddr_t. */ - sin = (struct sockaddr_in *)&sa_storage; - sin->sin_family = AF_INET; - sin->sin_port = 9090; - sin->sin_addr.s_addr = htonl(0x7f7f0102); /*127.127.1.2*/ - tor_addr_from_sockaddr(&t1, (struct sockaddr *)sin, NULL); - test_eq(tor_addr_family(&t1), AF_INET); - test_eq(tor_addr_to_ipv4h(&t1), 0x7f7f0102); - - memset(&sa_storage, 0, sizeof(sa_storage)); - test_eq(sizeof(struct sockaddr_in), - tor_addr_to_sockaddr(&t1, 1234, (struct sockaddr *)&sa_storage, - sizeof(sa_storage))); - test_eq(1234, ntohs(sin->sin_port)); - test_eq(0x7f7f0102, ntohl(sin->sin_addr.s_addr)); - - memset(&sa_storage, 0, sizeof(sa_storage)); - sin6 = (struct sockaddr_in6 *)&sa_storage; - sin6->sin6_family = AF_INET6; - sin6->sin6_port = htons(7070); - sin6->sin6_addr.s6_addr[0] = 128; - tor_addr_from_sockaddr(&t1, (struct sockaddr *)sin6, NULL); - test_eq(tor_addr_family(&t1), AF_INET6); - p1 = tor_addr_to_str(buf, &t1, sizeof(buf), 0); - test_streq(p1, "8000::"); - - memset(&sa_storage, 0, sizeof(sa_storage)); - test_eq(sizeof(struct sockaddr_in6), - tor_addr_to_sockaddr(&t1, 9999, (struct sockaddr *)&sa_storage, - sizeof(sa_storage))); - test_eq(AF_INET6, sin6->sin6_family); - test_eq(9999, ntohs(sin6->sin6_port)); - test_eq(0x80000000, ntohl(S6_ADDR32(sin6->sin6_addr)[0])); - - /* ==== tor_addr_lookup: static cases. (Can't test dns without knowing we - * have a good resolver. */ - test_eq(0, tor_addr_lookup("127.128.129.130", AF_UNSPEC, &t1)); - test_eq(AF_INET, tor_addr_family(&t1)); - test_eq(tor_addr_to_ipv4h(&t1), 0x7f808182); - - test_eq(0, tor_addr_lookup("9000::5", AF_UNSPEC, &t1)); - test_eq(AF_INET6, tor_addr_family(&t1)); - test_eq(0x90, tor_addr_to_in6_addr8(&t1)[0]); - test_assert(tor_mem_is_zero((char*)tor_addr_to_in6_addr8(&t1)+1, 14)); - test_eq(0x05, tor_addr_to_in6_addr8(&t1)[15]); - - /* === Test pton: valid af_inet6 */ - /* Simple, valid parsing. */ - r = tor_inet_pton(AF_INET6, - "0102:0304:0506:0708:090A:0B0C:0D0E:0F10", &a1); - test_assert(r==1); - for (i=0;i<16;++i) { test_eq(i+1, (int)a1.s6_addr[i]); } - /* ipv4 ending. */ - test_pton6_same("0102:0304:0506:0708:090A:0B0C:0D0E:0F10", - "0102:0304:0506:0708:090A:0B0C:13.14.15.16"); - /* shortened words. */ - test_pton6_same("0001:0099:BEEF:0000:0123:FFFF:0001:0001", - "1:99:BEEF:0:0123:FFFF:1:1"); - /* zeros at the beginning */ - test_pton6_same("0000:0000:0000:0000:0009:C0A8:0001:0001", - "::9:c0a8:1:1"); - test_pton6_same("0000:0000:0000:0000:0009:C0A8:0001:0001", - "::9:c0a8:0.1.0.1"); - /* zeros in the middle. */ - test_pton6_same("fe80:0000:0000:0000:0202:1111:0001:0001", - "fe80::202:1111:1:1"); - /* zeros at the end. */ - test_pton6_same("1000:0001:0000:0007:0000:0000:0000:0000", - "1000:1:0:7::"); - - /* === Test ntop: af_inet6 */ - test_ntop6_reduces("0:0:0:0:0:0:0:0", "::"); - - test_ntop6_reduces("0001:0099:BEEF:0006:0123:FFFF:0001:0001", - "1:99:beef:6:123:ffff:1:1"); - - //test_ntop6_reduces("0:0:0:0:0:0:c0a8:0101", "::192.168.1.1"); - test_ntop6_reduces("0:0:0:0:0:ffff:c0a8:0101", "::ffff:192.168.1.1"); - test_ntop6_reduces("002:0:0000:0:3::4", "2::3:0:0:4"); - test_ntop6_reduces("0:0::1:0:3", "::1:0:3"); - test_ntop6_reduces("008:0::0", "8::"); - test_ntop6_reduces("0:0:0:0:0:ffff::1", "::ffff:0.0.0.1"); - test_ntop6_reduces("abcd:0:0:0:0:0:7f00::", "abcd::7f00:0"); - test_ntop6_reduces("0000:0000:0000:0000:0009:C0A8:0001:0001", - "::9:c0a8:1:1"); - test_ntop6_reduces("fe80:0000:0000:0000:0202:1111:0001:0001", - "fe80::202:1111:1:1"); - test_ntop6_reduces("1000:0001:0000:0007:0000:0000:0000:0000", - "1000:1:0:7::"); - - /* === Test pton: invalid in6. */ - test_pton6_bad("foobar."); - test_pton6_bad("55555::"); - test_pton6_bad("9:-60::"); - test_pton6_bad("1:2:33333:4:0002:3::"); - //test_pton6_bad("1:2:3333:4:00002:3::");// BAD, but glibc doesn't say so. - test_pton6_bad("1:2:3333:4:fish:3::"); - test_pton6_bad("1:2:3:4:5:6:7:8:9"); - test_pton6_bad("1:2:3:4:5:6:7"); - test_pton6_bad("1:2:3:4:5:6:1.2.3.4.5"); - test_pton6_bad("1:2:3:4:5:6:1.2.3"); - test_pton6_bad("::1.2.3"); - test_pton6_bad("::1.2.3.4.5"); - test_pton6_bad("99"); - test_pton6_bad(""); - test_pton6_bad("1::2::3:4"); - test_pton6_bad("a:::b:c"); - test_pton6_bad(":::a:b:c"); - test_pton6_bad("a:b:c:::"); - - /* test internal checking */ - test_external_ip("fbff:ffff::2:7", 0); - test_internal_ip("fc01::2:7", 0); - test_internal_ip("fdff:ffff::f:f", 0); - test_external_ip("fe00::3:f", 0); - - test_external_ip("fe7f:ffff::2:7", 0); - test_internal_ip("fe80::2:7", 0); - test_internal_ip("febf:ffff::f:f", 0); - - test_internal_ip("fec0::2:7:7", 0); - test_internal_ip("feff:ffff::e:7:7", 0); - test_external_ip("ff00::e:7:7", 0); - - test_internal_ip("::", 0); - test_internal_ip("::1", 0); - test_internal_ip("::1", 1); - test_internal_ip("::", 0); - test_external_ip("::", 1); - test_external_ip("::2", 0); - test_external_ip("2001::", 0); - test_external_ip("ffff::", 0); - - test_external_ip("::ffff:0.0.0.0", 1); - test_internal_ip("::ffff:0.0.0.0", 0); - test_internal_ip("::ffff:0.255.255.255", 0); - test_external_ip("::ffff:1.0.0.0", 0); - - test_external_ip("::ffff:9.255.255.255", 0); - test_internal_ip("::ffff:10.0.0.0", 0); - test_internal_ip("::ffff:10.255.255.255", 0); - test_external_ip("::ffff:11.0.0.0", 0); - - test_external_ip("::ffff:126.255.255.255", 0); - test_internal_ip("::ffff:127.0.0.0", 0); - test_internal_ip("::ffff:127.255.255.255", 0); - test_external_ip("::ffff:128.0.0.0", 0); - - test_external_ip("::ffff:172.15.255.255", 0); - test_internal_ip("::ffff:172.16.0.0", 0); - test_internal_ip("::ffff:172.31.255.255", 0); - test_external_ip("::ffff:172.32.0.0", 0); - - test_external_ip("::ffff:192.167.255.255", 0); - test_internal_ip("::ffff:192.168.0.0", 0); - test_internal_ip("::ffff:192.168.255.255", 0); - test_external_ip("::ffff:192.169.0.0", 0); - - test_external_ip("::ffff:169.253.255.255", 0); - test_internal_ip("::ffff:169.254.0.0", 0); - test_internal_ip("::ffff:169.254.255.255", 0); - test_external_ip("::ffff:169.255.0.0", 0); - test_assert(is_internal_IP(0x7f000001, 0)); - - /* tor_addr_compare(tor_addr_t x2) */ - test_addr_compare("ffff::", ==, "ffff::0"); - test_addr_compare("0::3:2:1", <, "0::ffff:0.3.2.1"); - test_addr_compare("0::2:2:1", <, "0::ffff:0.3.2.1"); - test_addr_compare("0::ffff:0.3.2.1", >, "0::0:0:0"); - test_addr_compare("0::ffff:5.2.2.1", <, "::ffff:6.0.0.0"); /* XXXX wrong. */ - tor_addr_parse_mask_ports("[::ffff:2.3.4.5]", &t1, NULL, NULL, NULL); - tor_addr_parse_mask_ports("2.3.4.5", &t2, NULL, NULL, NULL); - test_assert(tor_addr_compare(&t1, &t2, CMP_SEMANTIC) == 0); - tor_addr_parse_mask_ports("[::ffff:2.3.4.4]", &t1, NULL, NULL, NULL); - tor_addr_parse_mask_ports("2.3.4.5", &t2, NULL, NULL, NULL); - test_assert(tor_addr_compare(&t1, &t2, CMP_SEMANTIC) < 0); - - /* test compare_masked */ - test_addr_compare_masked("ffff::", ==, "ffff::0", 128); - test_addr_compare_masked("ffff::", ==, "ffff::0", 64); - test_addr_compare_masked("0::2:2:1", <, "0::8000:2:1", 81); - test_addr_compare_masked("0::2:2:1", ==, "0::8000:2:1", 80); - - /* Test decorated addr_to_string. */ - test_eq(AF_INET6, tor_addr_from_str(&t1, "[123:45:6789::5005:11]")); - p1 = tor_addr_to_str(buf, &t1, sizeof(buf), 1); - test_streq(p1, "[123:45:6789::5005:11]"); - test_eq(AF_INET, tor_addr_from_str(&t1, "18.0.0.1")); - p1 = tor_addr_to_str(buf, &t1, sizeof(buf), 1); - test_streq(p1, "18.0.0.1"); - - /* Test tor_addr_parse_reverse_lookup_name */ - i = tor_addr_parse_reverse_lookup_name(&t1, "Foobar.baz", AF_UNSPEC, 0); - test_eq(0, i); - i = tor_addr_parse_reverse_lookup_name(&t1, "Foobar.baz", AF_UNSPEC, 1); - test_eq(0, i); - i = tor_addr_parse_reverse_lookup_name(&t1, "1.0.168.192.in-addr.arpa", - AF_UNSPEC, 1); - test_eq(1, i); - test_eq(tor_addr_family(&t1), AF_INET); - p1 = tor_addr_to_str(buf, &t1, sizeof(buf), 1); - test_streq(p1, "192.168.0.1"); - i = tor_addr_parse_reverse_lookup_name(&t1, "192.168.0.99", AF_UNSPEC, 0); - test_eq(0, i); - i = tor_addr_parse_reverse_lookup_name(&t1, "192.168.0.99", AF_UNSPEC, 1); - test_eq(1, i); - p1 = tor_addr_to_str(buf, &t1, sizeof(buf), 1); - test_streq(p1, "192.168.0.99"); - memset(&t1, 0, sizeof(t1)); - i = tor_addr_parse_reverse_lookup_name(&t1, - "0.1.2.3.4.5.6.7.8.9.a.b.c.d.e.f." - "f.e.e.b.1.e.b.e.e.f.f.e.e.e.d.9." - "ip6.ARPA", - AF_UNSPEC, 0); - test_eq(1, i); - p1 = tor_addr_to_str(buf, &t1, sizeof(buf), 1); - test_streq(p1, "[9dee:effe:ebe1:beef:fedc:ba98:7654:3210]"); - /* Failing cases. */ - i = tor_addr_parse_reverse_lookup_name(&t1, - "6.7.8.9.a.b.c.d.e.f." - "f.e.e.b.1.e.b.e.e.f.f.e.e.e.d.9." - "ip6.ARPA", - AF_UNSPEC, 0); - test_eq(i, -1); - i = tor_addr_parse_reverse_lookup_name(&t1, - "6.7.8.9.a.b.c.d.e.f.a.b.c.d.e.f.0." - "f.e.e.b.1.e.b.e.e.f.f.e.e.e.d.9." - "ip6.ARPA", - AF_UNSPEC, 0); - test_eq(i, -1); - i = tor_addr_parse_reverse_lookup_name(&t1, - "6.7.8.9.a.b.c.d.e.f.X.0.0.0.0.9." - "f.e.e.b.1.e.b.e.e.f.f.e.e.e.d.9." - "ip6.ARPA", - AF_UNSPEC, 0); - test_eq(i, -1); - i = tor_addr_parse_reverse_lookup_name(&t1, "32.1.1.in-addr.arpa", - AF_UNSPEC, 0); - test_eq(i, -1); - i = tor_addr_parse_reverse_lookup_name(&t1, ".in-addr.arpa", - AF_UNSPEC, 0); - test_eq(i, -1); - i = tor_addr_parse_reverse_lookup_name(&t1, "1.2.3.4.5.in-addr.arpa", - AF_UNSPEC, 0); - test_eq(i, -1); - i = tor_addr_parse_reverse_lookup_name(&t1, "1.2.3.4.5.in-addr.arpa", - AF_INET6, 0); - test_eq(i, -1); - i = tor_addr_parse_reverse_lookup_name(&t1, - "6.7.8.9.a.b.c.d.e.f.a.b.c.d.e.0." - "f.e.e.b.1.e.b.e.e.f.f.e.e.e.d.9." - "ip6.ARPA", - AF_INET, 0); - test_eq(i, -1); - - /* test tor_addr_parse_mask_ports */ - test_addr_mask_ports_parse("[::f]/17:47-95", AF_INET6, - 0, 0, 0, 0x0000000f, 17, 47, 95); - //test_addr_parse("[::fefe:4.1.1.7/120]:999-1000"); - //test_addr_parse_check("::fefe:401:107", 120, 999, 1000); - test_addr_mask_ports_parse("[::ffff:4.1.1.7]/120:443", AF_INET6, - 0, 0, 0x0000ffff, 0x04010107, 120, 443, 443); - test_addr_mask_ports_parse("[abcd:2::44a:0]:2-65000", AF_INET6, - 0xabcd0002, 0, 0, 0x044a0000, 128, 2, 65000); - - r=tor_addr_parse_mask_ports("[fefef::]/112", &t1, NULL, NULL, NULL); - test_assert(r == -1); - r=tor_addr_parse_mask_ports("efef::/112", &t1, NULL, NULL, NULL); - test_assert(r == -1); - r=tor_addr_parse_mask_ports("[f:f:f:f:f:f:f:f::]", &t1, NULL, NULL, NULL); - test_assert(r == -1); - r=tor_addr_parse_mask_ports("[::f:f:f:f:f:f:f:f]", &t1, NULL, NULL, NULL); - test_assert(r == -1); - r=tor_addr_parse_mask_ports("[f:f:f:f:f:f:f:f:f]", &t1, NULL, NULL, NULL); - test_assert(r == -1); - /* Test for V4-mapped address with mask < 96. (arguably not valid) */ - r=tor_addr_parse_mask_ports("[::ffff:1.1.2.2/33]", &t1, &mask, NULL, NULL); - test_assert(r == -1); - r=tor_addr_parse_mask_ports("1.1.2.2/33", &t1, &mask, NULL, NULL); - test_assert(r == -1); - r=tor_addr_parse_mask_ports("1.1.2.2/31", &t1, &mask, NULL, NULL); - test_assert(r == AF_INET); - r=tor_addr_parse_mask_ports("[efef::]/112", &t1, &mask, &port1, &port2); - test_assert(r == AF_INET6); - test_assert(port1 == 1); - test_assert(port2 == 65535); - - /* make sure inet address lengths >= max */ - test_assert(INET_NTOA_BUF_LEN >= sizeof("255.255.255.255")); - test_assert(TOR_ADDR_BUF_LEN >= - sizeof("ffff:ffff:ffff:ffff:ffff:ffff:255.255.255.255")); - - test_assert(sizeof(tor_addr_t) >= sizeof(struct in6_addr)); - - /* get interface addresses */ - r = get_interface_address6(LOG_DEBUG, AF_INET, &t1); - i = get_interface_address6(LOG_DEBUG, AF_INET6, &t2); -#if 0 - tor_inet_ntop(AF_INET, &t1.sa.sin_addr, buf, sizeof(buf)); - printf("\nv4 address: %s (family=%i)", buf, IN_FAMILY(&t1)); - tor_inet_ntop(AF_INET6, &t2.sa6.sin6_addr, buf, sizeof(buf)); - printf("\nv6 address: %s (family=%i)", buf, IN_FAMILY(&t2)); -#endif - - done: - ; -} - -/** Run unit tests for basic dynamic-sized array functionality. */ -static void -test_util_smartlist_basic(void) -{ - smartlist_t *sl; - - /* XXXX test sort_digests, uniq_strings, uniq_digests */ - - /* Test smartlist add, del_keeporder, insert, get. */ - sl = smartlist_create(); - smartlist_add(sl, (void*)1); - smartlist_add(sl, (void*)2); - smartlist_add(sl, (void*)3); - smartlist_add(sl, (void*)4); - smartlist_del_keeporder(sl, 1); - smartlist_insert(sl, 1, (void*)22); - smartlist_insert(sl, 0, (void*)0); - smartlist_insert(sl, 5, (void*)555); - test_eq_ptr((void*)0, smartlist_get(sl,0)); - test_eq_ptr((void*)1, smartlist_get(sl,1)); - test_eq_ptr((void*)22, smartlist_get(sl,2)); - test_eq_ptr((void*)3, smartlist_get(sl,3)); - test_eq_ptr((void*)4, smartlist_get(sl,4)); - test_eq_ptr((void*)555, smartlist_get(sl,5)); - /* Try deleting in the middle. */ - smartlist_del(sl, 1); - test_eq_ptr((void*)555, smartlist_get(sl, 1)); - /* Try deleting at the end. */ - smartlist_del(sl, 4); - test_eq(4, smartlist_len(sl)); - - /* test isin. */ - test_assert(smartlist_isin(sl, (void*)3)); - test_assert(!smartlist_isin(sl, (void*)99)); - - done: - smartlist_free(sl); -} - -/** Run unit tests for smartlist-of-strings functionality. */ -static void -test_util_smartlist_strings(void) -{ - smartlist_t *sl = smartlist_create(); - char *cp=NULL, *cp_alloc=NULL; - size_t sz; - - /* Test split and join */ - test_eq(0, smartlist_len(sl)); - smartlist_split_string(sl, "abc", ":", 0, 0); - test_eq(1, smartlist_len(sl)); - test_streq("abc", smartlist_get(sl, 0)); - smartlist_split_string(sl, "a::bc::", "::", 0, 0); - test_eq(4, smartlist_len(sl)); - test_streq("a", smartlist_get(sl, 1)); - test_streq("bc", smartlist_get(sl, 2)); - test_streq("", smartlist_get(sl, 3)); - cp_alloc = smartlist_join_strings(sl, "", 0, NULL); - test_streq(cp_alloc, "abcabc"); - tor_free(cp_alloc); - cp_alloc = smartlist_join_strings(sl, "!", 0, NULL); - test_streq(cp_alloc, "abc!a!bc!"); - tor_free(cp_alloc); - cp_alloc = smartlist_join_strings(sl, "XY", 0, NULL); - test_streq(cp_alloc, "abcXYaXYbcXY"); - tor_free(cp_alloc); - cp_alloc = smartlist_join_strings(sl, "XY", 1, NULL); - test_streq(cp_alloc, "abcXYaXYbcXYXY"); - tor_free(cp_alloc); - cp_alloc = smartlist_join_strings(sl, "", 1, NULL); - test_streq(cp_alloc, "abcabc"); - tor_free(cp_alloc); - - smartlist_split_string(sl, "/def/ /ghijk", "/", 0, 0); - test_eq(8, smartlist_len(sl)); - test_streq("", smartlist_get(sl, 4)); - test_streq("def", smartlist_get(sl, 5)); - test_streq(" ", smartlist_get(sl, 6)); - test_streq("ghijk", smartlist_get(sl, 7)); - SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); - smartlist_clear(sl); - - smartlist_split_string(sl, "a,bbd,cdef", ",", SPLIT_SKIP_SPACE, 0); - test_eq(3, smartlist_len(sl)); - test_streq("a", smartlist_get(sl,0)); - test_streq("bbd", smartlist_get(sl,1)); - test_streq("cdef", smartlist_get(sl,2)); - smartlist_split_string(sl, " z <> zhasd <> <> bnud<> ", "<>", - SPLIT_SKIP_SPACE, 0); - test_eq(8, smartlist_len(sl)); - test_streq("z", smartlist_get(sl,3)); - test_streq("zhasd", smartlist_get(sl,4)); - test_streq("", smartlist_get(sl,5)); - test_streq("bnud", smartlist_get(sl,6)); - test_streq("", smartlist_get(sl,7)); - - SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); - smartlist_clear(sl); - - smartlist_split_string(sl, " ab\tc \td ef ", NULL, - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); - test_eq(4, smartlist_len(sl)); - test_streq("ab", smartlist_get(sl,0)); - test_streq("c", smartlist_get(sl,1)); - test_streq("d", smartlist_get(sl,2)); - test_streq("ef", smartlist_get(sl,3)); - smartlist_split_string(sl, "ghi\tj", NULL, - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); - test_eq(6, smartlist_len(sl)); - test_streq("ghi", smartlist_get(sl,4)); - test_streq("j", smartlist_get(sl,5)); - - SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); - smartlist_clear(sl); - - cp_alloc = smartlist_join_strings(sl, "XY", 0, NULL); - test_streq(cp_alloc, ""); - tor_free(cp_alloc); - cp_alloc = smartlist_join_strings(sl, "XY", 1, NULL); - test_streq(cp_alloc, "XY"); - tor_free(cp_alloc); - - smartlist_split_string(sl, " z <> zhasd <> <> bnud<> ", "<>", - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); - test_eq(3, smartlist_len(sl)); - test_streq("z", smartlist_get(sl, 0)); - test_streq("zhasd", smartlist_get(sl, 1)); - test_streq("bnud", smartlist_get(sl, 2)); - smartlist_split_string(sl, " z <> zhasd <> <> bnud<> ", "<>", - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 2); - test_eq(5, smartlist_len(sl)); - test_streq("z", smartlist_get(sl, 3)); - test_streq("zhasd <> <> bnud<>", smartlist_get(sl, 4)); - SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); - smartlist_clear(sl); - - smartlist_split_string(sl, "abcd\n", "\n", - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); - test_eq(1, smartlist_len(sl)); - test_streq("abcd", smartlist_get(sl, 0)); - smartlist_split_string(sl, "efgh", "\n", - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); - test_eq(2, smartlist_len(sl)); - test_streq("efgh", smartlist_get(sl, 1)); - - SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); - smartlist_clear(sl); - - /* Test swapping, shuffling, and sorting. */ - smartlist_split_string(sl, "the,onion,router,by,arma,and,nickm", ",", 0, 0); - test_eq(7, smartlist_len(sl)); - smartlist_sort(sl, _compare_strs); - cp_alloc = smartlist_join_strings(sl, ",", 0, NULL); - test_streq(cp_alloc,"and,arma,by,nickm,onion,router,the"); - tor_free(cp_alloc); - smartlist_swap(sl, 1, 5); - cp_alloc = smartlist_join_strings(sl, ",", 0, NULL); - test_streq(cp_alloc,"and,router,by,nickm,onion,arma,the"); - tor_free(cp_alloc); - smartlist_shuffle(sl); - test_eq(7, smartlist_len(sl)); - test_assert(smartlist_string_isin(sl, "and")); - test_assert(smartlist_string_isin(sl, "router")); - test_assert(smartlist_string_isin(sl, "by")); - test_assert(smartlist_string_isin(sl, "nickm")); - test_assert(smartlist_string_isin(sl, "onion")); - test_assert(smartlist_string_isin(sl, "arma")); - test_assert(smartlist_string_isin(sl, "the")); - - /* Test bsearch. */ - smartlist_sort(sl, _compare_strs); - test_streq("nickm", smartlist_bsearch(sl, "zNicKM", - _compare_without_first_ch)); - test_streq("and", smartlist_bsearch(sl, " AND", _compare_without_first_ch)); - test_eq_ptr(NULL, smartlist_bsearch(sl, " ANz", _compare_without_first_ch)); - - /* Test bsearch_idx */ - { - int f; - test_eq(0, smartlist_bsearch_idx(sl," aaa",_compare_without_first_ch,&f)); - test_eq(f, 0); - test_eq(0, smartlist_bsearch_idx(sl," and",_compare_without_first_ch,&f)); - test_eq(f, 1); - test_eq(1, smartlist_bsearch_idx(sl," arm",_compare_without_first_ch,&f)); - test_eq(f, 0); - test_eq(1, smartlist_bsearch_idx(sl," arma",_compare_without_first_ch,&f)); - test_eq(f, 1); - test_eq(2, smartlist_bsearch_idx(sl," armb",_compare_without_first_ch,&f)); - test_eq(f, 0); - test_eq(7, smartlist_bsearch_idx(sl," zzzz",_compare_without_first_ch,&f)); - test_eq(f, 0); - } - - /* Test reverse() and pop_last() */ - smartlist_reverse(sl); - cp_alloc = smartlist_join_strings(sl, ",", 0, NULL); - test_streq(cp_alloc,"the,router,onion,nickm,by,arma,and"); - tor_free(cp_alloc); - cp_alloc = smartlist_pop_last(sl); - test_streq(cp_alloc, "and"); - tor_free(cp_alloc); - test_eq(smartlist_len(sl), 6); - SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); - smartlist_clear(sl); - cp_alloc = smartlist_pop_last(sl); - test_eq(cp_alloc, NULL); - - /* Test uniq() */ - smartlist_split_string(sl, - "50,noon,radar,a,man,a,plan,a,canal,panama,radar,noon,50", - ",", 0, 0); - smartlist_sort(sl, _compare_strs); - smartlist_uniq(sl, _compare_strs, _tor_free); - cp_alloc = smartlist_join_strings(sl, ",", 0, NULL); - test_streq(cp_alloc, "50,a,canal,man,noon,panama,plan,radar"); - tor_free(cp_alloc); - - /* Test string_isin and isin_case and num_isin */ - test_assert(smartlist_string_isin(sl, "noon")); - test_assert(!smartlist_string_isin(sl, "noonoon")); - test_assert(smartlist_string_isin_case(sl, "nOOn")); - test_assert(!smartlist_string_isin_case(sl, "nooNooN")); - test_assert(smartlist_string_num_isin(sl, 50)); - test_assert(!smartlist_string_num_isin(sl, 60)); - - /* Test smartlist_choose */ - { - int i; - int allsame = 1; - int allin = 1; - void *first = smartlist_choose(sl); - test_assert(smartlist_isin(sl, first)); - for (i = 0; i < 100; ++i) { - void *second = smartlist_choose(sl); - if (second != first) - allsame = 0; - if (!smartlist_isin(sl, second)) - allin = 0; - } - test_assert(!allsame); - test_assert(allin); - } - SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); - smartlist_clear(sl); - - /* Test string_remove and remove and join_strings2 */ - smartlist_split_string(sl, - "Some say the Earth will end in ice and some in fire", - " ", 0, 0); - cp = smartlist_get(sl, 4); - test_streq(cp, "will"); - smartlist_add(sl, cp); - smartlist_remove(sl, cp); - tor_free(cp); - cp_alloc = smartlist_join_strings(sl, ",", 0, NULL); - test_streq(cp_alloc, "Some,say,the,Earth,fire,end,in,ice,and,some,in"); - tor_free(cp_alloc); - smartlist_string_remove(sl, "in"); - cp_alloc = smartlist_join_strings2(sl, "+XX", 1, 0, &sz); - test_streq(cp_alloc, "Some+say+the+Earth+fire+end+some+ice+and"); - test_eq((int)sz, 40); - - done: - - SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); - smartlist_free(sl); - tor_free(cp_alloc); -} - -/** Run unit tests for smartlist set manipulation functions. */ -static void -test_util_smartlist_overlap(void) -{ - smartlist_t *sl = smartlist_create(); - smartlist_t *ints = smartlist_create(); - smartlist_t *odds = smartlist_create(); - smartlist_t *evens = smartlist_create(); - smartlist_t *primes = smartlist_create(); - int i; - for (i=1; i < 10; i += 2) - smartlist_add(odds, (void*)(uintptr_t)i); - for (i=0; i < 10; i += 2) - smartlist_add(evens, (void*)(uintptr_t)i); - - /* add_all */ - smartlist_add_all(ints, odds); - smartlist_add_all(ints, evens); - test_eq(smartlist_len(ints), 10); - - smartlist_add(primes, (void*)2); - smartlist_add(primes, (void*)3); - smartlist_add(primes, (void*)5); - smartlist_add(primes, (void*)7); - - /* overlap */ - test_assert(smartlist_overlap(ints, odds)); - test_assert(smartlist_overlap(odds, primes)); - test_assert(smartlist_overlap(evens, primes)); - test_assert(!smartlist_overlap(odds, evens)); - - /* intersect */ - smartlist_add_all(sl, odds); - smartlist_intersect(sl, primes); - test_eq(smartlist_len(sl), 3); - test_assert(smartlist_isin(sl, (void*)3)); - test_assert(smartlist_isin(sl, (void*)5)); - test_assert(smartlist_isin(sl, (void*)7)); - - /* subtract */ - smartlist_add_all(sl, primes); - smartlist_subtract(sl, odds); - test_eq(smartlist_len(sl), 1); - test_assert(smartlist_isin(sl, (void*)2)); - - done: - smartlist_free(odds); - smartlist_free(evens); - smartlist_free(ints); - smartlist_free(primes); - smartlist_free(sl); -} - -/** Run unit tests for smartlist-of-digests functions. */ -static void -test_util_smartlist_digests(void) -{ - smartlist_t *sl = smartlist_create(); - - /* digest_isin. */ - smartlist_add(sl, tor_memdup("AAAAAAAAAAAAAAAAAAAA", DIGEST_LEN)); - smartlist_add(sl, tor_memdup("\00090AAB2AAAAaasdAAAAA", DIGEST_LEN)); - smartlist_add(sl, tor_memdup("\00090AAB2AAAAaasdAAAAA", DIGEST_LEN)); - test_eq(0, smartlist_digest_isin(NULL, "AAAAAAAAAAAAAAAAAAAA")); - test_assert(smartlist_digest_isin(sl, "AAAAAAAAAAAAAAAAAAAA")); - test_assert(smartlist_digest_isin(sl, "\00090AAB2AAAAaasdAAAAA")); - test_eq(0, smartlist_digest_isin(sl, "\00090AAB2AAABaasdAAAAA")); - - /* sort digests */ - smartlist_sort_digests(sl); - test_memeq(smartlist_get(sl, 0), "\00090AAB2AAAAaasdAAAAA", DIGEST_LEN); - test_memeq(smartlist_get(sl, 1), "\00090AAB2AAAAaasdAAAAA", DIGEST_LEN); - test_memeq(smartlist_get(sl, 2), "AAAAAAAAAAAAAAAAAAAA", DIGEST_LEN); - test_eq(3, smartlist_len(sl)); - - /* uniq_digests */ - smartlist_uniq_digests(sl); - test_eq(2, smartlist_len(sl)); - test_memeq(smartlist_get(sl, 0), "\00090AAB2AAAAaasdAAAAA", DIGEST_LEN); - test_memeq(smartlist_get(sl, 1), "AAAAAAAAAAAAAAAAAAAA", DIGEST_LEN); - - done: - SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); - smartlist_free(sl); -} - -/** Run unit tests for concatenate-a-smartlist-of-strings functions. */ -static void -test_util_smartlist_join(void) -{ - smartlist_t *sl = smartlist_create(); - smartlist_t *sl2 = smartlist_create(), *sl3 = smartlist_create(), - *sl4 = smartlist_create(); - char *joined=NULL; - /* unique, sorted. */ - smartlist_split_string(sl, - "Abashments Ambush Anchorman Bacon Banks Borscht " - "Bunks Inhumane Insurance Knish Know Manners " - "Maraschinos Stamina Sunbonnets Unicorns Wombats", - " ", 0, 0); - /* non-unique, sorted. */ - smartlist_split_string(sl2, - "Ambush Anchorman Anchorman Anemias Anemias Bacon " - "Crossbowmen Inhumane Insurance Knish Know Manners " - "Manners Maraschinos Wombats Wombats Work", - " ", 0, 0); - SMARTLIST_FOREACH_JOIN(sl, char *, cp1, - sl2, char *, cp2, - strcmp(cp1,cp2), - smartlist_add(sl3, cp2)) { - test_streq(cp1, cp2); - smartlist_add(sl4, cp1); - } SMARTLIST_FOREACH_JOIN_END(cp1, cp2); - - SMARTLIST_FOREACH(sl3, const char *, cp, - test_assert(smartlist_isin(sl2, cp) && - !smartlist_string_isin(sl, cp))); - SMARTLIST_FOREACH(sl4, const char *, cp, - test_assert(smartlist_isin(sl, cp) && - smartlist_string_isin(sl2, cp))); - joined = smartlist_join_strings(sl3, ",", 0, NULL); - test_streq(joined, "Anemias,Anemias,Crossbowmen,Work"); - tor_free(joined); - joined = smartlist_join_strings(sl4, ",", 0, NULL); - test_streq(joined, "Ambush,Anchorman,Anchorman,Bacon,Inhumane,Insurance," - "Knish,Know,Manners,Manners,Maraschinos,Wombats,Wombats"); - tor_free(joined); - - done: - smartlist_free(sl4); - smartlist_free(sl3); - SMARTLIST_FOREACH(sl2, char *, cp, tor_free(cp)); - smartlist_free(sl2); - SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); - smartlist_free(sl); - tor_free(joined); -} - -/** Run unit tests for bitarray code */ -static void -test_util_bitarray(void) -{ - bitarray_t *ba = NULL; - int i, j, ok=1; - - ba = bitarray_init_zero(1); - test_assert(ba); - test_assert(! bitarray_is_set(ba, 0)); - bitarray_set(ba, 0); - test_assert(bitarray_is_set(ba, 0)); - bitarray_clear(ba, 0); - test_assert(! bitarray_is_set(ba, 0)); - bitarray_free(ba); - - ba = bitarray_init_zero(1023); - for (i = 1; i < 64; ) { - for (j = 0; j < 1023; ++j) { - if (j % i) - bitarray_set(ba, j); - else - bitarray_clear(ba, j); - } - for (j = 0; j < 1023; ++j) { - if (!bool_eq(bitarray_is_set(ba, j), j%i)) - ok = 0; - } - test_assert(ok); - if (i < 7) - ++i; - else if (i == 28) - i = 32; - else - i += 7; - } - - done: - if (ba) - bitarray_free(ba); -} - -/** Run unit tests for digest set code (implemented as a hashtable or as a - * bloom filter) */ -static void -test_util_digestset(void) -{ - smartlist_t *included = smartlist_create(); - char d[DIGEST_LEN]; - int i; - int ok = 1; - int false_positives = 0; - digestset_t *set = NULL; - - for (i = 0; i < 1000; ++i) { - crypto_rand(d, DIGEST_LEN); - smartlist_add(included, tor_memdup(d, DIGEST_LEN)); - } - set = digestset_new(1000); - SMARTLIST_FOREACH(included, const char *, cp, - if (digestset_isin(set, cp)) - ok = 0); - test_assert(ok); - SMARTLIST_FOREACH(included, const char *, cp, - digestset_add(set, cp)); - SMARTLIST_FOREACH(included, const char *, cp, - if (!digestset_isin(set, cp)) - ok = 0); - test_assert(ok); - for (i = 0; i < 1000; ++i) { - crypto_rand(d, DIGEST_LEN); - if (digestset_isin(set, d)) - ++false_positives; - } - test_assert(false_positives < 50); /* Should be far lower. */ - - done: - if (set) - digestset_free(set); - SMARTLIST_FOREACH(included, char *, cp, tor_free(cp)); - smartlist_free(included); -} - -/** mutex for thread test to stop the threads hitting data at the same time. */ -static tor_mutex_t *_thread_test_mutex = NULL; -/** mutexes for the thread test to make sure that the threads have to - * interleave somewhat. */ -static tor_mutex_t *_thread_test_start1 = NULL, - *_thread_test_start2 = NULL; -/** Shared strmap for the thread test. */ -static strmap_t *_thread_test_strmap = NULL; -/** The name of thread1 for the thread test */ -static char *_thread1_name = NULL; -/** The name of thread2 for the thread test */ -static char *_thread2_name = NULL; - -static void _thread_test_func(void* _s) ATTR_NORETURN; - -/** How many iterations have the threads in the unit test run? */ -static int t1_count = 0, t2_count = 0; - -/** Helper function for threading unit tests: This function runs in a - * subthread. It grabs its own mutex (start1 or start2) to make sure that it - * should start, then it repeatedly alters _test_thread_strmap protected by - * _thread_test_mutex. */ -static void -_thread_test_func(void* _s) -{ - char *s = _s; - int i, *count; - tor_mutex_t *m; - char buf[64]; - char **cp; - if (!strcmp(s, "thread 1")) { - m = _thread_test_start1; - cp = &_thread1_name; - count = &t1_count; - } else { - m = _thread_test_start2; - cp = &_thread2_name; - count = &t2_count; - } - tor_mutex_acquire(m); - - tor_snprintf(buf, sizeof(buf), "%lu", tor_get_thread_id()); - *cp = tor_strdup(buf); - - for (i=0; i<10000; ++i) { - tor_mutex_acquire(_thread_test_mutex); - strmap_set(_thread_test_strmap, "last to run", *cp); - ++*count; - tor_mutex_release(_thread_test_mutex); - } - tor_mutex_acquire(_thread_test_mutex); - strmap_set(_thread_test_strmap, s, *cp); - tor_mutex_release(_thread_test_mutex); - - tor_mutex_release(m); - - spawn_exit(); -} - -/** Run unit tests for threading logic. */ -static void -test_util_threads(void) -{ - char *s1 = NULL, *s2 = NULL; - int done = 0, timedout = 0; - time_t started; -#ifndef MS_WINDOWS - struct timeval tv; - tv.tv_sec=0; - tv.tv_usec=10; -#endif -#ifndef TOR_IS_MULTITHREADED - /* Skip this test if we aren't threading. We should be threading most - * everywhere by now. */ - if (1) - return; -#endif - _thread_test_mutex = tor_mutex_new(); - _thread_test_start1 = tor_mutex_new(); - _thread_test_start2 = tor_mutex_new(); - _thread_test_strmap = strmap_new(); - s1 = tor_strdup("thread 1"); - s2 = tor_strdup("thread 2"); - tor_mutex_acquire(_thread_test_start1); - tor_mutex_acquire(_thread_test_start2); - spawn_func(_thread_test_func, s1); - spawn_func(_thread_test_func, s2); - tor_mutex_release(_thread_test_start2); - tor_mutex_release(_thread_test_start1); - started = time(NULL); - while (!done) { - tor_mutex_acquire(_thread_test_mutex); - strmap_assert_ok(_thread_test_strmap); - if (strmap_get(_thread_test_strmap, "thread 1") && - strmap_get(_thread_test_strmap, "thread 2")) { - done = 1; - } else if (time(NULL) > started + 25) { - timedout = done = 1; - } - tor_mutex_release(_thread_test_mutex); -#ifndef MS_WINDOWS - /* Prevent the main thread from starving the worker threads. */ - select(0, NULL, NULL, NULL, &tv); -#endif - } - - tor_mutex_acquire(_thread_test_start1); - tor_mutex_release(_thread_test_start1); - tor_mutex_acquire(_thread_test_start2); - tor_mutex_release(_thread_test_start2); - - tor_mutex_free(_thread_test_mutex); - - if (timedout) { - printf("\nTimed out: %d %d", t1_count, t2_count); - test_assert(strmap_get(_thread_test_strmap, "thread 1")); - test_assert(strmap_get(_thread_test_strmap, "thread 2")); - test_assert(!timedout); - } - - /* different thread IDs. */ - test_assert(strcmp(strmap_get(_thread_test_strmap, "thread 1"), - strmap_get(_thread_test_strmap, "thread 2"))); - test_assert(!strcmp(strmap_get(_thread_test_strmap, "thread 1"), - strmap_get(_thread_test_strmap, "last to run")) || - !strcmp(strmap_get(_thread_test_strmap, "thread 2"), - strmap_get(_thread_test_strmap, "last to run"))); - - done: - tor_free(s1); - tor_free(s2); - tor_free(_thread1_name); - tor_free(_thread2_name); - if (_thread_test_strmap) - strmap_free(_thread_test_strmap, NULL); - if (_thread_test_start1) - tor_mutex_free(_thread_test_start1); - if (_thread_test_start2) - tor_mutex_free(_thread_test_start2); -} - -/** Helper: return a tristate based on comparing two strings. */ -static int -_compare_strings_for_pqueue(const void *s1, const void *s2) -{ - return strcmp((const char*)s1, (const char*)s2); -} - -/** Run unit tests for heap-based priority queue functions. */ -static void -test_util_pqueue(void) -{ - smartlist_t *sl = smartlist_create(); - int (*cmp)(const void *, const void*); -#define OK() smartlist_pqueue_assert_ok(sl, cmp) - - cmp = _compare_strings_for_pqueue; - - smartlist_pqueue_add(sl, cmp, (char*)"cows"); - smartlist_pqueue_add(sl, cmp, (char*)"zebras"); - smartlist_pqueue_add(sl, cmp, (char*)"fish"); - smartlist_pqueue_add(sl, cmp, (char*)"frogs"); - smartlist_pqueue_add(sl, cmp, (char*)"apples"); - smartlist_pqueue_add(sl, cmp, (char*)"squid"); - smartlist_pqueue_add(sl, cmp, (char*)"daschunds"); - smartlist_pqueue_add(sl, cmp, (char*)"eggplants"); - smartlist_pqueue_add(sl, cmp, (char*)"weissbier"); - smartlist_pqueue_add(sl, cmp, (char*)"lobsters"); - smartlist_pqueue_add(sl, cmp, (char*)"roquefort"); - - OK(); - - test_eq(smartlist_len(sl), 11); - test_streq(smartlist_get(sl, 0), "apples"); - test_streq(smartlist_pqueue_pop(sl, cmp), "apples"); - test_eq(smartlist_len(sl), 10); - OK(); - test_streq(smartlist_pqueue_pop(sl, cmp), "cows"); - test_streq(smartlist_pqueue_pop(sl, cmp), "daschunds"); - smartlist_pqueue_add(sl, cmp, (char*)"chinchillas"); - OK(); - smartlist_pqueue_add(sl, cmp, (char*)"fireflies"); - OK(); - test_streq(smartlist_pqueue_pop(sl, cmp), "chinchillas"); - test_streq(smartlist_pqueue_pop(sl, cmp), "eggplants"); - test_streq(smartlist_pqueue_pop(sl, cmp), "fireflies"); - OK(); - test_streq(smartlist_pqueue_pop(sl, cmp), "fish"); - test_streq(smartlist_pqueue_pop(sl, cmp), "frogs"); - test_streq(smartlist_pqueue_pop(sl, cmp), "lobsters"); - test_streq(smartlist_pqueue_pop(sl, cmp), "roquefort"); - OK(); - test_eq(smartlist_len(sl), 3); - test_streq(smartlist_pqueue_pop(sl, cmp), "squid"); - test_streq(smartlist_pqueue_pop(sl, cmp), "weissbier"); - test_streq(smartlist_pqueue_pop(sl, cmp), "zebras"); - test_eq(smartlist_len(sl), 0); - OK(); -#undef OK - - done: - - smartlist_free(sl); -} - -/** Run unit tests for compression functions */ -static void -test_util_gzip(void) -{ - char *buf1=NULL, *buf2=NULL, *buf3=NULL, *cp1, *cp2; - const char *ccp2; - size_t len1, len2; - tor_zlib_state_t *state = NULL; - - buf1 = tor_strdup("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZAAAAAAAAAAAAAAAAAAAZ"); - test_assert(detect_compression_method(buf1, strlen(buf1)) == UNKNOWN_METHOD); - if (is_gzip_supported()) { - test_assert(!tor_gzip_compress(&buf2, &len1, buf1, strlen(buf1)+1, - GZIP_METHOD)); - test_assert(buf2); - test_assert(!memcmp(buf2, "\037\213", 2)); /* Gzip magic. */ - test_assert(detect_compression_method(buf2, len1) == GZIP_METHOD); - - test_assert(!tor_gzip_uncompress(&buf3, &len2, buf2, len1, - GZIP_METHOD, 1, LOG_INFO)); - test_assert(buf3); - test_streq(buf1,buf3); - - tor_free(buf2); - tor_free(buf3); - } - - test_assert(!tor_gzip_compress(&buf2, &len1, buf1, strlen(buf1)+1, - ZLIB_METHOD)); - test_assert(buf2); - test_assert(!memcmp(buf2, "\x78\xDA", 2)); /* deflate magic. */ - test_assert(detect_compression_method(buf2, len1) == ZLIB_METHOD); - - test_assert(!tor_gzip_uncompress(&buf3, &len2, buf2, len1, - ZLIB_METHOD, 1, LOG_INFO)); - test_assert(buf3); - test_streq(buf1,buf3); - - /* Check whether we can uncompress concatenated, compressed strings. */ - tor_free(buf3); - buf2 = tor_realloc(buf2, len1*2); - memcpy(buf2+len1, buf2, len1); - test_assert(!tor_gzip_uncompress(&buf3, &len2, buf2, len1*2, - ZLIB_METHOD, 1, LOG_INFO)); - test_eq(len2, (strlen(buf1)+1)*2); - test_memeq(buf3, - "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZAAAAAAAAAAAAAAAAAAAZ\0" - "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZAAAAAAAAAAAAAAAAAAAZ\0", - (strlen(buf1)+1)*2); - - tor_free(buf1); - tor_free(buf2); - tor_free(buf3); - - /* Check whether we can uncompress partial strings. */ - buf1 = - tor_strdup("String with low redundancy that won't be compressed much."); - test_assert(!tor_gzip_compress(&buf2, &len1, buf1, strlen(buf1)+1, - ZLIB_METHOD)); - tor_assert(len1>16); - /* when we allow an incomplete string, we should succeed.*/ - tor_assert(!tor_gzip_uncompress(&buf3, &len2, buf2, len1-16, - ZLIB_METHOD, 0, LOG_INFO)); - buf3[len2]='\0'; - tor_assert(len2 > 5); - tor_assert(!strcmpstart(buf1, buf3)); - - /* when we demand a complete string, this must fail. */ - tor_free(buf3); - tor_assert(tor_gzip_uncompress(&buf3, &len2, buf2, len1-16, - ZLIB_METHOD, 1, LOG_INFO)); - tor_assert(!buf3); - - /* Now, try streaming compression. */ - tor_free(buf1); - tor_free(buf2); - tor_free(buf3); - state = tor_zlib_new(1, ZLIB_METHOD); - tor_assert(state); - cp1 = buf1 = tor_malloc(1024); - len1 = 1024; - ccp2 = "ABCDEFGHIJABCDEFGHIJ"; - len2 = 21; - test_assert(tor_zlib_process(state, &cp1, &len1, &ccp2, &len2, 0) - == TOR_ZLIB_OK); - test_eq(len2, 0); /* Make sure we compressed it all. */ - test_assert(cp1 > buf1); - - len2 = 0; - cp2 = cp1; - test_assert(tor_zlib_process(state, &cp1, &len1, &ccp2, &len2, 1) - == TOR_ZLIB_DONE); - test_eq(len2, 0); - test_assert(cp1 > cp2); /* Make sure we really added something. */ - - tor_assert(!tor_gzip_uncompress(&buf3, &len2, buf1, 1024-len1, - ZLIB_METHOD, 1, LOG_WARN)); - test_streq(buf3, "ABCDEFGHIJABCDEFGHIJ"); /*Make sure it compressed right.*/ - - done: - if (state) - tor_zlib_free(state); - tor_free(buf2); - tor_free(buf3); - tor_free(buf1); -} - -/** Run unit tests for string-to-void* map functions */ -static void -test_util_strmap(void) -{ - strmap_t *map; - strmap_iter_t *iter; - const char *k; - void *v; - char *visited = NULL; - smartlist_t *found_keys = NULL; - - map = strmap_new(); - test_assert(map); - test_eq(strmap_size(map), 0); - test_assert(strmap_isempty(map)); - v = strmap_set(map, "K1", (void*)99); - test_eq(v, NULL); - test_assert(!strmap_isempty(map)); - v = strmap_set(map, "K2", (void*)101); - test_eq(v, NULL); - v = strmap_set(map, "K1", (void*)100); - test_eq(v, (void*)99); - test_eq_ptr(strmap_get(map,"K1"), (void*)100); - test_eq_ptr(strmap_get(map,"K2"), (void*)101); - test_eq_ptr(strmap_get(map,"K-not-there"), NULL); - strmap_assert_ok(map); - - v = strmap_remove(map,"K2"); - strmap_assert_ok(map); - test_eq_ptr(v, (void*)101); - test_eq_ptr(strmap_get(map,"K2"), NULL); - test_eq_ptr(strmap_remove(map,"K2"), NULL); - - strmap_set(map, "K2", (void*)101); - strmap_set(map, "K3", (void*)102); - strmap_set(map, "K4", (void*)103); - test_eq(strmap_size(map), 4); - strmap_assert_ok(map); - strmap_set(map, "K5", (void*)104); - strmap_set(map, "K6", (void*)105); - strmap_assert_ok(map); - - /* Test iterator. */ - iter = strmap_iter_init(map); - found_keys = smartlist_create(); - while (!strmap_iter_done(iter)) { - strmap_iter_get(iter,&k,&v); - smartlist_add(found_keys, tor_strdup(k)); - test_eq_ptr(v, strmap_get(map, k)); - - if (!strcmp(k, "K2")) { - iter = strmap_iter_next_rmv(map,iter); - } else { - iter = strmap_iter_next(map,iter); - } - } - - /* Make sure we removed K2, but not the others. */ - test_eq_ptr(strmap_get(map, "K2"), NULL); - test_eq_ptr(strmap_get(map, "K5"), (void*)104); - /* Make sure we visited everyone once */ - smartlist_sort_strings(found_keys); - visited = smartlist_join_strings(found_keys, ":", 0, NULL); - test_streq(visited, "K1:K2:K3:K4:K5:K6"); - - strmap_assert_ok(map); - /* Clean up after ourselves. */ - strmap_free(map, NULL); - map = NULL; - - /* Now try some lc functions. */ - map = strmap_new(); - strmap_set_lc(map,"Ab.C", (void*)1); - test_eq_ptr(strmap_get(map,"ab.c"), (void*)1); - strmap_assert_ok(map); - test_eq_ptr(strmap_get_lc(map,"AB.C"), (void*)1); - test_eq_ptr(strmap_get(map,"AB.C"), NULL); - test_eq_ptr(strmap_remove_lc(map,"aB.C"), (void*)1); - strmap_assert_ok(map); - test_eq_ptr(strmap_get_lc(map,"AB.C"), NULL); - - done: - if (map) - strmap_free(map,NULL); - if (found_keys) { - SMARTLIST_FOREACH(found_keys, char *, cp, tor_free(cp)); - smartlist_free(found_keys); - } - tor_free(visited); -} - -/** Run unit tests for mmap() wrapper functionality. */ -static void -test_util_mmap(void) -{ - char *fname1 = tor_strdup(get_fname("mapped_1")); - char *fname2 = tor_strdup(get_fname("mapped_2")); - char *fname3 = tor_strdup(get_fname("mapped_3")); - const size_t buflen = 17000; - char *buf = tor_malloc(17000); - tor_mmap_t *mapping = NULL; - - crypto_rand(buf, buflen); - - mapping = tor_mmap_file(fname1); - test_assert(! mapping); - - write_str_to_file(fname1, "Short file.", 1); - write_bytes_to_file(fname2, buf, buflen, 1); - write_bytes_to_file(fname3, buf, 16384, 1); - - mapping = tor_mmap_file(fname1); - test_assert(mapping); - test_eq(mapping->size, strlen("Short file.")); - test_streq(mapping->data, "Short file."); -#ifdef MS_WINDOWS - tor_munmap_file(mapping); - mapping = NULL; - test_assert(unlink(fname1) == 0); -#else - /* make sure we can unlink. */ - test_assert(unlink(fname1) == 0); - test_streq(mapping->data, "Short file."); - tor_munmap_file(mapping); - mapping = NULL; -#endif - - /* Now a zero-length file. */ - write_str_to_file(fname1, "", 1); - mapping = tor_mmap_file(fname1); - test_eq(mapping, NULL); - test_eq(ERANGE, errno); - unlink(fname1); - - /* Make sure that we fail to map a no-longer-existent file. */ - mapping = tor_mmap_file(fname1); - test_assert(mapping == NULL); - - /* Now try a big file that stretches across a few pages and isn't aligned */ - mapping = tor_mmap_file(fname2); - test_assert(mapping); - test_eq(mapping->size, buflen); - test_memeq(mapping->data, buf, buflen); - tor_munmap_file(mapping); - mapping = NULL; - - /* Now try a big aligned file. */ - mapping = tor_mmap_file(fname3); - test_assert(mapping); - test_eq(mapping->size, 16384); - test_memeq(mapping->data, buf, 16384); - tor_munmap_file(mapping); - mapping = NULL; - - done: - unlink(fname1); - unlink(fname2); - unlink(fname3); - - tor_free(fname1); - tor_free(fname2); - tor_free(fname3); - tor_free(buf); - - if (mapping) - tor_munmap_file(mapping); -} - -/** Run unit tests for escaping/unescaping data for use by controllers. */ -static void -test_util_control_formats(void) -{ - char *out = NULL; - const char *inp = - "..This is a test\r\nof the emergency \nbroadcast\r\n..system.\r\nZ.\r\n"; - size_t sz; - - sz = read_escaped_data(inp, strlen(inp), &out); - test_streq(out, - ".This is a test\nof the emergency \nbroadcast\n.system.\nZ.\n"); - test_eq(sz, strlen(out)); - - done: - tor_free(out); -} - -static void -test_util_sscanf(void) -{ - unsigned u1, u2, u3; - char s1[10], s2[10], s3[10], ch; - int r; - - r = tor_sscanf("hello world", "hello world"); /* String match: success */ - test_eq(r, 0); - r = tor_sscanf("hello world 3", "hello worlb %u", &u1); /* String fail */ - test_eq(r, 0); - r = tor_sscanf("12345", "%u", &u1); /* Simple number */ - test_eq(r, 1); - test_eq(u1, 12345u); - r = tor_sscanf("", "%u", &u1); /* absent number */ - test_eq(r, 0); - r = tor_sscanf("A", "%u", &u1); /* bogus number */ - test_eq(r, 0); - r = tor_sscanf("4294967295", "%u", &u1); /* UINT32_MAX should work. */ - test_eq(r, 1); - test_eq(u1, 4294967295u); - r = tor_sscanf("4294967296", "%u", &u1); /* Always say -1 at 32 bits. */ - test_eq(r, 0); - r = tor_sscanf("123456", "%2u%u", &u1, &u2); /* Width */ - test_eq(r, 2); - test_eq(u1, 12u); - test_eq(u2, 3456u); - r = tor_sscanf("!12:3:456", "!%2u:%2u:%3u", &u1, &u2, &u3); /* separators */ - test_eq(r, 3); - test_eq(u1, 12u); - test_eq(u2, 3u); - test_eq(u3, 456u); - r = tor_sscanf("12:3:045", "%2u:%2u:%3u", &u1, &u2, &u3); /* 0s */ - test_eq(r, 3); - test_eq(u1, 12u); - test_eq(u2, 3u); - test_eq(u3, 45u); - /* %u does not match space.*/ - r = tor_sscanf("12:3: 45", "%2u:%2u:%3u", &u1, &u2, &u3); - test_eq(r, 2); - /* %u does not match negative numbers. */ - r = tor_sscanf("12:3:-4", "%2u:%2u:%3u", &u1, &u2, &u3); - test_eq(r, 2); - /* Arbitrary amounts of 0-padding are okay */ - r = tor_sscanf("12:03:000000000000000099", "%2u:%2u:%u", &u1, &u2, &u3); - test_eq(r, 3); - test_eq(u1, 12u); - test_eq(u2, 3u); - test_eq(u3, 99u); - - r = tor_sscanf("99% fresh", "%3u%% fresh", &u1); /* percents are scannable.*/ - test_eq(r, 1); - test_eq(u1, 99); - - r = tor_sscanf("hello", "%s", s1); /* %s needs a number. */ - test_eq(r, -1); - - r = tor_sscanf("hello", "%3s%7s", s1, s2); /* %s matches characters. */ - test_eq(r, 2); - test_streq(s1, "hel"); - test_streq(s2, "lo"); - r = tor_sscanf("WD40", "%2s%u", s3, &u1); /* %s%u */ - test_eq(r, 2); - test_streq(s3, "WD"); - test_eq(u1, 40); - r = tor_sscanf("76trombones", "%6u%9s", &u1, s1); /* %u%s */ - test_eq(r, 2); - test_eq(u1, 76); - test_streq(s1, "trombones"); - r = tor_sscanf("hello world", "%9s %9s", s1, s2); /* %s doesn't eat space. */ - test_eq(r, 2); - test_streq(s1, "hello"); - test_streq(s2, "world"); - r = tor_sscanf("hi", "%9s%9s%3s", s1, s2, s3); /* %s can be empty. */ - test_eq(r, 3); - test_streq(s1, "hi"); - test_streq(s2, ""); - test_streq(s3, ""); - - r = tor_sscanf("1.2.3", "%u.%u.%u%c", &u1, &u2, &u3, &ch); - test_eq(r, 3); - r = tor_sscanf("1.2.3 foobar", "%u.%u.%u%c", &u1, &u2, &u3, &ch); - test_eq(r, 4); - - done: - ; -} - -/** Run unit tests for the onion handshake code. */ -static void -test_onion_handshake(void) -{ - /* client-side */ - crypto_dh_env_t *c_dh = NULL; - char c_buf[ONIONSKIN_CHALLENGE_LEN]; - char c_keys[40]; - - /* server-side */ - char s_buf[ONIONSKIN_REPLY_LEN]; - char s_keys[40]; - - /* shared */ - crypto_pk_env_t *pk = NULL; - - pk = pk_generate(0); - - /* client handshake 1. */ - memset(c_buf, 0, ONIONSKIN_CHALLENGE_LEN); - test_assert(! onion_skin_create(pk, &c_dh, c_buf)); - - /* server handshake */ - memset(s_buf, 0, ONIONSKIN_REPLY_LEN); - memset(s_keys, 0, 40); - test_assert(! onion_skin_server_handshake(c_buf, pk, NULL, - s_buf, s_keys, 40)); - - /* client handshake 2 */ - memset(c_keys, 0, 40); - test_assert(! onion_skin_client_handshake(c_dh, s_buf, c_keys, 40)); - - if (memcmp(c_keys, s_keys, 40)) { - puts("Aiiiie"); - exit(1); - } - test_memeq(c_keys, s_keys, 40); - memset(s_buf, 0, 40); - test_memneq(c_keys, s_buf, 40); - - done: - if (c_dh) - crypto_dh_free(c_dh); - if (pk) - crypto_free_pk_env(pk); -} - -/** Run unit tests for router descriptor generation logic. */ -static void -test_dir_format(void) -{ - char buf[8192], buf2[8192]; - char platform[256]; - char fingerprint[FINGERPRINT_LEN+1]; - char *pk1_str = NULL, *pk2_str = NULL, *pk3_str = NULL, *cp; - size_t pk1_str_len, pk2_str_len, pk3_str_len; - routerinfo_t *r1=NULL, *r2=NULL; - crypto_pk_env_t *pk1 = NULL, *pk2 = NULL, *pk3 = NULL; - routerinfo_t *rp1 = NULL; - addr_policy_t *ex1, *ex2; - routerlist_t *dir1 = NULL, *dir2 = NULL; - tor_version_t ver1; - - pk1 = pk_generate(0); - pk2 = pk_generate(1); - pk3 = pk_generate(2); - - test_assert( is_legal_nickname("a")); - test_assert(!is_legal_nickname("")); - test_assert(!is_legal_nickname("abcdefghijklmnopqrst")); /* 20 chars */ - test_assert(!is_legal_nickname("hyphen-")); /* bad char */ - test_assert( is_legal_nickname("abcdefghijklmnopqrs")); /* 19 chars */ - test_assert(!is_legal_nickname("$AAAAAAAA01234AAAAAAAAAAAAAAAAAAAAAAAAAAA")); - /* valid */ - test_assert( is_legal_nickname_or_hexdigest( - "$AAAAAAAA01234AAAAAAAAAAAAAAAAAAAAAAAAAAA")); - test_assert( is_legal_nickname_or_hexdigest( - "$AAAAAAAA01234AAAAAAAAAAAAAAAAAAAAAAAAAAA=fred")); - test_assert( is_legal_nickname_or_hexdigest( - "$AAAAAAAA01234AAAAAAAAAAAAAAAAAAAAAAAAAAA~fred")); - /* too short */ - test_assert(!is_legal_nickname_or_hexdigest( - "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")); - /* illegal char */ - test_assert(!is_legal_nickname_or_hexdigest( - "$AAAAAAzAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")); - /* hex part too long */ - test_assert(!is_legal_nickname_or_hexdigest( - "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")); - test_assert(!is_legal_nickname_or_hexdigest( - "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=fred")); - /* Bad nickname */ - test_assert(!is_legal_nickname_or_hexdigest( - "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=")); - test_assert(!is_legal_nickname_or_hexdigest( - "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~")); - test_assert(!is_legal_nickname_or_hexdigest( - "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~hyphen-")); - test_assert(!is_legal_nickname_or_hexdigest( - "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~" - "abcdefghijklmnoppqrst")); - /* Bad extra char. */ - test_assert(!is_legal_nickname_or_hexdigest( - "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA!")); - test_assert(is_legal_nickname_or_hexdigest("xyzzy")); - test_assert(is_legal_nickname_or_hexdigest("abcdefghijklmnopqrs")); - test_assert(!is_legal_nickname_or_hexdigest("abcdefghijklmnopqrst")); - - get_platform_str(platform, sizeof(platform)); - r1 = tor_malloc_zero(sizeof(routerinfo_t)); - r1->address = tor_strdup("18.244.0.1"); - r1->addr = 0xc0a80001u; /* 192.168.0.1 */ - r1->cache_info.published_on = 0; - r1->or_port = 9000; - r1->dir_port = 9003; - r1->onion_pkey = crypto_pk_dup_key(pk1); - r1->identity_pkey = crypto_pk_dup_key(pk2); - r1->bandwidthrate = 1000; - r1->bandwidthburst = 5000; - r1->bandwidthcapacity = 10000; - r1->exit_policy = NULL; - r1->nickname = tor_strdup("Magri"); - r1->platform = tor_strdup(platform); - - ex1 = tor_malloc_zero(sizeof(addr_policy_t)); - ex2 = tor_malloc_zero(sizeof(addr_policy_t)); - ex1->policy_type = ADDR_POLICY_ACCEPT; - tor_addr_from_ipv4h(&ex1->addr, 0); - ex1->maskbits = 0; - ex1->prt_min = ex1->prt_max = 80; - ex2->policy_type = ADDR_POLICY_REJECT; - tor_addr_from_ipv4h(&ex2->addr, 18<<24); - ex2->maskbits = 8; - ex2->prt_min = ex2->prt_max = 24; - r2 = tor_malloc_zero(sizeof(routerinfo_t)); - r2->address = tor_strdup("1.1.1.1"); - r2->addr = 0x0a030201u; /* 10.3.2.1 */ - r2->platform = tor_strdup(platform); - r2->cache_info.published_on = 5; - r2->or_port = 9005; - r2->dir_port = 0; - r2->onion_pkey = crypto_pk_dup_key(pk2); - r2->identity_pkey = crypto_pk_dup_key(pk1); - r2->bandwidthrate = r2->bandwidthburst = r2->bandwidthcapacity = 3000; - r2->exit_policy = smartlist_create(); - smartlist_add(r2->exit_policy, ex2); - smartlist_add(r2->exit_policy, ex1); - r2->nickname = tor_strdup("Fred"); - - test_assert(!crypto_pk_write_public_key_to_string(pk1, &pk1_str, - &pk1_str_len)); - test_assert(!crypto_pk_write_public_key_to_string(pk2 , &pk2_str, - &pk2_str_len)); - test_assert(!crypto_pk_write_public_key_to_string(pk3 , &pk3_str, - &pk3_str_len)); - - memset(buf, 0, 2048); - test_assert(router_dump_router_to_string(buf, 2048, r1, pk2)>0); - - strlcpy(buf2, "router Magri 18.244.0.1 9000 0 9003\n" - "platform Tor "VERSION" on ", sizeof(buf2)); - strlcat(buf2, get_uname(), sizeof(buf2)); - strlcat(buf2, "\n" - "opt protocols Link 1 2 Circuit 1\n" - "published 1970-01-01 00:00:00\n" - "opt fingerprint ", sizeof(buf2)); - test_assert(!crypto_pk_get_fingerprint(pk2, fingerprint, 1)); - strlcat(buf2, fingerprint, sizeof(buf2)); - strlcat(buf2, "\nuptime 0\n" - /* XXX the "0" above is hard-coded, but even if we made it reflect - * uptime, that still wouldn't make it right, because the two - * descriptors might be made on different seconds... hm. */ - "bandwidth 1000 5000 10000\n" - "opt extra-info-digest 0000000000000000000000000000000000000000\n" - "onion-key\n", sizeof(buf2)); - strlcat(buf2, pk1_str, sizeof(buf2)); - strlcat(buf2, "signing-key\n", sizeof(buf2)); - strlcat(buf2, pk2_str, sizeof(buf2)); - strlcat(buf2, "opt hidden-service-dir\n", sizeof(buf2)); - strlcat(buf2, "reject *:*\nrouter-signature\n", sizeof(buf2)); - buf[strlen(buf2)] = '\0'; /* Don't compare the sig; it's never the same - * twice */ - - test_streq(buf, buf2); - - test_assert(router_dump_router_to_string(buf, 2048, r1, pk2)>0); - cp = buf; - rp1 = router_parse_entry_from_string((const char*)cp,NULL,1,0,NULL); - test_assert(rp1); - test_streq(rp1->address, r1->address); - test_eq(rp1->or_port, r1->or_port); - //test_eq(rp1->dir_port, r1->dir_port); - test_eq(rp1->bandwidthrate, r1->bandwidthrate); - test_eq(rp1->bandwidthburst, r1->bandwidthburst); - test_eq(rp1->bandwidthcapacity, r1->bandwidthcapacity); - test_assert(crypto_pk_cmp_keys(rp1->onion_pkey, pk1) == 0); - test_assert(crypto_pk_cmp_keys(rp1->identity_pkey, pk2) == 0); - //test_assert(rp1->exit_policy == NULL); - -#if 0 - /* XXX Once we have exit policies, test this again. XXX */ - strlcpy(buf2, "router tor.tor.tor 9005 0 0 3000\n", sizeof(buf2)); - strlcat(buf2, pk2_str, sizeof(buf2)); - strlcat(buf2, "signing-key\n", sizeof(buf2)); - strlcat(buf2, pk1_str, sizeof(buf2)); - strlcat(buf2, "accept *:80\nreject 18.*:24\n\n", sizeof(buf2)); - test_assert(router_dump_router_to_string(buf, 2048, &r2, pk2)>0); - test_streq(buf, buf2); - - cp = buf; - rp2 = router_parse_entry_from_string(&cp,1); - test_assert(rp2); - test_streq(rp2->address, r2.address); - test_eq(rp2->or_port, r2.or_port); - test_eq(rp2->dir_port, r2.dir_port); - test_eq(rp2->bandwidth, r2.bandwidth); - test_assert(crypto_pk_cmp_keys(rp2->onion_pkey, pk2) == 0); - test_assert(crypto_pk_cmp_keys(rp2->identity_pkey, pk1) == 0); - test_eq(rp2->exit_policy->policy_type, EXIT_POLICY_ACCEPT); - test_streq(rp2->exit_policy->string, "accept *:80"); - test_streq(rp2->exit_policy->address, "*"); - test_streq(rp2->exit_policy->port, "80"); - test_eq(rp2->exit_policy->next->policy_type, EXIT_POLICY_REJECT); - test_streq(rp2->exit_policy->next->string, "reject 18.*:24"); - test_streq(rp2->exit_policy->next->address, "18.*"); - test_streq(rp2->exit_policy->next->port, "24"); - test_assert(rp2->exit_policy->next->next == NULL); - - /* Okay, now for the directories. */ - { - fingerprint_list = smartlist_create(); - crypto_pk_get_fingerprint(pk2, buf, 1); - add_fingerprint_to_dir("Magri", buf, fingerprint_list); - crypto_pk_get_fingerprint(pk1, buf, 1); - add_fingerprint_to_dir("Fred", buf, fingerprint_list); - } - - { - char d[DIGEST_LEN]; - const char *m; - /* XXXX NM re-enable. */ - /* Make sure routers aren't too far in the past any more. */ - r1->cache_info.published_on = time(NULL); - r2->cache_info.published_on = time(NULL)-3*60*60; - test_assert(router_dump_router_to_string(buf, 2048, r1, pk2)>0); - test_eq(dirserv_add_descriptor(buf,&m,""), ROUTER_ADDED_NOTIFY_GENERATOR); - test_assert(router_dump_router_to_string(buf, 2048, r2, pk1)>0); - test_eq(dirserv_add_descriptor(buf,&m,""), ROUTER_ADDED_NOTIFY_GENERATOR); - get_options()->Nickname = tor_strdup("DirServer"); - test_assert(!dirserv_dump_directory_to_string(&cp,pk3, 0)); - crypto_pk_get_digest(pk3, d); - test_assert(!router_parse_directory(cp)); - test_eq(2, smartlist_len(dir1->routers)); - tor_free(cp); - } -#endif - dirserv_free_fingerprint_list(); - - /* Try out version parsing functionality */ - test_eq(0, tor_version_parse("0.3.4pre2-cvs", &ver1)); - test_eq(0, ver1.major); - test_eq(3, ver1.minor); - test_eq(4, ver1.micro); - test_eq(VER_PRE, ver1.status); - test_eq(2, ver1.patchlevel); - test_eq(0, tor_version_parse("0.3.4rc1", &ver1)); - test_eq(0, ver1.major); - test_eq(3, ver1.minor); - test_eq(4, ver1.micro); - test_eq(VER_RC, ver1.status); - test_eq(1, ver1.patchlevel); - test_eq(0, tor_version_parse("1.3.4", &ver1)); - test_eq(1, ver1.major); - test_eq(3, ver1.minor); - test_eq(4, ver1.micro); - test_eq(VER_RELEASE, ver1.status); - test_eq(0, ver1.patchlevel); - test_eq(0, tor_version_parse("1.3.4.999", &ver1)); - test_eq(1, ver1.major); - test_eq(3, ver1.minor); - test_eq(4, ver1.micro); - test_eq(VER_RELEASE, ver1.status); - test_eq(999, ver1.patchlevel); - test_eq(0, tor_version_parse("0.1.2.4-alpha", &ver1)); - test_eq(0, ver1.major); - test_eq(1, ver1.minor); - test_eq(2, ver1.micro); - test_eq(4, ver1.patchlevel); - test_eq(VER_RELEASE, ver1.status); - test_streq("alpha", ver1.status_tag); - test_eq(0, tor_version_parse("0.1.2.4", &ver1)); - test_eq(0, ver1.major); - test_eq(1, ver1.minor); - test_eq(2, ver1.micro); - test_eq(4, ver1.patchlevel); - test_eq(VER_RELEASE, ver1.status); - test_streq("", ver1.status_tag); - -#define test_eq_vs(vs1, vs2) test_eq_type(version_status_t, "%d", (vs1), (vs2)) -#define test_v_i_o(val, ver, lst) \ - test_eq_vs(val, tor_version_is_obsolete(ver, lst)) - - /* make sure tor_version_is_obsolete() works */ - test_v_i_o(VS_OLD, "0.0.1", "Tor 0.0.2"); - test_v_i_o(VS_OLD, "0.0.1", "0.0.2, Tor 0.0.3"); - test_v_i_o(VS_OLD, "0.0.1", "0.0.2,Tor 0.0.3"); - test_v_i_o(VS_OLD, "0.0.1","0.0.3,BetterTor 0.0.1"); - test_v_i_o(VS_RECOMMENDED, "0.0.2", "Tor 0.0.2,Tor 0.0.3"); - test_v_i_o(VS_NEW_IN_SERIES, "0.0.2", "Tor 0.0.2pre1,Tor 0.0.3"); - test_v_i_o(VS_OLD, "0.0.2", "Tor 0.0.2.1,Tor 0.0.3"); - test_v_i_o(VS_NEW, "0.1.0", "Tor 0.0.2,Tor 0.0.3"); - test_v_i_o(VS_RECOMMENDED, "0.0.7rc2", "0.0.7,Tor 0.0.7rc2,Tor 0.0.8"); - test_v_i_o(VS_OLD, "0.0.5.0", "0.0.5.1-cvs"); - test_v_i_o(VS_NEW_IN_SERIES, "0.0.5.1-cvs", "0.0.5, 0.0.6"); - /* Not on list, but newer than any in same series. */ - test_v_i_o(VS_NEW_IN_SERIES, "0.1.0.3", - "Tor 0.1.0.2,Tor 0.0.9.5,Tor 0.1.1.0"); - /* Series newer than any on list. */ - test_v_i_o(VS_NEW, "0.1.2.3", "Tor 0.1.0.2,Tor 0.0.9.5,Tor 0.1.1.0"); - /* Series older than any on list. */ - test_v_i_o(VS_OLD, "0.0.1.3", "Tor 0.1.0.2,Tor 0.0.9.5,Tor 0.1.1.0"); - /* Not on list, not newer than any on same series. */ - test_v_i_o(VS_UNRECOMMENDED, "0.1.0.1", - "Tor 0.1.0.2,Tor 0.0.9.5,Tor 0.1.1.0"); - /* On list, not newer than any on same series. */ - test_v_i_o(VS_UNRECOMMENDED, - "0.1.0.1", "Tor 0.1.0.2,Tor 0.0.9.5,Tor 0.1.1.0"); - test_eq(0, tor_version_as_new_as("Tor 0.0.5", "0.0.9pre1-cvs")); - test_eq(1, tor_version_as_new_as( - "Tor 0.0.8 on Darwin 64-121-192-100.c3-0." - "sfpo-ubr1.sfrn-sfpo.ca.cable.rcn.com Power Macintosh", - "0.0.8rc2")); - test_eq(0, tor_version_as_new_as( - "Tor 0.0.8 on Darwin 64-121-192-100.c3-0." - "sfpo-ubr1.sfrn-sfpo.ca.cable.rcn.com Power Macintosh", "0.0.8.2")); - - /* Now try svn revisions. */ - test_eq(1, tor_version_as_new_as("Tor 0.2.1.0-dev (r100)", - "Tor 0.2.1.0-dev (r99)")); - test_eq(1, tor_version_as_new_as("Tor 0.2.1.0-dev (r100) on Banana Jr", - "Tor 0.2.1.0-dev (r99) on Hal 9000")); - test_eq(1, tor_version_as_new_as("Tor 0.2.1.0-dev (r100)", - "Tor 0.2.1.0-dev on Colossus")); - test_eq(0, tor_version_as_new_as("Tor 0.2.1.0-dev (r99)", - "Tor 0.2.1.0-dev (r100)")); - test_eq(0, tor_version_as_new_as("Tor 0.2.1.0-dev (r99) on MCP", - "Tor 0.2.1.0-dev (r100) on AM")); - test_eq(0, tor_version_as_new_as("Tor 0.2.1.0-dev", - "Tor 0.2.1.0-dev (r99)")); - test_eq(1, tor_version_as_new_as("Tor 0.2.1.1", - "Tor 0.2.1.0-dev (r99)")); - done: - if (r1) - routerinfo_free(r1); - if (r2) - routerinfo_free(r2); - - tor_free(pk1_str); - tor_free(pk2_str); - tor_free(pk3_str); - if (pk1) crypto_free_pk_env(pk1); - if (pk2) crypto_free_pk_env(pk2); - if (pk3) crypto_free_pk_env(pk3); - if (rp1) routerinfo_free(rp1); - tor_free(dir1); /* XXXX And more !*/ - tor_free(dir2); /* And more !*/ -} - -/** Run unit tests for misc directory functions. */ -static void -test_dirutil(void) -{ - smartlist_t *sl = smartlist_create(); - fp_pair_t *pair; - - dir_split_resource_into_fingerprint_pairs( - /* Two pairs, out of order, with one duplicate. */ - "73656372657420646174612E0000000000FFFFFF-" - "557365204145532d32353620696e73746561642e+" - "73656372657420646174612E0000000000FFFFFF-" - "557365204145532d32353620696e73746561642e+" - "48657861646563696d616c2069736e277420736f-" - "676f6f6420666f7220686964696e6720796f7572.z", sl); - - test_eq(smartlist_len(sl), 2); - pair = smartlist_get(sl, 0); - test_memeq(pair->first, "Hexadecimal isn't so", DIGEST_LEN); - test_memeq(pair->second, "good for hiding your", DIGEST_LEN); - pair = smartlist_get(sl, 1); - test_memeq(pair->first, "secret data.\0\0\0\0\0\xff\xff\xff", DIGEST_LEN); - test_memeq(pair->second, "Use AES-256 instead.", DIGEST_LEN); - - done: - SMARTLIST_FOREACH(sl, fp_pair_t *, pair, tor_free(pair)); - smartlist_free(sl); -} - -extern const char AUTHORITY_CERT_1[]; -extern const char AUTHORITY_SIGNKEY_1[]; -extern const char AUTHORITY_CERT_2[]; -extern const char AUTHORITY_SIGNKEY_2[]; -extern const char AUTHORITY_CERT_3[]; -extern const char AUTHORITY_SIGNKEY_3[]; - -/** Helper: Test that two networkstatus_voter_info_t do in fact represent the - * same voting authority, and that they do in fact have all the same - * information. */ -static void -test_same_voter(networkstatus_voter_info_t *v1, - networkstatus_voter_info_t *v2) -{ - test_streq(v1->nickname, v2->nickname); - test_memeq(v1->identity_digest, v2->identity_digest, DIGEST_LEN); - test_streq(v1->address, v2->address); - test_eq(v1->addr, v2->addr); - test_eq(v1->dir_port, v2->dir_port); - test_eq(v1->or_port, v2->or_port); - test_streq(v1->contact, v2->contact); - test_memeq(v1->vote_digest, v2->vote_digest, DIGEST_LEN); - done: - ; -} - -/** Run unit tests for getting the median of a list. */ -static void -test_util_order_functions(void) -{ - int lst[25], n = 0; - // int a=12,b=24,c=25,d=60,e=77; - -#define median() median_int(lst, n) - - lst[n++] = 12; - test_eq(12, median()); /* 12 */ - lst[n++] = 77; - //smartlist_shuffle(sl); - test_eq(12, median()); /* 12, 77 */ - lst[n++] = 77; - //smartlist_shuffle(sl); - test_eq(77, median()); /* 12, 77, 77 */ - lst[n++] = 24; - test_eq(24, median()); /* 12,24,77,77 */ - lst[n++] = 60; - lst[n++] = 12; - lst[n++] = 25; - //smartlist_shuffle(sl); - test_eq(25, median()); /* 12,12,24,25,60,77,77 */ -#undef median - - done: - ; -} - -/** Helper: Make a new routerinfo containing the right information for a - * given vote_routerstatus_t. */ -static routerinfo_t * -generate_ri_from_rs(const vote_routerstatus_t *vrs) -{ - routerinfo_t *r; - const routerstatus_t *rs = &vrs->status; - static time_t published = 0; - - r = tor_malloc_zero(sizeof(routerinfo_t)); - memcpy(r->cache_info.identity_digest, rs->identity_digest, DIGEST_LEN); - memcpy(r->cache_info.signed_descriptor_digest, rs->descriptor_digest, - DIGEST_LEN); - r->cache_info.do_not_cache = 1; - r->cache_info.routerlist_index = -1; - r->cache_info.signed_descriptor_body = - tor_strdup("123456789012345678901234567890123"); - r->cache_info.signed_descriptor_len = - strlen(r->cache_info.signed_descriptor_body); - r->exit_policy = smartlist_create(); - r->cache_info.published_on = ++published + time(NULL); - return r; -} - -/** Run unit tests for generating and parsing V3 consensus networkstatus - * documents. */ -static void -test_v3_networkstatus(void) -{ - authority_cert_t *cert1=NULL, *cert2=NULL, *cert3=NULL; - crypto_pk_env_t *sign_skey_1=NULL, *sign_skey_2=NULL, *sign_skey_3=NULL; - crypto_pk_env_t *sign_skey_leg1=NULL; - const char *msg=NULL; - - time_t now = time(NULL); - networkstatus_voter_info_t *voter; - networkstatus_t *vote=NULL, *v1=NULL, *v2=NULL, *v3=NULL, *con=NULL; - vote_routerstatus_t *vrs; - routerstatus_t *rs; - char *v1_text=NULL, *v2_text=NULL, *v3_text=NULL, *consensus_text=NULL, *cp; - smartlist_t *votes = smartlist_create(); - - /* For generating the two other consensuses. */ - char *detached_text1=NULL, *detached_text2=NULL; - char *consensus_text2=NULL, *consensus_text3=NULL; - networkstatus_t *con2=NULL, *con3=NULL; - ns_detached_signatures_t *dsig1=NULL, *dsig2=NULL; - - /* Parse certificates and keys. */ - cert1 = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL); - test_assert(cert1); - test_assert(cert1->is_cross_certified); - cert2 = authority_cert_parse_from_string(AUTHORITY_CERT_2, NULL); - test_assert(cert2); - cert3 = authority_cert_parse_from_string(AUTHORITY_CERT_3, NULL); - test_assert(cert3); - sign_skey_1 = crypto_new_pk_env(); - sign_skey_2 = crypto_new_pk_env(); - sign_skey_3 = crypto_new_pk_env(); - sign_skey_leg1 = pk_generate(4); - - test_assert(!crypto_pk_read_private_key_from_string(sign_skey_1, - AUTHORITY_SIGNKEY_1)); - test_assert(!crypto_pk_read_private_key_from_string(sign_skey_2, - AUTHORITY_SIGNKEY_2)); - test_assert(!crypto_pk_read_private_key_from_string(sign_skey_3, - AUTHORITY_SIGNKEY_3)); - - test_assert(!crypto_pk_cmp_keys(sign_skey_1, cert1->signing_key)); - test_assert(!crypto_pk_cmp_keys(sign_skey_2, cert2->signing_key)); - - /* - * Set up a vote; generate it; try to parse it. - */ - vote = tor_malloc_zero(sizeof(networkstatus_t)); - vote->type = NS_TYPE_VOTE; - vote->published = now; - vote->valid_after = now+1000; - vote->fresh_until = now+2000; - vote->valid_until = now+3000; - vote->vote_seconds = 100; - vote->dist_seconds = 200; - vote->supported_methods = smartlist_create(); - smartlist_split_string(vote->supported_methods, "1 2 3", NULL, 0, -1); - vote->client_versions = tor_strdup("0.1.2.14,0.1.2.15"); - vote->server_versions = tor_strdup("0.1.2.14,0.1.2.15,0.1.2.16"); - vote->known_flags = smartlist_create(); - smartlist_split_string(vote->known_flags, - "Authority Exit Fast Guard Running Stable V2Dir Valid", - 0, SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); - vote->voters = smartlist_create(); - voter = tor_malloc_zero(sizeof(networkstatus_voter_info_t)); - voter->nickname = tor_strdup("Voter1"); - voter->address = tor_strdup("1.2.3.4"); - voter->addr = 0x01020304; - voter->dir_port = 80; - voter->or_port = 9000; - voter->contact = tor_strdup("voter@example.com"); - crypto_pk_get_digest(cert1->identity_key, voter->identity_digest); - smartlist_add(vote->voters, voter); - vote->cert = authority_cert_dup(cert1); - vote->routerstatus_list = smartlist_create(); - /* add the first routerstatus. */ - vrs = tor_malloc_zero(sizeof(vote_routerstatus_t)); - rs = &vrs->status; - vrs->version = tor_strdup("0.1.2.14"); - rs->published_on = now-1500; - strlcpy(rs->nickname, "router2", sizeof(rs->nickname)); - memset(rs->identity_digest, 3, DIGEST_LEN); - memset(rs->descriptor_digest, 78, DIGEST_LEN); - rs->addr = 0x99008801; - rs->or_port = 443; - rs->dir_port = 8000; - /* all flags but running cleared */ - rs->is_running = 1; - smartlist_add(vote->routerstatus_list, vrs); - test_assert(router_add_to_routerlist(generate_ri_from_rs(vrs), &msg,0,0)>=0); - - /* add the second routerstatus. */ - vrs = tor_malloc_zero(sizeof(vote_routerstatus_t)); - rs = &vrs->status; - vrs->version = tor_strdup("0.2.0.5"); - rs->published_on = now-1000; - strlcpy(rs->nickname, "router1", sizeof(rs->nickname)); - memset(rs->identity_digest, 5, DIGEST_LEN); - memset(rs->descriptor_digest, 77, DIGEST_LEN); - rs->addr = 0x99009901; - rs->or_port = 443; - rs->dir_port = 0; - rs->is_exit = rs->is_stable = rs->is_fast = rs->is_running = - rs->is_valid = rs->is_v2_dir = rs->is_possible_guard = 1; - smartlist_add(vote->routerstatus_list, vrs); - test_assert(router_add_to_routerlist(generate_ri_from_rs(vrs), &msg,0,0)>=0); - - /* add the third routerstatus. */ - vrs = tor_malloc_zero(sizeof(vote_routerstatus_t)); - rs = &vrs->status; - vrs->version = tor_strdup("0.1.0.3"); - rs->published_on = now-1000; - strlcpy(rs->nickname, "router3", sizeof(rs->nickname)); - memset(rs->identity_digest, 33, DIGEST_LEN); - memset(rs->descriptor_digest, 79, DIGEST_LEN); - rs->addr = 0xAA009901; - rs->or_port = 400; - rs->dir_port = 9999; - rs->is_authority = rs->is_exit = rs->is_stable = rs->is_fast = - rs->is_running = rs->is_valid = rs->is_v2_dir = rs->is_possible_guard = 1; - smartlist_add(vote->routerstatus_list, vrs); - test_assert(router_add_to_routerlist(generate_ri_from_rs(vrs), &msg,0,0)>=0); - - /* add a fourth routerstatus that is not running. */ - vrs = tor_malloc_zero(sizeof(vote_routerstatus_t)); - rs = &vrs->status; - vrs->version = tor_strdup("0.1.6.3"); - rs->published_on = now-1000; - strlcpy(rs->nickname, "router4", sizeof(rs->nickname)); - memset(rs->identity_digest, 34, DIGEST_LEN); - memset(rs->descriptor_digest, 48, DIGEST_LEN); - rs->addr = 0xC0000203; - rs->or_port = 500; - rs->dir_port = 1999; - /* Running flag (and others) cleared */ - smartlist_add(vote->routerstatus_list, vrs); - test_assert(router_add_to_routerlist(generate_ri_from_rs(vrs), &msg,0,0)>=0); - - /* dump the vote and try to parse it. */ - v1_text = format_networkstatus_vote(sign_skey_1, vote); - test_assert(v1_text); - v1 = networkstatus_parse_vote_from_string(v1_text, NULL, NS_TYPE_VOTE); - test_assert(v1); - - /* Make sure the parsed thing was right. */ - test_eq(v1->type, NS_TYPE_VOTE); - test_eq(v1->published, vote->published); - test_eq(v1->valid_after, vote->valid_after); - test_eq(v1->fresh_until, vote->fresh_until); - test_eq(v1->valid_until, vote->valid_until); - test_eq(v1->vote_seconds, vote->vote_seconds); - test_eq(v1->dist_seconds, vote->dist_seconds); - test_streq(v1->client_versions, vote->client_versions); - test_streq(v1->server_versions, vote->server_versions); - test_assert(v1->voters && smartlist_len(v1->voters)); - voter = smartlist_get(v1->voters, 0); - test_streq(voter->nickname, "Voter1"); - test_streq(voter->address, "1.2.3.4"); - test_eq(voter->addr, 0x01020304); - test_eq(voter->dir_port, 80); - test_eq(voter->or_port, 9000); - test_streq(voter->contact, "voter@example.com"); - test_assert(v1->cert); - test_assert(!crypto_pk_cmp_keys(sign_skey_1, v1->cert->signing_key)); - cp = smartlist_join_strings(v1->known_flags, ":", 0, NULL); - test_streq(cp, "Authority:Exit:Fast:Guard:Running:Stable:V2Dir:Valid"); - tor_free(cp); - test_eq(smartlist_len(v1->routerstatus_list), 4); - /* Check the first routerstatus. */ - vrs = smartlist_get(v1->routerstatus_list, 0); - rs = &vrs->status; - test_streq(vrs->version, "0.1.2.14"); - test_eq(rs->published_on, now-1500); - test_streq(rs->nickname, "router2"); - test_memeq(rs->identity_digest, - "\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3", - DIGEST_LEN); - test_memeq(rs->descriptor_digest, "NNNNNNNNNNNNNNNNNNNN", DIGEST_LEN); - test_eq(rs->addr, 0x99008801); - test_eq(rs->or_port, 443); - test_eq(rs->dir_port, 8000); - test_eq(vrs->flags, U64_LITERAL(16)); // no flags except "running" - /* Check the second routerstatus. */ - vrs = smartlist_get(v1->routerstatus_list, 1); - rs = &vrs->status; - test_streq(vrs->version, "0.2.0.5"); - test_eq(rs->published_on, now-1000); - test_streq(rs->nickname, "router1"); - test_memeq(rs->identity_digest, - "\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5", - DIGEST_LEN); - test_memeq(rs->descriptor_digest, "MMMMMMMMMMMMMMMMMMMM", DIGEST_LEN); - test_eq(rs->addr, 0x99009901); - test_eq(rs->or_port, 443); - test_eq(rs->dir_port, 0); - test_eq(vrs->flags, U64_LITERAL(254)); // all flags except "authority." - - /* Generate second vote. It disagrees on some of the times, - * and doesn't list versions, and knows some crazy flags */ - vote->published = now+1; - vote->fresh_until = now+3005; - vote->dist_seconds = 300; - authority_cert_free(vote->cert); - vote->cert = authority_cert_dup(cert2); - tor_free(vote->client_versions); - tor_free(vote->server_versions); - voter = smartlist_get(vote->voters, 0); - tor_free(voter->nickname); - tor_free(voter->address); - voter->nickname = tor_strdup("Voter2"); - voter->address = tor_strdup("2.3.4.5"); - voter->addr = 0x02030405; - crypto_pk_get_digest(cert2->identity_key, voter->identity_digest); - smartlist_add(vote->known_flags, tor_strdup("MadeOfCheese")); - smartlist_add(vote->known_flags, tor_strdup("MadeOfTin")); - smartlist_sort_strings(vote->known_flags); - vrs = smartlist_get(vote->routerstatus_list, 2); - smartlist_del_keeporder(vote->routerstatus_list, 2); - tor_free(vrs->version); - tor_free(vrs); - vrs = smartlist_get(vote->routerstatus_list, 0); - vrs->status.is_fast = 1; - /* generate and parse. */ - v2_text = format_networkstatus_vote(sign_skey_2, vote); - test_assert(v2_text); - v2 = networkstatus_parse_vote_from_string(v2_text, NULL, NS_TYPE_VOTE); - test_assert(v2); - /* Check that flags come out right.*/ - cp = smartlist_join_strings(v2->known_flags, ":", 0, NULL); - test_streq(cp, "Authority:Exit:Fast:Guard:MadeOfCheese:MadeOfTin:" - "Running:Stable:V2Dir:Valid"); - tor_free(cp); - vrs = smartlist_get(v2->routerstatus_list, 1); - /* 1023 - authority(1) - madeofcheese(16) - madeoftin(32) */ - test_eq(vrs->flags, U64_LITERAL(974)); - - /* Generate the third vote. */ - vote->published = now; - vote->fresh_until = now+2003; - vote->dist_seconds = 250; - authority_cert_free(vote->cert); - vote->cert = authority_cert_dup(cert3); - smartlist_add(vote->supported_methods, tor_strdup("4")); - vote->client_versions = tor_strdup("0.1.2.14,0.1.2.17"); - vote->server_versions = tor_strdup("0.1.2.10,0.1.2.15,0.1.2.16"); - voter = smartlist_get(vote->voters, 0); - tor_free(voter->nickname); - tor_free(voter->address); - voter->nickname = tor_strdup("Voter3"); - voter->address = tor_strdup("3.4.5.6"); - voter->addr = 0x03040506; - crypto_pk_get_digest(cert3->identity_key, voter->identity_digest); - /* This one has a legacy id. */ - memset(voter->legacy_id_digest, (int)'A', DIGEST_LEN); - vrs = smartlist_get(vote->routerstatus_list, 0); - smartlist_del_keeporder(vote->routerstatus_list, 0); - tor_free(vrs->version); - tor_free(vrs); - vrs = smartlist_get(vote->routerstatus_list, 0); - memset(vrs->status.descriptor_digest, (int)'Z', DIGEST_LEN); - test_assert(router_add_to_routerlist(generate_ri_from_rs(vrs), &msg,0,0)>=0); - - v3_text = format_networkstatus_vote(sign_skey_3, vote); - test_assert(v3_text); - - v3 = networkstatus_parse_vote_from_string(v3_text, NULL, NS_TYPE_VOTE); - test_assert(v3); - - /* Compute a consensus as voter 3. */ - smartlist_add(votes, v3); - smartlist_add(votes, v1); - smartlist_add(votes, v2); - consensus_text = networkstatus_compute_consensus(votes, 3, - cert3->identity_key, - sign_skey_3, - "AAAAAAAAAAAAAAAAAAAA", - sign_skey_leg1); - test_assert(consensus_text); - con = networkstatus_parse_vote_from_string(consensus_text, NULL, - NS_TYPE_CONSENSUS); - test_assert(con); - //log_notice(LD_GENERAL, "<<%s>>\n<<%s>>\n<<%s>>\n", - // v1_text, v2_text, v3_text); - - /* Check consensus contents. */ - test_assert(con->type == NS_TYPE_CONSENSUS); - test_eq(con->published, 0); /* this field only appears in votes. */ - test_eq(con->valid_after, now+1000); - test_eq(con->fresh_until, now+2003); /* median */ - test_eq(con->valid_until, now+3000); - test_eq(con->vote_seconds, 100); - test_eq(con->dist_seconds, 250); /* median */ - test_streq(con->client_versions, "0.1.2.14"); - test_streq(con->server_versions, "0.1.2.15,0.1.2.16"); - cp = smartlist_join_strings(v2->known_flags, ":", 0, NULL); - test_streq(cp, "Authority:Exit:Fast:Guard:MadeOfCheese:MadeOfTin:" - "Running:Stable:V2Dir:Valid"); - tor_free(cp); - test_eq(4, smartlist_len(con->voters)); /*3 voters, 1 legacy key.*/ - /* The voter id digests should be in this order. */ - test_assert(memcmp(cert2->cache_info.identity_digest, - cert1->cache_info.identity_digest,DIGEST_LEN)<0); - test_assert(memcmp(cert1->cache_info.identity_digest, - cert3->cache_info.identity_digest,DIGEST_LEN)<0); - test_same_voter(smartlist_get(con->voters, 1), - smartlist_get(v2->voters, 0)); - test_same_voter(smartlist_get(con->voters, 2), - smartlist_get(v1->voters, 0)); - test_same_voter(smartlist_get(con->voters, 3), - smartlist_get(v3->voters, 0)); - - test_assert(!con->cert); - test_eq(2, smartlist_len(con->routerstatus_list)); - /* There should be two listed routers: one with identity 3, one with - * identity 5. */ - /* This one showed up in 2 digests. */ - rs = smartlist_get(con->routerstatus_list, 0); - test_memeq(rs->identity_digest, - "\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3", - DIGEST_LEN); - test_memeq(rs->descriptor_digest, "NNNNNNNNNNNNNNNNNNNN", DIGEST_LEN); - test_assert(!rs->is_authority); - test_assert(!rs->is_exit); - test_assert(!rs->is_fast); - test_assert(!rs->is_possible_guard); - test_assert(!rs->is_stable); - test_assert(rs->is_running); /* If it wasn't running it wouldn't be here */ - test_assert(!rs->is_v2_dir); - test_assert(!rs->is_valid); - test_assert(!rs->is_named); - /* XXXX check version */ - - rs = smartlist_get(con->routerstatus_list, 1); - /* This one showed up in 3 digests. Twice with ID 'M', once with 'Z'. */ - test_memeq(rs->identity_digest, - "\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5", - DIGEST_LEN); - test_streq(rs->nickname, "router1"); - test_memeq(rs->descriptor_digest, "MMMMMMMMMMMMMMMMMMMM", DIGEST_LEN); - test_eq(rs->published_on, now-1000); - test_eq(rs->addr, 0x99009901); - test_eq(rs->or_port, 443); - test_eq(rs->dir_port, 0); - test_assert(!rs->is_authority); - test_assert(rs->is_exit); - test_assert(rs->is_fast); - test_assert(rs->is_possible_guard); - test_assert(rs->is_stable); - test_assert(rs->is_running); - test_assert(rs->is_v2_dir); - test_assert(rs->is_valid); - test_assert(!rs->is_named); - /* XXXX check version */ - // x231 - // x213 - - /* Check signatures. the first voter is a pseudo-entry with a legacy key. - * The second one hasn't signed. The fourth one has signed: validate it. */ - voter = smartlist_get(con->voters, 1); - test_assert(!voter->signature); - test_assert(!voter->good_signature); - test_assert(!voter->bad_signature); - - voter = smartlist_get(con->voters, 3); - test_assert(voter->signature); - test_assert(!voter->good_signature); - test_assert(!voter->bad_signature); - test_assert(!networkstatus_check_voter_signature(con, - smartlist_get(con->voters, 3), - cert3)); - test_assert(voter->signature); - test_assert(voter->good_signature); - test_assert(!voter->bad_signature); - - { - const char *msg=NULL; - /* Compute the other two signed consensuses. */ - smartlist_shuffle(votes); - consensus_text2 = networkstatus_compute_consensus(votes, 3, - cert2->identity_key, - sign_skey_2, NULL,NULL); - smartlist_shuffle(votes); - consensus_text3 = networkstatus_compute_consensus(votes, 3, - cert1->identity_key, - sign_skey_1, NULL,NULL); - test_assert(consensus_text2); - test_assert(consensus_text3); - con2 = networkstatus_parse_vote_from_string(consensus_text2, NULL, - NS_TYPE_CONSENSUS); - con3 = networkstatus_parse_vote_from_string(consensus_text3, NULL, - NS_TYPE_CONSENSUS); - test_assert(con2); - test_assert(con3); - - /* All three should have the same digest. */ - test_memeq(con->networkstatus_digest, con2->networkstatus_digest, - DIGEST_LEN); - test_memeq(con->networkstatus_digest, con3->networkstatus_digest, - DIGEST_LEN); - - /* Extract a detached signature from con3. */ - detached_text1 = networkstatus_get_detached_signatures(con3); - tor_assert(detached_text1); - /* Try to parse it. */ - dsig1 = networkstatus_parse_detached_signatures(detached_text1, NULL); - tor_assert(dsig1); - - /* Are parsed values as expected? */ - test_eq(dsig1->valid_after, con3->valid_after); - test_eq(dsig1->fresh_until, con3->fresh_until); - test_eq(dsig1->valid_until, con3->valid_until); - test_memeq(dsig1->networkstatus_digest, con3->networkstatus_digest, - DIGEST_LEN); - test_eq(1, smartlist_len(dsig1->signatures)); - voter = smartlist_get(dsig1->signatures, 0); - test_memeq(voter->identity_digest, cert1->cache_info.identity_digest, - DIGEST_LEN); - - /* Try adding it to con2. */ - detached_text2 = networkstatus_get_detached_signatures(con2); - test_eq(1, networkstatus_add_detached_signatures(con2, dsig1, &msg)); - tor_free(detached_text2); - detached_text2 = networkstatus_get_detached_signatures(con2); - //printf("\n<%s>\n", detached_text2); - dsig2 = networkstatus_parse_detached_signatures(detached_text2, NULL); - test_assert(dsig2); - /* - printf("\n"); - SMARTLIST_FOREACH(dsig2->signatures, networkstatus_voter_info_t *, vi, { - char hd[64]; - base16_encode(hd, sizeof(hd), vi->identity_digest, DIGEST_LEN); - printf("%s\n", hd); - }); - */ - test_eq(2, smartlist_len(dsig2->signatures)); - - /* Try adding to con2 twice; verify that nothing changes. */ - test_eq(0, networkstatus_add_detached_signatures(con2, dsig1, &msg)); - - /* Add to con. */ - test_eq(2, networkstatus_add_detached_signatures(con, dsig2, &msg)); - /* Check signatures */ - test_assert(!networkstatus_check_voter_signature(con, - smartlist_get(con->voters, 1), - cert2)); - test_assert(!networkstatus_check_voter_signature(con, - smartlist_get(con->voters, 2), - cert1)); - - } - - done: - smartlist_free(votes); - tor_free(v1_text); - tor_free(v2_text); - tor_free(v3_text); - tor_free(consensus_text); - - if (vote) - networkstatus_vote_free(vote); - if (v1) - networkstatus_vote_free(v1); - if (v2) - networkstatus_vote_free(v2); - if (v3) - networkstatus_vote_free(v3); - if (con) - networkstatus_vote_free(con); - if (sign_skey_1) - crypto_free_pk_env(sign_skey_1); - if (sign_skey_2) - crypto_free_pk_env(sign_skey_2); - if (sign_skey_3) - crypto_free_pk_env(sign_skey_3); - if (sign_skey_leg1) - crypto_free_pk_env(sign_skey_leg1); - if (cert1) - authority_cert_free(cert1); - if (cert2) - authority_cert_free(cert2); - if (cert3) - authority_cert_free(cert3); - - tor_free(consensus_text2); - tor_free(consensus_text3); - tor_free(detached_text1); - tor_free(detached_text2); - if (con2) - networkstatus_vote_free(con2); - if (con3) - networkstatus_vote_free(con3); - if (dsig1) - ns_detached_signatures_free(dsig1); - if (dsig2) - ns_detached_signatures_free(dsig2); -} - -/** Helper: Parse the exit policy string in <b>policy_str</b>, and make sure - * that policies_summarize() produces the string <b>expected_summary</b> from - * it. */ -static void -test_policy_summary_helper(const char *policy_str, - const char *expected_summary) -{ - config_line_t line; - smartlist_t *policy = smartlist_create(); - char *summary = NULL; - int r; - - line.key = (char*)"foo"; - line.value = (char *)policy_str; - line.next = NULL; - - r = policies_parse_exit_policy(&line, &policy, 0, NULL); - test_eq(r, 0); - summary = policy_summarize(policy); - - test_assert(summary != NULL); - test_streq(summary, expected_summary); - - done: - tor_free(summary); - if (policy) - addr_policy_list_free(policy); -} - -/** Run unit tests for generating summary lines of exit policies */ -static void -test_policies(void) -{ - int i; - smartlist_t *policy = NULL, *policy2 = NULL; - addr_policy_t *p; - tor_addr_t tar; - config_line_t line; - smartlist_t *sm = NULL; - char *policy_str = NULL; - - policy = smartlist_create(); - - p = router_parse_addr_policy_item_from_string("reject 192.168.0.0/16:*",-1); - test_assert(p != NULL); - test_eq(ADDR_POLICY_REJECT, p->policy_type); - tor_addr_from_ipv4h(&tar, 0xc0a80000u); - test_eq(0, tor_addr_compare(&p->addr, &tar, CMP_EXACT)); - test_eq(16, p->maskbits); - test_eq(1, p->prt_min); - test_eq(65535, p->prt_max); - - smartlist_add(policy, p); - - test_assert(ADDR_POLICY_ACCEPTED == - compare_addr_to_addr_policy(0x01020304u, 2, policy)); - test_assert(ADDR_POLICY_PROBABLY_ACCEPTED == - compare_addr_to_addr_policy(0, 2, policy)); - test_assert(ADDR_POLICY_REJECTED == - compare_addr_to_addr_policy(0xc0a80102, 2, policy)); - - policy2 = NULL; - test_assert(0 == policies_parse_exit_policy(NULL, &policy2, 1, NULL)); - test_assert(policy2); - - test_assert(!exit_policy_is_general_exit(policy)); - test_assert(exit_policy_is_general_exit(policy2)); - test_assert(!exit_policy_is_general_exit(NULL)); - - test_assert(cmp_addr_policies(policy, policy2)); - test_assert(cmp_addr_policies(policy, NULL)); - test_assert(!cmp_addr_policies(policy2, policy2)); - test_assert(!cmp_addr_policies(NULL, NULL)); - - test_assert(!policy_is_reject_star(policy2)); - test_assert(policy_is_reject_star(policy)); - test_assert(policy_is_reject_star(NULL)); - - addr_policy_list_free(policy); - policy = NULL; - - /* make sure compacting logic works. */ - policy = NULL; - line.key = (char*)"foo"; - line.value = (char*)"accept *:80,reject private:*,reject *:*"; - line.next = NULL; - test_assert(0 == policies_parse_exit_policy(&line, &policy, 0, NULL)); - test_assert(policy); - //test_streq(policy->string, "accept *:80"); - //test_streq(policy->next->string, "reject *:*"); - test_eq(smartlist_len(policy), 2); - - /* test policy summaries */ - /* check if we properly ignore private IP addresses */ - test_policy_summary_helper("reject 192.168.0.0/16:*," - "reject 0.0.0.0/8:*," - "reject 10.0.0.0/8:*," - "accept *:10-30," - "accept *:90," - "reject *:*", - "accept 10-30,90"); - /* check all accept policies, and proper counting of rejects */ - test_policy_summary_helper("reject 11.0.0.0/9:80," - "reject 12.0.0.0/9:80," - "reject 13.0.0.0/9:80," - "reject 14.0.0.0/9:80," - "accept *:*", "accept 1-65535"); - test_policy_summary_helper("reject 11.0.0.0/9:80," - "reject 12.0.0.0/9:80," - "reject 13.0.0.0/9:80," - "reject 14.0.0.0/9:80," - "reject 15.0.0.0:81," - "accept *:*", "accept 1-65535"); - test_policy_summary_helper("reject 11.0.0.0/9:80," - "reject 12.0.0.0/9:80," - "reject 13.0.0.0/9:80," - "reject 14.0.0.0/9:80," - "reject 15.0.0.0:80," - "accept *:*", - "reject 80"); - /* no exits */ - test_policy_summary_helper("accept 11.0.0.0/9:80," - "reject *:*", - "reject 1-65535"); - /* port merging */ - test_policy_summary_helper("accept *:80," - "accept *:81," - "accept *:100-110," - "accept *:111," - "reject *:*", - "accept 80-81,100-111"); - /* border ports */ - test_policy_summary_helper("accept *:1," - "accept *:3," - "accept *:65535," - "reject *:*", - "accept 1,3,65535"); - /* holes */ - test_policy_summary_helper("accept *:1," - "accept *:3," - "accept *:5," - "accept *:7," - "reject *:*", - "accept 1,3,5,7"); - test_policy_summary_helper("reject *:1," - "reject *:3," - "reject *:5," - "reject *:7," - "accept *:*", - "reject 1,3,5,7"); - - /* truncation ports */ - sm = smartlist_create(); - for (i=1; i<2000; i+=2) { - char buf[POLICY_BUF_LEN]; - tor_snprintf(buf, sizeof(buf), "reject *:%d", i); - smartlist_add(sm, tor_strdup(buf)); - } - smartlist_add(sm, tor_strdup("accept *:*")); - policy_str = smartlist_join_strings(sm, ",", 0, NULL); - test_policy_summary_helper( policy_str, - "accept 2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40,42,44," - "46,48,50,52,54,56,58,60,62,64,66,68,70,72,74,76,78,80,82,84,86,88,90," - "92,94,96,98,100,102,104,106,108,110,112,114,116,118,120,122,124,126,128," - "130,132,134,136,138,140,142,144,146,148,150,152,154,156,158,160,162,164," - "166,168,170,172,174,176,178,180,182,184,186,188,190,192,194,196,198,200," - "202,204,206,208,210,212,214,216,218,220,222,224,226,228,230,232,234,236," - "238,240,242,244,246,248,250,252,254,256,258,260,262,264,266,268,270,272," - "274,276,278,280,282,284,286,288,290,292,294,296,298,300,302,304,306,308," - "310,312,314,316,318,320,322,324,326,328,330,332,334,336,338,340,342,344," - "346,348,350,352,354,356,358,360,362,364,366,368,370,372,374,376,378,380," - "382,384,386,388,390,392,394,396,398,400,402,404,406,408,410,412,414,416," - "418,420,422,424,426,428,430,432,434,436,438,440,442,444,446,448,450,452," - "454,456,458,460,462,464,466,468,470,472,474,476,478,480,482,484,486,488," - "490,492,494,496,498,500,502,504,506,508,510,512,514,516,518,520,522"); - - done: - if (policy) - addr_policy_list_free(policy); - if (policy2) - addr_policy_list_free(policy2); - tor_free(policy_str); - if (sm) { - SMARTLIST_FOREACH(sm, char *, s, tor_free(s)); - smartlist_free(sm); - } -} - -/** Run unit tests for basic rendezvous functions. */ -static void -test_rend_fns(void) -{ - char address1[] = "fooaddress.onion"; - char address2[] = "aaaaaaaaaaaaaaaa.onion"; - char address3[] = "fooaddress.exit"; - char address4[] = "www.torproject.org"; - rend_service_descriptor_t *d1 = - tor_malloc_zero(sizeof(rend_service_descriptor_t)); - rend_service_descriptor_t *d2 = NULL; - char *encoded = NULL; - size_t len; - time_t now; - int i; - crypto_pk_env_t *pk1 = pk_generate(0), *pk2 = pk_generate(1); - - /* Test unversioned (v0) descriptor */ - d1->pk = crypto_pk_dup_key(pk1); - now = time(NULL); - d1->timestamp = now; - d1->version = 0; - d1->intro_nodes = smartlist_create(); - for (i = 0; i < 3; i++) { - rend_intro_point_t *intro = tor_malloc_zero(sizeof(rend_intro_point_t)); - intro->extend_info = tor_malloc_zero(sizeof(extend_info_t)); - crypto_rand(intro->extend_info->identity_digest, DIGEST_LEN); - intro->extend_info->nickname[0] = '$'; - base16_encode(intro->extend_info->nickname+1, HEX_DIGEST_LEN+1, - intro->extend_info->identity_digest, DIGEST_LEN); - smartlist_add(d1->intro_nodes, intro); - } - test_assert(! rend_encode_service_descriptor(d1, pk1, &encoded, &len)); - d2 = rend_parse_service_descriptor(encoded, len); - test_assert(d2); - - test_assert(!crypto_pk_cmp_keys(d1->pk, d2->pk)); - test_eq(d2->timestamp, now); - test_eq(d2->version, 0); - test_eq(d2->protocols, 1<<2); - test_eq(smartlist_len(d2->intro_nodes), 3); - for (i = 0; i < 3; i++) { - rend_intro_point_t *intro1 = smartlist_get(d1->intro_nodes, i); - rend_intro_point_t *intro2 = smartlist_get(d2->intro_nodes, i); - test_streq(intro1->extend_info->nickname, - intro2->extend_info->nickname); - } - - test_assert(BAD_HOSTNAME == parse_extended_hostname(address1)); - test_assert(ONION_HOSTNAME == parse_extended_hostname(address2)); - test_assert(EXIT_HOSTNAME == parse_extended_hostname(address3)); - test_assert(NORMAL_HOSTNAME == parse_extended_hostname(address4)); - - crypto_free_pk_env(pk1); - crypto_free_pk_env(pk2); - pk1 = pk2 = NULL; - rend_service_descriptor_free(d1); - rend_service_descriptor_free(d2); - d1 = d2 = NULL; - - done: - if (pk1) - crypto_free_pk_env(pk1); - if (pk2) - crypto_free_pk_env(pk2); - if (d1) - rend_service_descriptor_free(d1); - if (d2) - rend_service_descriptor_free(d2); - tor_free(encoded); -} - -/** Run AES performance benchmarks. */ -static void -bench_aes(void) -{ - int len, i; - char *b1, *b2; - crypto_cipher_env_t *c; - struct timeval start, end; - const int iters = 100000; - uint64_t nsec; - c = crypto_new_cipher_env(); - crypto_cipher_generate_key(c); - crypto_cipher_encrypt_init_cipher(c); - for (len = 1; len <= 8192; len *= 2) { - b1 = tor_malloc_zero(len); - b2 = tor_malloc_zero(len); - tor_gettimeofday(&start); - for (i = 0; i < iters; ++i) { - crypto_cipher_encrypt(c, b1, b2, len); - } - tor_gettimeofday(&end); - tor_free(b1); - tor_free(b2); - nsec = (uint64_t) tv_udiff(&start,&end); - nsec *= 1000; - nsec /= (iters*len); - printf("%d bytes: "U64_FORMAT" nsec per byte\n", len, - U64_PRINTF_ARG(nsec)); - } - crypto_free_cipher_env(c); -} - -/** Run digestmap_t performance benchmarks. */ -static void -bench_dmap(void) -{ - smartlist_t *sl = smartlist_create(); - smartlist_t *sl2 = smartlist_create(); - struct timeval start, end, pt2, pt3, pt4; - const int iters = 10000; - const int elts = 4000; - const int fpostests = 1000000; - char d[20]; - int i,n=0, fp = 0; - digestmap_t *dm = digestmap_new(); - digestset_t *ds = digestset_new(elts); - - for (i = 0; i < elts; ++i) { - crypto_rand(d, 20); - smartlist_add(sl, tor_memdup(d, 20)); - } - for (i = 0; i < elts; ++i) { - crypto_rand(d, 20); - smartlist_add(sl2, tor_memdup(d, 20)); - } - printf("nbits=%d\n", ds->mask+1); - - tor_gettimeofday(&start); - for (i = 0; i < iters; ++i) { - SMARTLIST_FOREACH(sl, const char *, cp, digestmap_set(dm, cp, (void*)1)); - } - tor_gettimeofday(&pt2); - for (i = 0; i < iters; ++i) { - SMARTLIST_FOREACH(sl, const char *, cp, digestmap_get(dm, cp)); - SMARTLIST_FOREACH(sl2, const char *, cp, digestmap_get(dm, cp)); - } - tor_gettimeofday(&pt3); - for (i = 0; i < iters; ++i) { - SMARTLIST_FOREACH(sl, const char *, cp, digestset_add(ds, cp)); - } - tor_gettimeofday(&pt4); - for (i = 0; i < iters; ++i) { - SMARTLIST_FOREACH(sl, const char *, cp, n += digestset_isin(ds, cp)); - SMARTLIST_FOREACH(sl2, const char *, cp, n += digestset_isin(ds, cp)); - } - tor_gettimeofday(&end); - - for (i = 0; i < fpostests; ++i) { - crypto_rand(d, 20); - if (digestset_isin(ds, d)) ++fp; - } - - printf("%ld\n",(unsigned long)tv_udiff(&start, &pt2)); - printf("%ld\n",(unsigned long)tv_udiff(&pt2, &pt3)); - printf("%ld\n",(unsigned long)tv_udiff(&pt3, &pt4)); - printf("%ld\n",(unsigned long)tv_udiff(&pt4, &end)); - printf("-- %d\n", n); - printf("++ %f\n", fp/(double)fpostests); - digestmap_free(dm, NULL); - digestset_free(ds); - SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); - SMARTLIST_FOREACH(sl2, char *, cp, tor_free(cp)); - smartlist_free(sl); - smartlist_free(sl2); -} - -/** Run unittests for memory pool allocator */ -static void -test_util_mempool(void) -{ - mp_pool_t *pool = NULL; - smartlist_t *allocated = NULL; - int i; - - pool = mp_pool_new(1, 100); - test_assert(pool); - test_assert(pool->new_chunk_capacity >= 100); - test_assert(pool->item_alloc_size >= sizeof(void*)+1); - mp_pool_destroy(pool); - pool = NULL; - - pool = mp_pool_new(241, 2500); - test_assert(pool); - test_assert(pool->new_chunk_capacity >= 10); - test_assert(pool->item_alloc_size >= sizeof(void*)+241); - test_eq(pool->item_alloc_size & 0x03, 0); - test_assert(pool->new_chunk_capacity < 60); - - allocated = smartlist_create(); - for (i = 0; i < 20000; ++i) { - if (smartlist_len(allocated) < 20 || crypto_rand_int(2)) { - void *m = mp_pool_get(pool); - memset(m, 0x09, 241); - smartlist_add(allocated, m); - //printf("%d: %p\n", i, m); - //mp_pool_assert_ok(pool); - } else { - int idx = crypto_rand_int(smartlist_len(allocated)); - void *m = smartlist_get(allocated, idx); - //printf("%d: free %p\n", i, m); - smartlist_del(allocated, idx); - mp_pool_release(m); - //mp_pool_assert_ok(pool); - } - if (crypto_rand_int(777)==0) - mp_pool_clean(pool, 1, 1); - - if (i % 777) - mp_pool_assert_ok(pool); - } - - done: - if (allocated) { - SMARTLIST_FOREACH(allocated, void *, m, mp_pool_release(m)); - mp_pool_assert_ok(pool); - mp_pool_clean(pool, 0, 0); - mp_pool_assert_ok(pool); - smartlist_free(allocated); - } - - if (pool) - mp_pool_destroy(pool); -} - -/** Run unittests for memory area allocator */ -static void -test_util_memarea(void) -{ - memarea_t *area = memarea_new(); - char *p1, *p2, *p3, *p1_orig; - void *malloced_ptr = NULL; - int i; - - test_assert(area); - - p1_orig = p1 = memarea_alloc(area,64); - p2 = memarea_alloc_zero(area,52); - p3 = memarea_alloc(area,11); - - test_assert(memarea_owns_ptr(area, p1)); - test_assert(memarea_owns_ptr(area, p2)); - test_assert(memarea_owns_ptr(area, p3)); - /* Make sure we left enough space. */ - test_assert(p1+64 <= p2); - test_assert(p2+52 <= p3); - /* Make sure we aligned. */ - test_eq(((uintptr_t)p1) % sizeof(void*), 0); - test_eq(((uintptr_t)p2) % sizeof(void*), 0); - test_eq(((uintptr_t)p3) % sizeof(void*), 0); - test_assert(!memarea_owns_ptr(area, p3+8192)); - test_assert(!memarea_owns_ptr(area, p3+30)); - test_assert(tor_mem_is_zero(p2, 52)); - /* Make sure we don't overalign. */ - p1 = memarea_alloc(area, 1); - p2 = memarea_alloc(area, 1); - test_eq(p1+sizeof(void*), p2); - { - malloced_ptr = tor_malloc(64); - test_assert(!memarea_owns_ptr(area, malloced_ptr)); - tor_free(malloced_ptr); - } - - /* memarea_memdup */ - { - malloced_ptr = tor_malloc(64); - crypto_rand((char*)malloced_ptr, 64); - p1 = memarea_memdup(area, malloced_ptr, 64); - test_assert(p1 != malloced_ptr); - test_memeq(p1, malloced_ptr, 64); - tor_free(malloced_ptr); - } - - /* memarea_strdup. */ - p1 = memarea_strdup(area,""); - p2 = memarea_strdup(area, "abcd"); - test_assert(p1); - test_assert(p2); - test_streq(p1, ""); - test_streq(p2, "abcd"); - - /* memarea_strndup. */ - { - const char *s = "Ad ogni porta batte la morte e grida: il nome!"; - /* (From Turandot, act 3.) */ - size_t len = strlen(s); - p1 = memarea_strndup(area, s, 1000); - p2 = memarea_strndup(area, s, 10); - test_streq(p1, s); - test_assert(p2 >= p1 + len + 1); - test_memeq(s, p2, 10); - test_eq(p2[10], '\0'); - p3 = memarea_strndup(area, s, len); - test_streq(p3, s); - p3 = memarea_strndup(area, s, len-1); - test_memeq(s, p3, len-1); - test_eq(p3[len-1], '\0'); - } - - memarea_clear(area); - p1 = memarea_alloc(area, 1); - test_eq(p1, p1_orig); - memarea_clear(area); - - /* Check for running over an area's size. */ - for (i = 0; i < 512; ++i) { - p1 = memarea_alloc(area, crypto_rand_int(5)+1); - test_assert(memarea_owns_ptr(area, p1)); - } - memarea_assert_ok(area); - /* Make sure we can allocate a too-big object. */ - p1 = memarea_alloc_zero(area, 9000); - p2 = memarea_alloc_zero(area, 16); - test_assert(memarea_owns_ptr(area, p1)); - test_assert(memarea_owns_ptr(area, p2)); - - done: - memarea_drop_all(area); - tor_free(malloced_ptr); -} - -/** Run unit tests for utility functions to get file names relative to - * the data directory. */ -static void -test_util_datadir(void) -{ - char buf[1024]; - char *f = NULL; - - f = get_datadir_fname(NULL); - test_streq(f, temp_dir); - tor_free(f); - f = get_datadir_fname("state"); - tor_snprintf(buf, sizeof(buf), "%s"PATH_SEPARATOR"state", temp_dir); - test_streq(f, buf); - tor_free(f); - f = get_datadir_fname2("cache", "thingy"); - tor_snprintf(buf, sizeof(buf), - "%s"PATH_SEPARATOR"cache"PATH_SEPARATOR"thingy", temp_dir); - test_streq(f, buf); - tor_free(f); - f = get_datadir_fname2_suffix("cache", "thingy", ".foo"); - tor_snprintf(buf, sizeof(buf), - "%s"PATH_SEPARATOR"cache"PATH_SEPARATOR"thingy.foo", temp_dir); - test_streq(f, buf); - tor_free(f); - f = get_datadir_fname_suffix("cache", ".foo"); - tor_snprintf(buf, sizeof(buf), "%s"PATH_SEPARATOR"cache.foo", - temp_dir); - test_streq(f, buf); - - done: - tor_free(f); -} - -/** Test AES-CTR encryption and decryption with IV. */ -static void -test_crypto_aes_iv(void) -{ - crypto_cipher_env_t *cipher; - char *plain, *encrypted1, *encrypted2, *decrypted1, *decrypted2; - char plain_1[1], plain_15[15], plain_16[16], plain_17[17]; - char key1[16], key2[16]; - ssize_t encrypted_size, decrypted_size; - - plain = tor_malloc(4095); - encrypted1 = tor_malloc(4095 + 1 + 16); - encrypted2 = tor_malloc(4095 + 1 + 16); - decrypted1 = tor_malloc(4095 + 1); - decrypted2 = tor_malloc(4095 + 1); - - crypto_rand(plain, 4095); - crypto_rand(key1, 16); - crypto_rand(key2, 16); - crypto_rand(plain_1, 1); - crypto_rand(plain_15, 15); - crypto_rand(plain_16, 16); - crypto_rand(plain_17, 17); - key1[0] = key2[0] + 128; /* Make sure that contents are different. */ - /* Encrypt and decrypt with the same key. */ - cipher = crypto_create_init_cipher(key1, 1); - encrypted_size = crypto_cipher_encrypt_with_iv(cipher, encrypted1, 16 + 4095, - plain, 4095); - crypto_free_cipher_env(cipher); - cipher = NULL; - test_eq(encrypted_size, 16 + 4095); - tor_assert(encrypted_size > 0); /* This is obviously true, since 4111 is - * greater than 0, but its truth is not - * obvious to all analysis tools. */ - cipher = crypto_create_init_cipher(key1, 0); - decrypted_size = crypto_cipher_decrypt_with_iv(cipher, decrypted1, 4095, - encrypted1, encrypted_size); - crypto_free_cipher_env(cipher); - cipher = NULL; - test_eq(decrypted_size, 4095); - tor_assert(decrypted_size > 0); - test_memeq(plain, decrypted1, 4095); - /* Encrypt a second time (with a new random initialization vector). */ - cipher = crypto_create_init_cipher(key1, 1); - encrypted_size = crypto_cipher_encrypt_with_iv(cipher, encrypted2, 16 + 4095, - plain, 4095); - crypto_free_cipher_env(cipher); - cipher = NULL; - test_eq(encrypted_size, 16 + 4095); - tor_assert(encrypted_size > 0); - cipher = crypto_create_init_cipher(key1, 0); - decrypted_size = crypto_cipher_decrypt_with_iv(cipher, decrypted2, 4095, - encrypted2, encrypted_size); - crypto_free_cipher_env(cipher); - cipher = NULL; - test_eq(decrypted_size, 4095); - tor_assert(decrypted_size > 0); - test_memeq(plain, decrypted2, 4095); - test_memneq(encrypted1, encrypted2, encrypted_size); - /* Decrypt with the wrong key. */ - cipher = crypto_create_init_cipher(key2, 0); - decrypted_size = crypto_cipher_decrypt_with_iv(cipher, decrypted2, 4095, - encrypted1, encrypted_size); - crypto_free_cipher_env(cipher); - cipher = NULL; - test_memneq(plain, decrypted2, encrypted_size); - /* Alter the initialization vector. */ - encrypted1[0] += 42; - cipher = crypto_create_init_cipher(key1, 0); - decrypted_size = crypto_cipher_decrypt_with_iv(cipher, decrypted1, 4095, - encrypted1, encrypted_size); - crypto_free_cipher_env(cipher); - cipher = NULL; - test_memneq(plain, decrypted2, 4095); - /* Special length case: 1. */ - cipher = crypto_create_init_cipher(key1, 1); - encrypted_size = crypto_cipher_encrypt_with_iv(cipher, encrypted1, 16 + 1, - plain_1, 1); - crypto_free_cipher_env(cipher); - cipher = NULL; - test_eq(encrypted_size, 16 + 1); - tor_assert(encrypted_size > 0); - cipher = crypto_create_init_cipher(key1, 0); - decrypted_size = crypto_cipher_decrypt_with_iv(cipher, decrypted1, 1, - encrypted1, encrypted_size); - crypto_free_cipher_env(cipher); - cipher = NULL; - test_eq(decrypted_size, 1); - tor_assert(decrypted_size > 0); - test_memeq(plain_1, decrypted1, 1); - /* Special length case: 15. */ - cipher = crypto_create_init_cipher(key1, 1); - encrypted_size = crypto_cipher_encrypt_with_iv(cipher, encrypted1, 16 + 15, - plain_15, 15); - crypto_free_cipher_env(cipher); - cipher = NULL; - test_eq(encrypted_size, 16 + 15); - tor_assert(encrypted_size > 0); - cipher = crypto_create_init_cipher(key1, 0); - decrypted_size = crypto_cipher_decrypt_with_iv(cipher, decrypted1, 15, - encrypted1, encrypted_size); - crypto_free_cipher_env(cipher); - cipher = NULL; - test_eq(decrypted_size, 15); - tor_assert(decrypted_size > 0); - test_memeq(plain_15, decrypted1, 15); - /* Special length case: 16. */ - cipher = crypto_create_init_cipher(key1, 1); - encrypted_size = crypto_cipher_encrypt_with_iv(cipher, encrypted1, 16 + 16, - plain_16, 16); - crypto_free_cipher_env(cipher); - cipher = NULL; - test_eq(encrypted_size, 16 + 16); - tor_assert(encrypted_size > 0); - cipher = crypto_create_init_cipher(key1, 0); - decrypted_size = crypto_cipher_decrypt_with_iv(cipher, decrypted1, 16, - encrypted1, encrypted_size); - crypto_free_cipher_env(cipher); - cipher = NULL; - test_eq(decrypted_size, 16); - tor_assert(decrypted_size > 0); - test_memeq(plain_16, decrypted1, 16); - /* Special length case: 17. */ - cipher = crypto_create_init_cipher(key1, 1); - encrypted_size = crypto_cipher_encrypt_with_iv(cipher, encrypted1, 16 + 17, - plain_17, 17); - crypto_free_cipher_env(cipher); - cipher = NULL; - test_eq(encrypted_size, 16 + 17); - tor_assert(encrypted_size > 0); - cipher = crypto_create_init_cipher(key1, 0); - decrypted_size = crypto_cipher_decrypt_with_iv(cipher, decrypted1, 17, - encrypted1, encrypted_size); - test_eq(decrypted_size, 17); - tor_assert(decrypted_size > 0); - test_memeq(plain_17, decrypted1, 17); - - done: - /* Free memory. */ - tor_free(plain); - tor_free(encrypted1); - tor_free(encrypted2); - tor_free(decrypted1); - tor_free(decrypted2); - if (cipher) - crypto_free_cipher_env(cipher); -} - -/** Test base32 decoding. */ -static void -test_crypto_base32_decode(void) -{ - char plain[60], encoded[96 + 1], decoded[60]; - int res; - crypto_rand(plain, 60); - /* Encode and decode a random string. */ - base32_encode(encoded, 96 + 1, plain, 60); - res = base32_decode(decoded, 60, encoded, 96); - test_eq(res, 0); - test_memeq(plain, decoded, 60); - /* Encode, uppercase, and decode a random string. */ - base32_encode(encoded, 96 + 1, plain, 60); - tor_strupper(encoded); - res = base32_decode(decoded, 60, encoded, 96); - test_eq(res, 0); - test_memeq(plain, decoded, 60); - /* Change encoded string and decode. */ - if (encoded[0] == 'A' || encoded[0] == 'a') - encoded[0] = 'B'; - else - encoded[0] = 'A'; - res = base32_decode(decoded, 60, encoded, 96); - test_eq(res, 0); - test_memneq(plain, decoded, 60); - /* Bad encodings. */ - encoded[0] = '!'; - res = base32_decode(decoded, 60, encoded, 96); - test_assert(res < 0); - - done: - ; -} - -/** Test encoding and parsing of v2 rendezvous service descriptors. */ -static void -test_rend_fns_v2(void) -{ - rend_service_descriptor_t *generated = NULL, *parsed = NULL; - char service_id[DIGEST_LEN]; - char service_id_base32[REND_SERVICE_ID_LEN_BASE32+1]; - const char *next_desc; - smartlist_t *descs = smartlist_create(); - char computed_desc_id[DIGEST_LEN]; - char parsed_desc_id[DIGEST_LEN]; - crypto_pk_env_t *pk1 = NULL, *pk2 = NULL; - time_t now; - char *intro_points_encrypted = NULL; - size_t intro_points_size; - size_t encoded_size; - int i; - pk1 = pk_generate(0); - pk2 = pk_generate(1); - generated = tor_malloc_zero(sizeof(rend_service_descriptor_t)); - generated->pk = crypto_pk_dup_key(pk1); - crypto_pk_get_digest(generated->pk, service_id); - base32_encode(service_id_base32, REND_SERVICE_ID_LEN_BASE32+1, - service_id, REND_SERVICE_ID_LEN); - now = time(NULL); - generated->timestamp = now; - generated->version = 2; - generated->protocols = 42; - generated->intro_nodes = smartlist_create(); - - for (i = 0; i < 3; i++) { - rend_intro_point_t *intro = tor_malloc_zero(sizeof(rend_intro_point_t)); - crypto_pk_env_t *okey = pk_generate(2 + i); - intro->extend_info = tor_malloc_zero(sizeof(extend_info_t)); - intro->extend_info->onion_key = okey; - crypto_pk_get_digest(intro->extend_info->onion_key, - intro->extend_info->identity_digest); - //crypto_rand(info->identity_digest, DIGEST_LEN); /* Would this work? */ - intro->extend_info->nickname[0] = '$'; - base16_encode(intro->extend_info->nickname + 1, - sizeof(intro->extend_info->nickname) - 1, - intro->extend_info->identity_digest, DIGEST_LEN); - /* Does not cover all IP addresses. */ - tor_addr_from_ipv4h(&intro->extend_info->addr, crypto_rand_int(65536)); - intro->extend_info->port = crypto_rand_int(65536); - intro->intro_key = crypto_pk_dup_key(pk2); - smartlist_add(generated->intro_nodes, intro); - } - test_assert(rend_encode_v2_descriptors(descs, generated, now, 0, - REND_NO_AUTH, NULL, NULL) > 0); - test_assert(rend_compute_v2_desc_id(computed_desc_id, service_id_base32, - NULL, now, 0) == 0); - test_memeq(((rend_encoded_v2_service_descriptor_t *) - smartlist_get(descs, 0))->desc_id, computed_desc_id, DIGEST_LEN); - test_assert(rend_parse_v2_service_descriptor(&parsed, parsed_desc_id, - &intro_points_encrypted, - &intro_points_size, - &encoded_size, - &next_desc, - ((rend_encoded_v2_service_descriptor_t *) - smartlist_get(descs, 0))->desc_str) == 0); - test_assert(parsed); - test_memeq(((rend_encoded_v2_service_descriptor_t *) - smartlist_get(descs, 0))->desc_id, parsed_desc_id, DIGEST_LEN); - test_eq(rend_parse_introduction_points(parsed, intro_points_encrypted, - intro_points_size), 3); - test_assert(!crypto_pk_cmp_keys(generated->pk, parsed->pk)); - test_eq(parsed->timestamp, now); - test_eq(parsed->version, 2); - test_eq(parsed->protocols, 42); - test_eq(smartlist_len(parsed->intro_nodes), 3); - for (i = 0; i < smartlist_len(parsed->intro_nodes); i++) { - rend_intro_point_t *par_intro = smartlist_get(parsed->intro_nodes, i), - *gen_intro = smartlist_get(generated->intro_nodes, i); - extend_info_t *par_info = par_intro->extend_info; - extend_info_t *gen_info = gen_intro->extend_info; - test_assert(!crypto_pk_cmp_keys(gen_info->onion_key, par_info->onion_key)); - test_memeq(gen_info->identity_digest, par_info->identity_digest, - DIGEST_LEN); - test_streq(gen_info->nickname, par_info->nickname); - test_assert(tor_addr_eq(&gen_info->addr, &par_info->addr)); - test_eq(gen_info->port, par_info->port); - } - - rend_service_descriptor_free(parsed); - rend_service_descriptor_free(generated); - parsed = generated = NULL; - - done: - if (descs) { - for (i = 0; i < smartlist_len(descs); i++) - rend_encoded_v2_service_descriptor_free(smartlist_get(descs, i)); - smartlist_free(descs); - } - if (parsed) - rend_service_descriptor_free(parsed); - if (generated) - rend_service_descriptor_free(generated); - if (pk1) - crypto_free_pk_env(pk1); - if (pk2) - crypto_free_pk_env(pk2); - tor_free(intro_points_encrypted); -} - -/** Run unit tests for GeoIP code. */ -static void -test_geoip(void) -{ - int i, j; - time_t now = time(NULL); - char *s = NULL; - - /* Populate the DB a bit. Add these in order, since we can't do the final - * 'sort' step. These aren't very good IP addresses, but they're perfectly - * fine uint32_t values. */ - test_eq(0, geoip_parse_entry("10,50,AB")); - test_eq(0, geoip_parse_entry("52,90,XY")); - test_eq(0, geoip_parse_entry("95,100,AB")); - test_eq(0, geoip_parse_entry("\"105\",\"140\",\"ZZ\"")); - test_eq(0, geoip_parse_entry("\"150\",\"190\",\"XY\"")); - test_eq(0, geoip_parse_entry("\"200\",\"250\",\"AB\"")); - - /* We should have 3 countries: ab, xy, zz. */ - test_eq(3, geoip_get_n_countries()); - /* Make sure that country ID actually works. */ -#define NAMEFOR(x) geoip_get_country_name(geoip_get_country_by_ip(x)) - test_streq("ab", NAMEFOR(32)); - test_streq("??", NAMEFOR(5)); - test_streq("??", NAMEFOR(51)); - test_streq("xy", NAMEFOR(150)); - test_streq("xy", NAMEFOR(190)); - test_streq("??", NAMEFOR(2000)); -#undef NAMEFOR - - get_options()->BridgeRelay = 1; - get_options()->BridgeRecordUsageByCountry = 1; - /* Put 9 observations in AB... */ - for (i=32; i < 40; ++i) - geoip_note_client_seen(GEOIP_CLIENT_CONNECT, i, now-7200); - geoip_note_client_seen(GEOIP_CLIENT_CONNECT, 225, now-7200); - /* and 3 observations in XY, several times. */ - for (j=0; j < 10; ++j) - for (i=52; i < 55; ++i) - geoip_note_client_seen(GEOIP_CLIENT_CONNECT, i, now-3600); - /* and 17 observations in ZZ... */ - for (i=110; i < 127; ++i) - geoip_note_client_seen(GEOIP_CLIENT_CONNECT, i, now); - s = geoip_get_client_history(now+5*24*60*60, GEOIP_CLIENT_CONNECT); - test_assert(s); - test_streq("zz=24,ab=16,xy=8", s); - tor_free(s); - - /* Now clear out all the AB observations. */ - geoip_remove_old_clients(now-6000); - s = geoip_get_client_history(now+5*24*60*60, GEOIP_CLIENT_CONNECT); - test_assert(s); - test_streq("zz=24,xy=8", s); - - done: - tor_free(s); -} - -/** For test_array. Declare an CLI-invocable off-by-default function in the - * unit tests, with function name and user-visible name <b>x</b>*/ -#define DISABLED(x) { #x, x, 0, 0, 0 } -/** For test_array. Declare an CLI-invocable unit test function, with function - * name test_<b>x</b>(), and user-visible name <b>x</b> */ -#define ENT(x) { #x, test_ ## x, 0, 0, 1 } -/** For test_array. Declare an CLI-invocable unit test function, with function - * name test_<b>x</b>_<b>y</b>(), and user-visible name - * <b>x</b>/<b>y</b>. This function will be treated as a subentry of <b>x</b>, - * so that invoking <b>x</b> from the CLI invokes this test too. */ -#define SUBENT(x,y) { #x "/" #y, test_ ## x ## _ ## y, 1, 0, 1 } - -/** An array of functions and information for all the unit tests we can run. */ -static struct { - const char *test_name; /**< How does the user refer to this test from the - * command line? */ - void (*test_fn)(void); /**< What function is called to run this test? */ - int is_subent; /**< Is this a subentry of a bigger set of related tests? */ - int selected; /**< Are we planning to run this one? */ - int is_default; /**< If the user doesn't say what tests they want, do they - * get this function by default? */ -} test_array[] = { - ENT(buffers), - ENT(crypto), - SUBENT(crypto, rng), - SUBENT(crypto, aes), - SUBENT(crypto, sha), - SUBENT(crypto, pk), - SUBENT(crypto, dh), - SUBENT(crypto, s2k), - SUBENT(crypto, aes_iv), - SUBENT(crypto, base32_decode), - ENT(util), - SUBENT(util, ip6_helpers), - SUBENT(util, gzip), - SUBENT(util, datadir), - SUBENT(util, smartlist_basic), - SUBENT(util, smartlist_strings), - SUBENT(util, smartlist_overlap), - SUBENT(util, smartlist_digests), - SUBENT(util, smartlist_join), - SUBENT(util, bitarray), - SUBENT(util, digestset), - SUBENT(util, mempool), - SUBENT(util, memarea), - SUBENT(util, strmap), - SUBENT(util, control_formats), - SUBENT(util, pqueue), - SUBENT(util, mmap), - SUBENT(util, threads), - SUBENT(util, order_functions), - SUBENT(util, sscanf), - ENT(onion_handshake), - ENT(dir_format), - ENT(dirutil), - ENT(v3_networkstatus), - ENT(policies), - ENT(rend_fns), - SUBENT(rend_fns, v2), - ENT(geoip), - - DISABLED(bench_aes), - DISABLED(bench_dmap), - { NULL, NULL, 0, 0, 0 }, -}; - -static void syntax(void) ATTR_NORETURN; - -/** Print a syntax usage message, and exit.*/ -static void -syntax(void) -{ - int i; - printf("Syntax:\n" - " test [-v|--verbose] [--warn|--notice|--info|--debug]\n" - " [testname...]\n" - "Recognized tests are:\n"); - for (i = 0; test_array[i].test_name; ++i) { - printf(" %s\n", test_array[i].test_name); - } - - exit(0); -} - -/** Main entry point for unit test code: parse the command line, and run - * some unit tests. */ -int -main(int c, char**v) -{ - or_options_t *options; - char *errmsg = NULL; - int i; - int verbose = 0, any_selected = 0; - int loglevel = LOG_ERR; - -#ifdef USE_DMALLOC - { - int r = CRYPTO_set_mem_ex_functions(_tor_malloc, _tor_realloc, _tor_free); - tor_assert(r); - } -#endif - - update_approx_time(time(NULL)); - options = options_new(); - tor_threads_init(); - init_logging(); - - for (i = 1; i < c; ++i) { - if (!strcmp(v[i], "-v") || !strcmp(v[i], "--verbose")) - verbose++; - else if (!strcmp(v[i], "--warn")) - loglevel = LOG_WARN; - else if (!strcmp(v[i], "--notice")) - loglevel = LOG_NOTICE; - else if (!strcmp(v[i], "--info")) - loglevel = LOG_INFO; - else if (!strcmp(v[i], "--debug")) - loglevel = LOG_DEBUG; - else if (!strcmp(v[i], "--help") || !strcmp(v[i], "-h") || v[i][0] == '-') - syntax(); - else { - int j, found=0; - for (j = 0; test_array[j].test_name; ++j) { - if (!strcmp(v[i], test_array[j].test_name) || - (test_array[j].is_subent && - !strcmpstart(test_array[j].test_name, v[i]) && - test_array[j].test_name[strlen(v[i])] == '/') || - (v[i][0] == '=' && !strcmp(v[i]+1, test_array[j].test_name))) { - test_array[j].selected = 1; - any_selected = 1; - found = 1; - } - } - if (!found) { - printf("Unknown test: %s\n", v[i]); - syntax(); - } - } - } - - if (!any_selected) { - for (i = 0; test_array[i].test_name; ++i) { - test_array[i].selected = test_array[i].is_default; - } - } - - { - log_severity_list_t s; - memset(&s, 0, sizeof(s)); - set_log_severity_config(loglevel, LOG_ERR, &s); - add_stream_log(&s, "", fileno(stdout)); - } - - options->command = CMD_RUN_UNITTESTS; - crypto_global_init(0); - rep_hist_init(); - network_init(); - setup_directory(); - options_init(options); - options->DataDirectory = tor_strdup(temp_dir); - if (set_options(options, &errmsg) < 0) { - printf("Failed to set initial options: %s\n", errmsg); - tor_free(errmsg); - return 1; - } - - crypto_seed_rng(1); - - atexit(remove_directory); - - printf("Running Tor unit tests on %s\n", get_uname()); - - for (i = 0; test_array[i].test_name; ++i) { - if (!test_array[i].selected) - continue; - if (!test_array[i].is_subent) { - printf("\n============================== %s\n",test_array[i].test_name); - } else if (test_array[i].is_subent && verbose) { - printf("\n%s", test_array[i].test_name); - } - test_array[i].test_fn(); - } - puts(""); - - free_pregenerated_keys(); -#ifdef USE_DMALLOC - tor_free_all(0); - dmalloc_log_unfreed(); -#endif - - if (have_failed) - return 1; - else - return 0; -} - diff --git a/src/or/tor_main.c b/src/or/tor_main.c index bb1bf37b4d..117369c565 100644 --- a/src/or/tor_main.c +++ b/src/or/tor_main.c @@ -7,7 +7,7 @@ * built from. This string is generated by a bit of shell kludging int * src/or/Makefile.am, and is usually right. */ -const char tor_svn_revision[] = +const char tor_git_revision[] = #ifndef _MSC_VER #include "micro-revision.i" #endif diff --git a/src/test/Makefile.am b/src/test/Makefile.am new file mode 100644 index 0000000000..546fa2f4b7 --- /dev/null +++ b/src/test/Makefile.am @@ -0,0 +1,30 @@ +TESTS = test + +noinst_PROGRAMS = test + +AM_CPPFLAGS = -DSHARE_DATADIR="\"$(datadir)\"" \ + -DLOCALSTATEDIR="\"$(localstatedir)\"" \ + -DBINDIR="\"$(bindir)\"" \ + -I"$(top_srcdir)/src/or" + +# -L flags need to go in LDFLAGS. -l flags need to go in LDADD. +# This seems to matter nowhere but on windows, but I assure you that it +# matters a lot there, and is quite hard to debug if you forget to do it. + +test_SOURCES = \ + test_data.c \ + test.c \ + test_addr.c \ + test_crypto.c \ + test_dir.c \ + test_containers.c \ + test_util.c \ + tinytest.c + +test_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ \ + @TOR_LDFLAGS_libevent@ +test_LDADD = ../or/libtor.a ../common/libor.a ../common/libor-crypto.a \ + ../common/libor-event.a \ + @TOR_ZLIB_LIBS@ -lm @TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ + +noinst_HEADERS = tinytest.h tinytest_macros.h test.h diff --git a/src/test/test.c b/src/test/test.c new file mode 100644 index 0000000000..2d396b5815 --- /dev/null +++ b/src/test/test.c @@ -0,0 +1,1351 @@ +/* Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2010, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/* Ordinarily defined in tor_main.c; this bit is just here to provide one + * since we're not linking to tor_main.c */ +const char tor_git_revision[] = ""; + +/** + * \file test.c + * \brief Unit tests for many pieces of the lower level Tor modules. + **/ + +#include "orconfig.h" + +#include <stdio.h> +#ifdef HAVE_FCNTL_H +#include <fcntl.h> +#endif + +#ifdef MS_WINDOWS +/* For mkdir() */ +#include <direct.h> +#else +#include <dirent.h> +#endif + +/* These macros pull in declarations for some functions and structures that + * are typically file-private. */ +#define BUFFERS_PRIVATE +#define CONFIG_PRIVATE +#define GEOIP_PRIVATE +#define ROUTER_PRIVATE +#define CIRCUIT_PRIVATE + +/* + * Linux doesn't provide lround in math.h by default, but mac os does... + * It's best just to leave math.h out of the picture entirely. + */ +//#include <math.h> +long int lround(double x); +double fabs(double x); + +#include "or.h" +#include "buffers.h" +#include "circuitbuild.h" +#include "config.h" +#include "connection_edge.h" +#include "geoip.h" +#include "rendcommon.h" +#include "test.h" +#include "torgzip.h" +#include "mempool.h" +#include "memarea.h" +#include "onion.h" +#include "policies.h" +#include "rephist.h" +#include "routerparse.h" + +#ifdef USE_DMALLOC +#include <dmalloc.h> +#include <openssl/crypto.h> +#endif + +/** Set to true if any unit test has failed. Mostly, this is set by the macros + * in test.h */ +int have_failed = 0; + +/** Temporary directory (set up by setup_directory) under which we store all + * our files during testing. */ +static char temp_dir[256]; +static pid_t temp_dir_setup_in_pid = 0; + +/** Select and create the temporary directory we'll use to run our unit tests. + * Store it in <b>temp_dir</b>. Exit immediately if we can't create it. + * idempotent. */ +static void +setup_directory(void) +{ + static int is_setup = 0; + int r; + if (is_setup) return; + +#ifdef MS_WINDOWS + // XXXX + tor_snprintf(temp_dir, sizeof(temp_dir), + "c:\\windows\\temp\\tor_test_%d", (int)getpid()); + r = mkdir(temp_dir); +#else + tor_snprintf(temp_dir, sizeof(temp_dir), "/tmp/tor_test_%d", (int) getpid()); + r = mkdir(temp_dir, 0700); +#endif + if (r) { + fprintf(stderr, "Can't create directory %s:", temp_dir); + perror(""); + exit(1); + } + is_setup = 1; + temp_dir_setup_in_pid = getpid(); +} + +/** Return a filename relative to our testing temporary directory */ +const char * +get_fname(const char *name) +{ + static char buf[1024]; + setup_directory(); + tor_snprintf(buf,sizeof(buf),"%s/%s",temp_dir,name); + return buf; +} + +/** Remove all files stored under the temporary directory, and the directory + * itself. Called by atexit(). */ +static void +remove_directory(void) +{ + smartlist_t *elements; + if (getpid() != temp_dir_setup_in_pid) { + /* Only clean out the tempdir when the main process is exiting. */ + return; + } + elements = tor_listdir(temp_dir); + if (elements) { + SMARTLIST_FOREACH(elements, const char *, cp, + { + size_t len = strlen(cp)+strlen(temp_dir)+16; + char *tmp = tor_malloc(len); + tor_snprintf(tmp, len, "%s"PATH_SEPARATOR"%s", temp_dir, cp); + unlink(tmp); + tor_free(tmp); + }); + SMARTLIST_FOREACH(elements, char *, cp, tor_free(cp)); + smartlist_free(elements); + } + rmdir(temp_dir); +} + +/** Define this if unit tests spend too much time generating public keys*/ +#undef CACHE_GENERATED_KEYS + +static crypto_pk_env_t *pregen_keys[5] = {NULL, NULL, NULL, NULL, NULL}; +#define N_PREGEN_KEYS ((int)(sizeof(pregen_keys)/sizeof(pregen_keys[0]))) + +/** Generate and return a new keypair for use in unit tests. If we're using + * the key cache optimization, we might reuse keys: we only guarantee that + * keys made with distinct values for <b>idx</b> are different. The value of + * <b>idx</b> must be at least 0, and less than N_PREGEN_KEYS. */ +crypto_pk_env_t * +pk_generate(int idx) +{ +#ifdef CACHE_GENERATED_KEYS + tor_assert(idx < N_PREGEN_KEYS); + if (! pregen_keys[idx]) { + pregen_keys[idx] = crypto_new_pk_env(); + tor_assert(!crypto_pk_generate_key(pregen_keys[idx])); + } + return crypto_pk_dup_key(pregen_keys[idx]); +#else + crypto_pk_env_t *result; + (void) idx; + result = crypto_new_pk_env(); + tor_assert(!crypto_pk_generate_key(result)); + return result; +#endif +} + +/** Free all storage used for the cached key optimization. */ +static void +free_pregenerated_keys(void) +{ + unsigned idx; + for (idx = 0; idx < N_PREGEN_KEYS; ++idx) { + if (pregen_keys[idx]) { + crypto_free_pk_env(pregen_keys[idx]); + pregen_keys[idx] = NULL; + } + } +} + +/** Run unit tests for buffers.c */ +static void +test_buffers(void) +{ + char str[256]; + char str2[256]; + + buf_t *buf = NULL, *buf2 = NULL; + const char *cp; + + int j; + size_t r; + + /**** + * buf_new + ****/ + if (!(buf = buf_new())) + test_fail(); + + //test_eq(buf_capacity(buf), 4096); + test_eq(buf_datalen(buf), 0); + + /**** + * General pointer frobbing + */ + for (j=0;j<256;++j) { + str[j] = (char)j; + } + write_to_buf(str, 256, buf); + write_to_buf(str, 256, buf); + test_eq(buf_datalen(buf), 512); + fetch_from_buf(str2, 200, buf); + test_memeq(str, str2, 200); + test_eq(buf_datalen(buf), 312); + memset(str2, 0, sizeof(str2)); + + fetch_from_buf(str2, 256, buf); + test_memeq(str+200, str2, 56); + test_memeq(str, str2+56, 200); + test_eq(buf_datalen(buf), 56); + memset(str2, 0, sizeof(str2)); + /* Okay, now we should be 512 bytes into the 4096-byte buffer. If we add + * another 3584 bytes, we hit the end. */ + for (j=0;j<15;++j) { + write_to_buf(str, 256, buf); + } + assert_buf_ok(buf); + test_eq(buf_datalen(buf), 3896); + fetch_from_buf(str2, 56, buf); + test_eq(buf_datalen(buf), 3840); + test_memeq(str+200, str2, 56); + for (j=0;j<15;++j) { + memset(str2, 0, sizeof(str2)); + fetch_from_buf(str2, 256, buf); + test_memeq(str, str2, 256); + } + test_eq(buf_datalen(buf), 0); + buf_free(buf); + buf = NULL; + + /* Okay, now make sure growing can work. */ + buf = buf_new_with_capacity(16); + //test_eq(buf_capacity(buf), 16); + write_to_buf(str+1, 255, buf); + //test_eq(buf_capacity(buf), 256); + fetch_from_buf(str2, 254, buf); + test_memeq(str+1, str2, 254); + //test_eq(buf_capacity(buf), 256); + assert_buf_ok(buf); + write_to_buf(str, 32, buf); + //test_eq(buf_capacity(buf), 256); + assert_buf_ok(buf); + write_to_buf(str, 256, buf); + assert_buf_ok(buf); + //test_eq(buf_capacity(buf), 512); + test_eq(buf_datalen(buf), 33+256); + fetch_from_buf(str2, 33, buf); + test_eq(*str2, str[255]); + + test_memeq(str2+1, str, 32); + //test_eq(buf_capacity(buf), 512); + test_eq(buf_datalen(buf), 256); + fetch_from_buf(str2, 256, buf); + test_memeq(str, str2, 256); + + /* now try shrinking: case 1. */ + buf_free(buf); + buf = buf_new_with_capacity(33668); + for (j=0;j<67;++j) { + write_to_buf(str,255, buf); + } + //test_eq(buf_capacity(buf), 33668); + test_eq(buf_datalen(buf), 17085); + for (j=0; j < 40; ++j) { + fetch_from_buf(str2, 255,buf); + test_memeq(str2, str, 255); + } + + /* now try shrinking: case 2. */ + buf_free(buf); + buf = buf_new_with_capacity(33668); + for (j=0;j<67;++j) { + write_to_buf(str,255, buf); + } + for (j=0; j < 20; ++j) { + fetch_from_buf(str2, 255,buf); + test_memeq(str2, str, 255); + } + for (j=0;j<80;++j) { + write_to_buf(str,255, buf); + } + //test_eq(buf_capacity(buf),33668); + for (j=0; j < 120; ++j) { + fetch_from_buf(str2, 255,buf); + test_memeq(str2, str, 255); + } + + /* Move from buf to buf. */ + buf_free(buf); + buf = buf_new_with_capacity(4096); + buf2 = buf_new_with_capacity(4096); + for (j=0;j<100;++j) + write_to_buf(str, 255, buf); + test_eq(buf_datalen(buf), 25500); + for (j=0;j<100;++j) { + r = 10; + move_buf_to_buf(buf2, buf, &r); + test_eq(r, 0); + } + test_eq(buf_datalen(buf), 24500); + test_eq(buf_datalen(buf2), 1000); + for (j=0;j<3;++j) { + fetch_from_buf(str2, 255, buf2); + test_memeq(str2, str, 255); + } + r = 8192; /*big move*/ + move_buf_to_buf(buf2, buf, &r); + test_eq(r, 0); + r = 30000; /* incomplete move */ + move_buf_to_buf(buf2, buf, &r); + test_eq(r, 13692); + for (j=0;j<97;++j) { + fetch_from_buf(str2, 255, buf2); + test_memeq(str2, str, 255); + } + buf_free(buf); + buf_free(buf2); + buf = buf2 = NULL; + + buf = buf_new_with_capacity(5); + cp = "Testing. This is a moderately long Testing string."; + for (j = 0; cp[j]; j++) + write_to_buf(cp+j, 1, buf); + test_eq(0, buf_find_string_offset(buf, "Testing", 7)); + test_eq(1, buf_find_string_offset(buf, "esting", 6)); + test_eq(1, buf_find_string_offset(buf, "est", 3)); + test_eq(39, buf_find_string_offset(buf, "ing str", 7)); + test_eq(35, buf_find_string_offset(buf, "Testing str", 11)); + test_eq(32, buf_find_string_offset(buf, "ng ", 3)); + test_eq(43, buf_find_string_offset(buf, "string.", 7)); + test_eq(-1, buf_find_string_offset(buf, "shrdlu", 6)); + test_eq(-1, buf_find_string_offset(buf, "Testing thing", 13)); + test_eq(-1, buf_find_string_offset(buf, "ngx", 3)); + buf_free(buf); + buf = NULL; + +#if 0 + { + int s; + int eof; + int i; + buf_t *buf2; + /**** + * read_to_buf + ****/ + s = open(get_fname("data"), O_WRONLY|O_CREAT|O_TRUNC, 0600); + write(s, str, 256); + close(s); + + s = open(get_fname("data"), O_RDONLY, 0); + eof = 0; + errno = 0; /* XXXX */ + i = read_to_buf(s, 10, buf, &eof); + printf("%s\n", strerror(errno)); + test_eq(i, 10); + test_eq(eof, 0); + //test_eq(buf_capacity(buf), 4096); + test_eq(buf_datalen(buf), 10); + + test_memeq(str, (char*)_buf_peek_raw_buffer(buf), 10); + + /* Test reading 0 bytes. */ + i = read_to_buf(s, 0, buf, &eof); + //test_eq(buf_capacity(buf), 512*1024); + test_eq(buf_datalen(buf), 10); + test_eq(eof, 0); + test_eq(i, 0); + + /* Now test when buffer is filled exactly. */ + buf2 = buf_new_with_capacity(6); + i = read_to_buf(s, 6, buf2, &eof); + //test_eq(buf_capacity(buf2), 6); + test_eq(buf_datalen(buf2), 6); + test_eq(eof, 0); + test_eq(i, 6); + test_memeq(str+10, (char*)_buf_peek_raw_buffer(buf2), 6); + buf_free(buf2); + buf2 = NULL; + + /* Now test when buffer is filled with more data to read. */ + buf2 = buf_new_with_capacity(32); + i = read_to_buf(s, 128, buf2, &eof); + //test_eq(buf_capacity(buf2), 128); + test_eq(buf_datalen(buf2), 32); + test_eq(eof, 0); + test_eq(i, 32); + buf_free(buf2); + buf2 = NULL; + + /* Now read to eof. */ + test_assert(buf_capacity(buf) > 256); + i = read_to_buf(s, 1024, buf, &eof); + test_eq(i, (256-32-10-6)); + test_eq(buf_capacity(buf), MAX_BUF_SIZE); + test_eq(buf_datalen(buf), 256-6-32); + test_memeq(str, (char*)_buf_peek_raw_buffer(buf), 10); /* XXX Check rest. */ + test_eq(eof, 0); + + i = read_to_buf(s, 1024, buf, &eof); + test_eq(i, 0); + test_eq(buf_capacity(buf), MAX_BUF_SIZE); + test_eq(buf_datalen(buf), 256-6-32); + test_eq(eof, 1); + } +#endif + + done: + if (buf) + buf_free(buf); + if (buf2) + buf_free(buf2); +} + +/** Run unit tests for the onion handshake code. */ +static void +test_onion_handshake(void) +{ + /* client-side */ + crypto_dh_env_t *c_dh = NULL; + char c_buf[ONIONSKIN_CHALLENGE_LEN]; + char c_keys[40]; + + /* server-side */ + char s_buf[ONIONSKIN_REPLY_LEN]; + char s_keys[40]; + + /* shared */ + crypto_pk_env_t *pk = NULL; + + pk = pk_generate(0); + + /* client handshake 1. */ + memset(c_buf, 0, ONIONSKIN_CHALLENGE_LEN); + test_assert(! onion_skin_create(pk, &c_dh, c_buf)); + + /* server handshake */ + memset(s_buf, 0, ONIONSKIN_REPLY_LEN); + memset(s_keys, 0, 40); + test_assert(! onion_skin_server_handshake(c_buf, pk, NULL, + s_buf, s_keys, 40)); + + /* client handshake 2 */ + memset(c_keys, 0, 40); + test_assert(! onion_skin_client_handshake(c_dh, s_buf, c_keys, 40)); + + if (memcmp(c_keys, s_keys, 40)) { + puts("Aiiiie"); + exit(1); + } + test_memeq(c_keys, s_keys, 40); + memset(s_buf, 0, 40); + test_memneq(c_keys, s_buf, 40); + + done: + if (c_dh) + crypto_dh_free(c_dh); + if (pk) + crypto_free_pk_env(pk); +} + +static void +test_circuit_timeout(void) +{ + /* Plan: + * 1. Generate 1000 samples + * 2. Estimate parameters + * 3. If difference, repeat + * 4. Save state + * 5. load state + * 6. Estimate parameters + * 7. compare differences + */ + circuit_build_times_t initial; + circuit_build_times_t estimate; + circuit_build_times_t final; + double timeout1, timeout2; + or_state_t state; + char *msg; + int i, runs; + double close_ms; + circuit_build_times_init(&initial); + circuit_build_times_init(&estimate); + circuit_build_times_init(&final); + + memset(&state, 0, sizeof(or_state_t)); + + circuitbuild_running_unit_tests(); +#define timeout0 (build_time_t)(30*1000.0) + initial.Xm = 3000; + circuit_build_times_initial_alpha(&initial, + CBT_DEFAULT_QUANTILE_CUTOFF/100.0, + timeout0); + close_ms = MAX(circuit_build_times_calculate_timeout(&initial, + CBT_DEFAULT_CLOSE_QUANTILE/100.0), + CBT_DEFAULT_TIMEOUT_INITIAL_VALUE); + do { + for (i=0; i < CBT_DEFAULT_MIN_CIRCUITS_TO_OBSERVE; i++) { + build_time_t sample = circuit_build_times_generate_sample(&initial,0,1); + + if (sample > close_ms) { + circuit_build_times_add_time(&estimate, CBT_BUILD_ABANDONED); + } else { + circuit_build_times_add_time(&estimate, sample); + } + } + circuit_build_times_update_alpha(&estimate); + timeout1 = circuit_build_times_calculate_timeout(&estimate, + CBT_DEFAULT_QUANTILE_CUTOFF/100.0); + circuit_build_times_set_timeout(&estimate); + log_notice(LD_CIRC, "Timeout1 is %lf, Xm is %d", timeout1, estimate.Xm); + /* 2% error */ + } while (fabs(circuit_build_times_cdf(&initial, timeout0) - + circuit_build_times_cdf(&initial, timeout1)) > 0.02); + + test_assert(estimate.total_build_times <= CBT_NCIRCUITS_TO_OBSERVE); + + circuit_build_times_update_state(&estimate, &state); + test_assert(circuit_build_times_parse_state(&final, &state, &msg) == 0); + + circuit_build_times_update_alpha(&final); + timeout2 = circuit_build_times_calculate_timeout(&final, + CBT_DEFAULT_QUANTILE_CUTOFF/100.0); + + circuit_build_times_set_timeout(&final); + log_notice(LD_CIRC, "Timeout2 is %lf, Xm is %d", timeout2, final.Xm); + + /* 5% here because some accuracy is lost due to histogram conversion */ + test_assert(fabs(circuit_build_times_cdf(&initial, timeout0) - + circuit_build_times_cdf(&initial, timeout2)) < 0.05); + + for (runs = 0; runs < 50; runs++) { + int build_times_idx = 0; + int total_build_times = 0; + + final.close_ms = final.timeout_ms = CBT_DEFAULT_TIMEOUT_INITIAL_VALUE; + estimate.close_ms = estimate.timeout_ms + = CBT_DEFAULT_TIMEOUT_INITIAL_VALUE; + + for (i = 0; i < CBT_DEFAULT_RECENT_CIRCUITS*2; i++) { + circuit_build_times_network_circ_success(&estimate); + circuit_build_times_add_time(&estimate, + circuit_build_times_generate_sample(&estimate, 0, + CBT_DEFAULT_QUANTILE_CUTOFF/100.0)); + + circuit_build_times_network_circ_success(&estimate); + circuit_build_times_add_time(&final, + circuit_build_times_generate_sample(&final, 0, + CBT_DEFAULT_QUANTILE_CUTOFF/100.0)); + } + + test_assert(!circuit_build_times_network_check_changed(&estimate)); + test_assert(!circuit_build_times_network_check_changed(&final)); + + /* Reset liveness to be non-live */ + final.liveness.network_last_live = 0; + estimate.liveness.network_last_live = 0; + + build_times_idx = estimate.build_times_idx; + total_build_times = estimate.total_build_times; + for (i = 0; i < CBT_NETWORK_NONLIVE_TIMEOUT_COUNT; i++) { + test_assert(circuit_build_times_network_check_live(&estimate)); + test_assert(circuit_build_times_network_check_live(&final)); + + circuit_build_times_count_close(&estimate, 0, + (time_t)(approx_time()-estimate.close_ms/1000.0-1)); + circuit_build_times_count_close(&final, 0, + (time_t)(approx_time()-final.close_ms/1000.0-1)); + } + + test_assert(!circuit_build_times_network_check_live(&estimate)); + test_assert(!circuit_build_times_network_check_live(&final)); + + for ( ; i < CBT_NETWORK_NONLIVE_DISCARD_COUNT; i++) { + circuit_build_times_count_close(&estimate, 0, + (time_t)(approx_time()-estimate.close_ms/1000.0-1)); + + if (i < CBT_NETWORK_NONLIVE_DISCARD_COUNT-1) { + circuit_build_times_count_close(&final, 0, + (time_t)(approx_time()-final.close_ms/1000.0-1)); + } + } + + test_assert(!circuit_build_times_network_check_live(&estimate)); + test_assert(!circuit_build_times_network_check_live(&final)); + + log_info(LD_CIRC, "idx: %d %d, tot: %d %d", + build_times_idx, estimate.build_times_idx, + total_build_times, estimate.total_build_times); + + /* Check rollback index. Should match top of loop. */ + test_assert(build_times_idx == estimate.build_times_idx); + // This can fail if estimate.total_build_times == 1000, because + // in that case, rewind actually causes us to lose timeouts + if (total_build_times != CBT_NCIRCUITS_TO_OBSERVE) + test_assert(total_build_times == estimate.total_build_times); + + /* Now simulate that the network has become live and we need + * a change */ + circuit_build_times_network_is_live(&estimate); + circuit_build_times_network_is_live(&final); + + for (i = 0; i < CBT_DEFAULT_MAX_RECENT_TIMEOUT_COUNT; i++) { + circuit_build_times_count_timeout(&estimate, 1); + + if (i < CBT_DEFAULT_MAX_RECENT_TIMEOUT_COUNT-1) { + circuit_build_times_count_timeout(&final, 1); + } + } + + test_assert(estimate.liveness.after_firsthop_idx == 0); + test_assert(final.liveness.after_firsthop_idx == + CBT_DEFAULT_MAX_RECENT_TIMEOUT_COUNT-1); + + test_assert(circuit_build_times_network_check_live(&estimate)); + test_assert(circuit_build_times_network_check_live(&final)); + + circuit_build_times_count_timeout(&final, 1); + } + +done: + return; +} + +/** Helper: Parse the exit policy string in <b>policy_str</b>, and make sure + * that policies_summarize() produces the string <b>expected_summary</b> from + * it. */ +static void +test_policy_summary_helper(const char *policy_str, + const char *expected_summary) +{ + config_line_t line; + smartlist_t *policy = smartlist_create(); + char *summary = NULL; + int r; + + line.key = (char*)"foo"; + line.value = (char *)policy_str; + line.next = NULL; + + r = policies_parse_exit_policy(&line, &policy, 0, NULL, 1); + test_eq(r, 0); + summary = policy_summarize(policy); + + test_assert(summary != NULL); + test_streq(summary, expected_summary); + + done: + tor_free(summary); + if (policy) + addr_policy_list_free(policy); +} + +/** Run unit tests for generating summary lines of exit policies */ +static void +test_policies(void) +{ + int i; + smartlist_t *policy = NULL, *policy2 = NULL, *policy3 = NULL, + *policy4 = NULL, *policy5 = NULL, *policy6 = NULL, + *policy7 = NULL; + addr_policy_t *p; + tor_addr_t tar; + config_line_t line; + smartlist_t *sm = NULL; + char *policy_str = NULL; + + policy = smartlist_create(); + + p = router_parse_addr_policy_item_from_string("reject 192.168.0.0/16:*",-1); + test_assert(p != NULL); + test_eq(ADDR_POLICY_REJECT, p->policy_type); + tor_addr_from_ipv4h(&tar, 0xc0a80000u); + test_eq(0, tor_addr_compare(&p->addr, &tar, CMP_EXACT)); + test_eq(16, p->maskbits); + test_eq(1, p->prt_min); + test_eq(65535, p->prt_max); + + smartlist_add(policy, p); + + test_assert(ADDR_POLICY_ACCEPTED == + compare_addr_to_addr_policy(0x01020304u, 2, policy)); + test_assert(ADDR_POLICY_PROBABLY_ACCEPTED == + compare_addr_to_addr_policy(0, 2, policy)); + test_assert(ADDR_POLICY_REJECTED == + compare_addr_to_addr_policy(0xc0a80102, 2, policy)); + + test_assert(0 == policies_parse_exit_policy(NULL, &policy2, 1, NULL, 1)); + test_assert(policy2); + + policy3 = smartlist_create(); + p = router_parse_addr_policy_item_from_string("reject *:*",-1); + test_assert(p != NULL); + smartlist_add(policy3, p); + p = router_parse_addr_policy_item_from_string("accept *:*",-1); + test_assert(p != NULL); + smartlist_add(policy3, p); + + policy4 = smartlist_create(); + p = router_parse_addr_policy_item_from_string("accept *:443",-1); + test_assert(p != NULL); + smartlist_add(policy4, p); + p = router_parse_addr_policy_item_from_string("accept *:443",-1); + test_assert(p != NULL); + smartlist_add(policy4, p); + + policy5 = smartlist_create(); + p = router_parse_addr_policy_item_from_string("reject 0.0.0.0/8:*",-1); + test_assert(p != NULL); + smartlist_add(policy5, p); + p = router_parse_addr_policy_item_from_string("reject 169.254.0.0/16:*",-1); + test_assert(p != NULL); + smartlist_add(policy5, p); + p = router_parse_addr_policy_item_from_string("reject 127.0.0.0/8:*",-1); + test_assert(p != NULL); + smartlist_add(policy5, p); + p = router_parse_addr_policy_item_from_string("reject 192.168.0.0/16:*",-1); + test_assert(p != NULL); + smartlist_add(policy5, p); + p = router_parse_addr_policy_item_from_string("reject 10.0.0.0/8:*",-1); + test_assert(p != NULL); + smartlist_add(policy5, p); + p = router_parse_addr_policy_item_from_string("reject 172.16.0.0/12:*",-1); + test_assert(p != NULL); + smartlist_add(policy5, p); + p = router_parse_addr_policy_item_from_string("reject 80.190.250.90:*",-1); + test_assert(p != NULL); + smartlist_add(policy5, p); + p = router_parse_addr_policy_item_from_string("reject *:1-65534",-1); + test_assert(p != NULL); + smartlist_add(policy5, p); + p = router_parse_addr_policy_item_from_string("reject *:65535",-1); + test_assert(p != NULL); + smartlist_add(policy5, p); + p = router_parse_addr_policy_item_from_string("accept *:1-65535",-1); + test_assert(p != NULL); + smartlist_add(policy5, p); + + policy6 = smartlist_create(); + p = router_parse_addr_policy_item_from_string("accept 43.3.0.0/9:*",-1); + test_assert(p != NULL); + smartlist_add(policy6, p); + + policy7 = smartlist_create(); + p = router_parse_addr_policy_item_from_string("accept 0.0.0.0/8:*",-1); + test_assert(p != NULL); + smartlist_add(policy7, p); + + test_assert(!exit_policy_is_general_exit(policy)); + test_assert(exit_policy_is_general_exit(policy2)); + test_assert(!exit_policy_is_general_exit(NULL)); + test_assert(!exit_policy_is_general_exit(policy3)); + test_assert(!exit_policy_is_general_exit(policy4)); + test_assert(!exit_policy_is_general_exit(policy5)); + test_assert(!exit_policy_is_general_exit(policy6)); + test_assert(!exit_policy_is_general_exit(policy7)); + + test_assert(cmp_addr_policies(policy, policy2)); + test_assert(cmp_addr_policies(policy, NULL)); + test_assert(!cmp_addr_policies(policy2, policy2)); + test_assert(!cmp_addr_policies(NULL, NULL)); + + test_assert(!policy_is_reject_star(policy2)); + test_assert(policy_is_reject_star(policy)); + test_assert(policy_is_reject_star(NULL)); + + addr_policy_list_free(policy); + policy = NULL; + + /* make sure compacting logic works. */ + policy = NULL; + line.key = (char*)"foo"; + line.value = (char*)"accept *:80,reject private:*,reject *:*"; + line.next = NULL; + test_assert(0 == policies_parse_exit_policy(&line, &policy, 0, NULL, 1)); + test_assert(policy); + //test_streq(policy->string, "accept *:80"); + //test_streq(policy->next->string, "reject *:*"); + test_eq(smartlist_len(policy), 2); + + /* test policy summaries */ + /* check if we properly ignore private IP addresses */ + test_policy_summary_helper("reject 192.168.0.0/16:*," + "reject 0.0.0.0/8:*," + "reject 10.0.0.0/8:*," + "accept *:10-30," + "accept *:90," + "reject *:*", + "accept 10-30,90"); + /* check all accept policies, and proper counting of rejects */ + test_policy_summary_helper("reject 11.0.0.0/9:80," + "reject 12.0.0.0/9:80," + "reject 13.0.0.0/9:80," + "reject 14.0.0.0/9:80," + "accept *:*", "accept 1-65535"); + test_policy_summary_helper("reject 11.0.0.0/9:80," + "reject 12.0.0.0/9:80," + "reject 13.0.0.0/9:80," + "reject 14.0.0.0/9:80," + "reject 15.0.0.0:81," + "accept *:*", "accept 1-65535"); + test_policy_summary_helper("reject 11.0.0.0/9:80," + "reject 12.0.0.0/9:80," + "reject 13.0.0.0/9:80," + "reject 14.0.0.0/9:80," + "reject 15.0.0.0:80," + "accept *:*", + "reject 80"); + /* no exits */ + test_policy_summary_helper("accept 11.0.0.0/9:80," + "reject *:*", + "reject 1-65535"); + /* port merging */ + test_policy_summary_helper("accept *:80," + "accept *:81," + "accept *:100-110," + "accept *:111," + "reject *:*", + "accept 80-81,100-111"); + /* border ports */ + test_policy_summary_helper("accept *:1," + "accept *:3," + "accept *:65535," + "reject *:*", + "accept 1,3,65535"); + /* holes */ + test_policy_summary_helper("accept *:1," + "accept *:3," + "accept *:5," + "accept *:7," + "reject *:*", + "accept 1,3,5,7"); + test_policy_summary_helper("reject *:1," + "reject *:3," + "reject *:5," + "reject *:7," + "accept *:*", + "reject 1,3,5,7"); + + /* truncation ports */ + sm = smartlist_create(); + for (i=1; i<2000; i+=2) { + char buf[POLICY_BUF_LEN]; + tor_snprintf(buf, sizeof(buf), "reject *:%d", i); + smartlist_add(sm, tor_strdup(buf)); + } + smartlist_add(sm, tor_strdup("accept *:*")); + policy_str = smartlist_join_strings(sm, ",", 0, NULL); + test_policy_summary_helper( policy_str, + "accept 2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40,42,44," + "46,48,50,52,54,56,58,60,62,64,66,68,70,72,74,76,78,80,82,84,86,88,90," + "92,94,96,98,100,102,104,106,108,110,112,114,116,118,120,122,124,126,128," + "130,132,134,136,138,140,142,144,146,148,150,152,154,156,158,160,162,164," + "166,168,170,172,174,176,178,180,182,184,186,188,190,192,194,196,198,200," + "202,204,206,208,210,212,214,216,218,220,222,224,226,228,230,232,234,236," + "238,240,242,244,246,248,250,252,254,256,258,260,262,264,266,268,270,272," + "274,276,278,280,282,284,286,288,290,292,294,296,298,300,302,304,306,308," + "310,312,314,316,318,320,322,324,326,328,330,332,334,336,338,340,342,344," + "346,348,350,352,354,356,358,360,362,364,366,368,370,372,374,376,378,380," + "382,384,386,388,390,392,394,396,398,400,402,404,406,408,410,412,414,416," + "418,420,422,424,426,428,430,432,434,436,438,440,442,444,446,448,450,452," + "454,456,458,460,462,464,466,468,470,472,474,476,478,480,482,484,486,488," + "490,492,494,496,498,500,502,504,506,508,510,512,514,516,518,520,522"); + + done: + addr_policy_list_free(policy); + addr_policy_list_free(policy2); + addr_policy_list_free(policy3); + addr_policy_list_free(policy4); + addr_policy_list_free(policy5); + addr_policy_list_free(policy6); + addr_policy_list_free(policy7); + tor_free(policy_str); + if (sm) { + SMARTLIST_FOREACH(sm, char *, s, tor_free(s)); + smartlist_free(sm); + } +} + +/** Run AES performance benchmarks. */ +static void +bench_aes(void) +{ + int len, i; + char *b1, *b2; + crypto_cipher_env_t *c; + struct timeval start, end; + const int iters = 100000; + uint64_t nsec; + c = crypto_new_cipher_env(); + crypto_cipher_generate_key(c); + crypto_cipher_encrypt_init_cipher(c); + for (len = 1; len <= 8192; len *= 2) { + b1 = tor_malloc_zero(len); + b2 = tor_malloc_zero(len); + tor_gettimeofday(&start); + for (i = 0; i < iters; ++i) { + crypto_cipher_encrypt(c, b1, b2, len); + } + tor_gettimeofday(&end); + tor_free(b1); + tor_free(b2); + nsec = (uint64_t) tv_udiff(&start,&end); + nsec *= 1000; + nsec /= (iters*len); + printf("%d bytes: "U64_FORMAT" nsec per byte\n", len, + U64_PRINTF_ARG(nsec)); + } + crypto_free_cipher_env(c); +} + +/** Run digestmap_t performance benchmarks. */ +static void +bench_dmap(void) +{ + smartlist_t *sl = smartlist_create(); + smartlist_t *sl2 = smartlist_create(); + struct timeval start, end, pt2, pt3, pt4; + const int iters = 10000; + const int elts = 4000; + const int fpostests = 1000000; + char d[20]; + int i,n=0, fp = 0; + digestmap_t *dm = digestmap_new(); + digestset_t *ds = digestset_new(elts); + + for (i = 0; i < elts; ++i) { + crypto_rand(d, 20); + smartlist_add(sl, tor_memdup(d, 20)); + } + for (i = 0; i < elts; ++i) { + crypto_rand(d, 20); + smartlist_add(sl2, tor_memdup(d, 20)); + } + printf("nbits=%d\n", ds->mask+1); + + tor_gettimeofday(&start); + for (i = 0; i < iters; ++i) { + SMARTLIST_FOREACH(sl, const char *, cp, digestmap_set(dm, cp, (void*)1)); + } + tor_gettimeofday(&pt2); + for (i = 0; i < iters; ++i) { + SMARTLIST_FOREACH(sl, const char *, cp, digestmap_get(dm, cp)); + SMARTLIST_FOREACH(sl2, const char *, cp, digestmap_get(dm, cp)); + } + tor_gettimeofday(&pt3); + for (i = 0; i < iters; ++i) { + SMARTLIST_FOREACH(sl, const char *, cp, digestset_add(ds, cp)); + } + tor_gettimeofday(&pt4); + for (i = 0; i < iters; ++i) { + SMARTLIST_FOREACH(sl, const char *, cp, n += digestset_isin(ds, cp)); + SMARTLIST_FOREACH(sl2, const char *, cp, n += digestset_isin(ds, cp)); + } + tor_gettimeofday(&end); + + for (i = 0; i < fpostests; ++i) { + crypto_rand(d, 20); + if (digestset_isin(ds, d)) ++fp; + } + + printf("%ld\n",(unsigned long)tv_udiff(&start, &pt2)); + printf("%ld\n",(unsigned long)tv_udiff(&pt2, &pt3)); + printf("%ld\n",(unsigned long)tv_udiff(&pt3, &pt4)); + printf("%ld\n",(unsigned long)tv_udiff(&pt4, &end)); + printf("-- %d\n", n); + printf("++ %f\n", fp/(double)fpostests); + digestmap_free(dm, NULL); + digestset_free(ds); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + SMARTLIST_FOREACH(sl2, char *, cp, tor_free(cp)); + smartlist_free(sl); + smartlist_free(sl2); +} + +/** Test encoding and parsing of rendezvous service descriptors. */ +static void +test_rend_fns(void) +{ + rend_service_descriptor_t *generated = NULL, *parsed = NULL; + char service_id[DIGEST_LEN]; + char service_id_base32[REND_SERVICE_ID_LEN_BASE32+1]; + const char *next_desc; + smartlist_t *descs = smartlist_create(); + char computed_desc_id[DIGEST_LEN]; + char parsed_desc_id[DIGEST_LEN]; + crypto_pk_env_t *pk1 = NULL, *pk2 = NULL; + time_t now; + char *intro_points_encrypted = NULL; + size_t intro_points_size; + size_t encoded_size; + int i; + char address1[] = "fooaddress.onion"; + char address2[] = "aaaaaaaaaaaaaaaa.onion"; + char address3[] = "fooaddress.exit"; + char address4[] = "www.torproject.org"; + + test_assert(BAD_HOSTNAME == parse_extended_hostname(address1, 1)); + test_assert(ONION_HOSTNAME == parse_extended_hostname(address2, 1)); + test_assert(EXIT_HOSTNAME == parse_extended_hostname(address3, 1)); + test_assert(NORMAL_HOSTNAME == parse_extended_hostname(address4, 1)); + + pk1 = pk_generate(0); + pk2 = pk_generate(1); + generated = tor_malloc_zero(sizeof(rend_service_descriptor_t)); + generated->pk = crypto_pk_dup_key(pk1); + crypto_pk_get_digest(generated->pk, service_id); + base32_encode(service_id_base32, REND_SERVICE_ID_LEN_BASE32+1, + service_id, REND_SERVICE_ID_LEN); + now = time(NULL); + generated->timestamp = now; + generated->version = 2; + generated->protocols = 42; + generated->intro_nodes = smartlist_create(); + + for (i = 0; i < 3; i++) { + rend_intro_point_t *intro = tor_malloc_zero(sizeof(rend_intro_point_t)); + crypto_pk_env_t *okey = pk_generate(2 + i); + intro->extend_info = tor_malloc_zero(sizeof(extend_info_t)); + intro->extend_info->onion_key = okey; + crypto_pk_get_digest(intro->extend_info->onion_key, + intro->extend_info->identity_digest); + //crypto_rand(info->identity_digest, DIGEST_LEN); /* Would this work? */ + intro->extend_info->nickname[0] = '$'; + base16_encode(intro->extend_info->nickname + 1, + sizeof(intro->extend_info->nickname) - 1, + intro->extend_info->identity_digest, DIGEST_LEN); + /* Does not cover all IP addresses. */ + tor_addr_from_ipv4h(&intro->extend_info->addr, crypto_rand_int(65536)); + intro->extend_info->port = 1 + crypto_rand_int(65535); + intro->intro_key = crypto_pk_dup_key(pk2); + smartlist_add(generated->intro_nodes, intro); + } + test_assert(rend_encode_v2_descriptors(descs, generated, now, 0, + REND_NO_AUTH, NULL, NULL) > 0); + test_assert(rend_compute_v2_desc_id(computed_desc_id, service_id_base32, + NULL, now, 0) == 0); + test_memeq(((rend_encoded_v2_service_descriptor_t *) + smartlist_get(descs, 0))->desc_id, computed_desc_id, DIGEST_LEN); + test_assert(rend_parse_v2_service_descriptor(&parsed, parsed_desc_id, + &intro_points_encrypted, + &intro_points_size, + &encoded_size, + &next_desc, + ((rend_encoded_v2_service_descriptor_t *) + smartlist_get(descs, 0))->desc_str) == 0); + test_assert(parsed); + test_memeq(((rend_encoded_v2_service_descriptor_t *) + smartlist_get(descs, 0))->desc_id, parsed_desc_id, DIGEST_LEN); + test_eq(rend_parse_introduction_points(parsed, intro_points_encrypted, + intro_points_size), 3); + test_assert(!crypto_pk_cmp_keys(generated->pk, parsed->pk)); + test_eq(parsed->timestamp, now); + test_eq(parsed->version, 2); + test_eq(parsed->protocols, 42); + test_eq(smartlist_len(parsed->intro_nodes), 3); + for (i = 0; i < smartlist_len(parsed->intro_nodes); i++) { + rend_intro_point_t *par_intro = smartlist_get(parsed->intro_nodes, i), + *gen_intro = smartlist_get(generated->intro_nodes, i); + extend_info_t *par_info = par_intro->extend_info; + extend_info_t *gen_info = gen_intro->extend_info; + test_assert(!crypto_pk_cmp_keys(gen_info->onion_key, par_info->onion_key)); + test_memeq(gen_info->identity_digest, par_info->identity_digest, + DIGEST_LEN); + test_streq(gen_info->nickname, par_info->nickname); + test_assert(tor_addr_eq(&gen_info->addr, &par_info->addr)); + test_eq(gen_info->port, par_info->port); + } + + rend_service_descriptor_free(parsed); + rend_service_descriptor_free(generated); + parsed = generated = NULL; + + done: + if (descs) { + for (i = 0; i < smartlist_len(descs); i++) + rend_encoded_v2_service_descriptor_free(smartlist_get(descs, i)); + smartlist_free(descs); + } + if (parsed) + rend_service_descriptor_free(parsed); + if (generated) + rend_service_descriptor_free(generated); + if (pk1) + crypto_free_pk_env(pk1); + if (pk2) + crypto_free_pk_env(pk2); + tor_free(intro_points_encrypted); +} + +/** Run unit tests for GeoIP code. */ +static void +test_geoip(void) +{ + int i, j; + time_t now = time(NULL); + char *s = NULL; + + /* Populate the DB a bit. Add these in order, since we can't do the final + * 'sort' step. These aren't very good IP addresses, but they're perfectly + * fine uint32_t values. */ + test_eq(0, geoip_parse_entry("10,50,AB")); + test_eq(0, geoip_parse_entry("52,90,XY")); + test_eq(0, geoip_parse_entry("95,100,AB")); + test_eq(0, geoip_parse_entry("\"105\",\"140\",\"ZZ\"")); + test_eq(0, geoip_parse_entry("\"150\",\"190\",\"XY\"")); + test_eq(0, geoip_parse_entry("\"200\",\"250\",\"AB\"")); + + /* We should have 4 countries: ??, ab, xy, zz. */ + test_eq(4, geoip_get_n_countries()); + /* Make sure that country ID actually works. */ +#define NAMEFOR(x) geoip_get_country_name(geoip_get_country_by_ip(x)) + test_streq("??", NAMEFOR(3)); + test_eq(0, geoip_get_country_by_ip(3)); + test_streq("ab", NAMEFOR(32)); + test_streq("??", NAMEFOR(5)); + test_streq("??", NAMEFOR(51)); + test_streq("xy", NAMEFOR(150)); + test_streq("xy", NAMEFOR(190)); + test_streq("??", NAMEFOR(2000)); +#undef NAMEFOR + + get_options()->BridgeRelay = 1; + get_options()->BridgeRecordUsageByCountry = 1; + /* Put 9 observations in AB... */ + for (i=32; i < 40; ++i) + geoip_note_client_seen(GEOIP_CLIENT_CONNECT, i, now-7200); + geoip_note_client_seen(GEOIP_CLIENT_CONNECT, 225, now-7200); + /* and 3 observations in XY, several times. */ + for (j=0; j < 10; ++j) + for (i=52; i < 55; ++i) + geoip_note_client_seen(GEOIP_CLIENT_CONNECT, i, now-3600); + /* and 17 observations in ZZ... */ + for (i=110; i < 127; ++i) + geoip_note_client_seen(GEOIP_CLIENT_CONNECT, i, now); + s = geoip_get_client_history(GEOIP_CLIENT_CONNECT); + test_assert(s); + test_streq("zz=24,ab=16,xy=8", s); + tor_free(s); + + /* Now clear out all the AB observations. */ + geoip_remove_old_clients(now-6000); + s = geoip_get_client_history(GEOIP_CLIENT_CONNECT); + test_assert(s); + test_streq("zz=24,xy=8", s); + + done: + tor_free(s); +} + +/** Run unit tests for stats code. */ +static void +test_stats(void) +{ + time_t now = 1281533250; /* 2010-08-11 13:27:30 UTC */ + char *s = NULL; + + /* We shouldn't collect exit stats without initializing them. */ + rep_hist_note_exit_stream_opened(80); + rep_hist_note_exit_bytes(80, 100, 10000); + s = rep_hist_format_exit_stats(now + 86400); + test_assert(!s); + + /* Initialize stats, note some streams and bytes, and generate history + * string. */ + rep_hist_exit_stats_init(now); + rep_hist_note_exit_stream_opened(80); + rep_hist_note_exit_bytes(80, 100, 10000); + rep_hist_note_exit_stream_opened(443); + rep_hist_note_exit_bytes(443, 100, 10000); + rep_hist_note_exit_bytes(443, 100, 10000); + s = rep_hist_format_exit_stats(now + 86400); + test_streq("exit-stats-end 2010-08-12 13:27:30 (86400 s)\n" + "exit-kibibytes-written 80=1,443=1,other=0\n" + "exit-kibibytes-read 80=10,443=20,other=0\n" + "exit-streams-opened 80=4,443=4,other=0\n", s); + tor_free(s); + + /* Stop collecting stats, add some bytes, and ensure we don't generate + * a history string. */ + rep_hist_exit_stats_term(); + rep_hist_note_exit_bytes(80, 100, 10000); + s = rep_hist_format_exit_stats(now + 86400); + test_assert(!s); + + /* Re-start stats, add some bytes, reset stats, and see what history we + * get when observing no streams or bytes at all. */ + rep_hist_exit_stats_init(now); + rep_hist_note_exit_stream_opened(80); + rep_hist_note_exit_bytes(80, 100, 10000); + rep_hist_reset_exit_stats(now); + s = rep_hist_format_exit_stats(now + 86400); + test_streq("exit-stats-end 2010-08-12 13:27:30 (86400 s)\n" + "exit-kibibytes-written other=0\n" + "exit-kibibytes-read other=0\n" + "exit-streams-opened other=0\n", s); + + done: + tor_free(s); +} + +static void * +legacy_test_setup(const struct testcase_t *testcase) +{ + return testcase->setup_data; +} + +void +legacy_test_helper(void *data) +{ + void (*fn)(void) = data; + fn(); +} + +static int +legacy_test_cleanup(const struct testcase_t *testcase, void *ptr) +{ + (void)ptr; + (void)testcase; + return 1; +} + +const struct testcase_setup_t legacy_setup = { + legacy_test_setup, legacy_test_cleanup +}; + +#define ENT(name) \ + { #name, legacy_test_helper, 0, &legacy_setup, test_ ## name } +#define SUBENT(group, name) \ + { #group "_" #name, legacy_test_helper, 0, &legacy_setup, \ + test_ ## group ## _ ## name } +#define DISABLED(name) \ + { #name, legacy_test_helper, TT_SKIP, &legacy_setup, name } +#define FORK(name) \ + { #name, legacy_test_helper, TT_FORK, &legacy_setup, test_ ## name } + +static struct testcase_t test_array[] = { + ENT(buffers), + ENT(onion_handshake), + ENT(circuit_timeout), + ENT(policies), + ENT(rend_fns), + ENT(geoip), + FORK(stats), + + DISABLED(bench_aes), + DISABLED(bench_dmap), + END_OF_TESTCASES +}; + +extern struct testcase_t addr_tests[]; +extern struct testcase_t crypto_tests[]; +extern struct testcase_t container_tests[]; +extern struct testcase_t util_tests[]; +extern struct testcase_t dir_tests[]; + +static struct testgroup_t testgroups[] = { + { "", test_array }, + { "addr/", addr_tests }, + { "crypto/", crypto_tests }, + { "container/", container_tests }, + { "util/", util_tests }, + { "dir/", dir_tests }, + END_OF_GROUPS +}; + +/** Main entry point for unit test code: parse the command line, and run + * some unit tests. */ +int +main(int c, const char **v) +{ + or_options_t *options; + char *errmsg = NULL; + int i, i_out; + int loglevel = LOG_ERR; + +#ifdef USE_DMALLOC + { + int r = CRYPTO_set_mem_ex_functions(_tor_malloc, _tor_realloc, _tor_free); + tor_assert(r); + } +#endif + + update_approx_time(time(NULL)); + options = options_new(); + tor_threads_init(); + init_logging(); + + for (i_out = i = 1; i < c; ++i) { + if (!strcmp(v[i], "--warn")) { + loglevel = LOG_WARN; + } else if (!strcmp(v[i], "--notice")) { + loglevel = LOG_NOTICE; + } else if (!strcmp(v[i], "--info")) { + loglevel = LOG_INFO; + } else if (!strcmp(v[i], "--debug")) { + loglevel = LOG_DEBUG; + } else { + v[i_out++] = v[i]; + } + } + c = i_out; + + { + log_severity_list_t s; + memset(&s, 0, sizeof(s)); + set_log_severity_config(loglevel, LOG_ERR, &s); + add_stream_log(&s, "", fileno(stdout)); + } + + options->command = CMD_RUN_UNITTESTS; + crypto_global_init(0, NULL, NULL); + rep_hist_init(); + network_init(); + setup_directory(); + options_init(options); + options->DataDirectory = tor_strdup(temp_dir); + options->EntryStatistics = 1; + if (set_options(options, &errmsg) < 0) { + printf("Failed to set initial options: %s\n", errmsg); + tor_free(errmsg); + return 1; + } + + crypto_seed_rng(1); + + atexit(remove_directory); + + have_failed = (tinytest_main(c, v, testgroups) != 0); + + free_pregenerated_keys(); +#ifdef USE_DMALLOC + tor_free_all(0); + dmalloc_log_unfreed(); +#endif + + if (have_failed) + return 1; + else + return 0; +} + diff --git a/src/test/test.h b/src/test/test.h new file mode 100644 index 0000000000..550c57a812 --- /dev/null +++ b/src/test/test.h @@ -0,0 +1,75 @@ +/* Copyright (c) 2001-2003, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2010, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef _TOR_TEST_H +#define _TOR_TEST_H + +/** + * \file test.h + * \brief Macros and functions used by unit tests. + */ + +#include "compat.h" +#include "tinytest.h" +#define TT_EXIT_TEST_FUNCTION STMT_BEGIN goto done; STMT_END +#include "tinytest_macros.h" + +#ifdef __GNUC__ +#define PRETTY_FUNCTION __PRETTY_FUNCTION__ +#else +#define PRETTY_FUNCTION "" +#endif + +#define test_fail_msg(msg) TT_DIE((msg)) + +#define test_fail() test_fail_msg("Assertion failed.") + +#define test_assert(expr) tt_assert(expr) + +#define test_eq(expr1, expr2) tt_int_op((expr1), ==, (expr2)) +#define test_eq_ptr(expr1, expr2) tt_ptr_op((expr1), ==, (expr2)) +#define test_neq(expr1, expr2) tt_int_op((expr1), !=, (expr2)) +#define test_neq_ptr(expr1, expr2) tt_ptr_op((expr1), !=, (expr2)) +#define test_streq(expr1, expr2) tt_str_op((expr1), ==, (expr2)) +#define test_strneq(expr1, expr2) tt_str_op((expr1), !=, (expr2)) +#define test_streq(expr1, expr2) tt_str_op((expr1), ==, (expr2)) + +#define test_mem_op(expr1, op, expr2, len) \ + tt_assert_test_fmt_type(expr1,expr2,#expr1" "#op" "#expr2, \ + const char *, \ + (memcmp(_val1, _val2, len) op 0), \ + char *, "%s", \ + { size_t printlen = (len)*2+1; \ + _print = tor_malloc(printlen); \ + base16_encode(_print, printlen, _value, \ + (len)); }, \ + { tor_free(_print); } \ + ); + +#define test_memeq(expr1, expr2, len) test_mem_op((expr1), ==, (expr2), len) +#define test_memneq(expr1, expr2, len) test_mem_op((expr1), !=, (expr2), len) + +/* As test_mem_op, but decodes 'hex' before comparing. There must be a + * local char* variable called mem_op_hex_tmp for this to work. */ +#define test_mem_op_hex(expr1, op, hex) \ + STMT_BEGIN \ + size_t length = strlen(hex); \ + tor_free(mem_op_hex_tmp); \ + mem_op_hex_tmp = tor_malloc(length/2); \ + tor_assert((length&1)==0); \ + base16_decode(mem_op_hex_tmp, length/2, hex, length); \ + test_mem_op(expr1, op, mem_op_hex_tmp, length/2); \ + STMT_END + +#define test_memeq_hex(expr1, hex) test_mem_op_hex(expr1, ==, hex) + +const char *get_fname(const char *name); +crypto_pk_env_t *pk_generate(int idx); + +void legacy_test_helper(void *data); +extern const struct testcase_setup_t legacy_setup; + +#endif + diff --git a/src/test/test_addr.c b/src/test/test_addr.c new file mode 100644 index 0000000000..bafc0a968e --- /dev/null +++ b/src/test/test_addr.c @@ -0,0 +1,497 @@ +/* Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2010, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "orconfig.h" +#include "or.h" +#include "test.h" + +static void +test_addr_basic(void) +{ + uint32_t u32; + uint16_t u16; + char *cp; + + /* Test parse_addr_port */ + cp = NULL; u32 = 3; u16 = 3; + test_assert(!parse_addr_port(LOG_WARN, "1.2.3.4", &cp, &u32, &u16)); + test_streq(cp, "1.2.3.4"); + test_eq(u32, 0x01020304u); + test_eq(u16, 0); + tor_free(cp); + test_assert(!parse_addr_port(LOG_WARN, "4.3.2.1:99", &cp, &u32, &u16)); + test_streq(cp, "4.3.2.1"); + test_eq(u32, 0x04030201u); + test_eq(u16, 99); + tor_free(cp); + test_assert(!parse_addr_port(LOG_WARN, "nonexistent.address:4040", + &cp, NULL, &u16)); + test_streq(cp, "nonexistent.address"); + test_eq(u16, 4040); + tor_free(cp); + test_assert(!parse_addr_port(LOG_WARN, "localhost:9999", &cp, &u32, &u16)); + test_streq(cp, "localhost"); + test_eq(u32, 0x7f000001u); + test_eq(u16, 9999); + tor_free(cp); + u32 = 3; + test_assert(!parse_addr_port(LOG_WARN, "localhost", NULL, &u32, &u16)); + test_eq(cp, NULL); + test_eq(u32, 0x7f000001u); + test_eq(u16, 0); + tor_free(cp); + test_eq(0, addr_mask_get_bits(0x0u)); + test_eq(32, addr_mask_get_bits(0xFFFFFFFFu)); + test_eq(16, addr_mask_get_bits(0xFFFF0000u)); + test_eq(31, addr_mask_get_bits(0xFFFFFFFEu)); + test_eq(1, addr_mask_get_bits(0x80000000u)); + + /* Test inet_ntop */ + { + char tmpbuf[TOR_ADDR_BUF_LEN]; + const char *ip = "176.192.208.224"; + struct in_addr in; + tor_inet_pton(AF_INET, ip, &in); + tor_inet_ntop(AF_INET, &in, tmpbuf, sizeof(tmpbuf)); + test_streq(tmpbuf, ip); + } + + done: + ; +} + +#define _test_op_ip6(a,op,b,e1,e2) \ + STMT_BEGIN \ + tt_assert_test_fmt_type(a,b,e1" "#op" "e2,struct in6_addr*, \ + (memcmp(_val1->s6_addr, _val2->s6_addr, 16) op 0), \ + char *, "%s", \ + { int i; char *cp; \ + cp = _print = tor_malloc(64); \ + for (i=0;i<16;++i) { \ + tor_snprintf(cp, 3,"%02x", (unsigned)_value->s6_addr[i]);\ + cp += 2; \ + if (i != 15) *cp++ = ':'; \ + } \ + }, { tor_free(_print); } \ + ); \ + STMT_END + +/** Helper: Assert that two strings both decode as IPv6 addresses with + * tor_inet_pton(), and both decode to the same address. */ +#define test_pton6_same(a,b) STMT_BEGIN \ + test_eq(tor_inet_pton(AF_INET6, a, &a1), 1); \ + test_eq(tor_inet_pton(AF_INET6, b, &a2), 1); \ + _test_op_ip6(&a1,==,&a2,#a,#b); \ + STMT_END + +/** Helper: Assert that <b>a</b> is recognized as a bad IPv6 address by + * tor_inet_pton(). */ +#define test_pton6_bad(a) \ + test_eq(0, tor_inet_pton(AF_INET6, a, &a1)) + +/** Helper: assert that <b>a</b>, when parsed by tor_inet_pton() and displayed + * with tor_inet_ntop(), yields <b>b</b>. Also assert that <b>b</b> parses to + * the same value as <b>a</b>. */ +#define test_ntop6_reduces(a,b) STMT_BEGIN \ + test_eq(tor_inet_pton(AF_INET6, a, &a1), 1); \ + test_streq(tor_inet_ntop(AF_INET6, &a1, buf, sizeof(buf)), b); \ + test_eq(tor_inet_pton(AF_INET6, b, &a2), 1); \ + _test_op_ip6(&a1, ==, &a2, a, b); \ + STMT_END + +/** Helper: assert that <b>a</b> parses by tor_inet_pton() into a address that + * passes tor_addr_is_internal() with <b>for_listening</b>. */ +#define test_internal_ip(a,for_listening) STMT_BEGIN \ + test_eq(tor_inet_pton(AF_INET6, a, &t1.addr.in6_addr), 1); \ + t1.family = AF_INET6; \ + if (!tor_addr_is_internal(&t1, for_listening)) \ + test_fail_msg( a "was not internal."); \ + STMT_END + +/** Helper: assert that <b>a</b> parses by tor_inet_pton() into a address that + * does not pass tor_addr_is_internal() with <b>for_listening</b>. */ +#define test_external_ip(a,for_listening) STMT_BEGIN \ + test_eq(tor_inet_pton(AF_INET6, a, &t1.addr.in6_addr), 1); \ + t1.family = AF_INET6; \ + if (tor_addr_is_internal(&t1, for_listening)) \ + test_fail_msg(a "was not external."); \ + STMT_END + +/** Helper: Assert that <b>a</b> and <b>b</b>, when parsed by + * tor_inet_pton(), give addresses that compare in the order defined by + * <b>op</b> with tor_addr_compare(). */ +#define test_addr_compare(a, op, b) STMT_BEGIN \ + test_eq(tor_inet_pton(AF_INET6, a, &t1.addr.in6_addr), 1); \ + test_eq(tor_inet_pton(AF_INET6, b, &t2.addr.in6_addr), 1); \ + t1.family = t2.family = AF_INET6; \ + r = tor_addr_compare(&t1,&t2,CMP_SEMANTIC); \ + if (!(r op 0)) \ + test_fail_msg("failed: tor_addr_compare("a","b") "#op" 0"); \ + STMT_END + +/** Helper: Assert that <b>a</b> and <b>b</b>, when parsed by + * tor_inet_pton(), give addresses that compare in the order defined by + * <b>op</b> with tor_addr_compare_masked() with <b>m</b> masked. */ +#define test_addr_compare_masked(a, op, b, m) STMT_BEGIN \ + test_eq(tor_inet_pton(AF_INET6, a, &t1.addr.in6_addr), 1); \ + test_eq(tor_inet_pton(AF_INET6, b, &t2.addr.in6_addr), 1); \ + t1.family = t2.family = AF_INET6; \ + r = tor_addr_compare_masked(&t1,&t2,m,CMP_SEMANTIC); \ + if (!(r op 0)) \ + test_fail_msg("failed: tor_addr_compare_masked("a","b","#m") "#op" 0"); \ + STMT_END + +/** Helper: assert that <b>xx</b> is parseable as a masked IPv6 address with + * ports by tor_parse_mask_addr_ports(), with family <b>f</b>, IP address + * as 4 32-bit words <b>ip1...ip4</b>, mask bits as <b>mm</b>, and port range + * as <b>pt1..pt2</b>. */ +#define test_addr_mask_ports_parse(xx, f, ip1, ip2, ip3, ip4, mm, pt1, pt2) \ + STMT_BEGIN \ + test_eq(tor_addr_parse_mask_ports(xx, &t1, &mask, &port1, &port2), f); \ + p1=tor_inet_ntop(AF_INET6, &t1.addr.in6_addr, bug, sizeof(bug)); \ + test_eq(htonl(ip1), tor_addr_to_in6_addr32(&t1)[0]); \ + test_eq(htonl(ip2), tor_addr_to_in6_addr32(&t1)[1]); \ + test_eq(htonl(ip3), tor_addr_to_in6_addr32(&t1)[2]); \ + test_eq(htonl(ip4), tor_addr_to_in6_addr32(&t1)[3]); \ + test_eq(mask, mm); \ + test_eq(port1, pt1); \ + test_eq(port2, pt2); \ + STMT_END + +/** Run unit tests for IPv6 encoding/decoding/manipulation functions. */ +static void +test_addr_ip6_helpers(void) +{ + char buf[TOR_ADDR_BUF_LEN], bug[TOR_ADDR_BUF_LEN]; + struct in6_addr a1, a2; + tor_addr_t t1, t2; + int r, i; + uint16_t port1, port2; + maskbits_t mask; + const char *p1; + struct sockaddr_storage sa_storage; + struct sockaddr_in *sin; + struct sockaddr_in6 *sin6; + + // struct in_addr b1, b2; + /* Test tor_inet_ntop and tor_inet_pton: IPv6 */ + + /* ==== Converting to and from sockaddr_t. */ + sin = (struct sockaddr_in *)&sa_storage; + sin->sin_family = AF_INET; + sin->sin_port = 9090; + sin->sin_addr.s_addr = htonl(0x7f7f0102); /*127.127.1.2*/ + tor_addr_from_sockaddr(&t1, (struct sockaddr *)sin, NULL); + test_eq(tor_addr_family(&t1), AF_INET); + test_eq(tor_addr_to_ipv4h(&t1), 0x7f7f0102); + + memset(&sa_storage, 0, sizeof(sa_storage)); + test_eq(sizeof(struct sockaddr_in), + tor_addr_to_sockaddr(&t1, 1234, (struct sockaddr *)&sa_storage, + sizeof(sa_storage))); + test_eq(1234, ntohs(sin->sin_port)); + test_eq(0x7f7f0102, ntohl(sin->sin_addr.s_addr)); + + memset(&sa_storage, 0, sizeof(sa_storage)); + sin6 = (struct sockaddr_in6 *)&sa_storage; + sin6->sin6_family = AF_INET6; + sin6->sin6_port = htons(7070); + sin6->sin6_addr.s6_addr[0] = 128; + tor_addr_from_sockaddr(&t1, (struct sockaddr *)sin6, NULL); + test_eq(tor_addr_family(&t1), AF_INET6); + p1 = tor_addr_to_str(buf, &t1, sizeof(buf), 0); + test_streq(p1, "8000::"); + + memset(&sa_storage, 0, sizeof(sa_storage)); + test_eq(sizeof(struct sockaddr_in6), + tor_addr_to_sockaddr(&t1, 9999, (struct sockaddr *)&sa_storage, + sizeof(sa_storage))); + test_eq(AF_INET6, sin6->sin6_family); + test_eq(9999, ntohs(sin6->sin6_port)); + test_eq(0x80000000, ntohl(S6_ADDR32(sin6->sin6_addr)[0])); + + /* ==== tor_addr_lookup: static cases. (Can't test dns without knowing we + * have a good resolver. */ + test_eq(0, tor_addr_lookup("127.128.129.130", AF_UNSPEC, &t1)); + test_eq(AF_INET, tor_addr_family(&t1)); + test_eq(tor_addr_to_ipv4h(&t1), 0x7f808182); + + test_eq(0, tor_addr_lookup("9000::5", AF_UNSPEC, &t1)); + test_eq(AF_INET6, tor_addr_family(&t1)); + test_eq(0x90, tor_addr_to_in6_addr8(&t1)[0]); + test_assert(tor_mem_is_zero((char*)tor_addr_to_in6_addr8(&t1)+1, 14)); + test_eq(0x05, tor_addr_to_in6_addr8(&t1)[15]); + + /* === Test pton: valid af_inet6 */ + /* Simple, valid parsing. */ + r = tor_inet_pton(AF_INET6, + "0102:0304:0506:0708:090A:0B0C:0D0E:0F10", &a1); + test_assert(r==1); + for (i=0;i<16;++i) { test_eq(i+1, (int)a1.s6_addr[i]); } + /* ipv4 ending. */ + test_pton6_same("0102:0304:0506:0708:090A:0B0C:0D0E:0F10", + "0102:0304:0506:0708:090A:0B0C:13.14.15.16"); + /* shortened words. */ + test_pton6_same("0001:0099:BEEF:0000:0123:FFFF:0001:0001", + "1:99:BEEF:0:0123:FFFF:1:1"); + /* zeros at the beginning */ + test_pton6_same("0000:0000:0000:0000:0009:C0A8:0001:0001", + "::9:c0a8:1:1"); + test_pton6_same("0000:0000:0000:0000:0009:C0A8:0001:0001", + "::9:c0a8:0.1.0.1"); + /* zeros in the middle. */ + test_pton6_same("fe80:0000:0000:0000:0202:1111:0001:0001", + "fe80::202:1111:1:1"); + /* zeros at the end. */ + test_pton6_same("1000:0001:0000:0007:0000:0000:0000:0000", + "1000:1:0:7::"); + + /* === Test ntop: af_inet6 */ + test_ntop6_reduces("0:0:0:0:0:0:0:0", "::"); + + test_ntop6_reduces("0001:0099:BEEF:0006:0123:FFFF:0001:0001", + "1:99:beef:6:123:ffff:1:1"); + + //test_ntop6_reduces("0:0:0:0:0:0:c0a8:0101", "::192.168.1.1"); + test_ntop6_reduces("0:0:0:0:0:ffff:c0a8:0101", "::ffff:192.168.1.1"); + test_ntop6_reduces("002:0:0000:0:3::4", "2::3:0:0:4"); + test_ntop6_reduces("0:0::1:0:3", "::1:0:3"); + test_ntop6_reduces("008:0::0", "8::"); + test_ntop6_reduces("0:0:0:0:0:ffff::1", "::ffff:0.0.0.1"); + test_ntop6_reduces("abcd:0:0:0:0:0:7f00::", "abcd::7f00:0"); + test_ntop6_reduces("0000:0000:0000:0000:0009:C0A8:0001:0001", + "::9:c0a8:1:1"); + test_ntop6_reduces("fe80:0000:0000:0000:0202:1111:0001:0001", + "fe80::202:1111:1:1"); + test_ntop6_reduces("1000:0001:0000:0007:0000:0000:0000:0000", + "1000:1:0:7::"); + + /* === Test pton: invalid in6. */ + test_pton6_bad("foobar."); + test_pton6_bad("55555::"); + test_pton6_bad("9:-60::"); + test_pton6_bad("1:2:33333:4:0002:3::"); + //test_pton6_bad("1:2:3333:4:00002:3::");// BAD, but glibc doesn't say so. + test_pton6_bad("1:2:3333:4:fish:3::"); + test_pton6_bad("1:2:3:4:5:6:7:8:9"); + test_pton6_bad("1:2:3:4:5:6:7"); + test_pton6_bad("1:2:3:4:5:6:1.2.3.4.5"); + test_pton6_bad("1:2:3:4:5:6:1.2.3"); + test_pton6_bad("::1.2.3"); + test_pton6_bad("::1.2.3.4.5"); + test_pton6_bad("99"); + test_pton6_bad(""); + test_pton6_bad("1::2::3:4"); + test_pton6_bad("a:::b:c"); + test_pton6_bad(":::a:b:c"); + test_pton6_bad("a:b:c:::"); + + /* test internal checking */ + test_external_ip("fbff:ffff::2:7", 0); + test_internal_ip("fc01::2:7", 0); + test_internal_ip("fdff:ffff::f:f", 0); + test_external_ip("fe00::3:f", 0); + + test_external_ip("fe7f:ffff::2:7", 0); + test_internal_ip("fe80::2:7", 0); + test_internal_ip("febf:ffff::f:f", 0); + + test_internal_ip("fec0::2:7:7", 0); + test_internal_ip("feff:ffff::e:7:7", 0); + test_external_ip("ff00::e:7:7", 0); + + test_internal_ip("::", 0); + test_internal_ip("::1", 0); + test_internal_ip("::1", 1); + test_internal_ip("::", 0); + test_external_ip("::", 1); + test_external_ip("::2", 0); + test_external_ip("2001::", 0); + test_external_ip("ffff::", 0); + + test_external_ip("::ffff:0.0.0.0", 1); + test_internal_ip("::ffff:0.0.0.0", 0); + test_internal_ip("::ffff:0.255.255.255", 0); + test_external_ip("::ffff:1.0.0.0", 0); + + test_external_ip("::ffff:9.255.255.255", 0); + test_internal_ip("::ffff:10.0.0.0", 0); + test_internal_ip("::ffff:10.255.255.255", 0); + test_external_ip("::ffff:11.0.0.0", 0); + + test_external_ip("::ffff:126.255.255.255", 0); + test_internal_ip("::ffff:127.0.0.0", 0); + test_internal_ip("::ffff:127.255.255.255", 0); + test_external_ip("::ffff:128.0.0.0", 0); + + test_external_ip("::ffff:172.15.255.255", 0); + test_internal_ip("::ffff:172.16.0.0", 0); + test_internal_ip("::ffff:172.31.255.255", 0); + test_external_ip("::ffff:172.32.0.0", 0); + + test_external_ip("::ffff:192.167.255.255", 0); + test_internal_ip("::ffff:192.168.0.0", 0); + test_internal_ip("::ffff:192.168.255.255", 0); + test_external_ip("::ffff:192.169.0.0", 0); + + test_external_ip("::ffff:169.253.255.255", 0); + test_internal_ip("::ffff:169.254.0.0", 0); + test_internal_ip("::ffff:169.254.255.255", 0); + test_external_ip("::ffff:169.255.0.0", 0); + test_assert(is_internal_IP(0x7f000001, 0)); + + /* tor_addr_compare(tor_addr_t x2) */ + test_addr_compare("ffff::", ==, "ffff::0"); + test_addr_compare("0::3:2:1", <, "0::ffff:0.3.2.1"); + test_addr_compare("0::2:2:1", <, "0::ffff:0.3.2.1"); + test_addr_compare("0::ffff:0.3.2.1", >, "0::0:0:0"); + test_addr_compare("0::ffff:5.2.2.1", <, "::ffff:6.0.0.0"); /* XXXX wrong. */ + tor_addr_parse_mask_ports("[::ffff:2.3.4.5]", &t1, NULL, NULL, NULL); + tor_addr_parse_mask_ports("2.3.4.5", &t2, NULL, NULL, NULL); + test_assert(tor_addr_compare(&t1, &t2, CMP_SEMANTIC) == 0); + tor_addr_parse_mask_ports("[::ffff:2.3.4.4]", &t1, NULL, NULL, NULL); + tor_addr_parse_mask_ports("2.3.4.5", &t2, NULL, NULL, NULL); + test_assert(tor_addr_compare(&t1, &t2, CMP_SEMANTIC) < 0); + + /* test compare_masked */ + test_addr_compare_masked("ffff::", ==, "ffff::0", 128); + test_addr_compare_masked("ffff::", ==, "ffff::0", 64); + test_addr_compare_masked("0::2:2:1", <, "0::8000:2:1", 81); + test_addr_compare_masked("0::2:2:1", ==, "0::8000:2:1", 80); + + /* Test decorated addr_to_string. */ + test_eq(AF_INET6, tor_addr_from_str(&t1, "[123:45:6789::5005:11]")); + p1 = tor_addr_to_str(buf, &t1, sizeof(buf), 1); + test_streq(p1, "[123:45:6789::5005:11]"); + test_eq(AF_INET, tor_addr_from_str(&t1, "18.0.0.1")); + p1 = tor_addr_to_str(buf, &t1, sizeof(buf), 1); + test_streq(p1, "18.0.0.1"); + + /* Test tor_addr_parse_reverse_lookup_name */ + i = tor_addr_parse_reverse_lookup_name(&t1, "Foobar.baz", AF_UNSPEC, 0); + test_eq(0, i); + i = tor_addr_parse_reverse_lookup_name(&t1, "Foobar.baz", AF_UNSPEC, 1); + test_eq(0, i); + i = tor_addr_parse_reverse_lookup_name(&t1, "1.0.168.192.in-addr.arpa", + AF_UNSPEC, 1); + test_eq(1, i); + test_eq(tor_addr_family(&t1), AF_INET); + p1 = tor_addr_to_str(buf, &t1, sizeof(buf), 1); + test_streq(p1, "192.168.0.1"); + i = tor_addr_parse_reverse_lookup_name(&t1, "192.168.0.99", AF_UNSPEC, 0); + test_eq(0, i); + i = tor_addr_parse_reverse_lookup_name(&t1, "192.168.0.99", AF_UNSPEC, 1); + test_eq(1, i); + p1 = tor_addr_to_str(buf, &t1, sizeof(buf), 1); + test_streq(p1, "192.168.0.99"); + memset(&t1, 0, sizeof(t1)); + i = tor_addr_parse_reverse_lookup_name(&t1, + "0.1.2.3.4.5.6.7.8.9.a.b.c.d.e.f." + "f.e.e.b.1.e.b.e.e.f.f.e.e.e.d.9." + "ip6.ARPA", + AF_UNSPEC, 0); + test_eq(1, i); + p1 = tor_addr_to_str(buf, &t1, sizeof(buf), 1); + test_streq(p1, "[9dee:effe:ebe1:beef:fedc:ba98:7654:3210]"); + /* Failing cases. */ + i = tor_addr_parse_reverse_lookup_name(&t1, + "6.7.8.9.a.b.c.d.e.f." + "f.e.e.b.1.e.b.e.e.f.f.e.e.e.d.9." + "ip6.ARPA", + AF_UNSPEC, 0); + test_eq(i, -1); + i = tor_addr_parse_reverse_lookup_name(&t1, + "6.7.8.9.a.b.c.d.e.f.a.b.c.d.e.f.0." + "f.e.e.b.1.e.b.e.e.f.f.e.e.e.d.9." + "ip6.ARPA", + AF_UNSPEC, 0); + test_eq(i, -1); + i = tor_addr_parse_reverse_lookup_name(&t1, + "6.7.8.9.a.b.c.d.e.f.X.0.0.0.0.9." + "f.e.e.b.1.e.b.e.e.f.f.e.e.e.d.9." + "ip6.ARPA", + AF_UNSPEC, 0); + test_eq(i, -1); + i = tor_addr_parse_reverse_lookup_name(&t1, "32.1.1.in-addr.arpa", + AF_UNSPEC, 0); + test_eq(i, -1); + i = tor_addr_parse_reverse_lookup_name(&t1, ".in-addr.arpa", + AF_UNSPEC, 0); + test_eq(i, -1); + i = tor_addr_parse_reverse_lookup_name(&t1, "1.2.3.4.5.in-addr.arpa", + AF_UNSPEC, 0); + test_eq(i, -1); + i = tor_addr_parse_reverse_lookup_name(&t1, "1.2.3.4.5.in-addr.arpa", + AF_INET6, 0); + test_eq(i, -1); + i = tor_addr_parse_reverse_lookup_name(&t1, + "6.7.8.9.a.b.c.d.e.f.a.b.c.d.e.0." + "f.e.e.b.1.e.b.e.e.f.f.e.e.e.d.9." + "ip6.ARPA", + AF_INET, 0); + test_eq(i, -1); + + /* test tor_addr_parse_mask_ports */ + test_addr_mask_ports_parse("[::f]/17:47-95", AF_INET6, + 0, 0, 0, 0x0000000f, 17, 47, 95); + //test_addr_parse("[::fefe:4.1.1.7/120]:999-1000"); + //test_addr_parse_check("::fefe:401:107", 120, 999, 1000); + test_addr_mask_ports_parse("[::ffff:4.1.1.7]/120:443", AF_INET6, + 0, 0, 0x0000ffff, 0x04010107, 120, 443, 443); + test_addr_mask_ports_parse("[abcd:2::44a:0]:2-65000", AF_INET6, + 0xabcd0002, 0, 0, 0x044a0000, 128, 2, 65000); + + r=tor_addr_parse_mask_ports("[fefef::]/112", &t1, NULL, NULL, NULL); + test_assert(r == -1); + r=tor_addr_parse_mask_ports("efef::/112", &t1, NULL, NULL, NULL); + test_assert(r == -1); + r=tor_addr_parse_mask_ports("[f:f:f:f:f:f:f:f::]", &t1, NULL, NULL, NULL); + test_assert(r == -1); + r=tor_addr_parse_mask_ports("[::f:f:f:f:f:f:f:f]", &t1, NULL, NULL, NULL); + test_assert(r == -1); + r=tor_addr_parse_mask_ports("[f:f:f:f:f:f:f:f:f]", &t1, NULL, NULL, NULL); + test_assert(r == -1); + /* Test for V4-mapped address with mask < 96. (arguably not valid) */ + r=tor_addr_parse_mask_ports("[::ffff:1.1.2.2/33]", &t1, &mask, NULL, NULL); + test_assert(r == -1); + r=tor_addr_parse_mask_ports("1.1.2.2/33", &t1, &mask, NULL, NULL); + test_assert(r == -1); + r=tor_addr_parse_mask_ports("1.1.2.2/31", &t1, &mask, NULL, NULL); + test_assert(r == AF_INET); + r=tor_addr_parse_mask_ports("[efef::]/112", &t1, &mask, &port1, &port2); + test_assert(r == AF_INET6); + test_assert(port1 == 1); + test_assert(port2 == 65535); + + /* make sure inet address lengths >= max */ + test_assert(INET_NTOA_BUF_LEN >= sizeof("255.255.255.255")); + test_assert(TOR_ADDR_BUF_LEN >= + sizeof("ffff:ffff:ffff:ffff:ffff:ffff:255.255.255.255")); + + test_assert(sizeof(tor_addr_t) >= sizeof(struct in6_addr)); + + /* get interface addresses */ + r = get_interface_address6(LOG_DEBUG, AF_INET, &t1); + i = get_interface_address6(LOG_DEBUG, AF_INET6, &t2); +#if 0 + tor_inet_ntop(AF_INET, &t1.sa.sin_addr, buf, sizeof(buf)); + printf("\nv4 address: %s (family=%i)", buf, IN_FAMILY(&t1)); + tor_inet_ntop(AF_INET6, &t2.sa6.sin6_addr, buf, sizeof(buf)); + printf("\nv6 address: %s (family=%i)", buf, IN_FAMILY(&t2)); +#endif + + done: + ; +} + +#define ADDR_LEGACY(name) \ + { #name, legacy_test_helper, 0, &legacy_setup, test_addr_ ## name } + +struct testcase_t addr_tests[] = { + ADDR_LEGACY(basic), + ADDR_LEGACY(ip6_helpers), + END_OF_TESTCASES +}; + diff --git a/src/test/test_containers.c b/src/test/test_containers.c new file mode 100644 index 0000000000..1fc248cb46 --- /dev/null +++ b/src/test/test_containers.c @@ -0,0 +1,765 @@ +/* Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2010, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "orconfig.h" +#include "or.h" +#include "test.h" + +/** Helper: return a tristate based on comparing the strings in *<b>a</b> and + * *<b>b</b>. */ +static int +_compare_strs(const void **a, const void **b) +{ + const char *s1 = *a, *s2 = *b; + return strcmp(s1, s2); +} + +/** Helper: return a tristate based on comparing the strings in *<b>a</b> and + * *<b>b</b>, excluding a's first character, and ignoring case. */ +static int +_compare_without_first_ch(const void *a, const void **b) +{ + const char *s1 = a, *s2 = *b; + return strcasecmp(s1+1, s2); +} + +/** Run unit tests for basic dynamic-sized array functionality. */ +static void +test_container_smartlist_basic(void) +{ + smartlist_t *sl; + + /* XXXX test sort_digests, uniq_strings, uniq_digests */ + + /* Test smartlist add, del_keeporder, insert, get. */ + sl = smartlist_create(); + smartlist_add(sl, (void*)1); + smartlist_add(sl, (void*)2); + smartlist_add(sl, (void*)3); + smartlist_add(sl, (void*)4); + smartlist_del_keeporder(sl, 1); + smartlist_insert(sl, 1, (void*)22); + smartlist_insert(sl, 0, (void*)0); + smartlist_insert(sl, 5, (void*)555); + test_eq_ptr((void*)0, smartlist_get(sl,0)); + test_eq_ptr((void*)1, smartlist_get(sl,1)); + test_eq_ptr((void*)22, smartlist_get(sl,2)); + test_eq_ptr((void*)3, smartlist_get(sl,3)); + test_eq_ptr((void*)4, smartlist_get(sl,4)); + test_eq_ptr((void*)555, smartlist_get(sl,5)); + /* Try deleting in the middle. */ + smartlist_del(sl, 1); + test_eq_ptr((void*)555, smartlist_get(sl, 1)); + /* Try deleting at the end. */ + smartlist_del(sl, 4); + test_eq(4, smartlist_len(sl)); + + /* test isin. */ + test_assert(smartlist_isin(sl, (void*)3)); + test_assert(!smartlist_isin(sl, (void*)99)); + + done: + smartlist_free(sl); +} + +/** Run unit tests for smartlist-of-strings functionality. */ +static void +test_container_smartlist_strings(void) +{ + smartlist_t *sl = smartlist_create(); + char *cp=NULL, *cp_alloc=NULL; + size_t sz; + + /* Test split and join */ + test_eq(0, smartlist_len(sl)); + smartlist_split_string(sl, "abc", ":", 0, 0); + test_eq(1, smartlist_len(sl)); + test_streq("abc", smartlist_get(sl, 0)); + smartlist_split_string(sl, "a::bc::", "::", 0, 0); + test_eq(4, smartlist_len(sl)); + test_streq("a", smartlist_get(sl, 1)); + test_streq("bc", smartlist_get(sl, 2)); + test_streq("", smartlist_get(sl, 3)); + cp_alloc = smartlist_join_strings(sl, "", 0, NULL); + test_streq(cp_alloc, "abcabc"); + tor_free(cp_alloc); + cp_alloc = smartlist_join_strings(sl, "!", 0, NULL); + test_streq(cp_alloc, "abc!a!bc!"); + tor_free(cp_alloc); + cp_alloc = smartlist_join_strings(sl, "XY", 0, NULL); + test_streq(cp_alloc, "abcXYaXYbcXY"); + tor_free(cp_alloc); + cp_alloc = smartlist_join_strings(sl, "XY", 1, NULL); + test_streq(cp_alloc, "abcXYaXYbcXYXY"); + tor_free(cp_alloc); + cp_alloc = smartlist_join_strings(sl, "", 1, NULL); + test_streq(cp_alloc, "abcabc"); + tor_free(cp_alloc); + + smartlist_split_string(sl, "/def/ /ghijk", "/", 0, 0); + test_eq(8, smartlist_len(sl)); + test_streq("", smartlist_get(sl, 4)); + test_streq("def", smartlist_get(sl, 5)); + test_streq(" ", smartlist_get(sl, 6)); + test_streq("ghijk", smartlist_get(sl, 7)); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + smartlist_split_string(sl, "a,bbd,cdef", ",", SPLIT_SKIP_SPACE, 0); + test_eq(3, smartlist_len(sl)); + test_streq("a", smartlist_get(sl,0)); + test_streq("bbd", smartlist_get(sl,1)); + test_streq("cdef", smartlist_get(sl,2)); + smartlist_split_string(sl, " z <> zhasd <> <> bnud<> ", "<>", + SPLIT_SKIP_SPACE, 0); + test_eq(8, smartlist_len(sl)); + test_streq("z", smartlist_get(sl,3)); + test_streq("zhasd", smartlist_get(sl,4)); + test_streq("", smartlist_get(sl,5)); + test_streq("bnud", smartlist_get(sl,6)); + test_streq("", smartlist_get(sl,7)); + + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + smartlist_split_string(sl, " ab\tc \td ef ", NULL, + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); + test_eq(4, smartlist_len(sl)); + test_streq("ab", smartlist_get(sl,0)); + test_streq("c", smartlist_get(sl,1)); + test_streq("d", smartlist_get(sl,2)); + test_streq("ef", smartlist_get(sl,3)); + smartlist_split_string(sl, "ghi\tj", NULL, + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); + test_eq(6, smartlist_len(sl)); + test_streq("ghi", smartlist_get(sl,4)); + test_streq("j", smartlist_get(sl,5)); + + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + cp_alloc = smartlist_join_strings(sl, "XY", 0, NULL); + test_streq(cp_alloc, ""); + tor_free(cp_alloc); + cp_alloc = smartlist_join_strings(sl, "XY", 1, NULL); + test_streq(cp_alloc, "XY"); + tor_free(cp_alloc); + + smartlist_split_string(sl, " z <> zhasd <> <> bnud<> ", "<>", + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); + test_eq(3, smartlist_len(sl)); + test_streq("z", smartlist_get(sl, 0)); + test_streq("zhasd", smartlist_get(sl, 1)); + test_streq("bnud", smartlist_get(sl, 2)); + smartlist_split_string(sl, " z <> zhasd <> <> bnud<> ", "<>", + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 2); + test_eq(5, smartlist_len(sl)); + test_streq("z", smartlist_get(sl, 3)); + test_streq("zhasd <> <> bnud<>", smartlist_get(sl, 4)); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + smartlist_split_string(sl, "abcd\n", "\n", + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); + test_eq(1, smartlist_len(sl)); + test_streq("abcd", smartlist_get(sl, 0)); + smartlist_split_string(sl, "efgh", "\n", + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); + test_eq(2, smartlist_len(sl)); + test_streq("efgh", smartlist_get(sl, 1)); + + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + /* Test swapping, shuffling, and sorting. */ + smartlist_split_string(sl, "the,onion,router,by,arma,and,nickm", ",", 0, 0); + test_eq(7, smartlist_len(sl)); + smartlist_sort(sl, _compare_strs); + cp_alloc = smartlist_join_strings(sl, ",", 0, NULL); + test_streq(cp_alloc,"and,arma,by,nickm,onion,router,the"); + tor_free(cp_alloc); + smartlist_swap(sl, 1, 5); + cp_alloc = smartlist_join_strings(sl, ",", 0, NULL); + test_streq(cp_alloc,"and,router,by,nickm,onion,arma,the"); + tor_free(cp_alloc); + smartlist_shuffle(sl); + test_eq(7, smartlist_len(sl)); + test_assert(smartlist_string_isin(sl, "and")); + test_assert(smartlist_string_isin(sl, "router")); + test_assert(smartlist_string_isin(sl, "by")); + test_assert(smartlist_string_isin(sl, "nickm")); + test_assert(smartlist_string_isin(sl, "onion")); + test_assert(smartlist_string_isin(sl, "arma")); + test_assert(smartlist_string_isin(sl, "the")); + + /* Test bsearch. */ + smartlist_sort(sl, _compare_strs); + test_streq("nickm", smartlist_bsearch(sl, "zNicKM", + _compare_without_first_ch)); + test_streq("and", smartlist_bsearch(sl, " AND", _compare_without_first_ch)); + test_eq_ptr(NULL, smartlist_bsearch(sl, " ANz", _compare_without_first_ch)); + + /* Test bsearch_idx */ + { + int f; + test_eq(0, smartlist_bsearch_idx(sl," aaa",_compare_without_first_ch,&f)); + test_eq(f, 0); + test_eq(0, smartlist_bsearch_idx(sl," and",_compare_without_first_ch,&f)); + test_eq(f, 1); + test_eq(1, smartlist_bsearch_idx(sl," arm",_compare_without_first_ch,&f)); + test_eq(f, 0); + test_eq(1, smartlist_bsearch_idx(sl," arma",_compare_without_first_ch,&f)); + test_eq(f, 1); + test_eq(2, smartlist_bsearch_idx(sl," armb",_compare_without_first_ch,&f)); + test_eq(f, 0); + test_eq(7, smartlist_bsearch_idx(sl," zzzz",_compare_without_first_ch,&f)); + test_eq(f, 0); + } + + /* Test reverse() and pop_last() */ + smartlist_reverse(sl); + cp_alloc = smartlist_join_strings(sl, ",", 0, NULL); + test_streq(cp_alloc,"the,router,onion,nickm,by,arma,and"); + tor_free(cp_alloc); + cp_alloc = smartlist_pop_last(sl); + test_streq(cp_alloc, "and"); + tor_free(cp_alloc); + test_eq(smartlist_len(sl), 6); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + cp_alloc = smartlist_pop_last(sl); + test_eq(cp_alloc, NULL); + + /* Test uniq() */ + smartlist_split_string(sl, + "50,noon,radar,a,man,a,plan,a,canal,panama,radar,noon,50", + ",", 0, 0); + smartlist_sort(sl, _compare_strs); + smartlist_uniq(sl, _compare_strs, _tor_free); + cp_alloc = smartlist_join_strings(sl, ",", 0, NULL); + test_streq(cp_alloc, "50,a,canal,man,noon,panama,plan,radar"); + tor_free(cp_alloc); + + /* Test string_isin and isin_case and num_isin */ + test_assert(smartlist_string_isin(sl, "noon")); + test_assert(!smartlist_string_isin(sl, "noonoon")); + test_assert(smartlist_string_isin_case(sl, "nOOn")); + test_assert(!smartlist_string_isin_case(sl, "nooNooN")); + test_assert(smartlist_string_num_isin(sl, 50)); + test_assert(!smartlist_string_num_isin(sl, 60)); + + /* Test smartlist_choose */ + { + int i; + int allsame = 1; + int allin = 1; + void *first = smartlist_choose(sl); + test_assert(smartlist_isin(sl, first)); + for (i = 0; i < 100; ++i) { + void *second = smartlist_choose(sl); + if (second != first) + allsame = 0; + if (!smartlist_isin(sl, second)) + allin = 0; + } + test_assert(!allsame); + test_assert(allin); + } + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + /* Test string_remove and remove and join_strings2 */ + smartlist_split_string(sl, + "Some say the Earth will end in ice and some in fire", + " ", 0, 0); + cp = smartlist_get(sl, 4); + test_streq(cp, "will"); + smartlist_add(sl, cp); + smartlist_remove(sl, cp); + tor_free(cp); + cp_alloc = smartlist_join_strings(sl, ",", 0, NULL); + test_streq(cp_alloc, "Some,say,the,Earth,fire,end,in,ice,and,some,in"); + tor_free(cp_alloc); + smartlist_string_remove(sl, "in"); + cp_alloc = smartlist_join_strings2(sl, "+XX", 1, 0, &sz); + test_streq(cp_alloc, "Some+say+the+Earth+fire+end+some+ice+and"); + test_eq((int)sz, 40); + + done: + + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_free(sl); + tor_free(cp_alloc); +} + +/** Run unit tests for smartlist set manipulation functions. */ +static void +test_container_smartlist_overlap(void) +{ + smartlist_t *sl = smartlist_create(); + smartlist_t *ints = smartlist_create(); + smartlist_t *odds = smartlist_create(); + smartlist_t *evens = smartlist_create(); + smartlist_t *primes = smartlist_create(); + int i; + for (i=1; i < 10; i += 2) + smartlist_add(odds, (void*)(uintptr_t)i); + for (i=0; i < 10; i += 2) + smartlist_add(evens, (void*)(uintptr_t)i); + + /* add_all */ + smartlist_add_all(ints, odds); + smartlist_add_all(ints, evens); + test_eq(smartlist_len(ints), 10); + + smartlist_add(primes, (void*)2); + smartlist_add(primes, (void*)3); + smartlist_add(primes, (void*)5); + smartlist_add(primes, (void*)7); + + /* overlap */ + test_assert(smartlist_overlap(ints, odds)); + test_assert(smartlist_overlap(odds, primes)); + test_assert(smartlist_overlap(evens, primes)); + test_assert(!smartlist_overlap(odds, evens)); + + /* intersect */ + smartlist_add_all(sl, odds); + smartlist_intersect(sl, primes); + test_eq(smartlist_len(sl), 3); + test_assert(smartlist_isin(sl, (void*)3)); + test_assert(smartlist_isin(sl, (void*)5)); + test_assert(smartlist_isin(sl, (void*)7)); + + /* subtract */ + smartlist_add_all(sl, primes); + smartlist_subtract(sl, odds); + test_eq(smartlist_len(sl), 1); + test_assert(smartlist_isin(sl, (void*)2)); + + done: + smartlist_free(odds); + smartlist_free(evens); + smartlist_free(ints); + smartlist_free(primes); + smartlist_free(sl); +} + +/** Run unit tests for smartlist-of-digests functions. */ +static void +test_container_smartlist_digests(void) +{ + smartlist_t *sl = smartlist_create(); + + /* digest_isin. */ + smartlist_add(sl, tor_memdup("AAAAAAAAAAAAAAAAAAAA", DIGEST_LEN)); + smartlist_add(sl, tor_memdup("\00090AAB2AAAAaasdAAAAA", DIGEST_LEN)); + smartlist_add(sl, tor_memdup("\00090AAB2AAAAaasdAAAAA", DIGEST_LEN)); + test_eq(0, smartlist_digest_isin(NULL, "AAAAAAAAAAAAAAAAAAAA")); + test_assert(smartlist_digest_isin(sl, "AAAAAAAAAAAAAAAAAAAA")); + test_assert(smartlist_digest_isin(sl, "\00090AAB2AAAAaasdAAAAA")); + test_eq(0, smartlist_digest_isin(sl, "\00090AAB2AAABaasdAAAAA")); + + /* sort digests */ + smartlist_sort_digests(sl); + test_memeq(smartlist_get(sl, 0), "\00090AAB2AAAAaasdAAAAA", DIGEST_LEN); + test_memeq(smartlist_get(sl, 1), "\00090AAB2AAAAaasdAAAAA", DIGEST_LEN); + test_memeq(smartlist_get(sl, 2), "AAAAAAAAAAAAAAAAAAAA", DIGEST_LEN); + test_eq(3, smartlist_len(sl)); + + /* uniq_digests */ + smartlist_uniq_digests(sl); + test_eq(2, smartlist_len(sl)); + test_memeq(smartlist_get(sl, 0), "\00090AAB2AAAAaasdAAAAA", DIGEST_LEN); + test_memeq(smartlist_get(sl, 1), "AAAAAAAAAAAAAAAAAAAA", DIGEST_LEN); + + done: + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_free(sl); +} + +/** Run unit tests for concatenate-a-smartlist-of-strings functions. */ +static void +test_container_smartlist_join(void) +{ + smartlist_t *sl = smartlist_create(); + smartlist_t *sl2 = smartlist_create(), *sl3 = smartlist_create(), + *sl4 = smartlist_create(); + char *joined=NULL; + /* unique, sorted. */ + smartlist_split_string(sl, + "Abashments Ambush Anchorman Bacon Banks Borscht " + "Bunks Inhumane Insurance Knish Know Manners " + "Maraschinos Stamina Sunbonnets Unicorns Wombats", + " ", 0, 0); + /* non-unique, sorted. */ + smartlist_split_string(sl2, + "Ambush Anchorman Anchorman Anemias Anemias Bacon " + "Crossbowmen Inhumane Insurance Knish Know Manners " + "Manners Maraschinos Wombats Wombats Work", + " ", 0, 0); + SMARTLIST_FOREACH_JOIN(sl, char *, cp1, + sl2, char *, cp2, + strcmp(cp1,cp2), + smartlist_add(sl3, cp2)) { + test_streq(cp1, cp2); + smartlist_add(sl4, cp1); + } SMARTLIST_FOREACH_JOIN_END(cp1, cp2); + + SMARTLIST_FOREACH(sl3, const char *, cp, + test_assert(smartlist_isin(sl2, cp) && + !smartlist_string_isin(sl, cp))); + SMARTLIST_FOREACH(sl4, const char *, cp, + test_assert(smartlist_isin(sl, cp) && + smartlist_string_isin(sl2, cp))); + joined = smartlist_join_strings(sl3, ",", 0, NULL); + test_streq(joined, "Anemias,Anemias,Crossbowmen,Work"); + tor_free(joined); + joined = smartlist_join_strings(sl4, ",", 0, NULL); + test_streq(joined, "Ambush,Anchorman,Anchorman,Bacon,Inhumane,Insurance," + "Knish,Know,Manners,Manners,Maraschinos,Wombats,Wombats"); + tor_free(joined); + + done: + smartlist_free(sl4); + smartlist_free(sl3); + SMARTLIST_FOREACH(sl2, char *, cp, tor_free(cp)); + smartlist_free(sl2); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_free(sl); + tor_free(joined); +} + +/** Run unit tests for bitarray code */ +static void +test_container_bitarray(void) +{ + bitarray_t *ba = NULL; + int i, j, ok=1; + + ba = bitarray_init_zero(1); + test_assert(ba); + test_assert(! bitarray_is_set(ba, 0)); + bitarray_set(ba, 0); + test_assert(bitarray_is_set(ba, 0)); + bitarray_clear(ba, 0); + test_assert(! bitarray_is_set(ba, 0)); + bitarray_free(ba); + + ba = bitarray_init_zero(1023); + for (i = 1; i < 64; ) { + for (j = 0; j < 1023; ++j) { + if (j % i) + bitarray_set(ba, j); + else + bitarray_clear(ba, j); + } + for (j = 0; j < 1023; ++j) { + if (!bool_eq(bitarray_is_set(ba, j), j%i)) + ok = 0; + } + test_assert(ok); + if (i < 7) + ++i; + else if (i == 28) + i = 32; + else + i += 7; + } + + done: + if (ba) + bitarray_free(ba); +} + +/** Run unit tests for digest set code (implemented as a hashtable or as a + * bloom filter) */ +static void +test_container_digestset(void) +{ + smartlist_t *included = smartlist_create(); + char d[DIGEST_LEN]; + int i; + int ok = 1; + int false_positives = 0; + digestset_t *set = NULL; + + for (i = 0; i < 1000; ++i) { + crypto_rand(d, DIGEST_LEN); + smartlist_add(included, tor_memdup(d, DIGEST_LEN)); + } + set = digestset_new(1000); + SMARTLIST_FOREACH(included, const char *, cp, + if (digestset_isin(set, cp)) + ok = 0); + test_assert(ok); + SMARTLIST_FOREACH(included, const char *, cp, + digestset_add(set, cp)); + SMARTLIST_FOREACH(included, const char *, cp, + if (!digestset_isin(set, cp)) + ok = 0); + test_assert(ok); + for (i = 0; i < 1000; ++i) { + crypto_rand(d, DIGEST_LEN); + if (digestset_isin(set, d)) + ++false_positives; + } + test_assert(false_positives < 50); /* Should be far lower. */ + + done: + if (set) + digestset_free(set); + SMARTLIST_FOREACH(included, char *, cp, tor_free(cp)); + smartlist_free(included); +} + +typedef struct pq_entry_t { + const char *val; + int idx; +} pq_entry_t; + +/** Helper: return a tristate based on comparing two pq_entry_t values. */ +static int +_compare_strings_for_pqueue(const void *p1, const void *p2) +{ + const pq_entry_t *e1=p1, *e2=p2; + return strcmp(e1->val, e2->val); +} + +/** Run unit tests for heap-based priority queue functions. */ +static void +test_container_pqueue(void) +{ + smartlist_t *sl = smartlist_create(); + int (*cmp)(const void *, const void*); + const int offset = STRUCT_OFFSET(pq_entry_t, idx); +#define ENTRY(s) pq_entry_t s = { #s, -1 } + ENTRY(cows); + ENTRY(zebras); + ENTRY(fish); + ENTRY(frogs); + ENTRY(apples); + ENTRY(squid); + ENTRY(daschunds); + ENTRY(eggplants); + ENTRY(weissbier); + ENTRY(lobsters); + ENTRY(roquefort); + ENTRY(chinchillas); + ENTRY(fireflies); + +#define OK() smartlist_pqueue_assert_ok(sl, cmp, offset) + + cmp = _compare_strings_for_pqueue; + smartlist_pqueue_add(sl, cmp, offset, &cows); + smartlist_pqueue_add(sl, cmp, offset, &zebras); + smartlist_pqueue_add(sl, cmp, offset, &fish); + smartlist_pqueue_add(sl, cmp, offset, &frogs); + smartlist_pqueue_add(sl, cmp, offset, &apples); + smartlist_pqueue_add(sl, cmp, offset, &squid); + smartlist_pqueue_add(sl, cmp, offset, &daschunds); + smartlist_pqueue_add(sl, cmp, offset, &eggplants); + smartlist_pqueue_add(sl, cmp, offset, &weissbier); + smartlist_pqueue_add(sl, cmp, offset, &lobsters); + smartlist_pqueue_add(sl, cmp, offset, &roquefort); + + OK(); + + test_eq(smartlist_len(sl), 11); + test_eq_ptr(smartlist_get(sl, 0), &apples); + test_eq_ptr(smartlist_pqueue_pop(sl, cmp, offset), &apples); + test_eq(smartlist_len(sl), 10); + OK(); + test_eq_ptr(smartlist_pqueue_pop(sl, cmp, offset), &cows); + test_eq_ptr(smartlist_pqueue_pop(sl, cmp, offset), &daschunds); + smartlist_pqueue_add(sl, cmp, offset, &chinchillas); + OK(); + smartlist_pqueue_add(sl, cmp, offset, &fireflies); + OK(); + test_eq_ptr(smartlist_pqueue_pop(sl, cmp, offset), &chinchillas); + test_eq_ptr(smartlist_pqueue_pop(sl, cmp, offset), &eggplants); + test_eq_ptr(smartlist_pqueue_pop(sl, cmp, offset), &fireflies); + OK(); + test_eq_ptr(smartlist_pqueue_pop(sl, cmp, offset), &fish); + test_eq_ptr(smartlist_pqueue_pop(sl, cmp, offset), &frogs); + test_eq_ptr(smartlist_pqueue_pop(sl, cmp, offset), &lobsters); + test_eq_ptr(smartlist_pqueue_pop(sl, cmp, offset), &roquefort); + OK(); + test_eq(smartlist_len(sl), 3); + test_eq_ptr(smartlist_pqueue_pop(sl, cmp, offset), &squid); + test_eq_ptr(smartlist_pqueue_pop(sl, cmp, offset), &weissbier); + test_eq_ptr(smartlist_pqueue_pop(sl, cmp, offset), &zebras); + test_eq(smartlist_len(sl), 0); + OK(); + + /* Now test remove. */ + smartlist_pqueue_add(sl, cmp, offset, &cows); + smartlist_pqueue_add(sl, cmp, offset, &fish); + smartlist_pqueue_add(sl, cmp, offset, &frogs); + smartlist_pqueue_add(sl, cmp, offset, &apples); + smartlist_pqueue_add(sl, cmp, offset, &squid); + smartlist_pqueue_add(sl, cmp, offset, &zebras); + test_eq(smartlist_len(sl), 6); + OK(); + smartlist_pqueue_remove(sl, cmp, offset, &zebras); + test_eq(smartlist_len(sl), 5); + OK(); + smartlist_pqueue_remove(sl, cmp, offset, &cows); + test_eq(smartlist_len(sl), 4); + OK(); + smartlist_pqueue_remove(sl, cmp, offset, &apples); + test_eq(smartlist_len(sl), 3); + OK(); + test_eq_ptr(smartlist_pqueue_pop(sl, cmp, offset), &fish); + test_eq_ptr(smartlist_pqueue_pop(sl, cmp, offset), &frogs); + test_eq_ptr(smartlist_pqueue_pop(sl, cmp, offset), &squid); + test_eq(smartlist_len(sl), 0); + OK(); + +#undef OK + + done: + + smartlist_free(sl); +} + +/** Run unit tests for string-to-void* map functions */ +static void +test_container_strmap(void) +{ + strmap_t *map; + strmap_iter_t *iter; + const char *k; + void *v; + char *visited = NULL; + smartlist_t *found_keys = NULL; + + map = strmap_new(); + test_assert(map); + test_eq(strmap_size(map), 0); + test_assert(strmap_isempty(map)); + v = strmap_set(map, "K1", (void*)99); + test_eq(v, NULL); + test_assert(!strmap_isempty(map)); + v = strmap_set(map, "K2", (void*)101); + test_eq(v, NULL); + v = strmap_set(map, "K1", (void*)100); + test_eq(v, (void*)99); + test_eq_ptr(strmap_get(map,"K1"), (void*)100); + test_eq_ptr(strmap_get(map,"K2"), (void*)101); + test_eq_ptr(strmap_get(map,"K-not-there"), NULL); + strmap_assert_ok(map); + + v = strmap_remove(map,"K2"); + strmap_assert_ok(map); + test_eq_ptr(v, (void*)101); + test_eq_ptr(strmap_get(map,"K2"), NULL); + test_eq_ptr(strmap_remove(map,"K2"), NULL); + + strmap_set(map, "K2", (void*)101); + strmap_set(map, "K3", (void*)102); + strmap_set(map, "K4", (void*)103); + test_eq(strmap_size(map), 4); + strmap_assert_ok(map); + strmap_set(map, "K5", (void*)104); + strmap_set(map, "K6", (void*)105); + strmap_assert_ok(map); + + /* Test iterator. */ + iter = strmap_iter_init(map); + found_keys = smartlist_create(); + while (!strmap_iter_done(iter)) { + strmap_iter_get(iter,&k,&v); + smartlist_add(found_keys, tor_strdup(k)); + test_eq_ptr(v, strmap_get(map, k)); + + if (!strcmp(k, "K2")) { + iter = strmap_iter_next_rmv(map,iter); + } else { + iter = strmap_iter_next(map,iter); + } + } + + /* Make sure we removed K2, but not the others. */ + test_eq_ptr(strmap_get(map, "K2"), NULL); + test_eq_ptr(strmap_get(map, "K5"), (void*)104); + /* Make sure we visited everyone once */ + smartlist_sort_strings(found_keys); + visited = smartlist_join_strings(found_keys, ":", 0, NULL); + test_streq(visited, "K1:K2:K3:K4:K5:K6"); + + strmap_assert_ok(map); + /* Clean up after ourselves. */ + strmap_free(map, NULL); + map = NULL; + + /* Now try some lc functions. */ + map = strmap_new(); + strmap_set_lc(map,"Ab.C", (void*)1); + test_eq_ptr(strmap_get(map,"ab.c"), (void*)1); + strmap_assert_ok(map); + test_eq_ptr(strmap_get_lc(map,"AB.C"), (void*)1); + test_eq_ptr(strmap_get(map,"AB.C"), NULL); + test_eq_ptr(strmap_remove_lc(map,"aB.C"), (void*)1); + strmap_assert_ok(map); + test_eq_ptr(strmap_get_lc(map,"AB.C"), NULL); + + done: + if (map) + strmap_free(map,NULL); + if (found_keys) { + SMARTLIST_FOREACH(found_keys, char *, cp, tor_free(cp)); + smartlist_free(found_keys); + } + tor_free(visited); +} + +/** Run unit tests for getting the median of a list. */ +static void +test_container_order_functions(void) +{ + int lst[25], n = 0; + // int a=12,b=24,c=25,d=60,e=77; + +#define median() median_int(lst, n) + + lst[n++] = 12; + test_eq(12, median()); /* 12 */ + lst[n++] = 77; + //smartlist_shuffle(sl); + test_eq(12, median()); /* 12, 77 */ + lst[n++] = 77; + //smartlist_shuffle(sl); + test_eq(77, median()); /* 12, 77, 77 */ + lst[n++] = 24; + test_eq(24, median()); /* 12,24,77,77 */ + lst[n++] = 60; + lst[n++] = 12; + lst[n++] = 25; + //smartlist_shuffle(sl); + test_eq(25, median()); /* 12,12,24,25,60,77,77 */ +#undef median + + done: + ; +} + +#define CONTAINER_LEGACY(name) \ + { #name, legacy_test_helper, 0, &legacy_setup, test_container_ ## name } + +struct testcase_t container_tests[] = { + CONTAINER_LEGACY(smartlist_basic), + CONTAINER_LEGACY(smartlist_strings), + CONTAINER_LEGACY(smartlist_overlap), + CONTAINER_LEGACY(smartlist_digests), + CONTAINER_LEGACY(smartlist_join), + CONTAINER_LEGACY(bitarray), + CONTAINER_LEGACY(digestset), + CONTAINER_LEGACY(strmap), + CONTAINER_LEGACY(pqueue), + CONTAINER_LEGACY(order_functions), + END_OF_TESTCASES +}; + diff --git a/src/test/test_crypto.c b/src/test/test_crypto.c new file mode 100644 index 0000000000..b475914b1b --- /dev/null +++ b/src/test/test_crypto.c @@ -0,0 +1,788 @@ +/* Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2010, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "orconfig.h" +#define CRYPTO_PRIVATE +#include "or.h" +#include "test.h" + +/** Run unit tests for Diffie-Hellman functionality. */ +static void +test_crypto_dh(void) +{ + crypto_dh_env_t *dh1 = crypto_dh_new(); + crypto_dh_env_t *dh2 = crypto_dh_new(); + char p1[DH_BYTES]; + char p2[DH_BYTES]; + char s1[DH_BYTES]; + char s2[DH_BYTES]; + ssize_t s1len, s2len; + + test_eq(crypto_dh_get_bytes(dh1), DH_BYTES); + test_eq(crypto_dh_get_bytes(dh2), DH_BYTES); + + memset(p1, 0, DH_BYTES); + memset(p2, 0, DH_BYTES); + test_memeq(p1, p2, DH_BYTES); + test_assert(! crypto_dh_get_public(dh1, p1, DH_BYTES)); + test_memneq(p1, p2, DH_BYTES); + test_assert(! crypto_dh_get_public(dh2, p2, DH_BYTES)); + test_memneq(p1, p2, DH_BYTES); + + memset(s1, 0, DH_BYTES); + memset(s2, 0xFF, DH_BYTES); + s1len = crypto_dh_compute_secret(LOG_WARN, dh1, p2, DH_BYTES, s1, 50); + s2len = crypto_dh_compute_secret(LOG_WARN, dh2, p1, DH_BYTES, s2, 50); + test_assert(s1len > 0); + test_eq(s1len, s2len); + test_memeq(s1, s2, s1len); + + { + /* XXXX Now fabricate some bad values and make sure they get caught, + * Check 0, 1, N-1, >= N, etc. + */ + } + + done: + crypto_dh_free(dh1); + crypto_dh_free(dh2); +} + +/** Run unit tests for our random number generation function and its wrappers. + */ +static void +test_crypto_rng(void) +{ + int i, j, allok; + char data1[100], data2[100]; + double d; + + /* Try out RNG. */ + test_assert(! crypto_seed_rng(0)); + crypto_rand(data1, 100); + crypto_rand(data2, 100); + test_memneq(data1,data2,100); + allok = 1; + for (i = 0; i < 100; ++i) { + uint64_t big; + char *host; + j = crypto_rand_int(100); + if (i < 0 || i >= 100) + allok = 0; + big = crypto_rand_uint64(U64_LITERAL(1)<<40); + if (big >= (U64_LITERAL(1)<<40)) + allok = 0; + big = crypto_rand_uint64(U64_LITERAL(5)); + if (big >= 5) + allok = 0; + d = crypto_rand_double(); + test_assert(d >= 0); + test_assert(d < 1.0); + host = crypto_random_hostname(3,8,"www.",".onion"); + if (strcmpstart(host,"www.") || + strcmpend(host,".onion") || + strlen(host) < 13 || + strlen(host) > 18) + allok = 0; + tor_free(host); + } + test_assert(allok); + done: + ; +} + +/** Run unit tests for our AES functionality */ +static void +test_crypto_aes(void) +{ + char *data1 = NULL, *data2 = NULL, *data3 = NULL; + crypto_cipher_env_t *env1 = NULL, *env2 = NULL; + int i, j; + char *mem_op_hex_tmp=NULL; + + data1 = tor_malloc(1024); + data2 = tor_malloc(1024); + data3 = tor_malloc(1024); + + /* Now, test encryption and decryption with stream cipher. */ + data1[0]='\0'; + for (i = 1023; i>0; i -= 35) + strncat(data1, "Now is the time for all good onions", i); + + memset(data2, 0, 1024); + memset(data3, 0, 1024); + env1 = crypto_new_cipher_env(); + test_neq(env1, 0); + env2 = crypto_new_cipher_env(); + test_neq(env2, 0); + j = crypto_cipher_generate_key(env1); + crypto_cipher_set_key(env2, crypto_cipher_get_key(env1)); + crypto_cipher_encrypt_init_cipher(env1); + crypto_cipher_decrypt_init_cipher(env2); + + /* Try encrypting 512 chars. */ + crypto_cipher_encrypt(env1, data2, data1, 512); + crypto_cipher_decrypt(env2, data3, data2, 512); + test_memeq(data1, data3, 512); + test_memneq(data1, data2, 512); + + /* Now encrypt 1 at a time, and get 1 at a time. */ + for (j = 512; j < 560; ++j) { + crypto_cipher_encrypt(env1, data2+j, data1+j, 1); + } + for (j = 512; j < 560; ++j) { + crypto_cipher_decrypt(env2, data3+j, data2+j, 1); + } + test_memeq(data1, data3, 560); + /* Now encrypt 3 at a time, and get 5 at a time. */ + for (j = 560; j < 1024-5; j += 3) { + crypto_cipher_encrypt(env1, data2+j, data1+j, 3); + } + for (j = 560; j < 1024-5; j += 5) { + crypto_cipher_decrypt(env2, data3+j, data2+j, 5); + } + test_memeq(data1, data3, 1024-5); + /* Now make sure that when we encrypt with different chunk sizes, we get + the same results. */ + crypto_free_cipher_env(env2); + env2 = NULL; + + memset(data3, 0, 1024); + env2 = crypto_new_cipher_env(); + test_neq(env2, 0); + crypto_cipher_set_key(env2, crypto_cipher_get_key(env1)); + crypto_cipher_encrypt_init_cipher(env2); + for (j = 0; j < 1024-16; j += 17) { + crypto_cipher_encrypt(env2, data3+j, data1+j, 17); + } + for (j= 0; j < 1024-16; ++j) { + if (data2[j] != data3[j]) { + printf("%d: %d\t%d\n", j, (int) data2[j], (int) data3[j]); + } + } + test_memeq(data2, data3, 1024-16); + crypto_free_cipher_env(env1); + env1 = NULL; + crypto_free_cipher_env(env2); + env2 = NULL; + + /* NIST test vector for aes. */ + env1 = crypto_new_cipher_env(); /* IV starts at 0 */ + crypto_cipher_set_key(env1, "\x80\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00"); + crypto_cipher_encrypt_init_cipher(env1); + crypto_cipher_encrypt(env1, data1, + "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00", 16); + test_memeq_hex(data1, "0EDD33D3C621E546455BD8BA1418BEC8"); + + /* Now test rollover. All these values are originally from a python + * script. */ + crypto_cipher_set_iv(env1, "\x00\x00\x00\x00\x00\x00\x00\x00" + "\xff\xff\xff\xff\xff\xff\xff\xff"); + memset(data2, 0, 1024); + crypto_cipher_encrypt(env1, data1, data2, 32); + test_memeq_hex(data1, "335fe6da56f843199066c14a00a40231" + "cdd0b917dbc7186908a6bfb5ffd574d3"); + + crypto_cipher_set_iv(env1, "\x00\x00\x00\x00\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff"); + memset(data2, 0, 1024); + crypto_cipher_encrypt(env1, data1, data2, 32); + test_memeq_hex(data1, "e627c6423fa2d77832a02b2794094b73" + "3e63c721df790d2c6469cc1953a3ffac"); + + crypto_cipher_set_iv(env1, "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff"); + memset(data2, 0, 1024); + crypto_cipher_encrypt(env1, data1, data2, 32); + test_memeq_hex(data1, "2aed2bff0de54f9328efd070bf48f70a" + "0EDD33D3C621E546455BD8BA1418BEC8"); + + /* Now check rollover on inplace cipher. */ + crypto_cipher_set_iv(env1, "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff"); + crypto_cipher_crypt_inplace(env1, data2, 64); + test_memeq_hex(data2, "2aed2bff0de54f9328efd070bf48f70a" + "0EDD33D3C621E546455BD8BA1418BEC8" + "93e2c5243d6839eac58503919192f7ae" + "1908e67cafa08d508816659c2e693191"); + crypto_cipher_set_iv(env1, "\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff"); + crypto_cipher_crypt_inplace(env1, data2, 64); + test_assert(tor_mem_is_zero(data2, 64)); + + done: + tor_free(mem_op_hex_tmp); + if (env1) + crypto_free_cipher_env(env1); + if (env2) + crypto_free_cipher_env(env2); + tor_free(data1); + tor_free(data2); + tor_free(data3); +} + +/** Run unit tests for our SHA-1 functionality */ +static void +test_crypto_sha(void) +{ + crypto_digest_env_t *d1 = NULL, *d2 = NULL; + int i; + char key[80]; + char digest[32]; + char data[50]; + char d_out1[DIGEST_LEN], d_out2[DIGEST256_LEN]; + char *mem_op_hex_tmp=NULL; + + /* Test SHA-1 with a test vector from the specification. */ + i = crypto_digest(data, "abc", 3); + test_memeq_hex(data, "A9993E364706816ABA3E25717850C26C9CD0D89D"); + + /* Test SHA-256 with a test vector from the specification. */ + i = crypto_digest256(data, "abc", 3, DIGEST_SHA256); + test_memeq_hex(data, "BA7816BF8F01CFEA414140DE5DAE2223B00361A3" + "96177A9CB410FF61F20015AD"); + + /* Test HMAC-SHA-1 with test cases from RFC2202. */ + + /* Case 1. */ + memset(key, 0x0b, 20); + crypto_hmac_sha1(digest, key, 20, "Hi There", 8); + test_streq(hex_str(digest, 20), + "B617318655057264E28BC0B6FB378C8EF146BE00"); + /* Case 2. */ + crypto_hmac_sha1(digest, "Jefe", 4, "what do ya want for nothing?", 28); + test_streq(hex_str(digest, 20), + "EFFCDF6AE5EB2FA2D27416D5F184DF9C259A7C79"); + + /* Case 4. */ + base16_decode(key, 25, + "0102030405060708090a0b0c0d0e0f10111213141516171819", 50); + memset(data, 0xcd, 50); + crypto_hmac_sha1(digest, key, 25, data, 50); + test_streq(hex_str(digest, 20), + "4C9007F4026250C6BC8414F9BF50C86C2D7235DA"); + + /* Case 5. */ + memset(key, 0xaa, 80); + crypto_hmac_sha1(digest, key, 80, + "Test Using Larger Than Block-Size Key - Hash Key First", + 54); + test_streq(hex_str(digest, 20), + "AA4AE5E15272D00E95705637CE8A3B55ED402112"); + + /* Incremental digest code. */ + d1 = crypto_new_digest_env(); + test_assert(d1); + crypto_digest_add_bytes(d1, "abcdef", 6); + d2 = crypto_digest_dup(d1); + test_assert(d2); + crypto_digest_add_bytes(d2, "ghijkl", 6); + crypto_digest_get_digest(d2, d_out1, sizeof(d_out1)); + crypto_digest(d_out2, "abcdefghijkl", 12); + test_memeq(d_out1, d_out2, DIGEST_LEN); + crypto_digest_assign(d2, d1); + crypto_digest_add_bytes(d2, "mno", 3); + crypto_digest_get_digest(d2, d_out1, sizeof(d_out1)); + crypto_digest(d_out2, "abcdefmno", 9); + test_memeq(d_out1, d_out2, DIGEST_LEN); + crypto_digest_get_digest(d1, d_out1, sizeof(d_out1)); + crypto_digest(d_out2, "abcdef", 6); + test_memeq(d_out1, d_out2, DIGEST_LEN); + crypto_free_digest_env(d1); + crypto_free_digest_env(d2); + + /* Incremental digest code with sha256 */ + d1 = crypto_new_digest256_env(DIGEST_SHA256); + test_assert(d1); + crypto_digest_add_bytes(d1, "abcdef", 6); + d2 = crypto_digest_dup(d1); + test_assert(d2); + crypto_digest_add_bytes(d2, "ghijkl", 6); + crypto_digest_get_digest(d2, d_out1, sizeof(d_out1)); + crypto_digest256(d_out2, "abcdefghijkl", 12, DIGEST_SHA256); + test_memeq(d_out1, d_out2, DIGEST_LEN); + crypto_digest_assign(d2, d1); + crypto_digest_add_bytes(d2, "mno", 3); + crypto_digest_get_digest(d2, d_out1, sizeof(d_out1)); + crypto_digest256(d_out2, "abcdefmno", 9, DIGEST_SHA256); + test_memeq(d_out1, d_out2, DIGEST_LEN); + crypto_digest_get_digest(d1, d_out1, sizeof(d_out1)); + crypto_digest256(d_out2, "abcdef", 6, DIGEST_SHA256); + test_memeq(d_out1, d_out2, DIGEST_LEN); + + done: + if (d1) + crypto_free_digest_env(d1); + if (d2) + crypto_free_digest_env(d2); + tor_free(mem_op_hex_tmp); +} + +/** Run unit tests for our public key crypto functions */ +static void +test_crypto_pk(void) +{ + crypto_pk_env_t *pk1 = NULL, *pk2 = NULL; + char *encoded = NULL; + char data1[1024], data2[1024], data3[1024]; + size_t size; + int i, j, p, len; + + /* Public-key ciphers */ + pk1 = pk_generate(0); + pk2 = crypto_new_pk_env(); + test_assert(pk1 && pk2); + test_assert(! crypto_pk_write_public_key_to_string(pk1, &encoded, &size)); + test_assert(! crypto_pk_read_public_key_from_string(pk2, encoded, size)); + test_eq(0, crypto_pk_cmp_keys(pk1, pk2)); + + test_eq(128, crypto_pk_keysize(pk1)); + test_eq(128, crypto_pk_keysize(pk2)); + + test_eq(128, crypto_pk_public_encrypt(pk2, data1, "Hello whirled.", 15, + PK_PKCS1_OAEP_PADDING)); + test_eq(128, crypto_pk_public_encrypt(pk1, data2, "Hello whirled.", 15, + PK_PKCS1_OAEP_PADDING)); + /* oaep padding should make encryption not match */ + test_memneq(data1, data2, 128); + test_eq(15, crypto_pk_private_decrypt(pk1, data3, data1, 128, + PK_PKCS1_OAEP_PADDING,1)); + test_streq(data3, "Hello whirled."); + memset(data3, 0, 1024); + test_eq(15, crypto_pk_private_decrypt(pk1, data3, data2, 128, + PK_PKCS1_OAEP_PADDING,1)); + test_streq(data3, "Hello whirled."); + /* Can't decrypt with public key. */ + test_eq(-1, crypto_pk_private_decrypt(pk2, data3, data2, 128, + PK_PKCS1_OAEP_PADDING,1)); + /* Try again with bad padding */ + memcpy(data2+1, "XYZZY", 5); /* This has fails ~ once-in-2^40 */ + test_eq(-1, crypto_pk_private_decrypt(pk1, data3, data2, 128, + PK_PKCS1_OAEP_PADDING,1)); + + /* File operations: save and load private key */ + test_assert(! crypto_pk_write_private_key_to_filename(pk1, + get_fname("pkey1"))); + /* failing case for read: can't read. */ + test_assert(crypto_pk_read_private_key_from_filename(pk2, + get_fname("xyzzy")) < 0); + write_str_to_file(get_fname("xyzzy"), "foobar", 6); + /* Failing case for read: no key. */ + test_assert(crypto_pk_read_private_key_from_filename(pk2, + get_fname("xyzzy")) < 0); + test_assert(! crypto_pk_read_private_key_from_filename(pk2, + get_fname("pkey1"))); + test_eq(15, crypto_pk_private_decrypt(pk2, data3, data1, 128, + PK_PKCS1_OAEP_PADDING,1)); + + /* Now try signing. */ + strlcpy(data1, "Ossifrage", 1024); + test_eq(128, crypto_pk_private_sign(pk1, data2, data1, 10)); + test_eq(10, crypto_pk_public_checksig(pk1, data3, data2, 128)); + test_streq(data3, "Ossifrage"); + /* Try signing digests. */ + test_eq(128, crypto_pk_private_sign_digest(pk1, data2, data1, 10)); + test_eq(20, crypto_pk_public_checksig(pk1, data3, data2, 128)); + test_eq(0, crypto_pk_public_checksig_digest(pk1, data1, 10, data2, 128)); + test_eq(-1, crypto_pk_public_checksig_digest(pk1, data1, 11, data2, 128)); + /*XXXX test failed signing*/ + + /* Try encoding */ + crypto_free_pk_env(pk2); + pk2 = NULL; + i = crypto_pk_asn1_encode(pk1, data1, 1024); + test_assert(i>0); + pk2 = crypto_pk_asn1_decode(data1, i); + test_assert(crypto_pk_cmp_keys(pk1,pk2) == 0); + + /* Try with hybrid encryption wrappers. */ + crypto_rand(data1, 1024); + for (i = 0; i < 3; ++i) { + for (j = 85; j < 140; ++j) { + memset(data2,0,1024); + memset(data3,0,1024); + if (i == 0 && j < 129) + continue; + p = (i==0)?PK_NO_PADDING: + (i==1)?PK_PKCS1_PADDING:PK_PKCS1_OAEP_PADDING; + len = crypto_pk_public_hybrid_encrypt(pk1,data2,data1,j,p,0); + test_assert(len>=0); + len = crypto_pk_private_hybrid_decrypt(pk1,data3,data2,len,p,1); + test_eq(len,j); + test_memeq(data1,data3,j); + } + } + + /* Try copy_full */ + crypto_free_pk_env(pk2); + pk2 = crypto_pk_copy_full(pk1); + test_assert(pk2 != NULL); + test_neq_ptr(pk1, pk2); + test_assert(crypto_pk_cmp_keys(pk1,pk2) == 0); + + done: + if (pk1) + crypto_free_pk_env(pk1); + if (pk2) + crypto_free_pk_env(pk2); + tor_free(encoded); +} + +/** Run unit tests for misc crypto formatting functionality (base64, base32, + * fingerprints, etc) */ +static void +test_crypto_formats(void) +{ + char *data1 = NULL, *data2 = NULL, *data3 = NULL; + int i, j, idx; + + data1 = tor_malloc(1024); + data2 = tor_malloc(1024); + data3 = tor_malloc(1024); + test_assert(data1 && data2 && data3); + + /* Base64 tests */ + memset(data1, 6, 1024); + for (idx = 0; idx < 10; ++idx) { + i = base64_encode(data2, 1024, data1, idx); + test_assert(i >= 0); + j = base64_decode(data3, 1024, data2, i); + test_eq(j,idx); + test_memeq(data3, data1, idx); + } + + strlcpy(data1, "Test string that contains 35 chars.", 1024); + strlcat(data1, " 2nd string that contains 35 chars.", 1024); + + i = base64_encode(data2, 1024, data1, 71); + test_assert(i >= 0); + j = base64_decode(data3, 1024, data2, i); + test_eq(j, 71); + test_streq(data3, data1); + test_assert(data2[i] == '\0'); + + crypto_rand(data1, DIGEST_LEN); + memset(data2, 100, 1024); + digest_to_base64(data2, data1); + test_eq(BASE64_DIGEST_LEN, strlen(data2)); + test_eq(100, data2[BASE64_DIGEST_LEN+2]); + memset(data3, 99, 1024); + test_eq(digest_from_base64(data3, data2), 0); + test_memeq(data1, data3, DIGEST_LEN); + test_eq(99, data3[DIGEST_LEN+1]); + + test_assert(digest_from_base64(data3, "###") < 0); + + /* Encoding SHA256 */ + crypto_rand(data2, DIGEST256_LEN); + memset(data2, 100, 1024); + digest256_to_base64(data2, data1); + test_eq(BASE64_DIGEST256_LEN, strlen(data2)); + test_eq(100, data2[BASE64_DIGEST256_LEN+2]); + memset(data3, 99, 1024); + test_eq(digest256_from_base64(data3, data2), 0); + test_memeq(data1, data3, DIGEST256_LEN); + test_eq(99, data3[DIGEST256_LEN+1]); + + /* Base32 tests */ + strlcpy(data1, "5chrs", 1024); + /* bit pattern is: [35 63 68 72 73] -> + * [00110101 01100011 01101000 01110010 01110011] + * By 5s: [00110 10101 10001 10110 10000 11100 10011 10011] + */ + base32_encode(data2, 9, data1, 5); + test_streq(data2, "gvrwq4tt"); + + strlcpy(data1, "\xFF\xF5\x6D\x44\xAE\x0D\x5C\xC9\x62\xC4", 1024); + base32_encode(data2, 30, data1, 10); + test_streq(data2, "772w2rfobvomsywe"); + + /* Base16 tests */ + strlcpy(data1, "6chrs\xff", 1024); + base16_encode(data2, 13, data1, 6); + test_streq(data2, "3663687273FF"); + + strlcpy(data1, "f0d678affc000100", 1024); + i = base16_decode(data2, 8, data1, 16); + test_eq(i,0); + test_memeq(data2, "\xf0\xd6\x78\xaf\xfc\x00\x01\x00",8); + + /* now try some failing base16 decodes */ + test_eq(-1, base16_decode(data2, 8, data1, 15)); /* odd input len */ + test_eq(-1, base16_decode(data2, 7, data1, 16)); /* dest too short */ + strlcpy(data1, "f0dz!8affc000100", 1024); + test_eq(-1, base16_decode(data2, 8, data1, 16)); + + tor_free(data1); + tor_free(data2); + tor_free(data3); + + /* Add spaces to fingerprint */ + { + data1 = tor_strdup("ABCD1234ABCD56780000ABCD1234ABCD56780000"); + test_eq(strlen(data1), 40); + data2 = tor_malloc(FINGERPRINT_LEN+1); + add_spaces_to_fp(data2, FINGERPRINT_LEN+1, data1); + test_streq(data2, "ABCD 1234 ABCD 5678 0000 ABCD 1234 ABCD 5678 0000"); + tor_free(data1); + tor_free(data2); + } + + /* Check fingerprint */ + { + test_assert(crypto_pk_check_fingerprint_syntax( + "ABCD 1234 ABCD 5678 0000 ABCD 1234 ABCD 5678 0000")); + test_assert(!crypto_pk_check_fingerprint_syntax( + "ABCD 1234 ABCD 5678 0000 ABCD 1234 ABCD 5678 000")); + test_assert(!crypto_pk_check_fingerprint_syntax( + "ABCD 1234 ABCD 5678 0000 ABCD 1234 ABCD 5678 00000")); + test_assert(!crypto_pk_check_fingerprint_syntax( + "ABCD 1234 ABCD 5678 0000 ABCD1234 ABCD 5678 0000")); + test_assert(!crypto_pk_check_fingerprint_syntax( + "ABCD 1234 ABCD 5678 0000 ABCD1234 ABCD 5678 00000")); + test_assert(!crypto_pk_check_fingerprint_syntax( + "ACD 1234 ABCD 5678 0000 ABCD 1234 ABCD 5678 00000")); + } + + done: + tor_free(data1); + tor_free(data2); + tor_free(data3); +} + +/** Run unit tests for our secret-to-key passphrase hashing functionality. */ +static void +test_crypto_s2k(void) +{ + char buf[29]; + char buf2[29]; + char *buf3 = NULL; + int i; + + memset(buf, 0, sizeof(buf)); + memset(buf2, 0, sizeof(buf2)); + buf3 = tor_malloc(65536); + memset(buf3, 0, 65536); + + secret_to_key(buf+9, 20, "", 0, buf); + crypto_digest(buf2+9, buf3, 1024); + test_memeq(buf, buf2, 29); + + memcpy(buf,"vrbacrda",8); + memcpy(buf2,"vrbacrda",8); + buf[8] = 96; + buf2[8] = 96; + secret_to_key(buf+9, 20, "12345678", 8, buf); + for (i = 0; i < 65536; i += 16) { + memcpy(buf3+i, "vrbacrda12345678", 16); + } + crypto_digest(buf2+9, buf3, 65536); + test_memeq(buf, buf2, 29); + + done: + tor_free(buf3); +} + +/** Test AES-CTR encryption and decryption with IV. */ +static void +test_crypto_aes_iv(void) +{ + crypto_cipher_env_t *cipher; + char *plain, *encrypted1, *encrypted2, *decrypted1, *decrypted2; + char plain_1[1], plain_15[15], plain_16[16], plain_17[17]; + char key1[16], key2[16]; + ssize_t encrypted_size, decrypted_size; + + plain = tor_malloc(4095); + encrypted1 = tor_malloc(4095 + 1 + 16); + encrypted2 = tor_malloc(4095 + 1 + 16); + decrypted1 = tor_malloc(4095 + 1); + decrypted2 = tor_malloc(4095 + 1); + + crypto_rand(plain, 4095); + crypto_rand(key1, 16); + crypto_rand(key2, 16); + crypto_rand(plain_1, 1); + crypto_rand(plain_15, 15); + crypto_rand(plain_16, 16); + crypto_rand(plain_17, 17); + key1[0] = key2[0] + 128; /* Make sure that contents are different. */ + /* Encrypt and decrypt with the same key. */ + cipher = crypto_create_init_cipher(key1, 1); + encrypted_size = crypto_cipher_encrypt_with_iv(cipher, encrypted1, 16 + 4095, + plain, 4095); + crypto_free_cipher_env(cipher); + cipher = NULL; + test_eq(encrypted_size, 16 + 4095); + tor_assert(encrypted_size > 0); /* This is obviously true, since 4111 is + * greater than 0, but its truth is not + * obvious to all analysis tools. */ + cipher = crypto_create_init_cipher(key1, 0); + decrypted_size = crypto_cipher_decrypt_with_iv(cipher, decrypted1, 4095, + encrypted1, encrypted_size); + crypto_free_cipher_env(cipher); + cipher = NULL; + test_eq(decrypted_size, 4095); + tor_assert(decrypted_size > 0); + test_memeq(plain, decrypted1, 4095); + /* Encrypt a second time (with a new random initialization vector). */ + cipher = crypto_create_init_cipher(key1, 1); + encrypted_size = crypto_cipher_encrypt_with_iv(cipher, encrypted2, 16 + 4095, + plain, 4095); + crypto_free_cipher_env(cipher); + cipher = NULL; + test_eq(encrypted_size, 16 + 4095); + tor_assert(encrypted_size > 0); + cipher = crypto_create_init_cipher(key1, 0); + decrypted_size = crypto_cipher_decrypt_with_iv(cipher, decrypted2, 4095, + encrypted2, encrypted_size); + crypto_free_cipher_env(cipher); + cipher = NULL; + test_eq(decrypted_size, 4095); + tor_assert(decrypted_size > 0); + test_memeq(plain, decrypted2, 4095); + test_memneq(encrypted1, encrypted2, encrypted_size); + /* Decrypt with the wrong key. */ + cipher = crypto_create_init_cipher(key2, 0); + decrypted_size = crypto_cipher_decrypt_with_iv(cipher, decrypted2, 4095, + encrypted1, encrypted_size); + crypto_free_cipher_env(cipher); + cipher = NULL; + test_memneq(plain, decrypted2, encrypted_size); + /* Alter the initialization vector. */ + encrypted1[0] += 42; + cipher = crypto_create_init_cipher(key1, 0); + decrypted_size = crypto_cipher_decrypt_with_iv(cipher, decrypted1, 4095, + encrypted1, encrypted_size); + crypto_free_cipher_env(cipher); + cipher = NULL; + test_memneq(plain, decrypted2, 4095); + /* Special length case: 1. */ + cipher = crypto_create_init_cipher(key1, 1); + encrypted_size = crypto_cipher_encrypt_with_iv(cipher, encrypted1, 16 + 1, + plain_1, 1); + crypto_free_cipher_env(cipher); + cipher = NULL; + test_eq(encrypted_size, 16 + 1); + tor_assert(encrypted_size > 0); + cipher = crypto_create_init_cipher(key1, 0); + decrypted_size = crypto_cipher_decrypt_with_iv(cipher, decrypted1, 1, + encrypted1, encrypted_size); + crypto_free_cipher_env(cipher); + cipher = NULL; + test_eq(decrypted_size, 1); + tor_assert(decrypted_size > 0); + test_memeq(plain_1, decrypted1, 1); + /* Special length case: 15. */ + cipher = crypto_create_init_cipher(key1, 1); + encrypted_size = crypto_cipher_encrypt_with_iv(cipher, encrypted1, 16 + 15, + plain_15, 15); + crypto_free_cipher_env(cipher); + cipher = NULL; + test_eq(encrypted_size, 16 + 15); + tor_assert(encrypted_size > 0); + cipher = crypto_create_init_cipher(key1, 0); + decrypted_size = crypto_cipher_decrypt_with_iv(cipher, decrypted1, 15, + encrypted1, encrypted_size); + crypto_free_cipher_env(cipher); + cipher = NULL; + test_eq(decrypted_size, 15); + tor_assert(decrypted_size > 0); + test_memeq(plain_15, decrypted1, 15); + /* Special length case: 16. */ + cipher = crypto_create_init_cipher(key1, 1); + encrypted_size = crypto_cipher_encrypt_with_iv(cipher, encrypted1, 16 + 16, + plain_16, 16); + crypto_free_cipher_env(cipher); + cipher = NULL; + test_eq(encrypted_size, 16 + 16); + tor_assert(encrypted_size > 0); + cipher = crypto_create_init_cipher(key1, 0); + decrypted_size = crypto_cipher_decrypt_with_iv(cipher, decrypted1, 16, + encrypted1, encrypted_size); + crypto_free_cipher_env(cipher); + cipher = NULL; + test_eq(decrypted_size, 16); + tor_assert(decrypted_size > 0); + test_memeq(plain_16, decrypted1, 16); + /* Special length case: 17. */ + cipher = crypto_create_init_cipher(key1, 1); + encrypted_size = crypto_cipher_encrypt_with_iv(cipher, encrypted1, 16 + 17, + plain_17, 17); + crypto_free_cipher_env(cipher); + cipher = NULL; + test_eq(encrypted_size, 16 + 17); + tor_assert(encrypted_size > 0); + cipher = crypto_create_init_cipher(key1, 0); + decrypted_size = crypto_cipher_decrypt_with_iv(cipher, decrypted1, 17, + encrypted1, encrypted_size); + test_eq(decrypted_size, 17); + tor_assert(decrypted_size > 0); + test_memeq(plain_17, decrypted1, 17); + + done: + /* Free memory. */ + tor_free(plain); + tor_free(encrypted1); + tor_free(encrypted2); + tor_free(decrypted1); + tor_free(decrypted2); + if (cipher) + crypto_free_cipher_env(cipher); +} + +/** Test base32 decoding. */ +static void +test_crypto_base32_decode(void) +{ + char plain[60], encoded[96 + 1], decoded[60]; + int res; + crypto_rand(plain, 60); + /* Encode and decode a random string. */ + base32_encode(encoded, 96 + 1, plain, 60); + res = base32_decode(decoded, 60, encoded, 96); + test_eq(res, 0); + test_memeq(plain, decoded, 60); + /* Encode, uppercase, and decode a random string. */ + base32_encode(encoded, 96 + 1, plain, 60); + tor_strupper(encoded); + res = base32_decode(decoded, 60, encoded, 96); + test_eq(res, 0); + test_memeq(plain, decoded, 60); + /* Change encoded string and decode. */ + if (encoded[0] == 'A' || encoded[0] == 'a') + encoded[0] = 'B'; + else + encoded[0] = 'A'; + res = base32_decode(decoded, 60, encoded, 96); + test_eq(res, 0); + test_memneq(plain, decoded, 60); + /* Bad encodings. */ + encoded[0] = '!'; + res = base32_decode(decoded, 60, encoded, 96); + test_assert(res < 0); + + done: + ; +} + +#define CRYPTO_LEGACY(name) \ + { #name, legacy_test_helper, 0, &legacy_setup, test_crypto_ ## name } + +struct testcase_t crypto_tests[] = { + CRYPTO_LEGACY(formats), + CRYPTO_LEGACY(rng), + CRYPTO_LEGACY(aes), + CRYPTO_LEGACY(sha), + CRYPTO_LEGACY(pk), + CRYPTO_LEGACY(dh), + CRYPTO_LEGACY(s2k), + CRYPTO_LEGACY(aes_iv), + CRYPTO_LEGACY(base32_decode), + END_OF_TESTCASES +}; + diff --git a/src/or/test_data.c b/src/test/test_data.c index f926ee17dd..f926ee17dd 100644 --- a/src/or/test_data.c +++ b/src/test/test_data.c diff --git a/src/test/test_dir.c b/src/test/test_dir.c new file mode 100644 index 0000000000..a129bf9777 --- /dev/null +++ b/src/test/test_dir.c @@ -0,0 +1,1318 @@ +/* Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2010, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "orconfig.h" +#define DIRSERV_PRIVATE +#define DIRVOTE_PRIVATE +#define ROUTER_PRIVATE +#include "or.h" +#include "directory.h" +#include "dirserv.h" +#include "dirvote.h" +#include "networkstatus.h" +#include "router.h" +#include "routerlist.h" +#include "routerparse.h" +#include "test.h" + +static void +test_dir_nicknames(void) +{ + test_assert( is_legal_nickname("a")); + test_assert(!is_legal_nickname("")); + test_assert(!is_legal_nickname("abcdefghijklmnopqrst")); /* 20 chars */ + test_assert(!is_legal_nickname("hyphen-")); /* bad char */ + test_assert( is_legal_nickname("abcdefghijklmnopqrs")); /* 19 chars */ + test_assert(!is_legal_nickname("$AAAAAAAA01234AAAAAAAAAAAAAAAAAAAAAAAAAAA")); + /* valid */ + test_assert( is_legal_nickname_or_hexdigest( + "$AAAAAAAA01234AAAAAAAAAAAAAAAAAAAAAAAAAAA")); + test_assert( is_legal_nickname_or_hexdigest( + "$AAAAAAAA01234AAAAAAAAAAAAAAAAAAAAAAAAAAA=fred")); + test_assert( is_legal_nickname_or_hexdigest( + "$AAAAAAAA01234AAAAAAAAAAAAAAAAAAAAAAAAAAA~fred")); + /* too short */ + test_assert(!is_legal_nickname_or_hexdigest( + "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")); + /* illegal char */ + test_assert(!is_legal_nickname_or_hexdigest( + "$AAAAAAzAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")); + /* hex part too long */ + test_assert(!is_legal_nickname_or_hexdigest( + "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")); + test_assert(!is_legal_nickname_or_hexdigest( + "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=fred")); + /* Bad nickname */ + test_assert(!is_legal_nickname_or_hexdigest( + "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=")); + test_assert(!is_legal_nickname_or_hexdigest( + "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~")); + test_assert(!is_legal_nickname_or_hexdigest( + "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~hyphen-")); + test_assert(!is_legal_nickname_or_hexdigest( + "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~" + "abcdefghijklmnoppqrst")); + /* Bad extra char. */ + test_assert(!is_legal_nickname_or_hexdigest( + "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA!")); + test_assert(is_legal_nickname_or_hexdigest("xyzzy")); + test_assert(is_legal_nickname_or_hexdigest("abcdefghijklmnopqrs")); + test_assert(!is_legal_nickname_or_hexdigest("abcdefghijklmnopqrst")); + done: + ; +} + +/** Run unit tests for router descriptor generation logic. */ +static void +test_dir_formats(void) +{ + char buf[8192], buf2[8192]; + char platform[256]; + char fingerprint[FINGERPRINT_LEN+1]; + char *pk1_str = NULL, *pk2_str = NULL, *pk3_str = NULL, *cp; + size_t pk1_str_len, pk2_str_len, pk3_str_len; + routerinfo_t *r1=NULL, *r2=NULL; + crypto_pk_env_t *pk1 = NULL, *pk2 = NULL, *pk3 = NULL; + routerinfo_t *rp1 = NULL; + addr_policy_t *ex1, *ex2; + routerlist_t *dir1 = NULL, *dir2 = NULL; + + pk1 = pk_generate(0); + pk2 = pk_generate(1); + pk3 = pk_generate(2); + + test_assert(pk1 && pk2 && pk3); + + get_platform_str(platform, sizeof(platform)); + r1 = tor_malloc_zero(sizeof(routerinfo_t)); + r1->address = tor_strdup("18.244.0.1"); + r1->addr = 0xc0a80001u; /* 192.168.0.1 */ + r1->cache_info.published_on = 0; + r1->or_port = 9000; + r1->dir_port = 9003; + r1->onion_pkey = crypto_pk_dup_key(pk1); + r1->identity_pkey = crypto_pk_dup_key(pk2); + r1->bandwidthrate = 1000; + r1->bandwidthburst = 5000; + r1->bandwidthcapacity = 10000; + r1->exit_policy = NULL; + r1->nickname = tor_strdup("Magri"); + r1->platform = tor_strdup(platform); + + ex1 = tor_malloc_zero(sizeof(addr_policy_t)); + ex2 = tor_malloc_zero(sizeof(addr_policy_t)); + ex1->policy_type = ADDR_POLICY_ACCEPT; + tor_addr_from_ipv4h(&ex1->addr, 0); + ex1->maskbits = 0; + ex1->prt_min = ex1->prt_max = 80; + ex2->policy_type = ADDR_POLICY_REJECT; + tor_addr_from_ipv4h(&ex2->addr, 18<<24); + ex2->maskbits = 8; + ex2->prt_min = ex2->prt_max = 24; + r2 = tor_malloc_zero(sizeof(routerinfo_t)); + r2->address = tor_strdup("1.1.1.1"); + r2->addr = 0x0a030201u; /* 10.3.2.1 */ + r2->platform = tor_strdup(platform); + r2->cache_info.published_on = 5; + r2->or_port = 9005; + r2->dir_port = 0; + r2->onion_pkey = crypto_pk_dup_key(pk2); + r2->identity_pkey = crypto_pk_dup_key(pk1); + r2->bandwidthrate = r2->bandwidthburst = r2->bandwidthcapacity = 3000; + r2->exit_policy = smartlist_create(); + smartlist_add(r2->exit_policy, ex2); + smartlist_add(r2->exit_policy, ex1); + r2->nickname = tor_strdup("Fred"); + + test_assert(!crypto_pk_write_public_key_to_string(pk1, &pk1_str, + &pk1_str_len)); + test_assert(!crypto_pk_write_public_key_to_string(pk2 , &pk2_str, + &pk2_str_len)); + test_assert(!crypto_pk_write_public_key_to_string(pk3 , &pk3_str, + &pk3_str_len)); + + memset(buf, 0, 2048); + test_assert(router_dump_router_to_string(buf, 2048, r1, pk2)>0); + + strlcpy(buf2, "router Magri 18.244.0.1 9000 0 9003\n" + "platform Tor "VERSION" on ", sizeof(buf2)); + strlcat(buf2, get_uname(), sizeof(buf2)); + strlcat(buf2, "\n" + "opt protocols Link 1 2 Circuit 1\n" + "published 1970-01-01 00:00:00\n" + "opt fingerprint ", sizeof(buf2)); + test_assert(!crypto_pk_get_fingerprint(pk2, fingerprint, 1)); + strlcat(buf2, fingerprint, sizeof(buf2)); + strlcat(buf2, "\nuptime 0\n" + /* XXX the "0" above is hard-coded, but even if we made it reflect + * uptime, that still wouldn't make it right, because the two + * descriptors might be made on different seconds... hm. */ + "bandwidth 1000 5000 10000\n" + "opt extra-info-digest 0000000000000000000000000000000000000000\n" + "onion-key\n", sizeof(buf2)); + strlcat(buf2, pk1_str, sizeof(buf2)); + strlcat(buf2, "signing-key\n", sizeof(buf2)); + strlcat(buf2, pk2_str, sizeof(buf2)); + strlcat(buf2, "opt hidden-service-dir\n", sizeof(buf2)); + strlcat(buf2, "reject *:*\nrouter-signature\n", sizeof(buf2)); + buf[strlen(buf2)] = '\0'; /* Don't compare the sig; it's never the same + * twice */ + + test_streq(buf, buf2); + + test_assert(router_dump_router_to_string(buf, 2048, r1, pk2)>0); + cp = buf; + rp1 = router_parse_entry_from_string((const char*)cp,NULL,1,0,NULL); + test_assert(rp1); + test_streq(rp1->address, r1->address); + test_eq(rp1->or_port, r1->or_port); + //test_eq(rp1->dir_port, r1->dir_port); + test_eq(rp1->bandwidthrate, r1->bandwidthrate); + test_eq(rp1->bandwidthburst, r1->bandwidthburst); + test_eq(rp1->bandwidthcapacity, r1->bandwidthcapacity); + test_assert(crypto_pk_cmp_keys(rp1->onion_pkey, pk1) == 0); + test_assert(crypto_pk_cmp_keys(rp1->identity_pkey, pk2) == 0); + //test_assert(rp1->exit_policy == NULL); + +#if 0 + /* XXX Once we have exit policies, test this again. XXX */ + strlcpy(buf2, "router tor.tor.tor 9005 0 0 3000\n", sizeof(buf2)); + strlcat(buf2, pk2_str, sizeof(buf2)); + strlcat(buf2, "signing-key\n", sizeof(buf2)); + strlcat(buf2, pk1_str, sizeof(buf2)); + strlcat(buf2, "accept *:80\nreject 18.*:24\n\n", sizeof(buf2)); + test_assert(router_dump_router_to_string(buf, 2048, &r2, pk2)>0); + test_streq(buf, buf2); + + cp = buf; + rp2 = router_parse_entry_from_string(&cp,1); + test_assert(rp2); + test_streq(rp2->address, r2.address); + test_eq(rp2->or_port, r2.or_port); + test_eq(rp2->dir_port, r2.dir_port); + test_eq(rp2->bandwidth, r2.bandwidth); + test_assert(crypto_pk_cmp_keys(rp2->onion_pkey, pk2) == 0); + test_assert(crypto_pk_cmp_keys(rp2->identity_pkey, pk1) == 0); + test_eq(rp2->exit_policy->policy_type, EXIT_POLICY_ACCEPT); + test_streq(rp2->exit_policy->string, "accept *:80"); + test_streq(rp2->exit_policy->address, "*"); + test_streq(rp2->exit_policy->port, "80"); + test_eq(rp2->exit_policy->next->policy_type, EXIT_POLICY_REJECT); + test_streq(rp2->exit_policy->next->string, "reject 18.*:24"); + test_streq(rp2->exit_policy->next->address, "18.*"); + test_streq(rp2->exit_policy->next->port, "24"); + test_assert(rp2->exit_policy->next->next == NULL); + + /* Okay, now for the directories. */ + { + fingerprint_list = smartlist_create(); + crypto_pk_get_fingerprint(pk2, buf, 1); + add_fingerprint_to_dir("Magri", buf, fingerprint_list); + crypto_pk_get_fingerprint(pk1, buf, 1); + add_fingerprint_to_dir("Fred", buf, fingerprint_list); + } + + { + char d[DIGEST_LEN]; + const char *m; + /* XXXX NM re-enable. */ + /* Make sure routers aren't too far in the past any more. */ + r1->cache_info.published_on = time(NULL); + r2->cache_info.published_on = time(NULL)-3*60*60; + test_assert(router_dump_router_to_string(buf, 2048, r1, pk2)>0); + test_eq(dirserv_add_descriptor(buf,&m,""), ROUTER_ADDED_NOTIFY_GENERATOR); + test_assert(router_dump_router_to_string(buf, 2048, r2, pk1)>0); + test_eq(dirserv_add_descriptor(buf,&m,""), ROUTER_ADDED_NOTIFY_GENERATOR); + get_options()->Nickname = tor_strdup("DirServer"); + test_assert(!dirserv_dump_directory_to_string(&cp,pk3, 0)); + crypto_pk_get_digest(pk3, d); + test_assert(!router_parse_directory(cp)); + test_eq(2, smartlist_len(dir1->routers)); + tor_free(cp); + } +#endif + dirserv_free_fingerprint_list(); + + done: + if (r1) + routerinfo_free(r1); + if (r2) + routerinfo_free(r2); + + tor_free(pk1_str); + tor_free(pk2_str); + tor_free(pk3_str); + if (pk1) crypto_free_pk_env(pk1); + if (pk2) crypto_free_pk_env(pk2); + if (pk3) crypto_free_pk_env(pk3); + if (rp1) routerinfo_free(rp1); + tor_free(dir1); /* XXXX And more !*/ + tor_free(dir2); /* And more !*/ +} + +static void +test_dir_versions(void) +{ + tor_version_t ver1; + + /* Try out version parsing functionality */ + test_eq(0, tor_version_parse("0.3.4pre2-cvs", &ver1)); + test_eq(0, ver1.major); + test_eq(3, ver1.minor); + test_eq(4, ver1.micro); + test_eq(VER_PRE, ver1.status); + test_eq(2, ver1.patchlevel); + test_eq(0, tor_version_parse("0.3.4rc1", &ver1)); + test_eq(0, ver1.major); + test_eq(3, ver1.minor); + test_eq(4, ver1.micro); + test_eq(VER_RC, ver1.status); + test_eq(1, ver1.patchlevel); + test_eq(0, tor_version_parse("1.3.4", &ver1)); + test_eq(1, ver1.major); + test_eq(3, ver1.minor); + test_eq(4, ver1.micro); + test_eq(VER_RELEASE, ver1.status); + test_eq(0, ver1.patchlevel); + test_eq(0, tor_version_parse("1.3.4.999", &ver1)); + test_eq(1, ver1.major); + test_eq(3, ver1.minor); + test_eq(4, ver1.micro); + test_eq(VER_RELEASE, ver1.status); + test_eq(999, ver1.patchlevel); + test_eq(0, tor_version_parse("0.1.2.4-alpha", &ver1)); + test_eq(0, ver1.major); + test_eq(1, ver1.minor); + test_eq(2, ver1.micro); + test_eq(4, ver1.patchlevel); + test_eq(VER_RELEASE, ver1.status); + test_streq("alpha", ver1.status_tag); + test_eq(0, tor_version_parse("0.1.2.4", &ver1)); + test_eq(0, ver1.major); + test_eq(1, ver1.minor); + test_eq(2, ver1.micro); + test_eq(4, ver1.patchlevel); + test_eq(VER_RELEASE, ver1.status); + test_streq("", ver1.status_tag); + +#define tt_versionstatus_op(vs1, op, vs2) \ + tt_assert_test_type(vs1,vs2,#vs1" "#op" "#vs2,version_status_t, \ + (_val1 op _val2),"%d") +#define test_v_i_o(val, ver, lst) \ + tt_versionstatus_op(val, ==, tor_version_is_obsolete(ver, lst)) + + /* make sure tor_version_is_obsolete() works */ + test_v_i_o(VS_OLD, "0.0.1", "Tor 0.0.2"); + test_v_i_o(VS_OLD, "0.0.1", "0.0.2, Tor 0.0.3"); + test_v_i_o(VS_OLD, "0.0.1", "0.0.2,Tor 0.0.3"); + test_v_i_o(VS_OLD, "0.0.1","0.0.3,BetterTor 0.0.1"); + test_v_i_o(VS_RECOMMENDED, "0.0.2", "Tor 0.0.2,Tor 0.0.3"); + test_v_i_o(VS_NEW_IN_SERIES, "0.0.2", "Tor 0.0.2pre1,Tor 0.0.3"); + test_v_i_o(VS_OLD, "0.0.2", "Tor 0.0.2.1,Tor 0.0.3"); + test_v_i_o(VS_NEW, "0.1.0", "Tor 0.0.2,Tor 0.0.3"); + test_v_i_o(VS_RECOMMENDED, "0.0.7rc2", "0.0.7,Tor 0.0.7rc2,Tor 0.0.8"); + test_v_i_o(VS_OLD, "0.0.5.0", "0.0.5.1-cvs"); + test_v_i_o(VS_NEW_IN_SERIES, "0.0.5.1-cvs", "0.0.5, 0.0.6"); + /* Not on list, but newer than any in same series. */ + test_v_i_o(VS_NEW_IN_SERIES, "0.1.0.3", + "Tor 0.1.0.2,Tor 0.0.9.5,Tor 0.1.1.0"); + /* Series newer than any on list. */ + test_v_i_o(VS_NEW, "0.1.2.3", "Tor 0.1.0.2,Tor 0.0.9.5,Tor 0.1.1.0"); + /* Series older than any on list. */ + test_v_i_o(VS_OLD, "0.0.1.3", "Tor 0.1.0.2,Tor 0.0.9.5,Tor 0.1.1.0"); + /* Not on list, not newer than any on same series. */ + test_v_i_o(VS_UNRECOMMENDED, "0.1.0.1", + "Tor 0.1.0.2,Tor 0.0.9.5,Tor 0.1.1.0"); + /* On list, not newer than any on same series. */ + test_v_i_o(VS_UNRECOMMENDED, + "0.1.0.1", "Tor 0.1.0.2,Tor 0.0.9.5,Tor 0.1.1.0"); + test_eq(0, tor_version_as_new_as("Tor 0.0.5", "0.0.9pre1-cvs")); + test_eq(1, tor_version_as_new_as( + "Tor 0.0.8 on Darwin 64-121-192-100.c3-0." + "sfpo-ubr1.sfrn-sfpo.ca.cable.rcn.com Power Macintosh", + "0.0.8rc2")); + test_eq(0, tor_version_as_new_as( + "Tor 0.0.8 on Darwin 64-121-192-100.c3-0." + "sfpo-ubr1.sfrn-sfpo.ca.cable.rcn.com Power Macintosh", "0.0.8.2")); + + /* Now try svn revisions. */ + test_eq(1, tor_version_as_new_as("Tor 0.2.1.0-dev (r100)", + "Tor 0.2.1.0-dev (r99)")); + test_eq(1, tor_version_as_new_as("Tor 0.2.1.0-dev (r100) on Banana Jr", + "Tor 0.2.1.0-dev (r99) on Hal 9000")); + test_eq(1, tor_version_as_new_as("Tor 0.2.1.0-dev (r100)", + "Tor 0.2.1.0-dev on Colossus")); + test_eq(0, tor_version_as_new_as("Tor 0.2.1.0-dev (r99)", + "Tor 0.2.1.0-dev (r100)")); + test_eq(0, tor_version_as_new_as("Tor 0.2.1.0-dev (r99) on MCP", + "Tor 0.2.1.0-dev (r100) on AM")); + test_eq(0, tor_version_as_new_as("Tor 0.2.1.0-dev", + "Tor 0.2.1.0-dev (r99)")); + test_eq(1, tor_version_as_new_as("Tor 0.2.1.1", + "Tor 0.2.1.0-dev (r99)")); + + /* Now try git revisions */ + test_eq(0, tor_version_parse("0.5.6.7 (git-ff00ff)", &ver1)); + test_eq(0, ver1.major); + test_eq(5, ver1.minor); + test_eq(6, ver1.micro); + test_eq(7, ver1.patchlevel); + test_eq(3, ver1.git_tag_len); + test_memeq(ver1.git_tag, "\xff\x00\xff", 3); + test_eq(-1, tor_version_parse("0.5.6.7 (git-ff00xx)", &ver1)); + test_eq(-1, tor_version_parse("0.5.6.7 (git-ff00fff)", &ver1)); + test_eq(0, tor_version_parse("0.5.6.7 (git ff00fff)", &ver1)); + done: + ; +} + +/** Run unit tests for directory fp_pair functions. */ +static void +test_dir_fp_pairs(void) +{ + smartlist_t *sl = smartlist_create(); + fp_pair_t *pair; + + dir_split_resource_into_fingerprint_pairs( + /* Two pairs, out of order, with one duplicate. */ + "73656372657420646174612E0000000000FFFFFF-" + "557365204145532d32353620696e73746561642e+" + "73656372657420646174612E0000000000FFFFFF-" + "557365204145532d32353620696e73746561642e+" + "48657861646563696d616c2069736e277420736f-" + "676f6f6420666f7220686964696e6720796f7572.z", sl); + + test_eq(smartlist_len(sl), 2); + pair = smartlist_get(sl, 0); + test_memeq(pair->first, "Hexadecimal isn't so", DIGEST_LEN); + test_memeq(pair->second, "good for hiding your", DIGEST_LEN); + pair = smartlist_get(sl, 1); + test_memeq(pair->first, "secret data.\0\0\0\0\0\xff\xff\xff", DIGEST_LEN); + test_memeq(pair->second, "Use AES-256 instead.", DIGEST_LEN); + + done: + SMARTLIST_FOREACH(sl, fp_pair_t *, pair, tor_free(pair)); + smartlist_free(sl); +} + +static void +test_dir_split_fps(void *testdata) +{ + smartlist_t *sl = smartlist_create(); + char *mem_op_hex_tmp = NULL; + (void)testdata; + + /* Some example hex fingerprints and their base64 equivalents */ +#define HEX1 "Fe0daff89127389bc67558691231234551193EEE" +#define HEX2 "Deadbeef99999991111119999911111111f00ba4" +#define HEX3 "b33ff00db33ff00db33ff00db33ff00db33ff00d" +#define HEX256_1 \ + "f3f3f3f3fbbbbf3f3f3f3fbbbf3f3f3f3fbbbbf3f3f3f3fbbbf3f3f3f3fbbbbf" +#define HEX256_2 \ + "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccCCc" +#define HEX256_3 \ + "0123456789ABCdef0123456789ABCdef0123456789ABCdef0123456789ABCdef" +#define B64_1 "/g2v+JEnOJvGdVhpEjEjRVEZPu4" +#define B64_2 "3q2+75mZmZERERmZmRERERHwC6Q" +#define B64_3 "sz/wDbM/8A2zP/ANsz/wDbM/8A0" +#define B64_256_1 "8/Pz8/u7vz8/Pz+7vz8/Pz+7u/Pz8/P7u/Pz8/P7u78" +#define B64_256_2 "zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMw" +#define B64_256_3 "ASNFZ4mrze8BI0VniavN7wEjRWeJq83vASNFZ4mrze8" + + /* no flags set */ + dir_split_resource_into_fingerprints("A+C+B", sl, NULL, 0); + tt_int_op(smartlist_len(sl), ==, 3); + tt_str_op(smartlist_get(sl, 0), ==, "A"); + tt_str_op(smartlist_get(sl, 1), ==, "C"); + tt_str_op(smartlist_get(sl, 2), ==, "B"); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + /* uniq strings. */ + dir_split_resource_into_fingerprints("A+C+B+A+B+B", sl, NULL, DSR_SORT_UNIQ); + tt_int_op(smartlist_len(sl), ==, 3); + tt_str_op(smartlist_get(sl, 0), ==, "A"); + tt_str_op(smartlist_get(sl, 1), ==, "B"); + tt_str_op(smartlist_get(sl, 2), ==, "C"); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + /* Decode hex. */ + dir_split_resource_into_fingerprints(HEX1"+"HEX2, sl, NULL, DSR_HEX); + tt_int_op(smartlist_len(sl), ==, 2); + test_mem_op_hex(smartlist_get(sl, 0), ==, HEX1); + test_mem_op_hex(smartlist_get(sl, 1), ==, HEX2); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + /* decode hex and drop weirdness. */ + dir_split_resource_into_fingerprints(HEX1"+bogus+"HEX2"+"HEX256_1, + sl, NULL, DSR_HEX); + tt_int_op(smartlist_len(sl), ==, 2); + test_mem_op_hex(smartlist_get(sl, 0), ==, HEX1); + test_mem_op_hex(smartlist_get(sl, 1), ==, HEX2); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + /* Decode long hex */ + dir_split_resource_into_fingerprints(HEX256_1"+"HEX256_2"+"HEX2"+"HEX256_3, + sl, NULL, DSR_HEX|DSR_DIGEST256); + tt_int_op(smartlist_len(sl), ==, 3); + test_mem_op_hex(smartlist_get(sl, 0), ==, HEX256_1); + test_mem_op_hex(smartlist_get(sl, 1), ==, HEX256_2); + test_mem_op_hex(smartlist_get(sl, 2), ==, HEX256_3); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + /* Decode hex and sort. */ + dir_split_resource_into_fingerprints(HEX1"+"HEX2"+"HEX3"+"HEX2, + sl, NULL, DSR_HEX|DSR_SORT_UNIQ); + tt_int_op(smartlist_len(sl), ==, 3); + test_mem_op_hex(smartlist_get(sl, 0), ==, HEX3); + test_mem_op_hex(smartlist_get(sl, 1), ==, HEX2); + test_mem_op_hex(smartlist_get(sl, 2), ==, HEX1); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + /* Decode long hex and sort */ + dir_split_resource_into_fingerprints(HEX256_1"+"HEX256_2"+"HEX256_3 + "+"HEX256_1, + sl, NULL, + DSR_HEX|DSR_DIGEST256|DSR_SORT_UNIQ); + tt_int_op(smartlist_len(sl), ==, 3); + test_mem_op_hex(smartlist_get(sl, 0), ==, HEX256_3); + test_mem_op_hex(smartlist_get(sl, 1), ==, HEX256_2); + test_mem_op_hex(smartlist_get(sl, 2), ==, HEX256_1); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + /* Decode base64 */ + dir_split_resource_into_fingerprints(B64_1"-"B64_2, sl, NULL, DSR_BASE64); + tt_int_op(smartlist_len(sl), ==, 2); + test_mem_op_hex(smartlist_get(sl, 0), ==, HEX1); + test_mem_op_hex(smartlist_get(sl, 1), ==, HEX2); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + /* Decode long base64 */ + dir_split_resource_into_fingerprints(B64_256_1"-"B64_256_2, + sl, NULL, DSR_BASE64|DSR_DIGEST256); + tt_int_op(smartlist_len(sl), ==, 2); + test_mem_op_hex(smartlist_get(sl, 0), ==, HEX256_1); + test_mem_op_hex(smartlist_get(sl, 1), ==, HEX256_2); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + dir_split_resource_into_fingerprints(B64_256_1, + sl, NULL, DSR_BASE64|DSR_DIGEST256); + tt_int_op(smartlist_len(sl), ==, 1); + test_mem_op_hex(smartlist_get(sl, 0), ==, HEX256_1); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + done: + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_free(sl); + tor_free(mem_op_hex_tmp); +} + +static void +test_dir_measured_bw(void) +{ + measured_bw_line_t mbwl; + int i; + const char *lines_pass[] = { + "node_id=$557365204145532d32353620696e73746561642e bw=1024\n", + "node_id=$557365204145532d32353620696e73746561642e\t bw=1024 \n", + " node_id=$557365204145532d32353620696e73746561642e bw=1024\n", + "\tnoise\tnode_id=$557365204145532d32353620696e73746561642e " + "bw=1024 junk=007\n", + "misc=junk node_id=$557365204145532d32353620696e73746561642e " + "bw=1024 junk=007\n", + "end" + }; + const char *lines_fail[] = { + /* Test possible python stupidity on input */ + "node_id=None bw=1024\n", + "node_id=$None bw=1024\n", + "node_id=$557365204145532d32353620696e73746561642e bw=None\n", + "node_id=$557365204145532d32353620696e73746561642e bw=1024.0\n", + "node_id=$557365204145532d32353620696e73746561642e bw=.1024\n", + "node_id=$557365204145532d32353620696e73746561642e bw=1.024\n", + "node_id=$557365204145532d32353620696e73746561642e bw=1024 bw=0\n", + "node_id=$557365204145532d32353620696e73746561642e bw=1024 bw=None\n", + "node_id=$557365204145532d32353620696e73746561642e bw=-1024\n", + /* Test incomplete writes due to race conditions, partial copies, etc */ + "node_i", + "node_i\n", + "node_id=", + "node_id=\n", + "node_id=$557365204145532d32353620696e73746561642e bw=", + "node_id=$557365204145532d32353620696e73746561642e bw=1024", + "node_id=$557365204145532d32353620696e73746561642e bw=\n", + "node_id=$557365204145532d32353620696e7374", + "node_id=$557365204145532d32353620696e7374\n", + "", + "\n", + " \n ", + " \n\n", + /* Test assorted noise */ + " node_id= ", + "node_id==$557365204145532d32353620696e73746561642e bw==1024\n", + "node_id=$55736520414552d32353620696e73746561642e bw=1024\n", + "node_id=557365204145532d32353620696e73746561642e bw=1024\n", + "node_id= $557365204145532d32353620696e73746561642e bw=0.23\n", + "end" + }; + + for (i = 0; strcmp(lines_fail[i], "end"); i++) { + //fprintf(stderr, "Testing: %s\n", lines_fail[i]); + test_assert(measured_bw_line_parse(&mbwl, lines_fail[i]) == -1); + } + + for (i = 0; strcmp(lines_pass[i], "end"); i++) { + //fprintf(stderr, "Testing: %s %d\n", lines_pass[i], TOR_ISSPACE('\n')); + test_assert(measured_bw_line_parse(&mbwl, lines_pass[i]) == 0); + test_assert(mbwl.bw == 1024); + test_assert(strcmp(mbwl.node_hex, + "557365204145532d32353620696e73746561642e") == 0); + } + +done: + return; +} + +static void +test_dir_param_voting(void) +{ + networkstatus_t vote1, vote2, vote3, vote4; + smartlist_t *votes = smartlist_create(); + char *res = NULL; + + /* dirvote_compute_params only looks at the net_params field of the votes, + so that's all we need to set. + */ + memset(&vote1, 0, sizeof(vote1)); + memset(&vote2, 0, sizeof(vote2)); + memset(&vote3, 0, sizeof(vote3)); + memset(&vote4, 0, sizeof(vote4)); + vote1.net_params = smartlist_create(); + vote2.net_params = smartlist_create(); + vote3.net_params = smartlist_create(); + vote4.net_params = smartlist_create(); + smartlist_split_string(vote1.net_params, + "ab=90 abcd=20 cw=50 x-yz=-99", NULL, 0, 0); + smartlist_split_string(vote2.net_params, + "ab=27 cw=5 x-yz=88", NULL, 0, 0); + smartlist_split_string(vote3.net_params, + "abcd=20 c=60 cw=500 x-yz=-9 zzzzz=101", NULL, 0, 0); + smartlist_split_string(vote4.net_params, + "ab=900 abcd=200 c=1 cw=51 x-yz=100", NULL, 0, 0); + test_eq(100, networkstatus_get_param(&vote4, "x-yz", 50)); + test_eq(222, networkstatus_get_param(&vote4, "foobar", 222)); + + smartlist_add(votes, &vote1); + smartlist_add(votes, &vote2); + smartlist_add(votes, &vote3); + smartlist_add(votes, &vote4); + + res = dirvote_compute_params(votes); + test_streq(res, + "ab=90 abcd=20 c=1 cw=50 x-yz=-9 zzzzz=101"); + + done: + tor_free(res); + SMARTLIST_FOREACH(vote1.net_params, char *, cp, tor_free(cp)); + SMARTLIST_FOREACH(vote2.net_params, char *, cp, tor_free(cp)); + SMARTLIST_FOREACH(vote3.net_params, char *, cp, tor_free(cp)); + SMARTLIST_FOREACH(vote4.net_params, char *, cp, tor_free(cp)); + smartlist_free(vote1.net_params); + smartlist_free(vote2.net_params); + smartlist_free(vote3.net_params); + smartlist_free(vote4.net_params); + smartlist_free(votes); + + return; +} + +extern const char AUTHORITY_CERT_1[]; +extern const char AUTHORITY_SIGNKEY_1[]; +extern const char AUTHORITY_CERT_2[]; +extern const char AUTHORITY_SIGNKEY_2[]; +extern const char AUTHORITY_CERT_3[]; +extern const char AUTHORITY_SIGNKEY_3[]; + +/** Helper: Test that two networkstatus_voter_info_t do in fact represent the + * same voting authority, and that they do in fact have all the same + * information. */ +static void +test_same_voter(networkstatus_voter_info_t *v1, + networkstatus_voter_info_t *v2) +{ + test_streq(v1->nickname, v2->nickname); + test_memeq(v1->identity_digest, v2->identity_digest, DIGEST_LEN); + test_streq(v1->address, v2->address); + test_eq(v1->addr, v2->addr); + test_eq(v1->dir_port, v2->dir_port); + test_eq(v1->or_port, v2->or_port); + test_streq(v1->contact, v2->contact); + test_memeq(v1->vote_digest, v2->vote_digest, DIGEST_LEN); + done: + ; +} + +/** Helper: Make a new routerinfo containing the right information for a + * given vote_routerstatus_t. */ +static routerinfo_t * +generate_ri_from_rs(const vote_routerstatus_t *vrs) +{ + routerinfo_t *r; + const routerstatus_t *rs = &vrs->status; + static time_t published = 0; + + r = tor_malloc_zero(sizeof(routerinfo_t)); + memcpy(r->cache_info.identity_digest, rs->identity_digest, DIGEST_LEN); + memcpy(r->cache_info.signed_descriptor_digest, rs->descriptor_digest, + DIGEST_LEN); + r->cache_info.do_not_cache = 1; + r->cache_info.routerlist_index = -1; + r->cache_info.signed_descriptor_body = + tor_strdup("123456789012345678901234567890123"); + r->cache_info.signed_descriptor_len = + strlen(r->cache_info.signed_descriptor_body); + r->exit_policy = smartlist_create(); + r->cache_info.published_on = ++published + time(NULL); + return r; +} + +/** Helper: get a detached signatures document for one or two + * consensuses. */ +static char * +get_detached_sigs(networkstatus_t *ns, networkstatus_t *ns2) +{ + char *r; + smartlist_t *sl; + tor_assert(ns && ns->flavor == FLAV_NS); + sl = smartlist_create(); + smartlist_add(sl,ns); + if (ns2) + smartlist_add(sl,ns2); + r = networkstatus_get_detached_signatures(sl); + smartlist_free(sl); + return r; +} + +/** Run unit tests for generating and parsing V3 consensus networkstatus + * documents. */ +static void +test_dir_v3_networkstatus(void) +{ + authority_cert_t *cert1=NULL, *cert2=NULL, *cert3=NULL; + crypto_pk_env_t *sign_skey_1=NULL, *sign_skey_2=NULL, *sign_skey_3=NULL; + crypto_pk_env_t *sign_skey_leg1=NULL; + const char *msg=NULL; + + time_t now = time(NULL); + networkstatus_voter_info_t *voter; + document_signature_t *sig; + networkstatus_t *vote=NULL, *v1=NULL, *v2=NULL, *v3=NULL, *con=NULL, + *con_md=NULL; + vote_routerstatus_t *vrs; + routerstatus_t *rs; + char *v1_text=NULL, *v2_text=NULL, *v3_text=NULL, *consensus_text=NULL, *cp; + smartlist_t *votes = smartlist_create(); + + /* For generating the two other consensuses. */ + char *detached_text1=NULL, *detached_text2=NULL; + char *consensus_text2=NULL, *consensus_text3=NULL; + char *consensus_text_md2=NULL, *consensus_text_md3=NULL; + char *consensus_text_md=NULL; + networkstatus_t *con2=NULL, *con_md2=NULL, *con3=NULL, *con_md3=NULL; + ns_detached_signatures_t *dsig1=NULL, *dsig2=NULL; + + /* Parse certificates and keys. */ + cert1 = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL); + test_assert(cert1); + test_assert(cert1->is_cross_certified); + cert2 = authority_cert_parse_from_string(AUTHORITY_CERT_2, NULL); + test_assert(cert2); + cert3 = authority_cert_parse_from_string(AUTHORITY_CERT_3, NULL); + test_assert(cert3); + sign_skey_1 = crypto_new_pk_env(); + sign_skey_2 = crypto_new_pk_env(); + sign_skey_3 = crypto_new_pk_env(); + sign_skey_leg1 = pk_generate(4); + + test_assert(!crypto_pk_read_private_key_from_string(sign_skey_1, + AUTHORITY_SIGNKEY_1)); + test_assert(!crypto_pk_read_private_key_from_string(sign_skey_2, + AUTHORITY_SIGNKEY_2)); + test_assert(!crypto_pk_read_private_key_from_string(sign_skey_3, + AUTHORITY_SIGNKEY_3)); + + test_assert(!crypto_pk_cmp_keys(sign_skey_1, cert1->signing_key)); + test_assert(!crypto_pk_cmp_keys(sign_skey_2, cert2->signing_key)); + + /* + * Set up a vote; generate it; try to parse it. + */ + vote = tor_malloc_zero(sizeof(networkstatus_t)); + vote->type = NS_TYPE_VOTE; + vote->published = now; + vote->valid_after = now+1000; + vote->fresh_until = now+2000; + vote->valid_until = now+3000; + vote->vote_seconds = 100; + vote->dist_seconds = 200; + vote->supported_methods = smartlist_create(); + smartlist_split_string(vote->supported_methods, "1 2 3", NULL, 0, -1); + vote->client_versions = tor_strdup("0.1.2.14,0.1.2.15"); + vote->server_versions = tor_strdup("0.1.2.14,0.1.2.15,0.1.2.16"); + vote->known_flags = smartlist_create(); + smartlist_split_string(vote->known_flags, + "Authority Exit Fast Guard Running Stable V2Dir Valid", + 0, SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); + vote->voters = smartlist_create(); + voter = tor_malloc_zero(sizeof(networkstatus_voter_info_t)); + voter->nickname = tor_strdup("Voter1"); + voter->address = tor_strdup("1.2.3.4"); + voter->addr = 0x01020304; + voter->dir_port = 80; + voter->or_port = 9000; + voter->contact = tor_strdup("voter@example.com"); + crypto_pk_get_digest(cert1->identity_key, voter->identity_digest); + smartlist_add(vote->voters, voter); + vote->cert = authority_cert_dup(cert1); + vote->net_params = smartlist_create(); + smartlist_split_string(vote->net_params, "circuitwindow=101 foo=990", + NULL, 0, 0); + vote->routerstatus_list = smartlist_create(); + /* add the first routerstatus. */ + vrs = tor_malloc_zero(sizeof(vote_routerstatus_t)); + rs = &vrs->status; + vrs->version = tor_strdup("0.1.2.14"); + rs->published_on = now-1500; + strlcpy(rs->nickname, "router2", sizeof(rs->nickname)); + memset(rs->identity_digest, 3, DIGEST_LEN); + memset(rs->descriptor_digest, 78, DIGEST_LEN); + rs->addr = 0x99008801; + rs->or_port = 443; + rs->dir_port = 8000; + /* all flags but running cleared */ + rs->is_running = 1; + smartlist_add(vote->routerstatus_list, vrs); + test_assert(router_add_to_routerlist(generate_ri_from_rs(vrs), &msg,0,0)>=0); + + /* add the second routerstatus. */ + vrs = tor_malloc_zero(sizeof(vote_routerstatus_t)); + rs = &vrs->status; + vrs->version = tor_strdup("0.2.0.5"); + rs->published_on = now-1000; + strlcpy(rs->nickname, "router1", sizeof(rs->nickname)); + memset(rs->identity_digest, 5, DIGEST_LEN); + memset(rs->descriptor_digest, 77, DIGEST_LEN); + rs->addr = 0x99009901; + rs->or_port = 443; + rs->dir_port = 0; + rs->is_exit = rs->is_stable = rs->is_fast = rs->is_running = + rs->is_valid = rs->is_v2_dir = rs->is_possible_guard = 1; + smartlist_add(vote->routerstatus_list, vrs); + test_assert(router_add_to_routerlist(generate_ri_from_rs(vrs), &msg,0,0)>=0); + + /* add the third routerstatus. */ + vrs = tor_malloc_zero(sizeof(vote_routerstatus_t)); + rs = &vrs->status; + vrs->version = tor_strdup("0.1.0.3"); + rs->published_on = now-1000; + strlcpy(rs->nickname, "router3", sizeof(rs->nickname)); + memset(rs->identity_digest, 33, DIGEST_LEN); + memset(rs->descriptor_digest, 79, DIGEST_LEN); + rs->addr = 0xAA009901; + rs->or_port = 400; + rs->dir_port = 9999; + rs->is_authority = rs->is_exit = rs->is_stable = rs->is_fast = + rs->is_running = rs->is_valid = rs->is_v2_dir = rs->is_possible_guard = 1; + smartlist_add(vote->routerstatus_list, vrs); + test_assert(router_add_to_routerlist(generate_ri_from_rs(vrs), &msg,0,0)>=0); + + /* add a fourth routerstatus that is not running. */ + vrs = tor_malloc_zero(sizeof(vote_routerstatus_t)); + rs = &vrs->status; + vrs->version = tor_strdup("0.1.6.3"); + rs->published_on = now-1000; + strlcpy(rs->nickname, "router4", sizeof(rs->nickname)); + memset(rs->identity_digest, 34, DIGEST_LEN); + memset(rs->descriptor_digest, 47, DIGEST_LEN); + rs->addr = 0xC0000203; + rs->or_port = 500; + rs->dir_port = 1999; + /* Running flag (and others) cleared */ + smartlist_add(vote->routerstatus_list, vrs); + test_assert(router_add_to_routerlist(generate_ri_from_rs(vrs), &msg,0,0)>=0); + + /* dump the vote and try to parse it. */ + v1_text = format_networkstatus_vote(sign_skey_1, vote); + test_assert(v1_text); + v1 = networkstatus_parse_vote_from_string(v1_text, NULL, NS_TYPE_VOTE); + test_assert(v1); + + /* Make sure the parsed thing was right. */ + test_eq(v1->type, NS_TYPE_VOTE); + test_eq(v1->published, vote->published); + test_eq(v1->valid_after, vote->valid_after); + test_eq(v1->fresh_until, vote->fresh_until); + test_eq(v1->valid_until, vote->valid_until); + test_eq(v1->vote_seconds, vote->vote_seconds); + test_eq(v1->dist_seconds, vote->dist_seconds); + test_streq(v1->client_versions, vote->client_versions); + test_streq(v1->server_versions, vote->server_versions); + test_assert(v1->voters && smartlist_len(v1->voters)); + voter = smartlist_get(v1->voters, 0); + test_streq(voter->nickname, "Voter1"); + test_streq(voter->address, "1.2.3.4"); + test_eq(voter->addr, 0x01020304); + test_eq(voter->dir_port, 80); + test_eq(voter->or_port, 9000); + test_streq(voter->contact, "voter@example.com"); + test_assert(v1->cert); + test_assert(!crypto_pk_cmp_keys(sign_skey_1, v1->cert->signing_key)); + cp = smartlist_join_strings(v1->known_flags, ":", 0, NULL); + test_streq(cp, "Authority:Exit:Fast:Guard:Running:Stable:V2Dir:Valid"); + tor_free(cp); + test_eq(smartlist_len(v1->routerstatus_list), 4); + /* Check the first routerstatus. */ + vrs = smartlist_get(v1->routerstatus_list, 0); + rs = &vrs->status; + test_streq(vrs->version, "0.1.2.14"); + test_eq(rs->published_on, now-1500); + test_streq(rs->nickname, "router2"); + test_memeq(rs->identity_digest, + "\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3", + DIGEST_LEN); + test_memeq(rs->descriptor_digest, "NNNNNNNNNNNNNNNNNNNN", DIGEST_LEN); + test_eq(rs->addr, 0x99008801); + test_eq(rs->or_port, 443); + test_eq(rs->dir_port, 8000); + test_eq(vrs->flags, U64_LITERAL(16)); // no flags except "running" + /* Check the second routerstatus. */ + vrs = smartlist_get(v1->routerstatus_list, 1); + rs = &vrs->status; + test_streq(vrs->version, "0.2.0.5"); + test_eq(rs->published_on, now-1000); + test_streq(rs->nickname, "router1"); + test_memeq(rs->identity_digest, + "\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5", + DIGEST_LEN); + test_memeq(rs->descriptor_digest, "MMMMMMMMMMMMMMMMMMMM", DIGEST_LEN); + test_eq(rs->addr, 0x99009901); + test_eq(rs->or_port, 443); + test_eq(rs->dir_port, 0); + test_eq(vrs->flags, U64_LITERAL(254)); // all flags except "authority." + + { + measured_bw_line_t mbw; + memset(mbw.node_id, 33, sizeof(mbw.node_id)); + mbw.bw = 1024; + test_assert(measured_bw_line_apply(&mbw, + v1->routerstatus_list) == 1); + vrs = smartlist_get(v1->routerstatus_list, 2); + test_assert(vrs->status.has_measured_bw && + vrs->status.measured_bw == 1024); + } + + /* Generate second vote. It disagrees on some of the times, + * and doesn't list versions, and knows some crazy flags */ + vote->published = now+1; + vote->fresh_until = now+3005; + vote->dist_seconds = 300; + authority_cert_free(vote->cert); + vote->cert = authority_cert_dup(cert2); + vote->net_params = smartlist_create(); + smartlist_split_string(vote->net_params, "bar=2000000000 circuitwindow=20", + NULL, 0, 0); + tor_free(vote->client_versions); + tor_free(vote->server_versions); + voter = smartlist_get(vote->voters, 0); + tor_free(voter->nickname); + tor_free(voter->address); + voter->nickname = tor_strdup("Voter2"); + voter->address = tor_strdup("2.3.4.5"); + voter->addr = 0x02030405; + crypto_pk_get_digest(cert2->identity_key, voter->identity_digest); + smartlist_add(vote->known_flags, tor_strdup("MadeOfCheese")); + smartlist_add(vote->known_flags, tor_strdup("MadeOfTin")); + smartlist_sort_strings(vote->known_flags); + vrs = smartlist_get(vote->routerstatus_list, 2); + smartlist_del_keeporder(vote->routerstatus_list, 2); + tor_free(vrs->version); + tor_free(vrs); + vrs = smartlist_get(vote->routerstatus_list, 0); + vrs->status.is_fast = 1; + /* generate and parse. */ + v2_text = format_networkstatus_vote(sign_skey_2, vote); + test_assert(v2_text); + v2 = networkstatus_parse_vote_from_string(v2_text, NULL, NS_TYPE_VOTE); + test_assert(v2); + /* Check that flags come out right.*/ + cp = smartlist_join_strings(v2->known_flags, ":", 0, NULL); + test_streq(cp, "Authority:Exit:Fast:Guard:MadeOfCheese:MadeOfTin:" + "Running:Stable:V2Dir:Valid"); + tor_free(cp); + vrs = smartlist_get(v2->routerstatus_list, 1); + /* 1023 - authority(1) - madeofcheese(16) - madeoftin(32) */ + test_eq(vrs->flags, U64_LITERAL(974)); + + /* Generate the third vote. */ + vote->published = now; + vote->fresh_until = now+2003; + vote->dist_seconds = 250; + authority_cert_free(vote->cert); + vote->cert = authority_cert_dup(cert3); + vote->net_params = smartlist_create(); + smartlist_split_string(vote->net_params, "circuitwindow=80 foo=660", + NULL, 0, 0); + smartlist_add(vote->supported_methods, tor_strdup("4")); + vote->client_versions = tor_strdup("0.1.2.14,0.1.2.17"); + vote->server_versions = tor_strdup("0.1.2.10,0.1.2.15,0.1.2.16"); + voter = smartlist_get(vote->voters, 0); + tor_free(voter->nickname); + tor_free(voter->address); + voter->nickname = tor_strdup("Voter3"); + voter->address = tor_strdup("3.4.5.6"); + voter->addr = 0x03040506; + crypto_pk_get_digest(cert3->identity_key, voter->identity_digest); + /* This one has a legacy id. */ + memset(voter->legacy_id_digest, (int)'A', DIGEST_LEN); + vrs = smartlist_get(vote->routerstatus_list, 0); + smartlist_del_keeporder(vote->routerstatus_list, 0); + tor_free(vrs->version); + tor_free(vrs); + vrs = smartlist_get(vote->routerstatus_list, 0); + memset(vrs->status.descriptor_digest, (int)'Z', DIGEST_LEN); + test_assert(router_add_to_routerlist(generate_ri_from_rs(vrs), &msg,0,0)>=0); + + v3_text = format_networkstatus_vote(sign_skey_3, vote); + test_assert(v3_text); + + v3 = networkstatus_parse_vote_from_string(v3_text, NULL, NS_TYPE_VOTE); + test_assert(v3); + + /* Compute a consensus as voter 3. */ + smartlist_add(votes, v3); + smartlist_add(votes, v1); + smartlist_add(votes, v2); + consensus_text = networkstatus_compute_consensus(votes, 3, + cert3->identity_key, + sign_skey_3, + "AAAAAAAAAAAAAAAAAAAA", + sign_skey_leg1, + FLAV_NS); + test_assert(consensus_text); + con = networkstatus_parse_vote_from_string(consensus_text, NULL, + NS_TYPE_CONSENSUS); + test_assert(con); + //log_notice(LD_GENERAL, "<<%s>>\n<<%s>>\n<<%s>>\n", + // v1_text, v2_text, v3_text); + consensus_text_md = networkstatus_compute_consensus(votes, 3, + cert3->identity_key, + sign_skey_3, + "AAAAAAAAAAAAAAAAAAAA", + sign_skey_leg1, + FLAV_MICRODESC); + test_assert(consensus_text_md); + con_md = networkstatus_parse_vote_from_string(consensus_text_md, NULL, + NS_TYPE_CONSENSUS); + test_assert(con_md); + test_eq(con_md->flavor, FLAV_MICRODESC); + + /* Check consensus contents. */ + test_assert(con->type == NS_TYPE_CONSENSUS); + test_eq(con->published, 0); /* this field only appears in votes. */ + test_eq(con->valid_after, now+1000); + test_eq(con->fresh_until, now+2003); /* median */ + test_eq(con->valid_until, now+3000); + test_eq(con->vote_seconds, 100); + test_eq(con->dist_seconds, 250); /* median */ + test_streq(con->client_versions, "0.1.2.14"); + test_streq(con->server_versions, "0.1.2.15,0.1.2.16"); + cp = smartlist_join_strings(v2->known_flags, ":", 0, NULL); + test_streq(cp, "Authority:Exit:Fast:Guard:MadeOfCheese:MadeOfTin:" + "Running:Stable:V2Dir:Valid"); + tor_free(cp); + cp = smartlist_join_strings(con->net_params, ":", 0, NULL); + test_streq(cp, "bar=2000000000:circuitwindow=80:foo=660"); + tor_free(cp); + + test_eq(4, smartlist_len(con->voters)); /*3 voters, 1 legacy key.*/ + /* The voter id digests should be in this order. */ + test_assert(memcmp(cert2->cache_info.identity_digest, + cert1->cache_info.identity_digest,DIGEST_LEN)<0); + test_assert(memcmp(cert1->cache_info.identity_digest, + cert3->cache_info.identity_digest,DIGEST_LEN)<0); + test_same_voter(smartlist_get(con->voters, 1), + smartlist_get(v2->voters, 0)); + test_same_voter(smartlist_get(con->voters, 2), + smartlist_get(v1->voters, 0)); + test_same_voter(smartlist_get(con->voters, 3), + smartlist_get(v3->voters, 0)); + + test_assert(!con->cert); + test_eq(2, smartlist_len(con->routerstatus_list)); + /* There should be two listed routers: one with identity 3, one with + * identity 5. */ + /* This one showed up in 2 digests. */ + rs = smartlist_get(con->routerstatus_list, 0); + test_memeq(rs->identity_digest, + "\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3", + DIGEST_LEN); + test_memeq(rs->descriptor_digest, "NNNNNNNNNNNNNNNNNNNN", DIGEST_LEN); + test_assert(!rs->is_authority); + test_assert(!rs->is_exit); + test_assert(!rs->is_fast); + test_assert(!rs->is_possible_guard); + test_assert(!rs->is_stable); + test_assert(rs->is_running); /* If it wasn't running it wouldn't be here */ + test_assert(!rs->is_v2_dir); + test_assert(!rs->is_valid); + test_assert(!rs->is_named); + /* XXXX check version */ + + rs = smartlist_get(con->routerstatus_list, 1); + /* This one showed up in 3 digests. Twice with ID 'M', once with 'Z'. */ + test_memeq(rs->identity_digest, + "\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5\x5", + DIGEST_LEN); + test_streq(rs->nickname, "router1"); + test_memeq(rs->descriptor_digest, "MMMMMMMMMMMMMMMMMMMM", DIGEST_LEN); + test_eq(rs->published_on, now-1000); + test_eq(rs->addr, 0x99009901); + test_eq(rs->or_port, 443); + test_eq(rs->dir_port, 0); + test_assert(!rs->is_authority); + test_assert(rs->is_exit); + test_assert(rs->is_fast); + test_assert(rs->is_possible_guard); + test_assert(rs->is_stable); + test_assert(rs->is_running); + test_assert(rs->is_v2_dir); + test_assert(rs->is_valid); + test_assert(!rs->is_named); + /* XXXX check version */ + + /* Check signatures. the first voter is a pseudo-entry with a legacy key. + * The second one hasn't signed. The fourth one has signed: validate it. */ + voter = smartlist_get(con->voters, 1); + test_eq(smartlist_len(voter->sigs), 0); + + voter = smartlist_get(con->voters, 3); + test_eq(smartlist_len(voter->sigs), 1); + sig = smartlist_get(voter->sigs, 0); + test_assert(sig->signature); + test_assert(!sig->good_signature); + test_assert(!sig->bad_signature); + + test_assert(!networkstatus_check_document_signature(con, sig, cert3)); + test_assert(sig->signature); + test_assert(sig->good_signature); + test_assert(!sig->bad_signature); + + { + const char *msg=NULL; + /* Compute the other two signed consensuses. */ + smartlist_shuffle(votes); + consensus_text2 = networkstatus_compute_consensus(votes, 3, + cert2->identity_key, + sign_skey_2, NULL,NULL, + FLAV_NS); + consensus_text_md2 = networkstatus_compute_consensus(votes, 3, + cert2->identity_key, + sign_skey_2, NULL,NULL, + FLAV_MICRODESC); + smartlist_shuffle(votes); + consensus_text3 = networkstatus_compute_consensus(votes, 3, + cert1->identity_key, + sign_skey_1, NULL,NULL, + FLAV_NS); + consensus_text_md3 = networkstatus_compute_consensus(votes, 3, + cert1->identity_key, + sign_skey_1, NULL,NULL, + FLAV_MICRODESC); + test_assert(consensus_text2); + test_assert(consensus_text3); + test_assert(consensus_text_md2); + test_assert(consensus_text_md3); + con2 = networkstatus_parse_vote_from_string(consensus_text2, NULL, + NS_TYPE_CONSENSUS); + con3 = networkstatus_parse_vote_from_string(consensus_text3, NULL, + NS_TYPE_CONSENSUS); + con_md2 = networkstatus_parse_vote_from_string(consensus_text_md2, NULL, + NS_TYPE_CONSENSUS); + con_md3 = networkstatus_parse_vote_from_string(consensus_text_md3, NULL, + NS_TYPE_CONSENSUS); + test_assert(con2); + test_assert(con3); + test_assert(con_md2); + test_assert(con_md3); + + /* All three should have the same digest. */ + test_memeq(&con->digests, &con2->digests, sizeof(digests_t)); + test_memeq(&con->digests, &con3->digests, sizeof(digests_t)); + + test_memeq(&con_md->digests, &con_md2->digests, sizeof(digests_t)); + test_memeq(&con_md->digests, &con_md3->digests, sizeof(digests_t)); + + /* Extract a detached signature from con3. */ + detached_text1 = get_detached_sigs(con3, con_md3); + tor_assert(detached_text1); + /* Try to parse it. */ + dsig1 = networkstatus_parse_detached_signatures(detached_text1, NULL); + tor_assert(dsig1); + + /* Are parsed values as expected? */ + test_eq(dsig1->valid_after, con3->valid_after); + test_eq(dsig1->fresh_until, con3->fresh_until); + test_eq(dsig1->valid_until, con3->valid_until); + { + digests_t *dsig_digests = strmap_get(dsig1->digests, "ns"); + test_assert(dsig_digests); + test_memeq(dsig_digests->d[DIGEST_SHA1], con3->digests.d[DIGEST_SHA1], + DIGEST_LEN); + dsig_digests = strmap_get(dsig1->digests, "microdesc"); + test_assert(dsig_digests); + test_memeq(dsig_digests->d[DIGEST_SHA256], + con_md3->digests.d[DIGEST_SHA256], + DIGEST256_LEN); + } + { + smartlist_t *dsig_signatures = strmap_get(dsig1->signatures, "ns"); + test_assert(dsig_signatures); + test_eq(1, smartlist_len(dsig_signatures)); + sig = smartlist_get(dsig_signatures, 0); + test_memeq(sig->identity_digest, cert1->cache_info.identity_digest, + DIGEST_LEN); + test_eq(sig->alg, DIGEST_SHA1); + + dsig_signatures = strmap_get(dsig1->signatures, "microdesc"); + test_assert(dsig_signatures); + test_eq(1, smartlist_len(dsig_signatures)); + sig = smartlist_get(dsig_signatures, 0); + test_memeq(sig->identity_digest, cert1->cache_info.identity_digest, + DIGEST_LEN); + test_eq(sig->alg, DIGEST_SHA256); + } + + /* Try adding it to con2. */ + detached_text2 = get_detached_sigs(con2,con_md2); + test_eq(1, networkstatus_add_detached_signatures(con2, dsig1, &msg)); + tor_free(detached_text2); + test_eq(1, networkstatus_add_detached_signatures(con_md2, dsig1, &msg)); + tor_free(detached_text2); + detached_text2 = get_detached_sigs(con2,con_md2); + //printf("\n<%s>\n", detached_text2); + dsig2 = networkstatus_parse_detached_signatures(detached_text2, NULL); + test_assert(dsig2); + /* + printf("\n"); + SMARTLIST_FOREACH(dsig2->signatures, networkstatus_voter_info_t *, vi, { + char hd[64]; + base16_encode(hd, sizeof(hd), vi->identity_digest, DIGEST_LEN); + printf("%s\n", hd); + }); + */ + test_eq(2, + smartlist_len((smartlist_t*)strmap_get(dsig2->signatures, "ns"))); + test_eq(2, + smartlist_len((smartlist_t*)strmap_get(dsig2->signatures, + "microdesc"))); + + /* Try adding to con2 twice; verify that nothing changes. */ + test_eq(0, networkstatus_add_detached_signatures(con2, dsig1, &msg)); + + /* Add to con. */ + test_eq(2, networkstatus_add_detached_signatures(con, dsig2, &msg)); + /* Check signatures */ + voter = smartlist_get(con->voters, 1); + sig = smartlist_get(voter->sigs, 0); + test_assert(sig); + test_assert(!networkstatus_check_document_signature(con, sig, cert2)); + voter = smartlist_get(con->voters, 2); + sig = smartlist_get(voter->sigs, 0); + test_assert(sig); + test_assert(!networkstatus_check_document_signature(con, sig, cert1)); + } + + done: + smartlist_free(votes); + tor_free(v1_text); + tor_free(v2_text); + tor_free(v3_text); + tor_free(consensus_text); + tor_free(consensus_text_md); + + if (vote) + networkstatus_vote_free(vote); + if (v1) + networkstatus_vote_free(v1); + if (v2) + networkstatus_vote_free(v2); + if (v3) + networkstatus_vote_free(v3); + if (con) + networkstatus_vote_free(con); + if (con_md) + networkstatus_vote_free(con_md); + if (sign_skey_1) + crypto_free_pk_env(sign_skey_1); + if (sign_skey_2) + crypto_free_pk_env(sign_skey_2); + if (sign_skey_3) + crypto_free_pk_env(sign_skey_3); + if (sign_skey_leg1) + crypto_free_pk_env(sign_skey_leg1); + if (cert1) + authority_cert_free(cert1); + if (cert2) + authority_cert_free(cert2); + if (cert3) + authority_cert_free(cert3); + + tor_free(consensus_text2); + tor_free(consensus_text3); + tor_free(consensus_text_md2); + tor_free(consensus_text_md3); + tor_free(detached_text1); + tor_free(detached_text2); + if (con2) + networkstatus_vote_free(con2); + if (con3) + networkstatus_vote_free(con3); + if (con_md2) + networkstatus_vote_free(con_md2); + if (con_md3) + networkstatus_vote_free(con_md3); + if (dsig1) + ns_detached_signatures_free(dsig1); + if (dsig2) + ns_detached_signatures_free(dsig2); +} + +#define DIR_LEGACY(name) \ + { #name, legacy_test_helper, 0, &legacy_setup, test_dir_ ## name } + +#define DIR(name) \ + { #name, test_dir_##name, 0, NULL, NULL } + +struct testcase_t dir_tests[] = { + DIR_LEGACY(nicknames), + DIR_LEGACY(formats), + DIR_LEGACY(versions), + DIR_LEGACY(fp_pairs), + DIR(split_fps), + DIR_LEGACY(measured_bw), + DIR_LEGACY(param_voting), + DIR_LEGACY(v3_networkstatus), + END_OF_TESTCASES +}; + diff --git a/src/test/test_util.c b/src/test/test_util.c new file mode 100644 index 0000000000..22f2707d75 --- /dev/null +++ b/src/test/test_util.c @@ -0,0 +1,1133 @@ +/* Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2010, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "orconfig.h" +#define CONTROL_PRIVATE +#define MEMPOOL_PRIVATE +#include "or.h" +#include "config.h" +#include "control.h" +#include "test.h" +#include "mempool.h" +#include "memarea.h" + +static void +test_util_time(void) +{ + struct timeval start, end; + struct tm a_time; + char timestr[RFC1123_TIME_LEN+1]; + time_t t_res; + int i; + + start.tv_sec = 5; + start.tv_usec = 5000; + + end.tv_sec = 5; + end.tv_usec = 5000; + + test_eq(0L, tv_udiff(&start, &end)); + + end.tv_usec = 7000; + + test_eq(2000L, tv_udiff(&start, &end)); + + end.tv_sec = 6; + + test_eq(1002000L, tv_udiff(&start, &end)); + + end.tv_usec = 0; + + test_eq(995000L, tv_udiff(&start, &end)); + + end.tv_sec = 4; + + test_eq(-1005000L, tv_udiff(&start, &end)); + + end.tv_usec = 999990; + start.tv_sec = 1; + start.tv_usec = 500; + + /* The test values here are confirmed to be correct on a platform + * with a working timegm. */ + a_time.tm_year = 2003-1900; + a_time.tm_mon = 7; + a_time.tm_mday = 30; + a_time.tm_hour = 6; + a_time.tm_min = 14; + a_time.tm_sec = 55; + test_eq((time_t) 1062224095UL, tor_timegm(&a_time)); + a_time.tm_year = 2004-1900; /* Try a leap year, after feb. */ + test_eq((time_t) 1093846495UL, tor_timegm(&a_time)); + a_time.tm_mon = 1; /* Try a leap year, in feb. */ + a_time.tm_mday = 10; + test_eq((time_t) 1076393695UL, tor_timegm(&a_time)); + + format_rfc1123_time(timestr, 0); + test_streq("Thu, 01 Jan 1970 00:00:00 GMT", timestr); + format_rfc1123_time(timestr, (time_t)1091580502UL); + test_streq("Wed, 04 Aug 2004 00:48:22 GMT", timestr); + + t_res = 0; + i = parse_rfc1123_time(timestr, &t_res); + test_eq(i,0); + test_eq(t_res, (time_t)1091580502UL); + test_eq(-1, parse_rfc1123_time("Wed, zz Aug 2004 99-99x99 GMT", &t_res)); + + tor_gettimeofday(&start); + /* now make sure time works. */ + tor_gettimeofday(&end); + /* We might've timewarped a little. */ + tt_int_op(tv_udiff(&start, &end), >=, -5000); + + done: + ; +} + +static void +test_util_config_line(void) +{ + char buf[1024]; + char *k=NULL, *v=NULL; + const char *str; + + /* Test parse_config_line_from_str */ + strlcpy(buf, "k v\n" " key value with spaces \n" "keykey val\n" + "k2\n" + "k3 \n" "\n" " \n" "#comment\n" + "k4#a\n" "k5#abc\n" "k6 val #with comment\n" + "kseven \"a quoted 'string\"\n" + "k8 \"a \\x71uoted\\n\\\"str\\\\ing\\t\\001\\01\\1\\\"\"\n" + , sizeof(buf)); + str = buf; + + str = parse_config_line_from_str(str, &k, &v); + test_streq(k, "k"); + test_streq(v, "v"); + tor_free(k); tor_free(v); + test_assert(!strcmpstart(str, "key value with")); + + str = parse_config_line_from_str(str, &k, &v); + test_streq(k, "key"); + test_streq(v, "value with spaces"); + tor_free(k); tor_free(v); + test_assert(!strcmpstart(str, "keykey")); + + str = parse_config_line_from_str(str, &k, &v); + test_streq(k, "keykey"); + test_streq(v, "val"); + tor_free(k); tor_free(v); + test_assert(!strcmpstart(str, "k2\n")); + + str = parse_config_line_from_str(str, &k, &v); + test_streq(k, "k2"); + test_streq(v, ""); + tor_free(k); tor_free(v); + test_assert(!strcmpstart(str, "k3 \n")); + + str = parse_config_line_from_str(str, &k, &v); + test_streq(k, "k3"); + test_streq(v, ""); + tor_free(k); tor_free(v); + test_assert(!strcmpstart(str, "#comment")); + + str = parse_config_line_from_str(str, &k, &v); + test_streq(k, "k4"); + test_streq(v, ""); + tor_free(k); tor_free(v); + test_assert(!strcmpstart(str, "k5#abc")); + + str = parse_config_line_from_str(str, &k, &v); + test_streq(k, "k5"); + test_streq(v, ""); + tor_free(k); tor_free(v); + test_assert(!strcmpstart(str, "k6")); + + str = parse_config_line_from_str(str, &k, &v); + test_streq(k, "k6"); + test_streq(v, "val"); + tor_free(k); tor_free(v); + test_assert(!strcmpstart(str, "kseven")); + + str = parse_config_line_from_str(str, &k, &v); + test_streq(k, "kseven"); + test_streq(v, "a quoted \'string"); + tor_free(k); tor_free(v); + test_assert(!strcmpstart(str, "k8 ")); + + str = parse_config_line_from_str(str, &k, &v); + test_streq(k, "k8"); + test_streq(v, "a quoted\n\"str\\ing\t\x01\x01\x01\""); + tor_free(k); tor_free(v); + test_streq(str, ""); + done: + tor_free(k); + tor_free(v); +} + +/** Test basic string functionality. */ +static void +test_util_strmisc(void) +{ + char buf[1024]; + int i; + char *cp; + + /* Tests for corner cases of strl operations */ + test_eq(5, strlcpy(buf, "Hello", 0)); + strlcpy(buf, "Hello", sizeof(buf)); + test_eq(10, strlcat(buf, "Hello", 5)); + + /* Test tor_strstrip() */ + strlcpy(buf, "Testing 1 2 3", sizeof(buf)); + tor_strstrip(buf, ",!"); + test_streq(buf, "Testing 1 2 3"); + strlcpy(buf, "!Testing 1 2 3?", sizeof(buf)); + tor_strstrip(buf, "!? "); + test_streq(buf, "Testing123"); + + /* Test tor_parse_long. */ + test_eq(10L, tor_parse_long("10",10,0,100,NULL,NULL)); + test_eq(0L, tor_parse_long("10",10,50,100,NULL,NULL)); + test_eq(-50L, tor_parse_long("-50",10,-100,100,NULL,NULL)); + + /* Test tor_parse_ulong */ + test_eq(10UL, tor_parse_ulong("10",10,0,100,NULL,NULL)); + test_eq(0UL, tor_parse_ulong("10",10,50,100,NULL,NULL)); + + /* Test tor_parse_uint64. */ + test_assert(U64_LITERAL(10) == tor_parse_uint64("10 x",10,0,100, &i, &cp)); + test_assert(i == 1); + test_streq(cp, " x"); + test_assert(U64_LITERAL(12345678901) == + tor_parse_uint64("12345678901",10,0,UINT64_MAX, &i, &cp)); + test_assert(i == 1); + test_streq(cp, ""); + test_assert(U64_LITERAL(0) == + tor_parse_uint64("12345678901",10,500,INT32_MAX, &i, &cp)); + test_assert(i == 0); + + { + /* Test tor_parse_double. */ + double d = tor_parse_double("10", 0, UINT64_MAX,&i,NULL); + test_assert(i == 1); + test_assert(DBL_TO_U64(d) == 10); + d = tor_parse_double("0", 0, UINT64_MAX,&i,NULL); + test_assert(i == 1); + test_assert(DBL_TO_U64(d) == 0); + d = tor_parse_double(" ", 0, UINT64_MAX,&i,NULL); + test_assert(i == 0); + d = tor_parse_double(".0a", 0, UINT64_MAX,&i,NULL); + test_assert(i == 0); + d = tor_parse_double(".0a", 0, UINT64_MAX,&i,&cp); + test_assert(i == 1); + d = tor_parse_double("-.0", 0, UINT64_MAX,&i,NULL); + test_assert(i == 1); + } + + /* Test failing snprintf cases */ + test_eq(-1, tor_snprintf(buf, 0, "Foo")); + test_eq(-1, tor_snprintf(buf, 2, "Foo")); + + /* Test printf with uint64 */ + tor_snprintf(buf, sizeof(buf), "x!"U64_FORMAT"!x", + U64_PRINTF_ARG(U64_LITERAL(12345678901))); + test_streq(buf, "x!12345678901!x"); + + /* Test for strcmpstart and strcmpend. */ + test_assert(strcmpstart("abcdef", "abcdef")==0); + test_assert(strcmpstart("abcdef", "abc")==0); + test_assert(strcmpstart("abcdef", "abd")<0); + test_assert(strcmpstart("abcdef", "abb")>0); + test_assert(strcmpstart("ab", "abb")<0); + + test_assert(strcmpend("abcdef", "abcdef")==0); + test_assert(strcmpend("abcdef", "def")==0); + test_assert(strcmpend("abcdef", "deg")<0); + test_assert(strcmpend("abcdef", "dee")>0); + test_assert(strcmpend("ab", "abb")<0); + + test_assert(strcasecmpend("AbcDEF", "abcdef")==0); + test_assert(strcasecmpend("abcdef", "dEF")==0); + test_assert(strcasecmpend("abcDEf", "deg")<0); + test_assert(strcasecmpend("abcdef", "DEE")>0); + test_assert(strcasecmpend("ab", "abB")<0); + + /* Test mem_is_zero */ + memset(buf,0,128); + buf[128] = 'x'; + test_assert(tor_digest_is_zero(buf)); + test_assert(tor_mem_is_zero(buf, 10)); + test_assert(tor_mem_is_zero(buf, 20)); + test_assert(tor_mem_is_zero(buf, 128)); + test_assert(!tor_mem_is_zero(buf, 129)); + buf[60] = (char)255; + test_assert(!tor_mem_is_zero(buf, 128)); + buf[0] = (char)1; + test_assert(!tor_mem_is_zero(buf, 10)); + + /* Test 'escaped' */ + test_streq("\"\"", escaped("")); + test_streq("\"abcd\"", escaped("abcd")); + test_streq("\"\\\\\\n\\r\\t\\\"\\'\"", escaped("\\\n\r\t\"\'")); + test_streq("\"z\\001abc\\277d\"", escaped("z\001abc\277d")); + test_assert(NULL == escaped(NULL)); + + /* Test strndup and memdup */ + { + const char *s = "abcdefghijklmnopqrstuvwxyz"; + cp = tor_strndup(s, 30); + test_streq(cp, s); /* same string, */ + test_neq(cp, s); /* but different pointers. */ + tor_free(cp); + + cp = tor_strndup(s, 5); + test_streq(cp, "abcde"); + tor_free(cp); + + s = "a\0b\0c\0d\0e\0"; + cp = tor_memdup(s,10); + test_memeq(cp, s, 10); /* same ram, */ + test_neq(cp, s); /* but different pointers. */ + tor_free(cp); + } + + /* Test str-foo functions */ + cp = tor_strdup("abcdef"); + test_assert(tor_strisnonupper(cp)); + cp[3] = 'D'; + test_assert(!tor_strisnonupper(cp)); + tor_strupper(cp); + test_streq(cp, "ABCDEF"); + test_assert(tor_strisprint(cp)); + cp[3] = 3; + test_assert(!tor_strisprint(cp)); + tor_free(cp); + + /* Test eat_whitespace. */ + { + const char *s = " \n a"; + test_eq_ptr(eat_whitespace(s), s+4); + s = "abcd"; + test_eq_ptr(eat_whitespace(s), s); + s = "#xyz\nab"; + test_eq_ptr(eat_whitespace(s), s+5); + } + + /* Test memmem and memstr */ + { + const char *haystack = "abcde"; + tor_assert(!tor_memmem(haystack, 5, "ef", 2)); + test_eq_ptr(tor_memmem(haystack, 5, "cd", 2), haystack + 2); + test_eq_ptr(tor_memmem(haystack, 5, "cde", 3), haystack + 2); + haystack = "ababcad"; + test_eq_ptr(tor_memmem(haystack, 7, "abc", 3), haystack + 2); + test_eq_ptr(tor_memstr(haystack, 7, "abc"), haystack + 2); + test_assert(!tor_memstr(haystack, 7, "fe")); + test_assert(!tor_memstr(haystack, 7, "longerthantheoriginal")); + } + + /* Test wrap_string */ + { + smartlist_t *sl = smartlist_create(); + wrap_string(sl, "This is a test of string wrapping functionality: woot.", + 10, "", ""); + cp = smartlist_join_strings(sl, "", 0, NULL); + test_streq(cp, + "This is a\ntest of\nstring\nwrapping\nfunctional\nity: woot.\n"); + tor_free(cp); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_clear(sl); + + wrap_string(sl, "This is a test of string wrapping functionality: woot.", + 16, "### ", "# "); + cp = smartlist_join_strings(sl, "", 0, NULL); + test_streq(cp, + "### This is a\n# test of string\n# wrapping\n# functionality:\n" + "# woot.\n"); + + tor_free(cp); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_free(sl); + } + done: + ; +} + +static void +test_util_pow2(void) +{ + /* Test tor_log2(). */ + test_eq(tor_log2(64), 6); + test_eq(tor_log2(65), 6); + test_eq(tor_log2(63), 5); + test_eq(tor_log2(1), 0); + test_eq(tor_log2(2), 1); + test_eq(tor_log2(3), 1); + test_eq(tor_log2(4), 2); + test_eq(tor_log2(5), 2); + test_eq(tor_log2(U64_LITERAL(40000000000000000)), 55); + test_eq(tor_log2(UINT64_MAX), 63); + + /* Test round_to_power_of_2 */ + test_eq(round_to_power_of_2(120), 128); + test_eq(round_to_power_of_2(128), 128); + test_eq(round_to_power_of_2(130), 128); + test_eq(round_to_power_of_2(U64_LITERAL(40000000000000000)), + U64_LITERAL(1)<<55); + test_eq(round_to_power_of_2(0), 2); + + done: + ; +} + +/** mutex for thread test to stop the threads hitting data at the same time. */ +static tor_mutex_t *_thread_test_mutex = NULL; +/** mutexes for the thread test to make sure that the threads have to + * interleave somewhat. */ +static tor_mutex_t *_thread_test_start1 = NULL, + *_thread_test_start2 = NULL; +/** Shared strmap for the thread test. */ +static strmap_t *_thread_test_strmap = NULL; +/** The name of thread1 for the thread test */ +static char *_thread1_name = NULL; +/** The name of thread2 for the thread test */ +static char *_thread2_name = NULL; + +static void _thread_test_func(void* _s) ATTR_NORETURN; + +/** How many iterations have the threads in the unit test run? */ +static int t1_count = 0, t2_count = 0; + +/** Helper function for threading unit tests: This function runs in a + * subthread. It grabs its own mutex (start1 or start2) to make sure that it + * should start, then it repeatedly alters _test_thread_strmap protected by + * _thread_test_mutex. */ +static void +_thread_test_func(void* _s) +{ + char *s = _s; + int i, *count; + tor_mutex_t *m; + char buf[64]; + char **cp; + if (!strcmp(s, "thread 1")) { + m = _thread_test_start1; + cp = &_thread1_name; + count = &t1_count; + } else { + m = _thread_test_start2; + cp = &_thread2_name; + count = &t2_count; + } + + tor_snprintf(buf, sizeof(buf), "%lu", tor_get_thread_id()); + *cp = tor_strdup(buf); + + tor_mutex_acquire(m); + + for (i=0; i<10000; ++i) { + tor_mutex_acquire(_thread_test_mutex); + strmap_set(_thread_test_strmap, "last to run", *cp); + ++*count; + tor_mutex_release(_thread_test_mutex); + } + tor_mutex_acquire(_thread_test_mutex); + strmap_set(_thread_test_strmap, s, *cp); + tor_mutex_release(_thread_test_mutex); + + tor_mutex_release(m); + + spawn_exit(); +} + +/** Run unit tests for threading logic. */ +static void +test_util_threads(void) +{ + char *s1 = NULL, *s2 = NULL; + int done = 0, timedout = 0; + time_t started; +#ifndef MS_WINDOWS + struct timeval tv; + tv.tv_sec=0; + tv.tv_usec=10; +#endif +#ifndef TOR_IS_MULTITHREADED + /* Skip this test if we aren't threading. We should be threading most + * everywhere by now. */ + if (1) + return; +#endif + _thread_test_mutex = tor_mutex_new(); + _thread_test_start1 = tor_mutex_new(); + _thread_test_start2 = tor_mutex_new(); + _thread_test_strmap = strmap_new(); + s1 = tor_strdup("thread 1"); + s2 = tor_strdup("thread 2"); + tor_mutex_acquire(_thread_test_start1); + tor_mutex_acquire(_thread_test_start2); + spawn_func(_thread_test_func, s1); + spawn_func(_thread_test_func, s2); + tor_mutex_release(_thread_test_start2); + tor_mutex_release(_thread_test_start1); + started = time(NULL); + while (!done) { + tor_mutex_acquire(_thread_test_mutex); + strmap_assert_ok(_thread_test_strmap); + if (strmap_get(_thread_test_strmap, "thread 1") && + strmap_get(_thread_test_strmap, "thread 2")) { + done = 1; + } else if (time(NULL) > started + 25) { + timedout = done = 1; + } + tor_mutex_release(_thread_test_mutex); +#ifndef MS_WINDOWS + /* Prevent the main thread from starving the worker threads. */ + select(0, NULL, NULL, NULL, &tv); +#endif + } + tor_mutex_acquire(_thread_test_start1); + tor_mutex_release(_thread_test_start1); + tor_mutex_acquire(_thread_test_start2); + tor_mutex_release(_thread_test_start2); + + tor_mutex_free(_thread_test_mutex); + + if (timedout) { + printf("\nTimed out: %d %d", t1_count, t2_count); + test_assert(strmap_get(_thread_test_strmap, "thread 1")); + test_assert(strmap_get(_thread_test_strmap, "thread 2")); + test_assert(!timedout); + } + + /* different thread IDs. */ + test_assert(strcmp(strmap_get(_thread_test_strmap, "thread 1"), + strmap_get(_thread_test_strmap, "thread 2"))); + test_assert(!strcmp(strmap_get(_thread_test_strmap, "thread 1"), + strmap_get(_thread_test_strmap, "last to run")) || + !strcmp(strmap_get(_thread_test_strmap, "thread 2"), + strmap_get(_thread_test_strmap, "last to run"))); + + done: + tor_free(s1); + tor_free(s2); + tor_free(_thread1_name); + tor_free(_thread2_name); + if (_thread_test_strmap) + strmap_free(_thread_test_strmap, NULL); + if (_thread_test_start1) + tor_mutex_free(_thread_test_start1); + if (_thread_test_start2) + tor_mutex_free(_thread_test_start2); +} + +/** Run unit tests for compression functions */ +static void +test_util_gzip(void) +{ + char *buf1=NULL, *buf2=NULL, *buf3=NULL, *cp1, *cp2; + const char *ccp2; + size_t len1, len2; + tor_zlib_state_t *state = NULL; + + buf1 = tor_strdup("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZAAAAAAAAAAAAAAAAAAAZ"); + test_assert(detect_compression_method(buf1, strlen(buf1)) == UNKNOWN_METHOD); + if (is_gzip_supported()) { + test_assert(!tor_gzip_compress(&buf2, &len1, buf1, strlen(buf1)+1, + GZIP_METHOD)); + test_assert(buf2); + test_assert(!memcmp(buf2, "\037\213", 2)); /* Gzip magic. */ + test_assert(detect_compression_method(buf2, len1) == GZIP_METHOD); + + test_assert(!tor_gzip_uncompress(&buf3, &len2, buf2, len1, + GZIP_METHOD, 1, LOG_INFO)); + test_assert(buf3); + test_streq(buf1,buf3); + + tor_free(buf2); + tor_free(buf3); + } + + test_assert(!tor_gzip_compress(&buf2, &len1, buf1, strlen(buf1)+1, + ZLIB_METHOD)); + test_assert(buf2); + test_assert(!memcmp(buf2, "\x78\xDA", 2)); /* deflate magic. */ + test_assert(detect_compression_method(buf2, len1) == ZLIB_METHOD); + + test_assert(!tor_gzip_uncompress(&buf3, &len2, buf2, len1, + ZLIB_METHOD, 1, LOG_INFO)); + test_assert(buf3); + test_streq(buf1,buf3); + + /* Check whether we can uncompress concatenated, compressed strings. */ + tor_free(buf3); + buf2 = tor_realloc(buf2, len1*2); + memcpy(buf2+len1, buf2, len1); + test_assert(!tor_gzip_uncompress(&buf3, &len2, buf2, len1*2, + ZLIB_METHOD, 1, LOG_INFO)); + test_eq(len2, (strlen(buf1)+1)*2); + test_memeq(buf3, + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZAAAAAAAAAAAAAAAAAAAZ\0" + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZAAAAAAAAAAAAAAAAAAAZ\0", + (strlen(buf1)+1)*2); + + tor_free(buf1); + tor_free(buf2); + tor_free(buf3); + + /* Check whether we can uncompress partial strings. */ + buf1 = + tor_strdup("String with low redundancy that won't be compressed much."); + test_assert(!tor_gzip_compress(&buf2, &len1, buf1, strlen(buf1)+1, + ZLIB_METHOD)); + tor_assert(len1>16); + /* when we allow an incomplete string, we should succeed.*/ + tor_assert(!tor_gzip_uncompress(&buf3, &len2, buf2, len1-16, + ZLIB_METHOD, 0, LOG_INFO)); + buf3[len2]='\0'; + tor_assert(len2 > 5); + tor_assert(!strcmpstart(buf1, buf3)); + + /* when we demand a complete string, this must fail. */ + tor_free(buf3); + tor_assert(tor_gzip_uncompress(&buf3, &len2, buf2, len1-16, + ZLIB_METHOD, 1, LOG_INFO)); + tor_assert(!buf3); + + /* Now, try streaming compression. */ + tor_free(buf1); + tor_free(buf2); + tor_free(buf3); + state = tor_zlib_new(1, ZLIB_METHOD); + tor_assert(state); + cp1 = buf1 = tor_malloc(1024); + len1 = 1024; + ccp2 = "ABCDEFGHIJABCDEFGHIJ"; + len2 = 21; + test_assert(tor_zlib_process(state, &cp1, &len1, &ccp2, &len2, 0) + == TOR_ZLIB_OK); + test_eq(len2, 0); /* Make sure we compressed it all. */ + test_assert(cp1 > buf1); + + len2 = 0; + cp2 = cp1; + test_assert(tor_zlib_process(state, &cp1, &len1, &ccp2, &len2, 1) + == TOR_ZLIB_DONE); + test_eq(len2, 0); + test_assert(cp1 > cp2); /* Make sure we really added something. */ + + tor_assert(!tor_gzip_uncompress(&buf3, &len2, buf1, 1024-len1, + ZLIB_METHOD, 1, LOG_WARN)); + test_streq(buf3, "ABCDEFGHIJABCDEFGHIJ"); /*Make sure it compressed right.*/ + + done: + if (state) + tor_zlib_free(state); + tor_free(buf2); + tor_free(buf3); + tor_free(buf1); +} + +/** Run unit tests for mmap() wrapper functionality. */ +static void +test_util_mmap(void) +{ + char *fname1 = tor_strdup(get_fname("mapped_1")); + char *fname2 = tor_strdup(get_fname("mapped_2")); + char *fname3 = tor_strdup(get_fname("mapped_3")); + const size_t buflen = 17000; + char *buf = tor_malloc(17000); + tor_mmap_t *mapping = NULL; + + crypto_rand(buf, buflen); + + mapping = tor_mmap_file(fname1); + test_assert(! mapping); + + write_str_to_file(fname1, "Short file.", 1); + write_bytes_to_file(fname2, buf, buflen, 1); + write_bytes_to_file(fname3, buf, 16384, 1); + + mapping = tor_mmap_file(fname1); + test_assert(mapping); + test_eq(mapping->size, strlen("Short file.")); + test_streq(mapping->data, "Short file."); +#ifdef MS_WINDOWS + tor_munmap_file(mapping); + mapping = NULL; + test_assert(unlink(fname1) == 0); +#else + /* make sure we can unlink. */ + test_assert(unlink(fname1) == 0); + test_streq(mapping->data, "Short file."); + tor_munmap_file(mapping); + mapping = NULL; +#endif + + /* Now a zero-length file. */ + write_str_to_file(fname1, "", 1); + mapping = tor_mmap_file(fname1); + test_eq(mapping, NULL); + test_eq(ERANGE, errno); + unlink(fname1); + + /* Make sure that we fail to map a no-longer-existent file. */ + mapping = tor_mmap_file(fname1); + test_assert(mapping == NULL); + + /* Now try a big file that stretches across a few pages and isn't aligned */ + mapping = tor_mmap_file(fname2); + test_assert(mapping); + test_eq(mapping->size, buflen); + test_memeq(mapping->data, buf, buflen); + tor_munmap_file(mapping); + mapping = NULL; + + /* Now try a big aligned file. */ + mapping = tor_mmap_file(fname3); + test_assert(mapping); + test_eq(mapping->size, 16384); + test_memeq(mapping->data, buf, 16384); + tor_munmap_file(mapping); + mapping = NULL; + + done: + unlink(fname1); + unlink(fname2); + unlink(fname3); + + tor_free(fname1); + tor_free(fname2); + tor_free(fname3); + tor_free(buf); + + if (mapping) + tor_munmap_file(mapping); +} + +/** Run unit tests for escaping/unescaping data for use by controllers. */ +static void +test_util_control_formats(void) +{ + char *out = NULL; + const char *inp = + "..This is a test\r\nof the emergency \nbroadcast\r\n..system.\r\nZ.\r\n"; + size_t sz; + + sz = read_escaped_data(inp, strlen(inp), &out); + test_streq(out, + ".This is a test\nof the emergency \nbroadcast\n.system.\nZ.\n"); + test_eq(sz, strlen(out)); + + done: + tor_free(out); +} + +static void +test_util_sscanf(void) +{ + unsigned u1, u2, u3; + char s1[10], s2[10], s3[10], ch; + int r; + + r = tor_sscanf("hello world", "hello world"); /* String match: success */ + test_eq(r, 0); + r = tor_sscanf("hello world 3", "hello worlb %u", &u1); /* String fail */ + test_eq(r, 0); + r = tor_sscanf("12345", "%u", &u1); /* Simple number */ + test_eq(r, 1); + test_eq(u1, 12345u); + r = tor_sscanf("", "%u", &u1); /* absent number */ + test_eq(r, 0); + r = tor_sscanf("A", "%u", &u1); /* bogus number */ + test_eq(r, 0); + r = tor_sscanf("4294967295", "%u", &u1); /* UINT32_MAX should work. */ + test_eq(r, 1); + test_eq(u1, 4294967295u); + r = tor_sscanf("4294967296", "%u", &u1); /* Always say -1 at 32 bits. */ + test_eq(r, 0); + r = tor_sscanf("123456", "%2u%u", &u1, &u2); /* Width */ + test_eq(r, 2); + test_eq(u1, 12u); + test_eq(u2, 3456u); + r = tor_sscanf("!12:3:456", "!%2u:%2u:%3u", &u1, &u2, &u3); /* separators */ + test_eq(r, 3); + test_eq(u1, 12u); + test_eq(u2, 3u); + test_eq(u3, 456u); + r = tor_sscanf("12:3:045", "%2u:%2u:%3u", &u1, &u2, &u3); /* 0s */ + test_eq(r, 3); + test_eq(u1, 12u); + test_eq(u2, 3u); + test_eq(u3, 45u); + /* %u does not match space.*/ + r = tor_sscanf("12:3: 45", "%2u:%2u:%3u", &u1, &u2, &u3); + test_eq(r, 2); + /* %u does not match negative numbers. */ + r = tor_sscanf("12:3:-4", "%2u:%2u:%3u", &u1, &u2, &u3); + test_eq(r, 2); + /* Arbitrary amounts of 0-padding are okay */ + r = tor_sscanf("12:03:000000000000000099", "%2u:%2u:%u", &u1, &u2, &u3); + test_eq(r, 3); + test_eq(u1, 12u); + test_eq(u2, 3u); + test_eq(u3, 99u); + + r = tor_sscanf("99% fresh", "%3u%% fresh", &u1); /* percents are scannable.*/ + test_eq(r, 1); + test_eq(u1, 99); + + r = tor_sscanf("hello", "%s", s1); /* %s needs a number. */ + test_eq(r, -1); + + r = tor_sscanf("hello", "%3s%7s", s1, s2); /* %s matches characters. */ + test_eq(r, 2); + test_streq(s1, "hel"); + test_streq(s2, "lo"); + r = tor_sscanf("WD40", "%2s%u", s3, &u1); /* %s%u */ + test_eq(r, 2); + test_streq(s3, "WD"); + test_eq(u1, 40); + r = tor_sscanf("76trombones", "%6u%9s", &u1, s1); /* %u%s */ + test_eq(r, 2); + test_eq(u1, 76); + test_streq(s1, "trombones"); + r = tor_sscanf("hello world", "%9s %9s", s1, s2); /* %s doesn't eat space. */ + test_eq(r, 2); + test_streq(s1, "hello"); + test_streq(s2, "world"); + r = tor_sscanf("hi", "%9s%9s%3s", s1, s2, s3); /* %s can be empty. */ + test_eq(r, 3); + test_streq(s1, "hi"); + test_streq(s2, ""); + test_streq(s3, ""); + + r = tor_sscanf("1.2.3", "%u.%u.%u%c", &u1, &u2, &u3, &ch); + test_eq(r, 3); + r = tor_sscanf("1.2.3 foobar", "%u.%u.%u%c", &u1, &u2, &u3, &ch); + test_eq(r, 4); + + done: + ; +} + +/** Run unittests for memory pool allocator */ +static void +test_util_mempool(void) +{ + mp_pool_t *pool = NULL; + smartlist_t *allocated = NULL; + int i; + + pool = mp_pool_new(1, 100); + test_assert(pool); + test_assert(pool->new_chunk_capacity >= 100); + test_assert(pool->item_alloc_size >= sizeof(void*)+1); + mp_pool_destroy(pool); + pool = NULL; + + pool = mp_pool_new(241, 2500); + test_assert(pool); + test_assert(pool->new_chunk_capacity >= 10); + test_assert(pool->item_alloc_size >= sizeof(void*)+241); + test_eq(pool->item_alloc_size & 0x03, 0); + test_assert(pool->new_chunk_capacity < 60); + + allocated = smartlist_create(); + for (i = 0; i < 20000; ++i) { + if (smartlist_len(allocated) < 20 || crypto_rand_int(2)) { + void *m = mp_pool_get(pool); + memset(m, 0x09, 241); + smartlist_add(allocated, m); + //printf("%d: %p\n", i, m); + //mp_pool_assert_ok(pool); + } else { + int idx = crypto_rand_int(smartlist_len(allocated)); + void *m = smartlist_get(allocated, idx); + //printf("%d: free %p\n", i, m); + smartlist_del(allocated, idx); + mp_pool_release(m); + //mp_pool_assert_ok(pool); + } + if (crypto_rand_int(777)==0) + mp_pool_clean(pool, 1, 1); + + if (i % 777) + mp_pool_assert_ok(pool); + } + + done: + if (allocated) { + SMARTLIST_FOREACH(allocated, void *, m, mp_pool_release(m)); + mp_pool_assert_ok(pool); + mp_pool_clean(pool, 0, 0); + mp_pool_assert_ok(pool); + smartlist_free(allocated); + } + + if (pool) + mp_pool_destroy(pool); +} + +/** Run unittests for memory area allocator */ +static void +test_util_memarea(void) +{ + memarea_t *area = memarea_new(); + char *p1, *p2, *p3, *p1_orig; + void *malloced_ptr = NULL; + int i; + + test_assert(area); + + p1_orig = p1 = memarea_alloc(area,64); + p2 = memarea_alloc_zero(area,52); + p3 = memarea_alloc(area,11); + + test_assert(memarea_owns_ptr(area, p1)); + test_assert(memarea_owns_ptr(area, p2)); + test_assert(memarea_owns_ptr(area, p3)); + /* Make sure we left enough space. */ + test_assert(p1+64 <= p2); + test_assert(p2+52 <= p3); + /* Make sure we aligned. */ + test_eq(((uintptr_t)p1) % sizeof(void*), 0); + test_eq(((uintptr_t)p2) % sizeof(void*), 0); + test_eq(((uintptr_t)p3) % sizeof(void*), 0); + test_assert(!memarea_owns_ptr(area, p3+8192)); + test_assert(!memarea_owns_ptr(area, p3+30)); + test_assert(tor_mem_is_zero(p2, 52)); + /* Make sure we don't overalign. */ + p1 = memarea_alloc(area, 1); + p2 = memarea_alloc(area, 1); + test_eq(p1+sizeof(void*), p2); + { + malloced_ptr = tor_malloc(64); + test_assert(!memarea_owns_ptr(area, malloced_ptr)); + tor_free(malloced_ptr); + } + + /* memarea_memdup */ + { + malloced_ptr = tor_malloc(64); + crypto_rand((char*)malloced_ptr, 64); + p1 = memarea_memdup(area, malloced_ptr, 64); + test_assert(p1 != malloced_ptr); + test_memeq(p1, malloced_ptr, 64); + tor_free(malloced_ptr); + } + + /* memarea_strdup. */ + p1 = memarea_strdup(area,""); + p2 = memarea_strdup(area, "abcd"); + test_assert(p1); + test_assert(p2); + test_streq(p1, ""); + test_streq(p2, "abcd"); + + /* memarea_strndup. */ + { + const char *s = "Ad ogni porta batte la morte e grida: il nome!"; + /* (From Turandot, act 3.) */ + size_t len = strlen(s); + p1 = memarea_strndup(area, s, 1000); + p2 = memarea_strndup(area, s, 10); + test_streq(p1, s); + test_assert(p2 >= p1 + len + 1); + test_memeq(s, p2, 10); + test_eq(p2[10], '\0'); + p3 = memarea_strndup(area, s, len); + test_streq(p3, s); + p3 = memarea_strndup(area, s, len-1); + test_memeq(s, p3, len-1); + test_eq(p3[len-1], '\0'); + } + + memarea_clear(area); + p1 = memarea_alloc(area, 1); + test_eq(p1, p1_orig); + memarea_clear(area); + + /* Check for running over an area's size. */ + for (i = 0; i < 512; ++i) { + p1 = memarea_alloc(area, crypto_rand_int(5)+1); + test_assert(memarea_owns_ptr(area, p1)); + } + memarea_assert_ok(area); + /* Make sure we can allocate a too-big object. */ + p1 = memarea_alloc_zero(area, 9000); + p2 = memarea_alloc_zero(area, 16); + test_assert(memarea_owns_ptr(area, p1)); + test_assert(memarea_owns_ptr(area, p2)); + + done: + memarea_drop_all(area); + tor_free(malloced_ptr); +} + +/** Run unit tests for utility functions to get file names relative to + * the data directory. */ +static void +test_util_datadir(void) +{ + char buf[1024]; + char *f = NULL; + char *temp_dir = NULL; + + temp_dir = get_datadir_fname(NULL); + f = get_datadir_fname("state"); + tor_snprintf(buf, sizeof(buf), "%s"PATH_SEPARATOR"state", temp_dir); + test_streq(f, buf); + tor_free(f); + f = get_datadir_fname2("cache", "thingy"); + tor_snprintf(buf, sizeof(buf), + "%s"PATH_SEPARATOR"cache"PATH_SEPARATOR"thingy", temp_dir); + test_streq(f, buf); + tor_free(f); + f = get_datadir_fname2_suffix("cache", "thingy", ".foo"); + tor_snprintf(buf, sizeof(buf), + "%s"PATH_SEPARATOR"cache"PATH_SEPARATOR"thingy.foo", temp_dir); + test_streq(f, buf); + tor_free(f); + f = get_datadir_fname_suffix("cache", ".foo"); + tor_snprintf(buf, sizeof(buf), "%s"PATH_SEPARATOR"cache.foo", + temp_dir); + test_streq(f, buf); + + done: + tor_free(f); + tor_free(temp_dir); +} + +static void +test_util_strtok(void) +{ + char buf[128]; + char buf2[128]; + char *cp1, *cp2; + strlcpy(buf, "Graved on the dark in gestures of descent", sizeof(buf)); + strlcpy(buf2, "they.seemed;their!own;most.perfect;monument", sizeof(buf2)); + /* -- "Year's End", Richard Wilbur */ + + test_streq("Graved", tor_strtok_r_impl(buf, " ", &cp1)); + test_streq("they", tor_strtok_r_impl(buf2, ".!..;!", &cp2)); +#define S1() tor_strtok_r_impl(NULL, " ", &cp1) +#define S2() tor_strtok_r_impl(NULL, ".!..;!", &cp2) + test_streq("on", S1()); + test_streq("the", S1()); + test_streq("dark", S1()); + test_streq("seemed", S2()); + test_streq("their", S2()); + test_streq("own", S2()); + test_streq("in", S1()); + test_streq("gestures", S1()); + test_streq("of", S1()); + test_streq("most", S2()); + test_streq("perfect", S2()); + test_streq("descent", S1()); + test_streq("monument", S2()); + test_assert(NULL == S1()); + test_assert(NULL == S2()); + done: + ; +} + +static void +test_util_find_str_at_start_of_line(void *ptr) +{ + const char *long_string = + "hello world. hello world. hello hello. howdy.\n" + "hello hello world\n"; + + (void)ptr; + + /* not-found case. */ + tt_assert(! find_str_at_start_of_line(long_string, "fred")); + + /* not-found case where haystack doesn't end with \n */ + tt_assert(! find_str_at_start_of_line("foobar\nbaz", "fred")); + + /* start-of-string case */ + tt_assert(long_string == + find_str_at_start_of_line(long_string, "hello world.")); + + /* start-of-line case */ + tt_assert(strchr(long_string,'\n')+1 == + find_str_at_start_of_line(long_string, "hello hello")); + done: + ; +} + +static void +test_util_asprintf(void *ptr) +{ +#define LOREMIPSUM \ + "Lorem ipsum dolor sit amet, consectetur adipisicing elit" + char *cp=NULL, *cp2=NULL; + int r; + (void)ptr; + + /* empty string. */ + r = tor_asprintf(&cp, "%s", ""); + tt_assert(cp); + tt_int_op(r, ==, strlen(cp)); + tt_str_op(cp, ==, ""); + + /* Short string with some printing in it. */ + r = tor_asprintf(&cp2, "First=%d, Second=%d", 101, 202); + tt_assert(cp2); + tt_int_op(r, ==, strlen(cp2)); + tt_str_op(cp2, ==, "First=101, Second=202"); + tt_assert(cp != cp2); + tor_free(cp); + tor_free(cp2); + + /* Glass-box test: a string exactly 128 characters long. */ + r = tor_asprintf(&cp, "Lorem1: %sLorem2: %s", LOREMIPSUM, LOREMIPSUM); + tt_assert(cp); + tt_int_op(r, ==, 128); + tt_assert(cp[128] == '\0'); + tt_str_op(cp, ==, + "Lorem1: "LOREMIPSUM"Lorem2: "LOREMIPSUM); + tor_free(cp); + + /* String longer than 128 characters */ + r = tor_asprintf(&cp, "1: %s 2: %s 3: %s", + LOREMIPSUM, LOREMIPSUM, LOREMIPSUM); + tt_assert(cp); + tt_int_op(r, ==, strlen(cp)); + tt_str_op(cp, ==, "1: "LOREMIPSUM" 2: "LOREMIPSUM" 3: "LOREMIPSUM); + + done: + tor_free(cp); + tor_free(cp2); +} + +#define UTIL_LEGACY(name) \ + { #name, legacy_test_helper, 0, &legacy_setup, test_util_ ## name } + +#define UTIL_TEST(name, flags) \ + { #name, test_util_ ## name, flags, NULL, NULL } + +struct testcase_t util_tests[] = { + UTIL_LEGACY(time), + UTIL_LEGACY(config_line), + UTIL_LEGACY(strmisc), + UTIL_LEGACY(pow2), + UTIL_LEGACY(gzip), + UTIL_LEGACY(datadir), + UTIL_LEGACY(mempool), + UTIL_LEGACY(memarea), + UTIL_LEGACY(control_formats), + UTIL_LEGACY(mmap), + UTIL_LEGACY(threads), + UTIL_LEGACY(sscanf), + UTIL_LEGACY(strtok), + UTIL_TEST(find_str_at_start_of_line, 0), + UTIL_TEST(asprintf, 0), + END_OF_TESTCASES +}; + diff --git a/src/test/tinytest.c b/src/test/tinytest.c new file mode 100644 index 0000000000..b358bb3a4d --- /dev/null +++ b/src/test/tinytest.c @@ -0,0 +1,369 @@ +/* tinytest.c -- Copyright 2009 Nick Mathewson + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <assert.h> + +#ifdef WIN32 +#include <windows.h> +#else +#include <sys/types.h> +#include <sys/wait.h> +#include <unistd.h> +#endif + +#ifndef __GNUC__ +#define __attribute__(x) +#endif + +#include "tinytest.h" +#include "tinytest_macros.h" + +#define LONGEST_TEST_NAME 16384 + +static int in_tinytest_main = 0; /**< true if we're in tinytest_main().*/ +static int n_ok = 0; /**< Number of tests that have passed */ +static int n_bad = 0; /**< Number of tests that have failed. */ +static int n_skipped = 0; /**< Number of tests that have been skipped. */ + +static int opt_forked = 0; /**< True iff we're called from inside a win32 fork*/ +static int opt_nofork = 0; /**< Suppress calls to fork() for debugging. */ +static int opt_verbosity = 1; /**< -==quiet,0==terse,1==normal,2==verbose */ +const char *verbosity_flag = ""; + +enum outcome { SKIP=2, OK=1, FAIL=0 }; +static enum outcome cur_test_outcome = 0; +const char *cur_test_prefix = NULL; /**< prefix of the current test group */ +/** Name of the current test, if we haven't logged is yet. Used for --quiet */ +const char *cur_test_name = NULL; + +#ifdef WIN32 +/** Pointer to argv[0] for win32. */ +static const char *commandname = NULL; +#endif + +static void usage(struct testgroup_t *groups, int list_groups) + __attribute__((noreturn)); + +static enum outcome +_testcase_run_bare(const struct testcase_t *testcase) +{ + void *env = NULL; + int outcome; + if (testcase->setup) { + env = testcase->setup->setup_fn(testcase); + if (!env) + return FAIL; + else if (env == (void*)TT_SKIP) + return SKIP; + } + + cur_test_outcome = OK; + testcase->fn(env); + outcome = cur_test_outcome; + + if (testcase->setup) { + if (testcase->setup->cleanup_fn(testcase, env) == 0) + outcome = FAIL; + } + + return outcome; +} + +#define MAGIC_EXITCODE 42 + +static enum outcome +_testcase_run_forked(const struct testgroup_t *group, + const struct testcase_t *testcase) +{ +#ifdef WIN32 + /* Fork? On Win32? How primitive! We'll do what the smart kids do: + we'll invoke our own exe (whose name we recall from the command + line) with a command line that tells it to run just the test we + want, and this time without forking. + + (No, threads aren't an option. The whole point of forking is to + share no state between tests.) + */ + int ok; + char buffer[LONGEST_TEST_NAME+256]; + STARTUPINFO si; + PROCESS_INFORMATION info; + DWORD exitcode; + + if (!in_tinytest_main) { + printf("\nERROR. On Windows, _testcase_run_forked must be" + " called from within tinytest_main.\n"); + abort(); + } + if (opt_verbosity>0) + printf("[forking] "); + + snprintf(buffer, sizeof(buffer), "%s --RUNNING-FORKED %s %s%s", + commandname, verbosity_flag, group->prefix, testcase->name); + + memset(&si, 0, sizeof(si)); + memset(&info, 0, sizeof(info)); + si.cb = sizeof(si); + + ok = CreateProcess(commandname, buffer, NULL, NULL, 0, + 0, NULL, NULL, &si, &info); + if (!ok) { + printf("CreateProcess failed!\n"); + return 0; + } + WaitForSingleObject(info.hProcess, INFINITE); + GetExitCodeProcess(info.hProcess, &exitcode); + CloseHandle(info.hProcess); + CloseHandle(info.hThread); + if (exitcode == 0) + return OK; + else if (exitcode == MAGIC_EXITCODE) + return SKIP; + else + return FAIL; +#else + int outcome_pipe[2]; + pid_t pid; + (void)group; + + if (pipe(outcome_pipe)) + perror("opening pipe"); + + if (opt_verbosity>0) + printf("[forking] "); + pid = fork(); + if (!pid) { + /* child. */ + int test_r, write_r; + char b[1]; + close(outcome_pipe[0]); + test_r = _testcase_run_bare(testcase); + assert(0<=(int)test_r && (int)test_r<=2); + b[0] = "NYS"[test_r]; + write_r = (int)write(outcome_pipe[1], b, 1); + if (write_r != 1) { + perror("write outcome to pipe"); + exit(1); + } + exit(0); + } else { + /* parent */ + int status, r; + char b[1]; + /* Close this now, so that if the other side closes it, + * our read fails. */ + close(outcome_pipe[1]); + r = (int)read(outcome_pipe[0], b, 1); + if (r == 0) { + printf("[Lost connection!] "); + return 0; + } else if (r != 1) { + perror("read outcome from pipe"); + } + waitpid(pid, &status, 0); + close(outcome_pipe[0]); + return b[0]=='Y' ? OK : (b[0]=='S' ? SKIP : FAIL); + } +#endif +} + +int +testcase_run_one(const struct testgroup_t *group, + const struct testcase_t *testcase) +{ + enum outcome outcome; + + if (testcase->flags & TT_SKIP) { + if (opt_verbosity>0) + printf("%s%s: SKIPPED\n", + group->prefix, testcase->name); + ++n_skipped; + return SKIP; + } + + if (opt_verbosity>0 && !opt_forked) { + printf("%s%s: ", group->prefix, testcase->name); + } else { + if (opt_verbosity==0) printf("."); + cur_test_prefix = group->prefix; + cur_test_name = testcase->name; + } + + if ((testcase->flags & TT_FORK) && !(opt_forked||opt_nofork)) { + outcome = _testcase_run_forked(group, testcase); + } else { + outcome = _testcase_run_bare(testcase); + } + + if (outcome == OK) { + ++n_ok; + if (opt_verbosity>0 && !opt_forked) + puts(opt_verbosity==1?"OK":""); + } else if (outcome == SKIP) { + ++n_skipped; + if (opt_verbosity>0 && !opt_forked) + puts("SKIPPED"); + } else { + ++n_bad; + if (!opt_forked) + printf("\n [%s FAILED]\n", testcase->name); + } + + if (opt_forked) { + exit(outcome==OK ? 0 : (outcome==SKIP?MAGIC_EXITCODE : 1)); + } else { + return (int)outcome; + } +} + +int +_tinytest_set_flag(struct testgroup_t *groups, const char *arg, unsigned long flag) +{ + int i, j; + size_t length = LONGEST_TEST_NAME; + char fullname[LONGEST_TEST_NAME]; + int found=0; + if (strstr(arg, "..")) + length = strstr(arg,"..")-arg; + for (i=0; groups[i].prefix; ++i) { + for (j=0; groups[i].cases[j].name; ++j) { + snprintf(fullname, sizeof(fullname), "%s%s", + groups[i].prefix, groups[i].cases[j].name); + if (!flag) /* Hack! */ + printf(" %s\n", fullname); + if (!strncmp(fullname, arg, length)) { + groups[i].cases[j].flags |= flag; + ++found; + } + } + } + return found; +} + +static void +usage(struct testgroup_t *groups, int list_groups) +{ + puts("Options are: [--verbose|--quiet|--terse] [--no-fork]"); + puts(" Specify tests by name, or using a prefix ending with '..'"); + puts(" Use --list-tests for a list of tests."); + if (list_groups) { + puts("Known tests are:"); + _tinytest_set_flag(groups, "..", 0); + } + exit(0); +} + +int +tinytest_main(int c, const char **v, struct testgroup_t *groups) +{ + int i, j, n=0; + +#ifdef WIN32 + commandname = v[0]; +#endif + for (i=1; i<c; ++i) { + if (v[i][0] == '-') { + if (!strcmp(v[i], "--RUNNING-FORKED")) { + opt_forked = 1; + } else if (!strcmp(v[i], "--no-fork")) { + opt_nofork = 1; + } else if (!strcmp(v[i], "--quiet")) { + opt_verbosity = -1; + verbosity_flag = "--quiet"; + } else if (!strcmp(v[i], "--verbose")) { + opt_verbosity = 2; + verbosity_flag = "--verbose"; + } else if (!strcmp(v[i], "--terse")) { + opt_verbosity = 0; + verbosity_flag = "--terse"; + } else if (!strcmp(v[i], "--help")) { + usage(groups, 0); + } else if (!strcmp(v[i], "--list-tests")) { + usage(groups, 1); + } else { + printf("Unknown option %s. Try --help\n",v[i]); + return -1; + } + } else { + ++n; + if (!_tinytest_set_flag(groups, v[i], _TT_ENABLED)) { + printf("No such test as %s!\n", v[i]); + return -1; + } + } + } + if (!n) + _tinytest_set_flag(groups, "..", _TT_ENABLED); + + setvbuf(stdout, NULL, _IONBF, 0); + + ++in_tinytest_main; + for (i=0; groups[i].prefix; ++i) + for (j=0; groups[i].cases[j].name; ++j) + if (groups[i].cases[j].flags & _TT_ENABLED) + testcase_run_one(&groups[i], + &groups[i].cases[j]); + + --in_tinytest_main; + + if (opt_verbosity==0) + puts(""); + + if (n_bad) + printf("%d/%d TESTS FAILED. (%d skipped)\n", n_bad, + n_bad+n_ok,n_skipped); + else if (opt_verbosity >= 1) + printf("%d tests ok. (%d skipped)\n", n_ok, n_skipped); + + return (n_bad == 0) ? 0 : 1; +} + +int +_tinytest_get_verbosity(void) +{ + return opt_verbosity; +} + +void +_tinytest_set_test_failed(void) +{ + if (opt_verbosity <= 0 && cur_test_name) { + if (opt_verbosity==0) puts(""); + printf("%s%s: ", cur_test_prefix, cur_test_name); + cur_test_name = NULL; + } + cur_test_outcome = 0; +} + +void +_tinytest_set_test_skipped(void) +{ + if (cur_test_outcome==OK) + cur_test_outcome = SKIP; +} + diff --git a/src/test/tinytest.h b/src/test/tinytest.h new file mode 100644 index 0000000000..a0cb913138 --- /dev/null +++ b/src/test/tinytest.h @@ -0,0 +1,87 @@ +/* tinytest.h -- Copyright 2009 Nick Mathewson + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _TINYTEST_H +#define _TINYTEST_H + +/** Flag for a test that needs to run in a subprocess. */ +#define TT_FORK (1<<0) +/** Runtime flag for a test we've decided to skip. */ +#define TT_SKIP (1<<1) +/** Internal runtime flag for a test we've decided to run. */ +#define _TT_ENABLED (1<<2) +/** If you add your own flags, make them start at this point. */ +#define TT_FIRST_USER_FLAG (1<<3) + +typedef void (*testcase_fn)(void *); + +struct testcase_t; + +/** Functions to initialize/teardown a structure for a testcase. */ +struct testcase_setup_t { + /** Return a new structure for use by a given testcase. */ + void *(*setup_fn)(const struct testcase_t *); + /** Clean/free a structure from setup_fn. Return 1 if ok, 0 on err. */ + int (*cleanup_fn)(const struct testcase_t *, void *); +}; + +/** A single test-case that you can run. */ +struct testcase_t { + const char *name; /**< An identifier for this case. */ + testcase_fn fn; /**< The function to run to implement this case. */ + unsigned long flags; /**< Bitfield of TT_* flags. */ + const struct testcase_setup_t *setup; /**< Optional setup/cleanup fns*/ + void *setup_data; /**< Extra data usable by setup function */ +}; +#define END_OF_TESTCASES { NULL, NULL, 0, NULL, NULL } + +/** A group of tests that are selectable together. */ +struct testgroup_t { + const char *prefix; /**< Prefix to prepend to testnames. */ + struct testcase_t *cases; /** Array, ending with END_OF_TESTCASES */ +}; +#define END_OF_GROUPS { NULL, NULL} + +/** Implementation: called from a test to indicate failure, before logging. */ +void _tinytest_set_test_failed(void); +/** Implementation: called from a test to indicate that we're skipping. */ +void _tinytest_set_test_skipped(void); +/** Implementation: return 0 for quiet, 1 for normal, 2 for loud. */ +int _tinytest_get_verbosity(void); +/** Implementation: Set a flag on tests matching a name; returns number + * of tests that matched. */ +int _tinytest_set_flag(struct testgroup_t *, const char *, unsigned long); + +/** Set all tests in 'groups' matching the name 'named' to be skipped. */ +#define tinytest_skip(groups, named) \ + _tinytest_set_flag(groups, named, TT_SKIP) + +/** Run a single testcase in a single group. */ +int testcase_run_one(const struct testgroup_t *,const struct testcase_t *); +/** Run a set of testcases from an END_OF_GROUPS-terminated array of groups, + as selected from the command line. */ +int tinytest_main(int argc, const char **argv, struct testgroup_t *groups); + +#endif diff --git a/src/test/tinytest_demo.c b/src/test/tinytest_demo.c new file mode 100644 index 0000000000..bd33cc37fa --- /dev/null +++ b/src/test/tinytest_demo.c @@ -0,0 +1,215 @@ +/* tinytest_demo.c -- Copyright 2009 Nick Mathewson + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + + +/* Welcome to the example file for tinytest! I'll show you how to set up + * some simple and not-so-simple testcases. */ + +/* Make sure you include these headers. */ +#include "tinytest.h" +#include "tinytest_macros.h" + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <errno.h> + +/* ============================================================ */ + +/* First, let's see if strcmp is working. (All your test cases should be + * functions declared to take a single void * as) an argument. */ +void +test_strcmp(void *data) +{ + (void)data; /* This testcase takes no data. */ + + /* Let's make sure the empty string is equal to itself */ + if (strcmp("","")) { + /* This macro tells tinytest to stop the current test + * and go straight to the "end" label. */ + tt_abort_msg("The empty string was not equal to itself"); + } + + /* Pretty often, calling tt_abort_msg to indicate failure is more + heavy-weight than you want. Instead, just say: */ + tt_assert(strcmp("testcase", "testcase") == 0); + + /* Occasionally, you don't want to stop the current testcase just + because a single assertion has failed. In that case, use + tt_want: */ + tt_want(strcmp("tinytest", "testcase") > 0); + + /* You can use the tt_*_op family of macros to compare values and to + fail unless they have the relationship you want. They produce + more useful output than tt_assert, since they display the actual + values of the failing things. + + Fail unless strcmp("abc, "abc") == 0 */ + tt_int_op(strcmp("abc", "abc"), ==, 0); + + /* Fail unless strcmp("abc, "abcd") is less than 0 */ + tt_int_op(strcmp("abc", "abcd"), < , 0); + + /* Incidentally, there's a test_str_op that uses strcmp internally. */ + tt_str_op("abc", <, "abcd"); + + + /* Every test-case function needs to finish with an "end:" + label and (optionally) code to clean up local variables. */ + end: + ; +} + +/* ============================================================ */ + +/* Now let's mess with setup and teardown functions! These are handy if + you have a bunch of tests that all need a similar environment, and you + want to reconstruct that environment freshly for each one. */ + +/* First you declare a type to hold the environment info, and functions to + set it up and tear it down. */ +struct data_buffer { + /* We're just going to have couple of character buffer. Using + setup/teardown functions is probably overkill for this case. + + You could also do file descriptors, complicated handles, temporary + files, etc. */ + char buffer1[512]; + char buffer2[512]; +}; +/* The setup function needs to take a const struct testcase_t and return + void* */ +void * +setup_data_buffer(const struct testcase_t *testcase) +{ + struct data_buffer *db = malloc(sizeof(struct data_buffer)); + + /* If you had a complicated set of setup rules, you might behave + differently here depending on testcase->flags or + testcase->setup_data or even or testcase->name. */ + + /* Returning a NULL here would mean that we couldn't set up for this + test, so we don't need to test db for null. */ + return db; +} +/* The clean function deallocates storage carefully and returns true on + success. */ +int +clean_data_buffer(const struct testcase_t *testcase, void *ptr) +{ + struct data_buffer *db = ptr; + + if (db) { + free(db); + return 1; + } + return 0; +} +/* Finally, declare a testcase_setup_t with these functions. */ +struct testcase_setup_t data_buffer_setup = { + setup_data_buffer, clean_data_buffer +}; + + +/* Now let's write our test. */ +void +test_memcpy(void *ptr) +{ + /* This time, we use the argument. */ + struct data_buffer *db = ptr; + + /* We'll also introduce a local variable that might need cleaning up. */ + char *mem = NULL; + + /* Let's make sure that memcpy does what we'd like. */ + strcpy(db->buffer1, "String 0"); + memcpy(db->buffer2, db->buffer1, sizeof(db->buffer1)); + tt_str_op(db->buffer1, ==, db->buffer2); + + /* Now we've allocated memory that's referenced by a local variable. + The end block of the function will clean it up. */ + mem = strdup("Hello world."); + tt_assert(mem); + + /* Another rather trivial test. */ + tt_str_op(db->buffer1, !=, mem); + + end: + /* This time our end block has something to do. */ + if (mem) + free(mem); +} + +/* ============================================================ */ + +/* Now we need to make sure that our tests get invoked. First, you take + a bunch of related tests and put them into an array of struct testcase_t. +*/ + +struct testcase_t demo_tests[] = { + /* Here's a really simple test: it has a name you can refer to it + with, and a function to invoke it. */ + { "strcmp", test_strcmp, }, + + /* The second test has a flag, "TT_FORK", to make it run in a + subprocess, and a pointer to the testcase_setup_t that configures + its environment. */ + { "memcpy", test_memcpy, TT_FORK, &data_buffer_setup }, + + /* The array has to end with END_OF_TESTCASES. */ + END_OF_TESTCASES +}; + +/* Next, we make an array of testgroups. This is mandatory. Unlike more + heavy-duty testing frameworks, groups can't nest. */ +struct testgroup_t groups[] = { + + /* Every group has a 'prefix', and an array of tests. That's it. */ + { "demo/", demo_tests }, + + END_OF_GROUPS +}; + + +int +main(int c, const char **v) +{ + /* Finally, just call tinytest_main(). It lets you specify verbose + or quiet output with --verbose and --quiet. You can list + specific tests: + + tinytest-demo demo/memcpy + + or use a ..-wildcard to select multiple tests with a common + prefix: + + tinytest-demo demo/.. + + If you list no tests, you get them all by default, so that + "tinytest-demo" and "tinytest-demo .." mean the same thing. + + */ + return tinytest_main(c, v, groups); +} diff --git a/src/test/tinytest_macros.h b/src/test/tinytest_macros.h new file mode 100644 index 0000000000..48c1fbdfb8 --- /dev/null +++ b/src/test/tinytest_macros.h @@ -0,0 +1,167 @@ +/* tinytest_macros.h -- Copyright 2009 Nick Mathewson + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _TINYTEST_MACROS_H +#define _TINYTEST_MACROS_H + +/* Helpers for defining statement-like macros */ +#define TT_STMT_BEGIN do { +#define TT_STMT_END } while(0) + +/* Redefine this if your test functions want to abort with something besides + * "goto end;" */ +#ifndef TT_EXIT_TEST_FUNCTION +#define TT_EXIT_TEST_FUNCTION TT_STMT_BEGIN goto end; TT_STMT_END +#endif + +/* Redefine this if you want to note success/failure in some different way. */ +#ifndef TT_DECLARE +#define TT_DECLARE(prefix, args) \ + TT_STMT_BEGIN \ + printf("\n %s %s:%d: ",prefix,__FILE__,__LINE__); \ + printf args ; \ + TT_STMT_END +#endif + +/* Announce a failure. Args are parenthesized printf args. */ +#define TT_GRIPE(args) TT_DECLARE("FAIL", args) + +/* Announce a non-failure if we're verbose. */ +#define TT_BLATHER(args) \ + TT_STMT_BEGIN \ + if (_tinytest_get_verbosity()>1) TT_DECLARE(" OK", args); \ + TT_STMT_END + +#define TT_DIE(args) \ + TT_STMT_BEGIN \ + _tinytest_set_test_failed(); \ + TT_GRIPE(args); \ + TT_EXIT_TEST_FUNCTION; \ + TT_STMT_END + +#define TT_FAIL(args) \ + TT_STMT_BEGIN \ + _tinytest_set_test_failed(); \ + TT_GRIPE(args); \ + TT_STMT_END + +/* Fail and abort the current test for the reason in msg */ +#define tt_abort_printf(msg) TT_DIE(msg) +#define tt_abort_perror(op) TT_DIE(("%s: %s [%d]",(op),strerror(errno), errno)) +#define tt_abort_msg(msg) TT_DIE(("%s", msg)) +#define tt_abort() TT_DIE(("%s", "(Failed.)")) + +/* Fail but do not abort the current test for the reason in msg. */ +#define tt_fail_printf(msg) TT_FAIL(msg) +#define tt_fail_perror(op) TT_FAIL(("%s: %s [%d]",(op),strerror(errno), errno)) +#define tt_fail_msg(msg) TT_FAIL(("%s", msg)) +#define tt_fail() TT_FAIL(("%s", "(Failed.)")) + +/* End the current test, and indicate we are skipping it. */ +#define tt_skip() \ + TT_STMT_BEGIN \ + _tinytest_set_test_skipped(); \ + TT_EXIT_TEST_FUNCTION; \ + TT_STMT_END + +#define _tt_want(b, msg, fail) \ + TT_STMT_BEGIN \ + if (!(b)) { \ + _tinytest_set_test_failed(); \ + TT_GRIPE((msg)); \ + fail; \ + } else { \ + TT_BLATHER((msg)); \ + } \ + TT_STMT_END + +/* Assert b, but do not stop the test if b fails. Log msg on failure. */ +#define tt_want_msg(b, msg) \ + _tt_want(b, msg, ); + +/* Assert b and stop the test if b fails. Log msg on failure. */ +#define tt_assert_msg(b, msg) \ + _tt_want(b, msg, TT_EXIT_TEST_FUNCTION); + +/* Assert b, but do not stop the test if b fails. */ +#define tt_want(b) tt_want_msg( (b), "want("#b")") +/* Assert b, and stop the test if b fails. */ +#define tt_assert(b) tt_assert_msg((b), "assert("#b")") + +#define tt_assert_test_fmt_type(a,b,str_test,type,test,printf_type,printf_fmt, \ + setup_block,cleanup_block) \ + TT_STMT_BEGIN \ + type _val1 = (type)(a); \ + type _val2 = (type)(b); \ + int _tt_status = (test); \ + if (!_tt_status || _tinytest_get_verbosity()>1) { \ + printf_type _print; \ + printf_type _print1; \ + printf_type _print2; \ + type _value = _val1; \ + setup_block; \ + _print1 = _print; \ + _value = _val2; \ + setup_block; \ + _print2 = _print; \ + TT_DECLARE(_tt_status?" OK":"FAIL", \ + ("assert(%s): "printf_fmt" vs "printf_fmt, \ + str_test, _print1, _print2)); \ + _print = _print1; \ + cleanup_block; \ + _print = _print2; \ + cleanup_block; \ + if (!_tt_status) { \ + _tinytest_set_test_failed(); \ + TT_EXIT_TEST_FUNCTION; \ + } \ + } \ + TT_STMT_END + +#define tt_assert_test_type(a,b,str_test,type,test,fmt) \ + tt_assert_test_fmt_type(a,b,str_test,type,test,type,fmt, \ + {_print=_value;},{}) + +/* Helper: assert that a op b, when cast to type. Format the values with + * printf format fmt on failure. */ +#define tt_assert_op_type(a,op,b,type,fmt) \ + tt_assert_test_type(a,b,#a" "#op" "#b,type,(_val1 op _val2),fmt) + +#define tt_int_op(a,op,b) \ + tt_assert_test_type(a,b,#a" "#op" "#b,long,(_val1 op _val2),"%ld") + +#define tt_uint_op(a,op,b) \ + tt_assert_test_type(a,b,#a" "#op" "#b,unsigned long, \ + (_val1 op _val2),"%lu") + +#define tt_ptr_op(a,op,b) \ + tt_assert_test_type(a,b,#a" "#op" "#b,void*, \ + (_val1 op _val2),"%p") + +#define tt_str_op(a,op,b) \ + tt_assert_test_type(a,b,#a" "#op" "#b,const char *, \ + (strcmp(_val1,_val2) op 0),"<%s>") + +#endif diff --git a/src/tools/Makefile.am b/src/tools/Makefile.am index 41786e4378..1bb5076849 100644 --- a/src/tools/Makefile.am +++ b/src/tools/Makefile.am @@ -3,16 +3,16 @@ noinst_PROGRAMS = tor-checkkey tor_resolve_SOURCES = tor-resolve.c tor_resolve_LDFLAGS = @TOR_LDFLAGS_libevent@ -tor_resolve_LDADD = ../common/libor.a @TOR_LIBEVENT_LIBS@ @TOR_LIB_WS32@ +tor_resolve_LDADD = ../common/libor.a -lm @TOR_LIBEVENT_LIBS@ @TOR_LIB_WS32@ tor_gencert_SOURCES = tor-gencert.c tor_gencert_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ \ @TOR_LDFLAGS_libevent@ tor_gencert_LDADD = ../common/libor.a ../common/libor-crypto.a \ - -lz @TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ + -lm @TOR_ZLIB_LIBS@ @TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ tor_checkkey_SOURCES = tor-checkkey.c tor_checkkey_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ \ @TOR_LDFLAGS_libevent@ tor_checkkey_LDADD = ../common/libor.a ../common/libor-crypto.a \ - -lz @TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ + -lm @TOR_ZLIB_LIBS@ @TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ diff --git a/src/tools/tor-checkkey.c b/src/tools/tor-checkkey.c index b29b52d8db..94c8cbd44c 100644 --- a/src/tools/tor-checkkey.c +++ b/src/tools/tor-checkkey.c @@ -6,19 +6,21 @@ #include <stdio.h> #include <stdlib.h> #include "crypto.h" -#include "log.h" -#include "util.h" +#include "torlog.h" +#include "../common/util.h" #include "compat.h" #include <openssl/bn.h> #include <openssl/rsa.h> -int main(int c, char **v) +int +main(int c, char **v) { crypto_pk_env_t *env; char *str; RSA *rsa; int wantdigest=0; int fname_idx; + char *fname=NULL; init_logging(); if (c < 2) { @@ -29,7 +31,7 @@ int main(int c, char **v) return 1; } - if (crypto_global_init(0)) { + if (crypto_global_init(0, NULL, NULL)) { fprintf(stderr, "Couldn't initialize crypto library.\n"); return 1; } @@ -46,7 +48,9 @@ int main(int c, char **v) fname_idx = 1; } - str = read_file_to_str(v[fname_idx], 0, NULL); + fname = expand_filename(v[fname_idx]); + str = read_file_to_str(fname, 0, NULL); + tor_free(fname); if (!str) { fprintf(stderr, "Couldn't read %s\n", v[fname_idx]); return 1; diff --git a/src/tools/tor-gencert.c b/src/tools/tor-gencert.c index 3f449dc66e..bc99c24bd7 100644 --- a/src/tools/tor-gencert.c +++ b/src/tools/tor-gencert.c @@ -13,6 +13,7 @@ #include <openssl/evp.h> #include <openssl/pem.h> +#include <openssl/rsa.h> #include <openssl/objects.h> #include <openssl/obj_mac.h> #include <openssl/err.h> @@ -27,8 +28,8 @@ #define CRYPTO_PRIVATE #include "compat.h" -#include "util.h" -#include "log.h" +#include "../common/util.h" +#include "../common/torlog.h" #include "crypto.h" #include "address.h" @@ -63,7 +64,6 @@ show_help(void) "[-c certificate_file]\n" " [-m lifetime_in_months] [-a address:port] " "[--passphrase-fd <fd>]\n"); - } /* XXXX copied from crypto.c */ @@ -218,6 +218,20 @@ parse_commandline(int argc, char **argv) return 0; } +static RSA * +generate_key(int bits) +{ + RSA *rsa = NULL; + crypto_pk_env_t *env = crypto_new_pk_env(); + if (crypto_pk_generate_key_with_bits(env,bits)<0) + goto done; + rsa = _crypto_pk_env_get_rsa(env); + rsa = RSAPrivateKey_dup(rsa); + done: + crypto_free_pk_env(env); + return rsa; +} + /** Try to read the identity key from <b>identity_key_file</b>. If no such * file exists and create_identity_key is set, make a new identity key and * store it. Return 0 on success, nonzero on failure. @@ -238,7 +252,7 @@ load_identity_key(void) } log_notice(LD_GENERAL, "Generating %d-bit RSA identity key.", IDENTITY_KEY_BITS); - if (!(key = RSA_generate_key(IDENTITY_KEY_BITS, 65537, NULL, NULL))) { + if (!(key = generate_key(IDENTITY_KEY_BITS))) { log_err(LD_GENERAL, "Couldn't generate identity key."); crypto_log_errors(LOG_ERR, "Generating identity key"); return 1; @@ -323,7 +337,7 @@ generate_signing_key(void) RSA *key; log_notice(LD_GENERAL, "Generating %d-bit RSA signing key.", SIGNING_KEY_BITS); - if (!(key = RSA_generate_key(SIGNING_KEY_BITS, 65537, NULL, NULL))) { + if (!(key = generate_key(SIGNING_KEY_BITS))) { log_err(LD_GENERAL, "Couldn't generate signing key."); crypto_log_errors(LOG_ERR, "Generating signing key"); return 1; @@ -393,7 +407,6 @@ get_fingerprint(EVP_PKEY *pkey, char *out) return r; } - /** Set <b>out</b> to the hex-encoded fingerprint of <b>pkey</b>. */ static int get_digest(EVP_PKEY *pkey, char *out) @@ -487,7 +500,6 @@ generate_certificate(void) return 0; } - /** Entry point to tor-gencert */ int main(int argc, char **argv) @@ -496,7 +508,7 @@ main(int argc, char **argv) init_logging(); /* Don't bother using acceleration. */ - if (crypto_global_init(0)) { + if (crypto_global_init(0, NULL, NULL)) { fprintf(stderr, "Couldn't initialize crypto library.\n"); return 1; } diff --git a/src/tools/tor-resolve.c b/src/tools/tor-resolve.c index 24c8a6406a..f5b0becaff 100644 --- a/src/tools/tor-resolve.c +++ b/src/tools/tor-resolve.c @@ -6,9 +6,9 @@ #include "orconfig.h" #include "compat.h" -#include "util.h" +#include "../common/util.h" #include "address.h" -#include "log.h" +#include "../common/torlog.h" #include <stdio.h> #include <stdlib.h> @@ -148,7 +148,7 @@ parse_socks4a_resolve_response(const char *hostname, static const char * socks5_reason_to_string(char reason) { - switch(reason) { + switch (reason) { case SOCKS5_SUCCEEDED: return "succeeded"; case SOCKS5_GENERAL_ERROR: @@ -251,7 +251,7 @@ do_resolve(const char *hostname, uint32_t sockshost, uint16_t socksport, } if (parse_socks4a_resolve_response(hostname, reply_buf, RESPONSE_LEN_4, - result_addr)<0){ + result_addr)<0) { return -1; } } else { diff --git a/src/win32/orconfig.h b/src/win32/orconfig.h index 65804fe19d..78e713f4c9 100644 --- a/src/win32/orconfig.h +++ b/src/win32/orconfig.h @@ -88,11 +88,18 @@ #define HAVE_STRING_H /* Define to 1 if you have the `strlcat' function. */ +#if defined (WINCE) +#define HAVE_STRLCAT +#else #undef HAVE_STRLCAT +#endif /* Define to 1 if you have the `strlcpy' function. */ +#if defined (WINCE) +#define HAVE_STRLCPY +#else #undef HAVE_STRLCPY - +#endif /* Define to 1 if you have the `strptime' function. */ #undef HAVE_STRPTIME @@ -226,6 +233,5 @@ #define USING_TWOS_COMPLEMENT /* Version number of package */ -#define VERSION "0.2.1.25" - +#define VERSION "0.2.2.14-alpha-dev" |