diff options
Diffstat (limited to 'src/common')
80 files changed, 4472 insertions, 1312 deletions
diff --git a/src/common/Makefile.nmake b/src/common/Makefile.nmake index b8c5dd4fea..a1c819fffa 100644 --- a/src/common/Makefile.nmake +++ b/src/common/Makefile.nmake @@ -7,8 +7,8 @@ LIBOR_OBJECTS = address.obj backtrace.obj compat.obj container.obj di_ops.obj \ log.obj memarea.obj mempool.obj procmon.obj sandbox.obj util.obj \ util_codedigest.obj -LIBOR_CRYPTO_OBJECTS = aes.obj crypto.obj crypto_format.obj torgzip.obj tortls.obj \ - crypto_curve25519.obj curve25519-donna.obj +LIBOR_CRYPTO_OBJECTS = aes.obj crypto.obj crypto_format.obj compress.obj compress_zlib.obj \ + tortls.obj crypto_curve25519.obj curve25519-donna.obj LIBOR_EVENT_OBJECTS = compat_libevent.obj diff --git a/src/common/address.c b/src/common/address.c index 96b99fa082..555f63da06 100644 --- a/src/common/address.c +++ b/src/common/address.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -159,6 +159,8 @@ tor_addr_from_sockaddr(tor_addr_t *a, const struct sockaddr *sa, tor_assert(a); tor_assert(sa); + /* This memset is redundant; leaving it in to avoid any future accidents, + however. */ memset(a, 0, sizeof(*a)); if (sa->sa_family == AF_INET) { @@ -235,8 +237,8 @@ tor_addr_make_null(tor_addr_t *a, sa_family_t family) * * Return 0 on success, -1 on failure; 1 on transient failure. */ -int -tor_addr_lookup(const char *name, uint16_t family, tor_addr_t *addr) +MOCK_IMPL(int, +tor_addr_lookup,(const char *name, uint16_t family, tor_addr_t *addr)) { /* Perhaps eventually this should be replaced by a tor_getaddrinfo or * something. @@ -562,8 +564,8 @@ tor_addr_parse_PTR_name(tor_addr_t *result, const char *address, /** Convert <b>addr</b> to an in-addr.arpa name or a .ip6.arpa name, * and store the result in the <b>outlen</b>-byte buffer at - * <b>out</b>. Return the number of chars written to <b>out</b>, not - * including the trailing \0, on success. Returns -1 on failure. */ + * <b>out</b>. Returns a non-negative integer on success. + * Returns -1 on failure. */ int tor_addr_to_PTR_name(char *out, size_t outlen, const tor_addr_t *addr) @@ -1804,9 +1806,10 @@ free_interface_address6_list(smartlist_t *addrs) * Returns NULL on failure. * Use free_interface_address6_list to free the returned list. */ -MOCK_IMPL(smartlist_t *,get_interface_address6_list,(int severity, - sa_family_t family, - int include_internal)) +MOCK_IMPL(smartlist_t *, +get_interface_address6_list,(int severity, + sa_family_t family, + int include_internal)) { smartlist_t *addrs; tor_addr_t addr; @@ -2074,7 +2077,8 @@ parse_port_range(const char *port, uint16_t *port_min_out, /** Given an IPv4 in_addr struct *<b>in</b> (in network order, as usual), * write it as a string into the <b>buf_len</b>-byte buffer in - * <b>buf</b>. + * <b>buf</b>. Returns a non-negative integer on success. + * Returns -1 on failure. */ int tor_inet_ntoa(const struct in_addr *in, char *buf, size_t buf_len) @@ -2125,7 +2129,8 @@ get_interface_address,(int severity, uint32_t *addr)) } /** Return true if we can tell that <b>name</b> is a canonical name for the - * loopback address. */ + * loopback address. Return true also for *.local hostnames, which are + * multicast DNS names for hosts on the local network. */ int tor_addr_hostname_is_local(const char *name) { @@ -2146,3 +2151,11 @@ tor_addr_port_new(const tor_addr_t *addr, uint16_t port) return ap; } +/** Return true iff <a>a</b> and <b>b</b> are the same address and port */ +int +tor_addr_port_eq(const tor_addr_port_t *a, + const tor_addr_port_t *b) +{ + return tor_addr_eq(&a->addr, &b->addr) && a->port == b->port; +} + diff --git a/src/common/address.h b/src/common/address.h index d57abd0d9e..8091f8cd7b 100644 --- a/src/common/address.h +++ b/src/common/address.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -190,7 +190,8 @@ tor_addr_eq_ipv4h(const tor_addr_t *a, uint32_t u) */ #define TOR_ADDR_BUF_LEN 48 -int tor_addr_lookup(const char *name, uint16_t family, tor_addr_t *addr_out); +MOCK_DECL(int, tor_addr_lookup,(const char *name, uint16_t family, + tor_addr_t *addr_out)); char *tor_addr_to_str_dup(const tor_addr_t *addr) ATTR_MALLOC; /** Wrapper function of fmt_addr_impl(). It does not decorate IPv6 @@ -344,6 +345,8 @@ get_interface_address_list(int severity, int include_internal) } tor_addr_port_t *tor_addr_port_new(const tor_addr_t *addr, uint16_t port); +int tor_addr_port_eq(const tor_addr_port_t *a, + const tor_addr_port_t *b); #ifdef ADDRESS_PRIVATE MOCK_DECL(smartlist_t *,get_interface_addresses_raw,(int severity, diff --git a/src/common/aes.c b/src/common/aes.c index 35c2d1e3a5..73abef143e 100644 --- a/src/common/aes.c +++ b/src/common/aes.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/aes.h b/src/common/aes.h index 1cda53f2fa..6fd9c3ea16 100644 --- a/src/common/aes.h +++ b/src/common/aes.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /* Implements a minimal interface to counter-mode AES. */ diff --git a/src/common/backtrace.c b/src/common/backtrace.c index 81e04e94eb..0f0fa857bf 100644 --- a/src/common/backtrace.c +++ b/src/common/backtrace.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2016, The Tor Project, Inc. */ +/* Copyright (c) 2013-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -76,7 +76,7 @@ clean_backtrace(void **stack, size_t depth, const ucontext_t *ctx) #ifdef PC_FROM_UCONTEXT #if defined(__linux__) const size_t n = 1; -#elif defined(__darwin__) || defined(__APPLE__) || defined(__OpenBSD__) \ +#elif defined(__darwin__) || defined(__APPLE__) || defined(OpenBSD) \ || defined(__FreeBSD__) const size_t n = 2; #else diff --git a/src/common/backtrace.h b/src/common/backtrace.h index b53fd2c668..8cd1dcf06c 100644 --- a/src/common/backtrace.h +++ b/src/common/backtrace.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2016, The Tor Project, Inc. */ +/* Copyright (c) 2013-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_BACKTRACE_H diff --git a/src/common/ciphers.inc b/src/common/ciphers.inc index 23f5fd2da4..0084b3e325 100644 --- a/src/common/ciphers.inc +++ b/src/common/ciphers.inc @@ -33,6 +33,26 @@ #else XCIPHER(0xc02f, TLS1_TXT_ECDHE_RSA_WITH_AES_128_GCM_SHA256) #endif +#ifdef TLS1_TXT_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + CIPHER(0xcca9, TLS1_TXT_ECDHE_ECDSA_WITH_CHACHA20_POLY1305) +#else + XCIPHER(0xcca9, TLS1_TXT_ECDHE_ECDSA_WITH_CHACHA20_POLY1305) +#endif +#ifdef TLS1_TXT_ECDHE_RSA_WITH_CHACHA20_POLY1305 + CIPHER(0xcca8, TLS1_TXT_ECDHE_RSA_WITH_CHACHA20_POLY1305) +#else + XCIPHER(0xcca8, TLS1_TXT_ECDHE_RSA_WITH_CHACHA20_POLY1305) +#endif +#ifdef TLS1_TXT_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + CIPHER(0xc02c, TLS1_TXT_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384) +#else + XCIPHER(0xc02c, TLS1_TXT_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384) +#endif +#ifdef TLS1_TXT_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + CIPHER(0xc030, TLS1_TXT_ECDHE_RSA_WITH_AES_256_GCM_SHA384) +#else + XCIPHER(0xc030, TLS1_TXT_ECDHE_RSA_WITH_AES_256_GCM_SHA384) +#endif #ifdef TLS1_TXT_ECDHE_ECDSA_WITH_AES_256_CBC_SHA CIPHER(0xc00a, TLS1_TXT_ECDHE_ECDSA_WITH_AES_256_CBC_SHA) #else @@ -53,88 +73,28 @@ #else XCIPHER(0xc014, TLS1_TXT_ECDHE_RSA_WITH_AES_256_CBC_SHA) #endif -#ifdef TLS1_TXT_ECDHE_RSA_WITH_DES_192_CBC3_SHA - CIPHER(0xc012, TLS1_TXT_ECDHE_RSA_WITH_DES_192_CBC3_SHA) -#else - XCIPHER(0xc012, TLS1_TXT_ECDHE_RSA_WITH_DES_192_CBC3_SHA) -#endif -#ifdef TLS1_TXT_ECDHE_ECDSA_WITH_RC4_128_SHA - CIPHER(0xc007, TLS1_TXT_ECDHE_ECDSA_WITH_RC4_128_SHA) -#else - XCIPHER(0xc007, TLS1_TXT_ECDHE_ECDSA_WITH_RC4_128_SHA) -#endif -#ifdef TLS1_TXT_ECDHE_RSA_WITH_RC4_128_SHA - CIPHER(0xc011, TLS1_TXT_ECDHE_RSA_WITH_RC4_128_SHA) -#else - XCIPHER(0xc011, TLS1_TXT_ECDHE_RSA_WITH_RC4_128_SHA) -#endif #ifdef TLS1_TXT_DHE_RSA_WITH_AES_128_SHA CIPHER(0x0033, TLS1_TXT_DHE_RSA_WITH_AES_128_SHA) #else XCIPHER(0x0033, TLS1_TXT_DHE_RSA_WITH_AES_128_SHA) #endif -#ifdef TLS1_TXT_DHE_DSS_WITH_AES_128_SHA - CIPHER(0x0032, TLS1_TXT_DHE_DSS_WITH_AES_128_SHA) -#else - XCIPHER(0x0032, TLS1_TXT_DHE_DSS_WITH_AES_128_SHA) -#endif -#ifdef TLS1_TXT_DHE_RSA_WITH_CAMELLIA_128_CBC_SHA - CIPHER(0x0045, TLS1_TXT_DHE_RSA_WITH_CAMELLIA_128_CBC_SHA) -#else - XCIPHER(0x0045, TLS1_TXT_DHE_RSA_WITH_CAMELLIA_128_CBC_SHA) -#endif #ifdef TLS1_TXT_DHE_RSA_WITH_AES_256_SHA CIPHER(0x0039, TLS1_TXT_DHE_RSA_WITH_AES_256_SHA) #else XCIPHER(0x0039, TLS1_TXT_DHE_RSA_WITH_AES_256_SHA) #endif -#ifdef TLS1_TXT_DHE_DSS_WITH_AES_256_SHA - CIPHER(0x0038, TLS1_TXT_DHE_DSS_WITH_AES_256_SHA) -#else - XCIPHER(0x0038, TLS1_TXT_DHE_DSS_WITH_AES_256_SHA) -#endif -#ifdef TLS1_TXT_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA - CIPHER(0x0088, TLS1_TXT_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA) -#else - XCIPHER(0x0088, TLS1_TXT_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA) -#endif -#ifdef SSL3_TXT_EDH_RSA_DES_192_CBC3_SHA - CIPHER(0x0016, SSL3_TXT_EDH_RSA_DES_192_CBC3_SHA) -#else - XCIPHER(0x0016, SSL3_TXT_EDH_RSA_DES_192_CBC3_SHA) -#endif #ifdef TLS1_TXT_RSA_WITH_AES_128_SHA CIPHER(0x002f, TLS1_TXT_RSA_WITH_AES_128_SHA) #else XCIPHER(0x002f, TLS1_TXT_RSA_WITH_AES_128_SHA) #endif -#ifdef TLS1_TXT_RSA_WITH_CAMELLIA_128_CBC_SHA - CIPHER(0x0041, TLS1_TXT_RSA_WITH_CAMELLIA_128_CBC_SHA) -#else - XCIPHER(0x0041, TLS1_TXT_RSA_WITH_CAMELLIA_128_CBC_SHA) -#endif #ifdef TLS1_TXT_RSA_WITH_AES_256_SHA CIPHER(0x0035, TLS1_TXT_RSA_WITH_AES_256_SHA) #else XCIPHER(0x0035, TLS1_TXT_RSA_WITH_AES_256_SHA) #endif -#ifdef TLS1_TXT_RSA_WITH_CAMELLIA_256_CBC_SHA - CIPHER(0x0084, TLS1_TXT_RSA_WITH_CAMELLIA_256_CBC_SHA) -#else - XCIPHER(0x0084, TLS1_TXT_RSA_WITH_CAMELLIA_256_CBC_SHA) -#endif #ifdef SSL3_TXT_RSA_DES_192_CBC3_SHA CIPHER(0x000a, SSL3_TXT_RSA_DES_192_CBC3_SHA) #else XCIPHER(0x000a, SSL3_TXT_RSA_DES_192_CBC3_SHA) #endif -#ifdef SSL3_TXT_RSA_RC4_128_SHA - CIPHER(0x0005, SSL3_TXT_RSA_RC4_128_SHA) -#else - XCIPHER(0x0005, SSL3_TXT_RSA_RC4_128_SHA) -#endif -#ifdef SSL3_TXT_RSA_RC4_128_MD5 - CIPHER(0x0004, SSL3_TXT_RSA_RC4_128_MD5) -#else - XCIPHER(0x0004, SSL3_TXT_RSA_RC4_128_MD5) -#endif diff --git a/src/common/compat.c b/src/common/compat.c index 4ac443c134..4a1f0013c6 100644 --- a/src/common/compat.c +++ b/src/common/compat.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -204,7 +204,15 @@ tor_rename(const char *path_old, const char *path_new) sandbox_intern_string(path_new)); } -#if defined(HAVE_SYS_MMAN_H) || defined(RUNNING_DOXYGEN) +/* Some MinGW builds have sys/mman.h, but not the corresponding symbols. + * Other configs rename the symbols using macros (including getpagesize). + * So check for sys/mman.h and unistd.h, and a getpagesize declaration. */ +#if (defined(HAVE_SYS_MMAN_H) && defined(HAVE_UNISTD_H) && \ + defined(HAVE_DECL_GETPAGESIZE)) +#define COMPAT_HAS_MMAN_AND_PAGESIZE +#endif + +#if defined(COMPAT_HAS_MMAN_AND_PAGESIZE) || defined(RUNNING_DOXYGEN) /** Try to create a memory mapping for <b>filename</b> and return it. On * failure, return NULL. Sets errno properly, using ERANGE to mean * "empty file". */ @@ -250,6 +258,12 @@ tor_mmap_file(const char *filename) page_size = getpagesize(); size += (size%page_size) ? page_size-(size%page_size) : 0; + if (st.st_size > SSIZE_T_CEILING || (off_t)size < st.st_size) { + log_warn(LD_FS, "File \"%s\" is too large. Ignoring.",filename); + errno = EFBIG; + close(fd); + return NULL; + } if (!size) { /* Zero-length file. If we call mmap on it, it will succeed but * return NULL, and bad things will happen. So just fail. */ @@ -2673,7 +2687,8 @@ static int uname_result_is_set = 0; /** Return a pointer to a description of our platform. */ -MOCK_IMPL(const char *, get_uname, (void)) +MOCK_IMPL(const char *, +get_uname,(void)) { #ifdef HAVE_UNAME struct utsname u; @@ -3246,7 +3261,7 @@ format_win32_error(DWORD err) FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, NULL, err, - MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), + MAKELANGID(LANG_ENGLISH, SUBLANG_DEFAULT), (LPVOID)&str, 0, NULL); diff --git a/src/common/compat.h b/src/common/compat.h index ee1c9454de..473ad2b957 100644 --- a/src/common/compat.h +++ b/src/common/compat.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_COMPAT_H @@ -414,10 +414,10 @@ struct tm *tor_gmtime_r(const time_t *timep, struct tm *result); #endif #ifndef timercmp -/** Replacement for timersub on platforms that do not have it: returns true +/** Replacement for timercmp on platforms that do not have it: returns true * iff the relational operator "op" makes the expression tv1 op tv2 true. * - * Note that while this definition should work for all boolean opeators, some + * Note that while this definition should work for all boolean operators, some * platforms' native timercmp definitions do not support >=, <=, or ==. So * don't use those. */ diff --git a/src/common/compat_libevent.c b/src/common/compat_libevent.c index 4a3b1af922..31eb4ac496 100644 --- a/src/common/compat_libevent.c +++ b/src/common/compat_libevent.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2009-2016, The Tor Project, Inc. */ +/* Copyright (c) 2009-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -280,6 +280,15 @@ tor_gettimeofday_cache_set(const struct timeval *tv) tor_assert(tv); memcpy(&cached_time_hires, tv, sizeof(*tv)); } + +/** For testing: called post-fork to make libevent reinitialize + * kernel structures. */ +void +tor_libevent_postfork(void) +{ + int r = event_reinit(tor_libevent_get_base()); + tor_assert(r == 0); +} #endif #endif diff --git a/src/common/compat_libevent.h b/src/common/compat_libevent.h index c2e34764e4..904938415c 100644 --- a/src/common/compat_libevent.h +++ b/src/common/compat_libevent.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2009-2016, The Tor Project, Inc. */ +/* Copyright (c) 2009-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_COMPAT_LIBEVENT_H @@ -54,6 +54,7 @@ void tor_gettimeofday_cached(struct timeval *tv); void tor_gettimeofday_cache_clear(void); #ifdef TOR_UNIT_TESTS void tor_gettimeofday_cache_set(const struct timeval *tv); +void tor_libevent_postfork(void); #endif #ifdef COMPAT_LIBEVENT_PRIVATE diff --git a/src/common/compat_openssl.h b/src/common/compat_openssl.h index 1bfe188075..2b94fe5b4e 100644 --- a/src/common/compat_openssl.h +++ b/src/common/compat_openssl.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_COMPAT_OPENSSL_H diff --git a/src/common/compat_pthreads.c b/src/common/compat_pthreads.c index c1ae66c1d2..8e4b833573 100644 --- a/src/common/compat_pthreads.c +++ b/src/common/compat_pthreads.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/compat_rust.c b/src/common/compat_rust.c new file mode 100644 index 0000000000..366fd4037b --- /dev/null +++ b/src/common/compat_rust.c @@ -0,0 +1,39 @@ +/* Copyright (c) 2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file rust_compat.c + * \brief Rust FFI compatibility functions and helpers. This file is only built + * if Rust is not used. + **/ + +#include "compat_rust.h" +#include "util.h" + +/** + * Free storage pointed to by <b>str</b>, and itself. + */ +void +rust_str_free(rust_str_t str) +{ + char *s = (char *)str; + tor_free(s); +} + +/** + * Return zero-terminated contained string. + */ +const char * +rust_str_get(const rust_str_t str) +{ + return (const char *)str; +} + +/* If we were using Rust, we'd say so on startup. */ +rust_str_t +rust_welcome_string(void) +{ + char *s = tor_malloc_zero(1); + return (rust_str_t)s; +} + diff --git a/src/common/compat_rust.h b/src/common/compat_rust.h new file mode 100644 index 0000000000..752a29b56c --- /dev/null +++ b/src/common/compat_rust.h @@ -0,0 +1,28 @@ +/* Copyright (c) 2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file rust_compat.h + * \brief Headers for rust_compat.c + **/ + +#ifndef TOR_RUST_COMPAT_H +#define TOR_RUST_COMPAT_H + +#include "torint.h" + +/** + * Strings allocated in Rust must be freed from Rust code again. Let's make + * it less likely to accidentally mess up and call tor_free() on it, because + * currently it'll just work but might break at any time. + */ +typedef uintptr_t rust_str_t; + +void rust_str_free(rust_str_t); + +const char *rust_str_get(const rust_str_t); + +rust_str_t rust_welcome_string(void); + +#endif + diff --git a/src/common/compat_threads.c b/src/common/compat_threads.c index f4809060d6..c593e9af8d 100644 --- a/src/common/compat_threads.c +++ b/src/common/compat_threads.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -94,51 +94,73 @@ in_main_thread(void) } #if defined(HAVE_EVENTFD) || defined(HAVE_PIPE) -/* As write(), but retry on EINTR */ +/* As write(), but retry on EINTR, and return the negative error code on + * error. */ static int write_ni(int fd, const void *buf, size_t n) { int r; again: r = (int) write(fd, buf, n); - if (r < 0 && errno == EINTR) - goto again; + if (r < 0) { + if (errno == EINTR) + goto again; + else + return -errno; + } return r; } -/* As read(), but retry on EINTR */ +/* As read(), but retry on EINTR, and return the negative error code on error. + */ static int read_ni(int fd, void *buf, size_t n) { int r; again: r = (int) read(fd, buf, n); - if (r < 0 && errno == EINTR) - goto again; + if (r < 0) { + if (errno == EINTR) + goto again; + else + return -errno; + } return r; } #endif -/** As send(), but retry on EINTR. */ +/** As send(), but retry on EINTR, and return the negative error code on + * error. */ static int send_ni(int fd, const void *buf, size_t n, int flags) { int r; again: r = (int) send(fd, buf, n, flags); - if (r < 0 && ERRNO_IS_EINTR(tor_socket_errno(fd))) - goto again; + if (r < 0) { + int error = tor_socket_errno(fd); + if (ERRNO_IS_EINTR(error)) + goto again; + else + return -error; + } return r; } -/** As recv(), but retry on EINTR. */ +/** As recv(), but retry on EINTR, and return the negative error code on + * error. */ static int recv_ni(int fd, void *buf, size_t n, int flags) { int r; again: r = (int) recv(fd, buf, n, flags); - if (r < 0 && ERRNO_IS_EINTR(tor_socket_errno(fd))) - goto again; + if (r < 0) { + int error = tor_socket_errno(fd); + if (ERRNO_IS_EINTR(error)) + goto again; + else + return -error; + } return r; } @@ -149,7 +171,7 @@ eventfd_alert(int fd) { uint64_t u = 1; int r = write_ni(fd, (void*)&u, sizeof(u)); - if (r < 0 && errno != EAGAIN) + if (r < 0 && -r != EAGAIN) return -1; return 0; } @@ -160,8 +182,8 @@ eventfd_drain(int fd) { uint64_t u = 0; int r = read_ni(fd, (void*)&u, sizeof(u)); - if (r < 0 && errno != EAGAIN) - return -1; + if (r < 0 && -r != EAGAIN) + return r; return 0; } #endif @@ -172,8 +194,8 @@ static int pipe_alert(int fd) { ssize_t r = write_ni(fd, "x", 1); - if (r < 0 && errno != EAGAIN) - return -1; + if (r < 0 && -r != EAGAIN) + return (int)r; return 0; } @@ -188,7 +210,7 @@ pipe_drain(int fd) r = read_ni(fd, buf, sizeof(buf)); } while (r > 0); if (r < 0 && errno != EAGAIN) - return -1; + return -errno; /* A value of r = 0 means EOF on the fd so successfully drained. */ return 0; } @@ -200,13 +222,13 @@ static int sock_alert(tor_socket_t fd) { ssize_t r = send_ni(fd, "x", 1, 0); - if (r < 0 && !ERRNO_IS_EAGAIN(tor_socket_errno(fd))) - return -1; + if (r < 0 && !ERRNO_IS_EAGAIN(-r)) + return (int)r; return 0; } /** Drain all the input from a socket <b>fd</b>, and ignore it. Return 0 on - * success, -1 on error. */ + * success, -errno on error. */ static int sock_drain(tor_socket_t fd) { @@ -215,8 +237,8 @@ sock_drain(tor_socket_t fd) do { r = recv_ni(fd, buf, sizeof(buf), 0); } while (r > 0); - if (r < 0 && !ERRNO_IS_EAGAIN(tor_socket_errno(fd))) - return -1; + if (r < 0 && !ERRNO_IS_EAGAIN(-r)) + return (int)r; /* A value of r = 0 means EOF on the fd so successfully drained. */ return 0; } @@ -330,3 +352,49 @@ alert_sockets_close(alert_sockets_t *socks) socks->read_fd = socks->write_fd = -1; } +/* + * XXXX We might be smart to move to compiler intrinsics or real atomic + * XXXX operations at some point. But not yet. + * + */ + +/** Initialize a new atomic counter with the value 0 */ +void +atomic_counter_init(atomic_counter_t *counter) +{ + memset(counter, 0, sizeof(*counter)); + tor_mutex_init_nonrecursive(&counter->mutex); +} +/** Clean up all resources held by an atomic counter. */ +void +atomic_counter_destroy(atomic_counter_t *counter) +{ + tor_mutex_uninit(&counter->mutex); + memset(counter, 0, sizeof(*counter)); +} +/** Add a value to an atomic counter. */ +void +atomic_counter_add(atomic_counter_t *counter, size_t add) +{ + tor_mutex_acquire(&counter->mutex); + counter->val += add; + tor_mutex_release(&counter->mutex); +} +/** Subtract a value from an atomic counter. */ +void +atomic_counter_sub(atomic_counter_t *counter, size_t sub) +{ + // this relies on unsigned overflow, but that's fine. + atomic_counter_add(counter, -sub); +} +/** Return the current value of an atomic counter */ +size_t +atomic_counter_get(atomic_counter_t *counter) +{ + size_t val; + tor_mutex_acquire(&counter->mutex); + val = counter->val; + tor_mutex_release(&counter->mutex); + return val; +} + diff --git a/src/common/compat_threads.h b/src/common/compat_threads.h index 171a9f93ff..9fa3d0d0b7 100644 --- a/src/common/compat_threads.h +++ b/src/common/compat_threads.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_COMPAT_THREADS_H @@ -147,5 +147,19 @@ void *tor_threadlocal_get(tor_threadlocal_t *threadlocal); */ void tor_threadlocal_set(tor_threadlocal_t *threadlocal, void *value); +/** + * Atomic counter type; holds a size_t value. + */ +typedef struct atomic_counter_t { + tor_mutex_t mutex; + size_t val; +} atomic_counter_t; + +void atomic_counter_init(atomic_counter_t *counter); +void atomic_counter_destroy(atomic_counter_t *counter); +void atomic_counter_add(atomic_counter_t *counter, size_t add); +void atomic_counter_sub(atomic_counter_t *counter, size_t sub); +size_t atomic_counter_get(atomic_counter_t *counter); + #endif diff --git a/src/common/compat_time.c b/src/common/compat_time.c index d044bbe1d7..2ccaa36e49 100644 --- a/src/common/compat_time.c +++ b/src/common/compat_time.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/compat_time.h b/src/common/compat_time.h index 2262446e57..90194c5ebc 100644 --- a/src/common/compat_time.h +++ b/src/common/compat_time.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/compat_winthreads.c b/src/common/compat_winthreads.c index 735be4ad17..915368b94f 100644 --- a/src/common/compat_winthreads.c +++ b/src/common/compat_winthreads.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/compress.c b/src/common/compress.c new file mode 100644 index 0000000000..beeff5fcb8 --- /dev/null +++ b/src/common/compress.c @@ -0,0 +1,658 @@ +/* Copyright (c) 2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file compress.c + * \brief Common compression API. + **/ + +#include "orconfig.h" + +#include <stdlib.h> +#include <stdio.h> +#include <assert.h> +#include <string.h> +#include "torint.h" + +#ifdef HAVE_NETINET_IN_H +#include <netinet/in.h> +#endif + +#include "util.h" +#include "torlog.h" +#include "compress.h" +#include "compress_lzma.h" +#include "compress_none.h" +#include "compress_zlib.h" +#include "compress_zstd.h" + +/** Total number of bytes allocated for compression state overhead. */ +static atomic_counter_t total_compress_allocation; + +/** @{ */ +/* These macros define the maximum allowable compression factor. Anything of + * size greater than CHECK_FOR_COMPRESSION_BOMB_AFTER is not allowed to + * have an uncompression factor (uncompressed size:compressed size ratio) of + * any greater than MAX_UNCOMPRESSION_FACTOR. + * + * Picking a value for MAX_UNCOMPRESSION_FACTOR is a trade-off: we want it to + * be small to limit the attack multiplier, but we also want it to be large + * enough so that no legitimate document --even ones we might invent in the + * future -- ever compresses by a factor of greater than + * MAX_UNCOMPRESSION_FACTOR. Within those parameters, there's a reasonably + * large range of possible values. IMO, anything over 8 is probably safe; IMO + * anything under 50 is probably sufficient. + */ +#define MAX_UNCOMPRESSION_FACTOR 25 +#define CHECK_FOR_COMPRESSION_BOMB_AFTER (1024*64) +/** @} */ + +/** Return true if uncompressing an input of size <b>in_size</b> to an input of + * size at least <b>size_out</b> looks like a compression bomb. */ +int +tor_compress_is_compression_bomb(size_t size_in, size_t size_out) +{ + if (size_in == 0 || size_out < CHECK_FOR_COMPRESSION_BOMB_AFTER) + return 0; + + return (size_out / size_in > MAX_UNCOMPRESSION_FACTOR); +} + +/** Guess the size that <b>in_len</b> will be after compression or + * decompression. */ +static size_t +guess_compress_size(int compress, compress_method_t method, + compression_level_t compression_level, + size_t in_len) +{ + // ignore these for now. + (void)compression_level; + if (method == NO_METHOD) { + /* Guess that we'll need an extra byte, to avoid a needless realloc + * for nul-termination */ + return (in_len < SIZE_MAX) ? in_len + 1 : in_len; + } + + /* Always guess a factor of 2. */ + if (compress) { + in_len /= 2; + } else { + if (in_len < SIZE_T_CEILING/2) + in_len *= 2; + } + return MAX(in_len, 1024); +} + +/** Internal function to implement tor_compress/tor_uncompress, depending on + * whether <b>compress</b> is set. All arguments are as for tor_compress or + * tor_uncompress. */ +static int +tor_compress_impl(int compress, + char **out, size_t *out_len, + const char *in, size_t in_len, + compress_method_t method, + compression_level_t compression_level, + int complete_only, + int protocol_warn_level) +{ + tor_compress_state_t *stream; + int rv; + + stream = tor_compress_new(compress, method, compression_level); + + if (stream == NULL) { + log_warn(LD_GENERAL, "NULL stream while %scompressing", + compress?"":"de"); + log_debug(LD_GENERAL, "method: %d level: %d at len: %lu", + method, compression_level, (unsigned long)in_len); + return -1; + } + + size_t in_len_orig = in_len; + size_t out_remaining, out_alloc; + char *outptr; + + out_remaining = out_alloc = + guess_compress_size(compress, method, compression_level, in_len); + *out = outptr = tor_malloc(out_remaining); + + const int finish = complete_only || compress; + + while (1) { + switch (tor_compress_process(stream, + &outptr, &out_remaining, + &in, &in_len, finish)) { + case TOR_COMPRESS_DONE: + if (in_len == 0 || compress) { + goto done; + } else { + // More data is present, and we're decompressing. So we may need to + // reinitialize the stream if we are handling multiple concatenated + // inputs. + tor_compress_free(stream); + stream = tor_compress_new(compress, method, compression_level); + if (stream == NULL) { + log_warn(LD_GENERAL, "NULL stream while %scompressing", + compress?"":"de"); + goto err; + } + } + break; + case TOR_COMPRESS_OK: + if (compress || complete_only) { + log_fn(protocol_warn_level, LD_PROTOCOL, + "Unexpected %s while %scompressing", + complete_only?"end of input":"result", + compress?"":"de"); + log_debug(LD_GENERAL, "method: %d level: %d at len: %lu", + method, compression_level, (unsigned long)in_len); + goto err; + } else { + if (in_len == 0) { + goto done; + } + } + break; + case TOR_COMPRESS_BUFFER_FULL: { + if (!compress && outptr < *out+out_alloc) { + // A buffer error in this case means that we have a problem + // with our input. + log_fn(protocol_warn_level, LD_PROTOCOL, + "Possible truncated or corrupt compressed data"); + goto err; + } + if (out_alloc >= SIZE_T_CEILING / 2) { + log_warn(LD_GENERAL, "While %scompresing data: ran out of space.", + compress?"":"un"); + goto err; + } + if (!compress && + tor_compress_is_compression_bomb(in_len_orig, out_alloc)) { + // This should already have been caught down in the backend logic. + // LCOV_EXCL_START + tor_assert_nonfatal_unreached(); + goto err; + // LCOV_EXCL_STOP + } + const size_t offset = outptr - *out; + out_alloc *= 2; + *out = tor_realloc(*out, out_alloc); + outptr = *out + offset; + out_remaining = out_alloc - offset; + break; + } + case TOR_COMPRESS_ERROR: + log_fn(protocol_warn_level, LD_GENERAL, + "Error while %scompresing data: bad input?", + compress?"":"un"); + goto err; // bad data. + default: + // LCOV_EXCL_START + tor_assert_nonfatal_unreached(); + goto err; + // LCOV_EXCL_STOP + } + } + done: + *out_len = outptr - *out; + if (compress && tor_compress_is_compression_bomb(*out_len, in_len_orig)) { + log_warn(LD_BUG, "We compressed something and got an insanely high " + "compression factor; other Tors would think this was a " + "compression bomb."); + goto err; + } + if (!compress) { + // NUL-terminate our output. + if (out_alloc == *out_len) + *out = tor_realloc(*out, out_alloc + 1); + (*out)[*out_len] = '\0'; + } + rv = 0; + goto out; + + err: + tor_free(*out); + *out_len = 0; + rv = -1; + goto out; + + out: + tor_compress_free(stream); + return rv; +} + +/** Given <b>in_len</b> bytes at <b>in</b>, compress them into a newly + * allocated buffer, using the method described in <b>method</b>. Store the + * compressed string in *<b>out</b>, and its length in *<b>out_len</b>. + * Return 0 on success, -1 on failure. + */ +int +tor_compress(char **out, size_t *out_len, + const char *in, size_t in_len, + compress_method_t method) +{ + return tor_compress_impl(1, out, out_len, in, in_len, method, + BEST_COMPRESSION, + 1, LOG_WARN); +} + +/** Given zero or more compressed strings of total length <b>in_len</b> bytes + * at <b>in</b>, uncompress them into a newly allocated buffer, using the + * method described in <b>method</b>. Store the uncompressed string in + * *<b>out</b>, and its length in *<b>out_len</b>. Return 0 on success, -1 on + * failure. + * + * If any bytes are written to <b>out</b>, an extra byte NUL is always + * written at the end, but not counted in <b>out_len</b>. This is a + * safety feature to ensure that the output can be treated as a + * NUL-terminated string -- though of course, callers should check + * out_len anyway. + * + * If <b>complete_only</b> is true, we consider a truncated input as a + * failure; otherwise we decompress as much as we can. Warn about truncated + * or corrupt inputs at <b>protocol_warn_level</b>. + */ +int +tor_uncompress(char **out, size_t *out_len, + const char *in, size_t in_len, + compress_method_t method, + int complete_only, + int protocol_warn_level) +{ + return tor_compress_impl(0, out, out_len, in, in_len, method, + BEST_COMPRESSION, + complete_only, protocol_warn_level); +} + +/** Try to tell whether the <b>in_len</b>-byte string in <b>in</b> is likely + * to be compressed or not. If it is, return the likeliest compression method. + * Otherwise, return UNKNOWN_METHOD. + */ +compress_method_t +detect_compression_method(const char *in, size_t in_len) +{ + if (in_len > 2 && fast_memeq(in, "\x1f\x8b", 2)) { + return GZIP_METHOD; + } else if (in_len > 2 && (in[0] & 0x0f) == 8 && + (ntohs(get_uint16(in)) % 31) == 0) { + return ZLIB_METHOD; + } else if (in_len > 2 && + fast_memeq(in, "\x5d\x00\x00", 3)) { + return LZMA_METHOD; + } else if (in_len > 3 && + fast_memeq(in, "\x28\xb5\x2f\xfd", 4)) { + return ZSTD_METHOD; + } else { + return UNKNOWN_METHOD; + } +} + +/** Return 1 if a given <b>method</b> is supported; otherwise 0. */ +int +tor_compress_supports_method(compress_method_t method) +{ + switch (method) { + case GZIP_METHOD: + case ZLIB_METHOD: + return tor_zlib_method_supported(); + case LZMA_METHOD: + return tor_lzma_method_supported(); + case ZSTD_METHOD: + return tor_zstd_method_supported(); + case NO_METHOD: + return 1; + case UNKNOWN_METHOD: + default: + return 0; + } +} + +/** + * Return a bitmask of the supported compression types, where 1<<m is + * set in the bitmask if and only if compression with method <b>m</b> is + * supported. + */ +unsigned +tor_compress_get_supported_method_bitmask(void) +{ + static unsigned supported = 0; + if (supported == 0) { + compress_method_t m; + for (m = NO_METHOD; m <= UNKNOWN_METHOD; ++m) { + if (tor_compress_supports_method(m)) { + supported |= (1u << m); + } + } + } + return supported; +} + +/** Table of compression method names. These should have an "x-" prefix, + * if they are not listed in the IANA content coding registry. */ +static const struct { + const char *name; + compress_method_t method; +} compression_method_names[] = { + { "gzip", GZIP_METHOD }, + { "deflate", ZLIB_METHOD }, + // We call this "x-tor-lzma" rather than "x-lzma", because we impose a + // lower maximum memory usage on the decoding side. + { "x-tor-lzma", LZMA_METHOD }, + { "x-zstd" , ZSTD_METHOD }, + { "identity", NO_METHOD }, + + /* Later entries in this table are not canonical; these are recognized but + * not emitted. */ + { "x-gzip", GZIP_METHOD }, +}; + +/** Return the canonical string representation of the compression method + * <b>method</b>, or NULL if the method isn't recognized. */ +const char * +compression_method_get_name(compress_method_t method) +{ + unsigned i; + for (i = 0; i < ARRAY_LENGTH(compression_method_names); ++i) { + if (method == compression_method_names[i].method) + return compression_method_names[i].name; + } + return NULL; +} + +/** Table of compression human readable method names. */ +static const struct { + compress_method_t method; + const char *name; +} compression_method_human_names[] = { + { NO_METHOD, "uncompressed" }, + { GZIP_METHOD, "gzipped" }, + { ZLIB_METHOD, "deflated" }, + { LZMA_METHOD, "LZMA compressed" }, + { ZSTD_METHOD, "Zstandard compressed" }, + { UNKNOWN_METHOD, "unknown encoding" }, +}; + +/** Return a human readable string representation of the compression method + * <b>method</b>, or NULL if the method isn't recognized. */ +const char * +compression_method_get_human_name(compress_method_t method) +{ + unsigned i; + for (i = 0; i < ARRAY_LENGTH(compression_method_human_names); ++i) { + if (method == compression_method_human_names[i].method) + return compression_method_human_names[i].name; + } + return NULL; +} + +/** Return the compression method represented by the string <b>name</b>, or + * UNKNOWN_METHOD if the string isn't recognized. */ +compress_method_t +compression_method_get_by_name(const char *name) +{ + unsigned i; + for (i = 0; i < ARRAY_LENGTH(compression_method_names); ++i) { + if (!strcmp(compression_method_names[i].name, name)) + return compression_method_names[i].method; + } + return UNKNOWN_METHOD; +} + +/** Return a string representation of the version of the library providing the + * compression method given in <b>method</b>. Returns NULL if <b>method</b> is + * unknown or unsupported. */ +const char * +tor_compress_version_str(compress_method_t method) +{ + switch (method) { + case GZIP_METHOD: + case ZLIB_METHOD: + return tor_zlib_get_version_str(); + case LZMA_METHOD: + return tor_lzma_get_version_str(); + case ZSTD_METHOD: + return tor_zstd_get_version_str(); + case NO_METHOD: + case UNKNOWN_METHOD: + default: + return NULL; + } +} + +/** Return a string representation of the version of the library, found at + * compile time, providing the compression method given in <b>method</b>. + * Returns NULL if <b>method</b> is unknown or unsupported. */ +const char * +tor_compress_header_version_str(compress_method_t method) +{ + switch (method) { + case GZIP_METHOD: + case ZLIB_METHOD: + return tor_zlib_get_header_version_str(); + case LZMA_METHOD: + return tor_lzma_get_header_version_str(); + case ZSTD_METHOD: + return tor_zstd_get_header_version_str(); + case NO_METHOD: + case UNKNOWN_METHOD: + default: + return NULL; + } +} + +/** Return the approximate number of bytes allocated for all + * supported compression schemas. */ +size_t +tor_compress_get_total_allocation(void) +{ + return atomic_counter_get(&total_compress_allocation) + + tor_zlib_get_total_allocation() + + tor_lzma_get_total_allocation() + + tor_zstd_get_total_allocation(); +} + +/** Internal state for an incremental compression/decompression. The body of + * this struct is not exposed. */ +struct tor_compress_state_t { + compress_method_t method; /**< The compression method. */ + + union { + tor_zlib_compress_state_t *zlib_state; + tor_lzma_compress_state_t *lzma_state; + tor_zstd_compress_state_t *zstd_state; + } u; /**< Compression backend state. */ +}; + +/** Construct and return a tor_compress_state_t object using <b>method</b>. If + * <b>compress</b>, it's for compression; otherwise it's for decompression. */ +tor_compress_state_t * +tor_compress_new(int compress, compress_method_t method, + compression_level_t compression_level) +{ + tor_compress_state_t *state; + + state = tor_malloc_zero(sizeof(tor_compress_state_t)); + state->method = method; + + switch (method) { + case GZIP_METHOD: + case ZLIB_METHOD: { + tor_zlib_compress_state_t *zlib_state = + tor_zlib_compress_new(compress, method, compression_level); + + if (zlib_state == NULL) + goto err; + + state->u.zlib_state = zlib_state; + break; + } + case LZMA_METHOD: { + tor_lzma_compress_state_t *lzma_state = + tor_lzma_compress_new(compress, method, compression_level); + + if (lzma_state == NULL) + goto err; + + state->u.lzma_state = lzma_state; + break; + } + case ZSTD_METHOD: { + tor_zstd_compress_state_t *zstd_state = + tor_zstd_compress_new(compress, method, compression_level); + + if (zstd_state == NULL) + goto err; + + state->u.zstd_state = zstd_state; + break; + } + case NO_METHOD: { + break; + } + case UNKNOWN_METHOD: + goto err; + } + + atomic_counter_add(&total_compress_allocation, + sizeof(tor_compress_state_t)); + return state; + + err: + tor_free(state); + return NULL; +} + +/** Compress/decompress some bytes using <b>state</b>. Read up to + * *<b>in_len</b> bytes from *<b>in</b>, and write up to *<b>out_len</b> bytes + * to *<b>out</b>, adjusting the values as we go. If <b>finish</b> is true, + * we've reached the end of the input. + * + * Return TOR_COMPRESS_DONE if we've finished the entire + * compression/decompression. + * Return TOR_COMPRESS_OK if we're processed everything from the input. + * Return TOR_COMPRESS_BUFFER_FULL if we're out of space on <b>out</b>. + * Return TOR_COMPRESS_ERROR if the stream is corrupt. + */ +tor_compress_output_t +tor_compress_process(tor_compress_state_t *state, + char **out, size_t *out_len, + const char **in, size_t *in_len, + int finish) +{ + tor_assert(state != NULL); + const size_t in_len_orig = *in_len; + const size_t out_len_orig = *out_len; + tor_compress_output_t rv; + + if (*out_len == 0 && (*in_len > 0 || finish)) { + // If we still have input data, but no space for output data, we might as + // well return early and let the caller do the reallocation of the out + // variable. + return TOR_COMPRESS_BUFFER_FULL; + } + + switch (state->method) { + case GZIP_METHOD: + case ZLIB_METHOD: + rv = tor_zlib_compress_process(state->u.zlib_state, + out, out_len, in, in_len, + finish); + break; + case LZMA_METHOD: + rv =tor_lzma_compress_process(state->u.lzma_state, + out, out_len, in, in_len, + finish); + break; + case ZSTD_METHOD: + rv = tor_zstd_compress_process(state->u.zstd_state, + out, out_len, in, in_len, + finish); + break; + case NO_METHOD: + rv = tor_cnone_compress_process(out, out_len, in, in_len, + finish); + break; + default: + case UNKNOWN_METHOD: + goto err; + } + if (BUG((rv == TOR_COMPRESS_OK) && + *in_len == in_len_orig && + *out_len == out_len_orig)) { + return TOR_COMPRESS_ERROR; + } + + return rv; + err: + return TOR_COMPRESS_ERROR; +} + +/** Deallocate <b>state</b>. */ +void +tor_compress_free(tor_compress_state_t *state) +{ + if (state == NULL) + return; + + switch (state->method) { + case GZIP_METHOD: + case ZLIB_METHOD: + tor_zlib_compress_free(state->u.zlib_state); + break; + case LZMA_METHOD: + tor_lzma_compress_free(state->u.lzma_state); + break; + case ZSTD_METHOD: + tor_zstd_compress_free(state->u.zstd_state); + break; + case NO_METHOD: + break; + case UNKNOWN_METHOD: + break; + } + + atomic_counter_sub(&total_compress_allocation, + sizeof(tor_compress_state_t)); + tor_free(state); +} + +/** Return the approximate number of bytes allocated for <b>state</b>. */ +size_t +tor_compress_state_size(const tor_compress_state_t *state) +{ + tor_assert(state != NULL); + + size_t size = sizeof(tor_compress_state_t); + + switch (state->method) { + case GZIP_METHOD: + case ZLIB_METHOD: + size += tor_zlib_compress_state_size(state->u.zlib_state); + break; + case LZMA_METHOD: + size += tor_lzma_compress_state_size(state->u.lzma_state); + break; + case ZSTD_METHOD: + size += tor_zstd_compress_state_size(state->u.zstd_state); + break; + case NO_METHOD: + case UNKNOWN_METHOD: + break; + } + + return size; +} + +/** Initialize all compression modules. */ +void +tor_compress_init(void) +{ + atomic_counter_init(&total_compress_allocation); + + tor_zlib_init(); + tor_lzma_init(); + tor_zstd_init(); +} + diff --git a/src/common/compress.h b/src/common/compress.h new file mode 100644 index 0000000000..59c8b7b9b5 --- /dev/null +++ b/src/common/compress.h @@ -0,0 +1,89 @@ +/* Copyright (c) 2003, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file compress.h + * \brief Headers for compress.c + **/ + +#ifndef TOR_COMPRESS_H +#define TOR_COMPRESS_H + +/** Enumeration of what kind of compression to use. Only ZLIB_METHOD and + * GZIP_METHOD is guaranteed to be supported by the compress/uncompress + * functions here. Call tor_compress_supports_method() to check if a given + * compression schema is supported by Tor. */ +typedef enum { + NO_METHOD=0, // This method must be first. + GZIP_METHOD=1, + ZLIB_METHOD=2, + LZMA_METHOD=3, + ZSTD_METHOD=4, + UNKNOWN_METHOD=5, // This method must be last. Add new ones in the middle. +} compress_method_t; + +/** + * Enumeration to define tradeoffs between memory usage and compression level. + * BEST_COMPRESSION saves the most bandwidth; LOW_COMPRESSION saves the most + * memory. + **/ +typedef enum { + BEST_COMPRESSION, HIGH_COMPRESSION, MEDIUM_COMPRESSION, LOW_COMPRESSION +} compression_level_t; + +int tor_compress(char **out, size_t *out_len, + const char *in, size_t in_len, + compress_method_t method); + +int tor_uncompress(char **out, size_t *out_len, + const char *in, size_t in_len, + compress_method_t method, + int complete_only, + int protocol_warn_level); + +compress_method_t detect_compression_method(const char *in, size_t in_len); + +int tor_compress_is_compression_bomb(size_t size_in, size_t size_out); + +int tor_compress_supports_method(compress_method_t method); +unsigned tor_compress_get_supported_method_bitmask(void); +const char *compression_method_get_name(compress_method_t method); +const char *compression_method_get_human_name(compress_method_t method); +compress_method_t compression_method_get_by_name(const char *name); + +const char *tor_compress_version_str(compress_method_t method); + +const char *tor_compress_header_version_str(compress_method_t method); + +size_t tor_compress_get_total_allocation(void); + +/** Return values from tor_compress_process; see that function's documentation + * for details. */ +typedef enum { + TOR_COMPRESS_OK, + TOR_COMPRESS_DONE, + TOR_COMPRESS_BUFFER_FULL, + TOR_COMPRESS_ERROR +} tor_compress_output_t; + +/** Internal state for an incremental compression/decompression. */ +typedef struct tor_compress_state_t tor_compress_state_t; + +tor_compress_state_t *tor_compress_new(int compress, + compress_method_t method, + compression_level_t level); + +tor_compress_output_t tor_compress_process(tor_compress_state_t *state, + char **out, size_t *out_len, + const char **in, size_t *in_len, + int finish); +void tor_compress_free(tor_compress_state_t *state); + +size_t tor_compress_state_size(const tor_compress_state_t *state); + +void tor_compress_init(void); + +#endif // TOR_COMPRESS_H. + diff --git a/src/common/compress_lzma.c b/src/common/compress_lzma.c new file mode 100644 index 0000000000..d453d9f718 --- /dev/null +++ b/src/common/compress_lzma.c @@ -0,0 +1,357 @@ +/* Copyright (c) 2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file compress_lzma.c + * \brief Compression backend for LZMA. + * + * This module should never be invoked directly. Use the compress module + * instead. + **/ + +#include "orconfig.h" + +#include "util.h" +#include "torlog.h" +#include "compress.h" +#include "compress_lzma.h" + +#ifdef HAVE_LZMA +#include <lzma.h> +#endif + +/** The maximum amount of memory we allow the LZMA decoder to use, in bytes. */ +#define MEMORY_LIMIT (16 * 1024 * 1024) + +/** Total number of bytes allocated for LZMA state. */ +static atomic_counter_t total_lzma_allocation; + +#ifdef HAVE_LZMA +/** Given <b>level</b> return the memory level. */ +static int +memory_level(compression_level_t level) +{ + switch (level) { + default: + case BEST_COMPRESSION: + case HIGH_COMPRESSION: return 6; + case MEDIUM_COMPRESSION: return 4; + case LOW_COMPRESSION: return 2; + } +} + +/** Convert a given <b>error</b> to a human readable error string. */ +static const char * +lzma_error_str(lzma_ret error) +{ + switch (error) { + case LZMA_OK: + return "Operation completed successfully"; + case LZMA_STREAM_END: + return "End of stream"; + case LZMA_NO_CHECK: + return "Input stream lacks integrity check"; + case LZMA_UNSUPPORTED_CHECK: + return "Unable to calculate integrity check"; + case LZMA_GET_CHECK: + return "Integrity check available"; + case LZMA_MEM_ERROR: + return "Unable to allocate memory"; + case LZMA_MEMLIMIT_ERROR: + return "Memory limit reached"; + case LZMA_FORMAT_ERROR: + return "Unknown file format"; + case LZMA_OPTIONS_ERROR: + return "Unsupported options"; + case LZMA_DATA_ERROR: + return "Corrupt input data"; + case LZMA_BUF_ERROR: + return "Unable to progress"; + case LZMA_PROG_ERROR: + return "Programming error"; + default: + return "Unknown LZMA error"; + } +} +#endif // HAVE_LZMA. + +/** Return 1 if LZMA compression is supported; otherwise 0. */ +int +tor_lzma_method_supported(void) +{ +#ifdef HAVE_LZMA + return 1; +#else + return 0; +#endif +} + +/** Return a string representation of the version of the currently running + * version of liblzma. Returns NULL if LZMA is unsupported. */ +const char * +tor_lzma_get_version_str(void) +{ +#ifdef HAVE_LZMA + return lzma_version_string(); +#else + return NULL; +#endif +} + +/** Return a string representation of the version of liblzma used at + * compilation time. Returns NULL if LZMA is unsupported. */ +const char * +tor_lzma_get_header_version_str(void) +{ +#ifdef HAVE_LZMA + return LZMA_VERSION_STRING; +#else + return NULL; +#endif +} + +/** Internal LZMA state for incremental compression/decompression. + * The body of this struct is not exposed. */ +struct tor_lzma_compress_state_t { +#ifdef HAVE_LZMA + lzma_stream stream; /**< The LZMA stream. */ +#endif + + int compress; /**< True if we are compressing; false if we are inflating */ + + /** Number of bytes read so far. Used to detect compression bombs. */ + size_t input_so_far; + /** Number of bytes written so far. Used to detect compression bombs. */ + size_t output_so_far; + + /** Approximate number of bytes allocated for this object. */ + size_t allocation; +}; + +#ifdef HAVE_LZMA +/** Return an approximate number of bytes stored in memory to hold the LZMA + * encoder/decoder state. */ +static size_t +tor_lzma_state_size_precalc(int compress, compression_level_t level) +{ + uint64_t memory_usage; + + if (compress) + memory_usage = lzma_easy_encoder_memusage(memory_level(level)); + else + memory_usage = lzma_easy_decoder_memusage(memory_level(level)); + + if (memory_usage == UINT64_MAX) { + // LCOV_EXCL_START + log_warn(LD_GENERAL, "Unsupported compression level passed to LZMA %s", + compress ? "encoder" : "decoder"); + goto err; + // LCOV_EXCL_STOP + } + + if (memory_usage + sizeof(tor_lzma_compress_state_t) > SIZE_MAX) + memory_usage = SIZE_MAX; + else + memory_usage += sizeof(tor_lzma_compress_state_t); + + return (size_t)memory_usage; + + err: + return 0; // LCOV_EXCL_LINE +} +#endif // HAVE_LZMA. + +/** Construct and return a tor_lzma_compress_state_t object using + * <b>method</b>. If <b>compress</b>, it's for compression; otherwise it's for + * decompression. */ +tor_lzma_compress_state_t * +tor_lzma_compress_new(int compress, + compress_method_t method, + compression_level_t level) +{ + tor_assert(method == LZMA_METHOD); + +#ifdef HAVE_LZMA + tor_lzma_compress_state_t *result; + lzma_ret retval; + lzma_options_lzma stream_options; + + // Note that we do not explicitly initialize the lzma_stream object here, + // since the LZMA_STREAM_INIT "just" initializes all members to 0, which is + // also what `tor_malloc_zero()` does. + result = tor_malloc_zero(sizeof(tor_lzma_compress_state_t)); + result->compress = compress; + result->allocation = tor_lzma_state_size_precalc(compress, level); + + if (compress) { + lzma_lzma_preset(&stream_options, memory_level(level)); + + retval = lzma_alone_encoder(&result->stream, &stream_options); + + if (retval != LZMA_OK) { + // LCOV_EXCL_START + log_warn(LD_GENERAL, "Error from LZMA encoder: %s (%u).", + lzma_error_str(retval), retval); + goto err; + // LCOV_EXCL_STOP + } + } else { + retval = lzma_alone_decoder(&result->stream, MEMORY_LIMIT); + + if (retval != LZMA_OK) { + // LCOV_EXCL_START + log_warn(LD_GENERAL, "Error from LZMA decoder: %s (%u).", + lzma_error_str(retval), retval); + goto err; + // LCOV_EXCL_STOP + } + } + + atomic_counter_add(&total_lzma_allocation, result->allocation); + return result; + + err: + tor_free(result); // LCOV_EXCL_LINE + return NULL; +#else // HAVE_LZMA. + (void)compress; + (void)method; + (void)level; + + return NULL; +#endif // HAVE_LZMA. +} + +/** Compress/decompress some bytes using <b>state</b>. Read up to + * *<b>in_len</b> bytes from *<b>in</b>, and write up to *<b>out_len</b> bytes + * to *<b>out</b>, adjusting the values as we go. If <b>finish</b> is true, + * we've reached the end of the input. + * + * Return TOR_COMPRESS_DONE if we've finished the entire + * compression/decompression. + * Return TOR_COMPRESS_OK if we're processed everything from the input. + * Return TOR_COMPRESS_BUFFER_FULL if we're out of space on <b>out</b>. + * Return TOR_COMPRESS_ERROR if the stream is corrupt. + */ +tor_compress_output_t +tor_lzma_compress_process(tor_lzma_compress_state_t *state, + char **out, size_t *out_len, + const char **in, size_t *in_len, + int finish) +{ +#ifdef HAVE_LZMA + lzma_ret retval; + lzma_action action; + + tor_assert(state != NULL); + tor_assert(*in_len <= UINT_MAX); + tor_assert(*out_len <= UINT_MAX); + + state->stream.next_in = (unsigned char *)*in; + state->stream.avail_in = *in_len; + state->stream.next_out = (unsigned char *)*out; + state->stream.avail_out = *out_len; + + action = finish ? LZMA_FINISH : LZMA_RUN; + + retval = lzma_code(&state->stream, action); + + state->input_so_far += state->stream.next_in - ((unsigned char *)*in); + state->output_so_far += state->stream.next_out - ((unsigned char *)*out); + + *out = (char *)state->stream.next_out; + *out_len = state->stream.avail_out; + *in = (const char *)state->stream.next_in; + *in_len = state->stream.avail_in; + + if (! state->compress && + tor_compress_is_compression_bomb(state->input_so_far, + state->output_so_far)) { + log_warn(LD_DIR, "Possible compression bomb; abandoning stream."); + return TOR_COMPRESS_ERROR; + } + + switch (retval) { + case LZMA_OK: + if (state->stream.avail_out == 0 || finish) + return TOR_COMPRESS_BUFFER_FULL; + + return TOR_COMPRESS_OK; + + case LZMA_BUF_ERROR: + if (state->stream.avail_in == 0 && !finish) + return TOR_COMPRESS_OK; + + return TOR_COMPRESS_BUFFER_FULL; + + case LZMA_STREAM_END: + return TOR_COMPRESS_DONE; + + // We list all the possible values of `lzma_ret` here to silence the + // `switch-enum` warning and to detect if a new member was added. + case LZMA_NO_CHECK: + case LZMA_UNSUPPORTED_CHECK: + case LZMA_GET_CHECK: + case LZMA_MEM_ERROR: + case LZMA_MEMLIMIT_ERROR: + case LZMA_FORMAT_ERROR: + case LZMA_OPTIONS_ERROR: + case LZMA_DATA_ERROR: + case LZMA_PROG_ERROR: + default: + log_warn(LD_GENERAL, "LZMA %s didn't finish: %s.", + state->compress ? "compression" : "decompression", + lzma_error_str(retval)); + return TOR_COMPRESS_ERROR; + } +#else // HAVE_LZMA. + (void)state; + (void)out; + (void)out_len; + (void)in; + (void)in_len; + (void)finish; + return TOR_COMPRESS_ERROR; +#endif // HAVE_LZMA. +} + +/** Deallocate <b>state</b>. */ +void +tor_lzma_compress_free(tor_lzma_compress_state_t *state) +{ + if (state == NULL) + return; + + atomic_counter_sub(&total_lzma_allocation, state->allocation); + +#ifdef HAVE_LZMA + lzma_end(&state->stream); +#endif + + tor_free(state); +} + +/** Return the approximate number of bytes allocated for <b>state</b>. */ +size_t +tor_lzma_compress_state_size(const tor_lzma_compress_state_t *state) +{ + tor_assert(state != NULL); + return state->allocation; +} + +/** Return the approximate number of bytes allocated for all LZMA states. */ +size_t +tor_lzma_get_total_allocation(void) +{ + return atomic_counter_get(&total_lzma_allocation); +} + +/** Initialize the lzma module */ +void +tor_lzma_init(void) +{ + atomic_counter_init(&total_lzma_allocation); +} + diff --git a/src/common/compress_lzma.h b/src/common/compress_lzma.h new file mode 100644 index 0000000000..1433c89f88 --- /dev/null +++ b/src/common/compress_lzma.h @@ -0,0 +1,43 @@ +/* Copyright (c) 2003, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file compress_lzma.h + * \brief Header for compress_lzma.c + **/ + +#ifndef TOR_COMPRESS_LZMA_H +#define TOR_COMPRESS_LZMA_H + +int tor_lzma_method_supported(void); + +const char *tor_lzma_get_version_str(void); + +const char *tor_lzma_get_header_version_str(void); + +/** Internal state for an incremental LZMA compression/decompression. */ +typedef struct tor_lzma_compress_state_t tor_lzma_compress_state_t; + +tor_lzma_compress_state_t * +tor_lzma_compress_new(int compress, + compress_method_t method, + compression_level_t compression_level); + +tor_compress_output_t +tor_lzma_compress_process(tor_lzma_compress_state_t *state, + char **out, size_t *out_len, + const char **in, size_t *in_len, + int finish); + +void tor_lzma_compress_free(tor_lzma_compress_state_t *state); + +size_t tor_lzma_compress_state_size(const tor_lzma_compress_state_t *state); + +size_t tor_lzma_get_total_allocation(void); + +void tor_lzma_init(void); + +#endif // TOR_COMPRESS_LZMA_H. + diff --git a/src/common/compress_none.c b/src/common/compress_none.c new file mode 100644 index 0000000000..34314e4af7 --- /dev/null +++ b/src/common/compress_none.c @@ -0,0 +1,53 @@ +/* Copyright (c) 2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file compress_none.c + * \brief Compression backend for identity compression. + * + * We actually define this backend so that we can treat the identity transform + * as another case of compression. + * + * This module should never be invoked directly. Use the compress module + * instead. + **/ + +#include "orconfig.h" + +#include "util.h" +#include "torlog.h" +#include "compress.h" +#include "compress_none.h" + +/** Transfer some bytes using the identity transformation. Read up to + * *<b>in_len</b> bytes from *<b>in</b>, and write up to *<b>out_len</b> bytes + * to *<b>out</b>, adjusting the values as we go. If <b>finish</b> is true, + * we've reached the end of the input. + * + * Return TOR_COMPRESS_DONE if we've finished the entire + * compression/decompression. + * Return TOR_COMPRESS_OK if we're processed everything from the input. + * Return TOR_COMPRESS_BUFFER_FULL if we're out of space on <b>out</b>. + * Return TOR_COMPRESS_ERROR if the stream is corrupt. + */ +tor_compress_output_t +tor_cnone_compress_process(char **out, size_t *out_len, + const char **in, size_t *in_len, + int finish) +{ + size_t n_to_copy = MIN(*in_len, *out_len); + + memcpy(*out, *in, n_to_copy); + *out += n_to_copy; + *in += n_to_copy; + *out_len -= n_to_copy; + *in_len -= n_to_copy; + if (*in_len == 0) { + return finish ? TOR_COMPRESS_DONE : TOR_COMPRESS_OK; + } else { + return TOR_COMPRESS_BUFFER_FULL; + } +} + diff --git a/src/common/compress_none.h b/src/common/compress_none.h new file mode 100644 index 0000000000..d1ebb4b625 --- /dev/null +++ b/src/common/compress_none.h @@ -0,0 +1,20 @@ +/* Copyright (c) 2003, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file compress_none.h + * \brief Header for compress_none.c + **/ + +#ifndef TOR_COMPRESS_NONE_H +#define TOR_COMPRESS_NONE_H + +tor_compress_output_t +tor_cnone_compress_process(char **out, size_t *out_len, + const char **in, size_t *in_len, + int finish); + +#endif // TOR_COMPRESS_NONE_H. + diff --git a/src/common/compress_zlib.c b/src/common/compress_zlib.c new file mode 100644 index 0000000000..284542e885 --- /dev/null +++ b/src/common/compress_zlib.c @@ -0,0 +1,304 @@ +/* Copyright (c) 2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file compress_zlib.c + * \brief Compression backend for gzip and zlib. + * + * This module should never be invoked directly. Use the compress module + * instead. + **/ + +#include "orconfig.h" + +#include "util.h" +#include "torlog.h" +#include "compress.h" +#include "compress_zlib.h" + +/* zlib 1.2.4 and 1.2.5 do some "clever" things with macros. Instead of + saying "(defined(FOO) ? FOO : 0)" they like to say "FOO-0", on the theory + that nobody will care if the compile outputs a no-such-identifier warning. + + Sorry, but we like -Werror over here, so I guess we need to define these. + I hope that zlib 1.2.6 doesn't break these too. +*/ +#ifndef _LARGEFILE64_SOURCE +#define _LARGEFILE64_SOURCE 0 +#endif +#ifndef _LFS64_LARGEFILE +#define _LFS64_LARGEFILE 0 +#endif +#ifndef _FILE_OFFSET_BITS +#define _FILE_OFFSET_BITS 0 +#endif +#ifndef off64_t +#define off64_t int64_t +#endif + +#include <zlib.h> + +#if defined ZLIB_VERNUM && ZLIB_VERNUM < 0x1200 +#error "We require zlib version 1.2 or later." +#endif + +static size_t tor_zlib_state_size_precalc(int inflate, + int windowbits, int memlevel); + +/** Total number of bytes allocated for zlib state */ +static atomic_counter_t total_zlib_allocation; + +/** Given <b>level</b> return the memory level. */ +static int +memory_level(compression_level_t level) +{ + switch (level) { + default: + case BEST_COMPRESSION: return 9; + case HIGH_COMPRESSION: return 8; + case MEDIUM_COMPRESSION: return 7; + case LOW_COMPRESSION: return 6; + } +} + +/** Return the 'bits' value to tell zlib to use <b>method</b>.*/ +static inline int +method_bits(compress_method_t method, compression_level_t level) +{ + /* Bits+16 means "use gzip" in zlib >= 1.2 */ + const int flag = method == GZIP_METHOD ? 16 : 0; + switch (level) { + default: + case BEST_COMPRESSION: + case HIGH_COMPRESSION: return flag + 15; + case MEDIUM_COMPRESSION: return flag + 13; + case LOW_COMPRESSION: return flag + 11; + } +} + +/** Return 1 if zlib/gzip compression is supported; otherwise 0. */ +int +tor_zlib_method_supported(void) +{ + /* We currently always support zlib/gzip, but we keep this function around in + * case we some day decide to deprecate zlib/gzip support. + */ + return 1; +} + +/** Return a string representation of the version of the currently running + * version of zlib. */ +const char * +tor_zlib_get_version_str(void) +{ + return zlibVersion(); +} + +/** Return a string representation of the version of the version of zlib +* used at compilation. */ +const char * +tor_zlib_get_header_version_str(void) +{ + return ZLIB_VERSION; +} + +/** Internal zlib state for an incremental compression/decompression. + * The body of this struct is not exposed. */ +struct tor_zlib_compress_state_t { + struct z_stream_s stream; /**< The zlib stream */ + int compress; /**< True if we are compressing; false if we are inflating */ + + /** Number of bytes read so far. Used to detect zlib bombs. */ + size_t input_so_far; + /** Number of bytes written so far. Used to detect zlib bombs. */ + size_t output_so_far; + + /** Approximate number of bytes allocated for this object. */ + size_t allocation; +}; + +/** Return an approximate number of bytes used in RAM to hold a state with + * window bits <b>windowBits</b> and compression level 'memlevel' */ +static size_t +tor_zlib_state_size_precalc(int inflate_, int windowbits, int memlevel) +{ + windowbits &= 15; + +#define A_FEW_KILOBYTES 2048 + + if (inflate_) { + /* From zconf.h: + + "The memory requirements for inflate are (in bytes) 1 << windowBits + that is, 32K for windowBits=15 (default value) plus a few kilobytes + for small objects." + */ + return sizeof(tor_zlib_compress_state_t) + sizeof(struct z_stream_s) + + (1 << 15) + A_FEW_KILOBYTES; + } else { + /* Also from zconf.h: + + "The memory requirements for deflate are (in bytes): + (1 << (windowBits+2)) + (1 << (memLevel+9)) + ... plus a few kilobytes for small objects." + */ + return sizeof(tor_zlib_compress_state_t) + sizeof(struct z_stream_s) + + (1 << (windowbits + 2)) + (1 << (memlevel + 9)) + A_FEW_KILOBYTES; + } +#undef A_FEW_KILOBYTES +} + +/** Construct and return a tor_zlib_compress_state_t object using + * <b>method</b>. If <b>compress</b>, it's for compression; otherwise it's for + * decompression. */ +tor_zlib_compress_state_t * +tor_zlib_compress_new(int compress_, + compress_method_t method, + compression_level_t compression_level) +{ + tor_zlib_compress_state_t *out; + int bits, memlevel; + + if (! compress_) { + /* use this setting for decompression, since we might have the + * max number of window bits */ + compression_level = BEST_COMPRESSION; + } + + out = tor_malloc_zero(sizeof(tor_zlib_compress_state_t)); + out->stream.zalloc = Z_NULL; + out->stream.zfree = Z_NULL; + out->stream.opaque = NULL; + out->compress = compress_; + bits = method_bits(method, compression_level); + memlevel = memory_level(compression_level); + if (compress_) { + if (deflateInit2(&out->stream, Z_BEST_COMPRESSION, Z_DEFLATED, + bits, memlevel, + Z_DEFAULT_STRATEGY) != Z_OK) + goto err; // LCOV_EXCL_LINE + } else { + if (inflateInit2(&out->stream, bits) != Z_OK) + goto err; // LCOV_EXCL_LINE + } + out->allocation = tor_zlib_state_size_precalc(!compress_, bits, memlevel); + + atomic_counter_add(&total_zlib_allocation, out->allocation); + + return out; + + err: + tor_free(out); + return NULL; +} + +/** Compress/decompress some bytes using <b>state</b>. Read up to + * *<b>in_len</b> bytes from *<b>in</b>, and write up to *<b>out_len</b> bytes + * to *<b>out</b>, adjusting the values as we go. If <b>finish</b> is true, + * we've reached the end of the input. + * + * Return TOR_COMPRESS_DONE if we've finished the entire + * compression/decompression. + * Return TOR_COMPRESS_OK if we're processed everything from the input. + * Return TOR_COMPRESS_BUFFER_FULL if we're out of space on <b>out</b>. + * Return TOR_COMPRESS_ERROR if the stream is corrupt. + */ +tor_compress_output_t +tor_zlib_compress_process(tor_zlib_compress_state_t *state, + char **out, size_t *out_len, + const char **in, size_t *in_len, + int finish) +{ + int err; + tor_assert(state != NULL); + if (*in_len > UINT_MAX || + *out_len > UINT_MAX) { + return TOR_COMPRESS_ERROR; + } + + state->stream.next_in = (unsigned char*) *in; + state->stream.avail_in = (unsigned int)*in_len; + state->stream.next_out = (unsigned char*) *out; + state->stream.avail_out = (unsigned int)*out_len; + + if (state->compress) { + err = deflate(&state->stream, finish ? Z_FINISH : Z_NO_FLUSH); + } else { + err = inflate(&state->stream, finish ? Z_FINISH : Z_SYNC_FLUSH); + } + + state->input_so_far += state->stream.next_in - ((unsigned char*)*in); + state->output_so_far += state->stream.next_out - ((unsigned char*)*out); + + *out = (char*) state->stream.next_out; + *out_len = state->stream.avail_out; + *in = (const char *) state->stream.next_in; + *in_len = state->stream.avail_in; + + if (! state->compress && + tor_compress_is_compression_bomb(state->input_so_far, + state->output_so_far)) { + log_warn(LD_DIR, "Possible zlib bomb; abandoning stream."); + return TOR_COMPRESS_ERROR; + } + + switch (err) + { + case Z_STREAM_END: + return TOR_COMPRESS_DONE; + case Z_BUF_ERROR: + if (state->stream.avail_in == 0 && !finish) + return TOR_COMPRESS_OK; + return TOR_COMPRESS_BUFFER_FULL; + case Z_OK: + if (state->stream.avail_out == 0 || finish) + return TOR_COMPRESS_BUFFER_FULL; + return TOR_COMPRESS_OK; + default: + log_warn(LD_GENERAL, "Gzip returned an error: %s", + state->stream.msg ? state->stream.msg : "<no message>"); + return TOR_COMPRESS_ERROR; + } +} + +/** Deallocate <b>state</b>. */ +void +tor_zlib_compress_free(tor_zlib_compress_state_t *state) +{ + if (state == NULL) + return; + + atomic_counter_sub(&total_zlib_allocation, state->allocation); + + if (state->compress) + deflateEnd(&state->stream); + else + inflateEnd(&state->stream); + + tor_free(state); +} + +/** Return the approximate number of bytes allocated for <b>state</b>. */ +size_t +tor_zlib_compress_state_size(const tor_zlib_compress_state_t *state) +{ + tor_assert(state != NULL); + return state->allocation; +} + +/** Return the approximate number of bytes allocated for all zlib states. */ +size_t +tor_zlib_get_total_allocation(void) +{ + return atomic_counter_get(&total_zlib_allocation); +} + +/** Set up global state for the zlib module */ +void +tor_zlib_init(void) +{ + atomic_counter_init(&total_zlib_allocation); +} + diff --git a/src/common/compress_zlib.h b/src/common/compress_zlib.h new file mode 100644 index 0000000000..df5c196ac7 --- /dev/null +++ b/src/common/compress_zlib.h @@ -0,0 +1,43 @@ +/* Copyright (c) 2003, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file compress_zlib.h + * \brief Header for compress_zlib.c + **/ + +#ifndef TOR_COMPRESS_ZLIB_H +#define TOR_COMPRESS_ZLIB_H + +int tor_zlib_method_supported(void); + +const char *tor_zlib_get_version_str(void); + +const char *tor_zlib_get_header_version_str(void); + +/** Internal state for an incremental zlib/gzip compression/decompression. */ +typedef struct tor_zlib_compress_state_t tor_zlib_compress_state_t; + +tor_zlib_compress_state_t * +tor_zlib_compress_new(int compress, + compress_method_t method, + compression_level_t compression_level); + +tor_compress_output_t +tor_zlib_compress_process(tor_zlib_compress_state_t *state, + char **out, size_t *out_len, + const char **in, size_t *in_len, + int finish); + +void tor_zlib_compress_free(tor_zlib_compress_state_t *state); + +size_t tor_zlib_compress_state_size(const tor_zlib_compress_state_t *state); + +size_t tor_zlib_get_total_allocation(void); + +void tor_zlib_init(void); + +#endif // TOR_COMPRESS_ZLIB_H. + diff --git a/src/common/compress_zstd.c b/src/common/compress_zstd.c new file mode 100644 index 0000000000..11edfffa0e --- /dev/null +++ b/src/common/compress_zstd.c @@ -0,0 +1,444 @@ +/* Copyright (c) 2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file compress_zstd.c + * \brief Compression backend for Zstandard. + * + * This module should never be invoked directly. Use the compress module + * instead. + **/ + +#include "orconfig.h" + +#include "util.h" +#include "torlog.h" +#include "compress.h" +#include "compress_zstd.h" + +#ifdef HAVE_ZSTD +DISABLE_GCC_WARNING(unused-const-variable) +#include <zstd.h> +ENABLE_GCC_WARNING(unused-const-variable) +#endif + +/** Total number of bytes allocated for Zstandard state. */ +static atomic_counter_t total_zstd_allocation; + +#ifdef HAVE_ZSTD +/** Given <b>level</b> return the memory level. */ +static int +memory_level(compression_level_t level) +{ + switch (level) { + default: + case BEST_COMPRESSION: + case HIGH_COMPRESSION: return 9; + case MEDIUM_COMPRESSION: return 8; + case LOW_COMPRESSION: return 7; + } +} +#endif // HAVE_ZSTD. + +/** Return 1 if Zstandard compression is supported; otherwise 0. */ +int +tor_zstd_method_supported(void) +{ +#ifdef HAVE_ZSTD + return 1; +#else + return 0; +#endif +} + +/** Return a string representation of the version of the currently running + * version of libzstd. Returns NULL if Zstandard is unsupported. */ +const char * +tor_zstd_get_version_str(void) +{ +#ifdef HAVE_ZSTD + static char version_str[16]; + size_t version_number; + + version_number = ZSTD_versionNumber(); + tor_snprintf(version_str, sizeof(version_str), + "%d.%d.%d", + (int) version_number / 10000 % 100, + (int) version_number / 100 % 100, + (int) version_number % 100); + + return version_str; +#else + return NULL; +#endif +} + +/** Return a string representation of the version of the version of libzstd + * used at compilation time. Returns NULL if Zstandard is unsupported. */ +const char * +tor_zstd_get_header_version_str(void) +{ +#ifdef HAVE_ZSTD + return ZSTD_VERSION_STRING; +#else + return NULL; +#endif +} + +/** Internal Zstandard state for incremental compression/decompression. + * The body of this struct is not exposed. */ +struct tor_zstd_compress_state_t { +#ifdef HAVE_ZSTD + union { + /** Compression stream. Used when <b>compress</b> is true. */ + ZSTD_CStream *compress_stream; + /** Decompression stream. Used when <b>compress</b> is false. */ + ZSTD_DStream *decompress_stream; + } u; /**< Zstandard stream objects. */ +#endif // HAVE_ZSTD. + + int compress; /**< True if we are compressing; false if we are inflating */ + int have_called_end; /**< True if we are compressing and we've called + * ZSTD_endStream */ + + /** Number of bytes read so far. Used to detect compression bombs. */ + size_t input_so_far; + /** Number of bytes written so far. Used to detect compression bombs. */ + size_t output_so_far; + + /** Approximate number of bytes allocated for this object. */ + size_t allocation; +}; + +#ifdef HAVE_ZSTD +/** Return an approximate number of bytes stored in memory to hold the + * Zstandard compression/decompression state. */ +static size_t +tor_zstd_state_size_precalc(int compress, int preset) +{ + tor_assert(preset > 0); + + size_t memory_usage = sizeof(tor_zstd_compress_state_t); + + // The Zstandard library provides a number of functions that would be useful + // here, but they are, unfortunately, still considered experimental and are + // thus only available in libzstd if we link against the library statically. + // + // The code in this function tries to approximate the calculations without + // being able to use the following: + // + // - We do not have access to neither the internal members of ZSTD_CStream + // and ZSTD_DStream and their internal context objects. + // + // - We cannot use ZSTD_sizeof_CStream() and ZSTD_sizeof_DStream() since they + // are unexposed. + // + // In the future it might be useful to check if libzstd have started + // providing these functions in a stable manner and simplify this function. + if (compress) { + // We try to approximate the ZSTD_sizeof_CStream(ZSTD_CStream *stream) + // function here. This function uses the following fields to make its + // estimate: + + // - sizeof(ZSTD_CStream): Around 192 bytes on a 64-bit machine: + memory_usage += 192; + + // - ZSTD_sizeof_CCtx(stream->cctx): This function requires access to + // variables that are not exposed via the public API. We use a _very_ + // simplified function to calculate the estimated amount of bytes used in + // this struct. + // memory_usage += (preset - 0.5) * 1024 * 1024; + memory_usage += (preset * 1024 * 1024) - (512 * 1024); + // - ZSTD_sizeof_CDict(stream->cdictLocal): Unused in Tor: 0 bytes. + // - stream->outBuffSize: 128 KB: + memory_usage += 128 * 1024; + // - stream->inBuffSize: 2048 KB: + memory_usage += 2048 * 1024; + } else { + // We try to approximate the ZSTD_sizeof_DStream(ZSTD_DStream *stream) + // function here. This function uses the following fields to make its + // estimate: + + // - sizeof(ZSTD_DStream): Around 208 bytes on a 64-bit machine: + memory_usage += 208; + // - ZSTD_sizeof_DCtx(stream->dctx): Around 150 KB. + memory_usage += 150 * 1024; + + // - ZSTD_sizeof_DDict(stream->ddictLocal): Unused in Tor: 0 bytes. + // - stream->inBuffSize: 0 KB. + // - stream->outBuffSize: 0 KB. + } + + return memory_usage; +} +#endif // HAVE_ZSTD. + +/** Construct and return a tor_zstd_compress_state_t object using + * <b>method</b>. If <b>compress</b>, it's for compression; otherwise it's for + * decompression. */ +tor_zstd_compress_state_t * +tor_zstd_compress_new(int compress, + compress_method_t method, + compression_level_t level) +{ + tor_assert(method == ZSTD_METHOD); + +#ifdef HAVE_ZSTD + const int preset = memory_level(level); + tor_zstd_compress_state_t *result; + size_t retval; + + result = tor_malloc_zero(sizeof(tor_zstd_compress_state_t)); + result->compress = compress; + result->allocation = tor_zstd_state_size_precalc(compress, preset); + + if (compress) { + result->u.compress_stream = ZSTD_createCStream(); + + if (result->u.compress_stream == NULL) { + // LCOV_EXCL_START + log_warn(LD_GENERAL, "Error while creating Zstandard compression " + "stream"); + goto err; + // LCOV_EXCL_STOP + } + + retval = ZSTD_initCStream(result->u.compress_stream, preset); + + if (ZSTD_isError(retval)) { + // LCOV_EXCL_START + log_warn(LD_GENERAL, "Zstandard stream initialization error: %s", + ZSTD_getErrorName(retval)); + goto err; + // LCOV_EXCL_STOP + } + } else { + result->u.decompress_stream = ZSTD_createDStream(); + + if (result->u.decompress_stream == NULL) { + // LCOV_EXCL_START + log_warn(LD_GENERAL, "Error while creating Zstandard decompression " + "stream"); + goto err; + // LCOV_EXCL_STOP + } + + retval = ZSTD_initDStream(result->u.decompress_stream); + + if (ZSTD_isError(retval)) { + // LCOV_EXCL_START + log_warn(LD_GENERAL, "Zstandard stream initialization error: %s", + ZSTD_getErrorName(retval)); + goto err; + // LCOV_EXCL_STOP + } + } + + atomic_counter_add(&total_zstd_allocation, result->allocation); + return result; + + err: + // LCOV_EXCL_START + if (compress) { + ZSTD_freeCStream(result->u.compress_stream); + } else { + ZSTD_freeDStream(result->u.decompress_stream); + } + + tor_free(result); + return NULL; + // LCOV_EXCL_STOP +#else // HAVE_ZSTD. + (void)compress; + (void)method; + (void)level; + + return NULL; +#endif // HAVE_ZSTD. +} + +/** Compress/decompress some bytes using <b>state</b>. Read up to + * *<b>in_len</b> bytes from *<b>in</b>, and write up to *<b>out_len</b> bytes + * to *<b>out</b>, adjusting the values as we go. If <b>finish</b> is true, + * we've reached the end of the input. + * + * Return TOR_COMPRESS_DONE if we've finished the entire + * compression/decompression. + * Return TOR_COMPRESS_OK if we're processed everything from the input. + * Return TOR_COMPRESS_BUFFER_FULL if we're out of space on <b>out</b>. + * Return TOR_COMPRESS_ERROR if the stream is corrupt. + */ +tor_compress_output_t +tor_zstd_compress_process(tor_zstd_compress_state_t *state, + char **out, size_t *out_len, + const char **in, size_t *in_len, + int finish) +{ +#ifdef HAVE_ZSTD + size_t retval; + + tor_assert(state != NULL); + tor_assert(*in_len <= UINT_MAX); + tor_assert(*out_len <= UINT_MAX); + + ZSTD_inBuffer input = { *in, *in_len, 0 }; + ZSTD_outBuffer output = { *out, *out_len, 0 }; + + if (BUG(finish == 0 && state->have_called_end)) { + finish = 1; + } + + if (state->compress) { + if (! state->have_called_end) + retval = ZSTD_compressStream(state->u.compress_stream, + &output, &input); + else + retval = 0; + } else { + retval = ZSTD_decompressStream(state->u.decompress_stream, + &output, &input); + } + + state->input_so_far += input.pos; + state->output_so_far += output.pos; + + *out = (char *)output.dst + output.pos; + *out_len = output.size - output.pos; + *in = (char *)input.src + input.pos; + *in_len = input.size - input.pos; + + if (! state->compress && + tor_compress_is_compression_bomb(state->input_so_far, + state->output_so_far)) { + log_warn(LD_DIR, "Possible compression bomb; abandoning stream."); + return TOR_COMPRESS_ERROR; + } + + if (ZSTD_isError(retval)) { + log_warn(LD_GENERAL, "Zstandard %s didn't finish: %s.", + state->compress ? "compression" : "decompression", + ZSTD_getErrorName(retval)); + return TOR_COMPRESS_ERROR; + } + + if (state->compress && !state->have_called_end) { + retval = ZSTD_flushStream(state->u.compress_stream, &output); + + *out = (char *)output.dst + output.pos; + *out_len = output.size - output.pos; + + if (ZSTD_isError(retval)) { + log_warn(LD_GENERAL, "Zstandard compression unable to flush: %s.", + ZSTD_getErrorName(retval)); + return TOR_COMPRESS_ERROR; + } + + // ZSTD_flushStream returns 0 if the frame is done, or >0 if it + // is incomplete. + if (retval > 0) { + return TOR_COMPRESS_BUFFER_FULL; + } + } + + if (!finish) { + // The caller says we're not done with the input, so no need to write an + // epilogue. + return TOR_COMPRESS_OK; + } else if (state->compress) { + if (*in_len) { + // We say that we're not done with the input, so we can't write an + // epilogue. + return TOR_COMPRESS_OK; + } + + retval = ZSTD_endStream(state->u.compress_stream, &output); + state->have_called_end = 1; + *out = (char *)output.dst + output.pos; + *out_len = output.size - output.pos; + + if (ZSTD_isError(retval)) { + log_warn(LD_GENERAL, "Zstandard compression unable to write " + "epilogue: %s.", + ZSTD_getErrorName(retval)); + return TOR_COMPRESS_ERROR; + } + + // endStream returns the number of bytes that is needed to write the + // epilogue. + if (retval > 0) + return TOR_COMPRESS_BUFFER_FULL; + + return TOR_COMPRESS_DONE; + } else /* if (!state->compress) */ { + // ZSTD_decompressStream returns 0 if the frame is done, or >0 if it + // is incomplete. + // We check this above. + tor_assert_nonfatal(!ZSTD_isError(retval)); + // Start a new frame if this frame is done + if (retval == 0) + return TOR_COMPRESS_DONE; + // Don't check out_len, it might have some space left if the next output + // chunk is larger than the remaining space + else if (*in_len > 0) + return TOR_COMPRESS_BUFFER_FULL; + else + return TOR_COMPRESS_OK; + } + +#else // HAVE_ZSTD. + (void)state; + (void)out; + (void)out_len; + (void)in; + (void)in_len; + (void)finish; + + return TOR_COMPRESS_ERROR; +#endif // HAVE_ZSTD. +} + +/** Deallocate <b>state</b>. */ +void +tor_zstd_compress_free(tor_zstd_compress_state_t *state) +{ + if (state == NULL) + return; + + atomic_counter_sub(&total_zstd_allocation, state->allocation); + +#ifdef HAVE_ZSTD + if (state->compress) { + ZSTD_freeCStream(state->u.compress_stream); + } else { + ZSTD_freeDStream(state->u.decompress_stream); + } +#endif // HAVE_ZSTD. + + tor_free(state); +} + +/** Return the approximate number of bytes allocated for <b>state</b>. */ +size_t +tor_zstd_compress_state_size(const tor_zstd_compress_state_t *state) +{ + tor_assert(state != NULL); + return state->allocation; +} + +/** Return the approximate number of bytes allocated for all Zstandard + * states. */ +size_t +tor_zstd_get_total_allocation(void) +{ + return atomic_counter_get(&total_zstd_allocation); +} + +/** Initialize the zstd module */ +void +tor_zstd_init(void) +{ + atomic_counter_init(&total_zstd_allocation); +} + diff --git a/src/common/compress_zstd.h b/src/common/compress_zstd.h new file mode 100644 index 0000000000..d3e65c2f16 --- /dev/null +++ b/src/common/compress_zstd.h @@ -0,0 +1,43 @@ +/* Copyright (c) 2003, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file compress_zstd.h + * \brief Header for compress_zstd.c + **/ + +#ifndef TOR_COMPRESS_ZSTD_H +#define TOR_COMPRESS_ZSTD_H + +int tor_zstd_method_supported(void); + +const char *tor_zstd_get_version_str(void); + +const char *tor_zstd_get_header_version_str(void); + +/** Internal state for an incremental Zstandard compression/decompression. */ +typedef struct tor_zstd_compress_state_t tor_zstd_compress_state_t; + +tor_zstd_compress_state_t * +tor_zstd_compress_new(int compress, + compress_method_t method, + compression_level_t compression_level); + +tor_compress_output_t +tor_zstd_compress_process(tor_zstd_compress_state_t *state, + char **out, size_t *out_len, + const char **in, size_t *in_len, + int finish); + +void tor_zstd_compress_free(tor_zstd_compress_state_t *state); + +size_t tor_zstd_compress_state_size(const tor_zstd_compress_state_t *state); + +size_t tor_zstd_get_total_allocation(void); + +void tor_zstd_init(void); + +#endif // TOR_COMPRESS_ZSTD_H. + diff --git a/src/common/confline.c b/src/common/confline.c new file mode 100644 index 0000000000..15fd96bf38 --- /dev/null +++ b/src/common/confline.c @@ -0,0 +1,527 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "compat.h" +#include "confline.h" +#include "torlog.h" +#include "util.h" +#include "container.h" + +static int config_get_lines_aux(const char *string, config_line_t **result, + int extended, int allow_include, + int *has_include, int recursion_level, + config_line_t **last); +static smartlist_t *config_get_file_list(const char *path); +static int config_get_included_list(const char *path, int recursion_level, + int extended, config_line_t **list, + config_line_t **list_last); +static int config_process_include(const char *path, int recursion_level, + int extended, config_line_t **list, + config_line_t **list_last); + +/** Helper: allocate a new configuration option mapping 'key' to 'val', + * append it to *<b>lst</b>. */ +void +config_line_append(config_line_t **lst, + const char *key, + const char *val) +{ + tor_assert(lst); + + config_line_t *newline; + + newline = tor_malloc_zero(sizeof(config_line_t)); + newline->key = tor_strdup(key); + newline->value = tor_strdup(val); + newline->next = NULL; + while (*lst) + lst = &((*lst)->next); + + (*lst) = newline; +} + +/** Helper: allocate a new configuration option mapping 'key' to 'val', + * and prepend it to *<b>lst</b> */ +void +config_line_prepend(config_line_t **lst, + const char *key, + const char *val) +{ + tor_assert(lst); + + config_line_t *newline; + + newline = tor_malloc_zero(sizeof(config_line_t)); + newline->key = tor_strdup(key); + newline->value = tor_strdup(val); + newline->next = *lst; + *lst = newline; +} + +/** Return the first line in <b>lines</b> whose key is exactly <b>key</b>, or + * NULL if no such key exists. + * + * (In options parsing, this is for handling commandline-only options only; + * other options should be looked up in the appropriate data structure.) */ +const config_line_t * +config_line_find(const config_line_t *lines, + const char *key) +{ + const config_line_t *cl; + for (cl = lines; cl; cl = cl->next) { + if (!strcmp(cl->key, key)) + return cl; + } + return NULL; +} + +/** Auxiliary function that does all the work of config_get_lines. + * <b>recursion_level</b> is the count of how many nested %includes we have. + * Returns the a pointer to the last element of the <b>result</b> in + * <b>last</b>. */ +static int +config_get_lines_aux(const char *string, config_line_t **result, int extended, + int allow_include, int *has_include, int recursion_level, + config_line_t **last) +{ + config_line_t *list = NULL, **next, *list_last = NULL; + char *k, *v; + const char *parse_err; + int include_used = 0; + + if (recursion_level > MAX_INCLUDE_RECURSION_LEVEL) { + log_warn(LD_CONFIG, "Error while parsing configuration: more than %d " + "nested %%includes.", MAX_INCLUDE_RECURSION_LEVEL); + return -1; + } + + next = &list; + do { + k = v = NULL; + string = parse_config_line_from_str_verbose(string, &k, &v, &parse_err); + if (!string) { + log_warn(LD_CONFIG, "Error while parsing configuration: %s", + parse_err?parse_err:"<unknown>"); + config_free_lines(list); + tor_free(k); + tor_free(v); + return -1; + } + if (k && v) { + unsigned command = CONFIG_LINE_NORMAL; + if (extended) { + if (k[0] == '+') { + char *k_new = tor_strdup(k+1); + tor_free(k); + k = k_new; + command = CONFIG_LINE_APPEND; + } else if (k[0] == '/') { + char *k_new = tor_strdup(k+1); + tor_free(k); + k = k_new; + tor_free(v); + v = tor_strdup(""); + command = CONFIG_LINE_CLEAR; + } + } + + if (allow_include && !strcmp(k, "%include")) { + tor_free(k); + include_used = 1; + + config_line_t *include_list; + if (config_process_include(v, recursion_level, extended, &include_list, + &list_last) < 0) { + log_warn(LD_CONFIG, "Error reading included configuration " + "file or directory: \"%s\".", v); + config_free_lines(list); + tor_free(v); + return -1; + } + *next = include_list; + if (list_last) + next = &list_last->next; + tor_free(v); + } else { + /* This list can get long, so we keep a pointer to the end of it + * rather than using config_line_append over and over and getting + * n^2 performance. */ + *next = tor_malloc_zero(sizeof(**next)); + (*next)->key = k; + (*next)->value = v; + (*next)->next = NULL; + (*next)->command = command; + list_last = *next; + next = &((*next)->next); + } + } else { + tor_free(k); + tor_free(v); + } + } while (*string); + + if (last) { + *last = list_last; + } + if (has_include) { + *has_include = include_used; + } + *result = list; + return 0; +} + +/** Helper: parse the config string and strdup into key/value + * strings. Set *result to the list, or NULL if parsing the string + * failed. Set *has_include to 1 if <b>result</b> has values from + * %included files. Return 0 on success, -1 on failure. Warn and ignore any + * misformatted lines. + * + * If <b>extended</b> is set, then treat keys beginning with / and with + as + * indicating "clear" and "append" respectively. */ +int +config_get_lines_include(const char *string, config_line_t **result, + int extended, int *has_include) +{ + return config_get_lines_aux(string, result, extended, 1, has_include, 1, + NULL); +} + +/** Same as config_get_lines_include but does not allow %include */ +int +config_get_lines(const char *string, config_line_t **result, int extended) +{ + return config_get_lines_aux(string, result, extended, 0, NULL, 1, NULL); +} + +/** Adds a list of configuration files present on <b>path</b> to + * <b>file_list</b>. <b>path</b> can be a file or a directory. If it is a file, + * only that file will be added to <b>file_list</b>. If it is a directory, + * all paths for files on that directory root (no recursion) except for files + * whose name starts with a dot will be added to <b>file_list</b>. + * Return 0 on success, -1 on failure. Ignores empty files. + */ +static smartlist_t * +config_get_file_list(const char *path) +{ + smartlist_t *file_list = smartlist_new(); + file_status_t file_type = file_status(path); + if (file_type == FN_FILE) { + smartlist_add_strdup(file_list, path); + return file_list; + } else if (file_type == FN_DIR) { + smartlist_t *all_files = tor_listdir(path); + if (!all_files) { + smartlist_free(file_list); + return NULL; + } + smartlist_sort_strings(all_files); + SMARTLIST_FOREACH_BEGIN(all_files, char *, f) { + if (f[0] == '.') { + tor_free(f); + continue; + } + + char *fullname; + tor_asprintf(&fullname, "%s"PATH_SEPARATOR"%s", path, f); + tor_free(f); + + if (file_status(fullname) != FN_FILE) { + tor_free(fullname); + continue; + } + smartlist_add(file_list, fullname); + } SMARTLIST_FOREACH_END(f); + smartlist_free(all_files); + return file_list; + } else if (file_type == FN_EMPTY) { + return file_list; + } else { + smartlist_free(file_list); + return NULL; + } +} + +/** Creates a list of config lines present on included <b>path</b>. + * Set <b>list</b> to the list and <b>list_last</b> to the last element of + * <b>list</b>. Return 0 on success, -1 on failure. */ +static int +config_get_included_list(const char *path, int recursion_level, int extended, + config_line_t **list, config_line_t **list_last) +{ + char *included_conf = read_file_to_str(path, 0, NULL); + if (!included_conf) { + return -1; + } + + if (config_get_lines_aux(included_conf, list, extended, 1, NULL, + recursion_level+1, list_last) < 0) { + tor_free(included_conf); + return -1; + } + + tor_free(included_conf); + return 0; +} + +/** Process an %include <b>path</b> in a config file. Set <b>list</b> to the + * list of configuration settings obtained and <b>list_last</b> to the last + * element of the same list. Return 0 on success, -1 on failure. */ +static int +config_process_include(const char *path, int recursion_level, int extended, + config_line_t **list, config_line_t **list_last) +{ + config_line_t *ret_list = NULL; + config_line_t **next = &ret_list; +#if 0 + // Disabled -- we already unescape_string() on the result. */ + char *unquoted_path = get_unquoted_path(path); + if (!unquoted_path) { + return -1; + } + + smartlist_t *config_files = config_get_file_list(unquoted_path); + if (!config_files) { + tor_free(unquoted_path); + return -1; + } + tor_free(unquoted_path); +#endif + smartlist_t *config_files = config_get_file_list(path); + if (!config_files) { + return -1; + } + + int rv = -1; + SMARTLIST_FOREACH_BEGIN(config_files, const char *, config_file) { + config_line_t *included_list = NULL; + if (config_get_included_list(config_file, recursion_level, extended, + &included_list, list_last) < 0) { + goto done; + } + + *next = included_list; + if (*list_last) + next = &(*list_last)->next; + + } SMARTLIST_FOREACH_END(config_file); + *list = ret_list; + rv = 0; + + done: + SMARTLIST_FOREACH(config_files, char *, f, tor_free(f)); + smartlist_free(config_files); + return rv; +} + +/** + * Free all the configuration lines on the linked list <b>front</b>. + */ +void +config_free_lines(config_line_t *front) +{ + config_line_t *tmp; + + while (front) { + tmp = front; + front = tmp->next; + + tor_free(tmp->key); + tor_free(tmp->value); + tor_free(tmp); + } +} + +/** Return a newly allocated deep copy of the lines in <b>inp</b>. */ +config_line_t * +config_lines_dup(const config_line_t *inp) +{ + return config_lines_dup_and_filter(inp, NULL); +} + +/** Return a newly allocated deep copy of the lines in <b>inp</b>, + * but only the ones that match <b>key</b>. */ +config_line_t * +config_lines_dup_and_filter(const config_line_t *inp, + const char *key) +{ + config_line_t *result = NULL; + config_line_t **next_out = &result; + while (inp) { + if (key && strcasecmpstart(inp->key, key)) { + inp = inp->next; + continue; + } + *next_out = tor_malloc_zero(sizeof(config_line_t)); + (*next_out)->key = tor_strdup(inp->key); + (*next_out)->value = tor_strdup(inp->value); + inp = inp->next; + next_out = &((*next_out)->next); + } + (*next_out) = NULL; + return result; +} + +/** Return true iff a and b contain identical keys and values in identical + * order. */ +int +config_lines_eq(config_line_t *a, config_line_t *b) +{ + while (a && b) { + if (strcasecmp(a->key, b->key) || strcmp(a->value, b->value)) + return 0; + a = a->next; + b = b->next; + } + if (a || b) + return 0; + return 1; +} + +/** Return the number of lines in <b>a</b> whose key is <b>key</b>. */ +int +config_count_key(const config_line_t *a, const char *key) +{ + int n = 0; + while (a) { + if (!strcasecmp(a->key, key)) { + ++n; + } + a = a->next; + } + return n; +} + +/** Given a string containing part of a configuration file or similar format, + * advance past comments and whitespace and try to parse a single line. If we + * parse a line successfully, set *<b>key_out</b> to a new string holding the + * key portion and *<b>value_out</b> to a new string holding the value portion + * of the line, and return a pointer to the start of the next line. If we run + * out of data, return a pointer to the end of the string. If we encounter an + * error, return NULL and set *<b>err_out</b> (if provided) to an error + * message. + */ +const char * +parse_config_line_from_str_verbose(const char *line, char **key_out, + char **value_out, + const char **err_out) +{ + /* + See torrc_format.txt for a description of the (silly) format this parses. + */ + const char *key, *val, *cp; + int continuation = 0; + + tor_assert(key_out); + tor_assert(value_out); + + *key_out = *value_out = NULL; + key = val = NULL; + /* Skip until the first keyword. */ + while (1) { + while (TOR_ISSPACE(*line)) + ++line; + if (*line == '#') { + while (*line && *line != '\n') + ++line; + } else { + break; + } + } + + if (!*line) { /* End of string? */ + *key_out = *value_out = NULL; + return line; + } + + /* Skip until the next space or \ followed by newline. */ + key = line; + while (*line && !TOR_ISSPACE(*line) && *line != '#' && + ! (line[0] == '\\' && line[1] == '\n')) + ++line; + *key_out = tor_strndup(key, line-key); + + /* Skip until the value. */ + while (*line == ' ' || *line == '\t') + ++line; + + val = line; + + /* Find the end of the line. */ + if (*line == '\"') { // XXX No continuation handling is done here + if (!(line = unescape_string(line, value_out, NULL))) { + if (err_out) + *err_out = "Invalid escape sequence in quoted string"; + return NULL; + } + while (*line == ' ' || *line == '\t') + ++line; + if (*line == '\r' && *(++line) == '\n') + ++line; + if (*line && *line != '#' && *line != '\n') { + if (err_out) + *err_out = "Excess data after quoted string"; + return NULL; + } + } else { + /* Look for the end of the line. */ + while (*line && *line != '\n' && (*line != '#' || continuation)) { + if (*line == '\\' && line[1] == '\n') { + continuation = 1; + line += 2; + } else if (*line == '#') { + do { + ++line; + } while (*line && *line != '\n'); + if (*line == '\n') + ++line; + } else { + ++line; + } + } + + if (*line == '\n') { + cp = line++; + } else { + cp = line; + } + /* Now back cp up to be the last nonspace character */ + while (cp>val && TOR_ISSPACE(*(cp-1))) + --cp; + + tor_assert(cp >= val); + + /* Now copy out and decode the value. */ + *value_out = tor_strndup(val, cp-val); + if (continuation) { + char *v_out, *v_in; + v_out = v_in = *value_out; + while (*v_in) { + if (*v_in == '#') { + do { + ++v_in; + } while (*v_in && *v_in != '\n'); + if (*v_in == '\n') + ++v_in; + } else if (v_in[0] == '\\' && v_in[1] == '\n') { + v_in += 2; + } else { + *v_out++ = *v_in++; + } + } + *v_out = '\0'; + } + } + + if (*line == '#') { + do { + ++line; + } while (*line && *line != '\n'); + } + while (TOR_ISSPACE(*line)) ++line; + + return line; +} + diff --git a/src/common/confline.h b/src/common/confline.h new file mode 100644 index 0000000000..eb863e8fd8 --- /dev/null +++ b/src/common/confline.h @@ -0,0 +1,53 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef TOR_CONFLINE_H +#define TOR_CONFLINE_H + +/** Ordinary configuration line. */ +#define CONFIG_LINE_NORMAL 0 +/** Appends to previous configuration for the same option, even if we + * would ordinary replace it. */ +#define CONFIG_LINE_APPEND 1 +/* Removes all previous configuration for an option. */ +#define CONFIG_LINE_CLEAR 2 + +#define MAX_INCLUDE_RECURSION_LEVEL 31 + +/** A linked list of lines in a config file, or elsewhere */ +typedef struct config_line_t { + char *key; + char *value; + struct config_line_t *next; + + /** What special treatment (if any) does this line require? */ + unsigned int command:2; + /** If true, subsequent assignments to this linelist should replace + * it, not extend it. Set only on the first item in a linelist in an + * or_options_t. */ + unsigned int fragile:1; +} config_line_t; + +void config_line_append(config_line_t **lst, + const char *key, const char *val); +void config_line_prepend(config_line_t **lst, + const char *key, const char *val); +config_line_t *config_lines_dup(const config_line_t *inp); +config_line_t *config_lines_dup_and_filter(const config_line_t *inp, + const char *key); +const config_line_t *config_line_find(const config_line_t *lines, + const char *key); +int config_lines_eq(config_line_t *a, config_line_t *b); +int config_count_key(const config_line_t *a, const char *key); +int config_get_lines(const char *string, config_line_t **result, int extended); +int config_get_lines_include(const char *string, config_line_t **result, + int extended, int *has_include); +void config_free_lines(config_line_t *front); +const char *parse_config_line_from_str_verbose(const char *line, + char **key_out, char **value_out, + const char **err_out); +#endif + diff --git a/src/common/container.c b/src/common/container.c index ec59dccf62..689e7e55e9 100644 --- a/src/common/container.c +++ b/src/common/container.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -132,6 +132,24 @@ smartlist_remove(smartlist_t *sl, const void *element) } } +/** As <b>smartlist_remove</b>, but do not change the order of + * any elements not removed */ +void +smartlist_remove_keeporder(smartlist_t *sl, const void *element) +{ + int i, j, num_used_orig = sl->num_used; + if (element == NULL) + return; + + for (i=j=0; j < num_used_orig; ++j) { + if (sl->list[j] == element) { + --sl->num_used; + } else { + sl->list[i++] = sl->list[j]; + } + } +} + /** If <b>sl</b> is nonempty, remove and return the final element. Otherwise, * return NULL. */ void * diff --git a/src/common/container.h b/src/common/container.h index 71495b660a..db68d892f5 100644 --- a/src/common/container.h +++ b/src/common/container.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_CONTAINER_H @@ -33,6 +33,7 @@ void smartlist_clear(smartlist_t *sl); void smartlist_add(smartlist_t *sl, void *element); void smartlist_add_all(smartlist_t *sl, const smartlist_t *s2); void smartlist_remove(smartlist_t *sl, const void *element); +void smartlist_remove_keeporder(smartlist_t *sl, const void *element); void *smartlist_pop_last(smartlist_t *sl); void smartlist_reverse(smartlist_t *sl); void smartlist_string_remove(smartlist_t *sl, const char *element); @@ -223,6 +224,7 @@ char *smartlist_join_strings2(smartlist_t *sl, const char *join, #define SMARTLIST_FOREACH_END(var) \ var = NULL; \ + (void) var ## _sl_idx; \ } STMT_END /** diff --git a/src/common/crypto.c b/src/common/crypto.c index f8495bb107..d2801e0d45 100644 --- a/src/common/crypto.c +++ b/src/common/crypto.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -467,7 +467,7 @@ crypto_new_pk_from_rsa_(RSA *rsa) return env; } -/** Helper, used by tor-checkkey.c and tor-gencert.c. Return the RSA from a +/** Helper, used by tor-gencert.c. Return the RSA from a * crypto_pk_t. */ RSA * crypto_pk_get_rsa_(crypto_pk_t *env) @@ -479,7 +479,7 @@ crypto_pk_get_rsa_(crypto_pk_t *env) * private is set, include the private-key portion of the key. Return a valid * pointer on success, and NULL on failure. */ MOCK_IMPL(EVP_PKEY *, - crypto_pk_get_evp_pkey_,(crypto_pk_t *env, int private)) +crypto_pk_get_evp_pkey_,(crypto_pk_t *env, int private)) { RSA *key = NULL; EVP_PKEY *pkey = NULL; @@ -516,7 +516,7 @@ crypto_dh_get_dh_(crypto_dh_t *dh) * be set. */ MOCK_IMPL(crypto_pk_t *, - crypto_pk_new,(void)) +crypto_pk_new,(void)) { RSA *rsa; @@ -606,7 +606,7 @@ crypto_cipher_free(crypto_cipher_t *env) * Return 0 on success, -1 on failure. */ MOCK_IMPL(int, - crypto_pk_generate_key_with_bits,(crypto_pk_t *env, int bits)) +crypto_pk_generate_key_with_bits,(crypto_pk_t *env, int bits)) { tor_assert(env); @@ -1122,10 +1122,10 @@ crypto_pk_private_decrypt(crypto_pk_t *env, char *to, * <b>tolen</b> is the number of writable bytes in <b>to</b>, and must be * at least the length of the modulus of <b>env</b>. */ -int -crypto_pk_public_checksig(const crypto_pk_t *env, char *to, - size_t tolen, - const char *from, size_t fromlen) +MOCK_IMPL(int, +crypto_pk_public_checksig,(const crypto_pk_t *env, char *to, + size_t tolen, + const char *from, size_t fromlen)) { int r; tor_assert(env); @@ -1149,9 +1149,10 @@ crypto_pk_public_checksig(const crypto_pk_t *env, char *to, * in <b>env</b>. Return 0 if <b>sig</b> is a correct signature for * SHA1(data). Else return -1. */ -int -crypto_pk_public_checksig_digest(crypto_pk_t *env, const char *data, - size_t datalen, const char *sig, size_t siglen) +MOCK_IMPL(int, +crypto_pk_public_checksig_digest,(crypto_pk_t *env, const char *data, + size_t datalen, const char *sig, + size_t siglen)) { char digest[DIGEST_LEN]; char *buf; @@ -1521,7 +1522,7 @@ crypto_pk_get_hashed_fingerprint(crypto_pk_t *pk, char *fp_out) if (crypto_pk_get_digest(pk, digest)) { return -1; } - if (crypto_digest(hashed_digest, digest, DIGEST_LEN)) { + if (crypto_digest(hashed_digest, digest, DIGEST_LEN) < 0) { return -1; } base16_encode(fp_out, FINGERPRINT_LEN + 1, hashed_digest, DIGEST_LEN); @@ -1715,19 +1716,21 @@ crypto_cipher_decrypt_with_iv(const char *key, /** Compute the SHA1 digest of the <b>len</b> bytes on data stored in * <b>m</b>. Write the DIGEST_LEN byte result into <b>digest</b>. - * Return 0 on success, 1 on failure. + * Return 0 on success, -1 on failure. */ int crypto_digest(char *digest, const char *m, size_t len) { tor_assert(m); tor_assert(digest); - return (SHA1((const unsigned char*)m,len,(unsigned char*)digest) == NULL); + if (SHA1((const unsigned char*)m,len,(unsigned char*)digest) == NULL) + return -1; + return 0; } /** Compute a 256-bit digest of <b>len</b> bytes in data stored in <b>m</b>, * using the algorithm <b>algorithm</b>. Write the DIGEST_LEN256-byte result - * into <b>digest</b>. Return 0 on success, 1 on failure. */ + * into <b>digest</b>. Return 0 on success, -1 on failure. */ int crypto_digest256(char *digest, const char *m, size_t len, digest_algorithm_t algorithm) @@ -1735,16 +1738,22 @@ crypto_digest256(char *digest, const char *m, size_t len, tor_assert(m); tor_assert(digest); tor_assert(algorithm == DIGEST_SHA256 || algorithm == DIGEST_SHA3_256); + + int ret = 0; if (algorithm == DIGEST_SHA256) - return (SHA256((const uint8_t*)m,len,(uint8_t*)digest) == NULL); + ret = (SHA256((const uint8_t*)m,len,(uint8_t*)digest) != NULL); else - return (sha3_256((uint8_t *)digest, DIGEST256_LEN,(const uint8_t *)m, len) - == -1); + ret = (sha3_256((uint8_t *)digest, DIGEST256_LEN,(const uint8_t *)m, len) + > -1); + + if (!ret) + return -1; + return 0; } /** Compute a 512-bit digest of <b>len</b> bytes in data stored in <b>m</b>, * using the algorithm <b>algorithm</b>. Write the DIGEST_LEN512-byte result - * into <b>digest</b>. Return 0 on success, 1 on failure. */ + * into <b>digest</b>. Return 0 on success, -1 on failure. */ int crypto_digest512(char *digest, const char *m, size_t len, digest_algorithm_t algorithm) @@ -1752,12 +1761,18 @@ crypto_digest512(char *digest, const char *m, size_t len, tor_assert(m); tor_assert(digest); tor_assert(algorithm == DIGEST_SHA512 || algorithm == DIGEST_SHA3_512); + + int ret = 0; if (algorithm == DIGEST_SHA512) - return (SHA512((const unsigned char*)m,len,(unsigned char*)digest) - == NULL); + ret = (SHA512((const unsigned char*)m,len,(unsigned char*)digest) + != NULL); else - return (sha3_512((uint8_t*)digest, DIGEST512_LEN, (const uint8_t*)m, len) - == -1); + ret = (sha3_512((uint8_t*)digest, DIGEST512_LEN, (const uint8_t*)m, len) + > -1); + + if (!ret) + return -1; + return 0; } /** Set the common_digests_t in <b>ds_out</b> to contain every digest on the @@ -2124,6 +2139,35 @@ crypto_hmac_sha256(char *hmac_out, tor_assert(rv); } +/** Compute a MAC using SHA3-256 of <b>msg_len</b> bytes in <b>msg</b> using a + * <b>key</b> of length <b>key_len</b> and a <b>salt</b> of length + * <b>salt_len</b>. Store the result of <b>len_out</b> bytes in in + * <b>mac_out</b>. This function can't fail. */ +void +crypto_mac_sha3_256(uint8_t *mac_out, size_t len_out, + const uint8_t *key, size_t key_len, + const uint8_t *msg, size_t msg_len) +{ + crypto_digest_t *digest; + + const uint64_t key_len_netorder = tor_htonll(key_len); + + tor_assert(mac_out); + tor_assert(key); + tor_assert(msg); + + digest = crypto_digest256_new(DIGEST_SHA3_256); + + /* Order matters here that is any subsystem using this function should + * expect this very precise ordering in the MAC construction. */ + crypto_digest_add_bytes(digest, (const char *) &key_len_netorder, + sizeof(key_len_netorder)); + crypto_digest_add_bytes(digest, (const char *) key, key_len); + crypto_digest_add_bytes(digest, (const char *) msg, msg_len); + crypto_digest_get_digest(digest, (char *) mac_out, len_out); + crypto_digest_free(digest); +} + /** Internal state for a eXtendable-Output Function (XOF). */ struct crypto_xof_t { keccak_state s; @@ -2643,7 +2687,7 @@ crypto_expand_key_material_TAP(const uint8_t *key_in, size_t key_in_len, for (cp = key_out, i=0; cp < key_out+key_out_len; ++i, cp += DIGEST_LEN) { tmp[key_in_len] = i; - if (crypto_digest((char*)digest, (const char *)tmp, key_in_len+1)) + if (crypto_digest((char*)digest, (const char *)tmp, key_in_len+1) < 0) goto exit; memcpy(cp, digest, MIN(DIGEST_LEN, key_out_len-(cp-key_out))); } @@ -2867,7 +2911,7 @@ crypto_strongest_rand_fallback(uint8_t *out, size_t out_len) size_t n; for (i = 0; filenames[i]; ++i) { - log_debug(LD_FS, "Opening %s for entropy", filenames[i]); + log_debug(LD_FS, "Considering %s for entropy", filenames[i]); fd = open(sandbox_intern_string(filenames[i]), O_RDONLY, 0); if (fd<0) continue; log_info(LD_CRYPTO, "Reading entropy from \"%s\"", filenames[i]); @@ -3430,3 +3474,15 @@ crypto_global_cleanup(void) /** @} */ +#ifdef USE_DMALLOC +/** Tell the crypto library to use Tor's allocation functions rather than + * calling libc's allocation functions directly. Return 0 on success, -1 + * on failure. */ +int +crypto_use_tor_alloc_functions(void) +{ + int r = CRYPTO_set_mem_ex_functions(tor_malloc_, tor_realloc_, tor_free_); + return r ? 0 : -1; +} +#endif + diff --git a/src/common/crypto.h b/src/common/crypto.h index 116e0a62fd..c70d91c262 100644 --- a/src/common/crypto.h +++ b/src/common/crypto.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -59,10 +59,12 @@ #define DIGEST256_LEN 32 /** Length of the output of our 64-bit optimized message digests (SHA512). */ #define DIGEST512_LEN 64 -/** Length of our symmetric cipher's keys. */ +/** Length of our symmetric cipher's keys of 128-bit. */ #define CIPHER_KEY_LEN 16 -/** Length of our symmetric cipher's IV. */ +/** Length of our symmetric cipher's IV of 128-bit. */ #define CIPHER_IV_LEN 16 +/** Length of our symmetric cipher's keys of 256-bit. */ +#define CIPHER256_KEY_LEN 32 /** Length of our public keys. */ #define PK_BYTES (1024/8) /** Length of our DH keys. */ @@ -129,6 +131,10 @@ int crypto_early_init(void) ATTR_WUR; int crypto_global_init(int hardwareAccel, const char *accelName, const char *accelPath) ATTR_WUR; +#ifdef USE_DMALLOC +int crypto_use_tor_alloc_functions(void); +#endif + void crypto_thread_cleanup(void); int crypto_global_cleanup(void); @@ -178,10 +184,12 @@ int crypto_pk_public_encrypt(crypto_pk_t *env, char *to, size_t tolen, int crypto_pk_private_decrypt(crypto_pk_t *env, char *to, size_t tolen, const char *from, size_t fromlen, int padding, int warnOnFailure); -int crypto_pk_public_checksig(const crypto_pk_t *env, char *to, size_t tolen, - const char *from, size_t fromlen); -int crypto_pk_public_checksig_digest(crypto_pk_t *env, const char *data, - size_t datalen, const char *sig, size_t siglen); +MOCK_DECL(int, crypto_pk_public_checksig,(const crypto_pk_t *env, + char *to, size_t tolen, + const char *from, size_t fromlen)); +MOCK_DECL(int, crypto_pk_public_checksig_digest,(crypto_pk_t *env, + const char *data, size_t datalen, + const char *sig, size_t siglen)); int crypto_pk_private_sign(const crypto_pk_t *env, char *to, size_t tolen, const char *from, size_t fromlen); int crypto_pk_private_sign_digest(crypto_pk_t *env, char *to, size_t tolen, @@ -255,6 +263,10 @@ void crypto_digest_assign(crypto_digest_t *into, void crypto_hmac_sha256(char *hmac_out, const char *key, size_t key_len, const char *msg, size_t msg_len); +void crypto_mac_sha3_256(uint8_t *mac_out, size_t len_out, + const uint8_t *key, size_t key_len, + const uint8_t *msg, size_t msg_len); + crypto_xof_t *crypto_xof_new(void); void crypto_xof_add_bytes(crypto_xof_t *xof, const uint8_t *data, size_t len); void crypto_xof_squeeze_bytes(crypto_xof_t *xof, uint8_t *out, size_t len); diff --git a/src/common/crypto_curve25519.c b/src/common/crypto_curve25519.c index fcbee3aba2..b99f13a93b 100644 --- a/src/common/crypto_curve25519.c +++ b/src/common/crypto_curve25519.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2012-2016, The Tor Project, Inc. */ +/* Copyright (c) 2012-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -80,7 +80,7 @@ curve25519_impl(uint8_t *output, const uint8_t *secret, /** * Helper function: Multiply the scalar "secret" by the Curve25519 * basepoint (X=9), and store the result in "output". Return 0 on - * success, -1 on false. + * success, -1 on failure. */ STATIC int curve25519_basepoint_impl(uint8_t *output, const uint8_t *secret) diff --git a/src/common/crypto_curve25519.h b/src/common/crypto_curve25519.h index 4011820949..e7790edac0 100644 --- a/src/common/crypto_curve25519.h +++ b/src/common/crypto_curve25519.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2012-2016, The Tor Project, Inc. */ +/* Copyright (c) 2012-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_CRYPTO_CURVE25519_H diff --git a/src/common/crypto_ed25519.c b/src/common/crypto_ed25519.c index 30ed772274..188e18c710 100644 --- a/src/common/crypto_ed25519.c +++ b/src/common/crypto_ed25519.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2016, The Tor Project, Inc. */ +/* Copyright (c) 2013-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -15,6 +15,7 @@ * keys to and from the corresponding Curve25519 keys. */ +#define CRYPTO_ED25519_PRIVATE #include "orconfig.h" #ifdef HAVE_SYS_STAT_H #include <sys/stat.h> @@ -31,10 +32,7 @@ #include "ed25519/ref10/ed25519_ref10.h" #include "ed25519/donna/ed25519_donna_tor.h" -#include <openssl/sha.h> - static void pick_ed25519_impl(void); -static int ed25519_impl_spot_check(void); /** An Ed25519 implementation, as a set of function pointers. */ typedef struct { @@ -211,6 +209,14 @@ ed25519_keypair_generate(ed25519_keypair_t *keypair_out, int extra_strong) return 0; } +/** Return true iff 'pubkey' is set to zero (eg to indicate that it is not + * set). */ +int +ed25519_public_key_is_zero(const ed25519_public_key_t *pubkey) +{ + return tor_mem_is_zero((char*)pubkey->pubkey, ED25519_PUBKEY_LEN); +} + /* Return a heap-allocated array that contains <b>msg</b> prefixed by the * string <b>prefix_str</b>. Set <b>final_msg_len_out</b> to the size of the * final array. If an error occured, return NULL. It's the resonsibility of the @@ -267,11 +273,11 @@ ed25519_sign(ed25519_signature_t *signature_out, * Like ed25519_sign(), but also prefix <b>msg</b> with <b>prefix_str</b> * before signing. <b>prefix_str</b> must be a NUL-terminated string. */ -int -ed25519_sign_prefixed(ed25519_signature_t *signature_out, - const uint8_t *msg, size_t msg_len, - const char *prefix_str, - const ed25519_keypair_t *keypair) +MOCK_IMPL(int, +ed25519_sign_prefixed,(ed25519_signature_t *signature_out, + const uint8_t *msg, size_t msg_len, + const char *prefix_str, + const ed25519_keypair_t *keypair)) { int retval; size_t prefixed_msg_len; @@ -300,10 +306,10 @@ ed25519_sign_prefixed(ed25519_signature_t *signature_out, * * Return 0 if the signature is valid; -1 if it isn't. */ -int -ed25519_checksig(const ed25519_signature_t *signature, - const uint8_t *msg, size_t len, - const ed25519_public_key_t *pubkey) +MOCK_IMPL(int, +ed25519_checksig,(const ed25519_signature_t *signature, + const uint8_t *msg, size_t len, + const ed25519_public_key_t *pubkey)) { return get_ed_impl()->open(signature->sig, msg, len, pubkey->pubkey) < 0 ? -1 : 0; @@ -346,10 +352,10 @@ ed25519_checksig_prefixed(const ed25519_signature_t *signature, * was valid. Otherwise return -N, where N is the number of invalid * signatures. */ -int -ed25519_checksig_batch(int *okay_out, - const ed25519_checkable_t *checkable, - int n_checkable) +MOCK_IMPL(int, +ed25519_checksig_batch,(int *okay_out, + const ed25519_checkable_t *checkable, + int n_checkable)) { int i, res; const ed25519_impl_t *impl = get_ed_impl(); @@ -434,14 +440,16 @@ ed25519_keypair_from_curve25519_keypair(ed25519_keypair_t *out, { const char string[] = "Derive high part of ed25519 key from curve25519 key"; ed25519_public_key_t pubkey_check; - SHA512_CTX ctx; - uint8_t sha512_output[64]; + crypto_digest_t *ctx; + uint8_t sha512_output[DIGEST512_LEN]; memcpy(out->seckey.seckey, inp->seckey.secret_key, 32); - SHA512_Init(&ctx); - SHA512_Update(&ctx, out->seckey.seckey, 32); - SHA512_Update(&ctx, string, sizeof(string)); - SHA512_Final(sha512_output, &ctx); + + ctx = crypto_digest512_new(DIGEST_SHA512); + crypto_digest_add_bytes(ctx, (const char*)out->seckey.seckey, 32); + crypto_digest_add_bytes(ctx, (const char*)string, sizeof(string)); + crypto_digest_get_digest(ctx, (char *)sha512_output, sizeof(sha512_output)); + crypto_digest_free(ctx); memcpy(out->seckey.seckey + 32, sha512_output, 32); ed25519_public_key_generate(&out->pubkey, &out->seckey); @@ -620,10 +628,22 @@ ed25519_pubkey_eq(const ed25519_public_key_t *key1, return tor_memeq(key1->pubkey, key2->pubkey, ED25519_PUBKEY_LEN); } +/** + * Set <b>dest</b> to contain the same key as <b>src</b>. + */ +void +ed25519_pubkey_copy(ed25519_public_key_t *dest, + const ed25519_public_key_t *src) +{ + tor_assert(dest); + tor_assert(src); + memcpy(dest, src, sizeof(ed25519_public_key_t)); +} + /** Check whether the given Ed25519 implementation seems to be working. * If so, return 0; otherwise return -1. */ -static int -ed25519_impl_spot_check(void) +MOCK_IMPL(STATIC int, +ed25519_impl_spot_check,(void)) { static const uint8_t alicesk[32] = { 0xc5,0xaa,0x8d,0xf4,0x3f,0x9f,0x83,0x7b, diff --git a/src/common/crypto_ed25519.h b/src/common/crypto_ed25519.h index 31afc49ccc..77a3313adc 100644 --- a/src/common/crypto_ed25519.h +++ b/src/common/crypto_ed25519.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2012-2016, The Tor Project, Inc. */ +/* Copyright (c) 2012-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_CRYPTO_ED25519_H @@ -51,21 +51,24 @@ int ed25519_keypair_generate(ed25519_keypair_t *keypair_out, int extra_strong); int ed25519_sign(ed25519_signature_t *signature_out, const uint8_t *msg, size_t len, const ed25519_keypair_t *key); -int ed25519_checksig(const ed25519_signature_t *signature, - const uint8_t *msg, size_t len, - const ed25519_public_key_t *pubkey); +MOCK_DECL(int,ed25519_checksig,(const ed25519_signature_t *signature, + const uint8_t *msg, size_t len, + const ed25519_public_key_t *pubkey)); + +MOCK_DECL(int, +ed25519_sign_prefixed,(ed25519_signature_t *signature_out, + const uint8_t *msg, size_t len, + const char *prefix_str, + const ed25519_keypair_t *keypair)); -int -ed25519_sign_prefixed(ed25519_signature_t *signature_out, - const uint8_t *msg, size_t len, - const char *prefix_str, - const ed25519_keypair_t *keypair); int ed25519_checksig_prefixed(const ed25519_signature_t *signature, const uint8_t *msg, size_t len, const char *prefix_str, const ed25519_public_key_t *pubkey); +int ed25519_public_key_is_zero(const ed25519_public_key_t *pubkey); + /** * A collection of information necessary to check an Ed25519 signature. Used * for batch verification. @@ -81,9 +84,9 @@ typedef struct { size_t len; } ed25519_checkable_t; -int ed25519_checksig_batch(int *okay_out, - const ed25519_checkable_t *checkable, - int n_checkable); +MOCK_DECL(int, ed25519_checksig_batch,(int *okay_out, + const ed25519_checkable_t *checkable, + int n_checkable)); int ed25519_keypair_from_curve25519_keypair(ed25519_keypair_t *out, int *signbit_out, @@ -118,6 +121,8 @@ void ed25519_keypair_free(ed25519_keypair_t *kp); int ed25519_pubkey_eq(const ed25519_public_key_t *key1, const ed25519_public_key_t *key2); +void ed25519_pubkey_copy(ed25519_public_key_t *dest, + const ed25519_public_key_t *src); void ed25519_set_impl_params(int use_donna); void ed25519_init(void); @@ -127,5 +132,9 @@ void crypto_ed25519_testing_force_impl(const char *name); void crypto_ed25519_testing_restore_impl(void); #endif +#ifdef CRYPTO_ED25519_PRIVATE +MOCK_DECL(STATIC int, ed25519_impl_spot_check, (void)); +#endif + #endif diff --git a/src/common/crypto_format.c b/src/common/crypto_format.c index 2f6d847c83..1d090a8770 100644 --- a/src/common/crypto_format.c +++ b/src/common/crypto_format.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -161,6 +161,27 @@ curve25519_public_from_base64(curve25519_public_key_t *pkey, } } +/** For logging convenience: Convert <b>pkey</b> to a statically allocated + * base64 string and return it. Not threadsafe. Format not meant to be + * computer-readable; it may change in the future. Subsequent calls invalidate + * previous returns. */ +const char * +ed25519_fmt(const ed25519_public_key_t *pkey) +{ + static char formatted[ED25519_BASE64_LEN+1]; + if (pkey) { + if (ed25519_public_key_is_zero(pkey)) { + strlcpy(formatted, "<unset>", sizeof(formatted)); + } else { + int r = ed25519_public_to_base64(formatted, pkey); + tor_assert(!r); + } + } else { + strlcpy(formatted, "<null>", sizeof(formatted)); + } + return formatted; +} + /** Try to decode the string <b>input</b> into an ed25519 public key. On * success, store the value in <b>pkey</b> and return 0. Otherwise return * -1. */ diff --git a/src/common/crypto_format.h b/src/common/crypto_format.h index 012e228cc4..390916cf04 100644 --- a/src/common/crypto_format.h +++ b/src/common/crypto_format.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_CRYPTO_FORMAT_H @@ -28,6 +28,7 @@ int ed25519_public_from_base64(ed25519_public_key_t *pkey, const char *input); int ed25519_public_to_base64(char *output, const ed25519_public_key_t *pkey); +const char *ed25519_fmt(const ed25519_public_key_t *pkey); /* XXXX move these to crypto_format.h */ #define ED25519_SIG_BASE64_LEN 86 diff --git a/src/common/crypto_pwbox.c b/src/common/crypto_pwbox.c index 31e37c007d..db8892e376 100644 --- a/src/common/crypto_pwbox.c +++ b/src/common/crypto_pwbox.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2014-2016, The Tor Project, Inc. */ +/* Copyright (c) 2014-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/crypto_s2k.c b/src/common/crypto_s2k.c index 5dbd2ad91f..076df815a9 100644 --- a/src/common/crypto_s2k.c +++ b/src/common/crypto_s2k.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/crypto_s2k.h b/src/common/crypto_s2k.h index 9b186450b1..04212b868a 100644 --- a/src/common/crypto_s2k.h +++ b/src/common/crypto_s2k.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_CRYPTO_S2K_H_INCLUDED diff --git a/src/common/di_ops.c b/src/common/di_ops.c index 4ed49e1164..e47998107d 100644 --- a/src/common/di_ops.c +++ b/src/common/di_ops.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2011-2016, The Tor Project, Inc. */ +/* Copyright (c) 2011-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/di_ops.h b/src/common/di_ops.h index 0a154302bf..e174fcc4e4 100644 --- a/src/common/di_ops.h +++ b/src/common/di_ops.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/handles.h b/src/common/handles.h index 1ee2322579..6d7262ab80 100644 --- a/src/common/handles.h +++ b/src/common/handles.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2016, The Tor Project, Inc. */ +/* Copyright (c) 2016-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/include.am b/src/common/include.am index cb307e9d5f..d12895b107 100644 --- a/src/common/include.am +++ b/src/common/include.am @@ -85,6 +85,7 @@ LIBOR_A_SRC = \ src/common/compat.c \ src/common/compat_threads.c \ src/common/compat_time.c \ + src/common/confline.c \ src/common/container.c \ src/common/log.c \ src/common/memarea.c \ @@ -94,21 +95,31 @@ LIBOR_A_SRC = \ src/common/util_format.c \ src/common/util_process.c \ src/common/sandbox.c \ + src/common/storagedir.c \ src/common/workqueue.c \ $(libor_extra_source) \ $(threads_impl_source) \ $(readpassphrase_source) +if USE_RUST +else +LIBOR_A_SRC += src/common/compat_rust.c +endif + src/common/src_common_libor_testing_a-log.$(OBJEXT) \ src/common/log.$(OBJEXT): micro-revision.i LIBOR_CRYPTO_A_SRC = \ src/common/aes.c \ + src/common/compress.c \ + src/common/compress_lzma.c \ + src/common/compress_none.c \ + src/common/compress_zlib.c \ + src/common/compress_zstd.c \ src/common/crypto.c \ src/common/crypto_pwbox.c \ src/common/crypto_s2k.c \ src/common/crypto_format.c \ - src/common/torgzip.c \ src/common/tortls.c \ src/common/crypto_curve25519.c \ src/common/crypto_ed25519.c @@ -143,8 +154,15 @@ COMMONHEADERS = \ src/common/compat.h \ src/common/compat_libevent.h \ src/common/compat_openssl.h \ + src/common/compat_rust.h \ src/common/compat_threads.h \ src/common/compat_time.h \ + src/common/compress.h \ + src/common/compress_lzma.h \ + src/common/compress_none.h \ + src/common/compress_zlib.h \ + src/common/compress_zstd.h \ + src/common/confline.h \ src/common/container.h \ src/common/crypto.h \ src/common/crypto_curve25519.h \ @@ -159,9 +177,9 @@ COMMONHEADERS = \ src/common/procmon.h \ src/common/pubsub.h \ src/common/sandbox.h \ + src/common/storagedir.h \ src/common/testsupport.h \ src/common/timers.h \ - src/common/torgzip.h \ src/common/torint.h \ src/common/torlog.h \ src/common/tortls.h \ diff --git a/src/common/log.c b/src/common/log.c index 4db1c9f0d0..87c260799d 100644 --- a/src/common/log.c +++ b/src/common/log.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -682,7 +682,7 @@ tor_log_get_logfile_names(smartlist_t *out) continue; if (lf->filename == NULL) continue; - smartlist_add(out, tor_strdup(lf->filename)); + smartlist_add_strdup(out, lf->filename); } UNLOCK_LOGS(); @@ -1086,7 +1086,7 @@ add_file_log(const log_severity_list_t *severity, const char *filename, int open_flags = O_WRONLY|O_CREAT; open_flags |= truncate_log ? O_TRUNC : O_APPEND; - fd = tor_open_cloexec(filename, open_flags, 0644); + fd = tor_open_cloexec(filename, open_flags, 0640); if (fd<0) return -1; if (tor_fd_seekend(fd)<0) { @@ -1177,7 +1177,7 @@ static const char *domain_list[] = { "GENERAL", "CRYPTO", "NET", "CONFIG", "FS", "PROTOCOL", "MM", "HTTP", "APP", "CONTROL", "CIRC", "REND", "BUG", "DIR", "DIRSERV", "OR", "EDGE", "ACCT", "HIST", "HANDSHAKE", "HEARTBEAT", "CHANNEL", - "SCHED", "DOS", NULL + "SCHED", "GUARD", "CONSDIFF", "DOS", NULL }; /** Return a bitmask for the log domain for which <b>domain</b> is the name, @@ -1319,10 +1319,8 @@ parse_log_severity_config(const char **cfg_ptr, if (got_an_unqualified_range > 1) return -1; - space = strchr(cfg, ' '); + space = find_whitespace(cfg); dash = strchr(cfg, '-'); - if (!space) - space = strchr(cfg, '\0'); if (dash && dash < space) { sev_lo = tor_strndup(cfg, dash-cfg); sev_hi = tor_strndup(dash+1, space-(dash+1)); diff --git a/src/common/memarea.c b/src/common/memarea.c index 7d16b702e3..659d1edf54 100644 --- a/src/common/memarea.c +++ b/src/common/memarea.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2008-2016, The Tor Project, Inc. */ +/* Copyright (c) 2008-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** \file memarea.c @@ -12,6 +12,9 @@ #include "util.h" #include "compat.h" #include "torlog.h" +#include "container.h" + +#ifndef DISABLE_MEMORY_SENTINELS /** If true, we try to detect any attempts to write beyond the length of a * memarea. */ @@ -304,3 +307,91 @@ memarea_assert_ok(memarea_t *area) } } +#else + +struct memarea_t { + smartlist_t *pieces; +}; + +memarea_t * +memarea_new(void) +{ + memarea_t *ma = tor_malloc_zero(sizeof(memarea_t)); + ma->pieces = smartlist_new(); + return ma; +} +void +memarea_drop_all(memarea_t *area) +{ + memarea_clear(area); + smartlist_free(area->pieces); + tor_free(area); +} +void +memarea_clear(memarea_t *area) +{ + SMARTLIST_FOREACH(area->pieces, void *, p, tor_free_(p)); + smartlist_clear(area->pieces); +} +int +memarea_owns_ptr(const memarea_t *area, const void *ptr) +{ + SMARTLIST_FOREACH(area->pieces, const void *, p, if (ptr == p) return 1;); + return 0; +} + +void * +memarea_alloc(memarea_t *area, size_t sz) +{ + void *result = tor_malloc(sz); + smartlist_add(area->pieces, result); + return result; +} + +void * +memarea_alloc_zero(memarea_t *area, size_t sz) +{ + void *result = tor_malloc_zero(sz); + smartlist_add(area->pieces, result); + return result; +} +void * +memarea_memdup(memarea_t *area, const void *s, size_t n) +{ + void *r = memarea_alloc(area, n); + memcpy(r, s, n); + return r; +} +char * +memarea_strdup(memarea_t *area, const char *s) +{ + size_t n = strlen(s); + char *r = memarea_alloc(area, n+1); + memcpy(r, s, n); + r[n] = 0; + return r; +} +char * +memarea_strndup(memarea_t *area, const char *s, size_t n) +{ + size_t ln = strnlen(s, n); + char *r = memarea_alloc(area, ln+1); + memcpy(r, s, ln); + r[ln] = 0; + return r; +} +void +memarea_get_stats(memarea_t *area, + size_t *allocated_out, size_t *used_out) +{ + (void)area; + *allocated_out = *used_out = 128; +} +void +memarea_assert_ok(memarea_t *area) +{ + (void)area; +} + +#endif + diff --git a/src/common/memarea.h b/src/common/memarea.h index 85bca51ad3..85012c1c34 100644 --- a/src/common/memarea.h +++ b/src/common/memarea.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2008-2016, The Tor Project, Inc. */ +/* Copyright (c) 2008-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /* Tor dependencies */ diff --git a/src/common/procmon.c b/src/common/procmon.c index c485c760c7..d49e7f18f5 100644 --- a/src/common/procmon.c +++ b/src/common/procmon.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2011-2016, The Tor Project, Inc. */ +/* Copyright (c) 2011-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/procmon.h b/src/common/procmon.h index 49ead24092..b07cff2c4a 100644 --- a/src/common/procmon.h +++ b/src/common/procmon.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2011-2016, The Tor Project, Inc. */ +/* Copyright (c) 2011-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/pubsub.c b/src/common/pubsub.c index b3faf40e00..336e8a6e7f 100644 --- a/src/common/pubsub.c +++ b/src/common/pubsub.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2016, The Tor Project, Inc. */ +/* Copyright (c) 2016-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/pubsub.h b/src/common/pubsub.h index bbb4f02a42..6f4ce08754 100644 --- a/src/common/pubsub.h +++ b/src/common/pubsub.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2016, The Tor Project, Inc. */ +/* Copyright (c) 2016-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/sandbox.c b/src/common/sandbox.c index 3d27ea66b5..312cf37722 100644 --- a/src/common/sandbox.c +++ b/src/common/sandbox.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -19,8 +19,14 @@ #define _LARGEFILE64_SOURCE #endif -/** Malloc mprotect limit in bytes. */ -#define MALLOC_MP_LIM 1048576 +/** Malloc mprotect limit in bytes. + * + * 28/06/2017: This value was increased from 16 MB to 20 MB after we introduced + * LZMA support in Tor (0.3.1.1-alpha). We limit our LZMA coder to 16 MB, but + * liblzma have a small overhead that we need to compensate for to avoid being + * killed by the sandbox. + */ +#define MALLOC_MP_LIM (20*1024*1024) #include <stdio.h> #include <string.h> @@ -1614,7 +1620,7 @@ sandbox_getaddrinfo(const char *name, const char *servname, return err; } - /* Otherwise, the sanbox is on. If we have an item, yield its cached + /* Otherwise, the sandbox is on. If we have an item, yield its cached result. */ if (item) { *res = item->res; diff --git a/src/common/sandbox.h b/src/common/sandbox.h index c5963e3119..a6b83153af 100644 --- a/src/common/sandbox.h +++ b/src/common/sandbox.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/storagedir.c b/src/common/storagedir.c new file mode 100644 index 0000000000..c471ea911f --- /dev/null +++ b/src/common/storagedir.c @@ -0,0 +1,586 @@ +/* Copyright (c) 2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "container.h" +#include "compat.h" +#include "confline.h" +#include "memarea.h" +#include "sandbox.h" +#include "storagedir.h" +#include "torlog.h" +#include "util.h" + +#ifdef HAVE_SYS_TYPES_H +#include <sys/types.h> +#endif +#ifdef HAVE_SYS_STAT_H +#include <sys/stat.h> +#endif +#ifdef HAVE_UNISTD_H +#include <unistd.h> +#endif + +#define FNAME_MIN_NUM 1000 + +/** A storage_dir_t represents a directory full of similar cached + * files. Filenames are decimal integers. Files can be cleaned as needed + * to limit total disk usage. */ +struct storage_dir_t { + /** Directory holding the files for this storagedir. */ + char *directory; + /** Either NULL, or a directory listing of the directory (as a smartlist + * of strings */ + smartlist_t *contents; + /** The largest number of non-temporary files we'll place in the + * directory. */ + int max_files; + /** If true, then 'usage' has been computed. */ + int usage_known; + /** The total number of bytes used in this directory */ + uint64_t usage; +}; + +/** Create or open a new storage directory at <b>dirname</b>, with + * capacity for up to <b>max_files</b> files. + */ +storage_dir_t * +storage_dir_new(const char *dirname, int max_files) +{ + if (check_private_dir(dirname, CPD_CREATE, NULL) < 0) + return NULL; + + storage_dir_t *d = tor_malloc_zero(sizeof(storage_dir_t)); + d->directory = tor_strdup(dirname); + d->max_files = max_files; + return d; +} + +/** + * Drop all in-RAM storage for <b>d</b>. Does not delete any files. + */ +void +storage_dir_free(storage_dir_t *d) +{ + if (d == NULL) + return; + tor_free(d->directory); + if (d->contents) { + SMARTLIST_FOREACH(d->contents, char *, cp, tor_free(cp)); + smartlist_free(d->contents); + } + tor_free(d); +} + +/** + * Tell the sandbox (if any) configured by <b>cfg</b> to allow the + * operations that <b>d</b> will need. + * + * The presence of this function is why we need an upper limit on the + * number of files in a storage_dir_t: we need to approve file operations + * one by one. + */ +int +storage_dir_register_with_sandbox(storage_dir_t *d, sandbox_cfg_t **cfg) +{ + int problems = 0; + int idx; + for (idx = FNAME_MIN_NUM; idx < FNAME_MIN_NUM + d->max_files; ++idx) { + char *path = NULL, *tmppath = NULL; + tor_asprintf(&path, "%s/%d", d->directory, idx); + tor_asprintf(&tmppath, "%s/%d.tmp", d->directory, idx); + + problems += sandbox_cfg_allow_open_filename(cfg, tor_strdup(path)); + problems += sandbox_cfg_allow_open_filename(cfg, tor_strdup(tmppath)); + problems += sandbox_cfg_allow_stat_filename(cfg, tor_strdup(path)); + problems += sandbox_cfg_allow_stat_filename(cfg, tor_strdup(tmppath)); + problems += sandbox_cfg_allow_rename(cfg, + tor_strdup(tmppath), tor_strdup(path)); + + tor_free(path); + tor_free(tmppath); + } + + return problems ? -1 : 0; +} + +/** + * Remove all files in <b>d</b> whose names end with ".tmp". + * + * Requires that the contents field of <b>d</b> is set. + */ +static void +storage_dir_clean_tmpfiles(storage_dir_t *d) +{ + if (!d->contents) + return; + SMARTLIST_FOREACH_BEGIN(d->contents, char *, fname) { + if (strcmpend(fname, ".tmp")) + continue; + char *path = NULL; + tor_asprintf(&path, "%s/%s", d->directory, fname); + if (unlink(sandbox_intern_string(path))) { + log_warn(LD_FS, "Unable to unlink %s while cleaning " + "temporary files: %s", escaped(path), strerror(errno)); + tor_free(path); + continue; + } + tor_free(path); + SMARTLIST_DEL_CURRENT(d->contents, fname); + tor_free(fname); + } SMARTLIST_FOREACH_END(fname); + + d->usage_known = 0; +} + +/** + * Re-scan the directory <b>d</b> to learn its contents. + */ +static int +storage_dir_rescan(storage_dir_t *d) +{ + if (d->contents) { + SMARTLIST_FOREACH(d->contents, char *, cp, tor_free(cp)); + smartlist_free(d->contents); + } + d->usage = 0; + d->usage_known = 0; + if (NULL == (d->contents = tor_listdir(d->directory))) { + return -1; + } + storage_dir_clean_tmpfiles(d); + return 0; +} + +/** + * Return a smartlist containing the filenames within <b>d</b>. + */ +const smartlist_t * +storage_dir_list(storage_dir_t *d) +{ + if (! d->contents) + storage_dir_rescan(d); + return d->contents; +} + +/** + * Return the total number of bytes used for storage in <b>d</b>. + */ +uint64_t +storage_dir_get_usage(storage_dir_t *d) +{ + if (d->usage_known) + return d->usage; + + uint64_t total = 0; + SMARTLIST_FOREACH_BEGIN(storage_dir_list(d), const char *, cp) { + char *path = NULL; + struct stat st; + tor_asprintf(&path, "%s/%s", d->directory, cp); + if (stat(sandbox_intern_string(path), &st) == 0) { + total += st.st_size; + } + tor_free(path); + } SMARTLIST_FOREACH_END(cp); + + d->usage = total; + d->usage_known = 1; + return d->usage; +} + +/** Mmap a specified file within <b>d</b>. + * + * On failure, return NULL and set errno as for tor_mmap_file(). */ +tor_mmap_t * +storage_dir_map(storage_dir_t *d, const char *fname) +{ + char *path = NULL; + tor_asprintf(&path, "%s/%s", d->directory, fname); + tor_mmap_t *result = tor_mmap_file(path); + int errval = errno; + tor_free(path); + if (result == NULL) + errno = errval; + return result; +} + +/** Read a file within <b>d</b> into a newly allocated buffer. Set + * *<b>sz_out</b> to its size. */ +uint8_t * +storage_dir_read(storage_dir_t *d, const char *fname, int bin, size_t *sz_out) +{ + const int flags = bin ? RFTS_BIN : 0; + + char *path = NULL; + tor_asprintf(&path, "%s/%s", d->directory, fname); + struct stat st; + char *contents = read_file_to_str(path, flags, &st); + if (contents && sz_out) { + // it fits in RAM, so we know its size is less than SIZE_MAX +#if UINT64_MAX > SIZE_MAX + tor_assert((uint64_t)st.st_size <= SIZE_MAX); +#endif + *sz_out = (size_t) st.st_size; + } + + tor_free(path); + return (uint8_t *) contents; +} + +/** Helper: Find an unused filename within the directory */ +static char * +find_unused_fname(storage_dir_t *d) +{ + if (!d->contents) { + if (storage_dir_rescan(d) < 0) + return NULL; + } + + char buf[16]; + int i; + /* Yuck; this is quadratic. Fortunately, that shouldn't matter much, + * since disk writes are more expensive by a lot. */ + for (i = FNAME_MIN_NUM; i < FNAME_MIN_NUM + d->max_files; ++i) { + tor_snprintf(buf, sizeof(buf), "%d", i); + if (!smartlist_contains_string(d->contents, buf)) { + return tor_strdup(buf); + } + } + return NULL; +} + +/** Helper: As storage_dir_save_bytes_to_file, but store a smartlist of + * sized_chunk_t rather than a single byte array. */ +static int +storage_dir_save_chunks_to_file(storage_dir_t *d, + const smartlist_t *chunks, + int binary, + char **fname_out) +{ + uint64_t total_length = 0; + char *fname = find_unused_fname(d); + if (!fname) + return -1; + + SMARTLIST_FOREACH(chunks, const sized_chunk_t *, ch, + total_length += ch->len); + + char *path = NULL; + tor_asprintf(&path, "%s/%s", d->directory, fname); + + int r = write_chunks_to_file(path, chunks, binary, 0); + if (r == 0) { + if (d->usage_known) + d->usage += total_length; + if (fname_out) { + *fname_out = tor_strdup(fname); + } + if (d->contents) + smartlist_add(d->contents, tor_strdup(fname)); + } + tor_free(fname); + tor_free(path); + return r; +} + +/** Try to write the <b>length</b> bytes at <b>data</b> into a new file + * in <b>d</b>. On success, return 0 and set *<b>fname_out</b> to a + * newly allocated string containing the filename. On failure, return + * -1. */ +int +storage_dir_save_bytes_to_file(storage_dir_t *d, + const uint8_t *data, + size_t length, + int binary, + char **fname_out) +{ + smartlist_t *chunks = smartlist_new(); + sized_chunk_t chunk = { (const char *)data, length }; + smartlist_add(chunks, &chunk); + int r = storage_dir_save_chunks_to_file(d, chunks, binary, fname_out); + smartlist_free(chunks); + return r; +} + +/** + * As storage_dir_save_bytes_to_file, but saves a NUL-terminated string + * <b>str</b>. + */ +int +storage_dir_save_string_to_file(storage_dir_t *d, + const char *str, + int binary, + char **fname_out) +{ + return storage_dir_save_bytes_to_file(d, + (const uint8_t*)str, strlen(str), binary, fname_out); +} + +/** + * As storage_dir_save_bytes_to_file, but associates the data with the + * key-value pairs in <b>labels</b>. Files stored in this format can be + * recovered with storage_dir_map_labeled() or storage_dir_read_labeled(). + */ +int +storage_dir_save_labeled_to_file(storage_dir_t *d, + const config_line_t *labels, + const uint8_t *data, + size_t length, + char **fname_out) +{ + /* + * The storage format is to prefix the data with the key-value pairs in + * <b>labels</b>, and a single NUL separator. But code outside this module + * MUST NOT rely on that format. + */ + + smartlist_t *chunks = smartlist_new(); + memarea_t *area = memarea_new(); + const config_line_t *line; + for (line = labels; line; line = line->next) { + sized_chunk_t *sz = memarea_alloc(area, sizeof(sized_chunk_t)); + sz->len = strlen(line->key) + 1 + strlen(line->value) + 1; + const size_t allocated = sz->len + 1; + char *bytes = memarea_alloc(area, allocated); + tor_snprintf(bytes, allocated, "%s %s\n", line->key, line->value); + sz->bytes = bytes; + smartlist_add(chunks, sz); + } + + sized_chunk_t *nul = memarea_alloc(area, sizeof(sized_chunk_t)); + nul->len = 1; + nul->bytes = "\0"; + smartlist_add(chunks, nul); + + sized_chunk_t *datachunk = memarea_alloc(area, sizeof(sized_chunk_t)); + datachunk->bytes = (const char *)data; + datachunk->len = length; + smartlist_add(chunks, datachunk); + + int r = storage_dir_save_chunks_to_file(d, chunks, 1, fname_out); + smartlist_free(chunks); + memarea_drop_all(area); + return r; +} + +/** + * Map a file that was created with storage_dir_save_labeled_to_file(). On + * failure, return NULL. On success, write a set of newly allocated labels + * into *<b>labels_out</b>, a pointer to the data into *<b>data_out</b>, and + * the data's size into *<b>sz_out</b>. On success, also return a tor_mmap_t + * object whose contents should not be used -- it needs to be kept around, + * though, for as long as <b>data_out</b> is going to be valid. + * + * On failure, set errno as for tor_mmap_file() if the file was missing or + * empty, and set errno to EINVAL if the file was not in the labeled + * format expected. + */ +tor_mmap_t * +storage_dir_map_labeled(storage_dir_t *dir, + const char *fname, + config_line_t **labels_out, + const uint8_t **data_out, + size_t *sz_out) +{ + tor_mmap_t *m = storage_dir_map(dir, fname); + int errval; + if (! m) { + errval = errno; + goto err; + } + const char *nulp = memchr(m->data, '\0', m->size); + if (! nulp) { + errval = EINVAL; + goto err; + } + if (labels_out && config_get_lines(m->data, labels_out, 0) < 0) { + errval = EINVAL; + goto err; + } + size_t offset = nulp - m->data + 1; + tor_assert(offset <= m->size); + *data_out = (const uint8_t *)(m->data + offset); + *sz_out = m->size - offset; + + return m; + err: + tor_munmap_file(m); + errno = errval; + return NULL; +} + +/** As storage_dir_map_labeled, but return a new byte array containing the + * data. */ +uint8_t * +storage_dir_read_labeled(storage_dir_t *dir, + const char *fname, + config_line_t **labels_out, + size_t *sz_out) +{ + const uint8_t *data = NULL; + tor_mmap_t *m = storage_dir_map_labeled(dir, fname, labels_out, + &data, sz_out); + if (m == NULL) + return NULL; + uint8_t *result = tor_memdup(data, *sz_out); + tor_munmap_file(m); + return result; +} + +/* Reduce the cached usage amount in <b>d</b> by <b>removed_file_size</b>. + * This function is a no-op if <b>d->usage_known</b> is 0. */ +static void +storage_dir_reduce_usage(storage_dir_t *d, uint64_t removed_file_size) +{ + if (d->usage_known) { + if (! BUG(d->usage < removed_file_size)) { + /* This bug can also be triggered if an external process resized a file + * between the call to storage_dir_get_usage() that last checked + * actual usage (rather than relaying on cached usage), and the call to + * this function. */ + d->usage -= removed_file_size; + } else { + /* If we underflowed the cached directory size, re-check the sizes of all + * the files in the directory. This makes storage_dir_shrink() quadratic, + * but only if a process is continually changing file sizes in the + * storage directory (in which case, we have bigger issues). + * + * We can't just reset usage_known, because storage_dir_shrink() relies + * on knowing the usage. */ + storage_dir_rescan(d); + (void)storage_dir_get_usage(d); + } + } +} + +/** + * Remove the file called <b>fname</b> from <b>d</b>. + */ +void +storage_dir_remove_file(storage_dir_t *d, + const char *fname) +{ + char *path = NULL; + tor_asprintf(&path, "%s/%s", d->directory, fname); + const char *ipath = sandbox_intern_string(path); + + uint64_t size = 0; + if (d->usage_known) { + struct stat st; + if (stat(ipath, &st) == 0) { + size = st.st_size; + } + } + if (unlink(ipath) == 0) { + storage_dir_reduce_usage(d, size); + } else { + log_warn(LD_FS, "Unable to unlink %s while removing file: %s", + escaped(path), strerror(errno)); + tor_free(path); + return; + } + if (d->contents) { + smartlist_string_remove(d->contents, fname); + } + + tor_free(path); +} + +/** Helper type: used to sort the members of storage directory by mtime. */ +typedef struct shrinking_dir_entry_t { + time_t mtime; + uint64_t size; + char *path; +} shrinking_dir_entry_t; + +/** Helper: use with qsort to sort shrinking_dir_entry_t structs. */ +static int +shrinking_dir_entry_compare(const void *a_, const void *b_) +{ + const shrinking_dir_entry_t *a = a_; + const shrinking_dir_entry_t *b = b_; + + if (a->mtime < b->mtime) + return -1; + else if (a->mtime > b->mtime) + return 1; + else + return 0; +} + +/** + * Try to free space by removing the oldest files in <b>d</b>. Delete + * until no more than <b>target_size</b> bytes are left, and at least + * <b>min_to_remove</b> files have been removed... or until there is + * nothing left to remove. + * + * Return 0 on success; -1 on failure. + */ +int +storage_dir_shrink(storage_dir_t *d, + uint64_t target_size, + int min_to_remove) +{ + if (d->usage_known && d->usage <= target_size && !min_to_remove) { + /* Already small enough. */ + return 0; + } + + if (storage_dir_rescan(d) < 0) + return -1; + + const uint64_t orig_usage = storage_dir_get_usage(d); + if (orig_usage <= target_size && !min_to_remove) { + /* Okay, small enough after rescan! */ + return 0; + } + + const int n = smartlist_len(d->contents); + shrinking_dir_entry_t *ents = tor_calloc(n, sizeof(shrinking_dir_entry_t)); + SMARTLIST_FOREACH_BEGIN(d->contents, const char *, fname) { + shrinking_dir_entry_t *ent = &ents[fname_sl_idx]; + struct stat st; + tor_asprintf(&ent->path, "%s/%s", d->directory, fname); + if (stat(sandbox_intern_string(ent->path), &st) == 0) { + ent->mtime = st.st_mtime; + ent->size = st.st_size; + } + } SMARTLIST_FOREACH_END(fname); + + qsort(ents, n, sizeof(shrinking_dir_entry_t), shrinking_dir_entry_compare); + + int idx = 0; + while ((d->usage > target_size || min_to_remove > 0) && idx < n) { + if (unlink(sandbox_intern_string(ents[idx].path)) == 0) { + storage_dir_reduce_usage(d, ents[idx].size); + --min_to_remove; + } + ++idx; + } + + for (idx = 0; idx < n; ++idx) { + tor_free(ents[idx].path); + } + tor_free(ents); + + storage_dir_rescan(d); + + return 0; +} + +/** Remove all files in <b>d</b>. */ +int +storage_dir_remove_all(storage_dir_t *d) +{ + return storage_dir_shrink(d, 0, d->max_files); +} + +/** + * Return the largest number of non-temporary files we're willing to + * store in <b>d</b>. + */ +int +storage_dir_get_max_files(storage_dir_t *d) +{ + return d->max_files; +} + diff --git a/src/common/storagedir.h b/src/common/storagedir.h new file mode 100644 index 0000000000..db25057e65 --- /dev/null +++ b/src/common/storagedir.h @@ -0,0 +1,51 @@ +/* Copyright (c) 2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef TOR_STORAGEDIR_H +#define TOR_STORAGEDIR_H + +typedef struct storage_dir_t storage_dir_t; +struct config_line_t; +struct sandbox_cfg_elem; + +storage_dir_t * storage_dir_new(const char *dirname, int n_files); +void storage_dir_free(storage_dir_t *d); +int storage_dir_register_with_sandbox(storage_dir_t *d, + struct sandbox_cfg_elem **cfg); +const smartlist_t *storage_dir_list(storage_dir_t *d); +uint64_t storage_dir_get_usage(storage_dir_t *d); +tor_mmap_t *storage_dir_map(storage_dir_t *d, const char *fname); +uint8_t *storage_dir_read(storage_dir_t *d, const char *fname, int bin, + size_t *sz_out); +int storage_dir_save_bytes_to_file(storage_dir_t *d, + const uint8_t *data, + size_t length, + int binary, + char **fname_out); +int storage_dir_save_string_to_file(storage_dir_t *d, + const char *data, + int binary, + char **fname_out); +int storage_dir_save_labeled_to_file(storage_dir_t *d, + const struct config_line_t *labels, + const uint8_t *data, + size_t length, + char **fname_out); +tor_mmap_t *storage_dir_map_labeled(storage_dir_t *dir, + const char *fname, + struct config_line_t **labels_out, + const uint8_t **data_out, + size_t *size_out); +uint8_t *storage_dir_read_labeled(storage_dir_t *d, const char *fname, + struct config_line_t **labels_out, + size_t *sz_out); +void storage_dir_remove_file(storage_dir_t *d, + const char *fname); +int storage_dir_shrink(storage_dir_t *d, + uint64_t target_size, + int min_to_remove); +int storage_dir_remove_all(storage_dir_t *d); +int storage_dir_get_max_files(storage_dir_t *d); + +#endif + diff --git a/src/common/testsupport.h b/src/common/testsupport.h index 9ad2ba77e0..9fc96199b4 100644 --- a/src/common/testsupport.h +++ b/src/common/testsupport.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2016, The Tor Project, Inc. */ +/* Copyright (c) 2013-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_TESTSUPPORT_H diff --git a/src/common/timers.c b/src/common/timers.c index 41b2008ac4..c43c49c083 100644 --- a/src/common/timers.c +++ b/src/common/timers.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2016, The Tor Project, Inc. */ +/* Copyright (c) 2016-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -29,6 +29,8 @@ #include "orconfig.h" +#define TOR_TIMERS_PRIVATE + #include "compat.h" #include "compat_libevent.h" #include "timers.h" @@ -148,6 +150,21 @@ libevent_timer_reschedule(void) event_add(global_timer_event, &d); } +/** Run the callback of every timer that has expired, based on the current + * output of monotime_get(). */ +STATIC void +timers_run_pending(void) +{ + monotime_t now; + monotime_get(&now); + timer_advance_to_cur_time(&now); + + tor_timer_t *t; + while ((t = timeouts_get(global_timeouts))) { + t->callback.cb(t, t->callback.arg, &now); + } +} + /** * Invoked when the libevent timer has expired: see which tor_timer_t events * have fired, activate their callbacks, and reschedule the libevent timer. @@ -159,14 +176,7 @@ libevent_timer_callback(evutil_socket_t fd, short what, void *arg) (void)what; (void)arg; - monotime_t now; - monotime_get(&now); - timer_advance_to_cur_time(&now); - - tor_timer_t *t; - while ((t = timeouts_get(global_timeouts))) { - t->callback.cb(t, t->callback.arg, &now); - } + timers_run_pending(); libevent_timer_reschedule(); } @@ -255,6 +265,20 @@ timer_set_cb(tor_timer_t *t, timer_cb_fn_t cb, void *arg) } /** + * Set *<b>cb_out</b> (if provided) to this timer's callback function, + * and *<b>arg_out</b> (if provided) to this timer's callback argument. + */ +void +timer_get_cb(const tor_timer_t *t, + timer_cb_fn_t *cb_out, void **arg_out) +{ + if (cb_out) + *cb_out = t->callback.cb; + if (arg_out) + *arg_out = t->callback.arg; +} + +/** * Schedule the timer t to fire at the current time plus a delay of * <b>delay</b> microseconds. All times are relative to monotime_get(). */ diff --git a/src/common/timers.h b/src/common/timers.h index 5f918f8e15..d9602cd2ae 100644 --- a/src/common/timers.h +++ b/src/common/timers.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2016, The Tor Project, Inc. */ +/* Copyright (c) 2016-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_TIMERS_H @@ -13,6 +13,8 @@ typedef void (*timer_cb_fn_t)(tor_timer_t *, void *, const struct monotime_t *); tor_timer_t *timer_new(timer_cb_fn_t cb, void *arg); void timer_set_cb(tor_timer_t *t, timer_cb_fn_t cb, void *arg); +void timer_get_cb(const tor_timer_t *t, + timer_cb_fn_t *cb_out, void **arg_out); void timer_schedule(tor_timer_t *t, const struct timeval *delay); void timer_disable(tor_timer_t *t); void timer_free(tor_timer_t *t); @@ -20,5 +22,9 @@ void timer_free(tor_timer_t *t); void timers_initialize(void); void timers_shutdown(void); +#ifdef TOR_TIMERS_PRIVATE +STATIC void timers_run_pending(void); +#endif + #endif diff --git a/src/common/torgzip.c b/src/common/torgzip.c deleted file mode 100644 index c44399aa74..0000000000 --- a/src/common/torgzip.c +++ /dev/null @@ -1,586 +0,0 @@ -/* Copyright (c) 2004, Roger Dingledine. - * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ -/* See LICENSE for licensing information */ - -/** - * \file torgzip.c - * \brief A simple in-memory gzip implementation. - **/ - -#include "orconfig.h" - -#include <stdlib.h> -#include <stdio.h> -#include <assert.h> -#include <string.h> -#include "torint.h" - -#ifdef HAVE_NETINET_IN_H -#include <netinet/in.h> -#endif - -#include "util.h" -#include "torlog.h" -#include "torgzip.h" - -/* zlib 1.2.4 and 1.2.5 do some "clever" things with macros. Instead of - saying "(defined(FOO) ? FOO : 0)" they like to say "FOO-0", on the theory - that nobody will care if the compile outputs a no-such-identifier warning. - - Sorry, but we like -Werror over here, so I guess we need to define these. - I hope that zlib 1.2.6 doesn't break these too. -*/ -#ifndef _LARGEFILE64_SOURCE -#define _LARGEFILE64_SOURCE 0 -#endif -#ifndef _LFS64_LARGEFILE -#define _LFS64_LARGEFILE 0 -#endif -#ifndef _FILE_OFFSET_BITS -#define _FILE_OFFSET_BITS 0 -#endif -#ifndef off64_t -#define off64_t int64_t -#endif - -#include <zlib.h> - -#if defined ZLIB_VERNUM && ZLIB_VERNUM < 0x1200 -#error "We require zlib version 1.2 or later." -#endif - -static size_t tor_zlib_state_size_precalc(int inflate, - int windowbits, int memlevel); - -/** Total number of bytes allocated for zlib state */ -static size_t total_zlib_allocation = 0; - -/** Return a string representation of the version of the currently running - * version of zlib. */ -const char * -tor_zlib_get_version_str(void) -{ - return zlibVersion(); -} - -/** Return a string representation of the version of the version of zlib -* used at compilation. */ -const char * -tor_zlib_get_header_version_str(void) -{ - return ZLIB_VERSION; -} - -/** Return the 'bits' value to tell zlib to use <b>method</b>.*/ -static inline int -method_bits(compress_method_t method, zlib_compression_level_t level) -{ - /* Bits+16 means "use gzip" in zlib >= 1.2 */ - const int flag = method == GZIP_METHOD ? 16 : 0; - switch (level) { - default: - case HIGH_COMPRESSION: return flag + 15; - case MEDIUM_COMPRESSION: return flag + 13; - case LOW_COMPRESSION: return flag + 11; - } -} - -static inline int -get_memlevel(zlib_compression_level_t level) -{ - switch (level) { - default: - case HIGH_COMPRESSION: return 8; - case MEDIUM_COMPRESSION: return 7; - case LOW_COMPRESSION: return 6; - } -} - -/** @{ */ -/* These macros define the maximum allowable compression factor. Anything of - * size greater than CHECK_FOR_COMPRESSION_BOMB_AFTER is not allowed to - * have an uncompression factor (uncompressed size:compressed size ratio) of - * any greater than MAX_UNCOMPRESSION_FACTOR. - * - * Picking a value for MAX_UNCOMPRESSION_FACTOR is a trade-off: we want it to - * be small to limit the attack multiplier, but we also want it to be large - * enough so that no legitimate document --even ones we might invent in the - * future -- ever compresses by a factor of greater than - * MAX_UNCOMPRESSION_FACTOR. Within those parameters, there's a reasonably - * large range of possible values. IMO, anything over 8 is probably safe; IMO - * anything under 50 is probably sufficient. - */ -#define MAX_UNCOMPRESSION_FACTOR 25 -#define CHECK_FOR_COMPRESSION_BOMB_AFTER (1024*64) -/** @} */ - -/** Return true if uncompressing an input of size <b>in_size</b> to an input - * of size at least <b>size_out</b> looks like a compression bomb. */ -static int -is_compression_bomb(size_t size_in, size_t size_out) -{ - if (size_in == 0 || size_out < CHECK_FOR_COMPRESSION_BOMB_AFTER) - return 0; - - return (size_out / size_in > MAX_UNCOMPRESSION_FACTOR); -} - -/** Given <b>in_len</b> bytes at <b>in</b>, compress them into a newly - * allocated buffer, using the method described in <b>method</b>. Store the - * compressed string in *<b>out</b>, and its length in *<b>out_len</b>. - * Return 0 on success, -1 on failure. - */ -int -tor_gzip_compress(char **out, size_t *out_len, - const char *in, size_t in_len, - compress_method_t method) -{ - struct z_stream_s *stream = NULL; - size_t out_size, old_size; - off_t offset; - - tor_assert(out); - tor_assert(out_len); - tor_assert(in); - tor_assert(in_len < UINT_MAX); - - *out = NULL; - - stream = tor_malloc_zero(sizeof(struct z_stream_s)); - stream->zalloc = Z_NULL; - stream->zfree = Z_NULL; - stream->opaque = NULL; - stream->next_in = (unsigned char*) in; - stream->avail_in = (unsigned int)in_len; - - if (deflateInit2(stream, Z_BEST_COMPRESSION, Z_DEFLATED, - method_bits(method, HIGH_COMPRESSION), - get_memlevel(HIGH_COMPRESSION), - Z_DEFAULT_STRATEGY) != Z_OK) { - //LCOV_EXCL_START -- we can only provoke failure by giving junk arguments. - log_warn(LD_GENERAL, "Error from deflateInit2: %s", - stream->msg?stream->msg:"<no message>"); - goto err; - //LCOV_EXCL_STOP - } - - /* Guess 50% compression. */ - out_size = in_len / 2; - if (out_size < 1024) out_size = 1024; - *out = tor_malloc(out_size); - stream->next_out = (unsigned char*)*out; - stream->avail_out = (unsigned int)out_size; - - while (1) { - switch (deflate(stream, Z_FINISH)) - { - case Z_STREAM_END: - goto done; - case Z_OK: - /* In case zlib doesn't work as I think .... */ - if (stream->avail_out >= stream->avail_in+16) - break; - /* Falls through. */ - case Z_BUF_ERROR: - offset = stream->next_out - ((unsigned char*)*out); - old_size = out_size; - out_size *= 2; - if (out_size < old_size) { - log_warn(LD_GENERAL, "Size overflow in compression."); - goto err; - } - *out = tor_realloc(*out, out_size); - stream->next_out = (unsigned char*)(*out + offset); - if (out_size - offset > UINT_MAX) { - log_warn(LD_BUG, "Ran over unsigned int limit of zlib while " - "uncompressing."); - goto err; - } - stream->avail_out = (unsigned int)(out_size - offset); - break; - default: - log_warn(LD_GENERAL, "Gzip compression didn't finish: %s", - stream->msg ? stream->msg : "<no message>"); - goto err; - } - } - done: - *out_len = stream->total_out; -#ifdef OPENBSD - /* "Hey Rocky! Watch me change an unsigned field to a signed field in a - * third-party API!" - * "Oh, that trick will just make people do unsafe casts to the unsigned - * type in their cross-platform code!" - * "Don't be foolish. I'm _sure_ they'll have the good sense to make sure - * the newly unsigned field isn't negative." */ - tor_assert(stream->total_out >= 0); -#endif - if (deflateEnd(stream)!=Z_OK) { - // LCOV_EXCL_START -- unreachable if we handled the zlib structure right - tor_assert_nonfatal_unreached(); - log_warn(LD_BUG, "Error freeing gzip structures"); - goto err; - // LCOV_EXCL_STOP - } - tor_free(stream); - - if (is_compression_bomb(*out_len, in_len)) { - log_warn(LD_BUG, "We compressed something and got an insanely high " - "compression factor; other Tors would think this was a zlib bomb."); - goto err; - } - - return 0; - err: - if (stream) { - deflateEnd(stream); - tor_free(stream); - } - tor_free(*out); - return -1; -} - -/** Given zero or more zlib-compressed or gzip-compressed strings of - * total length - * <b>in_len</b> bytes at <b>in</b>, uncompress them into a newly allocated - * buffer, using the method described in <b>method</b>. Store the uncompressed - * string in *<b>out</b>, and its length in *<b>out_len</b>. Return 0 on - * success, -1 on failure. - * - * If <b>complete_only</b> is true, we consider a truncated input as a - * failure; otherwise we decompress as much as we can. Warn about truncated - * or corrupt inputs at <b>protocol_warn_level</b>. - */ -int -tor_gzip_uncompress(char **out, size_t *out_len, - const char *in, size_t in_len, - compress_method_t method, - int complete_only, - int protocol_warn_level) -{ - struct z_stream_s *stream = NULL; - size_t out_size, old_size; - off_t offset; - int r; - - tor_assert(out); - tor_assert(out_len); - tor_assert(in); - tor_assert(in_len < UINT_MAX); - - *out = NULL; - - stream = tor_malloc_zero(sizeof(struct z_stream_s)); - stream->zalloc = Z_NULL; - stream->zfree = Z_NULL; - stream->opaque = NULL; - stream->next_in = (unsigned char*) in; - stream->avail_in = (unsigned int)in_len; - - if (inflateInit2(stream, - method_bits(method, HIGH_COMPRESSION)) != Z_OK) { - // LCOV_EXCL_START -- can only hit this if we give bad inputs. - log_warn(LD_GENERAL, "Error from inflateInit2: %s", - stream->msg?stream->msg:"<no message>"); - goto err; - // LCOV_EXCL_STOP - } - - out_size = in_len * 2; /* guess 50% compression. */ - if (out_size < 1024) out_size = 1024; - if (out_size >= SIZE_T_CEILING || out_size > UINT_MAX) - goto err; - - *out = tor_malloc(out_size); - stream->next_out = (unsigned char*)*out; - stream->avail_out = (unsigned int)out_size; - - while (1) { - switch (inflate(stream, complete_only ? Z_FINISH : Z_SYNC_FLUSH)) - { - case Z_STREAM_END: - if (stream->avail_in == 0) - goto done; - /* There may be more compressed data here. */ - if ((r = inflateEnd(stream)) != Z_OK) { - log_warn(LD_BUG, "Error freeing gzip structures"); - goto err; - } - if (inflateInit2(stream, - method_bits(method,HIGH_COMPRESSION)) != Z_OK) { - log_warn(LD_GENERAL, "Error from second inflateInit2: %s", - stream->msg?stream->msg:"<no message>"); - goto err; - } - break; - case Z_OK: - if (!complete_only && stream->avail_in == 0) - goto done; - /* In case zlib doesn't work as I think.... */ - if (stream->avail_out >= stream->avail_in+16) - break; - /* Falls through. */ - case Z_BUF_ERROR: - if (stream->avail_out > 0) { - log_fn(protocol_warn_level, LD_PROTOCOL, - "possible truncated or corrupt zlib data"); - goto err; - } - offset = stream->next_out - (unsigned char*)*out; - old_size = out_size; - out_size *= 2; - if (out_size < old_size) { - log_warn(LD_GENERAL, "Size overflow in uncompression."); - goto err; - } - if (is_compression_bomb(in_len, out_size)) { - log_warn(LD_GENERAL, "Input looks like a possible zlib bomb; " - "not proceeding."); - goto err; - } - if (out_size >= SIZE_T_CEILING) { - log_warn(LD_BUG, "Hit SIZE_T_CEILING limit while uncompressing."); - goto err; - } - *out = tor_realloc(*out, out_size); - stream->next_out = (unsigned char*)(*out + offset); - if (out_size - offset > UINT_MAX) { - log_warn(LD_BUG, "Ran over unsigned int limit of zlib while " - "uncompressing."); - goto err; - } - stream->avail_out = (unsigned int)(out_size - offset); - break; - default: - log_warn(LD_GENERAL, "Gzip decompression returned an error: %s", - stream->msg ? stream->msg : "<no message>"); - goto err; - } - } - done: - *out_len = stream->next_out - (unsigned char*)*out; - r = inflateEnd(stream); - tor_free(stream); - if (r != Z_OK) { - log_warn(LD_BUG, "Error freeing gzip structures"); - goto err; - } - - /* NUL-terminate output. */ - if (out_size == *out_len) - *out = tor_realloc(*out, out_size + 1); - (*out)[*out_len] = '\0'; - - return 0; - err: - if (stream) { - inflateEnd(stream); - tor_free(stream); - } - if (*out) { - tor_free(*out); - } - return -1; -} - -/** Try to tell whether the <b>in_len</b>-byte string in <b>in</b> is likely - * to be compressed or not. If it is, return the likeliest compression method. - * Otherwise, return UNKNOWN_METHOD. - */ -compress_method_t -detect_compression_method(const char *in, size_t in_len) -{ - if (in_len > 2 && fast_memeq(in, "\x1f\x8b", 2)) { - return GZIP_METHOD; - } else if (in_len > 2 && (in[0] & 0x0f) == 8 && - (ntohs(get_uint16(in)) % 31) == 0) { - return ZLIB_METHOD; - } else { - return UNKNOWN_METHOD; - } -} - -/** Internal state for an incremental zlib compression/decompression. The - * body of this struct is not exposed. */ -struct tor_zlib_state_t { - struct z_stream_s stream; /**< The zlib stream */ - int compress; /**< True if we are compressing; false if we are inflating */ - - /** Number of bytes read so far. Used to detect zlib bombs. */ - size_t input_so_far; - /** Number of bytes written so far. Used to detect zlib bombs. */ - size_t output_so_far; - - /** Approximate number of bytes allocated for this object. */ - size_t allocation; -}; - -/** Construct and return a tor_zlib_state_t object using <b>method</b>. If - * <b>compress</b>, it's for compression; otherwise it's for - * decompression. */ -tor_zlib_state_t * -tor_zlib_new(int compress_, compress_method_t method, - zlib_compression_level_t compression_level) -{ - tor_zlib_state_t *out; - int bits, memlevel; - - if (! compress_) { - /* use this setting for decompression, since we might have the - * max number of window bits */ - compression_level = HIGH_COMPRESSION; - } - - out = tor_malloc_zero(sizeof(tor_zlib_state_t)); - out->stream.zalloc = Z_NULL; - out->stream.zfree = Z_NULL; - out->stream.opaque = NULL; - out->compress = compress_; - bits = method_bits(method, compression_level); - memlevel = get_memlevel(compression_level); - if (compress_) { - if (deflateInit2(&out->stream, Z_BEST_COMPRESSION, Z_DEFLATED, - bits, memlevel, - Z_DEFAULT_STRATEGY) != Z_OK) - goto err; // LCOV_EXCL_LINE - } else { - if (inflateInit2(&out->stream, bits) != Z_OK) - goto err; // LCOV_EXCL_LINE - } - out->allocation = tor_zlib_state_size_precalc(!compress_, bits, memlevel); - - total_zlib_allocation += out->allocation; - - return out; - - err: - tor_free(out); - return NULL; -} - -/** Compress/decompress some bytes using <b>state</b>. Read up to - * *<b>in_len</b> bytes from *<b>in</b>, and write up to *<b>out_len</b> bytes - * to *<b>out</b>, adjusting the values as we go. If <b>finish</b> is true, - * we've reached the end of the input. - * - * Return TOR_ZLIB_DONE if we've finished the entire compression/decompression. - * Return TOR_ZLIB_OK if we're processed everything from the input. - * Return TOR_ZLIB_BUF_FULL if we're out of space on <b>out</b>. - * Return TOR_ZLIB_ERR if the stream is corrupt. - */ -tor_zlib_output_t -tor_zlib_process(tor_zlib_state_t *state, - char **out, size_t *out_len, - const char **in, size_t *in_len, - int finish) -{ - int err; - tor_assert(*in_len <= UINT_MAX); - tor_assert(*out_len <= UINT_MAX); - state->stream.next_in = (unsigned char*) *in; - state->stream.avail_in = (unsigned int)*in_len; - state->stream.next_out = (unsigned char*) *out; - state->stream.avail_out = (unsigned int)*out_len; - - if (state->compress) { - err = deflate(&state->stream, finish ? Z_FINISH : Z_NO_FLUSH); - } else { - err = inflate(&state->stream, finish ? Z_FINISH : Z_SYNC_FLUSH); - } - - state->input_so_far += state->stream.next_in - ((unsigned char*)*in); - state->output_so_far += state->stream.next_out - ((unsigned char*)*out); - - *out = (char*) state->stream.next_out; - *out_len = state->stream.avail_out; - *in = (const char *) state->stream.next_in; - *in_len = state->stream.avail_in; - - if (! state->compress && - is_compression_bomb(state->input_so_far, state->output_so_far)) { - log_warn(LD_DIR, "Possible zlib bomb; abandoning stream."); - return TOR_ZLIB_ERR; - } - - switch (err) - { - case Z_STREAM_END: - return TOR_ZLIB_DONE; - case Z_BUF_ERROR: - if (state->stream.avail_in == 0 && !finish) - return TOR_ZLIB_OK; - return TOR_ZLIB_BUF_FULL; - case Z_OK: - if (state->stream.avail_out == 0 || finish) - return TOR_ZLIB_BUF_FULL; - return TOR_ZLIB_OK; - default: - log_warn(LD_GENERAL, "Gzip returned an error: %s", - state->stream.msg ? state->stream.msg : "<no message>"); - return TOR_ZLIB_ERR; - } -} - -/** Deallocate <b>state</b>. */ -void -tor_zlib_free(tor_zlib_state_t *state) -{ - if (!state) - return; - - total_zlib_allocation -= state->allocation; - - if (state->compress) - deflateEnd(&state->stream); - else - inflateEnd(&state->stream); - - tor_free(state); -} - -/** Return an approximate number of bytes used in RAM to hold a state with - * window bits <b>windowBits</b> and compression level 'memlevel' */ -static size_t -tor_zlib_state_size_precalc(int inflate_, int windowbits, int memlevel) -{ - windowbits &= 15; - -#define A_FEW_KILOBYTES 2048 - - if (inflate_) { - /* From zconf.h: - - "The memory requirements for inflate are (in bytes) 1 << windowBits - that is, 32K for windowBits=15 (default value) plus a few kilobytes - for small objects." - */ - return sizeof(tor_zlib_state_t) + sizeof(struct z_stream_s) + - (1 << 15) + A_FEW_KILOBYTES; - } else { - /* Also from zconf.h: - - "The memory requirements for deflate are (in bytes): - (1 << (windowBits+2)) + (1 << (memLevel+9)) - ... plus a few kilobytes for small objects." - */ - return sizeof(tor_zlib_state_t) + sizeof(struct z_stream_s) + - (1 << (windowbits + 2)) + (1 << (memlevel + 9)) + A_FEW_KILOBYTES; - } -#undef A_FEW_KILOBYTES -} - -/** Return the approximate number of bytes allocated for <b>state</b>. */ -size_t -tor_zlib_state_size(const tor_zlib_state_t *state) -{ - return state->allocation; -} - -/** Return the approximate number of bytes allocated for all zlib states. */ -size_t -tor_zlib_get_total_allocation(void) -{ - return total_zlib_allocation; -} - diff --git a/src/common/torgzip.h b/src/common/torgzip.h deleted file mode 100644 index 00f62dcb45..0000000000 --- a/src/common/torgzip.h +++ /dev/null @@ -1,72 +0,0 @@ -/* Copyright (c) 2003, Roger Dingledine - * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ -/* See LICENSE for licensing information */ - -/** - * \file torgzip.h - * \brief Headers for torgzip.h - **/ - -#ifndef TOR_TORGZIP_H -#define TOR_TORGZIP_H - -/** Enumeration of what kind of compression to use. Only ZLIB_METHOD is - * guaranteed to be supported by the compress/uncompress functions here; - * GZIP_METHOD may be supported if we built against zlib version 1.2 or later - * and is_gzip_supported() returns true. */ -typedef enum { - NO_METHOD=0, GZIP_METHOD=1, ZLIB_METHOD=2, UNKNOWN_METHOD=3 -} compress_method_t; - -/** - * Enumeration to define tradeoffs between memory usage and compression level. - * HIGH_COMPRESSION saves the most bandwidth; LOW_COMPRESSION saves the most - * memory. - **/ -typedef enum { - HIGH_COMPRESSION, MEDIUM_COMPRESSION, LOW_COMPRESSION -} zlib_compression_level_t; - -int -tor_gzip_compress(char **out, size_t *out_len, - const char *in, size_t in_len, - compress_method_t method); -int -tor_gzip_uncompress(char **out, size_t *out_len, - const char *in, size_t in_len, - compress_method_t method, - int complete_only, - int protocol_warn_level); - -int is_gzip_supported(void); - -const char * -tor_zlib_get_version_str(void); - -const char * -tor_zlib_get_header_version_str(void); - -compress_method_t detect_compression_method(const char *in, size_t in_len); - -/** Return values from tor_zlib_process; see that function's documentation for - * details. */ -typedef enum { - TOR_ZLIB_OK, TOR_ZLIB_DONE, TOR_ZLIB_BUF_FULL, TOR_ZLIB_ERR -} tor_zlib_output_t; -/** Internal state for an incremental zlib compression/decompression. */ -typedef struct tor_zlib_state_t tor_zlib_state_t; -tor_zlib_state_t *tor_zlib_new(int compress, compress_method_t method, - zlib_compression_level_t level); - -tor_zlib_output_t tor_zlib_process(tor_zlib_state_t *state, - char **out, size_t *out_len, - const char **in, size_t *in_len, - int finish); -void tor_zlib_free(tor_zlib_state_t *state); - -size_t tor_zlib_state_size(const tor_zlib_state_t *state); -size_t tor_zlib_get_total_allocation(void); - -#endif - diff --git a/src/common/torint.h b/src/common/torint.h index 58c30f41a8..ee31459e94 100644 --- a/src/common/torint.h +++ b/src/common/torint.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/torlog.h b/src/common/torlog.h index 20b7d938f0..0149ce9a5b 100644 --- a/src/common/torlog.h +++ b/src/common/torlog.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -99,10 +99,14 @@ #define LD_CHANNEL (1u<<21) /** Scheduler */ #define LD_SCHED (1u<<22) +/** Guard nodes */ +#define LD_GUARD (1u<<23) +/** Generation and application of consensus diffs. */ +#define LD_CONSDIFF (1u<<24) /** Denial of Service mitigation. */ -#define LD_DOS (1u<<23) +#define LD_DOS (1u<<25) /** Number of logging domains in the code. */ -#define N_LOGGING_DOMAINS 24 +#define N_LOGGING_DOMAINS 26 /** This log message is not safe to send to a callback-based logger * immediately. Used as a flag, not a log domain. */ diff --git a/src/common/tortls.c b/src/common/tortls.c index a4e188603c..71de59896a 100644 --- a/src/common/tortls.c +++ b/src/common/tortls.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -17,6 +17,7 @@ #include "orconfig.h" #define TORTLS_PRIVATE +#define TORTLS_OPENSSL_PRIVATE #include <assert.h> #ifdef _WIN32 /*wrkard for dtls1.h >= 0.9.8m of "#include <winsock.h>"*/ @@ -136,6 +137,7 @@ static void tor_tls_context_decref(tor_tls_context_t *ctx); static void tor_tls_context_incref(tor_tls_context_t *ctx); static int check_cert_lifetime_internal(int severity, const X509 *cert, + time_t now, int past_tolerance, int future_tolerance); /** Global TLS contexts. We keep them here because nobody else needs @@ -458,11 +460,11 @@ tor_x509_name_new(const char *cname) * Return a certificate on success, NULL on failure. */ MOCK_IMPL(STATIC X509 *, - tor_tls_create_certificate,(crypto_pk_t *rsa, - crypto_pk_t *rsa_sign, - const char *cname, - const char *cname_sign, - unsigned int cert_lifetime)) +tor_tls_create_certificate,(crypto_pk_t *rsa, + crypto_pk_t *rsa_sign, + const char *cname, + const char *cname_sign, + unsigned int cert_lifetime)) { /* OpenSSL generates self-signed certificates with random 64-bit serial * numbers, so let's do that too. */ @@ -482,8 +484,22 @@ MOCK_IMPL(STATIC X509 *, * then we might pick a time where we're about to expire. Lastly, be * sure to start on a day boundary. */ time_t now = time(NULL); - start_time = crypto_rand_time_range(now - cert_lifetime, now) + 2*24*3600; - start_time -= start_time % (24*3600); + /* Our certificate lifetime will be cert_lifetime no matter what, but if we + * start cert_lifetime in the past, we'll have 0 real lifetime. instead we + * start up to (cert_lifetime - min_real_lifetime - start_granularity) in + * the past. */ + const time_t min_real_lifetime = 24*3600; + const time_t start_granularity = 24*3600; + time_t earliest_start_time = now - cert_lifetime + min_real_lifetime + + start_granularity; + /* Don't actually start in the future! */ + if (earliest_start_time >= now) + earliest_start_time = now - 1; + start_time = crypto_rand_time_range(earliest_start_time, now); + /* Round the start time back to the start of a day. */ + start_time -= start_time % start_granularity; + + end_time = start_time + cert_lifetime; tor_assert(rsa); tor_assert(cname); @@ -517,12 +533,12 @@ MOCK_IMPL(STATIC X509 *, if (!X509_time_adj(X509_get_notBefore(x509),0,&start_time)) goto error; - end_time = start_time + cert_lifetime; if (!X509_time_adj(X509_get_notAfter(x509),0,&end_time)) goto error; if (!X509_set_pubkey(x509, pkey)) goto error; - if (!X509_sign(x509, sign_pkey, EVP_sha1())) + + if (!X509_sign(x509, sign_pkey, EVP_sha256())) goto error; goto done; @@ -605,6 +621,12 @@ static const char UNRESTRICTED_SERVER_CIPHER_LIST[] = #ifdef TLS1_TXT_DHE_RSA_WITH_AES_128_GCM_SHA256 TLS1_TXT_DHE_RSA_WITH_AES_128_GCM_SHA256 ":" #endif +#ifdef TLS1_TXT_DHE_RSA_WITH_AES_256_CCM + TLS1_TXT_DHE_RSA_WITH_AES_256_CCM ":" +#endif +#ifdef TLS1_TXT_DHE_RSA_WITH_AES_128_CCM + TLS1_TXT_DHE_RSA_WITH_AES_128_CCM ":" +#endif #ifdef TLS1_TXT_DHE_RSA_WITH_AES_256_SHA256 TLS1_TXT_DHE_RSA_WITH_AES_256_SHA256 ":" #endif @@ -614,8 +636,14 @@ static const char UNRESTRICTED_SERVER_CIPHER_LIST[] = /* Required */ TLS1_TXT_DHE_RSA_WITH_AES_256_SHA ":" /* Required */ - TLS1_TXT_DHE_RSA_WITH_AES_128_SHA - ; + TLS1_TXT_DHE_RSA_WITH_AES_128_SHA ":" +#ifdef TLS1_TXT_ECDHE_RSA_WITH_CHACHA20_POLY1305 + TLS1_TXT_ECDHE_RSA_WITH_CHACHA20_POLY1305 ":" +#endif +#ifdef TLS1_TXT_DHE_RSA_WITH_CHACHA20_POLY1305 + TLS1_TXT_DHE_RSA_WITH_CHACHA20_POLY1305 +#endif + ; /* Note: to set up your own private testing network with link crypto * disabled, set your Tors' cipher list to @@ -656,7 +684,7 @@ tor_x509_cert_free(tor_x509_cert_t *cert) * Steals a reference to x509_cert. */ MOCK_IMPL(STATIC tor_x509_cert_t *, - tor_x509_cert_new,(X509 *x509_cert)) +tor_x509_cert_new,(X509 *x509_cert)) { tor_x509_cert_t *cert; EVP_PKEY *pkey; @@ -670,12 +698,7 @@ MOCK_IMPL(STATIC tor_x509_cert_t *, length = i2d_X509(x509_cert, &buf); cert = tor_malloc_zero(sizeof(tor_x509_cert_t)); if (length <= 0 || buf == NULL) { - /* LCOV_EXCL_START for the same reason as the exclusion above */ - tor_free(cert); - log_err(LD_CRYPTO, "Couldn't get length of encoded x509 certificate"); - X509_free(x509_cert); - return NULL; - /* LCOV_EXCL_STOP */ + goto err; } cert->encoded_len = (size_t) length; cert->encoded = tor_malloc(length); @@ -690,13 +713,25 @@ MOCK_IMPL(STATIC tor_x509_cert_t *, if ((pkey = X509_get_pubkey(x509_cert)) && (rsa = EVP_PKEY_get1_RSA(pkey))) { crypto_pk_t *pk = crypto_new_pk_from_rsa_(rsa); - crypto_pk_get_common_digests(pk, &cert->pkey_digests); + if (crypto_pk_get_common_digests(pk, &cert->pkey_digests) < 0) { + crypto_pk_free(pk); + EVP_PKEY_free(pkey); + goto err; + } + cert->pkey_digests_set = 1; crypto_pk_free(pk); EVP_PKEY_free(pkey); } return cert; + err: + /* LCOV_EXCL_START for the same reason as the exclusion above */ + tor_free(cert); + log_err(LD_CRYPTO, "Couldn't wrap encoded X509 certificate."); + X509_free(x509_cert); + return NULL; + /* LCOV_EXCL_STOP */ } /** Return a new copy of <b>cert</b>. */ @@ -800,8 +835,8 @@ tor_tls_context_decref(tor_tls_context_t *ctx) /** Set *<b>link_cert_out</b> and *<b>id_cert_out</b> to the link certificate * and ID certificate that we're currently using for our V3 in-protocol * handshake's certificate chain. If <b>server</b> is true, provide the certs - * that we use in server mode; otherwise, provide the certs that we use in - * client mode. */ + * that we use in server mode (auth, ID); otherwise, provide the certs that we + * use in client mode. (link, ID) */ int tor_tls_get_my_certs(int server, const tor_x509_cert_t **link_cert_out, @@ -831,7 +866,7 @@ tor_tls_get_my_client_auth_key(void) /** * Return a newly allocated copy of the public key that a certificate - * certifies. Return NULL if the cert's key is not RSA. + * certifies. Watch out! This returns NULL if the cert's key is not RSA. */ crypto_pk_t * tor_tls_cert_get_key(tor_x509_cert_t *cert) @@ -886,6 +921,7 @@ int tor_tls_cert_is_valid(int severity, const tor_x509_cert_t *cert, const tor_x509_cert_t *signing_cert, + time_t now, int check_rsa_1024) { check_no_tls_errors(); @@ -905,7 +941,7 @@ tor_tls_cert_is_valid(int severity, /* okay, the signature checked out right. Now let's check the check the * lifetime. */ - if (check_cert_lifetime_internal(severity, cert->cert, + if (check_cert_lifetime_internal(severity, cert->cert, now, 48*60*60, 30*24*60*60) < 0) goto bad; @@ -1050,6 +1086,8 @@ tor_tls_context_init_one(tor_tls_context_t **ppcontext, /** The group we should use for ecdhe when none was selected. */ #define NID_tor_default_ecdhe_group NID_X9_62_prime256v1 +#define RSA_LINK_KEY_BITS 2048 + /** Create a new TLS context for use with Tor TLS handshakes. * <b>identity</b> should be set to the identity key used to sign the * certificate. @@ -1075,7 +1113,7 @@ tor_tls_context_new(crypto_pk_t *identity, unsigned int key_lifetime, /* Generate short-term RSA key for use with TLS. */ if (!(rsa = crypto_pk_new())) goto error; - if (crypto_pk_generate_key(rsa)<0) + if (crypto_pk_generate_key_with_bits(rsa, RSA_LINK_KEY_BITS)<0) goto error; if (!is_client) { /* Generate short-term RSA key for use in the in-protocol ("v3") @@ -2073,13 +2111,13 @@ tor_tls_get_own_cert,(tor_tls_t *tls)) /** Warn that a certificate lifetime extends through a certain range. */ static void -log_cert_lifetime(int severity, const X509 *cert, const char *problem) +log_cert_lifetime(int severity, const X509 *cert, const char *problem, + time_t now) { BIO *bio = NULL; BUF_MEM *buf; char *s1=NULL, *s2=NULL; char mytime[33]; - time_t now = time(NULL); struct tm tm; size_t n; @@ -2227,6 +2265,7 @@ tor_tls_verify(int severity, tor_tls_t *tls, crypto_pk_t **identity_key) */ int tor_tls_check_lifetime(int severity, tor_tls_t *tls, + time_t now, int past_tolerance, int future_tolerance) { X509 *cert; @@ -2235,7 +2274,7 @@ tor_tls_check_lifetime(int severity, tor_tls_t *tls, if (!(cert = SSL_get_peer_certificate(tls->ssl))) goto done; - if (check_cert_lifetime_internal(severity, cert, + if (check_cert_lifetime_internal(severity, cert, now, past_tolerance, future_tolerance) < 0) goto done; @@ -2251,30 +2290,48 @@ tor_tls_check_lifetime(int severity, tor_tls_t *tls, /** Helper: check whether <b>cert</b> is expired give or take * <b>past_tolerance</b> seconds, or not-yet-valid give or take - * <b>future_tolerance</b> seconds. If it is live, return 0. If it is not - * live, log a message and return -1. */ + * <b>future_tolerance</b> seconds. (Relative to the current time + * <b>now</b>.) If it is live, return 0. If it is not live, log a message + * and return -1. */ static int check_cert_lifetime_internal(int severity, const X509 *cert, + time_t now, int past_tolerance, int future_tolerance) { - time_t now, t; - - now = time(NULL); + time_t t; t = now + future_tolerance; if (X509_cmp_time(X509_get_notBefore_const(cert), &t) > 0) { - log_cert_lifetime(severity, cert, "not yet valid"); + log_cert_lifetime(severity, cert, "not yet valid", now); return -1; } t = now - past_tolerance; if (X509_cmp_time(X509_get_notAfter_const(cert), &t) < 0) { - log_cert_lifetime(severity, cert, "already expired"); + log_cert_lifetime(severity, cert, "already expired", now); return -1; } return 0; } +#ifdef TOR_UNIT_TESTS +/* Testing only: return a new x509 cert with the same contents as <b>inp</b>, + but with the expiration time <b>new_expiration_time</b>, signed with + <b>signing_key</b>. */ +STATIC tor_x509_cert_t * +tor_x509_cert_replace_expiration(const tor_x509_cert_t *inp, + time_t new_expiration_time, + crypto_pk_t *signing_key) +{ + X509 *newc = X509_dup(inp->cert); + X509_time_adj(X509_get_notAfter(newc), 0, &new_expiration_time); + EVP_PKEY *pk = crypto_pk_get_evp_pkey_(signing_key, 1); + tor_assert(X509_sign(newc, pk, EVP_sha256())); + EVP_PKEY_free(pk); + return tor_x509_cert_new(newc); +} +#endif + /** Return the number of bytes available for reading from <b>tls</b>. */ int @@ -2493,6 +2550,28 @@ tor_tls_get_tlssecrets,(tor_tls_t *tls, uint8_t *secrets_out)) return 0; } +/** Using the RFC5705 key material exporting construction, and the + * provided <b>context</b> (<b>context_len</b> bytes long) and + * <b>label</b> (a NUL-terminated string), compute a 32-byte secret in + * <b>secrets_out</b> that only the parties to this TLS session can + * compute. Return 0 on success and -1 on failure. + */ +MOCK_IMPL(int, +tor_tls_export_key_material,(tor_tls_t *tls, uint8_t *secrets_out, + const uint8_t *context, + size_t context_len, + const char *label)) +{ + tor_assert(tls); + tor_assert(tls->ssl); + + int r = SSL_export_keying_material(tls->ssl, + secrets_out, DIGEST256_LEN, + label, strlen(label), + context, context_len, 1); + return (r == 1) ? 0 : -1; +} + /** Examine the amount of memory used and available for buffers in <b>tls</b>. * Set *<b>rbuf_capacity</b> to the amount of storage allocated for the read * buffer and *<b>rbuf_bytes</b> to the amount actually used. diff --git a/src/common/tortls.h b/src/common/tortls.h index f018c45c82..f430aff70b 100644 --- a/src/common/tortls.h +++ b/src/common/tortls.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_TORTLS_H @@ -63,12 +63,17 @@ typedef enum { } tor_tls_state_t; #define tor_tls_state_bitfield_t ENUM_BF(tor_tls_state_t) +struct x509_st; +struct ssl_st; +struct ssl_ctx_st; +struct ssl_session_st; + /** Holds a SSL_CTX object and related state used to configure TLS * connections. */ typedef struct tor_tls_context_t { int refcnt; - SSL_CTX *ctx; + struct ssl_ctx_st *ctx; tor_x509_cert_t *my_link_cert; tor_x509_cert_t *my_id_cert; tor_x509_cert_t *my_auth_cert; @@ -78,7 +83,7 @@ typedef struct tor_tls_context_t { /** Structure that we use for a single certificate. */ struct tor_x509_cert_t { - X509 *cert; + struct x509_st *cert; uint8_t *encoded; size_t encoded_len; unsigned pkey_digests_set : 1; @@ -92,7 +97,7 @@ struct tor_x509_cert_t { struct tor_tls_t { uint32_t magic; tor_tls_context_t *context; /** A link to the context object for this tls. */ - SSL *ssl; /**< An OpenSSL SSL object. */ + struct ssl_st *ssl; /**< An OpenSSL SSL object. */ int socket; /**< The underlying file descriptor for this TLS connection. */ char *address; /**< An address to log when describing this connection. */ tor_tls_state_bitfield_t state : 3; /**< The current SSL state, @@ -128,35 +133,45 @@ struct tor_tls_t { STATIC int tor_errno_to_tls_error(int e); STATIC int tor_tls_get_error(tor_tls_t *tls, int r, int extra, const char *doing, int severity, int domain); -STATIC tor_tls_t *tor_tls_get_by_ssl(const SSL *ssl); +STATIC tor_tls_t *tor_tls_get_by_ssl(const struct ssl_st *ssl); STATIC void tor_tls_allocate_tor_tls_object_ex_data_index(void); +#ifdef TORTLS_OPENSSL_PRIVATE STATIC int always_accept_verify_cb(int preverify_ok, X509_STORE_CTX *x509_ctx); -STATIC int tor_tls_classify_client_ciphers(const SSL *ssl, +STATIC int tor_tls_classify_client_ciphers(const struct ssl_st *ssl, STACK_OF(SSL_CIPHER) *peer_ciphers); -STATIC int tor_tls_client_is_using_v2_ciphers(const SSL *ssl); +#endif +STATIC int tor_tls_client_is_using_v2_ciphers(const struct ssl_st *ssl); MOCK_DECL(STATIC void, try_to_extract_certs_from_tls, - (int severity, tor_tls_t *tls, X509 **cert_out, X509 **id_cert_out)); + (int severity, tor_tls_t *tls, struct x509_st **cert_out, + struct x509_st **id_cert_out)); #ifndef HAVE_SSL_SESSION_GET_MASTER_KEY -STATIC size_t SSL_SESSION_get_master_key(SSL_SESSION *s, uint8_t *out, +STATIC size_t SSL_SESSION_get_master_key(struct ssl_session_st *s, + uint8_t *out, size_t len); #endif -STATIC void tor_tls_debug_state_callback(const SSL *ssl, int type, int val); -STATIC void tor_tls_server_info_callback(const SSL *ssl, int type, int val); -STATIC int tor_tls_session_secret_cb(SSL *ssl, void *secret, +STATIC void tor_tls_debug_state_callback(const struct ssl_st *ssl, + int type, int val); +STATIC void tor_tls_server_info_callback(const struct ssl_st *ssl, + int type, int val); +#ifdef TORTLS_OPENSSL_PRIVATE +STATIC int tor_tls_session_secret_cb(struct ssl_st *ssl, void *secret, int *secret_len, STACK_OF(SSL_CIPHER) *peer_ciphers, CONST_IF_OPENSSL_1_1_API SSL_CIPHER **cipher, void *arg); STATIC int find_cipher_by_id(const SSL *ssl, const SSL_METHOD *m, uint16_t cipher); -MOCK_DECL(STATIC X509*, tor_tls_create_certificate,(crypto_pk_t *rsa, +#endif +MOCK_DECL(STATIC struct x509_st *, tor_tls_create_certificate, + (crypto_pk_t *rsa, crypto_pk_t *rsa_sign, const char *cname, const char *cname_sign, unsigned int cert_lifetime)); STATIC tor_tls_context_t *tor_tls_context_new(crypto_pk_t *identity, unsigned int key_lifetime, unsigned flags, int is_client); -MOCK_DECL(STATIC tor_x509_cert_t *, tor_x509_cert_new,(X509 *x509_cert)); +MOCK_DECL(STATIC tor_x509_cert_t *, tor_x509_cert_new, + (struct x509_st *x509_cert)); STATIC int tor_tls_context_init_one(tor_tls_context_t **ppcontext, crypto_pk_t *identity, unsigned int key_lifetime, @@ -172,10 +187,16 @@ extern tor_tls_context_t *client_tls_context; extern uint16_t v2_cipher_list[]; extern uint64_t total_bytes_written_over_tls; extern uint64_t total_bytes_written_by_tls; + +STATIC tor_x509_cert_t *tor_x509_cert_replace_expiration( + const tor_x509_cert_t *inp, + time_t new_expiration_time, + crypto_pk_t *signing_key); #endif #endif /* endif TORTLS_PRIVATE */ +tor_x509_cert_t *tor_x509_cert_dup(const tor_x509_cert_t *cert); const char *tor_tls_err_to_string(int err); void tor_tls_get_state_description(tor_tls_t *tls, char *buf, size_t sz); @@ -197,12 +218,12 @@ void tor_tls_set_renegotiate_callback(tor_tls_t *tls, int tor_tls_is_server(tor_tls_t *tls); void tor_tls_free(tor_tls_t *tls); int tor_tls_peer_has_cert(tor_tls_t *tls); -tor_x509_cert_t *tor_x509_cert_dup(const tor_x509_cert_t *cert); MOCK_DECL(tor_x509_cert_t *,tor_tls_get_peer_cert,(tor_tls_t *tls)); MOCK_DECL(tor_x509_cert_t *,tor_tls_get_own_cert,(tor_tls_t *tls)); int tor_tls_verify(int severity, tor_tls_t *tls, crypto_pk_t **identity); int tor_tls_check_lifetime(int severity, - tor_tls_t *tls, int past_tolerance, + tor_tls_t *tls, time_t now, + int past_tolerance, int future_tolerance); MOCK_DECL(int, tor_tls_read, (tor_tls_t *tls, char *cp, size_t len)); int tor_tls_write(tor_tls_t *tls, const char *cp, size_t n); @@ -228,6 +249,11 @@ int tor_tls_used_v1_handshake(tor_tls_t *tls); int tor_tls_get_num_server_handshakes(tor_tls_t *tls); int tor_tls_server_got_renegotiate(tor_tls_t *tls); MOCK_DECL(int,tor_tls_get_tlssecrets,(tor_tls_t *tls, uint8_t *secrets_out)); +MOCK_DECL(int,tor_tls_export_key_material,( + tor_tls_t *tls, uint8_t *secrets_out, + const uint8_t *context, + size_t context_len, + const char *label)); /* Log and abort if there are unhandled TLS errors in OpenSSL's error stack. */ @@ -256,6 +282,7 @@ MOCK_DECL(int,tor_tls_cert_matches_key,(const tor_tls_t *tls, int tor_tls_cert_is_valid(int severity, const tor_x509_cert_t *cert, const tor_x509_cert_t *signing_cert, + time_t now, int check_rsa_1024); const char *tor_tls_get_ciphersuite_name(tor_tls_t *tls); diff --git a/src/common/util.c b/src/common/util.c index d2cbacde31..5b47028097 100644 --- a/src/common/util.c +++ b/src/common/util.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -79,7 +79,7 @@ #include <malloc/malloc.h> #endif #ifdef HAVE_MALLOC_H -#if !defined(OPENBSD) && !defined(__FreeBSD__) +#if !defined(OpenBSD) && !defined(__FreeBSD__) /* OpenBSD has a malloc.h, but for our purposes, it only exists in order to * scold us for being so stupid as to autodetect its presence. To be fair, * they've done this since 1996, when autoconf was only 5 years old. */ @@ -187,8 +187,9 @@ tor_malloc_zero_(size_t size DMALLOC_PARAMS) * 0xfffe0001. */ #define SQRT_SIZE_MAX_P1 (((size_t)1) << (sizeof(size_t)*4)) -/** Return non-zero if and only if the product of the arguments is exact. */ -static inline int +/** Return non-zero if and only if the product of the arguments is exact, + * and cannot overflow. */ +int size_mul_check(const size_t x, const size_t y) { /* This first check is equivalent to @@ -202,15 +203,6 @@ size_mul_check(const size_t x, const size_t y) x <= SIZE_MAX / y); } -#ifdef TOR_UNIT_TESTS -/** Exposed for unit tests only */ -int -size_mul_check__(const size_t x, const size_t y) -{ - return size_mul_check(x,y); -} -#endif - /** Allocate a chunk of <b>nmemb</b>*<b>size</b> bytes of memory, fill * the memory with zero bytes, and return a pointer to the result. * Log and terminate the process on error. (Same as @@ -712,6 +704,19 @@ tor_strisnonupper(const char *s) return 1; } +/** Return true iff every character in <b>s</b> is whitespace space; else + * return false. */ +int +tor_strisspace(const char *s) +{ + while (*s) { + if (!TOR_ISSPACE(*s)) + return 0; + s++; + } + return 1; +} + /** As strcmp, except that either string may be NULL. The NULL string is * considered to be before any non-NULL string. */ int @@ -1803,17 +1808,26 @@ format_iso_time_nospace_usec(char *buf, const struct timeval *tv) /** Given an ISO-formatted UTC time value (after the epoch) in <b>cp</b>, * parse it and store its value in *<b>t</b>. Return 0 on success, -1 on * failure. Ignore extraneous stuff in <b>cp</b> after the end of the time - * string, unless <b>strict</b> is set. */ + * string, unless <b>strict</b> is set. If <b>nospace</b> is set, + * expect the YYYY-MM-DDTHH:MM:SS format. */ int -parse_iso_time_(const char *cp, time_t *t, int strict) +parse_iso_time_(const char *cp, time_t *t, int strict, int nospace) { struct tm st_tm; unsigned int year=0, month=0, day=0, hour=0, minute=0, second=0; int n_fields; - char extra_char; - n_fields = tor_sscanf(cp, "%u-%2u-%2u %2u:%2u:%2u%c", &year, &month, - &day, &hour, &minute, &second, &extra_char); - if (strict ? (n_fields != 6) : (n_fields < 6)) { + char extra_char, separator_char; + n_fields = tor_sscanf(cp, "%u-%2u-%2u%c%2u:%2u:%2u%c", + &year, &month, &day, + &separator_char, + &hour, &minute, &second, &extra_char); + if (strict ? (n_fields != 7) : (n_fields < 7)) { + char *esc = esc_for_log(cp); + log_warn(LD_GENERAL, "ISO time %s was unparseable", esc); + tor_free(esc); + return -1; + } + if (separator_char != (nospace ? 'T' : ' ')) { char *esc = esc_for_log(cp); log_warn(LD_GENERAL, "ISO time %s was unparseable", esc); tor_free(esc); @@ -1855,7 +1869,16 @@ parse_iso_time_(const char *cp, time_t *t, int strict) int parse_iso_time(const char *cp, time_t *t) { - return parse_iso_time_(cp, t, 1); + return parse_iso_time_(cp, t, 1, 0); +} + +/** + * As parse_iso_time, but parses a time encoded by format_iso_time_nospace(). + */ +int +parse_iso_time_nospace(const char *cp, time_t *t) +{ + return parse_iso_time_(cp, t, 1, 1); } /** Given a <b>date</b> in one of the three formats allowed by HTTP (ugh), @@ -1930,7 +1953,7 @@ parse_http_time(const char *date, struct tm *tm) /** Given an <b>interval</b> in seconds, try to write it to the * <b>out_len</b>-byte buffer in <b>out</b> in a human-readable form. - * Return 0 on success, -1 on failure. + * Returns a non-negative integer on success, -1 on failure. */ int format_time_interval(char *out, size_t out_len, long interval) @@ -2095,7 +2118,7 @@ read_all(tor_socket_t fd, char *buf, size_t count, int isSocket) return -1; } - while (numread != count) { + while (numread < count) { if (isSocket) result = tor_socket_recv(fd, buf+numread, count-numread, 0); else @@ -2270,10 +2293,14 @@ check_private_dir,(const char *dirname, cpd_check_t check, * permissions on the directory will be checked again below.*/ fd = open(sandbox_intern_string(dirname), O_NOFOLLOW); - if (fd == -1) + if (fd == -1) { + log_warn(LD_FS, "Could not reopen recently created directory %s: %s", + dirname, + strerror(errno)); return -1; - else + } else { close(fd); + } } else if (!(check & CPD_CHECK)) { log_warn(LD_FS, "Directory %s does not exist.", dirname); @@ -2601,6 +2628,14 @@ finish_writing_to_file_impl(open_file_t *file_data, int abort_write) if (file_data->rename_on_close) { tor_assert(file_data->tempname && file_data->filename); + if (!abort_write) { + tor_assert(strcmp(file_data->filename, file_data->tempname)); + if (replace_file(file_data->tempname, file_data->filename)) { + log_warn(LD_FS, "Error replacing \"%s\": %s", file_data->filename, + strerror(errno)); + abort_write = r = -1; + } + } if (abort_write) { int res = unlink(file_data->tempname); if (res != 0) { @@ -2609,13 +2644,6 @@ finish_writing_to_file_impl(open_file_t *file_data, int abort_write) file_data->tempname, strerror(errno)); r = -1; } - } else { - tor_assert(strcmp(file_data->filename, file_data->tempname)); - if (replace_file(file_data->tempname, file_data->filename)) { - log_warn(LD_FS, "Error replacing \"%s\": %s", file_data->filename, - strerror(errno)); - r = -1; - } } } @@ -3017,135 +3045,39 @@ unescape_string(const char *s, char **result, size_t *size_out) } } -/** Given a string containing part of a configuration file or similar format, - * advance past comments and whitespace and try to parse a single line. If we - * parse a line successfully, set *<b>key_out</b> to a new string holding the - * key portion and *<b>value_out</b> to a new string holding the value portion - * of the line, and return a pointer to the start of the next line. If we run - * out of data, return a pointer to the end of the string. If we encounter an - * error, return NULL and set *<b>err_out</b> (if provided) to an error - * message. - */ -const char * -parse_config_line_from_str_verbose(const char *line, char **key_out, - char **value_out, - const char **err_out) +/** Removes enclosing quotes from <b>path</b> and unescapes quotes between the + * enclosing quotes. Backslashes are not unescaped. Return the unquoted + * <b>path</b> on sucess or 0 if <b>path</b> is not quoted correctly. */ +char * +get_unquoted_path(const char *path) { - /* - See torrc_format.txt for a description of the (silly) format this parses. - */ - const char *key, *val, *cp; - int continuation = 0; - - tor_assert(key_out); - tor_assert(value_out); + size_t len = strlen(path); - *key_out = *value_out = NULL; - key = val = NULL; - /* Skip until the first keyword. */ - while (1) { - while (TOR_ISSPACE(*line)) - ++line; - if (*line == '#') { - while (*line && *line != '\n') - ++line; - } else { - break; - } + if (len == 0) { + return tor_strdup(""); } - if (!*line) { /* End of string? */ - *key_out = *value_out = NULL; - return line; + int has_start_quote = (path[0] == '\"'); + int has_end_quote = (len > 0 && path[len-1] == '\"'); + if (has_start_quote != has_end_quote || (len == 1 && has_start_quote)) { + return NULL; } - /* Skip until the next space or \ followed by newline. */ - key = line; - while (*line && !TOR_ISSPACE(*line) && *line != '#' && - ! (line[0] == '\\' && line[1] == '\n')) - ++line; - *key_out = tor_strndup(key, line-key); - - /* Skip until the value. */ - while (*line == ' ' || *line == '\t') - ++line; - - val = line; - - /* Find the end of the line. */ - if (*line == '\"') { // XXX No continuation handling is done here - if (!(line = unescape_string(line, value_out, NULL))) { - if (err_out) - *err_out = "Invalid escape sequence in quoted string"; - return NULL; - } - while (*line == ' ' || *line == '\t') - ++line; - if (*line == '\r' && *(++line) == '\n') - ++line; - if (*line && *line != '#' && *line != '\n') { - if (err_out) - *err_out = "Excess data after quoted string"; + char *unquoted_path = tor_malloc(len - has_start_quote - has_end_quote + 1); + char *s = unquoted_path; + size_t i; + for (i = has_start_quote; i < len - has_end_quote; i++) { + if (path[i] == '\"' && (i > 0 && path[i-1] == '\\')) { + *(s-1) = path[i]; + } else if (path[i] != '\"') { + *s++ = path[i]; + } else { /* unescaped quote */ + tor_free(unquoted_path); return NULL; } - } else { - /* Look for the end of the line. */ - while (*line && *line != '\n' && (*line != '#' || continuation)) { - if (*line == '\\' && line[1] == '\n') { - continuation = 1; - line += 2; - } else if (*line == '#') { - do { - ++line; - } while (*line && *line != '\n'); - if (*line == '\n') - ++line; - } else { - ++line; - } - } - - if (*line == '\n') { - cp = line++; - } else { - cp = line; - } - /* Now back cp up to be the last nonspace character */ - while (cp>val && TOR_ISSPACE(*(cp-1))) - --cp; - - tor_assert(cp >= val); - - /* Now copy out and decode the value. */ - *value_out = tor_strndup(val, cp-val); - if (continuation) { - char *v_out, *v_in; - v_out = v_in = *value_out; - while (*v_in) { - if (*v_in == '#') { - do { - ++v_in; - } while (*v_in && *v_in != '\n'); - if (*v_in == '\n') - ++v_in; - } else if (v_in[0] == '\\' && v_in[1] == '\n') { - v_in += 2; - } else { - *v_out++ = *v_in++; - } - } - *v_out = '\0'; - } } - - if (*line == '#') { - do { - ++line; - } while (*line && *line != '\n'); - } - while (TOR_ISSPACE(*line)) ++line; - - return line; + *s = '\0'; + return unquoted_path; } /** Expand any homedir prefix on <b>filename</b>; return a newly allocated @@ -3534,6 +3466,17 @@ smartlist_add_vasprintf(struct smartlist_t *sl, const char *pattern, smartlist_add(sl, str); } +/** Append a copy of string to sl */ +void +smartlist_add_strdup(struct smartlist_t *sl, const char *string) +{ + char *copy; + + copy = tor_strdup(string); + + smartlist_add(sl, copy); +} + /** Return a new list containing the filenames in the directory <b>dirname</b>. * Return NULL on error or if <b>dirname</b> is not a directory. */ @@ -3567,7 +3510,7 @@ tor_listdir, (const char *dirname)) #endif if (strcmp(name, ".") && strcmp(name, "..")) { - smartlist_add(result, tor_strdup(name)); + smartlist_add_strdup(result, name); } if (!FindNextFile(handle, &findData)) { DWORD err; @@ -3593,7 +3536,7 @@ tor_listdir, (const char *dirname)) if (!strcmp(de->d_name, ".") || !strcmp(de->d_name, "..")) continue; - smartlist_add(result, tor_strdup(de->d_name)); + smartlist_add_strdup(result, de->d_name); } closedir(d); #endif @@ -4136,10 +4079,10 @@ tor_process_get_stdout_pipe(process_handle_t *process_handle) } #else /* DOCDOC tor_process_get_stdout_pipe */ -FILE * +int tor_process_get_stdout_pipe(process_handle_t *process_handle) { - return process_handle->stdout_handle; + return process_handle->stdout_pipe; } #endif @@ -4570,10 +4513,6 @@ tor_spawn_background(const char *const filename, const char **argv, log_warn(LD_GENERAL, "Failed to set stderror/stdout/stdin pipes " "nonblocking in parent process: %s", strerror(errno)); } - /* Open the buffered IO streams */ - process_handle->stdout_handle = fdopen(process_handle->stdout_pipe, "r"); - process_handle->stderr_handle = fdopen(process_handle->stderr_pipe, "r"); - process_handle->stdin_handle = fdopen(process_handle->stdin_pipe, "r"); *process_handle_out = process_handle; return process_handle->status; @@ -4620,14 +4559,9 @@ tor_process_handle_destroy,(process_handle_t *process_handle, if (process_handle->stdin_pipe) CloseHandle(process_handle->stdin_pipe); #else - if (process_handle->stdout_handle) - fclose(process_handle->stdout_handle); - - if (process_handle->stderr_handle) - fclose(process_handle->stderr_handle); - - if (process_handle->stdin_handle) - fclose(process_handle->stdin_handle); + close(process_handle->stdout_pipe); + close(process_handle->stderr_pipe); + close(process_handle->stdin_pipe); clear_waitpid_callback(process_handle->waitpid_cb); #endif @@ -4864,7 +4798,7 @@ get_current_process_environment_variables(void) char **environ_tmp; /* Not const char ** ? Really? */ for (environ_tmp = get_environment(); *environ_tmp; ++environ_tmp) { - smartlist_add(sl, tor_strdup(*environ_tmp)); + smartlist_add_strdup(sl, *environ_tmp); } return sl; @@ -4913,7 +4847,7 @@ tor_read_all_handle(HANDLE h, char *buf, size_t count, if (count > SIZE_T_CEILING || count > SSIZE_MAX) return -1; - while (numread != count) { + while (numread < count) { /* Check if there is anything to read */ retval = PeekNamedPipe(h, NULL, 0, NULL, &byte_count, NULL); if (!retval) { @@ -4959,19 +4893,19 @@ tor_read_all_handle(HANDLE h, char *buf, size_t count, return (ssize_t)numread; } #else -/** Read from a handle <b>h</b> into <b>buf</b>, up to <b>count</b> bytes. If +/** Read from a handle <b>fd</b> into <b>buf</b>, up to <b>count</b> bytes. If * <b>process</b> is NULL, the function will return immediately if there is * nothing more to read. Otherwise data will be read until end of file, or * <b>count</b> bytes are read. Returns the number of bytes read, or -1 on * error. Sets <b>eof</b> to true if <b>eof</b> is not NULL and the end of the * file has been reached. */ ssize_t -tor_read_all_handle(FILE *h, char *buf, size_t count, +tor_read_all_handle(int fd, char *buf, size_t count, const process_handle_t *process, int *eof) { size_t numread = 0; - char *retval; + ssize_t result; if (eof) *eof = 0; @@ -4979,34 +4913,28 @@ tor_read_all_handle(FILE *h, char *buf, size_t count, if (count > SIZE_T_CEILING || count > SSIZE_MAX) return -1; - while (numread != count) { - /* Use fgets because that is what we use in log_from_pipe() */ - retval = fgets(buf+numread, (int)(count-numread), h); - if (NULL == retval) { - if (feof(h)) { - log_debug(LD_GENERAL, "fgets() reached end of file"); - if (eof) - *eof = 1; + while (numread < count) { + result = read(fd, buf+numread, count-numread); + + if (result == 0) { + log_debug(LD_GENERAL, "read() reached end of file"); + if (eof) + *eof = 1; + break; + } else if (result < 0 && errno == EAGAIN) { + if (process) + continue; + else break; - } else { - if (EAGAIN == errno) { - if (process) - continue; - else - break; - } else { - log_warn(LD_GENERAL, "fgets() from handle failed: %s", - strerror(errno)); - return -1; - } - } + } else if (result < 0) { + log_warn(LD_GENERAL, "read() failed: %s", strerror(errno)); + return -1; } - tor_assert(retval != NULL); - tor_assert(strlen(retval) + numread <= count); - numread += strlen(retval); + + numread += result; } - log_debug(LD_GENERAL, "fgets() read %d bytes from handle", (int)numread); + log_debug(LD_GENERAL, "read() read %d bytes from handle", (int)numread); return (ssize_t)numread; } #endif @@ -5020,7 +4948,7 @@ tor_read_all_from_process_stdout(const process_handle_t *process_handle, return tor_read_all_handle(process_handle->stdout_pipe, buf, count, process_handle); #else - return tor_read_all_handle(process_handle->stdout_handle, buf, count, + return tor_read_all_handle(process_handle->stdout_pipe, buf, count, process_handle, NULL); #endif } @@ -5034,7 +4962,7 @@ tor_read_all_from_process_stderr(const process_handle_t *process_handle, return tor_read_all_handle(process_handle->stderr_pipe, buf, count, process_handle); #else - return tor_read_all_handle(process_handle->stderr_handle, buf, count, + return tor_read_all_handle(process_handle->stderr_pipe, buf, count, process_handle, NULL); #endif } @@ -5228,11 +5156,10 @@ log_from_handle(HANDLE *pipe, int severity) #else /** Return a smartlist containing lines outputted from - * <b>handle</b>. Return NULL on error, and set + * <b>fd</b>. Return NULL on error, and set * <b>stream_status_out</b> appropriately. */ MOCK_IMPL(smartlist_t *, -tor_get_lines_from_handle, (FILE *handle, - enum stream_status *stream_status_out)) +tor_get_lines_from_handle, (int fd, enum stream_status *stream_status_out)) { enum stream_status stream_status; char stdout_buf[400]; @@ -5241,13 +5168,13 @@ tor_get_lines_from_handle, (FILE *handle, while (1) { memset(stdout_buf, 0, sizeof(stdout_buf)); - stream_status = get_string_from_pipe(handle, + stream_status = get_string_from_pipe(fd, stdout_buf, sizeof(stdout_buf) - 1); if (stream_status != IO_STREAM_OKAY) goto done; if (!lines) lines = smartlist_new(); - smartlist_add(lines, tor_strdup(stdout_buf)); + smartlist_split_string(lines, stdout_buf, "\n", 0, 0); } done: @@ -5255,20 +5182,20 @@ tor_get_lines_from_handle, (FILE *handle, return lines; } -/** Read from stream, and send lines to log at the specified log level. +/** Read from fd, and send lines to log at the specified log level. * Returns 1 if stream is closed normally, -1 if there is a error reading, and * 0 otherwise. Handles lines from tor-fw-helper and * tor_spawn_background() specially. */ static int -log_from_pipe(FILE *stream, int severity, const char *executable, +log_from_pipe(int fd, int severity, const char *executable, int *child_status) { char buf[256]; enum stream_status r; for (;;) { - r = get_string_from_pipe(stream, buf, sizeof(buf) - 1); + r = get_string_from_pipe(fd, buf, sizeof(buf) - 1); if (r == IO_STREAM_CLOSED) { return 1; @@ -5293,7 +5220,7 @@ log_from_pipe(FILE *stream, int severity, const char *executable, } #endif -/** Reads from <b>stream</b> and stores input in <b>buf_out</b> making +/** Reads from <b>fd</b> and stores input in <b>buf_out</b> making * sure it's below <b>count</b> bytes. * If the string has a trailing newline, we strip it off. * @@ -5309,52 +5236,28 @@ log_from_pipe(FILE *stream, int severity, const char *executable, * IO_STREAM_OKAY: If everything went okay and we got a string * in <b>buf_out</b>. */ enum stream_status -get_string_from_pipe(FILE *stream, char *buf_out, size_t count) +get_string_from_pipe(int fd, char *buf_out, size_t count) { - char *retval; - size_t len; + ssize_t ret; tor_assert(count <= INT_MAX); - retval = fgets(buf_out, (int)count, stream); + ret = read(fd, buf_out, count); - if (!retval) { - if (feof(stream)) { - /* Program has closed stream (probably it exited) */ - /* TODO: check error */ - return IO_STREAM_CLOSED; - } else { - if (EAGAIN == errno) { - /* Nothing more to read, try again next time */ - return IO_STREAM_EAGAIN; - } else { - /* There was a problem, abandon this child process */ - return IO_STREAM_TERM; - } - } - } else { - len = strlen(buf_out); - if (len == 0) { - /* this probably means we got a NUL at the start of the string. */ - return IO_STREAM_EAGAIN; - } - - if (buf_out[len - 1] == '\n') { - /* Remove the trailing newline */ - buf_out[len - 1] = '\0'; - } else { - /* No newline; check whether we overflowed the buffer */ - if (!feof(stream)) - log_info(LD_GENERAL, - "Line from stream was truncated: %s", buf_out); - /* TODO: What to do with this error? */ - } + if (ret == 0) + return IO_STREAM_CLOSED; + else if (ret < 0 && errno == EAGAIN) + return IO_STREAM_EAGAIN; + else if (ret < 0) + return IO_STREAM_TERM; - return IO_STREAM_OKAY; - } + if (buf_out[ret - 1] == '\n') { + /* Remove the trailing newline */ + buf_out[ret - 1] = '\0'; + } else + buf_out[ret] = '\0'; - /* We should never get here */ - return IO_STREAM_TERM; + return IO_STREAM_OKAY; } /** Parse a <b>line</b> from tor-fw-helper and issue an appropriate @@ -5591,7 +5494,7 @@ tor_check_port_forwarding(const char *filename, #ifdef _WIN32 stderr_status = log_from_handle(child_handle->stderr_pipe, LOG_INFO); #else - stderr_status = log_from_pipe(child_handle->stderr_handle, + stderr_status = log_from_pipe(child_handle->stderr_pipe, LOG_INFO, filename, &retval); #endif if (handle_fw_helper_output(filename, child_handle) < 0) { diff --git a/src/common/util.h b/src/common/util.h index 479fc8d610..d56abcee2e 100644 --- a/src/common/util.h +++ b/src/common/util.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -163,9 +163,9 @@ int64_t clamp_double_to_int64(double number); void simplify_fraction64(uint64_t *numer, uint64_t *denom); /* Compute the CEIL of <b>a</b> divided by <b>b</b>, for nonnegative <b>a</b> - * and positive <b>b</b>. Works on integer types only. Not defined if a+b can - * overflow. */ -#define CEIL_DIV(a,b) (((a)+(b)-1)/(b)) + * and positive <b>b</b>. Works on integer types only. Not defined if a+(b-1) + * can overflow. */ +#define CEIL_DIV(a,b) (((a)+((b)-1))/(b)) /* Return <b>v</b> if it's between <b>min</b> and <b>max</b>. Otherwise * return <b>min</b> if <b>v</b> is smaller than <b>min</b>, or <b>max</b> if @@ -186,6 +186,7 @@ void tor_strlower(char *s) ATTR_NONNULL((1)); void tor_strupper(char *s) ATTR_NONNULL((1)); int tor_strisprint(const char *s) ATTR_NONNULL((1)); int tor_strisnonupper(const char *s) ATTR_NONNULL((1)); +int tor_strisspace(const char *s); int strcmp_opt(const char *s1, const char *s2); int strcmpstart(const char *s1, const char *s2) ATTR_NONNULL((1,2)); int strcmp_len(const char *s1, const char *s2, size_t len) ATTR_NONNULL((1,2)); @@ -239,6 +240,7 @@ void smartlist_add_asprintf(struct smartlist_t *sl, const char *pattern, ...) void smartlist_add_vasprintf(struct smartlist_t *sl, const char *pattern, va_list args) CHECK_PRINTF(2, 0); +void smartlist_add_strdup(struct smartlist_t *sl, const char *string); /* Time helpers */ long tv_udiff(const struct timeval *start, const struct timeval *end); @@ -254,8 +256,9 @@ void format_local_iso_time(char *buf, time_t t); void format_iso_time(char *buf, time_t t); void format_iso_time_nospace(char *buf, time_t t); void format_iso_time_nospace_usec(char *buf, const struct timeval *tv); -int parse_iso_time_(const char *cp, time_t *t, int strict); +int parse_iso_time_(const char *cp, time_t *t, int strict, int nospace); int parse_iso_time(const char *buf, time_t *t); +int parse_iso_time_nospace(const char *cp, time_t *t); int parse_http_time(const char *buf, struct tm *tm); int format_time_interval(char *out, size_t out_len, long interval); @@ -319,7 +322,7 @@ enum stream_status { const char *stream_status_to_string(enum stream_status stream_status); -enum stream_status get_string_from_pipe(FILE *stream, char *buf, size_t count); +enum stream_status get_string_from_pipe(int fd, char *buf, size_t count); MOCK_DECL(int,tor_unlink,(const char *pathname)); @@ -386,9 +389,7 @@ char *read_file_to_str_until_eof(int fd, size_t max_bytes_to_read, size_t *sz_out) ATTR_MALLOC; const char *unescape_string(const char *s, char **result, size_t *size_out); -const char *parse_config_line_from_str_verbose(const char *line, - char **key_out, char **value_out, - const char **err_out); +char *get_unquoted_path(const char *path); char *expand_filename(const char *filename); MOCK_DECL(struct smartlist_t *, tor_listdir, (const char *dirname)); int path_is_relative(const char *filename); @@ -460,9 +461,6 @@ struct process_handle_t { int stdin_pipe; int stdout_pipe; int stderr_pipe; - FILE *stdin_handle; - FILE *stdout_handle; - FILE *stderr_handle; pid_t pid; /** If the process has not given us a SIGCHLD yet, this has the * waitpid_callback_t that gets invoked once it has. Otherwise this @@ -485,7 +483,7 @@ int tor_split_lines(struct smartlist_t *sl, char *buf, int len); ssize_t tor_read_all_handle(HANDLE h, char *buf, size_t count, const process_handle_t *process); #else -ssize_t tor_read_all_handle(FILE *h, char *buf, size_t count, +ssize_t tor_read_all_handle(int fd, char *buf, size_t count, const process_handle_t *process, int *eof); #endif @@ -499,7 +497,7 @@ int tor_process_get_pid(process_handle_t *process_handle); #ifdef _WIN32 HANDLE tor_process_get_stdout_pipe(process_handle_t *process_handle); #else -FILE *tor_process_get_stdout_pipe(process_handle_t *process_handle); +int tor_process_get_stdout_pipe(process_handle_t *process_handle); #endif #ifdef _WIN32 @@ -508,7 +506,7 @@ tor_get_lines_from_handle,(HANDLE *handle, enum stream_status *stream_status)); #else MOCK_DECL(struct smartlist_t *, -tor_get_lines_from_handle,(FILE *handle, +tor_get_lines_from_handle,(int fd, enum stream_status *stream_status)); #endif @@ -551,9 +549,7 @@ STATIC int format_helper_exit_status(unsigned char child_state, #endif -#ifdef TOR_UNIT_TESTS -int size_mul_check__(const size_t x, const size_t y); -#endif +int size_mul_check(const size_t x, const size_t y); #define ARRAY_LENGTH(x) ((sizeof(x)) / sizeof(x[0])) diff --git a/src/common/util_bug.c b/src/common/util_bug.c index 08aba47974..3d990e3700 100644 --- a/src/common/util_bug.c +++ b/src/common/util_bug.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -44,7 +44,7 @@ static void add_captured_bug(const char *s) { --n_bugs_to_capture; - smartlist_add(bug_messages, tor_strdup(s)); + smartlist_add_strdup(bug_messages, s); } /** Set a callback to be invoked when we get any tor_bug_occurred_ * invocation. We use this in the unit tests so that a nonfatal diff --git a/src/common/util_bug.h b/src/common/util_bug.h index 0695806911..ae7e7a37fd 100644 --- a/src/common/util_bug.h +++ b/src/common/util_bug.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/util_format.c b/src/common/util_format.c index aef9db85c8..1f7b8b03aa 100644 --- a/src/common/util_format.c +++ b/src/common/util_format.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -22,13 +22,16 @@ #include <stdlib.h> /* Return the base32 encoded size in bytes using the source length srclen. - * The NUL terminated byte is added as well since every base32 encoding - * requires enough space for it. */ + * + * (WATCH OUT: This API counts the terminating NUL byte, but + * base64_encode_size does not.) + */ size_t base32_encoded_size(size_t srclen) { size_t enclen; - enclen = CEIL_DIV(srclen*8, 5) + 1; + tor_assert(srclen < SIZE_T_CEILING / 8); + enclen = BASE32_NOPAD_BUFSIZE(srclen); tor_assert(enclen < INT_MAX && enclen > srclen); return enclen; } @@ -41,7 +44,6 @@ base32_encode(char *dest, size_t destlen, const char *src, size_t srclen) size_t nbits = srclen * 8; size_t bit; - tor_assert(srclen < SIZE_T_CEILING/8); /* We need enough space for the encoded data and the extra NUL byte. */ tor_assert(base32_encoded_size(srclen) <= destlen); tor_assert(destlen < SIZE_T_CEILING); @@ -51,9 +53,10 @@ base32_encode(char *dest, size_t destlen, const char *src, size_t srclen) for (i=0,bit=0; bit < nbits; ++i, bit+=5) { /* set v to the 16-bit value starting at src[bits/8], 0-padded. */ - v = ((uint8_t)src[bit/8]) << 8; - if (bit+5<nbits) - v += (uint8_t)src[(bit/8)+1]; + size_t idx = bit / 8; + v = ((uint8_t)src[idx]) << 8; + if (idx+1 < srclen) + v += (uint8_t)src[idx+1]; /* set u to the 5-bit value at the bit'th bit of buf. */ u = (v >> (11-(bit%8))) & 0x1F; dest[i] = BASE32_CHARS[u]; @@ -133,6 +136,9 @@ base32_decode(char *dest, size_t destlen, const char *src, size_t srclen) /** Return the Base64 encoded size of <b>srclen</b> bytes of data in * bytes. * + * (WATCH OUT: This API <em>does not</em> count the terminating NUL byte, + * but base32_encoded_size does.) + * * If <b>flags</b>&BASE64_ENCODE_MULTILINE is true, return the size * of the encoded output as multiline output (64 character, `\n' terminated * lines). @@ -141,19 +147,16 @@ size_t base64_encode_size(size_t srclen, int flags) { size_t enclen; + + /* Use INT_MAX for overflow checking because base64_encode() returns int. */ tor_assert(srclen < INT_MAX); + tor_assert(CEIL_DIV(srclen, 3) < INT_MAX / 4); - if (srclen == 0) - return 0; + enclen = BASE64_LEN(srclen); + if (flags & BASE64_ENCODE_MULTILINE) + enclen += CEIL_DIV(enclen, BASE64_OPENSSL_LINELEN); - enclen = ((srclen - 1) / 3) * 4 + 4; - if (flags & BASE64_ENCODE_MULTILINE) { - size_t remainder = enclen % BASE64_OPENSSL_LINELEN; - enclen += enclen / BASE64_OPENSSL_LINELEN; - if (remainder) - enclen++; - } - tor_assert(enclen < INT_MAX && enclen > srclen); + tor_assert(enclen < INT_MAX && (enclen == 0 || enclen > srclen)); return enclen; } @@ -310,39 +313,6 @@ base64_encode_nopad(char *dest, size_t destlen, return (int)(out - dest); } -/** As base64_decode, but do not require any padding on the input */ -int -base64_decode_nopad(uint8_t *dest, size_t destlen, - const char *src, size_t srclen) -{ - if (srclen > SIZE_T_CEILING - 4) - return -1; - char *buf = tor_malloc(srclen + 4); - memcpy(buf, src, srclen+1); - size_t buflen; - switch (srclen % 4) - { - case 0: - default: - buflen = srclen; - break; - case 1: - tor_free(buf); - return -1; - case 2: - memcpy(buf+srclen, "==", 3); - buflen = srclen + 2; - break; - case 3: - memcpy(buf+srclen, "=", 2); - buflen = srclen + 1; - break; - } - int n = base64_decode((char*)dest, destlen, buf, buflen); - tor_free(buf); - return n; -} - #undef BASE64_OPENSSL_LINELEN /** @{ */ @@ -392,15 +362,9 @@ base64_decode(char *dest, size_t destlen, const char *src, size_t srclen) const char *eos = src+srclen; uint32_t n=0; int n_idx=0; - char *dest_orig = dest; + size_t di = 0; - /* Max number of bits == srclen*6. - * Number of bytes required to hold all bits == (srclen*6)/8. - * Yes, we want to round down: anything that hangs over the end of a - * byte is padding. */ - if (destlen < (srclen*3)/4) - return -1; - if (destlen > SIZE_T_CEILING) + if (destlen > INT_MAX) return -1; /* Make sure we leave no uninitialized data in the destination buffer. */ @@ -428,9 +392,11 @@ base64_decode(char *dest, size_t destlen, const char *src, size_t srclen) n = (n<<6) | v; if ((++n_idx) == 4) { /* We've accumulated 24 bits in n. Flush them. */ - *dest++ = (n>>16); - *dest++ = (n>>8) & 0xff; - *dest++ = (n) & 0xff; + if (destlen < 3 || di > destlen - 3) + return -1; + dest[di++] = (n>>16); + dest[di++] = (n>>8) & 0xff; + dest[di++] = (n) & 0xff; n_idx = 0; n = 0; } @@ -448,18 +414,21 @@ base64_decode(char *dest, size_t destlen, const char *src, size_t srclen) return -1; case 2: /* 12 leftover bits: The last 4 are padding and the first 8 are data. */ - *dest++ = n >> 4; + if (destlen < 1 || di > destlen - 1) + return -1; + dest[di++] = n >> 4; break; case 3: /* 18 leftover bits: The last 2 are padding and the first 16 are data. */ - *dest++ = n >> 10; - *dest++ = n >> 2; + if (destlen < 2 || di > destlen - 2) + return -1; + dest[di++] = n >> 10; + dest[di++] = n >> 2; } - tor_assert((dest-dest_orig) <= (ssize_t)destlen); - tor_assert((dest-dest_orig) <= INT_MAX); + tor_assert(di <= destlen); - return (int)(dest-dest_orig); + return (int)di; } #undef X #undef SP @@ -475,7 +444,8 @@ base16_encode(char *dest, size_t destlen, const char *src, size_t srclen) const char *end; char *cp; - tor_assert(destlen >= srclen*2+1); + tor_assert(srclen < SIZE_T_CEILING / 2 - 1); + tor_assert(destlen >= BASE16_BUFSIZE(srclen)); tor_assert(destlen < SIZE_T_CEILING); /* Make sure we leave no uninitialized data in the destination buffer. */ diff --git a/src/common/util_format.h b/src/common/util_format.h index 20ac711d10..4af8832bbe 100644 --- a/src/common/util_format.h +++ b/src/common/util_format.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_UTIL_FORMAT_H @@ -10,6 +10,26 @@ #include "testsupport.h" #include "torint.h" +/** @{ */ +/** These macros don't check for overflow. Use them only for constant inputs + * (like array declarations). The *_LEN macros are the raw encoding lengths + * (without terminating NUL), while the *_BUFSIZE macros count the terminating + * NUL. */ +#define BASE64_LEN(n) (CEIL_DIV((n), 3) * 4) +#define BASE32_LEN(n) (CEIL_DIV((n), 5) * 8) +#define BASE16_LEN(n) ((n) * 2) + +#define BASE64_BUFSIZE(n) (BASE64_LEN(n) + 1) +#define BASE32_BUFSIZE(n) (BASE32_LEN(n) + 1) +#define BASE16_BUFSIZE(n) (BASE16_LEN(n) + 1) + +#define BASE64_NOPAD_LEN(n) (CEIL_DIV((n) * 4, 3)) +#define BASE32_NOPAD_LEN(n) (CEIL_DIV((n) * 8, 5)) + +#define BASE64_NOPAD_BUFSIZE(n) (BASE64_NOPAD_LEN(n) + 1) +#define BASE32_NOPAD_BUFSIZE(n) (BASE32_NOPAD_LEN(n) + 1) +/** @} */ + #define BASE64_ENCODE_MULTILINE 1 size_t base64_encode_size(size_t srclen, int flags); int base64_encode(char *dest, size_t destlen, const char *src, size_t srclen, @@ -17,8 +37,6 @@ int base64_encode(char *dest, size_t destlen, const char *src, size_t srclen, int base64_decode(char *dest, size_t destlen, const char *src, size_t srclen); int base64_encode_nopad(char *dest, size_t destlen, const uint8_t *src, size_t srclen); -int base64_decode_nopad(uint8_t *dest, size_t destlen, - const char *src, size_t srclen); /** Characters that can appear (case-insensitively) in a base32 encoding. */ #define BASE32_CHARS "abcdefghijklmnopqrstuvwxyz234567" diff --git a/src/common/util_process.c b/src/common/util_process.c index abda63720c..9e9679b099 100644 --- a/src/common/util_process.c +++ b/src/common/util_process.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2016, The Tor Project, Inc. */ + * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/util_process.h b/src/common/util_process.h index d38301a354..c3a63498b5 100644 --- a/src/common/util_process.h +++ b/src/common/util_process.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2011-2016, The Tor Project, Inc. */ +/* Copyright (c) 2011-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/common/workqueue.c b/src/common/workqueue.c index e1fb663a2a..42723224d3 100644 --- a/src/common/workqueue.c +++ b/src/common/workqueue.c @@ -25,11 +25,19 @@ #include "orconfig.h" #include "compat.h" #include "compat_threads.h" +#include "crypto.h" #include "util.h" #include "workqueue.h" #include "tor_queue.h" #include "torlog.h" +#define WORKQUEUE_PRIORITY_FIRST WQ_PRI_HIGH +#define WORKQUEUE_PRIORITY_LAST WQ_PRI_LOW +#define WORKQUEUE_N_PRIORITIES (((int) WORKQUEUE_PRIORITY_LAST)+1) + +TOR_TAILQ_HEAD(work_tailq_t, workqueue_entry_s); +typedef struct work_tailq_t work_tailq_t; + struct threadpool_s { /** An array of pointers to workerthread_t: one for each running worker * thread. */ @@ -38,8 +46,12 @@ struct threadpool_s { /** Condition variable that we wait on when we have no work, and which * gets signaled when our queue becomes nonempty. */ tor_cond_t condition; - /** Queue of pending work that we have to do. */ - TOR_TAILQ_HEAD(, workqueue_entry_s) work; + /** Queues of pending work that we have to do. The queue with priority + * <b>p</b> is work[p]. */ + work_tailq_t work[WORKQUEUE_N_PRIORITIES]; + + /** Weak RNG, used to decide when to ignore priority. */ + tor_weak_rng_t weak_rng; /** The current 'update generation' of the threadpool. Any thread that is * at an earlier generation needs to run the update function. */ @@ -66,6 +78,11 @@ struct threadpool_s { void *new_thread_state_arg; }; +/** Used to put a workqueue_priority_t value into a bitfield. */ +#define workqueue_priority_bitfield_t ENUM_BF(workqueue_priority_t) +/** Number of bits needed to hold all legal values of workqueue_priority_t */ +#define WORKQUEUE_PRIORITY_BITS 2 + struct workqueue_entry_s { /** The next workqueue_entry_t that's pending on the same thread or * reply queue. */ @@ -76,6 +93,8 @@ struct workqueue_entry_s { struct threadpool_s *on_pool; /** True iff this entry is waiting for a worker to start processing it. */ uint8_t pending; + /** Priority of this entry. */ + workqueue_priority_bitfield_t priority : WORKQUEUE_PRIORITY_BITS; /** Function to run in the worker thread. */ workqueue_reply_t (*fn)(void *state, void *arg); /** Function to run while processing the reply queue. */ @@ -94,9 +113,7 @@ struct replyqueue_s { alert_sockets_t alert; }; -/** A worker thread represents a single thread in a thread pool. To avoid - * contention, each gets its own queue. This breaks the guarantee that that - * queued work will get executed strictly in order. */ +/** A worker thread represents a single thread in a thread pool. */ typedef struct workerthread_s { /** Which thread it this? In range 0..in_pool->n_threads-1 */ int index; @@ -109,6 +126,8 @@ typedef struct workerthread_s { replyqueue_t *reply_queue; /** The current update generation of this thread */ unsigned generation; + /** One over the probability of taking work from a lower-priority queue. */ + int32_t lower_priority_chance; } workerthread_t; static void queue_reply(replyqueue_t *queue, workqueue_entry_t *work); @@ -125,6 +144,7 @@ workqueue_entry_new(workqueue_reply_t (*fn)(void*, void*), ent->fn = fn; ent->reply_fn = reply_fn; ent->arg = arg; + ent->priority = WQ_PRI_HIGH; return ent; } @@ -161,8 +181,9 @@ workqueue_entry_cancel(workqueue_entry_t *ent) int cancelled = 0; void *result = NULL; tor_mutex_acquire(&ent->on_pool->lock); + workqueue_priority_t prio = ent->priority; if (ent->pending) { - TOR_TAILQ_REMOVE(&ent->on_pool->work, ent, next_work); + TOR_TAILQ_REMOVE(&ent->on_pool->work[prio], ent, next_work); cancelled = 1; result = ent->arg; } @@ -180,8 +201,46 @@ workqueue_entry_cancel(workqueue_entry_t *ent) static int worker_thread_has_work(workerthread_t *thread) { - return !TOR_TAILQ_EMPTY(&thread->in_pool->work) || - thread->generation != thread->in_pool->generation; + unsigned i; + for (i = WORKQUEUE_PRIORITY_FIRST; i <= WORKQUEUE_PRIORITY_LAST; ++i) { + if (!TOR_TAILQ_EMPTY(&thread->in_pool->work[i])) + return 1; + } + return thread->generation != thread->in_pool->generation; +} + +/** Extract the next workqueue_entry_t from the the thread's pool, removing + * it from the relevant queues and marking it as non-pending. + * + * The caller must hold the lock. */ +static workqueue_entry_t * +worker_thread_extract_next_work(workerthread_t *thread) +{ + threadpool_t *pool = thread->in_pool; + work_tailq_t *queue = NULL, *this_queue; + unsigned i; + for (i = WORKQUEUE_PRIORITY_FIRST; i <= WORKQUEUE_PRIORITY_LAST; ++i) { + this_queue = &pool->work[i]; + if (!TOR_TAILQ_EMPTY(this_queue)) { + queue = this_queue; + if (! tor_weak_random_one_in_n(&pool->weak_rng, + thread->lower_priority_chance)) { + /* Usually we'll just break now, so that we can get out of the loop + * and use the queue where we found work. But with a small + * probability, we'll keep looking for lower priority work, so that + * we don't ignore our low-priority queues entirely. */ + break; + } + } + } + + if (queue == NULL) + return NULL; + + workqueue_entry_t *work = TOR_TAILQ_FIRST(queue); + TOR_TAILQ_REMOVE(queue, work, next_work); + work->pending = 0; + return work; } /** @@ -217,9 +276,9 @@ worker_thread_main(void *thread_) tor_mutex_acquire(&pool->lock); continue; } - work = TOR_TAILQ_FIRST(&pool->work); - TOR_TAILQ_REMOVE(&pool->work, work, next_work); - work->pending = 0; + work = worker_thread_extract_next_work(thread); + if (BUG(work == NULL)) + break; tor_mutex_release(&pool->lock); /* We run the work function without holding the thread lock. This @@ -268,12 +327,14 @@ queue_reply(replyqueue_t *queue, workqueue_entry_t *work) /** Allocate and start a new worker thread to use state object <b>state</b>, * and send responses to <b>replyqueue</b>. */ static workerthread_t * -workerthread_new(void *state, threadpool_t *pool, replyqueue_t *replyqueue) +workerthread_new(int32_t lower_priority_chance, + void *state, threadpool_t *pool, replyqueue_t *replyqueue) { workerthread_t *thr = tor_malloc_zero(sizeof(workerthread_t)); thr->state = state; thr->reply_queue = replyqueue; thr->in_pool = pool; + thr->lower_priority_chance = lower_priority_chance; if (spawn_func(worker_thread_main, thr) < 0) { //LCOV_EXCL_START @@ -299,24 +360,34 @@ workerthread_new(void *state, threadpool_t *pool, replyqueue_t *replyqueue) * function's responsibility to free the work object. * * On success, return a workqueue_entry_t object that can be passed to - * workqueue_entry_cancel(). On failure, return NULL. + * workqueue_entry_cancel(). On failure, return NULL. (Failure is not + * currently possible, but callers should check anyway.) + * + * Items are executed in a loose priority order -- each thread will usually + * take from the queued work with the highest prioirity, but will occasionally + * visit lower-priority queues to keep them from starving completely. * - * Note that because each thread has its own work queue, work items may not + * Note that because of priorities and thread behavior, work items may not * be executed strictly in order. */ workqueue_entry_t * -threadpool_queue_work(threadpool_t *pool, - workqueue_reply_t (*fn)(void *, void *), - void (*reply_fn)(void *), - void *arg) +threadpool_queue_work_priority(threadpool_t *pool, + workqueue_priority_t prio, + workqueue_reply_t (*fn)(void *, void *), + void (*reply_fn)(void *), + void *arg) { + tor_assert(((int)prio) >= WORKQUEUE_PRIORITY_FIRST && + ((int)prio) <= WORKQUEUE_PRIORITY_LAST); + workqueue_entry_t *ent = workqueue_entry_new(fn, reply_fn, arg); ent->on_pool = pool; ent->pending = 1; + ent->priority = prio; tor_mutex_acquire(&pool->lock); - TOR_TAILQ_INSERT_TAIL(&pool->work, ent, next_work); + TOR_TAILQ_INSERT_TAIL(&pool->work[prio], ent, next_work); tor_cond_signal_one(&pool->condition); @@ -325,6 +396,16 @@ threadpool_queue_work(threadpool_t *pool, return ent; } +/** As threadpool_queue_work_priority(), but assumes WQ_PRI_HIGH */ +workqueue_entry_t * +threadpool_queue_work(threadpool_t *pool, + workqueue_reply_t (*fn)(void *, void *), + void (*reply_fn)(void *), + void *arg) +{ + return threadpool_queue_work_priority(pool, WQ_PRI_HIGH, fn, reply_fn, arg); +} + /** * Queue a copy of a work item for every thread in a pool. This can be used, * for example, to tell the threads to update some parameter in their states. @@ -388,6 +469,14 @@ threadpool_queue_update(threadpool_t *pool, /** Don't have more than this many threads per pool. */ #define MAX_THREADS 1024 +/** For half of our threads, choose lower priority queues with probability + * 1/N for each of these values. Both are chosen somewhat arbitrarily. If + * CHANCE_PERMISSIVE is too low, then we have a risk of low-priority tasks + * stalling forever. If it's too high, we have a risk of low-priority tasks + * grabbing half of the threads. */ +#define CHANCE_PERMISSIVE 37 +#define CHANCE_STRICT INT32_MAX + /** Launch threads until we have <b>n</b>. */ static int threadpool_start_threads(threadpool_t *pool, int n) @@ -404,8 +493,14 @@ threadpool_start_threads(threadpool_t *pool, int n) sizeof(workerthread_t*), n); while (pool->n_threads < n) { + /* For half of our threads, we'll choose lower priorities permissively; + * for the other half, we'll stick more strictly to higher priorities. + * This keeps slow low-priority tasks from taking over completely. */ + int32_t chance = (pool->n_threads & 1) ? CHANCE_STRICT : CHANCE_PERMISSIVE; + void *state = pool->new_thread_state_fn(pool->new_thread_state_arg); - workerthread_t *thr = workerthread_new(state, pool, pool->reply_queue); + workerthread_t *thr = workerthread_new(chance, + state, pool, pool->reply_queue); if (!thr) { //LCOV_EXCL_START @@ -441,7 +536,15 @@ threadpool_new(int n_threads, pool = tor_malloc_zero(sizeof(threadpool_t)); tor_mutex_init_nonrecursive(&pool->lock); tor_cond_init(&pool->condition); - TOR_TAILQ_INIT(&pool->work); + unsigned i; + for (i = WORKQUEUE_PRIORITY_FIRST; i <= WORKQUEUE_PRIORITY_LAST; ++i) { + TOR_TAILQ_INIT(&pool->work[i]); + } + { + unsigned seed; + crypto_rand((void*)&seed, sizeof(seed)); + tor_init_weak_random(&pool->weak_rng, seed); + } pool->new_thread_state_fn = new_thread_state_fn; pool->new_thread_state_arg = arg; @@ -510,12 +613,13 @@ replyqueue_get_socket(replyqueue_t *rq) void replyqueue_process(replyqueue_t *queue) { - if (queue->alert.drain_fn(queue->alert.read_fd) < 0) { + int r = queue->alert.drain_fn(queue->alert.read_fd); + if (r < 0) { //LCOV_EXCL_START static ratelim_t warn_limit = RATELIM_INIT(7200); log_fn_ratelim(&warn_limit, LOG_WARN, LD_GENERAL, "Failure from drain_fd: %s", - tor_socket_strerror(tor_socket_errno(queue->alert.read_fd))); + tor_socket_strerror(-r)); //LCOV_EXCL_STOP } diff --git a/src/common/workqueue.h b/src/common/workqueue.h index 54276767b0..d2508f5329 100644 --- a/src/common/workqueue.h +++ b/src/common/workqueue.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2016, The Tor Project, Inc. */ +/* Copyright (c) 2013-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_WORKQUEUE_H @@ -16,12 +16,26 @@ typedef struct threadpool_s threadpool_t; typedef struct workqueue_entry_s workqueue_entry_t; /** Possible return value from a work function: */ -typedef enum { +typedef enum workqueue_reply_t { WQ_RPL_REPLY = 0, /** indicates success */ WQ_RPL_ERROR = 1, /** indicates fatal error */ WQ_RPL_SHUTDOWN = 2, /** indicates thread is shutting down */ } workqueue_reply_t; +/** Possible priorities for work. Lower numeric values are more important. */ +typedef enum workqueue_priority_t { + WQ_PRI_HIGH = 0, + WQ_PRI_MED = 1, + WQ_PRI_LOW = 2, +} workqueue_priority_t; + +workqueue_entry_t *threadpool_queue_work_priority(threadpool_t *pool, + workqueue_priority_t prio, + workqueue_reply_t (*fn)(void *, + void *), + void (*reply_fn)(void *), + void *arg); + workqueue_entry_t *threadpool_queue_work(threadpool_t *pool, workqueue_reply_t (*fn)(void *, void *), |